ltmpl: Add version compare support to installpkg

This adds support for enforcing version requirements on installed
packages. See the documentation in ltmpl.installpkg for details.
This commit is contained in:
Brian C. Lane 2021-10-15 09:06:42 -07:00
parent f2c8d4f6bb
commit be166dc724
4 changed files with 241 additions and 156 deletions

View File

@ -1,5 +1,9 @@
## lorax template file: populate the ramdisk (runtime image)
<%page args="basearch, product"/>
<%
# This version of grub2 moves the font directory and is needed to keep the efi template from failing.
GRUB2VER="1:2.06-3"
%>
## anaconda package
installpkg anaconda anaconda-widgets kexec-tools-anaconda-addon anaconda-install-img-deps
@ -37,12 +41,13 @@ installpkg glibc-all-langpacks
## arch-specific packages (bootloaders etc.)
%if basearch == "aarch64":
installpkg efibootmgr
installpkg grub2-efi-aa64-cdboot shim-aa64
installpkg grub2-efi-aa64-cdboot>=${GRUB2VER}
installpkg shim-aa64
installpkg uboot-tools
%endif
%if basearch in ("arm", "armhfp"):
installpkg efibootmgr
installpkg grub2-efi-arm-cdboot
installpkg grub2-efi-arm-cdboot>=${GRUB2VER}
installpkg grubby-deprecated
installpkg kernel-lpae
installpkg uboot-tools
@ -51,19 +56,22 @@ installpkg glibc-all-langpacks
installpkg gpart
%endif
%if basearch == "x86_64":
installpkg grub2-tools-efi
installpkg grub2-tools-efi>=${GRUB2VER}
installpkg efibootmgr
installpkg shim-x64 grub2-efi-x64-cdboot
installpkg shim-ia32 grub2-efi-ia32-cdboot
installpkg shim-x64
installpkg grub2-efi-x64-cdboot>=${GRUB2VER}
installpkg shim-ia32
installpkg grub2-efi-ia32-cdboot>=${GRUB2VER}
%endif
%if basearch in ("i386", "x86_64"):
installpkg biosdevname syslinux
installpkg grub2-tools grub2-tools-minimal grub2-tools-extra
installpkg grub2-tools>=${GRUB2VER} grub2-tools-minimal>=${GRUB2VER}
installpkg grub2-tools-extra>=${GRUB2VER}
%endif
%if basearch == "ppc64le":
installpkg powerpc-utils lsvpd ppc64-diag
installpkg grub2-tools grub2-tools-minimal grub2-tools-extra
installpkg grub2-${basearch}
installpkg grub2-tools>=${GRUB2VER} grub2-tools-minimal>=${GRUB2VER}
installpkg grub2-tools-extra>=${GRUB2VER} grub2-${basearch}>=${GRUB2VER}
%endif
%if basearch == "s390x":
installpkg lsscsi s390utils-base s390utils-cmsfs-fuse s390utils-hmcdrvfs

View File

@ -185,8 +185,149 @@ class TemplateRunner(object):
raise
class InstallpkgMixin:
"""Helper class used with *Runner classes"""
def _pkgver(self, pkg_spec):
"""
Helper to parse package version compare operators
Returns a list of matching package objects or an empty list
Examples:
"bash>4.01"
"tmux>=3.1.4-5"
"grub2<2.06"
"""
# Always return the highest of the filtered results
if not any(g for g in ['=', '<', '>', '!'] if g in pkg_spec):
query = dnf.subject.Subject(pkg_spec).get_best_query(self.dbo.sack)
else:
pcv = re.split(r'([!<>=]+)', pkg_spec)
if not pcv[0]:
raise RuntimeError("Missing package name")
if not pcv[-1]:
raise RuntimeError("Missing version")
if len(pcv) != 3:
raise RuntimeError("Too many comparisons")
query = dnf.subject.Subject(pcv[0]).get_best_query(self.dbo.sack)
# Parse the comparison operators
if pcv[1] == "=" or pcv[1] == "==":
query.filterm(evr__eq = pcv[2])
elif pcv[1] == "!=" or pcv[1] == "<>":
query.filterm(evr__neq = pcv[2])
elif pcv[1] == ">":
query.filterm(evr__gt = pcv[2])
elif pcv[1] == ">=" or pcv[1] == "=>":
query.filterm(evr__gte = pcv[2])
elif pcv[1] == "<":
query.filterm(evr__lt = pcv[2])
elif pcv[1] == "<=" or pcv[1] == "=<":
query.filterm(evr__lte = pcv[2])
# MUST be added last. Otherwise it will only return the latest, not the latest of the
# filtered results.
query.filterm(latest=True)
return [pkg for pkg in query.apply()]
def installpkg(self, *pkgs):
'''
installpkg [--required|--optional] [--except PKGGLOB [--except PKGGLOB ...]] PKGGLOB [PKGGLOB ...]
Request installation of all packages matching the given globs.
Note that this is just a *request* - nothing is *actually* installed
until the 'run_pkg_transaction' command is given.
The non-except PKGGLOB can contain a version comparison. This should
not be used as a substitute for package dependencies, it should be
used to enforce installation of tools required by the templates. eg.
grub2 changed the font location in 2.06-2 so the current templates
require grub2 to be 2.06-2 or later.
installpkg tmux>=2.8 bash=5.0.0-1
It supports the =,!=,>,>=,<,<= operators. == is an alias for =, and
<> is an alias for !=
There should be no spaces between the package name, the compare
operator, and the version.
NOTE: When testing for equality you must include the version AND
release, otherwise it won't match anything.
--required is now the default. If the PKGGLOB can be missing pass --optional
'''
if pkgs[0] == '--optional':
pkgs = pkgs[1:]
required = False
elif pkgs[0] == '--required':
pkgs = pkgs[1:]
required = True
else:
required = True
excludes = []
while '--except' in pkgs:
idx = pkgs.index('--except')
if len(pkgs) == idx+1:
raise ValueError("installpkg needs an argument after --except")
# TODO: Check for bare version compare operators
excludes.append(pkgs[idx+1])
pkgs = pkgs[:idx] + pkgs[idx+2:]
errors = False
for p in pkgs:
# Did a version compare operatore end up in the list?
if p[0] in ['=', '<', '>', '!']:
raise RuntimeError("Version compare operators cannot be surrounded by spaces")
try:
# Start by using Subject to generate a package query, which will
# give us a query object similar to what dbo.install would select,
# minus the handling for multilib. This query may contain
# multiple arches. Pull the package names out of that, filter any
# that match the excludes patterns, and pass those names back to
# dbo.install to do the actual, arch and version and multilib
# aware, package selction.
# dnf queries don't have a concept of negative globs which is why
# the filtering is done the hard way.
# Get the latest package, or package matching the selected version
pkgnames = self._pkgver(p)
if not pkgnames:
raise dnf.exceptions.PackageNotFoundError("no package matched", p)
# Apply excludes to the name only
for exclude in excludes:
pkgnames = [pkg for pkg in pkgnames if not fnmatch.fnmatch(pkg.name, exclude)]
# Convert to a sorted NVR list for installation
pkgnvrs = sorted(["{}-{}-{}".format(pkg.name, pkg.version, pkg.release) for pkg in pkgnames])
# If the request is a glob, expand it in the log
if any(g for g in ['*','?','.'] if g in p):
logger.info("installpkg: %s expands to %s", p, ",".join(pkgnvrs))
for pkgnvr in pkgnvrs:
try:
self.dbo.install(pkgnvr)
except Exception as e: # pylint: disable=broad-except
if required:
raise
# Not required, log it and continue processing pkgs
logger.error("installpkg %s failed: %s", pkgnvr, str(e))
except Exception as e: # pylint: disable=broad-except
logger.error("installpkg %s failed: %s", p, str(e))
errors = True
if errors and required:
raise Exception("Required installpkg failed.")
# TODO: operate inside an actual chroot for safety? Not that RPM bothers..
class LoraxTemplateRunner(TemplateRunner):
class LoraxTemplateRunner(TemplateRunner, InstallpkgMixin):
'''
This class parses and executes Lorax templates. Sample usage:
@ -531,77 +672,6 @@ class LoraxTemplateRunner(TemplateRunner):
logger.error('command returned failure (%d)', e.returncode)
raise
def installpkg(self, *pkgs):
'''
installpkg [--required|--optional] [--except PKGGLOB [--except PKGGLOB ...]] PKGGLOB [PKGGLOB ...]
Request installation of all packages matching the given globs.
Note that this is just a *request* - nothing is *actually* installed
until the 'run_pkg_transaction' command is given.
--required is now the default. If the PKGGLOB can be missing pass --optional
'''
if pkgs[0] == '--optional':
pkgs = pkgs[1:]
required = False
elif pkgs[0] == '--required':
pkgs = pkgs[1:]
required = True
else:
required = True
excludes = []
while '--except' in pkgs:
idx = pkgs.index('--except')
if len(pkgs) == idx+1:
raise ValueError("installpkg needs an argument after --except")
excludes.append(pkgs[idx+1])
pkgs = pkgs[:idx] + pkgs[idx+2:]
errors = False
for p in pkgs:
try:
# Start by using Subject to generate a package query, which will
# give us a query object similar to what dbo.install would select,
# minus the handling for multilib. This query may contain
# multiple arches. Pull the package names out of that, filter any
# that match the excludes patterns, and pass those names back to
# dbo.install to do the actual, arch and version and multilib
# aware, package selction.
# dnf queries don't have a concept of negative globs which is why
# the filtering is done the hard way.
pkgnames = [pkg for pkg in dnf.subject.Subject(p).get_best_query(self.dbo.sack).filter(latest=True)]
if not pkgnames:
raise dnf.exceptions.PackageNotFoundError("no package matched", p)
# Apply excludes to the name only
for exclude in excludes:
pkgnames = [pkg for pkg in pkgnames if not fnmatch.fnmatch(pkg.name, exclude)]
# Convert to a sorted NVR list for installation
pkgnvrs = sorted(["{}-{}-{}".format(pkg.name, pkg.version, pkg.release) for pkg in pkgnames])
# If the request is a glob, expand it in the log
if any(g for g in ['*','?','.'] if g in p):
logger.info("installpkg: %s expands to %s", p, ",".join(pkgnvrs))
for pkgnvr in pkgnvrs:
try:
self.dbo.install(pkgnvr)
except Exception as e: # pylint: disable=broad-except
if required:
raise
# Not required, log it and continue processing pkgs
logger.error("installpkg %s failed: %s", pkgnvr, str(e))
except Exception as e: # pylint: disable=broad-except
logger.error("installpkg %s failed: %s", p, str(e))
errors = True
if errors and required:
raise Exception("Required installpkg failed.")
def removepkg(self, *pkgs):
'''
removepkg PKGGLOB [PKGGLOB...]
@ -800,7 +870,7 @@ class LoraxTemplateRunner(TemplateRunner):
except CalledProcessError:
pass
class LiveTemplateRunner(TemplateRunner):
class LiveTemplateRunner(TemplateRunner, InstallpkgMixin):
"""
This class parses and executes a limited Lorax template. Sample usage:
@ -817,68 +887,3 @@ class LiveTemplateRunner(TemplateRunner):
self.pkgnames = []
super(LiveTemplateRunner, self).__init__(fatalerrors, templatedir, defaults)
def installpkg(self, *pkgs):
'''
installpkg [--required|--optional] [--except PKGGLOB [--except PKGGLOB ...]] PKGGLOB [PKGGLOB ...]
Request installation of all packages matching the given globs.
Note that this is just a *request* - nothing is *actually* installed
until the 'run_pkg_transaction' command is given.
--required is now the default. If the PKGGLOB can be missing pass --optional
'''
if pkgs[0] == '--optional':
pkgs = pkgs[1:]
required = False
elif pkgs[0] == '--required':
pkgs = pkgs[1:]
required = True
else:
required = True
excludes = []
while '--except' in pkgs:
idx = pkgs.index('--except')
if len(pkgs) == idx+1:
raise ValueError("installpkg needs an argument after --except")
excludes.append(pkgs[idx+1])
pkgs = pkgs[:idx] + pkgs[idx+2:]
errors = False
for p in pkgs:
try:
# Start by using Subject to generate a package query, which will
# give us a query object similar to what dbo.install would select,
# minus the handling for multilib. This query may contain
# multiple arches. Pull the package names out of that, filter any
# that match the excludes patterns, and pass those names back to
# dbo.install to do the actual, arch and version and multilib
# aware, package selction.
# dnf queries don't have a concept of negative globs which is why
# the filtering is done the hard way.
pkgnames = [pkg for pkg in dnf.subject.Subject(p).get_best_query(self.dbo.sack).filter(latest=True)]
if not pkgnames:
raise dnf.exceptions.PackageNotFoundError("no package matched", p)
# Apply excludes to the name only
for exclude in excludes:
pkgnames = [pkg for pkg in pkgnames if not fnmatch.fnmatch(pkg.name, exclude)]
# Convert to a sorted NVR list for installation
pkgnvrs = sorted(["{}-{}-{}".format(pkg.name, pkg.version, pkg.release) for pkg in pkgnames])
# If the request is a glob, expand it in the log
if any(g for g in ['*','?','.'] if g in p):
logger.info("installpkg: %s expands to %s", p, ",".join(pkgnvrs))
self.pkgs.extend(pkgnvrs)
self.pkgnames.extend([pkg.name for pkg in pkgnames])
except Exception as e: # pylint: disable=broad-except
logger.error("installpkg %s failed: %s", p, str(e))
errors = True
if errors and required:
raise Exception("Required installpkg failed.")

View File

@ -1,8 +1,10 @@
<%page />
installpkg anaconda-core
installpkg --optional exact-1.3.17
installpkg --except fake-homer fake-*
installpkg --except fake-homer --except fake-milhouse --except fake-lisa fake-*
installpkg --required lots-of-files
installpkg known-path
-installpkg missing-package
installpkg fake-milhouse>1.0.0-4
installpkg fake-lisa<1.2.0-1
run_pkg_transaction

View File

@ -115,8 +115,10 @@ class LoraxTemplateRunnerTestCase(unittest.TestCase):
self.repo1_dir = tempfile.mkdtemp(prefix="lorax.test.repo.")
makeFakeRPM(self.repo1_dir, "anaconda-core", 0, "0.0.1", "1")
makeFakeRPM(self.repo1_dir, "exact", 0, "1.3.17", "1")
makeFakeRPM(self.repo1_dir, "fake-milhouse", 0, "1.0.0", "1")
makeFakeRPM(self.repo1_dir, "fake-milhouse", 0, "1.0.0", "1", ["/fake-milhouse/1.0.0-1"])
makeFakeRPM(self.repo1_dir, "fake-bart", 0, "1.0.0", "6")
makeFakeRPM(self.repo1_dir, "fake-bart", 2, "1.13.0", "6")
makeFakeRPM(self.repo1_dir, "fake-bart", 2, "2.3.0", "1")
makeFakeRPM(self.repo1_dir, "fake-homer", 0, "0.4.0", "2")
makeFakeRPM(self.repo1_dir, "lots-of-files", 0, "0.1.1", "1",
["/lorax-files/file-one.txt",
@ -126,12 +128,15 @@ class LoraxTemplateRunnerTestCase(unittest.TestCase):
os.system("createrepo_c " + self.repo1_dir)
self.repo2_dir = tempfile.mkdtemp(prefix="lorax.test.repo.")
makeFakeRPM(self.repo2_dir, "fake-milhouse", 0, "1.3.0", "1")
makeFakeRPM(self.repo2_dir, "fake-lisa", 0, "1.2.0", "1")
makeFakeRPM(self.repo2_dir, "fake-milhouse", 0, "1.0.0", "4", ["/fake-milhouse/1.0.0-4"])
makeFakeRPM(self.repo2_dir, "fake-milhouse", 0, "1.0.7", "1", ["/fake-milhouse/1.0.7-1"])
makeFakeRPM(self.repo2_dir, "fake-milhouse", 0, "1.3.0", "1", ["/fake-milhouse/1.3.0-1"])
makeFakeRPM(self.repo2_dir, "fake-lisa", 0, "1.2.0", "1", ["/fake-lisa/1.2.0-1"])
makeFakeRPM(self.repo2_dir, "fake-lisa", 0, "1.1.4", "5", ["/fake-lisa/1.1.4-5"])
os.system("createrepo_c " + self.repo2_dir)
self.repo3_dir = tempfile.mkdtemp(prefix="lorax.test.debug.repo.")
makeFakeRPM(self.repo3_dir, "fake-marge", 0, "2.3.0", "1", ["/fake-marge/file-one.txt"])
makeFakeRPM(self.repo3_dir, "fake-marge", 0, "2.3.0", "1", ["/fake-marge/2.3.0-1"])
makeFakeRPM(self.repo3_dir, "fake-marge-debuginfo", 0, "2.3.0", "1", ["/fake-marge/file-one-debuginfo.txt"])
os.system("createrepo_c " + self.repo3_dir)
@ -154,24 +159,89 @@ class LoraxTemplateRunnerTestCase(unittest.TestCase):
shutil.rmtree(self.repo2_dir)
shutil.rmtree(self.root_dir)
def test_00_runner_multi_repo(self):
def test_pkgver_errors(self):
"""Test error states of _pkgver"""
with self.assertRaises(RuntimeError) as e:
self.runner._pkgver("=")
self.assertEqual(str(e.exception), "Missing package name")
with self.assertRaises(RuntimeError) as e:
self.runner._pkgver("foopkg=")
self.assertEqual(str(e.exception), "Missing version")
with self.assertRaises(RuntimeError) as e:
self.runner._pkgver("foopkg>1.0.0-1<1.0.6-1")
self.assertEqual(str(e.exception), "Too many comparisons")
def test_00_pkgver(self):
"""Test all the version comparison operators with pkgver"""
matrix = [
("fake-milhouse>=2.1.0-1", ""), # Not available
("fake-bart>=2:3.0.0-2", ""), # Not available
("fake-bart>2:1.13.0-6", "fake-bart-2:2.3.0-1"),
("fake-bart<2:1.13.0-6", "fake-bart-1.0.0-6"),
("fake-milhouse==1.3.0-1", "fake-milhouse-1.3.0-1"),
("fake-milhouse=1.3.0-1", "fake-milhouse-1.3.0-1"),
("fake-milhouse=1.0.0-4", "fake-milhouse-1.0.0-4"),
("fake-milhouse!=1.3.0-1", "fake-milhouse-1.0.7-1"),
("fake-milhouse<>1.3.0-1", "fake-milhouse-1.0.7-1"),
("fake-milhouse>1.0.0-4", "fake-milhouse-1.3.0-1"),
("fake-milhouse>=1.3.0", "fake-milhouse-1.3.0-1"),
("fake-milhouse>=1.0.7-1", "fake-milhouse-1.3.0-1"),
("fake-milhouse=>1.0.0-4", "fake-milhouse-1.3.0-1"),
("fake-milhouse<=1.0.0-4", "fake-milhouse-1.0.0-4"),
("fake-milhouse=<1.0.7-1", "fake-milhouse-1.0.7-1"),
("fake-milhouse<1.3.0", "fake-milhouse-1.0.7-1"),
("fake-milhouse<1.3.0-1", "fake-milhouse-1.0.7-1"),
("fake-milhouse<1.0.7-1", "fake-milhouse-1.0.0-4"),
]
def nevra(pkg):
if pkg.epoch:
return "{}-{}:{}-{}".format(pkg.name, pkg.epoch, pkg.version, pkg.release)
else:
return "{}-{}-{}".format(pkg.name, pkg.version, pkg.release)
print([nevra(p) for p in list(self.dnfbase.sack.query().available())])
for t in matrix:
r = self.runner._pkgver(t[0])
if t[1]:
self.assertTrue(len(r) > 0, t[0])
self.assertEqual(nevra(self.runner._pkgver(t[0])[0]), t[1], t[0])
else:
self.assertEqual(r, [], t[0])
def test_01_runner_multi_repo(self):
"""Test installing packages with updates in a 2nd repo"""
# If this does not raise an error it means that:
# Installing a named package works (anaconda-core)
# Installing a pinned package works (exact-1.3.17)
# Installing a globbed set of package names from multiple repos works
# Installing a package using version compare
# removepkg removes a package's files
# removefrom removes some, but not all, of a package's files
#
# These all need to be done in one template because run_pkg_transaction can only run once
self.runner.run("install-test.tmpl")
self.runner.run("install-remove-test.tmpl")
self.assertFalse(os.path.exists(joinpaths(self.root_dir, "/known-path/file-one.txt")))
self.assertTrue(os.path.exists(joinpaths(self.root_dir, "/lorax-files/file-one.txt")))
self.assertFalse(os.path.exists(joinpaths(self.root_dir, "/lorax-files/file-two.txt")))
def exists(p):
return os.path.exists(joinpaths(self.root_dir, p))
self.assertFalse(exists("/known-path/file-one.txt"))
self.assertTrue(exists("/lorax-files/file-one.txt"))
self.assertFalse(exists("/lorax-files/file-two.txt"))
self.assertTrue(exists("/fake-marge/2.3.0-1"))
# Check the debug log
self.assertTrue(os.path.exists(joinpaths(self.root_dir, "/root/debug-pkgs.log")))
self.assertTrue(exists("/root/debug-pkgs.log"))
# Check package version installs
self.assertTrue(exists("/fake-lisa/1.1.4-5"))
self.assertFalse(exists("/fake-lisa/1.2.0-1"))
self.assertTrue(exists("/fake-milhouse/1.3.0-1"))
def test_install_file(self):
"""Test append, and install template commands"""