From be166dc724ba5eee74ee2a915f6b2f258e378cdb Mon Sep 17 00:00:00 2001 From: "Brian C. Lane" Date: Fri, 15 Oct 2021 09:06:42 -0700 Subject: [PATCH] ltmpl: Add version compare support to installpkg This adds support for enforcing version requirements on installed packages. See the documentation in ltmpl.installpkg for details. --- .../99-generic/runtime-install.tmpl | 24 +- src/pylorax/ltmpl.py | 281 +++++++++--------- tests/pylorax/templates/install-test.tmpl | 4 +- tests/pylorax/test_ltmpl.py | 88 +++++- 4 files changed, 241 insertions(+), 156 deletions(-) diff --git a/share/templates.d/99-generic/runtime-install.tmpl b/share/templates.d/99-generic/runtime-install.tmpl index fd186193..730f8c63 100644 --- a/share/templates.d/99-generic/runtime-install.tmpl +++ b/share/templates.d/99-generic/runtime-install.tmpl @@ -1,5 +1,9 @@ ## lorax template file: populate the ramdisk (runtime image) <%page args="basearch, product"/> +<% +# This version of grub2 moves the font directory and is needed to keep the efi template from failing. +GRUB2VER="1:2.06-3" +%> ## anaconda package installpkg anaconda anaconda-widgets kexec-tools-anaconda-addon anaconda-install-img-deps @@ -37,12 +41,13 @@ installpkg glibc-all-langpacks ## arch-specific packages (bootloaders etc.) %if basearch == "aarch64": installpkg efibootmgr - installpkg grub2-efi-aa64-cdboot shim-aa64 + installpkg grub2-efi-aa64-cdboot>=${GRUB2VER} + installpkg shim-aa64 installpkg uboot-tools %endif %if basearch in ("arm", "armhfp"): installpkg efibootmgr - installpkg grub2-efi-arm-cdboot + installpkg grub2-efi-arm-cdboot>=${GRUB2VER} installpkg grubby-deprecated installpkg kernel-lpae installpkg uboot-tools @@ -51,19 +56,22 @@ installpkg glibc-all-langpacks installpkg gpart %endif %if basearch == "x86_64": - installpkg grub2-tools-efi + installpkg grub2-tools-efi>=${GRUB2VER} installpkg efibootmgr - installpkg shim-x64 grub2-efi-x64-cdboot - installpkg shim-ia32 grub2-efi-ia32-cdboot + installpkg shim-x64 + installpkg grub2-efi-x64-cdboot>=${GRUB2VER} + installpkg shim-ia32 + installpkg grub2-efi-ia32-cdboot>=${GRUB2VER} %endif %if basearch in ("i386", "x86_64"): installpkg biosdevname syslinux - installpkg grub2-tools grub2-tools-minimal grub2-tools-extra + installpkg grub2-tools>=${GRUB2VER} grub2-tools-minimal>=${GRUB2VER} + installpkg grub2-tools-extra>=${GRUB2VER} %endif %if basearch == "ppc64le": installpkg powerpc-utils lsvpd ppc64-diag - installpkg grub2-tools grub2-tools-minimal grub2-tools-extra - installpkg grub2-${basearch} + installpkg grub2-tools>=${GRUB2VER} grub2-tools-minimal>=${GRUB2VER} + installpkg grub2-tools-extra>=${GRUB2VER} grub2-${basearch}>=${GRUB2VER} %endif %if basearch == "s390x": installpkg lsscsi s390utils-base s390utils-cmsfs-fuse s390utils-hmcdrvfs diff --git a/src/pylorax/ltmpl.py b/src/pylorax/ltmpl.py index 36e09e97..84e1e5dc 100644 --- a/src/pylorax/ltmpl.py +++ b/src/pylorax/ltmpl.py @@ -185,8 +185,149 @@ class TemplateRunner(object): raise +class InstallpkgMixin: + """Helper class used with *Runner classes""" + def _pkgver(self, pkg_spec): + """ + Helper to parse package version compare operators + + Returns a list of matching package objects or an empty list + + Examples: + "bash>4.01" + "tmux>=3.1.4-5" + "grub2<2.06" + """ + # Always return the highest of the filtered results + if not any(g for g in ['=', '<', '>', '!'] if g in pkg_spec): + query = dnf.subject.Subject(pkg_spec).get_best_query(self.dbo.sack) + else: + pcv = re.split(r'([!<>=]+)', pkg_spec) + if not pcv[0]: + raise RuntimeError("Missing package name") + if not pcv[-1]: + raise RuntimeError("Missing version") + if len(pcv) != 3: + raise RuntimeError("Too many comparisons") + + query = dnf.subject.Subject(pcv[0]).get_best_query(self.dbo.sack) + + # Parse the comparison operators + if pcv[1] == "=" or pcv[1] == "==": + query.filterm(evr__eq = pcv[2]) + elif pcv[1] == "!=" or pcv[1] == "<>": + query.filterm(evr__neq = pcv[2]) + elif pcv[1] == ">": + query.filterm(evr__gt = pcv[2]) + elif pcv[1] == ">=" or pcv[1] == "=>": + query.filterm(evr__gte = pcv[2]) + elif pcv[1] == "<": + query.filterm(evr__lt = pcv[2]) + elif pcv[1] == "<=" or pcv[1] == "=<": + query.filterm(evr__lte = pcv[2]) + + # MUST be added last. Otherwise it will only return the latest, not the latest of the + # filtered results. + query.filterm(latest=True) + return [pkg for pkg in query.apply()] + + def installpkg(self, *pkgs): + ''' + installpkg [--required|--optional] [--except PKGGLOB [--except PKGGLOB ...]] PKGGLOB [PKGGLOB ...] + Request installation of all packages matching the given globs. + Note that this is just a *request* - nothing is *actually* installed + until the 'run_pkg_transaction' command is given. + + The non-except PKGGLOB can contain a version comparison. This should + not be used as a substitute for package dependencies, it should be + used to enforce installation of tools required by the templates. eg. + grub2 changed the font location in 2.06-2 so the current templates + require grub2 to be 2.06-2 or later. + + installpkg tmux>=2.8 bash=5.0.0-1 + + It supports the =,!=,>,>=,<,<= operators. == is an alias for =, and + <> is an alias for != + + There should be no spaces between the package name, the compare + operator, and the version. + + NOTE: When testing for equality you must include the version AND + release, otherwise it won't match anything. + + --required is now the default. If the PKGGLOB can be missing pass --optional + ''' + if pkgs[0] == '--optional': + pkgs = pkgs[1:] + required = False + elif pkgs[0] == '--required': + pkgs = pkgs[1:] + required = True + else: + required = True + + excludes = [] + while '--except' in pkgs: + idx = pkgs.index('--except') + if len(pkgs) == idx+1: + raise ValueError("installpkg needs an argument after --except") + + # TODO: Check for bare version compare operators + excludes.append(pkgs[idx+1]) + pkgs = pkgs[:idx] + pkgs[idx+2:] + + errors = False + for p in pkgs: + # Did a version compare operatore end up in the list? + if p[0] in ['=', '<', '>', '!']: + raise RuntimeError("Version compare operators cannot be surrounded by spaces") + + try: + # Start by using Subject to generate a package query, which will + # give us a query object similar to what dbo.install would select, + # minus the handling for multilib. This query may contain + # multiple arches. Pull the package names out of that, filter any + # that match the excludes patterns, and pass those names back to + # dbo.install to do the actual, arch and version and multilib + # aware, package selction. + + # dnf queries don't have a concept of negative globs which is why + # the filtering is done the hard way. + + # Get the latest package, or package matching the selected version + pkgnames = self._pkgver(p) + if not pkgnames: + raise dnf.exceptions.PackageNotFoundError("no package matched", p) + + # Apply excludes to the name only + for exclude in excludes: + pkgnames = [pkg for pkg in pkgnames if not fnmatch.fnmatch(pkg.name, exclude)] + + # Convert to a sorted NVR list for installation + pkgnvrs = sorted(["{}-{}-{}".format(pkg.name, pkg.version, pkg.release) for pkg in pkgnames]) + + # If the request is a glob, expand it in the log + if any(g for g in ['*','?','.'] if g in p): + logger.info("installpkg: %s expands to %s", p, ",".join(pkgnvrs)) + + for pkgnvr in pkgnvrs: + try: + self.dbo.install(pkgnvr) + except Exception as e: # pylint: disable=broad-except + if required: + raise + # Not required, log it and continue processing pkgs + logger.error("installpkg %s failed: %s", pkgnvr, str(e)) + except Exception as e: # pylint: disable=broad-except + logger.error("installpkg %s failed: %s", p, str(e)) + errors = True + + if errors and required: + raise Exception("Required installpkg failed.") + + # TODO: operate inside an actual chroot for safety? Not that RPM bothers.. -class LoraxTemplateRunner(TemplateRunner): +class LoraxTemplateRunner(TemplateRunner, InstallpkgMixin): ''' This class parses and executes Lorax templates. Sample usage: @@ -531,77 +672,6 @@ class LoraxTemplateRunner(TemplateRunner): logger.error('command returned failure (%d)', e.returncode) raise - def installpkg(self, *pkgs): - ''' - installpkg [--required|--optional] [--except PKGGLOB [--except PKGGLOB ...]] PKGGLOB [PKGGLOB ...] - Request installation of all packages matching the given globs. - Note that this is just a *request* - nothing is *actually* installed - until the 'run_pkg_transaction' command is given. - - --required is now the default. If the PKGGLOB can be missing pass --optional - ''' - if pkgs[0] == '--optional': - pkgs = pkgs[1:] - required = False - elif pkgs[0] == '--required': - pkgs = pkgs[1:] - required = True - else: - required = True - - excludes = [] - while '--except' in pkgs: - idx = pkgs.index('--except') - if len(pkgs) == idx+1: - raise ValueError("installpkg needs an argument after --except") - - excludes.append(pkgs[idx+1]) - pkgs = pkgs[:idx] + pkgs[idx+2:] - - errors = False - for p in pkgs: - try: - # Start by using Subject to generate a package query, which will - # give us a query object similar to what dbo.install would select, - # minus the handling for multilib. This query may contain - # multiple arches. Pull the package names out of that, filter any - # that match the excludes patterns, and pass those names back to - # dbo.install to do the actual, arch and version and multilib - # aware, package selction. - - # dnf queries don't have a concept of negative globs which is why - # the filtering is done the hard way. - - pkgnames = [pkg for pkg in dnf.subject.Subject(p).get_best_query(self.dbo.sack).filter(latest=True)] - if not pkgnames: - raise dnf.exceptions.PackageNotFoundError("no package matched", p) - - # Apply excludes to the name only - for exclude in excludes: - pkgnames = [pkg for pkg in pkgnames if not fnmatch.fnmatch(pkg.name, exclude)] - - # Convert to a sorted NVR list for installation - pkgnvrs = sorted(["{}-{}-{}".format(pkg.name, pkg.version, pkg.release) for pkg in pkgnames]) - - # If the request is a glob, expand it in the log - if any(g for g in ['*','?','.'] if g in p): - logger.info("installpkg: %s expands to %s", p, ",".join(pkgnvrs)) - - for pkgnvr in pkgnvrs: - try: - self.dbo.install(pkgnvr) - except Exception as e: # pylint: disable=broad-except - if required: - raise - # Not required, log it and continue processing pkgs - logger.error("installpkg %s failed: %s", pkgnvr, str(e)) - except Exception as e: # pylint: disable=broad-except - logger.error("installpkg %s failed: %s", p, str(e)) - errors = True - - if errors and required: - raise Exception("Required installpkg failed.") - def removepkg(self, *pkgs): ''' removepkg PKGGLOB [PKGGLOB...] @@ -800,7 +870,7 @@ class LoraxTemplateRunner(TemplateRunner): except CalledProcessError: pass -class LiveTemplateRunner(TemplateRunner): +class LiveTemplateRunner(TemplateRunner, InstallpkgMixin): """ This class parses and executes a limited Lorax template. Sample usage: @@ -817,68 +887,3 @@ class LiveTemplateRunner(TemplateRunner): self.pkgnames = [] super(LiveTemplateRunner, self).__init__(fatalerrors, templatedir, defaults) - - def installpkg(self, *pkgs): - ''' - installpkg [--required|--optional] [--except PKGGLOB [--except PKGGLOB ...]] PKGGLOB [PKGGLOB ...] - Request installation of all packages matching the given globs. - Note that this is just a *request* - nothing is *actually* installed - until the 'run_pkg_transaction' command is given. - - --required is now the default. If the PKGGLOB can be missing pass --optional - ''' - if pkgs[0] == '--optional': - pkgs = pkgs[1:] - required = False - elif pkgs[0] == '--required': - pkgs = pkgs[1:] - required = True - else: - required = True - - excludes = [] - while '--except' in pkgs: - idx = pkgs.index('--except') - if len(pkgs) == idx+1: - raise ValueError("installpkg needs an argument after --except") - - excludes.append(pkgs[idx+1]) - pkgs = pkgs[:idx] + pkgs[idx+2:] - - errors = False - for p in pkgs: - try: - # Start by using Subject to generate a package query, which will - # give us a query object similar to what dbo.install would select, - # minus the handling for multilib. This query may contain - # multiple arches. Pull the package names out of that, filter any - # that match the excludes patterns, and pass those names back to - # dbo.install to do the actual, arch and version and multilib - # aware, package selction. - - # dnf queries don't have a concept of negative globs which is why - # the filtering is done the hard way. - - pkgnames = [pkg for pkg in dnf.subject.Subject(p).get_best_query(self.dbo.sack).filter(latest=True)] - if not pkgnames: - raise dnf.exceptions.PackageNotFoundError("no package matched", p) - - # Apply excludes to the name only - for exclude in excludes: - pkgnames = [pkg for pkg in pkgnames if not fnmatch.fnmatch(pkg.name, exclude)] - - # Convert to a sorted NVR list for installation - pkgnvrs = sorted(["{}-{}-{}".format(pkg.name, pkg.version, pkg.release) for pkg in pkgnames]) - - # If the request is a glob, expand it in the log - if any(g for g in ['*','?','.'] if g in p): - logger.info("installpkg: %s expands to %s", p, ",".join(pkgnvrs)) - - self.pkgs.extend(pkgnvrs) - self.pkgnames.extend([pkg.name for pkg in pkgnames]) - except Exception as e: # pylint: disable=broad-except - logger.error("installpkg %s failed: %s", p, str(e)) - errors = True - - if errors and required: - raise Exception("Required installpkg failed.") diff --git a/tests/pylorax/templates/install-test.tmpl b/tests/pylorax/templates/install-test.tmpl index 61c0203d..e7885218 100644 --- a/tests/pylorax/templates/install-test.tmpl +++ b/tests/pylorax/templates/install-test.tmpl @@ -1,8 +1,10 @@ <%page /> installpkg anaconda-core installpkg --optional exact-1.3.17 -installpkg --except fake-homer fake-* +installpkg --except fake-homer --except fake-milhouse --except fake-lisa fake-* installpkg --required lots-of-files installpkg known-path -installpkg missing-package +installpkg fake-milhouse>1.0.0-4 +installpkg fake-lisa<1.2.0-1 run_pkg_transaction diff --git a/tests/pylorax/test_ltmpl.py b/tests/pylorax/test_ltmpl.py index 8d1871a3..bc21a9d0 100644 --- a/tests/pylorax/test_ltmpl.py +++ b/tests/pylorax/test_ltmpl.py @@ -115,8 +115,10 @@ class LoraxTemplateRunnerTestCase(unittest.TestCase): self.repo1_dir = tempfile.mkdtemp(prefix="lorax.test.repo.") makeFakeRPM(self.repo1_dir, "anaconda-core", 0, "0.0.1", "1") makeFakeRPM(self.repo1_dir, "exact", 0, "1.3.17", "1") - makeFakeRPM(self.repo1_dir, "fake-milhouse", 0, "1.0.0", "1") + makeFakeRPM(self.repo1_dir, "fake-milhouse", 0, "1.0.0", "1", ["/fake-milhouse/1.0.0-1"]) + makeFakeRPM(self.repo1_dir, "fake-bart", 0, "1.0.0", "6") makeFakeRPM(self.repo1_dir, "fake-bart", 2, "1.13.0", "6") + makeFakeRPM(self.repo1_dir, "fake-bart", 2, "2.3.0", "1") makeFakeRPM(self.repo1_dir, "fake-homer", 0, "0.4.0", "2") makeFakeRPM(self.repo1_dir, "lots-of-files", 0, "0.1.1", "1", ["/lorax-files/file-one.txt", @@ -126,12 +128,15 @@ class LoraxTemplateRunnerTestCase(unittest.TestCase): os.system("createrepo_c " + self.repo1_dir) self.repo2_dir = tempfile.mkdtemp(prefix="lorax.test.repo.") - makeFakeRPM(self.repo2_dir, "fake-milhouse", 0, "1.3.0", "1") - makeFakeRPM(self.repo2_dir, "fake-lisa", 0, "1.2.0", "1") + makeFakeRPM(self.repo2_dir, "fake-milhouse", 0, "1.0.0", "4", ["/fake-milhouse/1.0.0-4"]) + makeFakeRPM(self.repo2_dir, "fake-milhouse", 0, "1.0.7", "1", ["/fake-milhouse/1.0.7-1"]) + makeFakeRPM(self.repo2_dir, "fake-milhouse", 0, "1.3.0", "1", ["/fake-milhouse/1.3.0-1"]) + makeFakeRPM(self.repo2_dir, "fake-lisa", 0, "1.2.0", "1", ["/fake-lisa/1.2.0-1"]) + makeFakeRPM(self.repo2_dir, "fake-lisa", 0, "1.1.4", "5", ["/fake-lisa/1.1.4-5"]) os.system("createrepo_c " + self.repo2_dir) self.repo3_dir = tempfile.mkdtemp(prefix="lorax.test.debug.repo.") - makeFakeRPM(self.repo3_dir, "fake-marge", 0, "2.3.0", "1", ["/fake-marge/file-one.txt"]) + makeFakeRPM(self.repo3_dir, "fake-marge", 0, "2.3.0", "1", ["/fake-marge/2.3.0-1"]) makeFakeRPM(self.repo3_dir, "fake-marge-debuginfo", 0, "2.3.0", "1", ["/fake-marge/file-one-debuginfo.txt"]) os.system("createrepo_c " + self.repo3_dir) @@ -154,24 +159,89 @@ class LoraxTemplateRunnerTestCase(unittest.TestCase): shutil.rmtree(self.repo2_dir) shutil.rmtree(self.root_dir) - def test_00_runner_multi_repo(self): + def test_pkgver_errors(self): + """Test error states of _pkgver""" + with self.assertRaises(RuntimeError) as e: + self.runner._pkgver("=") + self.assertEqual(str(e.exception), "Missing package name") + + + with self.assertRaises(RuntimeError) as e: + self.runner._pkgver("foopkg=") + self.assertEqual(str(e.exception), "Missing version") + + with self.assertRaises(RuntimeError) as e: + self.runner._pkgver("foopkg>1.0.0-1<1.0.6-1") + self.assertEqual(str(e.exception), "Too many comparisons") + + + def test_00_pkgver(self): + """Test all the version comparison operators with pkgver""" + matrix = [ + ("fake-milhouse>=2.1.0-1", ""), # Not available + ("fake-bart>=2:3.0.0-2", ""), # Not available + ("fake-bart>2:1.13.0-6", "fake-bart-2:2.3.0-1"), + ("fake-bart<2:1.13.0-6", "fake-bart-1.0.0-6"), + ("fake-milhouse==1.3.0-1", "fake-milhouse-1.3.0-1"), + ("fake-milhouse=1.3.0-1", "fake-milhouse-1.3.0-1"), + ("fake-milhouse=1.0.0-4", "fake-milhouse-1.0.0-4"), + ("fake-milhouse!=1.3.0-1", "fake-milhouse-1.0.7-1"), + ("fake-milhouse<>1.3.0-1", "fake-milhouse-1.0.7-1"), + ("fake-milhouse>1.0.0-4", "fake-milhouse-1.3.0-1"), + ("fake-milhouse>=1.3.0", "fake-milhouse-1.3.0-1"), + ("fake-milhouse>=1.0.7-1", "fake-milhouse-1.3.0-1"), + ("fake-milhouse=>1.0.0-4", "fake-milhouse-1.3.0-1"), + ("fake-milhouse<=1.0.0-4", "fake-milhouse-1.0.0-4"), + ("fake-milhouse=<1.0.7-1", "fake-milhouse-1.0.7-1"), + ("fake-milhouse<1.3.0", "fake-milhouse-1.0.7-1"), + ("fake-milhouse<1.3.0-1", "fake-milhouse-1.0.7-1"), + ("fake-milhouse<1.0.7-1", "fake-milhouse-1.0.0-4"), + ] + + def nevra(pkg): + if pkg.epoch: + return "{}-{}:{}-{}".format(pkg.name, pkg.epoch, pkg.version, pkg.release) + else: + return "{}-{}-{}".format(pkg.name, pkg.version, pkg.release) + + print([nevra(p) for p in list(self.dnfbase.sack.query().available())]) + for t in matrix: + r = self.runner._pkgver(t[0]) + if t[1]: + self.assertTrue(len(r) > 0, t[0]) + self.assertEqual(nevra(self.runner._pkgver(t[0])[0]), t[1], t[0]) + else: + self.assertEqual(r, [], t[0]) + + def test_01_runner_multi_repo(self): """Test installing packages with updates in a 2nd repo""" # If this does not raise an error it means that: # Installing a named package works (anaconda-core) # Installing a pinned package works (exact-1.3.17) # Installing a globbed set of package names from multiple repos works + # Installing a package using version compare # removepkg removes a package's files # removefrom removes some, but not all, of a package's files # # These all need to be done in one template because run_pkg_transaction can only run once self.runner.run("install-test.tmpl") self.runner.run("install-remove-test.tmpl") - self.assertFalse(os.path.exists(joinpaths(self.root_dir, "/known-path/file-one.txt"))) - self.assertTrue(os.path.exists(joinpaths(self.root_dir, "/lorax-files/file-one.txt"))) - self.assertFalse(os.path.exists(joinpaths(self.root_dir, "/lorax-files/file-two.txt"))) + + def exists(p): + return os.path.exists(joinpaths(self.root_dir, p)) + + self.assertFalse(exists("/known-path/file-one.txt")) + self.assertTrue(exists("/lorax-files/file-one.txt")) + self.assertFalse(exists("/lorax-files/file-two.txt")) + self.assertTrue(exists("/fake-marge/2.3.0-1")) # Check the debug log - self.assertTrue(os.path.exists(joinpaths(self.root_dir, "/root/debug-pkgs.log"))) + self.assertTrue(exists("/root/debug-pkgs.log")) + + # Check package version installs + self.assertTrue(exists("/fake-lisa/1.1.4-5")) + self.assertFalse(exists("/fake-lisa/1.2.0-1")) + self.assertTrue(exists("/fake-milhouse/1.3.0-1")) def test_install_file(self): """Test append, and install template commands"""