ltmpl: Add version compare support to installpkg

This adds support for enforcing version requirements on installed
packages. See the documentation in ltmpl.installpkg for details.

NOTE: On Fedora 34 grub2-2.06-7 is the first version with unicode.pf2 in
the new location.

share/templates.d/99-generic/runtime-install.tmpl

@@ -1,5 +1,9 @@
 ## lorax template file: populate the ramdisk (runtime image)
 <%page args="basearch, product"/>
+<%
+# This version of grub2 moves the font directory and is needed to keep the efi template from failing.
+GRUB2VER="1:2.06-7"
+%>
 
 ## anaconda package
 installpkg anaconda anaconda-widgets kexec-tools-anaconda-addon anaconda-install-img-deps
@@ -32,12 +36,13 @@ installpkg glibc-all-langpacks
 ## arch-specific packages (bootloaders etc.)
 %if basearch == "aarch64":
     installpkg efibootmgr
-    installpkg grub2-efi-aa64-cdboot shim-aa64
+    installpkg grub2-efi-aa64-cdboot>=${GRUB2VER}
+    installpkg shim-aa64
     installpkg uboot-tools
 %endif
 %if basearch in ("arm", "armhfp"):
     installpkg efibootmgr
-    installpkg grub2-efi-arm-cdboot
+    installpkg grub2-efi-arm-cdboot>=${GRUB2VER}
     installpkg grubby-deprecated
     installpkg kernel-lpae
     installpkg uboot-tools
@@ -46,19 +51,22 @@ installpkg glibc-all-langpacks
     installpkg gpart
 %endif
 %if basearch == "x86_64":
-    installpkg grub2-tools-efi
+    installpkg grub2-tools-efi>=${GRUB2VER}
     installpkg efibootmgr
-    installpkg shim-x64 grub2-efi-x64-cdboot
+    installpkg shim-x64
-    installpkg shim-ia32 grub2-efi-ia32-cdboot
+    installpkg grub2-efi-x64-cdboot>=${GRUB2VER}
+    installpkg shim-ia32
+    installpkg grub2-efi-ia32-cdboot>=${GRUB2VER}
 %endif
 %if basearch in ("i386", "x86_64"):
     installpkg biosdevname memtest86+ syslinux
-    installpkg grub2-tools grub2-tools-minimal grub2-tools-extra
+    installpkg grub2-tools>=${GRUB2VER} grub2-tools-minimal>=${GRUB2VER}
+    installpkg grub2-tools-extra>=${GRUB2VER}
 %endif
 %if basearch == "ppc64le":
     installpkg powerpc-utils lsvpd ppc64-diag
-    installpkg grub2-tools grub2-tools-minimal grub2-tools-extra
+    installpkg grub2-tools>=${GRUB2VER} grub2-tools-minimal>=${GRUB2VER}
-    installpkg grub2-${basearch}
+    installpkg grub2-tools-extra>=${GRUB2VER} grub2-${basearch}>=${GRUB2VER}
 %endif
 %if basearch == "s390x":
     installpkg lsscsi s390utils-base s390utils-cmsfs-fuse s390utils-hmcdrvfs

src/pylorax/ltmpl.py

@@ -185,8 +185,149 @@ class TemplateRunner(object):
                     raise
 
 
+class InstallpkgMixin:
+    """Helper class used with *Runner classes"""
+    def _pkgver(self, pkg_spec):
+        """
+        Helper to parse package version compare operators
+
+        Returns a list of matching package objects or an empty list
+
+        Examples:
+          "bash>4.01"
+          "tmux>=3.1.4-5"
+          "grub2<2.06"
+        """
+        # Always return the highest of the filtered results
+        if not any(g for g in ['=', '<', '>', '!'] if g in pkg_spec):
+            query = dnf.subject.Subject(pkg_spec).get_best_query(self.dbo.sack)
+        else:
+            pcv = re.split(r'([!<>=]+)', pkg_spec)
+            if not pcv[0]:
+                raise RuntimeError("Missing package name")
+            if not pcv[-1]:
+                raise RuntimeError("Missing version")
+            if len(pcv) != 3:
+                raise RuntimeError("Too many comparisons")
+
+            query = dnf.subject.Subject(pcv[0]).get_best_query(self.dbo.sack)
+
+            # Parse the comparison operators
+            if pcv[1] == "=" or pcv[1] == "==":
+                query.filterm(evr__eq = pcv[2])
+            elif pcv[1] == "!=" or pcv[1] == "<>":
+                query.filterm(evr__neq = pcv[2])
+            elif pcv[1] == ">":
+                query.filterm(evr__gt = pcv[2])
+            elif pcv[1] == ">=" or pcv[1] == "=>":
+                query.filterm(evr__gte = pcv[2])
+            elif pcv[1] == "<":
+                query.filterm(evr__lt = pcv[2])
+            elif pcv[1] == "<=" or pcv[1] == "=<":
+                query.filterm(evr__lte = pcv[2])
+
+        # MUST be added last. Otherwise it will only return the latest, not the latest of the
+        # filtered results.
+        query.filterm(latest=True)
+        return [pkg for pkg in query.apply()]
+
+    def installpkg(self, *pkgs):
+        '''
+        installpkg [--required|--optional] [--except PKGGLOB [--except PKGGLOB ...]] PKGGLOB [PKGGLOB ...]
+          Request installation of all packages matching the given globs.
+          Note that this is just a *request* - nothing is *actually* installed
+          until the 'run_pkg_transaction' command is given.
+
+          The non-except PKGGLOB can contain a version comparison. This should
+          not be used as a substitute for package dependencies, it should be
+          used to enforce installation of tools required by the templates. eg.
+          grub2 changed the font location in 2.06-2 so the current templates
+          require grub2 to be 2.06-2 or later.
+
+          installpkg tmux>=2.8 bash=5.0.0-1
+
+          It supports the =,!=,>,>=,<,<= operators. == is an alias for =, and
+          <> is an alias for !=
+
+          There should be no spaces between the package name, the compare
+          operator, and the version.
+
+          NOTE: When testing for equality you must include the version AND
+          release, otherwise it won't match anything.
+
+          --required is now the default. If the PKGGLOB can be missing pass --optional
+        '''
+        if pkgs[0] == '--optional':
+            pkgs = pkgs[1:]
+            required = False
+        elif pkgs[0] == '--required':
+            pkgs = pkgs[1:]
+            required = True
+        else:
+            required = True
+
+        excludes = []
+        while '--except' in pkgs:
+            idx = pkgs.index('--except')
+            if len(pkgs) == idx+1:
+                raise ValueError("installpkg needs an argument after --except")
+
+            # TODO: Check for bare version compare operators
+            excludes.append(pkgs[idx+1])
+            pkgs = pkgs[:idx] + pkgs[idx+2:]
+
+        errors = False
+        for p in pkgs:
+            # Did a version compare operatore end up in the list?
+            if p[0] in ['=', '<', '>', '!']:
+                raise RuntimeError("Version compare operators cannot be surrounded by spaces")
+
+            try:
+                # Start by using Subject to generate a package query, which will
+                # give us a query object similar to what dbo.install would select,
+                # minus the handling for multilib. This query may contain
+                # multiple arches. Pull the package names out of that, filter any
+                # that match the excludes patterns, and pass those names back to
+                # dbo.install to do the actual, arch and version and multilib
+                # aware, package selction.
+
+                # dnf queries don't have a concept of negative globs which is why
+                # the filtering is done the hard way.
+
+                # Get the latest package, or package matching the selected version
+                pkgnames = self._pkgver(p)
+                if not pkgnames:
+                    raise dnf.exceptions.PackageNotFoundError("no package matched", p)
+
+                # Apply excludes to the name only
+                for exclude in excludes:
+                    pkgnames = [pkg for pkg in pkgnames if not fnmatch.fnmatch(pkg.name, exclude)]
+
+                # Convert to a sorted NVR list for installation
+                pkgnvrs = sorted(["{}-{}-{}".format(pkg.name, pkg.version, pkg.release) for pkg in pkgnames])
+
+                # If the request is a glob, expand it in the log
+                if any(g for g in ['*','?','.'] if g in p):
+                    logger.info("installpkg: %s expands to %s", p, ",".join(pkgnvrs))
+
+                for pkgnvr in pkgnvrs:
+                    try:
+                        self.dbo.install(pkgnvr)
+                    except Exception as e: # pylint: disable=broad-except
+                        if required:
+                            raise
+                        # Not required, log it and continue processing pkgs
+                        logger.error("installpkg %s failed: %s", pkgnvr, str(e))
+            except Exception as e: # pylint: disable=broad-except
+                logger.error("installpkg %s failed: %s", p, str(e))
+                errors = True
+
+        if errors and required:
+            raise Exception("Required installpkg failed.")
+
+
 # TODO: operate inside an actual chroot for safety? Not that RPM bothers..
-class LoraxTemplateRunner(TemplateRunner):
+class LoraxTemplateRunner(TemplateRunner, InstallpkgMixin):
     '''
     This class parses and executes Lorax templates. Sample usage:
 
@@ -531,77 +672,6 @@ class LoraxTemplateRunner(TemplateRunner):
             logger.error('command returned failure (%d)', e.returncode)
             raise
 
-    def installpkg(self, *pkgs):
-        '''
-        installpkg [--required|--optional] [--except PKGGLOB [--except PKGGLOB ...]] PKGGLOB [PKGGLOB ...]
-          Request installation of all packages matching the given globs.
-          Note that this is just a *request* - nothing is *actually* installed
-          until the 'run_pkg_transaction' command is given.
-
-          --required is now the default. If the PKGGLOB can be missing pass --optional
-        '''
-        if pkgs[0] == '--optional':
-            pkgs = pkgs[1:]
-            required = False
-        elif pkgs[0] == '--required':
-            pkgs = pkgs[1:]
-            required = True
-        else:
-            required = True
-
-        excludes = []
-        while '--except' in pkgs:
-            idx = pkgs.index('--except')
-            if len(pkgs) == idx+1:
-                raise ValueError("installpkg needs an argument after --except")
-
-            excludes.append(pkgs[idx+1])
-            pkgs = pkgs[:idx] + pkgs[idx+2:]
-
-        errors = False
-        for p in pkgs:
-            try:
-                # Start by using Subject to generate a package query, which will
-                # give us a query object similar to what dbo.install would select,
-                # minus the handling for multilib. This query may contain
-                # multiple arches. Pull the package names out of that, filter any
-                # that match the excludes patterns, and pass those names back to
-                # dbo.install to do the actual, arch and version and multilib
-                # aware, package selction.
-
-                # dnf queries don't have a concept of negative globs which is why
-                # the filtering is done the hard way.
-
-                pkgnames = [pkg for pkg in dnf.subject.Subject(p).get_best_query(self.dbo.sack).filter(latest=True)]
-                if not pkgnames:
-                    raise dnf.exceptions.PackageNotFoundError("no package matched", p)
-
-                # Apply excludes to the name only
-                for exclude in excludes:
-                    pkgnames = [pkg for pkg in pkgnames if not fnmatch.fnmatch(pkg.name, exclude)]
-
-                # Convert to a sorted NVR list for installation
-                pkgnvrs = sorted(["{}-{}-{}".format(pkg.name, pkg.version, pkg.release) for pkg in pkgnames])
-
-                # If the request is a glob, expand it in the log
-                if any(g for g in ['*','?','.'] if g in p):
-                    logger.info("installpkg: %s expands to %s", p, ",".join(pkgnvrs))
-
-                for pkgnvr in pkgnvrs:
-                    try:
-                        self.dbo.install(pkgnvr)
-                    except Exception as e: # pylint: disable=broad-except
-                        if required:
-                            raise
-                        # Not required, log it and continue processing pkgs
-                        logger.error("installpkg %s failed: %s", pkgnvr, str(e))
-            except Exception as e: # pylint: disable=broad-except
-                logger.error("installpkg %s failed: %s", p, str(e))
-                errors = True
-
-        if errors and required:
-            raise Exception("Required installpkg failed.")
-
     def removepkg(self, *pkgs):
         '''
         removepkg PKGGLOB [PKGGLOB...]
@@ -800,7 +870,7 @@ class LoraxTemplateRunner(TemplateRunner):
             except CalledProcessError:
                 pass
 
-class LiveTemplateRunner(TemplateRunner):
+class LiveTemplateRunner(TemplateRunner, InstallpkgMixin):
     """
     This class parses and executes a limited Lorax template. Sample usage:
 
@@ -817,68 +887,3 @@ class LiveTemplateRunner(TemplateRunner):
         self.pkgnames = []
 
         super(LiveTemplateRunner, self).__init__(fatalerrors, templatedir, defaults)
-
-    def installpkg(self, *pkgs):
-        '''
-        installpkg [--required|--optional] [--except PKGGLOB [--except PKGGLOB ...]] PKGGLOB [PKGGLOB ...]
-          Request installation of all packages matching the given globs.
-          Note that this is just a *request* - nothing is *actually* installed
-          until the 'run_pkg_transaction' command is given.
-
-          --required is now the default. If the PKGGLOB can be missing pass --optional
-        '''
-        if pkgs[0] == '--optional':
-            pkgs = pkgs[1:]
-            required = False
-        elif pkgs[0] == '--required':
-            pkgs = pkgs[1:]
-            required = True
-        else:
-            required = True
-
-        excludes = []
-        while '--except' in pkgs:
-            idx = pkgs.index('--except')
-            if len(pkgs) == idx+1:
-                raise ValueError("installpkg needs an argument after --except")
-
-            excludes.append(pkgs[idx+1])
-            pkgs = pkgs[:idx] + pkgs[idx+2:]
-
-        errors = False
-        for p in pkgs:
-            try:
-                # Start by using Subject to generate a package query, which will
-                # give us a query object similar to what dbo.install would select,
-                # minus the handling for multilib. This query may contain
-                # multiple arches. Pull the package names out of that, filter any
-                # that match the excludes patterns, and pass those names back to
-                # dbo.install to do the actual, arch and version and multilib
-                # aware, package selction.
-
-                # dnf queries don't have a concept of negative globs which is why
-                # the filtering is done the hard way.
-
-                pkgnames = [pkg for pkg in dnf.subject.Subject(p).get_best_query(self.dbo.sack).filter(latest=True)]
-                if not pkgnames:
-                    raise dnf.exceptions.PackageNotFoundError("no package matched", p)
-
-                # Apply excludes to the name only
-                for exclude in excludes:
-                    pkgnames = [pkg for pkg in pkgnames if not fnmatch.fnmatch(pkg.name, exclude)]
-
-                # Convert to a sorted NVR list for installation
-                pkgnvrs = sorted(["{}-{}-{}".format(pkg.name, pkg.version, pkg.release) for pkg in pkgnames])
-
-                # If the request is a glob, expand it in the log
-                if any(g for g in ['*','?','.'] if g in p):
-                    logger.info("installpkg: %s expands to %s", p, ",".join(pkgnvrs))
-
-                self.pkgs.extend(pkgnvrs)
-                self.pkgnames.extend([pkg.name for pkg in pkgnames])
-            except Exception as e: # pylint: disable=broad-except
-                logger.error("installpkg %s failed: %s", p, str(e))
-                errors = True
-
-        if errors and required:
-            raise Exception("Required installpkg failed.")

tests/pylorax/templates/install-test.tmpl

@@ -1,8 +1,10 @@
 <%page />
 installpkg anaconda-core
 installpkg --optional exact-1.3.17
-installpkg --except fake-homer fake-*
+installpkg --except fake-homer --except fake-milhouse --except fake-lisa fake-*
 installpkg --required lots-of-files
 installpkg known-path
 -installpkg missing-package
+installpkg fake-milhouse>1.0.0-4
+installpkg fake-lisa<1.2.0-1
 run_pkg_transaction

tests/pylorax/test_ltmpl.py

@@ -115,8 +115,10 @@ class LoraxTemplateRunnerTestCase(unittest.TestCase):
         self.repo1_dir = tempfile.mkdtemp(prefix="lorax.test.repo.")
         makeFakeRPM(self.repo1_dir, "anaconda-core", 0, "0.0.1", "1")
         makeFakeRPM(self.repo1_dir, "exact", 0, "1.3.17", "1")
-        makeFakeRPM(self.repo1_dir, "fake-milhouse", 0, "1.0.0", "1")
+        makeFakeRPM(self.repo1_dir, "fake-milhouse", 0, "1.0.0", "1", ["/fake-milhouse/1.0.0-1"])
+        makeFakeRPM(self.repo1_dir, "fake-bart", 0, "1.0.0", "6")
         makeFakeRPM(self.repo1_dir, "fake-bart", 2, "1.13.0", "6")
+        makeFakeRPM(self.repo1_dir, "fake-bart", 2, "2.3.0", "1")
         makeFakeRPM(self.repo1_dir, "fake-homer", 0, "0.4.0", "2")
         makeFakeRPM(self.repo1_dir, "lots-of-files", 0, "0.1.1", "1",
                     ["/lorax-files/file-one.txt",
@@ -126,12 +128,15 @@ class LoraxTemplateRunnerTestCase(unittest.TestCase):
         os.system("createrepo_c " + self.repo1_dir)
 
         self.repo2_dir = tempfile.mkdtemp(prefix="lorax.test.repo.")
-        makeFakeRPM(self.repo2_dir, "fake-milhouse", 0, "1.3.0", "1")
+        makeFakeRPM(self.repo2_dir, "fake-milhouse", 0, "1.0.0", "4", ["/fake-milhouse/1.0.0-4"])
-        makeFakeRPM(self.repo2_dir, "fake-lisa", 0, "1.2.0", "1")
+        makeFakeRPM(self.repo2_dir, "fake-milhouse", 0, "1.0.7", "1", ["/fake-milhouse/1.0.7-1"])
+        makeFakeRPM(self.repo2_dir, "fake-milhouse", 0, "1.3.0", "1", ["/fake-milhouse/1.3.0-1"])
+        makeFakeRPM(self.repo2_dir, "fake-lisa", 0, "1.2.0", "1", ["/fake-lisa/1.2.0-1"])
+        makeFakeRPM(self.repo2_dir, "fake-lisa", 0, "1.1.4", "5", ["/fake-lisa/1.1.4-5"])
         os.system("createrepo_c " + self.repo2_dir)
 
         self.repo3_dir = tempfile.mkdtemp(prefix="lorax.test.debug.repo.")
-        makeFakeRPM(self.repo3_dir, "fake-marge", 0, "2.3.0", "1", ["/fake-marge/file-one.txt"])
+        makeFakeRPM(self.repo3_dir, "fake-marge", 0, "2.3.0", "1", ["/fake-marge/2.3.0-1"])
         makeFakeRPM(self.repo3_dir, "fake-marge-debuginfo", 0, "2.3.0", "1", ["/fake-marge/file-one-debuginfo.txt"])
         os.system("createrepo_c " + self.repo3_dir)
 
@@ -154,24 +159,89 @@ class LoraxTemplateRunnerTestCase(unittest.TestCase):
         shutil.rmtree(self.repo2_dir)
         shutil.rmtree(self.root_dir)
 
-    def test_00_runner_multi_repo(self):
+    def test_pkgver_errors(self):
+        """Test error states of _pkgver"""
+        with self.assertRaises(RuntimeError) as e:
+            self.runner._pkgver("=")
+        self.assertEqual(str(e.exception), "Missing package name")
+
+
+        with self.assertRaises(RuntimeError) as e:
+            self.runner._pkgver("foopkg=")
+        self.assertEqual(str(e.exception), "Missing version")
+
+        with self.assertRaises(RuntimeError) as e:
+            self.runner._pkgver("foopkg>1.0.0-1<1.0.6-1")
+        self.assertEqual(str(e.exception), "Too many comparisons")
+
+
+    def test_00_pkgver(self):
+        """Test all the version comparison operators with pkgver"""
+        matrix = [
+            ("fake-milhouse>=2.1.0-1", ""),                         # Not available
+            ("fake-bart>=2:3.0.0-2", ""),                           # Not available
+            ("fake-bart>2:1.13.0-6", "fake-bart-2:2.3.0-1"),
+            ("fake-bart<2:1.13.0-6", "fake-bart-1.0.0-6"),
+            ("fake-milhouse==1.3.0-1", "fake-milhouse-1.3.0-1"),
+            ("fake-milhouse=1.3.0-1", "fake-milhouse-1.3.0-1"),
+            ("fake-milhouse=1.0.0-4", "fake-milhouse-1.0.0-4"),
+            ("fake-milhouse!=1.3.0-1", "fake-milhouse-1.0.7-1"),
+            ("fake-milhouse<>1.3.0-1", "fake-milhouse-1.0.7-1"),
+            ("fake-milhouse>1.0.0-4", "fake-milhouse-1.3.0-1"),
+            ("fake-milhouse>=1.3.0", "fake-milhouse-1.3.0-1"),
+            ("fake-milhouse>=1.0.7-1", "fake-milhouse-1.3.0-1"),
+            ("fake-milhouse=>1.0.0-4", "fake-milhouse-1.3.0-1"),
+            ("fake-milhouse<=1.0.0-4", "fake-milhouse-1.0.0-4"),
+            ("fake-milhouse=<1.0.7-1", "fake-milhouse-1.0.7-1"),
+            ("fake-milhouse<1.3.0", "fake-milhouse-1.0.7-1"),
+            ("fake-milhouse<1.3.0-1", "fake-milhouse-1.0.7-1"),
+            ("fake-milhouse<1.0.7-1", "fake-milhouse-1.0.0-4"),
+        ]
+
+        def nevra(pkg):
+            if pkg.epoch:
+                return "{}-{}:{}-{}".format(pkg.name, pkg.epoch, pkg.version, pkg.release)
+            else:
+                return "{}-{}-{}".format(pkg.name, pkg.version, pkg.release)
+
+        print([nevra(p) for p in list(self.dnfbase.sack.query().available())])
+        for t in matrix:
+            r = self.runner._pkgver(t[0])
+            if t[1]:
+                self.assertTrue(len(r) > 0, t[0])
+                self.assertEqual(nevra(self.runner._pkgver(t[0])[0]), t[1], t[0])
+            else:
+                self.assertEqual(r, [], t[0])
+
+    def test_01_runner_multi_repo(self):
         """Test installing packages with updates in a 2nd repo"""
         # If this does not raise an error it means that:
         #   Installing a named package works (anaconda-core)
         #   Installing a pinned package works (exact-1.3.17)
         #   Installing a globbed set of package names from multiple repos works
+        #   Installing a package using version compare
         #   removepkg removes a package's files
         #   removefrom removes some, but not all, of a package's files
         #
         # These all need to be done in one template because run_pkg_transaction can only run once
         self.runner.run("install-test.tmpl")
         self.runner.run("install-remove-test.tmpl")
-        self.assertFalse(os.path.exists(joinpaths(self.root_dir, "/known-path/file-one.txt")))
+
-        self.assertTrue(os.path.exists(joinpaths(self.root_dir, "/lorax-files/file-one.txt")))
+        def exists(p):
-        self.assertFalse(os.path.exists(joinpaths(self.root_dir, "/lorax-files/file-two.txt")))
+            return os.path.exists(joinpaths(self.root_dir, p))
+
+        self.assertFalse(exists("/known-path/file-one.txt"))
+        self.assertTrue(exists("/lorax-files/file-one.txt"))
+        self.assertFalse(exists("/lorax-files/file-two.txt"))
+        self.assertTrue(exists("/fake-marge/2.3.0-1"))
 
         # Check the debug log
-        self.assertTrue(os.path.exists(joinpaths(self.root_dir, "/root/debug-pkgs.log")))
+        self.assertTrue(exists("/root/debug-pkgs.log"))
+
+        # Check package version installs
+        self.assertTrue(exists("/fake-lisa/1.1.4-5"))
+        self.assertFalse(exists("/fake-lisa/1.2.0-1"))
+        self.assertTrue(exists("/fake-milhouse/1.3.0-1"))
 
     def test_install_file(self):
         """Test append, and install template commands"""