Add support for version globs to blueprints

This uses dnf's version__glob filter to implement it. It amounts to '*'
wildcards and '?' for single character matching.

src/pylorax/api/compose.py

@@ -223,9 +223,9 @@ def start_build(cfg, dnflock, gitlock, branch, recipe_name, compose_type, test_m
 
     # Combine modules and packages and depsolve the list
     # TODO include the version/glob in the depsolving
-    module_names = [m["name"] for m in recipe["modules"] or []]
+    module_nver = recipe.module_nver
-    package_names = [p["name"] for p in recipe["packages"] or []]
+    package_nver = recipe.package_nver
-    projects = sorted(set(module_names+package_names), key=lambda n: n.lower())
+    projects = sorted(set(module_nver+package_nver), key=lambda p: p[0].lower())
     deps = []
     try:
         with dnflock.lock:
@@ -242,9 +242,10 @@ def start_build(cfg, dnflock, gitlock, branch, recipe_name, compose_type, test_m
     ks_version = makeVersion()
     ks = KickstartParser(ks_version, errorsAreFatal=False, missingIncludeIsFatal=False)
     ks.readKickstartFromString(ks_template+"\n%end\n")
+    ks_projects = [(name, "*") for name in ks.handler.packages.packageList]
     try:
         with dnflock.lock:
-            (template_size, _) = projects_depsolve_with_size(dnflock.dbo, ks.handler.packages.packageList,
+            (template_size, _) = projects_depsolve_with_size(dnflock.dbo, ks_projects,
                                                              with_core=not ks.handler.packages.nocore)
     except ProjectsError as e:
         log.error("start_build depsolve: %s", str(e))

src/pylorax/api/projects.py

@@ -179,29 +179,50 @@ def projects_info(dbo, project_names):
         pkgs = dbo.sack.query().available()
     return sorted(map(pkg_to_project_info, pkgs), key=lambda p: p["name"].lower())
 
+def _depsolve(dbo, projects):
+    """Add projects to a new transaction
 
-def projects_depsolve(dbo, project_names):
+    :param dbo: dnf base object
+    :type dbo: dnf.Base
+    :param projects: The projects and version globs to find the dependencies for
+    :type projects: List of tuples
+    :returns: None
+    :rtype: None
+    :raises: ProjectsError if there was a problem installing something
+    """
+    # This resets the transaction
+    dbo.reset(goal=True)
+    for name, version in projects:
+        try:
+            if not version:
+                version = "*"
+            pkgs = [pkg for pkg in dnf.subject.Subject(name).get_best_query(dbo.sack).filter(version__glob=version, latest=True)]
+            if not pkgs:
+                raise ProjectsError("No match for %s-%s" % (name, version))
+
+            for p in pkgs:
+                dbo.package_install(p)
+        except dnf.exceptions.MarkingError:
+            raise ProjectsError("No match for %s-%s" % (name, version))
+
+
+def projects_depsolve(dbo, projects):
     """Return the dependencies for a list of projects
 
     :param dbo: dnf base object
     :type dbo: dnf.Base
-    :param project_names: The projects to find the dependencies for
+    :param projects: The projects to find the dependencies for
-    :type project_names: List of Strings
+    :type projects: List of Strings
     :returns: NEVRA's of the project and its dependencies
     :rtype: list of dicts
+    :raises: ProjectsError if there was a problem installing something
     """
-    # This resets the transaction
+    _depsolve(dbo, projects)
-    dbo.reset(goal=True)
-    for p in project_names:
-        try:
-            dbo.install(p)
-        except dnf.exceptions.MarkingError:
-            raise ProjectsError("No match for %s" % p)
 
     try:
         dbo.resolve()
     except dnf.exceptions.DepsolveError as e:
-        raise ProjectsError("There was a problem depsolving %s: %s" % (project_names, str(e)))
+        raise ProjectsError("There was a problem depsolving %s: %s" % (projects, str(e)))
 
     if len(dbo.transaction) == 0:
         return []
@@ -230,7 +251,7 @@ def estimate_size(packages, block_size=6144):
     return installed_size
 
 
-def projects_depsolve_with_size(dbo, project_names, with_core=True):
+def projects_depsolve_with_size(dbo, projects, with_core=True):
     """Return the dependencies and installed size for a list of projects
 
     :param dbo: dnf base object
@@ -239,14 +260,9 @@ def projects_depsolve_with_size(dbo, project_names, with_core=True):
     :type project_names: List of Strings
     :returns: installed size and a list of NEVRA's of the project and its dependencies
     :rtype: tuple of (int, list of dicts)
+    :raises: ProjectsError if there was a problem installing something
     """
-    # This resets the transaction
+    _depsolve(dbo, projects)
-    dbo.reset(goal=True)
-    for p in project_names:
-        try:
-            dbo.install(p)
-        except dnf.exceptions.MarkingError:
-            raise ProjectsError("No match for %s" % p)
 
     if with_core:
         dbo.group_install("core", ['mandatory', 'default', 'optional'])
@@ -254,7 +270,7 @@ def projects_depsolve_with_size(dbo, project_names, with_core=True):
     try:
         dbo.resolve()
     except dnf.exceptions.DepsolveError as e:
-        raise ProjectsError("There was a problem depsolving %s: %s" % (project_names, str(e)))
+        raise ProjectsError("There was a problem depsolving %s: %s" % (projects, str(e)))
 
     if len(dbo.transaction) == 0:
         return (0, [])
@@ -299,6 +315,6 @@ def modules_info(dbo, module_names):
 
     # Add the dependency info to each one
     for module in modules:
-        module["dependencies"] = projects_depsolve(dbo, [module["name"]])
+        module["dependencies"] = projects_depsolve(dbo, [(module["name"], "*.*")])
 
     return modules

src/pylorax/api/recipes.py

@@ -73,11 +73,21 @@ class Recipe(dict):
         """Return the names of the packages"""
         return [p["name"] for p in self["packages"] or []]
 
+    @property
+    def package_nver(self):
+        """Return the names and version globs of the packages"""
+        return [(p["name"], p["version"]) for p in self["packages"] or []]
+
     @property
     def module_names(self):
         """Return the names of the modules"""
         return [m["name"] for m in self["modules"] or []]
 
+    @property
+    def module_nver(self):
+        """Return the names and version globs of the modules"""
+        return [(m["name"], m["version"]) for m in self["modules"] or []]
+
     @property
     def filename(self):
         """Return the Recipe's filename

src/pylorax/api/v0.py

@@ -1181,9 +1181,9 @@ def v0_api(api):
 
             # Combine modules and packages and depsolve the list
             # TODO include the version/glob in the depsolving
-            module_names = blueprint.module_names
+            module_nver = blueprint.module_nver
-            package_names = blueprint.package_names
+            package_nver = blueprint.package_nver
-            projects = sorted(set(module_names+package_names), key=lambda n: n.lower())
+            projects = sorted(set(module_nver+package_nver), key=lambda p: p[0].lower())
             deps = []
             try:
                 with api.config["DNFLOCK"].lock:
@@ -1233,9 +1233,9 @@ def v0_api(api):
 
             # Combine modules and packages and depsolve the list
             # TODO include the version/glob in the depsolving
-            module_names = [m["name"] for m in blueprint["modules"] or []]
+            module_nver = blueprint.module_nver
-            package_names = [p["name"] for p in blueprint["packages"] or []]
+            package_nver = blueprint.package_nver
-            projects = sorted(set(module_names+package_names), key=lambda n: n.lower())
+            projects = sorted(set(module_nver+package_nver), key=lambda p: p[0].lower())
             deps = []
             try:
                 with api.config["DNFLOCK"].lock:
@@ -1294,7 +1294,7 @@ def v0_api(api):
         """Return detailed information about the listed projects"""
         try:
             with api.config["DNFLOCK"].lock:
-                deps = projects_depsolve(api.config["DNFLOCK"].dbo, project_names.split(","))
+                deps = projects_depsolve(api.config["DNFLOCK"].dbo, [(n, "*") for n in project_names.split(",")])
         except ProjectsError as e:
             log.error("(v0_projects_depsolve) %s", str(e))
             return jsonify(status=False, errors=[str(e)]), 400

tests/pylorax/test_projects.py

@@ -151,13 +151,27 @@ class ProjectsTest(unittest.TestCase):
         self.assertEqual(projects[0]["builds"][0]["source"]["license"], "GPLv3+")
 
     def test_projects_depsolve(self):
-        deps = projects_depsolve(self.dbo, ["bash"])
+        deps = projects_depsolve(self.dbo, [("bash", "*.*")])
 
         self.assertEqual(deps[0]["name"], "basesystem")
 
+    def test_projects_depsolve_version(self):
+        """Test that depsolving with a partial wildcard version works"""
+        deps = projects_depsolve(self.dbo, [("bash", "4.*")])
+        self.assertEqual(deps[1]["name"], "bash")
+
+        deps = projects_depsolve(self.dbo, [("bash", "4.4.*")])
+        self.assertEqual(deps[1]["name"], "bash")
+
+    def test_projects_depsolve_oldversion(self):
+        """Test that depsolving a specific non-existant version fails"""
+        with self.assertRaises(ProjectsError):
+            deps = projects_depsolve(self.dbo, [("bash", "1.0.0")])
+            self.assertEqual(deps[1]["name"], "bash")
+
     def test_projects_depsolve_fail(self):
         with self.assertRaises(ProjectsError):
-            projects_depsolve(self.dbo, ["nada-package"])
+            projects_depsolve(self.dbo, [("nada-package", "*.*")])
 
     def test_modules_list_all(self):
         modules = modules_list(self.dbo, None)

tests/pylorax/test_server.py

@@ -173,7 +173,7 @@ class ServerTestCase(unittest.TestCase):
                        "modules":[{"name":"glusterfs", "version":"4.0.*"},
                                   {"name":"glusterfs-cli", "version":"4.0.*"}],
                        "packages":[{"name":"samba", "version":"4.8.*"},
-                                   {"name":"tmux", "version":"2.2"}]}
+                                   {"name":"tmux", "version":"2.7"}]}
 
         resp = self.server.post("/api/v0/blueprints/new",
                                 data=json.dumps(test_blueprint),
@@ -217,7 +217,7 @@ class ServerTestCase(unittest.TestCase):
                        "modules":[{"name":"glusterfs", "version":"4.0.*"},
                                   {"name":"glusterfs-cli", "version":"4.0.*"}],
                        "packages":[{"name":"samba", "version":"4.8.*"},
-                                   {"name":"tmux", "version":"2.2"}]}
+                                   {"name":"tmux", "version":"2.7"}]}
 
         resp = self.server.post("/api/v0/blueprints/workspace",
                                 data=json.dumps(test_blueprint),
@@ -243,7 +243,7 @@ class ServerTestCase(unittest.TestCase):
                        "modules":[{"name":"glusterfs", "version":"4.0.*"},
                                   {"name":"glusterfs-cli", "version":"4.0.*"}],
                        "packages":[{"name":"samba", "version":"4.8.*"},
-                                   {"name":"tmux", "version":"2.2"}]}
+                                   {"name":"tmux", "version":"2.7"}]}
 
         resp = self.server.post("/api/v0/blueprints/workspace",
                                 data=json.dumps(test_blueprint),
@@ -374,7 +374,7 @@ class ServerTestCase(unittest.TestCase):
                        "modules":[{"name":"glusterfs", "version":"4.0.*"},
                                   {"name":"glusterfs-cli", "version":"4.0.*"}],
                        "packages":[{"name":"samba", "version":"4.8.*"},
-                                   {"name":"tmux", "version":"2.2"}]}
+                                   {"name":"tmux", "version":"2.7"}]}
 
         resp = self.server.post("/api/v0/blueprints/workspace",
                                 data=json.dumps(test_blueprint),
@@ -390,7 +390,7 @@ class ServerTestCase(unittest.TestCase):
                              "old": {"Description": "An example GlusterFS server with samba"}},
                             {"new": {"Version": "0.3.0"},
                              "old": {"Version": "0.0.1"}},
-                            {"new": {"Package": {"version": "2.2", "name": "tmux"}},
+                            {"new": {"Package": {"version": "2.7", "name": "tmux"}},
                              "old": None}]}
         self.assertEqual(data, result)
 
@@ -437,6 +437,7 @@ class ServerTestCase(unittest.TestCase):
         blueprints = data.get("blueprints")
         self.assertNotEqual(blueprints, None)
         self.assertEqual(len(blueprints), 1)
+        self.assertTrue(len(blueprints[0]["blueprint"]["modules"]) > 0)
         self.assertEqual(blueprints[0]["blueprint"]["name"], "glusterfs")
         evra = blueprints[0]["blueprint"]["modules"][0]["version"]
         self.assertEqual(len(evra) > 10, True)
@@ -502,7 +503,7 @@ class ServerTestCase(unittest.TestCase):
                        "modules":[{"name":"glusterfs", "version":"4.0.*"},
                                   {"name":"glusterfs-cli", "version":"4.0.*"}],
                        "packages":[{"name":"samba", "version":"4.8.*"},
-                                   {"name":"tmux", "version":"2.2"}]}
+                                   {"name":"tmux", "version":"2.7"}]}
 
         resp = self.server.post("/api/v0/blueprints/new?branch=test",
                                 data=json.dumps(test_blueprint),