Add support for version globs to blueprints

You can use '*' wildcards and '?' for single character matching.

src/pylorax/api/compose.py

@@ -230,10 +230,9 @@ def start_build(cfg, yumlock, gitlock, branch, recipe_name, compose_type, test_m
         (commit_id, recipe) = read_recipe_and_id(gitlock.repo, branch, recipe_name)
 
     # Combine modules and packages and depsolve the list
-    # TODO include the version/glob in the depsolving
+    module_nver = recipe.module_nver
-    module_names = map(lambda m: m["name"], recipe["modules"] or [])
+    package_nver = recipe.package_nver
-    package_names = map(lambda p: p["name"], recipe["packages"] or [])
+    projects = sorted(set(module_nver+package_nver), key=lambda p: p[0].lower())
-    projects = sorted(set(module_names+package_names), key=lambda n: n.lower())
     deps = []
     try:
         with yumlock.lock:
@@ -250,9 +249,10 @@ def start_build(cfg, yumlock, gitlock, branch, recipe_name, compose_type, test_m
     ks_version = makeVersion(RHEL7)
     ks = KickstartParser(ks_version, errorsAreFatal=False, missingIncludeIsFatal=False)
     ks.readKickstartFromString(ks_template+"\n%end\n")
+    pkgs = [(name, "*") for name in ks.handler.packages.packageList]
     try:
         with yumlock.lock:
-            (template_size, _) = projects_depsolve_with_size(yumlock.yb, ks.handler.packages.packageList,
+            (template_size, _) = projects_depsolve_with_size(yumlock.yb, pkgs,
                                                              with_core=not ks.handler.packages.nocore)
     except ProjectsError as e:
         log.error("start_build depsolve: %s", str(e))

src/pylorax/api/projects.py

@@ -186,28 +186,31 @@ def projects_info(yb, project_names):
     return sorted(map(yaps_to_project_info, ybl.available), key=lambda p: p["name"].lower())
 
 
-def projects_depsolve(yb, project_names):
+def projects_depsolve(yb, projects):
     """Return the dependencies for a list of projects
 
     :param yb: yum base object
     :type yb: YumBase
-    :param project_names: The projects to find the dependencies for
+    :param projects: The projects and version globs to find the dependencies for
-    :type project_names: List of Strings
+    :type projects: List of tuples
     :returns: NEVRA's of the project and its dependencies
     :rtype: list of dicts
+    :raises: ProjectsError if there was a problem installing something
     """
     try:
         # This resets the transaction
         yb.closeRpmDB()
-        for p in project_names:
+        for name, version in projects:
-            yb.install(pattern=p)
+            if not version:
+                version = "*"
+            yb.install(pattern="%s-%s" % (name, version))
         (rc, msg) = yb.buildTransaction()
         if rc not in [0, 1, 2]:
-            raise ProjectsError("There was a problem depsolving %s: %s" % (project_names, msg))
+            raise ProjectsError("There was a problem depsolving %s: %s" % (projects, msg))
         yb.tsInfo.makelists()
         deps = sorted(map(tm_to_dep, yb.tsInfo.installed + yb.tsInfo.depinstalled), key=lambda p: p["name"].lower())
     except YumBaseError as e:
-        raise ProjectsError("There was a problem depsolving %s: %s" % (project_names, str(e)))
+        raise ProjectsError("There was a problem depsolving %s: %s" % (projects, str(e)))
     finally:
         yb.closeRpmDB()
     return deps
@@ -232,31 +235,34 @@ def estimate_size(packages, block_size=4096):
         installed_size += p.po.installedsize
     return installed_size
 
-def projects_depsolve_with_size(yb, project_names, with_core=True):
+def projects_depsolve_with_size(yb, projects, with_core=True):
     """Return the dependencies and installed size for a list of projects
 
     :param yb: yum base object
     :type yb: YumBase
-    :param project_names: The projects to find the dependencies for
+    :param projects: The projects and version globs to find the dependencies for
-    :type project_names: List of Strings
+    :type projects: List of tuples
     :returns: installed size and a list of NEVRA's of the project and its dependencies
     :rtype: tuple of (int, list of dicts)
+    :raises: ProjectsError if there was a problem installing something
     """
     try:
         # This resets the transaction
         yb.closeRpmDB()
-        for p in project_names:
+        for name, version in projects:
-            yb.install(pattern=p)
+            if not version:
+                version = "*"
+            yb.install(pattern="%s-%s" % (name, version))
         if with_core:
             yb.selectGroup("core", group_package_types=['mandatory', 'default', 'optional'])
         (rc, msg) = yb.buildTransaction()
         if rc not in [0, 1, 2]:
-            raise ProjectsError("There was a problem depsolving %s: %s" % (project_names, msg))
+            raise ProjectsError("There was a problem depsolving %s: %s" % (projects, msg))
         yb.tsInfo.makelists()
         installed_size = estimate_size(yb.tsInfo.installed + yb.tsInfo.depinstalled)
         deps = sorted(map(tm_to_dep, yb.tsInfo.installed + yb.tsInfo.depinstalled), key=lambda p: p["name"].lower())
     except YumBaseError as e:
-        raise ProjectsError("There was a problem depsolving %s: %s" % (project_names, str(e)))
+        raise ProjectsError("There was a problem depsolving %s: %s" % (projects, str(e)))
     finally:
         yb.closeRpmDB()
     return (installed_size, deps)
@@ -306,6 +312,6 @@ def modules_info(yb, module_names):
     modules = sorted(map(yaps_to_project, ybl.available), key=lambda p: p["name"].lower())
     # Add the dependency info to each one
     for module in modules:
-        module["dependencies"] = projects_depsolve(yb, [module["name"]])
+        module["dependencies"] = projects_depsolve(yb, [(module["name"], "*")])
 
     return modules

src/pylorax/api/recipes.py

@@ -73,11 +73,21 @@ class Recipe(dict):
         """Return the names of the packages"""
         return map(lambda p: p["name"], self["packages"] or [])
 
+    @property
+    def package_nver(self):
+        """Return the names and versions  of the packages"""
+        return [(p["name"], p["version"]) for p in self["packages"] or []]
+
     @property
     def module_names(self):
         """Return the names of the modules"""
         return map(lambda m: m["name"], self["modules"] or [])
 
+    @property
+    def module_nver(self):
+        """Return the names and versions of the modules"""
+        return [(m["name"], m["version"]) for m in self["modules"] or []]
+
     @property
     def filename(self):
         """Return the Recipe's filename

src/pylorax/api/v0.py

@@ -1181,9 +1181,9 @@ def v0_api(api):
 
             # Combine modules and packages and depsolve the list
             # TODO include the version/glob in the depsolving
-            module_names = blueprint.module_names
+            module_nver = blueprint.module_nver
-            package_names = blueprint.package_names
+            package_nver = blueprint.package_nver
-            projects = sorted(set(module_names+package_names), key=lambda n: n.lower())
+            projects = sorted(set(module_nver+package_nver), key=lambda p: p[0].lower())
             deps = []
             try:
                 with api.config["YUMLOCK"].lock:
@@ -1232,10 +1232,9 @@ def v0_api(api):
                 continue
 
             # Combine modules and packages and depsolve the list
-            # TODO include the version/glob in the depsolving
+            module_nver = blueprint.module_nver
-            module_names = map(lambda m: m["name"], blueprint["modules"] or [])
+            package_nver = blueprint.package_nver
-            package_names = map(lambda p: p["name"], blueprint["packages"] or [])
+            projects = sorted(set(module_nver+package_nver), key=lambda p: p[0].lower())
-            projects = sorted(set(module_names+package_names), key=lambda n: n.lower())
             deps = []
             try:
                 with api.config["YUMLOCK"].lock:
@@ -1294,7 +1293,7 @@ def v0_api(api):
         """Return detailed information about the listed projects"""
         try:
             with api.config["YUMLOCK"].lock:
-                deps = projects_depsolve(api.config["YUMLOCK"].yb, project_names.split(","))
+                deps = projects_depsolve(api.config["YUMLOCK"].yb, [(n, "*") for n in project_names.split(",")])
         except ProjectsError as e:
             log.error("(v0_projects_depsolve) %s", str(e))
             return jsonify(status=False, errors=[str(e)]), 400

tests/pylorax/blueprints/custom-base.toml

@@ -4,7 +4,7 @@ version = "0.0.1"
 
 [[packages]]
 name = "bash"
-version = "4.4.*"
+version = "4.2.*"
 
 [customizations]
 hostname = "custombase"

tests/pylorax/blueprints/glusterfs.toml

@@ -3,12 +3,12 @@ description = "An example GlusterFS server with samba"
 
 [[modules]]
 name = "glusterfs"
-version = "3.7.*"
+version = "3.8.*"
 
 [[modules]]
 name = "glusterfs-cli"
-version = "3.7.*"
+version = "3.8.*"
 
 [[packages]]
 name = "samba"
-version = "4.2.*"
+version = "4.7.*"

tests/pylorax/blueprints/http-server.toml

@@ -24,12 +24,12 @@ version = "5.4.*"
 
 [[packages]]
 name = "tmux"
-version = "2.2"
+version = "1.8"
 
 [[packages]]
 name = "openssh-server"
-version = "6.6.*"
+version = "7.*"
 
 [[packages]]
 name = "rsync"
-version = "3.0.*"
+version = "3.1.*"

tests/pylorax/blueprints/kubernetes.toml

@@ -3,23 +3,23 @@ description = "An example kubernetes master"
 
 [[modules]]
 name = "kubernetes"
-version = "1.2.*"
+version = "1.5.2"
 
 [[modules]]
 name = "docker"
-version = "1.10.*"
+version = "1.13.*"
 
 [[modules]]
 name = "docker-lvm-plugin"
-version = "1.10.*"
+version = "1.13.*"
 
 [[modules]]
 name = "etcd"
-version = "2.3.*"
+version = "3.2.*"
 
 [[modules]]
 name = "flannel"
-version = "0.5.*"
+version = "0.7.*"
 
 [[packages]]
 name = "oci-systemd-hook"

tests/pylorax/test_projects.py

@@ -178,13 +178,27 @@ class ProjectsTest(unittest.TestCase):
                 projects_info(self.yb, ["bash"])
 
     def test_projects_depsolve(self):
-        deps = projects_depsolve(self.yb, ["bash"])
+        deps = projects_depsolve(self.yb, [("bash", "*.*")])
+        self.assertTrue(len(deps) > 3)
+        self.assertEqual(deps[2]["name"], "basesystem")
 
-        self.assertEqual(deps[0]["name"], "basesystem")
+    def test_projects_depsolve_version(self):
+        """Test that depsolving with a partial wildcard version works"""
+        deps = projects_depsolve(self.yb, [("bash", "4.*")])
+        self.assertEqual(deps[1]["name"], "bash")
+
+        deps = projects_depsolve(self.yb, [("bash", "4.2.*")])
+        self.assertEqual(deps[1]["name"], "bash")
+
+    def test_projects_depsolve_oldversion(self):
+        """Test that depsolving a specific non-existant version fails"""
+        with self.assertRaises(ProjectsError):
+            deps = projects_depsolve(self.yb, [("bash", "1.0.0")])
+            self.assertEqual(deps[1]["name"], "bash")
 
     def test_projects_depsolve_fail(self):
         with self.assertRaises(ProjectsError):
-            projects_depsolve(self.yb, ["nada-package"])
+            projects_depsolve(self.yb, [("nada-package", "*.*")])
 
     def test_modules_list(self):
         modules = modules_list(self.yb, None)
@@ -204,8 +218,10 @@ class ProjectsTest(unittest.TestCase):
         modules = modules_info(self.yb, ["bash"])
 
         print(modules)
+        self.assertTrue(len(modules) > 0)
+        self.assertTrue(len(modules[0]["dependencies"]) > 3)
         self.assertEqual(modules[0]["name"], "bash")
-        self.assertEqual(modules[0]["dependencies"][0]["name"], "basesystem")
+        self.assertEqual(modules[0]["dependencies"][2]["name"], "basesystem")
 
     def test_modules_info_yum_raises_exception(self):
         with self.assertRaises(ProjectsError):

tests/pylorax/test_server.py

@@ -97,9 +97,9 @@ class ServerTestCase(unittest.TestCase):
                                               {"name":"php", "version":"5.4.*"},
                                               {"name": "php-mysql", "version":"5.4.*"}],
                                    "name":"http-server",
-                                   "packages": [{"name":"openssh-server", "version": "6.6.*"},
+                                   "packages": [{"name":"openssh-server", "version": "7.*"},
-                                                {"name": "rsync", "version": "3.0.*"},
+                                                {"name": "rsync", "version": "3.1.*"},
-                                                {"name": "tmux", "version": "2.2"}],
+                                                {"name": "tmux", "version": "1.8"}],
                                    "version": "0.0.1"}]}
         resp = self.server.get("/api/v0/blueprints/info/http-server")
         data = json.loads(resp.data)
@@ -109,10 +109,10 @@ class ServerTestCase(unittest.TestCase):
                                   {"changed":False, "name":"http-server"}],
                        "errors":[],
                        "blueprints":[{"description": "An example GlusterFS server with samba",
-                                   "modules":[{"name":"glusterfs", "version":"3.7.*"},
+                                   "modules":[{"name":"glusterfs", "version":"3.8.*"},
-                                              {"name":"glusterfs-cli", "version":"3.7.*"}],
+                                              {"name":"glusterfs-cli", "version":"3.8.*"}],
                                    "name":"glusterfs",
-                                   "packages":[{"name":"samba", "version":"4.2.*"}],
+                                   "packages":[{"name":"samba", "version":"4.7.*"}],
                                    "version": "0.0.1"},
                                   {"description":"An example http server with PHP and MySQL support.",
                                    "modules":[{"name":"httpd", "version":"2.4.*"},
@@ -121,9 +121,9 @@ class ServerTestCase(unittest.TestCase):
                                               {"name":"php", "version":"5.4.*"},
                                               {"name": "php-mysql", "version":"5.4.*"}],
                                    "name":"http-server",
-                                   "packages": [{"name":"openssh-server", "version": "6.6.*"},
+                                   "packages": [{"name":"openssh-server", "version": "7.*"},
-                                                {"name": "rsync", "version": "3.0.*"},
+                                                {"name": "rsync", "version": "3.1.*"},
-                                                {"name": "tmux", "version": "2.2"}],
+                                                {"name": "tmux", "version": "1.8"}],
                                    "version": "0.0.1"},
                                  ]}
         resp = self.server.get("/api/v0/blueprints/info/http-server,glusterfs")
@@ -164,10 +164,10 @@ class ServerTestCase(unittest.TestCase):
         test_blueprint = {"description": "An example GlusterFS server with samba",
                        "name":"glusterfs",
                        "version": "0.2.0",
-                       "modules":[{"name":"glusterfs", "version":"3.7.*"},
+                       "modules":[{"name":"glusterfs", "version":"3.8.*"},
-                                  {"name":"glusterfs-cli", "version":"3.7.*"}],
+                                  {"name":"glusterfs-cli", "version":"3.8.*"}],
-                       "packages":[{"name":"samba", "version":"4.2.*"},
+                       "packages":[{"name":"samba", "version":"4.7.*"},
-                                   {"name":"tmux", "version":"2.2"}]}
+                                   {"name":"tmux", "version":"1.8"}]}
 
         resp = self.server.post("/api/v0/blueprints/new",
                                 data=json.dumps(test_blueprint),
@@ -208,10 +208,10 @@ class ServerTestCase(unittest.TestCase):
         test_blueprint = {"description": "An example GlusterFS server with samba, ws version",
                        "name":"glusterfs",
                        "version": "0.3.0",
-                       "modules":[{"name":"glusterfs", "version":"3.7.*"},
+                       "modules":[{"name":"glusterfs", "version":"3.8.*"},
-                                  {"name":"glusterfs-cli", "version":"3.7.*"}],
+                                  {"name":"glusterfs-cli", "version":"3.8.*"}],
-                       "packages":[{"name":"samba", "version":"4.2.*"},
+                       "packages":[{"name":"samba", "version":"4.7.*"},
-                                   {"name":"tmux", "version":"2.2"}]}
+                                   {"name":"tmux", "version":"1.8"}]}
 
         resp = self.server.post("/api/v0/blueprints/workspace",
                                 data=json.dumps(test_blueprint),
@@ -234,10 +234,10 @@ class ServerTestCase(unittest.TestCase):
         test_blueprint = {"description": "An example GlusterFS server with samba, ws version",
                        "name":"glusterfs",
                        "version": "0.4.0",
-                       "modules":[{"name":"glusterfs", "version":"3.7.*"},
+                       "modules":[{"name":"glusterfs", "version":"3.8.*"},
-                                  {"name":"glusterfs-cli", "version":"3.7.*"}],
+                                  {"name":"glusterfs-cli", "version":"3.8.*"}],
-                       "packages":[{"name":"samba", "version":"4.2.*"},
+                       "packages":[{"name":"samba", "version":"4.7.*"},
-                                   {"name":"tmux", "version":"2.2"}]}
+                                   {"name":"tmux", "version":"1.8"}]}
 
         resp = self.server.post("/api/v0/blueprints/workspace",
                                 data=json.dumps(test_blueprint),
@@ -361,10 +361,10 @@ class ServerTestCase(unittest.TestCase):
         test_blueprint = {"description": "An example GlusterFS server with samba, ws version",
                        "name":"glusterfs",
                        "version": "0.3.0",
-                       "modules":[{"name":"glusterfs", "version":"3.7.*"},
+                       "modules":[{"name":"glusterfs", "version":"3.8.*"},
-                                  {"name":"glusterfs-cli", "version":"3.7.*"}],
+                                  {"name":"glusterfs-cli", "version":"3.8.*"}],
-                       "packages":[{"name":"samba", "version":"4.2.*"},
+                       "packages":[{"name":"samba", "version":"4.7.*"},
-                                   {"name":"tmux", "version":"2.2"}]}
+                                   {"name":"tmux", "version":"1.8"}]}
 
         resp = self.server.post("/api/v0/blueprints/workspace",
                                 data=json.dumps(test_blueprint),
@@ -380,7 +380,7 @@ class ServerTestCase(unittest.TestCase):
                              "old": {"Description": "An example GlusterFS server with samba"}},
                             {"new": {"Version": "0.3.0"},
                              "old": {"Version": "0.0.1"}},
-                            {"new": {"Package": {"version": "2.2", "name": "tmux"}},
+                            {"new": {"Package": {"version": "1.8", "name": "tmux"}},
                              "old": None}]}
         self.assertEqual(data, result)
 
@@ -392,6 +392,7 @@ class ServerTestCase(unittest.TestCase):
         blueprints = data.get("blueprints")
         self.assertNotEqual(blueprints, None)
         self.assertEqual(len(blueprints), 1)
+        self.assertTrue(len(blueprints[0]["blueprint"]["modules"]) > 0)
         self.assertEqual(blueprints[0]["blueprint"]["name"], "glusterfs")
         self.assertEqual(len(blueprints[0]["dependencies"]) > 10, True)
         self.assertFalse(data.get("errors"))
@@ -427,6 +428,7 @@ class ServerTestCase(unittest.TestCase):
         blueprints = data.get("blueprints")
         self.assertNotEqual(blueprints, None)
         self.assertEqual(len(blueprints), 1)
+        self.assertTrue(len(blueprints[0]["blueprint"]["modules"]) > 0)
         self.assertEqual(blueprints[0]["blueprint"]["name"], "glusterfs")
         evra = blueprints[0]["blueprint"]["modules"][0]["version"]
         self.assertEqual(len(evra) > 10, True)
@@ -446,6 +448,7 @@ class ServerTestCase(unittest.TestCase):
         self.assertNotEqual(data, None)
         projects = data.get("projects")
         self.assertEqual(len(projects), 1)
+        self.assertTrue(len(projects[0]["builds"]) > 0)
         self.assertEqual(projects[0]["name"], "bash")
         self.assertEqual(projects[0]["builds"][0]["source"]["license"], "GPLv3+")
 
@@ -456,7 +459,7 @@ class ServerTestCase(unittest.TestCase):
         self.assertNotEqual(data, None)
         deps = data.get("projects")
         self.assertEqual(len(deps) > 10, True)
-        self.assertEqual(deps[0]["name"], "basesystem")
+        self.assertEqual(deps[2]["name"], "basesystem")
 
     def test_modules_list(self):
         """Test /api/v0/modules/list"""
@@ -482,17 +485,17 @@ class ServerTestCase(unittest.TestCase):
         modules = data.get("modules")
         self.assertEqual(len(modules), 1)
         self.assertEqual(modules[0]["name"], "bash")
-        self.assertEqual(modules[0]["dependencies"][0]["name"], "basesystem")
+        self.assertEqual(modules[0]["dependencies"][2]["name"], "basesystem")
 
     def test_blueprint_new_branch(self):
         """Test the /api/v0/blueprints/new route with a new branch"""
         test_blueprint = {"description": "An example GlusterFS server with samba",
                        "name":"glusterfs",
                        "version": "0.2.0",
-                       "modules":[{"name":"glusterfs", "version":"3.7.*"},
+                       "modules":[{"name":"glusterfs", "version":"3.8.*"},
-                                  {"name":"glusterfs-cli", "version":"3.7.*"}],
+                                  {"name":"glusterfs-cli", "version":"3.8.*"}],
-                       "packages":[{"name":"samba", "version":"4.2.*"},
+                       "packages":[{"name":"samba", "version":"4.7.*"},
-                                   {"name":"tmux", "version":"2.2"}]}
+                                   {"name":"tmux", "version":"1.8"}]}
 
         resp = self.server.post("/api/v0/blueprints/new?branch=test",
                                 data=json.dumps(test_blueprint),