From 1af604014395832fcdc049bea825615765a0e478 Mon Sep 17 00:00:00 2001 From: "Brian C. Lane" Date: Fri, 18 May 2018 12:03:26 -0700 Subject: [PATCH] Add support for version globs to blueprints This uses dnf's version__glob filter to implement it. It amounts to '*' wildcards and '?' for single character matching. (cherry picked from commit 095829171ab7646af9ceb730bfe2b2b1a19d1f24) --- src/pylorax/api/compose.py | 9 +++--- src/pylorax/api/projects.py | 58 ++++++++++++++++++++++------------ src/pylorax/api/recipes.py | 10 ++++++ src/pylorax/api/v0.py | 14 ++++---- tests/pylorax/test_projects.py | 18 +++++++++-- tests/pylorax/test_server.py | 13 ++++---- 6 files changed, 82 insertions(+), 40 deletions(-) diff --git a/src/pylorax/api/compose.py b/src/pylorax/api/compose.py index 880919dc..b9824149 100644 --- a/src/pylorax/api/compose.py +++ b/src/pylorax/api/compose.py @@ -223,9 +223,9 @@ def start_build(cfg, dnflock, gitlock, branch, recipe_name, compose_type, test_m # Combine modules and packages and depsolve the list # TODO include the version/glob in the depsolving - module_names = [m["name"] for m in recipe["modules"] or []] - package_names = [p["name"] for p in recipe["packages"] or []] - projects = sorted(set(module_names+package_names), key=lambda n: n.lower()) + module_nver = recipe.module_nver + package_nver = recipe.package_nver + projects = sorted(set(module_nver+package_nver), key=lambda p: p[0].lower()) deps = [] try: with dnflock.lock: @@ -242,9 +242,10 @@ def start_build(cfg, dnflock, gitlock, branch, recipe_name, compose_type, test_m ks_version = makeVersion() ks = KickstartParser(ks_version, errorsAreFatal=False, missingIncludeIsFatal=False) ks.readKickstartFromString(ks_template+"\n%end\n") + ks_projects = [(name, "*") for name in ks.handler.packages.packageList] try: with dnflock.lock: - (template_size, _) = projects_depsolve_with_size(dnflock.dbo, ks.handler.packages.packageList, + (template_size, _) = projects_depsolve_with_size(dnflock.dbo, ks_projects, with_core=not ks.handler.packages.nocore) except ProjectsError as e: log.error("start_build depsolve: %s", str(e)) diff --git a/src/pylorax/api/projects.py b/src/pylorax/api/projects.py index 80ca351b..0292198e 100644 --- a/src/pylorax/api/projects.py +++ b/src/pylorax/api/projects.py @@ -179,29 +179,50 @@ def projects_info(dbo, project_names): pkgs = dbo.sack.query().available() return sorted(map(pkg_to_project_info, pkgs), key=lambda p: p["name"].lower()) +def _depsolve(dbo, projects): + """Add projects to a new transaction -def projects_depsolve(dbo, project_names): + :param dbo: dnf base object + :type dbo: dnf.Base + :param projects: The projects and version globs to find the dependencies for + :type projects: List of tuples + :returns: None + :rtype: None + :raises: ProjectsError if there was a problem installing something + """ + # This resets the transaction + dbo.reset(goal=True) + for name, version in projects: + try: + if not version: + version = "*" + pkgs = [pkg for pkg in dnf.subject.Subject(name).get_best_query(dbo.sack).filter(version__glob=version, latest=True)] + if not pkgs: + raise ProjectsError("No match for %s-%s" % (name, version)) + + for p in pkgs: + dbo.package_install(p) + except dnf.exceptions.MarkingError: + raise ProjectsError("No match for %s-%s" % (name, version)) + + +def projects_depsolve(dbo, projects): """Return the dependencies for a list of projects :param dbo: dnf base object :type dbo: dnf.Base - :param project_names: The projects to find the dependencies for - :type project_names: List of Strings + :param projects: The projects to find the dependencies for + :type projects: List of Strings :returns: NEVRA's of the project and its dependencies :rtype: list of dicts + :raises: ProjectsError if there was a problem installing something """ - # This resets the transaction - dbo.reset(goal=True) - for p in project_names: - try: - dbo.install(p) - except dnf.exceptions.MarkingError: - raise ProjectsError("No match for %s" % p) + _depsolve(dbo, projects) try: dbo.resolve() except dnf.exceptions.DepsolveError as e: - raise ProjectsError("There was a problem depsolving %s: %s" % (project_names, str(e))) + raise ProjectsError("There was a problem depsolving %s: %s" % (projects, str(e))) if len(dbo.transaction) == 0: return [] @@ -230,7 +251,7 @@ def estimate_size(packages, block_size=6144): return installed_size -def projects_depsolve_with_size(dbo, project_names, with_core=True): +def projects_depsolve_with_size(dbo, projects, with_core=True): """Return the dependencies and installed size for a list of projects :param dbo: dnf base object @@ -239,14 +260,9 @@ def projects_depsolve_with_size(dbo, project_names, with_core=True): :type project_names: List of Strings :returns: installed size and a list of NEVRA's of the project and its dependencies :rtype: tuple of (int, list of dicts) + :raises: ProjectsError if there was a problem installing something """ - # This resets the transaction - dbo.reset(goal=True) - for p in project_names: - try: - dbo.install(p) - except dnf.exceptions.MarkingError: - raise ProjectsError("No match for %s" % p) + _depsolve(dbo, projects) if with_core: dbo.group_install("core", ['mandatory', 'default', 'optional']) @@ -254,7 +270,7 @@ def projects_depsolve_with_size(dbo, project_names, with_core=True): try: dbo.resolve() except dnf.exceptions.DepsolveError as e: - raise ProjectsError("There was a problem depsolving %s: %s" % (project_names, str(e))) + raise ProjectsError("There was a problem depsolving %s: %s" % (projects, str(e))) if len(dbo.transaction) == 0: return (0, []) @@ -299,6 +315,6 @@ def modules_info(dbo, module_names): # Add the dependency info to each one for module in modules: - module["dependencies"] = projects_depsolve(dbo, [module["name"]]) + module["dependencies"] = projects_depsolve(dbo, [(module["name"], "*.*")]) return modules diff --git a/src/pylorax/api/recipes.py b/src/pylorax/api/recipes.py index 4e9af08f..a14ccbbd 100644 --- a/src/pylorax/api/recipes.py +++ b/src/pylorax/api/recipes.py @@ -73,11 +73,21 @@ class Recipe(dict): """Return the names of the packages""" return [p["name"] for p in self["packages"] or []] + @property + def package_nver(self): + """Return the names and version globs of the packages""" + return [(p["name"], p["version"]) for p in self["packages"] or []] + @property def module_names(self): """Return the names of the modules""" return [m["name"] for m in self["modules"] or []] + @property + def module_nver(self): + """Return the names and version globs of the modules""" + return [(m["name"], m["version"]) for m in self["modules"] or []] + @property def filename(self): """Return the Recipe's filename diff --git a/src/pylorax/api/v0.py b/src/pylorax/api/v0.py index 5e27f160..bfc1fb37 100644 --- a/src/pylorax/api/v0.py +++ b/src/pylorax/api/v0.py @@ -1181,9 +1181,9 @@ def v0_api(api): # Combine modules and packages and depsolve the list # TODO include the version/glob in the depsolving - module_names = blueprint.module_names - package_names = blueprint.package_names - projects = sorted(set(module_names+package_names), key=lambda n: n.lower()) + module_nver = blueprint.module_nver + package_nver = blueprint.package_nver + projects = sorted(set(module_nver+package_nver), key=lambda p: p[0].lower()) deps = [] try: with api.config["DNFLOCK"].lock: @@ -1233,9 +1233,9 @@ def v0_api(api): # Combine modules and packages and depsolve the list # TODO include the version/glob in the depsolving - module_names = [m["name"] for m in blueprint["modules"] or []] - package_names = [p["name"] for p in blueprint["packages"] or []] - projects = sorted(set(module_names+package_names), key=lambda n: n.lower()) + module_nver = blueprint.module_nver + package_nver = blueprint.package_nver + projects = sorted(set(module_nver+package_nver), key=lambda p: p[0].lower()) deps = [] try: with api.config["DNFLOCK"].lock: @@ -1294,7 +1294,7 @@ def v0_api(api): """Return detailed information about the listed projects""" try: with api.config["DNFLOCK"].lock: - deps = projects_depsolve(api.config["DNFLOCK"].dbo, project_names.split(",")) + deps = projects_depsolve(api.config["DNFLOCK"].dbo, [(n, "*") for n in project_names.split(",")]) except ProjectsError as e: log.error("(v0_projects_depsolve) %s", str(e)) return jsonify(status=False, errors=[str(e)]), 400 diff --git a/tests/pylorax/test_projects.py b/tests/pylorax/test_projects.py index 28ee6a10..d35f79c7 100644 --- a/tests/pylorax/test_projects.py +++ b/tests/pylorax/test_projects.py @@ -151,13 +151,27 @@ class ProjectsTest(unittest.TestCase): self.assertEqual(projects[0]["builds"][0]["source"]["license"], "GPLv3+") def test_projects_depsolve(self): - deps = projects_depsolve(self.dbo, ["bash"]) + deps = projects_depsolve(self.dbo, [("bash", "*.*")]) self.assertEqual(deps[0]["name"], "basesystem") + def test_projects_depsolve_version(self): + """Test that depsolving with a partial wildcard version works""" + deps = projects_depsolve(self.dbo, [("bash", "4.*")]) + self.assertEqual(deps[1]["name"], "bash") + + deps = projects_depsolve(self.dbo, [("bash", "4.4.*")]) + self.assertEqual(deps[1]["name"], "bash") + + def test_projects_depsolve_oldversion(self): + """Test that depsolving a specific non-existant version fails""" + with self.assertRaises(ProjectsError): + deps = projects_depsolve(self.dbo, [("bash", "1.0.0")]) + self.assertEqual(deps[1]["name"], "bash") + def test_projects_depsolve_fail(self): with self.assertRaises(ProjectsError): - projects_depsolve(self.dbo, ["nada-package"]) + projects_depsolve(self.dbo, [("nada-package", "*.*")]) def test_modules_list_all(self): modules = modules_list(self.dbo, None) diff --git a/tests/pylorax/test_server.py b/tests/pylorax/test_server.py index 63fa030b..3f0508eb 100644 --- a/tests/pylorax/test_server.py +++ b/tests/pylorax/test_server.py @@ -173,7 +173,7 @@ class ServerTestCase(unittest.TestCase): "modules":[{"name":"glusterfs", "version":"4.0.*"}, {"name":"glusterfs-cli", "version":"4.0.*"}], "packages":[{"name":"samba", "version":"4.8.*"}, - {"name":"tmux", "version":"2.2"}]} + {"name":"tmux", "version":"2.7"}]} resp = self.server.post("/api/v0/blueprints/new", data=json.dumps(test_blueprint), @@ -217,7 +217,7 @@ class ServerTestCase(unittest.TestCase): "modules":[{"name":"glusterfs", "version":"4.0.*"}, {"name":"glusterfs-cli", "version":"4.0.*"}], "packages":[{"name":"samba", "version":"4.8.*"}, - {"name":"tmux", "version":"2.2"}]} + {"name":"tmux", "version":"2.7"}]} resp = self.server.post("/api/v0/blueprints/workspace", data=json.dumps(test_blueprint), @@ -243,7 +243,7 @@ class ServerTestCase(unittest.TestCase): "modules":[{"name":"glusterfs", "version":"4.0.*"}, {"name":"glusterfs-cli", "version":"4.0.*"}], "packages":[{"name":"samba", "version":"4.8.*"}, - {"name":"tmux", "version":"2.2"}]} + {"name":"tmux", "version":"2.7"}]} resp = self.server.post("/api/v0/blueprints/workspace", data=json.dumps(test_blueprint), @@ -374,7 +374,7 @@ class ServerTestCase(unittest.TestCase): "modules":[{"name":"glusterfs", "version":"4.0.*"}, {"name":"glusterfs-cli", "version":"4.0.*"}], "packages":[{"name":"samba", "version":"4.8.*"}, - {"name":"tmux", "version":"2.2"}]} + {"name":"tmux", "version":"2.7"}]} resp = self.server.post("/api/v0/blueprints/workspace", data=json.dumps(test_blueprint), @@ -390,7 +390,7 @@ class ServerTestCase(unittest.TestCase): "old": {"Description": "An example GlusterFS server with samba"}}, {"new": {"Version": "0.3.0"}, "old": {"Version": "0.0.1"}}, - {"new": {"Package": {"version": "2.2", "name": "tmux"}}, + {"new": {"Package": {"version": "2.7", "name": "tmux"}}, "old": None}]} self.assertEqual(data, result) @@ -437,6 +437,7 @@ class ServerTestCase(unittest.TestCase): blueprints = data.get("blueprints") self.assertNotEqual(blueprints, None) self.assertEqual(len(blueprints), 1) + self.assertTrue(len(blueprints[0]["blueprint"]["modules"]) > 0) self.assertEqual(blueprints[0]["blueprint"]["name"], "glusterfs") evra = blueprints[0]["blueprint"]["modules"][0]["version"] self.assertEqual(len(evra) > 10, True) @@ -502,7 +503,7 @@ class ServerTestCase(unittest.TestCase): "modules":[{"name":"glusterfs", "version":"4.0.*"}, {"name":"glusterfs-cli", "version":"4.0.*"}], "packages":[{"name":"samba", "version":"4.8.*"}, - {"name":"tmux", "version":"2.2"}]} + {"name":"tmux", "version":"2.7"}]} resp = self.server.post("/api/v0/blueprints/new?branch=test", data=json.dumps(test_blueprint),