diff --git a/src/pylorax/api/compose.py b/src/pylorax/api/compose.py index 904e00dd..692d5f18 100644 --- a/src/pylorax/api/compose.py +++ b/src/pylorax/api/compose.py @@ -230,10 +230,9 @@ def start_build(cfg, yumlock, gitlock, branch, recipe_name, compose_type, test_m (commit_id, recipe) = read_recipe_and_id(gitlock.repo, branch, recipe_name) # Combine modules and packages and depsolve the list - # TODO include the version/glob in the depsolving - module_names = map(lambda m: m["name"], recipe["modules"] or []) - package_names = map(lambda p: p["name"], recipe["packages"] or []) - projects = sorted(set(module_names+package_names), key=lambda n: n.lower()) + module_nver = recipe.module_nver + package_nver = recipe.package_nver + projects = sorted(set(module_nver+package_nver), key=lambda p: p[0].lower()) deps = [] try: with yumlock.lock: @@ -250,9 +249,10 @@ def start_build(cfg, yumlock, gitlock, branch, recipe_name, compose_type, test_m ks_version = makeVersion(RHEL7) ks = KickstartParser(ks_version, errorsAreFatal=False, missingIncludeIsFatal=False) ks.readKickstartFromString(ks_template+"\n%end\n") + pkgs = [(name, "*") for name in ks.handler.packages.packageList] try: with yumlock.lock: - (template_size, _) = projects_depsolve_with_size(yumlock.yb, ks.handler.packages.packageList, + (template_size, _) = projects_depsolve_with_size(yumlock.yb, pkgs, with_core=not ks.handler.packages.nocore) except ProjectsError as e: log.error("start_build depsolve: %s", str(e)) diff --git a/src/pylorax/api/projects.py b/src/pylorax/api/projects.py index 35957cc5..5e8a5a4d 100644 --- a/src/pylorax/api/projects.py +++ b/src/pylorax/api/projects.py @@ -186,28 +186,31 @@ def projects_info(yb, project_names): return sorted(map(yaps_to_project_info, ybl.available), key=lambda p: p["name"].lower()) -def projects_depsolve(yb, project_names): +def projects_depsolve(yb, projects): """Return the dependencies for a list of projects :param yb: yum base object :type yb: YumBase - :param project_names: The projects to find the dependencies for - :type project_names: List of Strings + :param projects: The projects and version globs to find the dependencies for + :type projects: List of tuples :returns: NEVRA's of the project and its dependencies :rtype: list of dicts + :raises: ProjectsError if there was a problem installing something """ try: # This resets the transaction yb.closeRpmDB() - for p in project_names: - yb.install(pattern=p) + for name, version in projects: + if not version: + version = "*" + yb.install(pattern="%s-%s" % (name, version)) (rc, msg) = yb.buildTransaction() if rc not in [0, 1, 2]: - raise ProjectsError("There was a problem depsolving %s: %s" % (project_names, msg)) + raise ProjectsError("There was a problem depsolving %s: %s" % (projects, msg)) yb.tsInfo.makelists() deps = sorted(map(tm_to_dep, yb.tsInfo.installed + yb.tsInfo.depinstalled), key=lambda p: p["name"].lower()) except YumBaseError as e: - raise ProjectsError("There was a problem depsolving %s: %s" % (project_names, str(e))) + raise ProjectsError("There was a problem depsolving %s: %s" % (projects, str(e))) finally: yb.closeRpmDB() return deps @@ -232,31 +235,34 @@ def estimate_size(packages, block_size=4096): installed_size += p.po.installedsize return installed_size -def projects_depsolve_with_size(yb, project_names, with_core=True): +def projects_depsolve_with_size(yb, projects, with_core=True): """Return the dependencies and installed size for a list of projects :param yb: yum base object :type yb: YumBase - :param project_names: The projects to find the dependencies for - :type project_names: List of Strings + :param projects: The projects and version globs to find the dependencies for + :type projects: List of tuples :returns: installed size and a list of NEVRA's of the project and its dependencies :rtype: tuple of (int, list of dicts) + :raises: ProjectsError if there was a problem installing something """ try: # This resets the transaction yb.closeRpmDB() - for p in project_names: - yb.install(pattern=p) + for name, version in projects: + if not version: + version = "*" + yb.install(pattern="%s-%s" % (name, version)) if with_core: yb.selectGroup("core", group_package_types=['mandatory', 'default', 'optional']) (rc, msg) = yb.buildTransaction() if rc not in [0, 1, 2]: - raise ProjectsError("There was a problem depsolving %s: %s" % (project_names, msg)) + raise ProjectsError("There was a problem depsolving %s: %s" % (projects, msg)) yb.tsInfo.makelists() installed_size = estimate_size(yb.tsInfo.installed + yb.tsInfo.depinstalled) deps = sorted(map(tm_to_dep, yb.tsInfo.installed + yb.tsInfo.depinstalled), key=lambda p: p["name"].lower()) except YumBaseError as e: - raise ProjectsError("There was a problem depsolving %s: %s" % (project_names, str(e))) + raise ProjectsError("There was a problem depsolving %s: %s" % (projects, str(e))) finally: yb.closeRpmDB() return (installed_size, deps) @@ -306,6 +312,6 @@ def modules_info(yb, module_names): modules = sorted(map(yaps_to_project, ybl.available), key=lambda p: p["name"].lower()) # Add the dependency info to each one for module in modules: - module["dependencies"] = projects_depsolve(yb, [module["name"]]) + module["dependencies"] = projects_depsolve(yb, [(module["name"], "*")]) return modules diff --git a/src/pylorax/api/recipes.py b/src/pylorax/api/recipes.py index b1a72b35..9d22d181 100644 --- a/src/pylorax/api/recipes.py +++ b/src/pylorax/api/recipes.py @@ -73,11 +73,21 @@ class Recipe(dict): """Return the names of the packages""" return map(lambda p: p["name"], self["packages"] or []) + @property + def package_nver(self): + """Return the names and versions of the packages""" + return [(p["name"], p["version"]) for p in self["packages"] or []] + @property def module_names(self): """Return the names of the modules""" return map(lambda m: m["name"], self["modules"] or []) + @property + def module_nver(self): + """Return the names and versions of the modules""" + return [(m["name"], m["version"]) for m in self["modules"] or []] + @property def filename(self): """Return the Recipe's filename diff --git a/src/pylorax/api/v0.py b/src/pylorax/api/v0.py index cd4e0450..0aceeec7 100644 --- a/src/pylorax/api/v0.py +++ b/src/pylorax/api/v0.py @@ -1181,9 +1181,9 @@ def v0_api(api): # Combine modules and packages and depsolve the list # TODO include the version/glob in the depsolving - module_names = blueprint.module_names - package_names = blueprint.package_names - projects = sorted(set(module_names+package_names), key=lambda n: n.lower()) + module_nver = blueprint.module_nver + package_nver = blueprint.package_nver + projects = sorted(set(module_nver+package_nver), key=lambda p: p[0].lower()) deps = [] try: with api.config["YUMLOCK"].lock: @@ -1232,10 +1232,9 @@ def v0_api(api): continue # Combine modules and packages and depsolve the list - # TODO include the version/glob in the depsolving - module_names = map(lambda m: m["name"], blueprint["modules"] or []) - package_names = map(lambda p: p["name"], blueprint["packages"] or []) - projects = sorted(set(module_names+package_names), key=lambda n: n.lower()) + module_nver = blueprint.module_nver + package_nver = blueprint.package_nver + projects = sorted(set(module_nver+package_nver), key=lambda p: p[0].lower()) deps = [] try: with api.config["YUMLOCK"].lock: @@ -1294,7 +1293,7 @@ def v0_api(api): """Return detailed information about the listed projects""" try: with api.config["YUMLOCK"].lock: - deps = projects_depsolve(api.config["YUMLOCK"].yb, project_names.split(",")) + deps = projects_depsolve(api.config["YUMLOCK"].yb, [(n, "*") for n in project_names.split(",")]) except ProjectsError as e: log.error("(v0_projects_depsolve) %s", str(e)) return jsonify(status=False, errors=[str(e)]), 400 diff --git a/tests/pylorax/blueprints/custom-base.toml b/tests/pylorax/blueprints/custom-base.toml index e8d5dc1d..bb5ed486 100644 --- a/tests/pylorax/blueprints/custom-base.toml +++ b/tests/pylorax/blueprints/custom-base.toml @@ -4,7 +4,7 @@ version = "0.0.1" [[packages]] name = "bash" -version = "4.4.*" +version = "4.2.*" [customizations] hostname = "custombase" diff --git a/tests/pylorax/blueprints/glusterfs.toml b/tests/pylorax/blueprints/glusterfs.toml index b64fd883..0634c1df 100644 --- a/tests/pylorax/blueprints/glusterfs.toml +++ b/tests/pylorax/blueprints/glusterfs.toml @@ -3,12 +3,12 @@ description = "An example GlusterFS server with samba" [[modules]] name = "glusterfs" -version = "3.7.*" +version = "3.8.*" [[modules]] name = "glusterfs-cli" -version = "3.7.*" +version = "3.8.*" [[packages]] name = "samba" -version = "4.2.*" +version = "4.7.*" diff --git a/tests/pylorax/blueprints/http-server.toml b/tests/pylorax/blueprints/http-server.toml index c6d74b1c..a20718eb 100644 --- a/tests/pylorax/blueprints/http-server.toml +++ b/tests/pylorax/blueprints/http-server.toml @@ -24,12 +24,12 @@ version = "5.4.*" [[packages]] name = "tmux" -version = "2.2" +version = "1.8" [[packages]] name = "openssh-server" -version = "6.6.*" +version = "7.*" [[packages]] name = "rsync" -version = "3.0.*" +version = "3.1.*" diff --git a/tests/pylorax/blueprints/kubernetes.toml b/tests/pylorax/blueprints/kubernetes.toml index 08e20bb7..036fe241 100644 --- a/tests/pylorax/blueprints/kubernetes.toml +++ b/tests/pylorax/blueprints/kubernetes.toml @@ -3,23 +3,23 @@ description = "An example kubernetes master" [[modules]] name = "kubernetes" -version = "1.2.*" +version = "1.5.2" [[modules]] name = "docker" -version = "1.10.*" +version = "1.13.*" [[modules]] name = "docker-lvm-plugin" -version = "1.10.*" +version = "1.13.*" [[modules]] name = "etcd" -version = "2.3.*" +version = "3.2.*" [[modules]] name = "flannel" -version = "0.5.*" +version = "0.7.*" [[packages]] name = "oci-systemd-hook" diff --git a/tests/pylorax/test_projects.py b/tests/pylorax/test_projects.py index 67cc207a..037469ea 100644 --- a/tests/pylorax/test_projects.py +++ b/tests/pylorax/test_projects.py @@ -178,13 +178,27 @@ class ProjectsTest(unittest.TestCase): projects_info(self.yb, ["bash"]) def test_projects_depsolve(self): - deps = projects_depsolve(self.yb, ["bash"]) + deps = projects_depsolve(self.yb, [("bash", "*.*")]) + self.assertTrue(len(deps) > 3) + self.assertEqual(deps[2]["name"], "basesystem") - self.assertEqual(deps[0]["name"], "basesystem") + def test_projects_depsolve_version(self): + """Test that depsolving with a partial wildcard version works""" + deps = projects_depsolve(self.yb, [("bash", "4.*")]) + self.assertEqual(deps[1]["name"], "bash") + + deps = projects_depsolve(self.yb, [("bash", "4.2.*")]) + self.assertEqual(deps[1]["name"], "bash") + + def test_projects_depsolve_oldversion(self): + """Test that depsolving a specific non-existant version fails""" + with self.assertRaises(ProjectsError): + deps = projects_depsolve(self.yb, [("bash", "1.0.0")]) + self.assertEqual(deps[1]["name"], "bash") def test_projects_depsolve_fail(self): with self.assertRaises(ProjectsError): - projects_depsolve(self.yb, ["nada-package"]) + projects_depsolve(self.yb, [("nada-package", "*.*")]) def test_modules_list(self): modules = modules_list(self.yb, None) @@ -204,8 +218,10 @@ class ProjectsTest(unittest.TestCase): modules = modules_info(self.yb, ["bash"]) print(modules) + self.assertTrue(len(modules) > 0) + self.assertTrue(len(modules[0]["dependencies"]) > 3) self.assertEqual(modules[0]["name"], "bash") - self.assertEqual(modules[0]["dependencies"][0]["name"], "basesystem") + self.assertEqual(modules[0]["dependencies"][2]["name"], "basesystem") def test_modules_info_yum_raises_exception(self): with self.assertRaises(ProjectsError): diff --git a/tests/pylorax/test_server.py b/tests/pylorax/test_server.py index f23275f0..06b480b7 100644 --- a/tests/pylorax/test_server.py +++ b/tests/pylorax/test_server.py @@ -97,9 +97,9 @@ class ServerTestCase(unittest.TestCase): {"name":"php", "version":"5.4.*"}, {"name": "php-mysql", "version":"5.4.*"}], "name":"http-server", - "packages": [{"name":"openssh-server", "version": "6.6.*"}, - {"name": "rsync", "version": "3.0.*"}, - {"name": "tmux", "version": "2.2"}], + "packages": [{"name":"openssh-server", "version": "7.*"}, + {"name": "rsync", "version": "3.1.*"}, + {"name": "tmux", "version": "1.8"}], "version": "0.0.1"}]} resp = self.server.get("/api/v0/blueprints/info/http-server") data = json.loads(resp.data) @@ -109,10 +109,10 @@ class ServerTestCase(unittest.TestCase): {"changed":False, "name":"http-server"}], "errors":[], "blueprints":[{"description": "An example GlusterFS server with samba", - "modules":[{"name":"glusterfs", "version":"3.7.*"}, - {"name":"glusterfs-cli", "version":"3.7.*"}], + "modules":[{"name":"glusterfs", "version":"3.8.*"}, + {"name":"glusterfs-cli", "version":"3.8.*"}], "name":"glusterfs", - "packages":[{"name":"samba", "version":"4.2.*"}], + "packages":[{"name":"samba", "version":"4.7.*"}], "version": "0.0.1"}, {"description":"An example http server with PHP and MySQL support.", "modules":[{"name":"httpd", "version":"2.4.*"}, @@ -121,9 +121,9 @@ class ServerTestCase(unittest.TestCase): {"name":"php", "version":"5.4.*"}, {"name": "php-mysql", "version":"5.4.*"}], "name":"http-server", - "packages": [{"name":"openssh-server", "version": "6.6.*"}, - {"name": "rsync", "version": "3.0.*"}, - {"name": "tmux", "version": "2.2"}], + "packages": [{"name":"openssh-server", "version": "7.*"}, + {"name": "rsync", "version": "3.1.*"}, + {"name": "tmux", "version": "1.8"}], "version": "0.0.1"}, ]} resp = self.server.get("/api/v0/blueprints/info/http-server,glusterfs") @@ -164,10 +164,10 @@ class ServerTestCase(unittest.TestCase): test_blueprint = {"description": "An example GlusterFS server with samba", "name":"glusterfs", "version": "0.2.0", - "modules":[{"name":"glusterfs", "version":"3.7.*"}, - {"name":"glusterfs-cli", "version":"3.7.*"}], - "packages":[{"name":"samba", "version":"4.2.*"}, - {"name":"tmux", "version":"2.2"}]} + "modules":[{"name":"glusterfs", "version":"3.8.*"}, + {"name":"glusterfs-cli", "version":"3.8.*"}], + "packages":[{"name":"samba", "version":"4.7.*"}, + {"name":"tmux", "version":"1.8"}]} resp = self.server.post("/api/v0/blueprints/new", data=json.dumps(test_blueprint), @@ -208,10 +208,10 @@ class ServerTestCase(unittest.TestCase): test_blueprint = {"description": "An example GlusterFS server with samba, ws version", "name":"glusterfs", "version": "0.3.0", - "modules":[{"name":"glusterfs", "version":"3.7.*"}, - {"name":"glusterfs-cli", "version":"3.7.*"}], - "packages":[{"name":"samba", "version":"4.2.*"}, - {"name":"tmux", "version":"2.2"}]} + "modules":[{"name":"glusterfs", "version":"3.8.*"}, + {"name":"glusterfs-cli", "version":"3.8.*"}], + "packages":[{"name":"samba", "version":"4.7.*"}, + {"name":"tmux", "version":"1.8"}]} resp = self.server.post("/api/v0/blueprints/workspace", data=json.dumps(test_blueprint), @@ -234,10 +234,10 @@ class ServerTestCase(unittest.TestCase): test_blueprint = {"description": "An example GlusterFS server with samba, ws version", "name":"glusterfs", "version": "0.4.0", - "modules":[{"name":"glusterfs", "version":"3.7.*"}, - {"name":"glusterfs-cli", "version":"3.7.*"}], - "packages":[{"name":"samba", "version":"4.2.*"}, - {"name":"tmux", "version":"2.2"}]} + "modules":[{"name":"glusterfs", "version":"3.8.*"}, + {"name":"glusterfs-cli", "version":"3.8.*"}], + "packages":[{"name":"samba", "version":"4.7.*"}, + {"name":"tmux", "version":"1.8"}]} resp = self.server.post("/api/v0/blueprints/workspace", data=json.dumps(test_blueprint), @@ -361,10 +361,10 @@ class ServerTestCase(unittest.TestCase): test_blueprint = {"description": "An example GlusterFS server with samba, ws version", "name":"glusterfs", "version": "0.3.0", - "modules":[{"name":"glusterfs", "version":"3.7.*"}, - {"name":"glusterfs-cli", "version":"3.7.*"}], - "packages":[{"name":"samba", "version":"4.2.*"}, - {"name":"tmux", "version":"2.2"}]} + "modules":[{"name":"glusterfs", "version":"3.8.*"}, + {"name":"glusterfs-cli", "version":"3.8.*"}], + "packages":[{"name":"samba", "version":"4.7.*"}, + {"name":"tmux", "version":"1.8"}]} resp = self.server.post("/api/v0/blueprints/workspace", data=json.dumps(test_blueprint), @@ -380,7 +380,7 @@ class ServerTestCase(unittest.TestCase): "old": {"Description": "An example GlusterFS server with samba"}}, {"new": {"Version": "0.3.0"}, "old": {"Version": "0.0.1"}}, - {"new": {"Package": {"version": "2.2", "name": "tmux"}}, + {"new": {"Package": {"version": "1.8", "name": "tmux"}}, "old": None}]} self.assertEqual(data, result) @@ -392,6 +392,7 @@ class ServerTestCase(unittest.TestCase): blueprints = data.get("blueprints") self.assertNotEqual(blueprints, None) self.assertEqual(len(blueprints), 1) + self.assertTrue(len(blueprints[0]["blueprint"]["modules"]) > 0) self.assertEqual(blueprints[0]["blueprint"]["name"], "glusterfs") self.assertEqual(len(blueprints[0]["dependencies"]) > 10, True) self.assertFalse(data.get("errors")) @@ -427,6 +428,7 @@ class ServerTestCase(unittest.TestCase): blueprints = data.get("blueprints") self.assertNotEqual(blueprints, None) self.assertEqual(len(blueprints), 1) + self.assertTrue(len(blueprints[0]["blueprint"]["modules"]) > 0) self.assertEqual(blueprints[0]["blueprint"]["name"], "glusterfs") evra = blueprints[0]["blueprint"]["modules"][0]["version"] self.assertEqual(len(evra) > 10, True) @@ -446,6 +448,7 @@ class ServerTestCase(unittest.TestCase): self.assertNotEqual(data, None) projects = data.get("projects") self.assertEqual(len(projects), 1) + self.assertTrue(len(projects[0]["builds"]) > 0) self.assertEqual(projects[0]["name"], "bash") self.assertEqual(projects[0]["builds"][0]["source"]["license"], "GPLv3+") @@ -456,7 +459,7 @@ class ServerTestCase(unittest.TestCase): self.assertNotEqual(data, None) deps = data.get("projects") self.assertEqual(len(deps) > 10, True) - self.assertEqual(deps[0]["name"], "basesystem") + self.assertEqual(deps[2]["name"], "basesystem") def test_modules_list(self): """Test /api/v0/modules/list""" @@ -482,17 +485,17 @@ class ServerTestCase(unittest.TestCase): modules = data.get("modules") self.assertEqual(len(modules), 1) self.assertEqual(modules[0]["name"], "bash") - self.assertEqual(modules[0]["dependencies"][0]["name"], "basesystem") + self.assertEqual(modules[0]["dependencies"][2]["name"], "basesystem") def test_blueprint_new_branch(self): """Test the /api/v0/blueprints/new route with a new branch""" test_blueprint = {"description": "An example GlusterFS server with samba", "name":"glusterfs", "version": "0.2.0", - "modules":[{"name":"glusterfs", "version":"3.7.*"}, - {"name":"glusterfs-cli", "version":"3.7.*"}], - "packages":[{"name":"samba", "version":"4.2.*"}, - {"name":"tmux", "version":"2.2"}]} + "modules":[{"name":"glusterfs", "version":"3.8.*"}, + {"name":"glusterfs-cli", "version":"3.8.*"}], + "packages":[{"name":"samba", "version":"4.7.*"}, + {"name":"tmux", "version":"1.8"}]} resp = self.server.post("/api/v0/blueprints/new?branch=test", data=json.dumps(test_blueprint),