diff --git a/docs/lorax-composer.rst b/docs/lorax-composer.rst index b107073d..c219213d 100644 --- a/docs/lorax-composer.rst +++ b/docs/lorax-composer.rst @@ -137,6 +137,18 @@ NOTE: As of lorax-composer-29.2-1 the versions are not used for depsolving, that is planned for a future release. And currently there are no differences between ``packages`` and ``modules`` in ``lorax-composer``. +[[groups]] +~~~~~~~~~~ + +These entries describe a group of packages to be installed into the image. Package groups are +defined in the repository metadata. Each group has a descriptive name used primarily for display +in user interfaces and an ID more commonly used in kickstart files. Here, the ID is the expected +way of listing a group. + +Groups have three different ways of categorizing their packages: mandatory, default, and optional. +For purposes of blueprints, mandatory and default packages will be installed. There is no mechanism +for selecting optional packages. + Customizations ~~~~~~~~~~~~~~ diff --git a/src/pylorax/api/compose.py b/src/pylorax/api/compose.py index 08873bf0..a740c578 100644 --- a/src/pylorax/api/compose.py +++ b/src/pylorax/api/compose.py @@ -75,7 +75,7 @@ def test_templates(dbo, share_dir): ks.readKickstartFromString(ks_template+"\n%end\n") pkgs = [(name, "*") for name in ks.handler.packages.packageList] try: - _ = projects_depsolve(dbo, pkgs) + _ = projects_depsolve(dbo, pkgs, []) except ProjectsError as e: template_errors.append("Error depsolving %s: %s" % (compose_type, str(e))) @@ -258,7 +258,7 @@ def start_build(cfg, dnflock, gitlock, branch, recipe_name, compose_type, test_m deps = [] try: with dnflock.lock: - (installed_size, deps) = projects_depsolve_with_size(dnflock.dbo, projects, with_core=False) + (installed_size, deps) = projects_depsolve_with_size(dnflock.dbo, projects, recipe.group_names, with_core=False) except ProjectsError as e: log.error("start_build depsolve: %s", str(e)) raise RuntimeError("Problem depsolving %s: %s" % (recipe["name"], str(e))) @@ -274,7 +274,7 @@ def start_build(cfg, dnflock, gitlock, branch, recipe_name, compose_type, test_m ks_projects = [(name, "*") for name in ks.handler.packages.packageList] try: with dnflock.lock: - (template_size, _) = projects_depsolve_with_size(dnflock.dbo, ks_projects, + (template_size, _) = projects_depsolve_with_size(dnflock.dbo, ks_projects, [], with_core=not ks.handler.packages.nocore) except ProjectsError as e: log.error("start_build depsolve: %s", str(e)) diff --git a/src/pylorax/api/projects.py b/src/pylorax/api/projects.py index d84fc642..62233ffa 100644 --- a/src/pylorax/api/projects.py +++ b/src/pylorax/api/projects.py @@ -182,13 +182,15 @@ def projects_info(dbo, project_names): pkgs = dbo.sack.query().available() return sorted(map(pkg_to_project_info, pkgs), key=lambda p: p["name"].lower()) -def _depsolve(dbo, projects): +def _depsolve(dbo, projects, groups): """Add projects to a new transaction :param dbo: dnf base object :type dbo: dnf.Base :param projects: The projects and version globs to find the dependencies for :type projects: List of tuples + :param groups: The groups to include in dependency solving + :type groups: List of str :returns: None :rtype: None :raises: ProjectsError if there was a problem installing something @@ -196,6 +198,12 @@ def _depsolve(dbo, projects): # This resets the transaction dbo.reset(goal=True) install_errors = [] + for name in groups: + try: + dbo.group_install(name, ["mandatory", "default"]) + except dnf.exceptions.MarkingError as e: + install_errors.append(("Group %s" % (name), str(e))) + for name, version in projects: try: if not version: @@ -214,18 +222,20 @@ def _depsolve(dbo, projects): raise ProjectsError("The following package(s) had problems: %s" % ",".join(["%s (%s)" % (pattern, err) for pattern, err in install_errors])) -def projects_depsolve(dbo, projects): +def projects_depsolve(dbo, projects, groups): """Return the dependencies for a list of projects :param dbo: dnf base object :type dbo: dnf.Base :param projects: The projects to find the dependencies for :type projects: List of Strings + :param groups: The groups to include in dependency solving + :type groups: List of str :returns: NEVRA's of the project and its dependencies :rtype: list of dicts :raises: ProjectsError if there was a problem installing something """ - _depsolve(dbo, projects) + _depsolve(dbo, projects, groups) try: dbo.resolve() @@ -259,18 +269,20 @@ def estimate_size(packages, block_size=6144): return installed_size -def projects_depsolve_with_size(dbo, projects, with_core=True): +def projects_depsolve_with_size(dbo, projects, groups, with_core=True): """Return the dependencies and installed size for a list of projects :param dbo: dnf base object :type dbo: dnf.Base :param project_names: The projects to find the dependencies for :type project_names: List of Strings + :param groups: The groups to include in dependency solving + :type groups: List of str :returns: installed size and a list of NEVRA's of the project and its dependencies :rtype: tuple of (int, list of dicts) :raises: ProjectsError if there was a problem installing something """ - _depsolve(dbo, projects) + _depsolve(dbo, projects, groups) if with_core: dbo.group_install("core", ['mandatory', 'default', 'optional']) @@ -323,7 +335,7 @@ def modules_info(dbo, module_names): # Add the dependency info to each one for module in modules: - module["dependencies"] = projects_depsolve(dbo, [(module["name"], "*.*")]) + module["dependencies"] = projects_depsolve(dbo, [(module["name"], "*.*")], []) return modules diff --git a/src/pylorax/api/recipes.py b/src/pylorax/api/recipes.py index a14ccbbd..617044f5 100644 --- a/src/pylorax/api/recipes.py +++ b/src/pylorax/api/recipes.py @@ -47,21 +47,24 @@ class Recipe(dict): and adds a .filename property to return the recipe's filename, and a .toml() function to return the recipe as a TOML string. """ - def __init__(self, name, description, version, modules, packages, customizations=None): + def __init__(self, name, description, version, modules, packages, groups, customizations=None): # Check that version is empty or semver compatible if version: semver.Version(version) - # Make sure modules and packages are listed by their case-insensitive names + # Make sure modules, packages, and groups are listed by their case-insensitive names if modules is not None: modules = sorted(modules, key=lambda m: m["name"].lower()) if packages is not None: packages = sorted(packages, key=lambda p: p["name"].lower()) + if groups is not None: + groups = sorted(groups, key=lambda g: g["name"].lower()) dict.__init__(self, name=name, description=description, version=version, modules=modules, packages=packages, + groups=groups, customizations=customizations) # We don't want customizations=None to show up in the TOML so remove it @@ -88,6 +91,11 @@ class Recipe(dict): """Return the names and version globs of the modules""" return [(m["name"], m["version"]) for m in self["modules"] or []] + @property + def group_names(self): + """Return the names of the groups. Groups do not have versions.""" + return map(lambda g: g["name"], self["groups"] or []) + @property def filename(self): """Return the Recipe's filename @@ -144,21 +152,25 @@ class Recipe(dict): """ module_names = self.module_names package_names = self.package_names + group_names = self.group_names new_modules = [] new_packages = [] + new_groups = [] for dep in deps: if dep["name"] in package_names: new_packages.append(RecipePackage(dep["name"], dep_evra(dep))) elif dep["name"] in module_names: new_modules.append(RecipeModule(dep["name"], dep_evra(dep))) + elif dep["name"] in group_names: + new_groups.append(RecipeGroup(dep["name"])) if "customizations" in self: customizations = self["customizations"] else: customizations = None return Recipe(self["name"], self["description"], self["version"], - new_modules, new_packages, customizations) + new_modules, new_packages, new_groups, customizations) class RecipeModule(dict): def __init__(self, name, version): @@ -167,6 +179,10 @@ class RecipeModule(dict): class RecipePackage(RecipeModule): pass +class RecipeGroup(dict): + def __init__(self, name): + dict.__init__(self, name=name) + def recipe_from_file(recipe_path): """Return a recipe file as a Recipe object @@ -210,6 +226,10 @@ def recipe_from_dict(recipe_dict): packages = [RecipePackage(p.get("name"), p.get("version")) for p in recipe_dict["packages"]] else: packages = [] + if recipe_dict.get("groups"): + groups = [RecipeGroup(g.get("name")) for g in recipe_dict["groups"]] + else: + groups = [] name = recipe_dict["name"] description = recipe_dict["description"] version = recipe_dict.get("version", None) @@ -217,7 +237,7 @@ def recipe_from_dict(recipe_dict): except KeyError as e: raise RecipeError("There was a problem parsing the recipe: %s" % str(e)) - return Recipe(name, description, version, modules, packages, customizations) + return Recipe(name, description, version, modules, packages, groups, customizations) def gfile(path): """Convert a string path to GFile for use with Git""" @@ -897,5 +917,6 @@ def recipe_diff(old_recipe, new_recipe): diffs.extend(diff_items("Module", old_recipe["modules"], new_recipe["modules"])) diffs.extend(diff_items("Package", old_recipe["packages"], new_recipe["packages"])) + diffs.extend(diff_items("Group", old_recipe["groups"], new_recipe["groups"])) return diffs diff --git a/src/pylorax/api/v0.py b/src/pylorax/api/v0.py index a0adf4ad..e268809b 100644 --- a/src/pylorax/api/v0.py +++ b/src/pylorax/api/v0.py @@ -1278,7 +1278,7 @@ def v0_api(api): deps = [] try: with api.config["DNFLOCK"].lock: - deps = projects_depsolve(api.config["DNFLOCK"].dbo, projects) + deps = projects_depsolve(api.config["DNFLOCK"].dbo, projects, blueprint.group_names) except ProjectsError as e: errors.append("%s: %s" % (blueprint_name, str(e))) log.error("(v0_blueprints_freeze) %s", str(e)) @@ -1330,7 +1330,7 @@ def v0_api(api): deps = [] try: with api.config["DNFLOCK"].lock: - deps = projects_depsolve(api.config["DNFLOCK"].dbo, projects) + deps = projects_depsolve(api.config["DNFLOCK"].dbo, projects, blueprint.group_names) except ProjectsError as e: errors.append("%s: %s" % (blueprint_name, str(e))) log.error("(v0_blueprints_depsolve) %s", str(e)) @@ -1385,7 +1385,7 @@ def v0_api(api): """Return detailed information about the listed projects""" try: with api.config["DNFLOCK"].lock: - deps = projects_depsolve(api.config["DNFLOCK"].dbo, [(n, "*") for n in project_names.split(",")]) + deps = projects_depsolve(api.config["DNFLOCK"].dbo, [(n, "*") for n in project_names.split(",")], []) except ProjectsError as e: log.error("(v0_projects_depsolve) %s", str(e)) return jsonify(status=False, errors=[str(e)]), 400 diff --git a/tests/pylorax/results/custom-base.dict b/tests/pylorax/results/custom-base.dict index 0a39c626..2604cf34 100644 --- a/tests/pylorax/results/custom-base.dict +++ b/tests/pylorax/results/custom-base.dict @@ -1 +1 @@ -{'name': 'custom-base', 'description': 'A base system with customizations', 'version': '0.0.1', 'modules': [], 'packages': [{'name': 'bash', 'version': '4.4.*'}], 'customizations': {'hostname': 'custombase', 'sshkey': [{'user': 'root', 'key': 'A SSH KEY FOR ROOT'}]}} +{'name': 'custom-base', 'description': 'A base system with customizations', 'version': '0.0.1', 'groups': [], 'modules': [], 'packages': [{'name': 'bash', 'version': '4.4.*'}], 'customizations': {'hostname': 'custombase', 'sshkey': [{'user': 'root', 'key': 'A SSH KEY FOR ROOT'}]}} diff --git a/tests/pylorax/results/full-recipe.dict b/tests/pylorax/results/full-recipe.dict index 082d6de4..23a0ee4f 100644 --- a/tests/pylorax/results/full-recipe.dict +++ b/tests/pylorax/results/full-recipe.dict @@ -1 +1 @@ -{'description': u'An example http server with PHP and MySQL support.', 'packages': [{'version': u'6.6.*', 'name': u'openssh-server'}, {'version': u'3.0.*', 'name': u'rsync'}, {'version': u'2.2', 'name': u'tmux'}], 'modules': [{'version': u'2.4.*', 'name': u'httpd'}, {'version': u'5.4', 'name': u'mod_auth_kerb'}, {'version': u'2.4.*', 'name': u'mod_ssl'}, {'version': u'5.4.*', 'name': u'php'}, {'version': u'5.4.*', 'name': u'php-mysql'}], 'version': u'0.0.1', 'name': u'http-server'} +{'description': u'An example http server with PHP and MySQL support.', 'packages': [{'version': u'6.6.*', 'name': u'openssh-server'}, {'version': u'3.0.*', 'name': u'rsync'}, {'version': u'2.2', 'name': u'tmux'}], 'groups': [], 'modules': [{'version': u'2.4.*', 'name': u'httpd'}, {'version': u'5.4', 'name': u'mod_auth_kerb'}, {'version': u'2.4.*', 'name': u'mod_ssl'}, {'version': u'5.4.*', 'name': u'php'}, {'version': u'5.4.*', 'name': u'php-mysql'}], 'version': u'0.0.1', 'name': u'http-server'} diff --git a/tests/pylorax/results/minimal.dict b/tests/pylorax/results/minimal.dict index 4af1c108..81178d27 100644 --- a/tests/pylorax/results/minimal.dict +++ b/tests/pylorax/results/minimal.dict @@ -1 +1 @@ -{'description': u'An example http server with PHP and MySQL support.', 'packages': [], 'modules': [], 'version': u'0.0.1', 'name': u'http-server'} +{'description': u'An example http server with PHP and MySQL support.', 'packages': [], 'groups': [], 'modules': [], 'version': u'0.0.1', 'name': u'http-server'} diff --git a/tests/pylorax/results/modules-only.dict b/tests/pylorax/results/modules-only.dict index 64b74b31..81185b8c 100644 --- a/tests/pylorax/results/modules-only.dict +++ b/tests/pylorax/results/modules-only.dict @@ -1 +1 @@ -{'description': u'An example http server with PHP and MySQL support.', 'packages': [], 'modules': [{'version': u'2.4.*', 'name': u'httpd'}, {'version': u'5.4', 'name': u'mod_auth_kerb'}, {'version': u'2.4.*', 'name': u'mod_ssl'}, {'version': u'5.4.*', 'name': u'php'}, {'version': u'5.4.*', 'name': u'php-mysql'}], 'version': u'0.0.1', 'name': u'http-server'} +{'description': u'An example http server with PHP and MySQL support.', 'packages': [], 'groups': [], 'modules': [{'version': u'2.4.*', 'name': u'httpd'}, {'version': u'5.4', 'name': u'mod_auth_kerb'}, {'version': u'2.4.*', 'name': u'mod_ssl'}, {'version': u'5.4.*', 'name': u'php'}, {'version': u'5.4.*', 'name': u'php-mysql'}], 'version': u'0.0.1', 'name': u'http-server'} diff --git a/tests/pylorax/results/packages-only.dict b/tests/pylorax/results/packages-only.dict index 8ad5412c..9c542a53 100644 --- a/tests/pylorax/results/packages-only.dict +++ b/tests/pylorax/results/packages-only.dict @@ -1 +1 @@ -{'description': u'An example http server with PHP and MySQL support.', 'packages': [{'version': u'6.6.*', 'name': u'openssh-server'}, {'version': u'3.0.*', 'name': u'rsync'}, {'version': u'2.2', 'name': u'tmux'}], 'modules': [], 'version': u'0.0.1', 'name': u'http-server'} +{'description': u'An example http server with PHP and MySQL support.', 'packages': [{'version': u'6.6.*', 'name': u'openssh-server'}, {'version': u'3.0.*', 'name': u'rsync'}, {'version': u'2.2', 'name': u'tmux'}], 'groups': [], 'modules': [], 'version': u'0.0.1', 'name': u'http-server'} diff --git a/tests/pylorax/test_projects.py b/tests/pylorax/test_projects.py index 4ea57ad7..d22f3b44 100644 --- a/tests/pylorax/test_projects.py +++ b/tests/pylorax/test_projects.py @@ -156,27 +156,27 @@ class ProjectsTest(unittest.TestCase): self.assertEqual(projects[0]["builds"][0]["source"]["license"], "GPLv3+") def test_projects_depsolve(self): - deps = projects_depsolve(self.dbo, [("bash", "*.*")]) - + deps = projects_depsolve(self.dbo, [("bash", "*.*")], []) + self.assertTrue(len(deps) > 3) self.assertEqual(deps[0]["name"], "basesystem") def test_projects_depsolve_version(self): """Test that depsolving with a partial wildcard version works""" - deps = projects_depsolve(self.dbo, [("bash", "4.*")]) + deps = projects_depsolve(self.dbo, [("bash", "4.*")], []) self.assertEqual(deps[1]["name"], "bash") - deps = projects_depsolve(self.dbo, [("bash", "4.4.*")]) + deps = projects_depsolve(self.dbo, [("bash", "4.4.*")], []) self.assertEqual(deps[1]["name"], "bash") def test_projects_depsolve_oldversion(self): """Test that depsolving a specific non-existant version fails""" with self.assertRaises(ProjectsError): - deps = projects_depsolve(self.dbo, [("bash", "1.0.0")]) + deps = projects_depsolve(self.dbo, [("bash", "1.0.0")], []) self.assertEqual(deps[1]["name"], "bash") def test_projects_depsolve_fail(self): with self.assertRaises(ProjectsError): - projects_depsolve(self.dbo, [("nada-package", "*.*")]) + projects_depsolve(self.dbo, [("nada-package", "*.*")], []) def test_modules_list_all(self): modules = modules_list(self.dbo, None) diff --git a/tests/pylorax/test_recipes.py b/tests/pylorax/test_recipes.py index 1b77c800..3d5622f8 100644 --- a/tests/pylorax/test_recipes.py +++ b/tests/pylorax/test_recipes.py @@ -95,27 +95,27 @@ class BasicRecipeTest(unittest.TestCase): """Test the Recipe's version bump function""" # Neither have a version - recipe = recipes.Recipe("test-recipe", "A recipe used for testing", None, None, None) + recipe = recipes.Recipe("test-recipe", "A recipe used for testing", None, None, None, None) new_version = recipe.bump_version(None) self.assertEqual(new_version, "0.0.1") # Original has a version, new does not - recipe = recipes.Recipe("test-recipe", "A recipe used for testing", None, None, None) + recipe = recipes.Recipe("test-recipe", "A recipe used for testing", None, None, None, None) new_version = recipe.bump_version("0.0.1") self.assertEqual(new_version, "0.0.2") # Original has no version, new does - recipe = recipes.Recipe("test-recipe", "A recipe used for testing", "0.1.0", None, None) + recipe = recipes.Recipe("test-recipe", "A recipe used for testing", "0.1.0", None, None, None) new_version = recipe.bump_version(None) self.assertEqual(new_version, "0.1.0") # New and Original are the same - recipe = recipes.Recipe("test-recipe", "A recipe used for testing", "0.0.1", None, None) + recipe = recipes.Recipe("test-recipe", "A recipe used for testing", "0.0.1", None, None, None) new_version = recipe.bump_version("0.0.1") self.assertEqual(new_version, "0.0.2") # New is different from Original - recipe = recipes.Recipe("test-recipe", "A recipe used for testing", "0.1.1", None, None) + recipe = recipes.Recipe("test-recipe", "A recipe used for testing", "0.1.1", None, None, None) new_version = recipe.bump_version("0.0.1") self.assertEqual(new_version, "0.1.1") @@ -136,8 +136,8 @@ class BasicRecipeTest(unittest.TestCase): def recipe_diff_test(self): """Test the recipe_diff function""" - old_recipe = recipes.Recipe("test-recipe", "A recipe used for testing", "0.1.1", self.old_modules, self.old_packages) - new_recipe = recipes.Recipe("test-recipe", "A recipe used for testing", "0.3.1", self.new_modules, self.new_packages) + old_recipe = recipes.Recipe("test-recipe", "A recipe used for testing", "0.1.1", self.old_modules, self.old_packages, []) + new_recipe = recipes.Recipe("test-recipe", "A recipe used for testing", "0.3.1", self.new_modules, self.new_packages, []) result = [{'new': {'Version': '0.3.1'}, 'old': {'Version': '0.1.1'}}, {'new': {'Module': {'name': 'openssh', 'version': '2.8.1'}}, 'old': None}, {'new': None, 'old': {'Module': {'name': 'bash', 'version': '4.*'}}}, @@ -182,7 +182,7 @@ version = "2.7.*" def test_02_commit_recipe(self): """Test committing a Recipe object""" - recipe = recipes.Recipe("test-recipe", "A recipe used for testing", None, None, None) + recipe = recipes.Recipe("test-recipe", "A recipe used for testing", None, None, None, None) oid = recipes.commit_recipe(self.repo, "master", recipe) self.assertNotEqual(oid, None)