Include groups in depsolving.
This adds a new argument to projects_depsolve and
projects_depsolve_with_size that contains the group list, unfortunately.
I would have prefered adding a function that just returns a list of all
the contents of a group and then add that to what was being passed into
projects_depsolve. However, there does not appear to be any good way to
do that in yum aside from a lot of grubbing around in the comps object,
which I am unwilling to do.
(cherry picked from commit 0259f3564d)
This commit is contained in:
Chris Lumens
Brian C. Lane
parent
76c811b7b8
commit
22ddaa667e
@ -137,6 +137,18 @@ NOTE: As of lorax-composer-29.2-1 the versions are not used for depsolving,
|
||||
that is planned for a future release. And currently there are no differences
|
||||
between ``packages`` and ``modules`` in ``lorax-composer``.
|
||||
|
||||
[[groups]]
|
||||
~~~~~~~~~~
|
||||
|
||||
These entries describe a group of packages to be installed into the image. Package groups are
|
||||
defined in the repository metadata. Each group has a descriptive name used primarily for display
|
||||
in user interfaces and an ID more commonly used in kickstart files. Here, the ID is the expected
|
||||
way of listing a group.
|
||||
|
||||
Groups have three different ways of categorizing their packages: mandatory, default, and optional.
|
||||
For purposes of blueprints, mandatory and default packages will be installed. There is no mechanism
|
||||
for selecting optional packages.
|
||||
|
||||
Customizations
|
||||
~~~~~~~~~~~~~~
|
||||
|
||||
|
||||
@ -75,7 +75,7 @@ def test_templates(dbo, share_dir):
|
||||
ks.readKickstartFromString(ks_template+"\n%end\n")
|
||||
pkgs = [(name, "*") for name in ks.handler.packages.packageList]
|
||||
try:
|
||||
_ = projects_depsolve(dbo, pkgs)
|
||||
_ = projects_depsolve(dbo, pkgs, [])
|
||||
except ProjectsError as e:
|
||||
template_errors.append("Error depsolving %s: %s" % (compose_type, str(e)))
|
||||
|
||||
@ -258,7 +258,7 @@ def start_build(cfg, dnflock, gitlock, branch, recipe_name, compose_type, test_m
|
||||
deps = []
|
||||
try:
|
||||
with dnflock.lock:
|
||||
(installed_size, deps) = projects_depsolve_with_size(dnflock.dbo, projects, with_core=False)
|
||||
(installed_size, deps) = projects_depsolve_with_size(dnflock.dbo, projects, recipe.group_names, with_core=False)
|
||||
except ProjectsError as e:
|
||||
log.error("start_build depsolve: %s", str(e))
|
||||
raise RuntimeError("Problem depsolving %s: %s" % (recipe["name"], str(e)))
|
||||
@ -274,7 +274,7 @@ def start_build(cfg, dnflock, gitlock, branch, recipe_name, compose_type, test_m
|
||||
ks_projects = [(name, "*") for name in ks.handler.packages.packageList]
|
||||
try:
|
||||
with dnflock.lock:
|
||||
(template_size, _) = projects_depsolve_with_size(dnflock.dbo, ks_projects,
|
||||
(template_size, _) = projects_depsolve_with_size(dnflock.dbo, ks_projects, [],
|
||||
with_core=not ks.handler.packages.nocore)
|
||||
except ProjectsError as e:
|
||||
log.error("start_build depsolve: %s", str(e))
|
||||
|
||||
@ -182,13 +182,15 @@ def projects_info(dbo, project_names):
|
||||
pkgs = dbo.sack.query().available()
|
||||
return sorted(map(pkg_to_project_info, pkgs), key=lambda p: p["name"].lower())
|
||||
|
||||
def _depsolve(dbo, projects):
|
||||
def _depsolve(dbo, projects, groups):
|
||||
"""Add projects to a new transaction
|
||||
|
||||
:param dbo: dnf base object
|
||||
:type dbo: dnf.Base
|
||||
:param projects: The projects and version globs to find the dependencies for
|
||||
:type projects: List of tuples
|
||||
:param groups: The groups to include in dependency solving
|
||||
:type groups: List of str
|
||||
:returns: None
|
||||
:rtype: None
|
||||
:raises: ProjectsError if there was a problem installing something
|
||||
@ -196,6 +198,12 @@ def _depsolve(dbo, projects):
|
||||
# This resets the transaction
|
||||
dbo.reset(goal=True)
|
||||
install_errors = []
|
||||
for name in groups:
|
||||
try:
|
||||
dbo.group_install(name, ["mandatory", "default"])
|
||||
except dnf.exceptions.MarkingError as e:
|
||||
install_errors.append(("Group %s" % (name), str(e)))
|
||||
|
||||
for name, version in projects:
|
||||
try:
|
||||
if not version:
|
||||
@ -214,18 +222,20 @@ def _depsolve(dbo, projects):
|
||||
raise ProjectsError("The following package(s) had problems: %s" % ",".join(["%s (%s)" % (pattern, err) for pattern, err in install_errors]))
|
||||
|
||||
|
||||
def projects_depsolve(dbo, projects):
|
||||
def projects_depsolve(dbo, projects, groups):
|
||||
"""Return the dependencies for a list of projects
|
||||
|
||||
:param dbo: dnf base object
|
||||
:type dbo: dnf.Base
|
||||
:param projects: The projects to find the dependencies for
|
||||
:type projects: List of Strings
|
||||
:param groups: The groups to include in dependency solving
|
||||
:type groups: List of str
|
||||
:returns: NEVRA's of the project and its dependencies
|
||||
:rtype: list of dicts
|
||||
:raises: ProjectsError if there was a problem installing something
|
||||
"""
|
||||
_depsolve(dbo, projects)
|
||||
_depsolve(dbo, projects, groups)
|
||||
|
||||
try:
|
||||
dbo.resolve()
|
||||
@ -259,18 +269,20 @@ def estimate_size(packages, block_size=6144):
|
||||
return installed_size
|
||||
|
||||
|
||||
def projects_depsolve_with_size(dbo, projects, with_core=True):
|
||||
def projects_depsolve_with_size(dbo, projects, groups, with_core=True):
|
||||
"""Return the dependencies and installed size for a list of projects
|
||||
|
||||
:param dbo: dnf base object
|
||||
:type dbo: dnf.Base
|
||||
:param project_names: The projects to find the dependencies for
|
||||
:type project_names: List of Strings
|
||||
:param groups: The groups to include in dependency solving
|
||||
:type groups: List of str
|
||||
:returns: installed size and a list of NEVRA's of the project and its dependencies
|
||||
:rtype: tuple of (int, list of dicts)
|
||||
:raises: ProjectsError if there was a problem installing something
|
||||
"""
|
||||
_depsolve(dbo, projects)
|
||||
_depsolve(dbo, projects, groups)
|
||||
|
||||
if with_core:
|
||||
dbo.group_install("core", ['mandatory', 'default', 'optional'])
|
||||
@ -323,7 +335,7 @@ def modules_info(dbo, module_names):
|
||||
|
||||
# Add the dependency info to each one
|
||||
for module in modules:
|
||||
module["dependencies"] = projects_depsolve(dbo, [(module["name"], "*.*")])
|
||||
module["dependencies"] = projects_depsolve(dbo, [(module["name"], "*.*")], [])
|
||||
|
||||
return modules
|
||||
|
||||
|
||||
@ -47,21 +47,24 @@ class Recipe(dict):
|
||||
and adds a .filename property to return the recipe's filename,
|
||||
and a .toml() function to return the recipe as a TOML string.
|
||||
"""
|
||||
def __init__(self, name, description, version, modules, packages, customizations=None):
|
||||
def __init__(self, name, description, version, modules, packages, groups, customizations=None):
|
||||
# Check that version is empty or semver compatible
|
||||
if version:
|
||||
semver.Version(version)
|
||||
|
||||
# Make sure modules and packages are listed by their case-insensitive names
|
||||
# Make sure modules, packages, and groups are listed by their case-insensitive names
|
||||
if modules is not None:
|
||||
modules = sorted(modules, key=lambda m: m["name"].lower())
|
||||
if packages is not None:
|
||||
packages = sorted(packages, key=lambda p: p["name"].lower())
|
||||
if groups is not None:
|
||||
groups = sorted(groups, key=lambda g: g["name"].lower())
|
||||
dict.__init__(self, name=name,
|
||||
description=description,
|
||||
version=version,
|
||||
modules=modules,
|
||||
packages=packages,
|
||||
groups=groups,
|
||||
customizations=customizations)
|
||||
|
||||
# We don't want customizations=None to show up in the TOML so remove it
|
||||
@ -88,6 +91,11 @@ class Recipe(dict):
|
||||
"""Return the names and version globs of the modules"""
|
||||
return [(m["name"], m["version"]) for m in self["modules"] or []]
|
||||
|
||||
@property
|
||||
def group_names(self):
|
||||
"""Return the names of the groups. Groups do not have versions."""
|
||||
return map(lambda g: g["name"], self["groups"] or [])
|
||||
|
||||
@property
|
||||
def filename(self):
|
||||
"""Return the Recipe's filename
|
||||
@ -144,21 +152,25 @@ class Recipe(dict):
|
||||
"""
|
||||
module_names = self.module_names
|
||||
package_names = self.package_names
|
||||
group_names = self.group_names
|
||||
|
||||
new_modules = []
|
||||
new_packages = []
|
||||
new_groups = []
|
||||
for dep in deps:
|
||||
if dep["name"] in package_names:
|
||||
new_packages.append(RecipePackage(dep["name"], dep_evra(dep)))
|
||||
elif dep["name"] in module_names:
|
||||
new_modules.append(RecipeModule(dep["name"], dep_evra(dep)))
|
||||
elif dep["name"] in group_names:
|
||||
new_groups.append(RecipeGroup(dep["name"]))
|
||||
if "customizations" in self:
|
||||
customizations = self["customizations"]
|
||||
else:
|
||||
customizations = None
|
||||
|
||||
return Recipe(self["name"], self["description"], self["version"],
|
||||
new_modules, new_packages, customizations)
|
||||
new_modules, new_packages, new_groups, customizations)
|
||||
|
||||
class RecipeModule(dict):
|
||||
def __init__(self, name, version):
|
||||
@ -167,6 +179,10 @@ class RecipeModule(dict):
|
||||
class RecipePackage(RecipeModule):
|
||||
pass
|
||||
|
||||
class RecipeGroup(dict):
|
||||
def __init__(self, name):
|
||||
dict.__init__(self, name=name)
|
||||
|
||||
def recipe_from_file(recipe_path):
|
||||
"""Return a recipe file as a Recipe object
|
||||
|
||||
@ -210,6 +226,10 @@ def recipe_from_dict(recipe_dict):
|
||||
packages = [RecipePackage(p.get("name"), p.get("version")) for p in recipe_dict["packages"]]
|
||||
else:
|
||||
packages = []
|
||||
if recipe_dict.get("groups"):
|
||||
groups = [RecipeGroup(g.get("name")) for g in recipe_dict["groups"]]
|
||||
else:
|
||||
groups = []
|
||||
name = recipe_dict["name"]
|
||||
description = recipe_dict["description"]
|
||||
version = recipe_dict.get("version", None)
|
||||
@ -217,7 +237,7 @@ def recipe_from_dict(recipe_dict):
|
||||
except KeyError as e:
|
||||
raise RecipeError("There was a problem parsing the recipe: %s" % str(e))
|
||||
|
||||
return Recipe(name, description, version, modules, packages, customizations)
|
||||
return Recipe(name, description, version, modules, packages, groups, customizations)
|
||||
|
||||
def gfile(path):
|
||||
"""Convert a string path to GFile for use with Git"""
|
||||
@ -897,5 +917,6 @@ def recipe_diff(old_recipe, new_recipe):
|
||||
|
||||
diffs.extend(diff_items("Module", old_recipe["modules"], new_recipe["modules"]))
|
||||
diffs.extend(diff_items("Package", old_recipe["packages"], new_recipe["packages"]))
|
||||
diffs.extend(diff_items("Group", old_recipe["groups"], new_recipe["groups"]))
|
||||
|
||||
return diffs
|
||||
|
||||
@ -1278,7 +1278,7 @@ def v0_api(api):
|
||||
deps = []
|
||||
try:
|
||||
with api.config["DNFLOCK"].lock:
|
||||
deps = projects_depsolve(api.config["DNFLOCK"].dbo, projects)
|
||||
deps = projects_depsolve(api.config["DNFLOCK"].dbo, projects, blueprint.group_names)
|
||||
except ProjectsError as e:
|
||||
errors.append("%s: %s" % (blueprint_name, str(e)))
|
||||
log.error("(v0_blueprints_freeze) %s", str(e))
|
||||
@ -1330,7 +1330,7 @@ def v0_api(api):
|
||||
deps = []
|
||||
try:
|
||||
with api.config["DNFLOCK"].lock:
|
||||
deps = projects_depsolve(api.config["DNFLOCK"].dbo, projects)
|
||||
deps = projects_depsolve(api.config["DNFLOCK"].dbo, projects, blueprint.group_names)
|
||||
except ProjectsError as e:
|
||||
errors.append("%s: %s" % (blueprint_name, str(e)))
|
||||
log.error("(v0_blueprints_depsolve) %s", str(e))
|
||||
@ -1385,7 +1385,7 @@ def v0_api(api):
|
||||
"""Return detailed information about the listed projects"""
|
||||
try:
|
||||
with api.config["DNFLOCK"].lock:
|
||||
deps = projects_depsolve(api.config["DNFLOCK"].dbo, [(n, "*") for n in project_names.split(",")])
|
||||
deps = projects_depsolve(api.config["DNFLOCK"].dbo, [(n, "*") for n in project_names.split(",")], [])
|
||||
except ProjectsError as e:
|
||||
log.error("(v0_projects_depsolve) %s", str(e))
|
||||
return jsonify(status=False, errors=[str(e)]), 400
|
||||
|
||||
@ -1 +1 @@
|
||||
{'name': 'custom-base', 'description': 'A base system with customizations', 'version': '0.0.1', 'modules': [], 'packages': [{'name': 'bash', 'version': '4.4.*'}], 'customizations': {'hostname': 'custombase', 'sshkey': [{'user': 'root', 'key': 'A SSH KEY FOR ROOT'}]}}
|
||||
{'name': 'custom-base', 'description': 'A base system with customizations', 'version': '0.0.1', 'groups': [], 'modules': [], 'packages': [{'name': 'bash', 'version': '4.4.*'}], 'customizations': {'hostname': 'custombase', 'sshkey': [{'user': 'root', 'key': 'A SSH KEY FOR ROOT'}]}}
|
||||
|
||||
@ -1 +1 @@
|
||||
{'description': u'An example http server with PHP and MySQL support.', 'packages': [{'version': u'6.6.*', 'name': u'openssh-server'}, {'version': u'3.0.*', 'name': u'rsync'}, {'version': u'2.2', 'name': u'tmux'}], 'modules': [{'version': u'2.4.*', 'name': u'httpd'}, {'version': u'5.4', 'name': u'mod_auth_kerb'}, {'version': u'2.4.*', 'name': u'mod_ssl'}, {'version': u'5.4.*', 'name': u'php'}, {'version': u'5.4.*', 'name': u'php-mysql'}], 'version': u'0.0.1', 'name': u'http-server'}
|
||||
{'description': u'An example http server with PHP and MySQL support.', 'packages': [{'version': u'6.6.*', 'name': u'openssh-server'}, {'version': u'3.0.*', 'name': u'rsync'}, {'version': u'2.2', 'name': u'tmux'}], 'groups': [], 'modules': [{'version': u'2.4.*', 'name': u'httpd'}, {'version': u'5.4', 'name': u'mod_auth_kerb'}, {'version': u'2.4.*', 'name': u'mod_ssl'}, {'version': u'5.4.*', 'name': u'php'}, {'version': u'5.4.*', 'name': u'php-mysql'}], 'version': u'0.0.1', 'name': u'http-server'}
|
||||
|
||||
@ -1 +1 @@
|
||||
{'description': u'An example http server with PHP and MySQL support.', 'packages': [], 'modules': [], 'version': u'0.0.1', 'name': u'http-server'}
|
||||
{'description': u'An example http server with PHP and MySQL support.', 'packages': [], 'groups': [], 'modules': [], 'version': u'0.0.1', 'name': u'http-server'}
|
||||
|
||||
@ -1 +1 @@
|
||||
{'description': u'An example http server with PHP and MySQL support.', 'packages': [], 'modules': [{'version': u'2.4.*', 'name': u'httpd'}, {'version': u'5.4', 'name': u'mod_auth_kerb'}, {'version': u'2.4.*', 'name': u'mod_ssl'}, {'version': u'5.4.*', 'name': u'php'}, {'version': u'5.4.*', 'name': u'php-mysql'}], 'version': u'0.0.1', 'name': u'http-server'}
|
||||
{'description': u'An example http server with PHP and MySQL support.', 'packages': [], 'groups': [], 'modules': [{'version': u'2.4.*', 'name': u'httpd'}, {'version': u'5.4', 'name': u'mod_auth_kerb'}, {'version': u'2.4.*', 'name': u'mod_ssl'}, {'version': u'5.4.*', 'name': u'php'}, {'version': u'5.4.*', 'name': u'php-mysql'}], 'version': u'0.0.1', 'name': u'http-server'}
|
||||
|
||||
@ -1 +1 @@
|
||||
{'description': u'An example http server with PHP and MySQL support.', 'packages': [{'version': u'6.6.*', 'name': u'openssh-server'}, {'version': u'3.0.*', 'name': u'rsync'}, {'version': u'2.2', 'name': u'tmux'}], 'modules': [], 'version': u'0.0.1', 'name': u'http-server'}
|
||||
{'description': u'An example http server with PHP and MySQL support.', 'packages': [{'version': u'6.6.*', 'name': u'openssh-server'}, {'version': u'3.0.*', 'name': u'rsync'}, {'version': u'2.2', 'name': u'tmux'}], 'groups': [], 'modules': [], 'version': u'0.0.1', 'name': u'http-server'}
|
||||
|
||||
@ -156,27 +156,27 @@ class ProjectsTest(unittest.TestCase):
|
||||
self.assertEqual(projects[0]["builds"][0]["source"]["license"], "GPLv3+")
|
||||
|
||||
def test_projects_depsolve(self):
|
||||
deps = projects_depsolve(self.dbo, [("bash", "*.*")])
|
||||
|
||||
deps = projects_depsolve(self.dbo, [("bash", "*.*")], [])
|
||||
self.assertTrue(len(deps) > 3)
|
||||
self.assertEqual(deps[0]["name"], "basesystem")
|
||||
|
||||
def test_projects_depsolve_version(self):
|
||||
"""Test that depsolving with a partial wildcard version works"""
|
||||
deps = projects_depsolve(self.dbo, [("bash", "4.*")])
|
||||
deps = projects_depsolve(self.dbo, [("bash", "4.*")], [])
|
||||
self.assertEqual(deps[1]["name"], "bash")
|
||||
|
||||
deps = projects_depsolve(self.dbo, [("bash", "4.4.*")])
|
||||
deps = projects_depsolve(self.dbo, [("bash", "4.4.*")], [])
|
||||
self.assertEqual(deps[1]["name"], "bash")
|
||||
|
||||
def test_projects_depsolve_oldversion(self):
|
||||
"""Test that depsolving a specific non-existant version fails"""
|
||||
with self.assertRaises(ProjectsError):
|
||||
deps = projects_depsolve(self.dbo, [("bash", "1.0.0")])
|
||||
deps = projects_depsolve(self.dbo, [("bash", "1.0.0")], [])
|
||||
self.assertEqual(deps[1]["name"], "bash")
|
||||
|
||||
def test_projects_depsolve_fail(self):
|
||||
with self.assertRaises(ProjectsError):
|
||||
projects_depsolve(self.dbo, [("nada-package", "*.*")])
|
||||
projects_depsolve(self.dbo, [("nada-package", "*.*")], [])
|
||||
|
||||
def test_modules_list_all(self):
|
||||
modules = modules_list(self.dbo, None)
|
||||
|
||||
@ -95,27 +95,27 @@ class BasicRecipeTest(unittest.TestCase):
|
||||
"""Test the Recipe's version bump function"""
|
||||
|
||||
# Neither have a version
|
||||
recipe = recipes.Recipe("test-recipe", "A recipe used for testing", None, None, None)
|
||||
recipe = recipes.Recipe("test-recipe", "A recipe used for testing", None, None, None, None)
|
||||
new_version = recipe.bump_version(None)
|
||||
self.assertEqual(new_version, "0.0.1")
|
||||
|
||||
# Original has a version, new does not
|
||||
recipe = recipes.Recipe("test-recipe", "A recipe used for testing", None, None, None)
|
||||
recipe = recipes.Recipe("test-recipe", "A recipe used for testing", None, None, None, None)
|
||||
new_version = recipe.bump_version("0.0.1")
|
||||
self.assertEqual(new_version, "0.0.2")
|
||||
|
||||
# Original has no version, new does
|
||||
recipe = recipes.Recipe("test-recipe", "A recipe used for testing", "0.1.0", None, None)
|
||||
recipe = recipes.Recipe("test-recipe", "A recipe used for testing", "0.1.0", None, None, None)
|
||||
new_version = recipe.bump_version(None)
|
||||
self.assertEqual(new_version, "0.1.0")
|
||||
|
||||
# New and Original are the same
|
||||
recipe = recipes.Recipe("test-recipe", "A recipe used for testing", "0.0.1", None, None)
|
||||
recipe = recipes.Recipe("test-recipe", "A recipe used for testing", "0.0.1", None, None, None)
|
||||
new_version = recipe.bump_version("0.0.1")
|
||||
self.assertEqual(new_version, "0.0.2")
|
||||
|
||||
# New is different from Original
|
||||
recipe = recipes.Recipe("test-recipe", "A recipe used for testing", "0.1.1", None, None)
|
||||
recipe = recipes.Recipe("test-recipe", "A recipe used for testing", "0.1.1", None, None, None)
|
||||
new_version = recipe.bump_version("0.0.1")
|
||||
self.assertEqual(new_version, "0.1.1")
|
||||
|
||||
@ -136,8 +136,8 @@ class BasicRecipeTest(unittest.TestCase):
|
||||
|
||||
def recipe_diff_test(self):
|
||||
"""Test the recipe_diff function"""
|
||||
old_recipe = recipes.Recipe("test-recipe", "A recipe used for testing", "0.1.1", self.old_modules, self.old_packages)
|
||||
new_recipe = recipes.Recipe("test-recipe", "A recipe used for testing", "0.3.1", self.new_modules, self.new_packages)
|
||||
old_recipe = recipes.Recipe("test-recipe", "A recipe used for testing", "0.1.1", self.old_modules, self.old_packages, [])
|
||||
new_recipe = recipes.Recipe("test-recipe", "A recipe used for testing", "0.3.1", self.new_modules, self.new_packages, [])
|
||||
result = [{'new': {'Version': '0.3.1'}, 'old': {'Version': '0.1.1'}},
|
||||
{'new': {'Module': {'name': 'openssh', 'version': '2.8.1'}}, 'old': None},
|
||||
{'new': None, 'old': {'Module': {'name': 'bash', 'version': '4.*'}}},
|
||||
@ -182,7 +182,7 @@ version = "2.7.*"
|
||||
|
||||
def test_02_commit_recipe(self):
|
||||
"""Test committing a Recipe object"""
|
||||
recipe = recipes.Recipe("test-recipe", "A recipe used for testing", None, None, None)
|
||||
recipe = recipes.Recipe("test-recipe", "A recipe used for testing", None, None, None, None)
|
||||
oid = recipes.commit_recipe(self.repo, "master", recipe)
|
||||
self.assertNotEqual(oid, None)
|
||||
|
||||
|
||||
Loading…
Reference in New Issue
Block a user