almalinux/lorax
13
0
Fork 0
Code Issues Pull Requests Releases Activity

Add support for version globs to blueprints

Browse Source 
This uses dnf's version__glob filter to implement it. It amounts to '*'
wildcards and '?' for single character matching.
This commit is contained in:
Brian C. Lane 2018-05-18 12:03:26 -07:00
parent d406fbdf83
commit 095829171a
6 changed files with 82 additions and 40 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
src/pylorax/api/compose.py
View File

@ -223,9 +223,9 @@ def start_build(cfg, dnflock, gitlock, branch, recipe_name, compose_type, test_m
# Combine modules and packages and depsolve the list
# TODO include the version/glob in the depsolving
module_names = [m["name"] for m in recipe["modules"] or []]
package_names = [p["name"] for p in recipe["packages"] or []]
projects = sorted(set(module_names+package_names), key=lambda n: n.lower())
module_nver = recipe.module_nver
package_nver = recipe.package_nver
projects = sorted(set(module_nver+package_nver), key=lambda p: p[0].lower())
deps = []
try:
with dnflock.lock:
@ -242,9 +242,10 @@ def start_build(cfg, dnflock, gitlock, branch, recipe_name, compose_type, test_m
ks_version = makeVersion()
ks = KickstartParser(ks_version, errorsAreFatal=False, missingIncludeIsFatal=False)
ks.readKickstartFromString(ks_template+"\n%end\n")
ks_projects = [(name, "*") for name in ks.handler.packages.packageList]
try:
with dnflock.lock:
(template_size, _) = projects_depsolve_with_size(dnflock.dbo, ks.handler.packages.packageList,
(template_size, _) = projects_depsolve_with_size(dnflock.dbo, ks_projects,
with_core=not ks.handler.packages.nocore)
except ProjectsError as e:
log.error("start_build depsolve: %s", str(e))

58
src/pylorax/api/projects.py
View File

@ -179,29 +179,50 @@ def projects_info(dbo, project_names):
pkgs = dbo.sack.query().available()
return sorted(map(pkg_to_project_info, pkgs), key=lambda p: p["name"].lower())
def _depsolve(dbo, projects):
"""Add projects to a new transaction
def projects_depsolve(dbo, project_names):
:param dbo: dnf base object
:type dbo: dnf.Base
:param projects: The projects and version globs to find the dependencies for
:type projects: List of tuples
:returns: None
:rtype: None
:raises: ProjectsError if there was a problem installing something
"""
# This resets the transaction
dbo.reset(goal=True)
for name, version in projects:
try:
if not version:
version = "*"
pkgs = [pkg for pkg in dnf.subject.Subject(name).get_best_query(dbo.sack).filter(version__glob=version, latest=True)]
if not pkgs:
raise ProjectsError("No match for %s-%s" % (name, version))
for p in pkgs:
dbo.package_install(p)
except dnf.exceptions.MarkingError:
raise ProjectsError("No match for %s-%s" % (name, version))
def projects_depsolve(dbo, projects):
"""Return the dependencies for a list of projects
:param dbo: dnf base object
:type dbo: dnf.Base
:param project_names: The projects to find the dependencies for
:type project_names: List of Strings
:param projects: The projects to find the dependencies for
:type projects: List of Strings
:returns: NEVRA's of the project and its dependencies
:rtype: list of dicts
:raises: ProjectsError if there was a problem installing something
"""
# This resets the transaction
dbo.reset(goal=True)
for p in project_names:
try:
dbo.install(p)
except dnf.exceptions.MarkingError:
raise ProjectsError("No match for %s" % p)
_depsolve(dbo, projects)
try:
dbo.resolve()
except dnf.exceptions.DepsolveError as e:
raise ProjectsError("There was a problem depsolving %s: %s" % (project_names, str(e)))
raise ProjectsError("There was a problem depsolving %s: %s" % (projects, str(e)))
if len(dbo.transaction) == 0:
return []
@ -230,7 +251,7 @@ def estimate_size(packages, block_size=6144):
return installed_size
def projects_depsolve_with_size(dbo, project_names, with_core=True):
def projects_depsolve_with_size(dbo, projects, with_core=True):
"""Return the dependencies and installed size for a list of projects
:param dbo: dnf base object
@ -239,14 +260,9 @@ def projects_depsolve_with_size(dbo, project_names, with_core=True):
:type project_names: List of Strings
:returns: installed size and a list of NEVRA's of the project and its dependencies
:rtype: tuple of (int, list of dicts)
:raises: ProjectsError if there was a problem installing something
"""
# This resets the transaction
dbo.reset(goal=True)
for p in project_names:
try:
dbo.install(p)
except dnf.exceptions.MarkingError:
raise ProjectsError("No match for %s" % p)
_depsolve(dbo, projects)
if with_core:
dbo.group_install("core", ['mandatory', 'default', 'optional'])
@ -254,7 +270,7 @@ def projects_depsolve_with_size(dbo, project_names, with_core=True):
try:
dbo.resolve()
except dnf.exceptions.DepsolveError as e:
raise ProjectsError("There was a problem depsolving %s: %s" % (project_names, str(e)))
raise ProjectsError("There was a problem depsolving %s: %s" % (projects, str(e)))
if len(dbo.transaction) == 0:
return (0, [])
@ -299,6 +315,6 @@ def modules_info(dbo, module_names):
# Add the dependency info to each one
for module in modules:
module["dependencies"] = projects_depsolve(dbo, [module["name"]])
module["dependencies"] = projects_depsolve(dbo, [(module["name"], "*.*")])
return modules

10
src/pylorax/api/recipes.py
View File

@ -73,11 +73,21 @@ class Recipe(dict):
"""Return the names of the packages"""
return [p["name"] for p in self["packages"] or []]
@property
def package_nver(self):
"""Return the names and version globs of the packages"""
return [(p["name"], p["version"]) for p in self["packages"] or []]
@property
def module_names(self):
"""Return the names of the modules"""
return [m["name"] for m in self["modules"] or []]
@property
def module_nver(self):
"""Return the names and version globs of the modules"""
return [(m["name"], m["version"]) for m in self["modules"] or []]
@property
def filename(self):
"""Return the Recipe's filename

14
src/pylorax/api/v0.py
View File

@ -1181,9 +1181,9 @@ def v0_api(api):
# Combine modules and packages and depsolve the list
# TODO include the version/glob in the depsolving
module_names = blueprint.module_names
package_names = blueprint.package_names
projects = sorted(set(module_names+package_names), key=lambda n: n.lower())
module_nver = blueprint.module_nver
package_nver = blueprint.package_nver
projects = sorted(set(module_nver+package_nver), key=lambda p: p[0].lower())
deps = []
try:
with api.config["DNFLOCK"].lock:
@ -1233,9 +1233,9 @@ def v0_api(api):
# Combine modules and packages and depsolve the list
# TODO include the version/glob in the depsolving
module_names = [m["name"] for m in blueprint["modules"] or []]
package_names = [p["name"] for p in blueprint["packages"] or []]
projects = sorted(set(module_names+package_names), key=lambda n: n.lower())
module_nver = blueprint.module_nver
package_nver = blueprint.package_nver
projects = sorted(set(module_nver+package_nver), key=lambda p: p[0].lower())
deps = []
try:
with api.config["DNFLOCK"].lock:
@ -1294,7 +1294,7 @@ def v0_api(api):
"""Return detailed information about the listed projects"""
try:
with api.config["DNFLOCK"].lock:
deps = projects_depsolve(api.config["DNFLOCK"].dbo, project_names.split(","))
deps = projects_depsolve(api.config["DNFLOCK"].dbo, [(n, "*") for n in project_names.split(",")])
except ProjectsError as e:
log.error("(v0_projects_depsolve) %s", str(e))
return jsonify(status=False, errors=[str(e)]), 400

18
tests/pylorax/test_projects.py
View File

@ -151,13 +151,27 @@ class ProjectsTest(unittest.TestCase):
self.assertEqual(projects[0]["builds"][0]["source"]["license"], "GPLv3+")
def test_projects_depsolve(self):
deps = projects_depsolve(self.dbo, ["bash"])
deps = projects_depsolve(self.dbo, [("bash", "*.*")])
self.assertEqual(deps[0]["name"], "basesystem")
def test_projects_depsolve_version(self):
"""Test that depsolving with a partial wildcard version works"""
deps = projects_depsolve(self.dbo, [("bash", "4.*")])
self.assertEqual(deps[1]["name"], "bash")
deps = projects_depsolve(self.dbo, [("bash", "4.4.*")])
self.assertEqual(deps[1]["name"], "bash")
def test_projects_depsolve_oldversion(self):
"""Test that depsolving a specific non-existant version fails"""
with self.assertRaises(ProjectsError):
deps = projects_depsolve(self.dbo, [("bash", "1.0.0")])
self.assertEqual(deps[1]["name"], "bash")
def test_projects_depsolve_fail(self):
with self.assertRaises(ProjectsError):
projects_depsolve(self.dbo, ["nada-package"])
projects_depsolve(self.dbo, [("nada-package", "*.*")])
def test_modules_list_all(self):
modules = modules_list(self.dbo, None)

13
tests/pylorax/test_server.py
View File

@ -173,7 +173,7 @@ class ServerTestCase(unittest.TestCase):
"modules":[{"name":"glusterfs", "version":"4.0.*"},
{"name":"glusterfs-cli", "version":"4.0.*"}],
"packages":[{"name":"samba", "version":"4.8.*"},
{"name":"tmux", "version":"2.2"}]}
{"name":"tmux", "version":"2.7"}]}
resp = self.server.post("/api/v0/blueprints/new",
data=json.dumps(test_blueprint),
@ -217,7 +217,7 @@ class ServerTestCase(unittest.TestCase):
"modules":[{"name":"glusterfs", "version":"4.0.*"},
{"name":"glusterfs-cli", "version":"4.0.*"}],
"packages":[{"name":"samba", "version":"4.8.*"},
{"name":"tmux", "version":"2.2"}]}
{"name":"tmux", "version":"2.7"}]}
resp = self.server.post("/api/v0/blueprints/workspace",
data=json.dumps(test_blueprint),
@ -243,7 +243,7 @@ class ServerTestCase(unittest.TestCase):
"modules":[{"name":"glusterfs", "version":"4.0.*"},
{"name":"glusterfs-cli", "version":"4.0.*"}],
"packages":[{"name":"samba", "version":"4.8.*"},
{"name":"tmux", "version":"2.2"}]}
{"name":"tmux", "version":"2.7"}]}
resp = self.server.post("/api/v0/blueprints/workspace",
data=json.dumps(test_blueprint),
@ -374,7 +374,7 @@ class ServerTestCase(unittest.TestCase):
"modules":[{"name":"glusterfs", "version":"4.0.*"},
{"name":"glusterfs-cli", "version":"4.0.*"}],
"packages":[{"name":"samba", "version":"4.8.*"},
{"name":"tmux", "version":"2.2"}]}
{"name":"tmux", "version":"2.7"}]}
resp = self.server.post("/api/v0/blueprints/workspace",
data=json.dumps(test_blueprint),
@ -390,7 +390,7 @@ class ServerTestCase(unittest.TestCase):
"old": {"Description": "An example GlusterFS server with samba"}},
{"new": {"Version": "0.3.0"},
"old": {"Version": "0.0.1"}},
{"new": {"Package": {"version": "2.2", "name": "tmux"}},
{"new": {"Package": {"version": "2.7", "name": "tmux"}},
"old": None}]}
self.assertEqual(data, result)
@ -437,6 +437,7 @@ class ServerTestCase(unittest.TestCase):
blueprints = data.get("blueprints")
self.assertNotEqual(blueprints, None)
self.assertEqual(len(blueprints), 1)
self.assertTrue(len(blueprints[0]["blueprint"]["modules"]) > 0)
self.assertEqual(blueprints[0]["blueprint"]["name"], "glusterfs")
evra = blueprints[0]["blueprint"]["modules"][0]["version"]
self.assertEqual(len(evra) > 10, True)
@ -502,7 +503,7 @@ class ServerTestCase(unittest.TestCase):
"modules":[{"name":"glusterfs", "version":"4.0.*"},
{"name":"glusterfs-cli", "version":"4.0.*"}],
"packages":[{"name":"samba", "version":"4.8.*"},
{"name":"tmux", "version":"2.2"}]}
{"name":"tmux", "version":"2.7"}]}
resp = self.server.post("/api/v0/blueprints/new?branch=test",
data=json.dumps(test_blueprint),