almalinux/lorax
13
0
Fork 0
Code Issues Pull Requests Releases Activity

Add support for version globs to blueprints

Browse Source 
You can use '*' wildcards and '?' for single character matching.
This commit is contained in:
Brian C. Lane 2018-05-18 17:28:20 -07:00
parent 9e06f6e113
commit b99d8d7f6b
10 changed files with 110 additions and 76 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
src/pylorax/api/compose.py
View File

@ -230,10 +230,9 @@ def start_build(cfg, yumlock, gitlock, branch, recipe_name, compose_type, test_m
(commit_id, recipe) = read_recipe_and_id(gitlock.repo, branch, recipe_name)
# Combine modules and packages and depsolve the list
# TODO include the version/glob in the depsolving
module_names = map(lambda m: m["name"], recipe["modules"] or [])
package_names = map(lambda p: p["name"], recipe["packages"] or [])
projects = sorted(set(module_names+package_names), key=lambda n: n.lower())
module_nver = recipe.module_nver
package_nver = recipe.package_nver
projects = sorted(set(module_nver+package_nver), key=lambda p: p[0].lower())
deps = []
try:
with yumlock.lock:
@ -250,9 +249,10 @@ def start_build(cfg, yumlock, gitlock, branch, recipe_name, compose_type, test_m
ks_version = makeVersion(RHEL7)
ks = KickstartParser(ks_version, errorsAreFatal=False, missingIncludeIsFatal=False)
ks.readKickstartFromString(ks_template+"\n%end\n")
pkgs = [(name, "*") for name in ks.handler.packages.packageList]
try:
with yumlock.lock:
(template_size, _) = projects_depsolve_with_size(yumlock.yb, ks.handler.packages.packageList,
(template_size, _) = projects_depsolve_with_size(yumlock.yb, pkgs,
with_core=not ks.handler.packages.nocore)
except ProjectsError as e:
log.error("start_build depsolve: %s", str(e))

36
src/pylorax/api/projects.py
View File

@ -186,28 +186,31 @@ def projects_info(yb, project_names):
return sorted(map(yaps_to_project_info, ybl.available), key=lambda p: p["name"].lower())
def projects_depsolve(yb, project_names):
def projects_depsolve(yb, projects):
"""Return the dependencies for a list of projects
:param yb: yum base object
:type yb: YumBase
:param project_names: The projects to find the dependencies for
:type project_names: List of Strings
:param projects: The projects and version globs to find the dependencies for
:type projects: List of tuples
:returns: NEVRA's of the project and its dependencies
:rtype: list of dicts
:raises: ProjectsError if there was a problem installing something
"""
try:
# This resets the transaction
yb.closeRpmDB()
for p in project_names:
yb.install(pattern=p)
for name, version in projects:
if not version:
version = "*"
yb.install(pattern="%s-%s" % (name, version))
(rc, msg) = yb.buildTransaction()
if rc not in [0, 1, 2]:
raise ProjectsError("There was a problem depsolving %s: %s" % (project_names, msg))
raise ProjectsError("There was a problem depsolving %s: %s" % (projects, msg))
yb.tsInfo.makelists()
deps = sorted(map(tm_to_dep, yb.tsInfo.installed + yb.tsInfo.depinstalled), key=lambda p: p["name"].lower())
except YumBaseError as e:
raise ProjectsError("There was a problem depsolving %s: %s" % (project_names, str(e)))
raise ProjectsError("There was a problem depsolving %s: %s" % (projects, str(e)))
finally:
yb.closeRpmDB()
return deps
@ -232,31 +235,34 @@ def estimate_size(packages, block_size=4096):
installed_size += p.po.installedsize
return installed_size
def projects_depsolve_with_size(yb, project_names, with_core=True):
def projects_depsolve_with_size(yb, projects, with_core=True):
"""Return the dependencies and installed size for a list of projects
:param yb: yum base object
:type yb: YumBase
:param project_names: The projects to find the dependencies for
:type project_names: List of Strings
:param projects: The projects and version globs to find the dependencies for
:type projects: List of tuples
:returns: installed size and a list of NEVRA's of the project and its dependencies
:rtype: tuple of (int, list of dicts)
:raises: ProjectsError if there was a problem installing something
"""
try:
# This resets the transaction
yb.closeRpmDB()
for p in project_names:
yb.install(pattern=p)
for name, version in projects:
if not version:
version = "*"
yb.install(pattern="%s-%s" % (name, version))
if with_core:
yb.selectGroup("core", group_package_types=['mandatory', 'default', 'optional'])
(rc, msg) = yb.buildTransaction()
if rc not in [0, 1, 2]:
raise ProjectsError("There was a problem depsolving %s: %s" % (project_names, msg))
raise ProjectsError("There was a problem depsolving %s: %s" % (projects, msg))
yb.tsInfo.makelists()
installed_size = estimate_size(yb.tsInfo.installed + yb.tsInfo.depinstalled)
deps = sorted(map(tm_to_dep, yb.tsInfo.installed + yb.tsInfo.depinstalled), key=lambda p: p["name"].lower())
except YumBaseError as e:
raise ProjectsError("There was a problem depsolving %s: %s" % (project_names, str(e)))
raise ProjectsError("There was a problem depsolving %s: %s" % (projects, str(e)))
finally:
yb.closeRpmDB()
return (installed_size, deps)
@ -306,6 +312,6 @@ def modules_info(yb, module_names):
modules = sorted(map(yaps_to_project, ybl.available), key=lambda p: p["name"].lower())
# Add the dependency info to each one
for module in modules:
module["dependencies"] = projects_depsolve(yb, [module["name"]])
module["dependencies"] = projects_depsolve(yb, [(module["name"], "*")])
return modules

10
src/pylorax/api/recipes.py
View File

@ -73,11 +73,21 @@ class Recipe(dict):
"""Return the names of the packages"""
return map(lambda p: p["name"], self["packages"] or [])
@property
def package_nver(self):
"""Return the names and versions of the packages"""
return [(p["name"], p["version"]) for p in self["packages"] or []]
@property
def module_names(self):
"""Return the names of the modules"""
return map(lambda m: m["name"], self["modules"] or [])
@property
def module_nver(self):
"""Return the names and versions of the modules"""
return [(m["name"], m["version"]) for m in self["modules"] or []]
@property
def filename(self):
"""Return the Recipe's filename

15
src/pylorax/api/v0.py
View File

@ -1181,9 +1181,9 @@ def v0_api(api):
# Combine modules and packages and depsolve the list
# TODO include the version/glob in the depsolving
module_names = blueprint.module_names
package_names = blueprint.package_names
projects = sorted(set(module_names+package_names), key=lambda n: n.lower())
module_nver = blueprint.module_nver
package_nver = blueprint.package_nver
projects = sorted(set(module_nver+package_nver), key=lambda p: p[0].lower())
deps = []
try:
with api.config["YUMLOCK"].lock:
@ -1232,10 +1232,9 @@ def v0_api(api):
continue
# Combine modules and packages and depsolve the list
# TODO include the version/glob in the depsolving
module_names = map(lambda m: m["name"], blueprint["modules"] or [])
package_names = map(lambda p: p["name"], blueprint["packages"] or [])
projects = sorted(set(module_names+package_names), key=lambda n: n.lower())
module_nver = blueprint.module_nver
package_nver = blueprint.package_nver
projects = sorted(set(module_nver+package_nver), key=lambda p: p[0].lower())
deps = []
try:
with api.config["YUMLOCK"].lock:
@ -1294,7 +1293,7 @@ def v0_api(api):
"""Return detailed information about the listed projects"""
try:
with api.config["YUMLOCK"].lock:
deps = projects_depsolve(api.config["YUMLOCK"].yb, project_names.split(","))
deps = projects_depsolve(api.config["YUMLOCK"].yb, [(n, "*") for n in project_names.split(",")])
except ProjectsError as e:
log.error("(v0_projects_depsolve) %s", str(e))
return jsonify(status=False, errors=[str(e)]), 400

2
tests/pylorax/blueprints/custom-base.toml
View File

@ -4,7 +4,7 @@ version = "0.0.1"
[[packages]]
name = "bash"
version = "4.4.*"
version = "4.2.*"
[customizations]
hostname = "custombase"

6
tests/pylorax/blueprints/glusterfs.toml
View File

@ -3,12 +3,12 @@ description = "An example GlusterFS server with samba"
[[modules]]
name = "glusterfs"
version = "3.7.*"
version = "3.8.*"
[[modules]]
name = "glusterfs-cli"
version = "3.7.*"
version = "3.8.*"
[[packages]]
name = "samba"
version = "4.2.*"
version = "4.7.*"

6
tests/pylorax/blueprints/http-server.toml
View File

@ -24,12 +24,12 @@ version = "5.4.*"
[[packages]]
name = "tmux"
version = "2.2"
version = "1.8"
[[packages]]
name = "openssh-server"
version = "6.6.*"
version = "7.*"
[[packages]]
name = "rsync"
version = "3.0.*"
version = "3.1.*"

10
tests/pylorax/blueprints/kubernetes.toml
View File

@ -3,23 +3,23 @@ description = "An example kubernetes master"
[[modules]]
name = "kubernetes"
version = "1.2.*"
version = "1.5.2"
[[modules]]
name = "docker"
version = "1.10.*"
version = "1.13.*"
[[modules]]
name = "docker-lvm-plugin"
version = "1.10.*"
version = "1.13.*"
[[modules]]
name = "etcd"
version = "2.3.*"
version = "3.2.*"
[[modules]]
name = "flannel"
version = "0.5.*"
version = "0.7.*"
[[packages]]
name = "oci-systemd-hook"

24
tests/pylorax/test_projects.py
View File

@ -178,13 +178,27 @@ class ProjectsTest(unittest.TestCase):
projects_info(self.yb, ["bash"])
def test_projects_depsolve(self):
deps = projects_depsolve(self.yb, ["bash"])
deps = projects_depsolve(self.yb, [("bash", "*.*")])
self.assertTrue(len(deps) > 3)
self.assertEqual(deps[2]["name"], "basesystem")
self.assertEqual(deps[0]["name"], "basesystem")
def test_projects_depsolve_version(self):
"""Test that depsolving with a partial wildcard version works"""
deps = projects_depsolve(self.yb, [("bash", "4.*")])
self.assertEqual(deps[1]["name"], "bash")
deps = projects_depsolve(self.yb, [("bash", "4.2.*")])
self.assertEqual(deps[1]["name"], "bash")
def test_projects_depsolve_oldversion(self):
"""Test that depsolving a specific non-existant version fails"""
with self.assertRaises(ProjectsError):
deps = projects_depsolve(self.yb, [("bash", "1.0.0")])
self.assertEqual(deps[1]["name"], "bash")
def test_projects_depsolve_fail(self):
with self.assertRaises(ProjectsError):
projects_depsolve(self.yb, ["nada-package"])
projects_depsolve(self.yb, [("nada-package", "*.*")])
def test_modules_list(self):
modules = modules_list(self.yb, None)
@ -204,8 +218,10 @@ class ProjectsTest(unittest.TestCase):
modules = modules_info(self.yb, ["bash"])
print(modules)
self.assertTrue(len(modules) > 0)
self.assertTrue(len(modules[0]["dependencies"]) > 3)
self.assertEqual(modules[0]["name"], "bash")
self.assertEqual(modules[0]["dependencies"][0]["name"], "basesystem")
self.assertEqual(modules[0]["dependencies"][2]["name"], "basesystem")
def test_modules_info_yum_raises_exception(self):
with self.assertRaises(ProjectsError):

67
tests/pylorax/test_server.py
View File

@ -97,9 +97,9 @@ class ServerTestCase(unittest.TestCase):
{"name":"php", "version":"5.4.*"},
{"name": "php-mysql", "version":"5.4.*"}],
"name":"http-server",
"packages": [{"name":"openssh-server", "version": "6.6.*"},
{"name": "rsync", "version": "3.0.*"},
{"name": "tmux", "version": "2.2"}],
"packages": [{"name":"openssh-server", "version": "7.*"},
{"name": "rsync", "version": "3.1.*"},
{"name": "tmux", "version": "1.8"}],
"version": "0.0.1"}]}
resp = self.server.get("/api/v0/blueprints/info/http-server")
data = json.loads(resp.data)
@ -109,10 +109,10 @@ class ServerTestCase(unittest.TestCase):
{"changed":False, "name":"http-server"}],
"errors":[],
"blueprints":[{"description": "An example GlusterFS server with samba",
"modules":[{"name":"glusterfs", "version":"3.7.*"},
{"name":"glusterfs-cli", "version":"3.7.*"}],
"modules":[{"name":"glusterfs", "version":"3.8.*"},
{"name":"glusterfs-cli", "version":"3.8.*"}],
"name":"glusterfs",
"packages":[{"name":"samba", "version":"4.2.*"}],
"packages":[{"name":"samba", "version":"4.7.*"}],
"version": "0.0.1"},
{"description":"An example http server with PHP and MySQL support.",
"modules":[{"name":"httpd", "version":"2.4.*"},
@ -121,9 +121,9 @@ class ServerTestCase(unittest.TestCase):
{"name":"php", "version":"5.4.*"},
{"name": "php-mysql", "version":"5.4.*"}],
"name":"http-server",
"packages": [{"name":"openssh-server", "version": "6.6.*"},
{"name": "rsync", "version": "3.0.*"},
{"name": "tmux", "version": "2.2"}],
"packages": [{"name":"openssh-server", "version": "7.*"},
{"name": "rsync", "version": "3.1.*"},
{"name": "tmux", "version": "1.8"}],
"version": "0.0.1"},
]}
resp = self.server.get("/api/v0/blueprints/info/http-server,glusterfs")
@ -164,10 +164,10 @@ class ServerTestCase(unittest.TestCase):
test_blueprint = {"description": "An example GlusterFS server with samba",
"name":"glusterfs",
"version": "0.2.0",
"modules":[{"name":"glusterfs", "version":"3.7.*"},
{"name":"glusterfs-cli", "version":"3.7.*"}],
"packages":[{"name":"samba", "version":"4.2.*"},
{"name":"tmux", "version":"2.2"}]}
"modules":[{"name":"glusterfs", "version":"3.8.*"},
{"name":"glusterfs-cli", "version":"3.8.*"}],
"packages":[{"name":"samba", "version":"4.7.*"},
{"name":"tmux", "version":"1.8"}]}
resp = self.server.post("/api/v0/blueprints/new",
data=json.dumps(test_blueprint),
@ -208,10 +208,10 @@ class ServerTestCase(unittest.TestCase):
test_blueprint = {"description": "An example GlusterFS server with samba, ws version",
"name":"glusterfs",
"version": "0.3.0",
"modules":[{"name":"glusterfs", "version":"3.7.*"},
{"name":"glusterfs-cli", "version":"3.7.*"}],
"packages":[{"name":"samba", "version":"4.2.*"},
{"name":"tmux", "version":"2.2"}]}
"modules":[{"name":"glusterfs", "version":"3.8.*"},
{"name":"glusterfs-cli", "version":"3.8.*"}],
"packages":[{"name":"samba", "version":"4.7.*"},
{"name":"tmux", "version":"1.8"}]}
resp = self.server.post("/api/v0/blueprints/workspace",
data=json.dumps(test_blueprint),
@ -234,10 +234,10 @@ class ServerTestCase(unittest.TestCase):
test_blueprint = {"description": "An example GlusterFS server with samba, ws version",
"name":"glusterfs",
"version": "0.4.0",
"modules":[{"name":"glusterfs", "version":"3.7.*"},
{"name":"glusterfs-cli", "version":"3.7.*"}],
"packages":[{"name":"samba", "version":"4.2.*"},
{"name":"tmux", "version":"2.2"}]}
"modules":[{"name":"glusterfs", "version":"3.8.*"},
{"name":"glusterfs-cli", "version":"3.8.*"}],
"packages":[{"name":"samba", "version":"4.7.*"},
{"name":"tmux", "version":"1.8"}]}
resp = self.server.post("/api/v0/blueprints/workspace",
data=json.dumps(test_blueprint),
@ -361,10 +361,10 @@ class ServerTestCase(unittest.TestCase):
test_blueprint = {"description": "An example GlusterFS server with samba, ws version",
"name":"glusterfs",
"version": "0.3.0",
"modules":[{"name":"glusterfs", "version":"3.7.*"},
{"name":"glusterfs-cli", "version":"3.7.*"}],
"packages":[{"name":"samba", "version":"4.2.*"},
{"name":"tmux", "version":"2.2"}]}
"modules":[{"name":"glusterfs", "version":"3.8.*"},
{"name":"glusterfs-cli", "version":"3.8.*"}],
"packages":[{"name":"samba", "version":"4.7.*"},
{"name":"tmux", "version":"1.8"}]}
resp = self.server.post("/api/v0/blueprints/workspace",
data=json.dumps(test_blueprint),
@ -380,7 +380,7 @@ class ServerTestCase(unittest.TestCase):
"old": {"Description": "An example GlusterFS server with samba"}},
{"new": {"Version": "0.3.0"},
"old": {"Version": "0.0.1"}},
{"new": {"Package": {"version": "2.2", "name": "tmux"}},
{"new": {"Package": {"version": "1.8", "name": "tmux"}},
"old": None}]}
self.assertEqual(data, result)
@ -392,6 +392,7 @@ class ServerTestCase(unittest.TestCase):
blueprints = data.get("blueprints")
self.assertNotEqual(blueprints, None)
self.assertEqual(len(blueprints), 1)
self.assertTrue(len(blueprints[0]["blueprint"]["modules"]) > 0)
self.assertEqual(blueprints[0]["blueprint"]["name"], "glusterfs")
self.assertEqual(len(blueprints[0]["dependencies"]) > 10, True)
self.assertFalse(data.get("errors"))
@ -427,6 +428,7 @@ class ServerTestCase(unittest.TestCase):
blueprints = data.get("blueprints")
self.assertNotEqual(blueprints, None)
self.assertEqual(len(blueprints), 1)
self.assertTrue(len(blueprints[0]["blueprint"]["modules"]) > 0)
self.assertEqual(blueprints[0]["blueprint"]["name"], "glusterfs")
evra = blueprints[0]["blueprint"]["modules"][0]["version"]
self.assertEqual(len(evra) > 10, True)
@ -446,6 +448,7 @@ class ServerTestCase(unittest.TestCase):
self.assertNotEqual(data, None)
projects = data.get("projects")
self.assertEqual(len(projects), 1)
self.assertTrue(len(projects[0]["builds"]) > 0)
self.assertEqual(projects[0]["name"], "bash")
self.assertEqual(projects[0]["builds"][0]["source"]["license"], "GPLv3+")
@ -456,7 +459,7 @@ class ServerTestCase(unittest.TestCase):
self.assertNotEqual(data, None)
deps = data.get("projects")
self.assertEqual(len(deps) > 10, True)
self.assertEqual(deps[0]["name"], "basesystem")
self.assertEqual(deps[2]["name"], "basesystem")
def test_modules_list(self):
"""Test /api/v0/modules/list"""
@ -482,17 +485,17 @@ class ServerTestCase(unittest.TestCase):
modules = data.get("modules")
self.assertEqual(len(modules), 1)
self.assertEqual(modules[0]["name"], "bash")
self.assertEqual(modules[0]["dependencies"][0]["name"], "basesystem")
self.assertEqual(modules[0]["dependencies"][2]["name"], "basesystem")
def test_blueprint_new_branch(self):
"""Test the /api/v0/blueprints/new route with a new branch"""
test_blueprint = {"description": "An example GlusterFS server with samba",
"name":"glusterfs",
"version": "0.2.0",
"modules":[{"name":"glusterfs", "version":"3.7.*"},
{"name":"glusterfs-cli", "version":"3.7.*"}],
"packages":[{"name":"samba", "version":"4.2.*"},
{"name":"tmux", "version":"2.2"}]}
"modules":[{"name":"glusterfs", "version":"3.8.*"},
{"name":"glusterfs-cli", "version":"3.8.*"}],
"packages":[{"name":"samba", "version":"4.7.*"},
{"name":"tmux", "version":"1.8"}]}
resp = self.server.post("/api/v0/blueprints/new?branch=test",
data=json.dumps(test_blueprint),