From 460a277d4e451d7ce2a4fe1bb8e7972c2a19cc53 Mon Sep 17 00:00:00 2001 From: "Brian C. Lane" Date: Thu, 12 Jul 2018 15:26:21 -0700 Subject: [PATCH] Check the compose templates at startup Depsolve the packages included in the templates and report any errors using the /api/status 'msgs' field. This should help narrow down problems with package sources not being setup correctly. --- src/pylorax/api/compose.py | 31 ++++++++++++++++++++++++++++++- src/pylorax/api/projects.py | 11 ++++++++--- src/pylorax/api/server.py | 10 ++++++++-- src/sbin/lorax-composer | 4 ++++ tests/pylorax/test_server.py | 11 +++++++++-- 5 files changed, 59 insertions(+), 8 deletions(-) diff --git a/src/pylorax/api/compose.py b/src/pylorax/api/compose.py index b9824149..08873bf0 100644 --- a/src/pylorax/api/compose.py +++ b/src/pylorax/api/compose.py @@ -46,13 +46,42 @@ from pyanaconda.simpleconfig import SimpleConfigFile from pykickstart.parser import KickstartParser from pykickstart.version import makeVersion -from pylorax.api.projects import projects_depsolve_with_size, dep_nevra +from pylorax.api.projects import projects_depsolve, projects_depsolve_with_size, dep_nevra from pylorax.api.projects import ProjectsError from pylorax.api.recipes import read_recipe_and_id from pylorax.imgutils import default_image_name from pylorax.sysutils import joinpaths +def test_templates(dbo, share_dir): + """ Try depsolving each of the the templates and report any errors + + :param dbo: dnf base object + :type dbo: dnf.Base + :returns: List of template types and errors + :rtype: List of errors + + Return a list of templates and errors encountered or an empty list + """ + template_errors = [] + for compose_type in compose_types(share_dir): + # Read the kickstart template for this type + ks_template_path = joinpaths(share_dir, "composer", compose_type) + ".ks" + ks_template = open(ks_template_path, "r").read() + + # How much space will the packages in the default template take? + ks_version = makeVersion() + ks = KickstartParser(ks_version, errorsAreFatal=False, missingIncludeIsFatal=False) + ks.readKickstartFromString(ks_template+"\n%end\n") + pkgs = [(name, "*") for name in ks.handler.packages.packageList] + try: + _ = projects_depsolve(dbo, pkgs) + except ProjectsError as e: + template_errors.append("Error depsolving %s: %s" % (compose_type, str(e))) + + return template_errors + + def repo_to_ks(r, url="url"): """ Return a kickstart line with the correct args. :param r: DNF repository information diff --git a/src/pylorax/api/projects.py b/src/pylorax/api/projects.py index 0c466487..d84fc642 100644 --- a/src/pylorax/api/projects.py +++ b/src/pylorax/api/projects.py @@ -195,18 +195,23 @@ def _depsolve(dbo, projects): """ # This resets the transaction dbo.reset(goal=True) + install_errors = [] for name, version in projects: try: if not version: version = "*" pkgs = [pkg for pkg in dnf.subject.Subject(name).get_best_query(dbo.sack).filter(version__glob=version, latest=True)] if not pkgs: - raise ProjectsError("No match for %s-%s" % (name, version)) + install_errors.append(("%s-%s" % (name, version), "No match")) + continue for p in pkgs: dbo.package_install(p) - except dnf.exceptions.MarkingError: - raise ProjectsError("No match for %s-%s" % (name, version)) + except dnf.exceptions.MarkingError as e: + install_errors.append(("%s-%s" % (name, version), str(e))) + + if install_errors: + raise ProjectsError("The following package(s) had problems: %s" % ",".join(["%s (%s)" % (pattern, err) for pattern, err in install_errors])) def projects_depsolve(dbo, projects): diff --git a/src/pylorax/api/server.py b/src/pylorax/api/server.py index 9e668330..d0980aed 100644 --- a/src/pylorax/api/server.py +++ b/src/pylorax/api/server.py @@ -65,13 +65,19 @@ def v0_status(): "db_supported": true, "db_version": "0", "schema_version": "0", - "backend": "lorax-composer"} + "backend": "lorax-composer", + "msgs": []} + + The 'msgs' field can be a list of strings describing startup problems or status that + should be displayed to the user. eg. if the compose templates are not depsolving properly + the errors will be in 'msgs'. """ return jsonify(backend="lorax-composer", build=vernum, api="0", db_version="0", schema_version="0", - db_supported=True) + db_supported=True, + msgs=server.config["TEMPLATE_ERRORS"]) v0_api(server) diff --git a/src/sbin/lorax-composer b/src/sbin/lorax-composer index 4e39989a..41cba588 100755 --- a/src/sbin/lorax-composer +++ b/src/sbin/lorax-composer @@ -38,6 +38,7 @@ from gevent.pywsgi import WSGIServer from pylorax import vernum from pylorax.api.cmdline import lorax_composer_parser from pylorax.api.config import configure, make_dnf_dirs, make_queue_dirs +from pylorax.api.compose import test_templates from pylorax.api.queue import start_queue_monitor from pylorax.api.recipes import open_or_create_repo, commit_recipe_directory from pylorax.api.server import server, GitLock, DNFLock @@ -247,6 +248,9 @@ if __name__ == '__main__': dbo = get_base_object(server.config["COMPOSER_CFG"]) server.config["DNFLOCK"] = DNFLock(dbo=dbo, lock=Lock()) + # Depsolve the templates and make a note of the failures for /api/status to report + server.config["TEMPLATE_ERRORS"] = test_templates(dbo, server.config["COMPOSER_CFG"].get("composer", "share_dir")) + # Setup access to the git repo server.config["REPO_DIR"] = opts.BLUEPRINTS repo = open_or_create_repo(server.config["REPO_DIR"]) diff --git a/tests/pylorax/test_server.py b/tests/pylorax/test_server.py index 44c2c8f4..55dd4360 100644 --- a/tests/pylorax/test_server.py +++ b/tests/pylorax/test_server.py @@ -68,6 +68,9 @@ class ServerTestCase(unittest.TestCase): dbo = get_base_object(server.config["COMPOSER_CFG"]) server.config["DNFLOCK"] = DNFLock(dbo=dbo, lock=Lock()) + # Include a message in /api/status output + server.config["TEMPLATE_ERRORS"] = ["Test message"] + server.config['TESTING'] = True self.server = server.test_client() self.repo_dir = repo_dir @@ -91,12 +94,16 @@ class ServerTestCase(unittest.TestCase): def test_01_status(self): """Test the /api/status route""" - status_fields = ["build", "api", "db_version", "schema_version", "db_supported", "backend"] + status_fields = ["build", "api", "db_version", "schema_version", "db_supported", "backend", "msgs"] resp = self.server.get("/api/status") data = json.loads(resp.data) - # Just make sure the fields are present + # Make sure the fields are present self.assertEqual(sorted(data.keys()), sorted(status_fields)) + # Check for test message + self.assertEqual(data["msgs"], ["Test message"]) + + def test_02_blueprints_list(self): """Test the /api/v0/blueprints/list route""" list_dict = {"blueprints":["atlas", "custom-base", "development", "glusterfs", "http-server",