Check the compose templates at startup

Depsolve the packages included in the templates and report any errors
using the /api/status 'msgs' field. This should help narrow down
problems with package sources not being setup correctly.

src/pylorax/api/compose.py

@@ -46,13 +46,42 @@ from pyanaconda.simpleconfig import SimpleConfigFile
 from pykickstart.parser import KickstartParser
 from pykickstart.version import makeVersion
 
-from pylorax.api.projects import projects_depsolve_with_size, dep_nevra
+from pylorax.api.projects import projects_depsolve, projects_depsolve_with_size, dep_nevra
 from pylorax.api.projects import ProjectsError
 from pylorax.api.recipes import read_recipe_and_id
 from pylorax.imgutils import default_image_name
 from pylorax.sysutils import joinpaths
 
 
+def test_templates(dbo, share_dir):
+    """ Try depsolving each of the the templates and report any errors
+
+    :param dbo: dnf base object
+    :type dbo: dnf.Base
+    :returns: List of template types and errors
+    :rtype: List of errors
+
+    Return a list of templates and errors encountered or an empty list
+    """
+    template_errors = []
+    for compose_type in compose_types(share_dir):
+        # Read the kickstart template for this type
+        ks_template_path = joinpaths(share_dir, "composer", compose_type) + ".ks"
+        ks_template = open(ks_template_path, "r").read()
+
+        # How much space will the packages in the default template take?
+        ks_version = makeVersion()
+        ks = KickstartParser(ks_version, errorsAreFatal=False, missingIncludeIsFatal=False)
+        ks.readKickstartFromString(ks_template+"\n%end\n")
+        pkgs = [(name, "*") for name in ks.handler.packages.packageList]
+        try:
+            _ = projects_depsolve(dbo, pkgs)
+        except ProjectsError as e:
+            template_errors.append("Error depsolving %s: %s" % (compose_type, str(e)))
+
+    return template_errors
+
+
 def repo_to_ks(r, url="url"):
     """ Return a kickstart line with the correct args.
     :param r: DNF repository information

src/pylorax/api/projects.py

@@ -195,18 +195,23 @@ def _depsolve(dbo, projects):
     """
     # This resets the transaction
     dbo.reset(goal=True)
+    install_errors = []
     for name, version in projects:
         try:
             if not version:
                 version = "*"
             pkgs = [pkg for pkg in dnf.subject.Subject(name).get_best_query(dbo.sack).filter(version__glob=version, latest=True)]
             if not pkgs:
-                raise ProjectsError("No match for %s-%s" % (name, version))
+                install_errors.append(("%s-%s" % (name, version), "No match"))
+                continue
 
             for p in pkgs:
                 dbo.package_install(p)
-        except dnf.exceptions.MarkingError:
+        except dnf.exceptions.MarkingError as e:
-            raise ProjectsError("No match for %s-%s" % (name, version))
+            install_errors.append(("%s-%s" % (name, version), str(e)))
+
+    if install_errors:
+        raise ProjectsError("The following package(s) had problems: %s" % ",".join(["%s (%s)" % (pattern, err) for pattern, err in install_errors]))
 
 
 def projects_depsolve(dbo, projects):

src/pylorax/api/server.py

@@ -65,13 +65,19 @@ def v0_status():
             "db_supported": true,
             "db_version": "0",
             "schema_version": "0",
-            "backend": "lorax-composer"}
+            "backend": "lorax-composer",
+            "msgs": []}
+
+    The 'msgs' field can be a list of strings describing startup problems or status that
+    should be displayed to the user. eg. if the compose templates are not depsolving properly
+    the errors will be in 'msgs'.
     """
     return jsonify(backend="lorax-composer",
                    build=vernum,
                    api="0",
                    db_version="0",
                    schema_version="0",
-                   db_supported=True)
+                   db_supported=True,
+                   msgs=server.config["TEMPLATE_ERRORS"])
 
 v0_api(server)

src/sbin/lorax-composer

@@ -38,6 +38,7 @@ from gevent.pywsgi import WSGIServer
 from pylorax import vernum
 from pylorax.api.cmdline import lorax_composer_parser
 from pylorax.api.config import configure, make_dnf_dirs, make_queue_dirs
+from pylorax.api.compose import test_templates
 from pylorax.api.queue import start_queue_monitor
 from pylorax.api.recipes import open_or_create_repo, commit_recipe_directory
 from pylorax.api.server import server, GitLock, DNFLock
@@ -247,6 +248,9 @@ if __name__ == '__main__':
     dbo = get_base_object(server.config["COMPOSER_CFG"])
     server.config["DNFLOCK"] = DNFLock(dbo=dbo, lock=Lock())
 
+    # Depsolve the templates and make a note of the failures for /api/status to report
+    server.config["TEMPLATE_ERRORS"] = test_templates(dbo, server.config["COMPOSER_CFG"].get("composer", "share_dir"))
+
     # Setup access to the git repo
     server.config["REPO_DIR"] = opts.BLUEPRINTS
     repo = open_or_create_repo(server.config["REPO_DIR"])

tests/pylorax/test_server.py

@@ -68,6 +68,9 @@ class ServerTestCase(unittest.TestCase):
         dbo = get_base_object(server.config["COMPOSER_CFG"])
         server.config["DNFLOCK"] = DNFLock(dbo=dbo, lock=Lock())
 
+        # Include a message in /api/status output
+        server.config["TEMPLATE_ERRORS"] = ["Test message"]
+
         server.config['TESTING'] = True
         self.server = server.test_client()
         self.repo_dir = repo_dir
@@ -91,12 +94,16 @@ class ServerTestCase(unittest.TestCase):
 
     def test_01_status(self):
         """Test the /api/status route"""
-        status_fields = ["build", "api", "db_version", "schema_version", "db_supported", "backend"]
+        status_fields = ["build", "api", "db_version", "schema_version", "db_supported", "backend", "msgs"]
         resp = self.server.get("/api/status")
         data = json.loads(resp.data)
-        # Just make sure the fields are present
+        # Make sure the fields are present
         self.assertEqual(sorted(data.keys()), sorted(status_fields))
 
+        # Check for test message
+        self.assertEqual(data["msgs"], ["Test message"])
+
+
     def test_02_blueprints_list(self):
         """Test the /api/v0/blueprints/list route"""
         list_dict = {"blueprints":["atlas", "custom-base", "development", "glusterfs", "http-server",