Check the compose templates at startup

Depsolve the packages included in the templates and report any errors
using the /api/status 'msgs' field. This should help narrow down
problems with package sources not being setup correctly.
This commit is contained in:
Brian C. Lane 2018-07-12 15:26:21 -07:00
parent 9e41052992
commit 460a277d4e
5 changed files with 59 additions and 8 deletions

View File

@ -46,13 +46,42 @@ from pyanaconda.simpleconfig import SimpleConfigFile
from pykickstart.parser import KickstartParser from pykickstart.parser import KickstartParser
from pykickstart.version import makeVersion from pykickstart.version import makeVersion
from pylorax.api.projects import projects_depsolve_with_size, dep_nevra from pylorax.api.projects import projects_depsolve, projects_depsolve_with_size, dep_nevra
from pylorax.api.projects import ProjectsError from pylorax.api.projects import ProjectsError
from pylorax.api.recipes import read_recipe_and_id from pylorax.api.recipes import read_recipe_and_id
from pylorax.imgutils import default_image_name from pylorax.imgutils import default_image_name
from pylorax.sysutils import joinpaths from pylorax.sysutils import joinpaths
def test_templates(dbo, share_dir):
""" Try depsolving each of the the templates and report any errors
:param dbo: dnf base object
:type dbo: dnf.Base
:returns: List of template types and errors
:rtype: List of errors
Return a list of templates and errors encountered or an empty list
"""
template_errors = []
for compose_type in compose_types(share_dir):
# Read the kickstart template for this type
ks_template_path = joinpaths(share_dir, "composer", compose_type) + ".ks"
ks_template = open(ks_template_path, "r").read()
# How much space will the packages in the default template take?
ks_version = makeVersion()
ks = KickstartParser(ks_version, errorsAreFatal=False, missingIncludeIsFatal=False)
ks.readKickstartFromString(ks_template+"\n%end\n")
pkgs = [(name, "*") for name in ks.handler.packages.packageList]
try:
_ = projects_depsolve(dbo, pkgs)
except ProjectsError as e:
template_errors.append("Error depsolving %s: %s" % (compose_type, str(e)))
return template_errors
def repo_to_ks(r, url="url"): def repo_to_ks(r, url="url"):
""" Return a kickstart line with the correct args. """ Return a kickstart line with the correct args.
:param r: DNF repository information :param r: DNF repository information

View File

@ -195,18 +195,23 @@ def _depsolve(dbo, projects):
""" """
# This resets the transaction # This resets the transaction
dbo.reset(goal=True) dbo.reset(goal=True)
install_errors = []
for name, version in projects: for name, version in projects:
try: try:
if not version: if not version:
version = "*" version = "*"
pkgs = [pkg for pkg in dnf.subject.Subject(name).get_best_query(dbo.sack).filter(version__glob=version, latest=True)] pkgs = [pkg for pkg in dnf.subject.Subject(name).get_best_query(dbo.sack).filter(version__glob=version, latest=True)]
if not pkgs: if not pkgs:
raise ProjectsError("No match for %s-%s" % (name, version)) install_errors.append(("%s-%s" % (name, version), "No match"))
continue
for p in pkgs: for p in pkgs:
dbo.package_install(p) dbo.package_install(p)
except dnf.exceptions.MarkingError: except dnf.exceptions.MarkingError as e:
raise ProjectsError("No match for %s-%s" % (name, version)) install_errors.append(("%s-%s" % (name, version), str(e)))
if install_errors:
raise ProjectsError("The following package(s) had problems: %s" % ",".join(["%s (%s)" % (pattern, err) for pattern, err in install_errors]))
def projects_depsolve(dbo, projects): def projects_depsolve(dbo, projects):

View File

@ -65,13 +65,19 @@ def v0_status():
"db_supported": true, "db_supported": true,
"db_version": "0", "db_version": "0",
"schema_version": "0", "schema_version": "0",
"backend": "lorax-composer"} "backend": "lorax-composer",
"msgs": []}
The 'msgs' field can be a list of strings describing startup problems or status that
should be displayed to the user. eg. if the compose templates are not depsolving properly
the errors will be in 'msgs'.
""" """
return jsonify(backend="lorax-composer", return jsonify(backend="lorax-composer",
build=vernum, build=vernum,
api="0", api="0",
db_version="0", db_version="0",
schema_version="0", schema_version="0",
db_supported=True) db_supported=True,
msgs=server.config["TEMPLATE_ERRORS"])
v0_api(server) v0_api(server)

View File

@ -38,6 +38,7 @@ from gevent.pywsgi import WSGIServer
from pylorax import vernum from pylorax import vernum
from pylorax.api.cmdline import lorax_composer_parser from pylorax.api.cmdline import lorax_composer_parser
from pylorax.api.config import configure, make_dnf_dirs, make_queue_dirs from pylorax.api.config import configure, make_dnf_dirs, make_queue_dirs
from pylorax.api.compose import test_templates
from pylorax.api.queue import start_queue_monitor from pylorax.api.queue import start_queue_monitor
from pylorax.api.recipes import open_or_create_repo, commit_recipe_directory from pylorax.api.recipes import open_or_create_repo, commit_recipe_directory
from pylorax.api.server import server, GitLock, DNFLock from pylorax.api.server import server, GitLock, DNFLock
@ -247,6 +248,9 @@ if __name__ == '__main__':
dbo = get_base_object(server.config["COMPOSER_CFG"]) dbo = get_base_object(server.config["COMPOSER_CFG"])
server.config["DNFLOCK"] = DNFLock(dbo=dbo, lock=Lock()) server.config["DNFLOCK"] = DNFLock(dbo=dbo, lock=Lock())
# Depsolve the templates and make a note of the failures for /api/status to report
server.config["TEMPLATE_ERRORS"] = test_templates(dbo, server.config["COMPOSER_CFG"].get("composer", "share_dir"))
# Setup access to the git repo # Setup access to the git repo
server.config["REPO_DIR"] = opts.BLUEPRINTS server.config["REPO_DIR"] = opts.BLUEPRINTS
repo = open_or_create_repo(server.config["REPO_DIR"]) repo = open_or_create_repo(server.config["REPO_DIR"])

View File

@ -68,6 +68,9 @@ class ServerTestCase(unittest.TestCase):
dbo = get_base_object(server.config["COMPOSER_CFG"]) dbo = get_base_object(server.config["COMPOSER_CFG"])
server.config["DNFLOCK"] = DNFLock(dbo=dbo, lock=Lock()) server.config["DNFLOCK"] = DNFLock(dbo=dbo, lock=Lock())
# Include a message in /api/status output
server.config["TEMPLATE_ERRORS"] = ["Test message"]
server.config['TESTING'] = True server.config['TESTING'] = True
self.server = server.test_client() self.server = server.test_client()
self.repo_dir = repo_dir self.repo_dir = repo_dir
@ -91,12 +94,16 @@ class ServerTestCase(unittest.TestCase):
def test_01_status(self): def test_01_status(self):
"""Test the /api/status route""" """Test the /api/status route"""
status_fields = ["build", "api", "db_version", "schema_version", "db_supported", "backend"] status_fields = ["build", "api", "db_version", "schema_version", "db_supported", "backend", "msgs"]
resp = self.server.get("/api/status") resp = self.server.get("/api/status")
data = json.loads(resp.data) data = json.loads(resp.data)
# Just make sure the fields are present # Make sure the fields are present
self.assertEqual(sorted(data.keys()), sorted(status_fields)) self.assertEqual(sorted(data.keys()), sorted(status_fields))
# Check for test message
self.assertEqual(data["msgs"], ["Test message"])
def test_02_blueprints_list(self): def test_02_blueprints_list(self):
"""Test the /api/v0/blueprints/list route""" """Test the /api/v0/blueprints/list route"""
list_dict = {"blueprints":["atlas", "custom-base", "development", "glusterfs", "http-server", list_dict = {"blueprints":["atlas", "custom-base", "development", "glusterfs", "http-server",