From 67da4d6971af9c34fdba0e4ec3ecf80ff8b43435 Mon Sep 17 00:00:00 2001 From: "Brian C. Lane" Date: Mon, 29 Jan 2018 17:08:30 -0800 Subject: [PATCH] Add building an image, and the /compose route to start it This adds the ability to build a tar output image. The /compose and /compose/types API routes are now available. To start a build POST a JSON body to /compose, like this: {"recipe_name":"glusterfs", "compose_type":"tar", "branch":"master"} This will return a unique build id: { "build_id": "4d13abb6-aa4e-4c80-a671-0b867e6e77f6", "status": true } which will be used to keep track of the build status (routes for this do not exist yet). --- share/composer/tar.ks | 49 ++++++++++ src/pylorax/api/compose.py | 193 +++++++++++++++++++++++++++++++++++++ src/pylorax/api/queue.py | 6 +- src/pylorax/api/v0.py | 76 +++++++++++---- 4 files changed, 303 insertions(+), 21 deletions(-) create mode 100644 share/composer/tar.ks create mode 100644 src/pylorax/api/compose.py diff --git a/share/composer/tar.ks b/share/composer/tar.ks new file mode 100644 index 00000000..e40a019d --- /dev/null +++ b/share/composer/tar.ks @@ -0,0 +1,49 @@ +# Lorax Composer tar output kickstart template + +# +sshpw --username=root --plaintext randOmStrinGhERE +# Firewall configuration +firewall --enabled + +# Root password +rootpw --plaintext removethispw +# Network information +network --bootproto=dhcp --onboot=on --activate +# System authorization information +auth --useshadow --enablemd5 +# System keyboard +keyboard --xlayouts=us --vckeymap=us +# System language +lang en_US.UTF-8 +# SELinux configuration +selinux --enforcing +# Installation logging level +logging --level=info +# Shutdown after installation +shutdown +# System timezone +timezone US/Eastern +# System bootloader configuration +bootloader --location=mbr +# Clear the Master Boot Record +zerombr +# Partition clearing information +clearpart --all +# Disk partitioning information +part / --fstype="ext4" --size=4000 +part swap --size=1000 + +%post +# Remove root password +passwd -d root > /dev/null + +# Remove random-seed +rm /var/lib/systemd/random-seed +%end + +# NOTE Do NOT add any other sections after %packages +%packages +# Packages requires to support this output format go here + + +# NOTE lorax-composer will add the recipe packages below here, including the final %end diff --git a/src/pylorax/api/compose.py b/src/pylorax/api/compose.py new file mode 100644 index 00000000..e14b9422 --- /dev/null +++ b/src/pylorax/api/compose.py @@ -0,0 +1,193 @@ +# Copyright (C) 2018 Red Hat, Inc. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +""" Setup for composing an image + +Adding New Output Types +----------------------- + +The new output type must add a kickstart template to ./share/composer/ where the +name of the kickstart (without the trailing .ks) matches the entry in compose_args. + +The kickstart should not have any url or repo entries, these will be added at build +time. The %packages section should be the last thing, and while it can contain mandatory +packages required by the output type, it should not have the trailing %end because the +package NEVRAs will be appended to it at build time. + +compose_args should have a name matching the kickstart, and it should set the novirt_install +parameters needed to generate the desired output. Other types should be set to False. + +""" +import logging +log = logging.getLogger("lorax-composer") + +import os +from glob import glob +import pytoml as toml +import shutil +from uuid import uuid4 + +from pylorax.api.projects import projects_depsolve, dep_nevra +from pylorax.api.projects import ProjectsError +from pylorax.imgutils import default_image_name +from pylorax.sysutils import joinpaths + + +def repo_to_ks(r, url="url"): + """ Return a kickstart line with the correct args. + + Set url to "baseurl" if it is a repo, leave it as "url" for the installation url. + """ + cmd = "" + if r.metalink: + # XXX Total Hack + # RHEL7 kickstart doesn't support metalink. If the url has 'metalink' in it, rewrite it as 'mirrorlist' + if "metalink" in r.metalink: + log.info("RHEL7 does not support metalink, translating to mirrorlist") + cmd += '--mirrorlist="%s" ' % r.metalink.replace("metalink", "mirrorlist") + else: + log.error("Could not convert metalink to mirrorlist. %s", r.metalink) + raise RuntimeError("Cannot convert metalink to mirrorlist: %s" % r.metalink) + elif r.mirrorlist: + cmd += '--mirrorlist="%s" ' % r.mirrorlist + elif r.baseurl: + cmd += '--%s="%s" ' % (url, r.baseurl[0]) + else: + raise RuntimeError("Repo has no baseurl or mirror") + + if r.proxy: + cmd += '--proxy="%s" ' % r.proxy + + if not r.sslverify: + cmd += '--noverifyssl' + + return cmd + +def start_build(cfg, yumlock, recipe, compose_type): + """ Start the build + + :param cfg: Configuration object + :type cfg: ComposerConfig + :param yumlock: Lock and YumBase for depsolving + :type yumlock: YumLock + :param recipe: The recipe to build + :type recipe: str + :param compose_type: The type of output to create from the recipe + :type compose_type: str + :returns: Unique ID for the build that can be used to track its status + :rtype: str + """ + share_dir = cfg.get("composer", "share_dir") + lib_dir = cfg.get("composer", "lib_dir") + + # Make sure compose_type is valid + if compose_type not in compose_types(share_dir): + raise RuntimeError("Invalid compose type (%s), must be one of %s" % (compose_type, compose_types(share_dir))) + + # Combine modules and packages and depsolve the list + # TODO include the version/glob in the depsolving + module_names = map(lambda m: m["name"], recipe["modules"] or []) + package_names = map(lambda p: p["name"], recipe["packages"] or []) + projects = sorted(set(module_names+package_names), key=lambda n: n.lower()) + deps = [] + try: + with yumlock.lock: + deps = projects_depsolve(yumlock.yb, projects) + except ProjectsError as e: + log.error("start_build depsolve: %s", str(e)) + raise RuntimeError("Problem depsolving %s: %s" % (recipe["name"], str(e))) + + # Create the results directory + build_id = str(uuid4()) + results_dir = joinpaths(lib_dir, "results", build_id) + os.makedirs(results_dir) + + # Read the kickstart template for this type and copy it into the results + ks_template_path = joinpaths(share_dir, "composer", compose_type) + ".ks" + shutil.copy(ks_template_path, results_dir) + ks_template = open(ks_template_path, "r").read() + + # Write out the dependencies to the results dir + deps_path = joinpaths(results_dir, "deps.txt") + with open(deps_path, "w") as f: + for d in deps: + f.write(dep_nevra(d)+"\n") + + # Create the final kickstart with repos and package list + ks_path = joinpaths(results_dir, "final-kickstart.ks") + with open(ks_path, "w") as f: + with yumlock.lock: + repos = yumlock.yb.repos.listEnabled() + if not repos: + raise RuntimeError("No enabled repos, canceling build.") + + ks_url = repo_to_ks(repos[0], "url") + log.debug("url = %s", ks_url) + f.write('url %s\n' % ks_url) + for idx, r in enumerate(repos[1:]): + ks_repo = repo_to_ks(r, "baseurl") + log.debug("repo composer-%s = %s", idx, ks_repo) + f.write('repo --name="composer-%s" %s\n' % (idx, ks_repo)) + + f.write(ks_template) + + for d in deps: + f.write(dep_nevra(d)+"\n") + + f.write("%end\n") + + # Setup the config to pass to novirt_install + log_dir = joinpaths(results_dir, "logs/") + cfg_args = compose_args(compose_type) + cfg_args.update({ + "compression": "xz", + #"compress_args": ["-9"], + "compress_args": [], + "ks": [ks_path], + "anaconda_args": "", + "proxy": "", + "armplatform": "", + + "project": "Red Hat Enterprise Linux", + "releasever": "7", + + "logfile": log_dir + }) + with open(joinpaths(results_dir, "config.toml"), "w") as f: + f.write(toml.dumps(cfg_args).encode("UTF-8")) + + log.info("Starting compose %s with recipe %s output type %s", build_id, recipe["name"], compose_type) + os.symlink(results_dir, joinpaths(lib_dir, "queue/new/", build_id)) + + return build_id + +# Supported output types +def compose_types(share_dir): + """ Returns a list of the supported output types + + The output types come from the kickstart names in /usr/share/lorax/composer/*ks + """ + return [os.path.basename(ks)[:-3] for ks in glob(joinpaths(share_dir, "composer/*.ks"))] + +def compose_args(compose_type): + """ Returns the settings to pass to novirt_install for the compose type""" + _MAP = {"tar": {"make_tar": True, + "make_iso": False, + "make_fsimage": False, + "qcow2": False, + "image_name": default_image_name("xz", "root.tar")}, + } + + return _MAP[compose_type] diff --git a/src/pylorax/api/queue.py b/src/pylorax/api/queue.py index a75f7a52..ac938767 100644 --- a/src/pylorax/api/queue.py +++ b/src/pylorax/api/queue.py @@ -56,7 +56,6 @@ def monitor(cfg, cancel_q): while True: jobs = sorted(os.listdir(joinpaths(cfg.composer_dir, "queue/new")), key=queue_sort) - log.debug("jobs = %s", jobs) # Pick the oldest and move it into ./run/ if not jobs: @@ -97,11 +96,10 @@ def make_compose(cfg, results_dir): repo_url = ks.handler.method.url # Load the compose configuration - cfg_file = joinpaths(results_dir, "config.toml") + cfg_path = joinpaths(results_dir, "config.toml") if not os.path.exists(cfg_path): raise RuntimeError("Missing config.toml for %s" % results_dir) - cfg_dict = toml.loads(open(cfg_file, "r").read()) - cfg_dict["logfile"] = log_dict + cfg_dict = toml.loads(open(cfg_path, "r").read()) install_cfg = DataHolder(**cfg_dict) diff --git a/src/pylorax/api/v0.py b/src/pylorax/api/v0.py index 80834f4c..74972164 100644 --- a/src/pylorax/api/v0.py +++ b/src/pylorax/api/v0.py @@ -618,10 +618,7 @@ log = logging.getLogger("lorax-composer") from flask import jsonify, request -# Use pykickstart to calculate disk image size -from pykickstart.parser import KickstartParser -from pykickstart.version import makeVersion, RHEL7 - +from pylorax.api.compose import start_build, compose_types from pylorax.api.crossdomain import crossdomain from pylorax.api.projects import projects_list, projects_info, projects_depsolve, dep_evra from pylorax.api.projects import modules_list, modules_info, ProjectsError @@ -629,23 +626,10 @@ from pylorax.api.recipes import list_branch_files, read_recipe_commit, recipe_fi from pylorax.api.recipes import recipe_from_dict, recipe_from_toml, commit_recipe, delete_recipe, revert_recipe from pylorax.api.recipes import tag_recipe_commit, recipe_diff, Recipe, RecipePackage, RecipeModule from pylorax.api.workspace import workspace_read, workspace_write, workspace_delete -from pylorax.creator import DRACUT_DEFAULT, mount_boot_part_over_root -from pylorax.creator import make_appliance, make_image, make_livecd, make_live_images -from pylorax.creator import make_runtime, make_squashfs -from pylorax.imgutils import copytree -from pylorax.imgutils import Mount, PartitionMount, umount -from pylorax.installer import InstallError -from pylorax.sysutils import joinpaths # The API functions don't actually get called by any code here # pylint: disable=unused-variable -# no-virt mode doesn't need libvirt, so make it optional -try: - import libvirt -except ImportError: - libvirt = None - def take_limits(iterable, offset, limit): """ Apply offset and limit to an iterable object @@ -1099,3 +1083,61 @@ def v0_api(api): return jsonify(error={"msg":str(e)}), 400 return jsonify(modules=modules) + + @api.route("/api/v0/compose", methods=["POST"]) + @crossdomain(origin="*") + def v0_compose_start(): + """Start a compose + + The body of the post should have these fields: + recipe_name - The recipe name from /recipes/list/ + compose_type - The type of output to create, from /compose/types + branch - Optional, defaults to master, selects the git branch to use for the recipe. + """ + compose = request.get_json(cache=False) + + errors = [] + if not compose: + return jsonify(status=False, error={"msg":"Missing POST body"}), 400 + + if "recipe_name" not in compose: + errors.append("No 'recipe_name' in the JSON request") + else: + recipe_name = compose["recipe_name"] + + if "branch" not in compose or not compose["branch"]: + branch = "master" + else: + branch = compose["branch"] + + if "compose_type" not in compose: + errors.append("No 'compose_type' in the JSON request") + else: + compose_type = compose["compose_type"] + + if errors: + return jsonify(status=False, error={"msg":"\n".join(errors)}), 400 + + # Get the git version (if it exists) + try: + with api.config["GITLOCK"].lock: + recipe = read_recipe_commit(api.config["GITLOCK"].repo, branch, recipe_name) + except Exception as e: + log.error("Problem reading recipe %s: %s", recipe_name, str(e)) + return jsonify(status=False, error={"msg":str(e)}), 400 + try: + build_id = start_build(api.config["COMPOSER_CFG"], api.config["YUMLOCK"], recipe, compose_type) + except Exception as e: + return jsonify(status=False, error={"msg":str(e)}), 400 + + return jsonify(status=True, build_id=build_id) + + @api.route("/api/v0/compose/types") + @crossdomain(origin="*") + def v0_compose_types(): + """Return the list of enabled output types + + (only enabled types are returned) + """ + share_dir = api.config["COMPOSER_CFG"].get("composer", "share_dir") + return jsonify(types=[{"name": k, "enabled": True} for k in compose_types(share_dir)])