Add building an image, and the /compose route to start it

This adds the ability to build a tar output image. The /compose and
/compose/types API routes are now available.

To start a build POST a JSON body to /compose, like this:

{"recipe_name":"glusterfs", "compose_type":"tar", "branch":"master"}

This will return a unique build id:

{
  "build_id": "4d13abb6-aa4e-4c80-a671-0b867e6e77f6",
  "status": true
}

which will be used to keep track of the build status (routes for this
do not exist yet).

share/composer/tar.ks

@@ -0,0 +1,49 @@
+# Lorax Composer tar output kickstart template
+
+#
+sshpw --username=root --plaintext randOmStrinGhERE
+# Firewall configuration
+firewall --enabled
+
+# Root password
+rootpw --plaintext removethispw
+# Network information
+network  --bootproto=dhcp --onboot=on --activate
+# System authorization information
+auth --useshadow --enablemd5
+# System keyboard
+keyboard --xlayouts=us --vckeymap=us
+# System language
+lang en_US.UTF-8
+# SELinux configuration
+selinux --enforcing
+# Installation logging level
+logging --level=info
+# Shutdown after installation
+shutdown
+# System timezone
+timezone  US/Eastern
+# System bootloader configuration
+bootloader --location=mbr
+# Clear the Master Boot Record
+zerombr
+# Partition clearing information
+clearpart --all
+# Disk partitioning information
+part / --fstype="ext4" --size=4000
+part swap --size=1000
+
+%post
+# Remove root password
+passwd -d root > /dev/null
+
+# Remove random-seed
+rm /var/lib/systemd/random-seed
+%end
+
+# NOTE Do NOT add any other sections after %packages
+%packages
+# Packages requires to support this output format go here
+
+
+# NOTE lorax-composer will add the recipe packages below here, including the final %end

src/pylorax/api/compose.py

@@ -0,0 +1,193 @@
+# Copyright (C) 2018 Red Hat, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+""" Setup for composing an image
+
+Adding New Output Types
+-----------------------
+
+The new output type must add a kickstart template to ./share/composer/ where the
+name of the kickstart (without the trailing .ks) matches the entry in compose_args.
+
+The kickstart should not have any url or repo entries, these will be added at build
+time. The %packages section should be the last thing, and while it can contain mandatory
+packages required by the output type, it should not have the trailing %end because the
+package NEVRAs will be appended to it at build time.
+
+compose_args should have a name matching the kickstart, and it should set the novirt_install
+parameters needed to generate the desired output. Other types should be set to False.
+
+"""
+import logging
+log = logging.getLogger("lorax-composer")
+
+import os
+from glob import glob
+import pytoml as toml
+import shutil
+from uuid import uuid4
+
+from pylorax.api.projects import projects_depsolve, dep_nevra
+from pylorax.api.projects import ProjectsError
+from pylorax.imgutils import default_image_name
+from pylorax.sysutils import joinpaths
+
+
+def repo_to_ks(r, url="url"):
+    """ Return a kickstart line with the correct args.
+
+    Set url to "baseurl" if it is a repo, leave it as "url" for the installation url.
+    """
+    cmd = ""
+    if r.metalink:
+        # XXX Total Hack
+        # RHEL7 kickstart doesn't support metalink. If the url has 'metalink' in it, rewrite it as 'mirrorlist'
+        if "metalink" in r.metalink:
+            log.info("RHEL7 does not support metalink, translating to mirrorlist")
+            cmd += '--mirrorlist="%s" ' % r.metalink.replace("metalink", "mirrorlist")
+        else:
+            log.error("Could not convert metalink to mirrorlist. %s", r.metalink)
+            raise RuntimeError("Cannot convert metalink to mirrorlist: %s" % r.metalink)
+    elif r.mirrorlist:
+        cmd += '--mirrorlist="%s" ' % r.mirrorlist
+    elif r.baseurl:
+        cmd += '--%s="%s" ' % (url, r.baseurl[0])
+    else:
+        raise RuntimeError("Repo has no baseurl or mirror")
+
+    if r.proxy:
+        cmd += '--proxy="%s" ' % r.proxy
+
+    if not r.sslverify:
+        cmd += '--noverifyssl'
+
+    return cmd
+
+def start_build(cfg, yumlock, recipe, compose_type):
+    """ Start the build
+
+    :param cfg: Configuration object
+    :type cfg: ComposerConfig
+    :param yumlock: Lock and YumBase for depsolving
+    :type yumlock: YumLock
+    :param recipe: The recipe to build
+    :type recipe: str
+    :param compose_type: The type of output to create from the recipe
+    :type compose_type: str
+    :returns: Unique ID for the build that can be used to track its status
+    :rtype: str
+    """
+    share_dir = cfg.get("composer", "share_dir")
+    lib_dir = cfg.get("composer", "lib_dir")
+
+    # Make sure compose_type is valid
+    if compose_type not in compose_types(share_dir):
+        raise RuntimeError("Invalid compose type (%s), must be one of %s" % (compose_type, compose_types(share_dir)))
+
+    # Combine modules and packages and depsolve the list
+    # TODO include the version/glob in the depsolving
+    module_names = map(lambda m: m["name"], recipe["modules"] or [])
+    package_names = map(lambda p: p["name"], recipe["packages"] or [])
+    projects = sorted(set(module_names+package_names), key=lambda n: n.lower())
+    deps = []
+    try:
+        with yumlock.lock:
+            deps = projects_depsolve(yumlock.yb, projects)
+    except ProjectsError as e:
+        log.error("start_build depsolve: %s", str(e))
+        raise RuntimeError("Problem depsolving %s: %s" % (recipe["name"], str(e)))
+
+    # Create the results directory
+    build_id = str(uuid4())
+    results_dir = joinpaths(lib_dir, "results", build_id)
+    os.makedirs(results_dir)
+
+    # Read the kickstart template for this type and copy it into the results
+    ks_template_path = joinpaths(share_dir, "composer", compose_type) + ".ks"
+    shutil.copy(ks_template_path, results_dir)
+    ks_template = open(ks_template_path, "r").read()
+
+    # Write out the dependencies to the results dir
+    deps_path = joinpaths(results_dir, "deps.txt")
+    with open(deps_path, "w") as f:
+        for d in deps:
+            f.write(dep_nevra(d)+"\n")
+
+    # Create the final kickstart with repos and package list
+    ks_path = joinpaths(results_dir, "final-kickstart.ks")
+    with open(ks_path, "w") as f:
+        with yumlock.lock:
+            repos = yumlock.yb.repos.listEnabled()
+        if not repos:
+            raise RuntimeError("No enabled repos, canceling build.")
+
+        ks_url = repo_to_ks(repos[0], "url")
+        log.debug("url = %s", ks_url)
+        f.write('url %s\n' % ks_url)
+        for idx, r in enumerate(repos[1:]):
+            ks_repo = repo_to_ks(r, "baseurl")
+            log.debug("repo composer-%s = %s", idx, ks_repo)
+            f.write('repo --name="composer-%s" %s\n' % (idx, ks_repo))
+
+        f.write(ks_template)
+
+        for d in deps:
+            f.write(dep_nevra(d)+"\n")
+
+        f.write("%end\n")
+
+    # Setup the config to pass to novirt_install
+    log_dir = joinpaths(results_dir, "logs/")
+    cfg_args = compose_args(compose_type)
+    cfg_args.update({
+        "compression":      "xz",
+        #"compress_args":    ["-9"],
+        "compress_args":    [],
+        "ks":               [ks_path],
+        "anaconda_args":    "",
+        "proxy":            "",
+        "armplatform":      "",
+
+        "project":          "Red Hat Enterprise Linux",
+        "releasever":       "7",
+
+        "logfile":          log_dir
+    })
+    with open(joinpaths(results_dir, "config.toml"), "w") as f:
+        f.write(toml.dumps(cfg_args).encode("UTF-8"))
+
+    log.info("Starting compose %s with recipe %s output type %s", build_id, recipe["name"], compose_type)
+    os.symlink(results_dir, joinpaths(lib_dir, "queue/new/", build_id))
+
+    return build_id
+
+# Supported output types
+def compose_types(share_dir):
+    """ Returns a list of the supported output types
+
+    The output types come from the kickstart names in /usr/share/lorax/composer/*ks
+    """
+    return [os.path.basename(ks)[:-3] for ks in glob(joinpaths(share_dir, "composer/*.ks"))]
+
+def compose_args(compose_type):
+    """ Returns the settings to pass to novirt_install for the compose type"""
+    _MAP = {"tar": {"make_tar":     True,
+                    "make_iso":     False,
+                    "make_fsimage": False,
+                    "qcow2":        False,
+                    "image_name":   default_image_name("xz", "root.tar")},
+                    }
+
+    return _MAP[compose_type]

src/pylorax/api/queue.py

@@ -56,7 +56,6 @@ def monitor(cfg, cancel_q):
 
     while True:
         jobs = sorted(os.listdir(joinpaths(cfg.composer_dir, "queue/new")), key=queue_sort)
-        log.debug("jobs = %s", jobs)
 
         # Pick the oldest and move it into ./run/
         if not jobs:
@@ -97,11 +96,10 @@ def make_compose(cfg, results_dir):
     repo_url = ks.handler.method.url
 
     # Load the compose configuration
-    cfg_file = joinpaths(results_dir, "config.toml")
+    cfg_path = joinpaths(results_dir, "config.toml")
     if not os.path.exists(cfg_path):
         raise RuntimeError("Missing config.toml for %s" % results_dir)
-    cfg_dict = toml.loads(open(cfg_file, "r").read())
+    cfg_dict = toml.loads(open(cfg_path, "r").read())
-    cfg_dict["logfile"] = log_dict
 
     install_cfg = DataHolder(**cfg_dict)
 

src/pylorax/api/v0.py

@@ -618,10 +618,7 @@ log = logging.getLogger("lorax-composer")
 
 from flask import jsonify, request
 
-# Use pykickstart to calculate disk image size
+from pylorax.api.compose import start_build, compose_types
-from pykickstart.parser import KickstartParser
-from pykickstart.version import makeVersion, RHEL7
-
 from pylorax.api.crossdomain import crossdomain
 from pylorax.api.projects import projects_list, projects_info, projects_depsolve, dep_evra
 from pylorax.api.projects import modules_list, modules_info, ProjectsError
@@ -629,23 +626,10 @@ from pylorax.api.recipes import list_branch_files, read_recipe_commit, recipe_fi
 from pylorax.api.recipes import recipe_from_dict, recipe_from_toml, commit_recipe, delete_recipe, revert_recipe
 from pylorax.api.recipes import tag_recipe_commit, recipe_diff, Recipe, RecipePackage, RecipeModule
 from pylorax.api.workspace import workspace_read, workspace_write, workspace_delete
-from pylorax.creator import DRACUT_DEFAULT, mount_boot_part_over_root
-from pylorax.creator import make_appliance, make_image, make_livecd, make_live_images
-from pylorax.creator import make_runtime, make_squashfs
-from pylorax.imgutils import copytree
-from pylorax.imgutils import Mount, PartitionMount, umount
-from pylorax.installer import InstallError
-from pylorax.sysutils import joinpaths
 
 # The API functions don't actually get called by any code here
 # pylint: disable=unused-variable
 
-# no-virt mode doesn't need libvirt, so make it optional
-try:
-    import libvirt
-except ImportError:
-    libvirt = None
-
 def take_limits(iterable, offset, limit):
     """ Apply offset and limit to an iterable object
 
@@ -1099,3 +1083,61 @@ def v0_api(api):
             return jsonify(error={"msg":str(e)}), 400
 
         return jsonify(modules=modules)
+
+    @api.route("/api/v0/compose", methods=["POST"])
+    @crossdomain(origin="*")
+    def v0_compose_start():
+        """Start a compose
+
+        The body of the post should have these fields:
+          recipe_name   - The recipe name from /recipes/list/
+          compose_type  - The type of output to create, from /compose/types
+          branch        - Optional, defaults to master, selects the git branch to use for the recipe.
+        """
+        compose = request.get_json(cache=False)
+
+        errors = []
+        if not compose:
+            return jsonify(status=False, error={"msg":"Missing POST body"}), 400
+
+        if "recipe_name" not in compose:
+            errors.append("No 'recipe_name' in the JSON request")
+        else:
+            recipe_name = compose["recipe_name"]
+
+        if "branch" not in compose or not compose["branch"]:
+            branch = "master"
+        else:
+            branch = compose["branch"]
+
+        if "compose_type" not in compose:
+            errors.append("No 'compose_type' in the JSON request")
+        else:
+            compose_type = compose["compose_type"]
+
+        if errors:
+            return jsonify(status=False, error={"msg":"\n".join(errors)}), 400
+
+        # Get the git version (if it exists)
+        try:
+            with api.config["GITLOCK"].lock:
+                recipe = read_recipe_commit(api.config["GITLOCK"].repo, branch, recipe_name)
+        except Exception as e:
+            log.error("Problem reading recipe %s: %s", recipe_name, str(e))
+            return jsonify(status=False, error={"msg":str(e)}), 400
+        try:
+            build_id = start_build(api.config["COMPOSER_CFG"], api.config["YUMLOCK"], recipe, compose_type)
+        except Exception as e:
+            return jsonify(status=False, error={"msg":str(e)}), 400
+
+        return jsonify(status=True, build_id=build_id)
+
+    @api.route("/api/v0/compose/types")
+    @crossdomain(origin="*")
+    def v0_compose_types():
+        """Return the list of enabled output types
+
+        (only enabled types are returned)
+        """
+        share_dir = api.config["COMPOSER_CFG"].get("composer", "share_dir")
+        return jsonify(types=[{"name": k, "enabled": True} for k in compose_types(share_dir)])