almalinux/lorax
13
0
Fork 0
Code Issues Pull Requests Releases Activity

Add building an image, and the /compose route to start it

Browse Source 
This adds the ability to build a tar output image. The /compose and
/compose/types API routes are now available.

To start a build POST a JSON body to /compose, like this:

{"recipe_name":"glusterfs", "compose_type":"tar", "branch":"master"}

This will return a unique build id:

{
  "build_id": "4d13abb6-aa4e-4c80-a671-0b867e6e77f6",
  "status": true
}

which will be used to keep track of the build status (routes for this
do not exist yet).
This commit is contained in:
Brian C. Lane 2018-01-29 17:08:30 -08:00
parent 97fe514ceb
commit 0346a04dad
4 changed files with 303 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

49
share/composer/tar.ks Normal file
View File

@ -0,0 +1,49 @@
# Lorax Composer tar output kickstart template
#
sshpw --username=root --plaintext randOmStrinGhERE
# Firewall configuration
firewall --enabled
# Root password
rootpw --plaintext removethispw
# Network information
network --bootproto=dhcp --onboot=on --activate
# System authorization information
auth --useshadow --enablemd5
# System keyboard
keyboard --xlayouts=us --vckeymap=us
# System language
lang en_US.UTF-8
# SELinux configuration
selinux --enforcing
# Installation logging level
logging --level=info
# Shutdown after installation
shutdown
# System timezone
timezone US/Eastern
# System bootloader configuration
bootloader --location=mbr
# Clear the Master Boot Record
zerombr
# Partition clearing information
clearpart --all
# Disk partitioning information
part / --fstype="ext4" --size=4000
part swap --size=1000
%post
# Remove root password
passwd -d root > /dev/null
# Remove random-seed
rm /var/lib/systemd/random-seed
%end
# NOTE Do NOT add any other sections after %packages
%packages
# Packages requires to support this output format go here
# NOTE lorax-composer will add the recipe packages below here, including the final %end

193
src/pylorax/api/compose.py Normal file
View File

@ -0,0 +1,193 @@
# Copyright (C) 2018 Red Hat, Inc.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
""" Setup for composing an image
Adding New Output Types
-----------------------
The new output type must add a kickstart template to ./share/composer/ where the
name of the kickstart (without the trailing .ks) matches the entry in compose_args.
The kickstart should not have any url or repo entries, these will be added at build
time. The %packages section should be the last thing, and while it can contain mandatory
packages required by the output type, it should not have the trailing %end because the
package NEVRAs will be appended to it at build time.
compose_args should have a name matching the kickstart, and it should set the novirt_install
parameters needed to generate the desired output. Other types should be set to False.
"""
import logging
log = logging.getLogger("lorax-composer")
import os
from glob import glob
import pytoml as toml
import shutil
from uuid import uuid4
from pylorax.api.projects import projects_depsolve, dep_nevra
from pylorax.api.projects import ProjectsError
from pylorax.imgutils import default_image_name
from pylorax.sysutils import joinpaths
def repo_to_ks(r, url="url"):
""" Return a kickstart line with the correct args.
Set url to "baseurl" if it is a repo, leave it as "url" for the installation url.
"""
cmd = ""
if r.metalink:
# XXX Total Hack
# RHEL7 kickstart doesn't support metalink. If the url has 'metalink' in it, rewrite it as 'mirrorlist'
if "metalink" in r.metalink:
log.info("RHEL7 does not support metalink, translating to mirrorlist")
cmd += '--mirrorlist="%s" ' % r.metalink.replace("metalink", "mirrorlist")
else:
log.error("Could not convert metalink to mirrorlist. %s", r.metalink)
raise RuntimeError("Cannot convert metalink to mirrorlist: %s" % r.metalink)
elif r.mirrorlist:
cmd += '--mirrorlist="%s" ' % r.mirrorlist
elif r.baseurl:
cmd += '--%s="%s" ' % (url, r.baseurl[0])
else:
raise RuntimeError("Repo has no baseurl or mirror")
if r.proxy:
cmd += '--proxy="%s" ' % r.proxy
if not r.sslverify:
cmd += '--noverifyssl'
return cmd
def start_build(cfg, yumlock, recipe, compose_type):
""" Start the build
:param cfg: Configuration object
:type cfg: ComposerConfig
:param yumlock: Lock and YumBase for depsolving
:type yumlock: YumLock
:param recipe: The recipe to build
:type recipe: str
:param compose_type: The type of output to create from the recipe
:type compose_type: str
:returns: Unique ID for the build that can be used to track its status
:rtype: str
"""
share_dir = cfg.get("composer", "share_dir")
lib_dir = cfg.get("composer", "lib_dir")
# Make sure compose_type is valid
if compose_type not in compose_types(share_dir):
raise RuntimeError("Invalid compose type (%s), must be one of %s" % (compose_type, compose_types(share_dir)))
# Combine modules and packages and depsolve the list
# TODO include the version/glob in the depsolving
module_names = map(lambda m: m["name"], recipe["modules"] or [])
package_names = map(lambda p: p["name"], recipe["packages"] or [])
projects = sorted(set(module_names+package_names), key=lambda n: n.lower())
deps = []
try:
with yumlock.lock:
deps = projects_depsolve(yumlock.yb, projects)
except ProjectsError as e:
log.error("start_build depsolve: %s", str(e))
raise RuntimeError("Problem depsolving %s: %s" % (recipe["name"], str(e)))
# Create the results directory
build_id = str(uuid4())
results_dir = joinpaths(lib_dir, "results", build_id)
os.makedirs(results_dir)
# Read the kickstart template for this type and copy it into the results
ks_template_path = joinpaths(share_dir, "composer", compose_type) + ".ks"
shutil.copy(ks_template_path, results_dir)
ks_template = open(ks_template_path, "r").read()
# Write out the dependencies to the results dir
deps_path = joinpaths(results_dir, "deps.txt")
with open(deps_path, "w") as f:
for d in deps:
f.write(dep_nevra(d)+"\n")
# Create the final kickstart with repos and package list
ks_path = joinpaths(results_dir, "final-kickstart.ks")
with open(ks_path, "w") as f:
with yumlock.lock:
repos = yumlock.yb.repos.listEnabled()
if not repos:
raise RuntimeError("No enabled repos, canceling build.")
ks_url = repo_to_ks(repos[0], "url")
log.debug("url = %s", ks_url)
f.write('url %s\n' % ks_url)
for idx, r in enumerate(repos[1:]):
ks_repo = repo_to_ks(r, "baseurl")
log.debug("repo composer-%s = %s", idx, ks_repo)
f.write('repo --name="composer-%s" %s\n' % (idx, ks_repo))
f.write(ks_template)
for d in deps:
f.write(dep_nevra(d)+"\n")
f.write("%end\n")
# Setup the config to pass to novirt_install
log_dir = joinpaths(results_dir, "logs/")
cfg_args = compose_args(compose_type)
cfg_args.update({
"compression": "xz",
#"compress_args": ["-9"],
"compress_args": [],
"ks": [ks_path],
"anaconda_args": "",
"proxy": "",
"armplatform": "",
"project": "Red Hat Enterprise Linux",
"releasever": "7",
"logfile": log_dir
})
with open(joinpaths(results_dir, "config.toml"), "w") as f:
f.write(toml.dumps(cfg_args).encode("UTF-8"))
log.info("Starting compose %s with recipe %s output type %s", build_id, recipe["name"], compose_type)
os.symlink(results_dir, joinpaths(lib_dir, "queue/new/", build_id))
return build_id
# Supported output types
def compose_types(share_dir):
""" Returns a list of the supported output types
The output types come from the kickstart names in /usr/share/lorax/composer/*ks
"""
return [os.path.basename(ks)[:-3] for ks in glob(joinpaths(share_dir, "composer/*.ks"))]
def compose_args(compose_type):
""" Returns the settings to pass to novirt_install for the compose type"""
_MAP = {"tar": {"make_tar": True,
"make_iso": False,
"make_fsimage": False,
"qcow2": False,
"image_name": default_image_name("xz", "root.tar")},
}
return _MAP[compose_type]

6
src/pylorax/api/queue.py
View File

@ -56,7 +56,6 @@ def monitor(cfg, cancel_q):
while True:
jobs = sorted(os.listdir(joinpaths(cfg.composer_dir, "queue/new")), key=queue_sort)
log.debug("jobs = %s", jobs)
# Pick the oldest and move it into ./run/
if not jobs:
@ -97,11 +96,10 @@ def make_compose(cfg, results_dir):
repo_url = ks.handler.method.url
# Load the compose configuration
cfg_file = joinpaths(results_dir, "config.toml")
cfg_path = joinpaths(results_dir, "config.toml")
if not os.path.exists(cfg_path):
raise RuntimeError("Missing config.toml for %s" % results_dir)
cfg_dict = toml.loads(open(cfg_file, "r").read())
cfg_dict["logfile"] = log_dict
cfg_dict = toml.loads(open(cfg_path, "r").read())
install_cfg = DataHolder(**cfg_dict)

76
src/pylorax/api/v0.py
View File

@ -618,10 +618,7 @@ log = logging.getLogger("lorax-composer")
from flask import jsonify, request
# Use pykickstart to calculate disk image size
from pykickstart.parser import KickstartParser
from pykickstart.version import makeVersion, RHEL7
from pylorax.api.compose import start_build, compose_types
from pylorax.api.crossdomain import crossdomain
from pylorax.api.projects import projects_list, projects_info, projects_depsolve, dep_evra
from pylorax.api.projects import modules_list, modules_info, ProjectsError
@ -629,23 +626,10 @@ from pylorax.api.recipes import list_branch_files, read_recipe_commit, recipe_fi
from pylorax.api.recipes import recipe_from_dict, recipe_from_toml, commit_recipe, delete_recipe, revert_recipe
from pylorax.api.recipes import tag_recipe_commit, recipe_diff, Recipe, RecipePackage, RecipeModule
from pylorax.api.workspace import workspace_read, workspace_write, workspace_delete
from pylorax.creator import DRACUT_DEFAULT, mount_boot_part_over_root
from pylorax.creator import make_appliance, make_image, make_livecd, make_live_images
from pylorax.creator import make_runtime, make_squashfs
from pylorax.imgutils import copytree
from pylorax.imgutils import Mount, PartitionMount, umount
from pylorax.installer import InstallError
from pylorax.sysutils import joinpaths
# The API functions don't actually get called by any code here
# pylint: disable=unused-variable
# no-virt mode doesn't need libvirt, so make it optional
try:
import libvirt
except ImportError:
libvirt = None
def take_limits(iterable, offset, limit):
""" Apply offset and limit to an iterable object
@ -1099,3 +1083,61 @@ def v0_api(api):
return jsonify(error={"msg":str(e)}), 400
return jsonify(modules=modules)
@api.route("/api/v0/compose", methods=["POST"])
@crossdomain(origin="*")
def v0_compose_start():
"""Start a compose
The body of the post should have these fields:
recipe_name - The recipe name from /recipes/list/
compose_type - The type of output to create, from /compose/types
branch - Optional, defaults to master, selects the git branch to use for the recipe.
"""
compose = request.get_json(cache=False)
errors = []
if not compose:
return jsonify(status=False, error={"msg":"Missing POST body"}), 400
if "recipe_name" not in compose:
errors.append("No 'recipe_name' in the JSON request")
else:
recipe_name = compose["recipe_name"]
if "branch" not in compose or not compose["branch"]:
branch = "master"
else:
branch = compose["branch"]
if "compose_type" not in compose:
errors.append("No 'compose_type' in the JSON request")
else:
compose_type = compose["compose_type"]
if errors:
return jsonify(status=False, error={"msg":"\n".join(errors)}), 400
# Get the git version (if it exists)
try:
with api.config["GITLOCK"].lock:
recipe = read_recipe_commit(api.config["GITLOCK"].repo, branch, recipe_name)
except Exception as e:
log.error("Problem reading recipe %s: %s", recipe_name, str(e))
return jsonify(status=False, error={"msg":str(e)}), 400
try:
build_id = start_build(api.config["COMPOSER_CFG"], api.config["YUMLOCK"], recipe, compose_type)
except Exception as e:
return jsonify(status=False, error={"msg":str(e)}), 400
return jsonify(status=True, build_id=build_id)
@api.route("/api/v0/compose/types")
@crossdomain(origin="*")
def v0_compose_types():
"""Return the list of enabled output types
(only enabled types are returned)
"""
share_dir = api.config["COMPOSER_CFG"].get("composer", "share_dir")
return jsonify(types=[{"name": k, "enabled": True} for k in compose_types(share_dir)])