Add building an image, and the /compose route to start it
This adds the ability to build a tar output image. The /compose and
/compose/types API routes are now available.
To start a build POST a JSON body to /compose, like this:
{"recipe_name":"glusterfs", "compose_type":"tar", "branch":"master"}
This will return a unique build id:
{
"build_id": "4d13abb6-aa4e-4c80-a671-0b867e6e77f6",
"status": true
}
which will be used to keep track of the build status (routes for this
do not exist yet).
This commit is contained in:
Brian C. Lane
parent
de9f5b0456
commit
67da4d6971
|
|
@ -0,0 +1,49 @@
|
|||
# Lorax Composer tar output kickstart template
|
||||
|
||||
#
|
||||
sshpw --username=root --plaintext randOmStrinGhERE
|
||||
# Firewall configuration
|
||||
firewall --enabled
|
||||
|
||||
# Root password
|
||||
rootpw --plaintext removethispw
|
||||
# Network information
|
||||
network --bootproto=dhcp --onboot=on --activate
|
||||
# System authorization information
|
||||
auth --useshadow --enablemd5
|
||||
# System keyboard
|
||||
keyboard --xlayouts=us --vckeymap=us
|
||||
# System language
|
||||
lang en_US.UTF-8
|
||||
# SELinux configuration
|
||||
selinux --enforcing
|
||||
# Installation logging level
|
||||
logging --level=info
|
||||
# Shutdown after installation
|
||||
shutdown
|
||||
# System timezone
|
||||
timezone US/Eastern
|
||||
# System bootloader configuration
|
||||
bootloader --location=mbr
|
||||
# Clear the Master Boot Record
|
||||
zerombr
|
||||
# Partition clearing information
|
||||
clearpart --all
|
||||
# Disk partitioning information
|
||||
part / --fstype="ext4" --size=4000
|
||||
part swap --size=1000
|
||||
|
||||
%post
|
||||
# Remove root password
|
||||
passwd -d root > /dev/null
|
||||
|
||||
# Remove random-seed
|
||||
rm /var/lib/systemd/random-seed
|
||||
%end
|
||||
|
||||
# NOTE Do NOT add any other sections after %packages
|
||||
%packages
|
||||
# Packages requires to support this output format go here
|
||||
|
||||
|
||||
# NOTE lorax-composer will add the recipe packages below here, including the final %end
|
||||
|
|
@ -0,0 +1,193 @@
|
|||
# Copyright (C) 2018 Red Hat, Inc.
|
||||
#
|
||||
# This program is free software; you can redistribute it and/or modify
|
||||
# it under the terms of the GNU General Public License as published by
|
||||
# the Free Software Foundation; either version 2 of the License, or
|
||||
# (at your option) any later version.
|
||||
#
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU General Public License
|
||||
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
#
|
||||
""" Setup for composing an image
|
||||
|
||||
Adding New Output Types
|
||||
-----------------------
|
||||
|
||||
The new output type must add a kickstart template to ./share/composer/ where the
|
||||
name of the kickstart (without the trailing .ks) matches the entry in compose_args.
|
||||
|
||||
The kickstart should not have any url or repo entries, these will be added at build
|
||||
time. The %packages section should be the last thing, and while it can contain mandatory
|
||||
packages required by the output type, it should not have the trailing %end because the
|
||||
package NEVRAs will be appended to it at build time.
|
||||
|
||||
compose_args should have a name matching the kickstart, and it should set the novirt_install
|
||||
parameters needed to generate the desired output. Other types should be set to False.
|
||||
|
||||
"""
|
||||
import logging
|
||||
log = logging.getLogger("lorax-composer")
|
||||
|
||||
import os
|
||||
from glob import glob
|
||||
import pytoml as toml
|
||||
import shutil
|
||||
from uuid import uuid4
|
||||
|
||||
from pylorax.api.projects import projects_depsolve, dep_nevra
|
||||
from pylorax.api.projects import ProjectsError
|
||||
from pylorax.imgutils import default_image_name
|
||||
from pylorax.sysutils import joinpaths
|
||||
|
||||
|
||||
def repo_to_ks(r, url="url"):
|
||||
""" Return a kickstart line with the correct args.
|
||||
|
||||
Set url to "baseurl" if it is a repo, leave it as "url" for the installation url.
|
||||
"""
|
||||
cmd = ""
|
||||
if r.metalink:
|
||||
# XXX Total Hack
|
||||
# RHEL7 kickstart doesn't support metalink. If the url has 'metalink' in it, rewrite it as 'mirrorlist'
|
||||
if "metalink" in r.metalink:
|
||||
log.info("RHEL7 does not support metalink, translating to mirrorlist")
|
||||
cmd += '--mirrorlist="%s" ' % r.metalink.replace("metalink", "mirrorlist")
|
||||
else:
|
||||
log.error("Could not convert metalink to mirrorlist. %s", r.metalink)
|
||||
raise RuntimeError("Cannot convert metalink to mirrorlist: %s" % r.metalink)
|
||||
elif r.mirrorlist:
|
||||
cmd += '--mirrorlist="%s" ' % r.mirrorlist
|
||||
elif r.baseurl:
|
||||
cmd += '--%s="%s" ' % (url, r.baseurl[0])
|
||||
else:
|
||||
raise RuntimeError("Repo has no baseurl or mirror")
|
||||
|
||||
if r.proxy:
|
||||
cmd += '--proxy="%s" ' % r.proxy
|
||||
|
||||
if not r.sslverify:
|
||||
cmd += '--noverifyssl'
|
||||
|
||||
return cmd
|
||||
|
||||
def start_build(cfg, yumlock, recipe, compose_type):
|
||||
""" Start the build
|
||||
|
||||
:param cfg: Configuration object
|
||||
:type cfg: ComposerConfig
|
||||
:param yumlock: Lock and YumBase for depsolving
|
||||
:type yumlock: YumLock
|
||||
:param recipe: The recipe to build
|
||||
:type recipe: str
|
||||
:param compose_type: The type of output to create from the recipe
|
||||
:type compose_type: str
|
||||
:returns: Unique ID for the build that can be used to track its status
|
||||
:rtype: str
|
||||
"""
|
||||
share_dir = cfg.get("composer", "share_dir")
|
||||
lib_dir = cfg.get("composer", "lib_dir")
|
||||
|
||||
# Make sure compose_type is valid
|
||||
if compose_type not in compose_types(share_dir):
|
||||
raise RuntimeError("Invalid compose type (%s), must be one of %s" % (compose_type, compose_types(share_dir)))
|
||||
|
||||
# Combine modules and packages and depsolve the list
|
||||
# TODO include the version/glob in the depsolving
|
||||
module_names = map(lambda m: m["name"], recipe["modules"] or [])
|
||||
package_names = map(lambda p: p["name"], recipe["packages"] or [])
|
||||
projects = sorted(set(module_names+package_names), key=lambda n: n.lower())
|
||||
deps = []
|
||||
try:
|
||||
with yumlock.lock:
|
||||
deps = projects_depsolve(yumlock.yb, projects)
|
||||
except ProjectsError as e:
|
||||
log.error("start_build depsolve: %s", str(e))
|
||||
raise RuntimeError("Problem depsolving %s: %s" % (recipe["name"], str(e)))
|
||||
|
||||
# Create the results directory
|
||||
build_id = str(uuid4())
|
||||
results_dir = joinpaths(lib_dir, "results", build_id)
|
||||
os.makedirs(results_dir)
|
||||
|
||||
# Read the kickstart template for this type and copy it into the results
|
||||
ks_template_path = joinpaths(share_dir, "composer", compose_type) + ".ks"
|
||||
shutil.copy(ks_template_path, results_dir)
|
||||
ks_template = open(ks_template_path, "r").read()
|
||||
|
||||
# Write out the dependencies to the results dir
|
||||
deps_path = joinpaths(results_dir, "deps.txt")
|
||||
with open(deps_path, "w") as f:
|
||||
for d in deps:
|
||||
f.write(dep_nevra(d)+"\n")
|
||||
|
||||
# Create the final kickstart with repos and package list
|
||||
ks_path = joinpaths(results_dir, "final-kickstart.ks")
|
||||
with open(ks_path, "w") as f:
|
||||
with yumlock.lock:
|
||||
repos = yumlock.yb.repos.listEnabled()
|
||||
if not repos:
|
||||
raise RuntimeError("No enabled repos, canceling build.")
|
||||
|
||||
ks_url = repo_to_ks(repos[0], "url")
|
||||
log.debug("url = %s", ks_url)
|
||||
f.write('url %s\n' % ks_url)
|
||||
for idx, r in enumerate(repos[1:]):
|
||||
ks_repo = repo_to_ks(r, "baseurl")
|
||||
log.debug("repo composer-%s = %s", idx, ks_repo)
|
||||
f.write('repo --name="composer-%s" %s\n' % (idx, ks_repo))
|
||||
|
||||
f.write(ks_template)
|
||||
|
||||
for d in deps:
|
||||
f.write(dep_nevra(d)+"\n")
|
||||
|
||||
f.write("%end\n")
|
||||
|
||||
# Setup the config to pass to novirt_install
|
||||
log_dir = joinpaths(results_dir, "logs/")
|
||||
cfg_args = compose_args(compose_type)
|
||||
cfg_args.update({
|
||||
"compression": "xz",
|
||||
#"compress_args": ["-9"],
|
||||
"compress_args": [],
|
||||
"ks": [ks_path],
|
||||
"anaconda_args": "",
|
||||
"proxy": "",
|
||||
"armplatform": "",
|
||||
|
||||
"project": "Red Hat Enterprise Linux",
|
||||
"releasever": "7",
|
||||
|
||||
"logfile": log_dir
|
||||
})
|
||||
with open(joinpaths(results_dir, "config.toml"), "w") as f:
|
||||
f.write(toml.dumps(cfg_args).encode("UTF-8"))
|
||||
|
||||
log.info("Starting compose %s with recipe %s output type %s", build_id, recipe["name"], compose_type)
|
||||
os.symlink(results_dir, joinpaths(lib_dir, "queue/new/", build_id))
|
||||
|
||||
return build_id
|
||||
|
||||
# Supported output types
|
||||
def compose_types(share_dir):
|
||||
""" Returns a list of the supported output types
|
||||
|
||||
The output types come from the kickstart names in /usr/share/lorax/composer/*ks
|
||||
"""
|
||||
return [os.path.basename(ks)[:-3] for ks in glob(joinpaths(share_dir, "composer/*.ks"))]
|
||||
|
||||
def compose_args(compose_type):
|
||||
""" Returns the settings to pass to novirt_install for the compose type"""
|
||||
_MAP = {"tar": {"make_tar": True,
|
||||
"make_iso": False,
|
||||
"make_fsimage": False,
|
||||
"qcow2": False,
|
||||
"image_name": default_image_name("xz", "root.tar")},
|
||||
}
|
||||
|
||||
return _MAP[compose_type]
|
||||
|
|
@ -56,7 +56,6 @@ def monitor(cfg, cancel_q):
|
|||
|
||||
while True:
|
||||
jobs = sorted(os.listdir(joinpaths(cfg.composer_dir, "queue/new")), key=queue_sort)
|
||||
log.debug("jobs = %s", jobs)
|
||||
|
||||
# Pick the oldest and move it into ./run/
|
||||
if not jobs:
|
||||
|
|
@ -97,11 +96,10 @@ def make_compose(cfg, results_dir):
|
|||
repo_url = ks.handler.method.url
|
||||
|
||||
# Load the compose configuration
|
||||
cfg_file = joinpaths(results_dir, "config.toml")
|
||||
cfg_path = joinpaths(results_dir, "config.toml")
|
||||
if not os.path.exists(cfg_path):
|
||||
raise RuntimeError("Missing config.toml for %s" % results_dir)
|
||||
cfg_dict = toml.loads(open(cfg_file, "r").read())
|
||||
cfg_dict["logfile"] = log_dict
|
||||
cfg_dict = toml.loads(open(cfg_path, "r").read())
|
||||
|
||||
install_cfg = DataHolder(**cfg_dict)
|
||||
|
||||
|
|
|
|||
|
|
@ -618,10 +618,7 @@ log = logging.getLogger("lorax-composer")
|
|||
|
||||
from flask import jsonify, request
|
||||
|
||||
# Use pykickstart to calculate disk image size
|
||||
from pykickstart.parser import KickstartParser
|
||||
from pykickstart.version import makeVersion, RHEL7
|
||||
|
||||
from pylorax.api.compose import start_build, compose_types
|
||||
from pylorax.api.crossdomain import crossdomain
|
||||
from pylorax.api.projects import projects_list, projects_info, projects_depsolve, dep_evra
|
||||
from pylorax.api.projects import modules_list, modules_info, ProjectsError
|
||||
|
|
@ -629,23 +626,10 @@ from pylorax.api.recipes import list_branch_files, read_recipe_commit, recipe_fi
|
|||
from pylorax.api.recipes import recipe_from_dict, recipe_from_toml, commit_recipe, delete_recipe, revert_recipe
|
||||
from pylorax.api.recipes import tag_recipe_commit, recipe_diff, Recipe, RecipePackage, RecipeModule
|
||||
from pylorax.api.workspace import workspace_read, workspace_write, workspace_delete
|
||||
from pylorax.creator import DRACUT_DEFAULT, mount_boot_part_over_root
|
||||
from pylorax.creator import make_appliance, make_image, make_livecd, make_live_images
|
||||
from pylorax.creator import make_runtime, make_squashfs
|
||||
from pylorax.imgutils import copytree
|
||||
from pylorax.imgutils import Mount, PartitionMount, umount
|
||||
from pylorax.installer import InstallError
|
||||
from pylorax.sysutils import joinpaths
|
||||
|
||||
# The API functions don't actually get called by any code here
|
||||
# pylint: disable=unused-variable
|
||||
|
||||
# no-virt mode doesn't need libvirt, so make it optional
|
||||
try:
|
||||
import libvirt
|
||||
except ImportError:
|
||||
libvirt = None
|
||||
|
||||
def take_limits(iterable, offset, limit):
|
||||
""" Apply offset and limit to an iterable object
|
||||
|
||||
|
|
@ -1099,3 +1083,61 @@ def v0_api(api):
|
|||
return jsonify(error={"msg":str(e)}), 400
|
||||
|
||||
return jsonify(modules=modules)
|
||||
|
||||
@api.route("/api/v0/compose", methods=["POST"])
|
||||
@crossdomain(origin="*")
|
||||
def v0_compose_start():
|
||||
"""Start a compose
|
||||
|
||||
The body of the post should have these fields:
|
||||
recipe_name - The recipe name from /recipes/list/
|
||||
compose_type - The type of output to create, from /compose/types
|
||||
branch - Optional, defaults to master, selects the git branch to use for the recipe.
|
||||
"""
|
||||
compose = request.get_json(cache=False)
|
||||
|
||||
errors = []
|
||||
if not compose:
|
||||
return jsonify(status=False, error={"msg":"Missing POST body"}), 400
|
||||
|
||||
if "recipe_name" not in compose:
|
||||
errors.append("No 'recipe_name' in the JSON request")
|
||||
else:
|
||||
recipe_name = compose["recipe_name"]
|
||||
|
||||
if "branch" not in compose or not compose["branch"]:
|
||||
branch = "master"
|
||||
else:
|
||||
branch = compose["branch"]
|
||||
|
||||
if "compose_type" not in compose:
|
||||
errors.append("No 'compose_type' in the JSON request")
|
||||
else:
|
||||
compose_type = compose["compose_type"]
|
||||
|
||||
if errors:
|
||||
return jsonify(status=False, error={"msg":"\n".join(errors)}), 400
|
||||
|
||||
# Get the git version (if it exists)
|
||||
try:
|
||||
with api.config["GITLOCK"].lock:
|
||||
recipe = read_recipe_commit(api.config["GITLOCK"].repo, branch, recipe_name)
|
||||
except Exception as e:
|
||||
log.error("Problem reading recipe %s: %s", recipe_name, str(e))
|
||||
return jsonify(status=False, error={"msg":str(e)}), 400
|
||||
try:
|
||||
build_id = start_build(api.config["COMPOSER_CFG"], api.config["YUMLOCK"], recipe, compose_type)
|
||||
except Exception as e:
|
||||
return jsonify(status=False, error={"msg":str(e)}), 400
|
||||
|
||||
return jsonify(status=True, build_id=build_id)
|
||||
|
||||
@api.route("/api/v0/compose/types")
|
||||
@crossdomain(origin="*")
|
||||
def v0_compose_types():
|
||||
"""Return the list of enabled output types
|
||||
|
||||
(only enabled types are returned)
|
||||
"""
|
||||
share_dir = api.config["COMPOSER_CFG"].get("composer", "share_dir")
|
||||
return jsonify(types=[{"name": k, "enabled": True} for k in compose_types(share_dir)])
|
||||
|
|
|
|||
Loading…
Reference in New Issue