# Copyright (C) 2018 Red Hat, Inc.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
""" Setup for composing an image
Adding New Output Types
-----------------------
The new output type must add a kickstart template to ./share/composer/ where the
name of the kickstart (without the trailing .ks) matches the entry in compose_args.
The kickstart should not have any url or repo entries, these will be added at build
time. The %packages section should be the last thing, and while it can contain mandatory
packages required by the output type, it should not have the trailing %end because the
package NEVRAs will be appended to it at build time.
compose_args should have a name matching the kickstart, and it should set the novirt_install
parameters needed to generate the desired output. Other types should be set to False.
"""
import logging
log = logging.getLogger("lorax-composer")
import os
from glob import glob
from math import ceil
import pytoml as toml
import shutil
from uuid import uuid4
# Use pykickstart to calculate disk image size
from pykickstart.parser import KickstartParser
from pykickstart.version import makeVersion
from pylorax.api.projects import projects_depsolve, projects_depsolve_with_size, dep_nevra
from pylorax.api.projects import ProjectsError
from pylorax.api.recipes import read_recipe_and_id
from pylorax.api.timestamp import TS_CREATED, write_timestamp
from pylorax.imgutils import default_image_name
from pylorax.sysutils import joinpaths, flatconfig
[docs]def test_templates(dbo, share_dir):
""" Try depsolving each of the the templates and report any errors
:param dbo: dnf base object
:type dbo: dnf.Base
:returns: List of template types and errors
:rtype: List of errors
Return a list of templates and errors encountered or an empty list
"""
template_errors = []
for compose_type in compose_types(share_dir):
# Read the kickstart template for this type
ks_template_path = joinpaths(share_dir, "composer", compose_type) + ".ks"
ks_template = open(ks_template_path, "r").read()
# How much space will the packages in the default template take?
ks_version = makeVersion()
ks = KickstartParser(ks_version, errorsAreFatal=False, missingIncludeIsFatal=False)
ks.readKickstartFromString(ks_template+"\n%end\n")
pkgs = [(name, "*") for name in ks.handler.packages.packageList]
grps = [grp.name for grp in ks.handler.packages.groupList]
try:
_ = projects_depsolve(dbo, pkgs, grps)
except ProjectsError as e:
template_errors.append("Error depsolving %s: %s" % (compose_type, str(e)))
return template_errors
[docs]def repo_to_ks(r, url="url"):
""" Return a kickstart line with the correct args.
:param r: DNF repository information
:type r: dnf.Repo
:param url: "url" or "baseurl" to use for the baseurl parameter
:type url: str
:returns: kickstart command arguments for url/repo command
:rtype: str
Set url to "baseurl" if it is a repo, leave it as "url" for the installation url.
"""
cmd = ""
# url uses --url not --baseurl
if r.baseurl:
cmd += '--%s="%s" ' % (url, r.baseurl[0])
elif r.metalink:
cmd += '--metalink="%s" ' % r.metalink
elif r.mirrorlist:
cmd += '--mirrorlist="%s" ' % r.mirrorlist
else:
raise RuntimeError("Repo has no baseurl, metalink, or mirrorlist")
if r.proxy:
cmd += '--proxy="%s" ' % r.proxy
if not r.sslverify:
cmd += '--noverifyssl'
return cmd
[docs]def write_ks_user(f, user):
""" Write kickstart user and sshkey entry
:param f: kickstart file object
:type f: open file object
:param user: A blueprint user dictionary
:type user: dict
If the entry contains a ssh key, use sshkey to write it
All of the user fields are optional, except name, write out a kickstart user entry
with whatever options are relevant.
"""
if "name" not in user:
raise RuntimeError("user entry requires a name")
# ssh key uses the sshkey kickstart command
if "key" in user:
f.write('sshkey --user %s "%s"\n' % (user["name"], user["key"]))
# Write out the user kickstart command, much of it is optional
f.write("user --name %s" % user["name"])
if "home" in user:
f.write(" --homedir %s" % user["home"])
if "password" in user:
if any(user["password"].startswith(prefix) for prefix in ["$2b$", "$6$", "$5$"]):
log.debug("Detected pre-crypted password")
f.write(" --iscrypted")
else:
log.debug("Detected plaintext password")
f.write(" --plaintext")
f.write(" --password \"%s\"" % user["password"])
if "shell" in user:
f.write(" --shell %s" % user["shell"])
if "uid" in user:
f.write(" --uid %d" % int(user["uid"]))
if "gid" in user:
f.write(" --gid %d" % int(user["gid"]))
if "description" in user:
f.write(" --gecos \"%s\"" % user["description"])
if "groups" in user:
f.write(" --groups %s" % ",".join(user["groups"]))
f.write("\n")
[docs]def write_ks_group(f, group):
""" Write kickstart group entry
:param f: kickstart file object
:type f: open file object
:param group: A blueprint group dictionary
:type user: dict
gid is optional
"""
if "name" not in group:
raise RuntimeError("group entry requires a name")
f.write("group --name %s" % group["name"])
if "gid" in group:
f.write(" --gid %d" % int(group["gid"]))
f.write("\n")
[docs]def add_customizations(f, recipe):
""" Add customizations to the kickstart file
:param f: kickstart file object
:type f: open file object
:param recipe:
:type recipe: Recipe object
:returns: None
:raises: RuntimeError if there was a problem writing to the kickstart
"""
if "customizations" not in recipe:
return
customizations = recipe["customizations"]
if "hostname" in customizations:
f.write("network --hostname=%s\n" % customizations["hostname"])
# TODO - remove this, should use user section to define this
if "sshkey" in customizations:
# This is a list of entries
for sshkey in customizations["sshkey"]:
if "user" not in sshkey or "key" not in sshkey:
log.error("%s is incorrect, skipping", sshkey)
continue
f.write('sshkey --user %s "%s"\n' % (sshkey["user"], sshkey["key"]))
# Creating a user also creates a group. Make a list of the names for later
user_groups = []
if "user" in customizations:
# only name is required, everything else is optional
for user in customizations["user"]:
write_ks_user(f, user)
user_groups.append(user["name"])
if "group" in customizations:
for group in customizations["group"]:
if group["name"] not in user_groups:
write_ks_group(f, group)
else:
log.warning("Skipping group %s, already created by user", group["name"])
[docs]def start_build(cfg, dnflock, gitlock, branch, recipe_name, compose_type, test_mode=0):
""" Start the build
:param cfg: Configuration object
:type cfg: ComposerConfig
:param dnflock: Lock and YumBase for depsolving
:type dnflock: YumLock
:param recipe: The recipe to build
:type recipe: str
:param compose_type: The type of output to create from the recipe
:type compose_type: str
:returns: Unique ID for the build that can be used to track its status
:rtype: str
"""
share_dir = cfg.get("composer", "share_dir")
lib_dir = cfg.get("composer", "lib_dir")
# Make sure compose_type is valid
if compose_type not in compose_types(share_dir):
raise RuntimeError("Invalid compose type (%s), must be one of %s" % (compose_type, compose_types(share_dir)))
with gitlock.lock:
(commit_id, recipe) = read_recipe_and_id(gitlock.repo, branch, recipe_name)
# Combine modules and packages and depsolve the list
# TODO include the version/glob in the depsolving
module_nver = recipe.module_nver
package_nver = recipe.package_nver
projects = sorted(set(module_nver+package_nver), key=lambda p: p[0].lower())
deps = []
try:
with dnflock.lock:
(installed_size, deps) = projects_depsolve_with_size(dnflock.dbo, projects, recipe.group_names, with_core=False)
except ProjectsError as e:
log.error("start_build depsolve: %s", str(e))
raise RuntimeError("Problem depsolving %s: %s" % (recipe["name"], str(e)))
# Read the kickstart template for this type
ks_template_path = joinpaths(share_dir, "composer", compose_type) + ".ks"
ks_template = open(ks_template_path, "r").read()
# How much space will the packages in the default template take?
ks_version = makeVersion()
ks = KickstartParser(ks_version, errorsAreFatal=False, missingIncludeIsFatal=False)
ks.readKickstartFromString(ks_template+"\n%end\n")
pkgs = [(name, "*") for name in ks.handler.packages.packageList]
grps = [grp.name for grp in ks.handler.packages.groupList]
try:
with dnflock.lock:
(template_size, _) = projects_depsolve_with_size(dnflock.dbo, pkgs, grps, with_core=not ks.handler.packages.nocore)
except ProjectsError as e:
log.error("start_build depsolve: %s", str(e))
raise RuntimeError("Problem depsolving %s: %s" % (recipe["name"], str(e)))
log.debug("installed_size = %d, template_size=%d", installed_size, template_size)
# Minimum LMC disk size is 1GiB, and anaconda bumps the estimated size up by 10% (which doesn't always work).
# XXX BUT Anaconda has a bug, it won't execute a kickstart on a disk smaller than 3000 MB
# XXX There is an upstream patch pending, but until then, use that as the minimum
installed_size = max(3e9, int((installed_size+template_size))) * 1.2
log.debug("/ partition size = %d", installed_size)
# Create the results directory
build_id = str(uuid4())
results_dir = joinpaths(lib_dir, "results", build_id)
os.makedirs(results_dir)
# Write the recipe commit hash
commit_path = joinpaths(results_dir, "COMMIT")
with open(commit_path, "w") as f:
f.write(commit_id)
# Write the original recipe
recipe_path = joinpaths(results_dir, "blueprint.toml")
with open(recipe_path, "w") as f:
f.write(recipe.toml())
# Write the frozen recipe
frozen_recipe = recipe.freeze(deps)
recipe_path = joinpaths(results_dir, "frozen.toml")
with open(recipe_path, "w") as f:
f.write(frozen_recipe.toml())
# Write out the dependencies to the results dir
deps_path = joinpaths(results_dir, "deps.toml")
with open(deps_path, "w") as f:
f.write(toml.dumps({"packages":deps}))
# Save a copy of the original kickstart
shutil.copy(ks_template_path, results_dir)
with dnflock.lock:
repos = list(dnflock.dbo.repos.iter_enabled())
if not repos:
raise RuntimeError("No enabled repos, canceling build.")
# Create the final kickstart with repos and package list
ks_path = joinpaths(results_dir, "final-kickstart.ks")
with open(ks_path, "w") as f:
ks_url = repo_to_ks(repos[0], "url")
log.debug("url = %s", ks_url)
f.write('url %s\n' % ks_url)
for idx, r in enumerate(repos[1:]):
ks_repo = repo_to_ks(r, "baseurl")
log.debug("repo composer-%s = %s", idx, ks_repo)
f.write('repo --name="composer-%s" %s\n' % (idx, ks_repo))
# Setup the disk for booting
# TODO Add GPT and UEFI boot support
f.write('clearpart --all --initlabel\n')
# Write the root partition and it's size in MB (rounded up)
f.write('part / --fstype="ext4" --size=%d\n' % ceil(installed_size / 1024**2))
f.write(ks_template)
for d in deps:
f.write(dep_nevra(d)+"\n")
f.write("%end\n")
add_customizations(f, recipe)
# Setup the config to pass to novirt_install
log_dir = joinpaths(results_dir, "logs/")
cfg_args = compose_args(compose_type)
# Get the title, project, and release version from the host
if not os.path.exists("/etc/os-release"):
log.error("/etc/os-release is missing, cannot determine product or release version")
os_release = flatconfig("/etc/os-release")
log.debug("os_release = %s", dict(os_release.items()))
cfg_args["title"] = os_release.get("PRETTY_NAME", "")
cfg_args["project"] = os_release.get("NAME", "")
cfg_args["releasever"] = os_release.get("VERSION_ID", "")
cfg_args["volid"] = ""
cfg_args.update({
"compression": "xz",
"compress_args": [],
"ks": [ks_path],
"logfile": log_dir,
"timeout": 60, # 60 minute timeout
})
with open(joinpaths(results_dir, "config.toml"), "w") as f:
f.write(toml.dumps(cfg_args))
# Set the initial status
open(joinpaths(results_dir, "STATUS"), "w").write("WAITING")
# Set the test mode, if requested
if test_mode > 0:
open(joinpaths(results_dir, "TEST"), "w").write("%s" % test_mode)
write_timestamp(results_dir, TS_CREATED)
log.info("Adding %s (%s %s) to compose queue", build_id, recipe["name"], compose_type)
os.symlink(results_dir, joinpaths(lib_dir, "queue/new/", build_id))
return build_id
# Supported output types
[docs]def compose_types(share_dir):
r""" Returns a list of the supported output types
The output types come from the kickstart names in /usr/share/lorax/composer/\*ks
"""
return sorted([os.path.basename(ks)[:-3] for ks in glob(joinpaths(share_dir, "composer/*.ks"))])
[docs]def compose_args(compose_type):
""" Returns the settings to pass to novirt_install for the compose type
:param compose_type: The type of compose to create, from `compose_types()`
:type compose_type: str
This will return a dict of options that match the ArgumentParser options for livemedia-creator.
These are the ones the define the type of output, it's filename, etc.
Other options will be filled in by `make_compose()`
"""
_MAP = {"tar": {"make_iso": False,
"make_disk": False,
"make_fsimage": False,
"make_appliance": False,
"make_ami": False,
"make_tar": True,
"make_pxe_live": False,
"make_ostree_live": False,
"make_oci": False,
"make_vagrant": False,
"ostree": False,
"live_rootfs_keep_size": False,
"live_rootfs_size": 0,
"image_type": False, # False instead of None because of TOML
"qemu_args": [],
"image_name": default_image_name("xz", "root.tar"),
"image_only": True,
"app_name": None,
"app_template": None,
"app_file": None,
},
"live-iso": {"make_iso": True,
"make_disk": False,
"make_fsimage": False,
"make_appliance": False,
"make_ami": False,
"make_tar": False,
"make_pxe_live": False,
"make_ostree_live": False,
"make_oci": False,
"make_vagrant": False,
"ostree": False,
"live_rootfs_keep_size": False,
"live_rootfs_size": 0,
"image_type": False, # False instead of None because of TOML
"qemu_args": [],
"image_name": "live.iso",
"fs_label": "Anaconda", # Live booting may expect this to be 'Anaconda'
"image_only": False,
"app_name": None,
"app_template": None,
"app_file": None,
"iso_only": True,
"iso_name": "live.iso",
},
"partitioned-disk": {"make_iso": False,
"make_disk": True,
"make_fsimage": False,
"make_appliance": False,
"make_ami": False,
"make_tar": False,
"make_pxe_live": False,
"make_ostree_live": False,
"make_oci": False,
"make_vagrant": False,
"ostree": False,
"live_rootfs_keep_size": False,
"live_rootfs_size": 0,
"image_type": False, # False instead of None because of TOML
"qemu_args": [],
"image_name": "disk.img",
"fs_label": "",
"image_only": True,
"app_name": None,
"app_template": None,
"app_file": None,
},
"qcow2": {"make_iso": False,
"make_disk": True,
"make_fsimage": False,
"make_appliance": False,
"make_ami": False,
"make_tar": False,
"make_pxe_live": False,
"make_ostree_live": False,
"make_oci": False,
"make_vagrant": False,
"ostree": False,
"live_rootfs_keep_size": False,
"live_rootfs_size": 0,
"image_type": "qcow2",
"qemu_args": [],
"image_name": "disk.qcow2",
"fs_label": "",
"image_only": True,
"app_name": None,
"app_template": None,
"app_file": None,
},
"ext4-filesystem": {"make_iso": False,
"make_disk": False,
"make_fsimage": True,
"make_appliance": False,
"make_ami": False,
"make_tar": False,
"make_pxe_live": False,
"make_ostree_live": False,
"make_oci": False,
"make_vagrant": False,
"ostree": False,
"live_rootfs_keep_size": False,
"live_rootfs_size": 0,
"image_type": False, # False instead of None because of TOML
"qemu_args": [],
"image_name": "filesystem.img",
"fs_label": "",
"image_only": True,
"app_name": None,
"app_template": None,
"app_file": None,
},
}
return _MAP[compose_type]
[docs]def move_compose_results(cfg, results_dir):
"""Move the final image to the results_dir and cleanup the unneeded compose files
:param cfg: Build configuration
:type cfg: DataHolder
:param results_dir: Directory to put the results into
:type results_dir: str
"""
if cfg["make_tar"]:
shutil.move(joinpaths(cfg["result_dir"], cfg["image_name"]), results_dir)
elif cfg["make_iso"]:
# Output from live iso is always a boot.iso under images/, move and rename it
shutil.move(joinpaths(cfg["result_dir"], cfg["iso_name"]), joinpaths(results_dir, cfg["image_name"]))
elif cfg["make_disk"] or cfg["make_fsimage"]:
shutil.move(joinpaths(cfg["result_dir"], cfg["image_name"]), joinpaths(results_dir, cfg["image_name"]))
# Cleanup the compose directory, but only if it looks like a compose directory
if os.path.basename(cfg["result_dir"]) == "compose":
shutil.rmtree(cfg["result_dir"])
else:
log.error("Incorrect compose directory, not cleaning up")