From 8e90a2a32a05c1da8d0a0a069de3adda2d09ac47 Mon Sep 17 00:00:00 2001 From: Lubos Kocman Date: Tue, 1 Sep 2015 08:03:34 +0000 Subject: [PATCH] Add image-build support Signed-off-by: Lubos Kocman --- bin/pungi-koji | 3 + doc/configuration.rst | 77 ++++++++++++++++ pungi/paths.py | 67 ++++++++++++++ pungi/phases/__init__.py | 1 + pungi/phases/image_build.py | 166 ++++++++++++++++++++++++++++++++++ pungi/wrappers/kojiwrapper.py | 41 ++++++++- 6 files changed, 350 insertions(+), 5 deletions(-) create mode 100644 pungi/phases/image_build.py diff --git a/bin/pungi-koji b/bin/pungi-koji index c93c50e7..32f0e2cf 100755 --- a/bin/pungi-koji +++ b/bin/pungi-koji @@ -211,6 +211,7 @@ def run_compose(compose): extrafiles_phase = pungi.phases.ExtraFilesPhase(compose, pkgset_phase) createiso_phase = pungi.phases.CreateisoPhase(compose) liveimages_phase = pungi.phases.LiveImagesPhase(compose) + image_build_phase = pungi.phases.ImageBuildPhase(compose) test_phase = pungi.phases.TestPhase(compose) # check if all config options are set @@ -279,9 +280,11 @@ def run_compose(compose): # CREATEISO and LIVEIMAGES phases createiso_phase.start() liveimages_phase.start() + image_build_phase.start() createiso_phase.stop() liveimages_phase.stop() + image_build_phase.stop() # merge checksum files for variant in compose.get_variants(types=["variant", "layered-product"]): diff --git a/doc/configuration.rst b/doc/configuration.rst index 56958549..3566b1df 100644 --- a/doc/configuration.rst +++ b/doc/configuration.rst @@ -467,3 +467,80 @@ Example 'src': True }), ] + +Image Build Settings +==================== + +**image_build** + (*list*) -- config for koji image-build; format: [(variant_uid_regex, {arch|*: [{opt: value}])] + +.. note:: + Config can contain anything what is accepted by + koji image-build --config configfile.ini + Repo is currently the only option which is being automatically transformed + into a string. + + Please don't set install_tree as it would get overriden by pungi. + The 'format' attr is [('image_type', 'image_suffix'), ...]. + productmd should ideally contain all of image types and suffixes. + +Example +------- +:: + + image_build = [ + ('^Server$', { + 'x86_64': [ + { + 'format': [('docker', 'tar.gz'), ('qcow2', 'qcow2')] + 'name': 'fedora-qcow-and-docker-base', + 'target': 'koji-target-name', + 'ksversion': 'F23', # value from pykickstart + 'version': '23', + 'ksurl': 'https://git.fedorahosted.org/git/spin-kickstarts.git?somedirectoryifany#HEAD', + 'kickstart': "fedora-docker-base.ks", + 'repo': ["http://someextrarepos.org/repo", "ftp://rekcod.oi/repo]. + # 'install_tree': 'http://sometpath', # this is set automatically by pungi to os_dir for given variant/$arch + 'distro': 'Fedora-20', + 'disk_size': 3 + }, + { + 'format': [('qcow2','qcow2')] + 'name': 'fedora-qcow-base', + 'target': 'koji-target-name', + 'ksversion': 'F23', # value from pykickstart + 'version': '23', + 'ksurl': 'https://git.fedorahosted.org/git/spin-kickstarts.git?somedirectoryifany#HEAD', + 'kickstart': "fedora-docker-base.ks", + 'distro': 'Fedora-23' + } + ] + }), + ] + +Translate Paths Settings +======================== + +**translate_paths** + (*list*) -- list of paths to translate; format: [(path,translated_path)] + +.. note:: + This feature becomes useful when you need to transform compose location + into e.g. a http repo which is can be passed to koji image-build. + Translation needs to be invoked by a function call in pungi. + os.path.normpath() is applied on both path and translated_path + + +Example config +-------------- +:: + translate_paths = [ + ("/mnt/a", "http://b/dir"), + ] + +Example usage +------------- +:: + >>> from pungi.paths import translate_paths + >>> print translate_paths(compose_object_with_mapping, "/mnt/a/c/somefile") + http://b/dir/c/somefile diff --git a/pungi/paths.py b/pungi/paths.py index 02b9d083..73ca5aca 100644 --- a/pungi/paths.py +++ b/pungi/paths.py @@ -307,6 +307,34 @@ class WorkPaths(object): path = os.path.join(path, file_name) return path + def image_build_dir(self, arch, variant, create_dir=True): + """ + @param arch + @param variant + @param create_dir=True + + Examples: + work/x86_64/Server/image-build + """ + path = os.path.join(self.topdir(arch, create_dir=create_dir), variant.uid, "image-build") + if create_dir: + makedirs(path) + return path + + def image_build_conf(self, arch, variant, image_name, image_type, create_dir=True): + """ + @param arch + @param variant + @param image-name + @param image-type (e.g docker) + @param create_dir=True + + Examples: + work/x86_64/Server/image-build/docker_rhel-server-docker.cfg + """ + path = os.path.join(self.image_build_dir(arch, variant), "%s_%s.cfg" % (image_type, image_name)) + return path + class ComposePaths(object): def __init__(self, compose): @@ -511,6 +539,45 @@ class ComposePaths(object): result = os.path.join(path, file_name) return result + def image_dir(self, arch, variant, symlink_to=None, create_dir=True, relative=False): + """ + Examples: + compose/Server/x86_64/images + None + @param arch + @param variant + @param symlink_to=None + @param create_dir=True + @param relative=False + """ + # skip optional, addons and src architecture + if variant.type != "variant": + return None + if arch == "src": + return None + + path = os.path.join(self.topdir(arch, variant, create_dir=create_dir, relative=relative), "images") + if symlink_to: + topdir = self.compose.topdir.rstrip("/") + "/" + relative_dir = path[len(topdir):] + target_dir = os.path.join(symlink_to, self.compose.compose_id, relative_dir) + if create_dir and not relative: + makedirs(target_dir) + try: + os.symlink(target_dir, path) + except OSError as ex: + if ex.errno != errno.EEXIST: + raise + msg = "Symlink pointing to '%s' expected: %s" % (target_dir, path) + if not os.path.islink(path): + raise RuntimeError(msg) + if os.path.abspath(os.readlink(path)) != target_dir: + raise RuntimeError(msg) + else: + if create_dir and not relative: + makedirs(path) + return path + def jigdo_dir(self, arch, variant, create_dir=True, relative=False): """ Examples: diff --git a/pungi/phases/__init__.py b/pungi/phases/__init__.py index 1f11abe6..c3be00cd 100644 --- a/pungi/phases/__init__.py +++ b/pungi/phases/__init__.py @@ -25,4 +25,5 @@ from buildinstall import BuildinstallPhase # noqa from extra_files import ExtraFilesPhase # noqa from createiso import CreateisoPhase # noqa from live_images import LiveImagesPhase # noqa +from image_build import ImageBuildPhase # noqa from test import TestPhase # noqa diff --git a/pungi/phases/image_build.py b/pungi/phases/image_build.py new file mode 100644 index 00000000..688d2b6e --- /dev/null +++ b/pungi/phases/image_build.py @@ -0,0 +1,166 @@ +# -*- coding: utf-8 -*- + + +import os +import time +import pipes + +from pungi.util import get_arch_variant_data +from pungi.phases.base import PhaseBase +from pungi.linker import Linker +from pungi.paths import translate_path +from pungi.wrappers.kojiwrapper import KojiWrapper +from pungi.wrappers.iso import IsoWrapper +from kobo.shortcuts import run, read_checksum_file +from kobo.threads import ThreadPool, WorkerThread +from productmd.images import Image + +class ImageBuildPhase(PhaseBase): + """class for wrapping up koji image-build""" + name = "image_build" + + def __init__(self, compose): + PhaseBase.__init__(self, compose) + self.pool = ThreadPool(logger=self.compose._logger) + + def skip(self): + if PhaseBase.skip(self): + return True + if not self.compose.conf.get(self.name): + self.log_info("Section '%s' was not found. Skipping" % self.name) + return True + return False + + def run(self): + for arch in self.compose.get_arches(): # src will be skipped + for variant in self.compose.get_variants(arch=arch): + image_build_data = get_arch_variant_data(self.compose.conf, self.name, arch, variant) + for image_conf in image_build_data: + image_conf["arches"] = arch # passed to get_image_build_cmd as dict + image_conf["variant"] = variant # ^ + image_conf["install_tree"] = translate_path(self.compose, self.compose.paths.compose.os_tree(arch, variant)) # ^ + format = image_conf["format"] # transform format into right 'format' for image-build + image_conf["format"] = ",".join([x[0] for x in image_conf["format"]]) # 'docker,qcow2' + if image_conf.has_key("repos") and not isinstance(image_conf["repos"], str): + image_conf["repos"] = ",".join(image_conf["repos"]) # supply repos as str separated by , instead of list + cmd = { + "format": format, + "image_conf": image_conf, + "conf_file": self.compose.paths.work.image_build_conf(image_conf["arches"], image_conf['variant'], image_name=image_conf['name'], image_type=image_conf['format'].replace(",", "-")), + "image_dir": self.compose.paths.compose.image_dir(arch, variant), + "relative_image_dir": self.compose.paths.compose.image_dir(arch, variant, create_dir=False, relative=True), + "link_type": self.compose.conf.get("link_type", "hardlink-or-copy") + } + self.pool.add(CreateImageBuildThread(self.pool)) + self.pool.queue_put((self.compose, cmd)) + self.pool.start() + + def stop(self, *args, **kwargs): + PhaseBase.stop(self, *args, **kwargs) + if self.skip(): + return + +class CreateImageBuildThread(WorkerThread): + def fail(self, compose, cmd): + compose.log_error("CreateImageBuild failed.") + + def process(self, item, num): + compose, cmd = item + mounts = [compose.topdir] + if "mount" in cmd: + mounts.append(cmd["mount"]) + runroot = compose.conf.get("runroot", False) + log_file = compose.paths.log.log_file(cmd["image_conf"]["arches"], "imagebuild-%s-%s-%s" % (cmd["image_conf"]["arches"], cmd["image_conf"]["variant"], cmd['image_conf']['format'].replace(",","-"))) + msg = "Creating %s image (arch: %s, variant: %s)" % (cmd["image_conf"]["format"].replace(",","-"), cmd["image_conf"]["arches"], cmd["image_conf"]["variant"]) + self.pool.log_info("[BEGIN] %s" % msg) + + koji_wrapper = KojiWrapper(compose.conf["koji_profile"]) + # paths module doesn't hold compose object, so we have to generate path here + + # writes conf file for koji image-build + self.pool.log_info("Writing image-build config for %s.%s into %s" % (cmd["image_conf"]["variant"], cmd["image_conf"]["arches"], cmd["conf_file"])) + koji_cmd = koji_wrapper.get_image_build_cmd(cmd['image_conf'], conf_file_dest=cmd["conf_file"], wait=True, scratch=False) + + # avoid race conditions? + # Kerberos authentication failed: Permission denied in replay cache code (-1765328215) + time.sleep(num * 3) + output = koji_wrapper.run_create_image_cmd(koji_cmd, log_file=log_file) + self.pool.log_debug("build-image outputs: %s" % (output)) + if output["retcode"] != 0: + self.fail(compose, cmd) + raise RuntimeError("ImageBuild task failed: %s. See %s for more details." % (output["task_id"], log_file)) + + # copy image to images/ + image_infos = [] + + for filename in koji_wrapper.get_image_path(output["task_id"]): + # format is list of tuples [('qcow2', '.qcow2'), ('raw-xz', 'raw.xz'),] + for format, suffix in cmd['format']: + if filename.endswith(suffix): + image_infos.append({'filename': filename, 'suffix': suffix, 'type': format}) # the type/format ... image-build has it wrong + + if len(image_infos) != len(cmd['format']): + self.pool.log_error("Error in koji task %s. Expected to find same amount of images as in suffixes attr in image-build (%s). Got '%s'." % + (output["task_id"], len(cmd['image_conf']['format']), len(image_infos))) + self.fail(compose, cmd) + + # The usecase here is that you can run koji image-build with multiple --format + # It's ok to do it serialized since we're talking about max 2 images per single + # image_build record + linker = Linker(logger=compose._logger) + for image_info in image_infos: + # let's not change filename of koji outputs + image_dest = os.path.join(cmd["image_dir"], os.path.basename(image_info['filename'])) + linker.link(image_info['filename'], image_dest, link_type=cmd["link_type"]) + + iso = IsoWrapper(logger=compose._logger) # required for checksums only + checksum_cmd = ["cd %s" % pipes.quote(os.path.dirname(image_dest))] + checksum_cmd.extend(iso.get_checksum_cmds(os.path.basename(image_dest))) + checksum_cmd = " && ".join(checksum_cmd) + + if runroot: + packages = ["coreutils", "genisoimage", "isomd5sum", "jigdo", "strace", "lsof"] + runroot_channel = compose.conf.get("runroot_channel", None) + runroot_tag = compose.conf["runroot_tag"] + koji_cmd = koji_wrapper.get_runroot_cmd(runroot_tag, cmd["image_conf"]["arches"], checksum_cmd, channel=runroot_channel, use_shell=True, task_id=True, packages=packages, mounts=mounts) + + # avoid race conditions? + # Kerberos authentication failed: Permission denied in replay cache code (-1765328215) + time.sleep(num * 3) + + output = koji_wrapper.run_runroot_cmd(koji_cmd, log_file=log_file) + if output["retcode"] != 0: + self.fail(compose, cmd) + raise RuntimeError("Runroot task failed: %s. See %s for more details." % (output["task_id"], log_file)) + + else: + # run locally + try: + run(checksum_cmd, show_cmd=True, logfile=log_file) + except: + self.fail(compose, cmd) + raise + + # Update image manifest + img = Image(compose.im) + img.type = image_info['type'] + img.format = image_info['suffix'] + img.path = os.path.join(cmd["relative_image_dir"], os.path.basename(image_dest)) + img.mtime = int(os.stat(image_dest).st_mtime) + img.size = os.path.getsize(image_dest) + img.arch = cmd["image_conf"]["arches"] # arches should be always single arch + img.disc_number = 1 # We don't expect multiple disks + img.disc_count = 1 + for checksum_type in ("md5", "sha1", "sha256"): + checksum_path = image_dest + ".%sSUM" % checksum_type.upper() + checksum_value = None + if os.path.isfile(checksum_path): + checksum_value, image_name = read_checksum_file(checksum_path)[0] + if image_name != os.path.basename(img.path): + raise ValueError("Image name doesn't match checksum: %s" % checksum_path) + img.add_checksum(compose.paths.compose.topdir(), checksum_type=checksum_type, checksum_value=checksum_value) + img.bootable = False + # named keywords due portability (old productmd used arch, variant ... while new one uses variant, arch + compose.im.add(variant=cmd["image_conf"]["variant"].uid, arch=cmd["image_conf"]["arches"], image=img) + + self.pool.log_info("[DONE ] %s" % msg) diff --git a/pungi/wrappers/kojiwrapper.py b/pungi/wrappers/kojiwrapper.py index bb9f11e0..c3a4ebba 100644 --- a/pungi/wrappers/kojiwrapper.py +++ b/pungi/wrappers/kojiwrapper.py @@ -22,6 +22,7 @@ import re import koji import rpmUtils.arch from kobo.shortcuts import run +from ConfigParser import ConfigParser class KojiWrapper(object): @@ -97,6 +98,35 @@ class KojiWrapper(object): } return result + def get_image_build_cmd(self, config_options, conf_file_dest, wait=True, scratch=False): + """ + @param config_options + @param conf_file_dest - a destination in compose workdir for the conf file to be written + @param wait=True + @param scratch=False + """ + # Usage: koji image-build [options] [...] + sub_command = "image-build" + # The minimum set of options + min_options = ("name", "version", "target", "install_tree", "arches", "format", "kickstart", "ksurl", "distro") + assert set(min_options).issubset(set(config_options.keys())), "image-build requires at least %s got '%s'" % (", ".join(min_options), config_options) + cfg_parser = ConfigParser() + cfg_parser.add_section(sub_command) + for option, value in config_options.iteritems(): + cfg_parser.set(sub_command, option, value) + + fd = open(conf_file_dest, "w") + cfg_parser.write(fd) + fd.close() + + cmd = [self.executable, sub_command, "--config=%s" % conf_file_dest] + if wait: + cmd.append("--wait") + if scratch: + cmd.append("--scratch") + + return cmd + def get_create_image_cmd(self, name, version, target, arch, ks_file, repos, image_type="live", image_format=None, release=None, wait=True, archive=False, specfile=None): # Usage: koji spin-livecd [options] # Usage: koji spin-appliance [options] @@ -163,12 +193,14 @@ class KojiWrapper(object): def run_create_image_cmd(self, command, log_file=None): # spin-{livecd,appliance} is blocking by default -> you probably want to run it in a thread - - retcode, output = run(command, can_fail=True, logfile=log_file) + try: + retcode, output = run(command, can_fail=True, logfile=log_file) + except RuntimeError, e: + raise RuntimeError("%s. %s failed with '%s'" % (e, command, output)) match = re.search(r"Created task: (\d+)", output) if not match: - raise RuntimeError("Could not find task ID in output") + raise RuntimeError("Could not find task ID in output. Command '%s' returned '%s'." % (" ".join(command), output)) result = { "retcode": retcode, @@ -179,7 +211,6 @@ class KojiWrapper(object): def get_image_path(self, task_id): result = [] - # XXX: hardcoded URL koji_proxy = self.koji_module.ClientSession(self.koji_module.config.server) task_info_list = [] task_info_list.append(koji_proxy.getTaskInfo(task_id, request=True)) @@ -188,7 +219,7 @@ class KojiWrapper(object): # scan parent and child tasks for certain methods task_info = None for i in task_info_list: - if i["method"] in ("createAppliance", "createLiveCD"): + if i["method"] in ("createAppliance", "createLiveCD", 'createImage'): task_info = i break