Add image-build support

Signed-off-by: Lubos Kocman <lkocman@redhat.com>

bin/pungi-koji

@@ -211,6 +211,7 @@ def run_compose(compose):
     extrafiles_phase = pungi.phases.ExtraFilesPhase(compose, pkgset_phase)
     createiso_phase = pungi.phases.CreateisoPhase(compose)
     liveimages_phase = pungi.phases.LiveImagesPhase(compose)
+    image_build_phase = pungi.phases.ImageBuildPhase(compose)
     test_phase = pungi.phases.TestPhase(compose)
 
     # check if all config options are set
@@ -279,9 +280,11 @@ def run_compose(compose):
     # CREATEISO and LIVEIMAGES phases
     createiso_phase.start()
     liveimages_phase.start()
+    image_build_phase.start()
 
     createiso_phase.stop()
     liveimages_phase.stop()
+    image_build_phase.stop()
 
     # merge checksum files
     for variant in compose.get_variants(types=["variant", "layered-product"]):

doc/configuration.rst

@@ -467,3 +467,80 @@ Example
             'src': True
         }),
     ]
+
+Image Build Settings
+====================
+
+**image_build**
+    (*list*) -- config for koji image-build; format: [(variant_uid_regex, {arch|*: [{opt: value}])]
+
+.. note::
+    Config can contain anything what is accepted by
+    koji image-build --config configfile.ini
+    Repo is currently the only option which is being automatically transformed
+    into a string.
+
+    Please don't set install_tree as it would get overriden by pungi.
+    The 'format' attr is [('image_type', 'image_suffix'), ...].
+    productmd should ideally contain all of image types and suffixes.
+
+Example
+-------
+::
+
+    image_build = [
+        ('^Server$', {
+            'x86_64': [
+                {
+                    'format': [('docker', 'tar.gz'), ('qcow2', 'qcow2')]
+                    'name': 'fedora-qcow-and-docker-base',
+                    'target': 'koji-target-name',
+                    'ksversion': 'F23', # value from pykickstart
+                    'version': '23',
+                    'ksurl': 'https://git.fedorahosted.org/git/spin-kickstarts.git?somedirectoryifany#HEAD',
+                    'kickstart': "fedora-docker-base.ks",
+                    'repo': ["http://someextrarepos.org/repo", "ftp://rekcod.oi/repo].
+    #               'install_tree': 'http://sometpath',  # this is set automatically by pungi to os_dir for given variant/$arch 
+                    'distro': 'Fedora-20',
+                    'disk_size': 3
+                },
+                {
+                    'format': [('qcow2','qcow2')]
+                    'name': 'fedora-qcow-base',
+                    'target': 'koji-target-name',
+                    'ksversion': 'F23', # value from pykickstart
+                    'version': '23',
+                    'ksurl': 'https://git.fedorahosted.org/git/spin-kickstarts.git?somedirectoryifany#HEAD',
+                    'kickstart': "fedora-docker-base.ks",
+                    'distro': 'Fedora-23'
+                }
+            ]
+       }),
+    ]
+
+Translate Paths Settings
+========================
+
+**translate_paths**
+    (*list*) -- list of paths to translate; format: [(path,translated_path)]
+
+.. note::
+    This feature becomes useful when you need to transform compose location
+    into e.g. a http repo which is can be passed to koji image-build.
+    Translation needs to be invoked by a function call in pungi.
+    os.path.normpath() is applied on both path and translated_path
+    
+
+Example config
+--------------
+::
+    translate_paths = [
+        ("/mnt/a", "http://b/dir"),
+    ]
+
+Example usage
+-------------
+::
+    >>> from pungi.paths import translate_paths
+    >>> print translate_paths(compose_object_with_mapping, "/mnt/a/c/somefile")
+    http://b/dir/c/somefile

pungi/paths.py

@@ -307,6 +307,34 @@ class WorkPaths(object):
         path = os.path.join(path, file_name)
         return path
 
+    def image_build_dir(self, arch, variant, create_dir=True):
+        """
+        @param arch
+        @param variant
+        @param create_dir=True
+
+        Examples:
+            work/x86_64/Server/image-build
+        """
+        path = os.path.join(self.topdir(arch, create_dir=create_dir), variant.uid, "image-build")
+        if create_dir:
+            makedirs(path)
+        return path
+
+    def image_build_conf(self, arch, variant, image_name, image_type, create_dir=True):
+        """
+        @param arch
+        @param variant
+        @param image-name
+        @param image-type (e.g docker)
+        @param create_dir=True
+
+        Examples:
+            work/x86_64/Server/image-build/docker_rhel-server-docker.cfg
+        """
+        path = os.path.join(self.image_build_dir(arch, variant), "%s_%s.cfg" % (image_type, image_name))
+        return path
+
 
 class ComposePaths(object):
     def __init__(self, compose):
@@ -511,6 +539,45 @@ class ComposePaths(object):
         result = os.path.join(path, file_name)
         return result
 
+    def image_dir(self, arch, variant, symlink_to=None, create_dir=True, relative=False):
+        """
+        Examples:
+            compose/Server/x86_64/images
+            None
+        @param arch
+        @param variant
+        @param symlink_to=None
+        @param create_dir=True
+        @param relative=False
+        """
+        # skip optional, addons and src architecture
+        if variant.type != "variant":
+            return None
+        if arch == "src":
+            return None
+
+        path = os.path.join(self.topdir(arch, variant, create_dir=create_dir, relative=relative), "images")
+        if symlink_to:
+            topdir = self.compose.topdir.rstrip("/") + "/"
+            relative_dir = path[len(topdir):]
+            target_dir = os.path.join(symlink_to, self.compose.compose_id, relative_dir)
+            if create_dir and not relative:
+                makedirs(target_dir)
+            try:
+                os.symlink(target_dir, path)
+            except OSError as ex:
+                if ex.errno != errno.EEXIST:
+                    raise
+                msg = "Symlink pointing to '%s' expected: %s" % (target_dir, path)
+                if not os.path.islink(path):
+                    raise RuntimeError(msg)
+                if os.path.abspath(os.readlink(path)) != target_dir:
+                    raise RuntimeError(msg)
+        else:
+            if create_dir and not relative:
+                makedirs(path)
+        return path
+
     def jigdo_dir(self, arch, variant, create_dir=True, relative=False):
         """
         Examples:

pungi/phases/__init__.py

@@ -25,4 +25,5 @@ from buildinstall import BuildinstallPhase  # noqa
 from extra_files import ExtraFilesPhase  # noqa
 from createiso import CreateisoPhase  # noqa
 from live_images import LiveImagesPhase  # noqa
+from image_build import ImageBuildPhase  # noqa
 from test import TestPhase  # noqa

pungi/phases/image_build.py

@@ -0,0 +1,166 @@
+# -*- coding: utf-8 -*-
+
+
+import os
+import time
+import pipes
+
+from pungi.util import get_arch_variant_data
+from pungi.phases.base import PhaseBase
+from pungi.linker import Linker
+from pungi.paths import translate_path
+from pungi.wrappers.kojiwrapper import KojiWrapper
+from pungi.wrappers.iso import IsoWrapper
+from kobo.shortcuts import run, read_checksum_file
+from kobo.threads import ThreadPool, WorkerThread
+from productmd.images import Image
+
+class ImageBuildPhase(PhaseBase):
+    """class for wrapping up koji image-build"""
+    name = "image_build"
+
+    def __init__(self, compose):
+        PhaseBase.__init__(self, compose)
+        self.pool = ThreadPool(logger=self.compose._logger)
+
+    def skip(self):
+        if PhaseBase.skip(self):
+            return True
+        if not self.compose.conf.get(self.name):
+            self.log_info("Section '%s' was not found. Skipping" % self.name)
+            return True
+        return False
+
+    def run(self):
+        for arch in self.compose.get_arches(): # src will be skipped
+            for variant in self.compose.get_variants(arch=arch):
+                image_build_data = get_arch_variant_data(self.compose.conf, self.name, arch, variant)
+                for image_conf in image_build_data:
+                    image_conf["arches"] = arch # passed to get_image_build_cmd as dict
+                    image_conf["variant"] = variant # ^
+                    image_conf["install_tree"] = translate_path(self.compose, self.compose.paths.compose.os_tree(arch, variant)) # ^
+                    format = image_conf["format"] # transform format into right 'format' for image-build
+                    image_conf["format"] = ",".join([x[0] for x in image_conf["format"]]) # 'docker,qcow2'
+                    if image_conf.has_key("repos") and not isinstance(image_conf["repos"], str):
+                        image_conf["repos"] = ",".join(image_conf["repos"]) # supply repos as str separated by , instead of list
+                    cmd = {
+                        "format": format,
+                        "image_conf": image_conf,
+                        "conf_file": self.compose.paths.work.image_build_conf(image_conf["arches"], image_conf['variant'], image_name=image_conf['name'], image_type=image_conf['format'].replace(",", "-")),
+                        "image_dir": self.compose.paths.compose.image_dir(arch, variant),
+                        "relative_image_dir": self.compose.paths.compose.image_dir(arch, variant, create_dir=False, relative=True),
+                        "link_type": self.compose.conf.get("link_type", "hardlink-or-copy")
+                    }
+                    self.pool.add(CreateImageBuildThread(self.pool))
+                    self.pool.queue_put((self.compose, cmd))
+        self.pool.start()
+
+    def stop(self, *args, **kwargs):
+        PhaseBase.stop(self, *args, **kwargs)
+        if self.skip():
+            return
+
+class CreateImageBuildThread(WorkerThread):
+    def fail(self, compose, cmd):
+        compose.log_error("CreateImageBuild failed.")
+
+    def process(self, item, num):
+        compose, cmd = item
+        mounts = [compose.topdir]
+        if "mount" in cmd:
+            mounts.append(cmd["mount"])
+        runroot = compose.conf.get("runroot", False)
+        log_file = compose.paths.log.log_file(cmd["image_conf"]["arches"], "imagebuild-%s-%s-%s" % (cmd["image_conf"]["arches"], cmd["image_conf"]["variant"], cmd['image_conf']['format'].replace(",","-")))
+        msg = "Creating %s image (arch: %s, variant: %s)" % (cmd["image_conf"]["format"].replace(",","-"), cmd["image_conf"]["arches"], cmd["image_conf"]["variant"])
+        self.pool.log_info("[BEGIN] %s" % msg)
+
+        koji_wrapper = KojiWrapper(compose.conf["koji_profile"])
+        # paths module doesn't hold compose object, so we have to generate path here
+
+        # writes conf file for koji image-build
+        self.pool.log_info("Writing image-build config for %s.%s into %s" % (cmd["image_conf"]["variant"], cmd["image_conf"]["arches"], cmd["conf_file"]))
+        koji_cmd = koji_wrapper.get_image_build_cmd(cmd['image_conf'], conf_file_dest=cmd["conf_file"], wait=True, scratch=False)
+
+        # avoid race conditions?
+        # Kerberos authentication failed: Permission denied in replay cache code (-1765328215)
+        time.sleep(num * 3)
+        output = koji_wrapper.run_create_image_cmd(koji_cmd, log_file=log_file)
+        self.pool.log_debug("build-image outputs: %s" % (output))
+        if output["retcode"] != 0:
+            self.fail(compose, cmd)
+            raise RuntimeError("ImageBuild task failed: %s. See %s for more details." % (output["task_id"], log_file))
+
+        # copy image to images/
+        image_infos = []
+
+        for filename in koji_wrapper.get_image_path(output["task_id"]):
+            # format is list of tuples [('qcow2', '.qcow2'), ('raw-xz', 'raw.xz'),]
+            for format, suffix in cmd['format']:
+                if filename.endswith(suffix):
+                    image_infos.append({'filename': filename, 'suffix': suffix, 'type': format}) # the type/format ... image-build has it wrong
+
+        if len(image_infos) != len(cmd['format']):
+            self.pool.log_error("Error in koji task %s. Expected to find same amount of images as in suffixes attr in image-build (%s). Got '%s'." %
+                (output["task_id"], len(cmd['image_conf']['format']), len(image_infos)))
+            self.fail(compose, cmd)
+
+        # The usecase here is that you can run koji image-build with multiple --format
+        # It's ok to do it serialized since we're talking about max 2 images per single
+        # image_build record
+        linker = Linker(logger=compose._logger)
+        for image_info in image_infos:
+            # let's not change filename of koji outputs
+            image_dest = os.path.join(cmd["image_dir"], os.path.basename(image_info['filename']))
+            linker.link(image_info['filename'], image_dest, link_type=cmd["link_type"])
+
+            iso = IsoWrapper(logger=compose._logger) # required for checksums only
+            checksum_cmd = ["cd %s" % pipes.quote(os.path.dirname(image_dest))]
+            checksum_cmd.extend(iso.get_checksum_cmds(os.path.basename(image_dest)))
+            checksum_cmd = " && ".join(checksum_cmd)
+
+            if runroot:
+                packages = ["coreutils", "genisoimage", "isomd5sum", "jigdo", "strace", "lsof"]
+                runroot_channel = compose.conf.get("runroot_channel", None)
+                runroot_tag = compose.conf["runroot_tag"]
+                koji_cmd = koji_wrapper.get_runroot_cmd(runroot_tag, cmd["image_conf"]["arches"], checksum_cmd, channel=runroot_channel, use_shell=True, task_id=True, packages=packages, mounts=mounts)
+
+                # avoid race conditions?
+                # Kerberos authentication failed: Permission denied in replay cache code (-1765328215)
+                time.sleep(num * 3)
+
+                output = koji_wrapper.run_runroot_cmd(koji_cmd, log_file=log_file)
+                if output["retcode"] != 0:
+                    self.fail(compose, cmd)
+                    raise RuntimeError("Runroot task failed: %s. See %s for more details." % (output["task_id"], log_file))
+
+            else:
+                # run locally
+                try:
+                    run(checksum_cmd, show_cmd=True, logfile=log_file)
+                except:
+                    self.fail(compose, cmd)
+                    raise
+
+            # Update image manifest
+            img = Image(compose.im)
+            img.type = image_info['type']
+            img.format = image_info['suffix']
+            img.path = os.path.join(cmd["relative_image_dir"], os.path.basename(image_dest))
+            img.mtime = int(os.stat(image_dest).st_mtime)
+            img.size = os.path.getsize(image_dest)
+            img.arch = cmd["image_conf"]["arches"] # arches should be always single arch
+            img.disc_number = 1 # We don't expect multiple disks
+            img.disc_count = 1
+            for checksum_type in ("md5", "sha1", "sha256"):
+                checksum_path = image_dest + ".%sSUM" % checksum_type.upper()
+                checksum_value = None
+                if os.path.isfile(checksum_path):
+                    checksum_value, image_name = read_checksum_file(checksum_path)[0]
+                    if image_name != os.path.basename(img.path):
+                        raise ValueError("Image name doesn't match checksum: %s" % checksum_path)
+                img.add_checksum(compose.paths.compose.topdir(), checksum_type=checksum_type, checksum_value=checksum_value)
+            img.bootable = False
+            # named keywords due portability (old productmd used arch, variant ... while new one uses variant, arch
+            compose.im.add(variant=cmd["image_conf"]["variant"].uid, arch=cmd["image_conf"]["arches"], image=img)
+
+        self.pool.log_info("[DONE ] %s" % msg)

pungi/wrappers/kojiwrapper.py

@@ -22,6 +22,7 @@ import re
 import koji
 import rpmUtils.arch
 from kobo.shortcuts import run
+from ConfigParser import ConfigParser
 
 
 class KojiWrapper(object):
@@ -97,6 +98,35 @@ class KojiWrapper(object):
         }
         return result
 
+    def get_image_build_cmd(self, config_options, conf_file_dest, wait=True, scratch=False):
+        """
+        @param config_options
+        @param conf_file_dest -  a destination in compose workdir for the conf file to be written
+        @param wait=True
+        @param scratch=False
+        """
+        # Usage: koji image-build [options] <name> <version> <target> <install-tree-url> <arch> [<arch>...]
+        sub_command = "image-build"
+        # The minimum set of options
+        min_options = ("name", "version", "target", "install_tree", "arches", "format", "kickstart", "ksurl", "distro")
+        assert set(min_options).issubset(set(config_options.keys())), "image-build requires at least %s got '%s'" % (", ".join(min_options), config_options)
+        cfg_parser = ConfigParser()
+        cfg_parser.add_section(sub_command)
+        for option, value in config_options.iteritems():
+            cfg_parser.set(sub_command, option, value)
+
+        fd = open(conf_file_dest, "w")
+        cfg_parser.write(fd)
+        fd.close()
+
+        cmd = [self.executable, sub_command, "--config=%s" % conf_file_dest]
+        if wait:
+            cmd.append("--wait")
+        if scratch:
+            cmd.append("--scratch")
+
+        return cmd
+
     def get_create_image_cmd(self, name, version, target, arch, ks_file, repos, image_type="live", image_format=None, release=None, wait=True, archive=False, specfile=None):
         # Usage: koji spin-livecd [options] <name> <version> <target> <arch> <kickstart-file>
         # Usage: koji spin-appliance [options] <name> <version> <target> <arch> <kickstart-file>
@@ -163,12 +193,14 @@ class KojiWrapper(object):
 
     def run_create_image_cmd(self, command, log_file=None):
         # spin-{livecd,appliance} is blocking by default -> you probably want to run it in a thread
-
+        try:
             retcode, output = run(command, can_fail=True, logfile=log_file)
+        except RuntimeError, e:
+            raise RuntimeError("%s. %s failed with '%s'" % (e, command, output))
 
         match = re.search(r"Created task: (\d+)", output)
         if not match:
-            raise RuntimeError("Could not find task ID in output")
+            raise RuntimeError("Could not find task ID in output. Command '%s' returned '%s'." % (" ".join(command), output))
 
         result = {
             "retcode": retcode,
@@ -179,7 +211,6 @@ class KojiWrapper(object):
 
     def get_image_path(self, task_id):
         result = []
-        # XXX: hardcoded URL
         koji_proxy = self.koji_module.ClientSession(self.koji_module.config.server)
         task_info_list = []
         task_info_list.append(koji_proxy.getTaskInfo(task_id, request=True))
@@ -188,7 +219,7 @@ class KojiWrapper(object):
         # scan parent and child tasks for certain methods
         task_info = None
         for i in task_info_list:
-            if i["method"] in ("createAppliance", "createLiveCD"):
+            if i["method"] in ("createAppliance", "createLiveCD", 'createImage'):
                 task_info = i
                 break