From 9c1418eb0a30225965d80d30447fb6ebae9a85cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lubom=C3=ADr=20Sedl=C3=A1=C5=99?= Date: Tue, 5 Jan 2016 09:27:20 +0100 Subject: [PATCH] [image-build] Use single koji task per variant MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Given a list of arches, koji can build multiple images in one go (automatically starting children tasks for each one). This causes a bunch of changes: * The configuration no longer allows changing config based on architecture, only variants are allowed. It is however possible to filter which arches are used for the building in the variant. * The configuration files for koji image-build are stored in work/image-build/$variant (not split based on arch). This patch also changes the option name that is passed to koji image-build: the repos should be specified under key `repo` (without the trailing slash). Signed-off-by: Lubomír Sedlář --- doc/configuration.rst | 79 +++++----- pungi/paths.py | 27 ++-- pungi/phases/image_build.py | 152 +++++++++++------- pungi/util.py | 21 +++ pungi/wrappers/kojiwrapper.py | 35 +++++ tests/test_imagebuildphase.py | 281 ++++++++++++++++++++++++++++++++++ tests/test_koji_wrapper.py | 258 +++++++++++++++++++++++++++++++ tests/test_util.py | 34 ++++ 8 files changed, 780 insertions(+), 107 deletions(-) create mode 100755 tests/test_imagebuildphase.py create mode 100755 tests/test_koji_wrapper.py diff --git a/doc/configuration.rst b/doc/configuration.rst index a3cdb681..717d1551 100644 --- a/doc/configuration.rst +++ b/doc/configuration.rst @@ -598,18 +598,22 @@ Image Build Settings ==================== **image_build** - (*list*) -- config for koji image-build; format: [(variant_uid_regex, {arch|*: [{opt: value}])] + (*dict*) -- config for ``koji image-build``; format: {variant_uid_regex: [{opt: value}]} + + By default, images will be built for each binary arch valid for the + variant. The config can specify a list of arches to narrow this down. .. note:: Config can contain anything what is accepted by ``koji image-build --config configfile.ini`` - Repo is currently the only option which is being automatically transformed - into a string. + Repo can be specified either as a string or a list of strings. It will + automatically transformed into format suitable for ``koji``. A repo for the + currently built variant will be added as well. - Please don't set install_tree as it would get overriden by pungi. - The 'format' attr is [('image_type', 'image_suffix'), ...]. - productmd should ideally contain all of image types and suffixes. + Please don't set ``install_tree`` as it would get overriden by pungi. + The ``format`` attr is [('image_type', 'image_suffix'), ...]. + See productmd documentation for list of supported types and suffixes. If ``ksurl`` ends with ``#HEAD``, Pungi will figure out the SHA1 hash of current HEAD and use that instead. @@ -619,36 +623,39 @@ Example ------- :: - image_build = [ - ('^Server$', { - 'x86_64': [ - { - 'format': [('docker', 'tar.gz'), ('qcow2', 'qcow2')] - 'name': 'fedora-qcow-and-docker-base', - 'target': 'koji-target-name', - 'ksversion': 'F23', # value from pykickstart - 'version': '23', - # correct SHA1 hash will be put into the URL below automatically - 'ksurl': 'https://git.fedorahosted.org/git/spin-kickstarts.git?somedirectoryifany#HEAD', - 'kickstart': "fedora-docker-base.ks", - 'repo': ["http://someextrarepos.org/repo", "ftp://rekcod.oi/repo]. - # 'install_tree': 'http://sometpath', # this is set automatically by pungi to os_dir for given variant/$arch - 'distro': 'Fedora-20', - 'disk_size': 3 - }, - { - 'format': [('qcow2','qcow2')] - 'name': 'fedora-qcow-base', - 'target': 'koji-target-name', - 'ksversion': 'F23', # value from pykickstart - 'version': '23', - 'ksurl': 'https://git.fedorahosted.org/git/spin-kickstarts.git?somedirectoryifany#HEAD', - 'kickstart': "fedora-docker-base.ks", - 'distro': 'Fedora-23' - } - ] - }), - ] + image_build = { + '^Server$': [ + { + 'format': [('docker', 'tar.gz'), ('qcow2', 'qcow2')] + 'name': 'fedora-qcow-and-docker-base', + 'target': 'koji-target-name', + 'ksversion': 'F23', # value from pykickstart + 'version': '23', + # correct SHA1 hash will be put into the URL below automatically + 'ksurl': 'https://git.fedorahosted.org/git/spin-kickstarts.git?somedirectoryifany#HEAD', + 'kickstart': "fedora-docker-base.ks", + 'repo': ["http://someextrarepos.org/repo", "ftp://rekcod.oi/repo]. + 'distro': 'Fedora-20', + 'disk_size': 3, + + # this is set automatically by pungi to os_dir for given variant + # 'install_tree': 'http://sometpath', + }, + { + 'format': [('qcow2','qcow2')] + 'name': 'fedora-qcow-base', + 'target': 'koji-target-name', + 'ksversion': 'F23', # value from pykickstart + 'version': '23', + 'ksurl': 'https://git.fedorahosted.org/git/spin-kickstarts.git?somedirectoryifany#HEAD', + 'kickstart': "fedora-docker-base.ks", + 'distro': 'Fedora-23', + + # only build this type of image on x86_64 + 'arches': ['x86_64'] + } + ] + } Media Checksums Settings diff --git a/pungi/paths.py b/pungi/paths.py index 7cc7dd0c..64856932 100644 --- a/pungi/paths.py +++ b/pungi/paths.py @@ -305,32 +305,30 @@ class WorkPaths(object): path = os.path.join(path, file_name) return path - def image_build_dir(self, arch, variant, create_dir=True): + def image_build_dir(self, variant, create_dir=True): """ - @param arch @param variant @param create_dir=True Examples: - work/x86_64/Server/image-build + work/image-build/Server """ - path = os.path.join(self.topdir(arch, create_dir=create_dir), variant.uid, "image-build") + path = os.path.join(self.topdir('image-build', create_dir=create_dir), variant.uid) if create_dir: makedirs(path) return path - def image_build_conf(self, arch, variant, image_name, image_type, create_dir=True): + def image_build_conf(self, variant, image_name, image_type, create_dir=True): """ - @param arch @param variant @param image-name @param image-type (e.g docker) @param create_dir=True Examples: - work/x86_64/Server/image-build/docker_rhel-server-docker.cfg + work/image-build/Server/docker_rhel-server-docker.cfg """ - path = os.path.join(self.image_build_dir(arch, variant), "%s_%s.cfg" % (image_type, image_name)) + path = os.path.join(self.image_build_dir(variant), "%s_%s.cfg" % (image_type, image_name)) return path @@ -517,24 +515,23 @@ class ComposePaths(object): return os.path.join(path, filename) - def image_dir(self, arch, variant, symlink_to=None, create_dir=True, relative=False): + def image_dir(self, variant, symlink_to=None, create_dir=True, relative=False): """ + The arch is listed as literal '%(arch)s' Examples: - compose/Server/x86_64/images + compose/Server/%(arch)s/images None - @param arch @param variant @param symlink_to=None @param create_dir=True @param relative=False """ - # skip optional, addons and src architecture + # skip optional and addons if variant.type != "variant": return None - if arch == "src": - return None - path = os.path.join(self.topdir(arch, variant, create_dir=create_dir, relative=relative), "images") + path = os.path.join(self.topdir('%(arch)s', variant, create_dir=create_dir, relative=relative), + "images") if symlink_to: topdir = self.compose.topdir.rstrip("/") + "/" relative_dir = path[len(topdir):] diff --git a/pungi/phases/image_build.py b/pungi/phases/image_build.py index 6c88da78..436ccbc5 100644 --- a/pungi/phases/image_build.py +++ b/pungi/phases/image_build.py @@ -4,7 +4,7 @@ import copy import os import time -from pungi.util import get_arch_variant_data, resolve_git_url +from pungi.util import get_variant_data, resolve_git_url from pungi.phases.base import PhaseBase from pungi.linker import Linker from pungi.paths import translate_path @@ -30,46 +30,65 @@ class ImageBuildPhase(PhaseBase): return False def run(self): - for arch in self.compose.get_arches(): # src will be skipped - for variant in self.compose.get_variants(arch=arch): - image_build_data = get_arch_variant_data(self.compose.conf, self.name, arch, variant) - for image_conf in image_build_data: - # We will modify the data, so we need to make a copy to - # prevent problems in next iteration where the original - # value is needed. - image_conf = copy.deepcopy(image_conf) + for variant in self.compose.get_variants(): + arches = set([x for x in variant.arches if x != 'src']) - # Replace possible ambiguous ref name with explicit hash. - if 'ksurl' in image_conf: - image_conf['ksurl'] = resolve_git_url(image_conf['ksurl']) - image_conf["arches"] = arch # passed to get_image_build_cmd as dict - image_conf["variant"] = variant # ^ - image_conf["install_tree"] = translate_path(self.compose, self.compose.paths.compose.os_tree(arch, variant)) # ^ - format = image_conf["format"] # transform format into right 'format' for image-build - image_conf["format"] = ",".join([x[0] for x in image_conf["format"]]) # 'docker,qcow2' + for image_conf in get_variant_data(self.compose.conf, self.name, variant): + # We will modify the data, so we need to make a copy to + # prevent problems in next iteration where the original + # value is needed. + image_conf = copy.deepcopy(image_conf) - repos = image_conf.get('repos', []) - if isinstance(repos, str): - repos = [repos] - repos.append(translate_path(self.compose, self.compose.paths.compose.os_tree(arch, variant))) - image_conf['repos'] = ",".join(repos) # supply repos as str separated by , instead of list + # Replace possible ambiguous ref name with explicit hash. + if 'ksurl' in image_conf: + image_conf['ksurl'] = resolve_git_url(image_conf['ksurl']) + + # image_conf is passed to get_image_build_cmd as dict + + if 'arches' in image_conf: + image_conf["arches"] = ','.join(sorted(set(image_conf.get('arches', [])) & arches)) + else: + image_conf['arches'] = ','.join(sorted(arches)) + + if not image_conf['arches']: + continue + + image_conf["variant"] = variant + image_conf["install_tree"] = translate_path( + self.compose, + self.compose.paths.compose.os_tree('$arch', variant) + ) + # transform format into right 'format' for image-build + # e.g. 'docker,qcow2' + format = image_conf["format"] + image_conf["format"] = ",".join([x[0] for x in image_conf["format"]]) + + repo = image_conf.get('repo', []) + if isinstance(repo, str): + repo = [repo] + repo.append(translate_path(self.compose, self.compose.paths.compose.os_tree('$arch', variant))) + # supply repo as str separated by , instead of list + image_conf['repo'] = ",".join(repo) + + cmd = { + "format": format, + "image_conf": image_conf, + "conf_file": self.compose.paths.work.image_build_conf( + image_conf['variant'], + image_name=image_conf['name'], + image_type=image_conf['format'].replace(",", "-") + ), + "image_dir": self.compose.paths.compose.image_dir(variant), + "relative_image_dir": self.compose.paths.compose.image_dir( + variant, create_dir=False, relative=True + ), + "link_type": self.compose.conf.get("link_type", "hardlink-or-copy") + } + self.pool.add(CreateImageBuildThread(self.pool)) + self.pool.queue_put((self.compose, cmd)) - cmd = { - "format": format, - "image_conf": image_conf, - "conf_file": self.compose.paths.work.image_build_conf(image_conf["arches"], image_conf['variant'], image_name=image_conf['name'], image_type=image_conf['format'].replace(",", "-")), - "image_dir": self.compose.paths.compose.image_dir(arch, variant), - "relative_image_dir": self.compose.paths.compose.image_dir(arch, variant, create_dir=False, relative=True), - "link_type": self.compose.conf.get("link_type", "hardlink-or-copy") - } - self.pool.add(CreateImageBuildThread(self.pool)) - self.pool.queue_put((self.compose, cmd)) self.pool.start() - def stop(self, *args, **kwargs): - PhaseBase.stop(self, *args, **kwargs) - if self.skip(): - return class CreateImageBuildThread(WorkerThread): def fail(self, compose, cmd): @@ -77,19 +96,29 @@ class CreateImageBuildThread(WorkerThread): def process(self, item, num): compose, cmd = item + arches = cmd['image_conf']['arches'].split(',') + mounts = [compose.paths.compose.topdir()] if "mount" in cmd: mounts.append(cmd["mount"]) - log_file = compose.paths.log.log_file(cmd["image_conf"]["arches"], "imagebuild-%s-%s-%s" % (cmd["image_conf"]["arches"], cmd["image_conf"]["variant"], cmd['image_conf']['format'].replace(",","-"))) - msg = "Creating %s image (arch: %s, variant: %s)" % (cmd["image_conf"]["format"].replace(",","-"), cmd["image_conf"]["arches"], cmd["image_conf"]["variant"]) + log_file = compose.paths.log.log_file( + cmd["image_conf"]["arches"], + "imagebuild-%s-%s-%s" % ('-'.join(arches), + cmd["image_conf"]["variant"], + cmd['image_conf']['format'].replace(",", "-")) + ) + msg = "Creating %s image (arches: %s, variant: %s)" % (cmd["image_conf"]["format"].replace(",", "-"), + '-'.join(arches), + cmd["image_conf"]["variant"]) self.pool.log_info("[BEGIN] %s" % msg) koji_wrapper = KojiWrapper(compose.conf["koji_profile"]) - # paths module doesn't hold compose object, so we have to generate path here # writes conf file for koji image-build - self.pool.log_info("Writing image-build config for %s.%s into %s" % (cmd["image_conf"]["variant"], cmd["image_conf"]["arches"], cmd["conf_file"])) - koji_cmd = koji_wrapper.get_image_build_cmd(cmd['image_conf'], conf_file_dest=cmd["conf_file"], wait=True, scratch=False) + self.pool.log_info("Writing image-build config for %s.%s into %s" % ( + cmd["image_conf"]["variant"], '-'.join(arches), cmd["conf_file"])) + koji_cmd = koji_wrapper.get_image_build_cmd(cmd['image_conf'], + conf_file_dest=cmd["conf_file"]) # avoid race conditions? # Kerberos authentication failed: Permission denied in replay cache code (-1765328215) @@ -103,15 +132,22 @@ class CreateImageBuildThread(WorkerThread): # copy image to images/ image_infos = [] - for filename in koji_wrapper.get_image_path(output["task_id"]): - # format is list of tuples [('qcow2', '.qcow2'), ('raw-xz', 'raw.xz'),] - for format, suffix in cmd['format']: - if filename.endswith(suffix): - image_infos.append({'filename': filename, 'suffix': suffix, 'type': format}) # the type/format ... image-build has it wrong + paths = koji_wrapper.get_image_build_paths(output["task_id"]) - if len(image_infos) != len(cmd['format']): - self.pool.log_error("Error in koji task %s. Expected to find same amount of images as in suffixes attr in image-build (%s). Got '%s'." % - (output["task_id"], len(cmd['image_conf']['format']), len(image_infos))) + for arch, paths in paths.iteritems(): + for path in paths: + # format is list of tuples [('qcow2', '.qcow2'), ('raw-xz', 'raw.xz'),] + for format, suffix in cmd['format']: + if path.endswith(suffix): + image_infos.append({'path': path, 'suffix': suffix, 'type': format, 'arch': arch}) + break + + if len(image_infos) != len(cmd['format']) * len(arches): + self.pool.log_error( + "Error in koji task %s. Expected to find same amount of images " + "as in suffixes attr in image-build (%s) for each arch (%s). Got '%s'." % + (output["task_id"], len(cmd['format']), + len(arches), len(image_infos))) self.fail(compose, cmd) # The usecase here is that you can run koji image-build with multiple --format @@ -119,22 +155,26 @@ class CreateImageBuildThread(WorkerThread): # image_build record linker = Linker(logger=compose._logger) for image_info in image_infos: + image_dir = cmd["image_dir"] % {"arch": image_info['arch']} + relative_image_dir = cmd["relative_image_dir"] % {"arch": image_info['arch']} + # let's not change filename of koji outputs - image_dest = os.path.join(cmd["image_dir"], os.path.basename(image_info['filename'])) - linker.link(image_info['filename'], image_dest, link_type=cmd["link_type"]) + image_dest = os.path.join(image_dir, os.path.basename(image_info['path'])) + linker.link(image_info['path'], image_dest, link_type=cmd["link_type"]) # Update image manifest img = Image(compose.im) img.type = image_info['type'] img.format = image_info['suffix'] - img.path = os.path.join(cmd["relative_image_dir"], os.path.basename(image_dest)) + img.path = os.path.join(relative_image_dir, os.path.basename(image_dest)) img.mtime = int(os.stat(image_dest).st_mtime) img.size = os.path.getsize(image_dest) - img.arch = cmd["image_conf"]["arches"] # arches should be always single arch - img.disc_number = 1 # We don't expect multiple disks + img.arch = image_info['arch'] + img.disc_number = 1 # We don't expect multiple disks img.disc_count = 1 img.bootable = False - # named keywords due portability (old productmd used arch, variant ... while new one uses variant, arch - compose.im.add(variant=cmd["image_conf"]["variant"].uid, arch=cmd["image_conf"]["arches"], image=img) + compose.im.add(variant=cmd["image_conf"]["variant"].uid, + arch=image_info['arch'], + image=img) self.pool.log_info("[DONE ] %s" % msg) diff --git a/pungi/util.py b/pungi/util.py index facecd20..6828ae71 100644 --- a/pungi/util.py +++ b/pungi/util.py @@ -257,6 +257,27 @@ def get_arch_data(conf, var_name, arch): return result +def get_variant_data(conf, var_name, variant): + """Get configuration for variant. + + Expected config format is a mapping from variant_uid regexes to lists of + values. + + :param var_name: name of configuration key with which to work + :param variant: Variant object for which to get configuration + :rtype: a list of values + """ + result = [] + for conf_variant, conf_data in conf.get(var_name, {}).iteritems(): + if not re.match(conf_variant, variant.uid): + continue + if isinstance(conf_data, list): + result.extend(conf_data) + else: + result.append(conf_data) + return result + + def get_buildroot_rpms(compose, task_id): """Get build root RPMs - either from runroot or local""" result = [] diff --git a/pungi/wrappers/kojiwrapper.py b/pungi/wrappers/kojiwrapper.py index c3a4ebba..62e11751 100644 --- a/pungi/wrappers/kojiwrapper.py +++ b/pungi/wrappers/kojiwrapper.py @@ -209,6 +209,41 @@ class KojiWrapper(object): } return result + def get_image_build_paths(self, task_id): + """ + Given an image task in Koji, get a mapping from arches to a list of + paths to results of the task. + """ + result = {} + + # task = self.koji_proxy.getTaskInfo(task_id, request=True) + children_tasks = self.koji_proxy.getTaskChildren(task_id, request=True) + + for child_task in children_tasks: + if child_task['method'] != 'createImage': + continue + + is_scratch = child_task['request'][-1].get('scratch', False) + task_result = self.koji_proxy.getTaskResult(child_task['id']) + + if is_scratch: + topdir = os.path.join( + self.koji_module.pathinfo.work(), + self.koji_module.pathinfo.taskrelpath(child_task['id']) + ) + else: + build = self.koji_proxy.getImageBuild("%(name)s-%(version)s-%(release)s" % task_result) + build["name"] = task_result["name"] + build["version"] = task_result["version"] + build["release"] = task_result["release"] + build["arch"] = task_result["arch"] + topdir = self.koji_module.pathinfo.imagebuild(build) + + for i in task_result["files"]: + result.setdefault(task_result['arch'], []).append(os.path.join(topdir, i)) + + return result + def get_image_path(self, task_id): result = [] koji_proxy = self.koji_module.ClientSession(self.koji_module.config.server) diff --git a/tests/test_imagebuildphase.py b/tests/test_imagebuildphase.py new file mode 100755 index 00000000..448d783d --- /dev/null +++ b/tests/test_imagebuildphase.py @@ -0,0 +1,281 @@ +#!/usr/bin/env python2 +# -*- coding: utf-8 -*- + + +import unittest +import mock + +import os +import sys + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..")) + +from pungi.phases.image_build import ImageBuildPhase, CreateImageBuildThread + + +class _DummyCompose(object): + def __init__(self, config): + self.compose_date = '20151203' + self.compose_type_suffix = '.t' + self.compose_respin = 0 + self.ci_base = mock.Mock( + release_id='Test-1.0', + release=mock.Mock( + short='test', + version='1.0', + ), + ) + self.conf = config + self.paths = mock.Mock( + compose=mock.Mock( + topdir=mock.Mock(return_value='/a/b'), + os_tree=mock.Mock( + side_effect=lambda arch, variant: os.path.join('/ostree', arch, variant.uid) + ), + image_dir=mock.Mock( + side_effect=lambda variant, create_dir=False, relative=False: os.path.join( + '' if relative else '/', 'image_dir', variant.uid, '%(arch)s' + ) + ) + ), + work=mock.Mock( + image_build_conf=mock.Mock( + side_effect=lambda variant, image_name, image_type: + '-'.join([variant.uid, image_name, image_type]) + ) + ), + log=mock.Mock( + log_file=mock.Mock(return_value='/a/b/log/log_file') + ) + ) + self._logger = mock.Mock() + self.variants = [ + mock.Mock(uid='Server', arches=['x86_64', 'amd64']), + mock.Mock(uid='Client', arches=['amd64']), + ] + self.im = mock.Mock() + + def get_arches(self): + return ['x86_64', 'amd64'] + + def get_variants(self, arch=None, types=None): + return [v for v in self.variants if not arch or arch in v.arches] + + +class TestImageBuildPhase(unittest.TestCase): + + @mock.patch('pungi.phases.image_build.ThreadPool') + def test_image_build(self, ThreadPool): + compose = _DummyCompose({ + 'image_build': { + '^Client|Server$': [ + { + 'format': [('docker', 'tar.xz')], + 'name': 'Fedora-Docker-Base', + 'target': 'f24', + 'version': 'Rawhide', + 'ksurl': 'git://git.fedorahosted.org/git/spin-kickstarts.git', + 'kickstart': "fedora-docker-base.ks", + 'distro': 'Fedora-20', + 'disk_size': 3 + } + ] + }, + 'koji_profile': 'koji', + }) + + phase = ImageBuildPhase(compose) + + phase.run() + + # assert at least one thread was started + self.assertTrue(phase.pool.add.called) + client_args = { + "format": [('docker', 'tar.xz')], + "image_conf": { + 'install_tree': '/ostree/$arch/Client', + 'kickstart': 'fedora-docker-base.ks', + 'format': 'docker', + 'repo': '/ostree/$arch/Client', + 'variant': compose.variants[1], + 'target': 'f24', + 'disk_size': 3, + 'name': 'Fedora-Docker-Base', + 'arches': 'amd64', + 'version': 'Rawhide', + 'ksurl': 'git://git.fedorahosted.org/git/spin-kickstarts.git', + 'distro': 'Fedora-20', + }, + "conf_file": 'Client-Fedora-Docker-Base-docker', + "image_dir": '/image_dir/Client/%(arch)s', + "relative_image_dir": 'image_dir/Client/%(arch)s', + "link_type": 'hardlink-or-copy', + } + server_args = { + "format": [('docker', 'tar.xz')], + "image_conf": { + 'install_tree': '/ostree/$arch/Server', + 'kickstart': 'fedora-docker-base.ks', + 'format': 'docker', + 'repo': '/ostree/$arch/Server', + 'variant': compose.variants[0], + 'target': 'f24', + 'disk_size': 3, + 'name': 'Fedora-Docker-Base', + 'arches': 'amd64,x86_64', + 'version': 'Rawhide', + 'ksurl': 'git://git.fedorahosted.org/git/spin-kickstarts.git', + 'distro': 'Fedora-20', + }, + "conf_file": 'Server-Fedora-Docker-Base-docker', + "image_dir": '/image_dir/Server/%(arch)s', + "relative_image_dir": 'image_dir/Server/%(arch)s', + "link_type": 'hardlink-or-copy', + } + self.assertItemsEqual(phase.pool.queue_put.mock_calls, + [mock.call((compose, client_args)), + mock.call((compose, server_args))]) + + @mock.patch('pungi.phases.image_build.ThreadPool') + def test_image_build_filter_all_variants(self, ThreadPool): + compose = _DummyCompose({ + 'image_build': { + '^Client|Server$': [ + { + 'format': [('docker', 'tar.xz')], + 'name': 'Fedora-Docker-Base', + 'target': 'f24', + 'version': 'Rawhide', + 'ksurl': 'git://git.fedorahosted.org/git/spin-kickstarts.git', + 'kickstart': "fedora-docker-base.ks", + 'distro': 'Fedora-20', + 'disk_size': 3, + 'arches': ['non-existing'], + } + ] + }, + 'koji_profile': 'koji', + }) + + phase = ImageBuildPhase(compose) + + phase.run() + + # assert at least one thread was started + self.assertFalse(phase.pool.add.called) + self.assertFalse(phase.pool.queue_put.called) + + +class TestCreateImageBuildThread(unittest.TestCase): + + @mock.patch('pungi.phases.image_build.KojiWrapper') + @mock.patch('pungi.phases.image_build.Linker') + def test_process(self, Linker, KojiWrapper): + compose = _DummyCompose({ + 'koji_profile': 'koji' + }) + pool = mock.Mock() + cmd = { + "format": [('docker', 'tar.xz'), ('qcow2', 'qcow2')], + "image_conf": { + 'install_tree': '/ostree/$arch/Client', + 'kickstart': 'fedora-docker-base.ks', + 'format': 'docker', + 'repo': '/ostree/$arch/Client', + 'variant': compose.variants[1], + 'target': 'f24', + 'disk_size': 3, + 'name': 'Fedora-Docker-Base', + 'arches': 'amd64,x86_64', + 'version': 'Rawhide', + 'ksurl': 'git://git.fedorahosted.org/git/spin-kickstarts.git', + 'distro': 'Fedora-20', + }, + "conf_file": 'amd64,x86_64-Client-Fedora-Docker-Base-docker', + "image_dir": '/image_dir/Client/%(arch)s', + "relative_image_dir": 'image_dir/Client/%(arch)s', + "link_type": 'hardlink-or-copy', + } + koji_wrapper = KojiWrapper.return_value + koji_wrapper.run_create_image_cmd.return_value = { + "retcode": 0, + "output": None, + "task_id": 1234, + } + koji_wrapper.get_image_build_paths.return_value = { + 'amd64': [ + '/koji/task/1235/tdl-amd64.xml', + '/koji/task/1235/Fedora-Docker-Base-20160103.amd64.qcow2', + '/koji/task/1235/Fedora-Docker-Base-20160103.amd64.tar.xz' + ], + 'x86_64': [ + '/koji/task/1235/tdl-x86_64.xml', + '/koji/task/1235/Fedora-Docker-Base-20160103.x86_64.qcow2', + '/koji/task/1235/Fedora-Docker-Base-20160103.x86_64.tar.xz' + ] + } + + linker = Linker.return_value + + t = CreateImageBuildThread(pool) + with mock.patch('os.stat') as stat: + with mock.patch('os.path.getsize') as getsize: + with mock.patch('time.sleep'): + getsize.return_value = 1024 + stat.return_value.st_mtime = 13579 + t.process((compose, cmd), 1) + + self.assertItemsEqual( + linker.mock_calls, + [mock.call('/koji/task/1235/Fedora-Docker-Base-20160103.amd64.qcow2', + '/image_dir/Client/amd64/Fedora-Docker-Base-20160103.amd64.qcow2', + link_type='hardlink-or-copy'), + mock.call('/koji/task/1235/Fedora-Docker-Base-20160103.amd64.tar.xz', + '/image_dir/Client/amd64/Fedora-Docker-Base-20160103.amd64.tar.xz', + link_type='hardlink-or-copy'), + mock.call('/koji/task/1235/Fedora-Docker-Base-20160103.x86_64.qcow2', + '/image_dir/Client/x86_64/Fedora-Docker-Base-20160103.x86_64.qcow2', + link_type='hardlink-or-copy'), + mock.call('/koji/task/1235/Fedora-Docker-Base-20160103.x86_64.tar.xz', + '/image_dir/Client/x86_64/Fedora-Docker-Base-20160103.x86_64.tar.xz', + link_type='hardlink-or-copy')]) + + image_relative_paths = { + 'image_dir/Client/amd64/Fedora-Docker-Base-20160103.amd64.qcow2': { + 'format': 'qcow2', + 'type': 'qcow2', + 'arch': 'amd64', + }, + 'image_dir/Client/amd64/Fedora-Docker-Base-20160103.amd64.tar.xz': { + 'format': 'tar.xz', + 'type': 'docker', + 'arch': 'amd64', + }, + 'image_dir/Client/x86_64/Fedora-Docker-Base-20160103.x86_64.qcow2': { + 'format': 'qcow2', + 'type': 'qcow2', + 'arch': 'x86_64', + }, + 'image_dir/Client/x86_64/Fedora-Docker-Base-20160103.x86_64.tar.xz': { + 'format': 'tar.xz', + 'type': 'docker', + 'arch': 'x86_64', + }, + } + + # Assert there are 4 images added to manifest and the arguments are sane + self.assertEqual(len(compose.im.add.call_args_list), 4) + for call in compose.im.add.call_args_list: + _, kwargs = call + image = kwargs['image'] + self.assertEqual(kwargs['variant'], 'Client') + self.assertIn(kwargs['arch'], ('amd64', 'x86_64')) + self.assertEqual(kwargs['arch'], image.arch) + self.assertIn(image.path, image_relative_paths) + data = image_relative_paths.pop(image.path) + self.assertEqual(data['format'], image.format) + self.assertEqual(data['type'], image.type) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_koji_wrapper.py b/tests/test_koji_wrapper.py new file mode 100755 index 00000000..3db3c3e5 --- /dev/null +++ b/tests/test_koji_wrapper.py @@ -0,0 +1,258 @@ +#!/usr/bin/env python2 +# -*- coding: utf-8 -*- + +import mock +import unittest + +import os +import sys + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..")) + +from pungi.wrappers.kojiwrapper import KojiWrapper + + +class KojiWrapperTest(unittest.TestCase): + + def setUp(self): + self.koji_profile = mock.Mock() + with mock.patch('pungi.wrappers.kojiwrapper.koji') as koji: + koji.get_profile_module = mock.Mock( + return_value=mock.Mock( + pathinfo=mock.Mock( + work=mock.Mock(return_value='/koji'), + taskrelpath=mock.Mock(side_effect=lambda id: 'task/' + str(id)), + imagebuild=mock.Mock(side_effect=lambda id: '/koji/imagebuild/' + str(id)), + ) + ) + ) + self.koji_profile = koji.get_profile_module.return_value + self.koji = KojiWrapper('koji') + + @mock.patch('pungi.wrappers.kojiwrapper.open') + def test_get_image_build_cmd_without_required_data(self, mock_open): + with self.assertRaises(AssertionError): + self.koji.get_image_build_cmd( + { + 'name': 'test-name', + }, + '/tmp/file' + ) + + @mock.patch('pungi.wrappers.kojiwrapper.open') + def test_get_image_build_cmd_correct(self, mock_open): + cmd = self.koji.get_image_build_cmd( + { + 'name': 'test-name', + 'version': '1', + 'target': 'test-target', + 'install_tree': '/tmp/test/install_tree', + 'arches': 'x86_64', + 'format': 'docker,qcow2', + 'kickstart': 'test-kickstart', + 'ksurl': 'git://example.com/ks.git', + 'distro': 'test-distro', + }, + '/tmp/file' + ) + + self.assertEqual(cmd[0], 'koji') + self.assertEqual(cmd[1], 'image-build') + self.assertItemsEqual(cmd[2:], + ['--config=/tmp/file', '--wait']) + + output = mock_open.return_value + self.assertEqual(mock.call('[image-build]\n'), output.write.mock_calls[0]) + self.assertItemsEqual(output.write.mock_calls[1:], + [mock.call('name = test-name\n'), + mock.call('version = 1\n'), + mock.call('target = test-target\n'), + mock.call('install_tree = /tmp/test/install_tree\n'), + mock.call('arches = x86_64\n'), + mock.call('format = docker,qcow2\n'), + mock.call('kickstart = test-kickstart\n'), + mock.call('ksurl = git://example.com/ks.git\n'), + mock.call('distro = test-distro\n'), + mock.call('\n')]) + + def test_get_image_build_paths(self): + + # The data for this tests is obtained from the actual Koji build. It + # includes lots of fields that are not used, but for the sake of + # completeness is fully preserved. + + getTaskChildren_data = { + 12387273: [ + { + 'arch': 'i386', + 'awaited': False, + 'channel_id': 12, + 'completion_time': '2016-01-03 05:34:08.374262', + 'completion_ts': 1451799248.37426, + 'create_time': '2016-01-03 05:15:20.311599', + 'create_ts': 1451798120.3116, + 'host_id': 158, + 'id': 12387276, + 'label': 'i386', + 'method': 'createImage', + 'owner': 131, + 'parent': 12387273, + 'priority': 19, + 'request': [ + 'Fedora-Cloud-Base', + '23', + '20160103', + 'i386', + { + 'build_tag': 299, + 'build_tag_name': 'f23-build', + 'dest_tag': 294, + 'dest_tag_name': 'f23-updates-candidate', + 'id': 144, + 'name': 'f23-candidate' + }, + 299, + { + 'create_event': 14011966, + 'create_ts': 1451761803.33528, + 'creation_time': '2016-01-02 19:10:03.335283', + 'id': 563977, + 'state': 1 + }, + 'http://infrastructure.fedoraproject.org/pub/alt/releases/23/Cloud/i386/os/', + { + 'disk_size': '3', + 'distro': 'Fedora-20', + 'format': ['qcow2', 'raw-xz'], + 'kickstart': 'work/cli-image/1451798116.800155.wYJWTVHw/fedora-cloud-base-2878aa0.ks', + 'release': '20160103', + 'repo': ['http://infrastructure.fedoraproject.org/pub/alt/releases/23/Cloud/$arch/os/', + 'http://infrastructure.fedoraproject.org/pub/fedora/linux/updates/23/$arch/'], + 'scratch': True + } + ], + 'start_time': '2016-01-03 05:15:29.828081', + 'start_ts': 1451798129.82808, + 'state': 2, + 'waiting': None, + 'weight': 2.0 + }, { + 'arch': 'x86_64', + 'awaited': False, + 'channel_id': 12, + 'completion_time': '2016-01-03 05:33:20.066366', + 'completion_ts': 1451799200.06637, + 'create_time': '2016-01-03 05:15:20.754201', + 'create_ts': 1451798120.7542, + 'host_id': 156, + 'id': 12387277, + 'label': 'x86_64', + 'method': 'createImage', + 'owner': 131, + 'parent': 12387273, + 'priority': 19, + 'request': [ + 'Fedora-Cloud-Base', + '23', + '20160103', + 'x86_64', + { + 'build_tag': 299, + 'build_tag_name': 'f23-build', + 'dest_tag': 294, + 'dest_tag_name': 'f23-updates-candidate', + 'id': 144, + 'name': 'f23-candidate' + }, + 299, + { + 'create_event': 14011966, + 'create_ts': 1451761803.33528, + 'creation_time': '2016-01-02 19:10:03.335283', + 'id': 563977, + 'state': 1 + }, + 'http://infrastructure.fedoraproject.org/pub/alt/releases/23/Cloud/x86_64/os/', + { + 'disk_size': '3', + 'distro': 'Fedora-20', + 'format': ['qcow2', 'raw-xz'], + 'kickstart': 'work/cli-image/1451798116.800155.wYJWTVHw/fedora-cloud-base-2878aa0.ks', + 'release': '20160103', + 'repo': ['http://infrastructure.fedoraproject.org/pub/alt/releases/23/Cloud/$arch/os/', + 'http://infrastructure.fedoraproject.org/pub/fedora/linux/updates/23/$arch/'], + 'scratch': True + } + ], + 'start_time': '2016-01-03 05:15:35.196043', + 'start_ts': 1451798135.19604, + 'state': 2, + 'waiting': None, + 'weight': 2.0 + } + ] + } + + getTaskResult_data = { + 12387276: { + 'arch': 'i386', + 'files': ['tdl-i386.xml', + 'fedora-cloud-base-2878aa0.ks', + 'koji-f23-build-12387276-base.ks', + 'libvirt-qcow2-i386.xml', + 'Fedora-Cloud-Base-23-20160103.i386.qcow2', + 'libvirt-raw-xz-i386.xml', + 'Fedora-Cloud-Base-23-20160103.i386.raw.xz'], + 'logs': ['oz-i386.log'], + 'name': 'Fedora-Cloud-Base', + 'release': '20160103', + 'rpmlist': [], + 'task_id': 12387276, + 'version': '23' + }, + 12387277: { + 'arch': 'x86_64', + 'files': ['tdl-x86_64.xml', + 'fedora-cloud-base-2878aa0.ks', + 'koji-f23-build-12387277-base.ks', + 'libvirt-qcow2-x86_64.xml', + 'Fedora-Cloud-Base-23-20160103.x86_64.qcow2', + 'libvirt-raw-xz-x86_64.xml', + 'Fedora-Cloud-Base-23-20160103.x86_64.raw.xz'], + 'logs': ['oz-x86_64.log'], + 'name': 'Fedora-Cloud-Base', + 'release': '20160103', + 'rpmlist': [], + 'task_id': 12387277, + 'version': '23' + } + + } + + self.koji.koji_proxy = mock.Mock( + getTaskChildren=mock.Mock(side_effect=lambda task_id, request: getTaskChildren_data.get(task_id)), + getTaskResult=mock.Mock(side_effect=lambda task_id: getTaskResult_data.get(task_id)) + ) + result = self.koji.get_image_build_paths(12387273) + self.assertItemsEqual(result.keys(), ['i386', 'x86_64']) + self.maxDiff = None + self.assertItemsEqual(result['i386'], + ['/koji/task/12387276/tdl-i386.xml', + '/koji/task/12387276/fedora-cloud-base-2878aa0.ks', + '/koji/task/12387276/koji-f23-build-12387276-base.ks', + '/koji/task/12387276/libvirt-qcow2-i386.xml', + '/koji/task/12387276/Fedora-Cloud-Base-23-20160103.i386.qcow2', + '/koji/task/12387276/libvirt-raw-xz-i386.xml', + '/koji/task/12387276/Fedora-Cloud-Base-23-20160103.i386.raw.xz']) + self.assertItemsEqual(result['x86_64'], + ['/koji/task/12387277/tdl-x86_64.xml', + '/koji/task/12387277/fedora-cloud-base-2878aa0.ks', + '/koji/task/12387277/koji-f23-build-12387277-base.ks', + '/koji/task/12387277/libvirt-qcow2-x86_64.xml', + '/koji/task/12387277/Fedora-Cloud-Base-23-20160103.x86_64.qcow2', + '/koji/task/12387277/libvirt-raw-xz-x86_64.xml', + '/koji/task/12387277/Fedora-Cloud-Base-23-20160103.x86_64.raw.xz']) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_util.py b/tests/test_util.py index fe13a47f..c15013d7 100755 --- a/tests/test_util.py +++ b/tests/test_util.py @@ -46,5 +46,39 @@ class TestGitRefResolver(unittest.TestCase): run.assert_called_once_with(['git', 'ls-remote', 'https://git.example.com/repo.git', 'HEAD']) +class TestGetVariantData(unittest.TestCase): + def test_get_simple(self): + conf = { + 'foo': { + '^Client$': 1 + } + } + result = util.get_variant_data(conf, 'foo', mock.Mock(uid='Client')) + self.assertEqual(result, [1]) + + def test_get_make_list(self): + conf = { + 'foo': { + '^Client$': [1, 2], + '^.*$': 3, + } + } + result = util.get_variant_data(conf, 'foo', mock.Mock(uid='Client')) + self.assertItemsEqual(result, [1, 2, 3]) + + def test_not_matching_arch(self): + conf = { + 'foo': { + '^Client$': [1, 2], + } + } + result = util.get_variant_data(conf, 'foo', mock.Mock(uid='Server')) + self.assertItemsEqual(result, []) + + def test_handle_missing_config(self): + result = util.get_variant_data({}, 'foo', mock.Mock(uid='Client')) + self.assertItemsEqual(result, []) + + if __name__ == "__main__": unittest.main()