From d85f00818d608b925c49ed23985d370c3c866bf3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lubom=C3=ADr=20Sedl=C3=A1=C5=99?= Date: Thu, 28 Jan 2016 11:32:57 +0100 Subject: [PATCH 1/4] [image-build] Remove dead code MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit There are no mounts when running image-build tasks in Koji. This looks like a copy-paste error. Signed-off-by: Lubomír Sedlář --- pungi/phases/image_build.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/pungi/phases/image_build.py b/pungi/phases/image_build.py index d950f8ce..5fd67b33 100644 --- a/pungi/phases/image_build.py +++ b/pungi/phases/image_build.py @@ -154,9 +154,6 @@ class CreateImageBuildThread(WorkerThread): def worker(self, num, compose, cmd): arches = cmd['image_conf']['arches'].split(',') - mounts = [compose.paths.compose.topdir()] - if "mount" in cmd: - mounts.append(cmd["mount"]) log_file = compose.paths.log.log_file( cmd["image_conf"]["arches"], "imagebuild-%s-%s-%s" % ('-'.join(arches), From ae30c07553e6dc5ea8575bee8730dee995139426 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lubom=C3=ADr=20Sedl=C3=A1=C5=99?= Date: Thu, 28 Jan 2016 14:58:24 +0100 Subject: [PATCH 2/4] [koji-wrapper] Use more descriptive method names MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The methods mentioning image build are generic and can work for other task types. get_image_build_paths -> get_image_paths run_create_image_cmd -> run_blocking_cmd Signed-off-by: Lubomír Sedlář --- pungi/phases/image_build.py | 4 ++-- pungi/phases/live_images.py | 2 +- pungi/wrappers/kojiwrapper.py | 10 +++++++--- tests/test_imagebuildphase.py | 8 ++++---- tests/test_koji_wrapper.py | 4 ++-- tests/test_liveimagesphase.py | 6 +++--- 6 files changed, 19 insertions(+), 15 deletions(-) diff --git a/pungi/phases/image_build.py b/pungi/phases/image_build.py index 5fd67b33..ac54cef1 100644 --- a/pungi/phases/image_build.py +++ b/pungi/phases/image_build.py @@ -176,7 +176,7 @@ class CreateImageBuildThread(WorkerThread): # avoid race conditions? # Kerberos authentication failed: Permission denied in replay cache code (-1765328215) time.sleep(num * 3) - output = koji_wrapper.run_create_image_cmd(koji_cmd, log_file=log_file) + output = koji_wrapper.run_blocking_cmd(koji_cmd, log_file=log_file) self.pool.log_debug("build-image outputs: %s" % (output)) if output["retcode"] != 0: self.fail(compose, cmd) @@ -185,7 +185,7 @@ class CreateImageBuildThread(WorkerThread): # copy image to images/ image_infos = [] - paths = koji_wrapper.get_image_build_paths(output["task_id"]) + paths = koji_wrapper.get_image_paths(output["task_id"]) for arch, paths in paths.iteritems(): for path in paths: diff --git a/pungi/phases/live_images.py b/pungi/phases/live_images.py index 685bf4f5..3f7225e4 100644 --- a/pungi/phases/live_images.py +++ b/pungi/phases/live_images.py @@ -201,7 +201,7 @@ class CreateLiveImageThread(WorkerThread): # Kerberos authentication failed: Permission denied in replay cache code (-1765328215) time.sleep(num * 3) - output = koji_wrapper.run_create_image_cmd(koji_cmd, log_file=log_file) + output = koji_wrapper.run_blocking_cmd(koji_cmd, log_file=log_file) if output["retcode"] != 0: self.fail(compose, cmd) raise RuntimeError("LiveImage task failed: %s. See %s for more details." % (output["task_id"], log_file)) diff --git a/pungi/wrappers/kojiwrapper.py b/pungi/wrappers/kojiwrapper.py index 62e11751..5b5076b8 100644 --- a/pungi/wrappers/kojiwrapper.py +++ b/pungi/wrappers/kojiwrapper.py @@ -191,8 +191,12 @@ class KojiWrapper(object): return cmd - def run_create_image_cmd(self, command, log_file=None): - # spin-{livecd,appliance} is blocking by default -> you probably want to run it in a thread + def run_blocking_cmd(self, command, log_file=None): + """ + Run a blocking koji command. Returns a dict with output of the command, + its exit code and parsed task id. This method will block until the + command finishes. + """ try: retcode, output = run(command, can_fail=True, logfile=log_file) except RuntimeError, e: @@ -209,7 +213,7 @@ class KojiWrapper(object): } return result - def get_image_build_paths(self, task_id): + def get_image_paths(self, task_id): """ Given an image task in Koji, get a mapping from arches to a list of paths to results of the task. diff --git a/tests/test_imagebuildphase.py b/tests/test_imagebuildphase.py index f468e036..434db753 100755 --- a/tests/test_imagebuildphase.py +++ b/tests/test_imagebuildphase.py @@ -349,12 +349,12 @@ class TestCreateImageBuildThread(unittest.TestCase): "link_type": 'hardlink-or-copy', } koji_wrapper = KojiWrapper.return_value - koji_wrapper.run_create_image_cmd.return_value = { + koji_wrapper.run_blocking_cmd.return_value = { "retcode": 0, "output": None, "task_id": 1234, } - koji_wrapper.get_image_build_paths.return_value = { + koji_wrapper.get_image_paths.return_value = { 'amd64': [ '/koji/task/1235/tdl-amd64.xml', '/koji/task/1235/Fedora-Docker-Base-20160103.amd64.qcow2', @@ -468,7 +468,7 @@ class TestCreateImageBuildThread(unittest.TestCase): "link_type": 'hardlink-or-copy', } koji_wrapper = KojiWrapper.return_value - koji_wrapper.run_create_image_cmd.return_value = { + koji_wrapper.run_blocking_cmd.return_value = { "retcode": 1, "output": None, "task_id": 1234, @@ -520,7 +520,7 @@ class TestCreateImageBuildThread(unittest.TestCase): raise RuntimeError('BOOM') koji_wrapper = KojiWrapper.return_value - koji_wrapper.run_create_image_cmd.side_effect = boom + koji_wrapper.run_blocking_cmd.side_effect = boom t = CreateImageBuildThread(pool) with mock.patch('os.stat') as stat: diff --git a/tests/test_koji_wrapper.py b/tests/test_koji_wrapper.py index 3db3c3e5..f90dcc7d 100755 --- a/tests/test_koji_wrapper.py +++ b/tests/test_koji_wrapper.py @@ -75,7 +75,7 @@ class KojiWrapperTest(unittest.TestCase): mock.call('distro = test-distro\n'), mock.call('\n')]) - def test_get_image_build_paths(self): + def test_get_image_paths(self): # The data for this tests is obtained from the actual Koji build. It # includes lots of fields that are not used, but for the sake of @@ -233,7 +233,7 @@ class KojiWrapperTest(unittest.TestCase): getTaskChildren=mock.Mock(side_effect=lambda task_id, request: getTaskChildren_data.get(task_id)), getTaskResult=mock.Mock(side_effect=lambda task_id: getTaskResult_data.get(task_id)) ) - result = self.koji.get_image_build_paths(12387273) + result = self.koji.get_image_paths(12387273) self.assertItemsEqual(result.keys(), ['i386', 'x86_64']) self.maxDiff = None self.assertItemsEqual(result['i386'], diff --git a/tests/test_liveimagesphase.py b/tests/test_liveimagesphase.py index c403e9e8..ead447c1 100755 --- a/tests/test_liveimagesphase.py +++ b/tests/test_liveimagesphase.py @@ -129,7 +129,7 @@ class TestCreateLiveImageThread(unittest.TestCase): koji_wrapper = KojiWrapper.return_value koji_wrapper.get_create_image_cmd.return_value = 'koji spin-livecd ...' - koji_wrapper.run_create_image_cmd.return_value = { + koji_wrapper.run_blocking_cmd.return_value = { 'retcode': 0, 'output': 'some output', 'task_id': 123 @@ -140,7 +140,7 @@ class TestCreateLiveImageThread(unittest.TestCase): with mock.patch('time.sleep'): t.process((compose, cmd, compose.variants['Client'], 'amd64'), 1) - self.assertEqual(koji_wrapper.run_create_image_cmd.mock_calls, + self.assertEqual(koji_wrapper.run_blocking_cmd.mock_calls, [mock.call('koji spin-livecd ...', log_file='/a/b/log/log_file')]) self.assertEqual(koji_wrapper.get_image_path.mock_calls, [mock.call(123)]) self.assertEqual(copy2.mock_calls, @@ -178,7 +178,7 @@ class TestCreateLiveImageThread(unittest.TestCase): koji_wrapper = KojiWrapper.return_value koji_wrapper.get_create_image_cmd.return_value = 'koji spin-livecd ...' - koji_wrapper.run_create_image_cmd.return_value = { + koji_wrapper.run_blocking_cmd.return_value = { 'retcode': 1, 'output': 'some output', 'task_id': 123 From a5a0f3d69fcc7230fb688dfc6d1f77b1575639a9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lubom=C3=ADr=20Sedl=C3=A1=C5=99?= Date: Thu, 28 Jan 2016 14:56:22 +0100 Subject: [PATCH 3/4] [koji-wrapper] Add support for spin-livemedia MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This patch adds support for live media creator in Koji. The intended workflow is to create a command , run it and finally collect built artifacts. get_live_media_cmd() run_blocking_cmd() get_image_paths() Signed-off-by: Lubomír Sedlář --- pungi/wrappers/kojiwrapper.py | 32 ++++++++++++++++++++++++++-- tests/test_koji_wrapper.py | 40 +++++++++++++++++++++++++++++++++++ 2 files changed, 70 insertions(+), 2 deletions(-) diff --git a/pungi/wrappers/kojiwrapper.py b/pungi/wrappers/kojiwrapper.py index 5b5076b8..b5b7afba 100644 --- a/pungi/wrappers/kojiwrapper.py +++ b/pungi/wrappers/kojiwrapper.py @@ -127,6 +127,33 @@ class KojiWrapper(object): return cmd + def get_live_media_cmd(self, options, wait=True): + # Usage: koji spin-livemedia [options] + cmd = ['koji', 'spin-livemedia'] + + for key in ('name', 'version', 'target', 'arch', 'ksfile'): + if key not in options: + raise ValueError('Expected options to have key "%s"' % key) + cmd.append(pipes.quote(options[key])) + + if 'install_tree' not in options: + raise ValueError('Expected options to have key "install_tree"') + cmd.append('--install-tree=%s' % pipes.quote(options['install_tree'])) + + for repo in options.get('repo', []): + cmd.append('--repo=%s' % pipes.quote(repo)) + + if options.get('scratch'): + cmd.append('--scratch') + + if options.get('skip_tag'): + cmd.append('--skip-tag') + + if wait: + cmd.append('--wait') + + return cmd + def get_create_image_cmd(self, name, version, target, arch, ks_file, repos, image_type="live", image_format=None, release=None, wait=True, archive=False, specfile=None): # Usage: koji spin-livecd [options] # Usage: koji spin-appliance [options] @@ -204,7 +231,8 @@ class KojiWrapper(object): match = re.search(r"Created task: (\d+)", output) if not match: - raise RuntimeError("Could not find task ID in output. Command '%s' returned '%s'." % (" ".join(command), output)) + raise RuntimeError("Could not find task ID in output. Command '%s' returned '%s'." + % (" ".join(command), output)) result = { "retcode": retcode, @@ -224,7 +252,7 @@ class KojiWrapper(object): children_tasks = self.koji_proxy.getTaskChildren(task_id, request=True) for child_task in children_tasks: - if child_task['method'] != 'createImage': + if child_task['method'] not in ['createImage', 'createLiveMedia']: continue is_scratch = child_task['request'][-1].get('scratch', False) diff --git a/tests/test_koji_wrapper.py b/tests/test_koji_wrapper.py index f90dcc7d..a75a5eda 100755 --- a/tests/test_koji_wrapper.py +++ b/tests/test_koji_wrapper.py @@ -254,5 +254,45 @@ class KojiWrapperTest(unittest.TestCase): '/koji/task/12387277/Fedora-Cloud-Base-23-20160103.x86_64.raw.xz']) +class LiveMediaTestCase(unittest.TestCase): + def setUp(self): + self.koji_profile = mock.Mock() + with mock.patch('pungi.wrappers.kojiwrapper.koji') as koji: + koji.get_profile_module = mock.Mock( + return_value=mock.Mock( + pathinfo=mock.Mock( + work=mock.Mock(return_value='/koji'), + taskrelpath=mock.Mock(side_effect=lambda id: 'task/' + str(id)), + imagebuild=mock.Mock(side_effect=lambda id: '/koji/imagebuild/' + str(id)), + ) + ) + ) + self.koji_profile = koji.get_profile_module.return_value + self.koji = KojiWrapper('koji') + + def test_get_live_media_cmd_minimal(self): + opts = { + 'name': 'name', 'version': '1', 'target': 'tgt', 'arch': 'x,y,z', + 'ksfile': 'kickstart', 'install_tree': '/mnt/os' + } + cmd = self.koji.get_live_media_cmd(opts) + self.assertEqual(cmd, + ['koji', 'spin-livemedia', 'name', '1', 'tgt', 'x,y,z', 'kickstart', + '--install-tree=/mnt/os', '--wait']) + + def test_get_live_media_cmd_full(self): + opts = { + 'name': 'name', 'version': '1', 'target': 'tgt', 'arch': 'x,y,z', + 'ksfile': 'kickstart', 'install_tree': '/mnt/os', 'scratch': True, + 'repo': ['repo-1', 'repo-2'], 'skip_tag': True, + } + cmd = self.koji.get_live_media_cmd(opts) + self.assertEqual(cmd[:8], + ['koji', 'spin-livemedia', 'name', '1', 'tgt', 'x,y,z', 'kickstart', + '--install-tree=/mnt/os']) + self.assertItemsEqual(cmd[8:], + ['--repo=repo-1', '--repo=repo-2', '--skip-tag', '--scratch', '--wait']) + + if __name__ == "__main__": unittest.main() From 439622d576ed3b2b039e3d19e892be83e45b92b6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lubom=C3=ADr=20Sedl=C3=A1=C5=99?= Date: Thu, 28 Jan 2016 16:03:20 +0100 Subject: [PATCH 4/4] [live-media] Add live media phase MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This phase builds live media in Koji using the Live Media Creator. It runs in parallel with current live images, create ISO and image build phases. The documentation is updated to explain how to configure this. Signed-off-by: Lubomír Sedlář --- bin/pungi-koji | 6 +- doc/configuration.rst | 25 +++ pungi/phases/__init__.py | 1 + pungi/phases/livemedia_phase.py | 178 +++++++++++++++++ tests/test_livemediaphase.py | 328 ++++++++++++++++++++++++++++++++ 5 files changed, 537 insertions(+), 1 deletion(-) create mode 100644 pungi/phases/livemedia_phase.py create mode 100755 tests/test_livemediaphase.py diff --git a/bin/pungi-koji b/bin/pungi-koji index 3f1441d3..5740ddaf 100755 --- a/bin/pungi-koji +++ b/bin/pungi-koji @@ -229,6 +229,7 @@ def run_compose(compose): productimg_phase = pungi.phases.ProductimgPhase(compose, pkgset_phase) createiso_phase = pungi.phases.CreateisoPhase(compose) liveimages_phase = pungi.phases.LiveImagesPhase(compose) + livemedia_phase = pungi.phases.LiveMediaPhase(compose) image_build_phase = pungi.phases.ImageBuildPhase(compose) image_checksum_phase = pungi.phases.ImageChecksumPhase(compose) test_phase = pungi.phases.TestPhase(compose) @@ -237,7 +238,8 @@ def run_compose(compose): for phase in (init_phase, pkgset_phase, createrepo_phase, buildinstall_phase, productimg_phase, gather_phase, extrafiles_phase, createiso_phase, liveimages_phase, - image_build_phase, image_checksum_phase, test_phase): + livemedia_phase, image_build_phase, image_checksum_phase, + test_phase): if phase.skip(): continue try: @@ -302,10 +304,12 @@ def run_compose(compose): createiso_phase.start() liveimages_phase.start() image_build_phase.start() + livemedia_phase.start() createiso_phase.stop() liveimages_phase.stop() image_build_phase.stop() + livemedia_phase.stop() image_checksum_phase.start() image_checksum_phase.stop() diff --git a/doc/configuration.rst b/doc/configuration.rst index 59977bab..fc87be48 100644 --- a/doc/configuration.rst +++ b/doc/configuration.rst @@ -135,6 +135,7 @@ Options * iso * live * image-build + * live-media .. note:: @@ -646,6 +647,30 @@ Live Images Settings * ``scratch`` (*bool*) -- only RPM-wrapped images can use scratch builds, but by default this is turned off +Live Media Settings +=================== + +**live_media** + (*dict*) -- configuration for ``koji spin-livemedia``; format: + ``{variant_uid_regex: [{opt:value}]}`` + + Available options: + + * ``target`` (*str*) + * ``arches`` (*[str]*) -- what architectures to build the media for; by default uses + all arches for the variant. + * ``kickstart`` (*str*) -- name of the kickstart file + * ``ksurl`` (*str*) + * ``ksversion`` (*str*) + * ``scratch`` (*bool*) + * ``release`` (*str*) -- a string with the release, or explicit ``None`` + for using compose date and respin. + * ``skip_tag`` (*bool*) + * ``name`` (*str*) + * ``repo`` (*[str]*) -- external repo + * ``repo_from`` (*[str]*) -- list of variants to take extra repos from + * ``title`` (*str*) + Image Build Settings ==================== diff --git a/pungi/phases/__init__.py b/pungi/phases/__init__.py index 81e319d9..ba72887f 100644 --- a/pungi/phases/__init__.py +++ b/pungi/phases/__init__.py @@ -28,3 +28,4 @@ from live_images import LiveImagesPhase # noqa from image_build import ImageBuildPhase # noqa from test import TestPhase # noqa from image_checksum import ImageChecksumPhase # noqa +from livemedia_phase import LiveMediaPhase # noqa diff --git a/pungi/phases/livemedia_phase.py b/pungi/phases/livemedia_phase.py new file mode 100644 index 00000000..ddb8fefa --- /dev/null +++ b/pungi/phases/livemedia_phase.py @@ -0,0 +1,178 @@ +# -*- coding: utf-8 -*- + +import os +import time +from kobo import shortcuts + +from pungi.util import get_variant_data, resolve_git_url, makedirs +from pungi.phases.base import PhaseBase +from pungi.linker import Linker +from pungi.paths import translate_path +from pungi.wrappers.kojiwrapper import KojiWrapper +from kobo.threads import ThreadPool, WorkerThread +from productmd.images import Image + + +class LiveMediaPhase(PhaseBase): + """class for wrapping up koji spin-livemedia""" + name = 'live_media' + + def __init__(self, compose): + super(LiveMediaPhase, self).__init__(compose) + self.pool = ThreadPool(logger=self.compose._logger) + + def skip(self): + if super(LiveMediaPhase, self).skip(): + return True + if not self.compose.conf.get(self.name): + self.compose.log_info("Config section '%s' was not found. Skipping" % self.name) + return True + return False + + def _get_repos(self, image_conf, variant): + """ + Get a comma separated list of repos. First included are those + explicitly listed in config, followed by repos from other variants, + finally followed by repo for current variant. + + The `repo_from` key is removed from the dict (if present). + """ + repo = shortcuts.force_list(image_conf.get('repo', [])) + + extras = shortcuts.force_list(image_conf.pop('repo_from', [])) + extras.append(variant.uid) + + for extra in extras: + v = self.compose.variants.get(extra) + if not v: + raise RuntimeError( + 'There is no variant %s to get repo from when building live media for %s.' + % (extra, variant.uid)) + repo.append(translate_path( + self.compose, + self.compose.paths.compose.repository('$arch', v, create_dir=False))) + + return repo + + def _get_arches(self, image_conf, arches): + if 'arches' in image_conf: + arches = set(image_conf.get('arches', [])) & arches + return sorted(arches) + + def _get_release(self, image_conf): + """If release is set explicitly to None, replace it with date and respin.""" + if 'release' in image_conf and image_conf['release'] is None: + return '%s.%s' % (self.compose.compose_date, self.compose.compose_respin) + return image_conf.get('release', None) + + def run(self): + for variant in self.compose.get_variants(): + arches = set([x for x in variant.arches if x != 'src']) + + for image_conf in get_variant_data(self.compose.conf, self.name, variant): + config = { + 'target': image_conf['target'], + 'arches': self._get_arches(image_conf, arches), + 'kickstart': image_conf['kickstart'], + 'ksurl': resolve_git_url(image_conf['ksurl']), + 'ksversion': image_conf.get('ksversion'), + 'scratch': image_conf.get('scratch', False), + 'release': self._get_release(image_conf), + 'skip_tag': image_conf.get('skip_tag'), + 'name': image_conf['name'], + 'title': image_conf.get('title'), + 'repo': self._get_repos(image_conf, variant), + 'install_tree': translate_path( + self.compose, + self.compose.paths.compose.os_tree('$arch', variant, create_dir=False) + ) + } + self.pool.add(LiveMediaThread(self.pool)) + self.pool.queue_put((self.compose, variant, config)) + + self.pool.start() + + +class LiveMediaThread(WorkerThread): + def process(self, item, num): + compose, variant, config = item + self.num = num + try: + self.worker(compose, variant, config) + except: + if not compose.can_fail(variant, '*', 'live-media'): + raise + else: + msg = ('[FAIL] live-media for variant %s failed, but going on anyway.' + % variant.uid) + self.pool.log_info(msg) + + def _get_log_file(self, compose, variant, config): + arches = '-'.join(config['arches']) + return compose.paths.log.log_file(arches, 'livemedia-%s' % variant) + + def _run_command(self, koji_wrapper, cmd, compose, log_file): + time.sleep(self.num * 3) + output = koji_wrapper.run_blocking_cmd(cmd, log_file=log_file) + self.pool.log_debug('live media outputs: %s' % (output)) + if output['retcode'] != 0: + compose.log_error('Live media task failed.') + raise RuntimeError('Live media task failed: %s. See %s for more details.' + % (output['task_id'], log_file)) + return output + + def worker(self, compose, variant, config): + msg = 'Live media: %s (arches: %s, variant: %s)' % (config['name'], + ' '.join(config['arches']), + variant.uid) + self.pool.log_info('[BEGIN] %s' % msg) + + koji_wrapper = KojiWrapper(compose.conf['koji_profile']) + cmd = koji_wrapper.get_live_media_cmd(config) + + log_file = self._get_log_file(compose, variant, config) + output = self._run_command(koji_wrapper, cmd, compose, log_file) + + # collect results and update manifest + image_infos = [] + + paths = koji_wrapper.get_image_paths(output['task_id']) + + for arch, paths in paths.iteritems(): + for path in paths: + if path.endswith('.iso'): + image_infos.append({'path': path, 'arch': arch}) + + if len(image_infos) != len(config['arches']): + self.pool.log_error( + 'Error in koji task %s. Expected to find one image for each arch (%s). Got %s.' + % (output['task_id'], len(config['arches']), len(image_infos))) + raise RuntimeError('Image count mismatch in task %s.' % output['task_id']) + + linker = Linker(logger=compose._logger) + link_type = compose.conf.get("link_type", "hardlink-or-copy") + for image_info in image_infos: + image_dir = compose.paths.compose.image_dir(variant) % {"arch": image_info['arch']} + makedirs(image_dir) + relative_image_dir = ( + compose.paths.compose.image_dir(variant, relative=True) % {"arch": image_info['arch']} + ) + + # let's not change filename of koji outputs + image_dest = os.path.join(image_dir, os.path.basename(image_info['path'])) + linker.link(image_info['path'], image_dest, link_type=link_type) + + # Update image manifest + img = Image(compose.im) + img.type = 'live' + img.format = 'iso' + img.path = os.path.join(relative_image_dir, os.path.basename(image_dest)) + img.mtime = int(os.stat(image_dest).st_mtime) + img.size = os.path.getsize(image_dest) + img.arch = image_info['arch'] + img.disc_number = 1 # We don't expect multiple disks + img.disc_count = 1 + img.bootable = True + compose.im.add(variant=variant.uid, arch=image_info['arch'], image=img) + + self.pool.log_info('[DONE ] %s' % msg) diff --git a/tests/test_livemediaphase.py b/tests/test_livemediaphase.py new file mode 100755 index 00000000..6b0dbf9e --- /dev/null +++ b/tests/test_livemediaphase.py @@ -0,0 +1,328 @@ +#!/usr/bin/env python2 +# -*- coding: utf-8 -*- + +import unittest +import mock + +import sys +import os + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..")) + +from pungi.phases.livemedia_phase import LiveMediaPhase, LiveMediaThread +from pungi.util import get_arch_variant_data + + +class _DummyCompose(object): + def __init__(self, config): + self.compose_date = '20151203' + self.compose_type_suffix = '.t' + self.compose_respin = 0 + self.ci_base = mock.Mock( + release_id='Test-1.0', + release=mock.Mock( + short='test', + version='1.0', + ), + ) + self.conf = config + self.paths = mock.Mock( + compose=mock.Mock( + topdir=mock.Mock(return_value='/a/b'), + os_tree=mock.Mock( + side_effect=lambda arch, variant, create_dir=False: os.path.join('/ostree', arch, variant.uid) + ), + repository=mock.Mock( + side_effect=lambda arch, variant, create_dir=False: os.path.join('/repo', arch, variant.uid) + ), + image_dir=mock.Mock( + side_effect=lambda variant, relative=False: os.path.join( + '' if relative else '/', 'image_dir', variant.uid, '%(arch)s' + ) + ) + ), + work=mock.Mock( + image_build_conf=mock.Mock( + side_effect=lambda variant, image_name, image_type: + '-'.join([variant.uid, image_name, image_type]) + ) + ), + log=mock.Mock( + log_file=mock.Mock(return_value='/a/b/log/log_file') + ) + ) + self._logger = mock.Mock() + self.variants = { + 'Server': mock.Mock(uid='Server', arches=['x86_64', 'amd64']), + 'Client': mock.Mock(uid='Client', arches=['amd64']), + 'Everything': mock.Mock(uid='Everything', arches=['x86_64', 'amd64']), + } + self.im = mock.Mock() + self.log_error = mock.Mock() + + def get_variants(self, arch=None, types=None): + return [v for v in self.variants.values() if not arch or arch in v.arches] + + def can_fail(self, variant, arch, deliverable): + failable = get_arch_variant_data(self.conf, 'failable_deliverables', arch, variant) + return deliverable in failable + + +class TestLiveMediaPhase(unittest.TestCase): + @mock.patch('pungi.phases.livemedia_phase.ThreadPool') + def test_live_media_minimal(self, ThreadPool): + compose = _DummyCompose({ + 'live_media': { + '^Server$': [ + { + 'target': 'f24', + 'kickstart': 'file.ks', + 'ksurl': 'git://example.com/repo.git', + 'name': 'Fedora Server Live', + } + ] + }, + 'koji_profile': 'koji', + }) + + phase = LiveMediaPhase(compose) + + phase.run() + self.assertTrue(phase.pool.add.called) + self.assertEqual(phase.pool.queue_put.call_args_list, + [mock.call((compose, + compose.variants['Server'], + { + 'arches': ['amd64', 'x86_64'], + 'kickstart': 'file.ks', + 'ksurl': 'git://example.com/repo.git', + 'ksversion': None, + 'name': 'Fedora Server Live', + 'release': None, + 'repo': ['/repo/$arch/Server'], + 'scratch': False, + 'skip_tag': None, + 'target': 'f24', + 'title': None, + 'install_tree': '/ostree/$arch/Server', + }))]) + + @mock.patch('pungi.phases.livemedia_phase.resolve_git_url') + @mock.patch('pungi.phases.livemedia_phase.ThreadPool') + def test_live_media_full(self, ThreadPool, resolve_git_url): + compose = _DummyCompose({ + 'live_media': { + '^Server$': [ + { + 'target': 'f24', + 'kickstart': 'file.ks', + 'ksurl': 'git://example.com/repo.git#HEAD', + 'name': 'Fedora Server Live', + 'scratch': True, + 'skip_tag': True, + 'title': 'Custom Title', + 'repo_from': ['Everything'], + 'repo': ['http://example.com/extra_repo'], + 'arches': ['x86_64'], + 'ksversion': '24', + 'release': None + } + ] + } + }) + + resolve_git_url.return_value = 'resolved' + + phase = LiveMediaPhase(compose) + + phase.run() + self.assertTrue(phase.pool.add.called) + self.assertEqual(phase.pool.queue_put.call_args_list, + [mock.call((compose, + compose.variants['Server'], + { + 'arches': ['x86_64'], + 'kickstart': 'file.ks', + 'ksurl': 'resolved', + 'ksversion': '24', + 'name': 'Fedora Server Live', + 'release': '20151203.0', + 'repo': ['http://example.com/extra_repo', + '/repo/$arch/Everything', + '/repo/$arch/Server'], + 'scratch': True, + 'skip_tag': True, + 'target': 'f24', + 'title': 'Custom Title', + 'install_tree': '/ostree/$arch/Server', + }))]) + + +class TestCreateImageBuildThread(unittest.TestCase): + + @mock.patch('pungi.phases.livemedia_phase.KojiWrapper') + @mock.patch('pungi.phases.livemedia_phase.Linker') + @mock.patch('pungi.phases.livemedia_phase.makedirs') + def test_process(self, makedirs, Linker, KojiWrapper): + compose = _DummyCompose({ + 'koji_profile': 'koji' + }) + config = { + 'arches': ['amd64', 'x86_64'], + 'kickstart': 'file.ks', + 'ksurl': 'git://example.com/repo.git', + 'ksversion': None, + 'name': 'Fedora Server Live', + 'release': None, + 'repo': ['/repo/$arch/Server'], + 'scratch': False, + 'skip_tag': None, + 'target': 'f24', + 'title': None, + } + pool = mock.Mock() + + get_live_media_cmd = KojiWrapper.return_value.get_live_media_cmd + get_live_media_cmd.return_value = 'koji-spin-livemedia' + + run_blocking_cmd = KojiWrapper.return_value.run_blocking_cmd + run_blocking_cmd.return_value = { + 'task_id': 1234, + 'retcode': 0, + 'output': None, + } + + get_image_paths = KojiWrapper.return_value.get_image_paths + get_image_paths.return_value = { + 'x86_64': [ + '/koji/task/1235/tdl-amd64.xml', + '/koji/task/1235/Live-20160103.x86_64.iso', + '/koji/task/1235/Live-20160103.x86_64.tar.xz' + ], + 'amd64': [ + '/koji/task/1235/tdl-amd64.xml', + '/koji/task/1235/Live-20160103.amd64.iso', + '/koji/task/1235/Live-20160103.amd64.tar.xz' + ] + } + + t = LiveMediaThread(pool) + with mock.patch('os.stat') as stat: + with mock.patch('os.path.getsize') as getsize: + with mock.patch('time.sleep'): + getsize.return_value = 1024 + stat.return_value.st_mtime = 13579 + t.process((compose, compose.variants['Server'], config), 1) + + self.assertEqual(run_blocking_cmd.mock_calls, + [mock.call('koji-spin-livemedia', log_file='/a/b/log/log_file')]) + self.assertEqual(get_live_media_cmd.mock_calls, + [mock.call(config)]) + self.assertEqual(get_image_paths.mock_calls, + [mock.call(1234)]) + self.assertItemsEqual(makedirs.mock_calls, + [mock.call('/image_dir/Server/x86_64'), + mock.call('/image_dir/Server/amd64')]) + link = Linker.return_value.link + self.assertItemsEqual(link.mock_calls, + [mock.call('/koji/task/1235/Live-20160103.amd64.iso', + '/image_dir/Server/amd64/Live-20160103.amd64.iso', + link_type='hardlink-or-copy'), + mock.call('/koji/task/1235/Live-20160103.x86_64.iso', + '/image_dir/Server/x86_64/Live-20160103.x86_64.iso', + link_type='hardlink-or-copy')]) + + image_relative_paths = [ + 'image_dir/Server/amd64/Live-20160103.amd64.iso', + 'image_dir/Server/x86_64/Live-20160103.x86_64.iso' + ] + + self.assertEqual(len(compose.im.add.call_args_list), 2) + for call in compose.im.add.call_args_list: + _, kwargs = call + image = kwargs['image'] + self.assertEqual(kwargs['variant'], 'Server') + self.assertIn(kwargs['arch'], ('amd64', 'x86_64')) + self.assertEqual(kwargs['arch'], image.arch) + self.assertIn(image.path, image_relative_paths) + self.assertEqual('iso', image.format) + self.assertEqual('live', image.type) + + @mock.patch('pungi.phases.livemedia_phase.KojiWrapper') + def test_handle_koji_fail(self, KojiWrapper): + compose = _DummyCompose({ + 'koji_profile': 'koji', + 'failable_deliverables': [ + ('^.+$', {'*': ['live-media']}) + ] + }) + config = { + 'arches': ['amd64', 'x86_64'], + 'kickstart': 'file.ks', + 'ksurl': 'git://example.com/repo.git', + 'ksversion': None, + 'name': 'Fedora Server Live', + 'release': None, + 'repo': ['/repo/$arch/Server'], + 'scratch': False, + 'skip_tag': None, + 'target': 'f24', + 'title': None, + } + pool = mock.Mock() + + run_blocking_cmd = KojiWrapper.return_value.run_blocking_cmd + run_blocking_cmd.return_value = { + 'task_id': 1234, + 'retcode': 1, + 'output': None, + } + + t = LiveMediaThread(pool) + with mock.patch('os.stat') as stat: + with mock.patch('os.path.getsize') as getsize: + with mock.patch('time.sleep'): + getsize.return_value = 1024 + stat.return_value.st_mtime = 13579 + t.process((compose, compose.variants['Server'], config), 1) + + @mock.patch('pungi.phases.livemedia_phase.KojiWrapper') + def test_handle_exception(self, KojiWrapper): + compose = _DummyCompose({ + 'koji_profile': 'koji', + 'failable_deliverables': [ + ('^.+$', {'*': ['live-media']}) + ] + }) + config = { + 'arches': ['amd64', 'x86_64'], + 'kickstart': 'file.ks', + 'ksurl': 'git://example.com/repo.git', + 'ksversion': None, + 'name': 'Fedora Server Live', + 'release': None, + 'repo': ['/repo/$arch/Server'], + 'scratch': False, + 'skip_tag': None, + 'target': 'f24', + 'title': None, + } + pool = mock.Mock() + + def boom(*args, **kwargs): + raise Exception('BOOM') + + run_blocking_cmd = KojiWrapper.return_value.run_blocking_cmd + run_blocking_cmd.side_effect = boom + + t = LiveMediaThread(pool) + with mock.patch('os.stat') as stat: + with mock.patch('os.path.getsize') as getsize: + with mock.patch('time.sleep'): + getsize.return_value = 1024 + stat.return_value.st_mtime = 13579 + t.process((compose, compose.variants['Server'], config), 1) + + +if __name__ == "__main__": + unittest.main()