diff --git a/bin/pungi-koji b/bin/pungi-koji index 3f1441d3..5740ddaf 100755 --- a/bin/pungi-koji +++ b/bin/pungi-koji @@ -229,6 +229,7 @@ def run_compose(compose): productimg_phase = pungi.phases.ProductimgPhase(compose, pkgset_phase) createiso_phase = pungi.phases.CreateisoPhase(compose) liveimages_phase = pungi.phases.LiveImagesPhase(compose) + livemedia_phase = pungi.phases.LiveMediaPhase(compose) image_build_phase = pungi.phases.ImageBuildPhase(compose) image_checksum_phase = pungi.phases.ImageChecksumPhase(compose) test_phase = pungi.phases.TestPhase(compose) @@ -237,7 +238,8 @@ def run_compose(compose): for phase in (init_phase, pkgset_phase, createrepo_phase, buildinstall_phase, productimg_phase, gather_phase, extrafiles_phase, createiso_phase, liveimages_phase, - image_build_phase, image_checksum_phase, test_phase): + livemedia_phase, image_build_phase, image_checksum_phase, + test_phase): if phase.skip(): continue try: @@ -302,10 +304,12 @@ def run_compose(compose): createiso_phase.start() liveimages_phase.start() image_build_phase.start() + livemedia_phase.start() createiso_phase.stop() liveimages_phase.stop() image_build_phase.stop() + livemedia_phase.stop() image_checksum_phase.start() image_checksum_phase.stop() diff --git a/doc/configuration.rst b/doc/configuration.rst index 3f4f2a08..3b2a0747 100644 --- a/doc/configuration.rst +++ b/doc/configuration.rst @@ -135,6 +135,7 @@ Options * iso * live * image-build + * live-media .. note:: @@ -646,6 +647,30 @@ Live Images Settings * ``scratch`` (*bool*) -- only RPM-wrapped images can use scratch builds, but by default this is turned off +Live Media Settings +=================== + +**live_media** + (*dict*) -- configuration for ``koji spin-livemedia``; format: + ``{variant_uid_regex: [{opt:value}]}`` + + Available options: + + * ``target`` (*str*) + * ``arches`` (*[str]*) -- what architectures to build the media for; by default uses + all arches for the variant. + * ``kickstart`` (*str*) -- name of the kickstart file + * ``ksurl`` (*str*) + * ``ksversion`` (*str*) + * ``scratch`` (*bool*) + * ``release`` (*str*) -- a string with the release, or explicit ``None`` + for using compose date and respin. + * ``skip_tag`` (*bool*) + * ``name`` (*str*) + * ``repo`` (*[str]*) -- external repo + * ``repo_from`` (*[str]*) -- list of variants to take extra repos from + * ``title`` (*str*) + Image Build Settings ==================== diff --git a/pungi/phases/__init__.py b/pungi/phases/__init__.py index 81e319d9..ba72887f 100644 --- a/pungi/phases/__init__.py +++ b/pungi/phases/__init__.py @@ -28,3 +28,4 @@ from live_images import LiveImagesPhase # noqa from image_build import ImageBuildPhase # noqa from test import TestPhase # noqa from image_checksum import ImageChecksumPhase # noqa +from livemedia_phase import LiveMediaPhase # noqa diff --git a/pungi/phases/image_build.py b/pungi/phases/image_build.py index def1ecec..8a4a078f 100644 --- a/pungi/phases/image_build.py +++ b/pungi/phases/image_build.py @@ -155,9 +155,6 @@ class CreateImageBuildThread(WorkerThread): def worker(self, num, compose, cmd): arches = cmd['image_conf']['arches'].split(',') - mounts = [compose.paths.compose.topdir()] - if "mount" in cmd: - mounts.append(cmd["mount"]) log_file = compose.paths.log.log_file( cmd["image_conf"]["arches"], "imagebuild-%s-%s-%s" % ('-'.join(arches), @@ -181,7 +178,7 @@ class CreateImageBuildThread(WorkerThread): # avoid race conditions? # Kerberos authentication failed: Permission denied in replay cache code (-1765328215) time.sleep(num * 3) - output = koji_wrapper.run_create_image_cmd(koji_cmd, log_file=log_file) + output = koji_wrapper.run_blocking_cmd(koji_cmd, log_file=log_file) self.pool.log_debug("build-image outputs: %s" % (output)) if output["retcode"] != 0: self.fail(compose, cmd) @@ -190,7 +187,7 @@ class CreateImageBuildThread(WorkerThread): # copy image to images/ image_infos = [] - paths = koji_wrapper.get_image_build_paths(output["task_id"]) + paths = koji_wrapper.get_image_paths(output["task_id"]) for arch, paths in paths.iteritems(): for path in paths: diff --git a/pungi/phases/live_images.py b/pungi/phases/live_images.py index 685bf4f5..3f7225e4 100644 --- a/pungi/phases/live_images.py +++ b/pungi/phases/live_images.py @@ -201,7 +201,7 @@ class CreateLiveImageThread(WorkerThread): # Kerberos authentication failed: Permission denied in replay cache code (-1765328215) time.sleep(num * 3) - output = koji_wrapper.run_create_image_cmd(koji_cmd, log_file=log_file) + output = koji_wrapper.run_blocking_cmd(koji_cmd, log_file=log_file) if output["retcode"] != 0: self.fail(compose, cmd) raise RuntimeError("LiveImage task failed: %s. See %s for more details." % (output["task_id"], log_file)) diff --git a/pungi/phases/livemedia_phase.py b/pungi/phases/livemedia_phase.py new file mode 100644 index 00000000..ddb8fefa --- /dev/null +++ b/pungi/phases/livemedia_phase.py @@ -0,0 +1,178 @@ +# -*- coding: utf-8 -*- + +import os +import time +from kobo import shortcuts + +from pungi.util import get_variant_data, resolve_git_url, makedirs +from pungi.phases.base import PhaseBase +from pungi.linker import Linker +from pungi.paths import translate_path +from pungi.wrappers.kojiwrapper import KojiWrapper +from kobo.threads import ThreadPool, WorkerThread +from productmd.images import Image + + +class LiveMediaPhase(PhaseBase): + """class for wrapping up koji spin-livemedia""" + name = 'live_media' + + def __init__(self, compose): + super(LiveMediaPhase, self).__init__(compose) + self.pool = ThreadPool(logger=self.compose._logger) + + def skip(self): + if super(LiveMediaPhase, self).skip(): + return True + if not self.compose.conf.get(self.name): + self.compose.log_info("Config section '%s' was not found. Skipping" % self.name) + return True + return False + + def _get_repos(self, image_conf, variant): + """ + Get a comma separated list of repos. First included are those + explicitly listed in config, followed by repos from other variants, + finally followed by repo for current variant. + + The `repo_from` key is removed from the dict (if present). + """ + repo = shortcuts.force_list(image_conf.get('repo', [])) + + extras = shortcuts.force_list(image_conf.pop('repo_from', [])) + extras.append(variant.uid) + + for extra in extras: + v = self.compose.variants.get(extra) + if not v: + raise RuntimeError( + 'There is no variant %s to get repo from when building live media for %s.' + % (extra, variant.uid)) + repo.append(translate_path( + self.compose, + self.compose.paths.compose.repository('$arch', v, create_dir=False))) + + return repo + + def _get_arches(self, image_conf, arches): + if 'arches' in image_conf: + arches = set(image_conf.get('arches', [])) & arches + return sorted(arches) + + def _get_release(self, image_conf): + """If release is set explicitly to None, replace it with date and respin.""" + if 'release' in image_conf and image_conf['release'] is None: + return '%s.%s' % (self.compose.compose_date, self.compose.compose_respin) + return image_conf.get('release', None) + + def run(self): + for variant in self.compose.get_variants(): + arches = set([x for x in variant.arches if x != 'src']) + + for image_conf in get_variant_data(self.compose.conf, self.name, variant): + config = { + 'target': image_conf['target'], + 'arches': self._get_arches(image_conf, arches), + 'kickstart': image_conf['kickstart'], + 'ksurl': resolve_git_url(image_conf['ksurl']), + 'ksversion': image_conf.get('ksversion'), + 'scratch': image_conf.get('scratch', False), + 'release': self._get_release(image_conf), + 'skip_tag': image_conf.get('skip_tag'), + 'name': image_conf['name'], + 'title': image_conf.get('title'), + 'repo': self._get_repos(image_conf, variant), + 'install_tree': translate_path( + self.compose, + self.compose.paths.compose.os_tree('$arch', variant, create_dir=False) + ) + } + self.pool.add(LiveMediaThread(self.pool)) + self.pool.queue_put((self.compose, variant, config)) + + self.pool.start() + + +class LiveMediaThread(WorkerThread): + def process(self, item, num): + compose, variant, config = item + self.num = num + try: + self.worker(compose, variant, config) + except: + if not compose.can_fail(variant, '*', 'live-media'): + raise + else: + msg = ('[FAIL] live-media for variant %s failed, but going on anyway.' + % variant.uid) + self.pool.log_info(msg) + + def _get_log_file(self, compose, variant, config): + arches = '-'.join(config['arches']) + return compose.paths.log.log_file(arches, 'livemedia-%s' % variant) + + def _run_command(self, koji_wrapper, cmd, compose, log_file): + time.sleep(self.num * 3) + output = koji_wrapper.run_blocking_cmd(cmd, log_file=log_file) + self.pool.log_debug('live media outputs: %s' % (output)) + if output['retcode'] != 0: + compose.log_error('Live media task failed.') + raise RuntimeError('Live media task failed: %s. See %s for more details.' + % (output['task_id'], log_file)) + return output + + def worker(self, compose, variant, config): + msg = 'Live media: %s (arches: %s, variant: %s)' % (config['name'], + ' '.join(config['arches']), + variant.uid) + self.pool.log_info('[BEGIN] %s' % msg) + + koji_wrapper = KojiWrapper(compose.conf['koji_profile']) + cmd = koji_wrapper.get_live_media_cmd(config) + + log_file = self._get_log_file(compose, variant, config) + output = self._run_command(koji_wrapper, cmd, compose, log_file) + + # collect results and update manifest + image_infos = [] + + paths = koji_wrapper.get_image_paths(output['task_id']) + + for arch, paths in paths.iteritems(): + for path in paths: + if path.endswith('.iso'): + image_infos.append({'path': path, 'arch': arch}) + + if len(image_infos) != len(config['arches']): + self.pool.log_error( + 'Error in koji task %s. Expected to find one image for each arch (%s). Got %s.' + % (output['task_id'], len(config['arches']), len(image_infos))) + raise RuntimeError('Image count mismatch in task %s.' % output['task_id']) + + linker = Linker(logger=compose._logger) + link_type = compose.conf.get("link_type", "hardlink-or-copy") + for image_info in image_infos: + image_dir = compose.paths.compose.image_dir(variant) % {"arch": image_info['arch']} + makedirs(image_dir) + relative_image_dir = ( + compose.paths.compose.image_dir(variant, relative=True) % {"arch": image_info['arch']} + ) + + # let's not change filename of koji outputs + image_dest = os.path.join(image_dir, os.path.basename(image_info['path'])) + linker.link(image_info['path'], image_dest, link_type=link_type) + + # Update image manifest + img = Image(compose.im) + img.type = 'live' + img.format = 'iso' + img.path = os.path.join(relative_image_dir, os.path.basename(image_dest)) + img.mtime = int(os.stat(image_dest).st_mtime) + img.size = os.path.getsize(image_dest) + img.arch = image_info['arch'] + img.disc_number = 1 # We don't expect multiple disks + img.disc_count = 1 + img.bootable = True + compose.im.add(variant=variant.uid, arch=image_info['arch'], image=img) + + self.pool.log_info('[DONE ] %s' % msg) diff --git a/pungi/wrappers/kojiwrapper.py b/pungi/wrappers/kojiwrapper.py index 62e11751..b5b7afba 100644 --- a/pungi/wrappers/kojiwrapper.py +++ b/pungi/wrappers/kojiwrapper.py @@ -127,6 +127,33 @@ class KojiWrapper(object): return cmd + def get_live_media_cmd(self, options, wait=True): + # Usage: koji spin-livemedia [options] + cmd = ['koji', 'spin-livemedia'] + + for key in ('name', 'version', 'target', 'arch', 'ksfile'): + if key not in options: + raise ValueError('Expected options to have key "%s"' % key) + cmd.append(pipes.quote(options[key])) + + if 'install_tree' not in options: + raise ValueError('Expected options to have key "install_tree"') + cmd.append('--install-tree=%s' % pipes.quote(options['install_tree'])) + + for repo in options.get('repo', []): + cmd.append('--repo=%s' % pipes.quote(repo)) + + if options.get('scratch'): + cmd.append('--scratch') + + if options.get('skip_tag'): + cmd.append('--skip-tag') + + if wait: + cmd.append('--wait') + + return cmd + def get_create_image_cmd(self, name, version, target, arch, ks_file, repos, image_type="live", image_format=None, release=None, wait=True, archive=False, specfile=None): # Usage: koji spin-livecd [options] # Usage: koji spin-appliance [options] @@ -191,8 +218,12 @@ class KojiWrapper(object): return cmd - def run_create_image_cmd(self, command, log_file=None): - # spin-{livecd,appliance} is blocking by default -> you probably want to run it in a thread + def run_blocking_cmd(self, command, log_file=None): + """ + Run a blocking koji command. Returns a dict with output of the command, + its exit code and parsed task id. This method will block until the + command finishes. + """ try: retcode, output = run(command, can_fail=True, logfile=log_file) except RuntimeError, e: @@ -200,7 +231,8 @@ class KojiWrapper(object): match = re.search(r"Created task: (\d+)", output) if not match: - raise RuntimeError("Could not find task ID in output. Command '%s' returned '%s'." % (" ".join(command), output)) + raise RuntimeError("Could not find task ID in output. Command '%s' returned '%s'." + % (" ".join(command), output)) result = { "retcode": retcode, @@ -209,7 +241,7 @@ class KojiWrapper(object): } return result - def get_image_build_paths(self, task_id): + def get_image_paths(self, task_id): """ Given an image task in Koji, get a mapping from arches to a list of paths to results of the task. @@ -220,7 +252,7 @@ class KojiWrapper(object): children_tasks = self.koji_proxy.getTaskChildren(task_id, request=True) for child_task in children_tasks: - if child_task['method'] != 'createImage': + if child_task['method'] not in ['createImage', 'createLiveMedia']: continue is_scratch = child_task['request'][-1].get('scratch', False) diff --git a/tests/test_imagebuildphase.py b/tests/test_imagebuildphase.py index f3385a1c..e3c50fee 100755 --- a/tests/test_imagebuildphase.py +++ b/tests/test_imagebuildphase.py @@ -387,12 +387,12 @@ class TestCreateImageBuildThread(unittest.TestCase): "scratch": False, } koji_wrapper = KojiWrapper.return_value - koji_wrapper.run_create_image_cmd.return_value = { + koji_wrapper.run_blocking_cmd.return_value = { "retcode": 0, "output": None, "task_id": 1234, } - koji_wrapper.get_image_build_paths.return_value = { + koji_wrapper.get_image_paths.return_value = { 'amd64': [ '/koji/task/1235/tdl-amd64.xml', '/koji/task/1235/Fedora-Docker-Base-20160103.amd64.qcow2', @@ -506,7 +506,7 @@ class TestCreateImageBuildThread(unittest.TestCase): "link_type": 'hardlink-or-copy', } koji_wrapper = KojiWrapper.return_value - koji_wrapper.run_create_image_cmd.return_value = { + koji_wrapper.run_blocking_cmd.return_value = { "retcode": 1, "output": None, "task_id": 1234, @@ -558,7 +558,7 @@ class TestCreateImageBuildThread(unittest.TestCase): raise RuntimeError('BOOM') koji_wrapper = KojiWrapper.return_value - koji_wrapper.run_create_image_cmd.side_effect = boom + koji_wrapper.run_blocking_cmd.side_effect = boom t = CreateImageBuildThread(pool) with mock.patch('os.stat') as stat: diff --git a/tests/test_koji_wrapper.py b/tests/test_koji_wrapper.py index 3db3c3e5..a75a5eda 100755 --- a/tests/test_koji_wrapper.py +++ b/tests/test_koji_wrapper.py @@ -75,7 +75,7 @@ class KojiWrapperTest(unittest.TestCase): mock.call('distro = test-distro\n'), mock.call('\n')]) - def test_get_image_build_paths(self): + def test_get_image_paths(self): # The data for this tests is obtained from the actual Koji build. It # includes lots of fields that are not used, but for the sake of @@ -233,7 +233,7 @@ class KojiWrapperTest(unittest.TestCase): getTaskChildren=mock.Mock(side_effect=lambda task_id, request: getTaskChildren_data.get(task_id)), getTaskResult=mock.Mock(side_effect=lambda task_id: getTaskResult_data.get(task_id)) ) - result = self.koji.get_image_build_paths(12387273) + result = self.koji.get_image_paths(12387273) self.assertItemsEqual(result.keys(), ['i386', 'x86_64']) self.maxDiff = None self.assertItemsEqual(result['i386'], @@ -254,5 +254,45 @@ class KojiWrapperTest(unittest.TestCase): '/koji/task/12387277/Fedora-Cloud-Base-23-20160103.x86_64.raw.xz']) +class LiveMediaTestCase(unittest.TestCase): + def setUp(self): + self.koji_profile = mock.Mock() + with mock.patch('pungi.wrappers.kojiwrapper.koji') as koji: + koji.get_profile_module = mock.Mock( + return_value=mock.Mock( + pathinfo=mock.Mock( + work=mock.Mock(return_value='/koji'), + taskrelpath=mock.Mock(side_effect=lambda id: 'task/' + str(id)), + imagebuild=mock.Mock(side_effect=lambda id: '/koji/imagebuild/' + str(id)), + ) + ) + ) + self.koji_profile = koji.get_profile_module.return_value + self.koji = KojiWrapper('koji') + + def test_get_live_media_cmd_minimal(self): + opts = { + 'name': 'name', 'version': '1', 'target': 'tgt', 'arch': 'x,y,z', + 'ksfile': 'kickstart', 'install_tree': '/mnt/os' + } + cmd = self.koji.get_live_media_cmd(opts) + self.assertEqual(cmd, + ['koji', 'spin-livemedia', 'name', '1', 'tgt', 'x,y,z', 'kickstart', + '--install-tree=/mnt/os', '--wait']) + + def test_get_live_media_cmd_full(self): + opts = { + 'name': 'name', 'version': '1', 'target': 'tgt', 'arch': 'x,y,z', + 'ksfile': 'kickstart', 'install_tree': '/mnt/os', 'scratch': True, + 'repo': ['repo-1', 'repo-2'], 'skip_tag': True, + } + cmd = self.koji.get_live_media_cmd(opts) + self.assertEqual(cmd[:8], + ['koji', 'spin-livemedia', 'name', '1', 'tgt', 'x,y,z', 'kickstart', + '--install-tree=/mnt/os']) + self.assertItemsEqual(cmd[8:], + ['--repo=repo-1', '--repo=repo-2', '--skip-tag', '--scratch', '--wait']) + + if __name__ == "__main__": unittest.main() diff --git a/tests/test_liveimagesphase.py b/tests/test_liveimagesphase.py index c403e9e8..ead447c1 100755 --- a/tests/test_liveimagesphase.py +++ b/tests/test_liveimagesphase.py @@ -129,7 +129,7 @@ class TestCreateLiveImageThread(unittest.TestCase): koji_wrapper = KojiWrapper.return_value koji_wrapper.get_create_image_cmd.return_value = 'koji spin-livecd ...' - koji_wrapper.run_create_image_cmd.return_value = { + koji_wrapper.run_blocking_cmd.return_value = { 'retcode': 0, 'output': 'some output', 'task_id': 123 @@ -140,7 +140,7 @@ class TestCreateLiveImageThread(unittest.TestCase): with mock.patch('time.sleep'): t.process((compose, cmd, compose.variants['Client'], 'amd64'), 1) - self.assertEqual(koji_wrapper.run_create_image_cmd.mock_calls, + self.assertEqual(koji_wrapper.run_blocking_cmd.mock_calls, [mock.call('koji spin-livecd ...', log_file='/a/b/log/log_file')]) self.assertEqual(koji_wrapper.get_image_path.mock_calls, [mock.call(123)]) self.assertEqual(copy2.mock_calls, @@ -178,7 +178,7 @@ class TestCreateLiveImageThread(unittest.TestCase): koji_wrapper = KojiWrapper.return_value koji_wrapper.get_create_image_cmd.return_value = 'koji spin-livecd ...' - koji_wrapper.run_create_image_cmd.return_value = { + koji_wrapper.run_blocking_cmd.return_value = { 'retcode': 1, 'output': 'some output', 'task_id': 123 diff --git a/tests/test_livemediaphase.py b/tests/test_livemediaphase.py new file mode 100755 index 00000000..6b0dbf9e --- /dev/null +++ b/tests/test_livemediaphase.py @@ -0,0 +1,328 @@ +#!/usr/bin/env python2 +# -*- coding: utf-8 -*- + +import unittest +import mock + +import sys +import os + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..")) + +from pungi.phases.livemedia_phase import LiveMediaPhase, LiveMediaThread +from pungi.util import get_arch_variant_data + + +class _DummyCompose(object): + def __init__(self, config): + self.compose_date = '20151203' + self.compose_type_suffix = '.t' + self.compose_respin = 0 + self.ci_base = mock.Mock( + release_id='Test-1.0', + release=mock.Mock( + short='test', + version='1.0', + ), + ) + self.conf = config + self.paths = mock.Mock( + compose=mock.Mock( + topdir=mock.Mock(return_value='/a/b'), + os_tree=mock.Mock( + side_effect=lambda arch, variant, create_dir=False: os.path.join('/ostree', arch, variant.uid) + ), + repository=mock.Mock( + side_effect=lambda arch, variant, create_dir=False: os.path.join('/repo', arch, variant.uid) + ), + image_dir=mock.Mock( + side_effect=lambda variant, relative=False: os.path.join( + '' if relative else '/', 'image_dir', variant.uid, '%(arch)s' + ) + ) + ), + work=mock.Mock( + image_build_conf=mock.Mock( + side_effect=lambda variant, image_name, image_type: + '-'.join([variant.uid, image_name, image_type]) + ) + ), + log=mock.Mock( + log_file=mock.Mock(return_value='/a/b/log/log_file') + ) + ) + self._logger = mock.Mock() + self.variants = { + 'Server': mock.Mock(uid='Server', arches=['x86_64', 'amd64']), + 'Client': mock.Mock(uid='Client', arches=['amd64']), + 'Everything': mock.Mock(uid='Everything', arches=['x86_64', 'amd64']), + } + self.im = mock.Mock() + self.log_error = mock.Mock() + + def get_variants(self, arch=None, types=None): + return [v for v in self.variants.values() if not arch or arch in v.arches] + + def can_fail(self, variant, arch, deliverable): + failable = get_arch_variant_data(self.conf, 'failable_deliverables', arch, variant) + return deliverable in failable + + +class TestLiveMediaPhase(unittest.TestCase): + @mock.patch('pungi.phases.livemedia_phase.ThreadPool') + def test_live_media_minimal(self, ThreadPool): + compose = _DummyCompose({ + 'live_media': { + '^Server$': [ + { + 'target': 'f24', + 'kickstart': 'file.ks', + 'ksurl': 'git://example.com/repo.git', + 'name': 'Fedora Server Live', + } + ] + }, + 'koji_profile': 'koji', + }) + + phase = LiveMediaPhase(compose) + + phase.run() + self.assertTrue(phase.pool.add.called) + self.assertEqual(phase.pool.queue_put.call_args_list, + [mock.call((compose, + compose.variants['Server'], + { + 'arches': ['amd64', 'x86_64'], + 'kickstart': 'file.ks', + 'ksurl': 'git://example.com/repo.git', + 'ksversion': None, + 'name': 'Fedora Server Live', + 'release': None, + 'repo': ['/repo/$arch/Server'], + 'scratch': False, + 'skip_tag': None, + 'target': 'f24', + 'title': None, + 'install_tree': '/ostree/$arch/Server', + }))]) + + @mock.patch('pungi.phases.livemedia_phase.resolve_git_url') + @mock.patch('pungi.phases.livemedia_phase.ThreadPool') + def test_live_media_full(self, ThreadPool, resolve_git_url): + compose = _DummyCompose({ + 'live_media': { + '^Server$': [ + { + 'target': 'f24', + 'kickstart': 'file.ks', + 'ksurl': 'git://example.com/repo.git#HEAD', + 'name': 'Fedora Server Live', + 'scratch': True, + 'skip_tag': True, + 'title': 'Custom Title', + 'repo_from': ['Everything'], + 'repo': ['http://example.com/extra_repo'], + 'arches': ['x86_64'], + 'ksversion': '24', + 'release': None + } + ] + } + }) + + resolve_git_url.return_value = 'resolved' + + phase = LiveMediaPhase(compose) + + phase.run() + self.assertTrue(phase.pool.add.called) + self.assertEqual(phase.pool.queue_put.call_args_list, + [mock.call((compose, + compose.variants['Server'], + { + 'arches': ['x86_64'], + 'kickstart': 'file.ks', + 'ksurl': 'resolved', + 'ksversion': '24', + 'name': 'Fedora Server Live', + 'release': '20151203.0', + 'repo': ['http://example.com/extra_repo', + '/repo/$arch/Everything', + '/repo/$arch/Server'], + 'scratch': True, + 'skip_tag': True, + 'target': 'f24', + 'title': 'Custom Title', + 'install_tree': '/ostree/$arch/Server', + }))]) + + +class TestCreateImageBuildThread(unittest.TestCase): + + @mock.patch('pungi.phases.livemedia_phase.KojiWrapper') + @mock.patch('pungi.phases.livemedia_phase.Linker') + @mock.patch('pungi.phases.livemedia_phase.makedirs') + def test_process(self, makedirs, Linker, KojiWrapper): + compose = _DummyCompose({ + 'koji_profile': 'koji' + }) + config = { + 'arches': ['amd64', 'x86_64'], + 'kickstart': 'file.ks', + 'ksurl': 'git://example.com/repo.git', + 'ksversion': None, + 'name': 'Fedora Server Live', + 'release': None, + 'repo': ['/repo/$arch/Server'], + 'scratch': False, + 'skip_tag': None, + 'target': 'f24', + 'title': None, + } + pool = mock.Mock() + + get_live_media_cmd = KojiWrapper.return_value.get_live_media_cmd + get_live_media_cmd.return_value = 'koji-spin-livemedia' + + run_blocking_cmd = KojiWrapper.return_value.run_blocking_cmd + run_blocking_cmd.return_value = { + 'task_id': 1234, + 'retcode': 0, + 'output': None, + } + + get_image_paths = KojiWrapper.return_value.get_image_paths + get_image_paths.return_value = { + 'x86_64': [ + '/koji/task/1235/tdl-amd64.xml', + '/koji/task/1235/Live-20160103.x86_64.iso', + '/koji/task/1235/Live-20160103.x86_64.tar.xz' + ], + 'amd64': [ + '/koji/task/1235/tdl-amd64.xml', + '/koji/task/1235/Live-20160103.amd64.iso', + '/koji/task/1235/Live-20160103.amd64.tar.xz' + ] + } + + t = LiveMediaThread(pool) + with mock.patch('os.stat') as stat: + with mock.patch('os.path.getsize') as getsize: + with mock.patch('time.sleep'): + getsize.return_value = 1024 + stat.return_value.st_mtime = 13579 + t.process((compose, compose.variants['Server'], config), 1) + + self.assertEqual(run_blocking_cmd.mock_calls, + [mock.call('koji-spin-livemedia', log_file='/a/b/log/log_file')]) + self.assertEqual(get_live_media_cmd.mock_calls, + [mock.call(config)]) + self.assertEqual(get_image_paths.mock_calls, + [mock.call(1234)]) + self.assertItemsEqual(makedirs.mock_calls, + [mock.call('/image_dir/Server/x86_64'), + mock.call('/image_dir/Server/amd64')]) + link = Linker.return_value.link + self.assertItemsEqual(link.mock_calls, + [mock.call('/koji/task/1235/Live-20160103.amd64.iso', + '/image_dir/Server/amd64/Live-20160103.amd64.iso', + link_type='hardlink-or-copy'), + mock.call('/koji/task/1235/Live-20160103.x86_64.iso', + '/image_dir/Server/x86_64/Live-20160103.x86_64.iso', + link_type='hardlink-or-copy')]) + + image_relative_paths = [ + 'image_dir/Server/amd64/Live-20160103.amd64.iso', + 'image_dir/Server/x86_64/Live-20160103.x86_64.iso' + ] + + self.assertEqual(len(compose.im.add.call_args_list), 2) + for call in compose.im.add.call_args_list: + _, kwargs = call + image = kwargs['image'] + self.assertEqual(kwargs['variant'], 'Server') + self.assertIn(kwargs['arch'], ('amd64', 'x86_64')) + self.assertEqual(kwargs['arch'], image.arch) + self.assertIn(image.path, image_relative_paths) + self.assertEqual('iso', image.format) + self.assertEqual('live', image.type) + + @mock.patch('pungi.phases.livemedia_phase.KojiWrapper') + def test_handle_koji_fail(self, KojiWrapper): + compose = _DummyCompose({ + 'koji_profile': 'koji', + 'failable_deliverables': [ + ('^.+$', {'*': ['live-media']}) + ] + }) + config = { + 'arches': ['amd64', 'x86_64'], + 'kickstart': 'file.ks', + 'ksurl': 'git://example.com/repo.git', + 'ksversion': None, + 'name': 'Fedora Server Live', + 'release': None, + 'repo': ['/repo/$arch/Server'], + 'scratch': False, + 'skip_tag': None, + 'target': 'f24', + 'title': None, + } + pool = mock.Mock() + + run_blocking_cmd = KojiWrapper.return_value.run_blocking_cmd + run_blocking_cmd.return_value = { + 'task_id': 1234, + 'retcode': 1, + 'output': None, + } + + t = LiveMediaThread(pool) + with mock.patch('os.stat') as stat: + with mock.patch('os.path.getsize') as getsize: + with mock.patch('time.sleep'): + getsize.return_value = 1024 + stat.return_value.st_mtime = 13579 + t.process((compose, compose.variants['Server'], config), 1) + + @mock.patch('pungi.phases.livemedia_phase.KojiWrapper') + def test_handle_exception(self, KojiWrapper): + compose = _DummyCompose({ + 'koji_profile': 'koji', + 'failable_deliverables': [ + ('^.+$', {'*': ['live-media']}) + ] + }) + config = { + 'arches': ['amd64', 'x86_64'], + 'kickstart': 'file.ks', + 'ksurl': 'git://example.com/repo.git', + 'ksversion': None, + 'name': 'Fedora Server Live', + 'release': None, + 'repo': ['/repo/$arch/Server'], + 'scratch': False, + 'skip_tag': None, + 'target': 'f24', + 'title': None, + } + pool = mock.Mock() + + def boom(*args, **kwargs): + raise Exception('BOOM') + + run_blocking_cmd = KojiWrapper.return_value.run_blocking_cmd + run_blocking_cmd.side_effect = boom + + t = LiveMediaThread(pool) + with mock.patch('os.stat') as stat: + with mock.patch('os.path.getsize') as getsize: + with mock.patch('time.sleep'): + getsize.return_value = 1024 + stat.return_value.st_mtime = 13579 + t.process((compose, compose.variants['Server'], config), 1) + + +if __name__ == "__main__": + unittest.main()