diff --git a/bin/pungi-koji b/bin/pungi-koji index 3f1441d3..5740ddaf 100755 --- a/bin/pungi-koji +++ b/bin/pungi-koji @@ -229,6 +229,7 @@ def run_compose(compose): productimg_phase = pungi.phases.ProductimgPhase(compose, pkgset_phase) createiso_phase = pungi.phases.CreateisoPhase(compose) liveimages_phase = pungi.phases.LiveImagesPhase(compose) + livemedia_phase = pungi.phases.LiveMediaPhase(compose) image_build_phase = pungi.phases.ImageBuildPhase(compose) image_checksum_phase = pungi.phases.ImageChecksumPhase(compose) test_phase = pungi.phases.TestPhase(compose) @@ -237,7 +238,8 @@ def run_compose(compose): for phase in (init_phase, pkgset_phase, createrepo_phase, buildinstall_phase, productimg_phase, gather_phase, extrafiles_phase, createiso_phase, liveimages_phase, - image_build_phase, image_checksum_phase, test_phase): + livemedia_phase, image_build_phase, image_checksum_phase, + test_phase): if phase.skip(): continue try: @@ -302,10 +304,12 @@ def run_compose(compose): createiso_phase.start() liveimages_phase.start() image_build_phase.start() + livemedia_phase.start() createiso_phase.stop() liveimages_phase.stop() image_build_phase.stop() + livemedia_phase.stop() image_checksum_phase.start() image_checksum_phase.stop() diff --git a/doc/configuration.rst b/doc/configuration.rst index 59977bab..fc87be48 100644 --- a/doc/configuration.rst +++ b/doc/configuration.rst @@ -135,6 +135,7 @@ Options * iso * live * image-build + * live-media .. note:: @@ -646,6 +647,30 @@ Live Images Settings * ``scratch`` (*bool*) -- only RPM-wrapped images can use scratch builds, but by default this is turned off +Live Media Settings +=================== + +**live_media** + (*dict*) -- configuration for ``koji spin-livemedia``; format: + ``{variant_uid_regex: [{opt:value}]}`` + + Available options: + + * ``target`` (*str*) + * ``arches`` (*[str]*) -- what architectures to build the media for; by default uses + all arches for the variant. + * ``kickstart`` (*str*) -- name of the kickstart file + * ``ksurl`` (*str*) + * ``ksversion`` (*str*) + * ``scratch`` (*bool*) + * ``release`` (*str*) -- a string with the release, or explicit ``None`` + for using compose date and respin. + * ``skip_tag`` (*bool*) + * ``name`` (*str*) + * ``repo`` (*[str]*) -- external repo + * ``repo_from`` (*[str]*) -- list of variants to take extra repos from + * ``title`` (*str*) + Image Build Settings ==================== diff --git a/pungi/phases/__init__.py b/pungi/phases/__init__.py index 81e319d9..ba72887f 100644 --- a/pungi/phases/__init__.py +++ b/pungi/phases/__init__.py @@ -28,3 +28,4 @@ from live_images import LiveImagesPhase # noqa from image_build import ImageBuildPhase # noqa from test import TestPhase # noqa from image_checksum import ImageChecksumPhase # noqa +from livemedia_phase import LiveMediaPhase # noqa diff --git a/pungi/phases/livemedia_phase.py b/pungi/phases/livemedia_phase.py new file mode 100644 index 00000000..ddb8fefa --- /dev/null +++ b/pungi/phases/livemedia_phase.py @@ -0,0 +1,178 @@ +# -*- coding: utf-8 -*- + +import os +import time +from kobo import shortcuts + +from pungi.util import get_variant_data, resolve_git_url, makedirs +from pungi.phases.base import PhaseBase +from pungi.linker import Linker +from pungi.paths import translate_path +from pungi.wrappers.kojiwrapper import KojiWrapper +from kobo.threads import ThreadPool, WorkerThread +from productmd.images import Image + + +class LiveMediaPhase(PhaseBase): + """class for wrapping up koji spin-livemedia""" + name = 'live_media' + + def __init__(self, compose): + super(LiveMediaPhase, self).__init__(compose) + self.pool = ThreadPool(logger=self.compose._logger) + + def skip(self): + if super(LiveMediaPhase, self).skip(): + return True + if not self.compose.conf.get(self.name): + self.compose.log_info("Config section '%s' was not found. Skipping" % self.name) + return True + return False + + def _get_repos(self, image_conf, variant): + """ + Get a comma separated list of repos. First included are those + explicitly listed in config, followed by repos from other variants, + finally followed by repo for current variant. + + The `repo_from` key is removed from the dict (if present). + """ + repo = shortcuts.force_list(image_conf.get('repo', [])) + + extras = shortcuts.force_list(image_conf.pop('repo_from', [])) + extras.append(variant.uid) + + for extra in extras: + v = self.compose.variants.get(extra) + if not v: + raise RuntimeError( + 'There is no variant %s to get repo from when building live media for %s.' + % (extra, variant.uid)) + repo.append(translate_path( + self.compose, + self.compose.paths.compose.repository('$arch', v, create_dir=False))) + + return repo + + def _get_arches(self, image_conf, arches): + if 'arches' in image_conf: + arches = set(image_conf.get('arches', [])) & arches + return sorted(arches) + + def _get_release(self, image_conf): + """If release is set explicitly to None, replace it with date and respin.""" + if 'release' in image_conf and image_conf['release'] is None: + return '%s.%s' % (self.compose.compose_date, self.compose.compose_respin) + return image_conf.get('release', None) + + def run(self): + for variant in self.compose.get_variants(): + arches = set([x for x in variant.arches if x != 'src']) + + for image_conf in get_variant_data(self.compose.conf, self.name, variant): + config = { + 'target': image_conf['target'], + 'arches': self._get_arches(image_conf, arches), + 'kickstart': image_conf['kickstart'], + 'ksurl': resolve_git_url(image_conf['ksurl']), + 'ksversion': image_conf.get('ksversion'), + 'scratch': image_conf.get('scratch', False), + 'release': self._get_release(image_conf), + 'skip_tag': image_conf.get('skip_tag'), + 'name': image_conf['name'], + 'title': image_conf.get('title'), + 'repo': self._get_repos(image_conf, variant), + 'install_tree': translate_path( + self.compose, + self.compose.paths.compose.os_tree('$arch', variant, create_dir=False) + ) + } + self.pool.add(LiveMediaThread(self.pool)) + self.pool.queue_put((self.compose, variant, config)) + + self.pool.start() + + +class LiveMediaThread(WorkerThread): + def process(self, item, num): + compose, variant, config = item + self.num = num + try: + self.worker(compose, variant, config) + except: + if not compose.can_fail(variant, '*', 'live-media'): + raise + else: + msg = ('[FAIL] live-media for variant %s failed, but going on anyway.' + % variant.uid) + self.pool.log_info(msg) + + def _get_log_file(self, compose, variant, config): + arches = '-'.join(config['arches']) + return compose.paths.log.log_file(arches, 'livemedia-%s' % variant) + + def _run_command(self, koji_wrapper, cmd, compose, log_file): + time.sleep(self.num * 3) + output = koji_wrapper.run_blocking_cmd(cmd, log_file=log_file) + self.pool.log_debug('live media outputs: %s' % (output)) + if output['retcode'] != 0: + compose.log_error('Live media task failed.') + raise RuntimeError('Live media task failed: %s. See %s for more details.' + % (output['task_id'], log_file)) + return output + + def worker(self, compose, variant, config): + msg = 'Live media: %s (arches: %s, variant: %s)' % (config['name'], + ' '.join(config['arches']), + variant.uid) + self.pool.log_info('[BEGIN] %s' % msg) + + koji_wrapper = KojiWrapper(compose.conf['koji_profile']) + cmd = koji_wrapper.get_live_media_cmd(config) + + log_file = self._get_log_file(compose, variant, config) + output = self._run_command(koji_wrapper, cmd, compose, log_file) + + # collect results and update manifest + image_infos = [] + + paths = koji_wrapper.get_image_paths(output['task_id']) + + for arch, paths in paths.iteritems(): + for path in paths: + if path.endswith('.iso'): + image_infos.append({'path': path, 'arch': arch}) + + if len(image_infos) != len(config['arches']): + self.pool.log_error( + 'Error in koji task %s. Expected to find one image for each arch (%s). Got %s.' + % (output['task_id'], len(config['arches']), len(image_infos))) + raise RuntimeError('Image count mismatch in task %s.' % output['task_id']) + + linker = Linker(logger=compose._logger) + link_type = compose.conf.get("link_type", "hardlink-or-copy") + for image_info in image_infos: + image_dir = compose.paths.compose.image_dir(variant) % {"arch": image_info['arch']} + makedirs(image_dir) + relative_image_dir = ( + compose.paths.compose.image_dir(variant, relative=True) % {"arch": image_info['arch']} + ) + + # let's not change filename of koji outputs + image_dest = os.path.join(image_dir, os.path.basename(image_info['path'])) + linker.link(image_info['path'], image_dest, link_type=link_type) + + # Update image manifest + img = Image(compose.im) + img.type = 'live' + img.format = 'iso' + img.path = os.path.join(relative_image_dir, os.path.basename(image_dest)) + img.mtime = int(os.stat(image_dest).st_mtime) + img.size = os.path.getsize(image_dest) + img.arch = image_info['arch'] + img.disc_number = 1 # We don't expect multiple disks + img.disc_count = 1 + img.bootable = True + compose.im.add(variant=variant.uid, arch=image_info['arch'], image=img) + + self.pool.log_info('[DONE ] %s' % msg) diff --git a/tests/test_livemediaphase.py b/tests/test_livemediaphase.py new file mode 100755 index 00000000..6b0dbf9e --- /dev/null +++ b/tests/test_livemediaphase.py @@ -0,0 +1,328 @@ +#!/usr/bin/env python2 +# -*- coding: utf-8 -*- + +import unittest +import mock + +import sys +import os + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..")) + +from pungi.phases.livemedia_phase import LiveMediaPhase, LiveMediaThread +from pungi.util import get_arch_variant_data + + +class _DummyCompose(object): + def __init__(self, config): + self.compose_date = '20151203' + self.compose_type_suffix = '.t' + self.compose_respin = 0 + self.ci_base = mock.Mock( + release_id='Test-1.0', + release=mock.Mock( + short='test', + version='1.0', + ), + ) + self.conf = config + self.paths = mock.Mock( + compose=mock.Mock( + topdir=mock.Mock(return_value='/a/b'), + os_tree=mock.Mock( + side_effect=lambda arch, variant, create_dir=False: os.path.join('/ostree', arch, variant.uid) + ), + repository=mock.Mock( + side_effect=lambda arch, variant, create_dir=False: os.path.join('/repo', arch, variant.uid) + ), + image_dir=mock.Mock( + side_effect=lambda variant, relative=False: os.path.join( + '' if relative else '/', 'image_dir', variant.uid, '%(arch)s' + ) + ) + ), + work=mock.Mock( + image_build_conf=mock.Mock( + side_effect=lambda variant, image_name, image_type: + '-'.join([variant.uid, image_name, image_type]) + ) + ), + log=mock.Mock( + log_file=mock.Mock(return_value='/a/b/log/log_file') + ) + ) + self._logger = mock.Mock() + self.variants = { + 'Server': mock.Mock(uid='Server', arches=['x86_64', 'amd64']), + 'Client': mock.Mock(uid='Client', arches=['amd64']), + 'Everything': mock.Mock(uid='Everything', arches=['x86_64', 'amd64']), + } + self.im = mock.Mock() + self.log_error = mock.Mock() + + def get_variants(self, arch=None, types=None): + return [v for v in self.variants.values() if not arch or arch in v.arches] + + def can_fail(self, variant, arch, deliverable): + failable = get_arch_variant_data(self.conf, 'failable_deliverables', arch, variant) + return deliverable in failable + + +class TestLiveMediaPhase(unittest.TestCase): + @mock.patch('pungi.phases.livemedia_phase.ThreadPool') + def test_live_media_minimal(self, ThreadPool): + compose = _DummyCompose({ + 'live_media': { + '^Server$': [ + { + 'target': 'f24', + 'kickstart': 'file.ks', + 'ksurl': 'git://example.com/repo.git', + 'name': 'Fedora Server Live', + } + ] + }, + 'koji_profile': 'koji', + }) + + phase = LiveMediaPhase(compose) + + phase.run() + self.assertTrue(phase.pool.add.called) + self.assertEqual(phase.pool.queue_put.call_args_list, + [mock.call((compose, + compose.variants['Server'], + { + 'arches': ['amd64', 'x86_64'], + 'kickstart': 'file.ks', + 'ksurl': 'git://example.com/repo.git', + 'ksversion': None, + 'name': 'Fedora Server Live', + 'release': None, + 'repo': ['/repo/$arch/Server'], + 'scratch': False, + 'skip_tag': None, + 'target': 'f24', + 'title': None, + 'install_tree': '/ostree/$arch/Server', + }))]) + + @mock.patch('pungi.phases.livemedia_phase.resolve_git_url') + @mock.patch('pungi.phases.livemedia_phase.ThreadPool') + def test_live_media_full(self, ThreadPool, resolve_git_url): + compose = _DummyCompose({ + 'live_media': { + '^Server$': [ + { + 'target': 'f24', + 'kickstart': 'file.ks', + 'ksurl': 'git://example.com/repo.git#HEAD', + 'name': 'Fedora Server Live', + 'scratch': True, + 'skip_tag': True, + 'title': 'Custom Title', + 'repo_from': ['Everything'], + 'repo': ['http://example.com/extra_repo'], + 'arches': ['x86_64'], + 'ksversion': '24', + 'release': None + } + ] + } + }) + + resolve_git_url.return_value = 'resolved' + + phase = LiveMediaPhase(compose) + + phase.run() + self.assertTrue(phase.pool.add.called) + self.assertEqual(phase.pool.queue_put.call_args_list, + [mock.call((compose, + compose.variants['Server'], + { + 'arches': ['x86_64'], + 'kickstart': 'file.ks', + 'ksurl': 'resolved', + 'ksversion': '24', + 'name': 'Fedora Server Live', + 'release': '20151203.0', + 'repo': ['http://example.com/extra_repo', + '/repo/$arch/Everything', + '/repo/$arch/Server'], + 'scratch': True, + 'skip_tag': True, + 'target': 'f24', + 'title': 'Custom Title', + 'install_tree': '/ostree/$arch/Server', + }))]) + + +class TestCreateImageBuildThread(unittest.TestCase): + + @mock.patch('pungi.phases.livemedia_phase.KojiWrapper') + @mock.patch('pungi.phases.livemedia_phase.Linker') + @mock.patch('pungi.phases.livemedia_phase.makedirs') + def test_process(self, makedirs, Linker, KojiWrapper): + compose = _DummyCompose({ + 'koji_profile': 'koji' + }) + config = { + 'arches': ['amd64', 'x86_64'], + 'kickstart': 'file.ks', + 'ksurl': 'git://example.com/repo.git', + 'ksversion': None, + 'name': 'Fedora Server Live', + 'release': None, + 'repo': ['/repo/$arch/Server'], + 'scratch': False, + 'skip_tag': None, + 'target': 'f24', + 'title': None, + } + pool = mock.Mock() + + get_live_media_cmd = KojiWrapper.return_value.get_live_media_cmd + get_live_media_cmd.return_value = 'koji-spin-livemedia' + + run_blocking_cmd = KojiWrapper.return_value.run_blocking_cmd + run_blocking_cmd.return_value = { + 'task_id': 1234, + 'retcode': 0, + 'output': None, + } + + get_image_paths = KojiWrapper.return_value.get_image_paths + get_image_paths.return_value = { + 'x86_64': [ + '/koji/task/1235/tdl-amd64.xml', + '/koji/task/1235/Live-20160103.x86_64.iso', + '/koji/task/1235/Live-20160103.x86_64.tar.xz' + ], + 'amd64': [ + '/koji/task/1235/tdl-amd64.xml', + '/koji/task/1235/Live-20160103.amd64.iso', + '/koji/task/1235/Live-20160103.amd64.tar.xz' + ] + } + + t = LiveMediaThread(pool) + with mock.patch('os.stat') as stat: + with mock.patch('os.path.getsize') as getsize: + with mock.patch('time.sleep'): + getsize.return_value = 1024 + stat.return_value.st_mtime = 13579 + t.process((compose, compose.variants['Server'], config), 1) + + self.assertEqual(run_blocking_cmd.mock_calls, + [mock.call('koji-spin-livemedia', log_file='/a/b/log/log_file')]) + self.assertEqual(get_live_media_cmd.mock_calls, + [mock.call(config)]) + self.assertEqual(get_image_paths.mock_calls, + [mock.call(1234)]) + self.assertItemsEqual(makedirs.mock_calls, + [mock.call('/image_dir/Server/x86_64'), + mock.call('/image_dir/Server/amd64')]) + link = Linker.return_value.link + self.assertItemsEqual(link.mock_calls, + [mock.call('/koji/task/1235/Live-20160103.amd64.iso', + '/image_dir/Server/amd64/Live-20160103.amd64.iso', + link_type='hardlink-or-copy'), + mock.call('/koji/task/1235/Live-20160103.x86_64.iso', + '/image_dir/Server/x86_64/Live-20160103.x86_64.iso', + link_type='hardlink-or-copy')]) + + image_relative_paths = [ + 'image_dir/Server/amd64/Live-20160103.amd64.iso', + 'image_dir/Server/x86_64/Live-20160103.x86_64.iso' + ] + + self.assertEqual(len(compose.im.add.call_args_list), 2) + for call in compose.im.add.call_args_list: + _, kwargs = call + image = kwargs['image'] + self.assertEqual(kwargs['variant'], 'Server') + self.assertIn(kwargs['arch'], ('amd64', 'x86_64')) + self.assertEqual(kwargs['arch'], image.arch) + self.assertIn(image.path, image_relative_paths) + self.assertEqual('iso', image.format) + self.assertEqual('live', image.type) + + @mock.patch('pungi.phases.livemedia_phase.KojiWrapper') + def test_handle_koji_fail(self, KojiWrapper): + compose = _DummyCompose({ + 'koji_profile': 'koji', + 'failable_deliverables': [ + ('^.+$', {'*': ['live-media']}) + ] + }) + config = { + 'arches': ['amd64', 'x86_64'], + 'kickstart': 'file.ks', + 'ksurl': 'git://example.com/repo.git', + 'ksversion': None, + 'name': 'Fedora Server Live', + 'release': None, + 'repo': ['/repo/$arch/Server'], + 'scratch': False, + 'skip_tag': None, + 'target': 'f24', + 'title': None, + } + pool = mock.Mock() + + run_blocking_cmd = KojiWrapper.return_value.run_blocking_cmd + run_blocking_cmd.return_value = { + 'task_id': 1234, + 'retcode': 1, + 'output': None, + } + + t = LiveMediaThread(pool) + with mock.patch('os.stat') as stat: + with mock.patch('os.path.getsize') as getsize: + with mock.patch('time.sleep'): + getsize.return_value = 1024 + stat.return_value.st_mtime = 13579 + t.process((compose, compose.variants['Server'], config), 1) + + @mock.patch('pungi.phases.livemedia_phase.KojiWrapper') + def test_handle_exception(self, KojiWrapper): + compose = _DummyCompose({ + 'koji_profile': 'koji', + 'failable_deliverables': [ + ('^.+$', {'*': ['live-media']}) + ] + }) + config = { + 'arches': ['amd64', 'x86_64'], + 'kickstart': 'file.ks', + 'ksurl': 'git://example.com/repo.git', + 'ksversion': None, + 'name': 'Fedora Server Live', + 'release': None, + 'repo': ['/repo/$arch/Server'], + 'scratch': False, + 'skip_tag': None, + 'target': 'f24', + 'title': None, + } + pool = mock.Mock() + + def boom(*args, **kwargs): + raise Exception('BOOM') + + run_blocking_cmd = KojiWrapper.return_value.run_blocking_cmd + run_blocking_cmd.side_effect = boom + + t = LiveMediaThread(pool) + with mock.patch('os.stat') as stat: + with mock.patch('os.path.getsize') as getsize: + with mock.patch('time.sleep'): + getsize.return_value = 1024 + stat.return_value.st_mtime = 13579 + t.process((compose, compose.variants['Server'], config), 1) + + +if __name__ == "__main__": + unittest.main()