diff --git a/doc/configuration.rst b/doc/configuration.rst index a2b05c63..2f88856f 100644 --- a/doc/configuration.rst +++ b/doc/configuration.rst @@ -624,6 +624,29 @@ Example }), ] + +Live Images Settings +==================== + +**live_target** + (*str*) -- Koji build target for which to build the images. This gets + passed to ``koji spin-livecd``. + +**live_images** + (*list*) -- Configuration for the particular image. The elements of the + list should be tuples ``(variant_uid_regex, {arch|*: config})``. The config + should be a dict with these keys: + + * ``kickstart`` (*str|dict*) + * ``name`` (*str*) + * ``version`` (*str*) + * ``additional_repos`` (*list*) -- external repos specified by URL + * ``repos_from`` (*list*) -- repos from other variants + * ``specfile`` (*str*) -- for images wrapped in RPM + * ``scratch`` (*bool*) -- only RPM-wrapped images can use scratch builds, + but by default this is turned off + + Image Build Settings ==================== diff --git a/pungi/phases/live_images.py b/pungi/phases/live_images.py index e8ccf437..685bf4f5 100644 --- a/pungi/phases/live_images.py +++ b/pungi/phases/live_images.py @@ -67,9 +67,21 @@ class LiveImagesPhase(PhaseBase): return True return False + def _get_extra_repos(self, arch, variant, extras): + repo = [] + for extra in extras: + v = self.compose.variants.get(extra) + if not v: + raise RuntimeError( + 'There is no variant %s to get repo from when building live image for %s.' + % (extra, variant.uid)) + repo.append(translate_path( + self.compose, self.compose.paths.compose.repository(arch, v, create_dir=False))) + + return repo + def run(self): symlink_isos_to = self.compose.conf.get("symlink_isos_to", None) - iso = IsoWrapper() commands = [] for variant in self.compose.variants.values(): @@ -87,22 +99,21 @@ class LiveImagesPhase(PhaseBase): cmd = { "name": None, "version": None, - "arch": arch, - "variant": variant, "iso_path": None, "wrapped_rpms_path": iso_dir, "build_arch": arch, "ks_file": ks_file, "specfile": None, "scratch": False, - "cmd": [], "label": "", # currently not used } - cmd["repos"] = [translate_path(self.compose, self.compose.paths.compose.repository(arch, variant))] + cmd["repos"] = [translate_path( + self.compose, self.compose.paths.compose.repository(arch, variant, create_dir=False))] # additional repos data = get_arch_variant_data(self.compose.conf, "live_images", arch, variant) cmd["repos"].extend(data[0].get("additional_repos", [])) + cmd['repos'].extend(self._get_extra_repos(arch, variant, data[0].get('repos_from', []))) # Explicit name and version cmd["name"] = data[0].get("name", None) @@ -133,17 +144,7 @@ class LiveImagesPhase(PhaseBase): self.compose.log_warning("Skipping creating live image, it already exists: %s" % iso_path) continue cmd["iso_path"] = iso_path - iso_name = os.path.basename(iso_path) - # Additional commands - - chdir_cmd = "cd %s" % pipes.quote(iso_dir) - cmd["cmd"].append(chdir_cmd) - - # create iso manifest - cmd["cmd"].append(iso.get_manifest_cmd(iso_name)) - - cmd["cmd"] = " && ".join(cmd["cmd"]) commands.append((cmd, variant, arch)) for (cmd, variant, arch) in commands: @@ -170,7 +171,7 @@ class CreateLiveImageThread(WorkerThread): def process(self, item, num): compose, cmd, variant, arch = item try: - self.worker(compose, cmd, num) + self.worker(compose, cmd, variant, arch, num) except: if not compose.can_fail(variant, arch, 'live'): raise @@ -179,10 +180,10 @@ class CreateLiveImageThread(WorkerThread): % (variant.uid, arch)) self.pool.log_info(msg) - def worker(self, compose, cmd, num): - log_file = compose.paths.log.log_file(cmd["arch"], "createiso-%s" % os.path.basename(cmd["iso_path"])) + def worker(self, compose, cmd, variant, arch, num): + log_file = compose.paths.log.log_file(arch, "createiso-%s" % os.path.basename(cmd["iso_path"])) - msg = "Creating ISO (arch: %s, variant: %s): %s" % (cmd["arch"], cmd["variant"], os.path.basename(cmd["iso_path"])) + msg = "Creating ISO (arch: %s, variant: %s): %s" % (arch, variant, os.path.basename(cmd["iso_path"])) self.pool.log_info("[BEGIN] %s" % msg) koji_wrapper = KojiWrapper(compose.conf["koji_profile"]) @@ -217,11 +218,19 @@ class CreateLiveImageThread(WorkerThread): for rpm_path in rpm_paths: shutil.copy2(rpm_path, cmd["wrapped_rpms_path"]) - # write manifest - run(cmd["cmd"]) + self._write_manifest(cmd['iso_path']) self.pool.log_info("[DONE ] %s" % msg) + def _write_manifest(self, iso_path): + """Generate manifest for ISO at given path. + + :param iso_path: (str) absolute path to the ISO + """ + dir, filename = os.path.split(iso_path) + iso = IsoWrapper() + run("cd %s && %s" % (pipes.quote(dir), iso.get_manifest_cmd(filename))) + def get_ks_in(compose, arch, variant): data = get_arch_variant_data(compose.conf, "live_images", arch, variant) diff --git a/tests/test_liveimagesphase.py b/tests/test_liveimagesphase.py new file mode 100755 index 00000000..c403e9e8 --- /dev/null +++ b/tests/test_liveimagesphase.py @@ -0,0 +1,227 @@ +#!/usr/bin/env python2 +# -*- coding: utf-8 -*- + + +import unittest +import mock + +import os +import sys + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..")) + +from pungi.phases.live_images import LiveImagesPhase, CreateLiveImageThread +from pungi.util import get_arch_variant_data + + +class _DummyCompose(object): + def __init__(self, config): + self.compose_id = 'Test-20151203.0.t' + self.conf = config + self.paths = mock.Mock( + compose=mock.Mock( + repository=mock.Mock( + side_effect=lambda arch, variant, create_dir=False: os.path.join('/repo', arch, variant.uid) + ), + iso_dir=mock.Mock( + side_effect=lambda arch, variant, symlink_to: os.path.join( + '/iso_dir', arch, variant.uid + ) + ), + iso_path=mock.Mock( + side_effect=lambda arch, variant, filename, symlink_to: os.path.join( + '/iso_dir', arch, variant.uid, filename + ) + ) + ), + log=mock.Mock( + log_file=mock.Mock(return_value='/a/b/log/log_file') + ) + ) + self._logger = mock.Mock() + self.variants = { + 'Server': mock.Mock(uid='Server', arches=['x86_64', 'amd64']), + 'Client': mock.Mock(uid='Client', arches=['amd64']), + 'Everything': mock.Mock(uid='Everything', arches=['x86_64', 'amd64']), + } + self.log_error = mock.Mock() + self.get_image_name = mock.Mock(return_value='image-name') + + def get_arches(self): + return ['x86_64', 'amd64'] + + def get_variants(self, arch=None, types=None): + return [v for v in self.variants.values() if not arch or arch in v.arches] + + def can_fail(self, variant, arch, deliverable): + failable = get_arch_variant_data(self.conf, 'failable_deliverables', arch, variant) + return deliverable in failable + + +class TestLiveImagesPhase(unittest.TestCase): + + @mock.patch('pungi.phases.live_images.ThreadPool') + @mock.patch('pungi.phases.live_images.get_ks_in') + @mock.patch('pungi.phases.live_images.tweak_ks') + def test_image_build(self, tweak_ks, get_ks_in, ThreadPool): + compose = _DummyCompose({ + 'live_images': [ + ('^Client$', { + 'amd64': { + 'additional_repos': ['http://example.com/repo/'], + 'repos_from': ['Everything'], + } + }) + ], + }) + + get_ks_in.side_effect = (lambda compose, arch, variant: + None if variant.uid != 'Client' or arch != 'amd64' else '/path/to/ks_in') + tweak_ks.return_value = '/path/to/ks_file' + + phase = LiveImagesPhase(compose) + + phase.run() + + # assert at least one thread was started + self.assertTrue(phase.pool.add.called) + self.maxDiff = None + self.assertItemsEqual(phase.pool.queue_put.mock_calls, + [mock.call((compose, + {'ks_file': '/path/to/ks_file', + 'build_arch': 'amd64', + 'wrapped_rpms_path': '/iso_dir/amd64/Client', + 'scratch': False, + 'repos': ['/repo/amd64/Client', + 'http://example.com/repo/', + '/repo/amd64/Everything'], + 'label': '', + 'name': None, + 'iso_path': '/iso_dir/amd64/Client/image-name', + 'version': None, + 'specfile': None}, + compose.variants['Client'], + 'amd64'))]) + + +class TestCreateLiveImageThread(unittest.TestCase): + + @mock.patch('shutil.copy2') + @mock.patch('pungi.phases.live_images.run') + @mock.patch('pungi.phases.live_images.KojiWrapper') + def test_process(self, KojiWrapper, run, copy2): + compose = _DummyCompose({'koji_profile': 'koji'}) + pool = mock.Mock() + cmd = { + 'ks_file': '/path/to/ks_file', + 'build_arch': 'amd64', + 'wrapped_rpms_path': '/iso_dir/amd64/Client', + 'scratch': False, + 'repos': ['/repo/amd64/Client', + 'http://example.com/repo/', + '/repo/amd64/Everything'], + 'label': '', + 'name': None, + 'iso_path': '/iso_dir/amd64/Client/image-name', + 'version': None, + 'specfile': None + } + + koji_wrapper = KojiWrapper.return_value + koji_wrapper.get_create_image_cmd.return_value = 'koji spin-livecd ...' + koji_wrapper.run_create_image_cmd.return_value = { + 'retcode': 0, + 'output': 'some output', + 'task_id': 123 + } + koji_wrapper.get_image_path.return_value = ['/path/to/image'] + + t = CreateLiveImageThread(pool) + with mock.patch('time.sleep'): + t.process((compose, cmd, compose.variants['Client'], 'amd64'), 1) + + self.assertEqual(koji_wrapper.run_create_image_cmd.mock_calls, + [mock.call('koji spin-livecd ...', log_file='/a/b/log/log_file')]) + self.assertEqual(koji_wrapper.get_image_path.mock_calls, [mock.call(123)]) + self.assertEqual(copy2.mock_calls, + [mock.call('/path/to/image', '/iso_dir/amd64/Client/image-name')]) + + write_manifest_cmd = ' && '.join([ + 'cd /iso_dir/amd64/Client', + 'isoinfo -R -f -i image-name | grep -v \'/TRANS.TBL$\' | sort >> image-name.manifest' + ]) + self.assertEqual(run.mock_calls, [mock.call(write_manifest_cmd)]) + + @mock.patch('shutil.copy2') + @mock.patch('pungi.phases.live_images.run') + @mock.patch('pungi.phases.live_images.KojiWrapper') + def test_process_handles_fail(self, KojiWrapper, run, copy2): + compose = _DummyCompose({ + 'koji_profile': 'koji', + 'failable_deliverables': [('^.+$', {'*': ['live']})], + }) + pool = mock.Mock() + cmd = { + 'ks_file': '/path/to/ks_file', + 'build_arch': 'amd64', + 'wrapped_rpms_path': '/iso_dir/amd64/Client', + 'scratch': False, + 'repos': ['/repo/amd64/Client', + 'http://example.com/repo/', + '/repo/amd64/Everything'], + 'label': '', + 'name': None, + 'iso_path': '/iso_dir/amd64/Client/image-name', + 'version': None, + 'specfile': None + } + + koji_wrapper = KojiWrapper.return_value + koji_wrapper.get_create_image_cmd.return_value = 'koji spin-livecd ...' + koji_wrapper.run_create_image_cmd.return_value = { + 'retcode': 1, + 'output': 'some output', + 'task_id': 123 + } + + t = CreateLiveImageThread(pool) + with mock.patch('time.sleep'): + t.process((compose, cmd, compose.variants['Client'], 'amd64'), 1) + + @mock.patch('shutil.copy2') + @mock.patch('pungi.phases.live_images.run') + @mock.patch('pungi.phases.live_images.KojiWrapper') + def test_process_handles_exception(self, KojiWrapper, run, copy2): + compose = _DummyCompose({ + 'koji_profile': 'koji', + 'failable_deliverables': [('^.+$', {'*': ['live']})], + }) + pool = mock.Mock() + cmd = { + 'ks_file': '/path/to/ks_file', + 'build_arch': 'amd64', + 'wrapped_rpms_path': '/iso_dir/amd64/Client', + 'scratch': False, + 'repos': ['/repo/amd64/Client', + 'http://example.com/repo/', + '/repo/amd64/Everything'], + 'label': '', + 'name': None, + 'iso_path': '/iso_dir/amd64/Client/image-name', + 'version': None, + 'specfile': None + } + + def boom(*args, **kwargs): + raise RuntimeError('BOOM') + + koji_wrapper = KojiWrapper.return_value + koji_wrapper.get_create_image_cmd.side_effect = boom + + t = CreateLiveImageThread(pool) + with mock.patch('time.sleep'): + t.process((compose, cmd, compose.variants['Client'], 'amd64'), 1) + + +if __name__ == "__main__": + unittest.main()