diff --git a/doc/configuration.rst b/doc/configuration.rst index ae010266..a9960f76 100644 --- a/doc/configuration.rst +++ b/doc/configuration.rst @@ -135,6 +135,12 @@ Options * buildinstall * iso * live + * image-build + + .. note:: + + Image building is not run per-architecture. If you want to mark it + as failable, specify it in a block with arch set as ``*``. Please note that ``*`` as a wildcard matches all architectures but ``src``. @@ -618,7 +624,13 @@ Image Build Settings automatically transformed into format suitable for ``koji``. A repo for the currently built variant will be added as well. - Please don't set ``install_tree`` as it would get overriden by pungi. + You can also add extra variants to get repos from with key ``repo_from``. + The value should be a list of variant names. + + Please don't set ``install_tree``. This gets automatically set by *pungi* + based on current variant. You can use ``install_tree_from`` key to use + install tree from another variant. + The ``format`` attr is [('image_type', 'image_suffix'), ...]. See productmd documentation for list of supported types and suffixes. @@ -660,6 +672,10 @@ Example # only build this type of image on x86_64 'arches': ['x86_64'] + + # Use install tree and repo from Everything variant. + 'install_tree_from': 'Everything', + 'repo_from': ['Everything'], } ] } diff --git a/pungi/phases/image_build.py b/pungi/phases/image_build.py index 436ccbc5..0d85ae15 100644 --- a/pungi/phases/image_build.py +++ b/pungi/phases/image_build.py @@ -3,6 +3,7 @@ import copy import os import time +from kobo import shortcuts from pungi.util import get_variant_data, resolve_git_url from pungi.phases.base import PhaseBase @@ -29,6 +30,52 @@ class ImageBuildPhase(PhaseBase): return True return False + def _get_install_tree(self, image_conf, variant): + """ + Get a path to os tree for a variant specified in `install_tree_from` or + current variant. If the config is set, it will be removed from the + dict. + """ + install_tree_from = image_conf.pop('install_tree_from', variant.uid) + install_tree_source = self.compose.variants.get(install_tree_from) + if not install_tree_source: + raise RuntimeError( + 'There is no variant %s to get install tree from when building image for %s.' + % (install_tree_from, variant.uid)) + return translate_path( + self.compose, + self.compose.paths.compose.os_tree('$arch', install_tree_source) + ) + + def _get_repo(self, image_conf, variant): + """ + Get a comma separated list of repos. First included are those + explicitly listed in config, followed by repos from other variants, + finally followed by repo for current variant. + + The `repo_from` key is removed from the dict (if present). + """ + repo = shortcuts.force_list(image_conf.get('repo', [])) + + extras = shortcuts.force_list(image_conf.pop('repo_from', [])) + extras.append(variant.uid) + + for extra in extras: + v = self.compose.variants.get(extra) + if not v: + raise RuntimeError( + 'There is no variant %s to get repo from when building image for %s.' + % (extra, variant.uid)) + repo.append(translate_path(self.compose, + self.compose.paths.compose.os_tree('$arch', v))) + + return ",".join(repo) + + def _get_arches(self, image_conf, arches): + if 'arches' in image_conf: + arches = set(image_conf.get('arches', [])) & arches + return ','.join(sorted(arches)) + def run(self): for variant in self.compose.get_variants(): arches = set([x for x in variant.arches if x != 'src']) @@ -39,36 +86,26 @@ class ImageBuildPhase(PhaseBase): # value is needed. image_conf = copy.deepcopy(image_conf) + # image_conf is passed to get_image_build_cmd as dict + + image_conf['arches'] = self._get_arches(image_conf, arches) + if not image_conf['arches']: + continue + # Replace possible ambiguous ref name with explicit hash. if 'ksurl' in image_conf: image_conf['ksurl'] = resolve_git_url(image_conf['ksurl']) - # image_conf is passed to get_image_build_cmd as dict - - if 'arches' in image_conf: - image_conf["arches"] = ','.join(sorted(set(image_conf.get('arches', [])) & arches)) - else: - image_conf['arches'] = ','.join(sorted(arches)) - - if not image_conf['arches']: - continue - image_conf["variant"] = variant - image_conf["install_tree"] = translate_path( - self.compose, - self.compose.paths.compose.os_tree('$arch', variant) - ) + + image_conf["install_tree"] = self._get_install_tree(image_conf, variant) + # transform format into right 'format' for image-build # e.g. 'docker,qcow2' format = image_conf["format"] image_conf["format"] = ",".join([x[0] for x in image_conf["format"]]) - repo = image_conf.get('repo', []) - if isinstance(repo, str): - repo = [repo] - repo.append(translate_path(self.compose, self.compose.paths.compose.os_tree('$arch', variant))) - # supply repo as str separated by , instead of list - image_conf['repo'] = ",".join(repo) + image_conf['repo'] = self._get_repo(image_conf, variant) cmd = { "format": format, @@ -96,6 +133,17 @@ class CreateImageBuildThread(WorkerThread): def process(self, item, num): compose, cmd = item + try: + self.worker(num, compose, cmd) + except: + if not compose.can_fail(cmd['image_conf']['variant'], '*', 'image-build'): + raise + else: + msg = ('[FAIL] image-build for variant %s failed, but going on anyway.' + % cmd['image_conf']['variant']) + self.pool.log_info(msg) + + def worker(self, num, compose, cmd): arches = cmd['image_conf']['arches'].split(',') mounts = [compose.paths.compose.topdir()] diff --git a/tests/test_compose.py b/tests/test_compose.py index a23bf31f..556a49a8 100755 --- a/tests/test_compose.py +++ b/tests/test_compose.py @@ -40,6 +40,9 @@ class ComposeTestCase(unittest.TestCase): self.assertFalse(compose.can_fail(None, 'x86_64', 'live')) self.assertTrue(compose.can_fail(None, 'i386', 'live')) + self.assertTrue(compose.can_fail(variant, '*', 'buildinstall')) + self.assertFalse(compose.can_fail(variant, '*', 'live')) + @mock.patch('pungi.compose.ComposeInfo') def test_get_image_name(self, ci): conf = {} diff --git a/tests/test_imagebuildphase.py b/tests/test_imagebuildphase.py index 448d783d..91026eac 100755 --- a/tests/test_imagebuildphase.py +++ b/tests/test_imagebuildphase.py @@ -11,6 +11,7 @@ import sys sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..")) from pungi.phases.image_build import ImageBuildPhase, CreateImageBuildThread +from pungi.util import get_arch_variant_data class _DummyCompose(object): @@ -49,17 +50,23 @@ class _DummyCompose(object): ) ) self._logger = mock.Mock() - self.variants = [ - mock.Mock(uid='Server', arches=['x86_64', 'amd64']), - mock.Mock(uid='Client', arches=['amd64']), - ] + self.variants = { + 'Server': mock.Mock(uid='Server', arches=['x86_64', 'amd64']), + 'Client': mock.Mock(uid='Client', arches=['amd64']), + 'Everything': mock.Mock(uid='Everything', arches=['x86_64', 'amd64']), + } self.im = mock.Mock() + self.log_error = mock.Mock() def get_arches(self): return ['x86_64', 'amd64'] def get_variants(self, arch=None, types=None): - return [v for v in self.variants if not arch or arch in v.arches] + return [v for v in self.variants.values() if not arch or arch in v.arches] + + def can_fail(self, variant, arch, deliverable): + failable = get_arch_variant_data(self.conf, 'failable_deliverables', arch, variant) + return deliverable in failable class TestImageBuildPhase(unittest.TestCase): @@ -97,7 +104,7 @@ class TestImageBuildPhase(unittest.TestCase): 'kickstart': 'fedora-docker-base.ks', 'format': 'docker', 'repo': '/ostree/$arch/Client', - 'variant': compose.variants[1], + 'variant': compose.variants['Client'], 'target': 'f24', 'disk_size': 3, 'name': 'Fedora-Docker-Base', @@ -118,7 +125,7 @@ class TestImageBuildPhase(unittest.TestCase): 'kickstart': 'fedora-docker-base.ks', 'format': 'docker', 'repo': '/ostree/$arch/Server', - 'variant': compose.variants[0], + 'variant': compose.variants['Server'], 'target': 'f24', 'disk_size': 3, 'name': 'Fedora-Docker-Base', @@ -165,6 +172,116 @@ class TestImageBuildPhase(unittest.TestCase): self.assertFalse(phase.pool.add.called) self.assertFalse(phase.pool.queue_put.called) + @mock.patch('pungi.phases.image_build.ThreadPool') + def test_image_build_set_install_tree(self, ThreadPool): + compose = _DummyCompose({ + 'image_build': { + '^Server$': [ + { + 'format': [('docker', 'tar.xz')], + 'name': 'Fedora-Docker-Base', + 'target': 'f24', + 'version': 'Rawhide', + 'ksurl': 'git://git.fedorahosted.org/git/spin-kickstarts.git', + 'kickstart': "fedora-docker-base.ks", + 'distro': 'Fedora-20', + 'disk_size': 3, + 'arches': ['x86_64'], + 'install_tree_from': 'Everything', + } + ] + }, + 'koji_profile': 'koji', + }) + + phase = ImageBuildPhase(compose) + + phase.run() + + # assert at least one thread was started + self.assertTrue(phase.pool.add.called) + + self.assertTrue(phase.pool.queue_put.called_once) + args, kwargs = phase.pool.queue_put.call_args + self.assertEqual(args[0][0], compose) + self.maxDiff = None + self.assertDictEqual(args[0][1], { + "format": [('docker', 'tar.xz')], + "image_conf": { + 'install_tree': '/ostree/$arch/Everything', + 'kickstart': 'fedora-docker-base.ks', + 'format': 'docker', + 'repo': '/ostree/$arch/Server', + 'variant': compose.variants['Server'], + 'target': 'f24', + 'disk_size': 3, + 'name': 'Fedora-Docker-Base', + 'arches': 'x86_64', + 'version': 'Rawhide', + 'ksurl': 'git://git.fedorahosted.org/git/spin-kickstarts.git', + 'distro': 'Fedora-20', + }, + "conf_file": 'Server-Fedora-Docker-Base-docker', + "image_dir": '/image_dir/Server/%(arch)s', + "relative_image_dir": 'image_dir/Server/%(arch)s', + "link_type": 'hardlink-or-copy', + }) + + @mock.patch('pungi.phases.image_build.ThreadPool') + def test_image_build_set_extra_repos(self, ThreadPool): + compose = _DummyCompose({ + 'image_build': { + '^Server$': [ + { + 'format': [('docker', 'tar.xz')], + 'name': 'Fedora-Docker-Base', + 'target': 'f24', + 'version': 'Rawhide', + 'ksurl': 'git://git.fedorahosted.org/git/spin-kickstarts.git', + 'kickstart': "fedora-docker-base.ks", + 'distro': 'Fedora-20', + 'disk_size': 3, + 'arches': ['x86_64'], + 'repo_from': 'Everything', + } + ] + }, + 'koji_profile': 'koji', + }) + + phase = ImageBuildPhase(compose) + + phase.run() + + # assert at least one thread was started + self.assertTrue(phase.pool.add.called) + + self.assertTrue(phase.pool.queue_put.called_once) + args, kwargs = phase.pool.queue_put.call_args + self.assertEqual(args[0][0], compose) + self.maxDiff = None + self.assertDictEqual(args[0][1], { + "format": [('docker', 'tar.xz')], + "image_conf": { + 'install_tree': '/ostree/$arch/Server', + 'kickstart': 'fedora-docker-base.ks', + 'format': 'docker', + 'repo': '/ostree/$arch/Everything,/ostree/$arch/Server', + 'variant': compose.variants['Server'], + 'target': 'f24', + 'disk_size': 3, + 'name': 'Fedora-Docker-Base', + 'arches': 'x86_64', + 'version': 'Rawhide', + 'ksurl': 'git://git.fedorahosted.org/git/spin-kickstarts.git', + 'distro': 'Fedora-20', + }, + "conf_file": 'Server-Fedora-Docker-Base-docker', + "image_dir": '/image_dir/Server/%(arch)s', + "relative_image_dir": 'image_dir/Server/%(arch)s', + "link_type": 'hardlink-or-copy', + }) + class TestCreateImageBuildThread(unittest.TestCase): @@ -182,7 +299,7 @@ class TestCreateImageBuildThread(unittest.TestCase): 'kickstart': 'fedora-docker-base.ks', 'format': 'docker', 'repo': '/ostree/$arch/Client', - 'variant': compose.variants[1], + 'variant': compose.variants['Client'], 'target': 'f24', 'disk_size': 3, 'name': 'Fedora-Docker-Base', @@ -276,6 +393,102 @@ class TestCreateImageBuildThread(unittest.TestCase): self.assertEqual(data['format'], image.format) self.assertEqual(data['type'], image.type) + @mock.patch('pungi.phases.image_build.KojiWrapper') + @mock.patch('pungi.phases.image_build.Linker') + def test_process_handle_fail(self, Linker, KojiWrapper): + compose = _DummyCompose({ + 'koji_profile': 'koji', + 'failable_deliverables': [ + ('^.*$', { + '*': ['image-build'] + }) + ] + }) + pool = mock.Mock() + cmd = { + "format": [('docker', 'tar.xz'), ('qcow2', 'qcow2')], + "image_conf": { + 'install_tree': '/ostree/$arch/Client', + 'kickstart': 'fedora-docker-base.ks', + 'format': 'docker', + 'repo': '/ostree/$arch/Client', + 'variant': compose.variants['Client'], + 'target': 'f24', + 'disk_size': 3, + 'name': 'Fedora-Docker-Base', + 'arches': 'amd64,x86_64', + 'version': 'Rawhide', + 'ksurl': 'git://git.fedorahosted.org/git/spin-kickstarts.git', + 'distro': 'Fedora-20', + }, + "conf_file": 'amd64,x86_64-Client-Fedora-Docker-Base-docker', + "image_dir": '/image_dir/Client/%(arch)s', + "relative_image_dir": 'image_dir/Client/%(arch)s', + "link_type": 'hardlink-or-copy', + } + koji_wrapper = KojiWrapper.return_value + koji_wrapper.run_create_image_cmd.return_value = { + "retcode": 1, + "output": None, + "task_id": 1234, + } + + t = CreateImageBuildThread(pool) + with mock.patch('os.stat') as stat: + with mock.patch('os.path.getsize') as getsize: + with mock.patch('time.sleep'): + getsize.return_value = 1024 + stat.return_value.st_mtime = 13579 + t.process((compose, cmd), 1) + + @mock.patch('pungi.phases.image_build.KojiWrapper') + @mock.patch('pungi.phases.image_build.Linker') + def test_process_handle_exception(self, Linker, KojiWrapper): + compose = _DummyCompose({ + 'koji_profile': 'koji', + 'failable_deliverables': [ + ('^.*$', { + '*': ['image-build'] + }) + ] + }) + pool = mock.Mock() + cmd = { + "format": [('docker', 'tar.xz'), ('qcow2', 'qcow2')], + "image_conf": { + 'install_tree': '/ostree/$arch/Client', + 'kickstart': 'fedora-docker-base.ks', + 'format': 'docker', + 'repo': '/ostree/$arch/Client', + 'variant': compose.variants['Client'], + 'target': 'f24', + 'disk_size': 3, + 'name': 'Fedora-Docker-Base', + 'arches': 'amd64,x86_64', + 'version': 'Rawhide', + 'ksurl': 'git://git.fedorahosted.org/git/spin-kickstarts.git', + 'distro': 'Fedora-20', + }, + "conf_file": 'amd64,x86_64-Client-Fedora-Docker-Base-docker', + "image_dir": '/image_dir/Client/%(arch)s', + "relative_image_dir": 'image_dir/Client/%(arch)s', + "link_type": 'hardlink-or-copy', + } + + def boom(*args, **kwargs): + raise RuntimeError('BOOM') + + koji_wrapper = KojiWrapper.return_value + koji_wrapper.run_create_image_cmd.side_effect = boom + + t = CreateImageBuildThread(pool) + with mock.patch('os.stat') as stat: + with mock.patch('os.path.getsize') as getsize: + with mock.patch('time.sleep'): + getsize.return_value = 1024 + stat.return_value.st_mtime = 13579 + t.process((compose, cmd), 1) + if __name__ == "__main__": unittest.main()