From 463088d580f0cbff6e3d09f73a4ff990fe3e3c94 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lubom=C3=ADr=20Sedl=C3=A1=C5=99?= Date: Fri, 24 Jun 2016 09:44:40 +0200 Subject: [PATCH] Refactor failables MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This is a breaking change as big part of current failable_deliverables options will be ignored. There is no change for buildinstall and creatiso phase. Failability for artifacts in other phases is now configured per artifact. It already works correctly for ostree and ostree_installer phases (even per-arch). For OSBS phase there is currently only a binary switch as it does not handle multiple arches yet. When it gains that support, the option should contain list of non-blocking architectures. For live images, live media and image build phases each config block can configure list of failable arches. If the list is not empty, it can fail. Once we have a way to fail only some arches, the config will not need to change. Signed-off-by: Lubomír Sedlář --- doc/configuration.rst | 61 ++++++++++++++++------------ pungi/phases/buildinstall.py | 11 +++-- pungi/phases/createiso.py | 3 +- pungi/phases/image_build.py | 7 +++- pungi/phases/live_images.py | 7 +++- pungi/phases/livemedia_phase.py | 6 ++- pungi/phases/osbs.py | 2 +- pungi/phases/ostree.py | 4 +- pungi/phases/ostree_installer.py | 5 ++- pungi/phases/test.py | 4 +- pungi/util.py | 8 +++- tests/helpers.py | 2 +- tests/test_buildinstall.py | 18 ++++---- tests/test_imagebuildphase.py | 26 +++++------- tests/test_liveimagesphase.py | 21 ++++++---- tests/test_livemediaphase.py | 15 +++++-- tests/test_osbs_phase.py | 17 +++++++- tests/test_ostree_installer_phase.py | 8 +--- tests/test_ostree_phase.py | 8 +--- tests/test_test_phase.py | 12 ++---- 20 files changed, 148 insertions(+), 97 deletions(-) diff --git a/doc/configuration.rst b/doc/configuration.rst index 8f871e77..8d69254b 100644 --- a/doc/configuration.rst +++ b/doc/configuration.rst @@ -117,13 +117,6 @@ Example base_product_short = "Fedora" base_product_version = "23" -**tree_arches** - ([*str*]) -- list of architectures which should be included; if undefined, all architectures from variants.xml will be included - -**tree_variants** - ([*str*]) -- list of variants which should be included; if undefined, all variants from variants.xml will be included - - General Settings ================ @@ -138,28 +131,12 @@ Options **failable_deliverables** [optional] (*list*) -- list which deliverables on which variant and architecture can - fail and not abort the whole compose - - Currently handled deliverables are: - * buildinstall - * iso - * live - * image-build - * live-media - * ostree - * ostree-installer - * osbs - - .. note:: - - Image building is not run per-architecture. If you want to mark it - as failable, specify it in a block with arch set as ``*``. + fail and not abort the whole compose. This only applies to ``buildinstall`` + and ``iso`` parts. All other artifacts can be configured in their + respective part of configuration. Please note that ``*`` as a wildcard matches all architectures but ``src``. - tree_arches = ["x86_64"] - tree_variants = ["Server"] - **comps_filter_environments** [optional] (*bool*) -- When set to ``False``, the comps files for variants will not have their environments filtered to match the variant. @@ -168,6 +145,15 @@ Options (*list*) -- List of variants for which the original comps file will be copied without any modifications. Overwrites `comps_filter_environments`. +**tree_arches** + ([*str*]) -- list of architectures which should be included; if undefined, + all architectures from variants.xml will be included + +**tree_variants** + ([*str*]) -- list of variants which should be included; if undefined, all + variants from variants.xml will be included + + Example ------- :: @@ -196,6 +182,9 @@ Example }) ] + tree_arches = ["x86_64"] + tree_variants = ["Server"] + Image Naming ============ @@ -836,6 +825,14 @@ will be generated based on date, compose type and respin. * ``image_build_release`` * ``live_images_release`` +Each configuration block can also optionally specify a list of architectures +that are not release blocking with ``failable`` key. If any deliverable fails, +it will not abort the whole compose. Due to limitations in how the tasks are +done in Koji, if any architecture fails, all of them fail. Until this is +resolved, it is not possible to configure failability per architecture. An +empty list means required deliverable, non-empty list means non-blocking +deliverable. + Live Images Settings ==================== @@ -1059,6 +1056,8 @@ runroot environment. * ``config_branch`` -- (*str*) Git branch of the repo to use. Defaults to ``master``. + * ``failable`` -- (*[str]*) List of architectures for which this + deliverable is not release blocking. Example config @@ -1096,6 +1095,8 @@ an OSTree repository. This always runs in Koji as a ``runroot`` task. * ``release`` -- (*str*) Release value to set for the installer image. Set to ``None`` to use the date.respin format. + * ``failable`` -- (*[str]*) List of architectures for which this + deliverable is not release blocking. These optional keys are passed to ``lorax`` to customize the build. @@ -1163,6 +1164,14 @@ they are not scratch builds). Please see :ref:`git-urls` section for more details. * ``target`` -- (*str*) A Koji target to build the image for. + Optionally you can specify ``failable``. If it has a truthy value, failure + to create the image will not abort the whole compose. + + .. note:: + Once OSBS gains support for multiple architectures, the usage of this + option will most likely change to list architectures that are allowed + to fail. + The configuration will pass other attributes directly to the Koji task. This includes ``name``, ``version``, ``release``, ``scratch`` and ``priority``. diff --git a/pungi/phases/buildinstall.py b/pungi/phases/buildinstall.py index e69e1fc1..ff5a05f9 100644 --- a/pungi/phases/buildinstall.py +++ b/pungi/phases/buildinstall.py @@ -190,9 +190,10 @@ class BuildinstallPhase(PhaseBase): # TODO: label is not used label = "" volid = get_volid(self.compose, arch, variant, escape_spaces=False, disc_type=disc_type) - with failable(self.compose, variant, arch, 'buildinstall'): + can_fail = self.compose.can_fail(variant, arch, 'buildinstall') + with failable(self.compose, can_fail, variant, arch, 'buildinstall'): tweak_buildinstall(buildinstall_dir, os_tree, arch, variant.uid, label, volid, kickstart_file) - link_boot_iso(self.compose, arch, variant) + link_boot_iso(self.compose, arch, variant, can_fail) def get_kickstart_file(compose): @@ -315,7 +316,7 @@ def tweak_buildinstall(src, dst, arch, variant, label, volid, kickstart_file=Non shutil.rmtree(tmp_dir) -def link_boot_iso(compose, arch, variant): +def link_boot_iso(compose, arch, variant, can_fail): if arch == "src": return @@ -369,6 +370,7 @@ def link_boot_iso(compose, arch, variant): img.bootable = True img.subvariant = variant.name img.implant_md5 = implant_md5 + setattr(img, 'can_fail', can_fail) setattr(img, 'deliverable', 'buildinstall') try: img.volume_id = iso.get_volume_id(new_boot_iso_path) @@ -382,7 +384,8 @@ class BuildinstallThread(WorkerThread): def process(self, item, num): # The variant is None unless lorax is used as buildinstall method. compose, arch, variant, cmd = item - with failable(compose, variant, arch, 'buildinstall'): + can_fail = compose.can_fail(variant, arch, 'buildinstall') + with failable(compose, can_fail, variant, arch, 'buildinstall'): self.worker(compose, arch, variant, cmd, num) def worker(self, compose, arch, variant, cmd, num): diff --git a/pungi/phases/createiso.py b/pungi/phases/createiso.py index 682d2d52..762a9378 100644 --- a/pungi/phases/createiso.py +++ b/pungi/phases/createiso.py @@ -183,7 +183,8 @@ class CreateIsoThread(WorkerThread): def process(self, item, num): compose, cmd, variant, arch = item - with failable(compose, variant, arch, 'iso'): + can_fail = compose.can_fail(variant, arch, 'iso') + with failable(compose, can_fail, variant, arch, 'iso'): self.worker(compose, cmd, variant, arch, num) def worker(self, compose, cmd, variant, arch, num): diff --git a/pungi/phases/image_build.py b/pungi/phases/image_build.py index dcd74d3f..54c16589 100644 --- a/pungi/phases/image_build.py +++ b/pungi/phases/image_build.py @@ -157,6 +157,7 @@ class ImageBuildPhase(base.ImageConfigMixin, base.ConfigGuardedPhase): ), "link_type": self.compose.conf.get("link_type", "hardlink-or-copy"), "scratch": image_conf['image-build'].pop('scratch', False), + "failable_arches": image_conf.pop('failable', []), } self.pool.add(CreateImageBuildThread(self.pool)) self.pool.queue_put((self.compose, cmd)) @@ -172,7 +173,10 @@ class CreateImageBuildThread(WorkerThread): compose, cmd = item variant = cmd["image_conf"]["image-build"]["variant"] subvariant = cmd["image_conf"]["image-build"].get("subvariant", variant.uid) - with failable(compose, variant, '*', 'image-build', subvariant): + failable_arches = cmd.get('failable_arches', []) + self.can_fail = bool(failable_arches) + # TODO handle failure per architecture; currently not possible in single task + with failable(compose, self.can_fail, variant, '*', 'image-build', subvariant): self.worker(num, compose, variant, subvariant, cmd) def worker(self, num, compose, variant, subvariant, cmd): @@ -253,6 +257,7 @@ class CreateImageBuildThread(WorkerThread): img.disc_count = 1 img.bootable = False img.subvariant = subvariant + setattr(img, 'can_fail', self.can_fail) setattr(img, 'deliverable', 'image-build') compose.im.add(variant=variant.uid, arch=image_info['arch'], image=img) diff --git a/pungi/phases/live_images.py b/pungi/phases/live_images.py index f9b9d13d..ebd46b57 100644 --- a/pungi/phases/live_images.py +++ b/pungi/phases/live_images.py @@ -157,6 +157,7 @@ class LiveImagesPhase(base.ImageConfigMixin, base.ConfigGuardedPhase): "type": type, "label": "", # currently not used "subvariant": subvariant, + "failable_arches": data.get('failable', []), } cmd["repos"] = self._get_repos(arch, variant, data) @@ -199,7 +200,10 @@ class CreateLiveImageThread(WorkerThread): def process(self, item, num): compose, cmd, variant, arch = item - with failable(compose, variant, arch, 'live', cmd.get('subvariant')): + self.failable_arches = cmd.get('failable_arches', []) + # TODO handle failure per architecture; currently not possible in single task + self.can_fail = bool(self.failable_arches) + with failable(compose, self.can_fail, variant, arch, 'live', cmd.get('subvariant')): self.worker(compose, cmd, variant, arch, num) def worker(self, compose, cmd, variant, arch, num): @@ -290,6 +294,7 @@ class CreateLiveImageThread(WorkerThread): img.disc_count = 1 img.bootable = True img.subvariant = subvariant + setattr(img, 'can_fail', self.can_fail) setattr(img, 'deliverable', 'live') compose.im.add(variant=variant.uid, arch=arch, image=img) diff --git a/pungi/phases/livemedia_phase.py b/pungi/phases/livemedia_phase.py index e526831f..48279e03 100644 --- a/pungi/phases/livemedia_phase.py +++ b/pungi/phases/livemedia_phase.py @@ -116,6 +116,7 @@ class LiveMediaPhase(ImageConfigMixin, ConfigGuardedPhase): 'repo': self._get_repos(image_conf, variant), 'install_tree': self._get_install_tree(image_conf, variant), 'version': self.get_config(image_conf, 'version'), + 'failable_arches': image_conf.get('failable', []), } self.pool.add(LiveMediaThread(self.pool)) self.pool.queue_put((self.compose, variant, config)) @@ -127,8 +128,10 @@ class LiveMediaThread(WorkerThread): def process(self, item, num): compose, variant, config = item subvariant = config.pop('subvariant') + self.failable_arches = config.pop('failable_arches') self.num = num - with failable(compose, variant, '*', 'live-media', subvariant): + # TODO handle failure per architecture; currently not possible in single task + with failable(compose, bool(self.failable_arches), variant, '*', 'live-media', subvariant): self.worker(compose, variant, subvariant, config) def _get_log_file(self, compose, variant, subvariant, config): @@ -204,6 +207,7 @@ class LiveMediaThread(WorkerThread): img.disc_count = 1 img.bootable = True img.subvariant = subvariant + setattr(img, 'can_fail', bool(self.failable_arches)) setattr(img, 'deliverable', 'live-media') compose.im.add(variant=variant.uid, arch=image_info['arch'], image=img) diff --git a/pungi/phases/osbs.py b/pungi/phases/osbs.py index 428a6f33..43e6750b 100644 --- a/pungi/phases/osbs.py +++ b/pungi/phases/osbs.py @@ -47,7 +47,7 @@ class OSBSThread(WorkerThread): def process(self, item, num): compose, variant, config = item self.num = num - with util.failable(compose, variant, '*', 'osbs'): + with util.failable(compose, bool(config.pop('failable', None)), variant, '*', 'osbs'): self.worker(compose, variant, config) def worker(self, compose, variant, config): diff --git a/pungi/phases/ostree.py b/pungi/phases/ostree.py index 95813256..a3f7842c 100644 --- a/pungi/phases/ostree.py +++ b/pungi/phases/ostree.py @@ -39,7 +39,9 @@ class OSTreeThread(WorkerThread): def process(self, item, num): compose, variant, arch, config = item self.num = num - with util.failable(compose, variant, arch, 'ostree'): + failable_arches = config.get('failable', []) + with util.failable(compose, util.can_arch_fail(failable_arches, arch), + variant, arch, 'ostree'): self.worker(compose, variant, arch, config) def worker(self, compose, variant, arch, config): diff --git a/pungi/phases/ostree_installer.py b/pungi/phases/ostree_installer.py index 25827396..16b66c51 100644 --- a/pungi/phases/ostree_installer.py +++ b/pungi/phases/ostree_installer.py @@ -42,7 +42,9 @@ class OstreeInstallerThread(WorkerThread): def process(self, item, num): compose, variant, arch, config = item self.num = num - with util.failable(compose, variant, arch, 'ostree-installer'): + failable_arches = config.get('failable', []) + self.can_fail = util.can_arch_fail(failable_arches, arch) + with util.failable(compose, self.can_fail, variant, arch, 'ostree-installer'): self.worker(compose, variant, arch, config) def worker(self, compose, variant, arch, config): @@ -119,6 +121,7 @@ class OstreeInstallerThread(WorkerThread): img.bootable = True img.subvariant = variant.name img.implant_md5 = implant_md5 + setattr(img, 'can_fail', self.can_fail) setattr(img, 'deliverable', 'ostree-installer') try: img.volume_id = iso_wrapper.get_volume_id(full_iso_path) diff --git a/pungi/phases/test.py b/pungi/phases/test.py index 5650f6d9..61cbf199 100644 --- a/pungi/phases/test.py +++ b/pungi/phases/test.py @@ -125,7 +125,9 @@ def check(compose, variant, arch, image): result = True path = os.path.join(compose.paths.compose.topdir(), image.path) deliverable = getattr(image, 'deliverable') - with failable(compose, variant, arch, deliverable, subvariant=image.subvariant): + can_fail = getattr(image, 'can_fail', False) + with failable(compose, can_fail, variant, arch, deliverable, + subvariant=image.subvariant): with open(path) as f: iso = is_iso(f) if image.format == 'iso' and not iso: diff --git a/pungi/util.py b/pungi/util.py index 13a3fdbf..36ad624a 100644 --- a/pungi/util.py +++ b/pungi/util.py @@ -464,10 +464,9 @@ def process_args(fmt, args): @contextlib.contextmanager -def failable(compose, variant, arch, deliverable, subvariant=None): +def failable(compose, can_fail, variant, arch, deliverable, subvariant=None): """If a deliverable can fail, log a message and go on as if it succeeded.""" msg = deliverable.replace('-', ' ').capitalize() - can_fail = compose.can_fail(variant, arch, deliverable) if can_fail: compose.attempt_deliverable(variant, arch, deliverable, subvariant) else: @@ -489,6 +488,11 @@ def failable(compose, variant, arch, deliverable, subvariant=None): compose.log_debug(tb) +def can_arch_fail(failable_arches, arch): + """Check if `arch` is in `failable_arches` or `*` can fail.""" + return '*' in failable_arches or arch in failable_arches + + def get_format_substs(compose, **kwargs): """Return a dict of basic format substitutions. diff --git a/tests/helpers.py b/tests/helpers.py index b0aa5cf4..e52090f8 100644 --- a/tests/helpers.py +++ b/tests/helpers.py @@ -62,7 +62,7 @@ class DummyCompose(object): self.log_debug = mock.Mock() self.log_warning = mock.Mock() self.get_image_name = mock.Mock(return_value='image-name') - self.image = mock.Mock(path='Client/i386/iso/image.iso') + self.image = mock.Mock(path='Client/i386/iso/image.iso', can_fail=False) self.im = mock.Mock(images={'Client': {'amd64': [self.image]}}) self.old_composes = [] self.config_dir = '/home/releng/config' diff --git a/tests/test_buildinstall.py b/tests/test_buildinstall.py index b4c1653d..a92119bd 100755 --- a/tests/test_buildinstall.py +++ b/tests/test_buildinstall.py @@ -355,9 +355,9 @@ class TestCopyFiles(PungiTestCase): 'amd64', 'Client', '', 'Client.amd64', 'kickstart')]) self.assertItemsEqual( link_boot_iso.mock_calls, - [mock.call(compose, 'x86_64', compose.variants['Server']), - mock.call(compose, 'amd64', compose.variants['Client']), - mock.call(compose, 'amd64', compose.variants['Server'])]) + [mock.call(compose, 'x86_64', compose.variants['Server'], False), + mock.call(compose, 'amd64', compose.variants['Client'], False), + mock.call(compose, 'amd64', compose.variants['Server'], False)]) @mock.patch('pungi.phases.buildinstall.link_boot_iso') @mock.patch('pungi.phases.buildinstall.tweak_buildinstall') @@ -397,9 +397,9 @@ class TestCopyFiles(PungiTestCase): 'amd64', 'Client', '', 'Client.amd64', 'kickstart')]) self.assertItemsEqual( link_boot_iso.mock_calls, - [mock.call(compose, 'x86_64', compose.variants['Server']), - mock.call(compose, 'amd64', compose.variants['Client']), - mock.call(compose, 'amd64', compose.variants['Server'])]) + [mock.call(compose, 'x86_64', compose.variants['Server'], False), + mock.call(compose, 'amd64', compose.variants['Client'], False), + mock.call(compose, 'amd64', compose.variants['Server'], False)]) @mock.patch('pungi.phases.buildinstall.link_boot_iso') @mock.patch('pungi.phases.buildinstall.tweak_buildinstall') @@ -622,7 +622,7 @@ class TestSymlinkIso(PungiTestCase): get_file_size.return_value = 1024 get_mtime.return_value = 13579 - link_boot_iso(self.compose, 'x86_64', self.compose.variants['Server']) + link_boot_iso(self.compose, 'x86_64', self.compose.variants['Server'], False) tgt = self.topdir + '/compose/Server/x86_64/iso/image-name' self.assertTrue(os.path.isfile(tgt)) @@ -654,6 +654,7 @@ class TestSymlinkIso(PungiTestCase): self.assertEqual(image.disc_count, 1) self.assertEqual(image.bootable, True) self.assertEqual(image.implant_md5, IsoWrapper.get_implanted_md5.return_value) + self.assertEqual(image.can_fail, False) self.assertEqual(self.compose.im.add.mock_calls, [mock.call('Server', 'x86_64', image)]) @@ -671,7 +672,7 @@ class TestSymlinkIso(PungiTestCase): get_file_size.return_value = 1024 get_mtime.return_value = 13579 - link_boot_iso(self.compose, 'x86_64', self.compose.variants['Server']) + link_boot_iso(self.compose, 'x86_64', self.compose.variants['Server'], True) tgt = self.topdir + '/compose/Server/x86_64/iso/image-name' self.assertTrue(os.path.isfile(tgt)) @@ -703,6 +704,7 @@ class TestSymlinkIso(PungiTestCase): self.assertEqual(image.disc_count, 1) self.assertEqual(image.bootable, True) self.assertEqual(image.implant_md5, IsoWrapper.get_implanted_md5.return_value) + self.assertEqual(image.can_fail, True) self.assertEqual(self.compose.im.add.mock_calls, [mock.call('Server', 'x86_64', image)]) diff --git a/tests/test_imagebuildphase.py b/tests/test_imagebuildphase.py index f87fe6a0..1a5e2b12 100755 --- a/tests/test_imagebuildphase.py +++ b/tests/test_imagebuildphase.py @@ -71,6 +71,7 @@ class TestImageBuildPhase(PungiTestCase): "relative_image_dir": 'Client/%(arch)s/images', "link_type": 'hardlink-or-copy', "scratch": False, + "failable_arches": [], } server_args = { "format": [('docker', 'tar.xz')], @@ -95,6 +96,7 @@ class TestImageBuildPhase(PungiTestCase): "relative_image_dir": 'Server/%(arch)s/images', "link_type": 'hardlink-or-copy', "scratch": False, + "failable_arches": [], } self.maxDiff = None self.assertItemsEqual(phase.pool.queue_put.mock_calls, @@ -155,6 +157,7 @@ class TestImageBuildPhase(PungiTestCase): "relative_image_dir": 'Server/%(arch)s/images', "link_type": 'hardlink-or-copy', "scratch": False, + "failable_arches": [], } self.maxDiff = None self.assertItemsEqual(phase.pool.queue_put.mock_calls, @@ -249,6 +252,7 @@ class TestImageBuildPhase(PungiTestCase): "relative_image_dir": 'Server/%(arch)s/images', "link_type": 'hardlink-or-copy', "scratch": False, + "failable_arches": [], }) @mock.patch('pungi.phases.image_build.ThreadPool') @@ -310,6 +314,7 @@ class TestImageBuildPhase(PungiTestCase): "relative_image_dir": 'Server/%(arch)s/images', "link_type": 'hardlink-or-copy', "scratch": False, + "failable_arches": [], }) @mock.patch('pungi.phases.image_build.ThreadPool') @@ -370,6 +375,7 @@ class TestImageBuildPhase(PungiTestCase): "relative_image_dir": 'Server/%(arch)s/images', "link_type": 'hardlink-or-copy', "scratch": False, + "failable_arches": [], }) @mock.patch('pungi.phases.image_build.ThreadPool') @@ -621,14 +627,7 @@ class TestCreateImageBuildThread(PungiTestCase): @mock.patch('pungi.phases.image_build.KojiWrapper') @mock.patch('pungi.phases.image_build.Linker') def test_process_handle_fail(self, Linker, KojiWrapper): - compose = DummyCompose(self.topdir, { - 'koji_profile': 'koji', - 'failable_deliverables': [ - ('^.*$', { - '*': ['image-build'] - }) - ] - }) + compose = DummyCompose(self.topdir, {'koji_profile': 'koji'}) pool = mock.Mock() cmd = { "format": [('docker', 'tar.xz'), ('qcow2', 'qcow2')], @@ -653,6 +652,7 @@ class TestCreateImageBuildThread(PungiTestCase): "relative_image_dir": 'image_dir/Client/%(arch)s', "link_type": 'hardlink-or-copy', 'scratch': False, + "failable_arches": ['*'], } koji_wrapper = KojiWrapper.return_value koji_wrapper.run_blocking_cmd.return_value = { @@ -675,14 +675,7 @@ class TestCreateImageBuildThread(PungiTestCase): @mock.patch('pungi.phases.image_build.KojiWrapper') @mock.patch('pungi.phases.image_build.Linker') def test_process_handle_exception(self, Linker, KojiWrapper): - compose = DummyCompose(self.topdir, { - 'koji_profile': 'koji', - 'failable_deliverables': [ - ('^.*$', { - '*': ['image-build'] - }) - ] - }) + compose = DummyCompose(self.topdir, {'koji_profile': 'koji'}) pool = mock.Mock() cmd = { "format": [('docker', 'tar.xz'), ('qcow2', 'qcow2')], @@ -707,6 +700,7 @@ class TestCreateImageBuildThread(PungiTestCase): "relative_image_dir": 'image_dir/Client/%(arch)s', "link_type": 'hardlink-or-copy', 'scratch': False, + "failable_arches": ['*'], } koji_wrapper = KojiWrapper.return_value diff --git a/tests/test_liveimagesphase.py b/tests/test_liveimagesphase.py index 6ef44d2e..8a00acec 100755 --- a/tests/test_liveimagesphase.py +++ b/tests/test_liveimagesphase.py @@ -56,6 +56,7 @@ class TestLiveImagesPhase(PungiTestCase): 'type': 'live', 'release': '20151203.t.0', 'subvariant': 'Client', + 'failable_arches': [], 'ksurl': None}, compose.variants['Client'], 'amd64'))]) @@ -104,6 +105,7 @@ class TestLiveImagesPhase(PungiTestCase): 'type': 'live', 'release': '20151203.t.0', 'subvariant': 'Client', + 'failable_arches': [], 'ksurl': None}, compose.variants['Client'], 'amd64'))]) @@ -149,6 +151,7 @@ class TestLiveImagesPhase(PungiTestCase): 'type': 'live', 'release': '20151203.t.0', 'subvariant': 'Client', + 'failable_arches': [], 'ksurl': None}, compose.variants['Client'], 'amd64'))]) @@ -196,6 +199,7 @@ class TestLiveImagesPhase(PungiTestCase): 'type': 'live', 'release': None, 'subvariant': 'Client', + 'failable_arches': [], 'ksurl': None}, compose.variants['Client'], 'amd64')), @@ -216,6 +220,7 @@ class TestLiveImagesPhase(PungiTestCase): 'type': 'live', 'release': None, 'subvariant': 'Client', + 'failable_arches': [], 'ksurl': None}, compose.variants['Client'], 'amd64'))]) @@ -264,6 +269,7 @@ class TestLiveImagesPhase(PungiTestCase): 'type': 'appliance', 'release': None, 'subvariant': 'Client', + 'failable_arches': [], 'ksurl': 'https://git.example.com/kickstarts.git?#CAFEBABE'}, compose.variants['Client'], 'amd64'))]) @@ -316,6 +322,7 @@ class TestLiveImagesPhase(PungiTestCase): 'type': 'appliance', 'release': '20151203.t.0', 'subvariant': 'Client', + 'failable_arches': [], 'ksurl': 'https://git.example.com/kickstarts.git?#CAFEBABE'}, compose.variants['Client'], 'amd64'))]) @@ -368,6 +375,7 @@ class TestLiveImagesPhase(PungiTestCase): 'type': 'appliance', 'release': '20151203.t.0', 'subvariant': 'Client', + 'failable_arches': [], 'ksurl': 'https://git.example.com/kickstarts.git?#CAFEBABE'}, compose.variants['Client'], 'amd64'))]) @@ -415,6 +423,7 @@ class TestLiveImagesPhase(PungiTestCase): 'type': 'live', 'release': '20151203.t.0', 'subvariant': 'Client', + 'failable_arches': [], 'ksurl': None}, compose.variants['Client'], 'amd64'))]) @@ -665,10 +674,7 @@ class TestCreateLiveImageThread(PungiTestCase): @mock.patch('pungi.phases.live_images.run') @mock.patch('pungi.phases.live_images.KojiWrapper') def test_process_handles_fail(self, KojiWrapper, run, copy2): - compose = DummyCompose(self.topdir, { - 'koji_profile': 'koji', - 'failable_deliverables': [('^.+$', {'*': ['live']})], - }) + compose = DummyCompose(self.topdir, {'koji_profile': 'koji'}) pool = mock.Mock() cmd = { 'ks_file': '/path/to/ks_file', @@ -687,6 +693,7 @@ class TestCreateLiveImageThread(PungiTestCase): 'subvariant': 'Client', 'release': 'xyz', 'type': 'live', + 'failable_arches': ['*'], } koji_wrapper = KojiWrapper.return_value @@ -711,10 +718,7 @@ class TestCreateLiveImageThread(PungiTestCase): @mock.patch('pungi.phases.live_images.run') @mock.patch('pungi.phases.live_images.KojiWrapper') def test_process_handles_exception(self, KojiWrapper, run, copy2): - compose = DummyCompose(self.topdir, { - 'koji_profile': 'koji', - 'failable_deliverables': [('^.+$', {'*': ['live']})], - }) + compose = DummyCompose(self.topdir, {'koji_profile': 'koji'}) pool = mock.Mock() cmd = { 'ks_file': '/path/to/ks_file', @@ -733,6 +737,7 @@ class TestCreateLiveImageThread(PungiTestCase): 'subvariant': 'Client', 'release': 'xyz', 'type': 'live', + 'failable_arches': ['*'], } koji_wrapper = KojiWrapper.return_value diff --git a/tests/test_livemediaphase.py b/tests/test_livemediaphase.py index 365bc8d4..dee7afc4 100755 --- a/tests/test_livemediaphase.py +++ b/tests/test_livemediaphase.py @@ -73,6 +73,7 @@ class TestLiveMediaPhase(PungiTestCase): 'install_tree': self.topdir + '/compose/Server/$basearch/os', 'version': 'Rawhide', 'subvariant': 'Server', + 'failable_arches': [], }))]) @mock.patch('pungi.util.resolve_git_url') @@ -133,6 +134,7 @@ class TestLiveMediaPhase(PungiTestCase): 'install_tree': self.topdir + '/compose/Server/$basearch/os', 'version': 'Rawhide', 'subvariant': 'Server', + 'failable_arches': [], })), mock.call((compose, compose.variants['Server'], @@ -151,6 +153,7 @@ class TestLiveMediaPhase(PungiTestCase): 'install_tree': self.topdir + '/compose/Server/$basearch/os', 'version': 'Rawhide', 'subvariant': 'Server', + 'failable_arches': [], })), mock.call((compose, compose.variants['Server'], @@ -169,6 +172,7 @@ class TestLiveMediaPhase(PungiTestCase): 'install_tree': self.topdir + '/compose/Server/$basearch/os', 'version': '25', 'subvariant': 'Server', + 'failable_arches': [], }))]) @mock.patch('pungi.util.resolve_git_url') @@ -229,6 +233,7 @@ class TestLiveMediaPhase(PungiTestCase): 'install_tree': self.topdir + '/compose/Server/$basearch/os', 'version': 'Rawhide', 'subvariant': 'Server', + 'failable_arches': [], })), mock.call((compose, compose.variants['Server'], @@ -247,6 +252,7 @@ class TestLiveMediaPhase(PungiTestCase): 'install_tree': self.topdir + '/compose/Server/$basearch/os', 'version': 'Rawhide', 'subvariant': 'Server', + 'failable_arches': [], })), mock.call((compose, compose.variants['Server'], @@ -265,6 +271,7 @@ class TestLiveMediaPhase(PungiTestCase): 'install_tree': self.topdir + '/compose/Server/$basearch/os', 'version': '25', 'subvariant': 'Server', + 'failable_arches': [], }))]) @mock.patch('pungi.phases.livemedia_phase.ThreadPool') @@ -335,6 +342,7 @@ class TestLiveMediaPhase(PungiTestCase): 'version': 'Rawhide', 'install_tree_from': 'Everything', 'subvariant': 'Something', + 'failable': ['*'], } ] } @@ -366,6 +374,7 @@ class TestLiveMediaPhase(PungiTestCase): 'install_tree': self.topdir + '/compose/Everything/$basearch/os', 'version': 'Rawhide', 'subvariant': 'Something', + 'failable_arches': ['*'], }))]) @@ -393,6 +402,7 @@ class TestLiveMediaThread(PungiTestCase): 'title': None, 'version': 'Rawhide', 'subvariant': 'KDE', + 'failable_arches': [], } pool = mock.Mock() @@ -479,9 +489,6 @@ class TestLiveMediaThread(PungiTestCase): def test_handle_koji_fail(self, KojiWrapper, get_file_size, get_mtime): compose = DummyCompose(self.topdir, { 'koji_profile': 'koji', - 'failable_deliverables': [ - ('^.+$', {'*': ['live-media']}) - ] }) config = { 'arches': ['amd64', 'x86_64'], @@ -497,6 +504,7 @@ class TestLiveMediaThread(PungiTestCase): 'title': None, 'version': 'Rawhide', 'subvariant': 'KDE', + 'failable_arches': ['*'], } pool = mock.Mock() @@ -543,6 +551,7 @@ class TestLiveMediaThread(PungiTestCase): 'title': None, 'version': 'Rawhide', 'subvariant': 'KDE', + 'failable_arches': ['*'], } pool = mock.Mock() diff --git a/tests/test_osbs_phase.py b/tests/test_osbs_phase.py index a39a65d5..fc3aedd5 100755 --- a/tests/test_osbs_phase.py +++ b/tests/test_osbs_phase.py @@ -231,6 +231,21 @@ class OSBSThreadTest(helpers.PungiTestCase): self._assertCorrectCalls({}) self._assertCorrectMetadata() + @mock.patch('pungi.util.resolve_git_url') + @mock.patch('pungi.phases.osbs.kojiwrapper.KojiWrapper') + def test_run_failable(self, KojiWrapper, resolve_git_url): + cfg = { + 'url': 'git://example.com/repo?#HEAD', + 'target': 'f24-docker-candidate', + 'failable': ['*'] + } + self._setupMock(KojiWrapper, resolve_git_url) + + self.t.process((self.compose, self.compose.variants['Server'], cfg), 1) + + self._assertCorrectCalls({}) + self._assertCorrectMetadata() + @mock.patch('pungi.util.resolve_git_url') @mock.patch('pungi.phases.osbs.kojiwrapper.KojiWrapper') def test_run_with_more_args(self, KojiWrapper, resolve_git_url): @@ -353,10 +368,10 @@ class OSBSThreadTest(helpers.PungiTestCase): cfg = { 'url': 'git://example.com/repo?#HEAD', 'target': 'fedora-24-docker-candidate', + 'failable': ['*'] } self._setupMock(KojiWrapper, resolve_git_url) self.wrapper.watch_task.return_value = 1 - self.compose.conf['failable_deliverables'] = [('.*', {'*': ['osbs']})] self.t.process((self.compose, self.compose.variants['Server'], cfg), 1) diff --git a/tests/test_ostree_installer_phase.py b/tests/test_ostree_installer_phase.py index 512da4a6..b4f73c2f 100644 --- a/tests/test_ostree_installer_phase.py +++ b/tests/test_ostree_installer_phase.py @@ -432,14 +432,12 @@ class OstreeThreadTest(helpers.PungiTestCase): 'release_version': 'Rawhide', 'koji_profile': 'koji', 'runroot_tag': 'rrt', - 'failable_deliverables': [ - ('^.+$', {'*': ['ostree-installer']}) - ], }) pool = mock.Mock() cfg = { 'source_repo_from': 'Everything', 'release': None, + 'failable': ['x86_64'] } koji = KojiWrapper.return_value koji.run_runroot_cmd.side_effect = helpers.boom @@ -466,14 +464,12 @@ class OstreeThreadTest(helpers.PungiTestCase): 'release_version': 'Rawhide', 'koji_profile': 'koji', 'runroot_tag': 'rrt', - 'failable_deliverables': [ - ('^.+$', {'*': ['ostree-installer']}) - ], }) pool = mock.Mock() cfg = { 'source_repo_from': 'Everything', 'release': None, + 'failable': ['*'], } koji = KojiWrapper.return_value koji.run_runroot_cmd.return_value = { diff --git a/tests/test_ostree_phase.py b/tests/test_ostree_phase.py index a962d618..eec7e672 100755 --- a/tests/test_ostree_phase.py +++ b/tests/test_ostree_phase.py @@ -155,9 +155,6 @@ class OSTreeThreadTest(helpers.PungiTestCase): compose = helpers.DummyCompose(self.topdir, { 'koji_profile': 'koji', 'runroot_tag': 'rrt', - 'failable_deliverables': [ - ('^.*$', {'*': ['ostree']}) - ] }) pool = mock.Mock() cfg = { @@ -166,6 +163,7 @@ class OSTreeThreadTest(helpers.PungiTestCase): 'config_branch': 'f24', 'treefile': 'fedora-atomic-docker-host.json', 'ostree_repo': self.repo, + 'failable': ['*'] } koji = KojiWrapper.return_value koji.run_runroot_cmd.return_value = { @@ -192,9 +190,6 @@ class OSTreeThreadTest(helpers.PungiTestCase): compose = helpers.DummyCompose(self.topdir, { 'koji_profile': 'koji', 'runroot_tag': 'rrt', - 'failable_deliverables': [ - ('^.*$', {'*': ['ostree']}) - ] }) pool = mock.Mock() cfg = { @@ -203,6 +198,7 @@ class OSTreeThreadTest(helpers.PungiTestCase): 'config_branch': 'f24', 'treefile': 'fedora-atomic-docker-host.json', 'ostree_repo': self.repo, + 'failable': ['*'] } koji = KojiWrapper.return_value koji.run_runroot_cmd.side_effect = helpers.boom diff --git a/tests/test_test_phase.py b/tests/test_test_phase.py index 44e0275d..7780068e 100755 --- a/tests/test_test_phase.py +++ b/tests/test_test_phase.py @@ -17,12 +17,6 @@ import pungi.phases.test as test_phase from tests.helpers import DummyCompose, PungiTestCase, touch -FAILABLE_CONFIG = { - 'failable_deliverables': [ - ('^.+$', {'*': ['iso']}), - ] -} - PAD = '\0' * 100 UNBOOTABLE_ISO = ('\0' * 0x8001) + 'CD001' + PAD ISO_WITH_MBR = ('\0' * 0x1fe) + '\x55\xAA' + ('\0' * 0x7e01) + 'CD001' + PAD @@ -40,8 +34,9 @@ class TestCheckImageSanity(PungiTestCase): test_phase.check_image_sanity(compose) def test_missing_file_doesnt_report_if_failable(self): - compose = DummyCompose(self.topdir, FAILABLE_CONFIG) + compose = DummyCompose(self.topdir, {}) compose.image.deliverable = 'iso' + compose.image.can_fail = True try: test_phase.check_image_sanity(compose) @@ -83,10 +78,11 @@ class TestCheckImageSanity(PungiTestCase): str(ctx.exception)) def test_failable_bootable_iso_without_mbr_gpt_doesnt_raise(self): - compose = DummyCompose(self.topdir, FAILABLE_CONFIG) + compose = DummyCompose(self.topdir, {}) compose.image.format = 'iso' compose.image.bootable = True compose.image.deliverable = 'iso' + compose.image.can_fail = True touch(os.path.join(self.topdir, 'compose', compose.image.path), UNBOOTABLE_ISO) try: