From a6b673dbddd2eb6b014d8326197187710e8d55fd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lubom=C3=ADr=20Sedl=C3=A1=C5=99?= Date: Thu, 10 Dec 2015 15:49:00 +0100 Subject: [PATCH] Add option to specify non-failing stuff MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit There is a new configuration option that allows listing what can fail without aborting the whole compose. So far, only buildinstall, createiso and liveimages phases react to this option. Signed-off-by: Lubomír Sedlář --- doc/configuration.rst | 22 ++++++++++++++++++ pungi/compose.py | 10 +++++++- pungi/phases/buildinstall.py | 10 ++++++++ pungi/phases/createiso.py | 20 ++++++++++++---- pungi/phases/live_images.py | 18 +++++++++++---- tests/test_compose.py | 45 ++++++++++++++++++++++++++++++++++++ 6 files changed, 115 insertions(+), 10 deletions(-) create mode 100755 tests/test_compose.py diff --git a/doc/configuration.rst b/doc/configuration.rst index 8f6cf7f4..a3cdb681 100644 --- a/doc/configuration.rst +++ b/doc/configuration.rst @@ -127,6 +127,18 @@ Options **variants_file** [mandatory] (*scm_dict* or *str*) -- reference to variants XML file that defines release variants and architectures +**failable_deliverables** [optional] + (*list*) -- list which deliverables on which variant and architecture can + fail and not abort the whole compose + + Currently handled deliverables are: + * buildinstall + * iso + * live + + Please note that ``*`` as a wildcard matches all architectures but ``src``. + + Example ------- :: @@ -145,6 +157,16 @@ Example "file": "variants-fedora.xml", } + failable_deliverables = [ + ('^.*$', { + # Buildinstall can fail on any variant and any arch + '*': ['buildinstall'], + 'src': ['buildinstall'], + # Nothing on i386 blocks the compose + 'i386': ['buildinstall', 'iso', 'live'], + }) + ] + Image Naming ============ diff --git a/pungi/compose.py b/pungi/compose.py index 0d917f43..53375070 100644 --- a/pungi/compose.py +++ b/pungi/compose.py @@ -33,7 +33,7 @@ from productmd.images import Images from pungi.wrappers.variants import VariantsXmlParser from pungi.paths import Paths from pungi.wrappers.scm import get_file_from_scm -from pungi.util import makedirs +from pungi.util import makedirs, get_arch_variant_data from pungi.metadata import compose_to_composeinfo @@ -284,3 +284,11 @@ class Compose(kobo.log.LoggingBase): 'version': self.ci_base.release.version, } return format % args + + def can_fail(self, variant, arch, deliverable): + """Figure out if deliverable can fail on variant.arch. + + Variant can be None. + """ + failable = get_arch_variant_data(self.conf, 'failable_deliverables', arch, variant) + return deliverable in failable diff --git a/pungi/phases/buildinstall.py b/pungi/phases/buildinstall.py index 7f3d6f99..fb0faf60 100644 --- a/pungi/phases/buildinstall.py +++ b/pungi/phases/buildinstall.py @@ -357,6 +357,16 @@ def symlink_boot_iso(compose, arch, variant): class BuildinstallThread(WorkerThread): def process(self, item, num): compose, arch, cmd = item + try: + self.worker(compose, arch, cmd, num) + except Exception: + if not compose.can_fail(None, arch, 'buildinstall'): + raise + else: + self.pool.log_info( + '[FAIL] Buildinstall for arch %s failed, but going on anyway.' % arch) + + def worker(self, compose, arch, cmd, num): runroot = compose.conf.get("runroot", False) buildinstall_method = compose.conf["buildinstall_method"] log_file = compose.paths.log.log_file(arch, "buildinstall") diff --git a/pungi/phases/createiso.py b/pungi/phases/createiso.py index 7f9efa3b..81614d4c 100644 --- a/pungi/phases/createiso.py +++ b/pungi/phases/createiso.py @@ -184,15 +184,15 @@ class CreateisoPhase(PhaseBase): jigdo_cmd = jigdo.get_jigdo_cmd(iso_path, files, output_dir=jigdo_dir, no_servers=True, report="noprogress") jigdo_cmd = " ".join([pipes.quote(i) for i in jigdo_cmd]) cmd["cmd"].append(jigdo_cmd) - + cmd["cmd"] = " && ".join(cmd["cmd"]) - commands.append(cmd) + commands.append((cmd, variant, arch)) self.compose.notifier.send('createiso-targets', deliverables=deliverables) - for cmd in commands: + for (cmd, variant, arch) in commands: self.pool.add(CreateIsoThread(self.pool)) - self.pool.queue_put((self.compose, cmd)) + self.pool.queue_put((self.compose, cmd, variant, arch)) self.pool.start() @@ -217,8 +217,18 @@ class CreateIsoThread(WorkerThread): variant=str(cmd['variant'])) def process(self, item, num): - compose, cmd = item + compose, cmd, variant, arch = item + try: + self.worker(compose, cmd, num) + except Exception: + if not compose.can_fail(variant, arch, 'iso'): + raise + else: + msg = ('[FAIL] Creating iso for variant %s, arch %s failed, but going on anyway.' + % (variant.uid, arch)) + self.pool.log_info(msg) + def worker(self, compose, cmd, num): mounts = [compose.topdir] if "mount" in cmd: mounts.append(cmd["mount"]) diff --git a/pungi/phases/live_images.py b/pungi/phases/live_images.py index 93b082ee..e8ccf437 100644 --- a/pungi/phases/live_images.py +++ b/pungi/phases/live_images.py @@ -144,11 +144,11 @@ class LiveImagesPhase(PhaseBase): cmd["cmd"].append(iso.get_manifest_cmd(iso_name)) cmd["cmd"] = " && ".join(cmd["cmd"]) - commands.append(cmd) + commands.append((cmd, variant, arch)) - for cmd in commands: + for (cmd, variant, arch) in commands: self.pool.add(CreateLiveImageThread(self.pool)) - self.pool.queue_put((self.compose, cmd)) + self.pool.queue_put((self.compose, cmd, variant, arch)) self.pool.start() @@ -168,8 +168,18 @@ class CreateLiveImageThread(WorkerThread): pass def process(self, item, num): - compose, cmd = item + compose, cmd, variant, arch = item + try: + self.worker(compose, cmd, num) + except: + if not compose.can_fail(variant, arch, 'live'): + raise + else: + msg = ('[FAIL] Creating live image for variant %s, arch %s failed, but going on anyway.' + % (variant.uid, arch)) + self.pool.log_info(msg) + def worker(self, compose, cmd, num): log_file = compose.paths.log.log_file(cmd["arch"], "createiso-%s" % os.path.basename(cmd["iso_path"])) msg = "Creating ISO (arch: %s, variant: %s): %s" % (cmd["arch"], cmd["variant"], os.path.basename(cmd["iso_path"])) diff --git a/tests/test_compose.py b/tests/test_compose.py new file mode 100755 index 00000000..b3149c28 --- /dev/null +++ b/tests/test_compose.py @@ -0,0 +1,45 @@ +#!/usr/bin/env python2 +# -*- coding: utf-8 -*- + +import mock +import unittest +import os +import sys +import tempfile +import shutil + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..")) + +from pungi.compose import Compose + + +class ComposeTestCase(unittest.TestCase): + def setUp(self): + self.tmp_dir = tempfile.mkdtemp() + + def tearDown(self): + shutil.rmtree(self.tmp_dir) + + @mock.patch('pungi.compose.ComposeInfo') + def test_can_fail(self, ci): + conf = { + 'failable_deliverables': [ + ('^.*$', { + '*': ['buildinstall'], + 'i386': ['buildinstall', 'live', 'iso'], + }), + ] + } + compose = Compose(conf, self.tmp_dir) + variant = mock.Mock(uid='Server') + + self.assertTrue(compose.can_fail(variant, 'x86_64', 'buildinstall')) + self.assertFalse(compose.can_fail(variant, 'x86_64', 'live')) + self.assertTrue(compose.can_fail(variant, 'i386', 'live')) + + self.assertFalse(compose.can_fail(None, 'x86_64', 'live')) + self.assertTrue(compose.can_fail(None, 'i386', 'live')) + + +if __name__ == "__main__": + unittest.main()