Add option to specify non-failing stuff

There is a new configuration option that allows listing what can fail
without aborting the whole compose. So far, only buildinstall, createiso
and liveimages phases react to this option.

Signed-off-by: Lubomír Sedlář <lsedlar@redhat.com>

doc/configuration.rst

@@ -127,6 +127,18 @@ Options
 **variants_file** [mandatory]
     (*scm_dict* or *str*) -- reference to variants XML file that defines release variants and architectures
 
+**failable_deliverables** [optional]
+    (*list*) -- list which deliverables on which variant and architecture can
+    fail and not abort the whole compose
+
+    Currently handled deliverables are:
+     * buildinstall
+     * iso
+     * live
+
+    Please note that ``*`` as a wildcard matches all architectures but ``src``.
+
+
 Example
 -------
 ::
@@ -145,6 +157,16 @@ Example
         "file": "variants-fedora.xml",
     }
 
+    failable_deliverables = [
+        ('^.*$', {
+            # Buildinstall can fail on any variant and any arch
+            '*': ['buildinstall'],
+            'src': ['buildinstall'],
+            # Nothing on i386 blocks the compose
+            'i386': ['buildinstall', 'iso', 'live'],
+        })
+    ]
+
 
 Image Naming
 ============

pungi/compose.py

@@ -33,7 +33,7 @@ from productmd.images import Images
 from pungi.wrappers.variants import VariantsXmlParser
 from pungi.paths import Paths
 from pungi.wrappers.scm import get_file_from_scm
-from pungi.util import makedirs
+from pungi.util import makedirs, get_arch_variant_data
 from pungi.metadata import compose_to_composeinfo
 
 
@@ -284,3 +284,11 @@ class Compose(kobo.log.LoggingBase):
             'version': self.ci_base.release.version,
         }
         return format % args
+
+    def can_fail(self, variant, arch, deliverable):
+        """Figure out if deliverable can fail on variant.arch.
+
+        Variant can be None.
+        """
+        failable = get_arch_variant_data(self.conf, 'failable_deliverables', arch, variant)
+        return deliverable in failable

pungi/phases/buildinstall.py

@@ -357,6 +357,16 @@ def symlink_boot_iso(compose, arch, variant):
 class BuildinstallThread(WorkerThread):
     def process(self, item, num):
         compose, arch, cmd = item
+        try:
+            self.worker(compose, arch, cmd, num)
+        except Exception:
+            if not compose.can_fail(None, arch, 'buildinstall'):
+                raise
+            else:
+                self.pool.log_info(
+                    '[FAIL] Buildinstall for arch %s failed, but going on anyway.' % arch)
+
+    def worker(self, compose, arch, cmd, num):
         runroot = compose.conf.get("runroot", False)
         buildinstall_method = compose.conf["buildinstall_method"]
         log_file = compose.paths.log.log_file(arch, "buildinstall")

pungi/phases/createiso.py

@@ -184,15 +184,15 @@ class CreateisoPhase(PhaseBase):
                         jigdo_cmd = jigdo.get_jigdo_cmd(iso_path, files, output_dir=jigdo_dir, no_servers=True, report="noprogress")
                         jigdo_cmd = " ".join([pipes.quote(i) for i in jigdo_cmd])
                         cmd["cmd"].append(jigdo_cmd)
-    
+
                     cmd["cmd"] = " && ".join(cmd["cmd"])
-                    commands.append(cmd)
+                    commands.append((cmd, variant, arch))
 
         self.compose.notifier.send('createiso-targets', deliverables=deliverables)
 
-        for cmd in commands:
+        for (cmd, variant, arch) in commands:
             self.pool.add(CreateIsoThread(self.pool))
-            self.pool.queue_put((self.compose, cmd))
+            self.pool.queue_put((self.compose, cmd, variant, arch))
 
         self.pool.start()
 
@@ -217,8 +217,18 @@ class CreateIsoThread(WorkerThread):
                               variant=str(cmd['variant']))
 
     def process(self, item, num):
-        compose, cmd = item
+        compose, cmd, variant, arch = item
+        try:
+            self.worker(compose, cmd, num)
+        except Exception:
+            if not compose.can_fail(variant, arch, 'iso'):
+                raise
+            else:
+                msg = ('[FAIL] Creating iso for variant %s, arch %s failed, but going on anyway.'
+                       % (variant.uid, arch))
+                self.pool.log_info(msg)
 
+    def worker(self, compose, cmd, num):
         mounts = [compose.topdir]
         if "mount" in cmd:
             mounts.append(cmd["mount"])

pungi/phases/live_images.py

@@ -144,11 +144,11 @@ class LiveImagesPhase(PhaseBase):
                 cmd["cmd"].append(iso.get_manifest_cmd(iso_name))
 
                 cmd["cmd"] = " && ".join(cmd["cmd"])
-                commands.append(cmd)
+                commands.append((cmd, variant, arch))
 
-        for cmd in commands:
+        for (cmd, variant, arch) in commands:
             self.pool.add(CreateLiveImageThread(self.pool))
-            self.pool.queue_put((self.compose, cmd))
+            self.pool.queue_put((self.compose, cmd, variant, arch))
 
         self.pool.start()
 
@@ -168,8 +168,18 @@ class CreateLiveImageThread(WorkerThread):
             pass
 
     def process(self, item, num):
-        compose, cmd = item
+        compose, cmd, variant, arch = item
+        try:
+            self.worker(compose, cmd, num)
+        except:
+            if not compose.can_fail(variant, arch, 'live'):
+                raise
+            else:
+                msg = ('[FAIL] Creating live image for variant %s, arch %s failed, but going on anyway.'
+                       % (variant.uid, arch))
+                self.pool.log_info(msg)
 
+    def worker(self, compose, cmd, num):
         log_file = compose.paths.log.log_file(cmd["arch"], "createiso-%s" % os.path.basename(cmd["iso_path"]))
 
         msg = "Creating ISO (arch: %s, variant: %s): %s" % (cmd["arch"], cmd["variant"], os.path.basename(cmd["iso_path"]))

tests/test_compose.py

@@ -0,0 +1,45 @@
+#!/usr/bin/env python2
+# -*- coding: utf-8 -*-
+
+import mock
+import unittest
+import os
+import sys
+import tempfile
+import shutil
+
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), ".."))
+
+from pungi.compose import Compose
+
+
+class ComposeTestCase(unittest.TestCase):
+    def setUp(self):
+        self.tmp_dir = tempfile.mkdtemp()
+
+    def tearDown(self):
+        shutil.rmtree(self.tmp_dir)
+
+    @mock.patch('pungi.compose.ComposeInfo')
+    def test_can_fail(self, ci):
+        conf = {
+            'failable_deliverables': [
+                ('^.*$', {
+                    '*': ['buildinstall'],
+                    'i386': ['buildinstall', 'live', 'iso'],
+                }),
+            ]
+        }
+        compose = Compose(conf, self.tmp_dir)
+        variant = mock.Mock(uid='Server')
+
+        self.assertTrue(compose.can_fail(variant, 'x86_64', 'buildinstall'))
+        self.assertFalse(compose.can_fail(variant, 'x86_64', 'live'))
+        self.assertTrue(compose.can_fail(variant, 'i386', 'live'))
+
+        self.assertFalse(compose.can_fail(None, 'x86_64', 'live'))
+        self.assertTrue(compose.can_fail(None, 'i386', 'live'))
+
+
+if __name__ == "__main__":
+    unittest.main()