Merge #123 Live images: add repo from another variant

doc/configuration.rst

@@ -624,6 +624,29 @@ Example
         }),
     ]
 
+
+Live Images Settings
+====================
+
+**live_target**
+    (*str*) -- Koji build target for which to build the images. This gets
+    passed to ``koji spin-livecd``.
+
+**live_images**
+    (*list*) -- Configuration for the particular image. The elements of the
+    list should be tuples ``(variant_uid_regex, {arch|*: config})``. The config
+    should be a dict with these keys:
+
+      * ``kickstart`` (*str|dict*)
+      * ``name`` (*str*)
+      * ``version`` (*str*)
+      * ``additional_repos`` (*list*) -- external repos specified by URL
+      * ``repos_from`` (*list*) -- repos from other variants
+      * ``specfile`` (*str*) -- for images wrapped in RPM
+      * ``scratch`` (*bool*) -- only RPM-wrapped images can use scratch builds,
+        but by default this is turned off
+
+
 Image Build Settings
 ====================
 

pungi/phases/live_images.py

@@ -67,9 +67,21 @@ class LiveImagesPhase(PhaseBase):
             return True
         return False
 
+    def _get_extra_repos(self, arch, variant, extras):
+        repo = []
+        for extra in extras:
+            v = self.compose.variants.get(extra)
+            if not v:
+                raise RuntimeError(
+                    'There is no variant %s to get repo from when building live image for %s.'
+                    % (extra, variant.uid))
+            repo.append(translate_path(
+                self.compose, self.compose.paths.compose.repository(arch, v, create_dir=False)))
+
+        return repo
+
     def run(self):
         symlink_isos_to = self.compose.conf.get("symlink_isos_to", None)
-        iso = IsoWrapper()
         commands = []
 
         for variant in self.compose.variants.values():
@@ -87,22 +99,21 @@ class LiveImagesPhase(PhaseBase):
                 cmd = {
                     "name": None,
                     "version": None,
-                    "arch": arch,
-                    "variant": variant,
                     "iso_path": None,
                     "wrapped_rpms_path": iso_dir,
                     "build_arch": arch,
                     "ks_file": ks_file,
                     "specfile": None,
                     "scratch": False,
-                    "cmd": [],
                     "label": "",  # currently not used
                 }
-                cmd["repos"] = [translate_path(self.compose, self.compose.paths.compose.repository(arch, variant))]
+                cmd["repos"] = [translate_path(
+                    self.compose, self.compose.paths.compose.repository(arch, variant, create_dir=False))]
 
                 # additional repos
                 data = get_arch_variant_data(self.compose.conf, "live_images", arch, variant)
                 cmd["repos"].extend(data[0].get("additional_repos", []))
+                cmd['repos'].extend(self._get_extra_repos(arch, variant, data[0].get('repos_from', [])))
 
                 # Explicit name and version
                 cmd["name"] = data[0].get("name", None)
@@ -133,17 +144,7 @@ class LiveImagesPhase(PhaseBase):
                     self.compose.log_warning("Skipping creating live image, it already exists: %s" % iso_path)
                     continue
                 cmd["iso_path"] = iso_path
-                iso_name = os.path.basename(iso_path)
 
-                # Additional commands
-
-                chdir_cmd = "cd %s" % pipes.quote(iso_dir)
-                cmd["cmd"].append(chdir_cmd)
-
-                # create iso manifest
-                cmd["cmd"].append(iso.get_manifest_cmd(iso_name))
-
-                cmd["cmd"] = " && ".join(cmd["cmd"])
                 commands.append((cmd, variant, arch))
 
         for (cmd, variant, arch) in commands:
@@ -170,7 +171,7 @@ class CreateLiveImageThread(WorkerThread):
     def process(self, item, num):
         compose, cmd, variant, arch = item
         try:
-            self.worker(compose, cmd, num)
+            self.worker(compose, cmd, variant, arch, num)
         except:
             if not compose.can_fail(variant, arch, 'live'):
                 raise
@@ -179,10 +180,10 @@ class CreateLiveImageThread(WorkerThread):
                        % (variant.uid, arch))
                 self.pool.log_info(msg)
 
-    def worker(self, compose, cmd, num):
+    def worker(self, compose, cmd, variant, arch, num):
-        log_file = compose.paths.log.log_file(cmd["arch"], "createiso-%s" % os.path.basename(cmd["iso_path"]))
+        log_file = compose.paths.log.log_file(arch, "createiso-%s" % os.path.basename(cmd["iso_path"]))
 
-        msg = "Creating ISO (arch: %s, variant: %s): %s" % (cmd["arch"], cmd["variant"], os.path.basename(cmd["iso_path"]))
+        msg = "Creating ISO (arch: %s, variant: %s): %s" % (arch, variant, os.path.basename(cmd["iso_path"]))
         self.pool.log_info("[BEGIN] %s" % msg)
 
         koji_wrapper = KojiWrapper(compose.conf["koji_profile"])
@@ -217,11 +218,19 @@ class CreateLiveImageThread(WorkerThread):
             for rpm_path in rpm_paths:
                 shutil.copy2(rpm_path, cmd["wrapped_rpms_path"])
 
-        # write manifest
+        self._write_manifest(cmd['iso_path'])
-        run(cmd["cmd"])
 
         self.pool.log_info("[DONE ] %s" % msg)
 
+    def _write_manifest(self, iso_path):
+        """Generate manifest for ISO at given path.
+
+        :param iso_path: (str) absolute path to the ISO
+        """
+        dir, filename = os.path.split(iso_path)
+        iso = IsoWrapper()
+        run("cd %s && %s" % (pipes.quote(dir), iso.get_manifest_cmd(filename)))
+
 
 def get_ks_in(compose, arch, variant):
     data = get_arch_variant_data(compose.conf, "live_images", arch, variant)

tests/test_liveimagesphase.py

@@ -0,0 +1,227 @@
+#!/usr/bin/env python2
+# -*- coding: utf-8 -*-
+
+
+import unittest
+import mock
+
+import os
+import sys
+
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), ".."))
+
+from pungi.phases.live_images import LiveImagesPhase, CreateLiveImageThread
+from pungi.util import get_arch_variant_data
+
+
+class _DummyCompose(object):
+    def __init__(self, config):
+        self.compose_id = 'Test-20151203.0.t'
+        self.conf = config
+        self.paths = mock.Mock(
+            compose=mock.Mock(
+                repository=mock.Mock(
+                    side_effect=lambda arch, variant, create_dir=False: os.path.join('/repo', arch, variant.uid)
+                ),
+                iso_dir=mock.Mock(
+                    side_effect=lambda arch, variant, symlink_to: os.path.join(
+                        '/iso_dir', arch, variant.uid
+                    )
+                ),
+                iso_path=mock.Mock(
+                    side_effect=lambda arch, variant, filename, symlink_to: os.path.join(
+                        '/iso_dir', arch, variant.uid, filename
+                    )
+                )
+            ),
+            log=mock.Mock(
+                log_file=mock.Mock(return_value='/a/b/log/log_file')
+            )
+        )
+        self._logger = mock.Mock()
+        self.variants = {
+            'Server': mock.Mock(uid='Server', arches=['x86_64', 'amd64']),
+            'Client': mock.Mock(uid='Client', arches=['amd64']),
+            'Everything': mock.Mock(uid='Everything', arches=['x86_64', 'amd64']),
+        }
+        self.log_error = mock.Mock()
+        self.get_image_name = mock.Mock(return_value='image-name')
+
+    def get_arches(self):
+        return ['x86_64', 'amd64']
+
+    def get_variants(self, arch=None, types=None):
+        return [v for v in self.variants.values() if not arch or arch in v.arches]
+
+    def can_fail(self, variant, arch, deliverable):
+        failable = get_arch_variant_data(self.conf, 'failable_deliverables', arch, variant)
+        return deliverable in failable
+
+
+class TestLiveImagesPhase(unittest.TestCase):
+
+    @mock.patch('pungi.phases.live_images.ThreadPool')
+    @mock.patch('pungi.phases.live_images.get_ks_in')
+    @mock.patch('pungi.phases.live_images.tweak_ks')
+    def test_image_build(self, tweak_ks, get_ks_in, ThreadPool):
+        compose = _DummyCompose({
+            'live_images': [
+                ('^Client$', {
+                    'amd64': {
+                        'additional_repos': ['http://example.com/repo/'],
+                        'repos_from': ['Everything'],
+                    }
+                })
+            ],
+        })
+
+        get_ks_in.side_effect = (lambda compose, arch, variant:
+                                 None if variant.uid != 'Client' or arch != 'amd64' else '/path/to/ks_in')
+        tweak_ks.return_value = '/path/to/ks_file'
+
+        phase = LiveImagesPhase(compose)
+
+        phase.run()
+
+        # assert at least one thread was started
+        self.assertTrue(phase.pool.add.called)
+        self.maxDiff = None
+        self.assertItemsEqual(phase.pool.queue_put.mock_calls,
+                              [mock.call((compose,
+                                          {'ks_file': '/path/to/ks_file',
+                                           'build_arch': 'amd64',
+                                           'wrapped_rpms_path': '/iso_dir/amd64/Client',
+                                           'scratch': False,
+                                           'repos': ['/repo/amd64/Client',
+                                                     'http://example.com/repo/',
+                                                     '/repo/amd64/Everything'],
+                                           'label': '',
+                                           'name': None,
+                                           'iso_path': '/iso_dir/amd64/Client/image-name',
+                                           'version': None,
+                                           'specfile': None},
+                                          compose.variants['Client'],
+                                          'amd64'))])
+
+
+class TestCreateLiveImageThread(unittest.TestCase):
+
+    @mock.patch('shutil.copy2')
+    @mock.patch('pungi.phases.live_images.run')
+    @mock.patch('pungi.phases.live_images.KojiWrapper')
+    def test_process(self, KojiWrapper, run, copy2):
+        compose = _DummyCompose({'koji_profile': 'koji'})
+        pool = mock.Mock()
+        cmd = {
+            'ks_file': '/path/to/ks_file',
+            'build_arch': 'amd64',
+            'wrapped_rpms_path': '/iso_dir/amd64/Client',
+            'scratch': False,
+            'repos': ['/repo/amd64/Client',
+                      'http://example.com/repo/',
+                      '/repo/amd64/Everything'],
+            'label': '',
+            'name': None,
+            'iso_path': '/iso_dir/amd64/Client/image-name',
+            'version': None,
+            'specfile': None
+        }
+
+        koji_wrapper = KojiWrapper.return_value
+        koji_wrapper.get_create_image_cmd.return_value = 'koji spin-livecd ...'
+        koji_wrapper.run_create_image_cmd.return_value = {
+            'retcode': 0,
+            'output': 'some output',
+            'task_id': 123
+        }
+        koji_wrapper.get_image_path.return_value = ['/path/to/image']
+
+        t = CreateLiveImageThread(pool)
+        with mock.patch('time.sleep'):
+            t.process((compose, cmd, compose.variants['Client'], 'amd64'), 1)
+
+        self.assertEqual(koji_wrapper.run_create_image_cmd.mock_calls,
+                         [mock.call('koji spin-livecd ...', log_file='/a/b/log/log_file')])
+        self.assertEqual(koji_wrapper.get_image_path.mock_calls, [mock.call(123)])
+        self.assertEqual(copy2.mock_calls,
+                         [mock.call('/path/to/image', '/iso_dir/amd64/Client/image-name')])
+
+        write_manifest_cmd = ' && '.join([
+            'cd /iso_dir/amd64/Client',
+            'isoinfo -R -f -i image-name | grep -v \'/TRANS.TBL$\' | sort >> image-name.manifest'
+        ])
+        self.assertEqual(run.mock_calls, [mock.call(write_manifest_cmd)])
+
+    @mock.patch('shutil.copy2')
+    @mock.patch('pungi.phases.live_images.run')
+    @mock.patch('pungi.phases.live_images.KojiWrapper')
+    def test_process_handles_fail(self, KojiWrapper, run, copy2):
+        compose = _DummyCompose({
+            'koji_profile': 'koji',
+            'failable_deliverables': [('^.+$', {'*': ['live']})],
+        })
+        pool = mock.Mock()
+        cmd = {
+            'ks_file': '/path/to/ks_file',
+            'build_arch': 'amd64',
+            'wrapped_rpms_path': '/iso_dir/amd64/Client',
+            'scratch': False,
+            'repos': ['/repo/amd64/Client',
+                      'http://example.com/repo/',
+                      '/repo/amd64/Everything'],
+            'label': '',
+            'name': None,
+            'iso_path': '/iso_dir/amd64/Client/image-name',
+            'version': None,
+            'specfile': None
+        }
+
+        koji_wrapper = KojiWrapper.return_value
+        koji_wrapper.get_create_image_cmd.return_value = 'koji spin-livecd ...'
+        koji_wrapper.run_create_image_cmd.return_value = {
+            'retcode': 1,
+            'output': 'some output',
+            'task_id': 123
+        }
+
+        t = CreateLiveImageThread(pool)
+        with mock.patch('time.sleep'):
+            t.process((compose, cmd, compose.variants['Client'], 'amd64'), 1)
+
+    @mock.patch('shutil.copy2')
+    @mock.patch('pungi.phases.live_images.run')
+    @mock.patch('pungi.phases.live_images.KojiWrapper')
+    def test_process_handles_exception(self, KojiWrapper, run, copy2):
+        compose = _DummyCompose({
+            'koji_profile': 'koji',
+            'failable_deliverables': [('^.+$', {'*': ['live']})],
+        })
+        pool = mock.Mock()
+        cmd = {
+            'ks_file': '/path/to/ks_file',
+            'build_arch': 'amd64',
+            'wrapped_rpms_path': '/iso_dir/amd64/Client',
+            'scratch': False,
+            'repos': ['/repo/amd64/Client',
+                      'http://example.com/repo/',
+                      '/repo/amd64/Everything'],
+            'label': '',
+            'name': None,
+            'iso_path': '/iso_dir/amd64/Client/image-name',
+            'version': None,
+            'specfile': None
+        }
+
+        def boom(*args, **kwargs):
+            raise RuntimeError('BOOM')
+
+        koji_wrapper = KojiWrapper.return_value
+        koji_wrapper.get_create_image_cmd.side_effect = boom
+
+        t = CreateLiveImageThread(pool)
+        with mock.patch('time.sleep'):
+            t.process((compose, cmd, compose.variants['Client'], 'amd64'), 1)
+
+
+if __name__ == "__main__":
+    unittest.main()