[live-media] Add live media phase

This phase builds live media in Koji using the Live Media Creator. It
runs in parallel with current live images, create ISO and image build
phases.

The documentation is updated to explain how to configure this.

Signed-off-by: Lubomír Sedlář <lsedlar@redhat.com>

bin/pungi-koji

@@ -229,6 +229,7 @@ def run_compose(compose):
     productimg_phase = pungi.phases.ProductimgPhase(compose, pkgset_phase)
     createiso_phase = pungi.phases.CreateisoPhase(compose)
     liveimages_phase = pungi.phases.LiveImagesPhase(compose)
+    livemedia_phase = pungi.phases.LiveMediaPhase(compose)
     image_build_phase = pungi.phases.ImageBuildPhase(compose)
     image_checksum_phase = pungi.phases.ImageChecksumPhase(compose)
     test_phase = pungi.phases.TestPhase(compose)
@@ -237,7 +238,8 @@ def run_compose(compose):
     for phase in (init_phase, pkgset_phase, createrepo_phase,
                   buildinstall_phase, productimg_phase, gather_phase,
                   extrafiles_phase, createiso_phase, liveimages_phase,
-                  image_build_phase, image_checksum_phase, test_phase):
+                  livemedia_phase, image_build_phase, image_checksum_phase,
+                  test_phase):
         if phase.skip():
             continue
         try:
@@ -302,10 +304,12 @@ def run_compose(compose):
     createiso_phase.start()
     liveimages_phase.start()
     image_build_phase.start()
+    livemedia_phase.start()
 
     createiso_phase.stop()
     liveimages_phase.stop()
     image_build_phase.stop()
+    livemedia_phase.stop()
 
     image_checksum_phase.start()
     image_checksum_phase.stop()

doc/configuration.rst

@@ -135,6 +135,7 @@ Options
      * iso
      * live
      * image-build
+     * live-media
 
        .. note::
 
@@ -646,6 +647,30 @@ Live Images Settings
       * ``scratch`` (*bool*) -- only RPM-wrapped images can use scratch builds,
         but by default this is turned off
 
+Live Media Settings
+===================
+
+**live_media**
+    (*dict*) -- configuration for ``koji spin-livemedia``; format:
+    ``{variant_uid_regex: [{opt:value}]}``
+
+    Available options:
+
+      * ``target`` (*str*)
+      * ``arches`` (*[str]*) -- what architectures to build the media for; by default uses
+        all arches for the variant.
+      * ``kickstart`` (*str*) -- name of the kickstart file
+      * ``ksurl`` (*str*)
+      * ``ksversion`` (*str*)
+      * ``scratch`` (*bool*)
+      * ``release`` (*str*) -- a string with the release, or explicit ``None``
+        for using compose date and respin.
+      * ``skip_tag`` (*bool*)
+      * ``name`` (*str*)
+      * ``repo`` (*[str]*) -- external repo
+      * ``repo_from`` (*[str]*) -- list of variants to take extra repos from
+      * ``title`` (*str*)
+
 
 Image Build Settings
 ====================

pungi/phases/__init__.py

@@ -28,3 +28,4 @@ from live_images import LiveImagesPhase  # noqa
 from image_build import ImageBuildPhase  # noqa
 from test import TestPhase  # noqa
 from image_checksum import ImageChecksumPhase    # noqa
+from livemedia_phase import LiveMediaPhase  # noqa

pungi/phases/livemedia_phase.py

@@ -0,0 +1,178 @@
+# -*- coding: utf-8 -*-
+
+import os
+import time
+from kobo import shortcuts
+
+from pungi.util import get_variant_data, resolve_git_url, makedirs
+from pungi.phases.base import PhaseBase
+from pungi.linker import Linker
+from pungi.paths import translate_path
+from pungi.wrappers.kojiwrapper import KojiWrapper
+from kobo.threads import ThreadPool, WorkerThread
+from productmd.images import Image
+
+
+class LiveMediaPhase(PhaseBase):
+    """class for wrapping up koji spin-livemedia"""
+    name = 'live_media'
+
+    def __init__(self, compose):
+        super(LiveMediaPhase, self).__init__(compose)
+        self.pool = ThreadPool(logger=self.compose._logger)
+
+    def skip(self):
+        if super(LiveMediaPhase, self).skip():
+            return True
+        if not self.compose.conf.get(self.name):
+            self.compose.log_info("Config section '%s' was not found. Skipping" % self.name)
+            return True
+        return False
+
+    def _get_repos(self, image_conf, variant):
+        """
+        Get a comma separated list of repos. First included are those
+        explicitly listed in config, followed by repos from other variants,
+        finally followed by repo for current variant.
+
+        The `repo_from` key is removed from the dict (if present).
+        """
+        repo = shortcuts.force_list(image_conf.get('repo', []))
+
+        extras = shortcuts.force_list(image_conf.pop('repo_from', []))
+        extras.append(variant.uid)
+
+        for extra in extras:
+            v = self.compose.variants.get(extra)
+            if not v:
+                raise RuntimeError(
+                    'There is no variant %s to get repo from when building live media for %s.'
+                    % (extra, variant.uid))
+            repo.append(translate_path(
+                self.compose,
+                self.compose.paths.compose.repository('$arch', v, create_dir=False)))
+
+        return repo
+
+    def _get_arches(self, image_conf, arches):
+        if 'arches' in image_conf:
+            arches = set(image_conf.get('arches', [])) & arches
+        return sorted(arches)
+
+    def _get_release(self, image_conf):
+        """If release is set explicitly to None, replace it with date and respin."""
+        if 'release' in image_conf and image_conf['release'] is None:
+            return '%s.%s' % (self.compose.compose_date, self.compose.compose_respin)
+        return image_conf.get('release', None)
+
+    def run(self):
+        for variant in self.compose.get_variants():
+            arches = set([x for x in variant.arches if x != 'src'])
+
+            for image_conf in get_variant_data(self.compose.conf, self.name, variant):
+                config = {
+                    'target': image_conf['target'],
+                    'arches': self._get_arches(image_conf, arches),
+                    'kickstart': image_conf['kickstart'],
+                    'ksurl': resolve_git_url(image_conf['ksurl']),
+                    'ksversion': image_conf.get('ksversion'),
+                    'scratch': image_conf.get('scratch', False),
+                    'release': self._get_release(image_conf),
+                    'skip_tag': image_conf.get('skip_tag'),
+                    'name': image_conf['name'],
+                    'title': image_conf.get('title'),
+                    'repo': self._get_repos(image_conf, variant),
+                    'install_tree': translate_path(
+                        self.compose,
+                        self.compose.paths.compose.os_tree('$arch', variant, create_dir=False)
+                    )
+                }
+                self.pool.add(LiveMediaThread(self.pool))
+                self.pool.queue_put((self.compose, variant, config))
+
+        self.pool.start()
+
+
+class LiveMediaThread(WorkerThread):
+    def process(self, item, num):
+        compose, variant, config = item
+        self.num = num
+        try:
+            self.worker(compose, variant, config)
+        except:
+            if not compose.can_fail(variant, '*', 'live-media'):
+                raise
+            else:
+                msg = ('[FAIL] live-media for variant %s failed, but going on anyway.'
+                       % variant.uid)
+                self.pool.log_info(msg)
+
+    def _get_log_file(self, compose, variant, config):
+        arches = '-'.join(config['arches'])
+        return compose.paths.log.log_file(arches, 'livemedia-%s' % variant)
+
+    def _run_command(self, koji_wrapper, cmd, compose, log_file):
+        time.sleep(self.num * 3)
+        output = koji_wrapper.run_blocking_cmd(cmd, log_file=log_file)
+        self.pool.log_debug('live media outputs: %s' % (output))
+        if output['retcode'] != 0:
+            compose.log_error('Live media task failed.')
+            raise RuntimeError('Live media task failed: %s. See %s for more details.'
+                               % (output['task_id'], log_file))
+        return output
+
+    def worker(self, compose, variant, config):
+        msg = 'Live media: %s (arches: %s, variant: %s)' % (config['name'],
+                                                            ' '.join(config['arches']),
+                                                            variant.uid)
+        self.pool.log_info('[BEGIN] %s' % msg)
+
+        koji_wrapper = KojiWrapper(compose.conf['koji_profile'])
+        cmd = koji_wrapper.get_live_media_cmd(config)
+
+        log_file = self._get_log_file(compose, variant, config)
+        output = self._run_command(koji_wrapper, cmd, compose, log_file)
+
+        # collect results and update manifest
+        image_infos = []
+
+        paths = koji_wrapper.get_image_paths(output['task_id'])
+
+        for arch, paths in paths.iteritems():
+            for path in paths:
+                if path.endswith('.iso'):
+                    image_infos.append({'path': path, 'arch': arch})
+
+        if len(image_infos) != len(config['arches']):
+            self.pool.log_error(
+                'Error in koji task %s. Expected to find one image for each arch (%s). Got %s.'
+                % (output['task_id'], len(config['arches']), len(image_infos)))
+            raise RuntimeError('Image count mismatch in task %s.' % output['task_id'])
+
+        linker = Linker(logger=compose._logger)
+        link_type = compose.conf.get("link_type", "hardlink-or-copy")
+        for image_info in image_infos:
+            image_dir = compose.paths.compose.image_dir(variant) % {"arch": image_info['arch']}
+            makedirs(image_dir)
+            relative_image_dir = (
+                compose.paths.compose.image_dir(variant, relative=True) % {"arch": image_info['arch']}
+            )
+
+            # let's not change filename of koji outputs
+            image_dest = os.path.join(image_dir, os.path.basename(image_info['path']))
+            linker.link(image_info['path'], image_dest, link_type=link_type)
+
+            # Update image manifest
+            img = Image(compose.im)
+            img.type = 'live'
+            img.format = 'iso'
+            img.path = os.path.join(relative_image_dir, os.path.basename(image_dest))
+            img.mtime = int(os.stat(image_dest).st_mtime)
+            img.size = os.path.getsize(image_dest)
+            img.arch = image_info['arch']
+            img.disc_number = 1     # We don't expect multiple disks
+            img.disc_count = 1
+            img.bootable = True
+            compose.im.add(variant=variant.uid, arch=image_info['arch'], image=img)
+
+        self.pool.log_info('[DONE ] %s' % msg)

tests/test_livemediaphase.py

@@ -0,0 +1,328 @@
+#!/usr/bin/env python2
+# -*- coding: utf-8 -*-
+
+import unittest
+import mock
+
+import sys
+import os
+
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), ".."))
+
+from pungi.phases.livemedia_phase import LiveMediaPhase, LiveMediaThread
+from pungi.util import get_arch_variant_data
+
+
+class _DummyCompose(object):
+    def __init__(self, config):
+        self.compose_date = '20151203'
+        self.compose_type_suffix = '.t'
+        self.compose_respin = 0
+        self.ci_base = mock.Mock(
+            release_id='Test-1.0',
+            release=mock.Mock(
+                short='test',
+                version='1.0',
+            ),
+        )
+        self.conf = config
+        self.paths = mock.Mock(
+            compose=mock.Mock(
+                topdir=mock.Mock(return_value='/a/b'),
+                os_tree=mock.Mock(
+                    side_effect=lambda arch, variant, create_dir=False: os.path.join('/ostree', arch, variant.uid)
+                ),
+                repository=mock.Mock(
+                    side_effect=lambda arch, variant, create_dir=False: os.path.join('/repo', arch, variant.uid)
+                ),
+                image_dir=mock.Mock(
+                    side_effect=lambda variant, relative=False: os.path.join(
+                        '' if relative else '/', 'image_dir', variant.uid, '%(arch)s'
+                    )
+                )
+            ),
+            work=mock.Mock(
+                image_build_conf=mock.Mock(
+                    side_effect=lambda variant, image_name, image_type:
+                        '-'.join([variant.uid, image_name, image_type])
+                )
+            ),
+            log=mock.Mock(
+                log_file=mock.Mock(return_value='/a/b/log/log_file')
+            )
+        )
+        self._logger = mock.Mock()
+        self.variants = {
+            'Server': mock.Mock(uid='Server', arches=['x86_64', 'amd64']),
+            'Client': mock.Mock(uid='Client', arches=['amd64']),
+            'Everything': mock.Mock(uid='Everything', arches=['x86_64', 'amd64']),
+        }
+        self.im = mock.Mock()
+        self.log_error = mock.Mock()
+
+    def get_variants(self, arch=None, types=None):
+        return [v for v in self.variants.values() if not arch or arch in v.arches]
+
+    def can_fail(self, variant, arch, deliverable):
+        failable = get_arch_variant_data(self.conf, 'failable_deliverables', arch, variant)
+        return deliverable in failable
+
+
+class TestLiveMediaPhase(unittest.TestCase):
+    @mock.patch('pungi.phases.livemedia_phase.ThreadPool')
+    def test_live_media_minimal(self, ThreadPool):
+        compose = _DummyCompose({
+            'live_media': {
+                '^Server$': [
+                    {
+                        'target': 'f24',
+                        'kickstart': 'file.ks',
+                        'ksurl': 'git://example.com/repo.git',
+                        'name': 'Fedora Server Live',
+                    }
+                ]
+            },
+            'koji_profile': 'koji',
+        })
+
+        phase = LiveMediaPhase(compose)
+
+        phase.run()
+        self.assertTrue(phase.pool.add.called)
+        self.assertEqual(phase.pool.queue_put.call_args_list,
+                         [mock.call((compose,
+                                     compose.variants['Server'],
+                                     {
+                                         'arches': ['amd64', 'x86_64'],
+                                         'kickstart': 'file.ks',
+                                         'ksurl': 'git://example.com/repo.git',
+                                         'ksversion': None,
+                                         'name': 'Fedora Server Live',
+                                         'release': None,
+                                         'repo': ['/repo/$arch/Server'],
+                                         'scratch': False,
+                                         'skip_tag': None,
+                                         'target': 'f24',
+                                         'title': None,
+                                         'install_tree': '/ostree/$arch/Server',
+                                     }))])
+
+    @mock.patch('pungi.phases.livemedia_phase.resolve_git_url')
+    @mock.patch('pungi.phases.livemedia_phase.ThreadPool')
+    def test_live_media_full(self, ThreadPool, resolve_git_url):
+        compose = _DummyCompose({
+            'live_media': {
+                '^Server$': [
+                    {
+                        'target': 'f24',
+                        'kickstart': 'file.ks',
+                        'ksurl': 'git://example.com/repo.git#HEAD',
+                        'name': 'Fedora Server Live',
+                        'scratch': True,
+                        'skip_tag': True,
+                        'title': 'Custom Title',
+                        'repo_from': ['Everything'],
+                        'repo': ['http://example.com/extra_repo'],
+                        'arches': ['x86_64'],
+                        'ksversion': '24',
+                        'release': None
+                    }
+                ]
+            }
+        })
+
+        resolve_git_url.return_value = 'resolved'
+
+        phase = LiveMediaPhase(compose)
+
+        phase.run()
+        self.assertTrue(phase.pool.add.called)
+        self.assertEqual(phase.pool.queue_put.call_args_list,
+                         [mock.call((compose,
+                                     compose.variants['Server'],
+                                     {
+                                         'arches': ['x86_64'],
+                                         'kickstart': 'file.ks',
+                                         'ksurl': 'resolved',
+                                         'ksversion': '24',
+                                         'name': 'Fedora Server Live',
+                                         'release': '20151203.0',
+                                         'repo': ['http://example.com/extra_repo',
+                                                  '/repo/$arch/Everything',
+                                                  '/repo/$arch/Server'],
+                                         'scratch': True,
+                                         'skip_tag': True,
+                                         'target': 'f24',
+                                         'title': 'Custom Title',
+                                         'install_tree': '/ostree/$arch/Server',
+                                     }))])
+
+
+class TestCreateImageBuildThread(unittest.TestCase):
+
+    @mock.patch('pungi.phases.livemedia_phase.KojiWrapper')
+    @mock.patch('pungi.phases.livemedia_phase.Linker')
+    @mock.patch('pungi.phases.livemedia_phase.makedirs')
+    def test_process(self, makedirs, Linker, KojiWrapper):
+        compose = _DummyCompose({
+            'koji_profile': 'koji'
+        })
+        config = {
+            'arches': ['amd64', 'x86_64'],
+            'kickstart': 'file.ks',
+            'ksurl': 'git://example.com/repo.git',
+            'ksversion': None,
+            'name': 'Fedora Server Live',
+            'release': None,
+            'repo': ['/repo/$arch/Server'],
+            'scratch': False,
+            'skip_tag': None,
+            'target': 'f24',
+            'title': None,
+        }
+        pool = mock.Mock()
+
+        get_live_media_cmd = KojiWrapper.return_value.get_live_media_cmd
+        get_live_media_cmd.return_value = 'koji-spin-livemedia'
+
+        run_blocking_cmd = KojiWrapper.return_value.run_blocking_cmd
+        run_blocking_cmd.return_value = {
+            'task_id': 1234,
+            'retcode': 0,
+            'output': None,
+        }
+
+        get_image_paths = KojiWrapper.return_value.get_image_paths
+        get_image_paths.return_value = {
+            'x86_64': [
+                '/koji/task/1235/tdl-amd64.xml',
+                '/koji/task/1235/Live-20160103.x86_64.iso',
+                '/koji/task/1235/Live-20160103.x86_64.tar.xz'
+            ],
+            'amd64': [
+                '/koji/task/1235/tdl-amd64.xml',
+                '/koji/task/1235/Live-20160103.amd64.iso',
+                '/koji/task/1235/Live-20160103.amd64.tar.xz'
+            ]
+        }
+
+        t = LiveMediaThread(pool)
+        with mock.patch('os.stat') as stat:
+            with mock.patch('os.path.getsize') as getsize:
+                with mock.patch('time.sleep'):
+                    getsize.return_value = 1024
+                    stat.return_value.st_mtime = 13579
+                    t.process((compose, compose.variants['Server'], config), 1)
+
+        self.assertEqual(run_blocking_cmd.mock_calls,
+                         [mock.call('koji-spin-livemedia', log_file='/a/b/log/log_file')])
+        self.assertEqual(get_live_media_cmd.mock_calls,
+                         [mock.call(config)])
+        self.assertEqual(get_image_paths.mock_calls,
+                         [mock.call(1234)])
+        self.assertItemsEqual(makedirs.mock_calls,
+                              [mock.call('/image_dir/Server/x86_64'),
+                               mock.call('/image_dir/Server/amd64')])
+        link = Linker.return_value.link
+        self.assertItemsEqual(link.mock_calls,
+                              [mock.call('/koji/task/1235/Live-20160103.amd64.iso',
+                                         '/image_dir/Server/amd64/Live-20160103.amd64.iso',
+                                         link_type='hardlink-or-copy'),
+                               mock.call('/koji/task/1235/Live-20160103.x86_64.iso',
+                                         '/image_dir/Server/x86_64/Live-20160103.x86_64.iso',
+                                         link_type='hardlink-or-copy')])
+
+        image_relative_paths = [
+            'image_dir/Server/amd64/Live-20160103.amd64.iso',
+            'image_dir/Server/x86_64/Live-20160103.x86_64.iso'
+        ]
+
+        self.assertEqual(len(compose.im.add.call_args_list), 2)
+        for call in compose.im.add.call_args_list:
+            _, kwargs = call
+            image = kwargs['image']
+            self.assertEqual(kwargs['variant'], 'Server')
+            self.assertIn(kwargs['arch'], ('amd64', 'x86_64'))
+            self.assertEqual(kwargs['arch'], image.arch)
+            self.assertIn(image.path, image_relative_paths)
+            self.assertEqual('iso', image.format)
+            self.assertEqual('live', image.type)
+
+    @mock.patch('pungi.phases.livemedia_phase.KojiWrapper')
+    def test_handle_koji_fail(self, KojiWrapper):
+        compose = _DummyCompose({
+            'koji_profile': 'koji',
+            'failable_deliverables': [
+                ('^.+$', {'*': ['live-media']})
+            ]
+        })
+        config = {
+            'arches': ['amd64', 'x86_64'],
+            'kickstart': 'file.ks',
+            'ksurl': 'git://example.com/repo.git',
+            'ksversion': None,
+            'name': 'Fedora Server Live',
+            'release': None,
+            'repo': ['/repo/$arch/Server'],
+            'scratch': False,
+            'skip_tag': None,
+            'target': 'f24',
+            'title': None,
+        }
+        pool = mock.Mock()
+
+        run_blocking_cmd = KojiWrapper.return_value.run_blocking_cmd
+        run_blocking_cmd.return_value = {
+            'task_id': 1234,
+            'retcode': 1,
+            'output': None,
+        }
+
+        t = LiveMediaThread(pool)
+        with mock.patch('os.stat') as stat:
+            with mock.patch('os.path.getsize') as getsize:
+                with mock.patch('time.sleep'):
+                    getsize.return_value = 1024
+                    stat.return_value.st_mtime = 13579
+                    t.process((compose, compose.variants['Server'], config), 1)
+
+    @mock.patch('pungi.phases.livemedia_phase.KojiWrapper')
+    def test_handle_exception(self, KojiWrapper):
+        compose = _DummyCompose({
+            'koji_profile': 'koji',
+            'failable_deliverables': [
+                ('^.+$', {'*': ['live-media']})
+            ]
+        })
+        config = {
+            'arches': ['amd64', 'x86_64'],
+            'kickstart': 'file.ks',
+            'ksurl': 'git://example.com/repo.git',
+            'ksversion': None,
+            'name': 'Fedora Server Live',
+            'release': None,
+            'repo': ['/repo/$arch/Server'],
+            'scratch': False,
+            'skip_tag': None,
+            'target': 'f24',
+            'title': None,
+        }
+        pool = mock.Mock()
+
+        def boom(*args, **kwargs):
+            raise Exception('BOOM')
+
+        run_blocking_cmd = KojiWrapper.return_value.run_blocking_cmd
+        run_blocking_cmd.side_effect = boom
+
+        t = LiveMediaThread(pool)
+        with mock.patch('os.stat') as stat:
+            with mock.patch('os.path.getsize') as getsize:
+                with mock.patch('time.sleep'):
+                    getsize.return_value = 1024
+                    stat.return_value.st_mtime = 13579
+                    t.process((compose, compose.variants['Server'], config), 1)
+
+
+if __name__ == "__main__":
+    unittest.main()