[ostree] Add ostree phase

This phase runs the script to make ostree repository in koji runroot
task. It runs right after regular yum repos are created.

Signed-off-by: Lubomír Sedlář <lsedlar@redhat.com>

bin/pungi-koji

@@ -226,6 +226,7 @@ def run_compose(compose):
     gather_phase = pungi.phases.GatherPhase(compose, pkgset_phase)
     extrafiles_phase = pungi.phases.ExtraFilesPhase(compose, pkgset_phase)
     createrepo_phase = pungi.phases.CreaterepoPhase(compose)
+    ostree_phase = pungi.phases.OSTreePhase(compose)
     productimg_phase = pungi.phases.ProductimgPhase(compose, pkgset_phase)
     createiso_phase = pungi.phases.CreateisoPhase(compose)
     liveimages_phase = pungi.phases.LiveImagesPhase(compose)
@@ -239,7 +240,7 @@ def run_compose(compose):
                   buildinstall_phase, productimg_phase, gather_phase,
                   extrafiles_phase, createiso_phase, liveimages_phase,
                   livemedia_phase, image_build_phase, image_checksum_phase,
-                  test_phase):
+                  test_phase, ostree_phase):
         if phase.skip():
             continue
         try:
@@ -320,6 +321,9 @@ def run_compose(compose):
     if not buildinstall_phase.skip():
         buildinstall_phase.copy_files()
 
+    ostree_phase.start()
+    ostree_phase.stop()
+
     # PRODUCTIMG phase
     productimg_phase.start()
     productimg_phase.stop()

doc/configuration.rst

@@ -934,6 +934,45 @@ Example
     }
 
 
+OSTree Settings
+===============
+
+The ``ostree`` phase of *Pungi* can create ostree repositories in a Koji
+runroot environment.
+
+**ostree**
+    (*dict*) -- a variant/arch mapping of configuration. The format should be
+    ``[(variant_uid_regex, {arch|*: config_dict})]``.
+
+    The configuration dict for each variant arch pair must have these keys:
+
+    * ``treefile`` -- (*str*) Filename of configuration for ``rpm-ostree``.
+    * ``config_url`` -- (*str*) URL for Git repository with the ``treefile``.
+    * ``source_repo_from`` -- (*str*) Name of variant serving as source repository.
+    * ``atomic_repo`` -- (*str*) Where to put the atomic repository
+
+    These keys are optional:
+
+    * ``config_branch`` -- (*str*) Git branch of the repo to use. Defaults to
+      ``master``.
+
+
+Example config
+--------------
+::
+
+    ostree = [
+        ("^Atomic$", {
+            "x86_64": {
+                "treefile": "fedora-atomic-docker-host.json",
+                "config_url": "https://git.fedorahosted.org/git/fedora-atomic.git",
+                "source_repo_from": "Everything",
+                "atomic_repo": "/mnt/koji/compose/atomic/Rawhide/"
+            }
+        })
+    ]
+
+
 Media Checksums Settings
 ========================
 

pungi/phases/__init__.py

@@ -29,3 +29,4 @@ from image_build import ImageBuildPhase  # noqa
 from test import TestPhase  # noqa
 from image_checksum import ImageChecksumPhase    # noqa
 from livemedia_phase import LiveMediaPhase  # noqa
+from ostree import OSTreePhase  # noqa

pungi/phases/base.py

@@ -72,3 +72,15 @@ class PhaseBase(object):
 
     def run(self):
         raise NotImplementedError
+
+
+class ConfigGuardedPhase(PhaseBase):
+    """A phase that is skipped unless config option is set."""
+
+    def skip(self):
+        if super(ConfigGuardedPhase, self).skip():
+            return True
+        if not self.compose.conf.get(self.name):
+            self.compose.log_info("Config section '%s' was not found. Skipping." % self.name)
+            return True
+        return False

pungi/phases/ostree.py

@@ -0,0 +1,98 @@
+# -*- coding: utf-8 -*-
+
+import os
+from kobo.threads import ThreadPool, WorkerThread
+import re
+
+from .base import ConfigGuardedPhase
+from .. import util
+from ..paths import translate_path
+from ..wrappers import scm, kojiwrapper
+
+
+class OSTreePhase(ConfigGuardedPhase):
+    name = 'ostree'
+
+    config_options = (
+        {
+            "name": "ostree",
+            "expected_types": [dict],
+            "optional": True,
+        }
+    )
+
+    def __init__(self, compose):
+        super(OSTreePhase, self).__init__(compose)
+        self.pool = ThreadPool(logger=self.compose._logger)
+
+    def run(self):
+        for variant in self.compose.get_variants():
+            for arch in variant.arches:
+                for conf in util.get_arch_variant_data(self.compose.conf, self.name, arch, variant):
+                    self.pool.add(OSTreeThread(self.pool))
+                    self.pool.queue_put((self.compose, variant, arch, conf))
+
+        self.pool.start()
+
+
+class OSTreeThread(WorkerThread):
+    def process(self, item, num):
+        compose, variant, arch, config = item
+        self.num = num
+
+        msg = 'OSTree phase for variant %s, arch %s' % (variant.uid, arch)
+        self.pool.log_info('[BEGIN] %s' % msg)
+        workdir = compose.paths.work.topdir('atomic')
+        self.logdir = compose.paths.log.topdir('{}/atomic'.format(arch))
+        repodir = os.path.join(workdir, 'config_repo')
+
+        source_variant = compose.variants[config['source_repo_from']]
+        source_repo = translate_path(compose, compose.paths.compose.repository(arch, source_variant))
+
+        self._clone_repo(repodir, config['config_url'], config.get('config_branch', 'master'))
+        self._tweak_mirrorlist(repodir, source_repo)
+        self._run_atomic_cmd(compose, variant, arch, config, source_repo)
+
+        self.pool.log_info('[DONE ] %s' % msg)
+
+    def _run_atomic_cmd(self, compose, variant, arch, config, source_repo):
+        cmd = [
+            'pungi-make-ostree',
+            '--log-dir={}'.format(self.logdir),
+            '--treefile={}'.format(config['treefile']),
+            config['atomic_repo']
+        ]
+
+        runroot_channel = compose.conf.get("runroot_channel", None)
+        runroot_tag = compose.conf["runroot_tag"]
+
+        packages = ['pungi', 'ostree', 'rpm-ostree']
+        log_file = os.path.join(self.logdir, 'runroot.log')
+        koji = kojiwrapper.KojiWrapper(compose.conf["koji_profile"])
+        koji_cmd = koji.get_runroot_cmd(runroot_tag, arch, cmd,
+                                        channel=runroot_channel,
+                                        use_shell=True, task_id=True,
+                                        packages=packages, mounts=[compose.topdir])
+        output = koji.run_runroot_cmd(koji_cmd, log_file=log_file)
+        if output["retcode"] != 0:
+            raise RuntimeError("Runroot task failed: %s. See %s for more details."
+                               % (output["task_id"], log_file))
+
+    def _clone_repo(self, repodir, url, branch):
+        scm.get_dir_from_scm({'scm': 'git', 'repo': url, 'branch': branch, 'dir': '.'},
+                             repodir, logger=self.pool._logger)
+
+    def _tweak_mirrorlist(self, repodir, source_repo):
+        for file in os.listdir(repodir):
+            if file.endswith('.repo'):
+                tweak_file(os.path.join(repodir, file), source_repo)
+
+
+def tweak_file(path, source_repo):
+    """Replace mirrorlist line in repo file with baseurl pointing to source_repo."""
+    with open(path, 'r') as f:
+        contents = f.read()
+    replacement = 'baseurl={}'.format(source_repo)
+    contents = re.sub(r'^mirrorlist=.*$', replacement, contents)
+    with open(path, 'w') as f:
+        f.write(contents)

tests/test_ostree_phase.py

@@ -0,0 +1,98 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+
+import unittest
+import mock
+
+import os
+import sys
+
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..'))
+
+from tests import helpers
+from pungi.phases import ostree
+
+
+class OSTreePhaseTest(helpers.PungiTestCase):
+
+    @mock.patch('pungi.phases.ostree.ThreadPool')
+    def test_run(self, ThreadPool):
+        cfg = mock.Mock()
+        compose = helpers.DummyCompose(self.topdir, {
+            'ostree': [
+                ('^Everything$', {'x86_64': cfg})
+            ]
+        })
+
+        pool = ThreadPool.return_value
+
+        phase = ostree.OSTreePhase(compose)
+        phase.run()
+
+        self.assertEqual(len(pool.add.call_args_list), 1)
+        self.assertEqual(pool.queue_put.call_args_list,
+                         [mock.call((compose, compose.variants['Everything'], 'x86_64', cfg))])
+
+    @mock.patch('pungi.phases.ostree.ThreadPool')
+    def test_skip_without_config(self, ThreadPool):
+        compose = helpers.DummyCompose(self.topdir, {})
+        compose.just_phases = None
+        compose.skip_phases = []
+        phase = ostree.OSTreePhase(compose)
+        self.assertTrue(phase.skip())
+
+
+class OSTreeThreadTest(helpers.PungiTestCase):
+
+    def _dummy_config_repo(self, scm_dict, target, logger=None):
+        helpers.touch(os.path.join(target, 'fedora-atomic-docker-host.json'))
+        helpers.touch(os.path.join(target, 'fedora-rawhide.repo'))
+
+    @mock.patch('pungi.wrappers.scm.get_dir_from_scm')
+    @mock.patch('pungi.wrappers.kojiwrapper.KojiWrapper')
+    def test_run(self, KojiWrapper, get_dir_from_scm):
+        compose = helpers.DummyCompose(self.topdir, {
+            'koji_profile': 'koji',
+            'runroot_tag': 'rrt',
+        })
+        pool = mock.Mock()
+        cfg = {
+            'source_repo_from': 'Everything',
+            'config_url': 'https://git.fedorahosted.org/git/fedora-atomic.git',
+            'config_branch': 'f24',
+            'treefile': 'fedora-atomic-docker-host.json',
+            'atomic_repo': '/other/place/for/atomic'
+        }
+        get_dir_from_scm.side_effect = self._dummy_config_repo
+        koji = KojiWrapper.return_value
+        koji.run_runroot_cmd.return_value = {
+            'task_id': 1234,
+            'retcode': 0,
+            'output': 'Foo bar\n',
+        }
+
+        t = ostree.OSTreeThread(pool)
+
+        t.process((compose, compose.variants['Everything'], 'x86_64', cfg), 1)
+
+        self.assertEqual(get_dir_from_scm.call_args_list,
+                         [mock.call({'scm': 'git', 'repo': 'https://git.fedorahosted.org/git/fedora-atomic.git',
+                                     'branch': 'f24', 'dir': '.'},
+                                    self.topdir + '/work/atomic/config_repo', logger=pool._logger)])
+        self.assertEqual(koji.get_runroot_cmd.call_args_list,
+                         [mock.call('rrt', 'x86_64',
+                                    ['pungi-make-ostree',
+                                     '--log-dir={}/logs/x86_64/atomic'.format(self.topdir),
+                                     '--treefile=fedora-atomic-docker-host.json',
+                                     '/other/place/for/atomic'],
+                                    channel=None, mounts=[self.topdir],
+                                    packages=['pungi', 'ostree', 'rpm-ostree'],
+                                    task_id=True, use_shell=True)])
+        self.assertEqual(koji.run_runroot_cmd.call_args_list,
+                         [mock.call(koji.get_runroot_cmd.return_value,
+                                    log_file=self.topdir + '/logs/x86_64/atomic/runroot.log')])
+
+
+if __name__ == '__main__':
+    unittest.main()