diff --git a/bin/pungi-koji b/bin/pungi-koji index 9bbc7322..1734c9c7 100755 --- a/bin/pungi-koji +++ b/bin/pungi-koji @@ -226,6 +226,7 @@ def run_compose(compose): gather_phase = pungi.phases.GatherPhase(compose, pkgset_phase) extrafiles_phase = pungi.phases.ExtraFilesPhase(compose, pkgset_phase) createrepo_phase = pungi.phases.CreaterepoPhase(compose) + ostree_phase = pungi.phases.OSTreePhase(compose) productimg_phase = pungi.phases.ProductimgPhase(compose, pkgset_phase) createiso_phase = pungi.phases.CreateisoPhase(compose) liveimages_phase = pungi.phases.LiveImagesPhase(compose) @@ -239,7 +240,7 @@ def run_compose(compose): buildinstall_phase, productimg_phase, gather_phase, extrafiles_phase, createiso_phase, liveimages_phase, livemedia_phase, image_build_phase, image_checksum_phase, - test_phase): + test_phase, ostree_phase): if phase.skip(): continue try: @@ -320,6 +321,9 @@ def run_compose(compose): if not buildinstall_phase.skip(): buildinstall_phase.copy_files() + ostree_phase.start() + ostree_phase.stop() + # PRODUCTIMG phase productimg_phase.start() productimg_phase.stop() diff --git a/doc/configuration.rst b/doc/configuration.rst index 025fd70a..72daec54 100644 --- a/doc/configuration.rst +++ b/doc/configuration.rst @@ -934,6 +934,45 @@ Example } +OSTree Settings +=============== + +The ``ostree`` phase of *Pungi* can create ostree repositories in a Koji +runroot environment. + +**ostree** + (*dict*) -- a variant/arch mapping of configuration. The format should be + ``[(variant_uid_regex, {arch|*: config_dict})]``. + + The configuration dict for each variant arch pair must have these keys: + + * ``treefile`` -- (*str*) Filename of configuration for ``rpm-ostree``. + * ``config_url`` -- (*str*) URL for Git repository with the ``treefile``. + * ``source_repo_from`` -- (*str*) Name of variant serving as source repository. + * ``atomic_repo`` -- (*str*) Where to put the atomic repository + + These keys are optional: + + * ``config_branch`` -- (*str*) Git branch of the repo to use. Defaults to + ``master``. + + +Example config +-------------- +:: + + ostree = [ + ("^Atomic$", { + "x86_64": { + "treefile": "fedora-atomic-docker-host.json", + "config_url": "https://git.fedorahosted.org/git/fedora-atomic.git", + "source_repo_from": "Everything", + "atomic_repo": "/mnt/koji/compose/atomic/Rawhide/" + } + }) + ] + + Media Checksums Settings ======================== diff --git a/pungi/phases/__init__.py b/pungi/phases/__init__.py index ba72887f..bebda230 100644 --- a/pungi/phases/__init__.py +++ b/pungi/phases/__init__.py @@ -29,3 +29,4 @@ from image_build import ImageBuildPhase # noqa from test import TestPhase # noqa from image_checksum import ImageChecksumPhase # noqa from livemedia_phase import LiveMediaPhase # noqa +from ostree import OSTreePhase # noqa diff --git a/pungi/phases/base.py b/pungi/phases/base.py index 6c83bdf7..6e4446d0 100644 --- a/pungi/phases/base.py +++ b/pungi/phases/base.py @@ -72,3 +72,15 @@ class PhaseBase(object): def run(self): raise NotImplementedError + + +class ConfigGuardedPhase(PhaseBase): + """A phase that is skipped unless config option is set.""" + + def skip(self): + if super(ConfigGuardedPhase, self).skip(): + return True + if not self.compose.conf.get(self.name): + self.compose.log_info("Config section '%s' was not found. Skipping." % self.name) + return True + return False diff --git a/pungi/phases/ostree.py b/pungi/phases/ostree.py new file mode 100644 index 00000000..1cdd2ac8 --- /dev/null +++ b/pungi/phases/ostree.py @@ -0,0 +1,98 @@ +# -*- coding: utf-8 -*- + +import os +from kobo.threads import ThreadPool, WorkerThread +import re + +from .base import ConfigGuardedPhase +from .. import util +from ..paths import translate_path +from ..wrappers import scm, kojiwrapper + + +class OSTreePhase(ConfigGuardedPhase): + name = 'ostree' + + config_options = ( + { + "name": "ostree", + "expected_types": [dict], + "optional": True, + } + ) + + def __init__(self, compose): + super(OSTreePhase, self).__init__(compose) + self.pool = ThreadPool(logger=self.compose._logger) + + def run(self): + for variant in self.compose.get_variants(): + for arch in variant.arches: + for conf in util.get_arch_variant_data(self.compose.conf, self.name, arch, variant): + self.pool.add(OSTreeThread(self.pool)) + self.pool.queue_put((self.compose, variant, arch, conf)) + + self.pool.start() + + +class OSTreeThread(WorkerThread): + def process(self, item, num): + compose, variant, arch, config = item + self.num = num + + msg = 'OSTree phase for variant %s, arch %s' % (variant.uid, arch) + self.pool.log_info('[BEGIN] %s' % msg) + workdir = compose.paths.work.topdir('atomic') + self.logdir = compose.paths.log.topdir('{}/atomic'.format(arch)) + repodir = os.path.join(workdir, 'config_repo') + + source_variant = compose.variants[config['source_repo_from']] + source_repo = translate_path(compose, compose.paths.compose.repository(arch, source_variant)) + + self._clone_repo(repodir, config['config_url'], config.get('config_branch', 'master')) + self._tweak_mirrorlist(repodir, source_repo) + self._run_atomic_cmd(compose, variant, arch, config, source_repo) + + self.pool.log_info('[DONE ] %s' % msg) + + def _run_atomic_cmd(self, compose, variant, arch, config, source_repo): + cmd = [ + 'pungi-make-ostree', + '--log-dir={}'.format(self.logdir), + '--treefile={}'.format(config['treefile']), + config['atomic_repo'] + ] + + runroot_channel = compose.conf.get("runroot_channel", None) + runroot_tag = compose.conf["runroot_tag"] + + packages = ['pungi', 'ostree', 'rpm-ostree'] + log_file = os.path.join(self.logdir, 'runroot.log') + koji = kojiwrapper.KojiWrapper(compose.conf["koji_profile"]) + koji_cmd = koji.get_runroot_cmd(runroot_tag, arch, cmd, + channel=runroot_channel, + use_shell=True, task_id=True, + packages=packages, mounts=[compose.topdir]) + output = koji.run_runroot_cmd(koji_cmd, log_file=log_file) + if output["retcode"] != 0: + raise RuntimeError("Runroot task failed: %s. See %s for more details." + % (output["task_id"], log_file)) + + def _clone_repo(self, repodir, url, branch): + scm.get_dir_from_scm({'scm': 'git', 'repo': url, 'branch': branch, 'dir': '.'}, + repodir, logger=self.pool._logger) + + def _tweak_mirrorlist(self, repodir, source_repo): + for file in os.listdir(repodir): + if file.endswith('.repo'): + tweak_file(os.path.join(repodir, file), source_repo) + + +def tweak_file(path, source_repo): + """Replace mirrorlist line in repo file with baseurl pointing to source_repo.""" + with open(path, 'r') as f: + contents = f.read() + replacement = 'baseurl={}'.format(source_repo) + contents = re.sub(r'^mirrorlist=.*$', replacement, contents) + with open(path, 'w') as f: + f.write(contents) diff --git a/tests/test_ostree_phase.py b/tests/test_ostree_phase.py new file mode 100755 index 00000000..28f47d75 --- /dev/null +++ b/tests/test_ostree_phase.py @@ -0,0 +1,98 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + + +import unittest +import mock + +import os +import sys + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..')) + +from tests import helpers +from pungi.phases import ostree + + +class OSTreePhaseTest(helpers.PungiTestCase): + + @mock.patch('pungi.phases.ostree.ThreadPool') + def test_run(self, ThreadPool): + cfg = mock.Mock() + compose = helpers.DummyCompose(self.topdir, { + 'ostree': [ + ('^Everything$', {'x86_64': cfg}) + ] + }) + + pool = ThreadPool.return_value + + phase = ostree.OSTreePhase(compose) + phase.run() + + self.assertEqual(len(pool.add.call_args_list), 1) + self.assertEqual(pool.queue_put.call_args_list, + [mock.call((compose, compose.variants['Everything'], 'x86_64', cfg))]) + + @mock.patch('pungi.phases.ostree.ThreadPool') + def test_skip_without_config(self, ThreadPool): + compose = helpers.DummyCompose(self.topdir, {}) + compose.just_phases = None + compose.skip_phases = [] + phase = ostree.OSTreePhase(compose) + self.assertTrue(phase.skip()) + + +class OSTreeThreadTest(helpers.PungiTestCase): + + def _dummy_config_repo(self, scm_dict, target, logger=None): + helpers.touch(os.path.join(target, 'fedora-atomic-docker-host.json')) + helpers.touch(os.path.join(target, 'fedora-rawhide.repo')) + + @mock.patch('pungi.wrappers.scm.get_dir_from_scm') + @mock.patch('pungi.wrappers.kojiwrapper.KojiWrapper') + def test_run(self, KojiWrapper, get_dir_from_scm): + compose = helpers.DummyCompose(self.topdir, { + 'koji_profile': 'koji', + 'runroot_tag': 'rrt', + }) + pool = mock.Mock() + cfg = { + 'source_repo_from': 'Everything', + 'config_url': 'https://git.fedorahosted.org/git/fedora-atomic.git', + 'config_branch': 'f24', + 'treefile': 'fedora-atomic-docker-host.json', + 'atomic_repo': '/other/place/for/atomic' + } + get_dir_from_scm.side_effect = self._dummy_config_repo + koji = KojiWrapper.return_value + koji.run_runroot_cmd.return_value = { + 'task_id': 1234, + 'retcode': 0, + 'output': 'Foo bar\n', + } + + t = ostree.OSTreeThread(pool) + + t.process((compose, compose.variants['Everything'], 'x86_64', cfg), 1) + + self.assertEqual(get_dir_from_scm.call_args_list, + [mock.call({'scm': 'git', 'repo': 'https://git.fedorahosted.org/git/fedora-atomic.git', + 'branch': 'f24', 'dir': '.'}, + self.topdir + '/work/atomic/config_repo', logger=pool._logger)]) + self.assertEqual(koji.get_runroot_cmd.call_args_list, + [mock.call('rrt', 'x86_64', + ['pungi-make-ostree', + '--log-dir={}/logs/x86_64/atomic'.format(self.topdir), + '--treefile=fedora-atomic-docker-host.json', + '/other/place/for/atomic'], + channel=None, mounts=[self.topdir], + packages=['pungi', 'ostree', 'rpm-ostree'], + task_id=True, use_shell=True)]) + self.assertEqual(koji.run_runroot_cmd.call_args_list, + [mock.call(koji.get_runroot_cmd.return_value, + log_file=self.topdir + '/logs/x86_64/atomic/runroot.log')]) + + +if __name__ == '__main__': + unittest.main()