diff --git a/doc/configuration.rst b/doc/configuration.rst index 163a1528..e266c1a0 100644 --- a/doc/configuration.rst +++ b/doc/configuration.rst @@ -1244,7 +1244,9 @@ they are not scratch builds). This includes ``name``, ``version``, ``scratch`` and ``priority``. A value for ``yum_repourls`` will be created automatically and point at a - repository in the current compose. + repository in the current compose. You can add extra repositories with + ``repo`` key having a list of urls pointing to ``.repo`` files or + ``repo_from`` as a list of variants in current compose. Example config diff --git a/pungi/checks.py b/pungi/checks.py index 90544f9e..3c9de33a 100644 --- a/pungi/checks.py +++ b/pungi/checks.py @@ -828,6 +828,8 @@ def _make_schema(): "version": {"type": "string"}, "scratch": {"type": "boolean"}, "priority": {"type": "number"}, + "repo": {"$ref": "#/definitions/strings"}, + "repo_from": {"$ref": "#/definitions/strings"}, }, "required": ["url", "target"] } diff --git a/pungi/phases/osbs.py b/pungi/phases/osbs.py index 6b38a54d..cc24b6a1 100644 --- a/pungi/phases/osbs.py +++ b/pungi/phases/osbs.py @@ -3,6 +3,7 @@ import json import os from kobo.threads import ThreadPool, WorkerThread +from kobo import shortcuts from .base import ConfigGuardedPhase, PhaseLoggerMixin from .. import util @@ -50,15 +51,15 @@ class OSBSThread(WorkerThread): koji.login() # Start task - try: - source = util.resolve_git_url(config.pop('url')) - target = config.pop('target') - except KeyError as exc: - raise RuntimeError('OSBS: missing config key %s for %s' - % (exc, variant.uid)) + source = util.resolve_git_url(config.pop('url')) + target = config.pop('target') priority = config.pop('priority', None) + repos = shortcuts.force_list(config.pop('repo', [])) + compose_repos = [self._get_repo(compose, v) + for v in [variant.uid] + shortcuts.force_list( + config.pop('repo_from', []))] - config['yum_repourls'] = [self._get_repo(compose, variant)] + config['yum_repourls'] = compose_repos + repos task_id = koji.koji_proxy.buildContainer(source, target, config, priority=priority) @@ -106,11 +107,17 @@ class OSBSThread(WorkerThread): self.pool.metadata.setdefault( variant.uid, {}).setdefault(arch, []).append(data) - def _get_repo(self, compose, variant): + def _get_repo(self, compose, variant_uid): """ Write a .repo file pointing to current variant and return URL to the file. """ + try: + variant = compose.all_variants[variant_uid] + except KeyError: + raise RuntimeError( + 'There is no variant %s to get repo from to pass to OSBS.' + % (variant_uid)) os_tree = compose.paths.compose.os_tree('$basearch', variant, create_dir=False) repo_file = os.path.join(compose.paths.work.tmp_dir(None, variant), diff --git a/tests/test_osbs_phase.py b/tests/test_osbs_phase.py index 39cfee22..1f96efdd 100644 --- a/tests/test_osbs_phase.py +++ b/tests/test_osbs_phase.py @@ -8,12 +8,14 @@ except ImportError: import mock import json +import copy import os import sys sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..')) from tests import helpers +from pungi import checks from pungi.phases import osbs @@ -192,10 +194,28 @@ class OSBSThreadTest(helpers.PungiTestCase): mock.call.koji_proxy.getBuild(54321), mock.call.koji_proxy.listArchives(54321)]) - def _assertRepoFile(self): - with open(self.topdir + '/work/global/tmp-Server/compose-rpms-1.repo') as f: - lines = f.read().split('\n') - self.assertIn('baseurl=http://root/compose/Server/$baseurl/os', lines) + def _assertRepoFile(self, variants=None): + variants = variants or ['Server'] + for variant in variants: + with open(self.topdir + '/work/global/tmp-%s/compose-rpms-1.repo' % variant) as f: + lines = f.read().split('\n') + self.assertIn('baseurl=http://root/compose/%s/$basearch/os' % variant, lines) + + def _assertConfigCorrect(self, cfg): + config = copy.deepcopy(self.compose.conf) + config['osbs'] = { + '^Server$': cfg + } + self.assertEqual(([], []), checks.validate(config)) + + def _assertConfigMissing(self, cfg, key): + config = copy.deepcopy(self.compose.conf) + config['osbs'] = { + '^Server$': cfg + } + self.assertEqual( + (['Failed validation in osbs.^Server$: \'%s\' is a required property' % key], []), + checks.validate(config)) @mock.patch('pungi.util.resolve_git_url') @mock.patch('pungi.phases.osbs.kojiwrapper.KojiWrapper') @@ -205,11 +225,13 @@ class OSBSThreadTest(helpers.PungiTestCase): 'target': 'f24-docker-candidate', } self._setupMock(KojiWrapper, resolve_git_url) + self._assertConfigCorrect(cfg) self.t.process((self.compose, self.compose.variants['Server'], cfg), 1) self._assertCorrectCalls({}) self._assertCorrectMetadata() + self._assertRepoFile() @mock.patch('pungi.util.resolve_git_url') @mock.patch('pungi.phases.osbs.kojiwrapper.KojiWrapper') @@ -220,11 +242,13 @@ class OSBSThreadTest(helpers.PungiTestCase): 'failable': ['*'] } self._setupMock(KojiWrapper, resolve_git_url) + self._assertConfigCorrect(cfg) self.t.process((self.compose, self.compose.variants['Server'], cfg), 1) self._assertCorrectCalls({}) self._assertCorrectMetadata() + self._assertRepoFile() @mock.patch('pungi.util.resolve_git_url') @mock.patch('pungi.phases.osbs.kojiwrapper.KojiWrapper') @@ -236,39 +260,104 @@ class OSBSThreadTest(helpers.PungiTestCase): 'version': '1.0', } self._setupMock(KojiWrapper, resolve_git_url) + self._assertConfigCorrect(cfg) self.t.process((self.compose, self.compose.variants['Server'], cfg), 1) self._assertCorrectCalls({'name': 'my-name', 'version': '1.0'}) self._assertCorrectMetadata() + self._assertRepoFile() @mock.patch('pungi.util.resolve_git_url') @mock.patch('pungi.phases.osbs.kojiwrapper.KojiWrapper') - def test_run_with_missing_url(self, KojiWrapper, resolve_git_url): + def test_run_with_extra_repos(self, KojiWrapper, resolve_git_url): + cfg = { + 'url': 'git://example.com/repo?#HEAD', + 'target': 'f24-docker-candidate', + 'name': 'my-name', + 'version': '1.0', + 'repo': 'http://pkgs.example.com/my.repo', + 'repo_from': 'Everything', + } + self._setupMock(KojiWrapper, resolve_git_url) + self._assertConfigCorrect(cfg) + + self.t.process((self.compose, self.compose.variants['Server'], cfg), 1) + + options = { + 'name': 'my-name', + 'version': '1.0', + 'yum_repourls': [ + 'http://root/work/global/tmp-Server/compose-rpms-1.repo', + 'http://root/work/global/tmp-Everything/compose-rpms-1.repo', + 'http://pkgs.example.com/my.repo', + ] + } + self._assertCorrectCalls(options) + self._assertCorrectMetadata() + self._assertRepoFile(['Server', 'Everything']) + + @mock.patch('pungi.util.resolve_git_url') + @mock.patch('pungi.phases.osbs.kojiwrapper.KojiWrapper') + def test_run_with_extra_repos_in_list(self, KojiWrapper, resolve_git_url): + cfg = { + 'url': 'git://example.com/repo?#HEAD', + 'target': 'f24-docker-candidate', + 'name': 'my-name', + 'version': '1.0', + 'repo': ['http://pkgs.example.com/my.repo'], + 'repo_from': ['Everything', 'Client'], + } + self._assertConfigCorrect(cfg) + self._setupMock(KojiWrapper, resolve_git_url) + + self.t.process((self.compose, self.compose.variants['Server'], cfg), 1) + + options = { + 'name': 'my-name', + 'version': '1.0', + 'yum_repourls': [ + 'http://root/work/global/tmp-Server/compose-rpms-1.repo', + 'http://root/work/global/tmp-Everything/compose-rpms-1.repo', + 'http://root/work/global/tmp-Client/compose-rpms-1.repo', + 'http://pkgs.example.com/my.repo', + ] + } + self._assertCorrectCalls(options) + self._assertCorrectMetadata() + self._assertRepoFile(['Server', 'Everything', 'Client']) + + @mock.patch('pungi.util.resolve_git_url') + @mock.patch('pungi.phases.osbs.kojiwrapper.KojiWrapper') + def test_run_with_extra_repos_missing_variant(self, KojiWrapper, resolve_git_url): + cfg = { + 'url': 'git://example.com/repo?#HEAD', + 'target': 'f24-docker-candidate', + 'name': 'my-name', + 'version': '1.0', + 'repo_from': 'Gold', + } + self._assertConfigCorrect(cfg) + self._setupMock(KojiWrapper, resolve_git_url) + + with self.assertRaises(RuntimeError) as ctx: + self.t.process((self.compose, self.compose.variants['Server'], cfg), 1) + + self.assertIn('no variant Gold', str(ctx.exception)) + + def test_run_with_missing_url(self): cfg = { 'target': 'f24-docker-candidate', 'name': 'my-name', } - self._setupMock(KojiWrapper, resolve_git_url) + self._assertConfigMissing(cfg, 'url') - with self.assertRaises(RuntimeError) as ctx: - self.t.process((self.compose, self.compose.variants['Server'], cfg), 1) - - self.assertIn("missing config key 'url' for Server", str(ctx.exception)) - - @mock.patch('pungi.util.resolve_git_url') - @mock.patch('pungi.phases.osbs.kojiwrapper.KojiWrapper') - def test_run_with_missing_target(self, KojiWrapper, resolve_git_url): + def test_run_with_missing_target(self): cfg = { 'url': 'git://example.com/repo?#HEAD', 'name': 'my-name', } - self._setupMock(KojiWrapper, resolve_git_url) - - with self.assertRaises(RuntimeError) as ctx: - self.t.process((self.compose, self.compose.variants['Server'], cfg), 1) - - self.assertIn("missing config key 'target' for Server", str(ctx.exception)) + self._assertConfigMissing(cfg, 'target') @mock.patch('pungi.util.resolve_git_url') @mock.patch('pungi.phases.osbs.kojiwrapper.KojiWrapper') @@ -277,6 +366,7 @@ class OSBSThreadTest(helpers.PungiTestCase): 'url': 'git://example.com/repo?#HEAD', 'target': 'fedora-24-docker-candidate', } + self._assertConfigCorrect(cfg) self._setupMock(KojiWrapper, resolve_git_url) self.wrapper.watch_task.return_value = 1 @@ -293,6 +383,7 @@ class OSBSThreadTest(helpers.PungiTestCase): 'target': 'fedora-24-docker-candidate', 'failable': ['*'] } + self._assertConfigCorrect(cfg) self._setupMock(KojiWrapper, resolve_git_url) self.wrapper.watch_task.return_value = 1 @@ -306,6 +397,7 @@ class OSBSThreadTest(helpers.PungiTestCase): 'target': 'fedora-24-docker-candidate', 'scratch': True, } + self._assertConfigCorrect(cfg) self._setupMock(KojiWrapper, resolve_git_url) self.t.process((self.compose, self.compose.variants['Server'], cfg), 1)