From 814bf4484b046b807d76a0e1e3fdf406a63c308e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lubom=C3=ADr=20Sedl=C3=A1=C5=99?= Date: Wed, 4 Jan 2017 09:15:27 +0100 Subject: [PATCH 1/3] osbs: Enable specifying extra repos MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The same way live_media and image_build accept additional external repos or variants list, there is now a `repo` and `repo_from` configuration key to add these. Fixes: #486 Signed-off-by: Lubomír Sedlář --- doc/configuration.rst | 4 ++- pungi/checks.py | 2 ++ pungi/phases/osbs.py | 15 +++++++-- tests/test_osbs_phase.py | 72 ++++++++++++++++++++++++++++++++++++++++ 4 files changed, 90 insertions(+), 3 deletions(-) diff --git a/doc/configuration.rst b/doc/configuration.rst index 62d73612..6c08b8b5 100644 --- a/doc/configuration.rst +++ b/doc/configuration.rst @@ -1246,7 +1246,9 @@ they are not scratch builds). This includes ``name``, ``version``, ``scratch`` and ``priority``. A value for ``yum_repourls`` will be created automatically and point at a - repository in the current compose. + repository in the current compose. You can add extra repositories with + ``repo`` key having a list of urls pointing to ``.repo`` files or + ``repo_from`` as a list of variants in current compose. Example config diff --git a/pungi/checks.py b/pungi/checks.py index 90544f9e..3c9de33a 100644 --- a/pungi/checks.py +++ b/pungi/checks.py @@ -828,6 +828,8 @@ def _make_schema(): "version": {"type": "string"}, "scratch": {"type": "boolean"}, "priority": {"type": "number"}, + "repo": {"$ref": "#/definitions/strings"}, + "repo_from": {"$ref": "#/definitions/strings"}, }, "required": ["url", "target"] } diff --git a/pungi/phases/osbs.py b/pungi/phases/osbs.py index 6b38a54d..11ed9511 100644 --- a/pungi/phases/osbs.py +++ b/pungi/phases/osbs.py @@ -3,6 +3,7 @@ import json import os from kobo.threads import ThreadPool, WorkerThread +from kobo import shortcuts from .base import ConfigGuardedPhase, PhaseLoggerMixin from .. import util @@ -57,8 +58,12 @@ class OSBSThread(WorkerThread): raise RuntimeError('OSBS: missing config key %s for %s' % (exc, variant.uid)) priority = config.pop('priority', None) + repos = shortcuts.force_list(config.pop('repo', [])) + compose_repos = [self._get_repo(compose, v) + for v in [variant.uid] + shortcuts.force_list( + config.pop('repo_from', []))] - config['yum_repourls'] = [self._get_repo(compose, variant)] + config['yum_repourls'] = compose_repos + repos task_id = koji.koji_proxy.buildContainer(source, target, config, priority=priority) @@ -106,11 +111,17 @@ class OSBSThread(WorkerThread): self.pool.metadata.setdefault( variant.uid, {}).setdefault(arch, []).append(data) - def _get_repo(self, compose, variant): + def _get_repo(self, compose, variant_uid): """ Write a .repo file pointing to current variant and return URL to the file. """ + try: + variant = compose.all_variants[variant_uid] + except KeyError: + raise RuntimeError( + 'There is no variant %s to get repo from to pass to OSBS.' + % (variant_uid)) os_tree = compose.paths.compose.os_tree('$basearch', variant, create_dir=False) repo_file = os.path.join(compose.paths.work.tmp_dir(None, variant), diff --git a/tests/test_osbs_phase.py b/tests/test_osbs_phase.py index 39cfee22..007ea3eb 100644 --- a/tests/test_osbs_phase.py +++ b/tests/test_osbs_phase.py @@ -242,6 +242,78 @@ class OSBSThreadTest(helpers.PungiTestCase): self._assertCorrectCalls({'name': 'my-name', 'version': '1.0'}) self._assertCorrectMetadata() + @mock.patch('pungi.util.resolve_git_url') + @mock.patch('pungi.phases.osbs.kojiwrapper.KojiWrapper') + def test_run_with_extra_repos(self, KojiWrapper, resolve_git_url): + cfg = { + 'url': 'git://example.com/repo?#HEAD', + 'target': 'f24-docker-candidate', + 'name': 'my-name', + 'version': '1.0', + 'repo': 'http://pkgs.example.com/my.repo', + 'repo_from': 'Everything', + } + self._setupMock(KojiWrapper, resolve_git_url) + + self.t.process((self.compose, self.compose.variants['Server'], cfg), 1) + + options = { + 'name': 'my-name', + 'version': '1.0', + 'yum_repourls': [ + 'http://root/work/global/tmp-Server/compose-rpms-1.repo', + 'http://root/work/global/tmp-Everything/compose-rpms-1.repo', + 'http://pkgs.example.com/my.repo', + ] + } + self._assertCorrectCalls(options) + self._assertCorrectMetadata() + + @mock.patch('pungi.util.resolve_git_url') + @mock.patch('pungi.phases.osbs.kojiwrapper.KojiWrapper') + def test_run_with_extra_repos_in_list(self, KojiWrapper, resolve_git_url): + cfg = { + 'url': 'git://example.com/repo?#HEAD', + 'target': 'f24-docker-candidate', + 'name': 'my-name', + 'version': '1.0', + 'repo': ['http://pkgs.example.com/my.repo'], + 'repo_from': ['Everything', 'Client'], + } + self._setupMock(KojiWrapper, resolve_git_url) + + self.t.process((self.compose, self.compose.variants['Server'], cfg), 1) + + options = { + 'name': 'my-name', + 'version': '1.0', + 'yum_repourls': [ + 'http://root/work/global/tmp-Server/compose-rpms-1.repo', + 'http://root/work/global/tmp-Everything/compose-rpms-1.repo', + 'http://root/work/global/tmp-Client/compose-rpms-1.repo', + 'http://pkgs.example.com/my.repo', + ] + } + self._assertCorrectCalls(options) + self._assertCorrectMetadata() + + @mock.patch('pungi.util.resolve_git_url') + @mock.patch('pungi.phases.osbs.kojiwrapper.KojiWrapper') + def test_run_with_extra_repos_missing_variant(self, KojiWrapper, resolve_git_url): + cfg = { + 'url': 'git://example.com/repo?#HEAD', + 'target': 'f24-docker-candidate', + 'name': 'my-name', + 'version': '1.0', + 'repo_from': 'Gold', + } + self._setupMock(KojiWrapper, resolve_git_url) + + with self.assertRaises(RuntimeError) as ctx: + self.t.process((self.compose, self.compose.variants['Server'], cfg), 1) + + self.assertIn('no variant Gold', str(ctx.exception)) + @mock.patch('pungi.util.resolve_git_url') @mock.patch('pungi.phases.osbs.kojiwrapper.KojiWrapper') def test_run_with_missing_url(self, KojiWrapper, resolve_git_url): From 61a4c43db0de3841255962e666e76246d8d0750e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lubom=C3=ADr=20Sedl=C3=A1=C5=99?= Date: Wed, 4 Jan 2017 09:19:25 +0100 Subject: [PATCH 2/3] osbs: Verify the .repo files contain correct URL MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Lubomír Sedlář --- tests/test_osbs_phase.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/tests/test_osbs_phase.py b/tests/test_osbs_phase.py index 007ea3eb..72a7ed5c 100644 --- a/tests/test_osbs_phase.py +++ b/tests/test_osbs_phase.py @@ -192,10 +192,12 @@ class OSBSThreadTest(helpers.PungiTestCase): mock.call.koji_proxy.getBuild(54321), mock.call.koji_proxy.listArchives(54321)]) - def _assertRepoFile(self): - with open(self.topdir + '/work/global/tmp-Server/compose-rpms-1.repo') as f: - lines = f.read().split('\n') - self.assertIn('baseurl=http://root/compose/Server/$baseurl/os', lines) + def _assertRepoFile(self, variants=None): + variants = variants or ['Server'] + for variant in variants: + with open(self.topdir + '/work/global/tmp-%s/compose-rpms-1.repo' % variant) as f: + lines = f.read().split('\n') + self.assertIn('baseurl=http://root/compose/%s/$basearch/os' % variant, lines) @mock.patch('pungi.util.resolve_git_url') @mock.patch('pungi.phases.osbs.kojiwrapper.KojiWrapper') @@ -210,6 +212,7 @@ class OSBSThreadTest(helpers.PungiTestCase): self._assertCorrectCalls({}) self._assertCorrectMetadata() + self._assertRepoFile() @mock.patch('pungi.util.resolve_git_url') @mock.patch('pungi.phases.osbs.kojiwrapper.KojiWrapper') @@ -225,6 +228,7 @@ class OSBSThreadTest(helpers.PungiTestCase): self._assertCorrectCalls({}) self._assertCorrectMetadata() + self._assertRepoFile() @mock.patch('pungi.util.resolve_git_url') @mock.patch('pungi.phases.osbs.kojiwrapper.KojiWrapper') @@ -241,6 +245,7 @@ class OSBSThreadTest(helpers.PungiTestCase): self._assertCorrectCalls({'name': 'my-name', 'version': '1.0'}) self._assertCorrectMetadata() + self._assertRepoFile() @mock.patch('pungi.util.resolve_git_url') @mock.patch('pungi.phases.osbs.kojiwrapper.KojiWrapper') @@ -268,6 +273,7 @@ class OSBSThreadTest(helpers.PungiTestCase): } self._assertCorrectCalls(options) self._assertCorrectMetadata() + self._assertRepoFile(['Server', 'Everything']) @mock.patch('pungi.util.resolve_git_url') @mock.patch('pungi.phases.osbs.kojiwrapper.KojiWrapper') @@ -296,6 +302,7 @@ class OSBSThreadTest(helpers.PungiTestCase): } self._assertCorrectCalls(options) self._assertCorrectMetadata() + self._assertRepoFile(['Server', 'Everything', 'Client']) @mock.patch('pungi.util.resolve_git_url') @mock.patch('pungi.phases.osbs.kojiwrapper.KojiWrapper') From e7c8b2affdf8ac33c3223dfdb377e17a02982f73 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lubom=C3=ADr=20Sedl=C3=A1=C5=99?= Date: Wed, 4 Jan 2017 09:35:31 +0100 Subject: [PATCH 3/3] osbs: Validate config in tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This makes sure the test configurations will be accepted in real usage. It also enables us to remove some manual error checking that will be performed by validator. Signed-off-by: Lubomír Sedlář --- pungi/phases/osbs.py | 8 ++----- tests/test_osbs_phase.py | 49 +++++++++++++++++++++++++--------------- 2 files changed, 33 insertions(+), 24 deletions(-) diff --git a/pungi/phases/osbs.py b/pungi/phases/osbs.py index 11ed9511..cc24b6a1 100644 --- a/pungi/phases/osbs.py +++ b/pungi/phases/osbs.py @@ -51,12 +51,8 @@ class OSBSThread(WorkerThread): koji.login() # Start task - try: - source = util.resolve_git_url(config.pop('url')) - target = config.pop('target') - except KeyError as exc: - raise RuntimeError('OSBS: missing config key %s for %s' - % (exc, variant.uid)) + source = util.resolve_git_url(config.pop('url')) + target = config.pop('target') priority = config.pop('priority', None) repos = shortcuts.force_list(config.pop('repo', [])) compose_repos = [self._get_repo(compose, v) diff --git a/tests/test_osbs_phase.py b/tests/test_osbs_phase.py index 72a7ed5c..1f96efdd 100644 --- a/tests/test_osbs_phase.py +++ b/tests/test_osbs_phase.py @@ -8,12 +8,14 @@ except ImportError: import mock import json +import copy import os import sys sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..')) from tests import helpers +from pungi import checks from pungi.phases import osbs @@ -199,6 +201,22 @@ class OSBSThreadTest(helpers.PungiTestCase): lines = f.read().split('\n') self.assertIn('baseurl=http://root/compose/%s/$basearch/os' % variant, lines) + def _assertConfigCorrect(self, cfg): + config = copy.deepcopy(self.compose.conf) + config['osbs'] = { + '^Server$': cfg + } + self.assertEqual(([], []), checks.validate(config)) + + def _assertConfigMissing(self, cfg, key): + config = copy.deepcopy(self.compose.conf) + config['osbs'] = { + '^Server$': cfg + } + self.assertEqual( + (['Failed validation in osbs.^Server$: \'%s\' is a required property' % key], []), + checks.validate(config)) + @mock.patch('pungi.util.resolve_git_url') @mock.patch('pungi.phases.osbs.kojiwrapper.KojiWrapper') def test_minimal_run(self, KojiWrapper, resolve_git_url): @@ -207,6 +225,7 @@ class OSBSThreadTest(helpers.PungiTestCase): 'target': 'f24-docker-candidate', } self._setupMock(KojiWrapper, resolve_git_url) + self._assertConfigCorrect(cfg) self.t.process((self.compose, self.compose.variants['Server'], cfg), 1) @@ -223,6 +242,7 @@ class OSBSThreadTest(helpers.PungiTestCase): 'failable': ['*'] } self._setupMock(KojiWrapper, resolve_git_url) + self._assertConfigCorrect(cfg) self.t.process((self.compose, self.compose.variants['Server'], cfg), 1) @@ -240,6 +260,7 @@ class OSBSThreadTest(helpers.PungiTestCase): 'version': '1.0', } self._setupMock(KojiWrapper, resolve_git_url) + self._assertConfigCorrect(cfg) self.t.process((self.compose, self.compose.variants['Server'], cfg), 1) @@ -259,6 +280,7 @@ class OSBSThreadTest(helpers.PungiTestCase): 'repo_from': 'Everything', } self._setupMock(KojiWrapper, resolve_git_url) + self._assertConfigCorrect(cfg) self.t.process((self.compose, self.compose.variants['Server'], cfg), 1) @@ -286,6 +308,7 @@ class OSBSThreadTest(helpers.PungiTestCase): 'repo': ['http://pkgs.example.com/my.repo'], 'repo_from': ['Everything', 'Client'], } + self._assertConfigCorrect(cfg) self._setupMock(KojiWrapper, resolve_git_url) self.t.process((self.compose, self.compose.variants['Server'], cfg), 1) @@ -314,6 +337,7 @@ class OSBSThreadTest(helpers.PungiTestCase): 'version': '1.0', 'repo_from': 'Gold', } + self._assertConfigCorrect(cfg) self._setupMock(KojiWrapper, resolve_git_url) with self.assertRaises(RuntimeError) as ctx: @@ -321,33 +345,19 @@ class OSBSThreadTest(helpers.PungiTestCase): self.assertIn('no variant Gold', str(ctx.exception)) - @mock.patch('pungi.util.resolve_git_url') - @mock.patch('pungi.phases.osbs.kojiwrapper.KojiWrapper') - def test_run_with_missing_url(self, KojiWrapper, resolve_git_url): + def test_run_with_missing_url(self): cfg = { 'target': 'f24-docker-candidate', 'name': 'my-name', } - self._setupMock(KojiWrapper, resolve_git_url) + self._assertConfigMissing(cfg, 'url') - with self.assertRaises(RuntimeError) as ctx: - self.t.process((self.compose, self.compose.variants['Server'], cfg), 1) - - self.assertIn("missing config key 'url' for Server", str(ctx.exception)) - - @mock.patch('pungi.util.resolve_git_url') - @mock.patch('pungi.phases.osbs.kojiwrapper.KojiWrapper') - def test_run_with_missing_target(self, KojiWrapper, resolve_git_url): + def test_run_with_missing_target(self): cfg = { 'url': 'git://example.com/repo?#HEAD', 'name': 'my-name', } - self._setupMock(KojiWrapper, resolve_git_url) - - with self.assertRaises(RuntimeError) as ctx: - self.t.process((self.compose, self.compose.variants['Server'], cfg), 1) - - self.assertIn("missing config key 'target' for Server", str(ctx.exception)) + self._assertConfigMissing(cfg, 'target') @mock.patch('pungi.util.resolve_git_url') @mock.patch('pungi.phases.osbs.kojiwrapper.KojiWrapper') @@ -356,6 +366,7 @@ class OSBSThreadTest(helpers.PungiTestCase): 'url': 'git://example.com/repo?#HEAD', 'target': 'fedora-24-docker-candidate', } + self._assertConfigCorrect(cfg) self._setupMock(KojiWrapper, resolve_git_url) self.wrapper.watch_task.return_value = 1 @@ -372,6 +383,7 @@ class OSBSThreadTest(helpers.PungiTestCase): 'target': 'fedora-24-docker-candidate', 'failable': ['*'] } + self._assertConfigCorrect(cfg) self._setupMock(KojiWrapper, resolve_git_url) self.wrapper.watch_task.return_value = 1 @@ -385,6 +397,7 @@ class OSBSThreadTest(helpers.PungiTestCase): 'target': 'fedora-24-docker-candidate', 'scratch': True, } + self._assertConfigCorrect(cfg) self._setupMock(KojiWrapper, resolve_git_url) self.t.process((self.compose, self.compose.variants['Server'], cfg), 1)