From d52171195791360b5e52f74f947e8dea9bfcc002 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lubom=C3=ADr=20Sedl=C3=A1=C5=99?= Date: Fri, 5 Apr 2019 10:23:37 +0200 Subject: [PATCH] osbs: Rework configuration for image pushes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Embedding the registry configuration into OSBS config itself is simple, but makes it impossible to reuse the same configuration for multiple different composes. A nice example is a nightly pushing images to a testing registry, and production compose building the same images but pushing to staging location. The original design requires duplication of all the configuration just because registries are different. With this option, the push information is stored in a separate option as a mapping from NVR patterns to arbitrary data. The patterns are used to match finished builds to registry. The old configuration is marked as deprecated in code and will eventually be removed. The deprecation handling in config validation does not allow emitting warnings for nested values. JIRA: COMPOSE-3394 Signed-off-by: Lubomír Sedlář --- doc/configuration.rst | 19 +++++++++++-------- pungi/checks.py | 8 ++++++++ pungi/phases/osbs.py | 18 ++++++++++++++++-- tests/test_osbs_phase.py | 33 ++++++++++++++++++++++++++++++++- 4 files changed, 67 insertions(+), 11 deletions(-) diff --git a/doc/configuration.rst b/doc/configuration.rst index 3cab7378..b6ff6339 100644 --- a/doc/configuration.rst +++ b/doc/configuration.rst @@ -1533,14 +1533,6 @@ they are not scratch builds). option will most likely change to list architectures that are allowed to fail. - It is possible to configure extra information about where to push the image - (unless it is a scratch build). Pungi will take any value in ``registry`` - key in the configuration and collect them across all built images. The data - will be saved into ``logs/global/osbs-registries.json`` as a mapping from - Koji NVR to the registry data. The same data is also sent to the message - bus on ``osbs-request-push`` topic once the compose finishes successfully. - Handling the message and performing the actual push is outside of scope for - Pungi. The configuration will pass other attributes directly to the Koji task. This includes ``name``, ``version``, ``scratch`` and ``priority``. @@ -1551,6 +1543,17 @@ they are not scratch builds). variant uid, Pungi will create the .repo file for that variant. ``gpgkey`` can be specified to enable gpgcheck in repo files for variants. +**osbs_registries** + (*dict*) -- It is possible to configure extra information about where to + push the image (unless it is a scratch build). For each finished build, + Pungi will try to match NVR against a key in this mapping (using shell-style + globbing) and take the corresponding value and collect them across all built + images. The data will be saved into ``logs/global/osbs-registries.json`` as + a mapping from Koji NVR to the registry data. The same data is also sent to + the message bus on ``osbs-request-push`` topic once the compose finishes + successfully. Handling the message and performing the actual push is outside + of scope for Pungi. + Example config -------------- diff --git a/pungi/checks.py b/pungi/checks.py index 9976d0b1..0df15dd1 100644 --- a/pungi/checks.py +++ b/pungi/checks.py @@ -1225,6 +1225,14 @@ def make_schema(): }, "additionalProperties": False, }, + "osbs_registries": { + "type": "object", + "patternProperties": { + # There are no restrictions on what the value can be. + ".+": {} + }, + "additionalProperties": False, + }, "extra_files": _variant_arch_mapping({ "type": "array", diff --git a/pungi/phases/osbs.py b/pungi/phases/osbs.py index 88a9e110..7567cb20 100644 --- a/pungi/phases/osbs.py +++ b/pungi/phases/osbs.py @@ -1,5 +1,6 @@ # -*- coding: utf-8 -*- +import fnmatch import json import os from kobo.threads import ThreadPool, WorkerThread @@ -58,6 +59,16 @@ class OSBSPhase(PhaseLoggerMixin, ConfigGuardedPhase): ) +def get_registry(compose, nvr, fallback=None): + """Get a configured registry for the image from config matching given NVR. + If not present, return fallback value. + """ + for pattern, registry in compose.conf.get("osbs_registries", {}).items(): + if fnmatch.fnmatch(nvr, pattern): + return registry + return fallback + + class OSBSThread(WorkerThread): def process(self, item, num): compose, variant, config = item @@ -79,6 +90,7 @@ class OSBSThread(WorkerThread): gpgkey = config.pop('gpgkey', None) repos = [self._get_repo(compose, v, gpgkey=gpgkey) for v in [variant.uid] + shortcuts.force_list(config.pop('repo', []))] + # Deprecated in 4.1.36 registry = config.pop("registry", None) config['yum_repourls'] = repos @@ -98,8 +110,10 @@ class OSBSThread(WorkerThread): scratch = config.get('scratch', False) nvr = self._add_metadata(variant, task_id, compose, scratch) - if nvr and registry: - self.pool.registries[nvr] = registry + if nvr: + registry = get_registry(compose, nvr, registry) + if registry: + self.pool.registries[nvr] = registry self.pool.log_info('[DONE ] %s' % msg) diff --git a/tests/test_osbs_phase.py b/tests/test_osbs_phase.py index 5a7a9cd8..8670cf91 100644 --- a/tests/test_osbs_phase.py +++ b/tests/test_osbs_phase.py @@ -360,7 +360,7 @@ class OSBSThreadTest(helpers.PungiTestCase): self.assertIn("baseurl=http://example.com/repo\n", f) @mock.patch("pungi.phases.osbs.kojiwrapper.KojiWrapper") - def test_run_with_registry(self, KojiWrapper): + def test_run_with_deprecated_registry(self, KojiWrapper): cfg = { "url": "git://example.com/repo?#BEEFCAFE", "target": "f24-docker-candidate", @@ -390,6 +390,37 @@ class OSBSThreadTest(helpers.PungiTestCase): self._assertRepoFile(["Server", "Everything"]) self.assertEqual(self.t.pool.registries, {"my-name-1.0-1": {"foo": "bar"}}) + @mock.patch("pungi.phases.osbs.kojiwrapper.KojiWrapper") + def test_run_with_registry(self, KojiWrapper): + cfg = { + "url": "git://example.com/repo?#BEEFCAFE", + "target": "f24-docker-candidate", + "git_branch": "f24-docker", + "name": "my-name", + "version": "1.0", + "repo": ["Everything", "http://pkgs.example.com/my.repo"], + } + self.compose.conf["osbs_registries"] = {"my-name-1.0-*": [{"foo": "bar"}]} + self._setupMock(KojiWrapper) + self._assertConfigCorrect(cfg) + + self.t.process((self.compose, self.compose.variants["Server"], cfg), 1) + + options = { + "name": "my-name", + "version": "1.0", + "git_branch": "f24-docker", + "yum_repourls": [ + "http://root/work/global/tmp-Server/compose-rpms-Server-1.repo", + "http://root/work/global/tmp-Everything/compose-rpms-Everything-1.repo", + "http://pkgs.example.com/my.repo", + ] + } + self._assertCorrectCalls(options) + self._assertCorrectMetadata() + self._assertRepoFile(["Server", "Everything"]) + self.assertEqual(self.t.pool.registries, {"my-name-1.0-1": [{"foo": "bar"}]}) + @mock.patch('pungi.phases.osbs.kojiwrapper.KojiWrapper') def test_run_with_extra_repos_in_list(self, KojiWrapper): cfg = {