diff --git a/bin/pungi-koji b/bin/pungi-koji index d47f051e..32e4c447 100755 --- a/bin/pungi-koji +++ b/bin/pungi-koji @@ -430,6 +430,7 @@ def run_compose(compose, create_latest_link=True, latest_link_status=None): test_phase.stop() compose.write_status("FINISHED") + osbs_phase.request_push() latest_link = False if create_latest_link: if latest_link_status is None: diff --git a/doc/configuration.rst b/doc/configuration.rst index 1bde2bc3..dd6e3118 100644 --- a/doc/configuration.rst +++ b/doc/configuration.rst @@ -1455,12 +1455,12 @@ Example config OSBS Settings ============= -*Pungi* can build docker images in OSBS. The build is initiated through Koji +*Pungi* can build container images in OSBS. The build is initiated through Koji ``container-build`` plugin. The base image will be using RPMs from the current compose and a ``Dockerfile`` from specified Git repository. -Please note that the image is uploaded to a Docker v2 registry and not exported -into compose directory. There will be a metadata file in +Please note that the image is uploaded to a registry and not exported into +compose directory. There will be a metadata file in ``compose/metadata/osbs.json`` with details about the built images (assuming they are not scratch builds). @@ -1487,6 +1487,15 @@ they are not scratch builds). option will most likely change to list architectures that are allowed to fail. + It is possible to configure extra information about where to push the image + (unless it is a scratch build). Pungi will take any value in ``registry`` + key in the configuration and collect them across all built images. The data + will be saved into ``logs/global/osbs-registries.json`` as a mapping from + Koji NVR to the registry data. The same data is also sent to the message + bus on ``osbs-request-push`` topic once the compose finishes successfully. + Handling the message and performing the actual push is outside of scope for + Pungi. + The configuration will pass other attributes directly to the Koji task. This includes ``name``, ``version``, ``scratch`` and ``priority``. diff --git a/pungi/checks.py b/pungi/checks.py index 530e2a99..388ad445 100644 --- a/pungi/checks.py +++ b/pungi/checks.py @@ -497,6 +497,7 @@ def make_schema(): }, "gpgkey": {"type": "string"}, "git_branch": {"type": "string"}, + "registry": {"type": "object"}, }, "required": ["url", "target", "git_branch"] }, diff --git a/pungi/phases/osbs.py b/pungi/phases/osbs.py index a0e94564..96ff9515 100644 --- a/pungi/phases/osbs.py +++ b/pungi/phases/osbs.py @@ -17,6 +17,7 @@ class OSBSPhase(PhaseLoggerMixin, ConfigGuardedPhase): super(OSBSPhase, self).__init__(compose) self.pool = ThreadPool(logger=self.logger) self.pool.metadata = {} + self.pool.registries = {} def run(self): for variant in self.compose.get_variants(): @@ -34,6 +35,28 @@ class OSBSPhase(PhaseLoggerMixin, ConfigGuardedPhase): json.dump(self.pool.metadata, f, indent=4, sort_keys=True, separators=(',', ': ')) + def request_push(self): + """Store configuration data about where to push the created images and + then send the same data to message bus. + """ + if not self.pool.registries: + return + + # Write the data into a file. + registry_file = os.path.join( + self.compose.paths.log.topdir(), "osbs-registries.json" + ) + with open(registry_file, "w") as fh: + json.dump(self.pool.registries, fh) + + # Send a message with the data + if self.compose.notifier: + self.compose.notifier.send( + "osbs-request-push", + config_location=util.translate_path(self.compose, registry_file), + config=self.pool.registries, + ) + class OSBSThread(WorkerThread): def process(self, item, num): @@ -56,6 +79,7 @@ class OSBSThread(WorkerThread): gpgkey = config.pop('gpgkey', None) repos = [self._get_repo(compose, v, gpgkey=gpgkey) for v in [variant.uid] + shortcuts.force_list(config.pop('repo', []))] + registry = config.pop("registry", None) config['yum_repourls'] = repos @@ -73,7 +97,9 @@ class OSBSThread(WorkerThread): % (task_id, log_file)) scratch = config.get('scratch', False) - self._add_metadata(variant, task_id, compose, scratch) + nvr = self._add_metadata(variant, task_id, compose, scratch) + if nvr and registry: + self.pool.registries[nvr] = registry self.pool.log_info('[DONE ] %s' % msg) @@ -98,16 +124,20 @@ class OSBSThread(WorkerThread): # in same data structure as real builds. self.pool.metadata.setdefault( variant.uid, {}).setdefault('scratch', []).append(metadata) + return None + else: build_id = int(result['koji_builds'][0]) buildinfo = koji.koji_proxy.getBuild(build_id) archives = koji.koji_proxy.listArchives(build_id) + nvr = "%(name)s-%(version)s-%(release)s" % buildinfo + metadata.update({ 'name': buildinfo['name'], 'version': buildinfo['version'], 'release': buildinfo['release'], - 'nvr': '%(name)s-%(version)s-%(release)s' % buildinfo, + 'nvr': nvr, 'creation_time': buildinfo['creation_time'], }) for archive in archives: @@ -123,6 +153,7 @@ class OSBSThread(WorkerThread): metadata['name'], metadata['version'], metadata['release'], arch)) self.pool.metadata.setdefault( variant.uid, {}).setdefault(arch, []).append(data) + return nvr def _get_repo(self, compose, repo, gpgkey=None): """ diff --git a/tests/test_osbs_phase.py b/tests/test_osbs_phase.py index c6b360a1..5d8fe7f1 100644 --- a/tests/test_osbs_phase.py +++ b/tests/test_osbs_phase.py @@ -75,6 +75,29 @@ class OSBSPhaseTest(helpers.PungiTestCase): self.assertFalse(os.path.isfile(self.topdir + '/compose/metadata/osbs.json')) + @mock.patch("pungi.phases.osbs.ThreadPool") + def test_request_push(self, ThreadPool): + compose = helpers.DummyCompose(self.topdir, { + "osbs": {"^Everything$": {}} + }) + compose.just_phases = None + compose.skip_phases = [] + compose.notifier = mock.Mock() + phase = osbs.OSBSPhase(compose) + phase.start() + phase.stop() + phase.pool.registries = {"foo": "bar"} + phase.request_push() + + with open(os.path.join(self.topdir, "logs/global/osbs-registries.json")) as f: + data = json.load(f) + self.assertEqual(data, phase.pool.registries) + + self.assertEqual( + compose.notifier.call_args_list, + [], + ) + TASK_RESULT = { 'koji_builds': ['54321'], @@ -169,7 +192,7 @@ class OSBSThreadTest(helpers.PungiTestCase): def setUp(self): super(OSBSThreadTest, self).setUp() - self.pool = mock.Mock(metadata={}) + self.pool = mock.Mock(metadata={}, registries={}) self.t = osbs.OSBSThread(self.pool) self.compose = helpers.DummyCompose(self.topdir, { 'koji_profile': 'koji', @@ -331,6 +354,37 @@ class OSBSThreadTest(helpers.PungiTestCase): self._assertCorrectMetadata() self._assertRepoFile(['Server', 'Everything']) + @mock.patch("pungi.phases.osbs.kojiwrapper.KojiWrapper") + def test_run_with_registry(self, KojiWrapper): + cfg = { + "url": "git://example.com/repo?#BEEFCAFE", + "target": "f24-docker-candidate", + "git_branch": "f24-docker", + "name": "my-name", + "version": "1.0", + "repo": ["Everything", "http://pkgs.example.com/my.repo"], + "registry": {"foo": "bar"}, + } + self._setupMock(KojiWrapper) + self._assertConfigCorrect(cfg) + + self.t.process((self.compose, self.compose.variants["Server"], cfg), 1) + + options = { + "name": "my-name", + "version": "1.0", + "git_branch": "f24-docker", + "yum_repourls": [ + "http://root/work/global/tmp-Server/compose-rpms-Server-1.repo", + "http://root/work/global/tmp-Everything/compose-rpms-Everything-1.repo", + "http://pkgs.example.com/my.repo", + ] + } + self._assertCorrectCalls(options) + self._assertCorrectMetadata() + self._assertRepoFile(["Server", "Everything"]) + self.assertEqual(self.t.pool.registries, {"my-name-1.0-1": {"foo": "bar"}}) + @mock.patch('pungi.phases.osbs.kojiwrapper.KojiWrapper') def test_run_with_extra_repos_in_list(self, KojiWrapper): cfg = {