From 36373479db874ec663f6c4dbb835990253dc2759 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lubom=C3=ADr=20Sedl=C3=A1=C5=99?= Date: Mon, 8 Feb 2021 11:42:07 +0100 Subject: [PATCH] Move container metadata into compose object MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Rather than tracking this directly in OSBS phase, move this into Compose object, which will allow access to this from multiple phases. Signed-off-by: Lubomír Sedlář --- pungi/compose.py | 15 +++++++++++++++ pungi/phases/osbs.py | 14 ++------------ pungi/scripts/pungi_koji.py | 2 +- tests/helpers.py | 1 + tests/test_compose.py | 24 ++++++++++++++++++++++++ tests/test_osbs_phase.py | 32 ++------------------------------ 6 files changed, 45 insertions(+), 43 deletions(-) diff --git a/pungi/compose.py b/pungi/compose.py index 2121a69f..ec9c5640 100644 --- a/pungi/compose.py +++ b/pungi/compose.py @@ -284,6 +284,8 @@ class Compose(kobo.log.LoggingBase): self.im.compose.respin = self.compose_respin self.im.metadata_path = self.paths.compose.metadata() + self.containers_metadata = {} + # Stores list of deliverables that failed, but did not abort the # compose. # {deliverable: [(Variant.uid, arch, subvariant)]} @@ -575,6 +577,19 @@ class Compose(kobo.log.LoggingBase): path = os.path.join(self.paths.work.tmp_dir(arch=arch, variant=variant)) return tempfile.mkdtemp(suffix=suffix, prefix=prefix, dir=path) + def dump_containers_metadata(self): + """Create a file with container metadata if there are any containers.""" + if not self.containers_metadata: + return + with open(self.paths.compose.metadata("osbs.json"), "w") as f: + json.dump( + self.containers_metadata, + f, + indent=4, + sort_keys=True, + separators=(",", ": "), + ) + def get_ordered_variant_uids(compose): if not hasattr(compose, "_ordered_variant_uids"): diff --git a/pungi/phases/osbs.py b/pungi/phases/osbs.py index 148b3c3e..7ad2abc8 100644 --- a/pungi/phases/osbs.py +++ b/pungi/phases/osbs.py @@ -17,7 +17,6 @@ class OSBSPhase(PhaseLoggerMixin, ConfigGuardedPhase): def __init__(self, compose): super(OSBSPhase, self).__init__(compose) self.pool = ThreadPool(logger=self.logger) - self.pool.metadata = {} self.pool.registries = {} def run(self): @@ -28,15 +27,6 @@ class OSBSPhase(PhaseLoggerMixin, ConfigGuardedPhase): self.pool.start() - def dump_metadata(self): - """Create a file with image metadata if the phase actually ran.""" - if self._skipped: - return - with open(self.compose.paths.compose.metadata("osbs.json"), "w") as f: - json.dump( - self.pool.metadata, f, indent=4, sort_keys=True, separators=(",", ": ") - ) - def request_push(self): """Store configuration data about where to push the created images and then send the same data to message bus. @@ -146,7 +136,7 @@ class OSBSThread(WorkerThread): metadata.update({"repositories": result["repositories"]}) # add a fake arch of 'scratch', so we can construct the metadata # in same data structure as real builds. - self.pool.metadata.setdefault(variant.uid, {}).setdefault( + compose.containers_metadata.setdefault(variant.uid, {}).setdefault( "scratch", [] ).append(metadata) return None @@ -180,7 +170,7 @@ class OSBSThread(WorkerThread): "Created Docker base image %s-%s-%s.%s" % (metadata["name"], metadata["version"], metadata["release"], arch) ) - self.pool.metadata.setdefault(variant.uid, {}).setdefault( + compose.containers_metadata.setdefault(variant.uid, {}).setdefault( arch, [] ).append(data) return nvr diff --git a/pungi/scripts/pungi_koji.py b/pungi/scripts/pungi_koji.py index 1e631718..d0e53bb8 100644 --- a/pungi/scripts/pungi_koji.py +++ b/pungi/scripts/pungi_koji.py @@ -538,7 +538,7 @@ def run_compose( and osbuild_phase.skip() ): compose.im.dump(compose.paths.compose.metadata("images.json")) - osbs_phase.dump_metadata() + compose.dump_containers_metadata() test_phase.start() test_phase.stop() diff --git a/tests/helpers.py b/tests/helpers.py index a3969953..3fdd1eae 100644 --- a/tests/helpers.py +++ b/tests/helpers.py @@ -226,6 +226,7 @@ class DummyCompose(object): self.require_deliverable = mock.Mock() self.should_create_yum_database = True self.cache_region = None + self.containers_metadata = {} def setup_optional(self): self.all_variants["Server-optional"] = MockVariant( diff --git a/tests/test_compose.py b/tests/test_compose.py index 85590273..628973bd 100644 --- a/tests/test_compose.py +++ b/tests/test_compose.py @@ -753,3 +753,27 @@ class StatusTest(unittest.TestCase): self.compose.conf["gather_backend"] = "yum" self.compose.conf["createrepo_database"] = False self.assertFalse(self.compose.should_create_yum_database) + + +class DumpContainerMetadataTest(unittest.TestCase): + def setUp(self): + self.tmp_dir = tempfile.mkdtemp() + with mock.patch("pungi.compose.ComposeInfo"): + self.compose = Compose({}, self.tmp_dir) + + def tearDown(self): + shutil.rmtree(self.tmp_dir) + + def test_dump_metadata(self): + metadata = {"Server": {"x86_64": "Metadata"}} + self.compose.containers_metadata = metadata + self.compose.dump_containers_metadata() + + with open(self.tmp_dir + "/compose/metadata/osbs.json") as f: + data = json.load(f) + self.assertEqual(data, metadata) + + @mock.patch("pungi.phases.osbs.ThreadPool") + def test_dump_empty_metadata(self, ThreadPool): + self.compose.dump_containers_metadata() + self.assertFalse(os.path.isfile(self.tmp_dir + "/compose/metadata/osbs.json")) diff --git a/tests/test_osbs_phase.py b/tests/test_osbs_phase.py index 059ce828..03bd6740 100644 --- a/tests/test_osbs_phase.py +++ b/tests/test_osbs_phase.py @@ -36,34 +36,6 @@ class OSBSPhaseTest(helpers.PungiTestCase): phase = osbs.OSBSPhase(compose) self.assertTrue(phase.skip()) - @mock.patch("pungi.phases.osbs.ThreadPool") - def test_dump_metadata(self, ThreadPool): - compose = helpers.DummyCompose(self.topdir, {"osbs": {"^Everything$": {}}}) - compose.just_phases = None - compose.skip_phases = [] - compose.notifier = mock.Mock() - phase = osbs.OSBSPhase(compose) - phase.start() - phase.stop() - phase.pool.metadata = METADATA - phase.dump_metadata() - - with open(self.topdir + "/compose/metadata/osbs.json") as f: - data = json.load(f) - self.assertEqual(data, METADATA) - - @mock.patch("pungi.phases.osbs.ThreadPool") - def test_dump_metadata_after_skip(self, ThreadPool): - compose = helpers.DummyCompose(self.topdir, {}) - compose.just_phases = None - compose.skip_phases = [] - phase = osbs.OSBSPhase(compose) - phase.start() - phase.stop() - phase.dump_metadata() - - self.assertFalse(os.path.isfile(self.topdir + "/compose/metadata/osbs.json")) - @mock.patch("pungi.phases.osbs.ThreadPool") def test_request_push(self, ThreadPool): compose = helpers.DummyCompose(self.topdir, {"osbs": {"^Everything$": {}}}) @@ -190,7 +162,7 @@ SCRATCH_METADATA = { class OSBSThreadTest(helpers.PungiTestCase): def setUp(self): super(OSBSThreadTest, self).setUp() - self.pool = mock.Mock(metadata={}, registries={}) + self.pool = mock.Mock(registries={}) self.t = osbs.OSBSThread(self.pool) self.compose = helpers.DummyCompose( self.topdir, @@ -226,7 +198,7 @@ class OSBSThreadTest(helpers.PungiTestCase): metadata = copy.deepcopy(METADATA) metadata["Server"]["x86_64"][0]["compose_id"] = self.compose.compose_id metadata["Server"]["x86_64"][0]["koji_task"] = 12345 - self.assertEqual(self.pool.metadata, metadata) + self.assertEqual(self.compose.containers_metadata, metadata) def _assertCorrectCalls(self, opts, setupCalls=None, scratch=False): setupCalls = setupCalls or []