From 3bb1e3df119d676c55f645e7950f16f7e40c0671 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lubom=C3=ADr=20Sedl=C3=A1=C5=99?= Date: Mon, 8 Jun 2020 14:52:57 +0200 Subject: [PATCH] createrepo: Add extra modulemd files to the repo MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This is a workaround for modularity design issues and DNF bugs. If there were gaps in contexts, DNF has trouble handling the upgrades. Thus we may need to add module metadata for older versions of previously released module streams and add the missing contexts. JIRA: RHELCMP-982 Signed-off-by: Lubomír Sedlář --- doc/configuration.rst | 21 ++++++++++ pungi/checks.py | 7 ++++ pungi/phases/createrepo.py | 24 +++++++++++ tests/fixtures/fake-modulemd.yaml | 20 ++++++++++ tests/test_createrepophase.py | 66 +++++++++++++++++++++++++++++++ 5 files changed, 138 insertions(+) create mode 100644 tests/fixtures/fake-modulemd.yaml diff --git a/doc/configuration.rst b/doc/configuration.rst index 0481626c..09ca8ee4 100644 --- a/doc/configuration.rst +++ b/doc/configuration.rst @@ -439,6 +439,13 @@ Options ``createrepo_c`` executable. This could be useful for enabling zchunk generation and pointing it to correct dictionaries. +**createrepo_extra_modulemd** + (*dict*) -- a mapping of variant UID to :ref:`an scm dict `. + If specified, it should point to a directory with extra module metadata + YAML files that will be added to the repository for this variant. The + cloned files should be split into subdirectories for each architecture of + the variant. + **product_id** = None (:ref:`scm_dict `) -- If specified, it should point to a directory with certificates ``--*.pem``. Pungi will @@ -467,6 +474,20 @@ Example # Also Server.x86_64 should have them (but not on other arches). ('^Server$', {'x86_64': True}), ] + createrepo_extra_modulemd = { + "Server": { + "scm": "git", + "repo": "https://example.com/extra-server-modulemd.git", + "dir": ".", + # The directory should have this layout. Each architecture for the + # variant should be included (even if the directory is empty. + # . + # ├── aarch64 + # │ ├── some-file.yaml + # │ └ ... + # └── x86_64 + } + } Package Set Settings diff --git a/pungi/checks.py b/pungi/checks.py index 55fd55f6..09799ed9 100644 --- a/pungi/checks.py +++ b/pungi/checks.py @@ -659,6 +659,13 @@ def make_schema(): "default": [], "items": {"type": "string"}, }, + "createrepo_extra_modulemd": { + "type": "object", + "patternProperties": { + ".+": {"$ref": "#/definitions/scm_dict"}, + "additionalProperties": False, + }, + }, "repoclosure_strictness": _variant_arch_mapping( { "type": "string", diff --git a/pungi/phases/createrepo.py b/pungi/phases/createrepo.py index 529d31cb..5780068a 100644 --- a/pungi/phases/createrepo.py +++ b/pungi/phases/createrepo.py @@ -73,6 +73,14 @@ class CreaterepoPhase(PhaseBase): for variant in self.compose.get_variants(): if variant.is_empty: continue + + if variant.uid in self.compose.conf.get("createrepo_extra_modulemd", {}): + # Clone extra modulemd repository if it's configured. + get_dir_from_scm( + self.compose.conf["createrepo_extra_modulemd"][variant.uid], + self.compose.paths.work.tmp_dir(variant=variant, create_dir=False), + ) + self.pool.queue_put((self.compose, None, variant, "srpm")) for arch in variant.arches: self.pool.queue_put((self.compose, arch, variant, "rpm")) @@ -237,6 +245,22 @@ def create_variant_repo( defaults_dir, module_names, mod_index, overrides_dir=overrides_dir ) + # Add extra modulemd files + if variant.uid in compose.conf.get("createrepo_extra_modulemd", {}): + compose.log_debug("Adding extra modulemd for %s.%s", variant.uid, arch) + dirname = compose.paths.work.tmp_dir(variant=variant, create_dir=False) + for filepath in glob.glob(os.path.join(dirname, arch) + "/*.yaml"): + module_stream = Modulemd.ModuleStream.read_file(filepath, strict=True) + if not mod_index.add_module_stream(module_stream): + raise RuntimeError( + "Failed parsing modulemd data from %s" % filepath + ) + # Add the module to metadata with dummy tag. We can't leave the + # value empty, but we don't know what the correct tag is. + nsvc = module_stream.get_nsvc() + variant.module_uid_to_koji_tag[nsvc] = "DUMMY" + metadata.append((nsvc, [])) + log_file = compose.paths.log.log_file(arch, "modifyrepo-modules-%s" % variant) add_modular_metadata(repo, repo_dir, mod_index, log_file) diff --git a/tests/fixtures/fake-modulemd.yaml b/tests/fixtures/fake-modulemd.yaml new file mode 100644 index 00000000..bda71c6c --- /dev/null +++ b/tests/fixtures/fake-modulemd.yaml @@ -0,0 +1,20 @@ +--- +document: modulemd +version: 2 +data: + name: mymodule + stream: master + version: 1 + context: cafe + arch: x86_64 + summary: Dummy module + description: Dummy module + license: + module: + - Beerware + content: + - Beerware + artifacts: + rpms: + - foobar-0:1.0-1.noarch +... diff --git a/tests/test_createrepophase.py b/tests/test_createrepophase.py index 53f0f8e9..9f07f48f 100644 --- a/tests/test_createrepophase.py +++ b/tests/test_createrepophase.py @@ -128,6 +128,22 @@ class TestCreaterepoPhase(PungiTestCase): ], ) + @mock.patch("pungi.phases.createrepo.get_dir_from_scm") + @mock.patch("pungi.phases.createrepo.ThreadPool") + def test_clones_extra_modulemd(self, ThreadPoolCls, get_dir_from_scm): + scm = mock.Mock() + compose = DummyCompose( + self.topdir, {"createrepo_extra_modulemd": {"Server": scm}} + ) + + phase = CreaterepoPhase(compose) + phase.run() + + self.assertEqual( + get_dir_from_scm.call_args_list, + [mock.call(scm, os.path.join(compose.topdir, "work/global/tmp-Server"))], + ) + def make_mocked_modifyrepo_cmd(tc, module_artifacts): def mocked_modifyrepo_cmd(repodir, mmd_path, **kwargs): @@ -1190,6 +1206,56 @@ class TestCreateVariantRepo(PungiTestCase): [mock.call(repodata_dir, mock.ANY, compress_type="gz", mdtype="modules")], ) + @unittest.skipUnless(Modulemd is not None, "Skipped test, no module support.") + @mock.patch("pungi.phases.createrepo.find_file_in_repodata") + @mock.patch("pungi.phases.createrepo.run") + @mock.patch("pungi.phases.createrepo.CreaterepoWrapper") + def test_variant_repo_extra_modulemd( + self, CreaterepoWrapperCls, run, modulemd_filename + ): + compose = DummyCompose( + self.topdir, {"createrepo_extra_modulemd": {"Server": mock.Mock()}} + ) + compose.has_comps = False + + variant = compose.variants["Server"] + variant.arch_mmds["x86_64"] = {} + variant.module_uid_to_koji_tag = {} + + repo = CreaterepoWrapperCls.return_value + + copy_fixture("server-rpms.json", compose.paths.compose.metadata("rpms.json")) + copy_fixture( + "fake-modulemd.yaml", + os.path.join(compose.topdir, "work/global/tmp-Server/x86_64/*.yaml"), + ) + + repodata_dir = os.path.join( + compose.paths.compose.os_tree("x86_64", compose.variants["Server"]), + "repodata", + ) + + modules_metadata = ModulesMetadata(compose) + + modulemd_filename.return_value = "Server/x86_64/os/repodata/3511d16a723e1bd69826e591508f07e377d2212769b59178a9-modules.yaml.gz" # noqa: E501 + create_variant_repo( + compose, + "x86_64", + compose.variants["Server"], + "rpm", + self.pkgset, + modules_metadata, + ) + + self.assertEqual( + repo.get_modifyrepo_cmd.mock_calls, + [mock.call(repodata_dir, mock.ANY, compress_type="gz", mdtype="modules")], + ) + self.assertEqual( + list(modules_metadata.productmd_modules_metadata["Server"]["x86_64"]), + ["mymodule:master:1:cafe"], + ) + class TestGetProductIds(PungiTestCase): def mock_get(self, filenames):