diff --git a/doc/configuration.rst b/doc/configuration.rst index 0481626c..09ca8ee4 100644 --- a/doc/configuration.rst +++ b/doc/configuration.rst @@ -439,6 +439,13 @@ Options ``createrepo_c`` executable. This could be useful for enabling zchunk generation and pointing it to correct dictionaries. +**createrepo_extra_modulemd** + (*dict*) -- a mapping of variant UID to :ref:`an scm dict `. + If specified, it should point to a directory with extra module metadata + YAML files that will be added to the repository for this variant. The + cloned files should be split into subdirectories for each architecture of + the variant. + **product_id** = None (:ref:`scm_dict `) -- If specified, it should point to a directory with certificates ``--*.pem``. Pungi will @@ -467,6 +474,20 @@ Example # Also Server.x86_64 should have them (but not on other arches). ('^Server$', {'x86_64': True}), ] + createrepo_extra_modulemd = { + "Server": { + "scm": "git", + "repo": "https://example.com/extra-server-modulemd.git", + "dir": ".", + # The directory should have this layout. Each architecture for the + # variant should be included (even if the directory is empty. + # . + # ├── aarch64 + # │ ├── some-file.yaml + # │ └ ... + # └── x86_64 + } + } Package Set Settings diff --git a/pungi/checks.py b/pungi/checks.py index 55fd55f6..09799ed9 100644 --- a/pungi/checks.py +++ b/pungi/checks.py @@ -659,6 +659,13 @@ def make_schema(): "default": [], "items": {"type": "string"}, }, + "createrepo_extra_modulemd": { + "type": "object", + "patternProperties": { + ".+": {"$ref": "#/definitions/scm_dict"}, + "additionalProperties": False, + }, + }, "repoclosure_strictness": _variant_arch_mapping( { "type": "string", diff --git a/pungi/phases/createrepo.py b/pungi/phases/createrepo.py index 529d31cb..5780068a 100644 --- a/pungi/phases/createrepo.py +++ b/pungi/phases/createrepo.py @@ -73,6 +73,14 @@ class CreaterepoPhase(PhaseBase): for variant in self.compose.get_variants(): if variant.is_empty: continue + + if variant.uid in self.compose.conf.get("createrepo_extra_modulemd", {}): + # Clone extra modulemd repository if it's configured. + get_dir_from_scm( + self.compose.conf["createrepo_extra_modulemd"][variant.uid], + self.compose.paths.work.tmp_dir(variant=variant, create_dir=False), + ) + self.pool.queue_put((self.compose, None, variant, "srpm")) for arch in variant.arches: self.pool.queue_put((self.compose, arch, variant, "rpm")) @@ -237,6 +245,22 @@ def create_variant_repo( defaults_dir, module_names, mod_index, overrides_dir=overrides_dir ) + # Add extra modulemd files + if variant.uid in compose.conf.get("createrepo_extra_modulemd", {}): + compose.log_debug("Adding extra modulemd for %s.%s", variant.uid, arch) + dirname = compose.paths.work.tmp_dir(variant=variant, create_dir=False) + for filepath in glob.glob(os.path.join(dirname, arch) + "/*.yaml"): + module_stream = Modulemd.ModuleStream.read_file(filepath, strict=True) + if not mod_index.add_module_stream(module_stream): + raise RuntimeError( + "Failed parsing modulemd data from %s" % filepath + ) + # Add the module to metadata with dummy tag. We can't leave the + # value empty, but we don't know what the correct tag is. + nsvc = module_stream.get_nsvc() + variant.module_uid_to_koji_tag[nsvc] = "DUMMY" + metadata.append((nsvc, [])) + log_file = compose.paths.log.log_file(arch, "modifyrepo-modules-%s" % variant) add_modular_metadata(repo, repo_dir, mod_index, log_file) diff --git a/tests/fixtures/fake-modulemd.yaml b/tests/fixtures/fake-modulemd.yaml new file mode 100644 index 00000000..bda71c6c --- /dev/null +++ b/tests/fixtures/fake-modulemd.yaml @@ -0,0 +1,20 @@ +--- +document: modulemd +version: 2 +data: + name: mymodule + stream: master + version: 1 + context: cafe + arch: x86_64 + summary: Dummy module + description: Dummy module + license: + module: + - Beerware + content: + - Beerware + artifacts: + rpms: + - foobar-0:1.0-1.noarch +... diff --git a/tests/test_createrepophase.py b/tests/test_createrepophase.py index 53f0f8e9..9f07f48f 100644 --- a/tests/test_createrepophase.py +++ b/tests/test_createrepophase.py @@ -128,6 +128,22 @@ class TestCreaterepoPhase(PungiTestCase): ], ) + @mock.patch("pungi.phases.createrepo.get_dir_from_scm") + @mock.patch("pungi.phases.createrepo.ThreadPool") + def test_clones_extra_modulemd(self, ThreadPoolCls, get_dir_from_scm): + scm = mock.Mock() + compose = DummyCompose( + self.topdir, {"createrepo_extra_modulemd": {"Server": scm}} + ) + + phase = CreaterepoPhase(compose) + phase.run() + + self.assertEqual( + get_dir_from_scm.call_args_list, + [mock.call(scm, os.path.join(compose.topdir, "work/global/tmp-Server"))], + ) + def make_mocked_modifyrepo_cmd(tc, module_artifacts): def mocked_modifyrepo_cmd(repodir, mmd_path, **kwargs): @@ -1190,6 +1206,56 @@ class TestCreateVariantRepo(PungiTestCase): [mock.call(repodata_dir, mock.ANY, compress_type="gz", mdtype="modules")], ) + @unittest.skipUnless(Modulemd is not None, "Skipped test, no module support.") + @mock.patch("pungi.phases.createrepo.find_file_in_repodata") + @mock.patch("pungi.phases.createrepo.run") + @mock.patch("pungi.phases.createrepo.CreaterepoWrapper") + def test_variant_repo_extra_modulemd( + self, CreaterepoWrapperCls, run, modulemd_filename + ): + compose = DummyCompose( + self.topdir, {"createrepo_extra_modulemd": {"Server": mock.Mock()}} + ) + compose.has_comps = False + + variant = compose.variants["Server"] + variant.arch_mmds["x86_64"] = {} + variant.module_uid_to_koji_tag = {} + + repo = CreaterepoWrapperCls.return_value + + copy_fixture("server-rpms.json", compose.paths.compose.metadata("rpms.json")) + copy_fixture( + "fake-modulemd.yaml", + os.path.join(compose.topdir, "work/global/tmp-Server/x86_64/*.yaml"), + ) + + repodata_dir = os.path.join( + compose.paths.compose.os_tree("x86_64", compose.variants["Server"]), + "repodata", + ) + + modules_metadata = ModulesMetadata(compose) + + modulemd_filename.return_value = "Server/x86_64/os/repodata/3511d16a723e1bd69826e591508f07e377d2212769b59178a9-modules.yaml.gz" # noqa: E501 + create_variant_repo( + compose, + "x86_64", + compose.variants["Server"], + "rpm", + self.pkgset, + modules_metadata, + ) + + self.assertEqual( + repo.get_modifyrepo_cmd.mock_calls, + [mock.call(repodata_dir, mock.ANY, compress_type="gz", mdtype="modules")], + ) + self.assertEqual( + list(modules_metadata.productmd_modules_metadata["Server"]["x86_64"]), + ["mymodule:master:1:cafe"], + ) + class TestGetProductIds(PungiTestCase): def mock_get(self, filenames):