createrepo: Add extra modulemd files to the repo

This is a workaround for modularity design issues and DNF bugs. If there
were gaps in contexts, DNF has trouble handling the upgrades. Thus we
may need to add module metadata for older versions of previously
released module streams and add the missing contexts.

JIRA: RHELCMP-982
Signed-off-by: Lubomír Sedlář <lsedlar@redhat.com>

doc/configuration.rst

@@ -439,6 +439,13 @@ Options
     ``createrepo_c`` executable. This could be useful for enabling zchunk
     generation and pointing it to correct dictionaries.
 
+**createrepo_extra_modulemd**
+    (*dict*) -- a mapping of variant UID to :ref:`an scm dict <scm_support>`.
+    If specified, it should point to a directory with extra module metadata
+    YAML files that will be added to the repository for this variant. The
+    cloned files should be split into subdirectories for each architecture of
+    the variant.
+
 **product_id** = None
     (:ref:`scm_dict <scm_support>`) -- If specified, it should point to a
     directory with certificates ``<variant_uid>-<arch>-*.pem``. Pungi will
@@ -467,6 +474,20 @@ Example
         # Also Server.x86_64 should have them (but not on other arches).
         ('^Server$', {'x86_64': True}),
     ]
+    createrepo_extra_modulemd = {
+        "Server": {
+            "scm": "git",
+            "repo": "https://example.com/extra-server-modulemd.git",
+            "dir": ".",
+            # The directory should have this layout. Each architecture for the
+            # variant should be included (even if the directory is empty.
+            # .
+            # ├── aarch64
+            # │   ├── some-file.yaml
+            # │   └ ...
+            # └── x86_64
+        }
+    }
 
 
 Package Set Settings

pungi/checks.py

@@ -659,6 +659,13 @@ def make_schema():
                 "default": [],
                 "items": {"type": "string"},
             },
+            "createrepo_extra_modulemd": {
+                "type": "object",
+                "patternProperties": {
+                    ".+": {"$ref": "#/definitions/scm_dict"},
+                    "additionalProperties": False,
+                },
+            },
             "repoclosure_strictness": _variant_arch_mapping(
                 {
                     "type": "string",

pungi/phases/createrepo.py

@@ -73,6 +73,14 @@ class CreaterepoPhase(PhaseBase):
         for variant in self.compose.get_variants():
             if variant.is_empty:
                 continue
+
+            if variant.uid in self.compose.conf.get("createrepo_extra_modulemd", {}):
+                # Clone extra modulemd repository if it's configured.
+                get_dir_from_scm(
+                    self.compose.conf["createrepo_extra_modulemd"][variant.uid],
+                    self.compose.paths.work.tmp_dir(variant=variant, create_dir=False),
+                )
+
             self.pool.queue_put((self.compose, None, variant, "srpm"))
             for arch in variant.arches:
                 self.pool.queue_put((self.compose, arch, variant, "rpm"))
@@ -237,6 +245,22 @@ def create_variant_repo(
             defaults_dir, module_names, mod_index, overrides_dir=overrides_dir
         )
 
+        # Add extra modulemd files
+        if variant.uid in compose.conf.get("createrepo_extra_modulemd", {}):
+            compose.log_debug("Adding extra modulemd for %s.%s", variant.uid, arch)
+            dirname = compose.paths.work.tmp_dir(variant=variant, create_dir=False)
+            for filepath in glob.glob(os.path.join(dirname, arch) + "/*.yaml"):
+                module_stream = Modulemd.ModuleStream.read_file(filepath, strict=True)
+                if not mod_index.add_module_stream(module_stream):
+                    raise RuntimeError(
+                        "Failed parsing modulemd data from %s" % filepath
+                    )
+                # Add the module to metadata with dummy tag. We can't leave the
+                # value empty, but we don't know what the correct tag is.
+                nsvc = module_stream.get_nsvc()
+                variant.module_uid_to_koji_tag[nsvc] = "DUMMY"
+                metadata.append((nsvc, []))
+
         log_file = compose.paths.log.log_file(arch, "modifyrepo-modules-%s" % variant)
         add_modular_metadata(repo, repo_dir, mod_index, log_file)
 

tests/fixtures/fake-modulemd.yaml

@@ -0,0 +1,20 @@
+---
+document: modulemd
+version: 2
+data:
+  name: mymodule
+  stream: master
+  version: 1
+  context: cafe
+  arch: x86_64
+  summary: Dummy module
+  description: Dummy module
+  license:
+    module:
+    - Beerware
+    content:
+    - Beerware
+  artifacts:
+    rpms:
+      - foobar-0:1.0-1.noarch
+...

tests/test_createrepophase.py

@@ -128,6 +128,22 @@ class TestCreaterepoPhase(PungiTestCase):
             ],
         )
 
+    @mock.patch("pungi.phases.createrepo.get_dir_from_scm")
+    @mock.patch("pungi.phases.createrepo.ThreadPool")
+    def test_clones_extra_modulemd(self, ThreadPoolCls, get_dir_from_scm):
+        scm = mock.Mock()
+        compose = DummyCompose(
+            self.topdir, {"createrepo_extra_modulemd": {"Server": scm}}
+        )
+
+        phase = CreaterepoPhase(compose)
+        phase.run()
+
+        self.assertEqual(
+            get_dir_from_scm.call_args_list,
+            [mock.call(scm, os.path.join(compose.topdir, "work/global/tmp-Server"))],
+        )
+
 
 def make_mocked_modifyrepo_cmd(tc, module_artifacts):
     def mocked_modifyrepo_cmd(repodir, mmd_path, **kwargs):
@@ -1190,6 +1206,56 @@ class TestCreateVariantRepo(PungiTestCase):
             [mock.call(repodata_dir, mock.ANY, compress_type="gz", mdtype="modules")],
         )
 
+    @unittest.skipUnless(Modulemd is not None, "Skipped test, no module support.")
+    @mock.patch("pungi.phases.createrepo.find_file_in_repodata")
+    @mock.patch("pungi.phases.createrepo.run")
+    @mock.patch("pungi.phases.createrepo.CreaterepoWrapper")
+    def test_variant_repo_extra_modulemd(
+        self, CreaterepoWrapperCls, run, modulemd_filename
+    ):
+        compose = DummyCompose(
+            self.topdir, {"createrepo_extra_modulemd": {"Server": mock.Mock()}}
+        )
+        compose.has_comps = False
+
+        variant = compose.variants["Server"]
+        variant.arch_mmds["x86_64"] = {}
+        variant.module_uid_to_koji_tag = {}
+
+        repo = CreaterepoWrapperCls.return_value
+
+        copy_fixture("server-rpms.json", compose.paths.compose.metadata("rpms.json"))
+        copy_fixture(
+            "fake-modulemd.yaml",
+            os.path.join(compose.topdir, "work/global/tmp-Server/x86_64/*.yaml"),
+        )
+
+        repodata_dir = os.path.join(
+            compose.paths.compose.os_tree("x86_64", compose.variants["Server"]),
+            "repodata",
+        )
+
+        modules_metadata = ModulesMetadata(compose)
+
+        modulemd_filename.return_value = "Server/x86_64/os/repodata/3511d16a723e1bd69826e591508f07e377d2212769b59178a9-modules.yaml.gz"  # noqa: E501
+        create_variant_repo(
+            compose,
+            "x86_64",
+            compose.variants["Server"],
+            "rpm",
+            self.pkgset,
+            modules_metadata,
+        )
+
+        self.assertEqual(
+            repo.get_modifyrepo_cmd.mock_calls,
+            [mock.call(repodata_dir, mock.ANY, compress_type="gz", mdtype="modules")],
+        )
+        self.assertEqual(
+            list(modules_metadata.productmd_modules_metadata["Server"]["x86_64"]),
+            ["mymodule:master:1:cafe"],
+        )
+
 
 class TestGetProductIds(PungiTestCase):
     def mock_get(self, filenames):