Allow loading overrides for module defaults

This patch adds a new config option. This is expected to be a name of
subdirectory in the repo with module defaults. If supplied, overrides
from that location are loaded every time defaults are loaded.

This raises the minimal required version of libmodulemd to 2.8.0

JIRA: COMPOSE-3828
Signed-off-by: Lubomír Sedlář <lsedlar@redhat.com>

pungi/checks.py

@@ -746,6 +746,7 @@ def make_schema():
                 "default": True
             },
             "module_defaults_dir": {"$ref": "#/definitions/str_or_scm_dict"},
+            "module_defaults_override_dir": {"type": "string"},
 
             "pkgset_repos": {
                 "type": "object",

pungi/phases/createrepo.py

@@ -211,7 +211,10 @@ def create_variant_repo(compose, arch, variant, pkg_type, pkgset, modules_metada
 
         module_names = set(mod_index.get_module_names())
         defaults_dir = compose.paths.work.module_defaults_dir()
-        collect_module_defaults(defaults_dir, module_names, mod_index)
+        overrides_dir = compose.conf.get("module_defaults_override_dir")
+        collect_module_defaults(
+            defaults_dir, module_names, mod_index, overrides_dir=overrides_dir
+        )
 
         log_file = compose.paths.log.log_file(arch, "modifyrepo-modules-%s" % variant)
         add_modular_metadata(repo, repo_dir, mod_index, log_file)

pungi/phases/gather/__init__.py

@@ -382,7 +382,10 @@ def _make_lookaside_repo(compose, variant, arch, pkg_map, package_sets=None):
 
         module_names = set(mod_index.get_module_names())
         defaults_dir = compose.paths.work.module_defaults_dir()
-        collect_module_defaults(defaults_dir, module_names, mod_index)
+        overrides_dir = compose.conf.get("module_defaults_override_dir")
+        collect_module_defaults(
+            defaults_dir, module_names, mod_index, overrides_dir=overrides_dir
+        )
 
         log_file = compose.paths.log.log_file(
             arch, "lookaside_repo_modules_%s" % (variant.uid)

pungi/phases/pkgset/common.py

@@ -155,8 +155,9 @@ def _create_arch_repo(worker_thread, args, task_num):
     # Add modulemd to the repo for all modules in all variants on this architecture.
     if Modulemd and mmd:
         names = set(x.get_module_name() for x in mmd)
+        overrides_dir = compose.conf.get("module_defaults_override_dir")
         mod_index = collect_module_defaults(
-            compose.paths.work.module_defaults_dir(), names
+            compose.paths.work.module_defaults_dir(), names, overrides_dir=overrides_dir
         )
         for x in mmd:
             mod_index.add_module_stream(x)

pungi/util.py

@@ -934,9 +934,10 @@ def iter_module_defaults(path):
     # and work with it. However that does not allow for detecting conflicting
     # defaults. That should not happen in practice, but better safe than sorry.
     # Once libmodulemd can report the error, this code can be simplifed by a
-    # lot. It's implemented in
+    # lot. It was implemented in
     # https://github.com/fedora-modularity/libmodulemd/commit/3087e4a5c38a331041fec9b6b8f1a372f9ffe64d
-    # and released in 2.6.0
+    # and released in 2.6.0, but 2.8.0 added the need to merge overrides and
+    # that breaks this use case again.
     for file in glob.glob(os.path.join(path, "*.yaml")):
         index = Modulemd.ModuleIndex()
         index.update_from_file(file, strict=False)
@@ -944,7 +945,9 @@ def iter_module_defaults(path):
             yield module_name, index.get_module(module_name).get_defaults()
 
 
-def collect_module_defaults(defaults_dir, modules_to_load=None, mod_index=None):
+def collect_module_defaults(
+    defaults_dir, modules_to_load=None, mod_index=None, overrides_dir=None
+):
     """Load module defaults into index.
 
     If `modules_to_load` is passed in, it should be a set of module names. Only
@@ -954,7 +957,15 @@ def collect_module_defaults(defaults_dir, modules_to_load=None, mod_index=None):
     not, a new ModuleIndex will be created and returned
     """
     mod_index = mod_index or Modulemd.ModuleIndex()
-    for module_name, defaults in iter_module_defaults(defaults_dir):
+
+    temp_index = Modulemd.ModuleIndex.new()
+    temp_index.update_from_defaults_directory(
+        defaults_dir, overrides_path=overrides_dir, strict=False
+    )
+
+    for module_name in temp_index.get_module_names():
+        defaults = temp_index.get_module(module_name).get_defaults()
+
         if not modules_to_load or module_name in modules_to_load:
             mod_index.add_defaults(defaults)
 

tests/test_pkgset_common.py

@@ -105,6 +105,7 @@ class TestMaterializedPkgsetCreate(helpers.PungiTestCase):
         cmd.assert_called_once_with(
             os.path.join(self.topdir, "work/global/module_defaults"),
             set(x.get_module_name.return_value for x in mmd["x86_64"]),
+            overrides_dir=None,
         )
         amm.assert_called_once_with(
             mock.ANY,