Backport patch for module defaults overrides

0001-Allow-loading-overrides-for-module-defaults.patch

@@ -0,0 +1,144 @@
+From 2b112d53f7ea443dd11be584269f559d7d911227 Mon Sep 17 00:00:00 2001
+From: =?UTF-8?q?Lubom=C3=ADr=20Sedl=C3=A1=C5=99?= <lsedlar@redhat.com>
+Date: Wed, 18 Sep 2019 14:47:30 +0200
+Subject: [PATCH] Allow loading overrides for module defaults
+MIME-Version: 1.0
+Content-Type: text/plain; charset=UTF-8
+Content-Transfer-Encoding: 8bit
+
+This patch adds a new config option. This is expected to be a name of
+subdirectory in the repo with module defaults. If supplied, overrides
+from that location are loaded every time defaults are loaded.
+
+This raises the minimal required version of libmodulemd to 2.8.0
+
+JIRA: COMPOSE-3828
+Signed-off-by: Lubomír Sedlář <lsedlar@redhat.com>
+---
+ pungi/checks.py                 |  1 +
+ pungi/phases/createrepo.py      |  5 ++++-
+ pungi/phases/gather/__init__.py |  5 ++++-
+ pungi/phases/pkgset/common.py   |  3 ++-
+ pungi/util.py                   | 19 +++++++++++++++----
+ tests/test_pkgset_common.py     |  1 +
+ 6 files changed, 27 insertions(+), 7 deletions(-)
+
+diff --git a/pungi/checks.py b/pungi/checks.py
+index 17ff8b50..7e9f65a7 100644
+--- a/pungi/checks.py
++++ b/pungi/checks.py
+@@ -746,6 +746,7 @@ def make_schema():
+                 "default": True
+             },
+             "module_defaults_dir": {"$ref": "#/definitions/str_or_scm_dict"},
++            "module_defaults_override_dir": {"type": "string"},
+ 
+             "pkgset_repos": {
+                 "type": "object",
+diff --git a/pungi/phases/createrepo.py b/pungi/phases/createrepo.py
+index 64553594..b2c67ad7 100644
+--- a/pungi/phases/createrepo.py
++++ b/pungi/phases/createrepo.py
+@@ -211,7 +211,10 @@ def create_variant_repo(compose, arch, variant, pkg_type, pkgset, modules_metada
+ 
+         module_names = set(mod_index.get_module_names())
+         defaults_dir = compose.paths.work.module_defaults_dir()
+-        collect_module_defaults(defaults_dir, module_names, mod_index)
++        overrides_dir = compose.conf.get("module_defaults_override_dir")
++        collect_module_defaults(
++            defaults_dir, module_names, mod_index, overrides_dir=overrides_dir
++        )
+ 
+         log_file = compose.paths.log.log_file(arch, "modifyrepo-modules-%s" % variant)
+         add_modular_metadata(repo, repo_dir, mod_index, log_file)
+diff --git a/pungi/phases/gather/__init__.py b/pungi/phases/gather/__init__.py
+index afba72f6..736c10ec 100644
+--- a/pungi/phases/gather/__init__.py
++++ b/pungi/phases/gather/__init__.py
+@@ -382,7 +382,10 @@ def _make_lookaside_repo(compose, variant, arch, pkg_map, package_sets=None):
+ 
+         module_names = set(mod_index.get_module_names())
+         defaults_dir = compose.paths.work.module_defaults_dir()
+-        collect_module_defaults(defaults_dir, module_names, mod_index)
++        overrides_dir = compose.conf.get("module_defaults_override_dir")
++        collect_module_defaults(
++            defaults_dir, module_names, mod_index, overrides_dir=overrides_dir
++        )
+ 
+         log_file = compose.paths.log.log_file(
+             arch, "lookaside_repo_modules_%s" % (variant.uid)
+diff --git a/pungi/phases/pkgset/common.py b/pungi/phases/pkgset/common.py
+index 6ab86e53..d7e06cfc 100644
+--- a/pungi/phases/pkgset/common.py
++++ b/pungi/phases/pkgset/common.py
+@@ -155,8 +155,9 @@ def _create_arch_repo(worker_thread, args, task_num):
+     # Add modulemd to the repo for all modules in all variants on this architecture.
+     if Modulemd and mmd:
+         names = set(x.get_module_name() for x in mmd)
++        overrides_dir = compose.conf.get("module_defaults_override_dir")
+         mod_index = collect_module_defaults(
+-            compose.paths.work.module_defaults_dir(), names
++            compose.paths.work.module_defaults_dir(), names, overrides_dir=overrides_dir
+         )
+         for x in mmd:
+             mod_index.add_module_stream(x)
+diff --git a/pungi/util.py b/pungi/util.py
+index c2c9a948..bd3e022c 100644
+--- a/pungi/util.py
++++ b/pungi/util.py
+@@ -934,9 +934,10 @@ def iter_module_defaults(path):
+     # and work with it. However that does not allow for detecting conflicting
+     # defaults. That should not happen in practice, but better safe than sorry.
+     # Once libmodulemd can report the error, this code can be simplifed by a
+-    # lot. It's implemented in
++    # lot. It was implemented in
+     # https://github.com/fedora-modularity/libmodulemd/commit/3087e4a5c38a331041fec9b6b8f1a372f9ffe64d
+-    # and released in 2.6.0
++    # and released in 2.6.0, but 2.8.0 added the need to merge overrides and
++    # that breaks this use case again.
+     for file in glob.glob(os.path.join(path, "*.yaml")):
+         index = Modulemd.ModuleIndex()
+         index.update_from_file(file, strict=False)
+@@ -944,7 +945,9 @@ def iter_module_defaults(path):
+             yield module_name, index.get_module(module_name).get_defaults()
+ 
+ 
+-def collect_module_defaults(defaults_dir, modules_to_load=None, mod_index=None):
++def collect_module_defaults(
++    defaults_dir, modules_to_load=None, mod_index=None, overrides_dir=None
++):
+     """Load module defaults into index.
+ 
+     If `modules_to_load` is passed in, it should be a set of module names. Only
+@@ -954,7 +957,15 @@ def collect_module_defaults(defaults_dir, modules_to_load=None, mod_index=None):
+     not, a new ModuleIndex will be created and returned
+     """
+     mod_index = mod_index or Modulemd.ModuleIndex()
+-    for module_name, defaults in iter_module_defaults(defaults_dir):
++
++    temp_index = Modulemd.ModuleIndex.new()
++    temp_index.update_from_defaults_directory(
++        defaults_dir, overrides_path=overrides_dir, strict=False
++    )
++
++    for module_name in temp_index.get_module_names():
++        defaults = temp_index.get_module(module_name).get_defaults()
++
+         if not modules_to_load or module_name in modules_to_load:
+             mod_index.add_defaults(defaults)
+ 
+diff --git a/tests/test_pkgset_common.py b/tests/test_pkgset_common.py
+index 64366a42..d1e147fe 100755
+--- a/tests/test_pkgset_common.py
++++ b/tests/test_pkgset_common.py
+@@ -105,6 +105,7 @@ class TestMaterializedPkgsetCreate(helpers.PungiTestCase):
+         cmd.assert_called_once_with(
+             os.path.join(self.topdir, "work/global/module_defaults"),
+             set(x.get_module_name.return_value for x in mmd["x86_64"]),
++            overrides_dir=None,
+         )
+         amm.assert_called_once_with(
+             mock.ANY,
+-- 
+2.21.0
+

pungi.spec

@@ -2,12 +2,13 @@
 
 Name:           pungi
 Version:        4.1.39
-Release:        2%{?dist}
+Release:        3%{?dist}
 Summary:        Distribution compose tool
 
 License:        GPLv2
 URL:            https://pagure.io/pungi
 Source0:        https://pagure.io/releases/%{name}/%{name}-%{version}.tar.bz2
+Patch0:         0001-Allow-loading-overrides-for-module-defaults.patch
 
 BuildRequires:  python3-nose
 BuildRequires:  python3-mock
@@ -30,7 +31,7 @@ BuildRequires:  python3-koji
 BuildRequires:  python3-unittest2
 BuildRequires:  lorax
 BuildRequires:  python3-PyYAML
-BuildRequires:  python3-libmodulemd
+BuildRequires:  python3-libmodulemd >= 2.8.0
 BuildRequires:  python3-gobject
 BuildRequires:  python3-createrepo_c
 BuildRequires:  python3-dogpile-cache
@@ -60,7 +61,7 @@ Requires:       python3-dnf
 Requires:       python3-multilib
 Requires:       python3-libcomps
 Requires:       python3-koji
-Requires:       python3-libmodulemd1
+Requires:       python3-libmodulemd >= 2.8.0
 Requires:       python3-gobject
 Requires:       python3-createrepo_c
 Requires:       python3-PyYAML
@@ -142,6 +143,9 @@ nosetests-3 --exe
 %{_bindir}/%{name}-wait-for-signed-ostree-handler
 
 %changelog
+* Mon Sep 23 2019 Lubomír Sedlář <lsedlar@redhat.com> - 4.1.39-3
+- Backport patch for module defaults overrides
+
 * Mon Sep 09 2019 Lubomír Sedlář <lsedlar@redhat.com> - 4.1.39-2
 - Drop pungi-legacy subpackage due to broken dependencies