diff --git a/0001-Allow-loading-overrides-for-module-defaults.patch b/0001-Allow-loading-overrides-for-module-defaults.patch new file mode 100644 index 00000000..9f91fdea --- /dev/null +++ b/0001-Allow-loading-overrides-for-module-defaults.patch @@ -0,0 +1,144 @@ +From 2b112d53f7ea443dd11be584269f559d7d911227 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Lubom=C3=ADr=20Sedl=C3=A1=C5=99?= +Date: Wed, 18 Sep 2019 14:47:30 +0200 +Subject: [PATCH] Allow loading overrides for module defaults +MIME-Version: 1.0 +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 8bit + +This patch adds a new config option. This is expected to be a name of +subdirectory in the repo with module defaults. If supplied, overrides +from that location are loaded every time defaults are loaded. + +This raises the minimal required version of libmodulemd to 2.8.0 + +JIRA: COMPOSE-3828 +Signed-off-by: Lubomír Sedlář +--- + pungi/checks.py | 1 + + pungi/phases/createrepo.py | 5 ++++- + pungi/phases/gather/__init__.py | 5 ++++- + pungi/phases/pkgset/common.py | 3 ++- + pungi/util.py | 19 +++++++++++++++---- + tests/test_pkgset_common.py | 1 + + 6 files changed, 27 insertions(+), 7 deletions(-) + +diff --git a/pungi/checks.py b/pungi/checks.py +index 17ff8b50..7e9f65a7 100644 +--- a/pungi/checks.py ++++ b/pungi/checks.py +@@ -746,6 +746,7 @@ def make_schema(): + "default": True + }, + "module_defaults_dir": {"$ref": "#/definitions/str_or_scm_dict"}, ++ "module_defaults_override_dir": {"type": "string"}, + + "pkgset_repos": { + "type": "object", +diff --git a/pungi/phases/createrepo.py b/pungi/phases/createrepo.py +index 64553594..b2c67ad7 100644 +--- a/pungi/phases/createrepo.py ++++ b/pungi/phases/createrepo.py +@@ -211,7 +211,10 @@ def create_variant_repo(compose, arch, variant, pkg_type, pkgset, modules_metada + + module_names = set(mod_index.get_module_names()) + defaults_dir = compose.paths.work.module_defaults_dir() +- collect_module_defaults(defaults_dir, module_names, mod_index) ++ overrides_dir = compose.conf.get("module_defaults_override_dir") ++ collect_module_defaults( ++ defaults_dir, module_names, mod_index, overrides_dir=overrides_dir ++ ) + + log_file = compose.paths.log.log_file(arch, "modifyrepo-modules-%s" % variant) + add_modular_metadata(repo, repo_dir, mod_index, log_file) +diff --git a/pungi/phases/gather/__init__.py b/pungi/phases/gather/__init__.py +index afba72f6..736c10ec 100644 +--- a/pungi/phases/gather/__init__.py ++++ b/pungi/phases/gather/__init__.py +@@ -382,7 +382,10 @@ def _make_lookaside_repo(compose, variant, arch, pkg_map, package_sets=None): + + module_names = set(mod_index.get_module_names()) + defaults_dir = compose.paths.work.module_defaults_dir() +- collect_module_defaults(defaults_dir, module_names, mod_index) ++ overrides_dir = compose.conf.get("module_defaults_override_dir") ++ collect_module_defaults( ++ defaults_dir, module_names, mod_index, overrides_dir=overrides_dir ++ ) + + log_file = compose.paths.log.log_file( + arch, "lookaside_repo_modules_%s" % (variant.uid) +diff --git a/pungi/phases/pkgset/common.py b/pungi/phases/pkgset/common.py +index 6ab86e53..d7e06cfc 100644 +--- a/pungi/phases/pkgset/common.py ++++ b/pungi/phases/pkgset/common.py +@@ -155,8 +155,9 @@ def _create_arch_repo(worker_thread, args, task_num): + # Add modulemd to the repo for all modules in all variants on this architecture. + if Modulemd and mmd: + names = set(x.get_module_name() for x in mmd) ++ overrides_dir = compose.conf.get("module_defaults_override_dir") + mod_index = collect_module_defaults( +- compose.paths.work.module_defaults_dir(), names ++ compose.paths.work.module_defaults_dir(), names, overrides_dir=overrides_dir + ) + for x in mmd: + mod_index.add_module_stream(x) +diff --git a/pungi/util.py b/pungi/util.py +index c2c9a948..bd3e022c 100644 +--- a/pungi/util.py ++++ b/pungi/util.py +@@ -934,9 +934,10 @@ def iter_module_defaults(path): + # and work with it. However that does not allow for detecting conflicting + # defaults. That should not happen in practice, but better safe than sorry. + # Once libmodulemd can report the error, this code can be simplifed by a +- # lot. It's implemented in ++ # lot. It was implemented in + # https://github.com/fedora-modularity/libmodulemd/commit/3087e4a5c38a331041fec9b6b8f1a372f9ffe64d +- # and released in 2.6.0 ++ # and released in 2.6.0, but 2.8.0 added the need to merge overrides and ++ # that breaks this use case again. + for file in glob.glob(os.path.join(path, "*.yaml")): + index = Modulemd.ModuleIndex() + index.update_from_file(file, strict=False) +@@ -944,7 +945,9 @@ def iter_module_defaults(path): + yield module_name, index.get_module(module_name).get_defaults() + + +-def collect_module_defaults(defaults_dir, modules_to_load=None, mod_index=None): ++def collect_module_defaults( ++ defaults_dir, modules_to_load=None, mod_index=None, overrides_dir=None ++): + """Load module defaults into index. + + If `modules_to_load` is passed in, it should be a set of module names. Only +@@ -954,7 +957,15 @@ def collect_module_defaults(defaults_dir, modules_to_load=None, mod_index=None): + not, a new ModuleIndex will be created and returned + """ + mod_index = mod_index or Modulemd.ModuleIndex() +- for module_name, defaults in iter_module_defaults(defaults_dir): ++ ++ temp_index = Modulemd.ModuleIndex.new() ++ temp_index.update_from_defaults_directory( ++ defaults_dir, overrides_path=overrides_dir, strict=False ++ ) ++ ++ for module_name in temp_index.get_module_names(): ++ defaults = temp_index.get_module(module_name).get_defaults() ++ + if not modules_to_load or module_name in modules_to_load: + mod_index.add_defaults(defaults) + +diff --git a/tests/test_pkgset_common.py b/tests/test_pkgset_common.py +index 64366a42..d1e147fe 100755 +--- a/tests/test_pkgset_common.py ++++ b/tests/test_pkgset_common.py +@@ -105,6 +105,7 @@ class TestMaterializedPkgsetCreate(helpers.PungiTestCase): + cmd.assert_called_once_with( + os.path.join(self.topdir, "work/global/module_defaults"), + set(x.get_module_name.return_value for x in mmd["x86_64"]), ++ overrides_dir=None, + ) + amm.assert_called_once_with( + mock.ANY, +-- +2.21.0 + diff --git a/pungi.spec b/pungi.spec index c47a6f22..1c33e1b8 100644 --- a/pungi.spec +++ b/pungi.spec @@ -2,12 +2,13 @@ Name: pungi Version: 4.1.39 -Release: 2%{?dist} +Release: 3%{?dist} Summary: Distribution compose tool License: GPLv2 URL: https://pagure.io/pungi Source0: https://pagure.io/releases/%{name}/%{name}-%{version}.tar.bz2 +Patch0: 0001-Allow-loading-overrides-for-module-defaults.patch BuildRequires: python3-nose BuildRequires: python3-mock @@ -30,7 +31,7 @@ BuildRequires: python3-koji BuildRequires: python3-unittest2 BuildRequires: lorax BuildRequires: python3-PyYAML -BuildRequires: python3-libmodulemd +BuildRequires: python3-libmodulemd >= 2.8.0 BuildRequires: python3-gobject BuildRequires: python3-createrepo_c BuildRequires: python3-dogpile-cache @@ -60,7 +61,7 @@ Requires: python3-dnf Requires: python3-multilib Requires: python3-libcomps Requires: python3-koji -Requires: python3-libmodulemd1 +Requires: python3-libmodulemd >= 2.8.0 Requires: python3-gobject Requires: python3-createrepo_c Requires: python3-PyYAML @@ -142,6 +143,9 @@ nosetests-3 --exe %{_bindir}/%{name}-wait-for-signed-ostree-handler %changelog +* Mon Sep 23 2019 Lubomír Sedlář - 4.1.39-3 +- Backport patch for module defaults overrides + * Mon Sep 09 2019 Lubomír Sedlář - 4.1.39-2 - Drop pungi-legacy subpackage due to broken dependencies