diff --git a/.gitignore b/.gitignore index 101ccee0..6c20baf6 100644 --- a/.gitignore +++ b/.gitignore @@ -14,3 +14,5 @@ htmlcov/ .idea/ .tox .venv +.kdev4/ +pungi.kdev4 diff --git a/doc/examples.rst b/doc/examples.rst index 701a5f5a..0370fa69 100644 --- a/doc/examples.rst +++ b/doc/examples.rst @@ -30,9 +30,17 @@ This is a shortened configuration for Fedora Radhide compose as of 2019-10-14. module_defaults_dir = { 'scm': 'git', 'repo': 'https://pagure.io/releng/fedora-module-defaults.git', - 'branch': 'master', + 'branch': 'main', 'dir': '.' } + # Optional module obsoletes configuration which is merged + # into the module index and gets resolved + module_obsoletes_dir = { + 'scm': 'git', + 'repo': 'https://pagure.io/releng/fedora-module-defaults.git', + 'branch': 'main', + 'dir': 'obsoletes' + } variants_file='variants-fedora.xml' sigkeys = ['12C944D0'] diff --git a/pungi/checks.py b/pungi/checks.py index 30114110..81c11699 100644 --- a/pungi/checks.py +++ b/pungi/checks.py @@ -736,6 +736,7 @@ def make_schema(): "patternProperties": {".+": {"$ref": "#/definitions/strings"}}, "additionalProperties": False, }, + "module_obsoletes_dir": {"$ref": "#/definitions/str_or_scm_dict"}, "create_optional_isos": {"type": "boolean", "default": False}, "symlink_isos_to": {"type": "string"}, "dogpile_cache_backend": {"type": "string"}, diff --git a/pungi/compose.py b/pungi/compose.py index 47066807..ce0fdb74 100644 --- a/pungi/compose.py +++ b/pungi/compose.py @@ -377,6 +377,10 @@ class Compose(kobo.log.LoggingBase): def has_module_defaults(self): return bool(self.conf.get("module_defaults_dir", False)) + @property + def has_module_obsoletes(self): + return bool(self.conf.get("module_obsoletes_dir", False)) + @property def config_dir(self): return os.path.dirname(self.conf._open_file or "") diff --git a/pungi/module_util.py b/pungi/module_util.py index ab29a67c..94a0ec5a 100644 --- a/pungi/module_util.py +++ b/pungi/module_util.py @@ -25,9 +25,10 @@ except (ImportError, ValueError): Modulemd = None -def iter_module_defaults(path): +def iter_module_defaults_or_obsoletes(path, obsoletes=False): """Given a path to a directory with yaml files, yield each module default in there as a pair (module_name, ModuleDefaults instance). + The same happens for module obsoletes if the obsoletes switch is True. """ # It is really tempting to merge all the module indexes into a single one # and work with it. However that does not allow for detecting conflicting @@ -41,7 +42,10 @@ def iter_module_defaults(path): index = Modulemd.ModuleIndex() index.update_from_file(file, strict=False) for module_name in index.get_module_names(): - yield module_name, index.get_module(module_name).get_defaults() + if obsoletes: + yield module_name, index.get_module(module_name).get_obsoletes() + else: + yield module_name, index.get_module(module_name).get_defaults() def collect_module_defaults( @@ -69,3 +73,21 @@ def collect_module_defaults( mod_index.add_defaults(defaults) return mod_index + + +def collect_module_obsoletes(obsoletes_dir, modules_to_load, mod_index=None): + """Load module obsoletes into index. + + This works in a similar fashion as collect_module_defaults except the overrides_dir + feature. + """ + mod_index = mod_index or Modulemd.ModuleIndex() + + for module_name, obsoletes in iter_module_defaults_or_obsoletes( + obsoletes_dir, obsoletes=True + ): + for obsolete in obsoletes: + if not modules_to_load or module_name in modules_to_load: + mod_index.add_obsoletes(obsoletes) + + return mod_index diff --git a/pungi/paths.py b/pungi/paths.py index 6049b667..aff17a93 100644 --- a/pungi/paths.py +++ b/pungi/paths.py @@ -509,6 +509,16 @@ class WorkPaths(object): makedirs(path) return path + def module_obsoletes_dir(self, create_dir=True): + """ + Example: + work/global/module_obsoletes + """ + path = os.path.join(self.topdir(create_dir=create_dir), "module_obsoletes") + if create_dir: + makedirs(path) + return path + def pkgset_file_cache(self, pkgset_name): """ Returns the path to file in which the cached version of diff --git a/pungi/phases/createrepo.py b/pungi/phases/createrepo.py index 3df030a4..c170b382 100644 --- a/pungi/phases/createrepo.py +++ b/pungi/phases/createrepo.py @@ -29,7 +29,7 @@ import productmd.rpms from kobo.shortcuts import relative_path, run from kobo.threads import ThreadPool, WorkerThread -from ..module_util import Modulemd, collect_module_defaults +from ..module_util import Modulemd, collect_module_defaults, collect_module_obsoletes from ..util import ( get_arch_variant_data, read_single_module_stream_from_file, @@ -266,6 +266,9 @@ def create_variant_repo( defaults_dir, module_names, mod_index, overrides_dir=overrides_dir ) + obsoletes_dir = compose.paths.work.module_obsoletes_dir() + collect_module_obsoletes(obsoletes_dir, module_names, mod_index) + # Add extra modulemd files if variant.uid in compose.conf.get("createrepo_extra_modulemd", {}): compose.log_debug("Adding extra modulemd for %s.%s", variant.uid, arch) diff --git a/pungi/phases/gather/__init__.py b/pungi/phases/gather/__init__.py index 00af2ff1..09b57287 100644 --- a/pungi/phases/gather/__init__.py +++ b/pungi/phases/gather/__init__.py @@ -33,7 +33,11 @@ except ImportError: import pungi.wrappers.kojiwrapper from pungi.arch import get_compatible_arches, split_name_arch from pungi.compose import get_ordered_variant_uids -from pungi.module_util import Modulemd, collect_module_defaults +from pungi.module_util import ( + Modulemd, + collect_module_defaults, + collect_module_obsoletes, +) from pungi.phases.base import PhaseBase from pungi.phases.createrepo import add_modular_metadata from pungi.util import get_arch_data, get_arch_variant_data, get_variant_data, makedirs @@ -698,6 +702,8 @@ def _make_lookaside_repo(compose, variant, arch, pkg_map, package_sets=None): collect_module_defaults( defaults_dir, module_names, mod_index, overrides_dir=overrides_dir ) + obsoletes_dir = compose.paths.work.module_obsoletes_dir() + collect_module_obsoletes(obsoletes_dir, module_names, mod_index) log_file = compose.paths.log.log_file( arch, "lookaside_repo_modules_%s" % (variant.uid) diff --git a/pungi/phases/init.py b/pungi/phases/init.py index f0590128..30c8a742 100644 --- a/pungi/phases/init.py +++ b/pungi/phases/init.py @@ -24,7 +24,7 @@ from kobo.threads import run_in_threads from pungi.phases.base import PhaseBase from pungi.phases.gather import write_prepopulate_file from pungi.util import temp_dir -from pungi.module_util import iter_module_defaults +from pungi.module_util import iter_module_defaults_or_obsoletes from pungi.wrappers.comps import CompsWrapper from pungi.wrappers.createrepo import CreaterepoWrapper from pungi.wrappers.scm import get_dir_from_scm, get_file_from_scm @@ -68,10 +68,18 @@ class InitPhase(PhaseBase): # download module defaults if self.compose.has_module_defaults: write_module_defaults(self.compose) - validate_module_defaults( + validate_module_defaults_or_obsoletes( self.compose.paths.work.module_defaults_dir(create_dir=False) ) + # download module obsoletes + if self.compose.has_module_obsoletes: + write_module_obsoletes(self.compose) + validate_module_defaults_or_obsoletes( + self.compose.paths.work.module_obsoletes_dir(create_dir=False), + obsoletes=True, + ) + # write prepopulate file write_prepopulate_file(self.compose) @@ -218,28 +226,53 @@ def write_module_defaults(compose): ) -def validate_module_defaults(path): +def write_module_obsoletes(compose): + scm_dict = compose.conf["module_obsoletes_dir"] + if isinstance(scm_dict, dict): + if scm_dict["scm"] == "file": + scm_dict["dir"] = os.path.join(compose.config_dir, scm_dict["dir"]) + else: + scm_dict = os.path.join(compose.config_dir, scm_dict) + + with temp_dir(prefix="moduleobsoletes_") as tmp_dir: + get_dir_from_scm(scm_dict, tmp_dir, compose=compose) + compose.log_debug("Writing module obsoletes") + shutil.copytree( + tmp_dir, + compose.paths.work.module_obsoletes_dir(create_dir=False), + ignore=shutil.ignore_patterns(".git"), + ) + + +def validate_module_defaults_or_obsoletes(path, obsoletes=False): """Make sure there are no conflicting defaults. Each module name can only - have one default stream. + have one default stream or module obsolete. - :param str path: directory with cloned module defaults + :param str path: directory with cloned module defaults/obsoletes """ - seen_defaults = collections.defaultdict(set) + seen = collections.defaultdict(set) + mmd_type = "obsoletes" if obsoletes else "defaults" - for module_name, defaults in iter_module_defaults(path): - seen_defaults[module_name].add(defaults.get_default_stream()) + for module_name, defaults_or_obsoletes in iter_module_defaults_or_obsoletes( + path, obsoletes + ): + if obsoletes: + for obsolete in defaults_or_obsoletes: + seen[obsolete.props.module_name].add(obsolete) + else: + seen[module_name].add(defaults_or_obsoletes.get_default_stream()) errors = [] - for module_name, defaults in seen_defaults.items(): - if len(defaults) > 1: + for module_name, defaults_or_obsoletes in seen.items(): + if len(defaults_or_obsoletes) > 1: errors.append( - "Module %s has multiple defaults: %s" - % (module_name, ", ".join(sorted(defaults))) + "Module %s has multiple %s: %s" + % (module_name, mmd_type, ", ".join(sorted(defaults_or_obsoletes))) ) if errors: raise RuntimeError( - "There are duplicated module defaults:\n%s" % "\n".join(errors) + "There are duplicated module %s:\n%s" % (mmd_type, "\n".join(errors)) ) diff --git a/pungi/phases/pkgset/common.py b/pungi/phases/pkgset/common.py index f2a38457..14dad789 100644 --- a/pungi/phases/pkgset/common.py +++ b/pungi/phases/pkgset/common.py @@ -28,7 +28,11 @@ from pungi.util import ( PartialFuncWorkerThread, PartialFuncThreadPool, ) -from pungi.module_util import Modulemd, collect_module_defaults +from pungi.module_util import ( + Modulemd, + collect_module_defaults, + collect_module_obsoletes, +) from pungi.phases.createrepo import add_modular_metadata @@ -159,6 +163,9 @@ def _create_arch_repo(worker_thread, args, task_num): mod_index = collect_module_defaults( compose.paths.work.module_defaults_dir(), names, overrides_dir=overrides_dir ) + mod_index = collect_module_obsoletes( + compose.paths.work.module_obsoletes_dir(), names, mod_index + ) for x in mmd: mod_index.add_module_stream(x) add_modular_metadata( diff --git a/tests/test_initphase.py b/tests/test_initphase.py index afd2601e..f6fad433 100644 --- a/tests/test_initphase.py +++ b/tests/test_initphase.py @@ -24,7 +24,7 @@ from tests.helpers import ( @mock.patch("pungi.phases.init.run_in_threads", new=fake_run_in_threads) @mock.patch("pungi.phases.init.validate_comps") -@mock.patch("pungi.phases.init.validate_module_defaults") +@mock.patch("pungi.phases.init.validate_module_defaults_or_obsoletes") @mock.patch("pungi.phases.init.write_module_defaults") @mock.patch("pungi.phases.init.write_global_comps") @mock.patch("pungi.phases.init.write_arch_comps") @@ -46,6 +46,7 @@ class TestInitPhase(PungiTestCase): compose = DummyCompose(self.topdir, {}) compose.has_comps = True compose.has_module_defaults = False + compose.has_module_obsoletes = False compose.setup_optional() phase = init.InitPhase(compose) phase.run() @@ -100,6 +101,7 @@ class TestInitPhase(PungiTestCase): compose = DummyCompose(self.topdir, {}) compose.has_comps = True compose.has_module_defaults = False + compose.has_module_obsoletes = False compose.variants["Everything"].groups = [] compose.variants["Everything"].modules = [] phase = init.InitPhase(compose) @@ -156,6 +158,7 @@ class TestInitPhase(PungiTestCase): compose = DummyCompose(self.topdir, {}) compose.has_comps = False compose.has_module_defaults = False + compose.has_module_obsoletes = False phase = init.InitPhase(compose) phase.run() @@ -182,6 +185,7 @@ class TestInitPhase(PungiTestCase): compose = DummyCompose(self.topdir, {}) compose.has_comps = False compose.has_module_defaults = True + compose.has_module_obsoletes = False phase = init.InitPhase(compose) phase.run() @@ -620,13 +624,13 @@ class TestValidateModuleDefaults(PungiTestCase): def test_valid_files(self): self._write_defaults({"httpd": ["1"], "python": ["3.6"]}) - init.validate_module_defaults(self.topdir) + init.validate_module_defaults_or_obsoletes(self.topdir) def test_duplicated_stream(self): self._write_defaults({"httpd": ["1"], "python": ["3.6", "3.5"]}) with self.assertRaises(RuntimeError) as ctx: - init.validate_module_defaults(self.topdir) + init.validate_module_defaults_or_obsoletes(self.topdir) self.assertIn( "Module python has multiple defaults: 3.5, 3.6", str(ctx.exception) @@ -636,7 +640,7 @@ class TestValidateModuleDefaults(PungiTestCase): self._write_defaults({"httpd": ["1", "2"], "python": ["3.6", "3.5"]}) with self.assertRaises(RuntimeError) as ctx: - init.validate_module_defaults(self.topdir) + init.validate_module_defaults_or_obsoletes(self.topdir) self.assertIn("Module httpd has multiple defaults: 1, 2", str(ctx.exception)) self.assertIn( @@ -661,7 +665,7 @@ class TestValidateModuleDefaults(PungiTestCase): ), ) - init.validate_module_defaults(self.topdir) + init.validate_module_defaults_or_obsoletes(self.topdir) @mock.patch("pungi.phases.init.CompsWrapper")