Add module obsoletes feature

JIRA: MODULAR-113
Merges: https://pagure.io/pungi/pull-request/1578
Signed-off-by: Filip Valder <fvalder@redhat.com>

.gitignore

@@ -14,3 +14,5 @@ htmlcov/
 .idea/
 .tox
 .venv
+.kdev4/
+pungi.kdev4

doc/examples.rst

@@ -30,9 +30,17 @@ This is a shortened configuration for Fedora Radhide compose as of 2019-10-14.
     module_defaults_dir = {
         'scm': 'git',
         'repo': 'https://pagure.io/releng/fedora-module-defaults.git',
-        'branch': 'master',
+        'branch': 'main',
         'dir': '.'
     }
+    # Optional module obsoletes configuration which is merged
+    # into the module index and gets resolved
+    module_obsoletes_dir = {
+        'scm': 'git',
+        'repo': 'https://pagure.io/releng/fedora-module-defaults.git',
+        'branch': 'main',
+        'dir': 'obsoletes'
+    }
 
     variants_file='variants-fedora.xml'
     sigkeys = ['12C944D0']

pungi/checks.py

@@ -736,6 +736,7 @@ def make_schema():
                 "patternProperties": {".+": {"$ref": "#/definitions/strings"}},
                 "additionalProperties": False,
             },
+            "module_obsoletes_dir": {"$ref": "#/definitions/str_or_scm_dict"},
             "create_optional_isos": {"type": "boolean", "default": False},
             "symlink_isos_to": {"type": "string"},
             "dogpile_cache_backend": {"type": "string"},

pungi/compose.py

@@ -377,6 +377,10 @@ class Compose(kobo.log.LoggingBase):
     def has_module_defaults(self):
         return bool(self.conf.get("module_defaults_dir", False))
 
+    @property
+    def has_module_obsoletes(self):
+        return bool(self.conf.get("module_obsoletes_dir", False))
+
     @property
     def config_dir(self):
         return os.path.dirname(self.conf._open_file or "")

pungi/module_util.py

@@ -25,9 +25,10 @@ except (ImportError, ValueError):
     Modulemd = None
 
 
-def iter_module_defaults(path):
+def iter_module_defaults_or_obsoletes(path, obsoletes=False):
     """Given a path to a directory with yaml files, yield each module default
     in there as a pair (module_name, ModuleDefaults instance).
+    The same happens for module obsoletes if the obsoletes switch is True.
     """
     # It is really tempting to merge all the module indexes into a single one
     # and work with it. However that does not allow for detecting conflicting
@@ -41,7 +42,10 @@ def iter_module_defaults(path):
         index = Modulemd.ModuleIndex()
         index.update_from_file(file, strict=False)
         for module_name in index.get_module_names():
-            yield module_name, index.get_module(module_name).get_defaults()
+            if obsoletes:
+                yield module_name, index.get_module(module_name).get_obsoletes()
+            else:
+                yield module_name, index.get_module(module_name).get_defaults()
 
 
 def collect_module_defaults(
@@ -69,3 +73,21 @@ def collect_module_defaults(
             mod_index.add_defaults(defaults)
 
     return mod_index
+
+
+def collect_module_obsoletes(obsoletes_dir, modules_to_load, mod_index=None):
+    """Load module obsoletes into index.
+
+    This works in a similar fashion as collect_module_defaults except the overrides_dir
+    feature.
+    """
+    mod_index = mod_index or Modulemd.ModuleIndex()
+
+    for module_name, obsoletes in iter_module_defaults_or_obsoletes(
+        obsoletes_dir, obsoletes=True
+    ):
+        for obsolete in obsoletes:
+            if not modules_to_load or module_name in modules_to_load:
+                mod_index.add_obsoletes(obsoletes)
+
+    return mod_index

pungi/paths.py

@@ -509,6 +509,16 @@ class WorkPaths(object):
             makedirs(path)
         return path
 
+    def module_obsoletes_dir(self, create_dir=True):
+        """
+        Example:
+            work/global/module_obsoletes
+        """
+        path = os.path.join(self.topdir(create_dir=create_dir), "module_obsoletes")
+        if create_dir:
+            makedirs(path)
+        return path
+
     def pkgset_file_cache(self, pkgset_name):
         """
         Returns the path to file in which the cached version of

pungi/phases/createrepo.py

@@ -29,7 +29,7 @@ import productmd.rpms
 from kobo.shortcuts import relative_path, run
 from kobo.threads import ThreadPool, WorkerThread
 
-from ..module_util import Modulemd, collect_module_defaults
+from ..module_util import Modulemd, collect_module_defaults, collect_module_obsoletes
 from ..util import (
     get_arch_variant_data,
     read_single_module_stream_from_file,
@@ -266,6 +266,9 @@ def create_variant_repo(
             defaults_dir, module_names, mod_index, overrides_dir=overrides_dir
         )
 
+        obsoletes_dir = compose.paths.work.module_obsoletes_dir()
+        collect_module_obsoletes(obsoletes_dir, module_names, mod_index)
+
         # Add extra modulemd files
         if variant.uid in compose.conf.get("createrepo_extra_modulemd", {}):
             compose.log_debug("Adding extra modulemd for %s.%s", variant.uid, arch)

pungi/phases/gather/__init__.py

@@ -33,7 +33,11 @@ except ImportError:
 import pungi.wrappers.kojiwrapper
 from pungi.arch import get_compatible_arches, split_name_arch
 from pungi.compose import get_ordered_variant_uids
-from pungi.module_util import Modulemd, collect_module_defaults
+from pungi.module_util import (
+    Modulemd,
+    collect_module_defaults,
+    collect_module_obsoletes,
+)
 from pungi.phases.base import PhaseBase
 from pungi.phases.createrepo import add_modular_metadata
 from pungi.util import get_arch_data, get_arch_variant_data, get_variant_data, makedirs
@@ -698,6 +702,8 @@ def _make_lookaside_repo(compose, variant, arch, pkg_map, package_sets=None):
         collect_module_defaults(
             defaults_dir, module_names, mod_index, overrides_dir=overrides_dir
         )
+        obsoletes_dir = compose.paths.work.module_obsoletes_dir()
+        collect_module_obsoletes(obsoletes_dir, module_names, mod_index)
 
         log_file = compose.paths.log.log_file(
             arch, "lookaside_repo_modules_%s" % (variant.uid)

pungi/phases/init.py

@@ -24,7 +24,7 @@ from kobo.threads import run_in_threads
 from pungi.phases.base import PhaseBase
 from pungi.phases.gather import write_prepopulate_file
 from pungi.util import temp_dir
-from pungi.module_util import iter_module_defaults
+from pungi.module_util import iter_module_defaults_or_obsoletes
 from pungi.wrappers.comps import CompsWrapper
 from pungi.wrappers.createrepo import CreaterepoWrapper
 from pungi.wrappers.scm import get_dir_from_scm, get_file_from_scm
@@ -68,10 +68,18 @@ class InitPhase(PhaseBase):
         # download module defaults
         if self.compose.has_module_defaults:
             write_module_defaults(self.compose)
-            validate_module_defaults(
+            validate_module_defaults_or_obsoletes(
                 self.compose.paths.work.module_defaults_dir(create_dir=False)
             )
 
+        # download module obsoletes
+        if self.compose.has_module_obsoletes:
+            write_module_obsoletes(self.compose)
+            validate_module_defaults_or_obsoletes(
+                self.compose.paths.work.module_obsoletes_dir(create_dir=False),
+                obsoletes=True,
+            )
+
         # write prepopulate file
         write_prepopulate_file(self.compose)
 
@@ -218,28 +226,53 @@ def write_module_defaults(compose):
         )
 
 
-def validate_module_defaults(path):
+def write_module_obsoletes(compose):
+    scm_dict = compose.conf["module_obsoletes_dir"]
+    if isinstance(scm_dict, dict):
+        if scm_dict["scm"] == "file":
+            scm_dict["dir"] = os.path.join(compose.config_dir, scm_dict["dir"])
+    else:
+        scm_dict = os.path.join(compose.config_dir, scm_dict)
+
+    with temp_dir(prefix="moduleobsoletes_") as tmp_dir:
+        get_dir_from_scm(scm_dict, tmp_dir, compose=compose)
+        compose.log_debug("Writing module obsoletes")
+        shutil.copytree(
+            tmp_dir,
+            compose.paths.work.module_obsoletes_dir(create_dir=False),
+            ignore=shutil.ignore_patterns(".git"),
+        )
+
+
+def validate_module_defaults_or_obsoletes(path, obsoletes=False):
     """Make sure there are no conflicting defaults. Each module name can only
-    have one default stream.
+    have one default stream or module obsolete.
 
-    :param str path: directory with cloned module defaults
+    :param str path: directory with cloned module defaults/obsoletes
     """
-    seen_defaults = collections.defaultdict(set)
+    seen = collections.defaultdict(set)
+    mmd_type = "obsoletes" if obsoletes else "defaults"
 
-    for module_name, defaults in iter_module_defaults(path):
+    for module_name, defaults_or_obsoletes in iter_module_defaults_or_obsoletes(
-        seen_defaults[module_name].add(defaults.get_default_stream())
+        path, obsoletes
+    ):
+        if obsoletes:
+            for obsolete in defaults_or_obsoletes:
+                seen[obsolete.props.module_name].add(obsolete)
+        else:
+            seen[module_name].add(defaults_or_obsoletes.get_default_stream())
 
     errors = []
-    for module_name, defaults in seen_defaults.items():
+    for module_name, defaults_or_obsoletes in seen.items():
-        if len(defaults) > 1:
+        if len(defaults_or_obsoletes) > 1:
             errors.append(
-                "Module %s has multiple defaults: %s"
+                "Module %s has multiple %s: %s"
-                % (module_name, ", ".join(sorted(defaults)))
+                % (module_name, mmd_type, ", ".join(sorted(defaults_or_obsoletes)))
             )
 
     if errors:
         raise RuntimeError(
-            "There are duplicated module defaults:\n%s" % "\n".join(errors)
+            "There are duplicated module %s:\n%s" % (mmd_type, "\n".join(errors))
         )
 
 

pungi/phases/pkgset/common.py

@@ -28,7 +28,11 @@ from pungi.util import (
     PartialFuncWorkerThread,
     PartialFuncThreadPool,
 )
-from pungi.module_util import Modulemd, collect_module_defaults
+from pungi.module_util import (
+    Modulemd,
+    collect_module_defaults,
+    collect_module_obsoletes,
+)
 from pungi.phases.createrepo import add_modular_metadata
 
 
@@ -159,6 +163,9 @@ def _create_arch_repo(worker_thread, args, task_num):
         mod_index = collect_module_defaults(
             compose.paths.work.module_defaults_dir(), names, overrides_dir=overrides_dir
         )
+        mod_index = collect_module_obsoletes(
+            compose.paths.work.module_obsoletes_dir(), names, mod_index
+        )
         for x in mmd:
             mod_index.add_module_stream(x)
         add_modular_metadata(

tests/test_initphase.py

@@ -24,7 +24,7 @@ from tests.helpers import (
 
 @mock.patch("pungi.phases.init.run_in_threads", new=fake_run_in_threads)
 @mock.patch("pungi.phases.init.validate_comps")
-@mock.patch("pungi.phases.init.validate_module_defaults")
+@mock.patch("pungi.phases.init.validate_module_defaults_or_obsoletes")
 @mock.patch("pungi.phases.init.write_module_defaults")
 @mock.patch("pungi.phases.init.write_global_comps")
 @mock.patch("pungi.phases.init.write_arch_comps")
@@ -46,6 +46,7 @@ class TestInitPhase(PungiTestCase):
         compose = DummyCompose(self.topdir, {})
         compose.has_comps = True
         compose.has_module_defaults = False
+        compose.has_module_obsoletes = False
         compose.setup_optional()
         phase = init.InitPhase(compose)
         phase.run()
@@ -100,6 +101,7 @@ class TestInitPhase(PungiTestCase):
         compose = DummyCompose(self.topdir, {})
         compose.has_comps = True
         compose.has_module_defaults = False
+        compose.has_module_obsoletes = False
         compose.variants["Everything"].groups = []
         compose.variants["Everything"].modules = []
         phase = init.InitPhase(compose)
@@ -156,6 +158,7 @@ class TestInitPhase(PungiTestCase):
         compose = DummyCompose(self.topdir, {})
         compose.has_comps = False
         compose.has_module_defaults = False
+        compose.has_module_obsoletes = False
         phase = init.InitPhase(compose)
         phase.run()
 
@@ -182,6 +185,7 @@ class TestInitPhase(PungiTestCase):
         compose = DummyCompose(self.topdir, {})
         compose.has_comps = False
         compose.has_module_defaults = True
+        compose.has_module_obsoletes = False
         phase = init.InitPhase(compose)
         phase.run()
 
@@ -620,13 +624,13 @@ class TestValidateModuleDefaults(PungiTestCase):
     def test_valid_files(self):
         self._write_defaults({"httpd": ["1"], "python": ["3.6"]})
 
-        init.validate_module_defaults(self.topdir)
+        init.validate_module_defaults_or_obsoletes(self.topdir)
 
     def test_duplicated_stream(self):
         self._write_defaults({"httpd": ["1"], "python": ["3.6", "3.5"]})
 
         with self.assertRaises(RuntimeError) as ctx:
-            init.validate_module_defaults(self.topdir)
+            init.validate_module_defaults_or_obsoletes(self.topdir)
 
         self.assertIn(
             "Module python has multiple defaults: 3.5, 3.6", str(ctx.exception)
@@ -636,7 +640,7 @@ class TestValidateModuleDefaults(PungiTestCase):
         self._write_defaults({"httpd": ["1", "2"], "python": ["3.6", "3.5"]})
 
         with self.assertRaises(RuntimeError) as ctx:
-            init.validate_module_defaults(self.topdir)
+            init.validate_module_defaults_or_obsoletes(self.topdir)
 
         self.assertIn("Module httpd has multiple defaults: 1, 2", str(ctx.exception))
         self.assertIn(
@@ -661,7 +665,7 @@ class TestValidateModuleDefaults(PungiTestCase):
             ),
         )
 
-        init.validate_module_defaults(self.topdir)
+        init.validate_module_defaults_or_obsoletes(self.topdir)
 
 
 @mock.patch("pungi.phases.init.CompsWrapper")