pkgset: Allow filtering modules from Koji tags

Add a configuration option to enable skipping some modules found in the
configured tag.

Fixes: https://pagure.io/pungi/issue/1260
JIRA: COMPOSE-3794
Signed-off-by: Lubomír Sedlář <lsedlar@redhat.com>

doc/configuration.rst

@@ -700,6 +700,16 @@ Options
     See :ref:`additional_packages <additional_packages>` for details about
     package specification.
 
+**filter_modules**
+    (*list*) -- modules to be excluded from a variant and architecture;
+    format: ``[(variant_uid_regex, {arch|*: [name:stream]})]``
+
+    Both name and stream can use shell-style globs. If stream is omitted, all
+    streams are removed.
+
+    This option only applies to modules taken from Koji tags, not modules
+    explicitly listed in variants XML without any tags.
+
 **filter_system_release_packages**
     (*bool*) -- for each variant, figure out the best system release package
     and filter out all others. This will not work if a variant needs more than

pungi/checks.py

@@ -733,6 +733,10 @@ def make_schema():
                 "$ref": "#/definitions/package_mapping",
                 "default": [],
             },
+            "filter_modules": {
+                "$ref": "#/definitions/package_mapping",
+                "default": [],
+            },
             "sigkeys": {
                 "type": "array",
                 "items": {"$ref": "#/definitions/optional_string"},

pungi/phases/pkgset/sources/source_koji.py

@@ -17,6 +17,7 @@
 import os
 import json
 import re
+from fnmatch import fnmatch
 from itertools import groupby
 
 from kobo.rpmlib import parse_nvra
@@ -26,7 +27,7 @@ import pungi.wrappers.kojiwrapper
 from pungi.wrappers.comps import CompsWrapper
 import pungi.phases.pkgset.pkgsets
 from pungi.arch import getBaseArch
-from pungi.util import retry, find_old_compose
+from pungi.util import retry, find_old_compose, get_arch_variant_data
 from pungi import Modulemd
 
 from pungi.phases.pkgset.common import MaterializedPackageSet, get_all_arches
@@ -190,7 +191,9 @@ def get_pkgset_from_koji(compose, koji_wrapper, path_prefix):
     return populate_global_pkgset(compose, koji_wrapper, path_prefix, event_info)
 
 
-def _add_module_to_variant(koji_wrapper, variant, build, add_to_variant_modules=False):
+def _add_module_to_variant(
+    koji_wrapper, variant, build, add_to_variant_modules=False, compose=None
+):
     """
     Adds module defined by Koji build info to variant.
 
@@ -198,6 +201,7 @@ def _add_module_to_variant(koji_wrapper, variant, build, add_to_variant_modules=
     :param int: build id
     :param bool add_to_variant_modules: Adds the modules also to
         variant.modules.
+    :param compose: Compose object to get filters from
     """
     mmds = {}
     archives = koji_wrapper.koji_proxy.listArchives(build["id"])
@@ -225,26 +229,57 @@ def _add_module_to_variant(koji_wrapper, variant, build, add_to_variant_modules=
         # longer supported and should be rebuilt. Let's skip it.
         return
 
-    nsvc = "%(name)s:%(stream)s:%(version)s:%(context)s" % build["extra"]["typeinfo"]["module"]
+    info = build["extra"]["typeinfo"]["module"]
+    nsvc = "%(name)s:%(stream)s:%(version)s:%(context)s" % info
+
+    added = False
 
     for arch in variant.arches:
+        if _is_filtered_out(compose, variant, arch, info["name"], info["stream"]):
+            compose.log_debug("Module %s is filtered from %s.%s", nsvc, variant, arch)
+            continue
+
         try:
             mmd = Modulemd.ModuleStream.read_file(
                 mmds["modulemd.%s.txt" % arch], strict=True
             )
             variant.arch_mmds.setdefault(arch, {})[nsvc] = mmd
+            added = True
         except KeyError:
             # There is no modulemd for this arch. This could mean an arch was
             # added to the compose after the module was built. We don't want to
             # process this, let's skip this module.
             pass
 
+    if not added:
+        # The module is filtered on all arches of this variant.
+        return None
+
     if add_to_variant_modules:
         variant.modules.append({"name": nsvc, "glob": False})
 
     return nsvc
 
 
+def _is_filtered_out(compose, variant, arch, module_name, module_stream):
+    """Check if module with given name and stream is filter out from this stream.
+    """
+    if not compose:
+        return False
+
+    for filter in get_arch_variant_data(compose.conf, "filter_modules", arch, variant):
+        if ":" not in filter:
+            name_filter = filter
+            stream_filter = "*"
+        else:
+            name_filter, stream_filter = filter.split(":", 1)
+
+        if fnmatch(module_name, name_filter) and fnmatch(module_stream, stream_filter):
+            return True
+
+    return False
+
+
 def _get_modules_from_koji(
     compose, koji_wrapper, event, variant, variant_tags, tag_to_mmd
 ):
@@ -264,26 +299,34 @@ def _get_modules_from_koji(
     for module in variant.get_modules():
         koji_modules = get_koji_modules(compose, koji_wrapper, event, module["name"])
         for koji_module in koji_modules:
-            nsvc = _add_module_to_variant(koji_wrapper, variant, koji_module)
+            nsvc = _add_module_to_variant(
+                koji_wrapper, variant, koji_module, compose=compose
+            )
             if not nsvc:
                 continue
 
             tag = koji_module["tag"]
             variant_tags[variant].append(tag)
 
-            # Store mapping NSVC --> koji_tag into variant.
-            # This is needed in createrepo phase where metadata is exposed by producmd
-            variant.module_uid_to_koji_tag[nsvc] = tag
-
             tag_to_mmd.setdefault(tag, {})
             for arch in variant.arch_mmds:
-                tag_to_mmd[tag].setdefault(arch, set()).add(variant.arch_mmds[arch][nsvc])
+                try:
+                    mmd = variant.arch_mmds[arch][nsvc]
+                except KeyError:
+                    # Module was filtered from here
+                    continue
+                tag_to_mmd[tag].setdefault(arch, set()).add(mmd)
 
-            module_msg = (
-                "Module '{uid}' in variant '{variant}' will use Koji tag '{tag}' "
-                "(as a result of querying module '{module}')"
-            ).format(uid=nsvc, variant=variant, tag=tag, module=module["name"])
-            compose.log_info("%s" % module_msg)
+            if tag_to_mmd[tag]:
+                compose.log_info(
+                    "Module '%s' in variant '%s' will use Koji tag '%s' "
+                    "(as a result of querying module '%s')",
+                    nsvc, variant, tag, module["name"]
+                )
+
+                # Store mapping NSVC --> koji_tag into variant. This is needed
+                # in createrepo phase where metadata is exposed by producmd
+                variant.module_uid_to_koji_tag[nsvc] = tag
 
 
 def filter_inherited(koji_proxy, event, module_builds, top_tag):
@@ -449,21 +492,31 @@ def _get_modules_from_koji_tags(
 
             variant_tags[variant].append(module_tag)
 
-            nsvc = _add_module_to_variant(koji_wrapper, variant, build, True)
+            nsvc = _add_module_to_variant(
+                koji_wrapper, variant, build, True, compose=compose
+            )
             if not nsvc:
                 continue
 
-            # Store mapping module-uid --> koji_tag into variant.
-            # This is needed in createrepo phase where metadata is exposed by producmd
-            variant.module_uid_to_koji_tag[nsvc] = module_tag
-
             tag_to_mmd.setdefault(module_tag, {})
             for arch in variant.arch_mmds:
-                tag_to_mmd[module_tag].setdefault(arch, set()).add(variant.arch_mmds[arch][nsvc])
+                try:
+                    mmd = variant.arch_mmds[arch][nsvc]
+                except KeyError:
+                    # Module was filtered from here
+                    continue
+                tag_to_mmd[module_tag].setdefault(arch, set()).add(mmd)
 
-            module_msg = "Module {module} in variant {variant} will use Koji tag {tag}.".format(
-                variant=variant, tag=module_tag, module=build["nvr"])
-            compose.log_info("%s" % module_msg)
+            if tag_to_mmd[module_tag]:
+                compose.log_info(
+                    "Module %s in variant %s will use Koji tag %s.",
+                    nsvc, variant, module_tag
+                )
+
+                # Store mapping module-uid --> koji_tag into variant. This is
+                # needed in createrepo phase where metadata is exposed by
+                # productmd
+                variant.module_uid_to_koji_tag[nsvc] = module_tag
 
     if expected_modules:
         # There are some module names that were listed in configuration and not
@@ -624,7 +677,12 @@ def populate_global_pkgset(compose, koji_wrapper, path_prefix, event):
                     # Not current tag, skip it
                     continue
                 for arch_modules in variant.arch_mmds.values():
-                    for rpm_nevra in arch_modules[nsvc].get_rpm_artifacts():
+                    try:
+                        module = arch_modules[nsvc]
+                    except KeyError:
+                        # The module was filtered out
+                        continue
+                    for rpm_nevra in module.get_rpm_artifacts():
                         nevra = parse_nvra(rpm_nevra)
                         modular_packages.add((nevra["name"], nevra["arch"]))
 

tests/test_pkgset_source_koji.py

@@ -615,9 +615,10 @@ class MockModule(object):
 
 @mock.patch("pungi.Modulemd.ModuleStream.read_file", new=MockModule)
 @unittest.skipIf(Modulemd is None, "Skipping tests, no module support")
-class TestAddModuleToVariant(unittest.TestCase):
+class TestAddModuleToVariant(helpers.PungiTestCase):
 
     def setUp(self):
+        super(TestAddModuleToVariant, self).setUp()
         self.koji = mock.Mock()
         self.koji.koji_module.pathinfo.typedir.return_value = "/koji"
         files = ["modulemd.x86_64.txt", "modulemd.armv7hl.txt", "modulemd.txt"]
@@ -741,3 +742,61 @@ class TestAddModuleToVariant(unittest.TestCase):
                 {"name": "module:master:20190318:abcdef", "glob": False},
             ],
         )
+
+    def test_adding_module_but_filtered(self):
+        compose = helpers.DummyCompose(
+            self.topdir, {"filter_modules": [(".*", {"*": ["module:*"]})]}
+        )
+        variant = mock.Mock(
+            arches=["armhfp", "x86_64"], arch_mmds={}, modules=[], uid="Variant"
+        )
+
+        nsvc = source_koji._add_module_to_variant(
+            self.koji,
+            variant,
+            self.buildinfo,
+            add_to_variant_modules=True,
+            compose=compose,
+        )
+
+        self.assertIsNone(nsvc)
+        self.assertEqual(variant.arch_mmds, {})
+        self.assertEqual(variant.modules, [])
+
+
+class TestIsModuleFiltered(helpers.PungiTestCase):
+    def assertIsFiltered(self, name, stream):
+        self.assertTrue(
+            source_koji._is_filtered_out(
+                self.compose, self.compose.variants["Server"], "x86_64", name, stream
+            )
+        )
+
+    def assertIsNotFiltered(self, name, stream):
+        self.assertFalse(
+            source_koji._is_filtered_out(
+                self.compose, self.compose.variants["Server"], "x86_64", name, stream
+            )
+        )
+
+    def test_no_filters(self):
+        self.compose = helpers.DummyCompose(self.topdir, {})
+
+        self.assertIsNotFiltered("foo", "master")
+
+    def test_filter_by_name(self):
+        self.compose = helpers.DummyCompose(
+            self.topdir, {"filter_modules": [(".*", {"*": ["foo"]})]}
+        )
+
+        self.assertIsFiltered("foo", "master")
+        self.assertIsNotFiltered("bar", "master")
+
+    def test_filter_by_stream(self):
+        self.compose = helpers.DummyCompose(
+            self.topdir, {"filter_modules": [(".*", {"*": ["foo:master"]})]}
+        )
+
+        self.assertIsFiltered("foo", "master")
+        self.assertIsNotFiltered("bar", "master")
+        self.assertIsNotFiltered("foo", "stable")