diff --git a/doc/configuration.rst b/doc/configuration.rst index aad91e42..f221ee44 100644 --- a/doc/configuration.rst +++ b/doc/configuration.rst @@ -700,6 +700,16 @@ Options See :ref:`additional_packages ` for details about package specification. +**filter_modules** + (*list*) -- modules to be excluded from a variant and architecture; + format: ``[(variant_uid_regex, {arch|*: [name:stream]})]`` + + Both name and stream can use shell-style globs. If stream is omitted, all + streams are removed. + + This option only applies to modules taken from Koji tags, not modules + explicitly listed in variants XML without any tags. + **filter_system_release_packages** (*bool*) -- for each variant, figure out the best system release package and filter out all others. This will not work if a variant needs more than diff --git a/pungi/checks.py b/pungi/checks.py index 7e9f65a7..9451f493 100644 --- a/pungi/checks.py +++ b/pungi/checks.py @@ -733,6 +733,10 @@ def make_schema(): "$ref": "#/definitions/package_mapping", "default": [], }, + "filter_modules": { + "$ref": "#/definitions/package_mapping", + "default": [], + }, "sigkeys": { "type": "array", "items": {"$ref": "#/definitions/optional_string"}, diff --git a/pungi/phases/pkgset/sources/source_koji.py b/pungi/phases/pkgset/sources/source_koji.py index 2bc61d28..3a0dfb52 100644 --- a/pungi/phases/pkgset/sources/source_koji.py +++ b/pungi/phases/pkgset/sources/source_koji.py @@ -17,6 +17,7 @@ import os import json import re +from fnmatch import fnmatch from itertools import groupby from kobo.rpmlib import parse_nvra @@ -26,7 +27,7 @@ import pungi.wrappers.kojiwrapper from pungi.wrappers.comps import CompsWrapper import pungi.phases.pkgset.pkgsets from pungi.arch import getBaseArch -from pungi.util import retry, find_old_compose +from pungi.util import retry, find_old_compose, get_arch_variant_data from pungi import Modulemd from pungi.phases.pkgset.common import MaterializedPackageSet, get_all_arches @@ -190,7 +191,9 @@ def get_pkgset_from_koji(compose, koji_wrapper, path_prefix): return populate_global_pkgset(compose, koji_wrapper, path_prefix, event_info) -def _add_module_to_variant(koji_wrapper, variant, build, add_to_variant_modules=False): +def _add_module_to_variant( + koji_wrapper, variant, build, add_to_variant_modules=False, compose=None +): """ Adds module defined by Koji build info to variant. @@ -198,6 +201,7 @@ def _add_module_to_variant(koji_wrapper, variant, build, add_to_variant_modules= :param int: build id :param bool add_to_variant_modules: Adds the modules also to variant.modules. + :param compose: Compose object to get filters from """ mmds = {} archives = koji_wrapper.koji_proxy.listArchives(build["id"]) @@ -225,26 +229,57 @@ def _add_module_to_variant(koji_wrapper, variant, build, add_to_variant_modules= # longer supported and should be rebuilt. Let's skip it. return - nsvc = "%(name)s:%(stream)s:%(version)s:%(context)s" % build["extra"]["typeinfo"]["module"] + info = build["extra"]["typeinfo"]["module"] + nsvc = "%(name)s:%(stream)s:%(version)s:%(context)s" % info + + added = False for arch in variant.arches: + if _is_filtered_out(compose, variant, arch, info["name"], info["stream"]): + compose.log_debug("Module %s is filtered from %s.%s", nsvc, variant, arch) + continue + try: mmd = Modulemd.ModuleStream.read_file( mmds["modulemd.%s.txt" % arch], strict=True ) variant.arch_mmds.setdefault(arch, {})[nsvc] = mmd + added = True except KeyError: # There is no modulemd for this arch. This could mean an arch was # added to the compose after the module was built. We don't want to # process this, let's skip this module. pass + if not added: + # The module is filtered on all arches of this variant. + return None + if add_to_variant_modules: variant.modules.append({"name": nsvc, "glob": False}) return nsvc +def _is_filtered_out(compose, variant, arch, module_name, module_stream): + """Check if module with given name and stream is filter out from this stream. + """ + if not compose: + return False + + for filter in get_arch_variant_data(compose.conf, "filter_modules", arch, variant): + if ":" not in filter: + name_filter = filter + stream_filter = "*" + else: + name_filter, stream_filter = filter.split(":", 1) + + if fnmatch(module_name, name_filter) and fnmatch(module_stream, stream_filter): + return True + + return False + + def _get_modules_from_koji( compose, koji_wrapper, event, variant, variant_tags, tag_to_mmd ): @@ -264,26 +299,34 @@ def _get_modules_from_koji( for module in variant.get_modules(): koji_modules = get_koji_modules(compose, koji_wrapper, event, module["name"]) for koji_module in koji_modules: - nsvc = _add_module_to_variant(koji_wrapper, variant, koji_module) + nsvc = _add_module_to_variant( + koji_wrapper, variant, koji_module, compose=compose + ) if not nsvc: continue tag = koji_module["tag"] variant_tags[variant].append(tag) - # Store mapping NSVC --> koji_tag into variant. - # This is needed in createrepo phase where metadata is exposed by producmd - variant.module_uid_to_koji_tag[nsvc] = tag - tag_to_mmd.setdefault(tag, {}) for arch in variant.arch_mmds: - tag_to_mmd[tag].setdefault(arch, set()).add(variant.arch_mmds[arch][nsvc]) + try: + mmd = variant.arch_mmds[arch][nsvc] + except KeyError: + # Module was filtered from here + continue + tag_to_mmd[tag].setdefault(arch, set()).add(mmd) - module_msg = ( - "Module '{uid}' in variant '{variant}' will use Koji tag '{tag}' " - "(as a result of querying module '{module}')" - ).format(uid=nsvc, variant=variant, tag=tag, module=module["name"]) - compose.log_info("%s" % module_msg) + if tag_to_mmd[tag]: + compose.log_info( + "Module '%s' in variant '%s' will use Koji tag '%s' " + "(as a result of querying module '%s')", + nsvc, variant, tag, module["name"] + ) + + # Store mapping NSVC --> koji_tag into variant. This is needed + # in createrepo phase where metadata is exposed by producmd + variant.module_uid_to_koji_tag[nsvc] = tag def filter_inherited(koji_proxy, event, module_builds, top_tag): @@ -449,21 +492,31 @@ def _get_modules_from_koji_tags( variant_tags[variant].append(module_tag) - nsvc = _add_module_to_variant(koji_wrapper, variant, build, True) + nsvc = _add_module_to_variant( + koji_wrapper, variant, build, True, compose=compose + ) if not nsvc: continue - # Store mapping module-uid --> koji_tag into variant. - # This is needed in createrepo phase where metadata is exposed by producmd - variant.module_uid_to_koji_tag[nsvc] = module_tag - tag_to_mmd.setdefault(module_tag, {}) for arch in variant.arch_mmds: - tag_to_mmd[module_tag].setdefault(arch, set()).add(variant.arch_mmds[arch][nsvc]) + try: + mmd = variant.arch_mmds[arch][nsvc] + except KeyError: + # Module was filtered from here + continue + tag_to_mmd[module_tag].setdefault(arch, set()).add(mmd) - module_msg = "Module {module} in variant {variant} will use Koji tag {tag}.".format( - variant=variant, tag=module_tag, module=build["nvr"]) - compose.log_info("%s" % module_msg) + if tag_to_mmd[module_tag]: + compose.log_info( + "Module %s in variant %s will use Koji tag %s.", + nsvc, variant, module_tag + ) + + # Store mapping module-uid --> koji_tag into variant. This is + # needed in createrepo phase where metadata is exposed by + # productmd + variant.module_uid_to_koji_tag[nsvc] = module_tag if expected_modules: # There are some module names that were listed in configuration and not @@ -624,7 +677,12 @@ def populate_global_pkgset(compose, koji_wrapper, path_prefix, event): # Not current tag, skip it continue for arch_modules in variant.arch_mmds.values(): - for rpm_nevra in arch_modules[nsvc].get_rpm_artifacts(): + try: + module = arch_modules[nsvc] + except KeyError: + # The module was filtered out + continue + for rpm_nevra in module.get_rpm_artifacts(): nevra = parse_nvra(rpm_nevra) modular_packages.add((nevra["name"], nevra["arch"])) diff --git a/tests/test_pkgset_source_koji.py b/tests/test_pkgset_source_koji.py index 9dda3fba..bd939683 100644 --- a/tests/test_pkgset_source_koji.py +++ b/tests/test_pkgset_source_koji.py @@ -615,9 +615,10 @@ class MockModule(object): @mock.patch("pungi.Modulemd.ModuleStream.read_file", new=MockModule) @unittest.skipIf(Modulemd is None, "Skipping tests, no module support") -class TestAddModuleToVariant(unittest.TestCase): +class TestAddModuleToVariant(helpers.PungiTestCase): def setUp(self): + super(TestAddModuleToVariant, self).setUp() self.koji = mock.Mock() self.koji.koji_module.pathinfo.typedir.return_value = "/koji" files = ["modulemd.x86_64.txt", "modulemd.armv7hl.txt", "modulemd.txt"] @@ -741,3 +742,61 @@ class TestAddModuleToVariant(unittest.TestCase): {"name": "module:master:20190318:abcdef", "glob": False}, ], ) + + def test_adding_module_but_filtered(self): + compose = helpers.DummyCompose( + self.topdir, {"filter_modules": [(".*", {"*": ["module:*"]})]} + ) + variant = mock.Mock( + arches=["armhfp", "x86_64"], arch_mmds={}, modules=[], uid="Variant" + ) + + nsvc = source_koji._add_module_to_variant( + self.koji, + variant, + self.buildinfo, + add_to_variant_modules=True, + compose=compose, + ) + + self.assertIsNone(nsvc) + self.assertEqual(variant.arch_mmds, {}) + self.assertEqual(variant.modules, []) + + +class TestIsModuleFiltered(helpers.PungiTestCase): + def assertIsFiltered(self, name, stream): + self.assertTrue( + source_koji._is_filtered_out( + self.compose, self.compose.variants["Server"], "x86_64", name, stream + ) + ) + + def assertIsNotFiltered(self, name, stream): + self.assertFalse( + source_koji._is_filtered_out( + self.compose, self.compose.variants["Server"], "x86_64", name, stream + ) + ) + + def test_no_filters(self): + self.compose = helpers.DummyCompose(self.topdir, {}) + + self.assertIsNotFiltered("foo", "master") + + def test_filter_by_name(self): + self.compose = helpers.DummyCompose( + self.topdir, {"filter_modules": [(".*", {"*": ["foo"]})]} + ) + + self.assertIsFiltered("foo", "master") + self.assertIsNotFiltered("bar", "master") + + def test_filter_by_stream(self): + self.compose = helpers.DummyCompose( + self.topdir, {"filter_modules": [(".*", {"*": ["foo:master"]})]} + ) + + self.assertIsFiltered("foo", "master") + self.assertIsNotFiltered("bar", "master") + self.assertIsNotFiltered("foo", "stable")