diff --git a/pungi/phases/pkgset/sources/source_koji.py b/pungi/phases/pkgset/sources/source_koji.py index a5bd001f..b1069543 100644 --- a/pungi/phases/pkgset/sources/source_koji.py +++ b/pungi/phases/pkgset/sources/source_koji.py @@ -335,16 +335,20 @@ def filter_inherited(koji_proxy, event, module_builds, top_tag): return result -def filter_by_whitelist(compose, module_builds, input_modules): +def filter_by_whitelist(compose, module_builds, input_modules, expected_modules): """ Exclude modules from the list that do not match any pattern specified in - input_modules. Order may not be preserved. + input_modules. Order may not be preserved. The last argument is a set of + module patterns that are expected across module tags. When a matching + module is found, the corresponding pattern is removed from the set. """ - specs = set() nvr_patterns = set() for spec in input_modules: - # Do not do any filtering in case variant wants all the modules. + # Do not do any filtering in case variant wants all the modules. Also + # empty the set of remaining expected modules, as the check does not + # really make much sense here. if spec["name"] == "*": + expected_modules.clear() return module_builds info = variant_dict_from_str(compose, spec["name"]) @@ -355,10 +359,8 @@ def filter_by_whitelist(compose, module_builds, input_modules): info.get("context"), ) nvr_patterns.add((pattern, spec["name"])) - specs.add(spec["name"]) modules_to_keep = [] - used = set() for mb in module_builds: # Split release from the build into version and context @@ -377,15 +379,9 @@ def filter_by_whitelist(compose, module_builds, input_modules): and (not c or ctx == c) ): modules_to_keep.append(mb) - used.add(spec) + expected_modules.discard(spec) break - if used != specs: - raise RuntimeError( - "Configuration specified patterns (%s) that don't match any modules in the configured tags." - % ", ".join(specs - used) - ) - return modules_to_keep @@ -405,6 +401,9 @@ def _get_modules_from_koji_tags(compose, koji_wrapper, event_id, variant, varian compose_tags = [ {"name": tag} for tag in force_list(compose.conf["pkgset_koji_module_tag"]) ] + # Get set of configured module names for this variant. If nothing is + # configured, the set is empty. + expected_modules = set(spec["name"] for spec in variant.get_modules()) # Find out all modules in every variant and add their Koji tags # to variant and variant_tags list. koji_proxy = koji_wrapper.koji_proxy @@ -425,7 +424,9 @@ def _get_modules_from_koji_tags(compose, koji_wrapper, event_id, variant, varian # Apply whitelist of modules if specified. variant_modules = variant.get_modules() if variant_modules: - module_builds = filter_by_whitelist(compose, module_builds, variant_modules) + module_builds = filter_by_whitelist( + compose, module_builds, variant_modules, expected_modules + ) # Find the latest builds of all modules. This does following: # - Sorts the module_builds descending by Koji NVR (which maps to NSV @@ -476,6 +477,14 @@ def _get_modules_from_koji_tags(compose, koji_wrapper, event_id, variant, varian variant=variant, tag=module_tag, module=build["nvr"]) compose.log_info("%s" % module_msg) + if expected_modules: + # There are some module names that were listed in configuration and not + # found in any tag... + raise RuntimeError( + "Configuration specified patterns (%s) that don't match any modules in the configured tags." + % ", ".join(expected_modules) + ) + def _find_old_file_cache_path(compose): """ diff --git a/tests/test_pkgset_source_koji.py b/tests/test_pkgset_source_koji.py index 12568a32..9946a8e7 100644 --- a/tests/test_pkgset_source_koji.py +++ b/tests/test_pkgset_source_koji.py @@ -598,11 +598,11 @@ class TestFilterByWhitelist(unittest.TestCase): compose = mock.Mock() module_builds = [] input_modules = [{"name": "foo:1"}] + expected = set(["foo:1"]) - with self.assertRaises(RuntimeError) as ctx: - source_koji.filter_by_whitelist(compose, module_builds, input_modules) + source_koji.filter_by_whitelist(compose, module_builds, input_modules, expected) - self.assertIn("patterns (foo:1) that don't match", str(ctx.exception)) + self.assertEqual(expected, set(["foo:1"])) def test_filter_by_NS(self): compose = mock.Mock() @@ -612,10 +612,14 @@ class TestFilterByWhitelist(unittest.TestCase): self._build("foo", "2", "201809031047", "deadbeef"), ] input_modules = [{"name": "foo:1"}] + expected = set(["foo:1"]) - result = source_koji.filter_by_whitelist(compose, module_builds, input_modules) + result = source_koji.filter_by_whitelist( + compose, module_builds, input_modules, expected + ) self.assertItemsEqual(result, [module_builds[0], module_builds[1]]) + self.assertEqual(expected, set()) def test_filter_by_NSV(self): compose = mock.Mock() @@ -625,10 +629,14 @@ class TestFilterByWhitelist(unittest.TestCase): self._build("foo", "2", "201809031047", "deadbeef"), ] input_modules = [{"name": "foo:1:201809031047"}] + expected = set(["foo:1:201809031047"]) - result = source_koji.filter_by_whitelist(compose, module_builds, input_modules) + result = source_koji.filter_by_whitelist( + compose, module_builds, input_modules, expected + ) self.assertItemsEqual(result, [module_builds[1]]) + self.assertEqual(expected, set()) def test_filter_by_NSVC(self): compose = mock.Mock() @@ -639,10 +647,14 @@ class TestFilterByWhitelist(unittest.TestCase): self._build("foo", "2", "201809031047", "deadbeef"), ] input_modules = [{"name": "foo:1:201809031047:deadbeef"}] + expected = set() - result = source_koji.filter_by_whitelist(compose, module_builds, input_modules) + result = source_koji.filter_by_whitelist( + compose, module_builds, input_modules, expected + ) self.assertItemsEqual(result, [module_builds[1]]) + self.assertEqual(expected, set()) def test_filter_by_wildcard(self): compose = mock.Mock() @@ -652,10 +664,14 @@ class TestFilterByWhitelist(unittest.TestCase): self._build("foo", "2", "201809031047", "deadbeef"), ] input_modules = [{"name": "*"}] + expected = set(["*"]) - result = source_koji.filter_by_whitelist(compose, module_builds, input_modules) + result = source_koji.filter_by_whitelist( + compose, module_builds, input_modules, expected + ) self.assertItemsEqual(result, module_builds) + self.assertEqual(expected, set()) class MockModule(object):