From e68e17d8fe09856febfca326137c7cd5036c318a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lubom=C3=ADr=20Sedl=C3=A1=C5=99?= Date: Thu, 13 Jun 2019 10:10:30 +0200 Subject: [PATCH] pkgset: Check for unused module patterns across all tags MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Since there can be multiple tags, the check must be done once for all of them at the same time. Otherwise any module found only in some and not all tags would raise this error. The code builds a set of all existing patterns and then removes items from it. If there is something left once all tags are processed, it means such patterns were not matched by anything. JIRA: COMPOSE-3609 Signed-off-by: Lubomír Sedlář --- pungi/phases/pkgset/sources/source_koji.py | 37 ++++++++++++++-------- tests/test_pkgset_source_koji.py | 30 ++++++++++++++---- 2 files changed, 46 insertions(+), 21 deletions(-) diff --git a/pungi/phases/pkgset/sources/source_koji.py b/pungi/phases/pkgset/sources/source_koji.py index a5bd001f..b1069543 100644 --- a/pungi/phases/pkgset/sources/source_koji.py +++ b/pungi/phases/pkgset/sources/source_koji.py @@ -335,16 +335,20 @@ def filter_inherited(koji_proxy, event, module_builds, top_tag): return result -def filter_by_whitelist(compose, module_builds, input_modules): +def filter_by_whitelist(compose, module_builds, input_modules, expected_modules): """ Exclude modules from the list that do not match any pattern specified in - input_modules. Order may not be preserved. + input_modules. Order may not be preserved. The last argument is a set of + module patterns that are expected across module tags. When a matching + module is found, the corresponding pattern is removed from the set. """ - specs = set() nvr_patterns = set() for spec in input_modules: - # Do not do any filtering in case variant wants all the modules. + # Do not do any filtering in case variant wants all the modules. Also + # empty the set of remaining expected modules, as the check does not + # really make much sense here. if spec["name"] == "*": + expected_modules.clear() return module_builds info = variant_dict_from_str(compose, spec["name"]) @@ -355,10 +359,8 @@ def filter_by_whitelist(compose, module_builds, input_modules): info.get("context"), ) nvr_patterns.add((pattern, spec["name"])) - specs.add(spec["name"]) modules_to_keep = [] - used = set() for mb in module_builds: # Split release from the build into version and context @@ -377,15 +379,9 @@ def filter_by_whitelist(compose, module_builds, input_modules): and (not c or ctx == c) ): modules_to_keep.append(mb) - used.add(spec) + expected_modules.discard(spec) break - if used != specs: - raise RuntimeError( - "Configuration specified patterns (%s) that don't match any modules in the configured tags." - % ", ".join(specs - used) - ) - return modules_to_keep @@ -405,6 +401,9 @@ def _get_modules_from_koji_tags(compose, koji_wrapper, event_id, variant, varian compose_tags = [ {"name": tag} for tag in force_list(compose.conf["pkgset_koji_module_tag"]) ] + # Get set of configured module names for this variant. If nothing is + # configured, the set is empty. + expected_modules = set(spec["name"] for spec in variant.get_modules()) # Find out all modules in every variant and add their Koji tags # to variant and variant_tags list. koji_proxy = koji_wrapper.koji_proxy @@ -425,7 +424,9 @@ def _get_modules_from_koji_tags(compose, koji_wrapper, event_id, variant, varian # Apply whitelist of modules if specified. variant_modules = variant.get_modules() if variant_modules: - module_builds = filter_by_whitelist(compose, module_builds, variant_modules) + module_builds = filter_by_whitelist( + compose, module_builds, variant_modules, expected_modules + ) # Find the latest builds of all modules. This does following: # - Sorts the module_builds descending by Koji NVR (which maps to NSV @@ -476,6 +477,14 @@ def _get_modules_from_koji_tags(compose, koji_wrapper, event_id, variant, varian variant=variant, tag=module_tag, module=build["nvr"]) compose.log_info("%s" % module_msg) + if expected_modules: + # There are some module names that were listed in configuration and not + # found in any tag... + raise RuntimeError( + "Configuration specified patterns (%s) that don't match any modules in the configured tags." + % ", ".join(expected_modules) + ) + def _find_old_file_cache_path(compose): """ diff --git a/tests/test_pkgset_source_koji.py b/tests/test_pkgset_source_koji.py index 12568a32..9946a8e7 100644 --- a/tests/test_pkgset_source_koji.py +++ b/tests/test_pkgset_source_koji.py @@ -598,11 +598,11 @@ class TestFilterByWhitelist(unittest.TestCase): compose = mock.Mock() module_builds = [] input_modules = [{"name": "foo:1"}] + expected = set(["foo:1"]) - with self.assertRaises(RuntimeError) as ctx: - source_koji.filter_by_whitelist(compose, module_builds, input_modules) + source_koji.filter_by_whitelist(compose, module_builds, input_modules, expected) - self.assertIn("patterns (foo:1) that don't match", str(ctx.exception)) + self.assertEqual(expected, set(["foo:1"])) def test_filter_by_NS(self): compose = mock.Mock() @@ -612,10 +612,14 @@ class TestFilterByWhitelist(unittest.TestCase): self._build("foo", "2", "201809031047", "deadbeef"), ] input_modules = [{"name": "foo:1"}] + expected = set(["foo:1"]) - result = source_koji.filter_by_whitelist(compose, module_builds, input_modules) + result = source_koji.filter_by_whitelist( + compose, module_builds, input_modules, expected + ) self.assertItemsEqual(result, [module_builds[0], module_builds[1]]) + self.assertEqual(expected, set()) def test_filter_by_NSV(self): compose = mock.Mock() @@ -625,10 +629,14 @@ class TestFilterByWhitelist(unittest.TestCase): self._build("foo", "2", "201809031047", "deadbeef"), ] input_modules = [{"name": "foo:1:201809031047"}] + expected = set(["foo:1:201809031047"]) - result = source_koji.filter_by_whitelist(compose, module_builds, input_modules) + result = source_koji.filter_by_whitelist( + compose, module_builds, input_modules, expected + ) self.assertItemsEqual(result, [module_builds[1]]) + self.assertEqual(expected, set()) def test_filter_by_NSVC(self): compose = mock.Mock() @@ -639,10 +647,14 @@ class TestFilterByWhitelist(unittest.TestCase): self._build("foo", "2", "201809031047", "deadbeef"), ] input_modules = [{"name": "foo:1:201809031047:deadbeef"}] + expected = set() - result = source_koji.filter_by_whitelist(compose, module_builds, input_modules) + result = source_koji.filter_by_whitelist( + compose, module_builds, input_modules, expected + ) self.assertItemsEqual(result, [module_builds[1]]) + self.assertEqual(expected, set()) def test_filter_by_wildcard(self): compose = mock.Mock() @@ -652,10 +664,14 @@ class TestFilterByWhitelist(unittest.TestCase): self._build("foo", "2", "201809031047", "deadbeef"), ] input_modules = [{"name": "*"}] + expected = set(["*"]) - result = source_koji.filter_by_whitelist(compose, module_builds, input_modules) + result = source_koji.filter_by_whitelist( + compose, module_builds, input_modules, expected + ) self.assertItemsEqual(result, module_builds) + self.assertEqual(expected, set()) class MockModule(object):