almalinux/pungi
14
1
Fork 0
Code Issues Pull Requests Releases Activity

pkgset: Check for unused module patterns across all tags

Browse Source 
Since there can be multiple tags, the check must be done once for all of
them at the same time. Otherwise any module found only in some and not
all tags would raise this error.

The code builds a set of all existing patterns and then removes items
from it. If there is something left once all tags are processed, it
means such patterns were not matched by anything.

JIRA: COMPOSE-3609
Signed-off-by: Lubomír Sedlář <lsedlar@redhat.com>
This commit is contained in:
Lubomír Sedlář 2019-06-13 10:10:30 +02:00
parent f784bbd519
commit e68e17d8fe
2 changed files with 46 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

37
pungi/phases/pkgset/sources/source_koji.py
View File

@ -335,16 +335,20 @@ def filter_inherited(koji_proxy, event, module_builds, top_tag):
return result
def filter_by_whitelist(compose, module_builds, input_modules):
def filter_by_whitelist(compose, module_builds, input_modules, expected_modules):
"""
Exclude modules from the list that do not match any pattern specified in
input_modules. Order may not be preserved.
input_modules. Order may not be preserved. The last argument is a set of
module patterns that are expected across module tags. When a matching
module is found, the corresponding pattern is removed from the set.
"""
specs = set()
nvr_patterns = set()
for spec in input_modules:
# Do not do any filtering in case variant wants all the modules.
# Do not do any filtering in case variant wants all the modules. Also
# empty the set of remaining expected modules, as the check does not
# really make much sense here.
if spec["name"] == "*":
expected_modules.clear()
return module_builds
info = variant_dict_from_str(compose, spec["name"])
@ -355,10 +359,8 @@ def filter_by_whitelist(compose, module_builds, input_modules):
info.get("context"),
)
nvr_patterns.add((pattern, spec["name"]))
specs.add(spec["name"])
modules_to_keep = []
used = set()
for mb in module_builds:
# Split release from the build into version and context
@ -377,15 +379,9 @@ def filter_by_whitelist(compose, module_builds, input_modules):
and (not c or ctx == c)
):
modules_to_keep.append(mb)
used.add(spec)
expected_modules.discard(spec)
break
if used != specs:
raise RuntimeError(
"Configuration specified patterns (%s) that don't match any modules in the configured tags."
% ", ".join(specs - used)
)
return modules_to_keep
@ -405,6 +401,9 @@ def _get_modules_from_koji_tags(compose, koji_wrapper, event_id, variant, varian
compose_tags = [
{"name": tag} for tag in force_list(compose.conf["pkgset_koji_module_tag"])
]
# Get set of configured module names for this variant. If nothing is
# configured, the set is empty.
expected_modules = set(spec["name"] for spec in variant.get_modules())
# Find out all modules in every variant and add their Koji tags
# to variant and variant_tags list.
koji_proxy = koji_wrapper.koji_proxy
@ -425,7 +424,9 @@ def _get_modules_from_koji_tags(compose, koji_wrapper, event_id, variant, varian
# Apply whitelist of modules if specified.
variant_modules = variant.get_modules()
if variant_modules:
module_builds = filter_by_whitelist(compose, module_builds, variant_modules)
module_builds = filter_by_whitelist(
compose, module_builds, variant_modules, expected_modules
)
# Find the latest builds of all modules. This does following:
# - Sorts the module_builds descending by Koji NVR (which maps to NSV
@ -476,6 +477,14 @@ def _get_modules_from_koji_tags(compose, koji_wrapper, event_id, variant, varian
variant=variant, tag=module_tag, module=build["nvr"])
compose.log_info("%s" % module_msg)
if expected_modules:
# There are some module names that were listed in configuration and not
# found in any tag...
raise RuntimeError(
"Configuration specified patterns (%s) that don't match any modules in the configured tags."
% ", ".join(expected_modules)
)
def _find_old_file_cache_path(compose):
"""

30
tests/test_pkgset_source_koji.py
View File

@ -598,11 +598,11 @@ class TestFilterByWhitelist(unittest.TestCase):
compose = mock.Mock()
module_builds = []
input_modules = [{"name": "foo:1"}]
expected = set(["foo:1"])
with self.assertRaises(RuntimeError) as ctx:
source_koji.filter_by_whitelist(compose, module_builds, input_modules)
source_koji.filter_by_whitelist(compose, module_builds, input_modules, expected)
self.assertIn("patterns (foo:1) that don't match", str(ctx.exception))
self.assertEqual(expected, set(["foo:1"]))
def test_filter_by_NS(self):
compose = mock.Mock()
@ -612,10 +612,14 @@ class TestFilterByWhitelist(unittest.TestCase):
self._build("foo", "2", "201809031047", "deadbeef"),
]
input_modules = [{"name": "foo:1"}]
expected = set(["foo:1"])
result = source_koji.filter_by_whitelist(compose, module_builds, input_modules)
result = source_koji.filter_by_whitelist(
compose, module_builds, input_modules, expected
)
self.assertItemsEqual(result, [module_builds[0], module_builds[1]])
self.assertEqual(expected, set())
def test_filter_by_NSV(self):
compose = mock.Mock()
@ -625,10 +629,14 @@ class TestFilterByWhitelist(unittest.TestCase):
self._build("foo", "2", "201809031047", "deadbeef"),
]
input_modules = [{"name": "foo:1:201809031047"}]
expected = set(["foo:1:201809031047"])
result = source_koji.filter_by_whitelist(compose, module_builds, input_modules)
result = source_koji.filter_by_whitelist(
compose, module_builds, input_modules, expected
)
self.assertItemsEqual(result, [module_builds[1]])
self.assertEqual(expected, set())
def test_filter_by_NSVC(self):
compose = mock.Mock()
@ -639,10 +647,14 @@ class TestFilterByWhitelist(unittest.TestCase):
self._build("foo", "2", "201809031047", "deadbeef"),
]
input_modules = [{"name": "foo:1:201809031047:deadbeef"}]
expected = set()
result = source_koji.filter_by_whitelist(compose, module_builds, input_modules)
result = source_koji.filter_by_whitelist(
compose, module_builds, input_modules, expected
)
self.assertItemsEqual(result, [module_builds[1]])
self.assertEqual(expected, set())
def test_filter_by_wildcard(self):
compose = mock.Mock()
@ -652,10 +664,14 @@ class TestFilterByWhitelist(unittest.TestCase):
self._build("foo", "2", "201809031047", "deadbeef"),
]
input_modules = [{"name": "*"}]
expected = set(["*"])
result = source_koji.filter_by_whitelist(compose, module_builds, input_modules)
result = source_koji.filter_by_whitelist(
compose, module_builds, input_modules, expected
)
self.assertItemsEqual(result, module_builds)
self.assertEqual(expected, set())
class MockModule(object):