From a9b9ec97fb761c6c9b4eaf981746f0b33ba0d14a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lubom=C3=ADr=20Sedl=C3=A1=C5=99?= Date: Fri, 3 May 2019 14:44:16 +0200 Subject: [PATCH] pkgset: Fix whitelist for modules MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The prefix checking only works if there are no streams that would share prefixes. Let's instead check the value as a whole. There is extra complexity from the fact that version and context may not be specified. The stream as specified in input is processed to replace dashes (`-`) with underscores (`_`) to match how the builds are imported into Koji. JIRA: COMPOSE-3547 Signed-off-by: Lubomír Sedlář --- pungi/phases/pkgset/sources/source_koji.py | 29 ++++++++--- tests/test_pkgset_source_koji.py | 60 ++++++++++------------ 2 files changed, 47 insertions(+), 42 deletions(-) diff --git a/pungi/phases/pkgset/sources/source_koji.py b/pungi/phases/pkgset/sources/source_koji.py index 6f9a6432..a96c9730 100644 --- a/pungi/phases/pkgset/sources/source_koji.py +++ b/pungi/phases/pkgset/sources/source_koji.py @@ -324,28 +324,41 @@ def filter_by_whitelist(compose, module_builds, input_modules): input_modules. Order may not be preserved. """ specs = set() - nvr_prefixes = set() + nvr_patterns = set() for spec in input_modules: # Do not do any filtering in case variant wants all the modules. if spec["name"] == "*": return module_builds info = variant_dict_from_str(compose, spec["name"]) - prefix = ("%s-%s-%s.%s" % ( + pattern = ( info["name"], info["stream"].replace("-", "_"), - info.get("version", ""), - info.get("context", ""), - )).rstrip("-.") - nvr_prefixes.add((prefix, spec["name"])) + info.get("version"), + info.get("context"), + ) + nvr_patterns.add((pattern, spec["name"])) specs.add(spec["name"]) modules_to_keep = [] used = set() for mb in module_builds: - for (prefix, spec) in nvr_prefixes: - if mb["nvr"].startswith(prefix): + # Split release from the build into version and context + ver, ctx = mb["release"].split(".") + # Values in `mb` are from Koji build. There's nvr and name, version and + # release. The input pattern specifies modular name, stream, version + # and context. + for (n, s, v, c), spec in nvr_patterns: + if ( + # We always have a name and stream... + mb["name"] == n + and mb["version"] == s + # ...but version and context can be missing, in which case we + # don't want to check them. + and (not v or ver == v) + and (not c or ctx == c) + ): modules_to_keep.append(mb) used.add(spec) break diff --git a/tests/test_pkgset_source_koji.py b/tests/test_pkgset_source_koji.py index 6d328a9e..5c0a6509 100644 --- a/tests/test_pkgset_source_koji.py +++ b/tests/test_pkgset_source_koji.py @@ -585,6 +585,15 @@ class TestFilterInherited(unittest.TestCase): class TestFilterByWhitelist(unittest.TestCase): + def _build(self, n, s, v, c): + s = s.replace("-", "_") + return { + "nvr": "%s-%s-%s.%s" % (n, s, v, c), + "name": n, + "version": s, + "release": "%s.%s" % (v, c), + } + def test_no_modules(self): compose = mock.Mock() module_builds = [] @@ -598,72 +607,55 @@ class TestFilterByWhitelist(unittest.TestCase): def test_filter_by_NS(self): compose = mock.Mock() module_builds = [ - {"nvr": "foo-1-201809031048.cafebabe"}, - {"nvr": "foo-1-201809031047.deadbeef"}, - {"nvr": "foo-2-201809031047.deadbeef"}, + self._build("foo", "1", "201809031048", "cafebabe"), + self._build("foo", "1", "201809031047", "deadbeef"), + self._build("foo", "2", "201809031047", "deadbeef"), ] input_modules = [{"name": "foo:1"}] result = source_koji.filter_by_whitelist(compose, module_builds, input_modules) - self.assertItemsEqual( - result, - [ - {"nvr": "foo-1-201809031048.cafebabe"}, - {"nvr": "foo-1-201809031047.deadbeef"}, - ], - ) + self.assertItemsEqual(result, [module_builds[0], module_builds[1]]) def test_filter_by_NSV(self): compose = mock.Mock() module_builds = [ - {"nvr": "foo-1-201809031048.cafebabe"}, - {"nvr": "foo-1-201809031047.deadbeef"}, - {"nvr": "foo-2-201809031047.deadbeef"}, + self._build("foo", "1", "201809031048", "cafebabe"), + self._build("foo", "1", "201809031047", "deadbeef"), + self._build("foo", "2", "201809031047", "deadbeef"), ] input_modules = [{"name": "foo:1:201809031047"}] result = source_koji.filter_by_whitelist(compose, module_builds, input_modules) - self.assertItemsEqual( - result, [{"nvr": "foo-1-201809031047.deadbeef"}] - ) + self.assertItemsEqual(result, [module_builds[1]]) def test_filter_by_NSVC(self): compose = mock.Mock() module_builds = [ - {"nvr": "foo-1-201809031048.cafebabe"}, - {"nvr": "foo-1-201809031047.deadbeef"}, - {"nvr": "foo-1-201809031047.cafebabe"}, - {"nvr": "foo-2-201809031047.deadbeef"}, + self._build("foo", "1", "201809031048", "cafebabe"), + self._build("foo", "1", "201809031047", "deadbeef"), + self._build("foo", "1", "201809031047", "cafebabe"), + self._build("foo", "2", "201809031047", "deadbeef"), ] input_modules = [{"name": "foo:1:201809031047:deadbeef"}] result = source_koji.filter_by_whitelist(compose, module_builds, input_modules) - self.assertItemsEqual( - result, [{"nvr": "foo-1-201809031047.deadbeef"}] - ) + self.assertItemsEqual(result, [module_builds[1]]) def test_filter_by_wildcard(self): compose = mock.Mock() module_builds = [ - {"nvr": "foo-1-201809031048.cafebabe"}, - {"nvr": "foo-1-201809031047.deadbeef"}, - {"nvr": "foo-2-201809031047.deadbeef"}, + self._build("foo", "1", "201809031048", "cafebabe"), + self._build("foo", "1", "201809031047", "deadbeef"), + self._build("foo", "2", "201809031047", "deadbeef"), ] input_modules = [{"name": "*"}] result = source_koji.filter_by_whitelist(compose, module_builds, input_modules) - self.assertItemsEqual( - result, - [ - {"nvr": "foo-1-201809031048.cafebabe"}, - {"nvr": "foo-1-201809031047.deadbeef"}, - {"nvr": "foo-2-201809031047.deadbeef"}, - ], - ) + self.assertItemsEqual(result, module_builds) class MockModule(object):