From fbb739ef17332d692c38e61085bbab8400b167f7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lubom=C3=ADr=20Sedl=C3=A1=C5=99?= Date: Thu, 30 Aug 2018 11:12:58 +0200 Subject: [PATCH] pkgset: Apply whitelist to modules in the tag MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This patch changes the behaviour when both module tag and NSV?C? is specified. The NSVC are used as a whitelist and only matching modules will be included in the compose. Additionally this patch adds filtering based on inheritance: when finding the latest module for each N:S combination, only the top tag in which the module is tagged is used. Even if a newer build is available somewhere deeper in the inheritance, it's not going to be used. Example inheritance and tagged modules f29-compose (foo:1:2018:cafe) └─ f29-candidate (foo:1:2019:cafe) The compose will use 2018 version, because it's in the topmost tag. JIRA: COMPOSE-2685 Signed-off-by: Lubomír Sedlář --- doc/configuration.rst | 10 +- pungi/checks.py | 4 + pungi/phases/pkgset/sources/source_koji.py | 85 ++++++++++++- tests/test_pkgset_source_koji.py | 132 +++++++++++++++++++++ 4 files changed, 228 insertions(+), 3 deletions(-) diff --git a/doc/configuration.rst b/doc/configuration.rst index 71607673..9059d273 100644 --- a/doc/configuration.rst +++ b/doc/configuration.rst @@ -470,12 +470,20 @@ Options (*str|[str]*) -- tag(s) to read package set from. This option can be omitted for modular composes. +**pkgset_koji_module_tag** + (*str|[str]*) -- tags to read module from. This option works similarly to + listing tags in variants XML. If tags are specified and variants XML + specifies some modules via NSVC (or part of), only modules matching that + list will be used (and taken from the tag). Inheritance is used + automatically. + **pkgset_koji_inherit** = True (*bool*) -- inherit builds from parent tags; we can turn it off only if we have all builds tagged in a single tag **pkgset_koji_inherit_modules** = False - (*bool*) -- the same as above, but this only applies to modular tags + (*bool*) -- the same as above, but this only applies to modular tags. This + option applies to the content tags that contain the RPMs. **pkgset_repos** (*dict*) -- A mapping of architectures to repositories with RPMs: ``{arch: diff --git a/pungi/checks.py b/pungi/checks.py index b41128f1..7a2f9a8d 100644 --- a/pungi/checks.py +++ b/pungi/checks.py @@ -773,6 +773,10 @@ def make_schema(): "koji_profile": {"type": "string"}, "pkgset_koji_tag": {"$ref": "#/definitions/strings"}, + "pkgset_koji_module_tag": { + "$ref": "#/definitions/strings", + "default": [], + }, "pkgset_koji_inherit": { "type": "boolean", "default": True diff --git a/pungi/phases/pkgset/sources/source_koji.py b/pungi/phases/pkgset/sources/source_koji.py index 2bb652ae..3b4b5903 100644 --- a/pungi/phases/pkgset/sources/source_koji.py +++ b/pungi/phases/pkgset/sources/source_koji.py @@ -316,6 +316,75 @@ def _get_modules_from_koji( compose.log_info("%s" % module_msg) +def filter_inherited(koji_proxy, event, module_builds, top_tag): + """Look at the tag inheritance and keep builds only from the topmost tag. + + Using latest=True for listTagged() call would automatically do this, but it + does not understand streams, so we have to reimplement it here. + """ + inheritance = [ + tag["name"] for tag in koji_proxy.getFullInheritance(top_tag, event=event["id"]) + ] + + def keyfunc(mb): + return (mb["name"], mb["version"]) + + result = [] + + # Group modules by Name-Stream + for _, builds in groupby(sorted(module_builds, key=keyfunc), keyfunc): + builds = list(builds) + # For each N-S combination find out which tags it's in + available_in = set(build["tag_name"] for build in builds) + + # And find out which is the topmost tag + for tag in [top_tag] + inheritance: + if tag in available_in: + break + + # And keep only builds from that topmost tag + result.extend(build for build in builds if build["tag_name"] == tag) + + return result + + +def filter_by_whitelist(compose, module_builds, input_modules): + """ + Exclude modules from the list that do not match any pattern specified in + input_modules. Order may not be preserved. + """ + specs = set() + nvr_prefixes = set() + for spec in input_modules: + info = variant_dict_from_str(compose, spec["name"]) + prefix = ("%s-%s-%s.%s" % ( + info["name"], + info["stream"].replace("-", "_"), + info.get("version", ""), + info.get("context", ""), + )).rstrip("-.") + nvr_prefixes.add((prefix, spec["name"])) + specs.add(spec["name"]) + + modules_to_keep = [] + used = set() + + for mb in module_builds: + for (prefix, spec) in nvr_prefixes: + if mb["nvr"].startswith(prefix): + modules_to_keep.append(mb) + used.add(spec) + break + + if used != specs: + raise RuntimeError( + "Configuration specified patterns (%s) that don't match any modules in the configured tags." + % ", ".join(specs - used) + ) + + return modules_to_keep + + def _get_modules_from_koji_tags( compose, koji_wrapper, event_id, variant, variant_tags, module_tag_rpm_filter): """ @@ -329,10 +398,14 @@ def _get_modules_from_koji_tags( :param dict variant_tags: Dict populated by this method. Key is `variant` and value is list of Koji tags to get the RPMs from. """ + # Compose tags from configuration + compose_tags = [ + {"name": tag} for tag in force_list(compose.conf["pkgset_koji_module_tag"]) + ] # Find out all modules in every variant and add their Koji tags # to variant and variant_tags list. koji_proxy = koji_wrapper.koji_proxy - for modular_koji_tag in variant.get_modular_koji_tags(): + for modular_koji_tag in variant.get_modular_koji_tags() + compose_tags: tag = modular_koji_tag["name"] # List all the modular builds in the modular Koji tag. @@ -343,6 +416,14 @@ def _get_modules_from_koji_tags( module_builds = koji_proxy.listTagged( tag, event=event_id["id"], inherit=True, type="module") + # Filter out builds inherited from non-top tag + module_builds = filter_inherited(koji_proxy, event_id, module_builds, tag) + + # Apply whitelist of modules if specified. + variant_modules = variant.get_modules() + if variant_modules: + module_builds = filter_by_whitelist(compose, module_builds, variant_modules) + # Find the latest builds of all modules. This does following: # - Sorts the module_builds descending by Koji NVR (which maps to NSV # for modules). @@ -496,7 +577,7 @@ def populate_global_pkgset(compose, koji_wrapper, path_prefix, event): "support for modules is disabled, but compose contains " "modules.") - if modular_koji_tags: + if modular_koji_tags or (compose.conf["pkgset_koji_module_tag"] and variant.modules): included_modules_file = os.path.join( compose.paths.work.topdir(arch="global"), "koji-tag-module-%s.yaml" % variant.uid) diff --git a/tests/test_pkgset_source_koji.py b/tests/test_pkgset_source_koji.py index b0f11ea4..6b0c750a 100644 --- a/tests/test_pkgset_source_koji.py +++ b/tests/test_pkgset_source_koji.py @@ -653,5 +653,137 @@ class TestCorrectNVR(helpers.PungiTestCase): self.compose, 'foo:bar:baz:quux:qaar') +class TestFilterInherited(unittest.TestCase): + + def test_empty_module_list(self): + event = {"id": 123456} + koji_proxy = mock.Mock() + module_builds = [] + top_tag = "top-tag" + + koji_proxy.getFullInheritance.return_value = [ + {"name": "middle-tag"}, {"name": "bottom-tag"} + ] + + result = source_koji.filter_inherited(koji_proxy, event, module_builds, top_tag) + + self.assertItemsEqual(result, []) + self.assertEqual( + koji_proxy.mock_calls, + [mock.call.getFullInheritance("top-tag", event=123456)], + ) + + def test_exclude_middle_and_bottom_tag(self): + event = {"id": 123456} + koji_proxy = mock.Mock() + top_tag = "top-tag" + + koji_proxy.getFullInheritance.return_value = [ + {"name": "middle-tag"}, {"name": "bottom-tag"} + ] + module_builds = [ + {"name": "foo", "version": "1", "release": "1", "tag_name": "top-tag"}, + {"name": "foo", "version": "1", "release": "2", "tag_name": "bottom-tag"}, + {"name": "foo", "version": "1", "release": "3", "tag_name": "middle-tag"}, + ] + + result = source_koji.filter_inherited(koji_proxy, event, module_builds, top_tag) + + self.assertItemsEqual( + result, + [{"name": "foo", "version": "1", "release": "1", "tag_name": "top-tag"}], + ) + self.assertEqual( + koji_proxy.mock_calls, + [mock.call.getFullInheritance("top-tag", event=123456)], + ) + + def test_missing_from_top_tag(self): + event = {"id": 123456} + koji_proxy = mock.Mock() + top_tag = "top-tag" + + koji_proxy.getFullInheritance.return_value = [ + {"name": "middle-tag"}, {"name": "bottom-tag"} + ] + module_builds = [ + {"name": "foo", "version": "1", "release": "2", "tag_name": "bottom-tag"}, + {"name": "foo", "version": "1", "release": "3", "tag_name": "middle-tag"}, + ] + + result = source_koji.filter_inherited(koji_proxy, event, module_builds, top_tag) + + self.assertItemsEqual( + result, + [{"name": "foo", "version": "1", "release": "3", "tag_name": "middle-tag"}], + ) + self.assertEqual( + koji_proxy.mock_calls, + [mock.call.getFullInheritance("top-tag", event=123456)], + ) + + +class TestFilterByWhitelist(unittest.TestCase): + def test_no_modules(self): + compose = mock.Mock() + module_builds = [] + input_modules = [{"name": "foo:1"}] + + with self.assertRaises(RuntimeError) as ctx: + source_koji.filter_by_whitelist(compose, module_builds, input_modules) + + self.assertIn("patterns (foo:1) that don't match", str(ctx.exception)) + + def test_filter_by_NS(self): + compose = mock.Mock() + module_builds = [ + {"nvr": "foo-1-201809031048.cafebabe"}, + {"nvr": "foo-1-201809031047.deadbeef"}, + {"nvr": "foo-2-201809031047.deadbeef"}, + ] + input_modules = [{"name": "foo:1"}] + + result = source_koji.filter_by_whitelist(compose, module_builds, input_modules) + + self.assertItemsEqual( + result, + [ + {"nvr": "foo-1-201809031048.cafebabe"}, + {"nvr": "foo-1-201809031047.deadbeef"}, + ], + ) + + def test_filter_by_NSV(self): + compose = mock.Mock() + module_builds = [ + {"nvr": "foo-1-201809031048.cafebabe"}, + {"nvr": "foo-1-201809031047.deadbeef"}, + {"nvr": "foo-2-201809031047.deadbeef"}, + ] + input_modules = [{"name": "foo:1:201809031047"}] + + result = source_koji.filter_by_whitelist(compose, module_builds, input_modules) + + self.assertItemsEqual( + result, [{"nvr": "foo-1-201809031047.deadbeef"}] + ) + + def test_filter_by_NSVC(self): + compose = mock.Mock() + module_builds = [ + {"nvr": "foo-1-201809031048.cafebabe"}, + {"nvr": "foo-1-201809031047.deadbeef"}, + {"nvr": "foo-1-201809031047.cafebabe"}, + {"nvr": "foo-2-201809031047.deadbeef"}, + ] + input_modules = [{"name": "foo:1:201809031047:deadbeef"}] + + result = source_koji.filter_by_whitelist(compose, module_builds, input_modules) + + self.assertItemsEqual( + result, [{"nvr": "foo-1-201809031047.deadbeef"}] + ) + + if __name__ == "__main__": unittest.main()