almalinux/pungi
14
1
Fork 0
Code Issues Pull Requests Releases Activity

pkgset: Apply whitelist to modules in the tag

Browse Source 
This patch changes the behaviour when both module tag and NSV?C? is
specified. The NSVC are used as a whitelist and only matching modules
will be included in the compose.

Additionally this patch adds filtering based on inheritance: when
finding the latest module for each N:S combination, only the top tag in
which the module is tagged is used. Even if a newer build is available
somewhere deeper in the inheritance, it's not going to be used.

Example inheritance and tagged modules

    f29-compose (foo:1:2018:cafe)
    └─ f29-candidate (foo:1:2019:cafe)

The compose will use 2018 version, because it's in the topmost tag.

JIRA: COMPOSE-2685
Signed-off-by: Lubomír Sedlář <lsedlar@redhat.com>
This commit is contained in:
Lubomír Sedlář 2018-08-30 11:12:58 +02:00
parent 833ba64c51
commit fbb739ef17
4 changed files with 228 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
doc/configuration.rst
View File

@ -470,12 +470,20 @@ Options
(*str|[str]*) -- tag(s) to read package set from. This option can be
omitted for modular composes.
**pkgset_koji_module_tag**
(*str|[str]*) -- tags to read module from. This option works similarly to
listing tags in variants XML. If tags are specified and variants XML
specifies some modules via NSVC (or part of), only modules matching that
list will be used (and taken from the tag). Inheritance is used
automatically.
**pkgset_koji_inherit** = True
(*bool*) -- inherit builds from parent tags; we can turn it off only if we
have all builds tagged in a single tag
**pkgset_koji_inherit_modules** = False
(*bool*) -- the same as above, but this only applies to modular tags
(*bool*) -- the same as above, but this only applies to modular tags. This
option applies to the content tags that contain the RPMs.
**pkgset_repos**
(*dict*) -- A mapping of architectures to repositories with RPMs: ``{arch:

4
pungi/checks.py
View File

@ -773,6 +773,10 @@ def make_schema():
"koji_profile": {"type": "string"},
"pkgset_koji_tag": {"$ref": "#/definitions/strings"},
"pkgset_koji_module_tag": {
"$ref": "#/definitions/strings",
"default": [],
},
"pkgset_koji_inherit": {
"type": "boolean",
"default": True

85
pungi/phases/pkgset/sources/source_koji.py
View File

@ -316,6 +316,75 @@ def _get_modules_from_koji(
compose.log_info("%s" % module_msg)
def filter_inherited(koji_proxy, event, module_builds, top_tag):
"""Look at the tag inheritance and keep builds only from the topmost tag.
Using latest=True for listTagged() call would automatically do this, but it
does not understand streams, so we have to reimplement it here.
"""
inheritance = [
tag["name"] for tag in koji_proxy.getFullInheritance(top_tag, event=event["id"])
]
def keyfunc(mb):
return (mb["name"], mb["version"])
result = []
# Group modules by Name-Stream
for _, builds in groupby(sorted(module_builds, key=keyfunc), keyfunc):
builds = list(builds)
# For each N-S combination find out which tags it's in
available_in = set(build["tag_name"] for build in builds)
# And find out which is the topmost tag
for tag in [top_tag] + inheritance:
if tag in available_in:
break
# And keep only builds from that topmost tag
result.extend(build for build in builds if build["tag_name"] == tag)
return result
def filter_by_whitelist(compose, module_builds, input_modules):
"""
Exclude modules from the list that do not match any pattern specified in
input_modules. Order may not be preserved.
"""
specs = set()
nvr_prefixes = set()
for spec in input_modules:
info = variant_dict_from_str(compose, spec["name"])
prefix = ("%s-%s-%s.%s" % (
info["name"],
info["stream"].replace("-", "_"),
info.get("version", ""),
info.get("context", ""),
)).rstrip("-.")
nvr_prefixes.add((prefix, spec["name"]))
specs.add(spec["name"])
modules_to_keep = []
used = set()
for mb in module_builds:
for (prefix, spec) in nvr_prefixes:
if mb["nvr"].startswith(prefix):
modules_to_keep.append(mb)
used.add(spec)
break
if used != specs:
raise RuntimeError(
"Configuration specified patterns (%s) that don't match any modules in the configured tags."
% ", ".join(specs - used)
)
return modules_to_keep
def _get_modules_from_koji_tags(
compose, koji_wrapper, event_id, variant, variant_tags, module_tag_rpm_filter):
"""
@ -329,10 +398,14 @@ def _get_modules_from_koji_tags(
:param dict variant_tags: Dict populated by this method. Key is `variant`
and value is list of Koji tags to get the RPMs from.
"""
# Compose tags from configuration
compose_tags = [
{"name": tag} for tag in force_list(compose.conf["pkgset_koji_module_tag"])
]
# Find out all modules in every variant and add their Koji tags
# to variant and variant_tags list.
koji_proxy = koji_wrapper.koji_proxy
for modular_koji_tag in variant.get_modular_koji_tags():
for modular_koji_tag in variant.get_modular_koji_tags() + compose_tags:
tag = modular_koji_tag["name"]
# List all the modular builds in the modular Koji tag.
@ -343,6 +416,14 @@ def _get_modules_from_koji_tags(
module_builds = koji_proxy.listTagged(
tag, event=event_id["id"], inherit=True, type="module")
# Filter out builds inherited from non-top tag
module_builds = filter_inherited(koji_proxy, event_id, module_builds, tag)
# Apply whitelist of modules if specified.
variant_modules = variant.get_modules()
if variant_modules:
module_builds = filter_by_whitelist(compose, module_builds, variant_modules)
# Find the latest builds of all modules. This does following:
# - Sorts the module_builds descending by Koji NVR (which maps to NSV
# for modules).
@ -496,7 +577,7 @@ def populate_global_pkgset(compose, koji_wrapper, path_prefix, event):
"support for modules is disabled, but compose contains "
"modules.")
if modular_koji_tags:
if modular_koji_tags or (compose.conf["pkgset_koji_module_tag"] and variant.modules):
included_modules_file = os.path.join(
compose.paths.work.topdir(arch="global"),
"koji-tag-module-%s.yaml" % variant.uid)

132
tests/test_pkgset_source_koji.py
View File

@ -653,5 +653,137 @@ class TestCorrectNVR(helpers.PungiTestCase):
self.compose, 'foo:bar:baz:quux:qaar')
class TestFilterInherited(unittest.TestCase):
def test_empty_module_list(self):
event = {"id": 123456}
koji_proxy = mock.Mock()
module_builds = []
top_tag = "top-tag"
koji_proxy.getFullInheritance.return_value = [
{"name": "middle-tag"}, {"name": "bottom-tag"}
]
result = source_koji.filter_inherited(koji_proxy, event, module_builds, top_tag)
self.assertItemsEqual(result, [])
self.assertEqual(
koji_proxy.mock_calls,
[mock.call.getFullInheritance("top-tag", event=123456)],
)
def test_exclude_middle_and_bottom_tag(self):
event = {"id": 123456}
koji_proxy = mock.Mock()
top_tag = "top-tag"
koji_proxy.getFullInheritance.return_value = [
{"name": "middle-tag"}, {"name": "bottom-tag"}
]
module_builds = [
{"name": "foo", "version": "1", "release": "1", "tag_name": "top-tag"},
{"name": "foo", "version": "1", "release": "2", "tag_name": "bottom-tag"},
{"name": "foo", "version": "1", "release": "3", "tag_name": "middle-tag"},
]
result = source_koji.filter_inherited(koji_proxy, event, module_builds, top_tag)
self.assertItemsEqual(
result,
[{"name": "foo", "version": "1", "release": "1", "tag_name": "top-tag"}],
)
self.assertEqual(
koji_proxy.mock_calls,
[mock.call.getFullInheritance("top-tag", event=123456)],
)
def test_missing_from_top_tag(self):
event = {"id": 123456}
koji_proxy = mock.Mock()
top_tag = "top-tag"
koji_proxy.getFullInheritance.return_value = [
{"name": "middle-tag"}, {"name": "bottom-tag"}
]
module_builds = [
{"name": "foo", "version": "1", "release": "2", "tag_name": "bottom-tag"},
{"name": "foo", "version": "1", "release": "3", "tag_name": "middle-tag"},
]
result = source_koji.filter_inherited(koji_proxy, event, module_builds, top_tag)
self.assertItemsEqual(
result,
[{"name": "foo", "version": "1", "release": "3", "tag_name": "middle-tag"}],
)
self.assertEqual(
koji_proxy.mock_calls,
[mock.call.getFullInheritance("top-tag", event=123456)],
)
class TestFilterByWhitelist(unittest.TestCase):
def test_no_modules(self):
compose = mock.Mock()
module_builds = []
input_modules = [{"name": "foo:1"}]
with self.assertRaises(RuntimeError) as ctx:
source_koji.filter_by_whitelist(compose, module_builds, input_modules)
self.assertIn("patterns (foo:1) that don't match", str(ctx.exception))
def test_filter_by_NS(self):
compose = mock.Mock()
module_builds = [
{"nvr": "foo-1-201809031048.cafebabe"},
{"nvr": "foo-1-201809031047.deadbeef"},
{"nvr": "foo-2-201809031047.deadbeef"},
]
input_modules = [{"name": "foo:1"}]
result = source_koji.filter_by_whitelist(compose, module_builds, input_modules)
self.assertItemsEqual(
result,
[
{"nvr": "foo-1-201809031048.cafebabe"},
{"nvr": "foo-1-201809031047.deadbeef"},
],
)
def test_filter_by_NSV(self):
compose = mock.Mock()
module_builds = [
{"nvr": "foo-1-201809031048.cafebabe"},
{"nvr": "foo-1-201809031047.deadbeef"},
{"nvr": "foo-2-201809031047.deadbeef"},
]
input_modules = [{"name": "foo:1:201809031047"}]
result = source_koji.filter_by_whitelist(compose, module_builds, input_modules)
self.assertItemsEqual(
result, [{"nvr": "foo-1-201809031047.deadbeef"}]
)
def test_filter_by_NSVC(self):
compose = mock.Mock()
module_builds = [
{"nvr": "foo-1-201809031048.cafebabe"},
{"nvr": "foo-1-201809031047.deadbeef"},
{"nvr": "foo-1-201809031047.cafebabe"},
{"nvr": "foo-2-201809031047.deadbeef"},
]
input_modules = [{"name": "foo:1:201809031047:deadbeef"}]
result = source_koji.filter_by_whitelist(compose, module_builds, input_modules)
self.assertItemsEqual(
result, [{"nvr": "foo-1-201809031047.deadbeef"}]
)
if __name__ == "__main__":
unittest.main()