almalinux/pungi
15
2
Fork 0
Code Issues Pull Requests Releases Activity

pkgset: Allow filtering modules from Koji tags

Browse Source 
Add a configuration option to enable skipping some modules found in the
configured tag.

Fixes: https://pagure.io/pungi/issue/1260
JIRA: COMPOSE-3794
Signed-off-by: Lubomír Sedlář <lsedlar@redhat.com>
This commit is contained in:
Lubomír Sedlář 2019-09-23 14:31:57 +02:00
parent 44e551317a
commit 21d45eb243
4 changed files with 156 additions and 25 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
doc/configuration.rst
View File

@ -700,6 +700,16 @@ Options
See :ref:`additional_packages <additional_packages>` for details about
package specification.
**filter_modules**
(*list*) -- modules to be excluded from a variant and architecture;
format: ``[(variant_uid_regex, {arch|*: [name:stream]})]``
Both name and stream can use shell-style globs. If stream is omitted, all
streams are removed.
This option only applies to modules taken from Koji tags, not modules
explicitly listed in variants XML without any tags.
**filter_system_release_packages**
(*bool*) -- for each variant, figure out the best system release package
and filter out all others. This will not work if a variant needs more than

4
pungi/checks.py
View File

@ -733,6 +733,10 @@ def make_schema():
"$ref": "#/definitions/package_mapping",
"default": [],
},
"filter_modules": {
"$ref": "#/definitions/package_mapping",
"default": [],
},
"sigkeys": {
"type": "array",
"items": {"$ref": "#/definitions/optional_string"},

106
pungi/phases/pkgset/sources/source_koji.py
View File

@ -17,6 +17,7 @@
import os
import json
import re
from fnmatch import fnmatch
from itertools import groupby
from kobo.rpmlib import parse_nvra
@ -26,7 +27,7 @@ import pungi.wrappers.kojiwrapper
from pungi.wrappers.comps import CompsWrapper
import pungi.phases.pkgset.pkgsets
from pungi.arch import getBaseArch
from pungi.util import retry, find_old_compose
from pungi.util import retry, find_old_compose, get_arch_variant_data
from pungi import Modulemd
from pungi.phases.pkgset.common import MaterializedPackageSet, get_all_arches
@ -190,7 +191,9 @@ def get_pkgset_from_koji(compose, koji_wrapper, path_prefix):
return populate_global_pkgset(compose, koji_wrapper, path_prefix, event_info)
def _add_module_to_variant(koji_wrapper, variant, build, add_to_variant_modules=False):
def _add_module_to_variant(
koji_wrapper, variant, build, add_to_variant_modules=False, compose=None
):
"""
Adds module defined by Koji build info to variant.
@ -198,6 +201,7 @@ def _add_module_to_variant(koji_wrapper, variant, build, add_to_variant_modules=
:param int: build id
:param bool add_to_variant_modules: Adds the modules also to
variant.modules.
:param compose: Compose object to get filters from
"""
mmds = {}
archives = koji_wrapper.koji_proxy.listArchives(build["id"])
@ -225,26 +229,57 @@ def _add_module_to_variant(koji_wrapper, variant, build, add_to_variant_modules=
# longer supported and should be rebuilt. Let's skip it.
return
nsvc = "%(name)s:%(stream)s:%(version)s:%(context)s" % build["extra"]["typeinfo"]["module"]
info = build["extra"]["typeinfo"]["module"]
nsvc = "%(name)s:%(stream)s:%(version)s:%(context)s" % info
added = False
for arch in variant.arches:
if _is_filtered_out(compose, variant, arch, info["name"], info["stream"]):
compose.log_debug("Module %s is filtered from %s.%s", nsvc, variant, arch)
continue
try:
mmd = Modulemd.ModuleStream.read_file(
mmds["modulemd.%s.txt" % arch], strict=True
)
variant.arch_mmds.setdefault(arch, {})[nsvc] = mmd
added = True
except KeyError:
# There is no modulemd for this arch. This could mean an arch was
# added to the compose after the module was built. We don't want to
# process this, let's skip this module.
pass
if not added:
# The module is filtered on all arches of this variant.
return None
if add_to_variant_modules:
variant.modules.append({"name": nsvc, "glob": False})
return nsvc
def _is_filtered_out(compose, variant, arch, module_name, module_stream):
"""Check if module with given name and stream is filter out from this stream.
"""
if not compose:
return False
for filter in get_arch_variant_data(compose.conf, "filter_modules", arch, variant):
if ":" not in filter:
name_filter = filter
stream_filter = "*"
else:
name_filter, stream_filter = filter.split(":", 1)
if fnmatch(module_name, name_filter) and fnmatch(module_stream, stream_filter):
return True
return False
def _get_modules_from_koji(
compose, koji_wrapper, event, variant, variant_tags, tag_to_mmd
):
@ -264,26 +299,34 @@ def _get_modules_from_koji(
for module in variant.get_modules():
koji_modules = get_koji_modules(compose, koji_wrapper, event, module["name"])
for koji_module in koji_modules:
nsvc = _add_module_to_variant(koji_wrapper, variant, koji_module)
nsvc = _add_module_to_variant(
koji_wrapper, variant, koji_module, compose=compose
)
if not nsvc:
continue
tag = koji_module["tag"]
variant_tags[variant].append(tag)
# Store mapping NSVC --> koji_tag into variant.
# This is needed in createrepo phase where metadata is exposed by producmd
variant.module_uid_to_koji_tag[nsvc] = tag
tag_to_mmd.setdefault(tag, {})
for arch in variant.arch_mmds:
tag_to_mmd[tag].setdefault(arch, set()).add(variant.arch_mmds[arch][nsvc])
try:
mmd = variant.arch_mmds[arch][nsvc]
except KeyError:
# Module was filtered from here
continue
tag_to_mmd[tag].setdefault(arch, set()).add(mmd)
module_msg = (
"Module '{uid}' in variant '{variant}' will use Koji tag '{tag}' "
"(as a result of querying module '{module}')"
).format(uid=nsvc, variant=variant, tag=tag, module=module["name"])
compose.log_info("%s" % module_msg)
if tag_to_mmd[tag]:
compose.log_info(
"Module '%s' in variant '%s' will use Koji tag '%s' "
"(as a result of querying module '%s')",
nsvc, variant, tag, module["name"]
)
# Store mapping NSVC --> koji_tag into variant. This is needed
# in createrepo phase where metadata is exposed by producmd
variant.module_uid_to_koji_tag[nsvc] = tag
def filter_inherited(koji_proxy, event, module_builds, top_tag):
@ -449,21 +492,31 @@ def _get_modules_from_koji_tags(
variant_tags[variant].append(module_tag)
nsvc = _add_module_to_variant(koji_wrapper, variant, build, True)
nsvc = _add_module_to_variant(
koji_wrapper, variant, build, True, compose=compose
)
if not nsvc:
continue
# Store mapping module-uid --> koji_tag into variant.
# This is needed in createrepo phase where metadata is exposed by producmd
variant.module_uid_to_koji_tag[nsvc] = module_tag
tag_to_mmd.setdefault(module_tag, {})
for arch in variant.arch_mmds:
tag_to_mmd[module_tag].setdefault(arch, set()).add(variant.arch_mmds[arch][nsvc])
try:
mmd = variant.arch_mmds[arch][nsvc]
except KeyError:
# Module was filtered from here
continue
tag_to_mmd[module_tag].setdefault(arch, set()).add(mmd)
module_msg = "Module {module} in variant {variant} will use Koji tag {tag}.".format(
variant=variant, tag=module_tag, module=build["nvr"])
compose.log_info("%s" % module_msg)
if tag_to_mmd[module_tag]:
compose.log_info(
"Module %s in variant %s will use Koji tag %s.",
nsvc, variant, module_tag
)
# Store mapping module-uid --> koji_tag into variant. This is
# needed in createrepo phase where metadata is exposed by
# productmd
variant.module_uid_to_koji_tag[nsvc] = module_tag
if expected_modules:
# There are some module names that were listed in configuration and not
@ -624,7 +677,12 @@ def populate_global_pkgset(compose, koji_wrapper, path_prefix, event):
# Not current tag, skip it
continue
for arch_modules in variant.arch_mmds.values():
for rpm_nevra in arch_modules[nsvc].get_rpm_artifacts():
try:
module = arch_modules[nsvc]
except KeyError:
# The module was filtered out
continue
for rpm_nevra in module.get_rpm_artifacts():
nevra = parse_nvra(rpm_nevra)
modular_packages.add((nevra["name"], nevra["arch"]))

61
tests/test_pkgset_source_koji.py
View File

@ -615,9 +615,10 @@ class MockModule(object):
@mock.patch("pungi.Modulemd.ModuleStream.read_file", new=MockModule)
@unittest.skipIf(Modulemd is None, "Skipping tests, no module support")
class TestAddModuleToVariant(unittest.TestCase):
class TestAddModuleToVariant(helpers.PungiTestCase):
def setUp(self):
super(TestAddModuleToVariant, self).setUp()
self.koji = mock.Mock()
self.koji.koji_module.pathinfo.typedir.return_value = "/koji"
files = ["modulemd.x86_64.txt", "modulemd.armv7hl.txt", "modulemd.txt"]
@ -741,3 +742,61 @@ class TestAddModuleToVariant(unittest.TestCase):
{"name": "module:master:20190318:abcdef", "glob": False},
],
)
def test_adding_module_but_filtered(self):
compose = helpers.DummyCompose(
self.topdir, {"filter_modules": [(".*", {"*": ["module:*"]})]}
)
variant = mock.Mock(
arches=["armhfp", "x86_64"], arch_mmds={}, modules=[], uid="Variant"
)
nsvc = source_koji._add_module_to_variant(
self.koji,
variant,
self.buildinfo,
add_to_variant_modules=True,
compose=compose,
)
self.assertIsNone(nsvc)
self.assertEqual(variant.arch_mmds, {})
self.assertEqual(variant.modules, [])
class TestIsModuleFiltered(helpers.PungiTestCase):
def assertIsFiltered(self, name, stream):
self.assertTrue(
source_koji._is_filtered_out(
self.compose, self.compose.variants["Server"], "x86_64", name, stream
)
)
def assertIsNotFiltered(self, name, stream):
self.assertFalse(
source_koji._is_filtered_out(
self.compose, self.compose.variants["Server"], "x86_64", name, stream
)
)
def test_no_filters(self):
self.compose = helpers.DummyCompose(self.topdir, {})
self.assertIsNotFiltered("foo", "master")
def test_filter_by_name(self):
self.compose = helpers.DummyCompose(
self.topdir, {"filter_modules": [(".*", {"*": ["foo"]})]}
)
self.assertIsFiltered("foo", "master")
self.assertIsNotFiltered("bar", "master")
def test_filter_by_stream(self):
self.compose = helpers.DummyCompose(
self.topdir, {"filter_modules": [(".*", {"*": ["foo:master"]})]}
)
self.assertIsFiltered("foo", "master")
self.assertIsNotFiltered("bar", "master")
self.assertIsNotFiltered("foo", "stable")