almalinux/pungi
14
1
Fork 0
Code Issues Pull Requests Releases Activity

pkgset: Apply module filters on pkgset level

Browse Source 
If a module says to filter a package out, we can do it immediately when
getting the build information from Koji.

This avoids a possible problem of something pulling the module package
in as a dependency, but it should also make the package set slightly
smaller.

Signed-off-by: Lubomír Sedlář <lsedlar@redhat.com>
This commit is contained in:
Lubomír Sedlář 2018-06-25 14:35:32 +02:00
parent 470c0ab3be
commit 98e7106f3e
3 changed files with 86 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
pungi/phases/pkgset/pkgsets.py
View File

@ -361,15 +361,19 @@ class KojiPackageSet(PackageSetBase):
% (rpm_info, self.sigkey_ordering, paths))
return None
def populate(self, tag, event=None, inherit=True, logfile=None):
def populate(
self, tag, event=None, inherit=True, logfile=None, exclude_packages=None
):
"""Populate the package set with packages from given tag.
:param event: the Koji event to query at (or latest if not given)
:param inherit: whether to enable tag inheritance
:param logfile: path to file where package source tags should be logged
:param exclude_packages: an iterable of package names that should be ignored
"""
result_rpms = []
result_srpms = []
exclude_packages = set(exclude_packages or [])
if type(event) is dict:
event = event["id"]
@ -398,6 +402,12 @@ class KojiPackageSet(PackageSetBase):
skipped_arches.append(rpm_info["arch"])
continue
if exclude_packages and rpm_info["name"] in exclude_packages and rpm_info["arch"] != "src":
self.log_debug(
"Skipping %(name)s-%(version)s-%(release)s.%(arch)s" % rpm_info
)
continue
if (self.populate_only_packages and self.packages and
rpm_info['name'] not in self.packages):
skipped_packages_count += 1

41
pungi/phases/pkgset/sources/source_koji.py
View File

@ -256,7 +256,9 @@ def _log_modulemd(compose, variant, mmd):
% (variant.uid, mmd.dup_nsvc())))
def _get_modules_from_koji(compose, koji_wrapper, variant, variant_tags):
def _get_modules_from_koji(
compose, koji_wrapper, variant, variant_tags, module_tag_rpm_filter
):
"""
Loads modules for given `variant` from koji `session`, adds them to
the `variant` and also to `variant_tags` dict.
@ -287,6 +289,8 @@ def _get_modules_from_koji(compose, koji_wrapper, variant, variant_tags):
# This is needed in createrepo phase where metadata is exposed by producmd
variant.module_uid_to_koji_tag[uid] = tag
module_tag_rpm_filter[tag] = set(mmd.get_rpm_filter().get())
module_msg = (
"Module '{uid}' in variant '{variant}' will use Koji tag '{tag}' "
"(as a result of querying module '{module}')"
@ -295,7 +299,7 @@ def _get_modules_from_koji(compose, koji_wrapper, variant, variant_tags):
def _get_modules_from_koji_tags(
compose, koji_wrapper, event_id, variant, variant_tags):
compose, koji_wrapper, event_id, variant, variant_tags, module_tag_rpm_filter):
"""
Loads modules for given `variant` from Koji, adds them to
the `variant` and also to `variant_tags` dict.
@ -381,6 +385,8 @@ def _get_modules_from_koji_tags(
uid += ":{context}".format(**module_data)
variant.module_uid_to_koji_tag[uid] = module_tag
module_tag_rpm_filter[module_tag] = set(mmd.get_rpm_filter().get())
module_msg = "Module {module} in variant {variant} will use Koji tag {tag}.".format(
variant=variant, tag=module_tag, module=build["nvr"])
compose.log_info("%s" % module_msg)
@ -449,6 +455,13 @@ def populate_global_pkgset(compose, koji_wrapper, path_prefix, event_id):
# there are some packages with invalid sigkeys, it raises an exception.
allow_invalid_sigkeys = compose.conf["gather_method"] == "deps"
# Mapping from koji tags to sets of package names that should be filtered
# out. This is basically a workaround for tagging working on build level,
# not rpm level. A module tag may build a package but not want it included.
# This should exclude it from the package set to avoid pulling it in as a
# dependency.
module_tag_rpm_filter = {}
for variant in compose.all_variants.values():
# pkgset storing the packages belonging to this particular variant.
variant.pkgset = pungi.phases.pkgset.pkgsets.KojiPackageSet(
@ -470,12 +483,24 @@ def populate_global_pkgset(compose, koji_wrapper, path_prefix, event_id):
compose.paths.work.topdir(arch="global"),
"koji-tag-module-%s.yaml" % variant.uid)
_get_modules_from_koji_tags(
compose, koji_wrapper, event_id, variant, variant_tags)
compose,
koji_wrapper,
event_id,
variant,
variant_tags,
module_tag_rpm_filter,
)
elif variant.modules:
included_modules_file = os.path.join(
compose.paths.work.topdir(arch="global"),
"koji-module-%s.yaml" % variant.uid)
_get_modules_from_koji(compose, koji_wrapper, variant, variant_tags)
_get_modules_from_koji(
compose,
koji_wrapper,
variant,
variant_tags,
module_tag_rpm_filter,
)
# Ensure that every tag added to `variant_tags` is added also to
# `compose_tags`.
@ -536,7 +561,13 @@ def populate_global_pkgset(compose, koji_wrapper, path_prefix, event_id):
None, 'packages_from_%s' % compose_tag.replace('/', '_'))
is_traditional = compose_tag in compose.conf.get('pkgset_koji_tag', [])
should_inherit = inherit if is_traditional else inherit_modules
pkgset.populate(compose_tag, event_id, inherit=should_inherit, logfile=logfile)
pkgset.populate(
compose_tag,
event_id,
inherit=should_inherit,
logfile=logfile,
exclude_packages=module_tag_rpm_filter.get(compose_tag),
)
for variant in compose.all_variants.values():
if compose_tag in variant_tags[variant]:
# Optimization for case where we have just single compose

48
tests/test_pkgset_source_koji.py
View File

@ -113,11 +113,23 @@ class TestPopulateGlobalPkgset(helpers.PungiTestCase):
self.assertIs(pkgset, orig_pkgset)
self.assertEqual(
pkgset.mock_calls,
[mock.call.populate('f25', 123456, inherit=True,
logfile=self.topdir + '/logs/global/packages_from_f25.global.log'),
mock.call.save_file_list(self.topdir + '/work/global/package_list/global.conf',
remove_path_prefix='/prefix'),
mock.call.save_file_cache(self.topdir + '/work/global/pkgset_file_cache.pickle')])
[
mock.call.populate(
'f25',
123456,
inherit=True,
logfile=self.topdir + '/logs/global/packages_from_f25.global.log',
exclude_packages=None,
),
mock.call.save_file_list(
self.topdir + '/work/global/package_list/global.conf',
remove_path_prefix='/prefix',
),
mock.call.save_file_cache(
self.topdir + '/work/global/pkgset_file_cache.pickle'
),
]
)
self.assertItemsEqual(pickle_dumps.call_args_list,
[mock.call(orig_pkgset)])
with open(self.pkgset_path) as f:
@ -211,10 +223,28 @@ data:
self.compose, self.koji_wrapper, '/prefix', 123456)
self.assertIs(pkgset, orig_pkgset)
pkgset.assert_has_calls([mock.call.populate('f25', 123456, inherit=True,
logfile=self.topdir + '/logs/global/packages_from_f25.global.log')])
pkgset.assert_has_calls([mock.call.populate('f25-extra', 123456, inherit=True,
logfile=self.topdir + '/logs/global/packages_from_f25-extra.global.log')])
pkgset.assert_has_calls(
[
mock.call.populate(
'f25',
123456,
inherit=True,
logfile=self.topdir + '/logs/global/packages_from_f25.global.log',
exclude_packages=None,
),
]
)
pkgset.assert_has_calls(
[
mock.call.populate(
'f25-extra',
123456,
inherit=True,
logfile=self.topdir + '/logs/global/packages_from_f25-extra.global.log',
exclude_packages=None,
),
]
)
pkgset.assert_has_calls([mock.call.save_file_list(self.topdir + '/work/global/package_list/global.conf',
remove_path_prefix='/prefix')])
# for each tag, call pkgset.fast_merge once for each variant and once for global pkgset