almalinux/pungi
14
1
Fork 0
Code Issues Pull Requests Releases Activity

pkgset: Refactor hiding unused modular packages

Browse Source 
A module build can create packages that are tagged in the content tag,
but should not be included in the module. Originally Pungi didn't know
what exactly the module contains and so it needed to apply filters to
exclude stuff that was definitely out.

With getting the final MMD from Koji, we can actually make this a bit
more strict by only keeping packages that we know we need.

When processing each content tag, we can put into package set only
packages that are included in some module using that tag. This should
work with -devel modules as well. Both the regular and -devel modules
will contribute to the set and thus all packages will go to the package
set.

Signed-off-by: Lubomír Sedlář <lsedlar@redhat.com>
This commit is contained in:
Lubomír Sedlář 2019-03-18 13:18:42 +01:00
parent 9f9b784e64
commit 66a127c313
3 changed files with 41 additions and 41 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
pungi/phases/pkgset/pkgsets.py
View File

@ -410,18 +410,19 @@ class KojiPackageSet(PackageSetBase):
return None return None
def populate( def populate(
self, tag, event=None, inherit=True, logfile=None, exclude_packages=None self, tag, event=None, inherit=True, logfile=None, include_packages=None,
): ):
"""Populate the package set with packages from given tag. """Populate the package set with packages from given tag.
:param event: the Koji event to query at (or latest if not given) :param event: the Koji event to query at (or latest if not given)
:param inherit: whether to enable tag inheritance :param inherit: whether to enable tag inheritance
:param logfile: path to file where package source tags should be logged :param logfile: path to file where package source tags should be logged
:param exclude_packages: an iterable of package names that should be ignored :param include_packages: an iterable of package names that should be
included, all others are skipped.
""" """
result_rpms = [] result_rpms = []
result_srpms = [] result_srpms = []
exclude_packages = set(exclude_packages or []) include_packages = set(include_packages or [])
if type(event) is dict: if type(event) is dict:
event = event["id"] event = event["id"]
@ -453,7 +454,7 @@ class KojiPackageSet(PackageSetBase):
skipped_arches.append(rpm_info["arch"]) skipped_arches.append(rpm_info["arch"])
continue continue
if exclude_packages and rpm_info["name"] in exclude_packages and rpm_info["arch"] != "src": if include_packages and rpm_info["name"] not in include_packages and rpm_info["arch"] != "src":
self.log_debug( self.log_debug(
"Skipping %(name)s-%(version)s-%(release)s.%(arch)s" % rpm_info "Skipping %(name)s-%(version)s-%(release)s.%(arch)s" % rpm_info
) )

67
pungi/phases/pkgset/sources/source_koji.py
View File

@ -249,9 +249,7 @@ def _add_module_to_variant(koji_wrapper, variant, build, add_to_variant_modules=
return source_mmd return source_mmd
def _get_modules_from_koji( def _get_modules_from_koji(compose, koji_wrapper, event, variant, variant_tags):
compose, koji_wrapper, event, variant, variant_tags, module_tag_rpm_filter
):
""" """
Loads modules for given `variant` from koji `session`, adds them to Loads modules for given `variant` from koji `session`, adds them to
the `variant` and also to `variant_tags` dict. the `variant` and also to `variant_tags` dict.
@ -271,20 +269,17 @@ def _get_modules_from_koji(
mmd = _add_module_to_variant(koji_wrapper, variant, koji_module) mmd = _add_module_to_variant(koji_wrapper, variant, koji_module)
tag = koji_module["tag"] tag = koji_module["tag"]
uid = ':'.join([koji_module['name'], koji_module['stream'], nsvc = mmd.dup_nsvc()
koji_module['version'], koji_module['context']])
variant_tags[variant].append(tag) variant_tags[variant].append(tag)
# Store mapping module-uid --> koji_tag into variant. # Store mapping NSVC --> koji_tag into variant.
# This is needed in createrepo phase where metadata is exposed by producmd # This is needed in createrepo phase where metadata is exposed by producmd
variant.module_uid_to_koji_tag[uid] = tag variant.module_uid_to_koji_tag[nsvc] = tag
module_tag_rpm_filter[tag] = set(mmd.get_rpm_filter().get())
module_msg = ( module_msg = (
"Module '{uid}' in variant '{variant}' will use Koji tag '{tag}' " "Module '{uid}' in variant '{variant}' will use Koji tag '{tag}' "
"(as a result of querying module '{module}')" "(as a result of querying module '{module}')"
).format(uid=uid, variant=variant, tag=tag, module=module["name"]) ).format(uid=nsvc, variant=variant, tag=tag, module=module["name"])
compose.log_info("%s" % module_msg) compose.log_info("%s" % module_msg)
@ -361,8 +356,7 @@ def filter_by_whitelist(compose, module_builds, input_modules):
return modules_to_keep return modules_to_keep
def _get_modules_from_koji_tags( def _get_modules_from_koji_tags(compose, koji_wrapper, event_id, variant, variant_tags):
compose, koji_wrapper, event_id, variant, variant_tags, module_tag_rpm_filter):
""" """
Loads modules for given `variant` from Koji, adds them to Loads modules for given `variant` from Koji, adds them to
the `variant` and also to `variant_tags` dict. the `variant` and also to `variant_tags` dict.
@ -436,18 +430,8 @@ def _get_modules_from_koji_tags(
# Store mapping module-uid --> koji_tag into variant. # Store mapping module-uid --> koji_tag into variant.
# This is needed in createrepo phase where metadata is exposed by producmd # This is needed in createrepo phase where metadata is exposed by producmd
module_data = build.get("extra", {}).get("typeinfo", {}).get("module", {}) nsvc = mmd.dup_nsvc()
try: variant.module_uid_to_koji_tag[nsvc] = module_tag
uid = "{name}:{stream}".format(**module_data)
except KeyError as e:
raise KeyError("Unable to create uid in format name:stream %s" % e)
if module_data.get("version"):
uid += ":{version}".format(**module_data)
if module_data.get("context"):
uid += ":{context}".format(**module_data)
variant.module_uid_to_koji_tag[uid] = module_tag
module_tag_rpm_filter[module_tag] = set(mmd.get_rpm_filter().get())
module_msg = "Module {module} in variant {variant} will use Koji tag {tag}.".format( module_msg = "Module {module} in variant {variant} will use Koji tag {tag}.".format(
variant=variant, tag=module_tag, module=build["nvr"]) variant=variant, tag=module_tag, module=build["nvr"])
@ -517,13 +501,6 @@ def populate_global_pkgset(compose, koji_wrapper, path_prefix, event):
# there are some packages with invalid sigkeys, it raises an exception. # there are some packages with invalid sigkeys, it raises an exception.
allow_invalid_sigkeys = compose.conf["gather_method"] == "deps" allow_invalid_sigkeys = compose.conf["gather_method"] == "deps"
# Mapping from koji tags to sets of package names that should be filtered
# out. This is basically a workaround for tagging working on build level,
# not rpm level. A module tag may build a package but not want it included.
# This should exclude it from the package set to avoid pulling it in as a
# dependency.
module_tag_rpm_filter = {}
for variant in compose.all_variants.values(): for variant in compose.all_variants.values():
# pkgset storing the packages belonging to this particular variant. # pkgset storing the packages belonging to this particular variant.
variant.pkgset = pungi.phases.pkgset.pkgsets.KojiPackageSet( variant.pkgset = pungi.phases.pkgset.pkgsets.KojiPackageSet(
@ -550,7 +527,6 @@ def populate_global_pkgset(compose, koji_wrapper, path_prefix, event):
event, event,
variant, variant,
variant_tags, variant_tags,
module_tag_rpm_filter,
) )
elif variant.modules: elif variant.modules:
included_modules_file = os.path.join( included_modules_file = os.path.join(
@ -562,7 +538,6 @@ def populate_global_pkgset(compose, koji_wrapper, path_prefix, event):
event, event,
variant, variant,
variant_tags, variant_tags,
module_tag_rpm_filter,
) )
# Ensure that every tag added to `variant_tags` is added also to # Ensure that every tag added to `variant_tags` is added also to
@ -638,12 +613,36 @@ def populate_global_pkgset(compose, koji_wrapper, path_prefix, event):
None, 'packages_from_%s' % compose_tag.replace('/', '_')) None, 'packages_from_%s' % compose_tag.replace('/', '_'))
is_traditional = compose_tag in compose.conf.get('pkgset_koji_tag', []) is_traditional = compose_tag in compose.conf.get('pkgset_koji_tag', [])
should_inherit = inherit if is_traditional else inherit_modules should_inherit = inherit if is_traditional else inherit_modules
# If we're processing a modular tag, we have an exact list of
# packages that will be used. This is basically a workaround for
# tagging working on build level, not rpm level. A module tag may
# build a package but not want it included. This should include
# only packages that are actually in modules. It's possible two
# module builds will use the same tag, particularly a -devel module
# is sharing a tag with its regular version.
# The ultimate goal of the mapping is to avoid a package built in modular
# tag to be used as a dependency of some non-modular package.
modular_packages = set()
for variant in compose.all_variants.values():
for nsvc, modular_tag in variant.module_uid_to_koji_tag.items():
if modular_tag != compose_tag:
# Not current tag, skip it
continue
for arch_modules in variant.arch_mmds.values():
arch_mmd = arch_modules[nsvc]
if arch_mmd:
modular_packages.update(
nevra.rsplit("-", 2)[0]
for nevra in arch_mmd.get_rpm_artifacts().get()
)
pkgset.populate( pkgset.populate(
compose_tag, compose_tag,
event, event,
inherit=should_inherit, inherit=should_inherit,
logfile=logfile, logfile=logfile,
exclude_packages=module_tag_rpm_filter.get(compose_tag), include_packages=modular_packages,
) )
for variant in compose.all_variants.values(): for variant in compose.all_variants.values():
if compose_tag in variant_tags[variant]: if compose_tag in variant_tags[variant]:

6
tests/test_pkgset_source_koji.py
View File

@ -115,7 +115,7 @@ class TestPopulateGlobalPkgset(helpers.PungiTestCase):
123456, 123456,
inherit=True, inherit=True,
logfile=self.topdir + '/logs/global/packages_from_f25.global.log', logfile=self.topdir + '/logs/global/packages_from_f25.global.log',
exclude_packages=None, include_packages=set(),
), ),
mock.call.save_file_list( mock.call.save_file_list(
self.topdir + '/work/global/package_list/global.conf', self.topdir + '/work/global/package_list/global.conf',
@ -154,7 +154,7 @@ class TestPopulateGlobalPkgset(helpers.PungiTestCase):
123456, 123456,
inherit=True, inherit=True,
logfile=self.topdir + '/logs/global/packages_from_f25.global.log', logfile=self.topdir + '/logs/global/packages_from_f25.global.log',
exclude_packages=None, include_packages=set(),
), ),
] ]
) )
@ -165,7 +165,7 @@ class TestPopulateGlobalPkgset(helpers.PungiTestCase):
123456, 123456,
inherit=True, inherit=True,
logfile=self.topdir + '/logs/global/packages_from_f25-extra.global.log', logfile=self.topdir + '/logs/global/packages_from_f25-extra.global.log',
exclude_packages=None, include_packages=set(),
), ),
] ]
) )