diff --git a/pungi/phases/pkgset/pkgsets.py b/pungi/phases/pkgset/pkgsets.py index 92db809d..c1244df0 100644 --- a/pungi/phases/pkgset/pkgsets.py +++ b/pungi/phases/pkgset/pkgsets.py @@ -361,15 +361,19 @@ class KojiPackageSet(PackageSetBase): % (rpm_info, self.sigkey_ordering, paths)) return None - def populate(self, tag, event=None, inherit=True, logfile=None): + def populate( + self, tag, event=None, inherit=True, logfile=None, exclude_packages=None + ): """Populate the package set with packages from given tag. :param event: the Koji event to query at (or latest if not given) :param inherit: whether to enable tag inheritance :param logfile: path to file where package source tags should be logged + :param exclude_packages: an iterable of package names that should be ignored """ result_rpms = [] result_srpms = [] + exclude_packages = set(exclude_packages or []) if type(event) is dict: event = event["id"] @@ -398,6 +402,12 @@ class KojiPackageSet(PackageSetBase): skipped_arches.append(rpm_info["arch"]) continue + if exclude_packages and rpm_info["name"] in exclude_packages and rpm_info["arch"] != "src": + self.log_debug( + "Skipping %(name)s-%(version)s-%(release)s.%(arch)s" % rpm_info + ) + continue + if (self.populate_only_packages and self.packages and rpm_info['name'] not in self.packages): skipped_packages_count += 1 diff --git a/pungi/phases/pkgset/sources/source_koji.py b/pungi/phases/pkgset/sources/source_koji.py index 32502612..3381d792 100644 --- a/pungi/phases/pkgset/sources/source_koji.py +++ b/pungi/phases/pkgset/sources/source_koji.py @@ -256,7 +256,9 @@ def _log_modulemd(compose, variant, mmd): % (variant.uid, mmd.dup_nsvc()))) -def _get_modules_from_koji(compose, koji_wrapper, variant, variant_tags): +def _get_modules_from_koji( + compose, koji_wrapper, variant, variant_tags, module_tag_rpm_filter +): """ Loads modules for given `variant` from koji `session`, adds them to the `variant` and also to `variant_tags` dict. @@ -287,6 +289,8 @@ def _get_modules_from_koji(compose, koji_wrapper, variant, variant_tags): # This is needed in createrepo phase where metadata is exposed by producmd variant.module_uid_to_koji_tag[uid] = tag + module_tag_rpm_filter[tag] = set(mmd.get_rpm_filter().get()) + module_msg = ( "Module '{uid}' in variant '{variant}' will use Koji tag '{tag}' " "(as a result of querying module '{module}')" @@ -295,7 +299,7 @@ def _get_modules_from_koji(compose, koji_wrapper, variant, variant_tags): def _get_modules_from_koji_tags( - compose, koji_wrapper, event_id, variant, variant_tags): + compose, koji_wrapper, event_id, variant, variant_tags, module_tag_rpm_filter): """ Loads modules for given `variant` from Koji, adds them to the `variant` and also to `variant_tags` dict. @@ -381,6 +385,8 @@ def _get_modules_from_koji_tags( uid += ":{context}".format(**module_data) variant.module_uid_to_koji_tag[uid] = module_tag + module_tag_rpm_filter[module_tag] = set(mmd.get_rpm_filter().get()) + module_msg = "Module {module} in variant {variant} will use Koji tag {tag}.".format( variant=variant, tag=module_tag, module=build["nvr"]) compose.log_info("%s" % module_msg) @@ -449,6 +455,13 @@ def populate_global_pkgset(compose, koji_wrapper, path_prefix, event_id): # there are some packages with invalid sigkeys, it raises an exception. allow_invalid_sigkeys = compose.conf["gather_method"] == "deps" + # Mapping from koji tags to sets of package names that should be filtered + # out. This is basically a workaround for tagging working on build level, + # not rpm level. A module tag may build a package but not want it included. + # This should exclude it from the package set to avoid pulling it in as a + # dependency. + module_tag_rpm_filter = {} + for variant in compose.all_variants.values(): # pkgset storing the packages belonging to this particular variant. variant.pkgset = pungi.phases.pkgset.pkgsets.KojiPackageSet( @@ -470,12 +483,24 @@ def populate_global_pkgset(compose, koji_wrapper, path_prefix, event_id): compose.paths.work.topdir(arch="global"), "koji-tag-module-%s.yaml" % variant.uid) _get_modules_from_koji_tags( - compose, koji_wrapper, event_id, variant, variant_tags) + compose, + koji_wrapper, + event_id, + variant, + variant_tags, + module_tag_rpm_filter, + ) elif variant.modules: included_modules_file = os.path.join( compose.paths.work.topdir(arch="global"), "koji-module-%s.yaml" % variant.uid) - _get_modules_from_koji(compose, koji_wrapper, variant, variant_tags) + _get_modules_from_koji( + compose, + koji_wrapper, + variant, + variant_tags, + module_tag_rpm_filter, + ) # Ensure that every tag added to `variant_tags` is added also to # `compose_tags`. @@ -536,7 +561,13 @@ def populate_global_pkgset(compose, koji_wrapper, path_prefix, event_id): None, 'packages_from_%s' % compose_tag.replace('/', '_')) is_traditional = compose_tag in compose.conf.get('pkgset_koji_tag', []) should_inherit = inherit if is_traditional else inherit_modules - pkgset.populate(compose_tag, event_id, inherit=should_inherit, logfile=logfile) + pkgset.populate( + compose_tag, + event_id, + inherit=should_inherit, + logfile=logfile, + exclude_packages=module_tag_rpm_filter.get(compose_tag), + ) for variant in compose.all_variants.values(): if compose_tag in variant_tags[variant]: # Optimization for case where we have just single compose diff --git a/tests/test_pkgset_source_koji.py b/tests/test_pkgset_source_koji.py index 5ea4c427..b6a85f93 100644 --- a/tests/test_pkgset_source_koji.py +++ b/tests/test_pkgset_source_koji.py @@ -113,11 +113,23 @@ class TestPopulateGlobalPkgset(helpers.PungiTestCase): self.assertIs(pkgset, orig_pkgset) self.assertEqual( pkgset.mock_calls, - [mock.call.populate('f25', 123456, inherit=True, - logfile=self.topdir + '/logs/global/packages_from_f25.global.log'), - mock.call.save_file_list(self.topdir + '/work/global/package_list/global.conf', - remove_path_prefix='/prefix'), - mock.call.save_file_cache(self.topdir + '/work/global/pkgset_file_cache.pickle')]) + [ + mock.call.populate( + 'f25', + 123456, + inherit=True, + logfile=self.topdir + '/logs/global/packages_from_f25.global.log', + exclude_packages=None, + ), + mock.call.save_file_list( + self.topdir + '/work/global/package_list/global.conf', + remove_path_prefix='/prefix', + ), + mock.call.save_file_cache( + self.topdir + '/work/global/pkgset_file_cache.pickle' + ), + ] + ) self.assertItemsEqual(pickle_dumps.call_args_list, [mock.call(orig_pkgset)]) with open(self.pkgset_path) as f: @@ -211,10 +223,28 @@ data: self.compose, self.koji_wrapper, '/prefix', 123456) self.assertIs(pkgset, orig_pkgset) - pkgset.assert_has_calls([mock.call.populate('f25', 123456, inherit=True, - logfile=self.topdir + '/logs/global/packages_from_f25.global.log')]) - pkgset.assert_has_calls([mock.call.populate('f25-extra', 123456, inherit=True, - logfile=self.topdir + '/logs/global/packages_from_f25-extra.global.log')]) + pkgset.assert_has_calls( + [ + mock.call.populate( + 'f25', + 123456, + inherit=True, + logfile=self.topdir + '/logs/global/packages_from_f25.global.log', + exclude_packages=None, + ), + ] + ) + pkgset.assert_has_calls( + [ + mock.call.populate( + 'f25-extra', + 123456, + inherit=True, + logfile=self.topdir + '/logs/global/packages_from_f25-extra.global.log', + exclude_packages=None, + ), + ] + ) pkgset.assert_has_calls([mock.call.save_file_list(self.topdir + '/work/global/package_list/global.conf', remove_path_prefix='/prefix')]) # for each tag, call pkgset.fast_merge once for each variant and once for global pkgset