diff --git a/pungi/phases/gather/methods/method_deps.py b/pungi/phases/gather/methods/method_deps.py index e144415f..6c3e94a9 100644 --- a/pungi/phases/gather/methods/method_deps.py +++ b/pungi/phases/gather/methods/method_deps.py @@ -54,16 +54,16 @@ def raise_on_invalid_sigkeys(arch, variant, package_sets, result): Raises RuntimeError if some package in compose is signed with an invalid sigkey. """ - invalid_sigkeys_rpms = [] + invalid_sigkey_rpms = [] for package in result["rpm"]: name = parse_nvra(package["path"])["name"] - for forbidden_package in package_sets["global"].invalid_sigkeys_rpms(): + for forbidden_package in package_sets["global"].invalid_sigkey_rpms: if name == forbidden_package["name"]: - invalid_sigkeys_rpms.append(forbidden_package) + invalid_sigkey_rpms.append(forbidden_package) - if invalid_sigkeys_rpms: + if invalid_sigkey_rpms: package_sets["global"].raise_invalid_sigkeys_exception( - invalid_sigkeys_rpms) + invalid_sigkey_rpms) def _format_packages(pkgs): diff --git a/pungi/phases/pkgset/pkgsets.py b/pungi/phases/pkgset/pkgsets.py index 8ede7815..f7060ec9 100644 --- a/pungi/phases/pkgset/pkgsets.py +++ b/pungi/phases/pkgset/pkgsets.py @@ -79,7 +79,8 @@ class PackageSetBase(kobo.log.LoggingBase): self._invalid_sigkey_rpms = [] self._allow_invalid_sigkeys = allow_invalid_sigkeys - def invalid_sigkeys_rpms(self): + @property + def invalid_sigkey_rpms(self): return self._invalid_sigkey_rpms def __getitem__(self, name): @@ -234,13 +235,38 @@ class FilelistPackageSet(PackageSetBase): class KojiPackageSet(PackageSetBase): def __init__(self, koji_wrapper, sigkey_ordering, arches=None, logger=None, - packages=None, allow_invalid_sigkeys=False): + packages=None, allow_invalid_sigkeys=False, + populate_only_packages=False): + """ + Creates new KojiPackageSet. + + :param list sigkey_ordering: Ordered list of sigkey strings. When + getting package from Koji, KojiPackageSet tries to get the package + signed by sigkey from this list. If None or "" appears in this + list, unsigned package is used. + :param list arches: List of arches to get the packages for. + :param logging.Logger logger: Logger instance to use for logging. + :param list packages: List of package names to be used when + `allow_invalid_sigkeys` or `populate_only_packages` is set. + :param bool allow_invalid_sigkeys: When True, packages *not* listed in + the `packages` list are added to KojiPackageSet even if they have + invalid sigkey. This is useful in case Koji tag contains some + unsigned packages, but we know they won't appear in a compose. + When False, all packages in Koji tag must have valid sigkey as + defined in `sigkey_ordering`. + :param bool populate_only_packages. When True, only packages in + `packages` list are added to KojiPackageSet. This can save time + when generating compose from predefined list of packages from big + Koji tag. + When False, all packages from Koji tag are added to KojiPackageSet. + """ super(KojiPackageSet, self).__init__(sigkey_ordering=sigkey_ordering, arches=arches, logger=logger, allow_invalid_sigkeys=allow_invalid_sigkeys) self.koji_wrapper = koji_wrapper # Names of packages to look for in the Koji tag. self.packages = set(packages or []) + self.populate_only_packages = populate_only_packages def __getstate__(self): result = self.__dict__.copy() @@ -283,7 +309,7 @@ class KojiPackageSet(PackageSetBase): if os.path.isfile(rpm_path): return rpm_path - if self._allow_invalid_sigkeys: + if self._allow_invalid_sigkeys and rpm_info["name"] not in self.packages: # use an unsigned copy (if allowed) rpm_path = os.path.join(pathinfo.build(build_info), pathinfo.rpm(rpm_info)) paths.append(rpm_path) @@ -333,7 +359,8 @@ class KojiPackageSet(PackageSetBase): skipped_arches.append(rpm_info["arch"]) continue - if self.packages and rpm_info['name'] not in self.packages: + if (self.populate_only_packages and self.packages and + rpm_info['name'] not in self.packages): skipped_packages_count += 1 continue @@ -352,6 +379,14 @@ class KojiPackageSet(PackageSetBase): result = self.read_packages(result_rpms, result_srpms) + # Check that after reading the packages, every package that is + # included in a compose has the right sigkey. + if self._invalid_sigkey_rpms: + invalid_sigkey_rpms = [rpm for rpm in self._invalid_sigkey_rpms + if rpm["name"] in self.packages] + if invalid_sigkey_rpms: + self.raise_invalid_sigkeys_exception(invalid_sigkey_rpms) + # Create a log with package NEVRAs and the tag they are coming from if logfile: with open(logfile, 'w') as f: diff --git a/pungi/phases/pkgset/sources/source_koji.py b/pungi/phases/pkgset/sources/source_koji.py index 9cc0bc34..8a706a61 100644 --- a/pungi/phases/pkgset/sources/source_koji.py +++ b/pungi/phases/pkgset/sources/source_koji.py @@ -205,14 +205,16 @@ def populate_global_pkgset(compose, koji_wrapper, path_prefix, event_id): # are really needed to do the compose and safe lot of time and resources # here. This only works if we are not creating bootable images. Those could # include packages that are not in the compose. - packages_to_gather = [] + packages_to_gather, groups = get_packages_to_gather( + compose, include_arch=False, include_prepopulated=True) + if groups: + comps = CompsWrapper(compose.paths.work.comps()) + for group in groups: + packages_to_gather += comps.get_packages(group) if compose.conf["gather_method"] == "nodeps" and not compose.conf.get('bootable'): - packages_to_gather, groups = get_packages_to_gather( - compose, include_arch=False, include_prepopulated=True) - if groups: - comps = CompsWrapper(compose.paths.work.comps()) - for group in groups: - packages_to_gather += comps.get_packages(group) + populate_only_packages_to_gather = True + else: + populate_only_packages_to_gather = False # In case we use "deps" gather_method, there might be some packages in # the Koji tag which are not signed with proper sigkey. However, these @@ -221,16 +223,13 @@ def populate_global_pkgset(compose, koji_wrapper, path_prefix, event_id): # In this case, we allow even packages with invalid sigkeys to be returned # by PKGSET phase and later, the gather phase checks its results and if # there are some packages with invalid sigkeys, it raises an exception. - if compose.conf["gather_method"] == "deps": - allow_invalid_sigkeys = True - else: - allow_invalid_sigkeys = False + allow_invalid_sigkeys = compose.conf["gather_method"] == "deps" session = get_pdc_client_session(compose) for variant in compose.all_variants.values(): variant.pkgset = pungi.phases.pkgset.pkgsets.KojiPackageSet( koji_wrapper, compose.conf["sigkeys"], logger=compose._logger, - arches=all_arches, allow_invalid_sigkeys=allow_invalid_sigkeys) + arches=all_arches) variant_tags[variant] = [] pdc_module_file = os.path.join(compose.paths.work.topdir(arch="global"), "pdc-module-%s.json" % variant.uid) @@ -297,7 +296,7 @@ def populate_global_pkgset(compose, koji_wrapper, path_prefix, event_id): else: global_pkgset = pungi.phases.pkgset.pkgsets.KojiPackageSet( koji_wrapper, compose.conf["sigkeys"], logger=compose._logger, - arches=all_arches, allow_invalid_sigkeys=allow_invalid_sigkeys) + arches=all_arches) # Get package set for each compose tag and merge it to global package # list. Also prepare per-variant pkgset, because we do not have list # of binary RPMs in module definition - there is just list of SRPMs. @@ -307,7 +306,8 @@ def populate_global_pkgset(compose, koji_wrapper, path_prefix, event_id): pkgset = pungi.phases.pkgset.pkgsets.KojiPackageSet( koji_wrapper, compose.conf["sigkeys"], logger=compose._logger, arches=all_arches, packages=packages_to_gather, - allow_invalid_sigkeys=allow_invalid_sigkeys) + allow_invalid_sigkeys=allow_invalid_sigkeys, + populate_only_packages=populate_only_packages_to_gather) # Create a filename for log with package-to-tag mapping. The tag # name is included in filename, so any slashes in it are replaced # with underscores just to be safe. diff --git a/tests/test_pkgset_pkgsets.py b/tests/test_pkgset_pkgsets.py index 4479737c..c3312090 100644 --- a/tests/test_pkgset_pkgsets.py +++ b/tests/test_pkgset_pkgsets.py @@ -276,7 +276,7 @@ class TestKojiPkgset(PkgsetCompareMixin, helpers.PungiTestCase): [mock.call.listTaggedRPMS('f25', event=None, inherit=True, latest=True)]) with self.assertRaises(RuntimeError) as ctx: - pkgset.raise_invalid_sigkeys_exception(pkgset.invalid_sigkeys_rpms()) + pkgset.raise_invalid_sigkeys_exception(pkgset.invalid_sigkey_rpms) figure = re.compile( r'^RPM\(s\) not found for sigs: .+Check log for details.+bash-4\.3\.42-4\.fc24.+bash-debuginfo-4\.3\.42-4\.fc24$', @@ -309,7 +309,7 @@ class TestKojiPkgset(PkgsetCompareMixin, helpers.PungiTestCase): ]) pkgset = pkgsets.KojiPackageSet(self.koji_wrapper, [None], - packages=["bash"]) + packages=["bash"], populate_only_packages=True) result = pkgset.populate('f25', logfile=self.topdir + '/pkgset.log')