From ef058d1f9b16726996d9a89bbe848cd2d4a784d2 Mon Sep 17 00:00:00 2001 From: Jan Kaluza Date: Mon, 12 Feb 2018 13:21:07 +0100 Subject: [PATCH] Fail early if input packages are unsigned Use 'get_packages_to_gather' to fail early if these packages are not signed with right key. This prevents us from having to wait for the repo to be created and depsolving to finish. Unsigned dependencies will still be reported later than previously. Signed-off-by: Jan Kaluza --- pungi/phases/gather/methods/method_deps.py | 10 ++--- pungi/phases/pkgset/pkgsets.py | 43 ++++++++++++++++++++-- pungi/phases/pkgset/sources/source_koji.py | 28 +++++++------- tests/test_pkgset_pkgsets.py | 4 +- 4 files changed, 60 insertions(+), 25 deletions(-) diff --git a/pungi/phases/gather/methods/method_deps.py b/pungi/phases/gather/methods/method_deps.py index e144415f..6c3e94a9 100644 --- a/pungi/phases/gather/methods/method_deps.py +++ b/pungi/phases/gather/methods/method_deps.py @@ -54,16 +54,16 @@ def raise_on_invalid_sigkeys(arch, variant, package_sets, result): Raises RuntimeError if some package in compose is signed with an invalid sigkey. """ - invalid_sigkeys_rpms = [] + invalid_sigkey_rpms = [] for package in result["rpm"]: name = parse_nvra(package["path"])["name"] - for forbidden_package in package_sets["global"].invalid_sigkeys_rpms(): + for forbidden_package in package_sets["global"].invalid_sigkey_rpms: if name == forbidden_package["name"]: - invalid_sigkeys_rpms.append(forbidden_package) + invalid_sigkey_rpms.append(forbidden_package) - if invalid_sigkeys_rpms: + if invalid_sigkey_rpms: package_sets["global"].raise_invalid_sigkeys_exception( - invalid_sigkeys_rpms) + invalid_sigkey_rpms) def _format_packages(pkgs): diff --git a/pungi/phases/pkgset/pkgsets.py b/pungi/phases/pkgset/pkgsets.py index 8ede7815..f7060ec9 100644 --- a/pungi/phases/pkgset/pkgsets.py +++ b/pungi/phases/pkgset/pkgsets.py @@ -79,7 +79,8 @@ class PackageSetBase(kobo.log.LoggingBase): self._invalid_sigkey_rpms = [] self._allow_invalid_sigkeys = allow_invalid_sigkeys - def invalid_sigkeys_rpms(self): + @property + def invalid_sigkey_rpms(self): return self._invalid_sigkey_rpms def __getitem__(self, name): @@ -234,13 +235,38 @@ class FilelistPackageSet(PackageSetBase): class KojiPackageSet(PackageSetBase): def __init__(self, koji_wrapper, sigkey_ordering, arches=None, logger=None, - packages=None, allow_invalid_sigkeys=False): + packages=None, allow_invalid_sigkeys=False, + populate_only_packages=False): + """ + Creates new KojiPackageSet. + + :param list sigkey_ordering: Ordered list of sigkey strings. When + getting package from Koji, KojiPackageSet tries to get the package + signed by sigkey from this list. If None or "" appears in this + list, unsigned package is used. + :param list arches: List of arches to get the packages for. + :param logging.Logger logger: Logger instance to use for logging. + :param list packages: List of package names to be used when + `allow_invalid_sigkeys` or `populate_only_packages` is set. + :param bool allow_invalid_sigkeys: When True, packages *not* listed in + the `packages` list are added to KojiPackageSet even if they have + invalid sigkey. This is useful in case Koji tag contains some + unsigned packages, but we know they won't appear in a compose. + When False, all packages in Koji tag must have valid sigkey as + defined in `sigkey_ordering`. + :param bool populate_only_packages. When True, only packages in + `packages` list are added to KojiPackageSet. This can save time + when generating compose from predefined list of packages from big + Koji tag. + When False, all packages from Koji tag are added to KojiPackageSet. + """ super(KojiPackageSet, self).__init__(sigkey_ordering=sigkey_ordering, arches=arches, logger=logger, allow_invalid_sigkeys=allow_invalid_sigkeys) self.koji_wrapper = koji_wrapper # Names of packages to look for in the Koji tag. self.packages = set(packages or []) + self.populate_only_packages = populate_only_packages def __getstate__(self): result = self.__dict__.copy() @@ -283,7 +309,7 @@ class KojiPackageSet(PackageSetBase): if os.path.isfile(rpm_path): return rpm_path - if self._allow_invalid_sigkeys: + if self._allow_invalid_sigkeys and rpm_info["name"] not in self.packages: # use an unsigned copy (if allowed) rpm_path = os.path.join(pathinfo.build(build_info), pathinfo.rpm(rpm_info)) paths.append(rpm_path) @@ -333,7 +359,8 @@ class KojiPackageSet(PackageSetBase): skipped_arches.append(rpm_info["arch"]) continue - if self.packages and rpm_info['name'] not in self.packages: + if (self.populate_only_packages and self.packages and + rpm_info['name'] not in self.packages): skipped_packages_count += 1 continue @@ -352,6 +379,14 @@ class KojiPackageSet(PackageSetBase): result = self.read_packages(result_rpms, result_srpms) + # Check that after reading the packages, every package that is + # included in a compose has the right sigkey. + if self._invalid_sigkey_rpms: + invalid_sigkey_rpms = [rpm for rpm in self._invalid_sigkey_rpms + if rpm["name"] in self.packages] + if invalid_sigkey_rpms: + self.raise_invalid_sigkeys_exception(invalid_sigkey_rpms) + # Create a log with package NEVRAs and the tag they are coming from if logfile: with open(logfile, 'w') as f: diff --git a/pungi/phases/pkgset/sources/source_koji.py b/pungi/phases/pkgset/sources/source_koji.py index 9cc0bc34..8a706a61 100644 --- a/pungi/phases/pkgset/sources/source_koji.py +++ b/pungi/phases/pkgset/sources/source_koji.py @@ -205,14 +205,16 @@ def populate_global_pkgset(compose, koji_wrapper, path_prefix, event_id): # are really needed to do the compose and safe lot of time and resources # here. This only works if we are not creating bootable images. Those could # include packages that are not in the compose. - packages_to_gather = [] + packages_to_gather, groups = get_packages_to_gather( + compose, include_arch=False, include_prepopulated=True) + if groups: + comps = CompsWrapper(compose.paths.work.comps()) + for group in groups: + packages_to_gather += comps.get_packages(group) if compose.conf["gather_method"] == "nodeps" and not compose.conf.get('bootable'): - packages_to_gather, groups = get_packages_to_gather( - compose, include_arch=False, include_prepopulated=True) - if groups: - comps = CompsWrapper(compose.paths.work.comps()) - for group in groups: - packages_to_gather += comps.get_packages(group) + populate_only_packages_to_gather = True + else: + populate_only_packages_to_gather = False # In case we use "deps" gather_method, there might be some packages in # the Koji tag which are not signed with proper sigkey. However, these @@ -221,16 +223,13 @@ def populate_global_pkgset(compose, koji_wrapper, path_prefix, event_id): # In this case, we allow even packages with invalid sigkeys to be returned # by PKGSET phase and later, the gather phase checks its results and if # there are some packages with invalid sigkeys, it raises an exception. - if compose.conf["gather_method"] == "deps": - allow_invalid_sigkeys = True - else: - allow_invalid_sigkeys = False + allow_invalid_sigkeys = compose.conf["gather_method"] == "deps" session = get_pdc_client_session(compose) for variant in compose.all_variants.values(): variant.pkgset = pungi.phases.pkgset.pkgsets.KojiPackageSet( koji_wrapper, compose.conf["sigkeys"], logger=compose._logger, - arches=all_arches, allow_invalid_sigkeys=allow_invalid_sigkeys) + arches=all_arches) variant_tags[variant] = [] pdc_module_file = os.path.join(compose.paths.work.topdir(arch="global"), "pdc-module-%s.json" % variant.uid) @@ -297,7 +296,7 @@ def populate_global_pkgset(compose, koji_wrapper, path_prefix, event_id): else: global_pkgset = pungi.phases.pkgset.pkgsets.KojiPackageSet( koji_wrapper, compose.conf["sigkeys"], logger=compose._logger, - arches=all_arches, allow_invalid_sigkeys=allow_invalid_sigkeys) + arches=all_arches) # Get package set for each compose tag and merge it to global package # list. Also prepare per-variant pkgset, because we do not have list # of binary RPMs in module definition - there is just list of SRPMs. @@ -307,7 +306,8 @@ def populate_global_pkgset(compose, koji_wrapper, path_prefix, event_id): pkgset = pungi.phases.pkgset.pkgsets.KojiPackageSet( koji_wrapper, compose.conf["sigkeys"], logger=compose._logger, arches=all_arches, packages=packages_to_gather, - allow_invalid_sigkeys=allow_invalid_sigkeys) + allow_invalid_sigkeys=allow_invalid_sigkeys, + populate_only_packages=populate_only_packages_to_gather) # Create a filename for log with package-to-tag mapping. The tag # name is included in filename, so any slashes in it are replaced # with underscores just to be safe. diff --git a/tests/test_pkgset_pkgsets.py b/tests/test_pkgset_pkgsets.py index 4479737c..c3312090 100644 --- a/tests/test_pkgset_pkgsets.py +++ b/tests/test_pkgset_pkgsets.py @@ -276,7 +276,7 @@ class TestKojiPkgset(PkgsetCompareMixin, helpers.PungiTestCase): [mock.call.listTaggedRPMS('f25', event=None, inherit=True, latest=True)]) with self.assertRaises(RuntimeError) as ctx: - pkgset.raise_invalid_sigkeys_exception(pkgset.invalid_sigkeys_rpms()) + pkgset.raise_invalid_sigkeys_exception(pkgset.invalid_sigkey_rpms) figure = re.compile( r'^RPM\(s\) not found for sigs: .+Check log for details.+bash-4\.3\.42-4\.fc24.+bash-debuginfo-4\.3\.42-4\.fc24$', @@ -309,7 +309,7 @@ class TestKojiPkgset(PkgsetCompareMixin, helpers.PungiTestCase): ]) pkgset = pkgsets.KojiPackageSet(self.koji_wrapper, [None], - packages=["bash"]) + packages=["bash"], populate_only_packages=True) result = pkgset.populate('f25', logfile=self.topdir + '/pkgset.log')