From c75f4a1e96172dccafde20acd34ade24dbfcd33a Mon Sep 17 00:00:00 2001 From: Jan Kaluza Date: Mon, 12 Feb 2018 08:02:45 +0100 Subject: [PATCH] Allow composing from tag with unsigned packages There can be packages in the tag that will not end up in the compose. Instead of failing immediately with error, this patch delays the check until after depsolving finishes and only checks packages that will really be included. This is not an issue for nodeps compose, as that already pulls in only packages that will be composed and nothing else. Merges: https://pagure.io/pungi/pull-request/843 Signed-off-by: Jan Kaluza --- pungi/phases/gather/methods/method_deps.py | 19 ++++++++++ pungi/phases/pkgset/pkgsets.py | 43 ++++++++++++++++------ pungi/phases/pkgset/sources/source_koji.py | 19 ++++++++-- tests/test_pkgset_pkgsets.py | 18 +++++++++ 4 files changed, 85 insertions(+), 14 deletions(-) diff --git a/pungi/phases/gather/methods/method_deps.py b/pungi/phases/gather/methods/method_deps.py index 7c9e8fb6..e144415f 100644 --- a/pungi/phases/gather/methods/method_deps.py +++ b/pungi/phases/gather/methods/method_deps.py @@ -18,6 +18,7 @@ import os from kobo.shortcuts import run from kobo.pkgset import SimpleRpmWrapper, RpmWrapper +from kobo.rpmlib import parse_nvra from pungi.util import rmtree, get_arch_variant_data from pungi.wrappers.pungi import PungiWrapper @@ -43,10 +44,28 @@ class GatherMethodDeps(pungi.phases.gather.method.GatherMethodBase): fulltree_excludes=fulltree_excludes, prepopulate=prepopulate, source_name=self.source_name) result, missing_deps = resolve_deps(self.compose, arch, variant, source_name=self.source_name) + raise_on_invalid_sigkeys(arch, variant, package_sets, result) check_deps(self.compose, arch, variant, missing_deps) return result +def raise_on_invalid_sigkeys(arch, variant, package_sets, result): + """ + Raises RuntimeError if some package in compose is signed with an invalid + sigkey. + """ + invalid_sigkeys_rpms = [] + for package in result["rpm"]: + name = parse_nvra(package["path"])["name"] + for forbidden_package in package_sets["global"].invalid_sigkeys_rpms(): + if name == forbidden_package["name"]: + invalid_sigkeys_rpms.append(forbidden_package) + + if invalid_sigkeys_rpms: + package_sets["global"].raise_invalid_sigkeys_exception( + invalid_sigkeys_rpms) + + def _format_packages(pkgs): """Sort packages and merge name with arch.""" for pkg, pkg_arch in sorted(pkgs): diff --git a/pungi/phases/pkgset/pkgsets.py b/pungi/phases/pkgset/pkgsets.py index 4d902f4f..8ede7815 100644 --- a/pungi/phases/pkgset/pkgsets.py +++ b/pungi/phases/pkgset/pkgsets.py @@ -67,7 +67,8 @@ class ReaderThread(WorkerThread): class PackageSetBase(kobo.log.LoggingBase): - def __init__(self, sigkey_ordering, arches=None, logger=None): + def __init__(self, sigkey_ordering, arches=None, logger=None, + allow_invalid_sigkeys=False): super(PackageSetBase, self).__init__(logger=logger) self.file_cache = kobo.pkgset.FileCache(kobo.pkgset.SimpleRpmWrapper) self.sigkey_ordering = sigkey_ordering or [None] @@ -76,6 +77,10 @@ class PackageSetBase(kobo.log.LoggingBase): self.srpms_by_name = {} # RPMs not found for specified sigkeys self._invalid_sigkey_rpms = [] + self._allow_invalid_sigkeys = allow_invalid_sigkeys + + def invalid_sigkeys_rpms(self): + return self._invalid_sigkey_rpms def __getitem__(self, name): return self.file_cache[name] @@ -96,6 +101,19 @@ class PackageSetBase(kobo.log.LoggingBase): self._logger = None self.__dict__.update(data) + def raise_invalid_sigkeys_exception(self, rpminfos): + """ + Raises RuntimeError containing details of RPMs with invalid + sigkeys defined in `rpminfos`. + """ + def nvr_formatter(package_info): + # joins NVR parts of the package with '-' character. + return '-'.join((package_info['name'], package_info['version'], package_info['release'])) + raise RuntimeError( + "RPM(s) not found for sigs: %s. Check log for details. Unsigned packages:\n%s" % ( + self.sigkey_ordering, + '\n'.join(sorted(set([nvr_formatter(rpminfo) for rpminfo in rpminfos]))))) + def read_packages(self, rpms, srpms): srpm_pool = ReaderPool(self, self._logger) rpm_pool = ReaderPool(self, self._logger) @@ -123,14 +141,8 @@ class PackageSetBase(kobo.log.LoggingBase): rpm_pool.stop() self.log_debug("Package set: worker threads stopped (RPMs)") - if self._invalid_sigkey_rpms: - def nvr_formatter(package_info): - # joins NVR parts of the package with '-' character. - return '-'.join((package_info['name'], package_info['version'], package_info['release'])) - raise RuntimeError( - "RPM(s) not found for sigs: %s. Check log for details. Unsigned packages:\n%s" % ( - self.sigkey_ordering, - '\n'.join(sorted(set([nvr_formatter(rpminfo) for rpminfo in self._invalid_sigkey_rpms]))))) + if not self._allow_invalid_sigkeys and self._invalid_sigkey_rpms: + self.raise_invalid_sigkeys_exception(self._invalid_sigkey_rpms) return self.rpms_by_arch @@ -222,9 +234,10 @@ class FilelistPackageSet(PackageSetBase): class KojiPackageSet(PackageSetBase): def __init__(self, koji_wrapper, sigkey_ordering, arches=None, logger=None, - packages=None): + packages=None, allow_invalid_sigkeys=False): super(KojiPackageSet, self).__init__(sigkey_ordering=sigkey_ordering, - arches=arches, logger=logger) + arches=arches, logger=logger, + allow_invalid_sigkeys=allow_invalid_sigkeys) self.koji_wrapper = koji_wrapper # Names of packages to look for in the Koji tag. self.packages = set(packages or []) @@ -270,6 +283,14 @@ class KojiPackageSet(PackageSetBase): if os.path.isfile(rpm_path): return rpm_path + if self._allow_invalid_sigkeys: + # use an unsigned copy (if allowed) + rpm_path = os.path.join(pathinfo.build(build_info), pathinfo.rpm(rpm_info)) + paths.append(rpm_path) + if os.path.isfile(rpm_path): + self._invalid_sigkey_rpms.append(rpm_info) + return rpm_path + self._invalid_sigkey_rpms.append(rpm_info) self.log_error("RPM %s not found for sigs: %s. Paths checked: %s" % (rpm_info, self.sigkey_ordering, paths)) diff --git a/pungi/phases/pkgset/sources/source_koji.py b/pungi/phases/pkgset/sources/source_koji.py index 98eed625..9cc0bc34 100644 --- a/pungi/phases/pkgset/sources/source_koji.py +++ b/pungi/phases/pkgset/sources/source_koji.py @@ -214,11 +214,23 @@ def populate_global_pkgset(compose, koji_wrapper, path_prefix, event_id): for group in groups: packages_to_gather += comps.get_packages(group) + # In case we use "deps" gather_method, there might be some packages in + # the Koji tag which are not signed with proper sigkey. However, these + # packages might never end up in a compose depending on which packages + # from the Koji tag are requested how the deps are resolved in the end. + # In this case, we allow even packages with invalid sigkeys to be returned + # by PKGSET phase and later, the gather phase checks its results and if + # there are some packages with invalid sigkeys, it raises an exception. + if compose.conf["gather_method"] == "deps": + allow_invalid_sigkeys = True + else: + allow_invalid_sigkeys = False + session = get_pdc_client_session(compose) for variant in compose.all_variants.values(): variant.pkgset = pungi.phases.pkgset.pkgsets.KojiPackageSet( koji_wrapper, compose.conf["sigkeys"], logger=compose._logger, - arches=all_arches) + arches=all_arches, allow_invalid_sigkeys=allow_invalid_sigkeys) variant_tags[variant] = [] pdc_module_file = os.path.join(compose.paths.work.topdir(arch="global"), "pdc-module-%s.json" % variant.uid) @@ -285,7 +297,7 @@ def populate_global_pkgset(compose, koji_wrapper, path_prefix, event_id): else: global_pkgset = pungi.phases.pkgset.pkgsets.KojiPackageSet( koji_wrapper, compose.conf["sigkeys"], logger=compose._logger, - arches=all_arches) + arches=all_arches, allow_invalid_sigkeys=allow_invalid_sigkeys) # Get package set for each compose tag and merge it to global package # list. Also prepare per-variant pkgset, because we do not have list # of binary RPMs in module definition - there is just list of SRPMs. @@ -294,7 +306,8 @@ def populate_global_pkgset(compose, koji_wrapper, path_prefix, event_id): "'%s'" % compose_tag) pkgset = pungi.phases.pkgset.pkgsets.KojiPackageSet( koji_wrapper, compose.conf["sigkeys"], logger=compose._logger, - arches=all_arches, packages=packages_to_gather) + arches=all_arches, packages=packages_to_gather, + allow_invalid_sigkeys=allow_invalid_sigkeys) # Create a filename for log with package-to-tag mapping. The tag # name is included in filename, so any slashes in it are replaced # with underscores just to be safe. diff --git a/tests/test_pkgset_pkgsets.py b/tests/test_pkgset_pkgsets.py index 2c5d00e0..4479737c 100644 --- a/tests/test_pkgset_pkgsets.py +++ b/tests/test_pkgset_pkgsets.py @@ -265,6 +265,24 @@ class TestKojiPkgset(PkgsetCompareMixin, helpers.PungiTestCase): re.DOTALL) self.assertRegexpMatches(str(ctx.exception), figure) + def test_can_not_find_signed_package_allow_invalid_sigkeys(self): + pkgset = pkgsets.KojiPackageSet(self.koji_wrapper, ['cafebabe'], arches=['x86_64'], + allow_invalid_sigkeys=True) + + pkgset.populate('f25') + + self.assertEqual( + self.koji_wrapper.koji_proxy.mock_calls, + [mock.call.listTaggedRPMS('f25', event=None, inherit=True, latest=True)]) + + with self.assertRaises(RuntimeError) as ctx: + pkgset.raise_invalid_sigkeys_exception(pkgset.invalid_sigkeys_rpms()) + + figure = re.compile( + r'^RPM\(s\) not found for sigs: .+Check log for details.+bash-4\.3\.42-4\.fc24.+bash-debuginfo-4\.3\.42-4\.fc24$', + re.DOTALL) + self.assertRegexpMatches(str(ctx.exception), figure) + def test_can_not_find_any_package(self): pkgset = pkgsets.KojiPackageSet(self.koji_wrapper, ['cafebabe', None], arches=['x86_64'])