Allow composing from tag with unsigned packages

There can be packages in the tag that will not end up in the compose.
Instead of failing immediately with error, this patch delays the check
until after depsolving finishes and only checks packages that will
really be included.

This is not an issue for nodeps compose, as that already pulls in only
packages that will be composed and nothing else.

Merges: https://pagure.io/pungi/pull-request/843
Signed-off-by: Jan Kaluza <jkaluza@redhat.com>

pungi/phases/gather/methods/method_deps.py

@@ -18,6 +18,7 @@ import os
 
 from kobo.shortcuts import run
 from kobo.pkgset import SimpleRpmWrapper, RpmWrapper
+from kobo.rpmlib import parse_nvra
 
 from pungi.util import rmtree, get_arch_variant_data
 from pungi.wrappers.pungi import PungiWrapper
@@ -43,10 +44,28 @@ class GatherMethodDeps(pungi.phases.gather.method.GatherMethodBase):
                            fulltree_excludes=fulltree_excludes, prepopulate=prepopulate,
                            source_name=self.source_name)
         result, missing_deps = resolve_deps(self.compose, arch, variant, source_name=self.source_name)
+        raise_on_invalid_sigkeys(arch, variant, package_sets, result)
         check_deps(self.compose, arch, variant, missing_deps)
         return result
 
 
+def raise_on_invalid_sigkeys(arch, variant, package_sets, result):
+    """
+    Raises RuntimeError if some package in compose is signed with an invalid
+    sigkey.
+    """
+    invalid_sigkeys_rpms = []
+    for package in result["rpm"]:
+        name = parse_nvra(package["path"])["name"]
+        for forbidden_package in package_sets["global"].invalid_sigkeys_rpms():
+            if name == forbidden_package["name"]:
+                invalid_sigkeys_rpms.append(forbidden_package)
+
+    if invalid_sigkeys_rpms:
+        package_sets["global"].raise_invalid_sigkeys_exception(
+            invalid_sigkeys_rpms)
+
+
 def _format_packages(pkgs):
     """Sort packages and merge name with arch."""
     for pkg, pkg_arch in sorted(pkgs):

pungi/phases/pkgset/pkgsets.py

@@ -67,7 +67,8 @@ class ReaderThread(WorkerThread):
 
 class PackageSetBase(kobo.log.LoggingBase):
 
-    def __init__(self, sigkey_ordering, arches=None, logger=None):
+    def __init__(self, sigkey_ordering, arches=None, logger=None,
+                 allow_invalid_sigkeys=False):
         super(PackageSetBase, self).__init__(logger=logger)
         self.file_cache = kobo.pkgset.FileCache(kobo.pkgset.SimpleRpmWrapper)
         self.sigkey_ordering = sigkey_ordering or [None]
@@ -76,6 +77,10 @@ class PackageSetBase(kobo.log.LoggingBase):
         self.srpms_by_name = {}
         # RPMs not found for specified sigkeys
         self._invalid_sigkey_rpms = []
+        self._allow_invalid_sigkeys = allow_invalid_sigkeys
+
+    def invalid_sigkeys_rpms(self):
+        return self._invalid_sigkey_rpms
 
     def __getitem__(self, name):
         return self.file_cache[name]
@@ -96,6 +101,19 @@ class PackageSetBase(kobo.log.LoggingBase):
         self._logger = None
         self.__dict__.update(data)
 
+    def raise_invalid_sigkeys_exception(self, rpminfos):
+        """
+        Raises RuntimeError containing details of RPMs with invalid
+        sigkeys defined in `rpminfos`.
+        """
+        def nvr_formatter(package_info):
+            # joins NVR parts of the package with '-' character.
+            return '-'.join((package_info['name'], package_info['version'], package_info['release']))
+        raise RuntimeError(
+            "RPM(s) not found for sigs: %s. Check log for details. Unsigned packages:\n%s" % (
+                self.sigkey_ordering,
+                '\n'.join(sorted(set([nvr_formatter(rpminfo) for rpminfo in rpminfos])))))
+
     def read_packages(self, rpms, srpms):
         srpm_pool = ReaderPool(self, self._logger)
         rpm_pool = ReaderPool(self, self._logger)
@@ -123,14 +141,8 @@ class PackageSetBase(kobo.log.LoggingBase):
         rpm_pool.stop()
         self.log_debug("Package set: worker threads stopped (RPMs)")
 
-        if self._invalid_sigkey_rpms:
+        if not self._allow_invalid_sigkeys and self._invalid_sigkey_rpms:
-            def nvr_formatter(package_info):
+            self.raise_invalid_sigkeys_exception(self._invalid_sigkey_rpms)
-                # joins NVR parts of the package with '-' character.
-                return '-'.join((package_info['name'], package_info['version'], package_info['release']))
-            raise RuntimeError(
-                "RPM(s) not found for sigs: %s. Check log for details. Unsigned packages:\n%s" % (
-                    self.sigkey_ordering,
-                    '\n'.join(sorted(set([nvr_formatter(rpminfo) for rpminfo in self._invalid_sigkey_rpms])))))
 
         return self.rpms_by_arch
 
@@ -222,9 +234,10 @@ class FilelistPackageSet(PackageSetBase):
 
 class KojiPackageSet(PackageSetBase):
     def __init__(self, koji_wrapper, sigkey_ordering, arches=None, logger=None,
-                 packages=None):
+                 packages=None, allow_invalid_sigkeys=False):
         super(KojiPackageSet, self).__init__(sigkey_ordering=sigkey_ordering,
-                                             arches=arches, logger=logger)
+                                             arches=arches, logger=logger,
+                                             allow_invalid_sigkeys=allow_invalid_sigkeys)
         self.koji_wrapper = koji_wrapper
         # Names of packages to look for in the Koji tag.
         self.packages = set(packages or [])
@@ -270,6 +283,14 @@ class KojiPackageSet(PackageSetBase):
             if os.path.isfile(rpm_path):
                 return rpm_path
 
+        if self._allow_invalid_sigkeys:
+            # use an unsigned copy (if allowed)
+            rpm_path = os.path.join(pathinfo.build(build_info), pathinfo.rpm(rpm_info))
+            paths.append(rpm_path)
+            if os.path.isfile(rpm_path):
+                self._invalid_sigkey_rpms.append(rpm_info)
+                return rpm_path
+
         self._invalid_sigkey_rpms.append(rpm_info)
         self.log_error("RPM %s not found for sigs: %s. Paths checked: %s"
                        % (rpm_info, self.sigkey_ordering, paths))

pungi/phases/pkgset/sources/source_koji.py

@@ -214,11 +214,23 @@ def populate_global_pkgset(compose, koji_wrapper, path_prefix, event_id):
             for group in groups:
                 packages_to_gather += comps.get_packages(group)
 
+    # In case we use "deps" gather_method, there might be some packages in
+    # the Koji tag which are not signed with proper sigkey. However, these
+    # packages might never end up in a compose depending on which packages
+    # from the Koji tag are requested how the deps are resolved in the end.
+    # In this case, we allow even packages with invalid sigkeys to be returned
+    # by PKGSET phase and later, the gather phase checks its results and if
+    # there are some packages with invalid sigkeys, it raises an exception.
+    if compose.conf["gather_method"] == "deps":
+        allow_invalid_sigkeys = True
+    else:
+        allow_invalid_sigkeys = False
+
     session = get_pdc_client_session(compose)
     for variant in compose.all_variants.values():
         variant.pkgset = pungi.phases.pkgset.pkgsets.KojiPackageSet(
             koji_wrapper, compose.conf["sigkeys"], logger=compose._logger,
-            arches=all_arches)
+            arches=all_arches, allow_invalid_sigkeys=allow_invalid_sigkeys)
         variant_tags[variant] = []
         pdc_module_file = os.path.join(compose.paths.work.topdir(arch="global"),
                                        "pdc-module-%s.json" % variant.uid)
@@ -285,7 +297,7 @@ def populate_global_pkgset(compose, koji_wrapper, path_prefix, event_id):
     else:
         global_pkgset = pungi.phases.pkgset.pkgsets.KojiPackageSet(
             koji_wrapper, compose.conf["sigkeys"], logger=compose._logger,
-            arches=all_arches)
+            arches=all_arches, allow_invalid_sigkeys=allow_invalid_sigkeys)
         # Get package set for each compose tag and merge it to global package
         # list. Also prepare per-variant pkgset, because we do not have list
         # of binary RPMs in module definition - there is just list of SRPMs.
@@ -294,7 +306,8 @@ def populate_global_pkgset(compose, koji_wrapper, path_prefix, event_id):
                              "'%s'" % compose_tag)
             pkgset = pungi.phases.pkgset.pkgsets.KojiPackageSet(
                 koji_wrapper, compose.conf["sigkeys"], logger=compose._logger,
-                arches=all_arches, packages=packages_to_gather)
+                arches=all_arches, packages=packages_to_gather,
+                allow_invalid_sigkeys=allow_invalid_sigkeys)
             # Create a filename for log with package-to-tag mapping. The tag
             # name is included in filename, so any slashes in it are replaced
             # with underscores just to be safe.

tests/test_pkgset_pkgsets.py

@@ -265,6 +265,24 @@ class TestKojiPkgset(PkgsetCompareMixin, helpers.PungiTestCase):
             re.DOTALL)
         self.assertRegexpMatches(str(ctx.exception), figure)
 
+    def test_can_not_find_signed_package_allow_invalid_sigkeys(self):
+        pkgset = pkgsets.KojiPackageSet(self.koji_wrapper, ['cafebabe'], arches=['x86_64'],
+                                        allow_invalid_sigkeys=True)
+
+        pkgset.populate('f25')
+
+        self.assertEqual(
+            self.koji_wrapper.koji_proxy.mock_calls,
+            [mock.call.listTaggedRPMS('f25', event=None, inherit=True, latest=True)])
+
+        with self.assertRaises(RuntimeError) as ctx:
+            pkgset.raise_invalid_sigkeys_exception(pkgset.invalid_sigkeys_rpms())
+
+        figure = re.compile(
+            r'^RPM\(s\) not found for sigs: .+Check log for details.+bash-4\.3\.42-4\.fc24.+bash-debuginfo-4\.3\.42-4\.fc24$',
+            re.DOTALL)
+        self.assertRegexpMatches(str(ctx.exception), figure)
+
     def test_can_not_find_any_package(self):
         pkgset = pkgsets.KojiPackageSet(self.koji_wrapper, ['cafebabe', None], arches=['x86_64'])