almalinux/pungi
14
1
Fork 0
Code Issues Pull Requests Releases Activity

Allow composing from tag with unsigned packages

Browse Source 
There can be packages in the tag that will not end up in the compose.
Instead of failing immediately with error, this patch delays the check
until after depsolving finishes and only checks packages that will
really be included.

This is not an issue for nodeps compose, as that already pulls in only
packages that will be composed and nothing else.

Merges: https://pagure.io/pungi/pull-request/843
Signed-off-by: Jan Kaluza <jkaluza@redhat.com>
This commit is contained in:
Jan Kaluza 2018-02-12 08:02:45 +01:00 committed by Lubomír Sedlář
parent c7cc200246
commit c75f4a1e96
4 changed files with 85 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
pungi/phases/gather/methods/method_deps.py
View File

@ -18,6 +18,7 @@ import os
from kobo.shortcuts import run
from kobo.pkgset import SimpleRpmWrapper, RpmWrapper
from kobo.rpmlib import parse_nvra
from pungi.util import rmtree, get_arch_variant_data
from pungi.wrappers.pungi import PungiWrapper
@ -43,10 +44,28 @@ class GatherMethodDeps(pungi.phases.gather.method.GatherMethodBase):
fulltree_excludes=fulltree_excludes, prepopulate=prepopulate,
source_name=self.source_name)
result, missing_deps = resolve_deps(self.compose, arch, variant, source_name=self.source_name)
raise_on_invalid_sigkeys(arch, variant, package_sets, result)
check_deps(self.compose, arch, variant, missing_deps)
return result
def raise_on_invalid_sigkeys(arch, variant, package_sets, result):
"""
Raises RuntimeError if some package in compose is signed with an invalid
sigkey.
"""
invalid_sigkeys_rpms = []
for package in result["rpm"]:
name = parse_nvra(package["path"])["name"]
for forbidden_package in package_sets["global"].invalid_sigkeys_rpms():
if name == forbidden_package["name"]:
invalid_sigkeys_rpms.append(forbidden_package)
if invalid_sigkeys_rpms:
package_sets["global"].raise_invalid_sigkeys_exception(
invalid_sigkeys_rpms)
def _format_packages(pkgs):
"""Sort packages and merge name with arch."""
for pkg, pkg_arch in sorted(pkgs):

43
pungi/phases/pkgset/pkgsets.py
View File

@ -67,7 +67,8 @@ class ReaderThread(WorkerThread):
class PackageSetBase(kobo.log.LoggingBase):
def __init__(self, sigkey_ordering, arches=None, logger=None):
def __init__(self, sigkey_ordering, arches=None, logger=None,
allow_invalid_sigkeys=False):
super(PackageSetBase, self).__init__(logger=logger)
self.file_cache = kobo.pkgset.FileCache(kobo.pkgset.SimpleRpmWrapper)
self.sigkey_ordering = sigkey_ordering or [None]
@ -76,6 +77,10 @@ class PackageSetBase(kobo.log.LoggingBase):
self.srpms_by_name = {}
# RPMs not found for specified sigkeys
self._invalid_sigkey_rpms = []
self._allow_invalid_sigkeys = allow_invalid_sigkeys
def invalid_sigkeys_rpms(self):
return self._invalid_sigkey_rpms
def __getitem__(self, name):
return self.file_cache[name]
@ -96,6 +101,19 @@ class PackageSetBase(kobo.log.LoggingBase):
self._logger = None
self.__dict__.update(data)
def raise_invalid_sigkeys_exception(self, rpminfos):
"""
Raises RuntimeError containing details of RPMs with invalid
sigkeys defined in `rpminfos`.
"""
def nvr_formatter(package_info):
# joins NVR parts of the package with '-' character.
return '-'.join((package_info['name'], package_info['version'], package_info['release']))
raise RuntimeError(
"RPM(s) not found for sigs: %s. Check log for details. Unsigned packages:\n%s" % (
self.sigkey_ordering,
'\n'.join(sorted(set([nvr_formatter(rpminfo) for rpminfo in rpminfos])))))
def read_packages(self, rpms, srpms):
srpm_pool = ReaderPool(self, self._logger)
rpm_pool = ReaderPool(self, self._logger)
@ -123,14 +141,8 @@ class PackageSetBase(kobo.log.LoggingBase):
rpm_pool.stop()
self.log_debug("Package set: worker threads stopped (RPMs)")
if self._invalid_sigkey_rpms:
def nvr_formatter(package_info):
# joins NVR parts of the package with '-' character.
return '-'.join((package_info['name'], package_info['version'], package_info['release']))
raise RuntimeError(
"RPM(s) not found for sigs: %s. Check log for details. Unsigned packages:\n%s" % (
self.sigkey_ordering,
'\n'.join(sorted(set([nvr_formatter(rpminfo) for rpminfo in self._invalid_sigkey_rpms])))))
if not self._allow_invalid_sigkeys and self._invalid_sigkey_rpms:
self.raise_invalid_sigkeys_exception(self._invalid_sigkey_rpms)
return self.rpms_by_arch
@ -222,9 +234,10 @@ class FilelistPackageSet(PackageSetBase):
class KojiPackageSet(PackageSetBase):
def __init__(self, koji_wrapper, sigkey_ordering, arches=None, logger=None,
packages=None):
packages=None, allow_invalid_sigkeys=False):
super(KojiPackageSet, self).__init__(sigkey_ordering=sigkey_ordering,
arches=arches, logger=logger)
arches=arches, logger=logger,
allow_invalid_sigkeys=allow_invalid_sigkeys)
self.koji_wrapper = koji_wrapper
# Names of packages to look for in the Koji tag.
self.packages = set(packages or [])
@ -270,6 +283,14 @@ class KojiPackageSet(PackageSetBase):
if os.path.isfile(rpm_path):
return rpm_path
if self._allow_invalid_sigkeys:
# use an unsigned copy (if allowed)
rpm_path = os.path.join(pathinfo.build(build_info), pathinfo.rpm(rpm_info))
paths.append(rpm_path)
if os.path.isfile(rpm_path):
self._invalid_sigkey_rpms.append(rpm_info)
return rpm_path
self._invalid_sigkey_rpms.append(rpm_info)
self.log_error("RPM %s not found for sigs: %s. Paths checked: %s"
% (rpm_info, self.sigkey_ordering, paths))

19
pungi/phases/pkgset/sources/source_koji.py
View File

@ -214,11 +214,23 @@ def populate_global_pkgset(compose, koji_wrapper, path_prefix, event_id):
for group in groups:
packages_to_gather += comps.get_packages(group)
# In case we use "deps" gather_method, there might be some packages in
# the Koji tag which are not signed with proper sigkey. However, these
# packages might never end up in a compose depending on which packages
# from the Koji tag are requested how the deps are resolved in the end.
# In this case, we allow even packages with invalid sigkeys to be returned
# by PKGSET phase and later, the gather phase checks its results and if
# there are some packages with invalid sigkeys, it raises an exception.
if compose.conf["gather_method"] == "deps":
allow_invalid_sigkeys = True
else:
allow_invalid_sigkeys = False
session = get_pdc_client_session(compose)
for variant in compose.all_variants.values():
variant.pkgset = pungi.phases.pkgset.pkgsets.KojiPackageSet(
koji_wrapper, compose.conf["sigkeys"], logger=compose._logger,
arches=all_arches)
arches=all_arches, allow_invalid_sigkeys=allow_invalid_sigkeys)
variant_tags[variant] = []
pdc_module_file = os.path.join(compose.paths.work.topdir(arch="global"),
"pdc-module-%s.json" % variant.uid)
@ -285,7 +297,7 @@ def populate_global_pkgset(compose, koji_wrapper, path_prefix, event_id):
else:
global_pkgset = pungi.phases.pkgset.pkgsets.KojiPackageSet(
koji_wrapper, compose.conf["sigkeys"], logger=compose._logger,
arches=all_arches)
arches=all_arches, allow_invalid_sigkeys=allow_invalid_sigkeys)
# Get package set for each compose tag and merge it to global package
# list. Also prepare per-variant pkgset, because we do not have list
# of binary RPMs in module definition - there is just list of SRPMs.
@ -294,7 +306,8 @@ def populate_global_pkgset(compose, koji_wrapper, path_prefix, event_id):
"'%s'" % compose_tag)
pkgset = pungi.phases.pkgset.pkgsets.KojiPackageSet(
koji_wrapper, compose.conf["sigkeys"], logger=compose._logger,
arches=all_arches, packages=packages_to_gather)
arches=all_arches, packages=packages_to_gather,
allow_invalid_sigkeys=allow_invalid_sigkeys)
# Create a filename for log with package-to-tag mapping. The tag
# name is included in filename, so any slashes in it are replaced
# with underscores just to be safe.

18
tests/test_pkgset_pkgsets.py
View File

@ -265,6 +265,24 @@ class TestKojiPkgset(PkgsetCompareMixin, helpers.PungiTestCase):
re.DOTALL)
self.assertRegexpMatches(str(ctx.exception), figure)
def test_can_not_find_signed_package_allow_invalid_sigkeys(self):
pkgset = pkgsets.KojiPackageSet(self.koji_wrapper, ['cafebabe'], arches=['x86_64'],
allow_invalid_sigkeys=True)
pkgset.populate('f25')
self.assertEqual(
self.koji_wrapper.koji_proxy.mock_calls,
[mock.call.listTaggedRPMS('f25', event=None, inherit=True, latest=True)])
with self.assertRaises(RuntimeError) as ctx:
pkgset.raise_invalid_sigkeys_exception(pkgset.invalid_sigkeys_rpms())
figure = re.compile(
r'^RPM\(s\) not found for sigs: .+Check log for details.+bash-4\.3\.42-4\.fc24.+bash-debuginfo-4\.3\.42-4\.fc24$',
re.DOTALL)
self.assertRegexpMatches(str(ctx.exception), figure)
def test_can_not_find_any_package(self):
pkgset = pkgsets.KojiPackageSet(self.koji_wrapper, ['cafebabe', None], arches=['x86_64'])