almalinux/pungi
14
1
Fork 0
Code Issues Pull Requests Releases Activity

Fail early if input packages are unsigned

Browse Source 
Use 'get_packages_to_gather' to fail early if these packages are not
signed with right key. This prevents us from having to wait for the
repo to be created and depsolving to finish. Unsigned dependencies will
still be reported later than previously.

Signed-off-by: Jan Kaluza <jkaluza@redhat.com>
This commit is contained in:
Jan Kaluza 2018-02-12 13:21:07 +01:00 committed by Lubomír Sedlář
parent c75f4a1e96
commit ef058d1f9b
4 changed files with 60 additions and 25 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
pungi/phases/gather/methods/method_deps.py
View File

@ -54,16 +54,16 @@ def raise_on_invalid_sigkeys(arch, variant, package_sets, result):
Raises RuntimeError if some package in compose is signed with an invalid
sigkey.
"""
invalid_sigkeys_rpms = []
invalid_sigkey_rpms = []
for package in result["rpm"]:
name = parse_nvra(package["path"])["name"]
for forbidden_package in package_sets["global"].invalid_sigkeys_rpms():
for forbidden_package in package_sets["global"].invalid_sigkey_rpms:
if name == forbidden_package["name"]:
invalid_sigkeys_rpms.append(forbidden_package)
invalid_sigkey_rpms.append(forbidden_package)
if invalid_sigkeys_rpms:
if invalid_sigkey_rpms:
package_sets["global"].raise_invalid_sigkeys_exception(
invalid_sigkeys_rpms)
invalid_sigkey_rpms)
def _format_packages(pkgs):

43
pungi/phases/pkgset/pkgsets.py
View File

@ -79,7 +79,8 @@ class PackageSetBase(kobo.log.LoggingBase):
self._invalid_sigkey_rpms = []
self._allow_invalid_sigkeys = allow_invalid_sigkeys
def invalid_sigkeys_rpms(self):
@property
def invalid_sigkey_rpms(self):
return self._invalid_sigkey_rpms
def __getitem__(self, name):
@ -234,13 +235,38 @@ class FilelistPackageSet(PackageSetBase):
class KojiPackageSet(PackageSetBase):
def __init__(self, koji_wrapper, sigkey_ordering, arches=None, logger=None,
packages=None, allow_invalid_sigkeys=False):
packages=None, allow_invalid_sigkeys=False,
populate_only_packages=False):
"""
Creates new KojiPackageSet.
:param list sigkey_ordering: Ordered list of sigkey strings. When
getting package from Koji, KojiPackageSet tries to get the package
signed by sigkey from this list. If None or "" appears in this
list, unsigned package is used.
:param list arches: List of arches to get the packages for.
:param logging.Logger logger: Logger instance to use for logging.
:param list packages: List of package names to be used when
`allow_invalid_sigkeys` or `populate_only_packages` is set.
:param bool allow_invalid_sigkeys: When True, packages *not* listed in
the `packages` list are added to KojiPackageSet even if they have
invalid sigkey. This is useful in case Koji tag contains some
unsigned packages, but we know they won't appear in a compose.
When False, all packages in Koji tag must have valid sigkey as
defined in `sigkey_ordering`.
:param bool populate_only_packages. When True, only packages in
`packages` list are added to KojiPackageSet. This can save time
when generating compose from predefined list of packages from big
Koji tag.
When False, all packages from Koji tag are added to KojiPackageSet.
"""
super(KojiPackageSet, self).__init__(sigkey_ordering=sigkey_ordering,
arches=arches, logger=logger,
allow_invalid_sigkeys=allow_invalid_sigkeys)
self.koji_wrapper = koji_wrapper
# Names of packages to look for in the Koji tag.
self.packages = set(packages or [])
self.populate_only_packages = populate_only_packages
def __getstate__(self):
result = self.__dict__.copy()
@ -283,7 +309,7 @@ class KojiPackageSet(PackageSetBase):
if os.path.isfile(rpm_path):
return rpm_path
if self._allow_invalid_sigkeys:
if self._allow_invalid_sigkeys and rpm_info["name"] not in self.packages:
# use an unsigned copy (if allowed)
rpm_path = os.path.join(pathinfo.build(build_info), pathinfo.rpm(rpm_info))
paths.append(rpm_path)
@ -333,7 +359,8 @@ class KojiPackageSet(PackageSetBase):
skipped_arches.append(rpm_info["arch"])
continue
if self.packages and rpm_info['name'] not in self.packages:
if (self.populate_only_packages and self.packages and
rpm_info['name'] not in self.packages):
skipped_packages_count += 1
continue
@ -352,6 +379,14 @@ class KojiPackageSet(PackageSetBase):
result = self.read_packages(result_rpms, result_srpms)
# Check that after reading the packages, every package that is
# included in a compose has the right sigkey.
if self._invalid_sigkey_rpms:
invalid_sigkey_rpms = [rpm for rpm in self._invalid_sigkey_rpms
if rpm["name"] in self.packages]
if invalid_sigkey_rpms:
self.raise_invalid_sigkeys_exception(invalid_sigkey_rpms)
# Create a log with package NEVRAs and the tag they are coming from
if logfile:
with open(logfile, 'w') as f:

18
pungi/phases/pkgset/sources/source_koji.py
View File

@ -205,14 +205,16 @@ def populate_global_pkgset(compose, koji_wrapper, path_prefix, event_id):
# are really needed to do the compose and safe lot of time and resources
# here. This only works if we are not creating bootable images. Those could
# include packages that are not in the compose.
packages_to_gather = []
if compose.conf["gather_method"] == "nodeps" and not compose.conf.get('bootable'):
packages_to_gather, groups = get_packages_to_gather(
compose, include_arch=False, include_prepopulated=True)
if groups:
comps = CompsWrapper(compose.paths.work.comps())
for group in groups:
packages_to_gather += comps.get_packages(group)
if compose.conf["gather_method"] == "nodeps" and not compose.conf.get('bootable'):
populate_only_packages_to_gather = True
else:
populate_only_packages_to_gather = False
# In case we use "deps" gather_method, there might be some packages in
# the Koji tag which are not signed with proper sigkey. However, these
@ -221,16 +223,13 @@ def populate_global_pkgset(compose, koji_wrapper, path_prefix, event_id):
# In this case, we allow even packages with invalid sigkeys to be returned
# by PKGSET phase and later, the gather phase checks its results and if
# there are some packages with invalid sigkeys, it raises an exception.
if compose.conf["gather_method"] == "deps":
allow_invalid_sigkeys = True
else:
allow_invalid_sigkeys = False
allow_invalid_sigkeys = compose.conf["gather_method"] == "deps"
session = get_pdc_client_session(compose)
for variant in compose.all_variants.values():
variant.pkgset = pungi.phases.pkgset.pkgsets.KojiPackageSet(
koji_wrapper, compose.conf["sigkeys"], logger=compose._logger,
arches=all_arches, allow_invalid_sigkeys=allow_invalid_sigkeys)
arches=all_arches)
variant_tags[variant] = []
pdc_module_file = os.path.join(compose.paths.work.topdir(arch="global"),
"pdc-module-%s.json" % variant.uid)
@ -297,7 +296,7 @@ def populate_global_pkgset(compose, koji_wrapper, path_prefix, event_id):
else:
global_pkgset = pungi.phases.pkgset.pkgsets.KojiPackageSet(
koji_wrapper, compose.conf["sigkeys"], logger=compose._logger,
arches=all_arches, allow_invalid_sigkeys=allow_invalid_sigkeys)
arches=all_arches)
# Get package set for each compose tag and merge it to global package
# list. Also prepare per-variant pkgset, because we do not have list
# of binary RPMs in module definition - there is just list of SRPMs.
@ -307,7 +306,8 @@ def populate_global_pkgset(compose, koji_wrapper, path_prefix, event_id):
pkgset = pungi.phases.pkgset.pkgsets.KojiPackageSet(
koji_wrapper, compose.conf["sigkeys"], logger=compose._logger,
arches=all_arches, packages=packages_to_gather,
allow_invalid_sigkeys=allow_invalid_sigkeys)
allow_invalid_sigkeys=allow_invalid_sigkeys,
populate_only_packages=populate_only_packages_to_gather)
# Create a filename for log with package-to-tag mapping. The tag
# name is included in filename, so any slashes in it are replaced
# with underscores just to be safe.

4
tests/test_pkgset_pkgsets.py
View File

@ -276,7 +276,7 @@ class TestKojiPkgset(PkgsetCompareMixin, helpers.PungiTestCase):
[mock.call.listTaggedRPMS('f25', event=None, inherit=True, latest=True)])
with self.assertRaises(RuntimeError) as ctx:
pkgset.raise_invalid_sigkeys_exception(pkgset.invalid_sigkeys_rpms())
pkgset.raise_invalid_sigkeys_exception(pkgset.invalid_sigkey_rpms)
figure = re.compile(
r'^RPM\(s\) not found for sigs: .+Check log for details.+bash-4\.3\.42-4\.fc24.+bash-debuginfo-4\.3\.42-4\.fc24$',
@ -309,7 +309,7 @@ class TestKojiPkgset(PkgsetCompareMixin, helpers.PungiTestCase):
])
pkgset = pkgsets.KojiPackageSet(self.koji_wrapper, [None],
packages=["bash"])
packages=["bash"], populate_only_packages=True)
result = pkgset.populate('f25', logfile=self.topdir + '/pkgset.log')