From 9915c7d6441d91d30a59912f29fdbe9f764b2b56 Mon Sep 17 00:00:00 2001 From: Jan Kaluza Date: Mon, 23 Apr 2018 07:01:17 +0200 Subject: [PATCH] Allow reusing pkgset FileCache from old composes. Signed-off-by: Jan Kaluza --- pungi/paths.py | 8 +++++ pungi/phases/pkgset/pkgsets.py | 30 ++++++++++++++++++- pungi/phases/pkgset/sources/source_koji.py | 35 ++++++++++++++++++++-- tests/test_pkgset_source_koji.py | 6 ++-- 4 files changed, 74 insertions(+), 5 deletions(-) diff --git a/pungi/paths.py b/pungi/paths.py index 5234780e..49e64c3d 100644 --- a/pungi/paths.py +++ b/pungi/paths.py @@ -350,6 +350,14 @@ class WorkPaths(object): makedirs(path) return path + def pkgset_file_cache(self): + """ + Returns the path to file in which the cached version of + PackageSetBase.file_cache should be stored. + """ + return os.path.join( + self.topdir(arch="global"), "pkgset_file_cache.pickle") + class ComposePaths(object): def __init__(self, compose): diff --git a/pungi/phases/pkgset/pkgsets.py b/pungi/phases/pkgset/pkgsets.py index 2fcde67f..bc011bfb 100644 --- a/pungi/phases/pkgset/pkgsets.py +++ b/pungi/phases/pkgset/pkgsets.py @@ -21,6 +21,7 @@ It automatically finds a signed copies according to *sigkey_ordering*. import itertools import os +from six.moves import cPickle as pickle import kobo.log import kobo.pkgset @@ -50,7 +51,19 @@ class ReaderThread(WorkerThread): rpm_path = self.pool.package_set.get_package_path(item) if rpm_path is None: return - rpm_obj = self.pool.package_set.file_cache.add(rpm_path) + + # In case we have old file cache data, try to reuse it. + if self.pool.package_set.old_file_cache: + # The kobo.pkgset.FileCache does not have any method to check if + # the RPM is in cache. Instead it just re-uses the cached RPM data, + # if available, in .add() method. + # Therefore we call `old_file_cache.add()` which either returns the + # cached RPM object fast or just loads it from filesystem. We then + # add the returned RPM object to real `file_cache` directly. + rpm_obj = self.pool.package_set.old_file_cache.add(rpm_path) + self.pool.package_set.file_cache[rpm_path] = rpm_obj + else: + rpm_obj = self.pool.package_set.file_cache.add(rpm_path) self.pool.package_set.rpms_by_arch.setdefault(rpm_obj.arch, []).append(rpm_obj) if pkg_is_srpm(rpm_obj): @@ -71,6 +84,7 @@ class PackageSetBase(kobo.log.LoggingBase): allow_invalid_sigkeys=False): super(PackageSetBase, self).__init__(logger=logger) self.file_cache = kobo.pkgset.FileCache(kobo.pkgset.SimpleRpmWrapper) + self.old_file_cache = None self.sigkey_ordering = sigkey_ordering or [None] self.arches = arches self.rpms_by_arch = {} @@ -215,6 +229,20 @@ class PackageSetBase(kobo.log.LoggingBase): rpm_path = rpm_path[len(remove_path_prefix):] f.write("%s\n" % rpm_path) + def load_old_file_cache(self, file_path): + """ + Loads the cached FileCache stored in pickle format in `file_path`. + """ + with open(file_path, "rb") as f: + self.old_file_cache = pickle.load(f) + + def save_file_cache(self, file_path): + """ + Saves the current FileCache using the pickle module to `file_path`. + """ + with open(file_path, 'wb') as f: + pickle.dump(self.file_cache, f) + class FilelistPackageSet(PackageSetBase): def get_package_path(self, queue_item): diff --git a/pungi/phases/pkgset/sources/source_koji.py b/pungi/phases/pkgset/sources/source_koji.py index 9f8c801c..806e5a4a 100644 --- a/pungi/phases/pkgset/sources/source_koji.py +++ b/pungi/phases/pkgset/sources/source_koji.py @@ -19,14 +19,14 @@ from six.moves import cPickle as pickle import json import re from itertools import groupby -from kobo.shortcuts import force_list +from kobo.shortcuts import force_list, relative_path from kobo.rpmlib import make_nvra import pungi.wrappers.kojiwrapper from pungi.wrappers.comps import CompsWrapper import pungi.phases.pkgset.pkgsets from pungi.arch import get_valid_arches -from pungi.util import is_arch_multilib, retry +from pungi.util import is_arch_multilib, retry, find_old_compose from pungi import Modulemd from pungi.phases.pkgset.common import create_arch_repos, create_global_repo, populate_arch_pkgsets @@ -364,6 +364,30 @@ def _get_modules_from_koji_tags( compose.log_info("%s" % module_msg) +def _find_old_file_cache_path(compose): + """ + Finds the old compose with "pkgset_file_cache.pickled" and returns + the path to it. If no compose is found, returns None. + """ + old_compose_path = find_old_compose( + compose.old_composes, + compose.ci_base.release.short, + compose.ci_base.release.version, + compose.ci_base.release.type_suffix if compose.conf['old_composes_per_release_type'] else None, + compose.ci_base.base_product.short if compose.ci_base.release.is_layered else None, + compose.ci_base.base_product.version if compose.ci_base.release.is_layered else None, + ) + if not old_compose_path: + return None + + old_file_cache_dir = compose.paths.work.pkgset_file_cache() + rel_dir = relative_path(old_file_cache_dir, compose.topdir.rstrip('/') + '/') + old_file_cache_path = os.path.join(old_compose_path, rel_dir) + if not os.path.exists(old_file_cache_path): + return None + return old_file_cache_path + + def populate_global_pkgset(compose, koji_wrapper, path_prefix, event_id): all_arches = set(["src"]) for arch in compose.get_arches(): @@ -466,6 +490,10 @@ def populate_global_pkgset(compose, koji_wrapper, path_prefix, event_id): global_pkgset = pungi.phases.pkgset.pkgsets.KojiPackageSet( koji_wrapper, compose.conf["sigkeys"], logger=compose._logger, arches=all_arches) + old_file_cache_path = _find_old_file_cache_path(compose) + if old_file_cache_path: + compose.log_info("Reusing old PKGSET file cache from %s" % old_file_cache_path) + global_pkgset.load_old_file_cache(old_file_cache_path) # Get package set for each compose tag and merge it to global package # list. Also prepare per-variant pkgset, because we do not have list # of binary RPMs in module definition - there is just list of SRPMs. @@ -477,6 +505,8 @@ def populate_global_pkgset(compose, koji_wrapper, path_prefix, event_id): arches=all_arches, packages=packages_to_gather, allow_invalid_sigkeys=allow_invalid_sigkeys, populate_only_packages=populate_only_packages_to_gather) + if old_file_cache_path: + pkgset.load_old_file_cache(old_file_cache_path) # Create a filename for log with package-to-tag mapping. The tag # name is included in filename, so any slashes in it are replaced # with underscores just to be safe. @@ -505,6 +535,7 @@ def populate_global_pkgset(compose, koji_wrapper, path_prefix, event_id): global_pkgset.save_file_list( compose.paths.work.package_list(arch="global"), remove_path_prefix=path_prefix) + global_pkgset.save_file_cache(compose.paths.work.pkgset_file_cache()) return global_pkgset diff --git a/tests/test_pkgset_source_koji.py b/tests/test_pkgset_source_koji.py index e5d9a311..ae7d114c 100644 --- a/tests/test_pkgset_source_koji.py +++ b/tests/test_pkgset_source_koji.py @@ -116,7 +116,8 @@ class TestPopulateGlobalPkgset(helpers.PungiTestCase): [mock.call.populate('f25', 123456, inherit=True, logfile=self.topdir + '/logs/global/packages_from_f25.global.log'), mock.call.save_file_list(self.topdir + '/work/global/package_list/global.conf', - remove_path_prefix='/prefix')]) + remove_path_prefix='/prefix'), + mock.call.save_file_cache(self.topdir + '/work/global/pkgset_file_cache.pickle')]) self.assertItemsEqual(pickle_dumps.call_args_list, [mock.call(orig_pkgset)]) with open(self.pkgset_path) as f: @@ -215,7 +216,8 @@ data: self.assertEqual( pkgset.mock_calls, [mock.call.save_file_list(self.topdir + '/work/global/package_list/global.conf', - remove_path_prefix='/prefix')]) + remove_path_prefix='/prefix'), + mock.call.save_file_cache(self.topdir + '/work/global/pkgset_file_cache.pickle')]) @mock.patch('six.moves.cPickle.dumps') @mock.patch('pungi.phases.pkgset.pkgsets.KojiPackageSet.populate')