almalinux/pungi
13
1
Fork 0
Code Issues Pull Requests Releases Activity

pkgset: Only load cache once

Browse Source 
The file in old compose does not change, there is no need to load it
again for each tag.

One consequence of this change is that the same cache will be used by
all package sets. They add stuff to the cache as they use it. However
that should not be a problem for the same reason it's okay to use the
cache in the first place. If two packages have the same path, they are
actually the same file and it's okay to reuse the data.

JIRA: COMPOSE-3374
Signed-off-by: Lubomír Sedlář <lsedlar@redhat.com>
This commit is contained in:
Lubomír Sedlář 2019-03-07 14:32:12 +01:00
parent 2d39490909
commit 0fc797a06f
2 changed files with 16 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
pungi/phases/pkgset/pkgsets.py
View File

@ -236,12 +236,17 @@ class PackageSetBase(kobo.log.LoggingBase):
rpm_path = rpm_path[len(remove_path_prefix):]
f.write("%s\n" % rpm_path)
def load_old_file_cache(self, file_path):
@staticmethod
def load_old_file_cache(file_path):
"""
Loads the cached FileCache stored in pickle format in `file_path`.
"""
with open(file_path, "rb") as f:
self.old_file_cache = pickle.load(f)
return pickle.load(f)
def set_old_file_cache(self, old_file_cache):
"""Set cache of old files."""
self.old_file_cache = old_file_cache
def save_file_cache(self, file_path):
"""

12
pungi/phases/pkgset/sources/source_koji.py
View File

@ -617,10 +617,16 @@ def populate_global_pkgset(compose, koji_wrapper, path_prefix, event):
global_pkgset = pungi.phases.pkgset.pkgsets.KojiPackageSet(
koji_wrapper, compose.conf["sigkeys"], logger=compose._logger,
arches=all_arches)
old_file_cache_path = _find_old_file_cache_path(compose)
old_file_cache = None
if old_file_cache_path:
compose.log_info("Reusing old PKGSET file cache from %s" % old_file_cache_path)
global_pkgset.load_old_file_cache(old_file_cache_path)
old_file_cache = pungi.phases.pkgset.pkgsets.KojiPackageSet.load_old_file_cache(
old_file_cache_path
)
global_pkgset.set_old_file_cache(old_file_cache)
# Get package set for each compose tag and merge it to global package
# list. Also prepare per-variant pkgset, because we do not have list
# of binary RPMs in module definition - there is just list of SRPMs.
@ -638,8 +644,8 @@ def populate_global_pkgset(compose, koji_wrapper, path_prefix, event):
populate_only_packages=populate_only_packages_to_gather,
cache_region=compose.cache_region,
extra_builds=extra_builds)
if old_file_cache_path:
pkgset.load_old_file_cache(old_file_cache_path)
if old_file_cache:
pkgset.set_old_file_cache(old_file_cache)
# Create a filename for log with package-to-tag mapping. The tag
# name is included in filename, so any slashes in it are replaced
# with underscores just to be safe.