From 4562fba4594f51fad8ef39fd048c702b33f41c13 Mon Sep 17 00:00:00 2001 From: Jan Kaluza Date: Tue, 10 Nov 2020 17:10:09 +0100 Subject: [PATCH] Optimize link_files by creating temporary dict mapping path to pkg_obj. Previously, the pkg_object has been found by iterating over pkg_sets for each package. This was quite slow given the number of RPMs in the compose. In this commit, the temporary dict is created which stores mapping between path and pkg_obj and is used instead. In this commit, the get_package_path is called just once instead of twice in the loop which might also save some time. Signed-off-by: Jan Kaluza --- pungi/phases/gather/link.py | 54 +++++++++++++++---------------------- 1 file changed, 22 insertions(+), 32 deletions(-) diff --git a/pungi/phases/gather/link.py b/pungi/phases/gather/link.py index 6dcd98d1..1e7d6413 100644 --- a/pungi/phases/gather/link.py +++ b/pungi/phases/gather/link.py @@ -57,14 +57,6 @@ def get_package_path(filename, hashed_directory=False): return filename -def _find_by_path(pkg_sets, arch, path): - """Find object in an list of package sets by path.""" - for pkg_set in pkg_sets: - if path in pkg_set[arch]: - return pkg_set[arch][path] - raise RuntimeError("Path %r not found in any package set." % path) - - def link_files(compose, arch, variant, pkg_map, pkg_sets, manifest, srpm_map={}): # srpm_map instance is shared between link_files() runs @@ -76,25 +68,29 @@ def link_files(compose, arch, variant, pkg_map, pkg_sets, manifest, srpm_map={}) hashed_directories = compose.conf["hashed_directories"] + # Create temporary dict mapping package path to package object from pkgset + # so we do not have to search all pkg_sets for every package in pkg_map. + pkg_by_path = {} + for pkg_set in pkg_sets: + for path in pkg_set[arch]: + pkg_by_path[path] = pkg_set[arch][path] + packages_dir = compose.paths.compose.packages("src", variant) packages_dir_relpath = compose.paths.compose.packages("src", variant, relative=True) for pkg in pkg_map["srpm"]: if "lookaside" in pkg["flags"]: continue - dst = os.path.join( - packages_dir, - get_package_path(os.path.basename(pkg["path"]), hashed_directories), - ) - dst_relpath = os.path.join( - packages_dir_relpath, - get_package_path(os.path.basename(pkg["path"]), hashed_directories), + package_path = get_package_path( + os.path.basename(pkg["path"]), hashed_directories ) + dst = os.path.join(packages_dir, package_path) + dst_relpath = os.path.join(packages_dir_relpath, package_path) # link file pool.queue_put((pkg["path"], dst)) # update rpm manifest - pkg_obj = _find_by_path(pkg_sets, arch, pkg["path"]) + pkg_obj = pkg_by_path[pkg["path"]] nevra = pkg_obj.nevra manifest.add( variant.uid, @@ -113,20 +109,17 @@ def link_files(compose, arch, variant, pkg_map, pkg_sets, manifest, srpm_map={}) for pkg in pkg_map["rpm"]: if "lookaside" in pkg["flags"]: continue - dst = os.path.join( - packages_dir, - get_package_path(os.path.basename(pkg["path"]), hashed_directories), - ) - dst_relpath = os.path.join( - packages_dir_relpath, - get_package_path(os.path.basename(pkg["path"]), hashed_directories), + package_path = get_package_path( + os.path.basename(pkg["path"]), hashed_directories ) + dst = os.path.join(packages_dir, package_path) + dst_relpath = os.path.join(packages_dir_relpath, package_path) # link file pool.queue_put((pkg["path"], dst)) # update rpm manifest - pkg_obj = _find_by_path(pkg_sets, arch, pkg["path"]) + pkg_obj = pkg_by_path[pkg["path"]] nevra = pkg_obj.nevra src_nevra = _get_src_nevra(compose, pkg_obj, srpm_map) manifest.add( @@ -146,20 +139,17 @@ def link_files(compose, arch, variant, pkg_map, pkg_sets, manifest, srpm_map={}) for pkg in pkg_map["debuginfo"]: if "lookaside" in pkg["flags"]: continue - dst = os.path.join( - packages_dir, - get_package_path(os.path.basename(pkg["path"]), hashed_directories), - ) - dst_relpath = os.path.join( - packages_dir_relpath, - get_package_path(os.path.basename(pkg["path"]), hashed_directories), + package_path = get_package_path( + os.path.basename(pkg["path"]), hashed_directories ) + dst = os.path.join(packages_dir, package_path) + dst_relpath = os.path.join(packages_dir_relpath, package_path) # link file pool.queue_put((pkg["path"], dst)) # update rpm manifest - pkg_obj = _find_by_path(pkg_sets, arch, pkg["path"]) + pkg_obj = pkg_by_path[pkg["path"]] nevra = pkg_obj.nevra src_nevra = _get_src_nevra(compose, pkg_obj, srpm_map) manifest.add(