From 2ff1f2fac342d8386350f84b435dcc96429a0002 Mon Sep 17 00:00:00 2001 From: Haibo Lin Date: Fri, 11 Oct 2019 11:17:29 +0800 Subject: [PATCH] gather: Make depsolving parallel JIRA: COMPOSE-3827 Signed-off-by: Haibo Lin --- pungi/phases/gather/__init__.py | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/pungi/phases/gather/__init__.py b/pungi/phases/gather/__init__.py index 2deda172..dfcfa966 100644 --- a/pungi/phases/gather/__init__.py +++ b/pungi/phases/gather/__init__.py @@ -17,10 +17,15 @@ import json import os import shutil +import threading from kobo.rpmlib import parse_nvra from kobo.shortcuts import run from productmd.rpms import Rpms +try: + from queue import Queue +except ImportError: + from Queue import Queue from pungi.wrappers.scm import get_file_from_scm from .link import link_files @@ -435,6 +440,8 @@ def _gather_variants(result, compose, variant_type, package_sets, exclude_fulltr variant = compose.all_variants[variant_uid] if variant.type != variant_type: continue + threads_list = [] + que = Queue() for arch in variant.arches: fulltree_excludes = set() if exclude_fulltree: @@ -446,7 +453,21 @@ def _gather_variants(result, compose, variant_type, package_sets, exclude_fulltr # there. _update_lookaside_config(compose, variant, arch, result, package_sets) - pkg_map = gather_packages(compose, arch, variant, package_sets, fulltree_excludes=fulltree_excludes) + # Run gather_packages() in parallel with multi threads and store + # its return value in a Queue() for later use. + t = threading.Thread( + target=lambda q, arch, *args, **kwargs: q.put((arch, gather_packages(*args, **kwargs))), + args=(que, arch, compose, arch, variant, package_sets), + kwargs={'fulltree_excludes': fulltree_excludes}, + ) + threads_list.append(t) + t.start() + + for t in threads_list: + t.join() + + while not que.empty(): + arch, pkg_map = que.get() result.setdefault(arch, {})[variant.uid] = pkg_map # Remove the module -> pkgset mapping to save memory