almalinux/pungi
14
1
Fork 0
Code Issues Pull Requests Releases Activity

gather: Make depsolving parallel

Browse Source 
JIRA: COMPOSE-3827
Signed-off-by: Haibo Lin <hlin@redhat.com>
This commit is contained in:
Haibo Lin 2019-10-11 11:17:29 +08:00
parent 014560f8bd
commit 2ff1f2fac3
1 changed files with 22 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

23
pungi/phases/gather/__init__.py
View File

@ -17,10 +17,15 @@
import json import json
import os import os
import shutil import shutil
import threading
from kobo.rpmlib import parse_nvra from kobo.rpmlib import parse_nvra
from kobo.shortcuts import run from kobo.shortcuts import run
from productmd.rpms import Rpms from productmd.rpms import Rpms
try:
from queue import Queue
except ImportError:
from Queue import Queue
from pungi.wrappers.scm import get_file_from_scm from pungi.wrappers.scm import get_file_from_scm
from .link import link_files from .link import link_files
@ -435,6 +440,8 @@ def _gather_variants(result, compose, variant_type, package_sets, exclude_fulltr
variant = compose.all_variants[variant_uid] variant = compose.all_variants[variant_uid]
if variant.type != variant_type: if variant.type != variant_type:
continue continue
threads_list = []
que = Queue()
for arch in variant.arches: for arch in variant.arches:
fulltree_excludes = set() fulltree_excludes = set()
if exclude_fulltree: if exclude_fulltree:
@ -446,7 +453,21 @@ def _gather_variants(result, compose, variant_type, package_sets, exclude_fulltr
# there. # there.
_update_lookaside_config(compose, variant, arch, result, package_sets) _update_lookaside_config(compose, variant, arch, result, package_sets)
pkg_map = gather_packages(compose, arch, variant, package_sets, fulltree_excludes=fulltree_excludes) # Run gather_packages() in parallel with multi threads and store
# its return value in a Queue() for later use.
t = threading.Thread(
target=lambda q, arch, *args, **kwargs: q.put((arch, gather_packages(*args, **kwargs))),
args=(que, arch, compose, arch, variant, package_sets),
kwargs={'fulltree_excludes': fulltree_excludes},
)
threads_list.append(t)
t.start()
for t in threads_list:
t.join()
while not que.empty():
arch, pkg_map = que.get()
result.setdefault(arch, {})[variant.uid] = pkg_map result.setdefault(arch, {})[variant.uid] = pkg_map
# Remove the module -> pkgset mapping to save memory # Remove the module -> pkgset mapping to save memory