almalinux/pungi
14
1
Fork 0
Code Issues Pull Requests Releases Activity

init: Create comps repos in parallel

Browse Source 
This should be mostly IO intensive operation, so running multiple
commands at the same time should save us a tiny bit of time.

Fixes: https://pagure.io/pungi/issue/981
JIRA: COMPOSE-2646
Signed-off-by: Lubomír Sedlář <lsedlar@redhat.com>
This commit is contained in:
Lubomír Sedlář 2018-08-23 15:22:40 +02:00
parent c1a03c259b
commit 9da70ee7a9
1 changed files with 28 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

35
pungi/phases/init.py
View File

@ -19,6 +19,7 @@ import os
import shutil import shutil
from kobo.shortcuts import run from kobo.shortcuts import run
from kobo.threads import run_in_threads
from pungi.phases.base import PhaseBase from pungi.phases.base import PhaseBase
from pungi.phases.gather import write_prepopulate_file from pungi.phases.gather import write_prepopulate_file
@ -42,15 +43,23 @@ class InitPhase(PhaseBase):
# write global comps and arch comps, create comps repos # write global comps and arch comps, create comps repos
global_comps = write_global_comps(self.compose) global_comps = write_global_comps(self.compose)
validate_comps(global_comps) validate_comps(global_comps)
for arch in self.compose.get_arches(): num_workers = self.compose.conf['createrepo_num_threads']
write_arch_comps(self.compose, arch) run_in_threads(
create_comps_repo(self.compose, arch, None) _arch_worker,
[(self.compose, arch) for arch in self.compose.get_arches()],
threads=num_workers,
)
# write variant comps # write variant comps
for variant in self.compose.get_variants(): run_in_threads(
for arch in variant.arches: _variant_worker,
write_variant_comps(self.compose, arch, variant) [
create_comps_repo(self.compose, arch, variant) (self.compose, arch, variant)
for variant in self.compose.get_variants()
for arch in variant.arches
],
threads=num_workers,
)
# download variants.xml / product.xml? # download variants.xml / product.xml?
@ -65,6 +74,18 @@ class InitPhase(PhaseBase):
write_prepopulate_file(self.compose) write_prepopulate_file(self.compose)
def _arch_worker(_, args, num):
compose, arch = args
write_arch_comps(compose, arch)
create_comps_repo(compose, arch, None)
def _variant_worker(_, args, num):
compose, arch, variant = args
write_variant_comps(compose, arch, variant)
create_comps_repo(compose, arch, variant)
def write_global_comps(compose): def write_global_comps(compose):
comps_file_global = compose.paths.work.comps(arch="global") comps_file_global = compose.paths.work.comps(arch="global")
msg = "Writing global comps file: %s" % comps_file_global msg = "Writing global comps file: %s" % comps_file_global