almalinux/pungi
14
1
Fork 0
Code Issues Pull Requests Releases Activity

pkgset: Create arch repos in parallel

Browse Source 
This patch reuses the existing createrepo_num_threads options to limit
maximum number of parallel createrepo processes.

Fixes: https://pagure.io/pungi/issue/955
JIRA: COMPOSE-2575
Signed-off-by: Lubomír Sedlář <lsedlar@redhat.com>
This commit is contained in:
Lubomír Sedlář 2019-06-07 13:20:19 +02:00
parent f1263eeacb
commit 82580ed5b3
4 changed files with 16 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
pungi/phases/pkgset/common.py
View File

@ -17,6 +17,7 @@
import os import os
from kobo.shortcuts import run, relative_path from kobo.shortcuts import run, relative_path
from kobo.threads import run_in_threads
import pungi.phases.pkgset.pkgsets import pungi.phases.pkgset.pkgsets
from pungi.arch import get_valid_arches from pungi.arch import get_valid_arches
@ -87,7 +88,17 @@ def run_create_global_repo(compose, cmd):
compose.log_info("[DONE ] %s" % msg) compose.log_info("[DONE ] %s" % msg)
def create_arch_repos(compose, arch, path_prefix): def create_arch_repos(compose, path_prefix):
run_in_threads(
_create_arch_repo,
[(compose, arch, path_prefix) for arch in compose.get_arches()],
threads=compose.conf['createrepo_num_threads'],
)
def _create_arch_repo(worker_thread, args, task_num):
"""Create a single pkgset repo for given arch."""
compose, arch, path_prefix = args
createrepo_c = compose.conf["createrepo_c"] createrepo_c = compose.conf["createrepo_c"]
createrepo_checksum = compose.conf["createrepo_checksum"] createrepo_checksum = compose.conf["createrepo_checksum"]
repo = CreaterepoWrapper(createrepo_c=createrepo_c) repo = CreaterepoWrapper(createrepo_c=createrepo_c)

5
pungi/phases/pkgset/sources/source_koji.py
View File

@ -23,6 +23,7 @@ import threading
from kobo.rpmlib import parse_nvra from kobo.rpmlib import parse_nvra
from kobo.shortcuts import force_list, relative_path from kobo.shortcuts import force_list, relative_path
from kobo.threads import run_in_threads
import pungi.wrappers.kojiwrapper import pungi.wrappers.kojiwrapper
from pungi.wrappers.comps import CompsWrapper from pungi.wrappers.comps import CompsWrapper
@ -203,9 +204,7 @@ def get_pkgset_from_koji(compose, koji_wrapper, path_prefix):
t.join() t.join()
for arch in compose.get_arches(): create_arch_repos(compose, path_prefix)
# TODO: threads? runroot?
create_arch_repos(compose, arch, path_prefix)
return package_sets return package_sets

4
pungi/phases/pkgset/sources/source_repos.py
View File

@ -126,9 +126,7 @@ def get_pkgset_from_repos(compose):
t.join() t.join()
for arch in compose.get_arches(): create_arch_repos(compose, path_prefix)
# TODO: threads? runroot?
create_arch_repos(compose, arch, path_prefix)
package_sets["global"] = pkgset_global package_sets["global"] = pkgset_global
return package_sets, path_prefix return package_sets, path_prefix

4
tests/test_pkgset_source_koji.py
View File

@ -258,9 +258,7 @@ class TestGetPackageSetFromKoji(helpers.PungiTestCase):
[mock.call(self.compose, '/prefix')]) [mock.call(self.compose, '/prefix')])
self.assertEqual(rcgr.call_args_list, self.assertEqual(rcgr.call_args_list,
[mock.call(self.compose, gcgrc.return_value)]) [mock.call(self.compose, gcgrc.return_value)])
self.assertItemsEqual(car.call_args_list, self.assertItemsEqual(car.call_args_list, [mock.call(self.compose, '/prefix')])
[mock.call(self.compose, 'x86_64', '/prefix'),
mock.call(self.compose, 'amd64', '/prefix')])
self.assertEqual(pkgsets, expected) self.assertEqual(pkgsets, expected)