pkgset: Create arch repos in parallel

This patch reuses the existing createrepo_num_threads options to limit
maximum number of parallel createrepo processes.

Fixes: https://pagure.io/pungi/issue/955
JIRA: COMPOSE-2575
Signed-off-by: Lubomír Sedlář <lsedlar@redhat.com>

pungi/phases/pkgset/common.py

@@ -17,6 +17,7 @@
 import os
 
 from kobo.shortcuts import run, relative_path
+from kobo.threads import run_in_threads
 
 import pungi.phases.pkgset.pkgsets
 from pungi.arch import get_valid_arches
@@ -87,7 +88,17 @@ def run_create_global_repo(compose, cmd):
     compose.log_info("[DONE ] %s" % msg)
 
 
-def create_arch_repos(compose, arch, path_prefix):
+def create_arch_repos(compose, path_prefix):
+    run_in_threads(
+        _create_arch_repo,
+        [(compose, arch, path_prefix) for arch in compose.get_arches()],
+        threads=compose.conf['createrepo_num_threads'],
+    )
+
+
+def _create_arch_repo(worker_thread, args, task_num):
+    """Create a single pkgset repo for given arch."""
+    compose, arch, path_prefix = args
     createrepo_c = compose.conf["createrepo_c"]
     createrepo_checksum = compose.conf["createrepo_checksum"]
     repo = CreaterepoWrapper(createrepo_c=createrepo_c)

pungi/phases/pkgset/sources/source_koji.py

@@ -23,6 +23,7 @@ import threading
 
 from kobo.rpmlib import parse_nvra
 from kobo.shortcuts import force_list, relative_path
+from kobo.threads import run_in_threads
 
 import pungi.wrappers.kojiwrapper
 from pungi.wrappers.comps import CompsWrapper
@@ -203,9 +204,7 @@ def get_pkgset_from_koji(compose, koji_wrapper, path_prefix):
 
     t.join()
 
-    for arch in compose.get_arches():
-        # TODO: threads? runroot?
-        create_arch_repos(compose, arch, path_prefix)
+    create_arch_repos(compose, path_prefix)
 
     return package_sets
 

pungi/phases/pkgset/sources/source_repos.py

@@ -126,9 +126,7 @@ def get_pkgset_from_repos(compose):
 
     t.join()
 
-    for arch in compose.get_arches():
-        # TODO: threads? runroot?
-        create_arch_repos(compose, arch, path_prefix)
+    create_arch_repos(compose, path_prefix)
 
     package_sets["global"] = pkgset_global
     return package_sets, path_prefix

tests/test_pkgset_source_koji.py

@@ -258,9 +258,7 @@ class TestGetPackageSetFromKoji(helpers.PungiTestCase):
                          [mock.call(self.compose, '/prefix')])
         self.assertEqual(rcgr.call_args_list,
                          [mock.call(self.compose, gcgrc.return_value)])
-        self.assertItemsEqual(car.call_args_list,
-                              [mock.call(self.compose, 'x86_64', '/prefix'),
-                               mock.call(self.compose, 'amd64', '/prefix')])
+        self.assertItemsEqual(car.call_args_list, [mock.call(self.compose, '/prefix')])
 
         self.assertEqual(pkgsets, expected)