remove remaining hard coded createrepo threads

To be more precise, new createrepo parameter "workers" was customized and
new default value was set. This fixes issue #752.

Signed-off-by: Ondrej Nosek <onosek@redhat.com>

doc/configuration.rst

@@ -405,6 +405,9 @@ Options
     (*int*) -- how many concurrent ``createrepo`` process to run. The default
     is to use one thread per CPU available on the machine.
 
+**createrepo_num_workers**
+    (*int*) -- how many concurrent ``createrepo`` workers to run. Value defaults to 3.
+
 **product_id** = None
     (:ref:`scm_dict <scm_support>`) -- If specified, it should point to a
     directory with certificates ``<variant_uid>-<arch>-*.pem``. This

pungi/checks.py

@@ -611,6 +611,10 @@ def make_schema():
                 "type": "number",
                 "default": get_num_cpus(),
             },
+            "createrepo_num_workers": {
+                "type": "number",
+                "default": 3,
+            },
             "repoclosure_strictness": _variant_arch_mapping({
                 "type": "string",
                 "default": "lenient",

pungi/phases/createiso.py

@@ -418,7 +418,8 @@ def prepare_iso(compose, arch, variant, disc_num=1, disc_count=None, split_iso_d
             run("cp -a %s/repodata %s/" % (shlex_quote(tree_dir), shlex_quote(iso_dir)))
             with open(file_list, "w") as f:
                 f.write("\n".join(file_list_content))
-            cmd = repo.get_createrepo_cmd(tree_dir, update=True, database=True, skip_stat=True, pkglist=file_list, outputdir=iso_dir, workers=3, checksum=createrepo_checksum)
+            cmd = repo.get_createrepo_cmd(tree_dir, update=True, database=True, skip_stat=True, pkglist=file_list, outputdir=iso_dir,
+                                          workers=compose.conf["createrepo_num_workers"], checksum=createrepo_checksum)
             run(cmd)
             # add repodata/repomd.xml back to checksums
             ti.checksums.add("repodata/repomd.xml", "sha256", root_dir=iso_dir)

pungi/phases/createrepo.py

@@ -159,7 +159,8 @@ def create_variant_repo(compose, arch, variant, pkg_type):
     if compose.has_comps and pkg_type == "rpm":
         comps_path = compose.paths.work.comps(arch=arch, variant=variant)
     cmd = repo.get_createrepo_cmd(repo_dir, update=True, database=True, skip_stat=True,
-                                  pkglist=file_list, outputdir=repo_dir, workers=3,
+                                  pkglist=file_list, outputdir=repo_dir,
+                                  workers=compose.conf["createrepo_num_workers"],
                                   groupfile=comps_path, update_md_path=repo_dir_arch,
                                   checksum=createrepo_checksum,
                                   deltas=with_deltas,
@@ -202,8 +203,8 @@ def create_variant_repo(compose, arch, variant, pkg_type):
             with open(modules_path, "w") as outfile:
                 outfile.write(yaml.dump_all(modules, explicit_start=True))
             cmd = repo.get_modifyrepo_cmd(os.path.join(repo_dir, "repodata"),
-                                        modules_path, mdtype="modules",
-                                        compress_type="gz")
+                                          modules_path, mdtype="modules",
+                                          compress_type="gz")
             log_file = compose.paths.log.log_file(
                 arch, "modifyrepo-modules-%s" % variant)
             run(cmd, logfile=log_file, show_cmd=True)

pungi/phases/pkgset/common.py

@@ -67,7 +67,10 @@ def create_global_repo(compose, path_prefix):
                 update_md_path = old_repo_dir
 
     # IMPORTANT: must not use --skip-stat here -- to make sure that correctly signed files are pulled in
-    cmd = repo.get_createrepo_cmd(path_prefix, update=True, database=True, skip_stat=False, pkglist=compose.paths.work.package_list(arch="global"), outputdir=repo_dir_global, baseurl="file://%s" % path_prefix, workers=5, update_md_path=update_md_path, checksum=createrepo_checksum)
+    cmd = repo.get_createrepo_cmd(path_prefix, update=True, database=True, skip_stat=False,
+                                  pkglist=compose.paths.work.package_list(arch="global"), outputdir=repo_dir_global,
+                                  baseurl="file://%s" % path_prefix, workers=compose.conf["createrepo_num_workers"],
+                                  update_md_path=update_md_path, checksum=createrepo_checksum)
     run(cmd, logfile=compose.paths.log.log_file("global", "arch_repo"), show_cmd=True)
     compose.log_info("[DONE ] %s" % msg)
 
@@ -88,6 +91,9 @@ def create_arch_repos(compose, arch, path_prefix):
     comps_path = None
     if compose.has_comps:
         comps_path = compose.paths.work.comps(arch=arch)
-    cmd = repo.get_createrepo_cmd(path_prefix, update=True, database=True, skip_stat=True, pkglist=compose.paths.work.package_list(arch=arch), outputdir=repo_dir, baseurl="file://%s" % path_prefix, workers=5, groupfile=comps_path, update_md_path=repo_dir_global, checksum=createrepo_checksum)
+    cmd = repo.get_createrepo_cmd(path_prefix, update=True, database=True, skip_stat=True,
+                                  pkglist=compose.paths.work.package_list(arch=arch), outputdir=repo_dir,
+                                  baseurl="file://%s" % path_prefix, workers=compose.conf["createrepo_num_workers"],
+                                  groupfile=comps_path, update_md_path=repo_dir_global, checksum=createrepo_checksum)
     run(cmd, logfile=compose.paths.log.log_file(arch, "arch_repo"), show_cmd=True)
     compose.log_info("[DONE ] %s" % msg)

tests/test_createrepophase.py

@@ -251,6 +251,7 @@ class TestCreateVariantRepo(PungiTestCase):
     def test_variant_repo_is_idepotent(self, CreaterepoWrapperCls, run):
         compose = DummyCompose(self.topdir, {
             'createrepo_checksum': 'sha256',
+            'createrepo_num_workers': 10,
         })
         compose.DEBUG = False
         compose.has_comps = False
@@ -268,7 +269,7 @@ class TestCreateVariantRepo(PungiTestCase):
         self.assertItemsEqual(
             repo.get_createrepo_cmd.mock_calls,
             [mock.call(self.topdir + '/compose/Server/x86_64/os', checksum='sha256',
-                       database=True, groupfile=None, workers=3,
+                       database=True, groupfile=None, workers=10,
                        outputdir=self.topdir + '/compose/Server/x86_64/os',
                        pkglist=list_file, skip_stat=True, update=True,
                        update_md_path=self.topdir + '/work/x86_64/repo',