From e0308a74fff8b13aeef902581b26a1b97d872c37 Mon Sep 17 00:00:00 2001 From: Ondrej Nosek Date: Mon, 30 Oct 2017 10:07:49 +0100 Subject: [PATCH] remove remaining hard coded createrepo threads To be more precise, new createrepo parameter "workers" was customized and new default value was set. This fixes issue #752. Signed-off-by: Ondrej Nosek --- doc/configuration.rst | 3 +++ pungi/checks.py | 4 ++++ pungi/phases/createiso.py | 3 ++- pungi/phases/createrepo.py | 7 ++++--- pungi/phases/pkgset/common.py | 10 ++++++++-- tests/test_createrepophase.py | 3 ++- 6 files changed, 23 insertions(+), 7 deletions(-) diff --git a/doc/configuration.rst b/doc/configuration.rst index 8c65b906..081bd2f2 100644 --- a/doc/configuration.rst +++ b/doc/configuration.rst @@ -405,6 +405,9 @@ Options (*int*) -- how many concurrent ``createrepo`` process to run. The default is to use one thread per CPU available on the machine. +**createrepo_num_workers** + (*int*) -- how many concurrent ``createrepo`` workers to run. Value defaults to 3. + **product_id** = None (:ref:`scm_dict `) -- If specified, it should point to a directory with certificates ``--*.pem``. This diff --git a/pungi/checks.py b/pungi/checks.py index 5d7c1b90..307e87e7 100644 --- a/pungi/checks.py +++ b/pungi/checks.py @@ -611,6 +611,10 @@ def make_schema(): "type": "number", "default": get_num_cpus(), }, + "createrepo_num_workers": { + "type": "number", + "default": 3, + }, "repoclosure_strictness": _variant_arch_mapping({ "type": "string", "default": "lenient", diff --git a/pungi/phases/createiso.py b/pungi/phases/createiso.py index 2d8ea601..6121538e 100644 --- a/pungi/phases/createiso.py +++ b/pungi/phases/createiso.py @@ -418,7 +418,8 @@ def prepare_iso(compose, arch, variant, disc_num=1, disc_count=None, split_iso_d run("cp -a %s/repodata %s/" % (shlex_quote(tree_dir), shlex_quote(iso_dir))) with open(file_list, "w") as f: f.write("\n".join(file_list_content)) - cmd = repo.get_createrepo_cmd(tree_dir, update=True, database=True, skip_stat=True, pkglist=file_list, outputdir=iso_dir, workers=3, checksum=createrepo_checksum) + cmd = repo.get_createrepo_cmd(tree_dir, update=True, database=True, skip_stat=True, pkglist=file_list, outputdir=iso_dir, + workers=compose.conf["createrepo_num_workers"], checksum=createrepo_checksum) run(cmd) # add repodata/repomd.xml back to checksums ti.checksums.add("repodata/repomd.xml", "sha256", root_dir=iso_dir) diff --git a/pungi/phases/createrepo.py b/pungi/phases/createrepo.py index 1e6662d7..28608235 100644 --- a/pungi/phases/createrepo.py +++ b/pungi/phases/createrepo.py @@ -159,7 +159,8 @@ def create_variant_repo(compose, arch, variant, pkg_type): if compose.has_comps and pkg_type == "rpm": comps_path = compose.paths.work.comps(arch=arch, variant=variant) cmd = repo.get_createrepo_cmd(repo_dir, update=True, database=True, skip_stat=True, - pkglist=file_list, outputdir=repo_dir, workers=3, + pkglist=file_list, outputdir=repo_dir, + workers=compose.conf["createrepo_num_workers"], groupfile=comps_path, update_md_path=repo_dir_arch, checksum=createrepo_checksum, deltas=with_deltas, @@ -202,8 +203,8 @@ def create_variant_repo(compose, arch, variant, pkg_type): with open(modules_path, "w") as outfile: outfile.write(yaml.dump_all(modules, explicit_start=True)) cmd = repo.get_modifyrepo_cmd(os.path.join(repo_dir, "repodata"), - modules_path, mdtype="modules", - compress_type="gz") + modules_path, mdtype="modules", + compress_type="gz") log_file = compose.paths.log.log_file( arch, "modifyrepo-modules-%s" % variant) run(cmd, logfile=log_file, show_cmd=True) diff --git a/pungi/phases/pkgset/common.py b/pungi/phases/pkgset/common.py index ae1e40e1..c6c4f25d 100644 --- a/pungi/phases/pkgset/common.py +++ b/pungi/phases/pkgset/common.py @@ -67,7 +67,10 @@ def create_global_repo(compose, path_prefix): update_md_path = old_repo_dir # IMPORTANT: must not use --skip-stat here -- to make sure that correctly signed files are pulled in - cmd = repo.get_createrepo_cmd(path_prefix, update=True, database=True, skip_stat=False, pkglist=compose.paths.work.package_list(arch="global"), outputdir=repo_dir_global, baseurl="file://%s" % path_prefix, workers=5, update_md_path=update_md_path, checksum=createrepo_checksum) + cmd = repo.get_createrepo_cmd(path_prefix, update=True, database=True, skip_stat=False, + pkglist=compose.paths.work.package_list(arch="global"), outputdir=repo_dir_global, + baseurl="file://%s" % path_prefix, workers=compose.conf["createrepo_num_workers"], + update_md_path=update_md_path, checksum=createrepo_checksum) run(cmd, logfile=compose.paths.log.log_file("global", "arch_repo"), show_cmd=True) compose.log_info("[DONE ] %s" % msg) @@ -88,6 +91,9 @@ def create_arch_repos(compose, arch, path_prefix): comps_path = None if compose.has_comps: comps_path = compose.paths.work.comps(arch=arch) - cmd = repo.get_createrepo_cmd(path_prefix, update=True, database=True, skip_stat=True, pkglist=compose.paths.work.package_list(arch=arch), outputdir=repo_dir, baseurl="file://%s" % path_prefix, workers=5, groupfile=comps_path, update_md_path=repo_dir_global, checksum=createrepo_checksum) + cmd = repo.get_createrepo_cmd(path_prefix, update=True, database=True, skip_stat=True, + pkglist=compose.paths.work.package_list(arch=arch), outputdir=repo_dir, + baseurl="file://%s" % path_prefix, workers=compose.conf["createrepo_num_workers"], + groupfile=comps_path, update_md_path=repo_dir_global, checksum=createrepo_checksum) run(cmd, logfile=compose.paths.log.log_file(arch, "arch_repo"), show_cmd=True) compose.log_info("[DONE ] %s" % msg) diff --git a/tests/test_createrepophase.py b/tests/test_createrepophase.py index 53296e7f..2accb4cc 100644 --- a/tests/test_createrepophase.py +++ b/tests/test_createrepophase.py @@ -251,6 +251,7 @@ class TestCreateVariantRepo(PungiTestCase): def test_variant_repo_is_idepotent(self, CreaterepoWrapperCls, run): compose = DummyCompose(self.topdir, { 'createrepo_checksum': 'sha256', + 'createrepo_num_workers': 10, }) compose.DEBUG = False compose.has_comps = False @@ -268,7 +269,7 @@ class TestCreateVariantRepo(PungiTestCase): self.assertItemsEqual( repo.get_createrepo_cmd.mock_calls, [mock.call(self.topdir + '/compose/Server/x86_64/os', checksum='sha256', - database=True, groupfile=None, workers=3, + database=True, groupfile=None, workers=10, outputdir=self.topdir + '/compose/Server/x86_64/os', pkglist=list_file, skip_stat=True, update=True, update_md_path=self.topdir + '/work/x86_64/repo',