From bd9a0ceda2c3582a68423616e9a2f11bf5af180a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lubom=C3=ADr=20Sedl=C3=A1=C5=99?= Date: Wed, 2 May 2018 09:01:46 +0200 Subject: [PATCH] pkgset: Create global repo in parallel to merging pkgsets MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The work in creating the repo is done in a separate process. This can easily use two threads. Signed-off-by: Lubomír Sedlář --- pungi/phases/pkgset/common.py | 14 +++++++++----- pungi/phases/pkgset/sources/source_koji.py | 15 +++++++++++++-- pungi/phases/pkgset/sources/source_repos.py | 14 ++++++++++++-- tests/test_pkgset_source_koji.py | 9 ++++++--- 4 files changed, 40 insertions(+), 12 deletions(-) diff --git a/pungi/phases/pkgset/common.py b/pungi/phases/pkgset/common.py index 980e317c..bff1f392 100644 --- a/pungi/phases/pkgset/common.py +++ b/pungi/phases/pkgset/common.py @@ -38,19 +38,16 @@ def populate_arch_pkgsets(compose, path_prefix, global_pkgset): return result -def create_global_repo(compose, path_prefix): +def get_create_global_repo_cmd(compose, path_prefix): createrepo_c = compose.conf["createrepo_c"] createrepo_checksum = compose.conf["createrepo_checksum"] repo = CreaterepoWrapper(createrepo_c=createrepo_c) repo_dir_global = compose.paths.work.arch_repo(arch="global") - msg = "Running createrepo for the global package set" if compose.DEBUG and os.path.isdir(os.path.join(repo_dir_global, "repodata")): - compose.log_warning("[SKIP ] %s" % msg) + compose.log_warning("[SKIP ] Running createrepo for the global package set") return - compose.log_info("[BEGIN] %s" % msg) - # find an old compose suitable for repodata reuse old_compose_path = None update_md_path = None @@ -78,6 +75,13 @@ def create_global_repo(compose, path_prefix): pkglist=compose.paths.work.package_list(arch="global"), outputdir=repo_dir_global, baseurl="file://%s" % path_prefix, workers=compose.conf["createrepo_num_workers"], update_md_path=update_md_path, checksum=createrepo_checksum) + return cmd + + +def run_create_global_repo(compose, cmd): + msg = "Running createrepo for the global package set" + compose.log_info("[BEGIN] %s" % msg) + run(cmd, logfile=compose.paths.log.log_file("global", "arch_repo"), show_cmd=True) compose.log_info("[DONE ] %s" % msg) diff --git a/pungi/phases/pkgset/sources/source_koji.py b/pungi/phases/pkgset/sources/source_koji.py index a79afedc..67d4805f 100644 --- a/pungi/phases/pkgset/sources/source_koji.py +++ b/pungi/phases/pkgset/sources/source_koji.py @@ -19,6 +19,8 @@ from six.moves import cPickle as pickle import json import re from itertools import groupby +import threading + from kobo.shortcuts import force_list, relative_path from kobo.rpmlib import make_nvra @@ -29,7 +31,10 @@ from pungi.arch import get_valid_arches from pungi.util import is_arch_multilib, retry, find_old_compose from pungi import Modulemd -from pungi.phases.pkgset.common import create_arch_repos, create_global_repo, populate_arch_pkgsets +from pungi.phases.pkgset.common import (create_arch_repos, + populate_arch_pkgsets, + get_create_global_repo_cmd, + run_create_global_repo) from pungi.phases.gather import get_packages_to_gather import pungi.phases.pkgset.source @@ -179,10 +184,16 @@ class PkgsetSourceKoji(pungi.phases.pkgset.source.PkgsetSourceBase): def get_pkgset_from_koji(compose, koji_wrapper, path_prefix): event_info = get_koji_event_info(compose, koji_wrapper) pkgset_global = populate_global_pkgset(compose, koji_wrapper, path_prefix, event_info) + + cmd = get_create_global_repo_cmd(compose, path_prefix) + t = threading.Thread(target=run_create_global_repo, args=(compose, cmd)) + t.start() + package_sets = populate_arch_pkgsets(compose, path_prefix, pkgset_global) package_sets["global"] = pkgset_global - create_global_repo(compose, path_prefix) + t.join() + for arch in compose.get_arches(): # TODO: threads? runroot? create_arch_repos(compose, arch, path_prefix) diff --git a/pungi/phases/pkgset/sources/source_repos.py b/pungi/phases/pkgset/sources/source_repos.py index 6067ab0e..55aab529 100644 --- a/pungi/phases/pkgset/sources/source_repos.py +++ b/pungi/phases/pkgset/sources/source_repos.py @@ -15,6 +15,7 @@ import os +import threading from six.moves import cPickle as pickle from kobo.shortcuts import run @@ -24,7 +25,10 @@ from pungi.arch import get_valid_arches from pungi.util import makedirs, is_arch_multilib from pungi.wrappers.pungi import PungiWrapper -from pungi.phases.pkgset.common import create_global_repo, create_arch_repos, populate_arch_pkgsets +from pungi.phases.pkgset.common import (run_create_global_repo, + get_create_global_repo_cmd, + create_arch_repos, + populate_arch_pkgsets) from pungi.phases.gather import get_prepopulate_packages, get_packages_to_gather from pungi.linker import LinkerThread, LinkerPool @@ -112,10 +116,16 @@ def get_pkgset_from_repos(compose): flist = sorted(set(flist)) pkgset_global = populate_global_pkgset(compose, flist, path_prefix) + + cmd = get_create_global_repo_cmd(compose, path_prefix) + t = threading.Thread(target=run_create_global_repo, args=(compose, cmd)) + t.start() + # get_extra_packages(compose, pkgset_global) package_sets = populate_arch_pkgsets(compose, path_prefix, pkgset_global) - create_global_repo(compose, path_prefix) + t.join() + for arch in compose.get_arches(): # TODO: threads? runroot? create_arch_repos(compose, arch, path_prefix) diff --git a/tests/test_pkgset_source_koji.py b/tests/test_pkgset_source_koji.py index f75a0fbc..c71f415f 100644 --- a/tests/test_pkgset_source_koji.py +++ b/tests/test_pkgset_source_koji.py @@ -253,10 +253,11 @@ class TestGetPackageSetFromKoji(helpers.PungiTestCase): self.koji_wrapper.koji_proxy.getTag.return_value = TAG_INFO @mock.patch('pungi.phases.pkgset.sources.source_koji.create_arch_repos') - @mock.patch('pungi.phases.pkgset.sources.source_koji.create_global_repo') + @mock.patch('pungi.phases.pkgset.sources.source_koji.run_create_global_repo') + @mock.patch('pungi.phases.pkgset.sources.source_koji.get_create_global_repo_cmd') @mock.patch('pungi.phases.pkgset.sources.source_koji.populate_arch_pkgsets') @mock.patch('pungi.phases.pkgset.sources.source_koji.populate_global_pkgset') - def test_get_package_sets(self, pgp, pap, cgr, car): + def test_get_package_sets(self, pgp, pap, gcgrc, rcgr, car): expected = {'x86_64': mock.Mock()} pap.return_value = expected expected['global'] = pgp.return_value @@ -273,8 +274,10 @@ class TestGetPackageSetFromKoji(helpers.PungiTestCase): EVENT_INFO)]) self.assertEqual(pap.call_args_list, [mock.call(self.compose, '/prefix', pgp.return_value)]) - self.assertEqual(cgr.call_args_list, + self.assertEqual(gcgrc.call_args_list, [mock.call(self.compose, '/prefix')]) + self.assertEqual(rcgr.call_args_list, + [mock.call(self.compose, gcgrc.return_value)]) self.assertItemsEqual(car.call_args_list, [mock.call(self.compose, 'x86_64', '/prefix'), mock.call(self.compose, 'amd64', '/prefix')])