From 5b6e4689522d7ee0d6980199de7b64b814815b99 Mon Sep 17 00:00:00 2001 From: Ralph Bean Date: Fri, 1 Sep 2017 20:04:02 -0400 Subject: [PATCH] Allow extracting profiling information from pungi-gather. `pungi-gather` (the tool that underlies both the `pkgset` and `gather` phases) contains profiling code that will log statistics about how long different function calls take. However, pungi-koji did not contain a way to pass the ``--profiler`` argument to enable this. This change adds a new configuration option ``gather_profiler`` which, when set to true, simply passes the argument to `pungi-koji`. Hopefully this can help shed some light on what is happening in some of our longer-running composes. Merges: https://pagure.io/pungi/pull-request/727 Signed-off-by: Ralph Bean --- doc/configuration.rst | 5 +++++ doc/gathering.rst | 7 +++++++ pungi/checks.py | 4 ++++ pungi/phases/gather/methods/method_deps.py | 6 +++++- pungi/phases/pkgset/sources/source_repos.py | 5 ++++- pungi/wrappers/pungi.py | 7 +++++-- 6 files changed, 30 insertions(+), 4 deletions(-) diff --git a/doc/configuration.rst b/doc/configuration.rst index e9f4fef2..dcf6e6bb 100644 --- a/doc/configuration.rst +++ b/doc/configuration.rst @@ -650,6 +650,11 @@ Options path to JSON file with following mapping: ``{variant: {arch: {rpm_name: [rpm_arch|None]}}}``. +**gather_profiler** = False + (*bool*) -- When set to ``True`` the gather tool will produce additional + performance profiling information at the end of its logs. Only takes + effect when ``gather_backend = "dnf"``. + Example ------- diff --git a/doc/gathering.rst b/doc/gathering.rst index a81702fb..244fa5c0 100644 --- a/doc/gathering.rst +++ b/doc/gathering.rst @@ -67,3 +67,10 @@ Some configuration options are overridden for particular variant types. +-----------+--------------+--------------+ | optional | enabled | enabled | +-----------+--------------+--------------+ + + +Profiling +========= + +Profiling data on the ``pungi-gather`` tool can be enabled by setting the +``gather_profiler`` configuration option to ``True``. diff --git a/pungi/checks.py b/pungi/checks.py index df609b11..3796c01c 100644 --- a/pungi/checks.py +++ b/pungi/checks.py @@ -582,6 +582,10 @@ def make_schema(): "enum": ["yum", "dnf"], "default": "yum", }, + "gather_profiler": { + "type": "boolean", + "default": False, + }, "pkgset_source": { "type": "string", diff --git a/pungi/phases/gather/methods/method_deps.py b/pungi/phases/gather/methods/method_deps.py index dcffde8e..f4d0d6f8 100644 --- a/pungi/phases/gather/methods/method_deps.py +++ b/pungi/phases/gather/methods/method_deps.py @@ -117,6 +117,9 @@ def resolve_deps(compose, arch, variant): fulltree = compose.conf["gather_fulltree"] selfhosting = compose.conf["gather_selfhosting"] + # profiling + profiler = compose.conf["gather_profiler"] + # optional if variant.type == "optional": fulltree = True @@ -144,7 +147,8 @@ def resolve_deps(compose, arch, variant): cmd = get_cmd(pungi_conf, destdir=tmp_dir, name=variant.uid, selfhosting=selfhosting, fulltree=fulltree, arch=yum_arch, full_archlist=True, greedy=greedy_method, cache_dir=cache_dir, - lookaside_repos=lookaside_repos, multilib_methods=multilib_methods) + lookaside_repos=lookaside_repos, multilib_methods=multilib_methods, + profiler=profiler) # Use temp working directory directory as workaround for # https://bugzilla.redhat.com/show_bug.cgi?id=795137 tmp_dir = compose.mkdtemp(prefix="pungi_") diff --git a/pungi/phases/pkgset/sources/source_repos.py b/pungi/phases/pkgset/sources/source_repos.py index b8ca232d..2ff9d23a 100644 --- a/pungi/phases/pkgset/sources/source_repos.py +++ b/pungi/phases/pkgset/sources/source_repos.py @@ -45,6 +45,8 @@ def get_pkgset_from_repos(compose): # TODO: noarch hack - secondary arches, use x86_64 noarch where possible flist = [] + profiler = compose.conf["gather_profiler"] + link_type = compose.conf["link_type"] pool = LinkerPool(link_type, logger=compose._logger) for i in range(10): @@ -79,7 +81,8 @@ def get_pkgset_from_repos(compose): cmd = get_cmd(pungi_conf, destdir=pungi_dir, name="FOO", selfhosting=True, fulltree=True, multilib_methods=["all"], nodownload=False, full_archlist=True, arch=arch, - cache_dir=compose.paths.work.pungi_cache_dir(arch=arch)) + cache_dir=compose.paths.work.pungi_cache_dir(arch=arch), + profiler=profiler) if compose.conf['gather_backend'] == 'yum': cmd.append("--force") diff --git a/pungi/wrappers/pungi.py b/pungi/wrappers/pungi.py index b127f7ff..a2a613f3 100644 --- a/pungi/wrappers/pungi.py +++ b/pungi/wrappers/pungi.py @@ -109,7 +109,7 @@ class PungiWrapper(object): kickstart.close() - def get_pungi_cmd(self, config, destdir, name, version=None, flavor=None, selfhosting=False, fulltree=False, greedy=None, nodeps=False, nodownload=True, full_archlist=False, arch=None, cache_dir=None, lookaside_repos=None, multilib_methods=None): + def get_pungi_cmd(self, config, destdir, name, version=None, flavor=None, selfhosting=False, fulltree=False, greedy=None, nodeps=False, nodownload=True, full_archlist=False, arch=None, cache_dir=None, lookaside_repos=None, multilib_methods=None, profiler=False): cmd = ["pungi"] # Gather stage @@ -169,7 +169,7 @@ class PungiWrapper(object): return cmd - def get_pungi_cmd_dnf(self, config, destdir, name, version=None, flavor=None, selfhosting=False, fulltree=False, greedy=None, nodeps=False, nodownload=True, full_archlist=False, arch=None, cache_dir=None, lookaside_repos=None, multilib_methods=None): + def get_pungi_cmd_dnf(self, config, destdir, name, version=None, flavor=None, selfhosting=False, fulltree=False, greedy=None, nodeps=False, nodownload=True, full_archlist=False, arch=None, cache_dir=None, lookaside_repos=None, multilib_methods=None, profiler=False): cmd = ["pungi-gather"] # path to a kickstart file @@ -203,6 +203,9 @@ class PungiWrapper(object): for i in lookaside_repos: cmd.append("--lookaside=%s" % i) + if profiler: + cmd.append("--profiler") + return cmd def parse_log(self, f):