Allow extracting profiling information from pungi-gather.

`pungi-gather` (the tool that underlies both the `pkgset` and `gather`
phases) contains profiling code that will log statistics about how long
different function calls take.  However, pungi-koji did not contain a
way to pass the ``--profiler`` argument to enable this.

This change adds a new configuration option ``gather_profiler`` which,
when set to true, simply passes the argument to `pungi-koji`.  Hopefully
this can help shed some light on what is happening in some of our
longer-running composes.

Merges: https://pagure.io/pungi/pull-request/727
Signed-off-by: Ralph Bean <rbean@redhat.com>
This commit is contained in:
Ralph Bean 2017-09-01 20:04:02 -04:00 committed by Lubomír Sedlář
parent 40796c04f4
commit 5b6e468952
6 changed files with 30 additions and 4 deletions

View File

@ -650,6 +650,11 @@ Options
path to JSON file with following mapping: ``{variant: {arch: {rpm_name: path to JSON file with following mapping: ``{variant: {arch: {rpm_name:
[rpm_arch|None]}}}``. [rpm_arch|None]}}}``.
**gather_profiler** = False
(*bool*) -- When set to ``True`` the gather tool will produce additional
performance profiling information at the end of its logs. Only takes
effect when ``gather_backend = "dnf"``.
Example Example
------- -------

View File

@ -67,3 +67,10 @@ Some configuration options are overridden for particular variant types.
+-----------+--------------+--------------+ +-----------+--------------+--------------+
| optional | enabled | enabled | | optional | enabled | enabled |
+-----------+--------------+--------------+ +-----------+--------------+--------------+
Profiling
=========
Profiling data on the ``pungi-gather`` tool can be enabled by setting the
``gather_profiler`` configuration option to ``True``.

View File

@ -582,6 +582,10 @@ def make_schema():
"enum": ["yum", "dnf"], "enum": ["yum", "dnf"],
"default": "yum", "default": "yum",
}, },
"gather_profiler": {
"type": "boolean",
"default": False,
},
"pkgset_source": { "pkgset_source": {
"type": "string", "type": "string",

View File

@ -117,6 +117,9 @@ def resolve_deps(compose, arch, variant):
fulltree = compose.conf["gather_fulltree"] fulltree = compose.conf["gather_fulltree"]
selfhosting = compose.conf["gather_selfhosting"] selfhosting = compose.conf["gather_selfhosting"]
# profiling
profiler = compose.conf["gather_profiler"]
# optional # optional
if variant.type == "optional": if variant.type == "optional":
fulltree = True fulltree = True
@ -144,7 +147,8 @@ def resolve_deps(compose, arch, variant):
cmd = get_cmd(pungi_conf, destdir=tmp_dir, name=variant.uid, cmd = get_cmd(pungi_conf, destdir=tmp_dir, name=variant.uid,
selfhosting=selfhosting, fulltree=fulltree, arch=yum_arch, selfhosting=selfhosting, fulltree=fulltree, arch=yum_arch,
full_archlist=True, greedy=greedy_method, cache_dir=cache_dir, full_archlist=True, greedy=greedy_method, cache_dir=cache_dir,
lookaside_repos=lookaside_repos, multilib_methods=multilib_methods) lookaside_repos=lookaside_repos, multilib_methods=multilib_methods,
profiler=profiler)
# Use temp working directory directory as workaround for # Use temp working directory directory as workaround for
# https://bugzilla.redhat.com/show_bug.cgi?id=795137 # https://bugzilla.redhat.com/show_bug.cgi?id=795137
tmp_dir = compose.mkdtemp(prefix="pungi_") tmp_dir = compose.mkdtemp(prefix="pungi_")

View File

@ -45,6 +45,8 @@ def get_pkgset_from_repos(compose):
# TODO: noarch hack - secondary arches, use x86_64 noarch where possible # TODO: noarch hack - secondary arches, use x86_64 noarch where possible
flist = [] flist = []
profiler = compose.conf["gather_profiler"]
link_type = compose.conf["link_type"] link_type = compose.conf["link_type"]
pool = LinkerPool(link_type, logger=compose._logger) pool = LinkerPool(link_type, logger=compose._logger)
for i in range(10): for i in range(10):
@ -79,7 +81,8 @@ def get_pkgset_from_repos(compose):
cmd = get_cmd(pungi_conf, destdir=pungi_dir, name="FOO", cmd = get_cmd(pungi_conf, destdir=pungi_dir, name="FOO",
selfhosting=True, fulltree=True, multilib_methods=["all"], selfhosting=True, fulltree=True, multilib_methods=["all"],
nodownload=False, full_archlist=True, arch=arch, nodownload=False, full_archlist=True, arch=arch,
cache_dir=compose.paths.work.pungi_cache_dir(arch=arch)) cache_dir=compose.paths.work.pungi_cache_dir(arch=arch),
profiler=profiler)
if compose.conf['gather_backend'] == 'yum': if compose.conf['gather_backend'] == 'yum':
cmd.append("--force") cmd.append("--force")

View File

@ -109,7 +109,7 @@ class PungiWrapper(object):
kickstart.close() kickstart.close()
def get_pungi_cmd(self, config, destdir, name, version=None, flavor=None, selfhosting=False, fulltree=False, greedy=None, nodeps=False, nodownload=True, full_archlist=False, arch=None, cache_dir=None, lookaside_repos=None, multilib_methods=None): def get_pungi_cmd(self, config, destdir, name, version=None, flavor=None, selfhosting=False, fulltree=False, greedy=None, nodeps=False, nodownload=True, full_archlist=False, arch=None, cache_dir=None, lookaside_repos=None, multilib_methods=None, profiler=False):
cmd = ["pungi"] cmd = ["pungi"]
# Gather stage # Gather stage
@ -169,7 +169,7 @@ class PungiWrapper(object):
return cmd return cmd
def get_pungi_cmd_dnf(self, config, destdir, name, version=None, flavor=None, selfhosting=False, fulltree=False, greedy=None, nodeps=False, nodownload=True, full_archlist=False, arch=None, cache_dir=None, lookaside_repos=None, multilib_methods=None): def get_pungi_cmd_dnf(self, config, destdir, name, version=None, flavor=None, selfhosting=False, fulltree=False, greedy=None, nodeps=False, nodownload=True, full_archlist=False, arch=None, cache_dir=None, lookaside_repos=None, multilib_methods=None, profiler=False):
cmd = ["pungi-gather"] cmd = ["pungi-gather"]
# path to a kickstart file # path to a kickstart file
@ -203,6 +203,9 @@ class PungiWrapper(object):
for i in lookaside_repos: for i in lookaside_repos:
cmd.append("--lookaside=%s" % i) cmd.append("--lookaside=%s" % i)
if profiler:
cmd.append("--profiler")
return cmd return cmd
def parse_log(self, f): def parse_log(self, f):