Allow extracting profiling information from pungi-gather.

`pungi-gather` (the tool that underlies both the `pkgset` and `gather`
phases) contains profiling code that will log statistics about how long
different function calls take.  However, pungi-koji did not contain a
way to pass the ``--profiler`` argument to enable this.

This change adds a new configuration option ``gather_profiler`` which,
when set to true, simply passes the argument to `pungi-koji`.  Hopefully
this can help shed some light on what is happening in some of our
longer-running composes.

Merges: https://pagure.io/pungi/pull-request/727
Signed-off-by: Ralph Bean <rbean@redhat.com>

doc/configuration.rst

@@ -650,6 +650,11 @@ Options
     path to JSON file with following mapping: ``{variant: {arch: {rpm_name:
     [rpm_arch|None]}}}``.
 
+**gather_profiler** = False
+    (*bool*) -- When set to ``True`` the gather tool will produce additional
+    performance profiling information at the end of its logs.  Only takes
+    effect when ``gather_backend = "dnf"``.
+
 
 Example
 -------

doc/gathering.rst

@@ -67,3 +67,10 @@ Some configuration options are overridden for particular variant types.
    +-----------+--------------+--------------+
    | optional  | enabled      | enabled      |
    +-----------+--------------+--------------+
+
+
+Profiling
+=========
+
+Profiling data on the ``pungi-gather`` tool can be enabled by setting the
+``gather_profiler`` configuration option to ``True``.

pungi/checks.py

@@ -582,6 +582,10 @@ def make_schema():
                 "enum": ["yum", "dnf"],
                 "default": "yum",
             },
+            "gather_profiler": {
+                "type": "boolean",
+                "default": False,
+            },
 
             "pkgset_source": {
                 "type": "string",

pungi/phases/gather/methods/method_deps.py

@@ -117,6 +117,9 @@ def resolve_deps(compose, arch, variant):
     fulltree = compose.conf["gather_fulltree"]
     selfhosting = compose.conf["gather_selfhosting"]
 
+    # profiling
+    profiler = compose.conf["gather_profiler"]
+
     # optional
     if variant.type == "optional":
         fulltree = True
@@ -144,7 +147,8 @@ def resolve_deps(compose, arch, variant):
     cmd = get_cmd(pungi_conf, destdir=tmp_dir, name=variant.uid,
                   selfhosting=selfhosting, fulltree=fulltree, arch=yum_arch,
                   full_archlist=True, greedy=greedy_method, cache_dir=cache_dir,
-                  lookaside_repos=lookaside_repos, multilib_methods=multilib_methods)
+                  lookaside_repos=lookaside_repos, multilib_methods=multilib_methods,
+                  profiler=profiler)
     # Use temp working directory directory as workaround for
     # https://bugzilla.redhat.com/show_bug.cgi?id=795137
     tmp_dir = compose.mkdtemp(prefix="pungi_")

pungi/phases/pkgset/sources/source_repos.py

@@ -45,6 +45,8 @@ def get_pkgset_from_repos(compose):
     # TODO: noarch hack - secondary arches, use x86_64 noarch where possible
     flist = []
 
+    profiler = compose.conf["gather_profiler"]
+
     link_type = compose.conf["link_type"]
     pool = LinkerPool(link_type, logger=compose._logger)
     for i in range(10):
@@ -79,7 +81,8 @@ def get_pkgset_from_repos(compose):
         cmd = get_cmd(pungi_conf, destdir=pungi_dir, name="FOO",
                       selfhosting=True, fulltree=True, multilib_methods=["all"],
                       nodownload=False, full_archlist=True, arch=arch,
-                      cache_dir=compose.paths.work.pungi_cache_dir(arch=arch))
+                      cache_dir=compose.paths.work.pungi_cache_dir(arch=arch),
+                      profiler=profiler)
         if compose.conf['gather_backend'] == 'yum':
             cmd.append("--force")
 

pungi/wrappers/pungi.py

@@ -109,7 +109,7 @@ class PungiWrapper(object):
 
         kickstart.close()
 
-    def get_pungi_cmd(self, config, destdir, name, version=None, flavor=None, selfhosting=False, fulltree=False, greedy=None, nodeps=False, nodownload=True, full_archlist=False, arch=None, cache_dir=None, lookaside_repos=None, multilib_methods=None):
+    def get_pungi_cmd(self, config, destdir, name, version=None, flavor=None, selfhosting=False, fulltree=False, greedy=None, nodeps=False, nodownload=True, full_archlist=False, arch=None, cache_dir=None, lookaside_repos=None, multilib_methods=None, profiler=False):
         cmd = ["pungi"]
 
         # Gather stage
@@ -169,7 +169,7 @@ class PungiWrapper(object):
 
         return cmd
 
-    def get_pungi_cmd_dnf(self, config, destdir, name, version=None, flavor=None, selfhosting=False, fulltree=False, greedy=None, nodeps=False, nodownload=True, full_archlist=False, arch=None, cache_dir=None, lookaside_repos=None, multilib_methods=None):
+    def get_pungi_cmd_dnf(self, config, destdir, name, version=None, flavor=None, selfhosting=False, fulltree=False, greedy=None, nodeps=False, nodownload=True, full_archlist=False, arch=None, cache_dir=None, lookaside_repos=None, multilib_methods=None, profiler=False):
         cmd = ["pungi-gather"]
 
         # path to a kickstart file
@@ -203,6 +203,9 @@ class PungiWrapper(object):
             for i in lookaside_repos:
                 cmd.append("--lookaside=%s" % i)
 
+        if profiler:
+            cmd.append("--profiler")
+
         return cmd
 
     def parse_log(self, f):