Add profiler to pungi-gather.

Signed-off-by: Lubomír Sedlář <lsedlar@redhat.com>

bin/pungi-gather

@@ -9,10 +9,15 @@ import argparse
 import pungi.ks
 from pungi.dnf_wrapper import DnfWrapper, Conf
 from pungi.gather_dnf import Gather, GatherOptions
+from pungi.profiler import Profiler
 
 
 def get_parser():
     parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--profiler",
+        action="store_true",
+    )
     parser.add_argument(
         "--arch",
         required=True,
@@ -103,8 +108,9 @@ def main():
             continue
         dnf_obj.add_repo(ks_repo.name, ks_repo.baseurl, ks_repo.mirrorlist)
 
-    dnf_obj.fill_sack(load_system_repo=False, load_available_repos=True)
-    dnf_obj.read_comps()
+    with Profiler("DnfWrapper.fill_sack()"):
+        dnf_obj.fill_sack(load_system_repo=False, load_available_repos=True)
+        dnf_obj.read_comps()
 
     gather_opts.langpacks = dnf_obj.comps_wrapper.get_langpacks()
     gather_opts.multilib_blacklist = ksparser.handler.multilib_blacklist
@@ -123,6 +129,8 @@ def main():
     g.gather(packages, conditional_packages)
 
     print_rpms(g)
+    if ns.profiler:
+        Profiler.print_results()
 
 
 def _get_flags(gather_obj, pkg):

pungi/gather_dnf.py

@@ -27,6 +27,7 @@ from kobo.rpmlib import parse_nvra
 
 import pungi.dnf_wrapper
 import pungi.multilib_dnf
+from pungi.profiler import Profiler
 
 
 class GatherOptions(object):
@@ -130,6 +131,7 @@ def filter_binary_noarch_packages(q):
     result = result.filter(name__glob__not=["*-debuginfo", "*-debuginfo-*"])
     return result
 
+
 def cache_init(queue, *args, **kwargs):
     cache = {}
     if kwargs:
@@ -139,6 +141,7 @@ def cache_init(queue, *args, **kwargs):
         cache.setdefault(key, set()).add(pkg)
     return cache
 
+
 def cache_get(cache, *args):
     key = tuple(args)
     return cache.get(key, set())
@@ -313,6 +316,7 @@ class Gather(GatherBase):
 
         return result
 
+    @Profiler("Gather.add_initial_packages()")
     def add_initial_packages(self, pattern_list):
         added = set()
 
@@ -373,6 +377,7 @@ class Gather(GatherBase):
 
         return added
 
+    @Profiler("Gather.add_prepopulate_packages()")
     def add_prepopulate_packages(self):
         added = set()
 
@@ -390,6 +395,7 @@ class Gather(GatherBase):
 
         return added
 
+    @Profiler("Gather.add_binary_package_deps()")
     def add_binary_package_deps(self):
         added = set()
 
@@ -411,6 +417,7 @@ class Gather(GatherBase):
 
         return added
 
+    @Profiler("Gather.add_conditional_packages()")
     def add_conditional_packages(self):
         """
         For each binary package add their conditional dependencies as specified in comps.
@@ -444,6 +451,7 @@ class Gather(GatherBase):
 
         return added
 
+    @Profiler("Gather.add_source_package_deps()")
     def add_source_package_deps(self):
         added = set()
 
@@ -466,6 +474,7 @@ class Gather(GatherBase):
 
         return added
 
+    @Profiler("Gather.add_source_packages()")
     def add_source_packages(self):
         """
         For each binary package add it's source package.
@@ -503,6 +512,7 @@ class Gather(GatherBase):
 
         return added
 
+    @Profiler("Gather.add_debug_packages()")
     def add_debug_packages(self):
         """
         For each binary package add debuginfo packages built from the same source.
@@ -542,6 +552,7 @@ class Gather(GatherBase):
 
         return added
 
+    @Profiler("Gather.add_fulltree_packages()")
     def add_fulltree_packages(self):
         """
         For each binary package add all binary packages built from the same source.
@@ -603,6 +614,7 @@ class Gather(GatherBase):
 
         return added
 
+    @Profiler("Gather.add_langpack_packages()")
     def add_langpack_packages(self, langpack_patterns):
         """
         For each binary package add all matching langpack packages.
@@ -654,6 +666,7 @@ class Gather(GatherBase):
 
         return added
 
+    @Profiler("Gather.add_multilib_packages()")
     def add_multilib_packages(self):
         added = set()
 
@@ -692,6 +705,7 @@ class Gather(GatherBase):
 
         return added
 
+    @Profiler("Gather.gather()")
     def gather(self, pattern_list, conditional_packages=None):
         self.conditional_packages = conditional_packages or []
 

pungi/profiler.py

@@ -0,0 +1,74 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+
+
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Library General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+
+"""
+Simple profiler that collects time spent in functions
+or code blocks and also call counts.
+
+
+Usage
+=====
+
+@Profiler("label1")
+def func():
+    ...
+
+or
+
+with Profiler("label2"):
+    ...
+
+
+To print profiling data, run:
+Profiler.print_results()
+"""
+
+
+import functools
+import time
+
+
+class Profiler(object):
+    _data = {}
+
+    def __init__(self, name):
+        self.name = name
+        self._data.setdefault(name, {"time": 0, "calls": 0})
+
+    def __enter__(self):
+        self.start = time.time()
+
+    def __exit__(self, ty, val, tb):
+        delta = time.time() - self.start
+        self._data[self.name]["time"] += delta
+        self._data[self.name]["calls"] += 1
+
+    def __call__(self, func):
+        @functools.wraps(func)
+        def decorated(*args, **kwargs):
+            with self:
+                return func(*args, **kwargs)
+        return decorated
+
+    @classmethod
+    def print_results(cls):
+        print "Profiling results:"
+        results = cls._data.items()
+        results.sort(lambda x, y: cmp(x[1]["time"], y[1]["time"]), reverse=True)
+        for name, data in results:
+            print "  %6.2f %5d %s" % (data["time"], data["calls"], name)