From 06635a3917a7c4be764c48fd1c321129ac99132e Mon Sep 17 00:00:00 2001 From: Daniel Mach Date: Wed, 6 Apr 2016 04:30:50 -0400 Subject: [PATCH] Add profiler to pungi-gather. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Lubomír Sedlář --- bin/pungi-gather | 12 ++++++-- pungi/gather_dnf.py | 14 +++++++++ pungi/profiler.py | 74 +++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 98 insertions(+), 2 deletions(-) create mode 100644 pungi/profiler.py diff --git a/bin/pungi-gather b/bin/pungi-gather index 5aba2307..1bf92436 100755 --- a/bin/pungi-gather +++ b/bin/pungi-gather @@ -9,10 +9,15 @@ import argparse import pungi.ks from pungi.dnf_wrapper import DnfWrapper, Conf from pungi.gather_dnf import Gather, GatherOptions +from pungi.profiler import Profiler def get_parser(): parser = argparse.ArgumentParser() + parser.add_argument( + "--profiler", + action="store_true", + ) parser.add_argument( "--arch", required=True, @@ -103,8 +108,9 @@ def main(): continue dnf_obj.add_repo(ks_repo.name, ks_repo.baseurl, ks_repo.mirrorlist) - dnf_obj.fill_sack(load_system_repo=False, load_available_repos=True) - dnf_obj.read_comps() + with Profiler("DnfWrapper.fill_sack()"): + dnf_obj.fill_sack(load_system_repo=False, load_available_repos=True) + dnf_obj.read_comps() gather_opts.langpacks = dnf_obj.comps_wrapper.get_langpacks() gather_opts.multilib_blacklist = ksparser.handler.multilib_blacklist @@ -123,6 +129,8 @@ def main(): g.gather(packages, conditional_packages) print_rpms(g) + if ns.profiler: + Profiler.print_results() def _get_flags(gather_obj, pkg): diff --git a/pungi/gather_dnf.py b/pungi/gather_dnf.py index 14298cd6..1676227c 100644 --- a/pungi/gather_dnf.py +++ b/pungi/gather_dnf.py @@ -27,6 +27,7 @@ from kobo.rpmlib import parse_nvra import pungi.dnf_wrapper import pungi.multilib_dnf +from pungi.profiler import Profiler class GatherOptions(object): @@ -130,6 +131,7 @@ def filter_binary_noarch_packages(q): result = result.filter(name__glob__not=["*-debuginfo", "*-debuginfo-*"]) return result + def cache_init(queue, *args, **kwargs): cache = {} if kwargs: @@ -139,6 +141,7 @@ def cache_init(queue, *args, **kwargs): cache.setdefault(key, set()).add(pkg) return cache + def cache_get(cache, *args): key = tuple(args) return cache.get(key, set()) @@ -313,6 +316,7 @@ class Gather(GatherBase): return result + @Profiler("Gather.add_initial_packages()") def add_initial_packages(self, pattern_list): added = set() @@ -373,6 +377,7 @@ class Gather(GatherBase): return added + @Profiler("Gather.add_prepopulate_packages()") def add_prepopulate_packages(self): added = set() @@ -390,6 +395,7 @@ class Gather(GatherBase): return added + @Profiler("Gather.add_binary_package_deps()") def add_binary_package_deps(self): added = set() @@ -411,6 +417,7 @@ class Gather(GatherBase): return added + @Profiler("Gather.add_conditional_packages()") def add_conditional_packages(self): """ For each binary package add their conditional dependencies as specified in comps. @@ -444,6 +451,7 @@ class Gather(GatherBase): return added + @Profiler("Gather.add_source_package_deps()") def add_source_package_deps(self): added = set() @@ -466,6 +474,7 @@ class Gather(GatherBase): return added + @Profiler("Gather.add_source_packages()") def add_source_packages(self): """ For each binary package add it's source package. @@ -503,6 +512,7 @@ class Gather(GatherBase): return added + @Profiler("Gather.add_debug_packages()") def add_debug_packages(self): """ For each binary package add debuginfo packages built from the same source. @@ -542,6 +552,7 @@ class Gather(GatherBase): return added + @Profiler("Gather.add_fulltree_packages()") def add_fulltree_packages(self): """ For each binary package add all binary packages built from the same source. @@ -603,6 +614,7 @@ class Gather(GatherBase): return added + @Profiler("Gather.add_langpack_packages()") def add_langpack_packages(self, langpack_patterns): """ For each binary package add all matching langpack packages. @@ -654,6 +666,7 @@ class Gather(GatherBase): return added + @Profiler("Gather.add_multilib_packages()") def add_multilib_packages(self): added = set() @@ -692,6 +705,7 @@ class Gather(GatherBase): return added + @Profiler("Gather.gather()") def gather(self, pattern_list, conditional_packages=None): self.conditional_packages = conditional_packages or [] diff --git a/pungi/profiler.py b/pungi/profiler.py new file mode 100644 index 00000000..3d9592ed --- /dev/null +++ b/pungi/profiler.py @@ -0,0 +1,74 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- + + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Library General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + +""" +Simple profiler that collects time spent in functions +or code blocks and also call counts. + + +Usage +===== + +@Profiler("label1") +def func(): + ... + +or + +with Profiler("label2"): + ... + + +To print profiling data, run: +Profiler.print_results() +""" + + +import functools +import time + + +class Profiler(object): + _data = {} + + def __init__(self, name): + self.name = name + self._data.setdefault(name, {"time": 0, "calls": 0}) + + def __enter__(self): + self.start = time.time() + + def __exit__(self, ty, val, tb): + delta = time.time() - self.start + self._data[self.name]["time"] += delta + self._data[self.name]["calls"] += 1 + + def __call__(self, func): + @functools.wraps(func) + def decorated(*args, **kwargs): + with self: + return func(*args, **kwargs) + return decorated + + @classmethod + def print_results(cls): + print "Profiling results:" + results = cls._data.items() + results.sort(lambda x, y: cmp(x[1]["time"], y[1]["time"]), reverse=True) + for name, data in results: + print " %6.2f %5d %s" % (data["time"], data["calls"], name)