almalinux/pungi
13
1
Fork 0
Code Issues Pull Requests Releases Activity

speedup filter(sourcerpm=XXX, arch__neq=YYY)

Browse Source 
this is most demanding filter
it is called 10826 times and gets 210.277010s out of 15min run which is
23% spent here

Signed-off-by: Lubomír Sedlář <lsedlar@redhat.com>
This commit is contained in:
Michael Mraka 2015-09-15 14:55:52 +02:00 committed by Lubomír Sedlář
parent 36f57b26b8
commit d8fcaf95f7
1 changed files with 23 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

30
pungi/gather_dnf.py
View File

@ -130,6 +130,19 @@ def filter_binary_noarch_packages(q):
result = result.filter(name__glob__not=["*-debuginfo", "*-debuginfo-*"]) result = result.filter(name__glob__not=["*-debuginfo", "*-debuginfo-*"])
return result return result
def cache_init(queue, *args, **kwargs):
cache = {}
if kwargs:
queue = queue.filter(**kwargs)
for pkg in queue:
key = tuple(getattr(pkg, arg) for arg in args)
cache.setdefault(key, set()).add(pkg)
return cache
def cache_get(cache, *args):
key = tuple(args)
return cache.get(key, set())
class GatherBase(object): class GatherBase(object):
def __init__(self, dnf_obj): def __init__(self, dnf_obj):
@ -537,19 +550,22 @@ class Gather(GatherBase):
if not self.opts.fulltree: if not self.opts.fulltree:
return added return added
q_native_fulltree_pkgs_cache = cache_init(self.q_native_binary_packages,
'sourcerpm', arch__neq="noarch")
q_multilib_fulltree_pkgs_cache = cache_init(self.q_multilib_binary_packages,
'sourcerpm', arch__neq="noarch")
q_noarch_fulltree_pkgs_cache = cache_init(self.q_native_binary_packages,
'sourcerpm', arch="noarch")
for pkg in sorted(self.result_binary_packages): for pkg in sorted(self.result_binary_packages):
assert pkg is not None assert pkg is not None
try: try:
fulltree_pkgs = self.finished_add_fulltree_packages[pkg] fulltree_pkgs = self.finished_add_fulltree_packages[pkg]
except KeyError: except KeyError:
q_native_fulltree_pkgs = self.q_native_binary_packages.filter(sourcerpm=pkg.sourcerpm, arch__neq="noarch").apply() native_fulltree_pkgs = cache_get(q_native_fulltree_pkgs_cache, pkg.sourcerpm)
q_multilib_fulltree_pkgs = self.q_multilib_binary_packages.filter(sourcerpm=pkg.sourcerpm, arch__neq="noarch").apply() multilib_fulltree_pkgs = cache_get(q_multilib_fulltree_pkgs_cache, pkg.sourcerpm)
q_noarch_fulltree_pkgs = self.q_native_binary_packages.filter(sourcerpm=pkg.sourcerpm, arch="noarch").apply() noarch_fulltree_pkgs = cache_get(q_noarch_fulltree_pkgs_cache, pkg.sourcerpm)
native_fulltree_pkgs = set(q_native_fulltree_pkgs)
multilib_fulltree_pkgs = set(q_multilib_fulltree_pkgs)
noarch_fulltree_pkgs = set(q_noarch_fulltree_pkgs)
if not native_fulltree_pkgs: if not native_fulltree_pkgs:
# no existing native pkgs -> pull multilib # no existing native pkgs -> pull multilib