From 94d16d8c32d17daa8edabdc87bd001f875be24f1 Mon Sep 17 00:00:00 2001 From: Daniel Mach Date: Tue, 24 May 2016 03:50:33 -0400 Subject: [PATCH] Rewrite gathering to use new hawkey code that's part of libhif. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Lubomír Sedlář --- pungi/common.py | 34 +++++++++++ pungi/gather_dnf.py | 140 ++++++++++++++------------------------------ 2 files changed, 78 insertions(+), 96 deletions(-) create mode 100644 pungi/common.py diff --git a/pungi/common.py b/pungi/common.py new file mode 100644 index 00000000..f35eedc7 --- /dev/null +++ b/pungi/common.py @@ -0,0 +1,34 @@ +# -*- coding: utf-8 -*- + + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Library General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + +class OptionsBase(object): + + def __init__(self, **kwargs): + """ + inherit and initialize attributes + call self.merge_options(**kwargs) at the end + """ + pass + + def merge_options(self, **kwargs): + """ + override defaults with user defined values + """ + for key, value in kwargs.items(): + if not hasattr(self, key): + raise ValueError("Invalid option in %s: %s" % (self.__class__.__name__, key)) + setattr(self, key, value) diff --git a/pungi/gather_dnf.py b/pungi/gather_dnf.py index 2f37c1fa..7e0c30fb 100644 --- a/pungi/gather_dnf.py +++ b/pungi/gather_dnf.py @@ -16,16 +16,18 @@ # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - +import logging import hawkey -import logging +from kobo.rpmlib import parse_nvra + +import pungi.common import pungi.dnf_wrapper import pungi.multilib_dnf from pungi.profiler import Profiler -from kobo.rpmlib import parse_nvra -class GatherOptions(object): + +class GatherOptions(pungi.common.OptionsBase): def __init__(self, **kwargs): super(GatherOptions, self).__init__() @@ -56,75 +58,7 @@ class GatherOptions(object): # lookaside repos; packages will be flagged accordingly self.lookaside_repos = [] - for key, value in kwargs.items(): - if not hasattr(self, key): - raise ValueError("Invalid gather option: %s" % key) - setattr(self, key, value) - - -def filter_debug_packages(q, arch=None): - result = q.filter(arch__neq=["src", "nosrc"]) - if arch: - arches = pungi.dnf_wrapper.ArchWrapper(arch).all_arches - result = result.filter(arch=arches) - result = result.filter(name__glob=["*-debuginfo", "*-debuginfo-*"]) - return result - - -def filter_native_debug_packages(q, arch): - result = q.filter(arch__neq=["src", "nosrc"]) - arches = pungi.dnf_wrapper.ArchWrapper(arch).native_arches - result = result.filter(arch=arches) - result = result.filter(name__glob=["*-debuginfo", "*-debuginfo-*"]) - return result - - -def filter_multilib_debug_packages(q, arch): - result = q.filter(arch__neq=["src", "nosrc"]) - arches = pungi.dnf_wrapper.ArchWrapper(arch).multilib_arches - result = result.filter(arch=arches) - result = result.filter(name__glob=["*-debuginfo", "*-debuginfo-*"]) - return result - - -def filter_source_packages(q): - result = q.filter(arch=["src", "nosrc"]) - return result - - -def filter_binary_packages(q, arch=None): - result = q.filter(arch__neq=["src", "nosrc"]) - if arch: - arches = pungi.dnf_wrapper.ArchWrapper(arch).all_arches - result = result.filter(arch=arches) - result = result.filter(latest_per_arch=True) - result = result.filter(name__glob__not=["*-debuginfo", "*-debuginfo-*"]) - return result - - -def filter_native_binary_packages(q, arch): - result = q.filter(arch__neq=["src", "nosrc"]) - arches = pungi.dnf_wrapper.ArchWrapper(arch).native_arches - result = result.filter(arch=arches) - result = result.filter(latest_per_arch=True) - result = result.filter(name__glob__not=["*-debuginfo", "*-debuginfo-*"]) - return result - - -def filter_multilib_binary_packages(q, arch): - result = q.filter(arch__neq=["src", "nosrc"]) - arches = pungi.dnf_wrapper.ArchWrapper(arch).multilib_arches - result = result.filter(arch=arches) - result = result.filter(latest_per_arch=True) - result = result.filter(name__glob__not=["*-debuginfo", "*-debuginfo-*"]) - return result - - -def filter_binary_noarch_packages(q): - result = q.filter(arch="noarch") - result = result.filter(latest_per_arch=True) - result = result.filter(name__glob__not=["*-debuginfo", "*-debuginfo-*"]) - return result + self.merge_options(**kwargs) class QueryCache(object): @@ -152,14 +86,30 @@ class QueryCache(object): class GatherBase(object): def __init__(self, dnf_obj): self.dnf = dnf_obj - self.q_binary_packages = filter_binary_packages(self._query, arch=self.dnf.basearch).apply() - self.q_native_binary_packages = filter_native_binary_packages(self._query, arch=self.dnf.basearch).apply() - self.q_multilib_binary_packages = filter_multilib_binary_packages(self._query, arch=self.dnf.basearch).apply() - self.q_noarch_binary_packages = filter_binary_packages(self._query).apply() - self.q_debug_packages = filter_debug_packages(self._query, arch=self.dnf.basearch).apply() - self.q_native_debug_packages = filter_native_debug_packages(self._query, arch=self.dnf.basearch).apply() - self.q_multilib_debug_packages = filter_multilib_debug_packages(self._query, arch=self.dnf.basearch).apply() - self.q_source_packages = filter_source_packages(self._query).apply() + + q = self._query + q = q.filter(latest_per_arch=True).apply() + + # source packages + self.q_source_packages = q.filter(arch=self.dnf.arch_wrapper.source_arches).apply() + q = q.difference(self.q_source_packages) + + # filter arches + q = q.filter(arch=self.dnf.arch_wrapper.all_arches).apply() + q_noarch = q.filter(arch="noarch").apply() + q_native = q.filter(arch=self.dnf.arch_wrapper.native_arches).apply() + q_multilib = q.difference(q_native).union(q_noarch).apply() + + # debug packages + self.q_debug_packages = q.filter(name__glob=["*-debuginfo", "*-debuginfo-*"]).apply() + self.q_native_debug_packages = self.q_debug_packages.intersection(q_native) + self.q_multilib_debug_packages = self.q_debug_packages.intersection(q_multilib) + + # binary packages + self.q_binary_packages = q.difference(self.q_debug_packages) + self.q_native_binary_packages = q_native.difference(self.q_debug_packages) + self.q_multilib_binary_packages = q_multilib.difference(self.q_debug_packages) + self.q_noarch_binary_packages = q_noarch.difference(self.q_debug_packages) @property def _query(self): @@ -169,7 +119,7 @@ class GatherBase(object): return pkg.arch == "noarch" def is_native_package(self, pkg): - if pkg.arch in ["src", "nosrc"]: + if pkg.arch in self.dnf.arch_wrapper.source_arches: return False if pkg.arch == "noarch": return True @@ -178,7 +128,7 @@ class GatherBase(object): return False def is_multilib_package(self, pkg): - if pkg.arch in ["src", "nosrc"]: + if pkg.arch in self.dnf.arch_wrapper.source_arches: return False if pkg.arch == "noarch": return False @@ -236,7 +186,7 @@ class Gather(GatherBase): all_pkgs = list(package_list) native_pkgs = self.q_native_binary_packages.filter(pkg=all_pkgs).apply() - multilib_pkgs = [pkg for pkg in all_pkgs if pkg.arch != "noarch"] + multilib_pkgs = self.q_multilib_binary_packages.filter(pkg=all_pkgs).apply() result = set() @@ -342,19 +292,19 @@ class Gather(GatherBase): with Profiler("Gather.add_initial_packages():exclude"): # TODO: debug, source if pattern.endswith(".+"): - pkgs = self.q_multilib_binary_packages.filter_autoglob(name=pattern[:-2]) + pkgs = self.q_multilib_binary_packages.filter(name__glob=pattern[:-2]) else: - pkgs = self.q_binary_packages.filter_autoglob(name=pattern) + pkgs = self.q_binary_packages.filter(name__glob=pattern) exclude.update(pkgs) self.logger.debug("EXCLUDED: %s" % list(pkgs)) self.dnf._sack.add_excludes(pkgs) - # HACK - self.q_binary_packages = self.q_binary_packages.filter(pkg=[i for i in self.q_binary_packages if i not in exclude]).apply() - self.q_native_binary_packages = self.q_native_binary_packages.filter(pkg=[i for i in self.q_native_binary_packages if i not in exclude]).apply() - self.q_multilib_binary_packages = self.q_multilib_binary_packages.filter(pkg=[i for i in self.q_multilib_binary_packages if i not in exclude]).apply() - self.q_noarch_binary_packages = self.q_noarch_binary_packages.filter(pkg=[i for i in self.q_noarch_binary_packages if i not in exclude]).apply() + with Profiler("Gather.add_initial_packages():exclude-queries"): + self.q_binary_packages = self.q_binary_packages.filter(pkg__neq=exclude).apply() + self.q_native_binary_packages = self.q_native_binary_packages.filter(pkg__neq=exclude).apply() + self.q_multilib_binary_packages = self.q_multilib_binary_packages.filter(pkg__neq=exclude).apply() + self.q_noarch_binary_packages = self.q_noarch_binary_packages.filter(pkg__neq=exclude).apply() self.init_query_cache() @@ -364,9 +314,9 @@ class Gather(GatherBase): pkgs = self.q_binary_packages.filter(provides=hawkey.Reldep(self.dnf.sack, "system-release")).apply() else: if pattern.endswith(".+"): - pkgs = self.q_multilib_binary_packages.filter_autoglob(name=pattern[:-2]).apply() + pkgs = self.q_multilib_binary_packages.filter(name__glob=pattern[:-2]).apply() else: - pkgs = self.q_binary_packages.filter_autoglob(name=pattern).apply() + pkgs = self.q_binary_packages.filter(name__glob=pattern).apply() pkgs = self._get_best_package(pkgs) if pkgs: @@ -377,12 +327,10 @@ class Gather(GatherBase): for pkg in added: self._set_flag(pkg, "input") - native_binary_packages = set(self.q_native_binary_packages) - if self.opts.greedy_method == "build": for pkg in added.copy(): with Profiler("Gather.add_initial_packages():greedy-build"): - if pkg in native_binary_packages: + if pkg in self.q_native_binary_packages: greedy_build_packages = self.q_native_pkgs_by_sourcerpm_cache.get(pkg.sourcerpm) or [] else: greedy_build_packages = self.q_multilib_pkgs_by_sourcerpm_cache.get(pkg.sourcerpm) or []