1083 lines
39 KiB
Python
1083 lines
39 KiB
Python
# -*- coding: utf-8 -*-
|
|
|
|
|
|
# This program is free software; you can redistribute it and/or modify
|
|
# it under the terms of the GNU General Public License as published by
|
|
# the Free Software Foundation; version 2 of the License.
|
|
#
|
|
# This program is distributed in the hope that it will be useful,
|
|
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
# GNU Library General Public License for more details.
|
|
#
|
|
# You should have received a copy of the GNU General Public License
|
|
# along with this program; if not, see <https://gnu.org/licenses/>.
|
|
|
|
|
|
from enum import Enum
|
|
from functools import cmp_to_key
|
|
from itertools import count, groupby
|
|
import logging
|
|
import os
|
|
import re
|
|
|
|
from kobo.rpmlib import parse_nvra
|
|
import rpm
|
|
|
|
import pungi.common
|
|
import pungi.dnf_wrapper
|
|
import pungi.multilib_dnf
|
|
import pungi.util
|
|
from pungi import arch_utils
|
|
from pungi.linker import Linker
|
|
from pungi.profiler import Profiler
|
|
from pungi.util import DEBUG_PATTERNS
|
|
|
|
|
|
def get_source_name(pkg):
|
|
# Workaround for rhbz#1418298
|
|
return pkg.sourcerpm.rsplit("-", 2)[0]
|
|
|
|
|
|
class GatherOptions(pungi.common.OptionsBase):
|
|
def __init__(self, **kwargs):
|
|
super(GatherOptions, self).__init__()
|
|
|
|
# include all unused sub-packages of already included RPMs
|
|
self.fulltree = False
|
|
|
|
# A set of packages for which fulltree does not apply.
|
|
self.fulltree_excludes = set()
|
|
|
|
# include langpacks
|
|
self.langpacks = [] # format: [{"package": "langpack-pattern-%s"}]
|
|
|
|
# resolve dependencies
|
|
self.resolve_deps = True
|
|
|
|
# pull build dependencies
|
|
self.selfhosting = False
|
|
|
|
# none, all, build
|
|
# TODO: validate values
|
|
self.greedy_method = "none"
|
|
|
|
# multilib options
|
|
self.multilib_methods = []
|
|
self.multilib_blacklist = []
|
|
self.multilib_whitelist = []
|
|
|
|
# prepopulate
|
|
self.prepopulate = []
|
|
|
|
# lookaside repos; packages will be flagged accordingly
|
|
self.lookaside_repos = []
|
|
|
|
# exclude source packages
|
|
self.exclude_source = False
|
|
|
|
# exclude debug packages
|
|
self.exclude_debug = False
|
|
|
|
self.merge_options(**kwargs)
|
|
|
|
def __str__(self):
|
|
lines = [
|
|
"fulltree=%s" % self.fulltree,
|
|
"fulltree_excludes=%d items" % len(self.fulltree_excludes),
|
|
"resolve_deps=%s" % self.resolve_deps,
|
|
"selfhosting=%s" % self.selfhosting,
|
|
"greedy_method=%s" % self.greedy_method,
|
|
"langpacks=%s" % self.langpacks,
|
|
"multilib_methods=%s" % self.multilib_methods,
|
|
"multilib_blacklist=%d items" % len(self.multilib_blacklist),
|
|
"multilib_whitelist=%d items" % len(self.multilib_whitelist),
|
|
"lookaside_repos=%s" % self.lookaside_repos,
|
|
"prepopulate=%d items" % len(self.prepopulate),
|
|
"exclude_source=%s" % self.exclude_source,
|
|
"exclude_debug=%s" % self.exclude_debug,
|
|
]
|
|
return "[\n%s\n]" % "\n".join(" " + line for line in lines)
|
|
|
|
|
|
class QueryCache(object):
|
|
def __init__(self, queue, *args, **kwargs):
|
|
self.cache = {}
|
|
self.nargs = len(args)
|
|
|
|
if kwargs:
|
|
queue = queue.filter(**kwargs)
|
|
|
|
for pkg in queue:
|
|
key = tuple(getattr(pkg, arg) for arg in args)
|
|
pkgs = self.cache.setdefault(key, [])
|
|
if pkg not in pkgs:
|
|
# use list preserve package order
|
|
pkgs.append(pkg)
|
|
|
|
def get(self, *args):
|
|
if len(args) != self.nargs:
|
|
raise ValueError("Expected %s arguments, got %s" % (self.nargs, len(args)))
|
|
key = tuple(args)
|
|
return self.cache.get(key, None)
|
|
|
|
|
|
class PkgFlag(Enum):
|
|
lookaside = 1
|
|
input = 2
|
|
greedy_build = 4
|
|
prepopulate = 8
|
|
conditional = 16
|
|
self_hosting = 32
|
|
fulltree = 64
|
|
multilib = 128
|
|
langpack = 256
|
|
fulltree_exclude = 512
|
|
|
|
|
|
class GatherBase(object):
|
|
def __init__(self, dnf_obj):
|
|
self.dnf = dnf_obj
|
|
|
|
q = self._query
|
|
# We can not filter only latest packages yet, because we need to apply
|
|
# excludes only to main repos and not lookaside. Filtering latest here
|
|
# makes that impossible as it could remove older versions from
|
|
# lookaside.
|
|
|
|
# source packages
|
|
self.q_source_packages = q.filter(
|
|
arch=self.dnf.arch_wrapper.source_arches
|
|
).apply()
|
|
q = q.difference(self.q_source_packages)
|
|
|
|
# filter arches
|
|
q = q.filter(arch=self.dnf.arch_wrapper.all_arches).apply()
|
|
q_noarch = q.filter(arch="noarch").apply()
|
|
q_native = q.filter(arch=self.dnf.arch_wrapper.native_arches).apply()
|
|
q_multilib = q.difference(q_native).union(q_noarch).apply()
|
|
|
|
# debug packages
|
|
self.q_debug_packages = q.filter(name__glob=DEBUG_PATTERNS).apply()
|
|
self.q_native_debug_packages = self.q_debug_packages.intersection(q_native)
|
|
self.q_multilib_debug_packages = self.q_debug_packages.intersection(q_multilib)
|
|
|
|
# binary packages
|
|
self.q_binary_packages = q.difference(self.q_debug_packages)
|
|
self.q_native_binary_packages = q_native.difference(self.q_debug_packages)
|
|
self.q_multilib_binary_packages = q_multilib.difference(self.q_debug_packages)
|
|
self.q_noarch_binary_packages = q_noarch.difference(self.q_debug_packages)
|
|
|
|
@property
|
|
def _query(self):
|
|
return self.dnf._sack.query()
|
|
|
|
def is_noarch_package(self, pkg):
|
|
return pkg.arch == "noarch"
|
|
|
|
def is_native_package(self, pkg):
|
|
if pkg.arch in self.dnf.arch_wrapper.source_arches:
|
|
return False
|
|
if pkg.arch == "noarch":
|
|
return True
|
|
if pkg.arch in self.dnf.arch_wrapper.native_arches:
|
|
return True
|
|
return False
|
|
|
|
|
|
class Gather(GatherBase):
|
|
def __init__(self, dnf_obj, gather_options, logger=None):
|
|
super(Gather, self).__init__(dnf_obj)
|
|
self.logger = logger
|
|
if not self.logger:
|
|
# default logger
|
|
self.logger = logging.getLogger("gather_dnf")
|
|
self.logger.setLevel(logging.DEBUG)
|
|
|
|
if not self.logger.handlers:
|
|
# default logging handler
|
|
handler = logging.StreamHandler()
|
|
handler.setFormatter(
|
|
logging.Formatter(
|
|
"%(asctime)s [%(levelname)-8s] %(message)s",
|
|
datefmt="%Y-%m-%d %H:%M:%S",
|
|
)
|
|
)
|
|
handler.setLevel(logging.DEBUG)
|
|
self.logger.addHandler(handler)
|
|
|
|
self.opts = gather_options
|
|
self.logger.debug("Gather received gather_options=%s" % gather_options)
|
|
self._multilib = pungi.multilib_dnf.Multilib.from_globs(
|
|
self.dnf._sack,
|
|
gather_options.multilib_methods,
|
|
blacklist=self.opts.multilib_blacklist,
|
|
whitelist=self.opts.multilib_whitelist,
|
|
)
|
|
|
|
# already processed packages
|
|
self.finished_add_binary_package_deps = {} # {pkg: [deps]}
|
|
self.finished_add_debug_package_deps = {} # {pkg: [deps]}
|
|
self.finished_add_source_package_deps = {} # {pkg: [deps]}
|
|
|
|
self.finished_get_package_deps_reqs = {}
|
|
|
|
self.finished_add_conditional_packages = {} # {pkg: [pkgs]}
|
|
self.finished_add_source_packages = {} # {pkg: src-pkg|None}
|
|
self.sourcerpm_cache = {} # {src_nvra: src-pkg|None}
|
|
self.finished_add_debug_packages = {} # {pkg: [debug-pkgs]}
|
|
self.finished_add_fulltree_packages = {} # {pkg: [pkgs]}
|
|
self.finished_add_langpack_packages = {} # {pkg: [pkgs]}
|
|
self.finished_add_multilib_packages = {} # {pkg: pkg|None}
|
|
|
|
# result
|
|
self.result_binary_packages = set()
|
|
self.result_debug_packages = set()
|
|
self.result_source_packages = set()
|
|
self.result_package_flags = {}
|
|
|
|
def _set_flag(self, pkg, *flags):
|
|
self.result_package_flags.setdefault(pkg, set()).update(flags)
|
|
|
|
def _get_best_package(self, package_list, pkg=None, req=None, debuginfo=False):
|
|
if not package_list:
|
|
return []
|
|
|
|
# The list can contain packages from lookaside and outside of
|
|
# lookaside. If the package is in both, we want to prefer the version
|
|
# from lookaside. This can be achieved by removing any package that is
|
|
# also in lookaside from the list.
|
|
lookaside_pkgs = set()
|
|
|
|
if self.opts.lookaside_repos:
|
|
# We will call `latest()` to get the highest version packages only.
|
|
# However, that is per name and architecture. If a package switches
|
|
# from arched to noarch or the other way, it is possible that the
|
|
# package_list contains different versions in main repos and in
|
|
# lookaside repos.
|
|
# We need to manually filter the latest version.
|
|
def vercmp(x, y):
|
|
return rpm.labelCompare(x[1], y[1])
|
|
|
|
# Annotate the packages with their version.
|
|
versioned_packages = [
|
|
(pkg, (str(pkg.epoch) or "0", pkg.version, pkg.release))
|
|
for pkg in package_list
|
|
]
|
|
# Sort the packages newest first.
|
|
sorted_packages = sorted(
|
|
versioned_packages, key=cmp_to_key(vercmp), reverse=True
|
|
)
|
|
# Group packages by version, take the first group and discard the
|
|
# version info from the tuple.
|
|
package_list = list(
|
|
x[0] for x in next(groupby(sorted_packages, key=lambda x: x[1]))[1]
|
|
)
|
|
|
|
# Now we can decide what is used from lookaside.
|
|
for pkg in package_list:
|
|
if pkg.repoid in self.opts.lookaside_repos:
|
|
lookaside_pkgs.add("{0.name}-{0.evr}".format(pkg))
|
|
|
|
all_pkgs = []
|
|
for pkg in package_list:
|
|
# Remove packages that are also in lookaside
|
|
if (
|
|
"{0.name}-{0.evr}".format(pkg) not in lookaside_pkgs
|
|
or pkg.repoid in self.opts.lookaside_repos
|
|
):
|
|
all_pkgs.append(pkg)
|
|
|
|
if not debuginfo:
|
|
native_pkgs = set(
|
|
self.q_native_binary_packages.filter(pkg=all_pkgs).latest().apply()
|
|
)
|
|
multilib_pkgs = set(
|
|
self.q_multilib_binary_packages.filter(pkg=all_pkgs).latest().apply()
|
|
)
|
|
else:
|
|
native_pkgs = set(
|
|
self.q_native_debug_packages.filter(pkg=all_pkgs).latest().apply()
|
|
)
|
|
multilib_pkgs = set(
|
|
self.q_multilib_debug_packages.filter(pkg=all_pkgs).latest().apply()
|
|
)
|
|
|
|
if self.opts.greedy_method == "all":
|
|
return list(native_pkgs | multilib_pkgs)
|
|
|
|
result = set()
|
|
|
|
# try seen native packages first
|
|
seen_pkgs = native_pkgs & self.result_binary_packages
|
|
if seen_pkgs:
|
|
result = seen_pkgs
|
|
|
|
# then try seen multilib packages
|
|
if not result:
|
|
seen_pkgs = multilib_pkgs & self.result_binary_packages
|
|
if seen_pkgs:
|
|
result = seen_pkgs
|
|
|
|
if not result:
|
|
result = native_pkgs
|
|
|
|
if not result:
|
|
result = multilib_pkgs
|
|
|
|
if not result:
|
|
return []
|
|
|
|
# return package with shortest name, alphabetically ordered
|
|
result = list(result)
|
|
result.sort(key=lambda x: x.name)
|
|
result.sort(key=lambda x: len(x.name))
|
|
|
|
# best arch
|
|
arches = self.dnf.arch_wrapper.all_arches
|
|
result.sort(key=lambda x: arches.index(x.arch))
|
|
match = result[0]
|
|
|
|
if self.opts.greedy_method == "build" and req:
|
|
if self.is_native_package(match):
|
|
return [i for i in native_pkgs if i.sourcerpm == match.sourcerpm]
|
|
else:
|
|
return [i for i in multilib_pkgs if i.sourcerpm == match.sourcerpm]
|
|
return [match]
|
|
|
|
def is_from_lookaside(self, pkg):
|
|
if not self.opts.lookaside_repos:
|
|
return False
|
|
pkgs = self.dnf.sack.query().filter(
|
|
name=pkg.name,
|
|
version=pkg.version,
|
|
release=pkg.release,
|
|
arch=pkg.arch,
|
|
reponame=self.opts.lookaside_repos,
|
|
)
|
|
return pkg in pkgs
|
|
|
|
def _add_packages(self, packages, pulled_by=None, req=None, reason=None, dest=None):
|
|
dest = dest if dest is not None else self.result_binary_packages
|
|
for i in packages:
|
|
assert i is not None
|
|
if i not in dest:
|
|
pb = ""
|
|
if pulled_by:
|
|
pb = " (pulled by %s, repo: %s)" % (pulled_by, pulled_by.repo.id)
|
|
if req:
|
|
pb += " (Requires: %s)" % req
|
|
if reason:
|
|
pb += " (%s)" % reason
|
|
self.logger.debug("Added package %s%s" % (i, pb))
|
|
dest.add(i)
|
|
# lookaside
|
|
if self.is_from_lookaside(i):
|
|
self._set_flag(i, PkgFlag.lookaside)
|
|
if i.sourcerpm.rsplit("-", 2)[0] in self.opts.fulltree_excludes:
|
|
self._set_flag(i, PkgFlag.fulltree_exclude)
|
|
|
|
def _get_package_deps(self, pkg, debuginfo=False):
|
|
"""Return all direct (1st level) deps for a package.
|
|
|
|
The return value is a set of tuples (pkg, reldep). Each package is
|
|
tagged with the particular reldep that pulled it in. Requires_pre and
|
|
_post are not distinguished.
|
|
"""
|
|
queue = self.q_debug_packages if debuginfo else self.q_binary_packages
|
|
assert pkg is not None
|
|
result = set()
|
|
|
|
if pkg.repoid in self.opts.lookaside_repos:
|
|
# Don't resolve deps for stuff in lookaside.
|
|
return result
|
|
|
|
# DNF package has the _pre and _post attributes only if they are not
|
|
# empty.
|
|
requires = (
|
|
pkg.requires
|
|
+ getattr(pkg, "requires_pre", [])
|
|
+ getattr(pkg, "requires_post", [])
|
|
)
|
|
|
|
q = queue.filter(provides=requires).apply()
|
|
for req in requires:
|
|
deps = self.finished_get_package_deps_reqs.setdefault(str(req), set())
|
|
if deps:
|
|
result.update((dep, req) for dep in deps)
|
|
continue
|
|
|
|
# TODO: need query also debuginfo
|
|
deps = q.filter(provides=req)
|
|
if deps:
|
|
deps = self._get_best_package(deps, req=req, debuginfo=debuginfo)
|
|
self.finished_get_package_deps_reqs[str(req)].update(deps)
|
|
result.update((dep, req) for dep in deps)
|
|
else:
|
|
self.logger.warning(
|
|
"Unresolvable dependency %s in %s.%s", req, pkg.name, pkg.arch
|
|
)
|
|
|
|
return result
|
|
|
|
def _filter_queue(self, queue, exclude):
|
|
"""Given an name of a queue (stored as attribute in `self`), exclude
|
|
all given packages and keep only the latest per package name and arch.
|
|
"""
|
|
setattr(self, queue, getattr(self, queue).filter(pkg__neq=exclude).apply())
|
|
|
|
@Profiler("Gather._apply_excludes()")
|
|
def _apply_excludes(self, excludes):
|
|
"""Exclude packages from all queues. An excluded package will no longer
|
|
be visible in the depsolving process (as if it was not in the repo in
|
|
the first place).
|
|
|
|
All packages matching patterns in `excludes` argument are removed, plus
|
|
anything matched by multilib blacklist. Finally only latest versions of
|
|
each package (per arch) is preserved in the queue.
|
|
"""
|
|
exclude = set()
|
|
for pattern in excludes:
|
|
with Profiler("Gather._apply_excludes():exclude"):
|
|
if pattern.endswith(".+"):
|
|
pkgs = self.q_multilib_binary_packages.filter(
|
|
name__glob=pattern[:-2],
|
|
arch__neq="noarch",
|
|
reponame__neq=self.opts.lookaside_repos,
|
|
)
|
|
elif pattern.endswith(".src"):
|
|
pkgs = self.q_source_packages.filter(
|
|
name__glob=pattern[:-4], reponame__neq=self.opts.lookaside_repos
|
|
)
|
|
elif pungi.util.pkg_is_debug(pattern):
|
|
pkgs = self.q_debug_packages.filter(
|
|
name__glob=pattern, reponame__neq=self.opts.lookaside_repos
|
|
)
|
|
else:
|
|
pkgs = self.q_binary_packages.filter(
|
|
name__glob=pattern, reponame__neq=self.opts.lookaside_repos
|
|
)
|
|
|
|
exclude.update(pkgs)
|
|
self.logger.debug("EXCLUDED by %s: %s", pattern, [str(p) for p in pkgs])
|
|
self.dnf._sack.add_excludes(pkgs)
|
|
|
|
for pattern in self.opts.multilib_blacklist:
|
|
with Profiler("Gather._apply_excludes():exclude-multilib-blacklist"):
|
|
# TODO: does whitelist affect this in any way?
|
|
pkgs = self.q_multilib_binary_packages.filter(
|
|
name__glob=pattern, arch__neq="noarch"
|
|
)
|
|
exclude.update(pkgs)
|
|
self.logger.debug("EXCLUDED by %s: %s", pattern, [str(p) for p in pkgs])
|
|
self.dnf._sack.add_excludes(pkgs)
|
|
|
|
all_queues = [
|
|
"q_binary_packages",
|
|
"q_native_binary_packages",
|
|
"q_multilib_binary_packages",
|
|
"q_noarch_binary_packages",
|
|
"q_source_packages",
|
|
"q_native_debug_packages",
|
|
"q_multilib_debug_packages",
|
|
]
|
|
|
|
with Profiler("Gather._apply_excludes():exclude-queries"):
|
|
for queue in all_queues:
|
|
self._filter_queue(queue, exclude)
|
|
|
|
@Profiler("Gather.add_initial_packages()")
|
|
def add_initial_packages(self, pattern_list):
|
|
added = set()
|
|
added_debuginfo = set()
|
|
|
|
excludes = []
|
|
includes = []
|
|
for pattern in pattern_list:
|
|
if pattern.startswith("-"):
|
|
excludes.append(pattern[1:])
|
|
else:
|
|
includes.append(pattern)
|
|
|
|
self._apply_excludes(excludes)
|
|
self.init_query_cache()
|
|
|
|
for pattern in includes:
|
|
with Profiler("Gather.add_initial_packages():include"):
|
|
debuginfo = False
|
|
if pattern == "system-release" and self.opts.greedy_method == "all":
|
|
pkgs = self.q_binary_packages.filter(
|
|
provides="system-release"
|
|
).apply()
|
|
else:
|
|
for p in DEBUG_PATTERNS:
|
|
p = p.replace("*", ".+")
|
|
if re.match(p, pattern):
|
|
debuginfo = True
|
|
break
|
|
if debuginfo:
|
|
if pattern.endswith(".+"):
|
|
pkgs = self.q_multilib_debug_packages.filter(
|
|
name__glob=pattern[:-2]
|
|
).apply()
|
|
else:
|
|
pkgs = self.q_debug_packages.filter(
|
|
name__glob=pattern
|
|
).apply()
|
|
else:
|
|
if pattern.endswith(".+"):
|
|
pkgs = self.q_multilib_binary_packages.filter(
|
|
name__glob=pattern[:-2]
|
|
).apply()
|
|
else:
|
|
kwargs = {"name__glob": pattern}
|
|
if "." in pattern:
|
|
# The pattern could be name.arch. Check if the
|
|
# arch is valid, and if yes, make a more
|
|
# specific query.
|
|
name, arch = pattern.split(".", 1)
|
|
if arch in arch_utils.arches:
|
|
kwargs["name__glob"] = name
|
|
kwargs["arch__eq"] = arch
|
|
pkgs = self.q_binary_packages.filter(**kwargs).apply()
|
|
|
|
if not pkgs:
|
|
self.logger.error(
|
|
"Could not find a match for %s in any configured repo", pattern
|
|
)
|
|
|
|
# The pattern could have been a glob. In that case we want to
|
|
# group the packages by name and get best match in those
|
|
# smaller groups.
|
|
packages_by_name = {}
|
|
for po in pkgs:
|
|
packages_by_name.setdefault(po.name, []).append(po)
|
|
|
|
for name, packages in packages_by_name.items():
|
|
pkgs = self._get_best_package(packages, debuginfo=debuginfo)
|
|
if pkgs:
|
|
if debuginfo:
|
|
added_debuginfo.update(pkgs)
|
|
else:
|
|
added.update(pkgs)
|
|
|
|
for pkg in added | added_debuginfo:
|
|
self._set_flag(pkg, PkgFlag.input)
|
|
|
|
return added, added_debuginfo
|
|
|
|
@Profiler("Gather.init_query_cache()")
|
|
def init_query_cache(self):
|
|
# HACK: workaround for insufficient hawkey query performance
|
|
# Must be executed *after* add_initial_packages() to exclude packages properly.
|
|
|
|
# source
|
|
self.source_pkgs_cache = QueryCache(
|
|
self.q_source_packages, "name", "version", "release"
|
|
)
|
|
|
|
# debug
|
|
self.native_debug_packages_cache = QueryCache(
|
|
self.q_native_debug_packages, "sourcerpm"
|
|
)
|
|
self.multilib_debug_packages_cache = QueryCache(
|
|
self.q_multilib_debug_packages, "sourcerpm"
|
|
)
|
|
|
|
# packages by sourcerpm
|
|
self.q_native_pkgs_by_sourcerpm_cache = QueryCache(
|
|
self.q_native_binary_packages, "sourcerpm", arch__neq="noarch"
|
|
)
|
|
self.q_multilib_pkgs_by_sourcerpm_cache = QueryCache(
|
|
self.q_multilib_binary_packages, "sourcerpm", arch__neq="noarch"
|
|
)
|
|
self.q_noarch_pkgs_by_sourcerpm_cache = QueryCache(
|
|
self.q_native_binary_packages, "sourcerpm", arch="noarch"
|
|
)
|
|
|
|
# multilib
|
|
self.q_multilib_binary_packages_cache = QueryCache(
|
|
self.q_multilib_binary_packages,
|
|
"name",
|
|
"version",
|
|
"release",
|
|
arch__neq="noarch",
|
|
)
|
|
|
|
# prepopulate
|
|
self.prepopulate_cache = QueryCache(self.q_binary_packages, "name", "arch")
|
|
|
|
@Profiler("Gather.add_prepopulate_packages()")
|
|
def add_prepopulate_packages(self):
|
|
added = set()
|
|
|
|
for name_arch in self.opts.prepopulate:
|
|
name, arch = name_arch.rsplit(".", 1)
|
|
pkgs = self.prepopulate_cache.get(name, arch)
|
|
pkgs = self._get_best_package(pkgs)
|
|
if pkgs:
|
|
added.update(pkgs)
|
|
else:
|
|
self.logger.warning("Prepopulate: Doesn't match: %s", name_arch)
|
|
|
|
for pkg in added:
|
|
self._set_flag(pkg, PkgFlag.prepopulate)
|
|
|
|
return added
|
|
|
|
@Profiler("Gather.add_binary_package_deps()")
|
|
def add_binary_package_deps(self):
|
|
added = set()
|
|
|
|
if not self.opts.resolve_deps:
|
|
return added
|
|
|
|
for pkg in self.result_binary_packages.copy():
|
|
assert pkg is not None
|
|
|
|
if pkg not in self.finished_add_binary_package_deps:
|
|
deps = self._get_package_deps(pkg)
|
|
for i, req in deps:
|
|
if i not in self.result_binary_packages:
|
|
self._add_packages(
|
|
[i], pulled_by=pkg, req=req, reason="binary-dep"
|
|
)
|
|
added.add(i)
|
|
self.finished_add_binary_package_deps[pkg] = deps
|
|
|
|
return added
|
|
|
|
@Profiler("Gather.add_debug_package_deps()")
|
|
def add_debug_package_deps(self):
|
|
added = set()
|
|
|
|
if not self.opts.resolve_deps or self.opts.exclude_debug:
|
|
return added
|
|
|
|
for pkg in self.result_debug_packages.copy():
|
|
if pkg not in self.finished_add_debug_package_deps:
|
|
deps = self._get_package_deps(pkg, debuginfo=True)
|
|
for i, req in deps:
|
|
if i in self.result_binary_packages:
|
|
# The dependency is already satisfied by binary package
|
|
continue
|
|
if i not in self.result_debug_packages:
|
|
self._add_packages(
|
|
[i],
|
|
pulled_by=pkg,
|
|
req=req,
|
|
reason="debug-dep",
|
|
dest=self.result_debug_packages,
|
|
)
|
|
added.add(i)
|
|
self.finished_add_debug_package_deps[pkg] = deps
|
|
|
|
return added
|
|
|
|
@Profiler("Gather.add_conditional_packages()")
|
|
def add_conditional_packages(self):
|
|
"""
|
|
For each binary package add their conditional dependencies
|
|
as specified in comps.
|
|
Return newly added packages.
|
|
"""
|
|
added = set()
|
|
|
|
if not self.opts.resolve_deps:
|
|
return added
|
|
|
|
for pkg in self.result_binary_packages.copy():
|
|
assert pkg is not None
|
|
|
|
try:
|
|
deps = self.finished_add_conditional_packages[pkg]
|
|
except KeyError:
|
|
deps = set()
|
|
for cond in self.conditional_packages:
|
|
if cond["name"] != pkg.name:
|
|
continue
|
|
pkgs = self.q_binary_packages.filter(name=cond["install"]).apply()
|
|
pkgs = self._get_best_package(pkgs) # TODO: multilib?
|
|
deps.update(pkgs)
|
|
self.finished_add_conditional_packages[pkg] = deps
|
|
|
|
for i in deps:
|
|
if i not in self.result_binary_packages:
|
|
self._add_packages([i], pulled_by=pkg, reason="cond-dep")
|
|
self._set_flag(pkg, PkgFlag.conditional)
|
|
added.add(i)
|
|
|
|
return added
|
|
|
|
@Profiler("Gather.add_source_package_deps()")
|
|
def add_source_package_deps(self):
|
|
added = set()
|
|
|
|
if not self.opts.resolve_deps:
|
|
return added
|
|
if not self.opts.selfhosting:
|
|
return added
|
|
if self.opts.exclude_source:
|
|
return added
|
|
|
|
for pkg in self.result_source_packages:
|
|
assert pkg is not None
|
|
|
|
try:
|
|
deps = self.finished_add_source_package_deps[pkg]
|
|
except KeyError:
|
|
deps = self._get_package_deps(pkg)
|
|
self.finished_add_source_package_deps[pkg] = set(
|
|
dep for (dep, req) in deps
|
|
)
|
|
for i, req in deps:
|
|
if i not in self.result_binary_packages:
|
|
self._add_packages(
|
|
[i], pulled_by=pkg, req=req, reason="source-dep"
|
|
)
|
|
added.add(i)
|
|
self._set_flag(pkg, PkgFlag.self_hosting)
|
|
|
|
return added
|
|
|
|
def _get_matching_srpm(self, pkg, sources):
|
|
"""
|
|
Get a source rpm from the same repo that pkg comes from. If there is no
|
|
such package, get the first one.
|
|
"""
|
|
for srpm in sources:
|
|
if pkg.repo == srpm.repo:
|
|
return srpm
|
|
return sources[0]
|
|
|
|
@Profiler("Gather.add_source_packages()")
|
|
def add_source_packages(self):
|
|
"""
|
|
For each binary package add it's source package.
|
|
Return newly added source packages.
|
|
"""
|
|
added = set()
|
|
|
|
if self.opts.exclude_source:
|
|
return added
|
|
|
|
for pkg in self.result_binary_packages:
|
|
assert pkg is not None
|
|
|
|
try:
|
|
source_pkg = self.finished_add_source_packages[pkg]
|
|
except KeyError:
|
|
source_pkg = None
|
|
if pkg.sourcerpm:
|
|
source_pkg = self.sourcerpm_cache.get(pkg.sourcerpm, None)
|
|
if source_pkg is None:
|
|
nvra = parse_nvra(pkg.sourcerpm)
|
|
source_pkgs = self.source_pkgs_cache.get(
|
|
nvra["name"], nvra["version"], nvra["release"]
|
|
)
|
|
if source_pkgs:
|
|
source_pkg = self._get_matching_srpm(pkg, source_pkgs)
|
|
self.sourcerpm_cache[pkg.sourcerpm] = source_pkg
|
|
self.finished_add_source_packages[pkg] = source_pkg
|
|
|
|
if not source_pkg:
|
|
continue
|
|
|
|
if (
|
|
source_pkg.repoid in self.opts.lookaside_repos
|
|
or pkg.repoid in self.opts.lookaside_repos
|
|
):
|
|
self._set_flag(source_pkg, PkgFlag.lookaside)
|
|
if source_pkg not in self.result_source_packages:
|
|
added.add(source_pkg)
|
|
self.result_source_packages.add(source_pkg)
|
|
|
|
return added
|
|
|
|
@Profiler("Gather.add_debug_packages()")
|
|
def add_debug_packages(self):
|
|
"""
|
|
For each binary package add debuginfo packages built from the same source.
|
|
Return newly added debug packages.
|
|
"""
|
|
added = set()
|
|
|
|
if self.opts.exclude_debug:
|
|
return added
|
|
|
|
for pkg in self.result_binary_packages:
|
|
assert pkg is not None
|
|
|
|
if pkg in self.finished_add_debug_packages:
|
|
continue
|
|
|
|
candidates = []
|
|
if pkg.sourcerpm:
|
|
if self.is_native_package(pkg):
|
|
candidates = self.native_debug_packages_cache.get(pkg.sourcerpm)
|
|
else:
|
|
candidates = self.multilib_debug_packages_cache.get(pkg.sourcerpm)
|
|
|
|
if not candidates:
|
|
continue
|
|
|
|
debug_pkgs = []
|
|
for i in candidates:
|
|
if pkg.arch != i.arch:
|
|
continue
|
|
if "-debugsource" not in i.name and i.name != "%s-debuginfo" % pkg.name:
|
|
# If it's not debugsource package or does not match name of
|
|
# the package, we don't want it in.
|
|
continue
|
|
if self.is_from_lookaside(i):
|
|
self._set_flag(i, PkgFlag.lookaside)
|
|
if i not in self.result_debug_packages:
|
|
added.add(i)
|
|
debug_pkgs.append(i)
|
|
self.logger.debug(
|
|
"Added debuginfo %s (for %s, repo: %s)" % (i, pkg, i.repo.id)
|
|
)
|
|
|
|
self.finished_add_debug_packages[pkg] = debug_pkgs
|
|
self.result_debug_packages.update(debug_pkgs)
|
|
|
|
return added
|
|
|
|
@Profiler("Gather.add_fulltree_packages()")
|
|
def add_fulltree_packages(self):
|
|
"""
|
|
For each binary package add all binary packages built from the same source.
|
|
Return newly added binary packages.
|
|
"""
|
|
added = set()
|
|
|
|
if not self.opts.fulltree:
|
|
return added
|
|
|
|
for pkg in sorted(self.result_binary_packages):
|
|
assert pkg is not None
|
|
|
|
if get_source_name(pkg) in self.opts.fulltree_excludes:
|
|
self.logger.debug("No fulltree for %s due to exclude list", pkg)
|
|
continue
|
|
|
|
try:
|
|
fulltree_pkgs = self.finished_add_fulltree_packages[pkg]
|
|
except KeyError:
|
|
native_fulltree_pkgs = (
|
|
self.q_native_pkgs_by_sourcerpm_cache.get(pkg.sourcerpm) or []
|
|
)
|
|
multilib_fulltree_pkgs = (
|
|
self.q_multilib_pkgs_by_sourcerpm_cache.get(pkg.sourcerpm) or []
|
|
)
|
|
noarch_fulltree_pkgs = (
|
|
self.q_noarch_pkgs_by_sourcerpm_cache.get(pkg.sourcerpm) or []
|
|
)
|
|
|
|
if not native_fulltree_pkgs:
|
|
# no existing native pkgs -> pull multilib
|
|
pull_native = False
|
|
elif set(native_fulltree_pkgs) & self.result_binary_packages:
|
|
# native pkgs in result -> pull native
|
|
pull_native = True
|
|
elif set(multilib_fulltree_pkgs) & self.result_binary_packages:
|
|
# multilib pkgs in result -> pull multilib
|
|
pull_native = False
|
|
else:
|
|
# fallback / default
|
|
pull_native = True
|
|
|
|
# We pull packages determined by `pull_native`, or everything
|
|
# if we're greedy
|
|
fulltree_pkgs = []
|
|
if pull_native or self.opts.greedy_method == "all":
|
|
fulltree_pkgs.extend(native_fulltree_pkgs)
|
|
if not pull_native or self.opts.greedy_method == "all":
|
|
fulltree_pkgs.extend(multilib_fulltree_pkgs)
|
|
|
|
# always pull all noarch subpackages
|
|
fulltree_pkgs += noarch_fulltree_pkgs
|
|
|
|
for i in fulltree_pkgs:
|
|
if i not in self.result_binary_packages:
|
|
self._add_packages([i], reason="fulltree")
|
|
self._set_flag(i, PkgFlag.fulltree)
|
|
added.add(i)
|
|
|
|
# don't run fulltree on added packages
|
|
self.finished_add_fulltree_packages[i] = []
|
|
|
|
self.finished_add_fulltree_packages[pkg] = fulltree_pkgs
|
|
|
|
return added
|
|
|
|
@Profiler("Gather.add_langpack_packages()")
|
|
def add_langpack_packages(self, langpack_patterns):
|
|
"""
|
|
For each binary package add all matching langpack packages.
|
|
Return newly added binary packages.
|
|
|
|
langpack_patterns: [{"name": <str>, "install": <str>}]
|
|
"""
|
|
added = set()
|
|
|
|
if not self.opts.langpacks:
|
|
return added
|
|
|
|
exceptions = ["man-pages-overrides"]
|
|
|
|
for pkg in sorted(self.result_binary_packages):
|
|
assert pkg is not None
|
|
|
|
try:
|
|
langpack_pkgs = self.finished_add_langpack_packages[pkg]
|
|
except KeyError:
|
|
patterns = [
|
|
i["install"] for i in langpack_patterns if i["name"] == pkg.name
|
|
]
|
|
patterns = [i.replace("%s", "*") for i in patterns]
|
|
|
|
if not patterns:
|
|
self.finished_add_langpack_packages[pkg] = []
|
|
continue
|
|
|
|
langpack_pkgs = self.q_binary_packages.filter(
|
|
name__glob=patterns
|
|
).apply()
|
|
langpack_pkgs = langpack_pkgs.filter(
|
|
name__glob__not=["*-devel", "*-static"]
|
|
)
|
|
langpack_pkgs = langpack_pkgs.filter(name__neq=exceptions)
|
|
|
|
pkgs_by_name = {}
|
|
for i in langpack_pkgs:
|
|
pkgs_by_name.setdefault(i.name, set()).add(i)
|
|
|
|
langpack_pkgs = set()
|
|
for name in sorted(pkgs_by_name):
|
|
pkgs = pkgs_by_name[name]
|
|
i = self._get_best_package(pkgs)
|
|
if i:
|
|
# TODO: greedy
|
|
i = i[0]
|
|
langpack_pkgs.add(i)
|
|
self._set_flag(i, PkgFlag.langpack)
|
|
if i not in self.result_binary_packages:
|
|
self._add_packages([i], pulled_by=pkg, reason="langpack")
|
|
added.add(pkg)
|
|
self.finished_add_langpack_packages[pkg] = langpack_pkgs
|
|
|
|
return added
|
|
|
|
@Profiler("Gather.add_multilib_packages()")
|
|
def add_multilib_packages(self):
|
|
added = set()
|
|
|
|
for pkg in sorted(self.result_binary_packages):
|
|
if pkg in self.finished_add_multilib_packages:
|
|
continue
|
|
|
|
if pkg.arch in ("noarch", "src", "nosrc"):
|
|
self.finished_add_multilib_packages[pkg] = None
|
|
continue
|
|
|
|
if pkg.arch in self.dnf.arch_wrapper.multilib_arches:
|
|
self.finished_add_multilib_packages[pkg] = None
|
|
continue
|
|
|
|
pkgs = self.q_multilib_binary_packages_cache.get(
|
|
pkg.name, pkg.version, pkg.release
|
|
)
|
|
pkgs = self._get_best_package(pkgs)
|
|
multilib_pkgs = []
|
|
for i in pkgs:
|
|
is_multilib = self._multilib.is_multilib(i)
|
|
if is_multilib:
|
|
multilib_pkgs.append(i)
|
|
added.add(i)
|
|
self._set_flag(i, PkgFlag.multilib)
|
|
self._add_packages([i], reason="multilib:%s" % is_multilib)
|
|
self.finished_add_multilib_packages[pkg] = i
|
|
# TODO: ^^^ may get multiple results; i686, i586, etc.
|
|
|
|
return added
|
|
|
|
@Profiler("Gather.gather()")
|
|
def gather(self, pattern_list, conditional_packages=None):
|
|
self.conditional_packages = conditional_packages or []
|
|
|
|
self.logger.debug("INITIAL PACKAGES")
|
|
added, added_debuginfo = self.add_initial_packages(pattern_list)
|
|
self._add_packages(added)
|
|
if added_debuginfo:
|
|
self._add_packages(added_debuginfo, dest=self.result_debug_packages)
|
|
|
|
added = self.log_count("PREPOPULATE", self.add_prepopulate_packages)
|
|
self._add_packages(added, reason="prepopulate")
|
|
|
|
for pass_num in count(1):
|
|
self.logger.debug("PASS %s" % pass_num)
|
|
|
|
if self.log_count("CONDITIONAL DEPS", self.add_conditional_packages):
|
|
continue
|
|
|
|
# resolve deps
|
|
if self.log_count("BINARY DEPS", self.add_binary_package_deps):
|
|
continue
|
|
|
|
if self.log_count("SOURCE DEPS", self.add_source_package_deps):
|
|
continue
|
|
|
|
if self.log_count("SOURCE PACKAGES", self.add_source_packages):
|
|
continue
|
|
|
|
if self.log_count("DEBUG PACKAGES", self.add_debug_packages):
|
|
continue
|
|
|
|
if self.log_count("DEBUG DEPS", self.add_debug_package_deps):
|
|
continue
|
|
|
|
if self.log_count("FULLTREE", self.add_fulltree_packages):
|
|
continue
|
|
|
|
if self.log_count(
|
|
"LANGPACKS", self.add_langpack_packages, self.opts.langpacks
|
|
):
|
|
continue
|
|
|
|
if self.log_count("MULTILIB", self.add_multilib_packages):
|
|
continue
|
|
|
|
# nothing added -> break depsolving cycle
|
|
break
|
|
|
|
def download(self, destdir):
|
|
pkglist = (
|
|
self.result_binary_packages
|
|
| self.result_debug_packages
|
|
| self.result_source_packages
|
|
)
|
|
self.dnf.download_packages(pkglist)
|
|
linker = Linker(logger=self.logger)
|
|
|
|
for pkg in pkglist:
|
|
basename = os.path.basename(pkg.relativepath)
|
|
target = os.path.join(destdir, basename)
|
|
|
|
# Link downloaded package in (or link package from file repo)
|
|
try:
|
|
linker.link(pkg.localPkg(), target)
|
|
except Exception:
|
|
self.logger.error("Unable to link %s from the yum cache." % pkg.name)
|
|
raise
|
|
|
|
def log_count(self, msg, method, *args):
|
|
"""
|
|
Print a message, run the function with given arguments and log length
|
|
of result.
|
|
"""
|
|
self.logger.debug("%s", msg)
|
|
added = method(*args)
|
|
self.logger.debug("ADDED: %s", len(added))
|
|
return added
|