almalinux/pungi
14
1
Fork 0
Code Issues Pull Requests Releases Activity

gather: Resolve dependencies of debug packages

Browse Source 
Until now, the behaviour was that all debuginfo from a build would be
included if at least one package with the same arch was included.

This resulted in many debuginfo packages being included even though
their corresponding package was not present.

With this patch, we will always pull in debugsource, but foo-debuginfo
will only be included if foo is included for the same arch. As a
consequence, it is necessary to resolve dependencies of debuginfo
packages. There are cases where foo-debuginfo needs foo-debuginfo-common
for example.

This change means that DNF and YUM backends are no longer identical in
the output. The tests where this is demonstrated are duplicated and
their results are modified.

JIRA: COMPOSE-3823
Signed-off-by: Lubomír Sedlář <lsedlar@redhat.com>
This commit is contained in:
Lubomír Sedlář 2019-09-27 14:05:09 +02:00
parent 4413b0df24
commit 26962d94ca
6 changed files with 248 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

70
pungi/gather_dnf.py
View File

@ -220,7 +220,7 @@ class Gather(GatherBase):
def _set_flag(self, pkg, *flags):
self.result_package_flags.setdefault(pkg, set()).update(flags)
def _get_best_package(self, package_list, pkg=None, req=None):
def _get_best_package(self, package_list, pkg=None, req=None, debuginfo=False):
if not package_list:
return []
@ -245,27 +245,31 @@ class Gather(GatherBase):
):
all_pkgs.append(pkg)
native_pkgs = self.q_native_binary_packages.filter(pkg=all_pkgs).apply()
multilib_pkgs = self.q_multilib_binary_packages.filter(pkg=all_pkgs).apply()
if not debuginfo:
native_pkgs = set(self.q_native_binary_packages.filter(pkg=all_pkgs).apply())
multilib_pkgs = set(self.q_multilib_binary_packages.filter(pkg=all_pkgs).apply())
else:
native_pkgs = set(self.q_native_debug_packages.filter(pkg=all_pkgs).apply())
multilib_pkgs = set(self.q_multilib_debug_packages.filter(pkg=all_pkgs).apply())
result = set()
# try seen native packages first
seen_pkgs = set(native_pkgs) & self.result_binary_packages
seen_pkgs = native_pkgs & self.result_binary_packages
if seen_pkgs:
result = seen_pkgs
# then try seen multilib packages
if not result:
seen_pkgs = set(multilib_pkgs) & self.result_binary_packages
seen_pkgs = multilib_pkgs & self.result_binary_packages
if seen_pkgs:
result = seen_pkgs
if not result:
result = set(native_pkgs)
result = native_pkgs
if not result:
result = set(multilib_pkgs)
result = multilib_pkgs
if not result:
return []
@ -307,13 +311,14 @@ class Gather(GatherBase):
if i.sourcerpm.rsplit('-', 2)[0] in self.opts.fulltree_excludes:
self._set_flag(i, PkgFlag.fulltree_exclude)
def _get_package_deps(self, pkg):
def _get_package_deps(self, pkg, debuginfo=False):
"""Return all direct (1st level) deps for a package.
The return value is a set of tuples (pkg, reldep). Each package is
tagged with the particular reldep that pulled it in. Requires_pre and
_post are not distinguished.
"""
queue = self.q_debug_packages if debuginfo else self.q_binary_packages
assert pkg is not None
result = set()
@ -329,7 +334,7 @@ class Gather(GatherBase):
+ getattr(pkg, 'requires_post', [])
)
q = self.q_binary_packages.filter(provides=requires).apply()
q = queue.filter(provides=requires).apply()
for req in requires:
deps = self.finished_get_package_deps_reqs.setdefault(str(req), set())
if deps:
@ -339,7 +344,7 @@ class Gather(GatherBase):
# TODO: need query also debuginfo
deps = q.filter(provides=req)
if deps:
deps = self._get_best_package(deps, req=req)
deps = self._get_best_package(deps, req=req, debuginfo=debuginfo)
self.finished_get_package_deps_reqs[str(req)].update(deps)
result.update((dep, req) for dep in deps)
else:
@ -510,6 +515,34 @@ class Gather(GatherBase):
return added
@Profiler("Gather.add_debug_package_deps()")
def add_debug_package_deps(self):
added = set()
if not self.opts.resolve_deps:
return added
for pkg in self.result_debug_packages.copy():
if pkg not in self.finished_add_debug_package_deps:
deps = self._get_package_deps(pkg, debuginfo=True)
for i, req in deps:
if i in self.result_binary_packages:
# The dependency is already satisfied by binary package
continue
if i not in self.result_debug_packages:
self._add_packages(
[i],
pulled_by=pkg,
req=req,
reason="debug-dep",
dest=self.result_debug_packages,
)
added.add(i)
self.finished_add_debug_package_deps[pkg] = deps
return added
@Profiler("Gather.add_conditional_packages()")
def add_conditional_packages(self):
"""
@ -643,17 +676,20 @@ class Gather(GatherBase):
debug_pkgs = []
pkg_in_lookaside = pkg.repoid in self.opts.lookaside_repos
for i in candidates:
if pkg.arch == 'noarch' and i.arch != 'noarch':
# If the package is noarch, we will only pull debuginfo if
# it's noarch as well. This covers mingw use case, but
# means we don't for example pull debuginfo just because of
# -doc subpackage.
if pkg.arch != i.arch:
continue
if "-debugsource" not in i.name and i.name != "%s-debuginfo" % pkg.name:
# If it's not debugsource package or does not match name of
# the package, we don't want it in.
continue
if i.repoid in self.opts.lookaside_repos or pkg_in_lookaside:
self._set_flag(i, PkgFlag.lookaside)
if i not in self.result_debug_packages:
added.add(i)
debug_pkgs.append(i)
self.logger.debug(
"Added debuginfo %s (for %s, repo: %s)" % (i, pkg, i.repo.id)
)
self.finished_add_debug_packages[pkg] = debug_pkgs
self.result_debug_packages.update(debug_pkgs)
@ -834,7 +870,9 @@ class Gather(GatherBase):
if self.log_count('DEBUG PACKAGES', self.add_debug_packages):
continue
# TODO: debug deps
if self.log_count("DEBUG DEPS", self.add_debug_package_deps):
continue
if self.log_count('FULLTREE', self.add_fulltree_packages):
continue

BIN
tests/fixtures/repos/debuginfo/repodata/filelists.xml.gz vendored Normal file
View File

Binary file not shown.

BIN
tests/fixtures/repos/debuginfo/repodata/other.xml.gz vendored Normal file
View File

Binary file not shown.

BIN
tests/fixtures/repos/debuginfo/repodata/primary.xml.gz vendored Normal file
View File

Binary file not shown.

28
tests/fixtures/repos/debuginfo/repodata/repomd.xml vendored Normal file
View File

@ -0,0 +1,28 @@
<?xml version="1.0" encoding="UTF-8"?>
<repomd xmlns="http://linux.duke.edu/metadata/repo" xmlns:rpm="http://linux.duke.edu/metadata/rpm">
<revision>1569500166</revision>
<data type="primary">
<checksum type="sha512">d7043643dc30ad24861d17329ae0ea638cc0659af7b586cb4482eff7d47949c5a8066fc03fa5df98efdd47d8f077628eb60e2fca5f1b24e1e8a3e84dc5a2042c</checksum>
<open-checksum type="sha512">477adda50364b5532d161a1d5fdeb6ea36cea3282d1f8d24de1b66108a5564f3ce47c573de5227d8f21a79cdab5012f5e30b03d5469fce0db98402bcf3635810</open-checksum>
<location href="repodata/primary.xml.gz"/>
<timestamp>1569500166</timestamp>
<size>578</size>
<open-size>7159</open-size>
</data>
<data type="filelists">
<checksum type="sha512">8876e1c8b97828e5e5d2ce35f8722a6f5cc3e7ef5a0c9be0c29011b2dab2a2ce3320fa2ecbdfd7b9ef379c4e091d75a4da18d0ec49691aba2f06515f61ffd794</checksum>
<open-checksum type="sha512">c809d172a03d06ec86481d221bb086f15d90f8c31ea30f26dd61a86f19633a4bf04d25b4fe9f9ddfdf3876079ab671ffd7c483bf3d991677fd1c970817e4bd2b</open-checksum>
<location href="repodata/filelists.xml.gz"/>
<timestamp>1569500166</timestamp>
<size>123</size>
<open-size>125</open-size>
</data>
<data type="other">
<checksum type="sha512">49e704998e8c9e154ad355be4c58476c0cbc298d4804c268ef2102995b32e14be16e9ac1f785338ce2dd1373574b356705c306efd37d52824436792b812a5e3b</checksum>
<open-checksum type="sha512">c1ca53d1a6f4ca3c94decb035a9a5e82bbab46da4e4322be85fdea10c1f87c6ebf490a5d71dc7fb4718a2ce14ce21d93b8c4e8a7bccd77e4b3d2057c6019899c</open-checksum>
<location href="repodata/other.xml.gz"/>
<timestamp>1569500166</timestamp>
<size>123</size>
<open-size>121</open-size>
</data>
</repomd>

167
tests/test_gather.py
View File

@ -1856,4 +1856,169 @@ class DNFDepsolvingTestCase(DepsolvingBase, unittest.TestCase):
super(DNFDepsolvingTestCase, self).test_firefox_selfhosting_with_krb5_lookaside()
self.assertFlags("dummy-krb5-devel-1.10-5.x86_64", [PkgFlag.lookaside])
self.assertFlags("dummy-krb5-1.10-5.src", [PkgFlag.lookaside])
self.assertFlags("dummy-krb5-debuginfo-1.10-5.x86_64", [PkgFlag.lookaside])
def test_only_required_debuginfo_pulled_in(self):
self.repo = os.path.join(os.path.dirname(__file__), "fixtures/repos/debuginfo")
packages = ["libuser.+", "python-libuser"]
pkg_map = self.go(packages, None)
self.assertItemsEqual(pkg_map["rpm"], [
"libuser-1-1.i686.rpm",
"python-libuser-1-1.x86_64.rpm",
])
self.assertItemsEqual(pkg_map["srpm"], [
"libuser-1-1.src.rpm",
])
self.assertItemsEqual(pkg_map["debuginfo"], [
"libuser-debuginfo-1-1.i686.rpm",
"python-libuser-debuginfo-1-1.x86_64.rpm",
])
def test_skype(self):
packages = [
"dummy-skype",
]
pkg_map = self.go(packages, None, greedy="build", fulltree=False, arch="x86_64")
self.assertItemsEqual(pkg_map["rpm"], [
"dummy-skype-4.2.0.13-1.i586.rpm",
"dummy-basesystem-10.0-6.noarch.rpm",
"dummy-filesystem-4.2.37-6.x86_64.rpm",
"dummy-glibc-common-2.14-5.x86_64.rpm",
"dummy-glibc-2.14-5.i686.rpm",
])
# no SRPM for skype
self.assertItemsEqual(pkg_map["srpm"], [
"dummy-basesystem-10.0-6.src.rpm",
"dummy-glibc-2.14-5.src.rpm",
"dummy-filesystem-4.2.37-6.src.rpm",
])
self.assertItemsEqual(pkg_map["debuginfo"], [
"dummy-glibc-debuginfo-common-2.14-5.i686.rpm",
"dummy-glibc-debuginfo-2.14-5.i686.rpm",
])
def test_bash_multilib_exclude_debuginfo(self):
packages = [
'dummy-bash.+',
'-dummy-bash-debuginfo',
'-dummy-bash-debugsource',
]
pkg_map = self.go(packages, None, greedy="none")
self.assertItemsEqual(pkg_map["rpm"], [
"dummy-basesystem-10.0-6.noarch.rpm",
"dummy-bash-4.2.37-6.i686.rpm",
"dummy-filesystem-4.2.37-6.x86_64.rpm",
"dummy-glibc-2.14-5.i686.rpm",
"dummy-glibc-common-2.14-5.x86_64.rpm",
])
self.assertItemsEqual(pkg_map["srpm"], [
"dummy-basesystem-10.0-6.src.rpm",
"dummy-bash-4.2.37-6.src.rpm",
"dummy-filesystem-4.2.37-6.src.rpm",
"dummy-glibc-2.14-5.src.rpm",
])
self.assertItemsEqual(pkg_map["debuginfo"], [
"dummy-glibc-debuginfo-2.14-5.i686.rpm",
"dummy-glibc-debuginfo-common-2.14-5.i686.rpm",
])
def test_exclude_wildcards(self):
packages = [
"dummy-bash",
"-dummy-bas*",
"dummy-glibc",
]
pkg_map = self.go(packages, None,
greedy="none", nodeps=True, fulltree=True)
# neither dummy-bash or dummy-basesystem is pulled in
self.assertItemsEqual(pkg_map["rpm"], [
"dummy-glibc-2.14-5.x86_64.rpm",
"dummy-glibc-common-2.14-5.x86_64.rpm",
"dummy-nscd-2.14-5.x86_64.rpm",
])
self.assertItemsEqual(pkg_map["srpm"], [
"dummy-glibc-2.14-5.src.rpm",
])
self.assertItemsEqual(pkg_map["debuginfo"], [
"dummy-glibc-debuginfo-2.14-5.x86_64.rpm",
])
def test_bash_multilib_exclude_source(self):
packages = [
"dummy-bash.+",
"-dummy-bash.src",
]
pkg_map = self.go(packages, None, greedy="none")
self.assertNotIn("dummy-bash-4.2.37-6.src.rpm", pkg_map["srpm"])
self.assertItemsEqual(pkg_map["rpm"], [
"dummy-basesystem-10.0-6.noarch.rpm",
"dummy-bash-4.2.37-6.i686.rpm",
"dummy-filesystem-4.2.37-6.x86_64.rpm",
"dummy-glibc-2.14-5.i686.rpm",
"dummy-glibc-common-2.14-5.x86_64.rpm",
])
self.assertItemsEqual(pkg_map["srpm"], [
"dummy-basesystem-10.0-6.src.rpm",
"dummy-filesystem-4.2.37-6.src.rpm",
"dummy-glibc-2.14-5.src.rpm",
])
self.assertItemsEqual(pkg_map["debuginfo"], [
"dummy-bash-debuginfo-4.2.37-6.i686.rpm",
"dummy-bash-debugsource-4.2.37-6.i686.rpm",
"dummy-glibc-debuginfo-2.14-5.i686.rpm",
"dummy-glibc-debuginfo-common-2.14-5.i686.rpm",
])
def test_ipw3945_kmod(self):
# every package name is different
packages = [
"dummy-kmod-ipw3945",
]
pkg_map = self.go(packages, None, greedy="none", fulltree=True)
self.assertItemsEqual(pkg_map["rpm"], [
"dummy-kmod-ipw3945-1.2.0-4.20.x86_64.rpm", # Important
"dummy-kmod-ipw3945-xen-1.2.0-4.20.x86_64.rpm",
])
self.assertItemsEqual(pkg_map["srpm"], [
"dummy-ipw3945-kmod-1.2.0-4.20.src.rpm",
])
self.assertItemsEqual(pkg_map["debuginfo"], [])
def test_multilib_method_devel_runtime(self):
packages = [
"dummy-lvm2-devel",
]
pkg_map = self.go(packages, None, greedy="none", fulltree=False,
multilib_methods=["devel", "runtime"])
self.assertItemsEqual(pkg_map["rpm"], [
"dummy-basesystem-10.0-6.noarch.rpm",
"dummy-filesystem-4.2.37-6.x86_64.rpm",
"dummy-glibc-2.14-5.x86_64.rpm",
"dummy-glibc-2.14-5.i686.rpm",
"dummy-glibc-common-2.14-5.x86_64.rpm",
"dummy-lvm2-2.02.84-4.x86_64.rpm",
"dummy-lvm2-devel-2.02.84-4.i686.rpm", # Important
"dummy-lvm2-devel-2.02.84-4.x86_64.rpm", # Important
"dummy-lvm2-libs-2.02.84-4.x86_64.rpm",
])
self.assertItemsEqual(pkg_map["srpm"], [
"dummy-basesystem-10.0-6.src.rpm",
"dummy-filesystem-4.2.37-6.src.rpm",
"dummy-glibc-2.14-5.src.rpm",
"dummy-lvm2-2.02.84-4.src.rpm",
])
self.assertItemsEqual(pkg_map["debuginfo"], [
"dummy-glibc-debuginfo-2.14-5.x86_64.rpm",
"dummy-glibc-debuginfo-common-2.14-5.x86_64.rpm",
"dummy-glibc-debuginfo-2.14-5.i686.rpm",
"dummy-glibc-debuginfo-common-2.14-5.i686.rpm",
"dummy-lvm2-debuginfo-2.02.84-4.x86_64.rpm",
])