From 26962d94ca7277606cc28e34ddc746997b87c78d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lubom=C3=ADr=20Sedl=C3=A1=C5=99?= Date: Fri, 27 Sep 2019 14:05:09 +0200 Subject: [PATCH] gather: Resolve dependencies of debug packages MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Until now, the behaviour was that all debuginfo from a build would be included if at least one package with the same arch was included. This resulted in many debuginfo packages being included even though their corresponding package was not present. With this patch, we will always pull in debugsource, but foo-debuginfo will only be included if foo is included for the same arch. As a consequence, it is necessary to resolve dependencies of debuginfo packages. There are cases where foo-debuginfo needs foo-debuginfo-common for example. This change means that DNF and YUM backends are no longer identical in the output. The tests where this is demonstrated are duplicated and their results are modified. JIRA: COMPOSE-3823 Signed-off-by: Lubomír Sedlář --- pungi/gather_dnf.py | 70 ++++++-- .../repos/debuginfo/repodata/filelists.xml.gz | Bin 0 -> 123 bytes .../repos/debuginfo/repodata/other.xml.gz | Bin 0 -> 123 bytes .../repos/debuginfo/repodata/primary.xml.gz | Bin 0 -> 578 bytes .../repos/debuginfo/repodata/repomd.xml | 28 +++ tests/test_gather.py | 167 +++++++++++++++++- 6 files changed, 248 insertions(+), 17 deletions(-) create mode 100644 tests/fixtures/repos/debuginfo/repodata/filelists.xml.gz create mode 100644 tests/fixtures/repos/debuginfo/repodata/other.xml.gz create mode 100644 tests/fixtures/repos/debuginfo/repodata/primary.xml.gz create mode 100644 tests/fixtures/repos/debuginfo/repodata/repomd.xml diff --git a/pungi/gather_dnf.py b/pungi/gather_dnf.py index b8cc0bfd..714c8d2c 100644 --- a/pungi/gather_dnf.py +++ b/pungi/gather_dnf.py @@ -220,7 +220,7 @@ class Gather(GatherBase): def _set_flag(self, pkg, *flags): self.result_package_flags.setdefault(pkg, set()).update(flags) - def _get_best_package(self, package_list, pkg=None, req=None): + def _get_best_package(self, package_list, pkg=None, req=None, debuginfo=False): if not package_list: return [] @@ -245,27 +245,31 @@ class Gather(GatherBase): ): all_pkgs.append(pkg) - native_pkgs = self.q_native_binary_packages.filter(pkg=all_pkgs).apply() - multilib_pkgs = self.q_multilib_binary_packages.filter(pkg=all_pkgs).apply() + if not debuginfo: + native_pkgs = set(self.q_native_binary_packages.filter(pkg=all_pkgs).apply()) + multilib_pkgs = set(self.q_multilib_binary_packages.filter(pkg=all_pkgs).apply()) + else: + native_pkgs = set(self.q_native_debug_packages.filter(pkg=all_pkgs).apply()) + multilib_pkgs = set(self.q_multilib_debug_packages.filter(pkg=all_pkgs).apply()) result = set() # try seen native packages first - seen_pkgs = set(native_pkgs) & self.result_binary_packages + seen_pkgs = native_pkgs & self.result_binary_packages if seen_pkgs: result = seen_pkgs # then try seen multilib packages if not result: - seen_pkgs = set(multilib_pkgs) & self.result_binary_packages + seen_pkgs = multilib_pkgs & self.result_binary_packages if seen_pkgs: result = seen_pkgs if not result: - result = set(native_pkgs) + result = native_pkgs if not result: - result = set(multilib_pkgs) + result = multilib_pkgs if not result: return [] @@ -307,13 +311,14 @@ class Gather(GatherBase): if i.sourcerpm.rsplit('-', 2)[0] in self.opts.fulltree_excludes: self._set_flag(i, PkgFlag.fulltree_exclude) - def _get_package_deps(self, pkg): + def _get_package_deps(self, pkg, debuginfo=False): """Return all direct (1st level) deps for a package. The return value is a set of tuples (pkg, reldep). Each package is tagged with the particular reldep that pulled it in. Requires_pre and _post are not distinguished. """ + queue = self.q_debug_packages if debuginfo else self.q_binary_packages assert pkg is not None result = set() @@ -329,7 +334,7 @@ class Gather(GatherBase): + getattr(pkg, 'requires_post', []) ) - q = self.q_binary_packages.filter(provides=requires).apply() + q = queue.filter(provides=requires).apply() for req in requires: deps = self.finished_get_package_deps_reqs.setdefault(str(req), set()) if deps: @@ -339,7 +344,7 @@ class Gather(GatherBase): # TODO: need query also debuginfo deps = q.filter(provides=req) if deps: - deps = self._get_best_package(deps, req=req) + deps = self._get_best_package(deps, req=req, debuginfo=debuginfo) self.finished_get_package_deps_reqs[str(req)].update(deps) result.update((dep, req) for dep in deps) else: @@ -510,6 +515,34 @@ class Gather(GatherBase): return added + @Profiler("Gather.add_debug_package_deps()") + def add_debug_package_deps(self): + added = set() + + if not self.opts.resolve_deps: + return added + + for pkg in self.result_debug_packages.copy(): + + if pkg not in self.finished_add_debug_package_deps: + deps = self._get_package_deps(pkg, debuginfo=True) + for i, req in deps: + if i in self.result_binary_packages: + # The dependency is already satisfied by binary package + continue + if i not in self.result_debug_packages: + self._add_packages( + [i], + pulled_by=pkg, + req=req, + reason="debug-dep", + dest=self.result_debug_packages, + ) + added.add(i) + self.finished_add_debug_package_deps[pkg] = deps + + return added + @Profiler("Gather.add_conditional_packages()") def add_conditional_packages(self): """ @@ -643,17 +676,20 @@ class Gather(GatherBase): debug_pkgs = [] pkg_in_lookaside = pkg.repoid in self.opts.lookaside_repos for i in candidates: - if pkg.arch == 'noarch' and i.arch != 'noarch': - # If the package is noarch, we will only pull debuginfo if - # it's noarch as well. This covers mingw use case, but - # means we don't for example pull debuginfo just because of - # -doc subpackage. + if pkg.arch != i.arch: + continue + if "-debugsource" not in i.name and i.name != "%s-debuginfo" % pkg.name: + # If it's not debugsource package or does not match name of + # the package, we don't want it in. continue if i.repoid in self.opts.lookaside_repos or pkg_in_lookaside: self._set_flag(i, PkgFlag.lookaside) if i not in self.result_debug_packages: added.add(i) debug_pkgs.append(i) + self.logger.debug( + "Added debuginfo %s (for %s, repo: %s)" % (i, pkg, i.repo.id) + ) self.finished_add_debug_packages[pkg] = debug_pkgs self.result_debug_packages.update(debug_pkgs) @@ -834,7 +870,9 @@ class Gather(GatherBase): if self.log_count('DEBUG PACKAGES', self.add_debug_packages): continue - # TODO: debug deps + + if self.log_count("DEBUG DEPS", self.add_debug_package_deps): + continue if self.log_count('FULLTREE', self.add_fulltree_packages): continue diff --git a/tests/fixtures/repos/debuginfo/repodata/filelists.xml.gz b/tests/fixtures/repos/debuginfo/repodata/filelists.xml.gz new file mode 100644 index 0000000000000000000000000000000000000000..40471d4c2cf86b45008bc3efd7e82ca3d7d21c17 GIT binary patch literal 123 zcmb2|=3oE=X4f8TzGed+hjKr`%6upD5WyQ3j4}t5^tN)a2`+#1BPHwk_srkf-Os$! zAH~|(5=}#UmuG%xFTj%E{#q}wgDcdI|Ik{f>alERE&%ukv;#bRMq-+6e&N-7>oX literal 0 HcmV?d00001 diff --git a/tests/fixtures/repos/debuginfo/repodata/other.xml.gz b/tests/fixtures/repos/debuginfo/repodata/other.xml.gz new file mode 100644 index 0000000000000000000000000000000000000000..89d6635dbfe9e439c0f818c0b205765541977647 GIT binary patch literal 123 zcmb2|=3oE=X4^A{e9ZD1UsiNuqU2(JBg-E=wLhEI>CReVEI)6G&$lA08KK|JA4#diUR&O(`@2P2`>%+d a+!?>ePQvN&j&UFEZ!!N`XmY+1XeR&$;W5O0z3;EEa`7oRPzWqavX_2bdZ^UyVNaED24a!e zk?kPpudkgC0;fu?b}JThz@G7Q?3sBlPL$;7Se57iN^>dFU^j8j37j=rkydsn>**Tr|f?-t1ulc|!&iVX~^E2s-py)zu$oSWJLw%Jy^z z_WDs^7wt`CPBo<1B5?MAP$FmrS;;qz29=Q3a&D?vk!i&e()OIIj|)L9S&`xhyGTM9X_)EyJNV}qKE2>V}N86sB7A*#?hUUO`SapT?6A=4ta;`+#{?FypY;t zZO@VD+C4(GY!n0A8~7GZ!|BA5%!SfT8zJMg0;s)gsHm{)wTV+E#q&jE^JKT2;>B_q z<1P(k$iumUf5-ddD*hSIJ@q~H?~eLBUd5jHp7`&c_+bv4X2-==`n&&aiD0apm%|&# z!Ji0T4EG|2`eX_zLjPKDTUxj-n-{`s3&9@=UIuq41OLi! + + 1569500166 + + d7043643dc30ad24861d17329ae0ea638cc0659af7b586cb4482eff7d47949c5a8066fc03fa5df98efdd47d8f077628eb60e2fca5f1b24e1e8a3e84dc5a2042c + 477adda50364b5532d161a1d5fdeb6ea36cea3282d1f8d24de1b66108a5564f3ce47c573de5227d8f21a79cdab5012f5e30b03d5469fce0db98402bcf3635810 + + 1569500166 + 578 + 7159 + + + 8876e1c8b97828e5e5d2ce35f8722a6f5cc3e7ef5a0c9be0c29011b2dab2a2ce3320fa2ecbdfd7b9ef379c4e091d75a4da18d0ec49691aba2f06515f61ffd794 + c809d172a03d06ec86481d221bb086f15d90f8c31ea30f26dd61a86f19633a4bf04d25b4fe9f9ddfdf3876079ab671ffd7c483bf3d991677fd1c970817e4bd2b + + 1569500166 + 123 + 125 + + + 49e704998e8c9e154ad355be4c58476c0cbc298d4804c268ef2102995b32e14be16e9ac1f785338ce2dd1373574b356705c306efd37d52824436792b812a5e3b + c1ca53d1a6f4ca3c94decb035a9a5e82bbab46da4e4322be85fdea10c1f87c6ebf490a5d71dc7fb4718a2ce14ce21d93b8c4e8a7bccd77e4b3d2057c6019899c + + 1569500166 + 123 + 121 + + diff --git a/tests/test_gather.py b/tests/test_gather.py index db824240..4d9ac411 100644 --- a/tests/test_gather.py +++ b/tests/test_gather.py @@ -1856,4 +1856,169 @@ class DNFDepsolvingTestCase(DepsolvingBase, unittest.TestCase): super(DNFDepsolvingTestCase, self).test_firefox_selfhosting_with_krb5_lookaside() self.assertFlags("dummy-krb5-devel-1.10-5.x86_64", [PkgFlag.lookaside]) self.assertFlags("dummy-krb5-1.10-5.src", [PkgFlag.lookaside]) - self.assertFlags("dummy-krb5-debuginfo-1.10-5.x86_64", [PkgFlag.lookaside]) + + def test_only_required_debuginfo_pulled_in(self): + self.repo = os.path.join(os.path.dirname(__file__), "fixtures/repos/debuginfo") + packages = ["libuser.+", "python-libuser"] + + pkg_map = self.go(packages, None) + + self.assertItemsEqual(pkg_map["rpm"], [ + "libuser-1-1.i686.rpm", + "python-libuser-1-1.x86_64.rpm", + ]) + self.assertItemsEqual(pkg_map["srpm"], [ + "libuser-1-1.src.rpm", + ]) + self.assertItemsEqual(pkg_map["debuginfo"], [ + "libuser-debuginfo-1-1.i686.rpm", + "python-libuser-debuginfo-1-1.x86_64.rpm", + ]) + + def test_skype(self): + packages = [ + "dummy-skype", + ] + pkg_map = self.go(packages, None, greedy="build", fulltree=False, arch="x86_64") + + self.assertItemsEqual(pkg_map["rpm"], [ + "dummy-skype-4.2.0.13-1.i586.rpm", + "dummy-basesystem-10.0-6.noarch.rpm", + "dummy-filesystem-4.2.37-6.x86_64.rpm", + "dummy-glibc-common-2.14-5.x86_64.rpm", + "dummy-glibc-2.14-5.i686.rpm", + ]) + # no SRPM for skype + self.assertItemsEqual(pkg_map["srpm"], [ + "dummy-basesystem-10.0-6.src.rpm", + "dummy-glibc-2.14-5.src.rpm", + "dummy-filesystem-4.2.37-6.src.rpm", + ]) + self.assertItemsEqual(pkg_map["debuginfo"], [ + "dummy-glibc-debuginfo-common-2.14-5.i686.rpm", + "dummy-glibc-debuginfo-2.14-5.i686.rpm", + ]) + + def test_bash_multilib_exclude_debuginfo(self): + packages = [ + 'dummy-bash.+', + '-dummy-bash-debuginfo', + '-dummy-bash-debugsource', + ] + pkg_map = self.go(packages, None, greedy="none") + + self.assertItemsEqual(pkg_map["rpm"], [ + "dummy-basesystem-10.0-6.noarch.rpm", + "dummy-bash-4.2.37-6.i686.rpm", + "dummy-filesystem-4.2.37-6.x86_64.rpm", + "dummy-glibc-2.14-5.i686.rpm", + "dummy-glibc-common-2.14-5.x86_64.rpm", + ]) + self.assertItemsEqual(pkg_map["srpm"], [ + "dummy-basesystem-10.0-6.src.rpm", + "dummy-bash-4.2.37-6.src.rpm", + "dummy-filesystem-4.2.37-6.src.rpm", + "dummy-glibc-2.14-5.src.rpm", + ]) + self.assertItemsEqual(pkg_map["debuginfo"], [ + "dummy-glibc-debuginfo-2.14-5.i686.rpm", + "dummy-glibc-debuginfo-common-2.14-5.i686.rpm", + ]) + + def test_exclude_wildcards(self): + packages = [ + "dummy-bash", + "-dummy-bas*", + "dummy-glibc", + ] + pkg_map = self.go(packages, None, + greedy="none", nodeps=True, fulltree=True) + + # neither dummy-bash or dummy-basesystem is pulled in + self.assertItemsEqual(pkg_map["rpm"], [ + "dummy-glibc-2.14-5.x86_64.rpm", + "dummy-glibc-common-2.14-5.x86_64.rpm", + "dummy-nscd-2.14-5.x86_64.rpm", + ]) + self.assertItemsEqual(pkg_map["srpm"], [ + "dummy-glibc-2.14-5.src.rpm", + ]) + self.assertItemsEqual(pkg_map["debuginfo"], [ + "dummy-glibc-debuginfo-2.14-5.x86_64.rpm", + ]) + + def test_bash_multilib_exclude_source(self): + packages = [ + "dummy-bash.+", + "-dummy-bash.src", + ] + pkg_map = self.go(packages, None, greedy="none") + + self.assertNotIn("dummy-bash-4.2.37-6.src.rpm", pkg_map["srpm"]) + + self.assertItemsEqual(pkg_map["rpm"], [ + "dummy-basesystem-10.0-6.noarch.rpm", + "dummy-bash-4.2.37-6.i686.rpm", + "dummy-filesystem-4.2.37-6.x86_64.rpm", + "dummy-glibc-2.14-5.i686.rpm", + "dummy-glibc-common-2.14-5.x86_64.rpm", + ]) + self.assertItemsEqual(pkg_map["srpm"], [ + "dummy-basesystem-10.0-6.src.rpm", + "dummy-filesystem-4.2.37-6.src.rpm", + "dummy-glibc-2.14-5.src.rpm", + ]) + self.assertItemsEqual(pkg_map["debuginfo"], [ + "dummy-bash-debuginfo-4.2.37-6.i686.rpm", + "dummy-bash-debugsource-4.2.37-6.i686.rpm", + "dummy-glibc-debuginfo-2.14-5.i686.rpm", + "dummy-glibc-debuginfo-common-2.14-5.i686.rpm", + ]) + + def test_ipw3945_kmod(self): + # every package name is different + packages = [ + "dummy-kmod-ipw3945", + ] + pkg_map = self.go(packages, None, greedy="none", fulltree=True) + + self.assertItemsEqual(pkg_map["rpm"], [ + "dummy-kmod-ipw3945-1.2.0-4.20.x86_64.rpm", # Important + "dummy-kmod-ipw3945-xen-1.2.0-4.20.x86_64.rpm", + ]) + self.assertItemsEqual(pkg_map["srpm"], [ + "dummy-ipw3945-kmod-1.2.0-4.20.src.rpm", + ]) + self.assertItemsEqual(pkg_map["debuginfo"], []) + + def test_multilib_method_devel_runtime(self): + packages = [ + "dummy-lvm2-devel", + ] + pkg_map = self.go(packages, None, greedy="none", fulltree=False, + multilib_methods=["devel", "runtime"]) + + self.assertItemsEqual(pkg_map["rpm"], [ + "dummy-basesystem-10.0-6.noarch.rpm", + "dummy-filesystem-4.2.37-6.x86_64.rpm", + "dummy-glibc-2.14-5.x86_64.rpm", + "dummy-glibc-2.14-5.i686.rpm", + "dummy-glibc-common-2.14-5.x86_64.rpm", + "dummy-lvm2-2.02.84-4.x86_64.rpm", + "dummy-lvm2-devel-2.02.84-4.i686.rpm", # Important + "dummy-lvm2-devel-2.02.84-4.x86_64.rpm", # Important + "dummy-lvm2-libs-2.02.84-4.x86_64.rpm", + ]) + self.assertItemsEqual(pkg_map["srpm"], [ + "dummy-basesystem-10.0-6.src.rpm", + "dummy-filesystem-4.2.37-6.src.rpm", + "dummy-glibc-2.14-5.src.rpm", + "dummy-lvm2-2.02.84-4.src.rpm", + ]) + self.assertItemsEqual(pkg_map["debuginfo"], [ + "dummy-glibc-debuginfo-2.14-5.x86_64.rpm", + "dummy-glibc-debuginfo-common-2.14-5.x86_64.rpm", + "dummy-glibc-debuginfo-2.14-5.i686.rpm", + "dummy-glibc-debuginfo-common-2.14-5.i686.rpm", + "dummy-lvm2-debuginfo-2.02.84-4.x86_64.rpm", + ])