diff --git a/pungi/gather_dnf.py b/pungi/gather_dnf.py index b8cc0bfd..714c8d2c 100644 --- a/pungi/gather_dnf.py +++ b/pungi/gather_dnf.py @@ -220,7 +220,7 @@ class Gather(GatherBase): def _set_flag(self, pkg, *flags): self.result_package_flags.setdefault(pkg, set()).update(flags) - def _get_best_package(self, package_list, pkg=None, req=None): + def _get_best_package(self, package_list, pkg=None, req=None, debuginfo=False): if not package_list: return [] @@ -245,27 +245,31 @@ class Gather(GatherBase): ): all_pkgs.append(pkg) - native_pkgs = self.q_native_binary_packages.filter(pkg=all_pkgs).apply() - multilib_pkgs = self.q_multilib_binary_packages.filter(pkg=all_pkgs).apply() + if not debuginfo: + native_pkgs = set(self.q_native_binary_packages.filter(pkg=all_pkgs).apply()) + multilib_pkgs = set(self.q_multilib_binary_packages.filter(pkg=all_pkgs).apply()) + else: + native_pkgs = set(self.q_native_debug_packages.filter(pkg=all_pkgs).apply()) + multilib_pkgs = set(self.q_multilib_debug_packages.filter(pkg=all_pkgs).apply()) result = set() # try seen native packages first - seen_pkgs = set(native_pkgs) & self.result_binary_packages + seen_pkgs = native_pkgs & self.result_binary_packages if seen_pkgs: result = seen_pkgs # then try seen multilib packages if not result: - seen_pkgs = set(multilib_pkgs) & self.result_binary_packages + seen_pkgs = multilib_pkgs & self.result_binary_packages if seen_pkgs: result = seen_pkgs if not result: - result = set(native_pkgs) + result = native_pkgs if not result: - result = set(multilib_pkgs) + result = multilib_pkgs if not result: return [] @@ -307,13 +311,14 @@ class Gather(GatherBase): if i.sourcerpm.rsplit('-', 2)[0] in self.opts.fulltree_excludes: self._set_flag(i, PkgFlag.fulltree_exclude) - def _get_package_deps(self, pkg): + def _get_package_deps(self, pkg, debuginfo=False): """Return all direct (1st level) deps for a package. The return value is a set of tuples (pkg, reldep). Each package is tagged with the particular reldep that pulled it in. Requires_pre and _post are not distinguished. """ + queue = self.q_debug_packages if debuginfo else self.q_binary_packages assert pkg is not None result = set() @@ -329,7 +334,7 @@ class Gather(GatherBase): + getattr(pkg, 'requires_post', []) ) - q = self.q_binary_packages.filter(provides=requires).apply() + q = queue.filter(provides=requires).apply() for req in requires: deps = self.finished_get_package_deps_reqs.setdefault(str(req), set()) if deps: @@ -339,7 +344,7 @@ class Gather(GatherBase): # TODO: need query also debuginfo deps = q.filter(provides=req) if deps: - deps = self._get_best_package(deps, req=req) + deps = self._get_best_package(deps, req=req, debuginfo=debuginfo) self.finished_get_package_deps_reqs[str(req)].update(deps) result.update((dep, req) for dep in deps) else: @@ -510,6 +515,34 @@ class Gather(GatherBase): return added + @Profiler("Gather.add_debug_package_deps()") + def add_debug_package_deps(self): + added = set() + + if not self.opts.resolve_deps: + return added + + for pkg in self.result_debug_packages.copy(): + + if pkg not in self.finished_add_debug_package_deps: + deps = self._get_package_deps(pkg, debuginfo=True) + for i, req in deps: + if i in self.result_binary_packages: + # The dependency is already satisfied by binary package + continue + if i not in self.result_debug_packages: + self._add_packages( + [i], + pulled_by=pkg, + req=req, + reason="debug-dep", + dest=self.result_debug_packages, + ) + added.add(i) + self.finished_add_debug_package_deps[pkg] = deps + + return added + @Profiler("Gather.add_conditional_packages()") def add_conditional_packages(self): """ @@ -643,17 +676,20 @@ class Gather(GatherBase): debug_pkgs = [] pkg_in_lookaside = pkg.repoid in self.opts.lookaside_repos for i in candidates: - if pkg.arch == 'noarch' and i.arch != 'noarch': - # If the package is noarch, we will only pull debuginfo if - # it's noarch as well. This covers mingw use case, but - # means we don't for example pull debuginfo just because of - # -doc subpackage. + if pkg.arch != i.arch: + continue + if "-debugsource" not in i.name and i.name != "%s-debuginfo" % pkg.name: + # If it's not debugsource package or does not match name of + # the package, we don't want it in. continue if i.repoid in self.opts.lookaside_repos or pkg_in_lookaside: self._set_flag(i, PkgFlag.lookaside) if i not in self.result_debug_packages: added.add(i) debug_pkgs.append(i) + self.logger.debug( + "Added debuginfo %s (for %s, repo: %s)" % (i, pkg, i.repo.id) + ) self.finished_add_debug_packages[pkg] = debug_pkgs self.result_debug_packages.update(debug_pkgs) @@ -834,7 +870,9 @@ class Gather(GatherBase): if self.log_count('DEBUG PACKAGES', self.add_debug_packages): continue - # TODO: debug deps + + if self.log_count("DEBUG DEPS", self.add_debug_package_deps): + continue if self.log_count('FULLTREE', self.add_fulltree_packages): continue diff --git a/tests/fixtures/repos/debuginfo/repodata/filelists.xml.gz b/tests/fixtures/repos/debuginfo/repodata/filelists.xml.gz new file mode 100644 index 00000000..40471d4c Binary files /dev/null and b/tests/fixtures/repos/debuginfo/repodata/filelists.xml.gz differ diff --git a/tests/fixtures/repos/debuginfo/repodata/other.xml.gz b/tests/fixtures/repos/debuginfo/repodata/other.xml.gz new file mode 100644 index 00000000..89d6635d Binary files /dev/null and b/tests/fixtures/repos/debuginfo/repodata/other.xml.gz differ diff --git a/tests/fixtures/repos/debuginfo/repodata/primary.xml.gz b/tests/fixtures/repos/debuginfo/repodata/primary.xml.gz new file mode 100644 index 00000000..fc625dd2 Binary files /dev/null and b/tests/fixtures/repos/debuginfo/repodata/primary.xml.gz differ diff --git a/tests/fixtures/repos/debuginfo/repodata/repomd.xml b/tests/fixtures/repos/debuginfo/repodata/repomd.xml new file mode 100644 index 00000000..9bdc1e11 --- /dev/null +++ b/tests/fixtures/repos/debuginfo/repodata/repomd.xml @@ -0,0 +1,28 @@ + + + 1569500166 + + d7043643dc30ad24861d17329ae0ea638cc0659af7b586cb4482eff7d47949c5a8066fc03fa5df98efdd47d8f077628eb60e2fca5f1b24e1e8a3e84dc5a2042c + 477adda50364b5532d161a1d5fdeb6ea36cea3282d1f8d24de1b66108a5564f3ce47c573de5227d8f21a79cdab5012f5e30b03d5469fce0db98402bcf3635810 + + 1569500166 + 578 + 7159 + + + 8876e1c8b97828e5e5d2ce35f8722a6f5cc3e7ef5a0c9be0c29011b2dab2a2ce3320fa2ecbdfd7b9ef379c4e091d75a4da18d0ec49691aba2f06515f61ffd794 + c809d172a03d06ec86481d221bb086f15d90f8c31ea30f26dd61a86f19633a4bf04d25b4fe9f9ddfdf3876079ab671ffd7c483bf3d991677fd1c970817e4bd2b + + 1569500166 + 123 + 125 + + + 49e704998e8c9e154ad355be4c58476c0cbc298d4804c268ef2102995b32e14be16e9ac1f785338ce2dd1373574b356705c306efd37d52824436792b812a5e3b + c1ca53d1a6f4ca3c94decb035a9a5e82bbab46da4e4322be85fdea10c1f87c6ebf490a5d71dc7fb4718a2ce14ce21d93b8c4e8a7bccd77e4b3d2057c6019899c + + 1569500166 + 123 + 121 + + diff --git a/tests/test_gather.py b/tests/test_gather.py index db824240..4d9ac411 100644 --- a/tests/test_gather.py +++ b/tests/test_gather.py @@ -1856,4 +1856,169 @@ class DNFDepsolvingTestCase(DepsolvingBase, unittest.TestCase): super(DNFDepsolvingTestCase, self).test_firefox_selfhosting_with_krb5_lookaside() self.assertFlags("dummy-krb5-devel-1.10-5.x86_64", [PkgFlag.lookaside]) self.assertFlags("dummy-krb5-1.10-5.src", [PkgFlag.lookaside]) - self.assertFlags("dummy-krb5-debuginfo-1.10-5.x86_64", [PkgFlag.lookaside]) + + def test_only_required_debuginfo_pulled_in(self): + self.repo = os.path.join(os.path.dirname(__file__), "fixtures/repos/debuginfo") + packages = ["libuser.+", "python-libuser"] + + pkg_map = self.go(packages, None) + + self.assertItemsEqual(pkg_map["rpm"], [ + "libuser-1-1.i686.rpm", + "python-libuser-1-1.x86_64.rpm", + ]) + self.assertItemsEqual(pkg_map["srpm"], [ + "libuser-1-1.src.rpm", + ]) + self.assertItemsEqual(pkg_map["debuginfo"], [ + "libuser-debuginfo-1-1.i686.rpm", + "python-libuser-debuginfo-1-1.x86_64.rpm", + ]) + + def test_skype(self): + packages = [ + "dummy-skype", + ] + pkg_map = self.go(packages, None, greedy="build", fulltree=False, arch="x86_64") + + self.assertItemsEqual(pkg_map["rpm"], [ + "dummy-skype-4.2.0.13-1.i586.rpm", + "dummy-basesystem-10.0-6.noarch.rpm", + "dummy-filesystem-4.2.37-6.x86_64.rpm", + "dummy-glibc-common-2.14-5.x86_64.rpm", + "dummy-glibc-2.14-5.i686.rpm", + ]) + # no SRPM for skype + self.assertItemsEqual(pkg_map["srpm"], [ + "dummy-basesystem-10.0-6.src.rpm", + "dummy-glibc-2.14-5.src.rpm", + "dummy-filesystem-4.2.37-6.src.rpm", + ]) + self.assertItemsEqual(pkg_map["debuginfo"], [ + "dummy-glibc-debuginfo-common-2.14-5.i686.rpm", + "dummy-glibc-debuginfo-2.14-5.i686.rpm", + ]) + + def test_bash_multilib_exclude_debuginfo(self): + packages = [ + 'dummy-bash.+', + '-dummy-bash-debuginfo', + '-dummy-bash-debugsource', + ] + pkg_map = self.go(packages, None, greedy="none") + + self.assertItemsEqual(pkg_map["rpm"], [ + "dummy-basesystem-10.0-6.noarch.rpm", + "dummy-bash-4.2.37-6.i686.rpm", + "dummy-filesystem-4.2.37-6.x86_64.rpm", + "dummy-glibc-2.14-5.i686.rpm", + "dummy-glibc-common-2.14-5.x86_64.rpm", + ]) + self.assertItemsEqual(pkg_map["srpm"], [ + "dummy-basesystem-10.0-6.src.rpm", + "dummy-bash-4.2.37-6.src.rpm", + "dummy-filesystem-4.2.37-6.src.rpm", + "dummy-glibc-2.14-5.src.rpm", + ]) + self.assertItemsEqual(pkg_map["debuginfo"], [ + "dummy-glibc-debuginfo-2.14-5.i686.rpm", + "dummy-glibc-debuginfo-common-2.14-5.i686.rpm", + ]) + + def test_exclude_wildcards(self): + packages = [ + "dummy-bash", + "-dummy-bas*", + "dummy-glibc", + ] + pkg_map = self.go(packages, None, + greedy="none", nodeps=True, fulltree=True) + + # neither dummy-bash or dummy-basesystem is pulled in + self.assertItemsEqual(pkg_map["rpm"], [ + "dummy-glibc-2.14-5.x86_64.rpm", + "dummy-glibc-common-2.14-5.x86_64.rpm", + "dummy-nscd-2.14-5.x86_64.rpm", + ]) + self.assertItemsEqual(pkg_map["srpm"], [ + "dummy-glibc-2.14-5.src.rpm", + ]) + self.assertItemsEqual(pkg_map["debuginfo"], [ + "dummy-glibc-debuginfo-2.14-5.x86_64.rpm", + ]) + + def test_bash_multilib_exclude_source(self): + packages = [ + "dummy-bash.+", + "-dummy-bash.src", + ] + pkg_map = self.go(packages, None, greedy="none") + + self.assertNotIn("dummy-bash-4.2.37-6.src.rpm", pkg_map["srpm"]) + + self.assertItemsEqual(pkg_map["rpm"], [ + "dummy-basesystem-10.0-6.noarch.rpm", + "dummy-bash-4.2.37-6.i686.rpm", + "dummy-filesystem-4.2.37-6.x86_64.rpm", + "dummy-glibc-2.14-5.i686.rpm", + "dummy-glibc-common-2.14-5.x86_64.rpm", + ]) + self.assertItemsEqual(pkg_map["srpm"], [ + "dummy-basesystem-10.0-6.src.rpm", + "dummy-filesystem-4.2.37-6.src.rpm", + "dummy-glibc-2.14-5.src.rpm", + ]) + self.assertItemsEqual(pkg_map["debuginfo"], [ + "dummy-bash-debuginfo-4.2.37-6.i686.rpm", + "dummy-bash-debugsource-4.2.37-6.i686.rpm", + "dummy-glibc-debuginfo-2.14-5.i686.rpm", + "dummy-glibc-debuginfo-common-2.14-5.i686.rpm", + ]) + + def test_ipw3945_kmod(self): + # every package name is different + packages = [ + "dummy-kmod-ipw3945", + ] + pkg_map = self.go(packages, None, greedy="none", fulltree=True) + + self.assertItemsEqual(pkg_map["rpm"], [ + "dummy-kmod-ipw3945-1.2.0-4.20.x86_64.rpm", # Important + "dummy-kmod-ipw3945-xen-1.2.0-4.20.x86_64.rpm", + ]) + self.assertItemsEqual(pkg_map["srpm"], [ + "dummy-ipw3945-kmod-1.2.0-4.20.src.rpm", + ]) + self.assertItemsEqual(pkg_map["debuginfo"], []) + + def test_multilib_method_devel_runtime(self): + packages = [ + "dummy-lvm2-devel", + ] + pkg_map = self.go(packages, None, greedy="none", fulltree=False, + multilib_methods=["devel", "runtime"]) + + self.assertItemsEqual(pkg_map["rpm"], [ + "dummy-basesystem-10.0-6.noarch.rpm", + "dummy-filesystem-4.2.37-6.x86_64.rpm", + "dummy-glibc-2.14-5.x86_64.rpm", + "dummy-glibc-2.14-5.i686.rpm", + "dummy-glibc-common-2.14-5.x86_64.rpm", + "dummy-lvm2-2.02.84-4.x86_64.rpm", + "dummy-lvm2-devel-2.02.84-4.i686.rpm", # Important + "dummy-lvm2-devel-2.02.84-4.x86_64.rpm", # Important + "dummy-lvm2-libs-2.02.84-4.x86_64.rpm", + ]) + self.assertItemsEqual(pkg_map["srpm"], [ + "dummy-basesystem-10.0-6.src.rpm", + "dummy-filesystem-4.2.37-6.src.rpm", + "dummy-glibc-2.14-5.src.rpm", + "dummy-lvm2-2.02.84-4.src.rpm", + ]) + self.assertItemsEqual(pkg_map["debuginfo"], [ + "dummy-glibc-debuginfo-2.14-5.x86_64.rpm", + "dummy-glibc-debuginfo-common-2.14-5.x86_64.rpm", + "dummy-glibc-debuginfo-2.14-5.i686.rpm", + "dummy-glibc-debuginfo-common-2.14-5.i686.rpm", + "dummy-lvm2-debuginfo-2.02.84-4.x86_64.rpm", + ])