gather: Resolve dependencies of debug packages

Until now, the behaviour was that all debuginfo from a build would be
included if at least one package with the same arch was included.

This resulted in many debuginfo packages being included even though
their corresponding package was not present.

With this patch, we will always pull in debugsource, but foo-debuginfo
will only be included if foo is included for the same arch. As a
consequence, it is necessary to resolve dependencies of debuginfo
packages. There are cases where foo-debuginfo needs foo-debuginfo-common
for example.

This change means that DNF and YUM backends are no longer identical in
the output. The tests where this is demonstrated are duplicated and
their results are modified.

JIRA: COMPOSE-3823
Signed-off-by: Lubomír Sedlář <lsedlar@redhat.com>

pungi/gather_dnf.py

@@ -220,7 +220,7 @@ class Gather(GatherBase):
     def _set_flag(self, pkg, *flags):
         self.result_package_flags.setdefault(pkg, set()).update(flags)
 
-    def _get_best_package(self, package_list, pkg=None, req=None):
+    def _get_best_package(self, package_list, pkg=None, req=None, debuginfo=False):
         if not package_list:
             return []
 
@@ -245,27 +245,31 @@ class Gather(GatherBase):
             ):
                 all_pkgs.append(pkg)
 
-        native_pkgs = self.q_native_binary_packages.filter(pkg=all_pkgs).apply()
+        if not debuginfo:
-        multilib_pkgs = self.q_multilib_binary_packages.filter(pkg=all_pkgs).apply()
+            native_pkgs = set(self.q_native_binary_packages.filter(pkg=all_pkgs).apply())
+            multilib_pkgs = set(self.q_multilib_binary_packages.filter(pkg=all_pkgs).apply())
+        else:
+            native_pkgs = set(self.q_native_debug_packages.filter(pkg=all_pkgs).apply())
+            multilib_pkgs = set(self.q_multilib_debug_packages.filter(pkg=all_pkgs).apply())
 
         result = set()
 
         # try seen native packages first
-        seen_pkgs = set(native_pkgs) & self.result_binary_packages
+        seen_pkgs = native_pkgs & self.result_binary_packages
         if seen_pkgs:
             result = seen_pkgs
 
         # then try seen multilib packages
         if not result:
-            seen_pkgs = set(multilib_pkgs) & self.result_binary_packages
+            seen_pkgs = multilib_pkgs & self.result_binary_packages
             if seen_pkgs:
                 result = seen_pkgs
 
         if not result:
-            result = set(native_pkgs)
+            result = native_pkgs
 
         if not result:
-            result = set(multilib_pkgs)
+            result = multilib_pkgs
 
         if not result:
             return []
@@ -307,13 +311,14 @@ class Gather(GatherBase):
                 if i.sourcerpm.rsplit('-', 2)[0] in self.opts.fulltree_excludes:
                     self._set_flag(i, PkgFlag.fulltree_exclude)
 
-    def _get_package_deps(self, pkg):
+    def _get_package_deps(self, pkg, debuginfo=False):
         """Return all direct (1st level) deps for a package.
 
         The return value is a set of tuples (pkg, reldep). Each package is
         tagged with the particular reldep that pulled it in. Requires_pre and
         _post are not distinguished.
         """
+        queue = self.q_debug_packages if debuginfo else self.q_binary_packages
         assert pkg is not None
         result = set()
 
@@ -329,7 +334,7 @@ class Gather(GatherBase):
             + getattr(pkg, 'requires_post', [])
         )
 
-        q = self.q_binary_packages.filter(provides=requires).apply()
+        q = queue.filter(provides=requires).apply()
         for req in requires:
             deps = self.finished_get_package_deps_reqs.setdefault(str(req), set())
             if deps:
@@ -339,7 +344,7 @@ class Gather(GatherBase):
             # TODO: need query also debuginfo
             deps = q.filter(provides=req)
             if deps:
-                deps = self._get_best_package(deps, req=req)
+                deps = self._get_best_package(deps, req=req, debuginfo=debuginfo)
                 self.finished_get_package_deps_reqs[str(req)].update(deps)
                 result.update((dep, req) for dep in deps)
             else:
@@ -510,6 +515,34 @@ class Gather(GatherBase):
 
         return added
 
+    @Profiler("Gather.add_debug_package_deps()")
+    def add_debug_package_deps(self):
+        added = set()
+
+        if not self.opts.resolve_deps:
+            return added
+
+        for pkg in self.result_debug_packages.copy():
+
+            if pkg not in self.finished_add_debug_package_deps:
+                deps = self._get_package_deps(pkg, debuginfo=True)
+                for i, req in deps:
+                    if i in self.result_binary_packages:
+                        # The dependency is already satisfied by binary package
+                        continue
+                    if i not in self.result_debug_packages:
+                        self._add_packages(
+                            [i],
+                            pulled_by=pkg,
+                            req=req,
+                            reason="debug-dep",
+                            dest=self.result_debug_packages,
+                        )
+                        added.add(i)
+                self.finished_add_debug_package_deps[pkg] = deps
+
+        return added
+
     @Profiler("Gather.add_conditional_packages()")
     def add_conditional_packages(self):
         """
@@ -643,17 +676,20 @@ class Gather(GatherBase):
             debug_pkgs = []
             pkg_in_lookaside = pkg.repoid in self.opts.lookaside_repos
             for i in candidates:
-                if pkg.arch == 'noarch' and i.arch != 'noarch':
+                if pkg.arch != i.arch:
-                    # If the package is noarch, we will only pull debuginfo if
+                    continue
-                    # it's noarch as well. This covers mingw use case, but
+                if "-debugsource" not in i.name and i.name != "%s-debuginfo" % pkg.name:
-                    # means we don't for example pull debuginfo just because of
+                    # If it's not debugsource package or does not match name of
-                    # -doc subpackage.
+                    # the package, we don't want it in.
                     continue
                 if i.repoid in self.opts.lookaside_repos or pkg_in_lookaside:
                     self._set_flag(i, PkgFlag.lookaside)
                 if i not in self.result_debug_packages:
                     added.add(i)
                     debug_pkgs.append(i)
+                    self.logger.debug(
+                        "Added debuginfo %s (for %s, repo: %s)" % (i, pkg, i.repo.id)
+                    )
 
             self.finished_add_debug_packages[pkg] = debug_pkgs
             self.result_debug_packages.update(debug_pkgs)
@@ -834,7 +870,9 @@ class Gather(GatherBase):
 
             if self.log_count('DEBUG PACKAGES', self.add_debug_packages):
                 continue
-            # TODO: debug deps
+
+            if self.log_count("DEBUG DEPS", self.add_debug_package_deps):
+                continue
 
             if self.log_count('FULLTREE', self.add_fulltree_packages):
                 continue

tests/fixtures/repos/debuginfo/repodata/repomd.xml

@@ -0,0 +1,28 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<repomd xmlns="http://linux.duke.edu/metadata/repo" xmlns:rpm="http://linux.duke.edu/metadata/rpm">
+  <revision>1569500166</revision>
+  <data type="primary">
+    <checksum type="sha512">d7043643dc30ad24861d17329ae0ea638cc0659af7b586cb4482eff7d47949c5a8066fc03fa5df98efdd47d8f077628eb60e2fca5f1b24e1e8a3e84dc5a2042c</checksum>
+    <open-checksum type="sha512">477adda50364b5532d161a1d5fdeb6ea36cea3282d1f8d24de1b66108a5564f3ce47c573de5227d8f21a79cdab5012f5e30b03d5469fce0db98402bcf3635810</open-checksum>
+    <location href="repodata/primary.xml.gz"/>
+    <timestamp>1569500166</timestamp>
+    <size>578</size>
+    <open-size>7159</open-size>
+  </data>
+  <data type="filelists">
+    <checksum type="sha512">8876e1c8b97828e5e5d2ce35f8722a6f5cc3e7ef5a0c9be0c29011b2dab2a2ce3320fa2ecbdfd7b9ef379c4e091d75a4da18d0ec49691aba2f06515f61ffd794</checksum>
+    <open-checksum type="sha512">c809d172a03d06ec86481d221bb086f15d90f8c31ea30f26dd61a86f19633a4bf04d25b4fe9f9ddfdf3876079ab671ffd7c483bf3d991677fd1c970817e4bd2b</open-checksum>
+    <location href="repodata/filelists.xml.gz"/>
+    <timestamp>1569500166</timestamp>
+    <size>123</size>
+    <open-size>125</open-size>
+  </data>
+  <data type="other">
+    <checksum type="sha512">49e704998e8c9e154ad355be4c58476c0cbc298d4804c268ef2102995b32e14be16e9ac1f785338ce2dd1373574b356705c306efd37d52824436792b812a5e3b</checksum>
+    <open-checksum type="sha512">c1ca53d1a6f4ca3c94decb035a9a5e82bbab46da4e4322be85fdea10c1f87c6ebf490a5d71dc7fb4718a2ce14ce21d93b8c4e8a7bccd77e4b3d2057c6019899c</open-checksum>
+    <location href="repodata/other.xml.gz"/>
+    <timestamp>1569500166</timestamp>
+    <size>123</size>
+    <open-size>121</open-size>
+  </data>
+</repomd>

tests/test_gather.py

@@ -1856,4 +1856,169 @@ class DNFDepsolvingTestCase(DepsolvingBase, unittest.TestCase):
         super(DNFDepsolvingTestCase, self).test_firefox_selfhosting_with_krb5_lookaside()
         self.assertFlags("dummy-krb5-devel-1.10-5.x86_64", [PkgFlag.lookaside])
         self.assertFlags("dummy-krb5-1.10-5.src", [PkgFlag.lookaside])
-        self.assertFlags("dummy-krb5-debuginfo-1.10-5.x86_64", [PkgFlag.lookaside])
+
+    def test_only_required_debuginfo_pulled_in(self):
+        self.repo = os.path.join(os.path.dirname(__file__), "fixtures/repos/debuginfo")
+        packages = ["libuser.+", "python-libuser"]
+
+        pkg_map = self.go(packages, None)
+
+        self.assertItemsEqual(pkg_map["rpm"], [
+            "libuser-1-1.i686.rpm",
+            "python-libuser-1-1.x86_64.rpm",
+        ])
+        self.assertItemsEqual(pkg_map["srpm"], [
+            "libuser-1-1.src.rpm",
+        ])
+        self.assertItemsEqual(pkg_map["debuginfo"], [
+            "libuser-debuginfo-1-1.i686.rpm",
+            "python-libuser-debuginfo-1-1.x86_64.rpm",
+        ])
+
+    def test_skype(self):
+        packages = [
+            "dummy-skype",
+        ]
+        pkg_map = self.go(packages, None, greedy="build", fulltree=False, arch="x86_64")
+
+        self.assertItemsEqual(pkg_map["rpm"], [
+            "dummy-skype-4.2.0.13-1.i586.rpm",
+            "dummy-basesystem-10.0-6.noarch.rpm",
+            "dummy-filesystem-4.2.37-6.x86_64.rpm",
+            "dummy-glibc-common-2.14-5.x86_64.rpm",
+            "dummy-glibc-2.14-5.i686.rpm",
+        ])
+        # no SRPM for skype
+        self.assertItemsEqual(pkg_map["srpm"], [
+            "dummy-basesystem-10.0-6.src.rpm",
+            "dummy-glibc-2.14-5.src.rpm",
+            "dummy-filesystem-4.2.37-6.src.rpm",
+        ])
+        self.assertItemsEqual(pkg_map["debuginfo"], [
+            "dummy-glibc-debuginfo-common-2.14-5.i686.rpm",
+            "dummy-glibc-debuginfo-2.14-5.i686.rpm",
+        ])
+
+    def test_bash_multilib_exclude_debuginfo(self):
+        packages = [
+            'dummy-bash.+',
+            '-dummy-bash-debuginfo',
+            '-dummy-bash-debugsource',
+        ]
+        pkg_map = self.go(packages, None, greedy="none")
+
+        self.assertItemsEqual(pkg_map["rpm"], [
+            "dummy-basesystem-10.0-6.noarch.rpm",
+            "dummy-bash-4.2.37-6.i686.rpm",
+            "dummy-filesystem-4.2.37-6.x86_64.rpm",
+            "dummy-glibc-2.14-5.i686.rpm",
+            "dummy-glibc-common-2.14-5.x86_64.rpm",
+        ])
+        self.assertItemsEqual(pkg_map["srpm"], [
+            "dummy-basesystem-10.0-6.src.rpm",
+            "dummy-bash-4.2.37-6.src.rpm",
+            "dummy-filesystem-4.2.37-6.src.rpm",
+            "dummy-glibc-2.14-5.src.rpm",
+        ])
+        self.assertItemsEqual(pkg_map["debuginfo"], [
+            "dummy-glibc-debuginfo-2.14-5.i686.rpm",
+            "dummy-glibc-debuginfo-common-2.14-5.i686.rpm",
+        ])
+
+    def test_exclude_wildcards(self):
+        packages = [
+            "dummy-bash",
+            "-dummy-bas*",
+            "dummy-glibc",
+        ]
+        pkg_map = self.go(packages, None,
+                          greedy="none", nodeps=True, fulltree=True)
+
+        # neither dummy-bash or dummy-basesystem is pulled in
+        self.assertItemsEqual(pkg_map["rpm"], [
+            "dummy-glibc-2.14-5.x86_64.rpm",
+            "dummy-glibc-common-2.14-5.x86_64.rpm",
+            "dummy-nscd-2.14-5.x86_64.rpm",
+        ])
+        self.assertItemsEqual(pkg_map["srpm"], [
+            "dummy-glibc-2.14-5.src.rpm",
+        ])
+        self.assertItemsEqual(pkg_map["debuginfo"], [
+            "dummy-glibc-debuginfo-2.14-5.x86_64.rpm",
+        ])
+
+    def test_bash_multilib_exclude_source(self):
+        packages = [
+            "dummy-bash.+",
+            "-dummy-bash.src",
+        ]
+        pkg_map = self.go(packages, None, greedy="none")
+
+        self.assertNotIn("dummy-bash-4.2.37-6.src.rpm", pkg_map["srpm"])
+
+        self.assertItemsEqual(pkg_map["rpm"], [
+            "dummy-basesystem-10.0-6.noarch.rpm",
+            "dummy-bash-4.2.37-6.i686.rpm",
+            "dummy-filesystem-4.2.37-6.x86_64.rpm",
+            "dummy-glibc-2.14-5.i686.rpm",
+            "dummy-glibc-common-2.14-5.x86_64.rpm",
+        ])
+        self.assertItemsEqual(pkg_map["srpm"], [
+            "dummy-basesystem-10.0-6.src.rpm",
+            "dummy-filesystem-4.2.37-6.src.rpm",
+            "dummy-glibc-2.14-5.src.rpm",
+        ])
+        self.assertItemsEqual(pkg_map["debuginfo"], [
+            "dummy-bash-debuginfo-4.2.37-6.i686.rpm",
+            "dummy-bash-debugsource-4.2.37-6.i686.rpm",
+            "dummy-glibc-debuginfo-2.14-5.i686.rpm",
+            "dummy-glibc-debuginfo-common-2.14-5.i686.rpm",
+        ])
+
+    def test_ipw3945_kmod(self):
+        # every package name is different
+        packages = [
+            "dummy-kmod-ipw3945",
+        ]
+        pkg_map = self.go(packages, None, greedy="none", fulltree=True)
+
+        self.assertItemsEqual(pkg_map["rpm"], [
+            "dummy-kmod-ipw3945-1.2.0-4.20.x86_64.rpm",         # Important
+            "dummy-kmod-ipw3945-xen-1.2.0-4.20.x86_64.rpm",
+        ])
+        self.assertItemsEqual(pkg_map["srpm"], [
+            "dummy-ipw3945-kmod-1.2.0-4.20.src.rpm",
+        ])
+        self.assertItemsEqual(pkg_map["debuginfo"], [])
+
+    def test_multilib_method_devel_runtime(self):
+        packages = [
+            "dummy-lvm2-devel",
+        ]
+        pkg_map = self.go(packages, None, greedy="none", fulltree=False,
+                          multilib_methods=["devel", "runtime"])
+
+        self.assertItemsEqual(pkg_map["rpm"], [
+            "dummy-basesystem-10.0-6.noarch.rpm",
+            "dummy-filesystem-4.2.37-6.x86_64.rpm",
+            "dummy-glibc-2.14-5.x86_64.rpm",
+            "dummy-glibc-2.14-5.i686.rpm",
+            "dummy-glibc-common-2.14-5.x86_64.rpm",
+            "dummy-lvm2-2.02.84-4.x86_64.rpm",
+            "dummy-lvm2-devel-2.02.84-4.i686.rpm",          # Important
+            "dummy-lvm2-devel-2.02.84-4.x86_64.rpm",        # Important
+            "dummy-lvm2-libs-2.02.84-4.x86_64.rpm",
+        ])
+        self.assertItemsEqual(pkg_map["srpm"], [
+            "dummy-basesystem-10.0-6.src.rpm",
+            "dummy-filesystem-4.2.37-6.src.rpm",
+            "dummy-glibc-2.14-5.src.rpm",
+            "dummy-lvm2-2.02.84-4.src.rpm",
+        ])
+        self.assertItemsEqual(pkg_map["debuginfo"], [
+            "dummy-glibc-debuginfo-2.14-5.x86_64.rpm",
+            "dummy-glibc-debuginfo-common-2.14-5.x86_64.rpm",
+            "dummy-glibc-debuginfo-2.14-5.i686.rpm",
+            "dummy-glibc-debuginfo-common-2.14-5.i686.rpm",
+            "dummy-lvm2-debuginfo-2.02.84-4.x86_64.rpm",
+        ])