repoclosure: Extract logs from hybrid solver

There is no repoclosure that correctly understands modules. The best
thing we can offer is the errors reported by the depsolver.

JIRA: COMPOSE-2321
Signed-off-by: Lubomír Sedlář <lsedlar@redhat.com>

pungi/phases/gather/__init__.py

@@ -151,10 +151,7 @@ def get_parent_pkgs(arch, variant, result_dict):
     return result
 
 
-def gather_packages(compose, arch, variant, package_sets, fulltree_excludes=None):
+def get_gather_methods(compose, variant):
-    # multilib white/black-list is per-arch, common for all variants
-    multilib_whitelist = get_multilib_whitelist(compose, arch)
-    multilib_blacklist = get_multilib_blacklist(compose, arch)
     methods = compose.conf["gather_method"]
     global_method_name = methods
     if isinstance(methods, dict):
@@ -163,6 +160,14 @@ def gather_packages(compose, arch, variant, package_sets, fulltree_excludes=None
             global_method_name = None
         except IndexError:
             raise RuntimeError("Variant %s has no configured gather_method" % variant.uid)
+    return global_method_name, methods
+
+
+def gather_packages(compose, arch, variant, package_sets, fulltree_excludes=None):
+    # multilib white/black-list is per-arch, common for all variants
+    multilib_whitelist = get_multilib_whitelist(compose, arch)
+    multilib_blacklist = get_multilib_blacklist(compose, arch)
+    global_method_name, methods = get_gather_methods(compose, variant)
 
     msg = "Gathering packages (arch: %s, variant: %s)" % (arch, variant)
 

pungi/phases/test.py

@@ -14,6 +14,7 @@
 # along with this program; if not, see <https://gnu.org/licenses/>.
 
 
+import glob
 import os
 
 from kobo.shortcuts import run
@@ -21,7 +22,7 @@ from kobo.shortcuts import run
 from pungi.wrappers import repoclosure
 from pungi.arch import get_valid_arches
 from pungi.phases.base import PhaseBase
-from pungi.phases.gather import get_lookaside_repos
+from pungi.phases.gather import get_lookaside_repos, get_gather_methods
 from pungi.util import is_arch_multilib, failable, temp_dir, get_arch_variant_data
 
 
@@ -63,34 +64,42 @@ def run_repoclosure(compose):
             for i, lookaside_url in enumerate(get_lookaside_repos(compose, arch, variant)):
                 lookaside["lookaside-%s.%s-%s" % (variant.uid, arch, i)] = lookaside_url
 
-            cmd = repoclosure.get_repoclosure_cmd(backend=compose.conf['repoclosure_backend'],
+            logfile = compose.paths.log.log_file(arch, "repoclosure-%s" % variant)
-                                                  repos=repos, lookaside=lookaside, arch=arches)
+
-            # Use temp working directory directory as workaround for
+            try:
-            # https://bugzilla.redhat.com/show_bug.cgi?id=795137
+                _, methods = get_gather_methods(compose, variant)
-            with temp_dir(prefix='repoclosure_') as tmp_dir:
+                if methods == "hybrid":
-                # Ideally we would want show_cmd=True here to include the
+                    # Using hybrid solver, no repoclosure command is available.
-                # command in the logfile, but due to a bug in Kobo that would
+                    pattern = compose.paths.log.log_file(
-                # cause any error to be printed directly to stderr.
+                        arch, "hybrid-depsolver-%s-iter-*" % variant
-                #  https://github.com/release-engineering/kobo/pull/26
-                try:
-                    run(
-                        cmd,
-                        logfile=compose.paths.log.log_file(
-                            arch, "repoclosure-%s" % variant
-                        ),
-                        workdir=tmp_dir,
-                        show_cmd=True,
                     )
-                except RuntimeError as exc:
+                    fus_log = sorted(glob.glob(pattern))[-1]
-                    if conf and conf[-1] == 'fatal':
+                    repoclosure.extract_from_fus_log(fus_log, logfile)
-                        raise
+                else:
-                    else:
+                    _run_repoclosure_cmd(compose, repos, lookaside, arches, logfile)
-                        compose.log_warning('Repoclosure failed for %s.%s\n%s'
+            except RuntimeError as exc:
-                                            % (variant.uid, arch, exc))
+                if conf and conf[-1] == 'fatal':
+                    raise
+                else:
+                    compose.log_warning('Repoclosure failed for %s.%s\n%s'
+                                        % (variant.uid, arch, exc))
 
     compose.log_info("[DONE ] %s" % msg)
 
 
+def _run_repoclosure_cmd(compose, repos, lookaside, arches, logfile):
+    cmd = repoclosure.get_repoclosure_cmd(backend=compose.conf["repoclosure_backend"],
+                                          repos=repos, lookaside=lookaside, arch=arches)
+    # Use temp working directory directory as workaround for
+    # https://bugzilla.redhat.com/show_bug.cgi?id=795137
+    with temp_dir(prefix="repoclosure_") as tmp_dir:
+        # Ideally we would want show_cmd=True here to include the
+        # command in the logfile, but due to a bug in Kobo that would
+        # cause any error to be printed directly to stderr.
+        #  https://github.com/release-engineering/kobo/pull/26
+        run(cmd, logfile=logfile, workdir=tmp_dir, show_cmd=True)
+
+
 def check_image_sanity(compose):
     """
     Go through all images in manifest and make basic sanity tests on them. If

pungi/wrappers/repoclosure.py

@@ -57,3 +57,18 @@ def _to_url(path):
     if "://" not in path:
         return "file://%s" % os.path.abspath(path)
     return path
+
+
+def extract_from_fus_log(input, output):
+    """Extract depsolver error messages from fus log and write them to output
+    file.
+    """
+    has_error = False
+    with open(input) as fin:
+        with open(output, "w") as fout:
+            for line in fin:
+                if line.startswith("Problem ") or line.startswith(" "):
+                    fout.write(line)
+                    has_error = True
+    if has_error:
+        raise RuntimeError("Broken dependencies! See %s for details" % output)

tests/test_repoclosure_wrapper.py

@@ -1,10 +1,5 @@
 # -*- coding: utf-8 -*-
 
-try:
-    import unittest2 as unittest
-except ImportError:
-    import unittest
-
 import os
 import sys
 
@@ -12,8 +7,10 @@ sys.path.insert(0, os.path.join(os.path.dirname(__file__), ".."))
 
 from pungi.wrappers import repoclosure as rc
 
+from . import helpers
 
-class RepoclosureWrapperTestCase(unittest.TestCase):
+
+class RepoclosureWrapperTestCase(helpers.BaseTestCase):
     def test_minimal_command(self):
         self.assertEqual(rc.get_repoclosure_cmd(),
                          ['/usr/bin/repoclosure'])
@@ -89,3 +86,27 @@ class RepoclosureWrapperTestCase(unittest.TestCase):
              '--repofrompath=remote,http://kojipkgs.fp.o/repo',
              '--lookaside=local',
              '--lookaside=remote'])
+
+
+class FusExtractorTestCase(helpers.PungiTestCase):
+    def setUp(self):
+        super(FusExtractorTestCase, self).setUp()
+        self.input = os.path.join(self.topdir, "in")
+        self.output = os.path.join(self.topdir, "out")
+
+    def test_no_match(self):
+        helpers.touch(self.input, "fus-DEBUG: Installing foo\n")
+        rc.extract_from_fus_log(self.input, self.output)
+        self.assertFileContent(self.output, "")
+
+    def test_error(self):
+        helpers.touch(
+            self.input,
+            "fus-DEBUG: Installing bar\nProblem 1/1\n - nothing provides foo\n"
+        )
+        with self.assertRaises(RuntimeError) as ctx:
+            rc.extract_from_fus_log(self.input, self.output)
+
+        self.assertIn(self.output, str(ctx.exception))
+
+        self.assertFileContent(self.output, "Problem 1/1\n - nothing provides foo\n")

tests/test_test_phase.py

@@ -229,6 +229,34 @@ class TestRepoclosure(PungiTestCase):
              mock.call(backend='dnf', arch=['x86_64', 'noarch'], lookaside={},
                        repos=self._get_repo('Everything', 'x86_64'))])
 
+    @mock.patch("glob.glob")
+    @mock.patch("pungi.wrappers.repoclosure.extract_from_fus_log")
+    @mock.patch("pungi.wrappers.repoclosure.get_repoclosure_cmd")
+    @mock.patch("pungi.phases.test.run")
+    def test_repoclosure_hybrid_variant(self, mock_run, mock_grc, effl, glob):
+        compose = DummyCompose(
+            self.topdir, {"repoclosure_backend": "dnf", "gather_method": "hybrid"}
+        )
+        f = mock.Mock()
+        glob.return_value = [f]
+
+        def _log(a, v):
+            return compose.paths.log.log_file(a, "repoclosure-%s" % compose.variants[v])
+
+        test_phase.run_repoclosure(compose)
+
+        self.assertEqual(mock_grc.call_args_list, [])
+        self.assertItemsEqual(
+            effl.call_args_list,
+            [
+                mock.call(f, _log("amd64", "Everything")),
+                mock.call(f, _log("amd64", "Client")),
+                mock.call(f, _log("amd64", "Server")),
+                mock.call(f, _log("x86_64", "Server")),
+                mock.call(f, _log("x86_64", "Everything")),
+            ]
+        )
+
     @mock.patch('pungi.wrappers.repoclosure.get_repoclosure_cmd')
     @mock.patch('pungi.phases.test.run')
     def test_repoclosure_report_error(self, mock_run, mock_grc):