repoclosure: Parse all fus logs

Originally the list of solvables for fus was growing with each iteration
and nothing was ever removed. That later changed so that fus iterations
are only done on newly added stuff. It's great for performance, but
means that the last log is not a superset of all others.

To get all dependency problems we need to look into all log files, not
just the last one.

JIRA: COMPOSE-3964
Signed-off-by: Lubomír Sedlář <lsedlar@redhat.com>

pungi/phases/test.py

@@ -75,8 +75,8 @@ def run_repoclosure(compose):
                     pattern = compose.paths.log.log_file(
                         arch, "hybrid-depsolver-%s-iter-*" % variant
                     )
-                    fus_log = sorted(glob.glob(pattern))[-1]
+                    fus_logs = sorted(glob.glob(pattern))
-                    repoclosure.extract_from_fus_log(fus_log, logfile)
+                    repoclosure.extract_from_fus_logs(fus_logs, logfile)
                 else:
                     _run_repoclosure_cmd(compose, repos, lookaside, arches, logfile)
             except RuntimeError as exc:

pungi/wrappers/repoclosure.py

@@ -59,16 +59,17 @@ def _to_url(path):
     return path
 
 
-def extract_from_fus_log(input, output):
+def extract_from_fus_logs(inputs, output):
     """Extract depsolver error messages from fus log and write them to output
     file.
     """
     has_error = False
-    with open(input) as fin:
+    with open(output, "w") as fout:
-        with open(output, "w") as fout:
+        for input in inputs:
-            for line in fin:
+            with open(input) as fin:
-                if line.startswith("Problem ") or line.startswith(" "):
+                for line in fin:
-                    fout.write(line)
+                    if line.startswith("Problem ") or line.startswith(" "):
-                    has_error = True
+                        fout.write(line)
+                        has_error = True
     if has_error:
         raise RuntimeError("Broken dependencies! See %s for details" % output)

tests/test_repoclosure_wrapper.py

@@ -108,22 +108,42 @@ class RepoclosureWrapperTestCase(helpers.BaseTestCase):
 class FusExtractorTestCase(helpers.PungiTestCase):
     def setUp(self):
         super(FusExtractorTestCase, self).setUp()
-        self.input = os.path.join(self.topdir, "in")
+        self.input1 = os.path.join(self.topdir, "in1")
+        self.input2 = os.path.join(self.topdir, "in2")
         self.output = os.path.join(self.topdir, "out")
 
     def test_no_match(self):
-        helpers.touch(self.input, "fus-DEBUG: Installing foo\n")
+        helpers.touch(self.input1, "fus-DEBUG: Installing foo\n")
-        rc.extract_from_fus_log(self.input, self.output)
+        rc.extract_from_fus_logs([self.input1], self.output)
         self.assertFileContent(self.output, "")
 
     def test_error(self):
         helpers.touch(
-            self.input,
+            self.input1,
             "fus-DEBUG: Installing bar\nProblem 1/1\n - nothing provides foo\n"
         )
         with self.assertRaises(RuntimeError) as ctx:
-            rc.extract_from_fus_log(self.input, self.output)
+            rc.extract_from_fus_logs([self.input1], self.output)
 
         self.assertIn(self.output, str(ctx.exception))
 
         self.assertFileContent(self.output, "Problem 1/1\n - nothing provides foo\n")
+
+    def test_errors_in_multiple_files(self):
+        helpers.touch(
+            self.input1,
+            "fus-DEBUG: Installing bar\nProblem 1/1\n - nothing provides foo\n"
+        )
+        helpers.touch(
+            self.input2,
+            "fus-DEBUG: Installing baz\nProblem 1/1\n - nothing provides quux\n"
+        )
+        with self.assertRaises(RuntimeError) as ctx:
+            rc.extract_from_fus_logs([self.input1, self.input2], self.output)
+
+        self.assertIn(self.output, str(ctx.exception))
+
+        self.assertFileContent(
+            self.output,
+            "Problem 1/1\n - nothing provides foo\nProblem 1/1\n - nothing provides quux\n",
+        )

tests/test_test_phase.py

@@ -362,7 +362,7 @@ class TestRepoclosure(PungiTestCase):
                        repos=self._get_repo(compose.compose_id, 'Everything', 'x86_64'))])
 
     @mock.patch("glob.glob")
-    @mock.patch("pungi.wrappers.repoclosure.extract_from_fus_log")
+    @mock.patch("pungi.wrappers.repoclosure.extract_from_fus_logs")
     @mock.patch("pungi.wrappers.repoclosure.get_repoclosure_cmd")
     @mock.patch("pungi.phases.test.run")
     def test_repoclosure_hybrid_variant(self, mock_run, mock_grc, effl, glob):
@@ -382,11 +382,11 @@ class TestRepoclosure(PungiTestCase):
             self,
             effl.call_args_list,
             [
-                mock.call(f, _log("amd64", "Everything")),
+                mock.call([f], _log("amd64", "Everything")),
-                mock.call(f, _log("amd64", "Client")),
+                mock.call([f], _log("amd64", "Client")),
-                mock.call(f, _log("amd64", "Server")),
+                mock.call([f], _log("amd64", "Server")),
-                mock.call(f, _log("x86_64", "Server")),
+                mock.call([f], _log("x86_64", "Server")),
-                mock.call(f, _log("x86_64", "Everything")),
+                mock.call([f], _log("x86_64", "Everything")),
             ]
         )