fus: Write solvables to file

Instead of having one giant command line, write the long list into a
file and pass that to the solver.

The items on the input list are sorted for easy processing.

JIRA: COMPOSE-3012
Signed-off-by: Lubomír Sedlář <lsedlar@redhat.com>

pungi/paths.py

@@ -127,6 +127,17 @@ class WorkPaths(object):
         path = os.path.join(path, file_name)
         return path
 
+    def fus_conf(self, arch, variant, iteration, create_dir=True):
+        """
+        Examples:
+            work/x86_64/fus/Server-solvables.x86_64.conf
+        """
+        file_name = "%s-solvables-%d.%s.conf" % (variant.uid, iteration, arch)
+        path = os.path.join(self.topdir(arch, create_dir=create_dir), "fus")
+        if create_dir:
+            makedirs(path)
+        return os.path.join(path, file_name)
+
     def pungi_log(self, arch=None, variant=None, create_dir=True, source_name=None):
         """
         Examples:

pungi/phases/gather/methods/method_hybrid.py

@@ -206,12 +206,13 @@ class GatherMethodHybrid(pungi.phases.gather.method.GatherMethodBase):
 
         while True:
             step += 1
+            conf_file = self.compose.paths.work.fus_conf(arch, variant, step)
+            fus.write_config(conf_file, sorted(modules), sorted(input_packages))
             cmd = fus.get_cmd(
+                conf_file,
                 tree_arch_to_yum_arch(arch),
                 repos,
                 pungi.phases.gather.get_lookaside_repos(self.compose, arch, variant),
-                input_packages,
-                modules,
                 platform=platform,
                 filter_packages=filter_packages,
             )

pungi/wrappers/fus.py

@@ -26,11 +26,10 @@ Pungi).
 
 
 def get_cmd(
+    conf_file,
     arch,
     repos,
     lookasides,
-    packages,
-    modules,
     platform=None,
     filter_packages=None,
 ):
@@ -49,14 +48,19 @@ def get_cmd(
     for pkg in sorted(filter_packages or []):
         cmd.append("--exclude=%s" % pkg)
 
-    for module in modules:
+    cmd.append("@%s" % conf_file)
-        cmd.append("module(%s)" % module)
-
-    cmd.extend(packages)
 
     return cmd
 
 
+def write_config(conf_file, modules, packages):
+    with open(conf_file, "w") as f:
+        for module in modules:
+            f.write("module(%s)\n" % module)
+        for pkg in packages:
+            f.write("%s\n" % pkg)
+
+
 def _prep_path(path):
     """Strip file:// from the path if present or raise exception for other
     protocols.

tests/helpers.py

@@ -32,6 +32,14 @@ class BaseTestCase(unittest.TestCase):
                                               fromfile='EXPECTED', tofile='ACTUAL'))
         self.assertEqual(diff, '', 'Files differ:\n' + diff)
 
+    def assertFileContent(self, fn, expected):
+        with open(fn, 'rb') as f:
+            lines = f.read().decode('utf-8').splitlines()
+        diff = '\n'.join(difflib.unified_diff(
+            lines, expected.splitlines(), fromfile='EXPECTED', tofile='ACTUAL')
+        )
+        self.assertEqual(diff, '', 'Files differ:\n' + diff)
+
 
 class PungiTestCase(BaseTestCase):
     def setUp(self):

tests/test_fus_wrapper.py

@@ -5,6 +5,7 @@ try:
 except ImportError:
     import unittest
 import tempfile
+from textwrap import dedent
 
 import os
 import sys
@@ -13,21 +14,20 @@ sys.path.insert(0, os.path.join(os.path.dirname(__file__), ".."))
 
 from pungi.wrappers import fus
 
-from .helpers import touch
+from .helpers import touch, PungiTestCase
 
 
 class TestGetCmd(unittest.TestCase):
     def test_minimum_command(self):
-        cmd = fus.get_cmd("x86_64", [], [], [], [], [])
+        cmd = fus.get_cmd("conf", "x86_64", repos=[], lookasides=[])
-        self.assertEqual(cmd, ["fus", "--verbose", "--arch", "x86_64"])
+        self.assertEqual(cmd, ["fus", "--verbose", "--arch", "x86_64", "@conf"])
 
     def test_full_command(self):
         cmd = fus.get_cmd(
+            "conf",
             "x86_64",
             ["/tmp/first", "/tmp/second"],
             ["/tmp/fst", "/tmp/snd"],
-            ["pkg"],
-            ["mod:1.0"],
             platform="f29",
             filter_packages=["foo", "bar"],
         )
@@ -45,31 +45,62 @@ class TestGetCmd(unittest.TestCase):
                 "--platform=f29",
                 "--exclude=bar",
                 "--exclude=foo",
-                "module(mod:1.0)",
+                "@conf",
-                "pkg",
             ],
         )
 
     def test_strip_file_protocol(self):
-        cmd = fus.get_cmd("x86_64", ["file:///tmp"], [], [], [], [])
+        cmd = fus.get_cmd("conf", "x86_64", ["file:///tmp"], [])
         self.assertEqual(
-            cmd, ["fus", "--verbose", "--arch", "x86_64", "--repo=repo-0,repo,/tmp"]
+            cmd,
+            [
+                "fus",
+                "--verbose",
+                "--arch",
+                "x86_64",
+                "--repo=repo-0,repo,/tmp",
+                "@conf",
+            ],
         )
 
     def test_fail_on_http_repo(self):
         with self.assertRaises(ValueError):
-            fus.get_cmd("x86_64", ["http:///tmp"], [], [], [], [])
+            fus.get_cmd("conf", "x86_64", ["http:///tmp"], [])
 
     def test_strip_file_protocol_lookaside(self):
-        cmd = fus.get_cmd("x86_64", [], ["file:///r"], [], [], [])
+        cmd = fus.get_cmd("conf", "x86_64", [], ["file:///r"])
         self.assertEqual(
             cmd,
-            ["fus", "--verbose", "--arch", "x86_64", "--repo=lookaside-0,lookaside,/r"]
+            [
+                "fus",
+                "--verbose",
+                "--arch",
+                "x86_64",
+                "--repo=lookaside-0,lookaside,/r",
+                "@conf",
+            ],
         )
 
     def test_fail_on_http_repo_lookaside(self):
         with self.assertRaises(ValueError):
-            fus.get_cmd("x86_64", [], ["http:///tmp"], [], [], [])
+            fus.get_cmd("conf", "x86_64", [], ["http:///tmp"])
+
+
+class TestWriteConfig(PungiTestCase):
+
+    def test_write_sorted_mix(self):
+        f = os.path.join(self.topdir, "solvables")
+        fus.write_config(f, ["moda:master"], ["pkg", "foo"])
+        self.assertFileContent(
+            f,
+            dedent(
+                """\
+                module(moda:master)
+                pkg
+                foo
+                """
+            ),
+        )
 
 
 class TestParseOutput(unittest.TestCase):

tests/test_gather_method_hybrid.py

@@ -292,6 +292,7 @@ class ModifiedMagicMock(mock.MagicMock):
         )
 
 
+@mock.patch("pungi.wrappers.fus.write_config")
 @mock.patch("pungi.wrappers.fus.parse_output")
 @mock.patch("pungi.wrappers.fus.get_cmd", new_callable=ModifiedMagicMock)
 @mock.patch("pungi.phases.gather.methods.method_hybrid.run")
@@ -307,8 +308,14 @@ class TestRunSolver(HelperMixin, helpers.PungiTestCase):
         self.logfile2 = os.path.join(
             self.compose.topdir, "logs/x86_64/hybrid-depsolver-Server-iter-2.x86_64.log"
         )
+        self.config1 = os.path.join(
+            self.compose.topdir, "work/x86_64/fus/Server-solvables-1.x86_64.conf"
+        )
+        self.config2 = os.path.join(
+            self.compose.topdir, "work/x86_64/fus/Server-solvables-2.x86_64.conf"
+        )
 
-    def test_with_modules(self, run, gc, po):
+    def test_with_modules(self, run, gc, po, wc):
         self.compose.has_comps = False
         self.compose.variants["Server"].arch_mmds["x86_64"] = {
             "mod:master": mock.Mock(
@@ -346,22 +353,24 @@ class TestRunSolver(HelperMixin, helpers.PungiTestCase):
                 )
             ],
         )
+        self.assertEqual(
+            wc.call_args_list, [mock.call(self.config1, ["mod:master"], [])],
+        )
         self.assertEqual(
             gc.call_args_list,
             [
                 mock.call(
+                    self.config1,
                     "x86_64",
                     [self._repo("repo"), self._repo("module_repo_Server")],
                     [],
-                    [],
-                    ["mod:master"],
                     platform="pl",
                     filter_packages=["foo"],
                 )
             ],
         )
 
-    def test_with_modules_with_devel(self, run, gc, po):
+    def test_with_modules_with_devel(self, run, gc, po, wc):
         self.compose.has_comps = False
         self.compose.variants["Server"].arch_mmds["x86_64"] = {
             "mod:master": mock.Mock(
@@ -397,22 +406,25 @@ class TestRunSolver(HelperMixin, helpers.PungiTestCase):
                 )
             ],
         )
+        self.assertEqual(
+            wc.call_args_list,
+            [mock.call(self.config1, ["mod-devel:master", "mod:master"], [])],
+        )
         self.assertEqual(
             gc.call_args_list,
             [
                 mock.call(
+                    self.config1,
                     "x86_64",
                     [self._repo("repo"), self._repo("module_repo_Server")],
                     [],
-                    [],
-                    ["mod:master", "mod-devel:master"],
                     platform="pl",
                     filter_packages=["foo"],
                 )
             ],
         )
 
-    def test_with_comps(self, run, gc, po):
+    def test_with_comps(self, run, gc, po, wc):
         po.return_value = (mock.Mock(), mock.Mock())
         res = self.phase.run_solver(
             self.compose.variants["Server"],
@@ -432,22 +444,24 @@ class TestRunSolver(HelperMixin, helpers.PungiTestCase):
                 )
             ],
         )
+        self.assertEqual(
+            wc.call_args_list, [mock.call(self.config1, [], ["pkg"])],
+        )
         self.assertEqual(
             gc.call_args_list,
             [
                 mock.call(
+                    self.config1,
                     "x86_64",
                     [self._repo("repo")],
                     [],
-                    ["pkg"],
-                    [],
                     platform=None,
                     filter_packages=[],
                 )
             ],
         )
 
-    def test_with_langpacks(self, run, gc, po):
+    def test_with_langpacks(self, run, gc, po, wc):
         self.phase.langpacks = {"pkg": set(["pkg-en"])}
         final = [("pkg-1.0-1", "x86_64", []), ("pkg-en-1.0-1", "noarch", [])]
         po.side_effect = [([("pkg-1.0-1", "x86_64", [])], set()), (final, [])]
@@ -475,24 +489,29 @@ class TestRunSolver(HelperMixin, helpers.PungiTestCase):
                 ),
             ],
         )
+        self.assertEqual(
+            wc.call_args_list,
+            [
+                mock.call(self.config1, [], ["pkg"]),
+                mock.call(self.config2, [], ["pkg", "pkg-en"]),
+            ],
+        )
         self.assertEqual(
             gc.call_args_list,
             [
                 mock.call(
+                    self.config1,
                     "x86_64",
                     [self._repo("repo")],
                     [],
-                    ["pkg"],
-                    [],
                     platform=None,
                     filter_packages=["foo"],
                 ),
                 mock.call(
+                    self.config2,
                     "x86_64",
                     [self._repo("repo")],
                     [],
-                    ["pkg", "pkg-en"],
-                    [],
                     platform=None,
                     filter_packages=["foo"],
                 ),
@@ -500,7 +519,7 @@ class TestRunSolver(HelperMixin, helpers.PungiTestCase):
         )
 
     @mock.patch("pungi.phases.gather.methods.method_hybrid.cr")
-    def test_multilib_devel(self, cr, run, gc, po):
+    def test_multilib_devel(self, cr, run, gc, po, wc):
         self.phase.arch = "x86_64"
         self.phase.multilib_methods = ["devel"]
         self.phase.multilib = mock.Mock()
@@ -550,24 +569,29 @@ class TestRunSolver(HelperMixin, helpers.PungiTestCase):
                 ),
             ],
         )
+        self.assertEqual(
+            wc.call_args_list,
+            [
+                mock.call(self.config1, [], ["foo", "pkg-devel"]),
+                mock.call(self.config2, [], ["foo", "pkg-devel", "pkg-devel.i686"]),
+            ],
+        )
         self.assertEqual(
             gc.call_args_list,
             [
                 mock.call(
+                    self.config1,
                     "x86_64",
                     [self._repo("repo")],
                     [],
-                    ["pkg-devel", "foo"],
-                    [],
                     platform=None,
                     filter_packages=[],
                 ),
                 mock.call(
+                    self.config2,
                     "x86_64",
                     [self._repo("repo")],
                     [],
-                    ["pkg-devel", "foo", "pkg-devel.i686"],
-                    [],
                     platform=None,
                     filter_packages=[],
                 ),
@@ -575,7 +599,7 @@ class TestRunSolver(HelperMixin, helpers.PungiTestCase):
         )
 
     @mock.patch("pungi.phases.gather.methods.method_hybrid.cr")
-    def test_multilib_runtime(self, cr, run, gc, po):
+    def test_multilib_runtime(self, cr, run, gc, po, wc):
         packages = {
             "abc": NamedMock(
                 name="foo",
@@ -651,24 +675,29 @@ class TestRunSolver(HelperMixin, helpers.PungiTestCase):
                 ),
             ],
         )
+        self.assertEqual(
+            wc.call_args_list,
+            [
+                mock.call(self.config1, [], ["foo", "pkg-devel"]),
+                mock.call(self.config2, [], ["foo", "foo.i686", "pkg-devel"]),
+            ],
+        )
         self.assertEqual(
             gc.call_args_list,
             [
                 mock.call(
+                    self.config1,
                     "x86_64",
                     [self._repo("repo")],
                     [],
-                    ["pkg-devel", "foo"],
-                    [],
                     platform=None,
                     filter_packages=[],
                 ),
                 mock.call(
+                    self.config2,
                     "x86_64",
                     [self._repo("repo")],
                     [],
-                    ["pkg-devel", "foo", "foo.i686"],
-                    [],
                     platform=None,
                     filter_packages=[],
                 ),