almalinux
/
pungi
15
1
Fork 0
Code Issues Pull Requests Releases Activity

fus: Write solvables to file 

Instead of having one giant command line, write the long list into a
file and pass that to the solver.

The items on the input list are sorted for easy processing.

JIRA: COMPOSE-3012
Signed-off-by: Lubomír Sedlář <lsedlar@redhat.com>
This commit is contained in:
Lubomír Sedlář 2018-10-12 14:50:46 +02:00
parent 4f712259d7
commit 9afb8e6801
6 changed files with 129 additions and 45 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
pungi/paths.py
View File

@ -127,6 +127,17 @@ class WorkPaths(object):
path = os.path.join(path, file_name)
return path
def fus_conf(self, arch, variant, iteration, create_dir=True):
"""
Examples:
work/x86_64/fus/Server-solvables.x86_64.conf
"""
file_name = "%s-solvables-%d.%s.conf" % (variant.uid, iteration, arch)
path = os.path.join(self.topdir(arch, create_dir=create_dir), "fus")
if create_dir:
makedirs(path)
return os.path.join(path, file_name)
def pungi_log(self, arch=None, variant=None, create_dir=True, source_name=None):
"""
Examples:

5
pungi/phases/gather/methods/method_hybrid.py
View File

@ -206,12 +206,13 @@ class GatherMethodHybrid(pungi.phases.gather.method.GatherMethodBase):
while True:
step += 1
conf_file = self.compose.paths.work.fus_conf(arch, variant, step)
fus.write_config(conf_file, sorted(modules), sorted(input_packages))
cmd = fus.get_cmd(
conf_file,
tree_arch_to_yum_arch(arch),
repos,
pungi.phases.gather.get_lookaside_repos(self.compose, arch, variant),
input_packages,
modules,
platform=platform,
filter_packages=filter_packages,
)

16
pungi/wrappers/fus.py
View File

@ -26,11 +26,10 @@ Pungi).
def get_cmd(
conf_file,
arch,
repos,
lookasides,
packages,
modules,
platform=None,
filter_packages=None,
):
@ -49,14 +48,19 @@ def get_cmd(
for pkg in sorted(filter_packages or []):
cmd.append("--exclude=%s" % pkg)
for module in modules:
cmd.append("module(%s)" % module)
cmd.extend(packages)
cmd.append("@%s" % conf_file)
return cmd
def write_config(conf_file, modules, packages):
with open(conf_file, "w") as f:
for module in modules:
f.write("module(%s)\n" % module)
for pkg in packages:
f.write("%s\n" % pkg)
def _prep_path(path):
"""Strip file:// from the path if present or raise exception for other
protocols.

8
tests/helpers.py
View File

@ -32,6 +32,14 @@ class BaseTestCase(unittest.TestCase):
fromfile='EXPECTED', tofile='ACTUAL'))
self.assertEqual(diff, '', 'Files differ:\n' + diff)
def assertFileContent(self, fn, expected):
with open(fn, 'rb') as f:
lines = f.read().decode('utf-8').splitlines()
diff = '\n'.join(difflib.unified_diff(
lines, expected.splitlines(), fromfile='EXPECTED', tofile='ACTUAL')
)
self.assertEqual(diff, '', 'Files differ:\n' + diff)
class PungiTestCase(BaseTestCase):
def setUp(self):

57
tests/test_fus_wrapper.py
View File

@ -5,6 +5,7 @@ try:
except ImportError:
import unittest
import tempfile
from textwrap import dedent
import os
import sys
@ -13,21 +14,20 @@ sys.path.insert(0, os.path.join(os.path.dirname(__file__), ".."))
from pungi.wrappers import fus
from .helpers import touch
from .helpers import touch, PungiTestCase
class TestGetCmd(unittest.TestCase):
def test_minimum_command(self):
cmd = fus.get_cmd("x86_64", [], [], [], [], [])
self.assertEqual(cmd, ["fus", "--verbose", "--arch", "x86_64"])
cmd = fus.get_cmd("conf", "x86_64", repos=[], lookasides=[])
self.assertEqual(cmd, ["fus", "--verbose", "--arch", "x86_64", "@conf"])
def test_full_command(self):
cmd = fus.get_cmd(
"conf",
"x86_64",
["/tmp/first", "/tmp/second"],
["/tmp/fst", "/tmp/snd"],
["pkg"],
["mod:1.0"],
platform="f29",
filter_packages=["foo", "bar"],
)
@ -45,31 +45,62 @@ class TestGetCmd(unittest.TestCase):
"--platform=f29",
"--exclude=bar",
"--exclude=foo",
"module(mod:1.0)",
"pkg",
"@conf",
],
)
def test_strip_file_protocol(self):
cmd = fus.get_cmd("x86_64", ["file:///tmp"], [], [], [], [])
cmd = fus.get_cmd("conf", "x86_64", ["file:///tmp"], [])
self.assertEqual(
cmd, ["fus", "--verbose", "--arch", "x86_64", "--repo=repo-0,repo,/tmp"]
cmd,
[
"fus",
"--verbose",
"--arch",
"x86_64",
"--repo=repo-0,repo,/tmp",
"@conf",
],
)
def test_fail_on_http_repo(self):
with self.assertRaises(ValueError):
fus.get_cmd("x86_64", ["http:///tmp"], [], [], [], [])
fus.get_cmd("conf", "x86_64", ["http:///tmp"], [])
def test_strip_file_protocol_lookaside(self):
cmd = fus.get_cmd("x86_64", [], ["file:///r"], [], [], [])
cmd = fus.get_cmd("conf", "x86_64", [], ["file:///r"])
self.assertEqual(
cmd,
["fus", "--verbose", "--arch", "x86_64", "--repo=lookaside-0,lookaside,/r"]
[
"fus",
"--verbose",
"--arch",
"x86_64",
"--repo=lookaside-0,lookaside,/r",
"@conf",
],
)
def test_fail_on_http_repo_lookaside(self):
with self.assertRaises(ValueError):
fus.get_cmd("x86_64", [], ["http:///tmp"], [], [], [])
fus.get_cmd("conf", "x86_64", [], ["http:///tmp"])
class TestWriteConfig(PungiTestCase):
def test_write_sorted_mix(self):
f = os.path.join(self.topdir, "solvables")
fus.write_config(f, ["moda:master"], ["pkg", "foo"])
self.assertFileContent(
f,
dedent(
"""\
module(moda:master)
pkg
foo
"""
),
)
class TestParseOutput(unittest.TestCase):

77
tests/test_gather_method_hybrid.py
View File

@ -292,6 +292,7 @@ class ModifiedMagicMock(mock.MagicMock):
)
@mock.patch("pungi.wrappers.fus.write_config")
@mock.patch("pungi.wrappers.fus.parse_output")
@mock.patch("pungi.wrappers.fus.get_cmd", new_callable=ModifiedMagicMock)
@mock.patch("pungi.phases.gather.methods.method_hybrid.run")
@ -307,8 +308,14 @@ class TestRunSolver(HelperMixin, helpers.PungiTestCase):
self.logfile2 = os.path.join(
self.compose.topdir, "logs/x86_64/hybrid-depsolver-Server-iter-2.x86_64.log"
)
self.config1 = os.path.join(
self.compose.topdir, "work/x86_64/fus/Server-solvables-1.x86_64.conf"
)
self.config2 = os.path.join(
self.compose.topdir, "work/x86_64/fus/Server-solvables-2.x86_64.conf"
)
def test_with_modules(self, run, gc, po):
def test_with_modules(self, run, gc, po, wc):
self.compose.has_comps = False
self.compose.variants["Server"].arch_mmds["x86_64"] = {
"mod:master": mock.Mock(
@ -346,22 +353,24 @@ class TestRunSolver(HelperMixin, helpers.PungiTestCase):
)
],
)
self.assertEqual(
wc.call_args_list, [mock.call(self.config1, ["mod:master"], [])],
)
self.assertEqual(
gc.call_args_list,
[
mock.call(
self.config1,
"x86_64",
[self._repo("repo"), self._repo("module_repo_Server")],
[],
[],
["mod:master"],
platform="pl",
filter_packages=["foo"],
)
],
)
def test_with_modules_with_devel(self, run, gc, po):
def test_with_modules_with_devel(self, run, gc, po, wc):
self.compose.has_comps = False
self.compose.variants["Server"].arch_mmds["x86_64"] = {
"mod:master": mock.Mock(
@ -397,22 +406,25 @@ class TestRunSolver(HelperMixin, helpers.PungiTestCase):
)
],
)
self.assertEqual(
wc.call_args_list,
[mock.call(self.config1, ["mod-devel:master", "mod:master"], [])],
)
self.assertEqual(
gc.call_args_list,
[
mock.call(
self.config1,
"x86_64",
[self._repo("repo"), self._repo("module_repo_Server")],
[],
[],
["mod:master", "mod-devel:master"],
platform="pl",
filter_packages=["foo"],
)
],
)
def test_with_comps(self, run, gc, po):
def test_with_comps(self, run, gc, po, wc):
po.return_value = (mock.Mock(), mock.Mock())
res = self.phase.run_solver(
self.compose.variants["Server"],
@ -432,22 +444,24 @@ class TestRunSolver(HelperMixin, helpers.PungiTestCase):
)
],
)
self.assertEqual(
wc.call_args_list, [mock.call(self.config1, [], ["pkg"])],
)
self.assertEqual(
gc.call_args_list,
[
mock.call(
self.config1,
"x86_64",
[self._repo("repo")],
[],
["pkg"],
[],
platform=None,
filter_packages=[],
)
],
)
def test_with_langpacks(self, run, gc, po):
def test_with_langpacks(self, run, gc, po, wc):
self.phase.langpacks = {"pkg": set(["pkg-en"])}
final = [("pkg-1.0-1", "x86_64", []), ("pkg-en-1.0-1", "noarch", [])]
po.side_effect = [([("pkg-1.0-1", "x86_64", [])], set()), (final, [])]
@ -475,24 +489,29 @@ class TestRunSolver(HelperMixin, helpers.PungiTestCase):
),
],
)
self.assertEqual(
wc.call_args_list,
[
mock.call(self.config1, [], ["pkg"]),
mock.call(self.config2, [], ["pkg", "pkg-en"]),
],
)
self.assertEqual(
gc.call_args_list,
[
mock.call(
self.config1,
"x86_64",
[self._repo("repo")],
[],
["pkg"],
[],
platform=None,
filter_packages=["foo"],
),
mock.call(
self.config2,
"x86_64",
[self._repo("repo")],
[],
["pkg", "pkg-en"],
[],
platform=None,
filter_packages=["foo"],
),
@ -500,7 +519,7 @@ class TestRunSolver(HelperMixin, helpers.PungiTestCase):
)
@mock.patch("pungi.phases.gather.methods.method_hybrid.cr")
def test_multilib_devel(self, cr, run, gc, po):
def test_multilib_devel(self, cr, run, gc, po, wc):
self.phase.arch = "x86_64"
self.phase.multilib_methods = ["devel"]
self.phase.multilib = mock.Mock()
@ -550,24 +569,29 @@ class TestRunSolver(HelperMixin, helpers.PungiTestCase):
),
],
)
self.assertEqual(
wc.call_args_list,
[
mock.call(self.config1, [], ["foo", "pkg-devel"]),
mock.call(self.config2, [], ["foo", "pkg-devel", "pkg-devel.i686"]),
],
)
self.assertEqual(
gc.call_args_list,
[
mock.call(
self.config1,
"x86_64",
[self._repo("repo")],
[],
["pkg-devel", "foo"],
[],
platform=None,
filter_packages=[],
),
mock.call(
self.config2,
"x86_64",
[self._repo("repo")],
[],
["pkg-devel", "foo", "pkg-devel.i686"],
[],
platform=None,
filter_packages=[],
),
@ -575,7 +599,7 @@ class TestRunSolver(HelperMixin, helpers.PungiTestCase):
)
@mock.patch("pungi.phases.gather.methods.method_hybrid.cr")
def test_multilib_runtime(self, cr, run, gc, po):
def test_multilib_runtime(self, cr, run, gc, po, wc):
packages = {
"abc": NamedMock(
name="foo",
@ -651,24 +675,29 @@ class TestRunSolver(HelperMixin, helpers.PungiTestCase):
),
],
)
self.assertEqual(
wc.call_args_list,
[
mock.call(self.config1, [], ["foo", "pkg-devel"]),
mock.call(self.config2, [], ["foo", "foo.i686", "pkg-devel"]),
],
)
self.assertEqual(
gc.call_args_list,
[
mock.call(
self.config1,
"x86_64",
[self._repo("repo")],
[],
["pkg-devel", "foo"],
[],
platform=None,
filter_packages=[],
),
mock.call(
self.config2,
"x86_64",
[self._repo("repo")],
[],
["pkg-devel", "foo", "foo.i686"],
[],
platform=None,
filter_packages=[],
),