gather: Use fresh cache for each sequence of fus runs

Each depsolved tree will be using its own cache for fus. This should
still allow for faster loading of metadata after first iteration, but
should prevent errors from using cached files meant for another variant
or architecture. The cache is deleted after the last iteration.

JIRA: COMPOSE-3959
Signed-off-by: Lubomír Sedlář <lsedlar@redhat.com>
This commit is contained in:
Lubomír Sedlář 2019-11-18 11:17:37 +01:00
parent 722411dfcd
commit 4473b05f10
2 changed files with 22 additions and 4 deletions

View File

@ -27,7 +27,7 @@ from pungi import multilib_dnf
from pungi.module_util import Modulemd from pungi.module_util import Modulemd
from pungi.arch import get_valid_arches, tree_arch_to_yum_arch from pungi.arch import get_valid_arches, tree_arch_to_yum_arch
from pungi.phases.gather import _mk_pkg_map from pungi.phases.gather import _mk_pkg_map
from pungi.util import get_arch_variant_data, pkg_is_debug from pungi.util import get_arch_variant_data, pkg_is_debug, temp_dir
from pungi.wrappers import fus from pungi.wrappers import fus
from pungi.wrappers.comps import CompsWrapper from pungi.wrappers.comps import CompsWrapper
@ -210,7 +210,11 @@ class GatherMethodHybrid(pungi.phases.gather.method.GatherMethodBase):
# strings. # strings.
filters = [_fmt_pkg(*p) for p in filter_packages] filters = [_fmt_pkg(*p) for p in filter_packages]
nvrs, out_modules = self.run_solver(variant, arch, packages, platform, filters) cache_prefix = "fus-cache-%s-%s-%s-" % (self.compose.compose_id, variant, arch)
with temp_dir(prefix=cache_prefix) as cache_dir:
nvrs, out_modules = self.run_solver(
variant, arch, packages, platform, filters, cache_dir=cache_dir
)
filter_modules(variant, arch, out_modules) filter_modules(variant, arch, out_modules)
return expand_packages( return expand_packages(
self._get_pkg_map(arch), self._get_pkg_map(arch),
@ -228,7 +232,7 @@ class GatherMethodHybrid(pungi.phases.gather.method.GatherMethodBase):
repos.append(pkgset.paths[self.arch]) repos.append(pkgset.paths[self.arch])
return repos return repos
def run_solver(self, variant, arch, packages, platform, filter_packages): def run_solver(self, variant, arch, packages, platform, filter_packages, cache_dir):
repos = self.get_repos() repos = self.get_repos()
results = set() results = set()
result_modules = set() result_modules = set()
@ -263,6 +267,7 @@ class GatherMethodHybrid(pungi.phases.gather.method.GatherMethodBase):
# useless for us anyway). # useless for us anyway).
env = os.environ.copy() env = os.environ.copy()
env["G_MESSAGES_PREFIXED"] = "" env["G_MESSAGES_PREFIXED"] = ""
env["XDG_CACHE_HOME"] = cache_dir
self.compose.log_debug("[BEGIN] Running fus") self.compose.log_debug("[BEGIN] Running fus")
run(cmd, logfile=logfile, show_cmd=True, env=env) run(cmd, logfile=logfile, show_cmd=True, env=env)
output, out_modules = fus.parse_output(logfile) output, out_modules = fus.parse_output(logfile)

View File

@ -27,12 +27,13 @@ class NamedMock(mock.Mock):
class TestMethodHybrid(helpers.PungiTestCase): class TestMethodHybrid(helpers.PungiTestCase):
@mock.patch("pungi.phases.gather.methods.method_hybrid.temp_dir")
@mock.patch("pungi.phases.gather.methods.method_hybrid.CompsWrapper") @mock.patch("pungi.phases.gather.methods.method_hybrid.CompsWrapper")
@mock.patch("pungi.phases.gather.get_lookaside_repos") @mock.patch("pungi.phases.gather.get_lookaside_repos")
@mock.patch("pungi.phases.gather.methods.method_hybrid.expand_groups") @mock.patch("pungi.phases.gather.methods.method_hybrid.expand_groups")
@mock.patch("pungi.phases.gather.methods.method_hybrid.expand_packages") @mock.patch("pungi.phases.gather.methods.method_hybrid.expand_packages")
@mock.patch("pungi.phases.gather.methods.method_hybrid.get_platform") @mock.patch("pungi.phases.gather.methods.method_hybrid.get_platform")
def test_call_method(self, gp, ep, eg, glr, CW): def test_call_method(self, gp, ep, eg, glr, CW, td):
compose = helpers.DummyCompose(self.topdir, {}) compose = helpers.DummyCompose(self.topdir, {})
m = hybrid.GatherMethodHybrid(compose) m = hybrid.GatherMethodHybrid(compose)
m.run_solver = mock.Mock(return_value=(mock.Mock(), mock.Mock())) m.run_solver = mock.Mock(return_value=(mock.Mock(), mock.Mock()))
@ -73,9 +74,14 @@ class TestMethodHybrid(helpers.PungiTestCase):
set(["pkg", "foo", "bar", ("prep", "noarch")]), set(["pkg", "foo", "bar", ("prep", "noarch")]),
gp.return_value, gp.return_value,
[], [],
cache_dir=td.return_value.__enter__.return_value,
) )
], ],
) )
self.assertIn(
[mock.call(prefix="fus-cache-Test-20151203.0.t-Server-x86_64-")],
td.mock_calls,
)
self.assertEqual( self.assertEqual(
ep.call_args_list, ep.call_args_list,
[ [
@ -331,6 +337,7 @@ class TestRunSolver(HelperMixin, helpers.PungiTestCase):
[], [],
platform="pl", platform="pl",
filter_packages=[("foo", None)], filter_packages=[("foo", None)],
cache_dir="/cache",
) )
self.assertEqual(res[0], set()) self.assertEqual(res[0], set())
@ -386,6 +393,7 @@ class TestRunSolver(HelperMixin, helpers.PungiTestCase):
[], [],
platform="pl", platform="pl",
filter_packages=["foo"], filter_packages=["foo"],
cache_dir="/cache",
) )
self.assertEqual(res, (set([("p-1-1", "x86_64", frozenset())]), set(["m1"]))) self.assertEqual(res, (set([("p-1-1", "x86_64", frozenset())]), set(["m1"])))
@ -426,6 +434,7 @@ class TestRunSolver(HelperMixin, helpers.PungiTestCase):
[("pkg", None)], [("pkg", None)],
platform=None, platform=None,
filter_packages=[], filter_packages=[],
cache_dir="/cache",
) )
six.assertCountEqual(self, res[0], po.return_value[0]) six.assertCountEqual(self, res[0], po.return_value[0])
@ -481,6 +490,7 @@ class TestRunSolver(HelperMixin, helpers.PungiTestCase):
[("pkg", None)], [("pkg", None)],
platform=None, platform=None,
filter_packages=[], filter_packages=[],
cache_dir="/cache",
) )
six.assertCountEqual( six.assertCountEqual(
@ -553,6 +563,7 @@ class TestRunSolver(HelperMixin, helpers.PungiTestCase):
[("pkg", None)], [("pkg", None)],
platform=None, platform=None,
filter_packages=["foo"], filter_packages=["foo"],
cache_dir="/cache",
) )
six.assertCountEqual(self, res[0], final) six.assertCountEqual(self, res[0], final)
@ -640,6 +651,7 @@ class TestRunSolver(HelperMixin, helpers.PungiTestCase):
[("pkg-devel", None), ("foo", None)], [("pkg-devel", None), ("foo", None)],
platform=None, platform=None,
filter_packages=[], filter_packages=[],
cache_dir="/cache",
) )
six.assertCountEqual( six.assertCountEqual(
@ -762,6 +774,7 @@ class TestRunSolver(HelperMixin, helpers.PungiTestCase):
[("pkg-devel", None), ("foo", None)], [("pkg-devel", None), ("foo", None)],
platform=None, platform=None,
filter_packages=[], filter_packages=[],
cache_dir="/cache",
) )
six.assertCountEqual( six.assertCountEqual(