From fe39056431b00bd35793990d48bbe42ccb17194a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lubom=C3=ADr=20Sedl=C3=A1=C5=99?= Date: Mon, 22 Oct 2018 14:14:53 +0200 Subject: [PATCH] repoclosure: Extract logs from hybrid solver MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit There is no repoclosure that correctly understands modules. The best thing we can offer is the errors reported by the depsolver. JIRA: COMPOSE-2321 Signed-off-by: Lubomír Sedlář --- pungi/phases/gather/__init__.py | 13 ++++--- pungi/phases/test.py | 57 ++++++++++++++++++------------- pungi/wrappers/repoclosure.py | 15 ++++++++ tests/test_repoclosure_wrapper.py | 33 ++++++++++++++---- tests/test_test_phase.py | 28 +++++++++++++++ 5 files changed, 112 insertions(+), 34 deletions(-) diff --git a/pungi/phases/gather/__init__.py b/pungi/phases/gather/__init__.py index 5a1d0b9f..745bd57c 100644 --- a/pungi/phases/gather/__init__.py +++ b/pungi/phases/gather/__init__.py @@ -151,10 +151,7 @@ def get_parent_pkgs(arch, variant, result_dict): return result -def gather_packages(compose, arch, variant, package_sets, fulltree_excludes=None): - # multilib white/black-list is per-arch, common for all variants - multilib_whitelist = get_multilib_whitelist(compose, arch) - multilib_blacklist = get_multilib_blacklist(compose, arch) +def get_gather_methods(compose, variant): methods = compose.conf["gather_method"] global_method_name = methods if isinstance(methods, dict): @@ -163,6 +160,14 @@ def gather_packages(compose, arch, variant, package_sets, fulltree_excludes=None global_method_name = None except IndexError: raise RuntimeError("Variant %s has no configured gather_method" % variant.uid) + return global_method_name, methods + + +def gather_packages(compose, arch, variant, package_sets, fulltree_excludes=None): + # multilib white/black-list is per-arch, common for all variants + multilib_whitelist = get_multilib_whitelist(compose, arch) + multilib_blacklist = get_multilib_blacklist(compose, arch) + global_method_name, methods = get_gather_methods(compose, variant) msg = "Gathering packages (arch: %s, variant: %s)" % (arch, variant) diff --git a/pungi/phases/test.py b/pungi/phases/test.py index c749697e..1efe2953 100644 --- a/pungi/phases/test.py +++ b/pungi/phases/test.py @@ -14,6 +14,7 @@ # along with this program; if not, see . +import glob import os from kobo.shortcuts import run @@ -21,7 +22,7 @@ from kobo.shortcuts import run from pungi.wrappers import repoclosure from pungi.arch import get_valid_arches from pungi.phases.base import PhaseBase -from pungi.phases.gather import get_lookaside_repos +from pungi.phases.gather import get_lookaside_repos, get_gather_methods from pungi.util import is_arch_multilib, failable, temp_dir, get_arch_variant_data @@ -63,34 +64,42 @@ def run_repoclosure(compose): for i, lookaside_url in enumerate(get_lookaside_repos(compose, arch, variant)): lookaside["lookaside-%s.%s-%s" % (variant.uid, arch, i)] = lookaside_url - cmd = repoclosure.get_repoclosure_cmd(backend=compose.conf['repoclosure_backend'], - repos=repos, lookaside=lookaside, arch=arches) - # Use temp working directory directory as workaround for - # https://bugzilla.redhat.com/show_bug.cgi?id=795137 - with temp_dir(prefix='repoclosure_') as tmp_dir: - # Ideally we would want show_cmd=True here to include the - # command in the logfile, but due to a bug in Kobo that would - # cause any error to be printed directly to stderr. - # https://github.com/release-engineering/kobo/pull/26 - try: - run( - cmd, - logfile=compose.paths.log.log_file( - arch, "repoclosure-%s" % variant - ), - workdir=tmp_dir, - show_cmd=True, + logfile = compose.paths.log.log_file(arch, "repoclosure-%s" % variant) + + try: + _, methods = get_gather_methods(compose, variant) + if methods == "hybrid": + # Using hybrid solver, no repoclosure command is available. + pattern = compose.paths.log.log_file( + arch, "hybrid-depsolver-%s-iter-*" % variant ) - except RuntimeError as exc: - if conf and conf[-1] == 'fatal': - raise - else: - compose.log_warning('Repoclosure failed for %s.%s\n%s' - % (variant.uid, arch, exc)) + fus_log = sorted(glob.glob(pattern))[-1] + repoclosure.extract_from_fus_log(fus_log, logfile) + else: + _run_repoclosure_cmd(compose, repos, lookaside, arches, logfile) + except RuntimeError as exc: + if conf and conf[-1] == 'fatal': + raise + else: + compose.log_warning('Repoclosure failed for %s.%s\n%s' + % (variant.uid, arch, exc)) compose.log_info("[DONE ] %s" % msg) +def _run_repoclosure_cmd(compose, repos, lookaside, arches, logfile): + cmd = repoclosure.get_repoclosure_cmd(backend=compose.conf["repoclosure_backend"], + repos=repos, lookaside=lookaside, arch=arches) + # Use temp working directory directory as workaround for + # https://bugzilla.redhat.com/show_bug.cgi?id=795137 + with temp_dir(prefix="repoclosure_") as tmp_dir: + # Ideally we would want show_cmd=True here to include the + # command in the logfile, but due to a bug in Kobo that would + # cause any error to be printed directly to stderr. + # https://github.com/release-engineering/kobo/pull/26 + run(cmd, logfile=logfile, workdir=tmp_dir, show_cmd=True) + + def check_image_sanity(compose): """ Go through all images in manifest and make basic sanity tests on them. If diff --git a/pungi/wrappers/repoclosure.py b/pungi/wrappers/repoclosure.py index e89106e9..31f04048 100644 --- a/pungi/wrappers/repoclosure.py +++ b/pungi/wrappers/repoclosure.py @@ -57,3 +57,18 @@ def _to_url(path): if "://" not in path: return "file://%s" % os.path.abspath(path) return path + + +def extract_from_fus_log(input, output): + """Extract depsolver error messages from fus log and write them to output + file. + """ + has_error = False + with open(input) as fin: + with open(output, "w") as fout: + for line in fin: + if line.startswith("Problem ") or line.startswith(" "): + fout.write(line) + has_error = True + if has_error: + raise RuntimeError("Broken dependencies! See %s for details" % output) diff --git a/tests/test_repoclosure_wrapper.py b/tests/test_repoclosure_wrapper.py index 4b670369..bb7ae950 100755 --- a/tests/test_repoclosure_wrapper.py +++ b/tests/test_repoclosure_wrapper.py @@ -1,10 +1,5 @@ # -*- coding: utf-8 -*- -try: - import unittest2 as unittest -except ImportError: - import unittest - import os import sys @@ -12,8 +7,10 @@ sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..")) from pungi.wrappers import repoclosure as rc +from . import helpers -class RepoclosureWrapperTestCase(unittest.TestCase): + +class RepoclosureWrapperTestCase(helpers.BaseTestCase): def test_minimal_command(self): self.assertEqual(rc.get_repoclosure_cmd(), ['/usr/bin/repoclosure']) @@ -89,3 +86,27 @@ class RepoclosureWrapperTestCase(unittest.TestCase): '--repofrompath=remote,http://kojipkgs.fp.o/repo', '--lookaside=local', '--lookaside=remote']) + + +class FusExtractorTestCase(helpers.PungiTestCase): + def setUp(self): + super(FusExtractorTestCase, self).setUp() + self.input = os.path.join(self.topdir, "in") + self.output = os.path.join(self.topdir, "out") + + def test_no_match(self): + helpers.touch(self.input, "fus-DEBUG: Installing foo\n") + rc.extract_from_fus_log(self.input, self.output) + self.assertFileContent(self.output, "") + + def test_error(self): + helpers.touch( + self.input, + "fus-DEBUG: Installing bar\nProblem 1/1\n - nothing provides foo\n" + ) + with self.assertRaises(RuntimeError) as ctx: + rc.extract_from_fus_log(self.input, self.output) + + self.assertIn(self.output, str(ctx.exception)) + + self.assertFileContent(self.output, "Problem 1/1\n - nothing provides foo\n") diff --git a/tests/test_test_phase.py b/tests/test_test_phase.py index d2dda657..824a67d5 100644 --- a/tests/test_test_phase.py +++ b/tests/test_test_phase.py @@ -229,6 +229,34 @@ class TestRepoclosure(PungiTestCase): mock.call(backend='dnf', arch=['x86_64', 'noarch'], lookaside={}, repos=self._get_repo('Everything', 'x86_64'))]) + @mock.patch("glob.glob") + @mock.patch("pungi.wrappers.repoclosure.extract_from_fus_log") + @mock.patch("pungi.wrappers.repoclosure.get_repoclosure_cmd") + @mock.patch("pungi.phases.test.run") + def test_repoclosure_hybrid_variant(self, mock_run, mock_grc, effl, glob): + compose = DummyCompose( + self.topdir, {"repoclosure_backend": "dnf", "gather_method": "hybrid"} + ) + f = mock.Mock() + glob.return_value = [f] + + def _log(a, v): + return compose.paths.log.log_file(a, "repoclosure-%s" % compose.variants[v]) + + test_phase.run_repoclosure(compose) + + self.assertEqual(mock_grc.call_args_list, []) + self.assertItemsEqual( + effl.call_args_list, + [ + mock.call(f, _log("amd64", "Everything")), + mock.call(f, _log("amd64", "Client")), + mock.call(f, _log("amd64", "Server")), + mock.call(f, _log("x86_64", "Server")), + mock.call(f, _log("x86_64", "Everything")), + ] + ) + @mock.patch('pungi.wrappers.repoclosure.get_repoclosure_cmd') @mock.patch('pungi.phases.test.run') def test_repoclosure_report_error(self, mock_run, mock_grc):