almalinux
/
pungi
15
1
Fork 0
Code Issues Pull Requests Releases Activity

repoclosure: Extract logs from hybrid solver 

There is no repoclosure that correctly understands modules. The best
thing we can offer is the errors reported by the depsolver.

JIRA: COMPOSE-2321
Signed-off-by: Lubomír Sedlář <lsedlar@redhat.com>
This commit is contained in:
Lubomír Sedlář 2018-10-22 14:14:53 +02:00
parent 12f949fe84
commit fe39056431
5 changed files with 112 additions and 34 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
pungi/phases/gather/__init__.py
View File

@ -151,10 +151,7 @@ def get_parent_pkgs(arch, variant, result_dict):
return result
def gather_packages(compose, arch, variant, package_sets, fulltree_excludes=None):
# multilib white/black-list is per-arch, common for all variants
multilib_whitelist = get_multilib_whitelist(compose, arch)
multilib_blacklist = get_multilib_blacklist(compose, arch)
def get_gather_methods(compose, variant):
methods = compose.conf["gather_method"]
global_method_name = methods
if isinstance(methods, dict):
@ -163,6 +160,14 @@ def gather_packages(compose, arch, variant, package_sets, fulltree_excludes=None
global_method_name = None
except IndexError:
raise RuntimeError("Variant %s has no configured gather_method" % variant.uid)
return global_method_name, methods
def gather_packages(compose, arch, variant, package_sets, fulltree_excludes=None):
# multilib white/black-list is per-arch, common for all variants
multilib_whitelist = get_multilib_whitelist(compose, arch)
multilib_blacklist = get_multilib_blacklist(compose, arch)
global_method_name, methods = get_gather_methods(compose, variant)
msg = "Gathering packages (arch: %s, variant: %s)" % (arch, variant)

57
pungi/phases/test.py
View File

@ -14,6 +14,7 @@
# along with this program; if not, see <https://gnu.org/licenses/>.
import glob
import os
from kobo.shortcuts import run
@ -21,7 +22,7 @@ from kobo.shortcuts import run
from pungi.wrappers import repoclosure
from pungi.arch import get_valid_arches
from pungi.phases.base import PhaseBase
from pungi.phases.gather import get_lookaside_repos
from pungi.phases.gather import get_lookaside_repos, get_gather_methods
from pungi.util import is_arch_multilib, failable, temp_dir, get_arch_variant_data
@ -63,34 +64,42 @@ def run_repoclosure(compose):
for i, lookaside_url in enumerate(get_lookaside_repos(compose, arch, variant)):
lookaside["lookaside-%s.%s-%s" % (variant.uid, arch, i)] = lookaside_url
cmd = repoclosure.get_repoclosure_cmd(backend=compose.conf['repoclosure_backend'],
repos=repos, lookaside=lookaside, arch=arches)
# Use temp working directory directory as workaround for
# https://bugzilla.redhat.com/show_bug.cgi?id=795137
with temp_dir(prefix='repoclosure_') as tmp_dir:
# Ideally we would want show_cmd=True here to include the
# command in the logfile, but due to a bug in Kobo that would
# cause any error to be printed directly to stderr.
# https://github.com/release-engineering/kobo/pull/26
try:
run(
cmd,
logfile=compose.paths.log.log_file(
arch, "repoclosure-%s" % variant
),
workdir=tmp_dir,
show_cmd=True,
logfile = compose.paths.log.log_file(arch, "repoclosure-%s" % variant)
try:
_, methods = get_gather_methods(compose, variant)
if methods == "hybrid":
# Using hybrid solver, no repoclosure command is available.
pattern = compose.paths.log.log_file(
arch, "hybrid-depsolver-%s-iter-*" % variant
)
except RuntimeError as exc:
if conf and conf[-1] == 'fatal':
raise
else:
compose.log_warning('Repoclosure failed for %s.%s\n%s'
% (variant.uid, arch, exc))
fus_log = sorted(glob.glob(pattern))[-1]
repoclosure.extract_from_fus_log(fus_log, logfile)
else:
_run_repoclosure_cmd(compose, repos, lookaside, arches, logfile)
except RuntimeError as exc:
if conf and conf[-1] == 'fatal':
raise
else:
compose.log_warning('Repoclosure failed for %s.%s\n%s'
% (variant.uid, arch, exc))
compose.log_info("[DONE ] %s" % msg)
def _run_repoclosure_cmd(compose, repos, lookaside, arches, logfile):
cmd = repoclosure.get_repoclosure_cmd(backend=compose.conf["repoclosure_backend"],
repos=repos, lookaside=lookaside, arch=arches)
# Use temp working directory directory as workaround for
# https://bugzilla.redhat.com/show_bug.cgi?id=795137
with temp_dir(prefix="repoclosure_") as tmp_dir:
# Ideally we would want show_cmd=True here to include the
# command in the logfile, but due to a bug in Kobo that would
# cause any error to be printed directly to stderr.
# https://github.com/release-engineering/kobo/pull/26
run(cmd, logfile=logfile, workdir=tmp_dir, show_cmd=True)
def check_image_sanity(compose):
"""
Go through all images in manifest and make basic sanity tests on them. If

15
pungi/wrappers/repoclosure.py
View File

@ -57,3 +57,18 @@ def _to_url(path):
if "://" not in path:
return "file://%s" % os.path.abspath(path)
return path
def extract_from_fus_log(input, output):
"""Extract depsolver error messages from fus log and write them to output
file.
"""
has_error = False
with open(input) as fin:
with open(output, "w") as fout:
for line in fin:
if line.startswith("Problem ") or line.startswith(" "):
fout.write(line)
has_error = True
if has_error:
raise RuntimeError("Broken dependencies! See %s for details" % output)

33
tests/test_repoclosure_wrapper.py
View File

@ -1,10 +1,5 @@
# -*- coding: utf-8 -*-
try:
import unittest2 as unittest
except ImportError:
import unittest
import os
import sys
@ -12,8 +7,10 @@ sys.path.insert(0, os.path.join(os.path.dirname(__file__), ".."))
from pungi.wrappers import repoclosure as rc
from . import helpers
class RepoclosureWrapperTestCase(unittest.TestCase):
class RepoclosureWrapperTestCase(helpers.BaseTestCase):
def test_minimal_command(self):
self.assertEqual(rc.get_repoclosure_cmd(),
['/usr/bin/repoclosure'])
@ -89,3 +86,27 @@ class RepoclosureWrapperTestCase(unittest.TestCase):
'--repofrompath=remote,http://kojipkgs.fp.o/repo',
'--lookaside=local',
'--lookaside=remote'])
class FusExtractorTestCase(helpers.PungiTestCase):
def setUp(self):
super(FusExtractorTestCase, self).setUp()
self.input = os.path.join(self.topdir, "in")
self.output = os.path.join(self.topdir, "out")
def test_no_match(self):
helpers.touch(self.input, "fus-DEBUG: Installing foo\n")
rc.extract_from_fus_log(self.input, self.output)
self.assertFileContent(self.output, "")
def test_error(self):
helpers.touch(
self.input,
"fus-DEBUG: Installing bar\nProblem 1/1\n - nothing provides foo\n"
)
with self.assertRaises(RuntimeError) as ctx:
rc.extract_from_fus_log(self.input, self.output)
self.assertIn(self.output, str(ctx.exception))
self.assertFileContent(self.output, "Problem 1/1\n - nothing provides foo\n")

28
tests/test_test_phase.py
View File

@ -229,6 +229,34 @@ class TestRepoclosure(PungiTestCase):
mock.call(backend='dnf', arch=['x86_64', 'noarch'], lookaside={},
repos=self._get_repo('Everything', 'x86_64'))])
@mock.patch("glob.glob")
@mock.patch("pungi.wrappers.repoclosure.extract_from_fus_log")
@mock.patch("pungi.wrappers.repoclosure.get_repoclosure_cmd")
@mock.patch("pungi.phases.test.run")
def test_repoclosure_hybrid_variant(self, mock_run, mock_grc, effl, glob):
compose = DummyCompose(
self.topdir, {"repoclosure_backend": "dnf", "gather_method": "hybrid"}
)
f = mock.Mock()
glob.return_value = [f]
def _log(a, v):
return compose.paths.log.log_file(a, "repoclosure-%s" % compose.variants[v])
test_phase.run_repoclosure(compose)
self.assertEqual(mock_grc.call_args_list, [])
self.assertItemsEqual(
effl.call_args_list,
[
mock.call(f, _log("amd64", "Everything")),
mock.call(f, _log("amd64", "Client")),
mock.call(f, _log("amd64", "Server")),
mock.call(f, _log("x86_64", "Server")),
mock.call(f, _log("x86_64", "Everything")),
]
)
@mock.patch('pungi.wrappers.repoclosure.get_repoclosure_cmd')
@mock.patch('pungi.phases.test.run')
def test_repoclosure_report_error(self, mock_run, mock_grc):