Store extended traceback for gather errors

When a gathering thread raises an exception, it gets forwarded to the
main thread and re-raised there. However, during this transition it
loses details about exact location of the problem.

This patch creates an extended traceback in the worker, which should
make it easier to track the problem down later.

JIRA: RHELCMP-4259
Signed-off-by: Lubomír Sedlář <lsedlar@redhat.com>
This commit is contained in:
Lubomír Sedlář 2021-02-17 10:52:08 +01:00
parent 98359654cf
commit 477dcf37d9
4 changed files with 50 additions and 6 deletions

View File

@ -26,6 +26,7 @@ import shutil
import json import json
import kobo.log import kobo.log
import kobo.tback
from productmd.composeinfo import ComposeInfo from productmd.composeinfo import ComposeInfo
from productmd.images import Images from productmd.images import Images
from dogpile.cache import make_region from dogpile.cache import make_region
@ -590,6 +591,20 @@ class Compose(kobo.log.LoggingBase):
separators=(",", ": "), separators=(",", ": "),
) )
def traceback(self, detail=None):
"""Store an extended traceback. This method should only be called when
handling an exception.
:param str detail: Extra information appended to the filename
"""
basename = "traceback"
if detail:
basename += "-" + detail
tb_path = self.paths.log.log_file("global", basename)
self.log_error("Extended traceback in: %s", tb_path)
with open(tb_path, "wb") as f:
f.write(kobo.tback.Traceback().get_traceback())
def get_ordered_variant_uids(compose): def get_ordered_variant_uids(compose):
if not hasattr(compose, "_ordered_variant_uids"): if not hasattr(compose, "_ordered_variant_uids"):

View File

@ -730,6 +730,10 @@ def _gather_variants(
try: try:
que.put((arch, gather_packages(*args, **kwargs))) que.put((arch, gather_packages(*args, **kwargs)))
except Exception as exc: except Exception as exc:
compose.log_error(
"Error in gathering for %s.%s: %s", variant, arch, exc
)
compose.traceback("gather-%s-%s" % (variant, arch))
errors.put(exc) errors.put(exc)
# Run gather_packages() in parallel with multi threads and store # Run gather_packages() in parallel with multi threads and store

View File

@ -637,15 +637,10 @@ def cli_main():
main() main()
except (Exception, KeyboardInterrupt) as ex: except (Exception, KeyboardInterrupt) as ex:
if COMPOSE: if COMPOSE:
tb_path = COMPOSE.paths.log.log_file("global", "traceback")
COMPOSE.log_error("Compose run failed: %s" % ex) COMPOSE.log_error("Compose run failed: %s" % ex)
COMPOSE.log_error("Extended traceback in: %s" % tb_path) COMPOSE.traceback()
COMPOSE.log_critical("Compose failed: %s" % COMPOSE.topdir) COMPOSE.log_critical("Compose failed: %s" % COMPOSE.topdir)
COMPOSE.write_status("DOOMED") COMPOSE.write_status("DOOMED")
import kobo.tback
with open(tb_path, "wb") as f:
f.write(kobo.tback.Traceback().get_traceback())
else: else:
print("Exception: %s" % ex) print("Exception: %s" % ex)
raise raise

View File

@ -777,3 +777,33 @@ class DumpContainerMetadataTest(unittest.TestCase):
def test_dump_empty_metadata(self, ThreadPool): def test_dump_empty_metadata(self, ThreadPool):
self.compose.dump_containers_metadata() self.compose.dump_containers_metadata()
self.assertFalse(os.path.isfile(self.tmp_dir + "/compose/metadata/osbs.json")) self.assertFalse(os.path.isfile(self.tmp_dir + "/compose/metadata/osbs.json"))
class TracebackTest(unittest.TestCase):
def setUp(self):
self.tmp_dir = tempfile.mkdtemp()
with mock.patch("pungi.compose.ComposeInfo"):
self.compose = Compose({}, self.tmp_dir)
self.patcher = mock.patch("kobo.tback.Traceback")
self.Traceback = self.patcher.start()
self.Traceback.return_value.get_traceback.return_value = b"traceback"
def tearDown(self):
shutil.rmtree(self.tmp_dir)
self.patcher.stop()
def assertTraceback(self, filename):
self.assertTrue(
os.path.isfile("%s/logs/global/%s.global.log" % (self.tmp_dir, filename))
)
self.assertEqual(
self.Traceback.mock_calls, [mock.call(), mock.call().get_traceback()]
)
def test_traceback_default(self):
self.compose.traceback()
self.assertTraceback("traceback")
def test_with_detail(self):
self.compose.traceback("extra-info")
self.assertTraceback("traceback-extra-info")