From c8091899b2f0eeabeb270f75c8422393ba1180e5 Mon Sep 17 00:00:00 2001 From: Haibo Lin Date: Thu, 1 Apr 2021 11:18:18 +0800 Subject: [PATCH] gather: Copy old logs when reusing gather result This would be helpful for debugging. JIRA: RHELCMP-4594 Signed-off-by: Haibo Lin --- pungi/phases/gather/__init__.py | 30 ++++++++++++++++++++++++++++-- tests/test_gather_phase.py | 28 ++++++++++++++-------------- 2 files changed, 42 insertions(+), 16 deletions(-) diff --git a/pungi/phases/gather/__init__.py b/pungi/phases/gather/__init__.py index 7f77f15a..b95f6e46 100644 --- a/pungi/phases/gather/__init__.py +++ b/pungi/phases/gather/__init__.py @@ -15,6 +15,7 @@ import json +import glob import os import shutil import threading @@ -192,7 +193,7 @@ def load_old_compose_config(compose): return old_config -def reuse_old_gather_packages(compose, arch, variant, package_sets): +def reuse_old_gather_packages(compose, arch, variant, package_sets, methods): """ Tries to reuse `gather_packages` result from older compose. @@ -200,6 +201,7 @@ def reuse_old_gather_packages(compose, arch, variant, package_sets): :param str arch: Architecture to reuse old gather data for. :param str variant: Variant to reuse old gather data for. :param list package_sets: List of package sets to gather packages from. + :param str methods: Gather method. :return: Old `gather_packages` result or None if old result cannot be used. """ log_msg = "Cannot reuse old GATHER phase results - %s" @@ -372,6 +374,28 @@ def reuse_old_gather_packages(compose, arch, variant, package_sets): compose.log_info(log_msg % "some RPMs have been removed.") return + # Copy old gather log for debugging + try: + if methods == "hybrid": + log_dir = compose.paths.log.topdir(arch, create_dir=False) + old_log_dir = compose.paths.old_compose_path(log_dir) + for log_file in glob.glob( + os.path.join(old_log_dir, "hybrid-depsolver-%s-iter-*" % variant) + ): + compose.log_info( + "Copying old gather log %s to %s" % (log_file, log_dir) + ) + shutil.copy2(log_file, log_dir) + else: + log_dir = os.path.dirname( + compose.paths.work.pungi_log(arch, variant, create_dir=False) + ) + old_log_dir = compose.paths.old_compose_path(log_dir) + compose.log_info("Copying old gather log %s to %s" % (old_log_dir, log_dir)) + shutil.copytree(old_log_dir, log_dir) + except Exception as e: + compose.log_warning("Copying old gather log failed: %s" % str(e)) + return result @@ -398,7 +422,9 @@ def gather_packages(compose, arch, variant, package_sets, fulltree_excludes=None prepopulate = get_prepopulate_packages(compose, arch, variant) fulltree_excludes = fulltree_excludes or set() - reused_result = reuse_old_gather_packages(compose, arch, variant, package_sets) + reused_result = reuse_old_gather_packages( + compose, arch, variant, package_sets, methods + ) if reused_result: result = reused_result elif methods == "hybrid": diff --git a/tests/test_gather_phase.py b/tests/test_gather_phase.py index e120fea2..5ade1c26 100644 --- a/tests/test_gather_phase.py +++ b/tests/test_gather_phase.py @@ -1086,7 +1086,7 @@ class TestReuseOldGatherPackages(helpers.PungiTestCase): compose = helpers.DummyCompose(self.topdir, {"gather_allow_reuse": True}) result = gather.reuse_old_gather_packages( - compose, "x86_64", compose.variants["Server"], [] + compose, "x86_64", compose.variants["Server"], [], "deps" ) self.assertEqual(result, None) @@ -1105,7 +1105,7 @@ class TestReuseOldGatherPackages(helpers.PungiTestCase): load_old_compose_config.return_value = None result = gather.reuse_old_gather_packages( - compose, "x86_64", compose.variants["Server"], [] + compose, "x86_64", compose.variants["Server"], [], "deps" ) self.assertEqual(result, None) @@ -1126,7 +1126,7 @@ class TestReuseOldGatherPackages(helpers.PungiTestCase): load_old_compose_config.return_value = compose_conf_copy result = gather.reuse_old_gather_packages( - compose, "x86_64", compose.variants["Server"], [] + compose, "x86_64", compose.variants["Server"], [], "nodeps" ) self.assertEqual(result, None) @@ -1148,7 +1148,7 @@ class TestReuseOldGatherPackages(helpers.PungiTestCase): load_old_compose_config.return_value = compose_conf_copy result = gather.reuse_old_gather_packages( - compose, "x86_64", compose.variants["Server"], [] + compose, "x86_64", compose.variants["Server"], [], "deps" ) self.assertEqual(result, {"rpm": [], "srpm": [], "debuginfo": []}) @@ -1182,7 +1182,7 @@ class TestReuseOldGatherPackages(helpers.PungiTestCase): load_old_compose_config.return_value = compose.conf result = gather.reuse_old_gather_packages( - compose, "x86_64", compose.variants["Server"], package_sets + compose, "x86_64", compose.variants["Server"], package_sets, "deps" ) self.assertEqual( result, @@ -1206,7 +1206,7 @@ class TestReuseOldGatherPackages(helpers.PungiTestCase): gather._update_config(compose, "Server", "x86_64", compose.topdir) result = gather.reuse_old_gather_packages( - compose, "x86_64", compose.variants["Server"], package_sets + compose, "x86_64", compose.variants["Server"], package_sets, "deps" ) self.assertEqual( result, @@ -1232,7 +1232,7 @@ class TestReuseOldGatherPackages(helpers.PungiTestCase): gather._update_config(compose, "Server", "x86_64", compose.topdir) result = gather.reuse_old_gather_packages( - compose, "x86_64", compose.variants["Server"], package_sets + compose, "x86_64", compose.variants["Server"], package_sets, "deps" ) self.assertEqual(result, None) @@ -1252,7 +1252,7 @@ class TestReuseOldGatherPackages(helpers.PungiTestCase): gather._update_config(compose, "Server", "x86_64", compose.topdir) result = gather.reuse_old_gather_packages( - compose, "x86_64", compose.variants["Server"], package_sets + compose, "x86_64", compose.variants["Server"], package_sets, "deps" ) self.assertEqual(result, None) @@ -1271,7 +1271,7 @@ class TestReuseOldGatherPackages(helpers.PungiTestCase): load_old_compose_config.return_value = compose.conf result = gather.reuse_old_gather_packages( - compose, "x86_64", compose.variants["Server"], package_sets + compose, "x86_64", compose.variants["Server"], package_sets, "deps" ) self.assertEqual(result, None) @@ -1293,7 +1293,7 @@ class TestReuseOldGatherPackages(helpers.PungiTestCase): load_old_compose_config.return_value = compose.conf result = gather.reuse_old_gather_packages( - compose, "x86_64", compose.variants["Server"], package_sets + compose, "x86_64", compose.variants["Server"], package_sets, "deps" ) self.assertEqual(result, None) @@ -1318,7 +1318,7 @@ class TestReuseOldGatherPackages(helpers.PungiTestCase): load_old_compose_config.return_value = compose.conf result = gather.reuse_old_gather_packages( - compose, "x86_64", compose.variants["Server"], package_sets + compose, "x86_64", compose.variants["Server"], package_sets, "deps" ) self.assertEqual(result, None) @@ -1335,7 +1335,7 @@ class TestReuseOldGatherPackages(helpers.PungiTestCase): load_old_compose_config.return_value = compose.conf result = gather.reuse_old_gather_packages( - compose, "x86_64", compose.variants["Server"], package_sets + compose, "x86_64", compose.variants["Server"], package_sets, "deps" ) self.assertEqual(result, None) @@ -1351,7 +1351,7 @@ class TestReuseOldGatherPackages(helpers.PungiTestCase): load_old_compose_config.return_value = compose.conf result = gather.reuse_old_gather_packages( - compose, "x86_64", compose.variants["Server"], package_sets + compose, "x86_64", compose.variants["Server"], package_sets, "deps" ) self.assertEqual(result, None) @@ -1367,7 +1367,7 @@ class TestReuseOldGatherPackages(helpers.PungiTestCase): load_old_compose_config.return_value = compose.conf result = gather.reuse_old_gather_packages( - compose, "x86_64", compose.variants["Server"], package_sets + compose, "x86_64", compose.variants["Server"], package_sets, "deps" ) self.assertEqual(result, None)