almalinux/pungi
14
1
Fork 0
Code Issues Pull Requests Releases Activity

gather: Copy old logs when reusing gather result

Browse Source 
This would be helpful for debugging.

JIRA: RHELCMP-4594
Signed-off-by: Haibo Lin <hlin@redhat.com>
This commit is contained in:
Haibo Lin 2021-04-01 11:18:18 +08:00
parent 035b37c566
commit c8091899b2
2 changed files with 42 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

30
pungi/phases/gather/__init__.py
View File

@ -15,6 +15,7 @@
import json import json
import glob
import os import os
import shutil import shutil
import threading import threading
@ -192,7 +193,7 @@ def load_old_compose_config(compose):
return old_config return old_config
def reuse_old_gather_packages(compose, arch, variant, package_sets): def reuse_old_gather_packages(compose, arch, variant, package_sets, methods):
""" """
Tries to reuse `gather_packages` result from older compose. Tries to reuse `gather_packages` result from older compose.
@ -200,6 +201,7 @@ def reuse_old_gather_packages(compose, arch, variant, package_sets):
:param str arch: Architecture to reuse old gather data for. :param str arch: Architecture to reuse old gather data for.
:param str variant: Variant to reuse old gather data for. :param str variant: Variant to reuse old gather data for.
:param list package_sets: List of package sets to gather packages from. :param list package_sets: List of package sets to gather packages from.
:param str methods: Gather method.
:return: Old `gather_packages` result or None if old result cannot be used. :return: Old `gather_packages` result or None if old result cannot be used.
""" """
log_msg = "Cannot reuse old GATHER phase results - %s" log_msg = "Cannot reuse old GATHER phase results - %s"
@ -372,6 +374,28 @@ def reuse_old_gather_packages(compose, arch, variant, package_sets):
compose.log_info(log_msg % "some RPMs have been removed.") compose.log_info(log_msg % "some RPMs have been removed.")
return return
# Copy old gather log for debugging
try:
if methods == "hybrid":
log_dir = compose.paths.log.topdir(arch, create_dir=False)
old_log_dir = compose.paths.old_compose_path(log_dir)
for log_file in glob.glob(
os.path.join(old_log_dir, "hybrid-depsolver-%s-iter-*" % variant)
):
compose.log_info(
"Copying old gather log %s to %s" % (log_file, log_dir)
)
shutil.copy2(log_file, log_dir)
else:
log_dir = os.path.dirname(
compose.paths.work.pungi_log(arch, variant, create_dir=False)
)
old_log_dir = compose.paths.old_compose_path(log_dir)
compose.log_info("Copying old gather log %s to %s" % (old_log_dir, log_dir))
shutil.copytree(old_log_dir, log_dir)
except Exception as e:
compose.log_warning("Copying old gather log failed: %s" % str(e))
return result return result
@ -398,7 +422,9 @@ def gather_packages(compose, arch, variant, package_sets, fulltree_excludes=None
prepopulate = get_prepopulate_packages(compose, arch, variant) prepopulate = get_prepopulate_packages(compose, arch, variant)
fulltree_excludes = fulltree_excludes or set() fulltree_excludes = fulltree_excludes or set()
reused_result = reuse_old_gather_packages(compose, arch, variant, package_sets) reused_result = reuse_old_gather_packages(
compose, arch, variant, package_sets, methods
)
if reused_result: if reused_result:
result = reused_result result = reused_result
elif methods == "hybrid": elif methods == "hybrid":

28
tests/test_gather_phase.py
View File

@ -1086,7 +1086,7 @@ class TestReuseOldGatherPackages(helpers.PungiTestCase):
compose = helpers.DummyCompose(self.topdir, {"gather_allow_reuse": True}) compose = helpers.DummyCompose(self.topdir, {"gather_allow_reuse": True})
result = gather.reuse_old_gather_packages( result = gather.reuse_old_gather_packages(
compose, "x86_64", compose.variants["Server"], [] compose, "x86_64", compose.variants["Server"], [], "deps"
) )
self.assertEqual(result, None) self.assertEqual(result, None)
@ -1105,7 +1105,7 @@ class TestReuseOldGatherPackages(helpers.PungiTestCase):
load_old_compose_config.return_value = None load_old_compose_config.return_value = None
result = gather.reuse_old_gather_packages( result = gather.reuse_old_gather_packages(
compose, "x86_64", compose.variants["Server"], [] compose, "x86_64", compose.variants["Server"], [], "deps"
) )
self.assertEqual(result, None) self.assertEqual(result, None)
@ -1126,7 +1126,7 @@ class TestReuseOldGatherPackages(helpers.PungiTestCase):
load_old_compose_config.return_value = compose_conf_copy load_old_compose_config.return_value = compose_conf_copy
result = gather.reuse_old_gather_packages( result = gather.reuse_old_gather_packages(
compose, "x86_64", compose.variants["Server"], [] compose, "x86_64", compose.variants["Server"], [], "nodeps"
) )
self.assertEqual(result, None) self.assertEqual(result, None)
@ -1148,7 +1148,7 @@ class TestReuseOldGatherPackages(helpers.PungiTestCase):
load_old_compose_config.return_value = compose_conf_copy load_old_compose_config.return_value = compose_conf_copy
result = gather.reuse_old_gather_packages( result = gather.reuse_old_gather_packages(
compose, "x86_64", compose.variants["Server"], [] compose, "x86_64", compose.variants["Server"], [], "deps"
) )
self.assertEqual(result, {"rpm": [], "srpm": [], "debuginfo": []}) self.assertEqual(result, {"rpm": [], "srpm": [], "debuginfo": []})
@ -1182,7 +1182,7 @@ class TestReuseOldGatherPackages(helpers.PungiTestCase):
load_old_compose_config.return_value = compose.conf load_old_compose_config.return_value = compose.conf
result = gather.reuse_old_gather_packages( result = gather.reuse_old_gather_packages(
compose, "x86_64", compose.variants["Server"], package_sets compose, "x86_64", compose.variants["Server"], package_sets, "deps"
) )
self.assertEqual( self.assertEqual(
result, result,
@ -1206,7 +1206,7 @@ class TestReuseOldGatherPackages(helpers.PungiTestCase):
gather._update_config(compose, "Server", "x86_64", compose.topdir) gather._update_config(compose, "Server", "x86_64", compose.topdir)
result = gather.reuse_old_gather_packages( result = gather.reuse_old_gather_packages(
compose, "x86_64", compose.variants["Server"], package_sets compose, "x86_64", compose.variants["Server"], package_sets, "deps"
) )
self.assertEqual( self.assertEqual(
result, result,
@ -1232,7 +1232,7 @@ class TestReuseOldGatherPackages(helpers.PungiTestCase):
gather._update_config(compose, "Server", "x86_64", compose.topdir) gather._update_config(compose, "Server", "x86_64", compose.topdir)
result = gather.reuse_old_gather_packages( result = gather.reuse_old_gather_packages(
compose, "x86_64", compose.variants["Server"], package_sets compose, "x86_64", compose.variants["Server"], package_sets, "deps"
) )
self.assertEqual(result, None) self.assertEqual(result, None)
@ -1252,7 +1252,7 @@ class TestReuseOldGatherPackages(helpers.PungiTestCase):
gather._update_config(compose, "Server", "x86_64", compose.topdir) gather._update_config(compose, "Server", "x86_64", compose.topdir)
result = gather.reuse_old_gather_packages( result = gather.reuse_old_gather_packages(
compose, "x86_64", compose.variants["Server"], package_sets compose, "x86_64", compose.variants["Server"], package_sets, "deps"
) )
self.assertEqual(result, None) self.assertEqual(result, None)
@ -1271,7 +1271,7 @@ class TestReuseOldGatherPackages(helpers.PungiTestCase):
load_old_compose_config.return_value = compose.conf load_old_compose_config.return_value = compose.conf
result = gather.reuse_old_gather_packages( result = gather.reuse_old_gather_packages(
compose, "x86_64", compose.variants["Server"], package_sets compose, "x86_64", compose.variants["Server"], package_sets, "deps"
) )
self.assertEqual(result, None) self.assertEqual(result, None)
@ -1293,7 +1293,7 @@ class TestReuseOldGatherPackages(helpers.PungiTestCase):
load_old_compose_config.return_value = compose.conf load_old_compose_config.return_value = compose.conf
result = gather.reuse_old_gather_packages( result = gather.reuse_old_gather_packages(
compose, "x86_64", compose.variants["Server"], package_sets compose, "x86_64", compose.variants["Server"], package_sets, "deps"
) )
self.assertEqual(result, None) self.assertEqual(result, None)
@ -1318,7 +1318,7 @@ class TestReuseOldGatherPackages(helpers.PungiTestCase):
load_old_compose_config.return_value = compose.conf load_old_compose_config.return_value = compose.conf
result = gather.reuse_old_gather_packages( result = gather.reuse_old_gather_packages(
compose, "x86_64", compose.variants["Server"], package_sets compose, "x86_64", compose.variants["Server"], package_sets, "deps"
) )
self.assertEqual(result, None) self.assertEqual(result, None)
@ -1335,7 +1335,7 @@ class TestReuseOldGatherPackages(helpers.PungiTestCase):
load_old_compose_config.return_value = compose.conf load_old_compose_config.return_value = compose.conf
result = gather.reuse_old_gather_packages( result = gather.reuse_old_gather_packages(
compose, "x86_64", compose.variants["Server"], package_sets compose, "x86_64", compose.variants["Server"], package_sets, "deps"
) )
self.assertEqual(result, None) self.assertEqual(result, None)
@ -1351,7 +1351,7 @@ class TestReuseOldGatherPackages(helpers.PungiTestCase):
load_old_compose_config.return_value = compose.conf load_old_compose_config.return_value = compose.conf
result = gather.reuse_old_gather_packages( result = gather.reuse_old_gather_packages(
compose, "x86_64", compose.variants["Server"], package_sets compose, "x86_64", compose.variants["Server"], package_sets, "deps"
) )
self.assertEqual(result, None) self.assertEqual(result, None)
@ -1367,7 +1367,7 @@ class TestReuseOldGatherPackages(helpers.PungiTestCase):
load_old_compose_config.return_value = compose.conf load_old_compose_config.return_value = compose.conf
result = gather.reuse_old_gather_packages( result = gather.reuse_old_gather_packages(
compose, "x86_64", compose.variants["Server"], package_sets compose, "x86_64", compose.variants["Server"], package_sets, "deps"
) )
self.assertEqual(result, None) self.assertEqual(result, None)