From f1974d3f03f7320f74609b34b701cc5ad0ead7ba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lubom=C3=ADr=20Sedl=C3=A1=C5=99?= Date: Thu, 17 Mar 2016 09:02:17 +0100 Subject: [PATCH 1/3] [live-media] Use subvariants in logging MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Print the subvariant together with variant UID and arches. This should make the global log easier to understand. The output of koji command should now be split into multiple files in logs/{arch}/livemedia-{variant}-{subvariant}.{arch}.log instead of having them all in one file. The tests were updated to use more sensible value for the subvariant field. Signed-off-by: Lubomír Sedlář --- pungi/phases/livemedia_phase.py | 22 +++++++++++----------- tests/test_livemediaphase.py | 8 +++++--- 2 files changed, 16 insertions(+), 14 deletions(-) diff --git a/pungi/phases/livemedia_phase.py b/pungi/phases/livemedia_phase.py index 087a893b..92804744 100644 --- a/pungi/phases/livemedia_phase.py +++ b/pungi/phases/livemedia_phase.py @@ -157,22 +157,24 @@ class LiveMediaPhase(PhaseBase): class LiveMediaThread(WorkerThread): def process(self, item, num): compose, variant, config = item + subvariant = config.pop('subvariant') self.num = num try: - self.worker(compose, variant, config) + self.worker(compose, variant, subvariant, config) except Exception as exc: if not compose.can_fail(variant, '*', 'live-media'): raise else: - msg = ('[FAIL] live-media for variant %s failed, but going on anyway.\n%s' - % (variant.uid, exc)) + msg = ('[FAIL] live-media for variant %s (%s) failed, but going on anyway.\n%s' + % (variant.uid, subvariant, exc)) self.pool.log_info(msg) tb = traceback.format_exc() self.pool.log_debug(tb) - def _get_log_file(self, compose, variant, config): + def _get_log_file(self, compose, variant, subvariant, config): arches = '-'.join(config['arches']) - return compose.paths.log.log_file(arches, 'livemedia-%s' % variant.uid) + return compose.paths.log.log_file(arches, 'livemedia-%s-%s' + % (variant.uid, subvariant)) def _run_command(self, koji_wrapper, cmd, compose, log_file): time.sleep(self.num * 3) @@ -190,17 +192,15 @@ class LiveMediaThread(WorkerThread): copy['arch'] = ','.join(copy.pop('arches', [])) return koji_wrapper.get_live_media_cmd(copy) - def worker(self, compose, variant, config): - msg = 'Live media: %s (arches: %s, variant: %s)' % (config['name'], - ' '.join(config['arches']), - variant.uid) + def worker(self, compose, variant, subvariant, config): + msg = ('Live media: %s (arches: %s, variant: %s, subvariant: %s)' + % (config['name'], ' '.join(config['arches']), variant.uid, subvariant)) self.pool.log_info('[BEGIN] %s' % msg) - subvariant = config.pop('subvariant') koji_wrapper = KojiWrapper(compose.conf['koji_profile']) cmd = self._get_cmd(koji_wrapper, config) - log_file = self._get_log_file(compose, variant, config) + log_file = self._get_log_file(compose, variant, subvariant, config) output = self._run_command(koji_wrapper, cmd, compose, log_file) # collect results and update manifest diff --git a/tests/test_livemediaphase.py b/tests/test_livemediaphase.py index 7983068d..a9c1a404 100755 --- a/tests/test_livemediaphase.py +++ b/tests/test_livemediaphase.py @@ -296,7 +296,7 @@ class TestLiveMediaThread(PungiTestCase): 'target': 'f24', 'title': None, 'version': 'Rawhide', - 'subvariant': 'DATA', + 'subvariant': 'KDE', } pool = mock.Mock() @@ -333,7 +333,7 @@ class TestLiveMediaThread(PungiTestCase): self.assertEqual( run_blocking_cmd.mock_calls, [mock.call('koji-spin-livemedia', - log_file=self.topdir + '/logs/amd64-x86_64/livemedia-Server.amd64-x86_64.log')]) + log_file=self.topdir + '/logs/amd64-x86_64/livemedia-Server-KDE.amd64-x86_64.log')]) self.assertEqual(get_live_media_cmd.mock_calls, [mock.call({'arch': 'amd64,x86_64', 'ksfile': 'file.ks', @@ -375,7 +375,7 @@ class TestLiveMediaThread(PungiTestCase): self.assertIn(image.path, image_relative_paths) self.assertEqual('iso', image.format) self.assertEqual('live', image.type) - self.assertEqual('DATA', image.subvariant) + self.assertEqual('KDE', image.subvariant) @mock.patch('pungi.phases.livemedia_phase.KojiWrapper') def test_handle_koji_fail(self, KojiWrapper): @@ -398,6 +398,7 @@ class TestLiveMediaThread(PungiTestCase): 'target': 'f24', 'title': None, 'version': 'Rawhide', + 'subvariant': 'KDE', } pool = mock.Mock() @@ -437,6 +438,7 @@ class TestLiveMediaThread(PungiTestCase): 'target': 'f24', 'title': None, 'version': 'Rawhide', + 'subvariant': 'KDE', } pool = mock.Mock() From ea7f98ef4ec77ebd680c8fd8747a00904b6c25d4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lubom=C3=ADr=20Sedl=C3=A1=C5=99?= Date: Thu, 17 Mar 2016 09:21:35 +0100 Subject: [PATCH 2/3] [image-build] Use subvariants in logging output MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Lubomír Sedlář --- pungi/phases/image_build.py | 29 +++++++++++++++-------------- tests/test_imagebuildphase.py | 15 +++++++++++++++ 2 files changed, 30 insertions(+), 14 deletions(-) diff --git a/pungi/phases/image_build.py b/pungi/phases/image_build.py index badf53ef..a08c9828 100644 --- a/pungi/phases/image_build.py +++ b/pungi/phases/image_build.py @@ -142,35 +142,36 @@ class CreateImageBuildThread(WorkerThread): def process(self, item, num): compose, cmd = item + variant = cmd["image_conf"]["image-build"]["variant"] + subvariant = cmd["image_conf"]["image-build"].get("subvariant", variant.uid) try: - self.worker(num, compose, cmd) + self.worker(num, compose, variant, subvariant, cmd) except Exception as exc: if not compose.can_fail(cmd["image_conf"]["image-build"]['variant'], '*', 'image-build'): raise else: - msg = ('[FAIL] image-build for variant %s failed, but going on anyway.\n%s' - % (cmd['image_conf']['image-build']['variant'], exc)) + msg = ('[FAIL] image-build for variant %s (%s) failed, but going on anyway.\n%s' + % (variant.uid, subvariant, exc)) self.pool.log_info(msg) - def worker(self, num, compose, cmd): + def worker(self, num, compose, variant, subvariant, cmd): arches = cmd["image_conf"]["image-build"]['arches'].split(',') dash_arches = '-'.join(arches) log_file = compose.paths.log.log_file( dash_arches, - "imagebuild-%s-%s-%s" % (dash_arches, - cmd["image_conf"]["image-build"]["variant"], + "imagebuild-%s-%s-%s" % (variant.uid, subvariant, cmd["image_conf"]["image-build"]['format'].replace(",", "-")) ) - msg = "Creating %s image (arches: %s, variant: %s)" % (cmd["image_conf"]["image-build"]["format"].replace(",", "-"), - dash_arches, - cmd["image_conf"]["image-build"]["variant"]) + msg = ("Creating %s image (arches: %s, variant: %s, subvariant: %s)" + % (cmd["image_conf"]["image-build"]["format"].replace(",", "-"), + dash_arches, variant, subvariant)) self.pool.log_info("[BEGIN] %s" % msg) koji_wrapper = KojiWrapper(compose.conf["koji_profile"]) # writes conf file for koji image-build self.pool.log_info("Writing image-build config for %s.%s into %s" % ( - cmd["image_conf"]["image-build"]["variant"], dash_arches, cmd["conf_file"])) + variant, dash_arches, cmd["conf_file"])) koji_cmd = koji_wrapper.get_image_build_cmd(cmd["image_conf"], conf_file_dest=cmd["conf_file"], scratch=cmd['scratch']) @@ -182,7 +183,8 @@ class CreateImageBuildThread(WorkerThread): self.pool.log_debug("build-image outputs: %s" % (output)) if output["retcode"] != 0: self.fail(compose, cmd) - raise RuntimeError("ImageBuild task failed: %s. See %s for more details." % (output["task_id"], log_file)) + raise RuntimeError("ImageBuild task failed: %s. See %s for more details." + % (output["task_id"], log_file)) # copy image to images/ image_infos = [] @@ -229,8 +231,7 @@ class CreateImageBuildThread(WorkerThread): img.disc_number = 1 # We don't expect multiple disks img.disc_count = 1 img.bootable = False - varname = cmd["image_conf"]["image-build"]["variant"].uid - img.subvariant = cmd["image_conf"]["image-build"].get("subvariant", varname) - compose.im.add(variant=varname, arch=image_info['arch'], image=img) + img.subvariant = subvariant + compose.im.add(variant=variant.uid, arch=image_info['arch'], image=img) self.pool.log_info("[DONE ] %s" % msg) diff --git a/tests/test_imagebuildphase.py b/tests/test_imagebuildphase.py index 1ba2c9e4..a315daca 100755 --- a/tests/test_imagebuildphase.py +++ b/tests/test_imagebuildphase.py @@ -388,6 +388,7 @@ class TestCreateImageBuildThread(PungiTestCase): 'version': 'Rawhide', 'ksurl': 'git://git.fedorahosted.org/git/spin-kickstarts.git', 'distro': 'Fedora-20', + 'subvariant': 'KDE', } }, "conf_file": 'amd64,x86_64-Client-Fedora-Docker-Base-docker', @@ -423,6 +424,19 @@ class TestCreateImageBuildThread(PungiTestCase): with mock.patch('time.sleep'): t.process((compose, cmd), 1) + self.assertItemsEqual( + koji_wrapper.get_image_build_cmd.call_args_list, + [mock.call(cmd['image_conf'], + conf_file_dest='amd64,x86_64-Client-Fedora-Docker-Base-docker', + scratch=False)] + ) + + self.assertItemsEqual( + koji_wrapper.run_blocking_cmd.call_args_list, + [mock.call(koji_wrapper.get_image_build_cmd.return_value, + log_file=self.topdir + '/logs/amd64-x86_64/imagebuild-Client-KDE-docker.amd64-x86_64.log')] + ) + self.assertItemsEqual( linker.mock_calls, [mock.call('/koji/task/1235/Fedora-Docker-Base-20160103.amd64.qcow2', @@ -473,6 +487,7 @@ class TestCreateImageBuildThread(PungiTestCase): data = image_relative_paths.pop(image.path) self.assertEqual(data['format'], image.format) self.assertEqual(data['type'], image.type) + self.assertEqual('KDE', image.subvariant) self.assertTrue(os.path.isdir(self.topdir + '/compose/Client/amd64/images')) self.assertTrue(os.path.isdir(self.topdir + '/compose/Client/x86_64/images')) From 1a1cfc65cef7eb742f8c92a76da537e701094169 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lubom=C3=ADr=20Sedl=C3=A1=C5=99?= Date: Thu, 17 Mar 2016 10:21:06 +0100 Subject: [PATCH 3/3] [image-build] Add traceback on failure MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Lubomír Sedlář --- pungi/phases/image_build.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pungi/phases/image_build.py b/pungi/phases/image_build.py index a08c9828..bb20c447 100644 --- a/pungi/phases/image_build.py +++ b/pungi/phases/image_build.py @@ -4,6 +4,7 @@ import copy import os import time from kobo import shortcuts +import traceback from pungi.util import get_variant_data, resolve_git_url, makedirs, get_mtime, get_file_size from pungi.phases.base import PhaseBase @@ -153,6 +154,8 @@ class CreateImageBuildThread(WorkerThread): msg = ('[FAIL] image-build for variant %s (%s) failed, but going on anyway.\n%s' % (variant.uid, subvariant, exc)) self.pool.log_info(msg) + tb = traceback.format_exc() + self.pool.log_debug(tb) def worker(self, num, compose, variant, subvariant, cmd): arches = cmd["image_conf"]["image-build"]['arches'].split(',')