Merge #232 Improve logging by adding subvariants

This commit is contained in:
Dennis Gilmore 2016-03-22 14:15:26 +00:00
commit 6230b0ff3e
4 changed files with 49 additions and 28 deletions

View File

@ -4,6 +4,7 @@ import copy
import os import os
import time import time
from kobo import shortcuts from kobo import shortcuts
import traceback
from pungi.util import get_variant_data, resolve_git_url, makedirs, get_mtime, get_file_size from pungi.util import get_variant_data, resolve_git_url, makedirs, get_mtime, get_file_size
from pungi.phases.base import PhaseBase from pungi.phases.base import PhaseBase
@ -142,35 +143,38 @@ class CreateImageBuildThread(WorkerThread):
def process(self, item, num): def process(self, item, num):
compose, cmd = item compose, cmd = item
variant = cmd["image_conf"]["image-build"]["variant"]
subvariant = cmd["image_conf"]["image-build"].get("subvariant", variant.uid)
try: try:
self.worker(num, compose, cmd) self.worker(num, compose, variant, subvariant, cmd)
except Exception as exc: except Exception as exc:
if not compose.can_fail(cmd["image_conf"]["image-build"]['variant'], '*', 'image-build'): if not compose.can_fail(cmd["image_conf"]["image-build"]['variant'], '*', 'image-build'):
raise raise
else: else:
msg = ('[FAIL] image-build for variant %s failed, but going on anyway.\n%s' msg = ('[FAIL] image-build for variant %s (%s) failed, but going on anyway.\n%s'
% (cmd['image_conf']['image-build']['variant'], exc)) % (variant.uid, subvariant, exc))
self.pool.log_info(msg) self.pool.log_info(msg)
tb = traceback.format_exc()
self.pool.log_debug(tb)
def worker(self, num, compose, cmd): def worker(self, num, compose, variant, subvariant, cmd):
arches = cmd["image_conf"]["image-build"]['arches'].split(',') arches = cmd["image_conf"]["image-build"]['arches'].split(',')
dash_arches = '-'.join(arches) dash_arches = '-'.join(arches)
log_file = compose.paths.log.log_file( log_file = compose.paths.log.log_file(
dash_arches, dash_arches,
"imagebuild-%s-%s-%s" % (dash_arches, "imagebuild-%s-%s-%s" % (variant.uid, subvariant,
cmd["image_conf"]["image-build"]["variant"],
cmd["image_conf"]["image-build"]['format'].replace(",", "-")) cmd["image_conf"]["image-build"]['format'].replace(",", "-"))
) )
msg = "Creating %s image (arches: %s, variant: %s)" % (cmd["image_conf"]["image-build"]["format"].replace(",", "-"), msg = ("Creating %s image (arches: %s, variant: %s, subvariant: %s)"
dash_arches, % (cmd["image_conf"]["image-build"]["format"].replace(",", "-"),
cmd["image_conf"]["image-build"]["variant"]) dash_arches, variant, subvariant))
self.pool.log_info("[BEGIN] %s" % msg) self.pool.log_info("[BEGIN] %s" % msg)
koji_wrapper = KojiWrapper(compose.conf["koji_profile"]) koji_wrapper = KojiWrapper(compose.conf["koji_profile"])
# writes conf file for koji image-build # writes conf file for koji image-build
self.pool.log_info("Writing image-build config for %s.%s into %s" % ( self.pool.log_info("Writing image-build config for %s.%s into %s" % (
cmd["image_conf"]["image-build"]["variant"], dash_arches, cmd["conf_file"])) variant, dash_arches, cmd["conf_file"]))
koji_cmd = koji_wrapper.get_image_build_cmd(cmd["image_conf"], koji_cmd = koji_wrapper.get_image_build_cmd(cmd["image_conf"],
conf_file_dest=cmd["conf_file"], conf_file_dest=cmd["conf_file"],
scratch=cmd['scratch']) scratch=cmd['scratch'])
@ -182,7 +186,8 @@ class CreateImageBuildThread(WorkerThread):
self.pool.log_debug("build-image outputs: %s" % (output)) self.pool.log_debug("build-image outputs: %s" % (output))
if output["retcode"] != 0: if output["retcode"] != 0:
self.fail(compose, cmd) self.fail(compose, cmd)
raise RuntimeError("ImageBuild task failed: %s. See %s for more details." % (output["task_id"], log_file)) raise RuntimeError("ImageBuild task failed: %s. See %s for more details."
% (output["task_id"], log_file))
# copy image to images/ # copy image to images/
image_infos = [] image_infos = []
@ -229,8 +234,7 @@ class CreateImageBuildThread(WorkerThread):
img.disc_number = 1 # We don't expect multiple disks img.disc_number = 1 # We don't expect multiple disks
img.disc_count = 1 img.disc_count = 1
img.bootable = False img.bootable = False
varname = cmd["image_conf"]["image-build"]["variant"].uid img.subvariant = subvariant
img.subvariant = cmd["image_conf"]["image-build"].get("subvariant", varname) compose.im.add(variant=variant.uid, arch=image_info['arch'], image=img)
compose.im.add(variant=varname, arch=image_info['arch'], image=img)
self.pool.log_info("[DONE ] %s" % msg) self.pool.log_info("[DONE ] %s" % msg)

View File

@ -157,22 +157,24 @@ class LiveMediaPhase(PhaseBase):
class LiveMediaThread(WorkerThread): class LiveMediaThread(WorkerThread):
def process(self, item, num): def process(self, item, num):
compose, variant, config = item compose, variant, config = item
subvariant = config.pop('subvariant')
self.num = num self.num = num
try: try:
self.worker(compose, variant, config) self.worker(compose, variant, subvariant, config)
except Exception as exc: except Exception as exc:
if not compose.can_fail(variant, '*', 'live-media'): if not compose.can_fail(variant, '*', 'live-media'):
raise raise
else: else:
msg = ('[FAIL] live-media for variant %s failed, but going on anyway.\n%s' msg = ('[FAIL] live-media for variant %s (%s) failed, but going on anyway.\n%s'
% (variant.uid, exc)) % (variant.uid, subvariant, exc))
self.pool.log_info(msg) self.pool.log_info(msg)
tb = traceback.format_exc() tb = traceback.format_exc()
self.pool.log_debug(tb) self.pool.log_debug(tb)
def _get_log_file(self, compose, variant, config): def _get_log_file(self, compose, variant, subvariant, config):
arches = '-'.join(config['arches']) arches = '-'.join(config['arches'])
return compose.paths.log.log_file(arches, 'livemedia-%s' % variant.uid) return compose.paths.log.log_file(arches, 'livemedia-%s-%s'
% (variant.uid, subvariant))
def _run_command(self, koji_wrapper, cmd, compose, log_file): def _run_command(self, koji_wrapper, cmd, compose, log_file):
time.sleep(self.num * 3) time.sleep(self.num * 3)
@ -190,17 +192,15 @@ class LiveMediaThread(WorkerThread):
copy['arch'] = ','.join(copy.pop('arches', [])) copy['arch'] = ','.join(copy.pop('arches', []))
return koji_wrapper.get_live_media_cmd(copy) return koji_wrapper.get_live_media_cmd(copy)
def worker(self, compose, variant, config): def worker(self, compose, variant, subvariant, config):
msg = 'Live media: %s (arches: %s, variant: %s)' % (config['name'], msg = ('Live media: %s (arches: %s, variant: %s, subvariant: %s)'
' '.join(config['arches']), % (config['name'], ' '.join(config['arches']), variant.uid, subvariant))
variant.uid)
self.pool.log_info('[BEGIN] %s' % msg) self.pool.log_info('[BEGIN] %s' % msg)
subvariant = config.pop('subvariant')
koji_wrapper = KojiWrapper(compose.conf['koji_profile']) koji_wrapper = KojiWrapper(compose.conf['koji_profile'])
cmd = self._get_cmd(koji_wrapper, config) cmd = self._get_cmd(koji_wrapper, config)
log_file = self._get_log_file(compose, variant, config) log_file = self._get_log_file(compose, variant, subvariant, config)
output = self._run_command(koji_wrapper, cmd, compose, log_file) output = self._run_command(koji_wrapper, cmd, compose, log_file)
# collect results and update manifest # collect results and update manifest

View File

@ -388,6 +388,7 @@ class TestCreateImageBuildThread(PungiTestCase):
'version': 'Rawhide', 'version': 'Rawhide',
'ksurl': 'git://git.fedorahosted.org/git/spin-kickstarts.git', 'ksurl': 'git://git.fedorahosted.org/git/spin-kickstarts.git',
'distro': 'Fedora-20', 'distro': 'Fedora-20',
'subvariant': 'KDE',
} }
}, },
"conf_file": 'amd64,x86_64-Client-Fedora-Docker-Base-docker', "conf_file": 'amd64,x86_64-Client-Fedora-Docker-Base-docker',
@ -423,6 +424,19 @@ class TestCreateImageBuildThread(PungiTestCase):
with mock.patch('time.sleep'): with mock.patch('time.sleep'):
t.process((compose, cmd), 1) t.process((compose, cmd), 1)
self.assertItemsEqual(
koji_wrapper.get_image_build_cmd.call_args_list,
[mock.call(cmd['image_conf'],
conf_file_dest='amd64,x86_64-Client-Fedora-Docker-Base-docker',
scratch=False)]
)
self.assertItemsEqual(
koji_wrapper.run_blocking_cmd.call_args_list,
[mock.call(koji_wrapper.get_image_build_cmd.return_value,
log_file=self.topdir + '/logs/amd64-x86_64/imagebuild-Client-KDE-docker.amd64-x86_64.log')]
)
self.assertItemsEqual( self.assertItemsEqual(
linker.mock_calls, linker.mock_calls,
[mock.call('/koji/task/1235/Fedora-Docker-Base-20160103.amd64.qcow2', [mock.call('/koji/task/1235/Fedora-Docker-Base-20160103.amd64.qcow2',
@ -473,6 +487,7 @@ class TestCreateImageBuildThread(PungiTestCase):
data = image_relative_paths.pop(image.path) data = image_relative_paths.pop(image.path)
self.assertEqual(data['format'], image.format) self.assertEqual(data['format'], image.format)
self.assertEqual(data['type'], image.type) self.assertEqual(data['type'], image.type)
self.assertEqual('KDE', image.subvariant)
self.assertTrue(os.path.isdir(self.topdir + '/compose/Client/amd64/images')) self.assertTrue(os.path.isdir(self.topdir + '/compose/Client/amd64/images'))
self.assertTrue(os.path.isdir(self.topdir + '/compose/Client/x86_64/images')) self.assertTrue(os.path.isdir(self.topdir + '/compose/Client/x86_64/images'))

View File

@ -296,7 +296,7 @@ class TestLiveMediaThread(PungiTestCase):
'target': 'f24', 'target': 'f24',
'title': None, 'title': None,
'version': 'Rawhide', 'version': 'Rawhide',
'subvariant': 'DATA', 'subvariant': 'KDE',
} }
pool = mock.Mock() pool = mock.Mock()
@ -333,7 +333,7 @@ class TestLiveMediaThread(PungiTestCase):
self.assertEqual( self.assertEqual(
run_blocking_cmd.mock_calls, run_blocking_cmd.mock_calls,
[mock.call('koji-spin-livemedia', [mock.call('koji-spin-livemedia',
log_file=self.topdir + '/logs/amd64-x86_64/livemedia-Server.amd64-x86_64.log')]) log_file=self.topdir + '/logs/amd64-x86_64/livemedia-Server-KDE.amd64-x86_64.log')])
self.assertEqual(get_live_media_cmd.mock_calls, self.assertEqual(get_live_media_cmd.mock_calls,
[mock.call({'arch': 'amd64,x86_64', [mock.call({'arch': 'amd64,x86_64',
'ksfile': 'file.ks', 'ksfile': 'file.ks',
@ -375,7 +375,7 @@ class TestLiveMediaThread(PungiTestCase):
self.assertIn(image.path, image_relative_paths) self.assertIn(image.path, image_relative_paths)
self.assertEqual('iso', image.format) self.assertEqual('iso', image.format)
self.assertEqual('live', image.type) self.assertEqual('live', image.type)
self.assertEqual('DATA', image.subvariant) self.assertEqual('KDE', image.subvariant)
@mock.patch('pungi.phases.livemedia_phase.KojiWrapper') @mock.patch('pungi.phases.livemedia_phase.KojiWrapper')
def test_handle_koji_fail(self, KojiWrapper): def test_handle_koji_fail(self, KojiWrapper):
@ -398,6 +398,7 @@ class TestLiveMediaThread(PungiTestCase):
'target': 'f24', 'target': 'f24',
'title': None, 'title': None,
'version': 'Rawhide', 'version': 'Rawhide',
'subvariant': 'KDE',
} }
pool = mock.Mock() pool = mock.Mock()
@ -437,6 +438,7 @@ class TestLiveMediaThread(PungiTestCase):
'target': 'f24', 'target': 'f24',
'title': None, 'title': None,
'version': 'Rawhide', 'version': 'Rawhide',
'subvariant': 'KDE',
} }
pool = mock.Mock() pool = mock.Mock()