Merge #232 Improve logging by adding subvariants
This commit is contained in:
commit
6230b0ff3e
@ -4,6 +4,7 @@ import copy
|
|||||||
import os
|
import os
|
||||||
import time
|
import time
|
||||||
from kobo import shortcuts
|
from kobo import shortcuts
|
||||||
|
import traceback
|
||||||
|
|
||||||
from pungi.util import get_variant_data, resolve_git_url, makedirs, get_mtime, get_file_size
|
from pungi.util import get_variant_data, resolve_git_url, makedirs, get_mtime, get_file_size
|
||||||
from pungi.phases.base import PhaseBase
|
from pungi.phases.base import PhaseBase
|
||||||
@ -142,35 +143,38 @@ class CreateImageBuildThread(WorkerThread):
|
|||||||
|
|
||||||
def process(self, item, num):
|
def process(self, item, num):
|
||||||
compose, cmd = item
|
compose, cmd = item
|
||||||
|
variant = cmd["image_conf"]["image-build"]["variant"]
|
||||||
|
subvariant = cmd["image_conf"]["image-build"].get("subvariant", variant.uid)
|
||||||
try:
|
try:
|
||||||
self.worker(num, compose, cmd)
|
self.worker(num, compose, variant, subvariant, cmd)
|
||||||
except Exception as exc:
|
except Exception as exc:
|
||||||
if not compose.can_fail(cmd["image_conf"]["image-build"]['variant'], '*', 'image-build'):
|
if not compose.can_fail(cmd["image_conf"]["image-build"]['variant'], '*', 'image-build'):
|
||||||
raise
|
raise
|
||||||
else:
|
else:
|
||||||
msg = ('[FAIL] image-build for variant %s failed, but going on anyway.\n%s'
|
msg = ('[FAIL] image-build for variant %s (%s) failed, but going on anyway.\n%s'
|
||||||
% (cmd['image_conf']['image-build']['variant'], exc))
|
% (variant.uid, subvariant, exc))
|
||||||
self.pool.log_info(msg)
|
self.pool.log_info(msg)
|
||||||
|
tb = traceback.format_exc()
|
||||||
|
self.pool.log_debug(tb)
|
||||||
|
|
||||||
def worker(self, num, compose, cmd):
|
def worker(self, num, compose, variant, subvariant, cmd):
|
||||||
arches = cmd["image_conf"]["image-build"]['arches'].split(',')
|
arches = cmd["image_conf"]["image-build"]['arches'].split(',')
|
||||||
dash_arches = '-'.join(arches)
|
dash_arches = '-'.join(arches)
|
||||||
log_file = compose.paths.log.log_file(
|
log_file = compose.paths.log.log_file(
|
||||||
dash_arches,
|
dash_arches,
|
||||||
"imagebuild-%s-%s-%s" % (dash_arches,
|
"imagebuild-%s-%s-%s" % (variant.uid, subvariant,
|
||||||
cmd["image_conf"]["image-build"]["variant"],
|
|
||||||
cmd["image_conf"]["image-build"]['format'].replace(",", "-"))
|
cmd["image_conf"]["image-build"]['format'].replace(",", "-"))
|
||||||
)
|
)
|
||||||
msg = "Creating %s image (arches: %s, variant: %s)" % (cmd["image_conf"]["image-build"]["format"].replace(",", "-"),
|
msg = ("Creating %s image (arches: %s, variant: %s, subvariant: %s)"
|
||||||
dash_arches,
|
% (cmd["image_conf"]["image-build"]["format"].replace(",", "-"),
|
||||||
cmd["image_conf"]["image-build"]["variant"])
|
dash_arches, variant, subvariant))
|
||||||
self.pool.log_info("[BEGIN] %s" % msg)
|
self.pool.log_info("[BEGIN] %s" % msg)
|
||||||
|
|
||||||
koji_wrapper = KojiWrapper(compose.conf["koji_profile"])
|
koji_wrapper = KojiWrapper(compose.conf["koji_profile"])
|
||||||
|
|
||||||
# writes conf file for koji image-build
|
# writes conf file for koji image-build
|
||||||
self.pool.log_info("Writing image-build config for %s.%s into %s" % (
|
self.pool.log_info("Writing image-build config for %s.%s into %s" % (
|
||||||
cmd["image_conf"]["image-build"]["variant"], dash_arches, cmd["conf_file"]))
|
variant, dash_arches, cmd["conf_file"]))
|
||||||
koji_cmd = koji_wrapper.get_image_build_cmd(cmd["image_conf"],
|
koji_cmd = koji_wrapper.get_image_build_cmd(cmd["image_conf"],
|
||||||
conf_file_dest=cmd["conf_file"],
|
conf_file_dest=cmd["conf_file"],
|
||||||
scratch=cmd['scratch'])
|
scratch=cmd['scratch'])
|
||||||
@ -182,7 +186,8 @@ class CreateImageBuildThread(WorkerThread):
|
|||||||
self.pool.log_debug("build-image outputs: %s" % (output))
|
self.pool.log_debug("build-image outputs: %s" % (output))
|
||||||
if output["retcode"] != 0:
|
if output["retcode"] != 0:
|
||||||
self.fail(compose, cmd)
|
self.fail(compose, cmd)
|
||||||
raise RuntimeError("ImageBuild task failed: %s. See %s for more details." % (output["task_id"], log_file))
|
raise RuntimeError("ImageBuild task failed: %s. See %s for more details."
|
||||||
|
% (output["task_id"], log_file))
|
||||||
|
|
||||||
# copy image to images/
|
# copy image to images/
|
||||||
image_infos = []
|
image_infos = []
|
||||||
@ -229,8 +234,7 @@ class CreateImageBuildThread(WorkerThread):
|
|||||||
img.disc_number = 1 # We don't expect multiple disks
|
img.disc_number = 1 # We don't expect multiple disks
|
||||||
img.disc_count = 1
|
img.disc_count = 1
|
||||||
img.bootable = False
|
img.bootable = False
|
||||||
varname = cmd["image_conf"]["image-build"]["variant"].uid
|
img.subvariant = subvariant
|
||||||
img.subvariant = cmd["image_conf"]["image-build"].get("subvariant", varname)
|
compose.im.add(variant=variant.uid, arch=image_info['arch'], image=img)
|
||||||
compose.im.add(variant=varname, arch=image_info['arch'], image=img)
|
|
||||||
|
|
||||||
self.pool.log_info("[DONE ] %s" % msg)
|
self.pool.log_info("[DONE ] %s" % msg)
|
||||||
|
@ -157,22 +157,24 @@ class LiveMediaPhase(PhaseBase):
|
|||||||
class LiveMediaThread(WorkerThread):
|
class LiveMediaThread(WorkerThread):
|
||||||
def process(self, item, num):
|
def process(self, item, num):
|
||||||
compose, variant, config = item
|
compose, variant, config = item
|
||||||
|
subvariant = config.pop('subvariant')
|
||||||
self.num = num
|
self.num = num
|
||||||
try:
|
try:
|
||||||
self.worker(compose, variant, config)
|
self.worker(compose, variant, subvariant, config)
|
||||||
except Exception as exc:
|
except Exception as exc:
|
||||||
if not compose.can_fail(variant, '*', 'live-media'):
|
if not compose.can_fail(variant, '*', 'live-media'):
|
||||||
raise
|
raise
|
||||||
else:
|
else:
|
||||||
msg = ('[FAIL] live-media for variant %s failed, but going on anyway.\n%s'
|
msg = ('[FAIL] live-media for variant %s (%s) failed, but going on anyway.\n%s'
|
||||||
% (variant.uid, exc))
|
% (variant.uid, subvariant, exc))
|
||||||
self.pool.log_info(msg)
|
self.pool.log_info(msg)
|
||||||
tb = traceback.format_exc()
|
tb = traceback.format_exc()
|
||||||
self.pool.log_debug(tb)
|
self.pool.log_debug(tb)
|
||||||
|
|
||||||
def _get_log_file(self, compose, variant, config):
|
def _get_log_file(self, compose, variant, subvariant, config):
|
||||||
arches = '-'.join(config['arches'])
|
arches = '-'.join(config['arches'])
|
||||||
return compose.paths.log.log_file(arches, 'livemedia-%s' % variant.uid)
|
return compose.paths.log.log_file(arches, 'livemedia-%s-%s'
|
||||||
|
% (variant.uid, subvariant))
|
||||||
|
|
||||||
def _run_command(self, koji_wrapper, cmd, compose, log_file):
|
def _run_command(self, koji_wrapper, cmd, compose, log_file):
|
||||||
time.sleep(self.num * 3)
|
time.sleep(self.num * 3)
|
||||||
@ -190,17 +192,15 @@ class LiveMediaThread(WorkerThread):
|
|||||||
copy['arch'] = ','.join(copy.pop('arches', []))
|
copy['arch'] = ','.join(copy.pop('arches', []))
|
||||||
return koji_wrapper.get_live_media_cmd(copy)
|
return koji_wrapper.get_live_media_cmd(copy)
|
||||||
|
|
||||||
def worker(self, compose, variant, config):
|
def worker(self, compose, variant, subvariant, config):
|
||||||
msg = 'Live media: %s (arches: %s, variant: %s)' % (config['name'],
|
msg = ('Live media: %s (arches: %s, variant: %s, subvariant: %s)'
|
||||||
' '.join(config['arches']),
|
% (config['name'], ' '.join(config['arches']), variant.uid, subvariant))
|
||||||
variant.uid)
|
|
||||||
self.pool.log_info('[BEGIN] %s' % msg)
|
self.pool.log_info('[BEGIN] %s' % msg)
|
||||||
subvariant = config.pop('subvariant')
|
|
||||||
|
|
||||||
koji_wrapper = KojiWrapper(compose.conf['koji_profile'])
|
koji_wrapper = KojiWrapper(compose.conf['koji_profile'])
|
||||||
cmd = self._get_cmd(koji_wrapper, config)
|
cmd = self._get_cmd(koji_wrapper, config)
|
||||||
|
|
||||||
log_file = self._get_log_file(compose, variant, config)
|
log_file = self._get_log_file(compose, variant, subvariant, config)
|
||||||
output = self._run_command(koji_wrapper, cmd, compose, log_file)
|
output = self._run_command(koji_wrapper, cmd, compose, log_file)
|
||||||
|
|
||||||
# collect results and update manifest
|
# collect results and update manifest
|
||||||
|
@ -388,6 +388,7 @@ class TestCreateImageBuildThread(PungiTestCase):
|
|||||||
'version': 'Rawhide',
|
'version': 'Rawhide',
|
||||||
'ksurl': 'git://git.fedorahosted.org/git/spin-kickstarts.git',
|
'ksurl': 'git://git.fedorahosted.org/git/spin-kickstarts.git',
|
||||||
'distro': 'Fedora-20',
|
'distro': 'Fedora-20',
|
||||||
|
'subvariant': 'KDE',
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"conf_file": 'amd64,x86_64-Client-Fedora-Docker-Base-docker',
|
"conf_file": 'amd64,x86_64-Client-Fedora-Docker-Base-docker',
|
||||||
@ -423,6 +424,19 @@ class TestCreateImageBuildThread(PungiTestCase):
|
|||||||
with mock.patch('time.sleep'):
|
with mock.patch('time.sleep'):
|
||||||
t.process((compose, cmd), 1)
|
t.process((compose, cmd), 1)
|
||||||
|
|
||||||
|
self.assertItemsEqual(
|
||||||
|
koji_wrapper.get_image_build_cmd.call_args_list,
|
||||||
|
[mock.call(cmd['image_conf'],
|
||||||
|
conf_file_dest='amd64,x86_64-Client-Fedora-Docker-Base-docker',
|
||||||
|
scratch=False)]
|
||||||
|
)
|
||||||
|
|
||||||
|
self.assertItemsEqual(
|
||||||
|
koji_wrapper.run_blocking_cmd.call_args_list,
|
||||||
|
[mock.call(koji_wrapper.get_image_build_cmd.return_value,
|
||||||
|
log_file=self.topdir + '/logs/amd64-x86_64/imagebuild-Client-KDE-docker.amd64-x86_64.log')]
|
||||||
|
)
|
||||||
|
|
||||||
self.assertItemsEqual(
|
self.assertItemsEqual(
|
||||||
linker.mock_calls,
|
linker.mock_calls,
|
||||||
[mock.call('/koji/task/1235/Fedora-Docker-Base-20160103.amd64.qcow2',
|
[mock.call('/koji/task/1235/Fedora-Docker-Base-20160103.amd64.qcow2',
|
||||||
@ -473,6 +487,7 @@ class TestCreateImageBuildThread(PungiTestCase):
|
|||||||
data = image_relative_paths.pop(image.path)
|
data = image_relative_paths.pop(image.path)
|
||||||
self.assertEqual(data['format'], image.format)
|
self.assertEqual(data['format'], image.format)
|
||||||
self.assertEqual(data['type'], image.type)
|
self.assertEqual(data['type'], image.type)
|
||||||
|
self.assertEqual('KDE', image.subvariant)
|
||||||
|
|
||||||
self.assertTrue(os.path.isdir(self.topdir + '/compose/Client/amd64/images'))
|
self.assertTrue(os.path.isdir(self.topdir + '/compose/Client/amd64/images'))
|
||||||
self.assertTrue(os.path.isdir(self.topdir + '/compose/Client/x86_64/images'))
|
self.assertTrue(os.path.isdir(self.topdir + '/compose/Client/x86_64/images'))
|
||||||
|
@ -296,7 +296,7 @@ class TestLiveMediaThread(PungiTestCase):
|
|||||||
'target': 'f24',
|
'target': 'f24',
|
||||||
'title': None,
|
'title': None,
|
||||||
'version': 'Rawhide',
|
'version': 'Rawhide',
|
||||||
'subvariant': 'DATA',
|
'subvariant': 'KDE',
|
||||||
}
|
}
|
||||||
pool = mock.Mock()
|
pool = mock.Mock()
|
||||||
|
|
||||||
@ -333,7 +333,7 @@ class TestLiveMediaThread(PungiTestCase):
|
|||||||
self.assertEqual(
|
self.assertEqual(
|
||||||
run_blocking_cmd.mock_calls,
|
run_blocking_cmd.mock_calls,
|
||||||
[mock.call('koji-spin-livemedia',
|
[mock.call('koji-spin-livemedia',
|
||||||
log_file=self.topdir + '/logs/amd64-x86_64/livemedia-Server.amd64-x86_64.log')])
|
log_file=self.topdir + '/logs/amd64-x86_64/livemedia-Server-KDE.amd64-x86_64.log')])
|
||||||
self.assertEqual(get_live_media_cmd.mock_calls,
|
self.assertEqual(get_live_media_cmd.mock_calls,
|
||||||
[mock.call({'arch': 'amd64,x86_64',
|
[mock.call({'arch': 'amd64,x86_64',
|
||||||
'ksfile': 'file.ks',
|
'ksfile': 'file.ks',
|
||||||
@ -375,7 +375,7 @@ class TestLiveMediaThread(PungiTestCase):
|
|||||||
self.assertIn(image.path, image_relative_paths)
|
self.assertIn(image.path, image_relative_paths)
|
||||||
self.assertEqual('iso', image.format)
|
self.assertEqual('iso', image.format)
|
||||||
self.assertEqual('live', image.type)
|
self.assertEqual('live', image.type)
|
||||||
self.assertEqual('DATA', image.subvariant)
|
self.assertEqual('KDE', image.subvariant)
|
||||||
|
|
||||||
@mock.patch('pungi.phases.livemedia_phase.KojiWrapper')
|
@mock.patch('pungi.phases.livemedia_phase.KojiWrapper')
|
||||||
def test_handle_koji_fail(self, KojiWrapper):
|
def test_handle_koji_fail(self, KojiWrapper):
|
||||||
@ -398,6 +398,7 @@ class TestLiveMediaThread(PungiTestCase):
|
|||||||
'target': 'f24',
|
'target': 'f24',
|
||||||
'title': None,
|
'title': None,
|
||||||
'version': 'Rawhide',
|
'version': 'Rawhide',
|
||||||
|
'subvariant': 'KDE',
|
||||||
}
|
}
|
||||||
pool = mock.Mock()
|
pool = mock.Mock()
|
||||||
|
|
||||||
@ -437,6 +438,7 @@ class TestLiveMediaThread(PungiTestCase):
|
|||||||
'target': 'f24',
|
'target': 'f24',
|
||||||
'title': None,
|
'title': None,
|
||||||
'version': 'Rawhide',
|
'version': 'Rawhide',
|
||||||
|
'subvariant': 'KDE',
|
||||||
}
|
}
|
||||||
pool = mock.Mock()
|
pool = mock.Mock()
|
||||||
|
|
||||||
|
Loading…
Reference in New Issue
Block a user