Merge #114 Use install tree/repo from another variant for image build
This commit is contained in:
Dennis Gilmore
commit
5f7f23ed96
@ -135,6 +135,12 @@ Options
|
||||
* buildinstall
|
||||
* iso
|
||||
* live
|
||||
* image-build
|
||||
|
||||
.. note::
|
||||
|
||||
Image building is not run per-architecture. If you want to mark it
|
||||
as failable, specify it in a block with arch set as ``*``.
|
||||
|
||||
Please note that ``*`` as a wildcard matches all architectures but ``src``.
|
||||
|
||||
@ -618,7 +624,13 @@ Image Build Settings
|
||||
automatically transformed into format suitable for ``koji``. A repo for the
|
||||
currently built variant will be added as well.
|
||||
|
||||
Please don't set ``install_tree`` as it would get overriden by pungi.
|
||||
You can also add extra variants to get repos from with key ``repo_from``.
|
||||
The value should be a list of variant names.
|
||||
|
||||
Please don't set ``install_tree``. This gets automatically set by *pungi*
|
||||
based on current variant. You can use ``install_tree_from`` key to use
|
||||
install tree from another variant.
|
||||
|
||||
The ``format`` attr is [('image_type', 'image_suffix'), ...].
|
||||
See productmd documentation for list of supported types and suffixes.
|
||||
|
||||
@ -660,6 +672,10 @@ Example
|
||||
|
||||
# only build this type of image on x86_64
|
||||
'arches': ['x86_64']
|
||||
|
||||
# Use install tree and repo from Everything variant.
|
||||
'install_tree_from': 'Everything',
|
||||
'repo_from': ['Everything'],
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
@ -3,6 +3,7 @@
|
||||
import copy
|
||||
import os
|
||||
import time
|
||||
from kobo import shortcuts
|
||||
|
||||
from pungi.util import get_variant_data, resolve_git_url
|
||||
from pungi.phases.base import PhaseBase
|
||||
@ -29,6 +30,52 @@ class ImageBuildPhase(PhaseBase):
|
||||
return True
|
||||
return False
|
||||
|
||||
def _get_install_tree(self, image_conf, variant):
|
||||
"""
|
||||
Get a path to os tree for a variant specified in `install_tree_from` or
|
||||
current variant. If the config is set, it will be removed from the
|
||||
dict.
|
||||
"""
|
||||
install_tree_from = image_conf.pop('install_tree_from', variant.uid)
|
||||
install_tree_source = self.compose.variants.get(install_tree_from)
|
||||
if not install_tree_source:
|
||||
raise RuntimeError(
|
||||
'There is no variant %s to get install tree from when building image for %s.'
|
||||
% (install_tree_from, variant.uid))
|
||||
return translate_path(
|
||||
self.compose,
|
||||
self.compose.paths.compose.os_tree('$arch', install_tree_source)
|
||||
)
|
||||
|
||||
def _get_repo(self, image_conf, variant):
|
||||
"""
|
||||
Get a comma separated list of repos. First included are those
|
||||
explicitly listed in config, followed by repos from other variants,
|
||||
finally followed by repo for current variant.
|
||||
|
||||
The `repo_from` key is removed from the dict (if present).
|
||||
"""
|
||||
repo = shortcuts.force_list(image_conf.get('repo', []))
|
||||
|
||||
extras = shortcuts.force_list(image_conf.pop('repo_from', []))
|
||||
extras.append(variant.uid)
|
||||
|
||||
for extra in extras:
|
||||
v = self.compose.variants.get(extra)
|
||||
if not v:
|
||||
raise RuntimeError(
|
||||
'There is no variant %s to get repo from when building image for %s.'
|
||||
% (extra, variant.uid))
|
||||
repo.append(translate_path(self.compose,
|
||||
self.compose.paths.compose.os_tree('$arch', v)))
|
||||
|
||||
return ",".join(repo)
|
||||
|
||||
def _get_arches(self, image_conf, arches):
|
||||
if 'arches' in image_conf:
|
||||
arches = set(image_conf.get('arches', [])) & arches
|
||||
return ','.join(sorted(arches))
|
||||
|
||||
def run(self):
|
||||
for variant in self.compose.get_variants():
|
||||
arches = set([x for x in variant.arches if x != 'src'])
|
||||
@ -39,36 +86,26 @@ class ImageBuildPhase(PhaseBase):
|
||||
# value is needed.
|
||||
image_conf = copy.deepcopy(image_conf)
|
||||
|
||||
# image_conf is passed to get_image_build_cmd as dict
|
||||
|
||||
image_conf['arches'] = self._get_arches(image_conf, arches)
|
||||
if not image_conf['arches']:
|
||||
continue
|
||||
|
||||
# Replace possible ambiguous ref name with explicit hash.
|
||||
if 'ksurl' in image_conf:
|
||||
image_conf['ksurl'] = resolve_git_url(image_conf['ksurl'])
|
||||
|
||||
# image_conf is passed to get_image_build_cmd as dict
|
||||
|
||||
if 'arches' in image_conf:
|
||||
image_conf["arches"] = ','.join(sorted(set(image_conf.get('arches', [])) & arches))
|
||||
else:
|
||||
image_conf['arches'] = ','.join(sorted(arches))
|
||||
|
||||
if not image_conf['arches']:
|
||||
continue
|
||||
|
||||
image_conf["variant"] = variant
|
||||
image_conf["install_tree"] = translate_path(
|
||||
self.compose,
|
||||
self.compose.paths.compose.os_tree('$arch', variant)
|
||||
)
|
||||
|
||||
image_conf["install_tree"] = self._get_install_tree(image_conf, variant)
|
||||
|
||||
# transform format into right 'format' for image-build
|
||||
# e.g. 'docker,qcow2'
|
||||
format = image_conf["format"]
|
||||
image_conf["format"] = ",".join([x[0] for x in image_conf["format"]])
|
||||
|
||||
repo = image_conf.get('repo', [])
|
||||
if isinstance(repo, str):
|
||||
repo = [repo]
|
||||
repo.append(translate_path(self.compose, self.compose.paths.compose.os_tree('$arch', variant)))
|
||||
# supply repo as str separated by , instead of list
|
||||
image_conf['repo'] = ",".join(repo)
|
||||
image_conf['repo'] = self._get_repo(image_conf, variant)
|
||||
|
||||
cmd = {
|
||||
"format": format,
|
||||
@ -96,6 +133,17 @@ class CreateImageBuildThread(WorkerThread):
|
||||
|
||||
def process(self, item, num):
|
||||
compose, cmd = item
|
||||
try:
|
||||
self.worker(num, compose, cmd)
|
||||
except:
|
||||
if not compose.can_fail(cmd['image_conf']['variant'], '*', 'image-build'):
|
||||
raise
|
||||
else:
|
||||
msg = ('[FAIL] image-build for variant %s failed, but going on anyway.'
|
||||
% cmd['image_conf']['variant'])
|
||||
self.pool.log_info(msg)
|
||||
|
||||
def worker(self, num, compose, cmd):
|
||||
arches = cmd['image_conf']['arches'].split(',')
|
||||
|
||||
mounts = [compose.paths.compose.topdir()]
|
||||
|
||||
@ -40,6 +40,9 @@ class ComposeTestCase(unittest.TestCase):
|
||||
self.assertFalse(compose.can_fail(None, 'x86_64', 'live'))
|
||||
self.assertTrue(compose.can_fail(None, 'i386', 'live'))
|
||||
|
||||
self.assertTrue(compose.can_fail(variant, '*', 'buildinstall'))
|
||||
self.assertFalse(compose.can_fail(variant, '*', 'live'))
|
||||
|
||||
@mock.patch('pungi.compose.ComposeInfo')
|
||||
def test_get_image_name(self, ci):
|
||||
conf = {}
|
||||
|
||||
@ -11,6 +11,7 @@ import sys
|
||||
sys.path.insert(0, os.path.join(os.path.dirname(__file__), ".."))
|
||||
|
||||
from pungi.phases.image_build import ImageBuildPhase, CreateImageBuildThread
|
||||
from pungi.util import get_arch_variant_data
|
||||
|
||||
|
||||
class _DummyCompose(object):
|
||||
@ -49,17 +50,23 @@ class _DummyCompose(object):
|
||||
)
|
||||
)
|
||||
self._logger = mock.Mock()
|
||||
self.variants = [
|
||||
mock.Mock(uid='Server', arches=['x86_64', 'amd64']),
|
||||
mock.Mock(uid='Client', arches=['amd64']),
|
||||
]
|
||||
self.variants = {
|
||||
'Server': mock.Mock(uid='Server', arches=['x86_64', 'amd64']),
|
||||
'Client': mock.Mock(uid='Client', arches=['amd64']),
|
||||
'Everything': mock.Mock(uid='Everything', arches=['x86_64', 'amd64']),
|
||||
}
|
||||
self.im = mock.Mock()
|
||||
self.log_error = mock.Mock()
|
||||
|
||||
def get_arches(self):
|
||||
return ['x86_64', 'amd64']
|
||||
|
||||
def get_variants(self, arch=None, types=None):
|
||||
return [v for v in self.variants if not arch or arch in v.arches]
|
||||
return [v for v in self.variants.values() if not arch or arch in v.arches]
|
||||
|
||||
def can_fail(self, variant, arch, deliverable):
|
||||
failable = get_arch_variant_data(self.conf, 'failable_deliverables', arch, variant)
|
||||
return deliverable in failable
|
||||
|
||||
|
||||
class TestImageBuildPhase(unittest.TestCase):
|
||||
@ -97,7 +104,7 @@ class TestImageBuildPhase(unittest.TestCase):
|
||||
'kickstart': 'fedora-docker-base.ks',
|
||||
'format': 'docker',
|
||||
'repo': '/ostree/$arch/Client',
|
||||
'variant': compose.variants[1],
|
||||
'variant': compose.variants['Client'],
|
||||
'target': 'f24',
|
||||
'disk_size': 3,
|
||||
'name': 'Fedora-Docker-Base',
|
||||
@ -118,7 +125,7 @@ class TestImageBuildPhase(unittest.TestCase):
|
||||
'kickstart': 'fedora-docker-base.ks',
|
||||
'format': 'docker',
|
||||
'repo': '/ostree/$arch/Server',
|
||||
'variant': compose.variants[0],
|
||||
'variant': compose.variants['Server'],
|
||||
'target': 'f24',
|
||||
'disk_size': 3,
|
||||
'name': 'Fedora-Docker-Base',
|
||||
@ -165,6 +172,116 @@ class TestImageBuildPhase(unittest.TestCase):
|
||||
self.assertFalse(phase.pool.add.called)
|
||||
self.assertFalse(phase.pool.queue_put.called)
|
||||
|
||||
@mock.patch('pungi.phases.image_build.ThreadPool')
|
||||
def test_image_build_set_install_tree(self, ThreadPool):
|
||||
compose = _DummyCompose({
|
||||
'image_build': {
|
||||
'^Server$': [
|
||||
{
|
||||
'format': [('docker', 'tar.xz')],
|
||||
'name': 'Fedora-Docker-Base',
|
||||
'target': 'f24',
|
||||
'version': 'Rawhide',
|
||||
'ksurl': 'git://git.fedorahosted.org/git/spin-kickstarts.git',
|
||||
'kickstart': "fedora-docker-base.ks",
|
||||
'distro': 'Fedora-20',
|
||||
'disk_size': 3,
|
||||
'arches': ['x86_64'],
|
||||
'install_tree_from': 'Everything',
|
||||
}
|
||||
]
|
||||
},
|
||||
'koji_profile': 'koji',
|
||||
})
|
||||
|
||||
phase = ImageBuildPhase(compose)
|
||||
|
||||
phase.run()
|
||||
|
||||
# assert at least one thread was started
|
||||
self.assertTrue(phase.pool.add.called)
|
||||
|
||||
self.assertTrue(phase.pool.queue_put.called_once)
|
||||
args, kwargs = phase.pool.queue_put.call_args
|
||||
self.assertEqual(args[0][0], compose)
|
||||
self.maxDiff = None
|
||||
self.assertDictEqual(args[0][1], {
|
||||
"format": [('docker', 'tar.xz')],
|
||||
"image_conf": {
|
||||
'install_tree': '/ostree/$arch/Everything',
|
||||
'kickstart': 'fedora-docker-base.ks',
|
||||
'format': 'docker',
|
||||
'repo': '/ostree/$arch/Server',
|
||||
'variant': compose.variants['Server'],
|
||||
'target': 'f24',
|
||||
'disk_size': 3,
|
||||
'name': 'Fedora-Docker-Base',
|
||||
'arches': 'x86_64',
|
||||
'version': 'Rawhide',
|
||||
'ksurl': 'git://git.fedorahosted.org/git/spin-kickstarts.git',
|
||||
'distro': 'Fedora-20',
|
||||
},
|
||||
"conf_file": 'Server-Fedora-Docker-Base-docker',
|
||||
"image_dir": '/image_dir/Server/%(arch)s',
|
||||
"relative_image_dir": 'image_dir/Server/%(arch)s',
|
||||
"link_type": 'hardlink-or-copy',
|
||||
})
|
||||
|
||||
@mock.patch('pungi.phases.image_build.ThreadPool')
|
||||
def test_image_build_set_extra_repos(self, ThreadPool):
|
||||
compose = _DummyCompose({
|
||||
'image_build': {
|
||||
'^Server$': [
|
||||
{
|
||||
'format': [('docker', 'tar.xz')],
|
||||
'name': 'Fedora-Docker-Base',
|
||||
'target': 'f24',
|
||||
'version': 'Rawhide',
|
||||
'ksurl': 'git://git.fedorahosted.org/git/spin-kickstarts.git',
|
||||
'kickstart': "fedora-docker-base.ks",
|
||||
'distro': 'Fedora-20',
|
||||
'disk_size': 3,
|
||||
'arches': ['x86_64'],
|
||||
'repo_from': 'Everything',
|
||||
}
|
||||
]
|
||||
},
|
||||
'koji_profile': 'koji',
|
||||
})
|
||||
|
||||
phase = ImageBuildPhase(compose)
|
||||
|
||||
phase.run()
|
||||
|
||||
# assert at least one thread was started
|
||||
self.assertTrue(phase.pool.add.called)
|
||||
|
||||
self.assertTrue(phase.pool.queue_put.called_once)
|
||||
args, kwargs = phase.pool.queue_put.call_args
|
||||
self.assertEqual(args[0][0], compose)
|
||||
self.maxDiff = None
|
||||
self.assertDictEqual(args[0][1], {
|
||||
"format": [('docker', 'tar.xz')],
|
||||
"image_conf": {
|
||||
'install_tree': '/ostree/$arch/Server',
|
||||
'kickstart': 'fedora-docker-base.ks',
|
||||
'format': 'docker',
|
||||
'repo': '/ostree/$arch/Everything,/ostree/$arch/Server',
|
||||
'variant': compose.variants['Server'],
|
||||
'target': 'f24',
|
||||
'disk_size': 3,
|
||||
'name': 'Fedora-Docker-Base',
|
||||
'arches': 'x86_64',
|
||||
'version': 'Rawhide',
|
||||
'ksurl': 'git://git.fedorahosted.org/git/spin-kickstarts.git',
|
||||
'distro': 'Fedora-20',
|
||||
},
|
||||
"conf_file": 'Server-Fedora-Docker-Base-docker',
|
||||
"image_dir": '/image_dir/Server/%(arch)s',
|
||||
"relative_image_dir": 'image_dir/Server/%(arch)s',
|
||||
"link_type": 'hardlink-or-copy',
|
||||
})
|
||||
|
||||
|
||||
class TestCreateImageBuildThread(unittest.TestCase):
|
||||
|
||||
@ -182,7 +299,7 @@ class TestCreateImageBuildThread(unittest.TestCase):
|
||||
'kickstart': 'fedora-docker-base.ks',
|
||||
'format': 'docker',
|
||||
'repo': '/ostree/$arch/Client',
|
||||
'variant': compose.variants[1],
|
||||
'variant': compose.variants['Client'],
|
||||
'target': 'f24',
|
||||
'disk_size': 3,
|
||||
'name': 'Fedora-Docker-Base',
|
||||
@ -276,6 +393,102 @@ class TestCreateImageBuildThread(unittest.TestCase):
|
||||
self.assertEqual(data['format'], image.format)
|
||||
self.assertEqual(data['type'], image.type)
|
||||
|
||||
@mock.patch('pungi.phases.image_build.KojiWrapper')
|
||||
@mock.patch('pungi.phases.image_build.Linker')
|
||||
def test_process_handle_fail(self, Linker, KojiWrapper):
|
||||
compose = _DummyCompose({
|
||||
'koji_profile': 'koji',
|
||||
'failable_deliverables': [
|
||||
('^.*$', {
|
||||
'*': ['image-build']
|
||||
})
|
||||
]
|
||||
})
|
||||
pool = mock.Mock()
|
||||
cmd = {
|
||||
"format": [('docker', 'tar.xz'), ('qcow2', 'qcow2')],
|
||||
"image_conf": {
|
||||
'install_tree': '/ostree/$arch/Client',
|
||||
'kickstart': 'fedora-docker-base.ks',
|
||||
'format': 'docker',
|
||||
'repo': '/ostree/$arch/Client',
|
||||
'variant': compose.variants['Client'],
|
||||
'target': 'f24',
|
||||
'disk_size': 3,
|
||||
'name': 'Fedora-Docker-Base',
|
||||
'arches': 'amd64,x86_64',
|
||||
'version': 'Rawhide',
|
||||
'ksurl': 'git://git.fedorahosted.org/git/spin-kickstarts.git',
|
||||
'distro': 'Fedora-20',
|
||||
},
|
||||
"conf_file": 'amd64,x86_64-Client-Fedora-Docker-Base-docker',
|
||||
"image_dir": '/image_dir/Client/%(arch)s',
|
||||
"relative_image_dir": 'image_dir/Client/%(arch)s',
|
||||
"link_type": 'hardlink-or-copy',
|
||||
}
|
||||
koji_wrapper = KojiWrapper.return_value
|
||||
koji_wrapper.run_create_image_cmd.return_value = {
|
||||
"retcode": 1,
|
||||
"output": None,
|
||||
"task_id": 1234,
|
||||
}
|
||||
|
||||
t = CreateImageBuildThread(pool)
|
||||
with mock.patch('os.stat') as stat:
|
||||
with mock.patch('os.path.getsize') as getsize:
|
||||
with mock.patch('time.sleep'):
|
||||
getsize.return_value = 1024
|
||||
stat.return_value.st_mtime = 13579
|
||||
t.process((compose, cmd), 1)
|
||||
|
||||
@mock.patch('pungi.phases.image_build.KojiWrapper')
|
||||
@mock.patch('pungi.phases.image_build.Linker')
|
||||
def test_process_handle_exception(self, Linker, KojiWrapper):
|
||||
compose = _DummyCompose({
|
||||
'koji_profile': 'koji',
|
||||
'failable_deliverables': [
|
||||
('^.*$', {
|
||||
'*': ['image-build']
|
||||
})
|
||||
]
|
||||
})
|
||||
pool = mock.Mock()
|
||||
cmd = {
|
||||
"format": [('docker', 'tar.xz'), ('qcow2', 'qcow2')],
|
||||
"image_conf": {
|
||||
'install_tree': '/ostree/$arch/Client',
|
||||
'kickstart': 'fedora-docker-base.ks',
|
||||
'format': 'docker',
|
||||
'repo': '/ostree/$arch/Client',
|
||||
'variant': compose.variants['Client'],
|
||||
'target': 'f24',
|
||||
'disk_size': 3,
|
||||
'name': 'Fedora-Docker-Base',
|
||||
'arches': 'amd64,x86_64',
|
||||
'version': 'Rawhide',
|
||||
'ksurl': 'git://git.fedorahosted.org/git/spin-kickstarts.git',
|
||||
'distro': 'Fedora-20',
|
||||
},
|
||||
"conf_file": 'amd64,x86_64-Client-Fedora-Docker-Base-docker',
|
||||
"image_dir": '/image_dir/Client/%(arch)s',
|
||||
"relative_image_dir": 'image_dir/Client/%(arch)s',
|
||||
"link_type": 'hardlink-or-copy',
|
||||
}
|
||||
|
||||
def boom(*args, **kwargs):
|
||||
raise RuntimeError('BOOM')
|
||||
|
||||
koji_wrapper = KojiWrapper.return_value
|
||||
koji_wrapper.run_create_image_cmd.side_effect = boom
|
||||
|
||||
t = CreateImageBuildThread(pool)
|
||||
with mock.patch('os.stat') as stat:
|
||||
with mock.patch('os.path.getsize') as getsize:
|
||||
with mock.patch('time.sleep'):
|
||||
getsize.return_value = 1024
|
||||
stat.return_value.st_mtime = 13579
|
||||
t.process((compose, cmd), 1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
Loading…
Reference in New Issue
Block a user