pungi/tests/test_imagebuildphase.py

603 lines
23 KiB
Python
Raw Normal View History

#!/usr/bin/env python2
# -*- coding: utf-8 -*-
import unittest
import mock
import os
import sys
sys.path.insert(0, os.path.join(os.path.dirname(__file__), ".."))
from pungi.phases.image_build import ImageBuildPhase, CreateImageBuildThread
from pungi.util import get_arch_variant_data
class _DummyCompose(object):
def __init__(self, config):
self.compose_date = '20151203'
self.compose_type_suffix = '.t'
self.compose_respin = 0
self.ci_base = mock.Mock(
release_id='Test-1.0',
release=mock.Mock(
short='test',
version='1.0',
),
)
self.conf = config
self.paths = mock.Mock(
compose=mock.Mock(
topdir=mock.Mock(return_value='/a/b'),
os_tree=mock.Mock(
side_effect=lambda arch, variant, create_dir=False: os.path.join('/ostree', arch, variant.uid)
),
image_dir=mock.Mock(
side_effect=lambda variant, relative=False: os.path.join(
'' if relative else '/', 'image_dir', variant.uid, '%(arch)s'
)
)
),
work=mock.Mock(
image_build_conf=mock.Mock(
side_effect=lambda variant, image_name, image_type:
'-'.join([variant.uid, image_name, image_type])
)
),
log=mock.Mock(
log_file=mock.Mock(return_value='/a/b/log/log_file')
)
)
self._logger = mock.Mock()
self.variants = {
'Server': mock.Mock(uid='Server', arches=['x86_64', 'amd64']),
'Client': mock.Mock(uid='Client', arches=['amd64']),
'Everything': mock.Mock(uid='Everything', arches=['x86_64', 'amd64']),
}
self.im = mock.Mock()
self.log_error = mock.Mock()
def get_arches(self):
return ['x86_64', 'amd64']
def get_variants(self, arch=None, types=None):
return [v for v in self.variants.values() if not arch or arch in v.arches]
def can_fail(self, variant, arch, deliverable):
failable = get_arch_variant_data(self.conf, 'failable_deliverables', arch, variant)
return deliverable in failable
class TestImageBuildPhase(unittest.TestCase):
@mock.patch('pungi.phases.image_build.ThreadPool')
def test_image_build(self, ThreadPool):
compose = _DummyCompose({
'image_build': {
'^Client|Server$': [
{
'image-build': {
'format': [('docker', 'tar.xz')],
'name': 'Fedora-Docker-Base',
'target': 'f24',
'version': 'Rawhide',
'ksurl': 'git://git.fedorahosted.org/git/spin-kickstarts.git',
'kickstart': "fedora-docker-base.ks",
'distro': 'Fedora-20',
'disk_size': 3
}
}
]
},
'koji_profile': 'koji',
})
phase = ImageBuildPhase(compose)
phase.run()
self.maxDiff = None
# assert at least one thread was started
self.assertTrue(phase.pool.add.called)
client_args = {
"format": [('docker', 'tar.xz')],
"image_conf": {
'image-build': {
'install_tree': '/ostree/$arch/Client',
'kickstart': 'fedora-docker-base.ks',
'format': 'docker',
'repo': '/ostree/$arch/Client',
'variant': compose.variants['Client'],
'target': 'f24',
'disk_size': 3,
'name': 'Fedora-Docker-Base',
'arches': 'amd64',
'version': 'Rawhide',
'ksurl': 'git://git.fedorahosted.org/git/spin-kickstarts.git',
'distro': 'Fedora-20',
}
},
"conf_file": 'Client-Fedora-Docker-Base-docker',
"image_dir": '/image_dir/Client/%(arch)s',
"relative_image_dir": 'image_dir/Client/%(arch)s',
"link_type": 'hardlink-or-copy',
"scratch": False,
}
server_args = {
"format": [('docker', 'tar.xz')],
"image_conf": {
'image-build': {
'install_tree': '/ostree/$arch/Server',
'kickstart': 'fedora-docker-base.ks',
'format': 'docker',
'repo': '/ostree/$arch/Server',
'variant': compose.variants['Server'],
'target': 'f24',
'disk_size': 3,
'name': 'Fedora-Docker-Base',
'arches': 'amd64,x86_64',
'version': 'Rawhide',
'ksurl': 'git://git.fedorahosted.org/git/spin-kickstarts.git',
'distro': 'Fedora-20',
}
},
"conf_file": 'Server-Fedora-Docker-Base-docker',
"image_dir": '/image_dir/Server/%(arch)s',
"relative_image_dir": 'image_dir/Server/%(arch)s',
"link_type": 'hardlink-or-copy',
"scratch": False,
}
self.assertItemsEqual(phase.pool.queue_put.mock_calls,
[mock.call((compose, client_args)),
mock.call((compose, server_args))])
@mock.patch('pungi.phases.image_build.ThreadPool')
def test_image_build_filter_all_variants(self, ThreadPool):
compose = _DummyCompose({
'image_build': {
'^Client|Server$': [
{
'image-build': {
'format': [('docker', 'tar.xz')],
'name': 'Fedora-Docker-Base',
'target': 'f24',
'version': 'Rawhide',
'ksurl': 'git://git.fedorahosted.org/git/spin-kickstarts.git',
'kickstart': "fedora-docker-base.ks",
'distro': 'Fedora-20',
'disk_size': 3,
'arches': ['non-existing'],
}
}
]
},
'koji_profile': 'koji',
})
phase = ImageBuildPhase(compose)
phase.run()
# assert at least one thread was started
self.assertFalse(phase.pool.add.called)
self.assertFalse(phase.pool.queue_put.called)
@mock.patch('pungi.phases.image_build.ThreadPool')
def test_image_build_set_install_tree(self, ThreadPool):
compose = _DummyCompose({
'image_build': {
'^Server$': [
{
'image-build': {
'format': [('docker', 'tar.xz')],
'name': 'Fedora-Docker-Base',
'target': 'f24',
'version': 'Rawhide',
'ksurl': 'git://git.fedorahosted.org/git/spin-kickstarts.git',
'kickstart': "fedora-docker-base.ks",
'distro': 'Fedora-20',
'disk_size': 3,
'arches': ['x86_64'],
'install_tree_from': 'Everything',
}
}
]
},
'koji_profile': 'koji',
})
phase = ImageBuildPhase(compose)
phase.run()
# assert at least one thread was started
self.assertTrue(phase.pool.add.called)
self.assertTrue(phase.pool.queue_put.called_once)
args, kwargs = phase.pool.queue_put.call_args
self.assertEqual(args[0][0], compose)
self.maxDiff = None
self.assertDictEqual(args[0][1], {
"format": [('docker', 'tar.xz')],
"image_conf": {
'image-build': {
'install_tree': '/ostree/$arch/Everything',
'kickstart': 'fedora-docker-base.ks',
'format': 'docker',
'repo': '/ostree/$arch/Server',
'variant': compose.variants['Server'],
'target': 'f24',
'disk_size': 3,
'name': 'Fedora-Docker-Base',
'arches': 'x86_64',
'version': 'Rawhide',
'ksurl': 'git://git.fedorahosted.org/git/spin-kickstarts.git',
'distro': 'Fedora-20',
}
},
"conf_file": 'Server-Fedora-Docker-Base-docker',
"image_dir": '/image_dir/Server/%(arch)s',
"relative_image_dir": 'image_dir/Server/%(arch)s',
"link_type": 'hardlink-or-copy',
"scratch": False,
})
@mock.patch('pungi.phases.image_build.ThreadPool')
def test_image_build_set_extra_repos(self, ThreadPool):
compose = _DummyCompose({
'image_build': {
'^Server$': [
{
'image-build': {
'format': [('docker', 'tar.xz')],
'name': 'Fedora-Docker-Base',
'target': 'f24',
'version': 'Rawhide',
'ksurl': 'git://git.fedorahosted.org/git/spin-kickstarts.git',
'kickstart': "fedora-docker-base.ks",
'distro': 'Fedora-20',
'disk_size': 3,
'arches': ['x86_64'],
'repo_from': 'Everything',
}
}
]
},
'koji_profile': 'koji',
})
phase = ImageBuildPhase(compose)
phase.run()
# assert at least one thread was started
self.assertTrue(phase.pool.add.called)
self.assertTrue(phase.pool.queue_put.called_once)
args, kwargs = phase.pool.queue_put.call_args
self.assertEqual(args[0][0], compose)
self.maxDiff = None
self.assertDictEqual(args[0][1], {
"format": [('docker', 'tar.xz')],
"image_conf": {
'image-build': {
'install_tree': '/ostree/$arch/Server',
'kickstart': 'fedora-docker-base.ks',
'format': 'docker',
'repo': '/ostree/$arch/Everything,/ostree/$arch/Server',
'variant': compose.variants['Server'],
'target': 'f24',
'disk_size': 3,
'name': 'Fedora-Docker-Base',
'arches': 'x86_64',
'version': 'Rawhide',
'ksurl': 'git://git.fedorahosted.org/git/spin-kickstarts.git',
'distro': 'Fedora-20',
}
},
"conf_file": 'Server-Fedora-Docker-Base-docker',
"image_dir": '/image_dir/Server/%(arch)s',
"relative_image_dir": 'image_dir/Server/%(arch)s',
"link_type": 'hardlink-or-copy',
"scratch": False,
})
@mock.patch('pungi.phases.image_build.ThreadPool')
def test_image_build_create_release(self, ThreadPool):
compose = _DummyCompose({
'image_build': {
'^Server$': [
{
'image-build': {
'format': [('docker', 'tar.xz')],
'name': 'Fedora-Docker-Base',
'target': 'f24',
'version': 'Rawhide',
'ksurl': 'git://git.fedorahosted.org/git/spin-kickstarts.git',
'kickstart': "fedora-docker-base.ks",
'distro': 'Fedora-20',
'disk_size': 3,
'arches': ['x86_64'],
'release': None,
}
}
]
},
'koji_profile': 'koji',
})
phase = ImageBuildPhase(compose)
phase.run()
# assert at least one thread was started
self.assertTrue(phase.pool.add.called)
self.assertTrue(phase.pool.queue_put.called_once)
args, kwargs = phase.pool.queue_put.call_args
self.assertEqual(args[0][1].get('image_conf', {}).get('image-build', {}).get('release'),
'20151203.0')
@mock.patch('pungi.phases.image_build.ThreadPool')
def test_image_build_scratch_build(self, ThreadPool):
compose = _DummyCompose({
'image_build': {
'^Server$': [
{
'image-build': {
'format': [('docker', 'tar.xz')],
'name': 'Fedora-Docker-Base',
'target': 'f24',
'version': 'Rawhide',
'ksurl': 'git://git.fedorahosted.org/git/spin-kickstarts.git',
'kickstart': "fedora-docker-base.ks",
'distro': 'Fedora-20',
'disk_size': 3,
'arches': ['x86_64'],
'scratch': True,
}
}
]
},
'koji_profile': 'koji',
})
phase = ImageBuildPhase(compose)
phase.run()
# assert at least one thread was started
self.assertTrue(phase.pool.add.called)
self.assertTrue(phase.pool.queue_put.called_once)
args, kwargs = phase.pool.queue_put.call_args
self.assertTrue(args[0][1].get('scratch'))
class TestCreateImageBuildThread(unittest.TestCase):
@mock.patch('pungi.phases.image_build.KojiWrapper')
@mock.patch('pungi.phases.image_build.Linker')
@mock.patch('pungi.phases.image_build.makedirs')
def test_process(self, makedirs, Linker, KojiWrapper):
compose = _DummyCompose({
'koji_profile': 'koji'
})
pool = mock.Mock()
cmd = {
"format": [('docker', 'tar.xz'), ('qcow2', 'qcow2')],
"image_conf": {
'image-build': {
'install_tree': '/ostree/$arch/Client',
'kickstart': 'fedora-docker-base.ks',
'format': 'docker',
'repo': '/ostree/$arch/Client',
'variant': compose.variants['Client'],
'target': 'f24',
'disk_size': 3,
'name': 'Fedora-Docker-Base',
'arches': 'amd64,x86_64',
'version': 'Rawhide',
'ksurl': 'git://git.fedorahosted.org/git/spin-kickstarts.git',
'distro': 'Fedora-20',
}
},
"conf_file": 'amd64,x86_64-Client-Fedora-Docker-Base-docker',
"image_dir": '/image_dir/Client/%(arch)s',
"relative_image_dir": 'image_dir/Client/%(arch)s',
"link_type": 'hardlink-or-copy',
"scratch": False,
}
koji_wrapper = KojiWrapper.return_value
koji_wrapper.run_blocking_cmd.return_value = {
"retcode": 0,
"output": None,
"task_id": 1234,
}
koji_wrapper.get_image_paths.return_value = {
'amd64': [
'/koji/task/1235/tdl-amd64.xml',
'/koji/task/1235/Fedora-Docker-Base-20160103.amd64.qcow2',
'/koji/task/1235/Fedora-Docker-Base-20160103.amd64.tar.xz'
],
'x86_64': [
'/koji/task/1235/tdl-x86_64.xml',
'/koji/task/1235/Fedora-Docker-Base-20160103.x86_64.qcow2',
'/koji/task/1235/Fedora-Docker-Base-20160103.x86_64.tar.xz'
]
}
linker = Linker.return_value
t = CreateImageBuildThread(pool)
with mock.patch('os.stat') as stat:
with mock.patch('os.path.getsize') as getsize:
with mock.patch('time.sleep'):
getsize.return_value = 1024
stat.return_value.st_mtime = 13579
t.process((compose, cmd), 1)
self.assertItemsEqual(
linker.mock_calls,
[mock.call('/koji/task/1235/Fedora-Docker-Base-20160103.amd64.qcow2',
'/image_dir/Client/amd64/Fedora-Docker-Base-20160103.amd64.qcow2',
link_type='hardlink-or-copy'),
mock.call('/koji/task/1235/Fedora-Docker-Base-20160103.amd64.tar.xz',
'/image_dir/Client/amd64/Fedora-Docker-Base-20160103.amd64.tar.xz',
link_type='hardlink-or-copy'),
mock.call('/koji/task/1235/Fedora-Docker-Base-20160103.x86_64.qcow2',
'/image_dir/Client/x86_64/Fedora-Docker-Base-20160103.x86_64.qcow2',
link_type='hardlink-or-copy'),
mock.call('/koji/task/1235/Fedora-Docker-Base-20160103.x86_64.tar.xz',
'/image_dir/Client/x86_64/Fedora-Docker-Base-20160103.x86_64.tar.xz',
link_type='hardlink-or-copy')])
image_relative_paths = {
'image_dir/Client/amd64/Fedora-Docker-Base-20160103.amd64.qcow2': {
'format': 'qcow2',
'type': 'qcow2',
'arch': 'amd64',
},
'image_dir/Client/amd64/Fedora-Docker-Base-20160103.amd64.tar.xz': {
'format': 'tar.xz',
'type': 'docker',
'arch': 'amd64',
},
'image_dir/Client/x86_64/Fedora-Docker-Base-20160103.x86_64.qcow2': {
'format': 'qcow2',
'type': 'qcow2',
'arch': 'x86_64',
},
'image_dir/Client/x86_64/Fedora-Docker-Base-20160103.x86_64.tar.xz': {
'format': 'tar.xz',
'type': 'docker',
'arch': 'x86_64',
},
}
# Assert there are 4 images added to manifest and the arguments are sane
self.assertEqual(len(compose.im.add.call_args_list), 4)
for call in compose.im.add.call_args_list:
_, kwargs = call
image = kwargs['image']
self.assertEqual(kwargs['variant'], 'Client')
self.assertIn(kwargs['arch'], ('amd64', 'x86_64'))
self.assertEqual(kwargs['arch'], image.arch)
self.assertIn(image.path, image_relative_paths)
data = image_relative_paths.pop(image.path)
self.assertEqual(data['format'], image.format)
self.assertEqual(data['type'], image.type)
self.assertItemsEqual(makedirs.mock_calls,
[mock.call('/image_dir/Client/amd64'),
mock.call('/image_dir/Client/amd64'),
mock.call('/image_dir/Client/x86_64'),
mock.call('/image_dir/Client/x86_64')])
@mock.patch('pungi.phases.image_build.KojiWrapper')
@mock.patch('pungi.phases.image_build.Linker')
def test_process_handle_fail(self, Linker, KojiWrapper):
compose = _DummyCompose({
'koji_profile': 'koji',
'failable_deliverables': [
('^.*$', {
'*': ['image-build']
})
]
})
pool = mock.Mock()
cmd = {
"format": [('docker', 'tar.xz'), ('qcow2', 'qcow2')],
"image_conf": {
'image-build': {
'install_tree': '/ostree/$arch/Client',
'kickstart': 'fedora-docker-base.ks',
'format': 'docker',
'repo': '/ostree/$arch/Client',
'variant': compose.variants['Client'],
'target': 'f24',
'disk_size': 3,
'name': 'Fedora-Docker-Base',
'arches': 'amd64,x86_64',
'version': 'Rawhide',
'ksurl': 'git://git.fedorahosted.org/git/spin-kickstarts.git',
'distro': 'Fedora-20',
}
},
"conf_file": 'amd64,x86_64-Client-Fedora-Docker-Base-docker',
"image_dir": '/image_dir/Client/%(arch)s',
"relative_image_dir": 'image_dir/Client/%(arch)s',
"link_type": 'hardlink-or-copy',
'scratch': False,
}
koji_wrapper = KojiWrapper.return_value
koji_wrapper.run_blocking_cmd.return_value = {
"retcode": 1,
"output": None,
"task_id": 1234,
}
t = CreateImageBuildThread(pool)
with mock.patch('os.stat') as stat:
with mock.patch('os.path.getsize') as getsize:
with mock.patch('time.sleep'):
getsize.return_value = 1024
stat.return_value.st_mtime = 13579
t.process((compose, cmd), 1)
@mock.patch('pungi.phases.image_build.KojiWrapper')
@mock.patch('pungi.phases.image_build.Linker')
def test_process_handle_exception(self, Linker, KojiWrapper):
compose = _DummyCompose({
'koji_profile': 'koji',
'failable_deliverables': [
('^.*$', {
'*': ['image-build']
})
]
})
pool = mock.Mock()
cmd = {
"format": [('docker', 'tar.xz'), ('qcow2', 'qcow2')],
"image_conf": {
'image-build': {
'install_tree': '/ostree/$arch/Client',
'kickstart': 'fedora-docker-base.ks',
'format': 'docker',
'repo': '/ostree/$arch/Client',
'variant': compose.variants['Client'],
'target': 'f24',
'disk_size': 3,
'name': 'Fedora-Docker-Base',
'arches': 'amd64,x86_64',
'version': 'Rawhide',
'ksurl': 'git://git.fedorahosted.org/git/spin-kickstarts.git',
'distro': 'Fedora-20',
}
},
"conf_file": 'amd64,x86_64-Client-Fedora-Docker-Base-docker',
"image_dir": '/image_dir/Client/%(arch)s',
"relative_image_dir": 'image_dir/Client/%(arch)s',
"link_type": 'hardlink-or-copy',
'scratch': False,
}
def boom(*args, **kwargs):
raise RuntimeError('BOOM')
koji_wrapper = KojiWrapper.return_value
koji_wrapper.run_blocking_cmd.side_effect = boom
t = CreateImageBuildThread(pool)
with mock.patch('os.stat') as stat:
with mock.patch('os.path.getsize') as getsize:
with mock.patch('time.sleep'):
getsize.return_value = 1024
stat.return_value.st_mtime = 13579
t.process((compose, cmd), 1)
if __name__ == "__main__":
unittest.main()