almalinux/pungi
14
1
Fork 0
Code Issues Pull Requests Releases Activity

[image-build] Use single koji task per variant

Browse Source 
Given a list of arches, koji can build multiple images in one go
(automatically starting children tasks for each one).

This causes a bunch of changes:
 * The configuration no longer allows changing config based on
   architecture, only variants are allowed. It is however possible to
   filter which arches are used for the building in the variant.
 * The configuration files for koji image-build are stored in
   work/image-build/$variant (not split based on arch).

This patch also changes the option name that is passed to koji
image-build: the repos should be specified under key `repo` (without the
trailing slash).

Signed-off-by: Lubomír Sedlář <lsedlar@redhat.com>
This commit is contained in:
Lubomír Sedlář 2016-01-05 09:27:20 +01:00
parent 88bb6ac3d7
commit 9c1418eb0a
8 changed files with 780 additions and 107 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

79
doc/configuration.rst
View File

@ -598,18 +598,22 @@ Image Build Settings
==================== ====================
**image_build** **image_build**
(*list*) -- config for koji image-build; format: [(variant_uid_regex, {arch|*: [{opt: value}])] (*dict*) -- config for ``koji image-build``; format: {variant_uid_regex: [{opt: value}]}
By default, images will be built for each binary arch valid for the
variant. The config can specify a list of arches to narrow this down.
.. note:: .. note::
Config can contain anything what is accepted by Config can contain anything what is accepted by
``koji image-build --config configfile.ini`` ``koji image-build --config configfile.ini``
Repo is currently the only option which is being automatically transformed Repo can be specified either as a string or a list of strings. It will
into a string. automatically transformed into format suitable for ``koji``. A repo for the
currently built variant will be added as well.
Please don't set install_tree as it would get overriden by pungi. Please don't set ``install_tree`` as it would get overriden by pungi.
The 'format' attr is [('image_type', 'image_suffix'), ...]. The ``format`` attr is [('image_type', 'image_suffix'), ...].
productmd should ideally contain all of image types and suffixes. See productmd documentation for list of supported types and suffixes.
If ``ksurl`` ends with ``#HEAD``, Pungi will figure out the SHA1 hash of If ``ksurl`` ends with ``#HEAD``, Pungi will figure out the SHA1 hash of
current HEAD and use that instead. current HEAD and use that instead.
@ -619,36 +623,39 @@ Example
------- -------
:: ::
image_build = [ image_build = {
('^Server$', { '^Server$': [
'x86_64': [ {
{ 'format': [('docker', 'tar.gz'), ('qcow2', 'qcow2')]
'format': [('docker', 'tar.gz'), ('qcow2', 'qcow2')] 'name': 'fedora-qcow-and-docker-base',
'name': 'fedora-qcow-and-docker-base', 'target': 'koji-target-name',
'target': 'koji-target-name', 'ksversion': 'F23', # value from pykickstart
'ksversion': 'F23', # value from pykickstart 'version': '23',
'version': '23', # correct SHA1 hash will be put into the URL below automatically
# correct SHA1 hash will be put into the URL below automatically 'ksurl': 'https://git.fedorahosted.org/git/spin-kickstarts.git?somedirectoryifany#HEAD',
'ksurl': 'https://git.fedorahosted.org/git/spin-kickstarts.git?somedirectoryifany#HEAD', 'kickstart': "fedora-docker-base.ks",
'kickstart': "fedora-docker-base.ks", 'repo': ["http://someextrarepos.org/repo", "ftp://rekcod.oi/repo].
'repo': ["http://someextrarepos.org/repo", "ftp://rekcod.oi/repo]. 'distro': 'Fedora-20',
# 'install_tree': 'http://sometpath', # this is set automatically by pungi to os_dir for given variant/$arch 'disk_size': 3,
'distro': 'Fedora-20',
'disk_size': 3 # this is set automatically by pungi to os_dir for given variant
}, # 'install_tree': 'http://sometpath',
{ },
'format': [('qcow2','qcow2')] {
'name': 'fedora-qcow-base', 'format': [('qcow2','qcow2')]
'target': 'koji-target-name', 'name': 'fedora-qcow-base',
'ksversion': 'F23', # value from pykickstart 'target': 'koji-target-name',
'version': '23', 'ksversion': 'F23', # value from pykickstart
'ksurl': 'https://git.fedorahosted.org/git/spin-kickstarts.git?somedirectoryifany#HEAD', 'version': '23',
'kickstart': "fedora-docker-base.ks", 'ksurl': 'https://git.fedorahosted.org/git/spin-kickstarts.git?somedirectoryifany#HEAD',
'distro': 'Fedora-23' 'kickstart': "fedora-docker-base.ks",
} 'distro': 'Fedora-23',
]
}), # only build this type of image on x86_64
] 'arches': ['x86_64']
}
]
}
Media Checksums Settings Media Checksums Settings

27
pungi/paths.py
View File

@ -305,32 +305,30 @@ class WorkPaths(object):
path = os.path.join(path, file_name) path = os.path.join(path, file_name)
return path return path
def image_build_dir(self, arch, variant, create_dir=True): def image_build_dir(self, variant, create_dir=True):
""" """
@param arch
@param variant @param variant
@param create_dir=True @param create_dir=True
Examples: Examples:
work/x86_64/Server/image-build work/image-build/Server
""" """
path = os.path.join(self.topdir(arch, create_dir=create_dir), variant.uid, "image-build") path = os.path.join(self.topdir('image-build', create_dir=create_dir), variant.uid)
if create_dir: if create_dir:
makedirs(path) makedirs(path)
return path return path
def image_build_conf(self, arch, variant, image_name, image_type, create_dir=True): def image_build_conf(self, variant, image_name, image_type, create_dir=True):
""" """
@param arch
@param variant @param variant
@param image-name @param image-name
@param image-type (e.g docker) @param image-type (e.g docker)
@param create_dir=True @param create_dir=True
Examples: Examples:
work/x86_64/Server/image-build/docker_rhel-server-docker.cfg work/image-build/Server/docker_rhel-server-docker.cfg
""" """
path = os.path.join(self.image_build_dir(arch, variant), "%s_%s.cfg" % (image_type, image_name)) path = os.path.join(self.image_build_dir(variant), "%s_%s.cfg" % (image_type, image_name))
return path return path
@ -517,24 +515,23 @@ class ComposePaths(object):
return os.path.join(path, filename) return os.path.join(path, filename)
def image_dir(self, arch, variant, symlink_to=None, create_dir=True, relative=False): def image_dir(self, variant, symlink_to=None, create_dir=True, relative=False):
""" """
The arch is listed as literal '%(arch)s'
Examples: Examples:
compose/Server/x86_64/images compose/Server/%(arch)s/images
None None
@param arch
@param variant @param variant
@param symlink_to=None @param symlink_to=None
@param create_dir=True @param create_dir=True
@param relative=False @param relative=False
""" """
# skip optional, addons and src architecture # skip optional and addons
if variant.type != "variant": if variant.type != "variant":
return None return None
if arch == "src":
return None
path = os.path.join(self.topdir(arch, variant, create_dir=create_dir, relative=relative), "images") path = os.path.join(self.topdir('%(arch)s', variant, create_dir=create_dir, relative=relative),
"images")
if symlink_to: if symlink_to:
topdir = self.compose.topdir.rstrip("/") + "/" topdir = self.compose.topdir.rstrip("/") + "/"
relative_dir = path[len(topdir):] relative_dir = path[len(topdir):]

152
pungi/phases/image_build.py
View File

@ -4,7 +4,7 @@ import copy
import os import os
import time import time
from pungi.util import get_arch_variant_data, resolve_git_url from pungi.util import get_variant_data, resolve_git_url
from pungi.phases.base import PhaseBase from pungi.phases.base import PhaseBase
from pungi.linker import Linker from pungi.linker import Linker
from pungi.paths import translate_path from pungi.paths import translate_path
@ -30,46 +30,65 @@ class ImageBuildPhase(PhaseBase):
return False return False
def run(self): def run(self):
for arch in self.compose.get_arches(): # src will be skipped for variant in self.compose.get_variants():
for variant in self.compose.get_variants(arch=arch): arches = set([x for x in variant.arches if x != 'src'])
image_build_data = get_arch_variant_data(self.compose.conf, self.name, arch, variant)
for image_conf in image_build_data:
# We will modify the data, so we need to make a copy to
# prevent problems in next iteration where the original
# value is needed.
image_conf = copy.deepcopy(image_conf)
# Replace possible ambiguous ref name with explicit hash. for image_conf in get_variant_data(self.compose.conf, self.name, variant):
if 'ksurl' in image_conf: # We will modify the data, so we need to make a copy to
image_conf['ksurl'] = resolve_git_url(image_conf['ksurl']) # prevent problems in next iteration where the original
image_conf["arches"] = arch # passed to get_image_build_cmd as dict # value is needed.
image_conf["variant"] = variant # ^ image_conf = copy.deepcopy(image_conf)
image_conf["install_tree"] = translate_path(self.compose, self.compose.paths.compose.os_tree(arch, variant)) # ^
format = image_conf["format"] # transform format into right 'format' for image-build
image_conf["format"] = ",".join([x[0] for x in image_conf["format"]]) # 'docker,qcow2'
repos = image_conf.get('repos', []) # Replace possible ambiguous ref name with explicit hash.
if isinstance(repos, str): if 'ksurl' in image_conf:
repos = [repos] image_conf['ksurl'] = resolve_git_url(image_conf['ksurl'])
repos.append(translate_path(self.compose, self.compose.paths.compose.os_tree(arch, variant)))
image_conf['repos'] = ",".join(repos) # supply repos as str separated by , instead of list # image_conf is passed to get_image_build_cmd as dict
if 'arches' in image_conf:
image_conf["arches"] = ','.join(sorted(set(image_conf.get('arches', [])) & arches))
else:
image_conf['arches'] = ','.join(sorted(arches))
if not image_conf['arches']:
continue
image_conf["variant"] = variant
image_conf["install_tree"] = translate_path(
self.compose,
self.compose.paths.compose.os_tree('$arch', variant)
)
# transform format into right 'format' for image-build
# e.g. 'docker,qcow2'
format = image_conf["format"]
image_conf["format"] = ",".join([x[0] for x in image_conf["format"]])
repo = image_conf.get('repo', [])
if isinstance(repo, str):
repo = [repo]
repo.append(translate_path(self.compose, self.compose.paths.compose.os_tree('$arch', variant)))
# supply repo as str separated by , instead of list
image_conf['repo'] = ",".join(repo)
cmd = {
"format": format,
"image_conf": image_conf,
"conf_file": self.compose.paths.work.image_build_conf(
image_conf['variant'],
image_name=image_conf['name'],
image_type=image_conf['format'].replace(",", "-")
),
"image_dir": self.compose.paths.compose.image_dir(variant),
"relative_image_dir": self.compose.paths.compose.image_dir(
variant, create_dir=False, relative=True
),
"link_type": self.compose.conf.get("link_type", "hardlink-or-copy")
}
self.pool.add(CreateImageBuildThread(self.pool))
self.pool.queue_put((self.compose, cmd))
cmd = {
"format": format,
"image_conf": image_conf,
"conf_file": self.compose.paths.work.image_build_conf(image_conf["arches"], image_conf['variant'], image_name=image_conf['name'], image_type=image_conf['format'].replace(",", "-")),
"image_dir": self.compose.paths.compose.image_dir(arch, variant),
"relative_image_dir": self.compose.paths.compose.image_dir(arch, variant, create_dir=False, relative=True),
"link_type": self.compose.conf.get("link_type", "hardlink-or-copy")
}
self.pool.add(CreateImageBuildThread(self.pool))
self.pool.queue_put((self.compose, cmd))
self.pool.start() self.pool.start()
def stop(self, *args, **kwargs):
PhaseBase.stop(self, *args, **kwargs)
if self.skip():
return
class CreateImageBuildThread(WorkerThread): class CreateImageBuildThread(WorkerThread):
def fail(self, compose, cmd): def fail(self, compose, cmd):
@ -77,19 +96,29 @@ class CreateImageBuildThread(WorkerThread):
def process(self, item, num): def process(self, item, num):
compose, cmd = item compose, cmd = item
arches = cmd['image_conf']['arches'].split(',')
mounts = [compose.paths.compose.topdir()] mounts = [compose.paths.compose.topdir()]
if "mount" in cmd: if "mount" in cmd:
mounts.append(cmd["mount"]) mounts.append(cmd["mount"])
log_file = compose.paths.log.log_file(cmd["image_conf"]["arches"], "imagebuild-%s-%s-%s" % (cmd["image_conf"]["arches"], cmd["image_conf"]["variant"], cmd['image_conf']['format'].replace(",","-"))) log_file = compose.paths.log.log_file(
msg = "Creating %s image (arch: %s, variant: %s)" % (cmd["image_conf"]["format"].replace(",","-"), cmd["image_conf"]["arches"], cmd["image_conf"]["variant"]) cmd["image_conf"]["arches"],
"imagebuild-%s-%s-%s" % ('-'.join(arches),
cmd["image_conf"]["variant"],
cmd['image_conf']['format'].replace(",", "-"))
)
msg = "Creating %s image (arches: %s, variant: %s)" % (cmd["image_conf"]["format"].replace(",", "-"),
'-'.join(arches),
cmd["image_conf"]["variant"])
self.pool.log_info("[BEGIN] %s" % msg) self.pool.log_info("[BEGIN] %s" % msg)
koji_wrapper = KojiWrapper(compose.conf["koji_profile"]) koji_wrapper = KojiWrapper(compose.conf["koji_profile"])
# paths module doesn't hold compose object, so we have to generate path here
# writes conf file for koji image-build # writes conf file for koji image-build
self.pool.log_info("Writing image-build config for %s.%s into %s" % (cmd["image_conf"]["variant"], cmd["image_conf"]["arches"], cmd["conf_file"])) self.pool.log_info("Writing image-build config for %s.%s into %s" % (
koji_cmd = koji_wrapper.get_image_build_cmd(cmd['image_conf'], conf_file_dest=cmd["conf_file"], wait=True, scratch=False) cmd["image_conf"]["variant"], '-'.join(arches), cmd["conf_file"]))
koji_cmd = koji_wrapper.get_image_build_cmd(cmd['image_conf'],
conf_file_dest=cmd["conf_file"])
# avoid race conditions? # avoid race conditions?
# Kerberos authentication failed: Permission denied in replay cache code (-1765328215) # Kerberos authentication failed: Permission denied in replay cache code (-1765328215)
@ -103,15 +132,22 @@ class CreateImageBuildThread(WorkerThread):
# copy image to images/ # copy image to images/
image_infos = [] image_infos = []
for filename in koji_wrapper.get_image_path(output["task_id"]): paths = koji_wrapper.get_image_build_paths(output["task_id"])
# format is list of tuples [('qcow2', '.qcow2'), ('raw-xz', 'raw.xz'),]
for format, suffix in cmd['format']:
if filename.endswith(suffix):
image_infos.append({'filename': filename, 'suffix': suffix, 'type': format}) # the type/format ... image-build has it wrong
if len(image_infos) != len(cmd['format']): for arch, paths in paths.iteritems():
self.pool.log_error("Error in koji task %s. Expected to find same amount of images as in suffixes attr in image-build (%s). Got '%s'." % for path in paths:
(output["task_id"], len(cmd['image_conf']['format']), len(image_infos))) # format is list of tuples [('qcow2', '.qcow2'), ('raw-xz', 'raw.xz'),]
for format, suffix in cmd['format']:
if path.endswith(suffix):
image_infos.append({'path': path, 'suffix': suffix, 'type': format, 'arch': arch})
break
if len(image_infos) != len(cmd['format']) * len(arches):
self.pool.log_error(
"Error in koji task %s. Expected to find same amount of images "
"as in suffixes attr in image-build (%s) for each arch (%s). Got '%s'." %
(output["task_id"], len(cmd['format']),
len(arches), len(image_infos)))
self.fail(compose, cmd) self.fail(compose, cmd)
# The usecase here is that you can run koji image-build with multiple --format # The usecase here is that you can run koji image-build with multiple --format
@ -119,22 +155,26 @@ class CreateImageBuildThread(WorkerThread):
# image_build record # image_build record
linker = Linker(logger=compose._logger) linker = Linker(logger=compose._logger)
for image_info in image_infos: for image_info in image_infos:
image_dir = cmd["image_dir"] % {"arch": image_info['arch']}
relative_image_dir = cmd["relative_image_dir"] % {"arch": image_info['arch']}
# let's not change filename of koji outputs # let's not change filename of koji outputs
image_dest = os.path.join(cmd["image_dir"], os.path.basename(image_info['filename'])) image_dest = os.path.join(image_dir, os.path.basename(image_info['path']))
linker.link(image_info['filename'], image_dest, link_type=cmd["link_type"]) linker.link(image_info['path'], image_dest, link_type=cmd["link_type"])
# Update image manifest # Update image manifest
img = Image(compose.im) img = Image(compose.im)
img.type = image_info['type'] img.type = image_info['type']
img.format = image_info['suffix'] img.format = image_info['suffix']
img.path = os.path.join(cmd["relative_image_dir"], os.path.basename(image_dest)) img.path = os.path.join(relative_image_dir, os.path.basename(image_dest))
img.mtime = int(os.stat(image_dest).st_mtime) img.mtime = int(os.stat(image_dest).st_mtime)
img.size = os.path.getsize(image_dest) img.size = os.path.getsize(image_dest)
img.arch = cmd["image_conf"]["arches"] # arches should be always single arch img.arch = image_info['arch']
img.disc_number = 1 # We don't expect multiple disks img.disc_number = 1 # We don't expect multiple disks
img.disc_count = 1 img.disc_count = 1
img.bootable = False img.bootable = False
# named keywords due portability (old productmd used arch, variant ... while new one uses variant, arch compose.im.add(variant=cmd["image_conf"]["variant"].uid,
compose.im.add(variant=cmd["image_conf"]["variant"].uid, arch=cmd["image_conf"]["arches"], image=img) arch=image_info['arch'],
image=img)
self.pool.log_info("[DONE ] %s" % msg) self.pool.log_info("[DONE ] %s" % msg)

21
pungi/util.py
View File

@ -257,6 +257,27 @@ def get_arch_data(conf, var_name, arch):
return result return result
def get_variant_data(conf, var_name, variant):
"""Get configuration for variant.
Expected config format is a mapping from variant_uid regexes to lists of
values.
:param var_name: name of configuration key with which to work
:param variant: Variant object for which to get configuration
:rtype: a list of values
"""
result = []
for conf_variant, conf_data in conf.get(var_name, {}).iteritems():
if not re.match(conf_variant, variant.uid):
continue
if isinstance(conf_data, list):
result.extend(conf_data)
else:
result.append(conf_data)
return result
def get_buildroot_rpms(compose, task_id): def get_buildroot_rpms(compose, task_id):
"""Get build root RPMs - either from runroot or local""" """Get build root RPMs - either from runroot or local"""
result = [] result = []

35
pungi/wrappers/kojiwrapper.py
View File

@ -209,6 +209,41 @@ class KojiWrapper(object):
} }
return result return result
def get_image_build_paths(self, task_id):
"""
Given an image task in Koji, get a mapping from arches to a list of
paths to results of the task.
"""
result = {}
# task = self.koji_proxy.getTaskInfo(task_id, request=True)
children_tasks = self.koji_proxy.getTaskChildren(task_id, request=True)
for child_task in children_tasks:
if child_task['method'] != 'createImage':
continue
is_scratch = child_task['request'][-1].get('scratch', False)
task_result = self.koji_proxy.getTaskResult(child_task['id'])
if is_scratch:
topdir = os.path.join(
self.koji_module.pathinfo.work(),
self.koji_module.pathinfo.taskrelpath(child_task['id'])
)
else:
build = self.koji_proxy.getImageBuild("%(name)s-%(version)s-%(release)s" % task_result)
build["name"] = task_result["name"]
build["version"] = task_result["version"]
build["release"] = task_result["release"]
build["arch"] = task_result["arch"]
topdir = self.koji_module.pathinfo.imagebuild(build)
for i in task_result["files"]:
result.setdefault(task_result['arch'], []).append(os.path.join(topdir, i))
return result
def get_image_path(self, task_id): def get_image_path(self, task_id):
result = [] result = []
koji_proxy = self.koji_module.ClientSession(self.koji_module.config.server) koji_proxy = self.koji_module.ClientSession(self.koji_module.config.server)

281
tests/test_imagebuildphase.py Executable file
View File

@ -0,0 +1,281 @@
#!/usr/bin/env python2
# -*- coding: utf-8 -*-
import unittest
import mock
import os
import sys
sys.path.insert(0, os.path.join(os.path.dirname(__file__), ".."))
from pungi.phases.image_build import ImageBuildPhase, CreateImageBuildThread
class _DummyCompose(object):
def __init__(self, config):
self.compose_date = '20151203'
self.compose_type_suffix = '.t'
self.compose_respin = 0
self.ci_base = mock.Mock(
release_id='Test-1.0',
release=mock.Mock(
short='test',
version='1.0',
),
)
self.conf = config
self.paths = mock.Mock(
compose=mock.Mock(
topdir=mock.Mock(return_value='/a/b'),
os_tree=mock.Mock(
side_effect=lambda arch, variant: os.path.join('/ostree', arch, variant.uid)
),
image_dir=mock.Mock(
side_effect=lambda variant, create_dir=False, relative=False: os.path.join(
'' if relative else '/', 'image_dir', variant.uid, '%(arch)s'
)
)
),
work=mock.Mock(
image_build_conf=mock.Mock(
side_effect=lambda variant, image_name, image_type:
'-'.join([variant.uid, image_name, image_type])
)
),
log=mock.Mock(
log_file=mock.Mock(return_value='/a/b/log/log_file')
)
)
self._logger = mock.Mock()
self.variants = [
mock.Mock(uid='Server', arches=['x86_64', 'amd64']),
mock.Mock(uid='Client', arches=['amd64']),
]
self.im = mock.Mock()
def get_arches(self):
return ['x86_64', 'amd64']
def get_variants(self, arch=None, types=None):
return [v for v in self.variants if not arch or arch in v.arches]
class TestImageBuildPhase(unittest.TestCase):
@mock.patch('pungi.phases.image_build.ThreadPool')
def test_image_build(self, ThreadPool):
compose = _DummyCompose({
'image_build': {
'^Client|Server$': [
{
'format': [('docker', 'tar.xz')],
'name': 'Fedora-Docker-Base',
'target': 'f24',
'version': 'Rawhide',
'ksurl': 'git://git.fedorahosted.org/git/spin-kickstarts.git',
'kickstart': "fedora-docker-base.ks",
'distro': 'Fedora-20',
'disk_size': 3
}
]
},
'koji_profile': 'koji',
})
phase = ImageBuildPhase(compose)
phase.run()
# assert at least one thread was started
self.assertTrue(phase.pool.add.called)
client_args = {
"format": [('docker', 'tar.xz')],
"image_conf": {
'install_tree': '/ostree/$arch/Client',
'kickstart': 'fedora-docker-base.ks',
'format': 'docker',
'repo': '/ostree/$arch/Client',
'variant': compose.variants[1],
'target': 'f24',
'disk_size': 3,
'name': 'Fedora-Docker-Base',
'arches': 'amd64',
'version': 'Rawhide',
'ksurl': 'git://git.fedorahosted.org/git/spin-kickstarts.git',
'distro': 'Fedora-20',
},
"conf_file": 'Client-Fedora-Docker-Base-docker',
"image_dir": '/image_dir/Client/%(arch)s',
"relative_image_dir": 'image_dir/Client/%(arch)s',
"link_type": 'hardlink-or-copy',
}
server_args = {
"format": [('docker', 'tar.xz')],
"image_conf": {
'install_tree': '/ostree/$arch/Server',
'kickstart': 'fedora-docker-base.ks',
'format': 'docker',
'repo': '/ostree/$arch/Server',
'variant': compose.variants[0],
'target': 'f24',
'disk_size': 3,
'name': 'Fedora-Docker-Base',
'arches': 'amd64,x86_64',
'version': 'Rawhide',
'ksurl': 'git://git.fedorahosted.org/git/spin-kickstarts.git',
'distro': 'Fedora-20',
},
"conf_file": 'Server-Fedora-Docker-Base-docker',
"image_dir": '/image_dir/Server/%(arch)s',
"relative_image_dir": 'image_dir/Server/%(arch)s',
"link_type": 'hardlink-or-copy',
}
self.assertItemsEqual(phase.pool.queue_put.mock_calls,
[mock.call((compose, client_args)),
mock.call((compose, server_args))])
@mock.patch('pungi.phases.image_build.ThreadPool')
def test_image_build_filter_all_variants(self, ThreadPool):
compose = _DummyCompose({
'image_build': {
'^Client|Server$': [
{
'format': [('docker', 'tar.xz')],
'name': 'Fedora-Docker-Base',
'target': 'f24',
'version': 'Rawhide',
'ksurl': 'git://git.fedorahosted.org/git/spin-kickstarts.git',
'kickstart': "fedora-docker-base.ks",
'distro': 'Fedora-20',
'disk_size': 3,
'arches': ['non-existing'],
}
]
},
'koji_profile': 'koji',
})
phase = ImageBuildPhase(compose)
phase.run()
# assert at least one thread was started
self.assertFalse(phase.pool.add.called)
self.assertFalse(phase.pool.queue_put.called)
class TestCreateImageBuildThread(unittest.TestCase):
@mock.patch('pungi.phases.image_build.KojiWrapper')
@mock.patch('pungi.phases.image_build.Linker')
def test_process(self, Linker, KojiWrapper):
compose = _DummyCompose({
'koji_profile': 'koji'
})
pool = mock.Mock()
cmd = {
"format": [('docker', 'tar.xz'), ('qcow2', 'qcow2')],
"image_conf": {
'install_tree': '/ostree/$arch/Client',
'kickstart': 'fedora-docker-base.ks',
'format': 'docker',
'repo': '/ostree/$arch/Client',
'variant': compose.variants[1],
'target': 'f24',
'disk_size': 3,
'name': 'Fedora-Docker-Base',
'arches': 'amd64,x86_64',
'version': 'Rawhide',
'ksurl': 'git://git.fedorahosted.org/git/spin-kickstarts.git',
'distro': 'Fedora-20',
},
"conf_file": 'amd64,x86_64-Client-Fedora-Docker-Base-docker',
"image_dir": '/image_dir/Client/%(arch)s',
"relative_image_dir": 'image_dir/Client/%(arch)s',
"link_type": 'hardlink-or-copy',
}
koji_wrapper = KojiWrapper.return_value
koji_wrapper.run_create_image_cmd.return_value = {
"retcode": 0,
"output": None,
"task_id": 1234,
}
koji_wrapper.get_image_build_paths.return_value = {
'amd64': [
'/koji/task/1235/tdl-amd64.xml',
'/koji/task/1235/Fedora-Docker-Base-20160103.amd64.qcow2',
'/koji/task/1235/Fedora-Docker-Base-20160103.amd64.tar.xz'
],
'x86_64': [
'/koji/task/1235/tdl-x86_64.xml',
'/koji/task/1235/Fedora-Docker-Base-20160103.x86_64.qcow2',
'/koji/task/1235/Fedora-Docker-Base-20160103.x86_64.tar.xz'
]
}
linker = Linker.return_value
t = CreateImageBuildThread(pool)
with mock.patch('os.stat') as stat:
with mock.patch('os.path.getsize') as getsize:
with mock.patch('time.sleep'):
getsize.return_value = 1024
stat.return_value.st_mtime = 13579
t.process((compose, cmd), 1)
self.assertItemsEqual(
linker.mock_calls,
[mock.call('/koji/task/1235/Fedora-Docker-Base-20160103.amd64.qcow2',
'/image_dir/Client/amd64/Fedora-Docker-Base-20160103.amd64.qcow2',
link_type='hardlink-or-copy'),
mock.call('/koji/task/1235/Fedora-Docker-Base-20160103.amd64.tar.xz',
'/image_dir/Client/amd64/Fedora-Docker-Base-20160103.amd64.tar.xz',
link_type='hardlink-or-copy'),
mock.call('/koji/task/1235/Fedora-Docker-Base-20160103.x86_64.qcow2',
'/image_dir/Client/x86_64/Fedora-Docker-Base-20160103.x86_64.qcow2',
link_type='hardlink-or-copy'),
mock.call('/koji/task/1235/Fedora-Docker-Base-20160103.x86_64.tar.xz',
'/image_dir/Client/x86_64/Fedora-Docker-Base-20160103.x86_64.tar.xz',
link_type='hardlink-or-copy')])
image_relative_paths = {
'image_dir/Client/amd64/Fedora-Docker-Base-20160103.amd64.qcow2': {
'format': 'qcow2',
'type': 'qcow2',
'arch': 'amd64',
},
'image_dir/Client/amd64/Fedora-Docker-Base-20160103.amd64.tar.xz': {
'format': 'tar.xz',
'type': 'docker',
'arch': 'amd64',
},
'image_dir/Client/x86_64/Fedora-Docker-Base-20160103.x86_64.qcow2': {
'format': 'qcow2',
'type': 'qcow2',
'arch': 'x86_64',
},
'image_dir/Client/x86_64/Fedora-Docker-Base-20160103.x86_64.tar.xz': {
'format': 'tar.xz',
'type': 'docker',
'arch': 'x86_64',
},
}
# Assert there are 4 images added to manifest and the arguments are sane
self.assertEqual(len(compose.im.add.call_args_list), 4)
for call in compose.im.add.call_args_list:
_, kwargs = call
image = kwargs['image']
self.assertEqual(kwargs['variant'], 'Client')
self.assertIn(kwargs['arch'], ('amd64', 'x86_64'))
self.assertEqual(kwargs['arch'], image.arch)
self.assertIn(image.path, image_relative_paths)
data = image_relative_paths.pop(image.path)
self.assertEqual(data['format'], image.format)
self.assertEqual(data['type'], image.type)
if __name__ == "__main__":
unittest.main()

258
tests/test_koji_wrapper.py Executable file
View File

@ -0,0 +1,258 @@
#!/usr/bin/env python2
# -*- coding: utf-8 -*-
import mock
import unittest
import os
import sys
sys.path.insert(0, os.path.join(os.path.dirname(__file__), ".."))
from pungi.wrappers.kojiwrapper import KojiWrapper
class KojiWrapperTest(unittest.TestCase):
def setUp(self):
self.koji_profile = mock.Mock()
with mock.patch('pungi.wrappers.kojiwrapper.koji') as koji:
koji.get_profile_module = mock.Mock(
return_value=mock.Mock(
pathinfo=mock.Mock(
work=mock.Mock(return_value='/koji'),
taskrelpath=mock.Mock(side_effect=lambda id: 'task/' + str(id)),
imagebuild=mock.Mock(side_effect=lambda id: '/koji/imagebuild/' + str(id)),
)
)
)
self.koji_profile = koji.get_profile_module.return_value
self.koji = KojiWrapper('koji')
@mock.patch('pungi.wrappers.kojiwrapper.open')
def test_get_image_build_cmd_without_required_data(self, mock_open):
with self.assertRaises(AssertionError):
self.koji.get_image_build_cmd(
{
'name': 'test-name',
},
'/tmp/file'
)
@mock.patch('pungi.wrappers.kojiwrapper.open')
def test_get_image_build_cmd_correct(self, mock_open):
cmd = self.koji.get_image_build_cmd(
{
'name': 'test-name',
'version': '1',
'target': 'test-target',
'install_tree': '/tmp/test/install_tree',
'arches': 'x86_64',
'format': 'docker,qcow2',
'kickstart': 'test-kickstart',
'ksurl': 'git://example.com/ks.git',
'distro': 'test-distro',
},
'/tmp/file'
)
self.assertEqual(cmd[0], 'koji')
self.assertEqual(cmd[1], 'image-build')
self.assertItemsEqual(cmd[2:],
['--config=/tmp/file', '--wait'])
output = mock_open.return_value
self.assertEqual(mock.call('[image-build]\n'), output.write.mock_calls[0])
self.assertItemsEqual(output.write.mock_calls[1:],
[mock.call('name = test-name\n'),
mock.call('version = 1\n'),
mock.call('target = test-target\n'),
mock.call('install_tree = /tmp/test/install_tree\n'),
mock.call('arches = x86_64\n'),
mock.call('format = docker,qcow2\n'),
mock.call('kickstart = test-kickstart\n'),
mock.call('ksurl = git://example.com/ks.git\n'),
mock.call('distro = test-distro\n'),
mock.call('\n')])
def test_get_image_build_paths(self):
# The data for this tests is obtained from the actual Koji build. It
# includes lots of fields that are not used, but for the sake of
# completeness is fully preserved.
getTaskChildren_data = {
12387273: [
{
'arch': 'i386',
'awaited': False,
'channel_id': 12,
'completion_time': '2016-01-03 05:34:08.374262',
'completion_ts': 1451799248.37426,
'create_time': '2016-01-03 05:15:20.311599',
'create_ts': 1451798120.3116,
'host_id': 158,
'id': 12387276,
'label': 'i386',
'method': 'createImage',
'owner': 131,
'parent': 12387273,
'priority': 19,
'request': [
'Fedora-Cloud-Base',
'23',
'20160103',
'i386',
{
'build_tag': 299,
'build_tag_name': 'f23-build',
'dest_tag': 294,
'dest_tag_name': 'f23-updates-candidate',
'id': 144,
'name': 'f23-candidate'
},
299,
{
'create_event': 14011966,
'create_ts': 1451761803.33528,
'creation_time': '2016-01-02 19:10:03.335283',
'id': 563977,
'state': 1
},
'http://infrastructure.fedoraproject.org/pub/alt/releases/23/Cloud/i386/os/',
{
'disk_size': '3',
'distro': 'Fedora-20',
'format': ['qcow2', 'raw-xz'],
'kickstart': 'work/cli-image/1451798116.800155.wYJWTVHw/fedora-cloud-base-2878aa0.ks',
'release': '20160103',
'repo': ['http://infrastructure.fedoraproject.org/pub/alt/releases/23/Cloud/$arch/os/',
'http://infrastructure.fedoraproject.org/pub/fedora/linux/updates/23/$arch/'],
'scratch': True
}
],
'start_time': '2016-01-03 05:15:29.828081',
'start_ts': 1451798129.82808,
'state': 2,
'waiting': None,
'weight': 2.0
}, {
'arch': 'x86_64',
'awaited': False,
'channel_id': 12,
'completion_time': '2016-01-03 05:33:20.066366',
'completion_ts': 1451799200.06637,
'create_time': '2016-01-03 05:15:20.754201',
'create_ts': 1451798120.7542,
'host_id': 156,
'id': 12387277,
'label': 'x86_64',
'method': 'createImage',
'owner': 131,
'parent': 12387273,
'priority': 19,
'request': [
'Fedora-Cloud-Base',
'23',
'20160103',
'x86_64',
{
'build_tag': 299,
'build_tag_name': 'f23-build',
'dest_tag': 294,
'dest_tag_name': 'f23-updates-candidate',
'id': 144,
'name': 'f23-candidate'
},
299,
{
'create_event': 14011966,
'create_ts': 1451761803.33528,
'creation_time': '2016-01-02 19:10:03.335283',
'id': 563977,
'state': 1
},
'http://infrastructure.fedoraproject.org/pub/alt/releases/23/Cloud/x86_64/os/',
{
'disk_size': '3',
'distro': 'Fedora-20',
'format': ['qcow2', 'raw-xz'],
'kickstart': 'work/cli-image/1451798116.800155.wYJWTVHw/fedora-cloud-base-2878aa0.ks',
'release': '20160103',
'repo': ['http://infrastructure.fedoraproject.org/pub/alt/releases/23/Cloud/$arch/os/',
'http://infrastructure.fedoraproject.org/pub/fedora/linux/updates/23/$arch/'],
'scratch': True
}
],
'start_time': '2016-01-03 05:15:35.196043',
'start_ts': 1451798135.19604,
'state': 2,
'waiting': None,
'weight': 2.0
}
]
}
getTaskResult_data = {
12387276: {
'arch': 'i386',
'files': ['tdl-i386.xml',
'fedora-cloud-base-2878aa0.ks',
'koji-f23-build-12387276-base.ks',
'libvirt-qcow2-i386.xml',
'Fedora-Cloud-Base-23-20160103.i386.qcow2',
'libvirt-raw-xz-i386.xml',
'Fedora-Cloud-Base-23-20160103.i386.raw.xz'],
'logs': ['oz-i386.log'],
'name': 'Fedora-Cloud-Base',
'release': '20160103',
'rpmlist': [],
'task_id': 12387276,
'version': '23'
},
12387277: {
'arch': 'x86_64',
'files': ['tdl-x86_64.xml',
'fedora-cloud-base-2878aa0.ks',
'koji-f23-build-12387277-base.ks',
'libvirt-qcow2-x86_64.xml',
'Fedora-Cloud-Base-23-20160103.x86_64.qcow2',
'libvirt-raw-xz-x86_64.xml',
'Fedora-Cloud-Base-23-20160103.x86_64.raw.xz'],
'logs': ['oz-x86_64.log'],
'name': 'Fedora-Cloud-Base',
'release': '20160103',
'rpmlist': [],
'task_id': 12387277,
'version': '23'
}
}
self.koji.koji_proxy = mock.Mock(
getTaskChildren=mock.Mock(side_effect=lambda task_id, request: getTaskChildren_data.get(task_id)),
getTaskResult=mock.Mock(side_effect=lambda task_id: getTaskResult_data.get(task_id))
)
result = self.koji.get_image_build_paths(12387273)
self.assertItemsEqual(result.keys(), ['i386', 'x86_64'])
self.maxDiff = None
self.assertItemsEqual(result['i386'],
['/koji/task/12387276/tdl-i386.xml',
'/koji/task/12387276/fedora-cloud-base-2878aa0.ks',
'/koji/task/12387276/koji-f23-build-12387276-base.ks',
'/koji/task/12387276/libvirt-qcow2-i386.xml',
'/koji/task/12387276/Fedora-Cloud-Base-23-20160103.i386.qcow2',
'/koji/task/12387276/libvirt-raw-xz-i386.xml',
'/koji/task/12387276/Fedora-Cloud-Base-23-20160103.i386.raw.xz'])
self.assertItemsEqual(result['x86_64'],
['/koji/task/12387277/tdl-x86_64.xml',
'/koji/task/12387277/fedora-cloud-base-2878aa0.ks',
'/koji/task/12387277/koji-f23-build-12387277-base.ks',
'/koji/task/12387277/libvirt-qcow2-x86_64.xml',
'/koji/task/12387277/Fedora-Cloud-Base-23-20160103.x86_64.qcow2',
'/koji/task/12387277/libvirt-raw-xz-x86_64.xml',
'/koji/task/12387277/Fedora-Cloud-Base-23-20160103.x86_64.raw.xz'])
if __name__ == "__main__":
unittest.main()

34
tests/test_util.py
View File

@ -46,5 +46,39 @@ class TestGitRefResolver(unittest.TestCase):
run.assert_called_once_with(['git', 'ls-remote', 'https://git.example.com/repo.git', 'HEAD']) run.assert_called_once_with(['git', 'ls-remote', 'https://git.example.com/repo.git', 'HEAD'])
class TestGetVariantData(unittest.TestCase):
def test_get_simple(self):
conf = {
'foo': {
'^Client$': 1
}
}
result = util.get_variant_data(conf, 'foo', mock.Mock(uid='Client'))
self.assertEqual(result, [1])
def test_get_make_list(self):
conf = {
'foo': {
'^Client$': [1, 2],
'^.*$': 3,
}
}
result = util.get_variant_data(conf, 'foo', mock.Mock(uid='Client'))
self.assertItemsEqual(result, [1, 2, 3])
def test_not_matching_arch(self):
conf = {
'foo': {
'^Client$': [1, 2],
}
}
result = util.get_variant_data(conf, 'foo', mock.Mock(uid='Server'))
self.assertItemsEqual(result, [])
def test_handle_missing_config(self):
result = util.get_variant_data({}, 'foo', mock.Mock(uid='Client'))
self.assertItemsEqual(result, [])
if __name__ == "__main__": if __name__ == "__main__":
unittest.main() unittest.main()