Add image-build support
Signed-off-by: Lubos Kocman <lkocman@redhat.com>
This commit is contained in:
Lubos Kocman
parent
4fb20198db
commit
8e90a2a32a
@ -211,6 +211,7 @@ def run_compose(compose):
|
||||
extrafiles_phase = pungi.phases.ExtraFilesPhase(compose, pkgset_phase)
|
||||
createiso_phase = pungi.phases.CreateisoPhase(compose)
|
||||
liveimages_phase = pungi.phases.LiveImagesPhase(compose)
|
||||
image_build_phase = pungi.phases.ImageBuildPhase(compose)
|
||||
test_phase = pungi.phases.TestPhase(compose)
|
||||
|
||||
# check if all config options are set
|
||||
@ -279,9 +280,11 @@ def run_compose(compose):
|
||||
# CREATEISO and LIVEIMAGES phases
|
||||
createiso_phase.start()
|
||||
liveimages_phase.start()
|
||||
image_build_phase.start()
|
||||
|
||||
createiso_phase.stop()
|
||||
liveimages_phase.stop()
|
||||
image_build_phase.stop()
|
||||
|
||||
# merge checksum files
|
||||
for variant in compose.get_variants(types=["variant", "layered-product"]):
|
||||
|
||||
@ -467,3 +467,80 @@ Example
|
||||
'src': True
|
||||
}),
|
||||
]
|
||||
|
||||
Image Build Settings
|
||||
====================
|
||||
|
||||
**image_build**
|
||||
(*list*) -- config for koji image-build; format: [(variant_uid_regex, {arch|*: [{opt: value}])]
|
||||
|
||||
.. note::
|
||||
Config can contain anything what is accepted by
|
||||
koji image-build --config configfile.ini
|
||||
Repo is currently the only option which is being automatically transformed
|
||||
into a string.
|
||||
|
||||
Please don't set install_tree as it would get overriden by pungi.
|
||||
The 'format' attr is [('image_type', 'image_suffix'), ...].
|
||||
productmd should ideally contain all of image types and suffixes.
|
||||
|
||||
Example
|
||||
-------
|
||||
::
|
||||
|
||||
image_build = [
|
||||
('^Server$', {
|
||||
'x86_64': [
|
||||
{
|
||||
'format': [('docker', 'tar.gz'), ('qcow2', 'qcow2')]
|
||||
'name': 'fedora-qcow-and-docker-base',
|
||||
'target': 'koji-target-name',
|
||||
'ksversion': 'F23', # value from pykickstart
|
||||
'version': '23',
|
||||
'ksurl': 'https://git.fedorahosted.org/git/spin-kickstarts.git?somedirectoryifany#HEAD',
|
||||
'kickstart': "fedora-docker-base.ks",
|
||||
'repo': ["http://someextrarepos.org/repo", "ftp://rekcod.oi/repo].
|
||||
# 'install_tree': 'http://sometpath', # this is set automatically by pungi to os_dir for given variant/$arch
|
||||
'distro': 'Fedora-20',
|
||||
'disk_size': 3
|
||||
},
|
||||
{
|
||||
'format': [('qcow2','qcow2')]
|
||||
'name': 'fedora-qcow-base',
|
||||
'target': 'koji-target-name',
|
||||
'ksversion': 'F23', # value from pykickstart
|
||||
'version': '23',
|
||||
'ksurl': 'https://git.fedorahosted.org/git/spin-kickstarts.git?somedirectoryifany#HEAD',
|
||||
'kickstart': "fedora-docker-base.ks",
|
||||
'distro': 'Fedora-23'
|
||||
}
|
||||
]
|
||||
}),
|
||||
]
|
||||
|
||||
Translate Paths Settings
|
||||
========================
|
||||
|
||||
**translate_paths**
|
||||
(*list*) -- list of paths to translate; format: [(path,translated_path)]
|
||||
|
||||
.. note::
|
||||
This feature becomes useful when you need to transform compose location
|
||||
into e.g. a http repo which is can be passed to koji image-build.
|
||||
Translation needs to be invoked by a function call in pungi.
|
||||
os.path.normpath() is applied on both path and translated_path
|
||||
|
||||
|
||||
Example config
|
||||
--------------
|
||||
::
|
||||
translate_paths = [
|
||||
("/mnt/a", "http://b/dir"),
|
||||
]
|
||||
|
||||
Example usage
|
||||
-------------
|
||||
::
|
||||
>>> from pungi.paths import translate_paths
|
||||
>>> print translate_paths(compose_object_with_mapping, "/mnt/a/c/somefile")
|
||||
http://b/dir/c/somefile
|
||||
|
||||
@ -307,6 +307,34 @@ class WorkPaths(object):
|
||||
path = os.path.join(path, file_name)
|
||||
return path
|
||||
|
||||
def image_build_dir(self, arch, variant, create_dir=True):
|
||||
"""
|
||||
@param arch
|
||||
@param variant
|
||||
@param create_dir=True
|
||||
|
||||
Examples:
|
||||
work/x86_64/Server/image-build
|
||||
"""
|
||||
path = os.path.join(self.topdir(arch, create_dir=create_dir), variant.uid, "image-build")
|
||||
if create_dir:
|
||||
makedirs(path)
|
||||
return path
|
||||
|
||||
def image_build_conf(self, arch, variant, image_name, image_type, create_dir=True):
|
||||
"""
|
||||
@param arch
|
||||
@param variant
|
||||
@param image-name
|
||||
@param image-type (e.g docker)
|
||||
@param create_dir=True
|
||||
|
||||
Examples:
|
||||
work/x86_64/Server/image-build/docker_rhel-server-docker.cfg
|
||||
"""
|
||||
path = os.path.join(self.image_build_dir(arch, variant), "%s_%s.cfg" % (image_type, image_name))
|
||||
return path
|
||||
|
||||
|
||||
class ComposePaths(object):
|
||||
def __init__(self, compose):
|
||||
@ -511,6 +539,45 @@ class ComposePaths(object):
|
||||
result = os.path.join(path, file_name)
|
||||
return result
|
||||
|
||||
def image_dir(self, arch, variant, symlink_to=None, create_dir=True, relative=False):
|
||||
"""
|
||||
Examples:
|
||||
compose/Server/x86_64/images
|
||||
None
|
||||
@param arch
|
||||
@param variant
|
||||
@param symlink_to=None
|
||||
@param create_dir=True
|
||||
@param relative=False
|
||||
"""
|
||||
# skip optional, addons and src architecture
|
||||
if variant.type != "variant":
|
||||
return None
|
||||
if arch == "src":
|
||||
return None
|
||||
|
||||
path = os.path.join(self.topdir(arch, variant, create_dir=create_dir, relative=relative), "images")
|
||||
if symlink_to:
|
||||
topdir = self.compose.topdir.rstrip("/") + "/"
|
||||
relative_dir = path[len(topdir):]
|
||||
target_dir = os.path.join(symlink_to, self.compose.compose_id, relative_dir)
|
||||
if create_dir and not relative:
|
||||
makedirs(target_dir)
|
||||
try:
|
||||
os.symlink(target_dir, path)
|
||||
except OSError as ex:
|
||||
if ex.errno != errno.EEXIST:
|
||||
raise
|
||||
msg = "Symlink pointing to '%s' expected: %s" % (target_dir, path)
|
||||
if not os.path.islink(path):
|
||||
raise RuntimeError(msg)
|
||||
if os.path.abspath(os.readlink(path)) != target_dir:
|
||||
raise RuntimeError(msg)
|
||||
else:
|
||||
if create_dir and not relative:
|
||||
makedirs(path)
|
||||
return path
|
||||
|
||||
def jigdo_dir(self, arch, variant, create_dir=True, relative=False):
|
||||
"""
|
||||
Examples:
|
||||
|
||||
@ -25,4 +25,5 @@ from buildinstall import BuildinstallPhase # noqa
|
||||
from extra_files import ExtraFilesPhase # noqa
|
||||
from createiso import CreateisoPhase # noqa
|
||||
from live_images import LiveImagesPhase # noqa
|
||||
from image_build import ImageBuildPhase # noqa
|
||||
from test import TestPhase # noqa
|
||||
|
||||
166
pungi/phases/image_build.py
Normal file
166
pungi/phases/image_build.py
Normal file
@ -0,0 +1,166 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
|
||||
import os
|
||||
import time
|
||||
import pipes
|
||||
|
||||
from pungi.util import get_arch_variant_data
|
||||
from pungi.phases.base import PhaseBase
|
||||
from pungi.linker import Linker
|
||||
from pungi.paths import translate_path
|
||||
from pungi.wrappers.kojiwrapper import KojiWrapper
|
||||
from pungi.wrappers.iso import IsoWrapper
|
||||
from kobo.shortcuts import run, read_checksum_file
|
||||
from kobo.threads import ThreadPool, WorkerThread
|
||||
from productmd.images import Image
|
||||
|
||||
class ImageBuildPhase(PhaseBase):
|
||||
"""class for wrapping up koji image-build"""
|
||||
name = "image_build"
|
||||
|
||||
def __init__(self, compose):
|
||||
PhaseBase.__init__(self, compose)
|
||||
self.pool = ThreadPool(logger=self.compose._logger)
|
||||
|
||||
def skip(self):
|
||||
if PhaseBase.skip(self):
|
||||
return True
|
||||
if not self.compose.conf.get(self.name):
|
||||
self.log_info("Section '%s' was not found. Skipping" % self.name)
|
||||
return True
|
||||
return False
|
||||
|
||||
def run(self):
|
||||
for arch in self.compose.get_arches(): # src will be skipped
|
||||
for variant in self.compose.get_variants(arch=arch):
|
||||
image_build_data = get_arch_variant_data(self.compose.conf, self.name, arch, variant)
|
||||
for image_conf in image_build_data:
|
||||
image_conf["arches"] = arch # passed to get_image_build_cmd as dict
|
||||
image_conf["variant"] = variant # ^
|
||||
image_conf["install_tree"] = translate_path(self.compose, self.compose.paths.compose.os_tree(arch, variant)) # ^
|
||||
format = image_conf["format"] # transform format into right 'format' for image-build
|
||||
image_conf["format"] = ",".join([x[0] for x in image_conf["format"]]) # 'docker,qcow2'
|
||||
if image_conf.has_key("repos") and not isinstance(image_conf["repos"], str):
|
||||
image_conf["repos"] = ",".join(image_conf["repos"]) # supply repos as str separated by , instead of list
|
||||
cmd = {
|
||||
"format": format,
|
||||
"image_conf": image_conf,
|
||||
"conf_file": self.compose.paths.work.image_build_conf(image_conf["arches"], image_conf['variant'], image_name=image_conf['name'], image_type=image_conf['format'].replace(",", "-")),
|
||||
"image_dir": self.compose.paths.compose.image_dir(arch, variant),
|
||||
"relative_image_dir": self.compose.paths.compose.image_dir(arch, variant, create_dir=False, relative=True),
|
||||
"link_type": self.compose.conf.get("link_type", "hardlink-or-copy")
|
||||
}
|
||||
self.pool.add(CreateImageBuildThread(self.pool))
|
||||
self.pool.queue_put((self.compose, cmd))
|
||||
self.pool.start()
|
||||
|
||||
def stop(self, *args, **kwargs):
|
||||
PhaseBase.stop(self, *args, **kwargs)
|
||||
if self.skip():
|
||||
return
|
||||
|
||||
class CreateImageBuildThread(WorkerThread):
|
||||
def fail(self, compose, cmd):
|
||||
compose.log_error("CreateImageBuild failed.")
|
||||
|
||||
def process(self, item, num):
|
||||
compose, cmd = item
|
||||
mounts = [compose.topdir]
|
||||
if "mount" in cmd:
|
||||
mounts.append(cmd["mount"])
|
||||
runroot = compose.conf.get("runroot", False)
|
||||
log_file = compose.paths.log.log_file(cmd["image_conf"]["arches"], "imagebuild-%s-%s-%s" % (cmd["image_conf"]["arches"], cmd["image_conf"]["variant"], cmd['image_conf']['format'].replace(",","-")))
|
||||
msg = "Creating %s image (arch: %s, variant: %s)" % (cmd["image_conf"]["format"].replace(",","-"), cmd["image_conf"]["arches"], cmd["image_conf"]["variant"])
|
||||
self.pool.log_info("[BEGIN] %s" % msg)
|
||||
|
||||
koji_wrapper = KojiWrapper(compose.conf["koji_profile"])
|
||||
# paths module doesn't hold compose object, so we have to generate path here
|
||||
|
||||
# writes conf file for koji image-build
|
||||
self.pool.log_info("Writing image-build config for %s.%s into %s" % (cmd["image_conf"]["variant"], cmd["image_conf"]["arches"], cmd["conf_file"]))
|
||||
koji_cmd = koji_wrapper.get_image_build_cmd(cmd['image_conf'], conf_file_dest=cmd["conf_file"], wait=True, scratch=False)
|
||||
|
||||
# avoid race conditions?
|
||||
# Kerberos authentication failed: Permission denied in replay cache code (-1765328215)
|
||||
time.sleep(num * 3)
|
||||
output = koji_wrapper.run_create_image_cmd(koji_cmd, log_file=log_file)
|
||||
self.pool.log_debug("build-image outputs: %s" % (output))
|
||||
if output["retcode"] != 0:
|
||||
self.fail(compose, cmd)
|
||||
raise RuntimeError("ImageBuild task failed: %s. See %s for more details." % (output["task_id"], log_file))
|
||||
|
||||
# copy image to images/
|
||||
image_infos = []
|
||||
|
||||
for filename in koji_wrapper.get_image_path(output["task_id"]):
|
||||
# format is list of tuples [('qcow2', '.qcow2'), ('raw-xz', 'raw.xz'),]
|
||||
for format, suffix in cmd['format']:
|
||||
if filename.endswith(suffix):
|
||||
image_infos.append({'filename': filename, 'suffix': suffix, 'type': format}) # the type/format ... image-build has it wrong
|
||||
|
||||
if len(image_infos) != len(cmd['format']):
|
||||
self.pool.log_error("Error in koji task %s. Expected to find same amount of images as in suffixes attr in image-build (%s). Got '%s'." %
|
||||
(output["task_id"], len(cmd['image_conf']['format']), len(image_infos)))
|
||||
self.fail(compose, cmd)
|
||||
|
||||
# The usecase here is that you can run koji image-build with multiple --format
|
||||
# It's ok to do it serialized since we're talking about max 2 images per single
|
||||
# image_build record
|
||||
linker = Linker(logger=compose._logger)
|
||||
for image_info in image_infos:
|
||||
# let's not change filename of koji outputs
|
||||
image_dest = os.path.join(cmd["image_dir"], os.path.basename(image_info['filename']))
|
||||
linker.link(image_info['filename'], image_dest, link_type=cmd["link_type"])
|
||||
|
||||
iso = IsoWrapper(logger=compose._logger) # required for checksums only
|
||||
checksum_cmd = ["cd %s" % pipes.quote(os.path.dirname(image_dest))]
|
||||
checksum_cmd.extend(iso.get_checksum_cmds(os.path.basename(image_dest)))
|
||||
checksum_cmd = " && ".join(checksum_cmd)
|
||||
|
||||
if runroot:
|
||||
packages = ["coreutils", "genisoimage", "isomd5sum", "jigdo", "strace", "lsof"]
|
||||
runroot_channel = compose.conf.get("runroot_channel", None)
|
||||
runroot_tag = compose.conf["runroot_tag"]
|
||||
koji_cmd = koji_wrapper.get_runroot_cmd(runroot_tag, cmd["image_conf"]["arches"], checksum_cmd, channel=runroot_channel, use_shell=True, task_id=True, packages=packages, mounts=mounts)
|
||||
|
||||
# avoid race conditions?
|
||||
# Kerberos authentication failed: Permission denied in replay cache code (-1765328215)
|
||||
time.sleep(num * 3)
|
||||
|
||||
output = koji_wrapper.run_runroot_cmd(koji_cmd, log_file=log_file)
|
||||
if output["retcode"] != 0:
|
||||
self.fail(compose, cmd)
|
||||
raise RuntimeError("Runroot task failed: %s. See %s for more details." % (output["task_id"], log_file))
|
||||
|
||||
else:
|
||||
# run locally
|
||||
try:
|
||||
run(checksum_cmd, show_cmd=True, logfile=log_file)
|
||||
except:
|
||||
self.fail(compose, cmd)
|
||||
raise
|
||||
|
||||
# Update image manifest
|
||||
img = Image(compose.im)
|
||||
img.type = image_info['type']
|
||||
img.format = image_info['suffix']
|
||||
img.path = os.path.join(cmd["relative_image_dir"], os.path.basename(image_dest))
|
||||
img.mtime = int(os.stat(image_dest).st_mtime)
|
||||
img.size = os.path.getsize(image_dest)
|
||||
img.arch = cmd["image_conf"]["arches"] # arches should be always single arch
|
||||
img.disc_number = 1 # We don't expect multiple disks
|
||||
img.disc_count = 1
|
||||
for checksum_type in ("md5", "sha1", "sha256"):
|
||||
checksum_path = image_dest + ".%sSUM" % checksum_type.upper()
|
||||
checksum_value = None
|
||||
if os.path.isfile(checksum_path):
|
||||
checksum_value, image_name = read_checksum_file(checksum_path)[0]
|
||||
if image_name != os.path.basename(img.path):
|
||||
raise ValueError("Image name doesn't match checksum: %s" % checksum_path)
|
||||
img.add_checksum(compose.paths.compose.topdir(), checksum_type=checksum_type, checksum_value=checksum_value)
|
||||
img.bootable = False
|
||||
# named keywords due portability (old productmd used arch, variant ... while new one uses variant, arch
|
||||
compose.im.add(variant=cmd["image_conf"]["variant"].uid, arch=cmd["image_conf"]["arches"], image=img)
|
||||
|
||||
self.pool.log_info("[DONE ] %s" % msg)
|
||||
@ -22,6 +22,7 @@ import re
|
||||
import koji
|
||||
import rpmUtils.arch
|
||||
from kobo.shortcuts import run
|
||||
from ConfigParser import ConfigParser
|
||||
|
||||
|
||||
class KojiWrapper(object):
|
||||
@ -97,6 +98,35 @@ class KojiWrapper(object):
|
||||
}
|
||||
return result
|
||||
|
||||
def get_image_build_cmd(self, config_options, conf_file_dest, wait=True, scratch=False):
|
||||
"""
|
||||
@param config_options
|
||||
@param conf_file_dest - a destination in compose workdir for the conf file to be written
|
||||
@param wait=True
|
||||
@param scratch=False
|
||||
"""
|
||||
# Usage: koji image-build [options] <name> <version> <target> <install-tree-url> <arch> [<arch>...]
|
||||
sub_command = "image-build"
|
||||
# The minimum set of options
|
||||
min_options = ("name", "version", "target", "install_tree", "arches", "format", "kickstart", "ksurl", "distro")
|
||||
assert set(min_options).issubset(set(config_options.keys())), "image-build requires at least %s got '%s'" % (", ".join(min_options), config_options)
|
||||
cfg_parser = ConfigParser()
|
||||
cfg_parser.add_section(sub_command)
|
||||
for option, value in config_options.iteritems():
|
||||
cfg_parser.set(sub_command, option, value)
|
||||
|
||||
fd = open(conf_file_dest, "w")
|
||||
cfg_parser.write(fd)
|
||||
fd.close()
|
||||
|
||||
cmd = [self.executable, sub_command, "--config=%s" % conf_file_dest]
|
||||
if wait:
|
||||
cmd.append("--wait")
|
||||
if scratch:
|
||||
cmd.append("--scratch")
|
||||
|
||||
return cmd
|
||||
|
||||
def get_create_image_cmd(self, name, version, target, arch, ks_file, repos, image_type="live", image_format=None, release=None, wait=True, archive=False, specfile=None):
|
||||
# Usage: koji spin-livecd [options] <name> <version> <target> <arch> <kickstart-file>
|
||||
# Usage: koji spin-appliance [options] <name> <version> <target> <arch> <kickstart-file>
|
||||
@ -163,12 +193,14 @@ class KojiWrapper(object):
|
||||
|
||||
def run_create_image_cmd(self, command, log_file=None):
|
||||
# spin-{livecd,appliance} is blocking by default -> you probably want to run it in a thread
|
||||
|
||||
retcode, output = run(command, can_fail=True, logfile=log_file)
|
||||
try:
|
||||
retcode, output = run(command, can_fail=True, logfile=log_file)
|
||||
except RuntimeError, e:
|
||||
raise RuntimeError("%s. %s failed with '%s'" % (e, command, output))
|
||||
|
||||
match = re.search(r"Created task: (\d+)", output)
|
||||
if not match:
|
||||
raise RuntimeError("Could not find task ID in output")
|
||||
raise RuntimeError("Could not find task ID in output. Command '%s' returned '%s'." % (" ".join(command), output))
|
||||
|
||||
result = {
|
||||
"retcode": retcode,
|
||||
@ -179,7 +211,6 @@ class KojiWrapper(object):
|
||||
|
||||
def get_image_path(self, task_id):
|
||||
result = []
|
||||
# XXX: hardcoded URL
|
||||
koji_proxy = self.koji_module.ClientSession(self.koji_module.config.server)
|
||||
task_info_list = []
|
||||
task_info_list.append(koji_proxy.getTaskInfo(task_id, request=True))
|
||||
@ -188,7 +219,7 @@ class KojiWrapper(object):
|
||||
# scan parent and child tasks for certain methods
|
||||
task_info = None
|
||||
for i in task_info_list:
|
||||
if i["method"] in ("createAppliance", "createLiveCD"):
|
||||
if i["method"] in ("createAppliance", "createLiveCD", 'createImage'):
|
||||
task_info = i
|
||||
break
|
||||
|
||||
|
||||
Loading…
Reference in New Issue
Block a user