almalinux/pungi
14
1
Fork 0
Code Issues Pull Requests Releases Activity

Add image-build support

Browse Source 
Signed-off-by: Lubos Kocman <lkocman@redhat.com>
This commit is contained in:
Lubos Kocman 2015-09-01 08:03:34 +00:00
parent 4fb20198db
commit 8e90a2a32a
6 changed files with 350 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
bin/pungi-koji
View File

@ -211,6 +211,7 @@ def run_compose(compose):
extrafiles_phase = pungi.phases.ExtraFilesPhase(compose, pkgset_phase)
createiso_phase = pungi.phases.CreateisoPhase(compose)
liveimages_phase = pungi.phases.LiveImagesPhase(compose)
image_build_phase = pungi.phases.ImageBuildPhase(compose)
test_phase = pungi.phases.TestPhase(compose)
# check if all config options are set
@ -279,9 +280,11 @@ def run_compose(compose):
# CREATEISO and LIVEIMAGES phases
createiso_phase.start()
liveimages_phase.start()
image_build_phase.start()
createiso_phase.stop()
liveimages_phase.stop()
image_build_phase.stop()
# merge checksum files
for variant in compose.get_variants(types=["variant", "layered-product"]):

77
doc/configuration.rst
View File

@ -467,3 +467,80 @@ Example
'src': True
}),
]
Image Build Settings
====================
**image_build**
(*list*) -- config for koji image-build; format: [(variant_uid_regex, {arch|*: [{opt: value}])]
.. note::
Config can contain anything what is accepted by
koji image-build --config configfile.ini
Repo is currently the only option which is being automatically transformed
into a string.
Please don't set install_tree as it would get overriden by pungi.
The 'format' attr is [('image_type', 'image_suffix'), ...].
productmd should ideally contain all of image types and suffixes.
Example
-------
::
image_build = [
('^Server$', {
'x86_64': [
{
'format': [('docker', 'tar.gz'), ('qcow2', 'qcow2')]
'name': 'fedora-qcow-and-docker-base',
'target': 'koji-target-name',
'ksversion': 'F23', # value from pykickstart
'version': '23',
'ksurl': 'https://git.fedorahosted.org/git/spin-kickstarts.git?somedirectoryifany#HEAD',
'kickstart': "fedora-docker-base.ks",
'repo': ["http://someextrarepos.org/repo", "ftp://rekcod.oi/repo].
# 'install_tree': 'http://sometpath', # this is set automatically by pungi to os_dir for given variant/$arch
'distro': 'Fedora-20',
'disk_size': 3
},
{
'format': [('qcow2','qcow2')]
'name': 'fedora-qcow-base',
'target': 'koji-target-name',
'ksversion': 'F23', # value from pykickstart
'version': '23',
'ksurl': 'https://git.fedorahosted.org/git/spin-kickstarts.git?somedirectoryifany#HEAD',
'kickstart': "fedora-docker-base.ks",
'distro': 'Fedora-23'
}
]
}),
]
Translate Paths Settings
========================
**translate_paths**
(*list*) -- list of paths to translate; format: [(path,translated_path)]
.. note::
This feature becomes useful when you need to transform compose location
into e.g. a http repo which is can be passed to koji image-build.
Translation needs to be invoked by a function call in pungi.
os.path.normpath() is applied on both path and translated_path
Example config
--------------
::
translate_paths = [
("/mnt/a", "http://b/dir"),
]
Example usage
-------------
::
>>> from pungi.paths import translate_paths
>>> print translate_paths(compose_object_with_mapping, "/mnt/a/c/somefile")
http://b/dir/c/somefile

67
pungi/paths.py
View File

@ -307,6 +307,34 @@ class WorkPaths(object):
path = os.path.join(path, file_name)
return path
def image_build_dir(self, arch, variant, create_dir=True):
"""
@param arch
@param variant
@param create_dir=True
Examples:
work/x86_64/Server/image-build
"""
path = os.path.join(self.topdir(arch, create_dir=create_dir), variant.uid, "image-build")
if create_dir:
makedirs(path)
return path
def image_build_conf(self, arch, variant, image_name, image_type, create_dir=True):
"""
@param arch
@param variant
@param image-name
@param image-type (e.g docker)
@param create_dir=True
Examples:
work/x86_64/Server/image-build/docker_rhel-server-docker.cfg
"""
path = os.path.join(self.image_build_dir(arch, variant), "%s_%s.cfg" % (image_type, image_name))
return path
class ComposePaths(object):
def __init__(self, compose):
@ -511,6 +539,45 @@ class ComposePaths(object):
result = os.path.join(path, file_name)
return result
def image_dir(self, arch, variant, symlink_to=None, create_dir=True, relative=False):
"""
Examples:
compose/Server/x86_64/images
None
@param arch
@param variant
@param symlink_to=None
@param create_dir=True
@param relative=False
"""
# skip optional, addons and src architecture
if variant.type != "variant":
return None
if arch == "src":
return None
path = os.path.join(self.topdir(arch, variant, create_dir=create_dir, relative=relative), "images")
if symlink_to:
topdir = self.compose.topdir.rstrip("/") + "/"
relative_dir = path[len(topdir):]
target_dir = os.path.join(symlink_to, self.compose.compose_id, relative_dir)
if create_dir and not relative:
makedirs(target_dir)
try:
os.symlink(target_dir, path)
except OSError as ex:
if ex.errno != errno.EEXIST:
raise
msg = "Symlink pointing to '%s' expected: %s" % (target_dir, path)
if not os.path.islink(path):
raise RuntimeError(msg)
if os.path.abspath(os.readlink(path)) != target_dir:
raise RuntimeError(msg)
else:
if create_dir and not relative:
makedirs(path)
return path
def jigdo_dir(self, arch, variant, create_dir=True, relative=False):
"""
Examples:

1
pungi/phases/__init__.py
View File

@ -25,4 +25,5 @@ from buildinstall import BuildinstallPhase # noqa
from extra_files import ExtraFilesPhase # noqa
from createiso import CreateisoPhase # noqa
from live_images import LiveImagesPhase # noqa
from image_build import ImageBuildPhase # noqa
from test import TestPhase # noqa

166
pungi/phases/image_build.py Normal file
View File

@ -0,0 +1,166 @@
# -*- coding: utf-8 -*-
import os
import time
import pipes
from pungi.util import get_arch_variant_data
from pungi.phases.base import PhaseBase
from pungi.linker import Linker
from pungi.paths import translate_path
from pungi.wrappers.kojiwrapper import KojiWrapper
from pungi.wrappers.iso import IsoWrapper
from kobo.shortcuts import run, read_checksum_file
from kobo.threads import ThreadPool, WorkerThread
from productmd.images import Image
class ImageBuildPhase(PhaseBase):
"""class for wrapping up koji image-build"""
name = "image_build"
def __init__(self, compose):
PhaseBase.__init__(self, compose)
self.pool = ThreadPool(logger=self.compose._logger)
def skip(self):
if PhaseBase.skip(self):
return True
if not self.compose.conf.get(self.name):
self.log_info("Section '%s' was not found. Skipping" % self.name)
return True
return False
def run(self):
for arch in self.compose.get_arches(): # src will be skipped
for variant in self.compose.get_variants(arch=arch):
image_build_data = get_arch_variant_data(self.compose.conf, self.name, arch, variant)
for image_conf in image_build_data:
image_conf["arches"] = arch # passed to get_image_build_cmd as dict
image_conf["variant"] = variant # ^
image_conf["install_tree"] = translate_path(self.compose, self.compose.paths.compose.os_tree(arch, variant)) # ^
format = image_conf["format"] # transform format into right 'format' for image-build
image_conf["format"] = ",".join([x[0] for x in image_conf["format"]]) # 'docker,qcow2'
if image_conf.has_key("repos") and not isinstance(image_conf["repos"], str):
image_conf["repos"] = ",".join(image_conf["repos"]) # supply repos as str separated by , instead of list
cmd = {
"format": format,
"image_conf": image_conf,
"conf_file": self.compose.paths.work.image_build_conf(image_conf["arches"], image_conf['variant'], image_name=image_conf['name'], image_type=image_conf['format'].replace(",", "-")),
"image_dir": self.compose.paths.compose.image_dir(arch, variant),
"relative_image_dir": self.compose.paths.compose.image_dir(arch, variant, create_dir=False, relative=True),
"link_type": self.compose.conf.get("link_type", "hardlink-or-copy")
}
self.pool.add(CreateImageBuildThread(self.pool))
self.pool.queue_put((self.compose, cmd))
self.pool.start()
def stop(self, *args, **kwargs):
PhaseBase.stop(self, *args, **kwargs)
if self.skip():
return
class CreateImageBuildThread(WorkerThread):
def fail(self, compose, cmd):
compose.log_error("CreateImageBuild failed.")
def process(self, item, num):
compose, cmd = item
mounts = [compose.topdir]
if "mount" in cmd:
mounts.append(cmd["mount"])
runroot = compose.conf.get("runroot", False)
log_file = compose.paths.log.log_file(cmd["image_conf"]["arches"], "imagebuild-%s-%s-%s" % (cmd["image_conf"]["arches"], cmd["image_conf"]["variant"], cmd['image_conf']['format'].replace(",","-")))
msg = "Creating %s image (arch: %s, variant: %s)" % (cmd["image_conf"]["format"].replace(",","-"), cmd["image_conf"]["arches"], cmd["image_conf"]["variant"])
self.pool.log_info("[BEGIN] %s" % msg)
koji_wrapper = KojiWrapper(compose.conf["koji_profile"])
# paths module doesn't hold compose object, so we have to generate path here
# writes conf file for koji image-build
self.pool.log_info("Writing image-build config for %s.%s into %s" % (cmd["image_conf"]["variant"], cmd["image_conf"]["arches"], cmd["conf_file"]))
koji_cmd = koji_wrapper.get_image_build_cmd(cmd['image_conf'], conf_file_dest=cmd["conf_file"], wait=True, scratch=False)
# avoid race conditions?
# Kerberos authentication failed: Permission denied in replay cache code (-1765328215)
time.sleep(num * 3)
output = koji_wrapper.run_create_image_cmd(koji_cmd, log_file=log_file)
self.pool.log_debug("build-image outputs: %s" % (output))
if output["retcode"] != 0:
self.fail(compose, cmd)
raise RuntimeError("ImageBuild task failed: %s. See %s for more details." % (output["task_id"], log_file))
# copy image to images/
image_infos = []
for filename in koji_wrapper.get_image_path(output["task_id"]):
# format is list of tuples [('qcow2', '.qcow2'), ('raw-xz', 'raw.xz'),]
for format, suffix in cmd['format']:
if filename.endswith(suffix):
image_infos.append({'filename': filename, 'suffix': suffix, 'type': format}) # the type/format ... image-build has it wrong
if len(image_infos) != len(cmd['format']):
self.pool.log_error("Error in koji task %s. Expected to find same amount of images as in suffixes attr in image-build (%s). Got '%s'." %
(output["task_id"], len(cmd['image_conf']['format']), len(image_infos)))
self.fail(compose, cmd)
# The usecase here is that you can run koji image-build with multiple --format
# It's ok to do it serialized since we're talking about max 2 images per single
# image_build record
linker = Linker(logger=compose._logger)
for image_info in image_infos:
# let's not change filename of koji outputs
image_dest = os.path.join(cmd["image_dir"], os.path.basename(image_info['filename']))
linker.link(image_info['filename'], image_dest, link_type=cmd["link_type"])
iso = IsoWrapper(logger=compose._logger) # required for checksums only
checksum_cmd = ["cd %s" % pipes.quote(os.path.dirname(image_dest))]
checksum_cmd.extend(iso.get_checksum_cmds(os.path.basename(image_dest)))
checksum_cmd = " && ".join(checksum_cmd)
if runroot:
packages = ["coreutils", "genisoimage", "isomd5sum", "jigdo", "strace", "lsof"]
runroot_channel = compose.conf.get("runroot_channel", None)
runroot_tag = compose.conf["runroot_tag"]
koji_cmd = koji_wrapper.get_runroot_cmd(runroot_tag, cmd["image_conf"]["arches"], checksum_cmd, channel=runroot_channel, use_shell=True, task_id=True, packages=packages, mounts=mounts)
# avoid race conditions?
# Kerberos authentication failed: Permission denied in replay cache code (-1765328215)
time.sleep(num * 3)
output = koji_wrapper.run_runroot_cmd(koji_cmd, log_file=log_file)
if output["retcode"] != 0:
self.fail(compose, cmd)
raise RuntimeError("Runroot task failed: %s. See %s for more details." % (output["task_id"], log_file))
else:
# run locally
try:
run(checksum_cmd, show_cmd=True, logfile=log_file)
except:
self.fail(compose, cmd)
raise
# Update image manifest
img = Image(compose.im)
img.type = image_info['type']
img.format = image_info['suffix']
img.path = os.path.join(cmd["relative_image_dir"], os.path.basename(image_dest))
img.mtime = int(os.stat(image_dest).st_mtime)
img.size = os.path.getsize(image_dest)
img.arch = cmd["image_conf"]["arches"] # arches should be always single arch
img.disc_number = 1 # We don't expect multiple disks
img.disc_count = 1
for checksum_type in ("md5", "sha1", "sha256"):
checksum_path = image_dest + ".%sSUM" % checksum_type.upper()
checksum_value = None
if os.path.isfile(checksum_path):
checksum_value, image_name = read_checksum_file(checksum_path)[0]
if image_name != os.path.basename(img.path):
raise ValueError("Image name doesn't match checksum: %s" % checksum_path)
img.add_checksum(compose.paths.compose.topdir(), checksum_type=checksum_type, checksum_value=checksum_value)
img.bootable = False
# named keywords due portability (old productmd used arch, variant ... while new one uses variant, arch
compose.im.add(variant=cmd["image_conf"]["variant"].uid, arch=cmd["image_conf"]["arches"], image=img)
self.pool.log_info("[DONE ] %s" % msg)

41
pungi/wrappers/kojiwrapper.py
View File

@ -22,6 +22,7 @@ import re
import koji
import rpmUtils.arch
from kobo.shortcuts import run
from ConfigParser import ConfigParser
class KojiWrapper(object):
@ -97,6 +98,35 @@ class KojiWrapper(object):
}
return result
def get_image_build_cmd(self, config_options, conf_file_dest, wait=True, scratch=False):
"""
@param config_options
@param conf_file_dest - a destination in compose workdir for the conf file to be written
@param wait=True
@param scratch=False
"""
# Usage: koji image-build [options] <name> <version> <target> <install-tree-url> <arch> [<arch>...]
sub_command = "image-build"
# The minimum set of options
min_options = ("name", "version", "target", "install_tree", "arches", "format", "kickstart", "ksurl", "distro")
assert set(min_options).issubset(set(config_options.keys())), "image-build requires at least %s got '%s'" % (", ".join(min_options), config_options)
cfg_parser = ConfigParser()
cfg_parser.add_section(sub_command)
for option, value in config_options.iteritems():
cfg_parser.set(sub_command, option, value)
fd = open(conf_file_dest, "w")
cfg_parser.write(fd)
fd.close()
cmd = [self.executable, sub_command, "--config=%s" % conf_file_dest]
if wait:
cmd.append("--wait")
if scratch:
cmd.append("--scratch")
return cmd
def get_create_image_cmd(self, name, version, target, arch, ks_file, repos, image_type="live", image_format=None, release=None, wait=True, archive=False, specfile=None):
# Usage: koji spin-livecd [options] <name> <version> <target> <arch> <kickstart-file>
# Usage: koji spin-appliance [options] <name> <version> <target> <arch> <kickstart-file>
@ -163,12 +193,14 @@ class KojiWrapper(object):
def run_create_image_cmd(self, command, log_file=None):
# spin-{livecd,appliance} is blocking by default -> you probably want to run it in a thread
retcode, output = run(command, can_fail=True, logfile=log_file)
try:
retcode, output = run(command, can_fail=True, logfile=log_file)
except RuntimeError, e:
raise RuntimeError("%s. %s failed with '%s'" % (e, command, output))
match = re.search(r"Created task: (\d+)", output)
if not match:
raise RuntimeError("Could not find task ID in output")
raise RuntimeError("Could not find task ID in output. Command '%s' returned '%s'." % (" ".join(command), output))
result = {
"retcode": retcode,
@ -179,7 +211,6 @@ class KojiWrapper(object):
def get_image_path(self, task_id):
result = []
# XXX: hardcoded URL
koji_proxy = self.koji_module.ClientSession(self.koji_module.config.server)
task_info_list = []
task_info_list.append(koji_proxy.getTaskInfo(task_id, request=True))
@ -188,7 +219,7 @@ class KojiWrapper(object):
# scan parent and child tasks for certain methods
task_info = None
for i in task_info_list:
if i["method"] in ("createAppliance", "createLiveCD"):
if i["method"] in ("createAppliance", "createLiveCD", 'createImage'):
task_info = i
break