2f5d6d7dcd
Config option 'repo' and 'repo_from' are used in several phases, merge
them with one option 'repo'. 'append' in schema is used for appending
the values from deprecated options to 'repo', so it won't break on any
existing config files that have the old options of 'repo_from' and
'source_repo_from' (which is an alias of 'repo_from').
And 'repo' schema is updated to support repo dict as the value or an
item in the values, a repo dict is just a dict contains repo options,
'baseurl' is required in the dict, like:
{"baseurl": "http://example.com/url/to/repo"}
or:
{"baseurl": "Serer"}
currently this is used in ostree phase to support extra repo options
like:
{"baseurl": "Server", "exclude": "systemd-container"}
Signed-off-by: Qixiang Wan <qwan@redhat.com>
333 lines
14 KiB
Python
333 lines
14 KiB
Python
# -*- coding: utf-8 -*-
|
|
|
|
|
|
# This program is free software; you can redistribute it and/or modify
|
|
# it under the terms of the GNU General Public License as published by
|
|
# the Free Software Foundation; version 2 of the License.
|
|
#
|
|
# This program is distributed in the hope that it will be useful,
|
|
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
# GNU Library General Public License for more details.
|
|
#
|
|
# You should have received a copy of the GNU General Public License
|
|
# along with this program; if not, see <https://gnu.org/licenses/>.
|
|
|
|
|
|
import os
|
|
import sys
|
|
import time
|
|
import pipes
|
|
import shutil
|
|
|
|
from kobo.threads import ThreadPool, WorkerThread
|
|
from kobo.shortcuts import run, save_to_file, force_list
|
|
from productmd.images import Image
|
|
|
|
from pungi.wrappers.kojiwrapper import KojiWrapper
|
|
from pungi.wrappers import iso
|
|
from pungi.phases import base
|
|
from pungi.util import get_arch_variant_data, makedirs, get_mtime, get_file_size, failable
|
|
from pungi.util import get_repo_urls
|
|
|
|
|
|
# HACK: define cmp in python3
|
|
if sys.version_info[0] == 3:
|
|
def cmp(a, b):
|
|
return (a > b) - (a < b)
|
|
|
|
|
|
class LiveImagesPhase(base.PhaseLoggerMixin, base.ImageConfigMixin, base.ConfigGuardedPhase):
|
|
name = "live_images"
|
|
|
|
def __init__(self, compose):
|
|
super(LiveImagesPhase, self).__init__(compose)
|
|
self.pool = ThreadPool(logger=self.logger)
|
|
|
|
def _get_repos(self, arch, variant, data):
|
|
repos = []
|
|
if not variant.is_empty:
|
|
repos.append(variant.uid)
|
|
repos.extend(force_list(data.get('repo', [])))
|
|
return get_repo_urls(self.compose, repos, arch=arch)
|
|
|
|
def run(self):
|
|
symlink_isos_to = self.compose.conf.get("symlink_isos_to")
|
|
commands = []
|
|
|
|
for variant in self.compose.all_variants.values():
|
|
for arch in variant.arches + ["src"]:
|
|
for data in get_arch_variant_data(self.compose.conf, self.name, arch, variant):
|
|
subvariant = data.get('subvariant', variant.uid)
|
|
type = data.get('type', 'live')
|
|
|
|
if type == 'live':
|
|
dest_dir = self.compose.paths.compose.iso_dir(arch, variant, symlink_to=symlink_isos_to)
|
|
elif type == 'appliance':
|
|
dest_dir = self.compose.paths.compose.image_dir(variant, symlink_to=symlink_isos_to)
|
|
dest_dir = dest_dir % {'arch': arch}
|
|
makedirs(dest_dir)
|
|
else:
|
|
raise RuntimeError('Unknown live image type %s' % type)
|
|
if not dest_dir:
|
|
continue
|
|
|
|
cmd = {
|
|
"name": data.get('name'),
|
|
"version": self.get_version(data),
|
|
"release": self.get_release(data),
|
|
"dest_dir": dest_dir,
|
|
"build_arch": arch,
|
|
"ks_file": data['kickstart'],
|
|
"ksurl": self.get_ksurl(data),
|
|
# Used for images wrapped in RPM
|
|
"specfile": data.get("specfile", None),
|
|
# Scratch (only taken in consideration if specfile
|
|
# specified) For images wrapped in rpm is scratch
|
|
# disabled by default For other images is scratch
|
|
# always on
|
|
"scratch": data.get("scratch", False),
|
|
"sign": False,
|
|
"type": type,
|
|
"label": "", # currently not used
|
|
"subvariant": subvariant,
|
|
"failable_arches": data.get('failable', []),
|
|
}
|
|
|
|
cmd["repos"] = self._get_repos(arch, variant, data)
|
|
|
|
# Signing of the rpm wrapped image
|
|
if not cmd["scratch"] and data.get("sign"):
|
|
cmd["sign"] = True
|
|
|
|
cmd['filename'] = self._get_file_name(arch, variant, cmd['name'], cmd['version'])
|
|
|
|
commands.append((cmd, variant, arch))
|
|
|
|
for (cmd, variant, arch) in commands:
|
|
self.pool.add(CreateLiveImageThread(self.pool))
|
|
self.pool.queue_put((self.compose, cmd, variant, arch))
|
|
|
|
self.pool.start()
|
|
|
|
def _get_file_name(self, arch, variant, name=None, version=None):
|
|
if self.compose.conf['live_images_no_rename']:
|
|
return None
|
|
|
|
disc_type = self.compose.conf['disc_types'].get('live', 'live')
|
|
|
|
format = "%(compose_id)s-%(variant)s-%(arch)s-%(disc_type)s%(disc_num)s%(suffix)s"
|
|
# Custom name (prefix)
|
|
if name:
|
|
custom_iso_name = name
|
|
if version:
|
|
custom_iso_name += "-%s" % version
|
|
format = custom_iso_name + "-%(variant)s-%(arch)s-%(disc_type)s%(disc_num)s%(suffix)s"
|
|
|
|
# XXX: hardcoded disc_num
|
|
return self.compose.get_image_name(arch, variant, disc_type=disc_type,
|
|
disc_num=None, format=format)
|
|
|
|
|
|
class CreateLiveImageThread(WorkerThread):
|
|
EXTS = ('.iso', '.raw.xz')
|
|
|
|
def process(self, item, num):
|
|
compose, cmd, variant, arch = item
|
|
self.failable_arches = cmd.get('failable_arches', [])
|
|
self.can_fail = bool(self.failable_arches)
|
|
with failable(compose, self.can_fail, variant, arch, 'live', cmd.get('subvariant'),
|
|
logger=self.pool._logger):
|
|
self.worker(compose, cmd, variant, arch, num)
|
|
|
|
def worker(self, compose, cmd, variant, arch, num):
|
|
self.basename = '%(name)s-%(version)s-%(release)s' % cmd
|
|
log_file = compose.paths.log.log_file(arch, "liveimage-%s" % self.basename)
|
|
|
|
subvariant = cmd.pop('subvariant')
|
|
|
|
imgname = "%s-%s-%s-%s" % (compose.ci_base.release.short, subvariant,
|
|
'Live' if cmd['type'] == 'live' else 'Disk',
|
|
arch)
|
|
|
|
msg = "Creating ISO (arch: %s, variant: %s): %s" % (arch, variant, self.basename)
|
|
self.pool.log_info("[BEGIN] %s" % msg)
|
|
|
|
koji_wrapper = KojiWrapper(compose.conf["koji_profile"])
|
|
_, version = compose.compose_id.rsplit("-", 1)
|
|
name = cmd["name"] or imgname
|
|
version = cmd["version"] or version
|
|
archive = False
|
|
if cmd["specfile"] and not cmd["scratch"]:
|
|
# Non scratch build are allowed only for rpm wrapped images
|
|
archive = True
|
|
target = compose.conf["live_target"]
|
|
koji_cmd = koji_wrapper.get_create_image_cmd(name, version, target,
|
|
cmd["build_arch"],
|
|
cmd["ks_file"],
|
|
cmd["repos"],
|
|
image_type=cmd['type'],
|
|
wait=True,
|
|
archive=archive,
|
|
specfile=cmd["specfile"],
|
|
release=cmd['release'],
|
|
ksurl=cmd['ksurl'])
|
|
|
|
# avoid race conditions?
|
|
# Kerberos authentication failed: Permission denied in replay cache code (-1765328215)
|
|
time.sleep(num * 3)
|
|
|
|
output = koji_wrapper.run_blocking_cmd(koji_cmd, log_file=log_file)
|
|
if output["retcode"] != 0:
|
|
raise RuntimeError("LiveImage task failed: %s. See %s for more details." % (output["task_id"], log_file))
|
|
|
|
# copy finished image to isos/
|
|
image_path = [path for path in koji_wrapper.get_image_path(output["task_id"])
|
|
if self._is_image(path)]
|
|
if len(image_path) != 1:
|
|
raise RuntimeError('Got %d images from task %d, expected 1.'
|
|
% (len(image_path), output['task_id']))
|
|
image_path = image_path[0]
|
|
filename = cmd.get('filename') or os.path.basename(image_path)
|
|
destination = os.path.join(cmd['dest_dir'], filename)
|
|
shutil.copy2(image_path, destination)
|
|
|
|
# copy finished rpm to isos/ (if rpm wrapped ISO was built)
|
|
if cmd["specfile"]:
|
|
rpm_paths = koji_wrapper.get_wrapped_rpm_path(output["task_id"])
|
|
|
|
if cmd["sign"]:
|
|
# Sign the rpm wrapped images and get their paths
|
|
self.pool.log_info("Signing rpm wrapped images in task_id: %s (expected key ID: %s)"
|
|
% (output["task_id"], compose.conf.get("signing_key_id")))
|
|
signed_rpm_paths = self._sign_image(koji_wrapper, compose, cmd, output["task_id"])
|
|
if signed_rpm_paths:
|
|
rpm_paths = signed_rpm_paths
|
|
|
|
for rpm_path in rpm_paths:
|
|
shutil.copy2(rpm_path, cmd["dest_dir"])
|
|
|
|
if cmd['type'] == 'live':
|
|
# ISO manifest only makes sense for live images
|
|
self._write_manifest(destination)
|
|
|
|
self._add_to_images(compose, variant, subvariant, arch, cmd['type'], self._get_format(image_path), destination)
|
|
|
|
self.pool.log_info("[DONE ] %s (task id: %s)" % (msg, output['task_id']))
|
|
|
|
def _add_to_images(self, compose, variant, subvariant, arch, type, format, path):
|
|
"""Adds the image to images.json"""
|
|
img = Image(compose.im)
|
|
img.type = 'raw-xz' if type == 'appliance' else type
|
|
img.format = format
|
|
img.path = os.path.relpath(path, compose.paths.compose.topdir())
|
|
img.mtime = get_mtime(path)
|
|
img.size = get_file_size(path)
|
|
img.arch = arch
|
|
img.disc_number = 1 # We don't expect multiple disks
|
|
img.disc_count = 1
|
|
img.bootable = True
|
|
img.subvariant = subvariant
|
|
setattr(img, 'can_fail', self.can_fail)
|
|
setattr(img, 'deliverable', 'live')
|
|
compose.im.add(variant=variant.uid, arch=arch, image=img)
|
|
|
|
def _is_image(self, path):
|
|
for ext in self.EXTS:
|
|
if path.endswith(ext):
|
|
return True
|
|
return False
|
|
|
|
def _get_format(self, path):
|
|
"""Get format based on extension."""
|
|
for ext in self.EXTS:
|
|
if path.endswith(ext):
|
|
return ext[1:]
|
|
raise RuntimeError('Getting format for unknown image %s' % path)
|
|
|
|
def _write_manifest(self, iso_path):
|
|
"""Generate manifest for ISO at given path.
|
|
|
|
:param iso_path: (str) absolute path to the ISO
|
|
"""
|
|
dir, filename = os.path.split(iso_path)
|
|
run("cd %s && %s" % (pipes.quote(dir), iso.get_manifest_cmd(filename)))
|
|
|
|
def _sign_image(self, koji_wrapper, compose, cmd, koji_task_id):
|
|
signing_key_id = compose.conf.get("signing_key_id")
|
|
signing_command = compose.conf.get("signing_command")
|
|
|
|
if not signing_key_id:
|
|
self.pool.log_warning("Signing is enabled but signing_key_id is not specified")
|
|
self.pool.log_warning("Signing skipped")
|
|
return None
|
|
if not signing_command:
|
|
self.pool.log_warning("Signing is enabled but signing_command is not specified")
|
|
self.pool.log_warning("Signing skipped")
|
|
return None
|
|
|
|
# Prepare signing log file
|
|
signing_log_file = compose.paths.log.log_file(cmd["build_arch"],
|
|
"live_images-signing-%s" % self.basename)
|
|
|
|
# Sign the rpm wrapped images
|
|
try:
|
|
sign_builds_in_task(koji_wrapper, koji_task_id, signing_command,
|
|
log_file=signing_log_file,
|
|
signing_key_password=compose.conf.get("signing_key_password"))
|
|
except RuntimeError:
|
|
self.pool.log_error("Error while signing rpm wrapped images. See log: %s" % signing_log_file)
|
|
raise
|
|
|
|
# Get pats to the signed rpms
|
|
signing_key_id = signing_key_id.lower() # Koji uses lowercase in paths
|
|
rpm_paths = koji_wrapper.get_signed_wrapped_rpms_paths(koji_task_id, signing_key_id)
|
|
|
|
# Wait untill files are available
|
|
if wait_paths(rpm_paths, 60 * 15):
|
|
# Files are ready
|
|
return rpm_paths
|
|
|
|
# Signed RPMs are not available
|
|
self.pool.log_warning("Signed files are not available: %s" % rpm_paths)
|
|
self.pool.log_warning("Unsigned files will be used")
|
|
return None
|
|
|
|
|
|
def wait_paths(paths, timeout=60):
|
|
started = time.time()
|
|
remaining = paths[:]
|
|
while True:
|
|
for path in remaining[:]:
|
|
if os.path.exists(path):
|
|
remaining.remove(path)
|
|
if not remaining:
|
|
break
|
|
time.sleep(1)
|
|
if timeout >= 0 and (time.time() - started) > timeout:
|
|
return False
|
|
return True
|
|
|
|
|
|
def sign_builds_in_task(koji_wrapper, task_id, signing_command, log_file=None, signing_key_password=None):
|
|
# Get list of nvrs that should be signed
|
|
nvrs = koji_wrapper.get_build_nvrs(task_id)
|
|
if not nvrs:
|
|
# No builds are available (scratch build, etc.?)
|
|
return
|
|
|
|
# Append builds to sign_cmd
|
|
for nvr in nvrs:
|
|
signing_command += " '%s'" % nvr
|
|
|
|
# Log signing command before password is filled in it
|
|
if log_file:
|
|
save_to_file(log_file, signing_command, append=True)
|
|
|
|
# Fill password into the signing command
|
|
if signing_key_password:
|
|
signing_command = signing_command % {"signing_key_password": signing_key_password}
|
|
|
|
# Sign the builds
|
|
run(signing_command, can_fail=False, show_cmd=False, logfile=log_file)
|