pungi/pungi/phases/osbuild.py
Lubomír Sedlář 98ddc74c16 osbuild: Fix not failing on failable tasks
The task can only fail as a whole, thus it only makes sense to set all
architectures (`*`) as failable. The correct value needs to be checked
though.

JIRA: RHELCMP-3412
Signed-off-by: Lubomír Sedlář <lsedlar@redhat.com>
2020-12-02 14:30:20 +01:00

202 lines
7.1 KiB
Python

# -*- coding: utf-8 -*-
import os
from kobo.threads import ThreadPool, WorkerThread
from kobo import shortcuts
from productmd.images import Image
from . import base
from .. import util
from ..linker import Linker
from ..wrappers import kojiwrapper
from .image_build import EXTENSIONS
class OSBuildPhase(
base.PhaseLoggerMixin, base.ImageConfigMixin, base.ConfigGuardedPhase
):
name = "osbuild"
def __init__(self, compose):
super(OSBuildPhase, self).__init__(compose)
self.pool = ThreadPool(logger=self.logger)
def _get_arches(self, image_conf, arches):
"""Get an intersection of arches in the config dict and the given ones."""
if "arches" in image_conf:
arches = set(image_conf["arches"]) & arches
return sorted(arches)
def _get_repo(self, image_conf, variant):
"""
Get a list of repos. First included are those explicitly listed in
config, followed by by repo for current variant if it's not included in
the list already.
"""
repos = shortcuts.force_list(image_conf.get("repo", []))
if not variant.is_empty and variant.uid not in repos:
repos.append(variant.uid)
return util.get_repo_urls(self.compose, repos, arch="$arch")
def run(self):
for variant in self.compose.get_variants():
arches = set([x for x in variant.arches if x != "src"])
for image_conf in self.get_config_block(variant):
build_arches = self._get_arches(image_conf, arches)
if not build_arches:
self.log_debug("skip: no arches")
continue
release = self.get_release(image_conf)
version = self.get_version(image_conf)
target = self.get_config(image_conf, "target")
repo = self._get_repo(image_conf, variant)
can_fail = image_conf.pop("failable", [])
if can_fail == ["*"]:
can_fail = image_conf["arches"]
if can_fail:
can_fail = sorted(can_fail)
self.pool.add(RunOSBuildThread(self.pool))
self.pool.queue_put(
(
self.compose,
variant,
image_conf,
build_arches,
version,
release,
target,
repo,
can_fail,
)
)
self.pool.start()
class RunOSBuildThread(WorkerThread):
def process(self, item, num):
(
compose,
variant,
config,
arches,
version,
release,
target,
repo,
can_fail,
) = item
self.can_fail = can_fail
self.num = num
with util.failable(
compose, can_fail, variant, "*", "osbuild", logger=self.pool._logger,
):
self.worker(
compose, variant, config, arches, version, release, target, repo
)
def worker(self, compose, variant, config, arches, version, release, target, repo):
msg = "OSBuild task for variant %s" % variant.uid
self.pool.log_info("[BEGIN] %s" % msg)
koji = kojiwrapper.KojiWrapper(compose.conf["koji_profile"])
koji.login()
# Start task
opts = {"repo": repo}
if release:
opts["release"] = release
task_id = koji.koji_proxy.osbuildImage(
config["name"],
version,
config["distro"],
config["image_types"],
target,
arches,
opts=opts,
)
# Wait for it to finish and capture the output into log file.
log_dir = os.path.join(compose.paths.log.topdir(), "osbuild")
util.makedirs(log_dir)
log_file = os.path.join(
log_dir, "%s-%s-watch-task.log" % (variant.uid, self.num)
)
if koji.watch_task(task_id, log_file) != 0:
raise RuntimeError(
"OSBuild: task %s failed: see %s for details" % (task_id, log_file)
)
# Refresh koji session which may have timed out while the task was
# running. Watching is done via a subprocess, so the session is
# inactive.
koji = kojiwrapper.KojiWrapper(compose.conf["koji_profile"])
# Get build id via the task's result json data
result = koji.koji_proxy.getTaskResult(task_id)
build_id = result["koji"]["build"]
linker = Linker(logger=self.pool._logger)
# Process all images in the build. There should be one for each
# architecture, but we don't verify that.
build_info = koji.koji_proxy.getBuild(build_id)
for archive in koji.koji_proxy.listArchives(buildID=build_id):
if archive["type_name"] not in config["image_types"]:
# Ignore values that are not of required types.
continue
# Get architecture of the image from extra data.
try:
arch = archive["extra"]["image"]["arch"]
except KeyError:
raise RuntimeError("Image doesn't have any architecture!")
# image_dir is absolute path to which the image should be copied.
# We also need the same path as relative to compose directory for
# including in the metadata.
image_dir = compose.paths.compose.image_dir(variant) % {"arch": arch}
rel_image_dir = compose.paths.compose.image_dir(variant, relative=True) % {
"arch": arch
}
util.makedirs(image_dir)
image_dest = os.path.join(image_dir, archive["filename"])
src_file = os.path.join(
koji.koji_module.pathinfo.imagebuild(build_info), archive["filename"]
)
linker.link(src_file, image_dest, link_type=compose.conf["link_type"])
suffix = archive["filename"].rsplit(".", 1)[-1]
if suffix not in EXTENSIONS[archive["type_name"]]:
raise RuntimeError(
"Failed to generate metadata. Format %s doesn't match type %s"
% (suffix, archive["type_name"])
)
# Update image manifest
img = Image(compose.im)
img.type = archive["type_name"]
img.format = suffix
img.path = os.path.join(rel_image_dir, archive["filename"])
img.mtime = util.get_mtime(image_dest)
img.size = util.get_file_size(image_dest)
img.arch = arch
img.disc_number = 1 # We don't expect multiple disks
img.disc_count = 1
img.bootable = False
img.subvariant = config.get("subvariant", variant.uid)
setattr(img, "can_fail", self.can_fail)
setattr(img, "deliverable", "image-build")
compose.im.add(variant=variant.uid, arch=arch, image=img)
self.pool.log_info("[DONE ] %s (task id: %s)" % (msg, task_id))