pungi/pungi/phases/livemedia_phase.py
Haibo Lin 035b37c566 Cancel koji tasks when pungi terminated
JIRA: RHELCMP-4148
Signed-off-by: Haibo Lin <hlin@redhat.com>
2021-03-23 14:47:48 +08:00

204 lines
8.0 KiB
Python

# -*- coding: utf-8 -*-
import os
import time
from kobo import shortcuts
from pungi.util import makedirs, get_mtime, get_file_size, failable, log_failed_task
from pungi.util import translate_path, get_repo_urls
from pungi.phases.base import ConfigGuardedPhase, ImageConfigMixin, PhaseLoggerMixin
from pungi.linker import Linker
from pungi.wrappers.kojiwrapper import KojiWrapper
from kobo.threads import ThreadPool, WorkerThread
from productmd.images import Image
class LiveMediaPhase(PhaseLoggerMixin, ImageConfigMixin, ConfigGuardedPhase):
"""class for wrapping up koji spin-livemedia"""
name = "live_media"
def __init__(self, compose):
super(LiveMediaPhase, self).__init__(compose)
self.pool = ThreadPool(logger=self.logger)
def _get_repos(self, image_conf, variant):
"""
Get a list of repo urls. First included are those explicitly listed in config,
followed by repo for current variant if it's not present in the list.
"""
repos = shortcuts.force_list(image_conf.get("repo", []))
if not variant.is_empty:
if variant.uid not in repos:
repos.append(variant.uid)
return get_repo_urls(self.compose, repos)
def _get_arches(self, image_conf, arches):
if "arches" in image_conf:
arches = set(image_conf.get("arches", [])) & arches
return sorted(arches)
def _get_install_tree(self, image_conf, variant):
if "install_tree_from" in image_conf:
variant_uid = image_conf["install_tree_from"]
try:
variant = self.compose.all_variants[variant_uid]
except KeyError:
raise RuntimeError(
"There is no variant %s to get repo from when building "
"live media for %s." % (variant_uid, variant.uid)
)
return translate_path(
self.compose,
self.compose.paths.compose.os_tree("$basearch", variant, create_dir=False),
)
def run(self):
for variant in self.compose.get_variants():
arches = set([x for x in variant.arches if x != "src"])
for image_conf in self.get_config_block(variant):
subvariant = image_conf.get("subvariant", variant.uid)
name = image_conf.get(
"name",
"%s-%s-Live" % (self.compose.ci_base.release.short, subvariant),
)
config = {
"target": self.get_config(image_conf, "target"),
"arches": self._get_arches(image_conf, arches),
"ksfile": image_conf["kickstart"],
"ksurl": self.get_ksurl(image_conf),
"ksversion": image_conf.get("ksversion"),
"scratch": image_conf.get("scratch", False),
"release": self.get_release(image_conf),
"skip_tag": image_conf.get("skip_tag"),
"name": name,
"subvariant": subvariant,
"repo": self._get_repos(image_conf, variant),
"install_tree": self._get_install_tree(image_conf, variant),
"version": self.get_version(image_conf),
"failable_arches": image_conf.get("failable", []),
}
if config["failable_arches"] == ["*"]:
config["failable_arches"] = config["arches"]
self.pool.add(LiveMediaThread(self.pool))
self.pool.queue_put((self.compose, variant, config))
self.pool.start()
class LiveMediaThread(WorkerThread):
def process(self, item, num):
compose, variant, config = item
subvariant = config.pop("subvariant")
self.failable_arches = config.pop("failable_arches")
self.num = num
can_fail = set(self.failable_arches) == set(config["arches"])
with failable(
compose,
can_fail,
variant,
"*",
"live-media",
subvariant,
logger=self.pool._logger,
):
self.worker(compose, variant, subvariant, config)
def _get_log_file(self, compose, variant, subvariant, config):
arches = "-".join(config["arches"])
return compose.paths.log.log_file(
arches, "livemedia-%s-%s" % (variant.uid, subvariant)
)
def _run_command(self, koji_wrapper, cmd, compose, log_file):
time.sleep(self.num * 3)
output = koji_wrapper.run_blocking_cmd(cmd, log_file=log_file)
self.pool.log_debug("live media outputs: %s" % (output))
if output["retcode"] != 0:
self.pool.log_error("Live media task failed.")
raise RuntimeError(
"Live media task failed: %s. See %s for more details."
% (output["task_id"], log_file)
)
return output
def _get_cmd(self, koji_wrapper, config):
"""Replace `arches` (as list) with `arch` as a comma-separated string."""
copy = dict(config)
copy["arch"] = ",".join(copy.pop("arches", []))
copy["can_fail"] = self.failable_arches
return koji_wrapper.get_live_media_cmd(copy)
def worker(self, compose, variant, subvariant, config):
msg = "Live media: %s (arches: %s, variant: %s, subvariant: %s)" % (
config["name"],
" ".join(config["arches"]),
variant.uid,
subvariant,
)
self.pool.log_info("[BEGIN] %s" % msg)
koji_wrapper = KojiWrapper(compose)
cmd = self._get_cmd(koji_wrapper, config)
log_file = self._get_log_file(compose, variant, subvariant, config)
output = self._run_command(koji_wrapper, cmd, compose, log_file)
# collect results and update manifest
image_infos = []
paths = koji_wrapper.get_image_paths(
output["task_id"],
callback=lambda arch: log_failed_task(
compose, variant, arch, "live-media", subvariant
),
)
for arch, paths in paths.items():
for path in paths:
if path.endswith(".iso"):
image_infos.append({"path": path, "arch": arch})
if len(image_infos) < len(config["arches"]) - len(self.failable_arches):
self.pool.log_error(
"Error in koji task %s. Expected to find at least one image "
"for each required arch (%s). Got %s."
% (output["task_id"], len(config["arches"]), len(image_infos))
)
raise RuntimeError("Image count mismatch in task %s." % output["task_id"])
linker = Linker(logger=self.pool._logger)
link_type = compose.conf["link_type"]
for image_info in image_infos:
image_dir = compose.paths.compose.iso_dir(image_info["arch"], variant)
makedirs(image_dir)
relative_image_dir = compose.paths.compose.iso_dir(
image_info["arch"], variant, relative=True
)
# let's not change filename of koji outputs
image_dest = os.path.join(image_dir, os.path.basename(image_info["path"]))
src_file = os.path.realpath(image_info["path"])
linker.link(src_file, image_dest, link_type=link_type)
# Update image manifest
img = Image(compose.im)
img.type = "live"
img.format = "iso"
img.path = os.path.join(relative_image_dir, os.path.basename(image_dest))
img.mtime = get_mtime(image_dest)
img.size = get_file_size(image_dest)
img.arch = image_info["arch"]
img.disc_number = 1 # We don't expect multiple disks
img.disc_count = 1
img.bootable = True
img.subvariant = subvariant
setattr(img, "can_fail", bool(self.failable_arches))
setattr(img, "deliverable", "live-media")
compose.im.add(variant=variant.uid, arch=image_info["arch"], image=img)
self.pool.log_info("[DONE ] %s (task id: %s)" % (msg, output["task_id"]))