From 6aa674fbb34a916e185b268d1e07a20815be3fbb Mon Sep 17 00:00:00 2001 From: Haibo Lin Date: Thu, 1 Feb 2024 17:30:05 +0800 Subject: [PATCH] Support KiwiBuild Adding kiwibuild phase which is similar to osbuild. Fixes: https://pagure.io/pungi/issue/1710 Merges: https://pagure.io/pungi/pull-request/1720 JIRA: RHELCMP-13348 Signed-off-by: Haibo Lin (cherry picked from commit 3d630d3e8e56e17db6f61c907120fc25e13c8579) --- doc/_static/phases.svg | 190 +++++++++++++++------------ doc/configuration.rst | 28 ++++ doc/phases.rst | 6 + pungi/checks.py | 33 +++++ pungi/phases/__init__.py | 1 + pungi/phases/kiwibuild.py | 255 ++++++++++++++++++++++++++++++++++++ pungi/scripts/pungi_koji.py | 4 + 7 files changed, 430 insertions(+), 87 deletions(-) create mode 100644 pungi/phases/kiwibuild.py diff --git a/doc/_static/phases.svg b/doc/_static/phases.svg index b973798a..02cde085 100644 --- a/doc/_static/phases.svg +++ b/doc/_static/phases.svg @@ -1,22 +1,22 @@ + inkscape:export-ydpi="90" + xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape" + xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd" + xmlns="http://www.w3.org/2000/svg" + xmlns:svg="http://www.w3.org/2000/svg" + xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#" + xmlns:cc="http://creativecommons.org/ns#" + xmlns:dc="http://purl.org/dc/elements/1.1/"> + lock-margins="true" + inkscape:showpageshadow="2" + inkscape:pagecheckerboard="0" + inkscape:deskcolor="#d1d1d1" /> image/svg+xml - @@ -103,7 +105,7 @@ style="font-size:13.1479px;line-height:1.25">Pkgset - - + + ImageChecksum + style="font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;line-height:0%;font-family:sans-serif;-inkscape-font-specification:'sans-serif, Normal';text-align:start;letter-spacing:0px;word-spacing:0px;writing-mode:lr-tb;text-anchor:start;fill:#000000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1" + xml:space="preserve">ImageChecksum + @@ -417,16 +423,16 @@ id="rect290" width="26.295755" height="224.35098" - x="1063.5973" + x="1091.7223" y="378.43698" transform="matrix(0,1,1,0,0,0)" /> ExtraIsos - - - Repoclosure - Repoclosure + y="1140.3958">Repoclosure + id="g206" + transform="translate(0,-1.8749994)"> KiwiBuild + + + + + + ImageContainer + + + + + + OSBuild - - ImageContainer diff --git a/doc/configuration.rst b/doc/configuration.rst index 75b6a682..2a23453b 100644 --- a/doc/configuration.rst +++ b/doc/configuration.rst @@ -1604,6 +1604,34 @@ Example } +KiwiBuild Settings +================== + +**kiwibuild** + (*dict*) -- configuration for building images using kiwi by a Koji plugin. + Pungi will trigger a Koji task delegating to kiwi, which will build the image, + import it to Koji via content generators. + + Format: ``{variant_uid_regex: [{...}]}``. + + Required keys in the configuration dict: + + * ``target`` -- (*str*) which build target to use for the task. + * ``description_scm`` -- (*str*) scm URL of description kiwi description. + * ``description_path`` -- (*str*) path to kiwi description + * ``kiwi_profile`` -- (*str*) select profile from description file. + * ``release`` -- (*str*) release of the output image. + * ``arches`` -- (*[str]*) List of architectures. + * ``repos`` -- a list of repositories from which to consume packages for + building the image. By default only the variant repository is used. + * ``failable`` -- (*[str]*) List of architectures for which this + deliverable is not release blocking. + + Optional keys: + + * ``repos`` -- the repos used to install RPMs in the image. + + OSBuild Composer for building images ==================================== diff --git a/doc/phases.rst b/doc/phases.rst index 7119e7f8..2b926028 100644 --- a/doc/phases.rst +++ b/doc/phases.rst @@ -112,6 +112,12 @@ ImageBuild This phase wraps up ``koji image-build``. It also updates the metadata ultimately responsible for ``images.json`` manifest. +KiwiBuild +--------- + +Similarly to image build, this phases creates a koji `kiwiBuild` task. In the +background it uses Kiwi to create images. + OSBuild ------- diff --git a/pungi/checks.py b/pungi/checks.py index f966f403..a26ad9eb 100644 --- a/pungi/checks.py +++ b/pungi/checks.py @@ -1229,6 +1229,39 @@ def make_schema(): }, "additionalProperties": False, }, + "kiwibuild": { + "type": "object", + "patternProperties": { + # Warning: this pattern is a variant uid regex, but the + # format does not let us validate it as there is no regular + # expression to describe all regular expressions. + ".+": { + "type": "array", + "items": { + "type": "object", + "properties": { + "target": {"type": "string"}, + "description_scm": {"type": "string"}, + "description_path": {"type": "string"}, + "kiwi_profile": {"type": "string"}, + "release": {"type": "string"}, + "arches": {"$ref": "#/definitions/list_of_strings"}, + "repos": {"$ref": "#/definitions/list_of_strings"}, + "failable": {"$ref": "#/definitions/list_of_strings"}, + }, + "required": [ + "target", + "description_scm", + "description_path", + "kiwi_profile", + "release", + ], + "additionalProperties": False, + }, + } + }, + "additionalProperties": False, + }, "osbuild_target": {"type": "string"}, "osbuild_release": {"$ref": "#/definitions/optional_string"}, "osbuild_version": {"type": "string"}, diff --git a/pungi/phases/__init__.py b/pungi/phases/__init__.py index 24bdc365..7d94e485 100644 --- a/pungi/phases/__init__.py +++ b/pungi/phases/__init__.py @@ -28,6 +28,7 @@ from .extra_isos import ExtraIsosPhase # noqa from .live_images import LiveImagesPhase # noqa from .image_build import ImageBuildPhase # noqa from .image_container import ImageContainerPhase # noqa +from .kiwibuild import KiwiBuildPhase # noqa from .osbuild import OSBuildPhase # noqa from .repoclosure import RepoclosurePhase # noqa from .test import TestPhase # noqa diff --git a/pungi/phases/kiwibuild.py b/pungi/phases/kiwibuild.py new file mode 100644 index 00000000..6921d6ff --- /dev/null +++ b/pungi/phases/kiwibuild.py @@ -0,0 +1,255 @@ +# -*- coding: utf-8 -*- + +import os +from kobo.threads import ThreadPool, WorkerThread +from kobo import shortcuts +from productmd.images import Image + +from . import base +from .. import util +from ..linker import Linker +from ..wrappers import kojiwrapper +from .image_build import EXTENSIONS + + +class KiwiBuildPhase( + base.PhaseLoggerMixin, base.ImageConfigMixin, base.ConfigGuardedPhase +): + name = "kiwibuild" + + def __init__(self, compose): + super(KiwiBuildPhase, self).__init__(compose) + self.pool = ThreadPool(logger=self.logger) + + def _get_arches(self, image_conf, arches): + """Get an intersection of arches in the config dict and the given ones.""" + if "arches" in image_conf: + arches = set(image_conf["arches"]) & arches + return sorted(arches) + + @staticmethod + def _get_repo_urls(compose, repos, arch="$basearch"): + """ + Get list of repos with resolved repo URLs. Preserve repos defined + as dicts. + """ + resolved_repos = [] + + for repo in repos: + if isinstance(repo, dict): + try: + url = repo["baseurl"] + except KeyError: + raise RuntimeError( + "`baseurl` is required in repo dict %s" % str(repo) + ) + url = util.get_repo_url(compose, url, arch=arch) + if url is None: + raise RuntimeError("Failed to resolve repo URL for %s" % str(repo)) + repo["baseurl"] = url + resolved_repos.append(repo) + else: + repo = util.get_repo_url(compose, repo, arch=arch) + if repo is None: + raise RuntimeError("Failed to resolve repo URL for %s" % repo) + resolved_repos.append(repo) + + return resolved_repos + + def _get_repo(self, image_conf, variant): + """ + Get a list of repos. First included are those explicitly listed in + config, followed by by repo for current variant if it's not included in + the list already. + """ + repos = shortcuts.force_list(image_conf.get("repos", [])) + + if not variant.is_empty and variant.uid not in repos: + repos.append(variant.uid) + + return KiwiBuildPhase._get_repo_urls(self.compose, repos, arch="$arch") + + def run(self): + for variant in self.compose.get_variants(): + arches = set([x for x in variant.arches if x != "src"]) + + for image_conf in self.get_config_block(variant): + build_arches = self._get_arches(image_conf, arches) + if not build_arches: + self.log_debug("skip: no arches") + continue + + release = self.get_release(image_conf) + target = self.get_config(image_conf, "target") + + repo = self._get_repo(image_conf, variant) + + can_fail = image_conf.pop("failable", []) + if can_fail == ["*"]: + can_fail = image_conf["arches"] + if can_fail: + can_fail = sorted(can_fail) + + self.pool.add(RunKiwiBuildThread(self.pool)) + self.pool.queue_put( + ( + self.compose, + variant, + image_conf, + build_arches, + release, + target, + repo, + can_fail, + ) + ) + + self.pool.start() + + +class RunKiwiBuildThread(WorkerThread): + def process(self, item, num): + ( + compose, + variant, + config, + arches, + release, + target, + repo, + can_fail, + ) = item + self.can_fail = can_fail + self.num = num + with util.failable( + compose, + can_fail, + variant, + "*", + "kiwibuild", + logger=self.pool._logger, + ): + self.worker(compose, variant, config, arches, release, target, repo) + + def worker(self, compose, variant, config, arches, release, target, repo): + msg = "kiwibuild task for variant %s" % variant.uid + self.pool.log_info("[BEGIN] %s" % msg) + koji = kojiwrapper.KojiWrapper(compose) + koji.login() + + task_id = koji.koji_proxy.kiwiBuild( + target, + arches, + config["description_scm"], + config["description_path"], + profile=config["kiwi_profile"], + release=release, + repos=repo, + ) + + koji.save_task_id(task_id) + + # Wait for it to finish and capture the output into log file. + log_dir = os.path.join(compose.paths.log.topdir(), "kiwibuild") + util.makedirs(log_dir) + log_file = os.path.join( + log_dir, "%s-%s-watch-task.log" % (variant.uid, self.num) + ) + if koji.watch_task(task_id, log_file) != 0: + raise RuntimeError( + "kiwiBuild: task %s failed: see %s for details" % (task_id, log_file) + ) + + # Refresh koji session which may have timed out while the task was + # running. Watching is done via a subprocess, so the session is + # inactive. + koji = kojiwrapper.KojiWrapper(compose) + + linker = Linker(logger=self.pool._logger) + + # Process all images in the build. There should be one for each + # architecture, but we don't verify that. + build_info = koji.koji_proxy.listBuilds(taskID=task_id)[0] + for archive in koji.koji_proxy.listArchives(buildID=build_info["build_id"]): + if archive["type_name"] not in EXTENSIONS: + # Ignore values that are not of required types. + continue + + # Get architecture of the image from extra data. + try: + arch = archive["extra"]["image"]["arch"] + except KeyError: + raise RuntimeError("Image doesn't have any architecture!") + + # image_dir is absolute path to which the image should be copied. + # We also need the same path as relative to compose directory for + # including in the metadata. + if archive["type_name"] == "iso": + # If the produced image is actually an ISO, it should go to + # iso/ subdirectory. + image_dir = compose.paths.compose.iso_dir(arch, variant) + rel_image_dir = compose.paths.compose.iso_dir( + arch, variant, relative=True + ) + else: + image_dir = compose.paths.compose.image_dir(variant) % {"arch": arch} + rel_image_dir = compose.paths.compose.image_dir( + variant, relative=True + ) % {"arch": arch} + util.makedirs(image_dir) + + image_dest = os.path.join(image_dir, archive["filename"]) + + src_file = compose.koji_downloader.get_file( + os.path.join( + koji.koji_module.pathinfo.imagebuild(build_info), + archive["filename"], + ), + ) + + linker.link(src_file, image_dest, link_type=compose.conf["link_type"]) + + for suffix in EXTENSIONS[archive["type_name"]]: + if archive["filename"].endswith(suffix): + break + else: + # No suffix matched. + raise RuntimeError( + "Failed to generate metadata. Format %s doesn't match type %s" + % (suffix, archive["type_name"]) + ) + + # Update image manifest + img = Image(compose.im) + + # Get the manifest type from the config if supplied, otherwise we + # determine the manifest type based on the koji output + img.type = config.get("manifest_type") + if not img.type: + if archive["type_name"] != "iso": + img.type = archive["type_name"] + else: + fn = archive["filename"].lower() + if "ostree" in fn: + img.type = "dvd-ostree-osbuild" + elif "live" in fn: + img.type = "live-osbuild" + elif "netinst" in fn or "boot" in fn: + img.type = "boot" + else: + img.type = "dvd" + + img.format = suffix + img.path = os.path.join(rel_image_dir, archive["filename"]) + img.mtime = util.get_mtime(image_dest) + img.size = util.get_file_size(image_dest) + img.arch = arch + img.disc_number = 1 # We don't expect multiple disks + img.disc_count = 1 + img.bootable = False + img.subvariant = config.get("subvariant", variant.uid) + setattr(img, "can_fail", self.can_fail) + setattr(img, "deliverable", "image-build") + compose.im.add(variant=variant.uid, arch=arch, image=img) + + self.pool.log_info("[DONE ] %s (task id: %s)" % (msg, task_id)) diff --git a/pungi/scripts/pungi_koji.py b/pungi/scripts/pungi_koji.py index 87c2ab64..592e636b 100644 --- a/pungi/scripts/pungi_koji.py +++ b/pungi/scripts/pungi_koji.py @@ -429,6 +429,7 @@ def run_compose( liveimages_phase = pungi.phases.LiveImagesPhase(compose) livemedia_phase = pungi.phases.LiveMediaPhase(compose) image_build_phase = pungi.phases.ImageBuildPhase(compose, buildinstall_phase) + kiwibuild_phase = pungi.phases.KiwiBuildPhase(compose) osbuild_phase = pungi.phases.OSBuildPhase(compose) osbs_phase = pungi.phases.OSBSPhase(compose, pkgset_phase, buildinstall_phase) image_container_phase = pungi.phases.ImageContainerPhase(compose) @@ -457,6 +458,7 @@ def run_compose( osbs_phase, osbuild_phase, image_container_phase, + kiwibuild_phase, ): if phase.skip(): continue @@ -556,6 +558,7 @@ def run_compose( image_build_phase, livemedia_phase, osbuild_phase, + kiwibuild_phase, ) post_image_phase = pungi.phases.WeaverPhase( compose, (image_checksum_phase, image_container_phase) @@ -580,6 +583,7 @@ def run_compose( and liveimages_phase.skip() and livemedia_phase.skip() and image_build_phase.skip() + and kiwibuild_phase.skip() and osbuild_phase.skip() and ostree_container_phase.skip() ):