almalinux/pungi
14
1
Fork 0
Code Issues Pull Requests Releases Activity

Support KiwiBuild

Browse Source 
Adding kiwibuild phase which is similar to osbuild.

Fixes: https://pagure.io/pungi/issue/1710
Merges: https://pagure.io/pungi/pull-request/1720
JIRA: RHELCMP-13348
Signed-off-by: Haibo Lin <hlin@redhat.com>
(cherry picked from commit 3d630d3e8e)
This commit is contained in:
Haibo Lin 2024-02-01 17:30:05 +08:00 committed by Stepan Oksanichenko
parent 05d9651eba
commit 6aa674fbb3
Signed by: soksanichenko
GPG Key ID: AB9983172AB1E45B
7 changed files with 430 additions and 87 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

130
doc/_static/phases.svg vendored
View File

@ -1,22 +1,22 @@
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
<svg
xmlns:dc="http://purl.org/dc/elements/1.1/"
xmlns:cc="http://creativecommons.org/ns#"
xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
xmlns:svg="http://www.w3.org/2000/svg"
xmlns="http://www.w3.org/2000/svg"
xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
width="610.46454"
height="301.1662"
viewBox="0 0 610.46457 301.1662"
height="327.16599"
viewBox="0 0 610.46457 327.16599"
id="svg2"
version="1.1"
inkscape:version="1.0.2 (e86c870879, 2021-01-15)"
inkscape:version="1.3.2 (091e20e, 2023-11-25)"
sodipodi:docname="phases.svg"
inkscape:export-filename="/home/lsedlar/repos/pungi/doc/_static/phases.png"
inkscape:export-xdpi="90"
inkscape:export-ydpi="90">
inkscape:export-ydpi="90"
xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
xmlns="http://www.w3.org/2000/svg"
xmlns:svg="http://www.w3.org/2000/svg"
xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
xmlns:cc="http://creativecommons.org/ns#"
xmlns:dc="http://purl.org/dc/elements/1.1/">
<sodipodi:namedview
id="base"
pagecolor="#ffffff"
@ -25,15 +25,15 @@
inkscape:pageopacity="1"
inkscape:pageshadow="2"
inkscape:zoom="1.5"
inkscape:cx="9.4746397"
inkscape:cy="58.833855"
inkscape:cx="268"
inkscape:cy="260.66667"
inkscape:document-units="px"
inkscape:current-layer="layer1"
showgrid="false"
inkscape:window-width="2560"
inkscape:window-height="1376"
inkscape:window-width="1920"
inkscape:window-height="1027"
inkscape:window-x="0"
inkscape:window-y="0"
inkscape:window-y="25"
inkscape:window-maximized="1"
units="px"
inkscape:document-rotation="0"
@ -43,7 +43,10 @@
fit-margin-left="7.4"
fit-margin-right="7.4"
fit-margin-bottom="7.4"
lock-margins="true" />
lock-margins="true"
inkscape:showpageshadow="2"
inkscape:pagecheckerboard="0"
inkscape:deskcolor="#d1d1d1" />
<defs
id="defs4">
<marker
@ -70,7 +73,6 @@
<dc:format>image/svg+xml</dc:format>
<dc:type
rdf:resource="http://purl.org/dc/dcmitype/StillImage" />
<dc:title />
</cc:Work>
</rdf:RDF>
</metadata>
@ -103,7 +105,7 @@
style="font-size:13.1479px;line-height:1.25">Pkgset</tspan></text>
</g>
<g
transform="translate(58.253953,-80.817124)"
transform="translate(56.378954,-80.817124)"
id="g3398">
<rect
y="553.98242"
@ -301,13 +303,16 @@
</g>
</g>
</g>
<g
id="g2"
transform="translate(-1.4062678e-8,9.3749966)">
<rect
transform="matrix(0,1,1,0,0,0)"
style="fill:#e9b96e;fill-rule:evenodd;stroke:none;stroke-width:1.85901px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
id="rect3338-1"
width="90.874992"
width="103.12497"
height="115.80065"
x="872.67383"
x="863.29883"
y="486.55563" />
<text
id="text3384-0"
@ -320,6 +325,7 @@
sodipodi:role="line"
x="489.56451"
y="921.73846">ImageChecksum</tspan></text>
</g>
<g
transform="translate(-42.209584,-80.817124)"
id="g3458">
@ -417,16 +423,16 @@
id="rect290"
width="26.295755"
height="224.35098"
x="1063.5973"
x="1091.7223"
y="378.43698"
transform="matrix(0,1,1,0,0,0)" />
<text
xml:space="preserve"
style="font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;line-height:0%;font-family:sans-serif;-inkscape-font-specification:'sans-serif, Normal';text-align:start;letter-spacing:0px;word-spacing:0px;writing-mode:lr-tb;text-anchor:start;fill:#000000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
x="380.74133"
y="1080.3723"
y="1106.6223"
id="text294"><tspan
y="1080.3723"
y="1106.6223"
x="380.74133"
sodipodi:role="line"
id="tspan301"
@ -454,32 +460,9 @@
y="1069.0087"
id="tspan3812">ExtraIsos</tspan></text>
</g>
<g
id="g1031"
transform="translate(-40.740337,29.23522)">
<rect
transform="matrix(0,1,1,0,0,0)"
style="fill:#5ed4ec;fill-opacity:1;fill-rule:evenodd;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
id="rect206"
width="26.295755"
height="102.36562"
x="1066.8611"
y="418.66275" />
<text
id="text210"
y="1084.9105"
x="421.51923"
style="font-style:normal;font-weight:normal;line-height:0%;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
xml:space="preserve"><tspan
y="1084.9105"
x="421.51923"
id="tspan208"
sodipodi:role="line"
style="font-size:13.1479px;line-height:1.25">Repoclosure</tspan></text>
</g>
<rect
y="377.92242"
x="1096.0963"
x="1122.3463"
height="224.24059"
width="26.295755"
id="rect87"
@ -489,17 +472,18 @@
xml:space="preserve"
style="font-style:normal;font-weight:normal;line-height:0%;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
x="380.7789"
y="1114.1458"
y="1140.3958"
id="text91"><tspan
style="font-size:13.1479px;line-height:1.25"
sodipodi:role="line"
id="tspan89"
x="380.7789"
y="1114.1458">Repoclosure</tspan></text>
y="1140.3958">Repoclosure</tspan></text>
<g
id="g206">
id="g206"
transform="translate(0,-1.8749994)">
<rect
style="fill:#fcaf3e;fill-rule:evenodd;stroke:none;stroke-width:1.00033px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
style="fill:#fcd9a4;fill-opacity:1;fill-rule:evenodd;stroke:none;stroke-width:1.00033px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
id="rect290-6"
width="26.295755"
height="101.91849"
@ -516,19 +500,25 @@
x="380.23166"
sodipodi:role="line"
id="tspan301-5"
style="font-size:12px;line-height:0">OSBuild</tspan></text>
style="font-size:12px;line-height:0">KiwiBuild</tspan></text>
</g>
<g
id="g3">
<g
id="g1">
<g
id="g4">
<rect
transform="matrix(0,1,1,0,0,0)"
style="fill:#729fcf;fill-rule:evenodd;stroke:none;stroke-width:1.83502px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
id="rect3338-1-3"
width="88.544876"
width="103.12497"
height="115.80065"
x="970.31763"
x="983.44263"
y="486.55563" />
<text
id="text3384-0-6"
y="1018.2172"
y="1038.8422"
x="489.56451"
style="font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;line-height:0%;font-family:sans-serif;-inkscape-font-specification:'sans-serif, Normal';text-align:start;letter-spacing:0px;word-spacing:0px;writing-mode:lr-tb;text-anchor:start;fill:#000000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
xml:space="preserve"><tspan
@ -536,6 +526,32 @@
id="tspan3391-7"
sodipodi:role="line"
x="489.56451"
y="1018.2172">ImageContainer</tspan></text>
y="1038.8422">ImageContainer</tspan></text>
</g>
</g>
</g>
<g
id="g206-1"
transform="translate(-0.04628921,28.701853)">
<rect
style="fill:#fcaf3e;fill-rule:evenodd;stroke:none;stroke-width:1.00033px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
id="rect290-6-7"
width="26.295755"
height="101.91849"
x="1032.3469"
y="377.92731"
transform="matrix(0,1,1,0,0,0)" />
<text
xml:space="preserve"
style="font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;line-height:0%;font-family:sans-serif;-inkscape-font-specification:'sans-serif, Normal';text-align:start;letter-spacing:0px;word-spacing:0px;writing-mode:lr-tb;text-anchor:start;fill:#000000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
x="380.23166"
y="1049.1219"
id="text294-7-5"><tspan
y="1049.1219"
x="380.23166"
sodipodi:role="line"
id="tspan301-5-5"
style="font-size:12px;line-height:0">OSBuild</tspan></text>
</g>
</g>
</svg>

Side by Side Swipe Overlay

Before

Width:  |  Height:  |  Size: 22 KiB

After

Width:  |  Height:  |  Size: 23 KiB

28
doc/configuration.rst
View File

@ -1604,6 +1604,34 @@ Example
}
KiwiBuild Settings
==================
**kiwibuild**
(*dict*) -- configuration for building images using kiwi by a Koji plugin.
Pungi will trigger a Koji task delegating to kiwi, which will build the image,
import it to Koji via content generators.
Format: ``{variant_uid_regex: [{...}]}``.
Required keys in the configuration dict:
* ``target`` -- (*str*) which build target to use for the task.
* ``description_scm`` -- (*str*) scm URL of description kiwi description.
* ``description_path`` -- (*str*) path to kiwi description
* ``kiwi_profile`` -- (*str*) select profile from description file.
* ``release`` -- (*str*) release of the output image.
* ``arches`` -- (*[str]*) List of architectures.
* ``repos`` -- a list of repositories from which to consume packages for
building the image. By default only the variant repository is used.
* ``failable`` -- (*[str]*) List of architectures for which this
deliverable is not release blocking.
Optional keys:
* ``repos`` -- the repos used to install RPMs in the image.
OSBuild Composer for building images
====================================

6
doc/phases.rst
View File

@ -112,6 +112,12 @@ ImageBuild
This phase wraps up ``koji image-build``. It also updates the metadata
ultimately responsible for ``images.json`` manifest.
KiwiBuild
---------
Similarly to image build, this phases creates a koji `kiwiBuild` task. In the
background it uses Kiwi to create images.
OSBuild
-------

33
pungi/checks.py
View File

@ -1229,6 +1229,39 @@ def make_schema():
},
"additionalProperties": False,
},
"kiwibuild": {
"type": "object",
"patternProperties": {
# Warning: this pattern is a variant uid regex, but the
# format does not let us validate it as there is no regular
# expression to describe all regular expressions.
".+": {
"type": "array",
"items": {
"type": "object",
"properties": {
"target": {"type": "string"},
"description_scm": {"type": "string"},
"description_path": {"type": "string"},
"kiwi_profile": {"type": "string"},
"release": {"type": "string"},
"arches": {"$ref": "#/definitions/list_of_strings"},
"repos": {"$ref": "#/definitions/list_of_strings"},
"failable": {"$ref": "#/definitions/list_of_strings"},
},
"required": [
"target",
"description_scm",
"description_path",
"kiwi_profile",
"release",
],
"additionalProperties": False,
},
}
},
"additionalProperties": False,
},
"osbuild_target": {"type": "string"},
"osbuild_release": {"$ref": "#/definitions/optional_string"},
"osbuild_version": {"type": "string"},

1
pungi/phases/__init__.py
View File

@ -28,6 +28,7 @@ from .extra_isos import ExtraIsosPhase # noqa
from .live_images import LiveImagesPhase # noqa
from .image_build import ImageBuildPhase # noqa
from .image_container import ImageContainerPhase # noqa
from .kiwibuild import KiwiBuildPhase # noqa
from .osbuild import OSBuildPhase # noqa
from .repoclosure import RepoclosurePhase # noqa
from .test import TestPhase # noqa

255
pungi/phases/kiwibuild.py Normal file
View File

@ -0,0 +1,255 @@
# -*- coding: utf-8 -*-
import os
from kobo.threads import ThreadPool, WorkerThread
from kobo import shortcuts
from productmd.images import Image
from . import base
from .. import util
from ..linker import Linker
from ..wrappers import kojiwrapper
from .image_build import EXTENSIONS
class KiwiBuildPhase(
base.PhaseLoggerMixin, base.ImageConfigMixin, base.ConfigGuardedPhase
):
name = "kiwibuild"
def __init__(self, compose):
super(KiwiBuildPhase, self).__init__(compose)
self.pool = ThreadPool(logger=self.logger)
def _get_arches(self, image_conf, arches):
"""Get an intersection of arches in the config dict and the given ones."""
if "arches" in image_conf:
arches = set(image_conf["arches"]) & arches
return sorted(arches)
@staticmethod
def _get_repo_urls(compose, repos, arch="$basearch"):
"""
Get list of repos with resolved repo URLs. Preserve repos defined
as dicts.
"""
resolved_repos = []
for repo in repos:
if isinstance(repo, dict):
try:
url = repo["baseurl"]
except KeyError:
raise RuntimeError(
"`baseurl` is required in repo dict %s" % str(repo)
)
url = util.get_repo_url(compose, url, arch=arch)
if url is None:
raise RuntimeError("Failed to resolve repo URL for %s" % str(repo))
repo["baseurl"] = url
resolved_repos.append(repo)
else:
repo = util.get_repo_url(compose, repo, arch=arch)
if repo is None:
raise RuntimeError("Failed to resolve repo URL for %s" % repo)
resolved_repos.append(repo)
return resolved_repos
def _get_repo(self, image_conf, variant):
"""
Get a list of repos. First included are those explicitly listed in
config, followed by by repo for current variant if it's not included in
the list already.
"""
repos = shortcuts.force_list(image_conf.get("repos", []))
if not variant.is_empty and variant.uid not in repos:
repos.append(variant.uid)
return KiwiBuildPhase._get_repo_urls(self.compose, repos, arch="$arch")
def run(self):
for variant in self.compose.get_variants():
arches = set([x for x in variant.arches if x != "src"])
for image_conf in self.get_config_block(variant):
build_arches = self._get_arches(image_conf, arches)
if not build_arches:
self.log_debug("skip: no arches")
continue
release = self.get_release(image_conf)
target = self.get_config(image_conf, "target")
repo = self._get_repo(image_conf, variant)
can_fail = image_conf.pop("failable", [])
if can_fail == ["*"]:
can_fail = image_conf["arches"]
if can_fail:
can_fail = sorted(can_fail)
self.pool.add(RunKiwiBuildThread(self.pool))
self.pool.queue_put(
(
self.compose,
variant,
image_conf,
build_arches,
release,
target,
repo,
can_fail,
)
)
self.pool.start()
class RunKiwiBuildThread(WorkerThread):
def process(self, item, num):
(
compose,
variant,
config,
arches,
release,
target,
repo,
can_fail,
) = item
self.can_fail = can_fail
self.num = num
with util.failable(
compose,
can_fail,
variant,
"*",
"kiwibuild",
logger=self.pool._logger,
):
self.worker(compose, variant, config, arches, release, target, repo)
def worker(self, compose, variant, config, arches, release, target, repo):
msg = "kiwibuild task for variant %s" % variant.uid
self.pool.log_info("[BEGIN] %s" % msg)
koji = kojiwrapper.KojiWrapper(compose)
koji.login()
task_id = koji.koji_proxy.kiwiBuild(
target,
arches,
config["description_scm"],
config["description_path"],
profile=config["kiwi_profile"],
release=release,
repos=repo,
)
koji.save_task_id(task_id)
# Wait for it to finish and capture the output into log file.
log_dir = os.path.join(compose.paths.log.topdir(), "kiwibuild")
util.makedirs(log_dir)
log_file = os.path.join(
log_dir, "%s-%s-watch-task.log" % (variant.uid, self.num)
)
if koji.watch_task(task_id, log_file) != 0:
raise RuntimeError(
"kiwiBuild: task %s failed: see %s for details" % (task_id, log_file)
)
# Refresh koji session which may have timed out while the task was
# running. Watching is done via a subprocess, so the session is
# inactive.
koji = kojiwrapper.KojiWrapper(compose)
linker = Linker(logger=self.pool._logger)
# Process all images in the build. There should be one for each
# architecture, but we don't verify that.
build_info = koji.koji_proxy.listBuilds(taskID=task_id)[0]
for archive in koji.koji_proxy.listArchives(buildID=build_info["build_id"]):
if archive["type_name"] not in EXTENSIONS:
# Ignore values that are not of required types.
continue
# Get architecture of the image from extra data.
try:
arch = archive["extra"]["image"]["arch"]
except KeyError:
raise RuntimeError("Image doesn't have any architecture!")
# image_dir is absolute path to which the image should be copied.
# We also need the same path as relative to compose directory for
# including in the metadata.
if archive["type_name"] == "iso":
# If the produced image is actually an ISO, it should go to
# iso/ subdirectory.
image_dir = compose.paths.compose.iso_dir(arch, variant)
rel_image_dir = compose.paths.compose.iso_dir(
arch, variant, relative=True
)
else:
image_dir = compose.paths.compose.image_dir(variant) % {"arch": arch}
rel_image_dir = compose.paths.compose.image_dir(
variant, relative=True
) % {"arch": arch}
util.makedirs(image_dir)
image_dest = os.path.join(image_dir, archive["filename"])
src_file = compose.koji_downloader.get_file(
os.path.join(
koji.koji_module.pathinfo.imagebuild(build_info),
archive["filename"],
),
)
linker.link(src_file, image_dest, link_type=compose.conf["link_type"])
for suffix in EXTENSIONS[archive["type_name"]]:
if archive["filename"].endswith(suffix):
break
else:
# No suffix matched.
raise RuntimeError(
"Failed to generate metadata. Format %s doesn't match type %s"
% (suffix, archive["type_name"])
)
# Update image manifest
img = Image(compose.im)
# Get the manifest type from the config if supplied, otherwise we
# determine the manifest type based on the koji output
img.type = config.get("manifest_type")
if not img.type:
if archive["type_name"] != "iso":
img.type = archive["type_name"]
else:
fn = archive["filename"].lower()
if "ostree" in fn:
img.type = "dvd-ostree-osbuild"
elif "live" in fn:
img.type = "live-osbuild"
elif "netinst" in fn or "boot" in fn:
img.type = "boot"
else:
img.type = "dvd"
img.format = suffix
img.path = os.path.join(rel_image_dir, archive["filename"])
img.mtime = util.get_mtime(image_dest)
img.size = util.get_file_size(image_dest)
img.arch = arch
img.disc_number = 1 # We don't expect multiple disks
img.disc_count = 1
img.bootable = False
img.subvariant = config.get("subvariant", variant.uid)
setattr(img, "can_fail", self.can_fail)
setattr(img, "deliverable", "image-build")
compose.im.add(variant=variant.uid, arch=arch, image=img)
self.pool.log_info("[DONE ] %s (task id: %s)" % (msg, task_id))

4
pungi/scripts/pungi_koji.py
View File

@ -429,6 +429,7 @@ def run_compose(
liveimages_phase = pungi.phases.LiveImagesPhase(compose)
livemedia_phase = pungi.phases.LiveMediaPhase(compose)
image_build_phase = pungi.phases.ImageBuildPhase(compose, buildinstall_phase)
kiwibuild_phase = pungi.phases.KiwiBuildPhase(compose)
osbuild_phase = pungi.phases.OSBuildPhase(compose)
osbs_phase = pungi.phases.OSBSPhase(compose, pkgset_phase, buildinstall_phase)
image_container_phase = pungi.phases.ImageContainerPhase(compose)
@ -457,6 +458,7 @@ def run_compose(
osbs_phase,
osbuild_phase,
image_container_phase,
kiwibuild_phase,
):
if phase.skip():
continue
@ -556,6 +558,7 @@ def run_compose(
image_build_phase,
livemedia_phase,
osbuild_phase,
kiwibuild_phase,
)
post_image_phase = pungi.phases.WeaverPhase(
compose, (image_checksum_phase, image_container_phase)
@ -580,6 +583,7 @@ def run_compose(
and liveimages_phase.skip()
and livemedia_phase.skip()
and image_build_phase.skip()
and kiwibuild_phase.skip()
and osbuild_phase.skip()
and ostree_container_phase.skip()
):