almalinux
/
pungi
15
1
Fork 0
Code Issues Pull Requests Releases Activity

Add phase for building images with osbuild 

This is similar to image-build in terms of what it does, and somewhat
similar to OSBS phase in how it's implemented.

The phase reads configuration, submits the build via XMLRPC call and
waits for the task to finish. Then it downloads the built image and
includes it in the compose metadata.

JIRA: RHELCMP-315
Signed-off-by: Lubomír Sedlář <lsedlar@redhat.com>
This commit is contained in:
Lubomír Sedlář 2020-10-12 14:56:29 +02:00
parent 609a555597
commit a45f4969f3
6 changed files with 581 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

42
doc/configuration.rst
View File

@ -1305,6 +1305,7 @@ Target is specified by these settings.
* ``live_media_target``
* ``image_build_target``
* ``live_images_target``
* ``osbuild_target``
Version is specified by these options. If no version is set, a default value
will be provided according to :ref:`automatic versioning <auto-version>`.
@ -1313,6 +1314,7 @@ will be provided according to :ref:`automatic versioning <auto-version>`.
* ``live_media_version``
* ``image_build_version``
* ``live_images_version``
* ``osbuild_version``
Release is specified by these options. If set to a magic value to
``!RELEASE_FROM_LABEL_DATE_TYPE_RESPIN``, a value will be generated according
@ -1322,6 +1324,7 @@ to :ref:`automatic versioning <auto-version>`.
* ``live_media_release``
* ``image_build_release``
* ``live_images_release``
* ``osbuild_release``
Each configuration block can also optionally specify a ``failable`` key. For
live images it should have a boolean value. For live media and image build it
@ -1512,6 +1515,45 @@ Example
}
OSBuild Composer for building images
====================================
**osbuild**
(*dict*) -- configuration for building images in OSBuild Composer service
fronted by a Koji plugin. Pungi will trigger a Koji task delegating to the
OSBuild Composer, which will build the image, import it to Koji via content
generators.
Format: ``{variant_uid_regex: [{...}]}``.
Required keys in the configuration dict:
* ``name`` -- name of the Koji package
* ``distro`` -- image for which distribution should be build TODO examples
* ``image_type`` -- a list of image types to build (e.g. ``qcow2``)
Optional keys:
* ``target`` -- which build target to use for the task. Either this option
or the global ``osbuild_target`` is required.
* ``version`` -- version for the final build (as a string). This option is
required if the global ``osbuild_version`` is not specified.
* ``release`` -- release part of the final NVR. If neither this option nor
the global ``osbuild_release`` is set, Koji will automatically generate a
value.
* ``repo`` -- a list of repository URLs from which to consume packages for
building the image. By default only the variant repository is used.
* ``arches`` -- list of architectures for which to build the image. By
default, the variant arches are used. This option can only restrict it,
not add a new one.
.. note::
There is initial support for having this task as failable without aborting
the whole compose. This can be enabled by setting ``"failable": ["*"]`` in
the config for the image. It is an on/off switch without granularity per
arch.
OSTree Settings
===============

31
pungi/checks.py
View File

@ -1111,6 +1111,37 @@ def make_schema():
},
"additionalProperties": False,
},
"osbuild_target": {"type": "string"},
"osbuild_release": {"$ref": "#/definitions/optional_string"},
"osbuild_version": {"type": "string"},
"osbuild": {
"type": "object",
"patternProperties": {
# Warning: this pattern is a variant uid regex, but the
# format does not let us validate it as there is no regular
# expression to describe all regular expressions.
".+": {
"type": "array",
"items": {
"type": "object",
"properties": {
"name": {"type": "string"},
"version": {"type": "string"},
"distro": {"type": "string"},
"target": {"type": "string"},
"image_type": {"$ref": "#/definitions/strings"},
"arches": {"$ref": "#/definitions/list_of_strings"},
"release": {"type": "string"},
"repo": {"$ref": "#/definitions/list_of_strings"},
"failable": {"$ref": "#/definitions/list_of_strings"},
},
"subvariant": {"type": "string"},
},
"required": ["name", "distro", "image_type"],
"additionalProperties": False,
},
},
},
"lorax_options": _variant_arch_mapping(
{
"type": "object",

1
pungi/phases/__init__.py
View File

@ -27,6 +27,7 @@ from .createiso import CreateisoPhase # noqa
from .extra_isos import ExtraIsosPhase # noqa
from .live_images import LiveImagesPhase # noqa
from .image_build import ImageBuildPhase # noqa
from .osbuild import OSBuildPhase # noqa
from .repoclosure import RepoclosurePhase # noqa
from .test import TestPhase # noqa
from .image_checksum import ImageChecksumPhase # noqa

214
pungi/phases/osbuild.py Normal file
View File

@ -0,0 +1,214 @@
# -*- coding: utf-8 -*-
import os
import re
from kobo.threads import ThreadPool, WorkerThread
from kobo import shortcuts
from productmd.images import Image
from . import base
from .. import util
from ..linker import Linker
from ..wrappers import kojiwrapper
from .image_build import EXTENSIONS
class OSBuildPhase(
base.PhaseLoggerMixin, base.ImageConfigMixin, base.ConfigGuardedPhase
):
name = "osbuild"
def __init__(self, compose):
super(OSBuildPhase, self).__init__(compose)
self.pool = ThreadPool(logger=self.logger)
def _get_arches(self, image_conf, arches):
"""Get an intersection of arches in the config dict and the given ones."""
if "arches" in image_conf:
arches = set(image_conf["arches"]) & arches
return sorted(arches)
def _get_repo(self, image_conf, variant):
"""
Get a list of repos. First included are those explicitly listed in
config, followed by by repo for current variant if it's not included in
the list already.
"""
repos = shortcuts.force_list(image_conf.get("repo", []))
if not variant.is_empty and variant.uid not in repos:
repos.append(variant.uid)
return util.get_repo_urls(self.compose, repos, arch="$arch")
def run(self):
for variant in self.compose.get_variants():
arches = set([x for x in variant.arches if x != "src"])
for image_conf in self.get_config_block(variant):
build_arches = self._get_arches(image_conf, arches)
if not build_arches:
self.log_debug("skip: no arches")
continue
release = self.get_release(image_conf)
version = self.get_version(image_conf)
target = self.get_config(image_conf, "target")
repo = self._get_repo(image_conf, variant)
can_fail = image_conf.pop("failable", [])
if can_fail == ["*"]:
can_fail = image_conf["arches"]
if can_fail:
can_fail = sorted(can_fail)
self.pool.add(RunOSBuildThread(self.pool))
self.pool.queue_put(
(
self.compose,
variant,
image_conf,
build_arches,
version,
release,
target,
repo,
can_fail,
)
)
self.pool.start()
class RunOSBuildThread(WorkerThread):
def process(self, item, num):
(
compose,
variant,
config,
arches,
version,
release,
target,
repo,
can_fail,
) = item
self.can_fail = can_fail
self.num = num
with util.failable(
compose,
bool(config.get("failable")),
variant,
"*",
"osbuild",
logger=self.pool._logger,
):
self.worker(
compose, variant, config, arches, version, release, target, repo
)
def worker(self, compose, variant, config, arches, version, release, target, repo):
msg = "OSBuild task for variant %s" % variant.uid
self.pool.log_info("[BEGIN] %s" % msg)
koji = kojiwrapper.KojiWrapper(compose.conf["koji_profile"])
koji.login()
# Start task
opts = {"repo": repo, "release": release}
task_id = koji.koji_proxy.osbuildImage(
config["name"],
version,
config["distro"],
config["image_types"],
target,
arches,
opts=opts,
)
# Wait for it to finish and capture the output into log file.
log_dir = os.path.join(compose.paths.log.topdir(), "osbuild")
util.makedirs(log_dir)
log_file = os.path.join(
log_dir, "%s-%s-watch-task.log" % (variant.uid, self.num)
)
if koji.watch_task(task_id, log_file) != 0:
raise RuntimeError(
"OSBuild: task %s failed: see %s for details" % (task_id, log_file)
)
# Parse NVR from the task output. If release part of NVR was generated
# by Koji, we don't have enough information in the configuration.
nvr = get_nvr(log_file)
# Refresh koji session which may have timed out while the task was
# running. Watching is done via a subprocess, so the session is
# inactive.
koji = kojiwrapper.KojiWrapper(compose.conf["koji_profile"])
linker = Linker(logger=self.pool._logger)
# Process all images in the build. There should be one for each
# architecture, but we don't verify that.
build_info = koji.koji_proxy.getBuild(nvr)
for archive in koji.koji_proxy.listArchives(buildID=build_info["build_id"]):
if archive["type_name"] not in config["image_types"]:
# Ignore values that are not of required types.
continue
# Get architecture of the image from extra data.
try:
arch = archive["extra"]["image"]["arch"]
except KeyError:
raise RuntimeError("Image doesn't have any architecture!")
# image_dir is absolute path to which the image should be copied.
# We also need the same path as relative to compose directory for
# including in the metadata.
image_dir = compose.paths.compose.image_dir(variant) % {"arch": arch}
rel_image_dir = compose.paths.compose.image_dir(variant, relative=True) % {
"arch": arch
}
util.makedirs(image_dir)
image_dest = os.path.join(image_dir, archive["filename"])
src_file = os.path.join(
koji.koji_module.pathinfo.imagebuild(build_info), archive["filename"]
)
linker.link(src_file, image_dest, link_type=compose.conf["link_type"])
suffix = archive["filename"].rsplit(".", 1)[-1]
if suffix not in EXTENSIONS[archive["type_name"]]:
raise RuntimeError(
"Failed to generate metadata. Format %s doesn't match type %s"
% (suffix, archive["type_name"])
)
# Update image manifest
img = Image(compose.im)
img.type = archive["type_name"]
img.format = suffix
img.path = os.path.join(rel_image_dir, archive["filename"])
img.mtime = util.get_mtime(image_dest)
img.size = util.get_file_size(image_dest)
img.arch = arch
img.disc_number = 1 # We don't expect multiple disks
img.disc_count = 1
img.bootable = False
img.subvariant = config.get("subvariant", variant.uid)
setattr(img, "can_fail", self.can_fail)
setattr(img, "deliverable", "image-build")
compose.im.add(variant=variant.uid, arch=arch, image=img)
self.pool.log_info("[DONE ] %s (task id: %s)" % (msg, task_id))
def get_nvr(log_file):
with open(log_file) as f:
for line in f:
match = re.search("Creating compose: ([^ ]+) ", line)
if match:
return match.group(1)
raise RuntimeError("Failed to find image NVR in the output")

4
pungi/scripts/pungi_koji.py
View File

@ -374,6 +374,7 @@ def run_compose(
liveimages_phase = pungi.phases.LiveImagesPhase(compose)
livemedia_phase = pungi.phases.LiveMediaPhase(compose)
image_build_phase = pungi.phases.ImageBuildPhase(compose)
osbuild_phase = pungi.phases.OSBuildPhase(compose)
osbs_phase = pungi.phases.OSBSPhase(compose)
image_checksum_phase = pungi.phases.ImageChecksumPhase(compose)
repoclosure_phase = pungi.phases.RepoclosurePhase(compose)
@ -397,6 +398,7 @@ def run_compose(
ostree_installer_phase,
extra_isos_phase,
osbs_phase,
osbuild_phase,
):
if phase.skip():
continue
@ -494,6 +496,7 @@ def run_compose(
liveimages_phase,
image_build_phase,
livemedia_phase,
osbuild_phase,
)
compose_images_phase = pungi.phases.WeaverPhase(compose, compose_images_schema)
extra_phase_schema = (
@ -514,6 +517,7 @@ def run_compose(
and liveimages_phase.skip()
and livemedia_phase.skip()
and image_build_phase.skip()
and osbuild_phase.skip()
):
compose.im.dump(compose.paths.compose.metadata("images.json"))
osbs_phase.dump_metadata()

289
tests/test_osbuild_phase.py Normal file
View File

@ -0,0 +1,289 @@
# -*- coding: utf-8 -*-
import mock
import os
import koji as orig_koji
from tests import helpers
from pungi.phases import osbuild
class OSBuildPhaseTest(helpers.PungiTestCase):
@mock.patch("pungi.phases.osbuild.ThreadPool")
def test_run(self, ThreadPool):
cfg = {
"name": "test-image",
"version": "1",
"target": "image-target",
"arches": ["x86_64"],
"failable": ["x86_64"],
}
compose = helpers.DummyCompose(
self.topdir, {"osbuild": {"^Everything$": [cfg]}}
)
pool = ThreadPool.return_value
phase = osbuild.OSBuildPhase(compose)
phase.run()
self.assertEqual(len(pool.add.call_args_list), 1)
self.assertEqual(
pool.queue_put.call_args_list,
[
mock.call(
(
compose,
compose.variants["Everything"],
cfg,
["x86_64"],
"1",
None,
"image-target",
[self.topdir + "/compose/Everything/$arch/os"],
["x86_64"],
),
),
],
)
@mock.patch("pungi.phases.osbuild.ThreadPool")
def test_run_with_global_options(self, ThreadPool):
cfg = {"name": "test-image"}
compose = helpers.DummyCompose(
self.topdir,
{
"osbuild": {"^Everything$": [cfg]},
"osbuild_target": "image-target",
"osbuild_version": "1",
"osbuild_release": "2",
},
)
pool = ThreadPool.return_value
phase = osbuild.OSBuildPhase(compose)
phase.run()
self.assertEqual(len(pool.add.call_args_list), 1)
self.assertEqual(
pool.queue_put.call_args_list,
[
mock.call(
(
compose,
compose.variants["Everything"],
cfg,
sorted(compose.variants["Everything"].arches),
"1",
"2",
"image-target",
[self.topdir + "/compose/Everything/$arch/os"],
[],
),
),
],
)
@mock.patch("pungi.phases.osbuild.ThreadPool")
def test_skip_without_config(self, ThreadPool):
compose = helpers.DummyCompose(self.topdir, {})
compose.just_phases = None
compose.skip_phases = []
phase = osbuild.OSBuildPhase(compose)
self.assertTrue(phase.skip())
class RunOSBuildThreadTest(helpers.PungiTestCase):
def setUp(self):
super(RunOSBuildThreadTest, self).setUp()
self.pool = mock.Mock()
self.t = osbuild.RunOSBuildThread(self.pool)
self.compose = helpers.DummyCompose(
self.topdir,
{
"koji_profile": "koji",
"translate_paths": [(self.topdir, "http://root")],
},
)
def make_fake_watch(self, retval):
def inner(task_id, log_file):
with open(log_file, "w") as f:
f.write("Creating compose: test-image-1-1 1234\n")
return retval
return inner
@mock.patch("pungi.util.get_file_size", new=lambda fp: 65536)
@mock.patch("pungi.util.get_mtime", new=lambda fp: 1024)
@mock.patch("pungi.phases.osbuild.Linker")
@mock.patch("pungi.phases.osbuild.kojiwrapper.KojiWrapper")
def test_process(self, KojiWrapper, Linker):
cfg = {"name": "test-image", "distro": "rhel8", "image_types": ["qcow2"]}
koji = KojiWrapper.return_value
koji.watch_task.side_effect = self.make_fake_watch(0)
koji.koji_proxy.osbuildImage.return_value = 1234
koji.koji_proxy.getBuild.return_value = {
"build_id": 5678,
"name": "test-image",
"version": "1",
"release": "1",
}
koji.koji_proxy.listArchives.return_value = [
{
"extra": {"image": {"arch": "aarch64"}},
"filename": "disk.aarch64.qcow2",
"type_name": "qcow2",
},
{
"extra": {"image": {"arch": "x86_64"}},
"filename": "disk.x86_64.qcow2",
"type_name": "qcow2",
},
]
koji.koji_module.pathinfo = orig_koji.pathinfo
self.t.process(
(
self.compose,
self.compose.variants["Everything"],
cfg,
["aarch64", "x86_64"],
"1",
None,
"image-target",
[self.topdir + "/compose/Everything/$arch/os"],
["x86_64"],
),
1,
)
# Verify two Koji instances were created.
self.assertEqual(len(KojiWrapper.call_args), 2)
# Verify correct calls to Koji
self.assertEqual(
koji.mock_calls,
[
mock.call.login(),
mock.call.koji_proxy.osbuildImage(
"test-image",
"1",
"rhel8",
["qcow2"],
"image-target",
["aarch64", "x86_64"],
opts={
"release": None,
"repo": [self.topdir + "/compose/Everything/$arch/os"],
},
),
mock.call.watch_task(1234, mock.ANY),
mock.call.koji_proxy.getBuild("test-image-1-1"),
mock.call.koji_proxy.listArchives(buildID=5678),
],
)
# Assert there are 2 images added to manifest and the arguments are sane
self.assertEqual(
self.compose.im.add.call_args_list,
[
mock.call(arch="aarch64", variant="Everything", image=mock.ANY),
mock.call(arch="x86_64", variant="Everything", image=mock.ANY),
],
)
for call in self.compose.im.add.call_args_list:
_, kwargs = call
image = kwargs["image"]
self.assertEqual(kwargs["variant"], "Everything")
self.assertIn(kwargs["arch"], ("aarch64", "x86_64"))
self.assertEqual(kwargs["arch"], image.arch)
self.assertEqual(
"Everything/%(arch)s/images/disk.%(arch)s.qcow2" % {"arch": image.arch},
image.path,
)
self.assertEqual("qcow2", image.format)
self.assertEqual("qcow2", image.type)
self.assertEqual("Everything", image.subvariant)
self.assertTrue(
os.path.isdir(self.topdir + "/compose/Everything/aarch64/images")
)
self.assertTrue(
os.path.isdir(self.topdir + "/compose/Everything/x86_64/images")
)
self.assertEqual(
Linker.return_value.mock_calls,
[
mock.call.link(
"/mnt/koji/packages/test-image/1/1/images/disk.%(arch)s.qcow2"
% {"arch": arch},
self.topdir
+ "/compose/Everything/%(arch)s/images/disk.%(arch)s.qcow2"
% {"arch": arch},
link_type="hardlink-or-copy",
)
for arch in ["aarch64", "x86_64"]
],
)
@mock.patch("pungi.phases.osbuild.kojiwrapper.KojiWrapper")
def test_task_fails(self, KojiWrapper):
cfg = {"name": "test-image", "distro": "rhel8", "image_types": ["qcow2"]}
koji = KojiWrapper.return_value
koji.watch_task.side_effect = self.make_fake_watch(1)
koji.koji_proxy.osbuildImage.return_value = 1234
with self.assertRaises(RuntimeError):
self.t.process(
(
self.compose,
self.compose.variants["Everything"],
cfg,
["aarch64", "x86_64"],
"1",
None,
"image-target",
[self.topdir + "/compose/Everything/$arch/os"],
["x86_64"],
),
1,
)
@mock.patch("pungi.phases.osbuild.kojiwrapper.KojiWrapper")
def test_task_fails_but_is_failable(self, KojiWrapper):
cfg = {
"name": "test-image",
"distro": "rhel8",
"image_types": ["qcow2"],
"failable": ["x86_65"],
}
koji = KojiWrapper.return_value
koji.watch_task.side_effect = self.make_fake_watch(1)
koji.koji_proxy.osbuildImage.return_value = 1234
self.t.process(
(
self.compose,
self.compose.variants["Everything"],
cfg,
["aarch64", "x86_64"],
"1",
None,
"image-target",
[self.topdir + "/compose/Everything/$arch/os"],
["x86_64"],
),
1,
)
self.assertFalse(
os.path.isdir(self.topdir + "/compose/Everything/aarch64/images")
)
self.assertFalse(
os.path.isdir(self.topdir + "/compose/Everything/x86_64/images")
)
self.assertEqual(len(self.compose.im.add.call_args_list), 0)