diff --git a/doc/configuration.rst b/doc/configuration.rst index 73af4fb4..e89dc6ee 100644 --- a/doc/configuration.rst +++ b/doc/configuration.rst @@ -1305,6 +1305,7 @@ Target is specified by these settings. * ``live_media_target`` * ``image_build_target`` * ``live_images_target`` + * ``osbuild_target`` Version is specified by these options. If no version is set, a default value will be provided according to :ref:`automatic versioning `. @@ -1313,6 +1314,7 @@ will be provided according to :ref:`automatic versioning `. * ``live_media_version`` * ``image_build_version`` * ``live_images_version`` + * ``osbuild_version`` Release is specified by these options. If set to a magic value to ``!RELEASE_FROM_LABEL_DATE_TYPE_RESPIN``, a value will be generated according @@ -1322,6 +1324,7 @@ to :ref:`automatic versioning `. * ``live_media_release`` * ``image_build_release`` * ``live_images_release`` + * ``osbuild_release`` Each configuration block can also optionally specify a ``failable`` key. For live images it should have a boolean value. For live media and image build it @@ -1512,6 +1515,45 @@ Example } +OSBuild Composer for building images +==================================== + +**osbuild** + (*dict*) -- configuration for building images in OSBuild Composer service + fronted by a Koji plugin. Pungi will trigger a Koji task delegating to the + OSBuild Composer, which will build the image, import it to Koji via content + generators. + + Format: ``{variant_uid_regex: [{...}]}``. + + Required keys in the configuration dict: + + * ``name`` -- name of the Koji package + * ``distro`` -- image for which distribution should be build TODO examples + * ``image_type`` -- a list of image types to build (e.g. ``qcow2``) + + Optional keys: + + * ``target`` -- which build target to use for the task. Either this option + or the global ``osbuild_target`` is required. + * ``version`` -- version for the final build (as a string). This option is + required if the global ``osbuild_version`` is not specified. + * ``release`` -- release part of the final NVR. If neither this option nor + the global ``osbuild_release`` is set, Koji will automatically generate a + value. + * ``repo`` -- a list of repository URLs from which to consume packages for + building the image. By default only the variant repository is used. + * ``arches`` -- list of architectures for which to build the image. By + default, the variant arches are used. This option can only restrict it, + not add a new one. + +.. note:: + There is initial support for having this task as failable without aborting + the whole compose. This can be enabled by setting ``"failable": ["*"]`` in + the config for the image. It is an on/off switch without granularity per + arch. + + OSTree Settings =============== diff --git a/pungi/checks.py b/pungi/checks.py index 6f190c89..9aec2c49 100644 --- a/pungi/checks.py +++ b/pungi/checks.py @@ -1111,6 +1111,37 @@ def make_schema(): }, "additionalProperties": False, }, + "osbuild_target": {"type": "string"}, + "osbuild_release": {"$ref": "#/definitions/optional_string"}, + "osbuild_version": {"type": "string"}, + "osbuild": { + "type": "object", + "patternProperties": { + # Warning: this pattern is a variant uid regex, but the + # format does not let us validate it as there is no regular + # expression to describe all regular expressions. + ".+": { + "type": "array", + "items": { + "type": "object", + "properties": { + "name": {"type": "string"}, + "version": {"type": "string"}, + "distro": {"type": "string"}, + "target": {"type": "string"}, + "image_type": {"$ref": "#/definitions/strings"}, + "arches": {"$ref": "#/definitions/list_of_strings"}, + "release": {"type": "string"}, + "repo": {"$ref": "#/definitions/list_of_strings"}, + "failable": {"$ref": "#/definitions/list_of_strings"}, + }, + "subvariant": {"type": "string"}, + }, + "required": ["name", "distro", "image_type"], + "additionalProperties": False, + }, + }, + }, "lorax_options": _variant_arch_mapping( { "type": "object", diff --git a/pungi/phases/__init__.py b/pungi/phases/__init__.py index d99f8789..7b28e4e5 100644 --- a/pungi/phases/__init__.py +++ b/pungi/phases/__init__.py @@ -27,6 +27,7 @@ from .createiso import CreateisoPhase # noqa from .extra_isos import ExtraIsosPhase # noqa from .live_images import LiveImagesPhase # noqa from .image_build import ImageBuildPhase # noqa +from .osbuild import OSBuildPhase # noqa from .repoclosure import RepoclosurePhase # noqa from .test import TestPhase # noqa from .image_checksum import ImageChecksumPhase # noqa diff --git a/pungi/phases/osbuild.py b/pungi/phases/osbuild.py new file mode 100644 index 00000000..ddd70536 --- /dev/null +++ b/pungi/phases/osbuild.py @@ -0,0 +1,214 @@ +# -*- coding: utf-8 -*- + +import os +import re +from kobo.threads import ThreadPool, WorkerThread +from kobo import shortcuts +from productmd.images import Image + +from . import base +from .. import util +from ..linker import Linker +from ..wrappers import kojiwrapper +from .image_build import EXTENSIONS + + +class OSBuildPhase( + base.PhaseLoggerMixin, base.ImageConfigMixin, base.ConfigGuardedPhase +): + name = "osbuild" + + def __init__(self, compose): + super(OSBuildPhase, self).__init__(compose) + self.pool = ThreadPool(logger=self.logger) + + def _get_arches(self, image_conf, arches): + """Get an intersection of arches in the config dict and the given ones.""" + if "arches" in image_conf: + arches = set(image_conf["arches"]) & arches + return sorted(arches) + + def _get_repo(self, image_conf, variant): + """ + Get a list of repos. First included are those explicitly listed in + config, followed by by repo for current variant if it's not included in + the list already. + """ + repos = shortcuts.force_list(image_conf.get("repo", [])) + + if not variant.is_empty and variant.uid not in repos: + repos.append(variant.uid) + + return util.get_repo_urls(self.compose, repos, arch="$arch") + + def run(self): + for variant in self.compose.get_variants(): + arches = set([x for x in variant.arches if x != "src"]) + + for image_conf in self.get_config_block(variant): + build_arches = self._get_arches(image_conf, arches) + if not build_arches: + self.log_debug("skip: no arches") + continue + + release = self.get_release(image_conf) + version = self.get_version(image_conf) + target = self.get_config(image_conf, "target") + + repo = self._get_repo(image_conf, variant) + + can_fail = image_conf.pop("failable", []) + if can_fail == ["*"]: + can_fail = image_conf["arches"] + if can_fail: + can_fail = sorted(can_fail) + + self.pool.add(RunOSBuildThread(self.pool)) + self.pool.queue_put( + ( + self.compose, + variant, + image_conf, + build_arches, + version, + release, + target, + repo, + can_fail, + ) + ) + + self.pool.start() + + +class RunOSBuildThread(WorkerThread): + def process(self, item, num): + ( + compose, + variant, + config, + arches, + version, + release, + target, + repo, + can_fail, + ) = item + self.can_fail = can_fail + self.num = num + with util.failable( + compose, + bool(config.get("failable")), + variant, + "*", + "osbuild", + logger=self.pool._logger, + ): + self.worker( + compose, variant, config, arches, version, release, target, repo + ) + + def worker(self, compose, variant, config, arches, version, release, target, repo): + msg = "OSBuild task for variant %s" % variant.uid + self.pool.log_info("[BEGIN] %s" % msg) + koji = kojiwrapper.KojiWrapper(compose.conf["koji_profile"]) + koji.login() + + # Start task + opts = {"repo": repo, "release": release} + task_id = koji.koji_proxy.osbuildImage( + config["name"], + version, + config["distro"], + config["image_types"], + target, + arches, + opts=opts, + ) + + # Wait for it to finish and capture the output into log file. + log_dir = os.path.join(compose.paths.log.topdir(), "osbuild") + util.makedirs(log_dir) + log_file = os.path.join( + log_dir, "%s-%s-watch-task.log" % (variant.uid, self.num) + ) + if koji.watch_task(task_id, log_file) != 0: + raise RuntimeError( + "OSBuild: task %s failed: see %s for details" % (task_id, log_file) + ) + + # Parse NVR from the task output. If release part of NVR was generated + # by Koji, we don't have enough information in the configuration. + nvr = get_nvr(log_file) + + # Refresh koji session which may have timed out while the task was + # running. Watching is done via a subprocess, so the session is + # inactive. + koji = kojiwrapper.KojiWrapper(compose.conf["koji_profile"]) + + linker = Linker(logger=self.pool._logger) + + # Process all images in the build. There should be one for each + # architecture, but we don't verify that. + build_info = koji.koji_proxy.getBuild(nvr) + for archive in koji.koji_proxy.listArchives(buildID=build_info["build_id"]): + if archive["type_name"] not in config["image_types"]: + # Ignore values that are not of required types. + continue + + # Get architecture of the image from extra data. + try: + arch = archive["extra"]["image"]["arch"] + except KeyError: + raise RuntimeError("Image doesn't have any architecture!") + + # image_dir is absolute path to which the image should be copied. + # We also need the same path as relative to compose directory for + # including in the metadata. + image_dir = compose.paths.compose.image_dir(variant) % {"arch": arch} + rel_image_dir = compose.paths.compose.image_dir(variant, relative=True) % { + "arch": arch + } + util.makedirs(image_dir) + + image_dest = os.path.join(image_dir, archive["filename"]) + + src_file = os.path.join( + koji.koji_module.pathinfo.imagebuild(build_info), archive["filename"] + ) + + linker.link(src_file, image_dest, link_type=compose.conf["link_type"]) + + suffix = archive["filename"].rsplit(".", 1)[-1] + if suffix not in EXTENSIONS[archive["type_name"]]: + raise RuntimeError( + "Failed to generate metadata. Format %s doesn't match type %s" + % (suffix, archive["type_name"]) + ) + + # Update image manifest + img = Image(compose.im) + img.type = archive["type_name"] + img.format = suffix + img.path = os.path.join(rel_image_dir, archive["filename"]) + img.mtime = util.get_mtime(image_dest) + img.size = util.get_file_size(image_dest) + img.arch = arch + img.disc_number = 1 # We don't expect multiple disks + img.disc_count = 1 + img.bootable = False + img.subvariant = config.get("subvariant", variant.uid) + setattr(img, "can_fail", self.can_fail) + setattr(img, "deliverable", "image-build") + compose.im.add(variant=variant.uid, arch=arch, image=img) + + self.pool.log_info("[DONE ] %s (task id: %s)" % (msg, task_id)) + + +def get_nvr(log_file): + with open(log_file) as f: + for line in f: + match = re.search("Creating compose: ([^ ]+) ", line) + if match: + return match.group(1) + raise RuntimeError("Failed to find image NVR in the output") diff --git a/pungi/scripts/pungi_koji.py b/pungi/scripts/pungi_koji.py index 2ff89e61..6fec647f 100644 --- a/pungi/scripts/pungi_koji.py +++ b/pungi/scripts/pungi_koji.py @@ -374,6 +374,7 @@ def run_compose( liveimages_phase = pungi.phases.LiveImagesPhase(compose) livemedia_phase = pungi.phases.LiveMediaPhase(compose) image_build_phase = pungi.phases.ImageBuildPhase(compose) + osbuild_phase = pungi.phases.OSBuildPhase(compose) osbs_phase = pungi.phases.OSBSPhase(compose) image_checksum_phase = pungi.phases.ImageChecksumPhase(compose) repoclosure_phase = pungi.phases.RepoclosurePhase(compose) @@ -397,6 +398,7 @@ def run_compose( ostree_installer_phase, extra_isos_phase, osbs_phase, + osbuild_phase, ): if phase.skip(): continue @@ -494,6 +496,7 @@ def run_compose( liveimages_phase, image_build_phase, livemedia_phase, + osbuild_phase, ) compose_images_phase = pungi.phases.WeaverPhase(compose, compose_images_schema) extra_phase_schema = ( @@ -514,6 +517,7 @@ def run_compose( and liveimages_phase.skip() and livemedia_phase.skip() and image_build_phase.skip() + and osbuild_phase.skip() ): compose.im.dump(compose.paths.compose.metadata("images.json")) osbs_phase.dump_metadata() diff --git a/tests/test_osbuild_phase.py b/tests/test_osbuild_phase.py new file mode 100644 index 00000000..9fd04b4d --- /dev/null +++ b/tests/test_osbuild_phase.py @@ -0,0 +1,289 @@ +# -*- coding: utf-8 -*- + +import mock + +import os + +import koji as orig_koji + +from tests import helpers +from pungi.phases import osbuild + + +class OSBuildPhaseTest(helpers.PungiTestCase): + @mock.patch("pungi.phases.osbuild.ThreadPool") + def test_run(self, ThreadPool): + cfg = { + "name": "test-image", + "version": "1", + "target": "image-target", + "arches": ["x86_64"], + "failable": ["x86_64"], + } + compose = helpers.DummyCompose( + self.topdir, {"osbuild": {"^Everything$": [cfg]}} + ) + + pool = ThreadPool.return_value + + phase = osbuild.OSBuildPhase(compose) + phase.run() + + self.assertEqual(len(pool.add.call_args_list), 1) + self.assertEqual( + pool.queue_put.call_args_list, + [ + mock.call( + ( + compose, + compose.variants["Everything"], + cfg, + ["x86_64"], + "1", + None, + "image-target", + [self.topdir + "/compose/Everything/$arch/os"], + ["x86_64"], + ), + ), + ], + ) + + @mock.patch("pungi.phases.osbuild.ThreadPool") + def test_run_with_global_options(self, ThreadPool): + cfg = {"name": "test-image"} + compose = helpers.DummyCompose( + self.topdir, + { + "osbuild": {"^Everything$": [cfg]}, + "osbuild_target": "image-target", + "osbuild_version": "1", + "osbuild_release": "2", + }, + ) + + pool = ThreadPool.return_value + + phase = osbuild.OSBuildPhase(compose) + phase.run() + + self.assertEqual(len(pool.add.call_args_list), 1) + self.assertEqual( + pool.queue_put.call_args_list, + [ + mock.call( + ( + compose, + compose.variants["Everything"], + cfg, + sorted(compose.variants["Everything"].arches), + "1", + "2", + "image-target", + [self.topdir + "/compose/Everything/$arch/os"], + [], + ), + ), + ], + ) + + @mock.patch("pungi.phases.osbuild.ThreadPool") + def test_skip_without_config(self, ThreadPool): + compose = helpers.DummyCompose(self.topdir, {}) + compose.just_phases = None + compose.skip_phases = [] + phase = osbuild.OSBuildPhase(compose) + self.assertTrue(phase.skip()) + + +class RunOSBuildThreadTest(helpers.PungiTestCase): + def setUp(self): + super(RunOSBuildThreadTest, self).setUp() + self.pool = mock.Mock() + self.t = osbuild.RunOSBuildThread(self.pool) + self.compose = helpers.DummyCompose( + self.topdir, + { + "koji_profile": "koji", + "translate_paths": [(self.topdir, "http://root")], + }, + ) + + def make_fake_watch(self, retval): + def inner(task_id, log_file): + with open(log_file, "w") as f: + f.write("Creating compose: test-image-1-1 1234\n") + return retval + + return inner + + @mock.patch("pungi.util.get_file_size", new=lambda fp: 65536) + @mock.patch("pungi.util.get_mtime", new=lambda fp: 1024) + @mock.patch("pungi.phases.osbuild.Linker") + @mock.patch("pungi.phases.osbuild.kojiwrapper.KojiWrapper") + def test_process(self, KojiWrapper, Linker): + cfg = {"name": "test-image", "distro": "rhel8", "image_types": ["qcow2"]} + koji = KojiWrapper.return_value + koji.watch_task.side_effect = self.make_fake_watch(0) + koji.koji_proxy.osbuildImage.return_value = 1234 + koji.koji_proxy.getBuild.return_value = { + "build_id": 5678, + "name": "test-image", + "version": "1", + "release": "1", + } + koji.koji_proxy.listArchives.return_value = [ + { + "extra": {"image": {"arch": "aarch64"}}, + "filename": "disk.aarch64.qcow2", + "type_name": "qcow2", + }, + { + "extra": {"image": {"arch": "x86_64"}}, + "filename": "disk.x86_64.qcow2", + "type_name": "qcow2", + }, + ] + koji.koji_module.pathinfo = orig_koji.pathinfo + + self.t.process( + ( + self.compose, + self.compose.variants["Everything"], + cfg, + ["aarch64", "x86_64"], + "1", + None, + "image-target", + [self.topdir + "/compose/Everything/$arch/os"], + ["x86_64"], + ), + 1, + ) + + # Verify two Koji instances were created. + self.assertEqual(len(KojiWrapper.call_args), 2) + # Verify correct calls to Koji + self.assertEqual( + koji.mock_calls, + [ + mock.call.login(), + mock.call.koji_proxy.osbuildImage( + "test-image", + "1", + "rhel8", + ["qcow2"], + "image-target", + ["aarch64", "x86_64"], + opts={ + "release": None, + "repo": [self.topdir + "/compose/Everything/$arch/os"], + }, + ), + mock.call.watch_task(1234, mock.ANY), + mock.call.koji_proxy.getBuild("test-image-1-1"), + mock.call.koji_proxy.listArchives(buildID=5678), + ], + ) + + # Assert there are 2 images added to manifest and the arguments are sane + self.assertEqual( + self.compose.im.add.call_args_list, + [ + mock.call(arch="aarch64", variant="Everything", image=mock.ANY), + mock.call(arch="x86_64", variant="Everything", image=mock.ANY), + ], + ) + for call in self.compose.im.add.call_args_list: + _, kwargs = call + image = kwargs["image"] + self.assertEqual(kwargs["variant"], "Everything") + self.assertIn(kwargs["arch"], ("aarch64", "x86_64")) + self.assertEqual(kwargs["arch"], image.arch) + self.assertEqual( + "Everything/%(arch)s/images/disk.%(arch)s.qcow2" % {"arch": image.arch}, + image.path, + ) + self.assertEqual("qcow2", image.format) + self.assertEqual("qcow2", image.type) + self.assertEqual("Everything", image.subvariant) + + self.assertTrue( + os.path.isdir(self.topdir + "/compose/Everything/aarch64/images") + ) + self.assertTrue( + os.path.isdir(self.topdir + "/compose/Everything/x86_64/images") + ) + + self.assertEqual( + Linker.return_value.mock_calls, + [ + mock.call.link( + "/mnt/koji/packages/test-image/1/1/images/disk.%(arch)s.qcow2" + % {"arch": arch}, + self.topdir + + "/compose/Everything/%(arch)s/images/disk.%(arch)s.qcow2" + % {"arch": arch}, + link_type="hardlink-or-copy", + ) + for arch in ["aarch64", "x86_64"] + ], + ) + + @mock.patch("pungi.phases.osbuild.kojiwrapper.KojiWrapper") + def test_task_fails(self, KojiWrapper): + cfg = {"name": "test-image", "distro": "rhel8", "image_types": ["qcow2"]} + koji = KojiWrapper.return_value + koji.watch_task.side_effect = self.make_fake_watch(1) + koji.koji_proxy.osbuildImage.return_value = 1234 + + with self.assertRaises(RuntimeError): + self.t.process( + ( + self.compose, + self.compose.variants["Everything"], + cfg, + ["aarch64", "x86_64"], + "1", + None, + "image-target", + [self.topdir + "/compose/Everything/$arch/os"], + ["x86_64"], + ), + 1, + ) + + @mock.patch("pungi.phases.osbuild.kojiwrapper.KojiWrapper") + def test_task_fails_but_is_failable(self, KojiWrapper): + cfg = { + "name": "test-image", + "distro": "rhel8", + "image_types": ["qcow2"], + "failable": ["x86_65"], + } + koji = KojiWrapper.return_value + koji.watch_task.side_effect = self.make_fake_watch(1) + koji.koji_proxy.osbuildImage.return_value = 1234 + + self.t.process( + ( + self.compose, + self.compose.variants["Everything"], + cfg, + ["aarch64", "x86_64"], + "1", + None, + "image-target", + [self.topdir + "/compose/Everything/$arch/os"], + ["x86_64"], + ), + 1, + ) + + self.assertFalse( + os.path.isdir(self.topdir + "/compose/Everything/aarch64/images") + ) + self.assertFalse( + os.path.isdir(self.topdir + "/compose/Everything/x86_64/images") + ) + self.assertEqual(len(self.compose.im.add.call_args_list), 0)