diff --git a/doc/_static/phases.svg b/doc/_static/phases.svg index 5083e3b7..b973798a 100644 --- a/doc/_static/phases.svg +++ b/doc/_static/phases.svg @@ -12,7 +12,7 @@ viewBox="0 0 610.46457 301.1662" id="svg2" version="1.1" - inkscape:version="1.0.1 (3bc2e813f5, 2020-09-07)" + inkscape:version="1.0.2 (e86c870879, 2021-01-15)" sodipodi:docname="phases.svg" inkscape:export-filename="/home/lsedlar/repos/pungi/doc/_static/phases.png" inkscape:export-xdpi="90" @@ -24,9 +24,9 @@ borderopacity="1.0" inkscape:pageopacity="1" inkscape:pageshadow="2" - inkscape:zoom="2.1213203" - inkscape:cx="276.65806" - inkscape:cy="189.24198" + inkscape:zoom="1.5" + inkscape:cx="9.4746397" + inkscape:cy="58.833855" inkscape:document-units="px" inkscape:current-layer="layer1" showgrid="false" @@ -70,7 +70,7 @@ image/svg+xml - + @@ -303,15 +303,15 @@ ImageChecksum + y="921.73846">ImageChecksum @@ -518,5 +518,24 @@ id="tspan301-5" style="font-size:12px;line-height:0">OSBuild + + ImageContainer diff --git a/doc/configuration.rst b/doc/configuration.rst index d1175baa..810af973 100644 --- a/doc/configuration.rst +++ b/doc/configuration.rst @@ -1555,6 +1555,56 @@ OSBuild Composer for building images arch. +Image container +=============== + +This phase supports building containers in OSBS that embed an image created in +the same compose. This can be useful for delivering the image to users running +in containerized environments. + +Pungi will start a ``buildContainer`` task in Koji with configured source +repository. The ``Dockerfile`` can expect that a repo file will be injected +into the container that defines a repo named ``image-to-include``, and its +``baseurl`` will point to the image to include. It is possible to extract the +URL with a command like ``dnf config-manager --dump image-to-include | awk +'/baseurl =/{print $3}'``` + +**image_container** + (*dict*) -- configuration for building containers embedding an image. + + Format: ``{variant_uid_regex: [{...}]}``. + + The inner object will define a single container. These keys are required: + + * ``url``, ``target``, ``git_branch``. See OSBS section for definition of + these. + * ``image_spec`` -- (*object*) A string mapping of filters used to select + the image to embed. All images listed in metadata for the variant will be + processed. The keys of this filter are used to select metadata fields for + the image, and values are regular expression that need to match the + metadata value. + + The filter should match exactly one image. + + +Example config +-------------- +:: + + image_container = { + "^Server$": [{ + "url": "git://example.com/dockerfiles.git?#HEAD", + "target": "f24-container-candidate", + "git_branch": "f24", + "image_spec": { + "format": "qcow2", + "arch": "x86_64", + "path": ".*/guest-image-.*$", + } + }] + } + + OSTree Settings =============== diff --git a/doc/phases.rst b/doc/phases.rst index 2cb810a8..7ae5bcdc 100644 --- a/doc/phases.rst +++ b/doc/phases.rst @@ -115,16 +115,30 @@ ImageBuild This phase wraps up ``koji image-build``. It also updates the metadata ultimately responsible for ``images.json`` manifest. +OSBuild +------- + +Similarly to image build, this phases creates a koji `osbuild` task. In the +background it uses OSBuild Composer to create images. + OSBS ---- -This phase builds docker base images in `OSBS +This phase builds container base images in `OSBS `_. The finished images are available in registry provided by OSBS, but not downloaded directly into the compose. The is metadata about the created image in ``compose/metadata/osbs.json``. +ImageContainer +-------------- + +This phase builds a container image in OSBS, and stores the metadata in the +same file as OSBS phase. The container produced here wraps a different image, +created it ImageBuild or OSBuild phase. It can be useful to deliver a VM image +to containerized environments. + OSTreeInstaller --------------- diff --git a/pungi/checks.py b/pungi/checks.py index 94a40b77..cfefdfb2 100644 --- a/pungi/checks.py +++ b/pungi/checks.py @@ -1213,6 +1213,26 @@ def make_schema(): }, "additionalProperties": False, }, + "image_container": { + "type": "object", + "patternProperties": { + ".+": _one_or_list( + { + "type": "object", + "properties": { + "url": {"type": "url"}, + "target": {"type": "string"}, + "priority": {"type": "number"}, + "failable": {"type": "boolean"}, + "git_branch": {"type": "string"}, + "image_spec": {"type": "object"}, + }, + "required": ["url", "target", "git_branch", "image_spec"], + } + ), + }, + "additionalProperties": False, + }, "extra_files": _variant_arch_mapping( { "type": "array", diff --git a/pungi/phases/__init__.py b/pungi/phases/__init__.py index 7b28e4e5..3e124548 100644 --- a/pungi/phases/__init__.py +++ b/pungi/phases/__init__.py @@ -27,6 +27,7 @@ from .createiso import CreateisoPhase # noqa from .extra_isos import ExtraIsosPhase # noqa from .live_images import LiveImagesPhase # noqa from .image_build import ImageBuildPhase # noqa +from .image_container import ImageContainerPhase # noqa from .osbuild import OSBuildPhase # noqa from .repoclosure import RepoclosurePhase # noqa from .test import TestPhase # noqa diff --git a/pungi/phases/image_container.py b/pungi/phases/image_container.py new file mode 100644 index 00000000..cb72161f --- /dev/null +++ b/pungi/phases/image_container.py @@ -0,0 +1,120 @@ +# -*- coding: utf-8 -*- + +import os +import re +from kobo.threads import ThreadPool, WorkerThread + +from .base import ConfigGuardedPhase, PhaseLoggerMixin +from .. import util +from ..wrappers import kojiwrapper +from ..phases.osbs import add_metadata + + +class ImageContainerPhase(PhaseLoggerMixin, ConfigGuardedPhase): + name = "image_container" + + def __init__(self, compose): + super(ImageContainerPhase, self).__init__(compose) + self.pool = ThreadPool(logger=self.logger) + self.pool.metadata = {} + + def run(self): + for variant in self.compose.get_variants(): + for conf in self.get_config_block(variant): + self.pool.add(ImageContainerThread(self.pool)) + self.pool.queue_put((self.compose, variant, conf)) + + self.pool.start() + + +class ImageContainerThread(WorkerThread): + def process(self, item, num): + compose, variant, config = item + self.num = num + with util.failable( + compose, + bool(config.pop("failable", None)), + variant, + "*", + "osbs", + logger=self.pool._logger, + ): + self.worker(compose, variant, config) + + def worker(self, compose, variant, config): + msg = "Image container task for variant %s" % variant.uid + self.pool.log_info("[BEGIN] %s" % msg) + + source = config.pop("url") + target = config.pop("target") + priority = config.pop("priority", None) + + config["yum_repourls"] = [ + self._get_repo( + compose, + variant, + config.get("arch_override", "").split(" "), + config.pop("image_spec"), + ) + ] + + # Start task + koji = kojiwrapper.KojiWrapper(compose.conf["koji_profile"]) + koji.login() + task_id = koji.koji_proxy.buildContainer( + source, target, config, priority=priority + ) + + # Wait for it to finish and capture the output into log file (even + # though there is not much there). + log_dir = os.path.join(compose.paths.log.topdir(), "image_container") + util.makedirs(log_dir) + log_file = os.path.join( + log_dir, "%s-%s-watch-task.log" % (variant.uid, self.num) + ) + if koji.watch_task(task_id, log_file) != 0: + raise RuntimeError( + "ImageContainer: task %s failed: see %s for details" + % (task_id, log_file) + ) + + add_metadata(variant, task_id, compose, config.get("scratch", False)) + + self.pool.log_info("[DONE ] %s" % msg) + + def _get_repo(self, compose, variant, arches, image_spec): + """ + Return a repo file that points baseurl to the image specified by + image_spec. + """ + image_paths = set() + + for arch in arches or compose.im.images[variant.uid].keys(): + for image in compose.im.images[variant.uid].get(arch, []): + for key, value in image_spec.items(): + if not re.match(value, getattr(image, key)): + break + else: + image_paths.add(image.path) + + if len(image_paths) != 1: + raise RuntimeError( + "%d images matched specification. Only one was expected." + % len(image_paths) + ) + + image_path = image_paths.pop() + absolute_path = os.path.join(compose.paths.compose.topdir(), image_path) + + repo_file = os.path.join( + compose.paths.work.tmp_dir(None, variant), + "image-container-%s-%s.repo" % (variant, self.num), + ) + with open(repo_file, "w") as f: + f.write("[image-to-include]\n") + f.write("name=Location of image to embed\n") + f.write("baseurl=%s\n" % util.translate_path(compose, absolute_path)) + f.write("enabled=0\n") + f.write("gpgcheck=0\n") + + return util.translate_path(compose, repo_file) diff --git a/pungi/scripts/pungi_koji.py b/pungi/scripts/pungi_koji.py index d0e53bb8..54763e3b 100644 --- a/pungi/scripts/pungi_koji.py +++ b/pungi/scripts/pungi_koji.py @@ -394,6 +394,7 @@ def run_compose( image_build_phase = pungi.phases.ImageBuildPhase(compose) osbuild_phase = pungi.phases.OSBuildPhase(compose) osbs_phase = pungi.phases.OSBSPhase(compose) + image_container_phase = pungi.phases.ImageContainerPhase(compose) image_checksum_phase = pungi.phases.ImageChecksumPhase(compose) repoclosure_phase = pungi.phases.RepoclosurePhase(compose) test_phase = pungi.phases.TestPhase(compose) @@ -417,6 +418,7 @@ def run_compose( extra_isos_phase, osbs_phase, osbuild_phase, + image_container_phase, ): if phase.skip(): continue @@ -516,9 +518,12 @@ def run_compose( livemedia_phase, osbuild_phase, ) + post_image_phase = pungi.phases.WeaverPhase( + compose, (image_checksum_phase, image_container_phase) + ) compose_images_phase = pungi.phases.WeaverPhase(compose, compose_images_schema) extra_phase_schema = ( - (compose_images_phase, image_checksum_phase), + (compose_images_phase, post_image_phase), osbs_phase, repoclosure_phase, ) diff --git a/tests/test_image_container_phase.py b/tests/test_image_container_phase.py new file mode 100644 index 00000000..65745b2d --- /dev/null +++ b/tests/test_image_container_phase.py @@ -0,0 +1,262 @@ +# -*- coding: utf-8 -*- + +import mock + +import os + +from tests import helpers +from pungi import checks +from pungi.phases import image_container + + +class ImageContainerPhaseTest(helpers.PungiTestCase): + @mock.patch("pungi.phases.image_container.ThreadPool") + def test_run(self, ThreadPool): + cfg = helpers.IterableMock() + compose = helpers.DummyCompose( + self.topdir, {"image_container": {"^Everything$": cfg}} + ) + + pool = ThreadPool.return_value + + phase = image_container.ImageContainerPhase(compose) + phase.run() + + self.assertEqual(len(pool.add.call_args_list), 1) + self.assertEqual( + pool.queue_put.call_args_list, + [mock.call((compose, compose.variants["Everything"], cfg))], + ) + + @mock.patch("pungi.phases.image_container.ThreadPool") + def test_skip_without_config(self, ThreadPool): + compose = helpers.DummyCompose(self.topdir, {}) + compose.just_phases = None + compose.skip_phases = [] + phase = image_container.ImageContainerPhase(compose) + self.assertTrue(phase.skip()) + + +class ImageContainerConfigTest(helpers.PungiTestCase): + def assertConfigMissing(self, cfg, key): + conf = helpers.load_config( + helpers.PKGSET_REPOS, **{"image_container": {"^Server$": cfg}} + ) + errors, warnings = checks.validate(conf, offline=True) + self.assertIn( + "Failed validation in image_container.^Server$: %r is not valid under any of the given schemas" # noqa: E501 + % cfg, + errors, + ) + self.assertIn(" Possible reason: %r is a required property" % key, errors) + self.assertEqual([], warnings) + + def test_correct(self): + conf = helpers.load_config( + helpers.PKGSET_REPOS, + **{ + "image_container": { + "^Server$": [ + { + "url": "http://example.com/repo.git#HEAD", + "target": "container-candidate", + "git_branch": "main", + "image_spec": {"type": "qcow2"}, + } + ] + } + } + ) + errors, warnings = checks.validate(conf, offline=True) + self.assertEqual([], errors) + self.assertEqual([], warnings) + + def test_missing_url(self): + self.assertConfigMissing( + { + "target": "container-candidate", + "git_branch": "main", + "image_spec": {"type": "qcow2"}, + }, + "url", + ) + + def test_missing_target(self): + self.assertConfigMissing( + { + "url": "http://example.com/repo.git#HEAD", + "git_branch": "main", + "image_spec": {"type": "qcow2"}, + }, + "target", + ) + + def test_missing_git_branch(self): + self.assertConfigMissing( + { + "url": "http://example.com/repo.git#HEAD", + "target": "container-candidate", + "image_spec": {"type": "qcow2"}, + }, + "git_branch", + ) + + def test_missing_image_spec(self): + self.assertConfigMissing( + { + "url": "http://example.com/repo.git#HEAD", + "target": "container-candidate", + "git_branch": "main", + }, + "image_spec", + ) + + +class ImageContainerThreadTest(helpers.PungiTestCase): + def setUp(self): + super(ImageContainerThreadTest, self).setUp() + self.pool = mock.Mock() + self.repofile_path = "work/global/tmp-Server/image-container-Server-1.repo" + self.t = image_container.ImageContainerThread(self.pool) + self.compose = helpers.DummyCompose( + self.topdir, + { + "koji_profile": "koji", + "translate_paths": [(self.topdir, "http://root")], + }, + ) + self.cfg = { + "url": "git://example.com/repo?#BEEFCAFE", + "target": "f24-docker-candidate", + "git_branch": "f24-docker", + "image_spec": {"type": "qcow2"}, + "arch_override": "x86_64", + } + self.compose.im.images["Server"] = { + "x86_64": [ + mock.Mock(path="Server/x86_64/iso/image.iso", type="iso"), + mock.Mock(path="Server/x86_64/images/image.qcow2", type="qcow2"), + ] + } + + def _setupMock(self, KojiWrapper): + self.wrapper = KojiWrapper.return_value + self.wrapper.koji_proxy.buildContainer.return_value = 12345 + self.wrapper.watch_task.return_value = 0 + + def assertRepoFile(self): + repofile = os.path.join(self.topdir, self.repofile_path) + with open(repofile) as f: + repo_content = list(f) + self.assertIn("[image-to-include]\n", repo_content) + self.assertIn( + "baseurl=http://root/compose/Server/x86_64/images/image.qcow2\n", + repo_content, + ) + self.assertIn("enabled=0\n", repo_content) + + def assertKojiCalls(self, cfg, scratch=False): + opts = { + "git_branch": cfg["git_branch"], + "arch_override": cfg["arch_override"], + "yum_repourls": ["http://root/" + self.repofile_path], + } + if scratch: + opts["scratch"] = True + self.assertEqual( + self.wrapper.mock_calls, + [ + mock.call.login(), + mock.call.koji_proxy.buildContainer( + cfg["url"], cfg["target"], opts, priority=None, + ), + mock.call.watch_task( + 12345, + os.path.join( + self.topdir, + "logs/global/image_container/Server-1-watch-task.log", + ), + ), + ], + ) + + @mock.patch("pungi.phases.image_container.add_metadata") + @mock.patch("pungi.phases.image_container.kojiwrapper.KojiWrapper") + def test_success(self, KojiWrapper, add_metadata): + self._setupMock(KojiWrapper) + + self.t.process( + (self.compose, self.compose.variants["Server"], self.cfg.copy()), 1 + ) + + self.assertRepoFile() + self.assertKojiCalls(self.cfg) + self.assertEqual( + add_metadata.call_args_list, + [mock.call(self.compose.variants["Server"], 12345, self.compose, False)], + ) + + @mock.patch("pungi.phases.image_container.add_metadata") + @mock.patch("pungi.phases.image_container.kojiwrapper.KojiWrapper") + def test_scratch_build(self, KojiWrapper, add_metadata): + self.cfg["scratch"] = True + self._setupMock(KojiWrapper) + + self.t.process( + (self.compose, self.compose.variants["Server"], self.cfg.copy()), 1 + ) + + self.assertRepoFile() + self.assertKojiCalls(self.cfg, scratch=True) + self.assertEqual( + add_metadata.call_args_list, + [mock.call(self.compose.variants["Server"], 12345, self.compose, True)], + ) + + @mock.patch("pungi.phases.image_container.add_metadata") + @mock.patch("pungi.phases.image_container.kojiwrapper.KojiWrapper") + def test_task_fail(self, KojiWrapper, add_metadata): + self._setupMock(KojiWrapper) + self.wrapper.watch_task.return_value = 1 + + with self.assertRaises(RuntimeError) as ctx: + self.t.process( + (self.compose, self.compose.variants["Server"], self.cfg.copy()), 1 + ) + + self.assertRegex(str(ctx.exception), r"task 12345 failed: see .+ for details") + self.assertRepoFile() + self.assertKojiCalls(self.cfg) + self.assertEqual(add_metadata.call_args_list, []) + + @mock.patch("pungi.phases.image_container.add_metadata") + @mock.patch("pungi.phases.image_container.kojiwrapper.KojiWrapper") + def test_task_fail_failable(self, KojiWrapper, add_metadata): + self.cfg["failable"] = "*" + self._setupMock(KojiWrapper) + self.wrapper.watch_task.return_value = 1 + + self.t.process( + (self.compose, self.compose.variants["Server"], self.cfg.copy()), 1 + ) + + self.assertRepoFile() + self.assertKojiCalls(self.cfg) + self.assertEqual(add_metadata.call_args_list, []) + + @mock.patch("pungi.phases.image_container.add_metadata") + @mock.patch("pungi.phases.image_container.kojiwrapper.KojiWrapper") + def test_non_unique_spec(self, KojiWrapper, add_metadata): + self.cfg["image_spec"] = {"path": ".*/image\\..*"} + self._setupMock(KojiWrapper) + + with self.assertRaises(RuntimeError) as ctx: + self.t.process( + (self.compose, self.compose.variants["Server"], self.cfg.copy()), 1 + ) + + self.assertRegex( + str(ctx.exception), "2 images matched specification. Only one was expected." + ) + self.assertEqual(self.wrapper.mock_calls, []) + self.assertEqual(add_metadata.call_args_list, [])