doc/_static/phases.svg

@@ -12,7 +12,7 @@
    viewBox="0 0 610.46457 301.1662"
    id="svg2"
    version="1.1"
-   inkscape:version="1.0.1 (3bc2e813f5, 2020-09-07)"
+   inkscape:version="1.0.2 (e86c870879, 2021-01-15)"
    sodipodi:docname="phases.svg"
    inkscape:export-filename="/home/lsedlar/repos/pungi/doc/_static/phases.png"
    inkscape:export-xdpi="90"
@@ -24,9 +24,9 @@
      borderopacity="1.0"
      inkscape:pageopacity="1"
      inkscape:pageshadow="2"
-     inkscape:zoom="2.1213203"
+     inkscape:zoom="1.5"
-     inkscape:cx="276.65806"
+     inkscape:cx="9.4746397"
-     inkscape:cy="189.24198"
+     inkscape:cy="58.833855"
      inkscape:document-units="px"
      inkscape:current-layer="layer1"
      showgrid="false"
@@ -70,7 +70,7 @@
         <dc:format>image/svg+xml</dc:format>
         <dc:type
            rdf:resource="http://purl.org/dc/dcmitype/StillImage" />
-        <dc:title></dc:title>
+        <dc:title />
       </cc:Work>
     </rdf:RDF>
   </metadata>
@@ -303,15 +303,15 @@
     </g>
     <rect
        transform="matrix(0,1,1,0,0,0)"
-       style="fill:#e9b96e;fill-rule:evenodd;stroke:none;stroke-width:2.65937px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
+       style="fill:#e9b96e;fill-rule:evenodd;stroke:none;stroke-width:1.85901px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
        id="rect3338-1"
-       width="185.96895"
+       width="90.874992"
        height="115.80065"
        x="872.67383"
        y="486.55563" />
     <text
        id="text3384-0"
-       y="969.2854"
+       y="921.73846"
        x="489.56451"
        style="font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;line-height:0%;font-family:sans-serif;-inkscape-font-specification:'sans-serif, Normal';text-align:start;letter-spacing:0px;word-spacing:0px;writing-mode:lr-tb;text-anchor:start;fill:#000000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
        xml:space="preserve"><tspan
@@ -319,7 +319,7 @@
          id="tspan3391"
          sodipodi:role="line"
          x="489.56451"
-         y="969.2854">ImageChecksum</tspan></text>
+         y="921.73846">ImageChecksum</tspan></text>
     <g
        transform="translate(-42.209584,-80.817124)"
        id="g3458">
@@ -518,5 +518,24 @@
            id="tspan301-5"
            style="font-size:12px;line-height:0">OSBuild</tspan></text>
     </g>
+    <rect
+       transform="matrix(0,1,1,0,0,0)"
+       style="fill:#729fcf;fill-rule:evenodd;stroke:none;stroke-width:1.83502px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
+       id="rect3338-1-3"
+       width="88.544876"
+       height="115.80065"
+       x="970.31763"
+       y="486.55563" />
+    <text
+       id="text3384-0-6"
+       y="1018.2172"
+       x="489.56451"
+       style="font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;line-height:0%;font-family:sans-serif;-inkscape-font-specification:'sans-serif, Normal';text-align:start;letter-spacing:0px;word-spacing:0px;writing-mode:lr-tb;text-anchor:start;fill:#000000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
+       xml:space="preserve"><tspan
+         style="font-size:13.1475px;line-height:1.25"
+         id="tspan3391-7"
+         sodipodi:role="line"
+         x="489.56451"
+         y="1018.2172">ImageContainer</tspan></text>
   </g>
 </svg>

doc/configuration.rst

@@ -1555,6 +1555,56 @@ OSBuild Composer for building images
    arch.
 
 
+Image container
+===============
+
+This phase supports building containers in OSBS that embed an image created in
+the same compose. This can be useful for delivering the image to users running
+in containerized environments.
+
+Pungi will start a ``buildContainer`` task in Koji with configured source
+repository. The ``Dockerfile`` can expect that a repo file will be injected
+into the container that defines a repo named ``image-to-include``, and its
+``baseurl`` will point to the image to include. It is possible to extract the
+URL with a command like ``dnf config-manager --dump image-to-include | awk
+'/baseurl =/{print $3}'```
+
+**image_container**
+    (*dict*) -- configuration for building containers embedding an image.
+
+    Format: ``{variant_uid_regex: [{...}]}``.
+
+    The inner object will define a single container. These keys are required:
+
+    * ``url``, ``target``, ``git_branch``. See OSBS section for definition of
+      these.
+    * ``image_spec`` -- (*object*) A string mapping of filters used to select
+      the image to embed. All images listed in metadata for the variant will be
+      processed. The keys of this filter are used to select metadata fields for
+      the image, and values are regular expression that need to match the
+      metadata value.
+
+      The filter should match exactly one image.
+
+
+Example config
+--------------
+::
+
+    image_container = {
+        "^Server$": [{
+            "url": "git://example.com/dockerfiles.git?#HEAD",
+            "target": "f24-container-candidate",
+            "git_branch": "f24",
+            "image_spec": {
+                "format": "qcow2",
+                "arch": "x86_64",
+                "path": ".*/guest-image-.*$",
+            }
+        }]
+    }
+
+
 OSTree Settings
 ===============
 

doc/phases.rst

@@ -115,16 +115,30 @@ ImageBuild
 This phase wraps up ``koji image-build``. It also updates the metadata
 ultimately responsible for ``images.json`` manifest.
 
+OSBuild
+-------
+
+Similarly to image build, this phases creates a koji `osbuild` task. In the
+background it uses OSBuild Composer to create images.
+
 OSBS
 ----
 
-This phase builds docker base images in `OSBS
+This phase builds container base images in `OSBS
 <http://osbs.readthedocs.io/en/latest/index.html>`_.
 
 The finished images are available in registry provided by OSBS, but not
 downloaded directly into the compose. The is metadata about the created image
 in ``compose/metadata/osbs.json``.
 
+ImageContainer
+--------------
+
+This phase builds a container image in OSBS, and stores the metadata in the
+same file as OSBS phase. The container produced here wraps a different image,
+created it ImageBuild or OSBuild phase. It can be useful to deliver a VM image
+to containerized environments.
+
 OSTreeInstaller
 ---------------
 

pungi/checks.py

@@ -1213,6 +1213,26 @@ def make_schema():
                 },
                 "additionalProperties": False,
             },
+            "image_container": {
+                "type": "object",
+                "patternProperties": {
+                    ".+": _one_or_list(
+                        {
+                            "type": "object",
+                            "properties": {
+                                "url": {"type": "url"},
+                                "target": {"type": "string"},
+                                "priority": {"type": "number"},
+                                "failable": {"type": "boolean"},
+                                "git_branch": {"type": "string"},
+                                "image_spec": {"type": "object"},
+                            },
+                            "required": ["url", "target", "git_branch", "image_spec"],
+                        }
+                    ),
+                },
+                "additionalProperties": False,
+            },
             "extra_files": _variant_arch_mapping(
                 {
                     "type": "array",

pungi/phases/__init__.py

@@ -27,6 +27,7 @@ from .createiso import CreateisoPhase  # noqa
 from .extra_isos import ExtraIsosPhase  # noqa
 from .live_images import LiveImagesPhase  # noqa
 from .image_build import ImageBuildPhase  # noqa
+from .image_container import ImageContainerPhase  # noqa
 from .osbuild import OSBuildPhase  # noqa
 from .repoclosure import RepoclosurePhase  # noqa
 from .test import TestPhase  # noqa

pungi/phases/image_container.py

@@ -0,0 +1,120 @@
+# -*- coding: utf-8 -*-
+
+import os
+import re
+from kobo.threads import ThreadPool, WorkerThread
+
+from .base import ConfigGuardedPhase, PhaseLoggerMixin
+from .. import util
+from ..wrappers import kojiwrapper
+from ..phases.osbs import add_metadata
+
+
+class ImageContainerPhase(PhaseLoggerMixin, ConfigGuardedPhase):
+    name = "image_container"
+
+    def __init__(self, compose):
+        super(ImageContainerPhase, self).__init__(compose)
+        self.pool = ThreadPool(logger=self.logger)
+        self.pool.metadata = {}
+
+    def run(self):
+        for variant in self.compose.get_variants():
+            for conf in self.get_config_block(variant):
+                self.pool.add(ImageContainerThread(self.pool))
+                self.pool.queue_put((self.compose, variant, conf))
+
+        self.pool.start()
+
+
+class ImageContainerThread(WorkerThread):
+    def process(self, item, num):
+        compose, variant, config = item
+        self.num = num
+        with util.failable(
+            compose,
+            bool(config.pop("failable", None)),
+            variant,
+            "*",
+            "osbs",
+            logger=self.pool._logger,
+        ):
+            self.worker(compose, variant, config)
+
+    def worker(self, compose, variant, config):
+        msg = "Image container task for variant %s" % variant.uid
+        self.pool.log_info("[BEGIN] %s" % msg)
+
+        source = config.pop("url")
+        target = config.pop("target")
+        priority = config.pop("priority", None)
+
+        config["yum_repourls"] = [
+            self._get_repo(
+                compose,
+                variant,
+                config.get("arch_override", "").split(" "),
+                config.pop("image_spec"),
+            )
+        ]
+
+        # Start task
+        koji = kojiwrapper.KojiWrapper(compose.conf["koji_profile"])
+        koji.login()
+        task_id = koji.koji_proxy.buildContainer(
+            source, target, config, priority=priority
+        )
+
+        # Wait for it to finish and capture the output into log file (even
+        # though there is not much there).
+        log_dir = os.path.join(compose.paths.log.topdir(), "image_container")
+        util.makedirs(log_dir)
+        log_file = os.path.join(
+            log_dir, "%s-%s-watch-task.log" % (variant.uid, self.num)
+        )
+        if koji.watch_task(task_id, log_file) != 0:
+            raise RuntimeError(
+                "ImageContainer: task %s failed: see %s for details"
+                % (task_id, log_file)
+            )
+
+        add_metadata(variant, task_id, compose, config.get("scratch", False))
+
+        self.pool.log_info("[DONE ] %s" % msg)
+
+    def _get_repo(self, compose, variant, arches, image_spec):
+        """
+        Return a repo file that points baseurl to the image specified by
+        image_spec.
+        """
+        image_paths = set()
+
+        for arch in arches or compose.im.images[variant.uid].keys():
+            for image in compose.im.images[variant.uid].get(arch, []):
+                for key, value in image_spec.items():
+                    if not re.match(value, getattr(image, key)):
+                        break
+                else:
+                    image_paths.add(image.path)
+
+        if len(image_paths) != 1:
+            raise RuntimeError(
+                "%d images matched specification. Only one was expected."
+                % len(image_paths)
+            )
+
+        image_path = image_paths.pop()
+        absolute_path = os.path.join(compose.paths.compose.topdir(), image_path)
+
+        repo_file = os.path.join(
+            compose.paths.work.tmp_dir(None, variant),
+            "image-container-%s-%s.repo" % (variant, self.num),
+        )
+        with open(repo_file, "w") as f:
+            f.write("[image-to-include]\n")
+            f.write("name=Location of image to embed\n")
+            f.write("baseurl=%s\n" % util.translate_path(compose, absolute_path))
+            f.write("enabled=0\n")
+            f.write("gpgcheck=0\n")
+
+        return util.translate_path(compose, repo_file)

pungi/scripts/pungi_koji.py

@@ -394,6 +394,7 @@ def run_compose(
     image_build_phase = pungi.phases.ImageBuildPhase(compose)
     osbuild_phase = pungi.phases.OSBuildPhase(compose)
     osbs_phase = pungi.phases.OSBSPhase(compose)
+    image_container_phase = pungi.phases.ImageContainerPhase(compose)
     image_checksum_phase = pungi.phases.ImageChecksumPhase(compose)
     repoclosure_phase = pungi.phases.RepoclosurePhase(compose)
     test_phase = pungi.phases.TestPhase(compose)
@@ -417,6 +418,7 @@ def run_compose(
         extra_isos_phase,
         osbs_phase,
         osbuild_phase,
+        image_container_phase,
     ):
         if phase.skip():
             continue
@@ -516,9 +518,12 @@ def run_compose(
         livemedia_phase,
         osbuild_phase,
     )
+    post_image_phase = pungi.phases.WeaverPhase(
+        compose, (image_checksum_phase, image_container_phase)
+    )
     compose_images_phase = pungi.phases.WeaverPhase(compose, compose_images_schema)
     extra_phase_schema = (
-        (compose_images_phase, image_checksum_phase),
+        (compose_images_phase, post_image_phase),
         osbs_phase,
         repoclosure_phase,
     )

tests/test_image_container_phase.py

@@ -0,0 +1,262 @@
+# -*- coding: utf-8 -*-
+
+import mock
+
+import os
+
+from tests import helpers
+from pungi import checks
+from pungi.phases import image_container
+
+
+class ImageContainerPhaseTest(helpers.PungiTestCase):
+    @mock.patch("pungi.phases.image_container.ThreadPool")
+    def test_run(self, ThreadPool):
+        cfg = helpers.IterableMock()
+        compose = helpers.DummyCompose(
+            self.topdir, {"image_container": {"^Everything$": cfg}}
+        )
+
+        pool = ThreadPool.return_value
+
+        phase = image_container.ImageContainerPhase(compose)
+        phase.run()
+
+        self.assertEqual(len(pool.add.call_args_list), 1)
+        self.assertEqual(
+            pool.queue_put.call_args_list,
+            [mock.call((compose, compose.variants["Everything"], cfg))],
+        )
+
+    @mock.patch("pungi.phases.image_container.ThreadPool")
+    def test_skip_without_config(self, ThreadPool):
+        compose = helpers.DummyCompose(self.topdir, {})
+        compose.just_phases = None
+        compose.skip_phases = []
+        phase = image_container.ImageContainerPhase(compose)
+        self.assertTrue(phase.skip())
+
+
+class ImageContainerConfigTest(helpers.PungiTestCase):
+    def assertConfigMissing(self, cfg, key):
+        conf = helpers.load_config(
+            helpers.PKGSET_REPOS, **{"image_container": {"^Server$": cfg}}
+        )
+        errors, warnings = checks.validate(conf, offline=True)
+        self.assertIn(
+            "Failed validation in image_container.^Server$: %r is not valid under any of the given schemas"  # noqa: E501
+            % cfg,
+            errors,
+        )
+        self.assertIn("    Possible reason: %r is a required property" % key, errors)
+        self.assertEqual([], warnings)
+
+    def test_correct(self):
+        conf = helpers.load_config(
+            helpers.PKGSET_REPOS,
+            **{
+                "image_container": {
+                    "^Server$": [
+                        {
+                            "url": "http://example.com/repo.git#HEAD",
+                            "target": "container-candidate",
+                            "git_branch": "main",
+                            "image_spec": {"type": "qcow2"},
+                        }
+                    ]
+                }
+            }
+        )
+        errors, warnings = checks.validate(conf, offline=True)
+        self.assertEqual([], errors)
+        self.assertEqual([], warnings)
+
+    def test_missing_url(self):
+        self.assertConfigMissing(
+            {
+                "target": "container-candidate",
+                "git_branch": "main",
+                "image_spec": {"type": "qcow2"},
+            },
+            "url",
+        )
+
+    def test_missing_target(self):
+        self.assertConfigMissing(
+            {
+                "url": "http://example.com/repo.git#HEAD",
+                "git_branch": "main",
+                "image_spec": {"type": "qcow2"},
+            },
+            "target",
+        )
+
+    def test_missing_git_branch(self):
+        self.assertConfigMissing(
+            {
+                "url": "http://example.com/repo.git#HEAD",
+                "target": "container-candidate",
+                "image_spec": {"type": "qcow2"},
+            },
+            "git_branch",
+        )
+
+    def test_missing_image_spec(self):
+        self.assertConfigMissing(
+            {
+                "url": "http://example.com/repo.git#HEAD",
+                "target": "container-candidate",
+                "git_branch": "main",
+            },
+            "image_spec",
+        )
+
+
+class ImageContainerThreadTest(helpers.PungiTestCase):
+    def setUp(self):
+        super(ImageContainerThreadTest, self).setUp()
+        self.pool = mock.Mock()
+        self.repofile_path = "work/global/tmp-Server/image-container-Server-1.repo"
+        self.t = image_container.ImageContainerThread(self.pool)
+        self.compose = helpers.DummyCompose(
+            self.topdir,
+            {
+                "koji_profile": "koji",
+                "translate_paths": [(self.topdir, "http://root")],
+            },
+        )
+        self.cfg = {
+            "url": "git://example.com/repo?#BEEFCAFE",
+            "target": "f24-docker-candidate",
+            "git_branch": "f24-docker",
+            "image_spec": {"type": "qcow2"},
+            "arch_override": "x86_64",
+        }
+        self.compose.im.images["Server"] = {
+            "x86_64": [
+                mock.Mock(path="Server/x86_64/iso/image.iso", type="iso"),
+                mock.Mock(path="Server/x86_64/images/image.qcow2", type="qcow2"),
+            ]
+        }
+
+    def _setupMock(self, KojiWrapper):
+        self.wrapper = KojiWrapper.return_value
+        self.wrapper.koji_proxy.buildContainer.return_value = 12345
+        self.wrapper.watch_task.return_value = 0
+
+    def assertRepoFile(self):
+        repofile = os.path.join(self.topdir, self.repofile_path)
+        with open(repofile) as f:
+            repo_content = list(f)
+        self.assertIn("[image-to-include]\n", repo_content)
+        self.assertIn(
+            "baseurl=http://root/compose/Server/x86_64/images/image.qcow2\n",
+            repo_content,
+        )
+        self.assertIn("enabled=0\n", repo_content)
+
+    def assertKojiCalls(self, cfg, scratch=False):
+        opts = {
+            "git_branch": cfg["git_branch"],
+            "arch_override": cfg["arch_override"],
+            "yum_repourls": ["http://root/" + self.repofile_path],
+        }
+        if scratch:
+            opts["scratch"] = True
+        self.assertEqual(
+            self.wrapper.mock_calls,
+            [
+                mock.call.login(),
+                mock.call.koji_proxy.buildContainer(
+                    cfg["url"], cfg["target"], opts, priority=None,
+                ),
+                mock.call.watch_task(
+                    12345,
+                    os.path.join(
+                        self.topdir,
+                        "logs/global/image_container/Server-1-watch-task.log",
+                    ),
+                ),
+            ],
+        )
+
+    @mock.patch("pungi.phases.image_container.add_metadata")
+    @mock.patch("pungi.phases.image_container.kojiwrapper.KojiWrapper")
+    def test_success(self, KojiWrapper, add_metadata):
+        self._setupMock(KojiWrapper)
+
+        self.t.process(
+            (self.compose, self.compose.variants["Server"], self.cfg.copy()), 1
+        )
+
+        self.assertRepoFile()
+        self.assertKojiCalls(self.cfg)
+        self.assertEqual(
+            add_metadata.call_args_list,
+            [mock.call(self.compose.variants["Server"], 12345, self.compose, False)],
+        )
+
+    @mock.patch("pungi.phases.image_container.add_metadata")
+    @mock.patch("pungi.phases.image_container.kojiwrapper.KojiWrapper")
+    def test_scratch_build(self, KojiWrapper, add_metadata):
+        self.cfg["scratch"] = True
+        self._setupMock(KojiWrapper)
+
+        self.t.process(
+            (self.compose, self.compose.variants["Server"], self.cfg.copy()), 1
+        )
+
+        self.assertRepoFile()
+        self.assertKojiCalls(self.cfg, scratch=True)
+        self.assertEqual(
+            add_metadata.call_args_list,
+            [mock.call(self.compose.variants["Server"], 12345, self.compose, True)],
+        )
+
+    @mock.patch("pungi.phases.image_container.add_metadata")
+    @mock.patch("pungi.phases.image_container.kojiwrapper.KojiWrapper")
+    def test_task_fail(self, KojiWrapper, add_metadata):
+        self._setupMock(KojiWrapper)
+        self.wrapper.watch_task.return_value = 1
+
+        with self.assertRaises(RuntimeError) as ctx:
+            self.t.process(
+                (self.compose, self.compose.variants["Server"], self.cfg.copy()), 1
+            )
+
+        self.assertRegex(str(ctx.exception), r"task 12345 failed: see .+ for details")
+        self.assertRepoFile()
+        self.assertKojiCalls(self.cfg)
+        self.assertEqual(add_metadata.call_args_list, [])
+
+    @mock.patch("pungi.phases.image_container.add_metadata")
+    @mock.patch("pungi.phases.image_container.kojiwrapper.KojiWrapper")
+    def test_task_fail_failable(self, KojiWrapper, add_metadata):
+        self.cfg["failable"] = "*"
+        self._setupMock(KojiWrapper)
+        self.wrapper.watch_task.return_value = 1
+
+        self.t.process(
+            (self.compose, self.compose.variants["Server"], self.cfg.copy()), 1
+        )
+
+        self.assertRepoFile()
+        self.assertKojiCalls(self.cfg)
+        self.assertEqual(add_metadata.call_args_list, [])
+
+    @mock.patch("pungi.phases.image_container.add_metadata")
+    @mock.patch("pungi.phases.image_container.kojiwrapper.KojiWrapper")
+    def test_non_unique_spec(self, KojiWrapper, add_metadata):
+        self.cfg["image_spec"] = {"path": ".*/image\\..*"}
+        self._setupMock(KojiWrapper)
+
+        with self.assertRaises(RuntimeError) as ctx:
+            self.t.process(
+                (self.compose, self.compose.variants["Server"], self.cfg.copy()), 1
+            )
+
+        self.assertRegex(
+            str(ctx.exception), "2 images matched specification. Only one was expected."
+        )
+        self.assertEqual(self.wrapper.mock_calls, [])
+        self.assertEqual(add_metadata.call_args_list, [])