Allow using Pungi Koji plugin for ostree phases.

This commits changes `ostree` and `ostree_installer` phases
so they can run with Koji Pungi plugin instead of the plain runroot.

It is similar to `buildinstall` phase running with Koji plugin.

Signed-off-by: Jan Kaluza <jkaluza@redhat.com>

doc/configuration.rst

@@ -1489,7 +1489,6 @@ repository with a new commit.
       reference will not be created.
     * ``ostree_ref`` -- (*str*) To override value ``ref`` from ``treefile``.
 
-
 Example config
 --------------
 ::
@@ -1514,6 +1513,11 @@ Example config
         }
     }
 
+**ostree_use_koji_plugin** = False
+    (*bool*) -- When set to ``True``, the Koji pungi_ostree task will be
+    used to execute rpm-ostree instead of runroot. Use only if the Koji instance
+    has the pungi_ostree plugin installed.
+
 
 Ostree Installer Settings
 =========================
@@ -1566,6 +1570,11 @@ an OSTree repository. This always runs in Koji as a ``runroot`` task.
     With this option it is possible to opt-in for the overwriting. The
     traditional ``boot.iso`` will be in the ``iso/`` subdirectory.
 
+**ostree_installer_use_koji_plugin** = False
+    (*bool*) -- When set to ``True``, the Koji pungi_buildinstall task will be
+    used to execute Lorax instead of runroot. Use only if the Koji instance
+    has the pungi_buildinstall plugin installed.
+
 
 Example config
 --------------

pungi/checks.py

@@ -1032,6 +1032,8 @@ def make_schema():
                     "additionalProperties": False,
                 }
             ),
+            "ostree_use_koji_plugin": {"type": "boolean", "default": False},
+            "ostree_installer_use_koji_plugin": {"type": "boolean", "default": False},
             "ostree_installer_overwrite": {"type": "boolean", "default": False},
             "live_images": _variant_arch_mapping(
                 _one_or_list({"$ref": "#/definitions/live_image_config"})

pungi/phases/ostree.py

@@ -5,6 +5,7 @@ import json
 import os
 from kobo import shortcuts
 from kobo.threads import ThreadPool, WorkerThread
+from collections import OrderedDict
 
 from pungi.arch_utils import getBaseArch
 from pungi.runroot import Runroot
@@ -154,45 +155,49 @@ class OSTreeThread(WorkerThread):
     def _run_ostree_cmd(
         self, compose, variant, arch, config, config_repo, extra_config_file=None
     ):
-        cmd = [
+        args = OrderedDict(
-            "pungi-make-ostree",
+            [
-            "tree",
+                ("repo", config["ostree_repo"]),
-            "--repo=%s" % config["ostree_repo"],
+                ("log-dir", self.logdir),
-            "--log-dir=%s" % self.logdir,
+                ("treefile", os.path.join(config_repo, config["treefile"])),
-            "--treefile=%s" % os.path.join(config_repo, config["treefile"]),
+                ("version", util.version_generator(compose, config.get("version"))),
-        ]
+                ("extra-config", extra_config_file),
-
+                ("update-summary", config.get("update_summary", False)),
-        version = util.version_generator(compose, config.get("version"))
+                ("ostree-ref", config.get("ostree_ref")),
-        if version:
+                ("force-new-commit", config.get("force_new_commit", False)),
-            cmd.append("--version=%s" % version)
+            ]
-
+        )
-        if extra_config_file:
-            cmd.append("--extra-config=%s" % extra_config_file)
-
-        if config.get("update_summary", False):
-            cmd.append("--update-summary")
-
-        ostree_ref = config.get("ostree_ref")
-        if ostree_ref:
-            cmd.append("--ostree-ref=%s" % ostree_ref)
-
-        if config.get("force_new_commit", False):
-            cmd.append("--force-new-commit")
-
         packages = ["pungi", "ostree", "rpm-ostree"]
         log_file = os.path.join(self.logdir, "runroot.log")
         mounts = [compose.topdir, config["ostree_repo"]]
-
         runroot = Runroot(compose, phase="ostree")
-        runroot.run(
+
-            cmd,
+        if compose.conf["ostree_use_koji_plugin"]:
-            log_file=log_file,
+            runroot.run_pungi_ostree(
-            arch=arch,
+                dict(args),
-            packages=packages,
+                log_file=log_file,
-            mounts=mounts,
+                arch=arch,
-            new_chroot=True,
+                packages=packages,
-            weight=compose.conf["runroot_weights"].get("ostree"),
+                mounts=mounts,
-        )
+                weight=compose.conf["runroot_weights"].get("ostree"),
+            )
+        else:
+            cmd = ["pungi-make-ostree", "tree"]
+            for key, value in args.items():
+                if value is True:
+                    cmd.append("--%s" % key)
+                elif value:
+                    cmd.append("--%s=%s" % (key, value))
+
+            runroot.run(
+                cmd,
+                log_file=log_file,
+                arch=arch,
+                packages=packages,
+                mounts=mounts,
+                new_chroot=True,
+                weight=compose.conf["runroot_weights"].get("ostree"),
+            )
 
     def _clone_repo(self, compose, repodir, url, branch):
         scm.get_dir_from_scm(

pungi/phases/ostree_installer.py

@@ -10,7 +10,14 @@ from kobo import shortcuts
 from .base import ConfigGuardedPhase, PhaseLoggerMixin
 from .. import util
 from ..arch import get_valid_arches
-from ..util import get_volid, get_repo_urls, version_generator, translate_path
+from ..util import (
+    get_volid,
+    get_repo_urls,
+    version_generator,
+    translate_path,
+    move_all,
+    makedirs,
+)
 from ..wrappers import iso, lorax, scm
 from ..runroot import Runroot
 
@@ -200,43 +207,83 @@ class OstreeInstallerThread(WorkerThread):
     def _run_ostree_cmd(
         self, compose, variant, arch, config, source_repo, output_dir, volid
     ):
-        lorax_wrapper = lorax.LoraxWrapper()
-        lorax_cmd = lorax_wrapper.get_lorax_cmd(
-            compose.conf["release_name"],
-            compose.conf["release_version"],
-            self._get_release(compose, config),
-            repo_baseurl=source_repo,
-            output_dir=output_dir,
-            variant=variant.uid,
-            nomacboot=True,
-            volid=volid,
-            buildarch=get_valid_arches(arch)[0],
-            buildinstallpackages=config.get("installpkgs"),
-            add_template=self._get_templates(config, "add_template"),
-            add_arch_template=self._get_templates(config, "add_arch_template"),
-            add_template_var=config.get("add_template_var"),
-            add_arch_template_var=config.get("add_arch_template_var"),
-            rootfs_size=config.get("rootfs_size"),
-            is_final=compose.supported,
-            log_dir=self.logdir,
-        )
-        cmd = "rm -rf %s && %s" % (
-            shlex_quote(output_dir),
-            " ".join([shlex_quote(x) for x in lorax_cmd]),
-        )
-
         packages = ["pungi", "lorax", "ostree"]
         packages += config.get("extra_runroot_pkgs", [])
-
         log_file = os.path.join(self.logdir, "runroot.log")
-
         runroot = Runroot(compose, phase="ostree_installer")
-        runroot.run(
+
-            cmd,
+        if compose.conf["ostree_installer_use_koji_plugin"]:
-            log_file=log_file,
+            args = {
-            arch=arch,
+                "product": compose.conf["release_name"],
-            packages=packages,
+                "version": compose.conf["release_version"],
-            mounts=[compose.topdir],
+                "release": self._get_release(compose, config),
-            chown_paths=[output_dir],
+                "sources": shortcuts.force_list(source_repo),
-            weight=compose.conf["runroot_weights"].get("ostree_installer"),
+                "variant": variant.uid,
-        )
+                "nomacboot": True,
+                "volid": volid,
+                "buildarch": get_valid_arches(arch)[0],
+                "installpkgs": config.get("installpkgs"),
+                "add-template": self._get_templates(config, "add_template"),
+                "add-arch-template": self._get_templates(config, "add_arch_template"),
+                "add-template-var": config.get("add_template_var"),
+                "add-arch-template-var": config.get("add_arch_template_var"),
+                "rootfs-size": config.get("rootfs_size"),
+                "isfinal": compose.supported,
+                "outputdir": output_dir,
+            }
+
+            runroot.run_pungi_buildinstall(
+                args,
+                log_file=log_file,
+                arch=arch,
+                packages=packages,
+                mounts=[compose.topdir],
+                weight=compose.conf["runroot_weights"].get("ostree_installer"),
+            )
+
+            # If Koji pungi-buildinstall is used, then the buildinstall results are
+            # not stored directly in `output_dir` dir, but in "results" and "logs"
+            # subdirectories. We need to move them to final_output_dir.
+            results_dir = os.path.join(output_dir, "results")
+            move_all(results_dir, output_dir, rm_src_dir=True)
+
+            # Get the log_dir into which we should copy the resulting log files.
+            if not os.path.exists(self.logdir):
+                makedirs(self.logdir)
+            log_dir = os.path.join(output_dir, "logs")
+            move_all(log_dir, self.logdir, rm_src_dir=True)
+        else:
+            lorax_wrapper = lorax.LoraxWrapper()
+            lorax_cmd = lorax_wrapper.get_lorax_cmd(
+                compose.conf["release_name"],
+                compose.conf["release_version"],
+                self._get_release(compose, config),
+                repo_baseurl=source_repo,
+                output_dir=output_dir,
+                variant=variant.uid,
+                nomacboot=True,
+                volid=volid,
+                buildarch=get_valid_arches(arch)[0],
+                buildinstallpackages=config.get("installpkgs"),
+                add_template=self._get_templates(config, "add_template"),
+                add_arch_template=self._get_templates(config, "add_arch_template"),
+                add_template_var=config.get("add_template_var"),
+                add_arch_template_var=config.get("add_arch_template_var"),
+                rootfs_size=config.get("rootfs_size"),
+                is_final=compose.supported,
+                log_dir=self.logdir,
+            )
+            cmd = "rm -rf %s && %s" % (
+                shlex_quote(output_dir),
+                " ".join([shlex_quote(x) for x in lorax_cmd]),
+            )
+
+            runroot.run(
+                cmd,
+                log_file=log_file,
+                arch=arch,
+                packages=packages,
+                mounts=[compose.topdir],
+                chown_paths=[output_dir],
+                weight=compose.conf["runroot_weights"].get("ostree_installer"),
+            )

pungi/runroot.py

@@ -283,6 +283,35 @@ class Runroot(kobo.log.LoggingBase):
             )
         self._result = output
 
+    def run_pungi_ostree(self, args, log_file=None, arch=None, **kwargs):
+        """
+        Runs the OStree runroot command using the Pungi OSTree
+        Koji plugin as pungi_ostree task.
+
+        The **kwargs are optional and matches the
+        `KojiWrapper.get_pungi_buildinstall_cmd()` kwargs.
+
+        :param dict args: Arguments for the pungi_ostree Koji task.
+        :param str log_file: Log file into which the output of the task will
+            be logged.
+        :param str arch: Architecture on which the task should be executed.
+        """
+        runroot_channel = self.compose.conf.get("runroot_channel")
+        runroot_tag = self.compose.conf["runroot_tag"]
+
+        koji_wrapper = kojiwrapper.KojiWrapper(self.compose.conf["koji_profile"])
+        koji_cmd = koji_wrapper.get_pungi_ostree_cmd(
+            runroot_tag, arch, args, channel=runroot_channel, **kwargs
+        )
+
+        output = koji_wrapper.run_runroot_cmd(koji_cmd, log_file=log_file)
+        if output["retcode"] != 0:
+            raise RuntimeError(
+                "Pungi-buildinstall task failed: %s. See %s for more details."
+                % (output["task_id"], log_file)
+            )
+        self._result = output
+
     def get_buildroot_rpms(self):
         """
         Returns the list of RPMs installed in a buildroot in which the runroot

pungi/wrappers/kojiwrapper.py

@@ -201,6 +201,44 @@ class KojiWrapper(object):
 
         return cmd
 
+    def get_pungi_ostree_cmd(
+        self, target, arch, args, channel=None, packages=None, mounts=None, weight=None,
+    ):
+        cmd = self._get_cmd("pungi-ostree", "--nowait", "--task-id")
+
+        if channel:
+            cmd.append("--channel-override=%s" % channel)
+        else:
+            cmd.append("--channel-override=runroot-local")
+
+        if weight:
+            cmd.append("--weight=%s" % int(weight))
+
+        for package in packages or []:
+            cmd.append("--package=%s" % package)
+
+        for mount in mounts or []:
+            # directories are *not* created here
+            cmd.append("--mount=%s" % mount)
+
+        # IMPORTANT: all --opts have to be provided *before* args
+
+        cmd.append(target)
+
+        # i686 -> i386 etc.
+        arch = getBaseArch(arch)
+        cmd.append(arch)
+
+        for k, v in args.items():
+            if v:
+                if isinstance(v, bool):
+                    cmd.append(k)
+                else:
+                    for arg in force_list(v):
+                        cmd.append("%s=%s" % (k, shlex_quote(arg)))
+
+        return cmd
+
     @contextlib.contextmanager
     def get_koji_cmd_env(self):
         """Get environment variables for running a koji command.

tests/test_ostree_installer_phase.py

@@ -266,6 +266,111 @@ class OstreeThreadTest(helpers.PungiTestCase):
         self.assertImageAdded(self.compose, ImageCls, iso)
         self.assertAllCopied(copy_all)
 
+    @mock.patch("pungi.util.copy_all")
+    @mock.patch("productmd.images.Image")
+    @mock.patch("pungi.util.get_mtime")
+    @mock.patch("pungi.util.get_file_size")
+    @mock.patch("pungi.phases.ostree_installer.iso")
+    @mock.patch("os.link")
+    @mock.patch("pungi.wrappers.kojiwrapper.KojiWrapper")
+    @mock.patch("pungi.phases.ostree_installer.move_all")
+    def test_run_koji_plugin(
+        self,
+        move_all,
+        KojiWrapper,
+        link,
+        iso,
+        get_file_size,
+        get_mtime,
+        ImageCls,
+        copy_all,
+    ):
+        self.compose.supported = False
+        self.compose.conf["ostree_installer_use_koji_plugin"] = True
+        pool = mock.Mock()
+        cfg = {
+            "repo": "Everything",  # this variant-type repo is deprecated, in result will be replaced with default repo  # noqa: E501
+            "release": "20160321.n.0",
+        }
+        koji = KojiWrapper.return_value
+        koji.run_runroot_cmd.return_value = {
+            "task_id": 1234,
+            "retcode": 0,
+            "output": "Foo bar\n",
+        }
+        get_file_size.return_value = 1024
+        get_mtime.return_value = 13579
+        final_iso_path = self.topdir + "/compose/Everything/x86_64/iso/image-name"
+
+        t = ostree.OstreeInstallerThread(pool, ["http://example.com/repo/1"])
+
+        t.process((self.compose, self.compose.variants["Everything"], "x86_64", cfg), 1)
+
+        args = {
+            "product": "Fedora",
+            "version": "Rawhide",
+            "release": "20160321.n.0",
+            "sources": [
+                "http://example.com/repo/1",
+                "http://example.com/work/$basearch/comps_repo_Everything",
+            ],
+            "variant": "Everything",
+            "nomacboot": True,
+            "volid": "test-Everything-x86_64",
+            "buildarch": "x86_64",
+            "installpkgs": None,
+            "add-template": [],
+            "add-arch-template": [],
+            "add-template-var": None,
+            "add-arch-template-var": None,
+            "rootfs-size": None,
+            "isfinal": False,
+            "outputdir": self.topdir + "/work/x86_64/Everything/ostree_installer",
+        }
+        self.assertEqual(
+            koji.get_pungi_buildinstall_cmd.mock_calls,
+            [
+                mock.call(
+                    "rrt",
+                    "x86_64",
+                    args,
+                    channel=None,
+                    packages=["pungi", "lorax", "ostree"],
+                    mounts=[self.topdir],
+                    weight=None,
+                    chown_uid=os.getuid(),
+                )
+            ],
+        )
+        self.assertEqual(
+            koji.run_runroot_cmd.mock_calls,
+            [
+                mock.call(
+                    koji.get_pungi_buildinstall_cmd.return_value,
+                    log_file=os.path.join(self.topdir, LOG_PATH, "runroot.log"),
+                )
+            ],
+        )
+        self.assertEqual(
+            move_all.mock_calls,
+            [
+                mock.call(
+                    self.topdir + "/work/x86_64/Everything/ostree_installer/results",
+                    self.topdir + "/work/x86_64/Everything/ostree_installer",
+                    rm_src_dir=True,
+                ),
+                mock.call(
+                    self.topdir + "/work/x86_64/Everything/ostree_installer/logs",
+                    os.path.join(self.topdir, LOG_PATH),
+                    rm_src_dir=True,
+                ),
+            ],
+        )
+
+        self.assertIsoLinked(link, get_file_size, get_mtime, final_iso_path)
+        self.assertImageAdded(self.compose, ImageCls, iso)
+        self.assertAllCopied(copy_all)
+
     @mock.patch("pungi.util.copy_all")
     @mock.patch("productmd.images.Image")
     @mock.patch("pungi.util.get_mtime")

tests/test_ostree_phase.py

@@ -277,6 +277,78 @@ class OSTreeThreadTest(helpers.PungiTestCase):
         )
         self.assertTrue(os.path.isdir(self.repo))
 
+    @mock.patch("pungi.wrappers.scm.get_dir_from_scm")
+    @mock.patch("pungi.wrappers.kojiwrapper.KojiWrapper")
+    def test_run_use_koji_plugin(self, KojiWrapper, get_dir_from_scm):
+        get_dir_from_scm.side_effect = self._dummy_config_repo
+        self.compose.conf["runroot_weights"] = {"ostree": 123}
+        self.compose.conf["ostree_use_koji_plugin"] = True
+
+        koji = KojiWrapper.return_value
+        koji.run_runroot_cmd.side_effect = self._mock_runroot(0)
+
+        t = ostree.OSTreeThread(self.pool, ["http://example.com/repo/1"])
+
+        t.process(
+            (self.compose, self.compose.variants["Everything"], "x86_64", self.cfg), 1
+        )
+
+        self.assertEqual(
+            get_dir_from_scm.call_args_list,
+            [
+                mock.call(
+                    {
+                        "scm": "git",
+                        "repo": "https://git.fedorahosted.org/git/fedora-atomic.git",
+                        "branch": "f24",
+                        "dir": ".",
+                    },
+                    self.topdir + "/work/ostree-1/config_repo",
+                    compose=self.compose,
+                )
+            ],
+        )
+        self.assertEqual(
+            koji.get_pungi_ostree_cmd.call_args_list,
+            [
+                mock.call(
+                    "rrt",
+                    "x86_64",
+                    {
+                        "repo": self.repo,
+                        "log-dir": "%s/logs/x86_64/Everything/ostree-1" % self.topdir,
+                        "treefile": "%s/fedora-atomic-docker-host.json"
+                        % (self.topdir + "/work/ostree-1/config_repo"),
+                        "extra-config": "%s/extra_config.json"
+                        % (self.topdir + "/work/ostree-1"),
+                        "update-summary": False,
+                        "ostree-ref": None,
+                        "force-new-commit": False,
+                        "version": None,
+                    },
+                    channel=None,
+                    mounts=[self.topdir, self.repo],
+                    packages=["pungi", "ostree", "rpm-ostree"],
+                    weight=123,
+                )
+            ],
+        )
+        self.assertEqual(
+            koji.run_runroot_cmd.call_args_list,
+            [
+                mock.call(
+                    koji.get_pungi_ostree_cmd.return_value,
+                    log_file=self.topdir
+                    + "/logs/x86_64/Everything/ostree-1/runroot.log",
+                )
+            ],
+        )
+
+        self.assertTrue(
+            os.path.isfile(os.path.join(self.topdir, "work/ostree-1/extra_config.json"))
+        )
+        self.assertTrue(os.path.isdir(self.repo))
+
     @mock.patch("pungi.wrappers.scm.get_dir_from_scm")
     @mock.patch("pungi.wrappers.kojiwrapper.KojiWrapper")
     def test_run_fail(self, KojiWrapper, get_dir_from_scm):