From b8c3ca1abeaeffc92768e4728d6520165e57a60a Mon Sep 17 00:00:00 2001 From: Jan Kaluza Date: Wed, 13 May 2020 07:58:07 +0200 Subject: [PATCH] Allow using Pungi Koji plugin for ostree phases. This commits changes `ostree` and `ostree_installer` phases so they can run with Koji Pungi plugin instead of the plain runroot. It is similar to `buildinstall` phase running with Koji plugin. Signed-off-by: Jan Kaluza --- doc/configuration.rst | 11 ++- pungi/checks.py | 2 + pungi/phases/ostree.py | 75 +++++++++-------- pungi/phases/ostree_installer.py | 121 +++++++++++++++++++-------- pungi/runroot.py | 29 +++++++ pungi/wrappers/kojiwrapper.py | 38 +++++++++ tests/test_ostree_installer_phase.py | 105 +++++++++++++++++++++++ tests/test_ostree_phase.py | 72 ++++++++++++++++ 8 files changed, 380 insertions(+), 73 deletions(-) diff --git a/doc/configuration.rst b/doc/configuration.rst index 54532060..21a4e944 100644 --- a/doc/configuration.rst +++ b/doc/configuration.rst @@ -1489,7 +1489,6 @@ repository with a new commit. reference will not be created. * ``ostree_ref`` -- (*str*) To override value ``ref`` from ``treefile``. - Example config -------------- :: @@ -1514,6 +1513,11 @@ Example config } } +**ostree_use_koji_plugin** = False + (*bool*) -- When set to ``True``, the Koji pungi_ostree task will be + used to execute rpm-ostree instead of runroot. Use only if the Koji instance + has the pungi_ostree plugin installed. + Ostree Installer Settings ========================= @@ -1566,6 +1570,11 @@ an OSTree repository. This always runs in Koji as a ``runroot`` task. With this option it is possible to opt-in for the overwriting. The traditional ``boot.iso`` will be in the ``iso/`` subdirectory. +**ostree_installer_use_koji_plugin** = False + (*bool*) -- When set to ``True``, the Koji pungi_buildinstall task will be + used to execute Lorax instead of runroot. Use only if the Koji instance + has the pungi_buildinstall plugin installed. + Example config -------------- diff --git a/pungi/checks.py b/pungi/checks.py index c91760bf..55fd55f6 100644 --- a/pungi/checks.py +++ b/pungi/checks.py @@ -1032,6 +1032,8 @@ def make_schema(): "additionalProperties": False, } ), + "ostree_use_koji_plugin": {"type": "boolean", "default": False}, + "ostree_installer_use_koji_plugin": {"type": "boolean", "default": False}, "ostree_installer_overwrite": {"type": "boolean", "default": False}, "live_images": _variant_arch_mapping( _one_or_list({"$ref": "#/definitions/live_image_config"}) diff --git a/pungi/phases/ostree.py b/pungi/phases/ostree.py index f525439a..2fcfce6c 100644 --- a/pungi/phases/ostree.py +++ b/pungi/phases/ostree.py @@ -5,6 +5,7 @@ import json import os from kobo import shortcuts from kobo.threads import ThreadPool, WorkerThread +from collections import OrderedDict from pungi.arch_utils import getBaseArch from pungi.runroot import Runroot @@ -154,45 +155,49 @@ class OSTreeThread(WorkerThread): def _run_ostree_cmd( self, compose, variant, arch, config, config_repo, extra_config_file=None ): - cmd = [ - "pungi-make-ostree", - "tree", - "--repo=%s" % config["ostree_repo"], - "--log-dir=%s" % self.logdir, - "--treefile=%s" % os.path.join(config_repo, config["treefile"]), - ] - - version = util.version_generator(compose, config.get("version")) - if version: - cmd.append("--version=%s" % version) - - if extra_config_file: - cmd.append("--extra-config=%s" % extra_config_file) - - if config.get("update_summary", False): - cmd.append("--update-summary") - - ostree_ref = config.get("ostree_ref") - if ostree_ref: - cmd.append("--ostree-ref=%s" % ostree_ref) - - if config.get("force_new_commit", False): - cmd.append("--force-new-commit") - + args = OrderedDict( + [ + ("repo", config["ostree_repo"]), + ("log-dir", self.logdir), + ("treefile", os.path.join(config_repo, config["treefile"])), + ("version", util.version_generator(compose, config.get("version"))), + ("extra-config", extra_config_file), + ("update-summary", config.get("update_summary", False)), + ("ostree-ref", config.get("ostree_ref")), + ("force-new-commit", config.get("force_new_commit", False)), + ] + ) packages = ["pungi", "ostree", "rpm-ostree"] log_file = os.path.join(self.logdir, "runroot.log") mounts = [compose.topdir, config["ostree_repo"]] - runroot = Runroot(compose, phase="ostree") - runroot.run( - cmd, - log_file=log_file, - arch=arch, - packages=packages, - mounts=mounts, - new_chroot=True, - weight=compose.conf["runroot_weights"].get("ostree"), - ) + + if compose.conf["ostree_use_koji_plugin"]: + runroot.run_pungi_ostree( + dict(args), + log_file=log_file, + arch=arch, + packages=packages, + mounts=mounts, + weight=compose.conf["runroot_weights"].get("ostree"), + ) + else: + cmd = ["pungi-make-ostree", "tree"] + for key, value in args.items(): + if value is True: + cmd.append("--%s" % key) + elif value: + cmd.append("--%s=%s" % (key, value)) + + runroot.run( + cmd, + log_file=log_file, + arch=arch, + packages=packages, + mounts=mounts, + new_chroot=True, + weight=compose.conf["runroot_weights"].get("ostree"), + ) def _clone_repo(self, compose, repodir, url, branch): scm.get_dir_from_scm( diff --git a/pungi/phases/ostree_installer.py b/pungi/phases/ostree_installer.py index 4b6dbe9d..c1167810 100644 --- a/pungi/phases/ostree_installer.py +++ b/pungi/phases/ostree_installer.py @@ -10,7 +10,14 @@ from kobo import shortcuts from .base import ConfigGuardedPhase, PhaseLoggerMixin from .. import util from ..arch import get_valid_arches -from ..util import get_volid, get_repo_urls, version_generator, translate_path +from ..util import ( + get_volid, + get_repo_urls, + version_generator, + translate_path, + move_all, + makedirs, +) from ..wrappers import iso, lorax, scm from ..runroot import Runroot @@ -200,43 +207,83 @@ class OstreeInstallerThread(WorkerThread): def _run_ostree_cmd( self, compose, variant, arch, config, source_repo, output_dir, volid ): - lorax_wrapper = lorax.LoraxWrapper() - lorax_cmd = lorax_wrapper.get_lorax_cmd( - compose.conf["release_name"], - compose.conf["release_version"], - self._get_release(compose, config), - repo_baseurl=source_repo, - output_dir=output_dir, - variant=variant.uid, - nomacboot=True, - volid=volid, - buildarch=get_valid_arches(arch)[0], - buildinstallpackages=config.get("installpkgs"), - add_template=self._get_templates(config, "add_template"), - add_arch_template=self._get_templates(config, "add_arch_template"), - add_template_var=config.get("add_template_var"), - add_arch_template_var=config.get("add_arch_template_var"), - rootfs_size=config.get("rootfs_size"), - is_final=compose.supported, - log_dir=self.logdir, - ) - cmd = "rm -rf %s && %s" % ( - shlex_quote(output_dir), - " ".join([shlex_quote(x) for x in lorax_cmd]), - ) - packages = ["pungi", "lorax", "ostree"] packages += config.get("extra_runroot_pkgs", []) - log_file = os.path.join(self.logdir, "runroot.log") - runroot = Runroot(compose, phase="ostree_installer") - runroot.run( - cmd, - log_file=log_file, - arch=arch, - packages=packages, - mounts=[compose.topdir], - chown_paths=[output_dir], - weight=compose.conf["runroot_weights"].get("ostree_installer"), - ) + + if compose.conf["ostree_installer_use_koji_plugin"]: + args = { + "product": compose.conf["release_name"], + "version": compose.conf["release_version"], + "release": self._get_release(compose, config), + "sources": shortcuts.force_list(source_repo), + "variant": variant.uid, + "nomacboot": True, + "volid": volid, + "buildarch": get_valid_arches(arch)[0], + "installpkgs": config.get("installpkgs"), + "add-template": self._get_templates(config, "add_template"), + "add-arch-template": self._get_templates(config, "add_arch_template"), + "add-template-var": config.get("add_template_var"), + "add-arch-template-var": config.get("add_arch_template_var"), + "rootfs-size": config.get("rootfs_size"), + "isfinal": compose.supported, + "outputdir": output_dir, + } + + runroot.run_pungi_buildinstall( + args, + log_file=log_file, + arch=arch, + packages=packages, + mounts=[compose.topdir], + weight=compose.conf["runroot_weights"].get("ostree_installer"), + ) + + # If Koji pungi-buildinstall is used, then the buildinstall results are + # not stored directly in `output_dir` dir, but in "results" and "logs" + # subdirectories. We need to move them to final_output_dir. + results_dir = os.path.join(output_dir, "results") + move_all(results_dir, output_dir, rm_src_dir=True) + + # Get the log_dir into which we should copy the resulting log files. + if not os.path.exists(self.logdir): + makedirs(self.logdir) + log_dir = os.path.join(output_dir, "logs") + move_all(log_dir, self.logdir, rm_src_dir=True) + else: + lorax_wrapper = lorax.LoraxWrapper() + lorax_cmd = lorax_wrapper.get_lorax_cmd( + compose.conf["release_name"], + compose.conf["release_version"], + self._get_release(compose, config), + repo_baseurl=source_repo, + output_dir=output_dir, + variant=variant.uid, + nomacboot=True, + volid=volid, + buildarch=get_valid_arches(arch)[0], + buildinstallpackages=config.get("installpkgs"), + add_template=self._get_templates(config, "add_template"), + add_arch_template=self._get_templates(config, "add_arch_template"), + add_template_var=config.get("add_template_var"), + add_arch_template_var=config.get("add_arch_template_var"), + rootfs_size=config.get("rootfs_size"), + is_final=compose.supported, + log_dir=self.logdir, + ) + cmd = "rm -rf %s && %s" % ( + shlex_quote(output_dir), + " ".join([shlex_quote(x) for x in lorax_cmd]), + ) + + runroot.run( + cmd, + log_file=log_file, + arch=arch, + packages=packages, + mounts=[compose.topdir], + chown_paths=[output_dir], + weight=compose.conf["runroot_weights"].get("ostree_installer"), + ) diff --git a/pungi/runroot.py b/pungi/runroot.py index 221029f0..174f822d 100644 --- a/pungi/runroot.py +++ b/pungi/runroot.py @@ -283,6 +283,35 @@ class Runroot(kobo.log.LoggingBase): ) self._result = output + def run_pungi_ostree(self, args, log_file=None, arch=None, **kwargs): + """ + Runs the OStree runroot command using the Pungi OSTree + Koji plugin as pungi_ostree task. + + The **kwargs are optional and matches the + `KojiWrapper.get_pungi_buildinstall_cmd()` kwargs. + + :param dict args: Arguments for the pungi_ostree Koji task. + :param str log_file: Log file into which the output of the task will + be logged. + :param str arch: Architecture on which the task should be executed. + """ + runroot_channel = self.compose.conf.get("runroot_channel") + runroot_tag = self.compose.conf["runroot_tag"] + + koji_wrapper = kojiwrapper.KojiWrapper(self.compose.conf["koji_profile"]) + koji_cmd = koji_wrapper.get_pungi_ostree_cmd( + runroot_tag, arch, args, channel=runroot_channel, **kwargs + ) + + output = koji_wrapper.run_runroot_cmd(koji_cmd, log_file=log_file) + if output["retcode"] != 0: + raise RuntimeError( + "Pungi-buildinstall task failed: %s. See %s for more details." + % (output["task_id"], log_file) + ) + self._result = output + def get_buildroot_rpms(self): """ Returns the list of RPMs installed in a buildroot in which the runroot diff --git a/pungi/wrappers/kojiwrapper.py b/pungi/wrappers/kojiwrapper.py index f71209a5..5b8c4d27 100644 --- a/pungi/wrappers/kojiwrapper.py +++ b/pungi/wrappers/kojiwrapper.py @@ -201,6 +201,44 @@ class KojiWrapper(object): return cmd + def get_pungi_ostree_cmd( + self, target, arch, args, channel=None, packages=None, mounts=None, weight=None, + ): + cmd = self._get_cmd("pungi-ostree", "--nowait", "--task-id") + + if channel: + cmd.append("--channel-override=%s" % channel) + else: + cmd.append("--channel-override=runroot-local") + + if weight: + cmd.append("--weight=%s" % int(weight)) + + for package in packages or []: + cmd.append("--package=%s" % package) + + for mount in mounts or []: + # directories are *not* created here + cmd.append("--mount=%s" % mount) + + # IMPORTANT: all --opts have to be provided *before* args + + cmd.append(target) + + # i686 -> i386 etc. + arch = getBaseArch(arch) + cmd.append(arch) + + for k, v in args.items(): + if v: + if isinstance(v, bool): + cmd.append(k) + else: + for arg in force_list(v): + cmd.append("%s=%s" % (k, shlex_quote(arg))) + + return cmd + @contextlib.contextmanager def get_koji_cmd_env(self): """Get environment variables for running a koji command. diff --git a/tests/test_ostree_installer_phase.py b/tests/test_ostree_installer_phase.py index cabbc3eb..de705754 100644 --- a/tests/test_ostree_installer_phase.py +++ b/tests/test_ostree_installer_phase.py @@ -266,6 +266,111 @@ class OstreeThreadTest(helpers.PungiTestCase): self.assertImageAdded(self.compose, ImageCls, iso) self.assertAllCopied(copy_all) + @mock.patch("pungi.util.copy_all") + @mock.patch("productmd.images.Image") + @mock.patch("pungi.util.get_mtime") + @mock.patch("pungi.util.get_file_size") + @mock.patch("pungi.phases.ostree_installer.iso") + @mock.patch("os.link") + @mock.patch("pungi.wrappers.kojiwrapper.KojiWrapper") + @mock.patch("pungi.phases.ostree_installer.move_all") + def test_run_koji_plugin( + self, + move_all, + KojiWrapper, + link, + iso, + get_file_size, + get_mtime, + ImageCls, + copy_all, + ): + self.compose.supported = False + self.compose.conf["ostree_installer_use_koji_plugin"] = True + pool = mock.Mock() + cfg = { + "repo": "Everything", # this variant-type repo is deprecated, in result will be replaced with default repo # noqa: E501 + "release": "20160321.n.0", + } + koji = KojiWrapper.return_value + koji.run_runroot_cmd.return_value = { + "task_id": 1234, + "retcode": 0, + "output": "Foo bar\n", + } + get_file_size.return_value = 1024 + get_mtime.return_value = 13579 + final_iso_path = self.topdir + "/compose/Everything/x86_64/iso/image-name" + + t = ostree.OstreeInstallerThread(pool, ["http://example.com/repo/1"]) + + t.process((self.compose, self.compose.variants["Everything"], "x86_64", cfg), 1) + + args = { + "product": "Fedora", + "version": "Rawhide", + "release": "20160321.n.0", + "sources": [ + "http://example.com/repo/1", + "http://example.com/work/$basearch/comps_repo_Everything", + ], + "variant": "Everything", + "nomacboot": True, + "volid": "test-Everything-x86_64", + "buildarch": "x86_64", + "installpkgs": None, + "add-template": [], + "add-arch-template": [], + "add-template-var": None, + "add-arch-template-var": None, + "rootfs-size": None, + "isfinal": False, + "outputdir": self.topdir + "/work/x86_64/Everything/ostree_installer", + } + self.assertEqual( + koji.get_pungi_buildinstall_cmd.mock_calls, + [ + mock.call( + "rrt", + "x86_64", + args, + channel=None, + packages=["pungi", "lorax", "ostree"], + mounts=[self.topdir], + weight=None, + chown_uid=os.getuid(), + ) + ], + ) + self.assertEqual( + koji.run_runroot_cmd.mock_calls, + [ + mock.call( + koji.get_pungi_buildinstall_cmd.return_value, + log_file=os.path.join(self.topdir, LOG_PATH, "runroot.log"), + ) + ], + ) + self.assertEqual( + move_all.mock_calls, + [ + mock.call( + self.topdir + "/work/x86_64/Everything/ostree_installer/results", + self.topdir + "/work/x86_64/Everything/ostree_installer", + rm_src_dir=True, + ), + mock.call( + self.topdir + "/work/x86_64/Everything/ostree_installer/logs", + os.path.join(self.topdir, LOG_PATH), + rm_src_dir=True, + ), + ], + ) + + self.assertIsoLinked(link, get_file_size, get_mtime, final_iso_path) + self.assertImageAdded(self.compose, ImageCls, iso) + self.assertAllCopied(copy_all) + @mock.patch("pungi.util.copy_all") @mock.patch("productmd.images.Image") @mock.patch("pungi.util.get_mtime") diff --git a/tests/test_ostree_phase.py b/tests/test_ostree_phase.py index 2b97a332..b214a127 100644 --- a/tests/test_ostree_phase.py +++ b/tests/test_ostree_phase.py @@ -277,6 +277,78 @@ class OSTreeThreadTest(helpers.PungiTestCase): ) self.assertTrue(os.path.isdir(self.repo)) + @mock.patch("pungi.wrappers.scm.get_dir_from_scm") + @mock.patch("pungi.wrappers.kojiwrapper.KojiWrapper") + def test_run_use_koji_plugin(self, KojiWrapper, get_dir_from_scm): + get_dir_from_scm.side_effect = self._dummy_config_repo + self.compose.conf["runroot_weights"] = {"ostree": 123} + self.compose.conf["ostree_use_koji_plugin"] = True + + koji = KojiWrapper.return_value + koji.run_runroot_cmd.side_effect = self._mock_runroot(0) + + t = ostree.OSTreeThread(self.pool, ["http://example.com/repo/1"]) + + t.process( + (self.compose, self.compose.variants["Everything"], "x86_64", self.cfg), 1 + ) + + self.assertEqual( + get_dir_from_scm.call_args_list, + [ + mock.call( + { + "scm": "git", + "repo": "https://git.fedorahosted.org/git/fedora-atomic.git", + "branch": "f24", + "dir": ".", + }, + self.topdir + "/work/ostree-1/config_repo", + compose=self.compose, + ) + ], + ) + self.assertEqual( + koji.get_pungi_ostree_cmd.call_args_list, + [ + mock.call( + "rrt", + "x86_64", + { + "repo": self.repo, + "log-dir": "%s/logs/x86_64/Everything/ostree-1" % self.topdir, + "treefile": "%s/fedora-atomic-docker-host.json" + % (self.topdir + "/work/ostree-1/config_repo"), + "extra-config": "%s/extra_config.json" + % (self.topdir + "/work/ostree-1"), + "update-summary": False, + "ostree-ref": None, + "force-new-commit": False, + "version": None, + }, + channel=None, + mounts=[self.topdir, self.repo], + packages=["pungi", "ostree", "rpm-ostree"], + weight=123, + ) + ], + ) + self.assertEqual( + koji.run_runroot_cmd.call_args_list, + [ + mock.call( + koji.get_pungi_ostree_cmd.return_value, + log_file=self.topdir + + "/logs/x86_64/Everything/ostree-1/runroot.log", + ) + ], + ) + + self.assertTrue( + os.path.isfile(os.path.join(self.topdir, "work/ostree-1/extra_config.json")) + ) + self.assertTrue(os.path.isdir(self.repo)) + @mock.patch("pungi.wrappers.scm.get_dir_from_scm") @mock.patch("pungi.wrappers.kojiwrapper.KojiWrapper") def test_run_fail(self, KojiWrapper, get_dir_from_scm):