From 3cde5c3a87c45d7612785f566edca88e9eb4290b Mon Sep 17 00:00:00 2001 From: Jan Kaluza Date: Wed, 22 Jan 2020 07:55:37 +0100 Subject: [PATCH] Add support for new Pungi Buildinstall Koji plugin. We would like to start generating the buildinstall phase using the safer Koji Pungi Buildinstall plugin and stop the direct use of Runroot plugin. The plugin so far exists only as PR for Koji: https://pagure.io/koji/pull-request/1939 This commit adds support for this plugin when `lorax_use_koji_plugin` is set to `True`. Signed-off-by: Jan Kaluza --- doc/configuration.rst | 4 + pungi/checks.py | 4 + pungi/phases/buildinstall.py | 110 ++++++++++++++++--------- pungi/runroot.py | 29 +++++++ pungi/wrappers/kojiwrapper.py | 49 ++++++++++- tests/test_buildinstall.py | 148 ++++++++++++++++++++++++++++++++++ 6 files changed, 304 insertions(+), 40 deletions(-) diff --git a/doc/configuration.rst b/doc/configuration.rst index e130a991..ba753e5b 100644 --- a/doc/configuration.rst +++ b/doc/configuration.rst @@ -550,6 +550,10 @@ Options **lorax_extra_sources** (*list*) -- a variant/arch mapping with urls for extra source repositories added to Lorax command line. Either one repo or a list can be specified. +**lorax_use_koji_plugin** = False + (*bool*) -- When set to ``True``, the Koji pungi_buildinstall task will be + used to execute Lorax instead of runroot. Use only if the Koji instance + has the pungi_buildinstall plugin installed. **buildinstall_kickstart** (:ref:`scm_dict `) -- If specified, this kickstart file will be copied into each file and pointed to in boot configuration. diff --git a/pungi/checks.py b/pungi/checks.py index c57dbb63..7eb98a2a 100644 --- a/pungi/checks.py +++ b/pungi/checks.py @@ -1262,6 +1262,10 @@ def make_schema(): "lorax_extra_sources": _variant_arch_mapping({ "$ref": "#/definitions/strings", }), + "lorax_use_koji_plugin": { + "type": "boolean", + "default": False, + }, "signing_key_id": {"type": "string"}, "signing_key_password_file": {"type": "string"}, diff --git a/pungi/phases/buildinstall.py b/pungi/phases/buildinstall.py index 150f3937..c584dd63 100644 --- a/pungi/phases/buildinstall.py +++ b/pungi/phases/buildinstall.py @@ -21,7 +21,7 @@ import shutil import re from kobo.threads import ThreadPool, WorkerThread -from kobo.shortcuts import run +from kobo.shortcuts import run, force_list from productmd.images import Image from six.moves import shlex_quote @@ -46,6 +46,7 @@ class BuildinstallPhase(PhaseBase): # is needed to skip copying files for failed tasks. self.pool.finished_tasks = set() self.buildinstall_method = self.compose.conf.get("buildinstall_method") + self.lorax_use_koji_plugin = self.compose.conf.get("lorax_use_koji_plugin") self.used_lorax = self.buildinstall_method == 'lorax' self.pkgset_phase = pkgset_phase @@ -98,13 +99,6 @@ class BuildinstallPhase(PhaseBase): # only care about the directory anyway. log_dir = _get_log_dir(self.compose, variant, arch) - # If the buildinstall_topdir is set, it means Koji is used for - # buildinstall phase and the filesystem with Koji is read-only. - # In that case, we have to write logs to buildinstall_topdir and - # later copy them back to our local log directory. - if self.compose.conf.get("buildinstall_topdir", None): - output_dir = os.path.join(output_dir, "results") - repos = repo_baseurl[:] repos.extend( get_arch_variant_data(self.compose.conf, "lorax_extra_sources", arch, variant) @@ -115,31 +109,61 @@ class BuildinstallPhase(PhaseBase): comps_repo = translate_path(self.compose, comps_repo) repos.append(comps_repo) - lorax = LoraxWrapper() - lorax_cmd = lorax.get_lorax_cmd( - self.compose.conf["release_name"], - version, - version, - repos, - output_dir, - variant=variant.uid, - buildinstallpackages=variant.buildinstallpackages, - is_final=self.compose.supported, - buildarch=buildarch, - volid=volid, - nomacboot=nomacboot, - bugurl=bugurl, - add_template=add_template, - add_arch_template=add_arch_template, - add_template_var=add_template_var, - add_arch_template_var=add_arch_template_var, - noupgrade=noupgrade, - rootfs_size=rootfs_size, - log_dir=log_dir, - dracut_args=dracut_args, - ) - return 'rm -rf %s && %s' % (shlex_quote(output_topdir), - ' '.join([shlex_quote(x) for x in lorax_cmd])) + if self.lorax_use_koji_plugin: + return { + "product": self.compose.conf["release_name"], + "version": version, + "release": version, + "sources": force_list(repos), + "variant": variant.uid, + "installpkgs": variant.buildinstallpackages, + "isfinal": self.compose.supported, + "buildarch": buildarch, + "volid": volid, + "nomacboot": nomacboot, + "bugurl": bugurl, + "add-template": add_template, + "add-arch-template": add_arch_template, + "add-template-var": add_template_var, + "add-arch-template-var": add_arch_template_var, + "noupgrade": noupgrade, + "rootfs-size": rootfs_size, + "dracut-args": dracut_args, + "outputdir": output_dir, + } + else: + # If the buildinstall_topdir is set, it means Koji is used for + # buildinstall phase and the filesystem with Koji is read-only. + # In that case, we have to write logs to buildinstall_topdir and + # later copy them back to our local log directory. + if self.compose.conf.get("buildinstall_topdir", None): + output_dir = os.path.join(output_dir, "results") + + lorax = LoraxWrapper() + lorax_cmd = lorax.get_lorax_cmd( + self.compose.conf["release_name"], + version, + version, + repos, + output_dir, + variant=variant.uid, + buildinstallpackages=variant.buildinstallpackages, + is_final=self.compose.supported, + buildarch=buildarch, + volid=volid, + nomacboot=nomacboot, + bugurl=bugurl, + add_template=add_template, + add_arch_template=add_arch_template, + add_template_var=add_template_var, + add_arch_template_var=add_arch_template_var, + noupgrade=noupgrade, + rootfs_size=rootfs_size, + log_dir=log_dir, + dracut_args=dracut_args, + ) + return 'rm -rf %s && %s' % (shlex_quote(output_topdir), + ' '.join([shlex_quote(x) for x in lorax_cmd])) def get_repos(self, arch): repos = [] @@ -423,6 +447,7 @@ class BuildinstallThread(WorkerThread): def worker(self, compose, arch, variant, cmd, num): buildinstall_method = compose.conf["buildinstall_method"] + lorax_use_koji_plugin = compose.conf["lorax_use_koji_plugin"] log_filename = ('buildinstall-%s' % variant.uid) if variant else 'buildinstall' log_file = compose.paths.log.log_file(arch, log_filename) @@ -458,12 +483,19 @@ class BuildinstallThread(WorkerThread): # Start the runroot task. runroot = Runroot(compose, phase="buildinstall") - runroot.run( - cmd, log_file=log_file, arch=arch, packages=packages, - mounts=[compose.topdir], - weight=compose.conf['runroot_weights'].get('buildinstall'), - chown_paths=chown_paths, - ) + if buildinstall_method == "lorax" and lorax_use_koji_plugin: + runroot.run_pungi_buildinstall( + cmd, log_file=log_file, arch=arch, packages=packages, + mounts=[compose.topdir], + weight=compose.conf['runroot_weights'].get('buildinstall'), + ) + else: + runroot.run( + cmd, log_file=log_file, arch=arch, packages=packages, + mounts=[compose.topdir], + weight=compose.conf['runroot_weights'].get('buildinstall'), + chown_paths=chown_paths, + ) if final_output_dir != output_dir: if not os.path.exists(final_output_dir): diff --git a/pungi/runroot.py b/pungi/runroot.py index f4e7d9c3..b4ec5f81 100644 --- a/pungi/runroot.py +++ b/pungi/runroot.py @@ -236,6 +236,35 @@ class Runroot(kobo.log.LoggingBase): else: raise ValueError("Unknown runroot_method %r." % self.runroot_method) + def run_pungi_buildinstall(self, args, log_file=None, arch=None, **kwargs): + """ + Runs the Lorax buildinstall runroot command using the Pungi Buildinstall + Koji plugin as pungi_buildinstall task. + + The **kwargs are optional and matches the + `KojiWrapper.get_pungi_buildinstall_cmd()` kwargs. + + :param dict args: Arguments for the pungi_buildinstall Koji task. + :param str log_file: Log file into which the output of the task will + be logged. + :param str arch: Architecture on which the task should be executed. + """ + runroot_channel = self.compose.conf.get("runroot_channel") + runroot_tag = self.compose.conf["runroot_tag"] + + koji_wrapper = kojiwrapper.KojiWrapper(self.compose.conf["koji_profile"]) + koji_cmd = koji_wrapper.get_pungi_buildinstall_cmd( + runroot_tag, arch, args, channel=runroot_channel, + chown_uid=os.getuid(), **kwargs) + + output = koji_wrapper.run_runroot_cmd(koji_cmd, log_file=log_file) + if output["retcode"] != 0: + raise RuntimeError( + "Pungi-buildinstall task failed: %s. See %s for more details." + % (output["task_id"], log_file) + ) + self._result = output + def get_buildroot_rpms(self): """ Returns the list of RPMs installed in a buildroot in which the runroot diff --git a/pungi/wrappers/kojiwrapper.py b/pungi/wrappers/kojiwrapper.py index ca61160c..649c7d2f 100644 --- a/pungi/wrappers/kojiwrapper.py +++ b/pungi/wrappers/kojiwrapper.py @@ -21,7 +21,7 @@ import threading import contextlib import koji -from kobo.shortcuts import run +from kobo.shortcuts import run, force_list import six from six.moves import configparser, shlex_quote import six.moves.xmlrpc_client as xmlrpclib @@ -123,6 +123,47 @@ class KojiWrapper(object): return cmd + def get_pungi_buildinstall_cmd( + self, target, arch, args, channel=None, packages=None, + mounts=None, weight=None, chown_uid=None): + cmd = self._get_cmd("pungi-buildinstall", "--nowait", "--task-id") + + if channel: + cmd.append("--channel-override=%s" % channel) + else: + cmd.append("--channel-override=runroot-local") + + if weight: + cmd.append("--weight=%s" % int(weight)) + + for package in packages or []: + cmd.append("--package=%s" % package) + + for mount in mounts or []: + # directories are *not* created here + cmd.append("--mount=%s" % mount) + + if chown_uid: + cmd.append("--chown-uid=%s" % chown_uid) + + # IMPORTANT: all --opts have to be provided *before* args + + cmd.append(target) + + # i686 -> i386 etc. + arch = getBaseArch(arch) + cmd.append(arch) + + for k, v in args.items(): + if v: + if isinstance(v, bool): + cmd.append(k) + else: + for arg in force_list(v): + cmd.append("%s=%s" % (k, shlex_quote(arg))) + + return cmd + @contextlib.contextmanager def get_koji_cmd_env(self): """Get environment variables for running a koji command. @@ -606,6 +647,12 @@ def get_buildroot_rpms(compose, task_id): # runroot koji = KojiWrapper(compose.conf['koji_profile']) buildroot_infos = koji.koji_proxy.listBuildroots(taskID=task_id) + if not buildroot_infos: + children_tasks = koji.koji_proxy.getTaskChildren(task_id) + for child_task in children_tasks: + buildroot_infos = koji.koji_proxy.listBuildroots(taskID=child_task["id"]) + if buildroot_infos: + break buildroot_info = buildroot_infos[-1] data = koji.koji_proxy.listRPMs(componentBuildrootID=buildroot_info["id"]) for rpm_info in data: diff --git a/tests/test_buildinstall.py b/tests/test_buildinstall.py index 57d77c3c..8174670b 100644 --- a/tests/test_buildinstall.py +++ b/tests/test_buildinstall.py @@ -157,6 +157,85 @@ class TestBuildinstallPhase(PungiTestCase): mock.call(compose, 'amd64', variant=compose.variants['Client'], disc_type='DVD'), mock.call(compose, 'amd64', variant=compose.variants['Server'], disc_type='DVD')]) + @mock.patch('pungi.phases.buildinstall.ThreadPool') + @mock.patch('pungi.phases.buildinstall.get_volid') + def test_starts_threads_for_each_cmd_with_lorax_koji_plugin( + self, get_volid, poolCls): + compose = BuildInstallCompose(self.topdir, { + 'bootable': True, + 'release_name': 'Test', + 'release_short': 't', + 'release_version': '1', + 'buildinstall_method': 'lorax', + 'lorax_use_koji_plugin': True, + 'disc_types': {'dvd': 'DVD'}, + }) + + get_volid.return_value = 'vol_id' + + phase = BuildinstallPhase(compose, self._make_pkgset_phase(["p1", "p2"])) + + phase.run() + self.maxDiff = None + + expected_args = [ + { + 'product': 'Test', 'version': '1', 'release': '1', + 'sources': [self.topdir + "/work/amd64/repo/p1", + self.topdir + "/work/amd64/repo/p2", + self.topdir + '/work/amd64/comps_repo_Server'], + 'variant': 'Server', 'installpkgs': ['bash', 'vim'], + 'isfinal': True, 'buildarch': 'amd64', 'volid': 'vol_id', + 'nomacboot': True, 'bugurl': None, 'add-template': [], + 'add-arch-template': [], 'add-template-var': [], + 'add-arch-template-var': [], 'noupgrade': True, + 'rootfs-size': None, 'dracut-args': [], + 'outputdir': self.topdir + '/work/amd64/buildinstall/Server' + }, + { + 'product': 'Test', 'version': '1', 'release': '1', + 'sources': [self.topdir + "/work/amd64/repo/p1", + self.topdir + "/work/amd64/repo/p2", + self.topdir + '/work/amd64/comps_repo_Client'], + 'variant': 'Client', 'installpkgs': [], + 'isfinal': True, 'buildarch': 'amd64', 'volid': 'vol_id', + 'nomacboot': True, 'bugurl': None, 'add-template': [], + 'add-arch-template': [], 'add-template-var': [], + 'add-arch-template-var': [], 'noupgrade': True, + 'rootfs-size': None, 'dracut-args': [], + 'outputdir': self.topdir + '/work/amd64/buildinstall/Client' + }, + { + 'product': 'Test', 'version': '1', 'release': '1', + 'sources': [self.topdir + "/work/x86_64/repo/p1", + self.topdir + "/work/x86_64/repo/p2", + self.topdir + '/work/x86_64/comps_repo_Server'], + 'variant': 'Server', 'installpkgs': ['bash', 'vim'], + 'isfinal': True, 'buildarch': 'x86_64', 'volid': 'vol_id', + 'nomacboot': True, 'bugurl': None, 'add-template': [], + 'add-arch-template': [], 'add-template-var': [], + 'add-arch-template-var': [], 'noupgrade': True, + 'rootfs-size': None, 'dracut-args': [], + 'outputdir': self.topdir + '/work/x86_64/buildinstall/Server' + }, + ] + + # Three items added for processing in total. + # Server.x86_64, Client.amd64, Server.x86_64 + pool = poolCls.return_value + self.assertEqual(3, len(pool.queue_put.mock_calls)) + six.assertCountEqual( + self, + [call[0][0][3] for call in pool.queue_put.call_args_list], + expected_args) + + six.assertCountEqual( + self, + get_volid.mock_calls, + [mock.call(compose, 'x86_64', variant=compose.variants['Server'], disc_type='DVD'), + mock.call(compose, 'amd64', variant=compose.variants['Client'], disc_type='DVD'), + mock.call(compose, 'amd64', variant=compose.variants['Server'], disc_type='DVD')]) + @mock.patch('pungi.phases.buildinstall.ThreadPool') @mock.patch('pungi.phases.buildinstall.LoraxWrapper') @mock.patch('pungi.phases.buildinstall.get_volid') @@ -659,6 +738,75 @@ class BuildinstallThreadTestCase(PungiTestCase): [mock.call(compose, "x86_64", compose.variants["Server"], False)], ) + @mock.patch('pungi.phases.buildinstall.link_boot_iso') + @mock.patch('pungi.phases.buildinstall.tweak_buildinstall') + @mock.patch('pungi.wrappers.kojiwrapper.KojiWrapper') + @mock.patch('pungi.wrappers.kojiwrapper.get_buildroot_rpms') + @mock.patch('pungi.phases.buildinstall.run') + def test_buildinstall_thread_with_lorax_using_koji_plugin( + self, run, get_buildroot_rpms, KojiWrapperMock, mock_tweak, mock_link + ): + compose = BuildInstallCompose(self.topdir, { + 'buildinstall_method': 'lorax', + 'lorax_use_koji_plugin': True, + 'runroot_tag': 'rrt', + 'koji_profile': 'koji', + 'runroot_weights': {'buildinstall': 123}, + }) + + get_buildroot_rpms.return_value = ['bash', 'zsh'] + + get_pungi_buildinstall_cmd = KojiWrapperMock.return_value.get_pungi_buildinstall_cmd + + run_runroot_cmd = KojiWrapperMock.return_value.run_runroot_cmd + run_runroot_cmd.return_value = { + 'output': 'Foo bar baz', + 'retcode': 0, + 'task_id': 1234, + } + + t = BuildinstallThread(self.pool) + + with mock.patch('time.sleep'): + t.process((compose, 'x86_64', compose.variants['Server'], self.cmd), 0) + + destdir = os.path.join(self.topdir, "work/x86_64/buildinstall/Server") + self.assertEqual( + get_pungi_buildinstall_cmd.mock_calls, + [mock.call( + 'rrt', 'x86_64', self.cmd, channel=None, + packages=['lorax'], mounts=[self.topdir], + weight=123, chown_uid=os.getuid() + )]) + self.assertEqual( + run_runroot_cmd.mock_calls, + [mock.call(get_pungi_buildinstall_cmd.return_value, + log_file=self.topdir + '/logs/x86_64/buildinstall-Server.x86_64.log')]) + with open(self.topdir + '/logs/x86_64/buildinstall-Server-RPMs.x86_64.log') as f: + rpms = f.read().strip().split('\n') + six.assertCountEqual(self, rpms, ["bash", "zsh"]) + six.assertCountEqual(self, self.pool.finished_tasks, [("Server", "x86_64")]) + + self.assertEqual( + mock_tweak.call_args_list, + [ + mock.call( + compose, + destdir, + os.path.join(self.topdir, "compose/Server/x86_64/os"), + "x86_64", + "Server", + "", + "dummy-volid", + self.pool.kickstart_file, + ) + ], + ) + self.assertEqual( + mock_link.call_args_list, + [mock.call(compose, "x86_64", compose.variants["Server"], False)], + ) + @mock.patch('pungi.phases.buildinstall.link_boot_iso') @mock.patch('pungi.phases.buildinstall.tweak_buildinstall') @mock.patch('pungi.wrappers.kojiwrapper.KojiWrapper')