diff --git a/doc/configuration.rst b/doc/configuration.rst index e130a991..ba753e5b 100644 --- a/doc/configuration.rst +++ b/doc/configuration.rst @@ -550,6 +550,10 @@ Options **lorax_extra_sources** (*list*) -- a variant/arch mapping with urls for extra source repositories added to Lorax command line. Either one repo or a list can be specified. +**lorax_use_koji_plugin** = False + (*bool*) -- When set to ``True``, the Koji pungi_buildinstall task will be + used to execute Lorax instead of runroot. Use only if the Koji instance + has the pungi_buildinstall plugin installed. **buildinstall_kickstart** (:ref:`scm_dict `) -- If specified, this kickstart file will be copied into each file and pointed to in boot configuration. diff --git a/pungi/checks.py b/pungi/checks.py index c57dbb63..7eb98a2a 100644 --- a/pungi/checks.py +++ b/pungi/checks.py @@ -1262,6 +1262,10 @@ def make_schema(): "lorax_extra_sources": _variant_arch_mapping({ "$ref": "#/definitions/strings", }), + "lorax_use_koji_plugin": { + "type": "boolean", + "default": False, + }, "signing_key_id": {"type": "string"}, "signing_key_password_file": {"type": "string"}, diff --git a/pungi/phases/buildinstall.py b/pungi/phases/buildinstall.py index 150f3937..c584dd63 100644 --- a/pungi/phases/buildinstall.py +++ b/pungi/phases/buildinstall.py @@ -21,7 +21,7 @@ import shutil import re from kobo.threads import ThreadPool, WorkerThread -from kobo.shortcuts import run +from kobo.shortcuts import run, force_list from productmd.images import Image from six.moves import shlex_quote @@ -46,6 +46,7 @@ class BuildinstallPhase(PhaseBase): # is needed to skip copying files for failed tasks. self.pool.finished_tasks = set() self.buildinstall_method = self.compose.conf.get("buildinstall_method") + self.lorax_use_koji_plugin = self.compose.conf.get("lorax_use_koji_plugin") self.used_lorax = self.buildinstall_method == 'lorax' self.pkgset_phase = pkgset_phase @@ -98,13 +99,6 @@ class BuildinstallPhase(PhaseBase): # only care about the directory anyway. log_dir = _get_log_dir(self.compose, variant, arch) - # If the buildinstall_topdir is set, it means Koji is used for - # buildinstall phase and the filesystem with Koji is read-only. - # In that case, we have to write logs to buildinstall_topdir and - # later copy them back to our local log directory. - if self.compose.conf.get("buildinstall_topdir", None): - output_dir = os.path.join(output_dir, "results") - repos = repo_baseurl[:] repos.extend( get_arch_variant_data(self.compose.conf, "lorax_extra_sources", arch, variant) @@ -115,31 +109,61 @@ class BuildinstallPhase(PhaseBase): comps_repo = translate_path(self.compose, comps_repo) repos.append(comps_repo) - lorax = LoraxWrapper() - lorax_cmd = lorax.get_lorax_cmd( - self.compose.conf["release_name"], - version, - version, - repos, - output_dir, - variant=variant.uid, - buildinstallpackages=variant.buildinstallpackages, - is_final=self.compose.supported, - buildarch=buildarch, - volid=volid, - nomacboot=nomacboot, - bugurl=bugurl, - add_template=add_template, - add_arch_template=add_arch_template, - add_template_var=add_template_var, - add_arch_template_var=add_arch_template_var, - noupgrade=noupgrade, - rootfs_size=rootfs_size, - log_dir=log_dir, - dracut_args=dracut_args, - ) - return 'rm -rf %s && %s' % (shlex_quote(output_topdir), - ' '.join([shlex_quote(x) for x in lorax_cmd])) + if self.lorax_use_koji_plugin: + return { + "product": self.compose.conf["release_name"], + "version": version, + "release": version, + "sources": force_list(repos), + "variant": variant.uid, + "installpkgs": variant.buildinstallpackages, + "isfinal": self.compose.supported, + "buildarch": buildarch, + "volid": volid, + "nomacboot": nomacboot, + "bugurl": bugurl, + "add-template": add_template, + "add-arch-template": add_arch_template, + "add-template-var": add_template_var, + "add-arch-template-var": add_arch_template_var, + "noupgrade": noupgrade, + "rootfs-size": rootfs_size, + "dracut-args": dracut_args, + "outputdir": output_dir, + } + else: + # If the buildinstall_topdir is set, it means Koji is used for + # buildinstall phase and the filesystem with Koji is read-only. + # In that case, we have to write logs to buildinstall_topdir and + # later copy them back to our local log directory. + if self.compose.conf.get("buildinstall_topdir", None): + output_dir = os.path.join(output_dir, "results") + + lorax = LoraxWrapper() + lorax_cmd = lorax.get_lorax_cmd( + self.compose.conf["release_name"], + version, + version, + repos, + output_dir, + variant=variant.uid, + buildinstallpackages=variant.buildinstallpackages, + is_final=self.compose.supported, + buildarch=buildarch, + volid=volid, + nomacboot=nomacboot, + bugurl=bugurl, + add_template=add_template, + add_arch_template=add_arch_template, + add_template_var=add_template_var, + add_arch_template_var=add_arch_template_var, + noupgrade=noupgrade, + rootfs_size=rootfs_size, + log_dir=log_dir, + dracut_args=dracut_args, + ) + return 'rm -rf %s && %s' % (shlex_quote(output_topdir), + ' '.join([shlex_quote(x) for x in lorax_cmd])) def get_repos(self, arch): repos = [] @@ -423,6 +447,7 @@ class BuildinstallThread(WorkerThread): def worker(self, compose, arch, variant, cmd, num): buildinstall_method = compose.conf["buildinstall_method"] + lorax_use_koji_plugin = compose.conf["lorax_use_koji_plugin"] log_filename = ('buildinstall-%s' % variant.uid) if variant else 'buildinstall' log_file = compose.paths.log.log_file(arch, log_filename) @@ -458,12 +483,19 @@ class BuildinstallThread(WorkerThread): # Start the runroot task. runroot = Runroot(compose, phase="buildinstall") - runroot.run( - cmd, log_file=log_file, arch=arch, packages=packages, - mounts=[compose.topdir], - weight=compose.conf['runroot_weights'].get('buildinstall'), - chown_paths=chown_paths, - ) + if buildinstall_method == "lorax" and lorax_use_koji_plugin: + runroot.run_pungi_buildinstall( + cmd, log_file=log_file, arch=arch, packages=packages, + mounts=[compose.topdir], + weight=compose.conf['runroot_weights'].get('buildinstall'), + ) + else: + runroot.run( + cmd, log_file=log_file, arch=arch, packages=packages, + mounts=[compose.topdir], + weight=compose.conf['runroot_weights'].get('buildinstall'), + chown_paths=chown_paths, + ) if final_output_dir != output_dir: if not os.path.exists(final_output_dir): diff --git a/pungi/runroot.py b/pungi/runroot.py index f4e7d9c3..b4ec5f81 100644 --- a/pungi/runroot.py +++ b/pungi/runroot.py @@ -236,6 +236,35 @@ class Runroot(kobo.log.LoggingBase): else: raise ValueError("Unknown runroot_method %r." % self.runroot_method) + def run_pungi_buildinstall(self, args, log_file=None, arch=None, **kwargs): + """ + Runs the Lorax buildinstall runroot command using the Pungi Buildinstall + Koji plugin as pungi_buildinstall task. + + The **kwargs are optional and matches the + `KojiWrapper.get_pungi_buildinstall_cmd()` kwargs. + + :param dict args: Arguments for the pungi_buildinstall Koji task. + :param str log_file: Log file into which the output of the task will + be logged. + :param str arch: Architecture on which the task should be executed. + """ + runroot_channel = self.compose.conf.get("runroot_channel") + runroot_tag = self.compose.conf["runroot_tag"] + + koji_wrapper = kojiwrapper.KojiWrapper(self.compose.conf["koji_profile"]) + koji_cmd = koji_wrapper.get_pungi_buildinstall_cmd( + runroot_tag, arch, args, channel=runroot_channel, + chown_uid=os.getuid(), **kwargs) + + output = koji_wrapper.run_runroot_cmd(koji_cmd, log_file=log_file) + if output["retcode"] != 0: + raise RuntimeError( + "Pungi-buildinstall task failed: %s. See %s for more details." + % (output["task_id"], log_file) + ) + self._result = output + def get_buildroot_rpms(self): """ Returns the list of RPMs installed in a buildroot in which the runroot diff --git a/pungi/wrappers/kojiwrapper.py b/pungi/wrappers/kojiwrapper.py index ca61160c..649c7d2f 100644 --- a/pungi/wrappers/kojiwrapper.py +++ b/pungi/wrappers/kojiwrapper.py @@ -21,7 +21,7 @@ import threading import contextlib import koji -from kobo.shortcuts import run +from kobo.shortcuts import run, force_list import six from six.moves import configparser, shlex_quote import six.moves.xmlrpc_client as xmlrpclib @@ -123,6 +123,47 @@ class KojiWrapper(object): return cmd + def get_pungi_buildinstall_cmd( + self, target, arch, args, channel=None, packages=None, + mounts=None, weight=None, chown_uid=None): + cmd = self._get_cmd("pungi-buildinstall", "--nowait", "--task-id") + + if channel: + cmd.append("--channel-override=%s" % channel) + else: + cmd.append("--channel-override=runroot-local") + + if weight: + cmd.append("--weight=%s" % int(weight)) + + for package in packages or []: + cmd.append("--package=%s" % package) + + for mount in mounts or []: + # directories are *not* created here + cmd.append("--mount=%s" % mount) + + if chown_uid: + cmd.append("--chown-uid=%s" % chown_uid) + + # IMPORTANT: all --opts have to be provided *before* args + + cmd.append(target) + + # i686 -> i386 etc. + arch = getBaseArch(arch) + cmd.append(arch) + + for k, v in args.items(): + if v: + if isinstance(v, bool): + cmd.append(k) + else: + for arg in force_list(v): + cmd.append("%s=%s" % (k, shlex_quote(arg))) + + return cmd + @contextlib.contextmanager def get_koji_cmd_env(self): """Get environment variables for running a koji command. @@ -606,6 +647,12 @@ def get_buildroot_rpms(compose, task_id): # runroot koji = KojiWrapper(compose.conf['koji_profile']) buildroot_infos = koji.koji_proxy.listBuildroots(taskID=task_id) + if not buildroot_infos: + children_tasks = koji.koji_proxy.getTaskChildren(task_id) + for child_task in children_tasks: + buildroot_infos = koji.koji_proxy.listBuildroots(taskID=child_task["id"]) + if buildroot_infos: + break buildroot_info = buildroot_infos[-1] data = koji.koji_proxy.listRPMs(componentBuildrootID=buildroot_info["id"]) for rpm_info in data: diff --git a/tests/test_buildinstall.py b/tests/test_buildinstall.py index 57d77c3c..8174670b 100644 --- a/tests/test_buildinstall.py +++ b/tests/test_buildinstall.py @@ -157,6 +157,85 @@ class TestBuildinstallPhase(PungiTestCase): mock.call(compose, 'amd64', variant=compose.variants['Client'], disc_type='DVD'), mock.call(compose, 'amd64', variant=compose.variants['Server'], disc_type='DVD')]) + @mock.patch('pungi.phases.buildinstall.ThreadPool') + @mock.patch('pungi.phases.buildinstall.get_volid') + def test_starts_threads_for_each_cmd_with_lorax_koji_plugin( + self, get_volid, poolCls): + compose = BuildInstallCompose(self.topdir, { + 'bootable': True, + 'release_name': 'Test', + 'release_short': 't', + 'release_version': '1', + 'buildinstall_method': 'lorax', + 'lorax_use_koji_plugin': True, + 'disc_types': {'dvd': 'DVD'}, + }) + + get_volid.return_value = 'vol_id' + + phase = BuildinstallPhase(compose, self._make_pkgset_phase(["p1", "p2"])) + + phase.run() + self.maxDiff = None + + expected_args = [ + { + 'product': 'Test', 'version': '1', 'release': '1', + 'sources': [self.topdir + "/work/amd64/repo/p1", + self.topdir + "/work/amd64/repo/p2", + self.topdir + '/work/amd64/comps_repo_Server'], + 'variant': 'Server', 'installpkgs': ['bash', 'vim'], + 'isfinal': True, 'buildarch': 'amd64', 'volid': 'vol_id', + 'nomacboot': True, 'bugurl': None, 'add-template': [], + 'add-arch-template': [], 'add-template-var': [], + 'add-arch-template-var': [], 'noupgrade': True, + 'rootfs-size': None, 'dracut-args': [], + 'outputdir': self.topdir + '/work/amd64/buildinstall/Server' + }, + { + 'product': 'Test', 'version': '1', 'release': '1', + 'sources': [self.topdir + "/work/amd64/repo/p1", + self.topdir + "/work/amd64/repo/p2", + self.topdir + '/work/amd64/comps_repo_Client'], + 'variant': 'Client', 'installpkgs': [], + 'isfinal': True, 'buildarch': 'amd64', 'volid': 'vol_id', + 'nomacboot': True, 'bugurl': None, 'add-template': [], + 'add-arch-template': [], 'add-template-var': [], + 'add-arch-template-var': [], 'noupgrade': True, + 'rootfs-size': None, 'dracut-args': [], + 'outputdir': self.topdir + '/work/amd64/buildinstall/Client' + }, + { + 'product': 'Test', 'version': '1', 'release': '1', + 'sources': [self.topdir + "/work/x86_64/repo/p1", + self.topdir + "/work/x86_64/repo/p2", + self.topdir + '/work/x86_64/comps_repo_Server'], + 'variant': 'Server', 'installpkgs': ['bash', 'vim'], + 'isfinal': True, 'buildarch': 'x86_64', 'volid': 'vol_id', + 'nomacboot': True, 'bugurl': None, 'add-template': [], + 'add-arch-template': [], 'add-template-var': [], + 'add-arch-template-var': [], 'noupgrade': True, + 'rootfs-size': None, 'dracut-args': [], + 'outputdir': self.topdir + '/work/x86_64/buildinstall/Server' + }, + ] + + # Three items added for processing in total. + # Server.x86_64, Client.amd64, Server.x86_64 + pool = poolCls.return_value + self.assertEqual(3, len(pool.queue_put.mock_calls)) + six.assertCountEqual( + self, + [call[0][0][3] for call in pool.queue_put.call_args_list], + expected_args) + + six.assertCountEqual( + self, + get_volid.mock_calls, + [mock.call(compose, 'x86_64', variant=compose.variants['Server'], disc_type='DVD'), + mock.call(compose, 'amd64', variant=compose.variants['Client'], disc_type='DVD'), + mock.call(compose, 'amd64', variant=compose.variants['Server'], disc_type='DVD')]) + @mock.patch('pungi.phases.buildinstall.ThreadPool') @mock.patch('pungi.phases.buildinstall.LoraxWrapper') @mock.patch('pungi.phases.buildinstall.get_volid') @@ -659,6 +738,75 @@ class BuildinstallThreadTestCase(PungiTestCase): [mock.call(compose, "x86_64", compose.variants["Server"], False)], ) + @mock.patch('pungi.phases.buildinstall.link_boot_iso') + @mock.patch('pungi.phases.buildinstall.tweak_buildinstall') + @mock.patch('pungi.wrappers.kojiwrapper.KojiWrapper') + @mock.patch('pungi.wrappers.kojiwrapper.get_buildroot_rpms') + @mock.patch('pungi.phases.buildinstall.run') + def test_buildinstall_thread_with_lorax_using_koji_plugin( + self, run, get_buildroot_rpms, KojiWrapperMock, mock_tweak, mock_link + ): + compose = BuildInstallCompose(self.topdir, { + 'buildinstall_method': 'lorax', + 'lorax_use_koji_plugin': True, + 'runroot_tag': 'rrt', + 'koji_profile': 'koji', + 'runroot_weights': {'buildinstall': 123}, + }) + + get_buildroot_rpms.return_value = ['bash', 'zsh'] + + get_pungi_buildinstall_cmd = KojiWrapperMock.return_value.get_pungi_buildinstall_cmd + + run_runroot_cmd = KojiWrapperMock.return_value.run_runroot_cmd + run_runroot_cmd.return_value = { + 'output': 'Foo bar baz', + 'retcode': 0, + 'task_id': 1234, + } + + t = BuildinstallThread(self.pool) + + with mock.patch('time.sleep'): + t.process((compose, 'x86_64', compose.variants['Server'], self.cmd), 0) + + destdir = os.path.join(self.topdir, "work/x86_64/buildinstall/Server") + self.assertEqual( + get_pungi_buildinstall_cmd.mock_calls, + [mock.call( + 'rrt', 'x86_64', self.cmd, channel=None, + packages=['lorax'], mounts=[self.topdir], + weight=123, chown_uid=os.getuid() + )]) + self.assertEqual( + run_runroot_cmd.mock_calls, + [mock.call(get_pungi_buildinstall_cmd.return_value, + log_file=self.topdir + '/logs/x86_64/buildinstall-Server.x86_64.log')]) + with open(self.topdir + '/logs/x86_64/buildinstall-Server-RPMs.x86_64.log') as f: + rpms = f.read().strip().split('\n') + six.assertCountEqual(self, rpms, ["bash", "zsh"]) + six.assertCountEqual(self, self.pool.finished_tasks, [("Server", "x86_64")]) + + self.assertEqual( + mock_tweak.call_args_list, + [ + mock.call( + compose, + destdir, + os.path.join(self.topdir, "compose/Server/x86_64/os"), + "x86_64", + "Server", + "", + "dummy-volid", + self.pool.kickstart_file, + ) + ], + ) + self.assertEqual( + mock_link.call_args_list, + [mock.call(compose, "x86_64", compose.variants["Server"], False)], + ) + @mock.patch('pungi.phases.buildinstall.link_boot_iso') @mock.patch('pungi.phases.buildinstall.tweak_buildinstall') @mock.patch('pungi.wrappers.kojiwrapper.KojiWrapper')