From 4ea1916a87f64541291eb2522518abf96fd1b949 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lubom=C3=ADr=20Sedl=C3=A1=C5=99?= Date: Thu, 23 Feb 2017 10:28:49 +0100 Subject: [PATCH] Enable customizing runroot task weight MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit For different cases where runroot is used it's now possible to set custom weight. The usecase for this is to avoid one builder taking too many tasks. Especially buildinstall is quite resource intensive, so one builder taking multiple tasks at the same time leads to very slow compose time. Signed-off-by: Lubomír Sedlář --- doc/configuration.rst | 11 +++++++++++ pungi/checks.py | 13 ++++++++++++- pungi/phases/buildinstall.py | 10 ++++++---- pungi/phases/createiso.py | 4 +++- pungi/phases/ostree.py | 3 ++- pungi/phases/ostree_installer.py | 3 ++- tests/test_buildinstall.py | 5 +++-- tests/test_createiso_phase.py | 7 ++++--- tests/test_ostree_installer_phase.py | 8 +++++--- tests/test_ostree_phase.py | 7 ++++--- 10 files changed, 52 insertions(+), 19 deletions(-) diff --git a/doc/configuration.rst b/doc/configuration.rst index 78ee5a56..5c51e863 100644 --- a/doc/configuration.rst +++ b/doc/configuration.rst @@ -680,6 +680,17 @@ Options **runroot_tag** (*str*) -- name of koji **build** tag used for runroot +**runroot_weights** + (*dict*) -- customize task weights for various runroot tasks. The values in + the mapping should be integers, the keys can be selected from the following + list. By default no weight is assigned and Koji picks the default one + according to policy. + + * ``buildinstall`` + * ``createiso`` + * ``ostree`` + * ``ostree_installer`` + Example ------- diff --git a/pungi/checks.py b/pungi/checks.py index 8dadd0b6..119973c7 100644 --- a/pungi/checks.py +++ b/pungi/checks.py @@ -531,6 +531,17 @@ def _make_schema(): "runroot_channel": { "$ref": "#/definitions/optional_string", }, + "runroot_weights": { + "type": "object", + "default": {}, + "properties": { + "buildinstall": {"type": "number"}, + "createiso": {"type": "number"}, + "ostree": {"type": "number"}, + "ostree_installer": {"type": "number"}, + }, + "additionalProperties": False, + }, "createrepo_deltas": { "type": "boolean", "default": False, @@ -969,7 +980,7 @@ CONFIG_DEPS = { (lambda x: x, ["koji_profile", "runroot_tag", "runroot_channel"]), ), "conflicts": ( - (lambda x: not x, ["runroot_tag", "runroot_channel"]), + (lambda x: not x, ["runroot_tag", "runroot_channel", "runroot_weights"]), ), }, "product_id": { diff --git a/pungi/phases/buildinstall.py b/pungi/phases/buildinstall.py index a9562bb1..c75020af 100644 --- a/pungi/phases/buildinstall.py +++ b/pungi/phases/buildinstall.py @@ -390,10 +390,12 @@ class BuildinstallThread(WorkerThread): runroot_tag = compose.conf["runroot_tag"] koji_wrapper = KojiWrapper(compose.conf["koji_profile"]) - koji_cmd = koji_wrapper.get_runroot_cmd(runroot_tag, arch, cmd, - channel=runroot_channel, - use_shell=True, task_id=True, - packages=packages, mounts=[compose.topdir]) + koji_cmd = koji_wrapper.get_runroot_cmd( + runroot_tag, arch, cmd, + channel=runroot_channel, + use_shell=True, task_id=True, + packages=packages, mounts=[compose.topdir], + weight=compose.conf['runroot_weights'].get('buildinstall')) # avoid race conditions? # Kerberos authentication failed: Permission denied in replay cache code (-1765328215) diff --git a/pungi/phases/createiso.py b/pungi/phases/createiso.py index ce6044f5..f0549370 100644 --- a/pungi/phases/createiso.py +++ b/pungi/phases/createiso.py @@ -227,7 +227,9 @@ class CreateIsoThread(WorkerThread): koji_cmd = koji_wrapper.get_runroot_cmd( runroot_tag, build_arch, cmd["cmd"], channel=runroot_channel, use_shell=True, task_id=True, - packages=packages, mounts=mounts) + packages=packages, mounts=mounts, + weight=compose.conf['runroot_weights'].get('createiso') + ) # avoid race conditions? # Kerberos authentication failed: Permission denied in replay cache code (-1765328215) diff --git a/pungi/phases/ostree.py b/pungi/phases/ostree.py index 101de8a7..b33c004c 100644 --- a/pungi/phases/ostree.py +++ b/pungi/phases/ostree.py @@ -147,7 +147,8 @@ class OSTreeThread(WorkerThread): channel=runroot_channel, use_shell=True, task_id=True, packages=packages, mounts=mounts, - new_chroot=True) + new_chroot=True, + weight=compose.conf["runroot_weights"].get('ostree')) output = koji.run_runroot_cmd(koji_cmd, log_file=log_file) if output["retcode"] != 0: raise RuntimeError("Runroot task failed: %s. See %s for more details." diff --git a/pungi/phases/ostree_installer.py b/pungi/phases/ostree_installer.py index c9297593..77eddcd4 100644 --- a/pungi/phases/ostree_installer.py +++ b/pungi/phases/ostree_installer.py @@ -170,7 +170,8 @@ class OstreeInstallerThread(WorkerThread): koji_cmd = koji.get_runroot_cmd(runroot_tag, arch, cmd, channel=runroot_channel, use_shell=True, task_id=True, - packages=packages, mounts=[compose.topdir]) + packages=packages, mounts=[compose.topdir], + weight=compose.conf['runroot_weights'].get('ostree_installer')) output = koji.run_runroot_cmd(koji_cmd, log_file=log_file) if output["retcode"] != 0: raise RuntimeError("Runroot task failed: %s. See %s for more details." diff --git a/tests/test_buildinstall.py b/tests/test_buildinstall.py index ce5439bc..0993605d 100644 --- a/tests/test_buildinstall.py +++ b/tests/test_buildinstall.py @@ -467,6 +467,7 @@ class BuildinstallThreadTestCase(PungiTestCase): 'runroot': True, 'runroot_tag': 'rrt', 'koji_profile': 'koji', + 'runroot_weights': {'buildinstall': 123}, }) get_buildroot_rpms.return_value = ['bash', 'zsh'] @@ -491,7 +492,7 @@ class BuildinstallThreadTestCase(PungiTestCase): get_runroot_cmd.mock_calls, [mock.call('rrt', 'x86_64', cmd, channel=None, use_shell=True, task_id=True, - packages=['strace', 'lorax'], mounts=[self.topdir])]) + packages=['strace', 'lorax'], mounts=[self.topdir], weight=123)]) self.assertItemsEqual( run_runroot_cmd.mock_calls, [mock.call(get_runroot_cmd.return_value, @@ -533,7 +534,7 @@ class BuildinstallThreadTestCase(PungiTestCase): get_runroot_cmd.mock_calls, [mock.call('rrt', 'x86_64', cmd, channel=None, use_shell=True, task_id=True, - packages=['strace', 'anaconda'], mounts=[self.topdir])]) + packages=['strace', 'anaconda'], mounts=[self.topdir], weight=None)]) self.assertItemsEqual( run_runroot_cmd.mock_calls, [mock.call(get_runroot_cmd.return_value, diff --git a/tests/test_createiso_phase.py b/tests/test_createiso_phase.py index c977c3b8..c58cc9ba 100644 --- a/tests/test_createiso_phase.py +++ b/tests/test_createiso_phase.py @@ -252,7 +252,7 @@ class CreateisoThreadTest(helpers.PungiTestCase): mounts=[self.topdir], packages=['coreutils', 'genisoimage', 'isomd5sum', 'jigdo'], - task_id=True, use_shell=True)]) + task_id=True, use_shell=True, weight=None)]) self.assertEqual( run_runroot.call_args_list, [mock.call(get_runroot_cmd.return_value, @@ -286,6 +286,7 @@ class CreateisoThreadTest(helpers.PungiTestCase): 'runroot_tag': 'f25-build', 'koji_profile': 'koji', 'create_jigdo': False, + 'runroot_weights': {'createiso': 123}, }) cmd = { 'iso_path': '%s/compose/Server/x86_64/iso/image-name' % self.topdir, @@ -316,7 +317,7 @@ class CreateisoThreadTest(helpers.PungiTestCase): [mock.call('f25-build', 'x86_64', cmd['cmd'], channel=None, mounts=[self.topdir], packages=['coreutils', 'genisoimage', 'isomd5sum'], - task_id=True, use_shell=True)]) + task_id=True, use_shell=True, weight=123)]) self.assertEqual( run_runroot.call_args_list, [mock.call(get_runroot_cmd.return_value, @@ -382,7 +383,7 @@ class CreateisoThreadTest(helpers.PungiTestCase): mounts=[self.topdir], packages=['coreutils', 'genisoimage', 'isomd5sum', 'jigdo', 'lorax'], - task_id=True, use_shell=True)]) + task_id=True, use_shell=True, weight=None)]) self.assertEqual( run_runroot.call_args_list, [mock.call(get_runroot_cmd.return_value, diff --git a/tests/test_ostree_installer_phase.py b/tests/test_ostree_installer_phase.py index 6aeecdb6..0db6af80 100644 --- a/tests/test_ostree_installer_phase.py +++ b/tests/test_ostree_installer_phase.py @@ -72,7 +72,7 @@ class OstreeThreadTest(helpers.PungiTestCase): self.assertEqual(compose.im.add.mock_calls, [mock.call('Everything', 'x86_64', image)]) - def assertRunrootCall(self, koji, sources, release, isfinal=False, extra=[]): + def assertRunrootCall(self, koji, sources, release, isfinal=False, extra=[], weight=None): lorax_cmd = [ 'lorax', '--product=Fedora', @@ -101,7 +101,7 @@ class OstreeThreadTest(helpers.PungiTestCase): lorax_cmd, channel=None, mounts=[self.topdir], packages=['pungi', 'lorax', 'ostree'], - task_id=True, use_shell=True)]) + task_id=True, use_shell=True, weight=weight)]) self.assertEqual(koji.run_runroot_cmd.call_args_list, [mock.call(koji.get_runroot_cmd.return_value, log_file=self.topdir + '/logs/x86_64/ostree_installer/runroot.log')]) @@ -377,6 +377,7 @@ class OstreeThreadTest(helpers.PungiTestCase): "ostree_ref=fedora-atomic/Rawhide/x86_64/docker-host", ], } + self.compose.conf['runroot_weights'] = {'ostree_installer': 123} koji = KojiWrapper.return_value koji.run_runroot_cmd.return_value = { 'task_id': 1234, @@ -404,7 +405,8 @@ class OstreeThreadTest(helpers.PungiTestCase): '--add-arch-template-var=ostree_repo=https://kojipkgs.fedoraproject.org/compose/atomic/Rawhide/', '--add-arch-template-var=ostree_osname=fedora-atomic', '--add-arch-template-var=ostree_ref=fedora-atomic/Rawhide/x86_64/docker-host', - '--logfile=%s/logs/x86_64/ostree_installer/lorax.log' % self.topdir] + '--logfile=%s/logs/x86_64/ostree_installer/lorax.log' % self.topdir], + weight=123, ) self.assertIsoLinked(link, get_file_size, get_mtime, final_iso_path) self.assertImageAdded(self.compose, ImageCls, iso) diff --git a/tests/test_ostree_phase.py b/tests/test_ostree_phase.py index 0861d9e5..ec458c76 100644 --- a/tests/test_ostree_phase.py +++ b/tests/test_ostree_phase.py @@ -95,6 +95,7 @@ class OSTreeThreadTest(helpers.PungiTestCase): @mock.patch('pungi.wrappers.kojiwrapper.KojiWrapper') def test_run(self, KojiWrapper, get_dir_from_scm): get_dir_from_scm.side_effect = self._dummy_config_repo + self.compose.conf['runroot_weights'] = {'ostree': 123} koji = KojiWrapper.return_value koji.run_runroot_cmd.side_effect = self._mock_runroot(0) @@ -118,7 +119,7 @@ class OSTreeThreadTest(helpers.PungiTestCase): '--extra-config=%s/extra_config.json' % (self.topdir + '/work/ostree-1')], channel=None, mounts=[self.topdir, self.repo], packages=['pungi', 'ostree', 'rpm-ostree'], - task_id=True, use_shell=True, new_chroot=True)]) + task_id=True, use_shell=True, new_chroot=True, weight=123)]) self.assertEqual(koji.run_runroot_cmd.call_args_list, [mock.call(koji.get_runroot_cmd.return_value, log_file=self.topdir + '/logs/x86_64/Everything/ostree-1/runroot.log')]) @@ -255,7 +256,7 @@ class OSTreeThreadTest(helpers.PungiTestCase): '--update-summary'], channel=None, mounts=[self.topdir, self.repo], packages=['pungi', 'ostree', 'rpm-ostree'], - task_id=True, use_shell=True, new_chroot=True)]) + task_id=True, use_shell=True, new_chroot=True, weight=None)]) self.assertEqual(koji.run_runroot_cmd.call_args_list, [mock.call(koji.get_runroot_cmd.return_value, log_file=self.topdir + '/logs/x86_64/Everything/ostree-1/runroot.log')]) @@ -290,7 +291,7 @@ class OSTreeThreadTest(helpers.PungiTestCase): '--extra-config=%s/work/ostree-1/extra_config.json' % self.topdir], channel=None, mounts=[self.topdir, self.repo], packages=['pungi', 'ostree', 'rpm-ostree'], - task_id=True, use_shell=True, new_chroot=True)]) + task_id=True, use_shell=True, new_chroot=True, weight=None)]) self.assertEqual(koji.run_runroot_cmd.call_args_list, [mock.call(koji.get_runroot_cmd.return_value, log_file=self.topdir + '/logs/x86_64/Everything/ostree-1/runroot.log')])