almalinux
/
pungi
15
1
Fork 0
Code Issues Pull Requests Releases Activity

Enable customizing runroot task weight 

For different cases where runroot is used it's now possible to set
custom weight. The usecase for this is to avoid one builder taking too
many tasks. Especially buildinstall is quite resource intensive, so one
builder taking multiple tasks at the same time leads to very slow
compose time.

Signed-off-by: Lubomír Sedlář <lsedlar@redhat.com>
This commit is contained in:
Lubomír Sedlář 2017-02-23 10:28:49 +01:00
parent d9ab899920
commit 4ea1916a87
10 changed files with 52 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
doc/configuration.rst
View File

@ -680,6 +680,17 @@ Options
**runroot_tag**
(*str*) -- name of koji **build** tag used for runroot
**runroot_weights**
(*dict*) -- customize task weights for various runroot tasks. The values in
the mapping should be integers, the keys can be selected from the following
list. By default no weight is assigned and Koji picks the default one
according to policy.
* ``buildinstall``
* ``createiso``
* ``ostree``
* ``ostree_installer``
Example
-------

13
pungi/checks.py
View File

@ -531,6 +531,17 @@ def _make_schema():
"runroot_channel": {
"$ref": "#/definitions/optional_string",
},
"runroot_weights": {
"type": "object",
"default": {},
"properties": {
"buildinstall": {"type": "number"},
"createiso": {"type": "number"},
"ostree": {"type": "number"},
"ostree_installer": {"type": "number"},
},
"additionalProperties": False,
},
"createrepo_deltas": {
"type": "boolean",
"default": False,
@ -969,7 +980,7 @@ CONFIG_DEPS = {
(lambda x: x, ["koji_profile", "runroot_tag", "runroot_channel"]),
),
"conflicts": (
(lambda x: not x, ["runroot_tag", "runroot_channel"]),
(lambda x: not x, ["runroot_tag", "runroot_channel", "runroot_weights"]),
),
},
"product_id": {

10
pungi/phases/buildinstall.py
View File

@ -390,10 +390,12 @@ class BuildinstallThread(WorkerThread):
runroot_tag = compose.conf["runroot_tag"]
koji_wrapper = KojiWrapper(compose.conf["koji_profile"])
koji_cmd = koji_wrapper.get_runroot_cmd(runroot_tag, arch, cmd,
channel=runroot_channel,
use_shell=True, task_id=True,
packages=packages, mounts=[compose.topdir])
koji_cmd = koji_wrapper.get_runroot_cmd(
runroot_tag, arch, cmd,
channel=runroot_channel,
use_shell=True, task_id=True,
packages=packages, mounts=[compose.topdir],
weight=compose.conf['runroot_weights'].get('buildinstall'))
# avoid race conditions?
# Kerberos authentication failed: Permission denied in replay cache code (-1765328215)

4
pungi/phases/createiso.py
View File

@ -227,7 +227,9 @@ class CreateIsoThread(WorkerThread):
koji_cmd = koji_wrapper.get_runroot_cmd(
runroot_tag, build_arch, cmd["cmd"],
channel=runroot_channel, use_shell=True, task_id=True,
packages=packages, mounts=mounts)
packages=packages, mounts=mounts,
weight=compose.conf['runroot_weights'].get('createiso')
)
# avoid race conditions?
# Kerberos authentication failed: Permission denied in replay cache code (-1765328215)

3
pungi/phases/ostree.py
View File

@ -147,7 +147,8 @@ class OSTreeThread(WorkerThread):
channel=runroot_channel,
use_shell=True, task_id=True,
packages=packages, mounts=mounts,
new_chroot=True)
new_chroot=True,
weight=compose.conf["runroot_weights"].get('ostree'))
output = koji.run_runroot_cmd(koji_cmd, log_file=log_file)
if output["retcode"] != 0:
raise RuntimeError("Runroot task failed: %s. See %s for more details."

3
pungi/phases/ostree_installer.py
View File

@ -170,7 +170,8 @@ class OstreeInstallerThread(WorkerThread):
koji_cmd = koji.get_runroot_cmd(runroot_tag, arch, cmd,
channel=runroot_channel,
use_shell=True, task_id=True,
packages=packages, mounts=[compose.topdir])
packages=packages, mounts=[compose.topdir],
weight=compose.conf['runroot_weights'].get('ostree_installer'))
output = koji.run_runroot_cmd(koji_cmd, log_file=log_file)
if output["retcode"] != 0:
raise RuntimeError("Runroot task failed: %s. See %s for more details."

5
tests/test_buildinstall.py
View File

@ -467,6 +467,7 @@ class BuildinstallThreadTestCase(PungiTestCase):
'runroot': True,
'runroot_tag': 'rrt',
'koji_profile': 'koji',
'runroot_weights': {'buildinstall': 123},
})
get_buildroot_rpms.return_value = ['bash', 'zsh']
@ -491,7 +492,7 @@ class BuildinstallThreadTestCase(PungiTestCase):
get_runroot_cmd.mock_calls,
[mock.call('rrt', 'x86_64', cmd, channel=None,
use_shell=True, task_id=True,
packages=['strace', 'lorax'], mounts=[self.topdir])])
packages=['strace', 'lorax'], mounts=[self.topdir], weight=123)])
self.assertItemsEqual(
run_runroot_cmd.mock_calls,
[mock.call(get_runroot_cmd.return_value,
@ -533,7 +534,7 @@ class BuildinstallThreadTestCase(PungiTestCase):
get_runroot_cmd.mock_calls,
[mock.call('rrt', 'x86_64', cmd, channel=None,
use_shell=True, task_id=True,
packages=['strace', 'anaconda'], mounts=[self.topdir])])
packages=['strace', 'anaconda'], mounts=[self.topdir], weight=None)])
self.assertItemsEqual(
run_runroot_cmd.mock_calls,
[mock.call(get_runroot_cmd.return_value,

7
tests/test_createiso_phase.py
View File

@ -252,7 +252,7 @@ class CreateisoThreadTest(helpers.PungiTestCase):
mounts=[self.topdir],
packages=['coreutils', 'genisoimage', 'isomd5sum',
'jigdo'],
task_id=True, use_shell=True)])
task_id=True, use_shell=True, weight=None)])
self.assertEqual(
run_runroot.call_args_list,
[mock.call(get_runroot_cmd.return_value,
@ -286,6 +286,7 @@ class CreateisoThreadTest(helpers.PungiTestCase):
'runroot_tag': 'f25-build',
'koji_profile': 'koji',
'create_jigdo': False,
'runroot_weights': {'createiso': 123},
})
cmd = {
'iso_path': '%s/compose/Server/x86_64/iso/image-name' % self.topdir,
@ -316,7 +317,7 @@ class CreateisoThreadTest(helpers.PungiTestCase):
[mock.call('f25-build', 'x86_64', cmd['cmd'], channel=None,
mounts=[self.topdir],
packages=['coreutils', 'genisoimage', 'isomd5sum'],
task_id=True, use_shell=True)])
task_id=True, use_shell=True, weight=123)])
self.assertEqual(
run_runroot.call_args_list,
[mock.call(get_runroot_cmd.return_value,
@ -382,7 +383,7 @@ class CreateisoThreadTest(helpers.PungiTestCase):
mounts=[self.topdir],
packages=['coreutils', 'genisoimage', 'isomd5sum',
'jigdo', 'lorax'],
task_id=True, use_shell=True)])
task_id=True, use_shell=True, weight=None)])
self.assertEqual(
run_runroot.call_args_list,
[mock.call(get_runroot_cmd.return_value,

8
tests/test_ostree_installer_phase.py
View File

@ -72,7 +72,7 @@ class OstreeThreadTest(helpers.PungiTestCase):
self.assertEqual(compose.im.add.mock_calls,
[mock.call('Everything', 'x86_64', image)])
def assertRunrootCall(self, koji, sources, release, isfinal=False, extra=[]):
def assertRunrootCall(self, koji, sources, release, isfinal=False, extra=[], weight=None):
lorax_cmd = [
'lorax',
'--product=Fedora',
@ -101,7 +101,7 @@ class OstreeThreadTest(helpers.PungiTestCase):
lorax_cmd,
channel=None, mounts=[self.topdir],
packages=['pungi', 'lorax', 'ostree'],
task_id=True, use_shell=True)])
task_id=True, use_shell=True, weight=weight)])
self.assertEqual(koji.run_runroot_cmd.call_args_list,
[mock.call(koji.get_runroot_cmd.return_value,
log_file=self.topdir + '/logs/x86_64/ostree_installer/runroot.log')])
@ -377,6 +377,7 @@ class OstreeThreadTest(helpers.PungiTestCase):
"ostree_ref=fedora-atomic/Rawhide/x86_64/docker-host",
],
}
self.compose.conf['runroot_weights'] = {'ostree_installer': 123}
koji = KojiWrapper.return_value
koji.run_runroot_cmd.return_value = {
'task_id': 1234,
@ -404,7 +405,8 @@ class OstreeThreadTest(helpers.PungiTestCase):
'--add-arch-template-var=ostree_repo=https://kojipkgs.fedoraproject.org/compose/atomic/Rawhide/',
'--add-arch-template-var=ostree_osname=fedora-atomic',
'--add-arch-template-var=ostree_ref=fedora-atomic/Rawhide/x86_64/docker-host',
'--logfile=%s/logs/x86_64/ostree_installer/lorax.log' % self.topdir]
'--logfile=%s/logs/x86_64/ostree_installer/lorax.log' % self.topdir],
weight=123,
)
self.assertIsoLinked(link, get_file_size, get_mtime, final_iso_path)
self.assertImageAdded(self.compose, ImageCls, iso)

7
tests/test_ostree_phase.py
View File

@ -95,6 +95,7 @@ class OSTreeThreadTest(helpers.PungiTestCase):
@mock.patch('pungi.wrappers.kojiwrapper.KojiWrapper')
def test_run(self, KojiWrapper, get_dir_from_scm):
get_dir_from_scm.side_effect = self._dummy_config_repo
self.compose.conf['runroot_weights'] = {'ostree': 123}
koji = KojiWrapper.return_value
koji.run_runroot_cmd.side_effect = self._mock_runroot(0)
@ -118,7 +119,7 @@ class OSTreeThreadTest(helpers.PungiTestCase):
'--extra-config=%s/extra_config.json' % (self.topdir + '/work/ostree-1')],
channel=None, mounts=[self.topdir, self.repo],
packages=['pungi', 'ostree', 'rpm-ostree'],
task_id=True, use_shell=True, new_chroot=True)])
task_id=True, use_shell=True, new_chroot=True, weight=123)])
self.assertEqual(koji.run_runroot_cmd.call_args_list,
[mock.call(koji.get_runroot_cmd.return_value,
log_file=self.topdir + '/logs/x86_64/Everything/ostree-1/runroot.log')])
@ -255,7 +256,7 @@ class OSTreeThreadTest(helpers.PungiTestCase):
'--update-summary'],
channel=None, mounts=[self.topdir, self.repo],
packages=['pungi', 'ostree', 'rpm-ostree'],
task_id=True, use_shell=True, new_chroot=True)])
task_id=True, use_shell=True, new_chroot=True, weight=None)])
self.assertEqual(koji.run_runroot_cmd.call_args_list,
[mock.call(koji.get_runroot_cmd.return_value,
log_file=self.topdir + '/logs/x86_64/Everything/ostree-1/runroot.log')])
@ -290,7 +291,7 @@ class OSTreeThreadTest(helpers.PungiTestCase):
'--extra-config=%s/work/ostree-1/extra_config.json' % self.topdir],
channel=None, mounts=[self.topdir, self.repo],
packages=['pungi', 'ostree', 'rpm-ostree'],
task_id=True, use_shell=True, new_chroot=True)])
task_id=True, use_shell=True, new_chroot=True, weight=None)])
self.assertEqual(koji.run_runroot_cmd.call_args_list,
[mock.call(koji.get_runroot_cmd.return_value,
log_file=self.topdir + '/logs/x86_64/Everything/ostree-1/runroot.log')])