almalinux
/
pungi
15
1
Fork 0
Code Issues Pull Requests Releases Activity

Add support for new Pungi Buildinstall Koji plugin. 

We would like to start generating the buildinstall phase using the safer
Koji Pungi Buildinstall plugin and stop the direct use of Runroot plugin.

The plugin so far exists only as PR for Koji:
https://pagure.io/koji/pull-request/1939

This commit adds support for this plugin when `lorax_use_koji_plugin`
is set to `True`.

Signed-off-by: Jan Kaluza <jkaluza@redhat.com>
This commit is contained in:
Jan Kaluza 2020-01-22 07:55:37 +01:00
parent 6eb6511aa6
commit 3cde5c3a87
6 changed files with 304 additions and 40 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
doc/configuration.rst
View File

@ -550,6 +550,10 @@ Options
**lorax_extra_sources**
(*list*) -- a variant/arch mapping with urls for extra source repositories
added to Lorax command line. Either one repo or a list can be specified.
**lorax_use_koji_plugin** = False
(*bool*) -- When set to ``True``, the Koji pungi_buildinstall task will be
used to execute Lorax instead of runroot. Use only if the Koji instance
has the pungi_buildinstall plugin installed.
**buildinstall_kickstart**
(:ref:`scm_dict <scm_support>`) -- If specified, this kickstart file will
be copied into each file and pointed to in boot configuration.

4
pungi/checks.py
View File

@ -1262,6 +1262,10 @@ def make_schema():
"lorax_extra_sources": _variant_arch_mapping({
"$ref": "#/definitions/strings",
}),
"lorax_use_koji_plugin": {
"type": "boolean",
"default": False,
},
"signing_key_id": {"type": "string"},
"signing_key_password_file": {"type": "string"},

110
pungi/phases/buildinstall.py
View File

@ -21,7 +21,7 @@ import shutil
import re
from kobo.threads import ThreadPool, WorkerThread
from kobo.shortcuts import run
from kobo.shortcuts import run, force_list
from productmd.images import Image
from six.moves import shlex_quote
@ -46,6 +46,7 @@ class BuildinstallPhase(PhaseBase):
# is needed to skip copying files for failed tasks.
self.pool.finished_tasks = set()
self.buildinstall_method = self.compose.conf.get("buildinstall_method")
self.lorax_use_koji_plugin = self.compose.conf.get("lorax_use_koji_plugin")
self.used_lorax = self.buildinstall_method == 'lorax'
self.pkgset_phase = pkgset_phase
@ -98,13 +99,6 @@ class BuildinstallPhase(PhaseBase):
# only care about the directory anyway.
log_dir = _get_log_dir(self.compose, variant, arch)
# If the buildinstall_topdir is set, it means Koji is used for
# buildinstall phase and the filesystem with Koji is read-only.
# In that case, we have to write logs to buildinstall_topdir and
# later copy them back to our local log directory.
if self.compose.conf.get("buildinstall_topdir", None):
output_dir = os.path.join(output_dir, "results")
repos = repo_baseurl[:]
repos.extend(
get_arch_variant_data(self.compose.conf, "lorax_extra_sources", arch, variant)
@ -115,31 +109,61 @@ class BuildinstallPhase(PhaseBase):
comps_repo = translate_path(self.compose, comps_repo)
repos.append(comps_repo)
lorax = LoraxWrapper()
lorax_cmd = lorax.get_lorax_cmd(
self.compose.conf["release_name"],
version,
version,
repos,
output_dir,
variant=variant.uid,
buildinstallpackages=variant.buildinstallpackages,
is_final=self.compose.supported,
buildarch=buildarch,
volid=volid,
nomacboot=nomacboot,
bugurl=bugurl,
add_template=add_template,
add_arch_template=add_arch_template,
add_template_var=add_template_var,
add_arch_template_var=add_arch_template_var,
noupgrade=noupgrade,
rootfs_size=rootfs_size,
log_dir=log_dir,
dracut_args=dracut_args,
)
return 'rm -rf %s && %s' % (shlex_quote(output_topdir),
' '.join([shlex_quote(x) for x in lorax_cmd]))
if self.lorax_use_koji_plugin:
return {
"product": self.compose.conf["release_name"],
"version": version,
"release": version,
"sources": force_list(repos),
"variant": variant.uid,
"installpkgs": variant.buildinstallpackages,
"isfinal": self.compose.supported,
"buildarch": buildarch,
"volid": volid,
"nomacboot": nomacboot,
"bugurl": bugurl,
"add-template": add_template,
"add-arch-template": add_arch_template,
"add-template-var": add_template_var,
"add-arch-template-var": add_arch_template_var,
"noupgrade": noupgrade,
"rootfs-size": rootfs_size,
"dracut-args": dracut_args,
"outputdir": output_dir,
}
else:
# If the buildinstall_topdir is set, it means Koji is used for
# buildinstall phase and the filesystem with Koji is read-only.
# In that case, we have to write logs to buildinstall_topdir and
# later copy them back to our local log directory.
if self.compose.conf.get("buildinstall_topdir", None):
output_dir = os.path.join(output_dir, "results")
lorax = LoraxWrapper()
lorax_cmd = lorax.get_lorax_cmd(
self.compose.conf["release_name"],
version,
version,
repos,
output_dir,
variant=variant.uid,
buildinstallpackages=variant.buildinstallpackages,
is_final=self.compose.supported,
buildarch=buildarch,
volid=volid,
nomacboot=nomacboot,
bugurl=bugurl,
add_template=add_template,
add_arch_template=add_arch_template,
add_template_var=add_template_var,
add_arch_template_var=add_arch_template_var,
noupgrade=noupgrade,
rootfs_size=rootfs_size,
log_dir=log_dir,
dracut_args=dracut_args,
)
return 'rm -rf %s && %s' % (shlex_quote(output_topdir),
' '.join([shlex_quote(x) for x in lorax_cmd]))
def get_repos(self, arch):
repos = []
@ -423,6 +447,7 @@ class BuildinstallThread(WorkerThread):
def worker(self, compose, arch, variant, cmd, num):
buildinstall_method = compose.conf["buildinstall_method"]
lorax_use_koji_plugin = compose.conf["lorax_use_koji_plugin"]
log_filename = ('buildinstall-%s' % variant.uid) if variant else 'buildinstall'
log_file = compose.paths.log.log_file(arch, log_filename)
@ -458,12 +483,19 @@ class BuildinstallThread(WorkerThread):
# Start the runroot task.
runroot = Runroot(compose, phase="buildinstall")
runroot.run(
cmd, log_file=log_file, arch=arch, packages=packages,
mounts=[compose.topdir],
weight=compose.conf['runroot_weights'].get('buildinstall'),
chown_paths=chown_paths,
)
if buildinstall_method == "lorax" and lorax_use_koji_plugin:
runroot.run_pungi_buildinstall(
cmd, log_file=log_file, arch=arch, packages=packages,
mounts=[compose.topdir],
weight=compose.conf['runroot_weights'].get('buildinstall'),
)
else:
runroot.run(
cmd, log_file=log_file, arch=arch, packages=packages,
mounts=[compose.topdir],
weight=compose.conf['runroot_weights'].get('buildinstall'),
chown_paths=chown_paths,
)
if final_output_dir != output_dir:
if not os.path.exists(final_output_dir):

29
pungi/runroot.py
View File

@ -236,6 +236,35 @@ class Runroot(kobo.log.LoggingBase):
else:
raise ValueError("Unknown runroot_method %r." % self.runroot_method)
def run_pungi_buildinstall(self, args, log_file=None, arch=None, **kwargs):
"""
Runs the Lorax buildinstall runroot command using the Pungi Buildinstall
Koji plugin as pungi_buildinstall task.
The **kwargs are optional and matches the
`KojiWrapper.get_pungi_buildinstall_cmd()` kwargs.
:param dict args: Arguments for the pungi_buildinstall Koji task.
:param str log_file: Log file into which the output of the task will
be logged.
:param str arch: Architecture on which the task should be executed.
"""
runroot_channel = self.compose.conf.get("runroot_channel")
runroot_tag = self.compose.conf["runroot_tag"]
koji_wrapper = kojiwrapper.KojiWrapper(self.compose.conf["koji_profile"])
koji_cmd = koji_wrapper.get_pungi_buildinstall_cmd(
runroot_tag, arch, args, channel=runroot_channel,
chown_uid=os.getuid(), **kwargs)
output = koji_wrapper.run_runroot_cmd(koji_cmd, log_file=log_file)
if output["retcode"] != 0:
raise RuntimeError(
"Pungi-buildinstall task failed: %s. See %s for more details."
% (output["task_id"], log_file)
)
self._result = output
def get_buildroot_rpms(self):
"""
Returns the list of RPMs installed in a buildroot in which the runroot

49
pungi/wrappers/kojiwrapper.py
View File

@ -21,7 +21,7 @@ import threading
import contextlib
import koji
from kobo.shortcuts import run
from kobo.shortcuts import run, force_list
import six
from six.moves import configparser, shlex_quote
import six.moves.xmlrpc_client as xmlrpclib
@ -123,6 +123,47 @@ class KojiWrapper(object):
return cmd
def get_pungi_buildinstall_cmd(
self, target, arch, args, channel=None, packages=None,
mounts=None, weight=None, chown_uid=None):
cmd = self._get_cmd("pungi-buildinstall", "--nowait", "--task-id")
if channel:
cmd.append("--channel-override=%s" % channel)
else:
cmd.append("--channel-override=runroot-local")
if weight:
cmd.append("--weight=%s" % int(weight))
for package in packages or []:
cmd.append("--package=%s" % package)
for mount in mounts or []:
# directories are *not* created here
cmd.append("--mount=%s" % mount)
if chown_uid:
cmd.append("--chown-uid=%s" % chown_uid)
# IMPORTANT: all --opts have to be provided *before* args
cmd.append(target)
# i686 -> i386 etc.
arch = getBaseArch(arch)
cmd.append(arch)
for k, v in args.items():
if v:
if isinstance(v, bool):
cmd.append(k)
else:
for arg in force_list(v):
cmd.append("%s=%s" % (k, shlex_quote(arg)))
return cmd
@contextlib.contextmanager
def get_koji_cmd_env(self):
"""Get environment variables for running a koji command.
@ -606,6 +647,12 @@ def get_buildroot_rpms(compose, task_id):
# runroot
koji = KojiWrapper(compose.conf['koji_profile'])
buildroot_infos = koji.koji_proxy.listBuildroots(taskID=task_id)
if not buildroot_infos:
children_tasks = koji.koji_proxy.getTaskChildren(task_id)
for child_task in children_tasks:
buildroot_infos = koji.koji_proxy.listBuildroots(taskID=child_task["id"])
if buildroot_infos:
break
buildroot_info = buildroot_infos[-1]
data = koji.koji_proxy.listRPMs(componentBuildrootID=buildroot_info["id"])
for rpm_info in data:

148
tests/test_buildinstall.py
View File

@ -157,6 +157,85 @@ class TestBuildinstallPhase(PungiTestCase):
mock.call(compose, 'amd64', variant=compose.variants['Client'], disc_type='DVD'),
mock.call(compose, 'amd64', variant=compose.variants['Server'], disc_type='DVD')])
@mock.patch('pungi.phases.buildinstall.ThreadPool')
@mock.patch('pungi.phases.buildinstall.get_volid')
def test_starts_threads_for_each_cmd_with_lorax_koji_plugin(
self, get_volid, poolCls):
compose = BuildInstallCompose(self.topdir, {
'bootable': True,
'release_name': 'Test',
'release_short': 't',
'release_version': '1',
'buildinstall_method': 'lorax',
'lorax_use_koji_plugin': True,
'disc_types': {'dvd': 'DVD'},
})
get_volid.return_value = 'vol_id'
phase = BuildinstallPhase(compose, self._make_pkgset_phase(["p1", "p2"]))
phase.run()
self.maxDiff = None
expected_args = [
{
'product': 'Test', 'version': '1', 'release': '1',
'sources': [self.topdir + "/work/amd64/repo/p1",
self.topdir + "/work/amd64/repo/p2",
self.topdir + '/work/amd64/comps_repo_Server'],
'variant': 'Server', 'installpkgs': ['bash', 'vim'],
'isfinal': True, 'buildarch': 'amd64', 'volid': 'vol_id',
'nomacboot': True, 'bugurl': None, 'add-template': [],
'add-arch-template': [], 'add-template-var': [],
'add-arch-template-var': [], 'noupgrade': True,
'rootfs-size': None, 'dracut-args': [],
'outputdir': self.topdir + '/work/amd64/buildinstall/Server'
},
{
'product': 'Test', 'version': '1', 'release': '1',
'sources': [self.topdir + "/work/amd64/repo/p1",
self.topdir + "/work/amd64/repo/p2",
self.topdir + '/work/amd64/comps_repo_Client'],
'variant': 'Client', 'installpkgs': [],
'isfinal': True, 'buildarch': 'amd64', 'volid': 'vol_id',
'nomacboot': True, 'bugurl': None, 'add-template': [],
'add-arch-template': [], 'add-template-var': [],
'add-arch-template-var': [], 'noupgrade': True,
'rootfs-size': None, 'dracut-args': [],
'outputdir': self.topdir + '/work/amd64/buildinstall/Client'
},
{
'product': 'Test', 'version': '1', 'release': '1',
'sources': [self.topdir + "/work/x86_64/repo/p1",
self.topdir + "/work/x86_64/repo/p2",
self.topdir + '/work/x86_64/comps_repo_Server'],
'variant': 'Server', 'installpkgs': ['bash', 'vim'],
'isfinal': True, 'buildarch': 'x86_64', 'volid': 'vol_id',
'nomacboot': True, 'bugurl': None, 'add-template': [],
'add-arch-template': [], 'add-template-var': [],
'add-arch-template-var': [], 'noupgrade': True,
'rootfs-size': None, 'dracut-args': [],
'outputdir': self.topdir + '/work/x86_64/buildinstall/Server'
},
]
# Three items added for processing in total.
# Server.x86_64, Client.amd64, Server.x86_64
pool = poolCls.return_value
self.assertEqual(3, len(pool.queue_put.mock_calls))
six.assertCountEqual(
self,
[call[0][0][3] for call in pool.queue_put.call_args_list],
expected_args)
six.assertCountEqual(
self,
get_volid.mock_calls,
[mock.call(compose, 'x86_64', variant=compose.variants['Server'], disc_type='DVD'),
mock.call(compose, 'amd64', variant=compose.variants['Client'], disc_type='DVD'),
mock.call(compose, 'amd64', variant=compose.variants['Server'], disc_type='DVD')])
@mock.patch('pungi.phases.buildinstall.ThreadPool')
@mock.patch('pungi.phases.buildinstall.LoraxWrapper')
@mock.patch('pungi.phases.buildinstall.get_volid')
@ -659,6 +738,75 @@ class BuildinstallThreadTestCase(PungiTestCase):
[mock.call(compose, "x86_64", compose.variants["Server"], False)],
)
@mock.patch('pungi.phases.buildinstall.link_boot_iso')
@mock.patch('pungi.phases.buildinstall.tweak_buildinstall')
@mock.patch('pungi.wrappers.kojiwrapper.KojiWrapper')
@mock.patch('pungi.wrappers.kojiwrapper.get_buildroot_rpms')
@mock.patch('pungi.phases.buildinstall.run')
def test_buildinstall_thread_with_lorax_using_koji_plugin(
self, run, get_buildroot_rpms, KojiWrapperMock, mock_tweak, mock_link
):
compose = BuildInstallCompose(self.topdir, {
'buildinstall_method': 'lorax',
'lorax_use_koji_plugin': True,
'runroot_tag': 'rrt',
'koji_profile': 'koji',
'runroot_weights': {'buildinstall': 123},
})
get_buildroot_rpms.return_value = ['bash', 'zsh']
get_pungi_buildinstall_cmd = KojiWrapperMock.return_value.get_pungi_buildinstall_cmd
run_runroot_cmd = KojiWrapperMock.return_value.run_runroot_cmd
run_runroot_cmd.return_value = {
'output': 'Foo bar baz',
'retcode': 0,
'task_id': 1234,
}
t = BuildinstallThread(self.pool)
with mock.patch('time.sleep'):
t.process((compose, 'x86_64', compose.variants['Server'], self.cmd), 0)
destdir = os.path.join(self.topdir, "work/x86_64/buildinstall/Server")
self.assertEqual(
get_pungi_buildinstall_cmd.mock_calls,
[mock.call(
'rrt', 'x86_64', self.cmd, channel=None,
packages=['lorax'], mounts=[self.topdir],
weight=123, chown_uid=os.getuid()
)])
self.assertEqual(
run_runroot_cmd.mock_calls,
[mock.call(get_pungi_buildinstall_cmd.return_value,
log_file=self.topdir + '/logs/x86_64/buildinstall-Server.x86_64.log')])
with open(self.topdir + '/logs/x86_64/buildinstall-Server-RPMs.x86_64.log') as f:
rpms = f.read().strip().split('\n')
six.assertCountEqual(self, rpms, ["bash", "zsh"])
six.assertCountEqual(self, self.pool.finished_tasks, [("Server", "x86_64")])
self.assertEqual(
mock_tweak.call_args_list,
[
mock.call(
compose,
destdir,
os.path.join(self.topdir, "compose/Server/x86_64/os"),
"x86_64",
"Server",
"",
"dummy-volid",
self.pool.kickstart_file,
)
],
)
self.assertEqual(
mock_link.call_args_list,
[mock.call(compose, "x86_64", compose.variants["Server"], False)],
)
@mock.patch('pungi.phases.buildinstall.link_boot_iso')
@mock.patch('pungi.phases.buildinstall.tweak_buildinstall')
@mock.patch('pungi.wrappers.kojiwrapper.KojiWrapper')