almalinux/pungi
14
1
Fork 0
Code Issues Pull Requests Releases Activity

buildinstall: Allow using external dire for runroot task

Browse Source 
A new `buildinstall_topdir` option allows using buildinstall even when
the compose is created on a different volume that Koji is using.

The files are created in this external directory and then copies into
the usual location.

Merges: https://pagure.io/pungi/pull-request/807
Signed-off-by: Jan Kaluza <jkaluza@redhat.com>
This commit is contained in:
Jan Kaluza 2017-11-22 10:50:14 +01:00 committed by Lubomír Sedlář
parent cf114a7fab
commit 953fb4c54c
5 changed files with 189 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
doc/configuration.rst
View File

@ -496,6 +496,14 @@ Options
**buildinstall_kickstart**
(:ref:`scm_dict <scm_support>`) -- If specified, this kickstart file will
be copied into each file and pointed to in boot configuration.
**buildinstall_topdir**
(*str*) -- Full path to top directory where the runroot buildinstall
Koji tasks output should be stored. This is useful in situation when
the Pungi compose is not generated on the same storage as the Koji task
is running on. In this case, Pungi can provide input repository for runroot
task using HTTP and set the output directory for this task to
``buildinstall_topdir``. Once the runroot task finishes, Pungi will copy
the results of runroot tasks to the compose working directory.
Example
-------

1
pungi/checks.py
View File

@ -709,6 +709,7 @@ def make_schema():
"type": "string",
"enum": ["lorax", "buildinstall"],
},
"buildinstall_topdir": {"type": "string"},
"buildinstall_kickstart": {"$ref": "#/definitions/str_or_scm_dict"},
"global_ksurl": {"type": "string"},

18
pungi/paths.py
View File

@ -203,14 +203,28 @@ class WorkPaths(object):
makedirs(path)
return path
def buildinstall_dir(self, arch, create_dir=True):
def buildinstall_dir(self, arch, create_dir=True,
allow_topdir_override=False, variant=None):
"""
:param bool allow_topdir_override: When True, the
"buildinstall_topdir" will be used (if set) instead of real
"topdir".
Examples:
work/x86_64/buildinstall
"""
if arch == "global":
raise RuntimeError("Global buildinstall dir makes no sense.")
path = os.path.join(self.topdir(arch, create_dir=create_dir), "buildinstall")
buildinstall_topdir = self.compose.conf.get("buildinstall_topdir", "")
if allow_topdir_override and buildinstall_topdir:
topdir_basename = os.path.basename(self.compose.topdir)
path = os.path.join(
buildinstall_topdir, "buildinstall-%s" % topdir_basename, arch)
else:
path = os.path.join(self.topdir(arch, create_dir=create_dir), "buildinstall")
if variant:
path = os.path.join(path, variant.uid)
return path
def extra_files_dir(self, arch, variant, create_dir=True):

45
pungi/phases/buildinstall.py
View File

@ -28,6 +28,7 @@ from six.moves import shlex_quote
from pungi.arch import get_valid_arches
from pungi.util import get_volid, get_arch_variant_data
from pungi.util import get_file_size, get_mtime, failable, makedirs
from pungi.util import copy_all, translate_path
from pungi.wrappers.lorax import LoraxWrapper
from pungi.wrappers.kojiwrapper import get_buildroot_rpms, KojiWrapper
from pungi.wrappers import iso
@ -74,13 +75,24 @@ class BuildinstallPhase(PhaseBase):
add_template_var.extend(data.get('add_template_var', []))
add_arch_template_var.extend(data.get('add_arch_template_var', []))
output_dir = os.path.join(output_dir, variant.uid)
output_topdir = output_dir
# The paths module will modify the filename (by inserting arch). But we
# only care about the directory anyway.
log_filename = 'buildinstall-%s-logs/dumym' % variant.uid
log_filename = 'buildinstall-%s-logs/dummy' % variant.uid
log_dir = os.path.dirname(self.compose.paths.log.log_file(arch, log_filename))
makedirs(log_dir)
# If the buildinstall_topdir is set, it means Koji is used for
# buildinstall phase and the filesystem with Koji is read-only.
# In that case, we have to write logs to buildinstall_topdir and
# later copy them back to our local log directory.
if self.compose.conf.get("buildinstall_topdir", None):
log_dir = self.compose.paths.work.buildinstall_dir(
arch, allow_topdir_override=True, variant=variant)
log_dir = os.path.join(log_dir, "logs")
output_dir = os.path.join(output_dir, "results")
lorax = LoraxWrapper()
lorax_cmd = lorax.get_lorax_cmd(self.compose.conf["release_name"],
self.compose.conf["release_version"],
@ -100,7 +112,7 @@ class BuildinstallPhase(PhaseBase):
add_arch_template_var=add_arch_template_var,
noupgrade=noupgrade,
log_dir=log_dir)
return 'rm -rf %s && %s' % (shlex_quote(output_dir),
return 'rm -rf %s && %s' % (shlex_quote(output_topdir),
' '.join([shlex_quote(x) for x in lorax_cmd]))
def run(self):
@ -114,8 +126,11 @@ class BuildinstallPhase(PhaseBase):
for arch in self.compose.get_arches():
commands = []
output_dir = self.compose.paths.work.buildinstall_dir(arch, allow_topdir_override=True)
final_output_dir = self.compose.paths.work.buildinstall_dir(arch, allow_topdir_override=False)
repo_baseurl = self.compose.paths.work.arch_repo(arch)
output_dir = self.compose.paths.work.buildinstall_dir(arch)
if final_output_dir != output_dir:
repo_baseurl = translate_path(self.compose, repo_baseurl)
if buildinstall_method == "lorax":
buildarch = get_valid_arches(arch)[0]
@ -388,11 +403,13 @@ class BuildinstallThread(WorkerThread):
msg = "Running buildinstall for arch %s, variant %s" % (arch, variant)
output_dir = compose.paths.work.buildinstall_dir(arch)
if variant:
output_dir = os.path.join(output_dir, variant.uid)
output_dir = compose.paths.work.buildinstall_dir(
arch, allow_topdir_override=True, variant=variant)
final_output_dir = compose.paths.work.buildinstall_dir(
arch, variant=variant)
if os.path.isdir(output_dir) and os.listdir(output_dir):
if (os.path.isdir(output_dir) and os.listdir(output_dir) or
os.path.isdir(final_output_dir) and os.listdir(final_output_dir)):
# output dir is *not* empty -> SKIP
self.pool.log_warning(
'[SKIP ] Buildinstall for arch %s, variant %s' % (arch, variant))
@ -433,6 +450,20 @@ class BuildinstallThread(WorkerThread):
# run locally
run(cmd, show_cmd=True, logfile=log_file)
if final_output_dir != output_dir:
if not os.path.exists(final_output_dir):
makedirs(final_output_dir)
results_dir = os.path.join(output_dir, "results")
copy_all(results_dir, final_output_dir)
# Get the log_dir into which we should copy the resulting log files.
log_fname = 'buildinstall-%s-logs/dummy' % variant.uid
final_log_dir = os.path.dirname(compose.paths.log.log_file(arch, log_fname))
if not os.path.exists(final_log_dir):
makedirs(final_log_dir)
log_dir = os.path.join(output_dir, "logs")
copy_all(log_dir, final_log_dir)
log_file = compose.paths.log.log_file(arch, log_filename + '-RPMs')
rpms = get_buildroot_rpms(compose, task_id)
with open(log_file, "w") as f:

126
tests/test_buildinstall.py
View File

@ -345,6 +345,76 @@ class TestBuildinstallPhase(PungiTestCase):
mock.call(compose, 'amd64', variant=compose.variants['Server'], disc_type='dvd')])
@mock.patch('pungi.phases.buildinstall.ThreadPool')
@mock.patch('pungi.phases.buildinstall.LoraxWrapper')
@mock.patch('pungi.phases.buildinstall.get_volid')
def test_uses_lorax_options_buildinstall_topdir(self, get_volid, loraxCls, poolCls):
compose = BuildInstallCompose(self.topdir, {
'bootable': True,
'release_name': 'Test',
'release_short': 't',
'release_version': '1',
'release_is_layered': False,
'buildinstall_method': 'lorax',
'buildinstall_topdir': '/buildinstall_topdir',
'translate_paths': [(self.topdir, "http://localhost/")],
})
buildinstall_topdir = os.path.join(
"/buildinstall_topdir", "buildinstall-" + os.path.basename(self.topdir))
self.maxDiff = None
get_volid.return_value = 'vol_id'
loraxCls.return_value.get_lorax_cmd.return_value = ['lorax', '...']
phase = BuildinstallPhase(compose)
phase.run()
# Three items added for processing in total.
# Server.x86_64, Client.amd64, Server.x86_64
pool = poolCls.return_value
self.assertEqual(3, len(pool.queue_put.mock_calls))
self.assertItemsEqual(
[call[0][0][3] for call in pool.queue_put.call_args_list],
['rm -rf %s/amd64/Client && lorax ...' % buildinstall_topdir,
'rm -rf %s/amd64/Server && lorax ...' % buildinstall_topdir,
'rm -rf %s/x86_64/Server && lorax ...' % buildinstall_topdir])
# Obtained correct lorax commands.
self.assertItemsEqual(
loraxCls.return_value.get_lorax_cmd.mock_calls,
[mock.call('Test', '1', '1', 'http://localhost/work/x86_64/repo',
buildinstall_topdir + '/x86_64/Server/results',
buildarch='x86_64', is_final=True, nomacboot=True, noupgrade=True,
volid='vol_id', variant='Server', buildinstallpackages=['bash', 'vim'],
add_template=[], add_arch_template=[],
add_template_var=[], add_arch_template_var=[],
bugurl=None,
log_dir=buildinstall_topdir + '/x86_64/Server/logs'),
mock.call('Test', '1', '1', 'http://localhost/work/amd64/repo',
buildinstall_topdir + '/amd64/Server/results',
buildarch='amd64', is_final=True, nomacboot=True, noupgrade=True,
volid='vol_id', variant='Server', buildinstallpackages=['bash', 'vim'],
bugurl=None,
add_template=[], add_arch_template=[],
add_template_var=[], add_arch_template_var=[],
log_dir=buildinstall_topdir + '/amd64/Server/logs'),
mock.call('Test', '1', '1', 'http://localhost/work/amd64/repo',
buildinstall_topdir + '/amd64/Client/results',
buildarch='amd64', is_final=True, nomacboot=True, noupgrade=True,
volid='vol_id', variant='Client', buildinstallpackages=[],
bugurl=None,
add_template=[], add_arch_template=[],
add_template_var=[], add_arch_template_var=[],
log_dir=buildinstall_topdir + '/amd64/Client/logs')])
self.assertItemsEqual(
get_volid.mock_calls,
[mock.call(compose, 'x86_64', variant=compose.variants['Server'], disc_type='dvd'),
mock.call(compose, 'amd64', variant=compose.variants['Client'], disc_type='dvd'),
mock.call(compose, 'amd64', variant=compose.variants['Server'], disc_type='dvd')])
class TestCopyFiles(PungiTestCase):
@mock.patch('pungi.phases.buildinstall.link_boot_iso')
@ -706,6 +776,62 @@ class BuildinstallThreadTestCase(PungiTestCase):
self.assertTrue(os.path.exists(dummy_file))
self.assertItemsEqual(self.pool.finished_tasks, [])
@mock.patch('pungi.phases.buildinstall.KojiWrapper')
@mock.patch('pungi.phases.buildinstall.get_buildroot_rpms')
@mock.patch('pungi.phases.buildinstall.run')
@mock.patch('pungi.phases.buildinstall.copy_all')
def test_buildinstall_thread_with_lorax_custom_buildinstall_topdir(
self, copy_all, run, get_buildroot_rpms, KojiWrapperMock):
compose = BuildInstallCompose(self.topdir, {
'buildinstall_method': 'lorax',
'runroot': True,
'runroot_tag': 'rrt',
'koji_profile': 'koji',
'runroot_weights': {'buildinstall': 123},
'buildinstall_topdir': '/buildinstall_topdir',
})
get_buildroot_rpms.return_value = ['bash', 'zsh']
get_runroot_cmd = KojiWrapperMock.return_value.get_runroot_cmd
run_runroot_cmd = KojiWrapperMock.return_value.run_runroot_cmd
run_runroot_cmd.return_value = {
'output': 'Foo bar baz',
'retcode': 0,
'task_id': 1234,
}
t = BuildinstallThread(self.pool)
with mock.patch('time.sleep'):
t.process((compose, 'x86_64', compose.variants['Server'], self.cmd), 0)
self.assertItemsEqual(
get_runroot_cmd.mock_calls,
[mock.call('rrt', 'x86_64', self.cmd, channel=None,
use_shell=True, task_id=True,
packages=['strace', 'lorax'], mounts=[self.topdir], weight=123)])
self.assertItemsEqual(
run_runroot_cmd.mock_calls,
[mock.call(get_runroot_cmd.return_value,
log_file=self.topdir + '/logs/x86_64/buildinstall-Server.x86_64.log')])
with open(self.topdir + '/logs/x86_64/buildinstall-Server-RPMs.x86_64.log') as f:
rpms = f.read().strip().split('\n')
self.assertItemsEqual(rpms, ['bash', 'zsh'])
self.assertItemsEqual(self.pool.finished_tasks, [('Server', 'x86_64')])
buildinstall_topdir = os.path.join(
"/buildinstall_topdir", "buildinstall-" + os.path.basename(self.topdir))
self.assertItemsEqual(
copy_all.mock_calls,
[mock.call(os.path.join(buildinstall_topdir, 'x86_64/Server/results'),
os.path.join(self.topdir, 'work/x86_64/buildinstall/Server')),
mock.call(os.path.join(buildinstall_topdir, 'x86_64/Server/logs'),
os.path.join(self.topdir, 'logs/x86_64/buildinstall-Server-logs'))
]
)
class TestSymlinkIso(PungiTestCase):