From 953fb4c54c3539879dc62abe0a4bb5c69f3d7f60 Mon Sep 17 00:00:00 2001 From: Jan Kaluza Date: Wed, 22 Nov 2017 10:50:14 +0100 Subject: [PATCH] buildinstall: Allow using external dire for runroot task A new `buildinstall_topdir` option allows using buildinstall even when the compose is created on a different volume that Koji is using. The files are created in this external directory and then copies into the usual location. Merges: https://pagure.io/pungi/pull-request/807 Signed-off-by: Jan Kaluza --- doc/configuration.rst | 8 +++ pungi/checks.py | 1 + pungi/paths.py | 18 ++++- pungi/phases/buildinstall.py | 45 +++++++++++-- tests/test_buildinstall.py | 126 +++++++++++++++++++++++++++++++++++ 5 files changed, 189 insertions(+), 9 deletions(-) diff --git a/doc/configuration.rst b/doc/configuration.rst index 46c85d2d..c90ee0d7 100644 --- a/doc/configuration.rst +++ b/doc/configuration.rst @@ -496,6 +496,14 @@ Options **buildinstall_kickstart** (:ref:`scm_dict `) -- If specified, this kickstart file will be copied into each file and pointed to in boot configuration. +**buildinstall_topdir** + (*str*) -- Full path to top directory where the runroot buildinstall + Koji tasks output should be stored. This is useful in situation when + the Pungi compose is not generated on the same storage as the Koji task + is running on. In this case, Pungi can provide input repository for runroot + task using HTTP and set the output directory for this task to + ``buildinstall_topdir``. Once the runroot task finishes, Pungi will copy + the results of runroot tasks to the compose working directory. Example ------- diff --git a/pungi/checks.py b/pungi/checks.py index 370e80d4..bfacc283 100644 --- a/pungi/checks.py +++ b/pungi/checks.py @@ -709,6 +709,7 @@ def make_schema(): "type": "string", "enum": ["lorax", "buildinstall"], }, + "buildinstall_topdir": {"type": "string"}, "buildinstall_kickstart": {"$ref": "#/definitions/str_or_scm_dict"}, "global_ksurl": {"type": "string"}, diff --git a/pungi/paths.py b/pungi/paths.py index f32debf1..42d0e3eb 100644 --- a/pungi/paths.py +++ b/pungi/paths.py @@ -203,14 +203,28 @@ class WorkPaths(object): makedirs(path) return path - def buildinstall_dir(self, arch, create_dir=True): + def buildinstall_dir(self, arch, create_dir=True, + allow_topdir_override=False, variant=None): """ + :param bool allow_topdir_override: When True, the + "buildinstall_topdir" will be used (if set) instead of real + "topdir". Examples: work/x86_64/buildinstall """ if arch == "global": raise RuntimeError("Global buildinstall dir makes no sense.") - path = os.path.join(self.topdir(arch, create_dir=create_dir), "buildinstall") + + buildinstall_topdir = self.compose.conf.get("buildinstall_topdir", "") + if allow_topdir_override and buildinstall_topdir: + topdir_basename = os.path.basename(self.compose.topdir) + path = os.path.join( + buildinstall_topdir, "buildinstall-%s" % topdir_basename, arch) + else: + path = os.path.join(self.topdir(arch, create_dir=create_dir), "buildinstall") + + if variant: + path = os.path.join(path, variant.uid) return path def extra_files_dir(self, arch, variant, create_dir=True): diff --git a/pungi/phases/buildinstall.py b/pungi/phases/buildinstall.py index a09d44d3..bbc06ae6 100644 --- a/pungi/phases/buildinstall.py +++ b/pungi/phases/buildinstall.py @@ -28,6 +28,7 @@ from six.moves import shlex_quote from pungi.arch import get_valid_arches from pungi.util import get_volid, get_arch_variant_data from pungi.util import get_file_size, get_mtime, failable, makedirs +from pungi.util import copy_all, translate_path from pungi.wrappers.lorax import LoraxWrapper from pungi.wrappers.kojiwrapper import get_buildroot_rpms, KojiWrapper from pungi.wrappers import iso @@ -74,13 +75,24 @@ class BuildinstallPhase(PhaseBase): add_template_var.extend(data.get('add_template_var', [])) add_arch_template_var.extend(data.get('add_arch_template_var', [])) output_dir = os.path.join(output_dir, variant.uid) + output_topdir = output_dir # The paths module will modify the filename (by inserting arch). But we # only care about the directory anyway. - log_filename = 'buildinstall-%s-logs/dumym' % variant.uid + log_filename = 'buildinstall-%s-logs/dummy' % variant.uid log_dir = os.path.dirname(self.compose.paths.log.log_file(arch, log_filename)) makedirs(log_dir) + # If the buildinstall_topdir is set, it means Koji is used for + # buildinstall phase and the filesystem with Koji is read-only. + # In that case, we have to write logs to buildinstall_topdir and + # later copy them back to our local log directory. + if self.compose.conf.get("buildinstall_topdir", None): + log_dir = self.compose.paths.work.buildinstall_dir( + arch, allow_topdir_override=True, variant=variant) + log_dir = os.path.join(log_dir, "logs") + output_dir = os.path.join(output_dir, "results") + lorax = LoraxWrapper() lorax_cmd = lorax.get_lorax_cmd(self.compose.conf["release_name"], self.compose.conf["release_version"], @@ -100,7 +112,7 @@ class BuildinstallPhase(PhaseBase): add_arch_template_var=add_arch_template_var, noupgrade=noupgrade, log_dir=log_dir) - return 'rm -rf %s && %s' % (shlex_quote(output_dir), + return 'rm -rf %s && %s' % (shlex_quote(output_topdir), ' '.join([shlex_quote(x) for x in lorax_cmd])) def run(self): @@ -114,8 +126,11 @@ class BuildinstallPhase(PhaseBase): for arch in self.compose.get_arches(): commands = [] + output_dir = self.compose.paths.work.buildinstall_dir(arch, allow_topdir_override=True) + final_output_dir = self.compose.paths.work.buildinstall_dir(arch, allow_topdir_override=False) repo_baseurl = self.compose.paths.work.arch_repo(arch) - output_dir = self.compose.paths.work.buildinstall_dir(arch) + if final_output_dir != output_dir: + repo_baseurl = translate_path(self.compose, repo_baseurl) if buildinstall_method == "lorax": buildarch = get_valid_arches(arch)[0] @@ -388,11 +403,13 @@ class BuildinstallThread(WorkerThread): msg = "Running buildinstall for arch %s, variant %s" % (arch, variant) - output_dir = compose.paths.work.buildinstall_dir(arch) - if variant: - output_dir = os.path.join(output_dir, variant.uid) + output_dir = compose.paths.work.buildinstall_dir( + arch, allow_topdir_override=True, variant=variant) + final_output_dir = compose.paths.work.buildinstall_dir( + arch, variant=variant) - if os.path.isdir(output_dir) and os.listdir(output_dir): + if (os.path.isdir(output_dir) and os.listdir(output_dir) or + os.path.isdir(final_output_dir) and os.listdir(final_output_dir)): # output dir is *not* empty -> SKIP self.pool.log_warning( '[SKIP ] Buildinstall for arch %s, variant %s' % (arch, variant)) @@ -433,6 +450,20 @@ class BuildinstallThread(WorkerThread): # run locally run(cmd, show_cmd=True, logfile=log_file) + if final_output_dir != output_dir: + if not os.path.exists(final_output_dir): + makedirs(final_output_dir) + results_dir = os.path.join(output_dir, "results") + copy_all(results_dir, final_output_dir) + + # Get the log_dir into which we should copy the resulting log files. + log_fname = 'buildinstall-%s-logs/dummy' % variant.uid + final_log_dir = os.path.dirname(compose.paths.log.log_file(arch, log_fname)) + if not os.path.exists(final_log_dir): + makedirs(final_log_dir) + log_dir = os.path.join(output_dir, "logs") + copy_all(log_dir, final_log_dir) + log_file = compose.paths.log.log_file(arch, log_filename + '-RPMs') rpms = get_buildroot_rpms(compose, task_id) with open(log_file, "w") as f: diff --git a/tests/test_buildinstall.py b/tests/test_buildinstall.py index 6d0c8a20..450c9a19 100644 --- a/tests/test_buildinstall.py +++ b/tests/test_buildinstall.py @@ -345,6 +345,76 @@ class TestBuildinstallPhase(PungiTestCase): mock.call(compose, 'amd64', variant=compose.variants['Server'], disc_type='dvd')]) + @mock.patch('pungi.phases.buildinstall.ThreadPool') + @mock.patch('pungi.phases.buildinstall.LoraxWrapper') + @mock.patch('pungi.phases.buildinstall.get_volid') + def test_uses_lorax_options_buildinstall_topdir(self, get_volid, loraxCls, poolCls): + compose = BuildInstallCompose(self.topdir, { + 'bootable': True, + 'release_name': 'Test', + 'release_short': 't', + 'release_version': '1', + 'release_is_layered': False, + 'buildinstall_method': 'lorax', + 'buildinstall_topdir': '/buildinstall_topdir', + 'translate_paths': [(self.topdir, "http://localhost/")], + }) + + buildinstall_topdir = os.path.join( + "/buildinstall_topdir", "buildinstall-" + os.path.basename(self.topdir)) + self.maxDiff = None + + get_volid.return_value = 'vol_id' + loraxCls.return_value.get_lorax_cmd.return_value = ['lorax', '...'] + + phase = BuildinstallPhase(compose) + + phase.run() + + # Three items added for processing in total. + # Server.x86_64, Client.amd64, Server.x86_64 + pool = poolCls.return_value + self.assertEqual(3, len(pool.queue_put.mock_calls)) + self.assertItemsEqual( + [call[0][0][3] for call in pool.queue_put.call_args_list], + ['rm -rf %s/amd64/Client && lorax ...' % buildinstall_topdir, + 'rm -rf %s/amd64/Server && lorax ...' % buildinstall_topdir, + 'rm -rf %s/x86_64/Server && lorax ...' % buildinstall_topdir]) + + # Obtained correct lorax commands. + self.assertItemsEqual( + loraxCls.return_value.get_lorax_cmd.mock_calls, + [mock.call('Test', '1', '1', 'http://localhost/work/x86_64/repo', + buildinstall_topdir + '/x86_64/Server/results', + buildarch='x86_64', is_final=True, nomacboot=True, noupgrade=True, + volid='vol_id', variant='Server', buildinstallpackages=['bash', 'vim'], + add_template=[], add_arch_template=[], + add_template_var=[], add_arch_template_var=[], + bugurl=None, + log_dir=buildinstall_topdir + '/x86_64/Server/logs'), + mock.call('Test', '1', '1', 'http://localhost/work/amd64/repo', + buildinstall_topdir + '/amd64/Server/results', + buildarch='amd64', is_final=True, nomacboot=True, noupgrade=True, + volid='vol_id', variant='Server', buildinstallpackages=['bash', 'vim'], + bugurl=None, + add_template=[], add_arch_template=[], + add_template_var=[], add_arch_template_var=[], + log_dir=buildinstall_topdir + '/amd64/Server/logs'), + mock.call('Test', '1', '1', 'http://localhost/work/amd64/repo', + buildinstall_topdir + '/amd64/Client/results', + buildarch='amd64', is_final=True, nomacboot=True, noupgrade=True, + volid='vol_id', variant='Client', buildinstallpackages=[], + bugurl=None, + add_template=[], add_arch_template=[], + add_template_var=[], add_arch_template_var=[], + log_dir=buildinstall_topdir + '/amd64/Client/logs')]) + self.assertItemsEqual( + get_volid.mock_calls, + [mock.call(compose, 'x86_64', variant=compose.variants['Server'], disc_type='dvd'), + mock.call(compose, 'amd64', variant=compose.variants['Client'], disc_type='dvd'), + mock.call(compose, 'amd64', variant=compose.variants['Server'], disc_type='dvd')]) + + class TestCopyFiles(PungiTestCase): @mock.patch('pungi.phases.buildinstall.link_boot_iso') @@ -706,6 +776,62 @@ class BuildinstallThreadTestCase(PungiTestCase): self.assertTrue(os.path.exists(dummy_file)) self.assertItemsEqual(self.pool.finished_tasks, []) + @mock.patch('pungi.phases.buildinstall.KojiWrapper') + @mock.patch('pungi.phases.buildinstall.get_buildroot_rpms') + @mock.patch('pungi.phases.buildinstall.run') + @mock.patch('pungi.phases.buildinstall.copy_all') + def test_buildinstall_thread_with_lorax_custom_buildinstall_topdir( + self, copy_all, run, get_buildroot_rpms, KojiWrapperMock): + compose = BuildInstallCompose(self.topdir, { + 'buildinstall_method': 'lorax', + 'runroot': True, + 'runroot_tag': 'rrt', + 'koji_profile': 'koji', + 'runroot_weights': {'buildinstall': 123}, + 'buildinstall_topdir': '/buildinstall_topdir', + }) + + get_buildroot_rpms.return_value = ['bash', 'zsh'] + + get_runroot_cmd = KojiWrapperMock.return_value.get_runroot_cmd + + run_runroot_cmd = KojiWrapperMock.return_value.run_runroot_cmd + run_runroot_cmd.return_value = { + 'output': 'Foo bar baz', + 'retcode': 0, + 'task_id': 1234, + } + + t = BuildinstallThread(self.pool) + + with mock.patch('time.sleep'): + t.process((compose, 'x86_64', compose.variants['Server'], self.cmd), 0) + + self.assertItemsEqual( + get_runroot_cmd.mock_calls, + [mock.call('rrt', 'x86_64', self.cmd, channel=None, + use_shell=True, task_id=True, + packages=['strace', 'lorax'], mounts=[self.topdir], weight=123)]) + self.assertItemsEqual( + run_runroot_cmd.mock_calls, + [mock.call(get_runroot_cmd.return_value, + log_file=self.topdir + '/logs/x86_64/buildinstall-Server.x86_64.log')]) + with open(self.topdir + '/logs/x86_64/buildinstall-Server-RPMs.x86_64.log') as f: + rpms = f.read().strip().split('\n') + self.assertItemsEqual(rpms, ['bash', 'zsh']) + self.assertItemsEqual(self.pool.finished_tasks, [('Server', 'x86_64')]) + + buildinstall_topdir = os.path.join( + "/buildinstall_topdir", "buildinstall-" + os.path.basename(self.topdir)) + self.assertItemsEqual( + copy_all.mock_calls, + [mock.call(os.path.join(buildinstall_topdir, 'x86_64/Server/results'), + os.path.join(self.topdir, 'work/x86_64/buildinstall/Server')), + mock.call(os.path.join(buildinstall_topdir, 'x86_64/Server/logs'), + os.path.join(self.topdir, 'logs/x86_64/buildinstall-Server-logs')) + ] + ) + class TestSymlinkIso(PungiTestCase):