buildinstall: Allow using external dire for runroot task

A new `buildinstall_topdir` option allows using buildinstall even when
the compose is created on a different volume that Koji is using.

The files are created in this external directory and then copies into
the usual location.

Merges: https://pagure.io/pungi/pull-request/807
Signed-off-by: Jan Kaluza <jkaluza@redhat.com>

doc/configuration.rst

@@ -496,6 +496,14 @@ Options
 **buildinstall_kickstart**
     (:ref:`scm_dict <scm_support>`) -- If specified, this kickstart file will
     be copied into each file and pointed to in boot configuration.
+**buildinstall_topdir**
+    (*str*) -- Full path to top directory where the runroot buildinstall
+    Koji tasks output should be stored. This is useful in situation when
+    the Pungi compose is not generated on the same storage as the Koji task
+    is running on. In this case, Pungi can provide input repository for runroot
+    task using HTTP and set the output directory for this task to
+    ``buildinstall_topdir``. Once the runroot task finishes, Pungi will copy
+    the results of runroot tasks to the compose working directory.
 
 Example
 -------

pungi/checks.py

@@ -709,6 +709,7 @@ def make_schema():
                 "type": "string",
                 "enum": ["lorax", "buildinstall"],
             },
+            "buildinstall_topdir":  {"type": "string"},
             "buildinstall_kickstart": {"$ref": "#/definitions/str_or_scm_dict"},
 
             "global_ksurl": {"type": "string"},

pungi/paths.py

@@ -203,14 +203,28 @@ class WorkPaths(object):
             makedirs(path)
         return path
 
-    def buildinstall_dir(self, arch, create_dir=True):
+    def buildinstall_dir(self, arch, create_dir=True,
+                         allow_topdir_override=False, variant=None):
         """
+        :param bool allow_topdir_override: When True, the
+            "buildinstall_topdir" will be used (if set) instead of real
+            "topdir".
         Examples:
             work/x86_64/buildinstall
         """
         if arch == "global":
             raise RuntimeError("Global buildinstall dir makes no sense.")
-        path = os.path.join(self.topdir(arch, create_dir=create_dir), "buildinstall")
+
+        buildinstall_topdir = self.compose.conf.get("buildinstall_topdir", "")
+        if allow_topdir_override and buildinstall_topdir:
+            topdir_basename = os.path.basename(self.compose.topdir)
+            path = os.path.join(
+                buildinstall_topdir, "buildinstall-%s" % topdir_basename, arch)
+        else:
+            path = os.path.join(self.topdir(arch, create_dir=create_dir), "buildinstall")
+
+        if variant:
+            path = os.path.join(path, variant.uid)
         return path
 
     def extra_files_dir(self, arch, variant, create_dir=True):

pungi/phases/buildinstall.py

@@ -28,6 +28,7 @@ from six.moves import shlex_quote
 from pungi.arch import get_valid_arches
 from pungi.util import get_volid, get_arch_variant_data
 from pungi.util import get_file_size, get_mtime, failable, makedirs
+from pungi.util import copy_all, translate_path
 from pungi.wrappers.lorax import LoraxWrapper
 from pungi.wrappers.kojiwrapper import get_buildroot_rpms, KojiWrapper
 from pungi.wrappers import iso
@@ -74,13 +75,24 @@ class BuildinstallPhase(PhaseBase):
             add_template_var.extend(data.get('add_template_var', []))
             add_arch_template_var.extend(data.get('add_arch_template_var', []))
         output_dir = os.path.join(output_dir, variant.uid)
+        output_topdir = output_dir
 
         # The paths module will modify the filename (by inserting arch). But we
         # only care about the directory anyway.
-        log_filename = 'buildinstall-%s-logs/dumym' % variant.uid
+        log_filename = 'buildinstall-%s-logs/dummy' % variant.uid
         log_dir = os.path.dirname(self.compose.paths.log.log_file(arch, log_filename))
         makedirs(log_dir)
 
+        # If the buildinstall_topdir is set, it means Koji is used for
+        # buildinstall phase and the filesystem with Koji is read-only.
+        # In that case, we have to write logs to buildinstall_topdir and
+        # later copy them back to our local log directory.
+        if self.compose.conf.get("buildinstall_topdir", None):
+            log_dir = self.compose.paths.work.buildinstall_dir(
+                arch, allow_topdir_override=True, variant=variant)
+            log_dir = os.path.join(log_dir, "logs")
+            output_dir = os.path.join(output_dir, "results")
+
         lorax = LoraxWrapper()
         lorax_cmd = lorax.get_lorax_cmd(self.compose.conf["release_name"],
                                         self.compose.conf["release_version"],
@@ -100,7 +112,7 @@ class BuildinstallPhase(PhaseBase):
                                         add_arch_template_var=add_arch_template_var,
                                         noupgrade=noupgrade,
                                         log_dir=log_dir)
-        return 'rm -rf %s && %s' % (shlex_quote(output_dir),
+        return 'rm -rf %s && %s' % (shlex_quote(output_topdir),
                                     ' '.join([shlex_quote(x) for x in lorax_cmd]))
 
     def run(self):
@@ -114,8 +126,11 @@ class BuildinstallPhase(PhaseBase):
         for arch in self.compose.get_arches():
             commands = []
 
+            output_dir = self.compose.paths.work.buildinstall_dir(arch, allow_topdir_override=True)
+            final_output_dir = self.compose.paths.work.buildinstall_dir(arch, allow_topdir_override=False)
             repo_baseurl = self.compose.paths.work.arch_repo(arch)
-            output_dir = self.compose.paths.work.buildinstall_dir(arch)
+            if final_output_dir != output_dir:
+                repo_baseurl = translate_path(self.compose, repo_baseurl)
 
             if buildinstall_method == "lorax":
                 buildarch = get_valid_arches(arch)[0]
@@ -388,11 +403,13 @@ class BuildinstallThread(WorkerThread):
 
         msg = "Running buildinstall for arch %s, variant %s" % (arch, variant)
 
-        output_dir = compose.paths.work.buildinstall_dir(arch)
+        output_dir = compose.paths.work.buildinstall_dir(
-        if variant:
+            arch, allow_topdir_override=True, variant=variant)
-            output_dir = os.path.join(output_dir, variant.uid)
+        final_output_dir = compose.paths.work.buildinstall_dir(
+            arch, variant=variant)
 
-        if os.path.isdir(output_dir) and os.listdir(output_dir):
+        if (os.path.isdir(output_dir) and os.listdir(output_dir) or
+                os.path.isdir(final_output_dir) and os.listdir(final_output_dir)):
             # output dir is *not* empty -> SKIP
             self.pool.log_warning(
                 '[SKIP ] Buildinstall for arch %s, variant %s' % (arch, variant))
@@ -433,6 +450,20 @@ class BuildinstallThread(WorkerThread):
             # run locally
             run(cmd, show_cmd=True, logfile=log_file)
 
+        if final_output_dir != output_dir:
+            if not os.path.exists(final_output_dir):
+                makedirs(final_output_dir)
+            results_dir = os.path.join(output_dir, "results")
+            copy_all(results_dir, final_output_dir)
+
+            # Get the log_dir into which we should copy the resulting log files.
+            log_fname = 'buildinstall-%s-logs/dummy' % variant.uid
+            final_log_dir = os.path.dirname(compose.paths.log.log_file(arch, log_fname))
+            if not os.path.exists(final_log_dir):
+                makedirs(final_log_dir)
+            log_dir = os.path.join(output_dir, "logs")
+            copy_all(log_dir, final_log_dir)
+
         log_file = compose.paths.log.log_file(arch, log_filename + '-RPMs')
         rpms = get_buildroot_rpms(compose, task_id)
         with open(log_file, "w") as f:

tests/test_buildinstall.py

@@ -345,6 +345,76 @@ class TestBuildinstallPhase(PungiTestCase):
              mock.call(compose, 'amd64', variant=compose.variants['Server'], disc_type='dvd')])
 
 
+    @mock.patch('pungi.phases.buildinstall.ThreadPool')
+    @mock.patch('pungi.phases.buildinstall.LoraxWrapper')
+    @mock.patch('pungi.phases.buildinstall.get_volid')
+    def test_uses_lorax_options_buildinstall_topdir(self, get_volid, loraxCls, poolCls):
+        compose = BuildInstallCompose(self.topdir, {
+            'bootable': True,
+            'release_name': 'Test',
+            'release_short': 't',
+            'release_version': '1',
+            'release_is_layered': False,
+            'buildinstall_method': 'lorax',
+            'buildinstall_topdir': '/buildinstall_topdir',
+            'translate_paths': [(self.topdir, "http://localhost/")],
+        })
+
+        buildinstall_topdir = os.path.join(
+            "/buildinstall_topdir", "buildinstall-" + os.path.basename(self.topdir))
+        self.maxDiff = None
+
+        get_volid.return_value = 'vol_id'
+        loraxCls.return_value.get_lorax_cmd.return_value = ['lorax', '...']
+
+        phase = BuildinstallPhase(compose)
+
+        phase.run()
+
+        # Three items added for processing in total.
+        # Server.x86_64, Client.amd64, Server.x86_64
+        pool = poolCls.return_value
+        self.assertEqual(3, len(pool.queue_put.mock_calls))
+        self.assertItemsEqual(
+            [call[0][0][3] for call in pool.queue_put.call_args_list],
+            ['rm -rf %s/amd64/Client && lorax ...' % buildinstall_topdir,
+             'rm -rf %s/amd64/Server && lorax ...' % buildinstall_topdir,
+             'rm -rf %s/x86_64/Server && lorax ...' % buildinstall_topdir])
+
+        # Obtained correct lorax commands.
+        self.assertItemsEqual(
+            loraxCls.return_value.get_lorax_cmd.mock_calls,
+            [mock.call('Test', '1', '1', 'http://localhost/work/x86_64/repo',
+                       buildinstall_topdir + '/x86_64/Server/results',
+                       buildarch='x86_64', is_final=True, nomacboot=True, noupgrade=True,
+                       volid='vol_id', variant='Server', buildinstallpackages=['bash', 'vim'],
+                       add_template=[], add_arch_template=[],
+                       add_template_var=[], add_arch_template_var=[],
+                       bugurl=None,
+                       log_dir=buildinstall_topdir + '/x86_64/Server/logs'),
+             mock.call('Test', '1', '1', 'http://localhost/work/amd64/repo',
+                       buildinstall_topdir + '/amd64/Server/results',
+                       buildarch='amd64', is_final=True, nomacboot=True, noupgrade=True,
+                       volid='vol_id', variant='Server', buildinstallpackages=['bash', 'vim'],
+                       bugurl=None,
+                       add_template=[], add_arch_template=[],
+                       add_template_var=[], add_arch_template_var=[],
+                       log_dir=buildinstall_topdir + '/amd64/Server/logs'),
+             mock.call('Test', '1', '1', 'http://localhost/work/amd64/repo',
+                       buildinstall_topdir + '/amd64/Client/results',
+                       buildarch='amd64', is_final=True, nomacboot=True, noupgrade=True,
+                       volid='vol_id', variant='Client', buildinstallpackages=[],
+                       bugurl=None,
+                       add_template=[], add_arch_template=[],
+                       add_template_var=[], add_arch_template_var=[],
+                       log_dir=buildinstall_topdir + '/amd64/Client/logs')])
+        self.assertItemsEqual(
+            get_volid.mock_calls,
+            [mock.call(compose, 'x86_64', variant=compose.variants['Server'], disc_type='dvd'),
+             mock.call(compose, 'amd64', variant=compose.variants['Client'], disc_type='dvd'),
+             mock.call(compose, 'amd64', variant=compose.variants['Server'], disc_type='dvd')])
+
+
 class TestCopyFiles(PungiTestCase):
 
     @mock.patch('pungi.phases.buildinstall.link_boot_iso')
@@ -706,6 +776,62 @@ class BuildinstallThreadTestCase(PungiTestCase):
         self.assertTrue(os.path.exists(dummy_file))
         self.assertItemsEqual(self.pool.finished_tasks, [])
 
+    @mock.patch('pungi.phases.buildinstall.KojiWrapper')
+    @mock.patch('pungi.phases.buildinstall.get_buildroot_rpms')
+    @mock.patch('pungi.phases.buildinstall.run')
+    @mock.patch('pungi.phases.buildinstall.copy_all')
+    def test_buildinstall_thread_with_lorax_custom_buildinstall_topdir(
+            self, copy_all, run, get_buildroot_rpms, KojiWrapperMock):
+        compose = BuildInstallCompose(self.topdir, {
+            'buildinstall_method': 'lorax',
+            'runroot': True,
+            'runroot_tag': 'rrt',
+            'koji_profile': 'koji',
+            'runroot_weights': {'buildinstall': 123},
+            'buildinstall_topdir': '/buildinstall_topdir',
+        })
+
+        get_buildroot_rpms.return_value = ['bash', 'zsh']
+
+        get_runroot_cmd = KojiWrapperMock.return_value.get_runroot_cmd
+
+        run_runroot_cmd = KojiWrapperMock.return_value.run_runroot_cmd
+        run_runroot_cmd.return_value = {
+            'output': 'Foo bar baz',
+            'retcode': 0,
+            'task_id': 1234,
+        }
+
+        t = BuildinstallThread(self.pool)
+
+        with mock.patch('time.sleep'):
+            t.process((compose, 'x86_64', compose.variants['Server'], self.cmd), 0)
+
+        self.assertItemsEqual(
+            get_runroot_cmd.mock_calls,
+            [mock.call('rrt', 'x86_64', self.cmd, channel=None,
+                       use_shell=True, task_id=True,
+                       packages=['strace', 'lorax'], mounts=[self.topdir], weight=123)])
+        self.assertItemsEqual(
+            run_runroot_cmd.mock_calls,
+            [mock.call(get_runroot_cmd.return_value,
+                       log_file=self.topdir + '/logs/x86_64/buildinstall-Server.x86_64.log')])
+        with open(self.topdir + '/logs/x86_64/buildinstall-Server-RPMs.x86_64.log') as f:
+            rpms = f.read().strip().split('\n')
+        self.assertItemsEqual(rpms, ['bash', 'zsh'])
+        self.assertItemsEqual(self.pool.finished_tasks, [('Server', 'x86_64')])
+
+        buildinstall_topdir = os.path.join(
+            "/buildinstall_topdir", "buildinstall-" + os.path.basename(self.topdir))
+        self.assertItemsEqual(
+            copy_all.mock_calls,
+            [mock.call(os.path.join(buildinstall_topdir, 'x86_64/Server/results'),
+                       os.path.join(self.topdir, 'work/x86_64/buildinstall/Server')),
+             mock.call(os.path.join(buildinstall_topdir, 'x86_64/Server/logs'),
+                       os.path.join(self.topdir, 'logs/x86_64/buildinstall-Server-logs'))
+            ]
+        )
+
 
 class TestSymlinkIso(PungiTestCase):