Add support for new Pungi Buildinstall Koji plugin.

We would like to start generating the buildinstall phase using the safer
Koji Pungi Buildinstall plugin and stop the direct use of Runroot plugin.

The plugin so far exists only as PR for Koji:
https://pagure.io/koji/pull-request/1939

This commit adds support for this plugin when `lorax_use_koji_plugin`
is set to `True`.

Signed-off-by: Jan Kaluza <jkaluza@redhat.com>

doc/configuration.rst

@@ -550,6 +550,10 @@ Options
 **lorax_extra_sources**
     (*list*) -- a variant/arch mapping with urls for extra source repositories
     added to Lorax command line. Either one repo or a list can be specified.
+**lorax_use_koji_plugin** = False
+    (*bool*) -- When set to ``True``, the Koji pungi_buildinstall task will be
+    used to execute Lorax instead of runroot. Use only if the Koji instance
+    has the pungi_buildinstall plugin installed.
 **buildinstall_kickstart**
     (:ref:`scm_dict <scm_support>`) -- If specified, this kickstart file will
     be copied into each file and pointed to in boot configuration.

pungi/checks.py

@@ -1262,6 +1262,10 @@ def make_schema():
             "lorax_extra_sources": _variant_arch_mapping({
                 "$ref": "#/definitions/strings",
             }),
+            "lorax_use_koji_plugin": {
+                "type": "boolean",
+                "default": False,
+            },
 
             "signing_key_id": {"type": "string"},
             "signing_key_password_file": {"type": "string"},

pungi/phases/buildinstall.py

@@ -21,7 +21,7 @@ import shutil
 import re
 
 from kobo.threads import ThreadPool, WorkerThread
-from kobo.shortcuts import run
+from kobo.shortcuts import run, force_list
 from productmd.images import Image
 from six.moves import shlex_quote
 
@@ -46,6 +46,7 @@ class BuildinstallPhase(PhaseBase):
         # is needed to skip copying files for failed tasks.
         self.pool.finished_tasks = set()
         self.buildinstall_method = self.compose.conf.get("buildinstall_method")
+        self.lorax_use_koji_plugin = self.compose.conf.get("lorax_use_koji_plugin")
         self.used_lorax = self.buildinstall_method == 'lorax'
         self.pkgset_phase = pkgset_phase
 
@@ -98,13 +99,6 @@ class BuildinstallPhase(PhaseBase):
         # only care about the directory anyway.
         log_dir = _get_log_dir(self.compose, variant, arch)
 
-        # If the buildinstall_topdir is set, it means Koji is used for
-        # buildinstall phase and the filesystem with Koji is read-only.
-        # In that case, we have to write logs to buildinstall_topdir and
-        # later copy them back to our local log directory.
-        if self.compose.conf.get("buildinstall_topdir", None):
-            output_dir = os.path.join(output_dir, "results")
-
         repos = repo_baseurl[:]
         repos.extend(
             get_arch_variant_data(self.compose.conf, "lorax_extra_sources", arch, variant)
@@ -115,6 +109,36 @@ class BuildinstallPhase(PhaseBase):
                 comps_repo = translate_path(self.compose, comps_repo)
             repos.append(comps_repo)
 
+        if self.lorax_use_koji_plugin:
+            return {
+                "product": self.compose.conf["release_name"],
+                "version": version,
+                "release": version,
+                "sources": force_list(repos),
+                "variant": variant.uid,
+                "installpkgs": variant.buildinstallpackages,
+                "isfinal": self.compose.supported,
+                "buildarch": buildarch,
+                "volid": volid,
+                "nomacboot": nomacboot,
+                "bugurl": bugurl,
+                "add-template": add_template,
+                "add-arch-template": add_arch_template,
+                "add-template-var": add_template_var,
+                "add-arch-template-var": add_arch_template_var,
+                "noupgrade": noupgrade,
+                "rootfs-size": rootfs_size,
+                "dracut-args": dracut_args,
+                "outputdir": output_dir,
+            }
+        else:
+            # If the buildinstall_topdir is set, it means Koji is used for
+            # buildinstall phase and the filesystem with Koji is read-only.
+            # In that case, we have to write logs to buildinstall_topdir and
+            # later copy them back to our local log directory.
+            if self.compose.conf.get("buildinstall_topdir", None):
+                output_dir = os.path.join(output_dir, "results")
+
             lorax = LoraxWrapper()
             lorax_cmd = lorax.get_lorax_cmd(
                 self.compose.conf["release_name"],
@@ -423,6 +447,7 @@ class BuildinstallThread(WorkerThread):
 
     def worker(self, compose, arch, variant, cmd, num):
         buildinstall_method = compose.conf["buildinstall_method"]
+        lorax_use_koji_plugin = compose.conf["lorax_use_koji_plugin"]
         log_filename = ('buildinstall-%s' % variant.uid) if variant else 'buildinstall'
         log_file = compose.paths.log.log_file(arch, log_filename)
 
@@ -458,6 +483,13 @@ class BuildinstallThread(WorkerThread):
 
         # Start the runroot task.
         runroot = Runroot(compose, phase="buildinstall")
+        if buildinstall_method == "lorax" and lorax_use_koji_plugin:
+            runroot.run_pungi_buildinstall(
+                cmd, log_file=log_file, arch=arch, packages=packages,
+                mounts=[compose.topdir],
+                weight=compose.conf['runroot_weights'].get('buildinstall'),
+            )
+        else:
             runroot.run(
                 cmd, log_file=log_file, arch=arch, packages=packages,
                 mounts=[compose.topdir],

pungi/runroot.py

@@ -236,6 +236,35 @@ class Runroot(kobo.log.LoggingBase):
         else:
             raise ValueError("Unknown runroot_method %r." % self.runroot_method)
 
+    def run_pungi_buildinstall(self, args, log_file=None, arch=None, **kwargs):
+        """
+        Runs the Lorax buildinstall runroot command using the Pungi Buildinstall
+        Koji plugin as pungi_buildinstall task.
+
+        The **kwargs are optional and matches the
+        `KojiWrapper.get_pungi_buildinstall_cmd()` kwargs.
+
+        :param dict args: Arguments for the pungi_buildinstall Koji task.
+        :param str log_file: Log file into which the output of the task will
+            be logged.
+        :param str arch: Architecture on which the task should be executed.
+        """
+        runroot_channel = self.compose.conf.get("runroot_channel")
+        runroot_tag = self.compose.conf["runroot_tag"]
+
+        koji_wrapper = kojiwrapper.KojiWrapper(self.compose.conf["koji_profile"])
+        koji_cmd = koji_wrapper.get_pungi_buildinstall_cmd(
+            runroot_tag, arch, args, channel=runroot_channel,
+            chown_uid=os.getuid(), **kwargs)
+
+        output = koji_wrapper.run_runroot_cmd(koji_cmd, log_file=log_file)
+        if output["retcode"] != 0:
+            raise RuntimeError(
+                "Pungi-buildinstall task failed: %s. See %s for more details."
+                % (output["task_id"], log_file)
+            )
+        self._result = output
+
     def get_buildroot_rpms(self):
         """
         Returns the list of RPMs installed in a buildroot in which the runroot

pungi/wrappers/kojiwrapper.py

@@ -21,7 +21,7 @@ import threading
 import contextlib
 
 import koji
-from kobo.shortcuts import run
+from kobo.shortcuts import run, force_list
 import six
 from six.moves import configparser, shlex_quote
 import six.moves.xmlrpc_client as xmlrpclib
@@ -123,6 +123,47 @@ class KojiWrapper(object):
 
         return cmd
 
+    def get_pungi_buildinstall_cmd(
+            self, target, arch, args, channel=None, packages=None,
+            mounts=None, weight=None, chown_uid=None):
+        cmd = self._get_cmd("pungi-buildinstall", "--nowait", "--task-id")
+
+        if channel:
+            cmd.append("--channel-override=%s" % channel)
+        else:
+            cmd.append("--channel-override=runroot-local")
+
+        if weight:
+            cmd.append("--weight=%s" % int(weight))
+
+        for package in packages or []:
+            cmd.append("--package=%s" % package)
+
+        for mount in mounts or []:
+            # directories are *not* created here
+            cmd.append("--mount=%s" % mount)
+
+        if chown_uid:
+            cmd.append("--chown-uid=%s" % chown_uid)
+
+        # IMPORTANT: all --opts have to be provided *before* args
+
+        cmd.append(target)
+
+        # i686 -> i386 etc.
+        arch = getBaseArch(arch)
+        cmd.append(arch)
+
+        for k, v in args.items():
+            if v:
+                if isinstance(v, bool):
+                    cmd.append(k)
+                else:
+                    for arg in force_list(v):
+                        cmd.append("%s=%s" % (k, shlex_quote(arg)))
+
+        return cmd
+
     @contextlib.contextmanager
     def get_koji_cmd_env(self):
         """Get environment variables for running a koji command.
@@ -606,6 +647,12 @@ def get_buildroot_rpms(compose, task_id):
         # runroot
         koji = KojiWrapper(compose.conf['koji_profile'])
         buildroot_infos = koji.koji_proxy.listBuildroots(taskID=task_id)
+        if not buildroot_infos:
+            children_tasks = koji.koji_proxy.getTaskChildren(task_id)
+            for child_task in children_tasks:
+                buildroot_infos = koji.koji_proxy.listBuildroots(taskID=child_task["id"])
+                if buildroot_infos:
+                    break
         buildroot_info = buildroot_infos[-1]
         data = koji.koji_proxy.listRPMs(componentBuildrootID=buildroot_info["id"])
         for rpm_info in data:

tests/test_buildinstall.py

@@ -157,6 +157,85 @@ class TestBuildinstallPhase(PungiTestCase):
              mock.call(compose, 'amd64', variant=compose.variants['Client'], disc_type='DVD'),
              mock.call(compose, 'amd64', variant=compose.variants['Server'], disc_type='DVD')])
 
+    @mock.patch('pungi.phases.buildinstall.ThreadPool')
+    @mock.patch('pungi.phases.buildinstall.get_volid')
+    def test_starts_threads_for_each_cmd_with_lorax_koji_plugin(
+            self, get_volid, poolCls):
+        compose = BuildInstallCompose(self.topdir, {
+            'bootable': True,
+            'release_name': 'Test',
+            'release_short': 't',
+            'release_version': '1',
+            'buildinstall_method': 'lorax',
+            'lorax_use_koji_plugin': True,
+            'disc_types': {'dvd': 'DVD'},
+        })
+
+        get_volid.return_value = 'vol_id'
+
+        phase = BuildinstallPhase(compose, self._make_pkgset_phase(["p1", "p2"]))
+
+        phase.run()
+        self.maxDiff = None
+
+        expected_args = [
+            {
+                'product': 'Test', 'version': '1', 'release': '1',
+                'sources': [self.topdir + "/work/amd64/repo/p1",
+                            self.topdir + "/work/amd64/repo/p2",
+                            self.topdir + '/work/amd64/comps_repo_Server'],
+                'variant': 'Server', 'installpkgs': ['bash', 'vim'],
+                'isfinal': True, 'buildarch': 'amd64', 'volid': 'vol_id',
+                'nomacboot': True, 'bugurl': None, 'add-template': [],
+                'add-arch-template': [], 'add-template-var': [],
+                'add-arch-template-var': [], 'noupgrade': True,
+                'rootfs-size': None, 'dracut-args': [],
+                'outputdir': self.topdir + '/work/amd64/buildinstall/Server'
+            },
+            {
+                'product': 'Test', 'version': '1', 'release': '1',
+                'sources': [self.topdir + "/work/amd64/repo/p1",
+                            self.topdir + "/work/amd64/repo/p2",
+                            self.topdir + '/work/amd64/comps_repo_Client'],
+                'variant': 'Client', 'installpkgs': [],
+                'isfinal': True, 'buildarch': 'amd64', 'volid': 'vol_id',
+                'nomacboot': True, 'bugurl': None, 'add-template': [],
+                'add-arch-template': [], 'add-template-var': [],
+                'add-arch-template-var': [], 'noupgrade': True,
+                'rootfs-size': None, 'dracut-args': [],
+                'outputdir': self.topdir + '/work/amd64/buildinstall/Client'
+            },
+            {
+                'product': 'Test', 'version': '1', 'release': '1',
+                'sources': [self.topdir + "/work/x86_64/repo/p1",
+                            self.topdir + "/work/x86_64/repo/p2",
+                            self.topdir + '/work/x86_64/comps_repo_Server'],
+                'variant': 'Server', 'installpkgs': ['bash', 'vim'],
+                'isfinal': True, 'buildarch': 'x86_64', 'volid': 'vol_id',
+                'nomacboot': True, 'bugurl': None, 'add-template': [],
+                'add-arch-template': [], 'add-template-var': [],
+                'add-arch-template-var': [], 'noupgrade': True,
+                'rootfs-size': None, 'dracut-args': [],
+                'outputdir': self.topdir + '/work/x86_64/buildinstall/Server'
+            },
+        ]
+
+        # Three items added for processing in total.
+        # Server.x86_64, Client.amd64, Server.x86_64
+        pool = poolCls.return_value
+        self.assertEqual(3, len(pool.queue_put.mock_calls))
+        six.assertCountEqual(
+            self,
+            [call[0][0][3] for call in pool.queue_put.call_args_list],
+            expected_args)
+
+        six.assertCountEqual(
+            self,
+            get_volid.mock_calls,
+            [mock.call(compose, 'x86_64', variant=compose.variants['Server'], disc_type='DVD'),
+             mock.call(compose, 'amd64', variant=compose.variants['Client'], disc_type='DVD'),
+             mock.call(compose, 'amd64', variant=compose.variants['Server'], disc_type='DVD')])
+
     @mock.patch('pungi.phases.buildinstall.ThreadPool')
     @mock.patch('pungi.phases.buildinstall.LoraxWrapper')
     @mock.patch('pungi.phases.buildinstall.get_volid')
@@ -659,6 +738,75 @@ class BuildinstallThreadTestCase(PungiTestCase):
             [mock.call(compose, "x86_64", compose.variants["Server"], False)],
         )
 
+    @mock.patch('pungi.phases.buildinstall.link_boot_iso')
+    @mock.patch('pungi.phases.buildinstall.tweak_buildinstall')
+    @mock.patch('pungi.wrappers.kojiwrapper.KojiWrapper')
+    @mock.patch('pungi.wrappers.kojiwrapper.get_buildroot_rpms')
+    @mock.patch('pungi.phases.buildinstall.run')
+    def test_buildinstall_thread_with_lorax_using_koji_plugin(
+        self, run, get_buildroot_rpms, KojiWrapperMock, mock_tweak, mock_link
+    ):
+        compose = BuildInstallCompose(self.topdir, {
+            'buildinstall_method': 'lorax',
+            'lorax_use_koji_plugin': True,
+            'runroot_tag': 'rrt',
+            'koji_profile': 'koji',
+            'runroot_weights': {'buildinstall': 123},
+        })
+
+        get_buildroot_rpms.return_value = ['bash', 'zsh']
+
+        get_pungi_buildinstall_cmd = KojiWrapperMock.return_value.get_pungi_buildinstall_cmd
+
+        run_runroot_cmd = KojiWrapperMock.return_value.run_runroot_cmd
+        run_runroot_cmd.return_value = {
+            'output': 'Foo bar baz',
+            'retcode': 0,
+            'task_id': 1234,
+        }
+
+        t = BuildinstallThread(self.pool)
+
+        with mock.patch('time.sleep'):
+            t.process((compose, 'x86_64', compose.variants['Server'], self.cmd), 0)
+
+        destdir = os.path.join(self.topdir, "work/x86_64/buildinstall/Server")
+        self.assertEqual(
+            get_pungi_buildinstall_cmd.mock_calls,
+            [mock.call(
+                'rrt', 'x86_64', self.cmd, channel=None,
+                packages=['lorax'], mounts=[self.topdir],
+                weight=123, chown_uid=os.getuid()
+            )])
+        self.assertEqual(
+            run_runroot_cmd.mock_calls,
+            [mock.call(get_pungi_buildinstall_cmd.return_value,
+                       log_file=self.topdir + '/logs/x86_64/buildinstall-Server.x86_64.log')])
+        with open(self.topdir + '/logs/x86_64/buildinstall-Server-RPMs.x86_64.log') as f:
+            rpms = f.read().strip().split('\n')
+        six.assertCountEqual(self, rpms, ["bash", "zsh"])
+        six.assertCountEqual(self, self.pool.finished_tasks, [("Server", "x86_64")])
+
+        self.assertEqual(
+            mock_tweak.call_args_list,
+            [
+                mock.call(
+                    compose,
+                    destdir,
+                    os.path.join(self.topdir, "compose/Server/x86_64/os"),
+                    "x86_64",
+                    "Server",
+                    "",
+                    "dummy-volid",
+                    self.pool.kickstart_file,
+                )
+            ],
+        )
+        self.assertEqual(
+            mock_link.call_args_list,
+            [mock.call(compose, "x86_64", compose.variants["Server"], False)],
+        )
+
     @mock.patch('pungi.phases.buildinstall.link_boot_iso')
     @mock.patch('pungi.phases.buildinstall.tweak_buildinstall')
     @mock.patch('pungi.wrappers.kojiwrapper.KojiWrapper')