Extend "openssh" runroot_method to be able to execute "mock"

This adds few new config options which are well described in the
configuration documentation. Please refer to it for more information.

Merges: https://pagure.io/pungi/pull-request/1170
Signed-off-by: Jan Kaluza <jkaluza@redhat.com>

doc/configuration.rst

@@ -889,6 +889,22 @@ Options
      * ``ostree``
      * ``ostree_installer``
 
+Example
+-------
+::
+
+    koji_profile = "koji"
+    runroot = True
+    runroot_channel = "runroot"
+    runroot_tag = "f23-build"
+
+Runroot "openssh" method settings
+=================================
+
+
+Options
+-------
+
 **runroot_ssh_username**
     (*str*) -- For ``openssh`` runroot method, configures the username used to login
     the remote machine to run the runroot task. Defaults to "root".
@@ -898,14 +914,56 @@ Options
     architecture on which the runroot task should be running. Format:
     ``{"x86_64": "runroot-x86-64.localhost.tld", ...}``
 
-Example
+**runroot_ssh_init_command**
--------
+    (*str*) [optional] -- For ``openssh`` runroot method, defines the command
-::
+    to initializes the runroot task on the remote machine. This command is
+    executed as first command for each runroot task executed.
 
-    koji_profile = "koji"
+    The command can print a string which is then available as ``{runroot_key}``
-    runroot = True
+    for other SSH commands. This string might be used to keep the context
-    runroot_channel = "runroot"
+    across different SSH commands executed for single runroot task.
-    runroot_tag = "f23-build"
+
+    The goal of this command is setting up the environment for real runroot
+    commands. For example preparing the unique mock environment, mounting the
+    desired file-systems, ...
+
+    When not set, no init command is executed.
+
+**runroot_ssh_install_packages_template**
+    (*str*) [optional] -- For ``openssh`` runroot method, defines the template
+    for command to install the packages requested to run the runroot task.
+
+    The template string can contain following variables which are replaced by
+    the real values before executing the install command:
+
+    * ``{runroot_key}`` - Replaced with the string returned by
+      ``runroot_ssh_init_command`` if used. This can be used to keep the track
+      of context of SSH commands beloging to single runroot task.
+    * ``{packages}`` - White-list separated list of packages to install.
+
+    Example (The ``{runroot_key}`` is expected to be set to mock config file
+    using the ``runroot_ssh_init_command`` command.):
+    ``"mock -r {runroot_key} --install {packages}"``
+
+    When not set, no command to install packages on remote machine is executed.
+
+**runroot_ssh_run_template**
+    (*str*) [optional] -- For ``openssh`` runroot method, defines the template
+    for the main runroot command.
+
+    The template string can contain following variables which are replaced by
+    the real values before executing the install command:
+
+    * ``{runroot_key}`` - Replaced with the string returned by
+      ``runroot_ssh_init_command`` if used. This can be used to keep the track
+      of context of SSH commands beloging to single runroot task.
+    * ``{command}`` - Command to run.
+
+    Example (The ``{runroot_key}`` is expected to be set to mock config file
+    using the ``runroot_ssh_init_command`` command.):
+    ``"mock -r {runroot_key} chroot -- {command}"``
+
+    When not set, the runroot command is run directly.
 
 
 Extra Files Settings

pungi/runroot.py

@@ -84,7 +84,23 @@ class Runroot(kobo.log.LoggingBase):
             )
         self._result = output
 
-    def _run_openssh(self, command, log_file=None, arch=None, **kwargs):
+    def _ssh_run(self, hostname, user, command, fmt_dict=None, log_file=None):
+        """
+        Helper method to run the command using "ssh".
+
+        :param str hostname: Hostname.
+        :param str user: User for login.
+        :param str command: Command to run.
+        :param str fmt_dict: If set, the `command` is formatted like
+            `command.format(**fmt_dict)`.
+        :param str log_file: Log file.
+        :return str: Output of remote command.
+        """
+        formatted_cmd = command.format(**fmt_dict) if fmt_dict else command
+        ssh_cmd = ["ssh", "-oBatchMode=yes", "-n", "-l", user, hostname, formatted_cmd]
+        return run(ssh_cmd, show_cmd=True, logfile=log_file)[1]
+
+    def _run_openssh(self, command, log_file=None, arch=None, packages=None, **kwargs):
         """
         Runs the runroot command on remote machine using ssh.
         """
@@ -94,16 +110,51 @@ class Runroot(kobo.log.LoggingBase):
 
         hostname = runroot_ssh_hostnames[arch]
         user = self.compose.conf.get("runroot_ssh_username", "root")
+        init_command = self.compose.conf.get("runroot_ssh_init_command")
+        install_packages_template = self.compose.conf.get(
+            "runroot_ssh_install_packages_template"
+        )
+        run_template = self.compose.conf.get("runroot_ssh_run_template")
 
-        ssh_cmd = ["ssh", "-oBatchMode=yes", "-n", "-l", user, hostname, command]
+        # Init the runroot on remote machine and get the runroot_key.
-        run(ssh_cmd, show_cmd=True, logfile=log_file)
+        if init_command:
+            runroot_key = self._ssh_run(hostname, user, init_command, log_file=log_file)
+            runroot_key = runroot_key.rstrip("\n\r")
+        else:
+            runroot_key = None
 
-        # Get the buildroot RPMs.
+        # Install the packages needed for runroot task if configured.
-        ssh_cmd = ["ssh", "-oBatchMode=yes", "-n", "-l", user, hostname,
+        if install_packages_template and packages:
-                   "rpm -qa --qf='%{name}-%{version}-%{release}.%{arch}\n'"]
+            fmt_dict = {"packages": " ".join(packages)}
-        output = run(ssh_cmd, show_cmd=True)[1]
+            if runroot_key:
+                fmt_dict["runroot_key"] = runroot_key
+            self._ssh_run(
+                hostname, user, install_packages_template, fmt_dict, log_file=log_file
+            )
+
+        # Run the runroot task and get the buildroot RPMs.
+        if run_template:
+            fmt_dict = {"command": command}
+            if runroot_key:
+                fmt_dict["runroot_key"] = runroot_key
+            self._ssh_run(hostname, user, run_template, fmt_dict, log_file=log_file)
+
+            fmt_dict["command"] = "rpm -qa --qf='%{name}-%{version}-%{release}.%{arch}\n'"
+            buildroot_rpms = self._ssh_run(
+                hostname, user, run_template, fmt_dict, log_file=log_file
+            )
+        else:
+            self._ssh_run(hostname, user, command, log_file=log_file)
+            buildroot_rpms = self._ssh_run(
+                hostname,
+                user,
+                "rpm -qa --qf='%{name}-%{version}-%{release}.%{arch}\n'",
+                log_file=log_file,
+            )
+
+        # Parse the buildroot_rpms and store it in self._result.
         self._result = []
-        for i in output.splitlines():
+        for i in buildroot_rpms.splitlines():
             if not i:
                 continue
             self._result.append(i)
@@ -158,8 +209,8 @@ class Runroot(kobo.log.LoggingBase):
         """
         if not self._result:
             raise ValueError(
-                "Runroot.get_buildroot_rpms called before runroot task "
+                "Runroot.get_buildroot_rpms called before runroot task finished."
-                "finished.")
+            )
         if self.runroot_method in ["local", "koji"]:
             if self.runroot_method == "local":
                 task_id = None

tests/test_runroot.py

@@ -28,17 +28,23 @@ class TestRunrootOpenSSH(helpers.PungiTestCase):
         method = self.runroot.get_runroot_method()
         self.assertEqual(method, "openssh")
 
+    def _ssh_call(self, cmd):
+        """
+        Helper method returning default SSH mock.call with given command `cmd`.
+        """
+        return mock.call(
+            ['ssh', '-oBatchMode=yes', '-n', '-l', 'root', 'localhost', cmd],
+            logfile='/foo/runroot.log',
+            show_cmd=True,
+        )
+
     @mock.patch("pungi.runroot.run")
     def test_run(self, run):
+        run.return_value = (0, "dummy output\n")
         self.runroot.run("df -h", log_file="/foo/runroot.log", arch="x86_64")
         run.assert_has_calls([
-            mock.call(
+            self._ssh_call('df -h'),
-                ['ssh', '-oBatchMode=yes', '-n', '-l', 'root', 'localhost',
+            self._ssh_call("rpm -qa --qf='%{name}-%{version}-%{release}.%{arch}\n'"),
-                 'df -h'], logfile='/foo/runroot.log', show_cmd=True),
-            mock.call(
-                ['ssh', '-oBatchMode=yes', '-n', '-l', 'root', 'localhost',
-                 "rpm -qa --qf='%{name}-%{version}-%{release}.%{arch}\n'"],
-                show_cmd=True)
         ])
 
     @mock.patch("pungi.runroot.run")
@@ -50,3 +56,60 @@ class TestRunrootOpenSSH(helpers.PungiTestCase):
         rpms = self.runroot.get_buildroot_rpms()
         self.assertEqual(
             set(rpms), set(["foo-1-1.fc29.noarch", "bar-1-1.fc29.noarch"]))
+
+    @mock.patch("pungi.runroot.run")
+    def test_run_templates(self, run):
+        self.compose.conf["runroot_ssh_init_command"] = "/usr/sbin/init_runroot"
+        self.compose.conf["runroot_ssh_install_packages_template"] = \
+            "install {runroot_key} {packages}"
+        self.compose.conf["runroot_ssh_run_template"] = "run {runroot_key} {command}"
+
+        run.return_value = (0, "key\n")
+        self.runroot.run("df -h", log_file="/foo/runroot.log", arch="x86_64",
+                         packages=["lorax", "automake"])
+        run.assert_has_calls([
+            self._ssh_call('/usr/sbin/init_runroot'),
+            self._ssh_call('install key lorax automake'),
+            self._ssh_call('run key df -h'),
+            self._ssh_call("run key rpm -qa --qf='%{name}-%{version}-%{release}.%{arch}\n'"),
+        ])
+
+    @mock.patch("pungi.runroot.run")
+    def test_run_templates_no_init(self, run):
+        self.compose.conf["runroot_ssh_install_packages_template"] = \
+            "install {packages}"
+        self.compose.conf["runroot_ssh_run_template"] = "run {command}"
+
+        run.return_value = (0, "key\n")
+        self.runroot.run("df -h", log_file="/foo/runroot.log", arch="x86_64",
+                         packages=["lorax", "automake"])
+        run.assert_has_calls([
+            self._ssh_call('install lorax automake'),
+            self._ssh_call('run df -h'),
+            self._ssh_call("run rpm -qa --qf='%{name}-%{version}-%{release}.%{arch}\n'"),
+        ])
+
+    @mock.patch("pungi.runroot.run")
+    def test_run_templates_no_packages(self, run):
+        self.compose.conf["runroot_ssh_install_packages_template"] = \
+            "install {packages}"
+        self.compose.conf["runroot_ssh_run_template"] = "run {command}"
+
+        run.return_value = (0, "key\n")
+        self.runroot.run("df -h", log_file="/foo/runroot.log", arch="x86_64")
+        run.assert_has_calls([
+            self._ssh_call('run df -h'),
+            self._ssh_call("run rpm -qa --qf='%{name}-%{version}-%{release}.%{arch}\n'"),
+        ])
+
+    @mock.patch("pungi.runroot.run")
+    def test_run_templates_no_install_packages(self, run):
+        self.compose.conf["runroot_ssh_run_template"] = "run {command}"
+
+        run.return_value = (0, "key\n")
+        self.runroot.run("df -h", log_file="/foo/runroot.log", arch="x86_64",
+                         packages=["lorax", "automake"])
+        run.assert_has_calls([
+            self._ssh_call('run df -h'),
+            self._ssh_call("run rpm -qa --qf='%{name}-%{version}-%{release}.%{arch}\n'"),
+        ])