Use pungi_buildinstall without NFS

The plugin supports two modes of operation:
1. Mount a shared storage volume into the runroot and have the output
   written there.
2. Have the plugin create a tar.gz with the outputs and upload them to
   the hub, from where they can be downloaded.

This patch switches from option 1 to option 2.

This requires all input repositories to be passes in as URLs and not
paths. Once the task finishes, Pungi will download the output archives
and unpack them into the expected locations.

JIRA: RHELCMP-13284
Signed-off-by: Lubomír Sedlář <lsedlar@redhat.com>
(cherry picked from commit f25489d060)

pungi/phases/buildinstall.py

@@ -31,14 +31,14 @@ from six.moves import shlex_quote
 from pungi.arch import get_valid_arches
 from pungi.util import get_volid, get_arch_variant_data
 from pungi.util import get_file_size, get_mtime, failable, makedirs
-from pungi.util import copy_all, translate_path, move_all
+from pungi.util import copy_all, translate_path
 from pungi.wrappers.lorax import LoraxWrapper
 from pungi.wrappers import iso
 from pungi.wrappers.scm import get_file
 from pungi.wrappers.scm import get_file_from_scm
 from pungi.wrappers import kojiwrapper
 from pungi.phases.base import PhaseBase
-from pungi.runroot import Runroot
+from pungi.runroot import Runroot, download_and_extract_archive
 
 
 class BuildinstallPhase(PhaseBase):
@@ -144,7 +144,7 @@ class BuildinstallPhase(PhaseBase):
         )
         if self.compose.has_comps:
             comps_repo = self.compose.paths.work.comps_repo(arch, variant)
-            if final_output_dir != output_dir:
+            if final_output_dir != output_dir or self.lorax_use_koji_plugin:
                 comps_repo = translate_path(self.compose, comps_repo)
             repos.append(comps_repo)
 
@@ -169,7 +169,6 @@ class BuildinstallPhase(PhaseBase):
                 "rootfs-size": rootfs_size,
                 "dracut-args": dracut_args,
                 "skip_branding": skip_branding,
-                "outputdir": output_dir,
                 "squashfs_only": squashfs_only,
                 "configuration_file": configuration_file,
             }
@@ -235,7 +234,7 @@ class BuildinstallPhase(PhaseBase):
             )
             makedirs(final_output_dir)
             repo_baseurls = self.get_repos(arch)
-            if final_output_dir != output_dir:
+            if final_output_dir != output_dir or self.lorax_use_koji_plugin:
                 repo_baseurls = [translate_path(self.compose, r) for r in repo_baseurls]
 
             if self.buildinstall_method == "lorax":
@@ -833,13 +832,13 @@ class BuildinstallThread(WorkerThread):
 
         # Start the runroot task.
         runroot = Runroot(compose, phase="buildinstall")
+        task_id = None
         if buildinstall_method == "lorax" and lorax_use_koji_plugin:
-            runroot.run_pungi_buildinstall(
+            task_id = runroot.run_pungi_buildinstall(
                 cmd,
                 log_file=log_file,
                 arch=arch,
                 packages=packages,
-                mounts=[compose.topdir],
                 weight=compose.conf["runroot_weights"].get("buildinstall"),
             )
         else:
@@ -872,19 +871,17 @@ class BuildinstallThread(WorkerThread):
             log_dir = os.path.join(output_dir, "logs")
             copy_all(log_dir, final_log_dir)
         elif lorax_use_koji_plugin:
-            # If Koji pungi-buildinstall is used, then the buildinstall results are
+            # If Koji pungi-buildinstall is used, then the buildinstall results
-            # not stored directly in `output_dir` dir, but in "results" and "logs"
+            # are attached as outputs to the Koji task. Download and unpack
-            # subdirectories. We need to move them to final_output_dir.
+            # them to the correct location.
-            results_dir = os.path.join(output_dir, "results")
+            download_and_extract_archive(
-            move_all(results_dir, final_output_dir, rm_src_dir=True)
+                compose, task_id, "results.tar.gz", final_output_dir
+            )
 
-            # Get the log_dir into which we should copy the resulting log files.
+            # Download the logs into proper location too.
             log_fname = "buildinstall-%s-logs/dummy" % variant.uid
             final_log_dir = os.path.dirname(compose.paths.log.log_file(arch, log_fname))
-            if not os.path.exists(final_log_dir):
+            download_and_extract_archive(compose, task_id, "logs.tar.gz", final_log_dir)
-                makedirs(final_log_dir)
-            log_dir = os.path.join(output_dir, "logs")
-            move_all(log_dir, final_log_dir, rm_src_dir=True)
 
         rpms = runroot.get_buildroot_rpms()
         self._write_buildinstall_metadata(

pungi/runroot.py

@@ -13,13 +13,19 @@
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, see <https://gnu.org/licenses/>.
 
+import contextlib
 import os
 import re
+import shutil
+import tarfile
+
+import requests
 import six
 from six.moves import shlex_quote
 import kobo.log
 from kobo.shortcuts import run
 
+from pungi import util
 from pungi.wrappers import kojiwrapper
 
 
@@ -314,7 +320,8 @@ class Runroot(kobo.log.LoggingBase):
             arch,
             args,
             channel=runroot_channel,
-            chown_uid=os.getuid(),
+            # We want to change owner only if shared NFS directory is used.
+            chown_uid=os.getuid() if kwargs.get("mounts") else None,
             **kwargs
         )
 
@@ -325,6 +332,7 @@ class Runroot(kobo.log.LoggingBase):
                 % (output["task_id"], log_file)
             )
         self._result = output
+        return output["task_id"]
 
     def run_pungi_ostree(self, args, log_file=None, arch=None, **kwargs):
         """
@@ -381,3 +389,72 @@ class Runroot(kobo.log.LoggingBase):
             return self._result
         else:
             raise ValueError("Unknown runroot_method %r." % self.runroot_method)
+
+
+@util.retry(wait_on=requests.exceptions.RequestException)
+def _download_file(url, dest):
+    # contextlib.closing is only needed in requests<2.18
+    with contextlib.closing(requests.get(url, stream=True, timeout=5)) as r:
+        if r.status_code == 404:
+            raise RuntimeError("Archive %s not found" % url)
+        r.raise_for_status()
+        with open(dest, "wb") as f:
+            shutil.copyfileobj(r.raw, f)
+
+
+def _download_archive(task_id, fname, archive_url, dest_dir):
+    """Download file from URL to a destination, with retries."""
+    temp_file = os.path.join(dest_dir, fname)
+    _download_file(archive_url, temp_file)
+    return temp_file
+
+
+def _extract_archive(task_id, fname, archive_file, dest_path):
+    """Extract the archive into given destination.
+
+    All items of the archive must match the name of the archive, i.e. all
+    paths in foo.tar.gz must start with foo/.
+    """
+    basename = os.path.basename(fname).split(".")[0]
+    strip_prefix = basename + "/"
+    with tarfile.open(archive_file, "r") as archive:
+        for member in archive.getmembers():
+            # Check if each item is either the root directory or is within it.
+            if member.name != basename and not member.name.startswith(strip_prefix):
+                raise RuntimeError(
+                    "Archive %s from task %s contains file without expected prefix: %s"
+                    % (fname, task_id, member)
+                )
+            dest = os.path.join(dest_path, member.name[len(strip_prefix) :])
+            if member.isdir():
+                # Create directories where needed...
+                util.makedirs(dest)
+            elif member.isfile():
+                # ... and extract files into them.
+                with open(dest, "wb") as dest_obj:
+                    shutil.copyfileobj(archive.extractfile(member), dest_obj)
+            elif member.islnk():
+                # We have a hardlink. Let's also link it.
+                linked_file = os.path.join(
+                    dest_path, member.linkname[len(strip_prefix) :]
+                )
+                os.link(linked_file, dest)
+            else:
+                # Any other file type is an error.
+                raise RuntimeError(
+                    "Unexpected file type in %s from task %s: %s"
+                    % (fname, task_id, member)
+                )
+
+
+def download_and_extract_archive(compose, task_id, fname, destination):
+    """Download a tar archive from task outputs and extract it to the destination."""
+    koji = kojiwrapper.KojiWrapper(compose).koji_module
+    # Koji API provides downloadTaskOutput method, but it's not usable as it
+    # will attempt to load the entire file into memory.
+    # So instead let's generate a patch and attempt to convert it to a URL.
+    server_path = os.path.join(koji.pathinfo.task(task_id), fname)
+    archive_url = server_path.replace(koji.config.topdir, koji.config.topurl)
+    with util.temp_dir(prefix="buildinstall-download") as tmp_dir:
+        local_path = _download_archive(task_id, fname, archive_url, tmp_dir)
+        _extract_archive(task_id, fname, local_path, destination)

tests/test_buildinstall.py

@@ -254,6 +254,7 @@ class TestBuildinstallPhase(PungiTestCase):
     def test_starts_threads_for_each_cmd_with_lorax_koji_plugin(
         self, get_volid, poolCls
     ):
+        topurl = "https://example.com/composes/"
         compose = BuildInstallCompose(
             self.topdir,
             {
@@ -264,6 +265,7 @@ class TestBuildinstallPhase(PungiTestCase):
                 "buildinstall_method": "lorax",
                 "lorax_use_koji_plugin": True,
                 "disc_types": {"dvd": "DVD"},
+                "translate_paths": [(self.topdir, topurl)],
             },
         )
 
@@ -280,9 +282,9 @@ class TestBuildinstallPhase(PungiTestCase):
                 "version": "1",
                 "release": "1",
                 "sources": [
-                    self.topdir + "/work/amd64/repo/p1",
+                    topurl + "work/amd64/repo/p1",
-                    self.topdir + "/work/amd64/repo/p2",
+                    topurl + "work/amd64/repo/p2",
-                    self.topdir + "/work/amd64/comps_repo_Server",
+                    topurl + "work/amd64/comps_repo_Server",
                 ],
                 "variant": "Server",
                 "installpkgs": ["bash", "vim"],
@@ -299,7 +301,6 @@ class TestBuildinstallPhase(PungiTestCase):
                 "rootfs-size": None,
                 "dracut-args": [],
                 "skip_branding": False,
-                "outputdir": self.topdir + "/work/amd64/buildinstall/Server",
                 "squashfs_only": False,
                 "configuration_file": None,
             },
@@ -308,9 +309,9 @@ class TestBuildinstallPhase(PungiTestCase):
                 "version": "1",
                 "release": "1",
                 "sources": [
-                    self.topdir + "/work/amd64/repo/p1",
+                    topurl + "work/amd64/repo/p1",
-                    self.topdir + "/work/amd64/repo/p2",
+                    topurl + "work/amd64/repo/p2",
-                    self.topdir + "/work/amd64/comps_repo_Client",
+                    topurl + "work/amd64/comps_repo_Client",
                 ],
                 "variant": "Client",
                 "installpkgs": [],
@@ -327,7 +328,6 @@ class TestBuildinstallPhase(PungiTestCase):
                 "rootfs-size": None,
                 "dracut-args": [],
                 "skip_branding": False,
-                "outputdir": self.topdir + "/work/amd64/buildinstall/Client",
                 "squashfs_only": False,
                 "configuration_file": None,
             },
@@ -336,9 +336,9 @@ class TestBuildinstallPhase(PungiTestCase):
                 "version": "1",
                 "release": "1",
                 "sources": [
-                    self.topdir + "/work/x86_64/repo/p1",
+                    topurl + "work/x86_64/repo/p1",
-                    self.topdir + "/work/x86_64/repo/p2",
+                    topurl + "work/x86_64/repo/p2",
-                    self.topdir + "/work/x86_64/comps_repo_Server",
+                    topurl + "work/x86_64/comps_repo_Server",
                 ],
                 "variant": "Server",
                 "installpkgs": ["bash", "vim"],
@@ -355,7 +355,6 @@ class TestBuildinstallPhase(PungiTestCase):
                 "rootfs-size": None,
                 "dracut-args": [],
                 "skip_branding": False,
-                "outputdir": self.topdir + "/work/x86_64/buildinstall/Server",
                 "squashfs_only": False,
                 "configuration_file": None,
             },
@@ -1234,9 +1233,9 @@ class BuildinstallThreadTestCase(PungiTestCase):
     @mock.patch("pungi.wrappers.kojiwrapper.KojiWrapper")
     @mock.patch("pungi.wrappers.kojiwrapper.get_buildroot_rpms")
     @mock.patch("pungi.phases.buildinstall.run")
-    @mock.patch("pungi.phases.buildinstall.move_all")
+    @mock.patch("pungi.phases.buildinstall.download_and_extract_archive")
     def test_buildinstall_thread_with_lorax_using_koji_plugin(
-        self, move_all, run, get_buildroot_rpms, KojiWrapperMock, mock_tweak, mock_link
+        self, download, run, get_buildroot_rpms, KojiWrapperMock, mock_tweak, mock_link
     ):
         compose = BuildInstallCompose(
             self.topdir,
@@ -1282,9 +1281,8 @@ class BuildinstallThreadTestCase(PungiTestCase):
                     self.cmd,
                     channel=None,
                     packages=["lorax"],
-                    mounts=[self.topdir],
                     weight=123,
-                    chown_uid=os.getuid(),
+                    chown_uid=None,
                 )
             ],
         )
@@ -1325,13 +1323,14 @@ class BuildinstallThreadTestCase(PungiTestCase):
             [mock.call(compose, "x86_64", compose.variants["Server"], False)],
         )
         self.assertEqual(
-            move_all.call_args_list,
+            download.call_args_list,
             [
-                mock.call(os.path.join(destdir, "results"), destdir, rm_src_dir=True),
+                mock.call(compose, 1234, "results.tar.gz", destdir),
                 mock.call(
-                    os.path.join(destdir, "logs"),
+                    compose,
+                    1234,
+                    "logs.tar.gz",
                     os.path.join(self.topdir, "logs/x86_64/buildinstall-Server-logs"),
-                    rm_src_dir=True,
                 ),
             ],
         )