From f25489d060dc74545818cd4b3b0bf2f9431843b3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lubom=C3=ADr=20Sedl=C3=A1=C5=99?= Date: Mon, 15 Jan 2024 13:58:13 +0100 Subject: [PATCH] Use pungi_buildinstall without NFS MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The plugin supports two modes of operation: 1. Mount a shared storage volume into the runroot and have the output written there. 2. Have the plugin create a tar.gz with the outputs and upload them to the hub, from where they can be downloaded. This patch switches from option 1 to option 2. This requires all input repositories to be passes in as URLs and not paths. Once the task finishes, Pungi will download the output archives and unpack them into the expected locations. JIRA: RHELCMP-13284 Signed-off-by: Lubomír Sedlář --- pungi/phases/buildinstall.py | 31 +++++++------- pungi/runroot.py | 79 +++++++++++++++++++++++++++++++++++- tests/test_buildinstall.py | 39 +++++++++--------- 3 files changed, 111 insertions(+), 38 deletions(-) diff --git a/pungi/phases/buildinstall.py b/pungi/phases/buildinstall.py index 5c6b69d7..50d1b3e9 100644 --- a/pungi/phases/buildinstall.py +++ b/pungi/phases/buildinstall.py @@ -31,14 +31,14 @@ from six.moves import shlex_quote from pungi.arch import get_valid_arches from pungi.util import get_volid, get_arch_variant_data from pungi.util import get_file_size, get_mtime, failable, makedirs -from pungi.util import copy_all, translate_path, move_all +from pungi.util import copy_all, translate_path from pungi.wrappers.lorax import LoraxWrapper from pungi.wrappers import iso from pungi.wrappers.scm import get_file from pungi.wrappers.scm import get_file_from_scm from pungi.wrappers import kojiwrapper from pungi.phases.base import PhaseBase -from pungi.runroot import Runroot +from pungi.runroot import Runroot, download_and_extract_archive class BuildinstallPhase(PhaseBase): @@ -144,7 +144,7 @@ class BuildinstallPhase(PhaseBase): ) if self.compose.has_comps: comps_repo = self.compose.paths.work.comps_repo(arch, variant) - if final_output_dir != output_dir: + if final_output_dir != output_dir or self.lorax_use_koji_plugin: comps_repo = translate_path(self.compose, comps_repo) repos.append(comps_repo) @@ -169,7 +169,6 @@ class BuildinstallPhase(PhaseBase): "rootfs-size": rootfs_size, "dracut-args": dracut_args, "skip_branding": skip_branding, - "outputdir": output_dir, "squashfs_only": squashfs_only, "configuration_file": configuration_file, } @@ -235,7 +234,7 @@ class BuildinstallPhase(PhaseBase): ) makedirs(final_output_dir) repo_baseurls = self.get_repos(arch) - if final_output_dir != output_dir: + if final_output_dir != output_dir or self.lorax_use_koji_plugin: repo_baseurls = [translate_path(self.compose, r) for r in repo_baseurls] if self.buildinstall_method == "lorax": @@ -826,13 +825,13 @@ class BuildinstallThread(WorkerThread): # Start the runroot task. runroot = Runroot(compose, phase="buildinstall") + task_id = None if buildinstall_method == "lorax" and lorax_use_koji_plugin: - runroot.run_pungi_buildinstall( + task_id = runroot.run_pungi_buildinstall( cmd, log_file=log_file, arch=arch, packages=packages, - mounts=[compose.topdir], weight=compose.conf["runroot_weights"].get("buildinstall"), ) else: @@ -865,19 +864,17 @@ class BuildinstallThread(WorkerThread): log_dir = os.path.join(output_dir, "logs") copy_all(log_dir, final_log_dir) elif lorax_use_koji_plugin: - # If Koji pungi-buildinstall is used, then the buildinstall results are - # not stored directly in `output_dir` dir, but in "results" and "logs" - # subdirectories. We need to move them to final_output_dir. - results_dir = os.path.join(output_dir, "results") - move_all(results_dir, final_output_dir, rm_src_dir=True) + # If Koji pungi-buildinstall is used, then the buildinstall results + # are attached as outputs to the Koji task. Download and unpack + # them to the correct location. + download_and_extract_archive( + compose, task_id, "results.tar.gz", final_output_dir + ) - # Get the log_dir into which we should copy the resulting log files. + # Download the logs into proper location too. log_fname = "buildinstall-%s-logs/dummy" % variant.uid final_log_dir = os.path.dirname(compose.paths.log.log_file(arch, log_fname)) - if not os.path.exists(final_log_dir): - makedirs(final_log_dir) - log_dir = os.path.join(output_dir, "logs") - move_all(log_dir, final_log_dir, rm_src_dir=True) + download_and_extract_archive(compose, task_id, "logs.tar.gz", final_log_dir) rpms = runroot.get_buildroot_rpms() self._write_buildinstall_metadata( diff --git a/pungi/runroot.py b/pungi/runroot.py index eb7e2dc4..e545ac20 100644 --- a/pungi/runroot.py +++ b/pungi/runroot.py @@ -13,13 +13,19 @@ # You should have received a copy of the GNU General Public License # along with this program; if not, see . +import contextlib import os import re +import shutil +import tarfile + +import requests import six from six.moves import shlex_quote import kobo.log from kobo.shortcuts import run +from pungi import util from pungi.wrappers import kojiwrapper @@ -314,7 +320,8 @@ class Runroot(kobo.log.LoggingBase): arch, args, channel=runroot_channel, - chown_uid=os.getuid(), + # We want to change owner only if shared NFS directory is used. + chown_uid=os.getuid() if kwargs.get("mounts") else None, **kwargs ) @@ -325,6 +332,7 @@ class Runroot(kobo.log.LoggingBase): % (output["task_id"], log_file) ) self._result = output + return output["task_id"] def run_pungi_ostree(self, args, log_file=None, arch=None, **kwargs): """ @@ -381,3 +389,72 @@ class Runroot(kobo.log.LoggingBase): return self._result else: raise ValueError("Unknown runroot_method %r." % self.runroot_method) + + +@util.retry(wait_on=requests.exceptions.RequestException) +def _download_file(url, dest): + # contextlib.closing is only needed in requests<2.18 + with contextlib.closing(requests.get(url, stream=True, timeout=5)) as r: + if r.status_code == 404: + raise RuntimeError("Archive %s not found" % url) + r.raise_for_status() + with open(dest, "wb") as f: + shutil.copyfileobj(r.raw, f) + + +def _download_archive(task_id, fname, archive_url, dest_dir): + """Download file from URL to a destination, with retries.""" + temp_file = os.path.join(dest_dir, fname) + _download_file(archive_url, temp_file) + return temp_file + + +def _extract_archive(task_id, fname, archive_file, dest_path): + """Extract the archive into given destination. + + All items of the archive must match the name of the archive, i.e. all + paths in foo.tar.gz must start with foo/. + """ + basename = os.path.basename(fname).split(".")[0] + strip_prefix = basename + "/" + with tarfile.open(archive_file, "r") as archive: + for member in archive.getmembers(): + # Check if each item is either the root directory or is within it. + if member.name != basename and not member.name.startswith(strip_prefix): + raise RuntimeError( + "Archive %s from task %s contains file without expected prefix: %s" + % (fname, task_id, member) + ) + dest = os.path.join(dest_path, member.name[len(strip_prefix) :]) + if member.isdir(): + # Create directories where needed... + util.makedirs(dest) + elif member.isfile(): + # ... and extract files into them. + with open(dest, "wb") as dest_obj: + shutil.copyfileobj(archive.extractfile(member), dest_obj) + elif member.islnk(): + # We have a hardlink. Let's also link it. + linked_file = os.path.join( + dest_path, member.linkname[len(strip_prefix) :] + ) + os.link(linked_file, dest) + else: + # Any other file type is an error. + raise RuntimeError( + "Unexpected file type in %s from task %s: %s" + % (fname, task_id, member) + ) + + +def download_and_extract_archive(compose, task_id, fname, destination): + """Download a tar archive from task outputs and extract it to the destination.""" + koji = kojiwrapper.KojiWrapper(compose).koji_module + # Koji API provides downloadTaskOutput method, but it's not usable as it + # will attempt to load the entire file into memory. + # So instead let's generate a patch and attempt to convert it to a URL. + server_path = os.path.join(koji.pathinfo.task(task_id), fname) + archive_url = server_path.replace(koji.config.topdir, koji.config.topurl) + with util.temp_dir(prefix="buildinstall-download") as tmp_dir: + local_path = _download_archive(task_id, fname, archive_url, tmp_dir) + _extract_archive(task_id, fname, local_path, destination) diff --git a/tests/test_buildinstall.py b/tests/test_buildinstall.py index 8acc83e1..b92abe11 100644 --- a/tests/test_buildinstall.py +++ b/tests/test_buildinstall.py @@ -254,6 +254,7 @@ class TestBuildinstallPhase(PungiTestCase): def test_starts_threads_for_each_cmd_with_lorax_koji_plugin( self, get_volid, poolCls ): + topurl = "https://example.com/composes/" compose = BuildInstallCompose( self.topdir, { @@ -264,6 +265,7 @@ class TestBuildinstallPhase(PungiTestCase): "buildinstall_method": "lorax", "lorax_use_koji_plugin": True, "disc_types": {"dvd": "DVD"}, + "translate_paths": [(self.topdir, topurl)], }, ) @@ -280,9 +282,9 @@ class TestBuildinstallPhase(PungiTestCase): "version": "1", "release": "1", "sources": [ - self.topdir + "/work/amd64/repo/p1", - self.topdir + "/work/amd64/repo/p2", - self.topdir + "/work/amd64/comps_repo_Server", + topurl + "work/amd64/repo/p1", + topurl + "work/amd64/repo/p2", + topurl + "work/amd64/comps_repo_Server", ], "variant": "Server", "installpkgs": ["bash", "vim"], @@ -299,7 +301,6 @@ class TestBuildinstallPhase(PungiTestCase): "rootfs-size": None, "dracut-args": [], "skip_branding": False, - "outputdir": self.topdir + "/work/amd64/buildinstall/Server", "squashfs_only": False, "configuration_file": None, }, @@ -308,9 +309,9 @@ class TestBuildinstallPhase(PungiTestCase): "version": "1", "release": "1", "sources": [ - self.topdir + "/work/amd64/repo/p1", - self.topdir + "/work/amd64/repo/p2", - self.topdir + "/work/amd64/comps_repo_Client", + topurl + "work/amd64/repo/p1", + topurl + "work/amd64/repo/p2", + topurl + "work/amd64/comps_repo_Client", ], "variant": "Client", "installpkgs": [], @@ -327,7 +328,6 @@ class TestBuildinstallPhase(PungiTestCase): "rootfs-size": None, "dracut-args": [], "skip_branding": False, - "outputdir": self.topdir + "/work/amd64/buildinstall/Client", "squashfs_only": False, "configuration_file": None, }, @@ -336,9 +336,9 @@ class TestBuildinstallPhase(PungiTestCase): "version": "1", "release": "1", "sources": [ - self.topdir + "/work/x86_64/repo/p1", - self.topdir + "/work/x86_64/repo/p2", - self.topdir + "/work/x86_64/comps_repo_Server", + topurl + "work/x86_64/repo/p1", + topurl + "work/x86_64/repo/p2", + topurl + "work/x86_64/comps_repo_Server", ], "variant": "Server", "installpkgs": ["bash", "vim"], @@ -355,7 +355,6 @@ class TestBuildinstallPhase(PungiTestCase): "rootfs-size": None, "dracut-args": [], "skip_branding": False, - "outputdir": self.topdir + "/work/x86_64/buildinstall/Server", "squashfs_only": False, "configuration_file": None, }, @@ -1234,9 +1233,9 @@ class BuildinstallThreadTestCase(PungiTestCase): @mock.patch("pungi.wrappers.kojiwrapper.KojiWrapper") @mock.patch("pungi.wrappers.kojiwrapper.get_buildroot_rpms") @mock.patch("pungi.phases.buildinstall.run") - @mock.patch("pungi.phases.buildinstall.move_all") + @mock.patch("pungi.phases.buildinstall.download_and_extract_archive") def test_buildinstall_thread_with_lorax_using_koji_plugin( - self, move_all, run, get_buildroot_rpms, KojiWrapperMock, mock_tweak, mock_link + self, download, run, get_buildroot_rpms, KojiWrapperMock, mock_tweak, mock_link ): compose = BuildInstallCompose( self.topdir, @@ -1282,9 +1281,8 @@ class BuildinstallThreadTestCase(PungiTestCase): self.cmd, channel=None, packages=["lorax"], - mounts=[self.topdir], weight=123, - chown_uid=os.getuid(), + chown_uid=None, ) ], ) @@ -1325,13 +1323,14 @@ class BuildinstallThreadTestCase(PungiTestCase): [mock.call(compose, "x86_64", compose.variants["Server"], False)], ) self.assertEqual( - move_all.call_args_list, + download.call_args_list, [ - mock.call(os.path.join(destdir, "results"), destdir, rm_src_dir=True), + mock.call(compose, 1234, "results.tar.gz", destdir), mock.call( - os.path.join(destdir, "logs"), + compose, + 1234, + "logs.tar.gz", os.path.join(self.topdir, "logs/x86_64/buildinstall-Server-logs"), - rm_src_dir=True, ), ], )