diff --git a/pungi/phases/buildinstall.py b/pungi/phases/buildinstall.py index 5504b50f..136134ec 100644 --- a/pungi/phases/buildinstall.py +++ b/pungi/phases/buildinstall.py @@ -31,14 +31,14 @@ from six.moves import shlex_quote from pungi.arch import get_valid_arches from pungi.util import get_volid, get_arch_variant_data from pungi.util import get_file_size, get_mtime, failable, makedirs -from pungi.util import copy_all, translate_path, move_all +from pungi.util import copy_all, translate_path from pungi.wrappers.lorax import LoraxWrapper from pungi.wrappers import iso from pungi.wrappers.scm import get_file from pungi.wrappers.scm import get_file_from_scm from pungi.wrappers import kojiwrapper from pungi.phases.base import PhaseBase -from pungi.runroot import Runroot +from pungi.runroot import Runroot, download_and_extract_archive class BuildinstallPhase(PhaseBase): @@ -144,7 +144,7 @@ class BuildinstallPhase(PhaseBase): ) if self.compose.has_comps: comps_repo = self.compose.paths.work.comps_repo(arch, variant) - if final_output_dir != output_dir: + if final_output_dir != output_dir or self.lorax_use_koji_plugin: comps_repo = translate_path(self.compose, comps_repo) repos.append(comps_repo) @@ -169,7 +169,6 @@ class BuildinstallPhase(PhaseBase): "rootfs-size": rootfs_size, "dracut-args": dracut_args, "skip_branding": skip_branding, - "outputdir": output_dir, "squashfs_only": squashfs_only, "configuration_file": configuration_file, } @@ -235,7 +234,7 @@ class BuildinstallPhase(PhaseBase): ) makedirs(final_output_dir) repo_baseurls = self.get_repos(arch) - if final_output_dir != output_dir: + if final_output_dir != output_dir or self.lorax_use_koji_plugin: repo_baseurls = [translate_path(self.compose, r) for r in repo_baseurls] if self.buildinstall_method == "lorax": @@ -833,13 +832,13 @@ class BuildinstallThread(WorkerThread): # Start the runroot task. runroot = Runroot(compose, phase="buildinstall") + task_id = None if buildinstall_method == "lorax" and lorax_use_koji_plugin: - runroot.run_pungi_buildinstall( + task_id = runroot.run_pungi_buildinstall( cmd, log_file=log_file, arch=arch, packages=packages, - mounts=[compose.topdir], weight=compose.conf["runroot_weights"].get("buildinstall"), ) else: @@ -872,19 +871,17 @@ class BuildinstallThread(WorkerThread): log_dir = os.path.join(output_dir, "logs") copy_all(log_dir, final_log_dir) elif lorax_use_koji_plugin: - # If Koji pungi-buildinstall is used, then the buildinstall results are - # not stored directly in `output_dir` dir, but in "results" and "logs" - # subdirectories. We need to move them to final_output_dir. - results_dir = os.path.join(output_dir, "results") - move_all(results_dir, final_output_dir, rm_src_dir=True) + # If Koji pungi-buildinstall is used, then the buildinstall results + # are attached as outputs to the Koji task. Download and unpack + # them to the correct location. + download_and_extract_archive( + compose, task_id, "results.tar.gz", final_output_dir + ) - # Get the log_dir into which we should copy the resulting log files. + # Download the logs into proper location too. log_fname = "buildinstall-%s-logs/dummy" % variant.uid final_log_dir = os.path.dirname(compose.paths.log.log_file(arch, log_fname)) - if not os.path.exists(final_log_dir): - makedirs(final_log_dir) - log_dir = os.path.join(output_dir, "logs") - move_all(log_dir, final_log_dir, rm_src_dir=True) + download_and_extract_archive(compose, task_id, "logs.tar.gz", final_log_dir) rpms = runroot.get_buildroot_rpms() self._write_buildinstall_metadata( diff --git a/pungi/runroot.py b/pungi/runroot.py index 166e3988..96985b9d 100644 --- a/pungi/runroot.py +++ b/pungi/runroot.py @@ -13,13 +13,19 @@ # You should have received a copy of the GNU General Public License # along with this program; if not, see . +import contextlib import os import re +import shutil +import tarfile + +import requests import six from six.moves import shlex_quote import kobo.log from kobo.shortcuts import run +from pungi import util from pungi.wrappers import kojiwrapper @@ -314,7 +320,8 @@ class Runroot(kobo.log.LoggingBase): arch, args, channel=runroot_channel, - chown_uid=os.getuid(), + # We want to change owner only if shared NFS directory is used. + chown_uid=os.getuid() if kwargs.get("mounts") else None, **kwargs ) @@ -325,6 +332,7 @@ class Runroot(kobo.log.LoggingBase): % (output["task_id"], log_file) ) self._result = output + return output["task_id"] def run_pungi_ostree(self, args, log_file=None, arch=None, **kwargs): """ @@ -381,3 +389,72 @@ class Runroot(kobo.log.LoggingBase): return self._result else: raise ValueError("Unknown runroot_method %r." % self.runroot_method) + + +@util.retry(wait_on=requests.exceptions.RequestException) +def _download_file(url, dest): + # contextlib.closing is only needed in requests<2.18 + with contextlib.closing(requests.get(url, stream=True, timeout=5)) as r: + if r.status_code == 404: + raise RuntimeError("Archive %s not found" % url) + r.raise_for_status() + with open(dest, "wb") as f: + shutil.copyfileobj(r.raw, f) + + +def _download_archive(task_id, fname, archive_url, dest_dir): + """Download file from URL to a destination, with retries.""" + temp_file = os.path.join(dest_dir, fname) + _download_file(archive_url, temp_file) + return temp_file + + +def _extract_archive(task_id, fname, archive_file, dest_path): + """Extract the archive into given destination. + + All items of the archive must match the name of the archive, i.e. all + paths in foo.tar.gz must start with foo/. + """ + basename = os.path.basename(fname).split(".")[0] + strip_prefix = basename + "/" + with tarfile.open(archive_file, "r") as archive: + for member in archive.getmembers(): + # Check if each item is either the root directory or is within it. + if member.name != basename and not member.name.startswith(strip_prefix): + raise RuntimeError( + "Archive %s from task %s contains file without expected prefix: %s" + % (fname, task_id, member) + ) + dest = os.path.join(dest_path, member.name[len(strip_prefix) :]) + if member.isdir(): + # Create directories where needed... + util.makedirs(dest) + elif member.isfile(): + # ... and extract files into them. + with open(dest, "wb") as dest_obj: + shutil.copyfileobj(archive.extractfile(member), dest_obj) + elif member.islnk(): + # We have a hardlink. Let's also link it. + linked_file = os.path.join( + dest_path, member.linkname[len(strip_prefix) :] + ) + os.link(linked_file, dest) + else: + # Any other file type is an error. + raise RuntimeError( + "Unexpected file type in %s from task %s: %s" + % (fname, task_id, member) + ) + + +def download_and_extract_archive(compose, task_id, fname, destination): + """Download a tar archive from task outputs and extract it to the destination.""" + koji = kojiwrapper.KojiWrapper(compose).koji_module + # Koji API provides downloadTaskOutput method, but it's not usable as it + # will attempt to load the entire file into memory. + # So instead let's generate a patch and attempt to convert it to a URL. + server_path = os.path.join(koji.pathinfo.task(task_id), fname) + archive_url = server_path.replace(koji.config.topdir, koji.config.topurl) + with util.temp_dir(prefix="buildinstall-download") as tmp_dir: + local_path = _download_archive(task_id, fname, archive_url, tmp_dir) + _extract_archive(task_id, fname, local_path, destination) diff --git a/tests/test_buildinstall.py b/tests/test_buildinstall.py index c099714e..9d16b27d 100644 --- a/tests/test_buildinstall.py +++ b/tests/test_buildinstall.py @@ -254,6 +254,7 @@ class TestBuildinstallPhase(PungiTestCase): def test_starts_threads_for_each_cmd_with_lorax_koji_plugin( self, get_volid, poolCls ): + topurl = "https://example.com/composes/" compose = BuildInstallCompose( self.topdir, { @@ -264,6 +265,7 @@ class TestBuildinstallPhase(PungiTestCase): "buildinstall_method": "lorax", "lorax_use_koji_plugin": True, "disc_types": {"dvd": "DVD"}, + "translate_paths": [(self.topdir, topurl)], }, ) @@ -280,9 +282,9 @@ class TestBuildinstallPhase(PungiTestCase): "version": "1", "release": "1", "sources": [ - self.topdir + "/work/amd64/repo/p1", - self.topdir + "/work/amd64/repo/p2", - self.topdir + "/work/amd64/comps_repo_Server", + topurl + "work/amd64/repo/p1", + topurl + "work/amd64/repo/p2", + topurl + "work/amd64/comps_repo_Server", ], "variant": "Server", "installpkgs": ["bash", "vim"], @@ -299,7 +301,6 @@ class TestBuildinstallPhase(PungiTestCase): "rootfs-size": None, "dracut-args": [], "skip_branding": False, - "outputdir": self.topdir + "/work/amd64/buildinstall/Server", "squashfs_only": False, "configuration_file": None, }, @@ -308,9 +309,9 @@ class TestBuildinstallPhase(PungiTestCase): "version": "1", "release": "1", "sources": [ - self.topdir + "/work/amd64/repo/p1", - self.topdir + "/work/amd64/repo/p2", - self.topdir + "/work/amd64/comps_repo_Client", + topurl + "work/amd64/repo/p1", + topurl + "work/amd64/repo/p2", + topurl + "work/amd64/comps_repo_Client", ], "variant": "Client", "installpkgs": [], @@ -327,7 +328,6 @@ class TestBuildinstallPhase(PungiTestCase): "rootfs-size": None, "dracut-args": [], "skip_branding": False, - "outputdir": self.topdir + "/work/amd64/buildinstall/Client", "squashfs_only": False, "configuration_file": None, }, @@ -336,9 +336,9 @@ class TestBuildinstallPhase(PungiTestCase): "version": "1", "release": "1", "sources": [ - self.topdir + "/work/x86_64/repo/p1", - self.topdir + "/work/x86_64/repo/p2", - self.topdir + "/work/x86_64/comps_repo_Server", + topurl + "work/x86_64/repo/p1", + topurl + "work/x86_64/repo/p2", + topurl + "work/x86_64/comps_repo_Server", ], "variant": "Server", "installpkgs": ["bash", "vim"], @@ -355,7 +355,6 @@ class TestBuildinstallPhase(PungiTestCase): "rootfs-size": None, "dracut-args": [], "skip_branding": False, - "outputdir": self.topdir + "/work/x86_64/buildinstall/Server", "squashfs_only": False, "configuration_file": None, }, @@ -1234,9 +1233,9 @@ class BuildinstallThreadTestCase(PungiTestCase): @mock.patch("pungi.wrappers.kojiwrapper.KojiWrapper") @mock.patch("pungi.wrappers.kojiwrapper.get_buildroot_rpms") @mock.patch("pungi.phases.buildinstall.run") - @mock.patch("pungi.phases.buildinstall.move_all") + @mock.patch("pungi.phases.buildinstall.download_and_extract_archive") def test_buildinstall_thread_with_lorax_using_koji_plugin( - self, move_all, run, get_buildroot_rpms, KojiWrapperMock, mock_tweak, mock_link + self, download, run, get_buildroot_rpms, KojiWrapperMock, mock_tweak, mock_link ): compose = BuildInstallCompose( self.topdir, @@ -1282,9 +1281,8 @@ class BuildinstallThreadTestCase(PungiTestCase): self.cmd, channel=None, packages=["lorax"], - mounts=[self.topdir], weight=123, - chown_uid=os.getuid(), + chown_uid=None, ) ], ) @@ -1325,13 +1323,14 @@ class BuildinstallThreadTestCase(PungiTestCase): [mock.call(compose, "x86_64", compose.variants["Server"], False)], ) self.assertEqual( - move_all.call_args_list, + download.call_args_list, [ - mock.call(os.path.join(destdir, "results"), destdir, rm_src_dir=True), + mock.call(compose, 1234, "results.tar.gz", destdir), mock.call( - os.path.join(destdir, "logs"), + compose, + 1234, + "logs.tar.gz", os.path.join(self.topdir, "logs/x86_64/buildinstall-Server-logs"), - rm_src_dir=True, ), ], )