almalinux/pungi
14
1
Fork 0
Code Issues Pull Requests Releases Activity

Use pungi_buildinstall without NFS

Browse Source 
The plugin supports two modes of operation:
1. Mount a shared storage volume into the runroot and have the output
   written there.
2. Have the plugin create a tar.gz with the outputs and upload them to
   the hub, from where they can be downloaded.

This patch switches from option 1 to option 2.

This requires all input repositories to be passes in as URLs and not
paths. Once the task finishes, Pungi will download the output archives
and unpack them into the expected locations.

JIRA: RHELCMP-13284
Signed-off-by: Lubomír Sedlář <lsedlar@redhat.com>
This commit is contained in:
Lubomír Sedlář 2024-01-15 13:58:13 +01:00
parent 432b0bce04
commit f25489d060
3 changed files with 111 additions and 38 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

31
pungi/phases/buildinstall.py
View File

@ -31,14 +31,14 @@ from six.moves import shlex_quote
from pungi.arch import get_valid_arches
from pungi.util import get_volid, get_arch_variant_data
from pungi.util import get_file_size, get_mtime, failable, makedirs
from pungi.util import copy_all, translate_path, move_all
from pungi.util import copy_all, translate_path
from pungi.wrappers.lorax import LoraxWrapper
from pungi.wrappers import iso
from pungi.wrappers.scm import get_file
from pungi.wrappers.scm import get_file_from_scm
from pungi.wrappers import kojiwrapper
from pungi.phases.base import PhaseBase
from pungi.runroot import Runroot
from pungi.runroot import Runroot, download_and_extract_archive
class BuildinstallPhase(PhaseBase):
@ -144,7 +144,7 @@ class BuildinstallPhase(PhaseBase):
)
if self.compose.has_comps:
comps_repo = self.compose.paths.work.comps_repo(arch, variant)
if final_output_dir != output_dir:
if final_output_dir != output_dir or self.lorax_use_koji_plugin:
comps_repo = translate_path(self.compose, comps_repo)
repos.append(comps_repo)
@ -169,7 +169,6 @@ class BuildinstallPhase(PhaseBase):
"rootfs-size": rootfs_size,
"dracut-args": dracut_args,
"skip_branding": skip_branding,
"outputdir": output_dir,
"squashfs_only": squashfs_only,
"configuration_file": configuration_file,
}
@ -235,7 +234,7 @@ class BuildinstallPhase(PhaseBase):
)
makedirs(final_output_dir)
repo_baseurls = self.get_repos(arch)
if final_output_dir != output_dir:
if final_output_dir != output_dir or self.lorax_use_koji_plugin:
repo_baseurls = [translate_path(self.compose, r) for r in repo_baseurls]
if self.buildinstall_method == "lorax":
@ -826,13 +825,13 @@ class BuildinstallThread(WorkerThread):
# Start the runroot task.
runroot = Runroot(compose, phase="buildinstall")
task_id = None
if buildinstall_method == "lorax" and lorax_use_koji_plugin:
runroot.run_pungi_buildinstall(
task_id = runroot.run_pungi_buildinstall(
cmd,
log_file=log_file,
arch=arch,
packages=packages,
mounts=[compose.topdir],
weight=compose.conf["runroot_weights"].get("buildinstall"),
)
else:
@ -865,19 +864,17 @@ class BuildinstallThread(WorkerThread):
log_dir = os.path.join(output_dir, "logs")
copy_all(log_dir, final_log_dir)
elif lorax_use_koji_plugin:
# If Koji pungi-buildinstall is used, then the buildinstall results are
# not stored directly in `output_dir` dir, but in "results" and "logs"
# subdirectories. We need to move them to final_output_dir.
results_dir = os.path.join(output_dir, "results")
move_all(results_dir, final_output_dir, rm_src_dir=True)
# If Koji pungi-buildinstall is used, then the buildinstall results
# are attached as outputs to the Koji task. Download and unpack
# them to the correct location.
download_and_extract_archive(
compose, task_id, "results.tar.gz", final_output_dir
)
# Get the log_dir into which we should copy the resulting log files.
# Download the logs into proper location too.
log_fname = "buildinstall-%s-logs/dummy" % variant.uid
final_log_dir = os.path.dirname(compose.paths.log.log_file(arch, log_fname))
if not os.path.exists(final_log_dir):
makedirs(final_log_dir)
log_dir = os.path.join(output_dir, "logs")
move_all(log_dir, final_log_dir, rm_src_dir=True)
download_and_extract_archive(compose, task_id, "logs.tar.gz", final_log_dir)
rpms = runroot.get_buildroot_rpms()
self._write_buildinstall_metadata(

79
pungi/runroot.py
View File

@ -13,13 +13,19 @@
# You should have received a copy of the GNU General Public License
# along with this program; if not, see <https://gnu.org/licenses/>.
import contextlib
import os
import re
import shutil
import tarfile
import requests
import six
from six.moves import shlex_quote
import kobo.log
from kobo.shortcuts import run
from pungi import util
from pungi.wrappers import kojiwrapper
@ -314,7 +320,8 @@ class Runroot(kobo.log.LoggingBase):
arch,
args,
channel=runroot_channel,
chown_uid=os.getuid(),
# We want to change owner only if shared NFS directory is used.
chown_uid=os.getuid() if kwargs.get("mounts") else None,
**kwargs
)
@ -325,6 +332,7 @@ class Runroot(kobo.log.LoggingBase):
% (output["task_id"], log_file)
)
self._result = output
return output["task_id"]
def run_pungi_ostree(self, args, log_file=None, arch=None, **kwargs):
"""
@ -381,3 +389,72 @@ class Runroot(kobo.log.LoggingBase):
return self._result
else:
raise ValueError("Unknown runroot_method %r." % self.runroot_method)
@util.retry(wait_on=requests.exceptions.RequestException)
def _download_file(url, dest):
# contextlib.closing is only needed in requests<2.18
with contextlib.closing(requests.get(url, stream=True, timeout=5)) as r:
if r.status_code == 404:
raise RuntimeError("Archive %s not found" % url)
r.raise_for_status()
with open(dest, "wb") as f:
shutil.copyfileobj(r.raw, f)
def _download_archive(task_id, fname, archive_url, dest_dir):
"""Download file from URL to a destination, with retries."""
temp_file = os.path.join(dest_dir, fname)
_download_file(archive_url, temp_file)
return temp_file
def _extract_archive(task_id, fname, archive_file, dest_path):
"""Extract the archive into given destination.
All items of the archive must match the name of the archive, i.e. all
paths in foo.tar.gz must start with foo/.
"""
basename = os.path.basename(fname).split(".")[0]
strip_prefix = basename + "/"
with tarfile.open(archive_file, "r") as archive:
for member in archive.getmembers():
# Check if each item is either the root directory or is within it.
if member.name != basename and not member.name.startswith(strip_prefix):
raise RuntimeError(
"Archive %s from task %s contains file without expected prefix: %s"
% (fname, task_id, member)
)
dest = os.path.join(dest_path, member.name[len(strip_prefix) :])
if member.isdir():
# Create directories where needed...
util.makedirs(dest)
elif member.isfile():
# ... and extract files into them.
with open(dest, "wb") as dest_obj:
shutil.copyfileobj(archive.extractfile(member), dest_obj)
elif member.islnk():
# We have a hardlink. Let's also link it.
linked_file = os.path.join(
dest_path, member.linkname[len(strip_prefix) :]
)
os.link(linked_file, dest)
else:
# Any other file type is an error.
raise RuntimeError(
"Unexpected file type in %s from task %s: %s"
% (fname, task_id, member)
)
def download_and_extract_archive(compose, task_id, fname, destination):
"""Download a tar archive from task outputs and extract it to the destination."""
koji = kojiwrapper.KojiWrapper(compose).koji_module
# Koji API provides downloadTaskOutput method, but it's not usable as it
# will attempt to load the entire file into memory.
# So instead let's generate a patch and attempt to convert it to a URL.
server_path = os.path.join(koji.pathinfo.task(task_id), fname)
archive_url = server_path.replace(koji.config.topdir, koji.config.topurl)
with util.temp_dir(prefix="buildinstall-download") as tmp_dir:
local_path = _download_archive(task_id, fname, archive_url, tmp_dir)
_extract_archive(task_id, fname, local_path, destination)

39
tests/test_buildinstall.py
View File

@ -254,6 +254,7 @@ class TestBuildinstallPhase(PungiTestCase):
def test_starts_threads_for_each_cmd_with_lorax_koji_plugin(
self, get_volid, poolCls
):
topurl = "https://example.com/composes/"
compose = BuildInstallCompose(
self.topdir,
{
@ -264,6 +265,7 @@ class TestBuildinstallPhase(PungiTestCase):
"buildinstall_method": "lorax",
"lorax_use_koji_plugin": True,
"disc_types": {"dvd": "DVD"},
"translate_paths": [(self.topdir, topurl)],
},
)
@ -280,9 +282,9 @@ class TestBuildinstallPhase(PungiTestCase):
"version": "1",
"release": "1",
"sources": [
self.topdir + "/work/amd64/repo/p1",
self.topdir + "/work/amd64/repo/p2",
self.topdir + "/work/amd64/comps_repo_Server",
topurl + "work/amd64/repo/p1",
topurl + "work/amd64/repo/p2",
topurl + "work/amd64/comps_repo_Server",
],
"variant": "Server",
"installpkgs": ["bash", "vim"],
@ -299,7 +301,6 @@ class TestBuildinstallPhase(PungiTestCase):
"rootfs-size": None,
"dracut-args": [],
"skip_branding": False,
"outputdir": self.topdir + "/work/amd64/buildinstall/Server",
"squashfs_only": False,
"configuration_file": None,
},
@ -308,9 +309,9 @@ class TestBuildinstallPhase(PungiTestCase):
"version": "1",
"release": "1",
"sources": [
self.topdir + "/work/amd64/repo/p1",
self.topdir + "/work/amd64/repo/p2",
self.topdir + "/work/amd64/comps_repo_Client",
topurl + "work/amd64/repo/p1",
topurl + "work/amd64/repo/p2",
topurl + "work/amd64/comps_repo_Client",
],
"variant": "Client",
"installpkgs": [],
@ -327,7 +328,6 @@ class TestBuildinstallPhase(PungiTestCase):
"rootfs-size": None,
"dracut-args": [],
"skip_branding": False,
"outputdir": self.topdir + "/work/amd64/buildinstall/Client",
"squashfs_only": False,
"configuration_file": None,
},
@ -336,9 +336,9 @@ class TestBuildinstallPhase(PungiTestCase):
"version": "1",
"release": "1",
"sources": [
self.topdir + "/work/x86_64/repo/p1",
self.topdir + "/work/x86_64/repo/p2",
self.topdir + "/work/x86_64/comps_repo_Server",
topurl + "work/x86_64/repo/p1",
topurl + "work/x86_64/repo/p2",
topurl + "work/x86_64/comps_repo_Server",
],
"variant": "Server",
"installpkgs": ["bash", "vim"],
@ -355,7 +355,6 @@ class TestBuildinstallPhase(PungiTestCase):
"rootfs-size": None,
"dracut-args": [],
"skip_branding": False,
"outputdir": self.topdir + "/work/x86_64/buildinstall/Server",
"squashfs_only": False,
"configuration_file": None,
},
@ -1234,9 +1233,9 @@ class BuildinstallThreadTestCase(PungiTestCase):
@mock.patch("pungi.wrappers.kojiwrapper.KojiWrapper")
@mock.patch("pungi.wrappers.kojiwrapper.get_buildroot_rpms")
@mock.patch("pungi.phases.buildinstall.run")
@mock.patch("pungi.phases.buildinstall.move_all")
@mock.patch("pungi.phases.buildinstall.download_and_extract_archive")
def test_buildinstall_thread_with_lorax_using_koji_plugin(
self, move_all, run, get_buildroot_rpms, KojiWrapperMock, mock_tweak, mock_link
self, download, run, get_buildroot_rpms, KojiWrapperMock, mock_tweak, mock_link
):
compose = BuildInstallCompose(
self.topdir,
@ -1282,9 +1281,8 @@ class BuildinstallThreadTestCase(PungiTestCase):
self.cmd,
channel=None,
packages=["lorax"],
mounts=[self.topdir],
weight=123,
chown_uid=os.getuid(),
chown_uid=None,
)
],
)
@ -1325,13 +1323,14 @@ class BuildinstallThreadTestCase(PungiTestCase):
[mock.call(compose, "x86_64", compose.variants["Server"], False)],
)
self.assertEqual(
move_all.call_args_list,
download.call_args_list,
[
mock.call(os.path.join(destdir, "results"), destdir, rm_src_dir=True),
mock.call(compose, 1234, "results.tar.gz", destdir),
mock.call(
os.path.join(destdir, "logs"),
compose,
1234,
"logs.tar.gz",
os.path.join(self.topdir, "logs/x86_64/buildinstall-Server-logs"),
rm_src_dir=True,
),
],
)