Use pungi_buildinstall without NFS
The plugin supports two modes of operation:
1. Mount a shared storage volume into the runroot and have the output
written there.
2. Have the plugin create a tar.gz with the outputs and upload them to
the hub, from where they can be downloaded.
This patch switches from option 1 to option 2.
This requires all input repositories to be passes in as URLs and not
paths. Once the task finishes, Pungi will download the output archives
and unpack them into the expected locations.
JIRA: RHELCMP-13284
Signed-off-by: Lubomír Sedlář <lsedlar@redhat.com>
(cherry picked from commit f25489d060
)
This commit is contained in:
parent
4c0059e91b
commit
34fcd550b6
@ -31,14 +31,14 @@ from six.moves import shlex_quote
|
||||
from pungi.arch import get_valid_arches
|
||||
from pungi.util import get_volid, get_arch_variant_data
|
||||
from pungi.util import get_file_size, get_mtime, failable, makedirs
|
||||
from pungi.util import copy_all, translate_path, move_all
|
||||
from pungi.util import copy_all, translate_path
|
||||
from pungi.wrappers.lorax import LoraxWrapper
|
||||
from pungi.wrappers import iso
|
||||
from pungi.wrappers.scm import get_file
|
||||
from pungi.wrappers.scm import get_file_from_scm
|
||||
from pungi.wrappers import kojiwrapper
|
||||
from pungi.phases.base import PhaseBase
|
||||
from pungi.runroot import Runroot
|
||||
from pungi.runroot import Runroot, download_and_extract_archive
|
||||
|
||||
|
||||
class BuildinstallPhase(PhaseBase):
|
||||
@ -144,7 +144,7 @@ class BuildinstallPhase(PhaseBase):
|
||||
)
|
||||
if self.compose.has_comps:
|
||||
comps_repo = self.compose.paths.work.comps_repo(arch, variant)
|
||||
if final_output_dir != output_dir:
|
||||
if final_output_dir != output_dir or self.lorax_use_koji_plugin:
|
||||
comps_repo = translate_path(self.compose, comps_repo)
|
||||
repos.append(comps_repo)
|
||||
|
||||
@ -169,7 +169,6 @@ class BuildinstallPhase(PhaseBase):
|
||||
"rootfs-size": rootfs_size,
|
||||
"dracut-args": dracut_args,
|
||||
"skip_branding": skip_branding,
|
||||
"outputdir": output_dir,
|
||||
"squashfs_only": squashfs_only,
|
||||
"configuration_file": configuration_file,
|
||||
}
|
||||
@ -235,7 +234,7 @@ class BuildinstallPhase(PhaseBase):
|
||||
)
|
||||
makedirs(final_output_dir)
|
||||
repo_baseurls = self.get_repos(arch)
|
||||
if final_output_dir != output_dir:
|
||||
if final_output_dir != output_dir or self.lorax_use_koji_plugin:
|
||||
repo_baseurls = [translate_path(self.compose, r) for r in repo_baseurls]
|
||||
|
||||
if self.buildinstall_method == "lorax":
|
||||
@ -833,13 +832,13 @@ class BuildinstallThread(WorkerThread):
|
||||
|
||||
# Start the runroot task.
|
||||
runroot = Runroot(compose, phase="buildinstall")
|
||||
task_id = None
|
||||
if buildinstall_method == "lorax" and lorax_use_koji_plugin:
|
||||
runroot.run_pungi_buildinstall(
|
||||
task_id = runroot.run_pungi_buildinstall(
|
||||
cmd,
|
||||
log_file=log_file,
|
||||
arch=arch,
|
||||
packages=packages,
|
||||
mounts=[compose.topdir],
|
||||
weight=compose.conf["runroot_weights"].get("buildinstall"),
|
||||
)
|
||||
else:
|
||||
@ -872,19 +871,17 @@ class BuildinstallThread(WorkerThread):
|
||||
log_dir = os.path.join(output_dir, "logs")
|
||||
copy_all(log_dir, final_log_dir)
|
||||
elif lorax_use_koji_plugin:
|
||||
# If Koji pungi-buildinstall is used, then the buildinstall results are
|
||||
# not stored directly in `output_dir` dir, but in "results" and "logs"
|
||||
# subdirectories. We need to move them to final_output_dir.
|
||||
results_dir = os.path.join(output_dir, "results")
|
||||
move_all(results_dir, final_output_dir, rm_src_dir=True)
|
||||
# If Koji pungi-buildinstall is used, then the buildinstall results
|
||||
# are attached as outputs to the Koji task. Download and unpack
|
||||
# them to the correct location.
|
||||
download_and_extract_archive(
|
||||
compose, task_id, "results.tar.gz", final_output_dir
|
||||
)
|
||||
|
||||
# Get the log_dir into which we should copy the resulting log files.
|
||||
# Download the logs into proper location too.
|
||||
log_fname = "buildinstall-%s-logs/dummy" % variant.uid
|
||||
final_log_dir = os.path.dirname(compose.paths.log.log_file(arch, log_fname))
|
||||
if not os.path.exists(final_log_dir):
|
||||
makedirs(final_log_dir)
|
||||
log_dir = os.path.join(output_dir, "logs")
|
||||
move_all(log_dir, final_log_dir, rm_src_dir=True)
|
||||
download_and_extract_archive(compose, task_id, "logs.tar.gz", final_log_dir)
|
||||
|
||||
rpms = runroot.get_buildroot_rpms()
|
||||
self._write_buildinstall_metadata(
|
||||
|
@ -13,13 +13,19 @@
|
||||
# You should have received a copy of the GNU General Public License
|
||||
# along with this program; if not, see <https://gnu.org/licenses/>.
|
||||
|
||||
import contextlib
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import tarfile
|
||||
|
||||
import requests
|
||||
import six
|
||||
from six.moves import shlex_quote
|
||||
import kobo.log
|
||||
from kobo.shortcuts import run
|
||||
|
||||
from pungi import util
|
||||
from pungi.wrappers import kojiwrapper
|
||||
|
||||
|
||||
@ -314,7 +320,8 @@ class Runroot(kobo.log.LoggingBase):
|
||||
arch,
|
||||
args,
|
||||
channel=runroot_channel,
|
||||
chown_uid=os.getuid(),
|
||||
# We want to change owner only if shared NFS directory is used.
|
||||
chown_uid=os.getuid() if kwargs.get("mounts") else None,
|
||||
**kwargs
|
||||
)
|
||||
|
||||
@ -325,6 +332,7 @@ class Runroot(kobo.log.LoggingBase):
|
||||
% (output["task_id"], log_file)
|
||||
)
|
||||
self._result = output
|
||||
return output["task_id"]
|
||||
|
||||
def run_pungi_ostree(self, args, log_file=None, arch=None, **kwargs):
|
||||
"""
|
||||
@ -381,3 +389,72 @@ class Runroot(kobo.log.LoggingBase):
|
||||
return self._result
|
||||
else:
|
||||
raise ValueError("Unknown runroot_method %r." % self.runroot_method)
|
||||
|
||||
|
||||
@util.retry(wait_on=requests.exceptions.RequestException)
|
||||
def _download_file(url, dest):
|
||||
# contextlib.closing is only needed in requests<2.18
|
||||
with contextlib.closing(requests.get(url, stream=True, timeout=5)) as r:
|
||||
if r.status_code == 404:
|
||||
raise RuntimeError("Archive %s not found" % url)
|
||||
r.raise_for_status()
|
||||
with open(dest, "wb") as f:
|
||||
shutil.copyfileobj(r.raw, f)
|
||||
|
||||
|
||||
def _download_archive(task_id, fname, archive_url, dest_dir):
|
||||
"""Download file from URL to a destination, with retries."""
|
||||
temp_file = os.path.join(dest_dir, fname)
|
||||
_download_file(archive_url, temp_file)
|
||||
return temp_file
|
||||
|
||||
|
||||
def _extract_archive(task_id, fname, archive_file, dest_path):
|
||||
"""Extract the archive into given destination.
|
||||
|
||||
All items of the archive must match the name of the archive, i.e. all
|
||||
paths in foo.tar.gz must start with foo/.
|
||||
"""
|
||||
basename = os.path.basename(fname).split(".")[0]
|
||||
strip_prefix = basename + "/"
|
||||
with tarfile.open(archive_file, "r") as archive:
|
||||
for member in archive.getmembers():
|
||||
# Check if each item is either the root directory or is within it.
|
||||
if member.name != basename and not member.name.startswith(strip_prefix):
|
||||
raise RuntimeError(
|
||||
"Archive %s from task %s contains file without expected prefix: %s"
|
||||
% (fname, task_id, member)
|
||||
)
|
||||
dest = os.path.join(dest_path, member.name[len(strip_prefix) :])
|
||||
if member.isdir():
|
||||
# Create directories where needed...
|
||||
util.makedirs(dest)
|
||||
elif member.isfile():
|
||||
# ... and extract files into them.
|
||||
with open(dest, "wb") as dest_obj:
|
||||
shutil.copyfileobj(archive.extractfile(member), dest_obj)
|
||||
elif member.islnk():
|
||||
# We have a hardlink. Let's also link it.
|
||||
linked_file = os.path.join(
|
||||
dest_path, member.linkname[len(strip_prefix) :]
|
||||
)
|
||||
os.link(linked_file, dest)
|
||||
else:
|
||||
# Any other file type is an error.
|
||||
raise RuntimeError(
|
||||
"Unexpected file type in %s from task %s: %s"
|
||||
% (fname, task_id, member)
|
||||
)
|
||||
|
||||
|
||||
def download_and_extract_archive(compose, task_id, fname, destination):
|
||||
"""Download a tar archive from task outputs and extract it to the destination."""
|
||||
koji = kojiwrapper.KojiWrapper(compose).koji_module
|
||||
# Koji API provides downloadTaskOutput method, but it's not usable as it
|
||||
# will attempt to load the entire file into memory.
|
||||
# So instead let's generate a patch and attempt to convert it to a URL.
|
||||
server_path = os.path.join(koji.pathinfo.task(task_id), fname)
|
||||
archive_url = server_path.replace(koji.config.topdir, koji.config.topurl)
|
||||
with util.temp_dir(prefix="buildinstall-download") as tmp_dir:
|
||||
local_path = _download_archive(task_id, fname, archive_url, tmp_dir)
|
||||
_extract_archive(task_id, fname, local_path, destination)
|
||||
|
@ -254,6 +254,7 @@ class TestBuildinstallPhase(PungiTestCase):
|
||||
def test_starts_threads_for_each_cmd_with_lorax_koji_plugin(
|
||||
self, get_volid, poolCls
|
||||
):
|
||||
topurl = "https://example.com/composes/"
|
||||
compose = BuildInstallCompose(
|
||||
self.topdir,
|
||||
{
|
||||
@ -264,6 +265,7 @@ class TestBuildinstallPhase(PungiTestCase):
|
||||
"buildinstall_method": "lorax",
|
||||
"lorax_use_koji_plugin": True,
|
||||
"disc_types": {"dvd": "DVD"},
|
||||
"translate_paths": [(self.topdir, topurl)],
|
||||
},
|
||||
)
|
||||
|
||||
@ -280,9 +282,9 @@ class TestBuildinstallPhase(PungiTestCase):
|
||||
"version": "1",
|
||||
"release": "1",
|
||||
"sources": [
|
||||
self.topdir + "/work/amd64/repo/p1",
|
||||
self.topdir + "/work/amd64/repo/p2",
|
||||
self.topdir + "/work/amd64/comps_repo_Server",
|
||||
topurl + "work/amd64/repo/p1",
|
||||
topurl + "work/amd64/repo/p2",
|
||||
topurl + "work/amd64/comps_repo_Server",
|
||||
],
|
||||
"variant": "Server",
|
||||
"installpkgs": ["bash", "vim"],
|
||||
@ -299,7 +301,6 @@ class TestBuildinstallPhase(PungiTestCase):
|
||||
"rootfs-size": None,
|
||||
"dracut-args": [],
|
||||
"skip_branding": False,
|
||||
"outputdir": self.topdir + "/work/amd64/buildinstall/Server",
|
||||
"squashfs_only": False,
|
||||
"configuration_file": None,
|
||||
},
|
||||
@ -308,9 +309,9 @@ class TestBuildinstallPhase(PungiTestCase):
|
||||
"version": "1",
|
||||
"release": "1",
|
||||
"sources": [
|
||||
self.topdir + "/work/amd64/repo/p1",
|
||||
self.topdir + "/work/amd64/repo/p2",
|
||||
self.topdir + "/work/amd64/comps_repo_Client",
|
||||
topurl + "work/amd64/repo/p1",
|
||||
topurl + "work/amd64/repo/p2",
|
||||
topurl + "work/amd64/comps_repo_Client",
|
||||
],
|
||||
"variant": "Client",
|
||||
"installpkgs": [],
|
||||
@ -327,7 +328,6 @@ class TestBuildinstallPhase(PungiTestCase):
|
||||
"rootfs-size": None,
|
||||
"dracut-args": [],
|
||||
"skip_branding": False,
|
||||
"outputdir": self.topdir + "/work/amd64/buildinstall/Client",
|
||||
"squashfs_only": False,
|
||||
"configuration_file": None,
|
||||
},
|
||||
@ -336,9 +336,9 @@ class TestBuildinstallPhase(PungiTestCase):
|
||||
"version": "1",
|
||||
"release": "1",
|
||||
"sources": [
|
||||
self.topdir + "/work/x86_64/repo/p1",
|
||||
self.topdir + "/work/x86_64/repo/p2",
|
||||
self.topdir + "/work/x86_64/comps_repo_Server",
|
||||
topurl + "work/x86_64/repo/p1",
|
||||
topurl + "work/x86_64/repo/p2",
|
||||
topurl + "work/x86_64/comps_repo_Server",
|
||||
],
|
||||
"variant": "Server",
|
||||
"installpkgs": ["bash", "vim"],
|
||||
@ -355,7 +355,6 @@ class TestBuildinstallPhase(PungiTestCase):
|
||||
"rootfs-size": None,
|
||||
"dracut-args": [],
|
||||
"skip_branding": False,
|
||||
"outputdir": self.topdir + "/work/x86_64/buildinstall/Server",
|
||||
"squashfs_only": False,
|
||||
"configuration_file": None,
|
||||
},
|
||||
@ -1234,9 +1233,9 @@ class BuildinstallThreadTestCase(PungiTestCase):
|
||||
@mock.patch("pungi.wrappers.kojiwrapper.KojiWrapper")
|
||||
@mock.patch("pungi.wrappers.kojiwrapper.get_buildroot_rpms")
|
||||
@mock.patch("pungi.phases.buildinstall.run")
|
||||
@mock.patch("pungi.phases.buildinstall.move_all")
|
||||
@mock.patch("pungi.phases.buildinstall.download_and_extract_archive")
|
||||
def test_buildinstall_thread_with_lorax_using_koji_plugin(
|
||||
self, move_all, run, get_buildroot_rpms, KojiWrapperMock, mock_tweak, mock_link
|
||||
self, download, run, get_buildroot_rpms, KojiWrapperMock, mock_tweak, mock_link
|
||||
):
|
||||
compose = BuildInstallCompose(
|
||||
self.topdir,
|
||||
@ -1282,9 +1281,8 @@ class BuildinstallThreadTestCase(PungiTestCase):
|
||||
self.cmd,
|
||||
channel=None,
|
||||
packages=["lorax"],
|
||||
mounts=[self.topdir],
|
||||
weight=123,
|
||||
chown_uid=os.getuid(),
|
||||
chown_uid=None,
|
||||
)
|
||||
],
|
||||
)
|
||||
@ -1325,13 +1323,14 @@ class BuildinstallThreadTestCase(PungiTestCase):
|
||||
[mock.call(compose, "x86_64", compose.variants["Server"], False)],
|
||||
)
|
||||
self.assertEqual(
|
||||
move_all.call_args_list,
|
||||
download.call_args_list,
|
||||
[
|
||||
mock.call(os.path.join(destdir, "results"), destdir, rm_src_dir=True),
|
||||
mock.call(compose, 1234, "results.tar.gz", destdir),
|
||||
mock.call(
|
||||
os.path.join(destdir, "logs"),
|
||||
compose,
|
||||
1234,
|
||||
"logs.tar.gz",
|
||||
os.path.join(self.topdir, "logs/x86_64/buildinstall-Server-logs"),
|
||||
rm_src_dir=True,
|
||||
),
|
||||
],
|
||||
)
|
||||
|
Loading…
Reference in New Issue
Block a user