945 lines
36 KiB
Python
945 lines
36 KiB
Python
# -*- coding: utf-8 -*-
|
|
|
|
|
|
# This program is free software; you can redistribute it and/or modify
|
|
# it under the terms of the GNU General Public License as published by
|
|
# the Free Software Foundation; version 2 of the License.
|
|
#
|
|
# This program is distributed in the hope that it will be useful,
|
|
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
# GNU Library General Public License for more details.
|
|
#
|
|
# You should have received a copy of the GNU General Public License
|
|
# along with this program; if not, see <https://gnu.org/licenses/>.
|
|
|
|
|
|
import errno
|
|
import os
|
|
import time
|
|
import shutil
|
|
import re
|
|
from six.moves import cPickle as pickle
|
|
from copy import copy
|
|
|
|
from kobo.threads import ThreadPool, WorkerThread
|
|
from kobo.shortcuts import run, force_list
|
|
import kobo.rpmlib
|
|
from productmd.images import Image
|
|
from six.moves import shlex_quote
|
|
|
|
from pungi.arch import get_valid_arches
|
|
from pungi.util import get_volid, get_arch_variant_data
|
|
from pungi.util import get_file_size, get_mtime, failable, makedirs
|
|
from pungi.util import copy_all, translate_path, move_all
|
|
from pungi.wrappers.lorax import LoraxWrapper
|
|
from pungi.wrappers import iso
|
|
from pungi.wrappers.scm import get_file
|
|
from pungi.wrappers.scm import get_file_from_scm
|
|
from pungi.wrappers import kojiwrapper
|
|
from pungi.phases.base import PhaseBase
|
|
from pungi.runroot import Runroot
|
|
|
|
|
|
class BuildinstallPhase(PhaseBase):
|
|
name = "buildinstall"
|
|
|
|
def __init__(self, compose, pkgset_phase=None):
|
|
PhaseBase.__init__(self, compose)
|
|
self.pool = ThreadPool(logger=self.compose._logger)
|
|
# A set of (variant_uid, arch) pairs that completed successfully. This
|
|
# is needed to skip copying files for failed tasks.
|
|
self.pool.finished_tasks = set()
|
|
# A set of (variant_uid, arch) pairs that were reused from previous
|
|
# compose.
|
|
self.pool.reused_tasks = set()
|
|
self.buildinstall_method = self.compose.conf.get("buildinstall_method")
|
|
self.lorax_use_koji_plugin = self.compose.conf.get("lorax_use_koji_plugin")
|
|
self.used_lorax = self.buildinstall_method == "lorax"
|
|
self.pkgset_phase = pkgset_phase
|
|
|
|
self.warned_skipped = False
|
|
|
|
def skip(self):
|
|
if PhaseBase.skip(self):
|
|
return True
|
|
if not self.compose.conf.get("buildinstall_method"):
|
|
if not self.warned_skipped:
|
|
msg = "Not a bootable product. Skipping buildinstall."
|
|
self.compose.log_debug(msg)
|
|
self.warned_skipped = True
|
|
return True
|
|
return False
|
|
|
|
def _get_lorax_cmd(
|
|
self,
|
|
repo_baseurl,
|
|
output_dir,
|
|
variant,
|
|
arch,
|
|
buildarch,
|
|
volid,
|
|
final_output_dir,
|
|
):
|
|
noupgrade = True
|
|
bugurl = None
|
|
nomacboot = True
|
|
add_template = []
|
|
add_arch_template = []
|
|
add_template_var = []
|
|
add_arch_template_var = []
|
|
dracut_args = []
|
|
rootfs_size = None
|
|
skip_branding = False
|
|
squashfs_only = False
|
|
configuration_file = None
|
|
configuration_file_source = None
|
|
version = self.compose.conf.get(
|
|
"treeinfo_version", self.compose.conf["release_version"]
|
|
)
|
|
for data in get_arch_variant_data(
|
|
self.compose.conf, "lorax_options", arch, variant
|
|
):
|
|
if not data.get("noupgrade", True):
|
|
noupgrade = False
|
|
if data.get("bugurl"):
|
|
bugurl = data.get("bugurl")
|
|
if not data.get("nomacboot", True):
|
|
nomacboot = False
|
|
if "rootfs_size" in data:
|
|
rootfs_size = data.get("rootfs_size")
|
|
add_template.extend(data.get("add_template", []))
|
|
add_arch_template.extend(data.get("add_arch_template", []))
|
|
add_template_var.extend(data.get("add_template_var", []))
|
|
add_arch_template_var.extend(data.get("add_arch_template_var", []))
|
|
dracut_args.extend(data.get("dracut_args", []))
|
|
skip_branding = data.get("skip_branding", False)
|
|
configuration_file_source = data.get("configuration_file")
|
|
squashfs_only = data.get("squashfs_only", False)
|
|
if "version" in data:
|
|
version = data["version"]
|
|
output_dir = os.path.join(output_dir, variant.uid)
|
|
output_topdir = output_dir
|
|
|
|
# The paths module will modify the filename (by inserting arch). But we
|
|
# only care about the directory anyway.
|
|
log_dir = _get_log_dir(self.compose, variant, arch)
|
|
# Place the lorax.conf as specified by
|
|
# the configuration_file parameter of lorax_options to the log directory.
|
|
if configuration_file_source:
|
|
configuration_file_destination = os.path.join(log_dir, "lorax.conf")
|
|
# Obtain lorax.conf for the buildInstall phase
|
|
get_file(
|
|
configuration_file_source,
|
|
configuration_file_destination,
|
|
compose=self.compose,
|
|
)
|
|
configuration_file = configuration_file_destination
|
|
|
|
repos = repo_baseurl[:]
|
|
repos.extend(
|
|
get_arch_variant_data(
|
|
self.compose.conf, "lorax_extra_sources", arch, variant
|
|
)
|
|
)
|
|
if self.compose.has_comps:
|
|
comps_repo = self.compose.paths.work.comps_repo(arch, variant)
|
|
if final_output_dir != output_dir:
|
|
comps_repo = translate_path(self.compose, comps_repo)
|
|
repos.append(comps_repo)
|
|
|
|
if self.lorax_use_koji_plugin:
|
|
return {
|
|
"product": self.compose.conf["release_name"],
|
|
"version": version,
|
|
"release": version,
|
|
"sources": force_list(repos),
|
|
"variant": variant.uid,
|
|
"installpkgs": variant.buildinstallpackages,
|
|
"isfinal": self.compose.supported,
|
|
"buildarch": buildarch,
|
|
"volid": volid,
|
|
"nomacboot": nomacboot,
|
|
"bugurl": bugurl,
|
|
"add-template": add_template,
|
|
"add-arch-template": add_arch_template,
|
|
"add-template-var": add_template_var,
|
|
"add-arch-template-var": add_arch_template_var,
|
|
"noupgrade": noupgrade,
|
|
"rootfs-size": rootfs_size,
|
|
"dracut-args": dracut_args,
|
|
"skip_branding": skip_branding,
|
|
"outputdir": output_dir,
|
|
"squashfs_only": squashfs_only,
|
|
"configuration_file": configuration_file,
|
|
}
|
|
else:
|
|
# If the buildinstall_topdir is set, it means Koji is used for
|
|
# buildinstall phase and the filesystem with Koji is read-only.
|
|
# In that case, we have to write logs to buildinstall_topdir and
|
|
# later copy them back to our local log directory.
|
|
if self.compose.conf.get("buildinstall_topdir", None):
|
|
output_dir = os.path.join(output_dir, "results")
|
|
|
|
lorax = LoraxWrapper()
|
|
lorax_cmd = lorax.get_lorax_cmd(
|
|
self.compose.conf["release_name"],
|
|
version,
|
|
version,
|
|
repos,
|
|
output_dir,
|
|
variant=variant.uid,
|
|
buildinstallpackages=variant.buildinstallpackages,
|
|
is_final=self.compose.supported,
|
|
buildarch=buildarch,
|
|
volid=volid,
|
|
nomacboot=nomacboot,
|
|
bugurl=bugurl,
|
|
add_template=add_template,
|
|
add_arch_template=add_arch_template,
|
|
add_template_var=add_template_var,
|
|
add_arch_template_var=add_arch_template_var,
|
|
noupgrade=noupgrade,
|
|
rootfs_size=rootfs_size,
|
|
log_dir=log_dir,
|
|
dracut_args=dracut_args,
|
|
skip_branding=skip_branding,
|
|
squashfs_only=squashfs_only,
|
|
configuration_file=configuration_file,
|
|
)
|
|
return "rm -rf %s && %s" % (
|
|
shlex_quote(output_topdir),
|
|
" ".join([shlex_quote(x) for x in lorax_cmd]),
|
|
)
|
|
|
|
def get_repos(self, arch):
|
|
repos = []
|
|
for pkgset in self.pkgset_phase.package_sets:
|
|
repos.append(pkgset.paths[arch])
|
|
return repos
|
|
|
|
def run(self):
|
|
lorax = LoraxWrapper()
|
|
product = self.compose.conf["release_name"]
|
|
version = self.compose.conf["release_version"]
|
|
release = self.compose.conf["release_version"]
|
|
disc_type = self.compose.conf["disc_types"].get("dvd", "dvd")
|
|
|
|
# Prepare kickstart file for final images.
|
|
self.pool.kickstart_file = get_kickstart_file(self.compose)
|
|
|
|
for arch in self.compose.get_arches():
|
|
commands = []
|
|
|
|
output_dir = self.compose.paths.work.buildinstall_dir(
|
|
arch, allow_topdir_override=True
|
|
)
|
|
final_output_dir = self.compose.paths.work.buildinstall_dir(
|
|
arch, allow_topdir_override=False
|
|
)
|
|
makedirs(final_output_dir)
|
|
repo_baseurls = self.get_repos(arch)
|
|
if final_output_dir != output_dir:
|
|
repo_baseurls = [translate_path(self.compose, r) for r in repo_baseurls]
|
|
|
|
if self.buildinstall_method == "lorax":
|
|
buildarch = get_valid_arches(arch)[0]
|
|
for variant in self.compose.get_variants(arch=arch, types=["variant"]):
|
|
if variant.is_empty:
|
|
continue
|
|
|
|
skip = get_arch_variant_data(
|
|
self.compose.conf, "buildinstall_skip", arch, variant
|
|
)
|
|
if skip == [True]:
|
|
self.compose.log_info(
|
|
"Skipping buildinstall for %s.%s due to config option"
|
|
% (variant, arch)
|
|
)
|
|
continue
|
|
|
|
volid = get_volid(
|
|
self.compose, arch, variant=variant, disc_type=disc_type
|
|
)
|
|
commands.append(
|
|
(
|
|
variant,
|
|
self._get_lorax_cmd(
|
|
repo_baseurls,
|
|
output_dir,
|
|
variant,
|
|
arch,
|
|
buildarch,
|
|
volid,
|
|
final_output_dir,
|
|
),
|
|
)
|
|
)
|
|
elif self.buildinstall_method == "buildinstall":
|
|
volid = get_volid(self.compose, arch, disc_type=disc_type)
|
|
commands.append(
|
|
(
|
|
None,
|
|
lorax.get_buildinstall_cmd(
|
|
product,
|
|
version,
|
|
release,
|
|
repo_baseurls,
|
|
output_dir,
|
|
is_final=self.compose.supported,
|
|
buildarch=arch,
|
|
volid=volid,
|
|
),
|
|
)
|
|
)
|
|
else:
|
|
raise ValueError(
|
|
"Unsupported buildinstall method: %s" % self.buildinstall_method
|
|
)
|
|
|
|
for (variant, cmd) in commands:
|
|
self.pool.add(BuildinstallThread(self.pool))
|
|
self.pool.queue_put(
|
|
(self.compose, arch, variant, cmd, self.pkgset_phase)
|
|
)
|
|
|
|
self.pool.start()
|
|
|
|
def succeeded(self, variant, arch):
|
|
# If the phase is skipped, we can treat it as successful. Either there
|
|
# will be no output, or it's a debug run of compose where anything can
|
|
# happen.
|
|
return (
|
|
super(BuildinstallPhase, self).skip()
|
|
or (variant.uid if self.used_lorax else None, arch)
|
|
in self.pool.finished_tasks
|
|
)
|
|
|
|
def reused(self, variant, arch):
|
|
"""
|
|
Check if buildinstall phase reused previous results for given variant
|
|
and arch. If the phase is skipped, the results will be considered
|
|
reused as well.
|
|
"""
|
|
return (
|
|
super(BuildinstallPhase, self).skip()
|
|
or (variant.uid if self.used_lorax else None, arch)
|
|
in self.pool.reused_tasks
|
|
)
|
|
|
|
|
|
def get_kickstart_file(compose):
|
|
scm_dict = compose.conf.get("buildinstall_kickstart")
|
|
if not scm_dict:
|
|
compose.log_debug("Path to ks.cfg (buildinstall_kickstart) not specified.")
|
|
return
|
|
|
|
msg = "Getting ks.cfg"
|
|
kickstart_path = os.path.join(compose.paths.work.topdir(arch="global"), "ks.cfg")
|
|
if os.path.exists(kickstart_path):
|
|
compose.log_warning("[SKIP ] %s" % msg)
|
|
return kickstart_path
|
|
|
|
compose.log_info("[BEGIN] %s" % msg)
|
|
if isinstance(scm_dict, dict):
|
|
kickstart_name = os.path.basename(scm_dict["file"])
|
|
if scm_dict["scm"] == "file":
|
|
scm_dict["file"] = os.path.join(compose.config_dir, scm_dict["file"])
|
|
else:
|
|
kickstart_name = os.path.basename(scm_dict)
|
|
scm_dict = os.path.join(compose.config_dir, scm_dict)
|
|
|
|
tmp_dir = compose.mkdtemp(prefix="buildinstall_kickstart_")
|
|
get_file_from_scm(scm_dict, tmp_dir, compose=compose)
|
|
src = os.path.join(tmp_dir, kickstart_name)
|
|
shutil.copy2(src, kickstart_path)
|
|
compose.log_info("[DONE ] %s" % msg)
|
|
return kickstart_path
|
|
|
|
|
|
BOOT_CONFIGS = [
|
|
"isolinux/isolinux.cfg",
|
|
"etc/yaboot.conf",
|
|
"ppc/ppc64/yaboot.conf",
|
|
"EFI/BOOT/BOOTX64.conf",
|
|
"EFI/BOOT/grub.cfg",
|
|
]
|
|
|
|
|
|
def tweak_configs(path, volid, ks_file, configs=BOOT_CONFIGS, logger=None):
|
|
volid_escaped = volid.replace(" ", r"\x20").replace("\\", "\\\\")
|
|
volid_escaped_2 = volid_escaped.replace("\\", "\\\\")
|
|
found_configs = []
|
|
for config in configs:
|
|
config_path = os.path.join(path, config)
|
|
if not os.path.exists(config_path):
|
|
continue
|
|
found_configs.append(config)
|
|
|
|
with open(config_path, "r") as f:
|
|
data = original_data = f.read()
|
|
os.unlink(config_path) # break hadlink by removing file writing a new one
|
|
|
|
# double-escape volid in yaboot.conf
|
|
new_volid = volid_escaped_2 if "yaboot" in config else volid_escaped
|
|
|
|
ks = (" inst.ks=hd:LABEL=%s:/ks.cfg" % new_volid) if ks_file else ""
|
|
|
|
# pre-f18
|
|
data = re.sub(r":CDLABEL=[^ \n]*", r":CDLABEL=%s%s" % (new_volid, ks), data)
|
|
# f18+
|
|
data = re.sub(r":LABEL=[^ \n]*", r":LABEL=%s%s" % (new_volid, ks), data)
|
|
data = re.sub(r"(search .* -l) '[^'\n]*'", r"\1 '%s'" % volid, data)
|
|
|
|
with open(config_path, "w") as f:
|
|
f.write(data)
|
|
|
|
if logger and data != original_data:
|
|
logger.info("Boot config %s changed" % config_path)
|
|
|
|
return found_configs
|
|
|
|
|
|
# HACK: this is a hack!
|
|
# * it's quite trivial to replace volids
|
|
# * it's not easy to replace menu titles
|
|
# * we probably need to get this into lorax
|
|
def tweak_buildinstall(
|
|
compose, src, dst, arch, variant, label, volid, kickstart_file=None
|
|
):
|
|
tmp_dir = compose.mkdtemp(prefix="tweak_buildinstall_")
|
|
|
|
# verify src
|
|
if not os.path.isdir(src):
|
|
raise OSError(errno.ENOENT, "Directory does not exist: %s" % src)
|
|
|
|
# create dst
|
|
try:
|
|
os.makedirs(dst)
|
|
except OSError as ex:
|
|
if ex.errno != errno.EEXIST:
|
|
raise
|
|
|
|
# copy src to temp
|
|
# TODO: place temp on the same device as buildinstall dir so we can hardlink
|
|
cmd = "cp -dRv --preserve=mode,links,timestamps --remove-destination %s/* %s/" % (
|
|
shlex_quote(src),
|
|
shlex_quote(tmp_dir),
|
|
)
|
|
run(cmd)
|
|
|
|
found_configs = tweak_configs(
|
|
tmp_dir, volid, kickstart_file, logger=compose._logger
|
|
)
|
|
if kickstart_file and found_configs:
|
|
shutil.copy2(kickstart_file, os.path.join(dst, "ks.cfg"))
|
|
|
|
images = [
|
|
os.path.join(tmp_dir, "images", "efiboot.img"),
|
|
]
|
|
for image in images:
|
|
if not os.path.isfile(image):
|
|
continue
|
|
|
|
with iso.mount(
|
|
image,
|
|
logger=compose._logger,
|
|
use_guestmount=compose.conf.get("buildinstall_use_guestmount"),
|
|
) as mount_tmp_dir:
|
|
for config in BOOT_CONFIGS:
|
|
config_path = os.path.join(tmp_dir, config)
|
|
config_in_image = os.path.join(mount_tmp_dir, config)
|
|
|
|
if os.path.isfile(config_in_image):
|
|
cmd = [
|
|
"cp",
|
|
"-v",
|
|
"--remove-destination",
|
|
config_path,
|
|
config_in_image,
|
|
]
|
|
run(cmd)
|
|
|
|
# HACK: make buildinstall files world readable
|
|
run("chmod -R a+rX %s" % shlex_quote(tmp_dir))
|
|
|
|
# copy temp to dst
|
|
cmd = "cp -dRv --preserve=mode,links,timestamps --remove-destination %s/* %s/" % (
|
|
shlex_quote(tmp_dir),
|
|
shlex_quote(dst),
|
|
)
|
|
run(cmd)
|
|
|
|
shutil.rmtree(tmp_dir)
|
|
|
|
|
|
def link_boot_iso(compose, arch, variant, can_fail):
|
|
if arch == "src":
|
|
return
|
|
|
|
disc_type = compose.conf["disc_types"].get("boot", "boot")
|
|
|
|
symlink_isos_to = compose.conf.get("symlink_isos_to")
|
|
os_tree = compose.paths.compose.os_tree(arch, variant)
|
|
# TODO: find in treeinfo?
|
|
boot_iso_path = os.path.join(os_tree, "images", "boot.iso")
|
|
if not os.path.isfile(boot_iso_path):
|
|
return
|
|
|
|
msg = "Linking boot.iso (arch: %s, variant: %s)" % (arch, variant)
|
|
filename = compose.get_image_name(
|
|
arch, variant, disc_type=disc_type, disc_num=None, suffix=".iso"
|
|
)
|
|
new_boot_iso_path = compose.paths.compose.iso_path(
|
|
arch, variant, filename, symlink_to=symlink_isos_to
|
|
)
|
|
new_boot_iso_relative_path = compose.paths.compose.iso_path(
|
|
arch, variant, filename, relative=True
|
|
)
|
|
if os.path.exists(new_boot_iso_path):
|
|
# TODO: log
|
|
compose.log_warning("[SKIP ] %s" % msg)
|
|
return
|
|
|
|
compose.log_info("[BEGIN] %s" % msg)
|
|
# Try to hardlink, and copy if that fails
|
|
try:
|
|
os.link(boot_iso_path, new_boot_iso_path)
|
|
except OSError:
|
|
shutil.copy2(boot_iso_path, new_boot_iso_path)
|
|
|
|
implant_md5 = iso.get_implanted_md5(new_boot_iso_path)
|
|
iso_name = os.path.basename(new_boot_iso_path)
|
|
iso_dir = os.path.dirname(new_boot_iso_path)
|
|
|
|
# create iso manifest
|
|
run(iso.get_manifest_cmd(iso_name), workdir=iso_dir)
|
|
|
|
img = Image(compose.im)
|
|
img.path = new_boot_iso_relative_path
|
|
img.mtime = get_mtime(new_boot_iso_path)
|
|
img.size = get_file_size(new_boot_iso_path)
|
|
img.arch = arch
|
|
img.type = "boot"
|
|
img.format = "iso"
|
|
img.disc_number = 1
|
|
img.disc_count = 1
|
|
img.bootable = True
|
|
img.subvariant = variant.uid
|
|
img.implant_md5 = implant_md5
|
|
setattr(img, "can_fail", can_fail)
|
|
setattr(img, "deliverable", "buildinstall")
|
|
try:
|
|
img.volume_id = iso.get_volume_id(new_boot_iso_path)
|
|
except RuntimeError:
|
|
pass
|
|
compose.im.add(variant.uid, arch, img)
|
|
compose.log_info("[DONE ] %s" % msg)
|
|
|
|
|
|
class BuildinstallThread(WorkerThread):
|
|
def process(self, item, num):
|
|
# The variant is None unless lorax is used as buildinstall method.
|
|
compose, arch, variant, cmd, pkgset_phase = item
|
|
can_fail = compose.can_fail(variant, arch, "buildinstall")
|
|
with failable(compose, can_fail, variant, arch, "buildinstall"):
|
|
try:
|
|
self.worker(compose, arch, variant, cmd, pkgset_phase, num)
|
|
except RuntimeError:
|
|
self._print_depsolve_error(compose, arch, variant)
|
|
raise
|
|
|
|
def _print_depsolve_error(self, compose, arch, variant):
|
|
try:
|
|
log_file = os.path.join(_get_log_dir(compose, variant, arch), "pylorax.log")
|
|
with open(log_file) as f:
|
|
matched = False
|
|
for line in f:
|
|
if re.match("Dependency check failed", line):
|
|
matched = True
|
|
if matched:
|
|
compose.log_error(line.rstrip())
|
|
except Exception:
|
|
pass
|
|
|
|
def _generate_buildinstall_metadata(
|
|
self, compose, arch, variant, cmd, buildroot_rpms, pkgset_phase
|
|
):
|
|
"""
|
|
Generate buildinstall.metadata dict.
|
|
|
|
:param Compose compose: Current compose.
|
|
:param str arch: Current architecture.
|
|
:param Variant variant: Compose variant.
|
|
:param list cmd: List of command line arguments passed to buildinstall task.
|
|
:param list buildroot_rpms: List of NVRAs of all RPMs installed in the
|
|
buildinstall task's buildroot.
|
|
:param PkgsetPhase pkgset_phase: Package set phase instance.
|
|
:return: The buildinstall.metadata dict.
|
|
"""
|
|
# Load the list of packages installed in the boot.iso.
|
|
# The list of installed packages is logged by Lorax in the "pkglists"
|
|
# directory. There is one file for each installed RPM and the name
|
|
# of the file is the name of the RPM.
|
|
# We need to resolve the name of each RPM back to its NVRA.
|
|
installed_rpms = []
|
|
log_fname = "buildinstall-%s-logs/dummy" % variant.uid
|
|
log_dir = os.path.dirname(compose.paths.log.log_file(arch, log_fname, False))
|
|
pkglists_dir = os.path.join(log_dir, "pkglists")
|
|
if os.path.exists(pkglists_dir):
|
|
for pkg_name in os.listdir(pkglists_dir):
|
|
for pkgset in pkgset_phase.package_sets:
|
|
global_pkgset = pkgset["global"]
|
|
# We actually do not care from which package_set the RPM
|
|
# came from or if there are multiple versions/release of
|
|
# the single RPM in more packages sets. We simply include
|
|
# all RPMs with this name in the metadata.
|
|
# Later when deciding if the buildinstall phase results
|
|
# can be reused, we check that all the RPMs with this name
|
|
# are still the same in old/new compose.
|
|
for rpm_path, rpm_obj in global_pkgset.file_cache.items():
|
|
if rpm_obj.name == pkg_name:
|
|
installed_rpms.append(rpm_path)
|
|
|
|
# Store the metadata in `buildinstall.metadata`.
|
|
metadata = {
|
|
"cmd": cmd,
|
|
"buildroot_rpms": sorted(buildroot_rpms),
|
|
"installed_rpms": sorted(installed_rpms),
|
|
}
|
|
return metadata
|
|
|
|
def _write_buildinstall_metadata(
|
|
self, compose, arch, variant, cmd, buildroot_rpms, pkgset_phase
|
|
):
|
|
"""
|
|
Write buildinstall.metadata file containing all the information about
|
|
buildinstall phase input and environment.
|
|
|
|
This file is later used to decide whether old buildinstall results can
|
|
be reused instead of generating them again.
|
|
|
|
:param Compose compose: Current compose.
|
|
:param str arch: Current architecture.
|
|
:param Variant variant: Compose variant.
|
|
:param list cmd: List of command line arguments passed to buildinstall task.
|
|
:param list buildroot_rpms: List of NVRAs of all RPMs installed in the
|
|
buildinstall task's buildroot.
|
|
:param PkgsetPhase pkgset_phase: Package set phase instance.
|
|
"""
|
|
# Generate the list of `*-RPMs` log file.
|
|
log_filename = ("buildinstall-%s" % variant.uid) if variant else "buildinstall"
|
|
log_file = compose.paths.log.log_file(arch, log_filename + "-RPMs")
|
|
with open(log_file, "w") as f:
|
|
f.write("\n".join(buildroot_rpms))
|
|
|
|
# Write buildinstall.metadata only if particular variant is defined.
|
|
# The `variant` is `None` only if old "buildinstall" method is used.
|
|
if not variant:
|
|
return
|
|
|
|
metadata = self._generate_buildinstall_metadata(
|
|
compose, arch, variant, cmd, buildroot_rpms, pkgset_phase
|
|
)
|
|
|
|
log_fname = "buildinstall-%s-logs/dummy" % variant.uid
|
|
log_dir = os.path.dirname(compose.paths.log.log_file(arch, log_fname))
|
|
metadata_path = os.path.join(log_dir, "buildinstall.metadata")
|
|
with open(metadata_path, "wb") as f:
|
|
pickle.dump(metadata, f, protocol=pickle.HIGHEST_PROTOCOL)
|
|
|
|
def _load_old_buildinstall_metadata(self, compose, arch, variant):
|
|
"""
|
|
Helper method to load "buildinstall.metadata" from old compose.
|
|
|
|
:param Compose compose: Current compose.
|
|
:param str arch: Current architecture.
|
|
:param Variant variant: Compose variant.
|
|
"""
|
|
if not variant:
|
|
return None
|
|
|
|
log_fname = "buildinstall-%s-logs/dummy" % variant.uid
|
|
metadata = os.path.join(
|
|
os.path.dirname(compose.paths.log.log_file(arch, log_fname)),
|
|
"buildinstall.metadata",
|
|
)
|
|
old_metadata = compose.paths.old_compose_path(metadata)
|
|
if not old_metadata:
|
|
return None
|
|
|
|
compose.log_info("Loading old BUILDINSTALL phase metadata: %s", old_metadata)
|
|
with open(old_metadata, "rb") as f:
|
|
old_result = pickle.load(f)
|
|
return old_result
|
|
|
|
def _reuse_old_buildinstall_result(self, compose, arch, variant, cmd, pkgset_phase):
|
|
"""
|
|
Try to reuse old buildinstall results.
|
|
|
|
:param Compose compose: Current compose.
|
|
:param str arch: Current architecture.
|
|
:param Variant variant: Compose variant.
|
|
:param list cmd: List of command line arguments passed to buildinstall task.
|
|
:param list buildroot_rpms: List of NVRAs of all RPMs installed in the
|
|
buildinstall task's buildroot.
|
|
:param PkgsetPhase pkgset_phase: Package set phase instance.
|
|
:return: True if old buildinstall phase results have been reused.
|
|
"""
|
|
log_msg = "Cannot reuse old BUILDINSTALL phase results - %s"
|
|
|
|
if not compose.conf["buildinstall_allow_reuse"]:
|
|
compose.log_info(log_msg % "reuse of old buildinstall results is disabled.")
|
|
return
|
|
|
|
# Load the old buildinstall.metadata.
|
|
old_metadata = self._load_old_buildinstall_metadata(compose, arch, variant)
|
|
if old_metadata is None:
|
|
compose.log_info(log_msg % "no old BUILDINSTALL metadata.")
|
|
return
|
|
|
|
# For now try to reuse only if pungi_buildinstall plugin is used.
|
|
# This is the easiest approach, because we later need to filter out
|
|
# some parts of `cmd` and for pungi_buildinstall, the `cmd` is a dict
|
|
# which makes this easy.
|
|
if not isinstance(old_metadata["cmd"], dict) or not isinstance(cmd, dict):
|
|
compose.log_info(log_msg % "pungi_buildinstall plugin is not used.")
|
|
return
|
|
|
|
# Filter out "outputdir" and "sources" because they change every time.
|
|
# The "sources" are not important, because we check the buildinstall
|
|
# input on RPM level.
|
|
cmd_copy = copy(cmd)
|
|
for key in ["outputdir", "sources"]:
|
|
del cmd_copy[key]
|
|
del old_metadata["cmd"][key]
|
|
|
|
# Do not reuse if command line arguments are not the same.
|
|
if old_metadata["cmd"] != cmd_copy:
|
|
compose.log_info(log_msg % "lorax command line arguments differ.")
|
|
return
|
|
|
|
# Check that the RPMs installed in the old boot.iso exists in the very
|
|
# same versions/releases in this compose.
|
|
for rpm_path in old_metadata["installed_rpms"]:
|
|
found = False
|
|
for pkgset in pkgset_phase.package_sets:
|
|
global_pkgset = pkgset["global"]
|
|
if rpm_path in global_pkgset.file_cache:
|
|
found = True
|
|
break
|
|
if not found:
|
|
compose.log_info(
|
|
log_msg % "RPM %s does not exist in new compose." % rpm_path
|
|
)
|
|
return
|
|
|
|
# Ask Koji for all the RPMs in the `runroot_tag` and check that
|
|
# those installed in the old buildinstall buildroot are still in the
|
|
# very same versions/releases.
|
|
koji_wrapper = kojiwrapper.KojiWrapper(compose)
|
|
rpms = koji_wrapper.koji_proxy.listTaggedRPMS(
|
|
compose.conf.get("runroot_tag"), inherit=True, latest=True
|
|
)[0]
|
|
rpm_nvras = set()
|
|
for rpm in rpms:
|
|
rpm_nvras.add(kobo.rpmlib.make_nvra(rpm, add_rpm=False, force_epoch=False))
|
|
for old_nvra in old_metadata["buildroot_rpms"]:
|
|
if old_nvra not in rpm_nvras:
|
|
compose.log_info(
|
|
log_msg % "RPM %s does not exist in new buildroot." % old_nvra
|
|
)
|
|
return
|
|
|
|
# We can reuse the old buildinstall results!
|
|
compose.log_info("Reusing old BUILDINSTALL phase output")
|
|
|
|
# Copy old buildinstall output to this this compose.
|
|
final_output_dir = compose.paths.work.buildinstall_dir(arch, variant=variant)
|
|
old_final_output_dir = compose.paths.old_compose_path(final_output_dir)
|
|
copy_all(old_final_output_dir, final_output_dir)
|
|
|
|
# Copy old buildinstall logs to this compose.
|
|
log_fname = "buildinstall-%s-logs/dummy" % variant.uid
|
|
final_log_dir = os.path.dirname(compose.paths.log.log_file(arch, log_fname))
|
|
old_final_log_dir = compose.paths.old_compose_path(final_log_dir)
|
|
if not os.path.exists(final_log_dir):
|
|
makedirs(final_log_dir)
|
|
copy_all(old_final_log_dir, final_log_dir)
|
|
|
|
# Write the buildinstall metadata so next compose can reuse this compose.
|
|
self._write_buildinstall_metadata(
|
|
compose, arch, variant, cmd, old_metadata["buildroot_rpms"], pkgset_phase
|
|
)
|
|
|
|
return True
|
|
|
|
def worker(self, compose, arch, variant, cmd, pkgset_phase, num):
|
|
buildinstall_method = compose.conf["buildinstall_method"]
|
|
lorax_use_koji_plugin = compose.conf["lorax_use_koji_plugin"]
|
|
log_filename = ("buildinstall-%s" % variant.uid) if variant else "buildinstall"
|
|
log_file = compose.paths.log.log_file(arch, log_filename)
|
|
|
|
msg = "Running buildinstall for arch %s, variant %s" % (arch, variant)
|
|
|
|
output_dir = compose.paths.work.buildinstall_dir(
|
|
arch, allow_topdir_override=True, variant=variant
|
|
)
|
|
final_output_dir = compose.paths.work.buildinstall_dir(arch, variant=variant)
|
|
|
|
if (
|
|
os.path.isdir(output_dir)
|
|
and os.listdir(output_dir)
|
|
or os.path.isdir(final_output_dir)
|
|
and os.listdir(final_output_dir)
|
|
):
|
|
# output dir is *not* empty -> SKIP
|
|
self.pool.log_warning(
|
|
"[SKIP ] Buildinstall for arch %s, variant %s" % (arch, variant)
|
|
)
|
|
return
|
|
|
|
self.pool.log_info("[BEGIN] %s" % msg)
|
|
|
|
# Get list of packages which are needed in runroot.
|
|
packages = []
|
|
chown_paths = [output_dir]
|
|
if buildinstall_method == "lorax":
|
|
packages += ["lorax"]
|
|
chown_paths.append(_get_log_dir(compose, variant, arch))
|
|
elif buildinstall_method == "buildinstall":
|
|
packages += ["anaconda"]
|
|
|
|
if self._reuse_old_buildinstall_result(
|
|
compose, arch, variant, cmd, pkgset_phase
|
|
):
|
|
self.copy_files(compose, variant, arch)
|
|
self.pool.finished_tasks.add((variant.uid if variant else None, arch))
|
|
self.pool.reused_tasks.add((variant.uid if variant else None, arch))
|
|
self.pool.log_info("[DONE ] %s" % msg)
|
|
return
|
|
|
|
# This should avoid a possible race condition with multiple processes
|
|
# trying to get a kerberos ticket at the same time.
|
|
# Kerberos authentication failed:
|
|
# Permission denied in replay cache code (-1765328215)
|
|
time.sleep(num * 3)
|
|
|
|
# Start the runroot task.
|
|
runroot = Runroot(compose, phase="buildinstall")
|
|
if buildinstall_method == "lorax" and lorax_use_koji_plugin:
|
|
runroot.run_pungi_buildinstall(
|
|
cmd,
|
|
log_file=log_file,
|
|
arch=arch,
|
|
packages=packages,
|
|
mounts=[compose.topdir],
|
|
weight=compose.conf["runroot_weights"].get("buildinstall"),
|
|
)
|
|
else:
|
|
try:
|
|
lorax_log_dir = _get_log_dir(compose, variant, arch)
|
|
except Exception:
|
|
lorax_log_dir = None
|
|
runroot.run(
|
|
cmd,
|
|
log_file=log_file,
|
|
arch=arch,
|
|
packages=packages,
|
|
mounts=[compose.topdir],
|
|
weight=compose.conf["runroot_weights"].get("buildinstall"),
|
|
chown_paths=chown_paths,
|
|
log_dir=lorax_log_dir,
|
|
)
|
|
|
|
if final_output_dir != output_dir:
|
|
if not os.path.exists(final_output_dir):
|
|
makedirs(final_output_dir)
|
|
results_dir = os.path.join(output_dir, "results")
|
|
copy_all(results_dir, final_output_dir)
|
|
|
|
# Get the log_dir into which we should copy the resulting log files.
|
|
log_fname = "buildinstall-%s-logs/dummy" % variant.uid
|
|
final_log_dir = os.path.dirname(compose.paths.log.log_file(arch, log_fname))
|
|
if not os.path.exists(final_log_dir):
|
|
makedirs(final_log_dir)
|
|
log_dir = os.path.join(output_dir, "logs")
|
|
copy_all(log_dir, final_log_dir)
|
|
elif lorax_use_koji_plugin:
|
|
# If Koji pungi-buildinstall is used, then the buildinstall results are
|
|
# not stored directly in `output_dir` dir, but in "results" and "logs"
|
|
# subdirectories. We need to move them to final_output_dir.
|
|
results_dir = os.path.join(output_dir, "results")
|
|
move_all(results_dir, final_output_dir, rm_src_dir=True)
|
|
|
|
# Get the log_dir into which we should copy the resulting log files.
|
|
log_fname = "buildinstall-%s-logs/dummy" % variant.uid
|
|
final_log_dir = os.path.dirname(compose.paths.log.log_file(arch, log_fname))
|
|
if not os.path.exists(final_log_dir):
|
|
makedirs(final_log_dir)
|
|
log_dir = os.path.join(output_dir, "logs")
|
|
move_all(log_dir, final_log_dir, rm_src_dir=True)
|
|
|
|
rpms = runroot.get_buildroot_rpms()
|
|
self._write_buildinstall_metadata(
|
|
compose, arch, variant, cmd, rpms, pkgset_phase
|
|
)
|
|
|
|
self.copy_files(compose, variant, arch)
|
|
|
|
self.pool.finished_tasks.add((variant.uid if variant else None, arch))
|
|
|
|
self.pool.log_info("[DONE ] %s" % msg)
|
|
|
|
def copy_files(self, compose, variant, arch):
|
|
disc_type = compose.conf["disc_types"].get("dvd", "dvd")
|
|
|
|
buildinstall_dir = compose.paths.work.buildinstall_dir(arch)
|
|
|
|
# Lorax runs per-variant, so we need to tweak the source path
|
|
# to include variant.
|
|
if variant:
|
|
buildinstall_dir = os.path.join(buildinstall_dir, variant.uid)
|
|
|
|
# Find all relevant variants if lorax is not used.
|
|
variants = (
|
|
[variant]
|
|
if variant
|
|
else compose.get_variants(arch=arch, types=["self", "variant"])
|
|
)
|
|
for var in variants:
|
|
os_tree = compose.paths.compose.os_tree(arch, var)
|
|
# TODO: label is not used
|
|
label = ""
|
|
volid = get_volid(compose, arch, var, disc_type=disc_type)
|
|
can_fail = compose.can_fail(var, arch, "buildinstall")
|
|
tweak_buildinstall(
|
|
compose,
|
|
buildinstall_dir,
|
|
os_tree,
|
|
arch,
|
|
var.uid,
|
|
label,
|
|
volid,
|
|
self.pool.kickstart_file,
|
|
)
|
|
link_boot_iso(compose, arch, var, can_fail)
|
|
|
|
|
|
def _get_log_dir(compose, variant, arch):
|
|
"""Find directory where to store lorax logs in. If it's inside the compose,
|
|
create the directory.
|
|
"""
|
|
if compose.conf.get("buildinstall_topdir"):
|
|
log_dir = compose.paths.work.buildinstall_dir(
|
|
arch, allow_topdir_override=True, variant=variant
|
|
)
|
|
return os.path.join(log_dir, "logs")
|
|
|
|
# The paths module will modify the filename (by inserting arch). But we
|
|
# only care about the directory anyway.
|
|
log_filename = "buildinstall-%s-logs/dummy" % variant.uid
|
|
log_dir = os.path.dirname(compose.paths.log.log_file(arch, log_filename))
|
|
makedirs(log_dir)
|
|
return log_dir
|