pungi/pungi/phases/buildinstall.py
Lubomír Sedlář e1d7544c2b createiso: Update possibly changed file on DVD
There's no good way of detecting if buildinstall phase tweaked boot
configuration (and efiboot.img). We should update those files in the DVD
just to be sure.

The .discinfo file is always different and needs to be updated.

Relates: https://pagure.io/pungi/issue/1647
JIRA: RHELCMP-10811
Signed-off-by: Lubomír Sedlář <lsedlar@redhat.com>
2023-03-27 12:40:39 +00:00

967 lines
37 KiB
Python

# -*- coding: utf-8 -*-
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; version 2 of the License.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Library General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, see <https://gnu.org/licenses/>.
import errno
import os
import time
import shutil
import re
from six.moves import cPickle as pickle
from copy import copy
from kobo.threads import ThreadPool, WorkerThread
from kobo.shortcuts import run, force_list
import kobo.rpmlib
from productmd.images import Image
from six.moves import shlex_quote
from pungi.arch import get_valid_arches
from pungi.util import get_volid, get_arch_variant_data
from pungi.util import get_file_size, get_mtime, failable, makedirs
from pungi.util import copy_all, translate_path, move_all
from pungi.wrappers.lorax import LoraxWrapper
from pungi.wrappers import iso
from pungi.wrappers.scm import get_file
from pungi.wrappers.scm import get_file_from_scm
from pungi.wrappers import kojiwrapper
from pungi.phases.base import PhaseBase
from pungi.runroot import Runroot
class BuildinstallPhase(PhaseBase):
name = "buildinstall"
def __init__(self, compose, pkgset_phase=None):
PhaseBase.__init__(self, compose)
self.pool = ThreadPool(logger=self.compose._logger)
# A set of (variant_uid, arch) pairs that completed successfully. This
# is needed to skip copying files for failed tasks.
self.pool.finished_tasks = set()
# A set of (variant_uid, arch) pairs that were reused from previous
# compose.
self.pool.reused_tasks = set()
self.buildinstall_method = self.compose.conf.get("buildinstall_method")
self.lorax_use_koji_plugin = self.compose.conf.get("lorax_use_koji_plugin")
self.used_lorax = self.buildinstall_method == "lorax"
self.pkgset_phase = pkgset_phase
self.warned_skipped = False
def skip(self):
if PhaseBase.skip(self):
return True
if not self.compose.conf.get("buildinstall_method"):
if not self.warned_skipped:
msg = "Not a bootable product. Skipping buildinstall."
self.compose.log_debug(msg)
self.warned_skipped = True
return True
return False
def _get_lorax_cmd(
self,
repo_baseurl,
output_dir,
variant,
arch,
buildarch,
volid,
final_output_dir,
):
noupgrade = True
bugurl = None
nomacboot = True
add_template = []
add_arch_template = []
add_template_var = []
add_arch_template_var = []
dracut_args = []
rootfs_size = None
skip_branding = False
squashfs_only = False
configuration_file = None
configuration_file_source = None
version = self.compose.conf.get(
"treeinfo_version", self.compose.conf["release_version"]
)
for data in get_arch_variant_data(
self.compose.conf, "lorax_options", arch, variant
):
if not data.get("noupgrade", True):
noupgrade = False
if data.get("bugurl"):
bugurl = data.get("bugurl")
if not data.get("nomacboot", True):
nomacboot = False
if "rootfs_size" in data:
rootfs_size = data.get("rootfs_size")
add_template.extend(data.get("add_template", []))
add_arch_template.extend(data.get("add_arch_template", []))
add_template_var.extend(data.get("add_template_var", []))
add_arch_template_var.extend(data.get("add_arch_template_var", []))
dracut_args.extend(data.get("dracut_args", []))
skip_branding = data.get("skip_branding", False)
configuration_file_source = data.get("configuration_file")
squashfs_only = data.get("squashfs_only", False)
if "version" in data:
version = data["version"]
output_dir = os.path.join(output_dir, variant.uid)
output_topdir = output_dir
# The paths module will modify the filename (by inserting arch). But we
# only care about the directory anyway.
log_dir = _get_log_dir(self.compose, variant, arch)
# Place the lorax.conf as specified by
# the configuration_file parameter of lorax_options to the log directory.
if configuration_file_source:
configuration_file_destination = os.path.join(log_dir, "lorax.conf")
# Obtain lorax.conf for the buildInstall phase
get_file(
configuration_file_source,
configuration_file_destination,
compose=self.compose,
)
configuration_file = configuration_file_destination
repos = repo_baseurl[:]
repos.extend(
get_arch_variant_data(
self.compose.conf, "lorax_extra_sources", arch, variant
)
)
if self.compose.has_comps:
comps_repo = self.compose.paths.work.comps_repo(arch, variant)
if final_output_dir != output_dir:
comps_repo = translate_path(self.compose, comps_repo)
repos.append(comps_repo)
if self.lorax_use_koji_plugin:
return {
"product": self.compose.conf["release_name"],
"version": version,
"release": version,
"sources": force_list(repos),
"variant": variant.uid,
"installpkgs": variant.buildinstallpackages,
"isfinal": self.compose.supported,
"buildarch": buildarch,
"volid": volid,
"nomacboot": nomacboot,
"bugurl": bugurl,
"add-template": add_template,
"add-arch-template": add_arch_template,
"add-template-var": add_template_var,
"add-arch-template-var": add_arch_template_var,
"noupgrade": noupgrade,
"rootfs-size": rootfs_size,
"dracut-args": dracut_args,
"skip_branding": skip_branding,
"outputdir": output_dir,
"squashfs_only": squashfs_only,
"configuration_file": configuration_file,
}
else:
# If the buildinstall_topdir is set, it means Koji is used for
# buildinstall phase and the filesystem with Koji is read-only.
# In that case, we have to write logs to buildinstall_topdir and
# later copy them back to our local log directory.
if self.compose.conf.get("buildinstall_topdir", None):
output_dir = os.path.join(output_dir, "results")
lorax = LoraxWrapper()
lorax_cmd = lorax.get_lorax_cmd(
self.compose.conf["release_name"],
version,
version,
repos,
output_dir,
variant=variant.uid,
buildinstallpackages=variant.buildinstallpackages,
is_final=self.compose.supported,
buildarch=buildarch,
volid=volid,
nomacboot=nomacboot,
bugurl=bugurl,
add_template=add_template,
add_arch_template=add_arch_template,
add_template_var=add_template_var,
add_arch_template_var=add_arch_template_var,
noupgrade=noupgrade,
rootfs_size=rootfs_size,
log_dir=log_dir,
dracut_args=dracut_args,
skip_branding=skip_branding,
squashfs_only=squashfs_only,
configuration_file=configuration_file,
)
return "rm -rf %s && %s" % (
shlex_quote(output_topdir),
" ".join([shlex_quote(x) for x in lorax_cmd]),
)
def get_repos(self, arch):
repos = []
for pkgset in self.pkgset_phase.package_sets:
repos.append(pkgset.paths[arch])
return repos
def run(self):
lorax = LoraxWrapper()
product = self.compose.conf["release_name"]
version = self.compose.conf["release_version"]
release = self.compose.conf["release_version"]
disc_type = self.compose.conf["disc_types"].get("dvd", "dvd")
# Prepare kickstart file for final images.
self.pool.kickstart_file = get_kickstart_file(self.compose)
for arch in self.compose.get_arches():
commands = []
output_dir = self.compose.paths.work.buildinstall_dir(
arch, allow_topdir_override=True
)
final_output_dir = self.compose.paths.work.buildinstall_dir(
arch, allow_topdir_override=False
)
makedirs(final_output_dir)
repo_baseurls = self.get_repos(arch)
if final_output_dir != output_dir:
repo_baseurls = [translate_path(self.compose, r) for r in repo_baseurls]
if self.buildinstall_method == "lorax":
buildarch = get_valid_arches(arch)[0]
for variant in self.compose.get_variants(arch=arch, types=["variant"]):
if variant.is_empty:
continue
skip = get_arch_variant_data(
self.compose.conf, "buildinstall_skip", arch, variant
)
if skip == [True]:
self.compose.log_info(
"Skipping buildinstall for %s.%s due to config option"
% (variant, arch)
)
continue
volid = get_volid(
self.compose, arch, variant=variant, disc_type=disc_type
)
commands.append(
(
variant,
self._get_lorax_cmd(
repo_baseurls,
output_dir,
variant,
arch,
buildarch,
volid,
final_output_dir,
),
)
)
elif self.buildinstall_method == "buildinstall":
volid = get_volid(self.compose, arch, disc_type=disc_type)
commands.append(
(
None,
lorax.get_buildinstall_cmd(
product,
version,
release,
repo_baseurls,
output_dir,
is_final=self.compose.supported,
buildarch=arch,
volid=volid,
),
)
)
else:
raise ValueError(
"Unsupported buildinstall method: %s" % self.buildinstall_method
)
for variant, cmd in commands:
self.pool.add(BuildinstallThread(self.pool))
self.pool.queue_put(
(self.compose, arch, variant, cmd, self.pkgset_phase)
)
self.pool.start()
def succeeded(self, variant, arch):
# If the phase is skipped, we can treat it as successful. Either there
# will be no output, or it's a debug run of compose where anything can
# happen.
return (
super(BuildinstallPhase, self).skip()
or (variant.uid if self.used_lorax else None, arch)
in self.pool.finished_tasks
)
def reused(self, variant, arch):
"""
Check if buildinstall phase reused previous results for given variant
and arch. If the phase is skipped, the results will be considered
reused as well.
"""
return (
super(BuildinstallPhase, self).skip()
or (variant.uid if self.used_lorax else None, arch)
in self.pool.reused_tasks
)
def get_kickstart_file(compose):
scm_dict = compose.conf.get("buildinstall_kickstart")
if not scm_dict:
compose.log_debug("Path to ks.cfg (buildinstall_kickstart) not specified.")
return
msg = "Getting ks.cfg"
kickstart_path = os.path.join(compose.paths.work.topdir(arch="global"), "ks.cfg")
if os.path.exists(kickstart_path):
compose.log_warning("[SKIP ] %s" % msg)
return kickstart_path
compose.log_info("[BEGIN] %s" % msg)
if isinstance(scm_dict, dict):
kickstart_name = os.path.basename(scm_dict["file"])
if scm_dict["scm"] == "file":
scm_dict["file"] = os.path.join(compose.config_dir, scm_dict["file"])
else:
kickstart_name = os.path.basename(scm_dict)
scm_dict = os.path.join(compose.config_dir, scm_dict)
tmp_dir = compose.mkdtemp(prefix="buildinstall_kickstart_")
get_file_from_scm(scm_dict, tmp_dir, compose=compose)
src = os.path.join(tmp_dir, kickstart_name)
shutil.copy2(src, kickstart_path)
compose.log_info("[DONE ] %s" % msg)
return kickstart_path
BOOT_CONFIGS = [
"isolinux/isolinux.cfg",
"etc/yaboot.conf",
"ppc/ppc64/yaboot.conf",
"EFI/BOOT/BOOTX64.conf",
"EFI/BOOT/grub.cfg",
]
BOOT_IMAGES = [
"images/efiboot.img",
]
def tweak_configs(path, volid, ks_file, configs=BOOT_CONFIGS, logger=None):
"""
Put escaped volume ID and possibly kickstart file into the boot
configuration files.
:returns: list of paths to modified config files
"""
volid_escaped = volid.replace(" ", r"\x20").replace("\\", "\\\\")
volid_escaped_2 = volid_escaped.replace("\\", "\\\\")
found_configs = []
for config in configs:
config_path = os.path.join(path, config)
if not os.path.exists(config_path):
continue
with open(config_path, "r") as f:
data = original_data = f.read()
os.unlink(config_path) # break hadlink by removing file writing a new one
# double-escape volid in yaboot.conf
new_volid = volid_escaped_2 if "yaboot" in config else volid_escaped
ks = (" inst.ks=hd:LABEL=%s:/ks.cfg" % new_volid) if ks_file else ""
# pre-f18
data = re.sub(r":CDLABEL=[^ \n]*", r":CDLABEL=%s%s" % (new_volid, ks), data)
# f18+
data = re.sub(r":LABEL=[^ \n]*", r":LABEL=%s%s" % (new_volid, ks), data)
data = re.sub(r"(search .* -l) '[^'\n]*'", r"\1 '%s'" % volid, data)
with open(config_path, "w") as f:
f.write(data)
if data != original_data:
found_configs.append(config)
if logger:
# Generally lorax should create file with correct volume id
# already. If we don't have a kickstart, this function should
# be a no-op.
logger.info("Boot config %s changed" % config_path)
return found_configs
# HACK: this is a hack!
# * it's quite trivial to replace volids
# * it's not easy to replace menu titles
# * we probably need to get this into lorax
def tweak_buildinstall(
compose, src, dst, arch, variant, label, volid, kickstart_file=None
):
tmp_dir = compose.mkdtemp(prefix="tweak_buildinstall_")
# verify src
if not os.path.isdir(src):
raise OSError(errno.ENOENT, "Directory does not exist: %s" % src)
# create dst
try:
os.makedirs(dst)
except OSError as ex:
if ex.errno != errno.EEXIST:
raise
# copy src to temp
# TODO: place temp on the same device as buildinstall dir so we can hardlink
cmd = "cp -dRv --preserve=mode,links,timestamps --remove-destination %s/* %s/" % (
shlex_quote(src),
shlex_quote(tmp_dir),
)
run(cmd)
found_configs = tweak_configs(
tmp_dir, volid, kickstart_file, logger=compose._logger
)
if kickstart_file and found_configs:
shutil.copy2(kickstart_file, os.path.join(dst, "ks.cfg"))
images = [os.path.join(tmp_dir, img) for img in BOOT_IMAGES]
if found_configs:
for image in images:
if not os.path.isfile(image):
continue
with iso.mount(
image,
logger=compose._logger,
use_guestmount=compose.conf.get("buildinstall_use_guestmount"),
) as mount_tmp_dir:
for config in found_configs:
# Put each modified config file into the image (overwriting the
# original).
config_path = os.path.join(tmp_dir, config)
config_in_image = os.path.join(mount_tmp_dir, config)
if os.path.isfile(config_in_image):
cmd = [
"cp",
"-v",
"--remove-destination",
config_path,
config_in_image,
]
run(cmd)
# HACK: make buildinstall files world readable
run("chmod -R a+rX %s" % shlex_quote(tmp_dir))
# copy temp to dst
cmd = "cp -dRv --preserve=mode,links,timestamps --remove-destination %s/* %s/" % (
shlex_quote(tmp_dir),
shlex_quote(dst),
)
run(cmd)
shutil.rmtree(tmp_dir)
def link_boot_iso(compose, arch, variant, can_fail):
if arch == "src":
return
disc_type = compose.conf["disc_types"].get("boot", "boot")
symlink_isos_to = compose.conf.get("symlink_isos_to")
os_tree = compose.paths.compose.os_tree(arch, variant)
# TODO: find in treeinfo?
boot_iso_path = os.path.join(os_tree, "images", "boot.iso")
if not os.path.isfile(boot_iso_path):
return
msg = "Linking boot.iso (arch: %s, variant: %s)" % (arch, variant)
filename = compose.get_image_name(
arch, variant, disc_type=disc_type, disc_num=None, suffix=".iso"
)
new_boot_iso_path = compose.paths.compose.iso_path(
arch, variant, filename, symlink_to=symlink_isos_to
)
new_boot_iso_relative_path = compose.paths.compose.iso_path(
arch, variant, filename, relative=True
)
if os.path.exists(new_boot_iso_path):
# TODO: log
compose.log_warning("[SKIP ] %s" % msg)
return
compose.log_info("[BEGIN] %s" % msg)
# Try to hardlink, and copy if that fails
try:
os.link(boot_iso_path, new_boot_iso_path)
except OSError:
shutil.copy2(boot_iso_path, new_boot_iso_path)
implant_md5 = iso.get_implanted_md5(new_boot_iso_path)
iso_name = os.path.basename(new_boot_iso_path)
iso_dir = os.path.dirname(new_boot_iso_path)
# create iso manifest
run(iso.get_manifest_cmd(iso_name), workdir=iso_dir)
img = Image(compose.im)
img.path = new_boot_iso_relative_path
img.mtime = get_mtime(new_boot_iso_path)
img.size = get_file_size(new_boot_iso_path)
img.arch = arch
img.type = "boot"
img.format = "iso"
img.disc_number = 1
img.disc_count = 1
img.bootable = True
img.subvariant = variant.uid
img.implant_md5 = implant_md5
setattr(img, "can_fail", can_fail)
setattr(img, "deliverable", "buildinstall")
try:
img.volume_id = iso.get_volume_id(new_boot_iso_path)
except RuntimeError:
pass
compose.im.add(variant.uid, arch, img)
compose.log_info("[DONE ] %s" % msg)
class BuildinstallThread(WorkerThread):
def process(self, item, num):
# The variant is None unless lorax is used as buildinstall method.
compose, arch, variant, cmd, pkgset_phase = item
can_fail = compose.can_fail(variant, arch, "buildinstall")
with failable(compose, can_fail, variant, arch, "buildinstall"):
try:
self.worker(compose, arch, variant, cmd, pkgset_phase, num)
except RuntimeError:
self._print_depsolve_error(compose, arch, variant)
raise
def _print_depsolve_error(self, compose, arch, variant):
try:
log_file = os.path.join(_get_log_dir(compose, variant, arch), "pylorax.log")
with open(log_file) as f:
matched = False
for line in f:
if re.match("Dependency check failed", line):
matched = True
if matched:
compose.log_error(line.rstrip())
except Exception:
pass
def _generate_buildinstall_metadata(
self, compose, arch, variant, cmd, buildroot_rpms, pkgset_phase
):
"""
Generate buildinstall.metadata dict.
:param Compose compose: Current compose.
:param str arch: Current architecture.
:param Variant variant: Compose variant.
:param list cmd: List of command line arguments passed to buildinstall task.
:param list buildroot_rpms: List of NVRAs of all RPMs installed in the
buildinstall task's buildroot.
:param PkgsetPhase pkgset_phase: Package set phase instance.
:return: The buildinstall.metadata dict.
"""
# Load the list of packages installed in the boot.iso.
# The list of installed packages is logged by Lorax in the "pkglists"
# directory. There is one file for each installed RPM and the name
# of the file is the name of the RPM.
# We need to resolve the name of each RPM back to its NVRA.
installed_rpms = []
log_fname = "buildinstall-%s-logs/dummy" % variant.uid
log_dir = os.path.dirname(compose.paths.log.log_file(arch, log_fname, False))
pkglists_dir = os.path.join(log_dir, "pkglists")
if os.path.exists(pkglists_dir):
for pkg_name in os.listdir(pkglists_dir):
for pkgset in pkgset_phase.package_sets:
global_pkgset = pkgset["global"]
# We actually do not care from which package_set the RPM
# came from or if there are multiple versions/release of
# the single RPM in more packages sets. We simply include
# all RPMs with this name in the metadata.
# Later when deciding if the buildinstall phase results
# can be reused, we check that all the RPMs with this name
# are still the same in old/new compose.
for rpm_path, rpm_obj in global_pkgset.file_cache.items():
if rpm_obj.name == pkg_name:
installed_rpms.append(rpm_path)
# Store the metadata in `buildinstall.metadata`.
metadata = {
"cmd": cmd,
"buildroot_rpms": sorted(buildroot_rpms),
"installed_rpms": sorted(installed_rpms),
}
return metadata
def _write_buildinstall_metadata(
self, compose, arch, variant, cmd, buildroot_rpms, pkgset_phase
):
"""
Write buildinstall.metadata file containing all the information about
buildinstall phase input and environment.
This file is later used to decide whether old buildinstall results can
be reused instead of generating them again.
:param Compose compose: Current compose.
:param str arch: Current architecture.
:param Variant variant: Compose variant.
:param list cmd: List of command line arguments passed to buildinstall task.
:param list buildroot_rpms: List of NVRAs of all RPMs installed in the
buildinstall task's buildroot.
:param PkgsetPhase pkgset_phase: Package set phase instance.
"""
# Generate the list of `*-RPMs` log file.
log_filename = ("buildinstall-%s" % variant.uid) if variant else "buildinstall"
log_file = compose.paths.log.log_file(arch, log_filename + "-RPMs")
with open(log_file, "w") as f:
f.write("\n".join(buildroot_rpms))
# Write buildinstall.metadata only if particular variant is defined.
# The `variant` is `None` only if old "buildinstall" method is used.
if not variant:
return
metadata = self._generate_buildinstall_metadata(
compose, arch, variant, cmd, buildroot_rpms, pkgset_phase
)
log_fname = "buildinstall-%s-logs/dummy" % variant.uid
log_dir = os.path.dirname(compose.paths.log.log_file(arch, log_fname))
metadata_path = os.path.join(log_dir, "buildinstall.metadata")
with open(metadata_path, "wb") as f:
pickle.dump(metadata, f, protocol=pickle.HIGHEST_PROTOCOL)
def _load_old_buildinstall_metadata(self, compose, arch, variant):
"""
Helper method to load "buildinstall.metadata" from old compose.
:param Compose compose: Current compose.
:param str arch: Current architecture.
:param Variant variant: Compose variant.
"""
if not variant:
return None
log_fname = "buildinstall-%s-logs/dummy" % variant.uid
metadata = os.path.join(
os.path.dirname(compose.paths.log.log_file(arch, log_fname)),
"buildinstall.metadata",
)
old_metadata = compose.paths.old_compose_path(metadata)
if not old_metadata:
return None
compose.log_info("Loading old BUILDINSTALL phase metadata: %s", old_metadata)
try:
with open(old_metadata, "rb") as f:
old_result = pickle.load(f)
return old_result
except Exception as e:
compose.log_debug(
"Failed to load old BUILDINSTALL phase metadata %s : %s"
% (old_metadata, str(e))
)
return None
def _reuse_old_buildinstall_result(self, compose, arch, variant, cmd, pkgset_phase):
"""
Try to reuse old buildinstall results.
:param Compose compose: Current compose.
:param str arch: Current architecture.
:param Variant variant: Compose variant.
:param list cmd: List of command line arguments passed to buildinstall task.
:param list buildroot_rpms: List of NVRAs of all RPMs installed in the
buildinstall task's buildroot.
:param PkgsetPhase pkgset_phase: Package set phase instance.
:return: True if old buildinstall phase results have been reused.
"""
log_msg = "Cannot reuse old BUILDINSTALL phase results - %s"
if not compose.conf["buildinstall_allow_reuse"]:
compose.log_info(log_msg % "reuse of old buildinstall results is disabled.")
return
# Load the old buildinstall.metadata.
old_metadata = self._load_old_buildinstall_metadata(compose, arch, variant)
if old_metadata is None:
compose.log_info(log_msg % "no old BUILDINSTALL metadata.")
return
# For now try to reuse only if pungi_buildinstall plugin is used.
# This is the easiest approach, because we later need to filter out
# some parts of `cmd` and for pungi_buildinstall, the `cmd` is a dict
# which makes this easy.
if not isinstance(old_metadata["cmd"], dict) or not isinstance(cmd, dict):
compose.log_info(log_msg % "pungi_buildinstall plugin is not used.")
return
# Filter out "outputdir" and "sources" because they change every time.
# The "sources" are not important, because we check the buildinstall
# input on RPM level.
cmd_copy = copy(cmd)
for key in ["outputdir", "sources"]:
del cmd_copy[key]
del old_metadata["cmd"][key]
# Do not reuse if command line arguments are not the same.
if old_metadata["cmd"] != cmd_copy:
compose.log_info(log_msg % "lorax command line arguments differ.")
return
# Check that the RPMs installed in the old boot.iso exists in the very
# same versions/releases in this compose.
for rpm_path in old_metadata["installed_rpms"]:
found = False
for pkgset in pkgset_phase.package_sets:
global_pkgset = pkgset["global"]
if rpm_path in global_pkgset.file_cache:
found = True
break
if not found:
compose.log_info(
log_msg % "RPM %s does not exist in new compose." % rpm_path
)
return
# Ask Koji for all the RPMs in the `runroot_tag` and check that
# those installed in the old buildinstall buildroot are still in the
# very same versions/releases.
koji_wrapper = kojiwrapper.KojiWrapper(compose)
rpms = koji_wrapper.koji_proxy.listTaggedRPMS(
compose.conf.get("runroot_tag"), inherit=True, latest=True
)[0]
rpm_nvras = set()
for rpm in rpms:
rpm_nvras.add(kobo.rpmlib.make_nvra(rpm, add_rpm=False, force_epoch=False))
for old_nvra in old_metadata["buildroot_rpms"]:
if old_nvra not in rpm_nvras:
compose.log_info(
log_msg % "RPM %s does not exist in new buildroot." % old_nvra
)
return
# We can reuse the old buildinstall results!
compose.log_info("Reusing old BUILDINSTALL phase output")
# Copy old buildinstall output to this this compose.
final_output_dir = compose.paths.work.buildinstall_dir(arch, variant=variant)
old_final_output_dir = compose.paths.old_compose_path(final_output_dir)
copy_all(old_final_output_dir, final_output_dir)
# Copy old buildinstall logs to this compose.
log_fname = "buildinstall-%s-logs/dummy" % variant.uid
final_log_dir = os.path.dirname(compose.paths.log.log_file(arch, log_fname))
old_final_log_dir = compose.paths.old_compose_path(final_log_dir)
if not os.path.exists(final_log_dir):
makedirs(final_log_dir)
copy_all(old_final_log_dir, final_log_dir)
# Write the buildinstall metadata so next compose can reuse this compose.
self._write_buildinstall_metadata(
compose, arch, variant, cmd, old_metadata["buildroot_rpms"], pkgset_phase
)
return True
def worker(self, compose, arch, variant, cmd, pkgset_phase, num):
buildinstall_method = compose.conf["buildinstall_method"]
lorax_use_koji_plugin = compose.conf["lorax_use_koji_plugin"]
log_filename = ("buildinstall-%s" % variant.uid) if variant else "buildinstall"
log_file = compose.paths.log.log_file(arch, log_filename)
msg = "Running buildinstall for arch %s, variant %s" % (arch, variant)
output_dir = compose.paths.work.buildinstall_dir(
arch, allow_topdir_override=True, variant=variant
)
final_output_dir = compose.paths.work.buildinstall_dir(arch, variant=variant)
if (
os.path.isdir(output_dir)
and os.listdir(output_dir)
or os.path.isdir(final_output_dir)
and os.listdir(final_output_dir)
):
# output dir is *not* empty -> SKIP
self.pool.log_warning(
"[SKIP ] Buildinstall for arch %s, variant %s" % (arch, variant)
)
return
self.pool.log_info("[BEGIN] %s" % msg)
# Get list of packages which are needed in runroot.
packages = []
chown_paths = [output_dir]
if buildinstall_method == "lorax":
packages += ["lorax"]
chown_paths.append(_get_log_dir(compose, variant, arch))
elif buildinstall_method == "buildinstall":
packages += ["anaconda"]
packages += get_arch_variant_data(
compose.conf, "buildinstall_packages", arch, variant
)
if self._reuse_old_buildinstall_result(
compose, arch, variant, cmd, pkgset_phase
):
self.copy_files(compose, variant, arch)
self.pool.finished_tasks.add((variant.uid if variant else None, arch))
self.pool.reused_tasks.add((variant.uid if variant else None, arch))
self.pool.log_info("[DONE ] %s" % msg)
return
# This should avoid a possible race condition with multiple processes
# trying to get a kerberos ticket at the same time.
# Kerberos authentication failed:
# Permission denied in replay cache code (-1765328215)
time.sleep(num * 3)
# Start the runroot task.
runroot = Runroot(compose, phase="buildinstall")
if buildinstall_method == "lorax" and lorax_use_koji_plugin:
runroot.run_pungi_buildinstall(
cmd,
log_file=log_file,
arch=arch,
packages=packages,
mounts=[compose.topdir],
weight=compose.conf["runroot_weights"].get("buildinstall"),
)
else:
try:
lorax_log_dir = _get_log_dir(compose, variant, arch)
except Exception:
lorax_log_dir = None
runroot.run(
cmd,
log_file=log_file,
arch=arch,
packages=packages,
mounts=[compose.topdir],
weight=compose.conf["runroot_weights"].get("buildinstall"),
chown_paths=chown_paths,
log_dir=lorax_log_dir,
)
if final_output_dir != output_dir:
if not os.path.exists(final_output_dir):
makedirs(final_output_dir)
results_dir = os.path.join(output_dir, "results")
copy_all(results_dir, final_output_dir)
# Get the log_dir into which we should copy the resulting log files.
log_fname = "buildinstall-%s-logs/dummy" % variant.uid
final_log_dir = os.path.dirname(compose.paths.log.log_file(arch, log_fname))
if not os.path.exists(final_log_dir):
makedirs(final_log_dir)
log_dir = os.path.join(output_dir, "logs")
copy_all(log_dir, final_log_dir)
elif lorax_use_koji_plugin:
# If Koji pungi-buildinstall is used, then the buildinstall results are
# not stored directly in `output_dir` dir, but in "results" and "logs"
# subdirectories. We need to move them to final_output_dir.
results_dir = os.path.join(output_dir, "results")
move_all(results_dir, final_output_dir, rm_src_dir=True)
# Get the log_dir into which we should copy the resulting log files.
log_fname = "buildinstall-%s-logs/dummy" % variant.uid
final_log_dir = os.path.dirname(compose.paths.log.log_file(arch, log_fname))
if not os.path.exists(final_log_dir):
makedirs(final_log_dir)
log_dir = os.path.join(output_dir, "logs")
move_all(log_dir, final_log_dir, rm_src_dir=True)
rpms = runroot.get_buildroot_rpms()
self._write_buildinstall_metadata(
compose, arch, variant, cmd, rpms, pkgset_phase
)
self.copy_files(compose, variant, arch)
self.pool.finished_tasks.add((variant.uid if variant else None, arch))
self.pool.log_info("[DONE ] %s" % msg)
def copy_files(self, compose, variant, arch):
disc_type = compose.conf["disc_types"].get("dvd", "dvd")
buildinstall_dir = compose.paths.work.buildinstall_dir(arch)
# Lorax runs per-variant, so we need to tweak the source path
# to include variant.
if variant:
buildinstall_dir = os.path.join(buildinstall_dir, variant.uid)
# Find all relevant variants if lorax is not used.
variants = (
[variant]
if variant
else compose.get_variants(arch=arch, types=["self", "variant"])
)
for var in variants:
os_tree = compose.paths.compose.os_tree(arch, var)
# TODO: label is not used
label = ""
volid = get_volid(compose, arch, var, disc_type=disc_type)
can_fail = compose.can_fail(var, arch, "buildinstall")
tweak_buildinstall(
compose,
buildinstall_dir,
os_tree,
arch,
var.uid,
label,
volid,
self.pool.kickstart_file,
)
link_boot_iso(compose, arch, var, can_fail)
def _get_log_dir(compose, variant, arch):
"""Find directory where to store lorax logs in. If it's inside the compose,
create the directory.
"""
if compose.conf.get("buildinstall_topdir"):
log_dir = compose.paths.work.buildinstall_dir(
arch, allow_topdir_override=True, variant=variant
)
return os.path.join(log_dir, "logs")
# The paths module will modify the filename (by inserting arch). But we
# only care about the directory anyway.
log_filename = "buildinstall-%s-logs/dummy" % variant.uid
log_dir = os.path.dirname(compose.paths.log.log_file(arch, log_filename))
makedirs(log_dir)
return log_dir