# -*- coding: utf-8 -*- # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; version 2 of the License. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU Library General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. import errno import os import time import pipes import tempfile import shutil import re import errno from kobo.threads import ThreadPool, WorkerThread from kobo.shortcuts import run, read_checksum_file, relative_path from productmd.imagemanifest import Image from pypungi.util import get_buildroot_rpms, get_volid from pypungi.wrappers.lorax import LoraxWrapper from pypungi.wrappers.kojiwrapper import KojiWrapper from pypungi.wrappers.iso import IsoWrapper from pypungi.wrappers.scm import get_file_from_scm from pypungi.phases.base import PhaseBase class BuildinstallPhase(PhaseBase): name = "buildinstall" config_options = ( { "name": "bootable", "expected_types": [bool], "expected_values": [True], }, { "name": "buildinstall_method", "extected_types": [str], "expected_values": ["lorax", "buildinstall"], "requires": ( (lambda x: bool(x) is True, ["bootable"]), ), }, { "name": "buildinstall_upgrade_image", "expected_types": [bool], "optional": True, }, { "name": "buildinstall_kickstart", "expected_types": [str], "optional": True, }, ) def __init__(self, compose): PhaseBase.__init__(self, compose) self.pool = ThreadPool(logger=self.compose._logger) def skip(self): if PhaseBase.skip(self): return True if not self.compose.conf.get("bootable"): msg = "Not a bootable product. Skipping buildinstall." self.compose.log_debug(msg) return True return False def run(self): lorax = LoraxWrapper() product = self.compose.conf["product_name"] version = self.compose.conf["product_version"] release = self.compose.conf["product_version"] noupgrade = not self.compose.conf.get("buildinstall_upgrade_image", False) buildinstall_method = self.compose.conf["buildinstall_method"] for arch in self.compose.get_arches(): repo_baseurl = self.compose.paths.work.arch_repo(arch) output_dir = self.compose.paths.work.buildinstall_dir(arch) volid = get_volid(self.compose, arch) if buildinstall_method == "lorax": cmd = lorax.get_lorax_cmd(product, version, release, repo_baseurl, output_dir, is_final=self.compose.supported, buildarch=arch, volid=volid, nomacboot=True, noupgrade=noupgrade) elif buildinstall_method == "buildinstall": cmd = lorax.get_buildinstall_cmd(product, version, release, repo_baseurl, output_dir, is_final=self.compose.supported, buildarch=arch, volid=volid) else: raise ValueError("Unsupported buildinstall method: %s" % buildinstall_method) self.pool.add(BuildinstallThread(self.pool)) self.pool.queue_put((self.compose, arch, cmd)) self.pool.start() def copy_files(self): # copy buildinstall files to the 'os' dir kickstart_file = get_kickstart_file(self.compose) for arch in self.compose.get_arches(): for variant in self.compose.get_variants(arch=arch, types=["self", "variant"]): buildinstall_dir = self.compose.paths.work.buildinstall_dir(arch) if not os.path.isdir(buildinstall_dir) or not os.listdir(buildinstall_dir): continue os_tree = self.compose.paths.compose.os_tree(arch, variant) # TODO: label is not used label = "" volid = get_volid(self.compose, arch, variant, escape_spaces=False) tweak_buildinstall(buildinstall_dir, os_tree, arch, variant.uid, label, volid, kickstart_file) symlink_boot_iso(self.compose, arch, variant) def get_kickstart_file(compose): scm_dict = compose.conf.get("buildinstall_kickstart", None) if not scm_dict: compose.log_debug("Path to ks.cfg (buildinstall_kickstart) not specified.") return msg = "Getting ks.cfg" kickstart_path = os.path.join(compose.paths.work.topdir(arch="global"), "ks.cfg") if os.path.exists(kickstart_path): compose.log_warn("[SKIP ] %s" % msg) return kickstart_path compose.log_info("[BEGIN] %s" % msg) if isinstance(scm_dict, dict): kickstart_name = os.path.basename(scm_dict["file"]) if scm_dict["scm"] == "file": scm_dict["file"] = os.path.join(compose.config_dir, scm_dict["file"]) else: kickstart_name = os.path.basename(scm_dict) scm_dict = os.path.join(compose.config_dir, scm_dict) tmp_dir = tempfile.mkdtemp(prefix="buildinstall_kickstart_") get_file_from_scm(scm_dict, tmp_dir, logger=compose._logger) src = os.path.join(tmp_dir, kickstart_name) shutil.copy2(src, kickstart_path) compose.log_info("[DONE ] %s" % msg) return kickstart_path # HACK: this is a hack! # * it's quite trivial to replace volids # * it's not easy to replace menu titles # * we probably need to get this into lorax def tweak_buildinstall(src, dst, arch, variant, label, volid, kickstart_file=None): volid_escaped = volid.replace(" ", r"\x20").replace("\\", "\\\\") volid_escaped_2 = volid_escaped.replace("\\", "\\\\") tmp_dir = tempfile.mkdtemp(prefix="tweak_buildinstall_") # verify src if not os.path.isdir(src): raise OSError(errno.ENOENT, "Directory does not exist: %s" % src) # create dst try: os.makedirs(dst) except OSError as ex: if ex.errno != errno.EEXIST: raise # copy src to temp # TODO: place temp on the same device as buildinstall dir so we can hardlink cmd = "cp -av --remove-destination %s/* %s/" % (pipes.quote(src), pipes.quote(tmp_dir)) run(cmd) # tweak configs configs = [ "isolinux/isolinux.cfg", "etc/yaboot.conf", "ppc/ppc64/yaboot.conf", "EFI/BOOT/BOOTX64.conf", "EFI/BOOT/grub.cfg", ] for config in configs: config_path = os.path.join(tmp_dir, config) if not os.path.exists(config_path): continue data = open(config_path, "r").read() os.unlink(config_path) # break hadlink by removing file writing a new one new_volid = volid_escaped if "yaboot" in config: # double-escape volid in yaboot.conf new_volid = volid_escaped_2 ks = "" if kickstart_file: shutil.copy2(kickstart_file, os.path.join(dst, "ks.cfg")) ks = " ks=hd:LABEL=%s:/ks.cfg" % new_volid # pre-f18 data = re.sub(r":CDLABEL=[^ \n]*", r":CDLABEL=%s%s" % (new_volid, ks), data) # f18+ data = re.sub(r":LABEL=[^ \n]*", r":LABEL=%s%s" % (new_volid, ks), data) data = re.sub(r"(search .* -l) '[^'\n]*'", r"\1 '%s'" % volid, data) open(config_path, "w").write(data) images = [ os.path.join(tmp_dir, "images", "efiboot.img"), ] for image in images: if not os.path.isfile(image): continue mount_tmp_dir = tempfile.mkdtemp(prefix="tweak_buildinstall") cmd = ["mount", "-o", "loop", image, mount_tmp_dir] run(cmd) for config in configs: config_path = os.path.join(tmp_dir, config) config_in_image = os.path.join(mount_tmp_dir, config) if os.path.isfile(config_in_image): cmd = ["cp", "-v", "--remove-destination", config_path, config_in_image] run(cmd) cmd = ["umount", mount_tmp_dir] run(cmd) shutil.rmtree(mount_tmp_dir) # HACK: make buildinstall files world readable run("chmod -R a+rX %s" % pipes.quote(tmp_dir)) # copy temp to dst cmd = "cp -av --remove-destination %s/* %s/" % (pipes.quote(tmp_dir), pipes.quote(dst)) run(cmd) shutil.rmtree(tmp_dir) def symlink_boot_iso(compose, arch, variant): if arch == "src": return symlink_isos_to = compose.conf.get("symlink_isos_to", None) os_tree = compose.paths.compose.os_tree(arch, variant) # TODO: find in treeinfo? boot_iso_path = os.path.join(os_tree, "images", "boot.iso") if not os.path.isfile(boot_iso_path): return msg = "Symlinking boot.iso (arch: %s, variant: %s)" % (arch, variant) new_boot_iso_path = compose.paths.compose.iso_path(arch, variant, disc_type="boot", disc_num=None, suffix=".iso", symlink_to=symlink_isos_to) new_boot_iso_relative_path = compose.paths.compose.iso_path(arch, variant, disc_type="boot", disc_num=None, suffix=".iso", relative=True) if os.path.exists(new_boot_iso_path): # TODO: log compose.log_warning("[SKIP ] %s" % msg) return compose.log_info("[BEGIN] %s" % msg) # can't make a hardlink - possible cross-device link due to 'symlink_to' argument symlink_target = relative_path(boot_iso_path, new_boot_iso_path) os.symlink(symlink_target, new_boot_iso_path) iso = IsoWrapper() implant_md5 = iso.get_implanted_md5(new_boot_iso_path) # compute md5sum, sha1sum, sha256sum iso_name = os.path.basename(new_boot_iso_path) iso_dir = os.path.dirname(new_boot_iso_path) for cmd in iso.get_checksum_cmds(iso_name): run(cmd, workdir=iso_dir) # create iso manifest run(iso.get_manifest_cmd(iso_name), workdir=iso_dir) img = Image(compose.im) img.implant_md5 = iso.get_implanted_md5(new_boot_iso_path) img.path = new_boot_iso_relative_path img.mtime = int(os.stat(new_boot_iso_path).st_mtime) img.size = os.path.getsize(new_boot_iso_path) img.arch = arch img.type = "boot" img.format = "iso" img.disc_number = 1 img.disc_count = 1 for checksum_type in ("md5", "sha1", "sha256"): checksum_path = new_boot_iso_path + ".%sSUM" % checksum_type.upper() checksum_value = None if os.path.isfile(checksum_path): checksum_value, iso_name = read_checksum_file(checksum_path)[0] if iso_name != os.path.basename(img.path): # a bit paranoind check - this should never happen raise ValueError("Image name doesn't match checksum: %s" % checksum_path) img.add_checksum(compose.paths.compose.topdir(), checksum_type=checksum_type, checksum_value=checksum_value) img.bootable = True img.implant_md5 = implant_md5 try: img.volume_id = iso.get_volume_id(new_boot_iso_path) except RuntimeError: pass compose.im.add(arch, variant.uid, img) compose.log_info("[DONE ] %s" % msg) class BuildinstallThread(WorkerThread): def process(self, item, num): compose, arch, cmd = item runroot = compose.conf.get("runroot", False) buildinstall_method = compose.conf["buildinstall_method"] log_file = compose.paths.log.log_file(arch, "buildinstall") msg = "Runnging buildinstall for arch %s" % arch output_dir = compose.paths.work.buildinstall_dir(arch) if os.path.isdir(output_dir): if os.listdir(output_dir): # output dir is *not* empty -> SKIP self.pool.log_warning("[SKIP ] %s" % msg) return else: # output dir is empty -> remove it and run buildinstall self.pool.log_debug("Removing existing (but empty) buildinstall dir: %s" % output_dir) os.rmdir(output_dir) self.pool.log_info("[BEGIN] %s" % msg) task_id = None if runroot: # run in a koji build root # glibc32 is needed by yaboot on ppc64 packages = ["glibc32", "strace"] if buildinstall_method == "lorax": packages += ["lorax"] elif buildinstall_method == "buildinstall": packages += ["anaconda"] runroot_channel = compose.conf.get("runroot_channel", None) runroot_tag = compose.conf["runroot_tag"] koji_wrapper = KojiWrapper(compose.conf["koji_profile"]) koji_cmd = koji_wrapper.get_runroot_cmd(runroot_tag, arch, cmd, channel=runroot_channel, use_shell=True, task_id=True, packages=packages, mounts=[compose.topdir]) # avoid race conditions? # Kerberos authentication failed: Permission denied in replay cache code (-1765328215) time.sleep(num * 3) output = koji_wrapper.run_runroot_cmd(koji_cmd, log_file=log_file) task_id = int(output["task_id"]) if output["retcode"] != 0: raise RuntimeError("Runroot task failed: %s. See %s for more details." % (output["task_id"], log_file)) else: # run locally run(cmd, show_cmd=True, logfile=log_file) log_file = compose.paths.log.log_file(arch, "buildinstall-RPMs") rpms = get_buildroot_rpms(compose, task_id) open(log_file, "w").write("\n".join(rpms)) self.pool.log_info("[DONE ] %s" % msg)