From 89050f068dd24394b884ca85953b78cc97a9bdc5 Mon Sep 17 00:00:00 2001 From: "Brian C. Lane" Date: Thu, 26 Apr 2018 17:09:05 -0700 Subject: [PATCH] livemedia-creator: Move core functions into pylorax modules This reduces the amount of code in livemedia-creator to the cmdline parsing and calling of the installer functions. Moving them into other modules will allow them to be used by other projects, like the lorax-composer API server. --- src/pylorax/creator.py | 634 ++++++++++++++++++++ src/pylorax/installer.py | 569 ++++++++++++++++++ src/sbin/livemedia-creator | 1153 +----------------------------------- 3 files changed, 1212 insertions(+), 1144 deletions(-) create mode 100644 src/pylorax/creator.py create mode 100644 src/pylorax/installer.py diff --git a/src/pylorax/creator.py b/src/pylorax/creator.py new file mode 100644 index 00000000..e18e42c3 --- /dev/null +++ b/src/pylorax/creator.py @@ -0,0 +1,634 @@ +# +# Copyright (C) 2011-2018 Red Hat, Inc. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +import logging +log = logging.getLogger("pylorax") + + +import os +import tempfile +import subprocess +import shutil +import hashlib +import glob +import json +from math import ceil + +# Use Mako templates for appliance builder descriptions +from mako.template import Template +from mako.exceptions import text_error_template + +# Use the Lorax treebuilder branch for iso creation +from pylorax import ArchData +from pylorax.base import DataHolder +from pylorax.executils import execWithRedirect, runcmd +from pylorax.imgutils import PartitionMount, mkext4img +from pylorax.imgutils import mount, umount, Mount +from pylorax.imgutils import mksquashfs, mkrootfsimg +from pylorax.imgutils import copytree +from pylorax.installer import novirt_install, virt_install, InstallError +from pylorax.treebuilder import TreeBuilder, RuntimeBuilder +from pylorax.treebuilder import findkernels +from pylorax.sysutils import joinpaths, remove + + +# Default parameters for rebuilding initramfs, override with --dracut-args +DRACUT_DEFAULT = ["--xz", "--add", "livenet dmsquash-live convertfs pollcdrom qemu qemu-net", + "--omit", "plymouth", "--no-hostonly", "--debug", "--no-early-microcode"] + +RUNTIME = "images/install.img" + +class FakeDNF(object): + """ + A minimal DNF object suitable for passing to RuntimeBuilder + + lmc uses RuntimeBuilder to run the arch specific iso creation + templates, so the the installroot config value is the important part of + this. Everything else should be a nop. + """ + def __init__(self, conf): + self.conf = conf + + def reset(self): + pass + +def is_image_mounted(disk_img): + """ + Check to see if the disk_img is mounted + + :returns: True if disk_img is in /proc/mounts + :rtype: bool + """ + with open("/proc/mounts") as mounts: + for mnt in mounts: + fields = mnt.split() + if len(fields) > 2 and fields[1] == disk_img: + return True + return False + +def find_ostree_root(phys_root): + """ + Find root of ostree deployment + + :param str phys_root: Path to physical root + :returns: Relative path of ostree deployment root + :rtype: str + :raise Exception: More than one deployment roots were found + """ + ostree_root = "" + ostree_sysroots = glob.glob(joinpaths(phys_root, "ostree/boot.?/*/*/0")) + log.debug("ostree_sysroots = %s", ostree_sysroots) + if ostree_sysroots: + if len(ostree_sysroots) > 1: + raise Exception("Too many deployment roots found: %s" % ostree_sysroots) + ostree_root = os.path.relpath(ostree_sysroots[0], phys_root) + return ostree_root + +def get_arch(mount_dir): + """ + Get the kernel arch + + :returns: Arch of first kernel found at mount_dir/boot/ or i386 + :rtype: str + """ + kernels = findkernels(mount_dir) + if not kernels: + return "i386" + return kernels[0].arch + +def squashfs_args(opts): + """ Returns the compression type and args to use when making squashfs + + :param opts: ArgumentParser object with compression and compressopts + :returns: tuple of compression type and args + :rtype: tuple + """ + compression = opts.compression or "xz" + arch = ArchData(opts.arch or os.uname().machine) + if compression == "xz" and arch.bcj: + compressargs = ["-Xbcj", arch.bcj] + else: + compressargs = [] + return (compression, compressargs) + + +def make_appliance(disk_img, name, template, outfile, networks=None, ram=1024, + vcpus=1, arch=None, title="Linux", project="Linux", + releasever="29"): + """ + Generate an appliance description file + + :param str disk_img: Full path of the disk image + :param str name: Name of the appliance, passed to the template + :param str template: Full path of Mako template + :param str outfile: Full path of file to write, using template + :param list networks: List of networks(str) from the kickstart + :param int ram: Ram, in MiB, passed to template. Default is 1024 + :param int vcpus: CPUs, passed to template. Default is 1 + :param str arch: CPU architecture. Default is 'x86_64' + :param str title: Title, passed to template. Default is 'Linux' + :param str project: Project, passed to template. Default is 'Linux' + :param str releasever: Release version, passed to template. Default is 29 + """ + if not (disk_img and template and outfile): + return None + + log.info("Creating appliance definition using %s", template) + + if not arch: + arch = "x86_64" + + log.info("Calculating SHA256 checksum of %s", disk_img) + sha256 = hashlib.sha256() + with open(disk_img) as f: + while True: + data = f.read(1024**2) + if not data: + break + sha256.update(data) + log.info("SHA256 of %s is %s", disk_img, sha256.hexdigest()) + disk_info = DataHolder(name=os.path.basename(disk_img), format="raw", + checksum_type="sha256", checksum=sha256.hexdigest()) + try: + result = Template(filename=template).render(disks=[disk_info], name=name, + arch=arch, memory=ram, vcpus=vcpus, networks=networks, + title=title, project=project, releasever=releasever) + except Exception: + log.error(text_error_template().render()) + raise + + with open(outfile, "w") as f: + f.write(result) + + +def make_fsimage(diskimage, fsimage, img_size=None, label="Anaconda"): + """ + Copy the / partition of a partitioned disk image to an un-partitioned + disk image. + + :param str diskimage: The full path to partitioned disk image with a / + :param str fsimage: The full path of the output fs image file + :param int img_size: Optional size of the fsimage in MiB or None to make + it as small as possible + :param str label: The label to apply to the image. Defaults to "Anaconda" + """ + with PartitionMount(diskimage) as img_mount: + if not img_mount or not img_mount.mount_dir: + return None + + log.info("Creating fsimage %s (%s)", fsimage, img_size or "minimized") + if img_size: + # convert to Bytes + img_size *= 1024**2 + + mkext4img(img_mount.mount_dir, fsimage, size=img_size, label=label) + + +def make_runtime(opts, mount_dir, work_dir, size=None): + """ + Make the squashfs image from a directory + + :param opts: options passed to livemedia-creator + :type opts: argparse options + :param str mount_dir: Directory tree to compress + :param str work_dir: Output compressed image to work_dir+images/install.img + :param int size: Size of disk image, in GiB + """ + kernel_arch = get_arch(mount_dir) + + # Fake dnf object + fake_dbo = FakeDNF(conf=DataHolder(installroot=mount_dir)) + # Fake arch with only basearch set + arch = ArchData(kernel_arch) + # TODO: Need to get release info from someplace... + product = DataHolder(name=opts.project, version=opts.releasever, release="", + variant="", bugurl="", isfinal=False) + + # This is a mounted image partition, cannot hardlink to it, so just use it + # symlink mount_dir/images to work_dir/images so we don't run out of space + os.makedirs(joinpaths(work_dir, "images")) + + rb = RuntimeBuilder(product, arch, fake_dbo) + compression, compressargs = squashfs_args(opts) + log.info("Creating runtime") + rb.create_runtime(joinpaths(work_dir, RUNTIME), size=size, + compression=compression, compressargs=compressargs) + + +def rebuild_initrds_for_live(opts, sys_root_dir, results_dir): + """ + Rebuild intrds for pxe live image (root=live:http://) + + :param opts: options passed to livemedia-creator + :type opts: argparse options + :param str sys_root_dir: Path to root of the system + :param str results_dir: Path of directory for storing results + """ + if not opts.dracut_args: + dracut_args = DRACUT_DEFAULT + else: + dracut_args = [] + for arg in opts.dracut_args: + dracut_args += arg.split(" ", 1) + log.info("dracut args = %s", dracut_args) + + dracut = ["dracut", "--nomdadmconf", "--nolvmconf"] + dracut_args + + kdir = "boot" + if opts.ostree: + kernels_dir = glob.glob(joinpaths(sys_root_dir, "boot/ostree/*")) + if kernels_dir: + kdir = os.path.relpath(kernels_dir[0], sys_root_dir) + + kernels = [kernel for kernel in findkernels(sys_root_dir, kdir)] + if not kernels: + raise Exception("No initrds found, cannot rebuild_initrds") + + # Hush some dracut warnings. TODO: bind-mount proc in place? + open(joinpaths(sys_root_dir,"/proc/modules"),"w") + + if opts.ostree: + # Dracut assumes to have some dirs in disk image + # /var/tmp for temp files + vartmp_dir = joinpaths(sys_root_dir, "var/tmp") + if not os.path.isdir(vartmp_dir): + os.mkdir(vartmp_dir) + # /root (maybe not fatal) + root_dir = joinpaths(sys_root_dir, "var/roothome") + if not os.path.isdir(root_dir): + os.mkdir(root_dir) + # /tmp (maybe not fatal) + tmp_dir = joinpaths(sys_root_dir, "sysroot/tmp") + if not os.path.isdir(tmp_dir): + os.mkdir(tmp_dir) + + # Write the new initramfs directly to the results directory + os.mkdir(joinpaths(sys_root_dir, "results")) + mount(results_dir, opts="bind", mnt=joinpaths(sys_root_dir, "results")) + # Dracut runs out of space inside the minimal rootfs image + mount("/var/tmp", opts="bind", mnt=joinpaths(sys_root_dir, "var/tmp")) + for kernel in kernels: + if hasattr(kernel, "initrd"): + outfile = os.path.basename(kernel.initrd.path) + else: + # Construct an initrd from the kernel name + outfile = os.path.basename(kernel.path.replace("vmlinuz-", "initrd-") + ".img") + log.info("rebuilding %s", outfile) + + kver = kernel.version + + cmd = dracut + ["/results/"+outfile, kver] + runcmd(cmd, root=sys_root_dir) + + shutil.copy2(joinpaths(sys_root_dir, kernel.path), results_dir) + umount(joinpaths(sys_root_dir, "var/tmp"), delete=False) + umount(joinpaths(sys_root_dir, "results"), delete=False) + os.unlink(joinpaths(sys_root_dir,"/proc/modules")) + +def create_pxe_config(template, images_dir, live_image_name, add_args = None): + """ + Create template for pxe to live configuration + + :param str images_dir: Path of directory with images to be used + :param str live_image_name: Name of live rootfs image file + :param list add_args: Arguments to be added to initrd= pxe config + """ + + add_args = add_args or [] + + kernels = [kernel for kernel in findkernels(images_dir, kdir="") + if hasattr(kernel, "initrd")] + if not kernels: + return + + kernel = kernels[0] + + add_args_str = " ".join(add_args) + + + try: + result = Template(filename=template).render(kernel=kernel.path, + initrd=kernel.initrd.path, liveimg=live_image_name, + addargs=add_args_str) + except Exception: + log.error(text_error_template().render()) + raise + + with open (joinpaths(images_dir, "PXE_CONFIG"), "w") as f: + f.write(result) + + +def create_vagrant_metadata(path, size=0): + """ Create a default Vagrant metadata.json file + + :param str path: Path to metadata.json file + :param int size: Disk size in MiB + """ + metadata = { "provider":"libvirt", "format":"qcow2", "virtual_size": ceil(size / 1024) } + with open(path, "wt") as f: + json.dump(metadata, f, indent=4) + + +def update_vagrant_metadata(path, size): + """ Update the Vagrant metadata.json file + + :param str path: Path to metadata.json file + :param int size: Disk size in MiB + + This function makes sure that the provider, format and virtual size of the + metadata file are set correctly. All other values are left untouched. + """ + with open(path, "rt") as f: + try: + metadata = json.load(f) + except ValueError as e: + log.error("Problem reading metadata file %s: %s", path, e) + return + + metadata["provider"] = "libvirt" + metadata["format"] = "qcow2" + metadata["virtual_size"] = ceil(size / 1024) + with open(path, "wt") as f: + json.dump(metadata, f, indent=4) + + +def make_livecd(opts, mount_dir, work_dir): + """ + Take the content from the disk image and make a livecd out of it + + :param opts: options passed to livemedia-creator + :type opts: argparse options + :param str mount_dir: Directory tree to compress + :param str work_dir: Output compressed image to work_dir+images/install.img + + This uses wwood's squashfs live initramfs method: + * put the real / into LiveOS/rootfs.img + * make a squashfs of the LiveOS/rootfs.img tree + * This is loaded by dracut when the cmdline is passed to the kernel: + root=live:CDLABEL= rd.live.image + """ + kernel_arch = get_arch(mount_dir) + + arch = ArchData(kernel_arch) + # TODO: Need to get release info from someplace... + product = DataHolder(name=opts.project, version=opts.releasever, release="", + variant="", bugurl="", isfinal=False) + + # Link /images to work_dir/images to make the templates happy + if os.path.islink(joinpaths(mount_dir, "images")): + os.unlink(joinpaths(mount_dir, "images")) + execWithRedirect("/bin/ln", ["-s", joinpaths(work_dir, "images"), + joinpaths(mount_dir, "images")]) + + # The templates expect the config files to be in /tmp/config_files + # I think these should be release specific, not from lorax, but for now + configdir = joinpaths(opts.lorax_templates,"live/config_files/") + configdir_path = "tmp/config_files" + fullpath = joinpaths(mount_dir, configdir_path) + if os.path.exists(fullpath): + remove(fullpath) + copytree(configdir, fullpath) + + isolabel = opts.volid or "{0.name}-{0.version}-{1.basearch}".format(product, arch) + if len(isolabel) > 32: + isolabel = isolabel[:32] + log.warning("Truncating isolabel to 32 chars: %s", isolabel) + + tb = TreeBuilder(product=product, arch=arch, domacboot=opts.domacboot, + inroot=mount_dir, outroot=work_dir, + runtime=RUNTIME, isolabel=isolabel, + templatedir=joinpaths(opts.lorax_templates,"live/")) + log.info("Rebuilding initrds") + if not opts.dracut_args: + dracut_args = DRACUT_DEFAULT + else: + dracut_args = [] + for arg in opts.dracut_args: + dracut_args += arg.split(" ", 1) + log.info("dracut args = %s", dracut_args) + tb.rebuild_initrds(add_args=dracut_args) + log.info("Building boot.iso") + tb.build() + + return work_dir + +def mount_boot_part_over_root(img_mount): + """ + Mount boot partition to /boot of root fs mounted in img_mount + + Used for OSTree so it finds deployment configurations on live rootfs + + param img_mount: object with mounted disk image root partition + type img_mount: imgutils.PartitionMount + """ + root_dir = img_mount.mount_dir + is_boot_part = lambda dir: os.path.exists(dir+"/loader.0") + tmp_mount_dir = tempfile.mkdtemp(prefix="lmc-tmpdir-") + sysroot_boot_dir = None + for dev, _size in img_mount.loop_devices: + if dev is img_mount.mount_dev: + continue + try: + mount("/dev/mapper/"+dev, mnt=tmp_mount_dir) + if is_boot_part(tmp_mount_dir): + umount(tmp_mount_dir) + sysroot_boot_dir = joinpaths(root_dir, "boot") + mount("/dev/mapper/"+dev, mnt=sysroot_boot_dir) + break + else: + umount(tmp_mount_dir) + except subprocess.CalledProcessError as e: + log.debug("Looking for boot partition error: %s", e) + remove(tmp_mount_dir) + return sysroot_boot_dir + +def make_squashfs(opts, disk_img, work_dir): + """ + Create a squashfs image of an unpartitioned filesystem disk image + + :param str disk_img: Path to the unpartitioned filesystem disk image + :param str work_dir: Output compressed image to work_dir+images/install.img + :param str compression: Compression type to use + :returns: True if squashfs creation was successful. False if there was an error. + :rtype: bool + + Take disk_img and put it into LiveOS/rootfs.img and squashfs this + tree into work_dir+images/install.img + + fsck.ext4 is run on the disk image to make sure there are no errors and to zero + out any deleted blocks to make it compress better. If this fails for any reason + it will return False and log the error. + """ + # Make sure free blocks are actually zeroed so it will compress + rc = execWithRedirect("/usr/sbin/fsck.ext4", ["-y", "-f", "-E", "discard", disk_img]) + if rc != 0: + log.error("Problem zeroing free blocks of %s", disk_img) + return False + + liveos_dir = joinpaths(work_dir, "runtime/LiveOS") + os.makedirs(liveos_dir) + os.makedirs(os.path.dirname(joinpaths(work_dir, RUNTIME))) + + rc = execWithRedirect("/bin/ln", [disk_img, joinpaths(liveos_dir, "rootfs.img")]) + if rc != 0: + shutil.copy2(disk_img, joinpaths(liveos_dir, "rootfs.img")) + + compression, compressargs = squashfs_args(opts) + mksquashfs(joinpaths(work_dir, "runtime"), + joinpaths(work_dir, RUNTIME), compression, compressargs) + remove(joinpaths(work_dir, "runtime")) + return True + +def calculate_disk_size(opts, ks): + """ Calculate the disk size from the kickstart + + :param opts: options passed to livemedia-creator + :type opts: argparse options + :param str ks: Path to the kickstart to use for the installation + :returns: Disk size in MiB + :rtype: int + """ + # Disk size for a filesystem image should only be the size of / + # to prevent surprises when using the same kickstart for different installations. + unique_partitions = dict((p.mountpoint, p) for p in ks.handler.partition.partitions) + if opts.no_virt and (opts.make_iso or opts.make_fsimage): + disk_size = 2 + sum(p.size for p in unique_partitions.values() if p.mountpoint == "/") + else: + disk_size = 2 + sum(p.size for p in unique_partitions.values()) + log.info("Using disk size of %sMiB", disk_size) + return disk_size + +def make_image(opts, ks): + """ + Install to a disk image + + :param opts: options passed to livemedia-creator + :type opts: argparse options + :param str ks: Path to the kickstart to use for the installation + :returns: Path of the image created + :rtype: str + + Use qemu+boot.iso or anaconda to install to a disk image. + """ + if opts.image_name: + disk_img = joinpaths(opts.result_dir, opts.image_name) + else: + disk_img = tempfile.mktemp(prefix="lmc-disk-", suffix=".img", dir=opts.result_dir) + log.info("disk_img = %s", disk_img) + disk_size = calculate_disk_size(opts, ks) + try: + if opts.no_virt: + novirt_install(opts, disk_img, disk_size) + else: + install_log = os.path.abspath(os.path.dirname(opts.logfile))+"/virt-install.log" + log.info("install_log = %s", install_log) + + virt_install(opts, install_log, disk_img, disk_size) + except InstallError as e: + log.error("Install failed: %s", e) + if not opts.keep_image and os.path.exists(disk_img): + log.info("Removing bad disk image") + os.unlink(disk_img) + raise + + log.info("Disk Image install successful") + return disk_img + + +def make_live_images(opts, work_dir, disk_img): + """ + Create live images from direcory or rootfs image + + :param opts: options passed to livemedia-creator + :type opts: argparse options + :param str work_dir: Directory for storing results + :param str disk_img: Path to disk image (fsimage or partitioned) + :returns: Path of directory with created images or None + :rtype: str + + fsck.ext4 is run on the rootfs_image to make sure there are no errors and to zero + out any deleted blocks to make it compress better. If this fails for any reason + it will return None and log the error. + """ + sys_root = "" + + squashfs_root_dir = joinpaths(work_dir, "squashfs_root") + liveos_dir = joinpaths(squashfs_root_dir, "LiveOS") + os.makedirs(liveos_dir) + rootfs_img = joinpaths(liveos_dir, "rootfs.img") + + if opts.fs_image or opts.no_virt: + # Find the ostree root in the fsimage + if opts.ostree: + with Mount(disk_img, opts="loop") as mnt_dir: + sys_root = find_ostree_root(mnt_dir) + + # Try to hardlink the image, if that fails, copy it + rc = execWithRedirect("/bin/ln", [disk_img, rootfs_img]) + if rc != 0: + shutil.copy2(disk_img, rootfs_img) + else: + is_root_part = None + if opts.ostree: + is_root_part = lambda dir: os.path.exists(dir+"/ostree/deploy") + with PartitionMount(disk_img, mount_ok=is_root_part) as img_mount: + if img_mount and img_mount.mount_dir: + try: + mounted_sysroot_boot_dir = None + if opts.ostree: + sys_root = find_ostree_root(img_mount.mount_dir) + mounted_sysroot_boot_dir = mount_boot_part_over_root(img_mount) + if opts.live_rootfs_keep_size: + size = img_mount.mount_size / 1024**3 + else: + size = opts.live_rootfs_size or None + log.info("Creating live rootfs image") + mkrootfsimg(img_mount.mount_dir, rootfs_img, "LiveOS", size=size, sysroot=sys_root) + finally: + if mounted_sysroot_boot_dir: + umount(mounted_sysroot_boot_dir) + log.debug("sys_root = %s", sys_root) + + # Make sure free blocks are actually zeroed so it will compress + rc = execWithRedirect("/usr/sbin/fsck.ext4", ["-y", "-f", "-E", "discard", rootfs_img]) + if rc != 0: + log.error("Problem zeroing free blocks of %s", disk_img) + return None + + log.info("Packing live rootfs image") + add_pxe_args = [] + live_image_name = "live-rootfs.squashfs.img" + compression, compressargs = squashfs_args(opts) + mksquashfs(squashfs_root_dir, joinpaths(work_dir, live_image_name), compression, compressargs) + + log.info("Rebuilding initramfs for live") + with Mount(rootfs_img, opts="loop") as mnt_dir: + try: + mount(joinpaths(mnt_dir, "boot"), opts="bind", mnt=joinpaths(mnt_dir, sys_root, "boot")) + rebuild_initrds_for_live(opts, joinpaths(mnt_dir, sys_root), work_dir) + finally: + umount(joinpaths(mnt_dir, sys_root, "boot"), delete=False) + + remove(squashfs_root_dir) + + if opts.ostree: + add_pxe_args.append("ostree=/%s" % sys_root) + template = joinpaths(opts.lorax_templates, "pxe-live/pxe-config.tmpl") + create_pxe_config(template, work_dir, live_image_name, add_pxe_args) + + return work_dir + + diff --git a/src/pylorax/installer.py b/src/pylorax/installer.py new file mode 100644 index 00000000..9dccefcb --- /dev/null +++ b/src/pylorax/installer.py @@ -0,0 +1,569 @@ +# +# Copyright (C) 2011-2018 Red Hat, Inc. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +import logging +log = logging.getLogger("pylorax") + +import os +import tempfile +import subprocess +import shutil +import glob +import socket + +# Use the Lorax treebuilder branch for iso creation +from pylorax.creator import create_vagrant_metadata, update_vagrant_metadata, make_fsimage +from pylorax.executils import execWithRedirect, execReadlines +from pylorax.imgutils import PartitionMount, mksparse, mkext4img, loop_detach +from pylorax.imgutils import get_loop_name, dm_detach, mount, umount +from pylorax.imgutils import mkqemu_img, mktar +from pylorax.imgutils import mkcpio +from pylorax.monitor import LogMonitor +from pylorax.mount import IsoMountpoint +from pylorax.sysutils import joinpaths +from pylorax.treebuilder import udev_escape + + +ROOT_PATH = "/mnt/sysimage/" + +class InstallError(Exception): + pass + + +def find_free_port(start=5900, end=5999, host="127.0.0.1"): + """ Return first free port in range. + + :param int start: Starting port number + :param int end: Ending port number + :param str host: Host IP to search + :returns: First free port or -1 if none found + :rtype: int + """ + s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + for port in range(start, end+1): + try: + s.bind((host, port)) + s.close() + return port + except OSError: + pass + + return -1 + +def append_initrd(initrd, files): + """ Append files to an initrd. + + :param str initrd: Path to initrd + :param list files: list of file paths to add + :returns: Path to a new initrd + :rtype: str + + The files are added to the initrd by creating a cpio image + of the files (stored at /) and writing the cpio to the end of a + copy of the initrd. + + The initrd is not changed, a copy is made before appending the + cpio archive. + """ + qemu_initrd = tempfile.mktemp(prefix="lmc-initrd-", suffix=".img") + shutil.copy2(initrd, qemu_initrd) + ks_dir = tempfile.mkdtemp(prefix="lmc-ksdir-") + for ks in files: + shutil.copy2(ks, ks_dir) + ks_initrd = tempfile.mktemp(prefix="lmc-ks-", suffix=".img") + mkcpio(ks_dir, ks_initrd) + shutil.rmtree(ks_dir) + with open(qemu_initrd, "ab") as initrd_fp: + with open(ks_initrd, "rb") as ks_fp: + while True: + data = ks_fp.read(1024**2) + if not data: + break + initrd_fp.write(data) + os.unlink(ks_initrd) + + return qemu_initrd + +class QEMUInstall(object): + """ + Run qemu using an iso and a kickstart + """ + # Mapping of arch to qemu command + QEMU_CMDS = {"x86_64": "qemu-system-x86_64", + "i386": "qemu-system-i386", + "arm": "qemu-system-arm", + "aarch64": "qemu-system-aarch64", + "ppc": "qemu-system-ppc", + "ppc64": "qemu-system-ppc64" + } + + def __init__(self, opts, iso, ks_paths, disk_img, img_size=2048, + kernel_args=None, memory=1024, vcpus=None, vnc=None, arch=None, + log_check=None, virtio_host="127.0.0.1", virtio_port=6080, + image_type=None, boot_uefi=False, ovmf_path=None): + """ + Start the installation + + :param iso: Information about the iso to use for the installation + :type iso: IsoMountpoint + :param list ks_paths: Paths to kickstart files. All are injected, the + first one is the one executed. + :param str disk_img: Path to a disk image, created it it doesn't exist + :param int img_size: The image size, in MiB, to create if it doesn't exist + :param str kernel_args: Extra kernel arguments to pass on the kernel cmdline + :param int memory: Amount of RAM to assign to the virt, in MiB + :param int vcpus: Number of virtual cpus + :param str vnc: Arguments to pass to qemu -display + :param str arch: Optional architecture to use in the virt + :param log_check: Method that returns True if the installation fails + :type log_check: method + :param str virtio_host: Hostname to connect virtio log to + :param int virtio_port: Port to connect virtio log to + :param str image_type: Type of qemu-img disk to create, or None. + :param bool boot_uefi: Use OVMF to boot the VM in UEFI mode + :param str ovmf_path: Path to the OVMF firmware + """ + # Lookup qemu-system- for arch if passed, or try to guess using host arch + qemu_cmd = [self.QEMU_CMDS.get(arch or os.uname().machine, "qemu-system-"+os.uname().machine)] + if not os.path.exists("/usr/bin/"+qemu_cmd[0]): + raise InstallError("%s does not exist, cannot run qemu" % qemu_cmd[0]) + + qemu_cmd += ["-nodefconfig"] + qemu_cmd += ["-m", str(memory)] + if vcpus: + qemu_cmd += ["-smp", str(vcpus)] + + if not opts.no_kvm and os.path.exists("/dev/kvm"): + qemu_cmd += ["--machine", "accel=kvm"] + + # Copy the initrd from the iso, create a cpio archive of the kickstart files + # and append it to the temporary initrd. + qemu_initrd = append_initrd(iso.initrd, ks_paths) + qemu_cmd += ["-kernel", iso.kernel] + qemu_cmd += ["-initrd", qemu_initrd] + + # Add the disk and cdrom + if not os.path.isfile(disk_img): + mksparse(disk_img, img_size * 1024**2) + drive_args = "file=%s" % disk_img + drive_args += ",cache=unsafe,discard=unmap" + if image_type: + drive_args += ",format=%s" % image_type + else: + drive_args += ",format=raw" + qemu_cmd += ["-drive", drive_args] + + drive_args = "file=%s,media=cdrom,readonly=on" % iso.iso_path + qemu_cmd += ["-drive", drive_args] + + # Setup the cmdline args + # ====================== + cmdline_args = "ks=file:/%s" % os.path.basename(ks_paths[0]) + cmdline_args += " inst.stage2=hd:LABEL=%s" % udev_escape(iso.label) + if opts.proxy: + cmdline_args += " inst.proxy=%s" % opts.proxy + if kernel_args: + cmdline_args += " "+kernel_args + cmdline_args += " inst.text inst.cmdline" + + qemu_cmd += ["-append", cmdline_args] + + if not opts.vnc: + vnc_port = find_free_port() + if vnc_port == -1: + raise InstallError("No free VNC ports") + display_args = "vnc=127.0.0.1:%d" % (vnc_port - 5900) + else: + display_args = opts.vnc + log.info("qemu %s", display_args) + qemu_cmd += ["-nographic", "-display", display_args ] + + # Setup the virtio log port + qemu_cmd += ["-device", "virtio-serial-pci,id=virtio-serial0"] + qemu_cmd += ["-device", "virtserialport,bus=virtio-serial0.0,nr=1,chardev=charchannel0" + ",id=channel0,name=org.fedoraproject.anaconda.log.0"] + qemu_cmd += ["-chardev", "socket,id=charchannel0,host=%s,port=%s" % (virtio_host, virtio_port)] + + # PAss through rng from host + if opts.with_rng != "none": + qemu_cmd += ["-object", "rng-random,id=virtio-rng0,filename=%s" % opts.with_rng] + qemu_cmd += ["-device", "virtio-rng-pci,rng=virtio-rng0,id=rng0,bus=pci.0,addr=0x9"] + + if boot_uefi and ovmf_path: + qemu_cmd += ["-drive", "file=%s/OVMF_CODE.fd,if=pflash,format=raw,unit=0,readonly=on" % ovmf_path] + + # Make a copy of the OVMF_VARS.fd for this run + ovmf_vars = tempfile.mktemp(prefix="lmc-OVMF_VARS-", suffix=".fd") + shutil.copy2(joinpaths(ovmf_path, "/OVMF_VARS.fd"), ovmf_vars) + + qemu_cmd += ["-drive", "file=%s,if=pflash,format=raw,unit=1" % ovmf_vars] + + log.info("Running qemu") + log.debug(qemu_cmd) + try: + execWithRedirect(qemu_cmd[0], qemu_cmd[1:], reset_lang=False, raise_err=True, + callback=lambda p: not log_check()) + except subprocess.CalledProcessError as e: + log.error("Running qemu failed:") + log.error("cmd: %s", " ".join(e.cmd)) + log.error("output: %s", e.output or "") + raise InstallError("QEMUInstall failed") + except (OSError, KeyboardInterrupt) as e: + log.error("Running qemu failed: %s", str(e)) + raise InstallError("QEMUInstall failed") + finally: + os.unlink(qemu_initrd) + if boot_uefi and ovmf_path: + os.unlink(ovmf_vars) + + if log_check(): + log.error("Installation error detected. See logfile for details.") + raise InstallError("QEMUInstall failed") + else: + log.info("Installation finished without errors.") + + +def novirt_log_check(log_check, proc): + """ + Check to see if there has been an error in the logs + + :param log_check: method to call to check for an error in the logs + :param proc: Popen object for the anaconda process + :returns: True if the process has been terminated + + The log_check method should return a True if an error has been detected. + When an error is detected the process is terminated and this returns True + """ + if log_check(): + proc.terminate() + return True + return False + + +def anaconda_cleanup(dirinstall_path): + """ + Cleanup any leftover mounts from anaconda + + :param str dirinstall_path: Path where anaconda mounts things + :returns: True if cleanups were successful. False if any of them failed. + + If anaconda crashes it may leave things mounted under this path. It will + typically be set to /mnt/sysimage/ + + Attempts to cleanup may also fail. Catch these and continue trying the + other mountpoints. + """ + rc = True + dirinstall_path = os.path.abspath(dirinstall_path) + # unmount filesystems + for mounted in reversed(open("/proc/mounts").readlines()): + (_device, mountpoint, _rest) = mounted.split(" ", 2) + if mountpoint.startswith(dirinstall_path) and os.path.ismount(mountpoint): + try: + umount(mountpoint) + except subprocess.CalledProcessError: + log.error("Cleanup of %s failed. See program.log for details", mountpoint) + rc = False + return rc + + +def novirt_install(opts, disk_img, disk_size): + """ + Use Anaconda to install to a disk image + + :param opts: options passed to livemedia-creator + :type opts: argparse options + :param str disk_img: The full path to the disk image to be created + :param int disk_size: The size of the disk_img in MiB + + This method runs anaconda to create the image and then based on the opts + passed creates a qemu disk image or tarfile. + """ + dirinstall_path = ROOT_PATH + + # Clean up /tmp/ from previous runs to prevent stale info from being used + for path in ["/tmp/yum.repos.d/", "/tmp/yum.cache/"]: + if os.path.isdir(path): + shutil.rmtree(path) + + args = ["--kickstart", opts.ks[0], "--cmdline"] + if opts.anaconda_args: + for arg in opts.anaconda_args: + args += arg.split(" ", 1) + if opts.proxy: + args += ["--proxy", opts.proxy] + if opts.armplatform: + args += ["--armplatform", opts.armplatform] + + if opts.make_iso or opts.make_fsimage or opts.make_pxe_live: + # Make a blank fs image + args += ["--dirinstall"] + + mkext4img(None, disk_img, label=opts.fs_label, size=disk_size * 1024**2) + if not os.path.isdir(dirinstall_path): + os.mkdir(dirinstall_path) + mount(disk_img, opts="loop", mnt=dirinstall_path) + elif opts.make_tar or opts.make_oci: + # Install under dirinstall_path, make sure it starts clean + if os.path.exists(dirinstall_path): + shutil.rmtree(dirinstall_path) + + if opts.make_oci: + # OCI installs under /rootfs/ + dirinstall_path = joinpaths(dirinstall_path, "rootfs") + args += ["--dirinstall", dirinstall_path] + else: + args += ["--dirinstall"] + + os.makedirs(dirinstall_path) + else: + args += ["--image", disk_img] + + # Create the sparse image + mksparse(disk_img, disk_size * 1024**2) + + log_monitor = LogMonitor(timeout=opts.timeout) + args += ["--remotelog", "%s:%s" % (log_monitor.host, log_monitor.port)] + + # Make sure anaconda has the right product and release + log.info("Running anaconda.") + try: + for line in execReadlines("anaconda", args, reset_lang=False, + env_add={"ANACONDA_PRODUCTNAME": opts.project, + "ANACONDA_PRODUCTVERSION": opts.releasever}, + callback=lambda p: not novirt_log_check(log_monitor.server.log_check, p)): + log.info(line) + + # Make sure the new filesystem is correctly labeled + setfiles_args = ["-e", "/proc", "-e", "/sys", "-e", "/dev", + "/etc/selinux/targeted/contexts/files/file_contexts", "/"] + + # setfiles may not be available, warn instead of fail + try: + if "--dirinstall" in args: + execWithRedirect("setfiles", setfiles_args, root=dirinstall_path) + else: + with PartitionMount(disk_img) as img_mount: + if img_mount and img_mount.mount_dir: + execWithRedirect("setfiles", setfiles_args, root=img_mount.mount_dir) + except (subprocess.CalledProcessError, OSError) as e: + log.warning("Running setfiles on install tree failed: %s", str(e)) + + except (subprocess.CalledProcessError, OSError) as e: + log.error("Running anaconda failed: %s", e) + raise InstallError("novirt_install failed") + finally: + log_monitor.shutdown() + + # Move the anaconda logs over to a log directory + log_dir = os.path.abspath(os.path.dirname(opts.logfile)) + log_anaconda = joinpaths(log_dir, "anaconda") + if not os.path.isdir(log_anaconda): + os.mkdir(log_anaconda) + for l in glob.glob("/tmp/*log")+glob.glob("/tmp/anaconda-tb-*"): + shutil.copy2(l, log_anaconda) + os.unlink(l) + + # Make sure any leftover anaconda mounts have been cleaned up + if not anaconda_cleanup(dirinstall_path): + raise InstallError("novirt_install cleanup of anaconda mounts failed.") + + if not opts.make_iso and not opts.make_fsimage and not opts.make_pxe_live: + dm_name = os.path.splitext(os.path.basename(disk_img))[0] + dm_path = "/dev/mapper/"+dm_name + if os.path.exists(dm_path): + dm_detach(dm_path) + loop_detach(get_loop_name(disk_img)) + + # qemu disk image is used by bare qcow2 images and by Vagrant + if opts.image_type: + log.info("Converting %s to %s", disk_img, opts.image_type) + qemu_args = [] + for arg in opts.qemu_args: + qemu_args += arg.split(" ", 1) + + # convert the image to the selected format + if "-O" not in qemu_args: + qemu_args.extend(["-O", opts.image_type]) + qemu_img = tempfile.mktemp(prefix="lmc-disk-", suffix=".img") + execWithRedirect("qemu-img", ["convert"] + qemu_args + [disk_img, qemu_img], raise_err=True) + if not opts.make_vagrant: + execWithRedirect("mv", ["-f", qemu_img, disk_img], raise_err=True) + else: + # Take the new qcow2 image and package it up for Vagrant + compress_args = [] + for arg in opts.compress_args: + compress_args += arg.split(" ", 1) + + vagrant_dir = tempfile.mkdtemp(prefix="lmc-tmpdir-") + metadata_path = joinpaths(vagrant_dir, "metadata.json") + execWithRedirect("mv", ["-f", qemu_img, joinpaths(vagrant_dir, "box.img")], raise_err=True) + if opts.vagrant_metadata: + shutil.copy2(opts.vagrant_metadata, metadata_path) + else: + create_vagrant_metadata(metadata_path) + update_vagrant_metadata(metadata_path, disk_size) + if opts.vagrantfile: + shutil.copy2(opts.vagrantfile, joinpaths(vagrant_dir, "vagrantfile")) + + log.info("Creating Vagrant image") + rc = mktar(vagrant_dir, disk_img, opts.compression, compress_args, selinux=False) + if rc: + raise InstallError("novirt_install mktar failed: rc=%s" % rc) + shutil.rmtree(vagrant_dir) + elif opts.make_tar: + compress_args = [] + for arg in opts.compress_args: + compress_args += arg.split(" ", 1) + + rc = mktar(dirinstall_path, disk_img, opts.compression, compress_args) + shutil.rmtree(dirinstall_path) + + if rc: + raise InstallError("novirt_install mktar failed: rc=%s" % rc) + elif opts.make_oci: + # An OCI image places the filesystem under /rootfs/ and adds the json files at the top + # And then creates a tar of the whole thing. + compress_args = [] + for arg in opts.compress_args: + compress_args += arg.split(" ", 1) + + shutil.copy2(opts.oci_config, ROOT_PATH) + shutil.copy2(opts.oci_runtime, ROOT_PATH) + rc = mktar(ROOT_PATH, disk_img, opts.compression, compress_args) + + if rc: + raise InstallError("novirt_install mktar failed: rc=%s" % rc) + + +def virt_install(opts, install_log, disk_img, disk_size): + """ + Use qemu to install to a disk image + + :param opts: options passed to livemedia-creator + :type opts: argparse options + :param str install_log: The path to write the log from qemu + :param str disk_img: The full path to the disk image to be created + :param int disk_size: The size of the disk_img in MiB + + This uses qemu with a boot.iso and a kickstart to create a disk + image and then optionally, based on the opts passed, creates tarfile. + """ + iso_mount = IsoMountpoint(opts.iso, opts.location) + if not iso_mount.stage2: + iso_mount.umount() + raise InstallError("ISO is missing stage2, cannot continue") + + log_monitor = LogMonitor(install_log, timeout=opts.timeout) + + kernel_args = "" + if opts.kernel_args: + kernel_args += opts.kernel_args + if opts.proxy: + kernel_args += " proxy="+opts.proxy + + if opts.image_type and not opts.make_fsimage: + qemu_args = [] + for arg in opts.qemu_args: + qemu_args += arg.split(" ", 1) + if "-f" not in qemu_args: + qemu_args += ["-f", opts.image_type] + + mkqemu_img(disk_img, disk_size*1024**2, qemu_args) + + if opts.make_fsimage or opts.make_tar or opts.make_oci: + diskimg_path = tempfile.mktemp(prefix="lmc-disk-", suffix=".img") + else: + diskimg_path = disk_img + + try: + QEMUInstall(opts, iso_mount, opts.ks, diskimg_path, disk_size, + kernel_args, opts.ram, opts.vcpus, opts.vnc, opts.arch, + log_check = log_monitor.server.log_check, + virtio_host = log_monitor.host, + virtio_port = log_monitor.port, + image_type=opts.image_type, boot_uefi=opts.virt_uefi, + ovmf_path=opts.ovmf_path) + log_monitor.shutdown() + except InstallError as e: + log.error("VirtualInstall failed: %s", e) + raise + finally: + log.info("unmounting the iso") + iso_mount.umount() + + if log_monitor.server.log_check(): + if not log_monitor.server.error_line and opts.timeout: + msg = "virt_install failed due to timeout" + else: + msg = "virt_install failed on line: %s" % log_monitor.server.error_line + raise InstallError(msg) + + if opts.make_fsimage: + make_fsimage(diskimg_path, disk_img, disk_size, label=opts.fs_label) + os.unlink(diskimg_path) + elif opts.make_tar: + compress_args = [] + for arg in opts.compress_args: + compress_args += arg.split(" ", 1) + + with PartitionMount(diskimg_path) as img_mount: + if img_mount and img_mount.mount_dir: + rc = mktar(img_mount.mount_dir, disk_img, opts.compression, compress_args) + else: + rc = 1 + os.unlink(diskimg_path) + + if rc: + raise InstallError("virt_install failed") + elif opts.make_oci: + # An OCI image places the filesystem under /rootfs/ and adds the json files at the top + # And then creates a tar of the whole thing. + compress_args = [] + for arg in opts.compress_args: + compress_args += arg.split(" ", 1) + + with PartitionMount(diskimg_path, submount="rootfs") as img_mount: + if img_mount and img_mount.temp_dir: + shutil.copy2(opts.oci_config, img_mount.temp_dir) + shutil.copy2(opts.oci_runtime, img_mount.temp_dir) + rc = mktar(img_mount.temp_dir, disk_img, opts.compression, compress_args) + else: + rc = 1 + os.unlink(diskimg_path) + + if rc: + raise InstallError("virt_install failed") + elif opts.make_vagrant: + compress_args = [] + for arg in opts.compress_args: + compress_args += arg.split(" ", 1) + + vagrant_dir = tempfile.mkdtemp(prefix="lmc-tmpdir-") + metadata_path = joinpaths(vagrant_dir, "metadata.json") + execWithRedirect("mv", ["-f", disk_img, joinpaths(vagrant_dir, "box.img")], raise_err=True) + if opts.vagrant_metadata: + shutil.copy2(opts.vagrant_metadata, metadata_path) + else: + create_vagrant_metadata(metadata_path) + update_vagrant_metadata(metadata_path, disk_size) + if opts.vagrantfile: + shutil.copy2(opts.vagrantfile, joinpaths(vagrant_dir, "vagrantfile")) + + rc = mktar(vagrant_dir, disk_img, opts.compression, compress_args, selinux=False) + if rc: + raise InstallError("virt_install failed") + shutil.rmtree(vagrant_dir) diff --git a/src/sbin/livemedia-creator b/src/sbin/livemedia-creator index bee9c7b9..d3b90772 100755 --- a/src/sbin/livemedia-creator +++ b/src/sbin/livemedia-creator @@ -2,7 +2,7 @@ # # Live Media Creator # -# Copyright (C) 2011-2015 Red Hat, Inc. +# Copyright (C) 2011-2018 Red Hat, Inc. # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -17,21 +17,14 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . # -# Author(s): Brian C. Lane -# import logging log = logging.getLogger("livemedia-creator") import os import sys import tempfile -import subprocess import shutil -import hashlib import glob -import json -from math import ceil -import socket import selinux # Use pykickstart to calculate disk image size @@ -39,1144 +32,16 @@ from pykickstart.parser import KickstartParser from pykickstart.version import makeVersion from pykickstart.constants import KS_SHUTDOWN -# Use Mako templates for appliance builder descriptions -from mako.template import Template -from mako.exceptions import text_error_template - # Use the Lorax treebuilder branch for iso creation -from pylorax import ArchData, setup_logging, find_templates, vernum -from pylorax.base import DataHolder -from pylorax.treebuilder import TreeBuilder, RuntimeBuilder, udev_escape -from pylorax.treebuilder import findkernels -from pylorax.sysutils import joinpaths, remove -from pylorax.imgutils import PartitionMount, mksparse, mkext4img, loop_detach -from pylorax.imgutils import get_loop_name, dm_detach, mount, umount, Mount -from pylorax.imgutils import mksquashfs, mkqemu_img, mktar, mkrootfsimg -from pylorax.imgutils import copytree, mkcpio -from pylorax.executils import execWithRedirect, execReadlines, runcmd -from pylorax.monitor import LogMonitor -from pylorax.mount import IsoMountpoint +from pylorax import setup_logging, find_templates, vernum from pylorax.cmdline import lmc_parser - -# Default parameters for rebuilding initramfs, override with --dracut-args -DRACUT_DEFAULT = ["--xz", "--add", "livenet dmsquash-live convertfs pollcdrom qemu qemu-net", - "--omit", "plymouth", "--no-hostonly", "--debug", "--no-early-microcode"] - -ROOT_PATH = "/mnt/sysimage/" -RUNTIME = "images/install.img" - - -class InstallError(Exception): - pass - - -class FakeDNF(object): - """ - A minimal DNF object suitable for passing to RuntimeBuilder - - lmc uses RuntimeBuilder to run the arch specific iso creation - templates, so the the installroot config value is the important part of - this. Everything else should be a nop. - """ - def __init__(self, conf): - self.conf = conf - - def reset(self): - pass - - -def find_free_port(start=5900, end=5999, host="127.0.0.1"): - """ Return first free port in range. - - :param int start: Starting port number - :param int end: Ending port number - :param str host: Host IP to search - :returns: First free port or -1 if none found - :rtype: int - """ - s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - for port in range(start, end+1): - try: - s.bind((host, port)) - s.close() - return port - except OSError: - pass - - return -1 - - -def append_initrd(initrd, files): - """ Append files to an initrd. - - :param str initrd: Path to initrd - :param list files: list of file paths to add - :returns: Path to a new initrd - :rtype: str - - The files are added to the initrd by creating a cpio image - of the files (stored at /) and writing the cpio to the end of a - copy of the initrd. - - The initrd is not changed, a copy is made before appending the - cpio archive. - """ - qemu_initrd = tempfile.mktemp(prefix="lmc-initrd-", suffix=".img") - shutil.copy2(initrd, qemu_initrd) - ks_dir = tempfile.mkdtemp(prefix="lmc-ksdir-") - for ks in files: - shutil.copy2(ks, ks_dir) - ks_initrd = tempfile.mktemp(prefix="lmc-ks-", suffix=".img") - mkcpio(ks_dir, ks_initrd) - shutil.rmtree(ks_dir) - with open(qemu_initrd, "ab") as initrd_fp: - with open(ks_initrd, "rb") as ks_fp: - while True: - data = ks_fp.read(1024**2) - if not data: - break - initrd_fp.write(data) - os.unlink(ks_initrd) - - return qemu_initrd - - -class QEMUInstall(object): - """ - Run qemu using an iso and a kickstart - """ - # Mapping of arch to qemu command - QEMU_CMDS = {"x86_64": "qemu-system-x86_64", - "i386": "qemu-system-i386", - "arm": "qemu-system-arm", - "aarch64": "qemu-system-aarch64", - "ppc": "qemu-system-ppc", - "ppc64": "qemu-system-ppc64" - } - - def __init__(self, opts, iso, ks_paths, disk_img, img_size=2048, - kernel_args=None, memory=1024, vcpus=None, vnc=None, arch=None, - log_check=None, virtio_host="127.0.0.1", virtio_port=6080, - image_type=None, boot_uefi=False, ovmf_path=None): - """ - Start the installation - - :param iso: Information about the iso to use for the installation - :type iso: IsoMountpoint - :param list ks_paths: Paths to kickstart files. All are injected, the - first one is the one executed. - :param str disk_img: Path to a disk image, created it it doesn't exist - :param int img_size: The image size, in MiB, to create if it doesn't exist - :param str kernel_args: Extra kernel arguments to pass on the kernel cmdline - :param int memory: Amount of RAM to assign to the virt, in MiB - :param int vcpus: Number of virtual cpus - :param str vnc: Arguments to pass to qemu -display - :param str arch: Optional architecture to use in the virt - :param log_check: Method that returns True if the installation fails - :type log_check: method - :param str virtio_host: Hostname to connect virtio log to - :param int virtio_port: Port to connect virtio log to - :param str image_type: Type of qemu-img disk to create, or None. - :param bool boot_uefi: Use OVMF to boot the VM in UEFI mode - :param str ovmf_path: Path to the OVMF firmware - """ - # Lookup qemu-system- for arch if passed, or try to guess using host arch - qemu_cmd = [self.QEMU_CMDS.get(arch or os.uname().machine, "qemu-system-"+os.uname().machine)] - if not os.path.exists("/usr/bin/"+qemu_cmd[0]): - raise InstallError("%s does not exist, cannot run qemu" % qemu_cmd[0]) - - qemu_cmd += ["-nodefconfig"] - qemu_cmd += ["-m", str(memory)] - if vcpus: - qemu_cmd += ["-smp", str(vcpus)] - - if not opts.no_kvm and os.path.exists("/dev/kvm"): - qemu_cmd += ["--machine", "accel=kvm"] - - # Copy the initrd from the iso, create a cpio archive of the kickstart files - # and append it to the temporary initrd. - qemu_initrd = append_initrd(iso.initrd, ks_paths) - qemu_cmd += ["-kernel", iso.kernel] - qemu_cmd += ["-initrd", qemu_initrd] - - # Add the disk and cdrom - if not os.path.isfile(disk_img): - mksparse(disk_img, img_size * 1024**2) - drive_args = "file=%s" % disk_img - drive_args += ",cache=unsafe,discard=unmap" - if image_type: - drive_args += ",format=%s" % image_type - else: - drive_args += ",format=raw" - qemu_cmd += ["-drive", drive_args] - - drive_args = "file=%s,media=cdrom,readonly=on" % iso.iso_path - qemu_cmd += ["-drive", drive_args] - - # Setup the cmdline args - # ====================== - cmdline_args = "ks=file:/%s" % os.path.basename(ks_paths[0]) - cmdline_args += " inst.stage2=hd:LABEL=%s" % udev_escape(iso.label) - if opts.proxy: - cmdline_args += " inst.proxy=%s" % opts.proxy - if kernel_args: - cmdline_args += " "+kernel_args - cmdline_args += " inst.text inst.cmdline" - - qemu_cmd += ["-append", cmdline_args] - - if not opts.vnc: - vnc_port = find_free_port() - if vnc_port == -1: - raise InstallError("No free VNC ports") - display_args = "vnc=127.0.0.1:%d" % (vnc_port - 5900) - else: - display_args = opts.vnc - log.info("qemu %s", display_args) - qemu_cmd += ["-nographic", "-display", display_args ] - - # Setup the virtio log port - qemu_cmd += ["-device", "virtio-serial-pci,id=virtio-serial0"] - qemu_cmd += ["-device", "virtserialport,bus=virtio-serial0.0,nr=1,chardev=charchannel0" - ",id=channel0,name=org.fedoraproject.anaconda.log.0"] - qemu_cmd += ["-chardev", "socket,id=charchannel0,host=%s,port=%s" % (virtio_host, virtio_port)] - - # PAss through rng from host - if opts.with_rng != "none": - qemu_cmd += ["-object", "rng-random,id=virtio-rng0,filename=%s" % opts.with_rng] - qemu_cmd += ["-device", "virtio-rng-pci,rng=virtio-rng0,id=rng0,bus=pci.0,addr=0x9"] - - if boot_uefi and ovmf_path: - qemu_cmd += ["-drive", "file=%s/OVMF_CODE.fd,if=pflash,format=raw,unit=0,readonly=on" % ovmf_path] - - # Make a copy of the OVMF_VARS.fd for this run - ovmf_vars = tempfile.mktemp(prefix="lmc-OVMF_VARS-", suffix=".fd") - shutil.copy2(joinpaths(ovmf_path, "/OVMF_VARS.fd"), ovmf_vars) - - qemu_cmd += ["-drive", "file=%s,if=pflash,format=raw,unit=1" % ovmf_vars] - - log.info("Running qemu") - log.debug(qemu_cmd) - try: - execWithRedirect(qemu_cmd[0], qemu_cmd[1:], reset_lang=False, raise_err=True, - callback=lambda p: not log_check()) - except subprocess.CalledProcessError as e: - log.error("Running qemu failed:") - log.error("cmd: %s", " ".join(e.cmd)) - log.error("output: %s", e.output or "") - raise InstallError("QEMUInstall failed") - except (OSError, KeyboardInterrupt) as e: - log.error("Running qemu failed: %s", str(e)) - raise InstallError("QEMUInstall failed") - finally: - os.unlink(qemu_initrd) - if boot_uefi and ovmf_path: - os.unlink(ovmf_vars) - - if log_check(): - log.error("Installation error detected. See logfile for details.") - raise InstallError("QEMUInstall failed") - else: - log.info("Installation finished without errors.") - - -def is_image_mounted(disk_img): - """ - Check to see if the disk_img is mounted - - :returns: True if disk_img is in /proc/mounts - :rtype: bool - """ - with open("/proc/mounts") as mounts: - for mnt in mounts: - fields = mnt.split() - if len(fields) > 2 and fields[1] == disk_img: - return True - return False - -def find_ostree_root(phys_root): - """ - Find root of ostree deployment - - :param str phys_root: Path to physical root - :returns: Relative path of ostree deployment root - :rtype: str - :raise Exception: More than one deployment roots were found - """ - ostree_root = "" - ostree_sysroots = glob.glob(joinpaths(phys_root, "ostree/boot.?/*/*/0")) - log.debug("ostree_sysroots = %s", ostree_sysroots) - if ostree_sysroots: - if len(ostree_sysroots) > 1: - raise Exception("Too many deployment roots found: %s" % ostree_sysroots) - ostree_root = os.path.relpath(ostree_sysroots[0], phys_root) - return ostree_root - -def get_arch(mount_dir): - """ - Get the kernel arch - - :returns: Arch of first kernel found at mount_dir/boot/ or i386 - :rtype: str - """ - kernels = findkernels(mount_dir) - if not kernels: - return "i386" - return kernels[0].arch - -def squashfs_args(opts): - """ Returns the compression type and args to use when making squashfs - - :param opts: ArgumentParser object with compression and compressopts - :returns: tuple of compression type and args - :rtype: tuple - """ - compression = opts.compression or "xz" - arch = ArchData(opts.arch or os.uname().machine) - if compression == "xz" and arch.bcj: - compressargs = ["-Xbcj", arch.bcj] - else: - compressargs = [] - return (compression, compressargs) - - -def make_appliance(disk_img, name, template, outfile, networks=None, ram=1024, - vcpus=1, arch=None, title="Linux", project="Linux", - releasever="29"): - """ - Generate an appliance description file - - :param str disk_img: Full path of the disk image - :param str name: Name of the appliance, passed to the template - :param str template: Full path of Mako template - :param str outfile: Full path of file to write, using template - :param list networks: List of networks(str) from the kickstart - :param int ram: Ram, in MiB, passed to template. Default is 1024 - :param int vcpus: CPUs, passed to template. Default is 1 - :param str arch: CPU architecture. Default is 'x86_64' - :param str title: Title, passed to template. Default is 'Linux' - :param str project: Project, passed to template. Default is 'Linux' - :param str releasever: Release version, passed to template. Default is 29 - """ - if not (disk_img and template and outfile): - return None - - log.info("Creating appliance definition using %s", template) - - if not arch: - arch = "x86_64" - - log.info("Calculating SHA256 checksum of %s", disk_img) - sha256 = hashlib.sha256() - with open(disk_img) as f: - while True: - data = f.read(1024**2) - if not data: - break - sha256.update(data) - log.info("SHA256 of %s is %s", disk_img, sha256.hexdigest()) - disk_info = DataHolder(name=os.path.basename(disk_img), format="raw", - checksum_type="sha256", checksum=sha256.hexdigest()) - try: - result = Template(filename=template).render(disks=[disk_info], name=name, - arch=arch, memory=ram, vcpus=vcpus, networks=networks, - title=title, project=project, releasever=releasever) - except Exception: - log.error(text_error_template().render()) - raise - - with open(outfile, "w") as f: - f.write(result) - - -def make_fsimage(diskimage, fsimage, img_size=None, label="Anaconda"): - """ - Copy the / partition of a partitioned disk image to an un-partitioned - disk image. - - :param str diskimage: The full path to partitioned disk image with a / - :param str fsimage: The full path of the output fs image file - :param int img_size: Optional size of the fsimage in MiB or None to make - it as small as possible - :param str label: The label to apply to the image. Defaults to "Anaconda" - """ - with PartitionMount(diskimage) as img_mount: - if not img_mount or not img_mount.mount_dir: - return None - - log.info("Creating fsimage %s (%s)", fsimage, img_size or "minimized") - if img_size: - # convert to Bytes - img_size *= 1024**2 - - mkext4img(img_mount.mount_dir, fsimage, size=img_size, label=label) - - -def make_runtime(opts, mount_dir, work_dir, size=None): - """ - Make the squashfs image from a directory - - :param opts: options passed to livemedia-creator - :type opts: argparse options - :param str mount_dir: Directory tree to compress - :param str work_dir: Output compressed image to work_dir+images/install.img - :param int size: Size of disk image, in GiB - """ - kernel_arch = get_arch(mount_dir) - - # Fake dnf object - fake_dbo = FakeDNF(conf=DataHolder(installroot=mount_dir)) - # Fake arch with only basearch set - arch = ArchData(kernel_arch) - # TODO: Need to get release info from someplace... - product = DataHolder(name=opts.project, version=opts.releasever, release="", - variant="", bugurl="", isfinal=False) - - # This is a mounted image partition, cannot hardlink to it, so just use it - # symlink mount_dir/images to work_dir/images so we don't run out of space - os.makedirs(joinpaths(work_dir, "images")) - - rb = RuntimeBuilder(product, arch, fake_dbo) - compression, compressargs = squashfs_args(opts) - log.info("Creating runtime") - rb.create_runtime(joinpaths(work_dir, RUNTIME), size=size, - compression=compression, compressargs=compressargs) - -def rebuild_initrds_for_live(opts, sys_root_dir, results_dir): - """ - Rebuild intrds for pxe live image (root=live:http://) - - :param opts: options passed to livemedia-creator - :type opts: argparse options - :param str sys_root_dir: Path to root of the system - :param str results_dir: Path of directory for storing results - """ - if not opts.dracut_args: - dracut_args = DRACUT_DEFAULT - else: - dracut_args = [] - for arg in opts.dracut_args: - dracut_args += arg.split(" ", 1) - log.info("dracut args = %s", dracut_args) - - dracut = ["dracut", "--nomdadmconf", "--nolvmconf"] + dracut_args - - kdir = "boot" - if opts.ostree: - kernels_dir = glob.glob(joinpaths(sys_root_dir, "boot/ostree/*")) - if kernels_dir: - kdir = os.path.relpath(kernels_dir[0], sys_root_dir) - - kernels = [kernel for kernel in findkernels(sys_root_dir, kdir)] - if not kernels: - raise Exception("No initrds found, cannot rebuild_initrds") - - # Hush some dracut warnings. TODO: bind-mount proc in place? - open(joinpaths(sys_root_dir,"/proc/modules"),"w") - - if opts.ostree: - # Dracut assumes to have some dirs in disk image - # /var/tmp for temp files - vartmp_dir = joinpaths(sys_root_dir, "var/tmp") - if not os.path.isdir(vartmp_dir): - os.mkdir(vartmp_dir) - # /root (maybe not fatal) - root_dir = joinpaths(sys_root_dir, "var/roothome") - if not os.path.isdir(root_dir): - os.mkdir(root_dir) - # /tmp (maybe not fatal) - tmp_dir = joinpaths(sys_root_dir, "sysroot/tmp") - if not os.path.isdir(tmp_dir): - os.mkdir(tmp_dir) - - # Write the new initramfs directly to the results directory - os.mkdir(joinpaths(sys_root_dir, "results")) - mount(results_dir, opts="bind", mnt=joinpaths(sys_root_dir, "results")) - # Dracut runs out of space inside the minimal rootfs image - mount("/var/tmp", opts="bind", mnt=joinpaths(sys_root_dir, "var/tmp")) - for kernel in kernels: - if hasattr(kernel, "initrd"): - outfile = os.path.basename(kernel.initrd.path) - else: - # Construct an initrd from the kernel name - outfile = os.path.basename(kernel.path.replace("vmlinuz-", "initrd-") + ".img") - log.info("rebuilding %s", outfile) - - kver = kernel.version - - cmd = dracut + ["/results/"+outfile, kver] - runcmd(cmd, root=sys_root_dir) - - shutil.copy2(joinpaths(sys_root_dir, kernel.path), results_dir) - umount(joinpaths(sys_root_dir, "var/tmp"), delete=False) - umount(joinpaths(sys_root_dir, "results"), delete=False) - os.unlink(joinpaths(sys_root_dir,"/proc/modules")) - -def create_pxe_config(template, images_dir, live_image_name, add_args = None): - """ - Create template for pxe to live configuration - - :param str images_dir: Path of directory with images to be used - :param str live_image_name: Name of live rootfs image file - :param list add_args: Arguments to be added to initrd= pxe config - """ - - add_args = add_args or [] - - kernels = [kernel for kernel in findkernels(images_dir, kdir="") - if hasattr(kernel, "initrd")] - if not kernels: - return - - kernel = kernels[0] - - add_args_str = " ".join(add_args) - - - try: - result = Template(filename=template).render(kernel=kernel.path, - initrd=kernel.initrd.path, liveimg=live_image_name, - addargs=add_args_str) - except Exception: - log.error(text_error_template().render()) - raise - - with open (joinpaths(images_dir, "PXE_CONFIG"), "w") as f: - f.write(result) - - -def create_vagrant_metadata(path, size=0): - """ Create a default Vagrant metadata.json file - - :param str path: Path to metadata.json file - :param int size: Disk size in MiB - """ - metadata = { "provider":"libvirt", "format":"qcow2", "virtual_size": ceil(size / 1024) } - with open(path, "wt") as f: - json.dump(metadata, f, indent=4) - - -def update_vagrant_metadata(path, size): - """ Update the Vagrant metadata.json file - - :param str path: Path to metadata.json file - :param int size: Disk size in MiB - - This function makes sure that the provider, format and virtual size of the - metadata file are set correctly. All other values are left untouched. - """ - with open(path, "rt") as f: - try: - metadata = json.load(f) - except ValueError as e: - log.error("Problem reading metadata file %s: %s", path, e) - return - - metadata["provider"] = "libvirt" - metadata["format"] = "qcow2" - metadata["virtual_size"] = ceil(size / 1024) - with open(path, "wt") as f: - json.dump(metadata, f, indent=4) - - -def make_livecd(opts, mount_dir, work_dir): - """ - Take the content from the disk image and make a livecd out of it - - :param opts: options passed to livemedia-creator - :type opts: argparse options - :param str mount_dir: Directory tree to compress - :param str work_dir: Output compressed image to work_dir+images/install.img - - This uses wwood's squashfs live initramfs method: - * put the real / into LiveOS/rootfs.img - * make a squashfs of the LiveOS/rootfs.img tree - * This is loaded by dracut when the cmdline is passed to the kernel: - root=live:CDLABEL= rd.live.image - """ - kernel_arch = get_arch(mount_dir) - - arch = ArchData(kernel_arch) - # TODO: Need to get release info from someplace... - product = DataHolder(name=opts.project, version=opts.releasever, release="", - variant="", bugurl="", isfinal=False) - - # Link /images to work_dir/images to make the templates happy - if os.path.islink(joinpaths(mount_dir, "images")): - os.unlink(joinpaths(mount_dir, "images")) - execWithRedirect("/bin/ln", ["-s", joinpaths(work_dir, "images"), - joinpaths(mount_dir, "images")]) - - # The templates expect the config files to be in /tmp/config_files - # I think these should be release specific, not from lorax, but for now - configdir = joinpaths(opts.lorax_templates,"live/config_files/") - configdir_path = "tmp/config_files" - fullpath = joinpaths(mount_dir, configdir_path) - if os.path.exists(fullpath): - remove(fullpath) - copytree(configdir, fullpath) - - isolabel = opts.volid or "{0.name}-{0.version}-{1.basearch}".format(product, arch) - if len(isolabel) > 32: - isolabel = isolabel[:32] - log.warning("Truncating isolabel to 32 chars: %s", isolabel) - - tb = TreeBuilder(product=product, arch=arch, domacboot=opts.domacboot, - inroot=mount_dir, outroot=work_dir, - runtime=RUNTIME, isolabel=isolabel, - templatedir=joinpaths(opts.lorax_templates,"live/")) - log.info("Rebuilding initrds") - if not opts.dracut_args: - dracut_args = DRACUT_DEFAULT - else: - dracut_args = [] - for arg in opts.dracut_args: - dracut_args += arg.split(" ", 1) - log.info("dracut args = %s", dracut_args) - tb.rebuild_initrds(add_args=dracut_args) - log.info("Building boot.iso") - tb.build() - - return work_dir - -def mount_boot_part_over_root(img_mount): - """ - Mount boot partition to /boot of root fs mounted in img_mount - - Used for OSTree so it finds deployment configurations on live rootfs - - param img_mount: object with mounted disk image root partition - type img_mount: imgutils.PartitionMount - """ - root_dir = img_mount.mount_dir - is_boot_part = lambda dir: os.path.exists(dir+"/loader.0") - tmp_mount_dir = tempfile.mkdtemp(prefix="lmc-tmpdir-") - sysroot_boot_dir = None - for dev, _size in img_mount.loop_devices: - if dev is img_mount.mount_dev: - continue - try: - mount("/dev/mapper/"+dev, mnt=tmp_mount_dir) - if is_boot_part(tmp_mount_dir): - umount(tmp_mount_dir) - sysroot_boot_dir = joinpaths(root_dir, "boot") - mount("/dev/mapper/"+dev, mnt=sysroot_boot_dir) - break - else: - umount(tmp_mount_dir) - except subprocess.CalledProcessError as e: - log.debug("Looking for boot partition error: %s", e) - remove(tmp_mount_dir) - return sysroot_boot_dir - -def novirt_log_check(log_check, proc): - """ - Check to see if there has been an error in the logs - - :param log_check: method to call to check for an error in the logs - :param proc: Popen object for the anaconda process - :returns: True if the process has been terminated - - The log_check method should return a True if an error has been detected. - When an error is detected the process is terminated and this returns True - """ - if log_check(): - proc.terminate() - return True - return False - - -def anaconda_cleanup(dirinstall_path): - """ - Cleanup any leftover mounts from anaconda - - :param str dirinstall_path: Path where anaconda mounts things - :returns: True if cleanups were successful. False if any of them failed. - - If anaconda crashes it may leave things mounted under this path. It will - typically be set to /mnt/sysimage/ - - Attempts to cleanup may also fail. Catch these and continue trying the - other mountpoints. - """ - rc = True - dirinstall_path = os.path.abspath(dirinstall_path) - # unmount filesystems - for mounted in reversed(open("/proc/mounts").readlines()): - (_device, mountpoint, _rest) = mounted.split(" ", 2) - if mountpoint.startswith(dirinstall_path) and os.path.ismount(mountpoint): - try: - umount(mountpoint) - except subprocess.CalledProcessError: - log.error("Cleanup of %s failed. See program.log for details", mountpoint) - rc = False - return rc - - -def novirt_install(opts, disk_img, disk_size): - """ - Use Anaconda to install to a disk image - - :param opts: options passed to livemedia-creator - :type opts: argparse options - :param str disk_img: The full path to the disk image to be created - :param int disk_size: The size of the disk_img in MiB - - This method runs anaconda to create the image and then based on the opts - passed creates a qemu disk image or tarfile. - """ - dirinstall_path = ROOT_PATH - - # Clean up /tmp/ from previous runs to prevent stale info from being used - for path in ["/tmp/yum.repos.d/", "/tmp/yum.cache/"]: - if os.path.isdir(path): - shutil.rmtree(path) - - args = ["--kickstart", opts.ks[0], "--cmdline"] - if opts.anaconda_args: - for arg in opts.anaconda_args: - args += arg.split(" ", 1) - if opts.proxy: - args += ["--proxy", opts.proxy] - if opts.armplatform: - args += ["--armplatform", opts.armplatform] - - if opts.make_iso or opts.make_fsimage or opts.make_pxe_live: - # Make a blank fs image - args += ["--dirinstall"] - - mkext4img(None, disk_img, label=opts.fs_label, size=disk_size * 1024**2) - if not os.path.isdir(dirinstall_path): - os.mkdir(dirinstall_path) - mount(disk_img, opts="loop", mnt=dirinstall_path) - elif opts.make_tar or opts.make_oci: - # Install under dirinstall_path, make sure it starts clean - if os.path.exists(dirinstall_path): - shutil.rmtree(dirinstall_path) - - if opts.make_oci: - # OCI installs under /rootfs/ - dirinstall_path = joinpaths(dirinstall_path, "rootfs") - args += ["--dirinstall", dirinstall_path] - else: - args += ["--dirinstall"] - - os.makedirs(dirinstall_path) - else: - args += ["--image", disk_img] - - # Create the sparse image - mksparse(disk_img, disk_size * 1024**2) - - log_monitor = LogMonitor(timeout=opts.timeout) - args += ["--remotelog", "%s:%s" % (log_monitor.host, log_monitor.port)] - - # Make sure anaconda has the right product and release - log.info("Running anaconda.") - try: - for line in execReadlines("anaconda", args, reset_lang=False, - env_add={"ANACONDA_PRODUCTNAME": opts.project, - "ANACONDA_PRODUCTVERSION": opts.releasever}, - callback=lambda p: not novirt_log_check(log_monitor.server.log_check, p)): - log.info(line) - - # Make sure the new filesystem is correctly labeled - setfiles_args = ["-e", "/proc", "-e", "/sys", "-e", "/dev", - "/etc/selinux/targeted/contexts/files/file_contexts", "/"] - - # setfiles may not be available, warn instead of fail - try: - if "--dirinstall" in args: - execWithRedirect("setfiles", setfiles_args, root=dirinstall_path) - else: - with PartitionMount(disk_img) as img_mount: - if img_mount and img_mount.mount_dir: - execWithRedirect("setfiles", setfiles_args, root=img_mount.mount_dir) - except (subprocess.CalledProcessError, OSError) as e: - log.warning("Running setfiles on install tree failed: %s", str(e)) - - except (subprocess.CalledProcessError, OSError) as e: - log.error("Running anaconda failed: %s", e) - raise InstallError("novirt_install failed") - finally: - log_monitor.shutdown() - - # Move the anaconda logs over to a log directory - log_dir = os.path.abspath(os.path.dirname(opts.logfile)) - log_anaconda = joinpaths(log_dir, "anaconda") - if not os.path.isdir(log_anaconda): - os.mkdir(log_anaconda) - for l in glob.glob("/tmp/*log")+glob.glob("/tmp/anaconda-tb-*"): - shutil.copy2(l, log_anaconda) - os.unlink(l) - - # Make sure any leftover anaconda mounts have been cleaned up - if not anaconda_cleanup(dirinstall_path): - raise InstallError("novirt_install cleanup of anaconda mounts failed.") - - if not opts.make_iso and not opts.make_fsimage and not opts.make_pxe_live: - dm_name = os.path.splitext(os.path.basename(disk_img))[0] - dm_path = "/dev/mapper/"+dm_name - if os.path.exists(dm_path): - dm_detach(dm_path) - loop_detach(get_loop_name(disk_img)) - - # qemu disk image is used by bare qcow2 images and by Vagrant - if opts.image_type: - log.info("Converting %s to %s", disk_img, opts.image_type) - qemu_args = [] - for arg in opts.qemu_args: - qemu_args += arg.split(" ", 1) - - # convert the image to the selected format - if "-O" not in qemu_args: - qemu_args.extend(["-O", opts.image_type]) - qemu_img = tempfile.mktemp(prefix="lmc-disk-", suffix=".img") - execWithRedirect("qemu-img", ["convert"] + qemu_args + [disk_img, qemu_img], raise_err=True) - if not opts.make_vagrant: - execWithRedirect("mv", ["-f", qemu_img, disk_img], raise_err=True) - else: - # Take the new qcow2 image and package it up for Vagrant - compress_args = [] - for arg in opts.compress_args: - compress_args += arg.split(" ", 1) - - vagrant_dir = tempfile.mkdtemp(prefix="lmc-tmpdir-") - metadata_path = joinpaths(vagrant_dir, "metadata.json") - execWithRedirect("mv", ["-f", qemu_img, joinpaths(vagrant_dir, "box.img")], raise_err=True) - if opts.vagrant_metadata: - shutil.copy2(opts.vagrant_metadata, metadata_path) - else: - create_vagrant_metadata(metadata_path) - update_vagrant_metadata(metadata_path, disk_size) - if opts.vagrantfile: - shutil.copy2(opts.vagrantfile, joinpaths(vagrant_dir, "vagrantfile")) - - log.info("Creating Vagrant image") - rc = mktar(vagrant_dir, disk_img, opts.compression, compress_args, selinux=False) - if rc: - raise InstallError("novirt_install mktar failed: rc=%s" % rc) - shutil.rmtree(vagrant_dir) - elif opts.make_tar: - compress_args = [] - for arg in opts.compress_args: - compress_args += arg.split(" ", 1) - - rc = mktar(dirinstall_path, disk_img, opts.compression, compress_args) - shutil.rmtree(dirinstall_path) - - if rc: - raise InstallError("novirt_install mktar failed: rc=%s" % rc) - elif opts.make_oci: - # An OCI image places the filesystem under /rootfs/ and adds the json files at the top - # And then creates a tar of the whole thing. - compress_args = [] - for arg in opts.compress_args: - compress_args += arg.split(" ", 1) - - shutil.copy2(opts.oci_config, ROOT_PATH) - shutil.copy2(opts.oci_runtime, ROOT_PATH) - rc = mktar(ROOT_PATH, disk_img, opts.compression, compress_args) - - if rc: - raise InstallError("novirt_install mktar failed: rc=%s" % rc) - - -def virt_install(opts, install_log, disk_img, disk_size): - """ - Use qemu to install to a disk image - - :param opts: options passed to livemedia-creator - :type opts: argparse options - :param str install_log: The path to write the log from qemu - :param str disk_img: The full path to the disk image to be created - :param int disk_size: The size of the disk_img in MiB - - This uses qemu with a boot.iso and a kickstart to create a disk - image and then optionally, based on the opts passed, creates tarfile. - """ - iso_mount = IsoMountpoint(opts.iso, opts.location) - if not iso_mount.stage2: - iso_mount.umount() - raise InstallError("ISO is missing stage2, cannot continue") - - log_monitor = LogMonitor(install_log, timeout=opts.timeout) - - kernel_args = "" - if opts.kernel_args: - kernel_args += opts.kernel_args - if opts.proxy: - kernel_args += " proxy="+opts.proxy - - if opts.image_type and not opts.make_fsimage: - qemu_args = [] - for arg in opts.qemu_args: - qemu_args += arg.split(" ", 1) - if "-f" not in qemu_args: - qemu_args += ["-f", opts.image_type] - - mkqemu_img(disk_img, disk_size*1024**2, qemu_args) - - if opts.make_fsimage or opts.make_tar or opts.make_oci: - diskimg_path = tempfile.mktemp(prefix="lmc-disk-", suffix=".img") - else: - diskimg_path = disk_img - - try: - QEMUInstall(opts, iso_mount, opts.ks, diskimg_path, disk_size, - kernel_args, opts.ram, opts.vcpus, opts.vnc, opts.arch, - log_check = log_monitor.server.log_check, - virtio_host = log_monitor.host, - virtio_port = log_monitor.port, - image_type=opts.image_type, boot_uefi=opts.virt_uefi, - ovmf_path=opts.ovmf_path) - log_monitor.shutdown() - except InstallError as e: - log.error("VirtualInstall failed: %s", e) - raise - finally: - log.info("unmounting the iso") - iso_mount.umount() - - if log_monitor.server.log_check(): - if not log_monitor.server.error_line and opts.timeout: - msg = "virt_install failed due to timeout" - else: - msg = "virt_install failed on line: %s" % log_monitor.server.error_line - raise InstallError(msg) - - if opts.make_fsimage: - make_fsimage(diskimg_path, disk_img, disk_size, label=opts.fs_label) - os.unlink(diskimg_path) - elif opts.make_tar: - compress_args = [] - for arg in opts.compress_args: - compress_args += arg.split(" ", 1) - - with PartitionMount(diskimg_path) as img_mount: - if img_mount and img_mount.mount_dir: - rc = mktar(img_mount.mount_dir, disk_img, opts.compression, compress_args) - else: - rc = 1 - os.unlink(diskimg_path) - - if rc: - raise InstallError("virt_install failed") - elif opts.make_oci: - # An OCI image places the filesystem under /rootfs/ and adds the json files at the top - # And then creates a tar of the whole thing. - compress_args = [] - for arg in opts.compress_args: - compress_args += arg.split(" ", 1) - - with PartitionMount(diskimg_path, submount="rootfs") as img_mount: - if img_mount and img_mount.temp_dir: - shutil.copy2(opts.oci_config, img_mount.temp_dir) - shutil.copy2(opts.oci_runtime, img_mount.temp_dir) - rc = mktar(img_mount.temp_dir, disk_img, opts.compression, compress_args) - else: - rc = 1 - os.unlink(diskimg_path) - - if rc: - raise InstallError("virt_install failed") - elif opts.make_vagrant: - compress_args = [] - for arg in opts.compress_args: - compress_args += arg.split(" ", 1) - - vagrant_dir = tempfile.mkdtemp(prefix="lmc-tmpdir-") - metadata_path = joinpaths(vagrant_dir, "metadata.json") - execWithRedirect("mv", ["-f", disk_img, joinpaths(vagrant_dir, "box.img")], raise_err=True) - if opts.vagrant_metadata: - shutil.copy2(opts.vagrant_metadata, metadata_path) - else: - create_vagrant_metadata(metadata_path) - update_vagrant_metadata(metadata_path, disk_size) - if opts.vagrantfile: - shutil.copy2(opts.vagrantfile, joinpaths(vagrant_dir, "vagrantfile")) - - rc = mktar(vagrant_dir, disk_img, opts.compression, compress_args, selinux=False) - if rc: - raise InstallError("virt_install failed") - shutil.rmtree(vagrant_dir) - - -def make_squashfs(opts, disk_img, work_dir): - """ - Create a squashfs image of an unpartitioned filesystem disk image - - :param str disk_img: Path to the unpartitioned filesystem disk image - :param str work_dir: Output compressed image to work_dir+images/install.img - :param str compression: Compression type to use - :returns: True if squashfs creation was successful. False if there was an error. - :rtype: bool - - Take disk_img and put it into LiveOS/rootfs.img and squashfs this - tree into work_dir+images/install.img - - fsck.ext4 is run on the disk image to make sure there are no errors and to zero - out any deleted blocks to make it compress better. If this fails for any reason - it will return False and log the error. - """ - # Make sure free blocks are actually zeroed so it will compress - rc = execWithRedirect("/usr/sbin/fsck.ext4", ["-y", "-f", "-E", "discard", disk_img]) - if rc != 0: - log.error("Problem zeroing free blocks of %s", disk_img) - return False - - liveos_dir = joinpaths(work_dir, "runtime/LiveOS") - os.makedirs(liveos_dir) - os.makedirs(os.path.dirname(joinpaths(work_dir, RUNTIME))) - - rc = execWithRedirect("/bin/ln", [disk_img, joinpaths(liveos_dir, "rootfs.img")]) - if rc != 0: - shutil.copy2(disk_img, joinpaths(liveos_dir, "rootfs.img")) - - compression, compressargs = squashfs_args(opts) - mksquashfs(joinpaths(work_dir, "runtime"), - joinpaths(work_dir, RUNTIME), compression, compressargs) - remove(joinpaths(work_dir, "runtime")) - return True - -def calculate_disk_size(opts, ks): - """ Calculate the disk size from the kickstart - - :param opts: options passed to livemedia-creator - :type opts: argparse options - :param str ks: Path to the kickstart to use for the installation - :returns: Disk size in MiB - :rtype: int - """ - # Disk size for a filesystem image should only be the size of / - # to prevent surprises when using the same kickstart for different installations. - unique_partitions = dict((p.mountpoint, p) for p in ks.handler.partition.partitions) - if opts.no_virt and (opts.make_iso or opts.make_fsimage): - disk_size = 2 + sum(p.size for p in unique_partitions.values() if p.mountpoint == "/") - else: - disk_size = 2 + sum(p.size for p in unique_partitions.values()) - log.info("Using disk size of %sMiB", disk_size) - return disk_size - -def make_image(opts, ks): - """ - Install to a disk image - - :param opts: options passed to livemedia-creator - :type opts: argparse options - :param str ks: Path to the kickstart to use for the installation - :returns: Path of the image created - :rtype: str - - Use qemu+boot.iso or anaconda to install to a disk image. - """ - if opts.image_name: - disk_img = joinpaths(opts.result_dir, opts.image_name) - else: - disk_img = tempfile.mktemp(prefix="lmc-disk-", suffix=".img", dir=opts.result_dir) - log.info("disk_img = %s", disk_img) - disk_size = calculate_disk_size(opts, ks) - try: - if opts.no_virt: - novirt_install(opts, disk_img, disk_size) - else: - install_log = os.path.abspath(os.path.dirname(opts.logfile))+"/virt-install.log" - log.info("install_log = %s", install_log) - - virt_install(opts, install_log, disk_img, disk_size) - except InstallError as e: - log.error("Install failed: %s", e) - if not opts.keep_image and os.path.exists(disk_img): - log.info("Removing bad disk image") - os.unlink(disk_img) - raise - - log.info("Disk Image install successful") - return disk_img - - -def make_live_images(opts, work_dir, disk_img): - """ - Create live images from direcory or rootfs image - - :param opts: options passed to livemedia-creator - :type opts: argparse options - :param str work_dir: Directory for storing results - :param str disk_img: Path to disk image (fsimage or partitioned) - :returns: Path of directory with created images or None - :rtype: str - - fsck.ext4 is run on the rootfs_image to make sure there are no errors and to zero - out any deleted blocks to make it compress better. If this fails for any reason - it will return None and log the error. - """ - sys_root = "" - - squashfs_root_dir = joinpaths(work_dir, "squashfs_root") - liveos_dir = joinpaths(squashfs_root_dir, "LiveOS") - os.makedirs(liveos_dir) - rootfs_img = joinpaths(liveos_dir, "rootfs.img") - - if opts.fs_image or opts.no_virt: - # Find the ostree root in the fsimage - if opts.ostree: - with Mount(disk_img, opts="loop") as mnt_dir: - sys_root = find_ostree_root(mnt_dir) - - # Try to hardlink the image, if that fails, copy it - rc = execWithRedirect("/bin/ln", [disk_img, rootfs_img]) - if rc != 0: - shutil.copy2(disk_img, rootfs_img) - else: - is_root_part = None - if opts.ostree: - is_root_part = lambda dir: os.path.exists(dir+"/ostree/deploy") - with PartitionMount(disk_img, mount_ok=is_root_part) as img_mount: - if img_mount and img_mount.mount_dir: - try: - mounted_sysroot_boot_dir = None - if opts.ostree: - sys_root = find_ostree_root(img_mount.mount_dir) - mounted_sysroot_boot_dir = mount_boot_part_over_root(img_mount) - if opts.live_rootfs_keep_size: - size = img_mount.mount_size / 1024**3 - else: - size = opts.live_rootfs_size or None - log.info("Creating live rootfs image") - mkrootfsimg(img_mount.mount_dir, rootfs_img, "LiveOS", size=size, sysroot=sys_root) - finally: - if mounted_sysroot_boot_dir: - umount(mounted_sysroot_boot_dir) - log.debug("sys_root = %s", sys_root) - - # Make sure free blocks are actually zeroed so it will compress - rc = execWithRedirect("/usr/sbin/fsck.ext4", ["-y", "-f", "-E", "discard", rootfs_img]) - if rc != 0: - log.error("Problem zeroing free blocks of %s", disk_img) - return None - - log.info("Packing live rootfs image") - add_pxe_args = [] - live_image_name = "live-rootfs.squashfs.img" - compression, compressargs = squashfs_args(opts) - mksquashfs(squashfs_root_dir, joinpaths(work_dir, live_image_name), compression, compressargs) - - log.info("Rebuilding initramfs for live") - with Mount(rootfs_img, opts="loop") as mnt_dir: - try: - mount(joinpaths(mnt_dir, "boot"), opts="bind", mnt=joinpaths(mnt_dir, sys_root, "boot")) - rebuild_initrds_for_live(opts, joinpaths(mnt_dir, sys_root), work_dir) - finally: - umount(joinpaths(mnt_dir, sys_root, "boot"), delete=False) - - remove(squashfs_root_dir) - - if opts.ostree: - add_pxe_args.append("ostree=/%s" % sys_root) - template = joinpaths(opts.lorax_templates, "pxe-live/pxe-config.tmpl") - create_pxe_config(template, work_dir, live_image_name, add_pxe_args) - - return work_dir +from pylorax.creator import make_image, make_squashfs, make_livecd, make_runtime, make_appliance, make_live_images +from pylorax.creator import calculate_disk_size +from pylorax.imgutils import PartitionMount +from pylorax.imgutils import Mount +from pylorax.imgutils import copytree +from pylorax.installer import InstallError +from pylorax.sysutils import joinpaths def default_image_name(compression, basename):