diff --git a/src/pylorax/creator.py b/src/pylorax/creator.py new file mode 100644 index 00000000..bca52c75 --- /dev/null +++ b/src/pylorax/creator.py @@ -0,0 +1,479 @@ +# +# Copyright (C) 2011-2017 Red Hat, Inc. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +import logging +log = logging.getLogger("pylorax") + +import os +import sys +import tempfile +import subprocess +import threading +import shutil +import hashlib +import re +import glob + +# Use Mako templates for appliance builder descriptions +from mako.template import Template +from mako.exceptions import text_error_template + +# Use the Lorax treebuilder branch for iso creation +from pylorax import ArchData, vernum +from pylorax.base import DataHolder +from pylorax.treebuilder import TreeBuilder, RuntimeBuilder, udev_escape +from pylorax.treebuilder import findkernels +from pylorax.sysutils import joinpaths, remove +from pylorax.imgutils import mount, umount, Mount +from pylorax.imgutils import mksquashfs, mkrootfsimg +from pylorax.imgutils import copytree +from pylorax.executils import execWithRedirect, execWithCapture, runcmd +from pylorax.installer import InstallError, novirt_install, virt_install + +RUNTIME = "images/install.img" + +# Default parameters for rebuilding initramfs, override with --dracut-args +DRACUT_DEFAULT = ["--xz", "--add", "livenet dmsquash-live convertfs pollcdrom", + "--omit", "plymouth", "--no-hostonly", "--no-early-microcode"] + + +def is_image_mounted(disk_img): + """ + Return True if the disk_img is mounted + """ + with open("/proc/mounts") as mounts: + for mount in mounts: + fields = mount.split() + if len(fields) > 2 and fields[1] == disk_img: + return True + return False + +def find_ostree_root(phys_root): + """ + Find root of ostree deployment + + :param str phys_root: Path to physical root + :returns: Relative path of ostree deployment root + :rtype: str + :raise Exception: More than one deployment roots were found + """ + ostree_root = "" + ostree_sysroots = glob.glob(joinpaths(phys_root, "ostree/boot.0/*/*/0")) + if ostree_sysroots: + if len(ostree_sysroots) > 1: + raise Exception("Too many deployment roots found: %s" % ostree_sysroots) + ostree_root = os.path.relpath(ostree_sysroots[0], phys_root) + return ostree_root + +class KernelInfo(object): + """ + Info about the kernels in boot_dir + """ + def __init__(self, boot_dir): + self.boot_dir = boot_dir + self.list = self.get_kernels() + self.arch = self.get_kernel_arch() + log.debug("kernel_list for {0.boot_dir} = {0.list}".format(self)) + log.debug("kernel_arch is {0.arch}".format(self)) + + def get_kernels(self): + """ + Get a list of the kernels in the boot_dir + + Examine the vmlinuz-* versions and return a list of them + + Ignore any with -rescue- in them, these are dracut rescue images. + The user shoud add + -dracut-config-rescue + to the kickstart to remove them, but catch it here as well. + """ + files = os.listdir(self.boot_dir) + return [f[8:] for f in files if f.startswith("vmlinuz-") \ + and f.find("-rescue-") == -1] + + def get_kernel_arch(self): + """ + Get the arch of the first kernel in boot_dir + + Defaults to i386 + """ + if self.list: + kernel_arch = self.list[0].split(".")[-1] + else: + kernel_arch = "i386" + return kernel_arch + + +def make_appliance(disk_img, name, template, outfile, networks=None, ram=1024, + vcpus=1, arch=None, title="Linux", project="Linux", + releasever="7"): + """ + Generate an appliance description file + + disk_img Full path of the disk image + name Name of the appliance, passed to the template + template Full path of Mako template + outfile Full path of file to write, using template + networks List of networks from the kickstart + ram Ram, in MB, passed to template. Default is 1024 + vcpus CPUs, passed to template. Default is 1 + arch CPU architecture. Default is 'x86_64' + title Title, passed to template. Default is 'Linux' + project Project, passed to template. Default is 'Linux' + releasever Release version, passed to template. Default is 17 + """ + if not (disk_img and template and outfile): + return None + + log.info("Creating appliance definition using {0}".format(template)) + + if not arch: + arch = "x86_64" + + log.info("Calculating SHA256 checksum of {0}".format(disk_img)) + sha256 = hashlib.sha256() + with open(disk_img) as f: + while True: + data = f.read(1024*1024) + if not data: + break + sha256.update(data) + log.info("SHA256 of {0} is {1}".format(disk_img, sha256.hexdigest())) + disk_info = DataHolder(name=os.path.basename(disk_img), format="raw", + checksum_type="sha256", checksum=sha256.hexdigest()) + try: + result = Template(filename=template).render(disks=[disk_info], name=name, + arch=arch, memory=ram, vcpus=vcpus, networks=networks, + title=title, project=project, releasever=releasever) + except Exception: + log.error(text_error_template().render()) + raise + + with open(outfile, "w") as f: + f.write(result) + + +def make_runtime(opts, mount_dir, work_dir): + """ + Make the squashfs image from a directory + + Result is in work_dir+RUNTIME + """ + kernels = KernelInfo(joinpaths(mount_dir, "boot" )) + + # Fake yum object + fake_yum = DataHolder(conf=DataHolder(installroot=mount_dir)) + # Fake arch with only basearch set + arch = ArchData(kernels.arch) + # TODO: Need to get release info from someplace... + product = DataHolder(name=opts.project, version=opts.releasever, release="", + variant="", bugurl="", isfinal=False) + + # This is a mounted image partition, cannot hardlink to it, so just use it + # symlink mount_dir/images to work_dir/images so we don't run out of space + os.makedirs(joinpaths(work_dir, "images")) + + rb = RuntimeBuilder(product, arch, fake_yum) + log.info("Creating runtime") + rb.create_runtime(joinpaths(work_dir, RUNTIME), size=None) + +def rebuild_initrds_for_live(opts, sys_root_dir, results_dir): + """ + Rebuild intrds for pxe live image (root=live:http://) + + :param opts: options passed to livemedia-creator + :type opts: argparse options + :param str sys_root_dir: Path to root of the system + :param str results_dir: Path of directory for storing results + """ + if not opts.dracut_args: + dracut_args = DRACUT_DEFAULT + else: + dracut_args = [] + for arg in opts.dracut_args: + dracut_args += arg.split(" ", 1) + log.info("dracut args = {0}".format(dracut_args)) + + dracut = ["dracut", "--nomdadmconf", "--nolvmconf"] + dracut_args + + kdir = "boot" + if opts.ostree: + kernels_dir = glob.glob(joinpaths(sys_root_dir, "boot/ostree/*"))[0] + kdir = os.path.relpath(kernels_dir, sys_root_dir) + + kernels = [kernel for kernel in findkernels(sys_root_dir, kdir) + if hasattr(kernel, "initrd")] + if not kernels: + raise Exception("No initrds found, cannot rebuild_initrds") + + # Hush some dracut warnings. TODO: bind-mount proc in place? + open(joinpaths(sys_root_dir,"/proc/modules"),"w") + + if opts.ostree: + # Dracut assumes to have some dirs in disk image + # /var/tmp for temp files + vartmp_dir = joinpaths(sys_root_dir, "var/tmp") + if not os.path.isdir(vartmp_dir): + os.mkdir(vartmp_dir) + # /root (maybe not fatal) + root_dir = joinpaths(sys_root_dir, "var/roothome") + if not os.path.isdir(root_dir): + os.mkdir(root_dir) + # /tmp (maybe not fatal) + tmp_dir = joinpaths(sys_root_dir, "sysroot/tmp") + if not os.path.isdir(tmp_dir): + os.mkdir(tmp_dir) + + for kernel in kernels: + outfile = kernel.initrd.path + ".live" + log.info("rebuilding %s", outfile) + + kver = kernel.version + + cmd = dracut + [outfile, kver] + runcmd(cmd, root=sys_root_dir) + + new_initrd_path = joinpaths(results_dir, os.path.basename(kernel.initrd.path)) + shutil.move(joinpaths(sys_root_dir, outfile), new_initrd_path) + os.chmod(new_initrd_path, 0644) + shutil.copy2(joinpaths(sys_root_dir, kernel.path), results_dir) + + os.unlink(joinpaths(sys_root_dir,"/proc/modules")) + +def create_pxe_config(template, images_dir, live_image_name, add_args = None): + """ + Create template for pxe to live configuration + + :param str images_dir: Path of directory with images to be used + :param str live_image_name: Name of live rootfs image file + :param list add_args: Arguments to be added to initrd= pxe config + """ + + add_args = add_args or [] + + kernels = [kernel for kernel in findkernels(images_dir, kdir="") + if hasattr(kernel, "initrd")] + if not kernels: + return + + kernel = kernels[0] + + add_args_str = " ".join(add_args) + + + try: + result = Template(filename=template).render(kernel=kernel.path, + initrd=kernel.initrd.path, liveimg=live_image_name, + addargs=add_args_str) + except Exception: + log.error(text_error_template().render()) + raise + + with open (joinpaths(images_dir, "PXE_CONFIG"), "w") as f: + f.write(result) + +def make_livecd(opts, mount_dir, work_dir): + """ + Take the content from the disk image and make a livecd out of it + + This uses wwood's squashfs live initramfs method: + * put the real / into LiveOS/rootfs.img + * make a squashfs of the LiveOS/rootfs.img tree + * make a simple initramfs with the squashfs.img and /etc/cmdline in it + * make a cpio of that tree + * append the squashfs.cpio to a dracut initramfs for each kernel installed + + Then on boot dracut reads /etc/cmdline which points to the squashfs.img + mounts that and then mounts LiveOS/rootfs.img as / + + """ + kernels = KernelInfo(joinpaths(mount_dir, "boot" )) + + arch = ArchData(kernels.arch) + # TODO: Need to get release info from someplace... + product = DataHolder(name=opts.project, version=opts.releasever, release="", + variant="", bugurl="", isfinal=False) + + # Link /images to work_dir/images to make the templates happy + if os.path.islink(joinpaths(mount_dir, "images")): + os.unlink(joinpaths(mount_dir, "images")) + execWithRedirect("/bin/ln", ["-s", joinpaths(work_dir, "images"), + joinpaths(mount_dir, "images")]) + + # The templates expect the config files to be in /tmp/config_files + # I think these should be release specific, not from lorax, but for now + configdir = joinpaths(opts.lorax_templates,"live/config_files/") + configdir_path = "tmp/config_files" + fullpath = joinpaths(mount_dir, configdir_path) + if os.path.exists(fullpath): + remove(fullpath) + shutil.copytree(configdir, fullpath) + + isolabel = opts.volid or "{0.name} {0.version} {1.basearch}".format(product, arch) + if len(isolabel) > 32: + isolabel = isolabel[:32] + log.warn("Truncating isolabel to 32 chars: %s" % (isolabel,)) + + tb = TreeBuilder(product=product, arch=arch, domacboot=opts.domacboot, + inroot=mount_dir, outroot=work_dir, + runtime=RUNTIME, isolabel=isolabel, + templatedir=joinpaths(opts.lorax_templates,"live/")) + log.info( "Rebuilding initrds" ) + if not opts.dracut_args: + dracut_args = DRACUT_DEFAULT + else: + dracut_args = [] + for arg in opts.dracut_args: + dracut_args += arg.split(" ", 1) + log.info("dracut args = {0}".format(dracut_args)) + tb.rebuild_initrds(add_args=dracut_args) + log.info("Building boot.iso") + tb.build() + + return work_dir + +def mount_boot_part_over_root(img_mount): + """ + Mount boot partition to /boot of root fs mounted in img_mount + + Used for OSTree so it finds deployment configurations on live rootfs + + param img_mount: object with mounted disk image root partition + type img_mount: imgutils.PartitionMount + """ + root_dir = img_mount.mount_dir + is_boot_part = lambda dir: os.path.exists(dir+"/loader.0") + tmp_mount_dir = tempfile.mkdtemp() + sys_root = find_ostree_root(root_dir) + sysroot_boot_dir = None + for dev, _size in img_mount.loop_devices: + if dev is img_mount.mount_dev: + continue + try: + mount("/dev/mapper/"+dev, mnt=tmp_mount_dir) + if is_boot_part(tmp_mount_dir): + umount(tmp_mount_dir) + sysroot_boot_dir = joinpaths(joinpaths(root_dir, sys_root), "boot") + mount("/dev/mapper/"+dev, mnt=sysroot_boot_dir) + break + else: + umount(tmp_mount_dir) + except subprocess.CalledProcessError as e: + log.debug("Looking for boot partition error: %s", e) + remove(tmp_mount_dir) + return sysroot_boot_dir + +def make_squashfs(disk_img, work_dir, compression="xz"): + """ + Take disk_img and put it into LiveOS/rootfs.img and squashfs this + tree into work_dir+RUNTIME + """ + liveos_dir = joinpaths(work_dir, "runtime/LiveOS") + os.makedirs(liveos_dir) + os.makedirs(os.path.dirname(joinpaths(work_dir, RUNTIME))) + + rc = execWithRedirect("/bin/ln", [disk_img, joinpaths(liveos_dir, "rootfs.img")]) + if rc != 0: + shutil.copy2(disk_img, joinpaths(liveos_dir, "rootfs.img")) + + mksquashfs(joinpaths(work_dir, "runtime"), + joinpaths(work_dir, RUNTIME), compression) + remove(joinpaths(work_dir, "runtime")) + + +def make_image(opts, ks): + """ + Install to an image + + Use virt or anaconda to install to an image. + + Returns the full path of of the image created. + """ + disk_size = 1 + (sum([p.size for p in ks.handler.partition.partitions]) / 1024) + log.info("disk_size = %sGB", disk_size) + + if opts.image_name: + disk_img = joinpaths(opts.result_dir, opts.image_name) + else: + disk_img = tempfile.mktemp(prefix="disk", suffix=".img", dir=opts.result_dir) + log.info("disk_img = %s", disk_img) + + try: + if opts.no_virt: + novirt_install(opts, disk_img, disk_size, ks.handler.method.url) + else: + install_log = os.path.abspath(os.path.dirname(opts.logfile))+"/virt-install.log" + log.info("install_log = %s", install_log) + + virt_install(opts, install_log, disk_img, disk_size) + except InstallError as e: + log.error("Install failed: {0}".format(e)) + if not opts.keep_image: + log.info("Removing bad disk image") + os.unlink(disk_img) + raise + + log.info("Disk Image install successful") + return disk_img + + +def make_live_images(opts, work_dir, root_dir, rootfs_image=None, size=None): + """ + Create live images from direcory or rootfs image + + :param opts: options passed to livemedia-creator + :type opts: argparse options + :param str work_dir: Directory for storing results + :param str root_dir: Root directory of live filesystem tree + :param str rootfs_image: Path to live rootfs image to be used + :returns: Path of directory with created images + :rtype: str + """ + sys_root = "" + if opts.ostree: + sys_root = find_ostree_root(root_dir) + + squashfs_root_dir = joinpaths(work_dir, "squashfs_root") + liveos_dir = joinpaths(squashfs_root_dir, "LiveOS") + os.makedirs(liveos_dir) + + if rootfs_image: + rc = execWithRedirect("/bin/ln", [rootfs_image, joinpaths(liveos_dir, "rootfs.img")]) + if rc != 0: + shutil.copy2(rootfs_image, joinpaths(liveos_dir, "rootfs.img")) + else: + log.info("Creating live rootfs image") + mkrootfsimg(root_dir, joinpaths(liveos_dir, "rootfs.img"), "LiveOS", size=size, sysroot=sys_root) + + log.info("Packing live rootfs image") + add_pxe_args = [] + live_image_name = "live-rootfs.squashfs.img" + mksquashfs(squashfs_root_dir, + joinpaths(work_dir, live_image_name), + opts.compression, + opts.compress_args) + + remove(squashfs_root_dir) + + log.info("Rebuilding initramfs for live") + rebuild_initrds_for_live(opts, joinpaths(root_dir, sys_root), work_dir) + + if opts.ostree: + add_pxe_args.append("ostree=/%s" % sys_root) + template = joinpaths(opts.lorax_templates, "pxe-live/pxe-config.tmpl") + create_pxe_config(template, work_dir, live_image_name, add_pxe_args) + + return work_dir diff --git a/src/pylorax/imgutils.py b/src/pylorax/imgutils.py index 817a6586..4279912d 100644 --- a/src/pylorax/imgutils.py +++ b/src/pylorax/imgutils.py @@ -107,6 +107,22 @@ def mkrootfsimg(rootdir, outfile, label, size=2, sysroot=""): root = join(mnt, sysroot.lstrip("/")) runcmd(cmd, root=root) +def mkdiskfsimage(diskimage, fsimage, label="Anaconda"): + """ + Copy the / partition of a partitioned disk image to an un-partitioned + disk image. + + diskimage is the full path to partitioned disk image with a / + fsimage is the full path of the output fs image file + label is the label to apply to the image. Defaults to "Anaconda" + """ + with PartitionMount(diskimage) as img_mount: + if not img_mount or not img_mount.mount_dir: + return None + + logger.info("Creating fsimage %s", fsimage) + mkext4img(img_mount.mount_dir, fsimage, label=label) + ######## Utility functions ############################################### def mksparse(outfile, size): diff --git a/src/pylorax/installer.py b/src/pylorax/installer.py new file mode 100644 index 00000000..0d6914b6 --- /dev/null +++ b/src/pylorax/installer.py @@ -0,0 +1,410 @@ +# +# Copyright (C) 2011-2017 Red Hat, Inc. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +import logging +log = logging.getLogger("pylorax") + +import os +import shutil +import sys +import subprocess +import tempfile +from time import sleep +import uuid + +from pylorax.executils import execWithRedirect, execWithCapture, runcmd +from pylorax.imgutils import get_loop_name, dm_detach, mount, umount, Mount +from pylorax.imgutils import PartitionMount, mksparse, mkext4img, loop_detach +from pylorax.imgutils import mksquashfs, mktar, mkrootfsimg, mkdiskfsimage, mkqcow2 +from pylorax.logmonitor import LogMonitor +from pylorax.sysutils import joinpaths, remove +from pylorax.treebuilder import TreeBuilder, RuntimeBuilder, udev_escape + +ROOT_PATH = "/mnt/sysimage/" + +# no-virt mode doesn't need libvirt, so make it optional +try: + import libvirt +except ImportError: + libvirt = None + + +class InstallError(Exception): + pass + +class IsoMountpoint(object): + """ + Mount the iso on a temporary directory and check to make sure the + vmlinuz and initrd.img files exist + Check the iso for a LiveOS directory and set a flag. + Extract the iso's label. + + initrd_path can be used to point to a boot.iso tree with a newer + initrd.img than the iso has. The iso is still used for stage2. + """ + def __init__( self, iso_path, initrd_path=None ): + """ iso_path is the path to a boot.iso + initrd_path overrides mounting the iso for access to + initrd and vmlinuz. + """ + self.label = None + self.iso_path = iso_path + self.initrd_path = initrd_path + + if not self.initrd_path: + self.mount_dir = mount(self.iso_path, opts="loop") + else: + self.mount_dir = self.initrd_path + + kernel_list = [("/isolinux/vmlinuz", "/isolinux/initrd.img"), + ("/ppc/ppc64/vmlinuz", "/ppc/ppc64/initrd.img"), + ("/images/pxeboot/vmlinuz", "/images/pxeboot/initrd.img")] + if os.path.isdir( self.mount_dir+"/repodata" ): + self.repo = self.mount_dir + else: + self.repo = None + self.liveos = os.path.isdir( self.mount_dir+"/LiveOS" ) + + try: + for kernel, initrd in kernel_list: + if (os.path.isfile(self.mount_dir+kernel) and + os.path.isfile(self.mount_dir+initrd)): + self.kernel = self.mount_dir+kernel + self.initrd = self.mount_dir+initrd + break + else: + raise Exception("Missing kernel and initrd file in iso, failed" + " to search under: {0}".format(kernel_list)) + except: + self.umount() + raise + + self.get_iso_label() + + def umount( self ): + if not self.initrd_path: + umount(self.mount_dir) + + def get_iso_label( self ): + """ + Get the iso's label using isoinfo + """ + isoinfo_output = execWithCapture("isoinfo", ["-d", "-i", self.iso_path]) + log.debug( isoinfo_output ) + for line in isoinfo_output.splitlines(): + if line.startswith("Volume id: "): + self.label = line[11:] + return + + +class VirtualInstall( object ): + """ + Run virt-install using an iso and kickstart(s) + """ + def __init__( self, iso, ks_paths, disk_img, img_size=2, + kernel_args=None, memory=1024, vnc=None, arch=None, + log_check=None, virtio_host="127.0.0.1", virtio_port=6080, + qcow2=False, boot_uefi=False, ovmf_path=None): + """ + Start the installation + + :param iso: Information about the iso to use for the installation + :type iso: IsoMountpoint + :param list ks_paths: Paths to kickstart files. All are injected, the + first one is the one executed. + :param str disk_img: Path to a disk image, created it it doesn't exist + :param int img_size: The image size, in MiB, to create if it doesn't exist + :param str kernel_args: Extra kernel arguments to pass on the kernel cmdline + :param int memory: Amount of RAM to assign to the virt, in MiB + :param str vnc: Arguments to pass to virt-install --graphics + :param str arch: Optional architecture to use in the virt + :param log_check: Method that returns True if the installation fails + :type log_check: method + :param str virtio_host: Hostname to connect virtio log to + :param int virtio_port: Port to connect virtio log to + :param bool qcow2: Set to True if disk_img is a qcow2 + :param bool boot_uefi: Use OVMF to boot the VM in UEFI mode + :param str ovmf_path: Path to the OVMF firmware + """ + self.virt_name = "LiveOS-"+str(uuid.uuid4()) + # add --graphics none later + # add whatever serial cmds are needed later + args = ["-n", self.virt_name, + "-r", str(memory), + "--noreboot", + "--noautoconsole"] + + args.append("--graphics") + if vnc: + args.append(vnc) + else: + args.append("none") + + for ks in ks_paths: + args.append("--initrd-inject") + args.append(ks) + + disk_opts = "path={0}".format(disk_img) + if qcow2: + disk_opts += ",format=qcow2" + else: + disk_opts += ",format=raw" + if not os.path.isfile(disk_img): + disk_opts += ",size={0}".format(img_size) + args.append("--disk") + args.append(disk_opts) + + if iso.liveos: + disk_opts = "path={0},device=cdrom".format(iso.iso_path) + args.append("--disk") + args.append(disk_opts) + + extra_args = "ks=file:/{0}".format(os.path.basename(ks_paths[0])) + if not vnc: + extra_args += " inst.cmdline console=ttyS0" + if kernel_args: + extra_args += " "+kernel_args + if iso.liveos: + extra_args += " stage2=hd:LABEL={0}".format(udev_escape(iso.label)) + args.append("--extra-args") + args.append(extra_args) + + args.append("--location") + args.append(iso.mount_dir) + + channel_args = "tcp,host={0}:{1},mode=connect,target_type=virtio" \ + ",name=org.fedoraproject.anaconda.log.0".format( + virtio_host, virtio_port) + args.append("--channel") + args.append(channel_args) + + if arch: + args.append("--arch") + args.append(arch) + + if boot_uefi and ovmf_path: + args.append("--boot") + args.append("loader=%s/OVMF_CODE.fd,loader_ro=yes,loader_type=pflash,nvram_template=%s/OVMF_VARS.fd,loader_secure=no" % (ovmf_path, ovmf_path)) + + log.info("Running virt-install.") + try: + execWithRedirect("virt-install", args, raise_err=True) + except subprocess.CalledProcessError as e: + raise InstallError("Problem starting virtual install: %s" % e) + + conn = libvirt.openReadOnly(None) + dom = conn.lookupByName(self.virt_name) + + # TODO: If vnc has been passed, we should look up the port and print that + # for the user at this point + + while dom.isActive() and not log_check(): + sys.stdout.write(".") + sys.stdout.flush() + sleep(10) + print + + if log_check(): + log.info( "Installation error detected. See logfile." ) + else: + log.info( "Install finished. Or at least virt shut down." ) + + def destroy( self ): + """ + Make sure the virt has been shut down and destroyed + + Could use libvirt for this instead. + """ + log.info( "Shutting down {0}".format(self.virt_name) ) + subprocess.call(["virsh", "destroy", self.virt_name]) + + # Undefine the virt, UEFI installs need to have --nvram passed + subprocess.call(["virsh", "undefine", self.virt_name, "--nvram"]) + +def novirt_install(opts, disk_img, disk_size, repo_url): + """ + Use Anaconda to install to a disk image + """ + import selinux + + # Set selinux to Permissive if it is Enforcing + selinux_enforcing = False + if selinux.is_selinux_enabled() and selinux.security_getenforce(): + selinux_enforcing = True + selinux.security_setenforce(0) + + # Clean up /tmp/ from previous runs to prevent stale info from being used + for path in ["/tmp/yum.repos.d/", "/tmp/yum.cache/"]: + if os.path.isdir(path): + shutil.rmtree(path) + + args = ["--kickstart", opts.ks[0], "--cmdline", "--repo", repo_url] + if opts.anaconda_args: + for arg in opts.anaconda_args: + args += arg.split(" ", 1) + if opts.proxy: + args += ["--proxy", opts.proxy] + if opts.armplatform: + args += ["--armplatform", opts.armplatform] + + if opts.make_iso or opts.make_fsimage: + # Make a blank fs image + args += ["--dirinstall"] + + mkext4img(None, disk_img, label=opts.fs_label, size=disk_size * 1024**3) + if not os.path.isdir(ROOT_PATH): + os.mkdir(ROOT_PATH) + mount(disk_img, opts="loop", mnt=ROOT_PATH) + elif opts.make_tar: + args += ["--dirinstall"] + + # Install directly into ROOT_PATH, make sure it starts clean + if os.path.exists(ROOT_PATH): + shutil.rmtree(ROOT_PATH) + if not os.path.isdir(ROOT_PATH): + os.mkdir(ROOT_PATH) + else: + args += ["--image", disk_img] + + # Create the sparse image + mksparse(disk_img, disk_size * 1024**3) + + # Make sure anaconda has the right product and release + os.environ["ANACONDA_PRODUCTNAME"] = opts.project + os.environ["ANACONDA_PRODUCTVERSION"] = opts.releasever + rc = execWithRedirect("anaconda", args) + + # Move the anaconda logs over to a log directory + log_dir = os.path.abspath(os.path.dirname(opts.logfile)) + log_anaconda = joinpaths(log_dir, "anaconda") + if not os.path.isdir(log_anaconda): + os.mkdir(log_anaconda) + for l in ["anaconda.log", "ifcfg.log", "program.log", "storage.log", + "packaging.log", "yum.log"]: + if os.path.exists("/tmp/"+l): + shutil.copy2("/tmp/"+l, log_anaconda) + os.unlink("/tmp/"+l) + + if opts.make_iso or opts.make_fsimage: + umount(ROOT_PATH) + else: + # If anaconda failed the disk image may still be in use by dm + execWithRedirect("anaconda-cleanup", []) + dm_name = os.path.splitext(os.path.basename(disk_img))[0] + dm_path = "/dev/mapper/"+dm_name + if os.path.exists(dm_path): + dm_detach(dm_path) + loop_detach(get_loop_name(disk_img)) + + if selinux_enforcing: + selinux.security_setenforce(1) + + if rc: + raise InstallError("novirt_install failed") + + if opts.make_tar: + compress_args = [] + for arg in opts.compress_args: + compress_args += arg.split(" ", 1) + + rc = mktar(ROOT_PATH, disk_img, opts.compression, compress_args) + shutil.rmtree(ROOT_PATH) + + if rc: + raise InstallError("novirt_install failed") + elif opts.qcow2: + log.info("Converting %s to qcow2", disk_img) + qcow2_args = [] + for arg in opts.qcow2_args: + qcow2_args += arg.split(" ", 1) + + # convert the image to qcow2 format + if "-O" not in qcow2_args: + qcow2_args.extend(["-O", "qcow2"]) + qcow2_img = tempfile.mktemp(prefix="disk", suffix=".img") + execWithRedirect("qemu-img", ["convert"] + qcow2_args + [disk_img, qcow2_img], raise_err=True) + execWithRedirect("mv", ["-f", qcow2_img, disk_img], raise_err=True) + + +def virt_install(opts, install_log, disk_img, disk_size): + """ + Use virt-install to install to a disk image + + install_log is the path to write the log from virt-install + disk_img is the full path to the final disk or filesystem image + disk_size is the size of the disk to create in GiB + """ + iso_mount = IsoMountpoint(opts.iso, opts.location) + log_monitor = LogMonitor(install_log) + + kernel_args = "" + if opts.kernel_args: + kernel_args += opts.kernel_args + if opts.proxy: + kernel_args += " proxy="+opts.proxy + + if opts.qcow2 and not opts.make_fsimage: + # virt-install can't take all the qcow2 options so create the image first + qcow2_args = [] + for arg in opts.qcow2_args: + qcow2_args += arg.split(" ", 1) + + mkqcow2(disk_img, disk_size*1024**3, qcow2_args) + + if opts.make_fsimage or opts.make_tar: + diskimg_path = tempfile.mktemp(prefix="disk", suffix=".img") + else: + diskimg_path = disk_img + + try: + virt = VirtualInstall(iso_mount, opts.ks, diskimg_path, disk_size, + kernel_args, opts.ram, opts.vnc, opts.arch, + log_check = log_monitor.server.log_check, + virtio_host = log_monitor.host, + virtio_port = log_monitor.port, + qcow2=opts.qcow2, boot_uefi=opts.virt_uefi, + ovmf_path=opts.ovmf_path) + + virt.destroy() + log_monitor.shutdown() + except InstallError as e: + log.error("VirtualInstall failed: %s", e) + raise + finally: + log.info("unmounting the iso") + iso_mount.umount() + + if log_monitor.server.log_check(): + raise InstallError("virt_install failed") + + if opts.make_fsimage: + mkdiskfsimage(diskimg_path, disk_img, label=opts.fs_label) + os.unlink(diskimg_path) + elif opts.make_tar: + compress_args = [] + for arg in opts.compress_args: + compress_args += arg.split(" ", 1) + + with PartitionMount(diskimg_path) as img_mount: + if img_mount and img_mount.mount_dir: + rc = mktar(img_mount.mount_dir, disk_img, opts.compression, compress_args) + os.unlink(diskimg_path) + + if rc: + raise InstallError("virt_install failed") + + + diff --git a/src/pylorax/logmonitor.py b/src/pylorax/logmonitor.py new file mode 100644 index 00000000..07f7560c --- /dev/null +++ b/src/pylorax/logmonitor.py @@ -0,0 +1,123 @@ +# +# Copyright (C) 2011-2017 Red Hat, Inc. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +import re +import socket +import SocketServer +import threading + +class LogRequestHandler(SocketServer.BaseRequestHandler): + """ + Handle monitoring and saving the logfiles from the virtual install + """ + def setup(self): + if self.server.log_path: + self.fp = open(self.server.log_path, "w") + else: + print "no log_path specified" + self.request.settimeout(10) + + def handle(self): + """ + Handle writing incoming data to a logfile and + checking the logs for any Tracebacks or other errors that indicate + that the install failed. + """ + line = "" + while True: + if self.server.kill: + break + + try: + data = self.request.recv(4096) + self.fp.write(data) + self.fp.flush() + + # check the data for errors and set error flag + # need to assemble it into lines so we can test for the error + # string. + while data: + more = data.split("\n", 1) + line += more[0] + if len(more) > 1: + self.iserror(line) + line = "" + data = more[1] + else: + data = None + + except socket.timeout: + pass + except: + break + + def finish(self): + self.fp.close() + + def iserror(self, line): + """ + Check a line to see if it contains an error indicating install failure + """ + simple_tests = ["Traceback (", + "Out of memory:", + "Call Trace:", + "insufficient disk space:"] + re_tests = [r"packaging: base repo .* not valid"] + for t in simple_tests: + if line.find(t) > -1: + self.server.log_error = True + return + for t in re_tests: + if re.search(t, line): + self.server.log_error = True + return + + +class LogServer(SocketServer.TCPServer): + """ + Add path to logfile + Add log error flag + Add a kill switch + """ + def __init__(self, log_path, *args, **kwargs): + self.kill = False + self.log_error = False + self.log_path = log_path + SocketServer.TCPServer.__init__(self, *args, **kwargs) + + def log_check(self): + return self.log_error + + +class LogMonitor(object): + """ + Contains all the stuff needed to setup a thread to listen to the logs + from the virtual install + """ + def __init__(self, log_path, host="localhost", port=0): + """ + Fire up the thread listening for logs + """ + self.server = LogServer(log_path, (host, port), LogRequestHandler) + self.host, self.port = self.server.server_address + self.log_path = log_path + self.server_thread = threading.Thread(target=self.server.handle_request) + self.server_thread.daemon = True + self.server_thread.start() + + def shutdown(self): + self.server.kill = True + self.server_thread.join() diff --git a/src/sbin/livemedia-creator b/src/sbin/livemedia-creator index eb1cc44e..45a42929 100755 --- a/src/sbin/livemedia-creator +++ b/src/sbin/livemedia-creator @@ -17,47 +17,32 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . # -# Author(s): Brian C. Lane -# import logging log = logging.getLogger("livemedia-creator") program_log = logging.getLogger("program") pylorax_log = logging.getLogger("pylorax") -import os -import sys -import uuid -import tempfile -import subprocess -import socket -import threading -import SocketServer -from time import sleep -import shutil import argparse -import hashlib -import re -import glob +import os +import shutil +import sys +import tempfile # Use pykickstart to calculate disk image size from pykickstart.parser import KickstartParser from pykickstart.version import makeVersion, RHEL7 -# Use Mako templates for appliance builder descriptions -from mako.template import Template -from mako.exceptions import text_error_template - -# Use the Lorax treebuilder branch for iso creation -from pylorax import ArchData, vernum -from pylorax.base import DataHolder -from pylorax.treebuilder import TreeBuilder, RuntimeBuilder, udev_escape -from pylorax.treebuilder import findkernels -from pylorax.sysutils import joinpaths, remove -from pylorax.imgutils import PartitionMount, mksparse, mkext4img, loop_detach -from pylorax.imgutils import get_loop_name, dm_detach, mount, umount, Mount -from pylorax.imgutils import mksquashfs, mktar, mkrootfsimg, mkqcow2 +from pylorax import vernum +from pylorax.creator import DRACUT_DEFAULT, mount_boot_part_over_root +from pylorax.creator import make_appliance, make_image, make_livecd, make_live_images +from pylorax.creator import make_runtime, make_squashfs from pylorax.imgutils import copytree -from pylorax.executils import execWithRedirect, execWithCapture, runcmd +from pylorax.imgutils import Mount, PartitionMount, umount +from pylorax.installer import InstallError +from pylorax.sysutils import joinpaths + +VERSION = "{0}-{1}".format(os.path.basename(sys.argv[0]), vernum) + # no-virt mode doesn't need libvirt, so make it optional try: @@ -65,972 +50,9 @@ try: except ImportError: libvirt = None -VERSION = "{0}-{1}".format(os.path.basename(sys.argv[0]), vernum) +def lorax_parser(): + """ Return the ArgumentParser for lorax""" -# Default parameters for rebuilding initramfs, override with --dracut-arg -DRACUT_DEFAULT = ["--xz", "--add", "livenet dmsquash-live convertfs pollcdrom", - "--omit", "plymouth", "--no-hostonly", "--no-early-microcode"] - -ROOT_PATH = "/mnt/sysimage/" -RUNTIME = "images/install.img" - -class InstallError(Exception): - pass - - -class LogRequestHandler(SocketServer.BaseRequestHandler): - """ - Handle monitoring and saving the logfiles from the virtual install - """ - def setup(self): - if self.server.log_path: - self.fp = open(self.server.log_path, "w") - else: - print "no log_path specified" - self.request.settimeout(10) - - def handle(self): - """ - Handle writing incoming data to a logfile and - checking the logs for any Tracebacks or other errors that indicate - that the install failed. - """ - line = "" - while True: - if self.server.kill: - break - - try: - data = self.request.recv(4096) - self.fp.write(data) - self.fp.flush() - - # check the data for errors and set error flag - # need to assemble it into lines so we can test for the error - # string. - while data: - more = data.split("\n", 1) - line += more[0] - if len(more) > 1: - self.iserror(line) - line = "" - data = more[1] - else: - data = None - - except socket.timeout: - pass - except: - break - - def finish(self): - self.fp.close() - - def iserror(self, line): - """ - Check a line to see if it contains an error indicating install failure - """ - simple_tests = ["Traceback (", - "Out of memory:", - "Call Trace:", - "insufficient disk space:"] - re_tests = [r"packaging: base repo .* not valid"] - for t in simple_tests: - if line.find(t) > -1: - self.server.log_error = True - return - for t in re_tests: - if re.search(t, line): - self.server.log_error = True - return - - -class LogServer(SocketServer.TCPServer): - """ - Add path to logfile - Add log error flag - Add a kill switch - """ - def __init__(self, log_path, *args, **kwargs): - self.kill = False - self.log_error = False - self.log_path = log_path - SocketServer.TCPServer.__init__(self, *args, **kwargs) - - def log_check(self): - return self.log_error - - -class LogMonitor(object): - """ - Contains all the stuff needed to setup a thread to listen to the logs - from the virtual install - """ - def __init__(self, log_path, host="localhost", port=0): - """ - Fire up the thread listening for logs - """ - self.server = LogServer(log_path, (host, port), LogRequestHandler) - self.host, self.port = self.server.server_address - self.log_path = log_path - self.server_thread = threading.Thread(target=self.server.handle_request) - self.server_thread.daemon = True - self.server_thread.start() - - def shutdown(self): - self.server.kill = True - self.server_thread.join() - - -class IsoMountpoint(object): - """ - Mount the iso on a temporary directory and check to make sure the - vmlinuz and initrd.img files exist - Check the iso for a LiveOS directory and set a flag. - Extract the iso's label. - - initrd_path can be used to point to a boot.iso tree with a newer - initrd.img than the iso has. The iso is still used for stage2. - """ - def __init__( self, iso_path, initrd_path=None ): - """ iso_path is the path to a boot.iso - initrd_path overrides mounting the iso for access to - initrd and vmlinuz. - """ - self.label = None - self.iso_path = iso_path - self.initrd_path = initrd_path - - if not self.initrd_path: - self.mount_dir = mount(self.iso_path, opts="loop") - else: - self.mount_dir = self.initrd_path - - kernel_list = [("/isolinux/vmlinuz", "/isolinux/initrd.img"), - ("/ppc/ppc64/vmlinuz", "/ppc/ppc64/initrd.img"), - ("/images/pxeboot/vmlinuz", "/images/pxeboot/initrd.img")] - if os.path.isdir( self.mount_dir+"/repodata" ): - self.repo = self.mount_dir - else: - self.repo = None - self.liveos = os.path.isdir( self.mount_dir+"/LiveOS" ) - - try: - for kernel, initrd in kernel_list: - if (os.path.isfile(self.mount_dir+kernel) and - os.path.isfile(self.mount_dir+initrd)): - self.kernel = self.mount_dir+kernel - self.initrd = self.mount_dir+initrd - break - else: - raise Exception("Missing kernel and initrd file in iso, failed" - " to search under: {0}".format(kernel_list)) - except: - self.umount() - raise - - self.get_iso_label() - - def umount( self ): - if not self.initrd_path: - umount(self.mount_dir) - - def get_iso_label( self ): - """ - Get the iso's label using isoinfo - """ - isoinfo_output = execWithCapture("isoinfo", ["-d", "-i", self.iso_path]) - log.debug( isoinfo_output ) - for line in isoinfo_output.splitlines(): - if line.startswith("Volume id: "): - self.label = line[11:] - return - - -class VirtualInstall( object ): - """ - Run virt-install using an iso and kickstart(s) - """ - def __init__( self, iso, ks_paths, disk_img, img_size=2, - kernel_args=None, memory=1024, vnc=None, arch=None, - log_check=None, virtio_host="127.0.0.1", virtio_port=6080, - qcow2=False, boot_uefi=False, ovmf_path=None): - """ - Start the installation - - :param iso: Information about the iso to use for the installation - :type iso: IsoMountpoint - :param list ks_paths: Paths to kickstart files. All are injected, the - first one is the one executed. - :param str disk_img: Path to a disk image, created it it doesn't exist - :param int img_size: The image size, in MiB, to create if it doesn't exist - :param str kernel_args: Extra kernel arguments to pass on the kernel cmdline - :param int memory: Amount of RAM to assign to the virt, in MiB - :param str vnc: Arguments to pass to virt-install --graphics - :param str arch: Optional architecture to use in the virt - :param log_check: Method that returns True if the installation fails - :type log_check: method - :param str virtio_host: Hostname to connect virtio log to - :param int virtio_port: Port to connect virtio log to - :param bool qcow2: Set to True if disk_img is a qcow2 - :param bool boot_uefi: Use OVMF to boot the VM in UEFI mode - :param str ovmf_path: Path to the OVMF firmware - """ - self.virt_name = "LiveOS-"+str(uuid.uuid4()) - # add --graphics none later - # add whatever serial cmds are needed later - args = ["-n", self.virt_name, - "-r", str(memory), - "--noreboot", - "--noautoconsole"] - - args.append("--graphics") - if vnc: - args.append(vnc) - else: - args.append("none") - - for ks in ks_paths: - args.append("--initrd-inject") - args.append(ks) - - disk_opts = "path={0}".format(disk_img) - if qcow2: - disk_opts += ",format=qcow2" - else: - disk_opts += ",format=raw" - if not os.path.isfile(disk_img): - disk_opts += ",size={0}".format(img_size) - args.append("--disk") - args.append(disk_opts) - - if iso.liveos: - disk_opts = "path={0},device=cdrom".format(iso.iso_path) - args.append("--disk") - args.append(disk_opts) - - extra_args = "ks=file:/{0}".format(os.path.basename(ks_paths[0])) - if not vnc: - extra_args += " inst.cmdline console=ttyS0" - if kernel_args: - extra_args += " "+kernel_args - if iso.liveos: - extra_args += " stage2=hd:LABEL={0}".format(udev_escape(iso.label)) - args.append("--extra-args") - args.append(extra_args) - - args.append("--location") - args.append(iso.mount_dir) - - channel_args = "tcp,host={0}:{1},mode=connect,target_type=virtio" \ - ",name=org.fedoraproject.anaconda.log.0".format( - virtio_host, virtio_port) - args.append("--channel") - args.append(channel_args) - - if arch: - args.append("--arch") - args.append(arch) - - elif boot_uefi and ovmf_path: - args.append("--boot") - args.append("loader=%s/OVMF_CODE.fd,loader_ro=yes,loader_type=pflash,nvram_template=%s/OVMF_VARS.fd,loader_secure=no" % (ovmf_path, ovmf_path)) - - log.info("Running virt-install.") - try: - execWithRedirect("virt-install", args, raise_err=True) - except subprocess.CalledProcessError as e: - raise InstallError("Problem starting virtual install: %s" % e) - - conn = libvirt.openReadOnly(None) - dom = conn.lookupByName(self.virt_name) - - # TODO: If vnc has been passed, we should look up the port and print that - # for the user at this point - - while dom.isActive() and not log_check(): - sys.stdout.write(".") - sys.stdout.flush() - sleep(10) - print - - if log_check(): - log.info( "Installation error detected. See logfile." ) - else: - log.info( "Install finished. Or at least virt shut down." ) - - def destroy( self ): - """ - Make sure the virt has been shut down and destroyed - - Could use libvirt for this instead. - """ - log.info( "Shutting down {0}".format(self.virt_name) ) - subprocess.call(["virsh", "destroy", self.virt_name]) - - # Undefine the virt, UEFI installs need to have --nvram passed - subprocess.call(["virsh", "undefine", self.virt_name, "--nvram"]) - -def is_image_mounted(disk_img): - """ - Return True if the disk_img is mounted - """ - with open("/proc/mounts") as mounts: - for mount in mounts: - fields = mount.split() - if len(fields) > 2 and fields[1] == disk_img: - return True - return False - -def find_ostree_root(phys_root): - """ - Find root of ostree deployment - - :param str phys_root: Path to physical root - :returns: Relative path of ostree deployment root - :rtype: str - :raise Exception: More than one deployment roots were found - """ - ostree_root = "" - ostree_sysroots = glob.glob(joinpaths(phys_root, "ostree/boot.0/*/*/0")) - if ostree_sysroots: - if len(ostree_sysroots) > 1: - raise Exception("Too many deployment roots found: %s" % ostree_sysroots) - ostree_root = os.path.relpath(ostree_sysroots[0], phys_root) - return ostree_root - -class KernelInfo(object): - """ - Info about the kernels in boot_dir - """ - def __init__(self, boot_dir): - self.boot_dir = boot_dir - self.list = self.get_kernels() - self.arch = self.get_kernel_arch() - log.debug("kernel_list for {0.boot_dir} = {0.list}".format(self)) - log.debug("kernel_arch is {0.arch}".format(self)) - - def get_kernels(self): - """ - Get a list of the kernels in the boot_dir - - Examine the vmlinuz-* versions and return a list of them - - Ignore any with -rescue- in them, these are dracut rescue images. - The user shoud add - -dracut-config-rescue - to the kickstart to remove them, but catch it here as well. - """ - files = os.listdir(self.boot_dir) - return [f[8:] for f in files if f.startswith("vmlinuz-") \ - and f.find("-rescue-") == -1] - - def get_kernel_arch(self): - """ - Get the arch of the first kernel in boot_dir - - Defaults to i386 - """ - if self.list: - kernel_arch = self.list[0].split(".")[-1] - else: - kernel_arch = "i386" - return kernel_arch - - -def make_appliance(disk_img, name, template, outfile, networks=None, ram=1024, - vcpus=1, arch=None, title="Linux", project="Linux", - releasever="7"): - """ - Generate an appliance description file - - disk_img Full path of the disk image - name Name of the appliance, passed to the template - template Full path of Mako template - outfile Full path of file to write, using template - networks List of networks from the kickstart - ram Ram, in MB, passed to template. Default is 1024 - vcpus CPUs, passed to template. Default is 1 - arch CPU architecture. Default is 'x86_64' - title Title, passed to template. Default is 'Linux' - project Project, passed to template. Default is 'Linux' - releasever Release version, passed to template. Default is 17 - """ - if not (disk_img and template and outfile): - return None - - log.info("Creating appliance definition using {0}".format(template)) - - if not arch: - arch = "x86_64" - - log.info("Calculating SHA256 checksum of {0}".format(disk_img)) - sha256 = hashlib.sha256() - with open(disk_img) as f: - while True: - data = f.read(1024*1024) - if not data: - break - sha256.update(data) - log.info("SHA256 of {0} is {1}".format(disk_img, sha256.hexdigest())) - disk_info = DataHolder(name=os.path.basename(disk_img), format="raw", - checksum_type="sha256", checksum=sha256.hexdigest()) - try: - result = Template(filename=template).render(disks=[disk_info], name=name, - arch=arch, memory=ram, vcpus=vcpus, networks=networks, - title=title, project=project, releasever=releasever) - except Exception: - log.error(text_error_template().render()) - raise - - with open(outfile, "w") as f: - f.write(result) - - -def make_fsimage(diskimage, fsimage, label="Anaconda"): - """ - Copy the / partition of a partitioned disk image to an un-partitioned - disk image. - - diskimage is the full path to partitioned disk image with a / - fsimage is the full path of the output fs image file - label is the label to apply to the image. Defaults to "Anaconda" - """ - with PartitionMount(diskimage) as img_mount: - if not img_mount or not img_mount.mount_dir: - return None - - log.info("Creating fsimage %s", fsimage) - mkext4img(img_mount.mount_dir, fsimage, label=label) - - -def make_runtime(opts, mount_dir, work_dir): - """ - Make the squashfs image from a directory - - Result is in work_dir+RUNTIME - """ - kernels = KernelInfo(joinpaths(mount_dir, "boot" )) - - # Fake yum object - fake_yum = DataHolder(conf=DataHolder(installroot=mount_dir)) - # Fake arch with only basearch set - arch = ArchData(kernels.arch) - # TODO: Need to get release info from someplace... - product = DataHolder(name=opts.project, version=opts.releasever, release="", - variant="", bugurl="", isfinal=False) - - # This is a mounted image partition, cannot hardlink to it, so just use it - # symlink mount_dir/images to work_dir/images so we don't run out of space - os.makedirs(joinpaths(work_dir, "images")) - - rb = RuntimeBuilder(product, arch, fake_yum) - log.info("Creating runtime") - rb.create_runtime(joinpaths(work_dir, RUNTIME), size=None) - -def rebuild_initrds_for_live(opts, sys_root_dir, results_dir): - """ - Rebuild intrds for pxe live image (root=live:http://) - - :param opts: options passed to livemedia-creator - :type opts: argparse options - :param str sys_root_dir: Path to root of the system - :param str results_dir: Path of directory for storing results - """ - if not opts.dracut_args: - dracut_args = DRACUT_DEFAULT - else: - dracut_args = [] - for arg in opts.dracut_args: - dracut_args += arg.split(" ", 1) - log.info("dracut args = {0}".format(dracut_args)) - - dracut = ["dracut", "--nomdadmconf", "--nolvmconf"] + dracut_args - - kdir = "boot" - if opts.ostree: - kernels_dir = glob.glob(joinpaths(sys_root_dir, "boot/ostree/*"))[0] - kdir = os.path.relpath(kernels_dir, sys_root_dir) - - kernels = [kernel for kernel in findkernels(sys_root_dir, kdir) - if hasattr(kernel, "initrd")] - if not kernels: - raise Exception("No initrds found, cannot rebuild_initrds") - - # Hush some dracut warnings. TODO: bind-mount proc in place? - open(joinpaths(sys_root_dir,"/proc/modules"),"w") - - if opts.ostree: - # Dracut assumes to have some dirs in disk image - # /var/tmp for temp files - vartmp_dir = joinpaths(sys_root_dir, "var/tmp") - if not os.path.isdir(vartmp_dir): - os.mkdir(vartmp_dir) - # /root (maybe not fatal) - root_dir = joinpaths(sys_root_dir, "var/roothome") - if not os.path.isdir(root_dir): - os.mkdir(root_dir) - # /tmp (maybe not fatal) - tmp_dir = joinpaths(sys_root_dir, "sysroot/tmp") - if not os.path.isdir(tmp_dir): - os.mkdir(tmp_dir) - - for kernel in kernels: - outfile = kernel.initrd.path + ".live" - log.info("rebuilding %s", outfile) - - kver = kernel.version - - cmd = dracut + [outfile, kver] - runcmd(cmd, root=sys_root_dir) - - new_initrd_path = joinpaths(results_dir, os.path.basename(kernel.initrd.path)) - shutil.move(joinpaths(sys_root_dir, outfile), new_initrd_path) - os.chmod(new_initrd_path, 0644) - shutil.copy2(joinpaths(sys_root_dir, kernel.path), results_dir) - - os.unlink(joinpaths(sys_root_dir,"/proc/modules")) - -def create_pxe_config(template, images_dir, live_image_name, add_args = None): - """ - Create template for pxe to live configuration - - :param str images_dir: Path of directory with images to be used - :param str live_image_name: Name of live rootfs image file - :param list add_args: Arguments to be added to initrd= pxe config - """ - - add_args = add_args or [] - - kernels = [kernel for kernel in findkernels(images_dir, kdir="") - if hasattr(kernel, "initrd")] - if not kernels: - return - - kernel = kernels[0] - - add_args_str = " ".join(add_args) - - - try: - result = Template(filename=template).render(kernel=kernel.path, - initrd=kernel.initrd.path, liveimg=live_image_name, - addargs=add_args_str) - except Exception: - log.error(text_error_template().render()) - raise - - with open (joinpaths(images_dir, "PXE_CONFIG"), "w") as f: - f.write(result) - -def make_livecd(opts, mount_dir, work_dir): - """ - Take the content from the disk image and make a livecd out of it - - This uses wwood's squashfs live initramfs method: - * put the real / into LiveOS/rootfs.img - * make a squashfs of the LiveOS/rootfs.img tree - * make a simple initramfs with the squashfs.img and /etc/cmdline in it - * make a cpio of that tree - * append the squashfs.cpio to a dracut initramfs for each kernel installed - - Then on boot dracut reads /etc/cmdline which points to the squashfs.img - mounts that and then mounts LiveOS/rootfs.img as / - - """ - kernels = KernelInfo(joinpaths(mount_dir, "boot" )) - - arch = ArchData(kernels.arch) - # TODO: Need to get release info from someplace... - product = DataHolder(name=opts.project, version=opts.releasever, release="", - variant="", bugurl="", isfinal=False) - - # Link /images to work_dir/images to make the templates happy - if os.path.islink(joinpaths(mount_dir, "images")): - os.unlink(joinpaths(mount_dir, "images")) - execWithRedirect("/bin/ln", ["-s", joinpaths(work_dir, "images"), - joinpaths(mount_dir, "images")]) - - # The templates expect the config files to be in /tmp/config_files - # I think these should be release specific, not from lorax, but for now - configdir = joinpaths(opts.lorax_templates,"live/config_files/") - configdir_path = "tmp/config_files" - fullpath = joinpaths(mount_dir, configdir_path) - if os.path.exists(fullpath): - remove(fullpath) - shutil.copytree(configdir, fullpath) - - isolabel = opts.volid or "{0.name} {0.version} {1.basearch}".format(product, arch) - if len(isolabel) > 32: - isolabel = isolabel[:32] - log.warn("Truncating isolabel to 32 chars: %s" % (isolabel,)) - - tb = TreeBuilder(product=product, arch=arch, domacboot=opts.domacboot, - inroot=mount_dir, outroot=work_dir, - runtime=RUNTIME, isolabel=isolabel, - templatedir=joinpaths(opts.lorax_templates,"live/")) - log.info( "Rebuilding initrds" ) - if not opts.dracut_args: - dracut_args = DRACUT_DEFAULT - else: - dracut_args = [] - for arg in opts.dracut_args: - dracut_args += arg.split(" ", 1) - log.info("dracut args = {0}".format(dracut_args)) - tb.rebuild_initrds(add_args=dracut_args) - log.info("Building boot.iso") - tb.build() - - return work_dir - -def mount_boot_part_over_root(img_mount): - """ - Mount boot partition to /boot of root fs mounted in img_mount - - Used for OSTree so it finds deployment configurations on live rootfs - - param img_mount: object with mounted disk image root partition - type img_mount: imgutils.PartitionMount - """ - root_dir = img_mount.mount_dir - is_boot_part = lambda dir: os.path.exists(dir+"/loader.0") - tmp_mount_dir = tempfile.mkdtemp() - sys_root = find_ostree_root(root_dir) - sysroot_boot_dir = None - for dev, _size in img_mount.loop_devices: - if dev is img_mount.mount_dev: - continue - try: - mount("/dev/mapper/"+dev, mnt=tmp_mount_dir) - if is_boot_part(tmp_mount_dir): - umount(tmp_mount_dir) - sysroot_boot_dir = joinpaths(joinpaths(root_dir, sys_root), "boot") - mount("/dev/mapper/"+dev, mnt=sysroot_boot_dir) - break - else: - umount(tmp_mount_dir) - except subprocess.CalledProcessError as e: - log.debug("Looking for boot partition error: %s", e) - remove(tmp_mount_dir) - return sysroot_boot_dir - -def novirt_install(opts, disk_img, disk_size, repo_url): - """ - Use Anaconda to install to a disk image - """ - import selinux - - # Set selinux to Permissive if it is Enforcing - selinux_enforcing = False - if selinux.is_selinux_enabled() and selinux.security_getenforce(): - selinux_enforcing = True - selinux.security_setenforce(0) - - # Clean up /tmp/ from previous runs to prevent stale info from being used - for path in ["/tmp/yum.repos.d/", "/tmp/yum.cache/"]: - if os.path.isdir(path): - shutil.rmtree(path) - - args = ["--kickstart", opts.ks[0], "--cmdline", "--repo", repo_url] - if opts.anaconda_args: - for arg in opts.anaconda_args: - args += arg.split(" ", 1) - if opts.proxy: - args += ["--proxy", opts.proxy] - if opts.armplatform: - args += ["--armplatform", opts.armplatform] - - if opts.make_iso or opts.make_fsimage: - # Make a blank fs image - args += ["--dirinstall"] - - mkext4img(None, disk_img, label=opts.fs_label, size=disk_size * 1024**3) - if not os.path.isdir(ROOT_PATH): - os.mkdir(ROOT_PATH) - mount(disk_img, opts="loop", mnt=ROOT_PATH) - elif opts.make_tar: - args += ["--dirinstall"] - - # Install directly into ROOT_PATH, make sure it starts clean - if os.path.exists(ROOT_PATH): - shutil.rmtree(ROOT_PATH) - if not os.path.isdir(ROOT_PATH): - os.mkdir(ROOT_PATH) - else: - args += ["--image", disk_img] - - # Create the sparse image - mksparse(disk_img, disk_size * 1024**3) - - # Make sure anaconda has the right product and release - os.environ["ANACONDA_PRODUCTNAME"] = opts.project - os.environ["ANACONDA_PRODUCTVERSION"] = opts.releasever - rc = execWithRedirect("anaconda", args) - - # Move the anaconda logs over to a log directory - log_dir = os.path.abspath(os.path.dirname(opts.logfile)) - log_anaconda = joinpaths(log_dir, "anaconda") - if not os.path.isdir(log_anaconda): - os.mkdir(log_anaconda) - for l in ["anaconda.log", "ifcfg.log", "program.log", "storage.log", - "packaging.log", "yum.log"]: - if os.path.exists("/tmp/"+l): - shutil.copy2("/tmp/"+l, log_anaconda) - os.unlink("/tmp/"+l) - - if opts.make_iso or opts.make_fsimage: - umount(ROOT_PATH) - else: - # If anaconda failed the disk image may still be in use by dm - execWithRedirect("anaconda-cleanup", []) - dm_name = os.path.splitext(os.path.basename(disk_img))[0] - dm_path = "/dev/mapper/"+dm_name - if os.path.exists(dm_path): - dm_detach(dm_path) - loop_detach(get_loop_name(disk_img)) - - if selinux_enforcing: - selinux.security_setenforce(1) - - if rc: - raise InstallError("novirt_install failed") - - if opts.make_tar: - compress_args = [] - for arg in opts.compress_args: - compress_args += arg.split(" ", 1) - - rc = mktar(ROOT_PATH, disk_img, opts.compression, compress_args) - shutil.rmtree(ROOT_PATH) - - if rc: - raise InstallError("novirt_install failed") - elif opts.qcow2: - log.info("Converting %s to qcow2", disk_img) - qcow2_args = [] - for arg in opts.qcow2_args: - qcow2_args += arg.split(" ", 1) - - # convert the image to qcow2 format - if "-O" not in qcow2_args: - qcow2_args.extend(["-O", "qcow2"]) - qcow2_img = tempfile.mktemp(prefix="disk", suffix=".img") - execWithRedirect("qemu-img", ["convert"] + qcow2_args + [disk_img, qcow2_img], raise_err=True) - execWithRedirect("mv", ["-f", qcow2_img, disk_img], raise_err=True) - - -def virt_install(opts, install_log, disk_img, disk_size): - """ - Use virt-install to install to a disk image - - install_log is the path to write the log from virt-install - disk_img is the full path to the final disk or filesystem image - disk_size is the size of the disk to create in GiB - """ - iso_mount = IsoMountpoint(opts.iso, opts.location) - log_monitor = LogMonitor(install_log) - - kernel_args = "" - if opts.kernel_args: - kernel_args += opts.kernel_args - if opts.proxy: - kernel_args += " proxy="+opts.proxy - - if opts.qcow2 and not opts.make_fsimage: - # virt-install can't take all the qcow2 options so create the image first - qcow2_args = [] - for arg in opts.qcow2_args: - qcow2_args += arg.split(" ", 1) - - mkqcow2(disk_img, disk_size*1024**3, qcow2_args) - - if opts.make_fsimage or opts.make_tar: - diskimg_path = tempfile.mktemp(prefix="disk", suffix=".img") - else: - diskimg_path = disk_img - - try: - virt = VirtualInstall(iso_mount, opts.ks, diskimg_path, disk_size, - kernel_args, opts.ram, opts.vnc, opts.arch, - log_check = log_monitor.server.log_check, - virtio_host = log_monitor.host, - virtio_port = log_monitor.port, - qcow2=opts.qcow2, boot_uefi=opts.virt_uefi, - ovmf_path=opts.ovmf_path) - - virt.destroy() - log_monitor.shutdown() - except InstallError as e: - log.error("VirtualInstall failed: %s", e) - raise - finally: - log.info("unmounting the iso") - iso_mount.umount() - - if log_monitor.server.log_check(): - raise InstallError("virt_install failed") - - if opts.make_fsimage: - make_fsimage(diskimg_path, disk_img, label=opts.fs_label) - os.unlink(diskimg_path) - elif opts.make_tar: - compress_args = [] - for arg in opts.compress_args: - compress_args += arg.split(" ", 1) - - with PartitionMount(diskimg_path) as img_mount: - if img_mount and img_mount.mount_dir: - rc = mktar(img_mount.mount_dir, disk_img, opts.compression, compress_args) - os.unlink(diskimg_path) - - if rc: - raise InstallError("virt_install failed") - - -def make_squashfs(disk_img, work_dir, compression="xz"): - """ - Take disk_img and put it into LiveOS/rootfs.img and squashfs this - tree into work_dir+RUNTIME - """ - liveos_dir = joinpaths(work_dir, "runtime/LiveOS") - os.makedirs(liveos_dir) - os.makedirs(os.path.dirname(joinpaths(work_dir, RUNTIME))) - - rc = execWithRedirect("/bin/ln", [disk_img, joinpaths(liveos_dir, "rootfs.img")]) - if rc != 0: - shutil.copy2(disk_img, joinpaths(liveos_dir, "rootfs.img")) - - mksquashfs(joinpaths(work_dir, "runtime"), - joinpaths(work_dir, RUNTIME), compression) - remove(joinpaths(work_dir, "runtime")) - - -def make_image(opts, ks): - """ - Install to an image - - Use virt or anaconda to install to an image. - - Returns the full path of of the image created. - """ - disk_size = 1 + (sum([p.size for p in ks.handler.partition.partitions]) / 1024) - log.info("disk_size = %sGB", disk_size) - - if opts.image_name: - disk_img = joinpaths(opts.result_dir, opts.image_name) - else: - disk_img = tempfile.mktemp(prefix="disk", suffix=".img", dir=opts.result_dir) - log.info("disk_img = %s", disk_img) - - try: - if opts.no_virt: - novirt_install(opts, disk_img, disk_size, ks.handler.method.url) - else: - install_log = os.path.abspath(os.path.dirname(opts.logfile))+"/virt-install.log" - log.info("install_log = %s", install_log) - - virt_install(opts, install_log, disk_img, disk_size) - except InstallError as e: - log.error("Install failed: {0}".format(e)) - if not opts.keep_image: - log.info("Removing bad disk image") - os.unlink(disk_img) - raise - - log.info("Disk Image install successful") - return disk_img - - -def make_live_images(opts, work_dir, root_dir, rootfs_image=None, size=None): - """ - Create live images from direcory or rootfs image - - :param opts: options passed to livemedia-creator - :type opts: argparse options - :param str work_dir: Directory for storing results - :param str root_dir: Root directory of live filesystem tree - :param str rootfs_image: Path to live rootfs image to be used - :returns: Path of directory with created images - :rtype: str - """ - sys_root = "" - if opts.ostree: - sys_root = find_ostree_root(root_dir) - - squashfs_root_dir = joinpaths(work_dir, "squashfs_root") - liveos_dir = joinpaths(squashfs_root_dir, "LiveOS") - os.makedirs(liveos_dir) - - if rootfs_image: - rc = execWithRedirect("/bin/ln", [rootfs_image, joinpaths(liveos_dir, "rootfs.img")]) - if rc != 0: - shutil.copy2(rootfs_image, joinpaths(liveos_dir, "rootfs.img")) - else: - log.info("Creating live rootfs image") - mkrootfsimg(root_dir, joinpaths(liveos_dir, "rootfs.img"), "LiveOS", size=size, sysroot=sys_root) - - log.info("Packing live rootfs image") - add_pxe_args = [] - live_image_name = "live-rootfs.squashfs.img" - mksquashfs(squashfs_root_dir, - joinpaths(work_dir, live_image_name), - opts.compression, - opts.compress_args) - - remove(squashfs_root_dir) - - log.info("Rebuilding initramfs for live") - rebuild_initrds_for_live(opts, joinpaths(root_dir, sys_root), work_dir) - - if opts.ostree: - add_pxe_args.append("ostree=/%s" % sys_root) - template = joinpaths(opts.lorax_templates, "pxe-live/pxe-config.tmpl") - create_pxe_config(template, work_dir, live_image_name, add_pxe_args) - - return work_dir - - -def setup_logging(opts): - # Setup logging to console and to logfile - log.setLevel(logging.DEBUG) - pylorax_log.setLevel(logging.DEBUG) - - sh = logging.StreamHandler() - sh.setLevel(logging.INFO) - fmt = logging.Formatter("%(asctime)s: %(message)s") - sh.setFormatter(fmt) - log.addHandler(sh) - pylorax_log.addHandler(sh) - - fh = logging.FileHandler(filename=opts.logfile, mode="w") - fh.setLevel(logging.DEBUG) - fmt = logging.Formatter("%(asctime)s %(levelname)s %(name)s: %(message)s") - fh.setFormatter(fmt) - log.addHandler(fh) - pylorax_log.addHandler(fh) - - # External program output log - program_log.setLevel(logging.DEBUG) - logfile = os.path.abspath(os.path.dirname(opts.logfile))+"/program.log" - fh = logging.FileHandler(filename=logfile, mode="w") - fh.setLevel(logging.DEBUG) - program_log.addHandler(fh) - - -def default_image_name(compression, basename): - """ Return a default image name with the correct suffix for the compression type. - - :param str compression: Compression type - :param str basename: Base filename - :returns: basename with compression suffix - - If the compression is unknown it defaults to xz - """ - SUFFIXES = {"xz": ".xz", "gzip": ".gz", "bzip2": ".bz2", "lzma": ".lzma"} - return basename + SUFFIXES.get(compression, ".xz") - - -if __name__ == '__main__': parser = argparse.ArgumentParser( description="Create Live Install Media", fromfile_prefix_chars="@" ) @@ -1174,7 +196,52 @@ if __name__ == '__main__': parser.add_argument("-V", help="show program's version number and exit", action="version", version=VERSION) + return parser + + +def setup_logging(opts): + # Setup logging to console and to logfile + log.setLevel(logging.DEBUG) + pylorax_log.setLevel(logging.DEBUG) + + sh = logging.StreamHandler() + sh.setLevel(logging.INFO) + fmt = logging.Formatter("%(asctime)s: %(message)s") + sh.setFormatter(fmt) + log.addHandler(sh) + pylorax_log.addHandler(sh) + + fh = logging.FileHandler(filename=opts.logfile, mode="w") + fh.setLevel(logging.DEBUG) + fmt = logging.Formatter("%(asctime)s %(levelname)s %(name)s: %(message)s") + fh.setFormatter(fmt) + log.addHandler(fh) + pylorax_log.addHandler(fh) + + # External program output log + program_log.setLevel(logging.DEBUG) + logfile = os.path.abspath(os.path.dirname(opts.logfile))+"/program.log" + fh = logging.FileHandler(filename=logfile, mode="w") + fh.setLevel(logging.DEBUG) + program_log.addHandler(fh) + + +def default_image_name(compression, basename): + """ Return a default image name with the correct suffix for the compression type. + + :param str compression: Compression type + :param str basename: Base filename + :returns: basename with compression suffix + + If the compression is unknown it defaults to xz + """ + SUFFIXES = {"xz": ".xz", "gzip": ".gz", "bzip2": ".bz2", "lzma": ".lzma"} + return basename + SUFFIXES.get(compression, ".xz") + + +if __name__ == '__main__': # parse the arguments + parser = lorax_parser() opts = parser.parse_args() setup_logging(opts)