lorax/src/sbin/livemedia-creator

#!/usr/bin/python
#
# Live Media Creator
#
# Copyright (C) 2011-2014  Red Hat, Inc.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
# Author(s): Brian C. Lane <bcl@redhat.com>
#
import logging
log = logging.getLogger("livemedia-creator")
program_log = logging.getLogger("program")
pylorax_log = logging.getLogger("pylorax")

import os
import sys
import uuid
import tempfile
import subprocess
import socket
import threading
import SocketServer
from time import sleep
import shutil
import argparse
import hashlib
import re
import glob

# Use pykickstart to calculate disk image size
from pykickstart.parser import KickstartParser
from pykickstart.version import makeVersion, RHEL7

# Use Mako templates for appliance builder descriptions
from mako.template import Template
from mako.exceptions import text_error_template

# Use the Lorax treebuilder branch for iso creation
from pylorax import ArchData
from pylorax.base import DataHolder
from pylorax.treebuilder import TreeBuilder, RuntimeBuilder, udev_escape
from pylorax.treebuilder import findkernels
from pylorax.sysutils import joinpaths, remove
from pylorax.imgutils import PartitionMount, mksparse, mkext4img, loop_detach
from pylorax.imgutils import get_loop_name, dm_detach, mount, umount, Mount
from pylorax.imgutils import mksquashfs, mktar, mkrootfsimg, mkqcow2
from pylorax.executils import execWithRedirect, execWithCapture, runcmd

# no-virt mode doesn't need libvirt, so make it optional
try:
    import libvirt
except ImportError:
    libvirt = None

# Default parameters for rebuilding initramfs, override with --dracut-args
DRACUT_DEFAULT = ["--xz", "--add", "livenet dmsquash-live convertfs pollcdrom",
                  "--omit", "plymouth", "--no-hostonly", "--no-early-microcode"]

ROOT_PATH = "/mnt/sysimage/"
RUNTIME = "images/install.img"

class InstallError(Exception):
    pass


class LogRequestHandler(SocketServer.BaseRequestHandler):
    """
    Handle monitoring and saving the logfiles from the virtual install
    """
    def setup(self):
        if self.server.log_path:
            self.fp = open(self.server.log_path, "w")
        else:
            print "no log_path specified"
        self.request.settimeout(10)

    def handle(self):
        """
        Handle writing incoming data to a logfile and
        checking the logs for any Tracebacks or other errors that indicate
        that the install failed.
        """
        line = ""
        while True:
            if self.server.kill:
                break

            try:
                data = self.request.recv(4096)
                self.fp.write(data)
                self.fp.flush()

                # check the data for errors and set error flag
                # need to assemble it into lines so we can test for the error
                # string.
                while data:
                    more = data.split("\n", 1)
                    line += more[0]
                    if len(more) > 1:
                        self.iserror(line)
                        line = ""
                        data = more[1]
                    else:
                        data = None

            except socket.timeout:
                pass
            except:
                break

    def finish(self):
        self.fp.close()

    def iserror(self, line):
        """
        Check a line to see if it contains an error indicating install failure
        """
        simple_tests = ["Traceback (",
                        "Out of memory:",
                        "Call Trace:",
                        "insufficient disk space:"]
        re_tests =     [r"packaging: base repo .* not valid"]
        for t in simple_tests:
            if line.find(t) > -1:
                self.server.log_error = True
                return
        for t in re_tests:
            if re.search(t, line):
                self.server.log_error = True
                return


class LogServer(SocketServer.TCPServer):
    """
    Add path to logfile
    Add log error flag
    Add a kill switch
    """
    def __init__(self, log_path, *args, **kwargs):
        self.kill = False
        self.log_error = False
        self.log_path = log_path
        SocketServer.TCPServer.__init__(self, *args, **kwargs)

    def log_check(self):
        return self.log_error


class LogMonitor(object):
    """
    Contains all the stuff needed to setup a thread to listen to the logs
    from the virtual install
    """
    def __init__(self, log_path, host="localhost", port=0):
        """
        Fire up the thread listening for logs
        """
        self.server = LogServer(log_path, (host, port), LogRequestHandler)
        self.host, self.port = self.server.server_address
        self.log_path = log_path
        self.server_thread = threading.Thread(target=self.server.handle_request)
        self.server_thread.daemon = True
        self.server_thread.start()

    def shutdown(self):
        self.server.kill = True
        self.server_thread.join()


class IsoMountpoint(object):
    """
    Mount the iso on a temporary directory and check to make sure the
    vmlinuz and initrd.img files exist
    Check the iso for a LiveOS directory and set a flag.
    Extract the iso's label.

    initrd_path can be used to point to a boot.iso tree with a newer
    initrd.img than the iso has. The iso is still used for stage2.
    """
    def __init__( self, iso_path, initrd_path=None ):
        """ iso_path is the path to a  boot.iso
            initrd_path overrides mounting the iso for access to
                        initrd and vmlinuz.
        """
        self.label = None
        self.iso_path = iso_path
        self.initrd_path = initrd_path

        if not self.initrd_path:
            self.mount_dir = mount(self.iso_path, opts="loop")
        else:
            self.mount_dir = self.initrd_path

        self.kernel = self.mount_dir+"/isolinux/vmlinuz"
        self.initrd = self.mount_dir+"/isolinux/initrd.img"

        if os.path.isdir( self.mount_dir+"/repodata" ):
            self.repo = self.mount_dir
        else:
            self.repo = None
        self.liveos = os.path.isdir( self.mount_dir+"/LiveOS" )

        try:
            for f in [self.kernel, self.initrd]:
                if not os.path.isfile(f):
                    raise Exception("Missing file on iso: {0}".format(f))
        except:
            self.umount()
            raise

        self.get_iso_label()

    def umount( self ):
        if not self.initrd_path:
            umount(self.mount_dir)

    def get_iso_label( self ):
        """
        Get the iso's label using isoinfo
        """
        isoinfo_output = execWithCapture("isoinfo", ["-d", "-i", self.iso_path])
        log.debug( isoinfo_output )
        for line in isoinfo_output.splitlines():
            if line.startswith("Volume id: "):
                self.label = line[11:]
                return


class VirtualInstall( object ):
    """
    Run virt-install using an iso and kickstart(s)
    """
    def __init__( self, iso, ks_paths, disk_img, img_size=2, 
                  kernel_args=None, memory=1024, vnc=None, arch=None,
                  log_check=None, virtio_host="127.0.0.1", virtio_port=6080,
                  qcow2=False):
        """

        iso is an instance of IsoMountpoint
        ks_paths is a list of paths to a kickstart files. All are injected, the
                 first one is the one executed.
        disk_img is the path to a disk image (doesn't need to exist)
        img_size is the size, in GiB, of the image if it doesn't exist
        kernel_args are extra arguments to pass on the kernel cmdline
        memory is the amount of ram to assign to the virt
        vnc is passed to the --graphics command verbatim
        arch is the optional architecture to use in the virt
        log_check is a method that returns True of the log indicates an error
        virtio_host and virtio_port are used to communicate with the log monitor
        """
        self.virt_name = "LiveOS-"+str(uuid.uuid4())
        # add --graphics none later
        # add whatever serial cmds are needed later
        args = ["-n", self.virt_name,
                 "-r", str(memory),
                 "--noreboot",
                 "--noautoconsole"]

        args.append("--graphics")
        if vnc:
            args.append(vnc)
        else:
            args.append("none")

        for ks in ks_paths:
            args.append("--initrd-inject")
            args.append(ks)

        disk_opts = "path={0}".format(disk_img)
        if qcow2:
            disk_opts += ",format=qcow2"
        else:
            disk_opts += ",format=raw"
        if not os.path.isfile(disk_img):
            disk_opts += ",size={0}".format(img_size)
        args.append("--disk")
        args.append(disk_opts)

        if iso.liveos:
            disk_opts = "path={0},device=cdrom".format(iso.iso_path)
            args.append("--disk")
            args.append(disk_opts)

        extra_args = "ks=file:/{0}".format(os.path.basename(ks_paths[0]))
        if not vnc:
            extra_args += " inst.cmdline console=ttyS0"
        if kernel_args:
            extra_args += " "+kernel_args
        if iso.liveos:
            extra_args += " stage2=hd:LABEL={0}".format(udev_escape(iso.label))
        args.append("--extra-args")
        args.append(extra_args)

        args.append("--location")
        args.append(iso.mount_dir)

        channel_args = "tcp,host={0}:{1},mode=connect,target_type=virtio" \
                       ",name=org.fedoraproject.anaconda.log.0".format(
                       virtio_host, virtio_port)
        args.append("--channel")
        args.append(channel_args)

        if arch:
            args.append("--arch")
            args.append(arch)

        rc = execWithRedirect("virt-install", args)
        if rc:
            raise Exception("Problem starting virtual install")

        conn = libvirt.openReadOnly(None)
        dom = conn.lookupByName(self.virt_name)

        # TODO: If vnc has been passed, we should look up the port and print that
        # for the user at this point

        while dom.isActive() and not log_check():
            sys.stdout.write(".")
            sys.stdout.flush()
            sleep(10)
        print

        if log_check():
            log.info( "Installation error detected. See logfile." )
        else:
            log.info( "Install finished. Or at least virt shut down." )

    def destroy( self ):
        """
        Make sure the virt has been shut down and destroyed

        Could use libvirt for this instead.
        """
        log.info( "Shutting down {0}".format(self.virt_name) )
        subprocess.call(["virsh", "destroy", self.virt_name])
        subprocess.call(["virsh", "undefine", self.virt_name])

def is_image_mounted(disk_img):
    """
    Return True if the disk_img is mounted
    """
    with open("/proc/mounts") as mounts:
        for mount in mounts:
            fields = mount.split()
            if len(fields) > 2 and fields[1] == disk_img:
                return True
    return False

def find_ostree_root(phys_root):
    """
    Find root of ostree deployment

    :param str phys_root: Path to physical root
    :returns: Relative path of ostree deployment root
    :rtype: str
    :raise Exception: More than one deployment roots were found
    """
    ostree_root = ""
    ostree_sysroots = glob.glob(joinpaths(phys_root, "ostree/boot.0/*/*/0"))
    if ostree_sysroots:
        if len(ostree_sysroots) > 1:
            raise Exception("Too many deployment roots found: %s" % ostree_sysroots)
        ostree_root = os.path.relpath(ostree_sysroots[0], phys_root)
    return ostree_root

class KernelInfo(object):
    """
    Info about the kernels in boot_dir
    """
    def __init__(self, boot_dir):
        self.boot_dir = boot_dir
        self.list = self.get_kernels()
        self.arch = self.get_kernel_arch()
        log.debug("kernel_list for {0.boot_dir} = {0.list}".format(self))
        log.debug("kernel_arch is {0.arch}".format(self))

    def get_kernels(self):
        """
        Get a list of the kernels in the boot_dir

        Examine the vmlinuz-* versions and return a list of them

        Ignore any with -rescue- in them, these are dracut rescue images.
        The user shoud add
        -dracut-config-rescue
        to the kickstart to remove them, but catch it here as well.
        """
        files = os.listdir(self.boot_dir)
        return [f[8:] for f in files if f.startswith("vmlinuz-") \
                and f.find("-rescue-") == -1]

    def get_kernel_arch(self):
        """
        Get the arch of the first kernel in boot_dir

        Defaults to i386
        """
        if self.list:
            kernel_arch = self.list[0].split(".")[-1]
        else:
            kernel_arch = "i386"
        return kernel_arch


def make_appliance(disk_img, name, template, outfile, networks=None, ram=1024,
                   vcpus=1, arch=None, title="Linux", project="Linux",
                   releasever="7"):
    """
    Generate an appliance description file

    disk_img    Full path of the disk image
    name        Name of the appliance, passed to the template
    template    Full path of Mako template
    outfile     Full path of file to write, using template
    networks    List of networks from the kickstart
    ram         Ram, in MB, passed to template. Default is 1024
    vcpus       CPUs, passed to template. Default is 1
    arch        CPU architecture. Default is 'x86_64'
    title       Title, passed to template. Default is 'Linux'
    project     Project, passed to template. Default is 'Linux'
    releasever  Release version, passed to template. Default is 17
    """
    if not (disk_img and template and outfile):
        return None

    log.info("Creating appliance definition using {0}".format(template))

    if not arch:
        arch = "x86_64"

    log.info("Calculating SHA256 checksum of {0}".format(disk_img))
    sha256 = hashlib.sha256()
    with open(disk_img) as f:
        while True:
            data = f.read(1024*1024)
            if not data:
                break
            sha256.update(data)
    log.info("SHA256 of {0} is {1}".format(disk_img, sha256.hexdigest()))
    disk_info = DataHolder(name=os.path.basename(disk_img), format="raw",
                           checksum_type="sha256", checksum=sha256.hexdigest())
    try:
        result = Template(filename=template).render(disks=[disk_info], name=name,
                          arch=arch, memory=ram, vcpus=vcpus, networks=networks,
                          title=title, project=project, releasever=releasever)
    except Exception:
        log.error(text_error_template().render())
        raise

    with open(outfile, "w") as f:
        f.write(result)


def make_fsimage(diskimage, fsimage, label="Anaconda"):
    """
    Copy the / partition of a partitioned disk image to an un-partitioned
    disk image.

    diskimage is the full path to partitioned disk image with a /
    fsimage is the full path of the output fs image file
    label is the label to apply to the image. Defaults to "Anaconda"
    """
    with PartitionMount(diskimage) as img_mount:
        if not img_mount or not img_mount.mount_dir:
            return None

        log.info("Creating fsimage %s", fsimage)
        mkext4img(img_mount.mount_dir, fsimage, label=label)


def make_runtime(opts, mount_dir, work_dir):
    """
    Make the squashfs image from a directory

    Result is in work_dir+RUNTIME
    """
    kernels = KernelInfo(joinpaths(mount_dir, "boot" ))

    # Fake yum object
    fake_yum = DataHolder(conf=DataHolder(installroot=mount_dir))
    # Fake arch with only basearch set
    arch = ArchData(kernels.arch)
    # TODO: Need to get release info from someplace...
    product = DataHolder(name=opts.project, version=opts.releasever, release="",
                            variant="", bugurl="", isfinal=False)

    # This is a mounted image partition, cannot hardlink to it, so just use it
    # symlink mount_dir/images to work_dir/images so we don't run out of space
    os.makedirs(joinpaths(work_dir, "images"))

    rb = RuntimeBuilder(product, arch, fake_yum)
    log.info("Creating runtime")
    rb.create_runtime(joinpaths(work_dir, RUNTIME), size=None)

def rebuild_initrds_for_live(opts, sys_root_dir, results_dir):
    """
    Rebuild intrds for pxe live image (root=live:http://)

    :param opts: options passed to livemedia-creator
    :type opts: argparse options
    :param str sys_root_dir: Path to root of the system
    :param str results_dir: Path of directory for storing results
    """
    if not opts.dracut_args:
        dracut_args = DRACUT_DEFAULT
    else:
        dracut_args = []
        for arg in opts.dracut_args:
            dracut_args += arg.split(" ", 1)
    log.info("dracut args = {0}".format(dracut_args))

    dracut = ["dracut", "--nomdadmconf", "--nolvmconf"] + dracut_args

    kdir = "boot"
    if opts.ostree:
        kernels_dir = glob.glob(joinpaths(sys_root_dir, "boot/ostree/*"))[0]
        kdir = os.path.relpath(kernels_dir, sys_root_dir)

    kernels = [kernel for kernel in findkernels(sys_root_dir, kdir)
               if hasattr(kernel, "initrd")]
    if not kernels:
        raise Exception("No initrds found, cannot rebuild_initrds")

    # Hush some dracut warnings. TODO: bind-mount proc in place?
    open(joinpaths(sys_root_dir,"/proc/modules"),"w")

    if opts.ostree:
        # Dracut assumes to have some dirs in disk image
        # /var/tmp for temp files
        vartmp_dir = joinpaths(sys_root_dir, "var/tmp")
        if not os.path.isdir(vartmp_dir):
            os.mkdir(vartmp_dir)
        # /root (maybe not fatal)
        root_dir = joinpaths(sys_root_dir, "var/roothome")
        if not os.path.isdir(root_dir):
            os.mkdir(root_dir)
        # /tmp (maybe not fatal)
        tmp_dir = joinpaths(sys_root_dir, "sysroot/tmp")
        if not os.path.isdir(tmp_dir):
            os.mkdir(tmp_dir)

    for kernel in kernels:
        outfile = kernel.initrd.path + ".live"
        log.info("rebuilding %s", outfile)

        kver = kernel.version

        cmd = dracut + [outfile, kver]
        runcmd(cmd, root=sys_root_dir)

        new_initrd_path = joinpaths(results_dir, os.path.basename(kernel.initrd.path))
        shutil.move(joinpaths(sys_root_dir, outfile), new_initrd_path)
        os.chmod(new_initrd_path, 0644)
        shutil.copy2(joinpaths(sys_root_dir, kernel.path), results_dir)

    os.unlink(joinpaths(sys_root_dir,"/proc/modules"))

def create_pxe_config(template, images_dir, live_image_name, add_args = None):
    """
    Create template for pxe to live configuration

    :param str images_dir: Path of directory with images to be used
    :param str live_image_name: Name of live rootfs image file
    :param list add_args: Arguments to be added to initrd= pxe config
    """

    add_args = add_args or []

    kernels = [kernel for kernel in findkernels(images_dir, kdir="")
               if hasattr(kernel, "initrd")]
    if not kernels:
        return

    kernel = kernels[0]

    add_args_str = " ".join(add_args)


    try:
        result = Template(filename=template).render(kernel=kernel.path,
                          initrd=kernel.initrd.path, liveimg=live_image_name,
                          addargs=add_args_str)
    except Exception:
        log.error(text_error_template().render())
        raise

    with open (joinpaths(images_dir, "PXE_CONFIG"), "w") as f:
        f.write(result)

def make_livecd(opts, mount_dir, work_dir):
    """
    Take the content from the disk image and make a livecd out of it

    This uses wwood's squashfs live initramfs method:
     * put the real / into LiveOS/rootfs.img
     * make a squashfs of the LiveOS/rootfs.img tree
     * make a simple initramfs with the squashfs.img and /etc/cmdline in it
     * make a cpio of that tree
     * append the squashfs.cpio to a dracut initramfs for each kernel installed

    Then on boot dracut reads /etc/cmdline which points to the squashfs.img
    mounts that and then mounts LiveOS/rootfs.img as /

    """
    kernels = KernelInfo(joinpaths(mount_dir, "boot" ))

    arch = ArchData(kernels.arch)
    # TODO: Need to get release info from someplace...
    product = DataHolder(name=opts.project, version=opts.releasever, release="",
                            variant="", bugurl="", isfinal=False)

    # Link /images to work_dir/images to make the templates happy
    if os.path.islink(joinpaths(mount_dir, "images")):
        os.unlink(joinpaths(mount_dir, "images"))
    execWithRedirect("/bin/ln", ["-s", joinpaths(work_dir, "images"),
                                 joinpaths(mount_dir, "images")])

    # The templates expect the config files to be in /tmp/config_files
    # I think these should be release specific, not from lorax, but for now
    configdir = joinpaths(opts.lorax_templates,"live/config_files/")
    configdir_path = "tmp/config_files"
    fullpath = joinpaths(mount_dir, configdir_path)
    if os.path.exists(fullpath):
        remove(fullpath)
    shutil.copytree(configdir, fullpath)

    isolabel = opts.volid or "{0.name} {0.version} {1.basearch}".format(product, arch)
    if len(isolabel) > 32:
        isolabel = isolabel[:32]
        log.warn("Truncating isolabel to 32 chars: %s" % (isolabel,))

    tb = TreeBuilder(product=product, arch=arch, domacboot=opts.domacboot,
                     inroot=mount_dir, outroot=work_dir,
                     runtime=RUNTIME, isolabel=isolabel,
                     templatedir=joinpaths(opts.lorax_templates,"live/"))
    log.info( "Rebuilding initrds" )
    if not opts.dracut_args:
        dracut_args = DRACUT_DEFAULT
    else:
        dracut_args = []
        for arg in opts.dracut_args:
            dracut_args += arg.split(" ", 1)
    log.info("dracut args = {0}".format(dracut_args))
    tb.rebuild_initrds(add_args=dracut_args)
    log.info("Building boot.iso")
    tb.build()

    return work_dir

def mount_boot_part_over_root(img_mount):
    """
    Mount boot partition to /boot of root fs mounted in img_mount

    Used for OSTree so it finds deployment configurations on live rootfs

    param img_mount: object with mounted disk image root partition
    type img_mount: imgutils.PartitionMount
    """
    root_dir = img_mount.mount_dir
    is_boot_part = lambda dir: os.path.exists(dir+"/loader.0")
    tmp_mount_dir = tempfile.mkdtemp()
    sys_root = find_ostree_root(root_dir)
    sysroot_boot_dir = None
    for dev, _size in img_mount.loop_devices:
        if dev is img_mount.mount_dev:
            continue
        try:
            mount("/dev/mapper/"+dev, mnt=tmp_mount_dir)
            if is_boot_part(tmp_mount_dir):
                umount(tmp_mount_dir)
                sysroot_boot_dir = joinpaths(joinpaths(root_dir, sys_root), "boot")
                mount("/dev/mapper/"+dev, mnt=sysroot_boot_dir)
                break
            else:
                umount(tmp_mount_dir)
        except subprocess.CalledProcessError as e:
            log.debug("Looking for boot partition error: %s", e)
    remove(tmp_mount_dir)
    return sysroot_boot_dir

def novirt_install(opts, disk_img, disk_size, repo_url):
    """
    Use Anaconda to install to a disk image
    """
    import selinux

    # Set selinux to Permissive if it is Enforcing
    selinux_enforcing = False
    if selinux.is_selinux_enabled() and selinux.security_getenforce():
        selinux_enforcing = True
        selinux.security_setenforce(0)

    # Clean up /tmp/ from previous runs to prevent stale info from being used
    for path in ["/tmp/yum.repos.d/", "/tmp/yum.cache/"]:
        if os.path.isdir(path):
            shutil.rmtree(path)

    args = ["--kickstart", opts.ks[0], "--cmdline", "--repo", repo_url]
    if opts.anaconda_args:
        for arg in opts.anaconda_args:
            args += arg.split(" ", 1)
    if opts.proxy:
        args += ["--proxy", opts.proxy]
    if opts.armplatform:
        args += ["--armplatform", opts.armplatform]

    if opts.make_iso or opts.make_fsimage:
        # Make a blank fs image
        args += ["--dirinstall"]

        mkext4img(None, disk_img, label=opts.fs_label, size=disk_size * 1024**3)
        if not os.path.isdir(ROOT_PATH):
            os.mkdir(ROOT_PATH)
        mount(disk_img, opts="loop", mnt=ROOT_PATH)
    elif opts.make_tar:
        args += ["--dirinstall"]

        # Install directly into ROOT_PATH, make sure it starts clean
        if os.path.exists(ROOT_PATH):
            shutil.rmtree(ROOT_PATH)
        if not os.path.isdir(ROOT_PATH):
            os.mkdir(ROOT_PATH)
    else:
        args += ["--image", disk_img]

        # Create the sparse image
        mksparse(disk_img, disk_size * 1024**3)

    # Make sure anaconda has the right product and release
    os.environ["ANACONDA_PRODUCTNAME"] = opts.project
    os.environ["ANACONDA_PRODUCTVERSION"] = opts.releasever
    rc = execWithRedirect("anaconda", args)

    # Move the anaconda logs over to a log directory
    log_dir = os.path.abspath(os.path.dirname(opts.logfile))
    log_anaconda = joinpaths(log_dir, "anaconda")
    if not os.path.isdir(log_anaconda):
        os.mkdir(log_anaconda)
    for l in ["anaconda.log", "ifcfg.log", "program.log", "storage.log",
              "packaging.log", "yum.log"]:
        if os.path.exists("/tmp/"+l):
            shutil.copy2("/tmp/"+l, log_anaconda)
            os.unlink("/tmp/"+l)

    if opts.make_iso or opts.make_fsimage:
        umount(ROOT_PATH)
    else:
        # If anaconda failed the disk image may still be in use by dm
        execWithRedirect("anaconda-cleanup", [])
        dm_name = os.path.splitext(os.path.basename(disk_img))[0]
        dm_path = "/dev/mapper/"+dm_name
        if os.path.exists(dm_path):
            dm_detach(dm_path)
            loop_detach(get_loop_name(disk_img))

    if selinux_enforcing:
        selinux.security_setenforce(1)

    if rc:
        raise InstallError("novirt_install failed")

    if opts.make_tar:
        compress_args = []
        for arg in opts.compress_args:
            compress_args += arg.split(" ", 1)

        rc = mktar(ROOT_PATH, disk_img, opts.compression, compress_args)
        shutil.rmtree(ROOT_PATH)

        if rc:
            raise InstallError("novirt_install failed")
    elif opts.qcow2:
        log.info("Converting %s to qcow2", disk_img)
        qcow2_args = []
        for arg in opts.qcow2_args:
            qcow2_args += arg.split(" ", 1)

        # convert the image to qcow2 format
        if "-O" not in qcow2_args:
            qcow2_args.extend(["-O", "qcow2"])
        qcow2_img = tempfile.mktemp(prefix="disk", suffix=".img")
        execWithRedirect("qemu-img", ["convert"] + qcow2_args + [disk_img, qcow2_img], raise_err=True)
        execWithRedirect("mv", ["-f", qcow2_img, disk_img], raise_err=True)


def virt_install(opts, install_log, disk_img, disk_size):
    """
    Use virt-install to install to a disk image

    install_log is the path to write the log from virt-install
    disk_img is the full path to the final disk or filesystem image
    disk_size is the size of the disk to create in GiB
    """
    iso_mount = IsoMountpoint(opts.iso, opts.location)
    log_monitor = LogMonitor(install_log)

    kernel_args = ""
    if opts.kernel_args:
        kernel_args += opts.kernel_args
    if opts.proxy:
        kernel_args += " proxy="+opts.proxy

    if opts.qcow2 and not opts.make_fsimage:
        # virt-install can't take all the qcow2 options so create the image first
        qcow2_args = []
        for arg in opts.qcow2_args:
            qcow2_args += arg.split(" ", 1)

        mkqcow2(disk_img, disk_size*1024**3, qcow2_args)

    if opts.make_fsimage or opts.make_tar:
        diskimg_path = tempfile.mktemp(prefix="disk", suffix=".img")
    else:
        diskimg_path = disk_img

    virt = VirtualInstall(iso_mount, opts.ks, diskimg_path, disk_size,
                           kernel_args, opts.ram, opts.vnc, opts.arch,
                           log_check = log_monitor.server.log_check,
                           virtio_host = log_monitor.host,
                           virtio_port = log_monitor.port,
                           qcow2=opts.qcow2)

    virt.destroy()
    log_monitor.shutdown()
    iso_mount.umount()

    if log_monitor.server.log_check():
        raise InstallError("virt_install failed")

    if opts.make_fsimage:
        make_fsimage(diskimg_path, disk_img, label=opts.fs_label)
        os.unlink(diskimg_path)
    elif opts.make_tar:
        compress_args = []
        for arg in opts.compress_args:
            compress_args += arg.split(" ", 1)

        with PartitionMount(diskimg_path) as img_mount:
            if img_mount and img_mount.mount_dir:
                rc = mktar(img_mount.mount_dir, disk_img, opts.compression, compress_args)
        os.unlink(diskimg_path)

        if rc:
            raise InstallError("virt_install failed")


def make_squashfs(disk_img, work_dir, compression="xz"):
    """
    Take disk_img and put it into LiveOS/rootfs.img and squashfs this
    tree into work_dir+RUNTIME
    """
    liveos_dir = joinpaths(work_dir, "runtime/LiveOS")
    os.makedirs(liveos_dir)
    os.makedirs(os.path.dirname(joinpaths(work_dir, RUNTIME)))

    rc = execWithRedirect("/bin/ln", [disk_img, joinpaths(liveos_dir, "rootfs.img")])
    if rc != 0:
        shutil.copy2(disk_img, joinpaths(liveos_dir, "rootfs.img"))

    mksquashfs(joinpaths(work_dir, "runtime"),
               joinpaths(work_dir, RUNTIME), compression)
    remove(joinpaths(work_dir, "runtime"))


def make_image(opts, ks):
    """
    Install to an image

    Use virt or anaconda to install to an image.

    Returns the full path of of the image created.
    """
    disk_size = 1 + (sum([p.size for p in ks.handler.partition.partitions]) / 1024)
    log.info("disk_size = %sGB", disk_size)

    if opts.image_name:
        disk_img = joinpaths(opts.result_dir, opts.image_name)
    else:
        disk_img = tempfile.mktemp(prefix="disk", suffix=".img", dir=opts.result_dir)
    log.info("disk_img = %s", disk_img)

    try:
        if opts.no_virt:
            novirt_install(opts, disk_img, disk_size, ks.handler.method.url)
        else:
            install_log = os.path.abspath(os.path.dirname(opts.logfile))+"/virt-install.log"
            log.info("install_log = %s", install_log)

            virt_install(opts, install_log, disk_img, disk_size)
    except InstallError as e:
        log.error("Install failed: {0}".format(e))
        if not opts.keep_image:
            log.info("Removing bad disk image")
            os.unlink(disk_img)
        raise

    log.info("Disk Image install successful")
    return disk_img


def make_live_images(opts, work_dir, root_dir, rootfs_image=None, size=None):
    """
    Create live images from direcory or rootfs image

    :param opts: options passed to livemedia-creator
    :type opts: argparse options
    :param str work_dir: Directory for storing results
    :param str root_dir: Root directory of live filesystem tree
    :param str rootfs_image: Path to live rootfs image to be used
    :returns: Path of directory with created images
    :rtype: str
    """
    sys_root = ""
    if opts.ostree:
        sys_root = find_ostree_root(root_dir)

    squashfs_root_dir = joinpaths(work_dir, "squashfs_root")
    liveos_dir = joinpaths(squashfs_root_dir, "LiveOS")
    os.makedirs(liveos_dir)

    if rootfs_image:
        rc = execWithRedirect("/bin/ln", [rootfs_image, joinpaths(liveos_dir, "rootfs.img")])
        if rc != 0:
            shutil.copy2(rootfs_image, joinpaths(liveos_dir, "rootfs.img"))
    else:
        log.info("Creating live rootfs image")
        mkrootfsimg(root_dir, joinpaths(liveos_dir, "rootfs.img"), "LiveOS", size=size, sysroot=sys_root)

    log.info("Packing live rootfs image")
    add_pxe_args = []
    live_image_name = "live-rootfs.squashfs.img"
    mksquashfs(squashfs_root_dir,
                joinpaths(work_dir, live_image_name),
                opts.compression,
                opts.compress_args)

    remove(squashfs_root_dir)

    log.info("Rebuilding initramfs for live")
    rebuild_initrds_for_live(opts, joinpaths(root_dir, sys_root), work_dir)

    if opts.ostree:
        add_pxe_args.append("ostree=/%s" % sys_root)
    template = joinpaths(opts.lorax_templates, "pxe-live/pxe-config.tmpl")
    create_pxe_config(template, work_dir, live_image_name, add_pxe_args)

    return work_dir


def setup_logging(opts):
    # Setup logging to console and to logfile
    log.setLevel(logging.DEBUG)
    pylorax_log.setLevel(logging.DEBUG)

    sh = logging.StreamHandler()
    sh.setLevel(logging.INFO)
    fmt = logging.Formatter("%(asctime)s: %(message)s")
    sh.setFormatter(fmt)
    log.addHandler(sh)
    pylorax_log.addHandler(sh)

    fh = logging.FileHandler(filename=opts.logfile, mode="w")
    fh.setLevel(logging.DEBUG)
    fmt = logging.Formatter("%(asctime)s %(levelname)s %(name)s: %(message)s")
    fh.setFormatter(fmt)
    log.addHandler(fh)
    pylorax_log.addHandler(fh)

    # External program output log
    program_log.setLevel(logging.DEBUG)
    logfile = os.path.abspath(os.path.dirname(opts.logfile))+"/program.log"
    fh = logging.FileHandler(filename=logfile, mode="w")
    fh.setLevel(logging.DEBUG)
    program_log.addHandler(fh)


def default_image_name(compression, basename):
    """ Return a default image name with the correct suffix for the compression type.

    :param str compression: Compression type
    :param str basename: Base filename
    :returns: basename with compression suffix

    If the compression is unknown it defaults to xz
    """
    SUFFIXES = {"xz": ".xz", "gzip": ".gz", "bzip2": ".bz2", "lzma": ".lzma"}
    return basename + SUFFIXES.get(compression, ".xz")


if __name__ == '__main__':
    parser = argparse.ArgumentParser( description="Create Live Install Media",
                                      fromfile_prefix_chars="@" )

    # These are mutually exclusive, one is required
    action = parser.add_mutually_exclusive_group( required=True )
    action.add_argument( "--make-iso", action="store_true",
                         help="Build a live iso" )
    action.add_argument( "--make-disk", action="store_true",
                         help="Build a partitioned disk image" )
    action.add_argument( "--make-fsimage", action="store_true",
                         help="Build a filesystem image" )
    action.add_argument( "--make-appliance", action="store_true",
                         help="Build an appliance image and XML description" )
    action.add_argument( "--make-ami", action="store_true",
                         help="Build an ami image" )
    action.add_argument( "--make-tar", action="store_true",
                         help="Build a tar of the root filesystem" )
    action.add_argument( "--make-pxe-live", action="store_true",
                         help="Build a live pxe boot squashfs image" )
    action.add_argument( "--make-ostree-live", action="store_true",
                         help="Build a live pxe boot squashfs image of Atomic Host" )


    parser.add_argument( "--iso", type=os.path.abspath,
                        help="Anaconda installation .iso path to use for virt-install" )
    parser.add_argument( "--fs-label", default="Anaconda",
                        help="Label to set on fsimage, default is 'Anaconda'")
    parser.add_argument("--compression", default="xz",
                        help="Compression binary for make-tar. xz, lzma, gzip, and bzip2 are supported. xz is the default.")
    parser.add_argument("--compress-arg", action="append", dest="compress_args", default=[],
                        help="Arguments to pass to compression. Pass once for each argument")

    parser.add_argument( "--ks", action="append", type=os.path.abspath,
                         help="Kickstart file defining the install." )
    parser.add_argument( "--image-only", action="store_true",
                         help="Exit after creating fs/disk image." )

    parser.add_argument( "--no-virt", action="store_true",
                         help="Use Anaconda's image install instead of virt-install" )
    parser.add_argument( "--proxy",
                         help="proxy URL to use for the install" )
    parser.add_argument( "--anaconda-arg", action="append", dest="anaconda_args",
                         help="Additional argument to pass to anaconda (no-virt "
                              "mode). Pass once for each argument" )
    parser.add_argument( "--armplatform",
                         help="the platform to use when creating images for ARM, "
                              "i.e., highbank, mvebu, omap, tegra, etc." )
    parser.add_argument( "--location", default=None, type=os.path.abspath,
                         help="location of iso directory tree with initrd.img "
                              "and vmlinuz. Used to run virt-install with a "
                              "newer initrd than the iso." )

    parser.add_argument( "--logfile", default="./livemedia.log",
                         type=os.path.abspath,
                         help="Path to logfile" )
    parser.add_argument( "--lorax-templates", default="/usr/share/lorax/",
                         type=os.path.abspath,
                         help="Path to mako templates for lorax" )
    parser.add_argument( "--tmp", default="/var/tmp", type=os.path.abspath,
                         help="Top level temporary directory" )
    parser.add_argument( "--resultdir", default=None, dest="result_dir",
                         type=os.path.abspath,
                         help="Directory to copy the resulting images and iso into. "
                              "Defaults to the temporary working directory")

    parser.add_argument( "--macboot", action="store_true",
                         dest="domacboot")
    parser.add_argument( "--nomacboot", action="store_false", default=False,
                         dest="domacboot")

    image_group = parser.add_argument_group("disk/fs image arguments")
    image_group.add_argument( "--disk-image", type=os.path.abspath,
                        help="Path to disk image to use for creating final image" )
    image_group.add_argument( "--keep-image", action="store_true",
                         help="Keep raw disk image after .iso creation" )
    image_group.add_argument( "--fs-image", type=os.path.abspath,
                        help="Path to filesystem image to use for creating final image" )
    image_group.add_argument( "--image-name", default=None,
                         help="Name of fs/disk image to create. Default is a random name." )
    image_group.add_argument("--qcow2", action="store_true",
                        help="create qcow2 image instead of raw sparse image")
    image_group.add_argument("--qcow2-arg", action="append", dest="qcow2_args", default=[],
                        help="Arguments to pass to qemu-img. Pass once for each argument")

    # Group of arguments for appliance creation
    app_group = parser.add_argument_group("appliance arguments")
    app_group.add_argument( "--app-name", default=None,
                            help="Name of appliance to pass to template")
    app_group.add_argument( "--app-template", default=None,
                         help="Path to template to use for appliance data.")
    app_group.add_argument( "--app-file", default="appliance.xml",
                         help="Appliance template results file.")

    # Group of arguments to pass to virt-install
    if not libvirt:
        virt_group = parser.add_argument_group("virt-install arguments (DISABLED -- no libvirt)")
    else:
        virt_group = parser.add_argument_group("virt-install arguments")
    virt_group.add_argument("--ram", metavar="MEMORY", type=int, default=1024,
                            help="Memory to allocate for installer in megabytes." )
    virt_group.add_argument("--vcpus", default=1,
                            help="Passed to --vcpus command" )
    virt_group.add_argument("--vnc",
                            help="Passed to --graphics command" )
    virt_group.add_argument("--arch", default=None,
                            help="Passed to --arch command" )
    virt_group.add_argument( "--kernel-args",
                             help="Additional argument to pass to the installation kernel" )

    # dracut arguments
    dracut_group = parser.add_argument_group( "dracut arguments" )
    dracut_group.add_argument( "--dracut-arg", action="append", dest="dracut_args",
                               help="Argument to pass to dracut when "
                                    "rebuilding the initramfs. Pass this "
                                    "once for each argument. NOTE: this "
                                    "overrides the default. (default: %s)" % (DRACUT_DEFAULT,) )

    # pxe to live arguments
    pxelive_group = parser.add_argument_group( "pxe to live arguments" )
    pxelive_group.add_argument( "--live-rootfs-size", type=int, default=0,
                                 help="Size of root filesystem of live image in GiB" )
    pxelive_group.add_argument( "--live-rootfs-keep-size", action="store_true",
                                help="Keep the original size of root filesystem in live image " )


    parser.add_argument( "--title", default="Linux Live Media",
                         help="Substituted for @TITLE@ in bootloader config files" )
    parser.add_argument( "--project", default="Red Hat Enterprise Linux",
                         help="substituted for @PROJECT@ in bootloader config files" )
    parser.add_argument( "--releasever", default="7",
                         help="substituted for @VERSION@ in bootloader config files" )
    parser.add_argument( "--volid", default=None, help="volume id")
    parser.add_argument( "--squashfs_args",
                         help="additional squashfs args" )

    opts = parser.parse_args()

    setup_logging(opts)

    log.debug( opts )

    if os.getuid() != 0:
        log.error("You need to run this as root")
        sys.exit( 1 )

    if opts.make_iso and not os.path.exists( opts.lorax_templates ):
        log.error( "The lorax templates directory ({0}) doesn't"
                   " exist.".format( opts.lorax_templates ) )
        sys.exit( 1 )

    if opts.result_dir and os.path.exists(opts.result_dir):
        log.error( "The results_dir ({0}) should not exist, please delete or "
                   "move its contents".format( opts.result_dir ))
        sys.exit( 1 )

    # Default to putting results into tmp
    if not opts.result_dir:
        opts.result_dir = opts.tmp
    else:
        os.makedirs(opts.result_dir)

    if opts.iso and not os.path.exists( opts.iso ):
        log.error( "The iso {0} is missing.".format( opts.iso ) )
        sys.exit( 1 )

    if opts.disk_image and not os.path.exists( opts.disk_image ):
        log.error( "The disk image {0} is missing.".format( opts.disk_image ) )
        sys.exit( 1 )

    if opts.fs_image and not os.path.exists( opts.fs_image ):
        log.error( "The filesystem image {0} is missing.".format( opts.fs_image ) )
        sys.exit( 1 )

    is_install = not (opts.disk_image or opts.fs_image)
    if is_install and not opts.no_virt and not opts.iso:
        log.error( "virt install needs an install iso." )
        sys.exit( 1 )

    if opts.volid and len(opts.volid) > 32:
        log.fatal("the volume id cannot be longer than 32 characters")
        sys.exit(1)

    if is_install and not opts.no_virt and not libvirt:
        log.error("virt install requires libvirt-python to be installed.")
        sys.exit(1)

    if is_install and not opts.no_virt \
       and not os.path.exists("/usr/bin/virt-install"):
        log.error("virt-install needs to be installed.")
        sys.exit(1)

    if is_install and opts.no_virt \
       and not os.path.exists("/usr/sbin/anaconda"):
        log.error("no-virt requires anaconda to be installed.")
        sys.exit(1)

    if opts.make_appliance and not opts.app_template:
        opts.app_template = joinpaths(opts.lorax_templates,
                                            "appliance/libvirt.tmpl")

    if opts.make_appliance and not os.path.exists(opts.app_template):
        log.error("The appliance template ({0}) doesn't "
                  "exist".format(opts.app_template))
        sys.exit(1)

    if opts.image_name and os.path.exists(joinpaths(opts.result_dir, opts.image_name)):
        log.error("The disk image to be created should not exist.")
        sys.exit(1)

    if opts.qcow2 and not os.path.exists("/usr/bin/qemu-img"):
        log.error("qcow2 requires the qemu-img utility to be installed.")
        sys.exit(1)

    if opts.qcow2 and opts.make_iso:
        log.error("qcow2 cannot be used to make a bootable iso.")
        sys.exit(1)

    # AMI image is just a fsimage with an AMI label
    if opts.make_ami:
        opts.make_fsimage = True
        if not opts.image_name:
            opts.image_name = "ami-root.img"
        if opts.fs_label == "Anaconda":
            opts.fs_label = "AMI"
    elif opts.make_tar:
        if not opts.image_name:
            opts.image_name = default_image_name(opts.compression, "root.tar")
        if opts.compression == "xz" and not opts.compress_args:
            opts.compress_args = ["-9"]

    if opts.app_file:
        opts.app_file = joinpaths(opts.result_dir, opts.app_file)

    if opts.make_ostree_live:
        opts.make_pxe_live = True
        opts.ostree = True
    else:
        opts.ostree = False

    tempfile.tempdir = opts.tmp
    disk_img = None

    # Parse the kickstart
    if opts.ks:
        ks_version = makeVersion(RHEL7)
        ks = KickstartParser( ks_version, errorsAreFatal=False, missingIncludeIsFatal=False )
        ks.readKickstart( opts.ks[0] )

    # Make the disk or filesystem image
    if not opts.disk_image and not opts.fs_image:
        if not opts.ks:
            log.error("Image creation requires a kickstart file")
            sys.exit(1)

        errors = []
        if ks.handler.method.method != "url" and opts.no_virt:
            errors.append("Only url install method is currently supported. Please "
                          "fix your kickstart file." )

        if ks.handler.displaymode.displayMode is not None:
            errors.append("The kickstart must not set a display mode (text, cmdline, "
                          "graphical), this will interfere with livemedia-creator.")

        if opts.make_fsimage:
            # Make sure the kickstart isn't using autopart and only has a / mountpoint
            part_ok = not any(p for p in ks.handler.partition.partitions
                                 if p.mountpoint not in ["/", "swap"])
            if not part_ok or ks.handler.autopart.seen:
                errors.append("Filesystem images must use a single / part, not autopart or "
                              "multiple partitions. swap is allowed but not used.")

        if errors:
            for e in errors:
                log.error(e)
            sys.exit(1)

        # Make the image. Output of this is either a partitioned disk image or a fsimage
        try:
            disk_img = make_image(opts, ks)
        except InstallError as e:
            log.error("ERROR: Image creation failed: %s" % (e))
            sys.exit(1)

    if not opts.image_only:
        result_dir = None
        if opts.make_iso:
            work_dir = tempfile.mkdtemp()
            log.info("working dir is {0}".format(work_dir))

            if (opts.fs_image or opts.no_virt) and not opts.disk_image:
                # Create iso from a filesystem image
                disk_img = opts.fs_image or disk_img

                make_squashfs(disk_img, work_dir)
                with Mount(disk_img, opts="loop") as mount_dir:
                    result_dir = make_livecd(opts, mount_dir, work_dir)
            else:
                # Create iso from a partitioned disk image
                disk_img = opts.disk_image or disk_img
                with PartitionMount(disk_img) as img_mount:
                    if img_mount and img_mount.mount_dir:
                        make_runtime(opts, img_mount.mount_dir, work_dir)
                        result_dir = make_livecd(opts, img_mount.mount_dir, work_dir)

            # cleanup the mess
            # cleanup work_dir?
            if disk_img and not (opts.keep_image or opts.disk_image or opts.fs_image):
                os.unlink(disk_img)
                log.info("Disk image erased")
                disk_img = None
        elif opts.make_appliance:
            if not opts.ks:
                networks = []
            else:
                networks = ks.handler.network.network
            make_appliance(opts.disk_image or disk_img, opts.app_name,
                           opts.app_template, opts.app_file, networks, opts.ram,
                           opts.vcpus, opts.arch, opts.title, opts.project, opts.releasever)
        elif opts.make_pxe_live:
            work_dir = tempfile.mkdtemp()
            log.info("working dir is {0}".format(work_dir))

            if (opts.fs_image or opts.no_virt) and not opts.disk_image:
                # Create pxe live images from a filesystem image
                disk_img = opts.fs_image or disk_img
                with Mount(disk_img, opts="loop") as mnt_dir:
                    result_dir = make_live_images(opts, work_dir, mnt_dir, rootfs_image=disk_img)
            else:
                # Create pxe live images from a partitioned disk image
                disk_img = opts.disk_image or disk_img
                is_root_part = None
                if opts.ostree:
                    is_root_part = lambda dir: os.path.exists(dir+"/ostree/deploy")
                with PartitionMount(disk_img, mount_ok=is_root_part) as img_mount:
                    if img_mount and img_mount.mount_dir:
                        try:
                            mounted_sysroot_boot_dir = None
                            if opts.ostree:
                                mounted_sysroot_boot_dir = mount_boot_part_over_root(img_mount)
                            if opts.live_rootfs_keep_size:
                                size = img_mount.mount_size / 1024**3
                            else:
                                size = opts.live_rootfs_size or None
                            result_dir = make_live_images(opts, work_dir, img_mount.mount_dir, size=size)
                        finally:
                            if mounted_sysroot_boot_dir:
                                umount(mounted_sysroot_boot_dir)

        if opts.result_dir != opts.tmp and result_dir:
            shutil.copytree( result_dir, opts.result_dir )
            shutil.rmtree( result_dir )

    log.info("SUMMARY")
    log.info("-------")
    log.info("Logs are in {0}".format(os.path.abspath(os.path.dirname(opts.logfile))))
    if disk_img:
        log.info("Disk image is at {0}".format(disk_img))
    if opts.make_appliance:
        log.info("Appliance description is in {0}".format(opts.app_file))
    log.info("Results are in {0}".format(opts.result_dir))

    sys.exit( 0 )