livemedia-creator: Move core functions into pylorax modules

This reduces the amount of code in livemedia-creator to the cmdline
parsing and calling of the installer functions. Moving them into other
modules will allow them to be used by other projects, like the
lorax-composer API server.

src/pylorax/creator.py

@@ -0,0 +1,479 @@
+#
+# Copyright (C) 2011-2017  Red Hat, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+import logging
+log = logging.getLogger("pylorax")
+
+import os
+import sys
+import tempfile
+import subprocess
+import threading
+import shutil
+import hashlib
+import re
+import glob
+
+# Use Mako templates for appliance builder descriptions
+from mako.template import Template
+from mako.exceptions import text_error_template
+
+# Use the Lorax treebuilder branch for iso creation
+from pylorax import ArchData, vernum
+from pylorax.base import DataHolder
+from pylorax.treebuilder import TreeBuilder, RuntimeBuilder, udev_escape
+from pylorax.treebuilder import findkernels
+from pylorax.sysutils import joinpaths, remove
+from pylorax.imgutils import mount, umount, Mount
+from pylorax.imgutils import mksquashfs, mkrootfsimg
+from pylorax.imgutils import copytree
+from pylorax.executils import execWithRedirect, execWithCapture, runcmd
+from pylorax.installer import InstallError, novirt_install, virt_install
+
+RUNTIME = "images/install.img"
+
+# Default parameters for rebuilding initramfs, override with --dracut-args
+DRACUT_DEFAULT = ["--xz", "--add", "livenet dmsquash-live convertfs pollcdrom",
+                  "--omit", "plymouth", "--no-hostonly", "--no-early-microcode"]
+
+
+def is_image_mounted(disk_img):
+    """
+    Return True if the disk_img is mounted
+    """
+    with open("/proc/mounts") as mounts:
+        for mount in mounts:
+            fields = mount.split()
+            if len(fields) > 2 and fields[1] == disk_img:
+                return True
+    return False
+
+def find_ostree_root(phys_root):
+    """
+    Find root of ostree deployment
+
+    :param str phys_root: Path to physical root
+    :returns: Relative path of ostree deployment root
+    :rtype: str
+    :raise Exception: More than one deployment roots were found
+    """
+    ostree_root = ""
+    ostree_sysroots = glob.glob(joinpaths(phys_root, "ostree/boot.0/*/*/0"))
+    if ostree_sysroots:
+        if len(ostree_sysroots) > 1:
+            raise Exception("Too many deployment roots found: %s" % ostree_sysroots)
+        ostree_root = os.path.relpath(ostree_sysroots[0], phys_root)
+    return ostree_root
+
+class KernelInfo(object):
+    """
+    Info about the kernels in boot_dir
+    """
+    def __init__(self, boot_dir):
+        self.boot_dir = boot_dir
+        self.list = self.get_kernels()
+        self.arch = self.get_kernel_arch()
+        log.debug("kernel_list for {0.boot_dir} = {0.list}".format(self))
+        log.debug("kernel_arch is {0.arch}".format(self))
+
+    def get_kernels(self):
+        """
+        Get a list of the kernels in the boot_dir
+
+        Examine the vmlinuz-* versions and return a list of them
+
+        Ignore any with -rescue- in them, these are dracut rescue images.
+        The user shoud add
+        -dracut-config-rescue
+        to the kickstart to remove them, but catch it here as well.
+        """
+        files = os.listdir(self.boot_dir)
+        return [f[8:] for f in files if f.startswith("vmlinuz-") \
+                and f.find("-rescue-") == -1]
+
+    def get_kernel_arch(self):
+        """
+        Get the arch of the first kernel in boot_dir
+
+        Defaults to i386
+        """
+        if self.list:
+            kernel_arch = self.list[0].split(".")[-1]
+        else:
+            kernel_arch = "i386"
+        return kernel_arch
+
+
+def make_appliance(disk_img, name, template, outfile, networks=None, ram=1024,
+                   vcpus=1, arch=None, title="Linux", project="Linux",
+                   releasever="7"):
+    """
+    Generate an appliance description file
+
+    disk_img    Full path of the disk image
+    name        Name of the appliance, passed to the template
+    template    Full path of Mako template
+    outfile     Full path of file to write, using template
+    networks    List of networks from the kickstart
+    ram         Ram, in MB, passed to template. Default is 1024
+    vcpus       CPUs, passed to template. Default is 1
+    arch        CPU architecture. Default is 'x86_64'
+    title       Title, passed to template. Default is 'Linux'
+    project     Project, passed to template. Default is 'Linux'
+    releasever  Release version, passed to template. Default is 17
+    """
+    if not (disk_img and template and outfile):
+        return None
+
+    log.info("Creating appliance definition using {0}".format(template))
+
+    if not arch:
+        arch = "x86_64"
+
+    log.info("Calculating SHA256 checksum of {0}".format(disk_img))
+    sha256 = hashlib.sha256()
+    with open(disk_img) as f:
+        while True:
+            data = f.read(1024*1024)
+            if not data:
+                break
+            sha256.update(data)
+    log.info("SHA256 of {0} is {1}".format(disk_img, sha256.hexdigest()))
+    disk_info = DataHolder(name=os.path.basename(disk_img), format="raw",
+                           checksum_type="sha256", checksum=sha256.hexdigest())
+    try:
+        result = Template(filename=template).render(disks=[disk_info], name=name,
+                          arch=arch, memory=ram, vcpus=vcpus, networks=networks,
+                          title=title, project=project, releasever=releasever)
+    except Exception:
+        log.error(text_error_template().render())
+        raise
+
+    with open(outfile, "w") as f:
+        f.write(result)
+
+
+def make_runtime(opts, mount_dir, work_dir):
+    """
+    Make the squashfs image from a directory
+
+    Result is in work_dir+RUNTIME
+    """
+    kernels = KernelInfo(joinpaths(mount_dir, "boot" ))
+
+    # Fake yum object
+    fake_yum = DataHolder(conf=DataHolder(installroot=mount_dir))
+    # Fake arch with only basearch set
+    arch = ArchData(kernels.arch)
+    # TODO: Need to get release info from someplace...
+    product = DataHolder(name=opts.project, version=opts.releasever, release="",
+                            variant="", bugurl="", isfinal=False)
+
+    # This is a mounted image partition, cannot hardlink to it, so just use it
+    # symlink mount_dir/images to work_dir/images so we don't run out of space
+    os.makedirs(joinpaths(work_dir, "images"))
+
+    rb = RuntimeBuilder(product, arch, fake_yum)
+    log.info("Creating runtime")
+    rb.create_runtime(joinpaths(work_dir, RUNTIME), size=None)
+
+def rebuild_initrds_for_live(opts, sys_root_dir, results_dir):
+    """
+    Rebuild intrds for pxe live image (root=live:http://)
+
+    :param opts: options passed to livemedia-creator
+    :type opts: argparse options
+    :param str sys_root_dir: Path to root of the system
+    :param str results_dir: Path of directory for storing results
+    """
+    if not opts.dracut_args:
+        dracut_args = DRACUT_DEFAULT
+    else:
+        dracut_args = []
+        for arg in opts.dracut_args:
+            dracut_args += arg.split(" ", 1)
+    log.info("dracut args = {0}".format(dracut_args))
+
+    dracut = ["dracut", "--nomdadmconf", "--nolvmconf"] + dracut_args
+
+    kdir = "boot"
+    if opts.ostree:
+        kernels_dir = glob.glob(joinpaths(sys_root_dir, "boot/ostree/*"))[0]
+        kdir = os.path.relpath(kernels_dir, sys_root_dir)
+
+    kernels = [kernel for kernel in findkernels(sys_root_dir, kdir)
+               if hasattr(kernel, "initrd")]
+    if not kernels:
+        raise Exception("No initrds found, cannot rebuild_initrds")
+
+    # Hush some dracut warnings. TODO: bind-mount proc in place?
+    open(joinpaths(sys_root_dir,"/proc/modules"),"w")
+
+    if opts.ostree:
+        # Dracut assumes to have some dirs in disk image
+        # /var/tmp for temp files
+        vartmp_dir = joinpaths(sys_root_dir, "var/tmp")
+        if not os.path.isdir(vartmp_dir):
+            os.mkdir(vartmp_dir)
+        # /root (maybe not fatal)
+        root_dir = joinpaths(sys_root_dir, "var/roothome")
+        if not os.path.isdir(root_dir):
+            os.mkdir(root_dir)
+        # /tmp (maybe not fatal)
+        tmp_dir = joinpaths(sys_root_dir, "sysroot/tmp")
+        if not os.path.isdir(tmp_dir):
+            os.mkdir(tmp_dir)
+
+    for kernel in kernels:
+        outfile = kernel.initrd.path + ".live"
+        log.info("rebuilding %s", outfile)
+
+        kver = kernel.version
+
+        cmd = dracut + [outfile, kver]
+        runcmd(cmd, root=sys_root_dir)
+
+        new_initrd_path = joinpaths(results_dir, os.path.basename(kernel.initrd.path))
+        shutil.move(joinpaths(sys_root_dir, outfile), new_initrd_path)
+        os.chmod(new_initrd_path, 0644)
+        shutil.copy2(joinpaths(sys_root_dir, kernel.path), results_dir)
+
+    os.unlink(joinpaths(sys_root_dir,"/proc/modules"))
+
+def create_pxe_config(template, images_dir, live_image_name, add_args = None):
+    """
+    Create template for pxe to live configuration
+
+    :param str images_dir: Path of directory with images to be used
+    :param str live_image_name: Name of live rootfs image file
+    :param list add_args: Arguments to be added to initrd= pxe config
+    """
+
+    add_args = add_args or []
+
+    kernels = [kernel for kernel in findkernels(images_dir, kdir="")
+               if hasattr(kernel, "initrd")]
+    if not kernels:
+        return
+
+    kernel = kernels[0]
+
+    add_args_str = " ".join(add_args)
+
+
+    try:
+        result = Template(filename=template).render(kernel=kernel.path,
+                          initrd=kernel.initrd.path, liveimg=live_image_name,
+                          addargs=add_args_str)
+    except Exception:
+        log.error(text_error_template().render())
+        raise
+
+    with open (joinpaths(images_dir, "PXE_CONFIG"), "w") as f:
+        f.write(result)
+
+def make_livecd(opts, mount_dir, work_dir):
+    """
+    Take the content from the disk image and make a livecd out of it
+
+    This uses wwood's squashfs live initramfs method:
+     * put the real / into LiveOS/rootfs.img
+     * make a squashfs of the LiveOS/rootfs.img tree
+     * make a simple initramfs with the squashfs.img and /etc/cmdline in it
+     * make a cpio of that tree
+     * append the squashfs.cpio to a dracut initramfs for each kernel installed
+
+    Then on boot dracut reads /etc/cmdline which points to the squashfs.img
+    mounts that and then mounts LiveOS/rootfs.img as /
+
+    """
+    kernels = KernelInfo(joinpaths(mount_dir, "boot" ))
+
+    arch = ArchData(kernels.arch)
+    # TODO: Need to get release info from someplace...
+    product = DataHolder(name=opts.project, version=opts.releasever, release="",
+                            variant="", bugurl="", isfinal=False)
+
+    # Link /images to work_dir/images to make the templates happy
+    if os.path.islink(joinpaths(mount_dir, "images")):
+        os.unlink(joinpaths(mount_dir, "images"))
+    execWithRedirect("/bin/ln", ["-s", joinpaths(work_dir, "images"),
+                                 joinpaths(mount_dir, "images")])
+
+    # The templates expect the config files to be in /tmp/config_files
+    # I think these should be release specific, not from lorax, but for now
+    configdir = joinpaths(opts.lorax_templates,"live/config_files/")
+    configdir_path = "tmp/config_files"
+    fullpath = joinpaths(mount_dir, configdir_path)
+    if os.path.exists(fullpath):
+        remove(fullpath)
+    shutil.copytree(configdir, fullpath)
+
+    isolabel = opts.volid or "{0.name} {0.version} {1.basearch}".format(product, arch)
+    if len(isolabel) > 32:
+        isolabel = isolabel[:32]
+        log.warn("Truncating isolabel to 32 chars: %s" % (isolabel,))
+
+    tb = TreeBuilder(product=product, arch=arch, domacboot=opts.domacboot,
+                     inroot=mount_dir, outroot=work_dir,
+                     runtime=RUNTIME, isolabel=isolabel,
+                     templatedir=joinpaths(opts.lorax_templates,"live/"))
+    log.info( "Rebuilding initrds" )
+    if not opts.dracut_args:
+        dracut_args = DRACUT_DEFAULT
+    else:
+        dracut_args = []
+        for arg in opts.dracut_args:
+            dracut_args += arg.split(" ", 1)
+    log.info("dracut args = {0}".format(dracut_args))
+    tb.rebuild_initrds(add_args=dracut_args)
+    log.info("Building boot.iso")
+    tb.build()
+
+    return work_dir
+
+def mount_boot_part_over_root(img_mount):
+    """
+    Mount boot partition to /boot of root fs mounted in img_mount
+
+    Used for OSTree so it finds deployment configurations on live rootfs
+
+    param img_mount: object with mounted disk image root partition
+    type img_mount: imgutils.PartitionMount
+    """
+    root_dir = img_mount.mount_dir
+    is_boot_part = lambda dir: os.path.exists(dir+"/loader.0")
+    tmp_mount_dir = tempfile.mkdtemp()
+    sys_root = find_ostree_root(root_dir)
+    sysroot_boot_dir = None
+    for dev, _size in img_mount.loop_devices:
+        if dev is img_mount.mount_dev:
+            continue
+        try:
+            mount("/dev/mapper/"+dev, mnt=tmp_mount_dir)
+            if is_boot_part(tmp_mount_dir):
+                umount(tmp_mount_dir)
+                sysroot_boot_dir = joinpaths(joinpaths(root_dir, sys_root), "boot")
+                mount("/dev/mapper/"+dev, mnt=sysroot_boot_dir)
+                break
+            else:
+                umount(tmp_mount_dir)
+        except subprocess.CalledProcessError as e:
+            log.debug("Looking for boot partition error: %s", e)
+    remove(tmp_mount_dir)
+    return sysroot_boot_dir
+
+def make_squashfs(disk_img, work_dir, compression="xz"):
+    """
+    Take disk_img and put it into LiveOS/rootfs.img and squashfs this
+    tree into work_dir+RUNTIME
+    """
+    liveos_dir = joinpaths(work_dir, "runtime/LiveOS")
+    os.makedirs(liveos_dir)
+    os.makedirs(os.path.dirname(joinpaths(work_dir, RUNTIME)))
+
+    rc = execWithRedirect("/bin/ln", [disk_img, joinpaths(liveos_dir, "rootfs.img")])
+    if rc != 0:
+        shutil.copy2(disk_img, joinpaths(liveos_dir, "rootfs.img"))
+
+    mksquashfs(joinpaths(work_dir, "runtime"),
+               joinpaths(work_dir, RUNTIME), compression)
+    remove(joinpaths(work_dir, "runtime"))
+
+
+def make_image(opts, ks):
+    """
+    Install to an image
+
+    Use virt or anaconda to install to an image.
+
+    Returns the full path of of the image created.
+    """
+    disk_size = 1 + (sum([p.size for p in ks.handler.partition.partitions]) / 1024)
+    log.info("disk_size = %sGB", disk_size)
+
+    if opts.image_name:
+        disk_img = joinpaths(opts.result_dir, opts.image_name)
+    else:
+        disk_img = tempfile.mktemp(prefix="disk", suffix=".img", dir=opts.result_dir)
+    log.info("disk_img = %s", disk_img)
+
+    try:
+        if opts.no_virt:
+            novirt_install(opts, disk_img, disk_size, ks.handler.method.url)
+        else:
+            install_log = os.path.abspath(os.path.dirname(opts.logfile))+"/virt-install.log"
+            log.info("install_log = %s", install_log)
+
+            virt_install(opts, install_log, disk_img, disk_size)
+    except InstallError as e:
+        log.error("Install failed: {0}".format(e))
+        if not opts.keep_image:
+            log.info("Removing bad disk image")
+            os.unlink(disk_img)
+        raise
+
+    log.info("Disk Image install successful")
+    return disk_img
+
+
+def make_live_images(opts, work_dir, root_dir, rootfs_image=None, size=None):
+    """
+    Create live images from direcory or rootfs image
+
+    :param opts: options passed to livemedia-creator
+    :type opts: argparse options
+    :param str work_dir: Directory for storing results
+    :param str root_dir: Root directory of live filesystem tree
+    :param str rootfs_image: Path to live rootfs image to be used
+    :returns: Path of directory with created images
+    :rtype: str
+    """
+    sys_root = ""
+    if opts.ostree:
+        sys_root = find_ostree_root(root_dir)
+
+    squashfs_root_dir = joinpaths(work_dir, "squashfs_root")
+    liveos_dir = joinpaths(squashfs_root_dir, "LiveOS")
+    os.makedirs(liveos_dir)
+
+    if rootfs_image:
+        rc = execWithRedirect("/bin/ln", [rootfs_image, joinpaths(liveos_dir, "rootfs.img")])
+        if rc != 0:
+            shutil.copy2(rootfs_image, joinpaths(liveos_dir, "rootfs.img"))
+    else:
+        log.info("Creating live rootfs image")
+        mkrootfsimg(root_dir, joinpaths(liveos_dir, "rootfs.img"), "LiveOS", size=size, sysroot=sys_root)
+
+    log.info("Packing live rootfs image")
+    add_pxe_args = []
+    live_image_name = "live-rootfs.squashfs.img"
+    mksquashfs(squashfs_root_dir,
+                joinpaths(work_dir, live_image_name),
+                opts.compression,
+                opts.compress_args)
+
+    remove(squashfs_root_dir)
+
+    log.info("Rebuilding initramfs for live")
+    rebuild_initrds_for_live(opts, joinpaths(root_dir, sys_root), work_dir)
+
+    if opts.ostree:
+        add_pxe_args.append("ostree=/%s" % sys_root)
+    template = joinpaths(opts.lorax_templates, "pxe-live/pxe-config.tmpl")
+    create_pxe_config(template, work_dir, live_image_name, add_pxe_args)
+
+    return work_dir

src/pylorax/imgutils.py

@@ -107,6 +107,22 @@ def mkrootfsimg(rootdir, outfile, label, size=2, sysroot=""):
             root = join(mnt, sysroot.lstrip("/"))
             runcmd(cmd, root=root)
 
+def mkdiskfsimage(diskimage, fsimage, label="Anaconda"):
+    """
+    Copy the / partition of a partitioned disk image to an un-partitioned
+    disk image.
+
+    diskimage is the full path to partitioned disk image with a /
+    fsimage is the full path of the output fs image file
+    label is the label to apply to the image. Defaults to "Anaconda"
+    """
+    with PartitionMount(diskimage) as img_mount:
+        if not img_mount or not img_mount.mount_dir:
+            return None
+
+        logger.info("Creating fsimage %s", fsimage)
+        mkext4img(img_mount.mount_dir, fsimage, label=label)
+
 ######## Utility functions ###############################################
 
 def mksparse(outfile, size):

src/pylorax/installer.py

@@ -0,0 +1,410 @@
+#
+# Copyright (C) 2011-2017  Red Hat, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+import logging
+log = logging.getLogger("pylorax")
+
+import os
+import shutil
+import sys
+import subprocess
+import tempfile
+from time import sleep
+import uuid
+
+from pylorax.executils import execWithRedirect, execWithCapture, runcmd
+from pylorax.imgutils import get_loop_name, dm_detach, mount, umount, Mount
+from pylorax.imgutils import PartitionMount, mksparse, mkext4img, loop_detach
+from pylorax.imgutils import mksquashfs, mktar, mkrootfsimg, mkdiskfsimage, mkqcow2
+from pylorax.logmonitor import LogMonitor
+from pylorax.sysutils import joinpaths, remove
+from pylorax.treebuilder import TreeBuilder, RuntimeBuilder, udev_escape
+
+ROOT_PATH = "/mnt/sysimage/"
+
+# no-virt mode doesn't need libvirt, so make it optional
+try:
+    import libvirt
+except ImportError:
+    libvirt = None
+
+
+class InstallError(Exception):
+    pass
+
+class IsoMountpoint(object):
+    """
+    Mount the iso on a temporary directory and check to make sure the
+    vmlinuz and initrd.img files exist
+    Check the iso for a LiveOS directory and set a flag.
+    Extract the iso's label.
+
+    initrd_path can be used to point to a boot.iso tree with a newer
+    initrd.img than the iso has. The iso is still used for stage2.
+    """
+    def __init__( self, iso_path, initrd_path=None ):
+        """ iso_path is the path to a  boot.iso
+            initrd_path overrides mounting the iso for access to
+                        initrd and vmlinuz.
+        """
+        self.label = None
+        self.iso_path = iso_path
+        self.initrd_path = initrd_path
+
+        if not self.initrd_path:
+            self.mount_dir = mount(self.iso_path, opts="loop")
+        else:
+            self.mount_dir = self.initrd_path
+
+        kernel_list = [("/isolinux/vmlinuz", "/isolinux/initrd.img"),
+                       ("/ppc/ppc64/vmlinuz", "/ppc/ppc64/initrd.img"),
+                       ("/images/pxeboot/vmlinuz", "/images/pxeboot/initrd.img")]
+        if os.path.isdir( self.mount_dir+"/repodata" ):
+            self.repo = self.mount_dir
+        else:
+            self.repo = None
+        self.liveos = os.path.isdir( self.mount_dir+"/LiveOS" )
+
+        try:
+            for kernel, initrd in kernel_list:
+                if (os.path.isfile(self.mount_dir+kernel) and
+                    os.path.isfile(self.mount_dir+initrd)):
+                    self.kernel = self.mount_dir+kernel
+                    self.initrd = self.mount_dir+initrd
+                    break
+            else:
+                raise Exception("Missing kernel and initrd file in iso, failed"
+                                " to search under: {0}".format(kernel_list))
+        except:
+            self.umount()
+            raise
+
+        self.get_iso_label()
+
+    def umount( self ):
+        if not self.initrd_path:
+            umount(self.mount_dir)
+
+    def get_iso_label( self ):
+        """
+        Get the iso's label using isoinfo
+        """
+        isoinfo_output = execWithCapture("isoinfo", ["-d", "-i", self.iso_path])
+        log.debug( isoinfo_output )
+        for line in isoinfo_output.splitlines():
+            if line.startswith("Volume id: "):
+                self.label = line[11:]
+                return
+
+
+class VirtualInstall( object ):
+    """
+    Run virt-install using an iso and kickstart(s)
+    """
+    def __init__( self, iso, ks_paths, disk_img, img_size=2, 
+                  kernel_args=None, memory=1024, vnc=None, arch=None,
+                  log_check=None, virtio_host="127.0.0.1", virtio_port=6080,
+                  qcow2=False, boot_uefi=False, ovmf_path=None):
+        """
+        Start the installation
+
+        :param iso: Information about the iso to use for the installation
+        :type iso: IsoMountpoint
+        :param list ks_paths: Paths to kickstart files. All are injected, the
+           first one is the one executed.
+        :param str disk_img: Path to a disk image, created it it doesn't exist
+        :param int img_size: The image size, in MiB, to create if it doesn't exist
+        :param str kernel_args: Extra kernel arguments to pass on the kernel cmdline
+        :param int memory: Amount of RAM to assign to the virt, in MiB
+        :param str vnc: Arguments to pass to virt-install --graphics
+        :param str arch: Optional architecture to use in the virt
+        :param log_check: Method that returns True if the installation fails
+        :type log_check: method
+        :param str virtio_host: Hostname to connect virtio log to
+        :param int virtio_port: Port to connect virtio log to
+        :param bool qcow2: Set to True if disk_img is a qcow2
+        :param bool boot_uefi: Use OVMF to boot the VM in UEFI mode
+        :param str ovmf_path: Path to the OVMF firmware
+        """
+        self.virt_name = "LiveOS-"+str(uuid.uuid4())
+        # add --graphics none later
+        # add whatever serial cmds are needed later
+        args = ["-n", self.virt_name,
+                 "-r", str(memory),
+                 "--noreboot",
+                 "--noautoconsole"]
+
+        args.append("--graphics")
+        if vnc:
+            args.append(vnc)
+        else:
+            args.append("none")
+
+        for ks in ks_paths:
+            args.append("--initrd-inject")
+            args.append(ks)
+
+        disk_opts = "path={0}".format(disk_img)
+        if qcow2:
+            disk_opts += ",format=qcow2"
+        else:
+            disk_opts += ",format=raw"
+        if not os.path.isfile(disk_img):
+            disk_opts += ",size={0}".format(img_size)
+        args.append("--disk")
+        args.append(disk_opts)
+
+        if iso.liveos:
+            disk_opts = "path={0},device=cdrom".format(iso.iso_path)
+            args.append("--disk")
+            args.append(disk_opts)
+
+        extra_args = "ks=file:/{0}".format(os.path.basename(ks_paths[0]))
+        if not vnc:
+            extra_args += " inst.cmdline console=ttyS0"
+        if kernel_args:
+            extra_args += " "+kernel_args
+        if iso.liveos:
+            extra_args += " stage2=hd:LABEL={0}".format(udev_escape(iso.label))
+        args.append("--extra-args")
+        args.append(extra_args)
+
+        args.append("--location")
+        args.append(iso.mount_dir)
+
+        channel_args = "tcp,host={0}:{1},mode=connect,target_type=virtio" \
+                       ",name=org.fedoraproject.anaconda.log.0".format(
+                       virtio_host, virtio_port)
+        args.append("--channel")
+        args.append(channel_args)
+
+        if arch:
+            args.append("--arch")
+            args.append(arch)
+
+        if boot_uefi and ovmf_path:
+            args.append("--boot")
+            args.append("loader=%s/OVMF_CODE.fd,loader_ro=yes,loader_type=pflash,nvram_template=%s/OVMF_VARS.fd,loader_secure=no" % (ovmf_path, ovmf_path))
+
+        log.info("Running virt-install.")
+        try:
+            execWithRedirect("virt-install", args, raise_err=True)
+        except subprocess.CalledProcessError as e:
+            raise InstallError("Problem starting virtual install: %s" % e)
+
+        conn = libvirt.openReadOnly(None)
+        dom = conn.lookupByName(self.virt_name)
+
+        # TODO: If vnc has been passed, we should look up the port and print that
+        # for the user at this point
+
+        while dom.isActive() and not log_check():
+            sys.stdout.write(".")
+            sys.stdout.flush()
+            sleep(10)
+        print
+
+        if log_check():
+            log.info( "Installation error detected. See logfile." )
+        else:
+            log.info( "Install finished. Or at least virt shut down." )
+
+    def destroy( self ):
+        """
+        Make sure the virt has been shut down and destroyed
+
+        Could use libvirt for this instead.
+        """
+        log.info( "Shutting down {0}".format(self.virt_name) )
+        subprocess.call(["virsh", "destroy", self.virt_name])
+
+        # Undefine the virt, UEFI installs need to have --nvram passed
+        subprocess.call(["virsh", "undefine", self.virt_name, "--nvram"])
+
+def novirt_install(opts, disk_img, disk_size, repo_url):
+    """
+    Use Anaconda to install to a disk image
+    """
+    import selinux
+
+    # Set selinux to Permissive if it is Enforcing
+    selinux_enforcing = False
+    if selinux.is_selinux_enabled() and selinux.security_getenforce():
+        selinux_enforcing = True
+        selinux.security_setenforce(0)
+
+    # Clean up /tmp/ from previous runs to prevent stale info from being used
+    for path in ["/tmp/yum.repos.d/", "/tmp/yum.cache/"]:
+        if os.path.isdir(path):
+            shutil.rmtree(path)
+
+    args = ["--kickstart", opts.ks[0], "--cmdline", "--repo", repo_url]
+    if opts.anaconda_args:
+        for arg in opts.anaconda_args:
+            args += arg.split(" ", 1)
+    if opts.proxy:
+        args += ["--proxy", opts.proxy]
+    if opts.armplatform:
+        args += ["--armplatform", opts.armplatform]
+
+    if opts.make_iso or opts.make_fsimage:
+        # Make a blank fs image
+        args += ["--dirinstall"]
+
+        mkext4img(None, disk_img, label=opts.fs_label, size=disk_size * 1024**3)
+        if not os.path.isdir(ROOT_PATH):
+            os.mkdir(ROOT_PATH)
+        mount(disk_img, opts="loop", mnt=ROOT_PATH)
+    elif opts.make_tar:
+        args += ["--dirinstall"]
+
+        # Install directly into ROOT_PATH, make sure it starts clean
+        if os.path.exists(ROOT_PATH):
+            shutil.rmtree(ROOT_PATH)
+        if not os.path.isdir(ROOT_PATH):
+            os.mkdir(ROOT_PATH)
+    else:
+        args += ["--image", disk_img]
+
+        # Create the sparse image
+        mksparse(disk_img, disk_size * 1024**3)
+
+    # Make sure anaconda has the right product and release
+    os.environ["ANACONDA_PRODUCTNAME"] = opts.project
+    os.environ["ANACONDA_PRODUCTVERSION"] = opts.releasever
+    rc = execWithRedirect("anaconda", args)
+
+    # Move the anaconda logs over to a log directory
+    log_dir = os.path.abspath(os.path.dirname(opts.logfile))
+    log_anaconda = joinpaths(log_dir, "anaconda")
+    if not os.path.isdir(log_anaconda):
+        os.mkdir(log_anaconda)
+    for l in ["anaconda.log", "ifcfg.log", "program.log", "storage.log",
+              "packaging.log", "yum.log"]:
+        if os.path.exists("/tmp/"+l):
+            shutil.copy2("/tmp/"+l, log_anaconda)
+            os.unlink("/tmp/"+l)
+
+    if opts.make_iso or opts.make_fsimage:
+        umount(ROOT_PATH)
+    else:
+        # If anaconda failed the disk image may still be in use by dm
+        execWithRedirect("anaconda-cleanup", [])
+        dm_name = os.path.splitext(os.path.basename(disk_img))[0]
+        dm_path = "/dev/mapper/"+dm_name
+        if os.path.exists(dm_path):
+            dm_detach(dm_path)
+            loop_detach(get_loop_name(disk_img))
+
+    if selinux_enforcing:
+        selinux.security_setenforce(1)
+
+    if rc:
+        raise InstallError("novirt_install failed")
+
+    if opts.make_tar:
+        compress_args = []
+        for arg in opts.compress_args:
+            compress_args += arg.split(" ", 1)
+
+        rc = mktar(ROOT_PATH, disk_img, opts.compression, compress_args)
+        shutil.rmtree(ROOT_PATH)
+
+        if rc:
+            raise InstallError("novirt_install failed")
+    elif opts.qcow2:
+        log.info("Converting %s to qcow2", disk_img)
+        qcow2_args = []
+        for arg in opts.qcow2_args:
+            qcow2_args += arg.split(" ", 1)
+
+        # convert the image to qcow2 format
+        if "-O" not in qcow2_args:
+            qcow2_args.extend(["-O", "qcow2"])
+        qcow2_img = tempfile.mktemp(prefix="disk", suffix=".img")
+        execWithRedirect("qemu-img", ["convert"] + qcow2_args + [disk_img, qcow2_img], raise_err=True)
+        execWithRedirect("mv", ["-f", qcow2_img, disk_img], raise_err=True)
+
+
+def virt_install(opts, install_log, disk_img, disk_size):
+    """
+    Use virt-install to install to a disk image
+
+    install_log is the path to write the log from virt-install
+    disk_img is the full path to the final disk or filesystem image
+    disk_size is the size of the disk to create in GiB
+    """
+    iso_mount = IsoMountpoint(opts.iso, opts.location)
+    log_monitor = LogMonitor(install_log)
+
+    kernel_args = ""
+    if opts.kernel_args:
+        kernel_args += opts.kernel_args
+    if opts.proxy:
+        kernel_args += " proxy="+opts.proxy
+
+    if opts.qcow2 and not opts.make_fsimage:
+        # virt-install can't take all the qcow2 options so create the image first
+        qcow2_args = []
+        for arg in opts.qcow2_args:
+            qcow2_args += arg.split(" ", 1)
+
+        mkqcow2(disk_img, disk_size*1024**3, qcow2_args)
+
+    if opts.make_fsimage or opts.make_tar:
+        diskimg_path = tempfile.mktemp(prefix="disk", suffix=".img")
+    else:
+        diskimg_path = disk_img
+
+    try:
+        virt = VirtualInstall(iso_mount, opts.ks, diskimg_path, disk_size,
+                               kernel_args, opts.ram, opts.vnc, opts.arch,
+                               log_check = log_monitor.server.log_check,
+                               virtio_host = log_monitor.host,
+                               virtio_port = log_monitor.port,
+                               qcow2=opts.qcow2, boot_uefi=opts.virt_uefi,
+                               ovmf_path=opts.ovmf_path)
+
+        virt.destroy()
+        log_monitor.shutdown()
+    except InstallError as e:
+        log.error("VirtualInstall failed: %s", e)
+        raise
+    finally:
+        log.info("unmounting the iso")
+        iso_mount.umount()
+
+    if log_monitor.server.log_check():
+        raise InstallError("virt_install failed")
+
+    if opts.make_fsimage:
+        mkdiskfsimage(diskimg_path, disk_img, label=opts.fs_label)
+        os.unlink(diskimg_path)
+    elif opts.make_tar:
+        compress_args = []
+        for arg in opts.compress_args:
+            compress_args += arg.split(" ", 1)
+
+        with PartitionMount(diskimg_path) as img_mount:
+            if img_mount and img_mount.mount_dir:
+                rc = mktar(img_mount.mount_dir, disk_img, opts.compression, compress_args)
+        os.unlink(diskimg_path)
+
+        if rc:
+            raise InstallError("virt_install failed")
+
+
+

src/pylorax/logmonitor.py

@@ -0,0 +1,123 @@
+#
+# Copyright (C) 2011-2017  Red Hat, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+import re
+import socket
+import SocketServer
+import threading
+
+class LogRequestHandler(SocketServer.BaseRequestHandler):
+    """
+    Handle monitoring and saving the logfiles from the virtual install
+    """
+    def setup(self):
+        if self.server.log_path:
+            self.fp = open(self.server.log_path, "w")
+        else:
+            print "no log_path specified"
+        self.request.settimeout(10)
+
+    def handle(self):
+        """
+        Handle writing incoming data to a logfile and
+        checking the logs for any Tracebacks or other errors that indicate
+        that the install failed.
+        """
+        line = ""
+        while True:
+            if self.server.kill:
+                break
+
+            try:
+                data = self.request.recv(4096)
+                self.fp.write(data)
+                self.fp.flush()
+
+                # check the data for errors and set error flag
+                # need to assemble it into lines so we can test for the error
+                # string.
+                while data:
+                    more = data.split("\n", 1)
+                    line += more[0]
+                    if len(more) > 1:
+                        self.iserror(line)
+                        line = ""
+                        data = more[1]
+                    else:
+                        data = None
+
+            except socket.timeout:
+                pass
+            except:
+                break
+
+    def finish(self):
+        self.fp.close()
+
+    def iserror(self, line):
+        """
+        Check a line to see if it contains an error indicating install failure
+        """
+        simple_tests = ["Traceback (",
+                        "Out of memory:",
+                        "Call Trace:",
+                        "insufficient disk space:"]
+        re_tests =     [r"packaging: base repo .* not valid"]
+        for t in simple_tests:
+            if line.find(t) > -1:
+                self.server.log_error = True
+                return
+        for t in re_tests:
+            if re.search(t, line):
+                self.server.log_error = True
+                return
+
+
+class LogServer(SocketServer.TCPServer):
+    """
+    Add path to logfile
+    Add log error flag
+    Add a kill switch
+    """
+    def __init__(self, log_path, *args, **kwargs):
+        self.kill = False
+        self.log_error = False
+        self.log_path = log_path
+        SocketServer.TCPServer.__init__(self, *args, **kwargs)
+
+    def log_check(self):
+        return self.log_error
+
+
+class LogMonitor(object):
+    """
+    Contains all the stuff needed to setup a thread to listen to the logs
+    from the virtual install
+    """
+    def __init__(self, log_path, host="localhost", port=0):
+        """
+        Fire up the thread listening for logs
+        """
+        self.server = LogServer(log_path, (host, port), LogRequestHandler)
+        self.host, self.port = self.server.server_address
+        self.log_path = log_path
+        self.server_thread = threading.Thread(target=self.server.handle_request)
+        self.server_thread.daemon = True
+        self.server_thread.start()
+
+    def shutdown(self):
+        self.server.kill = True
+        self.server_thread.join()