installupgradeinitrd() (pylorax.ltmpl.LoraxTemplateRunner method)

run_pkg_transaction() (pylorax.ltmpl.LoraxTemplateRunner method)

create_ext4_runtime() (pylorax.treebuilder.RuntimeBuilder method)

create_squashfs_runtime() (pylorax.treebuilder.RuntimeBuilder method)

generate_module_data() (pylorax.treebuilder.RuntimeBuilder method)

ZERO-CLAUSE BSD LICENSE FOR CODE IN THE PYTHON 3.9.4 DOCUMENTATION

email.message.Message: Representing an email message using the compat32 API

What is the most efficient way to concatenate many strings together?

Why does a_tuple[i] += [‘item’] raise an exception when the addition works?

What does the slash(/) in the parameter list of a function mean?

PEP 3118: New memoryview implementation and buffer protocol documentation

PEP 476: Enabling certificate verification by default for stdlib http clients

PEP 471 - os.scandir() function – a better and faster directory iterator

Why must ‘self’ be used explicitly in method definitions and calls?

argparse — Parser for command-line options, arguments and sub-commands

functools — Higher-order functions and operations on callable objects

multiprocessing.shared_memory —  Provides shared memory for direct access across processes

sysconfig — Provide access to Python’s configuration information

#
# base.py
#
# Copyright (C) 2009-2015 Red Hat, Inc.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
# Red Hat Author(s):  Martin Gracik <mgracik@redhat.com>
#

from abc import ABCMeta, abstractmethod
import sys

import pylorax.output as output

class BaseLoraxClass(object, metaclass=ABCMeta):
    @abstractmethod
    def __init__(self):
        self.output = output.LoraxOutput()

    def pcritical(self, msg, fobj=sys.stdout):
        self.output.critical(msg, fobj)

    def perror(self, msg, fobj=sys.stdout):
        self.output.error(msg, fobj)

    def pwarning(self, msg, fobj=sys.stdout):
        self.output.warning(msg, fobj)

    def pinfo(self, msg, fobj=sys.stdout):
        self.output.info(msg, fobj)

    def pdebug(self, msg, fobj=sys.stdout):
        self.output.debug(msg, fobj)

class DataHolder(dict):

    def __init__(self, **kwargs):
        dict.__init__(self)

        for attr, value in kwargs.items():
            self[attr] = value

    def __getattr__(self, attr):
        if attr in self:
            return self[attr]
        else:
            raise AttributeError

    def __setattr__(self, attr, value):
        self[attr] = value

    def copy(self):
        return DataHolder(**dict.copy(self))

#
# buildstamp.py
#
# Copyright (C) 2010-2015 Red Hat, Inc.
#
#
#
#
#

import logging
logger = logging.getLogger("pylorax.buildstamp")

import datetime
import os

class BuildStamp(object):

    def __init__(self, product, version, bugurl, isfinal, buildarch, variant=""):
        self.product = product
        self.version = version
        self.bugurl = bugurl
        self.isfinal = isfinal
        self.variant = variant

        if 'SOURCE_DATE_EPOCH' in os.environ:
            now = datetime.datetime.utcfromtimestamp(
                int(os.environ['SOURCE_DATE_EPOCH']))
        else:
            now = datetime.datetime.now()
        now = now.strftime("%Y%m%d%H%M")
        self.uuid = "{0}.{1}".format(now, buildarch)

    def write(self, outfile):
        # get lorax version
        try:
            import pylorax.version
        except ImportError:
            vernum = "devel"
        else:
            vernum = pylorax.version.num

        logger.info("writing .buildstamp file")
        with open(outfile, "w") as fobj:
            fobj.write("[Main]\n")
            fobj.write("Product={0.product}\n".format(self))
            fobj.write("Version={0.version}\n".format(self))
            fobj.write("BugURL={0.bugurl}\n".format(self))
            fobj.write("IsFinal={0.isfinal}\n".format(self))
            fobj.write("UUID={0.uuid}\n".format(self))
            if self.variant:
                fobj.write("Variant={0.variant}\n".format(self))
            fobj.write("[Compose]\n")
            fobj.write("Lorax={0}\n".format(vernum))

#
# cmdline.py
#
# Copyright (C) 2016  Red Hat, Inc.
#
#
#
#
# Red Hat Author(s):  Brian C. Lane <bcl@redhat.com>

import os
import sys
import argparse

from pylorax import vernum

version = "{0}-{1}".format(os.path.basename(sys.argv[0]), vernum)

def lorax_parser(dracut_default=""):
    """ Return the ArgumentParser for lorax"""

    parser = argparse.ArgumentParser(description="Create the Anaconda boot.iso")

    # required arguments for image creation
    required = parser.add_argument_group("required arguments")
    required.add_argument("-p", "--product", help="product name", required=True, metavar="PRODUCT")
    required.add_argument("-v", "--version", help="version identifier", required=True, metavar="VERSION")
    required.add_argument("-r", "--release", help="release information", required=True, metavar="RELEASE")
    required.add_argument("-s", "--source", help="source repository (may be listed multiple times)",
                        metavar="REPOSITORY", action="append", default=[])
    required.add_argument("--repo", help="source dnf repository file", type=os.path.abspath,
                          dest="repos", metavar="REPOSITORY", action="append", default=[])

    # optional arguments
    optional = parser.add_argument_group("optional arguments")
    optional.add_argument("-m", "--mirrorlist",
                        help="mirrorlist repository (may be listed multiple times)",
    optional.add_argument("-t", "--variant", default="",
                        help="variant name", metavar="VARIANT")
    optional.add_argument("-b", "--bugurl",
                        help="bug reporting URL for the product", metavar="URL",
                        default="your distribution provided bug reporting tool")
    optional.add_argument("--isfinal", help="",
                        action="store_true", default=False, dest="isfinal")
    optional.add_argument("-c", "--config", default="/etc/lorax/lorax.conf",
                        help="config file", metavar="CONFIGFILE")
    optional.add_argument("--proxy", default=None,
                          help="repo proxy url:port", metavar="HOST")
    optional.add_argument("-i", "--installpkgs", default=[],
                        action="append", metavar="PACKAGE",
                        help="package glob to install before runtime-install.tmpl runs. (may be listed multiple times)")
    optional.add_argument("-e", "--excludepkgs", default=[],
                        help="package glob to remove before runtime-install.tmpl runs. (may be listed multiple times)")
    optional.add_argument("--buildarch", default=None,
                        help="build architecture", metavar="ARCH")
    optional.add_argument("--volid", default=None,
                        help="volume id", metavar="VOLID")
    optional.add_argument("--macboot", help="",
                        action="store_true", default=True, dest="domacboot")
    optional.add_argument("--nomacboot", help="",
                        action="store_false", dest="domacboot")
    optional.add_argument("--noupgrade", help="",
                        action="store_false", default=True, dest="doupgrade")
    optional.add_argument("--logfile", default="./lorax.log", type=os.path.abspath,
                        help="Path to logfile")
    optional.add_argument("--tmp", default="/var/tmp/lorax",
                        help="Top level temporary directory" )
    optional.add_argument("--cachedir", default=None, type=os.path.abspath,
                        help="DNF cache directory. Default is a temporary dir.")
    optional.add_argument("--workdir", default=None, type=os.path.abspath,
                        help="Work directory, overrides --tmp. Default is a temporary dir under /var/tmp/lorax")
    optional.add_argument("--force", default=False, action="store_true",
                        help="Run even when the destination directory exists")
    optional.add_argument("--add-template", dest="add_templates",
                        action="append", help="Additional template for runtime image",
                        default=[])
    optional.add_argument("--add-template-var", dest="add_template_vars",
                        action="append", help="Set variable for runtime image template",
    optional.add_argument("--add-arch-template", dest="add_arch_templates",
                        action="append", help="Additional template for architecture-specific image",
    optional.add_argument("--add-arch-template-var", dest="add_arch_template_vars",
                        action="append", help="Set variable for architecture-specific image",
    optional.add_argument("--noverify", action="store_false", default=True, dest="verify",
                        help="Do not verify the install root")
    optional.add_argument("--sharedir", metavar="SHAREDIR", type=os.path.abspath,
                          help="Directory containing all the templates. Overrides config file sharedir")
    optional.add_argument("--enablerepo", action="append", default=[], dest="enablerepos",
                          metavar="[repo]", help="Names of repos to enable")
    optional.add_argument("--disablerepo", action="append", default=[], dest="disablerepos",
                          metavar="[repo]", help="Names of repos to disable")
    optional.add_argument("--rootfs-size", type=int, default=3,
                          help="Size of root filesystem in GiB. Defaults to 3.")
    optional.add_argument("--noverifyssl", action="store_true", default=False,
                          help="Do not verify SSL certificates")
    optional.add_argument("--dnfplugin", action="append", default=[], dest="dnfplugins",
                          help="Enable a DNF plugin by name/glob, or * to enable all of them.")
    optional.add_argument("--squashfs-only", action="store_true", default=False,
                          help="Use a plain squashfs filesystem for the runtime.")
    optional.add_argument("--skip-branding", action="store_true", default=False,
                          help="Disable automatic branding package selection. Use --installpkgs to add custom branding.")

    # dracut arguments
    dracut_group = parser.add_argument_group("dracut arguments: (default: %s)" % dracut_default)
    dracut_group.add_argument("--dracut-conf",
                              help="Path to a dracut.conf file to use instead of the "
                                   "default arguments. See the dracut.conf(5) manpage.")
    dracut_group.add_argument("--dracut-arg", action="append", dest="dracut_args",
                              help="Argument to pass to dracut when "
                                   "rebuilding the initramfs. Pass this "
                                   "once for each argument. NOTE: this "
                                   "overrides the defaults.")

    # add the show version option
    parser.add_argument("-V", help="show program's version number and exit",
                      action="version", version=version)

    parser.add_argument("outputdir", help="Output directory", metavar="OUTPUTDIR", type=os.path.abspath)

    return parser

def lmc_parser(dracut_default=""):
    """ Return a ArgumentParser object for live-media-creator."""
    parser = argparse.ArgumentParser(description="Create Live Install Media",
                                     fromfile_prefix_chars="@")

    # These are mutually exclusive, one is required
    action = parser.add_mutually_exclusive_group(required=True)
    action.add_argument("--make-iso", action="store_true",
                        help="Build a live iso")
    action.add_argument("--make-disk", action="store_true",
                        help="Build a partitioned disk image")
    action.add_argument("--make-fsimage", action="store_true",
                        help="Build a filesystem image")
    action.add_argument("--make-appliance", action="store_true",
                        help="Build an appliance image and XML description")
    action.add_argument("--make-ami", action="store_true",
                        help="Build an ami image")
    action.add_argument("--make-tar", action="store_true",
                        help="Build a tar of the root filesystem")
    action.add_argument("--make-tar-disk", action="store_true",
                        help="Build a tar of a partitioned disk image")
    action.add_argument("--make-pxe-live", action="store_true",
                        help="Build a live pxe boot squashfs image")
    action.add_argument("--make-ostree-live", action="store_true",
                        help="Build a live pxe boot squashfs image of Atomic Host")
    action.add_argument("--make-oci", action="store_true",
                        help="Build an Open Container Initiative image")
    action.add_argument("--make-vagrant", action="store_true",
                        help="Build a Vagrant Box image")

    parser.add_argument("--iso", type=os.path.abspath,
                        help="Anaconda installation .iso path to use for qemu")
    parser.add_argument("--iso-only", action="store_true",
                        help="Remove all iso creation artifacts except the boot.iso, "
                             "combine with --iso-name to rename the boot.iso")
    parser.add_argument("--iso-name", default=None,
                        help="Name of output iso file for --iso-only. Default is boot.iso")
    parser.add_argument("--ks", action="append", type=os.path.abspath,
                        help="Kickstart file defining the install.")
    parser.add_argument("--image-only", action="store_true",
                        help="Exit after creating fs/disk image.")

    parser.add_argument("--no-virt", action="store_true",
                        help="Run anaconda directly on host instead of using qemu")
    parser.add_argument("--proxy",
                        help="proxy URL to use for the install")
    parser.add_argument("--anaconda-arg", action="append", dest="anaconda_args",
                        help="Additional argument to pass to anaconda (no-virt "
                             "mode). Pass once for each argument")
    parser.add_argument("--armplatform",
                        help="the platform to use when creating images for ARM, "
                             "i.e., highbank, mvebu, omap, tegra, etc.")
    parser.add_argument("--location", default=None, type=os.path.abspath,
                        help="location of iso directory tree with initrd.img "
                             "and vmlinuz. Used to run qemu with a newer initrd "
                             "than the iso.")

    parser.add_argument("--logfile", default="./livemedia.log",
                        type=os.path.abspath,
                        help="Name and path for primary logfile, other logs will "
                             "be created in the same directory.")
    parser.add_argument("--lorax-templates", default=None,
                        help="Path to mako templates for lorax")
    parser.add_argument("--tmp", default="/var/tmp", type=os.path.abspath,
                        help="Top level temporary directory")
    parser.add_argument("--resultdir", default=None, dest="result_dir",
                        help="Directory to copy the resulting images and iso into. "
                             "Defaults to the temporary working directory")

    parser.add_argument("--macboot", action="store_true", default=True,
                        dest="domacboot")
    parser.add_argument("--nomacboot", action="store_false",

    parser.add_argument("--extra-boot-args", default="", dest="extra_boot_args",
                        help="Extra arguments to add to the bootloader kernel cmdline in the templates")

    image_group = parser.add_argument_group("disk/fs image arguments")
    image_group.add_argument("--disk-image", type=os.path.abspath,
                             help="Path to existing disk image to use for creating final image.")
    image_group.add_argument("--keep-image", action="store_true",
                             help="Keep raw disk image after .iso creation")
    image_group.add_argument("--fs-image", type=os.path.abspath,
                             help="Path to existing filesystem image to use for creating final image.")
    image_group.add_argument("--image-name", default=None,
                             help="Name of output file to create. Used for tar, fs and disk image. Default is a random name.")
    image_group.add_argument("--tar-disk-name", default=None,
                             help="Name of the archive member for make-tar-disk.")
    image_group.add_argument("--fs-label", default="Anaconda",
                             help="Label to set on fsimage, default is 'Anaconda'")
    image_group.add_argument("--image-size-align", type=int, default=0,
                             help="Create a disk image with a size that is a multiple of this value in MiB.")
    image_group.add_argument("--image-type", default=None,
                             help="Create an image with qemu-img. See qemu-img --help for supported formats.")
    image_group.add_argument("--qemu-arg", action="append", dest="qemu_args", default=[],
                             help="Arguments to pass to qemu-img. Pass once for each argument, they will be used for ALL calls to qemu-img.")
    image_group.add_argument("--qcow2", action="store_true",
                             help="Create qcow2 image instead of raw sparse image when making disk images.")
    image_group.add_argument("--qcow2-arg", action="append", dest="qemu_args", default=[],
    image_group.add_argument("--compression", default="xz",
                             help="Compression binary for make-tar. xz, lzma, gzip, and bzip2 are supported. xz is the default.")
    image_group.add_argument("--compress-arg", action="append", dest="compress_args", default=[],
                             help="Arguments to pass to compression. Pass once for each argument")
    # Group of arguments for appliance creation
    app_group = parser.add_argument_group("appliance arguments")
    app_group.add_argument("--app-name", default=None,
                           help="Name of appliance to pass to template")
    app_group.add_argument("--app-template", default=None,
                           help="Path to template to use for appliance data.")
    app_group.add_argument("--app-file", default="appliance.xml",
                           help="Appliance template results file.")

    # Group of arguments to pass to qemu
    virt_group = parser.add_argument_group("qemu arguments")
    virt_group.add_argument("--ram", metavar="MEMORY", type=int, default=2048,
                            help="Memory to allocate for installer in megabytes.")
    virt_group.add_argument("--vcpus", type=int, default=None,
                            help="Passed to qemu -smp command")
    virt_group.add_argument("--vnc",
                            help="Passed to qemu -display command. eg. vnc=127.0.0.1:5, default is to "
                                 "choose the first unused vnc port.")
    virt_group.add_argument("--arch", default=None,
                            help="System arch to build for. Used to select qemu-system-* command. "
                                 "Defaults to qemu-system-<arch>")
    virt_group.add_argument("--kernel-args",
                            help="Additional argument to pass to the installation kernel")
    virt_group.add_argument("--ovmf-path", default="/usr/share/edk2/ovmf/",
                            help="Path to OVMF firmware")
    virt_group.add_argument("--virt-uefi", action="store_true", default=False,
                            help="Use OVMF firmware to boot the VM in UEFI mode")
    virt_group.add_argument("--no-kvm", action="store_true", default=False,
                            help="Skip using kvm with qemu even if it is available.")
    virt_group.add_argument("--with-rng", default="/dev/random",
                            help="RNG device for QEMU (none for no RNG)")

    # pxe to live arguments
    pxelive_group = parser.add_argument_group("pxe to live arguments")
    pxelive_group.add_argument("--live-rootfs-size", type=int, default=0,
                                help="Size of root filesystem of live image in GiB")
    pxelive_group.add_argument("--live-rootfs-keep-size", action="store_true",
                                help="Keep the original size of root filesystem in live image")

    # OCI specific commands
    oci_group = parser.add_argument_group("OCI arguments")
    oci_group.add_argument("--oci-config",
                              help="config.json OCI configuration file")
    oci_group.add_argument("--oci-runtime",
                              help="runtime.json OCI configuration file")

    # Vagrant specific commands
    vagrant_group = parser.add_argument_group("Vagrant arguments")
    vagrant_group.add_argument("--vagrant-metadata",
                               help="optional metadata.json file")
    vagrant_group.add_argument("--vagrantfile",
                               help="optional vagrantfile")

    parser.add_argument("--project", default="Linux",
                        help="substituted for @PROJECT@ in bootloader config files")
    parser.add_argument("--releasever", default="34",
                        help="substituted for @VERSION@ in bootloader config files")
    parser.add_argument("--volid", default=None, help="volume id")
    parser.add_argument("--squashfs-only", action="store_true", default=False,
                        help="Use a plain squashfs filesystem for the runtime.")
    parser.add_argument("--timeout", default=None, type=int,
                        help="Cancel installer after X minutes")

#
# Copyright (C) 2011-2018  Red Hat, Inc.
#
#
#
#
log = logging.getLogger("pylorax")

import os
import tempfile
import subprocess
import shutil
import hashlib
import glob

# Use Mako templates for appliance builder descriptions
from mako.template import Template
from mako.exceptions import text_error_template

# Use pykickstart to calculate disk image size
from pykickstart.parser import KickstartParser
from pykickstart.constants import KS_SHUTDOWN
from pykickstart.version import makeVersion

# Use the Lorax treebuilder branch for iso creation
from pylorax import ArchData
from pylorax.base import DataHolder
from pylorax.executils import execWithRedirect, runcmd
from pylorax.imgutils import PartitionMount
from pylorax.imgutils import mount, umount, Mount
from pylorax.imgutils import mksquashfs, mkrootfsimg
from pylorax.imgutils import copytree
from pylorax.installer import novirt_install, virt_install, InstallError
from pylorax.treebuilder import TreeBuilder, RuntimeBuilder
from pylorax.treebuilder import findkernels
from pylorax.sysutils import joinpaths, remove

# Default parameters for rebuilding initramfs, override with --dracut-arg or --dracut-conf
DRACUT_DEFAULT = ["--xz", "--add", "livenet dmsquash-live dmsquash-live-ntfs convertfs pollcdrom qemu qemu-net",
                  "--omit", "plymouth", "--no-hostonly", "--debug", "--no-early-microcode"]

RUNTIME = "images/install.img"

class FakeDNF(object):
    """
    A minimal DNF object suitable for passing to RuntimeBuilder

    lmc uses RuntimeBuilder to run the arch specific iso creation
    templates, so the the installroot config value is the important part of
    this. Everything else should be a nop.
    """
    def __init__(self, conf):
        self.conf = conf

    def reset(self):
        pass

def is_image_mounted(disk_img):
    """
    Check to see if the disk_img is mounted

    :returns: True if disk_img is in /proc/mounts
    :rtype: bool
    """
    with open("/proc/mounts") as mounts:
        for mnt in mounts:
            fields = mnt.split()
            if len(fields) > 2 and fields[1] == disk_img:
                return True
    return False

def find_ostree_root(phys_root):
    """
    Find root of ostree deployment

    :param str phys_root: Path to physical root
    :returns: Relative path of ostree deployment root
    :rtype: str
    :raise Exception: More than one deployment roots were found
    """
    ostree_root = ""
    ostree_sysroots = glob.glob(joinpaths(phys_root, "ostree/boot.?/*/*/0"))
    log.debug("ostree_sysroots = %s", ostree_sysroots)
    if ostree_sysroots:
        if len(ostree_sysroots) > 1:
            raise Exception("Too many deployment roots found: %s" % ostree_sysroots)
        ostree_root = os.path.relpath(ostree_sysroots[0], phys_root)
    return ostree_root

def get_arch(mount_dir):
    """
    Get the kernel arch

    :returns: Arch of first kernel found at mount_dir/boot/ or i386
    """
    kernels = findkernels(mount_dir)
    if not kernels:
        return "i386"
    return kernels[0].arch

def squashfs_args(opts):
    """ Returns the compression type and args to use when making squashfs

    :param opts: ArgumentParser object with compression and compressopts
    :returns: tuple of compression type and args
    :rtype: tuple
    """
    compression = opts.compression or "xz"
    arch = ArchData(opts.arch or os.uname().machine)
    if compression == "xz" and arch.bcj and not opts.compress_args:
        # default to bcj when using xz
        compressargs = ["-Xbcj", arch.bcj]
    elif opts.compress_args:
        compressargs = []
        for arg in opts.compress_args:
            compressargs += arg.split(" ", 1)
    else:
    return (compression, compressargs)

def dracut_args(opts):
    """Return a list of the args to pass to dracut

    Return the default argument list unless one of the dracut cmdline arguments
    has been used.
    """
    if opts.dracut_conf:
        return ["--conf", opts.dracut_conf]
    elif opts.dracut_args:
        args = []
        for arg in opts.dracut_args:
            args += arg.split(" ", 1)
        return args
    else:
        return DRACUT_DEFAULT

def make_appliance(disk_img, name, template, outfile, networks=None, ram=1024,
                   vcpus=1, arch=None, title="Linux", project="Linux",
                   releasever="34"):
    """
    Generate an appliance description file

    :param str disk_img: Full path of the disk image
    :param str name: Name of the appliance, passed to the template
    :param str template: Full path of Mako template
    :param str outfile: Full path of file to write, using template
    :param list networks: List of networks(str) from the kickstart
    :param int ram: Ram, in MiB, passed to template. Default is 1024
    :param int vcpus: CPUs, passed to template. Default is 1
    :param str arch: CPU architecture. Default is 'x86_64'
    :param str title: Title, passed to template. Default is 'Linux'
    :param str project: Project, passed to template. Default is 'Linux'
    :param str releasever: Release version, passed to template. Default is 34
    """
    if not (disk_img and template and outfile):
        return None

    log.info("Creating appliance definition using %s", template)

    if not arch:
        arch = "x86_64"

    log.info("Calculating SHA256 checksum of %s", disk_img)
    sha256 = hashlib.sha256()
    with open(disk_img, "rb") as f:
        while True:
            data = f.read(1024**2)
            if not data:
                break
            sha256.update(data)
    log.info("SHA256 of %s is %s", disk_img, sha256.hexdigest())
    disk_info = DataHolder(name=os.path.basename(disk_img), format="raw",
                           checksum_type="sha256", checksum=sha256.hexdigest())
    try:
        result = Template(filename=template).render(disks=[disk_info], name=name,
                          arch=arch, memory=ram, vcpus=vcpus, networks=networks,
                          title=title, project=project, releasever=releasever)
    except Exception:
        log.error(text_error_template().render())
        raise

    with open(outfile, "w") as f:
        f.write(result)

def make_runtime(opts, mount_dir, work_dir, size=None):
    """
    Make the squashfs image from a directory

    :param opts: options passed to livemedia-creator
    :type opts: argparse options
    :param str mount_dir: Directory tree to compress
    :param str work_dir: Output compressed image to work_dir+images/install.img
    :param int size: Size of disk image, in GiB
    :returns: rc of squashfs creation
    :rtype: int
    """
    kernel_arch = get_arch(mount_dir)

    # Fake dnf  object
    fake_dbo = FakeDNF(conf=DataHolder(installroot=mount_dir))
    # Fake arch with only basearch set
    arch = ArchData(kernel_arch)
    # TODO: Need to get release info from someplace...
    product = DataHolder(name=opts.project, version=opts.releasever, release="",
                            variant="", bugurl="", isfinal=False)

    rb = RuntimeBuilder(product, arch, fake_dbo)
    compression, compressargs = squashfs_args(opts)

    if opts.squashfs_only:
        log.info("Creating a squashfs only runtime")
        return rb.create_squashfs_runtime(joinpaths(work_dir, RUNTIME), size=size,
                  compression=compression, compressargs=compressargs)
    else:
        log.info("Creating a squashfs+ext4 runtime")
        return rb.create_ext4_runtime(joinpaths(work_dir, RUNTIME), size=size,

def rebuild_initrds_for_live(opts, sys_root_dir, results_dir):
    """
    Rebuild intrds for pxe live image (root=live:http://)

    :param str sys_root_dir: Path to root of the system
    :param str results_dir: Path of directory for storing results
    """
    # cmdline dracut args override the defaults, but need to be parsed
    log.info("dracut args = %s", dracut_args(opts))

    dracut = ["dracut", "--nomdadmconf", "--nolvmconf"] + dracut_args(opts)

    kdir = "boot"
    if opts.ostree:
        kernels_dir = glob.glob(joinpaths(sys_root_dir, "boot/ostree/*"))
        if kernels_dir:
            kdir = os.path.relpath(kernels_dir[0], sys_root_dir)

    kernels = [kernel for kernel in findkernels(sys_root_dir, kdir)]
        raise Exception("No initrds found, cannot rebuild_initrds")

        # Dracut assumes to have some dirs in disk image
        # /var/tmp for temp files
        vartmp_dir = joinpaths(sys_root_dir, "var/tmp")
        if not os.path.isdir(vartmp_dir):
            os.mkdir(vartmp_dir)
        # /root (maybe not fatal)
        root_dir = joinpaths(sys_root_dir, "var/roothome")
        if not os.path.isdir(root_dir):
            os.mkdir(root_dir)
        # /tmp (maybe not fatal)
        tmp_dir = joinpaths(sys_root_dir, "sysroot/tmp")
        if not os.path.isdir(tmp_dir):
            os.mkdir(tmp_dir)

    # Write the new initramfs directly to the results directory
    os.mkdir(joinpaths(sys_root_dir, "results"))
    mount(results_dir, opts="bind", mnt=joinpaths(sys_root_dir, "results"))
    # Dracut runs out of space inside the minimal rootfs image
    mount("/var/tmp", opts="bind", mnt=joinpaths(sys_root_dir, "var/tmp"))
    for kernel in kernels:
        if hasattr(kernel, "initrd"):
            outfile = os.path.basename(kernel.initrd.path)
        else:
            # Construct an initrd from the kernel name
            outfile = os.path.basename(kernel.path.replace("vmlinuz-", "initrd-") + ".img")
        log.info("rebuilding %s", outfile)
        log.info("dracut warnings about /proc are safe to ignore")

        kver = kernel.version
        cmd = dracut + ["/results/"+outfile, kver]
        runcmd(cmd, root=sys_root_dir)

        shutil.copy2(joinpaths(sys_root_dir, kernel.path), results_dir)
    umount(joinpaths(sys_root_dir, "var/tmp"), delete=False)
    umount(joinpaths(sys_root_dir, "results"), delete=False)

def create_pxe_config(template, images_dir, live_image_name, add_args = None):
    """
    Create template for pxe to live configuration

    :param str images_dir: Path of directory with images to be used
    :param str live_image_name: Name of live rootfs image file
    :param list add_args: Arguments to be added to initrd= pxe config
    """

    add_args = add_args or []

    kernels = [kernel for kernel in findkernels(images_dir, kdir="")
               if hasattr(kernel, "initrd")]
        return

    kernel = kernels[0]

    add_args_str = " ".join(add_args)

    try:
        result = Template(filename=template).render(kernel=kernel.path,
                          initrd=kernel.initrd.path, liveimg=live_image_name,
                          addargs=add_args_str)
        raise

    with open (joinpaths(images_dir, "PXE_CONFIG"), "w") as f:

def make_livecd(opts, mount_dir, work_dir):
    """
    Take the content from the disk image and make a livecd out of it

    This uses wwood's squashfs live initramfs method:
     * put the real / into LiveOS/rootfs.img
     * make a squashfs of the LiveOS/rootfs.img tree
     * This is loaded by dracut when the cmdline is passed to the kernel:
       root=live:CDLABEL=<volid> rd.live.image
    """

    # Link /images to work_dir/images to make the templates happy
    if os.path.islink(joinpaths(mount_dir, "images")):
        os.unlink(joinpaths(mount_dir, "images"))
    rc = execWithRedirect("/bin/ln", ["-s", joinpaths(work_dir, "images"),
                                     joinpaths(mount_dir, "images")])
    if rc:
        raise RuntimeError("Failed to symlink images from mount_dir to work_dir")

    # The templates expect the config files to be in /tmp/config_files
    # I think these should be release specific, not from lorax, but for now
    configdir = joinpaths(opts.lorax_templates,"live/config_files/")
    configdir_path = "tmp/config_files"
    fullpath = joinpaths(mount_dir, configdir_path)
    if os.path.exists(fullpath):
        remove(fullpath)
    copytree(configdir, fullpath)

    isolabel = opts.volid or "{0.name}-{0.version}-{1.basearch}".format(product, arch)
    if len(isolabel) > 32:
        isolabel = isolabel[:32]
        log.warning("Truncating isolabel to 32 chars: %s", isolabel)

    tb = TreeBuilder(product=product, arch=arch, domacboot=opts.domacboot,
                     inroot=mount_dir, outroot=work_dir,
                     runtime=RUNTIME, isolabel=isolabel,
                     templatedir=joinpaths(opts.lorax_templates,"live/"),
                     extra_boot_args=opts.extra_boot_args)
    log.info("Rebuilding initrds")
    tb.rebuild_initrds(add_args=dracut_args(opts))
    log.info("Building boot.iso")
    tb.build()

    return work_dir

def mount_boot_part_over_root(img_mount):
    """
    Mount boot partition to /boot of root fs mounted in img_mount

    Used for OSTree so it finds deployment configurations on live rootfs

    param img_mount: object with mounted disk image root partition
    type img_mount: imgutils.PartitionMount
    """
    root_dir = img_mount.mount_dir
    is_boot_part = lambda dir: os.path.exists(dir+"/loader.0")
    tmp_mount_dir = tempfile.mkdtemp(prefix="lmc-tmpdir-")
    sysroot_boot_dir = None
    for dev, _size in img_mount.loop_devices:
        if dev is img_mount.mount_dev:
            continue
        try:
            mount("/dev/mapper/"+dev, mnt=tmp_mount_dir)
            if is_boot_part(tmp_mount_dir):
                umount(tmp_mount_dir)
                sysroot_boot_dir = joinpaths(root_dir, "boot")
                mount("/dev/mapper/"+dev, mnt=sysroot_boot_dir)
                break
            else:
        except subprocess.CalledProcessError as e:
            log.debug("Looking for boot partition error: %s", e)
    remove(tmp_mount_dir)
    return sysroot_boot_dir

def calculate_disk_size(opts, ks):
    """ Calculate the disk size from the kickstart

    :param str ks: Path to the kickstart to use for the installation
    :returns: Disk size in MiB

    Also takes into account the use of reqpart or reqpart --add-boot
    """
    # Disk size for a filesystem image should only be the size of /
    # to prevent surprises when using the same kickstart for different installations.
    unique_partitions = dict((p.mountpoint, p) for p in ks.handler.partition.partitions)
    if opts.no_virt and (opts.make_iso or opts.make_fsimage):
        disk_size = 2 + sum(p.size for p in unique_partitions.values() if p.mountpoint == "/")
    else:
        disk_size = 2 + sum(p.size for p in unique_partitions.values())

        # reqpart can add 1M, 2M, 200M based on platform. Add 500M to be sure
        if ks.handler.reqpart.seen:
            log.info("Adding 500M for reqpart")
            disk_size += 500

            # It can also request adding /boot which is 1G
            if ks.handler.reqpart.addBoot:
                log.info("Adding 1024M for reqpart --addboot")
                disk_size += 1024

    if opts.image_size_align:
        disk_size += opts.image_size_align - (disk_size % opts.image_size_align)

    log.info("Using disk size of %sMiB", disk_size)
    return disk_size

def make_image(opts, ks, cancel_func=None):
    """
    Install to a disk image

    :param cancel_func: Function that returns True to cancel build
    :type cancel_func: function
    :returns: Path of the image created

    Use qemu+boot.iso or anaconda to install to a disk image.
    """

    # For make_tar_disk, opts.image_name is the name of the final tarball.
    # Use opts.tar_disk_name as the name of the disk image
    if opts.make_tar_disk:
        disk_img = joinpaths(opts.result_dir, opts.tar_disk_name)
    elif opts.image_name:
        disk_img = joinpaths(opts.result_dir, opts.image_name)
    else:
        disk_img = tempfile.mktemp(prefix="lmc-disk-", suffix=".img", dir=opts.result_dir)
    log.info("disk_img = %s", disk_img)
    disk_size = calculate_disk_size(opts, ks)

    # For make_tar_disk, pass a second path parameter for the final tarball
    # not the final output file.
        tar_img = joinpaths(opts.result_dir, opts.image_name)
    else:
        tar_img = None

    try:
        if opts.no_virt:
            novirt_install(opts, disk_img, disk_size, cancel_func=cancel_func, tar_img=tar_img)
        else:
            install_log = os.path.abspath(os.path.dirname(opts.logfile))+"/virt-install.log"
            log.info("install_log = %s", install_log)

            virt_install(opts, install_log, disk_img, disk_size, cancel_func=cancel_func, tar_img=tar_img)
    except InstallError as e:
        log.error("Install failed: %s", e)
        if not opts.keep_image:
            if os.path.exists(disk_img):
                log.info("Removing bad disk image")
                os.unlink(disk_img)
            if tar_img and os.path.exists(tar_img):
                log.info("Removing bad tar file")
                os.unlink(tar_img)
        raise

    log.info("Disk Image install successful")

        return tar_img

    return disk_img

def make_live_images(opts, work_dir, disk_img):
    """
    Create live images from direcory or rootfs image

    :param str work_dir: Directory for storing results
    :param str disk_img: Path to disk image (fsimage or partitioned)
    :returns: Path of directory with created images or None

    fsck.ext4 is run on the rootfs_image to make sure there are no errors and to zero
    out any deleted blocks to make it compress better. If this fails for any reason
    it will return None and log the error.
    """
    sys_root = ""

    squashfs_root_dir = joinpaths(work_dir, "squashfs_root")
    liveos_dir = joinpaths(squashfs_root_dir, "LiveOS")
    os.makedirs(liveos_dir)
    rootfs_img = joinpaths(liveos_dir, "rootfs.img")

    if opts.fs_image or opts.no_virt:
        # Find the ostree root in the fsimage
        if opts.ostree:
            with Mount(disk_img, opts="loop") as mnt_dir:
                sys_root = find_ostree_root(mnt_dir)

        # Try to hardlink the image, if that fails, copy it
        rc = execWithRedirect("/bin/ln", [disk_img, rootfs_img])
        if rc != 0:
            shutil.copy2(disk_img, rootfs_img)
    else:
        is_root_part = None
            is_root_part = lambda dir: os.path.exists(dir+"/ostree/deploy")
        with PartitionMount(disk_img, mount_ok=is_root_part) as img_mount:
            if img_mount and img_mount.mount_dir:
                try:
                    mounted_sysroot_boot_dir = None
                    if opts.ostree:
                        sys_root = find_ostree_root(img_mount.mount_dir)
                        mounted_sysroot_boot_dir = mount_boot_part_over_root(img_mount)
                    if opts.live_rootfs_keep_size:
                        size = img_mount.mount_size / 1024**3
                    else:
                        size = opts.live_rootfs_size or None
                    log.info("Creating live rootfs image")
                    mkrootfsimg(img_mount.mount_dir, rootfs_img, "LiveOS", size=size, sysroot=sys_root)
                finally:
                    if mounted_sysroot_boot_dir:
                        umount(mounted_sysroot_boot_dir)
    log.debug("sys_root = %s", sys_root)

    # Make sure free blocks are actually zeroed so it will compress
    rc = execWithRedirect("/usr/sbin/fsck.ext4", ["-y", "-f", "-E", "discard", rootfs_img])
    if rc != 0:
        log.error("Problem zeroing free blocks of %s", disk_img)

    log.info("Packing live rootfs image")
    add_pxe_args = []
    live_image_name = "live-rootfs.squashfs.img"
    rc = mksquashfs(squashfs_root_dir, joinpaths(work_dir, live_image_name), compression, compressargs)
    if rc != 0:
        log.error("mksquashfs failed to create %s", live_image_name)

    log.info("Rebuilding initramfs for live")
    with Mount(rootfs_img, opts="loop") as mnt_dir:
        try:
            mount(joinpaths(mnt_dir, "boot"), opts="bind", mnt=joinpaths(mnt_dir, sys_root, "boot"))
            rebuild_initrds_for_live(opts, joinpaths(mnt_dir, sys_root), work_dir)
        finally:
            umount(joinpaths(mnt_dir, sys_root, "boot"), delete=False)

    remove(squashfs_root_dir)

        add_pxe_args.append("ostree=/%s" % sys_root)
    template = joinpaths(opts.lorax_templates, "pxe-live/pxe-config.tmpl")
    create_pxe_config(template, work_dir, live_image_name, add_pxe_args)

def check_kickstart(ks, opts):
    """Check the parsed kickstart object for errors

    :param ks: Parsed Kickstart object
    :type ks: pykickstart.parser.KickstartParser
    :param opts: Commandline options to control the process
    :type opts: Either a DataHolder or ArgumentParser
    :returns: List of error strings or empty list
    :rtype: list
    """
    errors = []
    if opts.no_virt and ks.handler.method.method not in ("url", "nfs") \
       and not ks.handler.ostreesetup.seen:
        errors.append("Only url, nfs and ostreesetup install methods are currently supported."
                      "Please fix your kickstart file." )

    if ks.handler.repo.seen and ks.handler.method.method != "url":
        errors.append("repo can only be used with the url install method. Add url to your "
                      "kickstart file.")

    if ks.handler.method.method in ("url", "nfs") and not ks.handler.network.seen:
        errors.append("The kickstart must activate networking if "
                      "the url or nfs install method is used.")

    if ks.handler.displaymode.displayMode is not None:
        errors.append("The kickstart must not set a display mode (text, cmdline, "
                      "graphical), this will interfere with livemedia-creator.")

    if opts.make_fsimage or (opts.make_pxe_live and opts.no_virt):
        # Make sure the kickstart isn't using autopart and only has a / mountpoint
        part_ok = not any(p for p in ks.handler.partition.partitions
                             if p.mountpoint not in ["/", "swap"])
        if not part_ok or ks.handler.autopart.seen:
            errors.append("Filesystem images must use a single / part, not autopart or "
                          "multiple partitions. swap is allowed but not used.")

    if not opts.no_virt and ks.handler.reboot.action != KS_SHUTDOWN:
        errors.append("The kickstart must include shutdown when using virt installation.")

    return errors

def run_creator(opts, cancel_func=None):
    """Run the image creator process

    :returns: The result directory and the disk image path.
    :rtype: Tuple of str

    This function takes the opts arguments and creates the selected output image.
    See the cmdline --help for livemedia-creator for the possible options

    (Yes, this is not ideal, but we can fix that later)
    """
    result_dir = None

    # Parse the kickstart
    if opts.ks:
        ks_version = makeVersion()
        ks = KickstartParser(ks_version, errorsAreFatal=False, missingIncludeIsFatal=False)
        ks.readKickstart(opts.ks[0])

    # live iso usually needs dracut-live so warn the user if it is missing
    if opts.ks and opts.make_iso:
        if "dracut-live" not in ks.handler.packages.packageList:
            log.error("dracut-live package is missing from the kickstart.")
            raise RuntimeError("dracut-live package is missing from the kickstart.")

    # Make the disk or filesystem image
    if not opts.disk_image and not opts.fs_image:
        if not opts.ks:
            raise RuntimeError("Image creation requires a kickstart file")

        # Check the kickstart for problems
        errors = check_kickstart(ks, opts)
        if errors:
            list(log.error(e) for e in errors)
            raise RuntimeError("\n".join(errors))

        # Make the image. Output of this is either a partitioned disk image or a fsimage
        try:
            disk_img = make_image(opts, ks, cancel_func=cancel_func)
        except InstallError as e:
            log.error("ERROR: Image creation failed: %s", e)
            raise RuntimeError("Image creation failed: %s" % e)

    if opts.image_only:
        return (result_dir, disk_img)

    if opts.make_iso:
        work_dir = tempfile.mkdtemp(prefix="lmc-work-")
        log.info("working dir is %s", work_dir)

        if (opts.fs_image or opts.no_virt) and not opts.disk_image:
            # Create iso from a filesystem image
            disk_img = opts.fs_image or disk_img
            with Mount(disk_img, opts="loop") as mount_dir:
                rc = make_runtime(opts, mount_dir, work_dir, calculate_disk_size(opts, ks)/1024.0)
                if rc != 0:
                    log.error("make_runtime failed with rc = %d. See program.log", rc)
                    raise RuntimeError("make_runtime failed with rc = %d" % rc)
                if cancel_func and cancel_func():
                    raise RuntimeError("ISO creation canceled")

                result_dir = make_livecd(opts, mount_dir, work_dir)
        else:
            # Create iso from a partitioned disk image
            disk_img = opts.disk_image or disk_img
            with PartitionMount(disk_img) as img_mount:
                if img_mount and img_mount.mount_dir:
                    rc = make_runtime(opts, img_mount.mount_dir, work_dir, calculate_disk_size(opts, ks)/1024.0)
                    if rc != 0:
                        log.error("make_runtime failed with rc = %d. See program.log", rc)
                        raise RuntimeError("make_runtime failed with rc = %d" % rc)
                    result_dir = make_livecd(opts, img_mount.mount_dir, work_dir)

        # --iso-only removes the extra build artifacts, keeping only the boot.iso
        if opts.iso_only and result_dir:
            boot_iso = joinpaths(result_dir, "images/boot.iso")
            if not os.path.exists(boot_iso):
                log.error("%s is missing, skipping --iso-only.", boot_iso)
            else:
                iso_dir = tempfile.mkdtemp(prefix="lmc-result-")
                dest_file = joinpaths(iso_dir, opts.iso_name or "boot.iso")
                shutil.move(boot_iso, dest_file)
                shutil.rmtree(result_dir)
                result_dir = iso_dir

        # cleanup the mess
        # cleanup work_dir?
        if disk_img and not (opts.keep_image or opts.disk_image or opts.fs_image):
            os.unlink(disk_img)
            log.info("Disk image erased")
            disk_img = None
    elif opts.make_appliance:
            networks = []
        else:
            networks = ks.handler.network.network
        make_appliance(opts.disk_image or disk_img, opts.app_name,
                       opts.app_template, opts.app_file, networks, opts.ram,
                       opts.vcpus or 1, opts.arch, opts.title, opts.project, opts.releasever)
    elif opts.make_pxe_live:
        disk_img = opts.fs_image or opts.disk_image or disk_img
        log.debug("disk image is %s", disk_img)

        result_dir = make_live_images(opts, work_dir, disk_img)
        if result_dir is None:
            log.error("Creating PXE live image failed.")
            raise RuntimeError("Creating PXE live image failed.")

    if opts.result_dir != opts.tmp and result_dir:
        copytree(result_dir, opts.result_dir, preserve=False)
        shutil.rmtree(result_dir)
        result_dir = None

    return (result_dir, disk_img)

#
# decorators.py
#
#
#
#
#
#

def singleton(cls):
    instances = {}

    def get_instance():
        if cls not in instances:
            instances[cls] = cls()
        return instances[cls]

    return get_instance

#
# discinfo.py
#
# Copyright (C) 2010-2015  Red Hat, Inc.
#
#
#
#
#

logger = logging.getLogger("pylorax.discinfo")

import os
import time

class DiscInfo(object):

    def __init__(self, release, basearch):
        self.release = release
        self.basearch = basearch

            timestamp = int(os.environ['SOURCE_DATE_EPOCH'])
        else:
            timestamp = time.time()

        logger.info("writing .discinfo file")
            fobj.write("{0:f}\n".format(timestamp))
            fobj.write("{0.release}\n".format(self))
            fobj.write("{0.basearch}\n".format(self))

# Copyright (C) 2018 Red Hat, Inc.
#
#
#
#

import dnf
import os

from pylorax import DEFAULT_PLATFORM_ID
from pylorax.sysutils import flatconfig

def get_dnf_base_object(installroot, sources, mirrorlists=None, repos=None,
                        enablerepos=None, disablerepos=None,
                        tempdir="/var/tmp", proxy=None, releasever="34",
                        cachedir=None, logdir=None, sslverify=True, dnfplugins=None):
    """ Create a dnf Base object and setup the repositories and installroot

        :param string installroot: Full path to the installroot
        :param list sources: List of source repo urls to use for the installation
        :param list enablerepos: List of repo names to enable
        :param list disablerepos: List of repo names to disable
        :param list mirrorlist: List of mirrors to use
        :param string tempdir: Path of temporary directory
        :param string proxy: http proxy to use when fetching packages
        :param string releasever: Release version to pass to dnf
        :param string cachedir: Directory to use for caching packages
        :param bool noverifyssl: Set to True to ignore the CA of ssl certs. eg. use self-signed ssl for https repos.

        If tempdir is not set /var/tmp is used.
        If cachedir is None a dnf.cache directory is created inside tmpdir
    """
    def sanitize_repo(repo):
        """Convert bare paths to file:/// URIs, and silently reject protocols unhandled by yum"""
        if repo.startswith("/"):
            return "file://{0}".format(repo)
        elif any(repo.startswith(p) for p in ('http://', 'https://', 'ftp://', 'file://')):
            return repo
        else:
            return None

    mirrorlists = mirrorlists or []

    # sanitize the repositories
    sources = list(sanitize_repo(r) for r in sources)
    mirrorlists = list(sanitize_repo(r) for r in mirrorlists)

    # remove invalid repositories
    sources = list(r for r in sources if r)
    mirrorlists = list(r for r in mirrorlists if r)

    if not cachedir:
        cachedir = os.path.join(tempdir, "dnf.cache")
    if not os.path.isdir(cachedir):
        os.mkdir(cachedir)

    if not logdir:
        logdir = os.path.join(tempdir, "dnf.logs")
        if not os.path.isdir(logdir):
            os.mkdir(logdir)

    dnfbase = dnf.Base()
    # Enable DNF pluings
    # NOTE: These come from the HOST system's environment
    if dnfplugins:
        if dnfplugins[0] == "*":
            # Enable them all
            dnfbase.init_plugins()
        else:
            # Only enable the listed plugins
            dnfbase.init_plugins(disabled_glob=["*"], enable_plugins=dnfplugins)
    conf = dnfbase.conf
    conf.logdir = logdir
    conf.cachedir = cachedir

    conf.install_weak_deps = False
    conf.releasever = releasever
    conf.installroot = installroot
    conf.prepend_installroot('persistdir')
    # this is a weird 'AppendOption' thing that, when you set it,
    # actually appends. Doing this adds 'nodocs' to the existing list
    # of values, over in libdnf, it does not replace the existing values.
    conf.tsflags = ['nodocs']
    # Log details about the solver
    conf.debug_solver = True

    if proxy:
        conf.proxy = proxy

    if sslverify == False:
        conf.sslverify = False

    # DNF 3.2 needs to have module_platform_id set, otherwise depsolve won't work correctly
    if not os.path.exists("/etc/os-release"):
        log.warning("/etc/os-release is missing, cannot determine platform id, falling back to %s", DEFAULT_PLATFORM_ID)
        platform_id = DEFAULT_PLATFORM_ID
    else:
        os_release = flatconfig("/etc/os-release")
        platform_id = os_release.get("PLATFORM_ID", DEFAULT_PLATFORM_ID)
    log.info("Using %s for module_platform_id", platform_id)
    conf.module_platform_id = platform_id

    # Add .repo files
    if repos:
        reposdir = os.path.join(tempdir, "dnf.repos")
        if not os.path.isdir(reposdir):
            os.mkdir(reposdir)
        for r in repos:
            shutil.copy2(r, reposdir)
        conf.reposdir = [reposdir]
        dnfbase.read_all_repos()

    # add the sources
    for i, r in enumerate(sources):
        if "SRPM" in r or "srpm" in r:
            log.info("Skipping source repo: %s", r)
            continue
        repo_name = "lorax-repo-%d" % i
        repo = dnf.repo.Repo(repo_name, conf)
        repo.baseurl = [r]
        if proxy:
            repo.proxy = proxy
        repo.enable()
        dnfbase.repos.add(repo)
        log.info("Added '%s': %s", repo_name, r)
        log.info("Fetching metadata...")
        try:
            repo.load()
        except dnf.exceptions.RepoError as e:
            log.error("Error fetching metadata for %s: %s", repo_name, e)

    # add the mirrorlists
    for i, r in enumerate(mirrorlists):
            continue
        repo_name = "lorax-mirrorlist-%d" % i
        repo.mirrorlist = r
        if proxy:
        try:

    # Enable repos listed on the cmdline
    for r in enablerepos:
        repolist = dnfbase.repos.get_matching(r)
        if not repolist:
            log.warning("%s is an unknown repo, not enabling it", r)
        else:
            repolist.enable()
            log.info("Enabled repo %s", r)

    # Disable repos listed on the cmdline
    for r in disablerepos:
            log.warning("%s is an unknown repo, not disabling it", r)
        else:
            repolist.disable()
            log.info("Disabled repo %s", r)

    dnfbase.fill_sack(load_system_repo=False)
    dnfbase.read_comps()

    return dnfbase

#
# dnfhelper.py
#
#
#
#
#
#                     Brian C. Lane <bcl@redhat.com>
#

logger = logging.getLogger("pylorax.dnfhelper")
import dnf
import dnf.transaction
import collections

__all__ = ['LoraxDownloadCallback', 'LoraxRpmCallback']

def _paced(fn):
    """Execute `fn` no more often then every 2 seconds."""
    def paced_fn(self, *args):
        now = time.time()
        if now - self.last_time < 2:
            return
        self.last_time = now
        return fn(self, *args)
    return paced_fn

class LoraxDownloadCallback(dnf.callback.DownloadProgress):
        self.downloads = collections.defaultdict(int)
        self.last_time = time.time()
        self.total_files = 0
        self.total_size = 0

        self.pkgno = 0
        self.total = 0

    @_paced
    def _update(self):
        msg = "Downloading %(pkgno)s / %(total_files)s RPMs, " \
              "%(downloaded)s / %(total_size)s (%(percent)d%%) done.\n"
        downloaded = sum(self.downloads.values())
        vals = {
            'downloaded'  : downloaded,
            'percent'     : int(100 * downloaded/self.total_size),
            'pkgno'       : self.pkgno,
            'total_files' : self.total_files,
            'total_size'  : self.total_size
        }
        self.output.write(msg % vals)

    def end(self, payload, status, msg):
        nevra = str(payload)
        if status is dnf.callback.STATUS_OK:
            self.downloads[nevra] = payload.download_size
            self.pkgno += 1
            self._update()
            return
        logger.critical("Failed to download '%s': %d - %s", nevra, status, msg)

    def progress(self, payload, done):
        self.downloads[nevra] = done
        self._update()

    # dnf 2.5.0 adds a new argument, accept it if it is passed
    # pylint: disable=arguments-differ
    def start(self, total_files, total_size, total_drpms=0):
        self.total_files = total_files
        self.total_size = total_size

class LoraxRpmCallback(dnf.callback.TransactionProgress):
        super(LoraxRpmCallback, self).__init__()
        self._last_ts = None

    def progress(self, package, action, ti_done, ti_total, ts_done, ts_total):
        if action == dnf.transaction.PKG_INSTALL:
            # do not report same package twice
            if self._last_ts == ts_done:
                return
            self._last_ts = ts_done

            msg = '(%d/%d) %s' % (ts_done, ts_total, package)
            logger.info(msg)
        elif action == dnf.transaction.TRANS_POST:
            msg = "Performing post-installation setup tasks"

    def error(self, message):
        logger.warning(message)

#
# executil.py - subprocess execution utility functions
#
# Copyright (C) 1999-2015
# Red Hat, Inc.  All rights reserved.
#
#
#
#

import os
import select
from subprocess import TimeoutExpired
import signal

program_log = logging.getLogger("program")

# pylint: disable=not-context-manager
from threading import Lock
program_log_lock = Lock()

_child_env = {}

def setenv(name, value):
    """ Set an environment variable to be used by child processes.

        This method does not modify os.environ for the running process, which
        is not thread-safe. If setenv has already been called for a particular
        variable name, the old value is overwritten.

        :param str name: The name of the environment variable
        :param str value: The value of the environment variable
    """

    _child_env[name] = value

def augmentEnv():
    env = os.environ.copy()
    env.update(_child_env)
    return env

class ExecProduct(object):
    def __init__(self, rc, stdout, stderr):
        self.rc = rc
        self.stdout = stdout
        self.stderr = stderr

def startProgram(argv, root='/', stdin=None, stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
        env_prune=None, env_add=None, reset_handlers=True, reset_lang=True, **kwargs):
    """ Start an external program and return the Popen object.

        The root and reset_handlers arguments are handled by passing a
        preexec_fn argument to subprocess.Popen, but an additional preexec_fn
        can still be specified and will be run. The user preexec_fn will be run
        last.

        :param argv: The command to run and argument
        :param root: The directory to chroot to before running command.
        :param stdin: The file object to read stdin from.
        :param stdout: The file object to write stdout to.
        :param stderr: The file object to write stderr to.
        :param env_prune: environment variables to remove before execution
        :param env_add: environment variables to add before execution
        :param reset_handlers: whether to reset to SIG_DFL any signal handlers set to SIG_IGN
        :param reset_lang: whether to set the locale of the child process to C
        :param kwargs: Additional parameters to pass to subprocess.Popen
        :return: A Popen object for the running command.
        :keyword preexec_fn: A function to run before execution starts.
    """
    if env_prune is None:
        env_prune = []

    # Check for and save a preexec_fn argument
    preexec_fn = kwargs.pop("preexec_fn", None)

    def preexec():
        # If a target root was specificed, chroot into it
        if root and root != '/':
            os.chroot(root)
            os.chdir("/")

        # Signal handlers set to SIG_IGN persist across exec. Reset
        # these to SIG_DFL if requested. In particular this will include the
        # SIGPIPE handler set by python.
        if reset_handlers:
            for signum in range(1, signal.NSIG):
                if signal.getsignal(signum) == signal.SIG_IGN:
                    signal.signal(signum, signal.SIG_DFL)

        # If the user specified an additional preexec_fn argument, run it
        if preexec_fn is not None:
            preexec_fn()

    with program_log_lock:
        program_log.info("Running... %s", " ".join(argv))

    env = augmentEnv()
    for var in env_prune:
        env.pop(var, None)

    if reset_lang:
        env.update({"LC_ALL": "C"})

    if env_add:
        env.update(env_add)

    # pylint: disable=subprocess-popen-preexec-fn
    return subprocess.Popen(argv,
                            stdin=stdin,
                            stdout=stdout,
                            stderr=stderr,
                            close_fds=True,
                            preexec_fn=preexec, cwd=root, env=env, **kwargs)

def _run_program(argv, root='/', stdin=None, stdout=None, env_prune=None, log_output=True,
        binary_output=False, filter_stderr=False, raise_err=False, callback=None,
        env_add=None, reset_handlers=True, reset_lang=True):
    """ Run an external program, log the output and return it to the caller

        :param stdout: Optional file object to write the output to.
        :param env_prune: environment variable to remove before execution
        :param log_output: whether to log the output of command
        :param binary_output: whether to treat the output of command as binary data
        :param filter_stderr: whether to exclude the contents of stderr from the returned output
        :param raise_err: whether to raise a CalledProcessError if the returncode is non-zero
        :param callback: method to call while waiting for process to finish, passed Popen object
        :return: The return code of the command and the output
        :raises: OSError or CalledProcessError
    """
    try:
        if filter_stderr:
            stderr = subprocess.PIPE
        else:
            stderr = subprocess.STDOUT

        proc = startProgram(argv, root=root, stdin=stdin, stdout=subprocess.PIPE, stderr=stderr,
                            env_prune=env_prune, universal_newlines=not binary_output,
                            env_add=env_add, reset_handlers=reset_handlers, reset_lang=reset_lang)

        output_string = None
        err_string = None
        if callback:
            while callback(proc) and proc.poll() is None:
                try:
                    (output_string, err_string) = proc.communicate(timeout=1)
                    break
                except TimeoutExpired:
                    pass
        else:
            (output_string, err_string) = proc.communicate()
        if output_string:
            if binary_output:
                output_lines = [output_string]
            else:
                if output_string[-1] != "\n":
                    output_string = output_string + "\n"
                output_lines = output_string.splitlines(True)

            if log_output:
                with program_log_lock:
                    for line in output_lines:
                        program_log.info(line.strip())

            if stdout:
                stdout.write(output_string)

        # If stderr was filtered, log it separately
        if filter_stderr and err_string and log_output:
            err_lines = err_string.splitlines(True)

            with program_log_lock:
                for line in err_lines:
                    program_log.info(line.strip())

    except OSError as e:
        with program_log_lock:
            program_log.error("Error running %s: %s", argv[0], e.strerror)
        raise

        program_log.debug("Return code: %s", proc.returncode)

    if proc.returncode and raise_err:
        output = (output_string or "") + (err_string or "")
        raise subprocess.CalledProcessError(proc.returncode, argv, output)

    return (proc.returncode, output_string)

def execWithRedirect(command, argv, stdin=None, stdout=None, root='/', env_prune=None,
                     log_output=True, binary_output=False, raise_err=False, callback=None,
                     env_add=None, reset_handlers=True, reset_lang=True):
    """ Run an external program and redirect the output to a file.

        :param command: The command to run
        :param argv: The argument list
        :param stdout: Optional file object to redirect stdout and stderr to.
        :return: The return code of the command
    """
    argv = [command] + list(argv)
    return _run_program(argv, stdin=stdin, stdout=stdout, root=root, env_prune=env_prune,
            log_output=log_output, binary_output=binary_output, raise_err=raise_err, callback=callback,
            env_add=env_add, reset_handlers=reset_handlers, reset_lang=reset_lang)[0]

def execWithCapture(command, argv, stdin=None, root='/', log_output=True, filter_stderr=False,
                    raise_err=False, callback=None, env_add=None, reset_handlers=True, reset_lang=True):
    """ Run an external program and capture standard out and err.

        :param log_output: Whether to log the output of command
        :param filter_stderr: Whether stderr should be excluded from the returned output
        :return: The output of the command
    """
    return _run_program(argv, stdin=stdin, root=root, log_output=log_output, filter_stderr=filter_stderr,
                        raise_err=raise_err, callback=callback, env_add=env_add,
                        reset_handlers=reset_handlers, reset_lang=reset_lang)[1]

def execReadlines(command, argv, stdin=None, root='/', env_prune=None, filter_stderr=False,
                  callback=lambda x: True, env_add=None, reset_handlers=True, reset_lang=True):
    """ Execute an external command and return the line output of the command
        in real-time.

        This method assumes that there is a reasonably low delay between the
        end of output and the process exiting. If the child process closes
        stdout and then keeps on truckin' there will be problems.

        NOTE/WARNING: UnicodeDecodeError will be raised if the output of the
                      external command can't be decoded as UTF-8.

        :return: Iterator of the lines from the command

        Output from the file is not logged to program.log
        This returns an iterator with the lines from the command until it has finished
    """

    class ExecLineReader(object):
        """Iterator class for returning lines from a process and cleaning
           up the process when the output is no longer needed.
        """

        def __init__(self, proc, argv, callback):
            self._proc = proc
            self._argv = argv
            self._callback = callback
            self._data = ""

        def __iter__(self):
            return self

        def __del__(self):
            # See if the process is still running
            if self._proc.poll() is None:
                # Stop the process and ignore any problems that might arise
                try:
                    self._proc.terminate()
                except OSError:
                    pass

        def __next__(self):
            # Return lines from stdout while also calling _callback
            while True:
                # Check for input without blocking
                if select.select([self._proc.stdout], [], [], 0)[0]:
                    size = len(self._proc.stdout.peek(1))
                    if size > 0:
                        self._data += self._proc.stdout.read(size).decode("utf-8")

                if self._data.find("\n") >= 0:
                    line = self._data.split("\n", 1)
                    self._data = line[1]
                    return line[0]

                if self._proc.poll() is not None or not self._callback(self._proc):
                    # Output finished, wait 60s for the process to end
                    try:
                        self._proc.communicate(timeout=60)
                    except subprocess.TimeoutExpired:
                        # Did not exit in 60s, kill it and wait 30s more
                        self._proc.kill()
                        try:
                            self._proc.communicate(timeout=30)
                        except subprocess.TimeoutExpired:
                            pass

                    if self._proc.returncode is None:
                        raise OSError("process '%s' failed to be killed" % self._argv)
                    elif self._proc.returncode < 0:
                        raise OSError("process '%s' was killed by signal %s" %
                                (self._argv, -self._proc.returncode))
                    elif self._proc.returncode > 0:
                        raise OSError("process '%s' exited with status %s" %
                                (self._argv, self._proc.returncode))
                    raise StopIteration

                # Don't loop too fast with no input to read
                time.sleep(0.5)

    argv = [command] + argv

    if filter_stderr:
        stderr = subprocess.DEVNULL
    else:
        stderr = subprocess.STDOUT

    try:
                            env_prune=env_prune, env_add=env_add, reset_handlers=reset_handlers, reset_lang=reset_lang)
        raise

    return ExecLineReader(proc, argv, callback)

def runcmd(cmd, **kwargs):
    """ run execWithRedirect with raise_err=True
    """
    kwargs["raise_err"] = True
    return execWithRedirect(cmd[0], cmd[1:], **kwargs)

def runcmd_output(cmd, **kwargs):
    """ run execWithCapture with raise_err=True
    """
    return execWithCapture(cmd[0], cmd[1:], **kwargs)

# imgutils.py - utility functions/classes for building disk images
#
# Copyright (C) 2011-2018 Red Hat, Inc.
#
#
#
#
# Author(s):  Will Woods <wwoods@redhat.com>

logger = logging.getLogger("pylorax.imgutils")

import os, tempfile
from os.path import join, dirname
from subprocess import Popen, PIPE, CalledProcessError
import sys
import traceback
import multiprocessing
from time import sleep

from pylorax.sysutils import cpfile
from pylorax.executils import execWithRedirect, execWithCapture
from pylorax.executils import runcmd, runcmd_output

######## Functions for making container images (cpio, tar, squashfs) ##########

def compress(command, root, outfile, compression="xz", compressargs=None):
    '''Make a compressed archive of the given rootdir or file.
    command is a list of the archiver commands to run
    compression should be "xz", "gzip", "lzma", "bzip2", or None.
    compressargs will be used on the compression commandline.'''
    if compression not in (None, "xz", "gzip", "lzma", "bzip2"):
        raise ValueError("Unknown compression type %s" % compression)
    compressargs = compressargs or ["-9"]
    if compression == "xz":
        compressargs.insert(0, "--check=crc32")
    if compression is None:
        compression = "cat" # this is a little silly

    # make compression run with multiple threads if possible
    if compression in ("xz", "lzma"):
        compressargs.insert(0, "-T%d" % multiprocessing.cpu_count())
    elif compression == "gzip":
        compression = "pigz"
        compressargs.insert(0, "-p%d" % multiprocessing.cpu_count())
    elif compression == "bzip2":
        compression = "pbzip2"

    find, archive, comp = None, None, None

    try:
        if os.path.isdir(root):
            logger.debug("find %s -print0 |%s | %s %s > %s", root, " ".join(command),
                    compression, " ".join(compressargs), outfile)

            find = Popen(["find", ".", "-print0"], stdout=PIPE, cwd=root)
            archive = Popen(command, stdin=find.stdout, stdout=PIPE, cwd=root)
        else:
            logger.debug("echo %s |%s | %s %s > %s", root, " ".join(command),
                         compression, " ".join(compressargs), outfile)

            archive = Popen(command, stdin=PIPE, stdout=PIPE, cwd=os.path.dirname(root))
            archive.stdin.write(os.path.basename(root).encode("utf-8") + b"\0")
            archive.stdin.close()

        with open(outfile, "wb") as fout:
            comp = Popen([compression] + compressargs,
                         stdin=archive.stdout, stdout=fout)
            comp.wait()

        # Clean up the open fds and processes
        if find:
            find.wait()
            find.stdout.close()
        archive.wait()
        if archive.stdin:
        if archive.stdout:
            archive.stdout.close()
        return comp.returncode
        logger.error(e)
        # Kill off any hanging processes
        list(p.kill() for p in (find, archive, comp) if p)
        return 1

def mkcpio(root, outfile, compression="xz", compressargs=None):
    return compress(["cpio", "--null", "--quiet", "-H", "newc", "-o"],
                    root, outfile, compression, compressargs)

def mktar(root, outfile, compression="xz", compressargs=None, selinux=True):
    tar_cmd = ["tar", "--no-recursion"]
    if selinux:
        tar_cmd += ["--selinux", "--acls", "--xattrs"]
    tar_cmd += ["-cf-", "--null", "-T-"]
    return compress(tar_cmd, root, outfile, compression, compressargs)

def mksquashfs(rootdir, outfile, compression="default", compressargs=None):
    '''Make a squashfs image containing the given rootdir.'''
    compressargs = compressargs or []
    if compression != "default":
        compressargs = ["-comp", compression] + compressargs
    return execWithRedirect("mksquashfs", [rootdir, outfile] + compressargs)

def mkrootfsimg(rootdir, outfile, label, size=2, sysroot=""):
    """
    Make rootfs image from a directory

    :param str rootdir: Root directory
    :param str outfile: Path of output image file
    :param str label: Filesystem label
    :param int size: Size of the image in GiB, if None computed automatically
    :param str sysroot: path to system (deployment) root relative to physical root
    """
    if size:
        fssize = size * (1024*1024*1024) # 2GB sparse file compresses down to nothin'
    else:
        fssize = None       # Let mkext4img figure out the needed size

    mkext4img(rootdir, outfile, label=label, size=fssize)

######## Utility functions ###############################################

def mksparse(outfile, size):
    '''use os.ftruncate to create a sparse file of the given size.'''
    with open(outfile, "w") as fobj:
        os.ftruncate(fobj.fileno(), size)

def mkqcow2(outfile, size, options=None):
    '''use qemu-img to create a file of the given size.
       options is a list of options passed to qemu-img

       Default format is qcow2, override by passing "-f", fmt
       in options.
    '''
    mkqemu_img(outfile, size, options)

def mkqemu_img(outfile, size, options=None):

    '''
    options = options or []
    if "-f" not in options:
        options.extend(["-f", "qcow2"])
    runcmd(["qemu-img", "create"] + options + [outfile, str(size)])

def loop_waitfor(loop_dev, outfile):
    """Make sure the loop device is attached to the outfile.

    It seems that on rare occasions losetup can return before the /dev/loopX is
    ready for use, causing problems with mkfs. This tries to make sure that the
    loop device really is associated with the backing file before continuing.

    Raise RuntimeError if it isn't setup after 5 tries.
    """
    for _x in range(0,5):
        runcmd(["udevadm", "settle", "--timeout", "300"])
        ## XXX Note that losetup --list output can be truncated to 64 bytes in some
        ##     situations. Don't use it to lookup backing file, go the other way
        ##     and lookup the loop for the backing file. See util-linux lib/loopdev.c
        ##     loopcxt_get_backing_file()
        if get_loop_name(outfile) == os.path.basename(loop_dev):
            return

        # If this really is a race, give it some time to settle down
        time.sleep(1)

    raise RuntimeError("Unable to setup %s on %s" % (loop_dev, outfile))

def loop_attach(outfile):
    """Attach a loop device to the given file. Return the loop device name.

    On rare occasions it appears that the device never shows up, some experiments
    seem to indicate that it may be a race with another process using /dev/loop* devices.

    So we now try 3 times before actually failing.

    Raises CalledProcessError if losetup fails.
    """
    retries = 0
    while True:
        try:
            retries += 1
            dev = runcmd_output(["losetup", "--find", "--show", outfile]).strip()

            # Sometimes the loop device isn't ready yet, make extra sure before returning
            loop_waitfor(dev, outfile)
        except RuntimeError:
            # Try to setup the loop device 3 times
            if retries == 3:
                logger.error("loop_attach failed, retries exhausted.")
                raise
            logger.debug("Try %d failed, %s did not appear.", retries, dev)
        break
    return dev

def loop_detach(loopdev):
    '''Detach the given loop device. Return False on failure.'''
    return (execWithRedirect("losetup", ["--detach", loopdev]) == 0)

def get_loop_name(path):
    '''Return the loop device associated with the path.
    Raises RuntimeError if more than one loop is associated'''
    buf = runcmd_output(["losetup", "-j", path])
    if len(buf.splitlines()) > 1:
        # there should never be more than one loop device listed
        raise RuntimeError("multiple loops associated with %s" % path)
    name = os.path.basename(buf.split(":")[0])
    return name

def dm_attach(dev, size, name=None):
    '''Attach a devicemapper device to the given device, with the given size.
    If name is None, a random name will be chosen. Returns the device name.
    raises CalledProcessError if dmsetup fails.'''
    if name is None:
        name = tempfile.mktemp(prefix="lorax.imgutils.", dir="")
    runcmd(["dmsetup", "create", name, "--table",
                       "0 %i linear %s 0" % (size/512, dev)])

def dm_detach(dev):
    '''Detach the named devicemapper device. Returns False if dmsetup fails.'''
    dev = dev.replace("/dev/mapper/", "") # strip prefix, if it's there
    return execWithRedirect("dmsetup", ["remove", dev])

def mount(dev, opts="", mnt=None):
    '''Mount the given device at the given mountpoint, using the given opts.
    opts should be a comma-separated string of mount options.
    if mnt is none, a temporary directory will be created and its path will be
    returned.
    raises CalledProcessError if mount fails.'''
    if mnt is None:
        mnt = tempfile.mkdtemp(prefix="lorax.imgutils.")
        logger.debug("make tmp mountdir %s", mnt)
    cmd = ["mount"]
    if opts:
        cmd += ["-o", opts]
    cmd += [dev, mnt]
    runcmd(cmd)
    return mnt

def umount(mnt,  lazy=False, maxretry=3, retrysleep=1.0, delete=True):
    '''Unmount the given mountpoint. If lazy is True, do a lazy umount (-l).
    If the mount was a temporary dir created by mount, it will be deleted.
    raises CalledProcessError if umount fails.'''
    cmd = ["umount"]
    if lazy: cmd += ["-l"]
    cmd += [mnt]
    count = 0
    while maxretry > 0:
        try:
            rv = runcmd(cmd)
        except CalledProcessError:
            count += 1
            if count == maxretry:
                raise
            logger.warning("failed to unmount %s. retrying (%d/%d)...",
                           mnt, count, maxretry)
            if logger.getEffectiveLevel() <= logging.DEBUG:
                fuser = execWithCapture("fuser", ["-vm", mnt])
                logger.debug("fuser -vm:\n%s\n", fuser)
            sleep(retrysleep)
        else:
            break
    if delete and 'lorax.imgutils' in mnt:
        os.rmdir(mnt)
        logger.debug("remove tmp mountdir %s", mnt)
    return (rv == 0)

def copytree(src, dest, preserve=True):
    '''Copy a tree of files using cp -a, thus preserving modes, timestamps,
    links, acls, sparse files, xattrs, selinux contexts, etc.
    If preserve is False, uses cp -R (useful for modeless filesystems)
    raises CalledProcessError if copy fails.'''
    logger.debug("copytree %s %s", src, dest)
    cp = ["cp", "-a"] if preserve else ["cp", "-R", "-L", "--preserve=timestamps"]
    cp += [join(src, "."), os.path.abspath(dest)]
    runcmd(cp)

def do_grafts(grafts, dest, preserve=True):
    '''Copy each of the items listed in grafts into dest.
    If the key ends with '/' it's assumed to be a directory which should be
    created, otherwise just the leading directories will be created.'''
    for imgpath, filename in grafts.items():
        if imgpath[-1] == '/':
            targetdir = join(dest, imgpath)
            imgpath = imgpath[:-1]
        else:
            targetdir = join(dest, dirname(imgpath))
        if not os.path.isdir(targetdir):
            os.makedirs(targetdir)
        if os.path.isdir(filename):
            copytree(filename, join(dest, imgpath), preserve)
        else:
            cpfile(filename, join(dest, imgpath))

def round_to_blocks(size, blocksize):
    '''If size isn't a multiple of blocksize, round up to the next multiple'''
    diff = size % blocksize
    if diff or not size:
        size += blocksize - diff
    return size

# TODO: move filesystem data outside this function
def estimate_size(rootdir, graft=None, fstype=None, blocksize=4096, overhead=256):
    graft = graft or {}
    getsize = lambda f: os.lstat(f).st_size
    if fstype == "btrfs":
        overhead = 64*1024 # don't worry, it's all sparse
    if fstype == "hfsplus":
        overhead = 200 # hack to deal with two bootloader copies
    if fstype in ("vfat", "msdos"):
        blocksize = 2048
        getsize = lambda f: os.stat(f).st_size # no symlinks, count as copies
    total = overhead*blocksize
    dirlist = list(graft.values())
    if rootdir:
        dirlist.append(rootdir)
    for root in dirlist:
        for top, dirs, files in os.walk(root):
            for f in files + dirs:
                total += round_to_blocks(getsize(join(top,f)), blocksize)
        total = max(256*1024*1024, total) # btrfs minimum size: 256MB
    logger.info("Size of %s block %s fs at %s estimated to be %s", blocksize, fstype, rootdir, total)
    return total

######## Execution contexts - use with the 'with' statement ##############

class LoopDev(object):
    def __init__(self, filename, size=None):
        self.loopdev = None
        self.filename = filename
        if size:
            mksparse(self.filename, size)
    def __enter__(self):
        self.loopdev = loop_attach(self.filename)
        return self.loopdev
    def __exit__(self, exc_type, exc_value, tracebk):
        loop_detach(self.loopdev)

class DMDev(object):
    def __init__(self, dev, size, name=None):
        self.mapperdev = None
        (self.dev, self.size, self.name) = (dev, size, name)
        self.mapperdev = dm_attach(self.dev, self.size, self.name)
        return self.mapperdev
        dm_detach(self.mapperdev)

class Mount(object):
    def __init__(self, dev, opts="", mnt=None):
        (self.dev, self.opts, self.mnt) = (dev, opts, mnt)
        self.mnt = mount(self.dev, self.opts, self.mnt)
        return self.mnt
        umount(self.mnt)

def kpartx_disk_img(disk_img):
    """Attach a disk image's partitions to /dev/loopX using kpartx

    :param disk_img: The full path to a partitioned disk image
    :type disk_img: str
    :returns: list of (loopXpN, size)
    :rtype: list of tuples
    """
    # Example kpartx output
    # kpartx -p p -v -a /tmp/diskV2DiCW.im
    # add map loop2p1 (253:2): 0 3481600 linear /dev/loop2 2048
    # add map loop2p2 (253:3): 0 614400 linear /dev/loop2 3483648
    kpartx_output = runcmd_output(["kpartx", "-v", "-a", "-s", disk_img])
    logger.debug(kpartx_output)

    # list of (deviceName, sizeInBytes)
    loop_devices = []
    for line in kpartx_output.splitlines():
        # add map loop2p3 (253:4): 0 7139328 linear /dev/loop2 528384
        # 3rd element is size in 512 byte blocks
        if line.startswith("add map "):
            fields = line[8:].split()
            loop_devices.append( (fields[0], int(fields[3])*512) )
    return loop_devices

class PartitionMount(object):
    """ Mount a partitioned image file using kpartx """
    def __init__(self, disk_img, mount_ok=None, submount=None):
        """
        :param str disk_img: The full path to a partitioned disk image
        :param mount_ok: A function that is passed the mount point and
                         returns True if it should be mounted.
        :param str submount: Directory inside mount_dir to mount at

        If mount_ok is not set it will look for /etc/passwd

        If the partition is found it will be mounted under a temporary
        directory and self.temp_dir set to it. If submount is passed it will be
        created and mounted there instead, with self.mount_dir set to point to
        it. self.mount_dev is set to the loop device, and self.mount_size is
        set to the size of the partition.

        When no subdir is passed self.temp_dir and self.mount_dir will be the same.
        """
        self.mount_dev = None
        self.mount_size = None
        self.mount_dir = None
        self.disk_img = disk_img
        self.mount_ok = mount_ok
        self.submount = submount
        self.temp_dir = None

        # Default is to mount partition with /etc/passwd
        if not self.mount_ok:
            self.mount_ok = lambda mount_dir: os.path.isfile(mount_dir+"/etc/passwd")

        # list of (deviceName, sizeInBytes)
        self.loop_devices = kpartx_disk_img(self.disk_img)

        # Mount the device selected by mount_ok, if possible
        self.temp_dir = tempfile.mkdtemp()
        if self.submount:
            mount_dir = os.path.normpath(os.path.sep.join([self.temp_dir, self.submount]))
            os.makedirs(mount_dir, mode=0o755, exist_ok=True)
        else:
            mount_dir = self.temp_dir
        for dev, size in self.loop_devices:
            try:
                mount( "/dev/mapper/"+dev, mnt=mount_dir )
                if self.mount_ok(mount_dir):
                    self.mount_dir = mount_dir
                    self.mount_dev = dev
                    self.mount_size = size
                    break
                umount( mount_dir )
            except CalledProcessError:
                logger.debug(traceback.format_exc())
        if self.mount_dir:
            logger.info("Partition mounted on %s size=%s", self.mount_dir, self.mount_size)
        else:
            logger.debug("Unable to mount anything from %s", self.disk_img)
            os.rmdir(self.temp_dir)
            self.temp_dir = None
        return self

        if self.temp_dir:
            umount(self.mount_dir)
            shutil.rmtree(self.temp_dir)
            self.mount_dir = None
        execWithRedirect("kpartx", ["-d", "-s", self.disk_img])

######## Functions for making filesystem images ##########################

def mkfsimage(fstype, rootdir, outfile, size=None, mkfsargs=None, mountargs="", graft=None):
    '''Generic filesystem image creation function.
    fstype should be a filesystem type - "mkfs.${fstype}" must exist.
    graft should be a dict: {"some/path/in/image": "local/file/or/dir"};
    if the path ends with a '/' it's assumed to be a directory.
    Will raise CalledProcessError if something goes wrong.'''
    mkfsargs = mkfsargs or []
    preserve = (fstype not in ("msdos", "vfat"))
    if not size:
        size = estimate_size(rootdir, graft, fstype)
    with LoopDev(outfile, size) as loopdev:
        try:
            runcmd(["mkfs.%s" % fstype] + mkfsargs + [loopdev])
        except CalledProcessError as e:
            logger.error("mkfs exited with a non-zero return code: %d", e.returncode)
            logger.error(e.output)
            sys.exit(e.returncode)

        with Mount(loopdev, mountargs) as mnt:
            if rootdir:
                copytree(rootdir, mnt, preserve)
            do_grafts(graft, mnt, preserve)

            # Save information about filesystem usage
            execWithRedirect("df", [mnt])

    # Make absolutely sure that the data has been written
    runcmd(["sync"])

# convenience functions with useful defaults
def mkdosimg(rootdir, outfile, size=None, label="", mountargs="shortname=winnt,umask=0077", graft=None):
    mkfsargs = ["-n", label]
    if 'SOURCE_DATE_EPOCH' in os.environ:
        mkfsargs.extend(["-i",
                "{:x}".format(int(os.environ['SOURCE_DATE_EPOCH']))])
    mkfsimage("msdos", rootdir, outfile, size, mountargs=mountargs,
              mkfsargs=mkfsargs, graft=graft)

def mkext4img(rootdir, outfile, size=None, label="", mountargs="", graft=None):
    mkfsimage("ext4", rootdir, outfile, size, mountargs=mountargs,
              mkfsargs=["-L", label, "-b", "4096", "-m", "0"], graft=graft)

def mkbtrfsimg(rootdir, outfile, size=None, label="", mountargs="", graft=None):
    mkfsimage("btrfs", rootdir, outfile, size, mountargs=mountargs,
               mkfsargs=["-L", label], graft=graft)

def mkhfsimg(rootdir, outfile, size=None, label="", mountargs="", graft=None):
    mkfsimage("hfsplus", rootdir, outfile, size, mountargs=mountargs,
              mkfsargs=["-v", label], graft=graft)

def mkfsimage_from_disk(diskimage, fsimage, img_size=None, label="Anaconda"):
    """
    Copy the / partition of a partitioned disk image to an un-partitioned
    disk image.

    :param str diskimage: The full path to partitioned disk image with a /
    :param str fsimage: The full path of the output fs image file
    :param int img_size: Optional size of the fsimage in MiB or None to make
       it as small as possible
    :param str label: The label to apply to the image. Defaults to "Anaconda"
    """
    with PartitionMount(diskimage) as img_mount:
        if not img_mount or not img_mount.mount_dir:

        logger.info("Creating fsimage %s (%s)", fsimage, img_size or "minimized")
        if img_size:
            # convert to Bytes
            img_size *= 1024**2

        mkext4img(img_mount.mount_dir, fsimage, size=img_size, label=label)

def default_image_name(compression, basename):
    """ Return a default image name with the correct suffix for the compression type.

    :param str compression: Compression type
    :param str basename: Base filename
    :returns: basename with compression suffix

    If the compression is unknown it defaults to xz
    """
    SUFFIXES = {"xz": ".xz", "gzip": ".gz", "bzip2": ".bz2", "lzma": ".lzma"}
    return basename + SUFFIXES.get(compression, ".xz")

#
#
#
#
#

import json
from math import ceil
import os
import socket

from pylorax.executils import execWithRedirect, execReadlines
from pylorax.imgutils import PartitionMount, mksparse, mkext4img, loop_detach
from pylorax.imgutils import get_loop_name, dm_detach, mount, umount
from pylorax.imgutils import mkqemu_img, mktar, mkcpio, mkfsimage_from_disk
from pylorax.monitor import LogMonitor
from pylorax.mount import IsoMountpoint
from pylorax.sysutils import joinpaths
from pylorax.treebuilder import udev_escape

ROOT_PATH = "/mnt/sysimage/"

class InstallError(Exception):
    pass

def create_vagrant_metadata(path, size=0):
    """ Create a default Vagrant metadata.json file

    :param str path: Path to metadata.json file
    :param int size: Disk size in MiB
    """
    metadata = { "provider":"libvirt", "format":"qcow2", "virtual_size": ceil(size / 1024) }
    with open(path, "wt") as f:
        json.dump(metadata, f, indent=4)

def update_vagrant_metadata(path, size):
    """ Update the Vagrant metadata.json file

    This function makes sure that the provider, format and virtual size of the
    metadata file are set correctly. All other values are left untouched.
    """
    with open(path, "rt") as f:
        try:
            metadata = json.load(f)
        except ValueError as e:
            log.error("Problem reading metadata file %s: %s", path, e)
            return

    metadata["provider"] = "libvirt"
    metadata["format"] = "qcow2"
    metadata["virtual_size"] = ceil(size / 1024)

def find_free_port(start=5900, end=5999, host="127.0.0.1"):
    """ Return first free port in range.

    :param int start: Starting port number
    :param int end: Ending port number
    :param str host: Host IP to search
    :returns: First free port or -1 if none found
    """
    s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    for port in range(start, end+1):
        try:
            s.bind((host, port))
            s.close()
            return port
        except OSError:
            pass

    return -1

def append_initrd(initrd, files):
    """ Append files to an initrd.

    :param str initrd: Path to initrd
    :param list files: list of file paths to add
    :returns: Path to a new initrd

    The files are added to the initrd by creating a cpio image
    of the files (stored at /) and writing the cpio to the end of a
    copy of the initrd.

    The initrd is not changed, a copy is made before appending the
    cpio archive.
    """
    qemu_initrd = tempfile.mktemp(prefix="lmc-initrd-", suffix=".img")
    shutil.copy2(initrd, qemu_initrd)
    ks_dir = tempfile.mkdtemp(prefix="lmc-ksdir-")
    for ks in files:
        shutil.copy2(ks, ks_dir)
    ks_initrd = tempfile.mktemp(prefix="lmc-ks-", suffix=".img")
    mkcpio(ks_dir, ks_initrd)
    shutil.rmtree(ks_dir)
    with open(qemu_initrd, "ab") as initrd_fp:
        with open(ks_initrd, "rb") as ks_fp:
                data = ks_fp.read(1024**2)
                if not data:
                    break
                initrd_fp.write(data)
    os.unlink(ks_initrd)

    return qemu_initrd

class QEMUInstall(object):
    """
    Run qemu using an iso and a kickstart
    """
    # Mapping of arch to qemu command
    QEMU_CMDS = {"x86_64":  "qemu-system-x86_64",
                 "i386":    "qemu-system-i386",
                 "arm":     "qemu-system-arm",
                 "aarch64": "qemu-system-aarch64",
                 "ppc64le": "qemu-system-ppc64"
                }

    def __init__(self, opts, iso, ks_paths, disk_img, img_size=2048,
                 kernel_args=None, memory=1024, vcpus=None, vnc=None, arch=None,
                 cancel_func=None, virtio_host="127.0.0.1", virtio_port=6080,
                 image_type=None, boot_uefi=False, ovmf_path=None):
        """
        Start the installation

        :param iso: Information about the iso to use for the installation
        :type iso: IsoMountpoint
        :param list ks_paths: Paths to kickstart files. All are injected, the
           first one is the one executed.
        :param str disk_img: Path to a disk image, created it it doesn't exist
        :param int img_size: The image size, in MiB, to create if it doesn't exist
        :param str kernel_args: Extra kernel arguments to pass on the kernel cmdline
        :param int memory: Amount of RAM to assign to the virt, in MiB
        :param int vcpus: Number of virtual cpus
        :param str vnc: Arguments to pass to qemu -display
        :param str arch: Optional architecture to use in the virt
        :param cancel_func: Function that returns True if the installation fails
        :type cancel_func: function
        :param str virtio_host: Hostname to connect virtio log to
        :param int virtio_port: Port to connect virtio log to
        :param str image_type: Type of qemu-img disk to create, or None.
        :param bool boot_uefi: Use OVMF to boot the VM in UEFI mode
        :param str ovmf_path: Path to the OVMF firmware
        """
        # Lookup qemu-system- for arch if passed, or try to guess using host arch
        qemu_cmd = [self.QEMU_CMDS.get(arch or os.uname().machine, "qemu-system-"+os.uname().machine)]
        if not os.path.exists("/usr/bin/"+qemu_cmd[0]):
            raise InstallError("%s does not exist, cannot run qemu" % qemu_cmd[0])

        qemu_cmd += ["-no-user-config"]
        qemu_cmd += ["-m", str(memory)]
        if vcpus:
            qemu_cmd += ["-smp", str(vcpus)]

        if not opts.no_kvm and os.path.exists("/dev/kvm"):
            qemu_cmd += ["-machine", "accel=kvm"]

        if boot_uefi:
            qemu_cmd += ["-machine", "q35,smm=on"]
            qemu_cmd += ["-global", "driver=cfi.pflash01,property=secure,value=on"]

        # Copy the initrd from the iso, create a cpio archive of the kickstart files
        # and append it to the temporary initrd.
        qemu_initrd = append_initrd(iso.initrd, ks_paths)
        qemu_cmd += ["-kernel", iso.kernel]
        qemu_cmd += ["-initrd", qemu_initrd]

        # Add the disk and cdrom
        if not os.path.isfile(disk_img):
            mksparse(disk_img, img_size * 1024**2)
        drive_args = "file=%s" % disk_img
        drive_args += ",cache=unsafe,discard=unmap"
        if image_type:
            drive_args += ",format=%s" % image_type
        else:
            drive_args += ",format=raw"
        qemu_cmd += ["-drive", drive_args]

        drive_args = "file=%s,media=cdrom,readonly=on" % iso.iso_path

        # Setup the cmdline args
        # ======================
        cmdline_args = "ks=file:/%s" % os.path.basename(ks_paths[0])
        cmdline_args += " inst.stage2=hd:LABEL=%s" % udev_escape(iso.label)
        if opts.proxy:
            cmdline_args += " inst.proxy=%s" % opts.proxy
        if kernel_args:
            cmdline_args += " "+kernel_args
        cmdline_args += " inst.text inst.cmdline"

        qemu_cmd += ["-append", cmdline_args]

        if not opts.vnc:
            vnc_port = find_free_port()
            if vnc_port == -1:
                raise InstallError("No free VNC ports")
            display_args = "vnc=127.0.0.1:%d" % (vnc_port - 5900)
        else:
            display_args = opts.vnc
        log.info("qemu %s", display_args)
        qemu_cmd += ["-nographic", "-monitor", "none", "-serial", "null", "-display", display_args ]

        # Setup the virtio log port
        qemu_cmd += ["-device", "virtio-serial-pci,id=virtio-serial0"]
        qemu_cmd += ["-device", "virtserialport,bus=virtio-serial0.0,nr=1,chardev=charchannel0"
                                ",id=channel0,name=org.fedoraproject.anaconda.log.0"]
        qemu_cmd += ["-chardev", "socket,id=charchannel0,host=%s,port=%s" % (virtio_host, virtio_port)]

        # Pass through rng from host
        if opts.with_rng != "none":
            qemu_cmd += ["-object", "rng-random,id=virtio-rng0,filename=%s" % opts.with_rng]
            if boot_uefi:
                qemu_cmd += ["-device", "virtio-rng-pci,rng=virtio-rng0,id=rng0,bus=pcie.0,addr=0x9"]
            else:
                qemu_cmd += ["-device", "virtio-rng-pci,rng=virtio-rng0,id=rng0,bus=pci.0,addr=0x9"]

        if boot_uefi and ovmf_path:
            qemu_cmd += ["-drive", "file=%s/OVMF_CODE.secboot.fd,if=pflash,format=raw,unit=0,readonly=on" % ovmf_path]

            # Make a copy of the OVMF_VARS.secboot.fd for this run
            ovmf_vars = tempfile.mktemp(prefix="lmc-OVMF_VARS-", suffix=".fd")
            shutil.copy2(joinpaths(ovmf_path, "/OVMF_VARS.secboot.fd"), ovmf_vars)

            qemu_cmd += ["-drive", "file=%s,if=pflash,format=raw,unit=1" % ovmf_vars]

        log.info("Running qemu")
        log.debug(qemu_cmd)
        try:
            execWithRedirect(qemu_cmd[0], qemu_cmd[1:], reset_lang=False, raise_err=True,
                             callback=lambda p: not (cancel_func and cancel_func()))
            log.error("Running qemu failed:")
            log.error("cmd: %s", " ".join(e.cmd))
            log.error("output: %s", e.output or "")
            raise InstallError("QEMUInstall failed")
        except (OSError, KeyboardInterrupt) as e:
            log.error("Running qemu failed: %s", str(e))
        finally:
            os.unlink(qemu_initrd)
            if boot_uefi and ovmf_path:
                os.unlink(ovmf_vars)

        if cancel_func and cancel_func():
            log.error("Installation error detected. See logfile for details.")
        else:
            log.info("Installation finished without errors.")

def novirt_cancel_check(cancel_funcs, proc):
    """
    Check to see if there has been an error in the logs

    :param cancel_funcs: list of functions to call, True from any one cancels the build
    :type cancel_funcs: list
    :param proc: Popen object for the anaconda process
    :type proc: subprocess.Popen
    :returns: True if the process has been terminated

    The cancel_funcs functions should return a True if an error has been detected.
    When an error is detected the process is terminated and this returns True
    """
    for f in cancel_funcs:
        if f():
            # Anaconda runs from unshare, anaconda doesn't exit correctly so try to
            # send TERM to all of them directly
            import psutil
            for p in psutil.Process(proc.pid).children(recursive=True):
                p.terminate()
            psutil.Process(proc.pid).terminate()
            return True

def anaconda_cleanup(dirinstall_path):
    """
    Cleanup any leftover mounts from anaconda

    :param str dirinstall_path: Path where anaconda mounts things
    :returns: True if cleanups were successful. False if any of them failed.

    If anaconda crashes it may leave things mounted under this path. It will
    typically be set to /mnt/sysimage/

    Attempts to cleanup may also fail. Catch these and continue trying the
    other mountpoints.

    Anaconda may also leave /run/anaconda.pid behind, clean that up as well.
    """
    # Anaconda may not clean up its /var/run/anaconda.pid file
    # Make sure the process is really finished (it should be, since it was started from a subprocess call)
    # and then remove the pid file.
    if os.path.exists("/var/run/anaconda.pid"):
        # anaconda may be started using unshare so the pid is always 1
        if open("/var/run/anaconda.pid").read().strip() == "1":
            os.unlink("/var/run/anaconda.pid")

    rc = True
    dirinstall_path = os.path.abspath(dirinstall_path)
    # unmount filesystems
    for mounted in reversed(open("/proc/mounts").readlines()):
        (_device, mountpoint, _rest) = mounted.split(" ", 2)
        if mountpoint.startswith(dirinstall_path) and os.path.ismount(mountpoint):
            try:
                umount(mountpoint)
            except subprocess.CalledProcessError:
                log.error("Cleanup of %s failed. See program.log for details", mountpoint)
                rc = False
    return rc

def novirt_install(opts, disk_img, disk_size, cancel_func=None, tar_img=None):
    """
    Use Anaconda to install to a disk image

    :param str disk_img: The full path to the disk image to be created
    :param int disk_size: The size of the disk_img in MiB
    :param str tar_img: For make_tar_disk, the path to final tarball to be created

    This method runs anaconda to create the image and then based on the opts
    passed creates a qemu disk image or tarfile.
    """
    dirinstall_path = ROOT_PATH

    # Clean up /tmp/ from previous runs to prevent stale info from being used
    for path in ["/tmp/yum.repos.d/", "/tmp/yum.cache/"]:
        if os.path.isdir(path):
            shutil.rmtree(path)

    args = ["--kickstart", opts.ks[0], "--cmdline"]
    if opts.anaconda_args:
        for arg in opts.anaconda_args:
    if opts.proxy:
        args += ["--proxy", opts.proxy]
    if opts.armplatform:
        args += ["--armplatform", opts.armplatform]

    if opts.make_iso or opts.make_fsimage or opts.make_pxe_live:
        # Make a blank fs image
        args += ["--dirinstall"]

        mkext4img(None, disk_img, label=opts.fs_label, size=disk_size * 1024**2)
        if not os.path.isdir(dirinstall_path):
            os.mkdir(dirinstall_path)
        mount(disk_img, opts="loop", mnt=dirinstall_path)
    elif opts.make_tar or opts.make_oci:
        # Install under dirinstall_path, make sure it starts clean
        if os.path.exists(dirinstall_path):
            shutil.rmtree(dirinstall_path)

        if opts.make_oci:
            # OCI installs under /rootfs/
            dirinstall_path = joinpaths(dirinstall_path, "rootfs")
            args += ["--dirinstall", dirinstall_path]
        else:
            args += ["--dirinstall"]

        os.makedirs(dirinstall_path)
    else:
        args += ["--image", disk_img]

        # Create the sparse image
        mksparse(disk_img, disk_size * 1024**2)

    log_monitor = LogMonitor(timeout=opts.timeout)
    args += ["--remotelog", "%s:%s" % (log_monitor.host, log_monitor.port)]
    cancel_funcs = [log_monitor.server.log_check]
    if cancel_func is not None:
        cancel_funcs.append(cancel_func)

    # Make sure anaconda has the right product and release
    log.info("Running anaconda.")
    try:
        unshare_args = ["--pid", "--kill-child", "--mount", "--propagation", "unchanged", "anaconda"] + args
        for line in execReadlines("unshare", unshare_args, reset_lang=False,
                                  env_add={"ANACONDA_PRODUCTNAME": opts.project,
                                           "ANACONDA_PRODUCTVERSION": opts.releasever},
                                  callback=lambda p: not novirt_cancel_check(cancel_funcs, p)):
            log.info(line)

        # Make sure the new filesystem is correctly labeled
        setfiles_args = ["-e", "/proc", "-e", "/sys",
                         "/etc/selinux/targeted/contexts/files/file_contexts", "/"]

        if "--dirinstall" in args:
            # setfiles may not be available, warn instead of fail
            try:
                execWithRedirect("setfiles", setfiles_args, root=dirinstall_path)
            except (subprocess.CalledProcessError, OSError) as e:
                log.warning("Running setfiles on install tree failed: %s", str(e))
        else:
                    try:
                        execWithRedirect("setfiles", setfiles_args, root=img_mount.mount_dir)
                    except (subprocess.CalledProcessError, OSError) as e:
                        log.warning("Running setfiles on install tree failed: %s", str(e))

                    # For image installs, run fstrim to discard unused blocks. This way
                    # unused blocks do not need to be allocated for sparse image types
                    execWithRedirect("fstrim", [img_mount.mount_dir])

    except (subprocess.CalledProcessError, OSError) as e:
        log.error("Running anaconda failed: %s", e)
        raise InstallError("novirt_install failed")
    finally:
        log_monitor.shutdown()

        # Move the anaconda logs over to a log directory
        log_dir = os.path.abspath(os.path.dirname(opts.logfile))
        log_anaconda = joinpaths(log_dir, "anaconda")
        if not os.path.isdir(log_anaconda):
            os.mkdir(log_anaconda)
        for l in glob.glob("/tmp/*log")+glob.glob("/tmp/anaconda-tb-*"):
            shutil.copy2(l, log_anaconda)
            os.unlink(l)

        # Make sure any leftover anaconda mounts have been cleaned up
        if not anaconda_cleanup(dirinstall_path):
            raise InstallError("novirt_install cleanup of anaconda mounts failed.")

        if not opts.make_iso and not opts.make_fsimage and not opts.make_pxe_live:
            dm_name = os.path.splitext(os.path.basename(disk_img))[0]

            # Remove device-mapper for partitions and disk
            log.debug("Removing device-mapper setup on %s", dm_name)
            for d in sorted(glob.glob("/dev/mapper/"+dm_name+"*"), reverse=True):
                dm_detach(d)

            log.debug("Removing loop device for %s", disk_img)
            loop_detach("/dev/"+get_loop_name(disk_img))

    # qemu disk image is used by bare qcow2 images and by Vagrant
    if opts.image_type:
        log.info("Converting %s to %s", disk_img, opts.image_type)
        qemu_args = []
        for arg in opts.qemu_args:
            qemu_args += arg.split(" ", 1)

        # convert the image to the selected format
        if "-O" not in qemu_args:
            qemu_args.extend(["-O", opts.image_type])
        qemu_img = tempfile.mktemp(prefix="lmc-disk-", suffix=".img")
        execWithRedirect("qemu-img", ["convert"] + qemu_args + [disk_img, qemu_img], raise_err=True)
        if not opts.make_vagrant:
            execWithRedirect("mv", ["-f", qemu_img, disk_img], raise_err=True)
        else:
            # Take the new qcow2 image and package it up for Vagrant
            compress_args = []
            for arg in opts.compress_args:
                compress_args += arg.split(" ", 1)

            vagrant_dir = tempfile.mkdtemp(prefix="lmc-tmpdir-")
            metadata_path = joinpaths(vagrant_dir, "metadata.json")
            execWithRedirect("mv", ["-f", qemu_img, joinpaths(vagrant_dir, "box.img")], raise_err=True)
            if opts.vagrant_metadata:
                shutil.copy2(opts.vagrant_metadata, metadata_path)
            else:
                create_vagrant_metadata(metadata_path)
            update_vagrant_metadata(metadata_path, disk_size)
            if opts.vagrantfile:
                shutil.copy2(opts.vagrantfile, joinpaths(vagrant_dir, "vagrantfile"))

            log.info("Creating Vagrant image")
            rc = mktar(vagrant_dir, disk_img, opts.compression, compress_args, selinux=False)
            if rc:
                raise InstallError("novirt_install mktar failed: rc=%s" % rc)
            shutil.rmtree(vagrant_dir)
    elif opts.make_tar:
        compress_args = []
            compress_args += arg.split(" ", 1)

        rc = mktar(dirinstall_path, disk_img, opts.compression, compress_args)
        shutil.rmtree(dirinstall_path)

        if rc:
            raise InstallError("novirt_install mktar failed: rc=%s" % rc)
    elif opts.make_oci:
        # An OCI image places the filesystem under /rootfs/ and adds the json files at the top
        # And then creates a tar of the whole thing.

        shutil.copy2(opts.oci_config, ROOT_PATH)
        shutil.copy2(opts.oci_runtime, ROOT_PATH)
        rc = mktar(ROOT_PATH, disk_img, opts.compression, compress_args)

        if rc:
    else:
        # For raw disk images, use fallocate to deallocate unused space
        execWithRedirect("fallocate", ["--dig-holes", disk_img], raise_err=True)

    # For make_tar_disk, wrap the result in a tar file, and remove the original disk image.

        rc = mktar(disk_img, tar_img, opts.compression, compress_args, selinux=False)

        if rc:

        os.unlink(disk_img)

def virt_install(opts, install_log, disk_img, disk_size, cancel_func=None, tar_img=None):
    """
    Use qemu to install to a disk image

    :param str install_log: The path to write the log from qemu

    This uses qemu with a boot.iso and a kickstart to create a disk
    image and then optionally, based on the opts passed, creates tarfile.
    """
    iso_mount = IsoMountpoint(opts.iso, opts.location)
    if not iso_mount.stage2:
        iso_mount.umount()
        raise InstallError("ISO is missing stage2, cannot continue")

    log_monitor = LogMonitor(install_log, timeout=opts.timeout)

    kernel_args = ""
    if opts.kernel_args:
        kernel_args += opts.kernel_args
        kernel_args += " proxy="+opts.proxy

    if opts.image_type and not opts.make_fsimage:
        if "-f" not in qemu_args:
            qemu_args += ["-f", opts.image_type]

        mkqemu_img(disk_img, disk_size*1024**2, qemu_args)

    if opts.make_fsimage or opts.make_tar or opts.make_oci:
        diskimg_path = tempfile.mktemp(prefix="lmc-disk-", suffix=".img")
    else:
        diskimg_path = disk_img

    try:
        QEMUInstall(opts, iso_mount, opts.ks, diskimg_path, disk_size,
                    kernel_args, opts.ram, opts.vcpus, opts.vnc, opts.arch,
                    cancel_func = lambda : any(f() for f in cancel_funcs),
                    virtio_host = log_monitor.host,
                    virtio_port = log_monitor.port,
                    image_type=opts.image_type, boot_uefi=opts.virt_uefi,
                    ovmf_path=opts.ovmf_path)
        log.error("VirtualInstall failed: %s", e)
        raise
    finally:
        log.info("unmounting the iso")

    if log_monitor.server.log_check():
        if not log_monitor.server.error_line and opts.timeout:
            msg = "virt_install failed due to timeout"
        else:
            msg = "virt_install failed on line: %s" % log_monitor.server.error_line
        raise InstallError(msg)
    elif cancel_func and cancel_func():
        raise InstallError("virt_install canceled by cancel_func")

    if opts.make_fsimage:
        mkfsimage_from_disk(diskimg_path, disk_img, disk_size, label=opts.fs_label)
        os.unlink(diskimg_path)

        with PartitionMount(diskimg_path) as img_mount:
                rc = mktar(img_mount.mount_dir, disk_img, opts.compression, compress_args)
            else:
                rc = 1

        if rc:
            raise InstallError("virt_install failed")

        with PartitionMount(diskimg_path, submount="rootfs") as img_mount:
            if img_mount and img_mount.temp_dir:
                shutil.copy2(opts.oci_config, img_mount.temp_dir)
                shutil.copy2(opts.oci_runtime, img_mount.temp_dir)
                rc = mktar(img_mount.temp_dir, disk_img, opts.compression, compress_args)
            else:
                rc = 1

        if rc:
    elif opts.make_vagrant:

        vagrant_dir = tempfile.mkdtemp(prefix="lmc-tmpdir-")
        metadata_path = joinpaths(vagrant_dir, "metadata.json")
        execWithRedirect("mv", ["-f", disk_img, joinpaths(vagrant_dir, "box.img")], raise_err=True)
        if opts.vagrant_metadata:
            shutil.copy2(opts.vagrant_metadata, metadata_path)
        else:
            create_vagrant_metadata(metadata_path)
        update_vagrant_metadata(metadata_path, disk_size)
        if opts.vagrantfile:
            shutil.copy2(opts.vagrantfile, joinpaths(vagrant_dir, "vagrantfile"))

        rc = mktar(vagrant_dir, disk_img, opts.compression, compress_args, selinux=False)
        if rc:
        shutil.rmtree(vagrant_dir)

        if rc:
            raise InstallError("virt_install mktar failed: rc=%s" % rc)

#
# ltmpl.py
#
# Copyright (C) 2009-2018  Red Hat, Inc.
#
#
#
#
#                     Will Woods <wwoods@redhat.com>
#

logger = logging.getLogger("pylorax.ltmpl")

import os, re, glob, shlex, fnmatch
from os.path import basename, isdir
from subprocess import CalledProcessError

from pylorax.sysutils import joinpaths, cpfile, mvfile, replace, remove
from pylorax.dnfhelper import LoraxDownloadCallback, LoraxRpmCallback
from pylorax.imgutils import mkcpio

from mako.lookup import TemplateLookup
import sys, traceback
import struct
import dnf
import collections.abc

class LoraxTemplate(object):
    def __init__(self, directories=None):
        directories = directories or ["/usr/share/lorax"]
        # we have to add ["/"] to the template lookup directories or the
        # file includes won't work properly for absolute paths
        self.directories = ["/"] + directories

    def parse(self, template_file, variables):
        lookup = TemplateLookup(directories=self.directories)
        template = lookup.get_template(template_file)

        try:
            textbuf = template.render(**variables)
        except:
            logger.error("Problem rendering %s (%s):", template_file, variables)
            logger.error(text_error_template().render())
            raise

        # split, strip and remove empty lines
        lines = textbuf.splitlines()
        lines = [line.strip() for line in lines]
        lines = [line for line in lines if line]

        # remove comments
        lines = [line for line in lines if not line.startswith("#")]

        # split with shlex and perform brace expansion. This can fail, so we unroll the loop
        # for better error reporting.
        expanded_lines = []
        try:
            for line in lines:
                expanded_lines.append(split_and_expand(line))
        except Exception as e:
            logger.error('shlex error processing "%s": %s', line, str(e))
            raise
        return expanded_lines

def split_and_expand(line):
    return [exp for word in shlex.split(line) for exp in brace_expand(word)]

def brace_expand(s):
    if not ('{' in s and ',' in s and '}' in s):
        yield s
    else:
        right = s.find('}')
        left = s[:right].rfind('{')
        (prefix, choices, suffix) = (s[:left], s[left+1:right], s[right+1:])
        for choice in choices.split(','):
            for alt in brace_expand(prefix+choice+suffix):
                yield alt

def rglob(pathname, root="/", fatal=False):
    seen = set()
    rootlen = len(root)+1
    for f in glob.iglob(joinpaths(root, pathname)):
        if f not in seen:
            seen.add(f)
            yield f[rootlen:] # remove the root to produce relative path
    if fatal and not seen:
        raise IOError("nothing matching %s in %s" % (pathname, root))

def rexists(pathname, root=""):
    # Generator is always True, even with no values;
    # bool(rglob(...)) won't work here.
    for _path in rglob(pathname, root):
        return True

class TemplateRunner(object):
    '''
    This class parses and executes Lorax templates. Sample usage:

      # install a bunch of packages
      runner = LoraxTemplateRunner(inroot=rundir, outroot=rundir, dbo=dnf_obj)
      runner.run("install-packages.ltmpl")

    NOTES:

    * Parsing procedure is roughly:
      1. Mako template expansion (on the whole file)
      2. For each line of the result,

        a. Whitespace splitting (using shlex.split())
        b. Brace expansion (using brace_expand())
        c. If the first token is the name of a function, call that function
           with the rest of the line as arguments

    * Parsing and execution are *separate* passes - so you can't use the result
      of a command in an %if statement (or any other control statements)!
    '''
    def __init__(self, fatalerrors=True, templatedir=None, defaults=None, builtins=None):
        self.fatalerrors = fatalerrors
        self.templatedir = templatedir or "/usr/share/lorax"
        self.templatefile = None
        self.builtins = builtins or {}
        self.defaults = defaults or {}

    def run(self, templatefile, **variables):
        for k,v in list(self.defaults.items()) + list(self.builtins.items()):
            variables.setdefault(k,v)
        logger.debug("executing %s with variables=%s", templatefile, variables)
        self.templatefile = templatefile
        t = LoraxTemplate(directories=[self.templatedir])
        commands = t.parse(templatefile, variables)
        self._run(commands)

    def _run(self, parsed_template):
        logger.info("running %s", self.templatefile)
        for (num, line) in enumerate(parsed_template,1):
            logger.debug("template line %i: %s", num, " ".join(line))
            skiperror = False
            (cmd, args) = (line[0], line[1:])
            # Following Makefile convention, if the command is prefixed with
            # a dash ('-'), we'll ignore any errors on that line.
            if cmd.startswith('-'):
                cmd = cmd[1:]
                skiperror = True
            try:
                # grab the method named in cmd and pass it the given arguments
                f = getattr(self, cmd, None)
                if cmd[0] == '_' or cmd == 'run' or not isinstance(f, collections.abc.Callable):
                    raise ValueError("unknown command %s" % cmd)
                f(*args)
            except Exception: # pylint: disable=broad-except
                if skiperror:
                    logger.debug("ignoring error")
                    continue
                logger.error("template command error in %s:", self.templatefile)
                logger.error("  %s", " ".join(line))
                # format the exception traceback
                exclines = traceback.format_exception(*sys.exc_info())
                # skip the bit about "ltmpl.py, in _run()" - we know that
                exclines.pop(1)
                # log the "ErrorType: this is what happened" line
                logger.error("  %s", exclines[-1].strip())
                # and log the entire traceback to the debug log
                for _line in ''.join(exclines).splitlines():
                    logger.debug("  %s", _line)
                if self.fatalerrors:
                    raise

# TODO: operate inside an actual chroot for safety? Not that RPM bothers..
class LoraxTemplateRunner(TemplateRunner):
    '''

      # modify a runtime dir
      runner = LoraxTemplateRunner(inroot=rundir, outroot=newrun)
      runner.run("runtime-transmogrify.ltmpl")

    NOTES:

    * Commands that run external programs (e.g. systemctl) currently use
      the *host*'s copy of that program, which may cause problems if there's a
      big enough difference between the host and the image you're modifying.

    * The commands are not executed under a real chroot, so absolute symlinks
      will point *outside* the inroot/outroot. Be careful with symlinks!

    ADDING NEW COMMANDS:

    * Each template command is just a method of the LoraxTemplateRunner
      object - so adding a new command is as easy as adding a new function.

    * Each function gets arguments that correspond to the rest of the tokens
      on that line (after word splitting and brace expansion)

    * Commands should raise exceptions for errors - don't use sys.exit()
    '''
    def __init__(self, inroot, outroot, dbo=None, fatalerrors=True,
                                        templatedir=None, defaults=None):
        self.inroot = inroot
        self.outroot = outroot
        self.dbo = dbo
        builtins = DataHolder(exists=lambda p: rexists(p, root=inroot),
                              glob=lambda g: list(rglob(g, root=inroot)))
        self.results = DataHolder(treeinfo=dict()) # just treeinfo for now

        super(LoraxTemplateRunner, self).__init__(fatalerrors, templatedir, defaults, builtins)
        # TODO: set up custom logger with a filter to add line info

    def _out(self, path):
        return joinpaths(self.outroot, path)
    def _in(self, path):
        return joinpaths(self.inroot, path)

    def _filelist(self, *pkgs):
        """ Return the list of files in the packages """
        pkglist = []
        for pkg_glob in pkgs:
            pkglist += list(self.dbo.sack.query().installed().filter(name__glob=pkg_glob))

        # dnf/hawkey doesn't make any distinction between file, dir or ghost like yum did
        # so only return the files.
        return set(f for pkg in pkglist for f in pkg.files if not os.path.isdir(self._out(f)))

    def _getsize(self, *files):
        return sum(os.path.getsize(self._out(f)) for f in files if os.path.isfile(self._out(f)))

    def _write_package_log(self):
        """
        Write the list of installed packages to /root/ on the boot.iso

        If lorax is called with a debug repo find the corresponding debuginfo package
        names and write them to /root/debubg-pkgs.log on the boot.iso
        The non-debuginfo packages are written to /root/lorax-packages.log
        """
        os.makedirs(self._out("root/"), exist_ok=True)
        available = self.dbo.sack.query().available()
        pkgs = []
        debug_pkgs = []
        for p in list(self.dbo.transaction.install_set):
            pkgs.append(f"{p.name}-{p.version}-{p.release}.{p.arch}")
            if available.filter(name=p.name+"-debuginfo"):
                debug_pkgs.append(f"{p.name}-debuginfo-{p.epoch}:{p.version}-{p.release}")

        with open(self._out("root/lorax-packages.log"), "w") as f:
            f.write("\n".join(sorted(pkgs)))
            f.write("\n")

        if debug_pkgs:
            with open(self._out("root/debug-pkgs.log"), "w") as f:
                f.write("\n".join(sorted(debug_pkgs)))
                f.write("\n")

    def install(self, srcglob, dest):
        '''
        install SRC DEST
          Copy the given file (or files, if a glob is used) from the input
          tree to the given destination in the output tree.
          The path to DEST must exist in the output tree.
          If DEST is a directory, SRC will be copied into that directory.
          If DEST doesn't exist, SRC will be copied to a file with that name,
          assuming the rest of the path exists.
          This is pretty much like how the 'cp' command works.

          Examples:
            install usr/share/myconfig/grub.conf /boot
            install /usr/share/myconfig/grub.conf.in /boot/grub.conf
        '''
        for src in rglob(self._in(srcglob), fatal=True):
            try:
                cpfile(src, self._out(dest))
            except shutil.Error as e:
                logger.error(e)

    def installimg(self, *args):
        '''
        installimg [--xz|--gzip|--bzip2|--lzma] [-ARG|--ARG=OPTION] SRCDIR DESTFILE
          Create a compressed cpio archive of the contents of SRCDIR and place
          it in DESTFILE.

          If SRCDIR doesn't exist or is empty nothing is created.

          Examples:
            installimg ${LORAXDIR}/product/ images/product.img
            installimg ${LORAXDIR}/updates/ images/updates.img
            installimg --xz -6 ${LORAXDIR}/updates/ images/updates.img
            installimg --xz -9 --memlimit-compress=3700MiB ${LORAXDIR}/updates/ images/updates.img

          Optionally use a different compression type and override the default args
          passed to it. The default is xz -9
        '''
        COMPRESSORS = ("--xz", "--gzip", "--bzip2", "--lzma")
        if len(args) < 2:
            raise ValueError("Not enough args for installimg.")

        srcdir = args[-2]
        destfile = args[-1]
        if not os.path.isdir(self._in(srcdir)) or not os.listdir(self._in(srcdir)):
            return

        compression = "xz"
        if args[0] in COMPRESSORS:
            compression = args[0][2:]

        for arg in args[1:-2]:
            if arg.startswith('-'):
                compressargs.append(arg)
            else:
                raise ValueError("Argument is missing -")

        logger.info("Creating image file %s from contents of %s", self._out(destfile), self._in(srcdir))
        logger.debug("Using %s %s compression", compression, compressargs or "")
        mkcpio(self._in(srcdir), self._out(destfile), compression=compression, compressargs=compressargs)

    def mkdir(self, *dirs):
        '''
        mkdir DIR [DIR ...]
          Create the named DIR(s). Will create leading directories as needed.

          Example:
            mkdir /images
        '''
        for d in dirs:
            d = self._out(d)
            if not isdir(d):
                os.makedirs(d)

    def replace(self, pat, repl, *fileglobs):
        '''
        replace PATTERN REPLACEMENT FILEGLOB [FILEGLOB ...]
          Find-and-replace the given PATTERN (Python-style regex) with the given
          REPLACEMENT string for each of the files listed.

          Example:
            replace @VERSION@ ${product.version} /boot/grub.conf /boot/isolinux.cfg
        '''
        match = False
        for g in fileglobs:
            for f in rglob(self._out(g)):
                match = True
                replace(f, pat, repl)
        if not match:
            raise IOError("no files matched %s" % " ".join(fileglobs))

    def append(self, filename, data):
        '''
        append FILE STRING
          Append STRING (followed by a newline character) to FILE.
          Python character escape sequences ('\\n', '\\t', etc.) will be
          converted to the appropriate characters.

          Examples:

            append /etc/depmod.d/dd.conf "search updates built-in"
            append /etc/resolv.conf ""
        '''
        with open(self._out(filename), "a") as fobj:
            fobj.write(bytes(data, "utf8").decode('unicode_escape')+"\n")

    def treeinfo(self, section, key, *valuetoks):
        '''
        treeinfo SECTION KEY ARG [ARG ...]
          Add an item to the treeinfo data store.
          The given SECTION will have a new item added where
          KEY = ARG ARG ...

          Example:
            treeinfo images-${kernel.arch} boot.iso images/boot.iso
        '''
        if section not in self.results.treeinfo:
            self.results.treeinfo[section] = dict()
        self.results.treeinfo[section][key] = " ".join(valuetoks)

    def installkernel(self, section, src, dest):
        '''
        installkernel SECTION SRC DEST
          Install the kernel from SRC in the input tree to DEST in the output
          tree, and then add an item to the treeinfo data store, in the named
          SECTION, where "kernel" = DEST.

          Equivalent to:
            install SRC DEST
            treeinfo SECTION kernel DEST
        '''
        self.install(src, dest)
        self.treeinfo(section, "kernel", dest)

    def installinitrd(self, section, src, dest):
        '''
        installinitrd SECTION SRC DEST
          Same as installkernel, but for "initrd".
        '''
        self.chmod(dest, '644')
        self.treeinfo(section, "initrd", dest)

    def installupgradeinitrd(self, section, src, dest):
        '''
        installupgradeinitrd SECTION SRC DEST
          Same as installkernel, but for "upgrade".
        '''
        self.treeinfo(section, "upgrade", dest)

    def hardlink(self, src, dest):
        '''
        hardlink SRC DEST
          Create a hardlink at DEST which is linked to SRC.
        '''
        if isdir(self._out(dest)):
            dest = joinpaths(dest, basename(src))
        os.link(self._out(src), self._out(dest))

    def symlink(self, target, dest):
        '''
        symlink SRC DEST
          Create a symlink at DEST which points to SRC.
        '''
        if rexists(self._out(dest)):
            self.remove(dest)
        os.symlink(target, self._out(dest))

    def copy(self, src, dest):
        '''
        copy SRC DEST
          Copy SRC to DEST.
          If DEST is a directory, SRC will be copied inside it.
          If DEST doesn't exist, SRC will be copied to a file with
          that name, if the path leading to it exists.
        '''
        try:
            cpfile(self._out(src), self._out(dest))
        except shutil.Error as e:
            logger.error(e)

    def move(self, src, dest):
        '''
        move SRC DEST
          Move SRC to DEST.
        '''
        mvfile(self._out(src), self._out(dest))

    def remove(self, *fileglobs):
        '''
        remove FILEGLOB [FILEGLOB ...]
          Remove all the named files or directories.
          Will *not* raise exceptions if the file(s) are not found.
        '''
                remove(f)
                logger.debug("removed %s", f)

    def chmod(self, fileglob, mode):
        '''
        chmod FILEGLOB OCTALMODE
          Change the mode of all the files matching FILEGLOB to OCTALMODE.
        '''
        for f in rglob(self._out(fileglob), fatal=True):
            os.chmod(f, int(mode,8))

    def log(self, msg):
        '''
        log MESSAGE
          Emit the given log message. Be sure to put it in quotes!

          Example:
            log "Reticulating splines, please wait..."
        '''
        logger.info(msg)

    # TODO: add ssh-keygen, mkisofs(?), find, and other useful commands
    def runcmd(self, *cmdlist):
        '''
        runcmd CMD [ARG ...]
          Run the given command with the given arguments.

          NOTE: All paths given MUST be COMPLETE, ABSOLUTE PATHS to the file
          or files mentioned. ${root}/${inroot}/${outroot} are good for
          constructing these paths.

          FURTHER NOTE: Please use this command only as a last resort!
          Whenever possible, you should use the existing template commands.
          If the existing commands don't do what you need, fix them!

          Examples:
            (this should be replaced with a "find" function)
            runcmd find ${root} -name "*.pyo" -type f -delete
            %for f in find(root, name="*.pyo"):
            remove ${f}
            %endfor
        '''
        cmd = cmdlist
        logger.debug('running command: %s', cmd)
        if cmd[0].startswith("--chdir="):
            logger.error("--chdir is no longer supported for runcmd.")
            raise ValueError("--chdir is no longer supported for runcmd.")

        try:
            stdout = runcmd_output(cmd)
            if stdout:
                logger.debug('command output:\n%s', stdout)
            logger.debug("command finished successfully")
            if e.output:
                logger.error('command output:\n%s', e.output)
            logger.error('command returned failure (%d)', e.returncode)
            raise

    def installpkg(self, *pkgs):
        '''
        installpkg [--required|--optional] [--except PKGGLOB [--except PKGGLOB ...]] PKGGLOB [PKGGLOB ...]
          Request installation of all packages matching the given globs.
          Note that this is just a *request* - nothing is *actually* installed
          until the 'run_pkg_transaction' command is given.

          --required is now the default. If the PKGGLOB can be missing pass --optional
        '''
        if pkgs[0] == '--optional':
            pkgs = pkgs[1:]
            required = False
        elif pkgs[0] == '--required':
            required = True
        else:

        excludes = []
        while '--except' in pkgs:
            idx = pkgs.index('--except')
            if len(pkgs) == idx+1:
                raise ValueError("installpkg needs an argument after --except")

            excludes.append(pkgs[idx+1])
            pkgs = pkgs[:idx] + pkgs[idx+2:]

        errors = False
        for p in pkgs:
            try:
                # Start by using Subject to generate a package query, which will
                # give us a query object similar to what dbo.install would select,
                # minus the handling for multilib. This query may contain
                # multiple arches. Pull the package names out of that, filter any
                # that match the excludes patterns, and pass those names back to
                # dbo.install to do the actual, arch and version and multilib
                # aware, package selction.

                # dnf queries don't have a concept of negative globs which is why
                # the filtering is done the hard way.

                pkgnames = [pkg for pkg in dnf.subject.Subject(p).get_best_query(self.dbo.sack).filter(latest=True)]
                if not pkgnames:
                    raise dnf.exceptions.PackageNotFoundError("no package matched", p)

                # Apply excludes to the name only
                for exclude in excludes:
                    pkgnames = [pkg for pkg in pkgnames if not fnmatch.fnmatch(pkg.name, exclude)]

                # Convert to a sorted NVR list for installation
                pkgnvrs = sorted(["{}-{}-{}".format(pkg.name, pkg.version, pkg.release) for pkg in pkgnames])

                # If the request is a glob, expand it in the log
                if any(g for g in ['*','?','.'] if g in p):
                    logger.info("installpkg: %s expands to %s", p, ",".join(pkgnvrs))

                for pkgnvr in pkgnvrs:
                    try:
                        self.dbo.install(pkgnvr)
                    except Exception as e: # pylint: disable=broad-except
                        if required:
                            raise
                        # Not required, log it and continue processing pkgs
                        logger.error("installpkg %s failed: %s", pkgnvr, str(e))
            except Exception as e: # pylint: disable=broad-except
                logger.error("installpkg %s failed: %s", p, str(e))
                errors = True

        if errors and required:
            raise Exception("Required installpkg failed.")

    def removepkg(self, *pkgs):
        '''
        removepkg PKGGLOB [PKGGLOB...]
          Delete the named package(s).

          IMPLEMENTATION NOTES:
            RPM scriptlets (%preun/%postun) are *not* run.
            Files are deleted, but directories are left behind.
        '''
            filepaths = [f.lstrip('/') for f in self._filelist(p)]
            # TODO: also remove directories that aren't owned by anything else
            if filepaths:
                logger.debug("removepkg %s: %ikb", p, self._getsize(*filepaths)/1024)
                self.remove(*filepaths)
            else:
                logger.debug("removepkg %s: no files to remove!", p)

    def run_pkg_transaction(self):
        '''
        run_pkg_transaction
          Actually install all the packages requested by previous 'installpkg'
          commands.
        '''
        try:
            logger.info("Checking dependencies")
            self.dbo.resolve()
        except dnf.exceptions.DepsolveError as e:
            logger.error("Dependency check failed: %s", e)
            raise
        logger.info("%d packages selected", len(self.dbo.transaction))
        if len(self.dbo.transaction) == 0:
            raise Exception("No packages in transaction")

        # Write out the packages installed, including debuginfo packages
        self._write_package_log()

        pkgs_to_download = self.dbo.transaction.install_set
        logger.info("Downloading packages")
        progress = LoraxDownloadCallback()
        try:
            self.dbo.download_packages(pkgs_to_download, progress)
        except dnf.exceptions.DownloadError as e:
            logger.error("Failed to download the following packages: %s", e)
            raise

        logger.info("Preparing transaction from installation source")
        try:
            display = LoraxRpmCallback()
            self.dbo.do_transaction(display=display)
        except BaseException as e:
            logger.error("The transaction process has ended abruptly: %s", e)
            raise

        # Reset the package sack to pick up the installed packages
        self.dbo.reset(repos=False)
        self.dbo.fill_sack(load_system_repo=True, load_available_repos=False)

        # At this point dnf should know about the installed files. Double check that it really does.
        if len(self._filelist("anaconda-core")) == 0:
            raise Exception("Failed to reset dbo to installed package set")

    def removefrom(self, pkg, *globs):
        '''
        removefrom PKGGLOB [--allbut] FILEGLOB [FILEGLOB...]
          Remove all files matching the given file globs from the package
          (or packages) named.
          If '--allbut' is used, all the files from the given package(s) will
          be removed *except* the ones which match the file globs.

          Examples:
            removefrom usbutils /usr/bin/*
            removefrom xfsprogs --allbut /sbin/*
        '''
        cmd = "%s %s" % (pkg, " ".join(globs)) # save for later logging
        keepmatches = False
        if globs[0] == '--allbut':
            keepmatches = True
            globs = globs[1:]
        # get pkg filelist and find files that match the globs
        filelist = self._filelist(pkg)
        matches = set()
        for g in globs:
            globs_re = re.compile(fnmatch.translate(g))
            m = [f for f in filelist if globs_re.match(f)]
            if m:
                matches.update(m)
            else:
                logger.debug("removefrom %s %s: no files matched!", pkg, g)
        # are we removing the matches, or keeping only the matches?
        if keepmatches:
            remove_files = filelist.difference(matches)
        else:
            remove_files = matches
        # remove the files
        if remove_files:
            logger.debug("removefrom %s: removed %i/%i files, %ikb/%ikb", cmd,
                             len(remove_files), len(filelist),
                             self._getsize(*remove_files)/1024, self._getsize(*filelist)/1024)
            self.remove(*remove_files)
        else:
            logger.debug("removefrom %s: no files to remove!", cmd)

    # pylint: disable=anomalous-backslash-in-string
    def removekmod(self, *globs):
        '''
        removekmod GLOB [GLOB...] [--allbut] KEEPGLOB [KEEPGLOB...]
          Remove all files and directories matching the given file globs from the kernel
          modules directory.

          If '--allbut' is used, all the files from the modules will be removed *except*
          the ones which match the file globs. There must be at least one initial GLOB
          to search and one KEEPGLOB to keep. The KEEPGLOB is expanded to be *KEEPGLOB*
          so that it will match anywhere in the path.

          This only removes files from under /lib/modules/\\*/kernel/

          Examples:
            removekmod sound drivers/media drivers/hwmon drivers/video
            removekmod drivers/char --allbut virtio_console hw_random
        '''
        cmd = " ".join(globs)
        if "--allbut" in globs:
            idx = globs.index("--allbut")
            if idx == 0:
                raise ValueError("removekmod needs at least one GLOB before --allbut")

            # Apply keepglobs anywhere they appear in the path
            keepglobs = globs[idx+1:]
            if len(keepglobs) == 0:
                raise ValueError("removekmod needs at least one GLOB after --allbut")

            globs = globs[:idx]
        else:
            # Nothing to keep
            keepglobs = []

        filelist = set()
            for top_dir in rglob(self._out("/lib/modules/*/kernel/"+g)):
                for root, _dirs, files in os.walk(top_dir):
                    filelist.update(root+"/"+f for f in files)

        # Remove anything matching keepglobs from the list
        for g in keepglobs:
            globs_re = re.compile(fnmatch.translate("*"+g+"*"))
            if m:
            else:
                logger.debug("removekmod %s: no files matched!", g)
        remove_files = filelist.difference(matches)

            logger.debug("removekmod: removing %d files", len(remove_files))
            list(remove(f) for f in remove_files)
        else:
            logger.debug("removekmod %s: no files to remove!", cmd)

    def createaddrsize(self, addr, src, dest):
        '''
        createaddrsize INITRD_ADDRESS INITRD ADDRSIZE
          Create the initrd.addrsize file required in LPAR boot process.

          Examples:
            createaddrsize ${INITRD_ADDRESS} ${outroot}/${BOOTDIR}/initrd.img ${outroot}/${BOOTDIR}/initrd.addrsize
        '''
        addrsize = open(dest, "wb")
        addrsize_data = struct.pack(">iiii", 0, int(addr, 16), 0, os.stat(src).st_size)
        addrsize.write(addrsize_data)
        addrsize.close()

    def systemctl(self, cmd, *units):
        '''
        systemctl [enable|disable|mask] UNIT [UNIT...]
          Enable, disable, or mask the given systemd units.

          Examples:
            systemctl disable lvm2-monitor.service
            systemctl mask fedora-storage-init.service fedora-configure.service
        '''
        if cmd not in ('enable', 'disable', 'mask'):
            raise ValueError('unsupported systemctl cmd: %s' % cmd)
        if not units:
            logger.debug("systemctl: no units given for %s, ignoring", cmd)
            return
        self.mkdir("/run/systemd/system") # XXX workaround for systemctl bug
        systemctl = ['systemctl', '--root', self.outroot, '--no-reload', cmd]
        # When a unit doesn't exist systemd aborts the command. Run them one at a time.
        # XXX for some reason 'systemctl enable/disable' always returns 1
        for unit in units:
            try:
                cmd = systemctl + [unit]
                runcmd(cmd)
                pass

class LiveTemplateRunner(TemplateRunner):
    """
    This class parses and executes a limited Lorax template. Sample usage:

      runner = LiveTemplateRunner(dbo, templatedir, defaults)
      runner.run("live-install.tmpl")

      It is meant to be used with the live-install.tmpl which lists the per-arch
      pacages needed to build the live-iso output.
    """
    def __init__(self, dbo, fatalerrors=True, templatedir=None, defaults=None):
        self.pkgs = []
        self.pkgnames = []

        super(LiveTemplateRunner, self).__init__(fatalerrors, templatedir, defaults)

        '''

        '''
        else:

            try:

                self.pkgs.extend(pkgnvrs)
                self.pkgnames.extend([pkg.name for pkg in pkgnames])

# monitor.py
#
# Copyright (C) 2011-2015  Red Hat, Inc.
#
#
#
#
# Author(s): Brian C. Lane <bcl@redhat.com>
#
log = logging.getLogger("livemedia-creator")

import re
import socketserver
import threading

class LogRequestHandler(socketserver.BaseRequestHandler):
    """
    Handle monitoring and saving the logfiles from the virtual install

    Incoming data is written to self.server.log_path and each line is checked
    for patterns that would indicate that the installation failed.
    self.server.log_error is set True when this happens.
    """

    simple_tests = [
        "Traceback (",
        "traceback script(s) have been run",
        "Out of memory:",
        "Call Trace:",
        "insufficient disk space:",
        "Not enough disk space to download the packages",
        "error populating transaction after",
        "crashed on signal",
        "packaging: Missed: NoSuchPackage",
        "packaging: Installation failed",
        "The following error occurred while installing.  This is a fatal error",
        "Error in POSTIN scriptlet in rpm package"
    ]

    re_tests = [
        r"packaging: base repo .* not valid",
        r"packaging: .* requires .*"
    ]

    def setup(self):
        """Start writing to self.server.log_path"""

        if self.server.log_path:
            self.fp = open(self.server.log_path, "w") # pylint: disable=attribute-defined-outside-init
        else:
            self.fp = None
        self.request.settimeout(10)

    def handle(self):
        """
        Write incoming data to a logfile and check for errors

        Split incoming data into lines and check for any Tracebacks or other
        errors that indicate that the install failed.

        Loops until self.server.kill is True
        """
        log.info("Processing logs from %s", self.client_address)
        line = ""
            if self.server.kill:
                break

            try:
                data = str(self.request.recv(4096), "utf8")
                if self.fp:
                    self.fp.write(data)
                    self.fp.flush()

                # check the data for errors and set error flag
                # need to assemble it into lines so we can test for the error
                # string.
                while data:
                    more = data.split("\n", 1)
                    line += more[0]
                    if len(more) > 1:
                        self.iserror(line)
                        line = ""
                        data = more[1]
                    else:
                        data = None

            except socket.timeout:
                pass
            except Exception as e:       # pylint: disable=broad-except
                log.info("log processing killed by exception: %s", e)
                break

    def finish(self):
        log.info("Shutting down log processing")
        self.request.close()
        if self.fp:
            self.fp.close()

    def iserror(self, line):
        """
        Check a line to see if it contains an error indicating installation failure

        :param str line: log line to check for failure

        If the line contains IGNORED it will be skipped.
        """
        if "IGNORED" in line:
            return

        for t in self.simple_tests:
            if t in line:
                self.server.log_error = True
                self.server.error_line = line
                return
        for t in self.re_tests:
            if re.search(t, line):
                return

class LogServer(socketserver.TCPServer):
    """A TCP Server that listens for log data"""

    # Number of seconds to wait for a connection after startup
    timeout = 60

    def __init__(self, log_path, *args, **kwargs):
        """
        Setup the log server

        :param str log_path: Path to the log file to write
        """
        self.kill = False
        self.log_error = False
        self.error_line = ""
        self.log_path = log_path
        self._timeout = kwargs.pop("timeout", None)
        if self._timeout:
            self._start_time = time.time()
        socketserver.TCPServer.__init__(self, *args, **kwargs)

    def log_check(self):
        """
        Check to see if an error has been found in the log

        :returns: True if there has been an error
        :rtype: bool
        """
            taking_too_long = time.time() > self._start_time + (self._timeout * 60)
            if taking_too_long:
                log.error("Canceling installation due to timeout")
        else:
            taking_too_long = False
        return self.log_error or taking_too_long

class LogMonitor(object):
    """
    Setup a server to monitor the logs output by the installation

    This needs to be running before the virt-install runs, it expects
    there to be a listener on the port used for the virtio log port.
    """
    def __init__(self, log_path=None, host="localhost", port=0, timeout=None, log_request_handler_class=LogRequestHandler):
        """
        Start a thread to monitor the logs.

        :param str log_path: Path to the logfile to write
        :param str host: Host to bind to. Default is localhost.
        :param int port: Port to listen to or 0 to pick a port

        If 0 is passed for the port the dynamically assigned port will be
        available as self.port

        If log_path isn't set then it only monitors the logs, instead of
        also writing them to disk.
        """
        self.server = LogServer(log_path, (host, port), log_request_handler_class, timeout=timeout)
        self.host, self.port = self.server.server_address
        self.server_thread = threading.Thread(target=self.server.handle_request)
        self.server_thread.daemon = True
        self.server_thread.start()

    def shutdown(self):
        """Force shutdown of the monitoring thread"""
        self.server.kill = True
        self.server_thread.join()

# mount.py
#
#
#
#
#
#

import os
import pycdlib
from pycdlib.pycdlibexception import PyCdlibException

from pylorax.imgutils import mount, umount

class IsoMountpoint(object):
    """
    Mount the iso and check to make sure the vmlinuz and initrd.img files exist

    Also check the iso for a a stage2 image and set a flag and extract the
    iso's label.

    stage2 can be either LiveOS/squashfs.img or images/install.img
    """
    def __init__(self, iso_path, initrd_path=None):
        """
        Mount the iso

        :param str iso_path: Path to the iso to mount
        :param str initrd_path: Optional path to initrd

        initrd_path can be used to point to a tree with a newer
        initrd.img than the iso has. The iso is still used for stage2.

        self.kernel and self.initrd point to the kernel and initrd.
        self.stage2 is set to True if there is a stage2 image.
        self.repo is the path to the mounted iso if there is a /repodata dir.
        """
        self.label = None
        self.iso_path = iso_path
        self.initrd_path = initrd_path

        if not self.initrd_path:
            self.mount_dir = mount(self.iso_path, opts="loop")
        else:
            self.mount_dir = self.initrd_path

        kernel_list = [("/isolinux/vmlinuz", "/isolinux/initrd.img"),
                       ("/ppc/ppc64/vmlinuz", "/ppc/ppc64/initrd.img"),
                       ("/images/pxeboot/vmlinuz", "/images/pxeboot/initrd.img")]

        if os.path.isdir(self.mount_dir+"/repodata"):
            self.repo = self.mount_dir
        else:
            self.repo = None
        self.stage2 = os.path.exists(self.mount_dir+"/LiveOS/squashfs.img") or \
                      os.path.exists(self.mount_dir+"/images/install.img")

        try:
            for kernel, initrd in kernel_list:
                if (os.path.isfile(self.mount_dir+kernel) and
                    os.path.isfile(self.mount_dir+initrd)):
                    self.kernel = self.mount_dir+kernel
                    self.initrd = self.mount_dir+initrd
                    break
            else:
                raise Exception("Missing kernel and initrd file in iso, failed"
                                " to search under: {0}".format(kernel_list))
        except:
            self.umount()
            raise

        self.get_iso_label()

    def umount( self ):
        """Unmount the iso"""

    def get_iso_label(self):
        """
        Get the iso's label using isoinfo

        Sets self.label if one is found
        """
        try:
            iso = pycdlib.PyCdlib()
            iso.open(self.iso_path)
            self.label = iso.pvd.volume_identifier.decode("UTF-8").strip()
        except PyCdlibException as e:
            log.error("Problem reading label from %s: %s", self.iso_path, e)

#
# sysutils.py
#
# Copyright (C) 2009-2019 Red Hat, Inc.
#
#
#
#
#

__all__ = ["joinpaths", "touch", "replace", "chown_", "chmod_", "remove",
           "linktree"]

import sys
import os
import re
import fileinput
import pwd
import grp
import shlex
from configparser import ConfigParser

from pylorax.executils import runcmd

def joinpaths(*args, **kwargs):
    path = os.path.sep.join(args)

    if kwargs.get("follow_symlinks"):
        return os.path.realpath(path)
    else:
        return path

def touch(fname):
    with open(fname, "w") as f:
        f.write("")

def replace(fname, find, sub):
    fin = fileinput.input(fname, inplace=1)
    pattern = re.compile(find)

    for line in fin:
        line = pattern.sub(sub, line)
        sys.stdout.write(line)

    fin.close()

def chown_(path, user=None, group=None, recursive=False):
    uid = gid = -1

    if user is not None:
        uid = pwd.getpwnam(user)[2]
    if group is not None:
        gid = grp.getgrnam(group)[2]

    for fname in glob.iglob(path):
        os.chown(fname, uid, gid)

        if recursive and os.path.isdir(fname):
            for nested in os.listdir(fname):
                nested = joinpaths(fname, nested)
                chown_(nested, user, group, recursive)

def chmod_(path, mode, recursive=False):
        os.chmod(fname, mode)

                chmod_(nested, mode, recursive)

def cpfile(src, dst):
    shutil.copy2(src, dst)
    if os.path.isdir(dst):
        dst = joinpaths(dst, os.path.basename(src))

    return dst

def mvfile(src, dst):
    os.rename(src, dst)
    return dst

def remove(target):
    if os.path.isdir(target) and not os.path.islink(target):
        shutil.rmtree(target)
    else:
        os.unlink(target)

def linktree(src, dst):
    runcmd(["/bin/cp", "-alx", src, dst])

def unquote(s):
    return ' '.join(shlex.split(s))

class UnquotingConfigParser(ConfigParser):
    """A ConfigParser, only with unquoting of the values."""
    def get(self, *args, **kwargs):
        ret = super().get(*args, **kwargs)
        if ret:
            ret = unquote(ret)
        return ret

def flatconfig(filename):
    """Use UnquotingConfigParser to read a flat config file (without
    section headers) by adding a section header.
    """
    with open (filename, 'r') as conffh:
        conftext = "[main]\n" + conffh.read()
    config = UnquotingConfigParser()
    config.read_string(conftext)
    return config['main']

def read_tail(path, size):
    """Read up to `size` kibibytes from the end of a file"""

    # NOTE: In py3 text files are unicode, not bytes so we have to open it as bytes
    with open(path, "rb") as f:
        return _read_file_end(f, size)

def _read_file_end(f, size):
    """Read the end of a file

    This skips to the next line to avoid starting in the middle of a unicode character.
    And returns "" in the case of a UnicodeDecodeError
    """
    f.seek(0, 2)
    end = f.tell()
    if end < 1024 * size:
        f.seek(0, 0)
    else:
        f.seek(end - (1024 * size))
    data = f.read()
    try:
        # Find the first newline in the block
        newline = min(1+data.find(b'\n'), len(data))
        text = data[newline:].decode("UTF-8")
    except UnicodeDecodeError:
        return ""
    return text

# treebuilder.py - handle arch-specific tree building stuff using templates
#
# Copyright (C) 2011-2015 Red Hat, Inc.
#
#
#
#

logger = logging.getLogger("pylorax.treebuilder")

import os, re
from os.path import basename
from shutil import copytree, copy2
from pathlib import Path
import itertools

from pylorax.ltmpl import LoraxTemplateRunner
import pylorax.imgutils as imgutils
from pylorax.executils import runcmd, runcmd_output, execWithCapture

templatemap = {
    'i386':    'x86.tmpl',
    'x86_64':  'x86.tmpl',
    'ppc64le': 'ppc64le.tmpl',
    's390':    's390.tmpl',
    's390x':   's390.tmpl',
    'aarch64': 'aarch64.tmpl',
    'arm':     'arm.tmpl',
    'armhfp':  'arm.tmpl',
}

def generate_module_info(moddir, outfile=None):
    def module_desc(mod):
        output = runcmd_output(["modinfo", "-F", "description", mod])
        return output.strip()
    def read_module_set(name):
        return set(l.strip() for l in open(joinpaths(moddir,name)) if ".ko" in l)
    modsets = {'scsi':read_module_set("modules.block"),
               'eth':read_module_set("modules.networking")}

    modinfo = list()
    for root, _dirs, files in os.walk(moddir):
        for modtype, modset in modsets.items():
            for mod in modset.intersection(files):  # modules in this dir
                (name, _ext) = os.path.splitext(mod) # foo.ko -> (foo, .ko)
                desc = module_desc(joinpaths(root,mod)) or "%s driver" % name
                modinfo.append(dict(name=name, type=modtype, desc=desc))

    out = open(outfile or joinpaths(moddir,"module-info"), "w")
    out.write("Version 0\n")
    for mod in sorted(modinfo, key=lambda m: m.get('name')):
        out.write('{name}\n\t{type}\n\t"{desc:.65}"\n'.format(**mod))

class RuntimeBuilder(object):
    '''Builds the anaconda runtime image.'''
    def __init__(self, product, arch, dbo, templatedir=None,
                 installpkgs=None, excludepkgs=None,
                 add_templates=None,
                 add_template_vars=None,
                 skip_branding=False):
        root = dbo.conf.installroot
        # use a copy of product so we can modify it locally
        product = product.copy()
        product.name = product.name.lower()
        self.vars = DataHolder(arch=arch, product=product, dbo=dbo, root=root,
                               basearch=arch.basearch, libdir=arch.libdir)
        self._runner = LoraxTemplateRunner(inroot=root, outroot=root,
                                           dbo=dbo, templatedir=templatedir)
        self.add_templates = add_templates or []
        self.add_template_vars = add_template_vars or {}
        self._installpkgs = installpkgs or []
        self._excludepkgs = excludepkgs or []
        self._runner.defaults = self.vars
        self.dbo.reset()
        self._skip_branding = skip_branding

    def _install_branding(self):
        """Select the branding from the available 'system-release' packages
        The *best* way to control this is to have a single package in the repo provide 'system-release'
        When there are more than 1 package it will:
        - Make a list of the available packages
        - If variant is set look for a package ending with lower(variant) and use that
        - If there are one or more non-generic packages, use the first one after sorting
        """
        if self._skip_branding:
            return

        release = None
        q = self.dbo.sack.query()
        a = q.available()
        pkgs = sorted([p.name for p in a.filter(provides='system-release')
                                    if not p.name.startswith("generic")])
        if not pkgs:
            logger.error("No system-release packages found, could not get the release")
            return

        logger.debug("system-release packages: %s", pkgs)
        if self.vars.product.variant:
            variant = [p for p in pkgs if p.endswith("-"+self.vars.product.variant.lower())]
            if variant:
                release = variant[0]
        if not release:
            release = pkgs[0]

        # release
        logger.info('got release: %s', release)
        self._runner.installpkg(release)

        # logos
        release, _suffix = release.split('-', 1)
        self._runner.installpkg('%s-logos' % release)

    def install(self):
        '''Install packages and do initial setup with runtime-install.tmpl'''
        self._install_branding()
        if len(self._installpkgs) > 0:
            self._runner.installpkg(*self._installpkgs)
        if len(self._excludepkgs) > 0:
            self._runner.removepkg(*self._excludepkgs)
        self._runner.run("runtime-install.tmpl")
        for tmpl in self.add_templates:
            self._runner.run(tmpl, **self.add_template_vars)

    def writepkglists(self, pkglistdir):
        '''debugging data: write out lists of package contents'''
        if not os.path.isdir(pkglistdir):
            os.makedirs(pkglistdir)
        for pkgobj in q.installed():
            with open(joinpaths(pkglistdir, pkgobj.name), "w") as fobj:
                for fname in pkgobj.files:
                    fobj.write("{0}\n".format(fname))

    def postinstall(self):
        '''Do some post-install setup work with runtime-postinstall.tmpl'''
        # copy configdir into runtime root beforehand
        configdir = joinpaths(self._runner.templatedir,"config_files")
        configdir_path = "tmp/config_files"
        fullpath = joinpaths(self.vars.root, configdir_path)
        if os.path.exists(fullpath):
            remove(fullpath)
        copytree(configdir, fullpath)
        self._runner.run("runtime-postinstall.tmpl", configdir=configdir_path)

    def cleanup(self):
        '''Remove unneeded packages and files with runtime-cleanup.tmpl'''
        self._runner.run("runtime-cleanup.tmpl")

    def verify(self):
        '''Ensure that contents of the installroot can run'''
        status = True

        ELF_MAGIC = b'\x7fELF'

        # Iterate over all files in /usr/bin and /usr/sbin
        # For ELF files, gather them into a list and we'll check them all at
        # the end. For files with a #!, check them as we go
        elf_files = []
        usr_bin = Path(self.vars.root + '/usr/bin')
        usr_sbin = Path(self.vars.root + '/usr/sbin')
        for path in (str(x) for x in itertools.chain(usr_bin.iterdir(), usr_sbin.iterdir()) \
                     if x.is_file()):
            with open(path, "rb") as f:
                magic = f.read(4)
                if magic == ELF_MAGIC:
                    # Save the path, minus the chroot prefix
                    elf_files.append(path[len(self.vars.root):])
                elif magic[:2] == b'#!':
                    # Reopen the file as text and read the first line.
                    # Open as latin-1 so that stray 8-bit characters don't make
                    # things blow up. We only really care about ASCII parts.
                    with open(path, "rt", encoding="latin-1") as f_text:
                        # Remove the #!, split on space, and take the first part
                        shabang = f_text.readline()[2:].split()[0]

                    # Does the path exist?
                    if not os.path.exists(self.vars.root + shabang):
                        logger.error('%s, needed by %s, does not exist', shabang, path)
                        status = False

        # Now, run ldd on all the ELF files
        # Just run ldd once on everything so it isn't logged a million times.
        # At least one thing in the list isn't going to be a dynamic executable,
        # so use execWithCapture to ignore the exit code.
        filename = ''
        for line in execWithCapture('ldd', elf_files, root=self.vars.root,
                log_output=False, filter_stderr=True).split('\n'):
            if line and not line[0].isspace():
                # New filename header, strip the : at the end and save
                filename = line[:-1]
            elif 'not found' in line:
                logger.error('%s, needed by %s, not found', line.split()[0], filename)
                status = False

        return status

    def writepkgsizes(self, pkgsizefile):
        '''debugging data: write a big list of pkg sizes'''
        fobj = open(pkgsizefile, "w")
        getsize = lambda f: os.lstat(f).st_size if os.path.exists(f) else 0
        for p in sorted(q.installed()):
            pkgsize = sum(getsize(joinpaths(self.vars.root,f)) for f in p.files)
            fobj.write("{0.name}.{0.arch}: {1}\n".format(p, pkgsize))

    def generate_module_data(self):
        root = self.vars.root
        moddir = joinpaths(root, "lib/modules/")
        for kernel in findkernels(root=root):
            ksyms = joinpaths(root, "boot/System.map-%s" % kernel.version)
            logger.info("doing depmod and module-info for %s", kernel.version)
            runcmd(["depmod", "-a", "-F", ksyms, "-b", root, kernel.version])
            generate_module_info(moddir+kernel.version, outfile=moddir+"module-info")

    def create_squashfs_runtime(self, outfile="/var/tmp/squashfs.img", compression="xz", compressargs=None, size=2):
        """Create a plain squashfs runtime"""
        compressargs = compressargs or []
        os.makedirs(os.path.dirname(outfile))

        # squash the rootfs
        return imgutils.mksquashfs(self.vars.root, outfile, compression, compressargs)

    def create_ext4_runtime(self, outfile="/var/tmp/squashfs.img", compression="xz", compressargs=None, size=2):
        """Create a squashfs compressed ext4 runtime"""
        # make live rootfs image - must be named "LiveOS/rootfs.img" for dracut
        workdir = joinpaths(os.path.dirname(outfile), "runtime-workdir")
        os.makedirs(joinpaths(workdir, "LiveOS"))

        # Catch problems with the rootfs being too small and clearly log them
        try:
            imgutils.mkrootfsimg(self.vars.root, joinpaths(workdir, "LiveOS/rootfs.img"),
                                 "Anaconda", size=size)
            if e.stdout and "No space left on device" in e.stdout:
                logger.error("The rootfs ran out of space with size=%d", size)
            raise

        # squash the live rootfs and clean up workdir
        rc = imgutils.mksquashfs(workdir, outfile, compression, compressargs)
        remove(workdir)
        return rc

    def finished(self):
        """ Done using RuntimeBuilder

        Close the dnf base object
        """
        self.dbo.close()

class TreeBuilder(object):
    '''Builds the arch-specific boot images.
    inroot should be the installtree root (the newly-built runtime dir)'''
    def __init__(self, product, arch, inroot, outroot, runtime, isolabel, domacboot=True, doupgrade=True,
                 templatedir=None, add_templates=None, add_template_vars=None, workdir=None, extra_boot_args=""):

        # NOTE: if you pass an arg named "runtime" to a mako template it'll
        # clobber some mako internal variables - hence "runtime_img".
        self.vars = DataHolder(arch=arch, product=product, runtime_img=runtime,
                               runtime_base=basename(runtime),
                               inroot=inroot, outroot=outroot,
                               basearch=arch.basearch, libdir=arch.libdir,
                               isolabel=isolabel, udev=udev_escape, domacboot=domacboot, doupgrade=doupgrade,
                               workdir=workdir, lower=string_lower,
                               extra_boot_args=extra_boot_args)
        self._runner = LoraxTemplateRunner(inroot, outroot, templatedir=templatedir)
        self.templatedir = templatedir
        self.treeinfo_data = None

    @property
    def kernels(self):
        return findkernels(root=self.vars.inroot)

    def rebuild_initrds(self, add_args=None, backup="", prefix=""):
        '''Rebuild all the initrds in the tree. If backup is specified, each
        initrd will be renamed with backup as a suffix before rebuilding.
        If backup is empty, the existing initrd files will be overwritten.
        If suffix is specified, the existing initrd is untouched and a new
        image is built with the filename "${prefix}-${kernel.version}.img"

        If the initrd doesn't exist its name will be created based on the
        name of the kernel.
        '''
        add_args = add_args or []
        dracut = ["dracut", "--nomdadmconf", "--nolvmconf"] + add_args
        if not backup:
            dracut.append("--force")

        if not self.kernels:
            raise Exception("No kernels found, cannot rebuild_initrds")

        for kernel in self.kernels:
            if prefix:
                idir = os.path.dirname(kernel.path)
                outfile = joinpaths(idir, prefix+'-'+kernel.version+'.img')
            elif hasattr(kernel, "initrd"):
                # If there is an existing initrd, use that
                outfile = kernel.initrd.path
            else:
                # Construct an initrd from the kernel name
                outfile = kernel.path.replace("vmlinuz-", "initrd-") + ".img"
            logger.info("rebuilding %s", outfile)
            logger.info("dracut warnings about /proc are safe to ignore")

            if backup:
                initrd = joinpaths(self.vars.inroot, outfile)
                if os.path.exists(initrd):
                    os.rename(initrd, initrd + backup)
            cmd = dracut + [outfile, kernel.version]
            runcmd(cmd, root=self.vars.inroot)

    def build(self):
        templatefile = templatemap[self.vars.arch.basearch]
        self._runner.run(templatefile, kernels=self.kernels)
        self.treeinfo_data = self._runner.results.treeinfo
        self.implantisomd5()

    def implantisomd5(self):
        for _section, data in self.treeinfo_data.items():
            if 'boot.iso' in data:
                iso = joinpaths(self.vars.outroot, data['boot.iso'])
                runcmd(["implantisomd5", iso])

    @property
    def dracut_hooks_path(self):
        """ Return the path to the lorax dracut hooks scripts

            Use the configured share dir if it is setup,
            otherwise default to /usr/share/lorax/dracut_hooks
        """
        if self.templatedir:
            return joinpaths(self.templatedir, "dracut_hooks")
        else:
            return "/usr/share/lorax/dracut_hooks"

    def copy_dracut_hooks(self, hooks):
        """ Copy the hook scripts in hooks into the installroot's /tmp/
        and return a list of commands to pass to dracut when creating the
        initramfs

        hooks is a list of tuples with the name of the hook script and the
        target dracut hook directory
        (eg. [("99anaconda-copy-ks.sh", "/lib/dracut/hooks/pre-pivot")])
        """
        dracut_commands = []
        for hook_script, dracut_path in hooks:
            src = joinpaths(self.dracut_hooks_path, hook_script)
            if not os.path.exists(src):
                logger.error("Missing lorax dracut hook script %s", (src))
                continue
            dst = joinpaths(self.vars.inroot, "/tmp/", hook_script)
            copy2(src, dst)
            dracut_commands += ["--include", joinpaths("/tmp/", hook_script),
                                dracut_path]
        return dracut_commands

#### TreeBuilder helper functions

def findkernels(root="/", kdir="boot"):
    # To find possible flavors, awk '/BuildKernel/ { print $4 }' kernel.spec
    flavors = ('debug', 'PAE', 'PAEdebug', 'smp', 'xen', 'lpae')
    kre = re.compile(r"vmlinuz-(?P<version>.+?\.(?P<arch>[a-z0-9_]+)"
                     r"(.(?P<flavor>{0}))?)$".format("|".join(flavors)))
    kernels = []
    bootfiles = os.listdir(joinpaths(root, kdir))
    for f in bootfiles:
        match = kre.match(f)
        if match:
            kernel = DataHolder(path=joinpaths(kdir, f))
            kernel.update(match.groupdict()) # sets version, arch, flavor
            kernels.append(kernel)

    # look for associated initrd/initramfs/etc.
        for f in bootfiles:
            if f.endswith('-'+kernel.version+'.img'):
                imgtype, _rest = f.split('-',1)
                # special backwards-compat case
                if imgtype == 'initramfs':
                    imgtype = 'initrd'
                kernel[imgtype] = DataHolder(path=joinpaths(kdir, f))

    logger.debug("kernels=%s", kernels)
    return kernels

# udev whitelist: 'a-zA-Z0-9#+.:=@_-' (see is_whitelisted in libudev-util.c)
udev_blacklist=' !"$%&\'()*,/;<>?[\\]^`{|}~' # ASCII printable, minus whitelist
udev_blacklist += ''.join(chr(i) for i in range(32)) # ASCII non-printable
def udev_escape(label):
    out = ''
    for ch in label:
        out += ch if ch not in udev_blacklist else '\\x%02x' % ord(ch)
    return out

def string_lower(string):
    """ Return a lowercase string.

    :param string: String to lowercase

    This is used as a filter in the templates.
    """
    return string.lower()

#
# treeinfo.py
#
#
#
#
#
#

logger = logging.getLogger("pylorax.treeinfo")

import configparser
import os

class TreeInfo(object):

    def __init__(self, product, version, variant, basearch,
                 packagedir=""):

        self.c = configparser.ConfigParser()

            timestamp = os.environ['SOURCE_DATE_EPOCH']
        else:
            timestamp = str(time.time())

        section = "general"
        data = {"timestamp": timestamp,
                "family": product,
                "version": version,
                "name": "%s-%s" % (product, version),
                "variant": variant or "",
                "arch": basearch,
                "packagedir": packagedir}

        self.c.add_section(section)
        list(self.c.set(section, key, value) for key, value in data.items())

    def add_section(self, section, data):
        if not self.c.has_section(section):
            self.c.add_section(section)

        logger.info("writing .treeinfo file")
            self.c.write(fobj)

#
# __init__.py
#
#
#
#
#
#                     David Cantrell <dcantrell@redhat.com>

# set up logging
logger = logging.getLogger("pylorax")
logger.addHandler(logging.NullHandler())

import sys
import os
import locale
import selinux
from glob import glob

from pylorax.base import BaseLoraxClass, DataHolder

import dnf

from pylorax.sysutils import joinpaths, remove, linktree

from pylorax.treebuilder import RuntimeBuilder, TreeBuilder
from pylorax.buildstamp import BuildStamp
from pylorax.treeinfo import TreeInfo
from pylorax.discinfo import DiscInfo

# get lorax version
try:
    import pylorax.version
except ImportError:
    vernum = "devel"
else:
    vernum = pylorax.version.num

DRACUT_DEFAULT = ["--xz", "--install", "/.buildstamp", "--no-early-microcode", "--add", "fips"]

# Used for DNF conf.module_platform_id
DEFAULT_PLATFORM_ID = "platform:f34"

class ArchData(DataHolder):
    lib64_arches = ("x86_64", "ppc64le", "s390x", "ia64", "aarch64")
    bcj_arch = dict(i386="x86", x86_64="x86",
                    ppc64le="powerpc",
                    arm="arm", armhfp="arm")

    def __init__(self, buildarch):
        super(ArchData, self).__init__()
        self.buildarch = buildarch
        self.basearch = dnf.rpm.basearch(buildarch)
        self.libdir = "lib64" if self.basearch in self.lib64_arches else "lib"
        self.bcj = self.bcj_arch.get(self.basearch)

class Lorax(BaseLoraxClass):

        BaseLoraxClass.__init__(self)
        self._configured = False
        self.product = None
        self.workdir = None
        self.arch = None
        self.conf = None
        self.inroot = None
        self.debug = False
        self.outputdir = None
        self._templatedir = None

        # set locale to C
        locale.setlocale(locale.LC_ALL, 'C')

    def configure(self, conf_file="/etc/lorax/lorax.conf"):
        self.conf = configparser.SafeConfigParser()

        # set defaults
        self.conf.add_section("lorax")
        self.conf.set("lorax", "debug", "1")
        self.conf.set("lorax", "sharedir", "/usr/share/lorax")
        self.conf.set("lorax", "logdir", "/var/log/lorax")

        self.conf.add_section("output")
        self.conf.set("output", "colors", "1")
        self.conf.set("output", "encoding", "utf-8")
        self.conf.set("output", "ignorelist", "/usr/share/lorax/ignorelist")

        self.conf.add_section("templates")
        self.conf.set("templates", "ramdisk", "ramdisk.ltmpl")

        self.conf.add_section("compression")
        self.conf.set("compression", "type", "xz")
        self.conf.set("compression", "args", "")
        self.conf.set("compression", "bcj", "on")

        # read the config file
        if os.path.isfile(conf_file):
            self.conf.read(conf_file)

        # set up the output
        self.debug = self.conf.getboolean("lorax", "debug")
        output_level = output.DEBUG if self.debug else output.INFO

        if sys.stdout.isatty():
            colors = self.conf.getboolean("output", "colors")
        else:
            colors = False
        encoding = self.conf.get("output", "encoding")

        self.output.basic_config(output_level=output_level,
                                 colors=colors, encoding=encoding)

        ignorelist = self.conf.get("output", "ignorelist")
        if os.path.isfile(ignorelist):
            with open(ignorelist, "r") as fobj:
                for line in fobj:
                    line = line.strip()
                    if line and not line.startswith("#"):
                        self.output.ignore(line)

        # cron does not have sbin in PATH,
        # so we have to add it ourselves
        os.environ["PATH"] = "{0}:/sbin:/usr/sbin".format(os.environ["PATH"])

        # remove some environmental variables that can cause problems with package scripts
        env_remove = ('DISPLAY', 'DBUS_SESSION_BUS_ADDRESS')
        list(os.environ.pop(k) for k in env_remove if k in os.environ)

        self._configured = True

    @property
    def templatedir(self):
        """Find the template directory.

        Pick the first directory under sharedir/templates.d/ if it exists.
        Otherwise use the sharedir
        """
        if not self._templatedir:
            self._templatedir = find_templates(self.conf.get("lorax", "sharedir"))
            logger.info("Using templatedir %s", self._templatedir)
        return self._templatedir

    def init_stream_logging(self):
        sh = logging.StreamHandler()
        sh.setLevel(logging.INFO)
        logger.addHandler(sh)

    def init_file_logging(self, logdir, logname="pylorax.log"):
        fh = logging.FileHandler(filename=joinpaths(logdir, logname), mode="w")
        fh.setLevel(logging.DEBUG)
        logger.addHandler(fh)

    def run(self, dbo, product, version, release, variant="", bugurl="",
            isfinal=False, workdir=None, outputdir=None, buildarch=None, volid=None,
            domacboot=True, doupgrade=True, remove_temp=False,
            installpkgs=None, excludepkgs=None,
            size=2,
            add_templates=None,
            add_template_vars=None,
            add_arch_templates=None,
            add_arch_template_vars=None,
            verify=True,
            user_dracut_args=None,
            squashfs_only=False,
            skip_branding=False):

        assert self._configured

        installpkgs = installpkgs or []
        excludepkgs = excludepkgs or []

        if domacboot:
            try:
                runcmd(["rpm", "-q", "hfsplus-tools"])
                logger.critical("you need to install hfsplus-tools to create mac images")
                sys.exit(1)

        # set up work directory
        self.workdir = workdir or tempfile.mkdtemp(prefix="pylorax.work.")
        if not os.path.isdir(self.workdir):
            os.makedirs(self.workdir)

        # set up log directory
        logdir = self.conf.get("lorax", "logdir")
            os.makedirs(logdir)

        self.init_stream_logging()
        self.init_file_logging(logdir)

        logger.debug("version is %s", vernum)
        log_selinux_state()

        logger.debug("using work directory %s", self.workdir)
        logger.debug("using log directory %s", logdir)

        # set up output directory
        self.outputdir = outputdir or tempfile.mkdtemp(prefix="pylorax.out.")
        if not os.path.isdir(self.outputdir):
            os.makedirs(self.outputdir)
        logger.debug("using output directory %s", self.outputdir)

        # do we have root privileges?
        logger.info("checking for root privileges")
        if not os.geteuid() == 0:
            logger.critical("no root privileges")
            sys.exit(1)

        # do we have a proper dnf base object?
        logger.info("checking dnf base object")
        if not isinstance(dbo, dnf.Base):
            logger.critical("no dnf base object")
        self.inroot = dbo.conf.installroot
        logger.debug("using install root: %s", self.inroot)

        if not buildarch:
            buildarch = get_buildarch(dbo)

        logger.info("setting up build architecture")
        self.arch = ArchData(buildarch)
        for attr in ('buildarch', 'basearch', 'libdir'):
            logger.debug("self.arch.%s = %s", attr, getattr(self.arch,attr))

        logger.info("setting up build parameters")
        self.product = DataHolder(name=product, version=version, release=release,
                                 variant=variant, bugurl=bugurl, isfinal=isfinal)
        logger.debug("product data: %s", self.product)

        # NOTE: if you change isolabel, you need to change pungi to match, or
        # the pungi images won't boot.
        isolabel = volid or "%s-%s-%s" % (self.product.name, self.product.version, self.arch.basearch)

        if len(isolabel) > 32:
            logger.fatal("the volume id cannot be longer than 32 characters")

        # NOTE: rb.root = dbo.conf.installroot (== self.inroot)
        rb = RuntimeBuilder(product=self.product, arch=self.arch,
                            dbo=dbo, templatedir=self.templatedir,
                            installpkgs=installpkgs,
                            excludepkgs=excludepkgs,
                            add_templates=add_templates,
                            add_template_vars=add_template_vars,
                            skip_branding=skip_branding)

        logger.info("installing runtime packages")
        rb.install()

        # write .buildstamp
        buildstamp = BuildStamp(self.product.name, self.product.version,
                                self.product.bugurl, self.product.isfinal,
                                self.arch.buildarch, self.product.variant)

        buildstamp.write(joinpaths(self.inroot, ".buildstamp"))

        if self.debug:
            rb.writepkglists(joinpaths(logdir, "pkglists"))
            rb.writepkgsizes(joinpaths(logdir, "original-pkgsizes.txt"))

        logger.info("doing post-install configuration")
        rb.postinstall()

        # write .discinfo
        discinfo = DiscInfo(self.product.release, self.arch.basearch)
        discinfo.write(joinpaths(self.outputdir, ".discinfo"))

        logger.info("backing up installroot")
        installroot = joinpaths(self.workdir, "installroot")
        linktree(self.inroot, installroot)

        logger.info("generating kernel module metadata")
        rb.generate_module_data()

        logger.info("cleaning unneeded files")
        rb.cleanup()

        if verify:
            logger.info("verifying the installroot")
            if not rb.verify():
        else:
            logger.info("Skipping verify")

            rb.writepkgsizes(joinpaths(logdir, "final-pkgsizes.txt"))

        logger.info("creating the runtime image")
        runtime = "images/install.img"
        compression = self.conf.get("compression", "type")
        compressargs = self.conf.get("compression", "args").split()     # pylint: disable=no-member
        if self.conf.getboolean("compression", "bcj"):
            if self.arch.bcj:
                compressargs += ["-Xbcj", self.arch.bcj]
            else:
                logger.info("no BCJ filter for arch %s", self.arch.basearch)
        if squashfs_only:
            # Create an ext4 rootfs.img and compress it with squashfs
            rc = rb.create_squashfs_runtime(joinpaths(installroot,runtime),
                    compression=compression, compressargs=compressargs,
                    size=size)
        else:
            rc = rb.create_ext4_runtime(joinpaths(installroot,runtime),
        if rc != 0:
            logger.error("rootfs.img creation failed. See program.log")

        rb.finished()

        logger.info("preparing to build output tree and boot images")
        treebuilder = TreeBuilder(product=self.product, arch=self.arch,
                                  inroot=installroot, outroot=self.outputdir,
                                  runtime=runtime, isolabel=isolabel,
                                  domacboot=domacboot, doupgrade=doupgrade,
                                  templatedir=self.templatedir,
                                  add_templates=add_arch_templates,
                                  add_template_vars=add_arch_template_vars,
                                  workdir=self.workdir)

        logger.info("rebuilding initramfs images")
        if not user_dracut_args:
            dracut_args = DRACUT_DEFAULT
        else:
            dracut_args = []
            for arg in user_dracut_args:
                dracut_args += arg.split(" ", 1)

        anaconda_args = dracut_args + ["--add", "anaconda pollcdrom qemu qemu-net"]

        logger.info("dracut args = %s", dracut_args)
        logger.info("anaconda args = %s", anaconda_args)
        treebuilder.rebuild_initrds(add_args=anaconda_args)

        logger.info("populating output tree and building boot images")
        treebuilder.build()

        # write .treeinfo file and we're done
        treeinfo = TreeInfo(self.product.name, self.product.version,
                            self.product.variant, self.arch.basearch)
        for section, data in treebuilder.treeinfo_data.items():
            treeinfo.add_section(section, data)
        treeinfo.write(joinpaths(self.outputdir, ".treeinfo"))

        # cleanup
        if remove_temp:
            remove(self.workdir)

def get_buildarch(dbo):
    # get architecture of the available anaconda package
    buildarch = None
    q = dbo.sack.query()
    a = q.available()
    for anaconda in a.filter(name="anaconda-core"):
        if anaconda.arch != "src":
            buildarch = anaconda.arch
            break
    if not buildarch:
        logger.critical("no anaconda-core package in the repository")
        sys.exit(1)

    return buildarch

def setup_logging(logfile, theLogger):
    """
    Setup the various logs

    :param logfile: filename to write the log to
    :type logfile: string
    :param theLogger: top-level logger
    :type theLogger: logging.Logger
    """
    if not os.path.isdir(os.path.abspath(os.path.dirname(logfile))):
        os.makedirs(os.path.abspath(os.path.dirname(logfile)))

    # Setup logging to console and to logfile
    logger.setLevel(logging.DEBUG)
    theLogger.setLevel(logging.DEBUG)

    sh = logging.StreamHandler()
    sh.setLevel(logging.INFO)
    fmt = logging.Formatter("%(asctime)s: %(message)s")
    sh.setFormatter(fmt)
    logger.addHandler(sh)
    theLogger.addHandler(sh)

    fh = logging.FileHandler(filename=logfile, mode="w")
    fh.setLevel(logging.DEBUG)
    fmt = logging.Formatter("%(asctime)s %(levelname)s %(name)s: %(message)s")
    fh.setFormatter(fmt)
    logger.addHandler(fh)
    theLogger.addHandler(fh)

    # External program output log
    program_log.setLevel(logging.DEBUG)
    f = os.path.abspath(os.path.dirname(logfile))+"/program.log"
    fh = logging.FileHandler(filename=f, mode="w")
    fmt = logging.Formatter("%(asctime)s %(levelname)s: %(message)s")
    program_log.addHandler(fh)

def find_templates(templatedir="/usr/share/lorax"):
    """ Find the templates to use.

    :param str templatedir: Top directory to search for templates
    :returns: Path to templates

    If there is a templates.d directory under templatedir the
    lowest numbered directory entry is returned.

    eg. /usr/share/lorax/templates.d/99-generic/
    """
    if os.path.isdir(joinpaths(templatedir, "templates.d")):
        try:
            templatedir = sorted(glob(joinpaths(templatedir, "templates.d", "*")))[0]
        except IndexError:
            pass
    return templatedir

def log_selinux_state():
    """Log the current state of selinux"""
    if selinux.is_selinux_enabled():
        if selinux.security_getenforce():
            logger.info("selinux is enabled and in Enforcing mode")
        else:
            logger.info("selinux is enabled and in Permissive mode")
    else:
        logger.info("selinux is Disabled")