PEP 476: Enabling certificate verification by default for stdlib http clients

What is the most efficient way to concatenate many strings together?

PEP 3118: New memoryview implementation and buffer protocol documentation

Why must 'self' be used explicitly in method definitions and calls?

Why does a_tuple[i] += ['item'] raise an exception when the addition works?

PEP 471 - os.scandir() function -- a better and faster directory iterator

installupgradeinitrd() (pylorax.ltmpl.LoraxTemplateRunner method)

run_pkg_transaction() (pylorax.ltmpl.LoraxTemplateRunner method)

generate_module_data() (pylorax.treebuilder.RuntimeBuilder method)

#
# discinfo.py
#
# Copyright (C) 2010-2015  Red Hat, Inc.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
# Red Hat Author(s):  Martin Gracik <mgracik@redhat.com>
#

import logging
logger = logging.getLogger("pylorax.discinfo")

import time

class DiscInfo(object):

    def __init__(self, release, basearch):
        self.release = release
        self.basearch = basearch

    def write(self, outfile):
        logger.info("writing .discinfo file")
        with open(outfile, "w") as fobj:
            fobj.write("{0:f}\n".format(time.time()))
            fobj.write("{0.release}\n".format(self))
            fobj.write("{0.basearch}\n".format(self))

# mount.py
#
# Copyright (C) 2011-2015  Red Hat, Inc.
#
#
#
#
# Author(s): Brian C. Lane <bcl@redhat.com>
#
log = logging.getLogger("livemedia-creator")

import os

from pylorax.imgutils import mount, umount
from pylorax.executils import execWithCapture

class IsoMountpoint(object):
    """
    Mount the iso and check to make sure the vmlinuz and initrd.img files exist

    Also check the iso for a a stage2 image and set a flag and extract the
    iso's label.

    stage2 can be either LiveOS/squashfs.img or images/install.img
    """
    def __init__(self, iso_path, initrd_path=None):
        """
        Mount the iso

        :param str iso_path: Path to the iso to mount
        :param str initrd_path: Optional path to initrd

        initrd_path can be used to point to a tree with a newer
        initrd.img than the iso has. The iso is still used for stage2.

        self.kernel and self.initrd point to the kernel and initrd.
        self.stage2 is set to True if there is a stage2 image.
        self.repo is the path to the mounted iso if there is a /repodata dir.
        """
        self.label = None
        self.iso_path = iso_path
        self.initrd_path = initrd_path

        if not self.initrd_path:
            self.mount_dir = mount(self.iso_path, opts="loop")
        else:
            self.mount_dir = self.initrd_path

        self.kernel = self.mount_dir+"/isolinux/vmlinuz"
        self.initrd = self.mount_dir+"/isolinux/initrd.img"

        if os.path.isdir(self.mount_dir+"/repodata"):
            self.repo = self.mount_dir
        else:
            self.repo = None
        self.stage2 = os.path.exists(self.mount_dir+"/LiveOS/squashfs.img") or \
                      os.path.exists(self.mount_dir+"/images/install.img")

        try:
            for f in [self.kernel, self.initrd]:
                if not os.path.isfile(f):
                    raise Exception("Missing file on iso: {0}".format(f))
        except:
            self.umount()
            raise

        self.get_iso_label()

    def umount( self ):
        """Unmount the iso"""
            umount(self.mount_dir)

    def get_iso_label(self):
        """
        Get the iso's label using isoinfo

        Sets self.label if one is found
        """
        isoinfo_output = execWithCapture("isoinfo", ["-d", "-i", self.iso_path])
        log.debug(isoinfo_output)
        for line in isoinfo_output.splitlines():
            if line.startswith("Volume id: "):
                self.label = line[11:]
                return

#
# base.py
#
# Copyright (C) 2009-2015 Red Hat, Inc.
#
#
#
#
#

from abc import ABCMeta, abstractmethod
import sys

import pylorax.output as output

class BaseLoraxClass(object, metaclass=ABCMeta):
    @abstractmethod
    def __init__(self):
        self.output = output.LoraxOutput()

    def pcritical(self, msg, fobj=sys.stdout):
        self.output.critical(msg, fobj)

    def perror(self, msg, fobj=sys.stdout):
        self.output.error(msg, fobj)

    def pwarning(self, msg, fobj=sys.stdout):
        self.output.warning(msg, fobj)

    def pinfo(self, msg, fobj=sys.stdout):
        self.output.info(msg, fobj)

    def pdebug(self, msg, fobj=sys.stdout):
        self.output.debug(msg, fobj)

class DataHolder(dict):

    def __init__(self, **kwargs):
        dict.__init__(self)

        for attr, value in kwargs.items():
            self[attr] = value

    def __getattr__(self, attr):
        return self[attr]

    def __setattr__(self, attr, value):
        self[attr] = value

    def copy(self):
        return DataHolder(**dict.copy(self))

#
# dnfhelper.py
#
# Copyright (C) 2010-2015 Red Hat, Inc.
#
#
#
#
#                     Brian C. Lane <bcl@redhat.com>
#

logger = logging.getLogger("pylorax.dnfhelper")
import dnf
import collections

__all__ = ['LoraxDownloadCallback', 'LoraxRpmCallback']

def _paced(fn):
    """Execute `fn` no more often then every 2 seconds."""
    def paced_fn(self, *args):
        now = time.time()
        if now - self.last_time < 2:
            return
        self.last_time = now
        return fn(self, *args)
    return paced_fn

class LoraxDownloadCallback(dnf.callback.DownloadProgress):
        self.downloads = collections.defaultdict(int)
        self.last_time = time.time()
        self.total_files = 0
        self.total_size = 0

        self.pkgno = 0
        self.total = 0

    @_paced
    def _update(self):
        msg = "Downloading %(pkgno)s / %(total_files)s RPMs, " \
              "%(downloaded)s / %(total_size)s (%(percent)d%%) done.\n"
        downloaded = sum(self.downloads.values())
        vals = {
            'downloaded'  : downloaded,
            'percent'     : int(100 * downloaded/self.total_size),
            'pkgno'       : self.pkgno,
            'total_files' : self.total_files,
            'total_size'  : self.total_size
        }
        self.output.write(msg % vals)

    def end(self, payload, status, err_msg):
        nevra = str(payload)
        if status is dnf.callback.STATUS_OK:
            self.downloads[nevra] = payload.download_size
            self.pkgno += 1
            self._update()
            return
        logger.critical("Failed to download '%s': %d - %s", nevra, status, err_msg)

    def progress(self, payload, done):
        self.downloads[nevra] = done
        self._update()

    def start(self, total_files, total_size):
        self.total_files = total_files
        self.total_size = total_size

class LoraxRpmCallback(dnf.callback.TransactionProgress):
        super(LoraxRpmCallback, self).__init__()
        self._last_ts = None

    def progress(self, package, action, ti_done, ti_total, ts_done, ts_total):
        if action == self.PKG_INSTALL:
            # do not report same package twice
            if self._last_ts == ts_done:
                return
            self._last_ts = ts_done

            msg = '(%d/%d) %s.%s' % (ts_done, ts_total, package.name, package.arch)
            logger.info(msg)
        elif action == self.TRANS_POST:
            msg = "Performing post-installation setup tasks"

    def error(self, err_msg):
        logger.warning(err_msg)

# treebuilder.py - handle arch-specific tree building stuff using templates
#
# Copyright (C) 2011-2015 Red Hat, Inc.
#
#
#
#
# Author(s):  Will Woods <wwoods@redhat.com>

logger = logging.getLogger("pylorax.treebuilder")

import os, re
from os.path import basename
from shutil import copytree, copy2
from pathlib import Path
import itertools

from pylorax.sysutils import joinpaths, remove
from pylorax.base import DataHolder
from pylorax.ltmpl import LoraxTemplateRunner
import pylorax.imgutils as imgutils
from pylorax.executils import runcmd, runcmd_output, execWithCapture

templatemap = {
    'i386':    'x86.tmpl',
    'x86_64':  'x86.tmpl',
    'ppc':     'ppc.tmpl',
    'ppc64':   'ppc.tmpl',
    'ppc64le': 'ppc64le.tmpl',
    's390':    's390.tmpl',
    's390x':   's390.tmpl',
    'aarch64': 'aarch64.tmpl',
    'arm':     'arm.tmpl',
    'armhfp':  'arm.tmpl',
}

def generate_module_info(moddir, outfile=None):
    def module_desc(mod):
        output = runcmd_output(["modinfo", "-F", "description", mod])
        return output.strip()
    def read_module_set(name):
        return set(l.strip() for l in open(joinpaths(moddir,name)) if ".ko" in l)
    modsets = {'scsi':read_module_set("modules.block"),
               'eth':read_module_set("modules.networking")}

    modinfo = list()
    for root, _dirs, files in os.walk(moddir):
        for modtype, modset in modsets.items():
            for mod in modset.intersection(files):  # modules in this dir
                (name, _ext) = os.path.splitext(mod) # foo.ko -> (foo, .ko)
                desc = module_desc(joinpaths(root,mod)) or "%s driver" % name
                modinfo.append(dict(name=name, type=modtype, desc=desc))

    out = open(outfile or joinpaths(moddir,"module-info"), "w")
    out.write("Version 0\n")
    for mod in sorted(modinfo, key=lambda m: m.get('name')):
        out.write('{name}\n\t{type}\n\t"{desc:.65}"\n'.format(**mod))

class RuntimeBuilder(object):
    '''Builds the anaconda runtime image.'''
    def __init__(self, product, arch, dbo, templatedir=None,
                 installpkgs=None,
                 add_templates=None,
                 add_template_vars=None):
        root = dbo.conf.installroot
        # use a copy of product so we can modify it locally
        product = product.copy()
        product.name = product.name.lower()
        self.vars = DataHolder(arch=arch, product=product, dbo=dbo, root=root,
                               basearch=arch.basearch, libdir=arch.libdir)
        self.dbo = dbo
        self._runner = LoraxTemplateRunner(inroot=root, outroot=root,
                                           dbo=dbo, templatedir=templatedir)
        self.add_templates = add_templates or []
        self.add_template_vars = add_template_vars or {}
        self._installpkgs = installpkgs or []
        self._runner.defaults = self.vars
        self.dbo.reset()

    def _install_branding(self):
        release = None
        q = self.dbo.sack.query()
        a = q.available()
        for pkg in a.filter(provides='/etc/system-release'):
            if pkg.name.startswith('generic'):
                continue
            else:
                release = pkg.name
                break

        if not release:
            logger.error('could not get the release')
            return

        # release
        logger.info('got release: %s', release)
        self._runner.installpkg(release)

        # logos
        release, _suffix = release.split('-', 1)
        self._runner.installpkg('%s-logos' % release)

    def install(self):
        '''Install packages and do initial setup with runtime-install.tmpl'''
        self._install_branding()
        if len(self._installpkgs) > 0:
            self._runner.installpkg(*self._installpkgs)
        self._runner.run("runtime-install.tmpl")
        for tmpl in self.add_templates:
            self._runner.run(tmpl, **self.add_template_vars)

    def writepkglists(self, pkglistdir):
        '''debugging data: write out lists of package contents'''
        if not os.path.isdir(pkglistdir):
            os.makedirs(pkglistdir)
        for pkgobj in q.installed():
            with open(joinpaths(pkglistdir, pkgobj.name), "w") as fobj:
                for fname in pkgobj.files:
                    fobj.write("{0}\n".format(fname))

    def postinstall(self):
        '''Do some post-install setup work with runtime-postinstall.tmpl'''
        # copy configdir into runtime root beforehand
        configdir = joinpaths(self._runner.templatedir,"config_files")
        configdir_path = "tmp/config_files"
        fullpath = joinpaths(self.vars.root, configdir_path)
        if os.path.exists(fullpath):
            remove(fullpath)
        copytree(configdir, fullpath)
        self._runner.run("runtime-postinstall.tmpl", configdir=configdir_path)

    def cleanup(self):
        '''Remove unneeded packages and files with runtime-cleanup.tmpl'''
        self._runner.run("runtime-cleanup.tmpl")

    def verify(self):
        '''Ensure that contents of the installroot can run'''
        status = True

        ELF_MAGIC = b'\x7fELF'

        # Iterate over all files in /usr/bin and /usr/sbin
        # For ELF files, gather them into a list and we'll check them all at
        # the end. For files with a #!, check them as we go
        elf_files = []
        usr_bin = Path(self.vars.root + '/usr/bin')
        usr_sbin = Path(self.vars.root + '/usr/sbin')
        for path in (str(x) for x in itertools.chain(usr_bin.iterdir(), usr_sbin.iterdir()) \
                     if x.is_file()):
            with open(path, "rb") as f:
                magic = f.read(4)
                if magic == ELF_MAGIC:
                    # Save the path, minus the chroot prefix
                    elf_files.append(path[len(self.vars.root):])
                elif magic[:2] == b'#!':
                    # Reopen the file as text and read the first line.
                    # Open as latin-1 so that stray 8-bit characters don't make
                    # things blow up. We only really care about ASCII parts.
                    with open(path, "rt", encoding="latin-1") as f_text:
                        # Remove the #!, split on space, and take the first part
                        shabang = f_text.readline()[2:].split()[0]

                    # Does the path exist?
                    if not os.path.exists(self.vars.root + shabang):
                        logger.error('%s, needed by %s, does not exist', shabang, path)
                        status = False

        # Now, run ldd on all the ELF files
        # Just run ldd once on everything so it isn't logged a million times.
        # At least one thing in the list isn't going to be a dynamic executable,
        # so use execWithCapture to ignore the exit code.
        filename = ''
        for line in execWithCapture('ldd', elf_files, root=self.vars.root,
                log_output=False, filter_stderr=True).split('\n'):
            if line and not line[0].isspace():
                # New filename header, strip the : at the end and save
                filename = line[:-1]
            elif 'not found' in line:
                logger.error('%s, needed by %s, not found', line.split()[0], filename)
                status = False

        return status

    def writepkgsizes(self, pkgsizefile):
        '''debugging data: write a big list of pkg sizes'''
        fobj = open(pkgsizefile, "w")
        getsize = lambda f: os.lstat(f).st_size if os.path.exists(f) else 0
        for p in sorted(q.installed()):
            pkgsize = sum(getsize(joinpaths(self.vars.root,f)) for f in p.files)
            fobj.write("{0.name}.{0.arch}: {1}\n".format(p, pkgsize))

    def generate_module_data(self):
        root = self.vars.root
        moddir = joinpaths(root, "lib/modules/")
        for kver in os.listdir(moddir):
            ksyms = joinpaths(root, "boot/System.map-%s" % kver)
            logger.info("doing depmod and module-info for %s", kver)
            runcmd(["depmod", "-a", "-F", ksyms, "-b", root, kver])
            generate_module_info(moddir+kver, outfile=moddir+"module-info")

    def create_runtime(self, outfile="/var/tmp/squashfs.img", compression="xz", compressargs=None, size=2):
        # make live rootfs image - must be named "LiveOS/rootfs.img" for dracut
        compressargs = compressargs or []
        workdir = joinpaths(os.path.dirname(outfile), "runtime-workdir")
        os.makedirs(joinpaths(workdir, "LiveOS"))

        imgutils.mkrootfsimg(self.vars.root, joinpaths(workdir, "LiveOS/rootfs.img"),
                             "Anaconda", size=size)

        # squash the live rootfs and clean up workdir
        imgutils.mksquashfs(workdir, outfile, compression, compressargs)
        remove(workdir)

    def finished(self):
        """ Done using RuntimeBuilder

        Close the dnf base object
        """
        self.dbo.close()

class TreeBuilder(object):
    '''Builds the arch-specific boot images.
    inroot should be the installtree root (the newly-built runtime dir)'''
    def __init__(self, product, arch, inroot, outroot, runtime, isolabel, domacboot=True, doupgrade=True, templatedir=None, add_templates=None, add_template_vars=None, workdir=None):

        # NOTE: if you pass an arg named "runtime" to a mako template it'll
        # clobber some mako internal variables - hence "runtime_img".
        self.vars = DataHolder(arch=arch, product=product, runtime_img=runtime,
                               runtime_base=basename(runtime),
                               inroot=inroot, outroot=outroot,
                               basearch=arch.basearch, libdir=arch.libdir,
                               isolabel=isolabel, udev=udev_escape, domacboot=domacboot, doupgrade=doupgrade,
                               workdir=workdir, lower=string_lower)
        self._runner = LoraxTemplateRunner(inroot, outroot, templatedir=templatedir)
        self.templatedir = templatedir
        self.treeinfo_data = None

    @property
    def kernels(self):
        return findkernels(root=self.vars.inroot)

    def rebuild_initrds(self, add_args=None, backup="", prefix=""):
        '''Rebuild all the initrds in the tree. If backup is specified, each
        initrd will be renamed with backup as a suffix before rebuilding.
        If backup is empty, the existing initrd files will be overwritten.
        If suffix is specified, the existing initrd is untouched and a new
        image is built with the filename "${prefix}-${kernel.version}.img"
        '''
        add_args = add_args or []
        dracut = ["dracut", "--nomdadmconf", "--nolvmconf"] + add_args
        if not backup:
            dracut.append("--force")

        kernels = [kernel for kernel in self.kernels if hasattr(kernel, "initrd")]
        if not kernels:
            raise Exception("No initrds found, cannot rebuild_initrds")

        # Hush some dracut warnings. TODO: bind-mount proc in place?
        open(joinpaths(self.vars.inroot,"/proc/modules"),"w")
        for kernel in kernels:
            if prefix:
                idir = os.path.dirname(kernel.initrd.path)
                outfile = joinpaths(idir, prefix+'-'+kernel.version+'.img')
            else:
                outfile = kernel.initrd.path
            logger.info("rebuilding %s", outfile)
            if backup:
                initrd = joinpaths(self.vars.inroot, outfile)
                os.rename(initrd, initrd + backup)
            cmd = dracut + [outfile, kernel.version]
            runcmd(cmd, root=self.vars.inroot)

            # ppc64 cannot boot images > 32MiB, check size and warn
            if self.vars.arch.basearch in ("ppc64", "ppc64le") and os.path.exists(outfile):
                st = os.stat(outfile)
                if st.st_size > 32 * 1024 * 1024:
                    logging.warning("ppc64 initrd %s is > 32MiB", outfile)

        os.unlink(joinpaths(self.vars.inroot,"/proc/modules"))

    def build(self):
        templatefile = templatemap[self.vars.arch.basearch]
        self._runner.run(templatefile, kernels=self.kernels)
        self.treeinfo_data = self._runner.results.treeinfo
        self.implantisomd5()

    def implantisomd5(self):
        for _section, data in self.treeinfo_data.items():
            if 'boot.iso' in data:
                iso = joinpaths(self.vars.outroot, data['boot.iso'])
                runcmd(["implantisomd5", iso])

    @property
    def dracut_hooks_path(self):
        """ Return the path to the lorax dracut hooks scripts

            Use the configured share dir if it is setup,
            otherwise default to /usr/share/lorax/dracut_hooks
        """
        if self.templatedir:
            return joinpaths(self.templatedir, "dracut_hooks")
        else:
            return "/usr/share/lorax/dracut_hooks"

    def copy_dracut_hooks(self, hooks):
        """ Copy the hook scripts in hooks into the installroot's /tmp/
        and return a list of commands to pass to dracut when creating the
        initramfs

        hooks is a list of tuples with the name of the hook script and the
        target dracut hook directory
        (eg. [("99anaconda-copy-ks.sh", "/lib/dracut/hooks/pre-pivot")])
        """
        dracut_commands = []
        for hook_script, dracut_path in hooks:
            src = joinpaths(self.dracut_hooks_path, hook_script)
            if not os.path.exists(src):
                logger.error("Missing lorax dracut hook script %s", (src))
                continue
            dst = joinpaths(self.vars.inroot, "/tmp/", hook_script)
            copy2(src, dst)
            dracut_commands += ["--include", joinpaths("/tmp/", hook_script),
                                dracut_path]
        return dracut_commands

#### TreeBuilder helper functions

def findkernels(root="/", kdir="boot"):
    # To find possible flavors, awk '/BuildKernel/ { print $4 }' kernel.spec
    flavors = ('debug', 'PAE', 'PAEdebug', 'smp', 'xen', 'lpae')
    kre = re.compile(r"vmlinuz-(?P<version>.+?\.(?P<arch>[a-z0-9_]+)"
                     r"(.(?P<flavor>{0}))?)$".format("|".join(flavors)))
    kernels = []
    bootfiles = os.listdir(joinpaths(root, kdir))
    for f in bootfiles:
        match = kre.match(f)
        if match:
            kernel = DataHolder(path=joinpaths(kdir, f))
            kernel.update(match.groupdict()) # sets version, arch, flavor
            kernels.append(kernel)

    # look for associated initrd/initramfs/etc.
    for kernel in kernels:
        for f in bootfiles:
            if f.endswith('-'+kernel.version+'.img'):
                imgtype, _rest = f.split('-',1)
                # special backwards-compat case
                if imgtype == 'initramfs':
                    imgtype = 'initrd'
                kernel[imgtype] = DataHolder(path=joinpaths(kdir, f))

    logger.debug("kernels=%s", kernels)
    return kernels

# udev whitelist: 'a-zA-Z0-9#+.:=@_-' (see is_whitelisted in libudev-util.c)
udev_blacklist=' !"$%&\'()*,/;<>?[\\]^`{|}~' # ASCII printable, minus whitelist
udev_blacklist += ''.join(chr(i) for i in range(32)) # ASCII non-printable
def udev_escape(label):
    out = ''
    for ch in label:
        out += ch if ch not in udev_blacklist else '\\x%02x' % ord(ch)
    return out

def string_lower(string):
    """ Return a lowercase string.

    :param string: String to lowercase

    This is used as a filter in the templates.
    """
    return string.lower()

#
# decorators.py
#
#
#
#
#
#

def singleton(cls):
    instances = {}

    def get_instance():
        if cls not in instances:
            instances[cls] = cls()
        return instances[cls]

    return get_instance

#
# __init__.py
#
#
#
#
#
#                     David Cantrell <dcantrell@redhat.com>
#                     Will Woods <wwoods@redhat.com>

# set up logging
logger = logging.getLogger("pylorax")
logger.addHandler(logging.NullHandler())

program_log = logging.getLogger("program")

import sys
import os
import configparser
import tempfile
import locale
from subprocess import CalledProcessError
import selinux
from glob import glob

from pylorax.base import BaseLoraxClass, DataHolder

import dnf

from pylorax.sysutils import joinpaths, remove, linktree

from pylorax.treebuilder import RuntimeBuilder, TreeBuilder
from pylorax.buildstamp import BuildStamp
from pylorax.treeinfo import TreeInfo
from pylorax.discinfo import DiscInfo
from pylorax.executils import runcmd, runcmd_output

# List of drivers to remove on ppc64 arch to keep initrd < 32MiB
REMOVE_PPC64_DRIVERS = "floppy scsi_debug nouveau radeon cirrus mgag200"
REMOVE_PPC64_MODULES = "drm plymouth"

class ArchData(DataHolder):
    lib64_arches = ("x86_64", "ppc64", "ppc64le", "s390x", "ia64", "aarch64")
    bcj_arch = dict(i386="x86", x86_64="x86",
                    ppc="powerpc", ppc64="powerpc", ppc64le="powerpc",
                    arm="arm", armhfp="arm")

    def __init__(self, buildarch):
        super(ArchData, self).__init__()
        self.buildarch = buildarch
        self.basearch = dnf.rpm.basearch(buildarch)
        self.libdir = "lib64" if self.basearch in self.lib64_arches else "lib"
        self.bcj = self.bcj_arch.get(self.basearch)

class Lorax(BaseLoraxClass):

        BaseLoraxClass.__init__(self)
        self._configured = False
        self.product = None
        self.workdir = None
        self.arch = None
        self.conf = None
        self.inroot = None
        self.debug = False
        self.outputdir = None
        self._templatedir = None

        # set locale to C
        locale.setlocale(locale.LC_ALL, 'C')

    def configure(self, conf_file="/etc/lorax/lorax.conf"):
        self.conf = configparser.SafeConfigParser()

        # set defaults
        self.conf.add_section("lorax")
        self.conf.set("lorax", "debug", "1")
        self.conf.set("lorax", "sharedir", "/usr/share/lorax")
        self.conf.set("lorax", "logdir", "/var/log/lorax")

        self.conf.add_section("output")
        self.conf.set("output", "colors", "1")
        self.conf.set("output", "encoding", "utf-8")
        self.conf.set("output", "ignorelist", "/usr/share/lorax/ignorelist")

        self.conf.add_section("templates")
        self.conf.set("templates", "ramdisk", "ramdisk.ltmpl")

        self.conf.add_section("compression")
        self.conf.set("compression", "type", "xz")
        self.conf.set("compression", "args", "")
        self.conf.set("compression", "bcj", "on")

        # read the config file
        if os.path.isfile(conf_file):
            self.conf.read(conf_file)

        # set up the output
        self.debug = self.conf.getboolean("lorax", "debug")
        output_level = output.DEBUG if self.debug else output.INFO

        if sys.stdout.isatty():
            colors = self.conf.getboolean("output", "colors")
        else:
            colors = False
        encoding = self.conf.get("output", "encoding")

        self.output.basic_config(output_level=output_level,
                                 colors=colors, encoding=encoding)

        ignorelist = self.conf.get("output", "ignorelist")
        if os.path.isfile(ignorelist):
            with open(ignorelist, "r") as fobj:
                for line in fobj:
                    line = line.strip()
                    if line and not line.startswith("#"):
                        self.output.ignore(line)

        # cron does not have sbin in PATH,
        # so we have to add it ourselves
        os.environ["PATH"] = "{0}:/sbin:/usr/sbin".format(os.environ["PATH"])

        # remove some environmental variables that can cause problems with package scripts
        env_remove = ('DISPLAY', 'DBUS_SESSION_BUS_ADDRESS')
        list(os.environ.pop(k) for k in env_remove if k in os.environ)

        self._configured = True

    @property
    def templatedir(self):
        """Find the template directory.

        Pick the first directory under sharedir/templates.d/ if it exists.
        Otherwise use the sharedir
        """
        if not self._templatedir:
            self._templatedir = find_templates(self.conf.get("lorax", "sharedir"))
            logger.info("Using templatedir %s", self._templatedir)
        return self._templatedir

    def init_stream_logging(self):
        sh = logging.StreamHandler()
        sh.setLevel(logging.INFO)
        logger.addHandler(sh)

    def init_file_logging(self, logdir, logname="pylorax.log"):
        fh = logging.FileHandler(filename=joinpaths(logdir, logname), mode="w")
        fh.setLevel(logging.DEBUG)
        logger.addHandler(fh)

    def run(self, dbo, product, version, release, variant="", bugurl="",
            isfinal=False, workdir=None, outputdir=None, buildarch=None, volid=None,
            domacboot=True, doupgrade=True, remove_temp=False,
            installpkgs=None,
            size=2,
            add_templates=None,
            add_template_vars=None,
            add_arch_templates=None,
            add_arch_template_vars=None,
            verify=True):

        assert self._configured

        installpkgs = installpkgs or []

        # get lorax version
        try:
            import pylorax.version
        except ImportError:
            vernum = "devel"
        else:
            vernum = pylorax.version.num

        if domacboot:
            try:
                runcmd(["rpm", "-q", "hfsplus-tools"])
            except CalledProcessError:
                logger.critical("you need to install hfsplus-tools to create mac images")
                sys.exit(1)

        # set up work directory
        self.workdir = workdir or tempfile.mkdtemp(prefix="pylorax.work.")
        if not os.path.isdir(self.workdir):
            os.makedirs(self.workdir)

        # set up log directory
        logdir = self.conf.get("lorax", "logdir")
        if not os.path.isdir(logdir):
            os.makedirs(logdir)

        self.init_stream_logging()
        self.init_file_logging(logdir)

        logger.debug("version is %s", vernum)
        logger.debug("using work directory %s", self.workdir)
        logger.debug("using log directory %s", logdir)

        # set up output directory
        self.outputdir = outputdir or tempfile.mkdtemp(prefix="pylorax.out.")
        if not os.path.isdir(self.outputdir):
            os.makedirs(self.outputdir)
        logger.debug("using output directory %s", self.outputdir)

        # do we have root privileges?
        logger.info("checking for root privileges")
        if not os.geteuid() == 0:
            logger.critical("no root privileges")
            sys.exit(1)

        # is selinux disabled?
        # With selinux in enforcing mode the rpcbind package required for
        # dracut nfs module, which is in turn required by anaconda module,
        # will not get installed, because it's preinstall scriptlet fails,
        # resulting in an incomplete initial ramdisk image.
        # The reason is that the scriptlet runs tools from the shadow-utils
        # package in chroot, particularly groupadd and useradd to add the
        # required rpc group and rpc user. This operation fails, because
        # the selinux context on files in the chroot, that the shadow-utils
        # tools need to access (/etc/group, /etc/passwd, /etc/shadow etc.),
        # is wrong and selinux therefore disallows access to these files.
        logger.info("checking the selinux mode")
        if selinux.is_selinux_enabled() and selinux.security_getenforce():
            logger.critical("selinux must be disabled or in Permissive mode")

        # do we have a proper dnf base object?
        logger.info("checking dnf base object")
        if not isinstance(dbo, dnf.Base):
            logger.critical("no dnf base object")
        self.inroot = dbo.conf.installroot
        logger.debug("using install root: %s", self.inroot)

        if not buildarch:
            buildarch = get_buildarch(dbo)

        logger.info("setting up build architecture")
        self.arch = ArchData(buildarch)
        for attr in ('buildarch', 'basearch', 'libdir'):
            logger.debug("self.arch.%s = %s", attr, getattr(self.arch,attr))

        logger.info("setting up build parameters")
        self.product = DataHolder(name=product, version=version, release=release,
                                 variant=variant, bugurl=bugurl, isfinal=isfinal)
        logger.debug("product data: %s", self.product)

        # NOTE: if you change isolabel, you need to change pungi to match, or
        # the pungi images won't boot.
        isolabel = volid or "%s-%s-%s" % (self.product.name, self.product.version, self.arch.basearch)

        if len(isolabel) > 32:
            logger.fatal("the volume id cannot be longer than 32 characters")

        # NOTE: rb.root = dbo.conf.installroot (== self.inroot)
        rb = RuntimeBuilder(product=self.product, arch=self.arch,
                            dbo=dbo, templatedir=self.templatedir,
                            installpkgs=installpkgs,
                            add_templates=add_templates,
                            add_template_vars=add_template_vars)

        logger.info("installing runtime packages")
        rb.install()

        # write .buildstamp
        buildstamp = BuildStamp(self.product.name, self.product.version,
                                self.product.bugurl, self.product.isfinal, self.arch.buildarch)

        buildstamp.write(joinpaths(self.inroot, ".buildstamp"))

        if self.debug:
            rb.writepkglists(joinpaths(logdir, "pkglists"))
            rb.writepkgsizes(joinpaths(logdir, "original-pkgsizes.txt"))

        logger.info("doing post-install configuration")
        rb.postinstall()

        # write .discinfo
        discinfo = DiscInfo(self.product.release, self.arch.basearch)
        discinfo.write(joinpaths(self.outputdir, ".discinfo"))

        logger.info("backing up installroot")
        installroot = joinpaths(self.workdir, "installroot")
        linktree(self.inroot, installroot)

        logger.info("generating kernel module metadata")
        rb.generate_module_data()

        logger.info("cleaning unneeded files")
        rb.cleanup()

        if verify:
            logger.info("verifying the installroot")
            if not rb.verify():
        else:
            logger.info("Skipping verify")

            rb.writepkgsizes(joinpaths(logdir, "final-pkgsizes.txt"))

        logger.info("creating the runtime image")
        runtime = "images/install.img"
        compression = self.conf.get("compression", "type")
        compressargs = self.conf.get("compression", "args").split()     # pylint: disable=no-member
        if self.conf.getboolean("compression", "bcj"):
            if self.arch.bcj:
                compressargs += ["-Xbcj", self.arch.bcj]
            else:
                logger.info("no BCJ filter for arch %s", self.arch.basearch)
        rb.create_runtime(joinpaths(installroot,runtime),
                          compression=compression, compressargs=compressargs,
                          size=size)
        rb.finished()

        logger.info("preparing to build output tree and boot images")
        treebuilder = TreeBuilder(product=self.product, arch=self.arch,
                                  inroot=installroot, outroot=self.outputdir,
                                  runtime=runtime, isolabel=isolabel,
                                  domacboot=domacboot, doupgrade=doupgrade,
                                  templatedir=self.templatedir,
                                  add_templates=add_arch_templates,
                                  add_template_vars=add_arch_template_vars,
                                  workdir=self.workdir)

        logger.info("rebuilding initramfs images")
        dracut_args = ["--xz", "--install", "/.buildstamp", "--no-early-microcode"]
        anaconda_args = dracut_args + ["--add", "anaconda pollcdrom qemu qemu-net"]

        # ppc64 cannot boot an initrd > 32MiB so remove some drivers
        if self.arch.basearch in ("ppc64", "ppc64le"):
            dracut_args.extend(["--omit-drivers", REMOVE_PPC64_DRIVERS])

            # Only omit dracut modules from the initrd so that they're kept for
            # upgrade.img
            anaconda_args.extend(["--omit", REMOVE_PPC64_MODULES])

        treebuilder.rebuild_initrds(add_args=anaconda_args)

        logger.info("populating output tree and building boot images")
        treebuilder.build()

        # write .treeinfo file and we're done
        treeinfo = TreeInfo(self.product.name, self.product.version,
                            self.product.variant, self.arch.basearch)
        for section, data in treebuilder.treeinfo_data.items():
            treeinfo.add_section(section, data)
        treeinfo.write(joinpaths(self.outputdir, ".treeinfo"))

        # cleanup
        if remove_temp:
            remove(self.workdir)

def get_buildarch(dbo):
    # get architecture of the available anaconda package
    buildarch = None
    q = dbo.sack.query()
    a = q.available()
    for anaconda in a.filter(name="anaconda"):
        if anaconda.arch != "src":
            buildarch = anaconda.arch
            break
    if not buildarch:
        logger.critical("no anaconda package in the repository")
        sys.exit(1)

    return buildarch

def setup_logging(logfile, theLogger):
    """
    Setup the various logs

    :param logfile: filename to write the log to
    :type logfile: string
    :param theLogger: top-level logger
    :type theLogger: logging.Logger
    """
    if not os.path.isdir(os.path.abspath(os.path.dirname(logfile))):
        os.makedirs(os.path.abspath(os.path.dirname(logfile)))

    # Setup logging to console and to logfile
    logger.setLevel(logging.DEBUG)
    theLogger.setLevel(logging.DEBUG)

    sh = logging.StreamHandler()
    sh.setLevel(logging.INFO)
    fmt = logging.Formatter("%(asctime)s: %(message)s")
    sh.setFormatter(fmt)
    logger.addHandler(sh)
    theLogger.addHandler(sh)

    fh = logging.FileHandler(filename=logfile, mode="w")
    fh.setLevel(logging.DEBUG)
    fmt = logging.Formatter("%(asctime)s %(levelname)s %(name)s: %(message)s")
    fh.setFormatter(fmt)
    logger.addHandler(fh)
    theLogger.addHandler(fh)

    # External program output log
    program_log.setLevel(logging.DEBUG)
    f = os.path.abspath(os.path.dirname(logfile))+"/program.log"
    fh = logging.FileHandler(filename=f, mode="w")
    program_log.addHandler(fh)

def find_templates(templatedir="/usr/share/lorax"):
    """ Find the templates to use.

    :param str templatedir: Top directory to search for templates
    :returns: Path to templates
    :rtype: str

    If there is a templates.d directory under templatedir the
    lowest numbered directory entry is returned.

    eg. /usr/share/lorax/templates.d/99-generic/
    """
    if os.path.isdir(joinpaths(templatedir, "templates.d")):
        try:
            templatedir = sorted(glob(joinpaths(templatedir, "templates.d", "*")))[0]
        except IndexError:
            pass
    return templatedir

#
# executil.py - subprocess execution utility functions
#
# Copyright (C) 1999-2015
# Red Hat, Inc.  All rights reserved.
#
#
#
#

import os
import subprocess
from subprocess import TimeoutExpired
import signal

log = logging.getLogger("pylorax")

from threading import Lock
program_log_lock = Lock()

_child_env = {}

def setenv(name, value):
    """ Set an environment variable to be used by child processes.

        This method does not modify os.environ for the running process, which
        is not thread-safe. If setenv has already been called for a particular
        variable name, the old value is overwritten.

        :param str name: The name of the environment variable
        :param str value: The value of the environment variable
    """

    _child_env[name] = value

def augmentEnv():
    env = os.environ.copy()
    env.update(_child_env)
    return env

class ExecProduct(object):
    def __init__(self, rc, stdout, stderr):
        self.rc = rc
        self.stdout = stdout
        self.stderr = stderr

def startProgram(argv, root='/', stdin=None, stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
        env_prune=None, env_add=None, reset_handlers=True, reset_lang=True, **kwargs):
    """ Start an external program and return the Popen object.

        The root and reset_handlers arguments are handled by passing a
        preexec_fn argument to subprocess.Popen, but an additional preexec_fn
        can still be specified and will be run. The user preexec_fn will be run
        last.

        :param argv: The command to run and argument
        :param root: The directory to chroot to before running command.
        :param stdin: The file object to read stdin from.
        :param stdout: The file object to write stdout to.
        :param stderr: The file object to write stderr to.
        :param env_prune: environment variables to remove before execution
        :param env_add: environment variables to add before execution
        :param reset_handlers: whether to reset to SIG_DFL any signal handlers set to SIG_IGN
        :param reset_lang: whether to set the locale of the child process to C
        :param kwargs: Additional parameters to pass to subprocess.Popen
        :return: A Popen object for the running command.
        :keyword preexec_fn: A function to run before execution starts.
    """
    if env_prune is None:
        env_prune = []

    # Check for and save a preexec_fn argument
    preexec_fn = kwargs.pop("preexec_fn", None)

    def preexec():
        # If a target root was specificed, chroot into it
        if root and root != '/':
            os.chroot(root)
            os.chdir("/")

        # Signal handlers set to SIG_IGN persist across exec. Reset
        # these to SIG_DFL if requested. In particular this will include the
        # SIGPIPE handler set by python.
        if reset_handlers:
            for signum in range(1, signal.NSIG):
                if signal.getsignal(signum) == signal.SIG_IGN:
                    signal.signal(signum, signal.SIG_DFL)

        # If the user specified an additional preexec_fn argument, run it
        if preexec_fn is not None:
            preexec_fn()

    with program_log_lock:
        program_log.info("Running... %s", " ".join(argv))

    env = augmentEnv()
    for var in env_prune:
        env.pop(var, None)

    if reset_lang:
        env.update({"LC_ALL": "C"})

    if env_add:
        env.update(env_add)

    return subprocess.Popen(argv,
                            stdin=stdin,
                            stdout=stdout,
                            stderr=stderr,
                            close_fds=True,
                            preexec_fn=preexec, cwd=root, env=env, **kwargs)

def _run_program(argv, root='/', stdin=None, stdout=None, env_prune=None, log_output=True,
        binary_output=False, filter_stderr=False, raise_err=False, callback=None,
        env_add=None, reset_handlers=True, reset_lang=True):
    """ Run an external program, log the output and return it to the caller

        :param stdout: Optional file object to write the output to.
        :param env_prune: environment variable to remove before execution
        :param log_output: whether to log the output of command
        :param binary_output: whether to treat the output of command as binary data
        :param filter_stderr: whether to exclude the contents of stderr from the returned output
        :param raise_err: whether to raise a CalledProcessError if the returncode is non-zero
        :param callback: method to call while waiting for process to finish, passed Popen object
        :return: The return code of the command and the output
        :raises: OSError or CalledProcessError
    """
    try:
        if filter_stderr:
            stderr = subprocess.PIPE
        else:
            stderr = subprocess.STDOUT

        proc = startProgram(argv, root=root, stdin=stdin, stdout=subprocess.PIPE, stderr=stderr,
                            env_prune=env_prune, universal_newlines=not binary_output,
                            env_add=env_add, reset_handlers=reset_handlers, reset_lang=reset_lang)

        output_string = None
        err_string = None
        if callback:
            while callback(proc) and proc.poll() is None:
                try:
                    (output_string, err_string) = proc.communicate(timeout=1)
                    break
                except TimeoutExpired:
                    pass
        else:
            (output_string, err_string) = proc.communicate()
        if output_string:
            if binary_output:
                output_lines = [output_string]
            else:
                if output_string[-1] != "\n":
                    output_string = output_string + "\n"
                output_lines = output_string.splitlines(True)

            if log_output:
                with program_log_lock:
                    for line in output_lines:
                        program_log.info(line.strip())

            if stdout:
                stdout.write(output_string)

        # If stderr was filtered, log it separately
        if filter_stderr and err_string and log_output:
            err_lines = err_string.splitlines(True)

            with program_log_lock:
                for line in err_lines:
                    program_log.info(line.strip())

    except OSError as e:
        with program_log_lock:
            program_log.error("Error running %s: %s", argv[0], e.strerror)
        raise

        program_log.debug("Return code: %s", proc.returncode)

    if proc.returncode and raise_err:
        output = (output_string or "") + (err_string or "")
        raise subprocess.CalledProcessError(proc.returncode, argv, output)

    return (proc.returncode, output_string)

def execWithRedirect(command, argv, stdin=None, stdout=None, root='/', env_prune=None,
                     log_output=True, binary_output=False, raise_err=False, callback=None,
                     env_add=None, reset_handlers=True, reset_lang=True):
    """ Run an external program and redirect the output to a file.

        :param command: The command to run
        :param argv: The argument list
        :param stdout: Optional file object to redirect stdout and stderr to.
        :return: The return code of the command
    """
    argv = [command] + list(argv)
    return _run_program(argv, stdin=stdin, stdout=stdout, root=root, env_prune=env_prune,
            log_output=log_output, binary_output=binary_output, raise_err=raise_err, callback=callback,
            env_add=env_add, reset_handlers=reset_handlers, reset_lang=reset_lang)[0]

def execWithCapture(command, argv, stdin=None, root='/', log_output=True, filter_stderr=False,
                    raise_err=False, callback=None, env_add=None, reset_handlers=True, reset_lang=True):
    """ Run an external program and capture standard out and err.

        :param log_output: Whether to log the output of command
        :param filter_stderr: Whether stderr should be excluded from the returned output
        :return: The output of the command
    """
    return _run_program(argv, stdin=stdin, root=root, log_output=log_output, filter_stderr=filter_stderr,
                        raise_err=raise_err, callback=callback, env_add=env_add,
                        reset_handlers=reset_handlers, reset_lang=reset_lang)[1]

def execReadlines(command, argv, stdin=None, root='/', env_prune=None, filter_stderr=False,
                  callback=lambda x: True, env_add=None, reset_handlers=True, reset_lang=True):
    """ Execute an external command and return the line output of the command
        in real-time.

        This method assumes that there is a reasonably low delay between the
        end of output and the process exiting. If the child process closes
        stdout and then keeps on truckin' there will be problems.

        NOTE/WARNING: UnicodeDecodeError will be raised if the output of the
                      external command can't be decoded as UTF-8.

        :return: Iterator of the lines from the command

        Output from the file is not logged to program.log
        This returns an iterator with the lines from the command until it has finished
    """

    class ExecLineReader(object):
        """Iterator class for returning lines from a process and cleaning
           up the process when the output is no longer needed.
        """

        def __init__(self, proc, argv, callback):
            self._proc = proc
            self._argv = argv
            self._callback = callback

        def __iter__(self):
            return self

        def __del__(self):
            # See if the process is still running
            if self._proc.poll() is None:
                # Stop the process and ignore any problems that might arise
                try:
                    self._proc.terminate()
                except OSError:
                    pass

        def __next__(self):
            # Read the next line, blocking if a line is not yet available
            line = self._proc.stdout.readline().decode("utf-8")
            if line == '' or not self._callback(self._proc):
                # Output finished, wait for the process to end
                self._proc.communicate()

                # Check for successful exit
                if self._proc.returncode < 0:
                    raise OSError("process '%s' was killed by signal %s" %
                            (self._argv, -self._proc.returncode))
                elif self._proc.returncode > 0:
                    raise OSError("process '%s' exited with status %s" %
                            (self._argv, self._proc.returncode))
                raise StopIteration

            return line.strip()

    argv = [command] + argv

    if filter_stderr:
        stderr = subprocess.DEVNULL
    else:
        stderr = subprocess.STDOUT

    try:
        proc = startProgram(argv, root=root, stdin=stdin, stdout=subprocess.PIPE, stderr=stderr, bufsize=1,
                            env_prune=env_prune, env_add=env_add, reset_handlers=reset_handlers, reset_lang=reset_lang)
        raise

    return ExecLineReader(proc, argv, callback)

def runcmd(cmd, **kwargs):
    """ run execWithRedirect with raise_err=True
    """
    kwargs["raise_err"] = True
    return execWithRedirect(cmd[0], cmd[1:], **kwargs)

def runcmd_output(cmd, **kwargs):
    """ run execWithCapture with raise_err=True
    """
    return execWithCapture(cmd[0], cmd[1:], **kwargs)

#
# sysutils.py
#
#
#
#
#
#

__all__ = ["joinpaths", "touch", "replace", "chown_", "chmod_", "remove",
           "linktree"]

import sys
import os
import re
import fileinput
import pwd
import grp
import glob
import shutil

from pylorax.executils import runcmd

def joinpaths(*args, **kwargs):
    path = os.path.sep.join(args)

    if kwargs.get("follow_symlinks"):
        return os.path.realpath(path)
    else:
        return path

def touch(fname):
    # python closes the file when it goes out of scope
    open(fname, "w").write("")

def replace(fname, find, sub):
    fin = fileinput.input(fname, inplace=1)
    pattern = re.compile(find)

    for line in fin:
        line = pattern.sub(sub, line)
        sys.stdout.write(line)

    fin.close()

def chown_(path, user=None, group=None, recursive=False):
    uid = gid = -1

    if user is not None:
        uid = pwd.getpwnam(user)[2]
    if group is not None:
        gid = grp.getgrnam(group)[2]

    for fname in glob.iglob(path):
        os.chown(fname, uid, gid)

        if recursive and os.path.isdir(fname):
            for nested in os.listdir(fname):
                nested = joinpaths(fname, nested)
                chown_(nested, user, group, recursive)

def chmod_(path, mode, recursive=False):
        os.chmod(fname, mode)

                chmod_(nested, mode, recursive)

def cpfile(src, dst):
    shutil.copy2(src, dst)
    if os.path.isdir(dst):
        dst = joinpaths(dst, os.path.basename(src))

    return dst

def mvfile(src, dst):
    os.rename(src, dst)
    return dst

def remove(target):
    if os.path.isdir(target) and not os.path.islink(target):
        shutil.rmtree(target)
    else:
        os.unlink(target)

def linktree(src, dst):
    runcmd(["/bin/cp", "-alx", src, dst])

#
# cmdline.py
#
# Copyright (C) 2016  Red Hat, Inc.
#
#
#
#
# Red Hat Author(s):  Brian C. Lane <bcl@redhat.com>

import os
import sys
import argparse

def lorax_parser():
    """ Return the ArgumentParser for lorax"""

    # get lorax version
    try:
        from pylorax import version
        vernum = version.num
    except ImportError:
        vernum = "devel"

    version = "{0}-{1}".format(os.path.basename(sys.argv[0]), vernum)

    parser = argparse.ArgumentParser(description="Create the Anaconda boot.iso")

    # required arguments for image creation
    required = parser.add_argument_group("required arguments")
    required.add_argument("-p", "--product", help="product name", required=True, metavar="PRODUCT")
    required.add_argument("-v", "--version", help="version identifier", required=True, metavar="VERSION")
    required.add_argument("-r", "--release", help="release information", required=True, metavar="RELEASE")
    required.add_argument("-s", "--source", help="source repository (may be listed multiple times)",
                        metavar="REPOSITORY", action="append", default=[])
    required.add_argument("--repo", help="source dnf repository file", type=os.path.abspath,
                          dest="repos", metavar="REPOSITORY", action="append", default=[])

    # optional arguments
    optional = parser.add_argument_group("optional arguments")
    optional.add_argument("-m", "--mirrorlist",
                        help="mirrorlist repository (may be listed multiple times)",
    optional.add_argument("-t", "--variant",
                        help="variant name", metavar="VARIANT")
    optional.add_argument("-b", "--bugurl",
                        help="bug reporting URL for the product", metavar="URL",
                        default="your distribution provided bug reporting tool")
    optional.add_argument("--isfinal", help="",
                        action="store_true", default=False, dest="isfinal")
    optional.add_argument("-c", "--config", default="/etc/lorax/lorax.conf",
                        help="config file", metavar="CONFIGFILE")
    optional.add_argument("--proxy", default=None,
                          help="repo proxy url:port", metavar="HOST")
    optional.add_argument("-i", "--installpkgs", default=[],
                        action="append", metavar="PACKAGE",
                        help="package glob to install before runtime-install.tmpl runs. (may be listed multiple times)")
    optional.add_argument("--buildarch", default=None,
                        help="build architecture", metavar="ARCH")
    optional.add_argument("--volid", default=None,
                        help="volume id", metavar="VOLID")
    optional.add_argument("--macboot", help="",
                        action="store_true", default=True, dest="domacboot")
    optional.add_argument("--nomacboot", help="",
                        action="store_false", dest="domacboot")
    optional.add_argument("--noupgrade", help="",
                        action="store_false", default=True, dest="doupgrade")
    optional.add_argument("--logfile", default="./lorax.log", type=os.path.abspath,
                        help="Path to logfile")
    optional.add_argument("--tmp", default="/var/tmp",
                        help="Top level temporary directory" )
    optional.add_argument("--cachedir", default=None, type=os.path.abspath,
                        help="DNF cache directory. Default is a temporary dir.")
    optional.add_argument("--workdir", default=None, type=os.path.abspath,
                        help="Work directory, overrides --tmp. Default is a temporary dir under /var/tmp")
    optional.add_argument("--force", default=False, action="store_true",
                        help="Run even when the destination directory exists")
    optional.add_argument("--add-template", dest="add_templates",
                        action="append", help="Additional template for runtime image",
                        default=[])
    optional.add_argument("--add-template-var", dest="add_template_vars",
                        action="append", help="Set variable for runtime image template",
    optional.add_argument("--add-arch-template", dest="add_arch_templates",
                        action="append", help="Additional template for architecture-specific image",
    optional.add_argument("--add-arch-template-var", dest="add_arch_template_vars",
                        action="append", help="Set variable for architecture-specific image",
    optional.add_argument("--noverify", action="store_false", default=True, dest="verify",
                        help="Do not verify the install root")
    optional.add_argument("--sharedir", metavar="SHAREDIR", type=os.path.abspath,
                          help="Directory containing all the templates. Overrides config file sharedir")
    optional.add_argument("--enablerepo", action="append", default=[], dest="enablerepos",
                          metavar="[repo]", help="Names of repos to enable")
    optional.add_argument("--disablerepo", action="append", default=[], dest="disablerepos",
                          metavar="[repo]", help="Names of repos to disable")

    # add the show version option
    parser.add_argument("-V", help="show program's version number and exit",
                      action="version", version=version)

    parser.add_argument("outputdir", help="Output directory", metavar="OUTPUTDIR", type=os.path.abspath)

    return parser

def lmc_parser(dracut_default=""):
    """ Return a ArgumentParser object for live-media-creator."""
    parser = argparse.ArgumentParser(description="Create Live Install Media",
                                     fromfile_prefix_chars="@")

    # These are mutually exclusive, one is required
    action = parser.add_mutually_exclusive_group(required=True)
    action.add_argument("--make-iso", action="store_true",
                        help="Build a live iso")
    action.add_argument("--make-disk", action="store_true",
                        help="Build a partitioned disk image")
    action.add_argument("--make-fsimage", action="store_true",
                        help="Build a filesystem image")
    action.add_argument("--make-appliance", action="store_true",
                        help="Build an appliance image and XML description")
    action.add_argument("--make-ami", action="store_true",
                        help="Build an ami image")
    action.add_argument("--make-tar", action="store_true",
                        help="Build a tar of the root filesystem")
    action.add_argument("--make-pxe-live", action="store_true",
                        help="Build a live pxe boot squashfs image")
    action.add_argument("--make-ostree-live", action="store_true",
                        help="Build a live pxe boot squashfs image of Atomic Host")
    action.add_argument("--make-oci", action="store_true",
                        help="Build an Open Container Initiative image")
    action.add_argument("--make-vagrant", action="store_true",
                        help="Build a Vagrant Box image")

    parser.add_argument("--iso", type=os.path.abspath,
                        help="Anaconda installation .iso path to use for qemu")
    parser.add_argument("--iso-only", action="store_true",
                        help="Remove all iso creation artifacts except the boot.iso, "
                             "combine with --iso-name to rename the boot.iso")
    parser.add_argument("--iso-name", default=None,
                        help="Name of output iso file for --iso-only. Default is boot.iso")
    parser.add_argument("--ks", action="append", type=os.path.abspath,
                        help="Kickstart file defining the install.")
    parser.add_argument("--image-only", action="store_true",
                        help="Exit after creating fs/disk image.")

    parser.add_argument("--no-virt", action="store_true",
                        help="Run anaconda directly on host instead of using qemu")
    parser.add_argument("--proxy",
                        help="proxy URL to use for the install")
    parser.add_argument("--anaconda-arg", action="append", dest="anaconda_args",
                        help="Additional argument to pass to anaconda (no-virt "
                             "mode). Pass once for each argument")
    parser.add_argument("--armplatform",
                        help="the platform to use when creating images for ARM, "
                             "i.e., highbank, mvebu, omap, tegra, etc.")
    parser.add_argument("--location", default=None, type=os.path.abspath,
                        help="location of iso directory tree with initrd.img "
                             "and vmlinuz. Used to run qemu with a newer initrd "
                             "than the iso.")

    parser.add_argument("--logfile", default="./livemedia.log",
                        type=os.path.abspath,
                        help="Name and path for primary logfile, other logs will "
                             "be created in the same directory.")
    parser.add_argument("--lorax-templates", default=None,
                        help="Path to mako templates for lorax")
    parser.add_argument("--tmp", default="/var/tmp", type=os.path.abspath,
                        help="Top level temporary directory")
    parser.add_argument("--resultdir", default=None, dest="result_dir",
                        help="Directory to copy the resulting images and iso into. "
                             "Defaults to the temporary working directory")

    parser.add_argument("--macboot", action="store_true", default=True,
                        dest="domacboot")
    parser.add_argument("--nomacboot", action="store_false",

    image_group = parser.add_argument_group("disk/fs image arguments")
    image_group.add_argument("--disk-image", type=os.path.abspath,
                             help="Path to existing disk image to use for creating final image.")
    image_group.add_argument("--keep-image", action="store_true",
                             help="Keep raw disk image after .iso creation")
    image_group.add_argument("--fs-image", type=os.path.abspath,
                             help="Path to existing filesystem image to use for creating final image.")
    image_group.add_argument("--image-name", default=None,
                             help="Name of output file to create. Used for tar, fs and disk image. Default is a random name.")
    image_group.add_argument("--fs-label", default="Anaconda",
                             help="Label to set on fsimage, default is 'Anaconda'")
    image_group.add_argument("--image-type", default=None,
                             help="Create an image with qemu-img. See qemu-img --help for supported formats.")
    image_group.add_argument("--qemu-arg", action="append", dest="qemu_args", default=[],
                             help="Arguments to pass to qemu-img. Pass once for each argument, they will be used for ALL calls to qemu-img.")
    image_group.add_argument("--qcow2", action="store_true",
                             help="Create qcow2 image instead of raw sparse image when making disk images.")
    image_group.add_argument("--qcow2-arg", action="append", dest="qemu_args", default=[],
    image_group.add_argument("--compression", default="xz",
                             help="Compression binary for make-tar. xz, lzma, gzip, and bzip2 are supported. xz is the default.")
    image_group.add_argument("--compress-arg", action="append", dest="compress_args", default=[],
                             help="Arguments to pass to compression. Pass once for each argument")
    # Group of arguments for appliance creation
    app_group = parser.add_argument_group("appliance arguments")
    app_group.add_argument("--app-name", default=None,
                           help="Name of appliance to pass to template")
    app_group.add_argument("--app-template", default=None,
                           help="Path to template to use for appliance data.")
    app_group.add_argument("--app-file", default="appliance.xml",
                           help="Appliance template results file.")

    # Group of arguments to pass to qemu
    virt_group = parser.add_argument_group("qemu arguments")
    virt_group.add_argument("--ram", metavar="MEMORY", type=int, default=1024,
                            help="Memory to allocate for installer in megabytes.")
    virt_group.add_argument("--vcpus", type=int, default=None,
                            help="Passed to qemu -smp command")
    virt_group.add_argument("--vnc",
                            help="Passed to qemu -display command. eg. vnc=127.0.0.1:5, default is to "
                                 "choose the first unused vnc port.")
    virt_group.add_argument("--arch", default=None,
                            help="System arch to build for. Used to select qemu-system-* command. "
                                 "Defaults to qemu-system-<arch>")
    virt_group.add_argument("--kernel-args",
                            help="Additional argument to pass to the installation kernel")
    virt_group.add_argument("--ovmf-path", default="/usr/share/OVMF/",
                            help="Path to OVMF firmware")
    virt_group.add_argument("--virt-uefi", action="store_true", default=False,
                            help="Use OVMF firmware to boot the VM in UEFI mode")
    virt_group.add_argument("--no-kvm", action="store_true", default=False,
                            help="Skip using kvm with qemu even if it is available.")

    # dracut arguments
    dracut_group = parser.add_argument_group("dracut arguments")
    dracut_group.add_argument("--dracut-arg", action="append", dest="dracut_args",
                              help="Argument to pass to dracut when "
                                   "rebuilding the initramfs. Pass this "
                                   "once for each argument. NOTE: this "
                                   "overrides the default. (default: %s)" % dracut_default)

    # pxe to live arguments
    pxelive_group = parser.add_argument_group("pxe to live arguments")
    pxelive_group.add_argument("--live-rootfs-size", type=int, default=0,
                                help="Size of root filesystem of live image in GiB")
    pxelive_group.add_argument("--live-rootfs-keep-size", action="store_true",
                                help="Keep the original size of root filesystem in live image")

    # OCI specific commands
    oci_group = parser.add_argument_group("OCI arguments")
    oci_group.add_argument("--oci-config",
                              help="config.json OCI configuration file")
    oci_group.add_argument("--oci-runtime",
                              help="runtime.json OCI configuration file")

    # Vagrant specific commands
    vagrant_group = parser.add_argument_group("Vagrant arguments")
    vagrant_group.add_argument("--vagrant-metadata",
                               help="optional metadata.json file")
    vagrant_group.add_argument("--vagrantfile",
                               help="optional vagrantfile")

    parser.add_argument("--title", default="Linux Live Media",
                        help="Substituted for @TITLE@ in bootloader config files")
    parser.add_argument("--project", default="Linux",
                        help="substituted for @PROJECT@ in bootloader config files")
    parser.add_argument("--releasever", default="25",
                        help="substituted for @VERSION@ in bootloader config files")
    parser.add_argument("--volid", default=None, help="volume id")
    parser.add_argument("--squashfs_args",
                        help="additional squashfs args")
    parser.add_argument("--timeout", default=None, type=int,
                        help="Cancel installer after X minutes")

#
# ltmpl.py
#
# Copyright (C) 2009-2015  Red Hat, Inc.
#
#
#
#
#

logger = logging.getLogger("pylorax.ltmpl")

import os, re, glob, shlex, fnmatch
from os.path import basename, isdir

from pylorax.sysutils import joinpaths, cpfile, mvfile, replace, remove
from pylorax.dnfhelper import LoraxDownloadCallback, LoraxRpmCallback
from pylorax.imgutils import mkcpio

from mako.lookup import TemplateLookup
from mako.exceptions import text_error_template
import sys, traceback
import struct
import dnf

class LoraxTemplate(object):
    def __init__(self, directories=None):
        directories = directories or ["/usr/share/lorax"]
        # we have to add ["/"] to the template lookup directories or the
        # file includes won't work properly for absolute paths
        self.directories = ["/"] + directories

    def parse(self, template_file, variables):
        lookup = TemplateLookup(directories=self.directories)
        template = lookup.get_template(template_file)

        try:
            textbuf = template.render(**variables)
        except:
            logger.error("Problem rendering %s (%s):", template_file, variables)
            logger.error(text_error_template().render())
            raise

        # split, strip and remove empty lines
        lines = textbuf.splitlines()
        lines = [line.strip() for line in lines]
        lines = [line for line in lines if line]

        # remove comments
        lines = [line for line in lines if not line.startswith("#")]

        # split with shlex and perform brace expansion
        lines = [split_and_expand(line) for line in lines]

        return lines

def split_and_expand(line):
    return [exp for word in shlex.split(line) for exp in brace_expand(word)]

def brace_expand(s):
    if not ('{' in s and ',' in s and '}' in s):
        yield s
    else:
        right = s.find('}')
        left = s[:right].rfind('{')
        (prefix, choices, suffix) = (s[:left], s[left+1:right], s[right+1:])
        for choice in choices.split(','):
            for alt in brace_expand(prefix+choice+suffix):
                yield alt

def rglob(pathname, root="/", fatal=False):
    seen = set()
    rootlen = len(root)+1
    for f in glob.iglob(joinpaths(root, pathname)):
        if f not in seen:
            seen.add(f)
            yield f[rootlen:] # remove the root to produce relative path
    if fatal and not seen:
        raise IOError("nothing matching %s in %s" % (pathname, root))

def rexists(pathname, root=""):
    # Generator is always True, even with no values;
    # bool(rglob(...)) won't work here.
    for _path in rglob(pathname, root):
        return True
    return False

# TODO: operate inside an actual chroot for safety? Not that RPM bothers..
class LoraxTemplateRunner(object):
    '''
    This class parses and executes Lorax templates. Sample usage:

      # install a bunch of packages
      runner = LoraxTemplateRunner(inroot=rundir, outroot=rundir, dbo=dnf_obj)
      runner.run("install-packages.ltmpl")

      # modify a runtime dir
      runner = LoraxTemplateRunner(inroot=rundir, outroot=newrun)
      runner.run("runtime-transmogrify.ltmpl")

    NOTES:

    * Parsing procedure is roughly:
      1. Mako template expansion (on the whole file)
      2. For each line of the result,
        a. Whitespace splitting (using shlex.split())
        b. Brace expansion (using brace_expand())
        c. If the first token is the name of a function, call that function
           with the rest of the line as arguments

    * Parsing and execution are *separate* passes - so you can't use the result
      of a command in an %if statement (or any other control statements)!

    * Commands that run external programs (systemctl, gconfset) currently use
      the *host*'s copy of that program, which may cause problems if there's a
      big enough difference between the host and the image you're modifying.

    * The commands are not executed under a real chroot, so absolute symlinks
      will point *outside* the inroot/outroot. Be careful with symlinks!

    ADDING NEW COMMANDS:

    * Each template command is just a method of the LoraxTemplateRunner
      object - so adding a new command is as easy as adding a new function.

    * Each function gets arguments that correspond to the rest of the tokens
      on that line (after word splitting and brace expansion)

    * Commands should raise exceptions for errors - don't use sys.exit()
    '''
    def __init__(self, inroot, outroot, dbo=None, fatalerrors=True,
                                        templatedir=None, defaults=None):
        self.inroot = inroot
        self.outroot = outroot
        self.fatalerrors = fatalerrors
        self.templatedir = templatedir or "/usr/share/lorax"
        self.templatefile = None
        # some builtin methods
        self.builtins = DataHolder(exists=lambda p: rexists(p, root=inroot),
                                   glob=lambda g: list(rglob(g, root=inroot)))
        self.defaults = defaults or {}
        self.results = DataHolder(treeinfo=dict()) # just treeinfo for now
        # TODO: set up custom logger with a filter to add line info

    def _out(self, path):
        return joinpaths(self.outroot, path)
    def _in(self, path):
        return joinpaths(self.inroot, path)

    def _filelist(self, *pkgs):
        """ Return the list of files in the packages """
        pkglist = []
        for pkg_glob in pkgs:
            pkglist += list(self.dbo.sack.query().installed().filter(name__glob=pkg_glob))

        # dnf/hawkey doesn't make any distinction between file, dir or ghost like yum did
        # so only return the files.
        return set(f for pkg in pkglist for f in pkg.files if not os.path.isdir(self._out(f)))

    def _getsize(self, *files):
        return sum(os.path.getsize(self._out(f)) for f in files if os.path.isfile(self._out(f)))

    def _write_debuginfo_log(self):
        """
        Write a list of debuginfo packages to /root/debug-pkgs.log

        If lorax is called with a debug repo find the corresponding debuginfo package
        names and write them to /root/debubg-pkgs.log on the boot.iso
        """
        for repo in self.dbo.repos:
            repo = self.dbo.repos[repo]
            if any(True for url in repo.baseurl if "debug" in url):
                break
            if repo.metalink and "debug" in repo.metalink:
                break
            if repo.mirrorlist and "debug" in repo.mirrorlist:
                break
        else:
            # No debug repos
            return

        available = self.dbo.sack.query().available()
        debug_pkgs = []
        for p in list(self.dbo.transaction.install_set):
            if available.filter(name=p.name+"-debuginfo"):
                debug_pkgs += ["{0.name}-debuginfo-{0.epoch}:{0.version}-{0.release}".format(p)]

        os.makedirs(self._out("root/"), exist_ok=True)
        with open(self._out("root/debug-pkgs.log"), "w") as f:
            for pkg in debug_pkgs:
                f.write("%s\n" % pkg)

    def run(self, templatefile, **variables):
        for k,v in list(self.defaults.items()) + list(self.builtins.items()):
            variables.setdefault(k,v)
        logger.debug("executing %s with variables=%s", templatefile, variables)
        self.templatefile = templatefile
        t = LoraxTemplate(directories=[self.templatedir])
        commands = t.parse(templatefile, variables)
        self._run(commands)

    def _run(self, parsed_template):
        logger.info("running %s", self.templatefile)
        for (num, line) in enumerate(parsed_template,1):
            logger.debug("template line %i: %s", num, " ".join(line))
            skiperror = False
            (cmd, args) = (line[0], line[1:])
            # Following Makefile convention, if the command is prefixed with
            # a dash ('-'), we'll ignore any errors on that line.
            if cmd.startswith('-'):
                cmd = cmd[1:]
                skiperror = True
            try:
                # grab the method named in cmd and pass it the given arguments
                f = getattr(self, cmd, None)
                if cmd[0] == '_' or cmd == 'run' or not isinstance(f, collections.Callable):
                    raise ValueError("unknown command %s" % cmd)
                f(*args)
            except Exception: # pylint: disable=broad-except
                if skiperror:
                    logger.debug("ignoring error")
                    continue
                logger.error("template command error in %s:", self.templatefile)
                logger.error("  %s", " ".join(line))
                # format the exception traceback
                exclines = traceback.format_exception(*sys.exc_info())
                # skip the bit about "ltmpl.py, in _run()" - we know that
                exclines.pop(1)
                # log the "ErrorType: this is what happened" line
                logger.error("  " + exclines[-1].strip())
                # and log the entire traceback to the debug log
                for line in ''.join(exclines).splitlines():
                    logger.debug("  " + line)
                if self.fatalerrors:
                    raise

    def install(self, srcglob, dest):
        '''
        install SRC DEST
          Copy the given file (or files, if a glob is used) from the input
          tree to the given destination in the output tree.
          The path to DEST must exist in the output tree.
          If DEST is a directory, SRC will be copied into that directory.
          If DEST doesn't exist, SRC will be copied to a file with that name,
          assuming the rest of the path exists.
          This is pretty much like how the 'cp' command works.
          Examples:
            install usr/share/myconfig/grub.conf /boot
            install /usr/share/myconfig/grub.conf.in /boot/grub.conf
        '''
        for src in rglob(self._in(srcglob), fatal=True):
            cpfile(src, self._out(dest))

    def installimg(self, srcdir, destfile):
        '''
        installimg SRCDIR DESTFILE
          Create a compressed cpio archive of the contents of SRCDIR and place
          it in DESTFILE.

          If SRCDIR doesn't exist or is empty nothing is created.

          Examples:
            installimg ${LORAXDIR}/product/ images/product.img
            installimg ${LORAXDIR}/updates/ images/updates.img
        '''
        if not os.path.isdir(self._in(srcdir)) or not os.listdir(self._in(srcdir)):
            return
        logger.info("Creating image file %s from contents of %s", self._out(destfile), self._in(srcdir))
        mkcpio(self._in(srcdir), self._out(destfile))

    def mkdir(self, *dirs):
        '''
        mkdir DIR [DIR ...]
          Create the named DIR(s). Will create leading directories as needed.
          Example:
            mkdir /images
        '''
        for d in dirs:
            d = self._out(d)
            if not isdir(d):
                os.makedirs(d)

    def replace(self, pat, repl, *fileglobs):
        '''
        replace PATTERN REPLACEMENT FILEGLOB [FILEGLOB ...]
          Find-and-replace the given PATTERN (Python-style regex) with the given
          REPLACEMENT string for each of the files listed.
          Example:
            replace @VERSION@ ${product.version} /boot/grub.conf /boot/isolinux.cfg
        '''
        match = False
        for g in fileglobs:
            for f in rglob(self._out(g)):
                match = True
                replace(f, pat, repl)
        if not match:
            raise IOError("no files matched %s" % " ".join(fileglobs))

    def append(self, filename, data):
        '''
        append FILE STRING
          Append STRING (followed by a newline character) to FILE.
          Python character escape sequences ('\\n', '\\t', etc.) will be
          converted to the appropriate characters.
          Examples:
            append /etc/depmod.d/dd.conf "search updates built-in"
            append /etc/resolv.conf ""
        '''
        with open(self._out(filename), "a") as fobj:
            fobj.write(bytes(data, "utf8").decode('unicode_escape')+"\n")

    def treeinfo(self, section, key, *valuetoks):
        '''
        treeinfo SECTION KEY ARG [ARG ...]
          Add an item to the treeinfo data store.
          The given SECTION will have a new item added where
          KEY = ARG ARG ...
          Example:
            treeinfo images-${kernel.arch} boot.iso images/boot.iso
        '''
        if section not in self.results.treeinfo:
            self.results.treeinfo[section] = dict()
        self.results.treeinfo[section][key] = " ".join(valuetoks)

    def installkernel(self, section, src, dest):
        '''
        installkernel SECTION SRC DEST
          Install the kernel from SRC in the input tree to DEST in the output
          tree, and then add an item to the treeinfo data store, in the named
          SECTION, where "kernel" = DEST.

          Equivalent to:
            install SRC DEST
            treeinfo SECTION kernel DEST
        '''
        self.install(src, dest)
        self.treeinfo(section, "kernel", dest)

    def installinitrd(self, section, src, dest):
        '''
        installinitrd SECTION SRC DEST
          Same as installkernel, but for "initrd".
        '''
        self.chmod(dest, '644')
        self.treeinfo(section, "initrd", dest)

    def installupgradeinitrd(self, section, src, dest):
        '''
        installupgradeinitrd SECTION SRC DEST
          Same as installkernel, but for "upgrade".
        '''
        self.treeinfo(section, "upgrade", dest)

    def hardlink(self, src, dest):
        '''
        hardlink SRC DEST
          Create a hardlink at DEST which is linked to SRC.
        '''
        if isdir(self._out(dest)):
            dest = joinpaths(dest, basename(src))
        os.link(self._out(src), self._out(dest))

    def symlink(self, target, dest):
        '''
        symlink SRC DEST
          Create a symlink at DEST which points to SRC.
        '''
        if rexists(self._out(dest)):
            self.remove(dest)
        os.symlink(target, self._out(dest))

    def copy(self, src, dest):
        '''
        copy SRC DEST
          Copy SRC to DEST.
          If DEST is a directory, SRC will be copied inside it.
          If DEST doesn't exist, SRC will be copied to a file with
          that name, if the path leading to it exists.
        '''
        cpfile(self._out(src), self._out(dest))

    def move(self, src, dest):
        '''
        move SRC DEST
          Move SRC to DEST.
        '''
        mvfile(self._out(src), self._out(dest))

    def remove(self, *fileglobs):
        '''
        remove FILEGLOB [FILEGLOB ...]
          Remove all the named files or directories.
          Will *not* raise exceptions if the file(s) are not found.
        '''
                remove(f)
                logger.debug("removed %s", f)

    def chmod(self, fileglob, mode):
        '''
        chmod FILEGLOB OCTALMODE
          Change the mode of all the files matching FILEGLOB to OCTALMODE.
        '''
        for f in rglob(self._out(fileglob), fatal=True):
            os.chmod(f, int(mode,8))

    # TODO: do we need a new command for gsettings?
    def gconfset(self, path, keytype, value, outfile=None):
        '''
        gconfset PATH KEYTYPE VALUE [OUTFILE]
          Set the given gconf PATH, with type KEYTYPE, to the given value.
          OUTFILE defaults to /etc/gconf/gconf.xml.defaults if not given.
          Example:
            gconfset /apps/metacity/general/num_workspaces int 1
        '''
        if outfile is None:
            outfile = self._out("etc/gconf/gconf.xml.defaults")
        cmd = ["gconftool-2", "--direct",
                    "--config-source=xml:readwrite:%s" % outfile,
                    "--set", "--type", keytype, path, value]
        runcmd(cmd)

    def log(self, msg):
        '''
        log MESSAGE
          Emit the given log message. Be sure to put it in quotes!
          Example:
            log "Reticulating splines, please wait..."
        '''
        logger.info(msg)

    # TODO: add ssh-keygen, mkisofs(?), find, and other useful commands
    def runcmd(self, *cmdlist):
        '''
        runcmd CMD [ARG ...]
          Run the given command with the given arguments.

          NOTE: All paths given MUST be COMPLETE, ABSOLUTE PATHS to the file
          or files mentioned. ${root}/${inroot}/${outroot} are good for
          constructing these paths.

          FURTHER NOTE: Please use this command only as a last resort!
          Whenever possible, you should use the existing template commands.
          If the existing commands don't do what you need, fix them!

          Examples:
            (this should be replaced with a "find" function)
            runcmd find ${root} -name "*.pyo" -type f -delete
            %for f in find(root, name="*.pyo"):
                remove ${f}
            %endfor
        '''
        cmd = cmdlist
        logger.debug('running command: %s', cmd)
        if cmd[0].startswith("--chdir="):
            logger.error("--chdir is no longer supported for runcmd.")
            raise ValueError("--chdir is no longer supported for runcmd.")

        try:
            stdout = runcmd_output(cmd)
            if stdout:
                logger.debug('command output:\n%s', stdout)
            logger.debug("command finished successfully")
        except CalledProcessError as e:
            if e.output:
                logger.debug('command output:\n%s', e.output)
            logger.debug('command returned failure (%d)', e.returncode)
            raise

    def installpkg(self, *pkgs):
        '''
        installpkg [--required] PKGGLOB [PKGGLOB ...]
          Request installation of all packages matching the given globs.
          Note that this is just a *request* - nothing is *actually* installed
          until the 'run_pkg_transaction' command is given.
        '''
        required = False
        if pkgs[0] == '--required':
            pkgs = pkgs[1:]
            required = True

        for p in pkgs:
            try:
                self.dbo.install(p)
            except Exception as e: # pylint: disable=broad-except
                # FIXME: save exception and re-raise after the loop finishes
                logger.error("installpkg %s failed: %s", p, str(e))
                if required:
                    raise

    def removepkg(self, *pkgs):
        '''
        removepkg PKGGLOB [PKGGLOB...]
          Delete the named package(s).
          IMPLEMENTATION NOTES:
            RPM scriptlets (%preun/%postun) are *not* run.
            Files are deleted, but directories are left behind.
        '''
            filepaths = [f.lstrip('/') for f in self._filelist(p)]
            # TODO: also remove directories that aren't owned by anything else
            if filepaths:
                logger.debug("removepkg %s: %ikb", p, self._getsize(*filepaths)/1024)
                self.remove(*filepaths)
            else:
                logger.debug("removepkg %s: no files to remove!", p)

    def run_pkg_transaction(self):
        '''
        run_pkg_transaction
          Actually install all the packages requested by previous 'installpkg'
          commands.
        '''
        try:
            logger.info("Checking dependencies")
            self.dbo.resolve()
        except dnf.exceptions.DepsolveError as e:
            logger.error("Dependency check failed: %s", e)
            raise
        logger.info("%d packages selected", len(self.dbo.transaction))
        if len(self.dbo.transaction) == 0:
            raise Exception("No packages in transaction")

        # If a debug repo has been included, write out a list of debuginfo packages
        self._write_debuginfo_log()

        pkgs_to_download = self.dbo.transaction.install_set
        logger.info("Downloading packages")
        progress = LoraxDownloadCallback()
        try:
            self.dbo.download_packages(pkgs_to_download, progress)
        except dnf.exceptions.DownloadError as e:
            logger.error("Failed to download the following packages: %s", e)
            raise

        logger.info("Preparing transaction from installation source")
        try:
            display = LoraxRpmCallback()
            self.dbo.do_transaction(display=display)
        except BaseException as e:
            logger.error("The transaction process has ended abruptly: %s", e)
            raise

        # Reset the package sack to pick up the installed packages
        self.dbo.reset(repos=False)
        self.dbo.fill_sack(load_system_repo=True, load_available_repos=False)

        # At this point dnf should know about the installed files. Double check that it really does.
        if len(self._filelist("anaconda-core")) == 0:
            raise Exception("Failed to reset dbo to installed package set")

    def removefrom(self, pkg, *globs):
        '''
        removefrom PKGGLOB [--allbut] FILEGLOB [FILEGLOB...]
          Remove all files matching the given file globs from the package
          (or packages) named.
          If '--allbut' is used, all the files from the given package(s) will
          be removed *except* the ones which match the file globs.
          Examples:
            removefrom usbutils /usr/bin/*
            removefrom xfsprogs --allbut /sbin/*
        '''
        cmd = "%s %s" % (pkg, " ".join(globs)) # save for later logging
        keepmatches = False
        if globs[0] == '--allbut':
            keepmatches = True
            globs = globs[1:]
        # get pkg filelist and find files that match the globs
        filelist = self._filelist(pkg)
        matches = set()
        for g in globs:
            globs_re = re.compile(fnmatch.translate(g))
            m = [f for f in filelist if globs_re.match(f)]
            if m:
                matches.update(m)
            else:
                logger.debug("removefrom %s %s: no files matched!", pkg, g)
        # are we removing the matches, or keeping only the matches?
        if keepmatches:
            remove_files = filelist.difference(matches)
        else:
            remove_files = matches
        # remove the files
        if remove_files:
            logger.debug("removefrom %s: removed %i/%i files, %ikb/%ikb", cmd,
                             len(remove_files), len(filelist),
                             self._getsize(*remove_files)/1024, self._getsize(*filelist)/1024)
            self.remove(*remove_files)
        else:
            logger.debug("removefrom %s: no files to remove!", cmd)

    def removekmod(self, *globs):
        '''
        removekmod GLOB [GLOB...] [--allbut] KEEPGLOB [KEEPGLOB...]
          Remove all files and directories matching the given file globs from the kernel
          modules directory.

          If '--allbut' is used, all the files from the modules will be removed *except*
          the ones which match the file globs. There must be at least one initial GLOB
          to search and one KEEPGLOB to keep. The KEEPGLOB is expanded to be *KEEPGLOB*
          so that it will match anywhere in the path.

          This only removes files from under /lib/modules/*/kernel/

          Examples:
            removekmod sound drivers/media drivers/hwmon drivers/video
            removekmod drivers/char --allbut virtio_console hw_random
        '''
        cmd = " ".join(globs)
        if "--allbut" in globs:
            idx = globs.index("--allbut")
            if idx == 0:
                raise ValueError("removekmod needs at least one GLOB before --allbut")

            # Apply keepglobs anywhere they appear in the path
            keepglobs = globs[idx+1:]
            if len(keepglobs) == 0:
                raise ValueError("removekmod needs at least one GLOB after --allbut")

            globs = globs[:idx]
        else:
            # Nothing to keep
            keepglobs = []

        filelist = set()
            for top_dir in rglob(self._out("/lib/modules/*/kernel/"+g)):
                for root, _dirs, files in os.walk(top_dir):
                    filelist.update(root+"/"+f for f in files)

        # Remove anything matching keepglobs from the list
        for g in keepglobs:
            globs_re = re.compile(fnmatch.translate("*"+g+"*"))
            if m:
            else:
                logger.debug("removekmod %s: no files matched!", g)
        remove_files = filelist.difference(matches)

            logger.debug("removekmod: removing %d files", len(remove_files))
            list(remove(f) for f in remove_files)
        else:
            logger.debug("removekmod %s: no files to remove!", cmd)

    def createaddrsize(self, addr, src, dest):
        '''
        createaddrsize INITRD_ADDRESS INITRD ADDRSIZE
          Create the initrd.addrsize file required in LPAR boot process.
          Examples:
            createaddrsize ${INITRD_ADDRESS} ${outroot}/${BOOTDIR}/initrd.img ${outroot}/${BOOTDIR}/initrd.addrsize
        '''
        addrsize = open(dest, "wb")
        addrsize_data = struct.pack(">iiii", 0, int(addr, 16), 0, os.stat(src).st_size)
        addrsize.write(addrsize_data)
        addrsize.close()

    def systemctl(self, cmd, *units):
        '''
        systemctl [enable|disable|mask] UNIT [UNIT...]
          Enable, disable, or mask the given systemd units.
          Examples:
            systemctl disable lvm2-monitor.service
            systemctl mask fedora-storage-init.service fedora-configure.service
        '''
        if cmd not in ('enable', 'disable', 'mask'):
            raise ValueError('unsupported systemctl cmd: %s' % cmd)
        if not units:
            logger.debug("systemctl: no units given for %s, ignoring", cmd)
            return
        self.mkdir("/run/systemd/system") # XXX workaround for systemctl bug
        systemctl = ('systemctl', '--root', self.outroot, '--no-reload',
                     '--quiet', cmd)
        # XXX for some reason 'systemctl enable/disable' always returns 1
        try:
            cmd = systemctl + units
            runcmd(cmd)
        except CalledProcessError:
            pass

# imgutils.py - utility functions/classes for building disk images
#
#
#
#
#

logger = logging.getLogger("pylorax.imgutils")

import os, tempfile
from os.path import join, dirname
from subprocess import Popen, PIPE, CalledProcessError
import sys
import traceback
import multiprocessing
from time import sleep

from pylorax.sysutils import cpfile
from pylorax.executils import execWithRedirect, execWithCapture

######## Functions for making container images (cpio, tar, squashfs) ##########

def compress(command, rootdir, outfile, compression="xz", compressargs=None):
    '''Make a compressed archive of the given rootdir.
    command is a list of the archiver commands to run
    compression should be "xz", "gzip", "lzma", "bzip2", or None.
    compressargs will be used on the compression commandline.'''
    if compression not in (None, "xz", "gzip", "lzma", "bzip2"):
        raise ValueError("Unknown compression type %s" % compression)
    compressargs = compressargs or ["-9"]
    if compression == "xz":
        compressargs.insert(0, "--check=crc32")
    if compression is None:
        compression = "cat" # this is a little silly
        compressargs = []

    # make compression run with multiple threads if possible
    if compression in ("xz", "lzma"):
        compressargs.insert(0, "-T%d" % multiprocessing.cpu_count())
    elif compression == "gzip":
        compression = "pigz"
        compressargs.insert(0, "-p%d" % multiprocessing.cpu_count())
    elif compression == "bzip2":
        compression = "pbzip2"

    logger.debug("find %s -print0 |%s | %s %s > %s", rootdir, " ".join(command),
                 compression, " ".join(compressargs), outfile)
    find, archive, comp = None, None, None
    try:
        find = Popen(["find", ".", "-print0"], stdout=PIPE, cwd=rootdir)
        archive = Popen(command, stdin=find.stdout, stdout=PIPE, cwd=rootdir)
        comp = Popen([compression] + compressargs,
                     stdin=archive.stdout, stdout=open(outfile, "wb"))
        comp.wait()
        return comp.returncode
        logger.error(e)
        # Kill off any hanging processes
        list(p.kill() for p in (find, archive, comp) if p)
        return 1

def mkcpio(rootdir, outfile, compression="xz", compressargs=None):
    return compress(["cpio", "--null", "--quiet", "-H", "newc", "-o"],
                    rootdir, outfile, compression, compressargs)

def mktar(rootdir, outfile, compression="xz", compressargs=None, selinux=True):
    tar_cmd = ["tar", "--no-recursion"]
    if selinux:
        tar_cmd += ["--selinux", "--acls", "--xattrs"]
    tar_cmd += ["-cf-", "--null", "-T-"]
    return compress(tar_cmd, rootdir, outfile, compression, compressargs)

def mksquashfs(rootdir, outfile, compression="default", compressargs=None):
    '''Make a squashfs image containing the given rootdir.'''
    compressargs = compressargs or []
    if compression != "default":
        compressargs = ["-comp", compression] + compressargs
    return execWithRedirect("mksquashfs", [rootdir, outfile] + compressargs)

def mkrootfsimg(rootdir, outfile, label, size=2, sysroot=""):
    """
    Make rootfs image from a directory

    :param str rootdir: Root directory
    :param str outfile: Path of output image file
    :param str label: Filesystem label
    :param int size: Size of the image in GiB, if None computed automatically
    :param str sysroot: path to system (deployment) root relative to physical root
    """
    if size:
        fssize = size * (1024*1024*1024) # 2GB sparse file compresses down to nothin'
    else:
        fssize = None       # Let mkext4img figure out the needed size

    mkext4img(rootdir, outfile, label=label, size=fssize)
    # Reset selinux context on new rootfs
    with LoopDev(outfile) as loopdev:
        with Mount(loopdev) as mnt:
            cmd = [ "setfiles", "-e", "/proc", "-e", "/sys", "-e", "/dev", "-e", "/install",
                    "/etc/selinux/targeted/contexts/files/file_contexts", "/"]
            root = join(mnt, sysroot.lstrip("/"))
            runcmd(cmd, root=root)

######## Utility functions ###############################################

def mksparse(outfile, size):
    '''use os.ftruncate to create a sparse file of the given size.'''
    fobj = open(outfile, "w")
    os.ftruncate(fobj.fileno(), size)

def mkqcow2(outfile, size, options=None):
    '''use qemu-img to create a file of the given size.
       options is a list of options passed to qemu-img

       Default format is qcow2, override by passing "-f", fmt
       in options.
    '''
    mkqemu_img(outfile, size, options)

def mkqemu_img(outfile, size, options=None):

    '''
    options = options or []
    if "-f" not in options:
        options.extend(["-f", "qcow2"])
    runcmd(["qemu-img", "create"] + options + [outfile, str(size)])

def loop_attach(outfile):
    '''Attach a loop device to the given file. Return the loop device name.
    Raises CalledProcessError if losetup fails.'''
    dev = runcmd_output(["losetup", "--find", "--show", outfile])
    return dev.strip()

def loop_detach(loopdev):
    '''Detach the given loop device. Return False on failure.'''
    return (execWithRedirect("losetup", ["--detach", loopdev]) == 0)

def get_loop_name(path):
    '''Return the loop device associated with the path.
    Raises RuntimeError if more than one loop is associated'''
    buf = runcmd_output(["losetup", "-j", path])
    if len(buf.splitlines()) > 1:
        # there should never be more than one loop device listed
        raise RuntimeError("multiple loops associated with %s" % path)
    name = os.path.basename(buf.split(":")[0])
    return name

def dm_attach(dev, size, name=None):
    '''Attach a devicemapper device to the given device, with the given size.
    If name is None, a random name will be chosen. Returns the device name.
    raises CalledProcessError if dmsetup fails.'''
    if name is None:
        name = tempfile.mktemp(prefix="lorax.imgutils.", dir="")
    runcmd(["dmsetup", "create", name, "--table",
                       "0 %i linear %s 0" % (size/512, dev)])

def dm_detach(dev):
    '''Detach the named devicemapper device. Returns False if dmsetup fails.'''
    dev = dev.replace("/dev/mapper/", "") # strip prefix, if it's there
    return execWithRedirect("dmsetup", ["remove", dev])

def mount(dev, opts="", mnt=None):
    '''Mount the given device at the given mountpoint, using the given opts.
    opts should be a comma-separated string of mount options.
    if mnt is none, a temporary directory will be created and its path will be
    returned.
    raises CalledProcessError if mount fails.'''
    if mnt is None:
        mnt = tempfile.mkdtemp(prefix="lorax.imgutils.")
        logger.debug("make tmp mountdir %s", mnt)
    cmd = ["mount"]
    if opts:
        cmd += ["-o", opts]
    cmd += [dev, mnt]
    runcmd(cmd)
    return mnt

def umount(mnt,  lazy=False, maxretry=3, retrysleep=1.0):
    '''Unmount the given mountpoint. If lazy is True, do a lazy umount (-l).
    If the mount was a temporary dir created by mount, it will be deleted.
    raises CalledProcessError if umount fails.'''
    cmd = ["umount"]
    if lazy: cmd += ["-l"]
    cmd += [mnt]
    count = 0
    while maxretry > 0:
        try:
            rv = runcmd(cmd)
            count += 1
            if count == maxretry:
                raise
            logger.warning("failed to unmount %s. retrying (%d/%d)...",
                           mnt, count, maxretry)
            if logger.getEffectiveLevel() <= logging.DEBUG:
                fuser = execWithCapture("fuser", ["-vm", mnt])
                logger.debug("fuser -vm:\n%s\n", fuser)
            sleep(retrysleep)
        else:
            break
    if 'lorax.imgutils' in mnt:
        os.rmdir(mnt)
        logger.debug("remove tmp mountdir %s", mnt)
    return (rv == 0)

def copytree(src, dest, preserve=True):
    '''Copy a tree of files using cp -a, thus preserving modes, timestamps,
    links, acls, sparse files, xattrs, selinux contexts, etc.
    If preserve is False, uses cp -R (useful for modeless filesystems)
    raises CalledProcessError if copy fails.'''
    logger.debug("copytree %s %s", src, dest)
    cp = ["cp", "-a"] if preserve else ["cp", "-R", "-L"]
    cp += [join(src, "."), os.path.abspath(dest)]
    runcmd(cp)

def do_grafts(grafts, dest, preserve=True):
    '''Copy each of the items listed in grafts into dest.
    If the key ends with '/' it's assumed to be a directory which should be
    created, otherwise just the leading directories will be created.'''
    for imgpath, filename in grafts.items():
        if imgpath[-1] == '/':
            targetdir = join(dest, imgpath)
            imgpath = imgpath[:-1]
        else:
            targetdir = join(dest, dirname(imgpath))
        if not os.path.isdir(targetdir):
            os.makedirs(targetdir)
        if os.path.isdir(filename):
            copytree(filename, join(dest, imgpath), preserve)
        else:
            cpfile(filename, join(dest, imgpath))

def round_to_blocks(size, blocksize):
    '''If size isn't a multiple of blocksize, round up to the next multiple'''
    diff = size % blocksize
    if diff or not size:
        size += blocksize - diff
    return size

# TODO: move filesystem data outside this function
def estimate_size(rootdir, graft=None, fstype=None, blocksize=4096, overhead=256):
    graft = graft or {}
    getsize = lambda f: os.lstat(f).st_size
    if fstype == "btrfs":
        overhead = 64*1024 # don't worry, it's all sparse
    if fstype == "hfsplus":
        overhead = 200 # hack to deal with two bootloader copies
    if fstype in ("vfat", "msdos"):
        blocksize = 2048
        getsize = lambda f: os.stat(f).st_size # no symlinks, count as copies
    total = overhead*blocksize
    dirlist = list(graft.values())
    if rootdir:
        dirlist.append(rootdir)
    for root in dirlist:
        for top, dirs, files in os.walk(root):
            for f in files + dirs:
                total += round_to_blocks(getsize(join(top,f)), blocksize)
        total = max(256*1024*1024, total) # btrfs minimum size: 256MB
    logger.info("Size of %s block %s fs at %s estimated to be %s", blocksize, fstype, rootdir, total)
    return total

######## Execution contexts - use with the 'with' statement ##############

class LoopDev(object):
    def __init__(self, filename, size=None):
        self.loopdev = None
        self.filename = filename
        if size:
            mksparse(self.filename, size)
    def __enter__(self):
        self.loopdev = loop_attach(self.filename)
        return self.loopdev
    def __exit__(self, exc_type, exc_value, tracebk):
        loop_detach(self.loopdev)

class DMDev(object):
    def __init__(self, dev, size, name=None):
        self.mapperdev = None
        (self.dev, self.size, self.name) = (dev, size, name)
        self.mapperdev = dm_attach(self.dev, self.size, self.name)
        return self.mapperdev
        dm_detach(self.mapperdev)

class Mount(object):
    def __init__(self, dev, opts="", mnt=None):
        (self.dev, self.opts, self.mnt) = (dev, opts, mnt)
        self.mnt = mount(self.dev, self.opts, self.mnt)
        return self.mnt
        umount(self.mnt)

class PartitionMount(object):
    """ Mount a partitioned image file using kpartx """
    def __init__(self, disk_img, mount_ok=None, submount=None):
        """
        :param str disk_img: The full path to a partitioned disk image
        :param mount_ok: A function that is passed the mount point and
                         returns True if it should be mounted.
        :param str submount: Directory inside mount_dir to mount at

        If mount_ok is not set it will look for /etc/passwd

        If the partition is found it will be mounted under a temporary
        directory and self.temp_dir set to it. If submount is passed it will be
        created and mounted there instead, with self.mount_dir set to point to
        it. self.mount_dev is set to the loop device, and self.mount_size is
        set to the size of the partition.

        When no subdir is passed self.temp_dir and self.mount_dir will be the same.
        """
        self.mount_dev = None
        self.mount_size = None
        self.mount_dir = None
        self.disk_img = disk_img
        self.mount_ok = mount_ok
        self.submount = submount
        self.temp_dir = None

        # Default is to mount partition with /etc/passwd
        if not self.mount_ok:
            self.mount_ok = lambda mount_dir: os.path.isfile(mount_dir+"/etc/passwd")

        # Example kpartx output
        # kpartx -p p -v -a /tmp/diskV2DiCW.im
        # add map loop2p1 (253:2): 0 3481600 linear /dev/loop2 2048
        # add map loop2p2 (253:3): 0 614400 linear /dev/loop2 3483648
        kpartx_output = runcmd_output(["kpartx", "-v", "-a", "-s", self.disk_img])
        logger.debug(kpartx_output)

        # list of (deviceName, sizeInBytes)
        self.loop_devices = []
        for line in kpartx_output.splitlines():
            # add map loop2p3 (253:4): 0 7139328 linear /dev/loop2 528384
            # 3rd element is size in 512 byte blocks
            if line.startswith("add map "):
                fields = line[8:].split()
                self.loop_devices.append( (fields[0], int(fields[3])*512) )

        # Mount the device selected by mount_ok, if possible
        self.temp_dir = tempfile.mkdtemp()
        if self.submount:
            mount_dir = os.path.normpath(os.path.sep.join([self.temp_dir, self.submount]))
            os.makedirs(mount_dir, mode=0o755, exist_ok=True)
        else:
            mount_dir = self.temp_dir
        for dev, size in self.loop_devices:
            try:
                mount( "/dev/mapper/"+dev, mnt=mount_dir )
                if self.mount_ok(mount_dir):
                    self.mount_dir = mount_dir
                    self.mount_dev = dev
                    self.mount_size = size
                    break
                umount( mount_dir )
                logger.debug(traceback.format_exc())
        if self.mount_dir:
            logger.info("Partition mounted on %s size=%s", self.mount_dir, self.mount_size)
        else:
            logger.debug("Unable to mount anything from %s", self.disk_img)
            os.rmdir(self.temp_dir)
            self.temp_dir = None
        return self

        if self.temp_dir:
            shutil.rmtree(self.temp_dir)
            self.mount_dir = None
        execWithRedirect("kpartx", ["-d", "-s", self.disk_img])

######## Functions for making filesystem images ##########################

def mkfsimage(fstype, rootdir, outfile, size=None, mkfsargs=None, mountargs="", graft=None):
    '''Generic filesystem image creation function.
    fstype should be a filesystem type - "mkfs.${fstype}" must exist.
    graft should be a dict: {"some/path/in/image": "local/file/or/dir"};
      if the path ends with a '/' it's assumed to be a directory.
    Will raise CalledProcessError if something goes wrong.'''
    mkfsargs = mkfsargs or []
    preserve = (fstype not in ("msdos", "vfat"))
    if not size:
        size = estimate_size(rootdir, graft, fstype)
    with LoopDev(outfile, size) as loopdev:
        try:
            runcmd(["mkfs.%s" % fstype] + mkfsargs + [loopdev])
            logger.error("mkfs exited with a non-zero return code: %d", e.returncode)
            logger.error(e.output)
            sys.exit(e.returncode)

        with Mount(loopdev, mountargs) as mnt:
            if rootdir:
                copytree(rootdir, mnt, preserve)
            do_grafts(graft, mnt, preserve)

    # Make absolutely sure that the data has been written
    runcmd(["sync"])

# convenience functions with useful defaults
def mkdosimg(rootdir, outfile, size=None, label="", mountargs="shortname=winnt,umask=0077", graft=None):
    mkfsimage("msdos", rootdir, outfile, size, mountargs=mountargs,
              mkfsargs=["-n", label], graft=graft)

def mkext4img(rootdir, outfile, size=None, label="", mountargs="", graft=None):
    mkfsimage("ext4", rootdir, outfile, size, mountargs=mountargs,
              mkfsargs=["-L", label, "-b", "4096", "-m", "0"], graft=graft)

def mkbtrfsimg(rootdir, outfile, size=None, label="", mountargs="", graft=None):
    mkfsimage("btrfs", rootdir, outfile, size, mountargs=mountargs,
               mkfsargs=["-L", label], graft=graft)

def mkhfsimg(rootdir, outfile, size=None, label="", mountargs="", graft=None):
    mkfsimage("hfsplus", rootdir, outfile, size, mountargs=mountargs,
              mkfsargs=["-v", label], graft=graft)

#
# buildstamp.py
#
#
#
#
#
#

logger = logging.getLogger("pylorax.buildstamp")

import datetime

class BuildStamp(object):

    def __init__(self, product, version, bugurl, isfinal, buildarch):
        self.product = product
        self.version = version
        self.bugurl = bugurl
        self.isfinal = isfinal

        now = datetime.datetime.now()
        now = now.strftime("%Y%m%d%H%M")
        self.uuid = "{0}.{1}".format(now, buildarch)

        try:
        else:

        logger.info("writing .buildstamp file")
            fobj.write("[Main]\n")
            fobj.write("Product={0.product}\n".format(self))
            fobj.write("Version={0.version}\n".format(self))
            fobj.write("BugURL={0.bugurl}\n".format(self))
            fobj.write("IsFinal={0.isfinal}\n".format(self))
            fobj.write("UUID={0.uuid}\n".format(self))
            fobj.write("[Compose]\n")
            fobj.write("Lorax={0}\n".format(vernum))

# monitor.py
#
#
#
#
#
#

import re
import socket
import socketserver
import threading

class LogRequestHandler(socketserver.BaseRequestHandler):
    """
    Handle monitoring and saving the logfiles from the virtual install

    Incoming data is written to self.server.log_path and each line is checked
    for patterns that would indicate that the installation failed.
    self.server.log_error is set True when this happens.
    """
    def setup(self):
        """Start writing to self.server.log_path"""

        if self.server.log_path:
            self.fp = open(self.server.log_path, "w") # pylint: disable=attribute-defined-outside-init
        else:
            self.fp = None
        self.request.settimeout(10)

    def handle(self):
        """
        Write incoming data to a logfile and check for errors

        Split incoming data into lines and check for any Tracebacks or other
        errors that indicate that the install failed.

        Loops until self.server.kill is True
        """
        log.info("Processing logs from %s", self.client_address)
        line = ""
        while True:
            if self.server.kill:
                break

            try:
                data = str(self.request.recv(4096), "utf8")
                if self.fp:
                    self.fp.write(data)
                    self.fp.flush()

                # check the data for errors and set error flag
                # need to assemble it into lines so we can test for the error
                # string.
                while data:
                    more = data.split("\n", 1)
                    line += more[0]
                    if len(more) > 1:
                        self.iserror(line)
                        line = ""
                        data = more[1]
                    else:
                        data = None

            except socket.timeout:
                pass
            except Exception as e:       # pylint: disable=broad-except
                log.info("log processing killed by exception: %s", e)
                break

    def finish(self):
        log.info("Shutting down log processing")
        self.request.close()
        if self.fp:
            self.fp.close()

    def iserror(self, line):
        """
        Check a line to see if it contains an error indicating installation failure

        :param str line: log line to check for failure

        If the line contains IGNORED it will be skipped.
        """
        if "IGNORED" in line:
            return
        simple_tests = ["Traceback (",
                        "Out of memory:",
                        "Call Trace:",
                        "insufficient disk space:",
                        "error populating transaction after",
                        "traceback script(s) have been run",
                        "crashed on signal",
                        "packaging: Missed: NoSuchPackage"]
        re_tests =     [r"packaging: base repo .* not valid",
                        r"packaging: .* requires .*"]
        for t in simple_tests:
            if t in line:
                self.server.log_error = True
                self.server.error_line = line
                return
        for t in re_tests:
            if re.search(t, line):
                return

class LogServer(socketserver.TCPServer):
    """A TCP Server that listens for log data"""

    # Number of seconds to wait for a connection after startup
    timeout = 60

    def __init__(self, log_path, *args, **kwargs):
        """
        Setup the log server

        :param str log_path: Path to the log file to write
        """
        self.kill = False
        self.log_error = False
        self.error_line = ""
        self.log_path = log_path
        self._timeout = kwargs.pop("timeout", None)
        if self._timeout:
            self._start_time = time.time()
        socketserver.TCPServer.__init__(self, *args, **kwargs)

    def log_check(self):
        """
        Check to see if an error has been found in the log

        :returns: True if there has been an error
        :rtype: bool
        """
            taking_too_long = time.time() > self._start_time + (self._timeout * 60)
            if taking_too_long:
                log.error("Canceling installation due to timeout")
        else:
            taking_too_long = False
        return self.log_error or taking_too_long

class LogMonitor(object):
    """
    Setup a server to monitor the logs output by the installation

    This needs to be running before the virt-install runs, it expects
    there to be a listener on the port used for the virtio log port.
    """
    def __init__(self, log_path=None, host="localhost", port=0, timeout=None):
        """
        Start a thread to monitor the logs.

        :param str log_path: Path to the logfile to write
        :param str host: Host to bind to. Default is localhost.
        :param int port: Port to listen to or 0 to pick a port

        If 0 is passed for the port the dynamically assigned port will be
        available as self.port

        If log_path isn't set then it only monitors the logs, instead of
        also writing them to disk.
        """
        self.server = LogServer(log_path, (host, port), LogRequestHandler, timeout=timeout)
        self.host, self.port = self.server.server_address
        self.server_thread = threading.Thread(target=self.server.handle_request)
        self.server_thread.daemon = True
        self.server_thread.start()

    def shutdown(self):
        """Force shutdown of the monitoring thread"""
        self.server.kill = True
        self.server_thread.join()

#
# treeinfo.py
#
#
#
#
#
#

logger = logging.getLogger("pylorax.treeinfo")

class TreeInfo(object):

    def __init__(self, product, version, variant, basearch,
                 packagedir=""):

        self.c = configparser.ConfigParser()

        section = "general"
        data = {"timestamp": str(time.time()),
                "family": product,
                "version": version,
                "name": "%s-%s" % (product, version),
                "variant": variant or "",
                "arch": basearch,
                "packagedir": packagedir}

        self.c.add_section(section)
        list(self.c.set(section, key, value) for key, value in data.items())

    def add_section(self, section, data):
        if not self.c.has_section(section):
            self.c.add_section(section)

        logger.info("writing .treeinfo file")
            self.c.write(fobj)