#!/usr/bin/env python
#
# Live Media Creator
#
# Copyright (C) 2011-2012 Red Hat, Inc.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see .
#
# Author(s): Brian C. Lane
#
import logging
log = logging.getLogger("livemedia-creator")
program_log = logging.getLogger("program")
pylorax_log = logging.getLogger("pylorax")
import os
import sys
import uuid
import tempfile
import subprocess
import socket
import threading
import SocketServer
import libvirt
from time import sleep
import shutil
import traceback
import argparse
# Use pykickstart to calculate disk image size
from pykickstart.parser import KickstartParser
from pykickstart.version import makeVersion
# Use the Lorax treebuilder branch for iso creation
from pylorax.base import DataHolder
from pylorax.treebuilder import TreeBuilder, RuntimeBuilder, udev_escape
from pylorax.sysutils import joinpaths, remove, linktree
from pylorax.imgutils import PartitionMount, mksparse
from pylorax.executils import execWithRedirect, execWithCapture
# Default parameters for rebuilding initramfs, override with --dracut-args
DRACUT_DEFAULT = ["--xz", "--add", "livenet", "--add", "dmsquash-live",
"--add", "convertfs", "--omit", "plymouth"]
class LogRequestHandler(SocketServer.BaseRequestHandler):
"""
Handle monitoring and saving the logfiles from the virtual install
"""
def setup(self):
if self.server.log_path:
self.fp = open(self.server.log_path, "w")
else:
print "no log_path specified"
self.request.settimeout(10)
def handle(self):
"""
Handle writing incoming data to a logfile and
checking the logs for any Tracebacks or other errors that indicate
that the install failed.
"""
line = ""
while True:
if self.server.kill:
break
try:
data = self.request.recv(4096)
self.fp.write(data)
self.fp.flush()
# check the data for errors and set error flag
# need to assemble it into lines so we can test for the error
# string.
while data:
more = data.split("\n", 1)
line += more[0]
if len(more) > 1:
self.iserror(line)
line = ""
data = more[1]
else:
data = None
except socket.timeout:
pass
except:
break
def finish(self):
self.fp.close()
def iserror(self, line):
"""
Check a line to see if it contains an error indicating install failure
"""
simple_tests = [ "Traceback (",
"Out of memory:",
"Call Trace:",
"insufficient disk space:" ]
for t in simple_tests:
if line.find( t ) > -1:
self.server.log_error = True
return
class LogServer(SocketServer.TCPServer):
"""
Add path to logfile
Add log error flag
Add a kill switch
"""
def __init__(self, log_path, *args, **kwargs):
self.kill = False
self.log_error = False
self.log_path = log_path
SocketServer.TCPServer.__init__(self, *args, **kwargs)
def log_check(self):
return self.log_error
class LogMonitor(object):
"""
Contains all the stuff needed to setup a thread to listen to the logs
from the virtual install
"""
def __init__(self, log_path, host="localhost", port=0):
"""
Fire up the thread listening for logs
"""
self.server = LogServer(log_path, (host, port), LogRequestHandler)
self.host, self.port = self.server.server_address
self.log_path = log_path
self.server_thread = threading.Thread(target=self.server.handle_request)
self.server_thread.daemon = True
self.server_thread.start()
def shutdown(self):
self.server.kill = True
self.server_thread.join()
class IsoMountpoint(object):
"""
Mount the iso on a temporary directory and check to make sure the
vmlinuz and initrd.img files exist
Check the iso for a LiveOS directory and set a flag.
Extract the iso's label.
"""
def __init__( self, iso_path ):
self.label = None
self.iso_path = iso_path
self.mount_dir = tempfile.mkdtemp()
if not self.mount_dir:
raise Exception("Error creating temporary directory")
cmd = ["mount","-o","loop",self.iso_path,self.mount_dir]
log.debug( cmd )
execWithRedirect( cmd[0], cmd[1:] )
self.kernel = self.mount_dir+"/isolinux/vmlinuz"
self.initrd = self.mount_dir+"/isolinux/initrd.img"
if os.path.isdir( self.mount_dir+"/repodata" ):
self.repo = self.mount_dir
else:
self.repo = None
self.liveos = os.path.isdir( self.mount_dir+"/LiveOS" )
try:
for f in [self.kernel, self.initrd]:
if not os.path.isfile(f):
raise Exception("Missing file on iso: {0}".format(f))
except:
self.umount()
raise
self.get_iso_label()
def umount( self ):
cmd = ["umount", self.mount_dir]
log.debug( cmd )
execWithRedirect( cmd[0], cmd[1:] )
os.rmdir( self.mount_dir )
def get_iso_label( self ):
"""
Get the iso's label using isoinfo
"""
cmd = [ "isoinfo", "-d", "-i", self.iso_path ]
log.debug( cmd )
isoinfo_output = execWithCapture( cmd[0], cmd[1:] )
log.debug( isoinfo_output )
for line in isoinfo_output.splitlines():
if line.startswith("Volume id: "):
self.label = line[11:]
return
class ImageMount(object):
"""
# kpartx -p p -v -a /tmp/diskV2DiCW.im
# add map loop2p1 (253:2): 0 3481600 linear /dev/loop2 2048
# add map loop2p2 (253:3): 0 614400 linear /dev/loop2 3483648
Find the / partition and only mount that
"""
def __init__(self, disk_img):
"""
Use kpartx to mount an image onto temporary directories
return a list of the dirs
"""
self.disk_img = disk_img
# call kpartx and parse the output
cmd = [ "kpartx", "-v", "-p", "p", "-a", disk_img ]
log.debug( cmd )
kpartx_output = execWithCapture( cmd[0], cmd[1:] )
log.debug( kpartx_output )
self.loop_devices = []
for line in kpartx_output.splitlines():
# add map loop2p3 (253:4): 0 7139328 linear /dev/loop2 528384
# 3rd element is size in 512 byte blocks
if line.startswith("add map "):
fields = line[8:].split()
self.loop_devices.append((fields[0], int(fields[3])*512))
log.debug( self.loop_devices )
# Mount the devices, if possible
self.mount_dir = None
mount_dir = tempfile.mkdtemp()
for dev, size in self.loop_devices:
cmd = ["mount", "/dev/mapper/"+dev, mount_dir]
log.debug( cmd )
try:
execWithRedirect( cmd[0], cmd[1:] )
if os.path.isfile(mount_dir+"/etc/fstab"):
self.mount_dir = mount_dir
self.mount_dev = dev
self.mount_size = size
break
cmd = ["umount", mount_dir]
execWithRedirect( cmd[0], cmd[1:] )
except subprocess.CalledProcessError:
log.debug( traceback.format_exc() )
if self.mount_dir:
log.info( "Found root partition, mounted on {0}".format(self.mount_dir) )
log.info( "size = {0}".format(self.mount_size) )
def umount(self):
"""
unmount the disk image
"""
if self.mount_dir:
cmd = ["umount", self.mount_dir]
log.debug( cmd )
execWithRedirect( cmd[0], cmd[1:] )
os.rmdir(self.mount_dir)
self.mount_dir = None
cmd = ["kpartx", "-d", self.disk_img]
log.debug( cmd )
execWithRedirect( cmd[0], cmd[1:] )
class VirtualInstall( object ):
"""
Run virt-install using an iso and kickstart(s)
"""
def __init__( self, iso, ks_paths, disk_img, img_size=2,
kernel_args=None, memory=1024, vnc=None,
log_check=None, virtio_host="127.0.0.1", virtio_port=6080 ):
"""
iso is an instance of IsoMountpoint
ks_paths is a list of paths to a kickstart files. All are injected, the
first one is the one executed.
disk_img is the path to a disk image (doesn't need to exist)
img_size is the size, in GiB, of the image if it doesn't exist
kernel_args are extra arguments to pass on the kernel cmdline
memory is the amount of ram to assign to the virt
vnc is passed to the --graphics command verbatim
log_check is a method that returns True of the log indicates an error
virtio_host and virtio_port are used to communicate with the log monitor
"""
self.virt_name = "LiveOS-"+str(uuid.uuid4())
# add --graphics none later
# add whatever serial cmds are needed later
cmd = [ "virt-install", "-n", self.virt_name,
"-r", str(memory),
"--noreboot",
"--noautoconsole" ]
cmd.append("--graphics")
if vnc:
cmd.append(vnc)
else:
cmd.append("none")
for ks in ks_paths:
cmd.append("--initrd-inject")
cmd.append(ks)
disk_opts = "path={0}".format(disk_img)
disk_opts += ",format=raw"
if not os.path.isfile(disk_img):
disk_opts += ",size={0}".format(img_size)
cmd.append("--disk")
cmd.append(disk_opts)
if iso.liveos:
disk_opts = "path={0},device=cdrom".format(iso.iso_path)
cmd.append("--disk")
cmd.append(disk_opts)
extra_args = "ks=file:/{0}".format(os.path.basename(ks_paths[0]))
if kernel_args:
extra_args += " "+kernel_args
if not vnc:
extra_args += " console=/dev/ttyS0"
if iso.liveos:
extra_args += " root=live:CDLABEL={0}".format(udev_escape(iso.label))
cmd.append("--extra-args")
cmd.append(extra_args)
cmd.append("--location")
cmd.append(iso.mount_dir)
channel_args = "tcp,host={0}:{1},mode=connect,target_type=virtio" \
",name=org.fedoraproject.anaconda.log.0".format(
virtio_host, virtio_port)
cmd.append("--channel")
cmd.append(channel_args)
log.debug( cmd )
rc = execWithRedirect( cmd[0], cmd[1:] )
if rc:
raise Exception("Problem starting virtual install")
conn = libvirt.openReadOnly(None)
dom = conn.lookupByName(self.virt_name)
# TODO: If vnc has been passed, we should look up the port and print that
# for the user at this point
while dom.isActive() and not log_check():
sys.stdout.write(".")
sys.stdout.flush()
sleep(10)
print
if log_check():
log.info( "Installation error detected. See logfile." )
else:
log.info( "Install finished. Or at least virt shut down." )
def destroy( self ):
"""
Make sure the virt has been shut down and destroyed
Could use libvirt for this instead.
"""
log.info( "Shutting down {0}".format(self.virt_name) )
subprocess.call(["virsh","destroy",self.virt_name])
subprocess.call(["virsh","undefine",self.virt_name])
def anaconda_install( disk_img, disk_size, kickstart, repo, args ):
"""
"""
# Create the sparse image
mksparse( disk_img, disk_size * 1024**3 )
cmd = [ "anaconda", "--image", disk_img, "--kickstart", kickstart,
"--script", "--repo", repo_url ]
cmd += args
log.debug( cmd )
return execWithRedirect( cmd[0], cmd[1:] )
def get_kernels( boot_dir ):
"""
Examine the vmlinuz-* versions and return a list of them
"""
files = os.listdir(boot_dir)
return [f[8:] for f in files if f.startswith("vmlinuz-")]
def make_livecd( disk_img, squashfs_args="", templatedir=None,
title="Linux", project="Linux", releasever=16 ):
"""
Take the content from the disk image and make a livecd out of it
This uses wwood's squashfs live initramfs method:
* put the real / into LiveOS/rootfs.img
* make a squashfs of the LiveOS/rootfs.img tree
* make a simple initramfs with the squashfs.img and /etc/cmdline in it
* make a cpio of that tree
* append the squashfs.cpio to a dracut initramfs for each kernel installed
Then on boot dracut reads /etc/cmdline which points to the squashfs.img
mounts that and then mounts LiveOS/rootfs.img as /
"""
with PartitionMount( disk_img ) as img_mount:
kernel_list = get_kernels( joinpaths( img_mount.mount_dir, "boot" ) )
log.debug( "kernel_list = {0}".format(kernel_list) )
if kernel_list:
kernel_arch = kernel_list[0].split(".")[-1]
else:
kernel_arch = "i386"
log.debug( "kernel_arch = {0}".format(kernel_arch) )
work_dir = tempfile.mkdtemp()
log.info( "working dir is {0}".format(work_dir) )
runtime = "images/install.img"
# This is a mounted image partition, cannot hardlink to it, so just use it
installroot = img_mount.mount_dir
# symlink installroot/images to work_dir/images so we don't run out of space
os.makedirs( joinpaths( work_dir, "images" ) )
# TreeBuilder expects the config files to be in /tmp/config_files
# I think these should be release specific, not from lorax, but for now
configdir = joinpaths(templatedir,"config_files/live/")
configdir_path = "tmp/config_files"
fullpath = joinpaths(installroot, configdir_path)
if os.path.exists(fullpath):
remove(fullpath)
shutil.copytree(configdir, fullpath)
# Fake yum object
fake_yum = DataHolder( conf=DataHolder( installroot=installroot ) )
# Fake arch with only basearch set
arch = DataHolder( basearch=kernel_arch, libdir=None, buildarch=None )
# TODO: Need to get release info from someplace...
product = DataHolder( name=project, version=releasever, release="",
variant="", bugurl="", isfinal=False )
rb = RuntimeBuilder( product, arch, fake_yum )
log.info( "Creating runtime" )
rb.create_runtime( joinpaths( work_dir, runtime ), size=None )
# Link /images to work_dir/images to make the templates happy
if os.path.islink( joinpaths( installroot, "images" ) ):
os.unlink( joinpaths( installroot, "images" ) )
subprocess.check_call(["/bin/ln", "-s", joinpaths( work_dir, "images" ),
joinpaths( installroot, "images" )])
tb = TreeBuilder( product=product, arch=arch,
inroot=installroot, outroot=work_dir,
runtime=runtime, templatedir=templatedir)
log.info( "Rebuilding initrds" )
if not opts.dracut_args:
dracut_args = DRACUT_DEFAULT
else:
dracut_args = []
for arg in opts.dracut_args:
dracut_args += arg.split(" ", 1)
log.info( "dracut args = {0}".format( dracut_args ) )
tb.rebuild_initrds( add_args=dracut_args )
log.info( "Building boot.iso" )
tb.build()
return work_dir
if __name__ == '__main__':
parser = argparse.ArgumentParser( description="Create Live Install Media",
fromfile_prefix_chars="@" )
# These are mutually exclusive, one is required
action = parser.add_mutually_exclusive_group( required=True )
action.add_argument( "--make-iso", action="store_true",
help="Build a live iso" )
action.add_argument( "--make-disk", action="store_true",
help="Build a disk image" )
action.add_argument( "--make-appliance", action="store_true",
help="Build an appliance image and XML description" )
action.add_argument( "--make-ami", action="store_true",
help="Build an ami image" )
parser.add_argument( "--iso", type=os.path.abspath,
help="Anaconda installation .iso path to use for virt-install" )
parser.add_argument( "--disk-image", type=os.path.abspath,
help="Path to disk image to use for creating final image" )
parser.add_argument( "--ks", action="append", type=os.path.abspath,
help="Kickstart file defining the install." )
parser.add_argument( "--image-only", action="store_true",
help="Exit after creating disk image." )
parser.add_argument( "--keep-image", action="store_true",
help="Keep raw disk image after .iso creation" )
parser.add_argument( "--no-virt", action="store_true",
help="Use Anaconda's image install instead of virt-install" )
parser.add_argument( "--proxy",
help="proxy URL to use for the install" )
parser.add_argument( "--anaconda-arg", action="append", dest="anaconda_args",
help="Additional argument to pass to anaconda (no-virt "
"mode). Pass once for each argument" )
parser.add_argument( "--logfile", default="./livemedia.log",
type=os.path.abspath,
help="Path to logfile" )
parser.add_argument( "--lorax-templates", default="/usr/share/lorax/",
type=os.path.abspath,
help="Path to mako templates for lorax" )
parser.add_argument( "--tmp", default="/tmp", type=os.path.abspath,
help="Top level temporary directory" )
parser.add_argument( "--resultdir", default=None, dest="result_dir",
type=os.path.abspath,
help="Directory to copy the resulting images and iso into. "
"Defaults to the temporary working directory")
# Group of arguments to pass to virt-install
virt_group = parser.add_argument_group("virt-install arguments")
virt_group.add_argument("--ram", metavar="MEMORY", default=1024,
help="Memory to allocate for installer in megabytes." )
virt_group.add_argument("--vcpus", default=1,
help="Passed to --vcpus command" )
virt_group.add_argument("--vnc",
help="Passed to --graphics command" )
virt_group.add_argument("--arch",
help="Passed to --arch command" )
virt_group.add_argument( "--kernel-args",
help="Additional argument to pass to the installation kernel" )
# dracut arguments
dracut_group = parser.add_argument_group( "dracut arguments" )
dracut_group.add_argument( "--dracut-arg", action="append", dest="dracut_args",
help="Argument to pass to dracut when "
"rebuilding the initramfs. Pass this "
"once for each argument. NOTE: this "
"overrides the default. (default: %s)" % (DRACUT_DEFAULT,) )
parser.add_argument( "--title", default="Linux Live Media",
help="Substituted for @TITLE@ in bootloader config files" )
parser.add_argument( "--project", default="Linux",
help="substituted for @PROJECT@ in bootloader config files" )
parser.add_argument( "--releasever", type=int, default=16,
help="substituted for @VERSION@ in bootloader config files" )
parser.add_argument( "--squashfs_args",
help="additional squashfs args" )
opts = parser.parse_args()
# Setup logging to console and to logfile
log.setLevel( logging.DEBUG )
pylorax_log.setLevel( logging.DEBUG )
sh = logging.StreamHandler()
sh.setLevel( logging.INFO )
fmt = logging.Formatter("%(asctime)s: %(message)s")
sh.setFormatter( fmt )
log.addHandler( sh )
pylorax_log.addHandler( sh )
fh = logging.FileHandler( filename=opts.logfile, mode="w" )
fh.setLevel( logging.DEBUG )
fmt = logging.Formatter("%(asctime)s %(levelname)s %(name)s: %(message)s")
fh.setFormatter( fmt )
log.addHandler( fh )
pylorax_log.addHandler( fh )
# External program output log
program_log.setLevel(logging.DEBUG)
logfile = os.path.abspath(os.path.dirname(opts.logfile))+"/program.log"
fh = logging.FileHandler( filename=logfile, mode="w")
fh.setLevel( logging.DEBUG )
program_log.addHandler( fh )
log.debug( opts )
if os.getuid() != 0:
log.error("You need to run this as root")
sys.exit( 1 )
if not os.path.exists( opts.lorax_templates ):
log.error( "The lorax templates directory ({0}) doesn't"
" exist.".format( opts.lorax_templates ) )
sys.exit( 1 )
if opts.result_dir and os.path.exists(opts.result_dir):
log.error( "The results_dir ({0}) should not exist, please delete or "
"move its contents".format( opts.result_dir ))
sys.exit( 1 )
if opts.iso and not os.path.exists( opts.iso ):
log.error( "The iso {0} is missing.".format( opts.iso ) )
sys.exit( 1 )
if opts.disk_image and not os.path.exists( opts.disk_image ):
log.error( "The disk image {0} is missing.".format( opts.disk_image ) )
sys.exit( 1 )
if opts.make_appliance:
log.error( "--make-appliance is not yet implemented." )
sys.exit( 1 )
if opts.make_ami:
log.error( "--make-ami is not yet implemented." )
sys.exit( 1 )
if not opts.no_virt and not opts.iso and not opts.disk_image:
log.error( "virt-install needs an install iso." )
sys.exit( 1 )
# Make the disk image
if not opts.disk_image:
# Parse the kickstart to get the partition sizes
ks_version = makeVersion()
ks = KickstartParser( ks_version, errorsAreFatal=False, missingIncludeIsFatal=False )
ks.readKickstart( opts.ks[0] )
disk_size = 1 + (sum( [p.size for p in ks.handler.partition.partitions] ) / 1024)
log.info( "disk_size = {0}GB".format(disk_size) )
if ks.handler.method.method != "url":
log.error( "Only url install method is currently supported. Please "
"fix your kickstart file." )
sys.exit( 1 )
repo_url = ks.handler.method.url
disk_img = tempfile.mktemp( prefix="disk", suffix=".img", dir=opts.tmp )
install_log = os.path.abspath(os.path.dirname(opts.logfile))+"/virt-install.log"
log.info( "disk_img = {0}".format(disk_img) )
log.info( "install_log = {0}".format(install_log) )
if opts.no_virt:
anaconda_args = []
if opts.anaconda_args:
for arg in opts.anaconda_args:
anaconda_args += arg.split(" ", 1)
if opts.proxy:
anaconda_args += [ "--proxy", opts.proxy ]
# Use anaconda's image install
install_error = anaconda_install( disk_img, disk_size, opts.ks[0],
repo_url, anaconda_args )
# Move the anaconda logs over to a log directory
log_dir = os.path.abspath(os.path.dirname(opts.logfile))
log_anaconda = joinpaths( log_dir, "anaconda" )
if not os.path.isdir( log_anaconda ):
os.mkdir( log_anaconda )
for l in ["anaconda.log", "ifcfg.log", "program.log", "storage.log",
"yum.log"]:
if os.path.exists( "/tmp/"+l ):
shutil.copy2( "/tmp/"+l, log_anaconda )
os.unlink( "/tmp/"+l )
else:
iso_mount = IsoMountpoint( opts.iso )
log_monitor = LogMonitor( install_log )
kernel_args = ""
if opts.kernel_args:
kernel_args += opts.kernel_args
if opts.proxy:
kernel_args += " proxy="+opts.proxy
virt = VirtualInstall( iso_mount, opts.ks, disk_img, disk_size,
kernel_args, opts.ram, opts.vnc,
log_check = log_monitor.server.log_check,
virtio_host = log_monitor.host,
virtio_port = log_monitor.port )
virt.destroy()
log_monitor.shutdown()
iso_mount.umount()
install_error = log_monitor.server.log_check()
if install_error:
log.error( "Install failed" )
if not opts.keep_image:
log.info( "Removing bad disk image" )
os.unlink( disk_img )
sys.exit( 1 )
else:
log.info( "Disk Image install successful" )
result_dir = None
if opts.make_iso and not opts.image_only:
result_dir = make_livecd( opts.disk_image or disk_img, opts.squashfs_args,
opts.lorax_templates,
opts.title, opts.project, opts.releasever )
if not opts.keep_image and not opts.disk_image:
os.unlink( disk_img )
if opts.result_dir:
shutil.copytree( result_dir, opts.result_dir )
shutil.rmtree( result_dir )
log.info("SUMMARY")
log.info("-------")
log.info("Logs are in {0}".format(os.path.abspath(os.path.dirname(opts.logfile))))
if opts.keep_image or opts.make_disk:
log.info("Disk image is at {0}".format(disk_img))
else:
log.info("Disk image erased")
if result_dir:
log.info("Results are in {0}".format(opts.result_dir or result_dir))
sys.exit( 0 )