diff --git a/src/pylorax/creator.py b/src/pylorax/creator.py
new file mode 100644
index 00000000..e18e42c3
--- /dev/null
+++ b/src/pylorax/creator.py
@@ -0,0 +1,634 @@
+#
+# Copyright (C) 2011-2018 Red Hat, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see .
+#
+import logging
+log = logging.getLogger("pylorax")
+
+
+import os
+import tempfile
+import subprocess
+import shutil
+import hashlib
+import glob
+import json
+from math import ceil
+
+# Use Mako templates for appliance builder descriptions
+from mako.template import Template
+from mako.exceptions import text_error_template
+
+# Use the Lorax treebuilder branch for iso creation
+from pylorax import ArchData
+from pylorax.base import DataHolder
+from pylorax.executils import execWithRedirect, runcmd
+from pylorax.imgutils import PartitionMount, mkext4img
+from pylorax.imgutils import mount, umount, Mount
+from pylorax.imgutils import mksquashfs, mkrootfsimg
+from pylorax.imgutils import copytree
+from pylorax.installer import novirt_install, virt_install, InstallError
+from pylorax.treebuilder import TreeBuilder, RuntimeBuilder
+from pylorax.treebuilder import findkernels
+from pylorax.sysutils import joinpaths, remove
+
+
+# Default parameters for rebuilding initramfs, override with --dracut-args
+DRACUT_DEFAULT = ["--xz", "--add", "livenet dmsquash-live convertfs pollcdrom qemu qemu-net",
+ "--omit", "plymouth", "--no-hostonly", "--debug", "--no-early-microcode"]
+
+RUNTIME = "images/install.img"
+
+class FakeDNF(object):
+ """
+ A minimal DNF object suitable for passing to RuntimeBuilder
+
+ lmc uses RuntimeBuilder to run the arch specific iso creation
+ templates, so the the installroot config value is the important part of
+ this. Everything else should be a nop.
+ """
+ def __init__(self, conf):
+ self.conf = conf
+
+ def reset(self):
+ pass
+
+def is_image_mounted(disk_img):
+ """
+ Check to see if the disk_img is mounted
+
+ :returns: True if disk_img is in /proc/mounts
+ :rtype: bool
+ """
+ with open("/proc/mounts") as mounts:
+ for mnt in mounts:
+ fields = mnt.split()
+ if len(fields) > 2 and fields[1] == disk_img:
+ return True
+ return False
+
+def find_ostree_root(phys_root):
+ """
+ Find root of ostree deployment
+
+ :param str phys_root: Path to physical root
+ :returns: Relative path of ostree deployment root
+ :rtype: str
+ :raise Exception: More than one deployment roots were found
+ """
+ ostree_root = ""
+ ostree_sysroots = glob.glob(joinpaths(phys_root, "ostree/boot.?/*/*/0"))
+ log.debug("ostree_sysroots = %s", ostree_sysroots)
+ if ostree_sysroots:
+ if len(ostree_sysroots) > 1:
+ raise Exception("Too many deployment roots found: %s" % ostree_sysroots)
+ ostree_root = os.path.relpath(ostree_sysroots[0], phys_root)
+ return ostree_root
+
+def get_arch(mount_dir):
+ """
+ Get the kernel arch
+
+ :returns: Arch of first kernel found at mount_dir/boot/ or i386
+ :rtype: str
+ """
+ kernels = findkernels(mount_dir)
+ if not kernels:
+ return "i386"
+ return kernels[0].arch
+
+def squashfs_args(opts):
+ """ Returns the compression type and args to use when making squashfs
+
+ :param opts: ArgumentParser object with compression and compressopts
+ :returns: tuple of compression type and args
+ :rtype: tuple
+ """
+ compression = opts.compression or "xz"
+ arch = ArchData(opts.arch or os.uname().machine)
+ if compression == "xz" and arch.bcj:
+ compressargs = ["-Xbcj", arch.bcj]
+ else:
+ compressargs = []
+ return (compression, compressargs)
+
+
+def make_appliance(disk_img, name, template, outfile, networks=None, ram=1024,
+ vcpus=1, arch=None, title="Linux", project="Linux",
+ releasever="29"):
+ """
+ Generate an appliance description file
+
+ :param str disk_img: Full path of the disk image
+ :param str name: Name of the appliance, passed to the template
+ :param str template: Full path of Mako template
+ :param str outfile: Full path of file to write, using template
+ :param list networks: List of networks(str) from the kickstart
+ :param int ram: Ram, in MiB, passed to template. Default is 1024
+ :param int vcpus: CPUs, passed to template. Default is 1
+ :param str arch: CPU architecture. Default is 'x86_64'
+ :param str title: Title, passed to template. Default is 'Linux'
+ :param str project: Project, passed to template. Default is 'Linux'
+ :param str releasever: Release version, passed to template. Default is 29
+ """
+ if not (disk_img and template and outfile):
+ return None
+
+ log.info("Creating appliance definition using %s", template)
+
+ if not arch:
+ arch = "x86_64"
+
+ log.info("Calculating SHA256 checksum of %s", disk_img)
+ sha256 = hashlib.sha256()
+ with open(disk_img) as f:
+ while True:
+ data = f.read(1024**2)
+ if not data:
+ break
+ sha256.update(data)
+ log.info("SHA256 of %s is %s", disk_img, sha256.hexdigest())
+ disk_info = DataHolder(name=os.path.basename(disk_img), format="raw",
+ checksum_type="sha256", checksum=sha256.hexdigest())
+ try:
+ result = Template(filename=template).render(disks=[disk_info], name=name,
+ arch=arch, memory=ram, vcpus=vcpus, networks=networks,
+ title=title, project=project, releasever=releasever)
+ except Exception:
+ log.error(text_error_template().render())
+ raise
+
+ with open(outfile, "w") as f:
+ f.write(result)
+
+
+def make_fsimage(diskimage, fsimage, img_size=None, label="Anaconda"):
+ """
+ Copy the / partition of a partitioned disk image to an un-partitioned
+ disk image.
+
+ :param str diskimage: The full path to partitioned disk image with a /
+ :param str fsimage: The full path of the output fs image file
+ :param int img_size: Optional size of the fsimage in MiB or None to make
+ it as small as possible
+ :param str label: The label to apply to the image. Defaults to "Anaconda"
+ """
+ with PartitionMount(diskimage) as img_mount:
+ if not img_mount or not img_mount.mount_dir:
+ return None
+
+ log.info("Creating fsimage %s (%s)", fsimage, img_size or "minimized")
+ if img_size:
+ # convert to Bytes
+ img_size *= 1024**2
+
+ mkext4img(img_mount.mount_dir, fsimage, size=img_size, label=label)
+
+
+def make_runtime(opts, mount_dir, work_dir, size=None):
+ """
+ Make the squashfs image from a directory
+
+ :param opts: options passed to livemedia-creator
+ :type opts: argparse options
+ :param str mount_dir: Directory tree to compress
+ :param str work_dir: Output compressed image to work_dir+images/install.img
+ :param int size: Size of disk image, in GiB
+ """
+ kernel_arch = get_arch(mount_dir)
+
+ # Fake dnf object
+ fake_dbo = FakeDNF(conf=DataHolder(installroot=mount_dir))
+ # Fake arch with only basearch set
+ arch = ArchData(kernel_arch)
+ # TODO: Need to get release info from someplace...
+ product = DataHolder(name=opts.project, version=opts.releasever, release="",
+ variant="", bugurl="", isfinal=False)
+
+ # This is a mounted image partition, cannot hardlink to it, so just use it
+ # symlink mount_dir/images to work_dir/images so we don't run out of space
+ os.makedirs(joinpaths(work_dir, "images"))
+
+ rb = RuntimeBuilder(product, arch, fake_dbo)
+ compression, compressargs = squashfs_args(opts)
+ log.info("Creating runtime")
+ rb.create_runtime(joinpaths(work_dir, RUNTIME), size=size,
+ compression=compression, compressargs=compressargs)
+
+
+def rebuild_initrds_for_live(opts, sys_root_dir, results_dir):
+ """
+ Rebuild intrds for pxe live image (root=live:http://)
+
+ :param opts: options passed to livemedia-creator
+ :type opts: argparse options
+ :param str sys_root_dir: Path to root of the system
+ :param str results_dir: Path of directory for storing results
+ """
+ if not opts.dracut_args:
+ dracut_args = DRACUT_DEFAULT
+ else:
+ dracut_args = []
+ for arg in opts.dracut_args:
+ dracut_args += arg.split(" ", 1)
+ log.info("dracut args = %s", dracut_args)
+
+ dracut = ["dracut", "--nomdadmconf", "--nolvmconf"] + dracut_args
+
+ kdir = "boot"
+ if opts.ostree:
+ kernels_dir = glob.glob(joinpaths(sys_root_dir, "boot/ostree/*"))
+ if kernels_dir:
+ kdir = os.path.relpath(kernels_dir[0], sys_root_dir)
+
+ kernels = [kernel for kernel in findkernels(sys_root_dir, kdir)]
+ if not kernels:
+ raise Exception("No initrds found, cannot rebuild_initrds")
+
+ # Hush some dracut warnings. TODO: bind-mount proc in place?
+ open(joinpaths(sys_root_dir,"/proc/modules"),"w")
+
+ if opts.ostree:
+ # Dracut assumes to have some dirs in disk image
+ # /var/tmp for temp files
+ vartmp_dir = joinpaths(sys_root_dir, "var/tmp")
+ if not os.path.isdir(vartmp_dir):
+ os.mkdir(vartmp_dir)
+ # /root (maybe not fatal)
+ root_dir = joinpaths(sys_root_dir, "var/roothome")
+ if not os.path.isdir(root_dir):
+ os.mkdir(root_dir)
+ # /tmp (maybe not fatal)
+ tmp_dir = joinpaths(sys_root_dir, "sysroot/tmp")
+ if not os.path.isdir(tmp_dir):
+ os.mkdir(tmp_dir)
+
+ # Write the new initramfs directly to the results directory
+ os.mkdir(joinpaths(sys_root_dir, "results"))
+ mount(results_dir, opts="bind", mnt=joinpaths(sys_root_dir, "results"))
+ # Dracut runs out of space inside the minimal rootfs image
+ mount("/var/tmp", opts="bind", mnt=joinpaths(sys_root_dir, "var/tmp"))
+ for kernel in kernels:
+ if hasattr(kernel, "initrd"):
+ outfile = os.path.basename(kernel.initrd.path)
+ else:
+ # Construct an initrd from the kernel name
+ outfile = os.path.basename(kernel.path.replace("vmlinuz-", "initrd-") + ".img")
+ log.info("rebuilding %s", outfile)
+
+ kver = kernel.version
+
+ cmd = dracut + ["/results/"+outfile, kver]
+ runcmd(cmd, root=sys_root_dir)
+
+ shutil.copy2(joinpaths(sys_root_dir, kernel.path), results_dir)
+ umount(joinpaths(sys_root_dir, "var/tmp"), delete=False)
+ umount(joinpaths(sys_root_dir, "results"), delete=False)
+ os.unlink(joinpaths(sys_root_dir,"/proc/modules"))
+
+def create_pxe_config(template, images_dir, live_image_name, add_args = None):
+ """
+ Create template for pxe to live configuration
+
+ :param str images_dir: Path of directory with images to be used
+ :param str live_image_name: Name of live rootfs image file
+ :param list add_args: Arguments to be added to initrd= pxe config
+ """
+
+ add_args = add_args or []
+
+ kernels = [kernel for kernel in findkernels(images_dir, kdir="")
+ if hasattr(kernel, "initrd")]
+ if not kernels:
+ return
+
+ kernel = kernels[0]
+
+ add_args_str = " ".join(add_args)
+
+
+ try:
+ result = Template(filename=template).render(kernel=kernel.path,
+ initrd=kernel.initrd.path, liveimg=live_image_name,
+ addargs=add_args_str)
+ except Exception:
+ log.error(text_error_template().render())
+ raise
+
+ with open (joinpaths(images_dir, "PXE_CONFIG"), "w") as f:
+ f.write(result)
+
+
+def create_vagrant_metadata(path, size=0):
+ """ Create a default Vagrant metadata.json file
+
+ :param str path: Path to metadata.json file
+ :param int size: Disk size in MiB
+ """
+ metadata = { "provider":"libvirt", "format":"qcow2", "virtual_size": ceil(size / 1024) }
+ with open(path, "wt") as f:
+ json.dump(metadata, f, indent=4)
+
+
+def update_vagrant_metadata(path, size):
+ """ Update the Vagrant metadata.json file
+
+ :param str path: Path to metadata.json file
+ :param int size: Disk size in MiB
+
+ This function makes sure that the provider, format and virtual size of the
+ metadata file are set correctly. All other values are left untouched.
+ """
+ with open(path, "rt") as f:
+ try:
+ metadata = json.load(f)
+ except ValueError as e:
+ log.error("Problem reading metadata file %s: %s", path, e)
+ return
+
+ metadata["provider"] = "libvirt"
+ metadata["format"] = "qcow2"
+ metadata["virtual_size"] = ceil(size / 1024)
+ with open(path, "wt") as f:
+ json.dump(metadata, f, indent=4)
+
+
+def make_livecd(opts, mount_dir, work_dir):
+ """
+ Take the content from the disk image and make a livecd out of it
+
+ :param opts: options passed to livemedia-creator
+ :type opts: argparse options
+ :param str mount_dir: Directory tree to compress
+ :param str work_dir: Output compressed image to work_dir+images/install.img
+
+ This uses wwood's squashfs live initramfs method:
+ * put the real / into LiveOS/rootfs.img
+ * make a squashfs of the LiveOS/rootfs.img tree
+ * This is loaded by dracut when the cmdline is passed to the kernel:
+ root=live:CDLABEL= rd.live.image
+ """
+ kernel_arch = get_arch(mount_dir)
+
+ arch = ArchData(kernel_arch)
+ # TODO: Need to get release info from someplace...
+ product = DataHolder(name=opts.project, version=opts.releasever, release="",
+ variant="", bugurl="", isfinal=False)
+
+ # Link /images to work_dir/images to make the templates happy
+ if os.path.islink(joinpaths(mount_dir, "images")):
+ os.unlink(joinpaths(mount_dir, "images"))
+ execWithRedirect("/bin/ln", ["-s", joinpaths(work_dir, "images"),
+ joinpaths(mount_dir, "images")])
+
+ # The templates expect the config files to be in /tmp/config_files
+ # I think these should be release specific, not from lorax, but for now
+ configdir = joinpaths(opts.lorax_templates,"live/config_files/")
+ configdir_path = "tmp/config_files"
+ fullpath = joinpaths(mount_dir, configdir_path)
+ if os.path.exists(fullpath):
+ remove(fullpath)
+ copytree(configdir, fullpath)
+
+ isolabel = opts.volid or "{0.name}-{0.version}-{1.basearch}".format(product, arch)
+ if len(isolabel) > 32:
+ isolabel = isolabel[:32]
+ log.warning("Truncating isolabel to 32 chars: %s", isolabel)
+
+ tb = TreeBuilder(product=product, arch=arch, domacboot=opts.domacboot,
+ inroot=mount_dir, outroot=work_dir,
+ runtime=RUNTIME, isolabel=isolabel,
+ templatedir=joinpaths(opts.lorax_templates,"live/"))
+ log.info("Rebuilding initrds")
+ if not opts.dracut_args:
+ dracut_args = DRACUT_DEFAULT
+ else:
+ dracut_args = []
+ for arg in opts.dracut_args:
+ dracut_args += arg.split(" ", 1)
+ log.info("dracut args = %s", dracut_args)
+ tb.rebuild_initrds(add_args=dracut_args)
+ log.info("Building boot.iso")
+ tb.build()
+
+ return work_dir
+
+def mount_boot_part_over_root(img_mount):
+ """
+ Mount boot partition to /boot of root fs mounted in img_mount
+
+ Used for OSTree so it finds deployment configurations on live rootfs
+
+ param img_mount: object with mounted disk image root partition
+ type img_mount: imgutils.PartitionMount
+ """
+ root_dir = img_mount.mount_dir
+ is_boot_part = lambda dir: os.path.exists(dir+"/loader.0")
+ tmp_mount_dir = tempfile.mkdtemp(prefix="lmc-tmpdir-")
+ sysroot_boot_dir = None
+ for dev, _size in img_mount.loop_devices:
+ if dev is img_mount.mount_dev:
+ continue
+ try:
+ mount("/dev/mapper/"+dev, mnt=tmp_mount_dir)
+ if is_boot_part(tmp_mount_dir):
+ umount(tmp_mount_dir)
+ sysroot_boot_dir = joinpaths(root_dir, "boot")
+ mount("/dev/mapper/"+dev, mnt=sysroot_boot_dir)
+ break
+ else:
+ umount(tmp_mount_dir)
+ except subprocess.CalledProcessError as e:
+ log.debug("Looking for boot partition error: %s", e)
+ remove(tmp_mount_dir)
+ return sysroot_boot_dir
+
+def make_squashfs(opts, disk_img, work_dir):
+ """
+ Create a squashfs image of an unpartitioned filesystem disk image
+
+ :param str disk_img: Path to the unpartitioned filesystem disk image
+ :param str work_dir: Output compressed image to work_dir+images/install.img
+ :param str compression: Compression type to use
+ :returns: True if squashfs creation was successful. False if there was an error.
+ :rtype: bool
+
+ Take disk_img and put it into LiveOS/rootfs.img and squashfs this
+ tree into work_dir+images/install.img
+
+ fsck.ext4 is run on the disk image to make sure there are no errors and to zero
+ out any deleted blocks to make it compress better. If this fails for any reason
+ it will return False and log the error.
+ """
+ # Make sure free blocks are actually zeroed so it will compress
+ rc = execWithRedirect("/usr/sbin/fsck.ext4", ["-y", "-f", "-E", "discard", disk_img])
+ if rc != 0:
+ log.error("Problem zeroing free blocks of %s", disk_img)
+ return False
+
+ liveos_dir = joinpaths(work_dir, "runtime/LiveOS")
+ os.makedirs(liveos_dir)
+ os.makedirs(os.path.dirname(joinpaths(work_dir, RUNTIME)))
+
+ rc = execWithRedirect("/bin/ln", [disk_img, joinpaths(liveos_dir, "rootfs.img")])
+ if rc != 0:
+ shutil.copy2(disk_img, joinpaths(liveos_dir, "rootfs.img"))
+
+ compression, compressargs = squashfs_args(opts)
+ mksquashfs(joinpaths(work_dir, "runtime"),
+ joinpaths(work_dir, RUNTIME), compression, compressargs)
+ remove(joinpaths(work_dir, "runtime"))
+ return True
+
+def calculate_disk_size(opts, ks):
+ """ Calculate the disk size from the kickstart
+
+ :param opts: options passed to livemedia-creator
+ :type opts: argparse options
+ :param str ks: Path to the kickstart to use for the installation
+ :returns: Disk size in MiB
+ :rtype: int
+ """
+ # Disk size for a filesystem image should only be the size of /
+ # to prevent surprises when using the same kickstart for different installations.
+ unique_partitions = dict((p.mountpoint, p) for p in ks.handler.partition.partitions)
+ if opts.no_virt and (opts.make_iso or opts.make_fsimage):
+ disk_size = 2 + sum(p.size for p in unique_partitions.values() if p.mountpoint == "/")
+ else:
+ disk_size = 2 + sum(p.size for p in unique_partitions.values())
+ log.info("Using disk size of %sMiB", disk_size)
+ return disk_size
+
+def make_image(opts, ks):
+ """
+ Install to a disk image
+
+ :param opts: options passed to livemedia-creator
+ :type opts: argparse options
+ :param str ks: Path to the kickstart to use for the installation
+ :returns: Path of the image created
+ :rtype: str
+
+ Use qemu+boot.iso or anaconda to install to a disk image.
+ """
+ if opts.image_name:
+ disk_img = joinpaths(opts.result_dir, opts.image_name)
+ else:
+ disk_img = tempfile.mktemp(prefix="lmc-disk-", suffix=".img", dir=opts.result_dir)
+ log.info("disk_img = %s", disk_img)
+ disk_size = calculate_disk_size(opts, ks)
+ try:
+ if opts.no_virt:
+ novirt_install(opts, disk_img, disk_size)
+ else:
+ install_log = os.path.abspath(os.path.dirname(opts.logfile))+"/virt-install.log"
+ log.info("install_log = %s", install_log)
+
+ virt_install(opts, install_log, disk_img, disk_size)
+ except InstallError as e:
+ log.error("Install failed: %s", e)
+ if not opts.keep_image and os.path.exists(disk_img):
+ log.info("Removing bad disk image")
+ os.unlink(disk_img)
+ raise
+
+ log.info("Disk Image install successful")
+ return disk_img
+
+
+def make_live_images(opts, work_dir, disk_img):
+ """
+ Create live images from direcory or rootfs image
+
+ :param opts: options passed to livemedia-creator
+ :type opts: argparse options
+ :param str work_dir: Directory for storing results
+ :param str disk_img: Path to disk image (fsimage or partitioned)
+ :returns: Path of directory with created images or None
+ :rtype: str
+
+ fsck.ext4 is run on the rootfs_image to make sure there are no errors and to zero
+ out any deleted blocks to make it compress better. If this fails for any reason
+ it will return None and log the error.
+ """
+ sys_root = ""
+
+ squashfs_root_dir = joinpaths(work_dir, "squashfs_root")
+ liveos_dir = joinpaths(squashfs_root_dir, "LiveOS")
+ os.makedirs(liveos_dir)
+ rootfs_img = joinpaths(liveos_dir, "rootfs.img")
+
+ if opts.fs_image or opts.no_virt:
+ # Find the ostree root in the fsimage
+ if opts.ostree:
+ with Mount(disk_img, opts="loop") as mnt_dir:
+ sys_root = find_ostree_root(mnt_dir)
+
+ # Try to hardlink the image, if that fails, copy it
+ rc = execWithRedirect("/bin/ln", [disk_img, rootfs_img])
+ if rc != 0:
+ shutil.copy2(disk_img, rootfs_img)
+ else:
+ is_root_part = None
+ if opts.ostree:
+ is_root_part = lambda dir: os.path.exists(dir+"/ostree/deploy")
+ with PartitionMount(disk_img, mount_ok=is_root_part) as img_mount:
+ if img_mount and img_mount.mount_dir:
+ try:
+ mounted_sysroot_boot_dir = None
+ if opts.ostree:
+ sys_root = find_ostree_root(img_mount.mount_dir)
+ mounted_sysroot_boot_dir = mount_boot_part_over_root(img_mount)
+ if opts.live_rootfs_keep_size:
+ size = img_mount.mount_size / 1024**3
+ else:
+ size = opts.live_rootfs_size or None
+ log.info("Creating live rootfs image")
+ mkrootfsimg(img_mount.mount_dir, rootfs_img, "LiveOS", size=size, sysroot=sys_root)
+ finally:
+ if mounted_sysroot_boot_dir:
+ umount(mounted_sysroot_boot_dir)
+ log.debug("sys_root = %s", sys_root)
+
+ # Make sure free blocks are actually zeroed so it will compress
+ rc = execWithRedirect("/usr/sbin/fsck.ext4", ["-y", "-f", "-E", "discard", rootfs_img])
+ if rc != 0:
+ log.error("Problem zeroing free blocks of %s", disk_img)
+ return None
+
+ log.info("Packing live rootfs image")
+ add_pxe_args = []
+ live_image_name = "live-rootfs.squashfs.img"
+ compression, compressargs = squashfs_args(opts)
+ mksquashfs(squashfs_root_dir, joinpaths(work_dir, live_image_name), compression, compressargs)
+
+ log.info("Rebuilding initramfs for live")
+ with Mount(rootfs_img, opts="loop") as mnt_dir:
+ try:
+ mount(joinpaths(mnt_dir, "boot"), opts="bind", mnt=joinpaths(mnt_dir, sys_root, "boot"))
+ rebuild_initrds_for_live(opts, joinpaths(mnt_dir, sys_root), work_dir)
+ finally:
+ umount(joinpaths(mnt_dir, sys_root, "boot"), delete=False)
+
+ remove(squashfs_root_dir)
+
+ if opts.ostree:
+ add_pxe_args.append("ostree=/%s" % sys_root)
+ template = joinpaths(opts.lorax_templates, "pxe-live/pxe-config.tmpl")
+ create_pxe_config(template, work_dir, live_image_name, add_pxe_args)
+
+ return work_dir
+
+
diff --git a/src/pylorax/installer.py b/src/pylorax/installer.py
new file mode 100644
index 00000000..9dccefcb
--- /dev/null
+++ b/src/pylorax/installer.py
@@ -0,0 +1,569 @@
+#
+# Copyright (C) 2011-2018 Red Hat, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see .
+#
+import logging
+log = logging.getLogger("pylorax")
+
+import os
+import tempfile
+import subprocess
+import shutil
+import glob
+import socket
+
+# Use the Lorax treebuilder branch for iso creation
+from pylorax.creator import create_vagrant_metadata, update_vagrant_metadata, make_fsimage
+from pylorax.executils import execWithRedirect, execReadlines
+from pylorax.imgutils import PartitionMount, mksparse, mkext4img, loop_detach
+from pylorax.imgutils import get_loop_name, dm_detach, mount, umount
+from pylorax.imgutils import mkqemu_img, mktar
+from pylorax.imgutils import mkcpio
+from pylorax.monitor import LogMonitor
+from pylorax.mount import IsoMountpoint
+from pylorax.sysutils import joinpaths
+from pylorax.treebuilder import udev_escape
+
+
+ROOT_PATH = "/mnt/sysimage/"
+
+class InstallError(Exception):
+ pass
+
+
+def find_free_port(start=5900, end=5999, host="127.0.0.1"):
+ """ Return first free port in range.
+
+ :param int start: Starting port number
+ :param int end: Ending port number
+ :param str host: Host IP to search
+ :returns: First free port or -1 if none found
+ :rtype: int
+ """
+ s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+ for port in range(start, end+1):
+ try:
+ s.bind((host, port))
+ s.close()
+ return port
+ except OSError:
+ pass
+
+ return -1
+
+def append_initrd(initrd, files):
+ """ Append files to an initrd.
+
+ :param str initrd: Path to initrd
+ :param list files: list of file paths to add
+ :returns: Path to a new initrd
+ :rtype: str
+
+ The files are added to the initrd by creating a cpio image
+ of the files (stored at /) and writing the cpio to the end of a
+ copy of the initrd.
+
+ The initrd is not changed, a copy is made before appending the
+ cpio archive.
+ """
+ qemu_initrd = tempfile.mktemp(prefix="lmc-initrd-", suffix=".img")
+ shutil.copy2(initrd, qemu_initrd)
+ ks_dir = tempfile.mkdtemp(prefix="lmc-ksdir-")
+ for ks in files:
+ shutil.copy2(ks, ks_dir)
+ ks_initrd = tempfile.mktemp(prefix="lmc-ks-", suffix=".img")
+ mkcpio(ks_dir, ks_initrd)
+ shutil.rmtree(ks_dir)
+ with open(qemu_initrd, "ab") as initrd_fp:
+ with open(ks_initrd, "rb") as ks_fp:
+ while True:
+ data = ks_fp.read(1024**2)
+ if not data:
+ break
+ initrd_fp.write(data)
+ os.unlink(ks_initrd)
+
+ return qemu_initrd
+
+class QEMUInstall(object):
+ """
+ Run qemu using an iso and a kickstart
+ """
+ # Mapping of arch to qemu command
+ QEMU_CMDS = {"x86_64": "qemu-system-x86_64",
+ "i386": "qemu-system-i386",
+ "arm": "qemu-system-arm",
+ "aarch64": "qemu-system-aarch64",
+ "ppc": "qemu-system-ppc",
+ "ppc64": "qemu-system-ppc64"
+ }
+
+ def __init__(self, opts, iso, ks_paths, disk_img, img_size=2048,
+ kernel_args=None, memory=1024, vcpus=None, vnc=None, arch=None,
+ log_check=None, virtio_host="127.0.0.1", virtio_port=6080,
+ image_type=None, boot_uefi=False, ovmf_path=None):
+ """
+ Start the installation
+
+ :param iso: Information about the iso to use for the installation
+ :type iso: IsoMountpoint
+ :param list ks_paths: Paths to kickstart files. All are injected, the
+ first one is the one executed.
+ :param str disk_img: Path to a disk image, created it it doesn't exist
+ :param int img_size: The image size, in MiB, to create if it doesn't exist
+ :param str kernel_args: Extra kernel arguments to pass on the kernel cmdline
+ :param int memory: Amount of RAM to assign to the virt, in MiB
+ :param int vcpus: Number of virtual cpus
+ :param str vnc: Arguments to pass to qemu -display
+ :param str arch: Optional architecture to use in the virt
+ :param log_check: Method that returns True if the installation fails
+ :type log_check: method
+ :param str virtio_host: Hostname to connect virtio log to
+ :param int virtio_port: Port to connect virtio log to
+ :param str image_type: Type of qemu-img disk to create, or None.
+ :param bool boot_uefi: Use OVMF to boot the VM in UEFI mode
+ :param str ovmf_path: Path to the OVMF firmware
+ """
+ # Lookup qemu-system- for arch if passed, or try to guess using host arch
+ qemu_cmd = [self.QEMU_CMDS.get(arch or os.uname().machine, "qemu-system-"+os.uname().machine)]
+ if not os.path.exists("/usr/bin/"+qemu_cmd[0]):
+ raise InstallError("%s does not exist, cannot run qemu" % qemu_cmd[0])
+
+ qemu_cmd += ["-nodefconfig"]
+ qemu_cmd += ["-m", str(memory)]
+ if vcpus:
+ qemu_cmd += ["-smp", str(vcpus)]
+
+ if not opts.no_kvm and os.path.exists("/dev/kvm"):
+ qemu_cmd += ["--machine", "accel=kvm"]
+
+ # Copy the initrd from the iso, create a cpio archive of the kickstart files
+ # and append it to the temporary initrd.
+ qemu_initrd = append_initrd(iso.initrd, ks_paths)
+ qemu_cmd += ["-kernel", iso.kernel]
+ qemu_cmd += ["-initrd", qemu_initrd]
+
+ # Add the disk and cdrom
+ if not os.path.isfile(disk_img):
+ mksparse(disk_img, img_size * 1024**2)
+ drive_args = "file=%s" % disk_img
+ drive_args += ",cache=unsafe,discard=unmap"
+ if image_type:
+ drive_args += ",format=%s" % image_type
+ else:
+ drive_args += ",format=raw"
+ qemu_cmd += ["-drive", drive_args]
+
+ drive_args = "file=%s,media=cdrom,readonly=on" % iso.iso_path
+ qemu_cmd += ["-drive", drive_args]
+
+ # Setup the cmdline args
+ # ======================
+ cmdline_args = "ks=file:/%s" % os.path.basename(ks_paths[0])
+ cmdline_args += " inst.stage2=hd:LABEL=%s" % udev_escape(iso.label)
+ if opts.proxy:
+ cmdline_args += " inst.proxy=%s" % opts.proxy
+ if kernel_args:
+ cmdline_args += " "+kernel_args
+ cmdline_args += " inst.text inst.cmdline"
+
+ qemu_cmd += ["-append", cmdline_args]
+
+ if not opts.vnc:
+ vnc_port = find_free_port()
+ if vnc_port == -1:
+ raise InstallError("No free VNC ports")
+ display_args = "vnc=127.0.0.1:%d" % (vnc_port - 5900)
+ else:
+ display_args = opts.vnc
+ log.info("qemu %s", display_args)
+ qemu_cmd += ["-nographic", "-display", display_args ]
+
+ # Setup the virtio log port
+ qemu_cmd += ["-device", "virtio-serial-pci,id=virtio-serial0"]
+ qemu_cmd += ["-device", "virtserialport,bus=virtio-serial0.0,nr=1,chardev=charchannel0"
+ ",id=channel0,name=org.fedoraproject.anaconda.log.0"]
+ qemu_cmd += ["-chardev", "socket,id=charchannel0,host=%s,port=%s" % (virtio_host, virtio_port)]
+
+ # PAss through rng from host
+ if opts.with_rng != "none":
+ qemu_cmd += ["-object", "rng-random,id=virtio-rng0,filename=%s" % opts.with_rng]
+ qemu_cmd += ["-device", "virtio-rng-pci,rng=virtio-rng0,id=rng0,bus=pci.0,addr=0x9"]
+
+ if boot_uefi and ovmf_path:
+ qemu_cmd += ["-drive", "file=%s/OVMF_CODE.fd,if=pflash,format=raw,unit=0,readonly=on" % ovmf_path]
+
+ # Make a copy of the OVMF_VARS.fd for this run
+ ovmf_vars = tempfile.mktemp(prefix="lmc-OVMF_VARS-", suffix=".fd")
+ shutil.copy2(joinpaths(ovmf_path, "/OVMF_VARS.fd"), ovmf_vars)
+
+ qemu_cmd += ["-drive", "file=%s,if=pflash,format=raw,unit=1" % ovmf_vars]
+
+ log.info("Running qemu")
+ log.debug(qemu_cmd)
+ try:
+ execWithRedirect(qemu_cmd[0], qemu_cmd[1:], reset_lang=False, raise_err=True,
+ callback=lambda p: not log_check())
+ except subprocess.CalledProcessError as e:
+ log.error("Running qemu failed:")
+ log.error("cmd: %s", " ".join(e.cmd))
+ log.error("output: %s", e.output or "")
+ raise InstallError("QEMUInstall failed")
+ except (OSError, KeyboardInterrupt) as e:
+ log.error("Running qemu failed: %s", str(e))
+ raise InstallError("QEMUInstall failed")
+ finally:
+ os.unlink(qemu_initrd)
+ if boot_uefi and ovmf_path:
+ os.unlink(ovmf_vars)
+
+ if log_check():
+ log.error("Installation error detected. See logfile for details.")
+ raise InstallError("QEMUInstall failed")
+ else:
+ log.info("Installation finished without errors.")
+
+
+def novirt_log_check(log_check, proc):
+ """
+ Check to see if there has been an error in the logs
+
+ :param log_check: method to call to check for an error in the logs
+ :param proc: Popen object for the anaconda process
+ :returns: True if the process has been terminated
+
+ The log_check method should return a True if an error has been detected.
+ When an error is detected the process is terminated and this returns True
+ """
+ if log_check():
+ proc.terminate()
+ return True
+ return False
+
+
+def anaconda_cleanup(dirinstall_path):
+ """
+ Cleanup any leftover mounts from anaconda
+
+ :param str dirinstall_path: Path where anaconda mounts things
+ :returns: True if cleanups were successful. False if any of them failed.
+
+ If anaconda crashes it may leave things mounted under this path. It will
+ typically be set to /mnt/sysimage/
+
+ Attempts to cleanup may also fail. Catch these and continue trying the
+ other mountpoints.
+ """
+ rc = True
+ dirinstall_path = os.path.abspath(dirinstall_path)
+ # unmount filesystems
+ for mounted in reversed(open("/proc/mounts").readlines()):
+ (_device, mountpoint, _rest) = mounted.split(" ", 2)
+ if mountpoint.startswith(dirinstall_path) and os.path.ismount(mountpoint):
+ try:
+ umount(mountpoint)
+ except subprocess.CalledProcessError:
+ log.error("Cleanup of %s failed. See program.log for details", mountpoint)
+ rc = False
+ return rc
+
+
+def novirt_install(opts, disk_img, disk_size):
+ """
+ Use Anaconda to install to a disk image
+
+ :param opts: options passed to livemedia-creator
+ :type opts: argparse options
+ :param str disk_img: The full path to the disk image to be created
+ :param int disk_size: The size of the disk_img in MiB
+
+ This method runs anaconda to create the image and then based on the opts
+ passed creates a qemu disk image or tarfile.
+ """
+ dirinstall_path = ROOT_PATH
+
+ # Clean up /tmp/ from previous runs to prevent stale info from being used
+ for path in ["/tmp/yum.repos.d/", "/tmp/yum.cache/"]:
+ if os.path.isdir(path):
+ shutil.rmtree(path)
+
+ args = ["--kickstart", opts.ks[0], "--cmdline"]
+ if opts.anaconda_args:
+ for arg in opts.anaconda_args:
+ args += arg.split(" ", 1)
+ if opts.proxy:
+ args += ["--proxy", opts.proxy]
+ if opts.armplatform:
+ args += ["--armplatform", opts.armplatform]
+
+ if opts.make_iso or opts.make_fsimage or opts.make_pxe_live:
+ # Make a blank fs image
+ args += ["--dirinstall"]
+
+ mkext4img(None, disk_img, label=opts.fs_label, size=disk_size * 1024**2)
+ if not os.path.isdir(dirinstall_path):
+ os.mkdir(dirinstall_path)
+ mount(disk_img, opts="loop", mnt=dirinstall_path)
+ elif opts.make_tar or opts.make_oci:
+ # Install under dirinstall_path, make sure it starts clean
+ if os.path.exists(dirinstall_path):
+ shutil.rmtree(dirinstall_path)
+
+ if opts.make_oci:
+ # OCI installs under /rootfs/
+ dirinstall_path = joinpaths(dirinstall_path, "rootfs")
+ args += ["--dirinstall", dirinstall_path]
+ else:
+ args += ["--dirinstall"]
+
+ os.makedirs(dirinstall_path)
+ else:
+ args += ["--image", disk_img]
+
+ # Create the sparse image
+ mksparse(disk_img, disk_size * 1024**2)
+
+ log_monitor = LogMonitor(timeout=opts.timeout)
+ args += ["--remotelog", "%s:%s" % (log_monitor.host, log_monitor.port)]
+
+ # Make sure anaconda has the right product and release
+ log.info("Running anaconda.")
+ try:
+ for line in execReadlines("anaconda", args, reset_lang=False,
+ env_add={"ANACONDA_PRODUCTNAME": opts.project,
+ "ANACONDA_PRODUCTVERSION": opts.releasever},
+ callback=lambda p: not novirt_log_check(log_monitor.server.log_check, p)):
+ log.info(line)
+
+ # Make sure the new filesystem is correctly labeled
+ setfiles_args = ["-e", "/proc", "-e", "/sys", "-e", "/dev",
+ "/etc/selinux/targeted/contexts/files/file_contexts", "/"]
+
+ # setfiles may not be available, warn instead of fail
+ try:
+ if "--dirinstall" in args:
+ execWithRedirect("setfiles", setfiles_args, root=dirinstall_path)
+ else:
+ with PartitionMount(disk_img) as img_mount:
+ if img_mount and img_mount.mount_dir:
+ execWithRedirect("setfiles", setfiles_args, root=img_mount.mount_dir)
+ except (subprocess.CalledProcessError, OSError) as e:
+ log.warning("Running setfiles on install tree failed: %s", str(e))
+
+ except (subprocess.CalledProcessError, OSError) as e:
+ log.error("Running anaconda failed: %s", e)
+ raise InstallError("novirt_install failed")
+ finally:
+ log_monitor.shutdown()
+
+ # Move the anaconda logs over to a log directory
+ log_dir = os.path.abspath(os.path.dirname(opts.logfile))
+ log_anaconda = joinpaths(log_dir, "anaconda")
+ if not os.path.isdir(log_anaconda):
+ os.mkdir(log_anaconda)
+ for l in glob.glob("/tmp/*log")+glob.glob("/tmp/anaconda-tb-*"):
+ shutil.copy2(l, log_anaconda)
+ os.unlink(l)
+
+ # Make sure any leftover anaconda mounts have been cleaned up
+ if not anaconda_cleanup(dirinstall_path):
+ raise InstallError("novirt_install cleanup of anaconda mounts failed.")
+
+ if not opts.make_iso and not opts.make_fsimage and not opts.make_pxe_live:
+ dm_name = os.path.splitext(os.path.basename(disk_img))[0]
+ dm_path = "/dev/mapper/"+dm_name
+ if os.path.exists(dm_path):
+ dm_detach(dm_path)
+ loop_detach(get_loop_name(disk_img))
+
+ # qemu disk image is used by bare qcow2 images and by Vagrant
+ if opts.image_type:
+ log.info("Converting %s to %s", disk_img, opts.image_type)
+ qemu_args = []
+ for arg in opts.qemu_args:
+ qemu_args += arg.split(" ", 1)
+
+ # convert the image to the selected format
+ if "-O" not in qemu_args:
+ qemu_args.extend(["-O", opts.image_type])
+ qemu_img = tempfile.mktemp(prefix="lmc-disk-", suffix=".img")
+ execWithRedirect("qemu-img", ["convert"] + qemu_args + [disk_img, qemu_img], raise_err=True)
+ if not opts.make_vagrant:
+ execWithRedirect("mv", ["-f", qemu_img, disk_img], raise_err=True)
+ else:
+ # Take the new qcow2 image and package it up for Vagrant
+ compress_args = []
+ for arg in opts.compress_args:
+ compress_args += arg.split(" ", 1)
+
+ vagrant_dir = tempfile.mkdtemp(prefix="lmc-tmpdir-")
+ metadata_path = joinpaths(vagrant_dir, "metadata.json")
+ execWithRedirect("mv", ["-f", qemu_img, joinpaths(vagrant_dir, "box.img")], raise_err=True)
+ if opts.vagrant_metadata:
+ shutil.copy2(opts.vagrant_metadata, metadata_path)
+ else:
+ create_vagrant_metadata(metadata_path)
+ update_vagrant_metadata(metadata_path, disk_size)
+ if opts.vagrantfile:
+ shutil.copy2(opts.vagrantfile, joinpaths(vagrant_dir, "vagrantfile"))
+
+ log.info("Creating Vagrant image")
+ rc = mktar(vagrant_dir, disk_img, opts.compression, compress_args, selinux=False)
+ if rc:
+ raise InstallError("novirt_install mktar failed: rc=%s" % rc)
+ shutil.rmtree(vagrant_dir)
+ elif opts.make_tar:
+ compress_args = []
+ for arg in opts.compress_args:
+ compress_args += arg.split(" ", 1)
+
+ rc = mktar(dirinstall_path, disk_img, opts.compression, compress_args)
+ shutil.rmtree(dirinstall_path)
+
+ if rc:
+ raise InstallError("novirt_install mktar failed: rc=%s" % rc)
+ elif opts.make_oci:
+ # An OCI image places the filesystem under /rootfs/ and adds the json files at the top
+ # And then creates a tar of the whole thing.
+ compress_args = []
+ for arg in opts.compress_args:
+ compress_args += arg.split(" ", 1)
+
+ shutil.copy2(opts.oci_config, ROOT_PATH)
+ shutil.copy2(opts.oci_runtime, ROOT_PATH)
+ rc = mktar(ROOT_PATH, disk_img, opts.compression, compress_args)
+
+ if rc:
+ raise InstallError("novirt_install mktar failed: rc=%s" % rc)
+
+
+def virt_install(opts, install_log, disk_img, disk_size):
+ """
+ Use qemu to install to a disk image
+
+ :param opts: options passed to livemedia-creator
+ :type opts: argparse options
+ :param str install_log: The path to write the log from qemu
+ :param str disk_img: The full path to the disk image to be created
+ :param int disk_size: The size of the disk_img in MiB
+
+ This uses qemu with a boot.iso and a kickstart to create a disk
+ image and then optionally, based on the opts passed, creates tarfile.
+ """
+ iso_mount = IsoMountpoint(opts.iso, opts.location)
+ if not iso_mount.stage2:
+ iso_mount.umount()
+ raise InstallError("ISO is missing stage2, cannot continue")
+
+ log_monitor = LogMonitor(install_log, timeout=opts.timeout)
+
+ kernel_args = ""
+ if opts.kernel_args:
+ kernel_args += opts.kernel_args
+ if opts.proxy:
+ kernel_args += " proxy="+opts.proxy
+
+ if opts.image_type and not opts.make_fsimage:
+ qemu_args = []
+ for arg in opts.qemu_args:
+ qemu_args += arg.split(" ", 1)
+ if "-f" not in qemu_args:
+ qemu_args += ["-f", opts.image_type]
+
+ mkqemu_img(disk_img, disk_size*1024**2, qemu_args)
+
+ if opts.make_fsimage or opts.make_tar or opts.make_oci:
+ diskimg_path = tempfile.mktemp(prefix="lmc-disk-", suffix=".img")
+ else:
+ diskimg_path = disk_img
+
+ try:
+ QEMUInstall(opts, iso_mount, opts.ks, diskimg_path, disk_size,
+ kernel_args, opts.ram, opts.vcpus, opts.vnc, opts.arch,
+ log_check = log_monitor.server.log_check,
+ virtio_host = log_monitor.host,
+ virtio_port = log_monitor.port,
+ image_type=opts.image_type, boot_uefi=opts.virt_uefi,
+ ovmf_path=opts.ovmf_path)
+ log_monitor.shutdown()
+ except InstallError as e:
+ log.error("VirtualInstall failed: %s", e)
+ raise
+ finally:
+ log.info("unmounting the iso")
+ iso_mount.umount()
+
+ if log_monitor.server.log_check():
+ if not log_monitor.server.error_line and opts.timeout:
+ msg = "virt_install failed due to timeout"
+ else:
+ msg = "virt_install failed on line: %s" % log_monitor.server.error_line
+ raise InstallError(msg)
+
+ if opts.make_fsimage:
+ make_fsimage(diskimg_path, disk_img, disk_size, label=opts.fs_label)
+ os.unlink(diskimg_path)
+ elif opts.make_tar:
+ compress_args = []
+ for arg in opts.compress_args:
+ compress_args += arg.split(" ", 1)
+
+ with PartitionMount(diskimg_path) as img_mount:
+ if img_mount and img_mount.mount_dir:
+ rc = mktar(img_mount.mount_dir, disk_img, opts.compression, compress_args)
+ else:
+ rc = 1
+ os.unlink(diskimg_path)
+
+ if rc:
+ raise InstallError("virt_install failed")
+ elif opts.make_oci:
+ # An OCI image places the filesystem under /rootfs/ and adds the json files at the top
+ # And then creates a tar of the whole thing.
+ compress_args = []
+ for arg in opts.compress_args:
+ compress_args += arg.split(" ", 1)
+
+ with PartitionMount(diskimg_path, submount="rootfs") as img_mount:
+ if img_mount and img_mount.temp_dir:
+ shutil.copy2(opts.oci_config, img_mount.temp_dir)
+ shutil.copy2(opts.oci_runtime, img_mount.temp_dir)
+ rc = mktar(img_mount.temp_dir, disk_img, opts.compression, compress_args)
+ else:
+ rc = 1
+ os.unlink(diskimg_path)
+
+ if rc:
+ raise InstallError("virt_install failed")
+ elif opts.make_vagrant:
+ compress_args = []
+ for arg in opts.compress_args:
+ compress_args += arg.split(" ", 1)
+
+ vagrant_dir = tempfile.mkdtemp(prefix="lmc-tmpdir-")
+ metadata_path = joinpaths(vagrant_dir, "metadata.json")
+ execWithRedirect("mv", ["-f", disk_img, joinpaths(vagrant_dir, "box.img")], raise_err=True)
+ if opts.vagrant_metadata:
+ shutil.copy2(opts.vagrant_metadata, metadata_path)
+ else:
+ create_vagrant_metadata(metadata_path)
+ update_vagrant_metadata(metadata_path, disk_size)
+ if opts.vagrantfile:
+ shutil.copy2(opts.vagrantfile, joinpaths(vagrant_dir, "vagrantfile"))
+
+ rc = mktar(vagrant_dir, disk_img, opts.compression, compress_args, selinux=False)
+ if rc:
+ raise InstallError("virt_install failed")
+ shutil.rmtree(vagrant_dir)
diff --git a/src/sbin/livemedia-creator b/src/sbin/livemedia-creator
index 6033713e..d3b90772 100755
--- a/src/sbin/livemedia-creator
+++ b/src/sbin/livemedia-creator
@@ -2,7 +2,7 @@
#
# Live Media Creator
#
-# Copyright (C) 2011-2015 Red Hat, Inc.
+# Copyright (C) 2011-2018 Red Hat, Inc.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
@@ -17,21 +17,14 @@
# You should have received a copy of the GNU General Public License
# along with this program. If not, see .
#
-# Author(s): Brian C. Lane
-#
import logging
log = logging.getLogger("livemedia-creator")
import os
import sys
import tempfile
-import subprocess
import shutil
-import hashlib
import glob
-import json
-from math import ceil
-import socket
import selinux
# Use pykickstart to calculate disk image size
@@ -39,1144 +32,16 @@ from pykickstart.parser import KickstartParser
from pykickstart.version import makeVersion
from pykickstart.constants import KS_SHUTDOWN
-# Use Mako templates for appliance builder descriptions
-from mako.template import Template
-from mako.exceptions import text_error_template
-
# Use the Lorax treebuilder branch for iso creation
-from pylorax import ArchData, setup_logging, find_templates, vernum
-from pylorax.base import DataHolder
-from pylorax.treebuilder import TreeBuilder, RuntimeBuilder, udev_escape
-from pylorax.treebuilder import findkernels
-from pylorax.sysutils import joinpaths, remove
-from pylorax.imgutils import PartitionMount, mksparse, mkext4img, loop_detach
-from pylorax.imgutils import get_loop_name, dm_detach, mount, umount, Mount
-from pylorax.imgutils import mksquashfs, mkqemu_img, mktar, mkrootfsimg
-from pylorax.imgutils import copytree, mkcpio
-from pylorax.executils import execWithRedirect, execReadlines, runcmd
-from pylorax.monitor import LogMonitor
-from pylorax.mount import IsoMountpoint
+from pylorax import setup_logging, find_templates, vernum
from pylorax.cmdline import lmc_parser
-
-# Default parameters for rebuilding initramfs, override with --dracut-args
-DRACUT_DEFAULT = ["--xz", "--add", "livenet dmsquash-live convertfs pollcdrom qemu qemu-net",
- "--omit", "plymouth", "--no-hostonly", "--debug", "--no-early-microcode"]
-
-ROOT_PATH = "/mnt/sysimage/"
-RUNTIME = "images/install.img"
-
-
-class InstallError(Exception):
- pass
-
-
-class FakeDNF(object):
- """
- A minimal DNF object suitable for passing to RuntimeBuilder
-
- lmc uses RuntimeBuilder to run the arch specific iso creation
- templates, so the the installroot config value is the important part of
- this. Everything else should be a nop.
- """
- def __init__(self, conf):
- self.conf = conf
-
- def reset(self):
- pass
-
-
-def find_free_port(start=5900, end=5999, host="127.0.0.1"):
- """ Return first free port in range.
-
- :param int start: Starting port number
- :param int end: Ending port number
- :param str host: Host IP to search
- :returns: First free port or -1 if none found
- :rtype: int
- """
- s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
- for port in range(start, end+1):
- try:
- s.bind((host, port))
- s.close()
- return port
- except OSError:
- pass
-
- return -1
-
-
-def append_initrd(initrd, files):
- """ Append files to an initrd.
-
- :param str initrd: Path to initrd
- :param list files: list of file paths to add
- :returns: Path to a new initrd
- :rtype: str
-
- The files are added to the initrd by creating a cpio image
- of the files (stored at /) and writing the cpio to the end of a
- copy of the initrd.
-
- The initrd is not changed, a copy is made before appending the
- cpio archive.
- """
- qemu_initrd = tempfile.mktemp(prefix="lmc-initrd-", suffix=".img")
- shutil.copy2(initrd, qemu_initrd)
- ks_dir = tempfile.mkdtemp(prefix="lmc-ksdir-")
- for ks in files:
- shutil.copy2(ks, ks_dir)
- ks_initrd = tempfile.mktemp(prefix="lmc-ks-", suffix=".img")
- mkcpio(ks_dir, ks_initrd)
- shutil.rmtree(ks_dir)
- with open(qemu_initrd, "ab") as initrd_fp:
- with open(ks_initrd, "rb") as ks_fp:
- while True:
- data = ks_fp.read(1024**2)
- if not data:
- break
- initrd_fp.write(data)
- os.unlink(ks_initrd)
-
- return qemu_initrd
-
-
-class QEMUInstall(object):
- """
- Run qemu using an iso and a kickstart
- """
- # Mapping of arch to qemu command
- QEMU_CMDS = {"x86_64": "qemu-system-x86_64",
- "i386": "qemu-system-i386",
- "arm": "qemu-system-arm",
- "aarch64": "qemu-system-aarch64",
- "ppc": "qemu-system-ppc",
- "ppc64": "qemu-system-ppc64"
- }
-
- def __init__(self, opts, iso, ks_paths, disk_img, img_size=2048,
- kernel_args=None, memory=1024, vcpus=None, vnc=None, arch=None,
- log_check=None, virtio_host="127.0.0.1", virtio_port=6080,
- image_type=None, boot_uefi=False, ovmf_path=None):
- """
- Start the installation
-
- :param iso: Information about the iso to use for the installation
- :type iso: IsoMountpoint
- :param list ks_paths: Paths to kickstart files. All are injected, the
- first one is the one executed.
- :param str disk_img: Path to a disk image, created it it doesn't exist
- :param int img_size: The image size, in MiB, to create if it doesn't exist
- :param str kernel_args: Extra kernel arguments to pass on the kernel cmdline
- :param int memory: Amount of RAM to assign to the virt, in MiB
- :param int vcpus: Number of virtual cpus
- :param str vnc: Arguments to pass to qemu -display
- :param str arch: Optional architecture to use in the virt
- :param log_check: Method that returns True if the installation fails
- :type log_check: method
- :param str virtio_host: Hostname to connect virtio log to
- :param int virtio_port: Port to connect virtio log to
- :param str image_type: Type of qemu-img disk to create, or None.
- :param bool boot_uefi: Use OVMF to boot the VM in UEFI mode
- :param str ovmf_path: Path to the OVMF firmware
- """
- # Lookup qemu-system- for arch if passed, or try to guess using host arch
- qemu_cmd = [self.QEMU_CMDS.get(arch or os.uname().machine, "qemu-system-"+os.uname().machine)]
- if not os.path.exists("/usr/bin/"+qemu_cmd[0]):
- raise InstallError("%s does not exist, cannot run qemu" % qemu_cmd[0])
-
- qemu_cmd += ["-nodefconfig"]
- qemu_cmd += ["-m", str(memory)]
- if vcpus:
- qemu_cmd += ["-smp", str(vcpus)]
-
- if not opts.no_kvm and os.path.exists("/dev/kvm"):
- qemu_cmd += ["--machine", "accel=kvm"]
-
- # Copy the initrd from the iso, create a cpio archive of the kickstart files
- # and append it to the temporary initrd.
- qemu_initrd = append_initrd(iso.initrd, ks_paths)
- qemu_cmd += ["-kernel", iso.kernel]
- qemu_cmd += ["-initrd", qemu_initrd]
-
- # Add the disk and cdrom
- if not os.path.isfile(disk_img):
- mksparse(disk_img, img_size * 1024**2)
- drive_args = "file=%s" % disk_img
- drive_args += ",cache=unsafe,discard=unmap"
- if image_type:
- drive_args += ",format=%s" % image_type
- else:
- drive_args += ",format=raw"
- qemu_cmd += ["-drive", drive_args]
-
- drive_args = "file=%s,media=cdrom,readonly=on" % iso.iso_path
- qemu_cmd += ["-drive", drive_args]
-
- # Setup the cmdline args
- # ======================
- cmdline_args = "ks=file:/%s" % os.path.basename(ks_paths[0])
- cmdline_args += " inst.stage2=hd:LABEL=%s" % udev_escape(iso.label)
- if opts.proxy:
- cmdline_args += " inst.proxy=%s" % opts.proxy
- if kernel_args:
- cmdline_args += " "+kernel_args
- cmdline_args += " inst.text inst.cmdline"
-
- qemu_cmd += ["-append", cmdline_args]
-
- if not opts.vnc:
- vnc_port = find_free_port()
- if vnc_port == -1:
- raise InstallError("No free VNC ports")
- display_args = "vnc=127.0.0.1:%d" % (vnc_port - 5900)
- else:
- display_args = opts.vnc
- log.info("qemu %s", display_args)
- qemu_cmd += ["-nographic", "-display", display_args ]
-
- # Setup the virtio log port
- qemu_cmd += ["-device", "virtio-serial-pci,id=virtio-serial0"]
- qemu_cmd += ["-device", "virtserialport,bus=virtio-serial0.0,nr=1,chardev=charchannel0"
- ",id=channel0,name=org.fedoraproject.anaconda.log.0"]
- qemu_cmd += ["-chardev", "socket,id=charchannel0,host=%s,port=%s" % (virtio_host, virtio_port)]
-
- # PAss through rng from host
- if opts.with_rng != "none":
- qemu_cmd += ["-object", "rng-random,id=virtio-rng0,filename=%s" % opts.with_rng]
- qemu_cmd += ["-device", "virtio-rng-pci,rng=virtio-rng0,id=rng0,bus=pci.0,addr=0x9"]
-
- if boot_uefi and ovmf_path:
- qemu_cmd += ["-drive", "file=%s/OVMF_CODE.fd,if=pflash,format=raw,unit=0,readonly=on" % ovmf_path]
-
- # Make a copy of the OVMF_VARS.fd for this run
- ovmf_vars = tempfile.mktemp(prefix="lmc-OVMF_VARS-", suffix=".fd")
- shutil.copy2(joinpaths(ovmf_path, "/OVMF_VARS.fd"), ovmf_vars)
-
- qemu_cmd += ["-drive", "file=%s,if=pflash,format=raw,unit=1" % ovmf_vars]
-
- log.info("Running qemu")
- log.debug(qemu_cmd)
- try:
- execWithRedirect(qemu_cmd[0], qemu_cmd[1:], reset_lang=False, raise_err=True,
- callback=lambda p: not log_check())
- except subprocess.CalledProcessError as e:
- log.error("Running qemu failed:")
- log.error("cmd: %s", " ".join(e.cmd))
- log.error("output: %s", e.output or "")
- raise InstallError("QEMUInstall failed")
- except (OSError, KeyboardInterrupt) as e:
- log.error("Running qemu failed: %s", str(e))
- raise InstallError("QEMUInstall failed")
- finally:
- os.unlink(qemu_initrd)
- if boot_uefi and ovmf_path:
- os.unlink(ovmf_vars)
-
- if log_check():
- log.error("Installation error detected. See logfile for details.")
- raise InstallError("QEMUInstall failed")
- else:
- log.info("Installation finished without errors.")
-
-
-def is_image_mounted(disk_img):
- """
- Check to see if the disk_img is mounted
-
- :returns: True if disk_img is in /proc/mounts
- :rtype: bool
- """
- with open("/proc/mounts") as mounts:
- for mnt in mounts:
- fields = mnt.split()
- if len(fields) > 2 and fields[1] == disk_img:
- return True
- return False
-
-def find_ostree_root(phys_root):
- """
- Find root of ostree deployment
-
- :param str phys_root: Path to physical root
- :returns: Relative path of ostree deployment root
- :rtype: str
- :raise Exception: More than one deployment roots were found
- """
- ostree_root = ""
- ostree_sysroots = glob.glob(joinpaths(phys_root, "ostree/boot.?/*/*/0"))
- log.debug("ostree_sysroots = %s", ostree_sysroots)
- if ostree_sysroots:
- if len(ostree_sysroots) > 1:
- raise Exception("Too many deployment roots found: %s" % ostree_sysroots)
- ostree_root = os.path.relpath(ostree_sysroots[0], phys_root)
- return ostree_root
-
-def get_arch(mount_dir):
- """
- Get the kernel arch
-
- :returns: Arch of first kernel found at mount_dir/boot/ or i386
- :rtype: str
- """
- kernels = findkernels(mount_dir)
- if not kernels:
- return "i386"
- return kernels[0].arch
-
-def squashfs_args(opts):
- """ Returns the compression type and args to use when making squashfs
-
- :param opts: ArgumentParser object with compression and compressopts
- :returns: tuple of compression type and args
- :rtype: tuple
- """
- compression = opts.compression or "xz"
- arch = ArchData(opts.arch or os.uname().machine)
- if compression == "xz" and arch.bcj:
- compressargs = ["-Xbcj", arch.bcj]
- else:
- compressargs = []
- return (compression, compressargs)
-
-
-def make_appliance(disk_img, name, template, outfile, networks=None, ram=1024,
- vcpus=1, arch=None, title="Linux", project="Linux",
- releasever="28"):
- """
- Generate an appliance description file
-
- :param str disk_img: Full path of the disk image
- :param str name: Name of the appliance, passed to the template
- :param str template: Full path of Mako template
- :param str outfile: Full path of file to write, using template
- :param list networks: List of networks(str) from the kickstart
- :param int ram: Ram, in MiB, passed to template. Default is 1024
- :param int vcpus: CPUs, passed to template. Default is 1
- :param str arch: CPU architecture. Default is 'x86_64'
- :param str title: Title, passed to template. Default is 'Linux'
- :param str project: Project, passed to template. Default is 'Linux'
- :param str releasever: Release version, passed to template. Default is 28
- """
- if not (disk_img and template and outfile):
- return None
-
- log.info("Creating appliance definition using %s", template)
-
- if not arch:
- arch = "x86_64"
-
- log.info("Calculating SHA256 checksum of %s", disk_img)
- sha256 = hashlib.sha256()
- with open(disk_img) as f:
- while True:
- data = f.read(1024**2)
- if not data:
- break
- sha256.update(data)
- log.info("SHA256 of %s is %s", disk_img, sha256.hexdigest())
- disk_info = DataHolder(name=os.path.basename(disk_img), format="raw",
- checksum_type="sha256", checksum=sha256.hexdigest())
- try:
- result = Template(filename=template).render(disks=[disk_info], name=name,
- arch=arch, memory=ram, vcpus=vcpus, networks=networks,
- title=title, project=project, releasever=releasever)
- except Exception:
- log.error(text_error_template().render())
- raise
-
- with open(outfile, "w") as f:
- f.write(result)
-
-
-def make_fsimage(diskimage, fsimage, img_size=None, label="Anaconda"):
- """
- Copy the / partition of a partitioned disk image to an un-partitioned
- disk image.
-
- :param str diskimage: The full path to partitioned disk image with a /
- :param str fsimage: The full path of the output fs image file
- :param int img_size: Optional size of the fsimage in MiB or None to make
- it as small as possible
- :param str label: The label to apply to the image. Defaults to "Anaconda"
- """
- with PartitionMount(diskimage) as img_mount:
- if not img_mount or not img_mount.mount_dir:
- return None
-
- log.info("Creating fsimage %s (%s)", fsimage, img_size or "minimized")
- if img_size:
- # convert to Bytes
- img_size *= 1024**2
-
- mkext4img(img_mount.mount_dir, fsimage, size=img_size, label=label)
-
-
-def make_runtime(opts, mount_dir, work_dir, size=None):
- """
- Make the squashfs image from a directory
-
- :param opts: options passed to livemedia-creator
- :type opts: argparse options
- :param str mount_dir: Directory tree to compress
- :param str work_dir: Output compressed image to work_dir+images/install.img
- :param int size: Size of disk image, in GiB
- """
- kernel_arch = get_arch(mount_dir)
-
- # Fake dnf object
- fake_dbo = FakeDNF(conf=DataHolder(installroot=mount_dir))
- # Fake arch with only basearch set
- arch = ArchData(kernel_arch)
- # TODO: Need to get release info from someplace...
- product = DataHolder(name=opts.project, version=opts.releasever, release="",
- variant="", bugurl="", isfinal=False)
-
- # This is a mounted image partition, cannot hardlink to it, so just use it
- # symlink mount_dir/images to work_dir/images so we don't run out of space
- os.makedirs(joinpaths(work_dir, "images"))
-
- rb = RuntimeBuilder(product, arch, fake_dbo)
- compression, compressargs = squashfs_args(opts)
- log.info("Creating runtime")
- rb.create_runtime(joinpaths(work_dir, RUNTIME), size=size,
- compression=compression, compressargs=compressargs)
-
-def rebuild_initrds_for_live(opts, sys_root_dir, results_dir):
- """
- Rebuild intrds for pxe live image (root=live:http://)
-
- :param opts: options passed to livemedia-creator
- :type opts: argparse options
- :param str sys_root_dir: Path to root of the system
- :param str results_dir: Path of directory for storing results
- """
- if not opts.dracut_args:
- dracut_args = DRACUT_DEFAULT
- else:
- dracut_args = []
- for arg in opts.dracut_args:
- dracut_args += arg.split(" ", 1)
- log.info("dracut args = %s", dracut_args)
-
- dracut = ["dracut", "--nomdadmconf", "--nolvmconf"] + dracut_args
-
- kdir = "boot"
- if opts.ostree:
- kernels_dir = glob.glob(joinpaths(sys_root_dir, "boot/ostree/*"))
- if kernels_dir:
- kdir = os.path.relpath(kernels_dir[0], sys_root_dir)
-
- kernels = [kernel for kernel in findkernels(sys_root_dir, kdir)]
- if not kernels:
- raise Exception("No initrds found, cannot rebuild_initrds")
-
- # Hush some dracut warnings. TODO: bind-mount proc in place?
- open(joinpaths(sys_root_dir,"/proc/modules"),"w")
-
- if opts.ostree:
- # Dracut assumes to have some dirs in disk image
- # /var/tmp for temp files
- vartmp_dir = joinpaths(sys_root_dir, "var/tmp")
- if not os.path.isdir(vartmp_dir):
- os.mkdir(vartmp_dir)
- # /root (maybe not fatal)
- root_dir = joinpaths(sys_root_dir, "var/roothome")
- if not os.path.isdir(root_dir):
- os.mkdir(root_dir)
- # /tmp (maybe not fatal)
- tmp_dir = joinpaths(sys_root_dir, "sysroot/tmp")
- if not os.path.isdir(tmp_dir):
- os.mkdir(tmp_dir)
-
- # Write the new initramfs directly to the results directory
- os.mkdir(joinpaths(sys_root_dir, "results"))
- mount(results_dir, opts="bind", mnt=joinpaths(sys_root_dir, "results"))
- # Dracut runs out of space inside the minimal rootfs image
- mount("/var/tmp", opts="bind", mnt=joinpaths(sys_root_dir, "var/tmp"))
- for kernel in kernels:
- if hasattr(kernel, "initrd"):
- outfile = os.path.basename(kernel.initrd.path)
- else:
- # Construct an initrd from the kernel name
- outfile = os.path.basename(kernel.path.replace("vmlinuz-", "initrd-") + ".img")
- log.info("rebuilding %s", outfile)
-
- kver = kernel.version
-
- cmd = dracut + ["/results/"+outfile, kver]
- runcmd(cmd, root=sys_root_dir)
-
- shutil.copy2(joinpaths(sys_root_dir, kernel.path), results_dir)
- umount(joinpaths(sys_root_dir, "var/tmp"), delete=False)
- umount(joinpaths(sys_root_dir, "results"), delete=False)
- os.unlink(joinpaths(sys_root_dir,"/proc/modules"))
-
-def create_pxe_config(template, images_dir, live_image_name, add_args = None):
- """
- Create template for pxe to live configuration
-
- :param str images_dir: Path of directory with images to be used
- :param str live_image_name: Name of live rootfs image file
- :param list add_args: Arguments to be added to initrd= pxe config
- """
-
- add_args = add_args or []
-
- kernels = [kernel for kernel in findkernels(images_dir, kdir="")
- if hasattr(kernel, "initrd")]
- if not kernels:
- return
-
- kernel = kernels[0]
-
- add_args_str = " ".join(add_args)
-
-
- try:
- result = Template(filename=template).render(kernel=kernel.path,
- initrd=kernel.initrd.path, liveimg=live_image_name,
- addargs=add_args_str)
- except Exception:
- log.error(text_error_template().render())
- raise
-
- with open (joinpaths(images_dir, "PXE_CONFIG"), "w") as f:
- f.write(result)
-
-
-def create_vagrant_metadata(path, size=0):
- """ Create a default Vagrant metadata.json file
-
- :param str path: Path to metadata.json file
- :param int size: Disk size in MiB
- """
- metadata = { "provider":"libvirt", "format":"qcow2", "virtual_size": ceil(size / 1024) }
- with open(path, "wt") as f:
- json.dump(metadata, f, indent=4)
-
-
-def update_vagrant_metadata(path, size):
- """ Update the Vagrant metadata.json file
-
- :param str path: Path to metadata.json file
- :param int size: Disk size in MiB
-
- This function makes sure that the provider, format and virtual size of the
- metadata file are set correctly. All other values are left untouched.
- """
- with open(path, "rt") as f:
- try:
- metadata = json.load(f)
- except ValueError as e:
- log.error("Problem reading metadata file %s: %s", path, e)
- return
-
- metadata["provider"] = "libvirt"
- metadata["format"] = "qcow2"
- metadata["virtual_size"] = ceil(size / 1024)
- with open(path, "wt") as f:
- json.dump(metadata, f, indent=4)
-
-
-def make_livecd(opts, mount_dir, work_dir):
- """
- Take the content from the disk image and make a livecd out of it
-
- :param opts: options passed to livemedia-creator
- :type opts: argparse options
- :param str mount_dir: Directory tree to compress
- :param str work_dir: Output compressed image to work_dir+images/install.img
-
- This uses wwood's squashfs live initramfs method:
- * put the real / into LiveOS/rootfs.img
- * make a squashfs of the LiveOS/rootfs.img tree
- * This is loaded by dracut when the cmdline is passed to the kernel:
- root=live:CDLABEL= rd.live.image
- """
- kernel_arch = get_arch(mount_dir)
-
- arch = ArchData(kernel_arch)
- # TODO: Need to get release info from someplace...
- product = DataHolder(name=opts.project, version=opts.releasever, release="",
- variant="", bugurl="", isfinal=False)
-
- # Link /images to work_dir/images to make the templates happy
- if os.path.islink(joinpaths(mount_dir, "images")):
- os.unlink(joinpaths(mount_dir, "images"))
- execWithRedirect("/bin/ln", ["-s", joinpaths(work_dir, "images"),
- joinpaths(mount_dir, "images")])
-
- # The templates expect the config files to be in /tmp/config_files
- # I think these should be release specific, not from lorax, but for now
- configdir = joinpaths(opts.lorax_templates,"live/config_files/")
- configdir_path = "tmp/config_files"
- fullpath = joinpaths(mount_dir, configdir_path)
- if os.path.exists(fullpath):
- remove(fullpath)
- copytree(configdir, fullpath)
-
- isolabel = opts.volid or "{0.name}-{0.version}-{1.basearch}".format(product, arch)
- if len(isolabel) > 32:
- isolabel = isolabel[:32]
- log.warning("Truncating isolabel to 32 chars: %s", isolabel)
-
- tb = TreeBuilder(product=product, arch=arch, domacboot=opts.domacboot,
- inroot=mount_dir, outroot=work_dir,
- runtime=RUNTIME, isolabel=isolabel,
- templatedir=joinpaths(opts.lorax_templates,"live/"))
- log.info("Rebuilding initrds")
- if not opts.dracut_args:
- dracut_args = DRACUT_DEFAULT
- else:
- dracut_args = []
- for arg in opts.dracut_args:
- dracut_args += arg.split(" ", 1)
- log.info("dracut args = %s", dracut_args)
- tb.rebuild_initrds(add_args=dracut_args)
- log.info("Building boot.iso")
- tb.build()
-
- return work_dir
-
-def mount_boot_part_over_root(img_mount):
- """
- Mount boot partition to /boot of root fs mounted in img_mount
-
- Used for OSTree so it finds deployment configurations on live rootfs
-
- param img_mount: object with mounted disk image root partition
- type img_mount: imgutils.PartitionMount
- """
- root_dir = img_mount.mount_dir
- is_boot_part = lambda dir: os.path.exists(dir+"/loader.0")
- tmp_mount_dir = tempfile.mkdtemp(prefix="lmc-tmpdir-")
- sysroot_boot_dir = None
- for dev, _size in img_mount.loop_devices:
- if dev is img_mount.mount_dev:
- continue
- try:
- mount("/dev/mapper/"+dev, mnt=tmp_mount_dir)
- if is_boot_part(tmp_mount_dir):
- umount(tmp_mount_dir)
- sysroot_boot_dir = joinpaths(root_dir, "boot")
- mount("/dev/mapper/"+dev, mnt=sysroot_boot_dir)
- break
- else:
- umount(tmp_mount_dir)
- except subprocess.CalledProcessError as e:
- log.debug("Looking for boot partition error: %s", e)
- remove(tmp_mount_dir)
- return sysroot_boot_dir
-
-def novirt_log_check(log_check, proc):
- """
- Check to see if there has been an error in the logs
-
- :param log_check: method to call to check for an error in the logs
- :param proc: Popen object for the anaconda process
- :returns: True if the process has been terminated
-
- The log_check method should return a True if an error has been detected.
- When an error is detected the process is terminated and this returns True
- """
- if log_check():
- proc.terminate()
- return True
- return False
-
-
-def anaconda_cleanup(dirinstall_path):
- """
- Cleanup any leftover mounts from anaconda
-
- :param str dirinstall_path: Path where anaconda mounts things
- :returns: True if cleanups were successful. False if any of them failed.
-
- If anaconda crashes it may leave things mounted under this path. It will
- typically be set to /mnt/sysimage/
-
- Attempts to cleanup may also fail. Catch these and continue trying the
- other mountpoints.
- """
- rc = True
- dirinstall_path = os.path.abspath(dirinstall_path)
- # unmount filesystems
- for mounted in reversed(open("/proc/mounts").readlines()):
- (_device, mountpoint, _rest) = mounted.split(" ", 2)
- if mountpoint.startswith(dirinstall_path) and os.path.ismount(mountpoint):
- try:
- umount(mountpoint)
- except subprocess.CalledProcessError:
- log.error("Cleanup of %s failed. See program.log for details", mountpoint)
- rc = False
- return rc
-
-
-def novirt_install(opts, disk_img, disk_size):
- """
- Use Anaconda to install to a disk image
-
- :param opts: options passed to livemedia-creator
- :type opts: argparse options
- :param str disk_img: The full path to the disk image to be created
- :param int disk_size: The size of the disk_img in MiB
-
- This method runs anaconda to create the image and then based on the opts
- passed creates a qemu disk image or tarfile.
- """
- dirinstall_path = ROOT_PATH
-
- # Clean up /tmp/ from previous runs to prevent stale info from being used
- for path in ["/tmp/yum.repos.d/", "/tmp/yum.cache/"]:
- if os.path.isdir(path):
- shutil.rmtree(path)
-
- args = ["--kickstart", opts.ks[0], "--cmdline"]
- if opts.anaconda_args:
- for arg in opts.anaconda_args:
- args += arg.split(" ", 1)
- if opts.proxy:
- args += ["--proxy", opts.proxy]
- if opts.armplatform:
- args += ["--armplatform", opts.armplatform]
-
- if opts.make_iso or opts.make_fsimage or opts.make_pxe_live:
- # Make a blank fs image
- args += ["--dirinstall"]
-
- mkext4img(None, disk_img, label=opts.fs_label, size=disk_size * 1024**2)
- if not os.path.isdir(dirinstall_path):
- os.mkdir(dirinstall_path)
- mount(disk_img, opts="loop", mnt=dirinstall_path)
- elif opts.make_tar or opts.make_oci:
- # Install under dirinstall_path, make sure it starts clean
- if os.path.exists(dirinstall_path):
- shutil.rmtree(dirinstall_path)
-
- if opts.make_oci:
- # OCI installs under /rootfs/
- dirinstall_path = joinpaths(dirinstall_path, "rootfs")
- args += ["--dirinstall", dirinstall_path]
- else:
- args += ["--dirinstall"]
-
- os.makedirs(dirinstall_path)
- else:
- args += ["--image", disk_img]
-
- # Create the sparse image
- mksparse(disk_img, disk_size * 1024**2)
-
- log_monitor = LogMonitor(timeout=opts.timeout)
- args += ["--remotelog", "%s:%s" % (log_monitor.host, log_monitor.port)]
-
- # Make sure anaconda has the right product and release
- log.info("Running anaconda.")
- try:
- for line in execReadlines("anaconda", args, reset_lang=False,
- env_add={"ANACONDA_PRODUCTNAME": opts.project,
- "ANACONDA_PRODUCTVERSION": opts.releasever},
- callback=lambda p: not novirt_log_check(log_monitor.server.log_check, p)):
- log.info(line)
-
- # Make sure the new filesystem is correctly labeled
- setfiles_args = ["-e", "/proc", "-e", "/sys", "-e", "/dev",
- "/etc/selinux/targeted/contexts/files/file_contexts", "/"]
-
- # setfiles may not be available, warn instead of fail
- try:
- if "--dirinstall" in args:
- execWithRedirect("setfiles", setfiles_args, root=dirinstall_path)
- else:
- with PartitionMount(disk_img) as img_mount:
- if img_mount and img_mount.mount_dir:
- execWithRedirect("setfiles", setfiles_args, root=img_mount.mount_dir)
- except (subprocess.CalledProcessError, OSError) as e:
- log.warning("Running setfiles on install tree failed: %s", str(e))
-
- except (subprocess.CalledProcessError, OSError) as e:
- log.error("Running anaconda failed: %s", e)
- raise InstallError("novirt_install failed")
- finally:
- log_monitor.shutdown()
-
- # Move the anaconda logs over to a log directory
- log_dir = os.path.abspath(os.path.dirname(opts.logfile))
- log_anaconda = joinpaths(log_dir, "anaconda")
- if not os.path.isdir(log_anaconda):
- os.mkdir(log_anaconda)
- for l in glob.glob("/tmp/*log")+glob.glob("/tmp/anaconda-tb-*"):
- shutil.copy2(l, log_anaconda)
- os.unlink(l)
-
- # Make sure any leftover anaconda mounts have been cleaned up
- if not anaconda_cleanup(dirinstall_path):
- raise InstallError("novirt_install cleanup of anaconda mounts failed.")
-
- if not opts.make_iso and not opts.make_fsimage and not opts.make_pxe_live:
- dm_name = os.path.splitext(os.path.basename(disk_img))[0]
- dm_path = "/dev/mapper/"+dm_name
- if os.path.exists(dm_path):
- dm_detach(dm_path)
- loop_detach(get_loop_name(disk_img))
-
- # qemu disk image is used by bare qcow2 images and by Vagrant
- if opts.image_type:
- log.info("Converting %s to %s", disk_img, opts.image_type)
- qemu_args = []
- for arg in opts.qemu_args:
- qemu_args += arg.split(" ", 1)
-
- # convert the image to the selected format
- if "-O" not in qemu_args:
- qemu_args.extend(["-O", opts.image_type])
- qemu_img = tempfile.mktemp(prefix="lmc-disk-", suffix=".img")
- execWithRedirect("qemu-img", ["convert"] + qemu_args + [disk_img, qemu_img], raise_err=True)
- if not opts.make_vagrant:
- execWithRedirect("mv", ["-f", qemu_img, disk_img], raise_err=True)
- else:
- # Take the new qcow2 image and package it up for Vagrant
- compress_args = []
- for arg in opts.compress_args:
- compress_args += arg.split(" ", 1)
-
- vagrant_dir = tempfile.mkdtemp(prefix="lmc-tmpdir-")
- metadata_path = joinpaths(vagrant_dir, "metadata.json")
- execWithRedirect("mv", ["-f", qemu_img, joinpaths(vagrant_dir, "box.img")], raise_err=True)
- if opts.vagrant_metadata:
- shutil.copy2(opts.vagrant_metadata, metadata_path)
- else:
- create_vagrant_metadata(metadata_path)
- update_vagrant_metadata(metadata_path, disk_size)
- if opts.vagrantfile:
- shutil.copy2(opts.vagrantfile, joinpaths(vagrant_dir, "vagrantfile"))
-
- log.info("Creating Vagrant image")
- rc = mktar(vagrant_dir, disk_img, opts.compression, compress_args, selinux=False)
- if rc:
- raise InstallError("novirt_install mktar failed: rc=%s" % rc)
- shutil.rmtree(vagrant_dir)
- elif opts.make_tar:
- compress_args = []
- for arg in opts.compress_args:
- compress_args += arg.split(" ", 1)
-
- rc = mktar(dirinstall_path, disk_img, opts.compression, compress_args)
- shutil.rmtree(dirinstall_path)
-
- if rc:
- raise InstallError("novirt_install mktar failed: rc=%s" % rc)
- elif opts.make_oci:
- # An OCI image places the filesystem under /rootfs/ and adds the json files at the top
- # And then creates a tar of the whole thing.
- compress_args = []
- for arg in opts.compress_args:
- compress_args += arg.split(" ", 1)
-
- shutil.copy2(opts.oci_config, ROOT_PATH)
- shutil.copy2(opts.oci_runtime, ROOT_PATH)
- rc = mktar(ROOT_PATH, disk_img, opts.compression, compress_args)
-
- if rc:
- raise InstallError("novirt_install mktar failed: rc=%s" % rc)
-
-
-def virt_install(opts, install_log, disk_img, disk_size):
- """
- Use qemu to install to a disk image
-
- :param opts: options passed to livemedia-creator
- :type opts: argparse options
- :param str install_log: The path to write the log from qemu
- :param str disk_img: The full path to the disk image to be created
- :param int disk_size: The size of the disk_img in MiB
-
- This uses qemu with a boot.iso and a kickstart to create a disk
- image and then optionally, based on the opts passed, creates tarfile.
- """
- iso_mount = IsoMountpoint(opts.iso, opts.location)
- if not iso_mount.stage2:
- iso_mount.umount()
- raise InstallError("ISO is missing stage2, cannot continue")
-
- log_monitor = LogMonitor(install_log, timeout=opts.timeout)
-
- kernel_args = ""
- if opts.kernel_args:
- kernel_args += opts.kernel_args
- if opts.proxy:
- kernel_args += " proxy="+opts.proxy
-
- if opts.image_type and not opts.make_fsimage:
- qemu_args = []
- for arg in opts.qemu_args:
- qemu_args += arg.split(" ", 1)
- if "-f" not in qemu_args:
- qemu_args += ["-f", opts.image_type]
-
- mkqemu_img(disk_img, disk_size*1024**2, qemu_args)
-
- if opts.make_fsimage or opts.make_tar or opts.make_oci:
- diskimg_path = tempfile.mktemp(prefix="lmc-disk-", suffix=".img")
- else:
- diskimg_path = disk_img
-
- try:
- QEMUInstall(opts, iso_mount, opts.ks, diskimg_path, disk_size,
- kernel_args, opts.ram, opts.vcpus, opts.vnc, opts.arch,
- log_check = log_monitor.server.log_check,
- virtio_host = log_monitor.host,
- virtio_port = log_monitor.port,
- image_type=opts.image_type, boot_uefi=opts.virt_uefi,
- ovmf_path=opts.ovmf_path)
- log_monitor.shutdown()
- except InstallError as e:
- log.error("VirtualInstall failed: %s", e)
- raise
- finally:
- log.info("unmounting the iso")
- iso_mount.umount()
-
- if log_monitor.server.log_check():
- if not log_monitor.server.error_line and opts.timeout:
- msg = "virt_install failed due to timeout"
- else:
- msg = "virt_install failed on line: %s" % log_monitor.server.error_line
- raise InstallError(msg)
-
- if opts.make_fsimage:
- make_fsimage(diskimg_path, disk_img, disk_size, label=opts.fs_label)
- os.unlink(diskimg_path)
- elif opts.make_tar:
- compress_args = []
- for arg in opts.compress_args:
- compress_args += arg.split(" ", 1)
-
- with PartitionMount(diskimg_path) as img_mount:
- if img_mount and img_mount.mount_dir:
- rc = mktar(img_mount.mount_dir, disk_img, opts.compression, compress_args)
- else:
- rc = 1
- os.unlink(diskimg_path)
-
- if rc:
- raise InstallError("virt_install failed")
- elif opts.make_oci:
- # An OCI image places the filesystem under /rootfs/ and adds the json files at the top
- # And then creates a tar of the whole thing.
- compress_args = []
- for arg in opts.compress_args:
- compress_args += arg.split(" ", 1)
-
- with PartitionMount(diskimg_path, submount="rootfs") as img_mount:
- if img_mount and img_mount.temp_dir:
- shutil.copy2(opts.oci_config, img_mount.temp_dir)
- shutil.copy2(opts.oci_runtime, img_mount.temp_dir)
- rc = mktar(img_mount.temp_dir, disk_img, opts.compression, compress_args)
- else:
- rc = 1
- os.unlink(diskimg_path)
-
- if rc:
- raise InstallError("virt_install failed")
- elif opts.make_vagrant:
- compress_args = []
- for arg in opts.compress_args:
- compress_args += arg.split(" ", 1)
-
- vagrant_dir = tempfile.mkdtemp(prefix="lmc-tmpdir-")
- metadata_path = joinpaths(vagrant_dir, "metadata.json")
- execWithRedirect("mv", ["-f", disk_img, joinpaths(vagrant_dir, "box.img")], raise_err=True)
- if opts.vagrant_metadata:
- shutil.copy2(opts.vagrant_metadata, metadata_path)
- else:
- create_vagrant_metadata(metadata_path)
- update_vagrant_metadata(metadata_path, disk_size)
- if opts.vagrantfile:
- shutil.copy2(opts.vagrantfile, joinpaths(vagrant_dir, "vagrantfile"))
-
- rc = mktar(vagrant_dir, disk_img, opts.compression, compress_args, selinux=False)
- if rc:
- raise InstallError("virt_install failed")
- shutil.rmtree(vagrant_dir)
-
-
-def make_squashfs(opts, disk_img, work_dir):
- """
- Create a squashfs image of an unpartitioned filesystem disk image
-
- :param str disk_img: Path to the unpartitioned filesystem disk image
- :param str work_dir: Output compressed image to work_dir+images/install.img
- :param str compression: Compression type to use
- :returns: True if squashfs creation was successful. False if there was an error.
- :rtype: bool
-
- Take disk_img and put it into LiveOS/rootfs.img and squashfs this
- tree into work_dir+images/install.img
-
- fsck.ext4 is run on the disk image to make sure there are no errors and to zero
- out any deleted blocks to make it compress better. If this fails for any reason
- it will return False and log the error.
- """
- # Make sure free blocks are actually zeroed so it will compress
- rc = execWithRedirect("/usr/sbin/fsck.ext4", ["-y", "-f", "-E", "discard", disk_img])
- if rc != 0:
- log.error("Problem zeroing free blocks of %s", disk_img)
- return False
-
- liveos_dir = joinpaths(work_dir, "runtime/LiveOS")
- os.makedirs(liveos_dir)
- os.makedirs(os.path.dirname(joinpaths(work_dir, RUNTIME)))
-
- rc = execWithRedirect("/bin/ln", [disk_img, joinpaths(liveos_dir, "rootfs.img")])
- if rc != 0:
- shutil.copy2(disk_img, joinpaths(liveos_dir, "rootfs.img"))
-
- compression, compressargs = squashfs_args(opts)
- mksquashfs(joinpaths(work_dir, "runtime"),
- joinpaths(work_dir, RUNTIME), compression, compressargs)
- remove(joinpaths(work_dir, "runtime"))
- return True
-
-def calculate_disk_size(opts, ks):
- """ Calculate the disk size from the kickstart
-
- :param opts: options passed to livemedia-creator
- :type opts: argparse options
- :param str ks: Path to the kickstart to use for the installation
- :returns: Disk size in MiB
- :rtype: int
- """
- # Disk size for a filesystem image should only be the size of /
- # to prevent surprises when using the same kickstart for different installations.
- unique_partitions = dict((p.mountpoint, p) for p in ks.handler.partition.partitions)
- if opts.no_virt and (opts.make_iso or opts.make_fsimage):
- disk_size = 2 + sum(p.size for p in unique_partitions.values() if p.mountpoint == "/")
- else:
- disk_size = 2 + sum(p.size for p in unique_partitions.values())
- log.info("Using disk size of %sMiB", disk_size)
- return disk_size
-
-def make_image(opts, ks):
- """
- Install to a disk image
-
- :param opts: options passed to livemedia-creator
- :type opts: argparse options
- :param str ks: Path to the kickstart to use for the installation
- :returns: Path of the image created
- :rtype: str
-
- Use qemu+boot.iso or anaconda to install to a disk image.
- """
- if opts.image_name:
- disk_img = joinpaths(opts.result_dir, opts.image_name)
- else:
- disk_img = tempfile.mktemp(prefix="lmc-disk-", suffix=".img", dir=opts.result_dir)
- log.info("disk_img = %s", disk_img)
- disk_size = calculate_disk_size(opts, ks)
- try:
- if opts.no_virt:
- novirt_install(opts, disk_img, disk_size)
- else:
- install_log = os.path.abspath(os.path.dirname(opts.logfile))+"/virt-install.log"
- log.info("install_log = %s", install_log)
-
- virt_install(opts, install_log, disk_img, disk_size)
- except InstallError as e:
- log.error("Install failed: %s", e)
- if not opts.keep_image and os.path.exists(disk_img):
- log.info("Removing bad disk image")
- os.unlink(disk_img)
- raise
-
- log.info("Disk Image install successful")
- return disk_img
-
-
-def make_live_images(opts, work_dir, disk_img):
- """
- Create live images from direcory or rootfs image
-
- :param opts: options passed to livemedia-creator
- :type opts: argparse options
- :param str work_dir: Directory for storing results
- :param str disk_img: Path to disk image (fsimage or partitioned)
- :returns: Path of directory with created images or None
- :rtype: str
-
- fsck.ext4 is run on the rootfs_image to make sure there are no errors and to zero
- out any deleted blocks to make it compress better. If this fails for any reason
- it will return None and log the error.
- """
- sys_root = ""
-
- squashfs_root_dir = joinpaths(work_dir, "squashfs_root")
- liveos_dir = joinpaths(squashfs_root_dir, "LiveOS")
- os.makedirs(liveos_dir)
- rootfs_img = joinpaths(liveos_dir, "rootfs.img")
-
- if opts.fs_image or opts.no_virt:
- # Find the ostree root in the fsimage
- if opts.ostree:
- with Mount(disk_img, opts="loop") as mnt_dir:
- sys_root = find_ostree_root(mnt_dir)
-
- # Try to hardlink the image, if that fails, copy it
- rc = execWithRedirect("/bin/ln", [disk_img, rootfs_img])
- if rc != 0:
- shutil.copy2(disk_img, rootfs_img)
- else:
- is_root_part = None
- if opts.ostree:
- is_root_part = lambda dir: os.path.exists(dir+"/ostree/deploy")
- with PartitionMount(disk_img, mount_ok=is_root_part) as img_mount:
- if img_mount and img_mount.mount_dir:
- try:
- mounted_sysroot_boot_dir = None
- if opts.ostree:
- sys_root = find_ostree_root(img_mount.mount_dir)
- mounted_sysroot_boot_dir = mount_boot_part_over_root(img_mount)
- if opts.live_rootfs_keep_size:
- size = img_mount.mount_size / 1024**3
- else:
- size = opts.live_rootfs_size or None
- log.info("Creating live rootfs image")
- mkrootfsimg(img_mount.mount_dir, rootfs_img, "LiveOS", size=size, sysroot=sys_root)
- finally:
- if mounted_sysroot_boot_dir:
- umount(mounted_sysroot_boot_dir)
- log.debug("sys_root = %s", sys_root)
-
- # Make sure free blocks are actually zeroed so it will compress
- rc = execWithRedirect("/usr/sbin/fsck.ext4", ["-y", "-f", "-E", "discard", rootfs_img])
- if rc != 0:
- log.error("Problem zeroing free blocks of %s", disk_img)
- return None
-
- log.info("Packing live rootfs image")
- add_pxe_args = []
- live_image_name = "live-rootfs.squashfs.img"
- compression, compressargs = squashfs_args(opts)
- mksquashfs(squashfs_root_dir, joinpaths(work_dir, live_image_name), compression, compressargs)
-
- log.info("Rebuilding initramfs for live")
- with Mount(rootfs_img, opts="loop") as mnt_dir:
- try:
- mount(joinpaths(mnt_dir, "boot"), opts="bind", mnt=joinpaths(mnt_dir, sys_root, "boot"))
- rebuild_initrds_for_live(opts, joinpaths(mnt_dir, sys_root), work_dir)
- finally:
- umount(joinpaths(mnt_dir, sys_root, "boot"), delete=False)
-
- remove(squashfs_root_dir)
-
- if opts.ostree:
- add_pxe_args.append("ostree=/%s" % sys_root)
- template = joinpaths(opts.lorax_templates, "pxe-live/pxe-config.tmpl")
- create_pxe_config(template, work_dir, live_image_name, add_pxe_args)
-
- return work_dir
+from pylorax.creator import make_image, make_squashfs, make_livecd, make_runtime, make_appliance, make_live_images
+from pylorax.creator import calculate_disk_size
+from pylorax.imgutils import PartitionMount
+from pylorax.imgutils import Mount
+from pylorax.imgutils import copytree
+from pylorax.installer import InstallError
+from pylorax.sysutils import joinpaths
def default_image_name(compression, basename):