From 06c227598cc7b1c244d704746254d0515d05cc4d Mon Sep 17 00:00:00 2001 From: "Brian C. Lane" Date: Wed, 2 May 2018 09:55:46 -0700 Subject: [PATCH] Move core of livemedia-creator into pylorax.creator This moves everything except the cmdline checking into run_creator in pylorax.creator It also rearranges some functions to prevent import loops, and adds a utility function to imgutils (mkfsimage_from_disk for copying a partition into a filesystem image). --- src/pylorax/creator.py | 203 ++++++++++++++++++++++++++----------- src/pylorax/imgutils.py | 34 +++++++ src/pylorax/installer.py | 46 +++++++-- src/sbin/livemedia-creator | 152 ++------------------------- 4 files changed, 226 insertions(+), 209 deletions(-) diff --git a/src/pylorax/creator.py b/src/pylorax/creator.py index e18e42c3..6ef781f0 100644 --- a/src/pylorax/creator.py +++ b/src/pylorax/creator.py @@ -17,25 +17,27 @@ import logging log = logging.getLogger("pylorax") - import os import tempfile import subprocess import shutil import hashlib import glob -import json -from math import ceil # Use Mako templates for appliance builder descriptions from mako.template import Template from mako.exceptions import text_error_template +# Use pykickstart to calculate disk image size +from pykickstart.parser import KickstartParser +from pykickstart.constants import KS_SHUTDOWN +from pykickstart.version import makeVersion + # Use the Lorax treebuilder branch for iso creation from pylorax import ArchData from pylorax.base import DataHolder from pylorax.executils import execWithRedirect, runcmd -from pylorax.imgutils import PartitionMount, mkext4img +from pylorax.imgutils import PartitionMount from pylorax.imgutils import mount, umount, Mount from pylorax.imgutils import mksquashfs, mkrootfsimg from pylorax.imgutils import copytree @@ -174,29 +176,6 @@ def make_appliance(disk_img, name, template, outfile, networks=None, ram=1024, f.write(result) -def make_fsimage(diskimage, fsimage, img_size=None, label="Anaconda"): - """ - Copy the / partition of a partitioned disk image to an un-partitioned - disk image. - - :param str diskimage: The full path to partitioned disk image with a / - :param str fsimage: The full path of the output fs image file - :param int img_size: Optional size of the fsimage in MiB or None to make - it as small as possible - :param str label: The label to apply to the image. Defaults to "Anaconda" - """ - with PartitionMount(diskimage) as img_mount: - if not img_mount or not img_mount.mount_dir: - return None - - log.info("Creating fsimage %s (%s)", fsimage, img_size or "minimized") - if img_size: - # convert to Bytes - img_size *= 1024**2 - - mkext4img(img_mount.mount_dir, fsimage, size=img_size, label=label) - - def make_runtime(opts, mount_dir, work_dir, size=None): """ Make the squashfs image from a directory @@ -331,40 +310,6 @@ def create_pxe_config(template, images_dir, live_image_name, add_args = None): f.write(result) -def create_vagrant_metadata(path, size=0): - """ Create a default Vagrant metadata.json file - - :param str path: Path to metadata.json file - :param int size: Disk size in MiB - """ - metadata = { "provider":"libvirt", "format":"qcow2", "virtual_size": ceil(size / 1024) } - with open(path, "wt") as f: - json.dump(metadata, f, indent=4) - - -def update_vagrant_metadata(path, size): - """ Update the Vagrant metadata.json file - - :param str path: Path to metadata.json file - :param int size: Disk size in MiB - - This function makes sure that the provider, format and virtual size of the - metadata file are set correctly. All other values are left untouched. - """ - with open(path, "rt") as f: - try: - metadata = json.load(f) - except ValueError as e: - log.error("Problem reading metadata file %s: %s", path, e) - return - - metadata["provider"] = "libvirt" - metadata["format"] = "qcow2" - metadata["virtual_size"] = ceil(size / 1024) - with open(path, "wt") as f: - json.dump(metadata, f, indent=4) - - def make_livecd(opts, mount_dir, work_dir): """ Take the content from the disk image and make a livecd out of it @@ -631,4 +576,140 @@ def make_live_images(opts, work_dir, disk_img): return work_dir +def run_creator(opts, callback_func=None): + """Run the image creator process + + :param opts: Commandline options to control the process + :type opts: Either a DataHolder or ArgumentParser + :returns: The result directory and the disk image path. + :rtype: Tuple of str + + This function takes the opts arguments and creates the selected output image. + See the cmdline --help for livemedia-creator for the possible options + + (Yes, this is not ideal, but we can fix that later) + """ + result_dir = None + + # Parse the kickstart + if opts.ks: + ks_version = makeVersion() + ks = KickstartParser(ks_version, errorsAreFatal=False, missingIncludeIsFatal=False) + ks.readKickstart(opts.ks[0]) + + # live iso usually needs dracut-live so warn the user if it is missing + if opts.ks and opts.make_iso: + if "dracut-live" not in ks.handler.packages.packageList: + log.error("dracut-live package is missing from the kickstart.") + raise RuntimeError("dracut-live package is missing from the kickstart.") + + # Make the disk or filesystem image + if not opts.disk_image and not opts.fs_image: + if not opts.ks: + raise RuntimeError("Image creation requires a kickstart file") + + errors = [] + if opts.no_virt and ks.handler.method.method not in ("url", "nfs") \ + and not ks.handler.ostreesetup.seen: + errors.append("Only url, nfs and ostreesetup install methods are currently supported." + "Please fix your kickstart file." ) + + if ks.handler.method.method in ("url", "nfs") and not ks.handler.network.seen: + errors.append("The kickstart must activate networking if " + "the url or nfs install method is used.") + + if ks.handler.displaymode.displayMode is not None: + errors.append("The kickstart must not set a display mode (text, cmdline, " + "graphical), this will interfere with livemedia-creator.") + + if opts.make_fsimage or (opts.make_pxe_live and opts.no_virt): + # Make sure the kickstart isn't using autopart and only has a / mountpoint + part_ok = not any(p for p in ks.handler.partition.partitions + if p.mountpoint not in ["/", "swap"]) + if not part_ok or ks.handler.autopart.seen: + errors.append("Filesystem images must use a single / part, not autopart or " + "multiple partitions. swap is allowed but not used.") + + if not opts.no_virt and ks.handler.reboot.action != KS_SHUTDOWN: + errors.append("The kickstart must include shutdown when using virt installation.") + + if errors: + list(log.error(e) for e in errors) + raise RuntimeError("\n".join(errors)) + + # Make the image. Output of this is either a partitioned disk image or a fsimage + try: + disk_img = make_image(opts, ks) + except InstallError as e: + log.error("ERROR: Image creation failed: %s", e) + raise RuntimeError("Image creation failed: %s" % e) + + if opts.image_only: + return (result_dir, disk_img) + + if opts.make_iso: + work_dir = tempfile.mkdtemp(prefix="lmc-work-") + log.info("working dir is %s", work_dir) + + if (opts.fs_image or opts.no_virt) and not opts.disk_image: + # Create iso from a filesystem image + disk_img = opts.fs_image or disk_img + + if not make_squashfs(opts, disk_img, work_dir): + log.error("squashfs.img creation failed") + raise RuntimeError("squashfs.img creation failed") + + with Mount(disk_img, opts="loop") as mount_dir: + result_dir = make_livecd(opts, mount_dir, work_dir) + else: + # Create iso from a partitioned disk image + disk_img = opts.disk_image or disk_img + with PartitionMount(disk_img) as img_mount: + if img_mount and img_mount.mount_dir: + make_runtime(opts, img_mount.mount_dir, work_dir, calculate_disk_size(opts, ks)/1024.0) + result_dir = make_livecd(opts, img_mount.mount_dir, work_dir) + + # --iso-only removes the extra build artifacts, keeping only the boot.iso + if opts.iso_only and result_dir: + boot_iso = joinpaths(result_dir, "images/boot.iso") + if not os.path.exists(boot_iso): + log.error("%s is missing, skipping --iso-only.", boot_iso) + else: + iso_dir = tempfile.mkdtemp(prefix="lmc-result-") + dest_file = joinpaths(iso_dir, opts.iso_name or "boot.iso") + shutil.move(boot_iso, dest_file) + shutil.rmtree(result_dir) + result_dir = iso_dir + + # cleanup the mess + # cleanup work_dir? + if disk_img and not (opts.keep_image or opts.disk_image or opts.fs_image): + os.unlink(disk_img) + log.info("Disk image erased") + disk_img = None + elif opts.make_appliance: + if not opts.ks: + networks = [] + else: + networks = ks.handler.network.network + make_appliance(opts.disk_image or disk_img, opts.app_name, + opts.app_template, opts.app_file, networks, opts.ram, + opts.vcpus or 1, opts.arch, opts.title, opts.project, opts.releasever) + elif opts.make_pxe_live: + work_dir = tempfile.mkdtemp(prefix="lmc-work-") + log.info("working dir is %s", work_dir) + disk_img = opts.fs_image or opts.disk_image or disk_img + log.debug("disk image is %s", disk_img) + + result_dir = make_live_images(opts, work_dir, disk_img) + if result_dir is None: + log.error("Creating PXE live image failed.") + raise RuntimeError("Creating PXE live image failed.") + + if opts.result_dir != opts.tmp and result_dir: + copytree(result_dir, opts.result_dir, preserve=False) + shutil.rmtree(result_dir) + result_dir = None + + return (result_dir, disk_img) diff --git a/src/pylorax/imgutils.py b/src/pylorax/imgutils.py index af534906..a5066e3b 100644 --- a/src/pylorax/imgutils.py +++ b/src/pylorax/imgutils.py @@ -483,3 +483,37 @@ def mkhfsimg(rootdir, outfile, size=None, label="", mountargs="", graft=None): graft = graft or {} mkfsimage("hfsplus", rootdir, outfile, size, mountargs=mountargs, mkfsargs=["-v", label], graft=graft) + +def mkfsimage_from_disk(diskimage, fsimage, img_size=None, label="Anaconda"): + """ + Copy the / partition of a partitioned disk image to an un-partitioned + disk image. + + :param str diskimage: The full path to partitioned disk image with a / + :param str fsimage: The full path of the output fs image file + :param int img_size: Optional size of the fsimage in MiB or None to make + it as small as possible + :param str label: The label to apply to the image. Defaults to "Anaconda" + """ + with PartitionMount(diskimage) as img_mount: + if not img_mount or not img_mount.mount_dir: + return None + + logger.info("Creating fsimage %s (%s)", fsimage, img_size or "minimized") + if img_size: + # convert to Bytes + img_size *= 1024**2 + + mkext4img(img_mount.mount_dir, fsimage, size=img_size, label=label) + +def default_image_name(compression, basename): + """ Return a default image name with the correct suffix for the compression type. + + :param str compression: Compression type + :param str basename: Base filename + :returns: basename with compression suffix + + If the compression is unknown it defaults to xz + """ + SUFFIXES = {"xz": ".xz", "gzip": ".gz", "bzip2": ".bz2", "lzma": ".lzma"} + return basename + SUFFIXES.get(compression, ".xz") diff --git a/src/pylorax/installer.py b/src/pylorax/installer.py index 9dccefcb..1b7d305e 100644 --- a/src/pylorax/installer.py +++ b/src/pylorax/installer.py @@ -17,20 +17,20 @@ import logging log = logging.getLogger("pylorax") +import glob +import json +from math import ceil import os -import tempfile import subprocess import shutil -import glob import socket +import tempfile # Use the Lorax treebuilder branch for iso creation -from pylorax.creator import create_vagrant_metadata, update_vagrant_metadata, make_fsimage from pylorax.executils import execWithRedirect, execReadlines from pylorax.imgutils import PartitionMount, mksparse, mkext4img, loop_detach from pylorax.imgutils import get_loop_name, dm_detach, mount, umount -from pylorax.imgutils import mkqemu_img, mktar -from pylorax.imgutils import mkcpio +from pylorax.imgutils import mkqemu_img, mktar, mkcpio, mkfsimage_from_disk from pylorax.monitor import LogMonitor from pylorax.mount import IsoMountpoint from pylorax.sysutils import joinpaths @@ -43,6 +43,40 @@ class InstallError(Exception): pass +def create_vagrant_metadata(path, size=0): + """ Create a default Vagrant metadata.json file + + :param str path: Path to metadata.json file + :param int size: Disk size in MiB + """ + metadata = { "provider":"libvirt", "format":"qcow2", "virtual_size": ceil(size / 1024) } + with open(path, "wt") as f: + json.dump(metadata, f, indent=4) + + +def update_vagrant_metadata(path, size): + """ Update the Vagrant metadata.json file + + :param str path: Path to metadata.json file + :param int size: Disk size in MiB + + This function makes sure that the provider, format and virtual size of the + metadata file are set correctly. All other values are left untouched. + """ + with open(path, "rt") as f: + try: + metadata = json.load(f) + except ValueError as e: + log.error("Problem reading metadata file %s: %s", path, e) + return + + metadata["provider"] = "libvirt" + metadata["format"] = "qcow2" + metadata["virtual_size"] = ceil(size / 1024) + with open(path, "wt") as f: + json.dump(metadata, f, indent=4) + + def find_free_port(start=5900, end=5999, host="127.0.0.1"): """ Return first free port in range. @@ -513,7 +547,7 @@ def virt_install(opts, install_log, disk_img, disk_size): raise InstallError(msg) if opts.make_fsimage: - make_fsimage(diskimg_path, disk_img, disk_size, label=opts.fs_label) + mkfsimage_from_disk(diskimg_path, disk_img, disk_size, label=opts.fs_label) os.unlink(diskimg_path) elif opts.make_tar: compress_args = [] diff --git a/src/sbin/livemedia-creator b/src/sbin/livemedia-creator index d3b90772..e43436f0 100755 --- a/src/sbin/livemedia-creator +++ b/src/sbin/livemedia-creator @@ -20,43 +20,20 @@ import logging log = logging.getLogger("livemedia-creator") +import glob import os +import selinux import sys import tempfile -import shutil -import glob -import selinux - -# Use pykickstart to calculate disk image size -from pykickstart.parser import KickstartParser -from pykickstart.version import makeVersion -from pykickstart.constants import KS_SHUTDOWN # Use the Lorax treebuilder branch for iso creation from pylorax import setup_logging, find_templates, vernum from pylorax.cmdline import lmc_parser -from pylorax.creator import make_image, make_squashfs, make_livecd, make_runtime, make_appliance, make_live_images -from pylorax.creator import calculate_disk_size -from pylorax.imgutils import PartitionMount -from pylorax.imgutils import Mount -from pylorax.imgutils import copytree -from pylorax.installer import InstallError +from pylorax.creator import run_creator +from pylorax.imgutils import default_image_name from pylorax.sysutils import joinpaths -def default_image_name(compression, basename): - """ Return a default image name with the correct suffix for the compression type. - - :param str compression: Compression type - :param str basename: Base filename - :returns: basename with compression suffix - - If the compression is unknown it defaults to xz - """ - SUFFIXES = {"xz": ".xz", "gzip": ".gz", "bzip2": ".bz2", "lzma": ".lzma"} - return basename + SUFFIXES.get(compression, ".xz") - - def main(): parser = lmc_parser() opts = parser.parse_args() @@ -213,121 +190,12 @@ def main(): tempfile.tempdir = opts.tmp disk_img = None - # Parse the kickstart - if opts.ks: - ks_version = makeVersion() - ks = KickstartParser(ks_version, errorsAreFatal=False, missingIncludeIsFatal=False) - ks.readKickstart(opts.ks[0]) - - # live iso usually needs dracut-live so warn the user if it is missing - if opts.ks and opts.make_iso: - if "dracut-live" not in ks.handler.packages.packageList: - log.error("dracut-live package is missing from the kickstart.") - sys.exit(1) - - # Make the disk or filesystem image - if not opts.disk_image and not opts.fs_image: - errors = [] - if opts.no_virt and ks.handler.method.method not in ("url", "nfs") \ - and not ks.handler.ostreesetup.seen: - errors.append("Only url, nfs and ostreesetup install methods are currently supported." - "Please fix your kickstart file." ) - - if ks.handler.method.method in ("url", "nfs") and not ks.handler.network.seen: - errors.append("The kickstart must activate networking if " - "the url or nfs install method is used.") - - if ks.handler.displaymode.displayMode is not None: - errors.append("The kickstart must not set a display mode (text, cmdline, " - "graphical), this will interfere with livemedia-creator.") - - if opts.make_fsimage or (opts.make_pxe_live and opts.no_virt): - # Make sure the kickstart isn't using autopart and only has a / mountpoint - part_ok = not any(p for p in ks.handler.partition.partitions - if p.mountpoint not in ["/", "swap"]) - if not part_ok or ks.handler.autopart.seen: - errors.append("Filesystem images must use a single / part, not autopart or " - "multiple partitions. swap is allowed but not used.") - - if not opts.no_virt and ks.handler.reboot.action != KS_SHUTDOWN: - errors.append("The kickstart must include shutdown when using virt installation.") - - if errors: - list(log.error(e) for e in errors) - sys.exit(1) - - # Make the image. Output of this is either a partitioned disk image or a fsimage - try: - disk_img = make_image(opts, ks) - except InstallError as e: - log.error("ERROR: Image creation failed: %s", e) - sys.exit(1) - - result_dir = None - if not opts.image_only: - if opts.make_iso: - work_dir = tempfile.mkdtemp(prefix="lmc-work-") - log.info("working dir is %s", work_dir) - - if (opts.fs_image or opts.no_virt) and not opts.disk_image: - # Create iso from a filesystem image - disk_img = opts.fs_image or disk_img - - if not make_squashfs(opts, disk_img, work_dir): - log.error("squashfs.img creation failed") - sys.exit(1) - - with Mount(disk_img, opts="loop") as mount_dir: - result_dir = make_livecd(opts, mount_dir, work_dir) - else: - # Create iso from a partitioned disk image - disk_img = opts.disk_image or disk_img - with PartitionMount(disk_img) as img_mount: - if img_mount and img_mount.mount_dir: - make_runtime(opts, img_mount.mount_dir, work_dir, calculate_disk_size(opts, ks)/1024.0) - result_dir = make_livecd(opts, img_mount.mount_dir, work_dir) - - # --iso-only removes the extra build artifacts, keeping only the boot.iso - if opts.iso_only and result_dir: - boot_iso = joinpaths(result_dir, "images/boot.iso") - if not os.path.exists(boot_iso): - log.error("%s is missing, skipping --iso-only.", boot_iso) - else: - iso_dir = tempfile.mkdtemp(prefix="lmc-result-") - dest_file = joinpaths(iso_dir, opts.iso_name or "boot.iso") - shutil.move(boot_iso, dest_file) - shutil.rmtree(result_dir) - result_dir = iso_dir - - # cleanup the mess - # cleanup work_dir? - if disk_img and not (opts.keep_image or opts.disk_image or opts.fs_image): - os.unlink(disk_img) - log.info("Disk image erased") - disk_img = None - elif opts.make_appliance: - if not opts.ks: - networks = [] - else: - networks = ks.handler.network.network - make_appliance(opts.disk_image or disk_img, opts.app_name, - opts.app_template, opts.app_file, networks, opts.ram, - opts.vcpus or 1, opts.arch, opts.title, opts.project, opts.releasever) - elif opts.make_pxe_live: - work_dir = tempfile.mkdtemp(prefix="lmc-work-") - log.info("working dir is %s", work_dir) - disk_img = opts.fs_image or opts.disk_image or disk_img - log.debug("disk image is %s", disk_img) - - result_dir = make_live_images(opts, work_dir, disk_img) - if result_dir is None: - log.error("Creating PXE live image failed.") - sys.exit(1) - - if opts.result_dir != opts.tmp and result_dir: - copytree(result_dir, opts.result_dir, preserve=False) - shutil.rmtree(result_dir) - result_dir = None + try: + # TODO - Better API than passing in opts + (result_dir, disk_img) = run_creator(opts) + except Exception as e: # pylint: disable=broad-except + log.error(str(e)) + sys.exit(1) log.info("SUMMARY") log.info("-------")