From d04a99e8f47b0615a9eb7569988508bcbfae9671 Mon Sep 17 00:00:00 2001 From: "Brian C. Lane" Date: Fri, 4 Apr 2014 14:38:51 -0700 Subject: [PATCH] livemedia-creator: Add support for making tarfiles This adds the --make-tar option which will produce a xz compressed tar of the root filesystem. This works with either virt-install or no-virt modes. Use --image-name to set the output filename. --compression is used to set the compression type to use, which defaults to xz. Supported types are xz, lzma, gzip and bzip2. --compress-arg is used to pass arguments to the compression utility. --- README.livemedia-creator | 15 +++++++++++ docs/livemedia-creator.1 | 17 +++++++++++-- src/pylorax/imgutils.py | 49 +++++++++++++++++++++++++----------- src/sbin/livemedia-creator | 51 +++++++++++++++++++++++++++++++++++--- 4 files changed, 111 insertions(+), 21 deletions(-) diff --git a/README.livemedia-creator b/README.livemedia-creator index b9422216..419dce3a 100644 --- a/README.livemedia-creator +++ b/README.livemedia-creator @@ -242,6 +242,21 @@ livemedia-creator --make-fsimage --iso=/path/to/boot.iso --ks=./docs/fedora-mini You can name the output image with --image-name and set a label on the filesystem with --fs-label +TAR FILE CREATION +----------------- +The --make-tar command can be used to create a tar of the root filesystem. By +default it is compressed using xz, but this can be changed using the +--compression and --compress-arg options. This option works with both virt and +--no-virt install methods. + +As with --make-fsimage the kickstart should be limited to a single / partition. + +eg. + +livemedia-creator --make-tar --iso=/path/to/boot.iso --ks=./docs/fedora-minimal.ks \ +--image-name=fedora-root.tar.xz + + DEBUGGING PROBLEMS ------------------ Cleaning up an aborted (ctrl-c) virt-install run (as root): diff --git a/docs/livemedia-creator.1 b/docs/livemedia-creator.1 index 17afe575..1b0ad209 100644 --- a/docs/livemedia-creator.1 +++ b/docs/livemedia-creator.1 @@ -4,12 +4,13 @@ livemedia-creator \- Create live install media .SH SYNOPSIS livemedia-creator [-h] - (--make-iso | --make-disk | --make-fsimage | --make-appliance | --make-ami) + (--make-iso | --make-disk | --make-fsimage | --make-appliance | --make-ami | --make-tar) [--iso ISO] [--disk-image DISK_IMAGE] [--fs-image FS_IMAGE] [--ks KS] [--image-name IMAGE_NAME] [--image-only] [--fs-label FS_LABEL] [--qcow2] [--qcow2-arg QCOW2_ARGS] + [--compression] [--compress-arg] [--keep-image] [--no-virt] [--proxy PROXY] [--anaconda-arg ANACONDA_ARGS] [--armplatform ARMPLATFORM] [--location LOCATION] @@ -65,6 +66,10 @@ Build an appliance image and XML description \fB\-\-make\-ami\fR Build an ami image +.TP +\fB\-\-make\-tar\fR +Build a tar of the root filesystem. Defaults to root.tar.xz + .TP \fB\-\-iso ISO\fR Anaconda installation .iso path to use for virt-install @@ -82,9 +87,17 @@ Path to existing filesystem image to use for creating final image. Create qcow2 image instead of raw sparse image when making disk images. .TP -\fB\-\-qcow2\-args\fR +\fB\-\-qcow2\-arg\fR Arguments to pass to qemu-img. Pass once for each argument +.TP +\fB\-\-compress\fR +Compression binary for make-tar. xz, lzma, gzip, and bzip2 are supported. xz is the default. + +.TP +\fB\-\-compress\-arg\fR +Arguments to pass to compression. Pass once for each argument + .TP \fB\-\-ks KS\fR Kickstart file defining the install. diff --git a/src/pylorax/imgutils.py b/src/pylorax/imgutils.py index 6877cc10..9fa0fe1d 100644 --- a/src/pylorax/imgutils.py +++ b/src/pylorax/imgutils.py @@ -22,7 +22,7 @@ logger = logging.getLogger("pylorax.imgutils") import os, tempfile from os.path import join, dirname -from subprocess import CalledProcessError +from subprocess import Popen, PIPE, CalledProcessError import sys import traceback import multiprocessing @@ -32,13 +32,14 @@ from pylorax.sysutils import cpfile from pylorax.executils import execWithRedirect, execWithCapture from pylorax.executils import runcmd, runcmd_output -######## Functions for making container images (cpio, squashfs) ########## +######## Functions for making container images (cpio, tar, squashfs) ########## -def mkcpio(rootdir, outfile, compression="xz", compressargs=["-9"]): - '''Make a compressed CPIO archive of the given rootdir. - compression should be "xz", "gzip", "lzma", or None. +def compress(command, rootdir, outfile, compression="xz", compressargs=["-9"]): + '''Make a compressed archive of the given rootdir. + command is a list of the archiver commands to run + compression should be "xz", "gzip", "lzma", "bzip2", or None. compressargs will be used on the compression commandline.''' - if compression not in (None, "xz", "gzip", "lzma"): + if compression not in (None, "xz", "gzip", "lzma", "bzip2"): raise ValueError, "Unknown compression type %s" % compression if compression == "xz": compressargs.insert(0, "--check=crc32") @@ -51,16 +52,34 @@ def mkcpio(rootdir, outfile, compression="xz", compressargs=["-9"]): compressargs.insert(0, "-T%d" % multiprocessing.cpu_count()) elif compression == "gzip": compression = "pigz" + compressargs.insert(0, "-p%d" % multiprocessing.cpu_count()) + elif compression == "bzip2": + compression = "pbzip2" + compressargs.insert(0, "-p%d" % multiprocessing.cpu_count()) - logger.debug("mkcpio %s | %s %s > %s", rootdir, compression, - " ".join(compressargs), outfile) - find = Popen(["find", ".", "-print0"], stdout=PIPE, cwd=rootdir) - cpio = Popen(["cpio", "--null", "--quiet", "-H", "newc", "-o"], - stdin=find.stdout, stdout=PIPE, cwd=rootdir) - comp = Popen([compression] + compressargs, - stdin=cpio.stdout, stdout=open(outfile, "wb")) - comp.wait() - return comp.returncode + logger.debug("find %s -print0 |%s | %s %s > %s", rootdir, " ".join(command), + compression, " ".join(compressargs), outfile) + find, archive, comp = None, None, None + try: + find = Popen(["find", ".", "-print0"], stdout=PIPE, cwd=rootdir) + archive = Popen(command, stdin=find.stdout, stdout=PIPE, cwd=rootdir) + comp = Popen([compression] + compressargs, + stdin=archive.stdout, stdout=open(outfile, "wb")) + comp.wait() + return comp.returncode + except OSError as e: + logger.error(e) + # Kill off any hanging processes + [p.kill() for p in (find, archive, comp) if p] + return 1 + +def mkcpio(rootdir, outfile, compression="xz", compressargs=["-9"]): + return compress(["cpio", "--null", "--quiet", "-H", "newc", "-o"], + rootdir, outfile, compression, compressargs) + +def mktar(rootdir, outfile, compression="xz", compressargs=["-9"]): + return compress(["tar", "--selinux", "--acls", "--xattrs", "-cf-", "--null", "-T-"], + rootdir, outfile, compression, compressargs) def mksquashfs(rootdir, outfile, compression="default", compressargs=[]): '''Make a squashfs image containing the given rootdir.''' diff --git a/src/sbin/livemedia-creator b/src/sbin/livemedia-creator index df2e5f59..61edc8a8 100755 --- a/src/sbin/livemedia-creator +++ b/src/sbin/livemedia-creator @@ -2,7 +2,7 @@ # # Live Media Creator # -# Copyright (C) 2011-2013 Red Hat, Inc. +# Copyright (C) 2011-2014 Red Hat, Inc. # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -53,7 +53,7 @@ from pylorax.treebuilder import findkernels from pylorax.sysutils import joinpaths, remove from pylorax.imgutils import PartitionMount, mksparse, mkext4img, loop_detach from pylorax.imgutils import get_loop_name, dm_detach, mount, umount, Mount -from pylorax.imgutils import mksquashfs, mkqcow2 +from pylorax.imgutils import mksquashfs, mkqcow2, mktar from pylorax.executils import execWithRedirect, execWithCapture # no-virt mode doesn't need libvirt, so make it optional @@ -562,6 +562,13 @@ def novirt_install(opts, disk_img, disk_size, repo_url): if not os.path.isdir(ROOT_PATH): os.mkdir(ROOT_PATH) mount(disk_img, opts="loop", mnt=ROOT_PATH) + elif opts.make_tar: + args += ["--dirinstall"] + + # Install directly into ROOT_PATH, make sure it starts clean + if os.path.exists(ROOT_PATH): + shutil.rmtree(ROOT_PATH) + os.mkdir(ROOT_PATH) else: args += ["--image", disk_img] @@ -613,6 +620,16 @@ def novirt_install(opts, disk_img, disk_size, repo_url): qcow2_img = tempfile.mktemp(prefix="disk", suffix=".img", dir=opts.tmp) execWithRedirect("qemu-img", ["convert"] + qcow2_args + [disk_img, qcow2_img], raise_err=True) execWithRedirect("mv", ["-f", qcow2_img, disk_img], raise_err=True) + elif opts.make_tar: + compress_args = [] + for arg in opts.compress_args: + compress_args += arg.split(" ", 1) + + rc = mktar(ROOT_PATH, disk_img, opts.compression, compress_args) + shutil.rmtree(ROOT_PATH) + + if rc: + raise InstallError("novirt_install failed") def virt_install(opts, install_log, disk_img, disk_size): @@ -640,7 +657,7 @@ def virt_install(opts, install_log, disk_img, disk_size): mkqcow2(disk_img, disk_size*1024**2, qcow2_args) - if opts.make_fsimage: + if opts.make_fsimage or opts.make_tar: diskimg_path = tempfile.mktemp(prefix="disk", suffix=".img", dir=opts.tmp) else: diskimg_path = disk_img @@ -662,6 +679,18 @@ def virt_install(opts, install_log, disk_img, disk_size): if opts.make_fsimage: make_fsimage(diskimg_path, disk_img, disk_size, label=opts.fs_label) os.unlink(diskimg_path) + elif opts.make_tar: + compress_args = [] + for arg in opts.compress_args: + compress_args += arg.split(" ", 1) + + with PartitionMount(diskimg_path) as img_mount: + if img_mount and img_mount.mount_dir: + rc = mktar(img_mount.mount_dir, disk_img, opts.compression, compress_args) + os.unlink(diskimg_path) + + if rc: + raise InstallError("virt_install failed") def make_squashfs(disk_img, work_dir, compression="xz"): @@ -758,6 +787,8 @@ if __name__ == '__main__': help="Build an appliance image and XML description" ) action.add_argument( "--make-ami", action="store_true", help="Build an ami image" ) + action.add_argument( "--make-tar", action="store_true", + help="Build a tar of the root filesystem" ) parser.add_argument( "--iso", type=os.path.abspath, help="Anaconda installation .iso path to use for virt-install" ) @@ -807,13 +838,17 @@ if __name__ == '__main__': image_group.add_argument( "--fs-image", type=os.path.abspath, help="Path to existing filesystem image to use for creating final image." ) image_group.add_argument( "--image-name", default=None, - help="Name of fs/disk image to create. Default is a random name." ) + help="Name of output file to create. Used for tar, fs and disk image. Default is a random name." ) image_group.add_argument( "--fs-label", default="Anaconda", help="Label to set on fsimage, default is 'Anaconda'") image_group.add_argument("--qcow2", action="store_true", help="Create qcow2 image instead of raw sparse image when making disk images.") image_group.add_argument("--qcow2-arg", action="append", dest="qcow2_args", default=[], help="Arguments to pass to qemu-img. Pass once for each argument") + image_group.add_argument("--compression", default="xz", + help="Compression binary for make-tar. xz, lzma, gzip, and bzip2 are supported. xz is the default.") + image_group.add_argument("--compress-arg", action="append", dest="compress_args", default=[], + help="Arguments to pass to compression. Pass once for each argument") # Group of arguments for appliance creation app_group = parser.add_argument_group("appliance arguments") @@ -927,6 +962,9 @@ if __name__ == '__main__': if opts.make_fsimage and opts.qcow2: errors.append("qcow2 cannot be used to make filesystem images.") + if opts.make_tar and opts.qcow2: + errors.append("qcow2 cannot be used to make a tar.") + if os.getuid() != 0: errors.append("You need to run this as root") @@ -941,6 +979,11 @@ if __name__ == '__main__': opts.image_name = "ami-root.img" if opts.fs_label == "Anaconda": opts.fs_label = "AMI" + elif opts.make_tar: + if not opts.image_name: + opts.image_name = "root.tar.xz" + if opts.compression == "xz" and not opts.compress_args: + opts.compress_args = ["-9"] if opts.app_file: opts.app_file = joinpaths(opts.tmp, opts.app_file)