livemedia-creator: Add support for making tarfiles (#1144140)

This adds the --make-tar option which will produce a xz compressed tar
of the root filesystem. This works with either virt-install or no-virt
modes. Use --image-name to set the output filename.

--compression is used to set the compression type to use, which defaults
to xz. Supported types are xz, lzma, gzip and bzip2.

--compress-arg is used to pass arguments to the compression utility.

(cherry picked from commit d04a99e8f4)

Resolves: rhbz#1144140

README.livemedia-creator

@@ -224,6 +224,21 @@ The created image can be imported into libvirt using:
 virt-image appliance.xml
 
 
+TAR FILE CREATION
+-----------------
+The --make-tar command can be used to create a tar of the root filesystem. By
+default it is compressed using xz, but this can be changed using the
+--compression and --compress-arg options. This option works with both virt and
+--no-virt install methods.
+
+As with --make-fsimage the kickstart should be limited to a single / partition.
+
+eg.
+
+livemedia-creator --make-tar --iso=/path/to/boot.iso --ks=./docs/fedora-minimal.ks \
+--image-name=fedora-root.tar.xz
+
+
 DEBUGGING PROBLEMS
 ------------------
 Cleaning up an aborted (ctrl-c) virt-install run (as root):

docs/livemedia-creator.1

@@ -4,10 +4,12 @@ livemedia-creator \- Create live install media
 
 .SH SYNOPSIS
 livemedia-creator [-h]
-    (--make-iso | --make-disk | --make-fsimage | --make-appliance | --make-ami)
+    (--make-iso | --make-disk | --make-fsimage | --make-appliance | --make-ami | --make-tar)
     [--iso ISO] [--disk-image DISK_IMAGE]
     [--fs-image FS_IMAGE] [--ks KS]
     [--image-name IMAGE_NAME] [--image-only]
+    [--fs-label FS_LABEL]
+    [--compression] [--compress-arg]
     [--keep-image] [--no-virt] [--proxy PROXY]
     [--anaconda-arg ANACONDA_ARGS]
     [--armplatform ARMPLATFORM] [--location LOCATION]
@@ -63,6 +65,10 @@ Build an appliance image and XML description
 \fB\-\-make\-ami\fR
 Build an ami image
 
+.TP
+\fB\-\-make\-tar\fR
+Build a tar of the root filesystem. Defaults to root.tar.xz
+
 .TP
 \fB\-\-iso ISO\fR
 Anaconda installation .iso path to use for virt-install
@@ -73,7 +79,24 @@ Path to disk image to use for creating final image
 
 .TP
 \fB\-\-fs\-image FS_IMAGE\fR
-Path to filesystem image to use for creating final image
+Path to existing filesystem image to use for creating final image.
+
+.TP
+\fB\-\-qcow2\fR
+Create qcow2 image instead of raw sparse image when making disk images.
+
+.TP
+\fB\-\-qcow2\-arg\fR
+Arguments to pass to qemu-img. Pass once for each argument
+>>>>>>> d04a99e... livemedia-creator: Add support for making tarfiles
+
+.TP
+\fB\-\-compress\fR
+Compression binary for make-tar. xz, lzma, gzip, and bzip2 are supported. xz is the default.
+
+.TP
+\fB\-\-compress\-arg\fR
+Arguments to pass to compression. Pass once for each argument
 
 .TP
 \fB\-\-ks KS\fR

src/pylorax/imgutils.py

@@ -22,37 +22,64 @@ logger = logging.getLogger("pylorax.imgutils")
 
 import os, tempfile
 from os.path import join, dirname
-from subprocess import CalledProcessError
+from subprocess import Popen, PIPE, CalledProcessError
 import sys
 import traceback
+import multiprocessing
 from time import sleep
 
 from pylorax.sysutils import cpfile
 from pylorax.executils import execWithRedirect, execWithCapture
 from pylorax.executils import runcmd, runcmd_output
 
-######## Functions for making container images (cpio, squashfs) ##########
+######## Functions for making container images (cpio, tar, squashfs) ##########
 
-def mkcpio(rootdir, outfile, compression="xz", compressargs=["-9"]):
+def compress(command, rootdir, outfile, compression="xz", compressargs=["-9"]):
-    '''Make a compressed CPIO archive of the given rootdir.
+    '''Make a compressed archive of the given rootdir.
-    compression should be "xz", "gzip", "lzma", or None.
+    command is a list of the archiver commands to run
+    compression should be "xz", "gzip", "lzma", "bzip2", or None.
     compressargs will be used on the compression commandline.'''
-    if compression not in (None, "xz", "gzip", "lzma"):
+    if compression not in (None, "xz", "gzip", "lzma", "bzip2"):
         raise ValueError, "Unknown compression type %s" % compression
     if compression == "xz":
         compressargs.insert(0, "--check=crc32")
     if compression is None:
         compression = "cat" # this is a little silly
         compressargs = []
-    logger.debug("mkcpio %s | %s %s > %s", rootdir, compression,
+
-                                        " ".join(compressargs), outfile)
+    # make compression run with multiple threads if possible
-    find = Popen(["find", ".", "-print0"], stdout=PIPE, cwd=rootdir)
+    if compression in ("xz", "lzma"):
-    cpio = Popen(["cpio", "--null", "--quiet", "-H", "newc", "-o"],
+        compressargs.insert(0, "-T%d" % multiprocessing.cpu_count())
-                 stdin=find.stdout, stdout=PIPE, cwd=rootdir)
+    elif compression == "gzip":
-    comp = Popen([compression] + compressargs,
+        compression = "pigz"
-                 stdin=cpio.stdout, stdout=open(outfile, "wb"))
+        compressargs.insert(0, "-p%d" % multiprocessing.cpu_count())
-    comp.wait()
+    elif compression == "bzip2":
-    return comp.returncode
+        compression = "pbzip2"
+        compressargs.insert(0, "-p%d" % multiprocessing.cpu_count())
+
+    logger.debug("find %s -print0 |%s | %s %s > %s", rootdir, " ".join(command),
+                 compression, " ".join(compressargs), outfile)
+    find, archive, comp = None, None, None
+    try:
+        find = Popen(["find", ".", "-print0"], stdout=PIPE, cwd=rootdir)
+        archive = Popen(command, stdin=find.stdout, stdout=PIPE, cwd=rootdir)
+        comp = Popen([compression] + compressargs,
+                     stdin=archive.stdout, stdout=open(outfile, "wb"))
+        comp.wait()
+        return comp.returncode
+    except OSError as e:
+        logger.error(e)
+        # Kill off any hanging processes
+        [p.kill() for p in (find, archive, comp) if p]
+        return 1
+
+def mkcpio(rootdir, outfile, compression="xz", compressargs=["-9"]):
+    return compress(["cpio", "--null", "--quiet", "-H", "newc", "-o"],
+                    rootdir, outfile, compression, compressargs)
+
+def mktar(rootdir, outfile, compression="xz", compressargs=["-9"]):
+    return compress(["tar", "--selinux", "--acls", "--xattrs", "-cf-", "--null", "-T-"],
+                    rootdir, outfile, compression, compressargs)
 
 def mksquashfs(rootdir, outfile, compression="default", compressargs=[]):
     '''Make a squashfs image containing the given rootdir.'''

src/sbin/livemedia-creator

@@ -2,7 +2,7 @@
 #
 # Live Media Creator
 #
-# Copyright (C) 2011-2013  Red Hat, Inc.
+# Copyright (C) 2011-2014  Red Hat, Inc.
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -53,7 +53,7 @@ from pylorax.treebuilder import TreeBuilder, RuntimeBuilder, udev_escape
 from pylorax.sysutils import joinpaths, remove
 from pylorax.imgutils import PartitionMount, mksparse, mkext4img, loop_detach
 from pylorax.imgutils import get_loop_name, dm_detach, mount, umount, Mount
-from pylorax.imgutils import mksquashfs
+from pylorax.imgutils import mksquashfs, mktar
 from pylorax.executils import execWithRedirect, execWithCapture
 
 # no-virt mode doesn't need libvirt, so make it optional
@@ -578,6 +578,13 @@ def novirt_install(opts, disk_img, disk_size, repo_url):
         if not os.path.isdir(ROOT_PATH):
             os.mkdir(ROOT_PATH)
         mount(disk_img, opts="loop", mnt=ROOT_PATH)
+    elif opts.make_tar:
+        args += ["--dirinstall"]
+
+        # Install directly into ROOT_PATH, make sure it starts clean
+        if os.path.exists(ROOT_PATH):
+            shutil.rmtree(ROOT_PATH)
+            os.mkdir(ROOT_PATH)
     else:
         args += ["--image", disk_img]
 
@@ -617,6 +624,17 @@ def novirt_install(opts, disk_img, disk_size, repo_url):
     if rc:
         raise InstallError("novirt_install failed")
 
+    if opts.make_tar:
+        compress_args = []
+        for arg in opts.compress_args:
+            compress_args += arg.split(" ", 1)
+
+        rc = mktar(ROOT_PATH, disk_img, opts.compression, compress_args)
+        shutil.rmtree(ROOT_PATH)
+
+        if rc:
+            raise InstallError("novirt_install failed")
+
 
 def virt_install(opts, install_log, disk_img, disk_size):
     """
@@ -635,7 +653,7 @@ def virt_install(opts, install_log, disk_img, disk_size):
     if opts.proxy:
         kernel_args += " proxy="+opts.proxy
 
-    if opts.make_fsimage:
+    if opts.make_fsimage or opts.make_tar:
         diskimg_path = tempfile.mktemp(prefix="disk", suffix=".img", dir=opts.tmp)
     else:
         diskimg_path = disk_img
@@ -655,6 +673,18 @@ def virt_install(opts, install_log, disk_img, disk_size):
     if opts.make_fsimage:
         make_fsimage(diskimg_path, disk_img, label=opts.fs_label)
         os.unlink(diskimg_path)
+    elif opts.make_tar:
+        compress_args = []
+        for arg in opts.compress_args:
+            compress_args += arg.split(" ", 1)
+
+        with PartitionMount(diskimg_path) as img_mount:
+            if img_mount and img_mount.mount_dir:
+                rc = mktar(img_mount.mount_dir, disk_img, opts.compression, compress_args)
+        os.unlink(diskimg_path)
+
+        if rc:
+            raise InstallError("virt_install failed")
 
 
 def make_squashfs(disk_img, work_dir, compression="xz"):
@@ -754,6 +784,8 @@ if __name__ == '__main__':
                          help="Build an appliance image and XML description" )
     action.add_argument( "--make-ami", action="store_true",
                          help="Build an ami image" )
+    action.add_argument( "--make-tar", action="store_true",
+                         help="Build a tar of the root filesystem" )
 
     parser.add_argument( "--iso", type=os.path.abspath,
                         help="Anaconda installation .iso path to use for virt-install" )
@@ -763,6 +795,10 @@ if __name__ == '__main__':
                         help="Path to existing filesystem image to use for creating final image." )
     parser.add_argument( "--fs-label", default="Anaconda",
                         help="Label to set on fsimage, default is 'Anaconda'")
+    parser.add_argument("--compression", default="xz",
+                        help="Compression binary for make-tar. xz, lzma, gzip, and bzip2 are supported. xz is the default.")
+    parser.add_argument("--compress-arg", action="append", dest="compress_args", default=[],
+                        help="Arguments to pass to compression. Pass once for each argument")
 
     parser.add_argument( "--ks", action="append", type=os.path.abspath,
                          help="Kickstart file defining the install." )
@@ -923,6 +959,11 @@ if __name__ == '__main__':
             opts.image_name = "ami-root.img"
         if opts.fs_label == "Anaconda":
             opts.fs_label = "AMI"
+    elif opts.make_tar:
+        if not opts.image_name:
+            opts.image_name = "root.tar.xz"
+        if opts.compression == "xz" and not opts.compress_args:
+            opts.compress_args = ["-9"]
 
     if opts.app_file:
         opts.app_file = joinpaths(opts.tmp, opts.app_file)