From 4e144e15b738dbf8fa643fb53d1740165879bf8f Mon Sep 17 00:00:00 2001 From: David Shea Date: Thu, 28 Mar 2019 15:53:14 -0400 Subject: [PATCH] Add a new output type, tar-disk. This option will create an optionally compressed tarball containing a disk image. This format is used by Google's Compute Engine. This also adds a new option, tar_disk_name, to set the name of the disk image that will be wrapped in the final tarball. opts.image_name continues to be the final output file name. (cherry picked from commit c941b82b0c55429bd94f061777988dbb1ef4c4d4) (cherry picked from commit 121717ac4acdb7a7f7ff1a317a06bd9db31acf5a) --- src/pylorax/api/compose.py | 53 ++++++++++++++++++++++++++++++++++++-- src/pylorax/cmdline.py | 4 +++ src/pylorax/creator.py | 33 +++++++++++++++++++----- src/pylorax/installer.py | 31 ++++++++++++++++++++-- src/sbin/livemedia-creator | 16 ++++++++++-- 5 files changed, 125 insertions(+), 12 deletions(-) diff --git a/src/pylorax/api/compose.py b/src/pylorax/api/compose.py index 08d37e54..b641c82b 100644 --- a/src/pylorax/api/compose.py +++ b/src/pylorax/api/compose.py @@ -550,9 +550,13 @@ def start_build(cfg, dnflock, gitlock, branch, recipe_name, compose_type, test_m cfg_args["volid"] = "" cfg_args["extra_boot_args"] = get_kernel_append(recipe) + if "compression" not in cfg_args: + cfg_args["compression"] = "xz" + + if "compress_args" not in cfg_args: + cfg_args["compress_args"] = [] + cfg_args.update({ - "compression": "xz", - "compress_args": [], "ks": [ks_path], "logfile": log_dir, "timeout": 60, # 60 minute timeout @@ -597,6 +601,7 @@ def compose_args(compose_type): "make_appliance": False, "make_ami": False, "make_tar": True, + "make_tar_disk": False, "make_pxe_live": False, "make_ostree_live": False, "make_oci": False, @@ -608,6 +613,7 @@ def compose_args(compose_type): "image_type": False, # False instead of None because of TOML "qemu_args": [], "image_name": default_image_name("xz", "root.tar"), + "tar_disk_name": None, "image_only": True, "app_name": None, "app_template": None, @@ -619,6 +625,7 @@ def compose_args(compose_type): "make_appliance": False, "make_ami": False, "make_tar": False, + "make_tar_disk": False, "make_pxe_live": False, "make_ostree_live": False, "make_oci": False, @@ -630,6 +637,7 @@ def compose_args(compose_type): "image_type": False, # False instead of None because of TOML "qemu_args": [], "image_name": "live.iso", + "tar_disk_name": None, "fs_label": "Anaconda", # Live booting may expect this to be 'Anaconda' "image_only": False, "app_name": None, @@ -644,6 +652,7 @@ def compose_args(compose_type): "make_appliance": False, "make_ami": False, "make_tar": False, + "make_tar_disk": False, "make_pxe_live": False, "make_ostree_live": False, "make_oci": False, @@ -655,6 +664,7 @@ def compose_args(compose_type): "image_type": False, # False instead of None because of TOML "qemu_args": [], "image_name": "disk.img", + "tar_disk_name": None, "fs_label": "", "image_only": True, "app_name": None, @@ -667,6 +677,7 @@ def compose_args(compose_type): "make_appliance": False, "make_ami": False, "make_tar": False, + "make_tar_disk": False, "make_pxe_live": False, "make_ostree_live": False, "make_oci": False, @@ -678,6 +689,7 @@ def compose_args(compose_type): "image_type": "qcow2", "qemu_args": [], "image_name": "disk.qcow2", + "tar_disk_name": None, "fs_label": "", "image_only": True, "app_name": None, @@ -690,6 +702,7 @@ def compose_args(compose_type): "make_appliance": False, "make_ami": False, "make_tar": False, + "make_tar_disk": False, "make_pxe_live": False, "make_ostree_live": False, "make_oci": False, @@ -701,6 +714,7 @@ def compose_args(compose_type): "image_type": False, # False instead of None because of TOML "qemu_args": [], "image_name": "filesystem.img", + "tar_disk_name": None, "fs_label": "", "image_only": True, "app_name": None, @@ -713,6 +727,7 @@ def compose_args(compose_type): "make_appliance": False, "make_ami": False, "make_tar": False, + "make_tar_disk": False, "make_pxe_live": False, "make_ostree_live": False, "make_oci": False, @@ -724,6 +739,7 @@ def compose_args(compose_type): "image_type": False, "qemu_args": [], "image_name": "disk.ami", + "tar_disk_name": None, "fs_label": "", "image_only": True, "app_name": None, @@ -736,6 +752,7 @@ def compose_args(compose_type): "make_appliance": False, "make_ami": False, "make_tar": False, + "make_tar_disk": False, "make_pxe_live": False, "make_ostree_live": False, "make_oci": False, @@ -747,6 +764,7 @@ def compose_args(compose_type): "image_type": "vpc", "qemu_args": ["-o", "subformat=fixed,force_size"], "image_name": "disk.vhd", + "tar_disk_name": None, "fs_label": "", "image_only": True, "app_name": None, @@ -759,6 +777,7 @@ def compose_args(compose_type): "make_appliance": False, "make_ami": False, "make_tar": False, + "make_tar_disk": False, "make_pxe_live": False, "make_ostree_live": False, "make_oci": False, @@ -770,6 +789,7 @@ def compose_args(compose_type): "image_type": "vmdk", "qemu_args": [], "image_name": "disk.vmdk", + "tar_disk_name": None, "fs_label": "", "image_only": True, "app_name": None, @@ -782,6 +802,7 @@ def compose_args(compose_type): "make_appliance": False, "make_ami": False, "make_tar": False, + "make_tar_disk": False, "make_pxe_live": False, "make_ostree_live": False, "make_oci": False, @@ -793,6 +814,34 @@ def compose_args(compose_type): "image_type": "qcow2", "qemu_args": [], "image_name": "disk.qcow2", + "tar_disk_name": None, + "fs_label": "", + "image_only": True, + "app_name": None, + "app_template": None, + "app_file": None, + }, + "google": {"make_iso": False, + "make_disk": True, + "make_fsimage": False, + "make_appliance": False, + "make_ami": False, + "make_tar": False, + "make_tar_disk": True, + "make_pxe_live": False, + "make_ostree_live": False, + "make_oci": False, + "make_vagrant": False, + "ostree": False, + "live_rootfs_keep_size": False, + "live_rootfs_size": 0, + "image_size_align": 1024, + "image_type": False, # False instead of None because of TOML + "qemu_args": [], + "image_name": "disk.tar.gz", + "tar_disk_name": "disk.raw", + "compression": "gzip", + "compress_args": ["-9"], "fs_label": "", "image_only": True, "app_name": None, diff --git a/src/pylorax/cmdline.py b/src/pylorax/cmdline.py index c409b47e..f37d2a17 100644 --- a/src/pylorax/cmdline.py +++ b/src/pylorax/cmdline.py @@ -148,6 +148,8 @@ def lmc_parser(dracut_default=""): help="Build an ami image") action.add_argument("--make-tar", action="store_true", help="Build a tar of the root filesystem") + action.add_argument("--make-tar-disk", action="store_true", + help="Build a tar of a partitioned disk image") action.add_argument("--make-pxe-live", action="store_true", help="Build a live pxe boot squashfs image") action.add_argument("--make-ostree-live", action="store_true", @@ -215,6 +217,8 @@ def lmc_parser(dracut_default=""): help="Path to existing filesystem image to use for creating final image.") image_group.add_argument("--image-name", default=None, help="Name of output file to create. Used for tar, fs and disk image. Default is a random name.") + image_group.add_argument("--tar-disk-name", default=None, + help="Name of the archive member for make-tar-disk.") image_group.add_argument("--fs-label", default="Anaconda", help="Label to set on fsimage, default is 'Anaconda'") image_group.add_argument("--image-size-align", type=int, default=0, diff --git a/src/pylorax/creator.py b/src/pylorax/creator.py index a968b687..c004f34a 100644 --- a/src/pylorax/creator.py +++ b/src/pylorax/creator.py @@ -487,28 +487,49 @@ def make_image(opts, ks, cancel_func=None): Use qemu+boot.iso or anaconda to install to a disk image. """ - if opts.image_name: + + # For make_tar_disk, opts.image_name is the name of the final tarball. + # Use opts.tar_disk_name as the name of the disk image + if opts.make_tar_disk: + disk_img = joinpaths(opts.result_dir, opts.tar_disk_name) + elif opts.image_name: disk_img = joinpaths(opts.result_dir, opts.image_name) else: disk_img = tempfile.mktemp(prefix="lmc-disk-", suffix=".img", dir=opts.result_dir) log.info("disk_img = %s", disk_img) disk_size = calculate_disk_size(opts, ks) + + # For make_tar_disk, pass a second path parameter for the final tarball + # not the final output file. + if opts.make_tar_disk: + tar_img = joinpaths(opts.result_dir, opts.image_name) + else: + tar_img = None + try: if opts.no_virt: - novirt_install(opts, disk_img, disk_size, cancel_func=cancel_func) + novirt_install(opts, disk_img, disk_size, cancel_func=cancel_func, tar_img=tar_img) else: install_log = os.path.abspath(os.path.dirname(opts.logfile))+"/virt-install.log" log.info("install_log = %s", install_log) - virt_install(opts, install_log, disk_img, disk_size, cancel_func=cancel_func) + virt_install(opts, install_log, disk_img, disk_size, cancel_func=cancel_func, tar_img=tar_img) except InstallError as e: log.error("Install failed: %s", e) - if not opts.keep_image and os.path.exists(disk_img): - log.info("Removing bad disk image") - os.unlink(disk_img) + if not opts.keep_image: + if os.path.exists(disk_img): + log.info("Removing bad disk image") + os.unlink(disk_img) + if tar_img and os.path.exists(tar_img): + log.info("Removing bad tar file") + os.unlink(tar_img) raise log.info("Disk Image install successful") + + if opts.make_tar_disk: + return tar_img + return disk_img diff --git a/src/pylorax/installer.py b/src/pylorax/installer.py index a32b9b1f..29a19687 100644 --- a/src/pylorax/installer.py +++ b/src/pylorax/installer.py @@ -324,7 +324,7 @@ def anaconda_cleanup(dirinstall_path): return rc -def novirt_install(opts, disk_img, disk_size, cancel_func=None): +def novirt_install(opts, disk_img, disk_size, cancel_func=None, tar_img=None): """ Use Anaconda to install to a disk image @@ -334,6 +334,7 @@ def novirt_install(opts, disk_img, disk_size, cancel_func=None): :param int disk_size: The size of the disk_img in MiB :param cancel_func: Function that returns True to cancel build :type cancel_func: function + :param str tar_img: For make_tar_disk, the path to final tarball to be created This method runs anaconda to create the image and then based on the opts passed creates a qemu disk image or tarfile. @@ -511,8 +512,20 @@ def novirt_install(opts, disk_img, disk_size, cancel_func=None): # For raw disk images, use fallocate to deallocate unused space execWithRedirect("fallocate", ["--dig-holes", disk_img], raise_err=True) + # For make_tar_disk, wrap the result in a tar file, and remove the original disk image. + if opts.make_tar_disk: + compress_args = [] + for arg in opts.compress_args: + compress_args += arg.split(" ", 1) -def virt_install(opts, install_log, disk_img, disk_size, cancel_func=None): + rc = mktar(disk_img, tar_img, opts.compression, compress_args, selinux=False) + + if rc: + raise InstallError("novirt_install mktar failed: rc=%s" % rc) + + os.unlink(disk_img) + +def virt_install(opts, install_log, disk_img, disk_size, cancel_func=None, tar_img=None): """ Use qemu to install to a disk image @@ -523,6 +536,7 @@ def virt_install(opts, install_log, disk_img, disk_size, cancel_func=None): :param int disk_size: The size of the disk_img in MiB :param cancel_func: Function that returns True to cancel build :type cancel_func: function + :param str tar_img: For make_tar_disk, the path to final tarball to be created This uses qemu with a boot.iso and a kickstart to create a disk image and then optionally, based on the opts passed, creates tarfile. @@ -637,3 +651,16 @@ def virt_install(opts, install_log, disk_img, disk_size, cancel_func=None): if rc: raise InstallError("virt_install failed") shutil.rmtree(vagrant_dir) + + # For make_tar_disk, wrap the result in a tar file, and remove the original disk image. + if opts.make_tar_disk: + compress_args = [] + for arg in opts.compress_args: + compress_args += arg.split(" ", 1) + + rc = mktar(disk_img, tar_img, opts.compression, compress_args, selinux=False) + + if rc: + raise InstallError("virt_install mktar failed: rc=%s" % rc) + + os.unlink(disk_img) diff --git a/src/sbin/livemedia-creator b/src/sbin/livemedia-creator index 5539fed7..30881e48 100755 --- a/src/sbin/livemedia-creator +++ b/src/sbin/livemedia-creator @@ -99,8 +99,12 @@ def main(): errors.append("The appliance template (%s) doesn't " "exist" % opts.app_template) - if opts.image_name and os.path.exists(joinpaths(opts.result_dir, opts.image_name)): - errors.append("The disk image to be created should not exist.") + if opts.make_tar_disk: + if opts.tar_disk_name and os.path.exists(joinpaths(opts.result_dir, opts.tar_disk_name)): + errors.append("The disk image to be created should not exist.") + else: + if opts.image_name and os.path.exists(joinpaths(opts.result_dir, opts.image_name)): + errors.append("The disk image to be created should not exist.") # Vagrant creates a qcow2 inside a tar, turn on qcow2 if opts.make_vagrant: @@ -173,6 +177,14 @@ def main(): opts.image_name = default_image_name(opts.compression, "vagrant.tar") if opts.compression == "xz" and not opts.compress_args: opts.compress_args = ["-9"] + elif opts.make_tar_disk: + opts.make_disk = True + if not opts.image_name: + opts.image_name = "root.img" + if not opts.tar_disk_name: + opts.tar_disk_name = default_image_name(opts.compression, "root.tar") + if opts.compression == "xz" and not opts.compress_args: + opts.compress_args = ["-9"] if opts.app_file: opts.app_file = joinpaths(opts.result_dir, opts.app_file)