Move core of livemedia-creator into pylorax.creator

This moves everything except the cmdline checking into run_creator in
pylorax.creator

It also rearranges some functions to prevent import loops, and adds a
utility function to imgutils (mkfsimage_from_disk for copying a
partition into a filesystem image).
This commit is contained in:
Brian C. Lane 2018-05-02 09:55:46 -07:00
parent 454c74035c
commit 06c227598c
4 changed files with 226 additions and 209 deletions

View File

@ -17,25 +17,27 @@
import logging import logging
log = logging.getLogger("pylorax") log = logging.getLogger("pylorax")
import os import os
import tempfile import tempfile
import subprocess import subprocess
import shutil import shutil
import hashlib import hashlib
import glob import glob
import json
from math import ceil
# Use Mako templates for appliance builder descriptions # Use Mako templates for appliance builder descriptions
from mako.template import Template from mako.template import Template
from mako.exceptions import text_error_template from mako.exceptions import text_error_template
# Use pykickstart to calculate disk image size
from pykickstart.parser import KickstartParser
from pykickstart.constants import KS_SHUTDOWN
from pykickstart.version import makeVersion
# Use the Lorax treebuilder branch for iso creation # Use the Lorax treebuilder branch for iso creation
from pylorax import ArchData from pylorax import ArchData
from pylorax.base import DataHolder from pylorax.base import DataHolder
from pylorax.executils import execWithRedirect, runcmd from pylorax.executils import execWithRedirect, runcmd
from pylorax.imgutils import PartitionMount, mkext4img from pylorax.imgutils import PartitionMount
from pylorax.imgutils import mount, umount, Mount from pylorax.imgutils import mount, umount, Mount
from pylorax.imgutils import mksquashfs, mkrootfsimg from pylorax.imgutils import mksquashfs, mkrootfsimg
from pylorax.imgutils import copytree from pylorax.imgutils import copytree
@ -174,29 +176,6 @@ def make_appliance(disk_img, name, template, outfile, networks=None, ram=1024,
f.write(result) f.write(result)
def make_fsimage(diskimage, fsimage, img_size=None, label="Anaconda"):
"""
Copy the / partition of a partitioned disk image to an un-partitioned
disk image.
:param str diskimage: The full path to partitioned disk image with a /
:param str fsimage: The full path of the output fs image file
:param int img_size: Optional size of the fsimage in MiB or None to make
it as small as possible
:param str label: The label to apply to the image. Defaults to "Anaconda"
"""
with PartitionMount(diskimage) as img_mount:
if not img_mount or not img_mount.mount_dir:
return None
log.info("Creating fsimage %s (%s)", fsimage, img_size or "minimized")
if img_size:
# convert to Bytes
img_size *= 1024**2
mkext4img(img_mount.mount_dir, fsimage, size=img_size, label=label)
def make_runtime(opts, mount_dir, work_dir, size=None): def make_runtime(opts, mount_dir, work_dir, size=None):
""" """
Make the squashfs image from a directory Make the squashfs image from a directory
@ -331,40 +310,6 @@ def create_pxe_config(template, images_dir, live_image_name, add_args = None):
f.write(result) f.write(result)
def create_vagrant_metadata(path, size=0):
""" Create a default Vagrant metadata.json file
:param str path: Path to metadata.json file
:param int size: Disk size in MiB
"""
metadata = { "provider":"libvirt", "format":"qcow2", "virtual_size": ceil(size / 1024) }
with open(path, "wt") as f:
json.dump(metadata, f, indent=4)
def update_vagrant_metadata(path, size):
""" Update the Vagrant metadata.json file
:param str path: Path to metadata.json file
:param int size: Disk size in MiB
This function makes sure that the provider, format and virtual size of the
metadata file are set correctly. All other values are left untouched.
"""
with open(path, "rt") as f:
try:
metadata = json.load(f)
except ValueError as e:
log.error("Problem reading metadata file %s: %s", path, e)
return
metadata["provider"] = "libvirt"
metadata["format"] = "qcow2"
metadata["virtual_size"] = ceil(size / 1024)
with open(path, "wt") as f:
json.dump(metadata, f, indent=4)
def make_livecd(opts, mount_dir, work_dir): def make_livecd(opts, mount_dir, work_dir):
""" """
Take the content from the disk image and make a livecd out of it Take the content from the disk image and make a livecd out of it
@ -631,4 +576,140 @@ def make_live_images(opts, work_dir, disk_img):
return work_dir return work_dir
def run_creator(opts, callback_func=None):
"""Run the image creator process
:param opts: Commandline options to control the process
:type opts: Either a DataHolder or ArgumentParser
:returns: The result directory and the disk image path.
:rtype: Tuple of str
This function takes the opts arguments and creates the selected output image.
See the cmdline --help for livemedia-creator for the possible options
(Yes, this is not ideal, but we can fix that later)
"""
result_dir = None
# Parse the kickstart
if opts.ks:
ks_version = makeVersion()
ks = KickstartParser(ks_version, errorsAreFatal=False, missingIncludeIsFatal=False)
ks.readKickstart(opts.ks[0])
# live iso usually needs dracut-live so warn the user if it is missing
if opts.ks and opts.make_iso:
if "dracut-live" not in ks.handler.packages.packageList:
log.error("dracut-live package is missing from the kickstart.")
raise RuntimeError("dracut-live package is missing from the kickstart.")
# Make the disk or filesystem image
if not opts.disk_image and not opts.fs_image:
if not opts.ks:
raise RuntimeError("Image creation requires a kickstart file")
errors = []
if opts.no_virt and ks.handler.method.method not in ("url", "nfs") \
and not ks.handler.ostreesetup.seen:
errors.append("Only url, nfs and ostreesetup install methods are currently supported."
"Please fix your kickstart file." )
if ks.handler.method.method in ("url", "nfs") and not ks.handler.network.seen:
errors.append("The kickstart must activate networking if "
"the url or nfs install method is used.")
if ks.handler.displaymode.displayMode is not None:
errors.append("The kickstart must not set a display mode (text, cmdline, "
"graphical), this will interfere with livemedia-creator.")
if opts.make_fsimage or (opts.make_pxe_live and opts.no_virt):
# Make sure the kickstart isn't using autopart and only has a / mountpoint
part_ok = not any(p for p in ks.handler.partition.partitions
if p.mountpoint not in ["/", "swap"])
if not part_ok or ks.handler.autopart.seen:
errors.append("Filesystem images must use a single / part, not autopart or "
"multiple partitions. swap is allowed but not used.")
if not opts.no_virt and ks.handler.reboot.action != KS_SHUTDOWN:
errors.append("The kickstart must include shutdown when using virt installation.")
if errors:
list(log.error(e) for e in errors)
raise RuntimeError("\n".join(errors))
# Make the image. Output of this is either a partitioned disk image or a fsimage
try:
disk_img = make_image(opts, ks)
except InstallError as e:
log.error("ERROR: Image creation failed: %s", e)
raise RuntimeError("Image creation failed: %s" % e)
if opts.image_only:
return (result_dir, disk_img)
if opts.make_iso:
work_dir = tempfile.mkdtemp(prefix="lmc-work-")
log.info("working dir is %s", work_dir)
if (opts.fs_image or opts.no_virt) and not opts.disk_image:
# Create iso from a filesystem image
disk_img = opts.fs_image or disk_img
if not make_squashfs(opts, disk_img, work_dir):
log.error("squashfs.img creation failed")
raise RuntimeError("squashfs.img creation failed")
with Mount(disk_img, opts="loop") as mount_dir:
result_dir = make_livecd(opts, mount_dir, work_dir)
else:
# Create iso from a partitioned disk image
disk_img = opts.disk_image or disk_img
with PartitionMount(disk_img) as img_mount:
if img_mount and img_mount.mount_dir:
make_runtime(opts, img_mount.mount_dir, work_dir, calculate_disk_size(opts, ks)/1024.0)
result_dir = make_livecd(opts, img_mount.mount_dir, work_dir)
# --iso-only removes the extra build artifacts, keeping only the boot.iso
if opts.iso_only and result_dir:
boot_iso = joinpaths(result_dir, "images/boot.iso")
if not os.path.exists(boot_iso):
log.error("%s is missing, skipping --iso-only.", boot_iso)
else:
iso_dir = tempfile.mkdtemp(prefix="lmc-result-")
dest_file = joinpaths(iso_dir, opts.iso_name or "boot.iso")
shutil.move(boot_iso, dest_file)
shutil.rmtree(result_dir)
result_dir = iso_dir
# cleanup the mess
# cleanup work_dir?
if disk_img and not (opts.keep_image or opts.disk_image or opts.fs_image):
os.unlink(disk_img)
log.info("Disk image erased")
disk_img = None
elif opts.make_appliance:
if not opts.ks:
networks = []
else:
networks = ks.handler.network.network
make_appliance(opts.disk_image or disk_img, opts.app_name,
opts.app_template, opts.app_file, networks, opts.ram,
opts.vcpus or 1, opts.arch, opts.title, opts.project, opts.releasever)
elif opts.make_pxe_live:
work_dir = tempfile.mkdtemp(prefix="lmc-work-")
log.info("working dir is %s", work_dir)
disk_img = opts.fs_image or opts.disk_image or disk_img
log.debug("disk image is %s", disk_img)
result_dir = make_live_images(opts, work_dir, disk_img)
if result_dir is None:
log.error("Creating PXE live image failed.")
raise RuntimeError("Creating PXE live image failed.")
if opts.result_dir != opts.tmp and result_dir:
copytree(result_dir, opts.result_dir, preserve=False)
shutil.rmtree(result_dir)
result_dir = None
return (result_dir, disk_img)

View File

@ -483,3 +483,37 @@ def mkhfsimg(rootdir, outfile, size=None, label="", mountargs="", graft=None):
graft = graft or {} graft = graft or {}
mkfsimage("hfsplus", rootdir, outfile, size, mountargs=mountargs, mkfsimage("hfsplus", rootdir, outfile, size, mountargs=mountargs,
mkfsargs=["-v", label], graft=graft) mkfsargs=["-v", label], graft=graft)
def mkfsimage_from_disk(diskimage, fsimage, img_size=None, label="Anaconda"):
"""
Copy the / partition of a partitioned disk image to an un-partitioned
disk image.
:param str diskimage: The full path to partitioned disk image with a /
:param str fsimage: The full path of the output fs image file
:param int img_size: Optional size of the fsimage in MiB or None to make
it as small as possible
:param str label: The label to apply to the image. Defaults to "Anaconda"
"""
with PartitionMount(diskimage) as img_mount:
if not img_mount or not img_mount.mount_dir:
return None
logger.info("Creating fsimage %s (%s)", fsimage, img_size or "minimized")
if img_size:
# convert to Bytes
img_size *= 1024**2
mkext4img(img_mount.mount_dir, fsimage, size=img_size, label=label)
def default_image_name(compression, basename):
""" Return a default image name with the correct suffix for the compression type.
:param str compression: Compression type
:param str basename: Base filename
:returns: basename with compression suffix
If the compression is unknown it defaults to xz
"""
SUFFIXES = {"xz": ".xz", "gzip": ".gz", "bzip2": ".bz2", "lzma": ".lzma"}
return basename + SUFFIXES.get(compression, ".xz")

View File

@ -17,20 +17,20 @@
import logging import logging
log = logging.getLogger("pylorax") log = logging.getLogger("pylorax")
import glob
import json
from math import ceil
import os import os
import tempfile
import subprocess import subprocess
import shutil import shutil
import glob
import socket import socket
import tempfile
# Use the Lorax treebuilder branch for iso creation # Use the Lorax treebuilder branch for iso creation
from pylorax.creator import create_vagrant_metadata, update_vagrant_metadata, make_fsimage
from pylorax.executils import execWithRedirect, execReadlines from pylorax.executils import execWithRedirect, execReadlines
from pylorax.imgutils import PartitionMount, mksparse, mkext4img, loop_detach from pylorax.imgutils import PartitionMount, mksparse, mkext4img, loop_detach
from pylorax.imgutils import get_loop_name, dm_detach, mount, umount from pylorax.imgutils import get_loop_name, dm_detach, mount, umount
from pylorax.imgutils import mkqemu_img, mktar from pylorax.imgutils import mkqemu_img, mktar, mkcpio, mkfsimage_from_disk
from pylorax.imgutils import mkcpio
from pylorax.monitor import LogMonitor from pylorax.monitor import LogMonitor
from pylorax.mount import IsoMountpoint from pylorax.mount import IsoMountpoint
from pylorax.sysutils import joinpaths from pylorax.sysutils import joinpaths
@ -43,6 +43,40 @@ class InstallError(Exception):
pass pass
def create_vagrant_metadata(path, size=0):
""" Create a default Vagrant metadata.json file
:param str path: Path to metadata.json file
:param int size: Disk size in MiB
"""
metadata = { "provider":"libvirt", "format":"qcow2", "virtual_size": ceil(size / 1024) }
with open(path, "wt") as f:
json.dump(metadata, f, indent=4)
def update_vagrant_metadata(path, size):
""" Update the Vagrant metadata.json file
:param str path: Path to metadata.json file
:param int size: Disk size in MiB
This function makes sure that the provider, format and virtual size of the
metadata file are set correctly. All other values are left untouched.
"""
with open(path, "rt") as f:
try:
metadata = json.load(f)
except ValueError as e:
log.error("Problem reading metadata file %s: %s", path, e)
return
metadata["provider"] = "libvirt"
metadata["format"] = "qcow2"
metadata["virtual_size"] = ceil(size / 1024)
with open(path, "wt") as f:
json.dump(metadata, f, indent=4)
def find_free_port(start=5900, end=5999, host="127.0.0.1"): def find_free_port(start=5900, end=5999, host="127.0.0.1"):
""" Return first free port in range. """ Return first free port in range.
@ -513,7 +547,7 @@ def virt_install(opts, install_log, disk_img, disk_size):
raise InstallError(msg) raise InstallError(msg)
if opts.make_fsimage: if opts.make_fsimage:
make_fsimage(diskimg_path, disk_img, disk_size, label=opts.fs_label) mkfsimage_from_disk(diskimg_path, disk_img, disk_size, label=opts.fs_label)
os.unlink(diskimg_path) os.unlink(diskimg_path)
elif opts.make_tar: elif opts.make_tar:
compress_args = [] compress_args = []

View File

@ -20,43 +20,20 @@
import logging import logging
log = logging.getLogger("livemedia-creator") log = logging.getLogger("livemedia-creator")
import glob
import os import os
import selinux
import sys import sys
import tempfile import tempfile
import shutil
import glob
import selinux
# Use pykickstart to calculate disk image size
from pykickstart.parser import KickstartParser
from pykickstart.version import makeVersion
from pykickstart.constants import KS_SHUTDOWN
# Use the Lorax treebuilder branch for iso creation # Use the Lorax treebuilder branch for iso creation
from pylorax import setup_logging, find_templates, vernum from pylorax import setup_logging, find_templates, vernum
from pylorax.cmdline import lmc_parser from pylorax.cmdline import lmc_parser
from pylorax.creator import make_image, make_squashfs, make_livecd, make_runtime, make_appliance, make_live_images from pylorax.creator import run_creator
from pylorax.creator import calculate_disk_size from pylorax.imgutils import default_image_name
from pylorax.imgutils import PartitionMount
from pylorax.imgutils import Mount
from pylorax.imgutils import copytree
from pylorax.installer import InstallError
from pylorax.sysutils import joinpaths from pylorax.sysutils import joinpaths
def default_image_name(compression, basename):
""" Return a default image name with the correct suffix for the compression type.
:param str compression: Compression type
:param str basename: Base filename
:returns: basename with compression suffix
If the compression is unknown it defaults to xz
"""
SUFFIXES = {"xz": ".xz", "gzip": ".gz", "bzip2": ".bz2", "lzma": ".lzma"}
return basename + SUFFIXES.get(compression, ".xz")
def main(): def main():
parser = lmc_parser() parser = lmc_parser()
opts = parser.parse_args() opts = parser.parse_args()
@ -213,121 +190,12 @@ def main():
tempfile.tempdir = opts.tmp tempfile.tempdir = opts.tmp
disk_img = None disk_img = None
# Parse the kickstart try:
if opts.ks: # TODO - Better API than passing in opts
ks_version = makeVersion() (result_dir, disk_img) = run_creator(opts)
ks = KickstartParser(ks_version, errorsAreFatal=False, missingIncludeIsFatal=False) except Exception as e: # pylint: disable=broad-except
ks.readKickstart(opts.ks[0]) log.error(str(e))
sys.exit(1)
# live iso usually needs dracut-live so warn the user if it is missing
if opts.ks and opts.make_iso:
if "dracut-live" not in ks.handler.packages.packageList:
log.error("dracut-live package is missing from the kickstart.")
sys.exit(1)
# Make the disk or filesystem image
if not opts.disk_image and not opts.fs_image:
errors = []
if opts.no_virt and ks.handler.method.method not in ("url", "nfs") \
and not ks.handler.ostreesetup.seen:
errors.append("Only url, nfs and ostreesetup install methods are currently supported."
"Please fix your kickstart file." )
if ks.handler.method.method in ("url", "nfs") and not ks.handler.network.seen:
errors.append("The kickstart must activate networking if "
"the url or nfs install method is used.")
if ks.handler.displaymode.displayMode is not None:
errors.append("The kickstart must not set a display mode (text, cmdline, "
"graphical), this will interfere with livemedia-creator.")
if opts.make_fsimage or (opts.make_pxe_live and opts.no_virt):
# Make sure the kickstart isn't using autopart and only has a / mountpoint
part_ok = not any(p for p in ks.handler.partition.partitions
if p.mountpoint not in ["/", "swap"])
if not part_ok or ks.handler.autopart.seen:
errors.append("Filesystem images must use a single / part, not autopart or "
"multiple partitions. swap is allowed but not used.")
if not opts.no_virt and ks.handler.reboot.action != KS_SHUTDOWN:
errors.append("The kickstart must include shutdown when using virt installation.")
if errors:
list(log.error(e) for e in errors)
sys.exit(1)
# Make the image. Output of this is either a partitioned disk image or a fsimage
try:
disk_img = make_image(opts, ks)
except InstallError as e:
log.error("ERROR: Image creation failed: %s", e)
sys.exit(1)
result_dir = None
if not opts.image_only:
if opts.make_iso:
work_dir = tempfile.mkdtemp(prefix="lmc-work-")
log.info("working dir is %s", work_dir)
if (opts.fs_image or opts.no_virt) and not opts.disk_image:
# Create iso from a filesystem image
disk_img = opts.fs_image or disk_img
if not make_squashfs(opts, disk_img, work_dir):
log.error("squashfs.img creation failed")
sys.exit(1)
with Mount(disk_img, opts="loop") as mount_dir:
result_dir = make_livecd(opts, mount_dir, work_dir)
else:
# Create iso from a partitioned disk image
disk_img = opts.disk_image or disk_img
with PartitionMount(disk_img) as img_mount:
if img_mount and img_mount.mount_dir:
make_runtime(opts, img_mount.mount_dir, work_dir, calculate_disk_size(opts, ks)/1024.0)
result_dir = make_livecd(opts, img_mount.mount_dir, work_dir)
# --iso-only removes the extra build artifacts, keeping only the boot.iso
if opts.iso_only and result_dir:
boot_iso = joinpaths(result_dir, "images/boot.iso")
if not os.path.exists(boot_iso):
log.error("%s is missing, skipping --iso-only.", boot_iso)
else:
iso_dir = tempfile.mkdtemp(prefix="lmc-result-")
dest_file = joinpaths(iso_dir, opts.iso_name or "boot.iso")
shutil.move(boot_iso, dest_file)
shutil.rmtree(result_dir)
result_dir = iso_dir
# cleanup the mess
# cleanup work_dir?
if disk_img and not (opts.keep_image or opts.disk_image or opts.fs_image):
os.unlink(disk_img)
log.info("Disk image erased")
disk_img = None
elif opts.make_appliance:
if not opts.ks:
networks = []
else:
networks = ks.handler.network.network
make_appliance(opts.disk_image or disk_img, opts.app_name,
opts.app_template, opts.app_file, networks, opts.ram,
opts.vcpus or 1, opts.arch, opts.title, opts.project, opts.releasever)
elif opts.make_pxe_live:
work_dir = tempfile.mkdtemp(prefix="lmc-work-")
log.info("working dir is %s", work_dir)
disk_img = opts.fs_image or opts.disk_image or disk_img
log.debug("disk image is %s", disk_img)
result_dir = make_live_images(opts, work_dir, disk_img)
if result_dir is None:
log.error("Creating PXE live image failed.")
sys.exit(1)
if opts.result_dir != opts.tmp and result_dir:
copytree(result_dir, opts.result_dir, preserve=False)
shutil.rmtree(result_dir)
result_dir = None
log.info("SUMMARY") log.info("SUMMARY")
log.info("-------") log.info("-------")