Update local copy of lorax to current rhel7-branch

These are used when running tests, so catch-up with changes in lorax.
Includes adding rootfs-size, extra kernel args, and using locking to
prevent temporary files from being deleted by systemd tmpfiles handling.

Related: rhbz#1668520
Related: rhbz#1715116
Related: rhbz#1689314

src/pylorax/__init__.py

@@ -294,7 +294,8 @@ class Lorax(BaseLoraxClass):
             else:
                 logger.info("no BCJ filter for arch %s", self.arch.basearch)
         rb.create_runtime(joinpaths(installroot,runtime),
-                          compression=compression, compressargs=compressargs)
+                          compression=compression, compressargs=compressargs,
+                          size=size)
 
         logger.info("preparing to build output tree and boot images")
         treebuilder = TreeBuilder(product=self.product, arch=self.arch,

src/pylorax/creator.py

@@ -45,7 +45,7 @@ from pylorax.installer import InstallError, novirt_install, virt_install
 
 RUNTIME = "images/install.img"
 
-# Default parameters for rebuilding initramfs, override with --dracut-args
+# Default parameters for rebuilding initramfs, override with --dracut-arg
 DRACUT_DEFAULT = ["--xz", "--add", "livenet dmsquash-live convertfs pollcdrom",
                   "--omit", "plymouth", "--no-hostonly", "--no-early-microcode"]
 
@@ -341,7 +341,8 @@ def make_livecd(opts, mount_dir, work_dir):
     tb = TreeBuilder(product=product, arch=arch, domacboot=opts.domacboot,
                      inroot=mount_dir, outroot=work_dir,
                      runtime=RUNTIME, isolabel=isolabel,
-                     templatedir=joinpaths(opts.lorax_templates,"live/"))
+                     templatedir=joinpaths(opts.lorax_templates,"live/"),
+                     extra_boot_args=opts.extra_boot_args)
     log.info( "Rebuilding initrds" )
     if not opts.dracut_args:
         dracut_args = DRACUT_DEFAULT

src/pylorax/imgutils.py

@@ -102,6 +102,13 @@ def mkrootfsimg(rootdir, outfile, label, size=2, sysroot=""):
         fssize = None       # Let mkext4img figure out the needed size
 
     mkext4img(rootdir, outfile, label=label, size=fssize)
+    # Reset selinux context on new rootfs
+    with LoopDev(outfile) as loopdev:
+        with Mount(loopdev) as mnt:
+            cmd = [ "setfiles", "-e", "/proc", "-e", "/sys", "-e", "/dev", "-e", "/install",
+                    "/etc/selinux/targeted/contexts/files/file_contexts", "/"]
+            root = join(mnt, sysroot.lstrip("/"))
+            runcmd(cmd, root=root)
 
 def mkdiskfsimage(diskimage, fsimage, label="Anaconda"):
     """

src/pylorax/installer.py

@@ -240,14 +240,6 @@ def novirt_install(opts, disk_img, disk_size, repo_url, cancel_func=None):
     """
     Use Anaconda to install to a disk image
     """
-#    import selinux
-
-    # Set selinux to Permissive if it is Enforcing
-#    selinux_enforcing = False
-#    if selinux.is_selinux_enabled() and selinux.security_getenforce():
-#        selinux_enforcing = True
-#        selinux.security_setenforce(0)
-
     # Clean up /tmp/ from previous runs to prevent stale info from being used
     for path in ["/tmp/yum.repos.d/", "/tmp/yum.cache/", "/tmp/yum.root/", "/tmp/yum.pluginconf.d/"]:
         if os.path.isdir(path):
@@ -320,9 +312,6 @@ def novirt_install(opts, disk_img, disk_size, repo_url, cancel_func=None):
             log.debug("Removing loop device for %s", disk_img)
             loop_detach("/dev/"+get_loop_name(disk_img))
 
-#    if selinux_enforcing:
-#        selinux.security_setenforce(1)
-
     if rc:
         raise InstallError("novirt_install failed")
 

src/pylorax/ltmpl.py

@@ -69,11 +69,17 @@ class LoraxTemplate(object):
         # mako template now returns unicode strings
         lines = map(lambda line: line.encode("utf8"), lines)
 
-        # split with shlex and perform brace expansion
+        # split with shlex and perform brace expansion. This can fail, so we unroll the loop
-        lines = map(split_and_expand, lines)
+        # for better error reporting.
-
+        expanded_lines = []
-        self.lines = lines
+        try:
-        return lines
+            for line in lines:
+                expanded_lines.append(split_and_expand(line))
+        except Exception as e:
+            logger.error('shlex error processing "%s": %s', line, str(e))
+            raise
+        self.lines = expanded_lines
+        return expanded_lines
 
 def split_and_expand(line):
     return [exp for word in shlex.split(line) for exp in brace_expand(word)]

src/pylorax/treebuilder.py

@@ -173,7 +173,8 @@ class RuntimeBuilder(object):
 class TreeBuilder(object):
     '''Builds the arch-specific boot images.
     inroot should be the installtree root (the newly-built runtime dir)'''
-    def __init__(self, product, arch, inroot, outroot, runtime, isolabel, domacboot=False, doupgrade=True, templatedir=None, add_templates=None, add_template_vars=None, workdir=None):
+    def __init__(self, product, arch, inroot, outroot, runtime, isolabel, domacboot=False, doupgrade=True,
+                 templatedir=None, add_templates=None, add_template_vars=None, workdir=None, extra_boot_args=""):
 
         # NOTE: if you pass an arg named "runtime" to a mako template it'll
         # clobber some mako internal variables - hence "runtime_img".
@@ -182,7 +183,7 @@ class TreeBuilder(object):
                                inroot=inroot, outroot=outroot,
                                basearch=arch.basearch, libdir=arch.libdir,
                                isolabel=isolabel, udev=udev_escape, domacboot=domacboot, doupgrade=doupgrade,
-                               workdir=workdir)
+                               workdir=workdir, extra_boot_args=extra_boot_args)
         self._runner = LoraxTemplateRunner(inroot, outroot, templatedir=templatedir)
         self._runner.defaults = self.vars
         self.add_templates = add_templates or []

src/sbin/livemedia-creator

@@ -114,6 +114,9 @@ def lorax_parser():
     parser.add_argument( "--nomacboot", action="store_false", default=False,
                          dest="domacboot")
 
+    parser.add_argument("--extra-boot-args", default="", dest="extra_boot_args",
+                        help="Extra arguments to add to the bootloader kernel cmdline in the templates")
+
     image_group = parser.add_argument_group("disk/fs image arguments")
     image_group.add_argument( "--disk-image", type=os.path.abspath,
                         help="Path to disk image to use for creating final image" )

src/sbin/lorax

@@ -30,6 +30,9 @@ pylorax_log = logging.getLogger("pylorax")
 yum_log = logging.getLogger("yum")
 
 
+import atexit
+import fcntl
+from glob import glob
 import sys
 import os
 import shutil
@@ -46,6 +49,42 @@ from pylorax import DRACUT_DEFAULT, log_selinux_state
 
 VERSION = "{0}-{1}".format(os.path.basename(sys.argv[0]), pylorax.vernum)
 
+def exit_handler(tempdir):
+    """Handle cleanup of tmpdir, if it still exists
+    """
+    if not tempdir:
+        return
+    if os.path.exists(tempdir):
+        log.info("Cleaning up tempdir - %s", tempdir)
+        shutil.rmtree(tempdir)
+
+
+def remove_tempdirs():
+    """Delete all unlocked tempdirs under tempfile.gettempdir
+
+    When lorax crashes it can leave behind tempdirs, which cannot be cleaned up by
+    systemd-tmpfiles (SELinux restricts a complete cleanup).
+
+    So we lock them while in use and cleanup all the ones that are not locked
+    when lorax starts.
+    """
+    for d in glob(os.path.join(tempfile.gettempdir(), "lorax.*")):
+        if not os.path.isdir(d):
+            continue
+        try:
+            dir_fd = os.open(d, os.O_RDONLY|os.O_DIRECTORY)
+            fcntl.flock(dir_fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
+        except IOError:
+            # lock failed, skip this directory
+            os.close(dir_fd)
+            continue
+
+        # Lock succeeded, remove the directory
+        log.info("Removing old tempdir %s", d)
+        shutil.rmtree(d)
+        os.close(dir_fd)
+
+
 def setup_logging(opts):
     # Setup logging to console and to logfile
     log.setLevel(logging.DEBUG)
@@ -134,7 +173,7 @@ def main(args):
             action="store_false", default=True, dest="doupgrade")
     optional.add_option("--logfile", default="./lorax.log",
             help="Path to logfile")
-    optional.add_option("--tmp", default="/var/tmp",
+    optional.add_option("--tmp", default="/var/tmp/lorax",
             help="Top level temporary directory" )
     optional.add_option("--add-template", dest="add_templates",
             action="append", help="Additional template for runtime image",
@@ -150,6 +189,8 @@ def main(args):
             default=[])
     optional.add_option("--noverifyssl", action="store_true", default=False,
                         help="Do not verify SSL certificates")
+    optional.add_option("--rootfs-size", type=int, default=2,
+                        help="Size of root filesystem in GiB. Defaults to 2.")
 
     # dracut arguments
     dracut_group = OptionGroup(parser, "dracut arguments")
@@ -162,6 +203,7 @@ def main(args):
     # add the option groups to the parser
     parser.add_option_group(required)
     parser.add_option_group(optional)
+    parser.add_option_group(dracut_group)
 
     # add the show version option
     parser.add_option("-V", help="show program's version number and exit",
@@ -195,11 +237,21 @@ def main(args):
     setup_logging(opts)
 
     log.info("Lorax %s", pylorax.vernum)
+
+    if not os.path.exists(opts.tmp):
+        os.makedirs(opts.tmp)
+
     log_selinux_state()
     tempfile.tempdir = opts.tmp
 
+    # Remove any orphaned lorax tempdirs
+    remove_tempdirs()
+
     # create the temporary directory for lorax
-    tempdir = tempfile.mkdtemp(prefix="lorax.", dir=tempfile.gettempdir())
+    tempdir = tempfile.mkdtemp(prefix="lorax.")
+
+    # register an exit handler to cleanup the temporary directory
+    atexit.register(exit_handler, tempdir)
 
     # create the yumbase object
     installtree = os.path.join(tempdir, "installtree")
@@ -207,6 +259,10 @@ def main(args):
     yumtempdir = os.path.join(tempdir, "yum")
     os.mkdir(yumtempdir)
 
+    # Obtain an exclusive lock on the tempdir
+    dir_fd = os.open(tempdir, os.O_RDONLY|os.O_DIRECTORY)
+    fcntl.flock(dir_fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
+
     yb = get_yum_base_object(installtree, opts.source, opts.mirrorlist,
                              opts.repo_files,
                              yumtempdir, opts.proxy, opts.excludepkgs,
@@ -239,6 +295,7 @@ def main(args):
               workdir=tempdir, outputdir=outputdir, buildarch=opts.buildarch,
               volid=opts.volid, domacboot=opts.domacboot, doupgrade=opts.doupgrade,
               installpkgs=opts.installpkgs,
+              size=opts.rootfs_size,
               add_templates=opts.add_templates,
               add_template_vars=parsed_add_template_vars,
               add_arch_templates=opts.add_arch_templates,
@@ -246,6 +303,9 @@ def main(args):
               remove_temp=True,
               user_dracut_args=opts.dracut_args)
 
+    # Release the lock on the tempdir
+    os.close(dir_fd)
+
 
 def get_yum_base_object(installroot, repositories, mirrorlists=None, repo_files=None,
                         tempdir="/var/tmp", proxy=None, excludepkgs=None,