Removed the fixed partition size from composer ks templates

The default size is always going to be wrong, so try to estimate a more
reasonable amount of space. This is more complicated than you would
expect, yum's installedsize doesn't take into account the block size of
the filesystem, nor any extra artifacts generated by pre/post scripts.

So in the end we end up with a minimum image size of 1GiB, a partition
that is 40% larger than the estimated space needed, and a disk image
that increases size in 1GiB increments. This is still better than having
a fixed 4GiB / partition that was either too large or too small.

share/composer/live-iso.ks

@@ -34,9 +34,6 @@ zerombr
 # Partition clearing information
 clearpart --all
 # Disk partitioning information
-part biosboot --size=1
-part / --fstype="ext4" --size=5000
-part swap --size=1000
 
 %post
 # FIXME: it'd be better to get this installed from a package

share/composer/partitioned-disk.ks

@@ -29,9 +29,6 @@ bootloader --location=mbr
 zerombr
 # Partition clearing information
 clearpart --all
-# Disk partitioning information
-part / --fstype="ext4" --size=4000
-part swap --size=1000
 
 %post
 # Remove root password

share/composer/qcow2.ks

@@ -29,9 +29,6 @@ bootloader --location=mbr
 zerombr
 # Partition clearing information
 clearpart --all
-# Disk partitioning information
-part / --fstype="ext4" --size=4000
-part swap --size=1000
 
 %post
 # Remove root password

src/pylorax/api/compose.py

@@ -35,13 +35,18 @@ log = logging.getLogger("lorax-composer")
 
 import os
 from glob import glob
+from math import ceil
 import pytoml as toml
 import shutil
 from uuid import uuid4
 
 from pyanaconda.simpleconfig import SimpleConfigFile
 
-from pylorax.api.projects import projects_depsolve, dep_nevra
+# Use pykickstart to calculate disk image size
+from pykickstart.parser import KickstartParser
+from pykickstart.version import makeVersion, RHEL7
+
+from pylorax.api.projects import projects_depsolve_with_size, dep_nevra
 from pylorax.api.projects import ProjectsError
 from pylorax.api.recipes import read_recipe_and_id
 from pylorax.imgutils import default_image_name
@@ -118,11 +123,32 @@ def start_build(cfg, yumlock, gitlock, branch, recipe_name, compose_type, test_m
     deps = []
     try:
         with yumlock.lock:
-            deps = projects_depsolve(yumlock.yb, projects)
+            (installed_size, deps) = projects_depsolve_with_size(yumlock.yb, projects, with_core=False)
     except ProjectsError as e:
         log.error("start_build depsolve: %s", str(e))
         raise RuntimeError("Problem depsolving %s: %s" % (recipe["name"], str(e)))
 
+    # Read the kickstart template for this type
+    ks_template_path = joinpaths(share_dir, "composer", compose_type) + ".ks"
+    ks_template = open(ks_template_path, "r").read()
+
+    # How much space will the packages in the default template take?
+    ks_version = makeVersion(RHEL7)
+    ks = KickstartParser(ks_version, errorsAreFatal=False, missingIncludeIsFatal=False)
+    ks.readKickstartFromString(ks_template+"\n%end\n")
+    try:
+        with yumlock.lock:
+            (template_size, _) = projects_depsolve_with_size(yumlock.yb, ks.handler.packages.packageList,
+                                                             with_core=not ks.handler.packages.nocore)
+    except ProjectsError as e:
+        log.error("start_build depsolve: %s", str(e))
+        raise RuntimeError("Problem depsolving %s: %s" % (recipe["name"], str(e)))
+    log.debug("installed_size = %d, template_size=%d", installed_size, template_size)
+
+    # Minimum LMC disk size is 1GiB, and anaconda bumps the estimated size up by 35% (which doesn't always work).
+    installed_size = max(1024**3, int((installed_size+template_size) * 1.4))
+    log.debug("/ partition size = %d", installed_size)
+
     # Create the results directory
     build_id = str(uuid4())
     results_dir = joinpaths(lib_dir, "results", build_id)
@@ -144,16 +170,14 @@ def start_build(cfg, yumlock, gitlock, branch, recipe_name, compose_type, test_m
     with open(recipe_path, "w") as f:
         f.write(frozen_recipe.toml())
 
-    # Read the kickstart template for this type and copy it into the results
-    ks_template_path = joinpaths(share_dir, "composer", compose_type) + ".ks"
-    shutil.copy(ks_template_path, results_dir)
-    ks_template = open(ks_template_path, "r").read()
-
     # Write out the dependencies to the results dir
     deps_path = joinpaths(results_dir, "deps.toml")
     with open(deps_path, "w") as f:
         f.write(toml.dumps({"packages":deps}).encode("UTF-8"))
 
+    # Save a copy of the original kickstart
+    shutil.copy(ks_template_path, results_dir)
+
     # Create the final kickstart with repos and package list
     ks_path = joinpaths(results_dir, "final-kickstart.ks")
     with open(ks_path, "w") as f:
@@ -170,6 +194,9 @@ def start_build(cfg, yumlock, gitlock, branch, recipe_name, compose_type, test_m
             log.debug("repo composer-%s = %s", idx, ks_repo)
             f.write('repo --name="composer-%s" %s\n' % (idx, ks_repo))
 
+        # Write the root partition and it's size in MB (rounded up)
+        f.write('part / --fstype="ext4" --size=%d\n' % ceil(installed_size / 1024**2))
+
         f.write(ks_template)
 
         for d in deps:

src/pylorax/api/projects.py

@@ -212,6 +212,54 @@ def projects_depsolve(yb, project_names):
         yb.closeRpmDB()
     return deps
 
+def estimate_size(packages, block_size=4096):
+    """Estimate the installed size of a package list
+
+    :param packages: The packages to be installed
+    :type packages: list of TransactionMember objects
+    :param block_size: The block size to use for rounding up file sizes.
+    :type block_size: int
+    :returns: The estimated size of installed packages
+    :rtype: int
+
+    Estimating actual requirements is difficult without the actual file sizes, which
+    yum doesn't provide access to. So use the file count and block size to estimate
+    a minimum size for each package.
+    """
+    installed_size = 0
+    for p in packages:
+        installed_size += len(p.po.filelist) * block_size
+        installed_size += p.po.installedsize
+    return installed_size
+
+def projects_depsolve_with_size(yb, project_names, with_core=True):
+    """Return the dependencies and installed size for a list of projects
+
+    :param yb: yum base object
+    :type yb: YumBase
+    :param project_names: The projects to find the dependencies for
+    :type project_names: List of Strings
+    :returns: installed size and a list of NEVRA's of the project and its dependencies
+    :rtype: tuple of (int, list of dicts)
+    """
+    try:
+        # This resets the transaction
+        yb.closeRpmDB()
+        for p in project_names:
+            yb.install(pattern=p)
+        if with_core:
+            yb.selectGroup("core", group_package_types=['mandatory', 'default', 'optional'])
+        (rc, msg) = yb.buildTransaction()
+        if rc not in [0, 1, 2]:
+            raise ProjectsError("There was a problem depsolving %s: %s" % (project_names, msg))
+        yb.tsInfo.makelists()
+        installed_size = estimate_size(yb.tsInfo.installed + yb.tsInfo.depinstalled)
+        deps = sorted(map(tm_to_dep, yb.tsInfo.installed + yb.tsInfo.depinstalled), key=lambda p: p["name"].lower())
+    except YumBaseError as e:
+        raise ProjectsError("There was a problem depsolving %s: %s" % (project_names, str(e)))
+    finally:
+        yb.closeRpmDB()
+    return (installed_size, deps)
 
 def modules_list(yb, module_names):
     """Return a list of modules