diff --git a/share/composer/live-iso.ks b/share/composer/live-iso.ks index f56ecc8a..aeecadde 100644 --- a/share/composer/live-iso.ks +++ b/share/composer/live-iso.ks @@ -34,9 +34,6 @@ zerombr # Partition clearing information clearpart --all # Disk partitioning information -part biosboot --size=1 -part / --fstype="ext4" --size=5000 -part swap --size=1000 %post # FIXME: it'd be better to get this installed from a package diff --git a/share/composer/partitioned-disk.ks b/share/composer/partitioned-disk.ks index d9d0b9b1..d65bdceb 100644 --- a/share/composer/partitioned-disk.ks +++ b/share/composer/partitioned-disk.ks @@ -29,9 +29,6 @@ bootloader --location=mbr zerombr # Partition clearing information clearpart --all -# Disk partitioning information -part / --fstype="ext4" --size=4000 -part swap --size=1000 %post # Remove root password diff --git a/share/composer/qcow2.ks b/share/composer/qcow2.ks index 242ca216..4ffb3534 100644 --- a/share/composer/qcow2.ks +++ b/share/composer/qcow2.ks @@ -29,9 +29,6 @@ bootloader --location=mbr zerombr # Partition clearing information clearpart --all -# Disk partitioning information -part / --fstype="ext4" --size=4000 -part swap --size=1000 %post # Remove root password diff --git a/src/pylorax/api/compose.py b/src/pylorax/api/compose.py index 04e61643..0efb7384 100644 --- a/src/pylorax/api/compose.py +++ b/src/pylorax/api/compose.py @@ -35,13 +35,18 @@ log = logging.getLogger("lorax-composer") import os from glob import glob +from math import ceil import pytoml as toml import shutil from uuid import uuid4 from pyanaconda.simpleconfig import SimpleConfigFile -from pylorax.api.projects import projects_depsolve, dep_nevra +# Use pykickstart to calculate disk image size +from pykickstart.parser import KickstartParser +from pykickstart.version import makeVersion, RHEL7 + +from pylorax.api.projects import projects_depsolve_with_size, dep_nevra from pylorax.api.projects import ProjectsError from pylorax.api.recipes import read_recipe_and_id from pylorax.imgutils import default_image_name @@ -118,11 +123,32 @@ def start_build(cfg, yumlock, gitlock, branch, recipe_name, compose_type, test_m deps = [] try: with yumlock.lock: - deps = projects_depsolve(yumlock.yb, projects) + (installed_size, deps) = projects_depsolve_with_size(yumlock.yb, projects, with_core=False) except ProjectsError as e: log.error("start_build depsolve: %s", str(e)) raise RuntimeError("Problem depsolving %s: %s" % (recipe["name"], str(e))) + # Read the kickstart template for this type + ks_template_path = joinpaths(share_dir, "composer", compose_type) + ".ks" + ks_template = open(ks_template_path, "r").read() + + # How much space will the packages in the default template take? + ks_version = makeVersion(RHEL7) + ks = KickstartParser(ks_version, errorsAreFatal=False, missingIncludeIsFatal=False) + ks.readKickstartFromString(ks_template+"\n%end\n") + try: + with yumlock.lock: + (template_size, _) = projects_depsolve_with_size(yumlock.yb, ks.handler.packages.packageList, + with_core=not ks.handler.packages.nocore) + except ProjectsError as e: + log.error("start_build depsolve: %s", str(e)) + raise RuntimeError("Problem depsolving %s: %s" % (recipe["name"], str(e))) + log.debug("installed_size = %d, template_size=%d", installed_size, template_size) + + # Minimum LMC disk size is 1GiB, and anaconda bumps the estimated size up by 35% (which doesn't always work). + installed_size = max(1024**3, int((installed_size+template_size) * 1.4)) + log.debug("/ partition size = %d", installed_size) + # Create the results directory build_id = str(uuid4()) results_dir = joinpaths(lib_dir, "results", build_id) @@ -144,16 +170,14 @@ def start_build(cfg, yumlock, gitlock, branch, recipe_name, compose_type, test_m with open(recipe_path, "w") as f: f.write(frozen_recipe.toml()) - # Read the kickstart template for this type and copy it into the results - ks_template_path = joinpaths(share_dir, "composer", compose_type) + ".ks" - shutil.copy(ks_template_path, results_dir) - ks_template = open(ks_template_path, "r").read() - # Write out the dependencies to the results dir deps_path = joinpaths(results_dir, "deps.toml") with open(deps_path, "w") as f: f.write(toml.dumps({"packages":deps}).encode("UTF-8")) + # Save a copy of the original kickstart + shutil.copy(ks_template_path, results_dir) + # Create the final kickstart with repos and package list ks_path = joinpaths(results_dir, "final-kickstart.ks") with open(ks_path, "w") as f: @@ -170,6 +194,9 @@ def start_build(cfg, yumlock, gitlock, branch, recipe_name, compose_type, test_m log.debug("repo composer-%s = %s", idx, ks_repo) f.write('repo --name="composer-%s" %s\n' % (idx, ks_repo)) + # Write the root partition and it's size in MB (rounded up) + f.write('part / --fstype="ext4" --size=%d\n' % ceil(installed_size / 1024**2)) + f.write(ks_template) for d in deps: diff --git a/src/pylorax/api/projects.py b/src/pylorax/api/projects.py index c8e167fe..35957cc5 100644 --- a/src/pylorax/api/projects.py +++ b/src/pylorax/api/projects.py @@ -212,6 +212,54 @@ def projects_depsolve(yb, project_names): yb.closeRpmDB() return deps +def estimate_size(packages, block_size=4096): + """Estimate the installed size of a package list + + :param packages: The packages to be installed + :type packages: list of TransactionMember objects + :param block_size: The block size to use for rounding up file sizes. + :type block_size: int + :returns: The estimated size of installed packages + :rtype: int + + Estimating actual requirements is difficult without the actual file sizes, which + yum doesn't provide access to. So use the file count and block size to estimate + a minimum size for each package. + """ + installed_size = 0 + for p in packages: + installed_size += len(p.po.filelist) * block_size + installed_size += p.po.installedsize + return installed_size + +def projects_depsolve_with_size(yb, project_names, with_core=True): + """Return the dependencies and installed size for a list of projects + + :param yb: yum base object + :type yb: YumBase + :param project_names: The projects to find the dependencies for + :type project_names: List of Strings + :returns: installed size and a list of NEVRA's of the project and its dependencies + :rtype: tuple of (int, list of dicts) + """ + try: + # This resets the transaction + yb.closeRpmDB() + for p in project_names: + yb.install(pattern=p) + if with_core: + yb.selectGroup("core", group_package_types=['mandatory', 'default', 'optional']) + (rc, msg) = yb.buildTransaction() + if rc not in [0, 1, 2]: + raise ProjectsError("There was a problem depsolving %s: %s" % (project_names, msg)) + yb.tsInfo.makelists() + installed_size = estimate_size(yb.tsInfo.installed + yb.tsInfo.depinstalled) + deps = sorted(map(tm_to_dep, yb.tsInfo.installed + yb.tsInfo.depinstalled), key=lambda p: p["name"].lower()) + except YumBaseError as e: + raise ProjectsError("There was a problem depsolving %s: %s" % (project_names, str(e))) + finally: + yb.closeRpmDB() + return (installed_size, deps) def modules_list(yb, module_names): """Return a list of modules