b2f5fe2f60
The default size is always going to be wrong, so try to estimate a more reasonable amount of space. This is more complicated than you would expect, yum's installedsize doesn't take into account the block size of the filesystem, nor any extra artifacts generated by pre/post scripts. So in the end we end up with a minimum image size of 1GiB, a partition that is 40% larger than the estimated space needed, and a disk image that increases size in 1GiB increments. This is still better than having a fixed 4GiB / partition that was either too large or too small.
312 lines
10 KiB
Python
312 lines
10 KiB
Python
#
|
|
# Copyright (C) 2017 Red Hat, Inc.
|
|
#
|
|
# This program is free software; you can redistribute it and/or modify
|
|
# it under the terms of the GNU General Public License as published by
|
|
# the Free Software Foundation; either version 2 of the License, or
|
|
# (at your option) any later version.
|
|
#
|
|
# This program is distributed in the hope that it will be useful,
|
|
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
# GNU General Public License for more details.
|
|
#
|
|
# You should have received a copy of the GNU General Public License
|
|
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
|
#
|
|
import logging
|
|
log = logging.getLogger("lorax-composer")
|
|
|
|
import time
|
|
|
|
from yum.Errors import YumBaseError
|
|
|
|
TIME_FORMAT = "%Y-%m-%dT%H:%M:%S"
|
|
|
|
|
|
class ProjectsError(Exception):
|
|
pass
|
|
|
|
|
|
def api_time(t):
|
|
"""Convert time since epoch to a string
|
|
|
|
:param t: Seconds since epoch
|
|
:type t: int
|
|
:returns: Time string
|
|
:rtype: str
|
|
"""
|
|
return time.strftime(TIME_FORMAT, time.localtime(t))
|
|
|
|
|
|
def api_changelog(changelog):
|
|
"""Convert the changelog to a string
|
|
|
|
:param changelog: A list of time, author, string tuples.
|
|
:type changelog: tuple
|
|
:returns: The most recent changelog text or ""
|
|
:rtype: str
|
|
|
|
This returns only the most recent changelog entry.
|
|
"""
|
|
try:
|
|
entry = changelog[0][2]
|
|
except IndexError:
|
|
entry = ""
|
|
return entry
|
|
|
|
|
|
def yaps_to_project(yaps):
|
|
"""Extract the details from a YumAvailablePackageSqlite object
|
|
|
|
:param yaps: Yum object with package details
|
|
:type yaps: YumAvailablePackageSqlite
|
|
:returns: A dict with the name, summary, description, and url.
|
|
:rtype: dict
|
|
|
|
upstream_vcs is hard-coded to UPSTREAM_VCS
|
|
"""
|
|
return {"name": yaps.name,
|
|
"summary": yaps.summary,
|
|
"description": yaps.description,
|
|
"homepage": yaps.url,
|
|
"upstream_vcs": "UPSTREAM_VCS"}
|
|
|
|
|
|
def yaps_to_project_info(yaps):
|
|
"""Extract the details from a YumAvailablePackageSqlite object
|
|
|
|
:param yaps: Yum object with package details
|
|
:type yaps: YumAvailablePackageSqlite
|
|
:returns: A dict with the project details, as well as epoch, release, arch, build_time, changelog, ...
|
|
:rtype: dict
|
|
|
|
metadata entries are hard-coded to {}
|
|
"""
|
|
build = {"epoch": int(yaps.epoch),
|
|
"release": yaps.release,
|
|
"arch": yaps.arch,
|
|
"build_time": api_time(yaps.buildtime),
|
|
"changelog": api_changelog(yaps.returnChangelog()),
|
|
"build_config_ref": "BUILD_CONFIG_REF",
|
|
"build_env_ref": "BUILD_ENV_REF",
|
|
"metadata": {},
|
|
"source": {"license": yaps.license,
|
|
"version": yaps.version,
|
|
"source_ref": "SOURCE_REF",
|
|
"metadata": {}}}
|
|
|
|
return {"name": yaps.name,
|
|
"summary": yaps.summary,
|
|
"description": yaps.description,
|
|
"homepage": yaps.url,
|
|
"upstream_vcs": "UPSTREAM_VCS",
|
|
"builds": [build]}
|
|
|
|
|
|
def tm_to_dep(tm):
|
|
"""Extract the info from a TransactionMember object
|
|
|
|
:param tm: A Yum transaction object
|
|
:type tm: TransactionMember
|
|
:returns: A dict with name, epoch, version, release, arch
|
|
:rtype: dict
|
|
"""
|
|
return {"name": tm.name,
|
|
"epoch": int(tm.epoch),
|
|
"version": tm.version,
|
|
"release": tm.release,
|
|
"arch": tm.arch}
|
|
|
|
|
|
def yaps_to_module(yaps):
|
|
"""Extract the name from a YumAvailablePackageSqlite object
|
|
|
|
:param yaps: Yum object with package details
|
|
:type yaps: YumAvailablePackageSqlite
|
|
:returns: A dict with name, and group_type
|
|
:rtype: dict
|
|
|
|
group_type is hard-coded to "rpm"
|
|
"""
|
|
return {"name": yaps.name,
|
|
"group_type": "rpm"}
|
|
|
|
|
|
def dep_evra(dep):
|
|
"""Return the epoch:version-release.arch for the dep
|
|
|
|
:param dep: dependency dict
|
|
:type dep: dict
|
|
:returns: epoch:version-release.arch
|
|
:rtype: str
|
|
"""
|
|
if dep["epoch"] == 0:
|
|
return dep["version"]+"-"+dep["release"]+"."+dep["arch"]
|
|
else:
|
|
return str(dep["epoch"])+":"+dep["version"]+"-"+dep["release"]+"."+dep["arch"]
|
|
|
|
def dep_nevra(dep):
|
|
"""Return the name-epoch:version-release.arch"""
|
|
return dep["name"]+"-"+dep_evra(dep)
|
|
|
|
def projects_list(yb):
|
|
"""Return a list of projects
|
|
|
|
:param yb: yum base object
|
|
:type yb: YumBase
|
|
:returns: List of project info dicts with name, summary, description, homepage, upstream_vcs
|
|
:rtype: list of dicts
|
|
"""
|
|
try:
|
|
ybl = yb.doPackageLists(pkgnarrow="available", showdups=False)
|
|
except YumBaseError as e:
|
|
raise ProjectsError("There was a problem listing projects: %s" % str(e))
|
|
finally:
|
|
yb.closeRpmDB()
|
|
return sorted(map(yaps_to_project, ybl.available), key=lambda p: p["name"].lower())
|
|
|
|
|
|
def projects_info(yb, project_names):
|
|
"""Return details about specific projects
|
|
|
|
:param yb: yum base object
|
|
:type yb: YumBase
|
|
:param project_names: List of names of projects to get info about
|
|
:type project_names: str
|
|
:returns: List of project info dicts with yaps_to_project as well as epoch, version, release, etc.
|
|
:rtype: list of dicts
|
|
"""
|
|
try:
|
|
ybl = yb.doPackageLists(pkgnarrow="available", patterns=project_names, showdups=False)
|
|
except YumBaseError as e:
|
|
raise ProjectsError("There was a problem with info for %s: %s" % (project_names, str(e)))
|
|
finally:
|
|
yb.closeRpmDB()
|
|
return sorted(map(yaps_to_project_info, ybl.available), key=lambda p: p["name"].lower())
|
|
|
|
|
|
def projects_depsolve(yb, project_names):
|
|
"""Return the dependencies for a list of projects
|
|
|
|
:param yb: yum base object
|
|
:type yb: YumBase
|
|
:param project_names: The projects to find the dependencies for
|
|
:type project_names: List of Strings
|
|
:returns: NEVRA's of the project and its dependencies
|
|
:rtype: list of dicts
|
|
"""
|
|
try:
|
|
# This resets the transaction
|
|
yb.closeRpmDB()
|
|
for p in project_names:
|
|
yb.install(pattern=p)
|
|
(rc, msg) = yb.buildTransaction()
|
|
if rc not in [0, 1, 2]:
|
|
raise ProjectsError("There was a problem depsolving %s: %s" % (project_names, msg))
|
|
yb.tsInfo.makelists()
|
|
deps = sorted(map(tm_to_dep, yb.tsInfo.installed + yb.tsInfo.depinstalled), key=lambda p: p["name"].lower())
|
|
except YumBaseError as e:
|
|
raise ProjectsError("There was a problem depsolving %s: %s" % (project_names, str(e)))
|
|
finally:
|
|
yb.closeRpmDB()
|
|
return deps
|
|
|
|
def estimate_size(packages, block_size=4096):
|
|
"""Estimate the installed size of a package list
|
|
|
|
:param packages: The packages to be installed
|
|
:type packages: list of TransactionMember objects
|
|
:param block_size: The block size to use for rounding up file sizes.
|
|
:type block_size: int
|
|
:returns: The estimated size of installed packages
|
|
:rtype: int
|
|
|
|
Estimating actual requirements is difficult without the actual file sizes, which
|
|
yum doesn't provide access to. So use the file count and block size to estimate
|
|
a minimum size for each package.
|
|
"""
|
|
installed_size = 0
|
|
for p in packages:
|
|
installed_size += len(p.po.filelist) * block_size
|
|
installed_size += p.po.installedsize
|
|
return installed_size
|
|
|
|
def projects_depsolve_with_size(yb, project_names, with_core=True):
|
|
"""Return the dependencies and installed size for a list of projects
|
|
|
|
:param yb: yum base object
|
|
:type yb: YumBase
|
|
:param project_names: The projects to find the dependencies for
|
|
:type project_names: List of Strings
|
|
:returns: installed size and a list of NEVRA's of the project and its dependencies
|
|
:rtype: tuple of (int, list of dicts)
|
|
"""
|
|
try:
|
|
# This resets the transaction
|
|
yb.closeRpmDB()
|
|
for p in project_names:
|
|
yb.install(pattern=p)
|
|
if with_core:
|
|
yb.selectGroup("core", group_package_types=['mandatory', 'default', 'optional'])
|
|
(rc, msg) = yb.buildTransaction()
|
|
if rc not in [0, 1, 2]:
|
|
raise ProjectsError("There was a problem depsolving %s: %s" % (project_names, msg))
|
|
yb.tsInfo.makelists()
|
|
installed_size = estimate_size(yb.tsInfo.installed + yb.tsInfo.depinstalled)
|
|
deps = sorted(map(tm_to_dep, yb.tsInfo.installed + yb.tsInfo.depinstalled), key=lambda p: p["name"].lower())
|
|
except YumBaseError as e:
|
|
raise ProjectsError("There was a problem depsolving %s: %s" % (project_names, str(e)))
|
|
finally:
|
|
yb.closeRpmDB()
|
|
return (installed_size, deps)
|
|
|
|
def modules_list(yb, module_names):
|
|
"""Return a list of modules
|
|
|
|
:param yb: yum base object
|
|
:type yb: YumBase
|
|
:param offset: Number of modules to skip
|
|
:type limit: int
|
|
:param limit: Maximum number of modules to return
|
|
:type limit: int
|
|
:returns: List of module information and total count
|
|
:rtype: tuple of a list of dicts and an Int
|
|
|
|
Modules don't exist in RHEL7 so this only returns projects
|
|
and sets the type to "rpm"
|
|
"""
|
|
try:
|
|
ybl = yb.doPackageLists(pkgnarrow="available", patterns=module_names, showdups=False)
|
|
except YumBaseError as e:
|
|
raise ProjectsError("There was a problem listing modules: %s" % str(e))
|
|
finally:
|
|
yb.closeRpmDB()
|
|
return sorted(map(yaps_to_module, ybl.available), key=lambda p: p["name"].lower())
|
|
|
|
|
|
def modules_info(yb, module_names):
|
|
"""Return details about a module, including dependencies
|
|
|
|
:param yb: yum base object
|
|
:type yb: YumBase
|
|
:param module_names: Names of the modules to get info about
|
|
:type module_names: str
|
|
:returns: List of dicts with module details and dependencies.
|
|
:rtype: list of dicts
|
|
"""
|
|
try:
|
|
# Get the info about each module
|
|
ybl = yb.doPackageLists(pkgnarrow="available", patterns=module_names, showdups=False)
|
|
except YumBaseError as e:
|
|
raise ProjectsError("There was a problem with info for %s: %s" % (module_names, str(e)))
|
|
finally:
|
|
yb.closeRpmDB()
|
|
|
|
modules = sorted(map(yaps_to_project, ybl.available), key=lambda p: p["name"].lower())
|
|
# Add the dependency info to each one
|
|
for module in modules:
|
|
module["dependencies"] = projects_depsolve(yb, [module["name"]])
|
|
|
|
return modules
|