f9a6c8418f
The schema is written in Python to reduce duplication. When configuration is loaded, the validation checks if it's correct and fills in default values. There is a custom extension to the schema to report deprecated options. The config dependencies are implemented as a separate pass. While it's technically possible to express the dependencies in the schema itself, the error messages are not very helpful and it makes the schema much harder to read. Phases no longer define `config_options`. New options should be added to the schema. Since the default values are populated automatically during validation, there is no need to duplicate them into the code. The `pungi-config-validate` script is updated to use the schema and report errors even for deeply nested fields. The dependencies are updated: pungi now depends on `python-jsonschema` (which is already available in Fedora). Signed-off-by: Lubomír Sedlář <lsedlar@redhat.com>
585 lines
20 KiB
Python
585 lines
20 KiB
Python
# -*- coding: utf-8 -*-
|
|
|
|
|
|
# This program is free software; you can redistribute it and/or modify
|
|
# it under the terms of the GNU General Public License as published by
|
|
# the Free Software Foundation; version 2 of the License.
|
|
#
|
|
# This program is distributed in the hope that it will be useful,
|
|
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
# GNU Library General Public License for more details.
|
|
#
|
|
# You should have received a copy of the GNU General Public License
|
|
# along with this program; if not, write to the Free Software
|
|
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
|
|
|
|
|
|
__all__ = (
|
|
"Paths",
|
|
)
|
|
|
|
|
|
import errno
|
|
import os
|
|
|
|
from pungi.util import makedirs
|
|
|
|
|
|
def translate_path(compose, path):
|
|
"""
|
|
@param compose - required for access to config
|
|
@param path
|
|
"""
|
|
normpath = os.path.normpath(path)
|
|
mapping = compose.conf["translate_paths"]
|
|
|
|
for prefix, newvalue in mapping:
|
|
prefix = os.path.normpath(prefix)
|
|
if normpath.startswith(prefix):
|
|
# We can't call os.path.normpath on result since it is not actually
|
|
# a path - http:// would get changed to http:/ and so on.
|
|
# Only the first occurance should be replaced.
|
|
return normpath.replace(prefix, newvalue, 1)
|
|
|
|
return normpath
|
|
|
|
|
|
class Paths(object):
|
|
def __init__(self, compose):
|
|
paths_module_name = compose.conf.get("paths_module")
|
|
if paths_module_name:
|
|
# custom paths
|
|
compose.log_info("Using custom paths from module %s" % paths_module_name)
|
|
paths_module = __import__(paths_module_name, globals(), locals(), ["LogPaths", "WorkPaths", "ComposePaths"])
|
|
self.compose = paths_module.ComposePaths(compose)
|
|
self.log = paths_module.LogPaths(compose)
|
|
self.work = paths_module.WorkPaths(compose)
|
|
else:
|
|
# default paths
|
|
self.compose = ComposePaths(compose)
|
|
self.log = LogPaths(compose)
|
|
self.work = WorkPaths(compose)
|
|
# self.metadata ?
|
|
|
|
|
|
class LogPaths(object):
|
|
def __init__(self, compose):
|
|
self.compose = compose
|
|
|
|
def topdir(self, arch=None, create_dir=True):
|
|
"""
|
|
Examples:
|
|
log/global
|
|
log/x86_64
|
|
"""
|
|
arch = arch or "global"
|
|
path = os.path.join(self.compose.topdir, "logs", arch)
|
|
if create_dir:
|
|
makedirs(path)
|
|
return path
|
|
|
|
def log_file(self, arch, log_name, create_dir=True):
|
|
arch = arch or "global"
|
|
if log_name.endswith(".log"):
|
|
log_name = log_name[:-4]
|
|
return os.path.join(self.topdir(arch, create_dir=create_dir), "%s.%s.log" % (log_name, arch))
|
|
|
|
|
|
class WorkPaths(object):
|
|
def __init__(self, compose):
|
|
self.compose = compose
|
|
|
|
def topdir(self, arch=None, create_dir=True):
|
|
"""
|
|
Examples:
|
|
work/global
|
|
work/x86_64
|
|
"""
|
|
arch = arch or "global"
|
|
path = os.path.join(self.compose.topdir, "work", arch)
|
|
if create_dir:
|
|
makedirs(path)
|
|
return path
|
|
|
|
def variants_file(self, arch=None, create_dir=True):
|
|
"""
|
|
Examples:
|
|
work/global/variants.xml
|
|
"""
|
|
arch = "global"
|
|
path = os.path.join(self.topdir(arch, create_dir=create_dir), "variants.xml")
|
|
return path
|
|
|
|
def comps(self, arch=None, variant=None, create_dir=True):
|
|
"""
|
|
Examples:
|
|
work/x86_64/comps/comps-86_64.xml
|
|
work/x86_64/comps/comps-Server.x86_64.xml
|
|
"""
|
|
arch = arch or "global"
|
|
if variant is None:
|
|
file_name = "comps-%s.xml" % arch
|
|
else:
|
|
file_name = "comps-%s.%s.xml" % (variant.uid, arch)
|
|
path = os.path.join(self.topdir(arch, create_dir=create_dir), "comps")
|
|
if create_dir:
|
|
makedirs(path)
|
|
path = os.path.join(path, file_name)
|
|
return path
|
|
|
|
def pungi_conf(self, arch=None, variant=None, create_dir=True):
|
|
"""
|
|
Examples:
|
|
work/x86_64/pungi/x86_64.conf
|
|
work/x86_64/pungi/Server.x86_64.conf
|
|
"""
|
|
arch = arch or "global"
|
|
if variant is None:
|
|
file_name = "%s.conf" % arch
|
|
else:
|
|
file_name = "%s.%s.conf" % (variant.uid, arch)
|
|
path = os.path.join(self.topdir(arch, create_dir=create_dir), "pungi")
|
|
if create_dir:
|
|
makedirs(path)
|
|
path = os.path.join(path, file_name)
|
|
return path
|
|
|
|
def pungi_log(self, arch=None, variant=None, create_dir=True):
|
|
"""
|
|
Examples:
|
|
work/x86_64/pungi/x86_64.log
|
|
work/x86_64/pungi/Server.x86_64.log
|
|
"""
|
|
path = self.pungi_conf(arch, variant, create_dir=create_dir)
|
|
path = path[:-5] + ".log"
|
|
return path
|
|
|
|
def pungi_cache_dir(self, arch, variant=None, create_dir=True):
|
|
"""
|
|
Examples:
|
|
work/global/pungi-cache
|
|
"""
|
|
# WARNING: Using the same cache dir with repos of the same names may lead to a race condition
|
|
# We should use per arch variant cache dirs to workaround this.
|
|
path = os.path.join(self.topdir(arch, create_dir=create_dir), "pungi-cache")
|
|
if variant:
|
|
path = os.path.join(path, variant.uid)
|
|
if create_dir:
|
|
makedirs(path)
|
|
return path
|
|
|
|
def comps_repo(self, arch=None, create_dir=True):
|
|
"""
|
|
Examples:
|
|
work/x86_64/comps-repo
|
|
work/global/comps-repo
|
|
"""
|
|
arch = arch or "global"
|
|
path = os.path.join(self.topdir(arch, create_dir=create_dir), "comps_repo")
|
|
if create_dir:
|
|
makedirs(path)
|
|
return path
|
|
|
|
def arch_repo(self, arch=None, create_dir=True):
|
|
"""
|
|
Examples:
|
|
work/x86_64/repo
|
|
work/global/repo
|
|
"""
|
|
arch = arch or "global"
|
|
path = os.path.join(self.topdir(arch, create_dir=create_dir), "repo")
|
|
if create_dir:
|
|
makedirs(path)
|
|
return path
|
|
|
|
def package_list(self, arch=None, variant=None, pkg_type=None, create_dir=True):
|
|
"""
|
|
Examples:
|
|
work/x86_64/package_list/x86_64.conf
|
|
work/x86_64/package_list/Server.x86_64.conf
|
|
work/x86_64/package_list/Server.x86_64.rpm.conf
|
|
"""
|
|
arch = arch or "global"
|
|
if variant is not None:
|
|
file_name = "%s.%s" % (variant, arch)
|
|
else:
|
|
file_name = "%s" % arch
|
|
if pkg_type is not None:
|
|
file_name += ".%s" % pkg_type
|
|
file_name += ".conf"
|
|
path = os.path.join(self.topdir(arch, create_dir=create_dir), "package_list")
|
|
if create_dir:
|
|
makedirs(path)
|
|
path = os.path.join(path, file_name)
|
|
return path
|
|
|
|
def pungi_download_dir(self, arch, create_dir=True):
|
|
"""
|
|
Examples:
|
|
work/x86_64/pungi_download
|
|
"""
|
|
path = os.path.join(self.topdir(arch, create_dir=create_dir), "pungi_download")
|
|
if create_dir:
|
|
makedirs(path)
|
|
return path
|
|
|
|
def buildinstall_dir(self, arch, create_dir=True):
|
|
"""
|
|
Examples:
|
|
work/x86_64/buildinstall
|
|
"""
|
|
if arch == "global":
|
|
raise RuntimeError("Global buildinstall dir makes no sense.")
|
|
path = os.path.join(self.topdir(arch, create_dir=create_dir), "buildinstall")
|
|
return path
|
|
|
|
def extra_files_dir(self, arch, variant, create_dir=True):
|
|
"""
|
|
Examples:
|
|
work/x86_64/Server/extra-files
|
|
"""
|
|
if arch == "global":
|
|
raise RuntimeError("Global extra files dir makes no sense.")
|
|
path = os.path.join(self.topdir(arch, create_dir=create_dir), variant.uid, "extra-files")
|
|
if create_dir:
|
|
makedirs(path)
|
|
return path
|
|
|
|
def repo_package_list(self, arch, variant, pkg_type=None, create_dir=True):
|
|
"""
|
|
Examples:
|
|
work/x86_64/repo_package_list/Server.x86_64.rpm.conf
|
|
"""
|
|
file_name = "%s.%s" % (variant.uid, arch)
|
|
if pkg_type is not None:
|
|
file_name += ".%s" % pkg_type
|
|
file_name += ".conf"
|
|
path = os.path.join(self.topdir(arch, create_dir=create_dir), "repo_package_list")
|
|
if create_dir:
|
|
makedirs(path)
|
|
path = os.path.join(path, file_name)
|
|
return path
|
|
|
|
def product_img(self, variant, create_dir=True):
|
|
"""
|
|
Examples:
|
|
work/global/product-Server.img
|
|
"""
|
|
file_name = "product-%s.img" % variant
|
|
path = self.topdir(arch="global", create_dir=create_dir)
|
|
path = os.path.join(path, file_name)
|
|
return path
|
|
|
|
def iso_dir(self, arch, filename, create_dir=True):
|
|
"""
|
|
Examples:
|
|
work/x86_64/iso/Project-1.0-20151203.0-Client-x86_64-dvd1.iso
|
|
"""
|
|
path = os.path.join(self.topdir(arch, create_dir=create_dir), "iso", filename)
|
|
if create_dir:
|
|
makedirs(path)
|
|
return path
|
|
|
|
def tmp_dir(self, arch, variant=None, create_dir=True):
|
|
"""
|
|
Examples:
|
|
work/x86_64/tmp
|
|
work/x86_64/tmp-Server
|
|
"""
|
|
dir_name = "tmp"
|
|
if variant:
|
|
dir_name += "-%s" % variant.uid
|
|
path = os.path.join(self.topdir(arch, create_dir=create_dir), dir_name)
|
|
if create_dir:
|
|
makedirs(path)
|
|
return path
|
|
|
|
def product_id(self, arch, variant, create_dir=True):
|
|
"""
|
|
Examples:
|
|
work/x86_64/product_id/productid-Server.x86_64.pem/productid
|
|
"""
|
|
# file_name = "%s.%s.pem" % (variant, arch)
|
|
# HACK: modifyrepo doesn't handle renames -> $dir/productid
|
|
file_name = "productid"
|
|
path = os.path.join(self.topdir(arch, create_dir=create_dir), "product_id", "%s.%s.pem" % (variant, arch))
|
|
if create_dir:
|
|
makedirs(path)
|
|
path = os.path.join(path, file_name)
|
|
return path
|
|
|
|
def image_build_dir(self, variant, create_dir=True):
|
|
"""
|
|
@param variant
|
|
@param create_dir=True
|
|
|
|
Examples:
|
|
work/image-build/Server
|
|
"""
|
|
path = os.path.join(self.topdir('image-build', create_dir=create_dir), variant.uid)
|
|
if create_dir:
|
|
makedirs(path)
|
|
return path
|
|
|
|
def image_build_conf(self, variant, image_name, image_type, create_dir=True):
|
|
"""
|
|
@param variant
|
|
@param image-name
|
|
@param image-type (e.g docker)
|
|
@param create_dir=True
|
|
|
|
Examples:
|
|
work/image-build/Server/docker_rhel-server-docker.cfg
|
|
"""
|
|
path = os.path.join(self.image_build_dir(variant), "%s_%s.cfg" % (image_type, image_name))
|
|
return path
|
|
|
|
|
|
class ComposePaths(object):
|
|
def __init__(self, compose):
|
|
self.compose = compose
|
|
# TODO: TREES?
|
|
|
|
def topdir(self, arch=None, variant=None, create_dir=True, relative=False):
|
|
"""
|
|
Examples:
|
|
compose
|
|
compose/Server/x86_64
|
|
"""
|
|
if bool(arch) != bool(variant):
|
|
raise TypeError("topdir(): either none or 2 arguments are expected")
|
|
|
|
path = ""
|
|
if not relative:
|
|
path = os.path.join(self.compose.topdir, "compose")
|
|
|
|
if arch or variant:
|
|
if variant.type == "addon":
|
|
return self.topdir(arch, variant.parent, create_dir=create_dir, relative=relative)
|
|
path = os.path.join(path, variant.uid, arch)
|
|
if create_dir and not relative:
|
|
makedirs(path)
|
|
return path
|
|
|
|
def tree_dir(self, arch, variant, create_dir=True, relative=False):
|
|
"""
|
|
Examples:
|
|
compose/Server/x86_64/os
|
|
compose/Server-optional/x86_64/os
|
|
"""
|
|
if arch == "src":
|
|
arch = "source"
|
|
|
|
if arch == "source":
|
|
tree_dir = "tree"
|
|
else:
|
|
# use 'os' dir due to historical reasons
|
|
tree_dir = "os"
|
|
|
|
path = os.path.join(self.topdir(arch, variant, create_dir=create_dir, relative=relative), tree_dir)
|
|
if create_dir and not relative:
|
|
makedirs(path)
|
|
return path
|
|
|
|
def os_tree(self, arch, variant, create_dir=True, relative=False):
|
|
return self.tree_dir(arch, variant, create_dir=create_dir, relative=relative)
|
|
|
|
def repository(self, arch, variant, create_dir=True, relative=False):
|
|
"""
|
|
Examples:
|
|
compose/Server/x86_64/os
|
|
compose/Server/x86_64/addons/LoadBalancer
|
|
"""
|
|
if variant.type == "addon":
|
|
path = self.packages(arch, variant, create_dir=create_dir, relative=relative)
|
|
else:
|
|
path = self.tree_dir(arch, variant, create_dir=create_dir, relative=relative)
|
|
if create_dir and not relative:
|
|
makedirs(path)
|
|
return path
|
|
|
|
def packages(self, arch, variant, create_dir=True, relative=False):
|
|
"""
|
|
Examples:
|
|
compose/Server/x86_64/os/Packages
|
|
compose/Server/x86_64/os/addons/LoadBalancer
|
|
compose/Server-optional/x86_64/os/Packages
|
|
"""
|
|
if variant.type == "addon":
|
|
path = os.path.join(self.tree_dir(arch, variant, create_dir=create_dir, relative=relative), "addons", variant.id)
|
|
else:
|
|
path = os.path.join(self.tree_dir(arch, variant, create_dir=create_dir, relative=relative), "Packages")
|
|
if create_dir and not relative:
|
|
makedirs(path)
|
|
return path
|
|
|
|
def debug_topdir(self, arch, variant, create_dir=True, relative=False):
|
|
"""
|
|
Examples:
|
|
compose/Server/x86_64/debug
|
|
compose/Server-optional/x86_64/debug
|
|
"""
|
|
path = os.path.join(self.topdir(arch, variant, create_dir=create_dir, relative=relative), "debug")
|
|
if create_dir and not relative:
|
|
makedirs(path)
|
|
return path
|
|
|
|
def debug_tree(self, arch, variant, create_dir=True, relative=False):
|
|
"""
|
|
Examples:
|
|
compose/Server/x86_64/debug/tree
|
|
compose/Server-optional/x86_64/debug/tree
|
|
"""
|
|
path = os.path.join(self.debug_topdir(arch, variant, create_dir=create_dir, relative=relative), "tree")
|
|
if create_dir and not relative:
|
|
makedirs(path)
|
|
return path
|
|
|
|
def debug_packages(self, arch, variant, create_dir=True, relative=False):
|
|
"""
|
|
Examples:
|
|
compose/Server/x86_64/debug/tree/Packages
|
|
compose/Server/x86_64/debug/tree/addons/LoadBalancer
|
|
compose/Server-optional/x86_64/debug/tree/Packages
|
|
"""
|
|
if arch in ("source", "src"):
|
|
return None
|
|
if variant.type == "addon":
|
|
path = os.path.join(self.debug_tree(arch, variant, create_dir=create_dir, relative=relative), "addons", variant.id)
|
|
else:
|
|
path = os.path.join(self.debug_tree(arch, variant, create_dir=create_dir, relative=relative), "Packages")
|
|
if create_dir and not relative:
|
|
makedirs(path)
|
|
return path
|
|
|
|
def debug_repository(self, arch, variant, create_dir=True, relative=False):
|
|
"""
|
|
Examples:
|
|
compose/Server/x86_64/debug/tree
|
|
compose/Server/x86_64/debug/tree/addons/LoadBalancer
|
|
compose/Server-optional/x86_64/debug/tree
|
|
"""
|
|
if arch in ("source", "src"):
|
|
return None
|
|
if variant.type == "addon":
|
|
path = os.path.join(self.debug_tree(arch, variant, create_dir=create_dir, relative=relative), "addons", variant.id)
|
|
else:
|
|
path = self.debug_tree(arch, variant, create_dir=create_dir, relative=relative)
|
|
if create_dir and not relative:
|
|
makedirs(path)
|
|
return path
|
|
|
|
def iso_dir(self, arch, variant, symlink_to=None, create_dir=True, relative=False):
|
|
"""
|
|
Examples:
|
|
compose/Server/x86_64/iso
|
|
None
|
|
"""
|
|
if variant.type == "addon":
|
|
return None
|
|
if variant.type == "optional":
|
|
if not self.compose.conf.get("create_optional_isos", False):
|
|
return None
|
|
if arch == "src":
|
|
arch = "source"
|
|
path = os.path.join(self.topdir(arch, variant, create_dir=create_dir, relative=relative), "iso")
|
|
|
|
if symlink_to:
|
|
# TODO: create_dir
|
|
topdir = self.compose.topdir.rstrip("/") + "/"
|
|
relative_dir = path[len(topdir):]
|
|
target_dir = os.path.join(symlink_to, self.compose.compose_id, relative_dir)
|
|
if create_dir and not relative:
|
|
makedirs(target_dir)
|
|
try:
|
|
os.symlink(target_dir, path)
|
|
except OSError as ex:
|
|
if ex.errno != errno.EEXIST:
|
|
raise
|
|
msg = "Symlink pointing to '%s' expected: %s" % (target_dir, path)
|
|
if not os.path.islink(path):
|
|
raise RuntimeError(msg)
|
|
if os.path.abspath(os.readlink(path)) != target_dir:
|
|
raise RuntimeError(msg)
|
|
else:
|
|
if create_dir and not relative:
|
|
makedirs(path)
|
|
return path
|
|
|
|
def iso_path(self, arch, variant, filename, symlink_to=None, create_dir=True, relative=False):
|
|
"""
|
|
Examples:
|
|
compose/Server/x86_64/iso/rhel-7.0-20120127.0-Server-x86_64-dvd1.iso
|
|
None
|
|
"""
|
|
path = self.iso_dir(arch, variant, symlink_to=symlink_to, create_dir=create_dir, relative=relative)
|
|
if path is None:
|
|
return None
|
|
|
|
return os.path.join(path, filename)
|
|
|
|
def image_dir(self, variant, symlink_to=None, relative=False):
|
|
"""
|
|
The arch is listed as literal '%(arch)s'
|
|
Examples:
|
|
compose/Server/%(arch)s/images
|
|
None
|
|
@param variant
|
|
@param symlink_to=None
|
|
@param relative=False
|
|
"""
|
|
# skip optional and addons
|
|
if variant.type != "variant":
|
|
return None
|
|
|
|
path = os.path.join(self.topdir('%(arch)s', variant, create_dir=False, relative=relative),
|
|
"images")
|
|
if symlink_to:
|
|
topdir = self.compose.topdir.rstrip("/") + "/"
|
|
relative_dir = path[len(topdir):]
|
|
target_dir = os.path.join(symlink_to, self.compose.compose_id, relative_dir)
|
|
try:
|
|
os.symlink(target_dir, path)
|
|
except OSError as ex:
|
|
if ex.errno != errno.EEXIST:
|
|
raise
|
|
msg = "Symlink pointing to '%s' expected: %s" % (target_dir, path)
|
|
if not os.path.islink(path):
|
|
raise RuntimeError(msg)
|
|
if os.path.abspath(os.readlink(path)) != target_dir:
|
|
raise RuntimeError(msg)
|
|
return path
|
|
|
|
def jigdo_dir(self, arch, variant, create_dir=True, relative=False):
|
|
"""
|
|
Examples:
|
|
compose/Server/x86_64/jigdo
|
|
None
|
|
"""
|
|
if variant.type == "addon":
|
|
return None
|
|
if variant.type == "optional":
|
|
if not self.compose.conf.get("create_optional_isos", False):
|
|
return None
|
|
if arch == "src":
|
|
arch = "source"
|
|
path = os.path.join(self.topdir(arch, variant, create_dir=create_dir, relative=relative), "jigdo")
|
|
|
|
if create_dir and not relative:
|
|
makedirs(path)
|
|
return path
|
|
|
|
def metadata(self, file_name=None, create_dir=True, relative=False):
|
|
"""
|
|
Examples:
|
|
compose/metadata
|
|
compose/metadata/rpms.json
|
|
"""
|
|
path = os.path.join(self.topdir(create_dir=create_dir, relative=relative), "metadata")
|
|
if create_dir and not relative:
|
|
makedirs(path)
|
|
if file_name:
|
|
path = os.path.join(path, file_name)
|
|
return path
|