f43d3584c5
The line of log printing can result in "No handlers could be found for logger" warning in quiet mode when custom paths module is enabled, because file handler is setup after that, since this line is only printed to screen (when pungi-koji is not ran in quiet mode) and doesn't contain important info, it doesn't hurt to remove it. Signed-off-by: Qixiang Wan <qwan@redhat.com>
582 lines
20 KiB
Python
582 lines
20 KiB
Python
# -*- coding: utf-8 -*-
|
|
|
|
|
|
# This program is free software; you can redistribute it and/or modify
|
|
# it under the terms of the GNU General Public License as published by
|
|
# the Free Software Foundation; version 2 of the License.
|
|
#
|
|
# This program is distributed in the hope that it will be useful,
|
|
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
# GNU Library General Public License for more details.
|
|
#
|
|
# You should have received a copy of the GNU General Public License
|
|
# along with this program; if not, see <https://gnu.org/licenses/>.
|
|
|
|
|
|
__all__ = (
|
|
"Paths",
|
|
)
|
|
|
|
|
|
import errno
|
|
import os
|
|
|
|
from pungi.util import makedirs
|
|
|
|
|
|
def translate_path(compose, path):
|
|
"""
|
|
@param compose - required for access to config
|
|
@param path
|
|
"""
|
|
normpath = os.path.normpath(path)
|
|
mapping = compose.conf["translate_paths"]
|
|
|
|
for prefix, newvalue in mapping:
|
|
prefix = os.path.normpath(prefix)
|
|
# Strip trailing slashes: the prefix has them stripped by `normpath`.
|
|
newvalue = newvalue.rstrip('/')
|
|
if normpath.startswith(prefix):
|
|
# We can't call os.path.normpath on result since it is not actually
|
|
# a path - http:// would get changed to http:/ and so on.
|
|
# Only the first occurance should be replaced.
|
|
return normpath.replace(prefix, newvalue, 1)
|
|
|
|
return normpath
|
|
|
|
|
|
class Paths(object):
|
|
def __init__(self, compose):
|
|
paths_module_name = compose.conf.get("paths_module")
|
|
if paths_module_name:
|
|
# custom paths
|
|
paths_module = __import__(paths_module_name, globals(), locals(), ["LogPaths", "WorkPaths", "ComposePaths"])
|
|
self.compose = paths_module.ComposePaths(compose)
|
|
self.log = paths_module.LogPaths(compose)
|
|
self.work = paths_module.WorkPaths(compose)
|
|
else:
|
|
# default paths
|
|
self.compose = ComposePaths(compose)
|
|
self.log = LogPaths(compose)
|
|
self.work = WorkPaths(compose)
|
|
# self.metadata ?
|
|
|
|
|
|
class LogPaths(object):
|
|
def __init__(self, compose):
|
|
self.compose = compose
|
|
|
|
def topdir(self, arch=None, create_dir=True):
|
|
"""
|
|
Examples:
|
|
log/global
|
|
log/x86_64
|
|
"""
|
|
arch = arch or "global"
|
|
path = os.path.join(self.compose.topdir, "logs", arch)
|
|
if create_dir:
|
|
makedirs(path)
|
|
return path
|
|
|
|
def log_file(self, arch, log_name, create_dir=True):
|
|
arch = arch or "global"
|
|
if log_name.endswith(".log"):
|
|
log_name = log_name[:-4]
|
|
return os.path.join(self.topdir(arch, create_dir=create_dir), "%s.%s.log" % (log_name, arch))
|
|
|
|
|
|
class WorkPaths(object):
|
|
def __init__(self, compose):
|
|
self.compose = compose
|
|
|
|
def topdir(self, arch=None, create_dir=True):
|
|
"""
|
|
Examples:
|
|
work/global
|
|
work/x86_64
|
|
"""
|
|
arch = arch or "global"
|
|
path = os.path.join(self.compose.topdir, "work", arch)
|
|
if create_dir:
|
|
makedirs(path)
|
|
return path
|
|
|
|
def variants_file(self, arch=None, create_dir=True):
|
|
"""
|
|
Examples:
|
|
work/global/variants.xml
|
|
"""
|
|
arch = "global"
|
|
path = os.path.join(self.topdir(arch, create_dir=create_dir), "variants.xml")
|
|
return path
|
|
|
|
def comps(self, arch=None, variant=None, create_dir=True):
|
|
"""
|
|
Examples:
|
|
work/x86_64/comps/comps-86_64.xml
|
|
work/x86_64/comps/comps-Server.x86_64.xml
|
|
"""
|
|
arch = arch or "global"
|
|
if variant is None:
|
|
file_name = "comps-%s.xml" % arch
|
|
else:
|
|
file_name = "comps-%s.%s.xml" % (variant.uid, arch)
|
|
path = os.path.join(self.topdir(arch, create_dir=create_dir), "comps")
|
|
if create_dir:
|
|
makedirs(path)
|
|
path = os.path.join(path, file_name)
|
|
return path
|
|
|
|
def pungi_conf(self, arch=None, variant=None, create_dir=True):
|
|
"""
|
|
Examples:
|
|
work/x86_64/pungi/x86_64.conf
|
|
work/x86_64/pungi/Server.x86_64.conf
|
|
"""
|
|
arch = arch or "global"
|
|
if variant is None:
|
|
file_name = "%s.conf" % arch
|
|
else:
|
|
file_name = "%s.%s.conf" % (variant.uid, arch)
|
|
path = os.path.join(self.topdir(arch, create_dir=create_dir), "pungi")
|
|
if create_dir:
|
|
makedirs(path)
|
|
path = os.path.join(path, file_name)
|
|
return path
|
|
|
|
def pungi_log(self, arch=None, variant=None, create_dir=True):
|
|
"""
|
|
Examples:
|
|
work/x86_64/pungi/x86_64.log
|
|
work/x86_64/pungi/Server.x86_64.log
|
|
"""
|
|
path = self.pungi_conf(arch, variant, create_dir=create_dir)
|
|
path = path[:-5] + ".log"
|
|
return path
|
|
|
|
def pungi_cache_dir(self, arch, variant=None, create_dir=True):
|
|
"""
|
|
Examples:
|
|
work/global/pungi-cache
|
|
"""
|
|
# WARNING: Using the same cache dir with repos of the same names may lead to a race condition
|
|
# We should use per arch variant cache dirs to workaround this.
|
|
path = os.path.join(self.topdir(arch, create_dir=create_dir), "pungi-cache")
|
|
if variant:
|
|
path = os.path.join(path, variant.uid)
|
|
if create_dir:
|
|
makedirs(path)
|
|
return path
|
|
|
|
def comps_repo(self, arch=None, create_dir=True):
|
|
"""
|
|
Examples:
|
|
work/x86_64/comps-repo
|
|
work/global/comps-repo
|
|
"""
|
|
arch = arch or "global"
|
|
path = os.path.join(self.topdir(arch, create_dir=create_dir), "comps_repo")
|
|
if create_dir:
|
|
makedirs(path)
|
|
return path
|
|
|
|
def arch_repo(self, arch=None, create_dir=True):
|
|
"""
|
|
Examples:
|
|
work/x86_64/repo
|
|
work/global/repo
|
|
"""
|
|
arch = arch or "global"
|
|
path = os.path.join(self.topdir(arch, create_dir=create_dir), "repo")
|
|
if create_dir:
|
|
makedirs(path)
|
|
return path
|
|
|
|
def package_list(self, arch=None, variant=None, pkg_type=None, create_dir=True):
|
|
"""
|
|
Examples:
|
|
work/x86_64/package_list/x86_64.conf
|
|
work/x86_64/package_list/Server.x86_64.conf
|
|
work/x86_64/package_list/Server.x86_64.rpm.conf
|
|
"""
|
|
arch = arch or "global"
|
|
if variant is not None:
|
|
file_name = "%s.%s" % (variant, arch)
|
|
else:
|
|
file_name = "%s" % arch
|
|
if pkg_type is not None:
|
|
file_name += ".%s" % pkg_type
|
|
file_name += ".conf"
|
|
path = os.path.join(self.topdir(arch, create_dir=create_dir), "package_list")
|
|
if create_dir:
|
|
makedirs(path)
|
|
path = os.path.join(path, file_name)
|
|
return path
|
|
|
|
def pungi_download_dir(self, arch, create_dir=True):
|
|
"""
|
|
Examples:
|
|
work/x86_64/pungi_download
|
|
"""
|
|
path = os.path.join(self.topdir(arch, create_dir=create_dir), "pungi_download")
|
|
if create_dir:
|
|
makedirs(path)
|
|
return path
|
|
|
|
def buildinstall_dir(self, arch, create_dir=True):
|
|
"""
|
|
Examples:
|
|
work/x86_64/buildinstall
|
|
"""
|
|
if arch == "global":
|
|
raise RuntimeError("Global buildinstall dir makes no sense.")
|
|
path = os.path.join(self.topdir(arch, create_dir=create_dir), "buildinstall")
|
|
return path
|
|
|
|
def extra_files_dir(self, arch, variant, create_dir=True):
|
|
"""
|
|
Examples:
|
|
work/x86_64/Server/extra-files
|
|
"""
|
|
if arch == "global":
|
|
raise RuntimeError("Global extra files dir makes no sense.")
|
|
path = os.path.join(self.topdir(arch, create_dir=create_dir), variant.uid, "extra-files")
|
|
if create_dir:
|
|
makedirs(path)
|
|
return path
|
|
|
|
def repo_package_list(self, arch, variant, pkg_type=None, create_dir=True):
|
|
"""
|
|
Examples:
|
|
work/x86_64/repo_package_list/Server.x86_64.rpm.conf
|
|
"""
|
|
file_name = "%s.%s" % (variant.uid, arch)
|
|
if pkg_type is not None:
|
|
file_name += ".%s" % pkg_type
|
|
file_name += ".conf"
|
|
path = os.path.join(self.topdir(arch, create_dir=create_dir), "repo_package_list")
|
|
if create_dir:
|
|
makedirs(path)
|
|
path = os.path.join(path, file_name)
|
|
return path
|
|
|
|
def product_img(self, variant, create_dir=True):
|
|
"""
|
|
Examples:
|
|
work/global/product-Server.img
|
|
"""
|
|
file_name = "product-%s.img" % variant
|
|
path = self.topdir(arch="global", create_dir=create_dir)
|
|
path = os.path.join(path, file_name)
|
|
return path
|
|
|
|
def iso_dir(self, arch, filename, create_dir=True):
|
|
"""
|
|
Examples:
|
|
work/x86_64/iso/Project-1.0-20151203.0-Client-x86_64-dvd1.iso
|
|
"""
|
|
path = os.path.join(self.topdir(arch, create_dir=create_dir), "iso", filename)
|
|
if create_dir:
|
|
makedirs(path)
|
|
return path
|
|
|
|
def tmp_dir(self, arch=None, variant=None, create_dir=True):
|
|
"""
|
|
Examples:
|
|
work/global/tmp
|
|
work/x86_64/tmp
|
|
work/x86_64/tmp-Server
|
|
"""
|
|
dir_name = "tmp"
|
|
if variant:
|
|
dir_name += "-%s" % variant.uid
|
|
path = os.path.join(self.topdir(arch=arch, create_dir=create_dir), dir_name)
|
|
if create_dir:
|
|
makedirs(path)
|
|
return path
|
|
|
|
def product_id(self, arch, variant, create_dir=True):
|
|
"""
|
|
Examples:
|
|
work/x86_64/product_id/productid-Server.x86_64.pem/productid
|
|
"""
|
|
# file_name = "%s.%s.pem" % (variant, arch)
|
|
# HACK: modifyrepo doesn't handle renames -> $dir/productid
|
|
file_name = "productid"
|
|
path = os.path.join(self.topdir(arch, create_dir=create_dir), "product_id", "%s.%s.pem" % (variant, arch))
|
|
if create_dir:
|
|
makedirs(path)
|
|
path = os.path.join(path, file_name)
|
|
return path
|
|
|
|
def image_build_dir(self, variant, create_dir=True):
|
|
"""
|
|
@param variant
|
|
@param create_dir=True
|
|
|
|
Examples:
|
|
work/image-build/Server
|
|
"""
|
|
path = os.path.join(self.topdir('image-build', create_dir=create_dir), variant.uid)
|
|
if create_dir:
|
|
makedirs(path)
|
|
return path
|
|
|
|
def image_build_conf(self, variant, image_name, image_type, create_dir=True):
|
|
"""
|
|
@param variant
|
|
@param image-name
|
|
@param image-type (e.g docker)
|
|
@param create_dir=True
|
|
|
|
Examples:
|
|
work/image-build/Server/docker_rhel-server-docker.cfg
|
|
"""
|
|
path = os.path.join(self.image_build_dir(variant), "%s_%s.cfg" % (image_type, image_name))
|
|
return path
|
|
|
|
|
|
class ComposePaths(object):
|
|
def __init__(self, compose):
|
|
self.compose = compose
|
|
# TODO: TREES?
|
|
|
|
def topdir(self, arch=None, variant=None, create_dir=True, relative=False):
|
|
"""
|
|
Examples:
|
|
compose
|
|
compose/Server/x86_64
|
|
"""
|
|
if bool(arch) != bool(variant):
|
|
raise TypeError("topdir(): either none or 2 arguments are expected")
|
|
|
|
path = ""
|
|
if not relative:
|
|
path = os.path.join(self.compose.topdir, "compose")
|
|
|
|
if arch or variant:
|
|
if variant.type == "addon":
|
|
return self.topdir(arch, variant.parent, create_dir=create_dir, relative=relative)
|
|
path = os.path.join(path, variant.uid, arch)
|
|
if create_dir and not relative:
|
|
makedirs(path)
|
|
return path
|
|
|
|
def tree_dir(self, arch, variant, create_dir=True, relative=False):
|
|
"""
|
|
Examples:
|
|
compose/Server/x86_64/os
|
|
compose/Server-optional/x86_64/os
|
|
"""
|
|
if arch == "src":
|
|
arch = "source"
|
|
|
|
if arch == "source":
|
|
tree_dir = "tree"
|
|
else:
|
|
# use 'os' dir due to historical reasons
|
|
tree_dir = "os"
|
|
|
|
path = os.path.join(self.topdir(arch, variant, create_dir=create_dir, relative=relative), tree_dir)
|
|
if create_dir and not relative:
|
|
makedirs(path)
|
|
return path
|
|
|
|
def os_tree(self, arch, variant, create_dir=True, relative=False):
|
|
return self.tree_dir(arch, variant, create_dir=create_dir, relative=relative)
|
|
|
|
def repository(self, arch, variant, create_dir=True, relative=False):
|
|
"""
|
|
Examples:
|
|
compose/Server/x86_64/os
|
|
compose/Server/x86_64/addons/LoadBalancer
|
|
"""
|
|
if variant.type == "addon":
|
|
path = self.packages(arch, variant, create_dir=create_dir, relative=relative)
|
|
else:
|
|
path = self.tree_dir(arch, variant, create_dir=create_dir, relative=relative)
|
|
if create_dir and not relative:
|
|
makedirs(path)
|
|
return path
|
|
|
|
def packages(self, arch, variant, create_dir=True, relative=False):
|
|
"""
|
|
Examples:
|
|
compose/Server/x86_64/os/Packages
|
|
compose/Server/x86_64/os/addons/LoadBalancer
|
|
compose/Server-optional/x86_64/os/Packages
|
|
"""
|
|
if variant.type == "addon":
|
|
path = os.path.join(self.tree_dir(arch, variant, create_dir=create_dir, relative=relative), "addons", variant.id)
|
|
else:
|
|
path = os.path.join(self.tree_dir(arch, variant, create_dir=create_dir, relative=relative), "Packages")
|
|
if create_dir and not relative:
|
|
makedirs(path)
|
|
return path
|
|
|
|
def debug_topdir(self, arch, variant, create_dir=True, relative=False):
|
|
"""
|
|
Examples:
|
|
compose/Server/x86_64/debug
|
|
compose/Server-optional/x86_64/debug
|
|
"""
|
|
path = os.path.join(self.topdir(arch, variant, create_dir=create_dir, relative=relative), "debug")
|
|
if create_dir and not relative:
|
|
makedirs(path)
|
|
return path
|
|
|
|
def debug_tree(self, arch, variant, create_dir=True, relative=False):
|
|
"""
|
|
Examples:
|
|
compose/Server/x86_64/debug/tree
|
|
compose/Server-optional/x86_64/debug/tree
|
|
"""
|
|
path = os.path.join(self.debug_topdir(arch, variant, create_dir=create_dir, relative=relative), "tree")
|
|
if create_dir and not relative:
|
|
makedirs(path)
|
|
return path
|
|
|
|
def debug_packages(self, arch, variant, create_dir=True, relative=False):
|
|
"""
|
|
Examples:
|
|
compose/Server/x86_64/debug/tree/Packages
|
|
compose/Server/x86_64/debug/tree/addons/LoadBalancer
|
|
compose/Server-optional/x86_64/debug/tree/Packages
|
|
"""
|
|
if arch in ("source", "src"):
|
|
return None
|
|
if variant.type == "addon":
|
|
path = os.path.join(self.debug_tree(arch, variant, create_dir=create_dir, relative=relative), "addons", variant.id)
|
|
else:
|
|
path = os.path.join(self.debug_tree(arch, variant, create_dir=create_dir, relative=relative), "Packages")
|
|
if create_dir and not relative:
|
|
makedirs(path)
|
|
return path
|
|
|
|
def debug_repository(self, arch, variant, create_dir=True, relative=False):
|
|
"""
|
|
Examples:
|
|
compose/Server/x86_64/debug/tree
|
|
compose/Server/x86_64/debug/tree/addons/LoadBalancer
|
|
compose/Server-optional/x86_64/debug/tree
|
|
"""
|
|
if arch in ("source", "src"):
|
|
return None
|
|
if variant.type == "addon":
|
|
path = os.path.join(self.debug_tree(arch, variant, create_dir=create_dir, relative=relative), "addons", variant.id)
|
|
else:
|
|
path = self.debug_tree(arch, variant, create_dir=create_dir, relative=relative)
|
|
if create_dir and not relative:
|
|
makedirs(path)
|
|
return path
|
|
|
|
def iso_dir(self, arch, variant, symlink_to=None, create_dir=True, relative=False):
|
|
"""
|
|
Examples:
|
|
compose/Server/x86_64/iso
|
|
None
|
|
"""
|
|
if variant.type == "addon":
|
|
return None
|
|
if variant.type == "optional":
|
|
if not self.compose.conf.get("create_optional_isos", False):
|
|
return None
|
|
if arch == "src":
|
|
arch = "source"
|
|
path = os.path.join(self.topdir(arch, variant, create_dir=create_dir, relative=relative), "iso")
|
|
|
|
if symlink_to:
|
|
# TODO: create_dir
|
|
topdir = self.compose.topdir.rstrip("/") + "/"
|
|
relative_dir = path[len(topdir):]
|
|
target_dir = os.path.join(symlink_to, self.compose.compose_id, relative_dir)
|
|
if create_dir and not relative:
|
|
makedirs(target_dir)
|
|
try:
|
|
os.symlink(target_dir, path)
|
|
except OSError as ex:
|
|
if ex.errno != errno.EEXIST:
|
|
raise
|
|
msg = "Symlink pointing to '%s' expected: %s" % (target_dir, path)
|
|
if not os.path.islink(path):
|
|
raise RuntimeError(msg)
|
|
if os.path.abspath(os.readlink(path)) != target_dir:
|
|
raise RuntimeError(msg)
|
|
else:
|
|
if create_dir and not relative:
|
|
makedirs(path)
|
|
return path
|
|
|
|
def iso_path(self, arch, variant, filename, symlink_to=None, create_dir=True, relative=False):
|
|
"""
|
|
Examples:
|
|
compose/Server/x86_64/iso/rhel-7.0-20120127.0-Server-x86_64-dvd1.iso
|
|
None
|
|
"""
|
|
path = self.iso_dir(arch, variant, symlink_to=symlink_to, create_dir=create_dir, relative=relative)
|
|
if path is None:
|
|
return None
|
|
|
|
return os.path.join(path, filename)
|
|
|
|
def image_dir(self, variant, symlink_to=None, relative=False):
|
|
"""
|
|
The arch is listed as literal '%(arch)s'
|
|
Examples:
|
|
compose/Server/%(arch)s/images
|
|
None
|
|
@param variant
|
|
@param symlink_to=None
|
|
@param relative=False
|
|
"""
|
|
path = os.path.join(self.topdir('%(arch)s', variant, create_dir=False, relative=relative),
|
|
"images")
|
|
if symlink_to:
|
|
topdir = self.compose.topdir.rstrip("/") + "/"
|
|
relative_dir = path[len(topdir):]
|
|
target_dir = os.path.join(symlink_to, self.compose.compose_id, relative_dir)
|
|
try:
|
|
os.symlink(target_dir, path)
|
|
except OSError as ex:
|
|
if ex.errno != errno.EEXIST:
|
|
raise
|
|
msg = "Symlink pointing to '%s' expected: %s" % (target_dir, path)
|
|
if not os.path.islink(path):
|
|
raise RuntimeError(msg)
|
|
if os.path.abspath(os.readlink(path)) != target_dir:
|
|
raise RuntimeError(msg)
|
|
return path
|
|
|
|
def jigdo_dir(self, arch, variant, create_dir=True, relative=False):
|
|
"""
|
|
Examples:
|
|
compose/Server/x86_64/jigdo
|
|
None
|
|
"""
|
|
if variant.type == "addon":
|
|
return None
|
|
if variant.type == "optional":
|
|
if not self.compose.conf.get("create_optional_isos", False):
|
|
return None
|
|
if arch == "src":
|
|
arch = "source"
|
|
path = os.path.join(self.topdir(arch, variant, create_dir=create_dir, relative=relative), "jigdo")
|
|
|
|
if create_dir and not relative:
|
|
makedirs(path)
|
|
return path
|
|
|
|
def metadata(self, file_name=None, create_dir=True, relative=False):
|
|
"""
|
|
Examples:
|
|
compose/metadata
|
|
compose/metadata/rpms.json
|
|
"""
|
|
path = os.path.join(self.topdir(create_dir=create_dir, relative=relative), "metadata")
|
|
if create_dir and not relative:
|
|
makedirs(path)
|
|
if file_name:
|
|
path = os.path.join(path, file_name)
|
|
return path
|