pungi/pungi/paths.py
Jan Kaluza 953fb4c54c buildinstall: Allow using external dire for runroot task
A new `buildinstall_topdir` option allows using buildinstall even when
the compose is created on a different volume that Koji is using.

The files are created in this external directory and then copies into
the usual location.

Merges: https://pagure.io/pungi/pull-request/807
Signed-off-by: Jan Kaluza <jkaluza@redhat.com>
2017-11-30 10:55:52 +01:00

581 lines
20 KiB
Python

# -*- coding: utf-8 -*-
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; version 2 of the License.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Library General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, see <https://gnu.org/licenses/>.
__all__ = (
"Paths",
)
import errno
import os
from pungi.util import makedirs
class Paths(object):
def __init__(self, compose):
paths_module_name = compose.conf.get("paths_module")
if paths_module_name:
# custom paths
paths_module = __import__(paths_module_name, globals(), locals(), ["LogPaths", "WorkPaths", "ComposePaths"])
self.compose = paths_module.ComposePaths(compose)
self.log = paths_module.LogPaths(compose)
self.work = paths_module.WorkPaths(compose)
else:
# default paths
self.compose = ComposePaths(compose)
self.log = LogPaths(compose)
self.work = WorkPaths(compose)
# self.metadata ?
class LogPaths(object):
def __init__(self, compose):
self.compose = compose
def topdir(self, arch=None, create_dir=True):
"""
Examples:
log/global
log/x86_64
"""
arch = arch or "global"
path = os.path.join(self.compose.topdir, "logs", arch)
if create_dir:
makedirs(path)
return path
def log_file(self, arch, log_name, create_dir=True):
arch = arch or "global"
if log_name.endswith(".log"):
log_name = log_name[:-4]
return os.path.join(self.topdir(arch, create_dir=create_dir), "%s.%s.log" % (log_name, arch))
class WorkPaths(object):
def __init__(self, compose):
self.compose = compose
def topdir(self, arch=None, create_dir=True):
"""
Examples:
work/global
work/x86_64
"""
arch = arch or "global"
path = os.path.join(self.compose.topdir, "work", arch)
if create_dir:
makedirs(path)
return path
def variants_file(self, arch=None, create_dir=True):
"""
Examples:
work/global/variants.xml
"""
arch = "global"
path = os.path.join(self.topdir(arch, create_dir=create_dir), "variants.xml")
return path
def comps(self, arch=None, variant=None, create_dir=True):
"""
Examples:
work/x86_64/comps/comps-86_64.xml
work/x86_64/comps/comps-Server.x86_64.xml
"""
arch = arch or "global"
if variant is None:
file_name = "comps-%s.xml" % arch
else:
file_name = "comps-%s.%s.xml" % (variant.uid, arch)
path = os.path.join(self.topdir(arch, create_dir=create_dir), "comps")
if create_dir:
makedirs(path)
path = os.path.join(path, file_name)
return path
def pungi_conf(self, arch=None, variant=None, create_dir=True):
"""
Examples:
work/x86_64/pungi/x86_64.conf
work/x86_64/pungi/Server.x86_64.conf
"""
arch = arch or "global"
if variant is None:
file_name = "%s.conf" % arch
else:
file_name = "%s.%s.conf" % (variant.uid, arch)
path = os.path.join(self.topdir(arch, create_dir=create_dir), "pungi")
if create_dir:
makedirs(path)
path = os.path.join(path, file_name)
return path
def pungi_log(self, arch=None, variant=None, create_dir=True):
"""
Examples:
work/x86_64/pungi/x86_64.log
work/x86_64/pungi/Server.x86_64.log
"""
path = self.pungi_conf(arch, variant, create_dir=create_dir)
path = path[:-5] + ".log"
return path
def pungi_cache_dir(self, arch, variant=None, create_dir=True):
"""
Examples:
work/global/pungi-cache
"""
# WARNING: Using the same cache dir with repos of the same names may lead to a race condition
# We should use per arch variant cache dirs to workaround this.
path = os.path.join(self.topdir(arch, create_dir=create_dir), "pungi-cache")
if variant:
path = os.path.join(path, variant.uid)
if create_dir:
makedirs(path)
return path
def comps_repo(self, arch=None, create_dir=True):
"""
Examples:
work/x86_64/comps-repo
work/global/comps-repo
"""
arch = arch or "global"
path = os.path.join(self.topdir(arch, create_dir=create_dir), "comps_repo")
if create_dir:
makedirs(path)
return path
def arch_repo(self, arch=None, create_dir=True):
"""
Examples:
work/x86_64/repo
work/global/repo
"""
arch = arch or "global"
path = os.path.join(self.topdir(arch, create_dir=create_dir), "repo")
if create_dir:
makedirs(path)
return path
def package_list(self, arch=None, variant=None, pkg_type=None, create_dir=True):
"""
Examples:
work/x86_64/package_list/x86_64.conf
work/x86_64/package_list/Server.x86_64.conf
work/x86_64/package_list/Server.x86_64.rpm.conf
"""
arch = arch or "global"
if variant is not None:
file_name = "%s.%s" % (variant, arch)
else:
file_name = "%s" % arch
if pkg_type is not None:
file_name += ".%s" % pkg_type
file_name += ".conf"
path = os.path.join(self.topdir(arch, create_dir=create_dir), "package_list")
if create_dir:
makedirs(path)
path = os.path.join(path, file_name)
return path
def pungi_download_dir(self, arch, create_dir=True):
"""
Examples:
work/x86_64/pungi_download
"""
path = os.path.join(self.topdir(arch, create_dir=create_dir), "pungi_download")
if create_dir:
makedirs(path)
return path
def buildinstall_dir(self, arch, create_dir=True,
allow_topdir_override=False, variant=None):
"""
:param bool allow_topdir_override: When True, the
"buildinstall_topdir" will be used (if set) instead of real
"topdir".
Examples:
work/x86_64/buildinstall
"""
if arch == "global":
raise RuntimeError("Global buildinstall dir makes no sense.")
buildinstall_topdir = self.compose.conf.get("buildinstall_topdir", "")
if allow_topdir_override and buildinstall_topdir:
topdir_basename = os.path.basename(self.compose.topdir)
path = os.path.join(
buildinstall_topdir, "buildinstall-%s" % topdir_basename, arch)
else:
path = os.path.join(self.topdir(arch, create_dir=create_dir), "buildinstall")
if variant:
path = os.path.join(path, variant.uid)
return path
def extra_files_dir(self, arch, variant, create_dir=True):
"""
Examples:
work/x86_64/Server/extra-files
"""
if arch == "global":
raise RuntimeError("Global extra files dir makes no sense.")
path = os.path.join(self.topdir(arch, create_dir=create_dir), variant.uid, "extra-files")
if create_dir:
makedirs(path)
return path
def repo_package_list(self, arch, variant, pkg_type=None, create_dir=True):
"""
Examples:
work/x86_64/repo_package_list/Server.x86_64.rpm.conf
"""
file_name = "%s.%s" % (variant.uid, arch)
if pkg_type is not None:
file_name += ".%s" % pkg_type
file_name += ".conf"
path = os.path.join(self.topdir(arch, create_dir=create_dir), "repo_package_list")
if create_dir:
makedirs(path)
path = os.path.join(path, file_name)
return path
def product_img(self, variant, create_dir=True):
"""
Examples:
work/global/product-Server.img
"""
file_name = "product-%s.img" % variant
path = self.topdir(arch="global", create_dir=create_dir)
path = os.path.join(path, file_name)
return path
def iso_dir(self, arch, filename, create_dir=True):
"""
Examples:
work/x86_64/iso/Project-1.0-20151203.0-Client-x86_64-dvd1.iso
"""
path = os.path.join(self.topdir(arch, create_dir=create_dir), "iso", filename)
if create_dir:
makedirs(path)
return path
def tmp_dir(self, arch=None, variant=None, create_dir=True):
"""
Examples:
work/global/tmp
work/x86_64/tmp
work/x86_64/tmp-Server
"""
dir_name = "tmp"
if variant:
dir_name += "-%s" % variant.uid
path = os.path.join(self.topdir(arch=arch, create_dir=create_dir), dir_name)
if create_dir:
makedirs(path)
return path
def product_id(self, arch, variant, create_dir=True):
"""
Examples:
work/x86_64/product_id/productid-Server.x86_64.pem/productid
"""
# file_name = "%s.%s.pem" % (variant, arch)
# HACK: modifyrepo doesn't handle renames -> $dir/productid
file_name = "productid"
path = os.path.join(self.topdir(arch, create_dir=create_dir), "product_id", "%s.%s.pem" % (variant, arch))
if create_dir:
makedirs(path)
path = os.path.join(path, file_name)
return path
def image_build_dir(self, variant, create_dir=True):
"""
@param variant
@param create_dir=True
Examples:
work/image-build/Server
"""
path = os.path.join(self.topdir('image-build', create_dir=create_dir), variant.uid)
if create_dir:
makedirs(path)
return path
def image_build_conf(self, variant, image_name, image_type, arches=None, create_dir=True):
"""
@param variant
@param image-name
@param image-type (e.g docker)
@param arches
@param create_dir=True
Examples:
work/image-build/Server/docker_rhel-server-docker.cfg
work/image-build/Server/docker_rhel-server-docker_x86_64.cfg
work/image-build/Server/docker_rhel-server-docker_x86_64-ppc64le.cfg
"""
path = os.path.join(self.image_build_dir(variant), "%s_%s" % (image_type, image_name))
if arches is not None:
path = "%s_%s" % (path, '-'.join(list(arches)))
path = "%s.cfg" % path
return path
class ComposePaths(object):
def __init__(self, compose):
self.compose = compose
# TODO: TREES?
def topdir(self, arch=None, variant=None, create_dir=True, relative=False):
"""
Examples:
compose
compose/Server/x86_64
"""
if bool(arch) != bool(variant):
raise TypeError("topdir(): either none or 2 arguments are expected")
path = ""
if not relative:
path = os.path.join(self.compose.topdir, "compose")
if arch or variant:
if variant.type == "addon":
return self.topdir(arch, variant.parent, create_dir=create_dir, relative=relative)
path = os.path.join(path, variant.uid, arch)
if create_dir and not relative:
makedirs(path)
return path
def tree_dir(self, arch, variant, create_dir=True, relative=False):
"""
Examples:
compose/Server/x86_64/os
compose/Server-optional/x86_64/os
"""
if arch == "src":
arch = "source"
if arch == "source":
tree_dir = "tree"
else:
# use 'os' dir due to historical reasons
tree_dir = "os"
path = os.path.join(self.topdir(arch, variant, create_dir=create_dir, relative=relative), tree_dir)
if create_dir and not relative:
makedirs(path)
return path
def os_tree(self, arch, variant, create_dir=True, relative=False):
return self.tree_dir(arch, variant, create_dir=create_dir, relative=relative)
def repository(self, arch, variant, create_dir=True, relative=False):
"""
Examples:
compose/Server/x86_64/os
compose/Server/x86_64/addons/LoadBalancer
"""
if variant.type == "addon":
path = self.packages(arch, variant, create_dir=create_dir, relative=relative)
else:
path = self.tree_dir(arch, variant, create_dir=create_dir, relative=relative)
if create_dir and not relative:
makedirs(path)
return path
def packages(self, arch, variant, create_dir=True, relative=False):
"""
Examples:
compose/Server/x86_64/os/Packages
compose/Server/x86_64/os/addons/LoadBalancer
compose/Server-optional/x86_64/os/Packages
"""
if variant.type == "addon":
path = os.path.join(self.tree_dir(arch, variant, create_dir=create_dir, relative=relative), "addons", variant.id)
else:
path = os.path.join(self.tree_dir(arch, variant, create_dir=create_dir, relative=relative), "Packages")
if create_dir and not relative:
makedirs(path)
return path
def debug_topdir(self, arch, variant, create_dir=True, relative=False):
"""
Examples:
compose/Server/x86_64/debug
compose/Server-optional/x86_64/debug
"""
path = os.path.join(self.topdir(arch, variant, create_dir=create_dir, relative=relative), "debug")
if create_dir and not relative:
makedirs(path)
return path
def debug_tree(self, arch, variant, create_dir=True, relative=False):
"""
Examples:
compose/Server/x86_64/debug/tree
compose/Server-optional/x86_64/debug/tree
"""
path = os.path.join(self.debug_topdir(arch, variant, create_dir=create_dir, relative=relative), "tree")
if create_dir and not relative:
makedirs(path)
return path
def debug_packages(self, arch, variant, create_dir=True, relative=False):
"""
Examples:
compose/Server/x86_64/debug/tree/Packages
compose/Server/x86_64/debug/tree/addons/LoadBalancer
compose/Server-optional/x86_64/debug/tree/Packages
"""
if arch in ("source", "src"):
return None
if variant.type == "addon":
path = os.path.join(self.debug_tree(arch, variant, create_dir=create_dir, relative=relative), "addons", variant.id)
else:
path = os.path.join(self.debug_tree(arch, variant, create_dir=create_dir, relative=relative), "Packages")
if create_dir and not relative:
makedirs(path)
return path
def debug_repository(self, arch, variant, create_dir=True, relative=False):
"""
Examples:
compose/Server/x86_64/debug/tree
compose/Server/x86_64/debug/tree/addons/LoadBalancer
compose/Server-optional/x86_64/debug/tree
"""
if arch in ("source", "src"):
return None
if variant.type == "addon":
path = os.path.join(self.debug_tree(arch, variant, create_dir=create_dir, relative=relative), "addons", variant.id)
else:
path = self.debug_tree(arch, variant, create_dir=create_dir, relative=relative)
if create_dir and not relative:
makedirs(path)
return path
def iso_dir(self, arch, variant, symlink_to=None, create_dir=True, relative=False):
"""
Examples:
compose/Server/x86_64/iso
None
"""
if variant.type == "addon":
return None
if variant.type == "optional":
if not self.compose.conf.get("create_optional_isos", False):
return None
if arch == "src":
arch = "source"
path = os.path.join(self.topdir(arch, variant, create_dir=create_dir, relative=relative), "iso")
if symlink_to:
# TODO: create_dir
topdir = self.compose.topdir.rstrip("/") + "/"
relative_dir = path[len(topdir):]
target_dir = os.path.join(symlink_to, self.compose.compose_id, relative_dir)
if create_dir and not relative:
makedirs(target_dir)
try:
os.symlink(target_dir, path)
except OSError as ex:
if ex.errno != errno.EEXIST:
raise
msg = "Symlink pointing to '%s' expected: %s" % (target_dir, path)
if not os.path.islink(path):
raise RuntimeError(msg)
if os.path.abspath(os.readlink(path)) != target_dir:
raise RuntimeError(msg)
else:
if create_dir and not relative:
makedirs(path)
return path
def iso_path(self, arch, variant, filename, symlink_to=None, create_dir=True, relative=False):
"""
Examples:
compose/Server/x86_64/iso/rhel-7.0-20120127.0-Server-x86_64-dvd1.iso
None
"""
path = self.iso_dir(arch, variant, symlink_to=symlink_to, create_dir=create_dir, relative=relative)
if path is None:
return None
return os.path.join(path, filename)
def image_dir(self, variant, symlink_to=None, relative=False):
"""
The arch is listed as literal '%(arch)s'
Examples:
compose/Server/%(arch)s/images
None
@param variant
@param symlink_to=None
@param relative=False
"""
path = os.path.join(self.topdir('%(arch)s', variant, create_dir=False, relative=relative),
"images")
if symlink_to:
topdir = self.compose.topdir.rstrip("/") + "/"
relative_dir = path[len(topdir):]
target_dir = os.path.join(symlink_to, self.compose.compose_id, relative_dir)
try:
os.symlink(target_dir, path)
except OSError as ex:
if ex.errno != errno.EEXIST:
raise
msg = "Symlink pointing to '%s' expected: %s" % (target_dir, path)
if not os.path.islink(path):
raise RuntimeError(msg)
if os.path.abspath(os.readlink(path)) != target_dir:
raise RuntimeError(msg)
return path
def jigdo_dir(self, arch, variant, create_dir=True, relative=False):
"""
Examples:
compose/Server/x86_64/jigdo
None
"""
if variant.type == "addon":
return None
if variant.type == "optional":
if not self.compose.conf.get("create_optional_isos", False):
return None
if arch == "src":
arch = "source"
path = os.path.join(self.topdir(arch, variant, create_dir=create_dir, relative=relative), "jigdo")
if create_dir and not relative:
makedirs(path)
return path
def metadata(self, file_name=None, create_dir=True, relative=False):
"""
Examples:
compose/metadata
compose/metadata/rpms.json
"""
path = os.path.join(self.topdir(create_dir=create_dir, relative=relative), "metadata")
if create_dir and not relative:
makedirs(path)
if file_name:
path = os.path.join(path, file_name)
return path