pungi/pungi/paths.py
Filip Valder fe986d68b9 Add module obsoletes feature
JIRA: MODULAR-113
Merges: https://pagure.io/pungi/pull-request/1578
Signed-off-by: Filip Valder <fvalder@redhat.com>
2022-01-12 12:51:37 +01:00

842 lines
27 KiB
Python

# -*- coding: utf-8 -*-
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; version 2 of the License.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Library General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, see <https://gnu.org/licenses/>.
__all__ = ("Paths",)
import errno
import os
from kobo.shortcuts import relative_path
from pungi.util import makedirs, find_old_compose
class Paths(object):
def __init__(self, compose):
self._compose = compose
paths_module_name = compose.conf.get("paths_module")
if paths_module_name:
# custom paths
paths_module = __import__(
paths_module_name,
globals(),
locals(),
["LogPaths", "WorkPaths", "ComposePaths"],
)
self.compose = paths_module.ComposePaths(compose)
self.log = paths_module.LogPaths(compose)
self.work = paths_module.WorkPaths(compose)
else:
# default paths
self.compose = ComposePaths(compose)
self.log = LogPaths(compose)
self.work = WorkPaths(compose)
# self.metadata ?
def get_old_compose_topdir(self, **kwargs):
"""
Finds old compose using the `find_old_compose` function and returns
the path to it. The `kwargs` are passed to `find_old_compose`.
"""
is_layered = self._compose.ci_base.release.is_layered
return find_old_compose(
self._compose.old_composes,
self._compose.ci_base.release.short,
self._compose.ci_base.release.version,
self._compose.ci_base.release.type_suffix,
self._compose.ci_base.base_product.short if is_layered else None,
self._compose.ci_base.base_product.version if is_layered else None,
**kwargs
)
def old_compose_path(self, path, **kwargs):
"""
Translates `path` to the topdir of old compose.
:param str path: Path to translate.
:param kwargs: The kwargs passed to `find_old_compose` function.
:return: None if old compose cannot be used or if `path` does not exist
in the old compose topdir. Otherwise path translated to old_compose
topdir.
Example:
old_repo_dir = compose.old_compose_path(
compose.paths.work.pkgset_repo(pkgset.name, arch="global"))
"""
old_compose_topdir = self.get_old_compose_topdir(**kwargs)
if not old_compose_topdir:
return None
rel_path = relative_path(path, self._compose.topdir.rstrip("/") + "/")
old_path = os.path.join(old_compose_topdir, rel_path)
if not os.path.exists(old_path):
return None
return old_path
class LogPaths(object):
def __init__(self, compose):
self.compose = compose
def topdir(self, arch=None, create_dir=True):
"""
Examples:
log/global
log/x86_64
"""
arch = arch or "global"
path = os.path.join(self.compose.topdir, "logs", arch)
if create_dir:
makedirs(path)
return path
def koji_tasks_dir(self, create_dir=True):
"""
Examples:
logs/global/koji-tasks
"""
path = os.path.join(self.topdir(create_dir=create_dir), "koji-tasks")
if create_dir:
makedirs(path)
return path
def log_file(self, arch, log_name, create_dir=True, ext=None):
ext = ext or "log"
arch = arch or "global"
if log_name.endswith(".log"):
log_name = log_name[:-4]
return os.path.join(
self.topdir(arch, create_dir=create_dir), "%s.%s.%s" % (log_name, arch, ext)
)
class WorkPaths(object):
def __init__(self, compose):
self.compose = compose
def topdir(self, arch=None, create_dir=True):
"""
Examples:
work/global
work/x86_64
"""
arch = arch or "global"
path = os.path.join(self.compose.topdir, "work", arch)
if create_dir:
makedirs(path)
return path
def variants_file(self, arch=None, create_dir=True):
"""
Examples:
work/global/variants.xml
"""
arch = "global"
path = os.path.join(self.topdir(arch, create_dir=create_dir), "variants.xml")
return path
def comps(self, arch=None, variant=None, create_dir=True):
"""
Examples:
work/x86_64/comps/comps-86_64.xml
work/x86_64/comps/comps-Server.x86_64.xml
"""
arch = arch or "global"
if variant is None:
file_name = "comps-%s.xml" % arch
else:
file_name = "comps-%s.%s.xml" % (variant.uid, arch)
path = os.path.join(self.topdir(arch, create_dir=create_dir), "comps")
if create_dir:
makedirs(path)
path = os.path.join(path, file_name)
return path
def gather_result(self, arch=None, variant=None, create_dir=True):
"""
Examples:
work/x86_64/gather_result/x86_64.result
work/x86_64/gather_result/Server.x86_64.result
"""
arch = arch or "global"
file_name = ""
if variant:
file_name += variant.uid + "."
file_name += arch + "."
file_name += "result"
path = os.path.join(self.topdir(arch, create_dir=create_dir), "gather_result")
if create_dir:
makedirs(path)
path = os.path.join(path, file_name)
return path
def pungi_conf(self, arch=None, variant=None, create_dir=True, source_name=None):
"""
Examples:
work/x86_64/pungi/x86_64.conf
work/x86_64/pungi/Server.x86_64.conf
"""
arch = arch or "global"
file_name = ""
if variant:
file_name += variant.uid + "."
file_name += arch + "."
if source_name:
file_name += source_name + "."
file_name += "conf"
path = os.path.join(self.topdir(arch, create_dir=create_dir), "pungi")
if create_dir:
makedirs(path)
path = os.path.join(path, file_name)
return path
def fus_conf(self, arch, variant, iteration, create_dir=True):
"""
Examples:
work/x86_64/fus/Server-solvables.x86_64.conf
"""
file_name = "%s-solvables-%d.%s.conf" % (variant.uid, iteration, arch)
path = os.path.join(self.topdir(arch, create_dir=create_dir), "fus")
if create_dir:
makedirs(path)
return os.path.join(path, file_name)
def pungi_log(self, arch=None, variant=None, create_dir=True, source_name=None):
"""
Examples:
work/x86_64/pungi/x86_64.log
work/x86_64/pungi/Server.x86_64.log
"""
path = self.pungi_conf(arch, variant, create_dir=create_dir)
path = path[:-5]
if source_name:
path += "." + source_name
return path + ".log"
def pungi_cache_dir(self, arch, variant=None, create_dir=True):
"""
Examples:
work/global/pungi-cache
"""
# WARNING: Using the same cache dir with repos of the same names
# may lead to a race condition.
# We should use per arch variant cache dirs to workaround this.
path = os.path.join(self.topdir(arch, create_dir=create_dir), "pungi-cache")
if variant:
path = os.path.join(path, variant.uid)
if create_dir:
makedirs(path)
return path
def _repo(self, type, arch=None, variant=None, create_dir=True):
arch = arch or "global"
path = os.path.join(self.topdir(arch, create_dir=create_dir), "%s_repo" % type)
if variant:
path += "_" + variant.uid
if create_dir:
makedirs(path)
return path
def comps_repo(self, arch=None, variant=None, create_dir=True):
"""
Examples:
work/x86_64/comps_repo_Server
work/global/comps_repo
"""
return self._repo("comps", arch, variant, create_dir=create_dir)
def pkgset_repo(self, pkgset_name, arch=None, create_dir=True):
"""
Examples:
work/x86_64/repo/f30-compose
work/global/repo/f30-compose
"""
arch = arch or "global"
path = os.path.join(
self.topdir(arch, create_dir=create_dir), "repo", pkgset_name
)
if create_dir:
makedirs(path)
return path
def lookaside_repo(self, arch, variant, create_dir=True):
"""
Examples:
work/x86_64/Server/lookaside_repo
"""
path = os.path.join(
self.topdir(arch, create_dir=create_dir), variant.uid, "lookaside_repo"
)
if create_dir:
makedirs(path)
return path
def package_list(
self, arch=None, variant=None, pkgset=None, pkg_type=None, create_dir=True
):
"""
Examples:
work/x86_64/package_list/x86_64.conf
work/x86_64/package_list/Server.x86_64.conf
work/x86_64/package_list/Server.x86_64.rpm.conf
"""
arch = arch or "global"
if variant is not None:
file_name = "%s.%s" % (variant, arch)
else:
file_name = "%s" % arch
if pkgset:
file_name += "." + pkgset.name
if pkg_type is not None:
file_name += ".%s" % pkg_type
file_name += ".conf"
path = os.path.join(self.topdir(arch, create_dir=create_dir), "package_list")
if create_dir:
makedirs(path)
path = os.path.join(path, file_name)
return path
def lookaside_package_list(self, arch, variant, create_dir=True):
"""
Examples:
work/x86_64/package_list/Server.x86_64.lookaside.conf
"""
return self.package_list(
arch, variant, pkg_type="lookaside", create_dir=create_dir
)
def pungi_download_dir(self, arch, create_dir=True):
"""
Examples:
work/x86_64/pungi_download
"""
path = os.path.join(self.topdir(arch, create_dir=create_dir), "pungi_download")
if create_dir:
makedirs(path)
return path
def buildinstall_dir(
self, arch, create_dir=True, allow_topdir_override=False, variant=None
):
"""
:param bool allow_topdir_override: When True, the
"buildinstall_topdir" will be used (if set) instead of real
"topdir".
Examples:
work/x86_64/buildinstall
"""
if arch == "global":
raise RuntimeError("Global buildinstall dir makes no sense.")
buildinstall_topdir = self.compose.conf.get("buildinstall_topdir", "")
if allow_topdir_override and buildinstall_topdir:
topdir_basename = os.path.basename(self.compose.topdir)
path = os.path.join(
buildinstall_topdir, "buildinstall-%s" % topdir_basename, arch
)
else:
path = os.path.join(
self.topdir(arch, create_dir=create_dir), "buildinstall"
)
if variant:
path = os.path.join(path, variant.uid)
return path
def extra_files_dir(self, arch, variant, create_dir=True):
"""
Examples:
work/x86_64/Server/extra-files
"""
if arch == "global":
raise RuntimeError("Global extra files dir makes no sense.")
path = os.path.join(
self.topdir(arch, create_dir=create_dir), variant.uid, "extra-files"
)
if create_dir:
makedirs(path)
return path
def extra_iso_extra_files_dir(self, arch, variant, create_dir=True):
"""
Examples:
work/x86_64/Server/extra-iso-extra-files
"""
if arch == "global":
raise RuntimeError("Global extra files dir makes no sense.")
path = os.path.join(
self.topdir(arch, create_dir=create_dir),
variant.uid,
"extra-iso-extra-files",
)
if create_dir:
makedirs(path)
return path
def iso_staging_dir(self, arch, variant, filename, create_dir=True):
"""
Examples:
work/x86_64/Server/iso-staging-dir/file.iso/
"""
path = os.path.join(
self.topdir(arch, create_dir=create_dir),
variant.uid,
"iso-staging-dir",
filename,
)
if create_dir:
makedirs(path)
return path
def repo_package_list(self, arch, variant, pkg_type=None, create_dir=True):
"""
Examples:
work/x86_64/repo_package_list/Server.x86_64.rpm.conf
"""
file_name = "%s.%s" % (variant.uid, arch)
if pkg_type is not None:
file_name += ".%s" % pkg_type
file_name += ".conf"
path = os.path.join(
self.topdir(arch, create_dir=create_dir), "repo_package_list"
)
if create_dir:
makedirs(path)
path = os.path.join(path, file_name)
return path
def iso_dir(self, arch, filename, create_dir=True):
"""
Examples:
work/x86_64/iso/Project-1.0-20151203.0-Client-x86_64-dvd1.iso
"""
path = os.path.join(self.topdir(arch, create_dir=create_dir), "iso", filename)
if create_dir:
makedirs(path)
return path
def tmp_dir(self, arch=None, variant=None, create_dir=True):
"""
Examples:
work/global/tmp
work/x86_64/tmp
work/x86_64/tmp-Server
"""
dir_name = "tmp"
if variant:
dir_name += "-%s" % variant.uid
path = os.path.join(self.topdir(arch=arch, create_dir=create_dir), dir_name)
if create_dir:
makedirs(path)
return path
def product_id(self, arch, variant, create_dir=True):
"""
Examples:
work/x86_64/product_id/productid-Server.x86_64.pem/productid
"""
# file_name = "%s.%s.pem" % (variant, arch)
# HACK: modifyrepo doesn't handle renames -> $dir/productid
file_name = "productid"
path = os.path.join(
self.topdir(arch, create_dir=create_dir),
"product_id",
"%s.%s.pem" % (variant, arch),
)
if create_dir:
makedirs(path)
path = os.path.join(path, file_name)
return path
def image_build_dir(self, variant, create_dir=True):
"""
@param variant
@param create_dir=True
Examples:
work/image-build/Server
"""
path = os.path.join(
self.topdir("image-build", create_dir=create_dir), variant.uid
)
if create_dir:
makedirs(path)
return path
def image_build_conf(
self, variant, image_name, image_type, arches=None, create_dir=True
):
"""
@param variant
@param image-name
@param image-type (e.g docker)
@param arches
@param create_dir=True
Examples:
work/image-build/Server/docker_rhel-server-docker.cfg
work/image-build/Server/docker_rhel-server-docker_x86_64.cfg
work/image-build/Server/docker_rhel-server-docker_x86_64-ppc64le.cfg
"""
path = os.path.join(
self.image_build_dir(variant), "%s_%s" % (image_type, image_name)
)
if arches is not None:
path = "%s_%s" % (path, "-".join(list(arches)))
path = "%s.cfg" % path
return path
def module_defaults_dir(self, create_dir=True):
"""
Example:
work/global/module_defaults
"""
path = os.path.join(self.topdir(create_dir=create_dir), "module_defaults")
if create_dir:
makedirs(path)
return path
def module_obsoletes_dir(self, create_dir=True):
"""
Example:
work/global/module_obsoletes
"""
path = os.path.join(self.topdir(create_dir=create_dir), "module_obsoletes")
if create_dir:
makedirs(path)
return path
def pkgset_file_cache(self, pkgset_name):
"""
Returns the path to file in which the cached version of
PackageSetBase.file_cache should be stored.
Example:
work/global/pkgset_f33-compose_file_cache.pickle
"""
filename = "pkgset_%s_file_cache.pickle" % pkgset_name
return os.path.join(self.topdir(arch="global"), filename)
def pkgset_reuse_file(self, pkgset_name):
"""
Example:
work/global/pkgset_f30-compose_reuse.pickle
"""
filename = "pkgset_%s_reuse.pickle" % pkgset_name
return os.path.join(self.topdir(arch="global", create_dir=False), filename)
class ComposePaths(object):
def __init__(self, compose):
self.compose = compose
# TODO: TREES?
def topdir(self, arch=None, variant=None, create_dir=True, relative=False):
"""
Examples:
compose
compose/Server/x86_64
"""
if bool(arch) != bool(variant):
raise TypeError("topdir(): either none or 2 arguments are expected")
path = ""
if not relative:
path = os.path.join(self.compose.topdir, "compose")
if arch or variant:
if variant.type == "addon":
return self.topdir(
arch, variant.parent, create_dir=create_dir, relative=relative
)
path = os.path.join(path, variant.uid, arch)
if create_dir and not relative:
makedirs(path)
return path
def tree_dir(self, arch, variant, create_dir=True, relative=False):
"""
Examples:
compose/Server/x86_64/os
compose/Server-optional/x86_64/os
"""
if arch == "src":
arch = "source"
if arch == "source":
tree_dir = "tree"
else:
# use 'os' dir due to historical reasons
tree_dir = "os"
path = os.path.join(
self.topdir(arch, variant, create_dir=create_dir, relative=relative),
tree_dir,
)
if create_dir and not relative:
makedirs(path)
return path
def os_tree(self, arch, variant, create_dir=True, relative=False):
return self.tree_dir(arch, variant, create_dir=create_dir, relative=relative)
def repository(self, arch, variant, create_dir=True, relative=False):
"""
Examples:
compose/Server/x86_64/os
compose/Server/x86_64/addons/LoadBalancer
"""
if variant.type == "addon":
path = self.packages(
arch, variant, create_dir=create_dir, relative=relative
)
else:
path = self.tree_dir(
arch, variant, create_dir=create_dir, relative=relative
)
if create_dir and not relative:
makedirs(path)
return path
def packages(self, arch, variant, create_dir=True, relative=False):
"""
Examples:
compose/Server/x86_64/os/Packages
compose/Server/x86_64/os/addons/LoadBalancer
compose/Server-optional/x86_64/os/Packages
"""
if variant.type == "addon":
path = os.path.join(
self.tree_dir(arch, variant, create_dir=create_dir, relative=relative),
"addons",
variant.id,
)
else:
path = os.path.join(
self.tree_dir(arch, variant, create_dir=create_dir, relative=relative),
"Packages",
)
if create_dir and not relative:
makedirs(path)
return path
def debug_topdir(self, arch, variant, create_dir=True, relative=False):
"""
Examples:
compose/Server/x86_64/debug
compose/Server-optional/x86_64/debug
"""
path = os.path.join(
self.topdir(arch, variant, create_dir=create_dir, relative=relative),
"debug",
)
if create_dir and not relative:
makedirs(path)
return path
def debug_tree(self, arch, variant, create_dir=True, relative=False):
"""
Examples:
compose/Server/x86_64/debug/tree
compose/Server-optional/x86_64/debug/tree
"""
path = os.path.join(
self.debug_topdir(arch, variant, create_dir=create_dir, relative=relative),
"tree",
)
if create_dir and not relative:
makedirs(path)
return path
def debug_packages(self, arch, variant, create_dir=True, relative=False):
"""
Examples:
compose/Server/x86_64/debug/tree/Packages
compose/Server/x86_64/debug/tree/addons/LoadBalancer
compose/Server-optional/x86_64/debug/tree/Packages
"""
if arch in ("source", "src"):
return None
if variant.type == "addon":
path = os.path.join(
self.debug_tree(
arch, variant, create_dir=create_dir, relative=relative
),
"addons",
variant.id,
)
else:
path = os.path.join(
self.debug_tree(
arch, variant, create_dir=create_dir, relative=relative
),
"Packages",
)
if create_dir and not relative:
makedirs(path)
return path
def debug_repository(self, arch, variant, create_dir=True, relative=False):
"""
Examples:
compose/Server/x86_64/debug/tree
compose/Server/x86_64/debug/tree/addons/LoadBalancer
compose/Server-optional/x86_64/debug/tree
"""
if arch in ("source", "src"):
return None
if variant.type == "addon":
path = os.path.join(
self.debug_tree(
arch, variant, create_dir=create_dir, relative=relative
),
"addons",
variant.id,
)
else:
path = self.debug_tree(
arch, variant, create_dir=create_dir, relative=relative
)
if create_dir and not relative:
makedirs(path)
return path
def iso_dir(self, arch, variant, symlink_to=None, create_dir=True, relative=False):
"""
Examples:
compose/Server/x86_64/iso
None
"""
if variant.type == "addon":
return None
if variant.type == "optional":
if not self.compose.conf.get("create_optional_isos", False):
return None
if arch == "src":
arch = "source"
path = os.path.join(
self.topdir(arch, variant, create_dir=create_dir, relative=relative), "iso"
)
if symlink_to:
# TODO: create_dir
topdir = self.compose.topdir.rstrip("/") + "/"
relative_dir = path[len(topdir) :]
target_dir = os.path.join(symlink_to, self.compose.compose_id, relative_dir)
if create_dir and not relative:
makedirs(target_dir)
try:
os.symlink(target_dir, path)
except OSError as ex:
if ex.errno != errno.EEXIST:
raise
msg = "Symlink pointing to '%s' expected: %s" % (target_dir, path)
if not os.path.islink(path):
raise RuntimeError(msg)
if os.path.abspath(os.readlink(path)) != target_dir:
raise RuntimeError(msg)
else:
if create_dir and not relative:
makedirs(path)
return path
def iso_path(
self, arch, variant, filename, symlink_to=None, create_dir=True, relative=False
):
"""
Examples:
compose/Server/x86_64/iso/rhel-7.0-20120127.0-Server-x86_64-dvd1.iso
None
"""
path = self.iso_dir(
arch,
variant,
symlink_to=symlink_to,
create_dir=create_dir,
relative=relative,
)
if path is None:
return None
return os.path.join(path, filename)
def image_dir(self, variant, symlink_to=None, relative=False):
"""
The arch is listed as literal '%(arch)s'
Examples:
compose/Server/%(arch)s/images
None
@param variant
@param symlink_to=None
@param relative=False
"""
path = os.path.join(
self.topdir("%(arch)s", variant, create_dir=False, relative=relative),
"images",
)
if symlink_to:
topdir = self.compose.topdir.rstrip("/") + "/"
relative_dir = path[len(topdir) :]
target_dir = os.path.join(symlink_to, self.compose.compose_id, relative_dir)
try:
os.symlink(target_dir, path)
except OSError as ex:
if ex.errno != errno.EEXIST:
raise
msg = "Symlink pointing to '%s' expected: %s" % (target_dir, path)
if not os.path.islink(path):
raise RuntimeError(msg)
if os.path.abspath(os.readlink(path)) != target_dir:
raise RuntimeError(msg)
return path
def jigdo_dir(self, arch, variant, create_dir=True, relative=False):
"""
Examples:
compose/Server/x86_64/jigdo
None
"""
if variant.type == "addon":
return None
if variant.type == "optional":
if not self.compose.conf.get("create_optional_isos", False):
return None
if arch == "src":
arch = "source"
path = os.path.join(
self.topdir(arch, variant, create_dir=create_dir, relative=relative),
"jigdo",
)
if create_dir and not relative:
makedirs(path)
return path
def metadata(self, file_name=None, create_dir=True, relative=False):
"""
Examples:
compose/metadata
compose/metadata/rpms.json
"""
path = os.path.join(
self.topdir(create_dir=create_dir, relative=relative), "metadata"
)
if create_dir and not relative:
makedirs(path)
if file_name:
path = os.path.join(path, file_name)
return path