# -*- coding: utf-8 -*- # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; version 2 of the License. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU Library General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, see . __all__ = ("Paths",) import errno import os from kobo.shortcuts import relative_path from pungi.util import makedirs, find_old_compose class Paths(object): def __init__(self, compose): self._compose = compose paths_module_name = compose.conf.get("paths_module") if paths_module_name: # custom paths paths_module = __import__( paths_module_name, globals(), locals(), ["LogPaths", "WorkPaths", "ComposePaths"], ) self.compose = paths_module.ComposePaths(compose) self.log = paths_module.LogPaths(compose) self.work = paths_module.WorkPaths(compose) else: # default paths self.compose = ComposePaths(compose) self.log = LogPaths(compose) self.work = WorkPaths(compose) # self.metadata ? def get_old_compose_topdir(self, **kwargs): """ Finds old compose using the `find_old_compose` function and returns the path to it. The `kwargs` are passed to `find_old_compose`. """ is_layered = self._compose.ci_base.release.is_layered return find_old_compose( self._compose.old_composes, self._compose.ci_base.release.short, self._compose.ci_base.release.version, self._compose.ci_base.release.type_suffix, self._compose.ci_base.base_product.short if is_layered else None, self._compose.ci_base.base_product.version if is_layered else None, **kwargs ) def old_compose_path(self, path, **kwargs): """ Translates `path` to the topdir of old compose. :param str path: Path to translate. :param kwargs: The kwargs passed to `find_old_compose` function. :return: None if old compose cannot be used or if `path` does not exist in the old compose topdir. Otherwise path translated to old_compose topdir. Example: old_repo_dir = compose.old_compose_path( compose.paths.work.pkgset_repo(pkgset.name, arch="global")) """ old_compose_topdir = self.get_old_compose_topdir(**kwargs) if not old_compose_topdir: return None rel_path = relative_path(path, self._compose.topdir.rstrip("/") + "/") old_path = os.path.join(old_compose_topdir, rel_path) if not os.path.exists(old_path): return None return old_path class LogPaths(object): def __init__(self, compose): self.compose = compose def topdir(self, arch=None, create_dir=True): """ Examples: log/global log/x86_64 """ arch = arch or "global" path = os.path.join(self.compose.topdir, "logs", arch) if create_dir: makedirs(path) return path def koji_tasks_dir(self, create_dir=True): """ Examples: logs/global/koji-tasks """ path = os.path.join(self.topdir(create_dir=create_dir), "koji-tasks") if create_dir: makedirs(path) return path def log_file(self, arch, log_name, create_dir=True, ext=None): ext = ext or "log" arch = arch or "global" if log_name.endswith(".log"): log_name = log_name[:-4] return os.path.join( self.topdir(arch, create_dir=create_dir), "%s.%s.%s" % (log_name, arch, ext) ) class WorkPaths(object): def __init__(self, compose): self.compose = compose def topdir(self, arch=None, create_dir=True): """ Examples: work/global work/x86_64 """ arch = arch or "global" path = os.path.join(self.compose.topdir, "work", arch) if create_dir: makedirs(path) return path def variants_file(self, arch=None, create_dir=True): """ Examples: work/global/variants.xml """ arch = "global" path = os.path.join(self.topdir(arch, create_dir=create_dir), "variants.xml") return path def comps(self, arch=None, variant=None, create_dir=True): """ Examples: work/x86_64/comps/comps-86_64.xml work/x86_64/comps/comps-Server.x86_64.xml """ arch = arch or "global" if variant is None: file_name = "comps-%s.xml" % arch else: file_name = "comps-%s.%s.xml" % (variant.uid, arch) path = os.path.join(self.topdir(arch, create_dir=create_dir), "comps") if create_dir: makedirs(path) path = os.path.join(path, file_name) return path def gather_result(self, arch=None, variant=None, create_dir=True): """ Examples: work/x86_64/gather_result/x86_64.result work/x86_64/gather_result/Server.x86_64.result """ arch = arch or "global" file_name = "" if variant: file_name += variant.uid + "." file_name += arch + "." file_name += "result" path = os.path.join(self.topdir(arch, create_dir=create_dir), "gather_result") if create_dir: makedirs(path) path = os.path.join(path, file_name) return path def pungi_conf(self, arch=None, variant=None, create_dir=True, source_name=None): """ Examples: work/x86_64/pungi/x86_64.conf work/x86_64/pungi/Server.x86_64.conf """ arch = arch or "global" file_name = "" if variant: file_name += variant.uid + "." file_name += arch + "." if source_name: file_name += source_name + "." file_name += "conf" path = os.path.join(self.topdir(arch, create_dir=create_dir), "pungi") if create_dir: makedirs(path) path = os.path.join(path, file_name) return path def fus_conf(self, arch, variant, iteration, create_dir=True): """ Examples: work/x86_64/fus/Server-solvables.x86_64.conf """ file_name = "%s-solvables-%d.%s.conf" % (variant.uid, iteration, arch) path = os.path.join(self.topdir(arch, create_dir=create_dir), "fus") if create_dir: makedirs(path) return os.path.join(path, file_name) def pungi_log(self, arch=None, variant=None, create_dir=True, source_name=None): """ Examples: work/x86_64/pungi/x86_64.log work/x86_64/pungi/Server.x86_64.log """ path = self.pungi_conf(arch, variant, create_dir=create_dir) path = path[:-5] if source_name: path += "." + source_name return path + ".log" def pungi_cache_dir(self, arch, variant=None, create_dir=True): """ Examples: work/global/pungi-cache """ # WARNING: Using the same cache dir with repos of the same names # may lead to a race condition. # We should use per arch variant cache dirs to workaround this. path = os.path.join(self.topdir(arch, create_dir=create_dir), "pungi-cache") if variant: path = os.path.join(path, variant.uid) if create_dir: makedirs(path) return path def _repo(self, type, arch=None, variant=None, create_dir=True): arch = arch or "global" path = os.path.join(self.topdir(arch, create_dir=create_dir), "%s_repo" % type) if variant: path += "_" + variant.uid if create_dir: makedirs(path) return path def comps_repo(self, arch=None, variant=None, create_dir=True): """ Examples: work/x86_64/comps_repo_Server work/global/comps_repo """ return self._repo("comps", arch, variant, create_dir=create_dir) def pkgset_repo(self, pkgset_name, arch=None, create_dir=True): """ Examples: work/x86_64/repo/f30-compose work/global/repo/f30-compose """ arch = arch or "global" path = os.path.join( self.topdir(arch, create_dir=create_dir), "repo", pkgset_name ) if create_dir: makedirs(path) return path def lookaside_repo(self, arch, variant, create_dir=True): """ Examples: work/x86_64/Server/lookaside_repo """ path = os.path.join( self.topdir(arch, create_dir=create_dir), variant.uid, "lookaside_repo" ) if create_dir: makedirs(path) return path def package_list( self, arch=None, variant=None, pkgset=None, pkg_type=None, create_dir=True ): """ Examples: work/x86_64/package_list/x86_64.conf work/x86_64/package_list/Server.x86_64.conf work/x86_64/package_list/Server.x86_64.rpm.conf """ arch = arch or "global" if variant is not None: file_name = "%s.%s" % (variant, arch) else: file_name = "%s" % arch if pkgset: file_name += "." + pkgset.name if pkg_type is not None: file_name += ".%s" % pkg_type file_name += ".conf" path = os.path.join(self.topdir(arch, create_dir=create_dir), "package_list") if create_dir: makedirs(path) path = os.path.join(path, file_name) return path def lookaside_package_list(self, arch, variant, create_dir=True): """ Examples: work/x86_64/package_list/Server.x86_64.lookaside.conf """ return self.package_list( arch, variant, pkg_type="lookaside", create_dir=create_dir ) def pungi_download_dir(self, arch, create_dir=True): """ Examples: work/x86_64/pungi_download """ path = os.path.join(self.topdir(arch, create_dir=create_dir), "pungi_download") if create_dir: makedirs(path) return path def buildinstall_dir( self, arch, create_dir=True, allow_topdir_override=False, variant=None ): """ :param bool allow_topdir_override: When True, the "buildinstall_topdir" will be used (if set) instead of real "topdir". Examples: work/x86_64/buildinstall """ if arch == "global": raise RuntimeError("Global buildinstall dir makes no sense.") buildinstall_topdir = self.compose.conf.get("buildinstall_topdir", "") if allow_topdir_override and buildinstall_topdir: topdir_basename = os.path.basename(self.compose.topdir) path = os.path.join( buildinstall_topdir, "buildinstall-%s" % topdir_basename, arch ) else: path = os.path.join( self.topdir(arch, create_dir=create_dir), "buildinstall" ) if variant: path = os.path.join(path, variant.uid) return path def extra_files_dir(self, arch, variant, create_dir=True): """ Examples: work/x86_64/Server/extra-files """ if arch == "global": raise RuntimeError("Global extra files dir makes no sense.") path = os.path.join( self.topdir(arch, create_dir=create_dir), variant.uid, "extra-files" ) if create_dir: makedirs(path) return path def extra_iso_extra_files_dir(self, arch, variant, create_dir=True): """ Examples: work/x86_64/Server/extra-iso-extra-files """ if arch == "global": raise RuntimeError("Global extra files dir makes no sense.") path = os.path.join( self.topdir(arch, create_dir=create_dir), variant.uid, "extra-iso-extra-files", ) if create_dir: makedirs(path) return path def iso_staging_dir(self, arch, variant, filename, create_dir=True): """ Examples: work/x86_64/Server/iso-staging-dir/file.iso/ """ path = os.path.join( self.topdir(arch, create_dir=create_dir), variant.uid, "iso-staging-dir", filename, ) if create_dir: makedirs(path) return path def repo_package_list(self, arch, variant, pkg_type=None, create_dir=True): """ Examples: work/x86_64/repo_package_list/Server.x86_64.rpm.conf """ file_name = "%s.%s" % (variant.uid, arch) if pkg_type is not None: file_name += ".%s" % pkg_type file_name += ".conf" path = os.path.join( self.topdir(arch, create_dir=create_dir), "repo_package_list" ) if create_dir: makedirs(path) path = os.path.join(path, file_name) return path def iso_dir(self, arch, filename, create_dir=True): """ Examples: work/x86_64/iso/Project-1.0-20151203.0-Client-x86_64-dvd1.iso """ path = os.path.join(self.topdir(arch, create_dir=create_dir), "iso", filename) if create_dir: makedirs(path) return path def tmp_dir(self, arch=None, variant=None, create_dir=True): """ Examples: work/global/tmp work/x86_64/tmp work/x86_64/tmp-Server """ dir_name = "tmp" if variant: dir_name += "-%s" % variant.uid path = os.path.join(self.topdir(arch=arch, create_dir=create_dir), dir_name) if create_dir: makedirs(path) return path def product_id(self, arch, variant, create_dir=True): """ Examples: work/x86_64/product_id/productid-Server.x86_64.pem/productid """ # file_name = "%s.%s.pem" % (variant, arch) # HACK: modifyrepo doesn't handle renames -> $dir/productid file_name = "productid" path = os.path.join( self.topdir(arch, create_dir=create_dir), "product_id", "%s.%s.pem" % (variant, arch), ) if create_dir: makedirs(path) path = os.path.join(path, file_name) return path def image_build_dir(self, variant, create_dir=True): """ @param variant @param create_dir=True Examples: work/image-build/Server """ path = os.path.join( self.topdir("image-build", create_dir=create_dir), variant.uid ) if create_dir: makedirs(path) return path def image_build_conf( self, variant, image_name, image_type, arches=None, create_dir=True ): """ @param variant @param image-name @param image-type (e.g docker) @param arches @param create_dir=True Examples: work/image-build/Server/docker_rhel-server-docker.cfg work/image-build/Server/docker_rhel-server-docker_x86_64.cfg work/image-build/Server/docker_rhel-server-docker_x86_64-ppc64le.cfg """ path = os.path.join( self.image_build_dir(variant), "%s_%s" % (image_type, image_name) ) if arches is not None: path = "%s_%s" % (path, "-".join(list(arches))) path = "%s.cfg" % path return path def module_defaults_dir(self, create_dir=True): """ Example: work/global/module_defaults """ path = os.path.join(self.topdir(create_dir=create_dir), "module_defaults") if create_dir: makedirs(path) return path def module_obsoletes_dir(self, create_dir=True): """ Example: work/global/module_obsoletes """ path = os.path.join(self.topdir(create_dir=create_dir), "module_obsoletes") if create_dir: makedirs(path) return path def pkgset_file_cache(self, pkgset_name): """ Returns the path to file in which the cached version of PackageSetBase.file_cache should be stored. Example: work/global/pkgset_f33-compose_file_cache.pickle """ filename = "pkgset_%s_file_cache.pickle" % pkgset_name return os.path.join(self.topdir(arch="global"), filename) def pkgset_reuse_file(self, pkgset_name): """ Example: work/global/pkgset_f30-compose_reuse.pickle """ filename = "pkgset_%s_reuse.pickle" % pkgset_name return os.path.join(self.topdir(arch="global", create_dir=False), filename) class ComposePaths(object): def __init__(self, compose): self.compose = compose # TODO: TREES? def topdir(self, arch=None, variant=None, create_dir=True, relative=False): """ Examples: compose compose/Server/x86_64 """ if bool(arch) != bool(variant): raise TypeError("topdir(): either none or 2 arguments are expected") path = "" if not relative: path = os.path.join(self.compose.topdir, "compose") if arch or variant: if variant.type == "addon": return self.topdir( arch, variant.parent, create_dir=create_dir, relative=relative ) path = os.path.join(path, variant.uid, arch) if create_dir and not relative: makedirs(path) return path def tree_dir(self, arch, variant, create_dir=True, relative=False): """ Examples: compose/Server/x86_64/os compose/Server-optional/x86_64/os """ if arch == "src": arch = "source" if arch == "source": tree_dir = "tree" else: # use 'os' dir due to historical reasons tree_dir = "os" path = os.path.join( self.topdir(arch, variant, create_dir=create_dir, relative=relative), tree_dir, ) if create_dir and not relative: makedirs(path) return path def os_tree(self, arch, variant, create_dir=True, relative=False): return self.tree_dir(arch, variant, create_dir=create_dir, relative=relative) def repository(self, arch, variant, create_dir=True, relative=False): """ Examples: compose/Server/x86_64/os compose/Server/x86_64/addons/LoadBalancer """ if variant.type == "addon": path = self.packages( arch, variant, create_dir=create_dir, relative=relative ) else: path = self.tree_dir( arch, variant, create_dir=create_dir, relative=relative ) if create_dir and not relative: makedirs(path) return path def packages(self, arch, variant, create_dir=True, relative=False): """ Examples: compose/Server/x86_64/os/Packages compose/Server/x86_64/os/addons/LoadBalancer compose/Server-optional/x86_64/os/Packages """ if variant.type == "addon": path = os.path.join( self.tree_dir(arch, variant, create_dir=create_dir, relative=relative), "addons", variant.id, ) else: path = os.path.join( self.tree_dir(arch, variant, create_dir=create_dir, relative=relative), "Packages", ) if create_dir and not relative: makedirs(path) return path def debug_topdir(self, arch, variant, create_dir=True, relative=False): """ Examples: compose/Server/x86_64/debug compose/Server-optional/x86_64/debug """ path = os.path.join( self.topdir(arch, variant, create_dir=create_dir, relative=relative), "debug", ) if create_dir and not relative: makedirs(path) return path def debug_tree(self, arch, variant, create_dir=True, relative=False): """ Examples: compose/Server/x86_64/debug/tree compose/Server-optional/x86_64/debug/tree """ path = os.path.join( self.debug_topdir(arch, variant, create_dir=create_dir, relative=relative), "tree", ) if create_dir and not relative: makedirs(path) return path def debug_packages(self, arch, variant, create_dir=True, relative=False): """ Examples: compose/Server/x86_64/debug/tree/Packages compose/Server/x86_64/debug/tree/addons/LoadBalancer compose/Server-optional/x86_64/debug/tree/Packages """ if arch in ("source", "src"): return None if variant.type == "addon": path = os.path.join( self.debug_tree( arch, variant, create_dir=create_dir, relative=relative ), "addons", variant.id, ) else: path = os.path.join( self.debug_tree( arch, variant, create_dir=create_dir, relative=relative ), "Packages", ) if create_dir and not relative: makedirs(path) return path def debug_repository(self, arch, variant, create_dir=True, relative=False): """ Examples: compose/Server/x86_64/debug/tree compose/Server/x86_64/debug/tree/addons/LoadBalancer compose/Server-optional/x86_64/debug/tree """ if arch in ("source", "src"): return None if variant.type == "addon": path = os.path.join( self.debug_tree( arch, variant, create_dir=create_dir, relative=relative ), "addons", variant.id, ) else: path = self.debug_tree( arch, variant, create_dir=create_dir, relative=relative ) if create_dir and not relative: makedirs(path) return path def iso_dir(self, arch, variant, symlink_to=None, create_dir=True, relative=False): """ Examples: compose/Server/x86_64/iso None """ if variant.type == "addon": return None if variant.type == "optional": if not self.compose.conf.get("create_optional_isos", False): return None if arch == "src": arch = "source" path = os.path.join( self.topdir(arch, variant, create_dir=create_dir, relative=relative), "iso" ) if symlink_to: # TODO: create_dir topdir = self.compose.topdir.rstrip("/") + "/" relative_dir = path[len(topdir) :] target_dir = os.path.join(symlink_to, self.compose.compose_id, relative_dir) if create_dir and not relative: makedirs(target_dir) try: os.symlink(target_dir, path) except OSError as ex: if ex.errno != errno.EEXIST: raise msg = "Symlink pointing to '%s' expected: %s" % (target_dir, path) if not os.path.islink(path): raise RuntimeError(msg) if os.path.abspath(os.readlink(path)) != target_dir: raise RuntimeError(msg) else: if create_dir and not relative: makedirs(path) return path def iso_path( self, arch, variant, filename, symlink_to=None, create_dir=True, relative=False ): """ Examples: compose/Server/x86_64/iso/rhel-7.0-20120127.0-Server-x86_64-dvd1.iso None """ path = self.iso_dir( arch, variant, symlink_to=symlink_to, create_dir=create_dir, relative=relative, ) if path is None: return None return os.path.join(path, filename) def image_dir(self, variant, symlink_to=None, relative=False): """ The arch is listed as literal '%(arch)s' Examples: compose/Server/%(arch)s/images None @param variant @param symlink_to=None @param relative=False """ path = os.path.join( self.topdir("%(arch)s", variant, create_dir=False, relative=relative), "images", ) if symlink_to: topdir = self.compose.topdir.rstrip("/") + "/" relative_dir = path[len(topdir) :] target_dir = os.path.join(symlink_to, self.compose.compose_id, relative_dir) try: os.symlink(target_dir, path) except OSError as ex: if ex.errno != errno.EEXIST: raise msg = "Symlink pointing to '%s' expected: %s" % (target_dir, path) if not os.path.islink(path): raise RuntimeError(msg) if os.path.abspath(os.readlink(path)) != target_dir: raise RuntimeError(msg) return path def jigdo_dir(self, arch, variant, create_dir=True, relative=False): """ Examples: compose/Server/x86_64/jigdo None """ if variant.type == "addon": return None if variant.type == "optional": if not self.compose.conf.get("create_optional_isos", False): return None if arch == "src": arch = "source" path = os.path.join( self.topdir(arch, variant, create_dir=create_dir, relative=relative), "jigdo", ) if create_dir and not relative: makedirs(path) return path def metadata(self, file_name=None, create_dir=True, relative=False): """ Examples: compose/metadata compose/metadata/rpms.json """ path = os.path.join( self.topdir(create_dir=create_dir, relative=relative), "metadata" ) if create_dir and not relative: makedirs(path) if file_name: path = os.path.join(path, file_name) return path