364d7f5229
With this patch the gather_source option is no longer used. Instead, all sources are always used. If they return at least some input packages, then a configured method is used and the returned lists of packages from all sources are merged. The method used for gathering can be configured for each variant and gather source separately. Additional packages are only added to the comps source. Each gathering step is logged separately. All the logs are preserved for later inspection. Signed-off-by: Lubomír Sedlář <lsedlar@redhat.com>
586 lines
20 KiB
Python
586 lines
20 KiB
Python
# -*- coding: utf-8 -*-
|
|
|
|
|
|
# This program is free software; you can redistribute it and/or modify
|
|
# it under the terms of the GNU General Public License as published by
|
|
# the Free Software Foundation; version 2 of the License.
|
|
#
|
|
# This program is distributed in the hope that it will be useful,
|
|
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
# GNU Library General Public License for more details.
|
|
#
|
|
# You should have received a copy of the GNU General Public License
|
|
# along with this program; if not, see <https://gnu.org/licenses/>.
|
|
|
|
|
|
__all__ = (
|
|
"Paths",
|
|
)
|
|
|
|
|
|
import errno
|
|
import os
|
|
|
|
from pungi.util import makedirs
|
|
|
|
|
|
class Paths(object):
|
|
def __init__(self, compose):
|
|
paths_module_name = compose.conf.get("paths_module")
|
|
if paths_module_name:
|
|
# custom paths
|
|
paths_module = __import__(paths_module_name, globals(), locals(), ["LogPaths", "WorkPaths", "ComposePaths"])
|
|
self.compose = paths_module.ComposePaths(compose)
|
|
self.log = paths_module.LogPaths(compose)
|
|
self.work = paths_module.WorkPaths(compose)
|
|
else:
|
|
# default paths
|
|
self.compose = ComposePaths(compose)
|
|
self.log = LogPaths(compose)
|
|
self.work = WorkPaths(compose)
|
|
# self.metadata ?
|
|
|
|
|
|
class LogPaths(object):
|
|
def __init__(self, compose):
|
|
self.compose = compose
|
|
|
|
def topdir(self, arch=None, create_dir=True):
|
|
"""
|
|
Examples:
|
|
log/global
|
|
log/x86_64
|
|
"""
|
|
arch = arch or "global"
|
|
path = os.path.join(self.compose.topdir, "logs", arch)
|
|
if create_dir:
|
|
makedirs(path)
|
|
return path
|
|
|
|
def log_file(self, arch, log_name, create_dir=True):
|
|
arch = arch or "global"
|
|
if log_name.endswith(".log"):
|
|
log_name = log_name[:-4]
|
|
return os.path.join(self.topdir(arch, create_dir=create_dir), "%s.%s.log" % (log_name, arch))
|
|
|
|
|
|
class WorkPaths(object):
|
|
def __init__(self, compose):
|
|
self.compose = compose
|
|
|
|
def topdir(self, arch=None, create_dir=True):
|
|
"""
|
|
Examples:
|
|
work/global
|
|
work/x86_64
|
|
"""
|
|
arch = arch or "global"
|
|
path = os.path.join(self.compose.topdir, "work", arch)
|
|
if create_dir:
|
|
makedirs(path)
|
|
return path
|
|
|
|
def variants_file(self, arch=None, create_dir=True):
|
|
"""
|
|
Examples:
|
|
work/global/variants.xml
|
|
"""
|
|
arch = "global"
|
|
path = os.path.join(self.topdir(arch, create_dir=create_dir), "variants.xml")
|
|
return path
|
|
|
|
def comps(self, arch=None, variant=None, create_dir=True):
|
|
"""
|
|
Examples:
|
|
work/x86_64/comps/comps-86_64.xml
|
|
work/x86_64/comps/comps-Server.x86_64.xml
|
|
"""
|
|
arch = arch or "global"
|
|
if variant is None:
|
|
file_name = "comps-%s.xml" % arch
|
|
else:
|
|
file_name = "comps-%s.%s.xml" % (variant.uid, arch)
|
|
path = os.path.join(self.topdir(arch, create_dir=create_dir), "comps")
|
|
if create_dir:
|
|
makedirs(path)
|
|
path = os.path.join(path, file_name)
|
|
return path
|
|
|
|
def pungi_conf(self, arch=None, variant=None, create_dir=True, source_name=None):
|
|
"""
|
|
Examples:
|
|
work/x86_64/pungi/x86_64.conf
|
|
work/x86_64/pungi/Server.x86_64.conf
|
|
"""
|
|
arch = arch or "global"
|
|
file_name = ''
|
|
if variant:
|
|
file_name += variant.uid + '.'
|
|
file_name += arch + '.'
|
|
if source_name:
|
|
file_name += source_name + '.'
|
|
file_name += 'conf'
|
|
path = os.path.join(self.topdir(arch, create_dir=create_dir), "pungi")
|
|
if create_dir:
|
|
makedirs(path)
|
|
path = os.path.join(path, file_name)
|
|
return path
|
|
|
|
def pungi_log(self, arch=None, variant=None, create_dir=True, source_name=None):
|
|
"""
|
|
Examples:
|
|
work/x86_64/pungi/x86_64.log
|
|
work/x86_64/pungi/Server.x86_64.log
|
|
"""
|
|
path = self.pungi_conf(arch, variant, create_dir=create_dir)
|
|
path = path[:-5]
|
|
if source_name:
|
|
path += '.' + source_name
|
|
return path + ".log"
|
|
|
|
def pungi_cache_dir(self, arch, variant=None, create_dir=True):
|
|
"""
|
|
Examples:
|
|
work/global/pungi-cache
|
|
"""
|
|
# WARNING: Using the same cache dir with repos of the same names may lead to a race condition
|
|
# We should use per arch variant cache dirs to workaround this.
|
|
path = os.path.join(self.topdir(arch, create_dir=create_dir), "pungi-cache")
|
|
if variant:
|
|
path = os.path.join(path, variant.uid)
|
|
if create_dir:
|
|
makedirs(path)
|
|
return path
|
|
|
|
def comps_repo(self, arch=None, create_dir=True):
|
|
"""
|
|
Examples:
|
|
work/x86_64/comps-repo
|
|
work/global/comps-repo
|
|
"""
|
|
arch = arch or "global"
|
|
path = os.path.join(self.topdir(arch, create_dir=create_dir), "comps_repo")
|
|
if create_dir:
|
|
makedirs(path)
|
|
return path
|
|
|
|
def arch_repo(self, arch=None, create_dir=True):
|
|
"""
|
|
Examples:
|
|
work/x86_64/repo
|
|
work/global/repo
|
|
"""
|
|
arch = arch or "global"
|
|
path = os.path.join(self.topdir(arch, create_dir=create_dir), "repo")
|
|
if create_dir:
|
|
makedirs(path)
|
|
return path
|
|
|
|
def package_list(self, arch=None, variant=None, pkg_type=None, create_dir=True):
|
|
"""
|
|
Examples:
|
|
work/x86_64/package_list/x86_64.conf
|
|
work/x86_64/package_list/Server.x86_64.conf
|
|
work/x86_64/package_list/Server.x86_64.rpm.conf
|
|
"""
|
|
arch = arch or "global"
|
|
if variant is not None:
|
|
file_name = "%s.%s" % (variant, arch)
|
|
else:
|
|
file_name = "%s" % arch
|
|
if pkg_type is not None:
|
|
file_name += ".%s" % pkg_type
|
|
file_name += ".conf"
|
|
path = os.path.join(self.topdir(arch, create_dir=create_dir), "package_list")
|
|
if create_dir:
|
|
makedirs(path)
|
|
path = os.path.join(path, file_name)
|
|
return path
|
|
|
|
def pungi_download_dir(self, arch, create_dir=True):
|
|
"""
|
|
Examples:
|
|
work/x86_64/pungi_download
|
|
"""
|
|
path = os.path.join(self.topdir(arch, create_dir=create_dir), "pungi_download")
|
|
if create_dir:
|
|
makedirs(path)
|
|
return path
|
|
|
|
def buildinstall_dir(self, arch, create_dir=True,
|
|
allow_topdir_override=False, variant=None):
|
|
"""
|
|
:param bool allow_topdir_override: When True, the
|
|
"buildinstall_topdir" will be used (if set) instead of real
|
|
"topdir".
|
|
Examples:
|
|
work/x86_64/buildinstall
|
|
"""
|
|
if arch == "global":
|
|
raise RuntimeError("Global buildinstall dir makes no sense.")
|
|
|
|
buildinstall_topdir = self.compose.conf.get("buildinstall_topdir", "")
|
|
if allow_topdir_override and buildinstall_topdir:
|
|
topdir_basename = os.path.basename(self.compose.topdir)
|
|
path = os.path.join(
|
|
buildinstall_topdir, "buildinstall-%s" % topdir_basename, arch)
|
|
else:
|
|
path = os.path.join(self.topdir(arch, create_dir=create_dir), "buildinstall")
|
|
|
|
if variant:
|
|
path = os.path.join(path, variant.uid)
|
|
return path
|
|
|
|
def extra_files_dir(self, arch, variant, create_dir=True):
|
|
"""
|
|
Examples:
|
|
work/x86_64/Server/extra-files
|
|
"""
|
|
if arch == "global":
|
|
raise RuntimeError("Global extra files dir makes no sense.")
|
|
path = os.path.join(self.topdir(arch, create_dir=create_dir), variant.uid, "extra-files")
|
|
if create_dir:
|
|
makedirs(path)
|
|
return path
|
|
|
|
def repo_package_list(self, arch, variant, pkg_type=None, create_dir=True):
|
|
"""
|
|
Examples:
|
|
work/x86_64/repo_package_list/Server.x86_64.rpm.conf
|
|
"""
|
|
file_name = "%s.%s" % (variant.uid, arch)
|
|
if pkg_type is not None:
|
|
file_name += ".%s" % pkg_type
|
|
file_name += ".conf"
|
|
path = os.path.join(self.topdir(arch, create_dir=create_dir), "repo_package_list")
|
|
if create_dir:
|
|
makedirs(path)
|
|
path = os.path.join(path, file_name)
|
|
return path
|
|
|
|
def product_img(self, variant, create_dir=True):
|
|
"""
|
|
Examples:
|
|
work/global/product-Server.img
|
|
"""
|
|
file_name = "product-%s.img" % variant
|
|
path = self.topdir(arch="global", create_dir=create_dir)
|
|
path = os.path.join(path, file_name)
|
|
return path
|
|
|
|
def iso_dir(self, arch, filename, create_dir=True):
|
|
"""
|
|
Examples:
|
|
work/x86_64/iso/Project-1.0-20151203.0-Client-x86_64-dvd1.iso
|
|
"""
|
|
path = os.path.join(self.topdir(arch, create_dir=create_dir), "iso", filename)
|
|
if create_dir:
|
|
makedirs(path)
|
|
return path
|
|
|
|
def tmp_dir(self, arch=None, variant=None, create_dir=True):
|
|
"""
|
|
Examples:
|
|
work/global/tmp
|
|
work/x86_64/tmp
|
|
work/x86_64/tmp-Server
|
|
"""
|
|
dir_name = "tmp"
|
|
if variant:
|
|
dir_name += "-%s" % variant.uid
|
|
path = os.path.join(self.topdir(arch=arch, create_dir=create_dir), dir_name)
|
|
if create_dir:
|
|
makedirs(path)
|
|
return path
|
|
|
|
def product_id(self, arch, variant, create_dir=True):
|
|
"""
|
|
Examples:
|
|
work/x86_64/product_id/productid-Server.x86_64.pem/productid
|
|
"""
|
|
# file_name = "%s.%s.pem" % (variant, arch)
|
|
# HACK: modifyrepo doesn't handle renames -> $dir/productid
|
|
file_name = "productid"
|
|
path = os.path.join(self.topdir(arch, create_dir=create_dir), "product_id", "%s.%s.pem" % (variant, arch))
|
|
if create_dir:
|
|
makedirs(path)
|
|
path = os.path.join(path, file_name)
|
|
return path
|
|
|
|
def image_build_dir(self, variant, create_dir=True):
|
|
"""
|
|
@param variant
|
|
@param create_dir=True
|
|
|
|
Examples:
|
|
work/image-build/Server
|
|
"""
|
|
path = os.path.join(self.topdir('image-build', create_dir=create_dir), variant.uid)
|
|
if create_dir:
|
|
makedirs(path)
|
|
return path
|
|
|
|
def image_build_conf(self, variant, image_name, image_type, arches=None, create_dir=True):
|
|
"""
|
|
@param variant
|
|
@param image-name
|
|
@param image-type (e.g docker)
|
|
@param arches
|
|
@param create_dir=True
|
|
|
|
Examples:
|
|
work/image-build/Server/docker_rhel-server-docker.cfg
|
|
work/image-build/Server/docker_rhel-server-docker_x86_64.cfg
|
|
work/image-build/Server/docker_rhel-server-docker_x86_64-ppc64le.cfg
|
|
"""
|
|
path = os.path.join(self.image_build_dir(variant), "%s_%s" % (image_type, image_name))
|
|
if arches is not None:
|
|
path = "%s_%s" % (path, '-'.join(list(arches)))
|
|
path = "%s.cfg" % path
|
|
return path
|
|
|
|
|
|
class ComposePaths(object):
|
|
def __init__(self, compose):
|
|
self.compose = compose
|
|
# TODO: TREES?
|
|
|
|
def topdir(self, arch=None, variant=None, create_dir=True, relative=False):
|
|
"""
|
|
Examples:
|
|
compose
|
|
compose/Server/x86_64
|
|
"""
|
|
if bool(arch) != bool(variant):
|
|
raise TypeError("topdir(): either none or 2 arguments are expected")
|
|
|
|
path = ""
|
|
if not relative:
|
|
path = os.path.join(self.compose.topdir, "compose")
|
|
|
|
if arch or variant:
|
|
if variant.type == "addon":
|
|
return self.topdir(arch, variant.parent, create_dir=create_dir, relative=relative)
|
|
path = os.path.join(path, variant.uid, arch)
|
|
if create_dir and not relative:
|
|
makedirs(path)
|
|
return path
|
|
|
|
def tree_dir(self, arch, variant, create_dir=True, relative=False):
|
|
"""
|
|
Examples:
|
|
compose/Server/x86_64/os
|
|
compose/Server-optional/x86_64/os
|
|
"""
|
|
if arch == "src":
|
|
arch = "source"
|
|
|
|
if arch == "source":
|
|
tree_dir = "tree"
|
|
else:
|
|
# use 'os' dir due to historical reasons
|
|
tree_dir = "os"
|
|
|
|
path = os.path.join(self.topdir(arch, variant, create_dir=create_dir, relative=relative), tree_dir)
|
|
if create_dir and not relative:
|
|
makedirs(path)
|
|
return path
|
|
|
|
def os_tree(self, arch, variant, create_dir=True, relative=False):
|
|
return self.tree_dir(arch, variant, create_dir=create_dir, relative=relative)
|
|
|
|
def repository(self, arch, variant, create_dir=True, relative=False):
|
|
"""
|
|
Examples:
|
|
compose/Server/x86_64/os
|
|
compose/Server/x86_64/addons/LoadBalancer
|
|
"""
|
|
if variant.type == "addon":
|
|
path = self.packages(arch, variant, create_dir=create_dir, relative=relative)
|
|
else:
|
|
path = self.tree_dir(arch, variant, create_dir=create_dir, relative=relative)
|
|
if create_dir and not relative:
|
|
makedirs(path)
|
|
return path
|
|
|
|
def packages(self, arch, variant, create_dir=True, relative=False):
|
|
"""
|
|
Examples:
|
|
compose/Server/x86_64/os/Packages
|
|
compose/Server/x86_64/os/addons/LoadBalancer
|
|
compose/Server-optional/x86_64/os/Packages
|
|
"""
|
|
if variant.type == "addon":
|
|
path = os.path.join(self.tree_dir(arch, variant, create_dir=create_dir, relative=relative), "addons", variant.id)
|
|
else:
|
|
path = os.path.join(self.tree_dir(arch, variant, create_dir=create_dir, relative=relative), "Packages")
|
|
if create_dir and not relative:
|
|
makedirs(path)
|
|
return path
|
|
|
|
def debug_topdir(self, arch, variant, create_dir=True, relative=False):
|
|
"""
|
|
Examples:
|
|
compose/Server/x86_64/debug
|
|
compose/Server-optional/x86_64/debug
|
|
"""
|
|
path = os.path.join(self.topdir(arch, variant, create_dir=create_dir, relative=relative), "debug")
|
|
if create_dir and not relative:
|
|
makedirs(path)
|
|
return path
|
|
|
|
def debug_tree(self, arch, variant, create_dir=True, relative=False):
|
|
"""
|
|
Examples:
|
|
compose/Server/x86_64/debug/tree
|
|
compose/Server-optional/x86_64/debug/tree
|
|
"""
|
|
path = os.path.join(self.debug_topdir(arch, variant, create_dir=create_dir, relative=relative), "tree")
|
|
if create_dir and not relative:
|
|
makedirs(path)
|
|
return path
|
|
|
|
def debug_packages(self, arch, variant, create_dir=True, relative=False):
|
|
"""
|
|
Examples:
|
|
compose/Server/x86_64/debug/tree/Packages
|
|
compose/Server/x86_64/debug/tree/addons/LoadBalancer
|
|
compose/Server-optional/x86_64/debug/tree/Packages
|
|
"""
|
|
if arch in ("source", "src"):
|
|
return None
|
|
if variant.type == "addon":
|
|
path = os.path.join(self.debug_tree(arch, variant, create_dir=create_dir, relative=relative), "addons", variant.id)
|
|
else:
|
|
path = os.path.join(self.debug_tree(arch, variant, create_dir=create_dir, relative=relative), "Packages")
|
|
if create_dir and not relative:
|
|
makedirs(path)
|
|
return path
|
|
|
|
def debug_repository(self, arch, variant, create_dir=True, relative=False):
|
|
"""
|
|
Examples:
|
|
compose/Server/x86_64/debug/tree
|
|
compose/Server/x86_64/debug/tree/addons/LoadBalancer
|
|
compose/Server-optional/x86_64/debug/tree
|
|
"""
|
|
if arch in ("source", "src"):
|
|
return None
|
|
if variant.type == "addon":
|
|
path = os.path.join(self.debug_tree(arch, variant, create_dir=create_dir, relative=relative), "addons", variant.id)
|
|
else:
|
|
path = self.debug_tree(arch, variant, create_dir=create_dir, relative=relative)
|
|
if create_dir and not relative:
|
|
makedirs(path)
|
|
return path
|
|
|
|
def iso_dir(self, arch, variant, symlink_to=None, create_dir=True, relative=False):
|
|
"""
|
|
Examples:
|
|
compose/Server/x86_64/iso
|
|
None
|
|
"""
|
|
if variant.type == "addon":
|
|
return None
|
|
if variant.type == "optional":
|
|
if not self.compose.conf.get("create_optional_isos", False):
|
|
return None
|
|
if arch == "src":
|
|
arch = "source"
|
|
path = os.path.join(self.topdir(arch, variant, create_dir=create_dir, relative=relative), "iso")
|
|
|
|
if symlink_to:
|
|
# TODO: create_dir
|
|
topdir = self.compose.topdir.rstrip("/") + "/"
|
|
relative_dir = path[len(topdir):]
|
|
target_dir = os.path.join(symlink_to, self.compose.compose_id, relative_dir)
|
|
if create_dir and not relative:
|
|
makedirs(target_dir)
|
|
try:
|
|
os.symlink(target_dir, path)
|
|
except OSError as ex:
|
|
if ex.errno != errno.EEXIST:
|
|
raise
|
|
msg = "Symlink pointing to '%s' expected: %s" % (target_dir, path)
|
|
if not os.path.islink(path):
|
|
raise RuntimeError(msg)
|
|
if os.path.abspath(os.readlink(path)) != target_dir:
|
|
raise RuntimeError(msg)
|
|
else:
|
|
if create_dir and not relative:
|
|
makedirs(path)
|
|
return path
|
|
|
|
def iso_path(self, arch, variant, filename, symlink_to=None, create_dir=True, relative=False):
|
|
"""
|
|
Examples:
|
|
compose/Server/x86_64/iso/rhel-7.0-20120127.0-Server-x86_64-dvd1.iso
|
|
None
|
|
"""
|
|
path = self.iso_dir(arch, variant, symlink_to=symlink_to, create_dir=create_dir, relative=relative)
|
|
if path is None:
|
|
return None
|
|
|
|
return os.path.join(path, filename)
|
|
|
|
def image_dir(self, variant, symlink_to=None, relative=False):
|
|
"""
|
|
The arch is listed as literal '%(arch)s'
|
|
Examples:
|
|
compose/Server/%(arch)s/images
|
|
None
|
|
@param variant
|
|
@param symlink_to=None
|
|
@param relative=False
|
|
"""
|
|
path = os.path.join(self.topdir('%(arch)s', variant, create_dir=False, relative=relative),
|
|
"images")
|
|
if symlink_to:
|
|
topdir = self.compose.topdir.rstrip("/") + "/"
|
|
relative_dir = path[len(topdir):]
|
|
target_dir = os.path.join(symlink_to, self.compose.compose_id, relative_dir)
|
|
try:
|
|
os.symlink(target_dir, path)
|
|
except OSError as ex:
|
|
if ex.errno != errno.EEXIST:
|
|
raise
|
|
msg = "Symlink pointing to '%s' expected: %s" % (target_dir, path)
|
|
if not os.path.islink(path):
|
|
raise RuntimeError(msg)
|
|
if os.path.abspath(os.readlink(path)) != target_dir:
|
|
raise RuntimeError(msg)
|
|
return path
|
|
|
|
def jigdo_dir(self, arch, variant, create_dir=True, relative=False):
|
|
"""
|
|
Examples:
|
|
compose/Server/x86_64/jigdo
|
|
None
|
|
"""
|
|
if variant.type == "addon":
|
|
return None
|
|
if variant.type == "optional":
|
|
if not self.compose.conf.get("create_optional_isos", False):
|
|
return None
|
|
if arch == "src":
|
|
arch = "source"
|
|
path = os.path.join(self.topdir(arch, variant, create_dir=create_dir, relative=relative), "jigdo")
|
|
|
|
if create_dir and not relative:
|
|
makedirs(path)
|
|
return path
|
|
|
|
def metadata(self, file_name=None, create_dir=True, relative=False):
|
|
"""
|
|
Examples:
|
|
compose/metadata
|
|
compose/metadata/rpms.json
|
|
"""
|
|
path = os.path.join(self.topdir(create_dir=create_dir, relative=relative), "metadata")
|
|
if create_dir and not relative:
|
|
makedirs(path)
|
|
if file_name:
|
|
path = os.path.join(path, file_name)
|
|
return path
|