gather: Implement hashed directories.

Set 'hashed_directories = True' config option to enable the feature.
This commit is contained in:
Daniel Mach 2015-07-24 04:40:55 -04:00
parent 1f313b39ad
commit 0d8ad9a111
5 changed files with 27 additions and 6 deletions

View File

@ -268,6 +268,9 @@ Options
**gather_lookaside_repos** = [] **gather_lookaside_repos** = []
(*list*) -- lookaside repositories used for package gathering; format: [(variant_uid_regex, {arch|*: [repo_urls]})] (*list*) -- lookaside repositories used for package gathering; format: [(variant_uid_regex, {arch|*: [repo_urls]})]
**hashed_directories** = False
(*bool*) -- put packages into "hashed" directories, for example Packages/k/kernel-4.0.4-301.fc22.x86_64.rpm
Example Example
------- -------
@ -279,6 +282,7 @@ Example
multilib_methods = ["devel", "runtime"] multilib_methods = ["devel", "runtime"]
multilib_arches = ["ppc64", "s390x", "x86_64"] multilib_arches = ["ppc64", "s390x", "x86_64"]
check_deps = False check_deps = False
hashed_directories = True
additional_packages = [ additional_packages = [
# bz#123456 # bz#123456

View File

@ -40,6 +40,8 @@ class LinkerThread(WorkerThread):
if (num % 100 == 0) or (num == self.pool.queue_total): if (num % 100 == 0) or (num == self.pool.queue_total):
self.pool.log_debug("Linked %s out of %s packages" % (num, self.pool.queue_total)) self.pool.log_debug("Linked %s out of %s packages" % (num, self.pool.queue_total))
directory = os.path.dirname(dst)
makedirs(directory)
self.pool.linker.link(src, dst, link_type=self.pool.link_type) self.pool.linker.link(src, dst, link_type=self.pool.link_type)

View File

@ -81,6 +81,11 @@ class GatherPhase(PhaseBase):
"expected_types": [str, dict], "expected_types": [str, dict],
"optional": True, "optional": True,
}, },
{
"name": "hashed_directories",
"expected_types": [bool],
"optional": True,
},
# DEPRECATED OPTIONS # DEPRECATED OPTIONS
{ {
"name": "additional_packages_multiarch", "name": "additional_packages_multiarch",

View File

@ -38,6 +38,13 @@ def _get_src_nevra(compose, pkg_obj, srpm_map):
return result return result
def get_package_path(filename, hashed_directory=False):
if hashed_directory:
prefix = filename[0]
return os.path.join(prefix, filename)
return filename
def link_files(compose, arch, variant, pkg_map, pkg_sets, manifest, srpm_map={}): def link_files(compose, arch, variant, pkg_map, pkg_sets, manifest, srpm_map={}):
# srpm_map instance is shared between link_files() runs # srpm_map instance is shared between link_files() runs
pkg_set = pkg_sets[arch] pkg_set = pkg_sets[arch]
@ -50,11 +57,13 @@ def link_files(compose, arch, variant, pkg_map, pkg_sets, manifest, srpm_map={})
for i in range(10): for i in range(10):
pool.add(LinkerThread(pool)) pool.add(LinkerThread(pool))
hashed_directories = compose.conf.get("hashed_directories", False)
packages_dir = compose.paths.compose.packages("src", variant) packages_dir = compose.paths.compose.packages("src", variant)
packages_dir_relpath = compose.paths.compose.packages("src", variant, relative=True) packages_dir_relpath = compose.paths.compose.packages("src", variant, relative=True)
for pkg in pkg_map["srpm"]: for pkg in pkg_map["srpm"]:
dst = os.path.join(packages_dir, os.path.basename(pkg["path"])) dst = os.path.join(packages_dir, get_package_path(os.path.basename(pkg["path"]), hashed_directories))
dst_relpath = os.path.join(packages_dir_relpath, os.path.basename(pkg["path"])) dst_relpath = os.path.join(packages_dir_relpath, get_package_path(os.path.basename(pkg["path"]), hashed_directories))
# link file # link file
pool.queue_put((pkg["path"], dst)) pool.queue_put((pkg["path"], dst))
@ -70,8 +79,8 @@ def link_files(compose, arch, variant, pkg_map, pkg_sets, manifest, srpm_map={})
packages_dir = compose.paths.compose.packages(arch, variant) packages_dir = compose.paths.compose.packages(arch, variant)
packages_dir_relpath = compose.paths.compose.packages(arch, variant, relative=True) packages_dir_relpath = compose.paths.compose.packages(arch, variant, relative=True)
for pkg in pkg_map["rpm"]: for pkg in pkg_map["rpm"]:
dst = os.path.join(packages_dir, os.path.basename(pkg["path"])) dst = os.path.join(packages_dir, get_package_path(os.path.basename(pkg["path"]), hashed_directories))
dst_relpath = os.path.join(packages_dir_relpath, os.path.basename(pkg["path"])) dst_relpath = os.path.join(packages_dir_relpath, get_package_path(os.path.basename(pkg["path"]), hashed_directories))
# link file # link file
pool.queue_put((pkg["path"], dst)) pool.queue_put((pkg["path"], dst))
@ -85,8 +94,8 @@ def link_files(compose, arch, variant, pkg_map, pkg_sets, manifest, srpm_map={})
packages_dir = compose.paths.compose.debug_packages(arch, variant) packages_dir = compose.paths.compose.debug_packages(arch, variant)
packages_dir_relpath = compose.paths.compose.debug_packages(arch, variant, relative=True) packages_dir_relpath = compose.paths.compose.debug_packages(arch, variant, relative=True)
for pkg in pkg_map["debuginfo"]: for pkg in pkg_map["debuginfo"]:
dst = os.path.join(packages_dir, os.path.basename(pkg["path"])) dst = os.path.join(packages_dir, get_package_path(os.path.basename(pkg["path"]), hashed_directories))
dst_relpath = os.path.join(packages_dir_relpath, os.path.basename(pkg["path"])) dst_relpath = os.path.join(packages_dir_relpath, get_package_path(os.path.basename(pkg["path"]), hashed_directories))
# link file # link file
pool.queue_put((pkg["path"], dst)) pool.queue_put((pkg["path"], dst))

View File

@ -40,6 +40,7 @@ gather_source = "comps"
gather_method = "deps" gather_method = "deps"
greedy_method = "build" greedy_method = "build"
check_deps = False check_deps = False
hashed_directories = True
multilib_arches = ["ppc64", "x86_64", "s390x"] multilib_arches = ["ppc64", "x86_64", "s390x"]
multilib_methods = ["devel", "runtime"] multilib_methods = ["devel", "runtime"]