gather: Implement hashed directories.

Set 'hashed_directories = True' config option to enable the feature.

doc/configuration.rst

@@ -268,6 +268,9 @@ Options
 **gather_lookaside_repos** = []
     (*list*) -- lookaside repositories used for package gathering; format: [(variant_uid_regex, {arch|*: [repo_urls]})]
 
+**hashed_directories** = False
+    (*bool*) -- put packages into "hashed" directories, for example Packages/k/kernel-4.0.4-301.fc22.x86_64.rpm
+
 
 Example
 -------
@@ -279,6 +282,7 @@ Example
     multilib_methods = ["devel", "runtime"]
     multilib_arches = ["ppc64", "s390x", "x86_64"]
     check_deps = False
+    hashed_directories = True
 
     additional_packages = [
         # bz#123456

pungi/linker.py

@@ -40,6 +40,8 @@ class LinkerThread(WorkerThread):
         if (num % 100 == 0) or (num == self.pool.queue_total):
             self.pool.log_debug("Linked %s out of %s packages" % (num, self.pool.queue_total))
 
+        directory = os.path.dirname(dst)
+        makedirs(directory)
         self.pool.linker.link(src, dst, link_type=self.pool.link_type)
 
 

pungi/phases/gather/__init__.py

@@ -81,6 +81,11 @@ class GatherPhase(PhaseBase):
             "expected_types": [str, dict],
             "optional": True,
         },
+        {
+            "name": "hashed_directories",
+            "expected_types": [bool],
+            "optional": True,
+        },
         # DEPRECATED OPTIONS
         {
             "name": "additional_packages_multiarch",

pungi/phases/gather/link.py

@@ -38,6 +38,13 @@ def _get_src_nevra(compose, pkg_obj, srpm_map):
     return result
 
 
+def get_package_path(filename, hashed_directory=False):
+    if hashed_directory:
+        prefix = filename[0]
+        return os.path.join(prefix, filename)
+    return filename
+
+
 def link_files(compose, arch, variant, pkg_map, pkg_sets, manifest, srpm_map={}):
     # srpm_map instance is shared between link_files() runs
     pkg_set = pkg_sets[arch]
@@ -50,11 +57,13 @@ def link_files(compose, arch, variant, pkg_map, pkg_sets, manifest, srpm_map={})
     for i in range(10):
         pool.add(LinkerThread(pool))
 
+    hashed_directories = compose.conf.get("hashed_directories", False)
+
     packages_dir = compose.paths.compose.packages("src", variant)
     packages_dir_relpath = compose.paths.compose.packages("src", variant, relative=True)
     for pkg in pkg_map["srpm"]:
-        dst = os.path.join(packages_dir, os.path.basename(pkg["path"]))
-        dst_relpath = os.path.join(packages_dir_relpath, os.path.basename(pkg["path"]))
+        dst = os.path.join(packages_dir, get_package_path(os.path.basename(pkg["path"]), hashed_directories))
+        dst_relpath = os.path.join(packages_dir_relpath, get_package_path(os.path.basename(pkg["path"]), hashed_directories))
 
         # link file
         pool.queue_put((pkg["path"], dst))
@@ -70,8 +79,8 @@ def link_files(compose, arch, variant, pkg_map, pkg_sets, manifest, srpm_map={})
     packages_dir = compose.paths.compose.packages(arch, variant)
     packages_dir_relpath = compose.paths.compose.packages(arch, variant, relative=True)
     for pkg in pkg_map["rpm"]:
-        dst = os.path.join(packages_dir, os.path.basename(pkg["path"]))
-        dst_relpath = os.path.join(packages_dir_relpath, os.path.basename(pkg["path"]))
+        dst = os.path.join(packages_dir, get_package_path(os.path.basename(pkg["path"]), hashed_directories))
+        dst_relpath = os.path.join(packages_dir_relpath, get_package_path(os.path.basename(pkg["path"]), hashed_directories))
 
         # link file
         pool.queue_put((pkg["path"], dst))
@@ -85,8 +94,8 @@ def link_files(compose, arch, variant, pkg_map, pkg_sets, manifest, srpm_map={})
     packages_dir = compose.paths.compose.debug_packages(arch, variant)
     packages_dir_relpath = compose.paths.compose.debug_packages(arch, variant, relative=True)
     for pkg in pkg_map["debuginfo"]:
-        dst = os.path.join(packages_dir, os.path.basename(pkg["path"]))
-        dst_relpath = os.path.join(packages_dir_relpath, os.path.basename(pkg["path"]))
+        dst = os.path.join(packages_dir, get_package_path(os.path.basename(pkg["path"]), hashed_directories))
+        dst_relpath = os.path.join(packages_dir_relpath, get_package_path(os.path.basename(pkg["path"]), hashed_directories))
 
         # link file
         pool.queue_put((pkg["path"], dst))

tests/data/dummy-pungi.conf

@@ -40,6 +40,7 @@ gather_source = "comps"
 gather_method = "deps"
 greedy_method = "build"
 check_deps = False
+hashed_directories = True
 
 multilib_arches = ["ppc64", "x86_64", "s390x"]
 multilib_methods = ["devel", "runtime"]