createiso: Break hardlinks by copying files

If a file has multiple hard links, genisoimage will put the wrong number
on the ISO. This patch can work around it by copying hard-linked files
into a temporary staging directory.

JIRA: COMPOSE-2610
Signed-off-by: Lubomír Sedlář <lsedlar@redhat.com>

doc/configuration.rst

@@ -972,6 +972,16 @@ Options
     ``optional`` variants. By default only variants with type ``variant`` or
     ``layered-product`` will get ISOs.
 
+**createiso_break_hardlinks** = False
+    (*bool*) -- when set to ``True``, all files that should go on the ISO and
+    have a hardlink will be first copied into a staging directory. This should
+    work around a bug in ``genisoimage`` including incorrect link count in the
+    image, but it is at the cost of having to copy a potentially significant
+    amount of data.
+
+    The staging directory is deleted when ISO is successfully created. In that
+    case the same task to create the ISO will not be re-runnable.
+
 **iso_size** = 4700000000
     (*int|str*) -- size of ISO image. The value should either be an integer
     meaning size in bytes, or it can be a string with ``k``, ``M``, ``G``

pungi/checks.py

@@ -697,6 +697,10 @@ def make_schema():
             },
             "symlink_isos_to": {"type": "string"},
             "createiso_skip": _variant_arch_mapping({"type": "boolean"}),
+            "createiso_break_hardlinks": {
+                "type": "boolean",
+                "default": False,
+            },
             "multilib": _variant_arch_mapping({
                 "$ref": "#/definitions/list_of_strings"
             }),

pungi/paths.py

@@ -276,6 +276,18 @@ class WorkPaths(object):
             makedirs(path)
         return path
 
+    def iso_staging_dir(self, arch, variant, create_dir=True):
+        """
+        Examples:
+            work/x86_64/Server/iso-staging-dir
+        """
+        path = os.path.join(
+            self.topdir(arch, create_dir=create_dir), variant.uid, "iso-staging-dir"
+        )
+        if create_dir:
+            makedirs(path)
+        return path
+
     def repo_package_list(self, arch, variant, pkg_type=None, create_dir=True):
         """
         Examples:

pungi/phases/createiso.py

@@ -18,6 +18,7 @@ import os
 import time
 import random
 import shutil
+import stat
 
 import productmd.treeinfo
 from productmd.images import Image
@@ -206,6 +207,11 @@ class CreateIsoThread(WorkerThread):
         add_iso_to_metadata(compose, variant, arch, cmd["iso_path"],
                             cmd["bootable"], cmd["disc_num"], cmd["disc_count"])
 
+        # Delete staging directory if present.
+        staging_dir = compose.paths.work.iso_staging_dir(arch, variant)
+        if os.path.exists(staging_dir):
+            shutil.rmtree(staging_dir)
+
         self.pool.log_info("[DONE ] %s" % msg)
         if compose.notifier:
             compose.notifier.send('createiso-imagedone',
@@ -444,6 +450,10 @@ def prepare_iso(compose, arch, variant, disc_num=1, disc_count=None, split_iso_d
     else:
         data = iso.get_graft_points([iso._paths_from_list(tree_dir, split_iso_data["files"]), iso_dir])
 
+    if compose.conf["createiso_break_hardlinks"]:
+        compose.log_debug("Breaking hardlinks for ISO for %s.%s" % (variant, arch))
+        break_hardlinks(data, compose.paths.work.iso_staging_dir(arch, variant))
+
     # TODO: /content /graft-points
     gp = "%s-graft-points" % iso_dir
     iso.write_graft_points(gp, data, exclude=["*/lost+found", "*/boot.iso"])
@@ -460,3 +470,16 @@ def copy_boot_images(src, dest):
         if os.path.exists(src_path):
             makedirs(os.path.dirname(dst_path))
             shutil.copy2(src_path, dst_path)
+
+
+def break_hardlinks(graft_points, staging_dir):
+    """Iterate over graft points and copy any file that has more than 1
+    hardlink into the staging directory. Replace the entry in the dict.
+    """
+    for f in graft_points:
+        info = os.stat(graft_points[f])
+        if stat.S_ISREG(info.st_mode) and info.st_nlink > 1:
+            dest_path = os.path.join(staging_dir, graft_points[f].lstrip("/"))
+            makedirs(os.path.dirname(dest_path))
+            shutil.copy2(graft_points[f], dest_path)
+            graft_points[f] = dest_path

tests/test_createiso_phase.py

@@ -844,5 +844,43 @@ class SplitIsoTest(helpers.PungiTestCase):
         self.assertEqual(len(data), 1)
 
 
+class BreakHardlinksTest(helpers.PungiTestCase):
+
+    def setUp(self):
+        super(BreakHardlinksTest, self).setUp()
+        self.src = os.path.join(self.topdir, "src")
+        self.stage = os.path.join(self.topdir, "stage")
+
+    def test_not_modify_dir(self):
+        p = os.path.join(self.src, "dir")
+        os.makedirs(p)
+
+        d = {"dir": p}
+        createiso.break_hardlinks(d, self.stage)
+
+        self.assertEqual(d, {"dir": p})
+
+    def test_not_copy_file_with_one(self):
+        f = os.path.join(self.src, "file")
+        helpers.touch(f)
+
+        d = {"f": f}
+        createiso.break_hardlinks(d, self.stage)
+
+        self.assertEqual(d, {"f": f})
+
+    def test_copy(self):
+        f = os.path.join(self.src, "file")
+        helpers.touch(f)
+        os.link(f, os.path.join(self.topdir, "file"))
+
+        d = {"f": f}
+        createiso.break_hardlinks(d, self.stage)
+
+        expected = self.stage + f
+        self.assertEqual(d, {"f": expected})
+        self.assertTrue(os.path.exists(expected))
+
+
 if __name__ == '__main__':
     unittest.main()