diff --git a/doc/configuration.rst b/doc/configuration.rst index bd26d8bf..2365db19 100644 --- a/doc/configuration.rst +++ b/doc/configuration.rst @@ -972,6 +972,16 @@ Options ``optional`` variants. By default only variants with type ``variant`` or ``layered-product`` will get ISOs. +**createiso_break_hardlinks** = False + (*bool*) -- when set to ``True``, all files that should go on the ISO and + have a hardlink will be first copied into a staging directory. This should + work around a bug in ``genisoimage`` including incorrect link count in the + image, but it is at the cost of having to copy a potentially significant + amount of data. + + The staging directory is deleted when ISO is successfully created. In that + case the same task to create the ISO will not be re-runnable. + **iso_size** = 4700000000 (*int|str*) -- size of ISO image. The value should either be an integer meaning size in bytes, or it can be a string with ``k``, ``M``, ``G`` diff --git a/pungi/checks.py b/pungi/checks.py index c6ac840a..a54ed8fd 100644 --- a/pungi/checks.py +++ b/pungi/checks.py @@ -697,6 +697,10 @@ def make_schema(): }, "symlink_isos_to": {"type": "string"}, "createiso_skip": _variant_arch_mapping({"type": "boolean"}), + "createiso_break_hardlinks": { + "type": "boolean", + "default": False, + }, "multilib": _variant_arch_mapping({ "$ref": "#/definitions/list_of_strings" }), diff --git a/pungi/paths.py b/pungi/paths.py index bb07005b..eaaae581 100644 --- a/pungi/paths.py +++ b/pungi/paths.py @@ -276,6 +276,18 @@ class WorkPaths(object): makedirs(path) return path + def iso_staging_dir(self, arch, variant, create_dir=True): + """ + Examples: + work/x86_64/Server/iso-staging-dir + """ + path = os.path.join( + self.topdir(arch, create_dir=create_dir), variant.uid, "iso-staging-dir" + ) + if create_dir: + makedirs(path) + return path + def repo_package_list(self, arch, variant, pkg_type=None, create_dir=True): """ Examples: diff --git a/pungi/phases/createiso.py b/pungi/phases/createiso.py index 29e10e6f..1f8f7a7f 100644 --- a/pungi/phases/createiso.py +++ b/pungi/phases/createiso.py @@ -18,6 +18,7 @@ import os import time import random import shutil +import stat import productmd.treeinfo from productmd.images import Image @@ -206,6 +207,11 @@ class CreateIsoThread(WorkerThread): add_iso_to_metadata(compose, variant, arch, cmd["iso_path"], cmd["bootable"], cmd["disc_num"], cmd["disc_count"]) + # Delete staging directory if present. + staging_dir = compose.paths.work.iso_staging_dir(arch, variant) + if os.path.exists(staging_dir): + shutil.rmtree(staging_dir) + self.pool.log_info("[DONE ] %s" % msg) if compose.notifier: compose.notifier.send('createiso-imagedone', @@ -444,6 +450,10 @@ def prepare_iso(compose, arch, variant, disc_num=1, disc_count=None, split_iso_d else: data = iso.get_graft_points([iso._paths_from_list(tree_dir, split_iso_data["files"]), iso_dir]) + if compose.conf["createiso_break_hardlinks"]: + compose.log_debug("Breaking hardlinks for ISO for %s.%s" % (variant, arch)) + break_hardlinks(data, compose.paths.work.iso_staging_dir(arch, variant)) + # TODO: /content /graft-points gp = "%s-graft-points" % iso_dir iso.write_graft_points(gp, data, exclude=["*/lost+found", "*/boot.iso"]) @@ -460,3 +470,16 @@ def copy_boot_images(src, dest): if os.path.exists(src_path): makedirs(os.path.dirname(dst_path)) shutil.copy2(src_path, dst_path) + + +def break_hardlinks(graft_points, staging_dir): + """Iterate over graft points and copy any file that has more than 1 + hardlink into the staging directory. Replace the entry in the dict. + """ + for f in graft_points: + info = os.stat(graft_points[f]) + if stat.S_ISREG(info.st_mode) and info.st_nlink > 1: + dest_path = os.path.join(staging_dir, graft_points[f].lstrip("/")) + makedirs(os.path.dirname(dest_path)) + shutil.copy2(graft_points[f], dest_path) + graft_points[f] = dest_path diff --git a/tests/test_createiso_phase.py b/tests/test_createiso_phase.py index 712cd297..4714482c 100644 --- a/tests/test_createiso_phase.py +++ b/tests/test_createiso_phase.py @@ -844,5 +844,43 @@ class SplitIsoTest(helpers.PungiTestCase): self.assertEqual(len(data), 1) +class BreakHardlinksTest(helpers.PungiTestCase): + + def setUp(self): + super(BreakHardlinksTest, self).setUp() + self.src = os.path.join(self.topdir, "src") + self.stage = os.path.join(self.topdir, "stage") + + def test_not_modify_dir(self): + p = os.path.join(self.src, "dir") + os.makedirs(p) + + d = {"dir": p} + createiso.break_hardlinks(d, self.stage) + + self.assertEqual(d, {"dir": p}) + + def test_not_copy_file_with_one(self): + f = os.path.join(self.src, "file") + helpers.touch(f) + + d = {"f": f} + createiso.break_hardlinks(d, self.stage) + + self.assertEqual(d, {"f": f}) + + def test_copy(self): + f = os.path.join(self.src, "file") + helpers.touch(f) + os.link(f, os.path.join(self.topdir, "file")) + + d = {"f": f} + createiso.break_hardlinks(d, self.stage) + + expected = self.stage + f + self.assertEqual(d, {"f": expected}) + self.assertTrue(os.path.exists(expected)) + + if __name__ == '__main__': unittest.main()