createiso: Break hardlinks by copying files

If a file has multiple hard links, genisoimage will put the wrong number
on the ISO. This patch can work around it by copying hard-linked files
into a temporary staging directory.

JIRA: COMPOSE-2610
Signed-off-by: Lubomír Sedlář <lsedlar@redhat.com>
This commit is contained in:
Lubomír Sedlář 2018-06-20 11:12:08 +02:00
parent d8c03f6239
commit 38f1a8509e
5 changed files with 87 additions and 0 deletions

View File

@ -972,6 +972,16 @@ Options
``optional`` variants. By default only variants with type ``variant`` or ``optional`` variants. By default only variants with type ``variant`` or
``layered-product`` will get ISOs. ``layered-product`` will get ISOs.
**createiso_break_hardlinks** = False
(*bool*) -- when set to ``True``, all files that should go on the ISO and
have a hardlink will be first copied into a staging directory. This should
work around a bug in ``genisoimage`` including incorrect link count in the
image, but it is at the cost of having to copy a potentially significant
amount of data.
The staging directory is deleted when ISO is successfully created. In that
case the same task to create the ISO will not be re-runnable.
**iso_size** = 4700000000 **iso_size** = 4700000000
(*int|str*) -- size of ISO image. The value should either be an integer (*int|str*) -- size of ISO image. The value should either be an integer
meaning size in bytes, or it can be a string with ``k``, ``M``, ``G`` meaning size in bytes, or it can be a string with ``k``, ``M``, ``G``

View File

@ -697,6 +697,10 @@ def make_schema():
}, },
"symlink_isos_to": {"type": "string"}, "symlink_isos_to": {"type": "string"},
"createiso_skip": _variant_arch_mapping({"type": "boolean"}), "createiso_skip": _variant_arch_mapping({"type": "boolean"}),
"createiso_break_hardlinks": {
"type": "boolean",
"default": False,
},
"multilib": _variant_arch_mapping({ "multilib": _variant_arch_mapping({
"$ref": "#/definitions/list_of_strings" "$ref": "#/definitions/list_of_strings"
}), }),

View File

@ -276,6 +276,18 @@ class WorkPaths(object):
makedirs(path) makedirs(path)
return path return path
def iso_staging_dir(self, arch, variant, create_dir=True):
"""
Examples:
work/x86_64/Server/iso-staging-dir
"""
path = os.path.join(
self.topdir(arch, create_dir=create_dir), variant.uid, "iso-staging-dir"
)
if create_dir:
makedirs(path)
return path
def repo_package_list(self, arch, variant, pkg_type=None, create_dir=True): def repo_package_list(self, arch, variant, pkg_type=None, create_dir=True):
""" """
Examples: Examples:

View File

@ -18,6 +18,7 @@ import os
import time import time
import random import random
import shutil import shutil
import stat
import productmd.treeinfo import productmd.treeinfo
from productmd.images import Image from productmd.images import Image
@ -206,6 +207,11 @@ class CreateIsoThread(WorkerThread):
add_iso_to_metadata(compose, variant, arch, cmd["iso_path"], add_iso_to_metadata(compose, variant, arch, cmd["iso_path"],
cmd["bootable"], cmd["disc_num"], cmd["disc_count"]) cmd["bootable"], cmd["disc_num"], cmd["disc_count"])
# Delete staging directory if present.
staging_dir = compose.paths.work.iso_staging_dir(arch, variant)
if os.path.exists(staging_dir):
shutil.rmtree(staging_dir)
self.pool.log_info("[DONE ] %s" % msg) self.pool.log_info("[DONE ] %s" % msg)
if compose.notifier: if compose.notifier:
compose.notifier.send('createiso-imagedone', compose.notifier.send('createiso-imagedone',
@ -444,6 +450,10 @@ def prepare_iso(compose, arch, variant, disc_num=1, disc_count=None, split_iso_d
else: else:
data = iso.get_graft_points([iso._paths_from_list(tree_dir, split_iso_data["files"]), iso_dir]) data = iso.get_graft_points([iso._paths_from_list(tree_dir, split_iso_data["files"]), iso_dir])
if compose.conf["createiso_break_hardlinks"]:
compose.log_debug("Breaking hardlinks for ISO for %s.%s" % (variant, arch))
break_hardlinks(data, compose.paths.work.iso_staging_dir(arch, variant))
# TODO: /content /graft-points # TODO: /content /graft-points
gp = "%s-graft-points" % iso_dir gp = "%s-graft-points" % iso_dir
iso.write_graft_points(gp, data, exclude=["*/lost+found", "*/boot.iso"]) iso.write_graft_points(gp, data, exclude=["*/lost+found", "*/boot.iso"])
@ -460,3 +470,16 @@ def copy_boot_images(src, dest):
if os.path.exists(src_path): if os.path.exists(src_path):
makedirs(os.path.dirname(dst_path)) makedirs(os.path.dirname(dst_path))
shutil.copy2(src_path, dst_path) shutil.copy2(src_path, dst_path)
def break_hardlinks(graft_points, staging_dir):
"""Iterate over graft points and copy any file that has more than 1
hardlink into the staging directory. Replace the entry in the dict.
"""
for f in graft_points:
info = os.stat(graft_points[f])
if stat.S_ISREG(info.st_mode) and info.st_nlink > 1:
dest_path = os.path.join(staging_dir, graft_points[f].lstrip("/"))
makedirs(os.path.dirname(dest_path))
shutil.copy2(graft_points[f], dest_path)
graft_points[f] = dest_path

View File

@ -844,5 +844,43 @@ class SplitIsoTest(helpers.PungiTestCase):
self.assertEqual(len(data), 1) self.assertEqual(len(data), 1)
class BreakHardlinksTest(helpers.PungiTestCase):
def setUp(self):
super(BreakHardlinksTest, self).setUp()
self.src = os.path.join(self.topdir, "src")
self.stage = os.path.join(self.topdir, "stage")
def test_not_modify_dir(self):
p = os.path.join(self.src, "dir")
os.makedirs(p)
d = {"dir": p}
createiso.break_hardlinks(d, self.stage)
self.assertEqual(d, {"dir": p})
def test_not_copy_file_with_one(self):
f = os.path.join(self.src, "file")
helpers.touch(f)
d = {"f": f}
createiso.break_hardlinks(d, self.stage)
self.assertEqual(d, {"f": f})
def test_copy(self):
f = os.path.join(self.src, "file")
helpers.touch(f)
os.link(f, os.path.join(self.topdir, "file"))
d = {"f": f}
createiso.break_hardlinks(d, self.stage)
expected = self.stage + f
self.assertEqual(d, {"f": expected})
self.assertTrue(os.path.exists(expected))
if __name__ == '__main__': if __name__ == '__main__':
unittest.main() unittest.main()