From dc692bc604a43816f2aaba5dd862a4c955063e84 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lubom=C3=ADr=20Sedl=C3=A1=C5=99?= Date: Fri, 9 Nov 2018 10:29:28 +0100 Subject: [PATCH] extra_iso: Stop including variant extra files MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit They are not always wanted, so let's not include them by default. There's a new option to include the same files that extra files phases uses, or alternatively they can be configured specifically and put into the variant subdirectory. JIRA: COMPOSE-3084 Signed-off-by: Lubomír Sedlář --- doc/configuration.rst | 6 +- pungi/checks.py | 4 + pungi/phases/extra_isos.py | 25 +++-- tests/test_extra_isos_phase.py | 170 +++++++++++++++++++++++++++------ 4 files changed, 167 insertions(+), 38 deletions(-) diff --git a/doc/configuration.rst b/doc/configuration.rst index 08a1d419..c9ac3dcb 100644 --- a/doc/configuration.rst +++ b/doc/configuration.rst @@ -1554,6 +1554,10 @@ will reuse boot configuration from that variant. * ``skip_src`` -- (*bool*) allows to disable creating an image with source packages. + * ``inherit_extra_files`` -- (*bool*) by default extra files in variants + are ignored. If you want to include them in the ISO, set this option to + ``True``. + Example config -------------- :: @@ -1581,8 +1585,6 @@ Example config # │   │ └── b # │   └── repodata # ├── Server - # │   ├── extra_files.json # extra file from Server - # │   ├── LICENSE # extra file from Server # │   ├── Packages # │   │ ├── a # │   │ └── b diff --git a/pungi/checks.py b/pungi/checks.py index fbf925c9..823f14d1 100644 --- a/pungi/checks.py +++ b/pungi/checks.py @@ -956,6 +956,10 @@ def make_schema(): "type": "boolean", "default": False, }, + "inherit_extra_files": { + "type": "boolean", + "default": False, + }, }, "required": ["include_variants"], "additionalProperties": False diff --git a/pungi/phases/extra_isos.py b/pungi/phases/extra_isos.py index 924d2e2f..c17eab2d 100644 --- a/pungi/phases/extra_isos.py +++ b/pungi/phases/extra_isos.py @@ -88,9 +88,15 @@ class ExtraIsosThread(WorkerThread): bootable = arch != "src" and compose.conf['bootable'] - graft_points = get_iso_contents(compose, variant, arch, - config['include_variants'], - filename, bootable) + graft_points = get_iso_contents( + compose, + variant, + arch, + config['include_variants'], + filename, + bootable=bootable, + inherit_extra_files=config.get("inherit_extra_files", False), + ) opts = createiso.CreateIsoOpts( output_dir=iso_dir, @@ -142,7 +148,9 @@ def get_extra_files(compose, variant, arch, extra_files): getter(scm_dict, target_path, logger=compose._logger) -def get_iso_contents(compose, variant, arch, include_variants, filename, bootable): +def get_iso_contents( + compose, variant, arch, include_variants, filename, bootable, inherit_extra_files +): """Find all files that should be on the ISO. For bootable image we start with the boot configuration. Then for each variant we add packages, repodata and extra files. Finally we add top-level extra files. @@ -181,10 +189,11 @@ def get_iso_contents(compose, variant, arch, include_variants, filename, bootabl for k, v in iso.get_graft_points([repo_dir]).items(): files[os.path.join(var.uid, 'repodata', k)] = v - # Get extra files... - extra_files_dir = compose.paths.work.extra_files_dir(arch, var) - for k, v in iso.get_graft_points([extra_files_dir]).items(): - files[os.path.join(var.uid, k)] = v + if inherit_extra_files: + # Get extra files... + extra_files_dir = compose.paths.work.extra_files_dir(arch, var) + for k, v in iso.get_graft_points([extra_files_dir]).items(): + files[os.path.join(var.uid, k)] = v extra_files_dir = compose.paths.work.extra_iso_extra_files_dir(arch, variant) diff --git a/tests/test_extra_isos_phase.py b/tests/test_extra_isos_phase.py index 706199e3..93133eb8 100644 --- a/tests/test_extra_isos_phase.py +++ b/tests/test_extra_isos_phase.py @@ -133,8 +133,20 @@ class ExtraIsosThreadTest(helpers.PungiTestCase): [mock.call(compose, server, 'x86_64', [])]) self.assertEqual(gef.call_args_list, [mock.call(compose, server, 'x86_64', [])]) - self.assertEqual(gic.call_args_list, - [mock.call(compose, server, 'x86_64', ['Client'], 'my.iso', True)]) + self.assertEqual( + gic.call_args_list, + [ + mock.call( + compose, + server, + 'x86_64', + ['Client'], + 'my.iso', + bootable=True, + inherit_extra_files=False, + ), + ], + ) self.assertEqual( rcc.call_args_list, [mock.call(False, 1, compose, True, 'x86_64', @@ -177,8 +189,20 @@ class ExtraIsosThreadTest(helpers.PungiTestCase): [mock.call(compose, server, 'x86_64', [])]) self.assertEqual(gef.call_args_list, [mock.call(compose, server, 'x86_64', [])]) - self.assertEqual(gic.call_args_list, - [mock.call(compose, server, 'x86_64', ['Client'], 'my.iso', True)]) + self.assertEqual( + gic.call_args_list, + [ + mock.call( + compose, + server, + 'x86_64', + ['Client'], + 'my.iso', + bootable=True, + inherit_extra_files=False, + ), + ], + ) self.assertEqual( rcc.call_args_list, [mock.call(False, 1, compose, True, 'x86_64', @@ -219,8 +243,20 @@ class ExtraIsosThreadTest(helpers.PungiTestCase): [mock.call(compose, server, 'x86_64', ['v1', 'v2'])]) self.assertEqual(gef.call_args_list, [mock.call(compose, server, 'x86_64', [])]) - self.assertEqual(gic.call_args_list, - [mock.call(compose, server, 'x86_64', ['Client'], 'my.iso', False)]) + self.assertEqual( + gic.call_args_list, + [ + mock.call( + compose, + server, + 'x86_64', + ['Client'], + 'my.iso', + bootable=False, + inherit_extra_files=False, + ), + ], + ) self.assertEqual( rcc.call_args_list, [mock.call(False, 1, compose, False, 'x86_64', @@ -262,8 +298,20 @@ class ExtraIsosThreadTest(helpers.PungiTestCase): [mock.call(compose, server, 'src', [])]) self.assertEqual(gef.call_args_list, [mock.call(compose, server, 'src', [])]) - self.assertEqual(gic.call_args_list, - [mock.call(compose, server, 'src', ['Client'], 'my.iso', False)]) + self.assertEqual( + gic.call_args_list, + [ + mock.call( + compose, + server, + 'src', + ['Client'], + 'my.iso', + bootable=False, + inherit_extra_files=False, + ), + ], + ) self.assertEqual( rcc.call_args_list, [mock.call(False, 1, compose, False, 'src', @@ -387,8 +435,6 @@ class GetIsoContentsTest(helpers.PungiTestCase): 'compose/Client/x86_64/os/repodata': {'primary.xml': '/mnt/repodata/primary.xml'}, 'compose/Server/x86_64/os/Packages': {'b/bar.rpm': '/mnt/b/bar.rpm'}, 'compose/Server/x86_64/os/repodata': {'repomd.xml': '/mnt/repodata/repomd.xml'}, - 'work/x86_64/Client/extra-files': {'GPL': '/mnt/GPL'}, - 'work/x86_64/Server/extra-files': {'AUTHORS': '/mnt/AUTHORS'}, 'work/x86_64/Server/extra-iso-extra-files': {'EULA': '/mnt/EULA'}, } @@ -396,17 +442,22 @@ class GetIsoContentsTest(helpers.PungiTestCase): gp_file = os.path.join(self.topdir, 'work/x86_64/iso/my.iso-graft-points') self.assertEqual( - extra_isos.get_iso_contents(self.compose, self.variant, 'x86_64', - ['Client'], 'my.iso', False), - gp_file + extra_isos.get_iso_contents( + self.compose, + self.variant, + 'x86_64', + ['Client'], + 'my.iso', + False, + inherit_extra_files=False, + ), + gp_file, ) expected = { - 'Client/GPL': '/mnt/GPL', 'Client/Packages/f/foo.rpm': '/mnt/f/foo.rpm', 'Client/repodata/primary.xml': '/mnt/repodata/primary.xml', 'EULA': '/mnt/EULA', - 'Server/AUTHORS': '/mnt/AUTHORS', 'Server/Packages/b/bar.rpm': '/mnt/b/bar.rpm', 'Server/repodata/repomd.xml': '/mnt/repodata/repomd.xml', } @@ -436,14 +487,74 @@ class GetIsoContentsTest(helpers.PungiTestCase): ], ) + def test_inherit_extra_files(self, ggp, wgp, tt): + gp = { + "compose/Client/x86_64/os/Packages": {"f/foo.rpm": "/mnt/f/foo.rpm"}, + "compose/Client/x86_64/os/repodata": {"primary.xml": "/mnt/repodata/primary.xml"}, + "compose/Server/x86_64/os/Packages": {"b/bar.rpm": "/mnt/b/bar.rpm"}, + "compose/Server/x86_64/os/repodata": {"repomd.xml": "/mnt/repodata/repomd.xml"}, + "work/x86_64/Client/extra-files": {"GPL": "/mnt/GPL"}, + "work/x86_64/Server/extra-files": {"AUTHORS": "/mnt/AUTHORS"}, + "work/x86_64/Server/extra-iso-extra-files": {"EULA": "/mnt/EULA"}, + } + + ggp.side_effect = lambda x: gp[x[0][len(self.topdir) + 1:]] + gp_file = os.path.join(self.topdir, "work/x86_64/iso/my.iso-graft-points") + + self.assertEqual( + extra_isos.get_iso_contents( + self.compose, + self.variant, + "x86_64", + ["Client"], + "my.iso", + False, + inherit_extra_files=True, + ), + gp_file, + ) + + expected = { + "Client/GPL": "/mnt/GPL", + "Client/Packages/f/foo.rpm": "/mnt/f/foo.rpm", + "Client/repodata/primary.xml": "/mnt/repodata/primary.xml", + "EULA": "/mnt/EULA", + "Server/AUTHORS": "/mnt/AUTHORS", + "Server/Packages/b/bar.rpm": "/mnt/b/bar.rpm", + "Server/repodata/repomd.xml": "/mnt/repodata/repomd.xml", + } + + self.assertItemsEqual( + ggp.call_args_list, + [mock.call([os.path.join(self.topdir, x)]) for x in gp] + ) + self.assertEqual(len(wgp.call_args_list), 1) + self.assertEqual(wgp.call_args_list[0][0][0], gp_file) + self.assertDictEqual(dict(wgp.call_args_list[0][0][1]), expected) + self.assertEqual(wgp.call_args_list[0][1], {"exclude": ["*/lost+found", "*/boot.iso"]}) + + # Check correct call to tweak_treeinfo + self.assertEqual( + tt.call_args_list, + [ + mock.call( + self.compose, + ["Client"], + os.path.join(self.topdir, "compose/Server/x86_64/os/.treeinfo"), + os.path.join( + self.topdir, + "work/x86_64/Server/extra-iso-extra-files/.treeinfo", + ) + ), + ], + ) + def test_source(self, ggp, wgp, tt): gp = { 'compose/Client/source/tree/Packages': {'f/foo.rpm': '/mnt/f/foo.rpm'}, 'compose/Client/source/tree/repodata': {'primary.xml': '/mnt/repodata/primary.xml'}, 'compose/Server/source/tree/Packages': {'b/bar.rpm': '/mnt/b/bar.rpm'}, 'compose/Server/source/tree/repodata': {'repomd.xml': '/mnt/repodata/repomd.xml'}, - 'work/src/Client/extra-files': {'GPL': '/mnt/GPL'}, - 'work/src/Server/extra-files': {'AUTHORS': '/mnt/AUTHORS'}, 'work/src/Server/extra-iso-extra-files': {'EULA': '/mnt/EULA'}, } @@ -451,17 +562,22 @@ class GetIsoContentsTest(helpers.PungiTestCase): gp_file = os.path.join(self.topdir, 'work/src/iso/my.iso-graft-points') self.assertEqual( - extra_isos.get_iso_contents(self.compose, self.variant, 'src', - ['Client'], 'my.iso', False), - gp_file + extra_isos.get_iso_contents( + self.compose, + self.variant, + 'src', + ['Client'], + 'my.iso', + bootable=False, + inherit_extra_files=False, + ), + gp_file, ) expected = { - 'Client/GPL': '/mnt/GPL', 'Client/Packages/f/foo.rpm': '/mnt/f/foo.rpm', 'Client/repodata/primary.xml': '/mnt/repodata/primary.xml', 'EULA': '/mnt/EULA', - 'Server/AUTHORS': '/mnt/AUTHORS', 'Server/Packages/b/bar.rpm': '/mnt/b/bar.rpm', 'Server/repodata/repomd.xml': '/mnt/repodata/repomd.xml', } @@ -505,8 +621,6 @@ class GetIsoContentsTest(helpers.PungiTestCase): 'compose/Client/x86_64/os/repodata': {'primary.xml': '/mnt/repodata/primary.xml'}, 'compose/Server/x86_64/os/Packages': {'b/bar.rpm': '/mnt/b/bar.rpm'}, 'compose/Server/x86_64/os/repodata': {'repomd.xml': '/mnt/repodata/repomd.xml'}, - 'work/x86_64/Client/extra-files': {'GPL': '/mnt/GPL'}, - 'work/x86_64/Server/extra-files': {'AUTHORS': '/mnt/AUTHORS'}, 'work/x86_64/Server/extra-iso-extra-files': {'EULA': '/mnt/EULA'}, } bi_gp = { @@ -525,18 +639,18 @@ class GetIsoContentsTest(helpers.PungiTestCase): 'x86_64', ['Client'], 'my.iso', - True), - gp_file + bootable=True, + inherit_extra_files=False, + ), + gp_file, ) self.maxDiff = None expected = { - 'Client/GPL': '/mnt/GPL', 'Client/Packages/f/foo.rpm': '/mnt/f/foo.rpm', 'Client/repodata/primary.xml': '/mnt/repodata/primary.xml', 'EULA': '/mnt/EULA', - 'Server/AUTHORS': '/mnt/AUTHORS', 'Server/Packages/b/bar.rpm': '/mnt/b/bar.rpm', 'Server/repodata/repomd.xml': '/mnt/repodata/repomd.xml', 'isolinux/isolinux.bin': os.path.join(iso_dir, 'isolinux/isolinux.bin'),