From 6c6d4759f57fb8278fd94081da534746f3f109b1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lubom=C3=ADr=20Sedl=C3=A1=C5=99?= Date: Tue, 12 Feb 2019 12:09:08 +0100 Subject: [PATCH] isos: Check maximum expected size MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This patch allows the configuration to express maximum expected size for ISOs created in createiso and extra_isos phases. If the image is larger than this limit, a warning is emitted in test phase. The compose itself is not affected in any way. JIRA: COMPOSE-2824 Signed-off-by: Lubomír Sedlář --- doc/configuration.rst | 9 ++++++ pungi/checks.py | 2 ++ pungi/phases/createiso.py | 1 + pungi/phases/extra_isos.py | 3 +- pungi/phases/test.py | 24 ++++++++++++-- tests/helpers.py | 4 ++- tests/test_extra_isos_phase.py | 57 ++++++++++++++++++++++++++++++++++ tests/test_test_phase.py | 54 ++++++++++++++++++++++++++++++++ 8 files changed, 150 insertions(+), 4 deletions(-) diff --git a/doc/configuration.rst b/doc/configuration.rst index dd6e3118..fa9b3b2a 100644 --- a/doc/configuration.rst +++ b/doc/configuration.rst @@ -1003,6 +1003,12 @@ Options **createiso_skip** = False (*list*) -- mapping that defines which variants and arches to skip during createiso; format: [(variant_uid_regex, {arch|*: True})] +**createiso_max_size** + (*list*) -- mapping that defines maximum expected size for each variant and + arch. If the ISO is larger than the limit, a warning will be issued. + + Format: ``[(variant_uid_regex, {arch|*: number})]`` + **create_jigdo** = True (*bool*) -- controls the creation of jigdo from ISO @@ -1574,6 +1580,9 @@ will reuse boot configuration from that variant. are ignored. If you want to include them in the ISO, set this option to ``True``. + * ``max_size`` -- (*int*) expected maximum size in bytes. If the final + image is larger, a warning will be issued. + Example config -------------- :: diff --git a/pungi/checks.py b/pungi/checks.py index 388ad445..0f2ab356 100644 --- a/pungi/checks.py +++ b/pungi/checks.py @@ -730,6 +730,7 @@ def make_schema(): "default": {}, }, "createiso_skip": _variant_arch_mapping({"type": "boolean"}), + "createiso_max_size": _variant_arch_mapping({"type": "number"}), "createiso_break_hardlinks": { "type": "boolean", "default": False, @@ -969,6 +970,7 @@ def make_schema(): "type": "boolean", "default": False, }, + "max_size": {"type": "number"}, }, "required": ["include_variants"], "additionalProperties": False diff --git a/pungi/phases/createiso.py b/pungi/phases/createiso.py index cbe911e5..060cc0f7 100644 --- a/pungi/phases/createiso.py +++ b/pungi/phases/createiso.py @@ -263,6 +263,7 @@ def add_iso_to_metadata( compose.im.add(variant.uid, variant_arch, img) else: compose.im.add(variant.uid, arch, img) + return img def run_createiso_command(runroot, num, compose, bootable, arch, cmd, mounts, diff --git a/pungi/phases/extra_isos.py b/pungi/phases/extra_isos.py index 3510547a..6c4b5e0f 100644 --- a/pungi/phases/extra_isos.py +++ b/pungi/phases/extra_isos.py @@ -125,7 +125,7 @@ class ExtraIsosThread(WorkerThread): arch, "extraiso-%s" % os.path.basename(iso_path)), with_jigdo=compose.conf['create_jigdo']) - add_iso_to_metadata( + img = add_iso_to_metadata( compose, variant, arch, @@ -133,6 +133,7 @@ class ExtraIsosThread(WorkerThread): bootable, additional_variants=config["include_variants"], ) + img._max_size = config.get("max_size") self.pool.log_info("[DONE ] %s" % msg) diff --git a/pungi/phases/test.py b/pungi/phases/test.py index 1efe2953..a8fe4506 100644 --- a/pungi/phases/test.py +++ b/pungi/phases/test.py @@ -114,10 +114,11 @@ def check_image_sanity(compose): if arch not in im.images[variant.uid]: continue for img in im.images[variant.uid][arch]: - check(compose, variant, arch, img) + check_sanity(compose, variant, arch, img) + check_size_limit(compose, variant, arch, img) -def check(compose, variant, arch, image): +def check_sanity(compose, variant, arch, image): path = os.path.join(compose.paths.compose.topdir(), image.path) deliverable = getattr(image, 'deliverable') can_fail = getattr(image, 'can_fail', False) @@ -159,3 +160,22 @@ def has_gpt(f): def has_eltorito(f): return _check_magic(f, 0x8801, b'CD001\1EL TORITO SPECIFICATION') + + +def check_size_limit(compose, variant, arch, img): + """If a size of the ISO image is over the configured limit, report a + warning. Do nothing for other types of images. + """ + if img.format != "iso": + return + limits = get_arch_variant_data(compose.conf, "createiso_max_size", arch, variant) + if not limits and not getattr(img, "_max_size", None): + return + # For ISOs created in extra_isos phase we add an attribute with the limit, + # and there is a global option otherwise. + limit = getattr(img, "_max_size", None) or limits[0] + + if img.size > limit: + compose.log_warning( + "ISO %s is too big. Expected max %dB, got %dB" % (img.path, limit, img.size) + ) diff --git a/tests/helpers.py b/tests/helpers.py index 467d3897..7287f047 100644 --- a/tests/helpers.py +++ b/tests/helpers.py @@ -169,7 +169,9 @@ class DummyCompose(object): self.log_debug = mock.Mock() self.log_warning = mock.Mock() self.get_image_name = mock.Mock(return_value='image-name') - self.image = mock.Mock(path='Client/i386/iso/image.iso', can_fail=False, size=123) + self.image = mock.Mock( + path='Client/i386/iso/image.iso', can_fail=False, size=123, _max_size=None, + ) self.im = mock.Mock(images={'Client': {'amd64': [self.image]}}) self.old_composes = [] self.config_dir = '/home/releng/config' diff --git a/tests/test_extra_isos_phase.py b/tests/test_extra_isos_phase.py index eb24aae4..a19499ef 100644 --- a/tests/test_extra_isos_phase.py +++ b/tests/test_extra_isos_phase.py @@ -220,6 +220,63 @@ class ExtraIsosThreadTest(helpers.PungiTestCase): ) self.assertEqual(pmm.call_args_list, [mock.call(compose, server, "x86_64")]) + def test_image_with_max_size(self, aitm, rcc, gef, gic, gfn, gvi, pmm): + compose = helpers.DummyCompose(self.topdir, { + "bootable": True, + "buildinstall_method": "lorax" + }) + server = compose.variants["Server"] + cfg = { + "include_variants": ["Client"], + "max_size": 15, + } + + gfn.return_value = "my.iso" + gvi.return_value = "my volume id" + gic.return_value = "/tmp/iso-graft-points" + + t = extra_isos.ExtraIsosThread(mock.Mock()) + with mock.patch("time.sleep"): + t.process((compose, cfg, server, "x86_64"), 1) + + self.assertEqual(gfn.call_args_list, + [mock.call(compose, server, "x86_64", None)]) + self.assertEqual(gvi.call_args_list, + [mock.call(compose, server, "x86_64", [])]) + self.assertEqual(gef.call_args_list, + [mock.call(compose, server, "x86_64", [])]) + self.assertEqual( + gic.call_args_list, + [ + mock.call( + compose, + server, + "x86_64", + ["Client"], + "my.iso", + bootable=True, + inherit_extra_files=False, + ), + ], + ) + self.assertEqual( + rcc.call_args_list, + [mock.call(False, 1, compose, True, "x86_64", + ["bash", os.path.join(self.topdir, "work/x86_64/tmp-Server/extraiso-my.iso.sh")], + [self.topdir], + log_file=os.path.join(self.topdir, "logs/x86_64/extraiso-my.iso.x86_64.log"), + with_jigdo=True)] + + ) + self.assertEqual( + aitm.call_args_list, + [mock.call(compose, server, "x86_64", + os.path.join(self.topdir, "compose/Server/x86_64/iso/my.iso"), + True, additional_variants=["Client"])] + ) + self.assertEqual(aitm.return_value._max_size, 15) + self.assertEqual(pmm.call_args_list, [mock.call(compose, server, "x86_64")]) + def test_binary_image_custom_naming(self, aitm, rcc, gef, gic, gfn, gvi, pmm): compose = helpers.DummyCompose(self.topdir, {}) server = compose.variants['Server'] diff --git a/tests/test_test_phase.py b/tests/test_test_phase.py index 824a67d5..d974473c 100644 --- a/tests/test_test_phase.py +++ b/tests/test_test_phase.py @@ -166,6 +166,60 @@ class TestCheckImageSanity(PungiTestCase): except Exception: self.fail('Checking optional variant must not raise') + @mock.patch("pungi.phases.test.check_sanity", new=mock.Mock()) + def test_too_big_iso(self): + compose = DummyCompose(self.topdir, {"createiso_max_size": [(".*", {"*": 10})]}) + compose.image.format = 'iso' + compose.image.bootable = False + compose.image.size = 20 + + test_phase.check_image_sanity(compose) + + warnings = [call[0][0] for call in compose.log_warning.call_args_list] + self.assertIn( + "ISO Client/i386/iso/image.iso is too big. Expected max 10B, got 20B", + warnings, + ) + + @mock.patch("pungi.phases.test.check_sanity", new=mock.Mock()) + def test_too_big_unified(self): + compose = DummyCompose(self.topdir, {}) + compose.image.format = 'iso' + compose.image.bootable = False + compose.image.size = 20 + compose.image.unified = True + setattr(compose.image, "_max_size", 10) + + test_phase.check_image_sanity(compose) + + warnings = [call[0][0] for call in compose.log_warning.call_args_list] + self.assertIn( + "ISO Client/i386/iso/image.iso is too big. Expected max 10B, got 20B", + warnings, + ) + + @mock.patch("pungi.phases.test.check_sanity", new=mock.Mock()) + def test_fits_in_limit(self): + compose = DummyCompose(self.topdir, {"createiso_max_size": [(".*", {"*": 20})]}) + compose.image.format = 'iso' + compose.image.bootable = False + compose.image.size = 5 + + test_phase.check_image_sanity(compose) + + self.assertEqual(compose.log_warning.call_args_list, []) + + @mock.patch("pungi.phases.test.check_sanity", new=mock.Mock()) + def test_non_iso(self): + compose = DummyCompose(self.topdir, {"createiso_max_size": [(".*", {"*": 10})]}) + compose.image.format = 'qcow2' + compose.image.bootable = False + compose.image.size = 20 + + test_phase.check_image_sanity(compose) + + self.assertEqual(compose.log_warning.call_args_list, []) + class TestRepoclosure(PungiTestCase):