From 81cb0952ca837a8f7dade8db235bf974df9df593 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lubom=C3=ADr=20Sedl=C3=A1=C5=99?= Date: Wed, 12 Jul 2017 14:52:40 +0200 Subject: [PATCH] checksum: Refactor creating checksum files MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Instead of iterating over the images metadata and appending the checksum to relevant files immediately, we should store them and write only once. This avoid an issue when the same image is mentioned in the metadata multiple times. This happens for source images that are listed under each binary arch. The unified isos script is updated to use the exact same logic and code. This also uncovered a problem with the metadata for debuginfo unified isos: their paths in metadata were incorrect, which lead to missing checksums. Fixes: https://pagure.io/pungi/issue/667 Fixes: https://pagure.io/pungi/issue/668 Signed-off-by: Lubomír Sedlář --- pungi/phases/image_checksum.py | 76 +++++--------- pungi_utils/unified_isos.py | 38 +++---- tests/helpers.py | 2 +- tests/test_imagechecksumphase.py | 168 ++++++++----------------------- tests/test_unified_isos.py | 125 ++++++----------------- 5 files changed, 116 insertions(+), 293 deletions(-) diff --git a/pungi/phases/image_checksum.py b/pungi/phases/image_checksum.py index 111ef924..fcdba9bd 100644 --- a/pungi/phases/image_checksum.py +++ b/pungi/phases/image_checksum.py @@ -2,6 +2,7 @@ import os from kobo import shortcuts +from collections import defaultdict from .base import PhaseBase from ..util import get_format_substs, get_file_size @@ -69,15 +70,10 @@ class ImageChecksumPhase(PhaseBase): def run(self): topdir = self.compose.paths.compose.topdir() - for (variant, arch, path), images in get_images(topdir, self.compose.im).iteritems(): - base_checksum_name = self._get_base_filename(variant, arch) - make_checksums(variant, arch, path, images, - self.checksums, base_checksum_name, self.one_file) + make_checksums(topdir, self.compose.im, self.checksums, self.one_file, self._get_base_filename) -def make_checksums(variant, arch, path, images, checksum_types, base_checksum_name, one_file): - checksums = {} - filesizes = {} +def _compute_checksums(results, variant, arch, path, images, checksum_types, base_checksum_name, one_file): for image in images: filename = os.path.basename(image.path) full_path = os.path.join(path, filename) @@ -85,63 +81,45 @@ def make_checksums(variant, arch, path, images, checksum_types, base_checksum_na continue filesize = image.size or get_file_size(full_path) - filesizes[filename] = filesize digests = shortcuts.compute_file_checksums(full_path, checksum_types) for checksum, digest in digests.iteritems(): - checksums.setdefault(checksum, {})[filename] = digest + # Update metadata with the checksum image.add_checksum(None, checksum, digest) + # If not turned of, create the file-specific checksum file if not one_file: - checksum_filename = '%s.%sSUM' % (filename, checksum.upper()) - dump_filesizes(path, {filename: filesize}, checksum_filename) - dump_checksums(path, checksum, - {filename: digest}, - checksum_filename) + checksum_filename = os.path.join(path, '%s.%sSUM' % (filename, checksum.upper())) + results[checksum_filename].add((filename, filesize, checksum, digest)) - if not checksums: - return + if one_file: + checksum_filename = os.path.join(path, base_checksum_name + 'CHECKSUM') + else: + checksum_filename = os.path.join(path, '%s%sSUM' % (base_checksum_name, checksum.upper())) - if one_file: - checksum_filename = base_checksum_name + 'CHECKSUM' - dump_filesizes(path, filesizes, checksum_filename) - dump_checksums(path, checksum_types[0], - checksums[checksum_types[0]], - checksum_filename) - else: - for checksum in checksums: - checksum_filename = '%s%sSUM' % (base_checksum_name, checksum.upper()) - dump_filesizes(path, filesizes, checksum_filename) - dump_checksums(path, checksum, - checksums[checksum], - checksum_filename) + results[checksum_filename].add((filename, filesize, checksum, digest)) -def dump_filesizes(dir, filesizes, filename): - """Write filesizes to file with comment lines. +def make_checksums(topdir, im, checksum_types, one_file, base_checksum_name_gen): + results = defaultdict(set) + for (variant, arch, path), images in get_images(topdir, im).iteritems(): + base_checksum_name = base_checksum_name_gen(variant, arch) + _compute_checksums(results, variant, arch, path, images, + checksum_types, base_checksum_name, one_file) - :param dir: where to put the file - :param filesizes: mapping from filenames to filesizes - :param filename: what to call the file - """ - filesize_file = os.path.join(dir, filename) - with open(filesize_file, 'a') as f: - for file, filesize in filesizes.iteritems(): - f.write('# %s: %s bytes\n' % (file, filesize)) + for file in results: + dump_checksums(file, results[file]) -def dump_checksums(dir, alg, checksums, filename): +def dump_checksums(checksum_file, data): """Write checksums to file. - :param dir: where to put the file - :param alg: which method was used - :param checksums: mapping from filenames to checksums - :param filename: what to call the file + :param checksum_file: where to write the checksums + :param data: an iterable of tuples (filename, filesize, checksum_type, hash) """ - checksum_file = os.path.join(dir, filename) - with open(checksum_file, 'a') as f: - for file, checksum in checksums.iteritems(): - f.write('%s (%s) = %s\n' % (alg.upper(), file, checksum)) - return checksum_file + with open(checksum_file, 'w') as f: + for filename, filesize, alg, checksum in sorted(data): + f.write('# %s: %s bytes\n' % (filename, filesize)) + f.write('%s (%s) = %s\n' % (alg.upper(), filename, checksum)) def get_images(top_dir, manifest): diff --git a/pungi_utils/unified_isos.py b/pungi_utils/unified_isos.py index 565deccd..10de95cd 100644 --- a/pungi_utils/unified_isos.py +++ b/pungi_utils/unified_isos.py @@ -27,14 +27,14 @@ import productmd import productmd.compose import productmd.images import productmd.treeinfo -from kobo.shortcuts import run, compute_file_checksums +from kobo.shortcuts import run import pungi.linker import pungi.wrappers.createrepo from pungi.util import makedirs from pungi.compose_metadata.discinfo import write_discinfo as create_discinfo from pungi.wrappers import iso -from pungi.phases.image_checksum import dump_checksums, get_images, make_checksums +from pungi.phases.image_checksum import make_checksums def ti_merge(one, two): @@ -88,10 +88,14 @@ class UnifiedISO(object): self.createiso() self.link_to_compose() self.update_checksums() + self.dump_manifest() finally: if delete_temp: shutil.rmtree(self.temp_dir) + def dump_manifest(self): + self.compose.images.dump(os.path.join(self.compose_path, 'metadata', 'images.json')) + def _link_tree(self, dir, variant, arch): blacklist_files = [".treeinfo", ".discinfo", "boot.iso", "media.repo", "extra_files.json"] blacklist_dirs = ["repodata"] @@ -308,9 +312,6 @@ class UnifiedISO(object): supported = True run(iso.get_implantisomd5_cmd(iso_path, supported)) - checksums = compute_file_checksums( - iso_path, self.conf.get('media_checksums', DEFAULT_CHECKSUMS)) - # write manifest file run(iso.get_manifest_cmd(iso_path)) @@ -332,15 +333,6 @@ class UnifiedISO(object): self.images.setdefault(typed_arch, set()).add(iso_path) self.images.setdefault(typed_arch, set()).add(iso_path + ".manifest") - for checksum_type, checksum in checksums.iteritems(): - if not self.conf.get('media_checksum_one_file', False): - checksum_path = dump_checksums(iso_dir, checksum_type, - {iso_name: checksum}, - '%s.%sSUM' % (iso_name, checksum_type.upper())) - self.images.setdefault(typed_arch, set()).add(checksum_path) - - img.add_checksum(self.compose_path, checksum_type=checksum_type, checksum_value=checksum) - img.implant_md5 = iso.get_implanted_md5(iso_path) try: img.volume_id = iso.get_volume_id(iso_path) @@ -365,14 +357,16 @@ class UnifiedISO(object): variant_img.subvariant = variant.id paths_attr = 'isos' if arch != 'src' else 'source_isos' paths = getattr(self.ci.variants[variant.uid].paths, paths_attr) + path = paths.get(tree_arch, os.path.join(variant.uid, tree_arch, "iso")) + if variant_img.type == 'dvd-debuginfo': + prefix, isodir = path.rsplit('/', 1) + path = os.path.join(prefix, 'debug', isodir) variant_img.path = os.path.join( - paths.get(tree_arch, os.path.join(variant.uid, tree_arch, "iso")), + path, os.path.basename(img.path) ) im.add(variant.uid, tree_arch, variant_img) - im.dump(os.path.join(self.compose_path, 'metadata', 'images.json')) - def link_to_compose(self): for variant in self.ci.get_variants(recursive=False): for arch in variant.arches | set(['debug-' + a for a in variant.arches]) | set(['src']): @@ -412,9 +406,7 @@ class UnifiedISO(object): return base_name def update_checksums(self): - for (variant, arch, path), images in get_images(self.compose_path, self.compose.images).iteritems(): - base_checksum_name = self._get_base_filename(variant, arch) - make_checksums(variant, arch, path, images, - self.conf.get('media_checksums', DEFAULT_CHECKSUMS), - base_checksum_name, - self.conf.get('media_checksum_one_file', False)) + make_checksums(self.compose_path, self.compose.images, + self.conf.get('media_checksums', DEFAULT_CHECKSUMS), + self.conf.get('media_checksum_one_file', False), + self._get_base_filename) diff --git a/tests/helpers.py b/tests/helpers.py index 401175e0..dc1dadb1 100644 --- a/tests/helpers.py +++ b/tests/helpers.py @@ -94,7 +94,7 @@ class DummyCompose(object): self.log_debug = mock.Mock() self.log_warning = mock.Mock() self.get_image_name = mock.Mock(return_value='image-name') - self.image = mock.Mock(path='Client/i386/iso/image.iso', can_fail=False) + self.image = mock.Mock(path='Client/i386/iso/image.iso', can_fail=False, size=123) self.im = mock.Mock(images={'Client': {'amd64': [self.image]}}) self.old_composes = [] self.config_dir = '/home/releng/config' diff --git a/tests/test_imagechecksumphase.py b/tests/test_imagechecksumphase.py index 33e843e8..0515552c 100644 --- a/tests/test_imagechecksumphase.py +++ b/tests/test_imagechecksumphase.py @@ -14,7 +14,7 @@ import shutil sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..")) -from pungi.phases.image_checksum import ImageChecksumPhase, dump_checksums, dump_filesizes +from pungi.phases.image_checksum import ImageChecksumPhase, dump_checksums from tests.helpers import DummyCompose, PungiTestCase @@ -38,9 +38,8 @@ class TestImageChecksumPhase(PungiTestCase): @mock.patch('os.path.exists') @mock.patch('kobo.shortcuts.compute_file_checksums') - @mock.patch('pungi.phases.image_checksum.dump_filesizes') @mock.patch('pungi.phases.image_checksum.dump_checksums') - def test_checksum_one_file(self, dump_checksums, dump_filesizes, cc, exists): + def test_checksum_one_file(self, dump_checksums, cc, exists): compose = DummyCompose(self.topdir, { 'media_checksums': ['sha256'], 'media_checksum_one_file': True, @@ -53,15 +52,15 @@ class TestImageChecksumPhase(PungiTestCase): phase.run() - dump_checksums.assert_called_once_with(self.topdir + '/compose/Client/i386/iso', 'sha256', {'image.iso': 'cafebabe'}, 'CHECKSUM') + dump_checksums.assert_called_once_with(self.topdir + '/compose/Client/i386/iso/CHECKSUM', + set([('image.iso', 123, 'sha256', 'cafebabe')])) cc.assert_called_once_with(self.topdir + '/compose/Client/i386/iso/image.iso', ['sha256']) compose.image.add_checksum.assert_called_once_with(None, 'sha256', 'cafebabe') @mock.patch('os.path.exists') @mock.patch('kobo.shortcuts.compute_file_checksums') - @mock.patch('pungi.phases.image_checksum.dump_filesizes') @mock.patch('pungi.phases.image_checksum.dump_checksums') - def test_checksum_save_individuals(self, dump_checksums, dump_filesizes, cc, exists): + def test_checksum_save_individuals(self, dump_checksums, cc, exists): compose = DummyCompose(self.topdir, { 'media_checksums': ['md5', 'sha256'], }) @@ -74,14 +73,14 @@ class TestImageChecksumPhase(PungiTestCase): phase.run() dump_checksums.assert_has_calls( - [mock.call(self.topdir + '/compose/Client/i386/iso', 'md5', - {'image.iso': 'cafebabe'}, 'image.iso.MD5SUM'), - mock.call(self.topdir + '/compose/Client/i386/iso', 'sha256', - {'image.iso': 'deadbeef'}, 'image.iso.SHA256SUM'), - mock.call(self.topdir + '/compose/Client/i386/iso', 'md5', - {'image.iso': 'cafebabe'}, 'MD5SUM'), - mock.call(self.topdir + '/compose/Client/i386/iso', 'sha256', - {'image.iso': 'deadbeef'}, 'SHA256SUM')], + [mock.call(self.topdir + '/compose/Client/i386/iso/image.iso.MD5SUM', + set([('image.iso', 123, 'md5', 'cafebabe')])), + mock.call(self.topdir + '/compose/Client/i386/iso/image.iso.SHA256SUM', + set([('image.iso', 123, 'sha256', 'deadbeef')])), + mock.call(self.topdir + '/compose/Client/i386/iso/MD5SUM', + set([('image.iso', 123, 'md5', 'cafebabe')])), + mock.call(self.topdir + '/compose/Client/i386/iso/SHA256SUM', + set([('image.iso', 123, 'sha256', 'deadbeef')]))], any_order=True ) cc.assert_called_once_with(self.topdir + '/compose/Client/i386/iso/image.iso', ['md5', 'sha256']) @@ -91,9 +90,8 @@ class TestImageChecksumPhase(PungiTestCase): @mock.patch('os.path.exists') @mock.patch('kobo.shortcuts.compute_file_checksums') - @mock.patch('pungi.phases.image_checksum.dump_filesizes') @mock.patch('pungi.phases.image_checksum.dump_checksums') - def test_checksum_one_file_custom_name(self, dump_checksums, dump_filesizes, cc, exists): + def test_checksum_one_file_custom_name(self, dump_checksums, cc, exists): compose = DummyCompose(self.topdir, { 'media_checksums': ['sha256'], 'media_checksum_one_file': True, @@ -108,17 +106,16 @@ class TestImageChecksumPhase(PungiTestCase): phase.run() - dump_checksums.assert_called_once_with(self.topdir + '/compose/Client/i386/iso', 'sha256', - {'image.iso': 'cafebabe'}, - 'test-Client-1.0-20151203.t.0_Alpha-1.0-CHECKSUM') + dump_checksums.assert_called_once_with( + self.topdir + '/compose/Client/i386/iso/test-Client-1.0-20151203.t.0_Alpha-1.0-CHECKSUM', + set([('image.iso', 123, 'sha256', 'cafebabe')])) cc.assert_called_once_with(self.topdir + '/compose/Client/i386/iso/image.iso', ['sha256']) compose.image.add_checksum.assert_called_once_with(None, 'sha256', 'cafebabe') @mock.patch('os.path.exists') @mock.patch('kobo.shortcuts.compute_file_checksums') - @mock.patch('pungi.phases.image_checksum.dump_filesizes') @mock.patch('pungi.phases.image_checksum.dump_checksums') - def test_checksum_save_individuals_custom_name(self, dump_checksums, dump_filesizes, cc, exists): + def test_checksum_save_individuals_custom_name(self, dump_checksums, cc, exists): compose = DummyCompose(self.topdir, { 'media_checksums': ['md5', 'sha256'], 'media_checksum_base_filename': '%(release_short)s-%(variant)s-%(version)s-%(date)s%(type_suffix)s.%(respin)s' @@ -132,14 +129,14 @@ class TestImageChecksumPhase(PungiTestCase): phase.run() dump_checksums.assert_has_calls( - [mock.call(self.topdir + '/compose/Client/i386/iso', 'md5', - {'image.iso': 'cafebabe'}, 'image.iso.MD5SUM'), - mock.call(self.topdir + '/compose/Client/i386/iso', 'sha256', - {'image.iso': 'deadbeef'}, 'image.iso.SHA256SUM'), - mock.call(self.topdir + '/compose/Client/i386/iso', 'md5', {'image.iso': 'cafebabe'}, - 'test-Client-1.0-20151203.t.0-MD5SUM'), - mock.call(self.topdir + '/compose/Client/i386/iso', 'sha256', {'image.iso': 'deadbeef'}, - 'test-Client-1.0-20151203.t.0-SHA256SUM')], + [mock.call(self.topdir + '/compose/Client/i386/iso/image.iso.MD5SUM', + set([('image.iso', 123, 'md5', 'cafebabe')])), + mock.call(self.topdir + '/compose/Client/i386/iso/image.iso.SHA256SUM', + set([('image.iso', 123, 'sha256', 'deadbeef')])), + mock.call(self.topdir + '/compose/Client/i386/iso/test-Client-1.0-20151203.t.0-MD5SUM', + set([('image.iso', 123, 'md5', 'cafebabe')])), + mock.call(self.topdir + '/compose/Client/i386/iso/test-Client-1.0-20151203.t.0-SHA256SUM', + set([('image.iso', 123, 'sha256', 'deadbeef')]))], any_order=True ) cc.assert_called_once_with(self.topdir + '/compose/Client/i386/iso/image.iso', ['md5', 'sha256']) @@ -149,9 +146,8 @@ class TestImageChecksumPhase(PungiTestCase): @mock.patch('os.path.exists') @mock.patch('kobo.shortcuts.compute_file_checksums') - @mock.patch('pungi.phases.image_checksum.dump_filesizes') @mock.patch('pungi.phases.image_checksum.dump_checksums') - def test_checksum_save_individuals_custom_name_str_format(self, dump_checksums, dump_filesizes, cc, exists): + def test_checksum_save_individuals_custom_name_str_format(self, dump_checksums, cc, exists): compose = DummyCompose(self.topdir, { 'media_checksums': ['md5', 'sha256'], 'media_checksum_base_filename': '{release_short}-{variant}-{version}-{date}{type_suffix}.{respin}' @@ -165,14 +161,14 @@ class TestImageChecksumPhase(PungiTestCase): phase.run() dump_checksums.assert_has_calls( - [mock.call(self.topdir + '/compose/Client/i386/iso', 'md5', - {'image.iso': 'cafebabe'}, 'image.iso.MD5SUM'), - mock.call(self.topdir + '/compose/Client/i386/iso', 'sha256', - {'image.iso': 'deadbeef'}, 'image.iso.SHA256SUM'), - mock.call(self.topdir + '/compose/Client/i386/iso', 'md5', {'image.iso': 'cafebabe'}, - 'test-Client-1.0-20151203.t.0-MD5SUM'), - mock.call(self.topdir + '/compose/Client/i386/iso', 'sha256', {'image.iso': 'deadbeef'}, - 'test-Client-1.0-20151203.t.0-SHA256SUM')], + [mock.call(self.topdir + '/compose/Client/i386/iso/image.iso.MD5SUM', + set([('image.iso', 123, 'md5', 'cafebabe')])), + mock.call(self.topdir + '/compose/Client/i386/iso/image.iso.SHA256SUM', + set([('image.iso', 123, 'sha256', 'deadbeef')])), + mock.call(self.topdir + '/compose/Client/i386/iso/test-Client-1.0-20151203.t.0-MD5SUM', + set([('image.iso', 123, 'md5', 'cafebabe')])), + mock.call(self.topdir + '/compose/Client/i386/iso/test-Client-1.0-20151203.t.0-SHA256SUM', + set([('image.iso', 123, 'sha256', 'deadbeef')]))], any_order=True ) cc.assert_called_once_with(self.topdir + '/compose/Client/i386/iso/image.iso', ['md5', 'sha256']) @@ -180,67 +176,8 @@ class TestImageChecksumPhase(PungiTestCase): mock.call(None, 'md5', 'cafebabe')], any_order=True) - @mock.patch('os.path.exists') - @mock.patch('kobo.shortcuts.compute_file_checksums') - @mock.patch('pungi.phases.image_checksum.get_file_size') - @mock.patch('pungi.phases.image_checksum.dump_filesizes') - @mock.patch('pungi.phases.image_checksum.dump_checksums') - def test_dump_filesizes_one_file(self, dump_checksums, dump_filesizes, get_file_size, cc, exists): - compose = DummyCompose(self.topdir, { - 'media_checksums': ['md5', 'sha256'], - 'media_checksum_base_filename': '{release_short}-{variant}-{version}-{date}{type_suffix}.{respin}', - 'media_checksum_one_file': True - }) - compose.image.size = None - get_file_size.return_value = '12345' - phase = ImageChecksumPhase(compose) - - exists.return_value = True - cc.return_value = {'md5': 'cafebabe', 'sha256': 'deadbeef'} - - phase.run() - - dump_filesizes.assert_called_once_with( - self.topdir + '/compose/Client/i386/iso', {'image.iso': '12345'}, 'test-Client-1.0-20151203.t.0-CHECKSUM') - get_file_size.assert_called_once_with(self.topdir + '/compose/Client/i386/iso/image.iso') - - @mock.patch('os.path.exists') - @mock.patch('kobo.shortcuts.compute_file_checksums') - @mock.patch('pungi.phases.image_checksum.get_file_size') - @mock.patch('pungi.phases.image_checksum.dump_filesizes') - @mock.patch('pungi.phases.image_checksum.dump_checksums') - def test_dump_filesizes_save_individuals(self, dump_checksums, dump_filesizes, get_file_size, cc, exists): - compose = DummyCompose(self.topdir, { - 'media_checksums': ['md5', 'sha256'], - 'media_checksum_base_filename': '{release_short}-{variant}-{version}-{date}{type_suffix}.{respin}' - }) - compose.image.size = None - get_file_size.return_value = '12345' - - phase = ImageChecksumPhase(compose) - - exists.return_value = True - cc.return_value = {'md5': 'cafebabe', 'sha256': 'deadbeef'} - - phase.run() - - dump_filesizes.assert_has_calls( - [mock.call(self.topdir + '/compose/Client/i386/iso', - {'image.iso': '12345'}, 'image.iso.SHA256SUM'), - mock.call(self.topdir + '/compose/Client/i386/iso', - {'image.iso': '12345'}, 'image.iso.MD5SUM'), - mock.call(self.topdir + '/compose/Client/i386/iso', - {'image.iso': '12345'}, 'test-Client-1.0-20151203.t.0-SHA256SUM'), - mock.call(self.topdir + '/compose/Client/i386/iso', - {'image.iso': '12345'}, 'test-Client-1.0-20151203.t.0-MD5SUM')], - any_order=True - ) - - get_file_size.assert_called_once_with(self.topdir + '/compose/Client/i386/iso/image.iso') - - -class TestChecksums(unittest.TestCase): +class TestDumpChecksums(unittest.TestCase): def setUp(self): self.tmp_dir = tempfile.mkdtemp() @@ -248,39 +185,20 @@ class TestChecksums(unittest.TestCase): shutil.rmtree(self.tmp_dir) def test_dump_checksums(self): - dump_checksums(self.tmp_dir, - 'md5', - {'file1.iso': 'abcdef', 'file2.iso': 'cafebabe'}, - 'CHECKSUM') - - with open(os.path.join(self.tmp_dir, 'CHECKSUM'), 'r') as f: - data = f.read().rstrip().split('\n') - expected = [ - 'MD5 (file1.iso) = abcdef', - 'MD5 (file2.iso) = cafebabe', - ] - self.assertItemsEqual(expected, data) - - -class TestDumpFilesizes(unittest.TestCase): - def setUp(self): - self.tmp_dir = tempfile.mkdtemp() - - def tearDown(self): - shutil.rmtree(self.tmp_dir) - - def test_dump_files(self): - filesizes = {'file1.iso': 123, - 'file2.iso': 456} - dump_filesizes(self.tmp_dir, filesizes, 'CHECKSUM') + dump_checksums(os.path.join(self.tmp_dir, 'CHECKSUM'), + [('file2.iso', 456, 'md5', 'cafebabe'), + ('file1.iso', 123, 'md5', 'abcdef')]) with open(os.path.join(self.tmp_dir, 'CHECKSUM'), 'r') as f: data = f.read().rstrip().split('\n') expected = [ '# file1.iso: 123 bytes', + 'MD5 (file1.iso) = abcdef', '# file2.iso: 456 bytes', + 'MD5 (file2.iso) = cafebabe', ] - self.assertItemsEqual(expected, data) + self.assertEqual(expected, data) + if __name__ == "__main__": unittest.main() diff --git a/tests/test_unified_isos.py b/tests/test_unified_isos.py index e333e559..e352d4a5 100755 --- a/tests/test_unified_isos.py +++ b/tests/test_unified_isos.py @@ -1,7 +1,6 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -import json import mock import os import shutil @@ -36,6 +35,14 @@ class TestUnifiedIsos(PungiTestCase): self.assertRegexpMatches(isos.temp_dir, '^%s/' % os.path.join(self.topdir, COMPOSE_ID, 'work')) + def test_dump_manifest(self): + compose_path = os.path.join(self.topdir, COMPOSE_ID, 'compose') + isos = unified_isos.UnifiedISO(compose_path) + isos.compose._images = mock.Mock() + isos.dump_manifest() + self.assertEqual(isos.compose._images.mock_calls, + [mock.call.dump(compose_path + '/metadata/images.json')]) + class TestCreate(PungiTestCase): def setUp(self): @@ -47,7 +54,7 @@ class TestCreate(PungiTestCase): def test_create_method(self): methods = ('link_to_temp', 'createrepo', 'discinfo', 'createiso', - 'link_to_compose', 'update_checksums') + 'link_to_compose', 'update_checksums', 'dump_manifest') for attr in methods: setattr(self.isos, attr, mock.Mock()) @@ -391,25 +398,22 @@ class TestCreateiso(PungiTestCase): self.mkisofs_cmd = self.mkisofs_cmd or mock.Mock(name='mkisofs cmd') return self.mkisofs_cmd - def _img(self, arch, exts): - exts = ['manifest'] + exts + def _img(self, arch): base_path = os.path.join(self.isos.temp_dir, 'iso', arch, u'DP-1.0-20161013.t.4-%s-dvd.iso' % arch) yield base_path - for ext in exts: - yield base_path + '.' + ext + yield base_path + '.manifest' - def _imgs(self, arches, exts): + def _imgs(self, arches): images = {} - exts = [e + 'SUM' for e in exts] for arch in arches: file_arch = arch if arch.startswith('debug-'): file_arch = arch.split('-', 1)[-1] + '-debuginfo' - images[arch] = set(self._img(file_arch if arch != 'src' else 'source', exts)) + images[arch] = set(self._img(file_arch if arch != 'src' else 'source')) return images - def assertResults(self, iso, run, arches, checksums): + def assertResults(self, iso, run, arches): self.assertEqual( run.mock_calls, [mock.call(self.mkisofs_cmd), @@ -417,33 +421,19 @@ class TestCreateiso(PungiTestCase): mock.call(iso.get_manifest_cmd.return_value)] * len(arches) ) - self.assertEqual( - self.isos.images, - self._imgs(arches, checksums), - ) + self.assertEqual(self.isos.images, self._imgs(arches)) - with open(os.path.join(self.compose_path, 'metadata', 'images.json')) as f: - manifest = json.load(f) + images = self.isos.compose.images for v in ('Client', 'Server'): for a in arches: - for image in manifest['payload']['images'][v]['x86_64']: - arch = iso_arch = 'source' if image['arch'] == 'src' else image['arch'] + for image in images[v]['x86_64']: + arch = iso_arch = 'source' if image.arch == 'src' else image.arch if a.startswith('debug-'): iso_arch += '-debuginfo' a = a.split('-', 1)[1] path = '{0}/{1}/iso/DP-1.0-20161013.t.4-{1}-dvd.iso'.format(v, arch, iso_arch) - if image.get('unified', False) and image['arch'] == a and image['path'] == path: - checksum_file_base = os.path.join(self.isos.temp_dir, 'iso', - arch, os.path.basename(image['path'])) - for ch in checksums: - fp = '%s.%sSUM' % (checksum_file_base, ch) - with open(fp) as f: - self.assertEqual( - f.read(), - '%s (%s) = %s\n' % (ch, os.path.basename(image['path']), - CHECKSUMS[ch]) - ) + if image.unified and image.arch == a and image.path == path: break else: self.fail('Image for %s.%s missing' % (v, a)) @@ -460,7 +450,7 @@ class TestCreateiso(PungiTestCase): self.isos.createiso() - self.assertResults(iso, run, ['src', 'x86_64'], ['MD5', 'SHA1', 'SHA256']) + self.assertResults(iso, run, ['src', 'x86_64']) @mock.patch('pungi_utils.unified_isos.iso') @mock.patch('pungi_utils.unified_isos.run') @@ -475,39 +465,7 @@ class TestCreateiso(PungiTestCase): self.isos.createiso() - self.assertResults(iso, run, ['src', 'x86_64', 'debug-x86_64'], ['MD5', 'SHA1', 'SHA256']) - - @mock.patch('pungi_utils.unified_isos.iso') - @mock.patch('pungi_utils.unified_isos.run') - def test_createiso_checksum_one_file(self, run, iso): - iso.get_mkisofs_cmd.side_effect = self.mock_gmc - iso.get_implanted_md5.return_value = 'beefcafebabedeadbeefcafebabedead' - iso.get_volume_id.return_value = 'VOLID' - - self.isos.conf['media_checksum_one_file'] = True - - self.isos.treeinfo = {'x86_64': self.isos.treeinfo['x86_64'], - 'src': self.isos.treeinfo['src']} - - self.isos.createiso() - - self.assertResults(iso, run, ['src', 'x86_64'], []) - - @mock.patch('pungi_utils.unified_isos.iso') - @mock.patch('pungi_utils.unified_isos.run') - def test_createiso_single_checksum(self, run, iso): - iso.get_mkisofs_cmd.side_effect = self.mock_gmc - iso.get_implanted_md5.return_value = 'beefcafebabedeadbeefcafebabedead' - iso.get_volume_id.return_value = 'VOLID' - - self.isos.conf['media_checksums'] = ['sha256'] - - self.isos.treeinfo = {'x86_64': self.isos.treeinfo['x86_64'], - 'src': self.isos.treeinfo['src']} - - self.isos.createiso() - - self.assertResults(iso, run, ['src', 'x86_64'], ['SHA256']) + self.assertResults(iso, run, ['src', 'x86_64', 'debug-x86_64']) class TestLinkToCompose(PungiTestCase): @@ -575,15 +533,9 @@ class TestUpdateChecksums(PungiTestCase): self.isos.update_checksums() self.assertItemsEqual( mmc.call_args_list, - [self._call('Client', 'i386'), - self._call('Client', 'x86_64'), - self._call('Server', 's390x'), - self._call('Server', 'x86_64'), - self._call('Client', 'i386', source=True), - self._call('Client', 'x86_64', source=True), - self._call('Server', 's390x', source=True), - self._call('Server', 'x86_64', source=True)] - ) + [mock.call(self.compose_path, self.isos.compose.images, + unified_isos.DEFAULT_CHECKSUMS, False, + self.isos._get_base_filename)]) @mock.patch('pungi_utils.unified_isos.make_checksums') def test_update_checksums_one_file(self, mmc): @@ -591,28 +543,11 @@ class TestUpdateChecksums(PungiTestCase): self.isos.update_checksums() self.assertItemsEqual( mmc.call_args_list, - [self._call('Client', 'i386', one_file=True), - self._call('Client', 'x86_64', one_file=True), - self._call('Server', 's390x', one_file=True), - self._call('Server', 'x86_64', one_file=True), - self._call('Client', 'i386', source=True, one_file=True), - self._call('Client', 'x86_64', source=True, one_file=True), - self._call('Server', 's390x', source=True, one_file=True), - self._call('Server', 'x86_64', source=True, one_file=True)] - ) + [mock.call(self.compose_path, self.isos.compose.images, + unified_isos.DEFAULT_CHECKSUMS, True, + self.isos._get_base_filename)]) - @mock.patch('pungi_utils.unified_isos.make_checksums') - def test_update_checksums_basename(self, mmc): + def test_get_base_filename(self): self.isos.conf['media_checksum_base_filename'] = '{variant}-{arch}' - self.isos.update_checksums() - self.assertItemsEqual( - mmc.call_args_list, - [self._call('Client', 'i386', basename='Client-i386-'), - self._call('Client', 'x86_64', basename='Client-x86_64-'), - self._call('Server', 's390x', basename='Server-s390x-'), - self._call('Server', 'x86_64', basename='Server-x86_64-'), - self._call('Client', 'i386', source=True, basename='Client-i386-'), - self._call('Client', 'x86_64', source=True, basename='Client-x86_64-'), - self._call('Server', 's390x', source=True, basename='Server-s390x-'), - self._call('Server', 'x86_64', source=True, basename='Server-x86_64-')] - ) + self.assertEqual(self.isos._get_base_filename('Client', 'x86_64'), + 'Client-x86_64-')