diff --git a/pungi/phases/image_checksum.py b/pungi/phases/image_checksum.py index 1e665007..111ef924 100644 --- a/pungi/phases/image_checksum.py +++ b/pungi/phases/image_checksum.py @@ -4,7 +4,7 @@ import os from kobo import shortcuts from .base import PhaseBase -from ..util import get_format_substs +from ..util import get_format_substs, get_file_size MULTIPLE_CHECKSUMS_ERROR = ( @@ -77,37 +77,60 @@ class ImageChecksumPhase(PhaseBase): def make_checksums(variant, arch, path, images, checksum_types, base_checksum_name, one_file): checksums = {} + filesizes = {} for image in images: filename = os.path.basename(image.path) full_path = os.path.join(path, filename) if not os.path.exists(full_path): continue + filesize = image.size or get_file_size(full_path) + filesizes[filename] = filesize + digests = shortcuts.compute_file_checksums(full_path, checksum_types) for checksum, digest in digests.iteritems(): checksums.setdefault(checksum, {})[filename] = digest image.add_checksum(None, checksum, digest) if not one_file: + checksum_filename = '%s.%sSUM' % (filename, checksum.upper()) + dump_filesizes(path, {filename: filesize}, checksum_filename) dump_checksums(path, checksum, {filename: digest}, - '%s.%sSUM' % (filename, checksum.upper())) + checksum_filename) if not checksums: return if one_file: + checksum_filename = base_checksum_name + 'CHECKSUM' + dump_filesizes(path, filesizes, checksum_filename) dump_checksums(path, checksum_types[0], checksums[checksum_types[0]], - base_checksum_name + 'CHECKSUM') + checksum_filename) else: for checksum in checksums: + checksum_filename = '%s%sSUM' % (base_checksum_name, checksum.upper()) + dump_filesizes(path, filesizes, checksum_filename) dump_checksums(path, checksum, checksums[checksum], - '%s%sSUM' % (base_checksum_name, checksum.upper())) + checksum_filename) + + +def dump_filesizes(dir, filesizes, filename): + """Write filesizes to file with comment lines. + + :param dir: where to put the file + :param filesizes: mapping from filenames to filesizes + :param filename: what to call the file + """ + filesize_file = os.path.join(dir, filename) + with open(filesize_file, 'a') as f: + for file, filesize in filesizes.iteritems(): + f.write('# %s: %s bytes\n' % (file, filesize)) def dump_checksums(dir, alg, checksums, filename): - """Create file with checksums. + """Write checksums to file. :param dir: where to put the file :param alg: which method was used @@ -115,7 +138,7 @@ def dump_checksums(dir, alg, checksums, filename): :param filename: what to call the file """ checksum_file = os.path.join(dir, filename) - with open(checksum_file, 'w') as f: + with open(checksum_file, 'a') as f: for file, checksum in checksums.iteritems(): f.write('%s (%s) = %s\n' % (alg.upper(), file, checksum)) return checksum_file diff --git a/tests/test_imagechecksumphase.py b/tests/test_imagechecksumphase.py index d3faa008..7a395647 100644 --- a/tests/test_imagechecksumphase.py +++ b/tests/test_imagechecksumphase.py @@ -14,7 +14,7 @@ import shutil sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..")) -from pungi.phases.image_checksum import ImageChecksumPhase, dump_checksums +from pungi.phases.image_checksum import ImageChecksumPhase, dump_checksums, dump_filesizes from tests.helpers import DummyCompose, PungiTestCase @@ -38,8 +38,9 @@ class TestImageChecksumPhase(PungiTestCase): @mock.patch('os.path.exists') @mock.patch('kobo.shortcuts.compute_file_checksums') + @mock.patch('pungi.phases.image_checksum.dump_filesizes') @mock.patch('pungi.phases.image_checksum.dump_checksums') - def test_checksum_one_file(self, dump, cc, exists): + def test_checksum_one_file(self, dump_checksums, dump_filesizes, cc, exists): compose = DummyCompose(self.topdir, { 'media_checksums': ['sha256'], 'media_checksum_one_file': True, @@ -52,14 +53,15 @@ class TestImageChecksumPhase(PungiTestCase): phase.run() - dump.assert_called_once_with(self.topdir + '/compose/Client/i386/iso', 'sha256', {'image.iso': 'cafebabe'}, 'CHECKSUM') + dump_checksums.assert_called_once_with(self.topdir + '/compose/Client/i386/iso', 'sha256', {'image.iso': 'cafebabe'}, 'CHECKSUM') cc.assert_called_once_with(self.topdir + '/compose/Client/i386/iso/image.iso', ['sha256']) compose.image.add_checksum.assert_called_once_with(None, 'sha256', 'cafebabe') @mock.patch('os.path.exists') @mock.patch('kobo.shortcuts.compute_file_checksums') + @mock.patch('pungi.phases.image_checksum.dump_filesizes') @mock.patch('pungi.phases.image_checksum.dump_checksums') - def test_checksum_save_individuals(self, dump, cc, exists): + def test_checksum_save_individuals(self, dump_checksums, dump_filesizes, cc, exists): compose = DummyCompose(self.topdir, { 'media_checksums': ['md5', 'sha256'], }) @@ -71,7 +73,7 @@ class TestImageChecksumPhase(PungiTestCase): phase.run() - dump.assert_has_calls( + dump_checksums.assert_has_calls( [mock.call(self.topdir + '/compose/Client/i386/iso', 'md5', {'image.iso': 'cafebabe'}, 'image.iso.MD5SUM'), mock.call(self.topdir + '/compose/Client/i386/iso', 'sha256', @@ -89,8 +91,9 @@ class TestImageChecksumPhase(PungiTestCase): @mock.patch('os.path.exists') @mock.patch('kobo.shortcuts.compute_file_checksums') + @mock.patch('pungi.phases.image_checksum.dump_filesizes') @mock.patch('pungi.phases.image_checksum.dump_checksums') - def test_checksum_one_file_custom_name(self, dump, cc, exists): + def test_checksum_one_file_custom_name(self, dump_checksums, dump_filesizes, cc, exists): compose = DummyCompose(self.topdir, { 'media_checksums': ['sha256'], 'media_checksum_one_file': True, @@ -105,16 +108,17 @@ class TestImageChecksumPhase(PungiTestCase): phase.run() - dump.assert_called_once_with(self.topdir + '/compose/Client/i386/iso', 'sha256', - {'image.iso': 'cafebabe'}, - 'test-Client-1.0-20151203.t.0_Alpha-1.0-CHECKSUM') + dump_checksums.assert_called_once_with(self.topdir + '/compose/Client/i386/iso', 'sha256', + {'image.iso': 'cafebabe'}, + 'test-Client-1.0-20151203.t.0_Alpha-1.0-CHECKSUM') cc.assert_called_once_with(self.topdir + '/compose/Client/i386/iso/image.iso', ['sha256']) compose.image.add_checksum.assert_called_once_with(None, 'sha256', 'cafebabe') @mock.patch('os.path.exists') @mock.patch('kobo.shortcuts.compute_file_checksums') + @mock.patch('pungi.phases.image_checksum.dump_filesizes') @mock.patch('pungi.phases.image_checksum.dump_checksums') - def test_checksum_save_individuals_custom_name(self, dump, cc, exists): + def test_checksum_save_individuals_custom_name(self, dump_checksums, dump_filesizes, cc, exists): compose = DummyCompose(self.topdir, { 'media_checksums': ['md5', 'sha256'], 'media_checksum_base_filename': '%(release_short)s-%(variant)s-%(version)s-%(date)s%(type_suffix)s.%(respin)s' @@ -127,7 +131,7 @@ class TestImageChecksumPhase(PungiTestCase): phase.run() - dump.assert_has_calls( + dump_checksums.assert_has_calls( [mock.call(self.topdir + '/compose/Client/i386/iso', 'md5', {'image.iso': 'cafebabe'}, 'image.iso.MD5SUM'), mock.call(self.topdir + '/compose/Client/i386/iso', 'sha256', @@ -145,8 +149,9 @@ class TestImageChecksumPhase(PungiTestCase): @mock.patch('os.path.exists') @mock.patch('kobo.shortcuts.compute_file_checksums') + @mock.patch('pungi.phases.image_checksum.dump_filesizes') @mock.patch('pungi.phases.image_checksum.dump_checksums') - def test_checksum_save_individuals_custom_name_str_format(self, dump, cc, exists): + def test_checksum_save_individuals_custom_name_str_format(self, dump_checksums, dump_filesizes, cc, exists): compose = DummyCompose(self.topdir, { 'media_checksums': ['md5', 'sha256'], 'media_checksum_base_filename': '{release_short}-{variant}-{version}-{date}{type_suffix}.{respin}' @@ -159,7 +164,7 @@ class TestImageChecksumPhase(PungiTestCase): phase.run() - dump.assert_has_calls( + dump_checksums.assert_has_calls( [mock.call(self.topdir + '/compose/Client/i386/iso', 'md5', {'image.iso': 'cafebabe'}, 'image.iso.MD5SUM'), mock.call(self.topdir + '/compose/Client/i386/iso', 'sha256', @@ -175,6 +180,65 @@ class TestImageChecksumPhase(PungiTestCase): mock.call(None, 'md5', 'cafebabe')], any_order=True) + @mock.patch('os.path.exists') + @mock.patch('kobo.shortcuts.compute_file_checksums') + @mock.patch('pungi.phases.image_checksum.get_file_size') + @mock.patch('pungi.phases.image_checksum.dump_filesizes') + @mock.patch('pungi.phases.image_checksum.dump_checksums') + def test_dump_filesizes_one_file(self, dump_checksums, dump_filesizes, get_file_size, cc, exists): + compose = DummyCompose(self.topdir, { + 'media_checksums': ['md5', 'sha256'], + 'media_checksum_base_filename': '{release_short}-{variant}-{version}-{date}{type_suffix}.{respin}', + 'media_checksum_one_file': True + }) + compose.image.size = None + get_file_size.return_value = '12345' + + phase = ImageChecksumPhase(compose) + + exists.return_value = True + cc.return_value = {'md5': 'cafebabe', 'sha256': 'deadbeef'} + + phase.run() + + dump_filesizes.assert_called_once_with( + self.topdir + '/compose/Client/i386/iso', {'image.iso': '12345'}, 'test-Client-1.0-20151203.t.0-CHECKSUM') + get_file_size.assert_called_once_with(self.topdir + '/compose/Client/i386/iso/image.iso') + + @mock.patch('os.path.exists') + @mock.patch('kobo.shortcuts.compute_file_checksums') + @mock.patch('pungi.phases.image_checksum.get_file_size') + @mock.patch('pungi.phases.image_checksum.dump_filesizes') + @mock.patch('pungi.phases.image_checksum.dump_checksums') + def test_dump_filesizes_save_individuals(self, dump_checksums, dump_filesizes, get_file_size, cc, exists): + compose = DummyCompose(self.topdir, { + 'media_checksums': ['md5', 'sha256'], + 'media_checksum_base_filename': '{release_short}-{variant}-{version}-{date}{type_suffix}.{respin}' + }) + compose.image.size = None + get_file_size.return_value = '12345' + + phase = ImageChecksumPhase(compose) + + exists.return_value = True + cc.return_value = {'md5': 'cafebabe', 'sha256': 'deadbeef'} + + phase.run() + + dump_filesizes.assert_has_calls( + [mock.call(self.topdir + '/compose/Client/i386/iso', + {'image.iso': '12345'}, 'image.iso.SHA256SUM'), + mock.call(self.topdir + '/compose/Client/i386/iso', + {'image.iso': '12345'}, 'image.iso.MD5SUM'), + mock.call(self.topdir + '/compose/Client/i386/iso', + {'image.iso': '12345'}, 'test-Client-1.0-20151203.t.0-SHA256SUM'), + mock.call(self.topdir + '/compose/Client/i386/iso', + {'image.iso': '12345'}, 'test-Client-1.0-20151203.t.0-MD5SUM')], + any_order=True + ) + + get_file_size.assert_called_once_with(self.topdir + '/compose/Client/i386/iso/image.iso') + class TestChecksums(unittest.TestCase): def setUp(self): @@ -198,5 +262,27 @@ class TestChecksums(unittest.TestCase): ] self.assertItemsEqual(expected, data) + +class TestDumpFilesizes(unittest.TestCase): + def setUp(self): + _, name = tempfile.mkstemp() + self.tmp_dir = tempfile.mkdtemp() + + def tearDown(self): + shutil.rmtree(self.tmp_dir) + + def test_dump_files(self): + filesizes = {'file1.iso': 123, + 'file2.iso': 456} + dump_filesizes(self.tmp_dir, filesizes, 'CHECKSUM') + + with open(os.path.join(self.tmp_dir, 'CHECKSUM'), 'r') as f: + data = f.read().rstrip().split('\n') + expected = [ + '# file1.iso: 123 bytes', + '# file2.iso: 456 bytes', + ] + self.assertItemsEqual(expected, data) + if __name__ == "__main__": unittest.main()