almalinux/pungi
15
2
Fork 0
Code Issues Pull Requests Releases Activity

image_checksum: add file sizes to checksum files

Browse Source 
Write file sizes of images in checksum files with comment lines,
checksum files are in BSD-style which supports comments by starting
a line with '#'.

Example:

$ cat RHEL-7.4-20170123.n.4/compose/Server/x86_64/iso/RHEL-Server-7.4-x86_64-20170123.n.4-CHECKSUM
 # RHEL-7.4-20170123.n.4-Server-x86_64-dvd1.iso: 3725590528 bytes
 # RHEL-7.4-20170123.n.4-Server-x86_64-boot.iso: 377487360 bytes
 SHA256 (RHEL-7.4-20170123.n.4-Server-x86_64-dvd1.iso) = fa3de37fe4b859a0285f16ea1123f44f15aec169aea84bf010aa3821bd58fc41
 SHA256 (RHEL-7.4-20170123.n.4-Server-x86_64-boot.iso) = 74bf68c54665328adb08b09daf773c67e633b5907e3e2797338ab3c1b58fdf48

(No space at the start of line, because git commit message drops lines
start with '#', added one space to avoid that.)

When there are multiple checksum types specified and checksums are
written to individual files, file size of the image will also be
written to every checksum files.

Fixes: #493

Signed-off-by: Qixiang Wan <qwan@redhat.com>
This commit is contained in:
Qixiang Wan 2017-01-23 17:54:54 +08:00
parent 4f011fbd45
commit 40df2034a8
2 changed files with 128 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

35
pungi/phases/image_checksum.py
View File

@ -4,7 +4,7 @@ import os
from kobo import shortcuts
from .base import PhaseBase
from ..util import get_format_substs
from ..util import get_format_substs, get_file_size
MULTIPLE_CHECKSUMS_ERROR = (
@ -77,37 +77,60 @@ class ImageChecksumPhase(PhaseBase):
def make_checksums(variant, arch, path, images, checksum_types, base_checksum_name, one_file):
checksums = {}
filesizes = {}
for image in images:
filename = os.path.basename(image.path)
full_path = os.path.join(path, filename)
if not os.path.exists(full_path):
continue
filesize = image.size or get_file_size(full_path)
filesizes[filename] = filesize
digests = shortcuts.compute_file_checksums(full_path, checksum_types)
for checksum, digest in digests.iteritems():
checksums.setdefault(checksum, {})[filename] = digest
image.add_checksum(None, checksum, digest)
if not one_file:
checksum_filename = '%s.%sSUM' % (filename, checksum.upper())
dump_filesizes(path, {filename: filesize}, checksum_filename)
dump_checksums(path, checksum,
{filename: digest},
'%s.%sSUM' % (filename, checksum.upper()))
checksum_filename)
if not checksums:
return
if one_file:
checksum_filename = base_checksum_name + 'CHECKSUM'
dump_filesizes(path, filesizes, checksum_filename)
dump_checksums(path, checksum_types[0],
checksums[checksum_types[0]],
base_checksum_name + 'CHECKSUM')
checksum_filename)
else:
for checksum in checksums:
checksum_filename = '%s%sSUM' % (base_checksum_name, checksum.upper())
dump_filesizes(path, filesizes, checksum_filename)
dump_checksums(path, checksum,
checksums[checksum],
'%s%sSUM' % (base_checksum_name, checksum.upper()))
checksum_filename)
def dump_filesizes(dir, filesizes, filename):
"""Write filesizes to file with comment lines.
:param dir: where to put the file
:param filesizes: mapping from filenames to filesizes
:param filename: what to call the file
"""
filesize_file = os.path.join(dir, filename)
with open(filesize_file, 'a') as f:
for file, filesize in filesizes.iteritems():
f.write('# %s: %s bytes\n' % (file, filesize))
def dump_checksums(dir, alg, checksums, filename):
"""Create file with checksums.
"""Write checksums to file.
:param dir: where to put the file
:param alg: which method was used
@ -115,7 +138,7 @@ def dump_checksums(dir, alg, checksums, filename):
:param filename: what to call the file
"""
checksum_file = os.path.join(dir, filename)
with open(checksum_file, 'w') as f:
with open(checksum_file, 'a') as f:
for file, checksum in checksums.iteritems():
f.write('%s (%s) = %s\n' % (alg.upper(), file, checksum))
return checksum_file

112
tests/test_imagechecksumphase.py
View File

@ -14,7 +14,7 @@ import shutil
sys.path.insert(0, os.path.join(os.path.dirname(__file__), ".."))
from pungi.phases.image_checksum import ImageChecksumPhase, dump_checksums
from pungi.phases.image_checksum import ImageChecksumPhase, dump_checksums, dump_filesizes
from tests.helpers import DummyCompose, PungiTestCase
@ -38,8 +38,9 @@ class TestImageChecksumPhase(PungiTestCase):
@mock.patch('os.path.exists')
@mock.patch('kobo.shortcuts.compute_file_checksums')
@mock.patch('pungi.phases.image_checksum.dump_filesizes')
@mock.patch('pungi.phases.image_checksum.dump_checksums')
def test_checksum_one_file(self, dump, cc, exists):
def test_checksum_one_file(self, dump_checksums, dump_filesizes, cc, exists):
compose = DummyCompose(self.topdir, {
'media_checksums': ['sha256'],
'media_checksum_one_file': True,
@ -52,14 +53,15 @@ class TestImageChecksumPhase(PungiTestCase):
phase.run()
dump.assert_called_once_with(self.topdir + '/compose/Client/i386/iso', 'sha256', {'image.iso': 'cafebabe'}, 'CHECKSUM')
dump_checksums.assert_called_once_with(self.topdir + '/compose/Client/i386/iso', 'sha256', {'image.iso': 'cafebabe'}, 'CHECKSUM')
cc.assert_called_once_with(self.topdir + '/compose/Client/i386/iso/image.iso', ['sha256'])
compose.image.add_checksum.assert_called_once_with(None, 'sha256', 'cafebabe')
@mock.patch('os.path.exists')
@mock.patch('kobo.shortcuts.compute_file_checksums')
@mock.patch('pungi.phases.image_checksum.dump_filesizes')
@mock.patch('pungi.phases.image_checksum.dump_checksums')
def test_checksum_save_individuals(self, dump, cc, exists):
def test_checksum_save_individuals(self, dump_checksums, dump_filesizes, cc, exists):
compose = DummyCompose(self.topdir, {
'media_checksums': ['md5', 'sha256'],
})
@ -71,7 +73,7 @@ class TestImageChecksumPhase(PungiTestCase):
phase.run()
dump.assert_has_calls(
dump_checksums.assert_has_calls(
[mock.call(self.topdir + '/compose/Client/i386/iso', 'md5',
{'image.iso': 'cafebabe'}, 'image.iso.MD5SUM'),
mock.call(self.topdir + '/compose/Client/i386/iso', 'sha256',
@ -89,8 +91,9 @@ class TestImageChecksumPhase(PungiTestCase):
@mock.patch('os.path.exists')
@mock.patch('kobo.shortcuts.compute_file_checksums')
@mock.patch('pungi.phases.image_checksum.dump_filesizes')
@mock.patch('pungi.phases.image_checksum.dump_checksums')
def test_checksum_one_file_custom_name(self, dump, cc, exists):
def test_checksum_one_file_custom_name(self, dump_checksums, dump_filesizes, cc, exists):
compose = DummyCompose(self.topdir, {
'media_checksums': ['sha256'],
'media_checksum_one_file': True,
@ -105,16 +108,17 @@ class TestImageChecksumPhase(PungiTestCase):
phase.run()
dump.assert_called_once_with(self.topdir + '/compose/Client/i386/iso', 'sha256',
{'image.iso': 'cafebabe'},
'test-Client-1.0-20151203.t.0_Alpha-1.0-CHECKSUM')
dump_checksums.assert_called_once_with(self.topdir + '/compose/Client/i386/iso', 'sha256',
{'image.iso': 'cafebabe'},
'test-Client-1.0-20151203.t.0_Alpha-1.0-CHECKSUM')
cc.assert_called_once_with(self.topdir + '/compose/Client/i386/iso/image.iso', ['sha256'])
compose.image.add_checksum.assert_called_once_with(None, 'sha256', 'cafebabe')
@mock.patch('os.path.exists')
@mock.patch('kobo.shortcuts.compute_file_checksums')
@mock.patch('pungi.phases.image_checksum.dump_filesizes')
@mock.patch('pungi.phases.image_checksum.dump_checksums')
def test_checksum_save_individuals_custom_name(self, dump, cc, exists):
def test_checksum_save_individuals_custom_name(self, dump_checksums, dump_filesizes, cc, exists):
compose = DummyCompose(self.topdir, {
'media_checksums': ['md5', 'sha256'],
'media_checksum_base_filename': '%(release_short)s-%(variant)s-%(version)s-%(date)s%(type_suffix)s.%(respin)s'
@ -127,7 +131,7 @@ class TestImageChecksumPhase(PungiTestCase):
phase.run()
dump.assert_has_calls(
dump_checksums.assert_has_calls(
[mock.call(self.topdir + '/compose/Client/i386/iso', 'md5',
{'image.iso': 'cafebabe'}, 'image.iso.MD5SUM'),
mock.call(self.topdir + '/compose/Client/i386/iso', 'sha256',
@ -145,8 +149,9 @@ class TestImageChecksumPhase(PungiTestCase):
@mock.patch('os.path.exists')
@mock.patch('kobo.shortcuts.compute_file_checksums')
@mock.patch('pungi.phases.image_checksum.dump_filesizes')
@mock.patch('pungi.phases.image_checksum.dump_checksums')
def test_checksum_save_individuals_custom_name_str_format(self, dump, cc, exists):
def test_checksum_save_individuals_custom_name_str_format(self, dump_checksums, dump_filesizes, cc, exists):
compose = DummyCompose(self.topdir, {
'media_checksums': ['md5', 'sha256'],
'media_checksum_base_filename': '{release_short}-{variant}-{version}-{date}{type_suffix}.{respin}'
@ -159,7 +164,7 @@ class TestImageChecksumPhase(PungiTestCase):
phase.run()
dump.assert_has_calls(
dump_checksums.assert_has_calls(
[mock.call(self.topdir + '/compose/Client/i386/iso', 'md5',
{'image.iso': 'cafebabe'}, 'image.iso.MD5SUM'),
mock.call(self.topdir + '/compose/Client/i386/iso', 'sha256',
@ -175,6 +180,65 @@ class TestImageChecksumPhase(PungiTestCase):
mock.call(None, 'md5', 'cafebabe')],
any_order=True)
@mock.patch('os.path.exists')
@mock.patch('kobo.shortcuts.compute_file_checksums')
@mock.patch('pungi.phases.image_checksum.get_file_size')
@mock.patch('pungi.phases.image_checksum.dump_filesizes')
@mock.patch('pungi.phases.image_checksum.dump_checksums')
def test_dump_filesizes_one_file(self, dump_checksums, dump_filesizes, get_file_size, cc, exists):
compose = DummyCompose(self.topdir, {
'media_checksums': ['md5', 'sha256'],
'media_checksum_base_filename': '{release_short}-{variant}-{version}-{date}{type_suffix}.{respin}',
'media_checksum_one_file': True
})
compose.image.size = None
get_file_size.return_value = '12345'
phase = ImageChecksumPhase(compose)
exists.return_value = True
cc.return_value = {'md5': 'cafebabe', 'sha256': 'deadbeef'}
phase.run()
dump_filesizes.assert_called_once_with(
self.topdir + '/compose/Client/i386/iso', {'image.iso': '12345'}, 'test-Client-1.0-20151203.t.0-CHECKSUM')
get_file_size.assert_called_once_with(self.topdir + '/compose/Client/i386/iso/image.iso')
@mock.patch('os.path.exists')
@mock.patch('kobo.shortcuts.compute_file_checksums')
@mock.patch('pungi.phases.image_checksum.get_file_size')
@mock.patch('pungi.phases.image_checksum.dump_filesizes')
@mock.patch('pungi.phases.image_checksum.dump_checksums')
def test_dump_filesizes_save_individuals(self, dump_checksums, dump_filesizes, get_file_size, cc, exists):
compose = DummyCompose(self.topdir, {
'media_checksums': ['md5', 'sha256'],
'media_checksum_base_filename': '{release_short}-{variant}-{version}-{date}{type_suffix}.{respin}'
})
compose.image.size = None
get_file_size.return_value = '12345'
phase = ImageChecksumPhase(compose)
exists.return_value = True
cc.return_value = {'md5': 'cafebabe', 'sha256': 'deadbeef'}
phase.run()
dump_filesizes.assert_has_calls(
[mock.call(self.topdir + '/compose/Client/i386/iso',
{'image.iso': '12345'}, 'image.iso.SHA256SUM'),
mock.call(self.topdir + '/compose/Client/i386/iso',
{'image.iso': '12345'}, 'image.iso.MD5SUM'),
mock.call(self.topdir + '/compose/Client/i386/iso',
{'image.iso': '12345'}, 'test-Client-1.0-20151203.t.0-SHA256SUM'),
mock.call(self.topdir + '/compose/Client/i386/iso',
{'image.iso': '12345'}, 'test-Client-1.0-20151203.t.0-MD5SUM')],
any_order=True
)
get_file_size.assert_called_once_with(self.topdir + '/compose/Client/i386/iso/image.iso')
class TestChecksums(unittest.TestCase):
def setUp(self):
@ -198,5 +262,27 @@ class TestChecksums(unittest.TestCase):
]
self.assertItemsEqual(expected, data)
class TestDumpFilesizes(unittest.TestCase):
def setUp(self):
_, name = tempfile.mkstemp()
self.tmp_dir = tempfile.mkdtemp()
def tearDown(self):
shutil.rmtree(self.tmp_dir)
def test_dump_files(self):
filesizes = {'file1.iso': 123,
'file2.iso': 456}
dump_filesizes(self.tmp_dir, filesizes, 'CHECKSUM')
with open(os.path.join(self.tmp_dir, 'CHECKSUM'), 'r') as f:
data = f.read().rstrip().split('\n')
expected = [
'# file1.iso: 123 bytes',
'# file2.iso: 456 bytes',
]
self.assertItemsEqual(expected, data)
if __name__ == "__main__":
unittest.main()