checksum: Refactor creating checksum files

Instead of iterating over the images metadata and appending the checksum
to relevant files immediately, we should store them and write only once.

This avoid an issue when the same image is mentioned in the metadata
multiple times. This happens for source images that are listed under
each binary arch.

The unified isos script is updated to use the exact same logic and code.
This also uncovered a problem with the metadata for debuginfo unified
isos: their paths in metadata were incorrect, which lead to missing
checksums.

Fixes: https://pagure.io/pungi/issue/667
Fixes: https://pagure.io/pungi/issue/668
Signed-off-by: Lubomír Sedlář <lsedlar@redhat.com>

pungi/phases/image_checksum.py

@@ -2,6 +2,7 @@
 
 import os
 from kobo import shortcuts
+from collections import defaultdict
 
 from .base import PhaseBase
 from ..util import get_format_substs, get_file_size
@@ -69,15 +70,10 @@ class ImageChecksumPhase(PhaseBase):
 
     def run(self):
         topdir = self.compose.paths.compose.topdir()
-        for (variant, arch, path), images in get_images(topdir, self.compose.im).iteritems():
+        make_checksums(topdir, self.compose.im, self.checksums, self.one_file, self._get_base_filename)
-            base_checksum_name = self._get_base_filename(variant, arch)
-            make_checksums(variant, arch, path, images,
-                           self.checksums, base_checksum_name, self.one_file)
 
 
-def make_checksums(variant, arch, path, images, checksum_types, base_checksum_name, one_file):
+def _compute_checksums(results, variant, arch, path, images, checksum_types, base_checksum_name, one_file):
-    checksums = {}
-    filesizes = {}
     for image in images:
         filename = os.path.basename(image.path)
         full_path = os.path.join(path, filename)
@@ -85,63 +81,45 @@ def make_checksums(variant, arch, path, images, checksum_types, base_checksum_na
             continue
 
         filesize = image.size or get_file_size(full_path)
-        filesizes[filename] = filesize
 
         digests = shortcuts.compute_file_checksums(full_path, checksum_types)
         for checksum, digest in digests.iteritems():
-            checksums.setdefault(checksum, {})[filename] = digest
+            # Update metadata with the checksum
             image.add_checksum(None, checksum, digest)
+            # If not turned of, create the file-specific checksum file
             if not one_file:
-                checksum_filename = '%s.%sSUM' % (filename, checksum.upper())
+                checksum_filename = os.path.join(path, '%s.%sSUM' % (filename, checksum.upper()))
-                dump_filesizes(path, {filename: filesize}, checksum_filename)
+                results[checksum_filename].add((filename, filesize, checksum, digest))
-                dump_checksums(path, checksum,
-                               {filename: digest},
-                               checksum_filename)
 
-    if not checksums:
+            if one_file:
-        return
+                checksum_filename = os.path.join(path, base_checksum_name + 'CHECKSUM')
+            else:
+                checksum_filename = os.path.join(path, '%s%sSUM' % (base_checksum_name, checksum.upper()))
 
-    if one_file:
+            results[checksum_filename].add((filename, filesize, checksum, digest))
-        checksum_filename = base_checksum_name + 'CHECKSUM'
-        dump_filesizes(path, filesizes, checksum_filename)
-        dump_checksums(path, checksum_types[0],
-                       checksums[checksum_types[0]],
-                       checksum_filename)
-    else:
-        for checksum in checksums:
-            checksum_filename = '%s%sSUM' % (base_checksum_name, checksum.upper())
-            dump_filesizes(path, filesizes, checksum_filename)
-            dump_checksums(path, checksum,
-                           checksums[checksum],
-                           checksum_filename)
 
 
-def dump_filesizes(dir, filesizes, filename):
+def make_checksums(topdir, im, checksum_types, one_file, base_checksum_name_gen):
-    """Write filesizes to file with comment lines.
+    results = defaultdict(set)
+    for (variant, arch, path), images in get_images(topdir, im).iteritems():
+        base_checksum_name = base_checksum_name_gen(variant, arch)
+        _compute_checksums(results, variant, arch, path, images,
+                           checksum_types, base_checksum_name, one_file)
 
-    :param dir: where to put the file
+    for file in results:
-    :param filesizes: mapping from filenames to filesizes
+        dump_checksums(file, results[file])
-    :param filename: what to call the file
-    """
-    filesize_file = os.path.join(dir, filename)
-    with open(filesize_file, 'a') as f:
-        for file, filesize in filesizes.iteritems():
-            f.write('# %s: %s bytes\n' % (file, filesize))
 
 
-def dump_checksums(dir, alg, checksums, filename):
+def dump_checksums(checksum_file, data):
     """Write checksums to file.
 
-    :param dir: where to put the file
+    :param checksum_file: where to write the checksums
-    :param alg: which method was used
+    :param data: an iterable of tuples (filename, filesize, checksum_type, hash)
-    :param checksums: mapping from filenames to checksums
-    :param filename: what to call the file
     """
-    checksum_file = os.path.join(dir, filename)
+    with open(checksum_file, 'w') as f:
-    with open(checksum_file, 'a') as f:
+        for filename, filesize, alg, checksum in sorted(data):
-        for file, checksum in checksums.iteritems():
+            f.write('# %s: %s bytes\n' % (filename, filesize))
-            f.write('%s (%s) = %s\n' % (alg.upper(), file, checksum))
+            f.write('%s (%s) = %s\n' % (alg.upper(), filename, checksum))
-    return checksum_file
 
 
 def get_images(top_dir, manifest):

pungi_utils/unified_isos.py

@@ -27,14 +27,14 @@ import productmd
 import productmd.compose
 import productmd.images
 import productmd.treeinfo
-from kobo.shortcuts import run, compute_file_checksums
+from kobo.shortcuts import run
 
 import pungi.linker
 import pungi.wrappers.createrepo
 from pungi.util import makedirs
 from pungi.compose_metadata.discinfo import write_discinfo as create_discinfo
 from pungi.wrappers import iso
-from pungi.phases.image_checksum import dump_checksums, get_images, make_checksums
+from pungi.phases.image_checksum import make_checksums
 
 
 def ti_merge(one, two):
@@ -88,10 +88,14 @@ class UnifiedISO(object):
             self.createiso()
             self.link_to_compose()
             self.update_checksums()
+            self.dump_manifest()
         finally:
             if delete_temp:
                 shutil.rmtree(self.temp_dir)
 
+    def dump_manifest(self):
+        self.compose.images.dump(os.path.join(self.compose_path, 'metadata', 'images.json'))
+
     def _link_tree(self, dir, variant, arch):
         blacklist_files = [".treeinfo", ".discinfo", "boot.iso", "media.repo", "extra_files.json"]
         blacklist_dirs = ["repodata"]
@@ -308,9 +312,6 @@ class UnifiedISO(object):
             supported = True
             run(iso.get_implantisomd5_cmd(iso_path, supported))
 
-            checksums = compute_file_checksums(
-                iso_path, self.conf.get('media_checksums', DEFAULT_CHECKSUMS))
-
             # write manifest file
             run(iso.get_manifest_cmd(iso_path))
 
@@ -332,15 +333,6 @@ class UnifiedISO(object):
             self.images.setdefault(typed_arch, set()).add(iso_path)
             self.images.setdefault(typed_arch, set()).add(iso_path + ".manifest")
 
-            for checksum_type, checksum in checksums.iteritems():
-                if not self.conf.get('media_checksum_one_file', False):
-                    checksum_path = dump_checksums(iso_dir, checksum_type,
-                                                   {iso_name: checksum},
-                                                   '%s.%sSUM' % (iso_name, checksum_type.upper()))
-                    self.images.setdefault(typed_arch, set()).add(checksum_path)
-
-                img.add_checksum(self.compose_path, checksum_type=checksum_type, checksum_value=checksum)
-
             img.implant_md5 = iso.get_implanted_md5(iso_path)
             try:
                 img.volume_id = iso.get_volume_id(iso_path)
@@ -365,14 +357,16 @@ class UnifiedISO(object):
                     variant_img.subvariant = variant.id
                     paths_attr = 'isos' if arch != 'src' else 'source_isos'
                     paths = getattr(self.ci.variants[variant.uid].paths, paths_attr)
+                    path = paths.get(tree_arch, os.path.join(variant.uid, tree_arch, "iso"))
+                    if variant_img.type == 'dvd-debuginfo':
+                        prefix, isodir = path.rsplit('/', 1)
+                        path = os.path.join(prefix, 'debug', isodir)
                     variant_img.path = os.path.join(
-                        paths.get(tree_arch, os.path.join(variant.uid, tree_arch, "iso")),
+                        path,
                         os.path.basename(img.path)
                     )
                     im.add(variant.uid, tree_arch, variant_img)
 
-        im.dump(os.path.join(self.compose_path, 'metadata', 'images.json'))
-
     def link_to_compose(self):
         for variant in self.ci.get_variants(recursive=False):
             for arch in variant.arches | set(['debug-' + a for a in variant.arches]) | set(['src']):
@@ -412,9 +406,7 @@ class UnifiedISO(object):
         return base_name
 
     def update_checksums(self):
-        for (variant, arch, path), images in get_images(self.compose_path, self.compose.images).iteritems():
+        make_checksums(self.compose_path, self.compose.images,
-            base_checksum_name = self._get_base_filename(variant, arch)
+                       self.conf.get('media_checksums', DEFAULT_CHECKSUMS),
-            make_checksums(variant, arch, path, images,
+                       self.conf.get('media_checksum_one_file', False),
-                           self.conf.get('media_checksums', DEFAULT_CHECKSUMS),
+                       self._get_base_filename)
-                           base_checksum_name,
-                           self.conf.get('media_checksum_one_file', False))

tests/helpers.py

@@ -94,7 +94,7 @@ class DummyCompose(object):
         self.log_debug = mock.Mock()
         self.log_warning = mock.Mock()
         self.get_image_name = mock.Mock(return_value='image-name')
-        self.image = mock.Mock(path='Client/i386/iso/image.iso', can_fail=False)
+        self.image = mock.Mock(path='Client/i386/iso/image.iso', can_fail=False, size=123)
         self.im = mock.Mock(images={'Client': {'amd64': [self.image]}})
         self.old_composes = []
         self.config_dir = '/home/releng/config'

tests/test_imagechecksumphase.py

@@ -14,7 +14,7 @@ import shutil
 
 sys.path.insert(0, os.path.join(os.path.dirname(__file__), ".."))
 
-from pungi.phases.image_checksum import ImageChecksumPhase, dump_checksums, dump_filesizes
+from pungi.phases.image_checksum import ImageChecksumPhase, dump_checksums
 from tests.helpers import DummyCompose, PungiTestCase
 
 
@@ -38,9 +38,8 @@ class TestImageChecksumPhase(PungiTestCase):
 
     @mock.patch('os.path.exists')
     @mock.patch('kobo.shortcuts.compute_file_checksums')
-    @mock.patch('pungi.phases.image_checksum.dump_filesizes')
     @mock.patch('pungi.phases.image_checksum.dump_checksums')
-    def test_checksum_one_file(self, dump_checksums, dump_filesizes, cc, exists):
+    def test_checksum_one_file(self, dump_checksums, cc, exists):
         compose = DummyCompose(self.topdir, {
             'media_checksums': ['sha256'],
             'media_checksum_one_file': True,
@@ -53,15 +52,15 @@ class TestImageChecksumPhase(PungiTestCase):
 
         phase.run()
 
-        dump_checksums.assert_called_once_with(self.topdir + '/compose/Client/i386/iso', 'sha256', {'image.iso': 'cafebabe'}, 'CHECKSUM')
+        dump_checksums.assert_called_once_with(self.topdir + '/compose/Client/i386/iso/CHECKSUM',
+                                               set([('image.iso', 123, 'sha256', 'cafebabe')]))
         cc.assert_called_once_with(self.topdir + '/compose/Client/i386/iso/image.iso', ['sha256'])
         compose.image.add_checksum.assert_called_once_with(None, 'sha256', 'cafebabe')
 
     @mock.patch('os.path.exists')
     @mock.patch('kobo.shortcuts.compute_file_checksums')
-    @mock.patch('pungi.phases.image_checksum.dump_filesizes')
     @mock.patch('pungi.phases.image_checksum.dump_checksums')
-    def test_checksum_save_individuals(self, dump_checksums, dump_filesizes, cc, exists):
+    def test_checksum_save_individuals(self, dump_checksums, cc, exists):
         compose = DummyCompose(self.topdir, {
             'media_checksums': ['md5', 'sha256'],
         })
@@ -74,14 +73,14 @@ class TestImageChecksumPhase(PungiTestCase):
         phase.run()
 
         dump_checksums.assert_has_calls(
-            [mock.call(self.topdir + '/compose/Client/i386/iso', 'md5',
+            [mock.call(self.topdir + '/compose/Client/i386/iso/image.iso.MD5SUM',
-                       {'image.iso': 'cafebabe'}, 'image.iso.MD5SUM'),
+                       set([('image.iso', 123, 'md5', 'cafebabe')])),
-             mock.call(self.topdir + '/compose/Client/i386/iso', 'sha256',
+             mock.call(self.topdir + '/compose/Client/i386/iso/image.iso.SHA256SUM',
-                       {'image.iso': 'deadbeef'}, 'image.iso.SHA256SUM'),
+                       set([('image.iso', 123, 'sha256', 'deadbeef')])),
-             mock.call(self.topdir + '/compose/Client/i386/iso', 'md5',
+             mock.call(self.topdir + '/compose/Client/i386/iso/MD5SUM',
-                       {'image.iso': 'cafebabe'}, 'MD5SUM'),
+                       set([('image.iso', 123, 'md5', 'cafebabe')])),
-             mock.call(self.topdir + '/compose/Client/i386/iso', 'sha256',
+             mock.call(self.topdir + '/compose/Client/i386/iso/SHA256SUM',
-                       {'image.iso': 'deadbeef'}, 'SHA256SUM')],
+                       set([('image.iso', 123, 'sha256', 'deadbeef')]))],
             any_order=True
         )
         cc.assert_called_once_with(self.topdir + '/compose/Client/i386/iso/image.iso', ['md5', 'sha256'])
@@ -91,9 +90,8 @@ class TestImageChecksumPhase(PungiTestCase):
 
     @mock.patch('os.path.exists')
     @mock.patch('kobo.shortcuts.compute_file_checksums')
-    @mock.patch('pungi.phases.image_checksum.dump_filesizes')
     @mock.patch('pungi.phases.image_checksum.dump_checksums')
-    def test_checksum_one_file_custom_name(self, dump_checksums, dump_filesizes, cc, exists):
+    def test_checksum_one_file_custom_name(self, dump_checksums, cc, exists):
         compose = DummyCompose(self.topdir, {
             'media_checksums': ['sha256'],
             'media_checksum_one_file': True,
@@ -108,17 +106,16 @@ class TestImageChecksumPhase(PungiTestCase):
 
         phase.run()
 
-        dump_checksums.assert_called_once_with(self.topdir + '/compose/Client/i386/iso', 'sha256',
+        dump_checksums.assert_called_once_with(
-                                               {'image.iso': 'cafebabe'},
+            self.topdir + '/compose/Client/i386/iso/test-Client-1.0-20151203.t.0_Alpha-1.0-CHECKSUM',
-                                               'test-Client-1.0-20151203.t.0_Alpha-1.0-CHECKSUM')
+            set([('image.iso', 123, 'sha256', 'cafebabe')]))
         cc.assert_called_once_with(self.topdir + '/compose/Client/i386/iso/image.iso', ['sha256'])
         compose.image.add_checksum.assert_called_once_with(None, 'sha256', 'cafebabe')
 
     @mock.patch('os.path.exists')
     @mock.patch('kobo.shortcuts.compute_file_checksums')
-    @mock.patch('pungi.phases.image_checksum.dump_filesizes')
     @mock.patch('pungi.phases.image_checksum.dump_checksums')
-    def test_checksum_save_individuals_custom_name(self, dump_checksums, dump_filesizes, cc, exists):
+    def test_checksum_save_individuals_custom_name(self, dump_checksums, cc, exists):
         compose = DummyCompose(self.topdir, {
             'media_checksums': ['md5', 'sha256'],
             'media_checksum_base_filename': '%(release_short)s-%(variant)s-%(version)s-%(date)s%(type_suffix)s.%(respin)s'
@@ -132,14 +129,14 @@ class TestImageChecksumPhase(PungiTestCase):
         phase.run()
 
         dump_checksums.assert_has_calls(
-            [mock.call(self.topdir + '/compose/Client/i386/iso', 'md5',
+            [mock.call(self.topdir + '/compose/Client/i386/iso/image.iso.MD5SUM',
-                       {'image.iso': 'cafebabe'}, 'image.iso.MD5SUM'),
+                       set([('image.iso', 123, 'md5', 'cafebabe')])),
-             mock.call(self.topdir + '/compose/Client/i386/iso', 'sha256',
+             mock.call(self.topdir + '/compose/Client/i386/iso/image.iso.SHA256SUM',
-                       {'image.iso': 'deadbeef'}, 'image.iso.SHA256SUM'),
+                       set([('image.iso', 123, 'sha256', 'deadbeef')])),
-             mock.call(self.topdir + '/compose/Client/i386/iso', 'md5', {'image.iso': 'cafebabe'},
+             mock.call(self.topdir + '/compose/Client/i386/iso/test-Client-1.0-20151203.t.0-MD5SUM',
-                       'test-Client-1.0-20151203.t.0-MD5SUM'),
+                       set([('image.iso', 123, 'md5', 'cafebabe')])),
-             mock.call(self.topdir + '/compose/Client/i386/iso', 'sha256', {'image.iso': 'deadbeef'},
+             mock.call(self.topdir + '/compose/Client/i386/iso/test-Client-1.0-20151203.t.0-SHA256SUM',
-                       'test-Client-1.0-20151203.t.0-SHA256SUM')],
+                       set([('image.iso', 123, 'sha256', 'deadbeef')]))],
             any_order=True
         )
         cc.assert_called_once_with(self.topdir + '/compose/Client/i386/iso/image.iso', ['md5', 'sha256'])
@@ -149,9 +146,8 @@ class TestImageChecksumPhase(PungiTestCase):
 
     @mock.patch('os.path.exists')
     @mock.patch('kobo.shortcuts.compute_file_checksums')
-    @mock.patch('pungi.phases.image_checksum.dump_filesizes')
     @mock.patch('pungi.phases.image_checksum.dump_checksums')
-    def test_checksum_save_individuals_custom_name_str_format(self, dump_checksums, dump_filesizes, cc, exists):
+    def test_checksum_save_individuals_custom_name_str_format(self, dump_checksums, cc, exists):
         compose = DummyCompose(self.topdir, {
             'media_checksums': ['md5', 'sha256'],
             'media_checksum_base_filename': '{release_short}-{variant}-{version}-{date}{type_suffix}.{respin}'
@@ -165,14 +161,14 @@ class TestImageChecksumPhase(PungiTestCase):
         phase.run()
 
         dump_checksums.assert_has_calls(
-            [mock.call(self.topdir + '/compose/Client/i386/iso', 'md5',
+            [mock.call(self.topdir + '/compose/Client/i386/iso/image.iso.MD5SUM',
-                       {'image.iso': 'cafebabe'}, 'image.iso.MD5SUM'),
+                       set([('image.iso', 123, 'md5', 'cafebabe')])),
-             mock.call(self.topdir + '/compose/Client/i386/iso', 'sha256',
+             mock.call(self.topdir + '/compose/Client/i386/iso/image.iso.SHA256SUM',
-                       {'image.iso': 'deadbeef'}, 'image.iso.SHA256SUM'),
+                       set([('image.iso', 123, 'sha256', 'deadbeef')])),
-             mock.call(self.topdir + '/compose/Client/i386/iso', 'md5', {'image.iso': 'cafebabe'},
+             mock.call(self.topdir + '/compose/Client/i386/iso/test-Client-1.0-20151203.t.0-MD5SUM',
-                       'test-Client-1.0-20151203.t.0-MD5SUM'),
+                       set([('image.iso', 123, 'md5', 'cafebabe')])),
-             mock.call(self.topdir + '/compose/Client/i386/iso', 'sha256', {'image.iso': 'deadbeef'},
+             mock.call(self.topdir + '/compose/Client/i386/iso/test-Client-1.0-20151203.t.0-SHA256SUM',
-                       'test-Client-1.0-20151203.t.0-SHA256SUM')],
+                       set([('image.iso', 123, 'sha256', 'deadbeef')]))],
             any_order=True
         )
         cc.assert_called_once_with(self.topdir + '/compose/Client/i386/iso/image.iso', ['md5', 'sha256'])
@@ -180,67 +176,8 @@ class TestImageChecksumPhase(PungiTestCase):
                                                      mock.call(None, 'md5', 'cafebabe')],
                                                     any_order=True)
 
-    @mock.patch('os.path.exists')
-    @mock.patch('kobo.shortcuts.compute_file_checksums')
-    @mock.patch('pungi.phases.image_checksum.get_file_size')
-    @mock.patch('pungi.phases.image_checksum.dump_filesizes')
-    @mock.patch('pungi.phases.image_checksum.dump_checksums')
-    def test_dump_filesizes_one_file(self, dump_checksums, dump_filesizes, get_file_size, cc, exists):
-        compose = DummyCompose(self.topdir, {
-            'media_checksums': ['md5', 'sha256'],
-            'media_checksum_base_filename': '{release_short}-{variant}-{version}-{date}{type_suffix}.{respin}',
-            'media_checksum_one_file': True
-        })
-        compose.image.size = None
-        get_file_size.return_value = '12345'
 
-        phase = ImageChecksumPhase(compose)
+class TestDumpChecksums(unittest.TestCase):
-
-        exists.return_value = True
-        cc.return_value = {'md5': 'cafebabe', 'sha256': 'deadbeef'}
-
-        phase.run()
-
-        dump_filesizes.assert_called_once_with(
-            self.topdir + '/compose/Client/i386/iso', {'image.iso': '12345'}, 'test-Client-1.0-20151203.t.0-CHECKSUM')
-        get_file_size.assert_called_once_with(self.topdir + '/compose/Client/i386/iso/image.iso')
-
-    @mock.patch('os.path.exists')
-    @mock.patch('kobo.shortcuts.compute_file_checksums')
-    @mock.patch('pungi.phases.image_checksum.get_file_size')
-    @mock.patch('pungi.phases.image_checksum.dump_filesizes')
-    @mock.patch('pungi.phases.image_checksum.dump_checksums')
-    def test_dump_filesizes_save_individuals(self, dump_checksums, dump_filesizes, get_file_size, cc, exists):
-        compose = DummyCompose(self.topdir, {
-            'media_checksums': ['md5', 'sha256'],
-            'media_checksum_base_filename': '{release_short}-{variant}-{version}-{date}{type_suffix}.{respin}'
-        })
-        compose.image.size = None
-        get_file_size.return_value = '12345'
-
-        phase = ImageChecksumPhase(compose)
-
-        exists.return_value = True
-        cc.return_value = {'md5': 'cafebabe', 'sha256': 'deadbeef'}
-
-        phase.run()
-
-        dump_filesizes.assert_has_calls(
-            [mock.call(self.topdir + '/compose/Client/i386/iso',
-                       {'image.iso': '12345'}, 'image.iso.SHA256SUM'),
-             mock.call(self.topdir + '/compose/Client/i386/iso',
-                       {'image.iso': '12345'}, 'image.iso.MD5SUM'),
-             mock.call(self.topdir + '/compose/Client/i386/iso',
-                       {'image.iso': '12345'}, 'test-Client-1.0-20151203.t.0-SHA256SUM'),
-             mock.call(self.topdir + '/compose/Client/i386/iso',
-                       {'image.iso': '12345'}, 'test-Client-1.0-20151203.t.0-MD5SUM')],
-            any_order=True
-        )
-
-        get_file_size.assert_called_once_with(self.topdir + '/compose/Client/i386/iso/image.iso')
-
-
-class TestChecksums(unittest.TestCase):
     def setUp(self):
         self.tmp_dir = tempfile.mkdtemp()
 
@@ -248,39 +185,20 @@ class TestChecksums(unittest.TestCase):
         shutil.rmtree(self.tmp_dir)
 
     def test_dump_checksums(self):
-        dump_checksums(self.tmp_dir,
+        dump_checksums(os.path.join(self.tmp_dir, 'CHECKSUM'),
-                       'md5',
+                       [('file2.iso', 456, 'md5', 'cafebabe'),
-                       {'file1.iso': 'abcdef', 'file2.iso': 'cafebabe'},
+                        ('file1.iso', 123, 'md5', 'abcdef')])
-                       'CHECKSUM')
-
-        with open(os.path.join(self.tmp_dir, 'CHECKSUM'), 'r') as f:
-            data = f.read().rstrip().split('\n')
-            expected = [
-                'MD5 (file1.iso) = abcdef',
-                'MD5 (file2.iso) = cafebabe',
-            ]
-            self.assertItemsEqual(expected, data)
-
-
-class TestDumpFilesizes(unittest.TestCase):
-    def setUp(self):
-        self.tmp_dir = tempfile.mkdtemp()
-
-    def tearDown(self):
-        shutil.rmtree(self.tmp_dir)
-
-    def test_dump_files(self):
-        filesizes = {'file1.iso': 123,
-                     'file2.iso': 456}
-        dump_filesizes(self.tmp_dir, filesizes, 'CHECKSUM')
 
         with open(os.path.join(self.tmp_dir, 'CHECKSUM'), 'r') as f:
             data = f.read().rstrip().split('\n')
             expected = [
                 '# file1.iso: 123 bytes',
+                'MD5 (file1.iso) = abcdef',
                 '# file2.iso: 456 bytes',
+                'MD5 (file2.iso) = cafebabe',
             ]
-            self.assertItemsEqual(expected, data)
+            self.assertEqual(expected, data)
+
 
 if __name__ == "__main__":
     unittest.main()

tests/test_unified_isos.py

@@ -1,7 +1,6 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 
-import json
 import mock
 import os
 import shutil
@@ -36,6 +35,14 @@ class TestUnifiedIsos(PungiTestCase):
         self.assertRegexpMatches(isos.temp_dir,
                                  '^%s/' % os.path.join(self.topdir, COMPOSE_ID, 'work'))
 
+    def test_dump_manifest(self):
+        compose_path = os.path.join(self.topdir, COMPOSE_ID, 'compose')
+        isos = unified_isos.UnifiedISO(compose_path)
+        isos.compose._images = mock.Mock()
+        isos.dump_manifest()
+        self.assertEqual(isos.compose._images.mock_calls,
+                         [mock.call.dump(compose_path + '/metadata/images.json')])
+
 
 class TestCreate(PungiTestCase):
     def setUp(self):
@@ -47,7 +54,7 @@ class TestCreate(PungiTestCase):
 
     def test_create_method(self):
         methods = ('link_to_temp', 'createrepo', 'discinfo', 'createiso',
-                   'link_to_compose', 'update_checksums')
+                   'link_to_compose', 'update_checksums', 'dump_manifest')
         for attr in methods:
             setattr(self.isos, attr, mock.Mock())
 
@@ -391,25 +398,22 @@ class TestCreateiso(PungiTestCase):
         self.mkisofs_cmd = self.mkisofs_cmd or mock.Mock(name='mkisofs cmd')
         return self.mkisofs_cmd
 
-    def _img(self, arch, exts):
+    def _img(self, arch):
-        exts = ['manifest'] + exts
         base_path = os.path.join(self.isos.temp_dir, 'iso', arch,
                                  u'DP-1.0-20161013.t.4-%s-dvd.iso' % arch)
         yield base_path
-        for ext in exts:
+        yield base_path + '.manifest'
-            yield base_path + '.' + ext
 
-    def _imgs(self, arches, exts):
+    def _imgs(self, arches):
         images = {}
-        exts = [e + 'SUM' for e in exts]
         for arch in arches:
             file_arch = arch
             if arch.startswith('debug-'):
                 file_arch = arch.split('-', 1)[-1] + '-debuginfo'
-            images[arch] = set(self._img(file_arch if arch != 'src' else 'source', exts))
+            images[arch] = set(self._img(file_arch if arch != 'src' else 'source'))
         return images
 
-    def assertResults(self, iso, run, arches, checksums):
+    def assertResults(self, iso, run, arches):
         self.assertEqual(
             run.mock_calls,
             [mock.call(self.mkisofs_cmd),
@@ -417,33 +421,19 @@ class TestCreateiso(PungiTestCase):
              mock.call(iso.get_manifest_cmd.return_value)] * len(arches)
         )
 
-        self.assertEqual(
+        self.assertEqual(self.isos.images, self._imgs(arches))
-            self.isos.images,
-            self._imgs(arches, checksums),
-        )
 
-        with open(os.path.join(self.compose_path, 'metadata', 'images.json')) as f:
+        images = self.isos.compose.images
-            manifest = json.load(f)
 
         for v in ('Client', 'Server'):
             for a in arches:
-                for image in manifest['payload']['images'][v]['x86_64']:
+                for image in images[v]['x86_64']:
-                    arch = iso_arch = 'source' if image['arch'] == 'src' else image['arch']
+                    arch = iso_arch = 'source' if image.arch == 'src' else image.arch
                     if a.startswith('debug-'):
                         iso_arch += '-debuginfo'
                         a = a.split('-', 1)[1]
                     path = '{0}/{1}/iso/DP-1.0-20161013.t.4-{1}-dvd.iso'.format(v, arch, iso_arch)
-                    if image.get('unified', False) and image['arch'] == a and image['path'] == path:
+                    if image.unified and image.arch == a and image.path == path:
-                        checksum_file_base = os.path.join(self.isos.temp_dir, 'iso',
-                                                          arch, os.path.basename(image['path']))
-                        for ch in checksums:
-                            fp = '%s.%sSUM' % (checksum_file_base, ch)
-                            with open(fp) as f:
-                                self.assertEqual(
-                                    f.read(),
-                                    '%s (%s) = %s\n' % (ch, os.path.basename(image['path']),
-                                                        CHECKSUMS[ch])
-                                )
                         break
                 else:
                     self.fail('Image for %s.%s missing' % (v, a))
@@ -460,7 +450,7 @@ class TestCreateiso(PungiTestCase):
 
         self.isos.createiso()
 
-        self.assertResults(iso, run, ['src', 'x86_64'], ['MD5', 'SHA1', 'SHA256'])
+        self.assertResults(iso, run, ['src', 'x86_64'])
 
     @mock.patch('pungi_utils.unified_isos.iso')
     @mock.patch('pungi_utils.unified_isos.run')
@@ -475,39 +465,7 @@ class TestCreateiso(PungiTestCase):
 
         self.isos.createiso()
 
-        self.assertResults(iso, run, ['src', 'x86_64', 'debug-x86_64'], ['MD5', 'SHA1', 'SHA256'])
+        self.assertResults(iso, run, ['src', 'x86_64', 'debug-x86_64'])
-
-    @mock.patch('pungi_utils.unified_isos.iso')
-    @mock.patch('pungi_utils.unified_isos.run')
-    def test_createiso_checksum_one_file(self, run, iso):
-        iso.get_mkisofs_cmd.side_effect = self.mock_gmc
-        iso.get_implanted_md5.return_value = 'beefcafebabedeadbeefcafebabedead'
-        iso.get_volume_id.return_value = 'VOLID'
-
-        self.isos.conf['media_checksum_one_file'] = True
-
-        self.isos.treeinfo = {'x86_64': self.isos.treeinfo['x86_64'],
-                              'src': self.isos.treeinfo['src']}
-
-        self.isos.createiso()
-
-        self.assertResults(iso, run, ['src', 'x86_64'], [])
-
-    @mock.patch('pungi_utils.unified_isos.iso')
-    @mock.patch('pungi_utils.unified_isos.run')
-    def test_createiso_single_checksum(self, run, iso):
-        iso.get_mkisofs_cmd.side_effect = self.mock_gmc
-        iso.get_implanted_md5.return_value = 'beefcafebabedeadbeefcafebabedead'
-        iso.get_volume_id.return_value = 'VOLID'
-
-        self.isos.conf['media_checksums'] = ['sha256']
-
-        self.isos.treeinfo = {'x86_64': self.isos.treeinfo['x86_64'],
-                              'src': self.isos.treeinfo['src']}
-
-        self.isos.createiso()
-
-        self.assertResults(iso, run, ['src', 'x86_64'], ['SHA256'])
 
 
 class TestLinkToCompose(PungiTestCase):
@@ -575,15 +533,9 @@ class TestUpdateChecksums(PungiTestCase):
         self.isos.update_checksums()
         self.assertItemsEqual(
             mmc.call_args_list,
-            [self._call('Client', 'i386'),
+            [mock.call(self.compose_path, self.isos.compose.images,
-             self._call('Client', 'x86_64'),
+                       unified_isos.DEFAULT_CHECKSUMS, False,
-             self._call('Server', 's390x'),
+                       self.isos._get_base_filename)])
-             self._call('Server', 'x86_64'),
-             self._call('Client', 'i386', source=True),
-             self._call('Client', 'x86_64', source=True),
-             self._call('Server', 's390x', source=True),
-             self._call('Server', 'x86_64', source=True)]
-        )
 
     @mock.patch('pungi_utils.unified_isos.make_checksums')
     def test_update_checksums_one_file(self, mmc):
@@ -591,28 +543,11 @@ class TestUpdateChecksums(PungiTestCase):
         self.isos.update_checksums()
         self.assertItemsEqual(
             mmc.call_args_list,
-            [self._call('Client', 'i386', one_file=True),
+            [mock.call(self.compose_path, self.isos.compose.images,
-             self._call('Client', 'x86_64', one_file=True),
+                       unified_isos.DEFAULT_CHECKSUMS, True,
-             self._call('Server', 's390x', one_file=True),
+                       self.isos._get_base_filename)])
-             self._call('Server', 'x86_64', one_file=True),
-             self._call('Client', 'i386', source=True, one_file=True),
-             self._call('Client', 'x86_64', source=True, one_file=True),
-             self._call('Server', 's390x', source=True, one_file=True),
-             self._call('Server', 'x86_64', source=True, one_file=True)]
-        )
 
-    @mock.patch('pungi_utils.unified_isos.make_checksums')
+    def test_get_base_filename(self):
-    def test_update_checksums_basename(self, mmc):
         self.isos.conf['media_checksum_base_filename'] = '{variant}-{arch}'
-        self.isos.update_checksums()
+        self.assertEqual(self.isos._get_base_filename('Client', 'x86_64'),
-        self.assertItemsEqual(
+                         'Client-x86_64-')
-            mmc.call_args_list,
-            [self._call('Client', 'i386', basename='Client-i386-'),
-             self._call('Client', 'x86_64', basename='Client-x86_64-'),
-             self._call('Server', 's390x', basename='Server-s390x-'),
-             self._call('Server', 'x86_64', basename='Server-x86_64-'),
-             self._call('Client', 'i386', source=True, basename='Client-i386-'),
-             self._call('Client', 'x86_64', source=True, basename='Client-x86_64-'),
-             self._call('Server', 's390x', source=True, basename='Server-s390x-'),
-             self._call('Server', 'x86_64', source=True, basename='Server-x86_64-')]
-        )