createrepo: Allow selecting variants for delta RPMs

The configuration needs to be more granular than a single global option.
With this patch each tree can enable deltas separately.

Fixes: https://pagure.io/pungi/issue/715
Signed-off-by: Lubomír Sedlář <lsedlar@redhat.com>

doc/configuration.rst

@@ -390,8 +390,10 @@ Options
     (*bool*) -- use createrepo_c (True) or legacy createrepo (False)
 
 **createrepo_deltas** = False
-    (*bool*) -- generate delta RPMs against an older compose. This needs to be
+    (*list*) -- generate delta RPMs against an older compose. This needs to be
-    used together with ``--old-composes`` command line argument.
+    used together with ``--old-composes`` command line argument. The value
+    should be a mapping of variants and architectures that should enable
+    creating delta RPMs. Source and debuginfo repos never have deltas.
 
 **createrepo_use_xz** = False
     (*bool*) -- whether to pass ``--xz`` to the createrepo command. This will
@@ -413,6 +415,12 @@ Example
 ::
 
     createrepo_checksum = "sha"
+    createrepo_deltas = [
+        # All arches for Everything should have deltas.
+        ('^Everything$', {'*': True}),
+        # Also Server.x86_64 should have them (but not on other arches).
+        ('^Server$', {'x86_64': True}),
+    ]
 
 
 Package Set Settings

pungi/checks.py

@@ -681,8 +681,14 @@ def make_schema():
                 "additionalProperties": False,
             },
             "createrepo_deltas": {
-                "type": "boolean",
+                "anyOf": [
-                "default": False,
+                    # Deprecated in favour of more granular settings.
+                    {
+                        "type": "boolean",
+                        "default": False,
+                    },
+                    _variant_arch_mapping({"type": "boolean"})
+                ]
             },
 
             "buildinstall_method": {

pungi/phases/createrepo.py

@@ -32,7 +32,7 @@ from kobo.shortcuts import run, relative_path
 from ..wrappers.scm import get_dir_from_scm
 from ..wrappers.createrepo import CreaterepoWrapper
 from .base import PhaseBase
-from ..util import find_old_compose, temp_dir
+from ..util import find_old_compose, temp_dir, get_arch_variant_data
 
 import productmd.rpms
 
@@ -55,7 +55,7 @@ class CreaterepoPhase(PhaseBase):
         except ValueError as exc:
             errors = exc.message.split('\n')
 
-        if not self.compose.old_composes and self.compose.conf['createrepo_deltas']:
+        if not self.compose.old_composes and self.compose.conf.get('createrepo_deltas'):
             errors.append('Can not generate deltas without old compose')
 
         if errors:
@@ -120,7 +120,7 @@ def create_variant_repo(compose, arch, variant, pkg_type):
     compose.log_info("[BEGIN] %s" % msg)
 
     # We only want delta RPMs for binary repos.
-    with_deltas = compose.conf['createrepo_deltas'] and pkg_type == 'rpm'
+    with_deltas = pkg_type == 'rpm' and _has_deltas(compose, variant, arch)
 
     rpms = set()
     rpm_nevras = set()
@@ -298,3 +298,11 @@ def _find_package_dirs(base):
         # The directory does not exist, so no drpms for you!
         pass
     return sorted(buckets)
+
+
+def _has_deltas(compose, variant, arch):
+    """Check if delta RPMs are enabled for given variant and architecture."""
+    key = 'createrepo_deltas'
+    if isinstance(compose.conf.get(key), bool):
+        return compose.conf[key]
+    return any(get_arch_variant_data(compose.conf, key, arch, variant))

tests/test_createrepophase.py

@@ -21,6 +21,18 @@ from tests.helpers import DummyCompose, PungiTestCase, copy_fixture, touch
 
 
 class TestCreaterepoPhase(PungiTestCase):
+    @mock.patch('pungi.phases.createrepo.ThreadPool')
+    def test_validates_without_option(self, ThreadPoolCls):
+        compose = DummyCompose(self.topdir, {
+            'createrepo_checksum': 'sha256',
+        })
+
+        phase = CreaterepoPhase(compose)
+        try:
+            phase.validate()
+        except ValueError:
+            self.fail('Missing delta config should not fail validation')
+
     @mock.patch('pungi.phases.createrepo.ThreadPool')
     def test_fails_deltas_without_old_compose(self, ThreadPoolCls):
         compose = DummyCompose(self.topdir, {
@@ -34,6 +46,19 @@ class TestCreaterepoPhase(PungiTestCase):
 
         self.assertIn('deltas', str(ctx.exception))
 
+    @mock.patch('pungi.phases.createrepo.ThreadPool')
+    def test_fails_deltas_without_old_compose_granular_config(self, ThreadPoolCls):
+        compose = DummyCompose(self.topdir, {
+            'createrepo_checksum': 'sha256',
+            'createrepo_deltas': [('^Everything$', {'*': True})],
+        })
+
+        phase = CreaterepoPhase(compose)
+        with self.assertRaises(ValueError) as ctx:
+            phase.validate()
+
+        self.assertIn('deltas', str(ctx.exception))
+
     @mock.patch('pungi.phases.createrepo.ThreadPool')
     def test_starts_jobs(self, ThreadPoolCls):
         compose = DummyCompose(self.topdir, {})
@@ -317,6 +342,110 @@ class TestCreateVariantRepo(PungiTestCase):
         with open(list_file) as f:
             self.assertEqual(f.read(), 'Packages/b/bash-4.3.30-2.fc21.x86_64.rpm\n')
 
+    @mock.patch('pungi.phases.createrepo.run')
+    @mock.patch('pungi.phases.createrepo.CreaterepoWrapper')
+    def test_variant_repo_rpms_with_deltas_granular_config(self, CreaterepoWrapperCls, run):
+        compose = DummyCompose(self.topdir, {
+            'createrepo_checksum': 'sha256',
+            'createrepo_deltas': [('^Server$', {'*': True})],
+        })
+        compose.DEBUG = False
+        compose.has_comps = False
+        compose.old_composes = [self.topdir + '/old']
+        touch(os.path.join(self.topdir, 'old', 'test-1.0-20151203.0', 'STATUS'), 'FINISHED')
+
+        repo = CreaterepoWrapperCls.return_value
+        copy_fixture('server-rpms.json', compose.paths.compose.metadata('rpms.json'))
+
+        create_variant_repo(compose, 'x86_64', compose.variants['Server'], 'rpm')
+
+        list_file = self.topdir + '/work/x86_64/repo_package_list/Server.x86_64.rpm.conf'
+        self.assertEqual(CreaterepoWrapperCls.mock_calls[0],
+                         mock.call(createrepo_c=True))
+        self.assertItemsEqual(
+            repo.get_createrepo_cmd.mock_calls,
+            [mock.call(self.topdir + '/compose/Server/x86_64/os', checksum='sha256',
+                       database=True, groupfile=None, workers=3,
+                       outputdir=self.topdir + '/compose/Server/x86_64/os',
+                       pkglist=list_file, skip_stat=True, update=True,
+                       update_md_path=None, deltas=True,
+                       oldpackagedirs=self.topdir + '/old/test-1.0-20151203.0/compose/Server/x86_64/os/Packages',
+                       use_xz=False)])
+        self.assertItemsEqual(
+            repo.get_modifyrepo_cmd.mock_calls,
+            [])
+        with open(list_file) as f:
+            self.assertEqual(f.read(), 'Packages/b/bash-4.3.30-2.fc21.x86_64.rpm\n')
+
+    @mock.patch('pungi.phases.createrepo.run')
+    @mock.patch('pungi.phases.createrepo.CreaterepoWrapper')
+    def test_variant_repo_rpms_with_deltas_granular_config_no_match(self, CreaterepoWrapperCls, run):
+        compose = DummyCompose(self.topdir, {
+            'createrepo_checksum': 'sha256',
+            'createrepo_deltas': [('^Everything$', {'x86_64': True})],
+        })
+        compose.DEBUG = False
+        compose.has_comps = False
+        compose.old_composes = [self.topdir + '/old']
+        touch(os.path.join(self.topdir, 'old', 'test-1.0-20151203.0', 'STATUS'), 'FINISHED')
+
+        repo = CreaterepoWrapperCls.return_value
+        copy_fixture('server-rpms.json', compose.paths.compose.metadata('rpms.json'))
+
+        create_variant_repo(compose, 'x86_64', compose.variants['Server'], 'rpm')
+
+        list_file = self.topdir + '/work/x86_64/repo_package_list/Server.x86_64.rpm.conf'
+        self.assertEqual(CreaterepoWrapperCls.mock_calls[0],
+                         mock.call(createrepo_c=True))
+        self.assertItemsEqual(
+            repo.get_createrepo_cmd.mock_calls,
+            [mock.call(self.topdir + '/compose/Server/x86_64/os', checksum='sha256',
+                       database=True, groupfile=None, workers=3,
+                       outputdir=self.topdir + '/compose/Server/x86_64/os',
+                       pkglist=list_file, skip_stat=True, update=True,
+                       update_md_path=self.topdir + '/work/x86_64/repo',
+                       deltas=False, oldpackagedirs=None, use_xz=False)])
+        self.assertItemsEqual(
+            repo.get_modifyrepo_cmd.mock_calls,
+            [])
+        with open(list_file) as f:
+            self.assertEqual(f.read(), 'Packages/b/bash-4.3.30-2.fc21.x86_64.rpm\n')
+
+    @mock.patch('pungi.phases.createrepo.run')
+    @mock.patch('pungi.phases.createrepo.CreaterepoWrapper')
+    def test_variant_repo_rpms_with_deltas_granular_config_no_match_on_arch(
+            self, CreaterepoWrapperCls, run):
+        compose = DummyCompose(self.topdir, {
+            'createrepo_checksum': 'sha256',
+            'createrepo_deltas': [('^Server$', {'s390x': True})],
+        })
+        compose.DEBUG = False
+        compose.has_comps = False
+        compose.old_composes = [self.topdir + '/old']
+        touch(os.path.join(self.topdir, 'old', 'test-1.0-20151203.0', 'STATUS'), 'FINISHED')
+
+        repo = CreaterepoWrapperCls.return_value
+        copy_fixture('server-rpms.json', compose.paths.compose.metadata('rpms.json'))
+
+        create_variant_repo(compose, 'x86_64', compose.variants['Server'], 'rpm')
+
+        list_file = self.topdir + '/work/x86_64/repo_package_list/Server.x86_64.rpm.conf'
+        self.assertEqual(CreaterepoWrapperCls.mock_calls[0],
+                         mock.call(createrepo_c=True))
+        self.assertItemsEqual(
+            repo.get_createrepo_cmd.mock_calls,
+            [mock.call(self.topdir + '/compose/Server/x86_64/os', checksum='sha256',
+                       database=True, groupfile=None, workers=3,
+                       outputdir=self.topdir + '/compose/Server/x86_64/os',
+                       pkglist=list_file, skip_stat=True, update=True,
+                       update_md_path=self.topdir + '/work/x86_64/repo',
+                       deltas=False, oldpackagedirs=None, use_xz=False)])
+        self.assertItemsEqual(
+            repo.get_modifyrepo_cmd.mock_calls,
+            [])
+        with open(list_file) as f:
+            self.assertEqual(f.read(), 'Packages/b/bash-4.3.30-2.fc21.x86_64.rpm\n')
+
     @mock.patch('pungi.phases.createrepo.run')
     @mock.patch('pungi.phases.createrepo.CreaterepoWrapper')
     def test_variant_repo_rpms_with_deltas_hashed_dirs(self, CreaterepoWrapperCls, run):