diff --git a/doc/configuration.rst b/doc/configuration.rst index 97bbe146..e9f4fef2 100644 --- a/doc/configuration.rst +++ b/doc/configuration.rst @@ -390,8 +390,10 @@ Options (*bool*) -- use createrepo_c (True) or legacy createrepo (False) **createrepo_deltas** = False - (*bool*) -- generate delta RPMs against an older compose. This needs to be - used together with ``--old-composes`` command line argument. + (*list*) -- generate delta RPMs against an older compose. This needs to be + used together with ``--old-composes`` command line argument. The value + should be a mapping of variants and architectures that should enable + creating delta RPMs. Source and debuginfo repos never have deltas. **createrepo_use_xz** = False (*bool*) -- whether to pass ``--xz`` to the createrepo command. This will @@ -413,6 +415,12 @@ Example :: createrepo_checksum = "sha" + createrepo_deltas = [ + # All arches for Everything should have deltas. + ('^Everything$', {'*': True}), + # Also Server.x86_64 should have them (but not on other arches). + ('^Server$', {'x86_64': True}), + ] Package Set Settings diff --git a/pungi/checks.py b/pungi/checks.py index 4eccd882..df609b11 100644 --- a/pungi/checks.py +++ b/pungi/checks.py @@ -681,8 +681,14 @@ def make_schema(): "additionalProperties": False, }, "createrepo_deltas": { - "type": "boolean", - "default": False, + "anyOf": [ + # Deprecated in favour of more granular settings. + { + "type": "boolean", + "default": False, + }, + _variant_arch_mapping({"type": "boolean"}) + ] }, "buildinstall_method": { diff --git a/pungi/phases/createrepo.py b/pungi/phases/createrepo.py index 466e9fc2..22985fba 100644 --- a/pungi/phases/createrepo.py +++ b/pungi/phases/createrepo.py @@ -32,7 +32,7 @@ from kobo.shortcuts import run, relative_path from ..wrappers.scm import get_dir_from_scm from ..wrappers.createrepo import CreaterepoWrapper from .base import PhaseBase -from ..util import find_old_compose, temp_dir +from ..util import find_old_compose, temp_dir, get_arch_variant_data import productmd.rpms @@ -55,7 +55,7 @@ class CreaterepoPhase(PhaseBase): except ValueError as exc: errors = exc.message.split('\n') - if not self.compose.old_composes and self.compose.conf['createrepo_deltas']: + if not self.compose.old_composes and self.compose.conf.get('createrepo_deltas'): errors.append('Can not generate deltas without old compose') if errors: @@ -120,7 +120,7 @@ def create_variant_repo(compose, arch, variant, pkg_type): compose.log_info("[BEGIN] %s" % msg) # We only want delta RPMs for binary repos. - with_deltas = compose.conf['createrepo_deltas'] and pkg_type == 'rpm' + with_deltas = pkg_type == 'rpm' and _has_deltas(compose, variant, arch) rpms = set() rpm_nevras = set() @@ -298,3 +298,11 @@ def _find_package_dirs(base): # The directory does not exist, so no drpms for you! pass return sorted(buckets) + + +def _has_deltas(compose, variant, arch): + """Check if delta RPMs are enabled for given variant and architecture.""" + key = 'createrepo_deltas' + if isinstance(compose.conf.get(key), bool): + return compose.conf[key] + return any(get_arch_variant_data(compose.conf, key, arch, variant)) diff --git a/tests/test_createrepophase.py b/tests/test_createrepophase.py index 2e0334d6..0eba0e4f 100644 --- a/tests/test_createrepophase.py +++ b/tests/test_createrepophase.py @@ -21,6 +21,18 @@ from tests.helpers import DummyCompose, PungiTestCase, copy_fixture, touch class TestCreaterepoPhase(PungiTestCase): + @mock.patch('pungi.phases.createrepo.ThreadPool') + def test_validates_without_option(self, ThreadPoolCls): + compose = DummyCompose(self.topdir, { + 'createrepo_checksum': 'sha256', + }) + + phase = CreaterepoPhase(compose) + try: + phase.validate() + except ValueError: + self.fail('Missing delta config should not fail validation') + @mock.patch('pungi.phases.createrepo.ThreadPool') def test_fails_deltas_without_old_compose(self, ThreadPoolCls): compose = DummyCompose(self.topdir, { @@ -34,6 +46,19 @@ class TestCreaterepoPhase(PungiTestCase): self.assertIn('deltas', str(ctx.exception)) + @mock.patch('pungi.phases.createrepo.ThreadPool') + def test_fails_deltas_without_old_compose_granular_config(self, ThreadPoolCls): + compose = DummyCompose(self.topdir, { + 'createrepo_checksum': 'sha256', + 'createrepo_deltas': [('^Everything$', {'*': True})], + }) + + phase = CreaterepoPhase(compose) + with self.assertRaises(ValueError) as ctx: + phase.validate() + + self.assertIn('deltas', str(ctx.exception)) + @mock.patch('pungi.phases.createrepo.ThreadPool') def test_starts_jobs(self, ThreadPoolCls): compose = DummyCompose(self.topdir, {}) @@ -317,6 +342,110 @@ class TestCreateVariantRepo(PungiTestCase): with open(list_file) as f: self.assertEqual(f.read(), 'Packages/b/bash-4.3.30-2.fc21.x86_64.rpm\n') + @mock.patch('pungi.phases.createrepo.run') + @mock.patch('pungi.phases.createrepo.CreaterepoWrapper') + def test_variant_repo_rpms_with_deltas_granular_config(self, CreaterepoWrapperCls, run): + compose = DummyCompose(self.topdir, { + 'createrepo_checksum': 'sha256', + 'createrepo_deltas': [('^Server$', {'*': True})], + }) + compose.DEBUG = False + compose.has_comps = False + compose.old_composes = [self.topdir + '/old'] + touch(os.path.join(self.topdir, 'old', 'test-1.0-20151203.0', 'STATUS'), 'FINISHED') + + repo = CreaterepoWrapperCls.return_value + copy_fixture('server-rpms.json', compose.paths.compose.metadata('rpms.json')) + + create_variant_repo(compose, 'x86_64', compose.variants['Server'], 'rpm') + + list_file = self.topdir + '/work/x86_64/repo_package_list/Server.x86_64.rpm.conf' + self.assertEqual(CreaterepoWrapperCls.mock_calls[0], + mock.call(createrepo_c=True)) + self.assertItemsEqual( + repo.get_createrepo_cmd.mock_calls, + [mock.call(self.topdir + '/compose/Server/x86_64/os', checksum='sha256', + database=True, groupfile=None, workers=3, + outputdir=self.topdir + '/compose/Server/x86_64/os', + pkglist=list_file, skip_stat=True, update=True, + update_md_path=None, deltas=True, + oldpackagedirs=self.topdir + '/old/test-1.0-20151203.0/compose/Server/x86_64/os/Packages', + use_xz=False)]) + self.assertItemsEqual( + repo.get_modifyrepo_cmd.mock_calls, + []) + with open(list_file) as f: + self.assertEqual(f.read(), 'Packages/b/bash-4.3.30-2.fc21.x86_64.rpm\n') + + @mock.patch('pungi.phases.createrepo.run') + @mock.patch('pungi.phases.createrepo.CreaterepoWrapper') + def test_variant_repo_rpms_with_deltas_granular_config_no_match(self, CreaterepoWrapperCls, run): + compose = DummyCompose(self.topdir, { + 'createrepo_checksum': 'sha256', + 'createrepo_deltas': [('^Everything$', {'x86_64': True})], + }) + compose.DEBUG = False + compose.has_comps = False + compose.old_composes = [self.topdir + '/old'] + touch(os.path.join(self.topdir, 'old', 'test-1.0-20151203.0', 'STATUS'), 'FINISHED') + + repo = CreaterepoWrapperCls.return_value + copy_fixture('server-rpms.json', compose.paths.compose.metadata('rpms.json')) + + create_variant_repo(compose, 'x86_64', compose.variants['Server'], 'rpm') + + list_file = self.topdir + '/work/x86_64/repo_package_list/Server.x86_64.rpm.conf' + self.assertEqual(CreaterepoWrapperCls.mock_calls[0], + mock.call(createrepo_c=True)) + self.assertItemsEqual( + repo.get_createrepo_cmd.mock_calls, + [mock.call(self.topdir + '/compose/Server/x86_64/os', checksum='sha256', + database=True, groupfile=None, workers=3, + outputdir=self.topdir + '/compose/Server/x86_64/os', + pkglist=list_file, skip_stat=True, update=True, + update_md_path=self.topdir + '/work/x86_64/repo', + deltas=False, oldpackagedirs=None, use_xz=False)]) + self.assertItemsEqual( + repo.get_modifyrepo_cmd.mock_calls, + []) + with open(list_file) as f: + self.assertEqual(f.read(), 'Packages/b/bash-4.3.30-2.fc21.x86_64.rpm\n') + + @mock.patch('pungi.phases.createrepo.run') + @mock.patch('pungi.phases.createrepo.CreaterepoWrapper') + def test_variant_repo_rpms_with_deltas_granular_config_no_match_on_arch( + self, CreaterepoWrapperCls, run): + compose = DummyCompose(self.topdir, { + 'createrepo_checksum': 'sha256', + 'createrepo_deltas': [('^Server$', {'s390x': True})], + }) + compose.DEBUG = False + compose.has_comps = False + compose.old_composes = [self.topdir + '/old'] + touch(os.path.join(self.topdir, 'old', 'test-1.0-20151203.0', 'STATUS'), 'FINISHED') + + repo = CreaterepoWrapperCls.return_value + copy_fixture('server-rpms.json', compose.paths.compose.metadata('rpms.json')) + + create_variant_repo(compose, 'x86_64', compose.variants['Server'], 'rpm') + + list_file = self.topdir + '/work/x86_64/repo_package_list/Server.x86_64.rpm.conf' + self.assertEqual(CreaterepoWrapperCls.mock_calls[0], + mock.call(createrepo_c=True)) + self.assertItemsEqual( + repo.get_createrepo_cmd.mock_calls, + [mock.call(self.topdir + '/compose/Server/x86_64/os', checksum='sha256', + database=True, groupfile=None, workers=3, + outputdir=self.topdir + '/compose/Server/x86_64/os', + pkglist=list_file, skip_stat=True, update=True, + update_md_path=self.topdir + '/work/x86_64/repo', + deltas=False, oldpackagedirs=None, use_xz=False)]) + self.assertItemsEqual( + repo.get_modifyrepo_cmd.mock_calls, + []) + with open(list_file) as f: + self.assertEqual(f.read(), 'Packages/b/bash-4.3.30-2.fc21.x86_64.rpm\n') + @mock.patch('pungi.phases.createrepo.run') @mock.patch('pungi.phases.createrepo.CreaterepoWrapper') def test_variant_repo_rpms_with_deltas_hashed_dirs(self, CreaterepoWrapperCls, run):