createrepo: Allow selecting variants for delta RPMs
The configuration needs to be more granular than a single global option. With this patch each tree can enable deltas separately. Fixes: https://pagure.io/pungi/issue/715 Signed-off-by: Lubomír Sedlář <lsedlar@redhat.com>
This commit is contained in:
parent
90be25c14c
commit
e21a27bdc9
@ -390,8 +390,10 @@ Options
|
||||
(*bool*) -- use createrepo_c (True) or legacy createrepo (False)
|
||||
|
||||
**createrepo_deltas** = False
|
||||
(*bool*) -- generate delta RPMs against an older compose. This needs to be
|
||||
used together with ``--old-composes`` command line argument.
|
||||
(*list*) -- generate delta RPMs against an older compose. This needs to be
|
||||
used together with ``--old-composes`` command line argument. The value
|
||||
should be a mapping of variants and architectures that should enable
|
||||
creating delta RPMs. Source and debuginfo repos never have deltas.
|
||||
|
||||
**createrepo_use_xz** = False
|
||||
(*bool*) -- whether to pass ``--xz`` to the createrepo command. This will
|
||||
@ -413,6 +415,12 @@ Example
|
||||
::
|
||||
|
||||
createrepo_checksum = "sha"
|
||||
createrepo_deltas = [
|
||||
# All arches for Everything should have deltas.
|
||||
('^Everything$', {'*': True}),
|
||||
# Also Server.x86_64 should have them (but not on other arches).
|
||||
('^Server$', {'x86_64': True}),
|
||||
]
|
||||
|
||||
|
||||
Package Set Settings
|
||||
|
@ -681,8 +681,14 @@ def make_schema():
|
||||
"additionalProperties": False,
|
||||
},
|
||||
"createrepo_deltas": {
|
||||
"type": "boolean",
|
||||
"default": False,
|
||||
"anyOf": [
|
||||
# Deprecated in favour of more granular settings.
|
||||
{
|
||||
"type": "boolean",
|
||||
"default": False,
|
||||
},
|
||||
_variant_arch_mapping({"type": "boolean"})
|
||||
]
|
||||
},
|
||||
|
||||
"buildinstall_method": {
|
||||
|
@ -32,7 +32,7 @@ from kobo.shortcuts import run, relative_path
|
||||
from ..wrappers.scm import get_dir_from_scm
|
||||
from ..wrappers.createrepo import CreaterepoWrapper
|
||||
from .base import PhaseBase
|
||||
from ..util import find_old_compose, temp_dir
|
||||
from ..util import find_old_compose, temp_dir, get_arch_variant_data
|
||||
|
||||
import productmd.rpms
|
||||
|
||||
@ -55,7 +55,7 @@ class CreaterepoPhase(PhaseBase):
|
||||
except ValueError as exc:
|
||||
errors = exc.message.split('\n')
|
||||
|
||||
if not self.compose.old_composes and self.compose.conf['createrepo_deltas']:
|
||||
if not self.compose.old_composes and self.compose.conf.get('createrepo_deltas'):
|
||||
errors.append('Can not generate deltas without old compose')
|
||||
|
||||
if errors:
|
||||
@ -120,7 +120,7 @@ def create_variant_repo(compose, arch, variant, pkg_type):
|
||||
compose.log_info("[BEGIN] %s" % msg)
|
||||
|
||||
# We only want delta RPMs for binary repos.
|
||||
with_deltas = compose.conf['createrepo_deltas'] and pkg_type == 'rpm'
|
||||
with_deltas = pkg_type == 'rpm' and _has_deltas(compose, variant, arch)
|
||||
|
||||
rpms = set()
|
||||
rpm_nevras = set()
|
||||
@ -298,3 +298,11 @@ def _find_package_dirs(base):
|
||||
# The directory does not exist, so no drpms for you!
|
||||
pass
|
||||
return sorted(buckets)
|
||||
|
||||
|
||||
def _has_deltas(compose, variant, arch):
|
||||
"""Check if delta RPMs are enabled for given variant and architecture."""
|
||||
key = 'createrepo_deltas'
|
||||
if isinstance(compose.conf.get(key), bool):
|
||||
return compose.conf[key]
|
||||
return any(get_arch_variant_data(compose.conf, key, arch, variant))
|
||||
|
@ -21,6 +21,18 @@ from tests.helpers import DummyCompose, PungiTestCase, copy_fixture, touch
|
||||
|
||||
|
||||
class TestCreaterepoPhase(PungiTestCase):
|
||||
@mock.patch('pungi.phases.createrepo.ThreadPool')
|
||||
def test_validates_without_option(self, ThreadPoolCls):
|
||||
compose = DummyCompose(self.topdir, {
|
||||
'createrepo_checksum': 'sha256',
|
||||
})
|
||||
|
||||
phase = CreaterepoPhase(compose)
|
||||
try:
|
||||
phase.validate()
|
||||
except ValueError:
|
||||
self.fail('Missing delta config should not fail validation')
|
||||
|
||||
@mock.patch('pungi.phases.createrepo.ThreadPool')
|
||||
def test_fails_deltas_without_old_compose(self, ThreadPoolCls):
|
||||
compose = DummyCompose(self.topdir, {
|
||||
@ -34,6 +46,19 @@ class TestCreaterepoPhase(PungiTestCase):
|
||||
|
||||
self.assertIn('deltas', str(ctx.exception))
|
||||
|
||||
@mock.patch('pungi.phases.createrepo.ThreadPool')
|
||||
def test_fails_deltas_without_old_compose_granular_config(self, ThreadPoolCls):
|
||||
compose = DummyCompose(self.topdir, {
|
||||
'createrepo_checksum': 'sha256',
|
||||
'createrepo_deltas': [('^Everything$', {'*': True})],
|
||||
})
|
||||
|
||||
phase = CreaterepoPhase(compose)
|
||||
with self.assertRaises(ValueError) as ctx:
|
||||
phase.validate()
|
||||
|
||||
self.assertIn('deltas', str(ctx.exception))
|
||||
|
||||
@mock.patch('pungi.phases.createrepo.ThreadPool')
|
||||
def test_starts_jobs(self, ThreadPoolCls):
|
||||
compose = DummyCompose(self.topdir, {})
|
||||
@ -317,6 +342,110 @@ class TestCreateVariantRepo(PungiTestCase):
|
||||
with open(list_file) as f:
|
||||
self.assertEqual(f.read(), 'Packages/b/bash-4.3.30-2.fc21.x86_64.rpm\n')
|
||||
|
||||
@mock.patch('pungi.phases.createrepo.run')
|
||||
@mock.patch('pungi.phases.createrepo.CreaterepoWrapper')
|
||||
def test_variant_repo_rpms_with_deltas_granular_config(self, CreaterepoWrapperCls, run):
|
||||
compose = DummyCompose(self.topdir, {
|
||||
'createrepo_checksum': 'sha256',
|
||||
'createrepo_deltas': [('^Server$', {'*': True})],
|
||||
})
|
||||
compose.DEBUG = False
|
||||
compose.has_comps = False
|
||||
compose.old_composes = [self.topdir + '/old']
|
||||
touch(os.path.join(self.topdir, 'old', 'test-1.0-20151203.0', 'STATUS'), 'FINISHED')
|
||||
|
||||
repo = CreaterepoWrapperCls.return_value
|
||||
copy_fixture('server-rpms.json', compose.paths.compose.metadata('rpms.json'))
|
||||
|
||||
create_variant_repo(compose, 'x86_64', compose.variants['Server'], 'rpm')
|
||||
|
||||
list_file = self.topdir + '/work/x86_64/repo_package_list/Server.x86_64.rpm.conf'
|
||||
self.assertEqual(CreaterepoWrapperCls.mock_calls[0],
|
||||
mock.call(createrepo_c=True))
|
||||
self.assertItemsEqual(
|
||||
repo.get_createrepo_cmd.mock_calls,
|
||||
[mock.call(self.topdir + '/compose/Server/x86_64/os', checksum='sha256',
|
||||
database=True, groupfile=None, workers=3,
|
||||
outputdir=self.topdir + '/compose/Server/x86_64/os',
|
||||
pkglist=list_file, skip_stat=True, update=True,
|
||||
update_md_path=None, deltas=True,
|
||||
oldpackagedirs=self.topdir + '/old/test-1.0-20151203.0/compose/Server/x86_64/os/Packages',
|
||||
use_xz=False)])
|
||||
self.assertItemsEqual(
|
||||
repo.get_modifyrepo_cmd.mock_calls,
|
||||
[])
|
||||
with open(list_file) as f:
|
||||
self.assertEqual(f.read(), 'Packages/b/bash-4.3.30-2.fc21.x86_64.rpm\n')
|
||||
|
||||
@mock.patch('pungi.phases.createrepo.run')
|
||||
@mock.patch('pungi.phases.createrepo.CreaterepoWrapper')
|
||||
def test_variant_repo_rpms_with_deltas_granular_config_no_match(self, CreaterepoWrapperCls, run):
|
||||
compose = DummyCompose(self.topdir, {
|
||||
'createrepo_checksum': 'sha256',
|
||||
'createrepo_deltas': [('^Everything$', {'x86_64': True})],
|
||||
})
|
||||
compose.DEBUG = False
|
||||
compose.has_comps = False
|
||||
compose.old_composes = [self.topdir + '/old']
|
||||
touch(os.path.join(self.topdir, 'old', 'test-1.0-20151203.0', 'STATUS'), 'FINISHED')
|
||||
|
||||
repo = CreaterepoWrapperCls.return_value
|
||||
copy_fixture('server-rpms.json', compose.paths.compose.metadata('rpms.json'))
|
||||
|
||||
create_variant_repo(compose, 'x86_64', compose.variants['Server'], 'rpm')
|
||||
|
||||
list_file = self.topdir + '/work/x86_64/repo_package_list/Server.x86_64.rpm.conf'
|
||||
self.assertEqual(CreaterepoWrapperCls.mock_calls[0],
|
||||
mock.call(createrepo_c=True))
|
||||
self.assertItemsEqual(
|
||||
repo.get_createrepo_cmd.mock_calls,
|
||||
[mock.call(self.topdir + '/compose/Server/x86_64/os', checksum='sha256',
|
||||
database=True, groupfile=None, workers=3,
|
||||
outputdir=self.topdir + '/compose/Server/x86_64/os',
|
||||
pkglist=list_file, skip_stat=True, update=True,
|
||||
update_md_path=self.topdir + '/work/x86_64/repo',
|
||||
deltas=False, oldpackagedirs=None, use_xz=False)])
|
||||
self.assertItemsEqual(
|
||||
repo.get_modifyrepo_cmd.mock_calls,
|
||||
[])
|
||||
with open(list_file) as f:
|
||||
self.assertEqual(f.read(), 'Packages/b/bash-4.3.30-2.fc21.x86_64.rpm\n')
|
||||
|
||||
@mock.patch('pungi.phases.createrepo.run')
|
||||
@mock.patch('pungi.phases.createrepo.CreaterepoWrapper')
|
||||
def test_variant_repo_rpms_with_deltas_granular_config_no_match_on_arch(
|
||||
self, CreaterepoWrapperCls, run):
|
||||
compose = DummyCompose(self.topdir, {
|
||||
'createrepo_checksum': 'sha256',
|
||||
'createrepo_deltas': [('^Server$', {'s390x': True})],
|
||||
})
|
||||
compose.DEBUG = False
|
||||
compose.has_comps = False
|
||||
compose.old_composes = [self.topdir + '/old']
|
||||
touch(os.path.join(self.topdir, 'old', 'test-1.0-20151203.0', 'STATUS'), 'FINISHED')
|
||||
|
||||
repo = CreaterepoWrapperCls.return_value
|
||||
copy_fixture('server-rpms.json', compose.paths.compose.metadata('rpms.json'))
|
||||
|
||||
create_variant_repo(compose, 'x86_64', compose.variants['Server'], 'rpm')
|
||||
|
||||
list_file = self.topdir + '/work/x86_64/repo_package_list/Server.x86_64.rpm.conf'
|
||||
self.assertEqual(CreaterepoWrapperCls.mock_calls[0],
|
||||
mock.call(createrepo_c=True))
|
||||
self.assertItemsEqual(
|
||||
repo.get_createrepo_cmd.mock_calls,
|
||||
[mock.call(self.topdir + '/compose/Server/x86_64/os', checksum='sha256',
|
||||
database=True, groupfile=None, workers=3,
|
||||
outputdir=self.topdir + '/compose/Server/x86_64/os',
|
||||
pkglist=list_file, skip_stat=True, update=True,
|
||||
update_md_path=self.topdir + '/work/x86_64/repo',
|
||||
deltas=False, oldpackagedirs=None, use_xz=False)])
|
||||
self.assertItemsEqual(
|
||||
repo.get_modifyrepo_cmd.mock_calls,
|
||||
[])
|
||||
with open(list_file) as f:
|
||||
self.assertEqual(f.read(), 'Packages/b/bash-4.3.30-2.fc21.x86_64.rpm\n')
|
||||
|
||||
@mock.patch('pungi.phases.createrepo.run')
|
||||
@mock.patch('pungi.phases.createrepo.CreaterepoWrapper')
|
||||
def test_variant_repo_rpms_with_deltas_hashed_dirs(self, CreaterepoWrapperCls, run):
|
||||
|
Loading…
Reference in New Issue
Block a user