almalinux
/
pungi
15
1
Fork 0
Code Issues Pull Requests Releases Activity

[createrepo] Compute delta RPMS against old compose 

Signed-off-by: Lubomír Sedlář <lsedlar@redhat.com>
This commit is contained in:
Lubomír Sedlář 2016-02-29 12:58:21 +01:00
parent 19c3707aee
commit 27375788c2
4 changed files with 164 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
doc/configuration.rst
View File

@ -336,6 +336,10 @@ Options
**createrepo_c** = True
(*bool*) -- use createrepo_c (True) or legacy createrepo (False)
**createrepo_deltas** = False
(*bool*) -- generate delta RPMs against an older compose. This needs to be
used together with `--old-composes`` command line argument.
Example

44
pungi/phases/createrepo.py
View File

@ -29,9 +29,10 @@ import threading
from kobo.threads import ThreadPool, WorkerThread
from kobo.shortcuts import run, relative_path
from pungi.wrappers.scm import get_dir_from_scm
from pungi.wrappers.createrepo import CreaterepoWrapper
from pungi.phases.base import PhaseBase
from ..wrappers.scm import get_dir_from_scm
from ..wrappers.createrepo import CreaterepoWrapper
from .base import PhaseBase
from ..util import find_old_compose
import productmd.rpms
@ -54,6 +55,11 @@ class CreaterepoPhase(PhaseBase):
"expected_types": [str],
"expected_values": ["sha256", "sha"],
},
{
"name": "createrepo_deltas",
"expected_types": [bool],
"optional": True,
},
{
"name": "product_id",
"expected_types": [dict],
@ -70,6 +76,19 @@ class CreaterepoPhase(PhaseBase):
PhaseBase.__init__(self, compose)
self.pool = ThreadPool(logger=self.compose._logger)
def validate(self):
errors = []
try:
super(CreaterepoPhase, self).validate()
except ValueError as exc:
errors = exc.message.split('\n')
if not self.compose.old_composes and 'createrepo_deltas' in self.compose.conf:
errors.append('Can not generate deltas without old compose')
if errors:
raise ValueError('\n'.join(errors))
def run(self):
get_productids_from_scm(self.compose)
for i in range(3):
@ -102,6 +121,7 @@ def create_variant_repo(compose, arch, variant, pkg_type):
createrepo_c = compose.conf.get("createrepo_c", True)
createrepo_checksum = compose.conf["createrepo_checksum"]
createrepo_deltas = compose.conf.get("createrepo_deltas", False)
repo = CreaterepoWrapper(createrepo_c=createrepo_c)
repo_dir_arch = compose.paths.work.arch_repo(arch='global' if pkg_type == 'srpm' else arch)
@ -151,13 +171,29 @@ def create_variant_repo(compose, arch, variant, pkg_type):
for rel_path in sorted(rpms):
f.write("%s\n" % rel_path)
old_packages_dir = None
if createrepo_deltas:
old_compose_path = find_old_compose(
compose.old_composes,
compose.ci_base.release.short,
compose.ci_base.release.version,
compose.ci_base.base_product.short if compose.ci_base.release.is_layered else None,
compose.ci_base.base_product.version if compose.ci_base.release.is_layered else None
)
if not old_compose_path:
compose.log_info("No suitable old compose found in: %s" % compose.old_composes)
else:
rel_dir = relative_path(repo_dir, compose.topdir.rstrip('/') + '/')
old_packages_dir = os.path.join(old_compose_path, rel_dir)
comps_path = None
if compose.has_comps and pkg_type == "rpm":
comps_path = compose.paths.work.comps(arch=arch, variant=variant)
cmd = repo.get_createrepo_cmd(repo_dir, update=True, database=True, skip_stat=True,
pkglist=file_list, outputdir=repo_dir, workers=3,
groupfile=comps_path, update_md_path=repo_dir_arch,
checksum=createrepo_checksum)
checksum=createrepo_checksum, deltas=createrepo_deltas,
oldpackagedirs=old_packages_dir)
log_file = compose.paths.log.log_file(arch, "createrepo-%s" % variant)
run(cmd, logfile=log_file, show_cmd=True)

2
tests/helpers.py
View File

@ -30,6 +30,7 @@ class DummyCompose(object):
release=mock.Mock(
short='test',
version='1.0',
is_layered=False,
),
)
self.topdir = topdir
@ -51,6 +52,7 @@ class DummyCompose(object):
self.get_image_name = mock.Mock(return_value='image-name')
self.image = mock.Mock(path='Client/i386/iso/image.iso')
self.im = mock.Mock(images={'Client': {'i386': [self.image]}})
self.old_composes = []
def get_variants(self, arch=None, types=None):
return [v for v in self.variants.values() if not arch or arch in v.arches]

124
tests/test_createrepophase.py
View File

@ -11,10 +11,22 @@ import sys
sys.path.insert(0, os.path.join(os.path.dirname(__file__), ".."))
from pungi.phases.createrepo import CreaterepoPhase, create_variant_repo
from tests.helpers import DummyCompose, PungiTestCase, copy_fixture
from tests.helpers import DummyCompose, PungiTestCase, copy_fixture, touch
class TestCreaterepoPhase(PungiTestCase):
@mock.patch('pungi.phases.createrepo.ThreadPool')
def test_fails_deltas_without_old_compose(self, ThreadPoolCls):
compose = DummyCompose(self.topdir, {
'createrepo_checksum': 'sha256',
'createrepo_deltas': True,
})
phase = CreaterepoPhase(compose)
with self.assertRaises(ValueError) as ctx:
phase.validate()
self.assertIn('deltas', str(ctx.exception))
@mock.patch('pungi.phases.createrepo.ThreadPool')
def test_starts_jobs(self, ThreadPoolCls):
@ -94,7 +106,8 @@ class TestCreateRepoThread(PungiTestCase):
database=True, groupfile=None, workers=3,
outputdir=self.topdir + '/compose/Server/x86_64/os',
pkglist=list_file, skip_stat=True, update=True,
update_md_path=self.topdir + '/work/x86_64/repo')])
update_md_path=self.topdir + '/work/x86_64/repo',
deltas=False, oldpackagedirs=None)])
with open(list_file) as f:
self.assertEqual(f.read(), 'Packages/b/bash-4.3.30-2.fc21.x86_64.rpm\n')
@ -121,7 +134,8 @@ class TestCreateRepoThread(PungiTestCase):
database=True, groupfile=None, workers=3,
outputdir=self.topdir + '/compose/Server/source/tree',
pkglist=list_file, skip_stat=True, update=True,
update_md_path=self.topdir + '/work/global/repo')])
update_md_path=self.topdir + '/work/global/repo',
deltas=False, oldpackagedirs=None)])
with open(list_file) as f:
self.assertItemsEqual(
f.read().strip().split('\n'),
@ -151,7 +165,8 @@ class TestCreateRepoThread(PungiTestCase):
database=True, groupfile=None, workers=3,
outputdir=self.topdir + '/compose/Server/x86_64/debug/tree',
pkglist=list_file, skip_stat=True, update=True,
update_md_path=self.topdir + '/work/x86_64/repo')])
update_md_path=self.topdir + '/work/x86_64/repo',
deltas=False, oldpackagedirs=None)])
with open(list_file) as f:
self.assertEqual(f.read(), 'Packages/b/bash-debuginfo-4.3.30-2.fc21.x86_64.rpm\n')
@ -179,7 +194,8 @@ class TestCreateRepoThread(PungiTestCase):
database=True, groupfile=None, workers=3,
outputdir=self.topdir + '/compose/Server/x86_64/os',
pkglist=list_file, skip_stat=True, update=True,
update_md_path=self.topdir + '/work/x86_64/repo')])
update_md_path=self.topdir + '/work/x86_64/repo',
deltas=False, oldpackagedirs=None)])
with open(list_file) as f:
self.assertEqual(f.read(), 'Packages/b/bash-4.3.30-2.fc21.x86_64.rpm\n')
@ -208,10 +224,106 @@ class TestCreateRepoThread(PungiTestCase):
database=True, groupfile=None, workers=3,
outputdir=self.topdir + '/compose/Server/x86_64/os',
pkglist=list_file, skip_stat=True, update=True,
update_md_path=self.topdir + '/work/x86_64/repo')])
update_md_path=self.topdir + '/work/x86_64/repo',
deltas=False, oldpackagedirs=None)])
with open(list_file) as f:
self.assertEqual(f.read(), 'Packages/b/bash-4.3.30-2.fc21.x86_64.rpm\n')
@mock.patch('pungi.phases.createrepo.run')
@mock.patch('pungi.phases.createrepo.CreaterepoWrapper')
def test_variant_repo_rpms_with_deltas(self, CreaterepoWrapperCls, run):
compose = DummyCompose(self.topdir, {
'createrepo_checksum': 'sha256',
'createrepo_deltas': True,
})
compose.DEBUG = False
compose.has_comps = False
compose.old_composes = [self.topdir + '/old']
touch(os.path.join(self.topdir, 'old', 'test-1.0-20151203.0', 'STATUS'), 'FINISHED')
repo = CreaterepoWrapperCls.return_value
copy_fixture('server-rpms.json', compose.paths.compose.metadata('rpms.json'))
create_variant_repo(compose, 'x86_64', compose.variants['Server'], 'rpm')
list_file = self.topdir + '/work/x86_64/repo_package_list/Server.x86_64.rpm.conf'
self.assertEqual(CreaterepoWrapperCls.mock_calls[0],
mock.call(createrepo_c=True))
self.assertItemsEqual(
repo.get_createrepo_cmd.mock_calls,
[mock.call(self.topdir + '/compose/Server/x86_64/os', checksum='sha256',
database=True, groupfile=None, workers=3,
outputdir=self.topdir + '/compose/Server/x86_64/os',
pkglist=list_file, skip_stat=True, update=True,
update_md_path=self.topdir + '/work/x86_64/repo', deltas=True,
oldpackagedirs=self.topdir + '/old/test-1.0-20151203.0/compose/Server/x86_64/os')])
with open(list_file) as f:
self.assertEqual(f.read(), 'Packages/b/bash-4.3.30-2.fc21.x86_64.rpm\n')
@mock.patch('pungi.phases.createrepo.run')
@mock.patch('pungi.phases.createrepo.CreaterepoWrapper')
def test_variant_repo_source_with_deltas(self, CreaterepoWrapperCls, run):
compose = DummyCompose(self.topdir, {
'createrepo_checksum': 'sha256',
'createrepo_deltas': True,
})
compose.DEBUG = False
compose.has_comps = False
compose.old_composes = [self.topdir + '/old']
touch(os.path.join(self.topdir, 'old', 'test-1.0-20151203.0', 'STATUS'), 'FINISHED')
repo = CreaterepoWrapperCls.return_value
copy_fixture('server-rpms.json', compose.paths.compose.metadata('rpms.json'))
create_variant_repo(compose, None, compose.variants['Server'], 'srpm')
list_file = self.topdir + '/work/global/repo_package_list/Server.None.srpm.conf'
self.assertEqual(CreaterepoWrapperCls.mock_calls[0],
mock.call(createrepo_c=True))
self.assertItemsEqual(
repo.get_createrepo_cmd.mock_calls,
[mock.call(self.topdir + '/compose/Server/source/tree', checksum='sha256',
database=True, groupfile=None, workers=3,
outputdir=self.topdir + '/compose/Server/source/tree',
pkglist=list_file, skip_stat=True, update=True,
update_md_path=self.topdir + '/work/global/repo', deltas=True,
oldpackagedirs=self.topdir + '/old/test-1.0-20151203.0/compose/Server/source/tree')])
with open(list_file) as f:
self.assertItemsEqual(
f.read().strip().split('\n'),
['../SRPMS/b/bash-4.3.30-2.fc21.src.rpm'])
@mock.patch('pungi.phases.createrepo.run')
@mock.patch('pungi.phases.createrepo.CreaterepoWrapper')
def test_variant_repo_debug_with_deltas(self, CreaterepoWrapperCls, run):
compose = DummyCompose(self.topdir, {
'createrepo_checksum': 'sha256',
'createrepo_deltas': True,
})
compose.DEBUG = False
compose.has_comps = False
compose.old_composes = [self.topdir + '/old']
touch(os.path.join(self.topdir, 'old', 'test-1.0-20151203.0', 'STATUS'), 'FINISHED')
repo = CreaterepoWrapperCls.return_value
copy_fixture('server-rpms.json', compose.paths.compose.metadata('rpms.json'))
create_variant_repo(compose, 'x86_64', compose.variants['Server'], 'debuginfo')
list_file = self.topdir + '/work/x86_64/repo_package_list/Server.x86_64.debuginfo.conf'
self.assertEqual(CreaterepoWrapperCls.mock_calls[0],
mock.call(createrepo_c=True))
self.assertItemsEqual(
repo.get_createrepo_cmd.mock_calls,
[mock.call(self.topdir + '/compose/Server/x86_64/debug/tree', checksum='sha256',
database=True, groupfile=None, workers=3,
outputdir=self.topdir + '/compose/Server/x86_64/debug/tree',
pkglist=list_file, skip_stat=True, update=True,
update_md_path=self.topdir + '/work/x86_64/repo', deltas=True,
oldpackagedirs=self.topdir + '/old/test-1.0-20151203.0/compose/Server/x86_64/debug/tree')])
with open(list_file) as f:
self.assertEqual(f.read(), 'Packages/b/bash-debuginfo-4.3.30-2.fc21.x86_64.rpm\n')
if __name__ == "__main__":
unittest.main()