From 27375788c2962c63c1726712bfc17e022ec4197d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lubom=C3=ADr=20Sedl=C3=A1=C5=99?= Date: Mon, 29 Feb 2016 12:58:21 +0100 Subject: [PATCH] [createrepo] Compute delta RPMS against old compose MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Lubomír Sedlář --- doc/configuration.rst | 4 ++ pungi/phases/createrepo.py | 44 ++++++++++-- tests/helpers.py | 2 + tests/test_createrepophase.py | 124 ++++++++++++++++++++++++++++++++-- 4 files changed, 164 insertions(+), 10 deletions(-) diff --git a/doc/configuration.rst b/doc/configuration.rst index 00e92473..2e161c08 100644 --- a/doc/configuration.rst +++ b/doc/configuration.rst @@ -336,6 +336,10 @@ Options **createrepo_c** = True (*bool*) -- use createrepo_c (True) or legacy createrepo (False) +**createrepo_deltas** = False + (*bool*) -- generate delta RPMs against an older compose. This needs to be + used together with `--old-composes`` command line argument. + Example diff --git a/pungi/phases/createrepo.py b/pungi/phases/createrepo.py index 568b4951..761b55f1 100644 --- a/pungi/phases/createrepo.py +++ b/pungi/phases/createrepo.py @@ -29,9 +29,10 @@ import threading from kobo.threads import ThreadPool, WorkerThread from kobo.shortcuts import run, relative_path -from pungi.wrappers.scm import get_dir_from_scm -from pungi.wrappers.createrepo import CreaterepoWrapper -from pungi.phases.base import PhaseBase +from ..wrappers.scm import get_dir_from_scm +from ..wrappers.createrepo import CreaterepoWrapper +from .base import PhaseBase +from ..util import find_old_compose import productmd.rpms @@ -54,6 +55,11 @@ class CreaterepoPhase(PhaseBase): "expected_types": [str], "expected_values": ["sha256", "sha"], }, + { + "name": "createrepo_deltas", + "expected_types": [bool], + "optional": True, + }, { "name": "product_id", "expected_types": [dict], @@ -70,6 +76,19 @@ class CreaterepoPhase(PhaseBase): PhaseBase.__init__(self, compose) self.pool = ThreadPool(logger=self.compose._logger) + def validate(self): + errors = [] + try: + super(CreaterepoPhase, self).validate() + except ValueError as exc: + errors = exc.message.split('\n') + + if not self.compose.old_composes and 'createrepo_deltas' in self.compose.conf: + errors.append('Can not generate deltas without old compose') + + if errors: + raise ValueError('\n'.join(errors)) + def run(self): get_productids_from_scm(self.compose) for i in range(3): @@ -102,6 +121,7 @@ def create_variant_repo(compose, arch, variant, pkg_type): createrepo_c = compose.conf.get("createrepo_c", True) createrepo_checksum = compose.conf["createrepo_checksum"] + createrepo_deltas = compose.conf.get("createrepo_deltas", False) repo = CreaterepoWrapper(createrepo_c=createrepo_c) repo_dir_arch = compose.paths.work.arch_repo(arch='global' if pkg_type == 'srpm' else arch) @@ -151,13 +171,29 @@ def create_variant_repo(compose, arch, variant, pkg_type): for rel_path in sorted(rpms): f.write("%s\n" % rel_path) + old_packages_dir = None + if createrepo_deltas: + old_compose_path = find_old_compose( + compose.old_composes, + compose.ci_base.release.short, + compose.ci_base.release.version, + compose.ci_base.base_product.short if compose.ci_base.release.is_layered else None, + compose.ci_base.base_product.version if compose.ci_base.release.is_layered else None + ) + if not old_compose_path: + compose.log_info("No suitable old compose found in: %s" % compose.old_composes) + else: + rel_dir = relative_path(repo_dir, compose.topdir.rstrip('/') + '/') + old_packages_dir = os.path.join(old_compose_path, rel_dir) + comps_path = None if compose.has_comps and pkg_type == "rpm": comps_path = compose.paths.work.comps(arch=arch, variant=variant) cmd = repo.get_createrepo_cmd(repo_dir, update=True, database=True, skip_stat=True, pkglist=file_list, outputdir=repo_dir, workers=3, groupfile=comps_path, update_md_path=repo_dir_arch, - checksum=createrepo_checksum) + checksum=createrepo_checksum, deltas=createrepo_deltas, + oldpackagedirs=old_packages_dir) log_file = compose.paths.log.log_file(arch, "createrepo-%s" % variant) run(cmd, logfile=log_file, show_cmd=True) diff --git a/tests/helpers.py b/tests/helpers.py index e4071a17..b2048cb8 100644 --- a/tests/helpers.py +++ b/tests/helpers.py @@ -30,6 +30,7 @@ class DummyCompose(object): release=mock.Mock( short='test', version='1.0', + is_layered=False, ), ) self.topdir = topdir @@ -51,6 +52,7 @@ class DummyCompose(object): self.get_image_name = mock.Mock(return_value='image-name') self.image = mock.Mock(path='Client/i386/iso/image.iso') self.im = mock.Mock(images={'Client': {'i386': [self.image]}}) + self.old_composes = [] def get_variants(self, arch=None, types=None): return [v for v in self.variants.values() if not arch or arch in v.arches] diff --git a/tests/test_createrepophase.py b/tests/test_createrepophase.py index 5f9e519a..a1084d91 100755 --- a/tests/test_createrepophase.py +++ b/tests/test_createrepophase.py @@ -11,10 +11,22 @@ import sys sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..")) from pungi.phases.createrepo import CreaterepoPhase, create_variant_repo -from tests.helpers import DummyCompose, PungiTestCase, copy_fixture +from tests.helpers import DummyCompose, PungiTestCase, copy_fixture, touch class TestCreaterepoPhase(PungiTestCase): + @mock.patch('pungi.phases.createrepo.ThreadPool') + def test_fails_deltas_without_old_compose(self, ThreadPoolCls): + compose = DummyCompose(self.topdir, { + 'createrepo_checksum': 'sha256', + 'createrepo_deltas': True, + }) + + phase = CreaterepoPhase(compose) + with self.assertRaises(ValueError) as ctx: + phase.validate() + + self.assertIn('deltas', str(ctx.exception)) @mock.patch('pungi.phases.createrepo.ThreadPool') def test_starts_jobs(self, ThreadPoolCls): @@ -94,7 +106,8 @@ class TestCreateRepoThread(PungiTestCase): database=True, groupfile=None, workers=3, outputdir=self.topdir + '/compose/Server/x86_64/os', pkglist=list_file, skip_stat=True, update=True, - update_md_path=self.topdir + '/work/x86_64/repo')]) + update_md_path=self.topdir + '/work/x86_64/repo', + deltas=False, oldpackagedirs=None)]) with open(list_file) as f: self.assertEqual(f.read(), 'Packages/b/bash-4.3.30-2.fc21.x86_64.rpm\n') @@ -121,7 +134,8 @@ class TestCreateRepoThread(PungiTestCase): database=True, groupfile=None, workers=3, outputdir=self.topdir + '/compose/Server/source/tree', pkglist=list_file, skip_stat=True, update=True, - update_md_path=self.topdir + '/work/global/repo')]) + update_md_path=self.topdir + '/work/global/repo', + deltas=False, oldpackagedirs=None)]) with open(list_file) as f: self.assertItemsEqual( f.read().strip().split('\n'), @@ -151,7 +165,8 @@ class TestCreateRepoThread(PungiTestCase): database=True, groupfile=None, workers=3, outputdir=self.topdir + '/compose/Server/x86_64/debug/tree', pkglist=list_file, skip_stat=True, update=True, - update_md_path=self.topdir + '/work/x86_64/repo')]) + update_md_path=self.topdir + '/work/x86_64/repo', + deltas=False, oldpackagedirs=None)]) with open(list_file) as f: self.assertEqual(f.read(), 'Packages/b/bash-debuginfo-4.3.30-2.fc21.x86_64.rpm\n') @@ -179,7 +194,8 @@ class TestCreateRepoThread(PungiTestCase): database=True, groupfile=None, workers=3, outputdir=self.topdir + '/compose/Server/x86_64/os', pkglist=list_file, skip_stat=True, update=True, - update_md_path=self.topdir + '/work/x86_64/repo')]) + update_md_path=self.topdir + '/work/x86_64/repo', + deltas=False, oldpackagedirs=None)]) with open(list_file) as f: self.assertEqual(f.read(), 'Packages/b/bash-4.3.30-2.fc21.x86_64.rpm\n') @@ -208,10 +224,106 @@ class TestCreateRepoThread(PungiTestCase): database=True, groupfile=None, workers=3, outputdir=self.topdir + '/compose/Server/x86_64/os', pkglist=list_file, skip_stat=True, update=True, - update_md_path=self.topdir + '/work/x86_64/repo')]) + update_md_path=self.topdir + '/work/x86_64/repo', + deltas=False, oldpackagedirs=None)]) with open(list_file) as f: self.assertEqual(f.read(), 'Packages/b/bash-4.3.30-2.fc21.x86_64.rpm\n') + @mock.patch('pungi.phases.createrepo.run') + @mock.patch('pungi.phases.createrepo.CreaterepoWrapper') + def test_variant_repo_rpms_with_deltas(self, CreaterepoWrapperCls, run): + compose = DummyCompose(self.topdir, { + 'createrepo_checksum': 'sha256', + 'createrepo_deltas': True, + }) + compose.DEBUG = False + compose.has_comps = False + compose.old_composes = [self.topdir + '/old'] + touch(os.path.join(self.topdir, 'old', 'test-1.0-20151203.0', 'STATUS'), 'FINISHED') + + repo = CreaterepoWrapperCls.return_value + copy_fixture('server-rpms.json', compose.paths.compose.metadata('rpms.json')) + + create_variant_repo(compose, 'x86_64', compose.variants['Server'], 'rpm') + + list_file = self.topdir + '/work/x86_64/repo_package_list/Server.x86_64.rpm.conf' + self.assertEqual(CreaterepoWrapperCls.mock_calls[0], + mock.call(createrepo_c=True)) + self.assertItemsEqual( + repo.get_createrepo_cmd.mock_calls, + [mock.call(self.topdir + '/compose/Server/x86_64/os', checksum='sha256', + database=True, groupfile=None, workers=3, + outputdir=self.topdir + '/compose/Server/x86_64/os', + pkglist=list_file, skip_stat=True, update=True, + update_md_path=self.topdir + '/work/x86_64/repo', deltas=True, + oldpackagedirs=self.topdir + '/old/test-1.0-20151203.0/compose/Server/x86_64/os')]) + with open(list_file) as f: + self.assertEqual(f.read(), 'Packages/b/bash-4.3.30-2.fc21.x86_64.rpm\n') + + @mock.patch('pungi.phases.createrepo.run') + @mock.patch('pungi.phases.createrepo.CreaterepoWrapper') + def test_variant_repo_source_with_deltas(self, CreaterepoWrapperCls, run): + compose = DummyCompose(self.topdir, { + 'createrepo_checksum': 'sha256', + 'createrepo_deltas': True, + }) + compose.DEBUG = False + compose.has_comps = False + compose.old_composes = [self.topdir + '/old'] + touch(os.path.join(self.topdir, 'old', 'test-1.0-20151203.0', 'STATUS'), 'FINISHED') + + repo = CreaterepoWrapperCls.return_value + copy_fixture('server-rpms.json', compose.paths.compose.metadata('rpms.json')) + + create_variant_repo(compose, None, compose.variants['Server'], 'srpm') + + list_file = self.topdir + '/work/global/repo_package_list/Server.None.srpm.conf' + self.assertEqual(CreaterepoWrapperCls.mock_calls[0], + mock.call(createrepo_c=True)) + self.assertItemsEqual( + repo.get_createrepo_cmd.mock_calls, + [mock.call(self.topdir + '/compose/Server/source/tree', checksum='sha256', + database=True, groupfile=None, workers=3, + outputdir=self.topdir + '/compose/Server/source/tree', + pkglist=list_file, skip_stat=True, update=True, + update_md_path=self.topdir + '/work/global/repo', deltas=True, + oldpackagedirs=self.topdir + '/old/test-1.0-20151203.0/compose/Server/source/tree')]) + with open(list_file) as f: + self.assertItemsEqual( + f.read().strip().split('\n'), + ['../SRPMS/b/bash-4.3.30-2.fc21.src.rpm']) + + @mock.patch('pungi.phases.createrepo.run') + @mock.patch('pungi.phases.createrepo.CreaterepoWrapper') + def test_variant_repo_debug_with_deltas(self, CreaterepoWrapperCls, run): + compose = DummyCompose(self.topdir, { + 'createrepo_checksum': 'sha256', + 'createrepo_deltas': True, + }) + compose.DEBUG = False + compose.has_comps = False + compose.old_composes = [self.topdir + '/old'] + touch(os.path.join(self.topdir, 'old', 'test-1.0-20151203.0', 'STATUS'), 'FINISHED') + + repo = CreaterepoWrapperCls.return_value + copy_fixture('server-rpms.json', compose.paths.compose.metadata('rpms.json')) + + create_variant_repo(compose, 'x86_64', compose.variants['Server'], 'debuginfo') + + list_file = self.topdir + '/work/x86_64/repo_package_list/Server.x86_64.debuginfo.conf' + self.assertEqual(CreaterepoWrapperCls.mock_calls[0], + mock.call(createrepo_c=True)) + self.assertItemsEqual( + repo.get_createrepo_cmd.mock_calls, + [mock.call(self.topdir + '/compose/Server/x86_64/debug/tree', checksum='sha256', + database=True, groupfile=None, workers=3, + outputdir=self.topdir + '/compose/Server/x86_64/debug/tree', + pkglist=list_file, skip_stat=True, update=True, + update_md_path=self.topdir + '/work/x86_64/repo', deltas=True, + oldpackagedirs=self.topdir + '/old/test-1.0-20151203.0/compose/Server/x86_64/debug/tree')]) + with open(list_file) as f: + self.assertEqual(f.read(), 'Packages/b/bash-debuginfo-4.3.30-2.fc21.x86_64.rpm\n') + if __name__ == "__main__": unittest.main()