From bf46048cbf393b96ec07e2e2dcbb34336c48dc97 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lubom=C3=ADr=20Sedl=C3=A1=C5=99?= Date: Mon, 12 Sep 2016 11:01:45 +0200 Subject: [PATCH] createrepo: Add option to use xz compression MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This patch adds a new config option createrepo_use_xz, which when set to true will cause createrepo to compress sqlite databases with xz. The default setting is False. Fixes: #387 Signed-off-by: Lubomír Sedlář --- doc/configuration.rst | 4 +++ pungi/checks.py | 4 +++ pungi/phases/createrepo.py | 3 +- pungi/wrappers/createrepo.py | 5 +++- tests/test_createrepo_wrapper.py | 5 ++-- tests/test_createrepophase.py | 48 ++++++++++++++++++++++++++------ 6 files changed, 57 insertions(+), 12 deletions(-) diff --git a/doc/configuration.rst b/doc/configuration.rst index 3f4b9d53..481eeea4 100644 --- a/doc/configuration.rst +++ b/doc/configuration.rst @@ -362,6 +362,10 @@ Options (*bool*) -- generate delta RPMs against an older compose. This needs to be used together with `--old-composes`` command line argument. +**createrepo_use_xz** = False + (*bool*) -- whether to pass ``--xz`` to the createrepo command. This will + cause the SQLite databases to be compressed with xz. + **product_id** = None (*scm_dict*) -- If specified, it should point to a directory with certificates ``--*.pem``. This certificate will be diff --git a/pungi/checks.py b/pungi/checks.py index e9301c7b..2726826b 100644 --- a/pungi/checks.py +++ b/pungi/checks.py @@ -424,6 +424,10 @@ def _make_schema(): "type": "string", "enum": ["sha", "sha256"], }, + "createrepo_use_xz": { + "type": "boolean", + "default": False, + }, "hashed_directories": { "type": "boolean", diff --git a/pungi/phases/createrepo.py b/pungi/phases/createrepo.py index 4ffb4130..1738b78f 100644 --- a/pungi/phases/createrepo.py +++ b/pungi/phases/createrepo.py @@ -165,7 +165,8 @@ def create_variant_repo(compose, arch, variant, pkg_type): pkglist=file_list, outputdir=repo_dir, workers=3, groupfile=comps_path, update_md_path=repo_dir_arch, checksum=createrepo_checksum, deltas=createrepo_deltas, - oldpackagedirs=old_packages_dir) + oldpackagedirs=old_packages_dir, + use_xz=compose.conf['createrepo_use_xz']) log_file = compose.paths.log.log_file(arch, "createrepo-%s.%s" % (variant, pkg_type)) run(cmd, logfile=log_file, show_cmd=True) diff --git a/pungi/wrappers/createrepo.py b/pungi/wrappers/createrepo.py index 0e80391c..1c601e86 100644 --- a/pungi/wrappers/createrepo.py +++ b/pungi/wrappers/createrepo.py @@ -33,7 +33,7 @@ class CreaterepoWrapper(object): update_md_path=None, skip_stat=False, checkts=False, split=False, pretty=True, database=True, checksum=None, unique_md_filenames=True, distro=None, content=None, repo=None, revision=None, deltas=False, - oldpackagedirs=None, num_deltas=None, workers=None): + oldpackagedirs=None, num_deltas=None, workers=None, use_xz=False): # groupfile = /path/to/comps.xml cmd = [self.createrepo, directory, '--verbose'] @@ -114,6 +114,9 @@ class CreaterepoWrapper(object): if workers: cmd.append("--workers=%d" % int(workers)) + if use_xz: + cmd.append("--xz") + return cmd def get_mergerepo_cmd(self, outputdir, repos, database=True, pkglist=None, nogroups=False, noupdateinfo=None): diff --git a/tests/test_createrepo_wrapper.py b/tests/test_createrepo_wrapper.py index 29dadfb0..1b32b09c 100755 --- a/tests/test_createrepo_wrapper.py +++ b/tests/test_createrepo_wrapper.py @@ -31,7 +31,8 @@ class CreateRepoWrapperTest(unittest.TestCase): update=False, update_md_path='/test/md_path', skip_stat=True, checkts=True, split=True, pretty=False, database=False, checksum='sha256', unique_md_filenames=False, distro='Fedora', content=['c1', 'c2'], repo=['r1', 'r2'], revision='rev', deltas=True, - oldpackagedirs='/test/old', num_deltas=2, workers=3, outputdir='/test/output' + oldpackagedirs='/test/old', num_deltas=2, workers=3, outputdir='/test/output', + use_xz=True ) self.maxDiff = None @@ -43,7 +44,7 @@ class CreateRepoWrapperTest(unittest.TestCase): '--checksum=sha256', '--distro=Fedora', '--simple-md-filenames', '--no-database', '--content=c1', '--content=c2', '--repo=r1', '--repo=r2', '--revision=rev', '--deltas', '--oldpackagedirs=/test/old', '--num-deltas=2', '--workers=3', - '--outputdir=/test/output', '--verbose']) + '--outputdir=/test/output', '--verbose', '--xz']) def test_get_createrepo_cmd_minimal(self): repo = CreaterepoWrapper(False) diff --git a/tests/test_createrepophase.py b/tests/test_createrepophase.py index 43593bad..2a894f9d 100755 --- a/tests/test_createrepophase.py +++ b/tests/test_createrepophase.py @@ -110,7 +110,7 @@ class TestCreateRepoThread(PungiTestCase): outputdir=self.topdir + '/compose/Server/x86_64/os', pkglist=list_file, skip_stat=True, update=True, update_md_path=self.topdir + '/work/x86_64/repo', - deltas=False, oldpackagedirs=None)]) + deltas=False, oldpackagedirs=None, use_xz=False)]) with open(list_file) as f: self.assertEqual(f.read(), 'Packages/b/bash-4.3.30-2.fc21.x86_64.rpm\n') @@ -138,7 +138,7 @@ class TestCreateRepoThread(PungiTestCase): outputdir=self.topdir + '/compose/Server/source/tree', pkglist=list_file, skip_stat=True, update=True, update_md_path=self.topdir + '/work/global/repo', - deltas=False, oldpackagedirs=None)]) + deltas=False, oldpackagedirs=None, use_xz=False)]) with open(list_file) as f: self.assertItemsEqual( f.read().strip().split('\n'), @@ -169,7 +169,7 @@ class TestCreateRepoThread(PungiTestCase): outputdir=self.topdir + '/compose/Server/x86_64/debug/tree', pkglist=list_file, skip_stat=True, update=True, update_md_path=self.topdir + '/work/x86_64/repo', - deltas=False, oldpackagedirs=None)]) + deltas=False, oldpackagedirs=None, use_xz=False)]) with open(list_file) as f: self.assertEqual(f.read(), 'Packages/b/bash-debuginfo-4.3.30-2.fc21.x86_64.rpm\n') @@ -198,7 +198,7 @@ class TestCreateRepoThread(PungiTestCase): outputdir=self.topdir + '/compose/Server/x86_64/os', pkglist=list_file, skip_stat=True, update=True, update_md_path=self.topdir + '/work/x86_64/repo', - deltas=False, oldpackagedirs=None)]) + deltas=False, oldpackagedirs=None, use_xz=False)]) with open(list_file) as f: self.assertEqual(f.read(), 'Packages/b/bash-4.3.30-2.fc21.x86_64.rpm\n') @@ -228,7 +228,36 @@ class TestCreateRepoThread(PungiTestCase): outputdir=self.topdir + '/compose/Server/x86_64/os', pkglist=list_file, skip_stat=True, update=True, update_md_path=self.topdir + '/work/x86_64/repo', - deltas=False, oldpackagedirs=None)]) + deltas=False, oldpackagedirs=None, use_xz=False)]) + with open(list_file) as f: + self.assertEqual(f.read(), 'Packages/b/bash-4.3.30-2.fc21.x86_64.rpm\n') + + @mock.patch('pungi.phases.createrepo.run') + @mock.patch('pungi.phases.createrepo.CreaterepoWrapper') + def test_variant_repo_rpms_with_xz(self, CreaterepoWrapperCls, run): + compose = DummyCompose(self.topdir, { + 'createrepo_checksum': 'sha256', + 'createrepo_use_xz': True, + }) + compose.DEBUG = False + compose.has_comps = False + + repo = CreaterepoWrapperCls.return_value + copy_fixture('server-rpms.json', compose.paths.compose.metadata('rpms.json')) + + create_variant_repo(compose, 'x86_64', compose.variants['Server'], 'rpm') + + list_file = self.topdir + '/work/x86_64/repo_package_list/Server.x86_64.rpm.conf' + self.assertEqual(CreaterepoWrapperCls.mock_calls[0], + mock.call(createrepo_c=True)) + self.assertItemsEqual( + repo.get_createrepo_cmd.mock_calls, + [mock.call(self.topdir + '/compose/Server/x86_64/os', checksum='sha256', + database=True, groupfile=None, workers=3, + outputdir=self.topdir + '/compose/Server/x86_64/os', + pkglist=list_file, skip_stat=True, update=True, + update_md_path=self.topdir + '/work/x86_64/repo', deltas=False, + oldpackagedirs=None, use_xz=True)]) with open(list_file) as f: self.assertEqual(f.read(), 'Packages/b/bash-4.3.30-2.fc21.x86_64.rpm\n') @@ -259,7 +288,8 @@ class TestCreateRepoThread(PungiTestCase): outputdir=self.topdir + '/compose/Server/x86_64/os', pkglist=list_file, skip_stat=True, update=True, update_md_path=self.topdir + '/work/x86_64/repo', deltas=True, - oldpackagedirs=self.topdir + '/old/test-1.0-20151203.0/compose/Server/x86_64/os')]) + oldpackagedirs=self.topdir + '/old/test-1.0-20151203.0/compose/Server/x86_64/os', + use_xz=False)]) with open(list_file) as f: self.assertEqual(f.read(), 'Packages/b/bash-4.3.30-2.fc21.x86_64.rpm\n') @@ -290,7 +320,8 @@ class TestCreateRepoThread(PungiTestCase): outputdir=self.topdir + '/compose/Server/source/tree', pkglist=list_file, skip_stat=True, update=True, update_md_path=self.topdir + '/work/global/repo', deltas=True, - oldpackagedirs=self.topdir + '/old/test-1.0-20151203.0/compose/Server/source/tree')]) + oldpackagedirs=self.topdir + '/old/test-1.0-20151203.0/compose/Server/source/tree', + use_xz=False)]) with open(list_file) as f: self.assertItemsEqual( f.read().strip().split('\n'), @@ -323,7 +354,8 @@ class TestCreateRepoThread(PungiTestCase): outputdir=self.topdir + '/compose/Server/x86_64/debug/tree', pkglist=list_file, skip_stat=True, update=True, update_md_path=self.topdir + '/work/x86_64/repo', deltas=True, - oldpackagedirs=self.topdir + '/old/test-1.0-20151203.0/compose/Server/x86_64/debug/tree')]) + oldpackagedirs=self.topdir + '/old/test-1.0-20151203.0/compose/Server/x86_64/debug/tree', + use_xz=False)]) with open(list_file) as f: self.assertEqual(f.read(), 'Packages/b/bash-debuginfo-4.3.30-2.fc21.x86_64.rpm\n')