From 2fa1f09827b3accf5fd412ad1cf5d23d8c14af55 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lubom=C3=ADr=20Sedl=C3=A1=C5=99?= Date: Wed, 8 Aug 2018 16:15:51 +0200 Subject: [PATCH] createrepo: Allow passing arbitrary arguments MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This could be used to enable zchunk generation, which can require up to 4 different options. Instead of hardcoding every single one, let's just allow more direct access to the executed command. Signed-off-by: Lubomír Sedlář --- doc/configuration.rst | 5 +++++ pungi/checks.py | 7 +++++++ pungi/phases/createrepo.py | 3 ++- pungi/wrappers/createrepo.py | 6 +++++- tests/test_createrepo_wrapper.py | 5 +++-- tests/test_createrepophase.py | 36 ++++++++++++++++---------------- 6 files changed, 40 insertions(+), 22 deletions(-) diff --git a/doc/configuration.rst b/doc/configuration.rst index 4ac0be18..a83ae71d 100644 --- a/doc/configuration.rst +++ b/doc/configuration.rst @@ -423,6 +423,11 @@ Options repo. Default value depends on gather backend. For DNF it's turned off, for Yum the default is ``True``. +**createrepo_extra_args** + (*[str]*) -- a list of extra arguments passed on to ``createrepo`` or + ``createrepo_c`` executable. This could be useful for enabling zchunk + generation and pointing it to correct dictionaries. + **product_id** = None (:ref:`scm_dict `) -- If specified, it should point to a directory with certificates ``--*.pem``. This diff --git a/pungi/checks.py b/pungi/checks.py index 9c8bea78..53fc615e 100644 --- a/pungi/checks.py +++ b/pungi/checks.py @@ -638,6 +638,13 @@ def make_schema(): "createrepo_database": { "type": "boolean", }, + "createrepo_extra_args": { + "type": "array", + "default": [], + "items": { + "type": "string", + }, + }, "repoclosure_strictness": _variant_arch_mapping({ "type": "string", "default": "lenient", diff --git a/pungi/phases/createrepo.py b/pungi/phases/createrepo.py index 96aab927..3a06a024 100644 --- a/pungi/phases/createrepo.py +++ b/pungi/phases/createrepo.py @@ -194,7 +194,8 @@ def create_variant_repo(compose, arch, variant, pkg_type, modules_metadata=None) checksum=createrepo_checksum, deltas=with_deltas, oldpackagedirs=old_package_dirs, - use_xz=compose.conf['createrepo_use_xz']) + use_xz=compose.conf['createrepo_use_xz'], + extra_args=compose.conf["createrepo_extra_args"]) log_file = compose.paths.log.log_file(arch, "createrepo-%s.%s" % (variant, pkg_type)) run(cmd, logfile=log_file, show_cmd=True) diff --git a/pungi/wrappers/createrepo.py b/pungi/wrappers/createrepo.py index f856afe1..4e664256 100644 --- a/pungi/wrappers/createrepo.py +++ b/pungi/wrappers/createrepo.py @@ -33,7 +33,8 @@ class CreaterepoWrapper(object): update_md_path=None, skip_stat=False, checkts=False, split=False, pretty=True, database=True, checksum=None, unique_md_filenames=True, distro=None, content=None, repo=None, revision=None, deltas=False, - oldpackagedirs=None, num_deltas=None, workers=None, use_xz=False): + oldpackagedirs=None, num_deltas=None, workers=None, use_xz=False, + extra_args=None): # groupfile = /path/to/comps.xml cmd = [self.createrepo, directory] @@ -117,6 +118,9 @@ class CreaterepoWrapper(object): if use_xz: cmd.append("--xz") + if extra_args: + cmd.extend(force_list(extra_args)) + return cmd def get_mergerepo_cmd(self, outputdir, repos, database=True, pkglist=None, nogroups=False, noupdateinfo=None): diff --git a/tests/test_createrepo_wrapper.py b/tests/test_createrepo_wrapper.py index d9d615f5..3b75b53e 100644 --- a/tests/test_createrepo_wrapper.py +++ b/tests/test_createrepo_wrapper.py @@ -32,7 +32,8 @@ class CreateRepoWrapperTest(unittest.TestCase): split=True, pretty=False, database=False, checksum='sha256', unique_md_filenames=False, distro='Fedora', content=['c1', 'c2'], repo=['r1', 'r2'], revision='rev', deltas=True, oldpackagedirs='/test/old', num_deltas=2, workers=3, outputdir='/test/output', - use_xz=True + use_xz=True, + extra_args=["--zck", "--zck-primary-dict=/foo/bar"], ) self.maxDiff = None @@ -44,7 +45,7 @@ class CreateRepoWrapperTest(unittest.TestCase): '--checksum=sha256', '--distro=Fedora', '--simple-md-filenames', '--no-database', '--content=c1', '--content=c2', '--repo=r1', '--repo=r2', '--revision=rev', '--deltas', '--oldpackagedirs=/test/old', '--num-deltas=2', '--workers=3', - '--outputdir=/test/output', '--xz']) + '--outputdir=/test/output', '--xz', "--zck", "--zck-primary-dict=/foo/bar"]) def test_get_createrepo_cmd_minimal(self): repo = CreaterepoWrapper(False) diff --git a/tests/test_createrepophase.py b/tests/test_createrepophase.py index 34f5b38f..e4f982cb 100644 --- a/tests/test_createrepophase.py +++ b/tests/test_createrepophase.py @@ -147,7 +147,7 @@ class TestCreateVariantRepo(PungiTestCase): outputdir=self.topdir + '/compose/Server/x86_64/os', pkglist=list_file, skip_stat=True, update=True, update_md_path=self.topdir + '/work/x86_64/repo', - deltas=False, oldpackagedirs=None, use_xz=False)]) + deltas=False, oldpackagedirs=None, use_xz=False, extra_args=[])]) self.assertItemsEqual( repo.get_modifyrepo_cmd.mock_calls, []) @@ -179,7 +179,7 @@ class TestCreateVariantRepo(PungiTestCase): outputdir=self.topdir + '/compose/Server/x86_64/os', pkglist=list_file, skip_stat=True, update=True, update_md_path=self.topdir + '/work/x86_64/repo', - deltas=False, oldpackagedirs=None, use_xz=False)]) + deltas=False, oldpackagedirs=None, use_xz=False, extra_args=[])]) self.assertItemsEqual( repo.get_modifyrepo_cmd.mock_calls, []) @@ -210,7 +210,7 @@ class TestCreateVariantRepo(PungiTestCase): outputdir=self.topdir + '/compose/Server/source/tree', pkglist=list_file, skip_stat=True, update=True, update_md_path=self.topdir + '/work/global/repo', - deltas=False, oldpackagedirs=None, use_xz=False)]) + deltas=False, oldpackagedirs=None, use_xz=False, extra_args=[])]) self.assertItemsEqual( repo.get_modifyrepo_cmd.mock_calls, []) @@ -244,7 +244,7 @@ class TestCreateVariantRepo(PungiTestCase): outputdir=self.topdir + '/compose/Server/x86_64/debug/tree', pkglist=list_file, skip_stat=True, update=True, update_md_path=self.topdir + '/work/x86_64/repo', - deltas=False, oldpackagedirs=None, use_xz=False)]) + deltas=False, oldpackagedirs=None, use_xz=False, extra_args=[])]) self.assertItemsEqual( repo.get_modifyrepo_cmd.mock_calls, []) @@ -276,7 +276,7 @@ class TestCreateVariantRepo(PungiTestCase): outputdir=self.topdir + '/compose/Server/x86_64/os', pkglist=list_file, skip_stat=True, update=True, update_md_path=self.topdir + '/work/x86_64/repo', - deltas=False, oldpackagedirs=None, use_xz=False)]) + deltas=False, oldpackagedirs=None, use_xz=False, extra_args=[])]) self.assertItemsEqual( repo.get_modifyrepo_cmd.mock_calls, []) @@ -310,7 +310,7 @@ class TestCreateVariantRepo(PungiTestCase): outputdir=self.topdir + '/compose/Server/x86_64/os', pkglist=list_file, skip_stat=True, update=True, update_md_path=self.topdir + '/work/x86_64/repo', - deltas=False, oldpackagedirs=None, use_xz=False)]) + deltas=False, oldpackagedirs=None, use_xz=False, extra_args=[])]) self.assertItemsEqual( repo.get_modifyrepo_cmd.mock_calls, []) @@ -342,7 +342,7 @@ class TestCreateVariantRepo(PungiTestCase): outputdir=self.topdir + '/compose/Server/x86_64/os', pkglist=list_file, skip_stat=True, update=True, update_md_path=self.topdir + '/work/x86_64/repo', deltas=False, - oldpackagedirs=None, use_xz=True)]) + oldpackagedirs=None, use_xz=True, extra_args=[])]) self.assertItemsEqual( repo.get_modifyrepo_cmd.mock_calls, []) @@ -377,7 +377,7 @@ class TestCreateVariantRepo(PungiTestCase): pkglist=list_file, skip_stat=True, update=True, update_md_path=None, deltas=True, oldpackagedirs=self.topdir + '/old/test-1.0-20151203.0/compose/Server/x86_64/os/Packages', - use_xz=False)]) + use_xz=False, extra_args=[])]) self.assertItemsEqual( repo.get_modifyrepo_cmd.mock_calls, []) @@ -412,7 +412,7 @@ class TestCreateVariantRepo(PungiTestCase): pkglist=list_file, skip_stat=True, update=True, update_md_path=None, deltas=True, oldpackagedirs=self.topdir + '/old/test-1.0-20151203.0/compose/Server/x86_64/os/Packages', - use_xz=False)]) + use_xz=False, extra_args=[])]) self.assertItemsEqual( repo.get_modifyrepo_cmd.mock_calls, []) @@ -446,7 +446,7 @@ class TestCreateVariantRepo(PungiTestCase): outputdir=self.topdir + '/compose/Server/x86_64/os', pkglist=list_file, skip_stat=True, update=True, update_md_path=self.topdir + '/work/x86_64/repo', - deltas=False, oldpackagedirs=None, use_xz=False)]) + deltas=False, oldpackagedirs=None, use_xz=False, extra_args=[])]) self.assertItemsEqual( repo.get_modifyrepo_cmd.mock_calls, []) @@ -481,7 +481,7 @@ class TestCreateVariantRepo(PungiTestCase): outputdir=self.topdir + '/compose/Server/x86_64/os', pkglist=list_file, skip_stat=True, update=True, update_md_path=self.topdir + '/work/x86_64/repo', - deltas=False, oldpackagedirs=None, use_xz=False)]) + deltas=False, oldpackagedirs=None, use_xz=False, extra_args=[])]) self.assertItemsEqual( repo.get_modifyrepo_cmd.mock_calls, []) @@ -524,7 +524,7 @@ class TestCreateVariantRepo(PungiTestCase): self.topdir + '/old/test-1.0-20151203.0/compose/Server/x86_64/os/Packages/a', self.topdir + '/old/test-1.0-20151203.0/compose/Server/x86_64/os/Packages/b', ], - use_xz=False)]) + use_xz=False, extra_args=[])]) self.assertItemsEqual( repo.get_modifyrepo_cmd.mock_calls, []) @@ -561,7 +561,7 @@ class TestCreateVariantRepo(PungiTestCase): pkglist=list_file, skip_stat=True, update=True, update_md_path=self.topdir + '/work/x86_64/repo', deltas=True, oldpackagedirs=[], - use_xz=False)]) + use_xz=False, extra_args=[])]) self.assertItemsEqual( repo.get_modifyrepo_cmd.mock_calls, []) @@ -596,7 +596,7 @@ class TestCreateVariantRepo(PungiTestCase): outputdir=self.topdir + '/compose/Server/source/tree', pkglist=list_file, skip_stat=True, update=True, update_md_path=self.topdir + '/work/global/repo', - deltas=False, oldpackagedirs=None, use_xz=False)]) + deltas=False, oldpackagedirs=None, use_xz=False, extra_args=[])]) self.assertItemsEqual( repo.get_modifyrepo_cmd.mock_calls, []) @@ -633,7 +633,7 @@ class TestCreateVariantRepo(PungiTestCase): outputdir=self.topdir + '/compose/Server/x86_64/debug/tree', pkglist=list_file, skip_stat=True, update=True, update_md_path=self.topdir + '/work/x86_64/repo', - deltas=False, oldpackagedirs=None, use_xz=False)]) + deltas=False, oldpackagedirs=None, use_xz=False, extra_args=[])]) self.assertItemsEqual( repo.get_modifyrepo_cmd.mock_calls, []) @@ -673,7 +673,7 @@ class TestCreateVariantRepo(PungiTestCase): pkglist=list_file, skip_stat=True, update=True, update_md_path=self.topdir + '/work/x86_64/repo', deltas=False, oldpackagedirs=None, - use_xz=False)]) + use_xz=False, extra_args=[])]) self.assertItemsEqual( repo.get_modifyrepo_cmd.mock_calls, [mock.call(repodata_dir, product_id, compress_type='gz')]) @@ -707,7 +707,7 @@ class TestCreateVariantRepo(PungiTestCase): pkglist=list_file, skip_stat=True, update=True, update_md_path=self.topdir + '/work/x86_64/repo', deltas=False, oldpackagedirs=None, - use_xz=False)]) + use_xz=False, extra_args=[])]) self.assertItemsEqual(repo.get_modifyrepo_cmd.mock_calls, []) with open(list_file) as f: self.assertEqual(f.read(), 'Packages/b/bash-debuginfo-4.3.30-2.fc21.x86_64.rpm\n') @@ -739,7 +739,7 @@ class TestCreateVariantRepo(PungiTestCase): pkglist=list_file, skip_stat=True, update=True, update_md_path=self.topdir + '/work/global/repo', deltas=False, oldpackagedirs=None, - use_xz=False)]) + use_xz=False, extra_args=[])]) self.assertItemsEqual(repo.get_modifyrepo_cmd.mock_calls, []) with open(list_file) as f: self.assertEqual(f.read(), 'Packages/b/bash-4.3.30-2.fc21.src.rpm\n')