From b4e746aa71c31cb5312ca0309e1695c24fe422a1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lubom=C3=ADr=20Sedl=C3=A1=C5=99?= Date: Fri, 18 May 2018 10:17:41 +0200 Subject: [PATCH] createrepo: Allow disabling SQLite database MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This is an optimization for Yum. DNF does not care at all. The behaviour is configurable, but the default depends on gather backend, as that is what users should be using to consume the packages from the repo. Fixes: https://pagure.io/pungi/issue/951 Signed-off-by: Lubomír Sedlář --- doc/configuration.rst | 6 ++++++ pungi/checks.py | 3 +++ pungi/compose.py | 10 ++++++++++ pungi/phases/createrepo.py | 4 +++- tests/helpers.py | 1 + tests/test_compose.py | 18 ++++++++++++++++++ tests/test_createrepophase.py | 32 ++++++++++++++++++++++++++++++++ 7 files changed, 73 insertions(+), 1 deletion(-) diff --git a/doc/configuration.rst b/doc/configuration.rst index c8135093..35f28504 100644 --- a/doc/configuration.rst +++ b/doc/configuration.rst @@ -412,6 +412,12 @@ Options **createrepo_num_workers** (*int*) -- how many concurrent ``createrepo`` workers to run. Value defaults to 3. +**createrepo_database** + (*bool*) -- whether to create SQLite database as part of the repodata. This + is only useful as an optimization for clients using Yum to consume to the + repo. Default value depends on gather backend. For DNF it's turned off, for + Yum the default is ``True``. + **product_id** = None (:ref:`scm_dict `) -- If specified, it should point to a directory with certificates ``--*.pem``. This diff --git a/pungi/checks.py b/pungi/checks.py index 7d5e01c4..6a45133d 100644 --- a/pungi/checks.py +++ b/pungi/checks.py @@ -627,6 +627,9 @@ def make_schema(): "type": "number", "default": 3, }, + "createrepo_database": { + "type": "boolean", + }, "repoclosure_strictness": _variant_arch_mapping({ "type": "string", "default": "lenient", diff --git a/pungi/compose.py b/pungi/compose.py index 622193be..d884ad21 100644 --- a/pungi/compose.py +++ b/pungi/compose.py @@ -201,6 +201,16 @@ class Compose(kobo.log.LoggingBase): def config_dir(self): return os.path.dirname(self.conf._open_file or "") + @property + def should_create_yum_database(self): + """Explicit configuration trumps all. Otherwise check gather backend + and only create it for Yum. + """ + config = self.conf.get('createrepo_database') + if config is not None: + return config + return self.conf['gather_backend'] == 'yum' + def read_variants(self): # TODO: move to phases/init ? variants_file = self.paths.work.variants_file(arch="global") diff --git a/pungi/phases/createrepo.py b/pungi/phases/createrepo.py index b45efdd3..721f9bfe 100644 --- a/pungi/phases/createrepo.py +++ b/pungi/phases/createrepo.py @@ -163,7 +163,9 @@ def create_variant_repo(compose, arch, variant, pkg_type, modules_metadata=None) comps_path = None if compose.has_comps and pkg_type == "rpm": comps_path = compose.paths.work.comps(arch=arch, variant=variant) - cmd = repo.get_createrepo_cmd(repo_dir, update=True, database=True, skip_stat=True, + cmd = repo.get_createrepo_cmd(repo_dir, update=True, + database=compose.should_create_yum_database, + skip_stat=True, pkglist=file_list, outputdir=repo_dir, workers=compose.conf["createrepo_num_workers"], groupfile=comps_path, update_md_path=repo_dir_arch, diff --git a/tests/helpers.py b/tests/helpers.py index 0a5f8c4e..963acdff 100644 --- a/tests/helpers.py +++ b/tests/helpers.py @@ -149,6 +149,7 @@ class DummyCompose(object): self.attempt_deliverable = mock.Mock() self.fail_deliverable = mock.Mock() self.require_deliverable = mock.Mock() + self.should_create_yum_database = True def setup_optional(self): self.all_variants['Server-optional'] = MockVariant( diff --git a/tests/test_compose.py b/tests/test_compose.py index d6dcfc08..d7969a44 100644 --- a/tests/test_compose.py +++ b/tests/test_compose.py @@ -481,6 +481,24 @@ class StatusTest(unittest.TestCase): self.assertTrue(self.compose.notifier.send.call_count, 1) + def test_no_database_with_dnf_backend(self): + self.compose.conf['gather_backend'] = 'dnf' + self.assertFalse(self.compose.should_create_yum_database) + + def test_no_database_with_dnf_backend_config_override(self): + self.compose.conf['gather_backend'] = 'dnf' + self.compose.conf['createrepo_database'] = True + self.assertTrue(self.compose.should_create_yum_database) + + def test_no_database_with_yum_backend(self): + self.compose.conf['gather_backend'] = 'yum' + self.assertTrue(self.compose.should_create_yum_database) + + def test_no_database_with_yum_backend_config_override(self): + self.compose.conf['gather_backend'] = 'yum' + self.compose.conf['createrepo_database'] = False + self.assertFalse(self.compose.should_create_yum_database) + if __name__ == "__main__": unittest.main() diff --git a/tests/test_createrepophase.py b/tests/test_createrepophase.py index fa46272b..6f14196f 100644 --- a/tests/test_createrepophase.py +++ b/tests/test_createrepophase.py @@ -151,6 +151,38 @@ class TestCreateVariantRepo(PungiTestCase): with open(list_file) as f: self.assertEqual(f.read(), 'Packages/b/bash-4.3.30-2.fc21.x86_64.rpm\n') + @mock.patch('pungi.phases.createrepo.run') + @mock.patch('pungi.phases.createrepo.CreaterepoWrapper') + def test_variant_repo_rpms_without_database(self, CreaterepoWrapperCls, run): + compose = DummyCompose(self.topdir, { + 'createrepo_checksum': 'sha256', + }) + compose.should_create_yum_database = False + compose.DEBUG = False + compose.has_comps = False + + repo = CreaterepoWrapperCls.return_value + copy_fixture('server-rpms.json', compose.paths.compose.metadata('rpms.json')) + + create_variant_repo(compose, 'x86_64', compose.variants['Server'], 'rpm') + + list_file = self.topdir + '/work/x86_64/repo_package_list/Server.x86_64.rpm.conf' + self.assertEqual(CreaterepoWrapperCls.mock_calls[0], + mock.call(createrepo_c=True)) + self.assertItemsEqual( + repo.get_createrepo_cmd.mock_calls, + [mock.call(self.topdir + '/compose/Server/x86_64/os', checksum='sha256', + database=False, groupfile=None, workers=3, + outputdir=self.topdir + '/compose/Server/x86_64/os', + pkglist=list_file, skip_stat=True, update=True, + update_md_path=self.topdir + '/work/x86_64/repo', + deltas=False, oldpackagedirs=None, use_xz=False)]) + self.assertItemsEqual( + repo.get_modifyrepo_cmd.mock_calls, + []) + with open(list_file) as f: + self.assertEqual(f.read(), 'Packages/b/bash-4.3.30-2.fc21.x86_64.rpm\n') + @mock.patch('pungi.phases.createrepo.run') @mock.patch('pungi.phases.createrepo.CreaterepoWrapper') def test_variant_repo_source(self, CreaterepoWrapperCls, run):