createrepo: Allow disabling SQLite database

This is an optimization for Yum. DNF does not care at all.

The behaviour is configurable, but the default depends on gather
backend, as that is what users should be using to consume the packages
from the repo.

Fixes: https://pagure.io/pungi/issue/951
Signed-off-by: Lubomír Sedlář <lsedlar@redhat.com>

doc/configuration.rst

@@ -412,6 +412,12 @@ Options
 **createrepo_num_workers**
     (*int*) -- how many concurrent ``createrepo`` workers to run. Value defaults to 3.
 
+**createrepo_database**
+    (*bool*) -- whether to create SQLite database as part of the repodata. This
+    is only useful as an optimization for clients using Yum to consume to the
+    repo. Default value depends on gather backend. For DNF it's turned off, for
+    Yum the default is ``True``.
+
 **product_id** = None
     (:ref:`scm_dict <scm_support>`) -- If specified, it should point to a
     directory with certificates ``<variant_uid>-<arch>-*.pem``. This

pungi/checks.py

@@ -627,6 +627,9 @@ def make_schema():
                 "type": "number",
                 "default": 3,
             },
+            "createrepo_database": {
+                "type": "boolean",
+            },
             "repoclosure_strictness": _variant_arch_mapping({
                 "type": "string",
                 "default": "lenient",

pungi/compose.py

@@ -201,6 +201,16 @@ class Compose(kobo.log.LoggingBase):
     def config_dir(self):
         return os.path.dirname(self.conf._open_file or "")
 
+    @property
+    def should_create_yum_database(self):
+        """Explicit configuration trumps all. Otherwise check gather backend
+        and only create it for Yum.
+        """
+        config = self.conf.get('createrepo_database')
+        if config is not None:
+            return config
+        return self.conf['gather_backend'] == 'yum'
+
     def read_variants(self):
         # TODO: move to phases/init ?
         variants_file = self.paths.work.variants_file(arch="global")

pungi/phases/createrepo.py

@@ -163,7 +163,9 @@ def create_variant_repo(compose, arch, variant, pkg_type, modules_metadata=None)
     comps_path = None
     if compose.has_comps and pkg_type == "rpm":
         comps_path = compose.paths.work.comps(arch=arch, variant=variant)
-    cmd = repo.get_createrepo_cmd(repo_dir, update=True, database=True, skip_stat=True,
+    cmd = repo.get_createrepo_cmd(repo_dir, update=True,
+                                  database=compose.should_create_yum_database,
+                                  skip_stat=True,
                                   pkglist=file_list, outputdir=repo_dir,
                                   workers=compose.conf["createrepo_num_workers"],
                                   groupfile=comps_path, update_md_path=repo_dir_arch,

tests/helpers.py

@@ -149,6 +149,7 @@ class DummyCompose(object):
         self.attempt_deliverable = mock.Mock()
         self.fail_deliverable = mock.Mock()
         self.require_deliverable = mock.Mock()
+        self.should_create_yum_database = True
 
     def setup_optional(self):
         self.all_variants['Server-optional'] = MockVariant(

tests/test_compose.py

@@ -481,6 +481,24 @@ class StatusTest(unittest.TestCase):
 
         self.assertTrue(self.compose.notifier.send.call_count, 1)
 
+    def test_no_database_with_dnf_backend(self):
+        self.compose.conf['gather_backend'] = 'dnf'
+        self.assertFalse(self.compose.should_create_yum_database)
+
+    def test_no_database_with_dnf_backend_config_override(self):
+        self.compose.conf['gather_backend'] = 'dnf'
+        self.compose.conf['createrepo_database'] = True
+        self.assertTrue(self.compose.should_create_yum_database)
+
+    def test_no_database_with_yum_backend(self):
+        self.compose.conf['gather_backend'] = 'yum'
+        self.assertTrue(self.compose.should_create_yum_database)
+
+    def test_no_database_with_yum_backend_config_override(self):
+        self.compose.conf['gather_backend'] = 'yum'
+        self.compose.conf['createrepo_database'] = False
+        self.assertFalse(self.compose.should_create_yum_database)
+
 
 if __name__ == "__main__":
     unittest.main()

tests/test_createrepophase.py

@@ -151,6 +151,38 @@ class TestCreateVariantRepo(PungiTestCase):
         with open(list_file) as f:
             self.assertEqual(f.read(), 'Packages/b/bash-4.3.30-2.fc21.x86_64.rpm\n')
 
+    @mock.patch('pungi.phases.createrepo.run')
+    @mock.patch('pungi.phases.createrepo.CreaterepoWrapper')
+    def test_variant_repo_rpms_without_database(self, CreaterepoWrapperCls, run):
+        compose = DummyCompose(self.topdir, {
+            'createrepo_checksum': 'sha256',
+        })
+        compose.should_create_yum_database = False
+        compose.DEBUG = False
+        compose.has_comps = False
+
+        repo = CreaterepoWrapperCls.return_value
+        copy_fixture('server-rpms.json', compose.paths.compose.metadata('rpms.json'))
+
+        create_variant_repo(compose, 'x86_64', compose.variants['Server'], 'rpm')
+
+        list_file = self.topdir + '/work/x86_64/repo_package_list/Server.x86_64.rpm.conf'
+        self.assertEqual(CreaterepoWrapperCls.mock_calls[0],
+                         mock.call(createrepo_c=True))
+        self.assertItemsEqual(
+            repo.get_createrepo_cmd.mock_calls,
+            [mock.call(self.topdir + '/compose/Server/x86_64/os', checksum='sha256',
+                       database=False, groupfile=None, workers=3,
+                       outputdir=self.topdir + '/compose/Server/x86_64/os',
+                       pkglist=list_file, skip_stat=True, update=True,
+                       update_md_path=self.topdir + '/work/x86_64/repo',
+                       deltas=False, oldpackagedirs=None, use_xz=False)])
+        self.assertItemsEqual(
+            repo.get_modifyrepo_cmd.mock_calls,
+            [])
+        with open(list_file) as f:
+            self.assertEqual(f.read(), 'Packages/b/bash-4.3.30-2.fc21.x86_64.rpm\n')
+
     @mock.patch('pungi.phases.createrepo.run')
     @mock.patch('pungi.phases.createrepo.CreaterepoWrapper')
     def test_variant_repo_source(self, CreaterepoWrapperCls, run):