createrepo: Add option to use xz compression

This patch adds a new config option createrepo_use_xz, which when set to
true will cause createrepo to compress sqlite databases with xz. The
default setting is False.

Fixes: #387
Signed-off-by: Lubomír Sedlář <lsedlar@redhat.com>

doc/configuration.rst

@@ -362,6 +362,10 @@ Options
     (*bool*) -- generate delta RPMs against an older compose. This needs to be
     used together with `--old-composes`` command line argument.
 
+**createrepo_use_xz** = False
+    (*bool*) -- whether to pass ``--xz`` to the createrepo command. This will
+    cause the SQLite databases to be compressed with xz.
+
 **product_id** = None
     (*scm_dict*) -- If specified, it should point to a directory with
     certificates ``<variant_uid>-<arch>-*.pem``. This certificate will be

pungi/checks.py

@@ -424,6 +424,10 @@ def _make_schema():
                 "type": "string",
                 "enum": ["sha", "sha256"],
             },
+            "createrepo_use_xz": {
+                "type": "boolean",
+                "default": False,
+            },
 
             "hashed_directories": {
                 "type": "boolean",

pungi/phases/createrepo.py

@@ -165,7 +165,8 @@ def create_variant_repo(compose, arch, variant, pkg_type):
                                   pkglist=file_list, outputdir=repo_dir, workers=3,
                                   groupfile=comps_path, update_md_path=repo_dir_arch,
                                   checksum=createrepo_checksum, deltas=createrepo_deltas,
-                                  oldpackagedirs=old_packages_dir)
+                                  oldpackagedirs=old_packages_dir,
+                                  use_xz=compose.conf['createrepo_use_xz'])
     log_file = compose.paths.log.log_file(arch, "createrepo-%s.%s" % (variant, pkg_type))
     run(cmd, logfile=log_file, show_cmd=True)
 

pungi/wrappers/createrepo.py

@@ -33,7 +33,7 @@ class CreaterepoWrapper(object):
                            update_md_path=None, skip_stat=False, checkts=False, split=False,
                            pretty=True, database=True, checksum=None, unique_md_filenames=True,
                            distro=None, content=None, repo=None, revision=None, deltas=False,
-                           oldpackagedirs=None, num_deltas=None, workers=None):
+                           oldpackagedirs=None, num_deltas=None, workers=None, use_xz=False):
         # groupfile = /path/to/comps.xml
 
         cmd = [self.createrepo, directory, '--verbose']
@@ -114,6 +114,9 @@ class CreaterepoWrapper(object):
         if workers:
             cmd.append("--workers=%d" % int(workers))
 
+        if use_xz:
+            cmd.append("--xz")
+
         return cmd
 
     def get_mergerepo_cmd(self, outputdir, repos, database=True, pkglist=None, nogroups=False, noupdateinfo=None):

tests/test_createrepo_wrapper.py

@@ -31,7 +31,8 @@ class CreateRepoWrapperTest(unittest.TestCase):
             update=False, update_md_path='/test/md_path', skip_stat=True, checkts=True,
             split=True, pretty=False, database=False, checksum='sha256', unique_md_filenames=False,
             distro='Fedora', content=['c1', 'c2'], repo=['r1', 'r2'], revision='rev', deltas=True,
-            oldpackagedirs='/test/old', num_deltas=2, workers=3, outputdir='/test/output'
+            oldpackagedirs='/test/old', num_deltas=2, workers=3, outputdir='/test/output',
+            use_xz=True
         )
         self.maxDiff = None
 
@@ -43,7 +44,7 @@ class CreateRepoWrapperTest(unittest.TestCase):
                                '--checksum=sha256', '--distro=Fedora', '--simple-md-filenames', '--no-database',
                                '--content=c1', '--content=c2', '--repo=r1', '--repo=r2', '--revision=rev',
                                '--deltas', '--oldpackagedirs=/test/old', '--num-deltas=2', '--workers=3',
-                               '--outputdir=/test/output', '--verbose'])
+                               '--outputdir=/test/output', '--verbose', '--xz'])
 
     def test_get_createrepo_cmd_minimal(self):
         repo = CreaterepoWrapper(False)

tests/test_createrepophase.py

@@ -110,7 +110,7 @@ class TestCreateRepoThread(PungiTestCase):
                        outputdir=self.topdir + '/compose/Server/x86_64/os',
                        pkglist=list_file, skip_stat=True, update=True,
                        update_md_path=self.topdir + '/work/x86_64/repo',
-                       deltas=False, oldpackagedirs=None)])
+                       deltas=False, oldpackagedirs=None, use_xz=False)])
         with open(list_file) as f:
             self.assertEqual(f.read(), 'Packages/b/bash-4.3.30-2.fc21.x86_64.rpm\n')
 
@@ -138,7 +138,7 @@ class TestCreateRepoThread(PungiTestCase):
                        outputdir=self.topdir + '/compose/Server/source/tree',
                        pkglist=list_file, skip_stat=True, update=True,
                        update_md_path=self.topdir + '/work/global/repo',
-                       deltas=False, oldpackagedirs=None)])
+                       deltas=False, oldpackagedirs=None, use_xz=False)])
         with open(list_file) as f:
             self.assertItemsEqual(
                 f.read().strip().split('\n'),
@@ -169,7 +169,7 @@ class TestCreateRepoThread(PungiTestCase):
                        outputdir=self.topdir + '/compose/Server/x86_64/debug/tree',
                        pkglist=list_file, skip_stat=True, update=True,
                        update_md_path=self.topdir + '/work/x86_64/repo',
-                       deltas=False, oldpackagedirs=None)])
+                       deltas=False, oldpackagedirs=None, use_xz=False)])
         with open(list_file) as f:
             self.assertEqual(f.read(), 'Packages/b/bash-debuginfo-4.3.30-2.fc21.x86_64.rpm\n')
 
@@ -198,7 +198,7 @@ class TestCreateRepoThread(PungiTestCase):
                        outputdir=self.topdir + '/compose/Server/x86_64/os',
                        pkglist=list_file, skip_stat=True, update=True,
                        update_md_path=self.topdir + '/work/x86_64/repo',
-                       deltas=False, oldpackagedirs=None)])
+                       deltas=False, oldpackagedirs=None, use_xz=False)])
         with open(list_file) as f:
             self.assertEqual(f.read(), 'Packages/b/bash-4.3.30-2.fc21.x86_64.rpm\n')
 
@@ -228,7 +228,36 @@ class TestCreateRepoThread(PungiTestCase):
                        outputdir=self.topdir + '/compose/Server/x86_64/os',
                        pkglist=list_file, skip_stat=True, update=True,
                        update_md_path=self.topdir + '/work/x86_64/repo',
-                       deltas=False, oldpackagedirs=None)])
+                       deltas=False, oldpackagedirs=None, use_xz=False)])
+        with open(list_file) as f:
+            self.assertEqual(f.read(), 'Packages/b/bash-4.3.30-2.fc21.x86_64.rpm\n')
+
+    @mock.patch('pungi.phases.createrepo.run')
+    @mock.patch('pungi.phases.createrepo.CreaterepoWrapper')
+    def test_variant_repo_rpms_with_xz(self, CreaterepoWrapperCls, run):
+        compose = DummyCompose(self.topdir, {
+            'createrepo_checksum': 'sha256',
+            'createrepo_use_xz': True,
+        })
+        compose.DEBUG = False
+        compose.has_comps = False
+
+        repo = CreaterepoWrapperCls.return_value
+        copy_fixture('server-rpms.json', compose.paths.compose.metadata('rpms.json'))
+
+        create_variant_repo(compose, 'x86_64', compose.variants['Server'], 'rpm')
+
+        list_file = self.topdir + '/work/x86_64/repo_package_list/Server.x86_64.rpm.conf'
+        self.assertEqual(CreaterepoWrapperCls.mock_calls[0],
+                         mock.call(createrepo_c=True))
+        self.assertItemsEqual(
+            repo.get_createrepo_cmd.mock_calls,
+            [mock.call(self.topdir + '/compose/Server/x86_64/os', checksum='sha256',
+                       database=True, groupfile=None, workers=3,
+                       outputdir=self.topdir + '/compose/Server/x86_64/os',
+                       pkglist=list_file, skip_stat=True, update=True,
+                       update_md_path=self.topdir + '/work/x86_64/repo', deltas=False,
+                       oldpackagedirs=None, use_xz=True)])
         with open(list_file) as f:
             self.assertEqual(f.read(), 'Packages/b/bash-4.3.30-2.fc21.x86_64.rpm\n')
 
@@ -259,7 +288,8 @@ class TestCreateRepoThread(PungiTestCase):
                        outputdir=self.topdir + '/compose/Server/x86_64/os',
                        pkglist=list_file, skip_stat=True, update=True,
                        update_md_path=self.topdir + '/work/x86_64/repo', deltas=True,
-                       oldpackagedirs=self.topdir + '/old/test-1.0-20151203.0/compose/Server/x86_64/os')])
+                       oldpackagedirs=self.topdir + '/old/test-1.0-20151203.0/compose/Server/x86_64/os',
+                       use_xz=False)])
         with open(list_file) as f:
             self.assertEqual(f.read(), 'Packages/b/bash-4.3.30-2.fc21.x86_64.rpm\n')
 
@@ -290,7 +320,8 @@ class TestCreateRepoThread(PungiTestCase):
                        outputdir=self.topdir + '/compose/Server/source/tree',
                        pkglist=list_file, skip_stat=True, update=True,
                        update_md_path=self.topdir + '/work/global/repo', deltas=True,
-                       oldpackagedirs=self.topdir + '/old/test-1.0-20151203.0/compose/Server/source/tree')])
+                       oldpackagedirs=self.topdir + '/old/test-1.0-20151203.0/compose/Server/source/tree',
+                       use_xz=False)])
         with open(list_file) as f:
             self.assertItemsEqual(
                 f.read().strip().split('\n'),
@@ -323,7 +354,8 @@ class TestCreateRepoThread(PungiTestCase):
                        outputdir=self.topdir + '/compose/Server/x86_64/debug/tree',
                        pkglist=list_file, skip_stat=True, update=True,
                        update_md_path=self.topdir + '/work/x86_64/repo', deltas=True,
-                       oldpackagedirs=self.topdir + '/old/test-1.0-20151203.0/compose/Server/x86_64/debug/tree')])
+                       oldpackagedirs=self.topdir + '/old/test-1.0-20151203.0/compose/Server/x86_64/debug/tree',
+                       use_xz=False)])
         with open(list_file) as f:
             self.assertEqual(f.read(), 'Packages/b/bash-debuginfo-4.3.30-2.fc21.x86_64.rpm\n')