[createrepo] Compute delta RPMS against old compose

Signed-off-by: Lubomír Sedlář <lsedlar@redhat.com>

doc/configuration.rst

@@ -336,6 +336,10 @@ Options
 **createrepo_c** = True
     (*bool*) -- use createrepo_c (True) or legacy createrepo (False)
 
+**createrepo_deltas** = False
+    (*bool*) -- generate delta RPMs against an older compose. This needs to be
+    used together with `--old-composes`` command line argument.
+
 
 
 Example

pungi/phases/createrepo.py

@@ -29,9 +29,10 @@ import threading
 from kobo.threads import ThreadPool, WorkerThread
 from kobo.shortcuts import run, relative_path
 
-from pungi.wrappers.scm import get_dir_from_scm
+from ..wrappers.scm import get_dir_from_scm
-from pungi.wrappers.createrepo import CreaterepoWrapper
+from ..wrappers.createrepo import CreaterepoWrapper
-from pungi.phases.base import PhaseBase
+from .base import PhaseBase
+from ..util import find_old_compose
 
 import productmd.rpms
 
@@ -54,6 +55,11 @@ class CreaterepoPhase(PhaseBase):
             "expected_types": [str],
             "expected_values": ["sha256", "sha"],
         },
+        {
+            "name": "createrepo_deltas",
+            "expected_types": [bool],
+            "optional": True,
+        },
         {
             "name": "product_id",
             "expected_types": [dict],
@@ -70,6 +76,19 @@ class CreaterepoPhase(PhaseBase):
         PhaseBase.__init__(self, compose)
         self.pool = ThreadPool(logger=self.compose._logger)
 
+    def validate(self):
+        errors = []
+        try:
+            super(CreaterepoPhase, self).validate()
+        except ValueError as exc:
+            errors = exc.message.split('\n')
+
+        if not self.compose.old_composes and 'createrepo_deltas' in self.compose.conf:
+            errors.append('Can not generate deltas without old compose')
+
+        if errors:
+            raise ValueError('\n'.join(errors))
+
     def run(self):
         get_productids_from_scm(self.compose)
         for i in range(3):
@@ -102,6 +121,7 @@ def create_variant_repo(compose, arch, variant, pkg_type):
 
     createrepo_c = compose.conf.get("createrepo_c", True)
     createrepo_checksum = compose.conf["createrepo_checksum"]
+    createrepo_deltas = compose.conf.get("createrepo_deltas", False)
     repo = CreaterepoWrapper(createrepo_c=createrepo_c)
     repo_dir_arch = compose.paths.work.arch_repo(arch='global' if pkg_type == 'srpm' else arch)
 
@@ -151,13 +171,29 @@ def create_variant_repo(compose, arch, variant, pkg_type):
         for rel_path in sorted(rpms):
             f.write("%s\n" % rel_path)
 
+    old_packages_dir = None
+    if createrepo_deltas:
+        old_compose_path = find_old_compose(
+            compose.old_composes,
+            compose.ci_base.release.short,
+            compose.ci_base.release.version,
+            compose.ci_base.base_product.short if compose.ci_base.release.is_layered else None,
+            compose.ci_base.base_product.version if compose.ci_base.release.is_layered else None
+        )
+        if not old_compose_path:
+            compose.log_info("No suitable old compose found in: %s" % compose.old_composes)
+        else:
+            rel_dir = relative_path(repo_dir, compose.topdir.rstrip('/') + '/')
+            old_packages_dir = os.path.join(old_compose_path, rel_dir)
+
     comps_path = None
     if compose.has_comps and pkg_type == "rpm":
         comps_path = compose.paths.work.comps(arch=arch, variant=variant)
     cmd = repo.get_createrepo_cmd(repo_dir, update=True, database=True, skip_stat=True,
                                   pkglist=file_list, outputdir=repo_dir, workers=3,
                                   groupfile=comps_path, update_md_path=repo_dir_arch,
-                                  checksum=createrepo_checksum)
+                                  checksum=createrepo_checksum, deltas=createrepo_deltas,
+                                  oldpackagedirs=old_packages_dir)
     log_file = compose.paths.log.log_file(arch, "createrepo-%s" % variant)
     run(cmd, logfile=log_file, show_cmd=True)
 

tests/helpers.py

@@ -30,6 +30,7 @@ class DummyCompose(object):
             release=mock.Mock(
                 short='test',
                 version='1.0',
+                is_layered=False,
             ),
         )
         self.topdir = topdir
@@ -51,6 +52,7 @@ class DummyCompose(object):
         self.get_image_name = mock.Mock(return_value='image-name')
         self.image = mock.Mock(path='Client/i386/iso/image.iso')
         self.im = mock.Mock(images={'Client': {'i386': [self.image]}})
+        self.old_composes = []
 
     def get_variants(self, arch=None, types=None):
         return [v for v in self.variants.values() if not arch or arch in v.arches]

tests/test_createrepophase.py

@@ -11,10 +11,22 @@ import sys
 sys.path.insert(0, os.path.join(os.path.dirname(__file__), ".."))
 
 from pungi.phases.createrepo import CreaterepoPhase, create_variant_repo
-from tests.helpers import DummyCompose, PungiTestCase, copy_fixture
+from tests.helpers import DummyCompose, PungiTestCase, copy_fixture, touch
 
 
 class TestCreaterepoPhase(PungiTestCase):
+    @mock.patch('pungi.phases.createrepo.ThreadPool')
+    def test_fails_deltas_without_old_compose(self, ThreadPoolCls):
+        compose = DummyCompose(self.topdir, {
+            'createrepo_checksum': 'sha256',
+            'createrepo_deltas': True,
+        })
+
+        phase = CreaterepoPhase(compose)
+        with self.assertRaises(ValueError) as ctx:
+            phase.validate()
+
+        self.assertIn('deltas', str(ctx.exception))
 
     @mock.patch('pungi.phases.createrepo.ThreadPool')
     def test_starts_jobs(self, ThreadPoolCls):
@@ -94,7 +106,8 @@ class TestCreateRepoThread(PungiTestCase):
                        database=True, groupfile=None, workers=3,
                        outputdir=self.topdir + '/compose/Server/x86_64/os',
                        pkglist=list_file, skip_stat=True, update=True,
-                       update_md_path=self.topdir + '/work/x86_64/repo')])
+                       update_md_path=self.topdir + '/work/x86_64/repo',
+                       deltas=False, oldpackagedirs=None)])
         with open(list_file) as f:
             self.assertEqual(f.read(), 'Packages/b/bash-4.3.30-2.fc21.x86_64.rpm\n')
 
@@ -121,7 +134,8 @@ class TestCreateRepoThread(PungiTestCase):
                        database=True, groupfile=None, workers=3,
                        outputdir=self.topdir + '/compose/Server/source/tree',
                        pkglist=list_file, skip_stat=True, update=True,
-                       update_md_path=self.topdir + '/work/global/repo')])
+                       update_md_path=self.topdir + '/work/global/repo',
+                       deltas=False, oldpackagedirs=None)])
         with open(list_file) as f:
             self.assertItemsEqual(
                 f.read().strip().split('\n'),
@@ -151,7 +165,8 @@ class TestCreateRepoThread(PungiTestCase):
                        database=True, groupfile=None, workers=3,
                        outputdir=self.topdir + '/compose/Server/x86_64/debug/tree',
                        pkglist=list_file, skip_stat=True, update=True,
-                       update_md_path=self.topdir + '/work/x86_64/repo')])
+                       update_md_path=self.topdir + '/work/x86_64/repo',
+                       deltas=False, oldpackagedirs=None)])
         with open(list_file) as f:
             self.assertEqual(f.read(), 'Packages/b/bash-debuginfo-4.3.30-2.fc21.x86_64.rpm\n')
 
@@ -179,7 +194,8 @@ class TestCreateRepoThread(PungiTestCase):
                        database=True, groupfile=None, workers=3,
                        outputdir=self.topdir + '/compose/Server/x86_64/os',
                        pkglist=list_file, skip_stat=True, update=True,
-                       update_md_path=self.topdir + '/work/x86_64/repo')])
+                       update_md_path=self.topdir + '/work/x86_64/repo',
+                       deltas=False, oldpackagedirs=None)])
         with open(list_file) as f:
             self.assertEqual(f.read(), 'Packages/b/bash-4.3.30-2.fc21.x86_64.rpm\n')
 
@@ -208,10 +224,106 @@ class TestCreateRepoThread(PungiTestCase):
                        database=True, groupfile=None, workers=3,
                        outputdir=self.topdir + '/compose/Server/x86_64/os',
                        pkglist=list_file, skip_stat=True, update=True,
-                       update_md_path=self.topdir + '/work/x86_64/repo')])
+                       update_md_path=self.topdir + '/work/x86_64/repo',
+                       deltas=False, oldpackagedirs=None)])
         with open(list_file) as f:
             self.assertEqual(f.read(), 'Packages/b/bash-4.3.30-2.fc21.x86_64.rpm\n')
 
+    @mock.patch('pungi.phases.createrepo.run')
+    @mock.patch('pungi.phases.createrepo.CreaterepoWrapper')
+    def test_variant_repo_rpms_with_deltas(self, CreaterepoWrapperCls, run):
+        compose = DummyCompose(self.topdir, {
+            'createrepo_checksum': 'sha256',
+            'createrepo_deltas': True,
+        })
+        compose.DEBUG = False
+        compose.has_comps = False
+        compose.old_composes = [self.topdir + '/old']
+        touch(os.path.join(self.topdir, 'old', 'test-1.0-20151203.0', 'STATUS'), 'FINISHED')
+
+        repo = CreaterepoWrapperCls.return_value
+        copy_fixture('server-rpms.json', compose.paths.compose.metadata('rpms.json'))
+
+        create_variant_repo(compose, 'x86_64', compose.variants['Server'], 'rpm')
+
+        list_file = self.topdir + '/work/x86_64/repo_package_list/Server.x86_64.rpm.conf'
+        self.assertEqual(CreaterepoWrapperCls.mock_calls[0],
+                         mock.call(createrepo_c=True))
+        self.assertItemsEqual(
+            repo.get_createrepo_cmd.mock_calls,
+            [mock.call(self.topdir + '/compose/Server/x86_64/os', checksum='sha256',
+                       database=True, groupfile=None, workers=3,
+                       outputdir=self.topdir + '/compose/Server/x86_64/os',
+                       pkglist=list_file, skip_stat=True, update=True,
+                       update_md_path=self.topdir + '/work/x86_64/repo', deltas=True,
+                       oldpackagedirs=self.topdir + '/old/test-1.0-20151203.0/compose/Server/x86_64/os')])
+        with open(list_file) as f:
+            self.assertEqual(f.read(), 'Packages/b/bash-4.3.30-2.fc21.x86_64.rpm\n')
+
+    @mock.patch('pungi.phases.createrepo.run')
+    @mock.patch('pungi.phases.createrepo.CreaterepoWrapper')
+    def test_variant_repo_source_with_deltas(self, CreaterepoWrapperCls, run):
+        compose = DummyCompose(self.topdir, {
+            'createrepo_checksum': 'sha256',
+            'createrepo_deltas': True,
+        })
+        compose.DEBUG = False
+        compose.has_comps = False
+        compose.old_composes = [self.topdir + '/old']
+        touch(os.path.join(self.topdir, 'old', 'test-1.0-20151203.0', 'STATUS'), 'FINISHED')
+
+        repo = CreaterepoWrapperCls.return_value
+        copy_fixture('server-rpms.json', compose.paths.compose.metadata('rpms.json'))
+
+        create_variant_repo(compose, None, compose.variants['Server'], 'srpm')
+
+        list_file = self.topdir + '/work/global/repo_package_list/Server.None.srpm.conf'
+        self.assertEqual(CreaterepoWrapperCls.mock_calls[0],
+                         mock.call(createrepo_c=True))
+        self.assertItemsEqual(
+            repo.get_createrepo_cmd.mock_calls,
+            [mock.call(self.topdir + '/compose/Server/source/tree', checksum='sha256',
+                       database=True, groupfile=None, workers=3,
+                       outputdir=self.topdir + '/compose/Server/source/tree',
+                       pkglist=list_file, skip_stat=True, update=True,
+                       update_md_path=self.topdir + '/work/global/repo', deltas=True,
+                       oldpackagedirs=self.topdir + '/old/test-1.0-20151203.0/compose/Server/source/tree')])
+        with open(list_file) as f:
+            self.assertItemsEqual(
+                f.read().strip().split('\n'),
+                ['../SRPMS/b/bash-4.3.30-2.fc21.src.rpm'])
+
+    @mock.patch('pungi.phases.createrepo.run')
+    @mock.patch('pungi.phases.createrepo.CreaterepoWrapper')
+    def test_variant_repo_debug_with_deltas(self, CreaterepoWrapperCls, run):
+        compose = DummyCompose(self.topdir, {
+            'createrepo_checksum': 'sha256',
+            'createrepo_deltas': True,
+        })
+        compose.DEBUG = False
+        compose.has_comps = False
+        compose.old_composes = [self.topdir + '/old']
+        touch(os.path.join(self.topdir, 'old', 'test-1.0-20151203.0', 'STATUS'), 'FINISHED')
+
+        repo = CreaterepoWrapperCls.return_value
+        copy_fixture('server-rpms.json', compose.paths.compose.metadata('rpms.json'))
+
+        create_variant_repo(compose, 'x86_64', compose.variants['Server'], 'debuginfo')
+
+        list_file = self.topdir + '/work/x86_64/repo_package_list/Server.x86_64.debuginfo.conf'
+        self.assertEqual(CreaterepoWrapperCls.mock_calls[0],
+                         mock.call(createrepo_c=True))
+        self.assertItemsEqual(
+            repo.get_createrepo_cmd.mock_calls,
+            [mock.call(self.topdir + '/compose/Server/x86_64/debug/tree', checksum='sha256',
+                       database=True, groupfile=None, workers=3,
+                       outputdir=self.topdir + '/compose/Server/x86_64/debug/tree',
+                       pkglist=list_file, skip_stat=True, update=True,
+                       update_md_path=self.topdir + '/work/x86_64/repo', deltas=True,
+                       oldpackagedirs=self.topdir + '/old/test-1.0-20151203.0/compose/Server/x86_64/debug/tree')])
+        with open(list_file) as f:
+            self.assertEqual(f.read(), 'Packages/b/bash-debuginfo-4.3.30-2.fc21.x86_64.rpm\n')
+
 
 if __name__ == "__main__":
     unittest.main()