Use cachedir when createrepo

Then createrepo can reuse checksum values from cache to make it faster.

JIRA: RHELCMP-5984
Signed-off-by: Haibo Lin <hlin@redhat.com>

pungi.spec

@@ -91,7 +91,7 @@ rm -rf %{buildroot}
 %{_bindir}/comps_filter
 %{_bindir}/%{name}-make-ostree
 %{_datadir}/%{name}
-/var/cache/%{name}
+%dir %attr(1777, root, root) /var/cache/%{name}
 
 %files utils
 %{python_sitelib}/%{name}_utils

pungi/checks.py

@@ -660,6 +660,7 @@ def make_schema():
                 "default": "sha256",
                 "enum": ["sha1", "sha256", "sha512"],
             },
+            "createrepo_enable_cache": {"type": "boolean", "default": True},
             "createrepo_use_xz": {"type": "boolean", "default": False},
             "createrepo_num_threads": {"type": "number", "default": get_num_cpus()},
             "createrepo_num_workers": {"type": "number", "default": 3},

pungi/phases/createrepo.py

@@ -188,6 +188,23 @@ def create_variant_repo(
     comps_path = None
     if compose.has_comps and pkg_type == "rpm":
         comps_path = compose.paths.work.comps(arch=arch, variant=variant)
+
+    if compose.conf["createrepo_enable_cache"]:
+        cachedir = os.path.join(
+            "/var/cache/pungi/createrepo_c/",
+            "%s-%s" % (compose.conf["release_short"], os.getuid()),
+        )
+        if not os.path.exists(cachedir):
+            try:
+                os.makedirs(cachedir)
+            except Exception as e:
+                compose.log_warning(
+                    "Cache disabled because cannot create cache dir %s %s"
+                    % (cachedir, str(e))
+                )
+                cachedir = None
+    else:
+        cachedir = None
     cmd = repo.get_createrepo_cmd(
         repo_dir,
         update=True,
@@ -203,6 +220,7 @@ def create_variant_repo(
         oldpackagedirs=old_package_dirs,
         use_xz=compose.conf["createrepo_use_xz"],
         extra_args=compose.conf["createrepo_extra_args"],
+        cachedir=cachedir,
     )
     log_file = compose.paths.log.log_file(
         arch, "createrepo-%s.%s" % (variant, pkg_type)

tests/test_createrepophase.py

@@ -176,7 +176,10 @@ class TestCreateVariantRepo(PungiTestCase):
     @mock.patch("pungi.phases.createrepo.run")
     @mock.patch("pungi.phases.createrepo.CreaterepoWrapper")
     def test_variant_repo_rpms(self, CreaterepoWrapperCls, run):
-        compose = DummyCompose(self.topdir, {"createrepo_checksum": "sha256"})
+        compose = DummyCompose(
+            self.topdir,
+            {"createrepo_checksum": "sha256"},
+        )
         compose.has_comps = False
 
         repo = CreaterepoWrapperCls.return_value
@@ -210,6 +213,10 @@ class TestCreateVariantRepo(PungiTestCase):
                     oldpackagedirs=None,
                     use_xz=False,
                     extra_args=[],
+                    cachedir=os.path.join(
+                        "/var/cache/pungi/createrepo_c/",
+                        "%s-%s" % (compose.conf["release_short"], os.getuid()),
+                    ),
                 )
             ],
         )
@@ -219,7 +226,10 @@ class TestCreateVariantRepo(PungiTestCase):
     @mock.patch("pungi.phases.createrepo.run")
     @mock.patch("pungi.phases.createrepo.CreaterepoWrapper")
     def test_variant_repo_rpms_without_database(self, CreaterepoWrapperCls, run):
-        compose = DummyCompose(self.topdir, {"createrepo_checksum": "sha256"})
+        compose = DummyCompose(
+            self.topdir,
+            {"createrepo_checksum": "sha256", "createrepo_enable_cache": False},
+        )
         compose.should_create_yum_database = False
         compose.has_comps = False
 
@@ -254,6 +264,7 @@ class TestCreateVariantRepo(PungiTestCase):
                     oldpackagedirs=None,
                     use_xz=False,
                     extra_args=[],
+                    cachedir=None,
                 )
             ],
         )
@@ -263,7 +274,10 @@ class TestCreateVariantRepo(PungiTestCase):
     @mock.patch("pungi.phases.createrepo.run")
     @mock.patch("pungi.phases.createrepo.CreaterepoWrapper")
     def test_variant_repo_source(self, CreaterepoWrapperCls, run):
-        compose = DummyCompose(self.topdir, {"createrepo_checksum": "sha256"})
+        compose = DummyCompose(
+            self.topdir,
+            {"createrepo_checksum": "sha256", "createrepo_enable_cache": False},
+        )
         compose.has_comps = False
 
         repo = CreaterepoWrapperCls.return_value
@@ -295,6 +309,7 @@ class TestCreateVariantRepo(PungiTestCase):
                     oldpackagedirs=None,
                     use_xz=False,
                     extra_args=[],
+                    cachedir=None,
                 )
             ],
         )
@@ -304,7 +319,10 @@ class TestCreateVariantRepo(PungiTestCase):
     @mock.patch("pungi.phases.createrepo.run")
     @mock.patch("pungi.phases.createrepo.CreaterepoWrapper")
     def test_variant_repo_debug(self, CreaterepoWrapperCls, run):
-        compose = DummyCompose(self.topdir, {"createrepo_checksum": "sha256"})
+        compose = DummyCompose(
+            self.topdir,
+            {"createrepo_checksum": "sha256", "createrepo_enable_cache": False},
+        )
         compose.has_comps = False
 
         repo = CreaterepoWrapperCls.return_value
@@ -339,6 +357,7 @@ class TestCreateVariantRepo(PungiTestCase):
                     oldpackagedirs=None,
                     use_xz=False,
                     extra_args=[],
+                    cachedir=None,
                 )
             ],
         )
@@ -351,7 +370,12 @@ class TestCreateVariantRepo(PungiTestCase):
     @mock.patch("pungi.phases.createrepo.CreaterepoWrapper")
     def test_variant_repo_no_createrepo_c(self, CreaterepoWrapperCls, run):
         compose = DummyCompose(
-            self.topdir, {"createrepo_c": False, "createrepo_checksum": "sha256"}
+            self.topdir,
+            {
+                "createrepo_c": False,
+                "createrepo_enable_cache": False,
+                "createrepo_checksum": "sha256",
+            },
         )
         compose.has_comps = False
 
@@ -386,6 +410,7 @@ class TestCreateVariantRepo(PungiTestCase):
                     oldpackagedirs=None,
                     use_xz=False,
                     extra_args=[],
+                    cachedir=None,
                 )
             ],
         )
@@ -397,7 +422,11 @@ class TestCreateVariantRepo(PungiTestCase):
     def test_variant_repo_is_idepotent(self, CreaterepoWrapperCls, run):
         compose = DummyCompose(
             self.topdir,
-            {"createrepo_checksum": "sha256", "createrepo_num_workers": 10},
+            {
+                "createrepo_checksum": "sha256",
+                "createrepo_enable_cache": False,
+                "createrepo_num_workers": 10,
+            },
         )
         compose.has_comps = False
 
@@ -436,6 +465,7 @@ class TestCreateVariantRepo(PungiTestCase):
                     oldpackagedirs=None,
                     use_xz=False,
                     extra_args=[],
+                    cachedir=None,
                 )
             ],
         )
@@ -446,7 +476,12 @@ class TestCreateVariantRepo(PungiTestCase):
     @mock.patch("pungi.phases.createrepo.CreaterepoWrapper")
     def test_variant_repo_rpms_with_xz(self, CreaterepoWrapperCls, run):
         compose = DummyCompose(
-            self.topdir, {"createrepo_checksum": "sha256", "createrepo_use_xz": True}
+            self.topdir,
+            {
+                "createrepo_checksum": "sha256",
+                "createrepo_enable_cache": False,
+                "createrepo_use_xz": True,
+            },
         )
         compose.has_comps = False
 
@@ -481,6 +516,7 @@ class TestCreateVariantRepo(PungiTestCase):
                     oldpackagedirs=None,
                     use_xz=True,
                     extra_args=[],
+                    cachedir=None,
                 )
             ],
         )
@@ -491,7 +527,12 @@ class TestCreateVariantRepo(PungiTestCase):
     @mock.patch("pungi.phases.createrepo.CreaterepoWrapper")
     def test_variant_repo_rpms_with_deltas(self, CreaterepoWrapperCls, run):
         compose = DummyCompose(
-            self.topdir, {"createrepo_checksum": "sha256", "createrepo_deltas": True}
+            self.topdir,
+            {
+                "createrepo_checksum": "sha256",
+                "createrepo_deltas": True,
+                "createrepo_enable_cache": False,
+            },
         )
         compose.has_comps = False
         compose.old_composes = [self.topdir + "/old"]
@@ -536,6 +577,7 @@ class TestCreateVariantRepo(PungiTestCase):
                     + "/old/test-1.0-20151203.0/compose/Server/x86_64/os/Packages",
                     use_xz=False,
                     extra_args=[],
+                    cachedir=None,
                 )
             ],
         )
@@ -552,6 +594,7 @@ class TestCreateVariantRepo(PungiTestCase):
             {
                 "createrepo_checksum": "sha256",
                 "createrepo_deltas": [("^Server$", {"*": True})],
+                "createrepo_enable_cache": False,
             },
         )
         compose.has_comps = False
@@ -596,6 +639,7 @@ class TestCreateVariantRepo(PungiTestCase):
                     + "/old/test-1.0-20151203.0/compose/Server/x86_64/os/Packages",
                     use_xz=False,
                     extra_args=[],
+                    cachedir=None,
                 )
             ],
         )
@@ -612,6 +656,7 @@ class TestCreateVariantRepo(PungiTestCase):
             {
                 "createrepo_checksum": "sha256",
                 "createrepo_deltas": [("^Everything$", {"x86_64": True})],
+                "createrepo_enable_cache": False,
             },
         )
         compose.has_comps = False
@@ -652,6 +697,7 @@ class TestCreateVariantRepo(PungiTestCase):
                     oldpackagedirs=None,
                     use_xz=False,
                     extra_args=[],
+                    cachedir=None,
                 )
             ],
         )
@@ -668,6 +714,7 @@ class TestCreateVariantRepo(PungiTestCase):
             {
                 "createrepo_checksum": "sha256",
                 "createrepo_deltas": [("^Server$", {"s390x": True})],
+                "createrepo_enable_cache": False,
             },
         )
         compose.has_comps = False
@@ -708,6 +755,7 @@ class TestCreateVariantRepo(PungiTestCase):
                     oldpackagedirs=None,
                     use_xz=False,
                     extra_args=[],
+                    cachedir=None,
                 )
             ],
         )
@@ -722,6 +770,7 @@ class TestCreateVariantRepo(PungiTestCase):
             {
                 "createrepo_checksum": "sha256",
                 "createrepo_deltas": True,
+                "createrepo_enable_cache": False,
                 "hashed_directories": True,
             },
         )
@@ -776,6 +825,7 @@ class TestCreateVariantRepo(PungiTestCase):
                     ],
                     use_xz=False,
                     extra_args=[],
+                    cachedir=None,
                 )
             ],
         )
@@ -792,6 +842,7 @@ class TestCreateVariantRepo(PungiTestCase):
             {
                 "createrepo_checksum": "sha256",
                 "createrepo_deltas": True,
+                "createrepo_enable_cache": False,
                 "hashed_directories": True,
             },
         )
@@ -834,6 +885,7 @@ class TestCreateVariantRepo(PungiTestCase):
                     oldpackagedirs=None,
                     use_xz=False,
                     extra_args=[],
+                    cachedir=None,
                 )
             ],
         )
@@ -845,7 +897,12 @@ class TestCreateVariantRepo(PungiTestCase):
     def test_variant_repo_source_with_deltas(self, CreaterepoWrapperCls, run):
         # This should not actually create deltas, only binary repos do.
         compose = DummyCompose(
-            self.topdir, {"createrepo_checksum": "sha256", "createrepo_deltas": True}
+            self.topdir,
+            {
+                "createrepo_checksum": "sha256",
+                "createrepo_enable_cache": False,
+                "createrepo_deltas": True,
+            },
         )
         compose.has_comps = False
         compose.old_composes = [self.topdir + "/old"]
@@ -883,6 +940,7 @@ class TestCreateVariantRepo(PungiTestCase):
                     oldpackagedirs=None,
                     use_xz=False,
                     extra_args=[],
+                    cachedir=None,
                 )
             ],
         )
@@ -894,7 +952,12 @@ class TestCreateVariantRepo(PungiTestCase):
     def test_variant_repo_debug_with_deltas(self, CreaterepoWrapperCls, run):
         # This should not actually create deltas, only binary repos do.
         compose = DummyCompose(
-            self.topdir, {"createrepo_checksum": "sha256", "createrepo_deltas": True}
+            self.topdir,
+            {
+                "createrepo_checksum": "sha256",
+                "createrepo_deltas": True,
+                "createrepo_enable_cache": False,
+            },
         )
         compose.has_comps = False
         compose.old_composes = [self.topdir + "/old"]
@@ -934,6 +997,7 @@ class TestCreateVariantRepo(PungiTestCase):
                     oldpackagedirs=None,
                     use_xz=False,
                     extra_args=[],
+                    cachedir=None,
                 )
             ],
         )
@@ -949,6 +1013,7 @@ class TestCreateVariantRepo(PungiTestCase):
             self.topdir,
             {
                 "createrepo_checksum": "sha256",
+                "createrepo_enable_cache": False,
                 "product_id": "yes",  # Truthy value is enough for this test
             },
         )
@@ -993,6 +1058,7 @@ class TestCreateVariantRepo(PungiTestCase):
                     oldpackagedirs=None,
                     use_xz=False,
                     extra_args=[],
+                    cachedir=None,
                 )
             ],
         )
@@ -1009,6 +1075,7 @@ class TestCreateVariantRepo(PungiTestCase):
             self.topdir,
             {
                 "createrepo_checksum": "sha256",
+                "createrepo_enable_cache": False,
                 "product_id": "yes",  # Truthy value is enough for this test
             },
         )
@@ -1046,6 +1113,7 @@ class TestCreateVariantRepo(PungiTestCase):
                     oldpackagedirs=None,
                     use_xz=False,
                     extra_args=[],
+                    cachedir=None,
                 )
             ],
         )
@@ -1061,6 +1129,7 @@ class TestCreateVariantRepo(PungiTestCase):
             self.topdir,
             {
                 "createrepo_checksum": "sha256",
+                "createrepo_enable_cache": False,
                 "product_id": "yes",  # Truthy value is enough for this test
             },
         )
@@ -1096,6 +1165,7 @@ class TestCreateVariantRepo(PungiTestCase):
                     oldpackagedirs=None,
                     use_xz=False,
                     extra_args=[],
+                    cachedir=None,
                 )
             ],
         )