From d2f392fac89ccfa1c6522b38fc4772bb3197b975 Mon Sep 17 00:00:00 2001 From: Jan Kaluza Date: Wed, 15 Aug 2018 09:20:40 +0200 Subject: [PATCH] Use dogpile.cache to cache the listTaggedRPMS calls if possible If the same tag is queried with the same event, Pungi can cache the response and call the API again. Particularly for small composes this can save up significant amount of time. Merges: https://pagure.io/pungi/pull-request/1022 Signed-off-by: Jan Kaluza --- doc/configuration.rst | 26 ++++++++++++ doc/contributing.rst | 3 +- pungi.spec | 2 + pungi/checks.py | 6 +++ pungi/compose.py | 11 +++++ pungi/phases/pkgset/pkgsets.py | 23 ++++++++++- pungi/phases/pkgset/sources/source_koji.py | 3 +- setup.py | 1 + tests/helpers.py | 1 + tests/test_pkgset_pkgsets.py | 47 ++++++++++++++++++++++ 10 files changed, 119 insertions(+), 4 deletions(-) diff --git a/doc/configuration.rst b/doc/configuration.rst index 85a0112f..71607673 100644 --- a/doc/configuration.rst +++ b/doc/configuration.rst @@ -1674,3 +1674,29 @@ Miscellaneous Settings (*str*) -- If set, the ISO files from ``buildinstall``, ``createiso`` and ``live_images`` phases will be put into this destination, and a symlink pointing to this location will be created in actual compose directory. + +**dogpile_cache_backend** + (*str*) -- If set, Pungi will use the configured Dogpile cache backend to + cache various data between multiple Pungi calls. This can make Pungi + faster in case more similar composes are running regularly in short time. + + For list of available backends, please see the + https://dogpilecache.readthedocs.io documentation. + + Most typical configuration uses the ``dogpile.cache.dbm`` backend. + +**dogpile_cache_arguments** + (*dict*) -- Arguments to be used when creating the Dogpile cache backend. + See the particular backend's configuration for the list of possible + key/value pairs. + + For the ``dogpile.cache.dbm`` backend, the value can be for example + following: :: + + { + "filename": "/tmp/pungi_cache_file.dbm" + } + +**dogpile_cache_expiration_time** + (*int*) -- Defines the default expiration time in seconds of data stored + in the Dogpile cache. Defaults to 3600 seconds. diff --git a/doc/contributing.rst b/doc/contributing.rst index 7b4dc790..51e68296 100644 --- a/doc/contributing.rst +++ b/doc/contributing.rst @@ -25,6 +25,7 @@ These packages will have to installed: * libmodulemd * libselinux-python * lorax + * python-dogpile-cache * python-jsonschema * python-kickstart * python-libcomps @@ -60,7 +61,7 @@ packages above as they are used by calling an executable. :: $ for pkg in _deltarpm krbV _selinux deltarpm sqlitecachec _sqlitecache; do ln -vs "$(deactivate && python -c 'import os, '$pkg'; print('$pkg'.__file__)')" "$(virtualenvwrapper_get_site_packages_dir)"; done $ pip install -U pip $ PYCURL_SSL_LIBRARY=nss pip install pycurl --no-binary :all: - $ pip install beanbag jsonschema 'kobo>=0.6.0' lockfile lxml mock nose nose-cov productmd pyopenssl python-multilib requests requests-kerberos setuptools sphinx ordered_set koji PyYAML + $ pip install beanbag jsonschema 'kobo>=0.6.0' lockfile lxml mock nose nose-cov productmd pyopenssl python-multilib requests requests-kerberos setuptools sphinx ordered_set koji PyYAML dogpile.cache Now you should be able to run all existing tests. diff --git a/pungi.spec b/pungi.spec index 44668b11..774f1621 100644 --- a/pungi.spec +++ b/pungi.spec @@ -20,6 +20,7 @@ BuildRequires: python2-multilib BuildRequires: python2-libcomps BuildRequires: python2-six BuildRequires: python2-multilib +BuildRequires: python2-dogpile-cache Requires: createrepo >= 0.4.11 Requires: yum => 3.4.3-28 @@ -49,6 +50,7 @@ Requires: python2-dnf Requires: python2-multilib Requires: python2-libcomps Requires: python2-six +Requires: python2-dogpile-cache BuildArch: noarch diff --git a/pungi/checks.py b/pungi/checks.py index 53fc615e..b41128f1 100644 --- a/pungi/checks.py +++ b/pungi/checks.py @@ -711,6 +711,12 @@ def make_schema(): "default": False }, "symlink_isos_to": {"type": "string"}, + "dogpile_cache_backend": {"type": "string"}, + "dogpile_cache_expiration_time": {"type": "number"}, + "dogpile_cache_arguments": { + "type": "object", + "default": {}, + }, "createiso_skip": _variant_arch_mapping({"type": "boolean"}), "createiso_break_hardlinks": { "type": "boolean", diff --git a/pungi/compose.py b/pungi/compose.py index d884ad21..426a8879 100644 --- a/pungi/compose.py +++ b/pungi/compose.py @@ -29,6 +29,8 @@ import json import kobo.log from productmd.composeinfo import ComposeInfo from productmd.images import Images +from dogpile.cache import make_region + from pungi.wrappers.variants import VariantsXmlParser from pungi.paths import Paths @@ -156,6 +158,15 @@ class Compose(kobo.log.LoggingBase): self.attempted_deliverables = {} self.required_deliverables = {} + if self.conf.get("dogpile_cache_backend", None): + self.cache_region = make_region().configure( + self.conf.get("dogpile_cache_backend"), + expiration_time=self.conf.get("dogpile_cache_expiration_time", 3600), + arguments=self.conf.get("dogpile_cache_arguments", {}) + ) + else: + self.cache_region = make_region().configure('dogpile.cache.null') + get_compose_dir = staticmethod(get_compose_dir) def __getitem__(self, name): diff --git a/pungi/phases/pkgset/pkgsets.py b/pungi/phases/pkgset/pkgsets.py index c1244df0..48b5c349 100644 --- a/pungi/phases/pkgset/pkgsets.py +++ b/pungi/phases/pkgset/pkgsets.py @@ -275,7 +275,7 @@ class FilelistPackageSet(PackageSetBase): class KojiPackageSet(PackageSetBase): def __init__(self, koji_wrapper, sigkey_ordering, arches=None, logger=None, packages=None, allow_invalid_sigkeys=False, - populate_only_packages=False): + populate_only_packages=False, cache_region=None): """ Creates new KojiPackageSet. @@ -298,6 +298,10 @@ class KojiPackageSet(PackageSetBase): when generating compose from predefined list of packages from big Koji tag. When False, all packages from Koji tag are added to KojiPackageSet. + :param dogpile.cache.CacheRegion cache_region: If set, the CacheRegion + will be used to cache the list of RPMs per Koji tag, so next calls + of the KojiPackageSet.populate(...) method won't try fetching it + again. """ super(KojiPackageSet, self).__init__(sigkey_ordering=sigkey_ordering, arches=arches, logger=logger, @@ -306,12 +310,14 @@ class KojiPackageSet(PackageSetBase): # Names of packages to look for in the Koji tag. self.packages = set(packages or []) self.populate_only_packages = populate_only_packages + self.cache_region = cache_region def __getstate__(self): result = self.__dict__.copy() result["koji_profile"] = self.koji_wrapper.profile del result["koji_wrapper"] del result["_logger"] + del result["cache_region"] return result def __setstate__(self, data): @@ -325,7 +331,20 @@ class KojiPackageSet(PackageSetBase): return self.koji_wrapper.koji_proxy def get_latest_rpms(self, tag, event, inherit=True): - return self.koji_proxy.listTaggedRPMS(tag, event=event, inherit=inherit, latest=True) + if self.cache_region: + cache_key = "KojiPackageSet.get_latest_rpms_%s_%s_%s" % ( + tag, str(event), str(inherit)) + cached_response = self.cache_region.get(cache_key) + if cached_response: + return cached_response + else: + response = self.koji_proxy.listTaggedRPMS( + tag, event=event, inherit=inherit, latest=True) + self.cache_region.set(cache_key, response) + return response + else: + return self.koji_proxy.listTaggedRPMS( + tag, event=event, inherit=inherit, latest=True) def get_package_path(self, queue_item): rpm_info, build_info = queue_item diff --git a/pungi/phases/pkgset/sources/source_koji.py b/pungi/phases/pkgset/sources/source_koji.py index ebbff850..bc62c181 100644 --- a/pungi/phases/pkgset/sources/source_koji.py +++ b/pungi/phases/pkgset/sources/source_koji.py @@ -560,7 +560,8 @@ def populate_global_pkgset(compose, koji_wrapper, path_prefix, event_id): koji_wrapper, compose.conf["sigkeys"], logger=compose._logger, arches=all_arches, packages=packages_to_gather, allow_invalid_sigkeys=allow_invalid_sigkeys, - populate_only_packages=populate_only_packages_to_gather) + populate_only_packages=populate_only_packages_to_gather, + cache_region=compose.cache_region) if old_file_cache_path: pkgset.load_old_file_cache(old_file_cache_path) # Create a filename for log with package-to-tag mapping. The tag diff --git a/setup.py b/setup.py index c359b49d..8ff5b58b 100755 --- a/setup.py +++ b/setup.py @@ -62,6 +62,7 @@ setup( "lxml", "productmd", "six", + 'dogpile.cache', ], tests_require = [ "mock", diff --git a/tests/helpers.py b/tests/helpers.py index 672b6974..8405d73b 100644 --- a/tests/helpers.py +++ b/tests/helpers.py @@ -152,6 +152,7 @@ class DummyCompose(object): self.fail_deliverable = mock.Mock() self.require_deliverable = mock.Mock() self.should_create_yum_database = True + self.cache_region = None self.DEBUG = False diff --git a/tests/test_pkgset_pkgsets.py b/tests/test_pkgset_pkgsets.py index c3312090..f514b566 100644 --- a/tests/test_pkgset_pkgsets.py +++ b/tests/test_pkgset_pkgsets.py @@ -11,6 +11,7 @@ except ImportError: import json import tempfile import re +from dogpile.cache import make_region sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..")) @@ -322,6 +323,52 @@ class TestKojiPkgset(PkgsetCompareMixin, helpers.PungiTestCase): 'i686': ['rpms/bash@4.3.42@4.fc24@i686'], 'x86_64': ['rpms/bash@4.3.42@4.fc24@x86_64']}) + def test_get_latest_rpms_cache(self): + self._touch_files([ + 'rpms/bash@4.3.42@4.fc24@x86_64', + 'rpms/bash-debuginfo@4.3.42@4.fc24@x86_64', + ]) + + cache_region = make_region().configure("dogpile.cache.memory") + pkgset = pkgsets.KojiPackageSet(self.koji_wrapper, [None], arches=['x86_64'], + cache_region=cache_region) + + # Try calling the populate twice, but expect just single listTaggedRPMs + # call - that means the caching worked. + for i in range(2): + result = pkgset.populate('f25') + self.assertEqual( + self.koji_wrapper.koji_proxy.mock_calls, + [mock.call.listTaggedRPMS('f25', event=None, inherit=True, latest=True)]) + self.assertPkgsetEqual( + result, + {'x86_64': ['rpms/bash-debuginfo@4.3.42@4.fc24@x86_64', + 'rpms/bash@4.3.42@4.fc24@x86_64']}) + + def test_get_latest_rpms_cache_different_id(self): + self._touch_files([ + 'rpms/bash@4.3.42@4.fc24@x86_64', + 'rpms/bash-debuginfo@4.3.42@4.fc24@x86_64', + ]) + + cache_region = make_region().configure("dogpile.cache.memory") + pkgset = pkgsets.KojiPackageSet(self.koji_wrapper, [None], arches=['x86_64'], + cache_region=cache_region) + + # Try calling the populate twice with different event id. It must not + # cache anything. + expected_calls = [] + for i in range(2): + expected_calls.append( + mock.call.listTaggedRPMS('f25', event=i, inherit=True, latest=True)) + result = pkgset.populate('f25', event={"id": i}) + self.assertEqual( + self.koji_wrapper.koji_proxy.mock_calls, + expected_calls) + self.assertPkgsetEqual( + result, + {'x86_64': ['rpms/bash-debuginfo@4.3.42@4.fc24@x86_64', + 'rpms/bash@4.3.42@4.fc24@x86_64']}) @mock.patch('kobo.pkgset.FileCache', new=MockFileCache) class TestMergePackageSets(PkgsetCompareMixin, unittest.TestCase):