rpms/libldb
7
0
Fork 0
Code Issues Pull Requests Releases Activity

import CS libldb-2.9.1-2.el9

Browse Source
This commit is contained in:
eabdullin 2024-09-30 15:52:43 +00:00
parent fd6f5657c0
commit c7bb96342b
6 changed files with 265 additions and 62 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.gitignore vendored
View File

@ -1,2 +1,2 @@
SOURCES/ldb-2.8.0.tar.gz
SOURCES/ldb-2.9.1.tar.gz
SOURCES/ldb.keyring

2
.libldb.metadata
View File

@ -1,2 +1,2 @@
cf5c3d8a15c0666cc980a8cf7227ae711664f5a3 SOURCES/ldb-2.8.0.tar.gz
0a82c6f41d3ada818bf1e258038979ca76def986 SOURCES/ldb-2.9.1.tar.gz
8ac6d09878c4218fb8e365fcf5a877a621dd40f9 SOURCES/ldb.keyring

11
SOURCES/ldb-2.8.0.tar.asc
View File

@ -1,11 +0,0 @@
-----BEGIN PGP SIGNATURE-----
iQEzBAABCgAdFiEEkUejOXGVGO6QEby1R5ORYRMIQCUFAmTDr+oACgkQR5ORYRMI
QCXfwgf/cAqWCgnnKIT3hvUdL2i2F9edDXTOkBDZ1vxQ8hLO+T8JtWO9F40hEZjH
F5R5B2pxBP6T2Nd9NHVbHUqlIpjqqesweTXtabuW60oz5PZ13owpGDWWQKortH5/
j49v/ZKHD0NBjVN09AylRgoKQ4kRDtd0rMOAS951aRUcRTFRjK86hnaHPgvQeexy
SizGRtHlifnwM/lbgJlLkTDUDNA+7RwXRAv0pvLwYReGFoS8vyUbMOYt1lnoiNas
6cz+6yTKknGO7KSE6bjviDahv7Xg04Qy02eI/HYEZ8NG3aJqNsOqPchP4y/JgVv+
90FZR2cdZNpTdlZ5TPfihL2/zldXKQ==
=H6+4
-----END PGP SIGNATURE-----

11
SOURCES/ldb-2.9.1.tar.asc Normal file
View File

@ -0,0 +1,11 @@
-----BEGIN PGP SIGNATURE-----
iQEzBAABCgAdFiEEkUejOXGVGO6QEby1R5ORYRMIQCUFAmZy6sMACgkQR5ORYRMI
QCViOQf/Wo1d05OXbX0hVC1L79cpCCQ6WDui27POHwx0clHhg6rChXtxqNscBsR+
rvKX2qnnKtrByQdTlORK2/K3ynM/3CV4pCNwoKCxgpfLumrPkTEtyHP9sTVQ37PC
Pm85uIoqedcleahuVLM1w2+Ns1aTrKKrdqjv9eIuGfzU4U7nt5hitUejRiJWis7a
5OurMd1gvUDv5QAgvcQWqgfqZ8lGmDc0QFOMj2aSepU1n9MBSjYrtqBGwB9/DkT8
OTR0MY4JUe2RKYjK1OhUiJtEhB1PEj3uiLiS1UwZ9jSwUfEhjDFMwUD2KWiZROaF
GFZsqp8zeJaEgYk2soYDqrQMcV/wiw==
=FzI6
-----END PGP SIGNATURE-----

221
SOURCES/libldb-fix-indexes-performance.patch Normal file
View File

@ -0,0 +1,221 @@
From 1944fcf4b7e5ab4cf580e17031918ba5f441902b Mon Sep 17 00:00:00 2001
From: Douglas Bagnall <douglas.bagnall@catalyst.net.nz>
Date: Wed, 26 Jun 2024 11:05:49 +1200
Subject: [PATCH 1/2] ldb_kv_index: dn_list load sub transaction can re-use
keys
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
We don't want to modify the original list, but we can reuse the keys
if we treat them as immutable and don't free them. That makes it a lot
quicker if there are many keys (i.e. where an index is useful) and may
sub-transactions. In particular, it avoids O(n²) talloc_memdups.
A removed comment that says "We have to free the top level index
memory otherwise we would leak", and this will be addressed in the
next commit.
BUG: https://bugzilla.samba.org/show_bug.cgi?id=15590
Signed-off-by: Douglas Bagnall <douglas.bagnall@catalyst.net.nz>
Reviewed-by: Stefan Metzmacher <metze@samba.org>
(cherry picked from commit 5f0198d69843c864f2b98a7c0c6305ad789a68a0)
---
lib/ldb/ldb_key_value/ldb_kv_index.c | 96 +++++++++++++++++-----------
1 file changed, 57 insertions(+), 39 deletions(-)
diff --git a/lib/ldb/ldb_key_value/ldb_kv_index.c b/lib/ldb/ldb_key_value/ldb_kv_index.c
index 3f1a847f2b6..fed1033f492 100644
--- a/lib/ldb/ldb_key_value/ldb_kv_index.c
+++ b/lib/ldb/ldb_key_value/ldb_kv_index.c
@@ -446,34 +446,39 @@ static int ldb_kv_dn_list_load(struct ldb_module *module,
* There is an active index sub transaction, and the record was
* found in the primary index transaction cache. A copy of the
* record needs be taken to prevent the original entry being
- * altered, until the index sub transaction is committed.
+ * altered, until the index sub transaction is committed, but we
+ * don't copy the actual values, just the array of struct ldb_val
+ * that points to the values (which are offsets into a GUID array).
+ *
+ * As a reminder, our primary cache is an in-memory tdb that
+ * maps attributes to struct dn_list objects, which point to
+ * the actual index, which is an array of struct ldb_val, the
+ * contents of which are {.data = <binary GUID>, .length =
+ * 16}. The array is sorted by GUID data, and these GUIDs are
+ * used to look up index entries in the main database. There
+ * are more layers of indirection than necessary, but what
+ * makes the index useful is we can use a binary search to
+ * find if the array contains a GUID.
+ *
+ * What we do in a sub-transaction is make a copy of the struct
+ * dn_list and the array of struct ldb_val, but *not* of the
+ * .data that they point to. This copy is put into a new
+ * in-memory tdb which masks the primary cache for the duration
+ * of the sub-transaction.
+ *
+ * In an add operation in a sub-transaction, the new ldb_val
+ * is a child of the sub-transaction dn_list, which will
+ * become the main dn_list if the transaction succeeds.
+ *
+ * These acrobatics do not affect read-only operations.
*/
-
- {
- struct ldb_val *dns = NULL;
- size_t x = 0;
-
- dns = talloc_array(
- list,
- struct ldb_val,
- list2->count);
- if (dns == NULL) {
- return LDB_ERR_OPERATIONS_ERROR;
- }
- for (x = 0; x < list2->count; x++) {
- dns[x].length = list2->dn[x].length;
- dns[x].data = talloc_memdup(
- dns,
- list2->dn[x].data,
- list2->dn[x].length);
- if (dns[x].data == NULL) {
- TALLOC_FREE(dns);
- return LDB_ERR_OPERATIONS_ERROR;
- }
- }
- list->dn = dns;
- list->count = list2->count;
+ list->dn = talloc_memdup(list,
+ list2->dn,
+ talloc_get_size(list2->dn));
+ if (list->dn == NULL) {
+ return LDB_ERR_OPERATIONS_ERROR;
}
+ list->count = list2->count;
return LDB_SUCCESS;
/*
@@ -3852,9 +3857,7 @@ int ldb_kv_reindex(struct ldb_module *module)
* Copy the contents of the nested transaction index cache record to the
* transaction index cache.
*
- * During this 'commit' of the subtransaction to the main transaction
- * (cache), care must be taken to free any existing index at the top
- * level because otherwise we would leak memory.
+ * This is a 'commit' of the subtransaction to the main transaction cache.
*/
static int ldb_kv_sub_transaction_traverse(
struct tdb_context *tdb,
@@ -3883,8 +3886,7 @@ static int ldb_kv_sub_transaction_traverse(
/*
* Do we already have an entry in the primary transaction cache
- * If so free it's dn_list and replace it with the dn_list from
- * the secondary cache
+ * If so replace dn_list with the one from the subtransaction.
*
* The TDB and so the fetched rec contains NO DATA, just a
* pointer to data held in memory.
@@ -3897,21 +3899,37 @@ static int ldb_kv_sub_transaction_traverse(
abort();
}
/*
- * We had this key at the top level. However we made a copy
- * at the sub-transaction level so that we could possibly
- * roll back. We have to free the top level index memory
- * otherwise we would leak
+ * We had this key at the top level, and made a copy
+ * of the dn list for this sub-transaction level that
+ * borrowed the top level GUID data. We can't free the
+ * original dn list just yet.
+ *
+ * In this diagram, ... is the C pointer structure
+ * and --- is the talloc structure (::: is both).
+ *
+ * index_in_top_level ::: dn orig ..............
+ * | | :
+ * | `--GUID array :
+ * | |----- val1 data
+ * ldb_kv `----- val2 data
+ * | :
+ * index_in_subtransaction :: dn copy ..........:
+ * | :
+ * `------------ new val3 data
+ *
+ * So we don't free the index_in_top_level dn list yet,
+ * because we are (probably) borrowing most of its
+ * children.
*/
- if (index_in_top_level->count > 0) {
- TALLOC_FREE(index_in_top_level->dn);
- }
index_in_top_level->dn
= talloc_steal(index_in_top_level,
index_in_subtransaction->dn);
index_in_top_level->count = index_in_subtransaction->count;
return 0;
}
-
+ /*
+ * We found no top level index in the cache, so we put one in.
+ */
index_in_top_level = talloc(ldb_kv->idxptr, struct dn_list);
if (index_in_top_level == NULL) {
ldb_kv->idxptr->error = LDB_ERR_OPERATIONS_ERROR;
--
2.46.0
From 70d8b1b2f87cbb16b671d334e46244ba001fbd31 Mon Sep 17 00:00:00 2001
From: Douglas Bagnall <douglas.bagnall@catalyst.net.nz>
Date: Mon, 22 Jul 2024 22:22:15 +1200
Subject: [PATCH 2/2] ldb:kv_index: realloc away old dn list
We can't just free it, because has the GUID index list as a child, and
these are shared by the new dn list (from the subtransaction we are
committing). But if the dn list is long and the main transaction is
long-lived, we can save a lot of memory by turning this dn list into
an almost empty node in the talloc tree. This returns us to roughly
the situation we had prior to the last commit.
For example, with the repro.sh script on bug 15590 in indexes mode
with 10000 rules, The last 3 commits use this much memory at the end
of an unusually large transaction:
full talloc report on 'struct ldb_context' (total 4012222 bytes in 90058 blocks)
full talloc report on 'struct ldb_context' (total 2405482219 bytes in 90058 blocks)
full talloc report on 'struct ldb_context' (total 4282195 bytes in 90058 blocks)
That is, the last commit increased usage 500 fold, and this commit
brings it back to normal.
BUG: https://bugzilla.samba.org/show_bug.cgi?id=15590
Signed-off-by: Douglas Bagnall <douglas.bagnall@catalyst.net.nz>
Reviewed-by: Stefan Metzmacher <metze@samba.org>
(cherry picked from commit 1bf9ede94f0a6b41fb18e880e59a8e390f8c21d3)
---
lib/ldb/ldb_key_value/ldb_kv_index.c | 6 +++++-
1 file changed, 5 insertions(+), 1 deletion(-)
diff --git a/lib/ldb/ldb_key_value/ldb_kv_index.c b/lib/ldb/ldb_key_value/ldb_kv_index.c
index fed1033f492..11bdf00dc08 100644
--- a/lib/ldb/ldb_key_value/ldb_kv_index.c
+++ b/lib/ldb/ldb_key_value/ldb_kv_index.c
@@ -3919,8 +3919,12 @@ static int ldb_kv_sub_transaction_traverse(
*
* So we don't free the index_in_top_level dn list yet,
* because we are (probably) borrowing most of its
- * children.
+ * children. But we can save memory by discarding the
+ * values and keeping it as an almost empty talloc
+ * node.
*/
+ talloc_realloc(index_in_top_level,
+ index_in_top_level->dn, struct ldb_val *, 1);
index_in_top_level->dn
= talloc_steal(index_in_top_level,
index_in_subtransaction->dn);
--
2.46.0

80
SPECS/libldb.spec
View File

@ -1,5 +1,3 @@
%if ((0%{?fedora} || 0%{?rhel} > 7 || 0%{?epel} > 6))
# lmdb is not supported on 32 bit architectures
%ifarch aarch64 ppc64le s390x x86_64
%bcond_without lmdb
@ -8,24 +6,13 @@
#endif arch
%endif
%else
%bcond_with lmdb
#endif fedora || rhel || epel
%endif
%if 0%{?fedora} || 0%{?rhel} > 7
%bcond_without python3
%else
%bcond_with python3
%endif
%global talloc_version 2.4.1
%global tdb_version 1.4.9
%global tevent_version 0.15.0
%global talloc_version 2.4.2
%global tdb_version 1.4.10
%global tevent_version 0.16.1
Name: libldb
Version: 2.8.0
Release: 1%{?dist}
Version: 2.9.1
Release: 2%{?dist}
Summary: A schema-less, ldap like, API and database
Requires: libtalloc%{?_isa} >= %{talloc_version}
Requires: libtdb%{?_isa} >= %{tdb_version}
@ -36,31 +23,27 @@ Source0: https://www.samba.org/ftp/ldb/ldb-%{version}.tar.gz
Source1: https://www.samba.org/ftp/ldb/ldb-%{version}.tar.asc
# gpg2 --no-default-keyring --keyring ./ldb.keyring --recv-keys 9147A339719518EE9011BCB54793916113084025
Source2: ldb.keyring
Patch0: libldb-fix-indexes-performance.patch
# Patches
BuildRequires: docbook-style-xsl
BuildRequires: doxygen
BuildRequires: gcc
BuildRequires: gnupg2
BuildRequires: libcmocka-devel
BuildRequires: libtalloc-devel >= %{talloc_version}
BuildRequires: libtdb-devel >= %{tdb_version}
BuildRequires: libtevent-devel >= %{tevent_version}
BuildRequires: libxslt
BuildRequires: make
BuildRequires: openldap-devel
BuildRequires: popt-devel
BuildRequires: python3-devel
BuildRequires: python3-talloc-devel
BuildRequires: python3-tdb
BuildRequires: python3-tevent
%if %{with lmdb}
BuildRequires: lmdb-devel >= 0.9.16
%endif
BuildRequires: popt-devel
BuildRequires: libxslt
BuildRequires: docbook-style-xsl
%if %{with python3}
BuildRequires: python3-devel
BuildRequires: python3-tdb
BuildRequires: python3-talloc-devel
BuildRequires: python3-tevent
#endif with python
%endif
BuildRequires: doxygen
BuildRequires: openldap-devel
BuildRequires: libcmocka-devel
BuildRequires: gnupg2
BuildRequires: make
Provides: bundled(libreplace)
Obsoletes: python2-ldb < 2.0.5-1
@ -99,7 +82,6 @@ Provides: pyldb-devel%{?_isa} = %{version}-%{release}
Development files for the Python bindings for the LDB library.
This package includes files that aren't specific to a Python version.
%if %{with python3}
%package -n python3-ldb
Summary: Python bindings for the LDB library
Requires: libldb%{?_isa} = %{version}-%{release}
@ -119,17 +101,12 @@ Requires: python-ldb-devel-common%{?_isa} = %{version}-%{release}
%description -n python3-ldb-devel
Development files for the Python bindings for the LDB library
#endif with python
%endif
%prep
zcat %{SOURCE0} | gpgv2 --quiet --keyring %{SOURCE2} %{SOURCE1} -
%autosetup -n ldb-%{version} -p1
%autosetup -n ldb-%{version} -p3
%build
# workaround for https://bugzilla.redhat.com/show_bug.cgi?id=1217376
export python_LDFLAGS=""
%configure --disable-rpath \
--disable-rpath-install \
--bundled-libraries=NONE \
@ -153,13 +130,11 @@ make %{?_smp_mflags} check
%make_install
# Install API docs
cp -a apidocs/man/* $RPM_BUILD_ROOT/%{_mandir}
cp -a apidocs/man/* %{buildroot}%{_mandir}
# bug: remove manpage named after full file path
# not needed with el8+ and fc28+
rm -f $RPM_BUILD_ROOT/%{_mandir}/man3/_*
%ldconfig_scriptlets
rm -f %{buildroot}%{_mandir}/man3/_*
%files
%dir %{_libdir}/ldb
@ -205,7 +180,6 @@ rm -f $RPM_BUILD_ROOT/%{_mandir}/man3/_*
%{_includedir}/pyldb.h
%{_mandir}/man*/Py*.gz
%if %{with python3}
%files -n python3-ldb
%{python3_sitearch}/ldb.cpython-*.so
%{_libdir}/libpyldb-util.cpython-*.so.2*
@ -216,11 +190,19 @@ rm -f $RPM_BUILD_ROOT/%{_mandir}/man3/_*
%{_libdir}/libpyldb-util.cpython-*.so
%{_libdir}/pkgconfig/pyldb-util.cpython-*.pc
%ldconfig_scriptlets
%ldconfig_scriptlets -n python3-ldb
#endif with python
%endif
%changelog
* Mon Aug 12 2024 Andreas Schneider <asn@redhat.com> - 2.9.1-2
* resolves: RHEL-53994 - Fix performance regression with indexes
* Tue Jul 02 2024 Pavel Filipenský <pfilipen@redhat.com> - 2.9.1-1
- related: RHEL-33757 - Rebase version to 2.9.1
* Thu Apr 25 2024 Pavel Filipenský <pfilipen@redhat.com> - 2.9.0-1
- resolves: RHEL-33757 - Rebase version to 2.9.0
* Mon Dec 04 2023 Andreas Schneider <asn@redhat.com> - 2.8.0-1
- resolves: RHEL-16482 - Rebase version to 2.8.0