From d354d3760f9e8e7485d2902cc5bf30e6cbd3dfec Mon Sep 17 00:00:00 2001 From: Viktor Ashirov Date: Thu, 12 Feb 2026 11:59:09 +0100 Subject: [PATCH] Bump version to 2.8.0-3 - Resolves: RHEL-117050 - Replication online reinitialization of a large database gets stalled. [rhel-9] - Resolves: RHEL-123244 - Attribute uniqueness is not enforced upon modrdn operation [rhel-9] - Resolves: RHEL-123279 - The new ipahealthcheck test ipahealthcheck.ds.backends.BackendsCheck raises CRITICAL issue [rhel-9] - Resolves: RHEL-140275 - ipa-healthcheck is complaining about missing or incorrectly configured system indexes. [rhel-9] - Resolves: RHEL-142980 - Scalability issue of replication online initialization with large database [rhel-9] - Resolves: RHEL-146899 - memory corruption in alias entry plugin [rhel-9] - Resolves: RHEL-147212 - Access logs are not getting deleted as configured. [rhel-9] --- ...nfig_set-asserts-because-of-dynamic-.patch | 31 + ...st-Simplify-test_reserve_descriptor_.patch | 93 ++ ...0007-generates-incorrect-remediation.patch | 235 ++++ ...89-ds-base-OpenScanHub-Leaks-Detecte.patch | 53 + ...t-index-scan-limits-for-system-index.patch | 785 +++++++++++ ...pgrade-function-to-remove-nsIndexIDL.patch | 212 +++ ...t-and-log-index-ordering-mismatch-du.patch | 300 ++++ ...sctl-index-check-command-for-offline.patch | 1233 +++++++++++++++++ ...uring-replication-online-total-init-.patch | 135 ++ ...-6784-6214-Fix-CI-test-failures-7077.patch | 877 ++++++++++++ ...evert_cache-never-called-in-modrdn-7.patch | 57 + ...47-Fix-health_system_indexes_test.py.patch | 24 + ...eakSanitizer-various-leaks-during-re.patch | 71 + ...essed-access-log-rotations-skipped-a.patch | 551 ++++++++ 389-ds-base.spec | 75 +- 15 files changed, 4707 insertions(+), 25 deletions(-) create mode 100644 0006-Issue-7166-db_config_set-asserts-because-of-dynamic-.patch create mode 100644 0007-Issue-7224-CI-Test-Simplify-test_reserve_descriptor_.patch create mode 100644 0008-Issue-7189-DSBLE0007-generates-incorrect-remediation.patch create mode 100644 0009-Issue-7027-2nd-389-ds-base-OpenScanHub-Leaks-Detecte.patch create mode 100644 0010-Issue-7223-Revert-index-scan-limits-for-system-index.patch create mode 100644 0011-Issue-7223-Add-upgrade-function-to-remove-nsIndexIDL.patch create mode 100644 0012-Issue-7223-Detect-and-log-index-ordering-mismatch-du.patch create mode 100644 0013-Issue-7223-Add-dsctl-index-check-command-for-offline.patch create mode 100644 0014-Issue-7096-2nd-During-replication-online-total-init-.patch create mode 100644 0015-Issue-7076-6992-6784-6214-Fix-CI-test-failures-7077.patch create mode 100644 0016-Issue-7076-Fix-revert_cache-never-called-in-modrdn-7.patch create mode 100644 0017-Issue-6947-Fix-health_system_indexes_test.py.patch create mode 100644 0018-Issue-7121-2nd-LeakSanitizer-various-leaks-during-re.patch create mode 100644 0019-Issue-7150-Compressed-access-log-rotations-skipped-a.patch diff --git a/0006-Issue-7166-db_config_set-asserts-because-of-dynamic-.patch b/0006-Issue-7166-db_config_set-asserts-because-of-dynamic-.patch new file mode 100644 index 0000000..47a4845 --- /dev/null +++ b/0006-Issue-7166-db_config_set-asserts-because-of-dynamic-.patch @@ -0,0 +1,31 @@ +From 4eea3051931f552ee1df36e70c7278aaa36124a1 Mon Sep 17 00:00:00 2001 +From: progier389 +Date: Mon, 5 Jan 2026 14:38:38 +0100 +Subject: [PATCH] Issue 7166 - db_config_set asserts because of dynamic list + (#7167) + +Avoid assertion in db_config_set when args does not contains dynamic list attributes + +Issue: #7166 + +Reviewed by: @tbordaz (Thanks!) +--- + src/lib389/lib389/cli_conf/backend.py | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/src/lib389/lib389/cli_conf/backend.py b/src/lib389/lib389/cli_conf/backend.py +index dcf57d006..f8735e08e 100644 +--- a/src/lib389/lib389/cli_conf/backend.py ++++ b/src/lib389/lib389/cli_conf/backend.py +@@ -542,7 +542,7 @@ def db_config_set(inst, basedn, log, args): + did_something = False + replace_list = [] + +- if args.enable_dynamic_lists and args.disable_dynamic_lists: ++ if getattr(args,'enable_dynamic_lists', None) and getattr(args, 'disable_dynamic_lists', None): + raise ValueError("You can not enable and disable dynamic lists at the same time") + + for attr, value in list(attrs.items()): +-- +2.52.0 + diff --git a/0007-Issue-7224-CI-Test-Simplify-test_reserve_descriptor_.patch b/0007-Issue-7224-CI-Test-Simplify-test_reserve_descriptor_.patch new file mode 100644 index 0000000..49e10e0 --- /dev/null +++ b/0007-Issue-7224-CI-Test-Simplify-test_reserve_descriptor_.patch @@ -0,0 +1,93 @@ +From 8c7de629b83fe12a36cabe89e2b19124dcd6c937 Mon Sep 17 00:00:00 2001 +From: James Chapman +Date: Thu, 5 Feb 2026 15:33:08 +0000 +Subject: [PATCH] Issue 7224 - CI Test - Simplify + test_reserve_descriptor_validation (#7225) + +Description: +Previously, the test_reserve_descriptor_validation CItest calculated +the expected number of file descriptors based on backends, indexes, +SSL/FIPS mode, and compared it to the value returned by the server. +This approach is fragile, especially in FIPS mode. + +Fix: +The test has been updated to simply verify that the server corrects +the configured nsslapd-reservedescriptors value if it is set too low, +instead of calculating the expected total. + +Fixes: https://github.com/389ds/389-ds-base/issues/7224 + +Reviewed by: @bsimonova (Thank you) +--- + .../suites/resource_limits/fdlimits_test.py | 36 +++++++------------ + 1 file changed, 13 insertions(+), 23 deletions(-) + +diff --git a/dirsrvtests/tests/suites/resource_limits/fdlimits_test.py b/dirsrvtests/tests/suites/resource_limits/fdlimits_test.py +index edcae28a5..9d8fdec52 100644 +--- a/dirsrvtests/tests/suites/resource_limits/fdlimits_test.py ++++ b/dirsrvtests/tests/suites/resource_limits/fdlimits_test.py +@@ -27,7 +27,7 @@ RESRV_FD_ATTR = "nsslapd-reservedescriptors" + GLOBAL_LIMIT = resource.getrlimit(resource.RLIMIT_NOFILE)[1] + SYSTEMD_LIMIT = ensure_str(check_output("systemctl show -p LimitNOFILE dirsrv@standalone1".split(" ")).strip()).split('=')[1] + CUSTOM_VAL = str(int(SYSTEMD_LIMIT) - 10) +-RESRV_DESC_VAL = str(10) ++RESRV_DESC_VAL_LOW = 10 + TOO_HIGH_VAL = str(GLOBAL_LIMIT * 2) + TOO_HIGH_VAL2 = str(int(SYSTEMD_LIMIT) * 2) + TOO_LOW_VAL = "0" +@@ -86,40 +86,30 @@ def test_reserve_descriptor_validation(topology_st): + :id: 9bacdbcc-7754-4955-8a56-1d8c82bce274 + :setup: Standalone Instance + :steps: +- 1. Set attr nsslapd-reservedescriptors to a low value of RESRV_DESC_VAL (10) ++ 1. Set attr nsslapd-reservedescriptors to a low value (10) + 2. Verify low value has been set + 3. Restart instance (On restart the reservedescriptor attr will be validated) +- 4. Check updated value for nsslapd-reservedescriptors attr ++ 4. Verify corrected value for nsslapd-reservedescriptors > low value + :expectedresults: + 1. Success +- 2. A value of RESRV_DESC_VAL (10) is returned ++ 2. A value of RESRV_DESC_VAL_LOW (10) is returned + 3. Success +- 4. A value of STANDALONE_INST_RESRV_DESCS (55) is returned ++ 4. Corrected value for nsslapd-reservedescriptors > low value + """ + +- # Set nsslapd-reservedescriptors to a low value (RESRV_DESC_VAL:10) +- topology_st.standalone.config.set(RESRV_FD_ATTR, RESRV_DESC_VAL) +- resrv_fd = topology_st.standalone.config.get_attr_val_utf8(RESRV_FD_ATTR) +- assert resrv_fd == RESRV_DESC_VAL ++ # Set nsslapd-reservedescriptors to a low value (10) ++ topology_st.standalone.config.set(RESRV_FD_ATTR, str(RESRV_DESC_VAL_LOW)) ++ resrv_fd = int(topology_st.standalone.config.get_attr_val_utf8(RESRV_FD_ATTR)) ++ assert resrv_fd == RESRV_DESC_VAL_LOW + + # An instance restart triggers a validation of the configured nsslapd-reservedescriptors attribute + topology_st.standalone.restart() + +- """ +- A standalone instance contains a single backend with default indexes +- so we only check these. TODO add tests for repl, chaining, PTA, SSL +- """ +- STANDALONE_INST_RESRV_DESCS = 20 # 20 = Reserve descriptor constant +- backends = Backends(topology_st.standalone) +- STANDALONE_INST_RESRV_DESCS += (len(backends.list()) * 4) # 4 = Backend descriptor constant +- for be in backends.list() : +- STANDALONE_INST_RESRV_DESCS += len(be.get_indexes().list()) +- +- # Varify reservedescriptors has been updated +- resrv_fd = topology_st.standalone.config.get_attr_val_utf8(RESRV_FD_ATTR) +- assert resrv_fd == str(STANDALONE_INST_RESRV_DESCS) ++ # Get the corrected value ++ corrected_fd = int(topology_st.standalone.config.get_attr_val_utf8(RESRV_FD_ATTR)) ++ assert corrected_fd > RESRV_DESC_VAL_LOW + +- log.info("test_reserve_descriptor_validation PASSED") ++ log.info(f"test_reserve_descriptor_validation PASSED (corrected from {RESRV_DESC_VAL_LOW} to {corrected_fd})") + + @pytest.mark.skipif(ds_is_older("1.4.1.2"), reason="Not implemented") + def test_reserve_descriptors_high(topology_st): +-- +2.52.0 + diff --git a/0008-Issue-7189-DSBLE0007-generates-incorrect-remediation.patch b/0008-Issue-7189-DSBLE0007-generates-incorrect-remediation.patch new file mode 100644 index 0000000..15a7113 --- /dev/null +++ b/0008-Issue-7189-DSBLE0007-generates-incorrect-remediation.patch @@ -0,0 +1,235 @@ +From ed02c78055406b020089947459954142df3b1fea Mon Sep 17 00:00:00 2001 +From: Viktor Ashirov +Date: Tue, 20 Jan 2026 09:52:47 +0100 +Subject: [PATCH] Issue 7189 - DSBLE0007 generates incorrect remediation + commands for scan limits + +Bug Description: + +The generated dsconf commands for fixing missing system indexes had two issues: + +1. The --add-scanlimit value was not quoted, causing the shell to interpret + "limit=5000 type=eq flags=AND" as multiple arguments instead of a single + value, resulting in "unrecognized arguments: type=eq flags=AND" error. + +2. When both matching rule and scanlimit were missing, two separate commands + were generated where the second would fail because the matching rule was + already added by the first command. + +Fix Description: + +1. Quote the scanlimit value in all remediation commands + +2. Combine matching rule and scanlimit fixes into a single command when + both are missing for the same index instead of expected_scanlimit) + +Fixes: https://github.com/389ds/389-ds-base/issues/7189 + +Reviewed by: @progier389, @droideck (Thanks!) +--- + .../healthcheck/health_system_indexes_test.py | 126 ++++++++++++++++++ + src/lib389/lib389/backend.py | 39 +++--- + 2 files changed, 147 insertions(+), 18 deletions(-) + +diff --git a/dirsrvtests/tests/suites/healthcheck/health_system_indexes_test.py b/dirsrvtests/tests/suites/healthcheck/health_system_indexes_test.py +index 6293340ca..5eadf6283 100644 +--- a/dirsrvtests/tests/suites/healthcheck/health_system_indexes_test.py ++++ b/dirsrvtests/tests/suites/healthcheck/health_system_indexes_test.py +@@ -405,6 +405,132 @@ def test_retrocl_plugin_missing_matching_rule(topology_st, retrocl_plugin_enable + run_healthcheck_and_flush_log(topology_st, standalone, json=True, searched_code=JSON_OUTPUT) + + ++def test_missing_scanlimit(topology_st, log_buffering_enabled): ++ """Check if healthcheck returns DSBLE0007 code when parentId index is missing scanlimit ++ ++ :id: 40e1bf6a-2397-459b-bdf3-f787ca118b86 ++ :setup: Standalone instance ++ :steps: ++ 1. Create DS instance ++ 2. Remove nsIndexIDListScanLimit from parentId index ++ 3. Use healthcheck without --json option ++ 4. Use healthcheck with --json option ++ 5. Verify the remediation command has properly quoted scanlimit ++ 6. Re-add the scanlimit ++ 7. Use healthcheck without --json option ++ 8. Use healthcheck with --json option ++ :expectedresults: ++ 1. Success ++ 2. Success ++ 3. healthcheck reports DSBLE0007 code and related details ++ 4. healthcheck reports DSBLE0007 code and related details ++ 5. The scanlimit value is quoted in the remediation command ++ 6. Success ++ 7. healthcheck reports no issues found ++ 8. healthcheck reports no issues found ++ """ ++ ++ RET_CODE = "DSBLE0007" ++ PARENTID_DN = "cn=parentid,cn=index,cn=userroot,cn=ldbm database,cn=plugins,cn=config" ++ SCANLIMIT_VALUE = "limit=5000 type=eq flags=AND" ++ ++ standalone = topology_st.standalone ++ ++ log.info("Remove nsIndexIDListScanLimit from parentId index") ++ parentid_index = Index(standalone, PARENTID_DN) ++ parentid_index.remove("nsIndexIDListScanLimit", SCANLIMIT_VALUE) ++ ++ run_healthcheck_and_flush_log(topology_st, standalone, json=False, searched_code=RET_CODE) ++ ++ # Verify the remediation command has properly quoted scanlimit ++ args = FakeArgs() ++ args.instance = standalone.serverid ++ args.verbose = standalone.verbose ++ args.list_errors = False ++ args.list_checks = False ++ args.exclude_check = [] ++ args.check = ["backends"] ++ args.dry_run = False ++ args.json = False ++ health_check_run(standalone, topology_st.logcap.log, args) ++ # Check that the scanlimit is quoted in the output ++ assert topology_st.logcap.contains('--add-scanlimit "limit=5000 type=eq flags=AND"') ++ log.info("Verified scanlimit is properly quoted in remediation command") ++ topology_st.logcap.flush() ++ ++ run_healthcheck_and_flush_log(topology_st, standalone, json=True, searched_code=RET_CODE) ++ ++ log.info("Re-add the nsIndexIDListScanLimit") ++ parentid_index = Index(standalone, PARENTID_DN) ++ parentid_index.add("nsIndexIDListScanLimit", SCANLIMIT_VALUE) ++ ++ run_healthcheck_and_flush_log(topology_st, standalone, json=False, searched_code=CMD_OUTPUT) ++ run_healthcheck_and_flush_log(topology_st, standalone, json=True, searched_code=JSON_OUTPUT) ++ ++ ++def test_missing_matching_rule_and_scanlimit(topology_st, log_buffering_enabled): ++ """Check if healthcheck generates a single combined command when both matching rule and scanlimit are missing ++ ++ :id: af8214ad-5e4c-422a-8f74-3e99227551df ++ :setup: Standalone instance ++ :steps: ++ 1. Create DS instance ++ 2. Remove both integerOrderingMatch and nsIndexIDListScanLimit from parentId index ++ 3. Use healthcheck and verify a single combined command is generated ++ 4. Re-add the matching rule and scanlimit ++ 5. Use healthcheck without --json option ++ 6. Use healthcheck with --json option ++ :expectedresults: ++ 1. Success ++ 2. Success ++ 3. healthcheck reports DSBLE0007 and generates a single command with both --add-mr and --add-scanlimit ++ 4. Success ++ 5. healthcheck reports no issues found ++ 6. healthcheck reports no issues found ++ """ ++ ++ RET_CODE = "DSBLE0007" ++ PARENTID_DN = "cn=parentid,cn=index,cn=userroot,cn=ldbm database,cn=plugins,cn=config" ++ SCANLIMIT_VALUE = "limit=5000 type=eq flags=AND" ++ ++ standalone = topology_st.standalone ++ ++ log.info("Remove both integerOrderingMatch and nsIndexIDListScanLimit from parentId index") ++ parentid_index = Index(standalone, PARENTID_DN) ++ parentid_index.remove("nsMatchingRule", "integerOrderingMatch") ++ parentid_index.remove("nsIndexIDListScanLimit", SCANLIMIT_VALUE) ++ ++ # Run healthcheck and verify combined command ++ args = FakeArgs() ++ args.instance = standalone.serverid ++ args.verbose = standalone.verbose ++ args.list_errors = False ++ args.list_checks = False ++ args.exclude_check = [] ++ args.check = ["backends"] ++ args.dry_run = False ++ args.json = False ++ health_check_run(standalone, topology_st.logcap.log, args) ++ ++ # Verify DSBLE0007 is reported ++ assert topology_st.logcap.contains(RET_CODE) ++ log.info("healthcheck returned code: %s" % RET_CODE) ++ ++ # Verify a single combined command is generated with both --add-mr and --add-scanlimit ++ assert topology_st.logcap.contains('--add-mr integerOrderingMatch --add-scanlimit "limit=5000 type=eq flags=AND"') ++ log.info("Verified combined command with both --add-mr and --add-scanlimit") ++ ++ topology_st.logcap.flush() ++ ++ log.info("Re-add the integerOrderingMatch matching rule and scanlimit") ++ parentid_index = Index(standalone, PARENTID_DN) ++ parentid_index.add("nsMatchingRule", "integerOrderingMatch") ++ parentid_index.add("nsIndexIDListScanLimit", SCANLIMIT_VALUE) ++ ++ run_healthcheck_and_flush_log(topology_st, standalone, json=False, searched_code=CMD_OUTPUT) ++ run_healthcheck_and_flush_log(topology_st, standalone, json=True, searched_code=JSON_OUTPUT) ++ ++ + def test_multiple_missing_indexes(topology_st, log_buffering_enabled): + """Check if healthcheck returns DSBLE0007 code when multiple system indexes are missing + +diff --git a/src/lib389/lib389/backend.py b/src/lib389/lib389/backend.py +index 024de2adb..d3c9ccf35 100644 +--- a/src/lib389/lib389/backend.py ++++ b/src/lib389/lib389/backend.py +@@ -678,7 +678,7 @@ class Backend(DSLdapObject): + if expected_config.get('matching_rule'): + cmd += f" --matching-rule {expected_config['matching_rule']}" + if expected_config.get('scanlimit'): +- cmd += f" --add-scanlimit {expected_config['scanlimit']}" ++ cmd += f" --add-scanlimit \"{expected_config['scanlimit']}\"" + remediation_commands.append(cmd) + reindex_attrs.add(attr_name) # New index needs reindexing + else: +@@ -700,28 +700,31 @@ class Backend(DSLdapObject): + remediation_commands.append(cmd) + reindex_attrs.add(attr_name) + +- # Check matching rules ++ # Check matching rules and scanlimit together to generate a single combined command + expected_mr = expected_config.get('matching_rule') ++ expected_scanlimit = expected_config.get('scanlimit') ++ ++ missing_mr = False + if expected_mr: + actual_mrs_lower = [mr.lower() for mr in actual_mrs] + if expected_mr.lower() not in actual_mrs_lower: + discrepancies.append(f"Index {attr_name} missing matching rule: {expected_mr}") +- # Add the missing matching rule +- cmd = f"dsconf YOUR_INSTANCE backend index set {bename} --attr {attr_name} --add-mr {expected_mr}" +- remediation_commands.append(cmd) +- reindex_attrs.add(attr_name) +- +- # Check fine grain definitions for parentid ONLY +- expected_scanlimit = expected_config.get('scanlimit') +- if (attr_name.lower() == "parentid") and expected_scanlimit and (len(actual_scanlimit) == 0): +- discrepancies.append(f"Index {attr_name} missing fine grain definition of IDs limit: {expected_mr}") +- # Add the missing scanlimit +- if expected_mr: +- cmd = f"dsconf YOUR_INSTANCE backend index set {bename} --attr {attr_name} --add-mr {expected_mr} --add-scanlimit {expected_scanlimit}" +- else: +- cmd = f"dsconf YOUR_INSTANCE backend index set {bename} --attr {attr_name} --add-scanlimit {expected_scanlimit}" +- remediation_commands.append(cmd) +- reindex_attrs.add(attr_name) ++ missing_mr = True ++ ++ missing_scanlimit = False ++ if expected_scanlimit and (len(actual_scanlimit) == 0): ++ discrepancies.append(f"Index {attr_name} missing fine grain definition of IDs limit: {expected_scanlimit}") ++ missing_scanlimit = True ++ ++ # Generate a single combined command for all missing items ++ if missing_mr or missing_scanlimit: ++ cmd = f"dsconf YOUR_INSTANCE backend index set {bename} --attr {attr_name}" ++ if missing_mr: ++ cmd += f" --add-mr {expected_mr}" ++ if missing_scanlimit: ++ cmd += f" --add-scanlimit \"{expected_scanlimit}\"" ++ remediation_commands.append(cmd) ++ reindex_attrs.add(attr_name) + + except Exception as e: + self._log.debug(f"_lint_system_indexes - Error checking index {attr_name}: {e}") +-- +2.52.0 + diff --git a/0009-Issue-7027-2nd-389-ds-base-OpenScanHub-Leaks-Detecte.patch b/0009-Issue-7027-2nd-389-ds-base-OpenScanHub-Leaks-Detecte.patch new file mode 100644 index 0000000..ce44451 --- /dev/null +++ b/0009-Issue-7027-2nd-389-ds-base-OpenScanHub-Leaks-Detecte.patch @@ -0,0 +1,53 @@ +From 819a5898d1a4d913f786086f374693c306446abb Mon Sep 17 00:00:00 2001 +From: Viktor Ashirov +Date: Fri, 30 Jan 2026 12:00:13 +0100 +Subject: [PATCH] Issue 7027 - (2nd) 389-ds-base OpenScanHub Leaks Detected + (#7211) + +Fix Description: +Update coverity annotations. + +Relates: https://github.com/389ds/389-ds-base/issues/7027 + +Reviewed by: @aadhikar (Thanks!) +--- + ldap/servers/slapd/log.c | 6 +++--- + 1 file changed, 3 insertions(+), 3 deletions(-) + +diff --git a/ldap/servers/slapd/log.c b/ldap/servers/slapd/log.c +index 668a02454..6f57a3d9c 100644 +--- a/ldap/servers/slapd/log.c ++++ b/ldap/servers/slapd/log.c +@@ -203,8 +203,8 @@ compress_log_file(char *log_name, int32_t mode) + + if ((source = fopen(log_name, "r")) == NULL) { + /* Failed to open log file */ +- /* coverity[leaked_storage] gzclose does close FD */ + gzclose(outfile); ++ /* coverity[leaked_handle] gzclose does close FD */ + return -1; + } + +@@ -214,17 +214,17 @@ compress_log_file(char *log_name, int32_t mode) + if (bytes_written == 0) + { + fclose(source); +- /* coverity[leaked_storage] gzclose does close FD */ + gzclose(outfile); ++ /* coverity[leaked_handle] gzclose does close FD */ + return -1; + } + bytes_read = fread(buf, 1, LOG_CHUNK, source); + } +- /* coverity[leaked_storage] gzclose does close FD */ + gzclose(outfile); + fclose(source); + PR_Delete(log_name); /* remove the old uncompressed log */ + ++ /* coverity[leaked_handle] gzclose does close FD */ + return 0; + } + +-- +2.52.0 + diff --git a/0010-Issue-7223-Revert-index-scan-limits-for-system-index.patch b/0010-Issue-7223-Revert-index-scan-limits-for-system-index.patch new file mode 100644 index 0000000..d2ea7ef --- /dev/null +++ b/0010-Issue-7223-Revert-index-scan-limits-for-system-index.patch @@ -0,0 +1,785 @@ +From 41c72bf9900a7e28e0f589384f6e0ce292607e9d Mon Sep 17 00:00:00 2001 +From: Viktor Ashirov +Date: Thu, 5 Feb 2026 12:17:06 +0100 +Subject: [PATCH] Issue 7223 - Revert index scan limits for system indexes + +This reverts changes introduced by the following commits: +c6f458b42 Issue 7189 - DSBLE0007 generates incorrect remediation commands for scan limits +8b6b3a9f9 Issue 6966 - On large DB, unlimited IDL scan limit reduce the SRCH performance + +Relates: https://github.com/389ds/389-ds-base/issues/7223 + +Reviewed by: @progier389, @tbordaz, @droideck (Thanks!) +--- + .../tests/suites/config/config_test.py | 27 +--- + .../healthcheck/health_system_indexes_test.py | 135 +----------------- + .../paged_results/paged_results_test.py | 25 +--- + ldap/servers/slapd/back-ldbm/back-ldbm.h | 1 - + ldap/servers/slapd/back-ldbm/index.c | 2 - + ldap/servers/slapd/back-ldbm/instance.c | 106 +++----------- + ldap/servers/slapd/back-ldbm/ldbm_config.c | 30 ---- + ldap/servers/slapd/back-ldbm/ldbm_config.h | 1 - + .../slapd/back-ldbm/ldbm_index_config.c | 8 -- + src/lib389/lib389/backend.py | 50 ++----- + src/lib389/lib389/cli_conf/backend.py | 20 --- + 11 files changed, 41 insertions(+), 364 deletions(-) + +diff --git a/dirsrvtests/tests/suites/config/config_test.py b/dirsrvtests/tests/suites/config/config_test.py +index b9ba684d9..6f11e7fc8 100644 +--- a/dirsrvtests/tests/suites/config/config_test.py ++++ b/dirsrvtests/tests/suites/config/config_test.py +@@ -715,19 +715,17 @@ def test_ndn_cache_size_enforcement(topo, request): + + request.addfinalizer(fin) + +-def test_require_index(topo, request): ++def test_require_index(topo): + """Validate that unindexed searches are rejected + + :id: fb6e31f2-acc2-4e75-a195-5c356faeb803 + :setup: Standalone instance + :steps: + 1. Set "nsslapd-require-index" to "on" +- 2. ancestorid/idlscanlimit to 100 +- 3. Test an unindexed search is rejected ++ 2. Test an unindexed search is rejected + :expectedresults: + 1. Success + 2. Success +- 3. Success + """ + + # Set the config +@@ -738,10 +736,6 @@ def test_require_index(topo, request): + + db_cfg = DatabaseConfig(topo.standalone) + db_cfg.set([('nsslapd-idlistscanlimit', '100')]) +- backend = Backends(topo.standalone).get_backend(DEFAULT_SUFFIX) +- ancestorid_index = backend.get_index('ancestorid') +- ancestorid_index.replace("nsIndexIDListScanLimit", ensure_bytes("limit=100 type=eq flags=AND")) +- topo.standalone.restart() + + users = UserAccounts(topo.standalone, DEFAULT_SUFFIX) + for i in range(101): +@@ -752,15 +746,10 @@ def test_require_index(topo, request): + with pytest.raises(ldap.UNWILLING_TO_PERFORM): + raw_objects.filter("(description=test*)") + +- def fin(): +- ancestorid_index.replace("nsIndexIDListScanLimit", ensure_bytes("limit=5000 type=eq flags=AND")) +- +- request.addfinalizer(fin) +- + + + @pytest.mark.skipif(ds_is_older('1.4.2'), reason="The config setting only exists in 1.4.2 and higher") +-def test_require_internal_index(topo, request): ++def test_require_internal_index(topo): + """Ensure internal operations require indexed attributes + + :id: 22b94f30-59e3-4f27-89a1-c4f4be036f7f +@@ -791,10 +780,6 @@ def test_require_internal_index(topo, request): + # Create a bunch of users + db_cfg = DatabaseConfig(topo.standalone) + db_cfg.set([('nsslapd-idlistscanlimit', '100')]) +- backend = Backends(topo.standalone).get_backend(DEFAULT_SUFFIX) +- ancestorid_index = backend.get_index('ancestorid') +- ancestorid_index.replace("nsIndexIDListScanLimit", ensure_bytes("limit=100 type=eq flags=AND")) +- topo.standalone.restart() + users = UserAccounts(topo.standalone, DEFAULT_SUFFIX) + for i in range(102, 202): + users.create_test_user(uid=i) +@@ -819,12 +804,6 @@ def test_require_internal_index(topo, request): + with pytest.raises(ldap.UNWILLING_TO_PERFORM): + user.delete() + +- def fin(): +- ancestorid_index.replace("nsIndexIDListScanLimit", ensure_bytes("limit=5000 type=eq flags=AND")) +- +- request.addfinalizer(fin) +- +- + + def get_pstack(pid): + """Get a pstack of the pid.""" +diff --git a/dirsrvtests/tests/suites/healthcheck/health_system_indexes_test.py b/dirsrvtests/tests/suites/healthcheck/health_system_indexes_test.py +index 5eadf6283..61972d60c 100644 +--- a/dirsrvtests/tests/suites/healthcheck/health_system_indexes_test.py ++++ b/dirsrvtests/tests/suites/healthcheck/health_system_indexes_test.py +@@ -171,8 +171,7 @@ def test_missing_parentid(topology_st, log_buffering_enabled): + + log.info("Re-add the parentId index") + backend = Backends(standalone).get("userRoot") +- backend.add_index("parentid", ["eq"], matching_rules=["integerOrderingMatch"], +- idlistscanlimit=['limit=5000 type=eq flags=AND']) ++ backend.add_index("parentid", ["eq"], matching_rules=["integerOrderingMatch"]) + + run_healthcheck_and_flush_log(topology_st, standalone, json=False, searched_code=CMD_OUTPUT) + run_healthcheck_and_flush_log(topology_st, standalone, json=True, searched_code=JSON_OUTPUT) +@@ -260,8 +259,7 @@ def test_usn_plugin_missing_entryusn(topology_st, usn_plugin_enabled, log_buffer + + log.info("Re-add the entryusn index") + backend = Backends(standalone).get("userRoot") +- backend.add_index("entryusn", ["eq"], matching_rules=["integerOrderingMatch"], +- idlistscanlimit=['limit=5000 type=eq flags=AND']) ++ backend.add_index("entryusn", ["eq"], matching_rules=["integerOrderingMatch"]) + + run_healthcheck_and_flush_log(topology_st, standalone, json=False, searched_code=CMD_OUTPUT) + run_healthcheck_and_flush_log(topology_st, standalone, json=True, searched_code=JSON_OUTPUT) +@@ -405,132 +403,6 @@ def test_retrocl_plugin_missing_matching_rule(topology_st, retrocl_plugin_enable + run_healthcheck_and_flush_log(topology_st, standalone, json=True, searched_code=JSON_OUTPUT) + + +-def test_missing_scanlimit(topology_st, log_buffering_enabled): +- """Check if healthcheck returns DSBLE0007 code when parentId index is missing scanlimit +- +- :id: 40e1bf6a-2397-459b-bdf3-f787ca118b86 +- :setup: Standalone instance +- :steps: +- 1. Create DS instance +- 2. Remove nsIndexIDListScanLimit from parentId index +- 3. Use healthcheck without --json option +- 4. Use healthcheck with --json option +- 5. Verify the remediation command has properly quoted scanlimit +- 6. Re-add the scanlimit +- 7. Use healthcheck without --json option +- 8. Use healthcheck with --json option +- :expectedresults: +- 1. Success +- 2. Success +- 3. healthcheck reports DSBLE0007 code and related details +- 4. healthcheck reports DSBLE0007 code and related details +- 5. The scanlimit value is quoted in the remediation command +- 6. Success +- 7. healthcheck reports no issues found +- 8. healthcheck reports no issues found +- """ +- +- RET_CODE = "DSBLE0007" +- PARENTID_DN = "cn=parentid,cn=index,cn=userroot,cn=ldbm database,cn=plugins,cn=config" +- SCANLIMIT_VALUE = "limit=5000 type=eq flags=AND" +- +- standalone = topology_st.standalone +- +- log.info("Remove nsIndexIDListScanLimit from parentId index") +- parentid_index = Index(standalone, PARENTID_DN) +- parentid_index.remove("nsIndexIDListScanLimit", SCANLIMIT_VALUE) +- +- run_healthcheck_and_flush_log(topology_st, standalone, json=False, searched_code=RET_CODE) +- +- # Verify the remediation command has properly quoted scanlimit +- args = FakeArgs() +- args.instance = standalone.serverid +- args.verbose = standalone.verbose +- args.list_errors = False +- args.list_checks = False +- args.exclude_check = [] +- args.check = ["backends"] +- args.dry_run = False +- args.json = False +- health_check_run(standalone, topology_st.logcap.log, args) +- # Check that the scanlimit is quoted in the output +- assert topology_st.logcap.contains('--add-scanlimit "limit=5000 type=eq flags=AND"') +- log.info("Verified scanlimit is properly quoted in remediation command") +- topology_st.logcap.flush() +- +- run_healthcheck_and_flush_log(topology_st, standalone, json=True, searched_code=RET_CODE) +- +- log.info("Re-add the nsIndexIDListScanLimit") +- parentid_index = Index(standalone, PARENTID_DN) +- parentid_index.add("nsIndexIDListScanLimit", SCANLIMIT_VALUE) +- +- run_healthcheck_and_flush_log(topology_st, standalone, json=False, searched_code=CMD_OUTPUT) +- run_healthcheck_and_flush_log(topology_st, standalone, json=True, searched_code=JSON_OUTPUT) +- +- +-def test_missing_matching_rule_and_scanlimit(topology_st, log_buffering_enabled): +- """Check if healthcheck generates a single combined command when both matching rule and scanlimit are missing +- +- :id: af8214ad-5e4c-422a-8f74-3e99227551df +- :setup: Standalone instance +- :steps: +- 1. Create DS instance +- 2. Remove both integerOrderingMatch and nsIndexIDListScanLimit from parentId index +- 3. Use healthcheck and verify a single combined command is generated +- 4. Re-add the matching rule and scanlimit +- 5. Use healthcheck without --json option +- 6. Use healthcheck with --json option +- :expectedresults: +- 1. Success +- 2. Success +- 3. healthcheck reports DSBLE0007 and generates a single command with both --add-mr and --add-scanlimit +- 4. Success +- 5. healthcheck reports no issues found +- 6. healthcheck reports no issues found +- """ +- +- RET_CODE = "DSBLE0007" +- PARENTID_DN = "cn=parentid,cn=index,cn=userroot,cn=ldbm database,cn=plugins,cn=config" +- SCANLIMIT_VALUE = "limit=5000 type=eq flags=AND" +- +- standalone = topology_st.standalone +- +- log.info("Remove both integerOrderingMatch and nsIndexIDListScanLimit from parentId index") +- parentid_index = Index(standalone, PARENTID_DN) +- parentid_index.remove("nsMatchingRule", "integerOrderingMatch") +- parentid_index.remove("nsIndexIDListScanLimit", SCANLIMIT_VALUE) +- +- # Run healthcheck and verify combined command +- args = FakeArgs() +- args.instance = standalone.serverid +- args.verbose = standalone.verbose +- args.list_errors = False +- args.list_checks = False +- args.exclude_check = [] +- args.check = ["backends"] +- args.dry_run = False +- args.json = False +- health_check_run(standalone, topology_st.logcap.log, args) +- +- # Verify DSBLE0007 is reported +- assert topology_st.logcap.contains(RET_CODE) +- log.info("healthcheck returned code: %s" % RET_CODE) +- +- # Verify a single combined command is generated with both --add-mr and --add-scanlimit +- assert topology_st.logcap.contains('--add-mr integerOrderingMatch --add-scanlimit "limit=5000 type=eq flags=AND"') +- log.info("Verified combined command with both --add-mr and --add-scanlimit") +- +- topology_st.logcap.flush() +- +- log.info("Re-add the integerOrderingMatch matching rule and scanlimit") +- parentid_index = Index(standalone, PARENTID_DN) +- parentid_index.add("nsMatchingRule", "integerOrderingMatch") +- parentid_index.add("nsIndexIDListScanLimit", SCANLIMIT_VALUE) +- +- run_healthcheck_and_flush_log(topology_st, standalone, json=False, searched_code=CMD_OUTPUT) +- run_healthcheck_and_flush_log(topology_st, standalone, json=True, searched_code=JSON_OUTPUT) +- +- + def test_multiple_missing_indexes(topology_st, log_buffering_enabled): + """Check if healthcheck returns DSBLE0007 code when multiple system indexes are missing + +@@ -571,8 +443,7 @@ def test_multiple_missing_indexes(topology_st, log_buffering_enabled): + + log.info("Re-add the missing system indexes") + backend = Backends(standalone).get("userRoot") +- backend.add_index("parentid", ["eq"], matching_rules=["integerOrderingMatch"], +- idlistscanlimit=['limit=5000 type=eq flags=AND']) ++ backend.add_index("parentid", ["eq"], matching_rules=["integerOrderingMatch"]) + backend.add_index("nsuniqueid", ["eq"]) + + run_healthcheck_and_flush_log(topology_st, standalone, json=False, searched_code=CMD_OUTPUT) +diff --git a/dirsrvtests/tests/suites/paged_results/paged_results_test.py b/dirsrvtests/tests/suites/paged_results/paged_results_test.py +index 61d6702da..1bb94b53a 100644 +--- a/dirsrvtests/tests/suites/paged_results/paged_results_test.py ++++ b/dirsrvtests/tests/suites/paged_results/paged_results_test.py +@@ -306,19 +306,19 @@ def test_search_success(topology_st, create_user, page_size, users_num): + del_users(users_list) + + +-@pytest.mark.parametrize("page_size,users_num,suffix,attr_name,attr_value,expected_err, restart", [ ++@pytest.mark.parametrize("page_size,users_num,suffix,attr_name,attr_value,expected_err", [ + (50, 200, 'cn=config,%s' % DN_LDBM, 'nsslapd-idlistscanlimit', '100', +- ldap.UNWILLING_TO_PERFORM, True), ++ ldap.UNWILLING_TO_PERFORM), + (5, 15, DN_CONFIG, 'nsslapd-timelimit', '20', +- ldap.UNAVAILABLE_CRITICAL_EXTENSION, False), ++ ldap.UNAVAILABLE_CRITICAL_EXTENSION), + (21, 50, DN_CONFIG, 'nsslapd-sizelimit', '20', +- ldap.SIZELIMIT_EXCEEDED, False), ++ ldap.SIZELIMIT_EXCEEDED), + (21, 50, DN_CONFIG, 'nsslapd-pagedsizelimit', '5', +- ldap.SIZELIMIT_EXCEEDED, False), ++ ldap.SIZELIMIT_EXCEEDED), + (5, 50, 'cn=config,%s' % DN_LDBM, 'nsslapd-lookthroughlimit', '20', +- ldap.ADMINLIMIT_EXCEEDED, False)]) ++ ldap.ADMINLIMIT_EXCEEDED)]) + def test_search_limits_fail(topology_st, create_user, page_size, users_num, +- suffix, attr_name, attr_value, expected_err, restart): ++ suffix, attr_name, attr_value, expected_err): + """Verify that search with a simple paged results control + throws expected exceptoins when corresponding limits are + exceeded. +@@ -341,15 +341,6 @@ def test_search_limits_fail(topology_st, create_user, page_size, users_num, + + users_list = add_users(topology_st, users_num, DEFAULT_SUFFIX) + attr_value_bck = change_conf_attr(topology_st, suffix, attr_name, attr_value) +- ancestorid_index = None +- if attr_name == 'nsslapd-idlistscanlimit': +- backend = Backends(topology_st.standalone).get_backend(DEFAULT_SUFFIX) +- ancestorid_index = backend.get_index('ancestorid') +- ancestorid_index.replace("nsIndexIDListScanLimit", ensure_bytes("limit=100 type=eq flags=AND")) +- +- if (restart): +- log.info('Instance restarted') +- topology_st.standalone.restart() + conf_param_dict = {attr_name: attr_value} + search_flt = r'(uid=test*)' + searchreq_attrlist = ['dn', 'sn'] +@@ -402,8 +393,6 @@ def test_search_limits_fail(topology_st, create_user, page_size, users_num, + else: + break + finally: +- if ancestorid_index: +- ancestorid_index.replace("nsIndexIDListScanLimit", ensure_bytes("limit=5000 type=eq flags=AND")) + del_users(users_list) + change_conf_attr(topology_st, suffix, attr_name, attr_value_bck) + +diff --git a/ldap/servers/slapd/back-ldbm/back-ldbm.h b/ldap/servers/slapd/back-ldbm/back-ldbm.h +index 5e4988782..a4993208c 100644 +--- a/ldap/servers/slapd/back-ldbm/back-ldbm.h ++++ b/ldap/servers/slapd/back-ldbm/back-ldbm.h +@@ -561,7 +561,6 @@ struct ldbminfo + int li_mode; + int li_lookthroughlimit; + int li_allidsthreshold; +- int li_system_allidsthreshold; + char *li_directory; + int li_reslimit_lookthrough_handle; + uint64_t li_dbcachesize; +diff --git a/ldap/servers/slapd/back-ldbm/index.c b/ldap/servers/slapd/back-ldbm/index.c +index e06a8ee5f..90129b682 100644 +--- a/ldap/servers/slapd/back-ldbm/index.c ++++ b/ldap/servers/slapd/back-ldbm/index.c +@@ -1007,8 +1007,6 @@ index_read_ext_allids( + } + if (pb) { + slapi_pblock_get(pb, SLAPI_SEARCH_IS_AND, &is_and); +- } else if (strcasecmp(type, LDBM_ANCESTORID_STR) == 0) { +- is_and = 1; + } + ai_flags = is_and ? INDEX_ALLIDS_FLAG_AND : 0; + /* the caller can pass in a value of 0 - just ignore those - but if the index +diff --git a/ldap/servers/slapd/back-ldbm/instance.c b/ldap/servers/slapd/back-ldbm/instance.c +index 23bd70243..f9a546661 100644 +--- a/ldap/servers/slapd/back-ldbm/instance.c ++++ b/ldap/servers/slapd/back-ldbm/instance.c +@@ -16,7 +16,7 @@ + + /* Forward declarations */ + static void ldbm_instance_destructor(void **arg); +-Slapi_Entry *ldbm_instance_init_config_entry(char *cn_val, char *v1, char *v2, char *v3, char *v4, char *mr, char *scanlimit); ++Slapi_Entry *ldbm_instance_init_config_entry(char *cn_val, char *v1, char *v2, char *v3, char *v4, char *mr); + + + /* Creates and initializes a new ldbm_instance structure. +@@ -127,7 +127,7 @@ done: + * Take a bunch of strings, and create a index config entry + */ + Slapi_Entry * +-ldbm_instance_init_config_entry(char *cn_val, char *val1, char *val2, char *val3, char *val4, char *mr, char *scanlimit) ++ldbm_instance_init_config_entry(char *cn_val, char *val1, char *val2, char *val3, char *val4, char *mr) + { + Slapi_Entry *e = slapi_entry_alloc(); + struct berval *vals[2]; +@@ -168,11 +168,6 @@ ldbm_instance_init_config_entry(char *cn_val, char *val1, char *val2, char *val3 + slapi_entry_add_values(e, "nsMatchingRule", vals); + } + +- if (scanlimit) { +- val.bv_val = scanlimit; +- val.bv_len = strlen(scanlimit); +- slapi_entry_add_values(e, "nsIndexIDListScanLimit", vals); +- } + return e; + } + +@@ -185,60 +180,8 @@ ldbm_instance_create_default_indexes(backend *be) + { + Slapi_Entry *e; + ldbm_instance *inst = (ldbm_instance *)be->be_instance_info; +- struct ldbminfo *li = (struct ldbminfo *)be->be_database->plg_private; + /* write the dse file only on the final index */ + int flags = LDBM_INSTANCE_CONFIG_DONT_WRITE; +- char *ancestorid_indexes_limit = NULL; +- char *parentid_indexes_limit = NULL; +- struct attrinfo *ai = NULL; +- int index_already_configured = 0; +- struct index_idlistsizeinfo *iter; +- int cookie; +- int limit; +- +- ainfo_get(be, (char *)LDBM_ANCESTORID_STR, &ai); +- if (ai && ai->ai_idlistinfo) { +- iter = (struct index_idlistsizeinfo *)dl_get_first(ai->ai_idlistinfo, &cookie); +- if (iter) { +- limit = iter->ai_idlistsizelimit; +- slapi_log_err(SLAPI_LOG_BACKLDBM, "ldbm_instance_create_default_indexes", +- "set ancestorid limit to %d from attribute index\n", +- limit); +- } else { +- limit = li->li_system_allidsthreshold; +- slapi_log_err(SLAPI_LOG_BACKLDBM, "ldbm_instance_create_default_indexes", +- "set ancestorid limit to %d from default (fail to read limit)\n", +- limit); +- } +- ancestorid_indexes_limit = slapi_ch_smprintf("limit=%d type=eq flags=AND", limit); +- } else { +- ancestorid_indexes_limit = slapi_ch_smprintf("limit=%d type=eq flags=AND", li->li_system_allidsthreshold); +- slapi_log_err(SLAPI_LOG_BACKLDBM, "ldbm_instance_create_default_indexes", +- "set ancestorid limit to %d from default (no attribute or limit)\n", +- li->li_system_allidsthreshold); +- } +- +- ainfo_get(be, (char *)LDBM_PARENTID_STR, &ai); +- if (ai && ai->ai_idlistinfo) { +- iter = (struct index_idlistsizeinfo *)dl_get_first(ai->ai_idlistinfo, &cookie); +- if (iter) { +- limit = iter->ai_idlistsizelimit; +- slapi_log_err(SLAPI_LOG_BACKLDBM, "ldbm_instance_create_default_indexes", +- "set parentid limit to %d from attribute index\n", +- limit); +- } else { +- limit = li->li_system_allidsthreshold; +- slapi_log_err(SLAPI_LOG_BACKLDBM, "ldbm_instance_create_default_indexes", +- "set parentid limit to %d from default (fail to read limit)\n", +- limit); +- } +- parentid_indexes_limit = slapi_ch_smprintf("limit=%d type=eq flags=AND", limit); +- } else { +- parentid_indexes_limit = slapi_ch_smprintf("limit=%d type=eq flags=AND", li->li_system_allidsthreshold); +- slapi_log_err(SLAPI_LOG_BACKLDBM, "ldbm_instance_create_default_indexes", +- "set parentid limit to %d from default (no attribute or limit)\n", +- li->li_system_allidsthreshold); +- } + + /* + * Always index (entrydn or entryrdn), parentid, objectclass, +@@ -247,53 +190,47 @@ ldbm_instance_create_default_indexes(backend *be) + * ACL routines. + */ + if (entryrdn_get_switch()) { /* subtree-rename: on */ +- e = ldbm_instance_init_config_entry(LDBM_ENTRYRDN_STR, "subtree", 0, 0, 0, 0, 0); ++ e = ldbm_instance_init_config_entry(LDBM_ENTRYRDN_STR, "subtree", 0, 0, 0, 0); + ldbm_instance_config_add_index_entry(inst, e, flags); + slapi_entry_free(e); + } else { +- e = ldbm_instance_init_config_entry(LDBM_ENTRYDN_STR, "eq", 0, 0, 0, 0, 0); ++ e = ldbm_instance_init_config_entry(LDBM_ENTRYDN_STR, "eq", 0, 0, 0, 0); + ldbm_instance_config_add_index_entry(inst, e, flags); + slapi_entry_free(e); + } + +- ainfo_get(be, (char *)LDBM_PARENTID_STR, &ai); +- /* Check if the attrinfo is actually for parentid, not a fallback to .default */ +- index_already_configured = (ai != NULL && strcmp(ai->ai_type, LDBM_PARENTID_STR) == 0); +- if (!index_already_configured) { +- e = ldbm_instance_init_config_entry(LDBM_PARENTID_STR, "eq", 0, 0, 0, "integerOrderingMatch", parentid_indexes_limit); +- ldbm_instance_config_add_index_entry(inst, e, flags); +- attr_index_config(be, "ldbm index init", 0, e, 1, 0, NULL); +- slapi_entry_free(e); +- } ++ e = ldbm_instance_init_config_entry(LDBM_PARENTID_STR, "eq", 0, 0, 0, "integerOrderingMatch"); ++ ldbm_instance_config_add_index_entry(inst, e, flags); ++ slapi_entry_free(e); + +- e = ldbm_instance_init_config_entry("objectclass", "eq", 0, 0, 0, 0, 0); ++ e = ldbm_instance_init_config_entry("objectclass", "eq", 0, 0, 0, 0); + ldbm_instance_config_add_index_entry(inst, e, flags); + slapi_entry_free(e); + +- e = ldbm_instance_init_config_entry("aci", "pres", 0, 0, 0, 0, 0); ++ e = ldbm_instance_init_config_entry("aci", "pres", 0, 0, 0, 0); + ldbm_instance_config_add_index_entry(inst, e, flags); + slapi_entry_free(e); + +- e = ldbm_instance_init_config_entry(LDBM_NUMSUBORDINATES_STR, "pres", 0, 0, 0, 0, 0); ++ e = ldbm_instance_init_config_entry(LDBM_NUMSUBORDINATES_STR, "pres", 0, 0, 0, 0); + ldbm_instance_config_add_index_entry(inst, e, flags); + slapi_entry_free(e); + +- e = ldbm_instance_init_config_entry(SLAPI_ATTR_UNIQUEID, "eq", 0, 0, 0, 0, 0); ++ e = ldbm_instance_init_config_entry(SLAPI_ATTR_UNIQUEID, "eq", 0, 0, 0, 0); + ldbm_instance_config_add_index_entry(inst, e, flags); + slapi_entry_free(e); + + /* For MMR, we need this attribute (to replace use of dncomp in delete). */ +- e = ldbm_instance_init_config_entry(ATTR_NSDS5_REPLCONFLICT, "eq", "pres", 0, 0, 0, 0); ++ e = ldbm_instance_init_config_entry(ATTR_NSDS5_REPLCONFLICT, "eq", "pres", 0, 0, 0); + ldbm_instance_config_add_index_entry(inst, e, flags); + slapi_entry_free(e); + + /* write the dse file only on the final index */ +- e = ldbm_instance_init_config_entry(SLAPI_ATTR_NSCP_ENTRYDN, "eq", 0, 0, 0, 0, 0); ++ e = ldbm_instance_init_config_entry(SLAPI_ATTR_NSCP_ENTRYDN, "eq", 0, 0, 0, 0); + ldbm_instance_config_add_index_entry(inst, e, flags); + slapi_entry_free(e); + + /* ldbm_instance_config_add_index_entry(inst, 2, argv); */ +- e = ldbm_instance_init_config_entry(LDBM_PSEUDO_ATTR_DEFAULT, "none", 0, 0, 0, 0, 0); ++ e = ldbm_instance_init_config_entry(LDBM_PSEUDO_ATTR_DEFAULT, "none", 0, 0, 0, 0); + attr_index_config(be, "ldbm index init", 0, e, 1, 0, NULL); + slapi_entry_free(e); + +@@ -302,20 +239,11 @@ ldbm_instance_create_default_indexes(backend *be) + * ancestorid is special, there is actually no such attr type + * but we still want to use the attr index file APIs. + */ +- ainfo_get(be, (char *)LDBM_ANCESTORID_STR, &ai); +- /* Check if the attrinfo is actually for ancestorid, not a fallback to .default */ +- index_already_configured = (ai != NULL && strcmp(ai->ai_type, LDBM_ANCESTORID_STR) == 0); +- if (!index_already_configured) { +- e = ldbm_instance_init_config_entry(LDBM_ANCESTORID_STR, "eq", 0, 0, 0, "integerOrderingMatch", ancestorid_indexes_limit); +- ldbm_instance_config_add_index_entry(inst, e, flags); +- attr_index_config(be, "ldbm index init", 0, e, 1, 0, NULL); +- slapi_entry_free(e); +- } ++ e = ldbm_instance_init_config_entry(LDBM_ANCESTORID_STR, "eq", 0, 0, 0, "integerOrderingMatch"); ++ attr_index_config(be, "ldbm index init", 0, e, 1, 0, NULL); ++ slapi_entry_free(e); + } + +- slapi_ch_free_string(&ancestorid_indexes_limit); +- slapi_ch_free_string(&parentid_indexes_limit); +- + return 0; + } + +diff --git a/ldap/servers/slapd/back-ldbm/ldbm_config.c b/ldap/servers/slapd/back-ldbm/ldbm_config.c +index d8ffc4479..5cce0dbd3 100644 +--- a/ldap/servers/slapd/back-ldbm/ldbm_config.c ++++ b/ldap/servers/slapd/back-ldbm/ldbm_config.c +@@ -386,35 +386,6 @@ ldbm_config_allidsthreshold_set(void *arg, void *value, char *errorbuf __attribu + return retval; + } + +-static void * +-ldbm_config_system_allidsthreshold_get(void *arg) +-{ +- struct ldbminfo *li = (struct ldbminfo *)arg; +- +- return (void *)((uintptr_t)(li->li_system_allidsthreshold)); +-} +- +-static int +-ldbm_config_system_allidsthreshold_set(void *arg, void *value, char *errorbuf __attribute__((unused)), int phase __attribute__((unused)), int apply) +-{ +- struct ldbminfo *li = (struct ldbminfo *)arg; +- int retval = LDAP_SUCCESS; +- int val = (int)((uintptr_t)value); +- +- /* Do whatever we can to make sure the data is ok. */ +- +- /* Catch attempts to configure a stupidly low ancestorid allidsthreshold */ +- if ((val > -1) && (val < 5000)) { +- val = 5000; +- } +- +- if (apply) { +- li->li_system_allidsthreshold = val; +- } +- +- return retval; +-} +- + static void * + ldbm_config_pagedallidsthreshold_get(void *arg) + { +@@ -1133,7 +1104,6 @@ static config_info ldbm_config[] = { + {CONFIG_LOOKTHROUGHLIMIT, CONFIG_TYPE_INT, "5000", &ldbm_config_lookthroughlimit_get, &ldbm_config_lookthroughlimit_set, CONFIG_FLAG_ALWAYS_SHOW | CONFIG_FLAG_ALLOW_RUNNING_CHANGE}, + {CONFIG_MODE, CONFIG_TYPE_INT_OCTAL, "0600", &ldbm_config_mode_get, &ldbm_config_mode_set, CONFIG_FLAG_ALWAYS_SHOW | CONFIG_FLAG_ALLOW_RUNNING_CHANGE}, + {CONFIG_IDLISTSCANLIMIT, CONFIG_TYPE_INT, "2147483646", &ldbm_config_allidsthreshold_get, &ldbm_config_allidsthreshold_set, CONFIG_FLAG_ALWAYS_SHOW | CONFIG_FLAG_ALLOW_RUNNING_CHANGE}, +- {CONFIG_SYSTEMIDLISTSCANLIMIT, CONFIG_TYPE_INT, "5000", &ldbm_config_system_allidsthreshold_get, &ldbm_config_system_allidsthreshold_set, CONFIG_FLAG_ALWAYS_SHOW | CONFIG_FLAG_ALLOW_RUNNING_CHANGE}, + {CONFIG_DIRECTORY, CONFIG_TYPE_STRING, "", &ldbm_config_directory_get, &ldbm_config_directory_set, CONFIG_FLAG_ALWAYS_SHOW | CONFIG_FLAG_ALLOW_RUNNING_CHANGE | CONFIG_FLAG_SKIP_DEFAULT_SETTING}, + {CONFIG_MAXPASSBEFOREMERGE, CONFIG_TYPE_INT, "100", &ldbm_config_maxpassbeforemerge_get, &ldbm_config_maxpassbeforemerge_set, 0}, + +diff --git a/ldap/servers/slapd/back-ldbm/ldbm_config.h b/ldap/servers/slapd/back-ldbm/ldbm_config.h +index 30e3477ed..e1f05d7a2 100644 +--- a/ldap/servers/slapd/back-ldbm/ldbm_config.h ++++ b/ldap/servers/slapd/back-ldbm/ldbm_config.h +@@ -60,7 +60,6 @@ struct config_info + #define CONFIG_RANGELOOKTHROUGHLIMIT "nsslapd-rangelookthroughlimit" + #define CONFIG_PAGEDLOOKTHROUGHLIMIT "nsslapd-pagedlookthroughlimit" + #define CONFIG_IDLISTSCANLIMIT "nsslapd-idlistscanlimit" +-#define CONFIG_SYSTEMIDLISTSCANLIMIT "nsslapd-systemidlistscanlimit" + #define CONFIG_PAGEDIDLISTSCANLIMIT "nsslapd-pagedidlistscanlimit" + #define CONFIG_DIRECTORY "nsslapd-directory" + #define CONFIG_MODE "nsslapd-mode" +diff --git a/ldap/servers/slapd/back-ldbm/ldbm_index_config.c b/ldap/servers/slapd/back-ldbm/ldbm_index_config.c +index bae2a64b9..38e7368e1 100644 +--- a/ldap/servers/slapd/back-ldbm/ldbm_index_config.c ++++ b/ldap/servers/slapd/back-ldbm/ldbm_index_config.c +@@ -384,14 +384,6 @@ ldbm_instance_config_add_index_entry( + } + } + +- /* get nsIndexIDListScanLimit and its values, and add them */ +- if (0 == slapi_entry_attr_find(e, "nsIndexIDListScanLimit", &attr)) { +- for (j = slapi_attr_first_value(attr, &sval); j != -1; j = slapi_attr_next_value(attr, j, &sval)) { +- attrValue = slapi_value_get_berval(sval); +- eBuf = PR_sprintf_append(eBuf, "nsIndexIDListScanLimit: %s\n", attrValue->bv_val); +- } +- } +- + ldbm_config_add_dse_entry(li, eBuf, flags); + if (eBuf) { + PR_smprintf_free(eBuf); +diff --git a/src/lib389/lib389/backend.py b/src/lib389/lib389/backend.py +index d3c9ccf35..1d9be4683 100644 +--- a/src/lib389/lib389/backend.py ++++ b/src/lib389/lib389/backend.py +@@ -615,10 +615,11 @@ class Backend(DSLdapObject): + indexes = self.get_indexes() + + # Default system indexes taken from ldap/servers/slapd/back-ldbm/instance.c ++ # Note: entryrdn and ancestorid are internal system indexes that are not ++ # exposed in cn=config - they are managed internally by the server. ++ # Only parentid has a DSE config entry (for the integerOrderingMatch rule). + expected_system_indexes = { +- 'entryrdn': {'types': ['subtree'], 'matching_rule': None}, +- 'parentid': {'types': ['eq'], 'matching_rule': 'integerOrderingMatch', 'scanlimit': 'limit=5000 type=eq flags=AND'}, +- 'ancestorid': {'types': ['eq'], 'matching_rule': 'integerOrderingMatch', 'scanlimit': 'limit=5000 type=eq flags=AND'}, ++ 'parentid': {'types': ['eq'], 'matching_rule': 'integerOrderingMatch'}, + 'objectClass': {'types': ['eq'], 'matching_rule': None}, + 'aci': {'types': ['pres'], 'matching_rule': None}, + 'nscpEntryDN': {'types': ['eq'], 'matching_rule': None}, +@@ -675,17 +676,14 @@ class Backend(DSLdapObject): + # Generate remediation command + index_types = ' '.join([f"--index-type {t}" for t in expected_config['types']]) + cmd = f"dsconf YOUR_INSTANCE backend index add {bename} --attr {attr_name} {index_types}" +- if expected_config.get('matching_rule'): ++ if expected_config['matching_rule']: + cmd += f" --matching-rule {expected_config['matching_rule']}" +- if expected_config.get('scanlimit'): +- cmd += f" --add-scanlimit \"{expected_config['scanlimit']}\"" + remediation_commands.append(cmd) + reindex_attrs.add(attr_name) # New index needs reindexing + else: + # Index exists, check configuration + actual_types = index.get_attr_vals_utf8('nsIndexType') or [] + actual_mrs = index.get_attr_vals_utf8('nsMatchingRule') or [] +- actual_scanlimit = index.get_attr_vals_utf8('nsIndexIDListScanLimit') or [] + + # Normalize to lowercase for comparison + actual_types = [t.lower() for t in actual_types] +@@ -700,31 +698,16 @@ class Backend(DSLdapObject): + remediation_commands.append(cmd) + reindex_attrs.add(attr_name) + +- # Check matching rules and scanlimit together to generate a single combined command ++ # Check matching rules + expected_mr = expected_config.get('matching_rule') +- expected_scanlimit = expected_config.get('scanlimit') +- +- missing_mr = False + if expected_mr: + actual_mrs_lower = [mr.lower() for mr in actual_mrs] + if expected_mr.lower() not in actual_mrs_lower: + discrepancies.append(f"Index {attr_name} missing matching rule: {expected_mr}") +- missing_mr = True +- +- missing_scanlimit = False +- if expected_scanlimit and (len(actual_scanlimit) == 0): +- discrepancies.append(f"Index {attr_name} missing fine grain definition of IDs limit: {expected_scanlimit}") +- missing_scanlimit = True +- +- # Generate a single combined command for all missing items +- if missing_mr or missing_scanlimit: +- cmd = f"dsconf YOUR_INSTANCE backend index set {bename} --attr {attr_name}" +- if missing_mr: +- cmd += f" --add-mr {expected_mr}" +- if missing_scanlimit: +- cmd += f" --add-scanlimit \"{expected_scanlimit}\"" +- remediation_commands.append(cmd) +- reindex_attrs.add(attr_name) ++ # Add the missing matching rule ++ cmd = f"dsconf YOUR_INSTANCE backend index set {bename} --attr {attr_name} --add-mr {expected_mr}" ++ remediation_commands.append(cmd) ++ reindex_attrs.add(attr_name) + + except Exception as e: + self._log.debug(f"_lint_system_indexes - Error checking index {attr_name}: {e}") +@@ -963,13 +946,12 @@ class Backend(DSLdapObject): + return + raise ValueError("Can not delete index because it does not exist") + +- def add_index(self, attr_name, types, matching_rules=None, idlistscanlimit=None, reindex=False): ++ def add_index(self, attr_name, types, matching_rules=None, reindex=False): + """ Add an index. + + :param attr_name - name of the attribute to index + :param types - a List of index types(eq, pres, sub, approx) + :param matching_rules - a List of matching rules for the index +- :param idlistscanlimit - a List of fine grain definitions for scanning limit + :param reindex - If set to True then index the attribute after creating it. + """ + +@@ -999,15 +981,6 @@ class Backend(DSLdapObject): + # Only add if there are actually rules present in the list. + if len(mrs) > 0: + props['nsMatchingRule'] = mrs +- +- if idlistscanlimit is not None: +- scanlimits = [] +- for scanlimit in idlistscanlimit: +- scanlimits.append(scanlimit) +- # Only add if there are actually limits in the list. +- if len(scanlimits) > 0: +- props['nsIndexIDListScanLimit'] = scanlimits +- + new_index.create(properties=props, basedn="cn=index," + self._dn) + + if reindex: +@@ -1314,7 +1287,6 @@ class DatabaseConfig(DSLdapObject): + 'nsslapd-lookthroughlimit', + 'nsslapd-mode', + 'nsslapd-idlistscanlimit', +- 'nsslapd-systemidlistscanlimit', + 'nsslapd-directory', + 'nsslapd-import-cachesize', + 'nsslapd-idl-switch', +diff --git a/src/lib389/lib389/cli_conf/backend.py b/src/lib389/lib389/cli_conf/backend.py +index f8735e08e..ec9b5debe 100644 +--- a/src/lib389/lib389/cli_conf/backend.py ++++ b/src/lib389/lib389/cli_conf/backend.py +@@ -39,7 +39,6 @@ arg_to_attr = { + 'mode': 'nsslapd-mode', + 'state': 'nsslapd-state', + 'idlistscanlimit': 'nsslapd-idlistscanlimit', +- 'systemidlistscanlimit': 'nsslapd-systemidlistscanlimit', + 'directory': 'nsslapd-directory', + 'dbcachesize': 'nsslapd-dbcachesize', + 'logdirectory': 'nsslapd-db-logdirectory', +@@ -626,21 +625,6 @@ def backend_set_index(inst, basedn, log, args): + except ldap.NO_SUCH_ATTRIBUTE: + raise ValueError('Can not delete matching rule type because it does not exist') + +- if args.replace_scanlimit is not None: +- for replace_scanlimit in args.replace_scanlimit: +- index.replace('nsIndexIDListScanLimit', replace_scanlimit) +- +- if args.add_scanlimit is not None: +- for add_scanlimit in args.add_scanlimit: +- index.add('nsIndexIDListScanLimit', add_scanlimit) +- +- if args.del_scanlimit is not None: +- for del_scanlimit in args.del_scanlimit: +- try: +- index.remove('nsIndexIDListScanLimit', del_scanlimit) +- except ldap.NO_SUCH_ATTRIBUTE: +- raise ValueError('Can not delete a fine grain limit definition because it does not exist') +- + if args.reindex: + be.reindex(attrs=[args.attr]) + log.info("Index successfully updated") +@@ -962,9 +946,6 @@ def create_parser(subparsers): + edit_index_parser.add_argument('--del-type', action='append', help='Removes an index type from the index: (eq, sub, pres, or approx)') + edit_index_parser.add_argument('--add-mr', action='append', help='Adds a matching-rule to the index') + edit_index_parser.add_argument('--del-mr', action='append', help='Removes a matching-rule from the index') +- edit_index_parser.add_argument('--add-scanlimit', action='append', help='Adds a fine grain limit definiton to the index') +- edit_index_parser.add_argument('--replace-scanlimit', action='append', help='Replaces a fine grain limit definiton to the index') +- edit_index_parser.add_argument('--del-scanlimit', action='append', help='Removes a fine grain limit definiton to the index') + edit_index_parser.add_argument('--reindex', action='store_true', help='Re-indexes the database after editing the index') + edit_index_parser.add_argument('be_name', help='The backend name or suffix') + +@@ -1091,7 +1072,6 @@ def create_parser(subparsers): + 'will check when examining candidate entries in response to a search request') + set_db_config_parser.add_argument('--mode', help='Specifies the permissions used for newly created index files') + set_db_config_parser.add_argument('--idlistscanlimit', help='Specifies the number of entry IDs that are searched during a search operation') +- set_db_config_parser.add_argument('--systemidlistscanlimit', help='Specifies the number of entry IDs that are fetch from ancestorid/parentid indexes') + set_db_config_parser.add_argument('--directory', help='Specifies absolute path to database instance') + set_db_config_parser.add_argument('--dbcachesize', help='Specifies the database index cache size in bytes') + set_db_config_parser.add_argument('--logdirectory', help='Specifies the path to the directory that contains the database transaction logs') +-- +2.52.0 + diff --git a/0011-Issue-7223-Add-upgrade-function-to-remove-nsIndexIDL.patch b/0011-Issue-7223-Add-upgrade-function-to-remove-nsIndexIDL.patch new file mode 100644 index 0000000..f2bcdad --- /dev/null +++ b/0011-Issue-7223-Add-upgrade-function-to-remove-nsIndexIDL.patch @@ -0,0 +1,212 @@ +From 450015057d84bb032996e0bd76941b1ab6e1c08c Mon Sep 17 00:00:00 2001 +From: Viktor Ashirov +Date: Thu, 5 Feb 2026 12:17:06 +0100 +Subject: [PATCH] Issue 7223 - Add upgrade function to remove + nsIndexIDListScanLimit from parentid + +Description: +Add `upgrade_remove_index_scanlimit()` function that removes the +nsIndexIDListScanLimit attribute from parentid index configuration +if present. + +This attribute was incorrectly added by a previous version and can +cause issues with index configuration. The upgrade function runs +automatically on server startup and removes the attribute if found. + +Relates: https://github.com/389ds/389-ds-base/issues/7223 + +Reviewed by: @progier389, @tbordaz, @droideck (Thanks!) +--- + .../healthcheck/health_system_indexes_test.py | 52 +++++++++ + ldap/servers/slapd/upgrade.c | 105 ++++++++++++++++++ + 2 files changed, 157 insertions(+) + +diff --git a/dirsrvtests/tests/suites/healthcheck/health_system_indexes_test.py b/dirsrvtests/tests/suites/healthcheck/health_system_indexes_test.py +index 61972d60c..b0d7a99ec 100644 +--- a/dirsrvtests/tests/suites/healthcheck/health_system_indexes_test.py ++++ b/dirsrvtests/tests/suites/healthcheck/health_system_indexes_test.py +@@ -450,6 +450,58 @@ def test_multiple_missing_indexes(topology_st, log_buffering_enabled): + run_healthcheck_and_flush_log(topology_st, standalone, json=True, searched_code=JSON_OUTPUT) + + ++def test_upgrade_removes_parentid_scanlimit(topology_st): ++ """Check if upgrade function removes nsIndexIDListScanLimit from parentid index ++ ++ :id: 2808886e-c1c1-441d-b3a3-299c4ef1ab4a ++ :setup: Standalone instance ++ :steps: ++ 1. Create DS instance ++ 2. Stop the server ++ 3. Use DSEldif to add nsIndexIDListScanLimit to parentid index ++ 4. Start the server (triggers upgrade) ++ 5. Verify nsIndexIDListScanLimit is removed from parentid index ++ :expectedresults: ++ 1. Success ++ 2. Success ++ 3. Success ++ 4. Success ++ 5. nsIndexIDListScanLimit is no longer present ++ """ ++ from lib389.dseldif import DSEldif ++ ++ standalone = topology_st.standalone ++ PARENTID_DN = "cn=parentid,cn=index,cn=userroot,cn=ldbm database,cn=plugins,cn=config" ++ SCANLIMIT_VALUE = "limit=5000 type=eq flags=AND" ++ ++ log.info("Stop the server") ++ standalone.stop() ++ ++ log.info("Add nsIndexIDListScanLimit to parentid index using DSEldif") ++ dse_ldif = DSEldif(standalone) ++ dse_ldif.add(PARENTID_DN, "nsIndexIDListScanLimit", SCANLIMIT_VALUE) ++ ++ # Verify it was added ++ scanlimit = dse_ldif.get(PARENTID_DN, "nsIndexIDListScanLimit") ++ assert scanlimit is not None, "Failed to add nsIndexIDListScanLimit" ++ log.info(f"Added nsIndexIDListScanLimit: {scanlimit}") ++ ++ log.info("Start the server (triggers upgrade)") ++ standalone.start() ++ ++ log.info("Verify nsIndexIDListScanLimit was removed by upgrade") ++ # Check via LDAP - the upgrade should have removed it ++ parentid_index = Index(standalone, PARENTID_DN) ++ scanlimit_after = parentid_index.get_attr_vals_utf8("nsIndexIDListScanLimit") ++ log.info(f"nsIndexIDListScanLimit after upgrade: {scanlimit_after}") ++ ++ # The upgrade function should have removed nsIndexIDListScanLimit ++ assert not scanlimit_after, \ ++ f"nsIndexIDListScanLimit should have been removed but found: {scanlimit_after}" ++ ++ log.info("Upgrade successfully removed nsIndexIDListScanLimit from parentid index") ++ ++ + if __name__ == "__main__": + # Run isolated + # -s for DEBUG mode +diff --git a/ldap/servers/slapd/upgrade.c b/ldap/servers/slapd/upgrade.c +index 43906c1af..adfec63de 100644 +--- a/ldap/servers/slapd/upgrade.c ++++ b/ldap/servers/slapd/upgrade.c +@@ -279,6 +279,107 @@ upgrade_205_fixup_repl_dep(void) + return UPGRADE_SUCCESS; + } + ++/* ++ * Remove nsIndexIDListScanLimit from parentid index configuration. ++ * ++ * This attribute was incorrectly added by a previous version and can ++ * cause issues with index configuration. Remove it if present. ++ */ ++static upgrade_status ++upgrade_remove_index_scanlimit(void) ++{ ++ struct slapi_pblock *pb = slapi_pblock_new(); ++ Slapi_Entry **backends = NULL; ++ const char *be_base_dn = "cn=ldbm database,cn=plugins,cn=config"; ++ const char *be_filter = "(objectclass=nsBackendInstance)"; ++ const char *attrs_to_check[] = {"parentid", NULL}; ++ upgrade_status uresult = UPGRADE_SUCCESS; ++ ++ /* Search for all backend instances */ ++ slapi_search_internal_set_pb( ++ pb, be_base_dn, ++ LDAP_SCOPE_ONELEVEL, ++ be_filter, NULL, 0, NULL, NULL, ++ plugin_get_default_component_id(), 0); ++ slapi_search_internal_pb(pb); ++ slapi_pblock_get(pb, SLAPI_PLUGIN_INTOP_SEARCH_ENTRIES, &backends); ++ ++ if (backends) { ++ for (size_t be_idx = 0; backends[be_idx] != NULL; be_idx++) { ++ const char *be_dn = slapi_entry_get_dn_const(backends[be_idx]); ++ const char *be_name = slapi_entry_attr_get_ref(backends[be_idx], "cn"); ++ if (!be_dn || !be_name) { ++ continue; ++ } ++ ++ for (size_t attr_idx = 0; attrs_to_check[attr_idx] != NULL; attr_idx++) { ++ const char *attr_name = attrs_to_check[attr_idx]; ++ struct slapi_pblock *idx_pb = slapi_pblock_new(); ++ Slapi_Entry **idx_entries = NULL; ++ char *idx_dn = slapi_create_dn_string("cn=%s,cn=index,%s", ++ attr_name, be_dn); ++ char *idx_filter = "(objectclass=nsIndex)"; ++ ++ if (!idx_dn) { ++ slapi_pblock_destroy(idx_pb); ++ continue; ++ } ++ ++ slapi_search_internal_set_pb( ++ idx_pb, idx_dn, ++ LDAP_SCOPE_BASE, ++ idx_filter, NULL, 0, NULL, NULL, ++ plugin_get_default_component_id(), 0); ++ slapi_search_internal_pb(idx_pb); ++ slapi_pblock_get(idx_pb, SLAPI_PLUGIN_INTOP_SEARCH_ENTRIES, &idx_entries); ++ ++ if (idx_entries && idx_entries[0]) { ++ /* Check if nsIndexIDListScanLimit is present */ ++ if (slapi_entry_attr_get_ref(idx_entries[0], "nsIndexIDListScanLimit") != NULL) { ++ /* Remove nsIndexIDListScanLimit */ ++ Slapi_PBlock *mod_pb = slapi_pblock_new(); ++ Slapi_Mods smods; ++ int rc; ++ ++ slapi_mods_init(&smods, 1); ++ slapi_mods_add(&smods, LDAP_MOD_DELETE, "nsIndexIDListScanLimit", 0, NULL); ++ ++ slapi_modify_internal_set_pb( ++ mod_pb, idx_dn, ++ slapi_mods_get_ldapmods_byref(&smods), ++ NULL, NULL, ++ plugin_get_default_component_id(), 0); ++ slapi_modify_internal_pb(mod_pb); ++ slapi_pblock_get(mod_pb, SLAPI_PLUGIN_INTOP_RESULT, &rc); ++ ++ if (rc == LDAP_SUCCESS) { ++ slapi_log_err(SLAPI_LOG_NOTICE, "upgrade_remove_index_scanlimit", ++ "Removed 'nsIndexIDListScanLimit' from index '%s' in backend '%s'\n", ++ attr_name, be_name); ++ } else if (rc != LDAP_NO_SUCH_ATTRIBUTE) { ++ slapi_log_err(SLAPI_LOG_ERR, "upgrade_remove_index_scanlimit", ++ "Failed to remove 'nsIndexIDListScanLimit' from index '%s' in backend '%s': error %d\n", ++ attr_name, be_name, rc); ++ } ++ ++ slapi_mods_done(&smods); ++ slapi_pblock_destroy(mod_pb); ++ } ++ } ++ ++ slapi_ch_free_string(&idx_dn); ++ slapi_free_search_results_internal(idx_pb); ++ slapi_pblock_destroy(idx_pb); ++ } ++ } ++ } ++ ++ slapi_free_search_results_internal(pb); ++ slapi_pblock_destroy(pb); ++ ++ return uresult; ++} ++ + /* + * Check if parentid/ancestorid indexes are missing the integerOrderingMatch + * matching rule. +@@ -407,6 +508,10 @@ upgrade_server(void) + return UPGRADE_FAILURE; + } + ++ if (upgrade_remove_index_scanlimit() != UPGRADE_SUCCESS) { ++ return UPGRADE_FAILURE; ++ } ++ + if (upgrade_check_id_index_matching_rule() != UPGRADE_SUCCESS) { + return UPGRADE_FAILURE; + } +-- +2.52.0 + diff --git a/0012-Issue-7223-Detect-and-log-index-ordering-mismatch-du.patch b/0012-Issue-7223-Detect-and-log-index-ordering-mismatch-du.patch new file mode 100644 index 0000000..2661363 --- /dev/null +++ b/0012-Issue-7223-Detect-and-log-index-ordering-mismatch-du.patch @@ -0,0 +1,300 @@ +From 46154ef03c7543a452b17540460dc808805bd4b7 Mon Sep 17 00:00:00 2001 +From: Viktor Ashirov +Date: Thu, 5 Feb 2026 12:17:06 +0100 +Subject: [PATCH] Issue 7223 - Detect and log index ordering mismatch during + backend startup + +Description: +Add `ldbm_instance_check_index_config()` function that checks on-disk +index data and logs a message in case of a mismatch with DSE config entry. + +Relates: https://github.com/389ds/389-ds-base/issues/7223 + +Reviewed by: @progier389, @tbordaz, @droideck (Thanks!) +--- + ldap/servers/slapd/back-ldbm/instance.c | 262 ++++++++++++++++++++++++ + 1 file changed, 262 insertions(+) + +diff --git a/ldap/servers/slapd/back-ldbm/instance.c b/ldap/servers/slapd/back-ldbm/instance.c +index f9a546661..3fcdb5554 100644 +--- a/ldap/servers/slapd/back-ldbm/instance.c ++++ b/ldap/servers/slapd/back-ldbm/instance.c +@@ -248,6 +248,266 @@ ldbm_instance_create_default_indexes(backend *be) + } + + ++/* ++ * Check if an index has integerOrderingMatch configured in DSE. ++ * ++ * This function performs an internal LDAP search to check if the index ++ * configuration entry has nsMatchingRule: integerOrderingMatch. ++ * ++ * Parameters: ++ * inst_name - backend instance name (e.g., "userRoot") ++ * index_name - name of the index to check (e.g., "parentid", "ancestorid") ++ * ++ * Returns: ++ * PR_TRUE if integerOrderingMatch is configured ++ * PR_FALSE if not configured or index entry doesn't exist ++ */ ++static PRBool ++ldbm_instance_index_has_int_order_in_dse(const char *inst_name, const char *index_name) ++{ ++ Slapi_PBlock *pb = NULL; ++ Slapi_Entry **entries = NULL; ++ char *idx_dn = NULL; ++ PRBool has_int_order = PR_FALSE; ++ ++ idx_dn = slapi_create_dn_string("cn=%s,cn=index,cn=%s,cn=ldbm database,cn=plugins,cn=config", ++ index_name, inst_name); ++ if (idx_dn == NULL) { ++ return PR_FALSE; ++ } ++ ++ pb = slapi_pblock_new(); ++ slapi_search_internal_set_pb(pb, idx_dn, LDAP_SCOPE_BASE, ++ "(objectclass=nsIndex)", NULL, 0, NULL, NULL, ++ plugin_get_default_component_id(), 0); ++ slapi_search_internal_pb(pb); ++ slapi_pblock_get(pb, SLAPI_PLUGIN_INTOP_SEARCH_ENTRIES, &entries); ++ ++ if (entries && entries[0]) { ++ Slapi_Attr *mr_attr = NULL; ++ if (slapi_entry_attr_find(entries[0], "nsMatchingRule", &mr_attr) == 0) { ++ Slapi_Value *sval = NULL; ++ int idx; ++ for (idx = slapi_attr_first_value(mr_attr, &sval); ++ idx != -1; ++ idx = slapi_attr_next_value(mr_attr, idx, &sval)) { ++ const struct berval *bval = slapi_value_get_berval(sval); ++ if (bval && bval->bv_val && ++ strcasecmp(bval->bv_val, "integerOrderingMatch") == 0) { ++ has_int_order = PR_TRUE; ++ break; ++ } ++ } ++ } ++ } ++ ++ slapi_ch_free_string(&idx_dn); ++ slapi_free_search_results_internal(pb); ++ slapi_pblock_destroy(pb); ++ ++ return has_int_order; ++} ++ ++/* ++ * Check a system index for ordering mismatch between config and on-disk data. ++ * ++ * This function compares what's configured in DSE (nsMatchingRule) with ++ * what's actually on disk. A mismatch can occur in two scenarios: ++ * 1. Ordering rule is configured but disk has lexicographic order ++ * (rule was added after index was created) ++ * 2. No ordering rule configured but disk has integer order ++ * (rule was removed after index was created with it) ++ * ++ * This function reads the first keys from the specified index and checks ++ * if they are stored in lexicographic order (string: "1" < "10" < "2") or ++ * integer order (numeric: "1" < "2" < "10"). ++ * ++ * Parameters: ++ * be - backend ++ * index_name - name of the index to check (e.g., "parentid", "ancestorid") ++ * ++ */ ++static void ++ldbm_instance_check_index_config(backend *be, const char *index_name) ++{ ++ ldbm_instance *inst = (ldbm_instance *)be->be_instance_info; ++ struct attrinfo *ai = NULL; ++ dbi_db_t *db = NULL; ++ dbi_cursor_t dbc = {0}; ++ dbi_val_t key = {0}; ++ dbi_val_t data = {0}; ++ int ret = 0; ++ PRBool config_has_int_order = PR_FALSE; ++ PRBool disk_has_int_order = PR_TRUE; /* Assume integer order until proven otherwise */ ++ ID prev_id = 0; ++ int key_count = 0; ++ PRBool first_key = PR_TRUE; ++ PRBool found_ordering_evidence = PR_FALSE; ++ ++ slapi_log_err(SLAPI_LOG_DEBUG, "ldbm_instance_check_index_config", ++ "Backend '%s': checking %s index ordering...\n", ++ inst->inst_name, index_name); ++ ++ /* Check if integerOrderingMatch is configured in DSE */ ++ config_has_int_order = ldbm_instance_index_has_int_order_in_dse(inst->inst_name, index_name); ++ ++ /* Get attrinfo for the index */ ++ ainfo_get(be, (char *)index_name, &ai); ++ if (ai == NULL || strcmp(ai->ai_type, index_name) != 0) { ++ /* No index config found */ ++ slapi_log_err(SLAPI_LOG_DEBUG, "ldbm_instance_check_index_config", ++ "Backend '%s': no %s attrinfo found, skipping check\n", ++ inst->inst_name, index_name); ++ return; ++ } ++ ++ /* Open the index file */ ++ ret = dblayer_get_index_file(be, ai, &db, 0); ++ if (ret != 0 || db == NULL) { ++ /* Index file doesn't exist or can't be opened - this is fine for new instances */ ++ slapi_log_err(SLAPI_LOG_DEBUG, "ldbm_instance_check_index_config", ++ "Backend '%s': could not open %s index file (ret=%d), skipping order check\n", ++ inst->inst_name, index_name, ret); ++ return; ++ } ++ ++ /* Create a cursor to read keys */ ++ ret = dblayer_new_cursor(be, db, NULL, &dbc); ++ if (ret != 0) { ++ slapi_log_err(SLAPI_LOG_ERR, "ldbm_instance_check_index_config", ++ "Backend '%s': could not create cursor on %s index (ret=%d)\n", ++ inst->inst_name, index_name, ret); ++ dblayer_release_index_file(be, ai, db); ++ return; ++ } ++ ++ dblayer_value_init(be, &key); ++ dblayer_value_init(be, &data); ++ ++ /* ++ * Read up to 100 unique keys and check their ordering. ++ * With lexicographic ordering: "1" < "10" < "100" < "2" < "20" < "3" ++ * With integer ordering: "1" < "2" < "3" < "10" < "20" < "100" ++ * ++ * If we find a case where prev_id > current_id (numerically), but the ++ * keys are still in order (lexicographically), then the index uses ++ * lexicographic ordering. ++ */ ++ while (key_count < 100) { ++ ID current_id; ++ ++ ret = dblayer_cursor_op(&dbc, first_key ? DBI_OP_MOVE_TO_FIRST : DBI_OP_NEXT_KEY, &key, &data); ++ first_key = PR_FALSE; /* Always advance cursor on next iteration */ ++ if (ret != 0) { ++ break; /* No more keys or error */ ++ } ++ ++ /* Skip non-equality keys */ ++ if (key.size < 2 || *(char *)key.data != EQ_PREFIX) { ++ continue; ++ } ++ ++ /* Parse the ID from the key (format: "=") */ ++ current_id = (ID)strtoul((char *)key.data + 1, NULL, 10); ++ if (current_id == 0) { ++ continue; /* Invalid ID, skip */ ++ } ++ ++ key_count++; ++ ++ if (prev_id != 0) { ++ /* ++ * Check ordering: if prev_id > current_id numerically, ++ * but we got this key after prev in DB order, then ++ * the index is using lexicographic ordering. ++ * ++ * Example: if we see "10" followed by "2", that's lexicographic ++ * because "10" < "2" as strings, but 10 > 2 as integers. ++ */ ++ if (prev_id > current_id) { ++ /* Found evidence of lexicographic ordering */ ++ disk_has_int_order = PR_FALSE; ++ found_ordering_evidence = PR_TRUE; ++ break; ++ } else if (prev_id < current_id) { ++ /* ++ * This is consistent with integer ordering, but we need ++ * to find a case that proves lexicographic ordering. ++ * For example, seeing "1" followed by "2" is ambiguous, ++ * but seeing "1" followed by "10" (not "2") proves lexicographic. ++ * ++ * A definitive test: if we see an ID followed by a smaller ++ * ID, that's lexicographic. If all IDs are strictly increasing, ++ * it could be either (or the index only has sequential IDs). ++ */ ++ found_ordering_evidence = PR_TRUE; ++ } ++ } ++ prev_id = current_id; ++ } ++ ++ /* Close the cursor and free values */ ++ dblayer_cursor_op(&dbc, DBI_OP_CLOSE, NULL, NULL); ++ dblayer_value_free(be, &key); ++ dblayer_value_free(be, &data); ++ ++ /* Release the index file */ ++ dblayer_release_index_file(be, ai, db); ++ ++ /* ++ * Report findings and check for config/disk mismatch. ++ * Log an error if there's a discrepancy between what's configured ++ * in DSE and what's actually on disk. ++ */ ++ if (!found_ordering_evidence) { ++ slapi_log_err(SLAPI_LOG_DEBUG, "ldbm_instance_check_index_config", ++ "Backend '%s': %s index ordering check - " ++ "could not determine on-disk ordering (index may be empty or have sequential IDs only). " ++ "Config has integerOrderingMatch: %s\n", ++ inst->inst_name, index_name, config_has_int_order ? "yes" : "no"); ++ } else if (config_has_int_order && !disk_has_int_order) { ++ /* Config expects integer ordering, but disk has lexicographic - MISMATCH */ ++ slapi_log_err(SLAPI_LOG_ERR, "ldbm_instance_check_index_config", ++ "Backend '%s': MISMATCH - %s index has integerOrderingMatch configured, " ++ "but on-disk data uses lexicographic ordering. " ++ "This will cause searches to return incorrect or incomplete results. " ++ "Please reindex the %s attribute: " ++ "dsconf backend index reindex --attr %s %s\n", ++ inst->inst_name, index_name, index_name, index_name, inst->inst_name); ++ } else if (!config_has_int_order && disk_has_int_order) { ++ /* Config expects lexicographic ordering, but disk has integer - MISMATCH */ ++ slapi_log_err(SLAPI_LOG_ERR, "ldbm_instance_check_index_config", ++ "Backend '%s': MISMATCH - %s index does not have integerOrderingMatch configured, " ++ "but on-disk data uses integer ordering. " ++ "This will cause searches to return incorrect or incomplete results. " ++ "Please reindex the %s attribute: " ++ "dsconf backend index reindex --attr %s %s\n", ++ inst->inst_name, index_name, index_name, index_name, inst->inst_name); ++ } else { ++ /* Config and disk ordering match - no action needed */ ++ slapi_log_err(SLAPI_LOG_DEBUG, "ldbm_instance_check_index_config", ++ "Backend '%s': %s index ordering check passed - " ++ "config has integerOrderingMatch: %s, on-disk data matches.\n", ++ inst->inst_name, index_name, config_has_int_order ? "yes" : "no"); ++ } ++} ++ ++/* ++ * Check system indexes for ordering mismatches. ++ * If a mismatch is detected, log an error advising the administrator ++ * to reindex the affected attribute. ++ * ++ * Note: We only check parentid here. The ancestorid index is a special ++ * system index that has no DSE config entry - its ordering is hardcoded ++ * in ldbm_instance_init_config_entry() and cannot be changed by users. ++ */ ++static void ++ldbm_instance_check_indexes(backend *be) ++{ ++ /* Check parentid index */ ++ ldbm_instance_check_index_config(be, LDBM_PARENTID_STR); ++} ++ + /* Starts a backend instance */ + int + ldbm_instance_start(backend *be) +@@ -319,6 +579,8 @@ ldbm_instance_startall(struct ldbminfo *li) + ldbm_instance_register_modify_callback(inst); + vlv_init(inst); + slapi_mtn_be_started(inst->inst_be); ++ /* Check index configuration for potential issues */ ++ ldbm_instance_check_indexes(inst->inst_be); + } + if (slapi_exist_referral(inst->inst_be)) { + slapi_be_set_flag(inst->inst_be, SLAPI_BE_FLAG_CONTAINS_REFERRAL); +-- +2.52.0 + diff --git a/0013-Issue-7223-Add-dsctl-index-check-command-for-offline.patch b/0013-Issue-7223-Add-dsctl-index-check-command-for-offline.patch new file mode 100644 index 0000000..b42800c --- /dev/null +++ b/0013-Issue-7223-Add-dsctl-index-check-command-for-offline.patch @@ -0,0 +1,1233 @@ +From c741eceaf0ba751d95153f032826c20249838de5 Mon Sep 17 00:00:00 2001 +From: Viktor Ashirov +Date: Thu, 5 Feb 2026 12:17:06 +0100 +Subject: [PATCH] Issue 7223 - Add dsctl index-check command for offline index + repair + +Description: +Add `dsctl index-check [backend] [--fix]` command for offline +detection and repair of index ordering mismatches. This is needed after +upgrade from versions that didn't use integerOrderingMatch for +parentid/ancestorid system indexes. + +It's automatically executed as part of RPM %post scriptlet during +upgrade. + +Relates: https://github.com/389ds/389-ds-base/issues/7223 + +Reviewed by: @progier389, @tbordaz, @droideck (Thanks!) +--- + .../healthcheck/health_system_indexes_test.py | 593 ++++++++++++++++++ + rpm/389-ds-base.spec.in | 51 +- + src/lib389/lib389/cli_ctl/dbtasks.py | 402 ++++++++++++ + src/lib389/lib389/dseldif.py | 51 +- + 4 files changed, 1068 insertions(+), 29 deletions(-) + +diff --git a/dirsrvtests/tests/suites/healthcheck/health_system_indexes_test.py b/dirsrvtests/tests/suites/healthcheck/health_system_indexes_test.py +index b0d7a99ec..55b9d2f6d 100644 +--- a/dirsrvtests/tests/suites/healthcheck/health_system_indexes_test.py ++++ b/dirsrvtests/tests/suites/healthcheck/health_system_indexes_test.py +@@ -502,6 +502,599 @@ def test_upgrade_removes_parentid_scanlimit(topology_st): + log.info("Upgrade successfully removed nsIndexIDListScanLimit from parentid index") + + ++def test_index_check_basic(topology_st): ++ """Check if dsctl index-check works correctly ++ ++ :id: 8a4e5c2d-1f3b-4a7c-9e8d-2b6f0c4a5d3e ++ :setup: Standalone instance ++ :steps: ++ 1. Create DS instance ++ 2. Run dsctl index-check while server is running (should fail) ++ 3. Stop the server ++ 4. Run dsctl index-check (should pass) ++ 5. Start the server ++ :expectedresults: ++ 1. Success ++ 2. index-check returns False and logs error ++ 3. Success ++ 4. index-check returns True (no mismatches) ++ 5. Success ++ """ ++ from lib389.cli_ctl.dbtasks import dbtasks_index_check ++ ++ standalone = topology_st.standalone ++ ++ log.info("Run index-check while server is running") ++ args = FakeArgs() ++ args.backend = None ++ args.fix = False ++ ++ # Server should be running, index-check should fail ++ assert standalone.status() ++ result = dbtasks_index_check(standalone, topology_st.logcap.log, args) ++ assert result is False ++ assert topology_st.logcap.contains("index-check requires the instance to be stopped") ++ topology_st.logcap.flush() ++ ++ log.info("Stop the server") ++ standalone.stop() ++ ++ log.info("Run index-check with server stopped") ++ result = dbtasks_index_check(standalone, topology_st.logcap.log, args) ++ assert result is True ++ assert topology_st.logcap.contains("All checks passed") ++ topology_st.logcap.flush() ++ ++ log.info("Start the server") ++ standalone.start() ++ ++ ++def test_index_check_specific_backend(topology_st): ++ """Check if dsctl index-check works with a specific backend ++ ++ :id: 407d8fcc-62e0-43dd-90fa-70e7090a5cfd ++ :setup: Standalone instance ++ :steps: ++ 1. Create DS instance ++ 2. Stop the server ++ 3. Run dsctl index-check with specific backend (userRoot) ++ 4. Run dsctl index-check with non-existent backend ++ 5. Start the server ++ :expectedresults: ++ 1. Success ++ 2. Success ++ 3. index-check returns True for userRoot ++ 4. index-check returns False for non-existent backend ++ 5. Success ++ """ ++ from lib389.cli_ctl.dbtasks import dbtasks_index_check ++ ++ standalone = topology_st.standalone ++ ++ log.info("Stop the server") ++ standalone.stop() ++ ++ log.info("Run index-check for userRoot backend") ++ args = FakeArgs() ++ args.backend = "userRoot" ++ args.fix = False ++ ++ result = dbtasks_index_check(standalone, topology_st.logcap.log, args) ++ assert result is True ++ # Check for backend name in any case ++ assert topology_st.logcap.contains("Checking backend:") ++ topology_st.logcap.flush() ++ ++ log.info("Run index-check for non-existent backend") ++ args.backend = "nonExistentBackend" ++ result = dbtasks_index_check(standalone, topology_st.logcap.log, args) ++ assert result is False ++ assert topology_st.logcap.contains("not found") ++ topology_st.logcap.flush() ++ ++ log.info("Start the server") ++ standalone.start() ++ ++ ++def test_index_check_mismatch_detection(topology_st): ++ """Check if dsctl index-check detects ordering mismatch ++ ++ :id: 50d14520-b0bf-4243-9fe6-b097928d4351 ++ :setup: Standalone instance ++ :steps: ++ 1. Create DS instance ++ 2. Stop the server ++ 3. Run dsctl index-check (without --fix) ++ 4. Verify output format ++ 5. Start the server ++ :expectedresults: ++ 1. Success ++ 2. Success ++ 3. index-check returns True (no mismatch on fresh instance) ++ 4. Log contains expected format ++ 5. Success ++ """ ++ from lib389.cli_ctl.dbtasks import dbtasks_index_check ++ ++ standalone = topology_st.standalone ++ ++ log.info("Stop the server") ++ standalone.stop() ++ ++ log.info("Run index-check to verify detection logic") ++ args = FakeArgs() ++ args.backend = "userRoot" ++ args.fix = False ++ ++ # On a fresh instance, there should be no mismatch ++ result = dbtasks_index_check(standalone, topology_st.logcap.log, args) ++ # Fresh instance should have matching config and disk ordering ++ assert result is True ++ # Check that the backend was checked (may skip indexes if ordering can't be determined) ++ assert topology_st.logcap.contains("Checking backend:") ++ topology_st.logcap.flush() ++ ++ log.info("Start the server") ++ standalone.start() ++ ++ ++def test_index_check_with_fix(topology_st): ++ """Check if dsctl index-check --fix triggers reindexing ++ ++ :id: 38ae36e4-c861-4771-ae7d-354370376a2f ++ :setup: Standalone instance ++ :steps: ++ 1. Create DS instance ++ 2. Stop the server ++ 3. Run dsctl index-check --fix (should pass since no mismatch) ++ 4. Verify output indicates check passed ++ 5. Start the server ++ :expectedresults: ++ 1. Success ++ 2. Success ++ 3. index-check returns True ++ 4. Log contains "All checks passed" ++ 5. Success ++ """ ++ from lib389.cli_ctl.dbtasks import dbtasks_index_check ++ ++ standalone = topology_st.standalone ++ ++ log.info("Stop the server") ++ standalone.stop() ++ ++ log.info("Run index-check with --fix option") ++ args = FakeArgs() ++ args.backend = None ++ args.fix = True ++ ++ result = dbtasks_index_check(standalone, topology_st.logcap.log, args) ++ # On a fresh instance, there should be no mismatch, so no reindexing needed ++ assert result is True ++ assert topology_st.logcap.contains("All checks passed") ++ topology_st.logcap.flush() ++ ++ log.info("Start the server") ++ standalone.start() ++ ++ ++def test_index_check_fixes_scanlimit(topology_st): ++ """Check if dsctl index-check --fix removes nsIndexIDListScanLimit ++ ++ :id: 4a9b2c7d-8e1f-4b3a-9c5d-6e7f8a0b1c2d ++ :setup: Standalone instance ++ :steps: ++ 1. Create DS instance ++ 2. Stop the server ++ 3. Add nsIndexIDListScanLimit to parentid index using DSEldif ++ 4. Run dsctl index-check (should detect issue) ++ 5. Run dsctl index-check --fix ++ 6. Verify nsIndexIDListScanLimit was removed ++ 7. Start the server ++ :expectedresults: ++ 1. Success ++ 2. Success ++ 3. Success ++ 4. index-check returns False and detects scanlimit ++ 5. index-check returns True after fix ++ 6. nsIndexIDListScanLimit no longer present ++ 7. Success ++ """ ++ from lib389.cli_ctl.dbtasks import dbtasks_index_check ++ from lib389.dseldif import DSEldif ++ ++ standalone = topology_st.standalone ++ ++ log.info("Stop the server") ++ standalone.stop() ++ ++ log.info("Add nsIndexIDListScanLimit to parentid index using DSEldif") ++ dse_ldif = DSEldif(standalone) ++ parentid_dn = "cn=parentid,cn=index,cn=userRoot,cn=ldbm database,cn=plugins,cn=config" ++ dse_ldif.add(parentid_dn, "nsIndexIDListScanLimit", "4000") ++ ++ # Verify it was added ++ scanlimit = dse_ldif.get(parentid_dn, "nsIndexIDListScanLimit", single=True) ++ assert scanlimit == "4000", f"Failed to add nsIndexIDListScanLimit, got: {scanlimit}" ++ log.info("Added nsIndexIDListScanLimit to parentid index") ++ ++ log.info("Run index-check without --fix (should detect issue)") ++ args = FakeArgs() ++ args.backend = "userRoot" ++ args.fix = False ++ ++ result = dbtasks_index_check(standalone, topology_st.logcap.log, args) ++ assert result is False, "index-check should detect scanlimit issue" ++ assert topology_st.logcap.contains("nsIndexIDListScanLimit") ++ topology_st.logcap.flush() ++ ++ log.info("Run index-check with --fix") ++ args.fix = True ++ result = dbtasks_index_check(standalone, topology_st.logcap.log, args) ++ assert result is True, "index-check --fix should succeed" ++ assert topology_st.logcap.contains("Removed nsIndexIDListScanLimit") ++ topology_st.logcap.flush() ++ ++ log.info("Verify nsIndexIDListScanLimit was removed") ++ dse_ldif = DSEldif(standalone) # Reload to get fresh data ++ scanlimit = dse_ldif.get(parentid_dn, "nsIndexIDListScanLimit", single=True) ++ assert scanlimit is None, f"nsIndexIDListScanLimit should be removed, but got: {scanlimit}" ++ log.info("nsIndexIDListScanLimit successfully removed") ++ ++ log.info("Start the server") ++ standalone.start() ++ ++ ++def test_index_check_fixes_ancestorid_config(topology_st): ++ """Check if dsctl index-check --fix removes ancestorid config entries ++ ++ :id: 5b0c3d8e-9f2a-4c4b-0d6e-7f8a9b1c2d3e ++ :setup: Standalone instance ++ :steps: ++ 1. Create DS instance ++ 2. Stop the server ++ 3. Add ancestorid index config entry using DSEldif ++ 4. Run dsctl index-check (should detect issue) ++ 5. Run dsctl index-check --fix ++ 6. Verify ancestorid config entry was removed ++ 7. Start the server ++ :expectedresults: ++ 1. Success ++ 2. Success ++ 3. Success ++ 4. index-check returns False and detects ancestorid config ++ 5. index-check returns True after fix ++ 6. ancestorid config entry no longer present ++ 7. Success ++ """ ++ from lib389.cli_ctl.dbtasks import dbtasks_index_check ++ from lib389.dseldif import DSEldif ++ ++ standalone = topology_st.standalone ++ ++ log.info("Stop the server") ++ standalone.stop() ++ ++ log.info("Add ancestorid index config entry using DSEldif") ++ dse_ldif = DSEldif(standalone) ++ ancestorid_entry = [ ++ "dn: cn=ancestorid,cn=index,cn=userRoot,cn=ldbm database,cn=plugins,cn=config\n", ++ "objectClass: top\n", ++ "objectClass: nsIndex\n", ++ "cn: ancestorid\n", ++ "nsSystemIndex: true\n", ++ "nsIndexType: eq\n", ++ ] ++ dse_ldif.add_entry(ancestorid_entry) ++ ++ # Verify it was added ++ ancestorid_dn = "cn=ancestorid,cn=index,cn=userRoot,cn=ldbm database,cn=plugins,cn=config" ++ dse_ldif = DSEldif(standalone) # Reload ++ cn_value = dse_ldif.get(ancestorid_dn, "cn", single=True) ++ assert cn_value is not None, "Failed to add ancestorid index config entry" ++ log.info(f"Added ancestorid index entry with cn: {cn_value}") ++ ++ log.info("Run index-check without --fix (should detect issue)") ++ args = FakeArgs() ++ args.backend = "userRoot" ++ args.fix = False ++ ++ result = dbtasks_index_check(standalone, topology_st.logcap.log, args) ++ assert result is False, "index-check should detect ancestorid config issue" ++ assert topology_st.logcap.contains("ancestorid") and topology_st.logcap.contains("config entry exists") ++ topology_st.logcap.flush() ++ ++ log.info("Run index-check with --fix") ++ args.fix = True ++ result = dbtasks_index_check(standalone, topology_st.logcap.log, args) ++ assert result is True, "index-check --fix should succeed" ++ assert topology_st.logcap.contains("Removed ancestorid config entry") ++ topology_st.logcap.flush() ++ ++ log.info("Verify ancestorid config entry was removed") ++ dse_ldif = DSEldif(standalone) # Reload to get fresh data ++ cn_value = dse_ldif.get(ancestorid_dn, "cn", single=True) ++ assert cn_value is None, f"ancestorid config entry should be removed, but got: {cn_value}" ++ log.info("ancestorid config entry successfully removed") ++ ++ log.info("Start the server") ++ standalone.start() ++ ++ ++def test_index_check_fixes_missing_matching_rule(topology_st): ++ """Check if dsctl index-check --fix adds missing integerOrderingMatch ++ ++ :id: 6c1d4e9f-0a3b-4d5c-1e7f-8a9b0c2d3e4f ++ :setup: Standalone instance ++ :steps: ++ 1. Create DS instance ++ 2. Stop the server ++ 3. Remove integerOrderingMatch from parentid index using DSEldif ++ 4. Run dsctl index-check (should detect issue) ++ 5. Run dsctl index-check --fix ++ 6. Verify integerOrderingMatch was added back ++ 7. Start the server ++ :expectedresults: ++ 1. Success ++ 2. Success ++ 3. Success ++ 4. index-check returns False and detects missing matching rule ++ 5. index-check returns True after fix ++ 6. integerOrderingMatch is present ++ 7. Success ++ """ ++ from lib389.cli_ctl.dbtasks import dbtasks_index_check ++ from lib389.dseldif import DSEldif ++ ++ standalone = topology_st.standalone ++ ++ log.info("Stop the server") ++ standalone.stop() ++ ++ log.info("Remove integerOrderingMatch from parentid index using DSEldif") ++ dse_ldif = DSEldif(standalone) ++ parentid_dn = "cn=parentid,cn=index,cn=userRoot,cn=ldbm database,cn=plugins,cn=config" ++ ++ # Check current matching rules ++ matching_rules = dse_ldif.get(parentid_dn, "nsMatchingRule") ++ log.info(f"Current matching rules: {matching_rules}") ++ ++ # Remove integerOrderingMatch if present ++ if matching_rules: ++ for mr in matching_rules: ++ if "integerorderingmatch" in mr.lower(): ++ dse_ldif.delete(parentid_dn, "nsMatchingRule", mr) ++ log.info(f"Removed matching rule: {mr}") ++ ++ # Verify it was removed ++ dse_ldif = DSEldif(standalone) # Reload ++ matching_rules = dse_ldif.get(parentid_dn, "nsMatchingRule") ++ if matching_rules: ++ for mr in matching_rules: ++ assert "integerorderingmatch" not in mr.lower(), \ ++ f"integerOrderingMatch should be removed, but found: {mr}" ++ log.info("integerOrderingMatch removed from parentid index") ++ ++ log.info("Run index-check without --fix (should detect issue)") ++ args = FakeArgs() ++ args.backend = "userRoot" ++ args.fix = False ++ ++ result = dbtasks_index_check(standalone, topology_st.logcap.log, args) ++ assert result is False, "index-check should detect missing matching rule" ++ assert topology_st.logcap.contains("missing integerOrderingMatch") ++ topology_st.logcap.flush() ++ ++ log.info("Run index-check with --fix") ++ args.fix = True ++ result = dbtasks_index_check(standalone, topology_st.logcap.log, args) ++ assert result is True, "index-check --fix should succeed" ++ assert topology_st.logcap.contains("integerOrderingMatch") ++ topology_st.logcap.flush() ++ ++ log.info("Verify integerOrderingMatch was added back") ++ dse_ldif = DSEldif(standalone) # Reload to get fresh data ++ matching_rules = dse_ldif.get(parentid_dn, "nsMatchingRule") ++ assert matching_rules is not None, "nsMatchingRule should be present" ++ found_int_order = False ++ for mr in matching_rules: ++ if "integerorderingmatch" in mr.lower(): ++ found_int_order = True ++ break ++ assert found_int_order, f"integerOrderingMatch should be present, got: {matching_rules}" ++ log.info("integerOrderingMatch successfully added back") ++ ++ log.info("Start the server") ++ standalone.start() ++ ++ ++def test_index_check_fixes_default_ancestorid(topology_st): ++ """Check if dsctl index-check --fix removes ancestorid from default indexes ++ ++ :id: 7d2e5f0a-1b4c-4e6d-2f8a-9b0c1d3e4f5a ++ :setup: Standalone instance ++ :steps: ++ 1. Create DS instance ++ 2. Stop the server ++ 3. Add ancestorid to cn=default indexes using DSEldif ++ 4. Run dsctl index-check (should detect issue) ++ 5. Run dsctl index-check --fix ++ 6. Verify ancestorid was removed from default indexes ++ 7. Start the server ++ :expectedresults: ++ 1. Success ++ 2. Success ++ 3. Success ++ 4. index-check returns False and detects ancestorid in default indexes ++ 5. index-check returns True after fix ++ 6. ancestorid no longer in default indexes ++ 7. Success ++ """ ++ from lib389.cli_ctl.dbtasks import dbtasks_index_check ++ from lib389.dseldif import DSEldif ++ ++ standalone = topology_st.standalone ++ ++ log.info("Stop the server") ++ standalone.stop() ++ ++ log.info("Add ancestorid to cn=default indexes using DSEldif") ++ dse_ldif = DSEldif(standalone) ++ ancestorid_default_entry = [ ++ "dn: cn=ancestorid,cn=default indexes,cn=config,cn=ldbm database,cn=plugins,cn=config\n", ++ "objectClass: top\n", ++ "objectClass: nsIndex\n", ++ "cn: ancestorid\n", ++ "nsSystemIndex: true\n", ++ "nsIndexType: eq\n", ++ ] ++ dse_ldif.add_entry(ancestorid_default_entry) ++ ++ # Verify it was added ++ ancestorid_default_dn = "cn=ancestorid,cn=default indexes,cn=config,cn=ldbm database,cn=plugins,cn=config" ++ dse_ldif = DSEldif(standalone) # Reload ++ cn_value = dse_ldif.get(ancestorid_default_dn, "cn", single=True) ++ assert cn_value is not None, "Failed to add ancestorid to default indexes" ++ log.info(f"Added ancestorid to default indexes with cn: {cn_value}") ++ ++ log.info("Run index-check without --fix (should detect issue)") ++ args = FakeArgs() ++ args.backend = None # Check all backends including default indexes ++ args.fix = False ++ ++ result = dbtasks_index_check(standalone, topology_st.logcap.log, args) ++ assert result is False, "index-check should detect ancestorid in default indexes" ++ assert topology_st.logcap.contains("ancestorid found in cn=default indexes") ++ topology_st.logcap.flush() ++ ++ log.info("Run index-check with --fix") ++ args.fix = True ++ result = dbtasks_index_check(standalone, topology_st.logcap.log, args) ++ assert result is True, "index-check --fix should succeed" ++ assert topology_st.logcap.contains("Removed ancestorid from default indexes") ++ topology_st.logcap.flush() ++ ++ log.info("Verify ancestorid was removed from default indexes") ++ dse_ldif = DSEldif(standalone) # Reload to get fresh data ++ cn_value = dse_ldif.get(ancestorid_default_dn, "cn", single=True) ++ assert cn_value is None, f"ancestorid should be removed from default indexes, but got: {cn_value}" ++ log.info("ancestorid successfully removed from default indexes") ++ ++ log.info("Start the server") ++ standalone.start() ++ ++ ++def test_index_check_fixes_multiple_issues(topology_st): ++ """Check if dsctl index-check --fix handles multiple issues at once ++ ++ :id: 8e3f6a1b-2c5d-4f7e-3a9b-0c1d2e4f5a6b ++ :setup: Standalone instance ++ :steps: ++ 1. Create DS instance ++ 2. Stop the server ++ 3. Add multiple issues: scanlimit, ancestorid config, missing matching rule ++ 4. Run dsctl index-check (should detect all issues) ++ 5. Run dsctl index-check --fix ++ 6. Verify all issues were fixed ++ 7. Run dsctl index-check again (should pass) ++ 8. Start the server ++ :expectedresults: ++ 1. Success ++ 2. Success ++ 3. Success ++ 4. index-check returns False and detects all issues ++ 5. index-check returns True after fix ++ 6. All issues resolved ++ 7. index-check returns True (no issues) ++ 8. Success ++ """ ++ from lib389.cli_ctl.dbtasks import dbtasks_index_check ++ from lib389.dseldif import DSEldif ++ ++ standalone = topology_st.standalone ++ ++ log.info("Stop the server") ++ standalone.stop() ++ ++ dse_ldif = DSEldif(standalone) ++ parentid_dn = "cn=parentid,cn=index,cn=userRoot,cn=ldbm database,cn=plugins,cn=config" ++ ancestorid_dn = "cn=ancestorid,cn=index,cn=userRoot,cn=ldbm database,cn=plugins,cn=config" ++ ++ log.info("Add issue 1: nsIndexIDListScanLimit to parentid") ++ dse_ldif.add(parentid_dn, "nsIndexIDListScanLimit", "4000") ++ ++ log.info("Add issue 2: ancestorid index config entry") ++ ancestorid_entry = [ ++ f"dn: {ancestorid_dn}\n", ++ "objectClass: top\n", ++ "objectClass: nsIndex\n", ++ "cn: ancestorid\n", ++ "nsSystemIndex: true\n", ++ "nsIndexType: eq\n", ++ ] ++ dse_ldif.add_entry(ancestorid_entry) ++ ++ log.info("Add issue 3: Remove integerOrderingMatch from parentid") ++ dse_ldif = DSEldif(standalone) # Reload ++ matching_rules = dse_ldif.get(parentid_dn, "nsMatchingRule") ++ if matching_rules: ++ for mr in matching_rules: ++ if "integerorderingmatch" in mr.lower(): ++ dse_ldif.delete(parentid_dn, "nsMatchingRule", mr) ++ ++ log.info("Run index-check without --fix (should detect all issues)") ++ args = FakeArgs() ++ args.backend = "userRoot" ++ args.fix = False ++ ++ result = dbtasks_index_check(standalone, topology_st.logcap.log, args) ++ assert result is False, "index-check should detect multiple issues" ++ # Check that multiple issues were detected ++ assert topology_st.logcap.contains("nsIndexIDListScanLimit") ++ assert topology_st.logcap.contains("ancestorid") ++ topology_st.logcap.flush() ++ ++ log.info("Run index-check with --fix") ++ args.fix = True ++ result = dbtasks_index_check(standalone, topology_st.logcap.log, args) ++ assert result is True, "index-check --fix should succeed" ++ assert topology_st.logcap.contains("All issues fixed") ++ topology_st.logcap.flush() ++ ++ log.info("Verify all issues were fixed") ++ dse_ldif = DSEldif(standalone) # Reload ++ ++ # Check scanlimit removed ++ scanlimit = dse_ldif.get(parentid_dn, "nsIndexIDListScanLimit", single=True) ++ assert scanlimit is None, f"nsIndexIDListScanLimit should be removed, got: {scanlimit}" ++ ++ # Check ancestorid config removed ++ cn_value = dse_ldif.get(ancestorid_dn, "cn", single=True) ++ assert cn_value is None, f"ancestorid config should be removed, got: {cn_value}" ++ ++ # Check matching rule added back ++ matching_rules = dse_ldif.get(parentid_dn, "nsMatchingRule") ++ found_int_order = False ++ if matching_rules: ++ for mr in matching_rules: ++ if "integerorderingmatch" in mr.lower(): ++ found_int_order = True ++ break ++ assert found_int_order, f"integerOrderingMatch should be present, got: {matching_rules}" ++ ++ log.info("All issues verified as fixed") ++ ++ log.info("Run index-check again to confirm all clear") ++ args.fix = False ++ result = dbtasks_index_check(standalone, topology_st.logcap.log, args) ++ assert result is True, "index-check should pass after fix" ++ assert topology_st.logcap.contains("All checks passed") ++ topology_st.logcap.flush() ++ ++ log.info("Start the server") ++ standalone.start() ++ ++ + if __name__ == "__main__": + # Run isolated + # -s for DEBUG mode +diff --git a/rpm/389-ds-base.spec.in b/rpm/389-ds-base.spec.in +index 258d94698..94d8c04c9 100644 +--- a/rpm/389-ds-base.spec.in ++++ b/rpm/389-ds-base.spec.in +@@ -642,42 +642,45 @@ if ! getent passwd $USERNAME >/dev/null ; then + fi + + # Reload our sysctl before we restart (if we can) +-sysctl --system &> $output; true ++sysctl --system &> "$output"; true + +-# Gather the running instances so we can restart them ++# Gather running instances, stop them, run index-check, then restart + instbase="%{_sysconfdir}/%{pkgname}" ++instances="" + ninst=0 +-for dir in $instbase/slapd-* ; do +- echo dir = $dir >> $output 2>&1 || : ++ ++for dir in "$instbase"/slapd-* ; do ++ echo "dir = $dir" >> "$output" 2>&1 || : + if [ ! -d "$dir" ] ; then continue ; fi + case "$dir" in *.removed) continue ;; esac +- basename=`basename $dir` +- inst="%{pkgname}@`echo $basename | sed -e 's/slapd-//g'`" +- echo found instance $inst - getting status >> $output 2>&1 || : +- if /bin/systemctl -q is-active $inst ; then +- echo instance $inst is running >> $output 2>&1 || : ++ basename=$(basename "$dir") ++ inst="%{pkgname}@${basename#slapd-}" ++ inst_name="${basename#slapd-}" ++ echo "found instance $inst - getting status" >> "$output" 2>&1 || : ++ if /bin/systemctl -q is-active "$inst" ; then ++ echo "instance $inst is running - stopping for upgrade" >> "$output" 2>&1 || : + instances="$instances $inst" ++ /bin/systemctl stop "$inst" >> "$output" 2>&1 || : + else +- echo instance $inst is not running >> $output 2>&1 || : ++ echo "instance $inst is not running" >> "$output" 2>&1 || : + fi +- ninst=`expr $ninst + 1` ++ # Run index-check on all instances (running or not) ++ # This fixes index ordering mismatches from older versions ++ dsctl "$inst_name" index-check --fix >> "$output2" 2>&1 || : ++ ninst=$((ninst + 1)) + done ++ + if [ $ninst -eq 0 ] ; then +- echo no instances to upgrade >> $output 2>&1 || : +- exit 0 # have no instances to upgrade - just skip the rest +-else +- # restart running instances +- echo shutting down all instances . . . >> $output 2>&1 || : +- for inst in $instances ; do +- echo stopping instance $inst >> $output 2>&1 || : +- /bin/systemctl stop $inst >> $output 2>&1 || : +- done +- for inst in $instances ; do +- echo starting instance $inst >> $output 2>&1 || : +- /bin/systemctl start $inst >> $output 2>&1 || : +- done ++ echo "no instances to upgrade" >> "$output" 2>&1 || : ++ exit 0 + fi + ++# Restart previously running instances ++for inst in $instances ; do ++ echo "starting instance $inst" >> "$output" 2>&1 || : ++ /bin/systemctl start "$inst" >> "$output" 2>&1 || : ++done ++ + + %preun + if [ $1 -eq 0 ]; then # Final removal +diff --git a/src/lib389/lib389/cli_ctl/dbtasks.py b/src/lib389/lib389/cli_ctl/dbtasks.py +index 856639672..16da966d1 100644 +--- a/src/lib389/lib389/cli_ctl/dbtasks.py ++++ b/src/lib389/lib389/cli_ctl/dbtasks.py +@@ -7,12 +7,24 @@ + # See LICENSE for details. + # --- END COPYRIGHT BLOCK --- + ++import glob + import os ++import re ++import subprocess ++from enum import Enum + from lib389._constants import TaskWarning + from lib389.cli_base import CustomHelpFormatter ++from lib389.dseldif import DSEldif + from pathlib import Path + + ++class IndexOrdering(Enum): ++ """Represents the ordering type of an index.""" ++ INTEGER = "integer" ++ LEXICOGRAPHIC = "lexicographic" ++ UNKNOWN = "unknown" ++ ++ + def dbtasks_db2index(inst, log, args): + rtn = False + if not args.backend: +@@ -126,6 +138,387 @@ def dbtasks_verify(inst, log, args): + log.info("dbverify successful") + + ++def _get_db_dir(dse_ldif): ++ """Get the database directory. ++ ++ Args: ++ dse_ldif: DSEldif instance. ++ ++ Returns: ++ Path to the database directory, or None if not found. ++ """ ++ try: ++ db_dir = dse_ldif.get( ++ "cn=config,cn=ldbm database,cn=plugins,cn=config", ++ "nsslapd-directory", ++ single=True, ++ ) ++ return db_dir ++ except (ValueError, TypeError): ++ pass ++ return None ++ ++ ++ ++def _has_integer_ordering_match(dse_ldif, backend, index_name): ++ """Check if an index has integerOrderingMatch configured in DSE. ++ ++ Args: ++ dse_ldif: DSEldif instance. ++ backend: Backend name. ++ index_name: Name of the index to check. ++ ++ Returns: ++ True if integerOrderingMatch is configured, False otherwise. ++ """ ++ index_dn = "cn={},cn=index,cn={},cn=ldbm database,cn=plugins,cn=config".format( ++ index_name, backend ++ ) ++ matching_rules = dse_ldif.get(index_dn, "nsMatchingRule", lower=True) ++ if matching_rules: ++ return any(mr.lower() == "integerorderingmatch" for mr in matching_rules) ++ return False ++ ++ ++def _has_index_scan_limit(dse_ldif, backend, index_name): ++ """Check if an index has nsIndexIDListScanLimit configured. ++ ++ Args: ++ dse_ldif: DSEldif instance. ++ backend: Backend name. ++ index_name: Name of the index to check. ++ ++ Returns: ++ True if nsIndexIDListScanLimit is configured, False otherwise. ++ """ ++ index_dn = "cn={},cn=index,cn={},cn=ldbm database,cn=plugins,cn=config".format( ++ index_name, backend ++ ) ++ scan_limit = dse_ldif.get(index_dn, "nsIndexIDListScanLimit") ++ return scan_limit is not None ++ ++ ++def _index_config_exists(dse_ldif, backend, index_name): ++ """Check if an index configuration entry exists in DSE. ++ ++ Args: ++ dse_ldif: DSEldif instance. ++ backend: Backend name. ++ index_name: Name of the index to check. ++ ++ Returns: ++ True if the index config entry exists, False otherwise. ++ """ ++ index_dn = "cn={},cn=index,cn={},cn=ldbm database,cn=plugins,cn=config".format( ++ index_name, backend ++ ) ++ try: ++ cn = dse_ldif.get(index_dn, "cn") ++ return cn is not None ++ except (ValueError, KeyError): ++ return False ++ ++ ++def _default_index_exists(dse_ldif, index_name): ++ """Check if an index exists in cn=default indexes. ++ ++ Args: ++ dse_ldif: DSEldif instance. ++ index_name: Name of the index to check. ++ ++ Returns: ++ True if the index exists in default indexes, False otherwise. ++ """ ++ index_dn = "cn={},cn=default indexes,cn=config,cn=ldbm database,cn=plugins,cn=config".format( ++ index_name ++ ) ++ try: ++ cn = dse_ldif.get(index_dn, "cn") ++ return cn is not None ++ except (ValueError, KeyError): ++ return False ++ ++ ++def _check_disk_ordering(db_dir, backend, index_name, dbscan_path, is_mdb, log): ++ """Check if index on disk uses lexicographic or integer ordering. ++ ++ Args: ++ db_dir: Path to the database directory. ++ backend: Backend name. ++ index_name: Name of the index to check. ++ dbscan_path: Path to the dbscan binary. ++ is_mdb: True if using MDB backend. ++ log: Logger instance. ++ ++ Returns: ++ IndexOrdering: The detected ordering type. ++ """ ++ if is_mdb: ++ # MDB uses pseudo-paths: db_dir/backend/index.db ++ # dbscan accesses indexes via paths like: /var/lib/dirsrv/slapd-xxx/db/userroot/parentid.db ++ index_file = os.path.join(db_dir, backend, "{}.db".format(index_name)) ++ else: ++ # BDB has separate directories per backend with actual index files ++ backend_dir = os.path.join(db_dir, backend) ++ if not os.path.exists(backend_dir): ++ return IndexOrdering.UNKNOWN ++ index_file = None ++ pattern = os.path.join(backend_dir, "{}.db*".format(index_name)) ++ for f in glob.glob(pattern): ++ if os.path.isfile(f): ++ index_file = f ++ break ++ if not index_file: ++ return IndexOrdering.UNKNOWN ++ ++ try: ++ result = subprocess.run( ++ [dbscan_path, "-f", index_file], ++ stdout=subprocess.PIPE, ++ stderr=subprocess.PIPE, ++ universal_newlines=True, ++ timeout=60, ++ ) ++ ++ if result.returncode != 0: ++ log.warning(" dbscan returned non-zero exit code for %s", index_file) ++ return IndexOrdering.UNKNOWN ++ ++ # Parse keys from dbscan output ++ keys = [] ++ for line in result.stdout.split("\n"): ++ line = line.strip() ++ if line.startswith("="): ++ match = re.match(r"^=(\d+)", line) ++ if match: ++ keys.append(int(match.group(1))) ++ ++ if len(keys) < 2: ++ return IndexOrdering.UNKNOWN ++ ++ # Check if keys are in integer order by looking for decreasing numeric values ++ # (which would indicate lexicographic ordering, e.g., "3" < "30" < "4") ++ prev_id = keys[0] ++ for i in range(1, min(len(keys), 100)): ++ current_id = keys[i] ++ if prev_id > current_id: ++ return IndexOrdering.LEXICOGRAPHIC ++ prev_id = current_id ++ ++ return IndexOrdering.INTEGER ++ ++ except subprocess.TimeoutExpired: ++ log.warning(" dbscan timed out for %s", index_file) ++ return IndexOrdering.UNKNOWN ++ except OSError as e: ++ log.warning(" Error running dbscan: %s", e) ++ return IndexOrdering.UNKNOWN ++ ++ ++def dbtasks_index_check(inst, log, args): ++ """Check and optionally fix index ordering mismatches. ++ ++ This function detects mismatches between the configured ordering ++ (integerOrderingMatch in DSE) and the actual on-disk ordering of ++ parentid and ancestorid indexes. ++ ++ Args: ++ inst: DirSrv instance. ++ log: Logger instance. ++ args: Parsed command line arguments. ++ ++ Returns: ++ True if all checks passed, False if mismatches were detected. ++ """ ++ # Server must be stopped ++ if inst.status(): ++ log.error("index-check requires the instance to be stopped") ++ return False ++ ++ # Check for dbscan binary ++ dbscan_path = os.path.join(inst.ds_paths.bin_dir, "dbscan") ++ if not os.path.exists(dbscan_path): ++ log.error("dbscan utility not found at %s", dbscan_path) ++ return False ++ ++ # Load DSE ++ try: ++ dse_ldif = DSEldif(inst) ++ except Exception as e: ++ log.error("Failed to read dse.ldif: %s", e) ++ return False ++ ++ # Get backends to check ++ all_backends = dse_ldif.get_backends() ++ if not all_backends: ++ log.info("No backends found") ++ return True ++ ++ # Filter to specific backend if requested ++ if args.backend: ++ # Case-insensitive backend lookup ++ backend_lower = args.backend.lower() ++ matching_backend = None ++ for be in all_backends: ++ if be.lower() == backend_lower: ++ matching_backend = be ++ break ++ if matching_backend is None: ++ log.error("Backend '%s' not found. Available backends: %s", ++ args.backend, ", ".join(all_backends)) ++ return False ++ backends_to_check = [matching_backend] ++ else: ++ backends_to_check = all_backends ++ ++ # Get database directory and check database type ++ db_dir = _get_db_dir(dse_ldif) ++ if not db_dir or not os.path.exists(db_dir): ++ log.error("Database directory not found") ++ return False ++ ++ db_lib = inst.get_db_lib() ++ is_mdb = (db_lib == "mdb") ++ log.info("Database type: %s", db_lib.upper()) ++ ++ # Track all issues found ++ all_ok = True ++ mismatches = [] # (backend, index_name) tuples needing reindex ++ missing_matching_rules = [] # (backend, index_name) tuples missing integerOrderingMatch ++ scan_limits_to_remove = [] # (backend, index_name) tuples with nsIndexIDListScanLimit ++ ancestorid_configs_to_remove = [] # backend names with ancestorid config entries ++ remove_ancestorid_from_defaults = False # Flag to remove from cn=default indexes ++ ++ # Check if ancestorid exists in cn=default indexes (should be removed) ++ if _default_index_exists(dse_ldif, "ancestorid"): ++ log.warning("ancestorid found in cn=default indexes - should be removed") ++ remove_ancestorid_from_defaults = True ++ all_ok = False ++ ++ for backend in backends_to_check: ++ log.info("Checking backend: %s", backend) ++ ++ # Check for ancestorid config entry (should not exist) ++ if _index_config_exists(dse_ldif, backend, "ancestorid"): ++ log.warning(" ancestorid - config entry exists (should be removed)") ++ ancestorid_configs_to_remove.append(backend) ++ all_ok = False ++ ++ # Check parentid and ancestorid indexes ++ for index_name in ["parentid", "ancestorid"]: ++ # Check for scan limits (should be removed) ++ if _has_index_scan_limit(dse_ldif, backend, index_name): ++ log.warning(" %s - has nsIndexIDListScanLimit (should be removed)", index_name) ++ scan_limits_to_remove.append((backend, index_name)) ++ all_ok = False ++ ++ # Check disk ordering ++ disk_ordering = _check_disk_ordering(db_dir, backend, index_name, dbscan_path, is_mdb, log) ++ ++ if disk_ordering == IndexOrdering.UNKNOWN: ++ log.info(" %s - could not determine disk ordering, skipping", index_name) ++ # For parentid, still check if matching rule is missing ++ if index_name == "parentid": ++ config_has_int_order = _has_integer_ordering_match(dse_ldif, backend, index_name) ++ if not config_has_int_order: ++ log.warning(" %s - missing integerOrderingMatch in config", index_name) ++ missing_matching_rules.append((backend, index_name)) ++ all_ok = False ++ continue ++ ++ config_has_int_order = _has_integer_ordering_match(dse_ldif, backend, index_name) ++ config_desc = "integer" if config_has_int_order else "lexicographic" ++ log.info(" %s - config: %s, disk: %s", ++ index_name, config_desc, disk_ordering.value) ++ ++ # For parentid, the desired state is always integer ordering ++ if index_name == "parentid": ++ if not config_has_int_order: ++ log.warning(" %s - missing integerOrderingMatch in config", index_name) ++ if (backend, index_name) not in missing_matching_rules: ++ missing_matching_rules.append((backend, index_name)) ++ all_ok = False ++ ++ if disk_ordering == IndexOrdering.LEXICOGRAPHIC: ++ log.warning(" %s - disk ordering is lexicographic, needs reindex", index_name) ++ if (backend, index_name) not in mismatches: ++ mismatches.append((backend, index_name)) ++ all_ok = False ++ ++ # Handle issues ++ if not all_ok: ++ if args.fix: ++ log.info("Fixing issues...") ++ ++ # Remove ancestorid from cn=default indexes ++ if remove_ancestorid_from_defaults: ++ default_idx_dn = "cn=ancestorid,cn=default indexes,cn=config,cn=ldbm database,cn=plugins,cn=config" ++ log.info(" Removing ancestorid from default indexes...") ++ try: ++ dse_ldif.delete_dn(default_idx_dn) ++ log.info(" Removed ancestorid from default indexes") ++ except Exception as e: ++ log.error(" Failed to remove ancestorid from default indexes: %s", e) ++ return False ++ ++ # Remove scan limits (only for indexes that won't be deleted) ++ for backend, index_name in scan_limits_to_remove: ++ # Skip ancestorid if we're going to delete the whole entry anyway ++ if index_name == "ancestorid" and backend in ancestorid_configs_to_remove: ++ continue ++ index_dn = "cn={},cn=index,cn={},cn=ldbm database,cn=plugins,cn=config".format( ++ index_name, backend ++ ) ++ log.info(" Removing nsIndexIDListScanLimit from %s in backend %s...", index_name, backend) ++ try: ++ dse_ldif.delete(index_dn, "nsIndexIDListScanLimit") ++ log.info(" Removed nsIndexIDListScanLimit from %s", index_name) ++ except Exception as e: ++ log.error(" Failed to remove nsIndexIDListScanLimit from %s: %s", index_name, e) ++ return False ++ ++ # Remove ancestorid config entries from backends ++ for backend in ancestorid_configs_to_remove: ++ index_dn = "cn=ancestorid,cn=index,cn={},cn=ldbm database,cn=plugins,cn=config".format(backend) ++ log.info(" Removing ancestorid config entry from backend %s...", backend) ++ try: ++ dse_ldif.delete_dn(index_dn) ++ log.info(" Removed ancestorid config entry from backend %s", backend) ++ except Exception as e: ++ log.error(" Failed to remove ancestorid config from backend %s: %s", backend, e) ++ return False ++ ++ # Add missing matching rules to dse.ldif ++ for backend, index_name in missing_matching_rules: ++ index_dn = "cn={},cn=index,cn={},cn=ldbm database,cn=plugins,cn=config".format( ++ index_name, backend ++ ) ++ log.info(" Adding integerOrderingMatch to %s in backend %s...", index_name, backend) ++ try: ++ dse_ldif.add(index_dn, "nsMatchingRule", "integerOrderingMatch") ++ log.info(" Updated dse.ldif with integerOrderingMatch for %s", index_name) ++ except Exception as e: ++ log.error(" Failed to update dse.ldif for %s: %s", index_name, e) ++ return False ++ ++ # Reindex indexes with disk ordering issues ++ for backend, index_name in mismatches: ++ log.info(" Reindexing %s in backend %s...", index_name, backend) ++ if not inst.db2index(bename=backend, attrs=[index_name]): ++ log.error(" Failed to reindex %s", index_name) ++ return False ++ log.info(" Reindex of %s completed successfully", index_name) ++ ++ log.info("All issues fixed") ++ return True ++ else: ++ log.info("Issues detected. Run with --fix to repair.") ++ return False ++ else: ++ log.info("All checks passed - no issues found") ++ return True ++ ++ + def create_parser(subcommands): + db2index_parser = subcommands.add_parser('db2index', help="Initialise a reindex of the server database. The server must be stopped for this to proceed.", formatter_class=CustomHelpFormatter) + # db2index_parser.add_argument('suffix', help="The suffix to reindex. IE dc=example,dc=com.") +@@ -172,3 +565,12 @@ def create_parser(subcommands): + ldifs_parser = subcommands.add_parser('ldifs', help="List all the LDIF files located in the server's LDIF directory", formatter_class=CustomHelpFormatter) + ldifs_parser.add_argument('--delete', nargs=1, help="Delete LDIF file") + ldifs_parser.set_defaults(func=dbtasks_ldifs) ++ ++ index_check_parser = subcommands.add_parser('index-check', ++ help="Check for index ordering mismatches (parentid/ancestorid). The server must be stopped.", ++ formatter_class=CustomHelpFormatter) ++ index_check_parser.add_argument('backend', nargs='?', default=None, ++ help="Backend to check. If not specified, all backends are checked.") ++ index_check_parser.add_argument('--fix', action='store_true', default=False, ++ help="Fix mismatches by reindexing affected indexes") ++ index_check_parser.set_defaults(func=dbtasks_index_check) +diff --git a/src/lib389/lib389/dseldif.py b/src/lib389/lib389/dseldif.py +index d12c6424c..7834d9468 100644 +--- a/src/lib389/lib389/dseldif.py ++++ b/src/lib389/lib389/dseldif.py +@@ -125,11 +125,14 @@ class DSEldif(DSLint): + self._contents[i] = self._contents[i].replace(strfrom, strto) + self._update() + +- def _find_attr(self, entry_dn, attr): ++ def _find_attr(self, entry_dn, attr, lower=False): + """Find all attribute values and indexes under a given entry + + Returns entry dn index and attribute data dict: + relative attribute indexes and the attribute value ++ ++ :param lower: Use case-insensitive matching for attribute name ++ :type lower: boolean + """ + + entry_dn_i = self._contents.index("dn: {}\n".format(entry_dn.lower())) +@@ -146,7 +149,11 @@ class DSEldif(DSLint): + + # Find the attribute + for line in entry_slice: +- if line.startswith("{}:".format(attr)): ++ if lower: ++ match = line.lower().startswith("{}:".format(attr.lower())) ++ else: ++ match = line.startswith("{}:".format(attr)) ++ if match: + attr_value = line.split(" ", 1)[1][:-1] + attr_data.update({entry_slice.index(line): attr_value}) + +@@ -155,7 +162,7 @@ class DSEldif(DSLint): + + return entry_dn_i, attr_data + +- def get(self, entry_dn, attr, single=False): ++ def get(self, entry_dn, attr, single=False, lower=False): + """Return attribute values under a given entry + + :param entry_dn: a DN of entry we want to get attribute from +@@ -163,11 +170,13 @@ class DSEldif(DSLint): + :param attr: an attribute name + :type attr: str + :param single: Return a single value instead of a list +- :type sigle: boolean ++ :type single: boolean ++ :param lower: Use case-insensitive matching for attribute name ++ :type lower: boolean + """ + + try: +- _, attr_data = self._find_attr(entry_dn, attr) ++ _, attr_data = self._find_attr(entry_dn, attr, lower=lower) + except ValueError: + return None + +@@ -190,6 +199,38 @@ class DSEldif(DSLint): + + return indexes + ++ def get_backends(self): ++ """Return a list of backend names from DSE. ++ ++ Returns backend names preserving their original case, as the ++ database directory names on disk use the original case. ++ ++ Note: DSEldif lowercases DN lines, so we read the 'cn' attribute ++ from each entry to get the original case. ++ ++ :returns: List of backend names ++ """ ++ backends = [] ++ excluded = ("config", "monitor", "index", "encrypted attributes") ++ ++ for entry in self._contents: ++ if (entry.startswith("dn: cn=") and ++ ",cn=ldbm database,cn=plugins,cn=config" in entry): ++ parts = entry.split(",") ++ if len(parts) > 1: ++ cn_lower = parts[0].replace("dn: cn=", "") ++ if cn_lower not in excluded: ++ dn = entry.strip()[4:].strip() ++ try: ++ suffix = self.get(dn, "nsslapd-suffix") ++ if suffix: ++ cn_values = self.get(dn, "cn") ++ if cn_values: ++ backends.append(cn_values[0]) ++ except (ValueError, IndexError): ++ pass ++ ++ return list(set(backends)) + + def add_entry(self, entry): + """Add a new entry +-- +2.52.0 + diff --git a/0014-Issue-7096-2nd-During-replication-online-total-init-.patch b/0014-Issue-7096-2nd-During-replication-online-total-init-.patch new file mode 100644 index 0000000..7274644 --- /dev/null +++ b/0014-Issue-7096-2nd-During-replication-online-total-init-.patch @@ -0,0 +1,135 @@ +From d1886fbc7d97e49ac0b8bc5bd6ae7e3263bb0cfb Mon Sep 17 00:00:00 2001 +From: Viktor Ashirov +Date: Tue, 27 Jan 2026 14:26:29 +0100 +Subject: [PATCH] Issue 7096 - (2nd) During replication online total init the + function idl_id_is_in_idlist is not scaling with large database (#7205) + +Bug Description: +The fix for #7096 optimized the BDB backend's `idl_new_range_fetch()` +function to use ID ranges instead of checking the full ID list during +online total initialization. However, the LMDB backend's +`idl_lmdb_range_fetch()` function and its callback +`idl_range_add_id_cb()` were not updated and still use the non-scaling +`idl_id_is_in_idlist()` function. + +Fix Description: +Apply the same optimization to the LMDB backend. + +Fixes: https://github.com/389ds/389-ds-base/issues/7096 + +Reviewed by: @tbordaz, @droideck (Thanks!) +--- + ldap/servers/slapd/back-ldbm/idl_new.c | 39 ++++++++++---------------- + 1 file changed, 15 insertions(+), 24 deletions(-) + +diff --git a/ldap/servers/slapd/back-ldbm/idl_new.c b/ldap/servers/slapd/back-ldbm/idl_new.c +index 2d978353f..613d53815 100644 +--- a/ldap/servers/slapd/back-ldbm/idl_new.c ++++ b/ldap/servers/slapd/back-ldbm/idl_new.c +@@ -66,6 +66,7 @@ typedef struct { + size_t leftoverlen; + size_t leftovercnt; + IDList *idl; ++ IdRange_t *idrange_list; + int flag_err; + ID lastid; + ID suffix; +@@ -700,9 +701,9 @@ error: + } + } + } +- slapi_ch_free((void **)&leftover); +- idrange_free(&idrange_list); + } ++ slapi_ch_free((void **)&leftover); ++ idrange_free(&idrange_list); + slapi_log_err(SLAPI_LOG_FILTER, "idl_new_range_fetch", + "Found %d candidates; error code is: %d\n", + idl ? idl->b_nids : 0, *flag_err); +@@ -716,7 +717,6 @@ static int + idl_range_add_id_cb(dbi_val_t *key, dbi_val_t *data, void *ctx) + { + idl_range_ctx_t *rctx = ctx; +- int idl_rc = 0; + ID id = 0; + + if (key->data == NULL) { +@@ -779,10 +779,12 @@ idl_range_add_id_cb(dbi_val_t *key, dbi_val_t *data, void *ctx) + * found entry is the one from the suffix + */ + rctx->suffix = keyval; +- idl_rc = idl_append_extend(&rctx->idl, id); +- } else if ((keyval == rctx->suffix) || idl_id_is_in_idlist(rctx->idl, keyval)) { ++ idl_append_extend(&rctx->idl, id); ++ idrange_add_id(&rctx->idrange_list, id); ++ } else if ((keyval == rctx->suffix) || idl_id_is_in_idlist_ranges(rctx->idl, rctx->idrange_list, keyval)) { + /* the parent is the suffix or already in idl. */ +- idl_rc = idl_append_extend(&rctx->idl, id); ++ idl_append_extend(&rctx->idl, id); ++ idrange_add_id(&rctx->idrange_list, id); + } else { + /* Otherwise, keep the {keyval,id} in leftover array */ + if (!rctx->leftover) { +@@ -797,14 +799,7 @@ idl_range_add_id_cb(dbi_val_t *key, dbi_val_t *data, void *ctx) + rctx->leftovercnt++; + } + } else { +- idl_rc = idl_append_extend(&rctx->idl, id); +- } +- if (idl_rc) { +- slapi_log_err(SLAPI_LOG_ERR, "idl_lmdb_range_fetch", +- "Unable to extend id list (err=%d)\n", idl_rc); +- idl_free(&rctx->idl); +- rctx->flag_err = LDAP_UNWILLING_TO_PERFORM; +- return DBI_RC_NOTFOUND; ++ idl_append_extend(&rctx->idl, id); + } + #if defined(DB_ALLIDS_ON_READ) + /* enforce the allids read limit */ +@@ -841,7 +836,6 @@ idl_lmdb_range_fetch( + { + int ret = 0; + int ret2 = 0; +- int idl_rc = 0; + dbi_cursor_t cursor = {0}; + back_txn s_txn; + struct ldbminfo *li = (struct ldbminfo *)be->be_database->plg_private; +@@ -891,6 +885,7 @@ idl_lmdb_range_fetch( + idl_range_ctx.lastid = 0; + idl_range_ctx.count = 0; + idl_range_ctx.index_id = index_id; ++ idl_range_ctx.idrange_list = NULL; + if (operator & SLAPI_OP_RANGE_NO_IDL_SORT) { + struct _back_info_index_key bck_info; + /* We are doing a bulk import +@@ -966,22 +961,18 @@ error: + while(remaining > 0) { + for (size_t i = 0; i < idl_range_ctx.leftovercnt; i++) { + if (idl_range_ctx.leftover[i].key > 0 && +- idl_id_is_in_idlist(idl_range_ctx.idl, idl_range_ctx.leftover[i].key) != 0) { ++ idl_id_is_in_idlist_ranges(idl_range_ctx.idl, idl_range_ctx.idrange_list, idl_range_ctx.leftover[i].key) != 0) { + /* if the leftover key has its parent in the idl */ +- idl_rc = idl_append_extend(&idl_range_ctx.idl, idl_range_ctx.leftover[i].id); +- if (idl_rc) { +- slapi_log_err(SLAPI_LOG_ERR, "idl_lmdb_range_fetch", +- "Unable to extend id list (err=%d)\n", idl_rc); +- idl_free(&idl_range_ctx.idl); +- break; +- } ++ idl_append_extend(&idl_range_ctx.idl, idl_range_ctx.leftover[i].id); ++ idrange_add_id(&idl_range_ctx.idrange_list, idl_range_ctx.leftover[i].id); + idl_range_ctx.leftover[i].key = 0; + remaining--; + } + } + } +- slapi_ch_free((void **)&idl_range_ctx.leftover); + } ++ slapi_ch_free((void **)&idl_range_ctx.leftover); ++ idrange_free(&idl_range_ctx.idrange_list); + *flag_err = idl_range_ctx.flag_err; + slapi_log_err(SLAPI_LOG_FILTER, "idl_lmdb_range_fetch", + "Found %d candidates; error code is: %d\n", +-- +2.52.0 + diff --git a/0015-Issue-7076-6992-6784-6214-Fix-CI-test-failures-7077.patch b/0015-Issue-7076-6992-6784-6214-Fix-CI-test-failures-7077.patch new file mode 100644 index 0000000..e208679 --- /dev/null +++ b/0015-Issue-7076-6992-6784-6214-Fix-CI-test-failures-7077.patch @@ -0,0 +1,877 @@ +From 42c38a7a95e898a6185ac7c71ad89119ef406509 Mon Sep 17 00:00:00 2001 +From: Akshay Adhikari +Date: Tue, 18 Nov 2025 21:57:10 +0530 +Subject: [PATCH] Issue 7076, 6992, 6784, 6214 - Fix CI test failures (#7077) + +- Fixed import test bugs in regression_test.py (cleanup handler, LDIF permissions) - + https://github.com/389ds/389-ds-base/issues/6992 +- Fixed ModRDN cache corruption on failed operations (parent update check, cache cleanup) +- Fixed attribute uniqueness test fixture cleanup in attruniq_test.py +- mproved test stability by fixing race conditions in replication, healthcheck, + web UI, memberOf, and basic tests. +- Fixed entrycache_eviction_test.py to track incremental log counts instead of cumulative - + https://github.com/389ds/389-ds-base/issues/6784 + +Fixes: https://github.com/389ds/389-ds-base/issues/7076 +Relates: https://github.com/389ds/389-ds-base/issues/6992 +Relates: https://github.com/389ds/389-ds-base/issues/6784 +Fixes: https://github.com/389ds/389-ds-base/issues/6214 + +Reviewed by: @vashirov, @progier389 (Thanks!) +--- + dirsrvtests/tests/suites/basic/basic_test.py | 4 +- + .../healthcheck/health_system_indexes_test.py | 3 + + .../tests/suites/import/regression_test.py | 388 +++++++++++++++++- + .../suites/memberof_plugin/fixup_test.py | 8 +- + .../tests/suites/plugins/attruniq_test.py | 84 ++-- + .../suites/replication/regression_m2_test.py | 4 + + .../replication/repl_log_monitoring_test.py | 8 +- + .../suites/webui/database/database_test.py | 1 + + ldap/servers/slapd/back-ldbm/ldbm_modrdn.c | 44 +- + 9 files changed, 481 insertions(+), 63 deletions(-) + +diff --git a/dirsrvtests/tests/suites/basic/basic_test.py b/dirsrvtests/tests/suites/basic/basic_test.py +index be825efe9..e9b611439 100644 +--- a/dirsrvtests/tests/suites/basic/basic_test.py ++++ b/dirsrvtests/tests/suites/basic/basic_test.py +@@ -593,7 +593,7 @@ def test_basic_import_export(topology_st, import_example_ldif): + # + # Test online/offline LDIF imports + # +- topology_st.standalone.start() ++ topology_st.standalone.restart() + # topology_st.standalone.config.set('nsslapd-errorlog-level', '1') + + # Generate a test ldif (50k entries) +@@ -691,6 +691,8 @@ def test_basic_backup(topology_st, import_example_ldif): + + log.info('Running test_basic_backup...') + ++ topology_st.standalone.restart() ++ + backup_dir = topology_st.standalone.get_bak_dir() + '/backup_test_online' + log.info(f'Backup directory is {backup_dir}') + +diff --git a/dirsrvtests/tests/suites/healthcheck/health_system_indexes_test.py b/dirsrvtests/tests/suites/healthcheck/health_system_indexes_test.py +index 55b9d2f6d..da7673283 100644 +--- a/dirsrvtests/tests/suites/healthcheck/health_system_indexes_test.py ++++ b/dirsrvtests/tests/suites/healthcheck/health_system_indexes_test.py +@@ -172,6 +172,7 @@ def test_missing_parentid(topology_st, log_buffering_enabled): + log.info("Re-add the parentId index") + backend = Backends(standalone).get("userRoot") + backend.add_index("parentid", ["eq"], matching_rules=["integerOrderingMatch"]) ++ standalone.restart() + + run_healthcheck_and_flush_log(topology_st, standalone, json=False, searched_code=CMD_OUTPUT) + run_healthcheck_and_flush_log(topology_st, standalone, json=True, searched_code=JSON_OUTPUT) +@@ -215,6 +216,7 @@ def test_missing_matching_rule(topology_st, log_buffering_enabled): + log.info("Re-add the integerOrderingMatch matching rule") + parentid_index = Index(standalone, PARENTID_DN) + parentid_index.add("nsMatchingRule", "integerOrderingMatch") ++ standalone.restart() + + run_healthcheck_and_flush_log(topology_st, standalone, json=False, searched_code=CMD_OUTPUT) + run_healthcheck_and_flush_log(topology_st, standalone, json=True, searched_code=JSON_OUTPUT) +@@ -445,6 +447,7 @@ def test_multiple_missing_indexes(topology_st, log_buffering_enabled): + backend = Backends(standalone).get("userRoot") + backend.add_index("parentid", ["eq"], matching_rules=["integerOrderingMatch"]) + backend.add_index("nsuniqueid", ["eq"]) ++ standalone.restart() + + run_healthcheck_and_flush_log(topology_st, standalone, json=False, searched_code=CMD_OUTPUT) + run_healthcheck_and_flush_log(topology_st, standalone, json=True, searched_code=JSON_OUTPUT) +diff --git a/dirsrvtests/tests/suites/import/regression_test.py b/dirsrvtests/tests/suites/import/regression_test.py +index 18611de35..bdbb516e8 100644 +--- a/dirsrvtests/tests/suites/import/regression_test.py ++++ b/dirsrvtests/tests/suites/import/regression_test.py +@@ -5,6 +5,7 @@ + # See LICENSE for details. + # --- END COPYRIGHT BLOCK --- + # ++from abc import ABC, abstractmethod + from decimal import * + import ldap + import logging +@@ -16,9 +17,9 @@ from lib389.backend import Backends + from lib389.properties import TASK_WAIT + from lib389.topologies import topology_st as topo + from lib389.dbgen import dbgen_users +-from lib389._constants import DEFAULT_SUFFIX ++from lib389._constants import DEFAULT_SUFFIX, DEFAULT_BENAME + from lib389.tasks import * +-from lib389.idm.user import UserAccounts ++from lib389.idm.user import UserAccounts, UserAccount + from lib389.idm.directorymanager import DirectoryManager + from lib389.dbgen import * + from lib389.utils import * +@@ -92,7 +93,203 @@ class AddDelUsers(threading.Thread): + return self._ran + + +-def test_replay_import_operation(topo): ++def get_backend_by_name(inst, bename): ++ bes = Backends(inst) ++ bename = bename.lower() ++ be = [ be for be in bes if be.get_attr_val_utf8_l('cn') == bename ] ++ return be[0] if len(be) == 1 else None ++ ++ ++class LogHandler(): ++ def __init__(self, logfd, patterns): ++ self.logfd = logfd ++ self.patterns = [ p.lower() for p in patterns ] ++ self.pos = logfd.tell() ++ self.last_result = None ++ ++ def zero(self): ++ return [0 for _ in range(len(self.patterns))] ++ ++ def countCaptures(self): ++ res = self.zero() ++ self.logfd.seek(self.pos) ++ for line in iter(self.logfd.readline, ''): ++ # Ignore autotune messages that may confuse the counts ++ if 'bdb_start_autotune' in line: ++ continue ++ # Ignore LMDB size warnings that may confuse the counts ++ if 'dbmdb_ctx_t_db_max_size_set' in line: ++ continue ++ log.info(f'ERROR LOG line is {line.strip()}') ++ for idx,pattern in enumerate(self.patterns): ++ if pattern in line.lower(): ++ res[idx] += 1 ++ self.pos = self.logfd.tell() ++ self.last_result = res ++ log.info(f'ERROR LOG counts are: {res}') ++ return res ++ ++ def seek2end(self): ++ self.pos = os.fstat(self.logfd.fileno()).st_size ++ ++ def check(self, idx, val): ++ count = self.last_result[idx] ++ assert count == val , f"Should have {val} '{self.patterns[idx]}' messages but got: {count} - idx = {idx}" ++ ++ ++class IEHandler(ABC): ++ def __init__(self, inst, errlog, ldifname, bename=DEFAULT_BENAME, suffix=None): ++ self.inst = inst ++ self.errlog = errlog ++ self.ldifname = ldifname ++ self.bename = bename ++ self.suffix = suffix ++ self.ldif = ldifname if ldifname.startswith('/') else f'{inst.get_ldif_dir()}/{ldifname}.ldif' ++ ++ @abstractmethod ++ def get_name(self): ++ pass ++ ++ @abstractmethod ++ def _run_task_b(self): ++ pass ++ ++ @abstractmethod ++ def _run_task_s(self): ++ pass ++ ++ @abstractmethod ++ def _run_offline(self): ++ pass ++ ++ @abstractmethod ++ def _set_log_pattern(self, success): ++ pass ++ ++ def run(self, extra_checks, success=True): ++ if self.errlog: ++ self._set_log_pattern(success) ++ self.errlog.seek2end() ++ ++ if self.inst.status(): ++ if self.bename: ++ log.info(f"Performing online {self.get_name()} of backend {self.bename} into LDIF file {self.ldif}") ++ r = self._run_task_b() ++ else: ++ log.info(f"Performing online {self.get_name()} of suffix {self.suffix} into LDIF file {self.ldif}") ++ r = self._run_task_s() ++ r.wait() ++ time.sleep(1) ++ else: ++ if self.bename: ++ log.info(f"Performing offline {self.get_name()} of backend {self.bename} into LDIF file {self.ldif}") ++ else: ++ log.info(f"Performing offline {self.get_name()} of suffix {self.suffix} into LDIF file {self.ldif}") ++ self._run_offline() ++ if self.errlog: ++ expected_counts = ['*' for _ in range(len(self.errlog.patterns))] ++ for (idx, val) in extra_checks: ++ expected_counts[idx] = val ++ res = self.errlog.countCaptures() ++ log.info(f'Expected errorlog counts are: {expected_counts}') ++ if success is True or success is False: ++ log.info(f'Number of {self.errlog.patterns[0]} in errorlog is: {res[0]}') ++ assert res[0] >= 1 ++ for (idx, val) in extra_checks: ++ self.errlog.check(idx, val) ++ ++ def check_db(self): ++ assert self.inst.dbscan(bename=self.bename, index='id2entry') ++ ++ ++class Importer(IEHandler): ++ def get_name(self): ++ return "import" ++ ++ def _set_log_pattern(self, success): ++ if success is True: ++ self.errlog.patterns[0] = 'import complete' ++ elif success is False: ++ self.errlog.patterns[0] = 'import failed' ++ ++ def _run_task_b(self): ++ bes = Backends(self.inst) ++ r = bes.import_ldif(self.bename, [self.ldif,], include_suffixes=self.suffix) ++ return r ++ ++ def _run_task_s(self): ++ r = ImportTask(self.inst) ++ r.import_suffix_from_ldif(self.ldif, self.suffix) ++ return r ++ ++ def _run_offline(self): ++ log.info(f'self.inst.ldif2db({self.bename}, {self.suffix}, ...)') ++ if self.suffix is None: ++ self.inst.ldif2db(self.bename, self.suffix, None, False, self.ldif) ++ else: ++ self.inst.ldif2db(self.bename, [self.suffix, ], None, False, self.ldif) ++ ++ ++class Exporter(IEHandler): ++ def get_name(self): ++ return "export" ++ ++ def _set_log_pattern(self, success): ++ if success is True: ++ self.errlog.patterns[0] = 'export finished' ++ elif success is False: ++ self.errlog.patterns[0] = 'export failed' ++ ++ def _run_task_b(self): ++ bes = Backends(self.inst) ++ r = bes.export_ldif(self.bename, self.ldif, include_suffixes=self.suffix) ++ return r ++ ++ def _run_task_s(self): ++ r = ExportTask(self.inst) ++ r.export_suffix_to_ldif(self.ldif, self.suffix) ++ return r ++ ++ def _run_offline(self): ++ self.inst.db2ldif(self.bename, self.suffix, None, False, False, self.ldif) ++ ++ ++def preserve_func(topo, request, restart): ++ # Ensure that topology get preserved helper ++ inst = topo.standalone ++ ++ def fin(): ++ if restart: ++ inst.restart() ++ Importer(inst, None, "save").run(()) ++ ++ r = Exporter(inst, None, "save") ++ if not os.path.isfile(r.ldif): ++ r.run(()) ++ request.addfinalizer(fin) ++ ++ ++@pytest.fixture(scope="function") ++def preserve(topo, request): ++ # Ensure that topology get preserved (no restart) ++ preserve_func(topo, request, False) ++ ++ ++@pytest.fixture(scope="function") ++def preserve_r(topo, request): ++ # Ensure that topology get preserved (with restart) ++ preserve_func(topo, request, True) ++ ++ ++@pytest.fixture(scope="function") ++def verify(topo): ++ # Check that backend is not broken ++ inst = topo.standalone ++ dn=f'uid=demo_user,ou=people,{DEFAULT_SUFFIX}' ++ assert UserAccount(inst,dn).exists() ++ ++ ++def test_replay_import_operation(topo, preserve_r, verify): + """ Check after certain failed import operation, is it + possible to replay an import operation + +@@ -487,7 +684,190 @@ def test_ldif2db_after_backend_create(topo): + import_time_2 = create_backend_and_import(instance, ldif_file_2, 'o=test_2', 'test_2') + + log.info('Import times should be approximately the same') +- assert abs(import_time_1 - import_time_2) < 5 ++ assert abs(import_time_1 - import_time_2) < 15 ++ ++ ++def test_ldif_missing_suffix_entry(topo, request, verify): ++ """Test that ldif2db/import aborts if suffix entry is not in the ldif ++ ++ :id: 731bd0d6-8cc8-11f0-8ef2-c85309d5c3e3 ++ :setup: Standalone Instance ++ :steps: ++ 1. Prepare final cleanup ++ 2. Add a few users ++ 3. Export ou=people subtree ++ 4. Online import using backend name ou=people subtree ++ 5. Online import using suffix name ou=people subtree ++ 6. Stop the instance ++ 7. Offline import using backend name ou=people subtree ++ 8. Offline import using suffix name ou=people subtree ++ 9. Generate ldif with a far away suffix ++ 10. Offline import using backend name and "far" ldif ++ 11. Offline import using suffix name and "far" ldif ++ 12. Start the instance ++ 13. Online import using backend name and "far" ldif ++ 14. Online import using suffix name and "far" ldif ++ :expectedresults: ++ 1. Operation successful ++ 2. Operation successful ++ 3. Operation successful ++ 4. Import should success, skip all entries, db should exists ++ 5. Import should success, skip all entries, db should exists ++ 6. Operation successful ++ 7. Import should success, skip all entries, db should exists ++ 8. Import should success, skip all entries, db should exists ++ 9. Operation successful ++ 10. Import should success, skip all entries, db should exists ++ 11. Import should success, 10 entries skipped, db should exists ++ 12. Operation successful ++ 13. Import should success, skip all entries, db should exists ++ 14. Import should success, 10 entries skipped, db should exists ++ """ ++ ++ inst = topo.standalone ++ inst.config.set('nsslapd-errorlog-level', '266354688') ++ no_suffix_on = ( ++ (1, 0), # no errors are expected. ++ (2, 1), # 1 warning is expected. ++ (3, 0), # no 'no parent' warning is expected. ++ (4, 1), # 1 'all entries were skipped' warning ++ (5, 0), # no 'returning task warning' info message ++ ) ++ no_suffix_off = ( ++ (1, 0), # no errors are expected. ++ (2, 1), # 1 warning is expected. ++ (3, 0), # no 'no parent' warning is expected. ++ (4, 1), # 1 'all entries were skipped' warning ++ (5, 1), # 1 'returning task warning' info message ++ ) ++ ++ far_suffix_on = ( ++ (1, 0), # no errors are expected. ++ (2, 1), # 1 warning (consolidated, pre-check aborts after 4 entries) ++ (3, 0), # 0 'no parent' warnings (pre-check aborts before processing) ++ (4, 1), # 1 'all entries were skipped' warning (from pre-check) ++ (5, 0), # 0 'returning task warning' info message (online import) ++ ) ++ # Backend-specific behavior for orphan detection when suffix parameter is provided ++ nbw = 0 if get_default_db_lib() == "bdb" else 10 ++ far_suffix_with_suffix_on = ( ++ (1, 0), # no errors are expected. ++ (2, nbw), # 0 (BDB early filtering) or 10 (LMDB orphan detection) warnings ++ (3, nbw), # 0 (BDB early filtering) or 10 (LMDB orphan detection) 'no parent' warnings ++ (4, 0), # 0 'all entries were skipped' warning (no pre-check abort) ++ (5, 0), # 0 'returning task warning' info message (online import) ++ ) ++ far_suffix_off = ( ++ (1, 0), # no errors are expected. ++ (2, 1), # 1 warning (consolidated, pre-check detects missing suffix) ++ (3, 0), # 0 'no parent' warnings (pre-check aborts before processing) ++ (4, 1), # 1 'all entries were skipped' warning (from pre-check) ++ (5, 1), # 1 'returning task warning' info message (offline import) ++ ) ++ far_suffix_with_suffix_off = ( ++ (1, 0), # no errors are expected. ++ (2, nbw), # 0 (BDB early filtering) or 10 (LMDB orphan detection) warnings ++ (3, nbw), # 0 (BDB early filtering) or 10 (LMDB orphan detection) 'no parent' warnings ++ (4, 0), # 0 'all entries were skipped' warning (no pre-check abort) ++ (5, 0), # 0 'returning task warning' (rc=0, successful import of suffix) ++ ) ++ ++ with open(inst.ds_paths.error_log, 'at+') as fd: ++ patterns = ( ++ "Reserved for IEHandler", ++ " ERR ", ++ " WARN ", ++ "has no parent", ++ "all entries were skipped", ++ "returning task warning", ++ ) ++ errlog = LogHandler(fd, patterns) ++ no_errors = ((1, 0), (2, 0)) # no errors nor warnings are expected. ++ ++ ++ # 1. Prepare final cleanup ++ Exporter(inst, errlog, "full").run(no_errors) ++ ++ def fin(): ++ inst.start() ++ with open(inst.ds_paths.error_log, 'at+') as cleanup_fd: ++ cleanup_errlog = LogHandler(cleanup_fd, patterns) ++ Importer(inst, cleanup_errlog, "full").run(no_errors) ++ ++ if not DEBUGGING: ++ request.addfinalizer(fin) ++ ++ # 2. Add a few users ++ user = UserAccounts(inst, DEFAULT_SUFFIX) ++ users = [ user.create_test_user(uid=i) for i in range(10) ] ++ ++ # 3. Export ou=people subtree ++ e = Exporter(inst, errlog, "people", suffix=f'ou=people,{DEFAULT_SUFFIX}') ++ e.run(no_errors) # no errors nor warnings are expected. ++ ++ # 4. Online import using backend name ou=people subtree ++ e = Importer(inst, errlog, "people") ++ e.run(no_suffix_on) ++ e.check_db() ++ ++ # 5. Online import using suffix name ou=people subtree ++ e = Importer(inst, errlog, "people", suffix=DEFAULT_SUFFIX) ++ e.run(no_suffix_on) ++ e.check_db() ++ ++ # 6. Stop the instance ++ inst.stop() ++ ++ # 7. Offline import using backend name ou=people subtree ++ e = Importer(inst, errlog, "people") ++ e.run(no_suffix_off) ++ e.check_db() ++ ++ # 8. Offline import using suffix name ou=people subtree ++ e = Importer(inst, errlog, "people", suffix=DEFAULT_SUFFIX) ++ e.run(no_suffix_off) ++ e.check_db() ++ ++ # 9. Generate ldif with a far away suffix ++ e = Importer(inst, errlog, "full") ++ people_ldif = e.ldif ++ e = Importer(inst, errlog, "far") ++ with open(e.ldif, "wt") as fout: ++ with open(people_ldif, "rt") as fin: ++ # Copy version ++ line = fin.readline() ++ fout.write(line) ++ line = fin.readline() ++ fout.write(line) ++ # Generate fake entries ++ for idx in range(10): ++ fout.write(f"dn: uid=id{idx},dc=foo\nobjectclasses: extensibleObject\n\n") ++ for line in iter(fin.readline, ''): ++ fout.write(line) ++ ++ os.chmod(e.ldif, 0o644) ++ ++ # 10. Offline import using backend name ou=people subtree ++ e.run(far_suffix_off) ++ e.check_db() ++ ++ # 11. Offline import using suffix name ou=people subtree ++ e = Importer(inst, errlog, "far", suffix=DEFAULT_SUFFIX) ++ e.run(far_suffix_with_suffix_off) ++ e.check_db() ++ ++ # 12. Start the instance ++ inst.start() ++ ++ # 13. Online import using backend name ou=people subtree ++ e = Importer(inst, errlog, "far") ++ e.run(far_suffix_on) ++ e.check_db() ++ ++ # 14. Online import using suffix name ou=people subtree ++ e = Importer(inst, errlog, "far", suffix=DEFAULT_SUFFIX) ++ e.run(far_suffix_with_suffix_on) ++ e.check_db() + + + if __name__ == '__main__': +diff --git a/dirsrvtests/tests/suites/memberof_plugin/fixup_test.py b/dirsrvtests/tests/suites/memberof_plugin/fixup_test.py +index 5aac40d2b..44804bd1c 100644 +--- a/dirsrvtests/tests/suites/memberof_plugin/fixup_test.py ++++ b/dirsrvtests/tests/suites/memberof_plugin/fixup_test.py +@@ -44,7 +44,10 @@ def test_fixup_task_limit(topo): + group = groups.create(properties={'cn': 'test'}) + + users = UserAccounts(topo.standalone, DEFAULT_SUFFIX) +- for idx in range(400): ++ # Turn on access log buffering to speed up user creation ++ buffering = topo.standalone.config.get_attr_val_utf8('nsslapd-accesslog-logbuffering') ++ topo.standalone.config.set('nsslapd-accesslog-logbuffering', 'on') ++ for idx in range(6000): + user = users.create(properties={ + 'uid': 'testuser%s' % idx, + 'cn' : 'testuser%s' % idx, +@@ -55,6 +58,9 @@ def test_fixup_task_limit(topo): + }) + group.add('member', user.dn) + ++ # Restore access log buffering ++ topo.standalone.config.set('nsslapd-accesslog-logbuffering', buffering) ++ + # Configure memberOf plugin + memberof = MemberOfPlugin(topo.standalone) + memberof.enable() +diff --git a/dirsrvtests/tests/suites/plugins/attruniq_test.py b/dirsrvtests/tests/suites/plugins/attruniq_test.py +index 6eaee08a4..a2be413c8 100644 +--- a/dirsrvtests/tests/suites/plugins/attruniq_test.py ++++ b/dirsrvtests/tests/suites/plugins/attruniq_test.py +@@ -84,14 +84,23 @@ def containers(topology_st, request): + def attruniq(topology_st, request): + log.info('Setup attribute uniqueness plugin') + attruniq = AttributeUniquenessPlugin(topology_st.standalone, dn="cn=attruniq,cn=plugins,cn=config") +- attruniq.create(properties={'cn': 'attruniq'}) +- attruniq.add_unique_attribute('cn') ++ ++ if attruniq.exists(): ++ attruniq.delete() ++ topology_st.standalone.restart() ++ ++ attruniq.create(properties={ ++ 'cn': 'attruniq', ++ 'uniqueness-attribute-name': 'cn', ++ 'uniqueness-subtrees': 'cn=config', ++ 'nsslapd-pluginEnabled': 'on' ++ }) + topology_st.standalone.restart() + + def fin(): + if attruniq.exists(): +- attruniq.disable() + attruniq.delete() ++ topology_st.standalone.restart() + + request.addfinalizer(fin) + +@@ -250,8 +259,8 @@ def test_modrdn_attr_uniqueness(topology_st, attruniq): + group1 = groups.create(properties={'cn': 'group1'}) + group2 = groups.create(properties={'cn': 'group2'}) + +- attruniq.add_unique_attribute('mail') +- attruniq.add_unique_subtree(group2.dn) ++ attruniq.replace('uniqueness-attribute-name', 'mail') ++ attruniq.replace('uniqueness-subtrees', group2.dn) + attruniq.enable_all_subtrees() + log.debug(f'Enable PLUGIN_ATTR_UNIQUENESS plugin as "ON"') + attruniq.enable() +@@ -274,9 +283,6 @@ def test_modrdn_attr_uniqueness(topology_st, attruniq): + assert 'attribute value already exist' in str(excinfo.value) + log.debug(excinfo.value) + +- log.debug('Move user2 to group1') +- user2.rename(f'uid={user2.rdn}', group1.dn) +- + user1.delete() + user2.delete() + +@@ -302,17 +308,11 @@ def test_multiple_attr_uniqueness(topology_st, attruniq): + 6. Should raise CONSTRAINT_VIOLATION + """ + +- try: +- attruniq.add_unique_attribute('mail') +- attruniq.add_unique_attribute('mailAlternateAddress') +- attruniq.add_unique_subtree(DEFAULT_SUFFIX) +- attruniq.enable_all_subtrees() +- log.debug(f'Enable PLUGIN_ATTR_UNIQUENESS plugin as "ON"') +- attruniq.enable() +- except ldap.LDAPError as e: +- log.fatal('test_multiple_attribute_uniqueness: Failed to configure plugin for "mail": error {}'.format(e.args[0]['desc'])) +- assert False +- ++ attruniq.replace('uniqueness-attribute-name', ['mail', 'mailAlternateAddress']) ++ attruniq.replace('uniqueness-subtrees', DEFAULT_SUFFIX) ++ attruniq.enable_all_subtrees() ++ log.debug(f'Enable PLUGIN_ATTR_UNIQUENESS plugin as "ON"') ++ attruniq.enable() + topology_st.standalone.restart() + + users = UserAccounts(topology_st.standalone, DEFAULT_SUFFIX) +@@ -383,8 +383,9 @@ def test_exclude_subtrees(topology_st, attruniq): + 16. Success + 17. Success + """ +- attruniq.add_unique_attribute('telephonenumber') +- attruniq.add_unique_subtree(DEFAULT_SUFFIX) ++ # Replace dummy config with actual test config ++ attruniq.replace('uniqueness-attribute-name', 'telephonenumber') ++ attruniq.replace('uniqueness-subtrees', DEFAULT_SUFFIX) + attruniq.enable_all_subtrees() + attruniq.enable() + topology_st.standalone.restart() +@@ -517,10 +518,18 @@ def test_matchingrule_attr(topology_st): + """ + + inst = topology_st.standalone +- + attruniq = AttributeUniquenessPlugin(inst, + dn="cn=attribute uniqueness,cn=plugins,cn=config") +- attruniq.add_unique_attribute('cn:CaseExactMatch:') ++ ++ if attruniq.exists(): ++ attruniq.delete() ++ inst.restart() ++ ++ attruniq.create(properties={ ++ 'cn': 'attribute uniqueness', ++ 'uniqueness-attribute-name': 'cn:CaseExactMatch:', ++ 'uniqueness-subtrees': DEFAULT_SUFFIX ++ }) + attruniq.enable_all_subtrees() + attruniq.enable() + inst.restart() +@@ -595,7 +604,7 @@ def test_one_container_add(topology_st, attruniq, containers, active_user_1): + active_2.delete() + + log.info('Setup attribute uniqueness plugin for "cn" attribute') +- attruniq.add_unique_subtree(ACTIVE_DN) ++ attruniq.replace('uniqueness-subtrees', ACTIVE_DN) + attruniq.enable() + topology_st.standalone.restart() + +@@ -628,7 +637,7 @@ def test_one_container_mod(topology_st, attruniq, containers, + """ + + log.info('Setup attribute uniqueness plugin for "cn" attribute') +- attruniq.add_unique_subtree(ACTIVE_DN) ++ attruniq.replace('uniqueness-subtrees', ACTIVE_DN) + attruniq.enable() + topology_st.standalone.restart() + +@@ -656,7 +665,7 @@ def test_one_container_modrdn(topology_st, attruniq, containers, + """ + + log.info('Setup attribute uniqueness plugin for "cn" attribute') +- attruniq.add_unique_subtree(ACTIVE_DN) ++ attruniq.replace('uniqueness-subtrees', ACTIVE_DN) + attruniq.enable() + topology_st.standalone.restart() + +@@ -690,8 +699,7 @@ def test_multiple_containers_add(topology_st, attruniq, containers, + """ + + log.info('Setup attribute uniqueness plugin for "cn" attribute') +- attruniq.add_unique_subtree(ACTIVE_DN) +- attruniq.add_unique_subtree(STAGE_DN) ++ attruniq.replace('uniqueness-subtrees', [ACTIVE_DN, STAGE_DN]) + attruniq.enable() + topology_st.standalone.restart() + +@@ -789,8 +797,7 @@ def test_multiple_containers_mod(topology_st, attruniq, containers, + """ + + log.info('Setup attribute uniqueness plugin for "cn" attribute') +- attruniq.add_unique_subtree(ACTIVE_DN) +- attruniq.add_unique_subtree(STAGE_DN) ++ attruniq.replace('uniqueness-subtrees', [ACTIVE_DN, STAGE_DN]) + attruniq.enable() + topology_st.standalone.restart() + +@@ -874,8 +881,8 @@ def test_multiple_containers_modrdn(topology_st, attruniq, containers, + """ + + log.info('Setup attribute uniqueness plugin for "cn" attribute') +- attruniq.add_unique_subtree(ACTIVE_DN) +- attruniq.add_unique_subtree(STAGE_DN) ++ # Replace dummy subtree with actual test subtrees ++ attruniq.replace('uniqueness-subtrees', [ACTIVE_DN, STAGE_DN]) + attruniq.enable() + topology_st.standalone.restart() + +@@ -993,7 +1000,10 @@ def test_invalid_config_missing_attr_name(topology_st): + _config_file(topology_st, action='save') + + attruniq = AttributeUniquenessPlugin(topology_st.standalone, dn="cn=attruniq,cn=plugins,cn=config") +- attruniq.create(properties={'cn': 'attruniq'}) ++ attruniq.create(properties={ ++ 'cn': 'attruniq', ++ 'uniqueness-subtrees': DEFAULT_SUFFIX ++ }) + attruniq.enable() + + topology_st.standalone.errorlog_file = open(topology_st.standalone.errlog, "r") +@@ -1040,9 +1050,11 @@ def test_invalid_config_invalid_subtree(topology_st): + _config_file(topology_st, action='save') + + attruniq = AttributeUniquenessPlugin(topology_st.standalone, dn="cn=attruniq,cn=plugins,cn=config") +- attruniq.create(properties={'cn': 'attruniq'}) +- attruniq.add_unique_attribute('cn') +- attruniq.add_unique_subtree('invalid_subtree') ++ attruniq.create(properties={ ++ 'cn': 'attruniq', ++ 'uniqueness-attribute-name': 'cn', ++ 'uniqueness-subtrees': 'invalid_subtree' ++ }) + attruniq.enable() + + topology_st.standalone.errorlog_file = open(topology_st.standalone.errlog, "r") +diff --git a/dirsrvtests/tests/suites/replication/regression_m2_test.py b/dirsrvtests/tests/suites/replication/regression_m2_test.py +index ba1ffcc9c..db5140b0b 100644 +--- a/dirsrvtests/tests/suites/replication/regression_m2_test.py ++++ b/dirsrvtests/tests/suites/replication/regression_m2_test.py +@@ -1221,6 +1221,10 @@ def test_rid_starting_with_0(topo_m2, request): + for replica,rid in zip(replicas, ['010', '020']): + replica.replace('nsDS5ReplicaId', rid) + ++ # Restart required - replica IDs are loaded at startup and cached in memory ++ S1.restart() ++ S2.restart() ++ + # Restore replica id in finalizer + def fin(): + for replica,rid in zip(replicas, ['1', '2']): +diff --git a/dirsrvtests/tests/suites/replication/repl_log_monitoring_test.py b/dirsrvtests/tests/suites/replication/repl_log_monitoring_test.py +index 665fcb96f..b2b2d25a2 100644 +--- a/dirsrvtests/tests/suites/replication/repl_log_monitoring_test.py ++++ b/dirsrvtests/tests/suites/replication/repl_log_monitoring_test.py +@@ -369,11 +369,13 @@ def test_replication_log_monitoring_multi_suffix(topo_m4): + repl.ensure_agreement(s1, s2) + repl.ensure_agreement(s2, s1) + +- # Allow initial topology to settle before capturing metrics ++ # Wait for all the setup replication to settle, then clear the logs + for suffix in all_suffixes: + repl = ReplicationManager(suffix) +- repl.test_replication_topology(topo_m4) +- ++ for s1 in suppliers: ++ for s2 in suppliers: ++ if s1 != s2: ++ repl.wait_for_replication(s1, s2) + for supplier in suppliers: + supplier.deleteAccessLogs(restart=True) + +diff --git a/dirsrvtests/tests/suites/webui/database/database_test.py b/dirsrvtests/tests/suites/webui/database/database_test.py +index ef105d262..05ddb6b00 100644 +--- a/dirsrvtests/tests/suites/webui/database/database_test.py ++++ b/dirsrvtests/tests/suites/webui/database/database_test.py +@@ -138,6 +138,7 @@ def test_chaining_configuration_availability(topology_st, page, browser_name): + log.info('Click on Chaining Configuration and check if element is loaded.') + frame.get_by_role('tab', name='Database', exact=True).click() + frame.locator('#chaining-config').click() ++ frame.locator('#chaining-page').wait_for() + frame.locator('#defSizeLimit').wait_for() + assert frame.locator('#defSizeLimit').is_visible() + +diff --git a/ldap/servers/slapd/back-ldbm/ldbm_modrdn.c b/ldap/servers/slapd/back-ldbm/ldbm_modrdn.c +index e3b7e5783..b1a29ff7f 100644 +--- a/ldap/servers/slapd/back-ldbm/ldbm_modrdn.c ++++ b/ldap/servers/slapd/back-ldbm/ldbm_modrdn.c +@@ -1080,7 +1080,9 @@ ldbm_back_modrdn(Slapi_PBlock *pb) + } + } + } +- if (slapi_sdn_get_dn(dn_newsuperiordn) != NULL) { ++ /* Only update parent if we're actually moving to a NEW parent (not the same parent) */ ++ if (slapi_sdn_get_dn(dn_newsuperiordn) != NULL && ++ slapi_sdn_compare(dn_newsuperiordn, &dn_parentdn) != 0) { + /* Push out the db modifications from the parent entry */ + retval = modify_update_all(be, pb, &parent_modify_context, &txn); + if (DBI_RC_RETRY == retval) { +@@ -1335,11 +1337,6 @@ ldbm_back_modrdn(Slapi_PBlock *pb) + goto common_return; + + error_return: +- /* Revert the caches if this is the parent operation */ +- if (parent_op && betxn_callback_fails) { +- revert_cache(inst, &parent_time); +- } +- + /* result already sent above - just free stuff */ + if (postentry) { + slapi_entry_free(postentry); +@@ -1417,13 +1414,6 @@ error_return: + slapi_pblock_set(pb, SLAPI_PLUGIN_OPRETURN, ldap_result_code ? &ldap_result_code : &retval); + } + slapi_pblock_get(pb, SLAPI_PB_RESULT_TEXT, &ldap_result_message); +- +- /* As it is a BETXN plugin failure then +- * revert the caches if this is the parent operation +- */ +- if (parent_op) { +- revert_cache(inst, &parent_time); +- } + } + retval = plugin_call_mmr_plugin_postop(pb, NULL,SLAPI_PLUGIN_BE_TXN_POST_MODRDN_FN); + +@@ -1437,6 +1427,15 @@ error_return: + } + } + ++ /* Revert the caches if this is the parent operation and cache modifications were made. ++ * Cache modifications (via modify_switch_entries) only happen after BETXN PRE plugins succeed, ++ * so we should only revert if we got past that point (i.e., BETXN POST plugin failures). ++ * For BETXN PRE failures, no cache modifications were made to parent/newparent entries. ++ */ ++ if (parent_op && betxn_callback_fails && postentry) { ++ revert_cache(inst, &parent_time); ++ } ++ + common_return: + + /* result code could be used in the bepost plugin functions. */ +@@ -1482,12 +1481,22 @@ common_return: + "operation failed, the target entry is cleared from dncache (%s)\n", slapi_entry_get_dn(ec->ep_entry)); + CACHE_REMOVE(&inst->inst_dncache, bdn); + CACHE_RETURN(&inst->inst_dncache, &bdn); ++ ++ /* Also remove ec from entry cache and free it since the operation failed */ ++ if (inst && cache_is_in_cache(&inst->inst_cache, ec)) { ++ CACHE_REMOVE(&inst->inst_cache, ec); ++ CACHE_RETURN(&inst->inst_cache, &ec); ++ } else { ++ /* ec was not in cache, just free it */ ++ backentry_free(&ec); ++ } ++ ec = NULL; + } + + if (ec && inst) { + CACHE_RETURN(&inst->inst_cache, &ec); ++ ec = NULL; + } +- ec = NULL; + } + + if (inst) { +@@ -1817,6 +1826,8 @@ moddn_get_newdn(Slapi_PBlock *pb, Slapi_DN *dn_olddn, Slapi_DN *dn_newrdn, Slapi + + /* + * Return the entries to the cache. ++ * For the original entry 'e', we should NOT remove it from cache on failure, ++ * as it's still a valid entry in the directory. + */ + static void + moddn_unlock_and_return_entry( +@@ -1825,12 +1836,9 @@ moddn_unlock_and_return_entry( + { + ldbm_instance *inst = (ldbm_instance *)be->be_instance_info; + +- /* Something bad happened so we should give back all the entries */ ++ /* Unlock and return the entry to the cache */ + if (*targetentry != NULL) { + cache_unlock_entry(&inst->inst_cache, *targetentry); +- if (cache_is_in_cache(&inst->inst_cache, *targetentry)) { +- CACHE_REMOVE(&inst->inst_cache, *targetentry); +- } + CACHE_RETURN(&inst->inst_cache, targetentry); + *targetentry = NULL; + } +-- +2.52.0 + diff --git a/0016-Issue-7076-Fix-revert_cache-never-called-in-modrdn-7.patch b/0016-Issue-7076-Fix-revert_cache-never-called-in-modrdn-7.patch new file mode 100644 index 0000000..df4c71b --- /dev/null +++ b/0016-Issue-7076-Fix-revert_cache-never-called-in-modrdn-7.patch @@ -0,0 +1,57 @@ +From b5a8ab96ec5c2a0cc0d478c5a906f3d2bfb4f6c5 Mon Sep 17 00:00:00 2001 +From: Akshay Adhikari +Date: Thu, 5 Feb 2026 15:41:09 +0530 +Subject: [PATCH] Issue 7076 - Fix revert_cache() never called in modrdn + (#7220) + +Description: The postentry check in PR #7077 was broken - postentry is always NULL +at that point, fixed by removing the check. + +Relates: #7076 + +Reviewed by: @vashirov, @mreynolds389, @droideck (Thanks!) +--- + ldap/servers/slapd/back-ldbm/ldbm_modrdn.c | 12 +++++++----- + 1 file changed, 7 insertions(+), 5 deletions(-) + +diff --git a/ldap/servers/slapd/back-ldbm/ldbm_modrdn.c b/ldap/servers/slapd/back-ldbm/ldbm_modrdn.c +index b1a29ff7f..36377fc01 100644 +--- a/ldap/servers/slapd/back-ldbm/ldbm_modrdn.c ++++ b/ldap/servers/slapd/back-ldbm/ldbm_modrdn.c +@@ -103,6 +103,7 @@ ldbm_back_modrdn(Slapi_PBlock *pb) + Connection *pb_conn = NULL; + int32_t parent_op = 0; + int32_t betxn_callback_fails = 0; /* if a BETXN fails we need to revert entry cache */ ++ int32_t cache_mod_phase = 0; /* set when we reach the cache modification phase */ + struct timespec parent_time; + Slapi_Mods *smods_add_rdn = NULL; + +@@ -1229,6 +1230,8 @@ ldbm_back_modrdn(Slapi_PBlock *pb) + goto error_return; + } + ++ /* We're now past the BETXN PRE phase and entering the cache modification phase */ ++ cache_mod_phase = 1; + postentry = slapi_entry_dup(ec->ep_entry); + + if (parententry != NULL) { +@@ -1427,12 +1430,11 @@ error_return: + } + } + +- /* Revert the caches if this is the parent operation and cache modifications were made. +- * Cache modifications (via modify_switch_entries) only happen after BETXN PRE plugins succeed, +- * so we should only revert if we got past that point (i.e., BETXN POST plugin failures). +- * For BETXN PRE failures, no cache modifications were made to parent/newparent entries. ++ /* Revert the caches if this is the parent operation AND we reached the ++ * cache modification phase. If BETXN PRE fails, cache_mod_phase is 0 ++ * and we don't need to revert since no cache modifications were made. + */ +- if (parent_op && betxn_callback_fails && postentry) { ++ if (parent_op && betxn_callback_fails && cache_mod_phase) { + revert_cache(inst, &parent_time); + } + +-- +2.52.0 + diff --git a/0017-Issue-6947-Fix-health_system_indexes_test.py.patch b/0017-Issue-6947-Fix-health_system_indexes_test.py.patch new file mode 100644 index 0000000..b878751 --- /dev/null +++ b/0017-Issue-6947-Fix-health_system_indexes_test.py.patch @@ -0,0 +1,24 @@ +From cd530816544b9a583adc517db2ff34cdfa84fc43 Mon Sep 17 00:00:00 2001 +From: Viktor Ashirov +Date: Wed, 11 Feb 2026 19:53:44 +0100 +Subject: [PATCH] Issue 6947 - Fix health_system_indexes_test.py + +--- + .../tests/suites/healthcheck/health_system_indexes_test.py | 1 + + 1 file changed, 1 insertion(+) + +diff --git a/dirsrvtests/tests/suites/healthcheck/health_system_indexes_test.py b/dirsrvtests/tests/suites/healthcheck/health_system_indexes_test.py +index da7673283..932857dd6 100644 +--- a/dirsrvtests/tests/suites/healthcheck/health_system_indexes_test.py ++++ b/dirsrvtests/tests/suites/healthcheck/health_system_indexes_test.py +@@ -99,6 +99,7 @@ def run_healthcheck_and_flush_log(topology, instance, searched_code, json, searc + "memberof", + ] + args.dry_run = False ++ args.exclude_check = [] + + # If we are using BDB as a backend, we will get error DSBLE0006 on new versions + if ( +-- +2.52.0 + diff --git a/0018-Issue-7121-2nd-LeakSanitizer-various-leaks-during-re.patch b/0018-Issue-7121-2nd-LeakSanitizer-various-leaks-during-re.patch new file mode 100644 index 0000000..6783f40 --- /dev/null +++ b/0018-Issue-7121-2nd-LeakSanitizer-various-leaks-during-re.patch @@ -0,0 +1,71 @@ +From 64dcdbcbba3e8e2239a2e099787c40713c8a0a7e Mon Sep 17 00:00:00 2001 +From: Viktor Ashirov +Date: Mon, 9 Feb 2026 13:18:09 +0100 +Subject: [PATCH] Issue 7121 - (2nd) LeakSanitizer: various leaks during + replication (#7212) + +Bug Description: +With the previous fix 75e0e487545893a7b0d83f94f9264c10f8bb0353 applied, +server can crash in ber_bvcpy. + +``` +Program terminated with signal SIGSEGV, Segmentation fault. +#0 ber_bvcpy (bvs=0x7f1d00000000, bvd=0x7f1da2cd73c0) at ldap/servers/slapd/value.c:47 +47 len = bvs->bv_len; +[Current thread is 1 (Thread 0x7f1db47fe640 (LWP 36576))] +(gdb) bt +#0 ber_bvcpy (bvs=0x7f1d00000000, bvd=0x7f1da2cd73c0) at ldap/servers/slapd/value.c:47 +#1 ber_bvcpy (bvs=0x7f1d00000000, bvd=0x7f1da2cd73c0) at ldap/servers/slapd/value.c:40 +#2 slapi_value_set_berval (bval=0x7f1d00000000, value=0x7f1da2cd73c0) at ldap/servers/slapd/value.c:322 +#3 slapi_value_set_berval (value=value@entry=0x7f1da2cd73c0, bval=bval@entry=0x7f1d00000000) at ldap/servers/slapd/value.c:317 +#4 0x00007f1e48b7d787 in value_init (v=v@entry=0x7f1da2cd73c0, bval=bval@entry=0x7f1d00000000, t=t@entry=0 '\000', csn=csn@entry=0x0) + at ldap/servers/slapd/value.c:179 +#5 0x00007f1e48b7d884 in value_new (bval=bval@entry=0x7f1d00000000, t=t@entry=0 '\000', csn=csn@entry=0x0) at ldap/servers/slapd/value.c:158 +#6 0x00007f1e48b7ddb7 in slapi_value_dup (v=0x7f1d00000000) at ldap/servers/slapd/value.c:147 +#7 0x00007f1e48b7e262 in valueset_set_valueset (vs2=0x7f1d502b5218, vs1=0x7f1da2c5b358) at ldap/servers/slapd/valueset.c:1244 +#8 valueset_set_valueset (vs1=0x7f1da2c5b358, vs2=0x7f1d502b5218) at ldap/servers/slapd/valueset.c:1220 +#9 0x00007f1e48add4af in slapi_attr_dup (attr=0x7f1d502b51e0) at ldap/servers/slapd/attr.c:396 +#10 0x00007f1e48af0f60 in slapi_entry_dup (e=0x7f1da2c19000) at ldap/servers/slapd/entry.c:2036 +#11 0x00007f1e442c734e in ldbm_back_modify (pb=0x7f1da2c00000) at ldap/servers/slapd/back-ldbm/ldbm_modify.c:741 +#12 0x00007f1e48b30076 in op_shared_modify (pb=pb@entry=0x7f1da2c00000, pw_change=pw_change@entry=0, old_pw=0x0) + at ldap/servers/slapd/modify.c:1079 +#13 0x00007f1e48b30ced in do_modify (pb=pb@entry=0x7f1da2c00000) at ldap/servers/slapd/modify.c:377 +#14 0x000055e990e2fd1c in connection_dispatch_operation (pb=0x7f1da2c00000, op=, conn=) + at ldap/servers/slapd/connection.c:672 +#15 connection_threadmain (arg=) at ldap/servers/slapd/connection.c:1955 +#16 0x00007f1e48839bd4 in _pt_root (arg=0x7f1e439d9500) at pthreads/../../../../nspr/pr/src/pthreads/ptthread.c:191 +#17 0x00007f1e4868a19a in start_thread (arg=) at pthread_create.c:443 +#18 0x00007f1e4870f100 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81 +``` + +The fix changed from always setting `v_csnset = NULL` to only freeing it +inside the if-block. + +Fix Description: +Keep `csnset_free()` outside the if-block to handle all values, not just +those matching the condtion. + +Related: https://github.com/389ds/389-ds-base/issues/7121 + +Reviewed by: @progier389, @droideck (Thanks!) +--- + ldap/servers/slapd/entrywsi.c | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/ldap/servers/slapd/entrywsi.c b/ldap/servers/slapd/entrywsi.c +index e1bdc1bab..0d044092d 100644 +--- a/ldap/servers/slapd/entrywsi.c ++++ b/ldap/servers/slapd/entrywsi.c +@@ -1185,8 +1185,8 @@ resolve_attribute_state_deleted_to_present(Slapi_Entry *e, Slapi_Attr *a, Slapi_ + if ((csn_compare(vucsn, deletedcsn) >= 0) || + value_distinguished_at_csn(e, a, valuestoupdate[i], deletedcsn)) { + entry_deleted_value_to_present_value(a, valuestoupdate[i]); +- csnset_free(&valuestoupdate[i]->v_csnset); + } ++ csnset_free(&valuestoupdate[i]->v_csnset); + } + } + } +-- +2.52.0 + diff --git a/0019-Issue-7150-Compressed-access-log-rotations-skipped-a.patch b/0019-Issue-7150-Compressed-access-log-rotations-skipped-a.patch new file mode 100644 index 0000000..de880d8 --- /dev/null +++ b/0019-Issue-7150-Compressed-access-log-rotations-skipped-a.patch @@ -0,0 +1,551 @@ +From eaa2077433bc3a38dee60ae5255f067aa3113691 Mon Sep 17 00:00:00 2001 +From: Simon Pichugin +Date: Tue, 16 Dec 2025 15:48:35 -0800 +Subject: [PATCH] Issue 7150 - Compressed access log rotations skipped, + accesslog-list out of sync (#7151) + +Description: Accept `.gz`-suffixed rotated log filenames when +rebuilding rotation info and checking previous logs, preventing +compressed rotations from being dropped from the internal list. + +Add regression tests to stress log rotation with compression, +verify `nsslapd-accesslog-list` stays in sync, and guard against +crashes when flushing buffered logs during rotation. +Minor doc fix in test. + +Fixes: https://github.com/389ds/389-ds-base/issues/7150 + +Reviewed by: @progier389 (Thanks!) +--- + .../suites/logging/log_flush_rotation_test.py | 341 +++++++++++++++++- + ldap/servers/slapd/log.c | 99 +++-- + 2 files changed, 402 insertions(+), 38 deletions(-) + +diff --git a/dirsrvtests/tests/suites/logging/log_flush_rotation_test.py b/dirsrvtests/tests/suites/logging/log_flush_rotation_test.py +index b33a622e1..864ba9c5d 100644 +--- a/dirsrvtests/tests/suites/logging/log_flush_rotation_test.py ++++ b/dirsrvtests/tests/suites/logging/log_flush_rotation_test.py +@@ -6,6 +6,7 @@ + # See LICENSE for details. + # --- END COPYRIGHT BLOCK --- + # ++import glob + import os + import logging + import time +@@ -13,14 +14,351 @@ import pytest + from lib389._constants import DEFAULT_SUFFIX, PW_DM + from lib389.tasks import ImportTask + from lib389.idm.user import UserAccounts ++from lib389.idm.domain import Domain ++from lib389.idm.directorymanager import DirectoryManager + from lib389.topologies import topology_st as topo + + + log = logging.getLogger(__name__) + + ++def remove_rotated_access_logs(inst): ++ """ ++ Remove all rotated access log files to start fresh for each test. ++ This prevents log files from previous tests affecting current test results. ++ """ ++ log_dir = inst.get_log_dir() ++ patterns = [ ++ f'{log_dir}/access.2*', # Uncompressed rotated logs ++ f'{log_dir}/access.*.gz', # Compressed rotated logs ++ ] ++ for pattern in patterns: ++ for log_file in glob.glob(pattern): ++ try: ++ os.remove(log_file) ++ log.info(f"Removed old log file: {log_file}") ++ except OSError as e: ++ log.warning(f"Could not remove {log_file}: {e}") ++ ++ ++def reset_access_log_config(inst): ++ """ ++ Reset access log configuration to default values. ++ """ ++ inst.config.set('nsslapd-accesslog-compress', 'off') ++ inst.config.set('nsslapd-accesslog-maxlogsize', '100') ++ inst.config.set('nsslapd-accesslog-maxlogsperdir', '10') ++ inst.config.set('nsslapd-accesslog-logrotationsync-enabled', 'off') ++ inst.config.set('nsslapd-accesslog-logbuffering', 'on') ++ inst.config.set('nsslapd-accesslog-logexpirationtime', '-1') ++ inst.config.set('nsslapd-accesslog-logminfreediskspace', '5') ++ ++ ++def generate_heavy_load(inst, suffix, iterations=50): ++ """ ++ Generate heavy LDAP load to fill access log quickly. ++ Performs multiple operations: searches, modifies, binds to populate logs. ++ """ ++ for i in range(iterations): ++ suffix.replace('description', f'iteration_{i}') ++ suffix.get_attr_val('description') ++ ++ ++def count_access_logs(log_dir, compressed_only=False): ++ """ ++ Count access log files in the log directory. ++ Returns count of rotated access logs (not including the active 'access' file). ++ """ ++ if compressed_only: ++ pattern = f'{log_dir}/access.*.gz' ++ else: ++ pattern = f'{log_dir}/access.2*' ++ log_files = glob.glob(pattern) ++ return len(log_files) ++ ++ ++def test_log_pileup_with_compression(topo): ++ """Test that log rotation properly deletes old logs when compression is enabled. ++ ++ :id: fa1bfce8-b6d3-4520-a0a8-bead14fa5838 ++ :setup: Standalone Instance ++ :steps: ++ 1. Clean up existing rotated logs and reset configuration ++ 2. Enable access log compression ++ 3. Set strict log limits (small maxlogsperdir) ++ 4. Disable log expiration to test count-based deletion ++ 5. Generate heavy load to create many log rotations ++ 6. Verify log count does not exceed maxlogsperdir limit ++ :expectedresults: ++ 1. Success ++ 2. Success ++ 3. Success ++ 4. Success ++ 5. Success ++ 6. Log count should be at or below maxlogsperdir + small buffer ++ """ ++ ++ inst = topo.standalone ++ suffix = Domain(inst, DEFAULT_SUFFIX) ++ log_dir = inst.get_log_dir() ++ ++ # Clean up before test ++ remove_rotated_access_logs(inst) ++ reset_access_log_config(inst) ++ inst.restart() ++ ++ max_logs = 5 ++ inst.config.set('nsslapd-accesslog-compress', 'on') ++ inst.config.set('nsslapd-accesslog-maxlogsperdir', str(max_logs)) ++ inst.config.set('nsslapd-accesslog-maxlogsize', '1') # 1MB to trigger rotation ++ inst.config.set('nsslapd-accesslog-logrotationsync-enabled', 'off') ++ inst.config.set('nsslapd-accesslog-logbuffering', 'off') ++ ++ inst.config.set('nsslapd-accesslog-logexpirationtime', '-1') ++ ++ inst.config.set('nsslapd-accesslog-logminfreediskspace', '5') ++ ++ inst.restart() ++ time.sleep(2) ++ ++ target_logs = max_logs * 3 ++ for i in range(target_logs): ++ log.info(f"Generating load for log rotation {i+1}/{target_logs}") ++ generate_heavy_load(inst, suffix, iterations=150) ++ time.sleep(1) # Wait for rotation ++ ++ time.sleep(3) ++ ++ logs_on_disk = count_access_logs(log_dir) ++ log.info(f"Configured maxlogsperdir: {max_logs}") ++ log.info(f"Actual rotated logs on disk: {logs_on_disk}") ++ ++ all_access_logs = glob.glob(f'{log_dir}/access*') ++ log.info(f"All access log files: {all_access_logs}") ++ ++ max_allowed = max_logs + 2 ++ assert logs_on_disk <= max_allowed, ( ++ f"Log rotation failed to delete old files! " ++ f"Expected at most {max_allowed} rotated logs (maxlogsperdir={max_logs} + 2 buffer), " ++ f"but found {logs_on_disk}. The server has lost track of the file list." ++ ) ++ ++ ++@pytest.mark.parametrize("compress_enabled", ["on", "off"]) ++def test_accesslog_list_mismatch(topo, compress_enabled): ++ """Test that nsslapd-accesslog-list stays synchronized with actual log files. ++ ++ :id: 0a8a46a6-cae7-43bd-8b64-5e3481480cd3 ++ :parametrized: yes ++ :setup: Standalone Instance ++ :steps: ++ 1. Clean up existing rotated logs and reset configuration ++ 2. Configure log rotation with compression enabled/disabled ++ 3. Generate activity to trigger multiple rotations ++ 4. Get the nsslapd-accesslog-list attribute ++ 5. Compare with actual files on disk ++ 6. Verify they match (accounting for .gz extension when enabled) ++ :expectedresults: ++ 1. Success ++ 2. Success ++ 3. Success ++ 4. Success ++ 5. Success ++ 6. The list attribute should match actual files on disk ++ """ ++ ++ inst = topo.standalone ++ suffix = Domain(inst, DEFAULT_SUFFIX) ++ log_dir = inst.get_log_dir() ++ compression_on = compress_enabled == "on" ++ ++ # Clean up before test ++ remove_rotated_access_logs(inst) ++ reset_access_log_config(inst) ++ inst.restart() ++ ++ inst.config.set('nsslapd-accesslog-compress', compress_enabled) ++ inst.config.set('nsslapd-accesslog-maxlogsize', '1') ++ inst.config.set('nsslapd-accesslog-maxlogsperdir', '10') ++ inst.config.set('nsslapd-accesslog-logrotationsync-enabled', 'off') ++ inst.config.set('nsslapd-accesslog-logbuffering', 'off') ++ inst.config.set('nsslapd-accesslog-logexpirationtime', '-1') ++ ++ inst.restart() ++ time.sleep(2) ++ ++ for i in range(15): ++ suffix_note = "(no compression)" if not compression_on else "" ++ log.info(f"Generating load for rotation {i+1}/15 {suffix_note}") ++ generate_heavy_load(inst, suffix, iterations=150) ++ time.sleep(1) ++ ++ time.sleep(3) ++ ++ accesslog_list = inst.config.get_attr_vals_utf8('nsslapd-accesslog-list') ++ log.info(f"nsslapd-accesslog-list entries (compress={compress_enabled}): {len(accesslog_list)}") ++ log.info(f"nsslapd-accesslog-list (compress={compress_enabled}): {accesslog_list}") ++ ++ disk_files = glob.glob(f'{log_dir}/access.2*') ++ log.info(f"Actual files on disk (compress={compress_enabled}): {len(disk_files)}") ++ log.info(f"Disk files (compress={compress_enabled}): {disk_files}") ++ ++ disk_files_for_compare = set() ++ for fpath in disk_files: ++ if compression_on and fpath.endswith('.gz'): ++ disk_files_for_compare.add(fpath[:-3]) ++ else: ++ disk_files_for_compare.add(fpath) ++ ++ list_files_set = set(accesslog_list) ++ missing_from_disk = list_files_set - disk_files_for_compare ++ extra_on_disk = disk_files_for_compare - list_files_set ++ ++ if missing_from_disk: ++ log.error( ++ f"[compress={compress_enabled}] Files in list but NOT on disk: {missing_from_disk}" ++ ) ++ if extra_on_disk: ++ log.warning( ++ f"[compress={compress_enabled}] Files on disk but NOT in list: {extra_on_disk}" ++ ) ++ ++ assert not missing_from_disk, ( ++ f"nsslapd-accesslog-list mismatch (compress={compress_enabled})! " ++ f"Files listed but missing from disk: {missing_from_disk}. " ++ f"This indicates the server's internal list is out of sync with actual files." ++ ) ++ ++ if len(extra_on_disk) > 2: ++ log.warning( ++ f"Potential log tracking issue (compress={compress_enabled}): " ++ f"{len(extra_on_disk)} files on disk are not tracked in the accesslog-list: " ++ f"{extra_on_disk}" ++ ) ++ ++ ++def test_accesslog_list_mixed_compression(topo): ++ """Test that nsslapd-accesslog-list correctly tracks both compressed and uncompressed logs. ++ ++ :id: 11b088cd-23be-407d-ad16-4ce2e12da09e ++ :setup: Standalone Instance ++ :steps: ++ 1. Clean up existing rotated logs and reset configuration ++ 2. Create rotated logs with compression OFF ++ 3. Enable compression and create more rotated logs ++ 4. Get the nsslapd-accesslog-list attribute ++ 5. Compare with actual files on disk ++ 6. Verify all files are correctly tracked (uncompressed and compressed) ++ :expectedresults: ++ 1. Success ++ 2. Success - uncompressed rotated logs created ++ 3. Success - compressed rotated logs created ++ 4. Success ++ 5. Success ++ 6. The list should contain base filenames (without .gz) that ++ correspond to files on disk (either as-is or with .gz suffix) ++ """ ++ ++ inst = topo.standalone ++ suffix = Domain(inst, DEFAULT_SUFFIX) ++ log_dir = inst.get_log_dir() ++ ++ # Clean up before test ++ remove_rotated_access_logs(inst) ++ reset_access_log_config(inst) ++ inst.restart() ++ ++ inst.config.set('nsslapd-accesslog-compress', 'off') ++ inst.config.set('nsslapd-accesslog-maxlogsize', '1') ++ inst.config.set('nsslapd-accesslog-maxlogsperdir', '20') ++ inst.config.set('nsslapd-accesslog-logrotationsync-enabled', 'off') ++ inst.config.set('nsslapd-accesslog-logbuffering', 'off') ++ inst.config.set('nsslapd-accesslog-logexpirationtime', '-1') ++ ++ inst.restart() ++ time.sleep(2) ++ ++ for i in range(15): ++ log.info(f"Generating load for uncompressed rotation {i+1}/15") ++ generate_heavy_load(inst, suffix, iterations=150) ++ time.sleep(1) ++ ++ time.sleep(2) ++ ++ # Check what we have so far ++ uncompressed_files = glob.glob(f'{log_dir}/access.2*') ++ log.info(f"Files on disk after uncompressed phase: {uncompressed_files}") ++ ++ inst.config.set('nsslapd-accesslog-compress', 'on') ++ inst.restart() ++ time.sleep(2) ++ ++ for i in range(15): ++ log.info(f"Generating load for compressed rotation {i+1}/15") ++ generate_heavy_load(inst, suffix, iterations=150) ++ time.sleep(1) ++ ++ time.sleep(3) ++ ++ accesslog_list = inst.config.get_attr_vals_utf8('nsslapd-accesslog-list') ++ ++ disk_files = glob.glob(f'{log_dir}/access.2*') ++ ++ log.info(f"nsslapd-accesslog-list entries: {len(accesslog_list)}") ++ log.info(f"nsslapd-accesslog-list: {sorted(accesslog_list)}") ++ log.info(f"Actual files on disk: {len(disk_files)}") ++ log.info(f"Disk files: {sorted(disk_files)}") ++ ++ compressed_on_disk = [f for f in disk_files if f.endswith('.gz')] ++ uncompressed_on_disk = [f for f in disk_files if not f.endswith('.gz')] ++ log.info(f"Compressed files on disk: {compressed_on_disk}") ++ log.info(f"Uncompressed files on disk: {uncompressed_on_disk}") ++ ++ list_files_set = set(accesslog_list) ++ ++ disk_files_base = set() ++ for fpath in disk_files: ++ if fpath.endswith('.gz'): ++ disk_files_base.add(fpath[:-3]) # Strip .gz ++ else: ++ disk_files_base.add(fpath) ++ ++ missing_from_disk = list_files_set - disk_files_base ++ ++ extra_on_disk = disk_files_base - list_files_set ++ ++ if missing_from_disk: ++ log.error(f"Files in list but NOT on disk: {missing_from_disk}") ++ if extra_on_disk: ++ log.warning(f"Files on disk but NOT in list: {extra_on_disk}") ++ ++ assert not missing_from_disk, ( ++ f"nsslapd-accesslog-list contains stale entries! " ++ f"Files in list but not on disk (as base or .gz): {missing_from_disk}" ++ ) ++ ++ for list_file in accesslog_list: ++ exists_uncompressed = os.path.exists(list_file) ++ exists_compressed = os.path.exists(list_file + '.gz') ++ assert exists_uncompressed or exists_compressed, ( ++ f"File in accesslog-list does not exist on disk: {list_file} " ++ f"(checked both {list_file} and {list_file}.gz)" ++ ) ++ if exists_compressed and not exists_uncompressed: ++ log.info(f" {list_file} -> exists as .gz (compressed)") ++ elif exists_uncompressed: ++ log.info(f" {list_file} -> exists (uncompressed)") ++ ++ if len(extra_on_disk) > 1: ++ log.warning( ++ f"Some files on disk are not tracked in accesslog-list: {extra_on_disk}" ++ ) ++ ++ log.info("Mixed compression test completed successfully") ++ ++ + def test_log_flush_and_rotation_crash(topo): +- """Make sure server does not crash whening flushing a buffer and rotating ++ """Make sure server does not crash when flushing a buffer and rotating + the log at the same time + + :id: d4b0af2f-48b2-45f5-ae8b-f06f692c3133 +@@ -36,6 +374,7 @@ def test_log_flush_and_rotation_crash(topo): + 3. Success + 4. Success + """ ++ # NOTE: This test is placed last as it may affect the suffix state. + + inst = topo.standalone + +diff --git a/ldap/servers/slapd/log.c b/ldap/servers/slapd/log.c +index 6f57a3d9c..ca8d481e5 100644 +--- a/ldap/servers/slapd/log.c ++++ b/ldap/servers/slapd/log.c +@@ -135,6 +135,7 @@ static void vslapd_log_emergency_error(LOGFD fp, const char *msg, int locked); + static int get_syslog_loglevel(int loglevel); + static void log_external_libs_debug_openldap_print(char *buffer); + static int log__fix_rotationinfof(char *pathname); ++static int log__validate_rotated_logname(const char *timestamp_str, PRBool *is_compressed); + + static int + get_syslog_loglevel(int loglevel) +@@ -410,7 +411,7 @@ g_log_init() + loginfo.log_security_fdes = NULL; + loginfo.log_security_file = NULL; + loginfo.log_securityinfo_file = NULL; +- loginfo.log_numof_access_logs = 1; ++ loginfo.log_numof_security_logs = 1; + loginfo.log_security_logchain = NULL; + loginfo.log_security_buffer = log_create_buffer(LOG_BUFFER_MAXSIZE); + loginfo.log_security_compress = cfg->securitylog_compress; +@@ -3311,7 +3312,7 @@ log__open_accesslogfile(int logfile_state, int locked) + } + } else if (loginfo.log_access_compress) { + if (compress_log_file(newfile, loginfo.log_access_mode) != 0) { +- slapi_log_err(SLAPI_LOG_ERR, "log__open_auditfaillogfile", ++ slapi_log_err(SLAPI_LOG_ERR, "log__open_accesslogfile", + "failed to compress rotated access log (%s)\n", + newfile); + } else { +@@ -4710,6 +4711,50 @@ log__delete_rotated_logs() + loginfo.log_error_logchain = NULL; + } + ++/* ++ * log__validate_rotated_logname ++ * ++ * Validates that a log filename timestamp suffix matches the expected format: ++ * YYYYMMDD-HHMMSS (15 chars) or YYYYMMDD-HHMMSS.gz (18 chars) for compressed files. ++ * Uses regex pattern: ^[0-9]{8}-[0-9]{6}(\.gz)?$ ++ * ++ * \param timestamp_str The timestamp portion of the log filename (after the first '.') ++ * \param is_compressed Output parameter set to PR_TRUE if the file has .gz suffix ++ * \return 1 if valid, 0 if invalid ++ */ ++static int ++log__validate_rotated_logname(const char *timestamp_str, PRBool *is_compressed) ++{ ++ Slapi_Regex *re = NULL; ++ char *re_error = NULL; ++ int rc = 0; ++ ++ /* Match YYYYMMDD-HHMMSS with optional .gz suffix */ ++ static const char *pattern = "^[0-9]{8}-[0-9]{6}(\\.gz)?$"; ++ ++ *is_compressed = PR_FALSE; ++ ++ re = slapi_re_comp(pattern, &re_error); ++ if (re == NULL) { ++ slapi_log_err(SLAPI_LOG_ERR, "log__validate_rotated_logname", ++ "Failed to compile regex: %s\n", re_error ? re_error : "unknown error"); ++ slapi_ch_free_string(&re_error); ++ return 0; ++ } ++ ++ rc = slapi_re_exec_nt(re, timestamp_str); ++ if (rc == 1) { ++ /* Check if compressed by looking for .gz suffix */ ++ size_t len = strlen(timestamp_str); ++ if (len >= 3 && strcmp(timestamp_str + len - 3, ".gz") == 0) { ++ *is_compressed = PR_TRUE; ++ } ++ } ++ ++ slapi_re_free(re); ++ return rc == 1 ? 1 : 0; ++} ++ + #define ERRORSLOG 1 + #define ACCESSLOG 2 + #define AUDITLOG 3 +@@ -4792,31 +4837,19 @@ log__fix_rotationinfof(char *pathname) + } + } else if (0 == strncmp(log_type, dirent->name, strlen(log_type)) && + (p = strchr(dirent->name, '.')) != NULL && +- NULL != strchr(p, '-')) /* e.g., errors.20051123-165135 */ ++ NULL != strchr(p, '-')) /* e.g., errors.20051123-165135 or errors.20051123-165135.gz */ + { + struct logfileinfo *logp; +- char *q; +- int ignoreit = 0; +- +- for (q = ++p; q && *q; q++) { +- if (*q != '-' && +- *q != '.' && /* .gz */ +- *q != 'g' && +- *q != 'z' && +- !isdigit(*q)) +- { +- ignoreit = 1; +- } +- } +- if (ignoreit || (q - p != 15)) { ++ PRBool is_compressed = PR_FALSE; ++ ++ /* Skip the '.' to get the timestamp portion */ ++ p++; ++ if (!log__validate_rotated_logname(p, &is_compressed)) { + continue; + } + logp = (struct logfileinfo *)slapi_ch_malloc(sizeof(struct logfileinfo)); + logp->l_ctime = log_reverse_convert_time(p); +- logp->l_compressed = PR_FALSE; +- if (strcmp(p + strlen(p) - 3, ".gz") == 0) { +- logp->l_compressed = PR_TRUE; +- } ++ logp->l_compressed = is_compressed; + PR_snprintf(rotated_log, rotated_log_len, "%s/%s", + logsdir, dirent->name); + +@@ -4982,23 +5015,15 @@ log__check_prevlogs(FILE *fp, char *pathname) + for (dirent = PR_ReadDir(dirptr, dirflags); dirent; + dirent = PR_ReadDir(dirptr, dirflags)) { + if (0 == strncmp(log_type, dirent->name, strlen(log_type)) && +- (p = strrchr(dirent->name, '.')) != NULL && +- NULL != strchr(p, '-')) { /* e.g., errors.20051123-165135 */ +- char *q; +- int ignoreit = 0; +- +- for (q = ++p; q && *q; q++) { +- if (*q != '-' && +- *q != '.' && /* .gz */ +- *q != 'g' && +- *q != 'z' && +- !isdigit(*q)) +- { +- ignoreit = 1; +- } +- } +- if (ignoreit || (q - p != 15)) ++ (p = strchr(dirent->name, '.')) != NULL && ++ NULL != strchr(p, '-')) { /* e.g., errors.20051123-165135 or errors.20051123-165135.gz */ ++ PRBool is_compressed = PR_FALSE; ++ ++ /* Skip the '.' to get the timestamp portion */ ++ p++; ++ if (!log__validate_rotated_logname(p, &is_compressed)) { + continue; ++ } + + fseek(fp, 0, SEEK_SET); + buf[BUFSIZ - 1] = '\0'; +-- +2.52.0 + diff --git a/389-ds-base.spec b/389-ds-base.spec index d80861b..bbdfe78 100644 --- a/389-ds-base.spec +++ b/389-ds-base.spec @@ -47,7 +47,7 @@ ExcludeArch: i686 Summary: 389 Directory Server (base) Name: 389-ds-base Version: 2.8.0 -Release: 2%{?dist} +Release: 3%{?dist} License: GPL-3.0-or-later WITH GPL-3.0-389-ds-base-exception AND (Apache-2.0 OR Apache-2.0 WITH LLVM-exception OR MIT) AND (Apache-2.0 OR LGPL-2.1-or-later OR MIT) AND (Apache-2.0 OR MIT) AND (CC-BY-4.0 AND MIT) AND (MIT OR Apache-2.0) AND Unicode-3.0 AND (MIT OR CC0-1.0) AND (MIT OR Unlicense) AND 0BSD AND Apache-2.0 AND BSD-2-Clause AND BSD-3-Clause AND ISC AND MIT AND MIT AND ISC AND MPL-2.0 AND PSF-2.0 AND Zlib URL: https://www.port389.org Conflicts: selinux-policy-base < 3.9.8 @@ -471,7 +471,21 @@ Patch: 0002-Issue-7096-During-replication-online-total-init-the-.patc Patch: 0003-Issue-Revise-paged-result-search-locking.patch Patch: 0004-Issue-7172-Index-ordering-mismatch-after-upgrade-717.patch Patch: 0005-Issue-7172-2nd-Index-ordering-mismatch-after-upgrade.patch - +Patch: 0006-Issue-7166-db_config_set-asserts-because-of-dynamic-.patch +Patch: 0007-Issue-7224-CI-Test-Simplify-test_reserve_descriptor_.patch +Patch: 0008-Issue-7189-DSBLE0007-generates-incorrect-remediation.patch +Patch: 0009-Issue-7027-2nd-389-ds-base-OpenScanHub-Leaks-Detecte.patch +Patch: 0010-Issue-7223-Revert-index-scan-limits-for-system-index.patch +Patch: 0011-Issue-7223-Add-upgrade-function-to-remove-nsIndexIDL.patch +Patch: 0012-Issue-7223-Detect-and-log-index-ordering-mismatch-du.patch +Patch: 0013-Issue-7223-Add-dsctl-index-check-command-for-offline.patch +Patch: 0014-Issue-7096-2nd-During-replication-online-total-init-.patch +Patch: 0015-Issue-7076-6992-6784-6214-Fix-CI-test-failures-7077.patch +Patch: 0016-Issue-7076-Fix-revert_cache-never-called-in-modrdn-7.patch +Patch: 0017-Issue-6947-Fix-health_system_indexes_test.py.patch +Patch: 0018-Issue-7121-2nd-LeakSanitizer-various-leaks-during-re.patch +Patch: 0019-Issue-7150-Compressed-access-log-rotations-skipped-a.patch + %description 389 Directory Server is an LDAPv3 compliant server. The base package includes the LDAP server and command line utilities for server administration. @@ -748,40 +762,42 @@ fi # Reload our sysctl before we restart (if we can) sysctl --system &> $output; true -# Gather the running instances so we can restart them +# Gather running instances, stop them, run index-check, then restart instbase="%{_sysconfdir}/%{pkgname}" +instances="" ninst=0 -for dir in $instbase/slapd-* ; do - echo dir = $dir >> $output 2>&1 || : + +for dir in "$instbase"/slapd-* ; do + echo "dir = $dir" >> "$output" 2>&1 || : if [ ! -d "$dir" ] ; then continue ; fi case "$dir" in *.removed) continue ;; esac - basename=`basename $dir` - inst="%{pkgname}@`echo $basename | sed -e 's/slapd-//g'`" - echo found instance $inst - getting status >> $output 2>&1 || : - if /bin/systemctl -q is-active $inst ; then - echo instance $inst is running >> $output 2>&1 || : + basename=$(basename "$dir") + inst="%{pkgname}@${basename#slapd-}" + inst_name="${basename#slapd-}" + echo "found instance $inst - getting status" >> "$output" 2>&1 || : + if /bin/systemctl -q is-active "$inst" ; then + echo "instance $inst is running - stopping for upgrade" >> "$output" 2>&1 || : instances="$instances $inst" + /bin/systemctl stop "$inst" >> "$output" 2>&1 || : else - echo instance $inst is not running >> $output 2>&1 || : + echo "instance $inst is not running" >> "$output" 2>&1 || : fi - ninst=`expr $ninst + 1` + # Run index-check on all instances (running or not) + # This fixes index ordering mismatches from older versions + dsctl "$inst_name" index-check --fix >> "$output2" 2>&1 || : + ninst=$((ninst + 1)) done + if [ $ninst -eq 0 ] ; then - echo no instances to upgrade >> $output 2>&1 || : - exit 0 # have no instances to upgrade - just skip the rest -else - # restart running instances - echo shutting down all instances . . . >> $output 2>&1 || : - for inst in $instances ; do - echo stopping instance $inst >> $output 2>&1 || : - /bin/systemctl stop $inst >> $output 2>&1 || : - done - for inst in $instances ; do - echo starting instance $inst >> $output 2>&1 || : - /bin/systemctl start $inst >> $output 2>&1 || : - done + echo "no instances to upgrade" >> "$output" 2>&1 || : + exit 0 fi +# Restart previously running instances +for inst in $instances ; do + echo "starting instance $inst" >> "$output" 2>&1 || : + /bin/systemctl start "$inst" >> "$output" 2>&1 || : +done %preun if [ $1 -eq 0 ]; then # Final removal @@ -921,6 +937,15 @@ exit 0 %endif %changelog +* Thu Feb 12 2026 Viktor Ashirov - 2.8.0-3 +- Resolves: RHEL-117050 - Replication online reinitialization of a large database gets stalled. [rhel-9] +- Resolves: RHEL-123244 - Attribute uniqueness is not enforced upon modrdn operation [rhel-9] +- Resolves: RHEL-123279 - The new ipahealthcheck test ipahealthcheck.ds.backends.BackendsCheck raises CRITICAL issue [rhel-9] +- Resolves: RHEL-140275 - ipa-healthcheck is complaining about missing or incorrectly configured system indexes. [rhel-9] +- Resolves: RHEL-142980 - Scalability issue of replication online initialization with large database [rhel-9] +- Resolves: RHEL-146899 - memory corruption in alias entry plugin [rhel-9] +- Resolves: RHEL-147212 - Access logs are not getting deleted as configured. [rhel-9] + * Mon Jan 12 2026 Viktor Ashirov - 2.8.0-2 - Resolves: RHEL-140089 - Upgrading IDM to latest version: 389-ds-base and ipa-server breaks replication [rhel-9]