diff --git a/SOURCES/0076-Issue-7071-search-filter-cn-dn-groups-no-longer-retu.patch b/SOURCES/0076-Issue-7071-search-filter-cn-dn-groups-no-longer-retu.patch new file mode 100644 index 0000000..6c9f249 --- /dev/null +++ b/SOURCES/0076-Issue-7071-search-filter-cn-dn-groups-no-longer-retu.patch @@ -0,0 +1,91 @@ +From c160ef5a53f0ae9725790ebcd2667dcda2089f14 Mon Sep 17 00:00:00 2001 +From: Mark Reynolds +Date: Tue, 28 Oct 2025 10:49:18 -0400 +Subject: [PATCH 1/2] Issue 7071 - search filter (&(cn:dn:=groups)) no longer + returns results + +Description: + +When processing an "and" filter and it only contains one filter component then +the logic in the code breaks down and the filter is seen as not matching. + +The logic breaks down because we are not setting "nomatch" after the access +check is successful. If there are two components then it works fine +because we do the access check on the first filter component and set that +the access check was done(access_check_done), but "nomatch" is not set yet. +So when the next filter component is checked for access we see that the access +check was done and then we set "nomatch". + +To recap we always need to set "nomatch" when the access check is successful +in order to handle the case where an "and" fitler only has one component. + +Relates: https://github.com/389ds/389-ds-base/issues/7071 + +Reviewed by: spichugi(Thanks!) +--- + .../tests/suites/filter/complex_filters_test.py | 17 ++++++++++++++--- + ldap/servers/slapd/filterentry.c | 7 ++++++- + 2 files changed, 20 insertions(+), 4 deletions(-) + +diff --git a/dirsrvtests/tests/suites/filter/complex_filters_test.py b/dirsrvtests/tests/suites/filter/complex_filters_test.py +index 62be27381..3180ab229 100644 +--- a/dirsrvtests/tests/suites/filter/complex_filters_test.py ++++ b/dirsrvtests/tests/suites/filter/complex_filters_test.py +@@ -25,8 +25,15 @@ AND_FILTERS = [("(&(uid=uid1)(sn=last1)(givenname=first1))", 1), + ("(&(uid=*)(&(sn=last3)(givenname=*)))", 1), + ("(&(uid=uid5)(&(&(sn=*))(&(givenname=*))))", 1), + ("(&(objectclass=*)(uid=*)(sn=last*))", 5), +- ("(&(objectclass=*)(uid=*)(sn=last1))", 1)] +- ++ ("(&(objectclass=*)(uid=*)(sn=last1))", 1), ++ ("(&(sn:dn:=last1))", 1), ++ ("(&(sn:dn:=last1)(givenname:dn:=first1))", 1), ++ ("(&(sn:dn:=last1)(givenname=first1))", 1), ++ ("(&(sn=last1)(givenname=first1)(uid:dn:=uid1))", 1), ++ ("(&(uid:dn:=uid1))", 1), ++ ("(&(uid:dn:=uid1)(cn:dn:=full1))", 1), ++ ("(&(uid:dn:=uid1)(givenname:dn:=first1))", 1), ++ ("(&(uid:dn:=uid1)(givenname:dn:=first1)(cn:dn:=full1))", 1)] + OR_FILTERS = [("(|(uid=uid1)(sn=last1)(givenname=first1))", 1), + ("(|(uid=uid1)(|(sn=last1)(givenname=first1)))", 1), + ("(|(uid=uid1)(|(|(sn=last1))(|(givenname=first1))))", 1), +@@ -51,7 +58,11 @@ ZERO_AND_FILTERS = [("(&(uid=uid1)(sn=last1)(givenname=NULL))", 0), + ("(&(uid=uid1)(&(sn=last1)(givenname=NULL)))", 0), + ("(&(uid=uid1)(&(&(sn=last1))(&(givenname=NULL))))", 0), + ("(&(uid=uid1)(&(&(sn=last1))(&(givenname=NULL)(sn=*)))(|(sn=NULL)))", 0), +- ("(&(uid=uid1)(&(&(sn=last*))(&(givenname=first*)))(&(sn=NULL)))", 0)] ++ ("(&(uid=uid1)(&(&(sn=last*))(&(givenname=first*)))(&(sn=NULL)))", 0), ++ ("(&(uid:dn:=not_uid))", 0), ++ ("(&(uid:dn:=not_uid)(cn:dn:=full1))", 0), ++ ("(&(uid:dn:=uid1)(givenname:dn:=not_first1))", 0), ++ ("(&(uid:dn:=uid1)(givenname:dn:=first1)(cn:dn:=not_full1))", 0)] + + ZERO_OR_FILTERS = [("(|(uid=NULL)(sn=NULL)(givenname=NULL))", 0), + ("(|(uid=NULL)(|(sn=NULL)(givenname=NULL)))", 0), +diff --git a/ldap/servers/slapd/filterentry.c b/ldap/servers/slapd/filterentry.c +index cae5c7edc..1c513fe62 100644 +--- a/ldap/servers/slapd/filterentry.c ++++ b/ldap/servers/slapd/filterentry.c +@@ -1011,13 +1011,18 @@ vattr_test_filter_list_and( + nomatch = -1; + break; + } else { ++ /* We have a match, but we need to check access */ + if (!verify_access || (*access_check_done)) { + nomatch = 0; + } else { + /* check access */ + rc = slapi_vattr_filter_test_ext_internal(pb, e, f, verify_access, 1, access_check_done); +- if (rc) ++ if (rc) { + undefined = rc; ++ } else { ++ /* Access is good so mark this as a match */ ++ nomatch = 0; ++ } + } + } + } +-- +2.52.0 + diff --git a/SOURCES/0077-Issue-7189-DSBLE0007-generates-incorrect-remediation.patch b/SOURCES/0077-Issue-7189-DSBLE0007-generates-incorrect-remediation.patch new file mode 100644 index 0000000..3480cfd --- /dev/null +++ b/SOURCES/0077-Issue-7189-DSBLE0007-generates-incorrect-remediation.patch @@ -0,0 +1,235 @@ +From 05ce84ae76e0e71381bcc7b8491a74e7edcd888f Mon Sep 17 00:00:00 2001 +From: Viktor Ashirov +Date: Tue, 20 Jan 2026 09:52:47 +0100 +Subject: [PATCH 2/2] Issue 7189 - DSBLE0007 generates incorrect remediation + commands for scan limits + +Bug Description: + +The generated dsconf commands for fixing missing system indexes had two issues: + +1. The --add-scanlimit value was not quoted, causing the shell to interpret + "limit=5000 type=eq flags=AND" as multiple arguments instead of a single + value, resulting in "unrecognized arguments: type=eq flags=AND" error. + +2. When both matching rule and scanlimit were missing, two separate commands + were generated where the second would fail because the matching rule was + already added by the first command. + +Fix Description: + +1. Quote the scanlimit value in all remediation commands + +2. Combine matching rule and scanlimit fixes into a single command when + both are missing for the same index instead of expected_scanlimit) + +Fixes: https://github.com/389ds/389-ds-base/issues/7189 + +Reviewed by: @progier389, @droideck (Thanks!) +--- + .../healthcheck/health_system_indexes_test.py | 126 ++++++++++++++++++ + src/lib389/lib389/backend.py | 39 +++--- + 2 files changed, 147 insertions(+), 18 deletions(-) + +diff --git a/dirsrvtests/tests/suites/healthcheck/health_system_indexes_test.py b/dirsrvtests/tests/suites/healthcheck/health_system_indexes_test.py +index 1700ba207..f25de4214 100644 +--- a/dirsrvtests/tests/suites/healthcheck/health_system_indexes_test.py ++++ b/dirsrvtests/tests/suites/healthcheck/health_system_indexes_test.py +@@ -406,6 +406,132 @@ def test_retrocl_plugin_missing_matching_rule(topology_st, retrocl_plugin_enable + run_healthcheck_and_flush_log(topology_st, standalone, json=True, searched_code=JSON_OUTPUT) + + ++def test_missing_scanlimit(topology_st, log_buffering_enabled): ++ """Check if healthcheck returns DSBLE0007 code when parentId index is missing scanlimit ++ ++ :id: 40e1bf6a-2397-459b-bdf3-f787ca118b86 ++ :setup: Standalone instance ++ :steps: ++ 1. Create DS instance ++ 2. Remove nsIndexIDListScanLimit from parentId index ++ 3. Use healthcheck without --json option ++ 4. Use healthcheck with --json option ++ 5. Verify the remediation command has properly quoted scanlimit ++ 6. Re-add the scanlimit ++ 7. Use healthcheck without --json option ++ 8. Use healthcheck with --json option ++ :expectedresults: ++ 1. Success ++ 2. Success ++ 3. healthcheck reports DSBLE0007 code and related details ++ 4. healthcheck reports DSBLE0007 code and related details ++ 5. The scanlimit value is quoted in the remediation command ++ 6. Success ++ 7. healthcheck reports no issues found ++ 8. healthcheck reports no issues found ++ """ ++ ++ RET_CODE = "DSBLE0007" ++ PARENTID_DN = "cn=parentid,cn=index,cn=userroot,cn=ldbm database,cn=plugins,cn=config" ++ SCANLIMIT_VALUE = "limit=5000 type=eq flags=AND" ++ ++ standalone = topology_st.standalone ++ ++ log.info("Remove nsIndexIDListScanLimit from parentId index") ++ parentid_index = Index(standalone, PARENTID_DN) ++ parentid_index.remove("nsIndexIDListScanLimit", SCANLIMIT_VALUE) ++ ++ run_healthcheck_and_flush_log(topology_st, standalone, json=False, searched_code=RET_CODE) ++ ++ # Verify the remediation command has properly quoted scanlimit ++ args = FakeArgs() ++ args.instance = standalone.serverid ++ args.verbose = standalone.verbose ++ args.list_errors = False ++ args.list_checks = False ++ args.exclude_check = [] ++ args.check = ["backends"] ++ args.dry_run = False ++ args.json = False ++ health_check_run(standalone, topology_st.logcap.log, args) ++ # Check that the scanlimit is quoted in the output ++ assert topology_st.logcap.contains('--add-scanlimit "limit=5000 type=eq flags=AND"') ++ log.info("Verified scanlimit is properly quoted in remediation command") ++ topology_st.logcap.flush() ++ ++ run_healthcheck_and_flush_log(topology_st, standalone, json=True, searched_code=RET_CODE) ++ ++ log.info("Re-add the nsIndexIDListScanLimit") ++ parentid_index = Index(standalone, PARENTID_DN) ++ parentid_index.add("nsIndexIDListScanLimit", SCANLIMIT_VALUE) ++ ++ run_healthcheck_and_flush_log(topology_st, standalone, json=False, searched_code=CMD_OUTPUT) ++ run_healthcheck_and_flush_log(topology_st, standalone, json=True, searched_code=JSON_OUTPUT) ++ ++ ++def test_missing_matching_rule_and_scanlimit(topology_st, log_buffering_enabled): ++ """Check if healthcheck generates a single combined command when both matching rule and scanlimit are missing ++ ++ :id: af8214ad-5e4c-422a-8f74-3e99227551df ++ :setup: Standalone instance ++ :steps: ++ 1. Create DS instance ++ 2. Remove both integerOrderingMatch and nsIndexIDListScanLimit from parentId index ++ 3. Use healthcheck and verify a single combined command is generated ++ 4. Re-add the matching rule and scanlimit ++ 5. Use healthcheck without --json option ++ 6. Use healthcheck with --json option ++ :expectedresults: ++ 1. Success ++ 2. Success ++ 3. healthcheck reports DSBLE0007 and generates a single command with both --add-mr and --add-scanlimit ++ 4. Success ++ 5. healthcheck reports no issues found ++ 6. healthcheck reports no issues found ++ """ ++ ++ RET_CODE = "DSBLE0007" ++ PARENTID_DN = "cn=parentid,cn=index,cn=userroot,cn=ldbm database,cn=plugins,cn=config" ++ SCANLIMIT_VALUE = "limit=5000 type=eq flags=AND" ++ ++ standalone = topology_st.standalone ++ ++ log.info("Remove both integerOrderingMatch and nsIndexIDListScanLimit from parentId index") ++ parentid_index = Index(standalone, PARENTID_DN) ++ parentid_index.remove("nsMatchingRule", "integerOrderingMatch") ++ parentid_index.remove("nsIndexIDListScanLimit", SCANLIMIT_VALUE) ++ ++ # Run healthcheck and verify combined command ++ args = FakeArgs() ++ args.instance = standalone.serverid ++ args.verbose = standalone.verbose ++ args.list_errors = False ++ args.list_checks = False ++ args.exclude_check = [] ++ args.check = ["backends"] ++ args.dry_run = False ++ args.json = False ++ health_check_run(standalone, topology_st.logcap.log, args) ++ ++ # Verify DSBLE0007 is reported ++ assert topology_st.logcap.contains(RET_CODE) ++ log.info("healthcheck returned code: %s" % RET_CODE) ++ ++ # Verify a single combined command is generated with both --add-mr and --add-scanlimit ++ assert topology_st.logcap.contains('--add-mr integerOrderingMatch --add-scanlimit "limit=5000 type=eq flags=AND"') ++ log.info("Verified combined command with both --add-mr and --add-scanlimit") ++ ++ topology_st.logcap.flush() ++ ++ log.info("Re-add the integerOrderingMatch matching rule and scanlimit") ++ parentid_index = Index(standalone, PARENTID_DN) ++ parentid_index.add("nsMatchingRule", "integerOrderingMatch") ++ parentid_index.add("nsIndexIDListScanLimit", SCANLIMIT_VALUE) ++ ++ run_healthcheck_and_flush_log(topology_st, standalone, json=False, searched_code=CMD_OUTPUT) ++ run_healthcheck_and_flush_log(topology_st, standalone, json=True, searched_code=JSON_OUTPUT) ++ ++ + def test_multiple_missing_indexes(topology_st, log_buffering_enabled): + """Check if healthcheck returns DSBLE0007 code when multiple system indexes are missing + +diff --git a/src/lib389/lib389/backend.py b/src/lib389/lib389/backend.py +index 14b64d1d3..3cea0df36 100644 +--- a/src/lib389/lib389/backend.py ++++ b/src/lib389/lib389/backend.py +@@ -602,7 +602,7 @@ class Backend(DSLdapObject): + if expected_config.get('matching_rule'): + cmd += f" --matching-rule {expected_config['matching_rule']}" + if expected_config.get('scanlimit'): +- cmd += f" --add-scanlimit {expected_config['scanlimit']}" ++ cmd += f" --add-scanlimit \"{expected_config['scanlimit']}\"" + remediation_commands.append(cmd) + reindex_attrs.add(attr_name) # New index needs reindexing + else: +@@ -624,28 +624,31 @@ class Backend(DSLdapObject): + remediation_commands.append(cmd) + reindex_attrs.add(attr_name) + +- # Check matching rules ++ # Check matching rules and scanlimit together to generate a single combined command + expected_mr = expected_config.get('matching_rule') ++ expected_scanlimit = expected_config.get('scanlimit') ++ ++ missing_mr = False + if expected_mr: + actual_mrs_lower = [mr.lower() for mr in actual_mrs] + if expected_mr.lower() not in actual_mrs_lower: + discrepancies.append(f"Index {attr_name} missing matching rule: {expected_mr}") +- # Add the missing matching rule +- cmd = f"dsconf YOUR_INSTANCE backend index set {bename} --attr {attr_name} --add-mr {expected_mr}" +- remediation_commands.append(cmd) +- reindex_attrs.add(attr_name) +- +- # Check fine grain definitions for parentid ONLY +- expected_scanlimit = expected_config.get('scanlimit') +- if (attr_name.lower() == "parentid") and expected_scanlimit and (len(actual_scanlimit) == 0): +- discrepancies.append(f"Index {attr_name} missing fine grain definition of IDs limit: {expected_mr}") +- # Add the missing scanlimit +- if expected_mr: +- cmd = f"dsconf YOUR_INSTANCE backend index set {bename} --attr {attr_name} --add-mr {expected_mr} --add-scanlimit {expected_scanlimit}" +- else: +- cmd = f"dsconf YOUR_INSTANCE backend index set {bename} --attr {attr_name} --add-scanlimit {expected_scanlimit}" +- remediation_commands.append(cmd) +- reindex_attrs.add(attr_name) ++ missing_mr = True ++ ++ missing_scanlimit = False ++ if expected_scanlimit and (len(actual_scanlimit) == 0): ++ discrepancies.append(f"Index {attr_name} missing fine grain definition of IDs limit: {expected_scanlimit}") ++ missing_scanlimit = True ++ ++ # Generate a single combined command for all missing items ++ if missing_mr or missing_scanlimit: ++ cmd = f"dsconf YOUR_INSTANCE backend index set {bename} --attr {attr_name}" ++ if missing_mr: ++ cmd += f" --add-mr {expected_mr}" ++ if missing_scanlimit: ++ cmd += f" --add-scanlimit \"{expected_scanlimit}\"" ++ remediation_commands.append(cmd) ++ reindex_attrs.add(attr_name) + + except Exception as e: + self._log.debug(f"_lint_system_indexes - Error checking index {attr_name}: {e}") +-- +2.52.0 + diff --git a/SOURCES/0078-Issue-7223-Revert-index-scan-limits-for-system-index.patch b/SOURCES/0078-Issue-7223-Revert-index-scan-limits-for-system-index.patch new file mode 100644 index 0000000..342bc91 --- /dev/null +++ b/SOURCES/0078-Issue-7223-Revert-index-scan-limits-for-system-index.patch @@ -0,0 +1,789 @@ +From 850cb2ae142b1dc31081387e8e997699668e70f4 Mon Sep 17 00:00:00 2001 +From: Viktor Ashirov +Date: Thu, 5 Feb 2026 12:17:06 +0100 +Subject: [PATCH] Issue 7223 - Revert index scan limits for system indexes + +This reverts changes introduced by the following commits: +c6f458b42 Issue 7189 - DSBLE0007 generates incorrect remediation commands for scan limits +8b6b3a9f9 Issue 6966 - On large DB, unlimited IDL scan limit reduce the SRCH performance + +Relates: https://github.com/389ds/389-ds-base/issues/7223 + +Reviewed by: @progier389, @tbordaz (Thanks!) +--- + .../tests/suites/config/config_test.py | 27 +--- + .../healthcheck/health_system_indexes_test.py | 135 +----------------- + .../paged_results/paged_results_test.py | 25 +--- + ldap/servers/slapd/back-ldbm/back-ldbm.h | 1 - + ldap/servers/slapd/back-ldbm/index.c | 2 - + ldap/servers/slapd/back-ldbm/instance.c | 108 ++------------ + ldap/servers/slapd/back-ldbm/ldbm_config.c | 30 ---- + ldap/servers/slapd/back-ldbm/ldbm_config.h | 1 - + .../slapd/back-ldbm/ldbm_index_config.c | 8 -- + src/lib389/lib389/backend.py | 50 ++----- + src/lib389/lib389/cli_conf/backend.py | 20 --- + 11 files changed, 39 insertions(+), 368 deletions(-) + +diff --git a/dirsrvtests/tests/suites/config/config_test.py b/dirsrvtests/tests/suites/config/config_test.py +index 430176602..bbe13d248 100644 +--- a/dirsrvtests/tests/suites/config/config_test.py ++++ b/dirsrvtests/tests/suites/config/config_test.py +@@ -514,19 +514,17 @@ def test_ndn_cache_enabled(topo): + topo.standalone.config.set('nsslapd-ndn-cache-max-size', 'invalid_value') + + +-def test_require_index(topo, request): ++def test_require_index(topo): + """Validate that unindexed searches are rejected + + :id: fb6e31f2-acc2-4e75-a195-5c356faeb803 + :setup: Standalone instance + :steps: + 1. Set "nsslapd-require-index" to "on" +- 2. ancestorid/idlscanlimit to 100 +- 3. Test an unindexed search is rejected ++ 2. Test an unindexed search is rejected + :expectedresults: + 1. Success + 2. Success +- 3. Success + """ + + # Set the config +@@ -537,10 +535,6 @@ def test_require_index(topo, request): + + db_cfg = DatabaseConfig(topo.standalone) + db_cfg.set([('nsslapd-idlistscanlimit', '100')]) +- backend = Backends(topo.standalone).get_backend(DEFAULT_SUFFIX) +- ancestorid_index = backend.get_index('ancestorid') +- ancestorid_index.replace("nsIndexIDListScanLimit", ensure_bytes("limit=100 type=eq flags=AND")) +- topo.standalone.restart() + + users = UserAccounts(topo.standalone, DEFAULT_SUFFIX) + for i in range(101): +@@ -551,15 +545,10 @@ def test_require_index(topo, request): + with pytest.raises(ldap.UNWILLING_TO_PERFORM): + raw_objects.filter("(description=test*)") + +- def fin(): +- ancestorid_index.replace("nsIndexIDListScanLimit", ensure_bytes("limit=5000 type=eq flags=AND")) +- +- request.addfinalizer(fin) +- + + + @pytest.mark.skipif(ds_is_older('1.4.2'), reason="The config setting only exists in 1.4.2 and higher") +-def test_require_internal_index(topo, request): ++def test_require_internal_index(topo): + """Ensure internal operations require indexed attributes + + :id: 22b94f30-59e3-4f27-89a1-c4f4be036f7f +@@ -591,10 +580,6 @@ def test_require_internal_index(topo, request): + # Create a bunch of users + db_cfg = DatabaseConfig(topo.standalone) + db_cfg.set([('nsslapd-idlistscanlimit', '100')]) +- backend = Backends(topo.standalone).get_backend(DEFAULT_SUFFIX) +- ancestorid_index = backend.get_index('ancestorid') +- ancestorid_index.replace("nsIndexIDListScanLimit", ensure_bytes("limit=100 type=eq flags=AND")) +- topo.standalone.restart() + users = UserAccounts(topo.standalone, DEFAULT_SUFFIX) + for i in range(102, 202): + users.create_test_user(uid=i) +@@ -619,12 +604,6 @@ def test_require_internal_index(topo, request): + with pytest.raises(ldap.UNWILLING_TO_PERFORM): + user.delete() + +- def fin(): +- ancestorid_index.replace("nsIndexIDListScanLimit", ensure_bytes("limit=5000 type=eq flags=AND")) +- +- request.addfinalizer(fin) +- +- + + if __name__ == '__main__': + # Run isolated +diff --git a/dirsrvtests/tests/suites/healthcheck/health_system_indexes_test.py b/dirsrvtests/tests/suites/healthcheck/health_system_indexes_test.py +index f25de4214..842f7e8dd 100644 +--- a/dirsrvtests/tests/suites/healthcheck/health_system_indexes_test.py ++++ b/dirsrvtests/tests/suites/healthcheck/health_system_indexes_test.py +@@ -172,8 +172,7 @@ def test_missing_parentid(topology_st, log_buffering_enabled): + + log.info("Re-add the parentId index") + backend = Backends(standalone).get("userRoot") +- backend.add_index("parentid", ["eq"], matching_rules=["integerOrderingMatch"], +- idlistscanlimit=['limit=5000 type=eq flags=AND']) ++ backend.add_index("parentid", ["eq"], matching_rules=["integerOrderingMatch"]) + + run_healthcheck_and_flush_log(topology_st, standalone, json=False, searched_code=CMD_OUTPUT) + run_healthcheck_and_flush_log(topology_st, standalone, json=True, searched_code=JSON_OUTPUT) +@@ -261,8 +260,7 @@ def test_usn_plugin_missing_entryusn(topology_st, usn_plugin_enabled, log_buffer + + log.info("Re-add the entryusn index") + backend = Backends(standalone).get("userRoot") +- backend.add_index("entryusn", ["eq"], matching_rules=["integerOrderingMatch"], +- idlistscanlimit=['limit=5000 type=eq flags=AND']) ++ backend.add_index("entryusn", ["eq"], matching_rules=["integerOrderingMatch"]) + + run_healthcheck_and_flush_log(topology_st, standalone, json=False, searched_code=CMD_OUTPUT) + run_healthcheck_and_flush_log(topology_st, standalone, json=True, searched_code=JSON_OUTPUT) +@@ -406,132 +404,6 @@ def test_retrocl_plugin_missing_matching_rule(topology_st, retrocl_plugin_enable + run_healthcheck_and_flush_log(topology_st, standalone, json=True, searched_code=JSON_OUTPUT) + + +-def test_missing_scanlimit(topology_st, log_buffering_enabled): +- """Check if healthcheck returns DSBLE0007 code when parentId index is missing scanlimit +- +- :id: 40e1bf6a-2397-459b-bdf3-f787ca118b86 +- :setup: Standalone instance +- :steps: +- 1. Create DS instance +- 2. Remove nsIndexIDListScanLimit from parentId index +- 3. Use healthcheck without --json option +- 4. Use healthcheck with --json option +- 5. Verify the remediation command has properly quoted scanlimit +- 6. Re-add the scanlimit +- 7. Use healthcheck without --json option +- 8. Use healthcheck with --json option +- :expectedresults: +- 1. Success +- 2. Success +- 3. healthcheck reports DSBLE0007 code and related details +- 4. healthcheck reports DSBLE0007 code and related details +- 5. The scanlimit value is quoted in the remediation command +- 6. Success +- 7. healthcheck reports no issues found +- 8. healthcheck reports no issues found +- """ +- +- RET_CODE = "DSBLE0007" +- PARENTID_DN = "cn=parentid,cn=index,cn=userroot,cn=ldbm database,cn=plugins,cn=config" +- SCANLIMIT_VALUE = "limit=5000 type=eq flags=AND" +- +- standalone = topology_st.standalone +- +- log.info("Remove nsIndexIDListScanLimit from parentId index") +- parentid_index = Index(standalone, PARENTID_DN) +- parentid_index.remove("nsIndexIDListScanLimit", SCANLIMIT_VALUE) +- +- run_healthcheck_and_flush_log(topology_st, standalone, json=False, searched_code=RET_CODE) +- +- # Verify the remediation command has properly quoted scanlimit +- args = FakeArgs() +- args.instance = standalone.serverid +- args.verbose = standalone.verbose +- args.list_errors = False +- args.list_checks = False +- args.exclude_check = [] +- args.check = ["backends"] +- args.dry_run = False +- args.json = False +- health_check_run(standalone, topology_st.logcap.log, args) +- # Check that the scanlimit is quoted in the output +- assert topology_st.logcap.contains('--add-scanlimit "limit=5000 type=eq flags=AND"') +- log.info("Verified scanlimit is properly quoted in remediation command") +- topology_st.logcap.flush() +- +- run_healthcheck_and_flush_log(topology_st, standalone, json=True, searched_code=RET_CODE) +- +- log.info("Re-add the nsIndexIDListScanLimit") +- parentid_index = Index(standalone, PARENTID_DN) +- parentid_index.add("nsIndexIDListScanLimit", SCANLIMIT_VALUE) +- +- run_healthcheck_and_flush_log(topology_st, standalone, json=False, searched_code=CMD_OUTPUT) +- run_healthcheck_and_flush_log(topology_st, standalone, json=True, searched_code=JSON_OUTPUT) +- +- +-def test_missing_matching_rule_and_scanlimit(topology_st, log_buffering_enabled): +- """Check if healthcheck generates a single combined command when both matching rule and scanlimit are missing +- +- :id: af8214ad-5e4c-422a-8f74-3e99227551df +- :setup: Standalone instance +- :steps: +- 1. Create DS instance +- 2. Remove both integerOrderingMatch and nsIndexIDListScanLimit from parentId index +- 3. Use healthcheck and verify a single combined command is generated +- 4. Re-add the matching rule and scanlimit +- 5. Use healthcheck without --json option +- 6. Use healthcheck with --json option +- :expectedresults: +- 1. Success +- 2. Success +- 3. healthcheck reports DSBLE0007 and generates a single command with both --add-mr and --add-scanlimit +- 4. Success +- 5. healthcheck reports no issues found +- 6. healthcheck reports no issues found +- """ +- +- RET_CODE = "DSBLE0007" +- PARENTID_DN = "cn=parentid,cn=index,cn=userroot,cn=ldbm database,cn=plugins,cn=config" +- SCANLIMIT_VALUE = "limit=5000 type=eq flags=AND" +- +- standalone = topology_st.standalone +- +- log.info("Remove both integerOrderingMatch and nsIndexIDListScanLimit from parentId index") +- parentid_index = Index(standalone, PARENTID_DN) +- parentid_index.remove("nsMatchingRule", "integerOrderingMatch") +- parentid_index.remove("nsIndexIDListScanLimit", SCANLIMIT_VALUE) +- +- # Run healthcheck and verify combined command +- args = FakeArgs() +- args.instance = standalone.serverid +- args.verbose = standalone.verbose +- args.list_errors = False +- args.list_checks = False +- args.exclude_check = [] +- args.check = ["backends"] +- args.dry_run = False +- args.json = False +- health_check_run(standalone, topology_st.logcap.log, args) +- +- # Verify DSBLE0007 is reported +- assert topology_st.logcap.contains(RET_CODE) +- log.info("healthcheck returned code: %s" % RET_CODE) +- +- # Verify a single combined command is generated with both --add-mr and --add-scanlimit +- assert topology_st.logcap.contains('--add-mr integerOrderingMatch --add-scanlimit "limit=5000 type=eq flags=AND"') +- log.info("Verified combined command with both --add-mr and --add-scanlimit") +- +- topology_st.logcap.flush() +- +- log.info("Re-add the integerOrderingMatch matching rule and scanlimit") +- parentid_index = Index(standalone, PARENTID_DN) +- parentid_index.add("nsMatchingRule", "integerOrderingMatch") +- parentid_index.add("nsIndexIDListScanLimit", SCANLIMIT_VALUE) +- +- run_healthcheck_and_flush_log(topology_st, standalone, json=False, searched_code=CMD_OUTPUT) +- run_healthcheck_and_flush_log(topology_st, standalone, json=True, searched_code=JSON_OUTPUT) +- +- + def test_multiple_missing_indexes(topology_st, log_buffering_enabled): + """Check if healthcheck returns DSBLE0007 code when multiple system indexes are missing + +@@ -572,8 +444,7 @@ def test_multiple_missing_indexes(topology_st, log_buffering_enabled): + + log.info("Re-add the missing system indexes") + backend = Backends(standalone).get("userRoot") +- backend.add_index("parentid", ["eq"], matching_rules=["integerOrderingMatch"], +- idlistscanlimit=['limit=5000 type=eq flags=AND']) ++ backend.add_index("parentid", ["eq"], matching_rules=["integerOrderingMatch"]) + backend.add_index("nsuniqueid", ["eq"]) + + run_healthcheck_and_flush_log(topology_st, standalone, json=False, searched_code=CMD_OUTPUT) +diff --git a/dirsrvtests/tests/suites/paged_results/paged_results_test.py b/dirsrvtests/tests/suites/paged_results/paged_results_test.py +index 8835be8fa..1ed11c891 100644 +--- a/dirsrvtests/tests/suites/paged_results/paged_results_test.py ++++ b/dirsrvtests/tests/suites/paged_results/paged_results_test.py +@@ -317,19 +317,19 @@ def test_search_success(topology_st, create_user, page_size, users_num): + del_users(users_list) + + +-@pytest.mark.parametrize("page_size,users_num,suffix,attr_name,attr_value,expected_err, restart", [ ++@pytest.mark.parametrize("page_size,users_num,suffix,attr_name,attr_value,expected_err", [ + (50, 200, 'cn=config,%s' % DN_LDBM, 'nsslapd-idlistscanlimit', '100', +- ldap.UNWILLING_TO_PERFORM, True), ++ ldap.UNWILLING_TO_PERFORM), + (5, 15, DN_CONFIG, 'nsslapd-timelimit', '20', +- ldap.UNAVAILABLE_CRITICAL_EXTENSION, False), ++ ldap.UNAVAILABLE_CRITICAL_EXTENSION), + (21, 50, DN_CONFIG, 'nsslapd-sizelimit', '20', +- ldap.SIZELIMIT_EXCEEDED, False), ++ ldap.SIZELIMIT_EXCEEDED), + (21, 50, DN_CONFIG, 'nsslapd-pagedsizelimit', '5', +- ldap.SIZELIMIT_EXCEEDED, False), ++ ldap.SIZELIMIT_EXCEEDED), + (5, 50, 'cn=config,%s' % DN_LDBM, 'nsslapd-lookthroughlimit', '20', +- ldap.ADMINLIMIT_EXCEEDED, False)]) ++ ldap.ADMINLIMIT_EXCEEDED)]) + def test_search_limits_fail(topology_st, create_user, page_size, users_num, +- suffix, attr_name, attr_value, expected_err, restart): ++ suffix, attr_name, attr_value, expected_err): + """Verify that search with a simple paged results control + throws expected exceptoins when corresponding limits are + exceeded. +@@ -351,15 +351,6 @@ def test_search_limits_fail(topology_st, create_user, page_size, users_num, + + users_list = add_users(topology_st, users_num, DEFAULT_SUFFIX) + attr_value_bck = change_conf_attr(topology_st, suffix, attr_name, attr_value) +- ancestorid_index = None +- if attr_name == 'nsslapd-idlistscanlimit': +- backend = Backends(topology_st.standalone).get_backend(DEFAULT_SUFFIX) +- ancestorid_index = backend.get_index('ancestorid') +- ancestorid_index.replace("nsIndexIDListScanLimit", ensure_bytes("limit=100 type=eq flags=AND")) +- +- if (restart): +- log.info('Instance restarted') +- topology_st.standalone.restart() + conf_param_dict = {attr_name: attr_value} + search_flt = r'(uid=test*)' + searchreq_attrlist = ['dn', 'sn'] +@@ -412,8 +403,6 @@ def test_search_limits_fail(topology_st, create_user, page_size, users_num, + else: + break + finally: +- if ancestorid_index: +- ancestorid_index.replace("nsIndexIDListScanLimit", ensure_bytes("limit=5000 type=eq flags=AND")) + del_users(users_list) + change_conf_attr(topology_st, suffix, attr_name, attr_value_bck) + +diff --git a/ldap/servers/slapd/back-ldbm/back-ldbm.h b/ldap/servers/slapd/back-ldbm/back-ldbm.h +index cde30cedd..d17ec644b 100644 +--- a/ldap/servers/slapd/back-ldbm/back-ldbm.h ++++ b/ldap/servers/slapd/back-ldbm/back-ldbm.h +@@ -554,7 +554,6 @@ struct ldbminfo + int li_mode; + int li_lookthroughlimit; + int li_allidsthreshold; +- int li_system_allidsthreshold; + char *li_directory; + int li_reslimit_lookthrough_handle; + uint64_t li_dbcachesize; +diff --git a/ldap/servers/slapd/back-ldbm/index.c b/ldap/servers/slapd/back-ldbm/index.c +index 63f0196c1..30fa09ebb 100644 +--- a/ldap/servers/slapd/back-ldbm/index.c ++++ b/ldap/servers/slapd/back-ldbm/index.c +@@ -999,8 +999,6 @@ index_read_ext_allids( + } + if (pb) { + slapi_pblock_get(pb, SLAPI_SEARCH_IS_AND, &is_and); +- } else if (strcasecmp(type, LDBM_ANCESTORID_STR) == 0) { +- is_and = 1; + } + ai_flags = is_and ? INDEX_ALLIDS_FLAG_AND : 0; + /* the caller can pass in a value of 0 - just ignore those - but if the index +diff --git a/ldap/servers/slapd/back-ldbm/instance.c b/ldap/servers/slapd/back-ldbm/instance.c +index 29643f7e4..6098e04fc 100644 +--- a/ldap/servers/slapd/back-ldbm/instance.c ++++ b/ldap/servers/slapd/back-ldbm/instance.c +@@ -16,7 +16,7 @@ + + /* Forward declarations */ + static void ldbm_instance_destructor(void **arg); +-Slapi_Entry *ldbm_instance_init_config_entry(char *cn_val, char *v1, char *v2, char *v3, char *v4, char *mr, char *scanlimit); ++Slapi_Entry *ldbm_instance_init_config_entry(char *cn_val, char *v1, char *v2, char *v3, char *v4, char *mr); + + + /* Creates and initializes a new ldbm_instance structure. +@@ -127,7 +127,7 @@ done: + * Take a bunch of strings, and create a index config entry + */ + Slapi_Entry * +-ldbm_instance_init_config_entry(char *cn_val, char *val1, char *val2, char *val3, char *val4, char *mr, char *scanlimit) ++ldbm_instance_init_config_entry(char *cn_val, char *val1, char *val2, char *val3, char *val4, char *mr) + { + Slapi_Entry *e = slapi_entry_alloc(); + struct berval *vals[2]; +@@ -168,11 +168,6 @@ ldbm_instance_init_config_entry(char *cn_val, char *val1, char *val2, char *val3 + slapi_entry_add_values(e, "nsMatchingRule", vals); + } + +- if (scanlimit) { +- val.bv_val = scanlimit; +- val.bv_len = strlen(scanlimit); +- slapi_entry_add_values(e, "nsIndexIDListScanLimit", vals); +- } + return e; + } + +@@ -185,60 +180,8 @@ ldbm_instance_create_default_indexes(backend *be) + { + Slapi_Entry *e; + ldbm_instance *inst = (ldbm_instance *)be->be_instance_info; +- struct ldbminfo *li = (struct ldbminfo *)be->be_database->plg_private; + /* write the dse file only on the final index */ + int flags = LDBM_INSTANCE_CONFIG_DONT_WRITE; +- char *ancestorid_indexes_limit = NULL; +- char *parentid_indexes_limit = NULL; +- struct attrinfo *ai = NULL; +- int index_already_configured = 0; +- struct index_idlistsizeinfo *iter; +- int cookie; +- int limit; +- +- ainfo_get(be, (char *)LDBM_ANCESTORID_STR, &ai); +- if (ai && ai->ai_idlistinfo) { +- iter = (struct index_idlistsizeinfo *)dl_get_first(ai->ai_idlistinfo, &cookie); +- if (iter) { +- limit = iter->ai_idlistsizelimit; +- slapi_log_err(SLAPI_LOG_BACKLDBM, "ldbm_instance_create_default_indexes", +- "set ancestorid limit to %d from attribute index\n", +- limit); +- } else { +- limit = li->li_system_allidsthreshold; +- slapi_log_err(SLAPI_LOG_BACKLDBM, "ldbm_instance_create_default_indexes", +- "set ancestorid limit to %d from default (fail to read limit)\n", +- limit); +- } +- ancestorid_indexes_limit = slapi_ch_smprintf("limit=%d type=eq flags=AND", limit); +- } else { +- ancestorid_indexes_limit = slapi_ch_smprintf("limit=%d type=eq flags=AND", li->li_system_allidsthreshold); +- slapi_log_err(SLAPI_LOG_BACKLDBM, "ldbm_instance_create_default_indexes", +- "set ancestorid limit to %d from default (no attribute or limit)\n", +- li->li_system_allidsthreshold); +- } +- +- ainfo_get(be, (char *)LDBM_PARENTID_STR, &ai); +- if (ai && ai->ai_idlistinfo) { +- iter = (struct index_idlistsizeinfo *)dl_get_first(ai->ai_idlistinfo, &cookie); +- if (iter) { +- limit = iter->ai_idlistsizelimit; +- slapi_log_err(SLAPI_LOG_BACKLDBM, "ldbm_instance_create_default_indexes", +- "set parentid limit to %d from attribute index\n", +- limit); +- } else { +- limit = li->li_system_allidsthreshold; +- slapi_log_err(SLAPI_LOG_BACKLDBM, "ldbm_instance_create_default_indexes", +- "set parentid limit to %d from default (fail to read limit)\n", +- limit); +- } +- parentid_indexes_limit = slapi_ch_smprintf("limit=%d type=eq flags=AND", limit); +- } else { +- parentid_indexes_limit = slapi_ch_smprintf("limit=%d type=eq flags=AND", li->li_system_allidsthreshold); +- slapi_log_err(SLAPI_LOG_BACKLDBM, "ldbm_instance_create_default_indexes", +- "set parentid limit to %d from default (no attribute or limit)\n", +- li->li_system_allidsthreshold); +- } + + /* + * Always index (entrydn or entryrdn), parentid, objectclass, +@@ -247,59 +190,47 @@ ldbm_instance_create_default_indexes(backend *be) + * ACL routines. + */ + if (entryrdn_get_switch()) { /* subtree-rename: on */ +- e = ldbm_instance_init_config_entry(LDBM_ENTRYRDN_STR, "subtree", 0, 0, 0, 0, 0); ++ e = ldbm_instance_init_config_entry(LDBM_ENTRYRDN_STR, "subtree", 0, 0, 0, 0); + ldbm_instance_config_add_index_entry(inst, e, flags); + slapi_entry_free(e); + } else { +- e = ldbm_instance_init_config_entry(LDBM_ENTRYDN_STR, "eq", 0, 0, 0, 0, 0); ++ e = ldbm_instance_init_config_entry(LDBM_ENTRYDN_STR, "eq", 0, 0, 0, 0); + ldbm_instance_config_add_index_entry(inst, e, flags); + slapi_entry_free(e); + } + +- ainfo_get(be, (char *)LDBM_PARENTID_STR, &ai); +- /* Check if the attrinfo is actually for parentid, not a fallback to .default */ +- index_already_configured = (ai != NULL && strcmp(ai->ai_type, LDBM_PARENTID_STR) == 0); +- if (!index_already_configured) { +- e = ldbm_instance_init_config_entry(LDBM_PARENTID_STR, "eq", 0, 0, 0, "integerOrderingMatch", parentid_indexes_limit); +- ldbm_instance_config_add_index_entry(inst, e, flags); +- attr_index_config(be, "ldbm index init", 0, e, 1, 0, NULL); +- slapi_entry_free(e); +- } +- +- e = ldbm_instance_init_config_entry("objectclass", "eq", 0, 0, 0, 0, 0); ++ e = ldbm_instance_init_config_entry(LDBM_PARENTID_STR, "eq", 0, 0, 0, "integerOrderingMatch"); + ldbm_instance_config_add_index_entry(inst, e, flags); + slapi_entry_free(e); + +- e = ldbm_instance_init_config_entry("aci", "pres", 0, 0, 0, 0, 0); ++ e = ldbm_instance_init_config_entry("objectclass", "eq", 0, 0, 0, 0); + ldbm_instance_config_add_index_entry(inst, e, flags); + slapi_entry_free(e); + +- e = ldbm_instance_init_config_entry(LDBM_NUMSUBORDINATES_STR, "pres", 0, 0, 0, 0, 0); ++ e = ldbm_instance_init_config_entry("aci", "pres", 0, 0, 0, 0); + ldbm_instance_config_add_index_entry(inst, e, flags); + slapi_entry_free(e); + +-#if 0 /* don't need copiedfrom */ +- e = ldbm_instance_init_config_entry("copiedfrom","pres",0 ,0); ++ e = ldbm_instance_init_config_entry(LDBM_NUMSUBORDINATES_STR, "pres", 0, 0, 0, 0); + ldbm_instance_config_add_index_entry(inst, e, flags); + slapi_entry_free(e); +-#endif + +- e = ldbm_instance_init_config_entry(SLAPI_ATTR_UNIQUEID, "eq", 0, 0, 0, 0, 0); ++ e = ldbm_instance_init_config_entry(SLAPI_ATTR_UNIQUEID, "eq", 0, 0, 0, 0); + ldbm_instance_config_add_index_entry(inst, e, flags); + slapi_entry_free(e); + + /* For MMR, we need this attribute (to replace use of dncomp in delete). */ +- e = ldbm_instance_init_config_entry(ATTR_NSDS5_REPLCONFLICT, "eq", "pres", 0, 0, 0, 0); ++ e = ldbm_instance_init_config_entry(ATTR_NSDS5_REPLCONFLICT, "eq", "pres", 0, 0, 0); + ldbm_instance_config_add_index_entry(inst, e, flags); + slapi_entry_free(e); + + /* write the dse file only on the final index */ +- e = ldbm_instance_init_config_entry(SLAPI_ATTR_NSCP_ENTRYDN, "eq", 0, 0, 0, 0, 0); ++ e = ldbm_instance_init_config_entry(SLAPI_ATTR_NSCP_ENTRYDN, "eq", 0, 0, 0, 0); + ldbm_instance_config_add_index_entry(inst, e, flags); + slapi_entry_free(e); + + /* ldbm_instance_config_add_index_entry(inst, 2, argv); */ +- e = ldbm_instance_init_config_entry(LDBM_PSEUDO_ATTR_DEFAULT, "none", 0, 0, 0, 0, 0); ++ e = ldbm_instance_init_config_entry(LDBM_PSEUDO_ATTR_DEFAULT, "none", 0, 0, 0, 0); + attr_index_config(be, "ldbm index init", 0, e, 1, 0, NULL); + slapi_entry_free(e); + +@@ -308,20 +239,11 @@ ldbm_instance_create_default_indexes(backend *be) + * ancestorid is special, there is actually no such attr type + * but we still want to use the attr index file APIs. + */ +- ainfo_get(be, (char *)LDBM_ANCESTORID_STR, &ai); +- /* Check if the attrinfo is actually for ancestorid, not a fallback to .default */ +- index_already_configured = (ai != NULL && strcmp(ai->ai_type, LDBM_ANCESTORID_STR) == 0); +- if (!index_already_configured) { +- e = ldbm_instance_init_config_entry(LDBM_ANCESTORID_STR, "eq", 0, 0, 0, "integerOrderingMatch", ancestorid_indexes_limit); +- ldbm_instance_config_add_index_entry(inst, e, flags); +- attr_index_config(be, "ldbm index init", 0, e, 1, 0, NULL); +- slapi_entry_free(e); +- } ++ e = ldbm_instance_init_config_entry(LDBM_ANCESTORID_STR, "eq", 0, 0, 0, "integerOrderingMatch"); ++ attr_index_config(be, "ldbm index init", 0, e, 1, 0, NULL); ++ slapi_entry_free(e); + } + +- slapi_ch_free_string(&ancestorid_indexes_limit); +- slapi_ch_free_string(&parentid_indexes_limit); +- + return 0; + } + +diff --git a/ldap/servers/slapd/back-ldbm/ldbm_config.c b/ldap/servers/slapd/back-ldbm/ldbm_config.c +index f8d8f7474..b7bceabf2 100644 +--- a/ldap/servers/slapd/back-ldbm/ldbm_config.c ++++ b/ldap/servers/slapd/back-ldbm/ldbm_config.c +@@ -366,35 +366,6 @@ ldbm_config_allidsthreshold_set(void *arg, void *value, char *errorbuf __attribu + return retval; + } + +-static void * +-ldbm_config_system_allidsthreshold_get(void *arg) +-{ +- struct ldbminfo *li = (struct ldbminfo *)arg; +- +- return (void *)((uintptr_t)(li->li_system_allidsthreshold)); +-} +- +-static int +-ldbm_config_system_allidsthreshold_set(void *arg, void *value, char *errorbuf __attribute__((unused)), int phase __attribute__((unused)), int apply) +-{ +- struct ldbminfo *li = (struct ldbminfo *)arg; +- int retval = LDAP_SUCCESS; +- int val = (int)((uintptr_t)value); +- +- /* Do whatever we can to make sure the data is ok. */ +- +- /* Catch attempts to configure a stupidly low ancestorid allidsthreshold */ +- if ((val > -1) && (val < 5000)) { +- val = 5000; +- } +- +- if (apply) { +- li->li_system_allidsthreshold = val; +- } +- +- return retval; +-} +- + static void * + ldbm_config_pagedallidsthreshold_get(void *arg) + { +@@ -974,7 +945,6 @@ static config_info ldbm_config[] = { + {CONFIG_LOOKTHROUGHLIMIT, CONFIG_TYPE_INT, "5000", &ldbm_config_lookthroughlimit_get, &ldbm_config_lookthroughlimit_set, CONFIG_FLAG_ALWAYS_SHOW | CONFIG_FLAG_ALLOW_RUNNING_CHANGE}, + {CONFIG_MODE, CONFIG_TYPE_INT_OCTAL, "0600", &ldbm_config_mode_get, &ldbm_config_mode_set, CONFIG_FLAG_ALWAYS_SHOW | CONFIG_FLAG_ALLOW_RUNNING_CHANGE}, + {CONFIG_IDLISTSCANLIMIT, CONFIG_TYPE_INT, "2147483646", &ldbm_config_allidsthreshold_get, &ldbm_config_allidsthreshold_set, CONFIG_FLAG_ALWAYS_SHOW | CONFIG_FLAG_ALLOW_RUNNING_CHANGE}, +- {CONFIG_SYSTEMIDLISTSCANLIMIT, CONFIG_TYPE_INT, "5000", &ldbm_config_system_allidsthreshold_get, &ldbm_config_system_allidsthreshold_set, CONFIG_FLAG_ALWAYS_SHOW | CONFIG_FLAG_ALLOW_RUNNING_CHANGE}, + {CONFIG_DIRECTORY, CONFIG_TYPE_STRING, "", &ldbm_config_directory_get, &ldbm_config_directory_set, CONFIG_FLAG_ALWAYS_SHOW | CONFIG_FLAG_ALLOW_RUNNING_CHANGE | CONFIG_FLAG_SKIP_DEFAULT_SETTING}, + {CONFIG_MAXPASSBEFOREMERGE, CONFIG_TYPE_INT, "100", &ldbm_config_maxpassbeforemerge_get, &ldbm_config_maxpassbeforemerge_set, 0}, + +diff --git a/ldap/servers/slapd/back-ldbm/ldbm_config.h b/ldap/servers/slapd/back-ldbm/ldbm_config.h +index 004e5ea7e..48446193e 100644 +--- a/ldap/servers/slapd/back-ldbm/ldbm_config.h ++++ b/ldap/servers/slapd/back-ldbm/ldbm_config.h +@@ -60,7 +60,6 @@ struct config_info + #define CONFIG_RANGELOOKTHROUGHLIMIT "nsslapd-rangelookthroughlimit" + #define CONFIG_PAGEDLOOKTHROUGHLIMIT "nsslapd-pagedlookthroughlimit" + #define CONFIG_IDLISTSCANLIMIT "nsslapd-idlistscanlimit" +-#define CONFIG_SYSTEMIDLISTSCANLIMIT "nsslapd-systemidlistscanlimit" + #define CONFIG_PAGEDIDLISTSCANLIMIT "nsslapd-pagedidlistscanlimit" + #define CONFIG_DIRECTORY "nsslapd-directory" + #define CONFIG_MODE "nsslapd-mode" +diff --git a/ldap/servers/slapd/back-ldbm/ldbm_index_config.c b/ldap/servers/slapd/back-ldbm/ldbm_index_config.c +index bae2a64b9..38e7368e1 100644 +--- a/ldap/servers/slapd/back-ldbm/ldbm_index_config.c ++++ b/ldap/servers/slapd/back-ldbm/ldbm_index_config.c +@@ -384,14 +384,6 @@ ldbm_instance_config_add_index_entry( + } + } + +- /* get nsIndexIDListScanLimit and its values, and add them */ +- if (0 == slapi_entry_attr_find(e, "nsIndexIDListScanLimit", &attr)) { +- for (j = slapi_attr_first_value(attr, &sval); j != -1; j = slapi_attr_next_value(attr, j, &sval)) { +- attrValue = slapi_value_get_berval(sval); +- eBuf = PR_sprintf_append(eBuf, "nsIndexIDListScanLimit: %s\n", attrValue->bv_val); +- } +- } +- + ldbm_config_add_dse_entry(li, eBuf, flags); + if (eBuf) { + PR_smprintf_free(eBuf); +diff --git a/src/lib389/lib389/backend.py b/src/lib389/lib389/backend.py +index 3cea0df36..4babf6850 100644 +--- a/src/lib389/lib389/backend.py ++++ b/src/lib389/lib389/backend.py +@@ -539,10 +539,11 @@ class Backend(DSLdapObject): + indexes = self.get_indexes() + + # Default system indexes taken from ldap/servers/slapd/back-ldbm/instance.c ++ # Note: entryrdn and ancestorid are internal system indexes that are not ++ # exposed in cn=config - they are managed internally by the server. ++ # Only parentid has a DSE config entry (for the integerOrderingMatch rule). + expected_system_indexes = { +- 'entryrdn': {'types': ['subtree'], 'matching_rule': None}, +- 'parentid': {'types': ['eq'], 'matching_rule': 'integerOrderingMatch', 'scanlimit': 'limit=5000 type=eq flags=AND'}, +- 'ancestorid': {'types': ['eq'], 'matching_rule': 'integerOrderingMatch', 'scanlimit': 'limit=5000 type=eq flags=AND'}, ++ 'parentid': {'types': ['eq'], 'matching_rule': 'integerOrderingMatch'}, + 'objectClass': {'types': ['eq'], 'matching_rule': None}, + 'aci': {'types': ['pres'], 'matching_rule': None}, + 'nscpEntryDN': {'types': ['eq'], 'matching_rule': None}, +@@ -599,17 +600,14 @@ class Backend(DSLdapObject): + # Generate remediation command + index_types = ' '.join([f"--index-type {t}" for t in expected_config['types']]) + cmd = f"dsconf YOUR_INSTANCE backend index add {bename} --attr {attr_name} {index_types}" +- if expected_config.get('matching_rule'): ++ if expected_config['matching_rule']: + cmd += f" --matching-rule {expected_config['matching_rule']}" +- if expected_config.get('scanlimit'): +- cmd += f" --add-scanlimit \"{expected_config['scanlimit']}\"" + remediation_commands.append(cmd) + reindex_attrs.add(attr_name) # New index needs reindexing + else: + # Index exists, check configuration + actual_types = index.get_attr_vals_utf8('nsIndexType') or [] + actual_mrs = index.get_attr_vals_utf8('nsMatchingRule') or [] +- actual_scanlimit = index.get_attr_vals_utf8('nsIndexIDListScanLimit') or [] + + # Normalize to lowercase for comparison + actual_types = [t.lower() for t in actual_types] +@@ -624,31 +622,16 @@ class Backend(DSLdapObject): + remediation_commands.append(cmd) + reindex_attrs.add(attr_name) + +- # Check matching rules and scanlimit together to generate a single combined command ++ # Check matching rules + expected_mr = expected_config.get('matching_rule') +- expected_scanlimit = expected_config.get('scanlimit') +- +- missing_mr = False + if expected_mr: + actual_mrs_lower = [mr.lower() for mr in actual_mrs] + if expected_mr.lower() not in actual_mrs_lower: + discrepancies.append(f"Index {attr_name} missing matching rule: {expected_mr}") +- missing_mr = True +- +- missing_scanlimit = False +- if expected_scanlimit and (len(actual_scanlimit) == 0): +- discrepancies.append(f"Index {attr_name} missing fine grain definition of IDs limit: {expected_scanlimit}") +- missing_scanlimit = True +- +- # Generate a single combined command for all missing items +- if missing_mr or missing_scanlimit: +- cmd = f"dsconf YOUR_INSTANCE backend index set {bename} --attr {attr_name}" +- if missing_mr: +- cmd += f" --add-mr {expected_mr}" +- if missing_scanlimit: +- cmd += f" --add-scanlimit \"{expected_scanlimit}\"" +- remediation_commands.append(cmd) +- reindex_attrs.add(attr_name) ++ # Add the missing matching rule ++ cmd = f"dsconf YOUR_INSTANCE backend index set {bename} --attr {attr_name} --add-mr {expected_mr}" ++ remediation_commands.append(cmd) ++ reindex_attrs.add(attr_name) + + except Exception as e: + self._log.debug(f"_lint_system_indexes - Error checking index {attr_name}: {e}") +@@ -879,13 +862,12 @@ class Backend(DSLdapObject): + return + raise ValueError("Can not delete index because it does not exist") + +- def add_index(self, attr_name, types, matching_rules=None, idlistscanlimit=None, reindex=False): ++ def add_index(self, attr_name, types, matching_rules=None, reindex=False): + """ Add an index. + + :param attr_name - name of the attribute to index + :param types - a List of index types(eq, pres, sub, approx) + :param matching_rules - a List of matching rules for the index +- :param idlistscanlimit - a List of fine grain definitions for scanning limit + :param reindex - If set to True then index the attribute after creating it. + """ + +@@ -915,15 +897,6 @@ class Backend(DSLdapObject): + # Only add if there are actually rules present in the list. + if len(mrs) > 0: + props['nsMatchingRule'] = mrs +- +- if idlistscanlimit is not None: +- scanlimits = [] +- for scanlimit in idlistscanlimit: +- scanlimits.append(scanlimit) +- # Only add if there are actually limits in the list. +- if len(scanlimits) > 0: +- props['nsIndexIDListScanLimit'] = scanlimits +- + new_index.create(properties=props, basedn="cn=index," + self._dn) + + if reindex: +@@ -1230,7 +1203,6 @@ class DatabaseConfig(DSLdapObject): + 'nsslapd-lookthroughlimit', + 'nsslapd-mode', + 'nsslapd-idlistscanlimit', +- 'nsslapd-systemidlistscanlimit', + 'nsslapd-directory', + 'nsslapd-import-cachesize', + 'nsslapd-idl-switch', +diff --git a/src/lib389/lib389/cli_conf/backend.py b/src/lib389/lib389/cli_conf/backend.py +index d57cb9433..4dc67d563 100644 +--- a/src/lib389/lib389/cli_conf/backend.py ++++ b/src/lib389/lib389/cli_conf/backend.py +@@ -39,7 +39,6 @@ arg_to_attr = { + 'mode': 'nsslapd-mode', + 'state': 'nsslapd-state', + 'idlistscanlimit': 'nsslapd-idlistscanlimit', +- 'systemidlistscanlimit': 'nsslapd-systemidlistscanlimit', + 'directory': 'nsslapd-directory', + 'dbcachesize': 'nsslapd-dbcachesize', + 'logdirectory': 'nsslapd-db-logdirectory', +@@ -588,21 +587,6 @@ def backend_set_index(inst, basedn, log, args): + except ldap.NO_SUCH_ATTRIBUTE: + raise ValueError('Can not delete matching rule type because it does not exist') + +- if args.replace_scanlimit is not None: +- for replace_scanlimit in args.replace_scanlimit: +- index.replace('nsIndexIDListScanLimit', replace_scanlimit) +- +- if args.add_scanlimit is not None: +- for add_scanlimit in args.add_scanlimit: +- index.add('nsIndexIDListScanLimit', add_scanlimit) +- +- if args.del_scanlimit is not None: +- for del_scanlimit in args.del_scanlimit: +- try: +- index.remove('nsIndexIDListScanLimit', del_scanlimit) +- except ldap.NO_SUCH_ATTRIBUTE: +- raise ValueError('Can not delete a fine grain limit definition because it does not exist') +- + if args.reindex: + be.reindex(attrs=[args.attr]) + log.info("Index successfully updated") +@@ -924,9 +908,6 @@ def create_parser(subparsers): + edit_index_parser.add_argument('--del-type', action='append', help='Removes an index type from the index: (eq, sub, pres, or approx)') + edit_index_parser.add_argument('--add-mr', action='append', help='Adds a matching-rule to the index') + edit_index_parser.add_argument('--del-mr', action='append', help='Removes a matching-rule from the index') +- edit_index_parser.add_argument('--add-scanlimit', action='append', help='Adds a fine grain limit definiton to the index') +- edit_index_parser.add_argument('--replace-scanlimit', action='append', help='Replaces a fine grain limit definiton to the index') +- edit_index_parser.add_argument('--del-scanlimit', action='append', help='Removes a fine grain limit definiton to the index') + edit_index_parser.add_argument('--reindex', action='store_true', help='Re-indexes the database after editing the index') + edit_index_parser.add_argument('be_name', help='The backend name or suffix') + +@@ -1053,7 +1034,6 @@ def create_parser(subparsers): + 'will check when examining candidate entries in response to a search request') + set_db_config_parser.add_argument('--mode', help='Specifies the permissions used for newly created index files') + set_db_config_parser.add_argument('--idlistscanlimit', help='Specifies the number of entry IDs that are searched during a search operation') +- set_db_config_parser.add_argument('--systemidlistscanlimit', help='Specifies the number of entry IDs that are fetch from ancestorid/parentid indexes') + set_db_config_parser.add_argument('--directory', help='Specifies absolute path to database instance') + set_db_config_parser.add_argument('--dbcachesize', help='Specifies the database index cache size in bytes') + set_db_config_parser.add_argument('--logdirectory', help='Specifies the path to the directory that contains the database transaction logs') +-- +2.52.0 + diff --git a/SOURCES/0079-Issue-7223-Backport-upgrade-infrastructure-from-main.patch b/SOURCES/0079-Issue-7223-Backport-upgrade-infrastructure-from-main.patch new file mode 100644 index 0000000..0527a9a --- /dev/null +++ b/SOURCES/0079-Issue-7223-Backport-upgrade-infrastructure-from-main.patch @@ -0,0 +1,117 @@ +From e9f5082218c06f7dbcb1e7f70337841178f18271 Mon Sep 17 00:00:00 2001 +From: Viktor Ashirov +Date: Mon, 2 Feb 2026 09:30:13 +0100 +Subject: [PATCH] Issue 7223 - Backport upgrade infrastructure from main branch + +Backport the upgrade.c infrastructure from main/1.4.4 branches to provide +a mechanism for automatic configuration fixes during server startup. + +Relates: https://github.com/389ds/389-ds-base/issues/7223 + +Reviewed by: @progier389, @tbordaz (Thanks!) +--- + Makefile.am | 1 + + ldap/servers/slapd/main.c | 16 ++++++++++++++++ + ldap/servers/slapd/slap.h | 10 ++++++++++ + ldap/servers/slapd/upgrade.c | 29 +++++++++++++++++++++++++++++ + 4 files changed, 56 insertions(+) + create mode 100644 ldap/servers/slapd/upgrade.c + +diff --git a/Makefile.am b/Makefile.am +index 245d7cf19..41eab712b 100644 +--- a/Makefile.am ++++ b/Makefile.am +@@ -2346,6 +2346,7 @@ ns_slapd_SOURCES = ldap/servers/slapd/abandon.c \ + ldap/servers/slapd/stubs.c \ + ldap/servers/slapd/tempnam.c \ + ldap/servers/slapd/unbind.c \ ++ ldap/servers/slapd/upgrade.c \ + $(GETSOCKETPEER) + + ns_slapd_CPPFLAGS = $(AM_CPPFLAGS) $(DSPLUGIN_CPPFLAGS) $(SASL_CFLAGS) $(SVRCORE_INCLUDES) +diff --git a/ldap/servers/slapd/main.c b/ldap/servers/slapd/main.c +index 9b5b845cb..043dc54bc 100644 +--- a/ldap/servers/slapd/main.c ++++ b/ldap/servers/slapd/main.c +@@ -738,6 +738,22 @@ main(int argc, char **argv) + + mcfg.n_port = config_get_port(); + mcfg.s_port = config_get_secureport(); ++ ++ /* ++ * This step checks for any updates and changes on upgrade ++ * specifically, it manages assumptions about what plugins should exist, ++ * and their configurations, and potentially even the state of ++ * configurations on the server and their removal and deprecation. ++ * ++ * Has to be after dse to change config, but before plugins start ++ * so we can adjust these configurations. ++ */ ++ if (upgrade_server() != UPGRADE_SUCCESS) { ++ slapi_log_err(SLAPI_LOG_EMERG, "main", ++ "Server upgrade check failed. Please check the error log for more information.\n"); ++ return_value = 1; ++ goto cleanup; ++ } + } + + raise_process_limits(); /* should be done ASAP once config file read */ +diff --git a/ldap/servers/slapd/slap.h b/ldap/servers/slapd/slap.h +index 36d26bf4a..e6b49d366 100644 +--- a/ldap/servers/slapd/slap.h ++++ b/ldap/servers/slapd/slap.h +@@ -183,6 +183,16 @@ typedef void (*VFPV)(); /* takes undefined arguments */ + #include "slapi-private.h" + #include "pw.h" + ++/* ++ * SERVER UPGRADE INTERNALS ++ */ ++typedef enum _upgrade_status { ++ UPGRADE_SUCCESS = 0, ++ UPGRADE_FAILURE = 1, ++} upgrade_status; ++ ++upgrade_status upgrade_server(void); ++ + /* + * call the appropriate signal() function. + */ +diff --git a/ldap/servers/slapd/upgrade.c b/ldap/servers/slapd/upgrade.c +new file mode 100644 +index 000000000..2f124afaf +--- /dev/null ++++ b/ldap/servers/slapd/upgrade.c +@@ -0,0 +1,29 @@ ++/* BEGIN COPYRIGHT BLOCK ++ * Copyright (C) 2017 Red Hat, Inc. ++ * Copyright (C) 2020 William Brown ++ * All rights reserved. ++ * ++ * License: GPL (version 3 or any later version). ++ * See LICENSE for details. ++ * END COPYRIGHT BLOCK */ ++ ++#include ++#include ++ ++/* ++ * This is called on server startup *before* plugins start ++ * but after config dse is read for operations. This allows ++ * us to make internal assertions about the state of the configuration ++ * at start up, enable plugins, and more. ++ * ++ * The functions in this file are named as: ++ * upgrade_xxx_yyy, where xxx is the minimum version of the project ++ * and yyy is the feature that is having it's configuration upgrade ++ * or altered. ++ */ ++ ++upgrade_status ++upgrade_server(void) ++{ ++ return UPGRADE_SUCCESS; ++} +-- +2.52.0 + diff --git a/SOURCES/0080-Issue-7223-Add-upgrade-function-to-remove-nsIndexIDL.patch b/SOURCES/0080-Issue-7223-Add-upgrade-function-to-remove-nsIndexIDL.patch new file mode 100644 index 0000000..add5f19 --- /dev/null +++ b/SOURCES/0080-Issue-7223-Add-upgrade-function-to-remove-nsIndexIDL.patch @@ -0,0 +1,312 @@ +From c6e2911dca08aac40e98999b48019eac72cc74a6 Mon Sep 17 00:00:00 2001 +From: Viktor Ashirov +Date: Thu, 5 Feb 2026 12:17:06 +0100 +Subject: [PATCH] Issue 7223 - Add upgrade function to remove + nsIndexIDListScanLimit from parentid + +Description: +Add `upgrade_remove_index_scanlimit()` function that removes the +nsIndexIDListScanLimit attribute from parentid index configuration +if present. + +This attribute was incorrectly added by a previous version and can +cause issues with index configuration. The upgrade function runs +automatically on server startup and removes the attribute if found. + +Relates: https://github.com/389ds/389-ds-base/issues/7223 + +Reviewed by: @progier389, @tbordaz (Thanks!) +--- + .../healthcheck/health_system_indexes_test.py | 52 +++++ + ldap/servers/slapd/upgrade.c | 210 ++++++++++++++++++ + 2 files changed, 262 insertions(+) + +diff --git a/dirsrvtests/tests/suites/healthcheck/health_system_indexes_test.py b/dirsrvtests/tests/suites/healthcheck/health_system_indexes_test.py +index 842f7e8dd..72a04fdab 100644 +--- a/dirsrvtests/tests/suites/healthcheck/health_system_indexes_test.py ++++ b/dirsrvtests/tests/suites/healthcheck/health_system_indexes_test.py +@@ -451,6 +451,58 @@ def test_multiple_missing_indexes(topology_st, log_buffering_enabled): + run_healthcheck_and_flush_log(topology_st, standalone, json=True, searched_code=JSON_OUTPUT) + + ++def test_upgrade_removes_parentid_scanlimit(topology_st): ++ """Check if upgrade function removes nsIndexIDListScanLimit from parentid index ++ ++ :id: 2808886e-c1c1-441d-b3a3-299c4ef1ab4a ++ :setup: Standalone instance ++ :steps: ++ 1. Create DS instance ++ 2. Stop the server ++ 3. Use DSEldif to add nsIndexIDListScanLimit to parentid index ++ 4. Start the server (triggers upgrade) ++ 5. Verify nsIndexIDListScanLimit is removed from parentid index ++ :expectedresults: ++ 1. Success ++ 2. Success ++ 3. Success ++ 4. Success ++ 5. nsIndexIDListScanLimit is no longer present ++ """ ++ from lib389.dseldif import DSEldif ++ ++ standalone = topology_st.standalone ++ PARENTID_DN = "cn=parentid,cn=index,cn=userroot,cn=ldbm database,cn=plugins,cn=config" ++ SCANLIMIT_VALUE = "limit=5000 type=eq flags=AND" ++ ++ log.info("Stop the server") ++ standalone.stop() ++ ++ log.info("Add nsIndexIDListScanLimit to parentid index using DSEldif") ++ dse_ldif = DSEldif(standalone) ++ dse_ldif.add(PARENTID_DN, "nsIndexIDListScanLimit", SCANLIMIT_VALUE) ++ ++ # Verify it was added ++ scanlimit = dse_ldif.get(PARENTID_DN, "nsIndexIDListScanLimit") ++ assert scanlimit is not None, "Failed to add nsIndexIDListScanLimit" ++ log.info(f"Added nsIndexIDListScanLimit: {scanlimit}") ++ ++ log.info("Start the server (triggers upgrade)") ++ standalone.start() ++ ++ log.info("Verify nsIndexIDListScanLimit was removed by upgrade") ++ # Check via LDAP - the upgrade should have removed it ++ parentid_index = Index(standalone, PARENTID_DN) ++ scanlimit_after = parentid_index.get_attr_vals_utf8("nsIndexIDListScanLimit") ++ log.info(f"nsIndexIDListScanLimit after upgrade: {scanlimit_after}") ++ ++ # The upgrade function should have removed nsIndexIDListScanLimit ++ assert not scanlimit_after, \ ++ f"nsIndexIDListScanLimit should have been removed but found: {scanlimit_after}" ++ ++ log.info("Upgrade successfully removed nsIndexIDListScanLimit from parentid index") ++ ++ + if __name__ == "__main__": + # Run isolated + # -s for DEBUG mode +diff --git a/ldap/servers/slapd/upgrade.c b/ldap/servers/slapd/upgrade.c +index 2f124afaf..074c15e3c 100644 +--- a/ldap/servers/slapd/upgrade.c ++++ b/ldap/servers/slapd/upgrade.c +@@ -22,8 +22,218 @@ + * or altered. + */ + ++/* ++ * Remove nsIndexIDListScanLimit from parentid index configuration. ++ * ++ * This attribute was incorrectly added by a previous version and can ++ * cause issues with index configuration. Remove it if present. ++ */ ++static upgrade_status ++upgrade_remove_index_scanlimit(void) ++{ ++ struct slapi_pblock *pb = slapi_pblock_new(); ++ Slapi_Entry **backends = NULL; ++ const char *be_base_dn = "cn=ldbm database,cn=plugins,cn=config"; ++ const char *be_filter = "(objectclass=nsBackendInstance)"; ++ const char *attrs_to_check[] = {"parentid", NULL}; ++ upgrade_status uresult = UPGRADE_SUCCESS; ++ ++ /* Search for all backend instances */ ++ slapi_search_internal_set_pb( ++ pb, be_base_dn, ++ LDAP_SCOPE_ONELEVEL, ++ be_filter, NULL, 0, NULL, NULL, ++ plugin_get_default_component_id(), 0); ++ slapi_search_internal_pb(pb); ++ slapi_pblock_get(pb, SLAPI_PLUGIN_INTOP_SEARCH_ENTRIES, &backends); ++ ++ if (backends) { ++ for (size_t be_idx = 0; backends[be_idx] != NULL; be_idx++) { ++ const char *be_dn = slapi_entry_get_dn_const(backends[be_idx]); ++ const char *be_name = slapi_entry_attr_get_ref(backends[be_idx], "cn"); ++ if (!be_dn || !be_name) { ++ continue; ++ } ++ ++ for (size_t attr_idx = 0; attrs_to_check[attr_idx] != NULL; attr_idx++) { ++ const char *attr_name = attrs_to_check[attr_idx]; ++ struct slapi_pblock *idx_pb = slapi_pblock_new(); ++ Slapi_Entry **idx_entries = NULL; ++ char *idx_dn = slapi_create_dn_string("cn=%s,cn=index,%s", ++ attr_name, be_dn); ++ char *idx_filter = "(objectclass=nsIndex)"; ++ ++ if (!idx_dn) { ++ slapi_pblock_destroy(idx_pb); ++ continue; ++ } ++ ++ slapi_search_internal_set_pb( ++ idx_pb, idx_dn, ++ LDAP_SCOPE_BASE, ++ idx_filter, NULL, 0, NULL, NULL, ++ plugin_get_default_component_id(), 0); ++ slapi_search_internal_pb(idx_pb); ++ slapi_pblock_get(idx_pb, SLAPI_PLUGIN_INTOP_SEARCH_ENTRIES, &idx_entries); ++ ++ if (idx_entries && idx_entries[0]) { ++ /* Check if nsIndexIDListScanLimit is present */ ++ if (slapi_entry_attr_get_ref(idx_entries[0], "nsIndexIDListScanLimit") != NULL) { ++ /* Remove nsIndexIDListScanLimit */ ++ Slapi_PBlock *mod_pb = slapi_pblock_new(); ++ Slapi_Mods smods; ++ int rc; ++ ++ slapi_mods_init(&smods, 1); ++ slapi_mods_add(&smods, LDAP_MOD_DELETE, "nsIndexIDListScanLimit", 0, NULL); ++ ++ slapi_modify_internal_set_pb( ++ mod_pb, idx_dn, ++ slapi_mods_get_ldapmods_byref(&smods), ++ NULL, NULL, ++ plugin_get_default_component_id(), 0); ++ slapi_modify_internal_pb(mod_pb); ++ slapi_pblock_get(mod_pb, SLAPI_PLUGIN_INTOP_RESULT, &rc); ++ ++ if (rc == LDAP_SUCCESS) { ++ slapi_log_err(SLAPI_LOG_NOTICE, "upgrade_remove_index_scanlimit", ++ "Removed 'nsIndexIDListScanLimit' from index '%s' in backend '%s'\n", ++ attr_name, be_name); ++ } else if (rc != LDAP_NO_SUCH_ATTRIBUTE) { ++ slapi_log_err(SLAPI_LOG_ERR, "upgrade_remove_index_scanlimit", ++ "Failed to remove 'nsIndexIDListScanLimit' from index '%s' in backend '%s': error %d\n", ++ attr_name, be_name, rc); ++ } ++ ++ slapi_mods_done(&smods); ++ slapi_pblock_destroy(mod_pb); ++ } ++ } ++ ++ slapi_ch_free_string(&idx_dn); ++ slapi_free_search_results_internal(idx_pb); ++ slapi_pblock_destroy(idx_pb); ++ } ++ } ++ } ++ ++ slapi_free_search_results_internal(pb); ++ slapi_pblock_destroy(pb); ++ ++ return uresult; ++} ++ ++/* ++ * Check if parentid indexes are missing the integerOrderingMatch ++ * matching rule. ++ * ++ * This function logs a warning if we detect this condition, advising ++ * the administrator to reindex the affected attributes. ++ */ ++static upgrade_status ++upgrade_check_id_index_matching_rule(void) ++{ ++ struct slapi_pblock *pb = slapi_pblock_new(); ++ Slapi_Entry **backends = NULL; ++ const char *be_base_dn = "cn=ldbm database,cn=plugins,cn=config"; ++ const char *be_filter = "(objectclass=nsBackendInstance)"; ++ const char *attrs_to_check[] = {"parentid", "ancestorid", NULL}; ++ upgrade_status uresult = UPGRADE_SUCCESS; ++ ++ /* Search for all backend instances */ ++ slapi_search_internal_set_pb( ++ pb, be_base_dn, ++ LDAP_SCOPE_ONELEVEL, ++ be_filter, NULL, 0, NULL, NULL, ++ plugin_get_default_component_id(), 0); ++ slapi_search_internal_pb(pb); ++ slapi_pblock_get(pb, SLAPI_PLUGIN_INTOP_SEARCH_ENTRIES, &backends); ++ ++ if (backends) { ++ for (size_t be_idx = 0; backends[be_idx] != NULL; be_idx++) { ++ const char *be_name = slapi_entry_attr_get_ref(backends[be_idx], "cn"); ++ if (!be_name) { ++ continue; ++ } ++ ++ /* Check each attribute that should have integerOrderingMatch */ ++ for (size_t attr_idx = 0; attrs_to_check[attr_idx] != NULL; attr_idx++) { ++ const char *attr_name = attrs_to_check[attr_idx]; ++ struct slapi_pblock *idx_pb = slapi_pblock_new(); ++ Slapi_Entry **idx_entries = NULL; ++ char *idx_dn = slapi_create_dn_string("cn=%s,cn=index,cn=%s,%s", ++ attr_name, be_name, be_base_dn); ++ char *idx_filter = "(objectclass=nsIndex)"; ++ PRBool has_matching_rule = PR_FALSE; ++ ++ if (!idx_dn) { ++ slapi_pblock_destroy(idx_pb); ++ continue; ++ } ++ ++ slapi_search_internal_set_pb( ++ idx_pb, idx_dn, ++ LDAP_SCOPE_BASE, ++ idx_filter, NULL, 0, NULL, NULL, ++ plugin_get_default_component_id(), 0); ++ slapi_search_internal_pb(idx_pb); ++ slapi_pblock_get(idx_pb, SLAPI_PLUGIN_INTOP_SEARCH_ENTRIES, &idx_entries); ++ ++ if (idx_entries && idx_entries[0]) { ++ /* Index exists, check if it has integerOrderingMatch */ ++ Slapi_Attr *mr_attr = NULL; ++ if (slapi_entry_attr_find(idx_entries[0], "nsMatchingRule", &mr_attr) == 0) { ++ Slapi_Value *sval = NULL; ++ int idx; ++ for (idx = slapi_attr_first_value(mr_attr, &sval); ++ idx != -1; ++ idx = slapi_attr_next_value(mr_attr, idx, &sval)) { ++ const struct berval *bval = slapi_value_get_berval(sval); ++ if (bval && bval->bv_val && ++ strcasecmp(bval->bv_val, "integerOrderingMatch") == 0) { ++ has_matching_rule = PR_TRUE; ++ break; ++ } ++ } ++ } ++ ++ if (!has_matching_rule) { ++ /* Index exists but doesn't have integerOrderingMatch, log a warning */ ++ slapi_log_err(SLAPI_LOG_ERR, "upgrade_check_id_index_matching_rule", ++ "Index '%s' in backend '%s' is missing 'nsMatchingRule: integerOrderingMatch'. " ++ "Incorrectly configured system indexes can lead to poor search performance, replication issues, and other operational problems. " ++ "To fix this, add the matching rule and reindex: " ++ "dsconf backend index set --add-mr integerOrderingMatch --attr %s %s && " ++ "dsconf backend index reindex --attr %s %s. " ++ "WARNING: Reindexing can be resource-intensive and may impact server performance on a live system. " ++ "Consider scheduling reindexing during maintenance windows or periods of low activity.\n", ++ attr_name, be_name, attr_name, be_name, attr_name, be_name); ++ } ++ } ++ ++ slapi_ch_free_string(&idx_dn); ++ slapi_free_search_results_internal(idx_pb); ++ slapi_pblock_destroy(idx_pb); ++ } ++ } ++ } ++ ++ slapi_free_search_results_internal(pb); ++ slapi_pblock_destroy(pb); ++ ++ return uresult; ++} ++ + upgrade_status + upgrade_server(void) + { ++ if (upgrade_remove_index_scanlimit() != UPGRADE_SUCCESS) { ++ return UPGRADE_FAILURE; ++ } ++ ++ if (upgrade_check_id_index_matching_rule() != UPGRADE_SUCCESS) { ++ return UPGRADE_FAILURE; ++ } ++ + return UPGRADE_SUCCESS; + } +-- +2.52.0 + diff --git a/SOURCES/0081-Issue-7223-Add-upgrade-function-to-remove-ancestorid.patch b/SOURCES/0081-Issue-7223-Add-upgrade-function-to-remove-ancestorid.patch new file mode 100644 index 0000000..d28519a --- /dev/null +++ b/SOURCES/0081-Issue-7223-Add-upgrade-function-to-remove-ancestorid.patch @@ -0,0 +1,313 @@ +From 1669e65b36d543fb69ef5503e5072ea37e9a0e19 Mon Sep 17 00:00:00 2001 +From: Viktor Ashirov +Date: Mon, 9 Feb 2026 14:12:18 +0100 +Subject: [PATCH] Issue 7223 - Add upgrade function to remove ancestorid index + config entry + +Description: +Add `upgrade_remove_ancestorid_index_config()` function that removes: +* ancestorid from `cn=default indexes` +* ancestorid index config entries from each backend's `cn=index` + +Also remove ancestorid index configuration from template-dse.ldif. + +Relates: https://github.com/389ds/389-ds-base/issues/7223 + +Reviewed by: @progier389, @tbordaz (Thanks!) +--- + .../healthcheck/health_system_indexes_test.py | 85 +++++++++++ + ldap/ldif/template-dse.ldif.in | 8 -- + ldap/servers/slapd/upgrade.c | 133 +++++++++++++++++- + 3 files changed, 214 insertions(+), 12 deletions(-) + +diff --git a/dirsrvtests/tests/suites/healthcheck/health_system_indexes_test.py b/dirsrvtests/tests/suites/healthcheck/health_system_indexes_test.py +index 72a04fdab..db5d13bf8 100644 +--- a/dirsrvtests/tests/suites/healthcheck/health_system_indexes_test.py ++++ b/dirsrvtests/tests/suites/healthcheck/health_system_indexes_test.py +@@ -502,6 +502,91 @@ def test_upgrade_removes_parentid_scanlimit(topology_st): + + log.info("Upgrade successfully removed nsIndexIDListScanLimit from parentid index") + ++ # Verify idempotency - restart again and ensure no errors ++ log.info("Restart server again to verify idempotency (no errors on second run)") ++ standalone.restart() ++ # Verify the attribute is still absent ++ scanlimit_after_second = parentid_index.get_attr_vals_utf8("nsIndexIDListScanLimit") ++ assert not scanlimit_after_second, \ ++ f"nsIndexIDListScanLimit should still be absent after second restart but found: {scanlimit_after_second}" ++ log.info("Idempotency verified - no issues on second restart") ++ ++ ++def test_upgrade_removes_ancestorid_index_config(topology_st): ++ """Check if upgrade function removes ancestorid index config entry ++ ++ :id: 3f3d6e9b-75ac-4f0d-b2ce-7204e6eacd0a ++ :setup: Standalone instance ++ :steps: ++ 1. Create DS instance ++ 2. Stop the server ++ 3. Use DSEldif to add an ancestorid index config entry ++ 4. Start the server (triggers upgrade) ++ 5. Verify ancestorid index config entry is removed ++ :expectedresults: ++ 1. Success ++ 2. Success ++ 3. Success ++ 4. Success ++ 5. ancestorid index config entry is no longer present ++ """ ++ from lib389.dseldif import DSEldif ++ ++ standalone = topology_st.standalone ++ ANCESTORID_DN = "cn=ancestorid,cn=index,cn=userroot,cn=ldbm database,cn=plugins,cn=config" ++ ++ log.info("Stop the server") ++ standalone.stop() ++ ++ log.info("Add ancestorid index config entry using DSEldif") ++ dse_ldif = DSEldif(standalone) ++ ++ # Create a fake ancestorid index entry ++ ancestorid_entry = [ ++ "dn: {}\n".format(ANCESTORID_DN), ++ "objectClass: top\n", ++ "objectClass: nsIndex\n", ++ "cn: ancestorid\n", ++ "nsSystemIndex: true\n", ++ "nsIndexType: eq\n", ++ "nsMatchingRule: integerOrderingMatch\n", ++ "\n" ++ ] ++ dse_ldif.add_entry(ancestorid_entry) ++ ++ # Verify it was added by re-reading dse.ldif ++ dse_ldif2 = DSEldif(standalone) ++ cn_value = dse_ldif2.get(ANCESTORID_DN, "cn") ++ assert cn_value is not None, "Failed to add ancestorid index config entry" ++ log.info(f"Added ancestorid index entry with cn: {cn_value}") ++ ++ log.info("Start the server (triggers upgrade)") ++ standalone.start() ++ ++ log.info("Verify ancestorid index config entry was removed by upgrade") ++ # Check via LDAP - the upgrade should have removed the entry ++ try: ++ ancestorid_index = Index(standalone, ANCESTORID_DN) ++ # If we can get the entry, it wasn't removed - this is a failure ++ cn_after = ancestorid_index.get_attr_vals_utf8("cn") ++ assert False, f"ancestorid index config entry should have been removed but still exists: {cn_after}" ++ except Exception as e: ++ # Entry should not exist - this is expected ++ log.info(f"ancestorid index config entry correctly removed (got exception: {e})") ++ ++ log.info("Upgrade successfully removed ancestorid index config entry") ++ ++ # Verify idempotency - restart again and ensure no errors ++ log.info("Restart server again to verify idempotency (no errors on second run)") ++ standalone.restart() ++ # Verify the entry is still absent ++ try: ++ ancestorid_index = Index(standalone, ANCESTORID_DN) ++ cn_after_second = ancestorid_index.get_attr_vals_utf8("cn") ++ assert False, f"ancestorid index config entry should still be absent after second restart but found: {cn_after_second}" ++ except Exception as e: ++ log.info(f"Idempotency verified - ancestorid still absent after second restart (got exception: {e})") ++ + + if __name__ == "__main__": + # Run isolated +diff --git a/ldap/ldif/template-dse.ldif.in b/ldap/ldif/template-dse.ldif.in +index c2754adf8..2ddaf5fb3 100644 +--- a/ldap/ldif/template-dse.ldif.in ++++ b/ldap/ldif/template-dse.ldif.in +@@ -973,14 +973,6 @@ cn: aci + nssystemindex: true + nsindextype: pres + +-dn: cn=ancestorid,cn=default indexes, cn=config,cn=ldbm database,cn=plugins,cn=config +-objectclass: top +-objectclass: nsIndex +-cn: ancestorid +-nssystemindex: true +-nsindextype: eq +-nsmatchingrule: integerOrderingMatch +- + dn: cn=cn,cn=default indexes, cn=config,cn=ldbm database,cn=plugins,cn=config + objectclass: top + objectclass: nsIndex +diff --git a/ldap/servers/slapd/upgrade.c b/ldap/servers/slapd/upgrade.c +index 074c15e3c..aa51e72d2 100644 +--- a/ldap/servers/slapd/upgrade.c ++++ b/ldap/servers/slapd/upgrade.c +@@ -123,6 +123,126 @@ upgrade_remove_index_scanlimit(void) + return uresult; + } + ++/* ++ * Remove ancestorid index configuration entry if present. ++ * ++ * The ancestorid index is special - it has no corresponding attribute type ++ * and should not have a DSE config entry. If an entry exists, remove it. ++ * ++ * This function removes: ++ * 1. The ancestorid entry from cn=default indexes (to prevent re-creation on startup) ++ * 2. The ancestorid entry from each backend's cn=index (if it exists) ++ */ ++static upgrade_status ++upgrade_remove_ancestorid_index_config(void) ++{ ++ struct slapi_pblock *pb = slapi_pblock_new(); ++ Slapi_Entry **backends = NULL; ++ const char *be_base_dn = "cn=ldbm database,cn=plugins,cn=config"; ++ const char *be_filter = "(objectclass=nsBackendInstance)"; ++ upgrade_status uresult = UPGRADE_SUCCESS; ++ int rc; ++ ++ /* ++ * First, remove ancestorid from cn=default indexes to prevent ++ * ldbm_instance_create_default_user_indexes() from re-creating it. ++ */ ++ { ++ Slapi_PBlock *def_pb = slapi_pblock_new(); ++ char *def_idx_dn = slapi_create_dn_string( ++ "cn=ancestorid,cn=default indexes,cn=config,%s", be_base_dn); ++ ++ if (def_idx_dn) { ++ slapi_delete_internal_set_pb( ++ def_pb, def_idx_dn, NULL, NULL, ++ plugin_get_default_component_id(), 0); ++ slapi_delete_internal_pb(def_pb); ++ slapi_pblock_get(def_pb, SLAPI_PLUGIN_INTOP_RESULT, &rc); ++ ++ if (rc == LDAP_SUCCESS) { ++ slapi_log_err(SLAPI_LOG_NOTICE, "upgrade_remove_ancestorid_index_config", ++ "Removed 'ancestorid' from default indexes.\n"); ++ } else if (rc != LDAP_NO_SUCH_OBJECT) { ++ slapi_log_err(SLAPI_LOG_ERR, "upgrade_remove_ancestorid_index_config", ++ "Failed to remove 'ancestorid' from default indexes: error %d\n", rc); ++ } ++ ++ slapi_ch_free_string(&def_idx_dn); ++ } ++ slapi_pblock_destroy(def_pb); ++ } ++ ++ /* Search for all backend instances */ ++ slapi_search_internal_set_pb( ++ pb, be_base_dn, ++ LDAP_SCOPE_ONELEVEL, ++ be_filter, NULL, 0, NULL, NULL, ++ plugin_get_default_component_id(), 0); ++ slapi_search_internal_pb(pb); ++ slapi_pblock_get(pb, SLAPI_PLUGIN_INTOP_SEARCH_ENTRIES, &backends); ++ ++ if (backends) { ++ for (size_t be_idx = 0; backends[be_idx] != NULL; be_idx++) { ++ const char *be_dn = slapi_entry_get_dn_const(backends[be_idx]); ++ const char *be_name = slapi_entry_attr_get_ref(backends[be_idx], "cn"); ++ if (!be_dn || !be_name) { ++ continue; ++ } ++ ++ struct slapi_pblock *idx_pb = slapi_pblock_new(); ++ Slapi_Entry **idx_entries = NULL; ++ char *idx_dn = slapi_create_dn_string("cn=ancestorid,cn=index,%s", ++ be_dn); ++ char *idx_filter = "(objectclass=nsIndex)"; ++ ++ if (!idx_dn) { ++ slapi_pblock_destroy(idx_pb); ++ continue; ++ } ++ ++ slapi_search_internal_set_pb( ++ idx_pb, idx_dn, ++ LDAP_SCOPE_BASE, ++ idx_filter, NULL, 0, NULL, NULL, ++ plugin_get_default_component_id(), 0); ++ slapi_search_internal_pb(idx_pb); ++ slapi_pblock_get(idx_pb, SLAPI_PLUGIN_INTOP_SEARCH_ENTRIES, &idx_entries); ++ ++ if (idx_entries && idx_entries[0]) { ++ /* ancestorid index entry exists - delete it */ ++ Slapi_PBlock *del_pb = slapi_pblock_new(); ++ ++ slapi_delete_internal_set_pb( ++ del_pb, idx_dn, NULL, NULL, ++ plugin_get_default_component_id(), 0); ++ slapi_delete_internal_pb(del_pb); ++ slapi_pblock_get(del_pb, SLAPI_PLUGIN_INTOP_RESULT, &rc); ++ ++ if (rc == LDAP_SUCCESS) { ++ slapi_log_err(SLAPI_LOG_NOTICE, "upgrade_remove_ancestorid_index_config", ++ "Removed 'ancestorid' index config entry in backend '%s'.\n", ++ be_name); ++ } else if (rc != LDAP_NO_SUCH_OBJECT) { ++ slapi_log_err(SLAPI_LOG_ERR, "upgrade_remove_ancestorid_index_config", ++ "Failed to remove 'ancestorid' index config entry in backend '%s': error %d\n", ++ be_name, rc); ++ } ++ ++ slapi_pblock_destroy(del_pb); ++ } ++ ++ slapi_ch_free_string(&idx_dn); ++ slapi_free_search_results_internal(idx_pb); ++ slapi_pblock_destroy(idx_pb); ++ } ++ } ++ ++ slapi_free_search_results_internal(pb); ++ slapi_pblock_destroy(pb); ++ ++ return uresult; ++} ++ + /* + * Check if parentid indexes are missing the integerOrderingMatch + * matching rule. +@@ -137,7 +257,7 @@ upgrade_check_id_index_matching_rule(void) + Slapi_Entry **backends = NULL; + const char *be_base_dn = "cn=ldbm database,cn=plugins,cn=config"; + const char *be_filter = "(objectclass=nsBackendInstance)"; +- const char *attrs_to_check[] = {"parentid", "ancestorid", NULL}; ++ const char *attrs_to_check[] = {"parentid", NULL}; + upgrade_status uresult = UPGRADE_SUCCESS; + + /* Search for all backend instances */ +@@ -151,8 +271,9 @@ upgrade_check_id_index_matching_rule(void) + + if (backends) { + for (size_t be_idx = 0; backends[be_idx] != NULL; be_idx++) { ++ const char *be_dn = slapi_entry_get_dn_const(backends[be_idx]); + const char *be_name = slapi_entry_attr_get_ref(backends[be_idx], "cn"); +- if (!be_name) { ++ if (!be_dn || !be_name) { + continue; + } + +@@ -161,8 +282,8 @@ upgrade_check_id_index_matching_rule(void) + const char *attr_name = attrs_to_check[attr_idx]; + struct slapi_pblock *idx_pb = slapi_pblock_new(); + Slapi_Entry **idx_entries = NULL; +- char *idx_dn = slapi_create_dn_string("cn=%s,cn=index,cn=%s,%s", +- attr_name, be_name, be_base_dn); ++ char *idx_dn = slapi_create_dn_string("cn=%s,cn=index,%s", ++ attr_name, be_dn); + char *idx_filter = "(objectclass=nsIndex)"; + PRBool has_matching_rule = PR_FALSE; + +@@ -231,6 +352,10 @@ upgrade_server(void) + return UPGRADE_FAILURE; + } + ++ if (upgrade_remove_ancestorid_index_config() != UPGRADE_SUCCESS) { ++ return UPGRADE_FAILURE; ++ } ++ + if (upgrade_check_id_index_matching_rule() != UPGRADE_SUCCESS) { + return UPGRADE_FAILURE; + } +-- +2.52.0 + diff --git a/SOURCES/0082-Issue-7223-Detect-and-log-index-ordering-mismatch-du.patch b/SOURCES/0082-Issue-7223-Detect-and-log-index-ordering-mismatch-du.patch new file mode 100644 index 0000000..e9beb43 --- /dev/null +++ b/SOURCES/0082-Issue-7223-Detect-and-log-index-ordering-mismatch-du.patch @@ -0,0 +1,307 @@ +From f6dd2d6dc94290c85d221951b34792443bb08d80 Mon Sep 17 00:00:00 2001 +From: Viktor Ashirov +Date: Thu, 5 Feb 2026 12:17:06 +0100 +Subject: [PATCH] Issue 7223 - Detect and log index ordering mismatch during + backend startup + +Description: +Add `ldbm_instance_check_index_config()` function that checks on-disk +index data and logs a message in case of a mismatch with DSE config entry. + +Relates: https://github.com/389ds/389-ds-base/issues/7223 + +Reviewed by: @progier389, @tbordaz (Thanks!) +--- + ldap/servers/slapd/back-ldbm/instance.c | 269 ++++++++++++++++++++++++ + 1 file changed, 269 insertions(+) + +diff --git a/ldap/servers/slapd/back-ldbm/instance.c b/ldap/servers/slapd/back-ldbm/instance.c +index 6098e04fc..388dd6efb 100644 +--- a/ldap/servers/slapd/back-ldbm/instance.c ++++ b/ldap/servers/slapd/back-ldbm/instance.c +@@ -248,6 +248,273 @@ ldbm_instance_create_default_indexes(backend *be) + } + + ++/* ++ * Check if an index has integerOrderingMatch configured in DSE. ++ * ++ * This function performs an internal LDAP search to check if the index ++ * configuration entry has nsMatchingRule: integerOrderingMatch. ++ * ++ * Parameters: ++ * inst_name - backend instance name (e.g., "userRoot") ++ * index_name - name of the index to check (e.g., "parentid", "ancestorid") ++ * ++ * Returns: ++ * PR_TRUE if integerOrderingMatch is configured ++ * PR_FALSE if not configured or index entry doesn't exist ++ */ ++static PRBool ++ldbm_instance_index_has_int_order_in_dse(const char *inst_name, const char *index_name) ++{ ++ Slapi_PBlock *pb = NULL; ++ Slapi_Entry **entries = NULL; ++ char *idx_dn = NULL; ++ PRBool has_int_order = PR_FALSE; ++ ++ idx_dn = slapi_create_dn_string("cn=%s,cn=index,cn=%s,cn=ldbm database,cn=plugins,cn=config", ++ index_name, inst_name); ++ if (idx_dn == NULL) { ++ return PR_FALSE; ++ } ++ ++ pb = slapi_pblock_new(); ++ slapi_search_internal_set_pb(pb, idx_dn, LDAP_SCOPE_BASE, ++ "(objectclass=nsIndex)", NULL, 0, NULL, NULL, ++ plugin_get_default_component_id(), 0); ++ slapi_search_internal_pb(pb); ++ slapi_pblock_get(pb, SLAPI_PLUGIN_INTOP_SEARCH_ENTRIES, &entries); ++ ++ if (entries && entries[0]) { ++ Slapi_Attr *mr_attr = NULL; ++ if (slapi_entry_attr_find(entries[0], "nsMatchingRule", &mr_attr) == 0) { ++ Slapi_Value *sval = NULL; ++ int idx; ++ for (idx = slapi_attr_first_value(mr_attr, &sval); ++ idx != -1; ++ idx = slapi_attr_next_value(mr_attr, idx, &sval)) { ++ const struct berval *bval = slapi_value_get_berval(sval); ++ if (bval && bval->bv_val && ++ strcasecmp(bval->bv_val, "integerOrderingMatch") == 0) { ++ has_int_order = PR_TRUE; ++ break; ++ } ++ } ++ } ++ } ++ ++ slapi_ch_free_string(&idx_dn); ++ slapi_free_search_results_internal(pb); ++ slapi_pblock_destroy(pb); ++ ++ return has_int_order; ++} ++ ++/* ++ * Check a system index for ordering mismatch between config and on-disk data. ++ * ++ * This function compares what's configured in DSE (nsMatchingRule) with ++ * what's actually on disk. A mismatch can occur in two scenarios: ++ * 1. Ordering rule is configured but disk has lexicographic order ++ * (rule was added after index was created) ++ * 2. No ordering rule configured but disk has integer order ++ * (rule was removed after index was created with it) ++ * ++ * This function reads the first keys from the specified index and checks ++ * if they are stored in lexicographic order (string: "1" < "10" < "2") or ++ * integer order (numeric: "1" < "2" < "10"). ++ * ++ * Parameters: ++ * be - backend ++ * index_name - name of the index to check (e.g., "parentid", "ancestorid") ++ * ++ */ ++static void ++ldbm_instance_check_index_config(backend *be, const char *index_name) ++{ ++ ldbm_instance *inst = (ldbm_instance *)be->be_instance_info; ++ struct attrinfo *ai = NULL; ++ DB *db = NULL; ++ DBC *dbc = NULL; ++ DBT key; ++ DBT data; ++ int ret = 0; ++ PRBool config_has_int_order = PR_FALSE; ++ PRBool disk_has_int_order = PR_TRUE; /* Assume integer order until proven otherwise */ ++ ID prev_id = 0; ++ int key_count = 0; ++ PRBool first_key = PR_TRUE; ++ PRBool found_ordering_evidence = PR_FALSE; ++ ++ slapi_log_err(SLAPI_LOG_DEBUG, "ldbm_instance_check_index_config", ++ "Backend '%s': checking %s index ordering...\n", ++ inst->inst_name, index_name); ++ ++ /* Check if integerOrderingMatch is configured in DSE */ ++ config_has_int_order = ldbm_instance_index_has_int_order_in_dse(inst->inst_name, index_name); ++ ++ /* Get attrinfo for the index */ ++ ainfo_get(be, (char *)index_name, &ai); ++ if (ai == NULL || strcmp(ai->ai_type, index_name) != 0) { ++ /* No index config found */ ++ slapi_log_err(SLAPI_LOG_DEBUG, "ldbm_instance_check_index_config", ++ "Backend '%s': no %s attrinfo found, skipping check\n", ++ inst->inst_name, index_name); ++ return; ++ } ++ ++ /* Open the index file */ ++ ret = dblayer_get_index_file(be, ai, &db, 0); ++ if (ret != 0 || db == NULL) { ++ /* Index file doesn't exist or can't be opened - this is fine for new instances */ ++ slapi_log_err(SLAPI_LOG_DEBUG, "ldbm_instance_check_index_config", ++ "Backend '%s': could not open %s index file (ret=%d), skipping order check\n", ++ inst->inst_name, index_name, ret); ++ return; ++ } ++ ++ /* Create a cursor to read keys */ ++ ret = db->cursor(db, NULL, &dbc, 0); ++ if (ret != 0) { ++ slapi_log_err(SLAPI_LOG_ERR, "ldbm_instance_check_index_config", ++ "Backend '%s': could not create cursor on %s index (ret=%d)\n", ++ inst->inst_name, index_name, ret); ++ dblayer_release_index_file(be, ai, db); ++ return; ++ } ++ ++ memset(&key, 0, sizeof(key)); ++ memset(&data, 0, sizeof(data)); ++ key.flags = DB_DBT_MALLOC; ++ data.flags = DB_DBT_MALLOC; ++ ++ /* ++ * Read up to 100 unique keys and check their ordering. ++ * With lexicographic ordering: "1" < "10" < "100" < "2" < "20" < "3" ++ * With integer ordering: "1" < "2" < "3" < "10" < "20" < "100" ++ * ++ * If we find a case where prev_id > current_id (numerically), but the ++ * keys are still in order (lexicographically), then the index uses ++ * lexicographic ordering. ++ */ ++ while (key_count < 100) { ++ ID current_id; ++ ++ slapi_ch_free(&(key.data)); ++ slapi_ch_free(&(data.data)); ++ key.size = 0; ++ data.size = 0; ++ ++ ret = dbc->c_get(dbc, &key, &data, first_key ? DB_FIRST : DB_NEXT_NODUP); ++ first_key = PR_FALSE; /* Always advance cursor on next iteration */ ++ if (ret != 0) { ++ break; /* No more keys or error */ ++ } ++ ++ /* Skip non-equality keys */ ++ if (key.size < 2 || *(char *)key.data != EQ_PREFIX) { ++ continue; ++ } ++ ++ /* Parse the ID from the key (format: "=") */ ++ current_id = (ID)strtoul((char *)key.data + 1, NULL, 10); ++ if (current_id == 0) { ++ continue; /* Invalid ID, skip */ ++ } ++ ++ key_count++; ++ ++ if (prev_id != 0) { ++ /* ++ * Check ordering: if prev_id > current_id numerically, ++ * but we got this key after prev in DB order, then ++ * the index is using lexicographic ordering. ++ * ++ * Example: if we see "10" followed by "2", that's lexicographic ++ * because "10" < "2" as strings, but 10 > 2 as integers. ++ */ ++ if (prev_id > current_id) { ++ /* Found evidence of lexicographic ordering */ ++ disk_has_int_order = PR_FALSE; ++ found_ordering_evidence = PR_TRUE; ++ break; ++ } else if (prev_id < current_id) { ++ /* ++ * This is consistent with integer ordering, but we need ++ * to find a case that proves lexicographic ordering. ++ * For example, seeing "1" followed by "2" is ambiguous, ++ * but seeing "1" followed by "10" (not "2") proves lexicographic. ++ * ++ * A definitive test: if we see an ID followed by a smaller ++ * ID, that's lexicographic. If all IDs are strictly increasing, ++ * it could be either (or the index only has sequential IDs). ++ */ ++ found_ordering_evidence = PR_TRUE; ++ } ++ } ++ prev_id = current_id; ++ } ++ ++ /* Close the cursor and free values */ ++ slapi_ch_free(&(key.data)); ++ slapi_ch_free(&(data.data)); ++ dbc->c_close(dbc); ++ ++ /* Release the index file */ ++ dblayer_release_index_file(be, ai, db); ++ ++ /* ++ * Report findings and check for config/disk mismatch. ++ * Log an error if there's a discrepancy between what's configured ++ * in DSE and what's actually on disk. ++ */ ++ if (!found_ordering_evidence) { ++ slapi_log_err(SLAPI_LOG_DEBUG, "ldbm_instance_check_index_config", ++ "Backend '%s': %s index ordering check - " ++ "could not determine on-disk ordering (index may be empty or have sequential IDs only). " ++ "Config has integerOrderingMatch: %s\n", ++ inst->inst_name, index_name, config_has_int_order ? "yes" : "no"); ++ } else if (config_has_int_order && !disk_has_int_order) { ++ /* Config expects integer ordering, but disk has lexicographic - MISMATCH */ ++ slapi_log_err(SLAPI_LOG_ERR, "ldbm_instance_check_index_config", ++ "Backend '%s': MISMATCH - %s index has integerOrderingMatch configured, " ++ "but on-disk data uses lexicographic ordering. " ++ "This will cause searches to return incorrect or incomplete results. " ++ "Please reindex the %s attribute: " ++ "dsconf backend index reindex --attr %s %s\n", ++ inst->inst_name, index_name, index_name, index_name, inst->inst_name); ++ } else if (!config_has_int_order && disk_has_int_order) { ++ /* Config expects lexicographic ordering, but disk has integer - MISMATCH */ ++ slapi_log_err(SLAPI_LOG_ERR, "ldbm_instance_check_index_config", ++ "Backend '%s': MISMATCH - %s index does not have integerOrderingMatch configured, " ++ "but on-disk data uses integer ordering. " ++ "This will cause searches to return incorrect or incomplete results. " ++ "Please reindex the %s attribute: " ++ "dsconf backend index reindex --attr %s %s\n", ++ inst->inst_name, index_name, index_name, index_name, inst->inst_name); ++ } else { ++ /* Config and disk ordering match - no action needed */ ++ slapi_log_err(SLAPI_LOG_DEBUG, "ldbm_instance_check_index_config", ++ "Backend '%s': %s index ordering check passed - " ++ "config has integerOrderingMatch: %s, on-disk data matches.\n", ++ inst->inst_name, index_name, config_has_int_order ? "yes" : "no"); ++ } ++} ++ ++/* ++ * Check system indexes for ordering mismatches. ++ * If a mismatch is detected, log an error advising the administrator ++ * to reindex the affected attribute. ++ * ++ * Note: We only check parentid here. The ancestorid index is a special ++ * system index that has no DSE config entry - its ordering is hardcoded ++ * in ldbm_instance_init_config_entry() and cannot be changed by users. ++ */ ++static void ++ldbm_instance_check_indexes(backend *be) ++{ ++ /* Check parentid index */ ++ ldbm_instance_check_index_config(be, LDBM_PARENTID_STR); ++} ++ + /* Starts a backend instance */ + int + ldbm_instance_start(backend *be) +@@ -316,6 +583,8 @@ ldbm_instance_startall(struct ldbminfo *li) + ldbm_instance_register_modify_callback(inst); + vlv_init(inst); + slapi_mtn_be_started(inst->inst_be); ++ /* Check index configuration for potential issues */ ++ ldbm_instance_check_indexes(inst->inst_be); + } + if (slapi_exist_referral(inst->inst_be)) { + slapi_be_set_flag(inst->inst_be, SLAPI_BE_FLAG_CONTAINS_REFERRAL); +-- +2.52.0 + diff --git a/SOURCES/0083-Issue-7223-Add-dsctl-index-check-command-for-offline.patch b/SOURCES/0083-Issue-7223-Add-dsctl-index-check-command-for-offline.patch new file mode 100644 index 0000000..4dc39da --- /dev/null +++ b/SOURCES/0083-Issue-7223-Add-dsctl-index-check-command-for-offline.patch @@ -0,0 +1,1215 @@ +From 99470a4befbea7eda777b9096337e29113e1ddda Mon Sep 17 00:00:00 2001 +From: Viktor Ashirov +Date: Thu, 5 Feb 2026 12:17:06 +0100 +Subject: [PATCH] Issue 7223 - Add dsctl index-check command for offline index + repair + +Description: +Add `dsctl index-check [backend] [--fix]` command for offline +detection and repair of index ordering mismatches. This is needed after +upgrade from versions that didn't use integerOrderingMatch for +parentid/ancestorid system indexes. + +It's automatically executed as part of RPM %post scriptlet during +upgrade. + +Relates: https://github.com/389ds/389-ds-base/issues/7223 + +Reviewed by: @progier389, @tbordaz (Thanks!) +--- + .../healthcheck/health_system_indexes_test.py | 593 ++++++++++++++++++ + rpm/389-ds-base.spec.in | 40 +- + src/lib389/lib389/cli_ctl/dbtasks.py | 402 ++++++++++++ + src/lib389/lib389/dseldif.py | 56 +- + 4 files changed, 1085 insertions(+), 6 deletions(-) + +diff --git a/dirsrvtests/tests/suites/healthcheck/health_system_indexes_test.py b/dirsrvtests/tests/suites/healthcheck/health_system_indexes_test.py +index db5d13bf8..ce86239e5 100644 +--- a/dirsrvtests/tests/suites/healthcheck/health_system_indexes_test.py ++++ b/dirsrvtests/tests/suites/healthcheck/health_system_indexes_test.py +@@ -588,6 +588,599 @@ def test_upgrade_removes_ancestorid_index_config(topology_st): + log.info(f"Idempotency verified - ancestorid still absent after second restart (got exception: {e})") + + ++def test_index_check_basic(topology_st): ++ """Check if dsctl index-check works correctly ++ ++ :id: 8a4e5c2d-1f3b-4a7c-9e8d-2b6f0c4a5d3e ++ :setup: Standalone instance ++ :steps: ++ 1. Create DS instance ++ 2. Run dsctl index-check while server is running (should fail) ++ 3. Stop the server ++ 4. Run dsctl index-check (should pass) ++ 5. Start the server ++ :expectedresults: ++ 1. Success ++ 2. index-check returns False and logs error ++ 3. Success ++ 4. index-check returns True (no mismatches) ++ 5. Success ++ """ ++ from lib389.cli_ctl.dbtasks import dbtasks_index_check ++ ++ standalone = topology_st.standalone ++ ++ log.info("Run index-check while server is running") ++ args = FakeArgs() ++ args.backend = None ++ args.fix = False ++ ++ # Server should be running, index-check should fail ++ assert standalone.status() ++ result = dbtasks_index_check(standalone, topology_st.logcap.log, args) ++ assert result is False ++ assert topology_st.logcap.contains("index-check requires the instance to be stopped") ++ topology_st.logcap.flush() ++ ++ log.info("Stop the server") ++ standalone.stop() ++ ++ log.info("Run index-check with server stopped") ++ result = dbtasks_index_check(standalone, topology_st.logcap.log, args) ++ assert result is True ++ assert topology_st.logcap.contains("All checks passed") ++ topology_st.logcap.flush() ++ ++ log.info("Start the server") ++ standalone.start() ++ ++ ++def test_index_check_specific_backend(topology_st): ++ """Check if dsctl index-check works with a specific backend ++ ++ :id: 407d8fcc-62e0-43dd-90fa-70e7090a5cfd ++ :setup: Standalone instance ++ :steps: ++ 1. Create DS instance ++ 2. Stop the server ++ 3. Run dsctl index-check with specific backend (userRoot) ++ 4. Run dsctl index-check with non-existent backend ++ 5. Start the server ++ :expectedresults: ++ 1. Success ++ 2. Success ++ 3. index-check returns True for userRoot ++ 4. index-check returns False for non-existent backend ++ 5. Success ++ """ ++ from lib389.cli_ctl.dbtasks import dbtasks_index_check ++ ++ standalone = topology_st.standalone ++ ++ log.info("Stop the server") ++ standalone.stop() ++ ++ log.info("Run index-check for userRoot backend") ++ args = FakeArgs() ++ args.backend = "userRoot" ++ args.fix = False ++ ++ result = dbtasks_index_check(standalone, topology_st.logcap.log, args) ++ assert result is True ++ # Check for backend name in any case ++ assert topology_st.logcap.contains("Checking backend:") ++ topology_st.logcap.flush() ++ ++ log.info("Run index-check for non-existent backend") ++ args.backend = "nonExistentBackend" ++ result = dbtasks_index_check(standalone, topology_st.logcap.log, args) ++ assert result is False ++ assert topology_st.logcap.contains("not found") ++ topology_st.logcap.flush() ++ ++ log.info("Start the server") ++ standalone.start() ++ ++ ++def test_index_check_mismatch_detection(topology_st): ++ """Check if dsctl index-check detects ordering mismatch ++ ++ :id: 50d14520-b0bf-4243-9fe6-b097928d4351 ++ :setup: Standalone instance ++ :steps: ++ 1. Create DS instance ++ 2. Stop the server ++ 3. Run dsctl index-check (without --fix) ++ 4. Verify output format ++ 5. Start the server ++ :expectedresults: ++ 1. Success ++ 2. Success ++ 3. index-check returns True (no mismatch on fresh instance) ++ 4. Log contains expected format ++ 5. Success ++ """ ++ from lib389.cli_ctl.dbtasks import dbtasks_index_check ++ ++ standalone = topology_st.standalone ++ ++ log.info("Stop the server") ++ standalone.stop() ++ ++ log.info("Run index-check to verify detection logic") ++ args = FakeArgs() ++ args.backend = "userRoot" ++ args.fix = False ++ ++ # On a fresh instance, there should be no mismatch ++ result = dbtasks_index_check(standalone, topology_st.logcap.log, args) ++ # Fresh instance should have matching config and disk ordering ++ assert result is True ++ # Check that the backend was checked (may skip indexes if ordering can't be determined) ++ assert topology_st.logcap.contains("Checking backend:") ++ topology_st.logcap.flush() ++ ++ log.info("Start the server") ++ standalone.start() ++ ++ ++def test_index_check_with_fix(topology_st): ++ """Check if dsctl index-check --fix triggers reindexing ++ ++ :id: 38ae36e4-c861-4771-ae7d-354370376a2f ++ :setup: Standalone instance ++ :steps: ++ 1. Create DS instance ++ 2. Stop the server ++ 3. Run dsctl index-check --fix (should pass since no mismatch) ++ 4. Verify output indicates check passed ++ 5. Start the server ++ :expectedresults: ++ 1. Success ++ 2. Success ++ 3. index-check returns True ++ 4. Log contains "All checks passed" ++ 5. Success ++ """ ++ from lib389.cli_ctl.dbtasks import dbtasks_index_check ++ ++ standalone = topology_st.standalone ++ ++ log.info("Stop the server") ++ standalone.stop() ++ ++ log.info("Run index-check with --fix option") ++ args = FakeArgs() ++ args.backend = None ++ args.fix = True ++ ++ result = dbtasks_index_check(standalone, topology_st.logcap.log, args) ++ # On a fresh instance, there should be no mismatch, so no reindexing needed ++ assert result is True ++ assert topology_st.logcap.contains("All checks passed") ++ topology_st.logcap.flush() ++ ++ log.info("Start the server") ++ standalone.start() ++ ++ ++def test_index_check_fixes_scanlimit(topology_st): ++ """Check if dsctl index-check --fix removes nsIndexIDListScanLimit ++ ++ :id: 4a9b2c7d-8e1f-4b3a-9c5d-6e7f8a0b1c2d ++ :setup: Standalone instance ++ :steps: ++ 1. Create DS instance ++ 2. Stop the server ++ 3. Add nsIndexIDListScanLimit to parentid index using DSEldif ++ 4. Run dsctl index-check (should detect issue) ++ 5. Run dsctl index-check --fix ++ 6. Verify nsIndexIDListScanLimit was removed ++ 7. Start the server ++ :expectedresults: ++ 1. Success ++ 2. Success ++ 3. Success ++ 4. index-check returns False and detects scanlimit ++ 5. index-check returns True after fix ++ 6. nsIndexIDListScanLimit no longer present ++ 7. Success ++ """ ++ from lib389.cli_ctl.dbtasks import dbtasks_index_check ++ from lib389.dseldif import DSEldif ++ ++ standalone = topology_st.standalone ++ ++ log.info("Stop the server") ++ standalone.stop() ++ ++ log.info("Add nsIndexIDListScanLimit to parentid index using DSEldif") ++ dse_ldif = DSEldif(standalone) ++ parentid_dn = "cn=parentid,cn=index,cn=userRoot,cn=ldbm database,cn=plugins,cn=config" ++ dse_ldif.add(parentid_dn, "nsIndexIDListScanLimit", "4000") ++ ++ # Verify it was added ++ scanlimit = dse_ldif.get(parentid_dn, "nsIndexIDListScanLimit", single=True) ++ assert scanlimit == "4000", f"Failed to add nsIndexIDListScanLimit, got: {scanlimit}" ++ log.info("Added nsIndexIDListScanLimit to parentid index") ++ ++ log.info("Run index-check without --fix (should detect issue)") ++ args = FakeArgs() ++ args.backend = "userRoot" ++ args.fix = False ++ ++ result = dbtasks_index_check(standalone, topology_st.logcap.log, args) ++ assert result is False, "index-check should detect scanlimit issue" ++ assert topology_st.logcap.contains("nsIndexIDListScanLimit") ++ topology_st.logcap.flush() ++ ++ log.info("Run index-check with --fix") ++ args.fix = True ++ result = dbtasks_index_check(standalone, topology_st.logcap.log, args) ++ assert result is True, "index-check --fix should succeed" ++ assert topology_st.logcap.contains("Removed nsIndexIDListScanLimit") ++ topology_st.logcap.flush() ++ ++ log.info("Verify nsIndexIDListScanLimit was removed") ++ dse_ldif = DSEldif(standalone) # Reload to get fresh data ++ scanlimit = dse_ldif.get(parentid_dn, "nsIndexIDListScanLimit", single=True) ++ assert scanlimit is None, f"nsIndexIDListScanLimit should be removed, but got: {scanlimit}" ++ log.info("nsIndexIDListScanLimit successfully removed") ++ ++ log.info("Start the server") ++ standalone.start() ++ ++ ++def test_index_check_fixes_ancestorid_config(topology_st): ++ """Check if dsctl index-check --fix removes ancestorid config entries ++ ++ :id: 5b0c3d8e-9f2a-4c4b-0d6e-7f8a9b1c2d3e ++ :setup: Standalone instance ++ :steps: ++ 1. Create DS instance ++ 2. Stop the server ++ 3. Add ancestorid index config entry using DSEldif ++ 4. Run dsctl index-check (should detect issue) ++ 5. Run dsctl index-check --fix ++ 6. Verify ancestorid config entry was removed ++ 7. Start the server ++ :expectedresults: ++ 1. Success ++ 2. Success ++ 3. Success ++ 4. index-check returns False and detects ancestorid config ++ 5. index-check returns True after fix ++ 6. ancestorid config entry no longer present ++ 7. Success ++ """ ++ from lib389.cli_ctl.dbtasks import dbtasks_index_check ++ from lib389.dseldif import DSEldif ++ ++ standalone = topology_st.standalone ++ ++ log.info("Stop the server") ++ standalone.stop() ++ ++ log.info("Add ancestorid index config entry using DSEldif") ++ dse_ldif = DSEldif(standalone) ++ ancestorid_entry = [ ++ "dn: cn=ancestorid,cn=index,cn=userRoot,cn=ldbm database,cn=plugins,cn=config\n", ++ "objectClass: top\n", ++ "objectClass: nsIndex\n", ++ "cn: ancestorid\n", ++ "nsSystemIndex: true\n", ++ "nsIndexType: eq\n", ++ ] ++ dse_ldif.add_entry(ancestorid_entry) ++ ++ # Verify it was added ++ ancestorid_dn = "cn=ancestorid,cn=index,cn=userRoot,cn=ldbm database,cn=plugins,cn=config" ++ dse_ldif = DSEldif(standalone) # Reload ++ cn_value = dse_ldif.get(ancestorid_dn, "cn", single=True) ++ assert cn_value is not None, "Failed to add ancestorid index config entry" ++ log.info(f"Added ancestorid index entry with cn: {cn_value}") ++ ++ log.info("Run index-check without --fix (should detect issue)") ++ args = FakeArgs() ++ args.backend = "userRoot" ++ args.fix = False ++ ++ result = dbtasks_index_check(standalone, topology_st.logcap.log, args) ++ assert result is False, "index-check should detect ancestorid config issue" ++ assert topology_st.logcap.contains("ancestorid") and topology_st.logcap.contains("config entry exists") ++ topology_st.logcap.flush() ++ ++ log.info("Run index-check with --fix") ++ args.fix = True ++ result = dbtasks_index_check(standalone, topology_st.logcap.log, args) ++ assert result is True, "index-check --fix should succeed" ++ assert topology_st.logcap.contains("Removed ancestorid config entry") ++ topology_st.logcap.flush() ++ ++ log.info("Verify ancestorid config entry was removed") ++ dse_ldif = DSEldif(standalone) # Reload to get fresh data ++ cn_value = dse_ldif.get(ancestorid_dn, "cn", single=True) ++ assert cn_value is None, f"ancestorid config entry should be removed, but got: {cn_value}" ++ log.info("ancestorid config entry successfully removed") ++ ++ log.info("Start the server") ++ standalone.start() ++ ++ ++def test_index_check_fixes_missing_matching_rule(topology_st): ++ """Check if dsctl index-check --fix adds missing integerOrderingMatch ++ ++ :id: 6c1d4e9f-0a3b-4d5c-1e7f-8a9b0c2d3e4f ++ :setup: Standalone instance ++ :steps: ++ 1. Create DS instance ++ 2. Stop the server ++ 3. Remove integerOrderingMatch from parentid index using DSEldif ++ 4. Run dsctl index-check (should detect issue) ++ 5. Run dsctl index-check --fix ++ 6. Verify integerOrderingMatch was added back ++ 7. Start the server ++ :expectedresults: ++ 1. Success ++ 2. Success ++ 3. Success ++ 4. index-check returns False and detects missing matching rule ++ 5. index-check returns True after fix ++ 6. integerOrderingMatch is present ++ 7. Success ++ """ ++ from lib389.cli_ctl.dbtasks import dbtasks_index_check ++ from lib389.dseldif import DSEldif ++ ++ standalone = topology_st.standalone ++ ++ log.info("Stop the server") ++ standalone.stop() ++ ++ log.info("Remove integerOrderingMatch from parentid index using DSEldif") ++ dse_ldif = DSEldif(standalone) ++ parentid_dn = "cn=parentid,cn=index,cn=userRoot,cn=ldbm database,cn=plugins,cn=config" ++ ++ # Check current matching rules ++ matching_rules = dse_ldif.get(parentid_dn, "nsMatchingRule") ++ log.info(f"Current matching rules: {matching_rules}") ++ ++ # Remove integerOrderingMatch if present ++ if matching_rules: ++ for mr in matching_rules: ++ if "integerorderingmatch" in mr.lower(): ++ dse_ldif.delete(parentid_dn, "nsMatchingRule", mr) ++ log.info(f"Removed matching rule: {mr}") ++ ++ # Verify it was removed ++ dse_ldif = DSEldif(standalone) # Reload ++ matching_rules = dse_ldif.get(parentid_dn, "nsMatchingRule") ++ if matching_rules: ++ for mr in matching_rules: ++ assert "integerorderingmatch" not in mr.lower(), \ ++ f"integerOrderingMatch should be removed, but found: {mr}" ++ log.info("integerOrderingMatch removed from parentid index") ++ ++ log.info("Run index-check without --fix (should detect issue)") ++ args = FakeArgs() ++ args.backend = "userRoot" ++ args.fix = False ++ ++ result = dbtasks_index_check(standalone, topology_st.logcap.log, args) ++ assert result is False, "index-check should detect missing matching rule" ++ assert topology_st.logcap.contains("missing integerOrderingMatch") ++ topology_st.logcap.flush() ++ ++ log.info("Run index-check with --fix") ++ args.fix = True ++ result = dbtasks_index_check(standalone, topology_st.logcap.log, args) ++ assert result is True, "index-check --fix should succeed" ++ assert topology_st.logcap.contains("integerOrderingMatch") ++ topology_st.logcap.flush() ++ ++ log.info("Verify integerOrderingMatch was added back") ++ dse_ldif = DSEldif(standalone) # Reload to get fresh data ++ matching_rules = dse_ldif.get(parentid_dn, "nsMatchingRule") ++ assert matching_rules is not None, "nsMatchingRule should be present" ++ found_int_order = False ++ for mr in matching_rules: ++ if "integerorderingmatch" in mr.lower(): ++ found_int_order = True ++ break ++ assert found_int_order, f"integerOrderingMatch should be present, got: {matching_rules}" ++ log.info("integerOrderingMatch successfully added back") ++ ++ log.info("Start the server") ++ standalone.start() ++ ++ ++def test_index_check_fixes_default_ancestorid(topology_st): ++ """Check if dsctl index-check --fix removes ancestorid from default indexes ++ ++ :id: 7d2e5f0a-1b4c-4e6d-2f8a-9b0c1d3e4f5a ++ :setup: Standalone instance ++ :steps: ++ 1. Create DS instance ++ 2. Stop the server ++ 3. Add ancestorid to cn=default indexes using DSEldif ++ 4. Run dsctl index-check (should detect issue) ++ 5. Run dsctl index-check --fix ++ 6. Verify ancestorid was removed from default indexes ++ 7. Start the server ++ :expectedresults: ++ 1. Success ++ 2. Success ++ 3. Success ++ 4. index-check returns False and detects ancestorid in default indexes ++ 5. index-check returns True after fix ++ 6. ancestorid no longer in default indexes ++ 7. Success ++ """ ++ from lib389.cli_ctl.dbtasks import dbtasks_index_check ++ from lib389.dseldif import DSEldif ++ ++ standalone = topology_st.standalone ++ ++ log.info("Stop the server") ++ standalone.stop() ++ ++ log.info("Add ancestorid to cn=default indexes using DSEldif") ++ dse_ldif = DSEldif(standalone) ++ ancestorid_default_entry = [ ++ "dn: cn=ancestorid,cn=default indexes,cn=config,cn=ldbm database,cn=plugins,cn=config\n", ++ "objectClass: top\n", ++ "objectClass: nsIndex\n", ++ "cn: ancestorid\n", ++ "nsSystemIndex: true\n", ++ "nsIndexType: eq\n", ++ ] ++ dse_ldif.add_entry(ancestorid_default_entry) ++ ++ # Verify it was added ++ ancestorid_default_dn = "cn=ancestorid,cn=default indexes,cn=config,cn=ldbm database,cn=plugins,cn=config" ++ dse_ldif = DSEldif(standalone) # Reload ++ cn_value = dse_ldif.get(ancestorid_default_dn, "cn", single=True) ++ assert cn_value is not None, "Failed to add ancestorid to default indexes" ++ log.info(f"Added ancestorid to default indexes with cn: {cn_value}") ++ ++ log.info("Run index-check without --fix (should detect issue)") ++ args = FakeArgs() ++ args.backend = None # Check all backends including default indexes ++ args.fix = False ++ ++ result = dbtasks_index_check(standalone, topology_st.logcap.log, args) ++ assert result is False, "index-check should detect ancestorid in default indexes" ++ assert topology_st.logcap.contains("ancestorid found in cn=default indexes") ++ topology_st.logcap.flush() ++ ++ log.info("Run index-check with --fix") ++ args.fix = True ++ result = dbtasks_index_check(standalone, topology_st.logcap.log, args) ++ assert result is True, "index-check --fix should succeed" ++ assert topology_st.logcap.contains("Removed ancestorid from default indexes") ++ topology_st.logcap.flush() ++ ++ log.info("Verify ancestorid was removed from default indexes") ++ dse_ldif = DSEldif(standalone) # Reload to get fresh data ++ cn_value = dse_ldif.get(ancestorid_default_dn, "cn", single=True) ++ assert cn_value is None, f"ancestorid should be removed from default indexes, but got: {cn_value}" ++ log.info("ancestorid successfully removed from default indexes") ++ ++ log.info("Start the server") ++ standalone.start() ++ ++ ++def test_index_check_fixes_multiple_issues(topology_st): ++ """Check if dsctl index-check --fix handles multiple issues at once ++ ++ :id: 8e3f6a1b-2c5d-4f7e-3a9b-0c1d2e4f5a6b ++ :setup: Standalone instance ++ :steps: ++ 1. Create DS instance ++ 2. Stop the server ++ 3. Add multiple issues: scanlimit, ancestorid config, missing matching rule ++ 4. Run dsctl index-check (should detect all issues) ++ 5. Run dsctl index-check --fix ++ 6. Verify all issues were fixed ++ 7. Run dsctl index-check again (should pass) ++ 8. Start the server ++ :expectedresults: ++ 1. Success ++ 2. Success ++ 3. Success ++ 4. index-check returns False and detects all issues ++ 5. index-check returns True after fix ++ 6. All issues resolved ++ 7. index-check returns True (no issues) ++ 8. Success ++ """ ++ from lib389.cli_ctl.dbtasks import dbtasks_index_check ++ from lib389.dseldif import DSEldif ++ ++ standalone = topology_st.standalone ++ ++ log.info("Stop the server") ++ standalone.stop() ++ ++ dse_ldif = DSEldif(standalone) ++ parentid_dn = "cn=parentid,cn=index,cn=userRoot,cn=ldbm database,cn=plugins,cn=config" ++ ancestorid_dn = "cn=ancestorid,cn=index,cn=userRoot,cn=ldbm database,cn=plugins,cn=config" ++ ++ log.info("Add issue 1: nsIndexIDListScanLimit to parentid") ++ dse_ldif.add(parentid_dn, "nsIndexIDListScanLimit", "4000") ++ ++ log.info("Add issue 2: ancestorid index config entry") ++ ancestorid_entry = [ ++ f"dn: {ancestorid_dn}\n", ++ "objectClass: top\n", ++ "objectClass: nsIndex\n", ++ "cn: ancestorid\n", ++ "nsSystemIndex: true\n", ++ "nsIndexType: eq\n", ++ ] ++ dse_ldif.add_entry(ancestorid_entry) ++ ++ log.info("Add issue 3: Remove integerOrderingMatch from parentid") ++ dse_ldif = DSEldif(standalone) # Reload ++ matching_rules = dse_ldif.get(parentid_dn, "nsMatchingRule") ++ if matching_rules: ++ for mr in matching_rules: ++ if "integerorderingmatch" in mr.lower(): ++ dse_ldif.delete(parentid_dn, "nsMatchingRule", mr) ++ ++ log.info("Run index-check without --fix (should detect all issues)") ++ args = FakeArgs() ++ args.backend = "userRoot" ++ args.fix = False ++ ++ result = dbtasks_index_check(standalone, topology_st.logcap.log, args) ++ assert result is False, "index-check should detect multiple issues" ++ # Check that multiple issues were detected ++ assert topology_st.logcap.contains("nsIndexIDListScanLimit") ++ assert topology_st.logcap.contains("ancestorid") ++ topology_st.logcap.flush() ++ ++ log.info("Run index-check with --fix") ++ args.fix = True ++ result = dbtasks_index_check(standalone, topology_st.logcap.log, args) ++ assert result is True, "index-check --fix should succeed" ++ assert topology_st.logcap.contains("All issues fixed") ++ topology_st.logcap.flush() ++ ++ log.info("Verify all issues were fixed") ++ dse_ldif = DSEldif(standalone) # Reload ++ ++ # Check scanlimit removed ++ scanlimit = dse_ldif.get(parentid_dn, "nsIndexIDListScanLimit", single=True) ++ assert scanlimit is None, f"nsIndexIDListScanLimit should be removed, got: {scanlimit}" ++ ++ # Check ancestorid config removed ++ cn_value = dse_ldif.get(ancestorid_dn, "cn", single=True) ++ assert cn_value is None, f"ancestorid config should be removed, got: {cn_value}" ++ ++ # Check matching rule added back ++ matching_rules = dse_ldif.get(parentid_dn, "nsMatchingRule") ++ found_int_order = False ++ if matching_rules: ++ for mr in matching_rules: ++ if "integerorderingmatch" in mr.lower(): ++ found_int_order = True ++ break ++ assert found_int_order, f"integerOrderingMatch should be present, got: {matching_rules}" ++ ++ log.info("All issues verified as fixed") ++ ++ log.info("Run index-check again to confirm all clear") ++ args.fix = False ++ result = dbtasks_index_check(standalone, topology_st.logcap.log, args) ++ assert result is True, "index-check should pass after fix" ++ assert topology_st.logcap.contains("All checks passed") ++ topology_st.logcap.flush() ++ ++ log.info("Start the server") ++ standalone.start() ++ ++ + if __name__ == "__main__": + # Run isolated + # -s for DEBUG mode +diff --git a/rpm/389-ds-base.spec.in b/rpm/389-ds-base.spec.in +index b8c14cd14..16b8d14c5 100644 +--- a/rpm/389-ds-base.spec.in ++++ b/rpm/389-ds-base.spec.in +@@ -507,7 +507,45 @@ if ! getent passwd $USERNAME >/dev/null ; then + fi + + # Reload our sysctl before we restart (if we can) +-sysctl --system &> $output; true ++sysctl --system &> "$output"; true ++ ++# Gather running instances, stop them, run index-check, then restart ++instbase="%{_sysconfdir}/%{pkgname}" ++instances="" ++ninst=0 ++ ++for dir in "$instbase"/slapd-* ; do ++ echo "dir = $dir" >> "$output" 2>&1 || : ++ if [ ! -d "$dir" ] ; then continue ; fi ++ case "$dir" in *.removed) continue ;; esac ++ basename=$(basename "$dir") ++ inst="%{pkgname}@${basename#slapd-}" ++ inst_name="${basename#slapd-}" ++ echo "found instance $inst - getting status" >> "$output" 2>&1 || : ++ if /bin/systemctl -q is-active "$inst" ; then ++ echo "instance $inst is running - stopping for upgrade" >> "$output" 2>&1 || : ++ instances="$instances $inst" ++ /bin/systemctl stop "$inst" >> "$output" 2>&1 || : ++ else ++ echo "instance $inst is not running" >> "$output" 2>&1 || : ++ fi ++ # Run index-check on all instances (running or not) ++ # This fixes index ordering mismatches from older versions ++ dsctl "$inst_name" index-check --fix >> "$output2" 2>&1 || : ++ ninst=$((ninst + 1)) ++done ++ ++if [ $ninst -eq 0 ] ; then ++ echo "no instances to upgrade" >> "$output" 2>&1 || : ++ exit 0 ++fi ++ ++# Restart previously running instances ++for inst in $instances ; do ++ echo "starting instance $inst" >> "$output" 2>&1 || : ++ /bin/systemctl start "$inst" >> "$output" 2>&1 || : ++done ++ + + %preun + if [ $1 -eq 0 ]; then # Final removal +diff --git a/src/lib389/lib389/cli_ctl/dbtasks.py b/src/lib389/lib389/cli_ctl/dbtasks.py +index 856639672..16da966d1 100644 +--- a/src/lib389/lib389/cli_ctl/dbtasks.py ++++ b/src/lib389/lib389/cli_ctl/dbtasks.py +@@ -7,12 +7,24 @@ + # See LICENSE for details. + # --- END COPYRIGHT BLOCK --- + ++import glob + import os ++import re ++import subprocess ++from enum import Enum + from lib389._constants import TaskWarning + from lib389.cli_base import CustomHelpFormatter ++from lib389.dseldif import DSEldif + from pathlib import Path + + ++class IndexOrdering(Enum): ++ """Represents the ordering type of an index.""" ++ INTEGER = "integer" ++ LEXICOGRAPHIC = "lexicographic" ++ UNKNOWN = "unknown" ++ ++ + def dbtasks_db2index(inst, log, args): + rtn = False + if not args.backend: +@@ -126,6 +138,387 @@ def dbtasks_verify(inst, log, args): + log.info("dbverify successful") + + ++def _get_db_dir(dse_ldif): ++ """Get the database directory. ++ ++ Args: ++ dse_ldif: DSEldif instance. ++ ++ Returns: ++ Path to the database directory, or None if not found. ++ """ ++ try: ++ db_dir = dse_ldif.get( ++ "cn=config,cn=ldbm database,cn=plugins,cn=config", ++ "nsslapd-directory", ++ single=True, ++ ) ++ return db_dir ++ except (ValueError, TypeError): ++ pass ++ return None ++ ++ ++ ++def _has_integer_ordering_match(dse_ldif, backend, index_name): ++ """Check if an index has integerOrderingMatch configured in DSE. ++ ++ Args: ++ dse_ldif: DSEldif instance. ++ backend: Backend name. ++ index_name: Name of the index to check. ++ ++ Returns: ++ True if integerOrderingMatch is configured, False otherwise. ++ """ ++ index_dn = "cn={},cn=index,cn={},cn=ldbm database,cn=plugins,cn=config".format( ++ index_name, backend ++ ) ++ matching_rules = dse_ldif.get(index_dn, "nsMatchingRule", lower=True) ++ if matching_rules: ++ return any(mr.lower() == "integerorderingmatch" for mr in matching_rules) ++ return False ++ ++ ++def _has_index_scan_limit(dse_ldif, backend, index_name): ++ """Check if an index has nsIndexIDListScanLimit configured. ++ ++ Args: ++ dse_ldif: DSEldif instance. ++ backend: Backend name. ++ index_name: Name of the index to check. ++ ++ Returns: ++ True if nsIndexIDListScanLimit is configured, False otherwise. ++ """ ++ index_dn = "cn={},cn=index,cn={},cn=ldbm database,cn=plugins,cn=config".format( ++ index_name, backend ++ ) ++ scan_limit = dse_ldif.get(index_dn, "nsIndexIDListScanLimit") ++ return scan_limit is not None ++ ++ ++def _index_config_exists(dse_ldif, backend, index_name): ++ """Check if an index configuration entry exists in DSE. ++ ++ Args: ++ dse_ldif: DSEldif instance. ++ backend: Backend name. ++ index_name: Name of the index to check. ++ ++ Returns: ++ True if the index config entry exists, False otherwise. ++ """ ++ index_dn = "cn={},cn=index,cn={},cn=ldbm database,cn=plugins,cn=config".format( ++ index_name, backend ++ ) ++ try: ++ cn = dse_ldif.get(index_dn, "cn") ++ return cn is not None ++ except (ValueError, KeyError): ++ return False ++ ++ ++def _default_index_exists(dse_ldif, index_name): ++ """Check if an index exists in cn=default indexes. ++ ++ Args: ++ dse_ldif: DSEldif instance. ++ index_name: Name of the index to check. ++ ++ Returns: ++ True if the index exists in default indexes, False otherwise. ++ """ ++ index_dn = "cn={},cn=default indexes,cn=config,cn=ldbm database,cn=plugins,cn=config".format( ++ index_name ++ ) ++ try: ++ cn = dse_ldif.get(index_dn, "cn") ++ return cn is not None ++ except (ValueError, KeyError): ++ return False ++ ++ ++def _check_disk_ordering(db_dir, backend, index_name, dbscan_path, is_mdb, log): ++ """Check if index on disk uses lexicographic or integer ordering. ++ ++ Args: ++ db_dir: Path to the database directory. ++ backend: Backend name. ++ index_name: Name of the index to check. ++ dbscan_path: Path to the dbscan binary. ++ is_mdb: True if using MDB backend. ++ log: Logger instance. ++ ++ Returns: ++ IndexOrdering: The detected ordering type. ++ """ ++ if is_mdb: ++ # MDB uses pseudo-paths: db_dir/backend/index.db ++ # dbscan accesses indexes via paths like: /var/lib/dirsrv/slapd-xxx/db/userroot/parentid.db ++ index_file = os.path.join(db_dir, backend, "{}.db".format(index_name)) ++ else: ++ # BDB has separate directories per backend with actual index files ++ backend_dir = os.path.join(db_dir, backend) ++ if not os.path.exists(backend_dir): ++ return IndexOrdering.UNKNOWN ++ index_file = None ++ pattern = os.path.join(backend_dir, "{}.db*".format(index_name)) ++ for f in glob.glob(pattern): ++ if os.path.isfile(f): ++ index_file = f ++ break ++ if not index_file: ++ return IndexOrdering.UNKNOWN ++ ++ try: ++ result = subprocess.run( ++ [dbscan_path, "-f", index_file], ++ stdout=subprocess.PIPE, ++ stderr=subprocess.PIPE, ++ universal_newlines=True, ++ timeout=60, ++ ) ++ ++ if result.returncode != 0: ++ log.warning(" dbscan returned non-zero exit code for %s", index_file) ++ return IndexOrdering.UNKNOWN ++ ++ # Parse keys from dbscan output ++ keys = [] ++ for line in result.stdout.split("\n"): ++ line = line.strip() ++ if line.startswith("="): ++ match = re.match(r"^=(\d+)", line) ++ if match: ++ keys.append(int(match.group(1))) ++ ++ if len(keys) < 2: ++ return IndexOrdering.UNKNOWN ++ ++ # Check if keys are in integer order by looking for decreasing numeric values ++ # (which would indicate lexicographic ordering, e.g., "3" < "30" < "4") ++ prev_id = keys[0] ++ for i in range(1, min(len(keys), 100)): ++ current_id = keys[i] ++ if prev_id > current_id: ++ return IndexOrdering.LEXICOGRAPHIC ++ prev_id = current_id ++ ++ return IndexOrdering.INTEGER ++ ++ except subprocess.TimeoutExpired: ++ log.warning(" dbscan timed out for %s", index_file) ++ return IndexOrdering.UNKNOWN ++ except OSError as e: ++ log.warning(" Error running dbscan: %s", e) ++ return IndexOrdering.UNKNOWN ++ ++ ++def dbtasks_index_check(inst, log, args): ++ """Check and optionally fix index ordering mismatches. ++ ++ This function detects mismatches between the configured ordering ++ (integerOrderingMatch in DSE) and the actual on-disk ordering of ++ parentid and ancestorid indexes. ++ ++ Args: ++ inst: DirSrv instance. ++ log: Logger instance. ++ args: Parsed command line arguments. ++ ++ Returns: ++ True if all checks passed, False if mismatches were detected. ++ """ ++ # Server must be stopped ++ if inst.status(): ++ log.error("index-check requires the instance to be stopped") ++ return False ++ ++ # Check for dbscan binary ++ dbscan_path = os.path.join(inst.ds_paths.bin_dir, "dbscan") ++ if not os.path.exists(dbscan_path): ++ log.error("dbscan utility not found at %s", dbscan_path) ++ return False ++ ++ # Load DSE ++ try: ++ dse_ldif = DSEldif(inst) ++ except Exception as e: ++ log.error("Failed to read dse.ldif: %s", e) ++ return False ++ ++ # Get backends to check ++ all_backends = dse_ldif.get_backends() ++ if not all_backends: ++ log.info("No backends found") ++ return True ++ ++ # Filter to specific backend if requested ++ if args.backend: ++ # Case-insensitive backend lookup ++ backend_lower = args.backend.lower() ++ matching_backend = None ++ for be in all_backends: ++ if be.lower() == backend_lower: ++ matching_backend = be ++ break ++ if matching_backend is None: ++ log.error("Backend '%s' not found. Available backends: %s", ++ args.backend, ", ".join(all_backends)) ++ return False ++ backends_to_check = [matching_backend] ++ else: ++ backends_to_check = all_backends ++ ++ # Get database directory and check database type ++ db_dir = _get_db_dir(dse_ldif) ++ if not db_dir or not os.path.exists(db_dir): ++ log.error("Database directory not found") ++ return False ++ ++ db_lib = inst.get_db_lib() ++ is_mdb = (db_lib == "mdb") ++ log.info("Database type: %s", db_lib.upper()) ++ ++ # Track all issues found ++ all_ok = True ++ mismatches = [] # (backend, index_name) tuples needing reindex ++ missing_matching_rules = [] # (backend, index_name) tuples missing integerOrderingMatch ++ scan_limits_to_remove = [] # (backend, index_name) tuples with nsIndexIDListScanLimit ++ ancestorid_configs_to_remove = [] # backend names with ancestorid config entries ++ remove_ancestorid_from_defaults = False # Flag to remove from cn=default indexes ++ ++ # Check if ancestorid exists in cn=default indexes (should be removed) ++ if _default_index_exists(dse_ldif, "ancestorid"): ++ log.warning("ancestorid found in cn=default indexes - should be removed") ++ remove_ancestorid_from_defaults = True ++ all_ok = False ++ ++ for backend in backends_to_check: ++ log.info("Checking backend: %s", backend) ++ ++ # Check for ancestorid config entry (should not exist) ++ if _index_config_exists(dse_ldif, backend, "ancestorid"): ++ log.warning(" ancestorid - config entry exists (should be removed)") ++ ancestorid_configs_to_remove.append(backend) ++ all_ok = False ++ ++ # Check parentid and ancestorid indexes ++ for index_name in ["parentid", "ancestorid"]: ++ # Check for scan limits (should be removed) ++ if _has_index_scan_limit(dse_ldif, backend, index_name): ++ log.warning(" %s - has nsIndexIDListScanLimit (should be removed)", index_name) ++ scan_limits_to_remove.append((backend, index_name)) ++ all_ok = False ++ ++ # Check disk ordering ++ disk_ordering = _check_disk_ordering(db_dir, backend, index_name, dbscan_path, is_mdb, log) ++ ++ if disk_ordering == IndexOrdering.UNKNOWN: ++ log.info(" %s - could not determine disk ordering, skipping", index_name) ++ # For parentid, still check if matching rule is missing ++ if index_name == "parentid": ++ config_has_int_order = _has_integer_ordering_match(dse_ldif, backend, index_name) ++ if not config_has_int_order: ++ log.warning(" %s - missing integerOrderingMatch in config", index_name) ++ missing_matching_rules.append((backend, index_name)) ++ all_ok = False ++ continue ++ ++ config_has_int_order = _has_integer_ordering_match(dse_ldif, backend, index_name) ++ config_desc = "integer" if config_has_int_order else "lexicographic" ++ log.info(" %s - config: %s, disk: %s", ++ index_name, config_desc, disk_ordering.value) ++ ++ # For parentid, the desired state is always integer ordering ++ if index_name == "parentid": ++ if not config_has_int_order: ++ log.warning(" %s - missing integerOrderingMatch in config", index_name) ++ if (backend, index_name) not in missing_matching_rules: ++ missing_matching_rules.append((backend, index_name)) ++ all_ok = False ++ ++ if disk_ordering == IndexOrdering.LEXICOGRAPHIC: ++ log.warning(" %s - disk ordering is lexicographic, needs reindex", index_name) ++ if (backend, index_name) not in mismatches: ++ mismatches.append((backend, index_name)) ++ all_ok = False ++ ++ # Handle issues ++ if not all_ok: ++ if args.fix: ++ log.info("Fixing issues...") ++ ++ # Remove ancestorid from cn=default indexes ++ if remove_ancestorid_from_defaults: ++ default_idx_dn = "cn=ancestorid,cn=default indexes,cn=config,cn=ldbm database,cn=plugins,cn=config" ++ log.info(" Removing ancestorid from default indexes...") ++ try: ++ dse_ldif.delete_dn(default_idx_dn) ++ log.info(" Removed ancestorid from default indexes") ++ except Exception as e: ++ log.error(" Failed to remove ancestorid from default indexes: %s", e) ++ return False ++ ++ # Remove scan limits (only for indexes that won't be deleted) ++ for backend, index_name in scan_limits_to_remove: ++ # Skip ancestorid if we're going to delete the whole entry anyway ++ if index_name == "ancestorid" and backend in ancestorid_configs_to_remove: ++ continue ++ index_dn = "cn={},cn=index,cn={},cn=ldbm database,cn=plugins,cn=config".format( ++ index_name, backend ++ ) ++ log.info(" Removing nsIndexIDListScanLimit from %s in backend %s...", index_name, backend) ++ try: ++ dse_ldif.delete(index_dn, "nsIndexIDListScanLimit") ++ log.info(" Removed nsIndexIDListScanLimit from %s", index_name) ++ except Exception as e: ++ log.error(" Failed to remove nsIndexIDListScanLimit from %s: %s", index_name, e) ++ return False ++ ++ # Remove ancestorid config entries from backends ++ for backend in ancestorid_configs_to_remove: ++ index_dn = "cn=ancestorid,cn=index,cn={},cn=ldbm database,cn=plugins,cn=config".format(backend) ++ log.info(" Removing ancestorid config entry from backend %s...", backend) ++ try: ++ dse_ldif.delete_dn(index_dn) ++ log.info(" Removed ancestorid config entry from backend %s", backend) ++ except Exception as e: ++ log.error(" Failed to remove ancestorid config from backend %s: %s", backend, e) ++ return False ++ ++ # Add missing matching rules to dse.ldif ++ for backend, index_name in missing_matching_rules: ++ index_dn = "cn={},cn=index,cn={},cn=ldbm database,cn=plugins,cn=config".format( ++ index_name, backend ++ ) ++ log.info(" Adding integerOrderingMatch to %s in backend %s...", index_name, backend) ++ try: ++ dse_ldif.add(index_dn, "nsMatchingRule", "integerOrderingMatch") ++ log.info(" Updated dse.ldif with integerOrderingMatch for %s", index_name) ++ except Exception as e: ++ log.error(" Failed to update dse.ldif for %s: %s", index_name, e) ++ return False ++ ++ # Reindex indexes with disk ordering issues ++ for backend, index_name in mismatches: ++ log.info(" Reindexing %s in backend %s...", index_name, backend) ++ if not inst.db2index(bename=backend, attrs=[index_name]): ++ log.error(" Failed to reindex %s", index_name) ++ return False ++ log.info(" Reindex of %s completed successfully", index_name) ++ ++ log.info("All issues fixed") ++ return True ++ else: ++ log.info("Issues detected. Run with --fix to repair.") ++ return False ++ else: ++ log.info("All checks passed - no issues found") ++ return True ++ ++ + def create_parser(subcommands): + db2index_parser = subcommands.add_parser('db2index', help="Initialise a reindex of the server database. The server must be stopped for this to proceed.", formatter_class=CustomHelpFormatter) + # db2index_parser.add_argument('suffix', help="The suffix to reindex. IE dc=example,dc=com.") +@@ -172,3 +565,12 @@ def create_parser(subcommands): + ldifs_parser = subcommands.add_parser('ldifs', help="List all the LDIF files located in the server's LDIF directory", formatter_class=CustomHelpFormatter) + ldifs_parser.add_argument('--delete', nargs=1, help="Delete LDIF file") + ldifs_parser.set_defaults(func=dbtasks_ldifs) ++ ++ index_check_parser = subcommands.add_parser('index-check', ++ help="Check for index ordering mismatches (parentid/ancestorid). The server must be stopped.", ++ formatter_class=CustomHelpFormatter) ++ index_check_parser.add_argument('backend', nargs='?', default=None, ++ help="Backend to check. If not specified, all backends are checked.") ++ index_check_parser.add_argument('--fix', action='store_true', default=False, ++ help="Fix mismatches by reindexing affected indexes") ++ index_check_parser.set_defaults(func=dbtasks_index_check) +diff --git a/src/lib389/lib389/dseldif.py b/src/lib389/lib389/dseldif.py +index 3104a7b6f..49efc5dcb 100644 +--- a/src/lib389/lib389/dseldif.py ++++ b/src/lib389/lib389/dseldif.py +@@ -118,11 +118,19 @@ class DSEldif(DSLint): + with open(self.path, "w") as file_dse: + file_dse.write("".join(self._contents)) + +- def _find_attr(self, entry_dn, attr): ++ def globalSubstitute(self, strfrom, strto): ++ for i in range(0, len(self._contents)-1): ++ self._contents[i] = self._contents[i].replace(strfrom, strto) ++ self._update() ++ ++ def _find_attr(self, entry_dn, attr, lower=False): + """Find all attribute values and indexes under a given entry + + Returns entry dn index and attribute data dict: + relative attribute indexes and the attribute value ++ ++ :param lower: Use case-insensitive matching for attribute name ++ :type lower: boolean + """ + + entry_dn_i = self._contents.index("dn: {}\n".format(entry_dn.lower())) +@@ -139,7 +147,11 @@ class DSEldif(DSLint): + + # Find the attribute + for line in entry_slice: +- if line.startswith("{}:".format(attr)): ++ if lower: ++ match = line.lower().startswith("{}:".format(attr.lower())) ++ else: ++ match = line.startswith("{}:".format(attr)) ++ if match: + attr_value = line.split(" ", 1)[1][:-1] + attr_data.update({entry_slice.index(line): attr_value}) + +@@ -148,7 +160,7 @@ class DSEldif(DSLint): + + return entry_dn_i, attr_data + +- def get(self, entry_dn, attr, single=False): ++ def get(self, entry_dn, attr, single=False, lower=False): + """Return attribute values under a given entry + + :param entry_dn: a DN of entry we want to get attribute from +@@ -156,11 +168,13 @@ class DSEldif(DSLint): + :param attr: an attribute name + :type attr: str + :param single: Return a single value instead of a list +- :type sigle: boolean ++ :type single: boolean ++ :param lower: Use case-insensitive matching for attribute name ++ :type lower: boolean + """ + + try: +- _, attr_data = self._find_attr(entry_dn, attr) ++ _, attr_data = self._find_attr(entry_dn, attr, lower=lower) + except ValueError: + return None + +@@ -183,6 +197,38 @@ class DSEldif(DSLint): + + return indexes + ++ def get_backends(self): ++ """Return a list of backend names from DSE. ++ ++ Returns backend names preserving their original case, as the ++ database directory names on disk use the original case. ++ ++ Note: DSEldif lowercases DN lines, so we read the 'cn' attribute ++ from each entry to get the original case. ++ ++ :returns: List of backend names ++ """ ++ backends = [] ++ excluded = ("config", "monitor", "index", "encrypted attributes") ++ ++ for entry in self._contents: ++ if (entry.startswith("dn: cn=") and ++ ",cn=ldbm database,cn=plugins,cn=config" in entry): ++ parts = entry.split(",") ++ if len(parts) > 1: ++ cn_lower = parts[0].replace("dn: cn=", "") ++ if cn_lower not in excluded: ++ dn = entry.strip()[4:].strip() ++ try: ++ suffix = self.get(dn, "nsslapd-suffix") ++ if suffix: ++ cn_values = self.get(dn, "cn") ++ if cn_values: ++ backends.append(cn_values[0]) ++ except (ValueError, IndexError): ++ pass ++ ++ return list(set(backends)) + + def add_entry(self, entry): + """Add a new entry +-- +2.52.0 + diff --git a/SOURCES/0084-Issue-7223-Use-lexicographical-order-for-ancestorid.patch b/SOURCES/0084-Issue-7223-Use-lexicographical-order-for-ancestorid.patch new file mode 100644 index 0000000..367f8a2 --- /dev/null +++ b/SOURCES/0084-Issue-7223-Use-lexicographical-order-for-ancestorid.patch @@ -0,0 +1,34 @@ +From 384dbcaba418afe9e4de798d16a68d0468632a57 Mon Sep 17 00:00:00 2001 +From: Viktor Ashirov +Date: Fri, 13 Feb 2026 13:08:40 +0100 +Subject: [PATCH] Issue 7223 - Use lexicographical order for ancestorid + +Description: +`ldbm_instance_create_default_indexes()` configured ancestorid with +integerOrderingMatch in the in-memory attrinfo, but ancestorid on disk +might be using lexicographic ordering (data before the upgrade or after +ldif2db import). + +Relates: https://github.com/389ds/389-ds-base/issues/7223 + +Reviewed by: @progier389, @tbordaz (Thanks!) +--- + ldap/servers/slapd/back-ldbm/instance.c | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/ldap/servers/slapd/back-ldbm/instance.c b/ldap/servers/slapd/back-ldbm/instance.c +index 388dd6efb..f5342b99a 100644 +--- a/ldap/servers/slapd/back-ldbm/instance.c ++++ b/ldap/servers/slapd/back-ldbm/instance.c +@@ -239,7 +239,7 @@ ldbm_instance_create_default_indexes(backend *be) + * ancestorid is special, there is actually no such attr type + * but we still want to use the attr index file APIs. + */ +- e = ldbm_instance_init_config_entry(LDBM_ANCESTORID_STR, "eq", 0, 0, 0, "integerOrderingMatch"); ++ e = ldbm_instance_init_config_entry(LDBM_ANCESTORID_STR, "eq", 0, 0, 0, 0); + attr_index_config(be, "ldbm index init", 0, e, 1, 0, NULL); + slapi_entry_free(e); + } +-- +2.52.0 + diff --git a/SOURCES/0085-Issue-7223-Remove-integerOrderingMatch-requirement-f.patch b/SOURCES/0085-Issue-7223-Remove-integerOrderingMatch-requirement-f.patch new file mode 100644 index 0000000..0327a12 --- /dev/null +++ b/SOURCES/0085-Issue-7223-Remove-integerOrderingMatch-requirement-f.patch @@ -0,0 +1,537 @@ +From 67c3183380888f4af093b546e717f3e7451a41d6 Mon Sep 17 00:00:00 2001 +From: Viktor Ashirov +Date: Wed, 18 Feb 2026 09:26:57 +0100 +Subject: [PATCH] Issue 7223 - Remove integerOrderingMatch requirement for + parentid (#7264) + +Description: +integerOrderingMatch was introduced as a requirement for parentid and +ancestorid indexes for performance reasons. But after #7096 the order +for parentid doesn't make a lot of difference. + +Fix Description: +* Remove integerOrderingMatch requirement for parentid. +* Read only first 100 keys from dbscan in index ordering check +* Do not run dsctl index-check during RPM upgrade + +Relates: https://github.com/389ds/389-ds-base/pull/7223 + +Reviewed by: @progier389, @tbordaz (Thanks!) +--- + .../healthcheck/health_system_indexes_test.py | 83 ++++---------- + ldap/servers/slapd/upgrade.c | 106 ------------------ + rpm/389-ds-base.spec.in | 3 - + src/lib389/lib389/backend.py | 5 +- + src/lib389/lib389/cli_ctl/dbtasks.py | 99 ++++++++-------- + 5 files changed, 73 insertions(+), 223 deletions(-) + +diff --git a/dirsrvtests/tests/suites/healthcheck/health_system_indexes_test.py b/dirsrvtests/tests/suites/healthcheck/health_system_indexes_test.py +index ce86239e5..088b48587 100644 +--- a/dirsrvtests/tests/suites/healthcheck/health_system_indexes_test.py ++++ b/dirsrvtests/tests/suites/healthcheck/health_system_indexes_test.py +@@ -179,7 +179,8 @@ def test_missing_parentid(topology_st, log_buffering_enabled): + + + def test_missing_matching_rule(topology_st, log_buffering_enabled): +- """Check if healthcheck returns DSBLE0007 code when parentId index is missing integerOrderingMatch ++ """Check that healthcheck does NOT report DSBLE0007 when parentId index is missing integerOrderingMatch. ++ Both lexicographic and integer orderings are valid for parentid. + + :id: 7ffa71db-8995-430a-bed8-59bce944221c + :setup: Standalone instance +@@ -189,19 +190,14 @@ def test_missing_matching_rule(topology_st, log_buffering_enabled): + 3. Use healthcheck without --json option + 4. Use healthcheck with --json option + 5. Re-add the matching rule +- 6. Use healthcheck without --json option +- 7. Use healthcheck with --json option + :expectedresults: + 1. Success + 2. Success +- 3. healthcheck reports DSBLE0007 code and related details +- 4. healthcheck reports DSBLE0007 code and related details ++ 3. healthcheck reports no issues found ++ 4. healthcheck reports no issues found + 5. Success +- 6. healthcheck reports no issues found +- 7. healthcheck reports no issues found + """ + +- RET_CODE = "DSBLE0007" + PARENTID_DN = "cn=parentid,cn=index,cn=userroot,cn=ldbm database,cn=plugins,cn=config" + + standalone = topology_st.standalone +@@ -210,16 +206,13 @@ def test_missing_matching_rule(topology_st, log_buffering_enabled): + parentid_index = Index(standalone, PARENTID_DN) + parentid_index.remove("nsMatchingRule", "integerOrderingMatch") + +- run_healthcheck_and_flush_log(topology_st, standalone, json=False, searched_code=RET_CODE) +- run_healthcheck_and_flush_log(topology_st, standalone, json=True, searched_code=RET_CODE) ++ run_healthcheck_and_flush_log(topology_st, standalone, json=False, searched_code=CMD_OUTPUT) ++ run_healthcheck_and_flush_log(topology_st, standalone, json=True, searched_code=JSON_OUTPUT) + + log.info("Re-add the integerOrderingMatch matching rule") + parentid_index = Index(standalone, PARENTID_DN) + parentid_index.add("nsMatchingRule", "integerOrderingMatch") + +- run_healthcheck_and_flush_log(topology_st, standalone, json=False, searched_code=CMD_OUTPUT) +- run_healthcheck_and_flush_log(topology_st, standalone, json=True, searched_code=JSON_OUTPUT) +- + + def test_usn_plugin_missing_entryusn(topology_st, usn_plugin_enabled, log_buffering_enabled): + """Check if healthcheck returns DSBLE0007 code when USN plugin is enabled but entryusn index is missing +@@ -908,7 +901,9 @@ def test_index_check_fixes_ancestorid_config(topology_st): + + + def test_index_check_fixes_missing_matching_rule(topology_st): +- """Check if dsctl index-check --fix adds missing integerOrderingMatch ++ """Check that removing integerOrderingMatch from parentid config is not ++ flagged as an issue when disk ordering cannot be determined. ++ Both lexicographic and integer orderings are valid for parentid. + + :id: 6c1d4e9f-0a3b-4d5c-1e7f-8a9b0c2d3e4f + :setup: Standalone instance +@@ -916,18 +911,14 @@ def test_index_check_fixes_missing_matching_rule(topology_st): + 1. Create DS instance + 2. Stop the server + 3. Remove integerOrderingMatch from parentid index using DSEldif +- 4. Run dsctl index-check (should detect issue) +- 5. Run dsctl index-check --fix +- 6. Verify integerOrderingMatch was added back +- 7. Start the server ++ 4. Run dsctl index-check (should NOT detect issue since disk ordering is unknown) ++ 5. Start the server + :expectedresults: + 1. Success + 2. Success + 3. Success +- 4. index-check returns False and detects missing matching rule +- 5. index-check returns True after fix +- 6. integerOrderingMatch is present +- 7. Success ++ 4. index-check returns True (no issues, disk ordering unknown) ++ 5. Success + """ + from lib389.cli_ctl.dbtasks import dbtasks_index_check + from lib389.dseldif import DSEldif +@@ -961,34 +952,20 @@ def test_index_check_fixes_missing_matching_rule(topology_st): + f"integerOrderingMatch should be removed, but found: {mr}" + log.info("integerOrderingMatch removed from parentid index") + +- log.info("Run index-check without --fix (should detect issue)") ++ log.info("Run index-check (should NOT detect issue - disk ordering unknown)") + args = FakeArgs() + args.backend = "userRoot" + args.fix = False + + result = dbtasks_index_check(standalone, topology_st.logcap.log, args) +- assert result is False, "index-check should detect missing matching rule" +- assert topology_st.logcap.contains("missing integerOrderingMatch") ++ assert result is True, \ ++ "index-check should not flag missing integerOrderingMatch when disk ordering is unknown" ++ assert topology_st.logcap.contains("could not determine disk ordering") + topology_st.logcap.flush() + +- log.info("Run index-check with --fix") +- args.fix = True +- result = dbtasks_index_check(standalone, topology_st.logcap.log, args) +- assert result is True, "index-check --fix should succeed" +- assert topology_st.logcap.contains("integerOrderingMatch") +- topology_st.logcap.flush() +- +- log.info("Verify integerOrderingMatch was added back") +- dse_ldif = DSEldif(standalone) # Reload to get fresh data +- matching_rules = dse_ldif.get(parentid_dn, "nsMatchingRule") +- assert matching_rules is not None, "nsMatchingRule should be present" +- found_int_order = False +- for mr in matching_rules: +- if "integerorderingmatch" in mr.lower(): +- found_int_order = True +- break +- assert found_int_order, f"integerOrderingMatch should be present, got: {matching_rules}" +- log.info("integerOrderingMatch successfully added back") ++ log.info("Restore integerOrderingMatch and start the server") ++ dse_ldif = DSEldif(standalone) ++ dse_ldif.add(parentid_dn, "nsMatchingRule", "integerOrderingMatch") + + log.info("Start the server") + standalone.start() +@@ -1078,7 +1055,7 @@ def test_index_check_fixes_multiple_issues(topology_st): + :steps: + 1. Create DS instance + 2. Stop the server +- 3. Add multiple issues: scanlimit, ancestorid config, missing matching rule ++ 3. Add multiple issues: scanlimit and ancestorid config + 4. Run dsctl index-check (should detect all issues) + 5. Run dsctl index-check --fix + 6. Verify all issues were fixed +@@ -1120,14 +1097,6 @@ def test_index_check_fixes_multiple_issues(topology_st): + ] + dse_ldif.add_entry(ancestorid_entry) + +- log.info("Add issue 3: Remove integerOrderingMatch from parentid") +- dse_ldif = DSEldif(standalone) # Reload +- matching_rules = dse_ldif.get(parentid_dn, "nsMatchingRule") +- if matching_rules: +- for mr in matching_rules: +- if "integerorderingmatch" in mr.lower(): +- dse_ldif.delete(parentid_dn, "nsMatchingRule", mr) +- + log.info("Run index-check without --fix (should detect all issues)") + args = FakeArgs() + args.backend = "userRoot" +@@ -1158,16 +1127,6 @@ def test_index_check_fixes_multiple_issues(topology_st): + cn_value = dse_ldif.get(ancestorid_dn, "cn", single=True) + assert cn_value is None, f"ancestorid config should be removed, got: {cn_value}" + +- # Check matching rule added back +- matching_rules = dse_ldif.get(parentid_dn, "nsMatchingRule") +- found_int_order = False +- if matching_rules: +- for mr in matching_rules: +- if "integerorderingmatch" in mr.lower(): +- found_int_order = True +- break +- assert found_int_order, f"integerOrderingMatch should be present, got: {matching_rules}" +- + log.info("All issues verified as fixed") + + log.info("Run index-check again to confirm all clear") +diff --git a/ldap/servers/slapd/upgrade.c b/ldap/servers/slapd/upgrade.c +index aa51e72d2..4eb09ca38 100644 +--- a/ldap/servers/slapd/upgrade.c ++++ b/ldap/servers/slapd/upgrade.c +@@ -243,108 +243,6 @@ upgrade_remove_ancestorid_index_config(void) + return uresult; + } + +-/* +- * Check if parentid indexes are missing the integerOrderingMatch +- * matching rule. +- * +- * This function logs a warning if we detect this condition, advising +- * the administrator to reindex the affected attributes. +- */ +-static upgrade_status +-upgrade_check_id_index_matching_rule(void) +-{ +- struct slapi_pblock *pb = slapi_pblock_new(); +- Slapi_Entry **backends = NULL; +- const char *be_base_dn = "cn=ldbm database,cn=plugins,cn=config"; +- const char *be_filter = "(objectclass=nsBackendInstance)"; +- const char *attrs_to_check[] = {"parentid", NULL}; +- upgrade_status uresult = UPGRADE_SUCCESS; +- +- /* Search for all backend instances */ +- slapi_search_internal_set_pb( +- pb, be_base_dn, +- LDAP_SCOPE_ONELEVEL, +- be_filter, NULL, 0, NULL, NULL, +- plugin_get_default_component_id(), 0); +- slapi_search_internal_pb(pb); +- slapi_pblock_get(pb, SLAPI_PLUGIN_INTOP_SEARCH_ENTRIES, &backends); +- +- if (backends) { +- for (size_t be_idx = 0; backends[be_idx] != NULL; be_idx++) { +- const char *be_dn = slapi_entry_get_dn_const(backends[be_idx]); +- const char *be_name = slapi_entry_attr_get_ref(backends[be_idx], "cn"); +- if (!be_dn || !be_name) { +- continue; +- } +- +- /* Check each attribute that should have integerOrderingMatch */ +- for (size_t attr_idx = 0; attrs_to_check[attr_idx] != NULL; attr_idx++) { +- const char *attr_name = attrs_to_check[attr_idx]; +- struct slapi_pblock *idx_pb = slapi_pblock_new(); +- Slapi_Entry **idx_entries = NULL; +- char *idx_dn = slapi_create_dn_string("cn=%s,cn=index,%s", +- attr_name, be_dn); +- char *idx_filter = "(objectclass=nsIndex)"; +- PRBool has_matching_rule = PR_FALSE; +- +- if (!idx_dn) { +- slapi_pblock_destroy(idx_pb); +- continue; +- } +- +- slapi_search_internal_set_pb( +- idx_pb, idx_dn, +- LDAP_SCOPE_BASE, +- idx_filter, NULL, 0, NULL, NULL, +- plugin_get_default_component_id(), 0); +- slapi_search_internal_pb(idx_pb); +- slapi_pblock_get(idx_pb, SLAPI_PLUGIN_INTOP_SEARCH_ENTRIES, &idx_entries); +- +- if (idx_entries && idx_entries[0]) { +- /* Index exists, check if it has integerOrderingMatch */ +- Slapi_Attr *mr_attr = NULL; +- if (slapi_entry_attr_find(idx_entries[0], "nsMatchingRule", &mr_attr) == 0) { +- Slapi_Value *sval = NULL; +- int idx; +- for (idx = slapi_attr_first_value(mr_attr, &sval); +- idx != -1; +- idx = slapi_attr_next_value(mr_attr, idx, &sval)) { +- const struct berval *bval = slapi_value_get_berval(sval); +- if (bval && bval->bv_val && +- strcasecmp(bval->bv_val, "integerOrderingMatch") == 0) { +- has_matching_rule = PR_TRUE; +- break; +- } +- } +- } +- +- if (!has_matching_rule) { +- /* Index exists but doesn't have integerOrderingMatch, log a warning */ +- slapi_log_err(SLAPI_LOG_ERR, "upgrade_check_id_index_matching_rule", +- "Index '%s' in backend '%s' is missing 'nsMatchingRule: integerOrderingMatch'. " +- "Incorrectly configured system indexes can lead to poor search performance, replication issues, and other operational problems. " +- "To fix this, add the matching rule and reindex: " +- "dsconf backend index set --add-mr integerOrderingMatch --attr %s %s && " +- "dsconf backend index reindex --attr %s %s. " +- "WARNING: Reindexing can be resource-intensive and may impact server performance on a live system. " +- "Consider scheduling reindexing during maintenance windows or periods of low activity.\n", +- attr_name, be_name, attr_name, be_name, attr_name, be_name); +- } +- } +- +- slapi_ch_free_string(&idx_dn); +- slapi_free_search_results_internal(idx_pb); +- slapi_pblock_destroy(idx_pb); +- } +- } +- } +- +- slapi_free_search_results_internal(pb); +- slapi_pblock_destroy(pb); +- +- return uresult; +-} +- + upgrade_status + upgrade_server(void) + { +@@ -356,9 +254,5 @@ upgrade_server(void) + return UPGRADE_FAILURE; + } + +- if (upgrade_check_id_index_matching_rule() != UPGRADE_SUCCESS) { +- return UPGRADE_FAILURE; +- } +- + return UPGRADE_SUCCESS; + } +diff --git a/rpm/389-ds-base.spec.in b/rpm/389-ds-base.spec.in +index 16b8d14c5..59e3a748f 100644 +--- a/rpm/389-ds-base.spec.in ++++ b/rpm/389-ds-base.spec.in +@@ -529,9 +529,6 @@ for dir in "$instbase"/slapd-* ; do + else + echo "instance $inst is not running" >> "$output" 2>&1 || : + fi +- # Run index-check on all instances (running or not) +- # This fixes index ordering mismatches from older versions +- dsctl "$inst_name" index-check --fix >> "$output2" 2>&1 || : + ninst=$((ninst + 1)) + done + +diff --git a/src/lib389/lib389/backend.py b/src/lib389/lib389/backend.py +index 4babf6850..376596cd6 100644 +--- a/src/lib389/lib389/backend.py ++++ b/src/lib389/lib389/backend.py +@@ -541,9 +541,10 @@ class Backend(DSLdapObject): + # Default system indexes taken from ldap/servers/slapd/back-ldbm/instance.c + # Note: entryrdn and ancestorid are internal system indexes that are not + # exposed in cn=config - they are managed internally by the server. +- # Only parentid has a DSE config entry (for the integerOrderingMatch rule). ++ # parentid works correctly with both lexicographic and integer ordering, ++ # so integerOrderingMatch is not required. + expected_system_indexes = { +- 'parentid': {'types': ['eq'], 'matching_rule': 'integerOrderingMatch'}, ++ 'parentid': {'types': ['eq'], 'matching_rule': None}, + 'objectClass': {'types': ['eq'], 'matching_rule': None}, + 'aci': {'types': ['pres'], 'matching_rule': None}, + 'nscpEntryDN': {'types': ['eq'], 'matching_rule': None}, +diff --git a/src/lib389/lib389/cli_ctl/dbtasks.py b/src/lib389/lib389/cli_ctl/dbtasks.py +index 16da966d1..ea8a00cc3 100644 +--- a/src/lib389/lib389/cli_ctl/dbtasks.py ++++ b/src/lib389/lib389/cli_ctl/dbtasks.py +@@ -10,6 +10,7 @@ + import glob + import os + import re ++import signal + import subprocess + from enum import Enum + from lib389._constants import TaskWarning +@@ -271,45 +272,53 @@ def _check_disk_ordering(db_dir, backend, index_name, dbscan_path, is_mdb, log): + if not index_file: + return IndexOrdering.UNKNOWN + ++ # Only read the first 100 lines from dbscan to avoid scanning the ++ # entire index (which can take hours on large databases). + try: +- result = subprocess.run( ++ proc = subprocess.Popen( + [dbscan_path, "-f", index_file], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True, +- timeout=60, + ) + +- if result.returncode != 0: +- log.warning(" dbscan returned non-zero exit code for %s", index_file) +- return IndexOrdering.UNKNOWN +- +- # Parse keys from dbscan output + keys = [] +- for line in result.stdout.split("\n"): ++ line_count = 0 ++ assert proc.stdout is not None ++ for line in proc.stdout: ++ line_count += 1 ++ if line_count > 100: ++ break + line = line.strip() + if line.startswith("="): + match = re.match(r"^=(\d+)", line) + if match: + keys.append(int(match.group(1))) + ++ proc.terminate() ++ try: ++ proc.wait(timeout=5) ++ except subprocess.TimeoutExpired: ++ proc.kill() ++ proc.wait() ++ ++ if proc.returncode not in (0, -signal.SIGTERM): ++ log.warning(" dbscan returned non-zero exit code for %s", index_file) ++ return IndexOrdering.UNKNOWN ++ + if len(keys) < 2: + return IndexOrdering.UNKNOWN + + # Check if keys are in integer order by looking for decreasing numeric values + # (which would indicate lexicographic ordering, e.g., "3" < "30" < "4") + prev_id = keys[0] +- for i in range(1, min(len(keys), 100)): +- current_id = keys[i] ++ for current_id in keys[1:]: + if prev_id > current_id: + return IndexOrdering.LEXICOGRAPHIC + prev_id = current_id + + return IndexOrdering.INTEGER + +- except subprocess.TimeoutExpired: +- log.warning(" dbscan timed out for %s", index_file) +- return IndexOrdering.UNKNOWN + except OSError as e: + log.warning(" Error running dbscan: %s", e) + return IndexOrdering.UNKNOWN +@@ -383,8 +392,7 @@ def dbtasks_index_check(inst, log, args): + + # Track all issues found + all_ok = True +- mismatches = [] # (backend, index_name) tuples needing reindex +- missing_matching_rules = [] # (backend, index_name) tuples missing integerOrderingMatch ++ config_fixes = [] # (backend, index_name, action) tuples: action is "add_mr" or "remove_mr" + scan_limits_to_remove = [] # (backend, index_name) tuples with nsIndexIDListScanLimit + ancestorid_configs_to_remove = [] # backend names with ancestorid config entries + remove_ancestorid_from_defaults = False # Flag to remove from cn=default indexes +@@ -417,13 +425,6 @@ def dbtasks_index_check(inst, log, args): + + if disk_ordering == IndexOrdering.UNKNOWN: + log.info(" %s - could not determine disk ordering, skipping", index_name) +- # For parentid, still check if matching rule is missing +- if index_name == "parentid": +- config_has_int_order = _has_integer_ordering_match(dse_ldif, backend, index_name) +- if not config_has_int_order: +- log.warning(" %s - missing integerOrderingMatch in config", index_name) +- missing_matching_rules.append((backend, index_name)) +- all_ok = False + continue + + config_has_int_order = _has_integer_ordering_match(dse_ldif, backend, index_name) +@@ -431,18 +432,15 @@ def dbtasks_index_check(inst, log, args): + log.info(" %s - config: %s, disk: %s", + index_name, config_desc, disk_ordering.value) + +- # For parentid, the desired state is always integer ordering ++ # Both orderings are valid for parentid, but config must match disk. + if index_name == "parentid": +- if not config_has_int_order: +- log.warning(" %s - missing integerOrderingMatch in config", index_name) +- if (backend, index_name) not in missing_matching_rules: +- missing_matching_rules.append((backend, index_name)) ++ if config_has_int_order and disk_ordering == IndexOrdering.LEXICOGRAPHIC: ++ log.warning(" %s - MISMATCH: config has integerOrderingMatch but disk is lexicographic", index_name) ++ config_fixes.append((backend, index_name, "remove_mr")) + all_ok = False +- +- if disk_ordering == IndexOrdering.LEXICOGRAPHIC: +- log.warning(" %s - disk ordering is lexicographic, needs reindex", index_name) +- if (backend, index_name) not in mismatches: +- mismatches.append((backend, index_name)) ++ elif not config_has_int_order and disk_ordering == IndexOrdering.INTEGER: ++ log.warning(" %s - MISMATCH: config is lexicographic but disk has integer ordering", index_name) ++ config_fixes.append((backend, index_name, "add_mr")) + all_ok = False + + # Handle issues +@@ -488,26 +486,27 @@ def dbtasks_index_check(inst, log, args): + log.error(" Failed to remove ancestorid config from backend %s: %s", backend, e) + return False + +- # Add missing matching rules to dse.ldif +- for backend, index_name in missing_matching_rules: ++ # Fix config-vs-disk ordering mismatches by adjusting config to match disk ++ for backend, index_name, action in config_fixes: + index_dn = "cn={},cn=index,cn={},cn=ldbm database,cn=plugins,cn=config".format( + index_name, backend + ) +- log.info(" Adding integerOrderingMatch to %s in backend %s...", index_name, backend) +- try: +- dse_ldif.add(index_dn, "nsMatchingRule", "integerOrderingMatch") +- log.info(" Updated dse.ldif with integerOrderingMatch for %s", index_name) +- except Exception as e: +- log.error(" Failed to update dse.ldif for %s: %s", index_name, e) +- return False +- +- # Reindex indexes with disk ordering issues +- for backend, index_name in mismatches: +- log.info(" Reindexing %s in backend %s...", index_name, backend) +- if not inst.db2index(bename=backend, attrs=[index_name]): +- log.error(" Failed to reindex %s", index_name) +- return False +- log.info(" Reindex of %s completed successfully", index_name) ++ if action == "add_mr": ++ log.info(" Adding integerOrderingMatch to %s in backend %s...", index_name, backend) ++ try: ++ dse_ldif.add(index_dn, "nsMatchingRule", "integerOrderingMatch") ++ log.info(" Updated dse.ldif with integerOrderingMatch for %s", index_name) ++ except Exception as e: ++ log.error(" Failed to update dse.ldif for %s: %s", index_name, e) ++ return False ++ elif action == "remove_mr": ++ log.info(" Removing integerOrderingMatch from %s in backend %s...", index_name, backend) ++ try: ++ dse_ldif.delete(index_dn, "nsMatchingRule", "integerOrderingMatch") ++ log.info(" Removed integerOrderingMatch from %s", index_name) ++ except Exception as e: ++ log.error(" Failed to remove integerOrderingMatch from %s: %s", index_name, e) ++ return False + + log.info("All issues fixed") + return True +@@ -572,5 +571,5 @@ def create_parser(subcommands): + index_check_parser.add_argument('backend', nargs='?', default=None, + help="Backend to check. If not specified, all backends are checked.") + index_check_parser.add_argument('--fix', action='store_true', default=False, +- help="Fix mismatches by reindexing affected indexes") ++ help="Fix mismatches by adjusting config to match on-disk data") + index_check_parser.set_defaults(func=dbtasks_index_check) +-- +2.52.0 + diff --git a/SPECS/389-ds-base.spec b/SPECS/389-ds-base.spec index b93507d..d4611fd 100644 --- a/SPECS/389-ds-base.spec +++ b/SPECS/389-ds-base.spec @@ -52,7 +52,7 @@ ExcludeArch: i686 Summary: 389 Directory Server (base) Name: 389-ds-base Version: 1.4.3.39 -Release: %{?relprefix}20%{?prerel}%{?dist} +Release: %{?relprefix}22%{?prerel}%{?dist} License: GPL-3.0-or-later WITH GPL-3.0-389-ds-base-exception AND (0BSD OR Apache-2.0 OR MIT) AND (Apache-2.0 OR Apache-2.0 WITH LLVM-exception OR MIT) AND (Apache-2.0 OR BSD-2-Clause OR MIT) AND (Apache-2.0 OR BSL-1.0) AND (Apache-2.0 OR LGPL-2.1-or-later OR MIT) AND (Apache-2.0 OR MIT OR Zlib) AND (Apache-2.0 OR MIT) AND (MIT OR Apache-2.0) AND Unicode-3.0 AND (MIT OR Unlicense) AND Apache-2.0 AND BSD-3-Clause AND MIT AND MPL-2.0 URL: https://www.port389.org Group: System Environment/Daemons @@ -373,6 +373,16 @@ Patch72: 0072-Issue-6966-2nd-On-large-DB-unlimited-IDL-scan-limit-.patc Patch73: 0073-Issue-7056-DSBLE0007-doesn-t-generate-remediation-st.patch Patch74: 0074-Issue-7172-Index-ordering-mismatch-after-upgrade-717.patch Patch75: 0075-Issue-7172-2nd-Index-ordering-mismatch-after-upgrade.patch +Patch76: 0076-Issue-7071-search-filter-cn-dn-groups-no-longer-retu.patch +Patch77: 0077-Issue-7189-DSBLE0007-generates-incorrect-remediation.patch +Patch78: 0078-Issue-7223-Revert-index-scan-limits-for-system-index.patch +Patch79: 0079-Issue-7223-Backport-upgrade-infrastructure-from-main.patch +Patch80: 0080-Issue-7223-Add-upgrade-function-to-remove-nsIndexIDL.patch +Patch81: 0081-Issue-7223-Add-upgrade-function-to-remove-ancestorid.patch +Patch82: 0082-Issue-7223-Detect-and-log-index-ordering-mismatch-du.patch +Patch83: 0083-Issue-7223-Add-dsctl-index-check-command-for-offline.patch +Patch84: 0084-Issue-7223-Use-lexicographical-order-for-ancestorid.patch +Patch85: 0085-Issue-7223-Remove-integerOrderingMatch-requirement-f.patch #Patch100: cargo.patch @@ -681,7 +691,42 @@ if ! getent passwd $USERNAME >/dev/null ; then fi # Reload our sysctl before we restart (if we can) -sysctl --system &> $output; true +sysctl --system &> "$output"; true + +# Gather running instances, stop them, run index-check, then restart +instbase="%{_sysconfdir}/%{pkgname}" +instances="" +ninst=0 + +for dir in "$instbase"/slapd-* ; do + echo "dir = $dir" >> "$output" 2>&1 || : + if [ ! -d "$dir" ] ; then continue ; fi + case "$dir" in *.removed) continue ;; esac + basename=$(basename "$dir") + inst="%{pkgname}@${basename#slapd-}" + inst_name="${basename#slapd-}" + echo "found instance $inst - getting status" >> "$output" 2>&1 || : + if /bin/systemctl -q is-active "$inst" ; then + echo "instance $inst is running - stopping for upgrade" >> "$output" 2>&1 || : + instances="$instances $inst" + /bin/systemctl stop "$inst" >> "$output" 2>&1 || : + else + echo "instance $inst is not running" >> "$output" 2>&1 || : + fi + ninst=$((ninst + 1)) +done + +if [ $ninst -eq 0 ] ; then + echo "no instances to upgrade" >> "$output" 2>&1 || : + exit 0 +fi + +# Restart previously running instances +for inst in $instances ; do + echo "starting instance $inst" >> "$output" 2>&1 || : + /bin/systemctl start "$inst" >> "$output" 2>&1 || : +done + %preun if [ $1 -eq 0 ]; then # Final removal @@ -998,6 +1043,14 @@ exit 0 %doc README.md %changelog +* Wed Feb 18 2026 Viktor Ashirov - 1.4.3.39-22 +- Resolves: RHEL-148485 - Upgrading IDM to latest version: 389-ds-base and ipa-server breaks replication [rhel-8.10.z] + +* Fri Jan 23 2026 Arun Bansal - 1.4.3.39-21 +- Resolves: RHEL-141419 - (&(cn:dn:=groups)) no longer returns results [rhel-8.10.z] +- Resolves: RHEL-140272 - ipa-healthcheck is complaining about missing or + incorrectly configured system indexes. [rhel-8.10.z] + * Tue Jan 13 2026 Arun Bansal - 1.4.3.39-20 - Resolves: RHEL-140086 - Upgrading IDM to latest version: 389-ds-base and ipa-server breaks replication [rhel-8.10.z]