diff --git a/.gitignore b/.gitignore index 0e6a01d..16ed8b2 100644 --- a/.gitignore +++ b/.gitignore @@ -1,21 +1,4 @@ -SOURCES/389-ds-base-1.4.3.32.tar.bz2 -SOURCES/jemalloc-5.3.0.tar.bz2 -SOURCES/vendor-1.4.3.32-1.tar.gz -/389-ds-base-1.4.3.32.tar.bz2 -/jemalloc-5.3.0.tar.bz2 -/vendor-1.4.3.32-1.tar.gz -/vendor-1.4.3.35-1.tar.gz -/Cargo-1.4.3.35.lock -/389-ds-base-1.4.3.35.tar.bz2 -/389-ds-base-1.4.3.36.tar.bz2 -/vendor-1.4.3.36-1.tar.gz -/Cargo-1.4.3.36.lock -/389-ds-base-1.4.3.37.tar.bz2 -/vendor-1.4.3.37-1.tar.gz -/Cargo-1.4.3.37-1.lock -/389-ds-base-1.4.3.38.tar.bz2 -/vendor-1.4.3.38-1.tar.gz -/Cargo-1.4.3.38-1.lock -/389-ds-base-1.4.3.39.tar.bz2 -/vendor-1.4.3.39-1.tar.gz -/Cargo-1.4.3.39-1.lock +389-ds-base-*.tar.bz2 +jemalloc-5.3.0.tar.bz2 +vendor-*.tar.gz +Cargo-*.lock diff --git a/0023-Issue-6304-RFE-when-memberof-is-enabled-defer-update.patch b/0023-Issue-6304-RFE-when-memberof-is-enabled-defer-update.patch new file mode 100644 index 0000000..7c2c3cb --- /dev/null +++ b/0023-Issue-6304-RFE-when-memberof-is-enabled-defer-update.patch @@ -0,0 +1,3998 @@ +From 1bcb7f2fed1d3ac91b1e2b75041fe2d36ca7550d Mon Sep 17 00:00:00 2001 +From: Thierry Bordaz +Date: Thu, 29 Aug 2024 16:20:47 +0200 +Subject: [PATCH] Issue 6304 - RFE when memberof is enabled, defer updates of + members from the update of the group + +Bug Description: + When an update of a static group changes + impacts a large portion of its members, memberof triggers + a large number of internal updates on members. + All the updates are done a same TXN that may hold + sensitive DB pages and block others SRCH req + waiting for those pages. + In extreme condition, all workers are stuck on + SRCH req waiting for the completion of the update. + Then the server appears unresponsive. + +Fix Description: + The fix is to defer the update of the members. + + Memberof tests: + - for the verification of the membership, if deferred update is set, + then adds a delay before checking + - automember_plugin/automember_test.py + - automember_plugin/basic_test.py + - memberof_plugin/memberof_include_scopes_test.py + - plugins/acceptance_test.py + - plugins/entryusn_test.py + - plugins/memberof_test.py + - lib389/plugins.py + - original update (group) succeeds even if deferred updates fails (multiple TXN) + - betxns/betxn_test.py + - Check replication of memberof + - memberof_plugin/memberof_deferred_repl_test.py + - Check deferred update and shutdown + - memberof_plugin/memberof_include_scopes_test.py + + Core implementation: + + - Make sure that direct update (not internal) wait for + deferred update before returning a result + - back-ldbm/ldbm_add.c + - back-ldbm/ldbm_delete.c + - back-ldbm/ldbm_modify.c + - back-ldbm/ldbm_modrdn.c + + - Implementation of the deferred update + - memberof/memberof.h + - memberof/memberof.c + - memberof/memberof_config.c + - slapd/pblock.c + - slapd/pblock_v3.h + - slapd/schema.c + - slapd/slapi-plugin.h + memberof_be_postop_init + registers memberof_push_deferred_task that: + push deferred update to deferred thread + task taken from pblock (SLAPI_MEMBEROF_DEFERRED_TASK) + push to the memberof config deferred_list + + deferred thread (deferred_thread_func) + if 'memberOfNeedFixup: on' then run fixup task + loop until shutdown + fetch task (remove_deferred_task) from the memberof config deferred_list + proceed with the task + if it exits abruptly, it logs an alert and add 'memberOfNeedFixup: on' to the config + + memberof_postop_start + if deferred update is configured then it creates a deferred_list in the config + it spawn the deferred thread + CV + + Each postop_operation memberof_postop_modrdn, memberof_postop_del, memberof_postop_modify, memberof_postop_add + if deferred update it allocates a task and add it in the pblock (SLAPI_MEMBEROF_DEFERRED_TASK) + +Related: #6304 + +Reviewed by: Simon Pichugin (THanks !!!) +--- + .../automember_plugin/automember_test.py | 26 +- + .../suites/automember_plugin/basic_test.py | 90 +- + dirsrvtests/tests/suites/betxns/betxn_test.py | 34 +- + .../memberof_deferred_repl_test.py | 178 +++ + .../memberof_include_scopes_test.py | 6 + + .../suites/memberof_plugin/regression_test.py | 378 ++++- + .../tests/suites/plugins/acceptance_test.py | 16 + + .../tests/suites/plugins/entryusn_test.py | 7 + + .../tests/suites/plugins/memberof_test.py | 68 + + ldap/servers/plugins/memberof/memberof.c | 1312 +++++++++++++++-- + ldap/servers/plugins/memberof/memberof.h | 71 + + .../plugins/memberof/memberof_config.c | 26 +- + ldap/servers/slapd/back-ldbm/ldbm_add.c | 15 + + ldap/servers/slapd/back-ldbm/ldbm_delete.c | 15 + + ldap/servers/slapd/back-ldbm/ldbm_modify.c | 15 + + ldap/servers/slapd/back-ldbm/ldbm_modrdn.c | 16 + + ldap/servers/slapd/pblock.c | 17 + + ldap/servers/slapd/pblock_v3.h | 9 + + ldap/servers/slapd/schema.c | 21 + + ldap/servers/slapd/slapi-plugin.h | 7 + + ldap/servers/slapd/slapi-private.h | 3 + + src/lib389/lib389/plugins.py | 20 + + 22 files changed, 2168 insertions(+), 182 deletions(-) + create mode 100644 dirsrvtests/tests/suites/memberof_plugin/memberof_deferred_repl_test.py + +diff --git a/dirsrvtests/tests/suites/automember_plugin/automember_test.py b/dirsrvtests/tests/suites/automember_plugin/automember_test.py +index e1976bd78..fc3726221 100644 +--- a/dirsrvtests/tests/suites/automember_plugin/automember_test.py ++++ b/dirsrvtests/tests/suites/automember_plugin/automember_test.py +@@ -10,6 +10,7 @@ import logging + import pytest + import os + import ldap ++import time + from lib389.utils import ds_is_older + from lib389._constants import * + from lib389.plugins import AutoMembershipPlugin, AutoMembershipDefinition, AutoMembershipDefinitions, AutoMembershipRegexRule +@@ -172,6 +173,10 @@ def test_delete_default_group(automember_fixture, topo): + + from lib389.plugins import MemberOfPlugin + memberof = MemberOfPlugin(topo.standalone) ++ if (memberof.get_memberofdeferredupdate() and memberof.get_memberofdeferredupdate().lower() == "on"): ++ tries = 10 # to avoid any risk of transient failure ++ else: ++ tries = 1 + memberof.enable() + topo.standalone.restart() + topo.standalone.setLogLevel(65536) +@@ -182,8 +187,20 @@ def test_delete_default_group(automember_fixture, topo): + try: + assert group.is_member(user_1.dn) + group.delete() +- error_lines = topo.standalone.ds_error_log.match('.*auto-membership-plugin - automember_update_member_value - group .default or target. does not exist .%s.$' % group.dn) +- assert (len(error_lines) == 1) ++ # Check there is the expected message ++ while tries > 0: ++ error_lines = topo.standalone.ds_error_log.match('.*auto-membership-plugin - automember_update_member_value - group .default or target. does not exist .%s.$' % group.dn) ++ nb_match = len(error_lines) ++ log.info("len(error_lines)=%d" % nb_match) ++ for i in error_lines: ++ log.info(" - %s" % i) ++ assert nb_match <= 1 ++ if (nb_match == 1): ++ # we are done the test is successful ++ break ++ time.sleep(1) ++ tries -= 1 ++ assert tries > 0 + finally: + user_1.delete() + topo.standalone.setLogLevel(0) +@@ -285,6 +302,10 @@ def test_delete_target_group(automember_fixture, topo): + + from lib389.plugins import MemberOfPlugin + memberof = MemberOfPlugin(topo.standalone) ++ if (memberof.get_memberofdeferredupdate() and memberof.get_memberofdeferredupdate().lower() == "on"): ++ delay = 3 ++ else: ++ delay = 0 + memberof.enable() + + topo.standalone.restart() +@@ -300,6 +321,7 @@ def test_delete_target_group(automember_fixture, topo): + + # delete that target filter group + group_regex.delete() ++ time.sleep(delay) + error_lines = topo.standalone.ds_error_log.match('.*auto-membership-plugin - automember_update_member_value - group .default or target. does not exist .%s.$' % group_regex.dn) + # one line for default group and one for target group + assert (len(error_lines) == 1) +diff --git a/dirsrvtests/tests/suites/automember_plugin/basic_test.py b/dirsrvtests/tests/suites/automember_plugin/basic_test.py +index 075e85996..51acfeadc 100644 +--- a/dirsrvtests/tests/suites/automember_plugin/basic_test.py ++++ b/dirsrvtests/tests/suites/automember_plugin/basic_test.py +@@ -12,6 +12,7 @@ Will test AutoMememer Plugin with AotoMember Task and Retro Changelog + + import os + import pytest ++import time + from lib389.topologies import topology_m1 as topo + from lib389.idm.organizationalunit import OrganizationalUnits + from lib389.idm.domain import Domain +@@ -365,19 +366,23 @@ def test_ability_to_control_behavior_of_modifiers_name(topo, _create_all_entries + 7. Should success + """ + instance1 = topo.ms["supplier1"] ++ memberof = MemberOfPlugin(instance1) ++ if (memberof.get_memberofdeferredupdate() and memberof.get_memberofdeferredupdate().lower() == "on"): ++ delay = 3 ++ else: ++ delay = 0 + configure = Config(instance1) + configure.replace('nsslapd-plugin-binddn-tracking', 'on') + instance1.restart() + assert configure.get_attr_val_utf8('nsslapd-plugin-binddn-tracking') == 'on' + user = add_user(topo, "User_autoMembers_05", "ou=Employees,{}".format(TEST_BASE), + "19", "18", "Supervisor") ++ time.sleep(delay) + # search the User DN name for the creatorsname in user entry + assert user.get_attr_val_utf8('creatorsname') == 'cn=directory manager' + # search the User DN name for the internalCreatorsname in user entry + assert user.get_attr_val_utf8('internalCreatorsname') == \ + 'cn=ldbm database,cn=plugins,cn=config' +- # search the modifiersname in the user entry +- assert user.get_attr_val_utf8('modifiersname') == 'cn=directory manager' + # search the internalModifiersname in the user entry + assert user.get_attr_val_utf8('internalModifiersname') == \ + 'cn=MemberOf Plugin,cn=plugins,cn=config' +@@ -401,10 +406,18 @@ def test_posixaccount_objectclass_automemberdefaultgroup(topo, _create_all_entri + 2. Should success + """ + test_id = "autoMembers_05" ++ instance = topo.ms["supplier1"] ++ memberof = MemberOfPlugin(instance) ++ if (memberof.get_memberofdeferredupdate() and memberof.get_memberofdeferredupdate().lower() == "on"): ++ delay = 3 ++ else: ++ delay = 0 + default_group = "cn=TestDef1,CN=testuserGroups,{}".format(TEST_BASE) + user = add_user(topo, "User_{}".format(test_id), AUTO_MEM_SCOPE_TEST, "19", "18", "Supervisor") ++ time.sleep(delay) + assert check_groups(topo, default_group, user.dn, "member") + user.delete() ++ time.sleep(delay) + with pytest.raises(AssertionError): + assert check_groups(topo, default_group, user.dn, "member") + +@@ -430,13 +443,22 @@ def test_duplicated_member_attributes_added_when_the_entry_is_re_created(topo, _ + 6. Should success + """ + test_id = "autoMembers_06" ++ instance = topo.ms["supplier1"] ++ memberof = MemberOfPlugin(instance) ++ if (memberof.get_memberofdeferredupdate() and memberof.get_memberofdeferredupdate().lower() == "on"): ++ delay = 3 ++ else: ++ delay = 0 + default_group = "cn=TestDef1,CN=testuserGroups,{}".format(TEST_BASE) + user = add_user(topo, "User_{}".format(test_id), AUTO_MEM_SCOPE_TEST, "19", "16", "Supervisor") ++ time.sleep(delay) + assert check_groups(topo, default_group, user.dn, "member") + user.delete() ++ time.sleep(delay) + with pytest.raises(AssertionError): + assert check_groups(topo, default_group, user.dn, "member") + user = add_user(topo, "User_{}".format(test_id), AUTO_MEM_SCOPE_TEST, "19", "15", "Supervisor") ++ time.sleep(delay) + assert check_groups(topo, default_group, user.dn, "member") + user.delete() + +@@ -458,13 +480,21 @@ def test_multi_valued_automemberdefaultgroup_for_hostgroups(topo, _create_all_en + 4. Should success + """ + test_id = "autoMembers_07" ++ instance = topo.ms["supplier1"] ++ memberof = MemberOfPlugin(instance) ++ if (memberof.get_memberofdeferredupdate() and memberof.get_memberofdeferredupdate().lower() == "on"): ++ delay = 3 ++ else: ++ delay = 0 + default_group1 = "cn=TestDef1,CN=testuserGroups,{}".format(TEST_BASE) + default_group2 = "cn=TestDef2,CN=testuserGroups,{}".format(TEST_BASE) + default_group3 = "cn=TestDef3,CN=testuserGroups,{}".format(TEST_BASE) + user = add_user(topo, "User_{}".format(test_id), AUTO_MEM_SCOPE_TEST, "19", "14", "TestEngr") ++ time.sleep(delay) + for grp in [default_group1, default_group2, default_group3]: + assert check_groups(topo, grp, user.dn, "member") + user.delete() ++ time.sleep(delay) + with pytest.raises(AssertionError): + assert check_groups(topo, default_group1, user.dn, "member") + +@@ -485,6 +515,12 @@ def test_plugin_creates_member_attributes_of_the_automemberdefaultgroup(topo, _c + 3. Should success + """ + test_id = "autoMembers_08" ++ instance = topo.ms["supplier1"] ++ memberof = MemberOfPlugin(instance) ++ if (memberof.get_memberofdeferredupdate() and memberof.get_memberofdeferredupdate().lower() == "on"): ++ delay = 3 ++ else: ++ delay = 0 + default_group1 = "cn=TestDef1,CN=testuserGroups,{}".format(TEST_BASE) + default_group2 = "cn=TestDef5,CN=testuserGroups,{}".format(TEST_BASE) + default_group3 = "cn=TestDef3,CN=testuserGroups,{}".format(TEST_BASE) +@@ -495,6 +531,7 @@ def test_plugin_creates_member_attributes_of_the_automemberdefaultgroup(topo, _c + "cn=TestDef4,CN=testuserGroups,{}".format(TEST_BASE), + "uid=User_{},{}".format(test_id, AUTO_MEM_SCOPE_TEST), "member") + user = add_user(topo, "User_{}".format(test_id), AUTO_MEM_SCOPE_TEST, "19", "14", "TestEngr") ++ time.sleep(delay) + for grp in [default_group1, default_group2, default_group3]: + assert check_groups(topo, grp, user.dn, "member") + user.delete() +@@ -520,6 +557,11 @@ def test_multi_valued_automemberdefaultgroup_with_uniquemember(topo, _create_all + """ + test_id = "autoMembers_09" + instance = topo.ms["supplier1"] ++ memberof = MemberOfPlugin(instance) ++ if (memberof.get_memberofdeferredupdate() and memberof.get_memberofdeferredupdate().lower() == "on"): ++ delay = 3 ++ else: ++ delay = 0 + auto = AutoMembershipPlugin(topo.ms["supplier1"]) + # Modify automember config entry to use uniquemember: cn=testuserGroups,PLUGIN_AUTO + AutoMembershipDefinition( +@@ -570,11 +612,18 @@ def test_invalid_automembergroupingattr_member(topo, _create_all_entries): + 5. Should success + """ + test_id = "autoMembers_10" ++ instance = topo.ms["supplier1"] ++ memberof = MemberOfPlugin(instance) ++ if (memberof.get_memberofdeferredupdate() and memberof.get_memberofdeferredupdate().lower() == "on"): ++ delay = 3 ++ else: ++ delay = 0 + default_group = "cn=TestDef1,CN=testuserGroups,{}".format(TEST_BASE) + instance_of_group = Group(topo.ms["supplier1"], default_group) + change_grp_objclass("groupOfUniqueNames", "member", instance_of_group) + with pytest.raises(ldap.UNWILLING_TO_PERFORM): + add_user(topo, "User_{}".format(test_id), AUTO_MEM_SCOPE_TEST, "19", "20", "Invalid") ++ time.sleep(delay) + with pytest.raises(AssertionError): + assert check_groups(topo, default_group, + "uid=User_{},{}".format(test_id, AUTO_MEM_SCOPE_TEST), "member") +@@ -600,6 +649,14 @@ def test_valid_and_invalid_automembergroupingattr(topo, _create_all_entries): + 5. Should success + """ + test_id = "autoMembers_11" ++ instance = topo.ms["supplier1"] ++ memberof = MemberOfPlugin(instance) ++ if (memberof.get_memberofdeferredupdate() and memberof.get_memberofdeferredupdate().lower() == "on"): ++ singleTXN = False ++ delay = 3 ++ else: ++ singleTXN = True ++ delay = 0 + default_group_1 = "cn=TestDef1,CN=testuserGroups,{}".format(TEST_BASE) + default_group_2 = "cn=TestDef2,CN=testuserGroups,{}".format(TEST_BASE) + default_group_3 = "cn=TestDef3,CN=testuserGroups,{}".format(TEST_BASE) +@@ -611,6 +668,7 @@ def test_valid_and_invalid_automembergroupingattr(topo, _create_all_entries): + change_grp_objclass("groupOfUniqueNames", "member", instance_of_group) + with pytest.raises(ldap.UNWILLING_TO_PERFORM): + add_user(topo, "User_{}".format(test_id), AUTO_MEM_SCOPE_TEST, "19", "24", "MixUsers") ++ time.sleep(delay) + for grp in [default_group_1, default_group_2, default_group_3]: + assert not check_groups(topo, grp, "cn=User_{},{}".format(test_id, + AUTO_MEM_SCOPE_TEST), "member") +@@ -637,8 +695,15 @@ def test_add_regular_expressions_for_user_groups_and_check_for_member_attribute_ + 2. Should success + """ + test_id = "autoMembers_12" ++ instance = topo.ms["supplier1"] ++ memberof = MemberOfPlugin(instance) ++ if (memberof.get_memberofdeferredupdate() and memberof.get_memberofdeferredupdate().lower() == "on"): ++ delay = 3 ++ else: ++ delay = 0 + default_group = f'cn=SuffDef1,ou=userGroups,{BASE_SUFF}' + user = add_user(topo, "User_{}".format(test_id), AUTO_MEM_SCOPE_BASE, "19", "0", "HR") ++ time.sleep(delay) + assert check_groups(topo, default_group, user.dn, "member") + assert number_memberof(topo, user.dn, 5) + user.delete() +@@ -675,6 +740,13 @@ def test_matching_gid_role_inclusive_regular_expression(topo, _create_all_entrie + user1 = add_user(topo, "User_{}".format(testid), AUTO_MEM_SCOPE_BASE, uid, gid, role) + user2 = add_user(topo, "SecondUser_{}".format(testid), AUTO_MEM_SCOPE_BASE, + uid2, gid2, role) ++ instance = topo.ms["supplier1"] ++ memberof = MemberOfPlugin(instance) ++ if (memberof.get_memberofdeferredupdate() and memberof.get_memberofdeferredupdate().lower() == "on"): ++ delay = 3 ++ else: ++ delay = 0 ++ time.sleep(delay) + for user_dn in [user1.dn, user2.dn]: + assert check_groups(topo, contract_grp, user_dn, "member") + assert number_memberof(topo, user1.dn, 1) +@@ -715,9 +787,16 @@ def test_gid_and_role_inclusive_exclusive_regular_expression(topo, _create_all_e + 3. Should success + 4. Should success + """ ++ instance = topo.ms["supplier1"] ++ memberof = MemberOfPlugin(instance) ++ if (memberof.get_memberofdeferredupdate() and memberof.get_memberofdeferredupdate().lower() == "on"): ++ delay = 3 ++ else: ++ delay = 0 + contract_grp = f'cn={c_grp},ou=userGroups,{BASE_SUFF}' + default_group = f'cn={m_grp},ou=userGroups,{BASE_SUFF}' + user = add_user(topo, "User_{}".format(testid), AUTO_MEM_SCOPE_BASE, uid, gid, role) ++ time.sleep(delay) + with pytest.raises(AssertionError): + assert check_groups(topo, contract_grp, user.dn, "member") + check_groups(topo, default_group, user.dn, "member") +@@ -755,7 +834,14 @@ def test_managers_contractors_exclusive_regex_rules_member_uid(topo, _create_all + """ + default_group1 = f'cn={c_grp},{SUBSUFFIX}' + default_group2 = f'cn={m_grp},{SUBSUFFIX}' ++ instance = topo.ms["supplier1"] ++ memberof = MemberOfPlugin(instance) ++ if (memberof.get_memberofdeferredupdate() and memberof.get_memberofdeferredupdate().lower() == "on"): ++ delay = 3 ++ else: ++ delay = 0 + user = add_user(topo, "User_{}".format(testid), AUTO_MEM_SCOPE_BASE, uid, gid, role) ++ time.sleep(delay) + for group in [default_group1, default_group2]: + assert check_groups(topo, group, user.dn, "memberuid") + user.delete() +diff --git a/dirsrvtests/tests/suites/betxns/betxn_test.py b/dirsrvtests/tests/suites/betxns/betxn_test.py +index 76850a992..68fd72bd5 100644 +--- a/dirsrvtests/tests/suites/betxns/betxn_test.py ++++ b/dirsrvtests/tests/suites/betxns/betxn_test.py +@@ -159,6 +159,10 @@ def test_betxn_memberof(topology_st): + memberof = MemberOfPlugin(topology_st.standalone) + memberof.enable() + memberof.set_autoaddoc('referral') ++ if (memberof.get_memberofdeferredupdate() and memberof.get_memberofdeferredupdate().lower() == "on"): ++ singleTXN = False ++ else: ++ singleTXN = True + topology_st.standalone.restart() + + groups = Groups(topology_st.standalone, DEFAULT_SUFFIX) +@@ -171,11 +175,17 @@ def test_betxn_memberof(topology_st): + group2.remove('objectClass', 'nsMemberOf') + + # Add group2 to group1 - it should fail with objectclass violation +- with pytest.raises(ldap.OBJECT_CLASS_VIOLATION): ++ if singleTXN: ++ with pytest.raises(ldap.OBJECT_CLASS_VIOLATION): ++ group1.add_member(group2.dn) ++ ++ # verify entry cache reflects the current/correct state of group1 ++ assert not group1.is_member(group2.dn) ++ else: + group1.add_member(group2.dn) + +- # verify entry cache reflects the current/correct state of group1 +- assert not group1.is_member(group2.dn) ++ # verify entry cache reflects the current/correct state of group1 ++ assert group1.is_member(group2.dn) + + # Done + log.info('test_betxn_memberof: PASSED') +@@ -208,6 +218,10 @@ def test_betxn_modrdn_memberof_cache_corruption(topology_st): + memberof.set_autoaddoc('nsContainer') # Bad OC + memberof.set('memberOfEntryScope', peoplebase) + memberof.set('memberOfAllBackends', 'on') ++ if (memberof.get_memberofdeferredupdate() and memberof.get_memberofdeferredupdate().lower() == "on"): ++ singleTXN = False ++ else: ++ singleTXN = True + topology_st.standalone.restart() + + groups = Groups(topology_st.standalone, DEFAULT_SUFFIX) +@@ -223,12 +237,16 @@ def test_betxn_modrdn_memberof_cache_corruption(topology_st): + + group.add_member(user.dn) + +- # Attempt modrdn that should fail, but the original entry should stay in the cache +- with pytest.raises(ldap.OBJECT_CLASS_VIOLATION): +- group.rename('cn=group_to_people', newsuperior=peoplebase) ++ if singleTXN: ++ # Attempt modrdn that should fail, but the original entry should stay in the cache ++ with pytest.raises(ldap.OBJECT_CLASS_VIOLATION): ++ group.rename('cn=group_to_people', newsuperior=peoplebase) + +- # Should fail, but not with NO_SUCH_OBJECT as the original entry should still be in the cache +- with pytest.raises(ldap.OBJECT_CLASS_VIOLATION): ++ # Should fail, but not with NO_SUCH_OBJECT as the original entry should still be in the cache ++ with pytest.raises(ldap.OBJECT_CLASS_VIOLATION): ++ group.rename('cn=group_to_people', newsuperior=peoplebase) ++ else: ++ group.rename('cn=group_to_people', newsuperior=peoplebase) + group.rename('cn=group_to_people', newsuperior=peoplebase) + + # Done +diff --git a/dirsrvtests/tests/suites/memberof_plugin/memberof_deferred_repl_test.py b/dirsrvtests/tests/suites/memberof_plugin/memberof_deferred_repl_test.py +new file mode 100644 +index 000000000..e92df0661 +--- /dev/null ++++ b/dirsrvtests/tests/suites/memberof_plugin/memberof_deferred_repl_test.py +@@ -0,0 +1,178 @@ ++# --- BEGIN COPYRIGHT BLOCK --- ++# Copyright (C) 2024 Red Hat, Inc. ++# All rights reserved. ++# ++# License: GPL (version 3 or any later version). ++# See LICENSE for details. ++# --- END COPYRIGHT BLOCK --- ++# ++import logging ++import pytest ++import os ++import time ++from lib389._constants import DEFAULT_SUFFIX, AGMT_ATTR_LIST ++from lib389.topologies import topology_m2 as topo_m2 ++from lib389.agreement import Agreements ++from lib389.replica import Replicas ++from lib389.plugins import MemberOfPlugin ++from lib389.idm.user import UserAccounts ++from lib389.idm.group import Groups ++ ++log = logging.getLogger(__name__) ++ ++ ++def test_repl_deferred_updates(topo_m2): ++ """Test memberOf plugin deferred updates work in different types of ++ replicated environments ++ ++ :id: f7b20a60-7e52-411d-8693-cd7235df8e84 ++ :setup: 2 Supplier Instances ++ :steps: ++ 1. Enable memberOf with deferred updates on Supplier 1 ++ 2. Test deferred updates are replicated to Supplier 2 ++ 3. Enable memberOf with deferred updates on Supplier 2 ++ 4. Edit both agreements to strip memberOf updates ++ 5. Test that supplier 2 will update memberOf after receving replicated ++ group update ++ :expectedresults: ++ 1. Success ++ 2. Success ++ 3. Success ++ 4. Success ++ 5. Success ++ """ ++ s1 = topo_m2.ms["supplier1"] ++ s2 = topo_m2.ms["supplier2"] ++ ++ # Setup - create users and groups ++ s1_users = UserAccounts(s1, DEFAULT_SUFFIX) ++ s2_users = UserAccounts(s2, DEFAULT_SUFFIX) ++ user_dn_list = [] ++ for idx in range(5): ++ USER_NAME = f'user_{idx}' ++ user = s1_users.create(properties={ ++ 'uid': USER_NAME, ++ 'sn': USER_NAME, ++ 'cn': USER_NAME, ++ 'uidNumber': f'{idx}', ++ 'gidNumber': f'{idx}', ++ 'homeDirectory': f'/home/{USER_NAME}' ++ }) ++ user_dn_list.append(user.dn) ++ ++ groups = Groups(s1, DEFAULT_SUFFIX) ++ group = groups.create(properties={'cn': 'group'}) ++ ++ # ++ # Configure MO plugin Supplier 1, we're testing that deferred updates are ++ # replicated ++ # ++ memberof = MemberOfPlugin(s1) ++ memberof.enable() ++ memberof.set_autoaddoc('nsMemberOf') ++ memberof.set_memberofdeferredupdate('on') ++ #s1.config.set('nsslapd-errorlog','/dev/shm/slapd-supplier1/errors') ++ #s1.setLogLevel(65536) ++ #s1.setAccessLogLevel(260) ++ #s1.config.set('nsslapd-plugin-logging', 'on') ++ #s1.config.set('nsslapd-auditlog-logging-enabled', 'on') ++ s1.restart() ++ ++ # Update group ++ for dn in user_dn_list: ++ group.add('member', dn) ++ ++ # Check memberOf was added to all users in S1 and S2 ++ for count in range(10): ++ log.debug(f"Phase 1 - pass: {count}") ++ all_good = True ++ time.sleep(2) ++ # Check supplier 1 ++ users_s1 = s1_users.list() ++ for user in users_s1: ++ memberof = user.get_attr_vals('memberof') ++ log.debug("Checking %s" % user.dn) ++ log.debug("memberof: %s" % str(memberof)) ++ ++ for user in users_s1: ++ if not user.present('memberof'): ++ log.debug("missing memberof: %s !!!!" % user.dn) ++ all_good = False ++ break ++ else: ++ log.debug("Checking memberof: %s" % user.dn) ++ if not all_good: ++ continue ++ ++ # Supplier 1 is good, now check Supplier 2 ... ++ users_s2 = s2_users.list() ++ for user in users_s2: ++ if not user.present('memberof'): ++ all_good = False ++ break ++ ++ # If we are all good then we can break out ++ if all_good: ++ break ++ ++ assert all_good ++ ++ # ++ # New test that when a supplier receives a group update (memberOf is not ++ # replicated in this test) that it updates memberOf locally from the ++ # replicated group update ++ # ++ ++ # Exclude memberOf from replication ++ replica = Replicas(s1).get(DEFAULT_SUFFIX) ++ agmt = Agreements(s1, replica.dn).list()[0] ++ agmt.replace(AGMT_ATTR_LIST, '(objectclass=*) $ EXCLUDE memberOf') ++ agmt = Agreements(s2, replica.dn).list()[0] ++ agmt.replace(AGMT_ATTR_LIST, '(objectclass=*) $ EXCLUDE memberOf') ++ ++ # enable MO plugin on Supplier 2 ++ memberof = MemberOfPlugin(s2) ++ memberof.enable() ++ memberof.set_autoaddoc('nsMemberOf') ++ memberof.set_memberofdeferredupdate('on') ++ s1.restart() ++ s2.restart() ++ ++ # Remove members ++ group.remove_all('member') ++ ++ # Check memberOf is removed from users on S1 and S2 ++ all_good = True ++ for count in range(10): ++ log.debug(f"Phase 2 - pass: {count}") ++ all_good = True ++ time.sleep(2) ++ # Check supplier 1 ++ users_s1 = s1_users.list() ++ for user in users_s1: ++ if user.present('memberof'): ++ all_good = False ++ break ++ if not all_good: ++ continue ++ ++ # Supplier 1 is good, now check Supplier 2 ... ++ users_s2 = s2_users.list() ++ for user in users_s2: ++ if user.present('memberof'): ++ all_good = False ++ break ++ ++ # If we are all good then we can break out ++ if all_good: ++ break ++ ++ assert all_good ++ ++ ++if __name__ == '__main__': ++ # Run isolated ++ # -s for DEBUG mode ++ CURRENT_FILE = os.path.realpath(__file__) ++ pytest.main(["-s", CURRENT_FILE]) ++ +diff --git a/dirsrvtests/tests/suites/memberof_plugin/memberof_include_scopes_test.py b/dirsrvtests/tests/suites/memberof_plugin/memberof_include_scopes_test.py +index b310b15ea..e1d3b0a96 100644 +--- a/dirsrvtests/tests/suites/memberof_plugin/memberof_include_scopes_test.py ++++ b/dirsrvtests/tests/suites/memberof_plugin/memberof_include_scopes_test.py +@@ -9,6 +9,7 @@ + import pytest + import os + import ldap ++import time + from lib389.utils import ensure_str + from lib389.topologies import topology_st as topo + from lib389._constants import * +@@ -81,6 +82,10 @@ def test_multiple_scopes(topo): + memberof.enable() + memberof.add('memberOfEntryScope', SUBTREE_1) + memberof.add('memberOfEntryScope', SUBTREE_2) ++ if (memberof.get_memberofdeferredupdate() and memberof.get_memberofdeferredupdate().lower() == "on"): ++ delay = 3 ++ else: ++ delay = 0 + inst.restart() + + # Add setup entries +@@ -113,6 +118,7 @@ def test_multiple_scopes(topo): + user = UserAccount(topo.standalone, dn=INCLUDED_USER) + user.rename("uid=test_m1", newsuperior=EXCLUDED_SUBTREE) + ++ time.sleep(delay) + # Check memberOf and group are cleaned up + check_membership(inst, EXCLUDED_USER, GROUP_DN, False) + group = Group(topo.standalone, dn=GROUP_DN) +diff --git a/dirsrvtests/tests/suites/memberof_plugin/regression_test.py b/dirsrvtests/tests/suites/memberof_plugin/regression_test.py +index dd073dcc7..4c681a909 100644 +--- a/dirsrvtests/tests/suites/memberof_plugin/regression_test.py ++++ b/dirsrvtests/tests/suites/memberof_plugin/regression_test.py +@@ -10,7 +10,9 @@ import logging + import pytest + import os + import time ++import signal + import ldap ++from datetime import datetime + from random import sample + from lib389.utils import ds_is_older, ensure_list_bytes, ensure_bytes, ensure_str + from lib389.topologies import topology_m1h1c1 as topo, topology_st, topology_m2 as topo_m2 +@@ -22,6 +24,10 @@ from lib389.idm.group import Groups, Group + from lib389.replica import ReplicationManager + from lib389.tasks import * + from lib389.idm.nscontainer import nsContainers ++from lib389.idm.domain import Domain ++from lib389.dirsrv_log import DirsrvErrorLog ++from lib389.dseldif import DSEldif ++from contextlib import suppress + + + # Skip on older versions +@@ -302,6 +308,10 @@ def test_scheme_violation_errors_logged(topo_m2): + memberof = MemberOfPlugin(inst) + memberof.enable() + memberof.set_autoaddoc('nsMemberOf') ++ if (memberof.get_memberofdeferredupdate() and memberof.get_memberofdeferredupdate().lower() == "on"): ++ delay = 3 ++ else: ++ delay = 0 + inst.restart() + + users = UserAccounts(inst, SUFFIX) +@@ -315,6 +325,7 @@ def test_scheme_violation_errors_logged(topo_m2): + + testgroup.add('member', testuser.dn) + ++ time.sleep(delay) + user_memberof_attr = testuser.get_attr_val_utf8('memberof') + assert user_memberof_attr + log.info('memberOf attr value - {}'.format(user_memberof_attr)) +@@ -354,11 +365,19 @@ def test_memberof_with_changelog_reset(topo_m2): + + log.info("Configure memberof on M1 and M2") + memberof = MemberOfPlugin(m1) ++ if (memberof.get_memberofdeferredupdate() and memberof.get_memberofdeferredupdate().lower() == "on"): ++ # too difficult to make a large update work at shutdown ++ # need a dedicated test ++ return + memberof.enable() + memberof.set_autoaddoc('nsMemberOf') + m1.restart() + + memberof = MemberOfPlugin(m2) ++ if (memberof.get_memberofdeferredupdate() and memberof.get_memberofdeferredupdate().lower() == "on"): ++ # too difficult to make a large update work at shutdown ++ # need a dedicated test ++ return + memberof.enable() + memberof.set_autoaddoc('nsMemberOf') + m2.restart() +@@ -483,6 +502,10 @@ def test_memberof_group(topology_st): + memberof = MemberOfPlugin(inst) + memberof.enable() + memberof.replace('memberOfEntryScope', SUBTREE_1) ++ if (memberof.get_memberofdeferredupdate() and memberof.get_memberofdeferredupdate().lower() == "on"): ++ delay = 3 ++ else: ++ delay = 0 + inst.restart() + + add_container(inst, SUFFIX, 'sub1') +@@ -492,6 +515,7 @@ def test_memberof_group(topology_st): + add_group(inst, 'g1', SUBTREE_1) + add_group(inst, 'g2', SUBTREE_2) + ++ time.sleep(delay) + # _check_memberof + dn1 = '%s,%s' % ('uid=test_m1', SUBTREE_1) + dn2 = '%s,%s' % ('uid=test_m2', SUBTREE_1) +@@ -504,6 +528,7 @@ def test_memberof_group(topology_st): + + rename_entry(inst, 'cn=g2', SUBTREE_2, SUBTREE_1) + ++ time.sleep(delay) + g2n = '%s,%s' % ('cn=g2-new', SUBTREE_1) + _find_memberof_ext(inst, dn1, g1, True) + _find_memberof_ext(inst, dn2, g1, True) +@@ -566,6 +591,11 @@ def test_entrycache_on_modrdn_failure(topology_st): + + # only scopes peoplebase + _config_memberof_entrycache_on_modrdn_failure(topology_st.standalone) ++ memberof = MemberOfPlugin(topology_st.standalone) ++ if (memberof.get_memberofdeferredupdate() and memberof.get_memberofdeferredupdate().lower() == "on"): ++ delay = 3 ++ else: ++ delay = 0 + topology_st.standalone.restart(timeout=10) + + # create 10 users +@@ -587,6 +617,7 @@ def test_entrycache_on_modrdn_failure(topology_st): + ], + 'description': 'mygroup'}))) + ++ time.sleep(delay) + # Check the those entries have memberof with group0 + for i in range(2): + user_dn = 'cn=user%d,%s' % (i, peoplebase) +@@ -609,6 +640,7 @@ def test_entrycache_on_modrdn_failure(topology_st): + ], + 'description': 'mygroup'}))) + ++ time.sleep(delay) + # Check the those entries have not memberof with group1 + for i in range(2): + user_dn = 'cn=user%d,%s' % (i, peoplebase) +@@ -625,6 +657,8 @@ def test_entrycache_on_modrdn_failure(topology_st): + # move group1 into the scope and check user0 and user1 are memberof group1 + topology_st.standalone.rename_s(group1_dn, 'cn=group_in1', newsuperior=peoplebase, delold=0) + new_group1_dn = 'cn=group_in1,%s' % peoplebase ++ ++ time.sleep(delay) + for i in range(2): + user_dn = 'cn=user%d,%s' % (i, peoplebase) + ent = topology_st.standalone.getEntry(user_dn, ldap.SCOPE_BASE, "(objectclass=*)", ['memberof']) +@@ -646,7 +680,7 @@ def test_entrycache_on_modrdn_failure(topology_st): + 'cn=user3,%s' % peoplebase, + ], + 'description': entry_description}))) +- ++ time.sleep(delay) + # Check the those entries have not memberof with group2 + for i in (2, 3): + user_dn = 'cn=user%d,%s' % (i, peoplebase) +@@ -657,31 +691,36 @@ def test_entrycache_on_modrdn_failure(topology_st): + _disable_auto_oc_memberof(topology_st.standalone) + topology_st.standalone.restart(timeout=10) + +- # move group2 into the scope and check it fails +- try: ++ if (memberof.get_memberofdeferredupdate() and memberof.get_memberofdeferredupdate().lower() == "on"): ++ # move group2 into the scope and check it succeeds + topology_st.standalone.rename_s(group2_dn, 'cn=group_in2', newsuperior=peoplebase, delold=0) +- topology_st.standalone.log.info("This is unexpected, modrdn should fail as the member entry have not the appropriate objectclass") +- assert False +- except ldap.OBJECT_CLASS_VIOLATION: +- pass +- +- # retrieve the entry having the specific description value +- # check that the entry DN is the original group2 DN +- ents = topology_st.standalone.search_s(DEFAULT_SUFFIX, ldap.SCOPE_SUBTREE, '(cn=gr*)') +- found = False +- for ent in ents: +- topology_st.standalone.log.info("retrieve: %s with desc=%s" % (ent.dn, ent.getValue('description'))) +- if ent.getValue('description') == entry_description.encode(): +- found = True +- assert ent.dn == group2_dn +- assert found ++ topology_st.standalone.log.info("This is expected, modrdn does not fail only updates of members will fail") ++ else: ++ # move group2 into the scope and check it fails ++ try: ++ topology_st.standalone.rename_s(group2_dn, 'cn=group_in2', newsuperior=peoplebase, delold=0) ++ topology_st.standalone.log.info("This is unexpected, modrdn should fail as the member entry have not the appropriate objectclass") ++ assert False ++ except ldap.OBJECT_CLASS_VIOLATION: ++ pass ++ ++ # retrieve the entry having the specific description value ++ # check that the entry DN is the original group2 DN ++ ents = topology_st.standalone.search_s(DEFAULT_SUFFIX, ldap.SCOPE_SUBTREE, '(cn=gr*)') ++ found = False ++ for ent in ents: ++ topology_st.standalone.log.info("retrieve: %s with desc=%s" % (ent.dn, ent.getValue('description'))) ++ if ent.getValue('description') == entry_description.encode(): ++ found = True ++ assert ent.dn == group2_dn ++ assert found + + + def _config_memberof_silent_memberof_failure(server): + _config_memberof_entrycache_on_modrdn_failure(server) + + +-def test_silent_memberof_failure(topology_st): ++def test_silent_memberof_failure(topology_st, request): + """This test checks that if during a MODRDN, the memberof plugin fails + then MODRDN also fails + +@@ -720,6 +759,11 @@ def test_silent_memberof_failure(topology_st): + """ + # only scopes peoplebase + _config_memberof_silent_memberof_failure(topology_st.standalone) ++ memberof = MemberOfPlugin(topology_st.standalone) ++ if (memberof.get_memberofdeferredupdate() and memberof.get_memberofdeferredupdate().lower() == "on"): ++ delay = 3 ++ else: ++ delay = 0 + topology_st.standalone.restart(timeout=10) + + # first do some cleanup +@@ -727,10 +771,21 @@ def test_silent_memberof_failure(topology_st): + for i in range(10): + cn = 'user%d' % i + dn = 'cn=%s,%s' % (cn, peoplebase) +- topology_st.standalone.delete_s(dn) +- topology_st.standalone.delete_s('cn=group_in0,%s' % peoplebase) +- topology_st.standalone.delete_s('cn=group_in1,%s' % peoplebase) +- topology_st.standalone.delete_s('cn=group_out2,%s' % SUFFIX) ++ try: ++ topology_st.standalone.delete_s(dn) ++ except ldap.NO_SUCH_OBJECT: ++ pass ++ ++ for i in range(3): ++ try: ++ topology_st.standalone.delete_s('cn=group_in%d,%s' % (i, peoplebase)) ++ except ldap.NO_SUCH_OBJECT: ++ pass ++ ++ try: ++ topology_st.standalone.delete_s('cn=group_out2,%s' % SUFFIX) ++ except ldap.NO_SUCH_OBJECT: ++ pass + + # create 10 users + for i in range(10): +@@ -750,6 +805,7 @@ def test_silent_memberof_failure(topology_st): + ], + 'description': 'mygroup'}))) + ++ time.sleep(delay) + # Check the those entries have memberof with group0 + for i in range(2): + user_dn = 'cn=user%d,%s' % (i, peoplebase) +@@ -772,6 +828,7 @@ def test_silent_memberof_failure(topology_st): + ], + 'description': 'mygroup'}))) + ++ time.sleep(delay) + # Check the those entries have not memberof with group1 + for i in range(2): + user_dn = 'cn=user%d,%s' % (i, peoplebase) +@@ -788,6 +845,7 @@ def test_silent_memberof_failure(topology_st): + # move group1 into the scope and check user0 and user1 are memberof group1 + topology_st.standalone.rename_s(group1_dn, 'cn=group_in1', newsuperior=peoplebase, delold=0) + new_group1_dn = 'cn=group_in1,%s' % peoplebase ++ time.sleep(delay) + for i in range(2): + user_dn = 'cn=user%d,%s' % (i, peoplebase) + ent = topology_st.standalone.getEntry(user_dn, ldap.SCOPE_BASE, "(objectclass=*)", ['memberof']) +@@ -809,6 +867,7 @@ def test_silent_memberof_failure(topology_st): + ], + 'description': 'mygroup'}))) + ++ time.sleep(delay) + # Check the those entries have not memberof with group2 + for i in (2, 3): + user_dn = 'cn=user%d,%s' % (i, peoplebase) +@@ -819,14 +878,20 @@ def test_silent_memberof_failure(topology_st): + _disable_auto_oc_memberof(topology_st.standalone) + topology_st.standalone.restart(timeout=10) + +- # move group2 into the scope and check it fails +- try: ++ if (memberof.get_memberofdeferredupdate() and memberof.get_memberofdeferredupdate().lower() == "on"): ++ # move group2 into the scope and check it succeeds + topology_st.standalone.rename_s(group2_dn, 'cn=group_in2', newsuperior=peoplebase, delold=0) +- topology_st.standalone.log.info("This is unexpected, modrdn should fail as the member entry have not the appropriate objectclass") +- assert False +- except ldap.OBJECT_CLASS_VIOLATION: +- pass +- ++ topology_st.standalone.log.info("This is expected, modrdn does not fail only updates of members will fail") ++ else: ++ # move group2 into the scope and check it fails ++ try: ++ topology_st.standalone.rename_s(group2_dn, 'cn=group_in2', newsuperior=peoplebase, delold=0) ++ topology_st.standalone.log.info("This is unexpected, modrdn should fail as the member entry have not the appropriate objectclass") ++ assert False ++ except ldap.OBJECT_CLASS_VIOLATION: ++ pass ++ ++ time.sleep(delay) + # Check the those entries have not memberof + for i in (2, 3): + user_dn = 'cn=user%d,%s' % (i, peoplebase) +@@ -834,22 +899,32 @@ def test_silent_memberof_failure(topology_st): + topology_st.standalone.log.info("Should assert %s has memberof is %s" % (user_dn, ent.hasAttr('memberof'))) + assert not ent.hasAttr('memberof') + +- # Create a group3 in the scope +- group3_dn = 'cn=group3_in,%s' % peoplebase +- try: ++ if (memberof.get_memberofdeferredupdate() and memberof.get_memberofdeferredupdate().lower() == "on"): ++ # Create a group3 in the scope ++ group3_dn = 'cn=group3_in,%s' % peoplebase + topology_st.standalone.add_s(Entry((group3_dn, {'objectclass': ['top', 'groupofnames'], +- 'member': [ +- 'cn=user4,%s' % peoplebase, +- 'cn=user5,%s' % peoplebase, +- ], +- 'description': 'mygroup'}))) +- topology_st.standalone.log.info("This is unexpected, ADD should fail as the member entry have not the appropriate objectclass") +- assert False +- except ldap.OBJECT_CLASS_VIOLATION: +- pass +- except ldap.OPERATIONS_ERROR: +- pass +- ++ 'member': ['cn=user4,%s' % peoplebase, ++ 'cn=user5,%s' % peoplebase,], ++ 'description': 'mygroup'}))) ++ topology_st.standalone.log.info("This is expected, add does not fail only updates of members will fail") ++ else: ++ # Create a group3 in the scope ++ group3_dn = 'cn=group3_in,%s' % peoplebase ++ try: ++ topology_st.standalone.add_s(Entry((group3_dn, {'objectclass': ['top', 'groupofnames'], ++ 'member': [ ++ 'cn=user4,%s' % peoplebase, ++ 'cn=user5,%s' % peoplebase, ++ ], ++ 'description': 'mygroup'}))) ++ topology_st.standalone.log.info("This is unexpected, ADD should fail as the member entry have not the appropriate objectclass") ++ assert False ++ except ldap.OBJECT_CLASS_VIOLATION: ++ pass ++ except ldap.OPERATIONS_ERROR: ++ pass ++ ++ time.sleep(delay) + # Check the those entries do not have memberof + for i in (4, 5): + user_dn = 'cn=user%d,%s' % (i, peoplebase) +@@ -857,6 +932,219 @@ def test_silent_memberof_failure(topology_st): + topology_st.standalone.log.info("Should assert %s has memberof is %s" % (user_dn, ent.hasAttr('memberof'))) + assert not ent.hasAttr('memberof') + ++ def fin(): ++ # Cleanup the user[0-9]* entries ++ peoplebase = 'ou=people,%s' % SUFFIX ++ for i in range(10): ++ cn = 'user%d' % i ++ dn = 'cn=%s,%s' % (cn, peoplebase) ++ try: ++ topology_st.standalone.delete_s(dn) ++ except ldap.NO_SUCH_OBJECT: ++ pass ++ ++ # Cleanup the user_ entries ++ ents = topology_st.standalone.search_s(DEFAULT_SUFFIX, ldap.SCOPE_SUBTREE, '(uid=user_*)') ++ for ent in ents: ++ try: ++ topology_st.standalone.delete_s(ent.dn) ++ except ldap.NO_SUCH_OBJECT: ++ pass ++ ++ # Cleanup the test_ entries ++ ents = topology_st.standalone.search_s(DEFAULT_SUFFIX, ldap.SCOPE_SUBTREE, '(uid=test_*)') ++ for ent in ents: ++ try: ++ topology_st.standalone.delete_s(ent.dn) ++ except ldap.NO_SUCH_OBJECT: ++ pass ++ ++ for i in range(3): ++ try: ++ topology_st.standalone.delete_s('cn=group_in%d,%s' % (i, peoplebase)) ++ except ldap.NO_SUCH_OBJECT: ++ pass ++ ++ try: ++ topology_st.standalone.delete_s('cn=group_out2,%s' % SUFFIX) ++ except ldap.NO_SUCH_OBJECT: ++ pass ++ ++ request.addfinalizer(fin) ++ ++ ++def check_memberof_consistency(inst, group): ++ """This function checks that there is same number of: ++ - entries having 'memberOf' attribute ++ - members in the group ++ """ ++ suffix = Domain(inst, SUFFIX) ++ group_members = len(group.get_attr_vals('member')) ++ users_memberof = len(suffix.search(filter='(memberof=*)')) ++ assert group_members == users_memberof ++ ++ ++def count_global_fixup_message(errlog): ++ """This function returns a tuple (nbstarted, nsfinished) telling how many messages ++ (about Memberof pluging global fixed task) are found in the error log. ++ """ ++ nbstarted = 0 ++ nbfinished = 0 ++ for line in errlog.match('.*Memberof plugin [a-z]* the global fixup task.*'): ++ if 'started' in line: ++ nbstarted += 1 ++ elif 'finished' in line: ++ nbfinished += 1 ++ log.info(f'Global fixup start was started {nbstarted} times.') ++ log.info(f'Global fixup start was finished {nbfinished} times.') ++ return (nbstarted, nbfinished) ++ ++def _kill_instance(inst, sig=signal.SIGTERM, delay=None): ++ pid = None ++ try: ++ with open(inst.pid_file(), 'rb') as f: ++ for line in f.readlines(): ++ try: ++ pid = int(line.strip()) ++ break ++ except ValueError: ++ continue ++ except IOError: ++ pass ++ ++ if not pid or pid == 0: ++ pytest.raises(AssertionError) ++ ++ if delay: ++ time.sleep(delay) ++ os.kill(pid, signal.SIGKILL) ++ ++def test_shutdown_on_deferred_memberof(topology_st): ++ """This test checks that shutdown is handled properly if memberof updayes are deferred. ++ ++ :id: c5629cae-15a0-11ee-8807-482ae39447e5 ++ :setup: Standalone Instance ++ :steps: ++ 1. Enable memberof plugin to scope SUFFIX ++ 2. create 1000 users ++ 3. Create a large groups with 500 members ++ 4. Restart the instance (using the default 2 minutes timeout) ++ 5. Check that users memberof and group members are in sync. ++ 6. Modify the group to have 10 members. ++ 7. Restart the instance with short timeout ++ 8. Check that fixup task is in progress ++ 9. Wait until fixup task is completed ++ 10. Check that users memberof and group members are in sync. ++ :expectedresults: ++ 1. should succeed ++ 2. should succeed ++ 3. should succeed ++ 4. should succeed ++ 5. should succeed ++ 6. should succeed ++ 7. should succeed ++ 8. should succeed ++ 9. should succeed ++ 10. should succeed ++ """ ++ ++ inst = topology_st.standalone ++ inst.config.loglevel(vals=(ErrorLog.DEFAULT,ErrorLog.PLUGIN)) ++ errlog = DirsrvErrorLog(inst) ++ test_timeout = 900 ++ ++ # Step 1. Enable memberof plugin to scope SUFFIX ++ memberof = MemberOfPlugin(inst) ++ delay=0 ++ memberof.set_memberofdeferredupdate("on") ++ if (memberof.get_memberofdeferredupdate() and memberof.get_memberofdeferredupdate().lower() != "on"): ++ pytest.skip("Memberof deferred update not enabled or not supported."); ++ else: ++ delay=10 ++ memberof.set_attr('memberOf') ++ memberof.replace_groupattr('member') ++ memberof.remove_all_entryscope() ++ memberof.remove_all_excludescope() ++ memberof.remove_configarea() ++ memberof.remove_autoaddoc() ++ memberof.enable() ++ inst.restart() ++ ++ #Creates users and groups ++ users_dn = [] ++ ++ # Step 2. create 1000 users ++ for i in range(1000): ++ CN = '%s%d' % (USER_CN, i) ++ users = UserAccounts(inst, SUFFIX) ++ user_props = TEST_USER_PROPERTIES.copy() ++ user_props.update({'uid': CN, 'cn': CN, 'sn': '_%s' % CN}) ++ testuser = users.create(properties=user_props) ++ users_dn.append(testuser.dn) ++ ++ # Step 3. Create a large groups with 250 members ++ groups = Groups(inst, SUFFIX) ++ testgroup = groups.create(properties={'cn': 'group500', 'member': users_dn[0:249]}) ++ ++ # Step 4. Restart the instance (using the default 2 minutes timeout) ++ time.sleep(10) ++ log.info(f'Stopping instance at {datetime.now().strftime("%Y-%m-%d %H:%M:%S")}') ++ inst.stop() ++ log.info(f'Instance stopped at {datetime.now().strftime("%Y-%m-%d %H:%M:%S")}') ++ inst.start() ++ ++ time.sleep(delay) ++ # Step 5. Check that users memberof and group members are in sync. ++ check_memberof_consistency(inst, testgroup) ++ ++ # Step 6. Modify the group to get another big group. ++ testgroup.replace('member', users_dn[500:999]) ++ ++ # Step 7. Restart the instance with short timeout ++ pattern = 'deferred_thread_func - thread has stopped' ++ original_nbcleanstop = len(errlog.match(pattern)) ++ log.info(f'Stopping instance at {datetime.now().strftime("%Y-%m-%d %H:%M:%S")}') ++ _kill_instance(inst, sig=signal.SIGKILL, delay=5) ++ log.info(f'Instance stopped at {datetime.now().strftime("%Y-%m-%d %H:%M:%S")}') ++ # Double check that timeout occured during shutdown ++ # (i.e: no new 'deferred_thread_func - thread has stopped' message) ++ nbcleanstop = len(errlog.match(pattern)) ++ assert nbcleanstop == original_nbcleanstop ++ ++ original_nbfixupmsg = count_global_fixup_message(errlog) ++ log.info(f'Instance restarted after timeout at {datetime.now().strftime("%Y-%m-%d %H:%M:%S")}') ++ inst.restart() ++ assert inst.status() ++ log.info(f'Restart completed at {datetime.now().strftime("%Y-%m-%d %H:%M:%S")}') ++ ++ # Check that memberofneedfixup is present ++ dse = DSEldif(inst) ++ assert dse.get(memberof.dn, 'memberofneedfixup', single=True) ++ ++ # Step 8. Check that fixup task is in progress ++ # Note we have to wait as there may be some delay ++ elapsed_time = 0 ++ nbfixupmsg = count_global_fixup_message(errlog) ++ while nbfixupmsg[0] == original_nbfixupmsg[0]: ++ assert elapsed_time <= test_timeout ++ assert inst.status() ++ time.sleep(5) ++ elapsed_time += 5 ++ nbfixupmsg = count_global_fixup_message(errlog) ++ ++ # Step 9. Wait until fixup task is completed ++ while nbfixupmsg[1] == original_nbfixupmsg[1]: ++ assert elapsed_time <= test_timeout ++ assert inst.status() ++ time.sleep(10) ++ elapsed_time += 10 ++ nbfixupmsg = count_global_fixup_message(errlog) ++ ++ # Step 10. Check that users memberof and group members are in sync. ++ time.sleep(delay) ++ check_memberof_consistency(inst, testgroup) ++ ++ + if __name__ == '__main__': + # Run isolated + # -s for DEBUG mode +diff --git a/dirsrvtests/tests/suites/plugins/acceptance_test.py b/dirsrvtests/tests/suites/plugins/acceptance_test.py +index f34690483..193878f9a 100644 +--- a/dirsrvtests/tests/suites/plugins/acceptance_test.py ++++ b/dirsrvtests/tests/suites/plugins/acceptance_test.py +@@ -887,6 +887,10 @@ def test_memberof(topo, args=None): + + # stop the plugin, and start it + plugin = MemberOfPlugin(inst) ++ if (plugin.get_memberofdeferredupdate() and plugin.get_memberofdeferredupdate().lower() == "on"): ++ delay = 5 ++ else: ++ delay = 0 + plugin.disable() + plugin.enable() + +@@ -921,6 +925,7 @@ def test_memberof(topo, args=None): + 'memberOfGroupAttr': 'member', + 'memberOfAttr': MEMBER_ATTR}) + ++ time.sleep(delay) + # Check if the user now has a "memberOf" attribute + entries = inst.search_s(user1.dn, ldap.SCOPE_BASE, '({}=*)'.format(MEMBER_ATTR)) + assert entries +@@ -928,6 +933,7 @@ def test_memberof(topo, args=None): + # Remove "member" should remove "memberOf" from the entry + group.remove_all('member') + ++ time.sleep(delay) + # Check that "memberOf" was removed + entries = inst.search_s(user1.dn, ldap.SCOPE_BASE, '({}=*)'.format(MEMBER_ATTR)) + assert not entries +@@ -942,6 +948,7 @@ def test_memberof(topo, args=None): + ############################################################################ + group.replace('uniquemember', user1.dn) + ++ time.sleep(delay) + # Check if the user now has a "memberOf" attribute + entries = inst.search_s(user1.dn, ldap.SCOPE_BASE, '({}=*)'.format(MEMBER_ATTR)) + assert entries +@@ -949,6 +956,7 @@ def test_memberof(topo, args=None): + # Remove "uniquemember" should remove "memberOf" from the entry + group.remove_all('uniquemember') + ++ time.sleep(delay) + # Check that "memberOf" was removed + entries = inst.search_s(user1.dn, ldap.SCOPE_BASE, '({}=*)'.format(MEMBER_ATTR)) + assert not entries +@@ -970,6 +978,7 @@ def test_memberof(topo, args=None): + 'member': user1.dn}) + group.add('objectclass', 'groupOfUniqueNames') + ++ time.sleep(delay) + # Test the shared config + # Check if the user now has a "memberOf" attribute + entries = inst.search_s(user1.dn, ldap.SCOPE_BASE, '({}=*)'.format(MEMBER_ATTR)) +@@ -977,6 +986,7 @@ def test_memberof(topo, args=None): + + group.remove_all('member') + ++ time.sleep(delay) + # Check that "memberOf" was removed + entries = inst.search_s(user1.dn, ldap.SCOPE_BASE, '({}=*)'.format(MEMBER_ATTR)) + assert not entries +@@ -988,6 +998,7 @@ def test_memberof(topo, args=None): + + group.replace('uniquemember', user1.dn) + ++ time.sleep(delay) + # Check if the user now has a "memberOf" attribute + entries = inst.search_s(user1.dn, ldap.SCOPE_BASE, '({}=*)'.format(MEMBER_ATTR)) + assert entries +@@ -995,6 +1006,7 @@ def test_memberof(topo, args=None): + # Remove "uniquemember" should remove "memberOf" from the entry + group.remove_all('uniquemember') + ++ time.sleep(delay) + # Check that "memberOf" was removed + entries = inst.search_s(user1.dn, ldap.SCOPE_BASE, '({}=*)'.format(MEMBER_ATTR)) + assert not entries +@@ -1007,9 +1019,11 @@ def test_memberof(topo, args=None): + + # Remove shared config from plugin + plugin.remove_configarea() ++ inst.restart() + + group.replace('member', user1.dn) + ++ time.sleep(delay) + # Check if the user now has a "memberOf" attribute + entries = inst.search_s(user1.dn, ldap.SCOPE_BASE, '({}=*)'.format(MEMBER_ATTR)) + assert entries +@@ -1017,6 +1031,7 @@ def test_memberof(topo, args=None): + # Remove "uniquemember" should remove "memberOf" from the entry + group.remove_all('member') + ++ time.sleep(delay) + # Check that "memberOf" was removed + entries = inst.search_s(user1.dn, ldap.SCOPE_BASE, '({}=*)'.format(MEMBER_ATTR)) + assert not entries +@@ -1036,6 +1051,7 @@ def test_memberof(topo, args=None): + # Add uniquemember, should not update USER1 + group.replace('uniquemember', user1.dn) + ++ time.sleep(delay) + # Check for "memberOf" + entries = inst.search_s(user1.dn, ldap.SCOPE_BASE, '({}=*)'.format(MEMBER_ATTR)) + assert not entries +diff --git a/dirsrvtests/tests/suites/plugins/entryusn_test.py b/dirsrvtests/tests/suites/plugins/entryusn_test.py +index 60580c370..ab82f1df5 100644 +--- a/dirsrvtests/tests/suites/plugins/entryusn_test.py ++++ b/dirsrvtests/tests/suites/plugins/entryusn_test.py +@@ -115,6 +115,11 @@ def test_entryusn_no_duplicates(topology_st, setup): + """ + + inst = topology_st.standalone ++ plugin = MemberOfPlugin(inst) ++ if (plugin.get_memberofdeferredupdate() and plugin.get_memberofdeferredupdate().lower() == "on"): ++ delay = 3 ++ else: ++ delay = 0 + config = Config(inst) + config.replace('nsslapd-accesslog-level', '260') # Internal op + config.replace('nsslapd-errorlog-level', '65536') +@@ -125,6 +130,7 @@ def test_entryusn_no_duplicates(topology_st, setup): + groups = setup["groups"] + + groups[0].replace('member', users[0].dn) ++ time.sleep(delay) + entryusn_list.append(users[0].get_attr_val_int('entryusn')) + log.info(f"{users[0].dn}_1: {entryusn_list[-1:]}") + entryusn_list.append(groups[0].get_attr_val_int('entryusn')) +@@ -132,6 +138,7 @@ def test_entryusn_no_duplicates(topology_st, setup): + check_entryusn_no_duplicates(entryusn_list) + + groups[1].replace('member', [users[0].dn, users[1].dn]) ++ time.sleep(delay) + entryusn_list.append(users[0].get_attr_val_int('entryusn')) + log.info(f"{users[0].dn}_2: {entryusn_list[-1:]}") + entryusn_list.append(users[1].get_attr_val_int('entryusn')) +diff --git a/dirsrvtests/tests/suites/plugins/memberof_test.py b/dirsrvtests/tests/suites/plugins/memberof_test.py +index d3b32c856..2de1389fd 100644 +--- a/dirsrvtests/tests/suites/plugins/memberof_test.py ++++ b/dirsrvtests/tests/suites/plugins/memberof_test.py +@@ -11,6 +11,7 @@ from lib389.tasks import * + from lib389.utils import * + from lib389.topologies import topology_st + from lib389._constants import PLUGIN_MEMBER_OF, SUFFIX ++from lib389.plugins import MemberOfPlugin + + pytestmark = pytest.mark.tier1 + +@@ -34,6 +35,16 @@ USERS_CONTAINER = "ou=people,%s" % SUFFIX + GROUP_RDN = "group" + GROUPS_CONTAINER = "ou=groups,%s" % SUFFIX + ++def _memberof_checking_delay(inst): ++ memberof = MemberOfPlugin(inst) ++ if (memberof.get_memberofdeferredupdate() and memberof.get_memberofdeferredupdate().lower() == "on"): ++ # In case of deferred update then a safe delay ++ # to let the deferred thread processing is 3 sec ++ delay = 3 ++ else: ++ # Else it is the same TXN, no reason to wait ++ delay = 0 ++ return delay + + def _set_memberofgroupattr_add(topology_st, values): + topology_st.standalone.modify_s(MEMBEROF_PLUGIN_DN, +@@ -200,6 +211,10 @@ def test_member_add(topology_st): + 2. Success + 3. Success + """ ++ delay = _memberof_checking_delay(topology_st.standalone) ++ ++ topology_st.standalone.plugins.enable(name=PLUGIN_MEMBER_OF) ++ topology_st.standalone.restart() + + memofenh1 = _create_user(topology_st, 'memofenh1') + memofenh2 = _create_user(topology_st, 'memofenh2') +@@ -216,6 +231,7 @@ def test_member_add(topology_st): + log.info("Update %s is memberof %s (uniqueMember)" % (memofenh2, memofegrp2)) + topology_st.standalone.modify_s(ensure_str(memofegrp2), mods) + ++ time.sleep(delay) + # assert enh1 is member of grp1 and grp2 + assert _check_memberof(topology_st, member=memofenh1, group=memofegrp1) + assert _check_memberof(topology_st, member=memofenh1, group=memofegrp2) +@@ -238,6 +254,8 @@ def test_member_delete_gr1(topology_st): + 2. Success + """ + ++ delay = _memberof_checking_delay(topology_st.standalone) ++ + memofenh1 = _get_user_dn('memofenh1') + memofenh2 = _get_user_dn('memofenh2') + +@@ -247,6 +265,7 @@ def test_member_delete_gr1(topology_st): + mods = [(ldap.MOD_DELETE, 'member', memofenh1)] + topology_st.standalone.modify_s(ensure_str(memofegrp1), mods) + ++ time.sleep(delay) + # assert enh1 is NOT member of grp1 and is member of grp2 + assert not _check_memberof(topology_st, member=memofenh1, group=memofegrp1) + assert _check_memberof(topology_st, member=memofenh1, group=memofegrp2) +@@ -268,6 +287,7 @@ def test_member_delete_gr2(topology_st): + 1. Success + 2. Success + """ ++ delay = _memberof_checking_delay(topology_st.standalone) + + memofenh1 = _get_user_dn('memofenh1') + memofenh2 = _get_user_dn('memofenh2') +@@ -279,6 +299,7 @@ def test_member_delete_gr2(topology_st): + mods = [(ldap.MOD_DELETE, 'uniqueMember', memofenh2)] + topology_st.standalone.modify_s(ensure_str(memofegrp2), mods) + ++ time.sleep(delay) + # assert enh1 is NOT member of grp1 and is member of grp2 + assert not _check_memberof(topology_st, member=memofenh1, group=memofegrp1) + assert _check_memberof(topology_st, member=memofenh1, group=memofegrp2) +@@ -300,6 +321,7 @@ def test_member_delete_all(topology_st): + 1. Success + 2. Success + """ ++ delay = _memberof_checking_delay(topology_st.standalone) + + memofenh1 = _get_user_dn('memofenh1') + memofenh2 = _get_user_dn('memofenh2') +@@ -315,6 +337,7 @@ def test_member_delete_all(topology_st): + mods = [(ldap.MOD_DELETE, 'member', memofenh1)] + topology_st.standalone.modify_s(ensure_str(memofegrp2), mods) + ++ time.sleep(delay) + # assert enh1 is NOT member of grp1 and is member of grp2 + assert not _check_memberof(topology_st, member=memofenh1, group=memofegrp1) + assert not _check_memberof(topology_st, member=memofenh1, group=memofegrp2) +@@ -338,6 +361,7 @@ def test_member_after_restart(topology_st): + 2. Success + 3. Success + """ ++ delay = _memberof_checking_delay(topology_st.standalone) + + memofenh1 = _get_user_dn('memofenh1') + memofenh2 = _get_user_dn('memofenh2') +@@ -353,6 +377,7 @@ def test_member_after_restart(topology_st): + log.info("Update %s is memberof %s (uniqueMember)" % (memofenh2, memofegrp2)) + topology_st.standalone.modify_s(ensure_str(memofegrp2), mods) + ++ time.sleep(delay) + # assert enh1 is member of grp1 and is NOT member of grp2 + assert _check_memberof(topology_st, member=memofenh1, group=memofegrp1) + assert not _check_memberof(topology_st, member=memofenh1, group=memofegrp2) +@@ -548,6 +573,8 @@ def test_member_uniquemember_same_user(topology_st): + 6. Success + """ + ++ delay = _memberof_checking_delay(topology_st.standalone) ++ + memofenh1 = _get_user_dn('memofenh1') + memofenh2 = _get_user_dn('memofenh2') + +@@ -570,6 +597,7 @@ def test_member_uniquemember_same_user(topology_st): + log.info("Update %s is memberof %s (uniqueMember)" % (memofenh1, memofegrp3)) + topology_st.standalone.modify_s(ensure_str(memofegrp3), mods) + ++ time.sleep(delay) + # assert enh1 is member of + # - grp1 (member) + # - not grp2 +@@ -586,6 +614,7 @@ def test_member_uniquemember_same_user(topology_st): + log.info("Update %s is memberof %s (member)" % (memofenh2, memofegrp3)) + topology_st.standalone.modify_s(ensure_str(memofegrp3), mods) + ++ time.sleep(delay) + # assert enh1 is member of + # - grp1 (member) + # - not grp2 +@@ -658,6 +687,7 @@ def test_member_not_exists(topology_st): + 2. Success + 3. Success + """ ++ delay = _memberof_checking_delay(topology_st.standalone) + + memofenh1 = _get_user_dn('memofenh1') + memofenh2 = _get_user_dn('memofenh2') +@@ -700,6 +730,7 @@ def test_member_not_exists(topology_st): + assert ensure_bytes(dummy1) not in ent.getValues('uniqueMember') + assert ensure_bytes(dummy2) in ent.getValues('uniqueMember') + ++ time.sleep(delay) + # assert enh1 is member of + # - grp1 (member) + # - not grp2 +@@ -770,6 +801,7 @@ def test_member_not_exists_complex(topology_st): + 5. Success + 6. Success + """ ++ delay = _memberof_checking_delay(topology_st.standalone) + + memofenh1 = _get_user_dn('memofenh1') + memofenh2 = _get_user_dn('memofenh2') +@@ -807,6 +839,7 @@ def test_member_not_exists_complex(topology_st): + log.info("Update %s is memberof %s (uniqueMember)" % (memofenh1, memofegrp016)) + topology_st.standalone.modify_s(ensure_str(memofegrp016), mods) + ++ time.sleep(delay) + # assert enh1 is member of + # - grp1 (member) + # - not grp2 +@@ -851,6 +884,7 @@ def test_member_not_exists_complex(topology_st): + assert ent.hasAttr('uniqueMember') + assert ensure_bytes(dummy1) in ent.getValues('uniqueMember') + ++ time.sleep(delay) + # assert enh1 is member of + # - grp1 (member) + # - not grp2 +@@ -944,6 +978,8 @@ def test_complex_group_scenario_1(topology_st): + 8. Success + """ + ++ delay = _memberof_checking_delay(topology_st.standalone) ++ + memofenh1 = _get_user_dn('memofenh1') + memofenh2 = _get_user_dn('memofenh2') + +@@ -1013,6 +1049,7 @@ def test_complex_group_scenario_1(topology_st): + log.info("Update %s is memberof %s (memberuid)" % (memofuser3, memofegrp017)) + topology_st.standalone.modify_s(ensure_str(memofegrp017), mods) + ++ time.sleep(delay) + # assert enh1 is member of + # - grp1 (member) + # - not grp2 +@@ -1168,6 +1205,7 @@ def test_complex_group_scenario_2(topology_st): + 6. Success + 7. Success + """ ++ delay = _memberof_checking_delay(topology_st.standalone) + + memofenh1 = _get_user_dn('memofenh1') + memofenh2 = _get_user_dn('memofenh2') +@@ -1263,6 +1301,7 @@ def test_complex_group_scenario_2(topology_st): + log.info("Update %s is memberof %s (memberuid)" % (memofuser1, memofegrp017)) + topology_st.standalone.modify_s(ensure_str(memofegrp018), mods) + ++ time.sleep(delay) + # assert user1 is member of + # - not grp1 + # - not grp2 +@@ -1284,6 +1323,7 @@ def test_complex_group_scenario_2(topology_st): + log.info("Update %s is no longer memberof %s (uniqueMember)" % (memofuser1, memofegrp018)) + topology_st.standalone.modify_s(ensure_str(memofegrp018), mods) + ++ time.sleep(delay) + # assert user1 is member of + # - not grp1 + # - not grp2 +@@ -1306,6 +1346,7 @@ def test_complex_group_scenario_2(topology_st): + topology_st.standalone.delete_s(ensure_str(memofuser3)) + topology_st.standalone.delete_s(ensure_str(memofegrp017)) + ++ time.sleep(delay) + # assert enh1 is member of + # - grp1 (member) + # - not grp2 +@@ -1442,6 +1483,7 @@ def test_complex_group_scenario_3(topology_st): + 11. Success + 12. Success + """ ++ delay = _memberof_checking_delay(topology_st.standalone) + + memofenh1 = _get_user_dn('memofenh1') + memofenh2 = _get_user_dn('memofenh2') +@@ -1503,6 +1545,7 @@ def test_complex_group_scenario_3(topology_st): + mods = [(ldap.MOD_ADD, 'member', memofegrp019_2), (ldap.MOD_ADD, 'member', memofegrp019_3)] + topology_st.standalone.modify_s(ensure_str(memofegrp019_1), mods) + ++ time.sleep(delay) + # assert memofegrp019_1 is member of + # - not grp1 + # - not grp2 +@@ -1610,6 +1653,7 @@ def test_complex_group_scenario_3(topology_st): + topology_st.standalone.delete_s(ensure_str(memofegrp019_2)) + topology_st.standalone.delete_s(ensure_str(memofegrp019_3)) + ++ time.sleep(delay) + # assert enh1 is member of + # - grp1 (member) + # - not grp2 +@@ -1677,6 +1721,7 @@ def test_complex_group_scenario_4(topology_st): + 5. Success + 6. Success + """ ++ delay = _memberof_checking_delay(topology_st.standalone) + + memofenh1 = _get_user_dn('memofenh1') + memofenh2 = _get_user_dn('memofenh2') +@@ -1740,6 +1785,7 @@ def test_complex_group_scenario_4(topology_st): + mods.append((ldap.MOD_ADD, 'member', grp)) + topology_st.standalone.modify_s(ensure_str(memofegrp020_5), mods) + ++ time.sleep(delay) + assert _check_memberof(topology_st, member=memofuser1, group=memofegrp020_1) + assert _check_memberof(topology_st, member=memofuser1, group=memofegrp020_2) + assert _check_memberof(topology_st, member=memofuser1, group=memofegrp020_3) +@@ -1817,6 +1863,7 @@ def test_complex_group_scenario_5(topology_st): + 10. Success + 11. Success + """ ++ delay = _memberof_checking_delay(topology_st.standalone) + + memofenh1 = _get_user_dn('memofenh1') + memofenh2 = _get_user_dn('memofenh2') +@@ -1882,6 +1929,7 @@ def test_complex_group_scenario_5(topology_st): + mods.append((ldap.MOD_ADD, 'member', grp)) + topology_st.standalone.modify_s(ensure_str(memofegrp020_5), mods) + ++ time.sleep(delay) + # assert user[1-4] are member of grp20_5 + for user in [memofuser1, memofuser2, memofuser3, memofuser4]: + assert _check_memberof(topology_st, member=user, group=memofegrp020_5) +@@ -2009,6 +2057,7 @@ def test_complex_group_scenario_6(topology_st): + 7. Success + 8. Success + """ ++ delay = _memberof_checking_delay(topology_st.standalone) + + memofenh1 = _get_user_dn('memofenh1') + memofenh2 = _get_user_dn('memofenh2') +@@ -2108,6 +2157,7 @@ def test_complex_group_scenario_6(topology_st): + mods = [(ldap.MOD_ADD, 'member', x[1])] + topology_st.standalone.modify_s(ensure_str(x[0]), mods) + ++ time.sleep(delay) + # check that user[1-4] are 'member' and 'uniqueMember' of the grp20_[1-4] + for x in [(memofegrp020_1, memofuser1), + (memofegrp020_2, memofuser2), +@@ -2287,6 +2337,7 @@ def test_complex_group_scenario_7(topology_st): + |<--uniquemember-/ + + """ ++ delay = _memberof_checking_delay(topology_st.standalone) + + memofenh1 = _get_user_dn('memofenh1') + memofenh2 = _get_user_dn('memofenh2') +@@ -2412,6 +2463,7 @@ def test_complex_group_scenario_7(topology_st): + |----member ---> G4 ---member/uniqueMember -> U4 + |<--uniquemember-/ + """ ++ time.sleep(delay) + verify_post_023(topology_st, memofegrp020_1, memofegrp020_2, memofegrp020_3, memofegrp020_4, memofegrp020_5, + memofuser1, memofuser2, memofuser3, memofuser4) + +@@ -2504,6 +2556,7 @@ def test_complex_group_scenario_8(topology_st): + |<--uniquemember-/ + + """ ++ delay = _memberof_checking_delay(topology_st.standalone) + + memofuser1 = _get_user_dn('memofuser1') + memofuser2 = _get_user_dn('memofuser2') +@@ -2521,6 +2574,8 @@ def test_complex_group_scenario_8(topology_st): + # ADD user1 as 'member' of grp20_1 + mods = [(ldap.MOD_ADD, 'member', memofuser1)] + topology_st.standalone.modify_s(ensure_str(memofegrp020_1), mods) ++ ++ time.sleep(delay) + verify_post_024(topology_st, memofegrp020_1, memofegrp020_2, memofegrp020_3, memofegrp020_4, memofegrp020_5, + memofuser1, memofuser2, memofuser3, memofuser4) + +@@ -2599,6 +2654,11 @@ def test_complex_group_scenario_9(topology_st): + |----member ---> G4 + + """ ++ memberof = MemberOfPlugin(topology_st.standalone) ++ if (memberof.get_memberofdeferredupdate() and memberof.get_memberofdeferredupdate().lower() == "on"): ++ delay = 3 ++ else: ++ delay = 0 + + memofuser1 = _get_user_dn('memofuser1') + memofuser2 = _get_user_dn('memofuser2') +@@ -2652,6 +2712,7 @@ def test_complex_group_scenario_9(topology_st): + + """ + ++ time.sleep(delay) + verify_post_025(topology_st, memofegrp020_1, memofegrp020_2, memofegrp020_3, memofegrp020_4, memofegrp020_5, + memofuser1, memofuser2, memofuser3, memofuser4) + +@@ -2689,6 +2750,12 @@ def test_memberof_auto_add_oc(topology_st): + 11. Success + """ + ++ memberof = MemberOfPlugin(topology_st.standalone) ++ if (memberof.get_memberofdeferredupdate() and memberof.get_memberofdeferredupdate().lower() == "on"): ++ delay = 3 ++ else: ++ delay = 0 ++ + # enable dynamic plugins + try: + topology_st.standalone.modify_s(DN_CONFIG, +@@ -2731,6 +2798,7 @@ def test_memberof_auto_add_oc(topology_st): + log.fatal('Failed to add group entry, error: ' + e.message['desc']) + assert False + ++ time.sleep(delay) + # Assert memberOf on user1 + _check_memberof(topology_st, USER1_DN, GROUP_DN) + +diff --git a/ldap/servers/plugins/memberof/memberof.c b/ldap/servers/plugins/memberof/memberof.c +index a5f48d2c0..e75b99b14 100644 +--- a/ldap/servers/plugins/memberof/memberof.c ++++ b/ldap/servers/plugins/memberof/memberof.c +@@ -36,11 +36,13 @@ + + #include + #include ++#include + #include "slapi-plugin.h" + #include "string.h" + #include "nspr.h" + #include "plhash.h" + #include "memberof.h" ++#include "slap.h" + + static Slapi_PluginDesc pdesc = {"memberof", VENDOR, + DS_PACKAGE_VERSION, "memberof plugin"}; +@@ -108,6 +110,7 @@ typedef struct _task_data + /*** function prototypes ***/ + + /* exported functions */ ++static int memberof_be_postop_init(Slapi_PBlock *pb); + int memberof_postop_init(Slapi_PBlock *pb); + static int memberof_internal_postop_init(Slapi_PBlock *pb); + static int memberof_preop_init(Slapi_PBlock *pb); +@@ -119,6 +122,7 @@ static int memberof_postop_modify(Slapi_PBlock *pb); + static int memberof_postop_add(Slapi_PBlock *pb); + static int memberof_postop_start(Slapi_PBlock *pb); + static int memberof_postop_close(Slapi_PBlock *pb); ++int memberof_push_deferred_task(Slapi_PBlock *pb); + + /* supporting cast */ + static int memberof_oktodo(Slapi_PBlock *pb); +@@ -158,6 +162,7 @@ static void memberof_task_destructor(Slapi_Task *task); + static void memberof_fixup_task_thread(void *arg); + static int memberof_fix_memberof(MemberOfConfig *config, Slapi_Task *task, task_data *td); + static int memberof_fix_memberof_callback(Slapi_Entry *e, void *callback_data); ++static int memberof_fixup_memberof_callback(Slapi_Entry *e, void *callback_data); + static int memberof_entry_in_scope(MemberOfConfig *config, Slapi_DN *sdn); + static int memberof_add_objectclass(char *auto_add_oc, const char *dn); + static int memberof_add_memberof_attr(LDAPMod **mods, const char *dn, char *add_oc); +@@ -234,6 +239,22 @@ memberof_postop_init(Slapi_PBlock *pb) + slapi_pblock_set(pb, SLAPI_PLUGIN_START_FN, (void *)memberof_postop_start) != 0 || + slapi_pblock_set(pb, SLAPI_PLUGIN_CLOSE_FN, (void *)memberof_postop_close) != 0); + ++ if (!ret) { ++ ++ if (slapi_register_plugin("bepostoperation", /* op type */ ++ 1, /* Enabled */ ++ "memberof_bepostop_init", /* this function desc */ ++ memberof_be_postop_init, /* init func for be_post op */ ++ MEMBEROF_BEPOSTOP_DESC, /* plugin desc */ ++ NULL, ++ memberof_plugin_identity /* access control */)) { ++ slapi_log_err(SLAPI_LOG_ERR, MEMBEROF_PLUGIN_SUBSYSTEM, ++ "memberof_postop_init - memberof_be_postop_init Failed\n"); ++ ret = 1; ++ } ++ ++ } ++ + if (!ret && !usetxn && + slapi_register_plugin("internalpostoperation", /* op type */ + 1, /* Enabled */ +@@ -272,48 +293,793 @@ memberof_postop_init(Slapi_PBlock *pb) + slapi_log_err(SLAPI_LOG_TRACE, MEMBEROF_PLUGIN_SUBSYSTEM, + "<-- memberof_postop_init\n"); + +- return ret; ++ return ret; ++} ++ ++static int ++memberof_be_postop_init(Slapi_PBlock *pb) ++{ ++ int rc; ++ rc = slapi_pblock_set(pb, SLAPI_PLUGIN_BE_POST_ADD_FN, (void *)memberof_push_deferred_task); ++ rc |= slapi_pblock_set(pb, SLAPI_PLUGIN_BE_POST_DELETE_FN, (void *)memberof_push_deferred_task); ++ rc |= slapi_pblock_set(pb, SLAPI_PLUGIN_BE_POST_MODIFY_FN, (void *)memberof_push_deferred_task); ++ rc |= slapi_pblock_set(pb, SLAPI_PLUGIN_BE_POST_MODRDN_FN, (void *)memberof_push_deferred_task); ++ return (rc); ++} ++ ++static int ++memberof_preop_init(Slapi_PBlock *pb) ++{ ++ int status = 0; ++ ++ if (slapi_pblock_set(pb, SLAPI_PLUGIN_VERSION, SLAPI_PLUGIN_VERSION_01) != 0 || ++ slapi_pblock_set(pb, SLAPI_PLUGIN_DESCRIPTION, (void *)&pdesc) != 0 || ++ slapi_pblock_set(pb, premodfn, (void *)memberof_shared_config_validate) != 0) { ++ slapi_log_err(SLAPI_LOG_ERR, MEMBEROF_PLUGIN_SUBSYSTEM, ++ "memberof_preop_init: Failed to register plugin\n"); ++ status = -1; ++ } ++ ++ return status; ++} ++ ++static int ++memberof_internal_postop_init(Slapi_PBlock *pb) ++{ ++ int status = 0; ++ ++ if (slapi_pblock_set(pb, SLAPI_PLUGIN_VERSION, ++ SLAPI_PLUGIN_VERSION_01) != 0 || ++ slapi_pblock_set(pb, SLAPI_PLUGIN_DESCRIPTION, ++ (void *)&pdesc) != 0 || ++ slapi_pblock_set(pb, SLAPI_PLUGIN_INTERNAL_POST_DELETE_FN, ++ (void *)memberof_postop_del) != 0 || ++ slapi_pblock_set(pb, SLAPI_PLUGIN_INTERNAL_POST_MODRDN_FN, ++ (void *)memberof_postop_modrdn) != 0 || ++ slapi_pblock_set(pb, SLAPI_PLUGIN_INTERNAL_POST_MODIFY_FN, ++ (void *)memberof_postop_modify) != 0 || ++ slapi_pblock_set(pb, SLAPI_PLUGIN_INTERNAL_POST_ADD_FN, ++ (void *)memberof_postop_add) != 0) { ++ slapi_log_err(SLAPI_LOG_ERR, MEMBEROF_PLUGIN_SUBSYSTEM, ++ "memberof_internal_postop_init - Failed to register plugin\n"); ++ status = -1; ++ } ++ ++ return status; ++} ++ ++/* Caller must hold deferred_list->deferred_list_mutex ++ * deferred_list is FIFO ++ */ ++MemberofDeferredTask * ++remove_deferred_task(MemberofDeferredList *deferred_list) ++{ ++ MemberofDeferredTask *task; ++ if ((deferred_list == NULL) || (deferred_list->current_task == 0)) { ++ slapi_log_err(SLAPI_LOG_ERR, MEMBEROF_PLUGIN_SUBSYSTEM, ++ "Unexpected empty/not allocated deferred list\n"); ++ return NULL; ++ } ++ ++ /* extract the task from the queue */ ++ task = deferred_list->tasks_queue; ++ if (task == NULL) { ++ /* error condition current_task said there was a task available */ ++ slapi_log_err(SLAPI_LOG_ERR, MEMBEROF_PLUGIN_SUBSYSTEM, ++ "Unexpected current_task counter said there was %d task(s)\n", ++ deferred_list->current_task); ++ deferred_list->current_task = 0; ++ return NULL; ++ } ++ deferred_list->tasks_queue = task->prev; ++ if (deferred_list->tasks_queue) { ++ /* the queue is not empty ++ * Make this task the end of the queue ++ */ ++ deferred_list->tasks_queue->next = NULL; ++ } else { ++ /* The queue is now empty reset head */ ++ deferred_list->tasks_head = NULL; ++ } ++ task->prev = NULL; ++ task->next = NULL; ++ deferred_list->current_task--; ++ if (task) deferred_list->total_removed++; ++ ++ return task; ++} ++ ++/* ++ * deferred_list is FIFO ++ */ ++int ++add_deferred_task(MemberofDeferredList *deferred_list, MemberofDeferredTask *task) ++{ ++ if (deferred_list == NULL) { ++ slapi_log_err(SLAPI_LOG_ERR, MEMBEROF_PLUGIN_SUBSYSTEM, ++ "Not allocated deferred list\n"); ++ return -1; ++ } ++ pthread_mutex_lock(&deferred_list->deferred_list_mutex); ++ /* add the task at the head of the queue */ ++ if (deferred_list->tasks_head == NULL) { ++ /* this is the first task in the queue */ ++ task->next = NULL; ++ task->prev = NULL; ++ deferred_list->tasks_head = task; ++ deferred_list->tasks_queue = task; ++ deferred_list->current_task = 1; ++ } else { ++ deferred_list->tasks_head->prev = task; ++ task->next = deferred_list->tasks_head; ++ task->prev = NULL; ++ deferred_list->tasks_head = task; ++ deferred_list->current_task++; ++ } ++ deferred_list->total_added++; ++ /* wake up deferred_thread_func */ ++ pthread_cond_signal(&deferred_list->deferred_list_cv); ++ pthread_mutex_unlock(&deferred_list->deferred_list_mutex); ++ ++ return 0; ++} ++ ++typedef struct _memberof_del_dn_data ++{ ++ char *dn; ++ char *type; ++} memberof_del_dn_data; ++ ++int ++deferred_modrdn_func(MemberofDeferredModrdnTask *task) ++{ ++ Slapi_PBlock *pb; ++ MemberOfConfig *mainConfig = 0; ++ MemberOfConfig configCopy = {0}; ++ struct slapi_entry *pre_e = NULL; ++ struct slapi_entry *post_e = NULL; ++ Slapi_DN *pre_sdn = 0; ++ Slapi_DN *post_sdn = 0; ++ Slapi_DN *sdn = NULL; ++ int ret = SLAPI_PLUGIN_SUCCESS; ++ ++ pb = task->pb; ++ slapi_pblock_get(pb, SLAPI_TARGET_SDN, &sdn); ++ slapi_pblock_get(pb, SLAPI_ENTRY_PRE_OP, &pre_e); ++ slapi_pblock_get(pb, SLAPI_ENTRY_POST_OP, &post_e); ++ if (pre_e && post_e) { ++ pre_sdn = slapi_entry_get_sdn(pre_e); ++ post_sdn = slapi_entry_get_sdn(post_e); ++ } ++ slapi_log_err(SLAPI_LOG_PLUGIN, MEMBEROF_PLUGIN_SUBSYSTEM, ++ "deferred_mod_func: target %s\n", slapi_sdn_get_dn(sdn)); ++ ++ if (pre_sdn && post_sdn && slapi_sdn_compare(pre_sdn, post_sdn) == 0) { ++ /* Regarding memberof plugin, this rename is a no-op ++ * but it can be expensive to process it. So skip it ++ */ ++ slapi_log_err(SLAPI_LOG_PLUGIN, MEMBEROF_PLUGIN_SUBSYSTEM, ++ "deferred_modrdn_func: Skip modrdn operation because src/dst identical %s\n", ++ slapi_sdn_get_dn(post_sdn)); ++ goto skip_op; ++ } ++ ++ /* copy config so it doesn't change out from under us */ ++ memberof_rlock_config(); ++ mainConfig = memberof_get_config(); ++ memberof_copy_config(&configCopy, mainConfig); ++ memberof_unlock_config(); ++ ++ /* Need to check both the pre/post entries */ ++ if ((pre_sdn && !memberof_entry_in_scope(&configCopy, pre_sdn)) && ++ (post_sdn && !memberof_entry_in_scope(&configCopy, post_sdn))) ++ { ++ /* The entry is not in scope */ ++ goto bail; ++ } ++ ++ /* update any downstream members */ ++ if (pre_sdn && post_sdn && configCopy.group_filter && ++ 0 == slapi_filter_test_simple(post_e, configCopy.group_filter)) ++ { ++ Slapi_Attr *attr = 0; ++ ++ /* get a list of member attributes present in the group ++ * entry that is being renamed. */ ++ for (size_t i = 0; configCopy.groupattrs && configCopy.groupattrs[i]; i++) { ++ if (0 == slapi_entry_attr_find(post_e, configCopy.groupattrs[i], &attr)) { ++ if ((ret = memberof_moddn_attr_list(pb, &configCopy, pre_sdn, post_sdn, attr)) != 0) { ++ slapi_log_err(SLAPI_LOG_ERR, MEMBEROF_PLUGIN_SUBSYSTEM, ++ "deferred_modrdn_func - Update failed for (%s), error (%d)\n", ++ slapi_sdn_get_dn(pre_sdn), ret); ++ break; ++ } ++ } ++ } ++ } ++ ++ /* It's possible that this is an entry who is a member ++ * of other group entries. We need to update any member ++ * attributes to refer to the new name. */ ++ if (ret == LDAP_SUCCESS && pre_sdn && post_sdn) { ++ if (!memberof_entry_in_scope(&configCopy, post_sdn)) { ++ /* ++ * After modrdn the group contains both the pre and post DN's as ++ * members, so we need to cleanup both in this case. ++ */ ++ if ((ret = memberof_del_dn_from_groups(pb, &configCopy, pre_sdn))) { ++ slapi_log_err(SLAPI_LOG_ERR, MEMBEROF_PLUGIN_SUBSYSTEM, ++ "deferred_modrdn_func - Delete dn failed for preop entry(%s), error (%d)\n", ++ slapi_sdn_get_dn(pre_sdn), ret); ++ } ++ if ((ret = memberof_del_dn_from_groups(pb, &configCopy, post_sdn))) { ++ slapi_log_err(SLAPI_LOG_ERR, MEMBEROF_PLUGIN_SUBSYSTEM, ++ "deferred_modrdn_func - Delete dn failed for postop entry(%s), error (%d)\n", ++ slapi_sdn_get_dn(post_sdn), ret); ++ } ++ ++ if (ret == LDAP_SUCCESS && pre_e && configCopy.group_filter && ++ 0 == slapi_filter_test_simple(pre_e, configCopy.group_filter)) ++ { ++ /* is the entry of interest as a group? */ ++ Slapi_Attr *attr = 0; ++ ++ /* Loop through to find each grouping attribute separately. */ ++ for (size_t i = 0; configCopy.groupattrs && configCopy.groupattrs[i] && ret == LDAP_SUCCESS; i++) { ++ if (0 == slapi_entry_attr_find(pre_e, configCopy.groupattrs[i], &attr)) { ++ if ((ret = memberof_del_attr_list(pb, &configCopy, pre_sdn, attr))) { ++ slapi_log_err(SLAPI_LOG_ERR, MEMBEROF_PLUGIN_SUBSYSTEM, ++ "deferred_modrdn_func - Error deleting attr list - dn (%s). Error (%d)\n", ++ slapi_sdn_get_dn(pre_sdn), ret); ++ } ++ } ++ } ++ } ++ if (ret == LDAP_SUCCESS) { ++ memberof_del_dn_data del_data = {0, configCopy.memberof_attr}; ++ if ((ret = memberof_del_dn_type_callback(post_e, &del_data))) { ++ slapi_log_err(SLAPI_LOG_ERR, MEMBEROF_PLUGIN_SUBSYSTEM, ++ "deferred_modrdn_func - Delete dn callback failed for (%s), error (%d)\n", ++ slapi_entry_get_dn(post_e), ret); ++ } ++ } ++ } else { ++ if ((ret = memberof_replace_dn_from_groups(pb, &configCopy, pre_sdn, post_sdn))) { ++ slapi_log_err(SLAPI_LOG_ERR, MEMBEROF_PLUGIN_SUBSYSTEM, ++ "deferred_modrdn_func - Replace dn failed for (%s), error (%d)\n", ++ slapi_sdn_get_dn(pre_sdn), ret); ++ } ++ } ++ } ++bail: ++ memberof_free_config(&configCopy); ++ ++skip_op: ++ if (ret) { ++ slapi_log_err(SLAPI_LOG_ALERT, MEMBEROF_PLUGIN_SUBSYSTEM, ++ "Failed applying deferred updates: memberof values are invalid, please run fixup task\n"); ++ slapi_pblock_set(pb, SLAPI_RESULT_CODE, &ret); ++ ret = SLAPI_PLUGIN_FAILURE; ++ } ++ slapi_entry_free(pre_e); ++ slapi_entry_free(post_e); ++ slapi_sdn_free(&sdn); ++ slapi_pblock_destroy(pb); ++ ++ slapi_log_err(SLAPI_LOG_PLUGIN, MEMBEROF_PLUGIN_SUBSYSTEM, ++ "<-- deferred_modrdn_func\n"); ++ return ret; ++} ++ ++int ++deferred_del_func(MemberofDeferredDelTask *task) ++{ ++ Slapi_PBlock *pb; ++ struct slapi_entry *e = NULL; ++ Slapi_DN *sdn = 0; ++ MemberOfConfig configCopy = {0}; ++ PRBool free_configCopy = PR_FALSE; ++ MemberOfConfig *mainConfig; ++ int ret = SLAPI_PLUGIN_SUCCESS; ++ ++ pb = task->pb; ++ slapi_pblock_get(pb, SLAPI_ENTRY_PRE_OP, &e); ++ slapi_pblock_get(pb, SLAPI_TARGET_SDN, &sdn); ++ slapi_log_err(SLAPI_LOG_PLUGIN, MEMBEROF_PLUGIN_SUBSYSTEM, ++ "deferred_mod_func: target %s\n", slapi_sdn_get_dn(sdn)); ++ ++ memberof_rlock_config(); ++ mainConfig = memberof_get_config(); ++ if (!memberof_entry_in_scope(mainConfig, slapi_entry_get_sdn(e))) { ++ /* The entry is not in scope, bail...*/ ++ memberof_unlock_config(); ++ goto bail; ++ } ++ memberof_copy_config(&configCopy, memberof_get_config()); ++ free_configCopy = PR_TRUE; ++ memberof_unlock_config(); ++ ++ /* remove this DN from the ++ * membership lists of groups ++ */ ++ if ((ret = memberof_del_dn_from_groups(pb, &configCopy, sdn))) { ++ slapi_log_err(SLAPI_LOG_ERR, MEMBEROF_PLUGIN_SUBSYSTEM, ++ "deferred_del_func - Error deleting dn (%s) from group. Error (%d)\n", ++ slapi_sdn_get_dn(sdn), ret); ++ goto bail; ++ } ++ ++ /* is the entry of interest as a group? */ ++ if (e && configCopy.group_filter && 0 == slapi_filter_test_simple(e, configCopy.group_filter)) { ++ Slapi_Attr *attr = 0; ++ ++ /* Loop through to find each grouping attribute separately. */ ++ for (size_t i = 0; configCopy.groupattrs && configCopy.groupattrs[i] && ret == LDAP_SUCCESS; i++) { ++ if (0 == slapi_entry_attr_find(e, configCopy.groupattrs[i], &attr)) { ++ if ((ret = memberof_del_attr_list(pb, &configCopy, sdn, attr))) { ++ slapi_log_err(SLAPI_LOG_ERR, MEMBEROF_PLUGIN_SUBSYSTEM, ++ "deferred_del_func - Error deleting attr list - dn (%s). Error (%d)\n", ++ slapi_sdn_get_dn(sdn), ret); ++ } ++ } ++ } ++ } ++bail: ++ if (free_configCopy) { ++ memberof_free_config(&configCopy); ++ } ++ slapi_entry_free(e); ++ slapi_sdn_free(&sdn); ++ slapi_pblock_destroy(pb); ++ ++ if (ret) { ++ slapi_log_err(SLAPI_LOG_ALERT, MEMBEROF_PLUGIN_SUBSYSTEM, ++ "Failed applying deferred updates: memberof values are invalid, please run fixup task\n"); ++ slapi_pblock_set(pb, SLAPI_RESULT_CODE, &ret); ++ ret = SLAPI_PLUGIN_FAILURE; ++ } ++ ++ slapi_log_err(SLAPI_LOG_PLUGIN, MEMBEROF_PLUGIN_SUBSYSTEM, ++ "<-- deferred_del_func\n"); ++ ++ return ret; ++} ++int ++deferred_add_func(MemberofDeferredAddTask *task) ++{ ++ Slapi_PBlock *pb; ++ struct slapi_entry *e = NULL; ++ Slapi_DN *sdn = 0; ++ MemberOfConfig configCopy = {0}; ++ MemberOfConfig *mainConfig; ++ int interested = 0; ++ int ret = SLAPI_PLUGIN_SUCCESS; ++ ++ pb = task->pb; ++ ++ slapi_pblock_get(pb, SLAPI_TARGET_SDN, &sdn); ++ slapi_log_err(SLAPI_LOG_PLUGIN, MEMBEROF_PLUGIN_SUBSYSTEM, ++ "deferred_mod_func: target %s\n", slapi_sdn_get_dn(sdn)); ++ slapi_pblock_get(pb, SLAPI_ENTRY_POST_OP, &e); ++ ++ /* is the entry of interest? */ ++ memberof_rlock_config(); ++ mainConfig = memberof_get_config(); ++ if (e && mainConfig && mainConfig->group_filter && ++ 0 == slapi_filter_test_simple(e, mainConfig->group_filter)) ++ { ++ interested = 1; ++ if (!memberof_entry_in_scope(mainConfig, slapi_entry_get_sdn(e))) { ++ /* Entry is not in scope */ ++ memberof_unlock_config(); ++ goto bail; ++ } ++ memberof_copy_config(&configCopy, memberof_get_config()); ++ } ++ memberof_unlock_config(); ++ ++ if (interested) { ++ Slapi_Attr *attr = 0; ++ ++ for (size_t i = 0; configCopy.groupattrs && configCopy.groupattrs[i]; i++) { ++ if (0 == slapi_entry_attr_find(e, configCopy.groupattrs[i], &attr)) { ++ if ((ret = memberof_add_attr_list(pb, &configCopy, sdn, attr))) { ++ slapi_log_err(SLAPI_LOG_ERR, MEMBEROF_PLUGIN_SUBSYSTEM, ++ "deferred_add_func - Failed to add dn(%s), error (%d)\n", ++ slapi_sdn_get_dn(sdn), ret); ++ break; ++ } ++ } ++ } ++ memberof_free_config(&configCopy); ++ } ++ ++bail: ++ if (ret) { ++ slapi_log_err(SLAPI_LOG_ALERT, MEMBEROF_PLUGIN_SUBSYSTEM, ++ "Failed applying deferred updates: memberof values are invalid, please run fixup task\n"); ++ slapi_pblock_set(pb, SLAPI_RESULT_CODE, &ret); ++ ret = SLAPI_PLUGIN_FAILURE; ++ } ++ ++ slapi_entry_free(e); ++ slapi_sdn_free(&sdn); ++ slapi_pblock_destroy(pb); ++ ++ slapi_log_err(SLAPI_LOG_PLUGIN, MEMBEROF_PLUGIN_SUBSYSTEM, ++ "<-- deferred_add_func\n"); ++ ++ return ret; ++} ++int ++deferred_mod_func(MemberofDeferredModTask *task) ++{ ++ Slapi_PBlock *pb; ++ Slapi_Mod *next_mod = 0; ++ Slapi_Mods *smods = 0; ++ Slapi_Mod *smod = 0; ++ LDAPMod **mods; ++ Slapi_DN *sdn; ++ Slapi_Entry *pre_e = NULL; ++ Slapi_Entry *post_e = NULL; ++ int ret = SLAPI_PLUGIN_SUCCESS; ++ int config_copied = 0; ++ MemberOfConfig *mainConfig = 0; ++ MemberOfConfig configCopy = {0}; ++ ++ pb = task->pb; ++ slapi_pblock_get(pb, SLAPI_TARGET_SDN, &sdn); ++ slapi_log_err(SLAPI_LOG_PLUGIN, MEMBEROF_PLUGIN_SUBSYSTEM, ++ "deferred_mod_func: target %s\n", slapi_sdn_get_dn(sdn)); ++ /* get the mod set */ ++ slapi_pblock_get(pb, SLAPI_MODIFY_MODS, &mods); ++ smods = slapi_mods_new(); ++ slapi_mods_init_byref(smods, mods); ++ ++ next_mod = slapi_mod_new(); ++ smod = slapi_mods_get_first_smod(smods, next_mod); ++ while (smod) { ++ int interested = 0; ++ char *type = (char *)slapi_mod_get_type(smod); ++ /* We only want to copy the config if we encounter an ++ * operation that we need to act on. We also want to ++ * only copy the config the first time it's needed so ++ * it remains the same for all mods in the operation, ++ * despite any config changes that may be made. */ ++ if (!config_copied) { ++ memberof_rlock_config(); ++ mainConfig = memberof_get_config(); ++ if (memberof_is_grouping_attr(type, mainConfig)) { ++ interested = 1; ++ if (!memberof_entry_in_scope(mainConfig, sdn)) { ++ /* Entry is not in scope */ ++ memberof_unlock_config(); ++ goto bail; ++ } ++ /* copy config so it doesn't change out from under us */ ++ memberof_copy_config(&configCopy, mainConfig); ++ config_copied = 1; ++ } ++ memberof_unlock_config(); ++ } else if (memberof_is_grouping_attr(type, &configCopy)) { ++ interested = 1; ++ } ++ ++ if (interested) { ++ int op = slapi_mod_get_operation(smod); ++ ++ /* the modify op decides the function */ ++ switch (op & ~LDAP_MOD_BVALUES) { ++ case LDAP_MOD_ADD: { ++ /* add group DN to targets */ ++ if ((ret = memberof_add_smod_list(pb, &configCopy, sdn, smod))) { ++ slapi_log_err(SLAPI_LOG_ERR, MEMBEROF_PLUGIN_SUBSYSTEM, ++ "memberof_postop_modify - Failed to add dn (%s) to target. " ++ "Error (%d)\n", ++ slapi_sdn_get_dn(sdn), ret); ++ slapi_mod_done(next_mod); ++ goto bail; ++ } ++ break; ++ } ++ ++ case LDAP_MOD_DELETE: { ++ /* If there are no values in the smod, we should ++ * just do a replace instead. The user is just ++ * trying to delete all members from this group ++ * entry, which the replace code deals with. */ ++ if (slapi_mod_get_num_values(smod) == 0) { ++ if ((ret = memberof_replace_list(pb, &configCopy, sdn))) { ++ slapi_log_err(SLAPI_LOG_ERR, MEMBEROF_PLUGIN_SUBSYSTEM, ++ "memberof_postop_modify - Failed to replace list (%s). " ++ "Error (%d)\n", ++ slapi_sdn_get_dn(sdn), ret); ++ slapi_mod_done(next_mod); ++ goto bail; ++ } ++ } else { ++ /* remove group DN from target values in smod*/ ++ if ((ret = memberof_del_smod_list(pb, &configCopy, sdn, smod))) { ++ slapi_log_err(SLAPI_LOG_ERR, MEMBEROF_PLUGIN_SUBSYSTEM, ++ "memberof_postop_modify: failed to remove dn (%s). " ++ "Error (%d)\n", ++ slapi_sdn_get_dn(sdn), ret); ++ slapi_mod_done(next_mod); ++ goto bail; ++ } ++ } ++ break; ++ } ++ ++ case LDAP_MOD_REPLACE: { ++ /* replace current values */ ++ if ((ret = memberof_replace_list(pb, &configCopy, sdn))) { ++ slapi_log_err(SLAPI_LOG_ERR, MEMBEROF_PLUGIN_SUBSYSTEM, ++ "memberof_postop_modify - Failed to replace values in dn (%s). " ++ "Error (%d)\n", ++ slapi_sdn_get_dn(sdn), ret); ++ slapi_mod_done(next_mod); ++ goto bail; ++ } ++ break; ++ } ++ ++ default: { ++ slapi_log_err( ++ SLAPI_LOG_ERR, ++ MEMBEROF_PLUGIN_SUBSYSTEM, ++ "memberof_postop_modify - Unknown mod type\n"); ++ ret = SLAPI_PLUGIN_FAILURE; ++ break; ++ } ++ } ++ } ++ ++ slapi_mod_done(next_mod); ++ smod = slapi_mods_get_next_smod(smods, next_mod); ++ } ++ ++bail: ++ if (config_copied) { ++ memberof_free_config(&configCopy); ++ } ++ ++ slapi_mod_free(&next_mod); ++ slapi_mods_free(&smods); ++ slapi_pblock_get(pb, SLAPI_ENTRY_PRE_OP, &pre_e); ++ slapi_pblock_get(pb, SLAPI_ENTRY_POST_OP, &post_e); ++ slapi_entry_free(pre_e); ++ slapi_entry_free(post_e); ++ slapi_sdn_free(&sdn); ++ ldap_mods_free(task->mods, 1); ++ slapi_pblock_destroy(pb); ++ ++ if (ret) { ++ slapi_log_err(SLAPI_LOG_PLUGIN, MEMBEROF_PLUGIN_SUBSYSTEM, ++ "deferred_mod_func - fail to update new members of %s. Run fixup-task\n", ++ slapi_sdn_get_dn(task->target_sdn)); ++ slapi_log_err(SLAPI_LOG_ALERT, MEMBEROF_PLUGIN_SUBSYSTEM, ++ "Failed applying deferred updates: memberof values are invalid, please run fixup task\n"); ++ ret = SLAPI_PLUGIN_FAILURE; ++ } ++ ++ return ret; ++ ++} ++ ++/* Perform fixup (similar as fixup task) on all backends */ ++int ++perform_needed_fixup() ++{ ++ task_data td = {0}; ++ MemberOfConfig config = {0}; ++ Slapi_Backend *be = NULL; ++ char *cookie = NULL; ++ char **ocs = NULL; ++ size_t filter_size = 0; ++ char *filter = NULL; ++ int rc = 0; ++ ++ /* copy config so it doesn't change out from under us */ ++ memberof_rlock_config(); ++ memberof_copy_config(&config, memberof_get_config()); ++ memberof_unlock_config(); ++ slapi_log_err(SLAPI_LOG_INFO, MEMBEROF_PLUGIN_SUBSYSTEM, ++ "Memberof plugin started the global fixup task for attribute %s\n", config.memberof_attr); ++ /* Compute the filter for entries that may contains the attribute */ ++ ocs = schema_get_objectclasses_by_attribute(config.memberof_attr); ++ if (ocs == NULL) { ++ slapi_log_err(SLAPI_LOG_ALERT, MEMBEROF_PLUGIN_SUBSYSTEM, ++ "Failed to perform memberof fixup task because no objectclass contains the %s attribute.\n", ++ config.memberof_attr); ++ return -1; ++ } ++ filter_size = 4; /* For "(|...)\0" */ ++ for (size_t i=0; ocs[i]; i++) { ++ filter_size += 14 + strlen(ocs[i]); /* For "(objectclass=...)" */ ++ } ++ td.filter_str = filter = slapi_ch_malloc(filter_size); ++ strcpy(filter, "(|"); ++ for (size_t i=0; ocs[i]; i++) { ++ sprintf(filter+strlen(filter), "(objectclass=%s)", ocs[i]); ++ } ++ strcat(filter, ")"); ++ slapi_ch_array_free(ocs); ++ ocs = NULL; ++ td.bind_dn = slapi_ch_strdup(slapi_sdn_get_dn(memberof_get_config_area())); ++ /* Then perform fixup on all backends */ ++ be = slapi_get_first_backend(&cookie); ++ while (be) { ++ td.dn = (char*) slapi_sdn_get_dn(slapi_be_getsuffix(be, 0)); ++ if (td.dn) { ++ int rc1 = memberof_fix_memberof(&config, NULL, &td); ++ if (rc1) { ++ slapi_log_err(SLAPI_LOG_ERR, MEMBEROF_PLUGIN_SUBSYSTEM, ++ "memberof plugin failed to perform fixup on dn %s with filter %s - error: %d\n", ++ td.dn, td.filter_str, rc1); ++ rc = -1; ++ } ++ } ++ be = slapi_get_next_backend(cookie); ++ } ++ slapi_ch_free_string(&td.bind_dn); ++ slapi_ch_free_string(&td.filter_str); ++ slapi_log_err(SLAPI_LOG_INFO, MEMBEROF_PLUGIN_SUBSYSTEM, ++ "Memberof plugin finished the global fixup task for attribute %s\n", config.memberof_attr); ++ return rc; + } + +-static int +-memberof_preop_init(Slapi_PBlock *pb) ++/* Change memberOfNeedFixup attribute in config entry */ ++void ++modify_need_fixup(int set) + { +- int status = 0; ++ int rc = 0; ++ LDAPMod mod; ++ LDAPMod *mods[2] = { &mod, NULL }; ++ char *val[2] = { "true", NULL }; ++ Slapi_PBlock *mod_pb = 0; + +- if (slapi_pblock_set(pb, SLAPI_PLUGIN_VERSION, SLAPI_PLUGIN_VERSION_01) != 0 || +- slapi_pblock_set(pb, SLAPI_PLUGIN_DESCRIPTION, (void *)&pdesc) != 0 || +- slapi_pblock_set(pb, premodfn, (void *)memberof_shared_config_validate) != 0) { ++ if (set) { ++ slapi_log_err(SLAPI_LOG_PLUGIN, MEMBEROF_PLUGIN_SUBSYSTEM, ++ "modify_need_fixup - set memberOfNeedFixup in config entry.\n"); ++ } else { ++ slapi_log_err(SLAPI_LOG_PLUGIN, MEMBEROF_PLUGIN_SUBSYSTEM, ++ "modify_need_fixup - reset memberOfNeedFixup in config entry.\n"); ++ } ++ mod_pb = slapi_pblock_new(); ++ mod.mod_op = LDAP_MOD_REPLACE; ++ mod.mod_type = MEMBEROF_NEED_FIXUP; ++ mod.mod_values = set ? val : NULL; ++ slapi_modify_internal_set_pb_ext( ++ mod_pb, memberof_get_config_area(), ++ mods, 0, 0, ++ memberof_get_plugin_id(), SLAPI_OP_FLAG_BYPASS_REFERRALS); ++ ++ slapi_modify_internal_pb(mod_pb); ++ slapi_pblock_get(mod_pb, SLAPI_PLUGIN_INTOP_RESULT, &rc); ++ slapi_pblock_destroy(mod_pb); ++ if (rc) { + slapi_log_err(SLAPI_LOG_ERR, MEMBEROF_PLUGIN_SUBSYSTEM, +- "memberof_preop_init: Failed to register plugin\n"); +- status = -1; ++ "modify_need_fixup - failed to modify config entry. rc=%d\n", rc); ++ } else { ++ memberof_get_config()->need_fixup = set; + } ++} + +- return status; ++int ++is_memberof_plugin_started(struct slapdplugin **plg_addr) ++{ ++ volatile struct slapdplugin *plg = *plg_addr; ++ const char *plg_dn = slapi_sdn_get_ndn(memberof_get_config_area()); ++ if (!plg) { ++ /* Find our slapdplugin struct */ ++ for (int type = 0; type < PLUGIN_LIST_GLOBAL_MAX; type++) { ++ struct slapdplugin *plg_list = get_plugin_list(type); ++ for (struct slapdplugin *p = plg_list; plg == NULL && p != NULL; p = p->plg_next) { ++ if (strcmp(plg_dn, p->plg_dn) == 0) { ++ plg = *plg_addr = p; ++ } ++ } ++ } ++ } ++ if (!plg) { ++ slapi_log_err(SLAPI_LOG_PLUGIN, MEMBEROF_PLUGIN_SUBSYSTEM, ++ "Unable to find the struct slapdplugin entry for %s.\n", plg_dn); ++ return 0; ++ } ++ return plg->plg_started; + } + +-static int +-memberof_internal_postop_init(Slapi_PBlock *pb) ++void ++deferred_thread_func(void *arg) + { +- int status = 0; ++ MemberofDeferredList *deferred_list = (MemberofDeferredList *) arg; ++ MemberofDeferredTask *task; ++ struct slapdplugin *plg = NULL; ++ const char *dn = slapi_sdn_get_dn(memberof_get_config_area()); + +- if (slapi_pblock_set(pb, SLAPI_PLUGIN_VERSION, +- SLAPI_PLUGIN_VERSION_01) != 0 || +- slapi_pblock_set(pb, SLAPI_PLUGIN_DESCRIPTION, +- (void *)&pdesc) != 0 || +- slapi_pblock_set(pb, SLAPI_PLUGIN_INTERNAL_POST_DELETE_FN, +- (void *)memberof_postop_del) != 0 || +- slapi_pblock_set(pb, SLAPI_PLUGIN_INTERNAL_POST_MODRDN_FN, +- (void *)memberof_postop_modrdn) != 0 || +- slapi_pblock_set(pb, SLAPI_PLUGIN_INTERNAL_POST_MODIFY_FN, +- (void *)memberof_postop_modify) != 0 || +- slapi_pblock_set(pb, SLAPI_PLUGIN_INTERNAL_POST_ADD_FN, +- (void *)memberof_postop_add) != 0) { +- slapi_log_err(SLAPI_LOG_ERR, MEMBEROF_PLUGIN_SUBSYSTEM, +- "memberof_internal_postop_init - Failed to register plugin\n"); +- status = -1; ++ /* ++ * Wait until plugin is fully started. (Otherwise modify_need_fixup silently fails ++ * to update the dse.ldif file ++ */ ++ while (!is_memberof_plugin_started(&plg)) { ++ usleep(200); ++ slapi_log_err(SLAPI_LOG_PLUGIN, MEMBEROF_PLUGIN_SUBSYSTEM, ++ "deferred_thread_func wait for startup\n"); + } + +- return status; ++ /* ++ * keep running this thread until plugin is signaled to close ++ */ ++ g_incr_active_threadcnt(); ++ if (memberof_get_config()->need_fixup && perform_needed_fixup()) { ++ slapi_log_err(SLAPI_LOG_ALERT, MEMBEROF_PLUGIN_SUBSYSTEM, ++ "Failure occured during global fixup task: memberof values are invalid\n"); ++ } ++ slapi_log_err(SLAPI_LOG_PLUGIN, MEMBEROF_PLUGIN_SUBSYSTEM, ++ "deferred_thread_func - thread is starting " ++ "processing deferred updates for plugin %s\n", dn); ++ ++ /* Tells that global fixup should be done (in case of crash/kill -9) */ ++ modify_need_fixup(1); ++ while (1) { ++ pthread_mutex_lock(&deferred_list->deferred_list_mutex); ++ if (deferred_list->current_task) { ++ /* it exists a task, pick it up */ ++ task = remove_deferred_task(deferred_list); ++ } else { ++ struct timespec current_time = {0}; ++ if (g_get_shutdown()) { ++ /* In shutdown case, lets go on to loop until the queue is empty */ ++ slapi_log_err(SLAPI_LOG_INFO, MEMBEROF_PLUGIN_SUBSYSTEM, ++ "deferred_thread_func - ending with added %d / removed %d\n", ++ deferred_list->total_added, deferred_list->total_removed); ++ pthread_mutex_unlock(&deferred_list->deferred_list_mutex); ++ break; ++ } ++ /* let wait for a next notification */ ++ task = NULL; ++ clock_gettime(CLOCK_MONOTONIC, ¤t_time); ++ current_time.tv_sec += 1; ++ pthread_cond_timedwait(&deferred_list->deferred_list_cv, &deferred_list->deferred_list_mutex, ¤t_time); ++ } ++ pthread_mutex_unlock(&deferred_list->deferred_list_mutex); ++ ++ if (task) { ++ int deferred_op_running = 0; ++ switch(task->deferred_choice) { ++ case SLAPI_OPERATION_MODIFY: ++ deferred_mod_func(task->d_mod); ++ slapi_pblock_set(task->d_mod->pb_original, SLAPI_DEFERRED_MEMBEROF, &deferred_op_running); ++ slapi_ch_free((void **)&task->d_mod); ++ break; ++ case SLAPI_OPERATION_ADD: ++ deferred_add_func(task->d_add); ++ slapi_pblock_set(task->d_add->pb_original, SLAPI_DEFERRED_MEMBEROF, &deferred_op_running); ++ slapi_ch_free((void **)&task->d_add); ++ break; ++ case SLAPI_OPERATION_DELETE: ++ deferred_del_func(task->d_del); ++ slapi_pblock_set(task->d_del->pb_original, SLAPI_DEFERRED_MEMBEROF, &deferred_op_running); ++ slapi_ch_free((void **)&task->d_del); ++ break; ++ case SLAPI_OPERATION_MODRDN: ++ deferred_modrdn_func(task->d_modrdn); ++ slapi_pblock_set(task->d_modrdn->pb_original, SLAPI_DEFERRED_MEMBEROF, &deferred_op_running); ++ slapi_ch_free((void **)&task->d_modrdn); ++ break; ++ default: ++ slapi_log_err(SLAPI_LOG_ERR, MEMBEROF_PLUGIN_SUBSYSTEM, ++ "unsupported deferred operation %ld\n", task->deferred_choice); ++ } ++ slapi_ch_free((void **)&task); ++ } ++ } /* main loop */ ++ modify_need_fixup(0); ++ slapi_log_err(SLAPI_LOG_PLUGIN, MEMBEROF_PLUGIN_SUBSYSTEM, ++ "deferred_thread_func - thread has stopped " ++ "processing deferred updates for plugin %s\n", dn); ++ g_decr_active_threadcnt(); + } + + /* +@@ -328,6 +1094,7 @@ memberof_postop_start(Slapi_PBlock *pb) + Slapi_PBlock *search_pb = NULL; + Slapi_Entry **entries = NULL; + Slapi_Entry *config_e = NULL; /* entry containing plugin config */ ++ MemberOfConfig *mainConfig = NULL; + char *config_area = NULL; + int result = 0; + int rc = 0; +@@ -399,6 +1166,52 @@ memberof_postop_start(Slapi_PBlock *pb) + rc = -1; + goto bail; + } ++ memberof_rlock_config(); ++ mainConfig = memberof_get_config(); ++ /* if the update of the members is deferred then allocate mutex/cv */ ++ if (mainConfig->deferred_update) { ++ MemberofDeferredList *deferred_list; ++ pthread_condattr_t condAttr; ++ ++ deferred_list = (MemberofDeferredList *) slapi_ch_calloc(1, sizeof(struct memberof_deferred_list)); ++ ++ /* initialize the cv and lock */ ++ if ((rc = pthread_mutex_init(&deferred_list->deferred_list_mutex, NULL)) != 0) { ++ slapi_log_err(SLAPI_LOG_ERR, "memberof_postop_start", ++ "cannot create new lock. error %d (%s)\n", ++ rc, strerror(rc)); ++ exit(1); ++ } ++ if ((rc = pthread_condattr_init(&condAttr)) != 0) { ++ slapi_log_err(SLAPI_LOG_ERR, "memberof_postop_start", ++ "cannot create new condition attribute variable. error %d (%s)\n", ++ rc, strerror(rc)); ++ exit(1); ++ } ++ if ((rc = pthread_condattr_setclock(&condAttr, CLOCK_MONOTONIC)) != 0) { ++ slapi_log_err(SLAPI_LOG_ERR, "memberof_postop_start", ++ "cannot set condition attr clock. error %d (%s)\n", ++ rc, strerror(rc)); ++ exit(1); ++ } ++ if ((rc = pthread_cond_init(&deferred_list->deferred_list_cv, &condAttr)) != 0) { ++ slapi_log_err(SLAPI_LOG_ERR, "memberof_postop_start", ++ "cannot create new condition variable. error %d (%s)\n", ++ rc, strerror(rc)); ++ exit(1); ++ } ++ pthread_condattr_destroy(&condAttr); /* no longer needed */ ++ ++ deferred_list->deferred_tid = PR_CreateThread(PR_USER_THREAD, ++ deferred_thread_func, ++ deferred_list, ++ PR_PRIORITY_NORMAL, ++ PR_GLOBAL_THREAD, ++ PR_UNJOINABLE_THREAD, ++ SLAPD_DEFAULT_THREAD_STACKSIZE); ++ mainConfig->deferred_list = deferred_list; ++ } ++ memberof_unlock_config(); + + rc = slapi_plugin_task_register_handler("memberof task", memberof_task_add, pb); + if (rc) { +@@ -533,7 +1346,43 @@ memberof_postop_del(Slapi_PBlock *pb) + + if (memberof_oktodo(pb) && (sdn = memberof_getsdn(pb))) { + struct slapi_entry *e = NULL; ++ Slapi_DN *copied_sdn; ++ PRBool deferred_update; ++ ++ /* retrieve deferred update params that are valid until shutdown */ ++ memberof_rlock_config(); ++ mainConfig = memberof_get_config(); ++ deferred_update = mainConfig->deferred_update; ++ memberof_unlock_config(); + ++ if (deferred_update) { ++ MemberofDeferredTask* task; ++ Slapi_Operation *op; ++ int deferred_op_running = 1; ++ ++ /* Should be freed with slapi_sdn_free(copied_sdn) */ ++ copied_sdn = slapi_sdn_dup(sdn); ++ ++ task = (MemberofDeferredTask *)slapi_ch_calloc(1, sizeof(MemberofDeferredTask)); ++ task->d_del= (MemberofDeferredDelTask *)slapi_ch_calloc(1, sizeof(MemberofDeferredDelTask)); ++ slapi_pblock_set(pb, SLAPI_DEFERRED_MEMBEROF, &deferred_op_running); /* operation wait until the completion of the deferred update */ ++ task->d_del->pb_original = pb; ++ task->d_del->pb = slapi_pblock_new(); ++ op = internal_operation_new(SLAPI_OPERATION_DELETE, 0); ++ slapi_pblock_set(task->d_del->pb, SLAPI_OPERATION, op); ++ slapi_pblock_get(pb, SLAPI_ENTRY_PRE_OP, &e); ++ slapi_pblock_set(task->d_del->pb, SLAPI_ENTRY_PRE_OP, slapi_entry_dup(e)); ++ slapi_pblock_set(task->d_del->pb, SLAPI_TARGET_SDN, copied_sdn); ++ task->deferred_choice = SLAPI_OPERATION_DELETE; ++ /* store the task in the pblock that will be added to ++ * the deferred list during the backend postop (after txn_commit) ++ */ ++ slapi_pblock_set(pb, SLAPI_MEMBEROF_DEFERRED_TASK, (void *) task); ++ ret = SLAPI_PLUGIN_SUCCESS; ++ goto done; ++ } else { ++ slapi_pblock_set(pb, SLAPI_MEMBEROF_DEFERRED_TASK, NULL); ++ } + slapi_pblock_get(pb, SLAPI_ENTRY_PRE_OP, &e); + memberof_rlock_config(); + mainConfig = memberof_get_config(); +@@ -557,11 +1406,10 @@ memberof_postop_del(Slapi_PBlock *pb) + + /* is the entry of interest as a group? */ + if (e && configCopy.group_filter && 0 == slapi_filter_test_simple(e, configCopy.group_filter)) { +- int i = 0; + Slapi_Attr *attr = 0; + + /* Loop through to find each grouping attribute separately. */ +- for (i = 0; configCopy.groupattrs && configCopy.groupattrs[i] && ret == LDAP_SUCCESS; i++) { ++ for (size_t i = 0; configCopy.groupattrs && configCopy.groupattrs[i] && ret == LDAP_SUCCESS; i++) { + if (0 == slapi_entry_attr_find(e, configCopy.groupattrs[i], &attr)) { + if ((ret = memberof_del_attr_list(pb, &configCopy, sdn, attr))) { + slapi_log_err(SLAPI_LOG_ERR, MEMBEROF_PLUGIN_SUBSYSTEM, +@@ -571,10 +1419,11 @@ memberof_postop_del(Slapi_PBlock *pb) + } + } + } +- bail: ++bail: + memberof_free_config(&configCopy); + } + ++done: + if (ret) { + slapi_pblock_set(pb, SLAPI_RESULT_CODE, &ret); + ret = SLAPI_PLUGIN_FAILURE; +@@ -584,17 +1433,11 @@ memberof_postop_del(Slapi_PBlock *pb) + return ret; + } + +-typedef struct _memberof_del_dn_data +-{ +- char *dn; +- char *type; +-} memberof_del_dn_data; + + /* Deletes a member dn from all groups that refer to it. */ + static int + memberof_del_dn_from_groups(Slapi_PBlock *pb, MemberOfConfig *config, Slapi_DN *sdn) + { +- int i = 0; + char *groupattrs[2] = {0, 0}; + int rc = LDAP_SUCCESS; + int cached = 0; +@@ -602,7 +1445,7 @@ memberof_del_dn_from_groups(Slapi_PBlock *pb, MemberOfConfig *config, Slapi_DN * + /* Loop through each grouping attribute to find groups that have + * dn as a member. For any matches, delete the dn value from the + * same grouping attribute. */ +- for (i = 0; config->groupattrs && config->groupattrs[i] && rc == LDAP_SUCCESS; i++) { ++ for (size_t i = 0; config->groupattrs && config->groupattrs[i] && rc == LDAP_SUCCESS; i++) { + memberof_del_dn_data data = {(char *)slapi_sdn_get_dn(sdn), + config->groupattrs[i]}; + +@@ -725,7 +1568,6 @@ memberof_call_foreach_dn(Slapi_PBlock *pb __attribute__((unused)), Slapi_DN *sdn + int dn_len = slapi_sdn_get_ndn_len(sdn); + int free_it = 0; + int rc = 0; +- int i = 0; + + *cached = 0; + +@@ -769,7 +1611,7 @@ memberof_call_foreach_dn(Slapi_PBlock *pb __attribute__((unused)), Slapi_DN *sdn + escaped_filter_val = (char *)slapi_sdn_get_dn(sdn); + } + +- for (i = 0; types[i]; i++) { ++ for (size_t i = 0; types[i]; i++) { + /* Triggers one internal search per membership attribute. + * Assuming the attribute is indexed (eq), the search will + * bypass the evaluation of the filter (nsslapd-search-bypass-filter-test) +@@ -804,10 +1646,10 @@ memberof_call_foreach_dn(Slapi_PBlock *pb __attribute__((unused)), Slapi_DN *sdn + /* do nothing, entry scope is spanning + * multiple suffixes, start at suffix */ + } else if (config->entryScopes) { +- for (size_t i = 0; config->entryScopes[i]; i++) { +- if (slapi_sdn_issuffix(config->entryScopes[i], base_sdn)) { ++ for (size_t ii = 0; config->entryScopes[ii]; ii++) { ++ if (slapi_sdn_issuffix(config->entryScopes[ii], base_sdn)) { + /* Search each include scope */ +- slapi_search_internal_set_pb(search_pb, slapi_sdn_get_dn(config->entryScopes[i]), ++ slapi_search_internal_set_pb(search_pb, slapi_sdn_get_dn(config->entryScopes[ii]), + LDAP_SCOPE_SUBTREE, filter_str, 0, 0, 0, 0, + memberof_get_plugin_id(), 0); + slapi_search_internal_callback_pb(search_pb, callback_data, 0, callback, 0); +@@ -887,7 +1729,51 @@ memberof_postop_modrdn(Slapi_PBlock *pb) + struct slapi_entry *post_e = NULL; + Slapi_DN *pre_sdn = 0; + Slapi_DN *post_sdn = 0; ++ Slapi_DN *origin_sdn; ++ Slapi_DN *copied_sdn; ++ PRBool deferred_update; ++ ++ /* retrieve deferred update params that are valid until shutdown */ ++ memberof_rlock_config(); ++ mainConfig = memberof_get_config(); ++ deferred_update = mainConfig->deferred_update; ++ memberof_unlock_config(); + ++ if (deferred_update) { ++ MemberofDeferredTask* task; ++ Slapi_Operation *op; ++ int deferred_op_running = 1; ++ ++ task = (MemberofDeferredTask *)slapi_ch_calloc(1, sizeof(MemberofDeferredTask)); ++ task->d_modrdn= (MemberofDeferredModrdnTask *)slapi_ch_calloc(1, sizeof(MemberofDeferredModrdnTask)); ++ slapi_pblock_set(pb, SLAPI_DEFERRED_MEMBEROF, &deferred_op_running); /* operation wait until the completion of the deferred update */ ++ task->d_modrdn->pb_original = pb; ++ task->d_modrdn->pb = slapi_pblock_new(); ++ ++ op = internal_operation_new(SLAPI_OPERATION_MODRDN, 0); ++ slapi_pblock_set(task->d_modrdn->pb, SLAPI_OPERATION, op); ++ ++ slapi_pblock_get(pb, SLAPI_TARGET_SDN, &origin_sdn); ++ /* Should be freed with slapi_sdn_free(copied_sdn) */ ++ copied_sdn = slapi_sdn_dup(origin_sdn); ++ slapi_pblock_set(task->d_modrdn->pb, SLAPI_TARGET_SDN, copied_sdn); ++ ++ slapi_pblock_get(pb, SLAPI_ENTRY_PRE_OP, &pre_e); ++ slapi_pblock_set(task->d_modrdn->pb, SLAPI_ENTRY_PRE_OP, slapi_entry_dup(pre_e)); ++ ++ slapi_pblock_get(pb, SLAPI_ENTRY_POST_OP, &post_e); ++ slapi_pblock_set(task->d_modrdn->pb, SLAPI_ENTRY_POST_OP, slapi_entry_dup(post_e)); ++ ++ task->deferred_choice = SLAPI_OPERATION_MODRDN; ++ /* store the task in the pblock that will be added to ++ * the deferred list during the backend postop (after txn_commit) ++ */ ++ slapi_pblock_set(pb, SLAPI_MEMBEROF_DEFERRED_TASK, (void *) task); ++ ret = SLAPI_PLUGIN_SUCCESS; ++ goto skip_op; ++ } else { ++ slapi_pblock_set(pb, SLAPI_MEMBEROF_DEFERRED_TASK, NULL); ++ } + slapi_pblock_get(pb, SLAPI_ENTRY_PRE_OP, &pre_e); + slapi_pblock_get(pb, SLAPI_ENTRY_POST_OP, &post_e); + if (pre_e && post_e) { +@@ -921,12 +1807,11 @@ memberof_postop_modrdn(Slapi_PBlock *pb) + /* update any downstream members */ + if (pre_sdn && post_sdn && configCopy.group_filter && + 0 == slapi_filter_test_simple(post_e, configCopy.group_filter)) { +- int i = 0; + Slapi_Attr *attr = 0; + + /* get a list of member attributes present in the group + * entry that is being renamed. */ +- for (i = 0; configCopy.groupattrs && configCopy.groupattrs[i]; i++) { ++ for (size_t i = 0; configCopy.groupattrs && configCopy.groupattrs[i]; i++) { + if (0 == slapi_entry_attr_find(post_e, configCopy.groupattrs[i], &attr)) { + if ((ret = memberof_moddn_attr_list(pb, &configCopy, pre_sdn, post_sdn, attr)) != 0) { + slapi_log_err(SLAPI_LOG_ERR, MEMBEROF_PLUGIN_SUBSYSTEM, +@@ -961,11 +1846,10 @@ memberof_postop_modrdn(Slapi_PBlock *pb) + if (ret == LDAP_SUCCESS && pre_e && configCopy.group_filter && + 0 == slapi_filter_test_simple(pre_e, configCopy.group_filter)) { + /* is the entry of interest as a group? */ +- int i = 0; + Slapi_Attr *attr = 0; + + /* Loop through to find each grouping attribute separately. */ +- for (i = 0; configCopy.groupattrs && configCopy.groupattrs[i] && ret == LDAP_SUCCESS; i++) { ++ for (size_t i = 0; configCopy.groupattrs && configCopy.groupattrs[i] && ret == LDAP_SUCCESS; i++) { + if (0 == slapi_entry_attr_find(pre_e, configCopy.groupattrs[i], &attr)) { + if ((ret = memberof_del_attr_list(pb, &configCopy, pre_sdn, attr))) { + slapi_log_err(SLAPI_LOG_ERR, MEMBEROF_PLUGIN_SUBSYSTEM, +@@ -1019,7 +1903,6 @@ typedef struct _replace_dn_data + static int + memberof_replace_dn_from_groups(Slapi_PBlock *pb, MemberOfConfig *config, Slapi_DN *pre_sdn, Slapi_DN *post_sdn) + { +- int i = 0; + char *groupattrs[2] = {0, 0}; + int ret = LDAP_SUCCESS; + int cached = 0; +@@ -1027,7 +1910,7 @@ memberof_replace_dn_from_groups(Slapi_PBlock *pb, MemberOfConfig *config, Slapi_ + /* Loop through each grouping attribute to find groups that have + * pre_dn as a member. For any matches, replace pre_dn with post_dn + * using the same grouping attribute. */ +- for (i = 0; config->groupattrs && config->groupattrs[i]; i++) { ++ for (size_t i = 0; config->groupattrs && config->groupattrs[i]; i++) { + replace_dn_data data = {(char *)slapi_sdn_get_dn(pre_sdn), + (char *)slapi_sdn_get_dn(post_sdn), + config->groupattrs[i], +@@ -1083,7 +1966,25 @@ memberof_replace_dn_type_callback(Slapi_Entry *e, void *callback_data) + + return rc; + } +- ++LDAPMod ** ++my_copy_mods(LDAPMod **orig_mods) ++{ ++ LDAPMod **new_mods = NULL; ++ LDAPMod *mod; ++ Slapi_Mods smods_old; ++ Slapi_Mods smods_new; ++ slapi_mods_init_byref(&smods_old, orig_mods); ++ slapi_mods_init_passin(&smods_new, new_mods); ++ mod = slapi_mods_get_first_mod(&smods_old); ++ while (mod != NULL) { ++ slapi_mods_add_modbvps(&smods_new, mod->mod_op, mod->mod_type, mod->mod_bvalues); ++ mod = slapi_mods_get_next_mod(&smods_old); ++ } ++ new_mods = slapi_mods_get_ldapmods_passout(&smods_new); ++ slapi_mods_done(&smods_old); ++ slapi_mods_done(&smods_new); ++ return new_mods; ++} + /* + * memberof_postop_modify() + * +@@ -1148,6 +2049,55 @@ memberof_postop_modify(Slapi_PBlock *pb) + int config_copied = 0; + MemberOfConfig *mainConfig = 0; + MemberOfConfig configCopy = {0}; ++ PRBool deferred_update; ++ ++ /* retrieve deferred update params that are valid until shutdown */ ++ memberof_rlock_config(); ++ mainConfig = memberof_get_config(); ++ deferred_update = mainConfig->deferred_update; ++ memberof_unlock_config(); ++ ++ if (deferred_update) { ++ MemberofDeferredTask* task; ++ LDAPMod **copied_mods = NULL; ++ Slapi_DN *copied_sdn; ++ Slapi_Operation *op = NULL; ++ int deferred_op_running = 1; ++ struct slapi_entry *pre_e = NULL; ++ struct slapi_entry *post_e = NULL; ++ slapi_pblock_get(pb, SLAPI_MODIFY_MODS, &mods); ++ ++ /* Should be free with ldap_mods_free(copied_mods, 1);*/ ++ copied_mods = my_copy_mods(mods); ++ ++ /* Should be freed with slapi_sdn_free(copied_sdn) */ ++ copied_sdn = slapi_sdn_dup(sdn); ++ ++ task = (MemberofDeferredTask *)slapi_ch_calloc(1, sizeof(MemberofDeferredTask)); ++ task->d_mod = (MemberofDeferredModTask *)slapi_ch_calloc(1, sizeof(MemberofDeferredModTask)); ++ slapi_pblock_set(pb, SLAPI_DEFERRED_MEMBEROF, &deferred_op_running); /* operation wait until the completion of the deferred update */ ++ task->d_mod->pb_original = pb; ++ task->d_mod->pb = slapi_pblock_new(); ++ op = internal_operation_new(SLAPI_OPERATION_MODIFY, 0); ++ slapi_pblock_set(task->d_mod->pb, SLAPI_OPERATION, op); ++ slapi_pblock_set(task->d_mod->pb, SLAPI_MODIFY_MODS, copied_mods); ++ slapi_pblock_set(task->d_mod->pb, SLAPI_TARGET_SDN, copied_sdn); ++ slapi_pblock_get(pb, SLAPI_ENTRY_PRE_OP, &pre_e); ++ slapi_pblock_get(pb, SLAPI_ENTRY_POST_OP, &post_e); ++ slapi_pblock_set(task->d_mod->pb, SLAPI_ENTRY_PRE_OP, slapi_entry_dup(pre_e)); ++ slapi_pblock_set(task->d_mod->pb, SLAPI_ENTRY_POST_OP, slapi_entry_dup(post_e)); ++ task->d_mod->mods = copied_mods; // TODO - is this needed? ++ task->d_mod->target_sdn = copied_sdn; // TODO - is this needed? ++ task->deferred_choice = SLAPI_OPERATION_MODIFY; ++ /* store the task in the pblock that will be added to ++ * the deferred list during the backend postop (after txn_commit) ++ */ ++ slapi_pblock_set(pb, SLAPI_MEMBEROF_DEFERRED_TASK, (void *) task); ++ ret = SLAPI_PLUGIN_SUCCESS; ++ goto done; ++ } else { ++ slapi_pblock_set(pb, SLAPI_MEMBEROF_DEFERRED_TASK, NULL); ++ } + + /* get the mod set */ + slapi_pblock_get(pb, SLAPI_MODIFY_MODS, &mods); +@@ -1192,68 +2142,68 @@ memberof_postop_modify(Slapi_PBlock *pb) + + /* the modify op decides the function */ + switch (op & ~LDAP_MOD_BVALUES) { +- case LDAP_MOD_ADD: { +- /* add group DN to targets */ +- if ((ret = memberof_add_smod_list(pb, &configCopy, sdn, smod))) { +- slapi_log_err(SLAPI_LOG_ERR, MEMBEROF_PLUGIN_SUBSYSTEM, +- "memberof_postop_modify - Failed to add dn (%s) to target. " +- "Error (%d)\n", +- slapi_sdn_get_dn(sdn), ret); +- slapi_mod_done(next_mod); +- goto bail; +- } +- break; +- } +- +- case LDAP_MOD_DELETE: { +- /* If there are no values in the smod, we should +- * just do a replace instead. The user is just +- * trying to delete all members from this group +- * entry, which the replace code deals with. */ +- if (slapi_mod_get_num_values(smod) == 0) { +- if ((ret = memberof_replace_list(pb, &configCopy, sdn))) { ++ case LDAP_MOD_ADD: { ++ /* add group DN to targets */ ++ if ((ret = memberof_add_smod_list(pb, &configCopy, sdn, smod))) { + slapi_log_err(SLAPI_LOG_ERR, MEMBEROF_PLUGIN_SUBSYSTEM, +- "memberof_postop_modify - Failed to replace list (%s). " ++ "memberof_postop_modify - Failed to add dn (%s) to target. " + "Error (%d)\n", + slapi_sdn_get_dn(sdn), ret); + slapi_mod_done(next_mod); + goto bail; + } +- } else { +- /* remove group DN from target values in smod*/ +- if ((ret = memberof_del_smod_list(pb, &configCopy, sdn, smod))) { ++ break; ++ } ++ ++ case LDAP_MOD_DELETE: { ++ /* If there are no values in the smod, we should ++ * just do a replace instead. The user is just ++ * trying to delete all members from this group ++ * entry, which the replace code deals with. */ ++ if (slapi_mod_get_num_values(smod) == 0) { ++ if ((ret = memberof_replace_list(pb, &configCopy, sdn))) { ++ slapi_log_err(SLAPI_LOG_ERR, MEMBEROF_PLUGIN_SUBSYSTEM, ++ "memberof_postop_modify - Failed to replace list (%s). " ++ "Error (%d)\n", ++ slapi_sdn_get_dn(sdn), ret); ++ slapi_mod_done(next_mod); ++ goto bail; ++ } ++ } else { ++ /* remove group DN from target values in smod*/ ++ if ((ret = memberof_del_smod_list(pb, &configCopy, sdn, smod))) { ++ slapi_log_err(SLAPI_LOG_ERR, MEMBEROF_PLUGIN_SUBSYSTEM, ++ "memberof_postop_modify: failed to remove dn (%s). " ++ "Error (%d)\n", ++ slapi_sdn_get_dn(sdn), ret); ++ slapi_mod_done(next_mod); ++ goto bail; ++ } ++ } ++ break; ++ } ++ ++ case LDAP_MOD_REPLACE: { ++ /* replace current values */ ++ if ((ret = memberof_replace_list(pb, &configCopy, sdn))) { + slapi_log_err(SLAPI_LOG_ERR, MEMBEROF_PLUGIN_SUBSYSTEM, +- "memberof_postop_modify: failed to remove dn (%s). " ++ "memberof_postop_modify - Failed to replace values in dn (%s). " + "Error (%d)\n", + slapi_sdn_get_dn(sdn), ret); + slapi_mod_done(next_mod); + goto bail; + } ++ break; + } +- break; +- } + +- case LDAP_MOD_REPLACE: { +- /* replace current values */ +- if ((ret = memberof_replace_list(pb, &configCopy, sdn))) { +- slapi_log_err(SLAPI_LOG_ERR, MEMBEROF_PLUGIN_SUBSYSTEM, +- "memberof_postop_modify - Failed to replace values in dn (%s). " +- "Error (%d)\n", +- slapi_sdn_get_dn(sdn), ret); +- slapi_mod_done(next_mod); +- goto bail; ++ default: { ++ slapi_log_err( ++ SLAPI_LOG_ERR, ++ MEMBEROF_PLUGIN_SUBSYSTEM, ++ "memberof_postop_modify - Unknown mod type\n"); ++ ret = SLAPI_PLUGIN_FAILURE; ++ break; + } +- break; +- } +- +- default: { +- slapi_log_err( +- SLAPI_LOG_ERR, +- MEMBEROF_PLUGIN_SUBSYSTEM, +- "memberof_postop_modify - Unknown mod type\n"); +- ret = SLAPI_PLUGIN_FAILURE; +- break; +- } + } + } + +@@ -1281,6 +2231,53 @@ done: + return ret; + } + ++/* This callback is called during an operation be_postop ++ * So it is called AFTER the txn was committed ++ * In case there are deferred updates it is important so ++ * that the thread running the deferred updates uses an DB ++ * that is up to date. Especially, if it is using internal searches ++ * the indexes are valid. ++ * (required for mdb) ++ * The callback read the tasks, stored by the be_txn_postop ++ * in the pblock and push it to the list of tasks that the ++ * deferred update thread will process. ++ */ ++int ++memberof_push_deferred_task(Slapi_PBlock *pb) ++{ ++ int ret = SLAPI_PLUGIN_SUCCESS; ++ MemberOfConfig *mainConfig = NULL; ++ MemberofDeferredList* deferred_list; ++ MemberofDeferredTask* task = NULL; ++ ++ /* retrieve deferred update params that are valid until shutdown */ ++ memberof_rlock_config(); ++ mainConfig = memberof_get_config(); ++ if (mainConfig) { ++ deferred_list = mainConfig->deferred_list; ++ } ++ memberof_unlock_config(); ++ ++ if (!mainConfig) { ++ /* The configuration has not yet been uploaded. Get out of here */ ++ return ret; ++ } ++ ++ slapi_pblock_get(pb, SLAPI_MEMBEROF_DEFERRED_TASK, (void **) &task); ++ if (task) { ++ /* retrieve the task, registered during BE_TXN_POSTOP, and ++ * add it to the list of tasks that deferred update thread ++ * will process async ++ */ ++ slapi_pblock_set(pb, SLAPI_MEMBEROF_DEFERRED_TASK, NULL); ++ if (add_deferred_task(deferred_list, task)) { ++ slapi_log_err(SLAPI_LOG_ERR, MEMBEROF_PLUGIN_SUBSYSTEM, ++ "memberof_push_deferred_task - failure during deferred update. Run memberof fixup.\n"); ++ ret = SLAPI_PLUGIN_FAILURE; ++ } ++ } ++ return ret; ++} + + /* + * memberof_postop_add() +@@ -1311,6 +2308,43 @@ memberof_postop_add(Slapi_PBlock *pb) + struct slapi_entry *e = NULL; + MemberOfConfig configCopy = {0}; + MemberOfConfig *mainConfig; ++ Slapi_DN *copied_sdn; ++ PRBool deferred_update; ++ ++ /* retrieve deferred update params that are valid until shutdown */ ++ memberof_rlock_config(); ++ mainConfig = memberof_get_config(); ++ deferred_update = mainConfig->deferred_update; ++ memberof_unlock_config(); ++ ++ if (deferred_update) { ++ MemberofDeferredTask* task; ++ Slapi_Operation *op; ++ int deferred_op_running = 1; ++ ++ /* Should be freed with slapi_sdn_free(copied_sdn) */ ++ copied_sdn = slapi_sdn_dup(sdn); ++ ++ task = (MemberofDeferredTask *)slapi_ch_calloc(1, sizeof(MemberofDeferredTask)); ++ task->d_add = (MemberofDeferredAddTask *)slapi_ch_calloc(1, sizeof(MemberofDeferredAddTask)); ++ slapi_pblock_set(pb, SLAPI_DEFERRED_MEMBEROF, &deferred_op_running); /* operation wait until the completion of the deferred update */ ++ task->d_add->pb_original = pb; ++ task->d_add->pb = slapi_pblock_new(); ++ op = internal_operation_new(SLAPI_OPERATION_ADD, 0); ++ slapi_pblock_set(task->d_add->pb, SLAPI_OPERATION, op); ++ slapi_pblock_get(pb, SLAPI_ENTRY_POST_OP, &e); ++ slapi_pblock_set(task->d_add->pb, SLAPI_ENTRY_POST_OP, slapi_entry_dup(e)); ++ slapi_pblock_set(task->d_add->pb, SLAPI_TARGET_SDN, copied_sdn); ++ task->deferred_choice = SLAPI_OPERATION_ADD; ++ /* store the task in the pblock that will be added to ++ * the deferred list during the backend postop (after txn_commit) ++ */ ++ slapi_pblock_set(pb, SLAPI_MEMBEROF_DEFERRED_TASK, (void *) task); ++ ret = SLAPI_PLUGIN_SUCCESS; ++ goto bail; ++ } else { ++ slapi_pblock_set(pb, SLAPI_MEMBEROF_DEFERRED_TASK, NULL); ++ } + slapi_pblock_get(pb, SLAPI_ENTRY_POST_OP, &e); + + /* is the entry of interest? */ +@@ -1331,10 +2365,9 @@ memberof_postop_add(Slapi_PBlock *pb) + memberof_unlock_config(); + + if (interested) { +- int i = 0; + Slapi_Attr *attr = 0; + +- for (i = 0; configCopy.groupattrs && configCopy.groupattrs[i]; i++) { ++ for (size_t i = 0; configCopy.groupattrs && configCopy.groupattrs[i]; i++) { + if (0 == slapi_entry_attr_find(e, configCopy.groupattrs[i], &attr)) { + if ((ret = memberof_add_attr_list(pb, &configCopy, sdn, attr))) { + slapi_log_err(SLAPI_LOG_ERR, MEMBEROF_PLUGIN_SUBSYSTEM, +@@ -1525,6 +2558,10 @@ memberof_modop_one_replace_r(Slapi_PBlock *pb, MemberOfConfig *config, int mod_o + udn); + goto bail; + } ++ slapi_log_err(SLAPI_LOG_PLUGIN, MEMBEROF_PLUGIN_SUBSYSTEM, ++ "memberof_modop_one_replace_r - mod_op=%d op_to=%s op_this=%s.\n", ++ mod_op, op_to, op_this); ++ + /* op_this and op_to are both case-normalized */ + slapi_value_set_flags(this_dn_val, SLAPI_ATTR_FLAG_NORMALIZED_CIS); + slapi_value_set_flags(to_dn_val, SLAPI_ATTR_FLAG_NORMALIZED_CIS); +@@ -1631,7 +2668,6 @@ memberof_modop_one_replace_r(Slapi_PBlock *pb, MemberOfConfig *config, int mod_o + /* group */ + Slapi_Value *ll_dn_val = 0; + Slapi_Attr *members = 0; +- int i = 0; + + ll = stack; + +@@ -1643,9 +2679,7 @@ memberof_modop_one_replace_r(Slapi_PBlock *pb, MemberOfConfig *config, int mod_o + + if (0 == memberof_compare(config, &ll_dn_val, &to_dn_val)) { + slapi_value_free(&ll_dn_val); +- +- /* someone set up infinitely +- recursive groups - bail out */ ++ /* someone set up infinitely recursive groups - bail out */ + slapi_log_err(SLAPI_LOG_PLUGIN, + MEMBEROF_PLUGIN_SUBSYSTEM, + "memberof_modop_one_replace_r - Group recursion" +@@ -1653,7 +2687,6 @@ memberof_modop_one_replace_r(Slapi_PBlock *pb, MemberOfConfig *config, int mod_o + op_to); + goto bail; + } +- + slapi_value_free(&ll_dn_val); + ll = ll->next; + } +@@ -1669,11 +2702,14 @@ memberof_modop_one_replace_r(Slapi_PBlock *pb, MemberOfConfig *config, int mod_o + ll->next = stack; + + /* Go through each grouping attribute one at a time. */ +- for (i = 0; config->groupattrs && config->groupattrs[i]; i++) { ++ for (size_t i = 0; config->groupattrs && config->groupattrs[i]; i++) { + slapi_entry_attr_find(e, config->groupattrs[i], &members); + if (members) { + if ((rc = memberof_mod_attr_list_r(pb, config, mod_op, group_sdn, + op_this_sdn, members, ll)) != 0) { ++ slapi_log_err(SLAPI_LOG_PLUGIN, ++ MEMBEROF_PLUGIN_SUBSYSTEM, ++ "memberof_modop_one_replace_r - memberof_mod_attr_list_r failed.\n"); + goto bail; + } + } +@@ -1769,6 +2805,10 @@ memberof_modop_one_replace_r(Slapi_PBlock *pb, MemberOfConfig *config, int mod_o + } + + bail: ++ if (rc) { ++ slapi_log_err(SLAPI_LOG_PLUGIN, MEMBEROF_PLUGIN_SUBSYSTEM, ++ "memberof_modop_one_replace_r failed. rc=%d\n", rc); ++ } + slapi_value_free(&to_dn_val); + slapi_value_free(&this_dn_val); + slapi_search_get_entry_done(&entry_pb); +@@ -2077,8 +3117,7 @@ memberof_get_groups(MemberOfConfig *config, Slapi_DN *member_sdn) + void + dump_cache_entry(memberof_cached_value *double_check, const char *msg) + { +- int i; +- for (i = 0; double_check[i].valid; i++) { ++ for (size_t i = 0; double_check[i].valid; i++) { + slapi_log_err(SLAPI_LOG_PLUGIN, MEMBEROF_PLUGIN_SUBSYSTEM, "dump_cache_entry: %s -> %s\n", + msg ? msg : "", + double_check[i].group_dn_val ? double_check[i].group_dn_val : "NULL"); +@@ -2307,7 +3346,9 @@ memberof_get_groups_callback(Slapi_Entry *e, void *callback_data) + MemberOfConfig *config = ((memberof_get_groups_data *)callback_data)->config; + int rc = 0; + +- if (slapi_is_shutting_down()) { ++ if (!config->deferred_update && slapi_is_shutting_down()) { ++ slapi_log_err(SLAPI_LOG_PLUGIN, MEMBEROF_PLUGIN_SUBSYSTEM, ++ "memberof_get_groups_callback - aborted because shutdown is in progress\n"); + rc = -1; + goto bail; + } +@@ -2393,7 +3434,6 @@ memberof_is_direct_member(MemberOfConfig *config, Slapi_Value *groupdn, Slapi_Va + Slapi_DN *sdn = 0; + Slapi_Entry *group_e = 0; + Slapi_Attr *attr = 0; +- int i = 0; + + sdn = slapi_sdn_new_normdn_byref(slapi_value_get_string(groupdn)); + +@@ -2402,7 +3442,7 @@ memberof_is_direct_member(MemberOfConfig *config, Slapi_Value *groupdn, Slapi_Va + + if (group_e) { + /* See if memberdn is referred to by any of the group attributes. */ +- for (i = 0; config->groupattrs && config->groupattrs[i]; i++) { ++ for (size_t i = 0; config->groupattrs && config->groupattrs[i]; i++) { + slapi_entry_attr_find(group_e, config->groupattrs[i], &attr); + if (attr && (0 == slapi_attr_value_find(attr, slapi_value_get_berval(memberdn)))) { + rc = 1; +@@ -2427,9 +3467,8 @@ static int + memberof_is_grouping_attr(char *type, MemberOfConfig *config) + { + int match = 0; +- int i = 0; + +- for (i = 0; config && config->groupattrs && config->groupattrs[i]; i++) { ++ for (size_t i = 0; config && config->groupattrs && config->groupattrs[i]; i++) { + match = slapi_attr_types_equivalent(type, config->groupattrs[i]); + if (match) { + /* If we found a match, we're done. */ +@@ -2634,12 +3673,11 @@ memberof_replace_list(Slapi_PBlock *pb, MemberOfConfig *config, Slapi_DN *group_ + Slapi_Attr *pre_attr = 0; + Slapi_Attr *post_attr = 0; + int rc = 0; +- int i = 0; + + slapi_pblock_get(pb, SLAPI_ENTRY_PRE_OP, &pre_e); + slapi_pblock_get(pb, SLAPI_ENTRY_POST_OP, &post_e); + +- for (i = 0; config && config->groupattrs && config->groupattrs[i]; i++) { ++ for (size_t i = 0; config && config->groupattrs && config->groupattrs[i]; i++) { + if (pre_e && post_e) { + slapi_entry_attr_find(pre_e, config->groupattrs[i], &pre_attr); + slapi_entry_attr_find(post_e, config->groupattrs[i], &post_attr); +@@ -2872,11 +3910,14 @@ memberof_fixup_task_thread(void *arg) + if (be) { + fixup_pb = slapi_pblock_new(); + slapi_pblock_set(fixup_pb, SLAPI_BACKEND, be); +- rc = slapi_back_transaction_begin(fixup_pb); +- if (rc) { +- slapi_log_err(SLAPI_LOG_ERR, MEMBEROF_PLUGIN_SUBSYSTEM, +- "memberof_fixup_task_thread - Failed to start transaction\n"); +- goto done; ++ /* Start a txn but not in deferred case: Should not do big txn in txn mode */ ++ if (!configCopy.deferred_update) { ++ rc = slapi_back_transaction_begin(fixup_pb); ++ if (rc) { ++ slapi_log_err(SLAPI_LOG_ERR, MEMBEROF_PLUGIN_SUBSYSTEM, ++ "memberof_fixup_task_thread - Failed to start transaction\n"); ++ goto done; ++ } + } + } else { + slapi_log_err(SLAPI_LOG_ERR, MEMBEROF_PLUGIN_SUBSYSTEM, +@@ -3072,6 +4113,7 @@ memberof_task_destructor(Slapi_Task *task) + "memberof_task_destructor <--\n"); + } + ++/* The fixup task meat */ + int + memberof_fix_memberof(MemberOfConfig *config, Slapi_Task *task, task_data *td) + { +@@ -3086,7 +4128,7 @@ memberof_fix_memberof(MemberOfConfig *config, Slapi_Task *task, task_data *td) + + rc = slapi_search_internal_callback_pb(search_pb, + config, +- 0, memberof_fix_memberof_callback, ++ 0, memberof_fixup_memberof_callback, + 0); + if (rc) { + char *errmsg; +@@ -3096,7 +4138,9 @@ memberof_fix_memberof(MemberOfConfig *config, Slapi_Task *task, task_data *td) + errmsg = ldap_err2string(result); + slapi_log_err(SLAPI_LOG_ERR, MEMBEROF_PLUGIN_SUBSYSTEM, + "memberof_fix_memberof - Failed (%s)\n", errmsg); +- slapi_task_log_notice(task, "Memberof task failed (%s)", errmsg); ++ if (task) { ++ slapi_task_log_notice(task, "Memberof task failed (%s)", errmsg); ++ } + } + + slapi_pblock_destroy(search_pb); +@@ -3199,6 +4243,16 @@ ancestors_cache_add(MemberOfConfig *config, const void *key, void *value) + return e; + } + ++int ++memberof_fixup_memberof_callback(Slapi_Entry *e, void *callback_data) ++{ ++ /* Always check shutdown in fixup task */ ++ if (slapi_is_shutting_down()) { ++ return -1; ++ } ++ return memberof_fix_memberof_callback(e, callback_data); ++} ++ + /* memberof_fix_memberof_callback() + * Add initial and/or fix up broken group list in entry + * +@@ -3220,7 +4274,9 @@ memberof_fix_memberof_callback(Slapi_Entry *e, void *callback_data) + /* + * If the server is ordered to shutdown, stop the fixup and return an error. + */ +- if (slapi_is_shutting_down()) { ++ if (!config->deferred_update && slapi_is_shutting_down()) { ++ slapi_log_err(SLAPI_LOG_PLUGIN, MEMBEROF_PLUGIN_SUBSYSTEM, "memberof_fix_memberof_callback - " ++ "Aborted because shutdown is in progress. rc = -1\n"); + rc = -1; + goto bail; + } +@@ -3338,6 +4394,10 @@ memberof_fix_memberof_callback(Slapi_Entry *e, void *callback_data) + } + + bail: ++ if (rc) { ++ slapi_log_err(SLAPI_LOG_PLUGIN, MEMBEROF_PLUGIN_SUBSYSTEM, ++ "memberof_fix_memberof_callback failed. rc=%d\n", rc); ++ } + return rc; + } + +@@ -3383,6 +4443,8 @@ memberof_add_memberof_attr(LDAPMod **mods, const char *dn, char *add_oc) + slapi_pblock_destroy(mod_pb); + } else if (rc) { + /* Some other fatal error */ ++ slapi_log_err(SLAPI_LOG_PLUGIN, MEMBEROF_PLUGIN_SUBSYSTEM, ++ "memberof_add_memberof_attr - Internal modify failed. rc=%d\n", rc); + break; + } else { + /* success */ +@@ -3390,6 +4452,10 @@ memberof_add_memberof_attr(LDAPMod **mods, const char *dn, char *add_oc) + } + } + slapi_pblock_destroy(mod_pb); ++ if (rc) { ++ slapi_log_err(SLAPI_LOG_PLUGIN, MEMBEROF_PLUGIN_SUBSYSTEM, ++ "memberof_add_memberof_attr failed. rc=%d\n", rc); ++ } + + return rc; + } +diff --git a/ldap/servers/plugins/memberof/memberof.h b/ldap/servers/plugins/memberof/memberof.h +index 6cb46a447..c11d901ab 100644 +--- a/ldap/servers/plugins/memberof/memberof.h ++++ b/ldap/servers/plugins/memberof/memberof.h +@@ -35,21 +35,89 @@ + #define MEMBEROF_PLUGIN_SUBSYSTEM "memberof-plugin" /* used for logging */ + #define MEMBEROF_INT_PREOP_DESC "memberOf internal postop plugin" + #define MEMBEROF_PREOP_DESC "memberof preop plugin" ++#define MEMBEROF_BEPOSTOP_DESC "memberof backend postop plugin" + #define MEMBEROF_GROUP_ATTR "memberOfGroupAttr" + #define MEMBEROF_ATTR "memberOfAttr" + #define MEMBEROF_BACKEND_ATTR "memberOfAllBackends" + #define MEMBEROF_ENTRY_SCOPE_ATTR "memberOfEntryScope" + #define MEMBEROF_SKIP_NESTED_ATTR "memberOfSkipNested" ++#define MEMBEROF_DEFERRED_UPDATE_ATTR "memberOfDeferredUpdate" + #define MEMBEROF_AUTO_ADD_OC "memberOfAutoAddOC" ++#define MEMBEROF_NEED_FIXUP "memberOfNeedFixup" + #define NSMEMBEROF "nsMemberOf" + #define MEMBEROF_ENTRY_SCOPE_EXCLUDE_SUBTREE "memberOfEntryScopeExcludeSubtree" + #define DN_SYNTAX_OID "1.3.6.1.4.1.1466.115.121.1.12" + #define NAME_OPT_UID_SYNTAX_OID "1.3.6.1.4.1.1466.115.121.1.34" ++#define SHUTDOWN_TIMEOUT 60 /* systemctl timeout is by default 90s */ + + + /* + * structs + */ ++ ++typedef struct memberof_deferred_mod_task ++{ ++ Slapi_PBlock *pb_original; ++ Slapi_PBlock *pb; ++ LDAPMod **mods; ++ Slapi_DN *target_sdn; ++} MemberofDeferredModTask; ++typedef struct memberof_deferred_add_task ++{ ++ Slapi_PBlock *pb_original; ++ Slapi_PBlock *pb; ++ int foo; ++} MemberofDeferredAddTask; ++typedef struct memberof_deferred_del_task ++{ ++ Slapi_PBlock *pb_original; ++ Slapi_PBlock *pb; ++ int foo; ++} MemberofDeferredDelTask; ++typedef struct memberof_deferred_modrdn_task ++{ ++ Slapi_PBlock *pb_original; ++ Slapi_PBlock *pb; ++ int foo; ++} MemberofDeferredModrdnTask; ++typedef struct memberof_deferred_task ++{ ++ unsigned long deferred_choice; ++ union ++ { ++ /* modify */ ++ struct memberof_deferred_mod_task *d_un_mod; ++ ++ /* modify */ ++ struct memberof_deferred_add_task *d_un_add; ++ ++ /* modify */ ++ struct memberof_deferred_del_task *d_un_del; ++ ++ /* modify */ ++ struct memberof_deferred_modrdn_task *d_un_modrdn; ++ } d_un; ++#define d_mod d_un.d_un_mod ++#define d_add d_un.d_un_add ++#define d_del d_un.d_un_del ++#define d_modrdn d_un.d_un_modrdn ++ struct memberof_deferred_task *next; ++ struct memberof_deferred_task *prev; ++} MemberofDeferredTask; ++ ++typedef struct memberof_deferred_list ++{ ++ pthread_mutex_t deferred_list_mutex; ++ pthread_cond_t deferred_list_cv; ++ PRThread *deferred_tid; ++ int current_task; ++ int total_added; ++ int total_removed; ++ MemberofDeferredTask *tasks_head; ++ MemberofDeferredTask *tasks_queue; ++} MemberofDeferredList; ++ ++ + typedef struct memberofconfig + { + char **groupattrs; +@@ -64,9 +132,12 @@ typedef struct memberofconfig + int skip_nested; + int fixup_task; + char *auto_add_oc; ++ PRBool deferred_update; ++ MemberofDeferredList *deferred_list; + PLHashTable *ancestors_cache; + PLHashTable *fixup_cache; + Slapi_Task *task; ++ int need_fixup; + } MemberOfConfig; + + /* The key to access the hash table is the normalized DN +diff --git a/ldap/servers/plugins/memberof/memberof_config.c b/ldap/servers/plugins/memberof/memberof_config.c +index 586be11a9..89c44b014 100644 +--- a/ldap/servers/plugins/memberof/memberof_config.c ++++ b/ldap/servers/plugins/memberof/memberof_config.c +@@ -36,8 +36,8 @@ + */ + static void fixup_hashtable_empty( MemberOfConfig *config, char *msg); + static void ancestor_hashtable_empty(MemberOfConfig *config, char *msg); +-static int memberof_validate_config (Slapi_PBlock *pb, Slapi_Entry* entryBefore, Slapi_Entry* e, +- int *returncode, char *returntext, void *arg); ++static int memberof_validate_config (Slapi_PBlock *pb, Slapi_Entry* entryBefore, Slapi_Entry* e, ++ int *returncode, char *returntext, void *arg); + static int memberof_search (Slapi_PBlock *pb __attribute__((unused)), + Slapi_Entry* entryBefore __attribute__((unused)), + Slapi_Entry* e __attribute__((unused)), +@@ -469,7 +469,9 @@ memberof_apply_config(Slapi_PBlock *pb __attribute__((unused)), + char **entryScopeExcludeSubtrees = NULL; + char *sharedcfg = NULL; + const char *skip_nested = NULL; ++ const char *deferred_update = NULL; + char *auto_add_oc = NULL; ++ const char *needfixup = NULL; + int num_vals = 0; + + *returncode = LDAP_SUCCESS; +@@ -503,7 +505,9 @@ memberof_apply_config(Slapi_PBlock *pb __attribute__((unused)), + memberof_attr = slapi_entry_attr_get_charptr(e, MEMBEROF_ATTR); + allBackends = slapi_entry_attr_get_ref(e, MEMBEROF_BACKEND_ATTR); + skip_nested = slapi_entry_attr_get_ref(e, MEMBEROF_SKIP_NESTED_ATTR); ++ deferred_update = slapi_entry_attr_get_ref(e, MEMBEROF_DEFERRED_UPDATE_ATTR); + auto_add_oc = slapi_entry_attr_get_charptr(e, MEMBEROF_AUTO_ADD_OC); ++ needfixup = slapi_entry_attr_get_ref(e, MEMBEROF_NEED_FIXUP); + + if (auto_add_oc == NULL) { + auto_add_oc = slapi_ch_strdup(NSMEMBEROF); +@@ -514,6 +518,7 @@ memberof_apply_config(Slapi_PBlock *pb __attribute__((unused)), + * a memberOf operation, so we obtain an exclusive lock here + */ + memberof_wlock_config(); ++ theConfig.need_fixup = (needfixup != NULL); + + if (groupattrs) { + int i = 0; +@@ -615,6 +620,15 @@ memberof_apply_config(Slapi_PBlock *pb __attribute__((unused)), + } + } + ++ ++ if (deferred_update) { ++ if (strcasecmp(deferred_update, "on") == 0) { ++ theConfig.deferred_update = PR_TRUE; ++ } else { ++ theConfig.deferred_update = PR_FALSE; ++ } ++ } ++ + if (allBackends) { + if (strcasecmp(allBackends, "on") == 0) { + theConfig.allBackends = 1; +@@ -755,6 +769,14 @@ memberof_copy_config(MemberOfConfig *dest, MemberOfConfig *src) + slapi_ch_free_string(&dest->auto_add_oc); + dest->auto_add_oc = slapi_ch_strdup(src->auto_add_oc); + ++ dest->deferred_update = src->deferred_update; ++ dest->need_fixup = src->need_fixup; ++ /* ++ * deferred_list, ancestors_cache, fixup_cache are not config parameters ++ * but simple global parameters and should not be copied as ++ * and they are only meaningful in the original config (i.e: theConfig) ++ */ ++ + if (src->entryScopes) { + int num_vals = 0; + +diff --git a/ldap/servers/slapd/back-ldbm/ldbm_add.c b/ldap/servers/slapd/back-ldbm/ldbm_add.c +index ce4c314a1..bca660f22 100644 +--- a/ldap/servers/slapd/back-ldbm/ldbm_add.c ++++ b/ldap/servers/slapd/back-ldbm/ldbm_add.c +@@ -85,6 +85,7 @@ ldbm_back_add(Slapi_PBlock *pb) + int is_tombstone_operation = 0; + int is_fixup_operation = 0; + int is_remove_from_cache = 0; ++ int is_internal = 0; + int op_plugin_call = 1; + int is_ruv = 0; /* True if the current entry is RUV */ + CSN *opcsn = NULL; +@@ -125,6 +126,7 @@ ldbm_back_add(Slapi_PBlock *pb) + is_fixup_operation = operation_is_flag_set(operation, OP_FLAG_REPL_FIXUP); + is_ruv = operation_is_flag_set(operation, OP_FLAG_REPL_RUV); + is_remove_from_cache = operation_is_flag_set(operation, OP_FLAG_NEVER_CACHE); ++ is_internal = operation_is_flag_set(operation, OP_FLAG_INTERNAL); + if (operation_is_flag_set(operation,OP_FLAG_NOOP)) op_plugin_call = 0; + + inst = (ldbm_instance *)be->be_instance_info; +@@ -1441,6 +1443,19 @@ common_return: + ldap_result_code = LDAP_SUCCESS; + } + if (!result_sent) { ++ int deferred; ++ PRIntervalTime delay; ++ ++ if (!is_internal) { ++ slapi_pblock_get(pb, SLAPI_DEFERRED_MEMBEROF, &deferred); ++ if (deferred) { ++ delay = PR_MillisecondsToInterval(100); ++ } ++ while (deferred) { ++ DS_Sleep(delay); ++ slapi_pblock_get(pb, SLAPI_DEFERRED_MEMBEROF, &deferred); ++ } ++ } + slapi_send_ldap_result(pb, ldap_result_code, ldap_result_matcheddn, ldap_result_message, 0, NULL); + } + } +diff --git a/ldap/servers/slapd/back-ldbm/ldbm_delete.c b/ldap/servers/slapd/back-ldbm/ldbm_delete.c +index 27f0ac58a..c0f2516de 100644 +--- a/ldap/servers/slapd/back-ldbm/ldbm_delete.c ++++ b/ldap/servers/slapd/back-ldbm/ldbm_delete.c +@@ -57,6 +57,7 @@ ldbm_back_delete(Slapi_PBlock *pb) + int is_ruv = 0; /* True if the current entry is RUV */ + int is_replicated_operation = 0; + int is_tombstone_entry = 0; /* True if the current entry is alreday a tombstone */ ++ int is_internal; + int delete_tombstone_entry = 0; /* We must remove the given tombstone entry from the DB */ + int create_tombstone_entry = 0; /* We perform a "regular" LDAP delete but since we use */ + /* replication, we must create a new tombstone entry */ +@@ -141,6 +142,7 @@ ldbm_back_delete(Slapi_PBlock *pb) + is_fixup_operation = operation_is_flag_set(operation, OP_FLAG_REPL_FIXUP); + is_ruv = operation_is_flag_set(operation, OP_FLAG_REPL_RUV); + delete_tombstone_entry = operation_is_flag_set(operation, OP_FLAG_TOMBSTONE_ENTRY); ++ is_internal = operation_is_flag_set(operation, OP_FLAG_INTERNAL); + + inst = (ldbm_instance *)be->be_instance_info; + if (inst && inst->inst_ref_count) { +@@ -1567,6 +1569,19 @@ diskfull_return: + ldap_result_code = LDAP_SUCCESS; + } + if (!result_sent) { ++ int deferred; ++ PRIntervalTime delay; ++ ++ if (!is_internal) { ++ slapi_pblock_get(pb, SLAPI_DEFERRED_MEMBEROF, &deferred); ++ if (deferred) { ++ delay = PR_MillisecondsToInterval(100); ++ } ++ while (deferred) { ++ DS_Sleep(delay); ++ slapi_pblock_get(pb, SLAPI_DEFERRED_MEMBEROF, &deferred); ++ } ++ } + slapi_send_ldap_result(pb, ldap_result_code, NULL, ldap_result_message, 0, NULL); + } + } +diff --git a/ldap/servers/slapd/back-ldbm/ldbm_modify.c b/ldap/servers/slapd/back-ldbm/ldbm_modify.c +index 64b293001..0c62c87f0 100644 +--- a/ldap/servers/slapd/back-ldbm/ldbm_modify.c ++++ b/ldap/servers/slapd/back-ldbm/ldbm_modify.c +@@ -511,6 +511,7 @@ ldbm_back_modify(Slapi_PBlock *pb) + entry_address *addr; + int is_fixup_operation = 0; + int is_ruv = 0; /* True if the current entry is RUV */ ++ int is_internal = 0; + CSN *opcsn = NULL; + int repl_op; + int opreturn = 0; +@@ -534,6 +535,7 @@ ldbm_back_modify(Slapi_PBlock *pb) + slapi_pblock_get(pb, SLAPI_OPERATION, &operation); + + fixup_tombstone = operation_is_flag_set(operation, OP_FLAG_TOMBSTONE_FIXUP); ++ is_internal = operation_is_flag_set(operation, OP_FLAG_INTERNAL); + + dblayer_txn_init(li, &txn); /* must do this before first goto error_return */ + /* the calls to perform searches require the parent txn if any +@@ -1158,6 +1160,19 @@ common_return: + } + if (!result_sent) { + /* result is already sent in find_entry. */ ++ int deferred; ++ PRIntervalTime delay; ++ ++ if (!is_internal) { ++ slapi_pblock_get(pb, SLAPI_DEFERRED_MEMBEROF, &deferred); ++ if (deferred) { ++ delay = PR_MillisecondsToInterval(100); ++ } ++ while (deferred) { ++ DS_Sleep(delay); ++ slapi_pblock_get(pb, SLAPI_DEFERRED_MEMBEROF, &deferred); ++ } ++ } + slapi_send_ldap_result(pb, ldap_result_code, NULL, ldap_result_message, 0, NULL); + } + } +diff --git a/ldap/servers/slapd/back-ldbm/ldbm_modrdn.c b/ldap/servers/slapd/back-ldbm/ldbm_modrdn.c +index 6a7b73ade..fe35991b5 100644 +--- a/ldap/servers/slapd/back-ldbm/ldbm_modrdn.c ++++ b/ldap/servers/slapd/back-ldbm/ldbm_modrdn.c +@@ -81,6 +81,7 @@ ldbm_back_modrdn(Slapi_PBlock *pb) + int is_fixup_operation = 0; + int is_resurect_operation = 0; + int is_tombstone = 0; ++ int is_internal = 0; + entry_address new_addr; + entry_address *old_addr; + entry_address oldparent_addr; +@@ -124,6 +125,7 @@ ldbm_back_modrdn(Slapi_PBlock *pb) + is_fixup_operation = operation_is_flag_set(operation, OP_FLAG_REPL_FIXUP); + is_resurect_operation = operation_is_flag_set(operation, OP_FLAG_RESURECT_ENTRY); + is_tombstone = operation_is_flag_set(operation, OP_FLAG_TOMBSTONE_ENTRY); /* tombstone_to_glue on parent entry*/ ++ is_internal = operation_is_flag_set(operation, OP_FLAG_INTERNAL); + slapi_pblock_get(pb, SLAPI_CONNECTION, &pb_conn); + + if (NULL == sdn) { +@@ -1508,6 +1510,20 @@ common_return: + ldap_result_code = LDAP_SUCCESS; + } + if (!result_sent) { ++ int deferred; ++ PRIntervalTime delay; ++ ++ if (!is_internal) { ++ /* for direct operation, wait for members update */ ++ slapi_pblock_get(pb, SLAPI_DEFERRED_MEMBEROF, &deferred); ++ if (deferred) { ++ delay = PR_MillisecondsToInterval(100); ++ } ++ while (deferred) { ++ DS_Sleep(delay); ++ slapi_pblock_get(pb, SLAPI_DEFERRED_MEMBEROF, &deferred); ++ } ++ } + slapi_send_ldap_result(pb, ldap_result_code, ldap_result_matcheddn, + ldap_result_message, 0, NULL); + } +diff --git a/ldap/servers/slapd/pblock.c b/ldap/servers/slapd/pblock.c +index c78d1250f..7c1cdc81c 100644 +--- a/ldap/servers/slapd/pblock.c ++++ b/ldap/servers/slapd/pblock.c +@@ -443,6 +443,9 @@ slapi_pblock_get(Slapi_PBlock *pblock, int arg, void *value) + (*(char **)value) = (NULL == pblock->pb_conn->c_dn ? NULL : slapi_ch_strdup(pblock->pb_conn->c_dn)); + pthread_mutex_unlock(&(pblock->pb_conn->c_mutex)); + break; ++ case SLAPI_DEFERRED_MEMBEROF: ++ (*(int *)value) = pblock->pb_deferred_memberof; ++ break; + case SLAPI_CONN_AUTHTYPE: /* deprecated */ + if (pblock->pb_conn == NULL) { + slapi_log_err(SLAPI_LOG_ERR, +@@ -2496,6 +2499,13 @@ slapi_pblock_get(Slapi_PBlock *pblock, int arg, void *value) + (*(int *)value) = 0; + } + break; ++ case SLAPI_MEMBEROF_DEFERRED_TASK: ++ if (pblock->pb_intop != NULL) { ++ (*(void **)value) = pblock->pb_intop->memberof_deferred_task; ++ } else { ++ (*(void **)value) = NULL; ++ } ++ break; + + case SLAPI_USN_INCREMENT_FOR_TOMBSTONE: + if (pblock->pb_intop != NULL) { +@@ -2592,6 +2602,9 @@ slapi_pblock_set(Slapi_PBlock *pblock, int arg, void *value) + (char *)value, NULL, NULL, NULL, NULL); + slapi_ch_free((void **)&authtype); + break; ++ case SLAPI_DEFERRED_MEMBEROF: ++ pblock->pb_deferred_memberof = *((int *)value); ++ break; + case SLAPI_CONN_AUTHTYPE: /* deprecated */ + case SLAPI_CONN_AUTHMETHOD: + if (pblock->pb_conn == NULL) { +@@ -4190,6 +4203,10 @@ slapi_pblock_set(Slapi_PBlock *pblock, int arg, void *value) + case SLAPI_PAGED_RESULTS_COOKIE: + pblock->pb_intop->pb_paged_results_cookie = *(int *)value; + break; ++ case SLAPI_MEMBEROF_DEFERRED_TASK: ++ _pblock_assert_pb_intop(pblock); ++ pblock->pb_intop->memberof_deferred_task = (void *)value; ++ break; + + case SLAPI_USN_INCREMENT_FOR_TOMBSTONE: + pblock->pb_intop->pb_usn_tombstone_incremented = *((int32_t *)value); +diff --git a/ldap/servers/slapd/pblock_v3.h b/ldap/servers/slapd/pblock_v3.h +index b35d78565..cd484115f 100644 +--- a/ldap/servers/slapd/pblock_v3.h ++++ b/ldap/servers/slapd/pblock_v3.h +@@ -163,6 +163,14 @@ typedef struct _slapi_pblock_intop + int pb_paged_results_index; /* stash SLAPI_PAGED_RESULTS_INDEX */ + int pb_paged_results_cookie; /* stash SLAPI_PAGED_RESULTS_COOKIE */ + int32_t pb_usn_tombstone_incremented; /* stash SLAPI_PAGED_RESULTS_COOKIE */ ++ ++ /* For memberof deferred thread ++ * It is set by be_txn_postop with the task that ++ * will be processed by the memberof deferred thread ++ * It is reset by the be_postop, once the txn is committed ++ * when it pushes the task to list of deferred tasks ++ */ ++ void *memberof_deferred_task; + } slapi_pblock_intop; + + /* Stuff that is rarely used, but still present */ +@@ -216,6 +224,7 @@ typedef struct slapi_pblock + struct _slapi_pblock_intop *pb_intop; + struct _slapi_pblock_intplugin *pb_intplugin; + struct _slapi_pblock_deprecated *pb_deprecated; ++ int pb_deferred_memberof; + + #ifdef PBLOCK_ANALYTICS + uint32_t analytics_init; +diff --git a/ldap/servers/slapd/schema.c b/ldap/servers/slapd/schema.c +index a71b357ce..16f1861cf 100644 +--- a/ldap/servers/slapd/schema.c ++++ b/ldap/servers/slapd/schema.c +@@ -6598,3 +6598,24 @@ supplier_learn_new_definitions(struct berval **objectclasses, struct berval **at + modify_schema_free_new_definitions(at_list); + modify_schema_free_new_definitions(oc_list); + } ++ ++/* ++ * schema_get_objectclasses_by_attribute returns the name ++ * of all objectclass containing the attribute) ++ */ ++char ** ++schema_get_objectclasses_by_attribute(const char *attribute) ++{ ++ struct objclass *oc; ++ char **ocs = NULL; ++ ++ schema_dse_lock_read(); ++ for (oc = g_get_global_oc_nolock(); oc != NULL; oc = oc->oc_next) { ++ if (charray_inlist(oc->oc_required, (char*) attribute) || ++ charray_inlist(oc->oc_allowed, (char*) attribute)) { ++ charray_add(&ocs,slapi_ch_strdup(oc->oc_name)); ++ } ++ } ++ schema_dse_unlock(); ++ return ocs; ++} +diff --git a/ldap/servers/slapd/slapi-plugin.h b/ldap/servers/slapd/slapi-plugin.h +index b3b715583..9fdcaccc8 100644 +--- a/ldap/servers/slapd/slapi-plugin.h ++++ b/ldap/servers/slapd/slapi-plugin.h +@@ -7002,6 +7002,7 @@ slapi_timer_result slapi_timespec_expire_check(struct timespec *expire); + #define SLAPI_BE_LASTMOD 137 + #define SLAPI_CONN_ID 139 + #define SLAPI_BACKEND_COUNT 860 ++#define SLAPI_DEFERRED_MEMBEROF 861 + + /* operation */ + #define SLAPI_OPINITIATED_TIME 140 +@@ -7546,6 +7547,12 @@ typedef enum _slapi_op_note_t { + /* dbverify */ + #define SLAPI_DBVERIFY_DBDIR 1947 + ++/* task passed by memberof be_txn_post to the ++ * memberof be_post to be pushed in the list ++ * of memberof deferred updates ++ */ ++#define SLAPI_MEMBEROF_DEFERRED_TASK 1951 ++ + /* convenience macros for checking modify operation types */ + #define SLAPI_IS_MOD_ADD(x) (((x) & ~LDAP_MOD_BVALUES) == LDAP_MOD_ADD) + #define SLAPI_IS_MOD_DELETE(x) (((x) & ~LDAP_MOD_BVALUES) == LDAP_MOD_DELETE) +diff --git a/ldap/servers/slapd/slapi-private.h b/ldap/servers/slapd/slapi-private.h +index d6d74e8a7..4b6cf29eb 100644 +--- a/ldap/servers/slapd/slapi-private.h ++++ b/ldap/servers/slapd/slapi-private.h +@@ -756,6 +756,9 @@ char *slapi_schema_get_superior_name(const char *ocname_or_oid); + + CSN *dup_global_schema_csn(void); + ++/* schema access for memberof plugin */ ++char **schema_get_objectclasses_by_attribute(const char *attribute); ++ + /* misc function for the chaining backend */ + #define CHAIN_ROOT_UPDATE_REJECT 0 + #define CHAIN_ROOT_UPDATE_LOCAL 1 +diff --git a/src/lib389/lib389/plugins.py b/src/lib389/lib389/plugins.py +index a1ad0a45b..6bf1843ad 100644 +--- a/src/lib389/lib389/plugins.py ++++ b/src/lib389/lib389/plugins.py +@@ -907,6 +907,26 @@ class MemberOfPlugin(Plugin): + + self.set('memberofskipnested', 'off') + ++ def get_memberofdeferredupdate(self): ++ """Get memberOfDeferredUpdate attribute""" ++ ++ return self.get_attr_val_utf8_l('memberofdeferredupdate') ++ ++ def get_memberofdeferredupdate_formatted(self): ++ """Display memberofdeferredupdate attribute""" ++ ++ return self.display_attr('memberofdeferredupdate') ++ ++ def set_memberofdeferredupdate(self, value): ++ """Set memberofdeferredupdate attribute""" ++ ++ self.set('memberofdeferredupdate', value) ++ ++ def remove_memberofdeferredupdate(self): ++ """Remove all memberofdeferredupdate attributes""" ++ ++ self.remove_all('memberofdeferredupdate') ++ + def get_autoaddoc(self): + """Get memberofautoaddoc attribute""" + +-- +2.48.1 + diff --git a/0024-Issue-6436-MOD-on-a-large-group-slow-if-substring-in.patch b/0024-Issue-6436-MOD-on-a-large-group-slow-if-substring-in.patch new file mode 100644 index 0000000..b0951a8 --- /dev/null +++ b/0024-Issue-6436-MOD-on-a-large-group-slow-if-substring-in.patch @@ -0,0 +1,236 @@ +From 1845aed98becaba6b975342229cb5e0de79d208d Mon Sep 17 00:00:00 2001 +From: James Chapman +Date: Wed, 29 Jan 2025 17:41:55 +0000 +Subject: [PATCH] Issue 6436 - MOD on a large group slow if substring index is + present (#6437) + +Bug Description: If the substring index is configured for the group +membership attribute ( member or uniqueMember ), the removal of a +member from a large static group is pretty slow. + +Fix Description: A solution to this issue would be to introduce +a new index to track a membership atttribute index. In the interm, +we add a check to healthcheck to inform the user of the implications +of this configuration. + +Fixes: https://github.com/389ds/389-ds-base/issues/6436 + +Reviewed by: @Firstyear, @tbordaz, @droideck (Thanks) +--- + .../suites/healthcheck/health_config_test.py | 89 ++++++++++++++++++- + src/lib389/lib389/lint.py | 15 ++++ + src/lib389/lib389/plugins.py | 37 +++++++- + 3 files changed, 137 insertions(+), 4 deletions(-) + +diff --git a/dirsrvtests/tests/suites/healthcheck/health_config_test.py b/dirsrvtests/tests/suites/healthcheck/health_config_test.py +index 6d3d08bfa..747699486 100644 +--- a/dirsrvtests/tests/suites/healthcheck/health_config_test.py ++++ b/dirsrvtests/tests/suites/healthcheck/health_config_test.py +@@ -212,6 +212,7 @@ def test_healthcheck_RI_plugin_missing_indexes(topology_st): + MEMBER_DN = 'cn=member,cn=index,cn=userroot,cn=ldbm database,cn=plugins,cn=config' + + standalone = topology_st.standalone ++ standalone.config.set("nsslapd-accesslog-logbuffering", "on") + + log.info('Enable RI plugin') + plugin = ReferentialIntegrityPlugin(standalone) +@@ -233,7 +234,7 @@ def test_healthcheck_RI_plugin_missing_indexes(topology_st): + + + def test_healthcheck_MO_plugin_missing_indexes(topology_st): +- """Check if HealthCheck returns DSMOLE0002 code ++ """Check if HealthCheck returns DSMOLE0001 code + + :id: 236b0ec2-13da-48fb-b65a-db7406d56d5d + :setup: Standalone instance +@@ -248,8 +249,8 @@ def test_healthcheck_MO_plugin_missing_indexes(topology_st): + :expectedresults: + 1. Success + 2. Success +- 3. Healthcheck reports DSMOLE0002 code and related details +- 4. Healthcheck reports DSMOLE0002 code and related details ++ 3. Healthcheck reports DSMOLE0001 code and related details ++ 4. Healthcheck reports DSMOLE0001 code and related details + 5. Success + 6. Healthcheck reports no issue found + 7. Healthcheck reports no issue found +@@ -259,6 +260,7 @@ def test_healthcheck_MO_plugin_missing_indexes(topology_st): + MO_GROUP_ATTR = 'creatorsname' + + standalone = topology_st.standalone ++ standalone.config.set("nsslapd-accesslog-logbuffering", "on") + + log.info('Enable MO plugin') + plugin = MemberOfPlugin(standalone) +@@ -279,6 +281,87 @@ def test_healthcheck_MO_plugin_missing_indexes(topology_st): + run_healthcheck_and_flush_log(topology_st, standalone, json=True, searched_code=JSON_OUTPUT) + + ++def test_healthcheck_MO_plugin_substring_index(topology_st): ++ """Check if HealthCheck returns DSMOLE0002 code when the ++ member, uniquemember attribute contains a substring index type ++ ++ :id: 10954811-24ac-4886-8183-e30892f8e02d ++ :setup: Standalone instance ++ :steps: ++ 1. Create DS instance ++ 2. Configure the instance with MO Plugin ++ 3. Change index type to substring for member attribute ++ 4. Use HealthCheck without --json option ++ 5. Use HealthCheck with --json option ++ 6. Change index type back to equality for member attribute ++ 7. Use HealthCheck without --json option ++ 8. Use HealthCheck with --json option ++ 9. Change index type to substring for uniquemember attribute ++ 10. Use HealthCheck without --json option ++ 11. Use HealthCheck with --json option ++ 12. Change index type back to equality for uniquemember attribute ++ 13. Use HealthCheck without --json option ++ 14. Use HealthCheck with --json option ++ ++ :expectedresults: ++ 1. Success ++ 2. Success ++ 3. Success ++ 4. Healthcheck reports DSMOLE0002 code and related details ++ 5. Healthcheck reports DSMOLE0002 code and related details ++ 6. Success ++ 7. Healthcheck reports no issue found ++ 8. Healthcheck reports no issue found ++ 9. Success ++ 10. Healthcheck reports DSMOLE0002 code and related details ++ 11. Healthcheck reports DSMOLE0002 code and related details ++ 12. Success ++ 13. Healthcheck reports no issue found ++ 14. Healthcheck reports no issue found ++ """ ++ ++ RET_CODE = 'DSMOLE0002' ++ MEMBER_DN = 'cn=member,cn=index,cn=userroot,cn=ldbm database,cn=plugins,cn=config' ++ UNIQUE_MEMBER_DN = 'cn=uniquemember,cn=index,cn=userroot,cn=ldbm database,cn=plugins,cn=config' ++ ++ standalone = topology_st.standalone ++ standalone.config.set("nsslapd-accesslog-logbuffering", "on") ++ ++ log.info('Enable MO plugin') ++ plugin = MemberOfPlugin(standalone) ++ plugin.disable() ++ plugin.enable() ++ ++ log.info('Change the index type of the member attribute index to substring') ++ index = Index(topology_st.standalone, MEMBER_DN) ++ index.replace('nsIndexType', 'sub') ++ ++ run_healthcheck_and_flush_log(topology_st, standalone, json=False, searched_code=RET_CODE) ++ run_healthcheck_and_flush_log(topology_st, standalone, json=True, searched_code=RET_CODE) ++ ++ log.info('Set the index type of the member attribute index back to eq') ++ index.replace('nsIndexType', 'eq') ++ ++ run_healthcheck_and_flush_log(topology_st, standalone, json=False, searched_code=CMD_OUTPUT) ++ run_healthcheck_and_flush_log(topology_st, standalone, json=True, searched_code=JSON_OUTPUT) ++ ++ log.info('Change the index type of the uniquemember attribute index to substring') ++ index = Index(topology_st.standalone, UNIQUE_MEMBER_DN) ++ index.replace('nsIndexType', 'sub') ++ ++ run_healthcheck_and_flush_log(topology_st, standalone, json=False, searched_code=RET_CODE) ++ run_healthcheck_and_flush_log(topology_st, standalone, json=True, searched_code=RET_CODE) ++ ++ log.info('Set the index type of the uniquemember attribute index back to eq') ++ index.replace('nsIndexType', 'eq') ++ ++ run_healthcheck_and_flush_log(topology_st, standalone, json=False, searched_code=CMD_OUTPUT) ++ run_healthcheck_and_flush_log(topology_st, standalone, json=True, searched_code=JSON_OUTPUT) ++ ++ # Restart the instance after changing the plugin to avoid breaking the other tests ++ standalone.restart() ++ ++ + @pytest.mark.ds50873 + @pytest.mark.bz1685160 + @pytest.mark.xfail(ds_is_older("1.4.1"), reason="Not implemented") +diff --git a/src/lib389/lib389/lint.py b/src/lib389/lib389/lint.py +index 4d9cbb666..3d3c79ea3 100644 +--- a/src/lib389/lib389/lint.py ++++ b/src/lib389/lib389/lint.py +@@ -231,6 +231,21 @@ database after adding the missing index type. Here is an example using dsconf: + """ + } + ++DSMOLE0002 = { ++ 'dsle': 'DSMOLE0002', ++ 'severity': 'LOW', ++ 'description': 'Removal of a member can be slow ', ++ 'items': ['cn=memberof plugin,cn=plugins,cn=config', ], ++ 'detail': """If the substring index is configured for a membership attribute. The removal of a member ++from the large group can be slow. ++ ++""", ++ 'fix': """If not required, you can remove the substring index type using dsconf: ++ ++ # dsconf slapd-YOUR_INSTANCE backend index set --attr=ATTR BACKEND --del-type=sub ++""" ++} ++ + # Disk Space check. Note - PARTITION is replaced by the calling function + DSDSLE0001 = { + 'dsle': 'DSDSLE0001', +diff --git a/src/lib389/lib389/plugins.py b/src/lib389/lib389/plugins.py +index 6bf1843ad..185398e5b 100644 +--- a/src/lib389/lib389/plugins.py ++++ b/src/lib389/lib389/plugins.py +@@ -12,7 +12,7 @@ import copy + import os.path + from lib389 import tasks + from lib389._mapped_object import DSLdapObjects, DSLdapObject +-from lib389.lint import DSRILE0001, DSRILE0002, DSMOLE0001 ++from lib389.lint import DSRILE0001, DSRILE0002, DSMOLE0001, DSMOLE0002 + from lib389.utils import ensure_str, ensure_list_bytes + from lib389.schema import Schema + from lib389._constants import ( +@@ -827,6 +827,41 @@ class MemberOfPlugin(Plugin): + report['check'] = f'memberof:attr_indexes' + yield report + ++ def _lint_member_substring_index(self): ++ if self.status(): ++ from lib389.backend import Backends ++ backends = Backends(self._instance).list() ++ membership_attrs = ['member', 'uniquemember'] ++ container = self.get_attr_val_utf8_l("nsslapd-plugincontainerscope") ++ for backend in backends: ++ suffix = backend.get_attr_val_utf8_l('nsslapd-suffix') ++ if suffix == "cn=changelog": ++ # Always skip retro changelog ++ continue ++ if container is not None: ++ # Check if this backend is in the scope ++ if not container.endswith(suffix): ++ # skip this backend that is not in the scope ++ continue ++ indexes = backend.get_indexes() ++ for attr in membership_attrs: ++ report = copy.deepcopy(DSMOLE0002) ++ try: ++ index = indexes.get(attr) ++ types = index.get_attr_vals_utf8_l("nsIndexType") ++ if "sub" in types: ++ report['detail'] = report['detail'].replace('ATTR', attr) ++ report['detail'] = report['detail'].replace('BACKEND', suffix) ++ report['fix'] = report['fix'].replace('ATTR', attr) ++ report['fix'] = report['fix'].replace('BACKEND', suffix) ++ report['fix'] = report['fix'].replace('YOUR_INSTANCE', self._instance.serverid) ++ report['items'].append(suffix) ++ report['items'].append(attr) ++ report['check'] = f'attr:substring_index' ++ yield report ++ except KeyError: ++ continue ++ + def get_attr(self): + """Get memberofattr attribute""" + +-- +2.48.1 + diff --git a/0025-Issue-6494-Various-errors-when-using-extended-matchi.patch b/0025-Issue-6494-Various-errors-when-using-extended-matchi.patch new file mode 100644 index 0000000..2e707c0 --- /dev/null +++ b/0025-Issue-6494-Various-errors-when-using-extended-matchi.patch @@ -0,0 +1,651 @@ +From dba27e56161943fbcf54ecbc28337e2c81b07979 Mon Sep 17 00:00:00 2001 +From: progier389 +Date: Mon, 13 Jan 2025 18:03:07 +0100 +Subject: [PATCH] Issue 6494 - Various errors when using extended matching rule + on vlv sort filter (#6495) + +* Issue 6494 - Various errors when using extended matching rule on vlv sort filter + +Various issues when configuring and using extended matching rule within a vlv sort filter: + +Race condition about the keys storage while indexing leading to various heap and data corruption. (lmdb only) +Crash while indexing if vlv are misconfigured because NULL key is not checked. +Read after block because of data type mismatch between SlapiValue and berval +Memory leaks +Solution: + +Serialize the vlv index key generation if vlv filter has an extended matching rule. +Check null keys +Always provides SlapiValue even ifg we want to get keys as bervals +Free properly the resources +Issue: #6494 + +Reviewed by: @mreynolds389 (Thanks!) + +(cherry picked from commit 4bd27ecc4e1d21c8af5ab8cad795d70477179a98) +(cherry picked from commit 223a20250cbf29a546dcb398cfc76024d2f91347) +(cherry picked from commit 280043740a525eaf0438129fd8b99ca251c62366) +--- + .../tests/suites/indexes/regression_test.py | 29 +++ + .../tests/suites/vlv/regression_test.py | 183 ++++++++++++++++++ + ldap/servers/slapd/back-ldbm/cleanup.c | 8 + + ldap/servers/slapd/back-ldbm/dblayer.c | 22 ++- + ldap/servers/slapd/back-ldbm/ldbm_attr.c | 2 +- + ldap/servers/slapd/back-ldbm/matchrule.c | 8 +- + .../servers/slapd/back-ldbm/proto-back-ldbm.h | 3 +- + ldap/servers/slapd/back-ldbm/sort.c | 37 ++-- + ldap/servers/slapd/back-ldbm/vlv.c | 26 +-- + ldap/servers/slapd/back-ldbm/vlv_srch.c | 4 +- + ldap/servers/slapd/generation.c | 5 + + ldap/servers/slapd/plugin_mr.c | 12 +- + src/lib389/lib389/backend.py | 10 + + 13 files changed, 292 insertions(+), 57 deletions(-) + +diff --git a/dirsrvtests/tests/suites/indexes/regression_test.py b/dirsrvtests/tests/suites/indexes/regression_test.py +index fc6db727f..2196fb2ed 100644 +--- a/dirsrvtests/tests/suites/indexes/regression_test.py ++++ b/dirsrvtests/tests/suites/indexes/regression_test.py +@@ -227,6 +227,35 @@ def test_reject_virtual_attr_for_indexing(topo): + break + + ++def test_reindex_extended_matching_rule(topo, add_backend_and_ldif_50K_users): ++ """Check that index with extended matching rule are reindexed properly. ++ ++ :id: 8a3198e8-cc5a-11ef-a3e7-482ae39447e5 ++ :setup: Standalone instance + a second backend with 50K users ++ :steps: ++ 1. Configure uid with 2.5.13.2 matching rule ++ 1. Configure cn with 2.5.13.2 matching rule ++ 2. Reindex ++ :expectedresults: ++ 1. Success ++ 2. Success ++ """ ++ ++ inst = topo.standalone ++ tasks = Tasks(inst) ++ be2 = Backends(topo.standalone).get_backend(SUFFIX2) ++ index = be2.get_index('uid') ++ index.replace('nsMatchingRule', '2.5.13.2') ++ index = be2.get_index('cn') ++ index.replace('nsMatchingRule', '2.5.13.2') ++ ++ assert tasks.reindex( ++ suffix=SUFFIX2, ++ args={TASK_WAIT: True} ++ ) == 0 ++ ++ ++ + if __name__ == "__main__": + # Run isolated + # -s for DEBUG mode +diff --git a/dirsrvtests/tests/suites/vlv/regression_test.py b/dirsrvtests/tests/suites/vlv/regression_test.py +index 3b66de8b5..6ab709bd3 100644 +--- a/dirsrvtests/tests/suites/vlv/regression_test.py ++++ b/dirsrvtests/tests/suites/vlv/regression_test.py +@@ -22,6 +22,146 @@ logging.getLogger(__name__).setLevel(logging.DEBUG) + log = logging.getLogger(__name__) + + ++class BackendHandler: ++ def __init__(self, inst, bedict, scope=ldap.SCOPE_ONELEVEL): ++ self.inst = inst ++ self.bedict = bedict ++ self.bes = Backends(inst) ++ self.scope = scope ++ self.data = {} ++ ++ def find_backend(self, bename): ++ for be in self.bes.list(): ++ if be.get_attr_val_utf8_l('cn') == bename: ++ return be ++ return None ++ ++ def cleanup(self): ++ benames = list(self.bedict.keys()) ++ benames.reverse() ++ for bename in benames: ++ be = self.find_backend(bename) ++ if be: ++ be.delete() ++ ++ def setup(self): ++ # Create backends, add vlv index and populate the backends. ++ for bename,suffix in self.bedict.items(): ++ be = self.bes.create(properties={ ++ 'cn': bename, ++ 'nsslapd-suffix': suffix, ++ }) ++ # Add suffix entry ++ Organization(self.inst, dn=suffix).create(properties={ 'o': bename, }) ++ # Configure vlv ++ vlv_search, vlv_index = create_vlv_search_and_index( ++ self.inst, basedn=suffix, ++ bename=bename, scope=self.scope, ++ prefix=f'vlv_1lvl_{bename}') ++ # Reindex ++ reindex_task = Tasks(self.inst) ++ assert reindex_task.reindex( ++ suffix=suffix, ++ attrname=vlv_index.rdn, ++ args={TASK_WAIT: True}, ++ vlv=True ++ ) == 0 ++ # Add ou=People entry ++ OrganizationalUnits(self.inst, suffix).create(properties={'ou': 'People'}) ++ # Add another ou that will be deleted before the export ++ # so that import will change the vlv search basedn entryid ++ ou2 = OrganizationalUnits(self.inst, suffix).create(properties={'ou': 'dummy ou'}) ++ # Add a demo user so that vlv_check is happy ++ dn = f'uid=demo_user,ou=people,{suffix}' ++ UserAccount(self.inst, dn=dn).create( properties= { ++ 'uid': 'demo_user', ++ 'cn': 'Demo user', ++ 'sn': 'Demo user', ++ 'uidNumber': '99998', ++ 'gidNumber': '99998', ++ 'homeDirectory': '/var/empty', ++ 'loginShell': '/bin/false', ++ 'userpassword': DEMO_PW }) ++ # Add regular user ++ add_users(self.inst, 10, suffix=suffix) ++ # Removing ou2 ++ ou2.delete() ++ # And export ++ tasks = Tasks(self.inst) ++ ldif = f'{self.inst.get_ldif_dir()}/db-{bename}.ldif' ++ assert tasks.exportLDIF(suffix=suffix, ++ output_file=ldif, ++ args={TASK_WAIT: True}) == 0 ++ # Add the various parameters in topology_st.belist ++ self.data[bename] = { 'be': be, ++ 'suffix': suffix, ++ 'ldif': ldif, ++ 'vlv_search' : vlv_search, ++ 'vlv_index' : vlv_index, ++ 'dn' : dn} ++ ++ ++def create_vlv_search_and_index(inst, basedn=DEFAULT_SUFFIX, bename='userRoot', ++ scope=ldap.SCOPE_SUBTREE, prefix="vlv", vlvsort="cn"): ++ vlv_searches = VLVSearch(inst) ++ vlv_search_properties = { ++ "objectclass": ["top", "vlvSearch"], ++ "cn": f"{prefix}Srch", ++ "vlvbase": basedn, ++ "vlvfilter": "(uid=*)", ++ "vlvscope": str(scope), ++ } ++ vlv_searches.create( ++ basedn=f"cn={bename},cn=ldbm database,cn=plugins,cn=config", ++ properties=vlv_search_properties ++ ) ++ ++ vlv_index = VLVIndex(inst) ++ vlv_index_properties = { ++ "objectclass": ["top", "vlvIndex"], ++ "cn": f"{prefix}Idx", ++ "vlvsort": vlvsort, ++ } ++ vlv_index.create( ++ basedn=f"cn={prefix}Srch,cn={bename},cn=ldbm database,cn=plugins,cn=config", ++ properties=vlv_index_properties ++ ) ++ return vlv_searches, vlv_index ++ ++ ++@pytest.fixture ++def vlv_setup_with_uid_mr(topology_st, request): ++ inst = topology_st.standalone ++ bename = 'be1' ++ besuffix = f'o={bename}' ++ beh = BackendHandler(inst, { bename: besuffix }) ++ ++ def fin(): ++ # Cleanup function ++ if not DEBUGGING and inst.exists() and inst.status(): ++ beh.cleanup() ++ ++ request.addfinalizer(fin) ++ ++ # Make sure that our backend are not already present. ++ beh.cleanup() ++ ++ # Then add the new backend ++ beh.setup() ++ ++ index = Index(inst, f'cn=uid,cn=index,cn={bename},cn=ldbm database,cn=plugins,cn=config') ++ index.add('nsMatchingRule', '2.5.13.2') ++ reindex_task = Tasks(inst) ++ assert reindex_task.reindex( ++ suffix=besuffix, ++ attrname='uid', ++ args={TASK_WAIT: True} ++ ) == 0 ++ ++ topology_st.beh = beh ++ return topology_st ++ ++ + @pytest.mark.DS47966 + def test_bulk_import_when_the_backend_with_vlv_was_recreated(topology_m2): + """ +@@ -105,6 +245,49 @@ def test_bulk_import_when_the_backend_with_vlv_was_recreated(topology_m2): + entries = M2.search_s(DEFAULT_SUFFIX, ldap.SCOPE_SUBTREE, "(objectclass=*)") + + ++def test_vlv_with_mr(vlv_setup_with_uid_mr): ++ """ ++ Testing vlv having specific matching rule ++ ++ :id: 5e04afe2-beec-11ef-aa84-482ae39447e5 ++ :setup: Standalone with uid have a matching rule index ++ :steps: ++ 1. Append vlvIndex entries then vlvSearch entry in the dse.ldif ++ 2. Restart the server ++ :expectedresults: ++ 1. Should Success. ++ 2. Should Success. ++ """ ++ inst = vlv_setup_with_uid_mr.standalone ++ beh = vlv_setup_with_uid_mr.beh ++ bename, besuffix = next(iter(beh.bedict.items())) ++ vlv_searches, vlv_index = create_vlv_search_and_index( ++ inst, basedn=besuffix, bename=bename, ++ vlvsort="uid:2.5.13.2") ++ # Reindex the vlv ++ reindex_task = Tasks(inst) ++ assert reindex_task.reindex( ++ suffix=besuffix, ++ attrname=vlv_index.rdn, ++ args={TASK_WAIT: True}, ++ vlv=True ++ ) == 0 ++ ++ inst.restart() ++ users = UserAccounts(inst, besuffix) ++ user_properties = { ++ 'uid': f'a new testuser', ++ 'cn': f'a new testuser', ++ 'sn': 'user', ++ 'uidNumber': '0', ++ 'gidNumber': '0', ++ 'homeDirectory': 'foo' ++ } ++ user = users.create(properties=user_properties) ++ user.delete() ++ assert inst.status() ++ ++ + if __name__ == "__main__": + # Run isolated + # -s for DEBUG mode +diff --git a/ldap/servers/slapd/back-ldbm/cleanup.c b/ldap/servers/slapd/back-ldbm/cleanup.c +index 6b2e9faef..939d8bc4f 100644 +--- a/ldap/servers/slapd/back-ldbm/cleanup.c ++++ b/ldap/servers/slapd/back-ldbm/cleanup.c +@@ -15,12 +15,14 @@ + + #include "back-ldbm.h" + #include "dblayer.h" ++#include "vlv_srch.h" + + int + ldbm_back_cleanup(Slapi_PBlock *pb) + { + struct ldbminfo *li; + Slapi_Backend *be; ++ struct vlvSearch *nextp; + + slapi_log_err(SLAPI_LOG_TRACE, "ldbm_back_cleanup", "ldbm backend cleaning up\n"); + slapi_pblock_get(pb, SLAPI_PLUGIN_PRIVATE, &li); +@@ -45,6 +47,12 @@ ldbm_back_cleanup(Slapi_PBlock *pb) + return 0; + } + ++ /* Release the vlv list */ ++ for (struct vlvSearch *p=be->vlvSearchList; p; p=nextp) { ++ nextp = p->vlv_next; ++ vlvSearch_delete(&p); ++ } ++ + /* + * We check if li is NULL. Because of an issue in how we create backends + * we share the li and plugin info between many unique backends. This causes +diff --git a/ldap/servers/slapd/back-ldbm/dblayer.c b/ldap/servers/slapd/back-ldbm/dblayer.c +index 05cc5b891..6b8ce0016 100644 +--- a/ldap/servers/slapd/back-ldbm/dblayer.c ++++ b/ldap/servers/slapd/back-ldbm/dblayer.c +@@ -494,8 +494,12 @@ int + dblayer_close(struct ldbminfo *li, int dbmode) + { + dblayer_private *priv = (dblayer_private *)li->li_dblayer_private; +- +- return priv->dblayer_close_fn(li, dbmode); ++ int rc = priv->dblayer_close_fn(li, dbmode); ++ if (rc == 0) { ++ /* Clean thread specific data */ ++ dblayer_destroy_txn_stack(); ++ } ++ return rc; + } + + /* Routines for opening and closing random files in the DB_ENV. +@@ -621,6 +625,9 @@ dblayer_erase_index_file(backend *be, struct attrinfo *a, PRBool use_lock, int n + return 0; + } + struct ldbminfo *li = (struct ldbminfo *)be->be_database->plg_private; ++ if (NULL == li) { ++ return 0; ++ } + dblayer_private *priv = (dblayer_private *)li->li_dblayer_private; + + return priv->dblayer_rm_db_file_fn(be, a, use_lock, no_force_chkpt); +@@ -1382,3 +1389,14 @@ dblayer_pop_pvt_txn(void) + } + return; + } ++ ++void ++dblayer_destroy_txn_stack(void) ++{ ++ /* ++ * Cleanup for the main thread to avoid false/positive leaks from libasan ++ * Note: data is freed because PR_SetThreadPrivate calls the ++ * dblayer_cleanup_txn_stack callback ++ */ ++ PR_SetThreadPrivate(thread_private_txn_stack, NULL); ++} +diff --git a/ldap/servers/slapd/back-ldbm/ldbm_attr.c b/ldap/servers/slapd/back-ldbm/ldbm_attr.c +index 708756d3e..70700ca1d 100644 +--- a/ldap/servers/slapd/back-ldbm/ldbm_attr.c ++++ b/ldap/servers/slapd/back-ldbm/ldbm_attr.c +@@ -54,7 +54,7 @@ attrinfo_delete(struct attrinfo **pp) + idl_release_private(*pp); + (*pp)->ai_key_cmp_fn = NULL; + slapi_ch_free((void **)&((*pp)->ai_type)); +- slapi_ch_free((void **)(*pp)->ai_index_rules); ++ charray_free((*pp)->ai_index_rules); + slapi_ch_free((void **)&((*pp)->ai_attrcrypt)); + attr_done(&((*pp)->ai_sattr)); + attrinfo_delete_idlistinfo(&(*pp)->ai_idlistinfo); +diff --git a/ldap/servers/slapd/back-ldbm/matchrule.c b/ldap/servers/slapd/back-ldbm/matchrule.c +index 5d516b9f8..5365e8acf 100644 +--- a/ldap/servers/slapd/back-ldbm/matchrule.c ++++ b/ldap/servers/slapd/back-ldbm/matchrule.c +@@ -107,7 +107,7 @@ destroy_matchrule_indexer(Slapi_PBlock *pb) + * is destroyed + */ + int +-matchrule_values_to_keys(Slapi_PBlock *pb, struct berval **input_values, struct berval ***output_values) ++matchrule_values_to_keys(Slapi_PBlock *pb, Slapi_Value **input_values, struct berval ***output_values) + { + IFP mrINDEX = NULL; + +@@ -135,10 +135,8 @@ matchrule_values_to_keys_sv(Slapi_PBlock *pb, Slapi_Value **input_values, Slapi_ + slapi_pblock_get(pb, SLAPI_PLUGIN_MR_INDEX_SV_FN, &mrINDEX); + if (NULL == mrINDEX) { /* old school - does not have SV function */ + int rc; +- struct berval **bvi = NULL, **bvo = NULL; +- valuearray_get_bervalarray(input_values, &bvi); +- rc = matchrule_values_to_keys(pb, bvi, &bvo); +- ber_bvecfree(bvi); ++ struct berval **bvo = NULL; ++ rc = matchrule_values_to_keys(pb, input_values, &bvo); + /* note - the indexer owns bvo and will free it when destroyed */ + valuearray_init_bervalarray(bvo, output_values); + /* store output values in SV form - caller expects SLAPI_PLUGIN_MR_KEYS is Slapi_Value** */ +diff --git a/ldap/servers/slapd/back-ldbm/proto-back-ldbm.h b/ldap/servers/slapd/back-ldbm/proto-back-ldbm.h +index d93ff9239..157788fa4 100644 +--- a/ldap/servers/slapd/back-ldbm/proto-back-ldbm.h ++++ b/ldap/servers/slapd/back-ldbm/proto-back-ldbm.h +@@ -84,6 +84,7 @@ int dblayer_release_index_file(backend *be, struct attrinfo *a, DB *pDB); + int dblayer_erase_index_file(backend *be, struct attrinfo *a, PRBool use_lock, int no_force_chkpt); + int dblayer_get_id2entry(backend *be, DB **ppDB); + int dblayer_release_id2entry(backend *be, DB *pDB); ++void dblayer_destroy_txn_stack(void); + int dblayer_txn_init(struct ldbminfo *li, back_txn *txn); + int dblayer_txn_begin(backend *be, back_txnid parent_txn, back_txn *txn); + int dblayer_txn_begin_ext(struct ldbminfo *li, back_txnid parent_txn, back_txn *txn, PRBool use_lock); +@@ -560,7 +561,7 @@ int compute_allids_limit(Slapi_PBlock *pb, struct ldbminfo *li); + */ + int create_matchrule_indexer(Slapi_PBlock **pb, char *matchrule, char *type); + int destroy_matchrule_indexer(Slapi_PBlock *pb); +-int matchrule_values_to_keys(Slapi_PBlock *pb, struct berval **input_values, struct berval ***output_values); ++int matchrule_values_to_keys(Slapi_PBlock *pb, Slapi_Value **input_values, struct berval ***output_values); + int matchrule_values_to_keys_sv(Slapi_PBlock *pb, Slapi_Value **input_values, Slapi_Value ***output_values); + + /* +diff --git a/ldap/servers/slapd/back-ldbm/sort.c b/ldap/servers/slapd/back-ldbm/sort.c +index 70ac60803..196af753f 100644 +--- a/ldap/servers/slapd/back-ldbm/sort.c ++++ b/ldap/servers/slapd/back-ldbm/sort.c +@@ -536,30 +536,18 @@ compare_entries_sv(ID *id_a, ID *id_b, sort_spec *s, baggage_carrier *bc, int *e + valuearray_get_bervalarray(valueset_get_valuearray(&attr_b->a_present_values), &value_b); + } else { + /* Match rule case */ +- struct berval **actual_value_a = NULL; +- struct berval **actual_value_b = NULL; +- struct berval **temp_value = NULL; +- +- valuearray_get_bervalarray(valueset_get_valuearray(&attr_a->a_present_values), &actual_value_a); +- valuearray_get_bervalarray(valueset_get_valuearray(&attr_b->a_present_values), &actual_value_b); +- matchrule_values_to_keys(this_one->mr_pb, actual_value_a, &temp_value); +- /* Now copy it, so the second call doesn't crap on it */ +- value_a = slapi_ch_bvecdup(temp_value); /* Really, we'd prefer to not call the chXXX variant...*/ +- matchrule_values_to_keys(this_one->mr_pb, actual_value_b, &value_b); +- +- if ((actual_value_a && !value_a) || +- (actual_value_b && !value_b)) { +- ber_bvecfree(actual_value_a); +- ber_bvecfree(actual_value_b); +- CACHE_RETURN(&inst->inst_cache, &a); +- CACHE_RETURN(&inst->inst_cache, &b); +- *error = 1; +- return 0; ++ Slapi_Value **va_a = valueset_get_valuearray(&attr_a->a_present_values); ++ Slapi_Value **va_b = valueset_get_valuearray(&attr_b->a_present_values); ++ ++ matchrule_values_to_keys(this_one->mr_pb, va_a, &value_a); ++ /* Plugin owns the memory ==> duplicate the key before next call garble it */ ++ value_a = slapi_ch_bvecdup(value_a); ++ matchrule_values_to_keys(this_one->mr_pb, va_b, &value_b); ++ ++ if ((va_a && !value_a) || (va_b && !value_b)) { ++ result = 0; ++ goto bail; + } +- if (actual_value_a) +- ber_bvecfree(actual_value_a); +- if (actual_value_b) +- ber_bvecfree(actual_value_b); + } + /* Compare them */ + if (!order) { +@@ -582,9 +570,10 @@ compare_entries_sv(ID *id_a, ID *id_b, sort_spec *s, baggage_carrier *bc, int *e + } + /* If so, proceed to the next attribute for comparison */ + } ++ *error = 0; ++bail: + CACHE_RETURN(&inst->inst_cache, &a); + CACHE_RETURN(&inst->inst_cache, &b); +- *error = 0; + return result; + } + +diff --git a/ldap/servers/slapd/back-ldbm/vlv.c b/ldap/servers/slapd/back-ldbm/vlv.c +index 121fb3667..70e0bac85 100644 +--- a/ldap/servers/slapd/back-ldbm/vlv.c ++++ b/ldap/servers/slapd/back-ldbm/vlv.c +@@ -605,7 +605,7 @@ vlv_getindices(IFP callback_fn, void *param, backend *be) + * generate the same composite key, so we append the EntryID + * to ensure the uniqueness of the key. + * +- * Always creates a key. Never returns NULL. ++ * May return NULL in case of errors (typically in some configuration error cases) + */ + static struct vlv_key * + vlv_create_key(struct vlvIndex *p, struct backentry *e) +@@ -659,10 +659,8 @@ vlv_create_key(struct vlvIndex *p, struct backentry *e) + /* Matching rule. Do the magic mangling. Plugin owns the memory. */ + if (p->vlv_mrpb[sortattr] != NULL) { + /* xxxPINAKI */ +- struct berval **bval = NULL; + Slapi_Value **va = valueset_get_valuearray(&attr->a_present_values); +- valuearray_get_bervalarray(va, &bval); +- matchrule_values_to_keys(p->vlv_mrpb[sortattr], bval, &value); ++ matchrule_values_to_keys(p->vlv_mrpb[sortattr], va, &value); + } + } + +@@ -779,6 +777,13 @@ do_vlv_update_index(back_txn *txn, struct ldbminfo *li __attribute__((unused)), + } + + key = vlv_create_key(pIndex, entry); ++ if (key == NULL) { ++ slapi_log_err(SLAPI_LOG_ERR, "vlv_create_key", "Unable to generate vlv %s index key." ++ " There may be a configuration issue.\n", pIndex->vlv_name); ++ dblayer_release_index_file(be, pIndex->vlv_attrinfo, db); ++ return rc; ++ } ++ + if (NULL != txn) { + db_txn = txn->back_txn_txn; + } else { +@@ -949,11 +954,11 @@ vlv_create_matching_rule_value(Slapi_PBlock *pb, struct berval *original_value) + struct berval **value = NULL; + if (pb != NULL) { + struct berval **outvalue = NULL; +- struct berval *invalue[2]; +- invalue[0] = original_value; /* jcm: cast away const */ +- invalue[1] = NULL; ++ Slapi_Value v_in = {0}; ++ Slapi_Value *va_in[2] = { &v_in, NULL }; ++ slapi_value_init_berval(&v_in, original_value); + /* The plugin owns the memory it returns in outvalue */ +- matchrule_values_to_keys(pb, invalue, &outvalue); ++ matchrule_values_to_keys(pb, va_in, &outvalue); + if (outvalue != NULL) { + value = slapi_ch_bvecdup(outvalue); + } +@@ -1610,11 +1615,8 @@ retry: + PRBool needFree = PR_FALSE; + + if (sort_control->mr_pb != NULL) { +- struct berval **tmp_entry_value = NULL; +- +- valuearray_get_bervalarray(csn_value, &tmp_entry_value); + /* Matching rule. Do the magic mangling. Plugin owns the memory. */ +- matchrule_values_to_keys(sort_control->mr_pb, /* xxxPINAKI needs modification attr->a_vals */ tmp_entry_value, &entry_value); ++ matchrule_values_to_keys(sort_control->mr_pb, csn_value, &entry_value); + } else { + valuearray_get_bervalarray(csn_value, &entry_value); + needFree = PR_TRUE; /* entry_value is a copy */ +diff --git a/ldap/servers/slapd/back-ldbm/vlv_srch.c b/ldap/servers/slapd/back-ldbm/vlv_srch.c +index fe1208d59..11d1c715b 100644 +--- a/ldap/servers/slapd/back-ldbm/vlv_srch.c ++++ b/ldap/servers/slapd/back-ldbm/vlv_srch.c +@@ -203,6 +203,9 @@ vlvSearch_delete(struct vlvSearch **ppvs) + { + if (ppvs != NULL && *ppvs != NULL) { + struct vlvIndex *pi, *ni; ++ if ((*ppvs)->vlv_e) { ++ slapi_entry_free((struct slapi_entry *)((*ppvs)->vlv_e)); ++ } + slapi_sdn_free(&((*ppvs)->vlv_dn)); + slapi_ch_free((void **)&((*ppvs)->vlv_name)); + slapi_sdn_free(&((*ppvs)->vlv_base)); +@@ -217,7 +220,6 @@ vlvSearch_delete(struct vlvSearch **ppvs) + pi = ni; + } + slapi_ch_free((void **)ppvs); +- *ppvs = NULL; + } + } + +diff --git a/ldap/servers/slapd/generation.c b/ldap/servers/slapd/generation.c +index c4f20f793..89f097322 100644 +--- a/ldap/servers/slapd/generation.c ++++ b/ldap/servers/slapd/generation.c +@@ -93,9 +93,13 @@ get_server_dataversion() + lenstr *l = NULL; + Slapi_Backend *be; + char *cookie; ++ static pthread_mutex_t mutex = PTHREAD_MUTEX_INITIALIZER; + ++ /* Serialize to avoid race condition */ ++ pthread_mutex_lock(&mutex); + /* we already cached the copy - just return it */ + if (server_dataversion_id != NULL) { ++ pthread_mutex_unlock(&mutex); + return server_dataversion_id; + } + +@@ -130,5 +134,6 @@ get_server_dataversion() + server_dataversion_id = slapi_ch_strdup(l->ls_buf); + } + lenstr_free(&l); ++ pthread_mutex_unlock(&mutex); + return server_dataversion_id; + } +diff --git a/ldap/servers/slapd/plugin_mr.c b/ldap/servers/slapd/plugin_mr.c +index 13f76fe52..6cf88b7de 100644 +--- a/ldap/servers/slapd/plugin_mr.c ++++ b/ldap/servers/slapd/plugin_mr.c +@@ -391,28 +391,18 @@ mr_wrap_mr_index_sv_fn(Slapi_PBlock *pb) + return rc; + } + +-/* this function takes SLAPI_PLUGIN_MR_VALUES as struct berval ** and ++/* this function takes SLAPI_PLUGIN_MR_VALUES as Slapi_Value ** and + returns SLAPI_PLUGIN_MR_KEYS as struct berval ** + */ + static int + mr_wrap_mr_index_fn(Slapi_PBlock *pb) + { + int rc = -1; +- struct berval **in_vals = NULL; + struct berval **out_vals = NULL; + struct mr_private *mrpriv = NULL; +- Slapi_Value **in_vals_sv = NULL; + Slapi_Value **out_vals_sv = NULL; + +- slapi_pblock_get(pb, SLAPI_PLUGIN_MR_VALUES, &in_vals); /* get bervals */ +- /* convert bervals to sv ary */ +- valuearray_init_bervalarray(in_vals, &in_vals_sv); +- slapi_pblock_set(pb, SLAPI_PLUGIN_MR_VALUES, in_vals_sv); /* use sv */ + rc = mr_wrap_mr_index_sv_fn(pb); +- /* clean up in_vals_sv */ +- valuearray_free(&in_vals_sv); +- /* restore old in_vals */ +- slapi_pblock_set(pb, SLAPI_PLUGIN_MR_VALUES, in_vals); + /* get result sv keys */ + slapi_pblock_get(pb, SLAPI_PLUGIN_MR_KEYS, &out_vals_sv); + /* convert to bvec */ +diff --git a/src/lib389/lib389/backend.py b/src/lib389/lib389/backend.py +index 9acced205..cee073ea7 100644 +--- a/src/lib389/lib389/backend.py ++++ b/src/lib389/lib389/backend.py +@@ -1029,6 +1029,16 @@ class Backends(DSLdapObjects): + for be in sorted(self.list(), key=lambda be: len(be.get_suffix()), reverse=True): + be.delete() + ++ def get_backend(self, suffix): ++ """ ++ Return the backend associated with the provided suffix. ++ """ ++ suffix_l = suffix.lower() ++ for be in self.list(): ++ if be.get_attr_val_utf8_l('nsslapd-suffix') == suffix_l: ++ return be ++ return None ++ + + class DatabaseConfig(DSLdapObject): + """Backend Database configuration +-- +2.48.1 + diff --git a/0026-Issue-6004-idletimeout-may-be-ignored-6005.patch b/0026-Issue-6004-idletimeout-may-be-ignored-6005.patch new file mode 100644 index 0000000..b7e49b6 --- /dev/null +++ b/0026-Issue-6004-idletimeout-may-be-ignored-6005.patch @@ -0,0 +1,230 @@ +From bd2829d04491556c35a0b36b591c09a69baf6546 Mon Sep 17 00:00:00 2001 +From: progier389 +Date: Mon, 11 Dec 2023 11:58:40 +0100 +Subject: [PATCH] Issue 6004 - idletimeout may be ignored (#6005) + +* Issue 6004 - idletimeout may be ignored + +Problem: idletimeout is still not handled when binding as non root (unless there are some activity +on another connection) +Fix: +Add a slapi_eq_repeat_rel handler that walks all active connection every seconds and check if the timeout is expired. +Note about CI test: +Notice that idletimeout is never enforced for connections bound as root (i.e cn=directory manager). + +Issue #6004 + +Reviewed by: @droideck, @tbordaz (Thanks!) + +(cherry picked from commit 86b5969acbe124eec8c89bcf1ab2156b2b140c17) +(cherry picked from commit bdb0a72b4953678e5418406b3c202dfa2c7469a2) +(cherry picked from commit 61cebc191cd4090072dda691b9956dbde4cf7c48) +--- + .../tests/suites/config/regression_test.py | 82 ++++++++++++++++++- + ldap/servers/slapd/daemon.c | 52 +++++++++++- + 2 files changed, 128 insertions(+), 6 deletions(-) + +diff --git a/dirsrvtests/tests/suites/config/regression_test.py b/dirsrvtests/tests/suites/config/regression_test.py +index 0000dd82d..8dbba8cd2 100644 +--- a/dirsrvtests/tests/suites/config/regression_test.py ++++ b/dirsrvtests/tests/suites/config/regression_test.py +@@ -6,20 +6,49 @@ + # See LICENSE for details. + # --- END COPYRIGHT BLOCK --- + # ++import os + import logging + import pytest ++import time + from lib389.utils import * + from lib389.dseldif import DSEldif +-from lib389.config import LDBMConfig ++from lib389.config import BDB_LDBMConfig, LDBMConfig, Config + from lib389.backend import Backends + from lib389.topologies import topology_st as topo ++from lib389.idm.user import UserAccounts, TEST_USER_PROPERTIES ++from lib389._constants import DEFAULT_SUFFIX, PASSWORD, DN_DM + + pytestmark = pytest.mark.tier0 + + logging.getLogger(__name__).setLevel(logging.INFO) + log = logging.getLogger(__name__) + ++DEBUGGING = os.getenv("DEBUGGING", default=False) + CUSTOM_MEM = '9100100100' ++IDLETIMEOUT = 5 ++DN_TEST_USER = f'uid={TEST_USER_PROPERTIES["uid"]},ou=People,{DEFAULT_SUFFIX}' ++ ++ ++@pytest.fixture(scope="module") ++def idletimeout_topo(topo, request): ++ """Create an instance with a test user and set idletimeout""" ++ inst = topo.standalone ++ config = Config(inst) ++ ++ users = UserAccounts(inst, DEFAULT_SUFFIX) ++ user = users.create(properties={ ++ **TEST_USER_PROPERTIES, ++ 'userpassword' : PASSWORD, ++ }) ++ config.replace('nsslapd-idletimeout', str(IDLETIMEOUT)) ++ ++ def fin(): ++ if not DEBUGGING: ++ config.reset('nsslapd-idletimeout') ++ user.delete() ++ ++ request.addfinalizer(fin) ++ return topo + + + # Function to return value of available memory in kb +@@ -79,7 +108,7 @@ def test_maxbersize_repl(topo): + nsslapd-errorlog-logmaxdiskspace are set in certain order + + :id: 743e912c-2be4-4f5f-9c2a-93dcb18f51a0 +- :setup: MMR with two suppliers ++ :setup: Standalone Instance + :steps: + 1. Stop the instance + 2. Set nsslapd-errorlog-maxlogsize before/after +@@ -112,3 +141,52 @@ def test_maxbersize_repl(topo): + log.info("Assert no init_dse_file errors in the error log") + assert not inst.ds_error_log.match('.*ERR - init_dse_file.*') + ++ ++def test_bdb_config(topo): ++ """Check that bdb config entry exists ++ ++ :id: edbc6f54-7c98-11ee-b1c0-482ae39447e5 ++ :setup: standalone ++ :steps: ++ 1. Check that bdb config instance exists. ++ :expectedresults: ++ 1. Success ++ """ ++ ++ inst = topo.standalone ++ assert BDB_LDBMConfig(inst).exists() ++ ++ ++@pytest.mark.parametrize("dn,expected_result", [(DN_TEST_USER, True), (DN_DM, False)]) ++def test_idletimeout(idletimeout_topo, dn, expected_result): ++ """Check that bdb config entry exists ++ ++ :id: b20f2826-942a-11ee-827b-482ae39447e5 ++ :parametrized: yes ++ :setup: Standalone Instance with test user and idletimeout ++ :steps: ++ 1. Open new ldap connection ++ 2. Bind with the provided dn ++ 3. Wait longer than idletimeout ++ 4. Try to bind again the provided dn and check if ++ connection is closed or not. ++ 5. Check if result is the expected one. ++ :expectedresults: ++ 1. Success ++ 2. Success ++ 3. Success ++ 4. Success ++ 5. Success ++ """ ++ ++ inst = idletimeout_topo.standalone ++ ++ l = ldap.initialize(f'ldap://localhost:{inst.port}') ++ l.bind_s(dn, PASSWORD) ++ time.sleep(IDLETIMEOUT+1) ++ try: ++ l.bind_s(dn, PASSWORD) ++ result = False ++ except ldap.SERVER_DOWN: ++ result = True ++ assert expected_result == result +diff --git a/ldap/servers/slapd/daemon.c b/ldap/servers/slapd/daemon.c +index 57e07e5f5..6df109760 100644 +--- a/ldap/servers/slapd/daemon.c ++++ b/ldap/servers/slapd/daemon.c +@@ -68,6 +68,8 @@ + #define SLAPD_ACCEPT_WAKEUP_TIMER 250 + #endif + ++#define MILLISECONDS_PER_SECOND 1000 ++ + int slapd_wakeup_timer = SLAPD_WAKEUP_TIMER; /* time in ms to wakeup */ + int slapd_accept_wakeup_timer = SLAPD_ACCEPT_WAKEUP_TIMER; /* time in ms to wakeup */ + #ifdef notdef /* GGOODREPL */ +@@ -1045,6 +1047,48 @@ slapd_sockets_ports_free(daemon_ports_t *ports_info) + #endif + } + ++/* ++ * Tells if idle timeout has expired ++ */ ++static inline int __attribute__((always_inline)) ++has_idletimeout_expired(Connection *c, time_t curtime) ++{ ++ return (c->c_state != CONN_STATE_FREE && !c->c_gettingber && ++ c->c_idletimeout > 0 && NULL == c->c_ops && ++ curtime - c->c_idlesince >= c->c_idletimeout); ++} ++ ++/* ++ * slapi_eq_repeat_rel callback that checks that idletimeout has not expired. ++ */ ++void ++check_idletimeout(time_t when __attribute__((unused)), void *arg __attribute__((unused)) ) ++{ ++ Connection_Table *ct = the_connection_table; ++ time_t curtime = slapi_current_rel_time_t(); ++ /* Walk all active connections of all connection listeners */ ++ for (int list_num = 0; list_num < ct->list_num; list_num++) { ++ for (Connection *c = connection_table_get_first_active_connection(ct, list_num); ++ c != NULL; c = connection_table_get_next_active_connection(ct, c)) { ++ if (!has_idletimeout_expired(c, curtime)) { ++ continue; ++ } ++ /* Looks like idletimeout has expired, lets acquire the lock ++ * and double check. ++ */ ++ if (pthread_mutex_trylock(&(c->c_mutex)) == EBUSY) { ++ continue; ++ } ++ if (has_idletimeout_expired(c, curtime)) { ++ /* idle timeout has expired */ ++ disconnect_server_nomutex(c, c->c_connid, -1, ++ SLAPD_DISCONNECT_IDLE_TIMEOUT, ETIMEDOUT); ++ } ++ pthread_mutex_unlock(&(c->c_mutex)); ++ } ++ } ++} ++ + void + slapd_daemon(daemon_ports_t *ports) + { +@@ -1258,7 +1302,9 @@ slapd_daemon(daemon_ports_t *ports) + "MAINPID=%lu", + (unsigned long)getpid()); + #endif +- ++ slapi_eq_repeat_rel(check_idletimeout, NULL, ++ slapi_current_rel_time_t(), ++ MILLISECONDS_PER_SECOND); + /* The meat of the operation is in a loop on a call to select */ + while (!g_get_shutdown()) { + int select_return = 0; +@@ -1734,9 +1780,7 @@ handle_pr_read_ready(Connection_Table *ct, PRIntn num_poll __attribute__((unused + disconnect_server_nomutex(c, c->c_connid, -1, + SLAPD_DISCONNECT_POLL, EPIPE); + } +- } else if (c->c_idletimeout > 0 && +- (curtime - c->c_idlesince) >= c->c_idletimeout && +- NULL == c->c_ops) { ++ } else if (has_idletimeout_expired(c, curtime)) { + /* idle timeout */ + disconnect_server_nomutex(c, c->c_connid, -1, + SLAPD_DISCONNECT_IDLE_TIMEOUT, ETIMEDOUT); +-- +2.48.1 + diff --git a/0027-Issue-6004-2nd-idletimeout-may-be-ignored-6569.patch b/0027-Issue-6004-2nd-idletimeout-may-be-ignored-6569.patch new file mode 100644 index 0000000..807421a --- /dev/null +++ b/0027-Issue-6004-2nd-idletimeout-may-be-ignored-6569.patch @@ -0,0 +1,69 @@ +From e9fe6e074130406328b8e932a5c2efa814d190a0 Mon Sep 17 00:00:00 2001 +From: tbordaz +Date: Wed, 5 Feb 2025 09:41:30 +0100 +Subject: [PATCH] Issue 6004 - (2nd) idletimeout may be ignored (#6569) + +Problem: + multiple listener threads was implemented in 2.x and after + This is missing in 1.4.3 so the cherry pick should be adapted +Fix: + skip the loop with listeners + +Issue #6004 + +Reviewed by: Jamie Chapman (Thanks !) +--- + ldap/servers/slapd/daemon.c | 36 +++++++++++++++++------------------- + 1 file changed, 17 insertions(+), 19 deletions(-) + +diff --git a/ldap/servers/slapd/daemon.c b/ldap/servers/slapd/daemon.c +index 6df109760..bef75e4a3 100644 +--- a/ldap/servers/slapd/daemon.c ++++ b/ldap/servers/slapd/daemon.c +@@ -1066,26 +1066,24 @@ check_idletimeout(time_t when __attribute__((unused)), void *arg __attribute__(( + { + Connection_Table *ct = the_connection_table; + time_t curtime = slapi_current_rel_time_t(); +- /* Walk all active connections of all connection listeners */ +- for (int list_num = 0; list_num < ct->list_num; list_num++) { +- for (Connection *c = connection_table_get_first_active_connection(ct, list_num); +- c != NULL; c = connection_table_get_next_active_connection(ct, c)) { +- if (!has_idletimeout_expired(c, curtime)) { +- continue; +- } +- /* Looks like idletimeout has expired, lets acquire the lock +- * and double check. +- */ +- if (pthread_mutex_trylock(&(c->c_mutex)) == EBUSY) { +- continue; +- } +- if (has_idletimeout_expired(c, curtime)) { +- /* idle timeout has expired */ +- disconnect_server_nomutex(c, c->c_connid, -1, +- SLAPD_DISCONNECT_IDLE_TIMEOUT, ETIMEDOUT); +- } +- pthread_mutex_unlock(&(c->c_mutex)); ++ /* Walk all active connections */ ++ for (Connection *c = connection_table_get_first_active_connection(ct); ++ c != NULL; c = connection_table_get_next_active_connection(ct, c)) { ++ if (!has_idletimeout_expired(c, curtime)) { ++ continue; ++ } ++ /* Looks like idletimeout has expired, lets acquire the lock ++ * and double check. ++ */ ++ if (pthread_mutex_trylock(&(c->c_mutex)) == EBUSY) { ++ continue; ++ } ++ if (has_idletimeout_expired(c, curtime)) { ++ /* idle timeout has expired */ ++ disconnect_server_nomutex(c, c->c_connid, -1, ++ SLAPD_DISCONNECT_IDLE_TIMEOUT, ETIMEDOUT); + } ++ pthread_mutex_unlock(&(c->c_mutex)); + } + } + +-- +2.48.1 + diff --git a/0028-Issue-6485-Fix-double-free-in-USN-cleanup-task.patch b/0028-Issue-6485-Fix-double-free-in-USN-cleanup-task.patch new file mode 100644 index 0000000..25ba272 --- /dev/null +++ b/0028-Issue-6485-Fix-double-free-in-USN-cleanup-task.patch @@ -0,0 +1,52 @@ +From b2edc371c5ca4fd24ef469c64829c48824098e7f Mon Sep 17 00:00:00 2001 +From: Mark Reynolds +Date: Wed, 8 Jan 2025 12:57:52 -0500 +Subject: [PATCH] Issue 6485 - Fix double free in USN cleanup task + +Description: + +ASAN report shows double free of bind dn in the USN cleanup task data. The bind +dn was passed as a reference so it should never have to be freed by the cleanup +task. + +Relates: https://github.com/389ds/389-ds-base/issues/6485 + +Reviewed by: tbordaz(Thanks!) +--- + ldap/servers/plugins/usn/usn_cleanup.c | 6 ++---- + 1 file changed, 2 insertions(+), 4 deletions(-) + +diff --git a/ldap/servers/plugins/usn/usn_cleanup.c b/ldap/servers/plugins/usn/usn_cleanup.c +index bdb55e6b1..7eaf0f88f 100644 +--- a/ldap/servers/plugins/usn/usn_cleanup.c ++++ b/ldap/servers/plugins/usn/usn_cleanup.c +@@ -240,7 +240,7 @@ usn_cleanup_add(Slapi_PBlock *pb, + char *suffix = NULL; + char *backend = NULL; + char *maxusn = NULL; +- char *bind_dn; ++ char *bind_dn = NULL; + struct usn_cleanup_data *cleanup_data = NULL; + int rv = SLAPI_DSE_CALLBACK_OK; + Slapi_Task *task = NULL; +@@ -323,8 +323,7 @@ usn_cleanup_add(Slapi_PBlock *pb, + suffix = NULL; /* don't free in this function */ + cleanup_data->maxusn_to_delete = maxusn; + maxusn = NULL; /* don't free in this function */ +- cleanup_data->bind_dn = bind_dn; +- bind_dn = NULL; /* don't free in this function */ ++ cleanup_data->bind_dn = slapi_ch_strdup(bind_dn); + slapi_task_set_data(task, cleanup_data); + + /* start the USN tombstone cleanup task as a separate thread */ +@@ -363,7 +362,6 @@ usn_cleanup_task_destructor(Slapi_Task *task) + slapi_ch_free_string(&mydata->suffix); + slapi_ch_free_string(&mydata->maxusn_to_delete); + slapi_ch_free_string(&mydata->bind_dn); +- /* Need to cast to avoid a compiler warning */ + slapi_ch_free((void **)&mydata); + } + } +-- +2.48.1 + diff --git a/0029-Issue-6553-Update-concread-to-0.5.4-and-refactor-sta.patch b/0029-Issue-6553-Update-concread-to-0.5.4-and-refactor-sta.patch new file mode 100644 index 0000000..7b35d50 --- /dev/null +++ b/0029-Issue-6553-Update-concread-to-0.5.4-and-refactor-sta.patch @@ -0,0 +1,894 @@ +From c243d7aa593046a7037188f0bf060caa1865a1f8 Mon Sep 17 00:00:00 2001 +From: Simon Pichugin +Date: Wed, 19 Feb 2025 18:56:34 -0800 +Subject: [PATCH] Issue 6553 - Update concread to 0.5.4 and refactor statistics + tracking (#6607) + +Description: Implement new cache statistics tracking with atomic counters +and dedicated stats structs. +Update concread dependency to 0.5.4 for improved cache performance. +Add tests for cache statistics functionality. + +Fixes: https://github.com/389ds/389-ds-base/issues/6553 + +Reviewed by: @firstyear +--- + ldap/servers/slapd/dn.c | 4 +- + src/Cargo.lock | 272 +++++++++++++----------------- + src/librslapd/Cargo.toml | 5 +- + src/librslapd/src/cache.rs | 331 +++++++++++++++++++++++++++++++++---- + 4 files changed, 418 insertions(+), 194 deletions(-) + +diff --git a/ldap/servers/slapd/dn.c b/ldap/servers/slapd/dn.c +index 518e091d5..469ba6a71 100644 +--- a/ldap/servers/slapd/dn.c ++++ b/ldap/servers/slapd/dn.c +@@ -101,7 +101,7 @@ struct ndn_cache { + + /* + * This means we need 1 MB minimum per thread +- * ++ * + */ + #define NDN_CACHE_MINIMUM_CAPACITY 1048576 + /* +@@ -3404,7 +3404,7 @@ ndn_cache_get_stats(uint64_t *hits, uint64_t *tries, uint64_t *size, uint64_t *m + uint64_t freq_evicts; + uint64_t recent_evicts; + uint64_t p_weight; +- cache_char_stats(cache, ++ cache_char_stats(cache, + &reader_hits, + &reader_includes, + &write_hits, +diff --git a/src/Cargo.lock b/src/Cargo.lock +index 4667a17f1..908b5f639 100644 +--- a/src/Cargo.lock ++++ b/src/Cargo.lock +@@ -19,23 +19,28 @@ checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" + + [[package]] + name = "ahash" +-version = "0.7.7" ++version = "0.8.11" + source = "registry+https://github.com/rust-lang/crates.io-index" +-checksum = "5a824f2aa7e75a0c98c5a504fceb80649e9c35265d44525b5f94de4771a395cd" ++checksum = "e89da841a80418a9b391ebaea17f5c112ffaaa96f621d2c285b5174da76b9011" + dependencies = [ +- "getrandom", ++ "cfg-if", ++ "getrandom 0.2.15", + "once_cell", + "version_check", ++ "zerocopy", + ] + + [[package]] +-name = "ansi_term" +-version = "0.12.1" ++name = "allocator-api2" ++version = "0.2.21" + source = "registry+https://github.com/rust-lang/crates.io-index" +-checksum = "d52a9bb7ec0cf484c551830a7ce27bd20d67eac647e1befb56b0be4ee39a55d2" +-dependencies = [ +- "winapi", +-] ++checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" ++ ++[[package]] ++name = "arc-swap" ++version = "1.7.1" ++source = "registry+https://github.com/rust-lang/crates.io-index" ++checksum = "69f7f8c3906b62b754cd5326047894316021dcfe5a194c8ea52bdd94934a3457" + + [[package]] + name = "atty" +@@ -143,51 +148,20 @@ dependencies = [ + + [[package]] + name = "concread" +-version = "0.2.21" ++version = "0.5.4" + source = "registry+https://github.com/rust-lang/crates.io-index" +-checksum = "dcc9816f5ac93ebd51c37f7f9a6bf2b40dfcd42978ad2aea5d542016e9244cf6" ++checksum = "0a06c26e76cd1d7a88a44324d0cf18b11589be552e97af09bee345f7e7334c6d" + dependencies = [ + "ahash", +- "crossbeam", ++ "arc-swap", + "crossbeam-epoch", ++ "crossbeam-queue", + "crossbeam-utils", + "lru", +- "parking_lot", +- "rand", + "smallvec", ++ "sptr", + "tokio", +-] +- +-[[package]] +-name = "crossbeam" +-version = "0.8.4" +-source = "registry+https://github.com/rust-lang/crates.io-index" +-checksum = "1137cd7e7fc0fb5d3c5a8678be38ec56e819125d8d7907411fe24ccb943faca8" +-dependencies = [ +- "crossbeam-channel", +- "crossbeam-deque", +- "crossbeam-epoch", +- "crossbeam-queue", +- "crossbeam-utils", +-] +- +-[[package]] +-name = "crossbeam-channel" +-version = "0.5.11" +-source = "registry+https://github.com/rust-lang/crates.io-index" +-checksum = "176dc175b78f56c0f321911d9c8eb2b77a78a4860b9c19db83835fea1a46649b" +-dependencies = [ +- "crossbeam-utils", +-] +- +-[[package]] +-name = "crossbeam-deque" +-version = "0.8.5" +-source = "registry+https://github.com/rust-lang/crates.io-index" +-checksum = "613f8cc01fe9cf1a3eb3d7f488fd2fa8388403e97039e2f73692932e291a770d" +-dependencies = [ +- "crossbeam-epoch", +- "crossbeam-utils", ++ "tracing", + ] + + [[package]] +@@ -236,6 +210,12 @@ dependencies = [ + "uuid", + ] + ++[[package]] ++name = "equivalent" ++version = "1.0.2" ++source = "registry+https://github.com/rust-lang/crates.io-index" ++checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" ++ + [[package]] + name = "errno" + version = "0.3.8" +@@ -265,6 +245,12 @@ dependencies = [ + "zeroize", + ] + ++[[package]] ++name = "foldhash" ++version = "0.1.4" ++source = "registry+https://github.com/rust-lang/crates.io-index" ++checksum = "a0d2fde1f7b3d48b8395d5f2de76c18a528bd6a9cdde438df747bfcba3e05d6f" ++ + [[package]] + name = "foreign-types" + version = "0.3.2" +@@ -302,8 +288,16 @@ name = "hashbrown" + version = "0.12.3" + source = "registry+https://github.com/rust-lang/crates.io-index" + checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" ++ ++[[package]] ++name = "hashbrown" ++version = "0.15.2" ++source = "registry+https://github.com/rust-lang/crates.io-index" ++checksum = "bf151400ff0baff5465007dd2f3e717f3fe502074ca563069ce3a6629d07b289" + dependencies = [ +- "ahash", ++ "allocator-api2", ++ "equivalent", ++ "foldhash", + ] + + [[package]] +@@ -316,12 +310,13 @@ dependencies = [ + ] + + [[package]] +-name = "instant" +-version = "0.1.12" ++name = "indexmap" ++version = "1.9.3" + source = "registry+https://github.com/rust-lang/crates.io-index" +-checksum = "7a5bbe824c507c5da5956355e86a746d82e0e1464f65d862cc5e71da70e94b2c" ++checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" + dependencies = [ +- "cfg-if", ++ "autocfg", ++ "hashbrown 0.12.3", + ] + + [[package]] +@@ -370,16 +365,6 @@ version = "0.4.12" + source = "registry+https://github.com/rust-lang/crates.io-index" + checksum = "c4cd1a83af159aa67994778be9070f0ae1bd732942279cabb14f86f986a21456" + +-[[package]] +-name = "lock_api" +-version = "0.4.11" +-source = "registry+https://github.com/rust-lang/crates.io-index" +-checksum = "3c168f8615b12bc01f9c17e2eb0cc07dcae1940121185446edc3744920e8ef45" +-dependencies = [ +- "autocfg", +- "scopeguard", +-] +- + [[package]] + name = "log" + version = "0.4.20" +@@ -388,11 +373,11 @@ checksum = "b5e6163cb8c49088c2c36f57875e58ccd8c87c7427f7fbd50ea6710b2f3f2e8f" + + [[package]] + name = "lru" +-version = "0.7.8" ++version = "0.13.0" + source = "registry+https://github.com/rust-lang/crates.io-index" +-checksum = "e999beba7b6e8345721bd280141ed958096a2e4abdf74f67ff4ce49b4b54e47a" ++checksum = "227748d55f2f0ab4735d87fd623798cb6b664512fe979705f829c9f81c934465" + dependencies = [ +- "hashbrown", ++ "hashbrown 0.15.2", + ] + + [[package]] +@@ -464,29 +449,10 @@ dependencies = [ + ] + + [[package]] +-name = "parking_lot" +-version = "0.11.2" ++name = "os_str_bytes" ++version = "6.6.1" + source = "registry+https://github.com/rust-lang/crates.io-index" +-checksum = "7d17b78036a60663b797adeaee46f5c9dfebb86948d1255007a1d6be0271ff99" +-dependencies = [ +- "instant", +- "lock_api", +- "parking_lot_core", +-] +- +-[[package]] +-name = "parking_lot_core" +-version = "0.8.6" +-source = "registry+https://github.com/rust-lang/crates.io-index" +-checksum = "60a2cfe6f0ad2bfc16aefa463b497d5c7a5ecd44a23efa72aa342d90177356dc" +-dependencies = [ +- "cfg-if", +- "instant", +- "libc", +- "redox_syscall 0.2.16", +- "smallvec", +- "winapi", +-] ++checksum = "e2355d85b9a3786f481747ced0e0ff2ba35213a1f9bd406ed906554d7af805a1" + + [[package]] + name = "paste" +@@ -519,12 +485,6 @@ version = "0.3.28" + source = "registry+https://github.com/rust-lang/crates.io-index" + checksum = "69d3587f8a9e599cc7ec2c00e331f71c4e69a5f9a4b8a6efd5b07466b9736f9a" + +-[[package]] +-name = "ppv-lite86" +-version = "0.2.17" +-source = "registry+https://github.com/rust-lang/crates.io-index" +-checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" +- + [[package]] + name = "proc-macro-hack" + version = "0.5.20+deprecated" +@@ -562,54 +522,6 @@ dependencies = [ + "proc-macro2", + ] + +-[[package]] +-name = "rand" +-version = "0.8.5" +-source = "registry+https://github.com/rust-lang/crates.io-index" +-checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" +-dependencies = [ +- "libc", +- "rand_chacha", +- "rand_core", +-] +- +-[[package]] +-name = "rand_chacha" +-version = "0.3.1" +-source = "registry+https://github.com/rust-lang/crates.io-index" +-checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +-dependencies = [ +- "ppv-lite86", +- "rand_core", +-] +- +-[[package]] +-name = "rand_core" +-version = "0.6.4" +-source = "registry+https://github.com/rust-lang/crates.io-index" +-checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +-dependencies = [ +- "getrandom", +-] +- +-[[package]] +-name = "redox_syscall" +-version = "0.2.16" +-source = "registry+https://github.com/rust-lang/crates.io-index" +-checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a" +-dependencies = [ +- "bitflags 1.3.2", +-] +- +-[[package]] +-name = "redox_syscall" +-version = "0.4.1" +-source = "registry+https://github.com/rust-lang/crates.io-index" +-checksum = "4722d768eff46b75989dd134e5c353f0d6296e5aaa3132e776cbdb56be7731aa" +-dependencies = [ +- "bitflags 1.3.2", +-] +- + [[package]] + name = "rsds" + version = "0.1.0" +@@ -639,12 +551,6 @@ version = "1.0.16" + source = "registry+https://github.com/rust-lang/crates.io-index" + checksum = "f98d2aa92eebf49b69786be48e4477826b256916e84a57ff2a4f21923b48eb4c" + +-[[package]] +-name = "scopeguard" +-version = "1.2.0" +-source = "registry+https://github.com/rust-lang/crates.io-index" +-checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" +- + [[package]] + name = "serde" + version = "1.0.195" +@@ -698,6 +604,12 @@ version = "1.12.0" + source = "registry+https://github.com/rust-lang/crates.io-index" + checksum = "2593d31f82ead8df961d8bd23a64c2ccf2eb5dd34b0a34bfb4dd54011c72009e" + ++[[package]] ++name = "sptr" ++version = "0.3.2" ++source = "registry+https://github.com/rust-lang/crates.io-index" ++checksum = "3b9b39299b249ad65f3b7e96443bad61c02ca5cd3589f46cb6d610a0fd6c0d6a" ++ + [[package]] + name = "strsim" + version = "0.8.0" +@@ -756,27 +668,46 @@ checksum = "c89b4efa943be685f629b149f53829423f8f5531ea21249408e8e2f8671ec104" + dependencies = [ + "backtrace", + "pin-project-lite", +- "tokio-macros", + ] + + [[package]] +-name = "tokio-macros" +-version = "2.2.0" ++name = "toml" ++version = "0.5.11" + source = "registry+https://github.com/rust-lang/crates.io-index" +-checksum = "5b8a1e28f2deaa14e508979454cb3a223b10b938b45af148bc0986de36f1923b" ++checksum = "f4f7f0dd8d50a853a531c426359045b1998f04219d88799810762cd4ad314234" ++dependencies = [ ++ "serde", ++] ++ ++[[package]] ++name = "tracing" ++version = "0.1.41" ++source = "registry+https://github.com/rust-lang/crates.io-index" ++checksum = "784e0ac535deb450455cbfa28a6f0df145ea1bb7ae51b821cf5e7927fdcfbdd0" ++dependencies = [ ++ "pin-project-lite", ++ "tracing-attributes", ++ "tracing-core", ++] ++ ++[[package]] ++name = "tracing-attributes" ++version = "0.1.28" ++source = "registry+https://github.com/rust-lang/crates.io-index" ++checksum = "395ae124c09f9e6918a2310af6038fba074bcf474ac352496d5910dd59a2226d" + dependencies = [ + "proc-macro2", + "quote", +- "syn 2.0.48", ++ "syn 2.0.98", + ] + + [[package]] +-name = "toml" +-version = "0.5.11" ++name = "tracing-core" ++version = "0.1.33" + source = "registry+https://github.com/rust-lang/crates.io-index" +-checksum = "f4f7f0dd8d50a853a531c426359045b1998f04219d88799810762cd4ad314234" ++checksum = "e672c95779cf947c5311f83787af4fa8fffd12fb27e4993211a84bdfd9610f9c" + dependencies = [ +- "serde", ++ "once_cell", + ] + + [[package]] +@@ -910,7 +841,36 @@ checksum = "1a657e1e9d3f514745a572a6846d3c7aa7dbe1658c056ed9c3344c4109a6949e" + name = "windows_x86_64_msvc" + version = "0.52.0" + source = "registry+https://github.com/rust-lang/crates.io-index" +-checksum = "dff9641d1cd4be8d1a070daf9e3773c5f67e78b4d9d42263020c057706765c04" ++checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" ++ ++[[package]] ++name = "wit-bindgen-rt" ++version = "0.33.0" ++source = "registry+https://github.com/rust-lang/crates.io-index" ++checksum = "3268f3d866458b787f390cf61f4bbb563b922d091359f9608842999eaee3943c" ++dependencies = [ ++ "bitflags 2.8.0", ++] ++ ++[[package]] ++name = "zerocopy" ++version = "0.7.35" ++source = "registry+https://github.com/rust-lang/crates.io-index" ++checksum = "1b9b4fd18abc82b8136838da5d50bae7bdea537c574d8dc1a34ed098d6c166f0" ++dependencies = [ ++ "zerocopy-derive", ++] ++ ++[[package]] ++name = "zerocopy-derive" ++version = "0.7.35" ++source = "registry+https://github.com/rust-lang/crates.io-index" ++checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e" ++dependencies = [ ++ "proc-macro2", ++ "quote", ++ "syn 2.0.98", ++] + + [[package]] + name = "zeroize" +diff --git a/src/librslapd/Cargo.toml b/src/librslapd/Cargo.toml +index fb445c251..6d9b621fc 100644 +--- a/src/librslapd/Cargo.toml ++++ b/src/librslapd/Cargo.toml +@@ -16,8 +16,7 @@ crate-type = ["staticlib", "lib"] + [dependencies] + slapd = { path = "../slapd" } + libc = "0.2" +-concread = "^0.2.20" ++concread = "0.5.4" + + [build-dependencies] +-cbindgen = "0.9" +- ++cbindgen = "0.26" +diff --git a/src/librslapd/src/cache.rs b/src/librslapd/src/cache.rs +index b025c830a..e3c692865 100644 +--- a/src/librslapd/src/cache.rs ++++ b/src/librslapd/src/cache.rs +@@ -1,38 +1,171 @@ + // This exposes C-FFI capable bindings for the concread concurrently readable cache. ++use concread::arcache::stats::{ARCacheWriteStat, ReadCountStat}; + use concread::arcache::{ARCache, ARCacheBuilder, ARCacheReadTxn, ARCacheWriteTxn}; +-use std::convert::TryInto; ++use concread::cowcell::CowCell; + use std::ffi::{CStr, CString}; + use std::os::raw::c_char; + ++#[derive(Clone, Debug, Default)] ++struct CacheStats { ++ reader_hits: u64, // Hits from read transactions (main + local) ++ reader_includes: u64, // Number of includes from read transactions ++ write_hits: u64, // Hits from write transactions ++ write_inc_or_mod: u64, // Number of includes/modifications from write transactions ++ freq_evicts: u64, // Number of evictions from frequent set ++ recent_evicts: u64, // Number of evictions from recent set ++ p_weight: u64, // Current cache weight between recent and frequent. ++ shared_max: u64, // Maximum number of items in the shared cache. ++ freq: u64, // Number of items in the frequent set at this point in time. ++ recent: u64, // Number of items in the recent set at this point in time. ++ all_seen_keys: u64, // Number of total keys seen through the cache's lifetime. ++} ++ ++impl CacheStats { ++ fn new() -> Self { ++ CacheStats::default() ++ } ++ ++ fn update_from_read_stat(&mut self, stat: ReadCountStat) { ++ self.reader_hits += stat.main_hit + stat.local_hit; ++ self.reader_includes += stat.include + stat.local_include; ++ } ++ ++ fn update_from_write_stat(&mut self, stat: &FFIWriteStat) { ++ self.write_hits += stat.read_hits; ++ self.write_inc_or_mod += stat.includes + stat.modifications; ++ self.freq_evicts += stat.freq_evictions; ++ self.recent_evicts += stat.recent_evictions; ++ self.p_weight = stat.p_weight; ++ self.shared_max = stat.shared_max; ++ self.freq = stat.freq; ++ self.recent = stat.recent; ++ self.all_seen_keys = stat.all_seen_keys; ++ } ++} ++ ++#[derive(Debug, Default)] ++pub struct FFIWriteStat { ++ pub read_ops: u64, ++ pub read_hits: u64, ++ pub p_weight: u64, ++ pub shared_max: u64, ++ pub freq: u64, ++ pub recent: u64, ++ pub all_seen_keys: u64, ++ pub includes: u64, ++ pub modifications: u64, ++ pub freq_evictions: u64, ++ pub recent_evictions: u64, ++ pub ghost_freq_revives: u64, ++ pub ghost_rec_revives: u64, ++ pub haunted_includes: u64, ++} ++ ++impl ARCacheWriteStat for FFIWriteStat { ++ fn cache_clear(&mut self) { ++ self.read_ops = 0; ++ self.read_hits = 0; ++ } ++ ++ fn cache_read(&mut self) { ++ self.read_ops += 1; ++ } ++ ++ fn cache_hit(&mut self) { ++ self.read_hits += 1; ++ } ++ ++ fn p_weight(&mut self, p: u64) { ++ self.p_weight = p; ++ } ++ ++ fn shared_max(&mut self, i: u64) { ++ self.shared_max = i; ++ } ++ ++ fn freq(&mut self, i: u64) { ++ self.freq = i; ++ } ++ ++ fn recent(&mut self, i: u64) { ++ self.recent = i; ++ } ++ ++ fn all_seen_keys(&mut self, i: u64) { ++ self.all_seen_keys = i; ++ } ++ ++ fn include(&mut self, _k: &K) { ++ self.includes += 1; ++ } ++ ++ fn include_haunted(&mut self, _k: &K) { ++ self.haunted_includes += 1; ++ } ++ ++ fn modify(&mut self, _k: &K) { ++ self.modifications += 1; ++ } ++ ++ fn ghost_frequent_revive(&mut self, _k: &K) { ++ self.ghost_freq_revives += 1; ++ } ++ ++ fn ghost_recent_revive(&mut self, _k: &K) { ++ self.ghost_rec_revives += 1; ++ } ++ ++ fn evict_from_recent(&mut self, _k: &K) { ++ self.recent_evictions += 1; ++ } ++ ++ fn evict_from_frequent(&mut self, _k: &K) { ++ self.freq_evictions += 1; ++ } ++} ++ + pub struct ARCacheChar { + inner: ARCache, ++ stats: CowCell, + } + + pub struct ARCacheCharRead<'a> { +- inner: ARCacheReadTxn<'a, CString, CString>, ++ inner: ARCacheReadTxn<'a, CString, CString, ReadCountStat>, ++ cache: &'a ARCacheChar, + } + + pub struct ARCacheCharWrite<'a> { +- inner: ARCacheWriteTxn<'a, CString, CString>, ++ inner: ARCacheWriteTxn<'a, CString, CString, FFIWriteStat>, ++ cache: &'a ARCacheChar, ++} ++ ++impl ARCacheChar { ++ fn new(max: usize, read_max: usize) -> Option { ++ ARCacheBuilder::new() ++ .set_size(max, read_max) ++ .set_reader_quiesce(false) ++ .build() ++ .map(|inner| Self { ++ inner, ++ stats: CowCell::new(CacheStats::new()), ++ }) ++ } + } + + #[no_mangle] + pub extern "C" fn cache_char_create(max: usize, read_max: usize) -> *mut ARCacheChar { +- let inner = if let Some(cache) = ARCacheBuilder::new().set_size(max, read_max).build() { +- cache ++ if let Some(cache) = ARCacheChar::new(max, read_max) { ++ Box::into_raw(Box::new(cache)) + } else { +- return std::ptr::null_mut(); +- }; +- let cache: Box = Box::new(ARCacheChar { inner }); +- Box::into_raw(cache) ++ std::ptr::null_mut() ++ } + } + + #[no_mangle] + pub extern "C" fn cache_char_free(cache: *mut ARCacheChar) { +- // Should we be responsible to drain and free everything? + debug_assert!(!cache.is_null()); + unsafe { +- let _drop = Box::from_raw(cache); ++ drop(Box::from_raw(cache)); + } + } + +@@ -53,22 +186,22 @@ pub extern "C" fn cache_char_stats( + ) { + let cache_ref = unsafe { + debug_assert!(!cache.is_null()); +- &(*cache) as &ARCacheChar ++ &(*cache) + }; +- let stats = cache_ref.inner.view_stats(); +- *reader_hits = stats.reader_hits.try_into().unwrap(); +- *reader_includes = stats.reader_includes.try_into().unwrap(); +- *write_hits = stats.write_hits.try_into().unwrap(); +- *write_inc_or_mod = (stats.write_includes + stats.write_modifies) +- .try_into() +- .unwrap(); +- *shared_max = stats.shared_max.try_into().unwrap(); +- *freq = stats.freq.try_into().unwrap(); +- *recent = stats.recent.try_into().unwrap(); +- *freq_evicts = stats.freq_evicts.try_into().unwrap(); +- *recent_evicts = stats.recent_evicts.try_into().unwrap(); +- *p_weight = stats.p_weight.try_into().unwrap(); +- *all_seen_keys = stats.all_seen_keys.try_into().unwrap(); ++ ++ // Get stats snapshot ++ let stats_read = cache_ref.stats.read(); ++ *reader_hits = stats_read.reader_hits; ++ *reader_includes = stats_read.reader_includes; ++ *write_hits = stats_read.write_hits; ++ *write_inc_or_mod = stats_read.write_inc_or_mod; ++ *freq_evicts = stats_read.freq_evicts; ++ *recent_evicts = stats_read.recent_evicts; ++ *p_weight = stats_read.p_weight; ++ *shared_max = stats_read.shared_max; ++ *freq = stats_read.freq; ++ *recent = stats_read.recent; ++ *all_seen_keys = stats_read.all_seen_keys; + } + + // start read +@@ -79,7 +212,8 @@ pub extern "C" fn cache_char_read_begin(cache: *mut ARCacheChar) -> *mut ARCache + &(*cache) as &ARCacheChar + }; + let read_txn = Box::new(ARCacheCharRead { +- inner: cache_ref.inner.read(), ++ inner: cache_ref.inner.read_stats(ReadCountStat::default()), ++ cache: cache_ref, + }); + Box::into_raw(read_txn) + } +@@ -87,8 +221,20 @@ pub extern "C" fn cache_char_read_begin(cache: *mut ARCacheChar) -> *mut ARCache + #[no_mangle] + pub extern "C" fn cache_char_read_complete(read_txn: *mut ARCacheCharRead) { + debug_assert!(!read_txn.is_null()); ++ + unsafe { +- let _drop = Box::from_raw(read_txn); ++ let read_txn_box = Box::from_raw(read_txn); ++ let read_stats = read_txn_box.inner.finish(); ++ let write_stats = read_txn_box ++ .cache ++ .inner ++ .try_quiesce_stats(FFIWriteStat::default()); ++ ++ // Update stats ++ let mut stats_write = read_txn_box.cache.stats.write(); ++ stats_write.update_from_read_stat(read_stats); ++ stats_write.update_from_write_stat(&write_stats); ++ stats_write.commit(); + } + } + +@@ -141,7 +287,8 @@ pub extern "C" fn cache_char_write_begin( + &(*cache) as &ARCacheChar + }; + let write_txn = Box::new(ARCacheCharWrite { +- inner: cache_ref.inner.write(), ++ inner: cache_ref.inner.write_stats(FFIWriteStat::default()), ++ cache: cache_ref, + }); + Box::into_raw(write_txn) + } +@@ -149,15 +296,21 @@ pub extern "C" fn cache_char_write_begin( + #[no_mangle] + pub extern "C" fn cache_char_write_commit(write_txn: *mut ARCacheCharWrite) { + debug_assert!(!write_txn.is_null()); +- let wr = unsafe { Box::from_raw(write_txn) }; +- (*wr).inner.commit(); ++ unsafe { ++ let write_txn_box = Box::from_raw(write_txn); ++ let current_stats = write_txn_box.inner.commit(); ++ ++ let mut stats_write = write_txn_box.cache.stats.write(); ++ stats_write.update_from_write_stat(¤t_stats); ++ stats_write.commit(); ++ } + } + + #[no_mangle] + pub extern "C" fn cache_char_write_rollback(write_txn: *mut ARCacheCharWrite) { + debug_assert!(!write_txn.is_null()); + unsafe { +- let _drop = Box::from_raw(write_txn); ++ drop(Box::from_raw(write_txn)); + } + } + +@@ -182,7 +335,7 @@ pub extern "C" fn cache_char_write_include( + + #[cfg(test)] + mod tests { +- use crate::cache::*; ++ use super::*; + + #[test] + fn test_cache_basic() { +@@ -199,4 +352,116 @@ mod tests { + cache_char_read_complete(read_txn); + cache_char_free(cache_ptr); + } ++ ++ #[test] ++ fn test_cache_stats() { ++ let cache = cache_char_create(100, 8); ++ ++ // Variables to store stats ++ let mut reader_hits = 0; ++ let mut reader_includes = 0; ++ let mut write_hits = 0; ++ let mut write_inc_or_mod = 0; ++ let mut shared_max = 0; ++ let mut freq = 0; ++ let mut recent = 0; ++ let mut freq_evicts = 0; ++ let mut recent_evicts = 0; ++ let mut p_weight = 0; ++ let mut all_seen_keys = 0; ++ ++ // Do some operations ++ let key = CString::new("stats_test").unwrap(); ++ let value = CString::new("value").unwrap(); ++ ++ let write_txn = cache_char_write_begin(cache); ++ cache_char_write_include(write_txn, key.as_ptr(), value.as_ptr()); ++ cache_char_write_commit(write_txn); ++ ++ let read_txn = cache_char_read_begin(cache); ++ let _ = cache_char_read_get(read_txn, key.as_ptr()); ++ cache_char_read_complete(read_txn); ++ ++ // Get stats ++ cache_char_stats( ++ cache, ++ &mut reader_hits, ++ &mut reader_includes, ++ &mut write_hits, ++ &mut write_inc_or_mod, ++ &mut shared_max, ++ &mut freq, ++ &mut recent, ++ &mut freq_evicts, ++ &mut recent_evicts, ++ &mut p_weight, ++ &mut all_seen_keys, ++ ); ++ ++ // Verify that stats were updated ++ assert!(write_inc_or_mod > 0); ++ assert!(all_seen_keys > 0); ++ ++ cache_char_free(cache); ++ } ++ ++ #[test] ++ fn test_cache_read_write_operations() { ++ let cache = cache_char_create(100, 8); ++ ++ // Create test data ++ let key = CString::new("test_key").unwrap(); ++ let value = CString::new("test_value").unwrap(); ++ ++ // Test write operation ++ let write_txn = cache_char_write_begin(cache); ++ cache_char_write_include(write_txn, key.as_ptr(), value.as_ptr()); ++ cache_char_write_commit(write_txn); ++ ++ // Test read operation ++ let read_txn = cache_char_read_begin(cache); ++ let result = cache_char_read_get(read_txn, key.as_ptr()); ++ assert!(!result.is_null()); ++ ++ // Verify the value ++ let retrieved_value = unsafe { CStr::from_ptr(result) }; ++ assert_eq!(retrieved_value.to_bytes(), value.as_bytes()); ++ ++ cache_char_read_complete(read_txn); ++ cache_char_free(cache); ++ } ++ ++ #[test] ++ fn test_cache_miss() { ++ let cache = cache_char_create(100, 8); ++ let read_txn = cache_char_read_begin(cache); ++ ++ let missing_key = CString::new("nonexistent").unwrap(); ++ let result = cache_char_read_get(read_txn, missing_key.as_ptr()); ++ assert!(result.is_null()); ++ ++ cache_char_read_complete(read_txn); ++ cache_char_free(cache); ++ } ++ ++ #[test] ++ fn test_write_rollback() { ++ let cache = cache_char_create(100, 8); ++ ++ let key = CString::new("rollback_test").unwrap(); ++ let value = CString::new("value").unwrap(); ++ ++ // Start write transaction and rollback ++ let write_txn = cache_char_write_begin(cache); ++ cache_char_write_include(write_txn, key.as_ptr(), value.as_ptr()); ++ cache_char_write_rollback(write_txn); ++ ++ // Verify key doesn't exist ++ let read_txn = cache_char_read_begin(cache); ++ let result = cache_char_read_get(read_txn, key.as_ptr()); ++ assert!(result.is_null()); ++ ++ cache_char_read_complete(read_txn); ++ cache_char_free(cache); ++ } + } +-- +2.48.1 + diff --git a/0030-Issue-5841-dsconf-incorrectly-setting-up-Pass-Throug.patch b/0030-Issue-5841-dsconf-incorrectly-setting-up-Pass-Throug.patch new file mode 100644 index 0000000..fc37f73 --- /dev/null +++ b/0030-Issue-5841-dsconf-incorrectly-setting-up-Pass-Throug.patch @@ -0,0 +1,38 @@ +From 679262c0c292413851d2d004b588ecfd7d91c85a Mon Sep 17 00:00:00 2001 +From: James Chapman +Date: Tue, 11 Feb 2025 18:06:34 +0000 +Subject: [PATCH] Issue 5841 - dsconf incorrectly setting up Pass-Through + Authentication (#6601) + +Bug description: +During init, PAMPassThroughAuthConfigs defines an "objectclass=nsslapdplugin" +plugin object. During filter creation, dsconf fails as objectclass=nsslapdplugin +is not present in the PAM PT config entry. This objectclass has been removed in +all other branches, branch 1.4.3 was skipped as there are cherry pick conflicts. + +Fix description: +Remove nsslapdplugin from the plugin objecti, objectclass list. + +Fixes: https://github.com/389ds/389-ds-base/issues/5841 + +Reviewed by: @progier389 (Thank you) +--- + src/lib389/lib389/plugins.py | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/src/lib389/lib389/plugins.py b/src/lib389/lib389/plugins.py +index 185398e5b..25b49dae4 100644 +--- a/src/lib389/lib389/plugins.py ++++ b/src/lib389/lib389/plugins.py +@@ -1579,7 +1579,7 @@ class PAMPassThroughAuthConfigs(DSLdapObjects): + + def __init__(self, instance, basedn="cn=PAM Pass Through Auth,cn=plugins,cn=config"): + super(PAMPassThroughAuthConfigs, self).__init__(instance) +- self._objectclasses = ['top', 'extensibleObject', 'nsslapdplugin', 'pamConfig'] ++ self._objectclasses = ['top', 'extensibleObject', 'pamConfig'] + self._filterattrs = ['cn'] + self._scope = ldap.SCOPE_ONELEVEL + self._childobject = PAMPassThroughAuthConfig +-- +2.48.1 + diff --git a/0031-Issue-6067-Add-hidden-v-and-j-options-to-each-CLI-su.patch b/0031-Issue-6067-Add-hidden-v-and-j-options-to-each-CLI-su.patch new file mode 100644 index 0000000..43c2920 --- /dev/null +++ b/0031-Issue-6067-Add-hidden-v-and-j-options-to-each-CLI-su.patch @@ -0,0 +1,3275 @@ +From eee2876a2a80da1b3fc8911e3cea6448f26dcf06 Mon Sep 17 00:00:00 2001 +From: Simon Pichugin +Date: Wed, 14 Feb 2024 10:37:21 -0800 +Subject: [PATCH] Issue 6067 - Add hidden -v and -j options to each CLI + subcommand (#6088) + +Description: There is no [-v] option before instance_name mentioned, +so user will not know he can use it unless he runs "dsctl -h". +Add a custom HelpFormatter to each subcommand. The formatter_class adds +[-v] [-j] to the usage line and adds the options' description to the full help output. + +Related: https://github.com/389ds/389-ds-base/issues/6067 + +Reviewed by: @vashirov (Thanks!) +--- + src/lib389/cli/dsconf | 11 +- + src/lib389/cli/dsctl | 14 +-- + src/lib389/cli/dsidm | 11 +- + src/lib389/lib389/cli_base/__init__.py | 37 +++++++ + src/lib389/lib389/cli_conf/__init__.py | 9 +- + src/lib389/lib389/cli_conf/backend.py | 67 ++++++------ + src/lib389/lib389/cli_conf/backup.py | 7 +- + src/lib389/lib389/cli_conf/chaining.py | 19 ++-- + src/lib389/lib389/cli_conf/config.py | 11 +- + src/lib389/lib389/cli_conf/conflicts.py | 17 +-- + .../lib389/cli_conf/directory_manager.py | 6 +- + src/lib389/lib389/cli_conf/monitor.py | 18 ++-- + src/lib389/lib389/cli_conf/plugin.py | 9 +- + .../lib389/cli_conf/plugins/accountpolicy.py | 15 +-- + .../lib389/cli_conf/plugins/attruniq.py | 19 ++-- + .../lib389/cli_conf/plugins/automember.py | 35 +++--- + src/lib389/lib389/cli_conf/plugins/dna.py | 27 ++--- + .../lib389/cli_conf/plugins/entryuuid.py | 7 +- + .../cli_conf/plugins/ldappassthrough.py | 12 +-- + .../lib389/cli_conf/plugins/linkedattr.py | 19 ++-- + .../lib389/cli_conf/plugins/managedentries.py | 27 ++--- + .../lib389/cli_conf/plugins/memberof.py | 19 ++-- + .../lib389/cli_conf/plugins/pampassthrough.py | 14 +-- + .../lib389/cli_conf/plugins/posix_winsync.py | 7 +- + .../lib389/cli_conf/plugins/referint.py | 13 +-- + .../lib389/cli_conf/plugins/retrochangelog.py | 9 +- + .../lib389/cli_conf/plugins/rootdn_ac.py | 5 +- + src/lib389/lib389/cli_conf/plugins/usn.py | 11 +- + src/lib389/lib389/cli_conf/pwpolicy.py | 21 ++-- + src/lib389/lib389/cli_conf/replication.py | 100 +++++++++--------- + src/lib389/lib389/cli_conf/saslmappings.py | 15 +-- + src/lib389/lib389/cli_conf/schema.py | 40 +++---- + src/lib389/lib389/cli_conf/security.py | 26 ++--- + src/lib389/lib389/cli_ctl/cockpit.py | 11 +- + src/lib389/lib389/cli_ctl/dbgen.py | 17 +-- + src/lib389/lib389/cli_ctl/dbtasks.py | 17 +-- + src/lib389/lib389/cli_ctl/dsrc.py | 11 +- + src/lib389/lib389/cli_ctl/health.py | 2 +- + src/lib389/lib389/cli_ctl/instance.py | 13 +-- + src/lib389/lib389/cli_ctl/nsstate.py | 1 + + src/lib389/lib389/cli_ctl/tls.py | 33 +++--- + src/lib389/lib389/cli_idm/account.py | 25 ++--- + src/lib389/lib389/cli_idm/client_config.py | 1 + + src/lib389/lib389/cli_idm/group.py | 22 ++-- + src/lib389/lib389/cli_idm/initialise.py | 3 +- + .../lib389/cli_idm/organizationalunit.py | 18 ++-- + src/lib389/lib389/cli_idm/posixgroup.py | 16 +-- + src/lib389/lib389/cli_idm/role.py | 29 ++--- + src/lib389/lib389/cli_idm/service.py | 18 ++-- + src/lib389/lib389/cli_idm/uniquegroup.py | 22 ++-- + src/lib389/lib389/cli_idm/user.py | 16 +-- + 51 files changed, 504 insertions(+), 448 deletions(-) + +diff --git a/src/lib389/cli/dsconf b/src/lib389/cli/dsconf +index 23068efdd..b8a980877 100755 +--- a/src/lib389/cli/dsconf ++++ b/src/lib389/cli/dsconf +@@ -33,16 +33,13 @@ from lib389.cli_base import disconnect_instance, connect_instance + from lib389.cli_base.dsrc import dsrc_to_ldap, dsrc_arg_concat + from lib389.cli_base import setup_script_logger + from lib389.cli_base import format_error_to_dict ++from lib389.cli_base import parent_argparser + from lib389.utils import instance_choices + +-parser = argparse.ArgumentParser(allow_abbrev=True) ++parser = argparse.ArgumentParser(allow_abbrev=True, parents=[parent_argparser]) + parser.add_argument('instance', + help="The name of the instance or its LDAP URL, such as ldap://server.example.com:389", + ).completer = instance_choices +-parser.add_argument('-v', '--verbose', +- help="Display verbose operation tracing during command execution", +- action='store_true', default=False +- ) + parser.add_argument('-D', '--binddn', + help="The account to bind as for executing operations", + default=None +@@ -67,10 +64,6 @@ parser.add_argument('-Z', '--starttls', + help="Connect with StartTLS", + default=False, action='store_true' + ) +-parser.add_argument('-j', '--json', +- help="Return result in JSON object", +- default=False, action='store_true' +- ) + + subparsers = parser.add_subparsers(help="resources to act upon") + +diff --git a/src/lib389/cli/dsctl b/src/lib389/cli/dsctl +index 2881d7ac2..1ae8bb5f4 100755 +--- a/src/lib389/cli/dsctl ++++ b/src/lib389/cli/dsctl +@@ -29,21 +29,15 @@ from lib389.cli_ctl.instance import instance_remove_all + from lib389.cli_base import ( + disconnect_instance, + setup_script_logger, +- format_error_to_dict) ++ format_error_to_dict, ++ parent_argparser ++ ) + from lib389._constants import DSRC_CONTAINER + +-parser = argparse.ArgumentParser() +-parser.add_argument('-v', '--verbose', +- help="Display verbose operation tracing during command execution", +- action='store_true', default=False +- ) ++parser = argparse.ArgumentParser(parents=[parent_argparser]) + parser.add_argument('instance', nargs='?', default=False, + help="The name of the instance to act upon", + ).completer = instance_choices +-parser.add_argument('-j', '--json', +- help="Return result in JSON object", +- default=False, action='store_true' +- ) + parser.add_argument('-l', '--list', + help="List available Directory Server instances", + default=False, action='store_true' +diff --git a/src/lib389/cli/dsidm b/src/lib389/cli/dsidm +index 1b6762646..1b739b103 100755 +--- a/src/lib389/cli/dsidm ++++ b/src/lib389/cli/dsidm +@@ -32,8 +32,9 @@ from lib389.cli_idm import service as cli_service + from lib389.cli_base import connect_instance, disconnect_instance, setup_script_logger + from lib389.cli_base.dsrc import dsrc_to_ldap, dsrc_arg_concat + from lib389.cli_base import format_error_to_dict ++from lib389.cli_base import parent_argparser + +-parser = argparse.ArgumentParser(allow_abbrev=True) ++parser = argparse.ArgumentParser(allow_abbrev=True, parents=[parent_argparser]) + # First, add the LDAP options + parser.add_argument('instance', + help="The name of the instance or its LDAP URL, such as ldap://server.example.com:389", +@@ -42,10 +43,6 @@ parser.add_argument('-b', '--basedn', + help="Base DN (root naming context) of the instance to manage", + default=None + ) +-parser.add_argument('-v', '--verbose', +- help="Display verbose operation tracing during command execution", +- action='store_true', default=False +- ) + parser.add_argument('-D', '--binddn', + help="The account to bind as for executing operations", + default=None +@@ -66,10 +63,6 @@ parser.add_argument('-Z', '--starttls', + help="Connect with StartTLS", + default=False, action='store_true' + ) +-parser.add_argument('-j', '--json', +- help="Return result in JSON object", +- default=False, action='store_true' +- ) + subparsers = parser.add_subparsers(help="resources to act upon") + # Call all the other cli modules to register their bits + cli_account.create_parser(subparsers) +diff --git a/src/lib389/lib389/cli_base/__init__.py b/src/lib389/lib389/cli_base/__init__.py +index 60dd6cd70..c431f066d 100644 +--- a/src/lib389/lib389/cli_base/__init__.py ++++ b/src/lib389/lib389/cli_base/__init__.py +@@ -7,6 +7,7 @@ + # See LICENSE for details. + # --- END COPYRIGHT BLOCK --- + ++import argparse + import ast + import logging + import sys +@@ -365,6 +366,42 @@ def _generic_modify_dn(inst, basedn, log, manager_class, dn, args=None): + _generic_modify_inner(log, o, args.changes) + + ++# We need parent parser to be able to add -v and -j to all subparsers ++# because we use parent_arguments in CustomHelpFormatter ++parent_arguments = [] ++parent_argparser = argparse.ArgumentParser(add_help=False) ++parent_arguments.append(parent_argparser.add_argument('-v', '--verbose', ++ help="Display verbose operation tracing during command execution", ++ action='store_true', default=False ++ )) ++parent_arguments.append(parent_argparser.add_argument('-j', '--json', ++ help="Return result in JSON object", ++ default=False, action='store_true' ++ )) ++ ++ ++class CustomHelpFormatter(argparse.HelpFormatter): ++ """Custom help formatter to add [-v] [-j] to the usage line and add these options' ++ description to the full help output ++ """ ++ def add_arguments(self, actions): ++ if len(actions) > 0 and actions[0].option_strings: ++ actions = parent_arguments + actions ++ super(CustomHelpFormatter, self).add_arguments(actions) ++ ++ def _format_usage(self, usage, actions, groups, prefix): ++ usage = super(CustomHelpFormatter, self)._format_usage(usage, actions, groups, prefix) ++ formatted_options = self._format_actions_usage(parent_arguments, []) ++ # If formatted_options already in usage - remove them ++ if formatted_options in usage: ++ usage = usage.replace(f' {formatted_options}', '') ++ usage = usage.split(' ') ++ usage.insert(2, formatted_options) ++ usage = ' '.join(usage) ++ ++ return usage ++ ++ + class LogCapture(logging.Handler): + """ + This useful class is for intercepting logs, and then making assertions about +diff --git a/src/lib389/lib389/cli_conf/__init__.py b/src/lib389/lib389/cli_conf/__init__.py +index ef9db9166..44f39ed2f 100644 +--- a/src/lib389/lib389/cli_conf/__init__.py ++++ b/src/lib389/lib389/cli_conf/__init__.py +@@ -7,6 +7,7 @@ + # --- END COPYRIGHT BLOCK --- + import ldap + from lib389 import ensure_list_str ++from lib389.cli_base import CustomHelpFormatter + + + def _args_to_attrs(args, arg_to_attr): +@@ -155,16 +156,16 @@ def generic_status(inst, basedn, log, args): + + + def add_generic_plugin_parsers(subparser, plugin_cls): +- show_parser = subparser.add_parser('show', help='Displays the plugin configuration') ++ show_parser = subparser.add_parser('show', help='Displays the plugin configuration', formatter_class=CustomHelpFormatter) + show_parser.set_defaults(func=generic_show, plugin_cls=plugin_cls) + +- enable_parser = subparser.add_parser('enable', help='Enables the plugin') ++ enable_parser = subparser.add_parser('enable', help='Enables the plugin', formatter_class=CustomHelpFormatter) + enable_parser.set_defaults(func=generic_enable, plugin_cls=plugin_cls) + +- disable_parser = subparser.add_parser('disable', help='Disables the plugin') ++ disable_parser = subparser.add_parser('disable', help='Disables the plugin', formatter_class=CustomHelpFormatter) + disable_parser.set_defaults(func=generic_disable, plugin_cls=plugin_cls) + +- status_parser = subparser.add_parser('status', help='Displays the plugin status') ++ status_parser = subparser.add_parser('status', help='Displays the plugin status', formatter_class=CustomHelpFormatter) + status_parser.set_defaults(func=generic_status, plugin_cls=plugin_cls) + + +diff --git a/src/lib389/lib389/cli_conf/backend.py b/src/lib389/lib389/cli_conf/backend.py +index 5bcc098c7..4dc67d563 100644 +--- a/src/lib389/lib389/cli_conf/backend.py ++++ b/src/lib389/lib389/cli_conf/backend.py +@@ -28,6 +28,7 @@ from lib389.cli_base import ( + _generic_get_dn, + _get_arg, + _warn, ++ CustomHelpFormatter + ) + import json + import ldap +@@ -834,39 +835,39 @@ def backend_compact(inst, basedn, log, args): + + + def create_parser(subparsers): +- backend_parser = subparsers.add_parser('backend', help="Manage database suffixes and backends") ++ backend_parser = subparsers.add_parser('backend', help="Manage database suffixes and backends", formatter_class=CustomHelpFormatter) + subcommands = backend_parser.add_subparsers(help="action") + + ##################################################### + # Suffix parser + ##################################################### +- suffix_parser = subcommands.add_parser('suffix', help="Manage backend suffixes") ++ suffix_parser = subcommands.add_parser('suffix', help="Manage backend suffixes", formatter_class=CustomHelpFormatter) + suffix_subcommands = suffix_parser.add_subparsers(help="action") + + # List backends/suffixes +- list_parser = suffix_subcommands.add_parser('list', help="List active backends and suffixes") ++ list_parser = suffix_subcommands.add_parser('list', help="List active backends and suffixes", formatter_class=CustomHelpFormatter) + list_parser.set_defaults(func=backend_list) + list_parser.add_argument('--suffix', action='store_true', help='Displays the suffixes without backend name') + list_parser.add_argument('--skip-subsuffixes', action='store_true', help='Displays the list of suffixes without sub-suffixes') + + # Get backend +- get_parser = suffix_subcommands.add_parser('get', help='Display the suffix entry') ++ get_parser = suffix_subcommands.add_parser('get', help='Display the suffix entry', formatter_class=CustomHelpFormatter) + get_parser.set_defaults(func=backend_get) + get_parser.add_argument('selector', nargs='?', help='The backend database name to search for') + + # Get the DN of a backend +- get_dn_parser = suffix_subcommands.add_parser('get-dn', help='Display the DN of a backend') ++ get_dn_parser = suffix_subcommands.add_parser('get-dn', help='Display the DN of a backend', formatter_class=CustomHelpFormatter) + get_dn_parser.set_defaults(func=backend_get_dn) + get_dn_parser.add_argument('dn', nargs='?', help='The DN to the database entry in cn=ldbm database,cn=plugins,cn=config') + + # Get subsuffixes +- get_subsuffix_parser = suffix_subcommands.add_parser('get-sub-suffixes', help='Display sub-suffixes') ++ get_subsuffix_parser = suffix_subcommands.add_parser('get-sub-suffixes', help='Display sub-suffixes', formatter_class=CustomHelpFormatter) + get_subsuffix_parser.set_defaults(func=backend_get_subsuffixes) + get_subsuffix_parser.add_argument('--suffix', action='store_true', help='Displays the list of suffixes without backend name') + get_subsuffix_parser.add_argument('be_name', help='The backend name or suffix') + + # Set the backend/suffix configuration +- set_backend_parser = suffix_subcommands.add_parser('set', help='Set configuration settings for a specific backend') ++ set_backend_parser = suffix_subcommands.add_parser('set', help='Set configuration settings for a specific backend', formatter_class=CustomHelpFormatter) + set_backend_parser.set_defaults(func=backend_set) + set_backend_parser.add_argument('--enable-readonly', action='store_true', help='Enables read-only mode for the backend database') + set_backend_parser.add_argument('--disable-readonly', action='store_true', help='Disables read-only mode for the backend database') +@@ -887,11 +888,11 @@ def create_parser(subparsers): + ######################################### + # Index parser + ######################################### +- index_parser = subcommands.add_parser('index', help="Manage backend indexes") ++ index_parser = subcommands.add_parser('index', help="Manage backend indexes", formatter_class=CustomHelpFormatter) + index_subcommands = index_parser.add_subparsers(help="action") + + # Create index +- add_index_parser = index_subcommands.add_parser('add', help='Add an index') ++ add_index_parser = index_subcommands.add_parser('add', help='Add an index', formatter_class=CustomHelpFormatter) + add_index_parser.set_defaults(func=backend_add_index) + add_index_parser.add_argument('--index-type', required=True, action='append', help='Sets the indexing type (eq, sub, pres, or approx)') + add_index_parser.add_argument('--matching-rule', action='append', help='Sets the matching rule for the index') +@@ -900,7 +901,7 @@ def create_parser(subparsers): + add_index_parser.add_argument('be_name', help='The backend name or suffix') + + # Edit index +- edit_index_parser = index_subcommands.add_parser('set', help='Update an index') ++ edit_index_parser = index_subcommands.add_parser('set', help='Update an index', formatter_class=CustomHelpFormatter) + edit_index_parser.set_defaults(func=backend_set_index) + edit_index_parser.add_argument('--attr', required=True, help='Sets the indexed attribute to update') + edit_index_parser.add_argument('--add-type', action='append', help='Adds an index type to the index (eq, sub, pres, or approx)') +@@ -911,25 +912,25 @@ def create_parser(subparsers): + edit_index_parser.add_argument('be_name', help='The backend name or suffix') + + # Get index +- get_index_parser = index_subcommands.add_parser('get', help='Display an index entry') ++ get_index_parser = index_subcommands.add_parser('get', help='Display an index entry', formatter_class=CustomHelpFormatter) + get_index_parser.set_defaults(func=backend_get_index) + get_index_parser.add_argument('--attr', required=True, action='append', help='Sets the index name to display') + get_index_parser.add_argument('be_name', help='The backend name or suffix') + + # list indexes +- list_index_parser = index_subcommands.add_parser('list', help='Display the index') ++ list_index_parser = index_subcommands.add_parser('list', help='Display the index', formatter_class=CustomHelpFormatter) + list_index_parser.set_defaults(func=backend_list_index) + list_index_parser.add_argument('--just-names', action='store_true', help='Displays only the names of indexed attributes') + list_index_parser.add_argument('be_name', help='The backend name or suffix') + + # Delete index +- del_index_parser = index_subcommands.add_parser('delete', help='Delete an index') ++ del_index_parser = index_subcommands.add_parser('delete', help='Delete an index', formatter_class=CustomHelpFormatter) + del_index_parser.set_defaults(func=backend_del_index) + del_index_parser.add_argument('--attr', action='append', help='Sets the name of the attribute to delete from the index') + del_index_parser.add_argument('be_name', help='The backend name or suffix') + + # reindex index +- reindex_parser = index_subcommands.add_parser('reindex', help='Re-index the database for a single index or all indexes') ++ reindex_parser = index_subcommands.add_parser('reindex', help='Re-index the database for a single index or all indexes', formatter_class=CustomHelpFormatter) + reindex_parser.set_defaults(func=backend_reindex) + reindex_parser.add_argument('--attr', action='append', help='Sets the name of the attribute to re-index. Omit this argument to re-index all attributes') + reindex_parser.add_argument('--wait', action='store_true', help='Waits for the index task to complete and reports the status') +@@ -938,17 +939,17 @@ def create_parser(subparsers): + ############################################# + # VLV parser + ############################################# +- vlv_parser = subcommands.add_parser('vlv-index', help="Manage VLV searches and indexes") ++ vlv_parser = subcommands.add_parser('vlv-index', help="Manage VLV searches and indexes", formatter_class=CustomHelpFormatter) + vlv_subcommands = vlv_parser.add_subparsers(help="action") + + # List VLV Searches +- list_vlv_search_parser = vlv_subcommands.add_parser('list', help='List VLV search and index entries') ++ list_vlv_search_parser = vlv_subcommands.add_parser('list', help='List VLV search and index entries', formatter_class=CustomHelpFormatter) + list_vlv_search_parser.set_defaults(func=backend_list_vlv) + list_vlv_search_parser.add_argument('--just-names', action='store_true', help='Displays only the names of VLV search entries') + list_vlv_search_parser.add_argument('be_name', help='The backend name of the VLV index') + + # Get VLV search entry and indexes +- get_vlv_search_parser = vlv_subcommands.add_parser('get', help='Display a VLV search and indexes') ++ get_vlv_search_parser = vlv_subcommands.add_parser('get', help='Display a VLV search and indexes', formatter_class=CustomHelpFormatter) + get_vlv_search_parser.set_defaults(func=backend_get_vlv) + get_vlv_search_parser.add_argument('--name', help='Displays the VLV search entry and its index entries') + get_vlv_search_parser.add_argument('be_name', help='The backend name of the VLV index') +@@ -965,7 +966,7 @@ def create_parser(subparsers): + add_vlv_search_parser.add_argument('be_name', help='The backend name of the VLV index') + + # Edit vlv search +- edit_vlv_search_parser = vlv_subcommands.add_parser('edit-search', help='Update a VLV search and index') ++ edit_vlv_search_parser = vlv_subcommands.add_parser('edit-search', help='Update a VLV search and index', formatter_class=CustomHelpFormatter) + edit_vlv_search_parser.set_defaults(func=backend_edit_vlv) + edit_vlv_search_parser.add_argument('--name', required=True, help='Sets the name of the VLV index') + edit_vlv_search_parser.add_argument('--search-base', help='Sets the VLV search base') +@@ -975,13 +976,13 @@ def create_parser(subparsers): + edit_vlv_search_parser.add_argument('be_name', help='The backend name of the VLV index to update') + + # Delete vlv search(and index) +- del_vlv_search_parser = vlv_subcommands.add_parser('del-search', help='Delete VLV search & index') ++ del_vlv_search_parser = vlv_subcommands.add_parser('del-search', help='Delete VLV search & index', formatter_class=CustomHelpFormatter) + del_vlv_search_parser.set_defaults(func=backend_del_vlv) + del_vlv_search_parser.add_argument('--name', required=True, help='Sets the name of the VLV search index') + del_vlv_search_parser.add_argument('be_name', help='The backend name of the VLV index') + + # Create VLV Index +- add_vlv_index_parser = vlv_subcommands.add_parser('add-index', help='Create a VLV index under a VLV search entry (parent entry). ' ++ add_vlv_index_parser = vlv_subcommands.add_parser('add-index', help='Create a VLV index under a VLV search entry (parent entry, formatter_class=CustomHelpFormatter). ' + 'The VLV index specifies the attributes to sort') + add_vlv_index_parser.set_defaults(func=backend_create_vlv_index) + add_vlv_index_parser.add_argument('--parent-name', required=True, help='Sets the name or "cn" attribute of the parent VLV search entry') +@@ -991,7 +992,7 @@ def create_parser(subparsers): + add_vlv_index_parser.add_argument('be_name', help='The backend name of the VLV index') + + # Delete VLV Index +- del_vlv_index_parser = vlv_subcommands.add_parser('del-index', help='Delete a VLV index under a VLV search entry (parent entry)') ++ del_vlv_index_parser = vlv_subcommands.add_parser('del-index', help='Delete a VLV index under a VLV search entry (parent entry)', formatter_class=CustomHelpFormatter) + del_vlv_index_parser.set_defaults(func=backend_delete_vlv_index) + del_vlv_index_parser.add_argument('--parent-name', required=True, help='Sets the name or "cn" attribute value of the parent VLV search entry') + del_vlv_index_parser.add_argument('--index-name', help='Sets the name of the VLV index to delete') +@@ -999,7 +1000,7 @@ def create_parser(subparsers): + del_vlv_index_parser.add_argument('be_name', help='The backend name of the VLV index') + + # Reindex VLV +- reindex_vlv_parser = vlv_subcommands.add_parser('reindex', help='Index/re-index the VLV database index') ++ reindex_vlv_parser = vlv_subcommands.add_parser('reindex', help='Index/re-index the VLV database index', formatter_class=CustomHelpFormatter) + reindex_vlv_parser.set_defaults(func=backend_reindex_vlv) + reindex_vlv_parser.add_argument('--index-name', help='Sets the name of the VLV index entry to re-index. If not set, all indexes are re-indexed') + reindex_vlv_parser.add_argument('--parent-name', required=True, help='Sets the name or "cn" attribute value of the parent VLV search entry') +@@ -1008,7 +1009,7 @@ def create_parser(subparsers): + ############################################ + # Encrypted Attributes + ############################################ +- attr_encrypt_parser = subcommands.add_parser('attr-encrypt', help='Manage encrypted attribute settings') ++ attr_encrypt_parser = subcommands.add_parser('attr-encrypt', help='Manage encrypted attribute settings', formatter_class=CustomHelpFormatter) + attr_encrypt_parser.set_defaults(func=backend_attr_encrypt) + attr_encrypt_parser.add_argument('--list', action='store_true', help='Lists all encrypted attributes in the backend') + attr_encrypt_parser.add_argument('--just-names', action='store_true', help='List only the names of the encrypted attributes when used with --list') +@@ -1019,15 +1020,15 @@ def create_parser(subparsers): + ############################################ + # Global DB Config + ############################################ +- db_parser = subcommands.add_parser('config', help="Manage the global database configuration settings") ++ db_parser = subcommands.add_parser('config', help="Manage the global database configuration settings", formatter_class=CustomHelpFormatter) + db_subcommands = db_parser.add_subparsers(help="action") + + # Get the global database configuration +- get_db_config_parser = db_subcommands.add_parser('get', help='Display the global database configuration') ++ get_db_config_parser = db_subcommands.add_parser('get', help='Display the global database configuration', formatter_class=CustomHelpFormatter) + get_db_config_parser.set_defaults(func=db_config_get) + + # Update the global database configuration +- set_db_config_parser = db_subcommands.add_parser('set', help='Set the global database configuration') ++ set_db_config_parser = db_subcommands.add_parser('set', help='Set the global database configuration', formatter_class=CustomHelpFormatter) + set_db_config_parser.set_defaults(func=db_config_set) + set_db_config_parser.add_argument('--lookthroughlimit', help='Specifies the maximum number of entries that the server ' + 'will check when examining candidate entries in response to a search request') +@@ -1084,14 +1085,14 @@ def create_parser(subparsers): + ####################################################### + # Database & Suffix Monitor + ####################################################### +- get_monitor_parser = subcommands.add_parser('monitor', help="Displays global database or suffix monitoring information") ++ get_monitor_parser = subcommands.add_parser('monitor', help="Displays global database or suffix monitoring information", formatter_class=CustomHelpFormatter) + get_monitor_parser.set_defaults(func=get_monitor) + get_monitor_parser.add_argument('--suffix', help='Displays monitoring information only for the specified suffix') + + ####################################################### + # Import LDIF + ####################################################### +- import_parser = subcommands.add_parser('import', help="Online import of a suffix") ++ import_parser = subcommands.add_parser('import', help="Online import of a suffix", formatter_class=CustomHelpFormatter) + import_parser.set_defaults(func=backend_import) + import_parser.add_argument('be_name', nargs='?', + help='The backend name or the root suffix') +@@ -1122,7 +1123,7 @@ def create_parser(subparsers): + ####################################################### + # Export LDIF + ####################################################### +- export_parser = subcommands.add_parser('export', help='Online export of a suffix') ++ export_parser = subcommands.add_parser('export', help='Online export of a suffix', formatter_class=CustomHelpFormatter) + export_parser.set_defaults(func=backend_export) + export_parser.add_argument('be_names', nargs='+', + help="The backend names or the root suffixes") +@@ -1153,7 +1154,7 @@ def create_parser(subparsers): + ####################################################### + # Create a new backend database + ####################################################### +- create_parser = subcommands.add_parser('create', help='Create a backend database') ++ create_parser = subcommands.add_parser('create', help='Create a backend database', formatter_class=CustomHelpFormatter) + create_parser.set_defaults(func=backend_create) + create_parser.add_argument('--parent-suffix', default=False, + help="Sets the parent suffix only if this backend is a sub-suffix") +@@ -1166,20 +1167,20 @@ def create_parser(subparsers): + ####################################################### + # Delete backend + ####################################################### +- delete_parser = subcommands.add_parser('delete', help='Delete a backend database') ++ delete_parser = subcommands.add_parser('delete', help='Delete a backend database', formatter_class=CustomHelpFormatter) + delete_parser.set_defaults(func=backend_delete) + delete_parser.add_argument('be_name', help='The backend name or suffix') + + ####################################################### + # Get Suffix Tree (for use in web console) + ####################################################### +- get_tree_parser = subcommands.add_parser('get-tree', help='Display the suffix tree') ++ get_tree_parser = subcommands.add_parser('get-tree', help='Display the suffix tree', formatter_class=CustomHelpFormatter) + get_tree_parser.set_defaults(func=backend_get_tree) + + ####################################################### + # Run the db compaction task + ####################################################### +- compact_parser = subcommands.add_parser('compact-db', help='Compact the database and the replication changelog') ++ compact_parser = subcommands.add_parser('compact-db', help='Compact the database and the replication changelog', formatter_class=CustomHelpFormatter) + compact_parser.set_defaults(func=backend_compact) + compact_parser.add_argument('--only-changelog', action='store_true', help='Compacts only the replication change log') + compact_parser.add_argument('--timeout', default=0, type=int, +diff --git a/src/lib389/lib389/cli_conf/backup.py b/src/lib389/lib389/cli_conf/backup.py +index 32f5db2f1..2996c0bd2 100644 +--- a/src/lib389/lib389/cli_conf/backup.py ++++ b/src/lib389/lib389/cli_conf/backup.py +@@ -6,6 +6,7 @@ + # See LICENSE for details. + # --- END COPYRIGHT BLOCK --- + ++from lib389.cli_base import CustomHelpFormatter + + def backup_create(inst, basedn, log, args): + log = log.getChild('backup_create') +@@ -41,11 +42,11 @@ def backup_restore(inst, basedn, log, args): + + + def create_parser(subparsers): +- backup_parser = subparsers.add_parser('backup', help="Manage online backups") ++ backup_parser = subparsers.add_parser('backup', help="Manage online backups", formatter_class=CustomHelpFormatter) + + subcommands = backup_parser.add_subparsers(help="action") + +- create_backup_parser = subcommands.add_parser('create', help="Creates a backup of the database") ++ create_backup_parser = subcommands.add_parser('create', help="Creates a backup of the database", formatter_class=CustomHelpFormatter) + create_backup_parser.set_defaults(func=backup_create) + create_backup_parser.add_argument('archive', nargs='?', default=None, + help="Sets the directory where to store the backup files. " +@@ -56,7 +57,7 @@ def create_parser(subparsers): + create_backup_parser.add_argument('--timeout', type=int, default=120, + help="Sets the task timeout. Default is 120 seconds,") + +- restore_parser = subcommands.add_parser('restore', help="Restores a database from a backup") ++ restore_parser = subcommands.add_parser('restore', help="Restores a database from a backup", formatter_class=CustomHelpFormatter) + restore_parser.set_defaults(func=backup_restore) + restore_parser.add_argument('archive', help="Set the directory that contains the backup files") + restore_parser.add_argument('-t', '--db-type', default="ldbm database", +diff --git a/src/lib389/lib389/cli_conf/chaining.py b/src/lib389/lib389/cli_conf/chaining.py +index f76e7f991..d0f691a50 100644 +--- a/src/lib389/lib389/cli_conf/chaining.py ++++ b/src/lib389/lib389/cli_conf/chaining.py +@@ -13,6 +13,7 @@ from lib389.cli_base import ( + _generic_list, + _generic_get, + _get_arg, ++ CustomHelpFormatter + ) + from lib389.cli_conf.monitor import _format_status + from lib389.utils import get_passwd_from_file +@@ -220,25 +221,25 @@ def list_links(inst, basedn, log, args): + + + def create_parser(subparsers): +- chaining_parser = subparsers.add_parser('chaining', help="Manage database chaining and database links") ++ chaining_parser = subparsers.add_parser('chaining', help="Manage database chaining and database links", formatter_class=CustomHelpFormatter) + subcommands = chaining_parser.add_subparsers(help="action") + +- config_get_parser = subcommands.add_parser('config-get', help='Display the chaining controls and server component lists') ++ config_get_parser = subcommands.add_parser('config-get', help='Display the chaining controls and server component lists', formatter_class=CustomHelpFormatter) + config_get_parser.set_defaults(func=config_get) + config_get_parser.add_argument('--avail-controls', action='store_true', help="Lists available chaining controls") + config_get_parser.add_argument('--avail-comps', action='store_true', help="Lists available chaining plugin components") + +- config_set_parser = subcommands.add_parser('config-set', help='Set the chaining controls and server component lists') ++ config_set_parser = subcommands.add_parser('config-set', help='Set the chaining controls and server component lists', formatter_class=CustomHelpFormatter) + config_set_parser.set_defaults(func=config_set) + config_set_parser.add_argument('--add-control', action='append', help="Adds a transmitted control OID") + config_set_parser.add_argument('--del-control', action='append', help="Deletes a transmitted control OID") + config_set_parser.add_argument('--add-comp', action='append', help="Adds a chaining component") + config_set_parser.add_argument('--del-comp', action='append', help="Deletes a chaining component") + +- def_config_get_parser = subcommands.add_parser('config-get-def', help='Display the default creation parameters for new database links') ++ def_config_get_parser = subcommands.add_parser('config-get-def', help='Display the default creation parameters for new database links', formatter_class=CustomHelpFormatter) + def_config_get_parser.set_defaults(func=def_config_get) + +- def_config_set_parser = subcommands.add_parser('config-set-def', help='Set the default creation parameters for new database links') ++ def_config_set_parser = subcommands.add_parser('config-set-def', help='Set the default creation parameters for new database links', formatter_class=CustomHelpFormatter) + def_config_set_parser.set_defaults(func=def_config_set) + def_config_set_parser.add_argument('--conn-bind-limit', + help="Sets the maximum number of BIND connections the database link establishes " +@@ -293,7 +294,7 @@ def create_parser(subparsers): + create_link_parser.add_argument('--bind-pw-file', help="File containing the password") + create_link_parser.add_argument('--bind-pw-prompt', action='store_true', help="Prompt for password") + +- get_link_parser = subcommands.add_parser('link-get', help='Displays chaining database links') ++ get_link_parser = subcommands.add_parser('link-get', help='Displays chaining database links', formatter_class=CustomHelpFormatter) + get_link_parser.set_defaults(func=get_link) + get_link_parser.add_argument('CHAIN_NAME', nargs=1, help='The chaining link name or suffix to retrieve') + +@@ -312,13 +313,13 @@ def create_parser(subparsers): + edit_link_parser.add_argument('--bind-pw-file', help="File containing the password") + edit_link_parser.add_argument('--bind-pw-prompt', action='store_true', help="Prompt for password") + +- delete_link_parser = subcommands.add_parser('link-delete', help='Delete a database link') ++ delete_link_parser = subcommands.add_parser('link-delete', help='Delete a database link', formatter_class=CustomHelpFormatter) + delete_link_parser.set_defaults(func=delete_link) + delete_link_parser.add_argument('CHAIN_NAME', nargs=1, help='The name of the database link') + +- monitor_link_parser = subcommands.add_parser('monitor', help='Display monitor information for a database chaining link') ++ monitor_link_parser = subcommands.add_parser('monitor', help='Display monitor information for a database chaining link', formatter_class=CustomHelpFormatter) + monitor_link_parser.set_defaults(func=monitor_link) + monitor_link_parser.add_argument('CHAIN_NAME', nargs=1, help='The name of the database link') + +- list_link_parser = subcommands.add_parser('link-list', help='List database links') ++ list_link_parser = subcommands.add_parser('link-list', help='List database links', formatter_class=CustomHelpFormatter) + list_link_parser.set_defaults(func=list_links) +diff --git a/src/lib389/lib389/cli_conf/config.py b/src/lib389/lib389/cli_conf/config.py +index 6fbf54ed3..0af479ae3 100644 +--- a/src/lib389/lib389/cli_conf/config.py ++++ b/src/lib389/lib389/cli_conf/config.py +@@ -13,6 +13,7 @@ from lib389.cli_base import ( + _generic_get_entry, + _generic_get_attr, + _generic_replace_attr, ++ CustomHelpFormatter + ) + + OpType = Enum("OpType", "add delete") +@@ -118,22 +119,22 @@ def config_del_attr(inst, basedn, log, args): + + + def create_parser(subparsers): +- config_parser = subparsers.add_parser('config', help="Manage the server configuration") ++ config_parser = subparsers.add_parser('config', help="Manage the server configuration", formatter_class=CustomHelpFormatter) + + subcommands = config_parser.add_subparsers(help="action") + +- get_parser = subcommands.add_parser('get', help='get') ++ get_parser = subcommands.add_parser('get', help='get', formatter_class=CustomHelpFormatter) + get_parser.set_defaults(func=config_get) + get_parser.add_argument('attrs', nargs='*', help='Configuration attribute(s) to get') + +- add_attr_parser = subcommands.add_parser('add', help='Add attribute value to configuration') ++ add_attr_parser = subcommands.add_parser('add', help='Add attribute value to configuration', formatter_class=CustomHelpFormatter) + add_attr_parser.set_defaults(func=config_add_attr) + add_attr_parser.add_argument('attr', nargs='*', help='Configuration attribute to add') + +- replace_attr_parser = subcommands.add_parser('replace', help='Replace attribute value in configuration') ++ replace_attr_parser = subcommands.add_parser('replace', help='Replace attribute value in configuration', formatter_class=CustomHelpFormatter) + replace_attr_parser.set_defaults(func=config_replace_attr) + replace_attr_parser.add_argument('attr', nargs='*', help='Configuration attribute to replace') + +- del_attr_parser = subcommands.add_parser('delete', help='Delete attribute value in configuration') ++ del_attr_parser = subcommands.add_parser('delete', help='Delete attribute value in configuration', formatter_class=CustomHelpFormatter) + del_attr_parser.set_defaults(func=config_del_attr) + del_attr_parser.add_argument('attr', nargs='*', help='Configuration attribute to delete') +diff --git a/src/lib389/lib389/cli_conf/conflicts.py b/src/lib389/lib389/cli_conf/conflicts.py +index c6758fb70..5d2ef7227 100644 +--- a/src/lib389/lib389/cli_conf/conflicts.py ++++ b/src/lib389/lib389/cli_conf/conflicts.py +@@ -8,6 +8,7 @@ + + import json + from lib389.conflicts import (ConflictEntries, ConflictEntry, GlueEntries, GlueEntry) ++from lib389.cli_base import CustomHelpFormatter + + conflict_attrs = ['nsds5replconflict', '*'] + +@@ -84,23 +85,23 @@ def convert_glue(inst, basedn, log, args): + + + def create_parser(subparsers): +- conflict_parser = subparsers.add_parser('repl-conflict', help="Manage replication conflicts") ++ conflict_parser = subparsers.add_parser('repl-conflict', help="Manage replication conflicts", formatter_class=CustomHelpFormatter) + subcommands = conflict_parser.add_subparsers(help='action') + + # coinflict entry arguments +- list_parser = subcommands.add_parser('list', help="List conflict entries") ++ list_parser = subcommands.add_parser('list', help="List conflict entries", formatter_class=CustomHelpFormatter) + list_parser.add_argument('suffix', help='Sets the backend name, or suffix, to look for conflict entries') + list_parser.set_defaults(func=list_conflicts) + +- cmp_parser = subcommands.add_parser('compare', help="Compare the conflict entry with its valid counterpart") ++ cmp_parser = subcommands.add_parser('compare', help="Compare the conflict entry with its valid counterpart", formatter_class=CustomHelpFormatter) + cmp_parser.add_argument('DN', help='The DN of the conflict entry') + cmp_parser.set_defaults(func=cmp_conflict) + +- del_parser = subcommands.add_parser('delete', help="Delete a conflict entry") ++ del_parser = subcommands.add_parser('delete', help="Delete a conflict entry", formatter_class=CustomHelpFormatter) + del_parser.add_argument('DN', help='The DN of the conflict entry') + del_parser.set_defaults(func=del_conflict) + +- replace_parser = subcommands.add_parser('swap', help="Replace the valid entry with the conflict entry") ++ replace_parser = subcommands.add_parser('swap', help="Replace the valid entry with the conflict entry", formatter_class=CustomHelpFormatter) + replace_parser.add_argument('DN', help='The DN of the conflict entry') + replace_parser.set_defaults(func=swap_conflict) + +@@ -114,14 +115,14 @@ def create_parser(subparsers): + replace_parser.set_defaults(func=convert_conflict) + + # Glue entry arguments +- list_glue_parser = subcommands.add_parser('list-glue', help="List replication glue entries") ++ list_glue_parser = subcommands.add_parser('list-glue', help="List replication glue entries", formatter_class=CustomHelpFormatter) + list_glue_parser.add_argument('suffix', help='The backend name, or suffix, to look for glue entries') + list_glue_parser.set_defaults(func=list_glue) + +- del_glue_parser = subcommands.add_parser('delete-glue', help="Delete the glue entry and its child entries") ++ del_glue_parser = subcommands.add_parser('delete-glue', help="Delete the glue entry and its child entries", formatter_class=CustomHelpFormatter) + del_glue_parser.add_argument('DN', help='The DN of the glue entry') + del_glue_parser.set_defaults(func=del_glue) + +- convert_glue_parser = subcommands.add_parser('convert-glue', help="Convert the glue entry into a regular entry") ++ convert_glue_parser = subcommands.add_parser('convert-glue', help="Convert the glue entry into a regular entry", formatter_class=CustomHelpFormatter) + convert_glue_parser.add_argument('DN', help='The DN of the glue entry') + convert_glue_parser.set_defaults(func=convert_glue) +diff --git a/src/lib389/lib389/cli_conf/directory_manager.py b/src/lib389/lib389/cli_conf/directory_manager.py +index 0886d506d..b7d4ada03 100644 +--- a/src/lib389/lib389/cli_conf/directory_manager.py ++++ b/src/lib389/lib389/cli_conf/directory_manager.py +@@ -7,7 +7,7 @@ + # --- END COPYRIGHT BLOCK --- + + from lib389.idm.directorymanager import DirectoryManager +-from lib389.cli_base import _get_arg ++from lib389.cli_base import _get_arg, CustomHelpFormatter + + + def password_change(inst, basedn, log, args): +@@ -19,11 +19,11 @@ def password_change(inst, basedn, log, args): + + + def create_parsers(subparsers): +- directory_manager_parser = subparsers.add_parser('directory_manager', help="Manage the Directory Manager account") ++ directory_manager_parser = subparsers.add_parser('directory_manager', help="Manage the Directory Manager account", formatter_class=CustomHelpFormatter) + + subcommands = directory_manager_parser.add_subparsers(help='action') + +- password_change_parser = subcommands.add_parser('password_change', help="Changes the password of the Directory Manager account") ++ password_change_parser = subcommands.add_parser('password_change', help="Changes the password of the Directory Manager account", formatter_class=CustomHelpFormatter) + password_change_parser.set_defaults(func=password_change) + # This is to put in a dummy attr that args can work with. We do this + # because the actual test case will over-ride it, but it prevents +diff --git a/src/lib389/lib389/cli_conf/monitor.py b/src/lib389/lib389/cli_conf/monitor.py +index a56ad2fee..d69e2f06b 100644 +--- a/src/lib389/lib389/cli_conf/monitor.py ++++ b/src/lib389/lib389/cli_conf/monitor.py +@@ -14,7 +14,7 @@ from lib389.monitor import (Monitor, MonitorLDBM, MonitorSNMP, MonitorDiskSpace) + from lib389.chaining import (ChainingLinks) + from lib389.backend import Backends + from lib389.utils import convert_bytes +-from lib389.cli_base import _format_status ++from lib389.cli_base import _format_status, CustomHelpFormatter + + + def monitor(inst, basedn, log, args): +@@ -297,30 +297,30 @@ def db_monitor(inst, basedn, log, args): + + + def create_parser(subparsers): +- monitor_parser = subparsers.add_parser('monitor', help="Monitor the state of the instance") ++ monitor_parser = subparsers.add_parser('monitor', help="Monitor the state of the instance", formatter_class=CustomHelpFormatter) + subcommands = monitor_parser.add_subparsers(help='action') + +- server_parser = subcommands.add_parser('server', help="Displays the server statistics, connections, and operations") ++ server_parser = subcommands.add_parser('server', help="Displays the server statistics, connections, and operations", formatter_class=CustomHelpFormatter) + server_parser.set_defaults(func=monitor) + +- dbmon_parser = subcommands.add_parser('dbmon', help="Monitor all database statistics in a single report") ++ dbmon_parser = subcommands.add_parser('dbmon', help="Monitor all database statistics in a single report", formatter_class=CustomHelpFormatter) + dbmon_parser.set_defaults(func=db_monitor) + dbmon_parser.add_argument('-b', '--backends', help="Specifies a list of space-separated backends to monitor. Default is all backends.") + dbmon_parser.add_argument('-x', '--indexes', action='store_true', default=False, help="Shows index stats for each backend") + +- ldbm_parser = subcommands.add_parser('ldbm', help="Monitor the LDBM statistics, such as dbcache") ++ ldbm_parser = subcommands.add_parser('ldbm', help="Monitor the LDBM statistics, such as dbcache", formatter_class=CustomHelpFormatter) + ldbm_parser.set_defaults(func=ldbm_monitor) + +- backend_parser = subcommands.add_parser('backend', help="Monitor the behavior of a backend database") ++ backend_parser = subcommands.add_parser('backend', help="Monitor the behavior of a backend database", formatter_class=CustomHelpFormatter) + backend_parser.add_argument('backend', nargs='?', help="The optional name of the backend to monitor") + backend_parser.set_defaults(func=backend_monitor) + +- snmp_parser = subcommands.add_parser('snmp', help="Displays the SNMP statistics") ++ snmp_parser = subcommands.add_parser('snmp', help="Displays the SNMP statistics", formatter_class=CustomHelpFormatter) + snmp_parser.set_defaults(func=snmp_monitor) + +- chaining_parser = subcommands.add_parser('chaining', help="Monitor database chaining statistics") ++ chaining_parser = subcommands.add_parser('chaining', help="Monitor database chaining statistics", formatter_class=CustomHelpFormatter) + chaining_parser.add_argument('backend', nargs='?', help="The optional name of the chaining backend to monitor") + chaining_parser.set_defaults(func=chaining_monitor) + +- disk_parser = subcommands.add_parser('disk', help="Displays the disk space statistics. All values are in bytes.") ++ disk_parser = subcommands.add_parser('disk', help="Displays the disk space statistics. All values are in bytes.", formatter_class=CustomHelpFormatter) + disk_parser.set_defaults(func=disk_monitor) +diff --git a/src/lib389/lib389/cli_conf/plugin.py b/src/lib389/lib389/cli_conf/plugin.py +index 465a35f2d..c49d85255 100644 +--- a/src/lib389/lib389/cli_conf/plugin.py ++++ b/src/lib389/lib389/cli_conf/plugin.py +@@ -12,6 +12,7 @@ from lib389.utils import ensure_dict_str + from lib389.cli_base import ( + _generic_get, + _get_arg, ++ CustomHelpFormatter + ) + from lib389.cli_conf import generic_object_edit + from lib389.cli_conf.plugins import memberof as cli_memberof +@@ -98,7 +99,7 @@ def plugin_edit(inst, basedn, log, args): + + + def create_parser(subparsers): +- plugin_parser = subparsers.add_parser('plugin', help="Manage plug-ins available on the server") ++ plugin_parser = subparsers.add_parser('plugin', help="Manage plug-ins available on the server", formatter_class=CustomHelpFormatter) + + subcommands = plugin_parser.add_subparsers(help="Plugins") + +@@ -118,14 +119,14 @@ def create_parser(subparsers): + cli_posix_winsync.create_parser(subcommands) + cli_entryuuid.create_parser(subcommands) + +- list_parser = subcommands.add_parser('list', help="List current configured (enabled and disabled) plugins") ++ list_parser = subcommands.add_parser('list', help="List current configured (enabled and disabled) plugins", formatter_class=CustomHelpFormatter) + list_parser.set_defaults(func=plugin_list) + +- get_parser = subcommands.add_parser('show', help='Show the plugin data') ++ get_parser = subcommands.add_parser('show', help='Show the plugin data', formatter_class=CustomHelpFormatter) + get_parser.set_defaults(func=plugin_get) + get_parser.add_argument('selector', nargs='?', help='The plugin to search for') + +- edit_parser = subcommands.add_parser('set', help='Edit the plugin settings') ++ edit_parser = subcommands.add_parser('set', help='Edit the plugin settings', formatter_class=CustomHelpFormatter) + edit_parser.set_defaults(func=plugin_edit) + edit_parser.add_argument('selector', nargs='?', help='The plugin to edit') + edit_parser.add_argument('--type', help='The type of plugin.') +diff --git a/src/lib389/lib389/cli_conf/plugins/accountpolicy.py b/src/lib389/lib389/cli_conf/plugins/accountpolicy.py +index 531173c18..b6a878ef5 100644 +--- a/src/lib389/lib389/cli_conf/plugins/accountpolicy.py ++++ b/src/lib389/lib389/cli_conf/plugins/accountpolicy.py +@@ -9,6 +9,7 @@ + import ldap + from lib389.plugins import AccountPolicyPlugin, AccountPolicyConfig + from lib389.cli_conf import add_generic_plugin_parsers, generic_object_edit, generic_object_add ++from lib389.cli_base import CustomHelpFormatter + + arg_to_attr = { + 'config_entry': 'nsslapd_pluginconfigarea' +@@ -101,31 +102,31 @@ def _add_parser_args(parser): + + + def create_parser(subparsers): +- accountpolicy = subparsers.add_parser('account-policy', help='Manage and configure Account Policy plugin') ++ accountpolicy = subparsers.add_parser('account-policy', help='Manage and configure Account Policy plugin', formatter_class=CustomHelpFormatter) + subcommands = accountpolicy.add_subparsers(help='action') + add_generic_plugin_parsers(subcommands, AccountPolicyPlugin) + +- edit = subcommands.add_parser('set', help='Edit the plugin settings') ++ edit = subcommands.add_parser('set', help='Edit the plugin settings', formatter_class=CustomHelpFormatter) + edit.set_defaults(func=accountpolicy_edit) + edit.add_argument('--config-entry', help='Sets the nsslapd-pluginConfigArea attribute') + +- config = subcommands.add_parser('config-entry', help='Manage the config entry') ++ config = subcommands.add_parser('config-entry', help='Manage the config entry', formatter_class=CustomHelpFormatter) + config_subcommands = config.add_subparsers(help='action') + +- add_config = config_subcommands.add_parser('add', help='Add the config entry') ++ add_config = config_subcommands.add_parser('add', help='Add the config entry', formatter_class=CustomHelpFormatter) + add_config.set_defaults(func=accountpolicy_add_config) + add_config.add_argument('DN', help='The full DN of the config entry') + _add_parser_args(add_config) + +- edit_config = config_subcommands.add_parser('set', help='Edit the config entry') ++ edit_config = config_subcommands.add_parser('set', help='Edit the config entry', formatter_class=CustomHelpFormatter) + edit_config.set_defaults(func=accountpolicy_edit_config) + edit_config.add_argument('DN', help='The full DN of the config entry') + _add_parser_args(edit_config) + +- show_config_parser = config_subcommands.add_parser('show', help='Display the config entry') ++ show_config_parser = config_subcommands.add_parser('show', help='Display the config entry', formatter_class=CustomHelpFormatter) + show_config_parser.set_defaults(func=accountpolicy_show_config) + show_config_parser.add_argument('DN', help='The full DN of the config entry') + +- del_config_parser = config_subcommands.add_parser('delete', help='Delete the config entry') ++ del_config_parser = config_subcommands.add_parser('delete', help='Delete the config entry', formatter_class=CustomHelpFormatter) + del_config_parser.set_defaults(func=accountpolicy_del_config) + del_config_parser.add_argument('DN', help='The full DN of the config entry') +diff --git a/src/lib389/lib389/cli_conf/plugins/attruniq.py b/src/lib389/lib389/cli_conf/plugins/attruniq.py +index f9bacab78..0c398b944 100644 +--- a/src/lib389/lib389/cli_conf/plugins/attruniq.py ++++ b/src/lib389/lib389/cli_conf/plugins/attruniq.py +@@ -10,6 +10,7 @@ import json + import ldap + from lib389.plugins import AttributeUniquenessPlugin, AttributeUniquenessPlugins + from lib389.cli_conf import (generic_object_edit, generic_object_add) ++from lib389.cli_base import CustomHelpFormatter + + arg_to_attr = { + 'enabled': 'nsslapd-pluginenabled', +@@ -136,37 +137,37 @@ def _add_parser_args(parser): + + + def create_parser(subparsers): +- attruniq = subparsers.add_parser('attr-uniq', help='Manage and configure Attribute Uniqueness plugin') ++ attruniq = subparsers.add_parser('attr-uniq', help='Manage and configure Attribute Uniqueness plugin', formatter_class=CustomHelpFormatter) + subcommands = attruniq.add_subparsers(help='action') + # We can't use the add_generic_plugin_parsers as we need named sub instances. + +- list = subcommands.add_parser('list', help='Lists available plugin configs') ++ list = subcommands.add_parser('list', help='Lists available plugin configs', formatter_class=CustomHelpFormatter) + list.set_defaults(func=attruniq_list) + +- add = subcommands.add_parser('add', help='Add the config entry') ++ add = subcommands.add_parser('add', help='Add the config entry', formatter_class=CustomHelpFormatter) + add.set_defaults(func=attruniq_add) + _add_parser_args(add) + +- edit = subcommands.add_parser('set', help='Edit the config entry') ++ edit = subcommands.add_parser('set', help='Edit the config entry', formatter_class=CustomHelpFormatter) + edit.set_defaults(func=attruniq_edit) + _add_parser_args(edit) + +- show = subcommands.add_parser('show', help='Display the config entry') ++ show = subcommands.add_parser('show', help='Display the config entry', formatter_class=CustomHelpFormatter) + show.add_argument('NAME', help='The name of the plug-in configuration record') + show.set_defaults(func=attruniq_show) + +- delete = subcommands.add_parser('delete', help='Delete the config entry') ++ delete = subcommands.add_parser('delete', help='Delete the config entry', formatter_class=CustomHelpFormatter) + delete.add_argument('NAME', help='The name of the plug-in configuration record') + delete.set_defaults(func=attruniq_del) + +- enable = subcommands.add_parser('enable', help='enable plugin') ++ enable = subcommands.add_parser('enable', help='enable plugin', formatter_class=CustomHelpFormatter) + enable.add_argument('NAME', help='The name of the plug-in configuration record') + enable.set_defaults(func=attruniq_enable) + +- disable = subcommands.add_parser('disable', help='disable plugin') ++ disable = subcommands.add_parser('disable', help='disable plugin', formatter_class=CustomHelpFormatter) + disable.add_argument('NAME', help='The name of the plug-in configuration record') + disable.set_defaults(func=attruniq_disable) + +- status = subcommands.add_parser('status', help='display plugin status') ++ status = subcommands.add_parser('status', help='display plugin status', formatter_class=CustomHelpFormatter) + status.add_argument('NAME', help='The name of the plug-in configuration record') + status.set_defaults(func=attruniq_status) +diff --git a/src/lib389/lib389/cli_conf/plugins/automember.py b/src/lib389/lib389/cli_conf/plugins/automember.py +index 568586ad8..febabad3d 100644 +--- a/src/lib389/lib389/cli_conf/plugins/automember.py ++++ b/src/lib389/lib389/cli_conf/plugins/automember.py +@@ -11,6 +11,7 @@ import json + from lib389.plugins import (AutoMembershipPlugin, AutoMembershipDefinition, AutoMembershipDefinitions, + AutoMembershipRegexRule, AutoMembershipRegexRules, AutoMembershipFixupTasks) + from lib389.cli_conf import add_generic_plugin_parsers, generic_object_edit, generic_object_add ++from lib389.cli_base import CustomHelpFormatter + from lib389.utils import get_task_status + + arg_to_attr_definition = { +@@ -221,49 +222,49 @@ def _add_parser_args_regex(parser): + + + def create_parser(subparsers): +- automember = subparsers.add_parser('automember', help="Manage and configure Automembership plugin") ++ automember = subparsers.add_parser('automember', help="Manage and configure Automembership plugin", formatter_class=CustomHelpFormatter) + subcommands = automember.add_subparsers(help='action') + add_generic_plugin_parsers(subcommands, AutoMembershipPlugin) + +- automember_list = subcommands.add_parser('list', help='List Automembership definitions or regex rules.') ++ automember_list = subcommands.add_parser('list', help='List Automembership definitions or regex rules.', formatter_class=CustomHelpFormatter) + subcommands_list = automember_list.add_subparsers(help='action') +- list_definitions = subcommands_list.add_parser('definitions', help='Lists Automembership definitions.') ++ list_definitions = subcommands_list.add_parser('definitions', help='Lists Automembership definitions.', formatter_class=CustomHelpFormatter) + list_definitions.set_defaults(func=definition_list) +- list_regexes = subcommands_list.add_parser('regexes', help='List Automembership regex rules.') ++ list_regexes = subcommands_list.add_parser('regexes', help='List Automembership regex rules.', formatter_class=CustomHelpFormatter) + list_regexes.add_argument('DEFNAME', help='The definition entry CN') + list_regexes.set_defaults(func=regex_list) + +- definition = subcommands.add_parser('definition', help='Manage Automembership definition.') ++ definition = subcommands.add_parser('definition', help='Manage Automembership definition.', formatter_class=CustomHelpFormatter) + definition.add_argument('DEFNAME', help='The definition entry CN.') + subcommands_definition = definition.add_subparsers(help='action') + +- add_def = subcommands_definition.add_parser('add', help='Creates Automembership definition.') ++ add_def = subcommands_definition.add_parser('add', help='Creates Automembership definition.', formatter_class=CustomHelpFormatter) + add_def.set_defaults(func=definition_add) + _add_parser_args_definition(add_def) +- edit_def = subcommands_definition.add_parser('set', help='Edits Automembership definition.') ++ edit_def = subcommands_definition.add_parser('set', help='Edits Automembership definition.', formatter_class=CustomHelpFormatter) + edit_def.set_defaults(func=definition_edit) + _add_parser_args_definition(edit_def) +- delete_def = subcommands_definition.add_parser('delete', help='Removes Automembership definition.') ++ delete_def = subcommands_definition.add_parser('delete', help='Removes Automembership definition.', formatter_class=CustomHelpFormatter) + delete_def.set_defaults(func=definition_del) +- show_def = subcommands_definition.add_parser('show', help='Displays Automembership definition.') ++ show_def = subcommands_definition.add_parser('show', help='Displays Automembership definition.', formatter_class=CustomHelpFormatter) + show_def.set_defaults(func=definition_show) + +- regex = subcommands_definition.add_parser('regex', help='Manage Automembership regex rules.') ++ regex = subcommands_definition.add_parser('regex', help='Manage Automembership regex rules.', formatter_class=CustomHelpFormatter) + regex.add_argument('REGEXNAME', help='The regex entry CN') + subcommands_regex = regex.add_subparsers(help='action') + +- add_regex = subcommands_regex.add_parser('add', help='Creates Automembership regex.') ++ add_regex = subcommands_regex.add_parser('add', help='Creates Automembership regex.', formatter_class=CustomHelpFormatter) + add_regex.set_defaults(func=regex_add) + _add_parser_args_regex(add_regex) +- edit_regex = subcommands_regex.add_parser('set', help='Edits Automembership regex.') ++ edit_regex = subcommands_regex.add_parser('set', help='Edits Automembership regex.', formatter_class=CustomHelpFormatter) + edit_regex.set_defaults(func=regex_edit) + _add_parser_args_regex(edit_regex) +- delete_regex = subcommands_regex.add_parser('delete', help='Removes Automembership regex.') ++ delete_regex = subcommands_regex.add_parser('delete', help='Removes Automembership regex.', formatter_class=CustomHelpFormatter) + delete_regex.set_defaults(func=regex_del) +- show_regex = subcommands_regex.add_parser('show', help='Displays Automembership regex.') ++ show_regex = subcommands_regex.add_parser('show', help='Displays Automembership regex.', formatter_class=CustomHelpFormatter) + show_regex.set_defaults(func=regex_show) + +- fixup_task = subcommands.add_parser('fixup', help='Run a rebuild membership task.') ++ fixup_task = subcommands.add_parser('fixup', help='Run a rebuild membership task.', formatter_class=CustomHelpFormatter) + fixup_task.set_defaults(func=fixup) + fixup_task.add_argument('DN', help="Base DN that contains entries to fix up") + fixup_task.add_argument('-f', '--filter', required=True, help='Sets the LDAP filter for entries to fix up') +@@ -276,14 +277,14 @@ def create_parser(subparsers): + fixup_task.add_argument('--timeout', default=0, type=int, + help="Set a timeout to wait for the fixup task. Default is 0 (no timeout)") + +- fixup_status = subcommands.add_parser('fixup-status', help='Check the status of a fix-up task') ++ fixup_status = subcommands.add_parser('fixup-status', help='Check the status of a fix-up task', formatter_class=CustomHelpFormatter) + fixup_status.set_defaults(func=do_fixup_status) + fixup_status.add_argument('--dn', help="The task entry's DN") + fixup_status.add_argument('--show-log', action='store_true', help="Display the task log") + fixup_status.add_argument('--watch', action='store_true', + help="Watch the task's status and wait for it to finish") + +- abort_fixup = subcommands.add_parser('abort-fixup', help='Abort the rebuild membership task.') ++ abort_fixup = subcommands.add_parser('abort-fixup', help='Abort the rebuild membership task.', formatter_class=CustomHelpFormatter) + abort_fixup.set_defaults(func=abort) + abort_fixup.add_argument('--timeout', default=0, type=int, + help="Set a timeout to wait for the abort task. Default is 0 (no timeout)") +diff --git a/src/lib389/lib389/cli_conf/plugins/dna.py b/src/lib389/lib389/cli_conf/plugins/dna.py +index 20d5aa584..bc766ecac 100644 +--- a/src/lib389/lib389/cli_conf/plugins/dna.py ++++ b/src/lib389/lib389/cli_conf/plugins/dna.py +@@ -10,6 +10,7 @@ import json + import ldap + from lib389.plugins import DNAPlugin, DNAPluginConfig, DNAPluginConfigs, DNAPluginSharedConfig, DNAPluginSharedConfigs + from lib389.cli_conf import add_generic_plugin_parsers, generic_object_edit, generic_object_add, _args_to_attrs ++from lib389.cli_base import CustomHelpFormatter + + arg_to_attr = { + 'type': 'dnaType', +@@ -201,43 +202,43 @@ def _add_parser_args(parser): + 'can request a range from a new server (dnaRangeRequestTimeout)') + + def create_parser(subparsers): +- dna = subparsers.add_parser('dna', help='Manage and configure DNA plugin') ++ dna = subparsers.add_parser('dna', help='Manage and configure DNA plugin', formatter_class=CustomHelpFormatter) + subcommands = dna.add_subparsers(help='action') + add_generic_plugin_parsers(subcommands, DNAPlugin) + +- list = subcommands.add_parser('list', help='List available plugin configs') ++ list = subcommands.add_parser('list', help='List available plugin configs', formatter_class=CustomHelpFormatter) + subcommands_list = list.add_subparsers(help='action') +- list_configs = subcommands_list.add_parser('configs', help='List main DNA plugin config entries') ++ list_configs = subcommands_list.add_parser('configs', help='List main DNA plugin config entries', formatter_class=CustomHelpFormatter) + list_configs.set_defaults(func=dna_list) +- list_shared_configs = subcommands_list.add_parser('shared-configs', help='List DNA plugin shared config entries') ++ list_shared_configs = subcommands_list.add_parser('shared-configs', help='List DNA plugin shared config entries', formatter_class=CustomHelpFormatter) + list_shared_configs.add_argument('BASEDN', help='The search DN') + list_shared_configs.set_defaults(func=dna_config_list) + +- config = subcommands.add_parser('config', help='Manage plugin configs') ++ config = subcommands.add_parser('config', help='Manage plugin configs', formatter_class=CustomHelpFormatter) + config.add_argument('NAME', help='The DNA configuration name') + config_subcommands = config.add_subparsers(help='action') +- add = config_subcommands.add_parser('add', help='Add the config entry') ++ add = config_subcommands.add_parser('add', help='Add the config entry', formatter_class=CustomHelpFormatter) + add.set_defaults(func=dna_add) + _add_parser_args(add) +- edit = config_subcommands.add_parser('set', help='Edit the config entry') ++ edit = config_subcommands.add_parser('set', help='Edit the config entry', formatter_class=CustomHelpFormatter) + edit.set_defaults(func=dna_edit) + _add_parser_args(edit) +- show = config_subcommands.add_parser('show', help='Display the config entry') ++ show = config_subcommands.add_parser('show', help='Display the config entry', formatter_class=CustomHelpFormatter) + show.set_defaults(func=dna_show) +- delete = config_subcommands.add_parser('delete', help='Delete the config entry') ++ delete = config_subcommands.add_parser('delete', help='Delete the config entry', formatter_class=CustomHelpFormatter) + delete.set_defaults(func=dna_del) + +- shared_config = config_subcommands.add_parser('shared-config-entry', help='Manage the shared config entry') ++ shared_config = config_subcommands.add_parser('shared-config-entry', help='Manage the shared config entry', formatter_class=CustomHelpFormatter) + shared_config.add_argument('SHARED_CFG', + help='Use HOSTNAME:PORT for this argument to identify the host name and port of a server in a shared range, as part of the DNA range ' + 'configuration for that specific host in multi-supplier replication. (dnaHostname+dnaPortNum)') + shared_config_subcommands = shared_config.add_subparsers(help='action') +- edit_config = shared_config_subcommands.add_parser('set', help='Edit the shared config entry') ++ edit_config = shared_config_subcommands.add_parser('set', help='Edit the shared config entry', formatter_class=CustomHelpFormatter) + edit_config.set_defaults(func=dna_config_edit) + edit_config.add_argument('--remote-bind-method', help='Specifies the remote bind method "SIMPLE", "SSL" (for SSL client auth), "SASL/GSSAPI", or "SASL/DIGEST-MD5" (dnaRemoteBindMethod)') + edit_config.add_argument('--remote-conn-protocol', help='Specifies the remote connection protocol "LDAP", or "TLS" (dnaRemoteConnProtocol)') + +- show_config_parser = shared_config_subcommands.add_parser('show', help='Display the shared config entry') ++ show_config_parser = shared_config_subcommands.add_parser('show', help='Display the shared config entry', formatter_class=CustomHelpFormatter) + show_config_parser.set_defaults(func=dna_config_show) +- del_config_parser = shared_config_subcommands.add_parser('delete', help='Delete the shared config entry') ++ del_config_parser = shared_config_subcommands.add_parser('delete', help='Delete the shared config entry', formatter_class=CustomHelpFormatter) + del_config_parser.set_defaults(func=dna_config_del) +diff --git a/src/lib389/lib389/cli_conf/plugins/entryuuid.py b/src/lib389/lib389/cli_conf/plugins/entryuuid.py +index af5ffd4e3..98506b6fc 100644 +--- a/src/lib389/lib389/cli_conf/plugins/entryuuid.py ++++ b/src/lib389/lib389/cli_conf/plugins/entryuuid.py +@@ -11,6 +11,7 @@ + from lib389.plugins import EntryUUIDPlugin, EntryUUIDFixupTasks + from lib389.cli_conf import add_generic_plugin_parsers + from lib389.utils import get_task_status ++from lib389.cli_base import CustomHelpFormatter + + + def do_fixup(inst, basedn, log, args): +@@ -41,12 +42,12 @@ def do_fixup_status(inst, basedn, log, args): + + + def create_parser(subparsers): +- referint = subparsers.add_parser('entryuuid', help='Manage and configure EntryUUID plugin') ++ referint = subparsers.add_parser('entryuuid', help='Manage and configure EntryUUID plugin', formatter_class=CustomHelpFormatter) + subcommands = referint.add_subparsers(help='action') + + add_generic_plugin_parsers(subcommands, EntryUUIDPlugin) + +- fixup = subcommands.add_parser('fixup', help='Run the fix-up task for EntryUUID plugin') ++ fixup = subcommands.add_parser('fixup', help='Run the fix-up task for EntryUUID plugin', formatter_class=CustomHelpFormatter) + fixup.set_defaults(func=do_fixup) + fixup.add_argument('DN', help="Base DN that contains entries to fix up") + fixup.add_argument('-f', '--filter', +@@ -57,7 +58,7 @@ def create_parser(subparsers): + fixup.add_argument('--timeout', type=int, default=0, + help="Sets the task timeout. Default is 0 (no timeout)") + +- fixup_status = subcommands.add_parser('fixup-status', help='Check the status of a fix-up task') ++ fixup_status = subcommands.add_parser('fixup-status', help='Check the status of a fix-up task', formatter_class=CustomHelpFormatter) + fixup_status.set_defaults(func=do_fixup_status) + fixup_status.add_argument('--dn', help="The task entry's DN") + fixup_status.add_argument('--show-log', action='store_true', help="Display the task log") +diff --git a/src/lib389/lib389/cli_conf/plugins/ldappassthrough.py b/src/lib389/lib389/cli_conf/plugins/ldappassthrough.py +index 584297ed4..b540b3462 100644 +--- a/src/lib389/lib389/cli_conf/plugins/ldappassthrough.py ++++ b/src/lib389/lib389/cli_conf/plugins/ldappassthrough.py +@@ -10,7 +10,7 @@ + import json + import ldap + from lib389.plugins import (PassThroughAuthenticationPlugin) +- ++from lib389.cli_base import CustomHelpFormatter + from lib389.cli_conf import add_generic_plugin_parsers, generic_object_edit, generic_object_add, generic_show, generic_enable, generic_disable, generic_status + + +@@ -129,20 +129,20 @@ def create_parser(subparsers): + + add_generic_plugin_parsers(subcommands, PassThroughAuthenticationPlugin) + +- list_urls = subcommands.add_parser('list', help='Lists LDAP URLs') ++ list_urls = subcommands.add_parser('list', help='Lists LDAP URLs', formatter_class=CustomHelpFormatter) + list_urls.set_defaults(func=pta_list) + +- # url = subcommands.add_parser('url', help='Manage PTA LDAP URL configurations') ++ # url = subcommands.add_parser('url', help='Manage PTA LDAP URL configurations', formatter_class=CustomHelpFormatter) + # subcommands_url = url.add_subparsers(help='action') + +- add_url = subcommands.add_parser('add', help='Add an LDAP url to the config entry') ++ add_url = subcommands.add_parser('add', help='Add an LDAP url to the config entry', formatter_class=CustomHelpFormatter) + add_url.add_argument('URL', + help='The full LDAP URL in format ' + '"ldap|ldaps://authDS/subtree maxconns,maxops,timeout,ldver,connlifetime,startTLS". ' + 'If one optional parameter is specified the rest should be specified too') + add_url.set_defaults(func=pta_add) + +- edit_url = subcommands.add_parser('modify', help='Edit the LDAP pass through config entry') ++ edit_url = subcommands.add_parser('modify', help='Edit the LDAP pass through config entry', formatter_class=CustomHelpFormatter) + edit_url.add_argument('OLD_URL', help='The full LDAP URL you get from the "list" command') + edit_url.add_argument('NEW_URL', + help='Sets the full LDAP URL in format ' +@@ -150,7 +150,7 @@ def create_parser(subparsers): + 'If one optional parameter is specified the rest should be specified too.') + edit_url.set_defaults(func=pta_edit) + +- delete_url = subcommands.add_parser('delete', help='Delete a URL from the config entry') ++ delete_url = subcommands.add_parser('delete', help='Delete a URL from the config entry', formatter_class=CustomHelpFormatter) + delete_url.add_argument('URL', help='The full LDAP URL you get from the "list" command') + delete_url.set_defaults(func=pta_del) + +diff --git a/src/lib389/lib389/cli_conf/plugins/linkedattr.py b/src/lib389/lib389/cli_conf/plugins/linkedattr.py +index 3af8c3a64..3c400a582 100644 +--- a/src/lib389/lib389/cli_conf/plugins/linkedattr.py ++++ b/src/lib389/lib389/cli_conf/plugins/linkedattr.py +@@ -11,6 +11,7 @@ import json + import ldap + from lib389.plugins import LinkedAttributesPlugin, LinkedAttributesConfig, LinkedAttributesConfigs, LinkedAttributesFixupTasks + from lib389.cli_conf import add_generic_plugin_parsers, generic_object_edit, generic_object_add ++from lib389.cli_base import CustomHelpFormatter + from lib389.utils import get_task_status + + arg_to_attr = { +@@ -110,36 +111,36 @@ def _add_parser_args(parser): + + + def create_parser(subparsers): +- linkedattr_parser = subparsers.add_parser('linked-attr', help='Manage and configure Linked Attributes plugin') ++ linkedattr_parser = subparsers.add_parser('linked-attr', help='Manage and configure Linked Attributes plugin', formatter_class=CustomHelpFormatter) + subcommands = linkedattr_parser.add_subparsers(help='action') + add_generic_plugin_parsers(subcommands, LinkedAttributesPlugin) + +- fixup_parser = subcommands.add_parser('fixup', help='Run the fix-up task for linked attributes plugin') ++ fixup_parser = subcommands.add_parser('fixup', help='Run the fix-up task for linked attributes plugin', formatter_class=CustomHelpFormatter) + fixup_parser.add_argument('-l', '--linkdn', help="Sets the base DN that contains entries to fix up") + fixup_parser.add_argument('--wait', action='store_true', + help="Wait for the task to finish, this could take a long time") + fixup_parser.set_defaults(func=fixup) + +- fixup_status = subcommands.add_parser('fixup-status', help='Check the status of a fix-up task') ++ fixup_status = subcommands.add_parser('fixup-status', help='Check the status of a fix-up task', formatter_class=CustomHelpFormatter) + fixup_status.set_defaults(func=do_fixup_status) + fixup_status.add_argument('--dn', help="The task entry's DN") + fixup_status.add_argument('--show-log', action='store_true', help="Display the task log") + fixup_status.add_argument('--watch', action='store_true', + help="Watch the task's status and wait for it to finish") + +- list = subcommands.add_parser('list', help='List available plugin configs') ++ list = subcommands.add_parser('list', help='List available plugin configs', formatter_class=CustomHelpFormatter) + list.set_defaults(func=linkedattr_list) + +- config = subcommands.add_parser('config', help='Manage plugin configs') ++ config = subcommands.add_parser('config', help='Manage plugin configs', formatter_class=CustomHelpFormatter) + config.add_argument('NAME', help='The Linked Attributes configuration name') + config_subcommands = config.add_subparsers(help='action') +- add = config_subcommands.add_parser('add', help='Add the config entry') ++ add = config_subcommands.add_parser('add', help='Add the config entry', formatter_class=CustomHelpFormatter) + add.set_defaults(func=linkedattr_add) + _add_parser_args(add) +- edit = config_subcommands.add_parser('set', help='Edit the config entry') ++ edit = config_subcommands.add_parser('set', help='Edit the config entry', formatter_class=CustomHelpFormatter) + edit.set_defaults(func=linkedattr_edit) + _add_parser_args(edit) +- show = config_subcommands.add_parser('show', help='Display the config entry') ++ show = config_subcommands.add_parser('show', help='Display the config entry', formatter_class=CustomHelpFormatter) + show.set_defaults(func=linkedattr_show) +- delete = config_subcommands.add_parser('delete', help='Delete the config entry') ++ delete = config_subcommands.add_parser('delete', help='Delete the config entry', formatter_class=CustomHelpFormatter) + delete.set_defaults(func=linkedattr_del) +diff --git a/src/lib389/lib389/cli_conf/plugins/managedentries.py b/src/lib389/lib389/cli_conf/plugins/managedentries.py +index 4dd9cce53..f8d133c13 100644 +--- a/src/lib389/lib389/cli_conf/plugins/managedentries.py ++++ b/src/lib389/lib389/cli_conf/plugins/managedentries.py +@@ -11,6 +11,7 @@ import json + from lib389.backend import Backends + from lib389.plugins import ManagedEntriesPlugin, MEPConfig, MEPConfigs, MEPTemplate, MEPTemplates + from lib389.cli_conf import add_generic_plugin_parsers, generic_object_edit, generic_object_add ++from lib389.cli_base import CustomHelpFormatter + + arg_to_attr = { + 'config_area': 'nsslapd-pluginconfigarea' +@@ -201,15 +202,15 @@ def _add_parser_args_template(parser): + + + def create_parser(subparsers): +- mep = subparsers.add_parser('managed-entries', help='Manage and configure Managed Entries Plugin') ++ mep = subparsers.add_parser('managed-entries', help='Manage and configure Managed Entries Plugin', formatter_class=CustomHelpFormatter) + subcommands = mep.add_subparsers(help='action') + add_generic_plugin_parsers(subcommands, ManagedEntriesPlugin) + +- edit = subcommands.add_parser('set', help='Edit the plugin settings') ++ edit = subcommands.add_parser('set', help='Edit the plugin settings', formatter_class=CustomHelpFormatter) + edit.set_defaults(func=mep_edit) + edit.add_argument('--config-area', help='Sets the value of the nsslapd-pluginConfigArea attribute') + +- list = subcommands.add_parser('list', help='List Managed Entries Plugin configs and templates') ++ list = subcommands.add_parser('list', help='List Managed Entries Plugin configs and templates', formatter_class=CustomHelpFormatter) + subcommands_list = list.add_subparsers(help='action') + list_configs = subcommands_list.add_parser('configs', help='List Managed Entries Plugin configs (list config-area ' + 'if specified in the main plugin entry)') +@@ -219,30 +220,30 @@ def create_parser(subparsers): + list_templates.add_argument('BASEDN', nargs='?', help='The base DN where to search the templates') + list_templates.set_defaults(func=mep_template_list) + +- config = subcommands.add_parser('config', help='Handle Managed Entries Plugin configs') ++ config = subcommands.add_parser('config', help='Handle Managed Entries Plugin configs', formatter_class=CustomHelpFormatter) + config.add_argument('NAME', help='The config entry CN') + config_subcommands = config.add_subparsers(help='action') +- add = config_subcommands.add_parser('add', help='Add the config entry') ++ add = config_subcommands.add_parser('add', help='Add the config entry', formatter_class=CustomHelpFormatter) + add.set_defaults(func=mep_config_add) + _add_parser_args_config(add) +- edit = config_subcommands.add_parser('set', help='Edit the config entry') ++ edit = config_subcommands.add_parser('set', help='Edit the config entry', formatter_class=CustomHelpFormatter) + edit.set_defaults(func=mep_config_edit) + _add_parser_args_config(edit) +- show = config_subcommands.add_parser('show', help='Display the config entry') ++ show = config_subcommands.add_parser('show', help='Display the config entry', formatter_class=CustomHelpFormatter) + show.set_defaults(func=mep_config_show) +- delete = config_subcommands.add_parser('delete', help='Delete the config entry') ++ delete = config_subcommands.add_parser('delete', help='Delete the config entry', formatter_class=CustomHelpFormatter) + delete.set_defaults(func=mep_config_del) + +- template = subcommands.add_parser('template', help='Handle Managed Entries Plugin templates') ++ template = subcommands.add_parser('template', help='Handle Managed Entries Plugin templates', formatter_class=CustomHelpFormatter) + template.add_argument('DN', help='The template entry DN.') + template_subcommands = template.add_subparsers(help='action') +- add = template_subcommands.add_parser('add', help='Add the template entry') ++ add = template_subcommands.add_parser('add', help='Add the template entry', formatter_class=CustomHelpFormatter) + add.set_defaults(func=mep_template_add) + _add_parser_args_template(add) +- edit = template_subcommands.add_parser('set', help='Edit the template entry') ++ edit = template_subcommands.add_parser('set', help='Edit the template entry', formatter_class=CustomHelpFormatter) + edit.set_defaults(func=mep_template_edit) + _add_parser_args_template(edit) +- show = template_subcommands.add_parser('show', help='Display the template entry') ++ show = template_subcommands.add_parser('show', help='Display the template entry', formatter_class=CustomHelpFormatter) + show.set_defaults(func=mep_template_show) +- delete = template_subcommands.add_parser('delete', help='Delete the template entry') ++ delete = template_subcommands.add_parser('delete', help='Delete the template entry', formatter_class=CustomHelpFormatter) + delete.set_defaults(func=mep_template_del) +diff --git a/src/lib389/lib389/cli_conf/plugins/memberof.py b/src/lib389/lib389/cli_conf/plugins/memberof.py +index 2d0f80c57..90c1af2c3 100644 +--- a/src/lib389/lib389/cli_conf/plugins/memberof.py ++++ b/src/lib389/lib389/cli_conf/plugins/memberof.py +@@ -12,6 +12,7 @@ import ldap + from lib389.plugins import MemberOfPlugin, MemberOfSharedConfig, MemberOfFixupTasks + from lib389.utils import get_task_status + from lib389.cli_conf import add_generic_plugin_parsers, generic_object_edit, generic_object_add ++from lib389.cli_base import CustomHelpFormatter + + arg_to_attr = { + 'initfunc': 'nsslapd-pluginInitfunc', +@@ -121,35 +122,35 @@ def _add_parser_args(parser): + + + def create_parser(subparsers): +- memberof = subparsers.add_parser('memberof', help='Manage and configure MemberOf plugin') ++ memberof = subparsers.add_parser('memberof', help='Manage and configure MemberOf plugin', formatter_class=CustomHelpFormatter) + + subcommands = memberof.add_subparsers(help='action') + + add_generic_plugin_parsers(subcommands, MemberOfPlugin) + +- edit = subcommands.add_parser('set', help='Edit the plugin settings') ++ edit = subcommands.add_parser('set', help='Edit the plugin settings', formatter_class=CustomHelpFormatter) + edit.set_defaults(func=memberof_edit) + _add_parser_args(edit) + edit.add_argument('--config-entry', help='The value to set as nsslapd-pluginConfigArea') + +- config = subcommands.add_parser('config-entry', help='Manage the config entry') ++ config = subcommands.add_parser('config-entry', help='Manage the config entry', formatter_class=CustomHelpFormatter) + config_subcommands = config.add_subparsers(help='action') +- add_config = config_subcommands.add_parser('add', help='Add the config entry') ++ add_config = config_subcommands.add_parser('add', help='Add the config entry', formatter_class=CustomHelpFormatter) + add_config.set_defaults(func=memberof_add_config) + add_config.add_argument('DN', help='The config entry full DN') + _add_parser_args(add_config) +- edit_config = config_subcommands.add_parser('set', help='Edit the config entry') ++ edit_config = config_subcommands.add_parser('set', help='Edit the config entry', formatter_class=CustomHelpFormatter) + edit_config.set_defaults(func=memberof_edit_config) + edit_config.add_argument('DN', help='The config entry full DN') + _add_parser_args(edit_config) +- show_config = config_subcommands.add_parser('show', help='Display the config entry') ++ show_config = config_subcommands.add_parser('show', help='Display the config entry', formatter_class=CustomHelpFormatter) + show_config.set_defaults(func=memberof_show_config) + show_config.add_argument('DN', help='The config entry full DN') +- del_config_ = config_subcommands.add_parser('delete', help='Delete the config entry') ++ del_config_ = config_subcommands.add_parser('delete', help='Delete the config entry', formatter_class=CustomHelpFormatter) + del_config_.set_defaults(func=memberof_del_config) + del_config_.add_argument('DN', help='The config entry full DN') + +- fixup = subcommands.add_parser('fixup', help='Run the fix-up task for memberOf plugin') ++ fixup = subcommands.add_parser('fixup', help='Run the fix-up task for memberOf plugin', formatter_class=CustomHelpFormatter) + fixup.set_defaults(func=do_fixup) + fixup.add_argument('DN', help="Base DN that contains entries to fix up") + fixup.add_argument('-f', '--filter', +@@ -161,7 +162,7 @@ def create_parser(subparsers): + fixup.add_argument('--timeout', type=int, default=0, + help="Sets the task timeout. ,Default is 0 (no timeout)") + +- fixup_status = subcommands.add_parser('fixup-status', help='Check the status of a fix-up task') ++ fixup_status = subcommands.add_parser('fixup-status', help='Check the status of a fix-up task', formatter_class=CustomHelpFormatter) + fixup_status.set_defaults(func=do_fixup_status) + fixup_status.add_argument('--dn', help="The task entry's DN") + fixup_status.add_argument('--show-log', action='store_true', help="Display the task log") +diff --git a/src/lib389/lib389/cli_conf/plugins/pampassthrough.py b/src/lib389/lib389/cli_conf/plugins/pampassthrough.py +index 810f24422..5dbfacec5 100644 +--- a/src/lib389/lib389/cli_conf/plugins/pampassthrough.py ++++ b/src/lib389/lib389/cli_conf/plugins/pampassthrough.py +@@ -10,7 +10,7 @@ import json + import ldap + from lib389.plugins import (PAMPassThroughAuthPlugin, + PAMPassThroughAuthConfigs, PAMPassThroughAuthConfig) +- ++from lib389.cli_base import CustomHelpFormatter + from lib389.cli_conf import add_generic_plugin_parsers, generic_object_edit, generic_object_add + + +@@ -114,20 +114,20 @@ def create_parser(subparsers): + + add_generic_plugin_parsers(subcommands, PAMPassThroughAuthPlugin) + +- list_pam = subcommands.add_parser('list', help='Lists PAM configurations') ++ list_pam = subcommands.add_parser('list', help='Lists PAM configurations', formatter_class=CustomHelpFormatter) + list_pam.set_defaults(func=pam_pta_list) + +- pam = subcommands.add_parser('config', help='Manage PAM PTA configurations.') ++ pam = subcommands.add_parser('config', help='Manage PAM PTA configurations.', formatter_class=CustomHelpFormatter) + pam.add_argument('NAME', help='The PAM PTA configuration name') + subcommands_pam = pam.add_subparsers(help='action') + +- add = subcommands_pam.add_parser('add', help='Add the config entry') ++ add = subcommands_pam.add_parser('add', help='Add the config entry', formatter_class=CustomHelpFormatter) + add.set_defaults(func=pam_pta_add) + _add_parser_args_pam(add) +- edit = subcommands_pam.add_parser('set', help='Edit the config entry') ++ edit = subcommands_pam.add_parser('set', help='Edit the config entry', formatter_class=CustomHelpFormatter) + edit.set_defaults(func=pam_pta_edit) + _add_parser_args_pam(edit) +- show = subcommands_pam.add_parser('show', help='Display the config entry') ++ show = subcommands_pam.add_parser('show', help='Display the config entry', formatter_class=CustomHelpFormatter) + show.set_defaults(func=pam_pta_show) +- delete = subcommands_pam.add_parser('delete', help='Delete the config entry') ++ delete = subcommands_pam.add_parser('delete', help='Delete the config entry', formatter_class=CustomHelpFormatter) + delete.set_defaults(func=pam_pta_del) +diff --git a/src/lib389/lib389/cli_conf/plugins/posix_winsync.py b/src/lib389/lib389/cli_conf/plugins/posix_winsync.py +index 8a97ef422..4c7377330 100644 +--- a/src/lib389/lib389/cli_conf/plugins/posix_winsync.py ++++ b/src/lib389/lib389/cli_conf/plugins/posix_winsync.py +@@ -8,6 +8,7 @@ + + from lib389.plugins import POSIXWinsyncPlugin + from lib389.cli_conf import add_generic_plugin_parsers, generic_object_edit ++from lib389.cli_base import CustomHelpFormatter + + arg_to_attr = { + 'create_memberof_task': 'posixWinsyncCreateMemberOfTask', +@@ -61,15 +62,15 @@ def _add_parser_args(parser): + + + def create_parser(subparsers): +- winsync = subparsers.add_parser('posix-winsync', help='Manage and configure the Posix Winsync API plugin') ++ winsync = subparsers.add_parser('posix-winsync', help='Manage and configure the Posix Winsync API plugin', formatter_class=CustomHelpFormatter) + subcommands = winsync.add_subparsers(help='action') + add_generic_plugin_parsers(subcommands, POSIXWinsyncPlugin) + +- edit = subcommands.add_parser('set', help='Edit the plugin settings') ++ edit = subcommands.add_parser('set', help='Edit the plugin settings', formatter_class=CustomHelpFormatter) + edit.set_defaults(func=winsync_edit) + _add_parser_args(edit) + +- fixup = subcommands.add_parser('fixup', help='Run the memberOf fix-up task to correct mismatched member and uniquemember values for synced users') ++ fixup = subcommands.add_parser('fixup', help='Run the memberOf fix-up task to correct mismatched member and uniquemember values for synced users', formatter_class=CustomHelpFormatter) + fixup.set_defaults(func=do_fixup) + fixup.add_argument('DN', help="Set the base DN that contains entries to fix up") + fixup.add_argument('-f', '--filter', +diff --git a/src/lib389/lib389/cli_conf/plugins/referint.py b/src/lib389/lib389/cli_conf/plugins/referint.py +index c46da6d75..b3982f3a2 100644 +--- a/src/lib389/lib389/cli_conf/plugins/referint.py ++++ b/src/lib389/lib389/cli_conf/plugins/referint.py +@@ -9,6 +9,7 @@ + import ldap + from lib389.plugins import ReferentialIntegrityPlugin, ReferentialIntegrityConfig + from lib389.cli_conf import add_generic_plugin_parsers, generic_object_edit, generic_object_add ++from lib389.cli_base import CustomHelpFormatter + + arg_to_attr = { + 'update_delay': 'referint-update-delay', +@@ -95,24 +96,24 @@ def create_parser(subparsers): + + add_generic_plugin_parsers(subcommands, ReferentialIntegrityPlugin) + +- edit = subcommands.add_parser('set', help='Edit the plugin settings') ++ edit = subcommands.add_parser('set', help='Edit the plugin settings', formatter_class=CustomHelpFormatter) + edit.set_defaults(func=referint_edit) + _add_parser_args(edit) + edit.add_argument('--config-entry', help='The value to set as nsslapd-pluginConfigArea') + +- config = subcommands.add_parser('config-entry', help='Manage the config entry') ++ config = subcommands.add_parser('config-entry', help='Manage the config entry', formatter_class=CustomHelpFormatter) + config_subcommands = config.add_subparsers(help='action') +- add_config = config_subcommands.add_parser('add', help='Add the config entry') ++ add_config = config_subcommands.add_parser('add', help='Add the config entry', formatter_class=CustomHelpFormatter) + add_config.set_defaults(func=referint_add_config) + add_config.add_argument('DN', help='The config entry full DN') + _add_parser_args(add_config) +- edit_config = config_subcommands.add_parser('set', help='Edit the config entry') ++ edit_config = config_subcommands.add_parser('set', help='Edit the config entry', formatter_class=CustomHelpFormatter) + edit_config.set_defaults(func=referint_edit_config) + edit_config.add_argument('DN', help='The config entry full DN') + _add_parser_args(edit_config) +- show_config = config_subcommands.add_parser('show', help='Display the config entry') ++ show_config = config_subcommands.add_parser('show', help='Display the config entry', formatter_class=CustomHelpFormatter) + show_config.set_defaults(func=referint_show_config) + show_config.add_argument('DN', help='The config entry full DN') +- del_config_ = config_subcommands.add_parser('delete', help='Delete the config entry') ++ del_config_ = config_subcommands.add_parser('delete', help='Delete the config entry', formatter_class=CustomHelpFormatter) + del_config_.set_defaults(func=referint_del_config) + del_config_.add_argument('DN', help='The config entry full DN') +diff --git a/src/lib389/lib389/cli_conf/plugins/retrochangelog.py b/src/lib389/lib389/cli_conf/plugins/retrochangelog.py +index a33c77c39..336f7e83f 100644 +--- a/src/lib389/lib389/cli_conf/plugins/retrochangelog.py ++++ b/src/lib389/lib389/cli_conf/plugins/retrochangelog.py +@@ -7,6 +7,7 @@ + # --- END COPYRIGHT BLOCK --- + from lib389.plugins import RetroChangelogPlugin + from lib389.cli_conf import add_generic_plugin_parsers, generic_object_edit, generic_object_add_attr, generic_object_del_attr ++from lib389.cli_base import CustomHelpFormatter + + arg_to_attr = { + 'is_replicated': 'isReplicated', +@@ -58,18 +59,18 @@ def _add_parser_args(parser): + + + def create_parser(subparsers): +- retrochangelog = subparsers.add_parser('retro-changelog', help='Manage and configure Retro Changelog plugin') ++ retrochangelog = subparsers.add_parser('retro-changelog', help='Manage and configure Retro Changelog plugin', formatter_class=CustomHelpFormatter) + subcommands = retrochangelog.add_subparsers(help='action') + add_generic_plugin_parsers(subcommands, RetroChangelogPlugin) + +- edit = subcommands.add_parser('set', help='Edit the plugin') ++ edit = subcommands.add_parser('set', help='Edit the plugin', formatter_class=CustomHelpFormatter) + edit.set_defaults(func=retrochangelog_edit) + _add_parser_args(edit) + +- addp = subcommands.add_parser('add', help='Add attributes to the plugin') ++ addp = subcommands.add_parser('add', help='Add attributes to the plugin', formatter_class=CustomHelpFormatter) + addp.set_defaults(func=retrochangelog_add) + _add_parser_args(addp) + +- delp = subcommands.add_parser('del', help='Delete an attribute from plugin scope') ++ delp = subcommands.add_parser('del', help='Delete an attribute from plugin scope', formatter_class=CustomHelpFormatter) + delp.set_defaults(func=retrochangelog_del) + _add_parser_args(delp) +diff --git a/src/lib389/lib389/cli_conf/plugins/rootdn_ac.py b/src/lib389/lib389/cli_conf/plugins/rootdn_ac.py +index 8c36f3186..65486fff8 100644 +--- a/src/lib389/lib389/cli_conf/plugins/rootdn_ac.py ++++ b/src/lib389/lib389/cli_conf/plugins/rootdn_ac.py +@@ -10,6 +10,7 @@ import socket + from lib389.plugins import RootDNAccessControlPlugin + from lib389.utils import is_valid_hostname + from lib389.cli_conf import add_generic_plugin_parsers, generic_object_edit ++from lib389.cli_base import CustomHelpFormatter + + arg_to_attr = { + 'allow_host': 'rootdn-allow-host', +@@ -124,11 +125,11 @@ def _add_parser_args(parser): + + + def create_parser(subparsers): +- rootdnac_parser = subparsers.add_parser('root-dn', help='Manage and configure RootDN Access Control plugin') ++ rootdnac_parser = subparsers.add_parser('root-dn', help='Manage and configure RootDN Access Control plugin', formatter_class=CustomHelpFormatter) + subcommands = rootdnac_parser.add_subparsers(help='action') + add_generic_plugin_parsers(subcommands, RootDNAccessControlPlugin) + +- edit = subcommands.add_parser('set', help='Edit the plugin settings') ++ edit = subcommands.add_parser('set', help='Edit the plugin settings', formatter_class=CustomHelpFormatter) + edit.set_defaults(func=rootdn_edit) + _add_parser_args(edit) + +diff --git a/src/lib389/lib389/cli_conf/plugins/usn.py b/src/lib389/lib389/cli_conf/plugins/usn.py +index bbc55da75..ed62856fe 100644 +--- a/src/lib389/lib389/cli_conf/plugins/usn.py ++++ b/src/lib389/lib389/cli_conf/plugins/usn.py +@@ -8,6 +8,7 @@ + + from lib389.plugins import USNPlugin + from lib389.cli_conf import add_generic_plugin_parsers ++from lib389.cli_base import CustomHelpFormatter + + + def display_usn_mode(inst, basedn, log, args): +@@ -48,19 +49,19 @@ def tombstone_cleanup(inst, basedn, log, args): + + + def create_parser(subparsers): +- usn_parser = subparsers.add_parser('usn', help='Manage and configure USN plugin') ++ usn_parser = subparsers.add_parser('usn', help='Manage and configure USN plugin', formatter_class=CustomHelpFormatter) + subcommands = usn_parser.add_subparsers(help='action') + add_generic_plugin_parsers(subcommands, USNPlugin) + +- global_mode_parser = subcommands.add_parser('global', help='Get or manage global USN mode (nsslapd-entryusn-global)') ++ global_mode_parser = subcommands.add_parser('global', help='Get or manage global USN mode (nsslapd-entryusn-global)', formatter_class=CustomHelpFormatter) + global_mode_parser.set_defaults(func=display_usn_mode) + global_mode_subcommands = global_mode_parser.add_subparsers(help='action') +- on_global_mode_parser = global_mode_subcommands.add_parser('on', help='Enables USN global mode') ++ on_global_mode_parser = global_mode_subcommands.add_parser('on', help='Enables USN global mode', formatter_class=CustomHelpFormatter) + on_global_mode_parser.set_defaults(func=enable_global_mode) +- off_global_mode_parser = global_mode_subcommands.add_parser('off', help='Disables USN global mode') ++ off_global_mode_parser = global_mode_subcommands.add_parser('off', help='Disables USN global mode', formatter_class=CustomHelpFormatter) + off_global_mode_parser.set_defaults(func=disable_global_mode) + +- cleanup_parser = subcommands.add_parser('cleanup', help='Runs the USN tombstone cleanup task') ++ cleanup_parser = subcommands.add_parser('cleanup', help='Runs the USN tombstone cleanup task', formatter_class=CustomHelpFormatter) + cleanup_parser.set_defaults(func=tombstone_cleanup) + cleanup_group = cleanup_parser.add_mutually_exclusive_group(required=True) + cleanup_group.add_argument('-s', '--suffix', +diff --git a/src/lib389/lib389/cli_conf/pwpolicy.py b/src/lib389/lib389/cli_conf/pwpolicy.py +index bce40604a..2d4ba9b21 100644 +--- a/src/lib389/lib389/cli_conf/pwpolicy.py ++++ b/src/lib389/lib389/cli_conf/pwpolicy.py +@@ -13,6 +13,7 @@ from lib389.utils import ensure_str + from lib389.pwpolicy import PwPolicyEntries, PwPolicyManager + from lib389.password_plugins import PasswordPlugins + from lib389.idm.account import Account ++from lib389.cli_base import CustomHelpFormatter + + + def _args_to_attrs(args, arg_to_attr): +@@ -212,23 +213,23 @@ def list_schemes(inst, basedn, log, args): + + def create_parser(subparsers): + # Create our two parsers for local and global policies +- globalpwp_parser = subparsers.add_parser('pwpolicy', help='Manage the global password policy settings') +- localpwp_parser = subparsers.add_parser('localpwp', help='Manage the local user and subtree password policies') ++ globalpwp_parser = subparsers.add_parser('pwpolicy', help='Manage the global password policy settings', formatter_class=CustomHelpFormatter) ++ localpwp_parser = subparsers.add_parser('localpwp', help='Manage the local user and subtree password policies', formatter_class=CustomHelpFormatter) + + ############################################ + # Local password policies + ############################################ + local_subcommands = localpwp_parser.add_subparsers(help='Local password policy') + # List all the local policies +- list_parser = local_subcommands.add_parser('list', help='List all the local password policies') ++ list_parser = local_subcommands.add_parser('list', help='List all the local password policies', formatter_class=CustomHelpFormatter) + list_parser.set_defaults(func=list_policies) + list_parser.add_argument('DN', nargs='?', help='Suffix to search for local password policies') + # Get a local policy +- get_parser = local_subcommands.add_parser('get', help='Get local password policy entry') ++ get_parser = local_subcommands.add_parser('get', help='Get local password policy entry', formatter_class=CustomHelpFormatter) + get_parser.set_defaults(func=get_local_policy) + get_parser.add_argument('DN', nargs=1, help='Get the local policy for this entry DN') + # The "set" arguments... +- set_parser = local_subcommands.add_parser('set', help='Set an attribute in a local password policy') ++ set_parser = local_subcommands.add_parser('set', help='Set an attribute in a local password policy', formatter_class=CustomHelpFormatter) + set_parser.set_defaults(func=set_local_policy) + # General settings + set_parser.add_argument('--pwdscheme', help="The password storage scheme") +@@ -278,18 +279,18 @@ def create_parser(subparsers): + set_parser.add_argument('--pwptprdelayexpireat', help="Number of seconds after which a reset password expires") + set_parser.add_argument('--pwptprdelayvalidfrom', help="Number of seconds to wait before using a reset password to authenticated") + # delete local password policy +- del_parser = local_subcommands.add_parser('remove', help='Remove a local password policy') ++ del_parser = local_subcommands.add_parser('remove', help='Remove a local password policy', formatter_class=CustomHelpFormatter) + del_parser.set_defaults(func=del_local_policy) + del_parser.add_argument('DN', nargs=1, help='Remove local policy for this entry DN') + # + # create USER local password policy + # +- add_user_parser = local_subcommands.add_parser('adduser', add_help=False, parents=[set_parser], help='Add new user password policy') ++ add_user_parser = local_subcommands.add_parser('adduser', add_help=False, parents=[set_parser], help='Add new user password policy', formatter_class=CustomHelpFormatter) + add_user_parser.set_defaults(func=create_user_policy) + # + # create SUBTREE local password policy + # +- add_subtree_parser = local_subcommands.add_parser('addsubtree', add_help=False, parents=[set_parser], help='Add new subtree password policy') ++ add_subtree_parser = local_subcommands.add_parser('addsubtree', add_help=False, parents=[set_parser], help='Add new subtree password policy', formatter_class=CustomHelpFormatter) + add_subtree_parser.set_defaults(func=create_subtree_policy) + + ########################################### +@@ -297,7 +298,7 @@ def create_parser(subparsers): + ########################################### + global_subcommands = globalpwp_parser.add_subparsers(help='Global password policy') + # Get policy +- get_global_parser = global_subcommands.add_parser('get', help='Get the global password policy entry') ++ get_global_parser = global_subcommands.add_parser('get', help='Get the global password policy entry', formatter_class=CustomHelpFormatter) + get_global_parser.set_defaults(func=get_global_policy) + # Set policy + set_global_parser = global_subcommands.add_parser('set', add_help=False, parents=[set_parser], +@@ -308,7 +309,7 @@ def create_parser(subparsers): + set_global_parser.add_argument('--pwdallowhash', help="Set to \"on\" to allow adding prehashed passwords") + set_global_parser.add_argument('--pwpinheritglobal', help="Set to \"on\" to allow local policies to inherit the global policy") + # list password storage schemes +- list_scehmes_parser = global_subcommands.add_parser('list-schemes', help='Get a list of the current password storage schemes') ++ list_scehmes_parser = global_subcommands.add_parser('list-schemes', help='Get a list of the current password storage schemes', formatter_class=CustomHelpFormatter) + list_scehmes_parser.set_defaults(func=list_schemes) + + ############################################# +diff --git a/src/lib389/lib389/cli_conf/replication.py b/src/lib389/lib389/cli_conf/replication.py +index ccc394255..399d0d2f8 100644 +--- a/src/lib389/lib389/cli_conf/replication.py ++++ b/src/lib389/lib389/cli_conf/replication.py +@@ -16,7 +16,7 @@ from shutil import copyfile + from getpass import getpass + from lib389._constants import ReplicaRole, DSRC_HOME + from lib389.cli_base.dsrc import dsrc_to_repl_monitor +-from lib389.cli_base import _get_arg ++from lib389.cli_base import _get_arg, CustomHelpFormatter + from lib389.utils import is_a_dn, copy_with_permissions, get_passwd_from_file + from lib389.replica import Replicas, ReplicationMonitor, BootstrapReplicationManager, Changelog5, ChangelogLDIF + from lib389.tasks import CleanAllRUVTask, AbortCleanAllRUVTask +@@ -1210,10 +1210,10 @@ def create_parser(subparsers): + # Replication Configuration + ############################################ + +- repl_parser = subparsers.add_parser('replication', help='Manage replication for a suffix') ++ repl_parser = subparsers.add_parser('replication', help='Manage replication for a suffix', formatter_class=CustomHelpFormatter) + repl_subcommands = repl_parser.add_subparsers(help='Replication Configuration') + +- repl_enable_parser = repl_subcommands.add_parser('enable', help='Enable replication for a suffix') ++ repl_enable_parser = repl_subcommands.add_parser('enable', help='Enable replication for a suffix', formatter_class=CustomHelpFormatter) + repl_enable_parser.set_defaults(func=enable_replication) + repl_enable_parser.add_argument('--suffix', required=True, help='Sets the DN of the suffix to be enabled for replication') + repl_enable_parser.add_argument('--role', required=True, help="Sets the replication role: \"supplier\", \"hub\", or \"consumer\"") +@@ -1226,18 +1226,18 @@ def create_parser(subparsers): + repl_enable_parser.add_argument('--bind-passwd-file', help="File containing the password") + repl_enable_parser.add_argument('--bind-passwd-prompt', action='store_true', help="Prompt for password") + +- repl_disable_parser = repl_subcommands.add_parser('disable', help='Disable replication for a suffix') ++ repl_disable_parser = repl_subcommands.add_parser('disable', help='Disable replication for a suffix', formatter_class=CustomHelpFormatter) + repl_disable_parser.set_defaults(func=disable_replication) + repl_disable_parser.add_argument('--suffix', required=True, help='Sets the DN of the suffix to have replication disabled') + +- repl_ruv_parser = repl_subcommands.add_parser('get-ruv', help='Display the database RUV entry for a suffix') ++ repl_ruv_parser = repl_subcommands.add_parser('get-ruv', help='Display the database RUV entry for a suffix', formatter_class=CustomHelpFormatter) + repl_ruv_parser.set_defaults(func=get_ruv) + repl_ruv_parser.add_argument('--suffix', required=True, help='Sets the DN of the replicated suffix') + +- repl_list_parser = repl_subcommands.add_parser('list', help='Lists all the replicated suffixes') ++ repl_list_parser = repl_subcommands.add_parser('list', help='Lists all the replicated suffixes', formatter_class=CustomHelpFormatter) + repl_list_parser.set_defaults(func=list_suffixes) + +- repl_status_parser = repl_subcommands.add_parser('status', help='Display the current status of all the replication agreements') ++ repl_status_parser = repl_subcommands.add_parser('status', help='Display the current status of all the replication agreements', formatter_class=CustomHelpFormatter) + repl_status_parser.set_defaults(func=get_repl_status) + repl_status_parser.add_argument('--suffix', required=True, help="Sets the DN of the replication suffix") + repl_status_parser.add_argument('--bind-dn', help="Sets the DN to use to authenticate to the consumer. If not set, current instance's root DN will be used. It will be used for all agreements") +@@ -1254,7 +1254,7 @@ def create_parser(subparsers): + repl_winsync_status_parser.add_argument('--bind-passwd-file', help="File containing the password. Currectly not used") + repl_winsync_status_parser.add_argument('--bind-passwd-prompt', action='store_true', help="Prompt for password. Currectly not used") + +- repl_promote_parser = repl_subcommands.add_parser('promote', help='Promote a replica to a hub or supplier') ++ repl_promote_parser = repl_subcommands.add_parser('promote', help='Promote a replica to a hub or supplier', formatter_class=CustomHelpFormatter) + repl_promote_parser.set_defaults(func=promote_replica) + repl_promote_parser.add_argument('--suffix', required=True, help="Sets the DN of the replication suffix to promote") + repl_promote_parser.add_argument('--newrole', required=True, help='Sets the new replica role to \"hub\" or \"supplier\"') +@@ -1262,7 +1262,7 @@ def create_parser(subparsers): + repl_promote_parser.add_argument('--bind-group-dn', help="Sets a group entry DN containing members that are \"bind/supplier\" DNs") + repl_promote_parser.add_argument('--bind-dn', help="Sets the bind or supplier DN that can make replication updates") + +- repl_add_manager_parser = repl_subcommands.add_parser('create-manager', help='Create a replication manager entry') ++ repl_add_manager_parser = repl_subcommands.add_parser('create-manager', help='Create a replication manager entry', formatter_class=CustomHelpFormatter) + repl_add_manager_parser.set_defaults(func=create_repl_manager) + repl_add_manager_parser.add_argument('--name', help="Sets the name of the new replication manager entry.For example, " + + "if the name is \"replication manager\" then the new manager " + +@@ -1273,28 +1273,28 @@ def create_parser(subparsers): + repl_add_manager_parser.add_argument('--suffix', help='The DN of the replication suffix whose replication ' + + 'configuration you want to add this new manager to (OPTIONAL)') + +- repl_del_manager_parser = repl_subcommands.add_parser('delete-manager', help='Delete a replication manager entry') ++ repl_del_manager_parser = repl_subcommands.add_parser('delete-manager', help='Delete a replication manager entry', formatter_class=CustomHelpFormatter) + repl_del_manager_parser.set_defaults(func=del_repl_manager) + repl_del_manager_parser.add_argument('--name', help="Sets the name of the replication manager entry under cn=config: \"cn=NAME,cn=config\"") + repl_del_manager_parser.add_argument('--suffix', help='Sets the DN of the replication suffix whose replication ' + + 'configuration you want to remove this manager from (OPTIONAL)') + +- repl_demote_parser = repl_subcommands.add_parser('demote', help='Demote replica to a hub or consumer') ++ repl_demote_parser = repl_subcommands.add_parser('demote', help='Demote replica to a hub or consumer', formatter_class=CustomHelpFormatter) + repl_demote_parser.set_defaults(func=demote_replica) + repl_demote_parser.add_argument('--suffix', required=True, help="Sets the DN of the replication suffix") + repl_demote_parser.add_argument('--newrole', required=True, help="Sets the new replication role to \"hub\", or \"consumer\"") + +- repl_get_parser = repl_subcommands.add_parser('get', help='Display the replication configuration') ++ repl_get_parser = repl_subcommands.add_parser('get', help='Display the replication configuration', formatter_class=CustomHelpFormatter) + repl_get_parser.set_defaults(func=get_repl_config) + repl_get_parser.add_argument('--suffix', required=True, help='Sets the suffix DN for the replication configuration to display') + +- repl_create_cl = repl_subcommands.add_parser('create-changelog', help='Create the replication changelog') ++ repl_create_cl = repl_subcommands.add_parser('create-changelog', help='Create the replication changelog', formatter_class=CustomHelpFormatter) + repl_create_cl.set_defaults(func=create_cl) + +- repl_delete_cl = repl_subcommands.add_parser('delete-changelog', help='Delete the replication changelog. This will invalidate any existing replication agreements') ++ repl_delete_cl = repl_subcommands.add_parser('delete-changelog', help='Delete the replication changelog. This will invalidate any existing replication agreements', formatter_class=CustomHelpFormatter) + repl_delete_cl.set_defaults(func=delete_cl) + +- repl_set_cl = repl_subcommands.add_parser('set-changelog', help='Set replication changelog attributes.') ++ repl_set_cl = repl_subcommands.add_parser('set-changelog', help='Set replication changelog attributes.', formatter_class=CustomHelpFormatter) + repl_set_cl.set_defaults(func=set_cl) + repl_set_cl.add_argument('--cl-dir', help="The replication changelog location on the filesystem") + repl_set_cl.add_argument('--max-entries', help="The maximum number of entries to get in the replication changelog") +@@ -1304,10 +1304,10 @@ def create_parser(subparsers): + 'has been reached: Use this format to set the hour and minute: HH:MM') + repl_set_cl.add_argument('--trim-interval', help="The interval to check if the replication changelog can be trimmed") + +- repl_get_cl = repl_subcommands.add_parser('get-changelog', help='Display replication changelog attributes.') ++ repl_get_cl = repl_subcommands.add_parser('get-changelog', help='Display replication changelog attributes.', formatter_class=CustomHelpFormatter) + repl_get_cl.set_defaults(func=get_cl) + +- repl_dump_cl = repl_subcommands.add_parser('dump-changelog', help='Decode Directory Server replication change log and dump it to an LDIF') ++ repl_dump_cl = repl_subcommands.add_parser('dump-changelog', help='Decode Directory Server replication change log and dump it to an LDIF', formatter_class=CustomHelpFormatter) + repl_dump_cl.set_defaults(func=dump_cl) + repl_dump_cl.add_argument('-c', '--csn-only', action='store_true', + help="Dump and interpret CSN only. This option can be used with or without -i option.") +@@ -1322,9 +1322,9 @@ def create_parser(subparsers): + "roots may be seperated by comma. All the replica roots would be dumped if the option is omitted.") + + repl_restore_cl = repl_subcommands.add_parser('restore-changelog', +- help='Restore Directory Server replication change log from LDIF file or change log directory') ++ help='Restore Directory Server replication change log from LDIF file or change log directory', formatter_class=CustomHelpFormatter) + restore_subcommands = repl_restore_cl.add_subparsers(help='Restore Replication Changelog') +- restore_ldif = restore_subcommands.add_parser('from-ldif', help='Restore a single LDIF file.') ++ restore_ldif = restore_subcommands.add_parser('from-ldif', help='Restore a single LDIF file.', formatter_class=CustomHelpFormatter) + restore_ldif.set_defaults(func=restore_cl_ldif) + restore_ldif.add_argument('LDIF_PATH', nargs=1, help='The path of changelog LDIF file.') + restore_ldif.add_argument('-r', '--replica-root', nargs=1, required=True, +@@ -1342,7 +1342,7 @@ def create_parser(subparsers): + help="Specify replica roots whose changelog you want to restore. The replica " + "roots may be separated by comma. All the replica roots would be dumped if the option is omitted.") + +- repl_set_parser = repl_subcommands.add_parser('set', help='Set an attribute in the replication configuration') ++ repl_set_parser = repl_subcommands.add_parser('set', help='Set an attribute in the replication configuration', formatter_class=CustomHelpFormatter) + repl_set_parser.set_defaults(func=set_repl_config) + repl_set_parser.add_argument('--suffix', required=True, help='Sets the DN of the replication suffix') + repl_set_parser.add_argument('--repl-add-bind-dn', help="Adds a bind (supplier) DN") +@@ -1366,7 +1366,7 @@ def create_parser(subparsers): + "an internal update to keep the RUV from getting stale. " + "The default is 1 hour (3600 seconds)") + +- repl_monitor_parser = repl_subcommands.add_parser('monitor', help='Display the full replication topology report') ++ repl_monitor_parser = repl_subcommands.add_parser('monitor', help='Display the full replication topology report', formatter_class=CustomHelpFormatter) + repl_monitor_parser.set_defaults(func=get_repl_monitor_info) + repl_monitor_parser.add_argument('-c', '--connections', nargs="*", + help="Sets the connection values for monitoring other not connected topologies. " +@@ -1381,47 +1381,47 @@ def create_parser(subparsers): + # Replication Agmts + ############################################ + +- agmt_parser = subparsers.add_parser('repl-agmt', help='Manage replication agreements') ++ agmt_parser = subparsers.add_parser('repl-agmt', help='Manage replication agreements', formatter_class=CustomHelpFormatter) + agmt_subcommands = agmt_parser.add_subparsers(help='Replication Agreement Configuration') + + # List +- agmt_list_parser = agmt_subcommands.add_parser('list', help='List all replication agreements') ++ agmt_list_parser = agmt_subcommands.add_parser('list', help='List all replication agreements', formatter_class=CustomHelpFormatter) + agmt_list_parser.set_defaults(func=list_agmts) + agmt_list_parser.add_argument('--suffix', required=True, help='Sets the DN of the suffix to look up replication agreements for') + agmt_list_parser.add_argument('--entry', help='Returns the entire entry for each agreement') + + # Enable +- agmt_enable_parser = agmt_subcommands.add_parser('enable', help='Enable replication agreement') ++ agmt_enable_parser = agmt_subcommands.add_parser('enable', help='Enable replication agreement', formatter_class=CustomHelpFormatter) + agmt_enable_parser.set_defaults(func=enable_agmt) + agmt_enable_parser.add_argument('AGMT_NAME', nargs=1, help='The name of the replication agreement') + agmt_enable_parser.add_argument('--suffix', required=True, help="Sets the DN of the replication suffix") + + # Disable +- agmt_disable_parser = agmt_subcommands.add_parser('disable', help='Disable replication agreement') ++ agmt_disable_parser = agmt_subcommands.add_parser('disable', help='Disable replication agreement', formatter_class=CustomHelpFormatter) + agmt_disable_parser.set_defaults(func=disable_agmt) + agmt_disable_parser.add_argument('AGMT_NAME', nargs=1, help='The name of the replication agreement') + agmt_disable_parser.add_argument('--suffix', required=True, help="Sets the DN of the replication suffix") + + # Initialize +- agmt_init_parser = agmt_subcommands.add_parser('init', help='Initialize replication agreement') ++ agmt_init_parser = agmt_subcommands.add_parser('init', help='Initialize replication agreement', formatter_class=CustomHelpFormatter) + agmt_init_parser.set_defaults(func=init_agmt) + agmt_init_parser.add_argument('AGMT_NAME', nargs=1, help='The name of the replication agreement') + agmt_init_parser.add_argument('--suffix', required=True, help="Sets the DN of the replication suffix") + + # Check Initialization progress +- agmt_check_init_parser = agmt_subcommands.add_parser('init-status', help='Check the agreement initialization status') ++ agmt_check_init_parser = agmt_subcommands.add_parser('init-status', help='Check the agreement initialization status', formatter_class=CustomHelpFormatter) + agmt_check_init_parser.set_defaults(func=check_init_agmt) + agmt_check_init_parser.add_argument('AGMT_NAME', nargs=1, help='The name of the replication agreement') + agmt_check_init_parser.add_argument('--suffix', required=True, help="Sets the DN of the replication suffix") + + # Send Updates Now +- agmt_poke_parser = agmt_subcommands.add_parser('poke', help='Trigger replication to send updates now') ++ agmt_poke_parser = agmt_subcommands.add_parser('poke', help='Trigger replication to send updates now', formatter_class=CustomHelpFormatter) + agmt_poke_parser.set_defaults(func=poke_agmt) + agmt_poke_parser.add_argument('AGMT_NAME', nargs=1, help='The name of the replication agreement') + agmt_poke_parser.add_argument('--suffix', required=True, help="Sets the DN of the replication suffix") + + # Status +- agmt_status_parser = agmt_subcommands.add_parser('status', help='Displays the current status of the replication agreement') ++ agmt_status_parser = agmt_subcommands.add_parser('status', help='Displays the current status of the replication agreement', formatter_class=CustomHelpFormatter) + agmt_status_parser.set_defaults(func=get_agmt_status) + agmt_status_parser.add_argument('AGMT_NAME', nargs=1, help='The name of the replication agreement') + agmt_status_parser.add_argument('--suffix', required=True, help="Sets the DN of the replication suffix") +@@ -1431,13 +1431,13 @@ def create_parser(subparsers): + agmt_status_parser.add_argument('--bind-passwd-prompt', action='store_true', help="Prompt for passwords for each agreement's instance separately") + + # Delete +- agmt_del_parser = agmt_subcommands.add_parser('delete', help='Delete replication agreement') ++ agmt_del_parser = agmt_subcommands.add_parser('delete', help='Delete replication agreement', formatter_class=CustomHelpFormatter) + agmt_del_parser.set_defaults(func=delete_agmt) + agmt_del_parser.add_argument('AGMT_NAME', nargs=1, help='The name of the replication agreement') + agmt_del_parser.add_argument('--suffix', required=True, help="Sets the DN of the replication suffix") + + # Create +- agmt_add_parser = agmt_subcommands.add_parser('create', help='Initialize replication agreement') ++ agmt_add_parser = agmt_subcommands.add_parser('create', help='Initialize replication agreement', formatter_class=CustomHelpFormatter) + agmt_add_parser.set_defaults(func=add_agmt) + agmt_add_parser.add_argument('AGMT_NAME', nargs=1, help='The name of the replication agreement') + agmt_add_parser.add_argument('--suffix', required=True, help="Sets the DN of the replication suffix") +@@ -1482,7 +1482,7 @@ def create_parser(subparsers): + agmt_add_parser.add_argument('--init', action='store_true', default=False, help="Initializes the agreement after creating it") + + # Set - Note can not use add's parent args because for "set" there are no "required=True" args +- agmt_set_parser = agmt_subcommands.add_parser('set', help='Set an attribute in the replication agreement') ++ agmt_set_parser = agmt_subcommands.add_parser('set', help='Set an attribute in the replication agreement', formatter_class=CustomHelpFormatter) + agmt_set_parser.set_defaults(func=set_agmt) + agmt_set_parser.add_argument('AGMT_NAME', nargs=1, help='The name of the replication agreement') + agmt_set_parser.add_argument('--suffix', required=True, help="Sets the DN of the replication suffix") +@@ -1525,7 +1525,7 @@ def create_parser(subparsers): + agmt_set_parser.add_argument('--bootstrap-bind-method', help="Sets the bind method: \"SIMPLE\", or \"SSLCLIENTAUTH\"") + + # Get +- agmt_get_parser = agmt_subcommands.add_parser('get', help='Get replication configuration') ++ agmt_get_parser = agmt_subcommands.add_parser('get', help='Get replication configuration', formatter_class=CustomHelpFormatter) + agmt_get_parser.set_defaults(func=get_repl_agmt) + agmt_get_parser.add_argument('AGMT_NAME', nargs=1, help='The suffix DN for which to display the replication configuration') + agmt_get_parser.add_argument('--suffix', required=True, help="Sets the DN of the replication suffix") +@@ -1534,58 +1534,58 @@ def create_parser(subparsers): + # Replication Winsync Agmts + ############################################ + +- winsync_parser = subparsers.add_parser('repl-winsync-agmt', help='Manage Winsync agreements') ++ winsync_parser = subparsers.add_parser('repl-winsync-agmt', help='Manage Winsync agreements', formatter_class=CustomHelpFormatter) + winsync_agmt_subcommands = winsync_parser.add_subparsers(help='Replication Winsync Agreement configuration') + + # List +- winsync_agmt_list_parser = winsync_agmt_subcommands.add_parser('list', help='List all the replication winsync agreements') ++ winsync_agmt_list_parser = winsync_agmt_subcommands.add_parser('list', help='List all the replication winsync agreements', formatter_class=CustomHelpFormatter) + winsync_agmt_list_parser.set_defaults(func=list_winsync_agmts) + winsync_agmt_list_parser.add_argument('--suffix', required=True, help='Sets the DN of the suffix to look up replication winsync agreements') + + # Enable +- winsync_agmt_enable_parser = winsync_agmt_subcommands.add_parser('enable', help='Enable replication winsync agreement') ++ winsync_agmt_enable_parser = winsync_agmt_subcommands.add_parser('enable', help='Enable replication winsync agreement', formatter_class=CustomHelpFormatter) + winsync_agmt_enable_parser.set_defaults(func=enable_winsync_agmt) + winsync_agmt_enable_parser.add_argument('AGMT_NAME', nargs=1, help='The name of the replication winsync agreement') + winsync_agmt_enable_parser.add_argument('--suffix', required=True, help="Sets the DN of the replication winsync suffix") + + # Disable +- winsync_agmt_disable_parser = winsync_agmt_subcommands.add_parser('disable', help='Disable replication winsync agreement') ++ winsync_agmt_disable_parser = winsync_agmt_subcommands.add_parser('disable', help='Disable replication winsync agreement', formatter_class=CustomHelpFormatter) + winsync_agmt_disable_parser.set_defaults(func=disable_winsync_agmt) + winsync_agmt_disable_parser.add_argument('AGMT_NAME', nargs=1, help='The name of the replication winsync agreement') + winsync_agmt_disable_parser.add_argument('--suffix', required=True, help="Sets the DN of the replication winsync suffix") + + # Initialize +- winsync_agmt_init_parser = winsync_agmt_subcommands.add_parser('init', help='Initialize replication winsync agreement') ++ winsync_agmt_init_parser = winsync_agmt_subcommands.add_parser('init', help='Initialize replication winsync agreement', formatter_class=CustomHelpFormatter) + winsync_agmt_init_parser.set_defaults(func=init_winsync_agmt) + winsync_agmt_init_parser.add_argument('AGMT_NAME', nargs=1, help='The name of the replication winsync agreement') + winsync_agmt_init_parser.add_argument('--suffix', required=True, help="Sets the DN of the replication winsync suffix") + + # Check Initialization progress +- winsync_agmt_check_init_parser = winsync_agmt_subcommands.add_parser('init-status', help='Check the agreement initialization status') ++ winsync_agmt_check_init_parser = winsync_agmt_subcommands.add_parser('init-status', help='Check the agreement initialization status', formatter_class=CustomHelpFormatter) + winsync_agmt_check_init_parser.set_defaults(func=check_winsync_init_agmt) + winsync_agmt_check_init_parser.add_argument('AGMT_NAME', nargs=1, help='The name of the replication agreement') + winsync_agmt_check_init_parser.add_argument('--suffix', required=True, help="Sets the DN of the replication suffix") + + # Send Updates Now +- winsync_agmt_poke_parser = winsync_agmt_subcommands.add_parser('poke', help='Trigger replication to send updates now') ++ winsync_agmt_poke_parser = winsync_agmt_subcommands.add_parser('poke', help='Trigger replication to send updates now', formatter_class=CustomHelpFormatter) + winsync_agmt_poke_parser.set_defaults(func=poke_winsync_agmt) + winsync_agmt_poke_parser.add_argument('AGMT_NAME', nargs=1, help='The name of the replication winsync agreement') + winsync_agmt_poke_parser.add_argument('--suffix', required=True, help="Sets the DN of the replication winsync suffix") + + # Status +- winsync_agmt_status_parser = winsync_agmt_subcommands.add_parser('status', help='Display the current status of the replication agreement') ++ winsync_agmt_status_parser = winsync_agmt_subcommands.add_parser('status', help='Display the current status of the replication agreement', formatter_class=CustomHelpFormatter) + winsync_agmt_status_parser.set_defaults(func=get_winsync_agmt_status) + winsync_agmt_status_parser.add_argument('AGMT_NAME', nargs=1, help='The name of the replication agreement') + winsync_agmt_status_parser.add_argument('--suffix', required=True, help="Sets the DN of the replication suffix") + + # Delete +- winsync_agmt_del_parser = winsync_agmt_subcommands.add_parser('delete', help='Delete replication winsync agreement') ++ winsync_agmt_del_parser = winsync_agmt_subcommands.add_parser('delete', help='Delete replication winsync agreement', formatter_class=CustomHelpFormatter) + winsync_agmt_del_parser.set_defaults(func=delete_winsync_agmt) + winsync_agmt_del_parser.add_argument('AGMT_NAME', nargs=1, help='The name of the replication winsync agreement') + winsync_agmt_del_parser.add_argument('--suffix', required=True, help="Sets the DN of the replication winsync suffix") + + # Create +- winsync_agmt_add_parser = winsync_agmt_subcommands.add_parser('create', help='Initialize replication winsync agreement') ++ winsync_agmt_add_parser = winsync_agmt_subcommands.add_parser('create', help='Initialize replication winsync agreement', formatter_class=CustomHelpFormatter) + winsync_agmt_add_parser.set_defaults(func=add_winsync_agmt) + winsync_agmt_add_parser.add_argument('AGMT_NAME', nargs=1, help='The name of the replication winsync agreement') + winsync_agmt_add_parser.add_argument('--suffix', required=True, help="Sets the DN of the replication winsync suffix") +@@ -1623,7 +1623,7 @@ def create_parser(subparsers): + winsync_agmt_add_parser.add_argument('--init', action='store_true', default=False, help="Initializes the agreement after creating it") + + # Set - Note can not use add's parent args because for "set" there are no "required=True" args +- winsync_agmt_set_parser = winsync_agmt_subcommands.add_parser('set', help='Set an attribute in the replication winsync agreement') ++ winsync_agmt_set_parser = winsync_agmt_subcommands.add_parser('set', help='Set an attribute in the replication winsync agreement', formatter_class=CustomHelpFormatter) + winsync_agmt_set_parser.set_defaults(func=set_winsync_agmt) + winsync_agmt_set_parser.add_argument('AGMT_NAME', nargs=1, help='The name of the replication winsync agreement') + winsync_agmt_set_parser.add_argument('--suffix', help="Sets the DN of the replication winsync suffix") +@@ -1658,7 +1658,7 @@ def create_parser(subparsers): + help="Sets the amount of time in seconds a supplier should wait between update sessions") + + # Get +- winsync_agmt_get_parser = winsync_agmt_subcommands.add_parser('get', help='Display replication configuration') ++ winsync_agmt_get_parser = winsync_agmt_subcommands.add_parser('get', help='Display replication configuration', formatter_class=CustomHelpFormatter) + winsync_agmt_get_parser.set_defaults(func=get_winsync_agmt) + winsync_agmt_get_parser.add_argument('AGMT_NAME', nargs=1, help='The suffix DN for the replication configuration to display') + winsync_agmt_get_parser.add_argument('--suffix', required=True, help="Sets the DN of the replication suffix") +@@ -1667,29 +1667,29 @@ def create_parser(subparsers): + # Replication Tasks (cleanalruv) + ############################################ + +- tasks_parser = subparsers.add_parser('repl-tasks', help='Manage replication tasks') ++ tasks_parser = subparsers.add_parser('repl-tasks', help='Manage replication tasks', formatter_class=CustomHelpFormatter) + task_subcommands = tasks_parser.add_subparsers(help='Replication tasks') + + # Cleanallruv +- task_cleanallruv = task_subcommands.add_parser('cleanallruv', help='Cleanup old/removed replica IDs') ++ task_cleanallruv = task_subcommands.add_parser('cleanallruv', help='Cleanup old/removed replica IDs', formatter_class=CustomHelpFormatter) + task_cleanallruv.set_defaults(func=run_cleanallruv) + task_cleanallruv.add_argument('--suffix', required=True, help="Sets the Directory Server suffix") + task_cleanallruv.add_argument('--replica-id', required=True, help="Sets the replica ID to remove/clean") + task_cleanallruv.add_argument('--force-cleaning', action='store_true', default=False, + help="Ignores errors and make a best attempt to clean all replicas") + +- task_cleanallruv_list = task_subcommands.add_parser('list-cleanruv-tasks', help='List all the running CleanAllRUV tasks') ++ task_cleanallruv_list = task_subcommands.add_parser('list-cleanruv-tasks', help='List all the running CleanAllRUV tasks', formatter_class=CustomHelpFormatter) + task_cleanallruv_list.set_defaults(func=list_cleanallruv) + task_cleanallruv_list.add_argument('--suffix', help="Lists only tasks for the specified suffix") + + # Abort cleanallruv +- task_abort_cleanallruv = task_subcommands.add_parser('abort-cleanallruv', help='Abort cleanallruv tasks') ++ task_abort_cleanallruv = task_subcommands.add_parser('abort-cleanallruv', help='Abort cleanallruv tasks', formatter_class=CustomHelpFormatter) + task_abort_cleanallruv.set_defaults(func=abort_cleanallruv) + task_abort_cleanallruv.add_argument('--suffix', required=True, help="Sets the Directory Server suffix") + task_abort_cleanallruv.add_argument('--replica-id', required=True, help="Sets the replica ID of the cleaning task to abort") + task_abort_cleanallruv.add_argument('--certify', action='store_true', default=False, + help="Enforces that the abort task completed on all replicas") + +- task_abort_cleanallruv_list = task_subcommands.add_parser('list-abortruv-tasks', help='List all the running CleanAllRUV abort tasks') ++ task_abort_cleanallruv_list = task_subcommands.add_parser('list-abortruv-tasks', help='List all the running CleanAllRUV abort tasks', formatter_class=CustomHelpFormatter) + task_abort_cleanallruv_list.set_defaults(func=list_abort_cleanallruv) + task_abort_cleanallruv_list.add_argument('--suffix', help="Lists only tasks for the specified suffix") +diff --git a/src/lib389/lib389/cli_conf/saslmappings.py b/src/lib389/lib389/cli_conf/saslmappings.py +index 6c11da35e..ebef86650 100644 +--- a/src/lib389/lib389/cli_conf/saslmappings.py ++++ b/src/lib389/lib389/cli_conf/saslmappings.py +@@ -18,6 +18,7 @@ from lib389.cli_base import ( + _get_arg, + _get_attributes, + _warn, ++ CustomHelpFormatter + ) + + SINGULAR = SaslMapping +@@ -96,30 +97,30 @@ def sasl_get_available(inst, basedn, log, args): + + + def create_parser(subparsers): +- sasl_parser = subparsers.add_parser('sasl', help='Manage SASL mappings') ++ sasl_parser = subparsers.add_parser('sasl', help='Manage SASL mappings', formatter_class=CustomHelpFormatter) + + subcommands = sasl_parser.add_subparsers(help='sasl') + +- list_mappings_parser = subcommands.add_parser('list', help='Display available SASL mappings') ++ list_mappings_parser = subcommands.add_parser('list', help='Display available SASL mappings', formatter_class=CustomHelpFormatter) + list_mappings_parser.set_defaults(func=sasl_map_list) + list_mappings_parser.add_argument('--details', action='store_true', default=False, + help="Displays each SASL mapping in detail") + +- get_mech_parser= subcommands.add_parser('get-mechs', help='Display the SASL mechanisms that the server will accept') ++ get_mech_parser= subcommands.add_parser('get-mechs', help='Display the SASL mechanisms that the server will accept', formatter_class=CustomHelpFormatter) + get_mech_parser.set_defaults(func=sasl_get_supported) + +- get_mech_parser= subcommands.add_parser('get-available-mechs', help='Display the SASL mechanisms that are available to the server') ++ get_mech_parser= subcommands.add_parser('get-available-mechs', help='Display the SASL mechanisms that are available to the server', formatter_class=CustomHelpFormatter) + get_mech_parser.set_defaults(func=sasl_get_available) + +- get_parser = subcommands.add_parser('get', help='Displays SASL mappings') ++ get_parser = subcommands.add_parser('get', help='Displays SASL mappings', formatter_class=CustomHelpFormatter) + get_parser.set_defaults(func=sasl_map_get) + get_parser.add_argument('selector', nargs='?', help='The SASL mapping name to display') + +- create_parser = subcommands.add_parser('create', help='Create a SASL mapping ') ++ create_parser = subcommands.add_parser('create', help='Create a SASL mapping ', formatter_class=CustomHelpFormatter) + create_parser.set_defaults(func=sasl_map_create) + populate_attr_arguments(create_parser, SaslMapping._must_attributes) + +- delete_parser = subcommands.add_parser('delete', help='Deletes the SASL object') ++ delete_parser = subcommands.add_parser('delete', help='Deletes the SASL object', formatter_class=CustomHelpFormatter) + delete_parser.set_defaults(func=sasl_map_delete) + delete_parser.add_argument('map_name', help='The SASL mapping name ("cn" value)') + +diff --git a/src/lib389/lib389/cli_conf/schema.py b/src/lib389/lib389/cli_conf/schema.py +index 60bfa02d8..577b4cea0 100644 +--- a/src/lib389/lib389/cli_conf/schema.py ++++ b/src/lib389/lib389/cli_conf/schema.py +@@ -8,7 +8,7 @@ + # --- END COPYRIGHT BLOCK --- + + from json import dumps as dump_json +-from lib389.cli_base import _get_arg ++from lib389.cli_base import _get_arg, CustomHelpFormatter + from lib389.schema import Schema, AttributeUsage, ObjectclassKind + from lib389.migrate.openldap.config import olSchema + from lib389.migrate.plan import Migration +@@ -334,59 +334,59 @@ def _add_parser_args(parser, type): + + + def create_parser(subparsers): +- schema_parser = subparsers.add_parser('schema', help='Manage the directory schema') ++ schema_parser = subparsers.add_parser('schema', help='Manage the directory schema', formatter_class=CustomHelpFormatter) + + schema_subcommands = schema_parser.add_subparsers(help='schema') +- schema_list_parser = schema_subcommands.add_parser('list', help='List all schema objects on this system') ++ schema_list_parser = schema_subcommands.add_parser('list', help='List all schema objects on this system', formatter_class=CustomHelpFormatter) + schema_list_parser.set_defaults(func=list_all) + +- attributetypes_parser = schema_subcommands.add_parser('attributetypes', help='Work with attribute types on this system') ++ attributetypes_parser = schema_subcommands.add_parser('attributetypes', help='Work with attribute types on this system', formatter_class=CustomHelpFormatter) + attributetypes_subcommands = attributetypes_parser.add_subparsers(help='schema') +- at_get_syntaxes_parser = attributetypes_subcommands.add_parser('get_syntaxes', help='List all available attribute type syntaxes') ++ at_get_syntaxes_parser = attributetypes_subcommands.add_parser('get_syntaxes', help='List all available attribute type syntaxes', formatter_class=CustomHelpFormatter) + at_get_syntaxes_parser.set_defaults(func=get_syntaxes) +- at_list_parser = attributetypes_subcommands.add_parser('list', help='List available attribute types on this system') ++ at_list_parser = attributetypes_subcommands.add_parser('list', help='List available attribute types on this system', formatter_class=CustomHelpFormatter) + at_list_parser.set_defaults(func=list_attributetypes) +- at_query_parser = attributetypes_subcommands.add_parser('query', help='Query an attribute to determine object classes that may or must take it') ++ at_query_parser = attributetypes_subcommands.add_parser('query', help='Query an attribute to determine object classes that may or must take it', formatter_class=CustomHelpFormatter) + at_query_parser.set_defaults(func=query_attributetype) + at_query_parser.add_argument('name', nargs='?', help='Attribute type to query') +- at_add_parser = attributetypes_subcommands.add_parser('add', help='Add an attribute type to this system') ++ at_add_parser = attributetypes_subcommands.add_parser('add', help='Add an attribute type to this system', formatter_class=CustomHelpFormatter) + at_add_parser.set_defaults(func=add_attributetype) + _add_parser_args(at_add_parser, 'attributetypes') + at_add_parser.add_argument('--syntax', required=True, help='OID of the LDAP syntax assigned to the attribute') +- at_edit_parser = attributetypes_subcommands.add_parser('replace', help='Replace an attribute type on this system') ++ at_edit_parser = attributetypes_subcommands.add_parser('replace', help='Replace an attribute type on this system', formatter_class=CustomHelpFormatter) + at_edit_parser.set_defaults(func=edit_attributetype) + _add_parser_args(at_edit_parser, 'attributetypes') + at_edit_parser.add_argument('--syntax', help='OID of the LDAP syntax assigned to the attribute') +- at_remove_parser = attributetypes_subcommands.add_parser('remove', help='Remove an attribute type on this system') ++ at_remove_parser = attributetypes_subcommands.add_parser('remove', help='Remove an attribute type on this system', formatter_class=CustomHelpFormatter) + at_remove_parser.set_defaults(func=remove_attributetype) + at_remove_parser.add_argument('name', help='NAME of the object') + +- objectclasses_parser = schema_subcommands.add_parser('objectclasses', help='Work with objectClasses on this system') ++ objectclasses_parser = schema_subcommands.add_parser('objectclasses', help='Work with objectClasses on this system', formatter_class=CustomHelpFormatter) + objectclasses_subcommands = objectclasses_parser.add_subparsers(help='schema') +- oc_list_parser = objectclasses_subcommands.add_parser('list', help='List available objectClasses on this system') ++ oc_list_parser = objectclasses_subcommands.add_parser('list', help='List available objectClasses on this system', formatter_class=CustomHelpFormatter) + oc_list_parser.set_defaults(func=list_objectclasses) +- oc_query_parser = objectclasses_subcommands.add_parser('query', help='Query an objectClass') ++ oc_query_parser = objectclasses_subcommands.add_parser('query', help='Query an objectClass', formatter_class=CustomHelpFormatter) + oc_query_parser.set_defaults(func=query_objectclass) + oc_query_parser.add_argument('name', nargs='?', help='ObjectClass to query') +- oc_add_parser = objectclasses_subcommands.add_parser('add', help='Add an objectClass to this system') ++ oc_add_parser = objectclasses_subcommands.add_parser('add', help='Add an objectClass to this system', formatter_class=CustomHelpFormatter) + oc_add_parser.set_defaults(func=add_objectclass) + _add_parser_args(oc_add_parser, 'objectclasses') +- oc_edit_parser = objectclasses_subcommands.add_parser('replace', help='Replace an objectClass on this system') ++ oc_edit_parser = objectclasses_subcommands.add_parser('replace', help='Replace an objectClass on this system', formatter_class=CustomHelpFormatter) + oc_edit_parser.set_defaults(func=edit_objectclass) + _add_parser_args(oc_edit_parser, 'objectclasses') +- oc_remove_parser = objectclasses_subcommands.add_parser('remove', help='Remove an objectClass on this system') ++ oc_remove_parser = objectclasses_subcommands.add_parser('remove', help='Remove an objectClass on this system', formatter_class=CustomHelpFormatter) + oc_remove_parser.set_defaults(func=remove_objectclass) + oc_remove_parser.add_argument('name', help='NAME of the object') + +- matchingrules_parser = schema_subcommands.add_parser('matchingrules', help='Work with matching rules on this system') ++ matchingrules_parser = schema_subcommands.add_parser('matchingrules', help='Work with matching rules on this system', formatter_class=CustomHelpFormatter) + matchingrules_subcommands = matchingrules_parser.add_subparsers(help='schema') +- mr_list_parser = matchingrules_subcommands.add_parser('list', help='List available matching rules on this system') ++ mr_list_parser = matchingrules_subcommands.add_parser('list', help='List available matching rules on this system', formatter_class=CustomHelpFormatter) + mr_list_parser.set_defaults(func=list_matchingrules) +- mr_query_parser = matchingrules_subcommands.add_parser('query', help='Query a matching rule') ++ mr_query_parser = matchingrules_subcommands.add_parser('query', help='Query a matching rule', formatter_class=CustomHelpFormatter) + mr_query_parser.set_defaults(func=query_matchingrule) + mr_query_parser.add_argument('name', nargs='?', help='Matching rule to query') + +- reload_parser = schema_subcommands.add_parser('reload', help='Dynamically reload schema while server is running') ++ reload_parser = schema_subcommands.add_parser('reload', help='Dynamically reload schema while server is running', formatter_class=CustomHelpFormatter) + reload_parser.set_defaults(func=reload_schema) + reload_parser.add_argument('-d', '--schemadir', help="directory where schema files are located") + reload_parser.add_argument('--wait', action='store_true', default=False, help="Wait for the reload task to complete") +diff --git a/src/lib389/lib389/cli_conf/security.py b/src/lib389/lib389/cli_conf/security.py +index db0e9cf20..eba138feb 100644 +--- a/src/lib389/lib389/cli_conf/security.py ++++ b/src/lib389/lib389/cli_conf/security.py +@@ -11,7 +11,7 @@ import json + import os + from lib389.config import Config, Encryption, RSA + from lib389.nss_ssl import NssSsl, CERT_NAME, CA_NAME +-from lib389.cli_base import _warn ++from lib389.cli_base import _warn, CustomHelpFormatter + + + Props = namedtuple('Props', ['cls', 'attr', 'help', 'values']) +@@ -103,13 +103,13 @@ def _security_generic_set(inst, basedn, log, args, attrs_map): + + + def _security_generic_get_parser(parent, attrs_map, help): +- p = parent.add_parser('get', help=help) ++ p = parent.add_parser('get', help=help, formatter_class=CustomHelpFormatter) + p.set_defaults(func=lambda *args: _security_generic_get(*args, attrs_map)) + return p + + + def _security_generic_set_parser(parent, attrs_map, help, description): +- p = parent.add_parser('set', help=help, description=description) ++ p = parent.add_parser('set', help=help, description=description, formatter_class=CustomHelpFormatter) + p.set_defaults(func=lambda *args: _security_generic_set(*args, attrs_map)) + for opt, params in attrs_map.items(): + p.add_argument(f'--{opt}', help=f'{params[2]} ({params[1]})') +@@ -133,7 +133,7 @@ def _security_generic_toggle(inst, basedn, log, args, cls, attr, value, thing): + + def _security_generic_toggle_parsers(parent, cls, attr, help_pattern): + def add_parser(action, value): +- p = parent.add_parser(action.lower(), help=help_pattern.format(action)) ++ p = parent.add_parser(action.lower(), help=help_pattern.format(action), formatter_class=CustomHelpFormatter) + p.set_defaults(func=lambda *args: _security_generic_toggle(*args, cls, attr, value, action)) + return p + +@@ -476,7 +476,7 @@ def export_cert(inst, basedn, log, args): + + + def create_parser(subparsers): +- security = subparsers.add_parser('security', help='Manage security settings') ++ security = subparsers.add_parser('security', help='Manage security settings', formatter_class=CustomHelpFormatter) + security_sub = security.add_subparsers(help='security') + + # Core security management +@@ -499,7 +499,7 @@ def create_parser(subparsers): + security_disable_plain_parser.set_defaults(func=security_disable_plaintext_port) + + # Server certificate management +- certs = security_sub.add_parser('certificate', help='Manage TLS certificates') ++ certs = security_sub.add_parser('certificate', help='Manage TLS certificates', formatter_class=CustomHelpFormatter) + certs_sub = certs.add_subparsers(help='certificate') + cert_add_parser = certs_sub.add_parser('add', help='Add a server certificate', description=( + 'Add a server certificate to the NSS database')) +@@ -533,7 +533,7 @@ def create_parser(subparsers): + cert_list_parser.set_defaults(func=cert_list) + + # CA certificate management +- cacerts = security_sub.add_parser('ca-certificate', help='Manage TLS certificate authorities') ++ cacerts = security_sub.add_parser('ca-certificate', help='Manage TLS certificate authorities', formatter_class=CustomHelpFormatter) + cacerts_sub = cacerts.add_subparsers(help='ca-certificate') + cacert_add_parser = cacerts_sub.add_parser('add', help='Add a Certificate Authority', description=( + 'Add a Certificate Authority to the NSS database')) +@@ -566,7 +566,7 @@ def create_parser(subparsers): + cacert_list_parser.set_defaults(func=cacert_list) + + # RSA management +- rsa = security_sub.add_parser('rsa', help='Query and update RSA security options') ++ rsa = security_sub.add_parser('rsa', help='Query and update RSA security options', formatter_class=CustomHelpFormatter) + rsa_sub = rsa.add_subparsers(help='rsa') + _security_generic_set_parser(rsa_sub, RSA_ATTRS_MAP, 'Set RSA security options', + ('Use this command for setting RSA (private key) related options located in cn=RSA,cn=encryption,cn=config.' +@@ -575,7 +575,7 @@ def create_parser(subparsers): + _security_generic_toggle_parsers(rsa_sub, RSA, 'nsSSLActivation', '{} RSA') + + # Cipher management +- ciphers = security_sub.add_parser('ciphers', help='Manage secure ciphers') ++ ciphers = security_sub.add_parser('ciphers', help='Manage secure ciphers', formatter_class=CustomHelpFormatter) + ciphers_sub = ciphers.add_subparsers(help='ciphers') + + ciphers_enable = ciphers_sub.add_parser('enable', help='Enable ciphers', description=( +@@ -612,7 +612,7 @@ def create_parser(subparsers): + help='Lists only supported ciphers but without enabled ciphers') + + # Certificate Signing Request Management +- csr = security_sub.add_parser('csr', help='Manage certificate signing requests') ++ csr = security_sub.add_parser('csr', help='Manage certificate signing requests', formatter_class=CustomHelpFormatter) + csr_sub = csr.add_subparsers(help='csr') + + csr_list_parser = csr_sub.add_parser('list', help='List CSRs', description=('List all CSR files in instance' +@@ -633,15 +633,15 @@ def create_parser(subparsers): + help="CSR alternative names. These are auto-detected if not provided") + csr_req_parser.set_defaults(func=csr_gen) + +- csr_delete_parser = csr_sub.add_parser('del', help='Delete a CSR file', description=('Delete a CSR file')) ++ csr_delete_parser = csr_sub.add_parser('del', help='Delete a CSR file', description=('Delete a CSR file'), formatter_class=CustomHelpFormatter) + csr_delete_parser.add_argument('name', help="Name of the CSR file to delete") + csr_delete_parser.set_defaults(func=csr_del) + + # Key Management +- key = security_sub.add_parser('key', help='Manage keys in NSS DB') ++ key = security_sub.add_parser('key', help='Manage keys in NSS DB', formatter_class=CustomHelpFormatter) + key_sub = key.add_subparsers(help='key') + +- key_list_parser = key_sub.add_parser('list', help='List all keys in NSS DB') ++ key_list_parser = key_sub.add_parser('list', help='List all keys in NSS DB', formatter_class=CustomHelpFormatter) + key_list_parser.add_argument('--orphan', action='store_true', help='List orphan keys (An orphan key is' + ' a private key in the NSS DB for which there is NO cert with the corresponding ' + ' public key). An orphan key is created during CSR generation, when the associated certificate is imported' +diff --git a/src/lib389/lib389/cli_ctl/cockpit.py b/src/lib389/lib389/cli_ctl/cockpit.py +index 22d79725e..afc724733 100644 +--- a/src/lib389/lib389/cli_ctl/cockpit.py ++++ b/src/lib389/lib389/cli_ctl/cockpit.py +@@ -8,6 +8,7 @@ + + import os + import subprocess ++from lib389.cli_base import CustomHelpFormatter + + + def enable_cockpit(inst, log, args): +@@ -67,22 +68,22 @@ def close_firewall(inst, log, args): + + + def create_parser(subparsers): +- cockpit_parser = subparsers.add_parser('cockpit', help="Enable the Cockpit interface/UI") ++ cockpit_parser = subparsers.add_parser('cockpit', help="Enable the Cockpit interface/UI", formatter_class=CustomHelpFormatter) + subcommands = cockpit_parser.add_subparsers(help="action") + + # Enable socket +- enable_parser = subcommands.add_parser('enable', help='Enable the Cockpit socket') ++ enable_parser = subcommands.add_parser('enable', help='Enable the Cockpit socket', formatter_class=CustomHelpFormatter) + enable_parser.set_defaults(func=enable_cockpit) + + # Open firewall +- open_parser = subcommands.add_parser('open-firewall', help='Open the firewall for the "cockpit" service') ++ open_parser = subcommands.add_parser('open-firewall', help='Open the firewall for the "cockpit" service', formatter_class=CustomHelpFormatter) + open_parser.add_argument('--zone', help="The firewall zone") + open_parser.set_defaults(func=open_firewall) + + # Disable socket +- disable_parser = subcommands.add_parser('disable', help='Disable the Cockpit socket') ++ disable_parser = subcommands.add_parser('disable', help='Disable the Cockpit socket', formatter_class=CustomHelpFormatter) + disable_parser.set_defaults(func=disable_cockpit) + + # Close firewall +- close_parser = subcommands.add_parser('close-firewall', help='Remove the "cockpit" service from the firewall settings') ++ close_parser = subcommands.add_parser('close-firewall', help='Remove the "cockpit" service from the firewall settings', formatter_class=CustomHelpFormatter) + close_parser.set_defaults(func=close_firewall) +diff --git a/src/lib389/lib389/cli_ctl/dbgen.py b/src/lib389/lib389/cli_ctl/dbgen.py +index 56db2fdcd..5efbc404a 100644 +--- a/src/lib389/lib389/cli_ctl/dbgen.py ++++ b/src/lib389/lib389/cli_ctl/dbgen.py +@@ -16,6 +16,7 @@ from lib389.dbgen import ( + dbgen_nested_ldif, + ) + from lib389.utils import is_a_dn ++from lib389.cli_base import CustomHelpFormatter + + DEFAULT_LDIF = "/ldifgen.ldif" + +@@ -499,11 +500,11 @@ def dbgen_create_nested(inst, log, args): + + + def create_parser(subparsers): +- db_gen_parser = subparsers.add_parser('ldifgen', help="LDIF generator to make sample LDIF files for testing") ++ db_gen_parser = subparsers.add_parser('ldifgen', help="LDIF generator to make sample LDIF files for testing", formatter_class=CustomHelpFormatter) + subcommands = db_gen_parser.add_subparsers(help="action") + + # Create just users +- dbgen_users_parser = subcommands.add_parser('users', help='Generate a LDIF containing user entries') ++ dbgen_users_parser = subcommands.add_parser('users', help='Generate a LDIF containing user entries', formatter_class=CustomHelpFormatter) + dbgen_users_parser.set_defaults(func=dbgen_create_users) + dbgen_users_parser.add_argument('--number', help="The number of users to create.") + dbgen_users_parser.add_argument('--suffix', help="The database suffix where the entries will be created.") +@@ -515,7 +516,7 @@ def create_parser(subparsers): + dbgen_users_parser.add_argument('--ldif-file', default="ldifgen.ldif", help=f"The LDIF file name. Default location is the server's LDIF directory using the name 'ldifgen.ldif'") + + # Create static groups +- dbgen_groups_parser = subcommands.add_parser('groups', help='Generate a LDIF containing groups and members') ++ dbgen_groups_parser = subcommands.add_parser('groups', help='Generate a LDIF containing groups and members', formatter_class=CustomHelpFormatter) + dbgen_groups_parser.set_defaults(func=dbgen_create_groups) + dbgen_groups_parser.add_argument('NAME', help="The group name.") + dbgen_groups_parser.add_argument('--number', default=1, help="The number of groups to create.") +@@ -528,7 +529,7 @@ def create_parser(subparsers): + dbgen_groups_parser.add_argument('--ldif-file', default="ldifgen.ldif", help=f"The LDIF file name. Default location is the server's LDIF directory using the name 'ldifgen.ldif'") + + # Create a COS definition +- dbgen_cos_def_parser = subcommands.add_parser('cos-def', help='Generate a LDIF containing a COS definition (classic, pointer, or indirect)') ++ dbgen_cos_def_parser = subcommands.add_parser('cos-def', help='Generate a LDIF containing a COS definition (classic, pointer, or indirect)', formatter_class=CustomHelpFormatter) + dbgen_cos_def_parser.set_defaults(func=dbgen_create_cos_def) + dbgen_cos_def_parser.add_argument('NAME', help="The COS definition name.") + dbgen_cos_def_parser.add_argument('--type', help="The COS definition type: \"classic\", \"pointer\", or \"indirect\".") +@@ -540,7 +541,7 @@ def create_parser(subparsers): + dbgen_cos_def_parser.add_argument('--ldif-file', default="ldifgen.ldif", help=f"The LDIF file name. Default location is the server's LDIF directory using the name 'ldifgen.ldif'") + + # Create a COS Template +- dbgen_cos_tmp_parser = subcommands.add_parser('cos-template', help='Generate a LDIF containing a COS template') ++ dbgen_cos_tmp_parser = subcommands.add_parser('cos-template', help='Generate a LDIF containing a COS template', formatter_class=CustomHelpFormatter) + dbgen_cos_tmp_parser.set_defaults(func=dbgen_create_cos_tmp) + dbgen_cos_tmp_parser.add_argument('NAME', help="The COS template name.") + dbgen_cos_tmp_parser.add_argument('--parent', help="The DN of the entry to store the COS template entry under.") +@@ -550,7 +551,7 @@ def create_parser(subparsers): + dbgen_cos_tmp_parser.add_argument('--ldif-file', default="ldifgen.ldif", help=f"The LDIF file name. Default location is the server's LDIF directory using the name 'ldifgen.ldif'") + + # Create Role entries +- dbgen_roles_parser = subcommands.add_parser('roles', help='Generate a LDIF containing a role entry (managed, filtered, or indirect)') ++ dbgen_roles_parser = subcommands.add_parser('roles', help='Generate a LDIF containing a role entry (managed, filtered, or indirect)', formatter_class=CustomHelpFormatter) + dbgen_roles_parser.set_defaults(func=dbgen_create_role) + dbgen_roles_parser.add_argument('NAME', help="The Role name.") + dbgen_roles_parser.add_argument('--type', help="The Role type: \"managed\", \"filtered\", or \"nested\".") +@@ -561,7 +562,7 @@ def create_parser(subparsers): + dbgen_roles_parser.add_argument('--ldif-file', default="ldifgen.ldif", help=f"The LDIF file name. Default location is the server's LDIF directory using the name 'ldifgen.ldif'") + + # Create a modification LDIF +- dbgen_mod_load_parser = subcommands.add_parser('mod-load', help='Generate a LDIF containing modify operations. This is intended to be consumed by ldapmodify.') ++ dbgen_mod_load_parser = subcommands.add_parser('mod-load', help='Generate a LDIF containing modify operations. This is intended to be consumed by ldapmodify.', formatter_class=CustomHelpFormatter) + dbgen_mod_load_parser.set_defaults(func=dbgen_create_mods) + dbgen_mod_load_parser.add_argument('--create-users', action='store_true', help="Create the entries that will be modified or deleted. By default the script assumes the user entries already exist.") + dbgen_mod_load_parser.add_argument('--delete-users', action='store_true', help="Delete all the user entries at the end of the LDIF.") +@@ -577,7 +578,7 @@ def create_parser(subparsers): + dbgen_mod_load_parser.add_argument('--ldif-file', default="ldifgen.ldif", help=f"The LDIF file name. Default location is the server's LDIF directory using the name 'ldifgen.ldif'") + + # Create a heavily nested LDIF +- dbgen_nested_parser = subcommands.add_parser('nested', help='Generate a heavily nested database LDIF in a cascading/fractal tree design') ++ dbgen_nested_parser = subcommands.add_parser('nested', help='Generate a heavily nested database LDIF in a cascading/fractal tree design', formatter_class=CustomHelpFormatter) + dbgen_nested_parser.set_defaults(func=dbgen_create_nested) + dbgen_nested_parser.add_argument('--num-users', help="The total number of user entries to create in the entire LDIF (does not include the container entries).") + dbgen_nested_parser.add_argument('--node-limit', help="The total number of user entries to create under each node/subtree") +diff --git a/src/lib389/lib389/cli_ctl/dbtasks.py b/src/lib389/lib389/cli_ctl/dbtasks.py +index 4bcd1f2f5..856639672 100644 +--- a/src/lib389/lib389/cli_ctl/dbtasks.py ++++ b/src/lib389/lib389/cli_ctl/dbtasks.py +@@ -9,6 +9,7 @@ + + import os + from lib389._constants import TaskWarning ++from lib389.cli_base import CustomHelpFormatter + from pathlib import Path + + +@@ -126,18 +127,18 @@ def dbtasks_verify(inst, log, args): + + + def create_parser(subcommands): +- db2index_parser = subcommands.add_parser('db2index', help="Initialise a reindex of the server database. The server must be stopped for this to proceed.") ++ db2index_parser = subcommands.add_parser('db2index', help="Initialise a reindex of the server database. The server must be stopped for this to proceed.", formatter_class=CustomHelpFormatter) + # db2index_parser.add_argument('suffix', help="The suffix to reindex. IE dc=example,dc=com.") + db2index_parser.add_argument('backend', nargs="?", help="The backend to reindex. IE userRoot", default=False) + db2index_parser.add_argument('--attr', nargs="*", help="The attribute's to reindex. IE --attr aci cn givenname", default=False) + db2index_parser.set_defaults(func=dbtasks_db2index) + +- db2bak_parser = subcommands.add_parser('db2bak', help="Initialise a BDB backup of the database. The server must be stopped for this to proceed.") ++ db2bak_parser = subcommands.add_parser('db2bak', help="Initialise a BDB backup of the database. The server must be stopped for this to proceed.", formatter_class=CustomHelpFormatter) + db2bak_parser.add_argument('archive', help="The destination for the archive. This will be created during the db2bak process.", + nargs='?', default=None) + db2bak_parser.set_defaults(func=dbtasks_db2bak) + +- db2ldif_parser = subcommands.add_parser('db2ldif', help="Initialise an LDIF dump of the database. The server must be stopped for this to proceed.") ++ db2ldif_parser = subcommands.add_parser('db2ldif', help="Initialise an LDIF dump of the database. The server must be stopped for this to proceed.", formatter_class=CustomHelpFormatter) + db2ldif_parser.add_argument('backend', help="The backend to output as an LDIF. IE userRoot") + db2ldif_parser.add_argument('ldif', help="The path to the ldif output location.", nargs='?', default=None) + db2ldif_parser.add_argument('--replication', help="Export replication information, suitable for importing on a new consumer or backups.", +@@ -148,15 +149,15 @@ def create_parser(subcommands): + db2ldif_parser.add_argument('--encrypted', help="Export encrypted attributes", default=False, action='store_true') + db2ldif_parser.set_defaults(func=dbtasks_db2ldif) + +- dbverify_parser = subcommands.add_parser('dbverify', help="Perform a db verification. You should only do this at direction of support") ++ dbverify_parser = subcommands.add_parser('dbverify', help="Perform a db verification. You should only do this at direction of support", formatter_class=CustomHelpFormatter) + dbverify_parser.add_argument('backend', help="The backend to verify. IE userRoot") + dbverify_parser.set_defaults(func=dbtasks_verify) + +- bak2db_parser = subcommands.add_parser('bak2db', help="Restore a BDB backup of the database. The server must be stopped for this to proceed.") ++ bak2db_parser = subcommands.add_parser('bak2db', help="Restore a BDB backup of the database. The server must be stopped for this to proceed.", formatter_class=CustomHelpFormatter) + bak2db_parser.add_argument('archive', help="The archive to restore. This will erase all current server databases.") + bak2db_parser.set_defaults(func=dbtasks_bak2db) + +- ldif2db_parser = subcommands.add_parser('ldif2db', help="Restore an LDIF dump of the database. The server must be stopped for this to proceed.") ++ ldif2db_parser = subcommands.add_parser('ldif2db', help="Restore an LDIF dump of the database. The server must be stopped for this to proceed.", formatter_class=CustomHelpFormatter) + ldif2db_parser.add_argument('backend', help="The backend to restore from an LDIF. IE userRoot") + ldif2db_parser.add_argument('ldif', help="The path to the ldif to import") + ldif2db_parser.add_argument('--encrypted', help="Import encrypted attributes", default=False, action='store_true') +@@ -164,10 +165,10 @@ def create_parser(subcommands): + # default=False, action='store_true') + ldif2db_parser.set_defaults(func=dbtasks_ldif2db) + +- backups_parser = subcommands.add_parser('backups', help="List backup's found in the server's default backup directory") ++ backups_parser = subcommands.add_parser('backups', help="List backup's found in the server's default backup directory", formatter_class=CustomHelpFormatter) + backups_parser.add_argument('--delete', nargs=1, help="Delete backup directory") + backups_parser.set_defaults(func=dbtasks_backups) + +- ldifs_parser = subcommands.add_parser('ldifs', help="List all the LDIF files located in the server's LDIF directory") ++ ldifs_parser = subcommands.add_parser('ldifs', help="List all the LDIF files located in the server's LDIF directory", formatter_class=CustomHelpFormatter) + ldifs_parser.add_argument('--delete', nargs=1, help="Delete LDIF file") + ldifs_parser.set_defaults(func=dbtasks_ldifs) +diff --git a/src/lib389/lib389/cli_ctl/dsrc.py b/src/lib389/lib389/cli_ctl/dsrc.py +index b54f68885..54a8b233e 100644 +--- a/src/lib389/lib389/cli_ctl/dsrc.py ++++ b/src/lib389/lib389/cli_ctl/dsrc.py +@@ -12,6 +12,7 @@ from os import path, remove + from ldapurl import isLDAPUrl + from ldap.dn import is_dn + import configparser ++from lib389.cli_base import CustomHelpFormatter + + + def create_dsrc(inst, log, args): +@@ -293,11 +294,11 @@ def display_dsrc(inst, log, args): + + + def create_parser(subparsers): +- dsrc_parser = subparsers.add_parser('dsrc', help="Manage the .dsrc file") ++ dsrc_parser = subparsers.add_parser('dsrc', help="Manage the .dsrc file", formatter_class=CustomHelpFormatter) + subcommands = dsrc_parser.add_subparsers(help="action") + + # Create .dsrc file +- dsrc_create_parser = subcommands.add_parser('create', help='Generate the .dsrc file') ++ dsrc_create_parser = subcommands.add_parser('create', help='Generate the .dsrc file', formatter_class=CustomHelpFormatter) + dsrc_create_parser.set_defaults(func=create_dsrc) + dsrc_create_parser.add_argument('--uri', help="The URI (LDAP URL) for the Directory Server instance.") + dsrc_create_parser.add_argument('--basedn', help="The default database suffix.") +@@ -315,7 +316,7 @@ def create_parser(subparsers): + dsrc_create_parser.add_argument('--pwdfile', help="The absolute path to a file containing the Bind DN's password.") + dsrc_create_parser.add_argument('--do-it', action='store_true', help="Create the file without any confirmation.") + +- dsrc_modify_parser = subcommands.add_parser('modify', help='Modify the .dsrc file') ++ dsrc_modify_parser = subcommands.add_parser('modify', help='Modify the .dsrc file', formatter_class=CustomHelpFormatter) + dsrc_modify_parser.set_defaults(func=modify_dsrc) + dsrc_modify_parser.add_argument('--uri', nargs='?', const='', help="The URI (LDAP URL) for the Directory Server instance.") + dsrc_modify_parser.add_argument('--basedn', nargs='?', const='', help="The default database suffix.") +@@ -333,11 +334,11 @@ def create_parser(subparsers): + dsrc_modify_parser.add_argument('--do-it', action='store_true', help="Update the file without any confirmation.") + + # Delete the instance from the .dsrc file +- dsrc_delete_parser = subcommands.add_parser('delete', help='Delete instance configuration from the .dsrc file.') ++ dsrc_delete_parser = subcommands.add_parser('delete', help='Delete instance configuration from the .dsrc file.', formatter_class=CustomHelpFormatter) + dsrc_delete_parser.set_defaults(func=delete_dsrc) + dsrc_delete_parser.add_argument('--do-it', action='store_true', + help="Delete this instance's configuration from the .dsrc file.") + + # Display .dsrc file +- dsrc_display_parser = subcommands.add_parser('display', help='Display the contents of the .dsrc file.') ++ dsrc_display_parser = subcommands.add_parser('display', help='Display the contents of the .dsrc file.', formatter_class=CustomHelpFormatter) + dsrc_display_parser.set_defaults(func=display_dsrc) +diff --git a/src/lib389/lib389/cli_ctl/health.py b/src/lib389/lib389/cli_ctl/health.py +index e242a4042..d85e3906a 100644 +--- a/src/lib389/lib389/cli_ctl/health.py ++++ b/src/lib389/lib389/cli_ctl/health.py +@@ -10,7 +10,7 @@ import json + import re + from lib389._mapped_object import DSLdapObjects + from lib389._mapped_object_lint import DSLint +-from lib389.cli_base import connect_instance, disconnect_instance ++from lib389.cli_base import connect_instance, disconnect_instance, CustomHelpFormatter + from lib389.cli_base.dsrc import dsrc_to_ldap, dsrc_arg_concat + from lib389.backend import Backends + from lib389.config import Encryption, Config +diff --git a/src/lib389/lib389/cli_ctl/instance.py b/src/lib389/lib389/cli_ctl/instance.py +index ae2bb1a88..23dec7810 100644 +--- a/src/lib389/lib389/cli_ctl/instance.py ++++ b/src/lib389/lib389/cli_ctl/instance.py +@@ -10,6 +10,7 @@ import os + import json + import time + from lib389 import DirSrv ++from lib389.cli_base import CustomHelpFormatter + from lib389.instance.setup import SetupDs + from lib389.utils import get_instance_list + from lib389.instance.remove import remove_ds_instance +@@ -171,23 +172,23 @@ def instance_remove(inst, log, args): + + + def create_parser(subcommands): +- # list_parser = subcommands.add_parser('list', help="List installed instances of Directory Server") ++ # list_parser = subcommands.add_parser('list', help="List installed instances of Directory Server", formatter_class=CustomHelpFormatter) + # list_parser.set_defaults(func=instance_list) + # list_parser.set_defaults(noinst=True) + +- restart_parser = subcommands.add_parser('restart', help="Restart an instance of Directory Server, if it is running: else start it.") ++ restart_parser = subcommands.add_parser('restart', help="Restart an instance of Directory Server, if it is running: else start it.", formatter_class=CustomHelpFormatter) + restart_parser.set_defaults(func=instance_restart) + +- start_parser = subcommands.add_parser('start', help="Start an instance of Directory Server, if it is not currently running") ++ start_parser = subcommands.add_parser('start', help="Start an instance of Directory Server, if it is not currently running", formatter_class=CustomHelpFormatter) + start_parser.set_defaults(func=instance_start) + +- stop_parser = subcommands.add_parser('stop', help="Stop an instance of Directory Server, if it is currently running") ++ stop_parser = subcommands.add_parser('stop', help="Stop an instance of Directory Server, if it is currently running", formatter_class=CustomHelpFormatter) + stop_parser.set_defaults(func=instance_stop) + +- status_parser = subcommands.add_parser('status', help="Check running status of an instance of Directory Server") ++ status_parser = subcommands.add_parser('status', help="Check running status of an instance of Directory Server", formatter_class=CustomHelpFormatter) + status_parser.set_defaults(func=instance_status) + +- remove_parser = subcommands.add_parser('remove', help="Destroy an instance of Directory Server, and remove all data.") ++ remove_parser = subcommands.add_parser('remove', help="Destroy an instance of Directory Server, and remove all data.", formatter_class=CustomHelpFormatter) + remove_parser.set_defaults(func=instance_remove) + remove_parser.add_argument('--do-it', dest="ack", help="By default we do a dry run. This actually initiates the removal of the instance.", + action='store_true', default=False) +diff --git a/src/lib389/lib389/cli_ctl/nsstate.py b/src/lib389/lib389/cli_ctl/nsstate.py +index 28acd99bf..00e3d407e 100644 +--- a/src/lib389/lib389/cli_ctl/nsstate.py ++++ b/src/lib389/lib389/cli_ctl/nsstate.py +@@ -8,6 +8,7 @@ + + import json + from lib389.dseldif import DSEldif ++from lib389.cli_base import CustomHelpFormatter + + + def get_nsstate(inst, log, args): +diff --git a/src/lib389/lib389/cli_ctl/tls.py b/src/lib389/lib389/cli_ctl/tls.py +index 9bfabd132..8834f5758 100644 +--- a/src/lib389/lib389/cli_ctl/tls.py ++++ b/src/lib389/lib389/cli_ctl/tls.py +@@ -8,7 +8,7 @@ + # + import os + from lib389.nss_ssl import NssSsl, CERT_NAME, CA_NAME +-from lib389.cli_base import _warn ++from lib389.cli_base import _warn, CustomHelpFormatter + + + def show_servercert(inst, log, args): +@@ -97,26 +97,27 @@ def export_cert(inst, log, args): + + + def create_parser(subparsers): +- tls_parser = subparsers.add_parser('tls', help="Manage TLS certificates") ++ tls_parser = subparsers.add_parser('tls', help="Manage TLS certificates", formatter_class=CustomHelpFormatter) + + subcommands = tls_parser.add_subparsers(help='action') + +- list_ca_parser = subcommands.add_parser('list-ca', help='list server certificate authorities including intermediates') ++ list_ca_parser = subcommands.add_parser('list-ca', help='list server certificate authorities including intermediates', formatter_class=CustomHelpFormatter) + list_ca_parser.set_defaults(func=list_cas) + +- list_client_ca_parser = subcommands.add_parser('list-client-ca', help='list client certificate authorities including intermediates') ++ list_client_ca_parser = subcommands.add_parser('list-client-ca', help='list client certificate authorities including intermediates', formatter_class=CustomHelpFormatter) + list_client_ca_parser.set_defaults(func=list_client_cas) + +- show_servercert_parser = subcommands.add_parser('show-server-cert', help='Show the active server certificate that clients will see and verify') ++ show_servercert_parser = subcommands.add_parser('show-server-cert', help='Show the active server certificate that clients will see and verify', formatter_class=CustomHelpFormatter) + show_servercert_parser.set_defaults(func=show_servercert) + +- show_cert_parser = subcommands.add_parser('show-cert', help='Show a certificate\'s details referenced by it\'s nickname. This is analogous to certutil -L -d -n ') ++ show_cert_parser = subcommands.add_parser('show-cert', help='Show a certificate\'s details referenced by it\'s nickname. This is analogous to certutil -L -d -n ', formatter_class=CustomHelpFormatter) + show_cert_parser.add_argument('nickname', help="The nickname (friendly name) of the certificate to display") + show_cert_parser.set_defaults(func=show_cert) + + generate_server_cert_csr_parser = subcommands.add_parser( + 'generate-server-cert-csr', +- help="Generate a Server-Cert certificate signing request - the csr is then submitted to a CA for verification, and when signed you import with import-ca and import-server-cert" ++ help="Generate a Server-Cert certificate signing request - the csr is then submitted to a CA for verification, and when signed you import with import-ca and import-server-cert", ++ formatter_class=CustomHelpFormatter + ) + generate_server_cert_csr_parser.add_argument('--subject', '-s', + default=None, +@@ -127,7 +128,8 @@ def create_parser(subparsers): + + import_client_ca_parser = subcommands.add_parser( + 'import-client-ca', +- help="Import a CA trusted to issue user (client) certificates. This is part of how client certificate authentication functions." ++ help="Import a CA trusted to issue user (client) certificates. This is part of how client certificate authentication functions.", ++ formatter_class=CustomHelpFormatter + ) + import_client_ca_parser.add_argument('cert_path', + help="The path to the x509 cert to import as a client trust root") +@@ -137,7 +139,8 @@ def create_parser(subparsers): + import_ca_parser = subcommands.add_parser( + 'import-ca', + help="Import a CA or intermediate CA for signing this servers certificates (aka Server-Cert). " +- "You should import all the CA's in the chain as required. PEM bundles are accepted" ++ "You should import all the CA's in the chain as required. PEM bundles are accepted", ++ formatter_class=CustomHelpFormatter + ) + import_ca_parser.add_argument('cert_path', + help="The path to the x509 cert to import as a server CA") +@@ -146,7 +149,8 @@ def create_parser(subparsers): + + import_server_cert_parser = subcommands.add_parser( + 'import-server-cert', +- help="Import a new Server-Cert after the csr has been signed from a CA." ++ help="Import a new Server-Cert after the csr has been signed from a CA.", ++ formatter_class=CustomHelpFormatter + ) + import_server_cert_parser.add_argument('cert_path', + help="The path to the x509 cert to import as Server-Cert") +@@ -154,7 +158,8 @@ def create_parser(subparsers): + + import_server_key_cert_parser = subcommands.add_parser( + 'import-server-key-cert', +- help="Import a new key and Server-Cert after having been signed from a CA. This is used if you have an external csr tool or a service like lets encrypt that generates PEM keys externally." ++ help="Import a new key and Server-Cert after having been signed from a CA. This is used if you have an external csr tool or a service like lets encrypt that generates PEM keys externally.", ++ formatter_class=CustomHelpFormatter + ) + import_server_key_cert_parser.add_argument('cert_path', + help="The path to the x509 cert to import as Server-Cert") +@@ -164,14 +169,16 @@ def create_parser(subparsers): + + remove_cert_parser = subcommands.add_parser( + 'remove-cert', +- help="Delete a certificate from this database. This will remove it from acting as a CA, a client CA or the Server-Cert role." ++ help="Delete a certificate from this database. This will remove it from acting as a CA, a client CA or the Server-Cert role.", ++ formatter_class=CustomHelpFormatter + ) + remove_cert_parser.add_argument('nickname', help="The name of the certificate to delete") + remove_cert_parser.set_defaults(func=remove_cert) + + export_cert_parser = subcommands.add_parser( + 'export-cert', +- help="Export a certificate to PEM or DER/Binary format. PEM format is the default" ++ help="Export a certificate to PEM or DER/Binary format. PEM format is the default", ++ formatter_class=CustomHelpFormatter + ) + export_cert_parser.add_argument('nickname', help="The name of the certificate to export") + export_cert_parser.add_argument('--binary-format', action='store_true', +diff --git a/src/lib389/lib389/cli_idm/account.py b/src/lib389/lib389/cli_idm/account.py +index 6c7aebce1..5d7b9cc77 100644 +--- a/src/lib389/lib389/cli_idm/account.py ++++ b/src/lib389/lib389/cli_idm/account.py +@@ -20,6 +20,7 @@ from lib389.cli_base import ( + _get_arg, + _get_dn_arg, + _warn, ++ CustomHelpFormatter + ) + from lib389.cli_idm import _generic_rename_dn + +@@ -213,42 +214,42 @@ like modify, locking and unlocking. To create an account, see "user" subcommand + + subcommands = account_parser.add_subparsers(help='action') + +- list_parser = subcommands.add_parser('list', help='list accounts that could login to the directory') ++ list_parser = subcommands.add_parser('list', help='list accounts that could login to the directory', formatter_class=CustomHelpFormatter) + list_parser.set_defaults(func=list) + +- get_dn_parser = subcommands.add_parser('get-by-dn', help='get-by-dn ') ++ get_dn_parser = subcommands.add_parser('get-by-dn', help='get-by-dn ', formatter_class=CustomHelpFormatter) + get_dn_parser.set_defaults(func=get_dn) + get_dn_parser.add_argument('dn', nargs='?', help='The dn to get and display') + +- modify_dn_parser = subcommands.add_parser('modify-by-dn', help='modify-by-dn :: ...') ++ modify_dn_parser = subcommands.add_parser('modify-by-dn', help='modify-by-dn :: ...', formatter_class=CustomHelpFormatter) + modify_dn_parser.set_defaults(func=modify) + modify_dn_parser.add_argument('dn', nargs=1, help='The dn to get and display') + modify_dn_parser.add_argument('changes', nargs='+', help="A list of changes to apply in format: ::") + +- rename_dn_parser = subcommands.add_parser('rename-by-dn', help='rename the object') ++ rename_dn_parser = subcommands.add_parser('rename-by-dn', help='rename the object', formatter_class=CustomHelpFormatter) + rename_dn_parser.set_defaults(func=rename) + rename_dn_parser.add_argument('dn', help='The dn to rename') + rename_dn_parser.add_argument('new_dn', help='A new role dn') + rename_dn_parser.add_argument('--keep-old-rdn', action='store_true', help="Specify whether the old RDN (i.e. 'cn: old_role') should be kept as an attribute of the entry or not") + +- delete_parser = subcommands.add_parser('delete', help='deletes the account') ++ delete_parser = subcommands.add_parser('delete', help='deletes the account', formatter_class=CustomHelpFormatter) + delete_parser.set_defaults(func=delete) + delete_parser.add_argument('dn', nargs='?', help='The dn of the account to delete') + +- lock_parser = subcommands.add_parser('lock', help='lock') ++ lock_parser = subcommands.add_parser('lock', help='lock', formatter_class=CustomHelpFormatter) + lock_parser.set_defaults(func=lock) + lock_parser.add_argument('dn', nargs='?', help='The dn to lock') + +- unlock_parser = subcommands.add_parser('unlock', help='unlock') ++ unlock_parser = subcommands.add_parser('unlock', help='unlock', formatter_class=CustomHelpFormatter) + unlock_parser.set_defaults(func=unlock) + unlock_parser.add_argument('dn', nargs='?', help='The dn to unlock') + +- status_parser = subcommands.add_parser('entry-status', help='status of a single entry') ++ status_parser = subcommands.add_parser('entry-status', help='status of a single entry', formatter_class=CustomHelpFormatter) + status_parser.set_defaults(func=entry_status) + status_parser.add_argument('dn', nargs='?', help='The single entry dn to check') + status_parser.add_argument('-V', '--details', action='store_true', help="Print more account policy details about the entry") + +- status_parser = subcommands.add_parser('subtree-status', help='status of a subtree') ++ status_parser = subcommands.add_parser('subtree-status', help='status of a subtree', formatter_class=CustomHelpFormatter) + status_parser.set_defaults(func=subtree_status) + status_parser.add_argument('basedn', help="Search base for finding entries") + status_parser.add_argument('-V', '--details', action='store_true', help="Print more account policy details about the entries") +@@ -258,18 +259,18 @@ like modify, locking and unlocking. To create an account, see "user" subcommand + status_parser.add_argument('-o', '--become-inactive-on', + help="Only display entries that will become inactive before specified date (in a format 2007-04-25T14:30)") + +- reset_pw_parser = subcommands.add_parser('reset_password', help='Reset the password of an account. This should be performed by a directory admin.') ++ reset_pw_parser = subcommands.add_parser('reset_password', help='Reset the password of an account. This should be performed by a directory admin.', formatter_class=CustomHelpFormatter) + reset_pw_parser.set_defaults(func=reset_password) + reset_pw_parser.add_argument('dn', nargs='?', help='The dn to reset the password for') + reset_pw_parser.add_argument('new_password', nargs='?', help='The new password to set') + +- change_pw_parser = subcommands.add_parser('change_password', help='Change the password of an account. This can be performed by any user (with correct rights)') ++ change_pw_parser = subcommands.add_parser('change_password', help='Change the password of an account. This can be performed by any user (with correct rights)', formatter_class=CustomHelpFormatter) + change_pw_parser.set_defaults(func=change_password) + change_pw_parser.add_argument('dn', nargs='?', help='The dn to change the password for') + change_pw_parser.add_argument('new_password', nargs='?', help='The new password to set') + change_pw_parser.add_argument('current_password', nargs='?', help='The accounts current password') + +- bulk_update_parser = subcommands.add_parser('bulk_update', help='Perform a common operation to a set of entries') ++ bulk_update_parser = subcommands.add_parser('bulk_update', help='Perform a common operation to a set of entries', formatter_class=CustomHelpFormatter) + bulk_update_parser.set_defaults(func=bulk_update) + bulk_update_parser.add_argument('basedn', help="Search base for finding entries, only the children of this DN are processed") + bulk_update_parser.add_argument('-f', '--filter', help="Search filter for finding entries, default is '(objectclass=*)'") +diff --git a/src/lib389/lib389/cli_idm/client_config.py b/src/lib389/lib389/cli_idm/client_config.py +index 027723c64..dd932f132 100644 +--- a/src/lib389/lib389/cli_idm/client_config.py ++++ b/src/lib389/lib389/cli_idm/client_config.py +@@ -10,6 +10,7 @@ from lib389.idm.user import nsUserAccounts + from lib389.idm.group import Groups + from lib389.plugins import MemberOfPlugin + from lib389.utils import basedn_to_ldap_dns_uri ++from lib389.cli_base import CustomHelpFormatter + + SSSD_CONF_TEMPLATE = """ + # +diff --git a/src/lib389/lib389/cli_idm/group.py b/src/lib389/lib389/cli_idm/group.py +index 3aafdf72d..08bdb36e8 100644 +--- a/src/lib389/lib389/cli_idm/group.py ++++ b/src/lib389/lib389/cli_idm/group.py +@@ -8,7 +8,7 @@ + # --- END COPYRIGHT BLOCK --- + + from lib389.idm.group import Group, Groups, MUST_ATTRIBUTES +-from lib389.cli_base import populate_attr_arguments, _generic_modify ++from lib389.cli_base import populate_attr_arguments, _generic_modify, CustomHelpFormatter + from lib389.cli_idm import ( + _generic_list, + _generic_get, +@@ -103,46 +103,46 @@ def create_parser(subparsers): + '"member"') + subcommands = group_parser.add_subparsers(help='action') + +- list_parser = subcommands.add_parser('list', help='list') ++ list_parser = subcommands.add_parser('list', help='list', formatter_class=CustomHelpFormatter) + list_parser.set_defaults(func=list) + +- get_parser = subcommands.add_parser('get', help='get') ++ get_parser = subcommands.add_parser('get', help='get', formatter_class=CustomHelpFormatter) + get_parser.set_defaults(func=get) + get_parser.add_argument('selector', nargs='?', help='The term to search for') + +- get_dn_parser = subcommands.add_parser('get_dn', help='get_dn') ++ get_dn_parser = subcommands.add_parser('get_dn', help='get_dn', formatter_class=CustomHelpFormatter) + get_dn_parser.set_defaults(func=get_dn) + get_dn_parser.add_argument('dn', nargs='?', help='The dn to get') + +- create_parser = subcommands.add_parser('create', help='create') ++ create_parser = subcommands.add_parser('create', help='create', formatter_class=CustomHelpFormatter) + create_parser.set_defaults(func=create) + populate_attr_arguments(create_parser, MUST_ATTRIBUTES) + +- delete_parser = subcommands.add_parser('delete', help='deletes the object') ++ delete_parser = subcommands.add_parser('delete', help='deletes the object', formatter_class=CustomHelpFormatter) + delete_parser.set_defaults(func=delete) + delete_parser.add_argument('dn', nargs='?', help='The dn to delete') + +- modify_parser = subcommands.add_parser('modify', help='modify :: ...') ++ modify_parser = subcommands.add_parser('modify', help='modify :: ...', formatter_class=CustomHelpFormatter) + modify_parser.set_defaults(func=modify) + modify_parser.add_argument('selector', nargs=1, help='The %s to modify' % RDN) + modify_parser.add_argument('changes', nargs='+', help="A list of changes to apply in format: ::") + +- rename_parser = subcommands.add_parser('rename', help='rename the object') ++ rename_parser = subcommands.add_parser('rename', help='rename the object', formatter_class=CustomHelpFormatter) + rename_parser.set_defaults(func=rename) + rename_parser.add_argument('selector', help='The %s to rename' % RDN) + rename_parser.add_argument('new_name', help='A new group name') + rename_parser.add_argument('--keep-old-rdn', action='store_true', help="Specify whether the old RDN (i.e. 'cn: old_group') should be kept as an attribute of the entry or not") + +- members_parser = subcommands.add_parser('members', help="List member dns of a group") ++ members_parser = subcommands.add_parser('members', help="List member dns of a group", formatter_class=CustomHelpFormatter) + members_parser.set_defaults(func=members) + members_parser.add_argument('cn', nargs='?', help="cn of group to list members of") + +- add_member_parser = subcommands.add_parser('add_member', help="Add a member to a group") ++ add_member_parser = subcommands.add_parser('add_member', help="Add a member to a group", formatter_class=CustomHelpFormatter) + add_member_parser.set_defaults(func=add_member) + add_member_parser.add_argument('cn', nargs='?', help="cn of group to add member to") + add_member_parser.add_argument('dn', nargs='?', help="dn of object to add to group as member") + +- remove_member_parser = subcommands.add_parser('remove_member', help="Remove a member from a group") ++ remove_member_parser = subcommands.add_parser('remove_member', help="Remove a member from a group", formatter_class=CustomHelpFormatter) + remove_member_parser.set_defaults(func=remove_member) + remove_member_parser.add_argument('cn', nargs='?', help="cn of group to remove member from") + remove_member_parser.add_argument('dn', nargs='?', help="dn of object to remove from group as member") +diff --git a/src/lib389/lib389/cli_idm/initialise.py b/src/lib389/lib389/cli_idm/initialise.py +index 26ae02d46..172fed7e6 100644 +--- a/src/lib389/lib389/cli_idm/initialise.py ++++ b/src/lib389/lib389/cli_idm/initialise.py +@@ -8,6 +8,7 @@ + + from lib389._constants import INSTALL_LATEST_CONFIG + from lib389.configurations import get_sample_entries ++from lib389.cli_base import CustomHelpFormatter + + def initialise(inst, basedn, log, args): + sample_entries = get_sample_entries(args.version) +@@ -16,7 +17,7 @@ def initialise(inst, basedn, log, args): + s_ent.apply() + + def create_parser(subparsers): +- initialise_parser = subparsers.add_parser('initialise', help="Initialise a backend with domain information and sample entries") ++ initialise_parser = subparsers.add_parser('initialise', help="Initialise a backend with domain information and sample entries", formatter_class=CustomHelpFormatter) + initialise_parser.set_defaults(func=initialise) + initialise_parser.add_argument('--version', help="The version of entries to create.", default=INSTALL_LATEST_CONFIG) + +diff --git a/src/lib389/lib389/cli_idm/organizationalunit.py b/src/lib389/lib389/cli_idm/organizationalunit.py +index 3e430dd09..1ec8a1a63 100644 +--- a/src/lib389/lib389/cli_idm/organizationalunit.py ++++ b/src/lib389/lib389/cli_idm/organizationalunit.py +@@ -7,7 +7,7 @@ + # --- END COPYRIGHT BLOCK --- + + from lib389.idm.organizationalunit import OrganizationalUnit, OrganizationalUnits, MUST_ATTRIBUTES +-from lib389.cli_base import populate_attr_arguments, _generic_modify ++from lib389.cli_base import populate_attr_arguments, _generic_modify, CustomHelpFormatter + from lib389.cli_idm import ( + _generic_list, + _generic_get, +@@ -56,35 +56,35 @@ def rename(inst, basedn, log, args, warn=True): + _generic_rename(inst, basedn, log.getChild('_generic_rename'), MANY, rdn, args) + + def create_parser(subparsers): +- ou_parser = subparsers.add_parser('organizationalunit', help='Manage organizational units') ++ ou_parser = subparsers.add_parser('organizationalunit', help='Manage organizational units', formatter_class=CustomHelpFormatter) + + subcommands = ou_parser.add_subparsers(help='action') + +- list_parser = subcommands.add_parser('list', help='list') ++ list_parser = subcommands.add_parser('list', help='list', formatter_class=CustomHelpFormatter) + list_parser.set_defaults(func=list) + +- get_parser = subcommands.add_parser('get', help='get') ++ get_parser = subcommands.add_parser('get', help='get', formatter_class=CustomHelpFormatter) + get_parser.set_defaults(func=get) + get_parser.add_argument('selector', nargs='?', help='The term to search for') + +- get_dn_parser = subcommands.add_parser('get_dn', help='get_dn') ++ get_dn_parser = subcommands.add_parser('get_dn', help='get_dn', formatter_class=CustomHelpFormatter) + get_dn_parser.set_defaults(func=get_dn) + get_dn_parser.add_argument('dn', nargs='?', help='The dn to get') + +- create_parser = subcommands.add_parser('create', help='create') ++ create_parser = subcommands.add_parser('create', help='create', formatter_class=CustomHelpFormatter) + create_parser.set_defaults(func=create) + populate_attr_arguments(create_parser, MUST_ATTRIBUTES) + +- delete_parser = subcommands.add_parser('delete', help='deletes the object') ++ delete_parser = subcommands.add_parser('delete', help='deletes the object', formatter_class=CustomHelpFormatter) + delete_parser.set_defaults(func=delete) + delete_parser.add_argument('dn', nargs='?', help='The dn to delete') + +- modify_parser = subcommands.add_parser('modify', help='modify :: ...') ++ modify_parser = subcommands.add_parser('modify', help='modify :: ...', formatter_class=CustomHelpFormatter) + modify_parser.set_defaults(func=modify) + modify_parser.add_argument('selector', nargs=1, help='The %s to modify' % RDN) + modify_parser.add_argument('changes', nargs='+', help="A list of changes to apply in format: ::") + +- rename_parser = subcommands.add_parser('rename', help='rename the object') ++ rename_parser = subcommands.add_parser('rename', help='rename the object', formatter_class=CustomHelpFormatter) + rename_parser.set_defaults(func=rename) + rename_parser.add_argument('selector', help='The %s to rename' % RDN) + rename_parser.add_argument('new_name', help='A new organizational unit name') +diff --git a/src/lib389/lib389/cli_idm/posixgroup.py b/src/lib389/lib389/cli_idm/posixgroup.py +index 2f5381a56..0b71d97c0 100644 +--- a/src/lib389/lib389/cli_idm/posixgroup.py ++++ b/src/lib389/lib389/cli_idm/posixgroup.py +@@ -8,7 +8,7 @@ + # --- END COPYRIGHT BLOCK --- + + from lib389.idm.posixgroup import PosixGroup, PosixGroups, MUST_ATTRIBUTES +-from lib389.cli_base import populate_attr_arguments, _generic_modify ++from lib389.cli_base import populate_attr_arguments, _generic_modify, CustomHelpFormatter + from lib389.cli_idm import ( + _generic_list, + _generic_get, +@@ -69,31 +69,31 @@ def create_parser(subparsers): + 'ou=groups") needs to exist prior to managing posix groups.') + subcommands = posixgroup_parser.add_subparsers(help='action') + +- list_parser = subcommands.add_parser('list', help='list') ++ list_parser = subcommands.add_parser('list', help='list', formatter_class=CustomHelpFormatter) + list_parser.set_defaults(func=list) + +- get_parser = subcommands.add_parser('get', help='get') ++ get_parser = subcommands.add_parser('get', help='get', formatter_class=CustomHelpFormatter) + get_parser.set_defaults(func=get) + get_parser.add_argument('selector', nargs='?', help='The term to search for') + +- get_dn_parser = subcommands.add_parser('get_dn', help='get_dn') ++ get_dn_parser = subcommands.add_parser('get_dn', help='get_dn', formatter_class=CustomHelpFormatter) + get_dn_parser.set_defaults(func=get_dn) + get_dn_parser.add_argument('dn', nargs='?', help='The dn to get') + +- create_group_parser = subcommands.add_parser('create', help='create') ++ create_group_parser = subcommands.add_parser('create', help='create', formatter_class=CustomHelpFormatter) + create_group_parser.set_defaults(func=create) + populate_attr_arguments(create_group_parser, MUST_ATTRIBUTES) + +- delete_parser = subcommands.add_parser('delete', help='deletes the object') ++ delete_parser = subcommands.add_parser('delete', help='deletes the object', formatter_class=CustomHelpFormatter) + delete_parser.set_defaults(func=delete) + delete_parser.add_argument('dn', nargs='?', help='The dn to delete') + +- modify_parser = subcommands.add_parser('modify', help='modify :: ...') ++ modify_parser = subcommands.add_parser('modify', help='modify :: ...', formatter_class=CustomHelpFormatter) + modify_parser.set_defaults(func=modify) + modify_parser.add_argument('selector', nargs=1, help='The %s to modify' % RDN) + modify_parser.add_argument('changes', nargs='+', help="A list of changes to apply in format: ::") + +- rename_parser = subcommands.add_parser('rename', help='rename the object') ++ rename_parser = subcommands.add_parser('rename', help='rename the object', formatter_class=CustomHelpFormatter) + rename_parser.set_defaults(func=rename) + rename_parser.add_argument('selector', help='The %s to rename' % RDN) + rename_parser.add_argument('new_name', help='A new posix group name') +diff --git a/src/lib389/lib389/cli_idm/role.py b/src/lib389/lib389/cli_idm/role.py +index 5bbec9ce1..4b09e1af4 100644 +--- a/src/lib389/lib389/cli_idm/role.py ++++ b/src/lib389/lib389/cli_idm/role.py +@@ -30,6 +30,7 @@ from lib389.cli_base import ( + _generic_create, + _get_dn_arg, + _warn, ++ CustomHelpFormatter + ) + from lib389.cli_idm import _generic_rename_dn + +@@ -132,61 +133,61 @@ def unlock(inst, basedn, log, args): + + + def create_parser(subparsers): +- role_parser = subparsers.add_parser('role', help='''Manage roles.''') ++ role_parser = subparsers.add_parser('role', help='''Manage roles.''', formatter_class=CustomHelpFormatter) + + subcommands = role_parser.add_subparsers(help='action') + +- list_parser = subcommands.add_parser('list', help='list roles that could login to the directory') ++ list_parser = subcommands.add_parser('list', help='list roles that could login to the directory', formatter_class=CustomHelpFormatter) + list_parser.set_defaults(func=list) + +- get_parser = subcommands.add_parser('get', help='get') ++ get_parser = subcommands.add_parser('get', help='get', formatter_class=CustomHelpFormatter) + get_parser.set_defaults(func=get) + get_parser.add_argument('selector', nargs='?', help='The term to search for') + +- get_dn_parser = subcommands.add_parser('get-by-dn', help='get-by-dn ') ++ get_dn_parser = subcommands.add_parser('get-by-dn', help='get-by-dn ', formatter_class=CustomHelpFormatter) + get_dn_parser.set_defaults(func=get_dn) + get_dn_parser.add_argument('dn', nargs='?', help='The dn to get and display') + +- create_managed_parser = subcommands.add_parser('create-managed', help='create') ++ create_managed_parser = subcommands.add_parser('create-managed', help='create', formatter_class=CustomHelpFormatter) + create_managed_parser.set_defaults(func=create_managed) + populate_attr_arguments(create_managed_parser, MUST_ATTRIBUTES) + +- create_filtered_parser = subcommands.add_parser('create-filtered', help='create') ++ create_filtered_parser = subcommands.add_parser('create-filtered', help='create', formatter_class=CustomHelpFormatter) + create_filtered_parser.set_defaults(func=create_filtered) + populate_attr_arguments(create_filtered_parser, MUST_ATTRIBUTES) + +- create_nested_parser = subcommands.add_parser('create-nested', help='create') ++ create_nested_parser = subcommands.add_parser('create-nested', help='create', formatter_class=CustomHelpFormatter) + create_nested_parser.set_defaults(func=create_nested) + populate_attr_arguments(create_nested_parser, MUST_ATTRIBUTES_NESTED) + +- modify_dn_parser = subcommands.add_parser('modify-by-dn', help='modify-by-dn :: ...') ++ modify_dn_parser = subcommands.add_parser('modify-by-dn', help='modify-by-dn :: ...', formatter_class=CustomHelpFormatter) + modify_dn_parser.set_defaults(func=modify) + modify_dn_parser.add_argument('dn', nargs=1, help='The dn to modify') + modify_dn_parser.add_argument('changes', nargs='+', help="A list of changes to apply in format: ::") + +- rename_dn_parser = subcommands.add_parser('rename-by-dn', help='rename the object') ++ rename_dn_parser = subcommands.add_parser('rename-by-dn', help='rename the object', formatter_class=CustomHelpFormatter) + rename_dn_parser.set_defaults(func=rename) + rename_dn_parser.add_argument('dn', help='The dn to rename') + rename_dn_parser.add_argument('new_dn', help='A new account dn') + rename_dn_parser.add_argument('--keep-old-rdn', action='store_true', help="Specify whether the old RDN (i.e. 'cn: old_account') should be kept as an attribute of the entry or not") + +- delete_parser = subcommands.add_parser('delete', help='deletes the role') ++ delete_parser = subcommands.add_parser('delete', help='deletes the role', formatter_class=CustomHelpFormatter) + delete_parser.set_defaults(func=delete) + delete_parser.add_argument('dn', nargs='?', help='The dn of the role to delete') + +- lock_parser = subcommands.add_parser('lock', help='lock') ++ lock_parser = subcommands.add_parser('lock', help='lock', formatter_class=CustomHelpFormatter) + lock_parser.set_defaults(func=lock) + lock_parser.add_argument('dn', nargs='?', help='The dn to lock') + +- unlock_parser = subcommands.add_parser('unlock', help='unlock') ++ unlock_parser = subcommands.add_parser('unlock', help='unlock', formatter_class=CustomHelpFormatter) + unlock_parser.set_defaults(func=unlock) + unlock_parser.add_argument('dn', nargs='?', help='The dn to unlock') + +- status_parser = subcommands.add_parser('entry-status', help='status of a single entry') ++ status_parser = subcommands.add_parser('entry-status', help='status of a single entry', formatter_class=CustomHelpFormatter) + status_parser.set_defaults(func=entry_status) + status_parser.add_argument('dn', nargs='?', help='The single entry dn to check') + +- status_parser = subcommands.add_parser('subtree-status', help='status of a subtree') ++ status_parser = subcommands.add_parser('subtree-status', help='status of a subtree', formatter_class=CustomHelpFormatter) + status_parser.set_defaults(func=subtree_status) + status_parser.add_argument('basedn', help="Search base for finding entries") + status_parser.add_argument('-f', '--filter', help="Search filter for finding entries") +diff --git a/src/lib389/lib389/cli_idm/service.py b/src/lib389/lib389/cli_idm/service.py +index da2342d12..c62fc12d1 100644 +--- a/src/lib389/lib389/cli_idm/service.py ++++ b/src/lib389/lib389/cli_idm/service.py +@@ -8,7 +8,7 @@ + # --- END COPYRIGHT BLOCK --- + + from lib389.idm.services import ServiceAccount, ServiceAccounts +-from lib389.cli_base import populate_attr_arguments, _generic_modify ++from lib389.cli_base import populate_attr_arguments, _generic_modify, CustomHelpFormatter + from lib389.cli_idm import ( + _generic_list, + _generic_get, +@@ -57,37 +57,37 @@ def rename(inst, basedn, log, args, warn=True): + _generic_rename(inst, basedn, log.getChild('_generic_rename'), MANY, rdn, args) + + def create_parser(subparsers): +- service_parser = subparsers.add_parser('service', help='Manage service accounts') ++ service_parser = subparsers.add_parser('service', help='Manage service accounts', formatter_class=CustomHelpFormatter) + + subcommands = service_parser.add_subparsers(help='action') + +- list_parser = subcommands.add_parser('list', help='list') ++ list_parser = subcommands.add_parser('list', help='list', formatter_class=CustomHelpFormatter) + list_parser.set_defaults(func=list) + +- get_parser = subcommands.add_parser('get', help='get') ++ get_parser = subcommands.add_parser('get', help='get', formatter_class=CustomHelpFormatter) + get_parser.set_defaults(func=get) + get_parser.add_argument('selector', nargs='?', help='The term to search for') + +- get_dn_parser = subcommands.add_parser('get_dn', help='get_dn') ++ get_dn_parser = subcommands.add_parser('get_dn', help='get_dn', formatter_class=CustomHelpFormatter) + get_dn_parser.set_defaults(func=get_dn) + get_dn_parser.add_argument('dn', nargs='?', help='The dn to get') + +- create_parser = subcommands.add_parser('create', help='create') ++ create_parser = subcommands.add_parser('create', help='create', formatter_class=CustomHelpFormatter) + create_parser.set_defaults(func=create) + populate_attr_arguments(create_parser, SINGULAR._must_attributes) + +- modify_parser = subcommands.add_parser('modify', help='modify :: ...') ++ modify_parser = subcommands.add_parser('modify', help='modify :: ...', formatter_class=CustomHelpFormatter) + modify_parser.set_defaults(func=modify) + modify_parser.add_argument('selector', nargs=1, help='The %s to modify' % RDN) + modify_parser.add_argument('changes', nargs='+', help="A list of changes to apply in format: ::") + +- rename_parser = subcommands.add_parser('rename', help='rename the object') ++ rename_parser = subcommands.add_parser('rename', help='rename the object', formatter_class=CustomHelpFormatter) + rename_parser.set_defaults(func=rename) + rename_parser.add_argument('selector', help='The %s to modify' % RDN) + rename_parser.add_argument('new_name', help='A new service name') + rename_parser.add_argument('--keep-old-rdn', action='store_true', help="Specify whether the old RDN (i.e. 'cn: old_service') should be kept as an attribute of the entry or not") + +- delete_parser = subcommands.add_parser('delete', help='deletes the object') ++ delete_parser = subcommands.add_parser('delete', help='deletes the object', formatter_class=CustomHelpFormatter) + delete_parser.set_defaults(func=delete) + delete_parser.add_argument('dn', nargs='?', help='The dn to delete') + +diff --git a/src/lib389/lib389/cli_idm/uniquegroup.py b/src/lib389/lib389/cli_idm/uniquegroup.py +index 19cabd570..fc3d6ae36 100644 +--- a/src/lib389/lib389/cli_idm/uniquegroup.py ++++ b/src/lib389/lib389/cli_idm/uniquegroup.py +@@ -7,7 +7,7 @@ + # --- END COPYRIGHT BLOCK --- + + from lib389.idm.group import UniqueGroup, UniqueGroups, MUST_ATTRIBUTES +-from lib389.cli_base import populate_attr_arguments, _generic_modify ++from lib389.cli_base import populate_attr_arguments, _generic_modify, CustomHelpFormatter + from lib389.cli_idm import ( + _generic_list, + _generic_get, +@@ -102,46 +102,46 @@ def create_parser(subparsers): + '"uniquemember"') + subcommands = group_parser.add_subparsers(help='action') + +- list_parser = subcommands.add_parser('list', help='list') ++ list_parser = subcommands.add_parser('list', help='list', formatter_class=CustomHelpFormatter) + list_parser.set_defaults(func=list) + +- get_parser = subcommands.add_parser('get', help='get') ++ get_parser = subcommands.add_parser('get', help='get', formatter_class=CustomHelpFormatter) + get_parser.set_defaults(func=get) + get_parser.add_argument('selector', nargs='?', help='The term to search for') + +- get_dn_parser = subcommands.add_parser('get_dn', help='get_dn') ++ get_dn_parser = subcommands.add_parser('get_dn', help='get_dn', formatter_class=CustomHelpFormatter) + get_dn_parser.set_defaults(func=get_dn) + get_dn_parser.add_argument('dn', nargs='?', help='The dn to get') + +- create_parser = subcommands.add_parser('create', help='create') ++ create_parser = subcommands.add_parser('create', help='create', formatter_class=CustomHelpFormatter) + create_parser.set_defaults(func=create) + populate_attr_arguments(create_parser, MUST_ATTRIBUTES) + +- delete_parser = subcommands.add_parser('delete', help='deletes the object') ++ delete_parser = subcommands.add_parser('delete', help='deletes the object', formatter_class=CustomHelpFormatter) + delete_parser.set_defaults(func=delete) + delete_parser.add_argument('dn', nargs='?', help='The dn to delete') + +- modify_parser = subcommands.add_parser('modify', help='modify :: ...') ++ modify_parser = subcommands.add_parser('modify', help='modify :: ...', formatter_class=CustomHelpFormatter) + modify_parser.set_defaults(func=modify) + modify_parser.add_argument('selector', nargs=1, help='The %s to modify' % RDN) + modify_parser.add_argument('changes', nargs='+', help="A list of changes to apply in format: ::") + +- rename_parser = subcommands.add_parser('rename', help='rename the object') ++ rename_parser = subcommands.add_parser('rename', help='rename the object', formatter_class=CustomHelpFormatter) + rename_parser.set_defaults(func=rename) + rename_parser.add_argument('selector', help='The %s to rename' % RDN) + rename_parser.add_argument('new_name', help='A new group name') + rename_parser.add_argument('--keep-old-rdn', action='store_true', help="Specify whether the old RDN (i.e. 'cn: old_group') should be kept as an attribute of the entry or not") + +- members_parser = subcommands.add_parser('members', help="List member dns of a group") ++ members_parser = subcommands.add_parser('members', help="List member dns of a group", formatter_class=CustomHelpFormatter) + members_parser.set_defaults(func=members) + members_parser.add_argument('cn', nargs='?', help="cn of group to list members of") + +- add_member_parser = subcommands.add_parser('add_member', help="Add a member to a group") ++ add_member_parser = subcommands.add_parser('add_member', help="Add a member to a group", formatter_class=CustomHelpFormatter) + add_member_parser.set_defaults(func=add_member) + add_member_parser.add_argument('cn', nargs='?', help="cn of group to add member to") + add_member_parser.add_argument('dn', nargs='?', help="dn of object to add to group as member") + +- remove_member_parser = subcommands.add_parser('remove_member', help="Remove a member from a group") ++ remove_member_parser = subcommands.add_parser('remove_member', help="Remove a member from a group", formatter_class=CustomHelpFormatter) + remove_member_parser.set_defaults(func=remove_member) + remove_member_parser.add_argument('cn', nargs='?', help="cn of group to remove member from") + remove_member_parser.add_argument('dn', nargs='?', help="dn of object to remove from group as member") +diff --git a/src/lib389/lib389/cli_idm/user.py b/src/lib389/lib389/cli_idm/user.py +index e25dec347..a29d8a479 100644 +--- a/src/lib389/lib389/cli_idm/user.py ++++ b/src/lib389/lib389/cli_idm/user.py +@@ -8,7 +8,7 @@ + # --- END COPYRIGHT BLOCK --- + + from lib389.idm.user import nsUserAccount, nsUserAccounts +-from lib389.cli_base import populate_attr_arguments, _generic_modify ++from lib389.cli_base import populate_attr_arguments, _generic_modify, CustomHelpFormatter + from lib389.cli_idm import ( + _generic_list, + _generic_get, +@@ -71,33 +71,33 @@ def create_parser(subparsers): + + subcommands = user_parser.add_subparsers(help='action') + +- list_parser = subcommands.add_parser('list', help='list') ++ list_parser = subcommands.add_parser('list', help='list', formatter_class=CustomHelpFormatter) + list_parser.set_defaults(func=list) + +- get_parser = subcommands.add_parser('get', help='get') ++ get_parser = subcommands.add_parser('get', help='get', formatter_class=CustomHelpFormatter) + get_parser.set_defaults(func=get) + get_parser.add_argument('selector', nargs='?', help='The term to search for') + +- get_dn_parser = subcommands.add_parser('get_dn', help='get_dn') ++ get_dn_parser = subcommands.add_parser('get_dn', help='get_dn', formatter_class=CustomHelpFormatter) + get_dn_parser.set_defaults(func=get_dn) + get_dn_parser.add_argument('dn', nargs='?', help='The dn to get') + +- create_user_parser = subcommands.add_parser('create', help='create') ++ create_user_parser = subcommands.add_parser('create', help='create', formatter_class=CustomHelpFormatter) + create_user_parser.set_defaults(func=create) + populate_attr_arguments(create_user_parser, SINGULAR._must_attributes) + +- modify_parser = subcommands.add_parser('modify', help='modify :: ...') ++ modify_parser = subcommands.add_parser('modify', help='modify :: ...', formatter_class=CustomHelpFormatter) + modify_parser.set_defaults(func=modify) + modify_parser.add_argument('selector', nargs=1, help='The %s to modify' % RDN) + modify_parser.add_argument('changes', nargs='+', help="A list of changes to apply in format: ::") + +- rename_parser = subcommands.add_parser('rename', help='rename the object') ++ rename_parser = subcommands.add_parser('rename', help='rename the object', formatter_class=CustomHelpFormatter) + rename_parser.set_defaults(func=rename) + rename_parser.add_argument('selector', help='The %s to modify' % RDN) + rename_parser.add_argument('new_name', help='A new user name') + rename_parser.add_argument('--keep-old-rdn', action='store_true', help="Specify whether the old RDN (i.e. 'cn: old_user') should be kept as an attribute of the entry or not") + +- delete_parser = subcommands.add_parser('delete', help='deletes the object') ++ delete_parser = subcommands.add_parser('delete', help='deletes the object', formatter_class=CustomHelpFormatter) + delete_parser.set_defaults(func=delete) + delete_parser.add_argument('dn', nargs='?', help='The dn to delete') + +-- +2.48.1 + diff --git a/0032-Issue-6067-Improve-dsidm-CLI-No-Such-Entry-handling-.patch b/0032-Issue-6067-Improve-dsidm-CLI-No-Such-Entry-handling-.patch new file mode 100644 index 0000000..38d2ca1 --- /dev/null +++ b/0032-Issue-6067-Improve-dsidm-CLI-No-Such-Entry-handling-.patch @@ -0,0 +1,319 @@ +From 7d534efdcd96b13524dae587c3c5994ed01924ab Mon Sep 17 00:00:00 2001 +From: Simon Pichugin +Date: Fri, 16 Feb 2024 13:52:36 -0800 +Subject: [PATCH] Issue 6067 - Improve dsidm CLI No Such Entry handling (#6079) + +Description: Add additional error processing to dsidm CLI tool for when basedn +or OU subentries are absent. + +Related: https://github.com/389ds/389-ds-base/issues/6067 + +Reviewed by: @vashirov (Thanks!) +--- + src/lib389/cli/dsidm | 21 ++++++++------- + src/lib389/lib389/cli_idm/__init__.py | 38 ++++++++++++++++++++++++++- + src/lib389/lib389/cli_idm/account.py | 4 +-- + src/lib389/lib389/cli_idm/service.py | 4 ++- + src/lib389/lib389/idm/group.py | 10 ++++--- + src/lib389/lib389/idm/posixgroup.py | 5 ++-- + src/lib389/lib389/idm/services.py | 5 ++-- + src/lib389/lib389/idm/user.py | 5 ++-- + 8 files changed, 67 insertions(+), 25 deletions(-) + +diff --git a/src/lib389/cli/dsidm b/src/lib389/cli/dsidm +index 1b739b103..970973f4f 100755 +--- a/src/lib389/cli/dsidm ++++ b/src/lib389/cli/dsidm +@@ -2,7 +2,7 @@ + + # --- BEGIN COPYRIGHT BLOCK --- + # Copyright (C) 2016, William Brown +-# Copyright (C) 2023 Red Hat, Inc. ++# Copyright (C) 2024 Red Hat, Inc. + # All rights reserved. + # + # License: GPL (version 3 or any later version). +@@ -19,6 +19,7 @@ import argparse + import argcomplete + from lib389.utils import get_instance_list, instance_choices + from lib389._constants import DSRC_HOME ++from lib389.cli_idm import _get_basedn_arg + from lib389.cli_idm import account as cli_account + from lib389.cli_idm import initialise as cli_init + from lib389.cli_idm import organizationalunit as cli_ou +@@ -117,14 +118,6 @@ if __name__ == '__main__': + parser.print_help() + sys.exit(1) + +- if dsrc_inst['basedn'] is None: +- errmsg = "Must provide a basedn!" +- if args.json: +- sys.stderr.write('{"desc": "%s"}\n' % errmsg) +- else: +- log.error(errmsg) +- sys.exit(1) +- + if not args.verbose: + signal.signal(signal.SIGINT, signal_handler) + +@@ -135,7 +128,15 @@ if __name__ == '__main__': + result = False + try: + inst = connect_instance(dsrc_inst=dsrc_inst, verbose=args.verbose, args=args) +- result = args.func(inst, dsrc_inst['basedn'], log, args) ++ basedn = _get_basedn_arg(inst, args, log, msg="Enter basedn") ++ if basedn is None: ++ errmsg = "Must provide a basedn!" ++ if args.json: ++ sys.stderr.write('{"desc": "%s"}\n' % errmsg) ++ else: ++ log.error(errmsg) ++ sys.exit(1) ++ result = args.func(inst, basedn, log, args) + if args.verbose: + log.info("Command successful.") + except Exception as e: +diff --git a/src/lib389/lib389/cli_idm/__init__.py b/src/lib389/lib389/cli_idm/__init__.py +index 0dab54847..e3622246d 100644 +--- a/src/lib389/lib389/cli_idm/__init__.py ++++ b/src/lib389/lib389/cli_idm/__init__.py +@@ -1,15 +1,30 @@ + # --- BEGIN COPYRIGHT BLOCK --- + # Copyright (C) 2016, William Brown +-# Copyright (C) 2023 Red Hat, Inc. ++# Copyright (C) 2024 Red Hat, Inc. + # All rights reserved. + # + # License: GPL (version 3 or any later version). + # See LICENSE for details. + # --- END COPYRIGHT BLOCK --- + ++import sys + import ldap + from getpass import getpass + import json ++from lib389._mapped_object import DSLdapObject ++from lib389.cli_base import _get_dn_arg ++from lib389.idm.user import DEFAULT_BASEDN_RDN as DEFAULT_BASEDN_RDN_USER ++from lib389.idm.group import DEFAULT_BASEDN_RDN as DEFAULT_BASEDN_RDN_GROUP ++from lib389.idm.posixgroup import DEFAULT_BASEDN_RDN as DEFAULT_BASEDN_RDN_POSIXGROUP ++from lib389.idm.services import DEFAULT_BASEDN_RDN as DEFAULT_BASEDN_RDN_SERVICES ++ ++# The key is module name, the value is default RDN ++BASEDN_RDNS = { ++ 'user': DEFAULT_BASEDN_RDN_USER, ++ 'group': DEFAULT_BASEDN_RDN_GROUP, ++ 'posixgroup': DEFAULT_BASEDN_RDN_POSIXGROUP, ++ 'service': DEFAULT_BASEDN_RDN_SERVICES, ++} + + + def _get_arg(args, msg=None): +@@ -37,6 +52,27 @@ def _get_args(args, kws): + return kwargs + + ++def _get_basedn_arg(inst, args, log, msg=None): ++ basedn_arg = _get_dn_arg(args.basedn, msg="Enter basedn") ++ if not DSLdapObject(inst, basedn_arg).exists(): ++ raise ValueError(f'The base DN "{basedn_arg}" does not exist.') ++ ++ # Get the RDN based on the last part of the module name if applicable ++ # (lib389.cli_idm.user -> user) ++ try: ++ command_name = args.func.__module__.split('.')[-1] ++ object_rdn = BASEDN_RDNS[command_name] ++ # Check if the DN for our command exists ++ command_basedn = f'{object_rdn},{basedn_arg}' ++ if not DSLdapObject(inst, command_basedn).exists(): ++ errmsg = f'The DN "{command_basedn}" does not exist.' ++ errmsg += f' It is required for "{command_name}" subcommand. Please create it first.' ++ raise ValueError(errmsg) ++ except KeyError: ++ pass ++ return basedn_arg ++ ++ + # This is really similar to get_args, but generates from an array + def _get_attributes(args, attrs): + kwargs = {} +diff --git a/src/lib389/lib389/cli_idm/account.py b/src/lib389/lib389/cli_idm/account.py +index 5d7b9cc77..15f766588 100644 +--- a/src/lib389/lib389/cli_idm/account.py ++++ b/src/lib389/lib389/cli_idm/account.py +@@ -1,5 +1,5 @@ + # --- BEGIN COPYRIGHT BLOCK --- +-# Copyright (C) 2023, Red Hat inc, ++# Copyright (C) 2024, Red Hat inc, + # Copyright (C) 2018, William Brown + # All rights reserved. + # +@@ -91,7 +91,6 @@ def entry_status(inst, basedn, log, args): + + + def subtree_status(inst, basedn, log, args): +- basedn = _get_dn_arg(args.basedn, msg="Enter basedn to check") + filter = "" + scope = ldap.SCOPE_SUBTREE + epoch_inactive_time = None +@@ -121,7 +120,6 @@ def subtree_status(inst, basedn, log, args): + + + def bulk_update(inst, basedn, log, args): +- basedn = _get_dn_arg(args.basedn, msg="Enter basedn to search") + search_filter = "(objectclass=*)" + scope = ldap.SCOPE_SUBTREE + scope_str = "sub" +diff --git a/src/lib389/lib389/cli_idm/service.py b/src/lib389/lib389/cli_idm/service.py +index c62fc12d1..c2b2c8c84 100644 +--- a/src/lib389/lib389/cli_idm/service.py ++++ b/src/lib389/lib389/cli_idm/service.py +@@ -57,7 +57,9 @@ def rename(inst, basedn, log, args, warn=True): + _generic_rename(inst, basedn, log.getChild('_generic_rename'), MANY, rdn, args) + + def create_parser(subparsers): +- service_parser = subparsers.add_parser('service', help='Manage service accounts', formatter_class=CustomHelpFormatter) ++ service_parser = subparsers.add_parser('service', ++ help='Manage service accounts. The organizationalUnit (by default "ou=Services") ' ++ 'needs to exist prior to managing service accounts.', formatter_class=CustomHelpFormatter) + + subcommands = service_parser.add_subparsers(help='action') + +diff --git a/src/lib389/lib389/idm/group.py b/src/lib389/lib389/idm/group.py +index 1b60a1f51..2cf2c7b23 100644 +--- a/src/lib389/lib389/idm/group.py ++++ b/src/lib389/lib389/idm/group.py +@@ -1,6 +1,6 @@ + # --- BEGIN COPYRIGHT BLOCK --- + # Copyright (C) 2016, William Brown +-# Copyright (C) 2023 Red Hat, Inc. ++# Copyright (C) 2024 Red Hat, Inc. + # All rights reserved. + # + # License: GPL (version 3 or any later version). +@@ -16,6 +16,8 @@ MUST_ATTRIBUTES = [ + 'cn', + ] + RDN = 'cn' ++DEFAULT_BASEDN_RDN = 'ou=Groups' ++DEFAULT_BASEDN_RDN_ADMIN_GROUPS = 'ou=People' + + + class Group(DSLdapObject): +@@ -93,7 +95,7 @@ class Groups(DSLdapObjects): + :type basedn: str + """ + +- def __init__(self, instance, basedn, rdn='ou=Groups'): ++ def __init__(self, instance, basedn, rdn=DEFAULT_BASEDN_RDN): + super(Groups, self).__init__(instance) + self._objectclasses = [ + 'groupOfNames', +@@ -140,7 +142,7 @@ class UniqueGroup(DSLdapObject): + class UniqueGroups(DSLdapObjects): + # WARNING!!! + # Use group, not unique group!!! +- def __init__(self, instance, basedn, rdn='ou=Groups'): ++ def __init__(self, instance, basedn, rdn=DEFAULT_BASEDN_RDN): + super(UniqueGroups, self).__init__(instance) + self._objectclasses = [ + 'groupOfUniqueNames', +@@ -203,7 +205,7 @@ class nsAdminGroups(DSLdapObjects): + :type rdn: str + """ + +- def __init__(self, instance, basedn, rdn='ou=People'): ++ def __init__(self, instance, basedn, rdn=DEFAULT_BASEDN_RDN_ADMIN_GROUPS): + super(nsAdminGroups, self).__init__(instance) + self._objectclasses = [ + 'nsAdminGroup' +diff --git a/src/lib389/lib389/idm/posixgroup.py b/src/lib389/lib389/idm/posixgroup.py +index d1debcf12..45735c579 100644 +--- a/src/lib389/lib389/idm/posixgroup.py ++++ b/src/lib389/lib389/idm/posixgroup.py +@@ -1,6 +1,6 @@ + # --- BEGIN COPYRIGHT BLOCK --- + # Copyright (C) 2016, William Brown +-# Copyright (C) 2023 Red Hat, Inc. ++# Copyright (C) 2024 Red Hat, Inc. + # All rights reserved. + # + # License: GPL (version 3 or any later version). +@@ -17,6 +17,7 @@ MUST_ATTRIBUTES = [ + 'gidNumber', + ] + RDN = 'cn' ++DEFAULT_BASEDN_RDN = 'ou=Groups' + + + class PosixGroup(DSLdapObject): +@@ -72,7 +73,7 @@ class PosixGroups(DSLdapObjects): + :type basedn: str + """ + +- def __init__(self, instance, basedn, rdn='ou=Groups'): ++ def __init__(self, instance, basedn, rdn=DEFAULT_BASEDN_RDN): + super(PosixGroups, self).__init__(instance) + self._objectclasses = [ + 'groupOfNames', +diff --git a/src/lib389/lib389/idm/services.py b/src/lib389/lib389/idm/services.py +index d1e5b4693..e750a32c4 100644 +--- a/src/lib389/lib389/idm/services.py ++++ b/src/lib389/lib389/idm/services.py +@@ -1,6 +1,6 @@ + # --- BEGIN COPYRIGHT BLOCK --- + # Copyright (C) 2016, William Brown +-# Copyright (C) 2021 Red Hat, Inc. ++# Copyright (C) 2024 Red Hat, Inc. + # All rights reserved. + # + # License: GPL (version 3 or any later version). +@@ -16,6 +16,7 @@ RDN = 'cn' + MUST_ATTRIBUTES = [ + 'cn', + ] ++DEFAULT_BASEDN_RDN = 'ou=Services' + + class ServiceAccount(Account): + """A single instance of Service entry +@@ -59,7 +60,7 @@ class ServiceAccounts(DSLdapObjects): + :type basedn: str + """ + +- def __init__(self, instance, basedn, rdn='ou=Services'): ++ def __init__(self, instance, basedn, rdn=DEFAULT_BASEDN_RDN): + super(ServiceAccounts, self).__init__(instance) + self._objectclasses = [ + 'applicationProcess', +diff --git a/src/lib389/lib389/idm/user.py b/src/lib389/lib389/idm/user.py +index 1206a6e08..3b21ccf1c 100644 +--- a/src/lib389/lib389/idm/user.py ++++ b/src/lib389/lib389/idm/user.py +@@ -1,6 +1,6 @@ + # --- BEGIN COPYRIGHT BLOCK --- + # Copyright (C) 2016, William Brown +-# Copyright (C) 2023 Red Hat, Inc. ++# Copyright (C) 2024 Red Hat, Inc. + # All rights reserved. + # + # License: GPL (version 3 or any later version). +@@ -23,6 +23,7 @@ MUST_ATTRIBUTES = [ + 'homeDirectory', + ] + RDN = 'uid' ++DEFAULT_BASEDN_RDN = 'ou=People' + + TEST_USER_PROPERTIES = { + 'uid': 'testuser', +@@ -201,7 +202,7 @@ class UserAccounts(DSLdapObjects): + :type rdn: str + """ + +- def __init__(self, instance, basedn, rdn='ou=People'): ++ def __init__(self, instance, basedn, rdn=DEFAULT_BASEDN_RDN): + super(UserAccounts, self).__init__(instance) + self._objectclasses = [ + 'account', +-- +2.48.1 + diff --git a/0033-Issue-6067-Update-dsidm-to-prioritize-basedn-from-.d.patch b/0033-Issue-6067-Update-dsidm-to-prioritize-basedn-from-.d.patch new file mode 100644 index 0000000..fe2e97b --- /dev/null +++ b/0033-Issue-6067-Update-dsidm-to-prioritize-basedn-from-.d.patch @@ -0,0 +1,52 @@ +From ee03e8443a108cff0cc4c7a03962fdc3a1fbf94d Mon Sep 17 00:00:00 2001 +From: Simon Pichugin +Date: Wed, 16 Oct 2024 19:24:55 -0700 +Subject: [PATCH] Issue 6067 - Update dsidm to prioritize basedn from .dsrc + over interactive input (#6362) + +Description: Modifies dsidm CLI tool to check for the basedn in the .dsrc configuration file +when the -b option is not provided. +Previously, users were required to always specify the basedn interactively if -b was omitted, +even if it was available in .dsrc. +Now, the basedn is determined by first checking the -b option, then the .dsrc file, and finally +prompting the user if neither is set. + +Related: https://github.com/389ds/389-ds-base/issues/6067 + +Reviewed by: @Firstyear (Thanks!) +--- + src/lib389/cli/dsidm | 2 +- + src/lib389/lib389/cli_idm/__init__.py | 4 ++-- + 2 files changed, 3 insertions(+), 3 deletions(-) + +diff --git a/src/lib389/cli/dsidm b/src/lib389/cli/dsidm +index 970973f4f..d318664bc 100755 +--- a/src/lib389/cli/dsidm ++++ b/src/lib389/cli/dsidm +@@ -128,7 +128,7 @@ if __name__ == '__main__': + result = False + try: + inst = connect_instance(dsrc_inst=dsrc_inst, verbose=args.verbose, args=args) +- basedn = _get_basedn_arg(inst, args, log, msg="Enter basedn") ++ basedn = _get_basedn_arg(inst, args, dsrc_inst['basedn'], log, msg="Enter basedn") + if basedn is None: + errmsg = "Must provide a basedn!" + if args.json: +diff --git a/src/lib389/lib389/cli_idm/__init__.py b/src/lib389/lib389/cli_idm/__init__.py +index e3622246d..1f3e2dc86 100644 +--- a/src/lib389/lib389/cli_idm/__init__.py ++++ b/src/lib389/lib389/cli_idm/__init__.py +@@ -52,8 +52,8 @@ def _get_args(args, kws): + return kwargs + + +-def _get_basedn_arg(inst, args, log, msg=None): +- basedn_arg = _get_dn_arg(args.basedn, msg="Enter basedn") ++def _get_basedn_arg(inst, args, basedn, log, msg=None): ++ basedn_arg = _get_dn_arg(basedn, msg="Enter basedn") + if not DSLdapObject(inst, basedn_arg).exists(): + raise ValueError(f'The base DN "{basedn_arg}" does not exist.') + +-- +2.48.1 + diff --git a/389-ds-base.spec b/389-ds-base.spec index 9a75c29..45da920 100644 --- a/389-ds-base.spec +++ b/389-ds-base.spec @@ -48,8 +48,8 @@ ExcludeArch: i686 Summary: 389 Directory Server (base) Name: 389-ds-base Version: 1.4.3.39 -Release: %{?relprefix}11%{?prerel}%{?dist} -License: GPLv3+ and (ASL 2.0 or MIT) +Release: %{?relprefix}12%{?prerel}%{?dist} +License: GPL-3.0-or-later WITH GPL-3.0-389-ds-base-exception AND (0BSD OR Apache-2.0 OR MIT) AND (Apache-2.0 OR Apache-2.0 WITH LLVM-exception OR MIT) AND (Apache-2.0 OR BSD-2-Clause OR MIT) AND (Apache-2.0 OR BSL-1.0) AND (Apache-2.0 OR LGPL-2.1-or-later OR MIT) AND (Apache-2.0 OR MIT OR Zlib) AND (Apache-2.0 OR MIT) AND (MIT OR Apache-2.0) AND Unicode-3.0 AND (MIT OR Unlicense) AND Apache-2.0 AND MIT AND MPL-2.0 AND Zlib URL: https://www.port389.org Group: System Environment/Daemons Conflicts: selinux-policy-base < 3.9.8 @@ -58,114 +58,111 @@ Obsoletes: %{name} <= 1.4.0.9 Provides: ldif2ldbm >= 0 ##### Bundled cargo crates list - START ##### -Provides: bundled(crate(addr2line)) = 0.21.0 -Provides: bundled(crate(adler)) = 1.0.2 -Provides: bundled(crate(ahash)) = 0.7.7 +Provides: bundled(crate(addr2line)) = 0.24.2 +Provides: bundled(crate(adler2)) = 2.0.0 +Provides: bundled(crate(ahash)) = 0.8.11 +Provides: bundled(crate(allocator-api2)) = 0.2.21 Provides: bundled(crate(ansi_term)) = 0.12.1 +Provides: bundled(crate(arc-swap)) = 1.7.1 Provides: bundled(crate(atty)) = 0.2.14 -Provides: bundled(crate(autocfg)) = 1.1.0 -Provides: bundled(crate(backtrace)) = 0.3.69 +Provides: bundled(crate(autocfg)) = 1.4.0 +Provides: bundled(crate(backtrace)) = 0.3.74 Provides: bundled(crate(base64)) = 0.13.1 -Provides: bundled(crate(bitflags)) = 1.3.2 -Provides: bundled(crate(bitflags)) = 2.4.1 +Provides: bundled(crate(bitflags)) = 2.9.0 Provides: bundled(crate(byteorder)) = 1.5.0 -Provides: bundled(crate(cbindgen)) = 0.9.1 -Provides: bundled(crate(cc)) = 1.0.83 +Provides: bundled(crate(cbindgen)) = 0.26.0 +Provides: bundled(crate(cc)) = 1.2.17 Provides: bundled(crate(cfg-if)) = 1.0.0 -Provides: bundled(crate(clap)) = 2.34.0 -Provides: bundled(crate(concread)) = 0.2.21 -Provides: bundled(crate(crossbeam)) = 0.8.4 -Provides: bundled(crate(crossbeam-channel)) = 0.5.11 -Provides: bundled(crate(crossbeam-deque)) = 0.8.5 +Provides: bundled(crate(clap)) = 3.2.25 +Provides: bundled(crate(clap_lex)) = 0.2.4 +Provides: bundled(crate(concread)) = 0.5.5 Provides: bundled(crate(crossbeam-epoch)) = 0.9.18 -Provides: bundled(crate(crossbeam-queue)) = 0.3.11 -Provides: bundled(crate(crossbeam-utils)) = 0.8.19 -Provides: bundled(crate(entryuuid)) = 0.1.0 -Provides: bundled(crate(entryuuid_syntax)) = 0.1.0 -Provides: bundled(crate(errno)) = 0.3.8 -Provides: bundled(crate(fastrand)) = 2.0.1 +Provides: bundled(crate(crossbeam-queue)) = 0.3.12 +Provides: bundled(crate(crossbeam-utils)) = 0.8.21 +Provides: bundled(crate(equivalent)) = 1.0.2 +Provides: bundled(crate(errno)) = 0.3.10 +Provides: bundled(crate(fastrand)) = 2.3.0 Provides: bundled(crate(fernet)) = 0.1.4 +Provides: bundled(crate(foldhash)) = 0.1.5 Provides: bundled(crate(foreign-types)) = 0.3.2 Provides: bundled(crate(foreign-types-shared)) = 0.1.1 -Provides: bundled(crate(getrandom)) = 0.2.12 -Provides: bundled(crate(gimli)) = 0.28.1 -Provides: bundled(crate(hashbrown)) = 0.12.3 +Provides: bundled(crate(getrandom)) = 0.3.2 +Provides: bundled(crate(gimli)) = 0.31.1 +Provides: bundled(crate(hashbrown)) = 0.15.2 +Provides: bundled(crate(heck)) = 0.4.1 Provides: bundled(crate(hermit-abi)) = 0.1.19 -Provides: bundled(crate(instant)) = 0.1.12 -Provides: bundled(crate(itoa)) = 1.0.10 -Provides: bundled(crate(jobserver)) = 0.1.27 -Provides: bundled(crate(libc)) = 0.2.152 -Provides: bundled(crate(librnsslapd)) = 0.1.0 -Provides: bundled(crate(librslapd)) = 0.1.0 -Provides: bundled(crate(linux-raw-sys)) = 0.4.12 -Provides: bundled(crate(lock_api)) = 0.4.11 -Provides: bundled(crate(log)) = 0.4.20 -Provides: bundled(crate(lru)) = 0.7.8 -Provides: bundled(crate(memchr)) = 2.7.1 -Provides: bundled(crate(miniz_oxide)) = 0.7.1 -Provides: bundled(crate(object)) = 0.32.2 -Provides: bundled(crate(once_cell)) = 1.19.0 -Provides: bundled(crate(openssl)) = 0.10.62 +Provides: bundled(crate(indexmap)) = 1.9.3 +Provides: bundled(crate(itoa)) = 1.0.15 +Provides: bundled(crate(jobserver)) = 0.1.33 +Provides: bundled(crate(libc)) = 0.2.171 +Provides: bundled(crate(linux-raw-sys)) = 0.9.3 +Provides: bundled(crate(log)) = 0.4.27 +Provides: bundled(crate(lru)) = 0.13.0 +Provides: bundled(crate(memchr)) = 2.7.4 +Provides: bundled(crate(miniz_oxide)) = 0.8.5 +Provides: bundled(crate(object)) = 0.36.7 +Provides: bundled(crate(once_cell)) = 1.21.3 +Provides: bundled(crate(openssl)) = 0.10.71 Provides: bundled(crate(openssl-macros)) = 0.1.1 -Provides: bundled(crate(openssl-sys)) = 0.9.98 -Provides: bundled(crate(parking_lot)) = 0.11.2 -Provides: bundled(crate(parking_lot_core)) = 0.8.6 +Provides: bundled(crate(openssl-sys)) = 0.9.106 +Provides: bundled(crate(os_str_bytes)) = 6.6.1 Provides: bundled(crate(paste)) = 0.1.18 Provides: bundled(crate(paste-impl)) = 0.1.18 -Provides: bundled(crate(pin-project-lite)) = 0.2.13 -Provides: bundled(crate(pkg-config)) = 0.3.28 -Provides: bundled(crate(ppv-lite86)) = 0.2.17 +Provides: bundled(crate(pin-project-lite)) = 0.2.16 +Provides: bundled(crate(pkg-config)) = 0.3.32 Provides: bundled(crate(proc-macro-hack)) = 0.5.20+deprecated -Provides: bundled(crate(proc-macro2)) = 1.0.76 -Provides: bundled(crate(pwdchan)) = 0.1.0 -Provides: bundled(crate(quote)) = 1.0.35 -Provides: bundled(crate(rand)) = 0.8.5 -Provides: bundled(crate(rand_chacha)) = 0.3.1 -Provides: bundled(crate(rand_core)) = 0.6.4 -Provides: bundled(crate(redox_syscall)) = 0.2.16 -Provides: bundled(crate(redox_syscall)) = 0.4.1 +Provides: bundled(crate(proc-macro2)) = 1.0.94 +Provides: bundled(crate(quote)) = 1.0.40 +Provides: bundled(crate(r-efi)) = 5.2.0 Provides: bundled(crate(rsds)) = 0.1.0 -Provides: bundled(crate(rustc-demangle)) = 0.1.23 -Provides: bundled(crate(rustix)) = 0.38.30 -Provides: bundled(crate(ryu)) = 1.0.16 -Provides: bundled(crate(scopeguard)) = 1.2.0 -Provides: bundled(crate(serde)) = 1.0.195 -Provides: bundled(crate(serde_derive)) = 1.0.195 -Provides: bundled(crate(serde_json)) = 1.0.111 -Provides: bundled(crate(slapd)) = 0.1.0 -Provides: bundled(crate(slapi_r_plugin)) = 0.1.0 -Provides: bundled(crate(smallvec)) = 1.12.0 -Provides: bundled(crate(strsim)) = 0.8.0 -Provides: bundled(crate(syn)) = 1.0.109 -Provides: bundled(crate(syn)) = 2.0.48 -Provides: bundled(crate(tempfile)) = 3.9.0 -Provides: bundled(crate(textwrap)) = 0.11.0 -Provides: bundled(crate(tokio)) = 1.35.1 -Provides: bundled(crate(tokio-macros)) = 2.2.0 +Provides: bundled(crate(rustc-demangle)) = 0.1.24 +Provides: bundled(crate(rustix)) = 1.0.5 +Provides: bundled(crate(ryu)) = 1.0.20 +Provides: bundled(crate(serde)) = 1.0.219 +Provides: bundled(crate(serde_derive)) = 1.0.219 +Provides: bundled(crate(serde_json)) = 1.0.140 +Provides: bundled(crate(shlex)) = 1.3.0 +Provides: bundled(crate(smallvec)) = 1.14.0 +Provides: bundled(crate(sptr)) = 0.3.2 +Provides: bundled(crate(strsim)) = 0.10.0 +Provides: bundled(crate(syn)) = 2.0.100 +Provides: bundled(crate(tempfile)) = 3.19.1 +Provides: bundled(crate(termcolor)) = 1.4.1 +Provides: bundled(crate(textwrap)) = 0.16.2 +Provides: bundled(crate(tokio)) = 1.44.1 Provides: bundled(crate(toml)) = 0.5.11 -Provides: bundled(crate(unicode-ident)) = 1.0.12 -Provides: bundled(crate(unicode-width)) = 0.1.11 +Provides: bundled(crate(tracing)) = 0.1.41 +Provides: bundled(crate(tracing-attributes)) = 0.1.28 +Provides: bundled(crate(tracing-core)) = 0.1.33 +Provides: bundled(crate(unicode-ident)) = 1.0.18 +Provides: bundled(crate(unicode-width)) = 0.1.14 Provides: bundled(crate(uuid)) = 0.8.2 Provides: bundled(crate(vcpkg)) = 0.2.15 Provides: bundled(crate(vec_map)) = 0.8.2 -Provides: bundled(crate(version_check)) = 0.9.4 -Provides: bundled(crate(wasi)) = 0.11.0+wasi_snapshot_preview1 +Provides: bundled(crate(version_check)) = 0.9.5 +Provides: bundled(crate(wasi)) = 0.14.2+wasi_0.2.4 Provides: bundled(crate(winapi)) = 0.3.9 Provides: bundled(crate(winapi-i686-pc-windows-gnu)) = 0.4.0 +Provides: bundled(crate(winapi-util)) = 0.1.9 Provides: bundled(crate(winapi-x86_64-pc-windows-gnu)) = 0.4.0 -Provides: bundled(crate(windows-sys)) = 0.52.0 -Provides: bundled(crate(windows-targets)) = 0.52.0 -Provides: bundled(crate(windows_aarch64_gnullvm)) = 0.52.0 -Provides: bundled(crate(windows_aarch64_msvc)) = 0.52.0 -Provides: bundled(crate(windows_i686_gnu)) = 0.52.0 -Provides: bundled(crate(windows_i686_msvc)) = 0.52.0 -Provides: bundled(crate(windows_x86_64_gnu)) = 0.52.0 -Provides: bundled(crate(windows_x86_64_gnullvm)) = 0.52.0 -Provides: bundled(crate(windows_x86_64_msvc)) = 0.52.0 -Provides: bundled(crate(zeroize)) = 1.7.0 +Provides: bundled(crate(windows-sys)) = 0.59.0 +Provides: bundled(crate(windows-targets)) = 0.52.6 +Provides: bundled(crate(windows_aarch64_gnullvm)) = 0.52.6 +Provides: bundled(crate(windows_aarch64_msvc)) = 0.52.6 +Provides: bundled(crate(windows_i686_gnu)) = 0.52.6 +Provides: bundled(crate(windows_i686_gnullvm)) = 0.52.6 +Provides: bundled(crate(windows_i686_msvc)) = 0.52.6 +Provides: bundled(crate(windows_x86_64_gnu)) = 0.52.6 +Provides: bundled(crate(windows_x86_64_gnullvm)) = 0.52.6 +Provides: bundled(crate(windows_x86_64_msvc)) = 0.52.6 +Provides: bundled(crate(wit-bindgen-rt)) = 0.39.0 +Provides: bundled(crate(zerocopy)) = 0.7.35 +Provides: bundled(crate(zerocopy-derive)) = 0.7.35 +Provides: bundled(crate(zeroize)) = 1.8.1 Provides: bundled(crate(zeroize_derive)) = 1.4.2 ##### Bundled cargo crates list - END ##### + BuildRequires: nspr-devel >= 4.32 BuildRequires: nss-devel >= 3.67.0-7 BuildRequires: perl-generators @@ -289,8 +286,8 @@ Source2: %{name}-devel.README Source3: https://github.com/jemalloc/%{jemalloc_name}/releases/download/%{jemalloc_ver}/%{jemalloc_name}-%{jemalloc_ver}.tar.bz2 %endif %if %{use_rust} -Source4: vendor-%{version}-1.tar.gz -Source5: Cargo-%{version}-1.lock +Source4: vendor-%{version}-2.tar.gz +Source5: Cargo-%{version}-2.lock %endif Patch01: 0001-issue-5647-covscan-memory-leak-in-audit-log-when-add.patch @@ -315,6 +312,19 @@ Patch19: 0019-Issue-6417-2nd-fix-typo.patch Patch20: 0020-Issue-6417-3rd-If-an-entry-RDN-is-identical-to-the-s.patch Patch21: 0021-Issue-6509-Race-condition-with-Paged-Result-searches.patch Patch22: 0022-Issue-6509-Fix-cherry-pick-issue-race-condition-in-P.patch +Patch23: 0023-Issue-6304-RFE-when-memberof-is-enabled-defer-update.patch +Patch24: 0024-Issue-6436-MOD-on-a-large-group-slow-if-substring-in.patch +Patch25: 0025-Issue-6494-Various-errors-when-using-extended-matchi.patch +Patch26: 0026-Issue-6004-idletimeout-may-be-ignored-6005.patch +Patch27: 0027-Issue-6004-2nd-idletimeout-may-be-ignored-6569.patch +Patch28: 0028-Issue-6485-Fix-double-free-in-USN-cleanup-task.patch +Patch29: 0029-Issue-6553-Update-concread-to-0.5.4-and-refactor-sta.patch +Patch30: 0030-Issue-5841-dsconf-incorrectly-setting-up-Pass-Throug.patch +Patch31: 0031-Issue-6067-Add-hidden-v-and-j-options-to-each-CLI-su.patch +Patch32: 0032-Issue-6067-Improve-dsidm-CLI-No-Such-Entry-handling-.patch +Patch33: 0033-Issue-6067-Update-dsidm-to-prioritize-basedn-from-.d.patch + +Patch100: cargo.patch %description 389 Directory Server is an LDAPv3 compliant server. The base package includes @@ -444,6 +454,7 @@ A cockpit UI Plugin for configuring and administering the 389 Directory Server %prep %autosetup -p1 -n %{name}-%{version}%{?prerel} %if %{use_rust} +rm -rf vendor tar xzf %{SOURCE4} cp %{SOURCE5} src/Cargo.lock %endif @@ -936,6 +947,16 @@ exit 0 %doc README.md %changelog +* Thu Apr 03 2025 Viktor Ashirov - 1.4.3.39-12 +- Resolves: RHEL-85499 - [RFE] defer memberof nested updates [rhel-8.10.z] +- Resolves: RHEL-65663 - dsconf incorrectly setting up Pass-Through Authentication +- Resolves: RHEL-80704 - Increased memory consumption caused by NDN cache [rhel-8.10.z] +- Resolves: RHEL-81127 - nsslapd-idletimeout is ignored [rhel-8.10.z] +- Resolves: RHEL-81136 - Healthcheck tool should warn admin about creating a substring index on membership attribute [rhel-8.10.z] +- Resolves: RHEL-81143 - 389DirectoryServer Process Stops When Setting up Sorted VLV Index [rhel-8.10.z] +- Resolves: RHEL-81152 - AddressSanitizer: double-free [rhel-8.10.z] +- Resolves: RHEL-81176 - Verbose option for dsctl is not shown in help of actions [rhel-8.10.z] + * Thu Jan 23 2025 Viktor Ashirov - 1.4.3.39-11 - Resolves: RHEL-72487 - IPA LDAP error code T3 when no exceeded time limit from a paged search result [rhel-8.10.z] diff --git a/Cargo.lock b/Cargo.lock deleted file mode 100644 index 18078a8..0000000 --- a/Cargo.lock +++ /dev/null @@ -1,801 +0,0 @@ -# This file is automatically @generated by Cargo. -# It is not intended for manual editing. -version = 3 - -[[package]] -name = "ahash" -version = "0.7.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fcb51a0695d8f838b1ee009b3fbf66bda078cd64590202a864a8f3e8c4315c47" -dependencies = [ - "getrandom", - "once_cell", - "version_check", -] - -[[package]] -name = "ansi_term" -version = "0.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d52a9bb7ec0cf484c551830a7ce27bd20d67eac647e1befb56b0be4ee39a55d2" -dependencies = [ - "winapi", -] - -[[package]] -name = "atty" -version = "0.2.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" -dependencies = [ - "hermit-abi", - "libc", - "winapi", -] - -[[package]] -name = "autocfg" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" - -[[package]] -name = "base64" -version = "0.13.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" - -[[package]] -name = "bitflags" -version = "1.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" - -[[package]] -name = "byteorder" -version = "1.4.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610" - -[[package]] -name = "cbindgen" -version = "0.9.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9daec6140ab4dcd38c3dd57e580b59a621172a526ac79f1527af760a55afeafd" -dependencies = [ - "clap", - "log", - "proc-macro2", - "quote", - "serde", - "serde_json", - "syn", - "tempfile", - "toml", -] - -[[package]] -name = "cc" -version = "1.0.76" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76a284da2e6fe2092f2353e51713435363112dfd60030e22add80be333fb928f" -dependencies = [ - "jobserver", -] - -[[package]] -name = "cfg-if" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" - -[[package]] -name = "clap" -version = "2.34.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a0610544180c38b88101fecf2dd634b174a62eef6946f84dfc6a7127512b381c" -dependencies = [ - "ansi_term", - "atty", - "bitflags", - "strsim", - "textwrap", - "unicode-width", - "vec_map", -] - -[[package]] -name = "concread" -version = "0.2.21" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dcc9816f5ac93ebd51c37f7f9a6bf2b40dfcd42978ad2aea5d542016e9244cf6" -dependencies = [ - "ahash", - "crossbeam", - "crossbeam-epoch", - "crossbeam-utils", - "lru", - "parking_lot", - "rand", - "smallvec", - "tokio", -] - -[[package]] -name = "crossbeam" -version = "0.8.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2801af0d36612ae591caa9568261fddce32ce6e08a7275ea334a06a4ad021a2c" -dependencies = [ - "cfg-if", - "crossbeam-channel", - "crossbeam-deque", - "crossbeam-epoch", - "crossbeam-queue", - "crossbeam-utils", -] - -[[package]] -name = "crossbeam-channel" -version = "0.5.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c2dd04ddaf88237dc3b8d8f9a3c1004b506b54b3313403944054d23c0870c521" -dependencies = [ - "cfg-if", - "crossbeam-utils", -] - -[[package]] -name = "crossbeam-deque" -version = "0.8.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "715e8152b692bba2d374b53d4875445368fdf21a94751410af607a5ac677d1fc" -dependencies = [ - "cfg-if", - "crossbeam-epoch", - "crossbeam-utils", -] - -[[package]] -name = "crossbeam-epoch" -version = "0.9.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f916dfc5d356b0ed9dae65f1db9fc9770aa2851d2662b988ccf4fe3516e86348" -dependencies = [ - "autocfg", - "cfg-if", - "crossbeam-utils", - "memoffset", - "scopeguard", -] - -[[package]] -name = "crossbeam-queue" -version = "0.3.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1cd42583b04998a5363558e5f9291ee5a5ff6b49944332103f251e7479a82aa7" -dependencies = [ - "cfg-if", - "crossbeam-utils", -] - -[[package]] -name = "crossbeam-utils" -version = "0.8.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "edbafec5fa1f196ca66527c1b12c2ec4745ca14b50f1ad8f9f6f720b55d11fac" -dependencies = [ - "cfg-if", -] - -[[package]] -name = "entryuuid" -version = "0.1.0" -dependencies = [ - "cc", - "libc", - "paste", - "slapi_r_plugin", - "uuid", -] - -[[package]] -name = "entryuuid_syntax" -version = "0.1.0" -dependencies = [ - "cc", - "libc", - "paste", - "slapi_r_plugin", - "uuid", -] - -[[package]] -name = "fastrand" -version = "1.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7a407cfaa3385c4ae6b23e84623d48c2798d06e3e6a1878f7f59f17b3f86499" -dependencies = [ - "instant", -] - -[[package]] -name = "fernet" -version = "0.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93804560e638370a8be6d59ce71ed803e55e230abdbf42598e666b41adda9b1f" -dependencies = [ - "base64", - "byteorder", - "getrandom", - "openssl", - "zeroize", -] - -[[package]] -name = "foreign-types" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" -dependencies = [ - "foreign-types-shared", -] - -[[package]] -name = "foreign-types-shared" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" - -[[package]] -name = "getrandom" -version = "0.2.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c05aeb6a22b8f62540c194aac980f2115af067bfe15a0734d7277a768d396b31" -dependencies = [ - "cfg-if", - "libc", - "wasi", -] - -[[package]] -name = "hashbrown" -version = "0.12.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" -dependencies = [ - "ahash", -] - -[[package]] -name = "hermit-abi" -version = "0.1.19" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33" -dependencies = [ - "libc", -] - -[[package]] -name = "instant" -version = "0.1.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a5bbe824c507c5da5956355e86a746d82e0e1464f65d862cc5e71da70e94b2c" -dependencies = [ - "cfg-if", -] - -[[package]] -name = "itoa" -version = "1.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4217ad341ebadf8d8e724e264f13e593e0648f5b3e94b3896a5df283be015ecc" - -[[package]] -name = "jobserver" -version = "0.1.25" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "068b1ee6743e4d11fb9c6a1e6064b3693a1b600e7f5f5988047d98b3dc9fb90b" -dependencies = [ - "libc", -] - -[[package]] -name = "libc" -version = "0.2.137" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc7fcc620a3bff7cdd7a365be3376c97191aeaccc2a603e600951e452615bf89" - -[[package]] -name = "librnsslapd" -version = "0.1.0" -dependencies = [ - "cbindgen", - "libc", - "slapd", -] - -[[package]] -name = "librslapd" -version = "0.1.0" -dependencies = [ - "cbindgen", - "concread", - "libc", - "slapd", -] - -[[package]] -name = "lock_api" -version = "0.4.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "435011366fe56583b16cf956f9df0095b405b82d76425bc8981c0e22e60ec4df" -dependencies = [ - "autocfg", - "scopeguard", -] - -[[package]] -name = "log" -version = "0.4.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "abb12e687cfb44aa40f41fc3978ef76448f9b6038cad6aef4259d3c095a2382e" -dependencies = [ - "cfg-if", -] - -[[package]] -name = "lru" -version = "0.7.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e999beba7b6e8345721bd280141ed958096a2e4abdf74f67ff4ce49b4b54e47a" -dependencies = [ - "hashbrown", -] - -[[package]] -name = "memoffset" -version = "0.6.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5aa361d4faea93603064a027415f07bd8e1d5c88c9fbf68bf56a285428fd79ce" -dependencies = [ - "autocfg", -] - -[[package]] -name = "once_cell" -version = "1.16.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86f0b0d4bf799edbc74508c1e8bf170ff5f41238e5f8225603ca7caaae2b7860" - -[[package]] -name = "openssl" -version = "0.10.42" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12fc0523e3bd51a692c8850d075d74dc062ccf251c0110668cbd921917118a13" -dependencies = [ - "bitflags", - "cfg-if", - "foreign-types", - "libc", - "once_cell", - "openssl-macros", - "openssl-sys", -] - -[[package]] -name = "openssl-macros" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b501e44f11665960c7e7fcf062c7d96a14ade4aa98116c004b2e37b5be7d736c" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "openssl-sys" -version = "0.9.77" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b03b84c3b2d099b81f0953422b4d4ad58761589d0229b5506356afca05a3670a" -dependencies = [ - "autocfg", - "cc", - "libc", - "pkg-config", - "vcpkg", -] - -[[package]] -name = "parking_lot" -version = "0.11.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d17b78036a60663b797adeaee46f5c9dfebb86948d1255007a1d6be0271ff99" -dependencies = [ - "instant", - "lock_api", - "parking_lot_core", -] - -[[package]] -name = "parking_lot_core" -version = "0.8.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d76e8e1493bcac0d2766c42737f34458f1c8c50c0d23bcb24ea953affb273216" -dependencies = [ - "cfg-if", - "instant", - "libc", - "redox_syscall", - "smallvec", - "winapi", -] - -[[package]] -name = "paste" -version = "0.1.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "45ca20c77d80be666aef2b45486da86238fabe33e38306bd3118fe4af33fa880" -dependencies = [ - "paste-impl", - "proc-macro-hack", -] - -[[package]] -name = "paste-impl" -version = "0.1.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d95a7db200b97ef370c8e6de0088252f7e0dfff7d047a28528e47456c0fc98b6" -dependencies = [ - "proc-macro-hack", -] - -[[package]] -name = "pin-project-lite" -version = "0.2.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e0a7ae3ac2f1173085d398531c705756c94a4c56843785df85a60c1a0afac116" - -[[package]] -name = "pkg-config" -version = "0.3.26" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ac9a59f73473f1b8d852421e59e64809f025994837ef743615c6d0c5b305160" - -[[package]] -name = "ppv-lite86" -version = "0.2.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" - -[[package]] -name = "proc-macro-hack" -version = "0.5.19" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dbf0c48bc1d91375ae5c3cd81e3722dff1abcf81a30960240640d223f59fe0e5" - -[[package]] -name = "proc-macro2" -version = "1.0.47" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ea3d908b0e36316caf9e9e2c4625cdde190a7e6f440d794667ed17a1855e725" -dependencies = [ - "unicode-ident", -] - -[[package]] -name = "pwdchan" -version = "0.1.0" -dependencies = [ - "base64", - "cc", - "libc", - "openssl", - "paste", - "slapi_r_plugin", - "uuid", -] - -[[package]] -name = "quote" -version = "1.0.21" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbe448f377a7d6961e30f5955f9b8d106c3f5e449d493ee1b125c1d43c2b5179" -dependencies = [ - "proc-macro2", -] - -[[package]] -name = "rand" -version = "0.8.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" -dependencies = [ - "libc", - "rand_chacha", - "rand_core", -] - -[[package]] -name = "rand_chacha" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" -dependencies = [ - "ppv-lite86", - "rand_core", -] - -[[package]] -name = "rand_core" -version = "0.6.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" -dependencies = [ - "getrandom", -] - -[[package]] -name = "redox_syscall" -version = "0.2.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a" -dependencies = [ - "bitflags", -] - -[[package]] -name = "remove_dir_all" -version = "0.5.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3acd125665422973a33ac9d3dd2df85edad0f4ae9b00dafb1a05e43a9f5ef8e7" -dependencies = [ - "winapi", -] - -[[package]] -name = "rsds" -version = "0.1.0" - -[[package]] -name = "ryu" -version = "1.0.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4501abdff3ae82a1c1b477a17252eb69cee9e66eb915c1abaa4f44d873df9f09" - -[[package]] -name = "scopeguard" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" - -[[package]] -name = "serde" -version = "1.0.147" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d193d69bae983fc11a79df82342761dfbf28a99fc8d203dca4c3c1b590948965" -dependencies = [ - "serde_derive", -] - -[[package]] -name = "serde_derive" -version = "1.0.147" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4f1d362ca8fc9c3e3a7484440752472d68a6caa98f1ab81d99b5dfe517cec852" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "serde_json" -version = "1.0.87" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ce777b7b150d76b9cf60d28b55f5847135a003f7d7350c6be7a773508ce7d45" -dependencies = [ - "itoa", - "ryu", - "serde", -] - -[[package]] -name = "slapd" -version = "0.1.0" -dependencies = [ - "fernet", -] - -[[package]] -name = "slapi_r_plugin" -version = "0.1.0" -dependencies = [ - "libc", - "paste", - "uuid", -] - -[[package]] -name = "smallvec" -version = "1.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a507befe795404456341dfab10cef66ead4c041f62b8b11bbb92bffe5d0953e0" - -[[package]] -name = "strsim" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ea5119cdb4c55b55d432abb513a0429384878c15dde60cc77b1c99de1a95a6a" - -[[package]] -name = "syn" -version = "1.0.103" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a864042229133ada95abf3b54fdc62ef5ccabe9515b64717bcb9a1919e59445d" -dependencies = [ - "proc-macro2", - "quote", - "unicode-ident", -] - -[[package]] -name = "synstructure" -version = "0.12.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f36bdaa60a83aca3921b5259d5400cbf5e90fc51931376a9bd4a0eb79aa7210f" -dependencies = [ - "proc-macro2", - "quote", - "syn", - "unicode-xid", -] - -[[package]] -name = "tempfile" -version = "3.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5cdb1ef4eaeeaddc8fbd371e5017057064af0911902ef36b39801f67cc6d79e4" -dependencies = [ - "cfg-if", - "fastrand", - "libc", - "redox_syscall", - "remove_dir_all", - "winapi", -] - -[[package]] -name = "textwrap" -version = "0.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d326610f408c7a4eb6f51c37c330e496b08506c9457c9d34287ecc38809fb060" -dependencies = [ - "unicode-width", -] - -[[package]] -name = "tokio" -version = "1.21.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9e03c497dc955702ba729190dc4aac6f2a0ce97f913e5b1b5912fc5039d9099" -dependencies = [ - "autocfg", - "pin-project-lite", - "tokio-macros", -] - -[[package]] -name = "tokio-macros" -version = "1.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9724f9a975fb987ef7a3cd9be0350edcbe130698af5b8f7a631e23d42d052484" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "toml" -version = "0.5.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d82e1a7758622a465f8cee077614c73484dac5b836c02ff6a40d5d1010324d7" -dependencies = [ - "serde", -] - -[[package]] -name = "unicode-ident" -version = "1.0.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ceab39d59e4c9499d4e5a8ee0e2735b891bb7308ac83dfb4e80cad195c9f6f3" - -[[package]] -name = "unicode-width" -version = "0.1.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c0edd1e5b14653f783770bce4a4dabb4a5108a5370a5f5d8cfe8710c361f6c8b" - -[[package]] -name = "unicode-xid" -version = "0.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f962df74c8c05a667b5ee8bcf162993134c104e96440b663c8daa176dc772d8c" - -[[package]] -name = "uuid" -version = "0.8.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc5cf98d8186244414c848017f0e2676b3fcb46807f6668a97dfe67359a3c4b7" -dependencies = [ - "getrandom", -] - -[[package]] -name = "vcpkg" -version = "0.2.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" - -[[package]] -name = "vec_map" -version = "0.8.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1bddf1187be692e79c5ffeab891132dfb0f236ed36a43c7ed39f1165ee20191" - -[[package]] -name = "version_check" -version = "0.9.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" - -[[package]] -name = "wasi" -version = "0.11.0+wasi-snapshot-preview1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" - -[[package]] -name = "winapi" -version = "0.3.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" -dependencies = [ - "winapi-i686-pc-windows-gnu", - "winapi-x86_64-pc-windows-gnu", -] - -[[package]] -name = "winapi-i686-pc-windows-gnu" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" - -[[package]] -name = "winapi-x86_64-pc-windows-gnu" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" - -[[package]] -name = "zeroize" -version = "1.5.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c394b5bd0c6f669e7275d9c20aa90ae064cb22e75a1cad54e1b34088034b149f" -dependencies = [ - "zeroize_derive", -] - -[[package]] -name = "zeroize_derive" -version = "1.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f8f187641dad4f680d25c4bfc4225b418165984179f26ca76ec4fb6441d3a17" -dependencies = [ - "proc-macro2", - "quote", - "syn", - "synstructure", -] diff --git a/cargo.patch b/cargo.patch new file mode 100644 index 0000000..f0146be --- /dev/null +++ b/cargo.patch @@ -0,0 +1,27 @@ +diff --git a/.cargo/config.in b/.cargo/config.in +index d7d8ff4d4..d61993c54 100644 +--- a/.cargo/config.in ++++ b/.cargo/config.in +@@ -2,5 +2,10 @@ + registry = "https://github.com/rust-lang/crates.io-index" + @rust_vendor_sources@ + ++[source."git+https://github.com/389ds/concread?branch=unstable_name_collisions"] ++git = "https://github.com/389ds/concread" ++branch = "unstable_name_collisions" ++replace-with = "vendored-sources" ++ + [source.vendored-sources] + directory = "./vendor" +diff --git a/src/Cargo.toml b/src/Cargo.toml +index 95c1ae3f5..4daf9cf7b 100644 +--- a/src/Cargo.toml ++++ b/src/Cargo.toml +@@ -15,4 +15,6 @@ members = [ + panic = "abort" + lto = true + ++[patch.crates-io] ++concread = { git = "https://github.com/389ds/concread", branch = "unstable_name_collisions" } + + diff --git a/sources b/sources index ce2475d..1e13421 100644 --- a/sources +++ b/sources @@ -1,4 +1,4 @@ SHA512 (jemalloc-5.3.0.tar.bz2) = 22907bb052096e2caffb6e4e23548aecc5cc9283dce476896a2b1127eee64170e3562fa2e7db9571298814a7a2c7df6e8d1fbe152bd3f3b0c1abec22a2de34b1 SHA512 (389-ds-base-1.4.3.39.tar.bz2) = de139895d4488d02f4930ba5a61809139e8bf6c9c46160de6780a01fff4578fce965434cea4bc14e0e6c1711f9f4ac1f31a5d25b374ee6e015962d4ab70db037 -SHA512 (vendor-1.4.3.39-1.tar.gz) = 892ec44032eb3feda987a7adf76721fe4222e9efb291e27238bcb4d01f8b444bbe5cf1ab4a8280cb485f35f16b51e151b08d064badf9b292cfbfb98df1be46f0 -SHA512 (Cargo-1.4.3.39-1.lock) = 76a2e8f737e576f7a9936d8cf70c424f944caab257a27e3f4b4177d35bd4e8b14a59e665b675e1f59300169b47af28afbc64fd3733460f3884e9b0c948777177 +SHA512 (Cargo-1.4.3.39-2.lock) = 79c7f4ade0ff8d9a05dc6fe69360d4ad25a874f4689eef48f1b74866e6348ac8037dd4699e688cf824f9fa014b4a4d1e169cd5c6697d57bdd36ed95de00a1717 +SHA512 (vendor-1.4.3.39-2.tar.gz) = 96a1949fb7ffc252a4afda11b44d6ade3e9fab12e57a7778f3a0398084e8aad4ded543b48bf104057a1a846a8df7d631d0c92ca4b2fe0d12f09a07ba7efab4a6