import 389-ds-base-1.4.3.16-7.module+el8.4.0+9324+a82a8f71
This commit is contained in:
parent
584f0d770f
commit
4c3ed9ae97
@ -0,0 +1,374 @@
|
||||
From d7b49259ff2f9e0295bbfeaf128369ed33421974 Mon Sep 17 00:00:00 2001
|
||||
From: James Chapman <jachapma@redhat.com>
|
||||
Date: Mon, 30 Nov 2020 15:28:05 +0000
|
||||
Subject: [PATCH 1/6] Issue 4418 - ldif2db - offline. Warn the user of skipped
|
||||
entries
|
||||
|
||||
Bug Description: During an ldif2db import entries that do not
|
||||
conform to various constraints will be skipped and not imported.
|
||||
On completition of an import with skipped entries, the server
|
||||
returns a success exit code and logs the skipped entry detail to
|
||||
the error logs. The success exit code could lead the user to
|
||||
believe that all entries were successfully imported.
|
||||
|
||||
Fix Description: If a skipped entry occurs during import, the
|
||||
import will continue and a warning will be returned to the user.
|
||||
|
||||
CLI tools for offline import updated to handle warning code.
|
||||
|
||||
Test added to generate an incorrect ldif entry and perform an
|
||||
import.
|
||||
|
||||
Fixes: #4418
|
||||
|
||||
Reviewed by: Firstyear, droideck (Thanks)
|
||||
|
||||
(cherry picked from commit a98fe54292e9b183a2163efbc7bdfe208d4abfb0)
|
||||
---
|
||||
.../tests/suites/import/import_test.py | 54 ++++++++++++++++++-
|
||||
.../slapd/back-ldbm/db-bdb/bdb_import.c | 22 ++++++--
|
||||
ldap/servers/slapd/main.c | 8 +++
|
||||
ldap/servers/slapd/pblock.c | 24 +++++++++
|
||||
ldap/servers/slapd/pblock_v3.h | 1 +
|
||||
ldap/servers/slapd/slapi-private.h | 14 +++++
|
||||
src/lib389/lib389/__init__.py | 18 +++----
|
||||
src/lib389/lib389/_constants.py | 7 +++
|
||||
src/lib389/lib389/cli_ctl/dbtasks.py | 8 ++-
|
||||
9 files changed, 140 insertions(+), 16 deletions(-)
|
||||
|
||||
diff --git a/dirsrvtests/tests/suites/import/import_test.py b/dirsrvtests/tests/suites/import/import_test.py
|
||||
index 3803ecf43..b47db96ed 100644
|
||||
--- a/dirsrvtests/tests/suites/import/import_test.py
|
||||
+++ b/dirsrvtests/tests/suites/import/import_test.py
|
||||
@@ -15,7 +15,7 @@ import pytest
|
||||
import time
|
||||
import glob
|
||||
from lib389.topologies import topology_st as topo
|
||||
-from lib389._constants import DEFAULT_SUFFIX
|
||||
+from lib389._constants import DEFAULT_SUFFIX, TaskWarning
|
||||
from lib389.dbgen import dbgen_users
|
||||
from lib389.tasks import ImportTask
|
||||
from lib389.index import Indexes
|
||||
@@ -139,6 +139,38 @@ def _create_bogus_ldif(topo):
|
||||
return import_ldif1
|
||||
|
||||
|
||||
+def _create_syntax_err_ldif(topo):
|
||||
+ """
|
||||
+ Create an incorrect ldif entry that violates syntax check
|
||||
+ """
|
||||
+ ldif_dir = topo.standalone.get_ldif_dir()
|
||||
+ line1 = """dn: dc=example,dc=com
|
||||
+objectClass: top
|
||||
+objectClass: domain
|
||||
+dc: example
|
||||
+dn: ou=groups,dc=example,dc=com
|
||||
+objectClass: top
|
||||
+objectClass: organizationalUnit
|
||||
+ou: groups
|
||||
+dn: uid=JHunt,ou=groups,dc=example,dc=com
|
||||
+objectClass: top
|
||||
+objectClass: person
|
||||
+objectClass: organizationalPerson
|
||||
+objectClass: inetOrgPerson
|
||||
+objectclass: inetUser
|
||||
+cn: James Hunt
|
||||
+sn: Hunt
|
||||
+uid: JHunt
|
||||
+givenName:
|
||||
+"""
|
||||
+ with open(f'{ldif_dir}/syntax_err.ldif', 'w') as out:
|
||||
+ out.write(f'{line1}')
|
||||
+ os.chmod(out.name, 0o777)
|
||||
+ out.close()
|
||||
+ import_ldif1 = ldif_dir + '/syntax_err.ldif'
|
||||
+ return import_ldif1
|
||||
+
|
||||
+
|
||||
def test_import_with_index(topo, _import_clean):
|
||||
"""
|
||||
Add an index, then import via cn=tasks
|
||||
@@ -214,6 +246,26 @@ def test_ldif2db_allows_entries_without_a_parent_to_be_imported(topo, _import_cl
|
||||
topo.standalone.start()
|
||||
|
||||
|
||||
+def test_ldif2db_syntax_check(topo):
|
||||
+ """ldif2db should return a warning when a skipped entry has occured.
|
||||
+ :id: 85e75670-42c5-4062-9edc-7f117c97a06f
|
||||
+ :setup:
|
||||
+ 1. Standalone Instance
|
||||
+ 2. Ldif entry that violates syntax check rule (empty givenname)
|
||||
+ :steps:
|
||||
+ 1. Create an ldif file which violates the syntax checking rule
|
||||
+ 2. Stop the server and import ldif file with ldif2db
|
||||
+ :expected results:
|
||||
+ 1. ldif2db import returns a warning to signify skipped entries
|
||||
+ """
|
||||
+ import_ldif1 = _create_syntax_err_ldif(topo)
|
||||
+ # Import the offending LDIF data - offline
|
||||
+ topo.standalone.stop()
|
||||
+ ret = topo.standalone.ldif2db('userRoot', None, None, None, import_ldif1)
|
||||
+ assert ret == TaskWarning.WARN_SKIPPED_IMPORT_ENTRY
|
||||
+ topo.standalone.start()
|
||||
+
|
||||
+
|
||||
def test_issue_a_warning_if_the_cache_size_is_smaller(topo, _import_clean):
|
||||
"""Report during startup if nsslapd-cachememsize is too small
|
||||
|
||||
diff --git a/ldap/servers/slapd/back-ldbm/db-bdb/bdb_import.c b/ldap/servers/slapd/back-ldbm/db-bdb/bdb_import.c
|
||||
index e7da0517f..1e4830e99 100644
|
||||
--- a/ldap/servers/slapd/back-ldbm/db-bdb/bdb_import.c
|
||||
+++ b/ldap/servers/slapd/back-ldbm/db-bdb/bdb_import.c
|
||||
@@ -2563,7 +2563,7 @@ error:
|
||||
slapi_task_dec_refcount(job->task);
|
||||
}
|
||||
import_all_done(job, ret);
|
||||
- ret = 1;
|
||||
+ ret |= WARN_UPGARDE_DN_FORMAT_ALL;
|
||||
} else if (NEED_DN_NORM == ret) {
|
||||
import_log_notice(job, SLAPI_LOG_NOTICE, "bdb_import_main",
|
||||
"%s complete. %s needs upgradednformat.",
|
||||
@@ -2572,7 +2572,7 @@ error:
|
||||
slapi_task_dec_refcount(job->task);
|
||||
}
|
||||
import_all_done(job, ret);
|
||||
- ret = 2;
|
||||
+ ret |= WARN_UPGRADE_DN_FORMAT;
|
||||
} else if (NEED_DN_NORM_SP == ret) {
|
||||
import_log_notice(job, SLAPI_LOG_NOTICE, "bdb_import_main",
|
||||
"%s complete. %s needs upgradednformat spaces.",
|
||||
@@ -2581,7 +2581,7 @@ error:
|
||||
slapi_task_dec_refcount(job->task);
|
||||
}
|
||||
import_all_done(job, ret);
|
||||
- ret = 3;
|
||||
+ ret |= WARN_UPGRADE_DN_FORMAT_SPACE;
|
||||
} else {
|
||||
ret = -1;
|
||||
if (job->task != NULL) {
|
||||
@@ -2600,6 +2600,11 @@ error:
|
||||
import_all_done(job, ret);
|
||||
}
|
||||
|
||||
+ /* set task warning if there are no errors */
|
||||
+ if((!ret) && (job->skipped)) {
|
||||
+ ret |= WARN_SKIPPED_IMPORT_ENTRY;
|
||||
+ }
|
||||
+
|
||||
/* This instance isn't busy anymore */
|
||||
instance_set_not_busy(job->inst);
|
||||
|
||||
@@ -2637,6 +2642,7 @@ bdb_back_ldif2db(Slapi_PBlock *pb)
|
||||
int total_files, i;
|
||||
int up_flags = 0;
|
||||
PRThread *thread = NULL;
|
||||
+ int ret = 0;
|
||||
|
||||
slapi_pblock_get(pb, SLAPI_BACKEND, &be);
|
||||
if (be == NULL) {
|
||||
@@ -2764,7 +2770,15 @@ bdb_back_ldif2db(Slapi_PBlock *pb)
|
||||
}
|
||||
|
||||
/* old style -- do it all synchronously (THIS IS GOING AWAY SOON) */
|
||||
- return import_main_offline((void *)job);
|
||||
+ ret = import_main_offline((void *)job);
|
||||
+
|
||||
+ /* no error just warning, reset ret */
|
||||
+ if(ret &= WARN_SKIPPED_IMPORT_ENTRY) {
|
||||
+ slapi_pblock_set_task_warning(pb, WARN_SKIPPED_IMPORT_ENTRY);
|
||||
+ ret = 0;
|
||||
+ }
|
||||
+
|
||||
+ return ret;
|
||||
}
|
||||
|
||||
struct _import_merge_thang
|
||||
diff --git a/ldap/servers/slapd/main.c b/ldap/servers/slapd/main.c
|
||||
index 694375b22..104f6826c 100644
|
||||
--- a/ldap/servers/slapd/main.c
|
||||
+++ b/ldap/servers/slapd/main.c
|
||||
@@ -2069,6 +2069,14 @@ slapd_exemode_ldif2db(struct main_config *mcfg)
|
||||
plugin->plg_name);
|
||||
return_value = -1;
|
||||
}
|
||||
+
|
||||
+ /* check for task warnings */
|
||||
+ if(!return_value) {
|
||||
+ if((return_value = slapi_pblock_get_task_warning(pb))) {
|
||||
+ slapi_log_err(SLAPI_LOG_INFO, "slapd_exemode_ldif2db","returning task warning: %d\n", return_value);
|
||||
+ }
|
||||
+ }
|
||||
+
|
||||
slapi_pblock_destroy(pb);
|
||||
charray_free(instances);
|
||||
charray_free(mcfg->cmd_line_instance_names);
|
||||
diff --git a/ldap/servers/slapd/pblock.c b/ldap/servers/slapd/pblock.c
|
||||
index 454ea9cc3..1ad9d0399 100644
|
||||
--- a/ldap/servers/slapd/pblock.c
|
||||
+++ b/ldap/servers/slapd/pblock.c
|
||||
@@ -28,12 +28,14 @@
|
||||
#define SLAPI_LDIF_DUMP_REPLICA 2003
|
||||
#define SLAPI_PWDPOLICY 2004
|
||||
#define SLAPI_PW_ENTRY 2005
|
||||
+#define SLAPI_TASK_WARNING 2006
|
||||
|
||||
/* Used for checking assertions about pblocks in some cases. */
|
||||
#define SLAPI_HINT 9999
|
||||
|
||||
static PRLock *pblock_analytics_lock = NULL;
|
||||
|
||||
+
|
||||
static PLHashNumber
|
||||
hash_int_func(const void *key)
|
||||
{
|
||||
@@ -4315,6 +4317,28 @@ slapi_pblock_set_ldif_dump_replica(Slapi_PBlock *pb, int32_t dump_replica)
|
||||
pb->pb_task->ldif_dump_replica = dump_replica;
|
||||
}
|
||||
|
||||
+int32_t
|
||||
+slapi_pblock_get_task_warning(Slapi_PBlock *pb)
|
||||
+{
|
||||
+#ifdef PBLOCK_ANALYTICS
|
||||
+ pblock_analytics_record(pb, SLAPI_TASK_WARNING);
|
||||
+#endif
|
||||
+ if (pb->pb_task != NULL) {
|
||||
+ return pb->pb_task->task_warning;
|
||||
+ }
|
||||
+ return 0;
|
||||
+}
|
||||
+
|
||||
+void
|
||||
+slapi_pblock_set_task_warning(Slapi_PBlock *pb, task_warning warning)
|
||||
+{
|
||||
+#ifdef PBLOCK_ANALYTICS
|
||||
+ pblock_analytics_record(pb, SLAPI_TASK_WARNING);
|
||||
+#endif
|
||||
+ _pblock_assert_pb_task(pb);
|
||||
+ pb->pb_task->task_warning = warning;
|
||||
+}
|
||||
+
|
||||
void *
|
||||
slapi_pblock_get_vattr_context(Slapi_PBlock *pb)
|
||||
{
|
||||
diff --git a/ldap/servers/slapd/pblock_v3.h b/ldap/servers/slapd/pblock_v3.h
|
||||
index 90498c0b0..b35d78565 100644
|
||||
--- a/ldap/servers/slapd/pblock_v3.h
|
||||
+++ b/ldap/servers/slapd/pblock_v3.h
|
||||
@@ -67,6 +67,7 @@ typedef struct _slapi_pblock_task
|
||||
int ldif2db_noattrindexes;
|
||||
int ldif_printkey;
|
||||
int task_flags;
|
||||
+ int32_t task_warning;
|
||||
int import_state;
|
||||
|
||||
int server_running; /* indicate that server is running */
|
||||
diff --git a/ldap/servers/slapd/slapi-private.h b/ldap/servers/slapd/slapi-private.h
|
||||
index c98c1947c..31cb33472 100644
|
||||
--- a/ldap/servers/slapd/slapi-private.h
|
||||
+++ b/ldap/servers/slapd/slapi-private.h
|
||||
@@ -1465,6 +1465,20 @@ void slapi_pblock_set_operation_notes(Slapi_PBlock *pb, uint32_t opnotes);
|
||||
void slapi_pblock_set_flag_operation_notes(Slapi_PBlock *pb, uint32_t opflag);
|
||||
void slapi_pblock_set_result_text_if_empty(Slapi_PBlock *pb, char *text);
|
||||
|
||||
+/* task warnings */
|
||||
+typedef enum task_warning_t{
|
||||
+ WARN_UPGARDE_DN_FORMAT_ALL = (1 << 0),
|
||||
+ WARN_UPGRADE_DN_FORMAT = (1 << 1),
|
||||
+ WARN_UPGRADE_DN_FORMAT_SPACE = (1 << 2),
|
||||
+ WARN_SKIPPED_IMPORT_ENTRY = (1 << 3)
|
||||
+} task_warning;
|
||||
+
|
||||
+int32_t slapi_pblock_get_task_warning(Slapi_PBlock *pb);
|
||||
+void slapi_pblock_set_task_warning(Slapi_PBlock *pb, task_warning warn);
|
||||
+
|
||||
+
|
||||
+int slapi_exists_or_add_internal(Slapi_DN *dn, const char *filter, const char *entry, const char *modifier_name);
|
||||
+
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
diff --git a/src/lib389/lib389/__init__.py b/src/lib389/lib389/__init__.py
|
||||
index 4e6a1905a..5b36a79e1 100644
|
||||
--- a/src/lib389/lib389/__init__.py
|
||||
+++ b/src/lib389/lib389/__init__.py
|
||||
@@ -2683,7 +2683,7 @@ class DirSrv(SimpleLDAPObject, object):
|
||||
# server is stopped)
|
||||
#
|
||||
def ldif2db(self, bename, suffixes, excludeSuffixes, encrypt,
|
||||
- import_file):
|
||||
+ import_file, import_cl):
|
||||
"""
|
||||
@param bename - The backend name of the database to import
|
||||
@param suffixes - List/tuple of suffixes to import
|
||||
@@ -2731,14 +2731,14 @@ class DirSrv(SimpleLDAPObject, object):
|
||||
try:
|
||||
result = subprocess.check_output(cmd, encoding='utf-8')
|
||||
except subprocess.CalledProcessError as e:
|
||||
- self.log.debug("Command: %s failed with the return code %s and the error %s",
|
||||
- format_cmd_list(cmd), e.returncode, e.output)
|
||||
- return False
|
||||
-
|
||||
- self.log.debug("ldif2db output: BEGIN")
|
||||
- for line in result.split("\n"):
|
||||
- self.log.debug(line)
|
||||
- self.log.debug("ldif2db output: END")
|
||||
+ if e.returncode == TaskWarning.WARN_SKIPPED_IMPORT_ENTRY:
|
||||
+ self.log.debug("Command: %s skipped import entry warning %s",
|
||||
+ format_cmd_list(cmd), e.returncode)
|
||||
+ return e.returncode
|
||||
+ else:
|
||||
+ self.log.debug("Command: %s failed with the return code %s and the error %s",
|
||||
+ format_cmd_list(cmd), e.returncode, e.output)
|
||||
+ return False
|
||||
|
||||
return True
|
||||
|
||||
diff --git a/src/lib389/lib389/_constants.py b/src/lib389/lib389/_constants.py
|
||||
index e28c602a3..38ba04565 100644
|
||||
--- a/src/lib389/lib389/_constants.py
|
||||
+++ b/src/lib389/lib389/_constants.py
|
||||
@@ -162,6 +162,13 @@ DB2BAK = 'db2bak'
|
||||
DB2INDEX = 'db2index'
|
||||
DBSCAN = 'dbscan'
|
||||
|
||||
+# Task warnings
|
||||
+class TaskWarning(IntEnum):
|
||||
+ WARN_UPGARDE_DN_FORMAT_ALL = (1 << 0)
|
||||
+ WARN_UPGRADE_DN_FORMAT = (1 << 1)
|
||||
+ WARN_UPGRADE_DN_FORMAT_SPACE = (1 << 2)
|
||||
+ WARN_SKIPPED_IMPORT_ENTRY = (1 << 3)
|
||||
+
|
||||
RDN_REPLICA = "cn=replica"
|
||||
|
||||
RETROCL_SUFFIX = "cn=changelog"
|
||||
diff --git a/src/lib389/lib389/cli_ctl/dbtasks.py b/src/lib389/lib389/cli_ctl/dbtasks.py
|
||||
index 590a1ea0e..02830239c 100644
|
||||
--- a/src/lib389/lib389/cli_ctl/dbtasks.py
|
||||
+++ b/src/lib389/lib389/cli_ctl/dbtasks.py
|
||||
@@ -7,6 +7,7 @@
|
||||
# See LICENSE for details.
|
||||
# --- END COPYRIGHT BLOCK ---
|
||||
|
||||
+from lib389._constants import TaskWarning
|
||||
|
||||
def dbtasks_db2index(inst, log, args):
|
||||
if not inst.db2index(bename=args.backend):
|
||||
@@ -44,10 +45,13 @@ def dbtasks_db2ldif(inst, log, args):
|
||||
|
||||
|
||||
def dbtasks_ldif2db(inst, log, args):
|
||||
- if not inst.ldif2db(bename=args.backend, encrypt=args.encrypted, import_file=args.ldif,
|
||||
- suffixes=None, excludeSuffixes=None):
|
||||
+ ret = inst.ldif2db(bename=args.backend, encrypt=args.encrypted, import_file=args.ldif,
|
||||
+ suffixes=None, excludeSuffixes=None, import_cl=False)
|
||||
+ if not ret:
|
||||
log.fatal("ldif2db failed")
|
||||
return False
|
||||
+ elif ret == TaskWarning.WARN_SKIPPED_IMPORT_ENTRY:
|
||||
+ log.warn("ldif2db successful with skipped entries")
|
||||
else:
|
||||
log.info("ldif2db successful")
|
||||
|
||||
--
|
||||
2.26.2
|
||||
|
@ -0,0 +1,52 @@
|
||||
From 97bdef2d562e447d521202beb485c3948b0e7214 Mon Sep 17 00:00:00 2001
|
||||
From: James Chapman <jachapma@redhat.com>
|
||||
Date: Mon, 30 Nov 2020 15:28:05 +0000
|
||||
Subject: [PATCH 2/6] Issue 4418 - ldif2db - offline. Warn the user of skipped
|
||||
entries
|
||||
|
||||
Bug Description: During an ldif2db import entries that do not
|
||||
conform to various constraints will be skipped and not imported.
|
||||
On completition of an import with skipped entries, the server
|
||||
returns a success exit code and logs the skipped entry detail to
|
||||
the error logs. The success exit code could lead the user to
|
||||
believe that all entries were successfully imported.
|
||||
|
||||
Fix Description: If a skipped entry occurs during import, the
|
||||
import will continue and a warning will be returned to the user.
|
||||
|
||||
CLI tools for offline import updated to handle warning code.
|
||||
|
||||
Test added to generate an incorrect ldif entry and perform an
|
||||
import.
|
||||
|
||||
Fixes: #4418
|
||||
|
||||
Reviewed by: Firstyear, droideck (Thanks)
|
||||
---
|
||||
ldap/servers/slapd/slapi-private.h | 10 ++++++++++
|
||||
1 file changed, 10 insertions(+)
|
||||
|
||||
diff --git a/ldap/servers/slapd/slapi-private.h b/ldap/servers/slapd/slapi-private.h
|
||||
index 31cb33472..e0092d571 100644
|
||||
--- a/ldap/servers/slapd/slapi-private.h
|
||||
+++ b/ldap/servers/slapd/slapi-private.h
|
||||
@@ -1476,6 +1476,16 @@ typedef enum task_warning_t{
|
||||
int32_t slapi_pblock_get_task_warning(Slapi_PBlock *pb);
|
||||
void slapi_pblock_set_task_warning(Slapi_PBlock *pb, task_warning warn);
|
||||
|
||||
+/* task warnings */
|
||||
+typedef enum task_warning_t{
|
||||
+ WARN_UPGARDE_DN_FORMAT_ALL = (1 << 0),
|
||||
+ WARN_UPGRADE_DN_FORMAT = (1 << 1),
|
||||
+ WARN_UPGRADE_DN_FORMAT_SPACE = (1 << 2),
|
||||
+ WARN_SKIPPED_IMPORT_ENTRY = (1 << 3)
|
||||
+} task_warning;
|
||||
+
|
||||
+int32_t slapi_pblock_get_task_warning(Slapi_PBlock *pb);
|
||||
+void slapi_pblock_set_task_warning(Slapi_PBlock *pb, task_warning warn);
|
||||
|
||||
int slapi_exists_or_add_internal(Slapi_DN *dn, const char *filter, const char *entry, const char *modifier_name);
|
||||
|
||||
--
|
||||
2.26.2
|
||||
|
34
SOURCES/0022-Fix-cherry-pick-erorr.patch
Normal file
34
SOURCES/0022-Fix-cherry-pick-erorr.patch
Normal file
@ -0,0 +1,34 @@
|
||||
From 22fb8b2690a5fa364d252846f06b77b5fec8c602 Mon Sep 17 00:00:00 2001
|
||||
From: Mark Reynolds <mreynolds@redhat.com>
|
||||
Date: Thu, 7 Jan 2021 10:27:43 -0500
|
||||
Subject: [PATCH 3/6] Fix cherry-pick erorr
|
||||
|
||||
---
|
||||
ldap/servers/slapd/slapi-private.h | 11 -----------
|
||||
1 file changed, 11 deletions(-)
|
||||
|
||||
diff --git a/ldap/servers/slapd/slapi-private.h b/ldap/servers/slapd/slapi-private.h
|
||||
index e0092d571..d5abe8ac1 100644
|
||||
--- a/ldap/servers/slapd/slapi-private.h
|
||||
+++ b/ldap/servers/slapd/slapi-private.h
|
||||
@@ -1476,17 +1476,6 @@ typedef enum task_warning_t{
|
||||
int32_t slapi_pblock_get_task_warning(Slapi_PBlock *pb);
|
||||
void slapi_pblock_set_task_warning(Slapi_PBlock *pb, task_warning warn);
|
||||
|
||||
-/* task warnings */
|
||||
-typedef enum task_warning_t{
|
||||
- WARN_UPGARDE_DN_FORMAT_ALL = (1 << 0),
|
||||
- WARN_UPGRADE_DN_FORMAT = (1 << 1),
|
||||
- WARN_UPGRADE_DN_FORMAT_SPACE = (1 << 2),
|
||||
- WARN_SKIPPED_IMPORT_ENTRY = (1 << 3)
|
||||
-} task_warning;
|
||||
-
|
||||
-int32_t slapi_pblock_get_task_warning(Slapi_PBlock *pb);
|
||||
-void slapi_pblock_set_task_warning(Slapi_PBlock *pb, task_warning warn);
|
||||
-
|
||||
int slapi_exists_or_add_internal(Slapi_DN *dn, const char *filter, const char *entry, const char *modifier_name);
|
||||
|
||||
#ifdef __cplusplus
|
||||
--
|
||||
2.26.2
|
||||
|
@ -0,0 +1,393 @@
|
||||
From 43f8a317bcd9040874b27cad905347a9e6bc8a6f Mon Sep 17 00:00:00 2001
|
||||
From: James Chapman <jachapma@redhat.com>
|
||||
Date: Wed, 9 Dec 2020 22:42:59 +0000
|
||||
Subject: [PATCH 4/6] Issue 4419 - Warn users of skipped entries during ldif2db
|
||||
online import (#4476)
|
||||
|
||||
Bug Description: During an online ldif2db import entries that do not
|
||||
conform to various constraints will be skipped and
|
||||
not imported. On completition of an import with skipped
|
||||
entries, the server responds with a success message
|
||||
and logs the skipped entry detail to the error logs.
|
||||
The success messgae could lead the user to believe
|
||||
that all entries were successfully imported.
|
||||
|
||||
Fix Description: If a skipped entry occurs during import, the import
|
||||
will continue and a warning message will be displayed.
|
||||
The schema is extended with a nsTaskWarning attribute
|
||||
which is used to capture and retrieve any task
|
||||
warnings.
|
||||
|
||||
CLI tools for online import updated.
|
||||
|
||||
Test added to generate an incorrect ldif entry and perform an
|
||||
online import.
|
||||
|
||||
Fixes: https://github.com/389ds/389-ds-base/issues/4419
|
||||
|
||||
Reviewed by: tbordaz, mreynolds389, droideck, Firstyear (Thanks)
|
||||
---
|
||||
.../tests/suites/import/import_test.py | 39 +++++++++++++++++--
|
||||
ldap/schema/02common.ldif | 3 +-
|
||||
.../back-ldbm/db-bdb/bdb_import_threads.c | 5 +++
|
||||
ldap/servers/slapd/slap.h | 1 +
|
||||
ldap/servers/slapd/slapi-plugin.h | 11 ++++++
|
||||
ldap/servers/slapd/slapi-private.h | 8 ----
|
||||
ldap/servers/slapd/task.c | 29 +++++++++++++-
|
||||
src/lib389/lib389/cli_conf/backend.py | 6 ++-
|
||||
src/lib389/lib389/tasks.py | 23 +++++++++--
|
||||
9 files changed, 108 insertions(+), 17 deletions(-)
|
||||
|
||||
diff --git a/dirsrvtests/tests/suites/import/import_test.py b/dirsrvtests/tests/suites/import/import_test.py
|
||||
index b47db96ed..77c915026 100644
|
||||
--- a/dirsrvtests/tests/suites/import/import_test.py
|
||||
+++ b/dirsrvtests/tests/suites/import/import_test.py
|
||||
@@ -65,6 +65,9 @@ def _import_clean(request, topo):
|
||||
import_ldif = ldif_dir + '/basic_import.ldif'
|
||||
if os.path.exists(import_ldif):
|
||||
os.remove(import_ldif)
|
||||
+ syntax_err_ldif = ldif_dir + '/syntax_err.dif'
|
||||
+ if os.path.exists(syntax_err_ldif):
|
||||
+ os.remove(syntax_err_ldif)
|
||||
|
||||
request.addfinalizer(finofaci)
|
||||
|
||||
@@ -141,17 +144,19 @@ def _create_bogus_ldif(topo):
|
||||
|
||||
def _create_syntax_err_ldif(topo):
|
||||
"""
|
||||
- Create an incorrect ldif entry that violates syntax check
|
||||
+ Create an ldif file, which contains an entry that violates syntax check
|
||||
"""
|
||||
ldif_dir = topo.standalone.get_ldif_dir()
|
||||
line1 = """dn: dc=example,dc=com
|
||||
objectClass: top
|
||||
objectClass: domain
|
||||
dc: example
|
||||
+
|
||||
dn: ou=groups,dc=example,dc=com
|
||||
objectClass: top
|
||||
objectClass: organizationalUnit
|
||||
ou: groups
|
||||
+
|
||||
dn: uid=JHunt,ou=groups,dc=example,dc=com
|
||||
objectClass: top
|
||||
objectClass: person
|
||||
@@ -201,6 +206,34 @@ def test_import_with_index(topo, _import_clean):
|
||||
assert f'{place}/userRoot/roomNumber.db' in glob.glob(f'{place}/userRoot/*.db', recursive=True)
|
||||
|
||||
|
||||
+def test_online_import_with_warning(topo, _import_clean):
|
||||
+ """
|
||||
+ Import an ldif file with syntax errors, verify skipped entry warning code
|
||||
+
|
||||
+ :id: 5bf75c47-a283-430e-a65c-3c5fd8dbadb8
|
||||
+ :setup: Standalone Instance
|
||||
+ :steps:
|
||||
+ 1. Create standalone Instance
|
||||
+ 2. Create an ldif file with an entry that violates syntax check (empty givenname)
|
||||
+ 3. Online import of troublesome ldif file
|
||||
+ :expected results:
|
||||
+ 1. Successful import with skipped entry warning
|
||||
+ """
|
||||
+ topo.standalone.restart()
|
||||
+
|
||||
+ import_task = ImportTask(topo.standalone)
|
||||
+ import_ldif1 = _create_syntax_err_ldif(topo)
|
||||
+
|
||||
+ # Importing the offending ldif file - online
|
||||
+ import_task.import_suffix_from_ldif(ldiffile=import_ldif1, suffix=DEFAULT_SUFFIX)
|
||||
+
|
||||
+ # There is just a single entry in this ldif
|
||||
+ import_task.wait(5)
|
||||
+
|
||||
+ # Check for the task nsTaskWarning attr, make sure its set to skipped entry code
|
||||
+ assert import_task.present('nstaskwarning')
|
||||
+ assert TaskWarning.WARN_SKIPPED_IMPORT_ENTRY == import_task.get_task_warn()
|
||||
+
|
||||
def test_crash_on_ldif2db(topo, _import_clean):
|
||||
"""
|
||||
Delete the cn=monitor entry for an LDBM backend instance. Doing this will
|
||||
@@ -246,7 +279,7 @@ def test_ldif2db_allows_entries_without_a_parent_to_be_imported(topo, _import_cl
|
||||
topo.standalone.start()
|
||||
|
||||
|
||||
-def test_ldif2db_syntax_check(topo):
|
||||
+def test_ldif2db_syntax_check(topo, _import_clean):
|
||||
"""ldif2db should return a warning when a skipped entry has occured.
|
||||
:id: 85e75670-42c5-4062-9edc-7f117c97a06f
|
||||
:setup:
|
||||
@@ -261,7 +294,7 @@ def test_ldif2db_syntax_check(topo):
|
||||
import_ldif1 = _create_syntax_err_ldif(topo)
|
||||
# Import the offending LDIF data - offline
|
||||
topo.standalone.stop()
|
||||
- ret = topo.standalone.ldif2db('userRoot', None, None, None, import_ldif1)
|
||||
+ ret = topo.standalone.ldif2db('userRoot', None, None, None, import_ldif1, None)
|
||||
assert ret == TaskWarning.WARN_SKIPPED_IMPORT_ENTRY
|
||||
topo.standalone.start()
|
||||
|
||||
diff --git a/ldap/schema/02common.ldif b/ldap/schema/02common.ldif
|
||||
index c6dc074db..821640d03 100644
|
||||
--- a/ldap/schema/02common.ldif
|
||||
+++ b/ldap/schema/02common.ldif
|
||||
@@ -145,6 +145,7 @@ attributeTypes: ( 2.16.840.1.113730.3.1.2356 NAME 'nsTaskExitCode' DESC 'Slapi T
|
||||
attributeTypes: ( 2.16.840.1.113730.3.1.2357 NAME 'nsTaskCurrentItem' DESC 'Slapi Task item' SYNTAX 1.3.6.1.4.1.1466.115.121.1.27 SINGLE-VALUE X-ORIGIN '389 Directory Server' )
|
||||
attributeTypes: ( 2.16.840.1.113730.3.1.2358 NAME 'nsTaskTotalItems' DESC 'Slapi Task total items' SYNTAX 1.3.6.1.4.1.1466.115.121.1.27 SINGLE-VALUE X-ORIGIN '389 Directory Server' )
|
||||
attributeTypes: ( 2.16.840.1.113730.3.1.2359 NAME 'nsTaskCreated' DESC 'Slapi Task creation date' SYNTAX 1.3.6.1.4.1.1466.115.121.1.24 SINGLE-VALUE X-ORIGIN '389 Directory Server' )
|
||||
+attributeTypes: ( 2.16.840.1.113730.3.1.2375 NAME 'nsTaskWarning' DESC 'Slapi Task warning code' SYNTAX 1.3.6.1.4.1.1466.115.121.1.27 SINGLE-VALUE X-ORIGIN '389 Directory Server' )
|
||||
#
|
||||
# objectclasses:
|
||||
#
|
||||
@@ -177,5 +178,5 @@ objectClasses: ( 2.16.840.1.113730.3.2.503 NAME 'nsDSWindowsReplicationAgreement
|
||||
objectClasses: ( 2.16.840.1.113730.3.2.128 NAME 'costemplate' DESC 'Netscape defined objectclass' SUP top MAY ( cn $ cospriority ) X-ORIGIN 'Netscape Directory Server' )
|
||||
objectClasses: ( 2.16.840.1.113730.3.2.304 NAME 'nsView' DESC 'Netscape defined objectclass' SUP top AUXILIARY MAY ( nsViewFilter $ description ) X-ORIGIN 'Netscape Directory Server' )
|
||||
objectClasses: ( 2.16.840.1.113730.3.2.316 NAME 'nsAttributeEncryption' DESC 'Netscape defined objectclass' SUP top MUST ( cn $ nsEncryptionAlgorithm ) X-ORIGIN 'Netscape Directory Server' )
|
||||
-objectClasses: ( 2.16.840.1.113730.3.2.335 NAME 'nsSlapiTask' DESC 'Slapi_Task objectclass' SUP top MUST ( cn ) MAY ( ttl $ nsTaskLog $ nsTaskStatus $ nsTaskExitCode $ nsTaskCurrentItem $ nsTaskTotalItems $ nsTaskCreated ) X-ORIGIN '389 Directory Server' )
|
||||
+objectClasses: ( 2.16.840.1.113730.3.2.335 NAME 'nsSlapiTask' DESC 'Slapi_Task objectclass' SUP top MUST ( cn ) MAY ( ttl $ nsTaskLog $ nsTaskStatus $ nsTaskExitCode $ nsTaskCurrentItem $ nsTaskTotalItems $ nsTaskCreated $ nsTaskWarning ) X-ORIGIN '389 Directory Server' )
|
||||
|
||||
diff --git a/ldap/servers/slapd/back-ldbm/db-bdb/bdb_import_threads.c b/ldap/servers/slapd/back-ldbm/db-bdb/bdb_import_threads.c
|
||||
index 310893884..5c7d9c8f7 100644
|
||||
--- a/ldap/servers/slapd/back-ldbm/db-bdb/bdb_import_threads.c
|
||||
+++ b/ldap/servers/slapd/back-ldbm/db-bdb/bdb_import_threads.c
|
||||
@@ -747,6 +747,11 @@ import_producer(void *param)
|
||||
}
|
||||
}
|
||||
|
||||
+ /* capture skipped entry warnings for this task */
|
||||
+ if((job) && (job->skipped)) {
|
||||
+ slapi_task_set_warning(job->task, WARN_SKIPPED_IMPORT_ENTRY);
|
||||
+ }
|
||||
+
|
||||
slapi_value_free(&(job->usn_value));
|
||||
import_free_ldif(&c);
|
||||
info->state = FINISHED;
|
||||
diff --git a/ldap/servers/slapd/slap.h b/ldap/servers/slapd/slap.h
|
||||
index 53c9161d1..be4d38739 100644
|
||||
--- a/ldap/servers/slapd/slap.h
|
||||
+++ b/ldap/servers/slapd/slap.h
|
||||
@@ -1753,6 +1753,7 @@ typedef struct slapi_task
|
||||
int task_progress; /* number between 0 and task_work */
|
||||
int task_work; /* "units" of work to be done */
|
||||
int task_flags; /* (see above) */
|
||||
+ task_warning task_warn; /* task warning */
|
||||
char *task_status; /* transient status info */
|
||||
char *task_log; /* appended warnings, etc */
|
||||
char task_date[SLAPI_TIMESTAMP_BUFSIZE]; /* Date/time when task was created */
|
||||
diff --git a/ldap/servers/slapd/slapi-plugin.h b/ldap/servers/slapd/slapi-plugin.h
|
||||
index 96313ef2c..ddb11bc7c 100644
|
||||
--- a/ldap/servers/slapd/slapi-plugin.h
|
||||
+++ b/ldap/servers/slapd/slapi-plugin.h
|
||||
@@ -6638,6 +6638,15 @@ int slapi_config_remove_callback(int operation, int flags, const char *base, int
|
||||
/* task flags (set by the task-control code) */
|
||||
#define SLAPI_TASK_DESTROYING 0x01 /* queued event for destruction */
|
||||
|
||||
+/* task warnings */
|
||||
+typedef enum task_warning_t{
|
||||
+ WARN_UPGARDE_DN_FORMAT_ALL = (1 << 0),
|
||||
+ WARN_UPGRADE_DN_FORMAT = (1 << 1),
|
||||
+ WARN_UPGRADE_DN_FORMAT_SPACE = (1 << 2),
|
||||
+ WARN_SKIPPED_IMPORT_ENTRY = (1 << 3)
|
||||
+} task_warning;
|
||||
+
|
||||
+
|
||||
int slapi_task_register_handler(const char *name, dseCallbackFn func);
|
||||
int slapi_plugin_task_register_handler(const char *name, dseCallbackFn func, Slapi_PBlock *plugin_pb);
|
||||
int slapi_plugin_task_unregister_handler(const char *name, dseCallbackFn func);
|
||||
@@ -6654,6 +6663,8 @@ int slapi_task_get_refcount(Slapi_Task *task);
|
||||
void slapi_task_set_destructor_fn(Slapi_Task *task, TaskCallbackFn func);
|
||||
void slapi_task_set_cancel_fn(Slapi_Task *task, TaskCallbackFn func);
|
||||
void slapi_task_status_changed(Slapi_Task *task);
|
||||
+void slapi_task_set_warning(Slapi_Task *task, task_warning warn);
|
||||
+int slapi_task_get_warning(Slapi_Task *task);
|
||||
void slapi_task_log_status(Slapi_Task *task, char *format, ...)
|
||||
#ifdef __GNUC__
|
||||
__attribute__((format(printf, 2, 3)));
|
||||
diff --git a/ldap/servers/slapd/slapi-private.h b/ldap/servers/slapd/slapi-private.h
|
||||
index d5abe8ac1..b956ebe63 100644
|
||||
--- a/ldap/servers/slapd/slapi-private.h
|
||||
+++ b/ldap/servers/slapd/slapi-private.h
|
||||
@@ -1465,14 +1465,6 @@ void slapi_pblock_set_operation_notes(Slapi_PBlock *pb, uint32_t opnotes);
|
||||
void slapi_pblock_set_flag_operation_notes(Slapi_PBlock *pb, uint32_t opflag);
|
||||
void slapi_pblock_set_result_text_if_empty(Slapi_PBlock *pb, char *text);
|
||||
|
||||
-/* task warnings */
|
||||
-typedef enum task_warning_t{
|
||||
- WARN_UPGARDE_DN_FORMAT_ALL = (1 << 0),
|
||||
- WARN_UPGRADE_DN_FORMAT = (1 << 1),
|
||||
- WARN_UPGRADE_DN_FORMAT_SPACE = (1 << 2),
|
||||
- WARN_SKIPPED_IMPORT_ENTRY = (1 << 3)
|
||||
-} task_warning;
|
||||
-
|
||||
int32_t slapi_pblock_get_task_warning(Slapi_PBlock *pb);
|
||||
void slapi_pblock_set_task_warning(Slapi_PBlock *pb, task_warning warn);
|
||||
|
||||
diff --git a/ldap/servers/slapd/task.c b/ldap/servers/slapd/task.c
|
||||
index 936c64920..806077a16 100644
|
||||
--- a/ldap/servers/slapd/task.c
|
||||
+++ b/ldap/servers/slapd/task.c
|
||||
@@ -46,6 +46,7 @@ static uint64_t shutting_down = 0;
|
||||
#define TASK_PROGRESS_NAME "nsTaskCurrentItem"
|
||||
#define TASK_WORK_NAME "nsTaskTotalItems"
|
||||
#define TASK_DATE_NAME "nsTaskCreated"
|
||||
+#define TASK_WARNING_NAME "nsTaskWarning"
|
||||
|
||||
#define DEFAULT_TTL "3600" /* seconds */
|
||||
#define TASK_SYSCONFIG_FILE_ATTR "sysconfigfile" /* sysconfig reload task file attr */
|
||||
@@ -332,7 +333,7 @@ slapi_task_status_changed(Slapi_Task *task)
|
||||
LDAPMod modlist[20];
|
||||
LDAPMod *mod[20];
|
||||
int cur = 0, i;
|
||||
- char s1[20], s2[20], s3[20];
|
||||
+ char s1[20], s2[20], s3[20], s4[20];
|
||||
|
||||
if (shutting_down) {
|
||||
/* don't care about task status updates anymore */
|
||||
@@ -346,9 +347,11 @@ slapi_task_status_changed(Slapi_Task *task)
|
||||
sprintf(s1, "%d", task->task_exitcode);
|
||||
sprintf(s2, "%d", task->task_progress);
|
||||
sprintf(s3, "%d", task->task_work);
|
||||
+ sprintf(s4, "%d", task->task_warn);
|
||||
NEXTMOD(TASK_PROGRESS_NAME, s2);
|
||||
NEXTMOD(TASK_WORK_NAME, s3);
|
||||
NEXTMOD(TASK_DATE_NAME, task->task_date);
|
||||
+ NEXTMOD(TASK_WARNING_NAME, s4);
|
||||
/* only add the exit code when the job is done */
|
||||
if ((task->task_state == SLAPI_TASK_FINISHED) ||
|
||||
(task->task_state == SLAPI_TASK_CANCELLED)) {
|
||||
@@ -452,6 +455,30 @@ slapi_task_get_refcount(Slapi_Task *task)
|
||||
return 0; /* return value not currently used */
|
||||
}
|
||||
|
||||
+/*
|
||||
+ * Return task warning
|
||||
+ */
|
||||
+int
|
||||
+slapi_task_get_warning(Slapi_Task *task)
|
||||
+{
|
||||
+ if (task) {
|
||||
+ return task->task_warn;
|
||||
+ }
|
||||
+
|
||||
+ return 0; /* return value not currently used */
|
||||
+}
|
||||
+
|
||||
+/*
|
||||
+ * Set task warning
|
||||
+ */
|
||||
+void
|
||||
+slapi_task_set_warning(Slapi_Task *task, task_warning warn)
|
||||
+{
|
||||
+ if (task) {
|
||||
+ return task->task_warn |= warn;
|
||||
+ }
|
||||
+}
|
||||
+
|
||||
int
|
||||
slapi_plugin_task_unregister_handler(const char *name, dseCallbackFn func)
|
||||
{
|
||||
diff --git a/src/lib389/lib389/cli_conf/backend.py b/src/lib389/lib389/cli_conf/backend.py
|
||||
index d7a6e670c..6bfbcb036 100644
|
||||
--- a/src/lib389/lib389/cli_conf/backend.py
|
||||
+++ b/src/lib389/lib389/cli_conf/backend.py
|
||||
@@ -243,9 +243,13 @@ def backend_import(inst, basedn, log, args):
|
||||
exclude_suffixes=args.exclude_suffixes)
|
||||
task.wait(timeout=None)
|
||||
result = task.get_exit_code()
|
||||
+ warning = task.get_task_warn()
|
||||
|
||||
if task.is_complete() and result == 0:
|
||||
- log.info("The import task has finished successfully")
|
||||
+ if warning is None or (warning == 0):
|
||||
+ log.info("The import task has finished successfully")
|
||||
+ else:
|
||||
+ log.info("The import task has finished successfully, with warning code {}, check the logs for more detail".format(warning))
|
||||
else:
|
||||
raise ValueError("Import task failed\n-------------------------\n{}".format(ensure_str(task.get_task_log())))
|
||||
|
||||
diff --git a/src/lib389/lib389/tasks.py b/src/lib389/lib389/tasks.py
|
||||
index dc7bb9206..bf20d1e61 100644
|
||||
--- a/src/lib389/lib389/tasks.py
|
||||
+++ b/src/lib389/lib389/tasks.py
|
||||
@@ -38,6 +38,7 @@ class Task(DSLdapObject):
|
||||
self._protected = False
|
||||
self._exit_code = None
|
||||
self._task_log = ""
|
||||
+ self._task_warn = None
|
||||
|
||||
def status(self):
|
||||
"""Return the decoded status of the task
|
||||
@@ -49,6 +50,7 @@ class Task(DSLdapObject):
|
||||
|
||||
self._exit_code = self.get_attr_val_utf8("nsTaskExitCode")
|
||||
self._task_log = self.get_attr_val_utf8("nsTaskLog")
|
||||
+ self._task_warn = self.get_attr_val_utf8("nsTaskWarning")
|
||||
if not self.exists():
|
||||
self._log.debug("complete: task has self cleaned ...")
|
||||
# The task cleaned it self up.
|
||||
@@ -77,6 +79,15 @@ class Task(DSLdapObject):
|
||||
return None
|
||||
return None
|
||||
|
||||
+ def get_task_warn(self):
|
||||
+ """Return task's warning code if task is complete, else None."""
|
||||
+ if self.is_complete():
|
||||
+ try:
|
||||
+ return int(self._task_warn)
|
||||
+ except TypeError:
|
||||
+ return None
|
||||
+ return None
|
||||
+
|
||||
def wait(self, timeout=120):
|
||||
"""Wait until task is complete."""
|
||||
|
||||
@@ -390,14 +401,17 @@ class Tasks(object):
|
||||
running, true if done - if true, second is the exit code - if dowait
|
||||
is True, this function will block until the task is complete'''
|
||||
attrlist = ['nsTaskLog', 'nsTaskStatus', 'nsTaskExitCode',
|
||||
- 'nsTaskCurrentItem', 'nsTaskTotalItems']
|
||||
+ 'nsTaskCurrentItem', 'nsTaskTotalItems', 'nsTaskWarning']
|
||||
done = False
|
||||
exitCode = 0
|
||||
+ warningCode = 0
|
||||
dn = entry.dn
|
||||
while not done:
|
||||
entry = self.conn.getEntry(dn, attrlist=attrlist)
|
||||
self.log.debug("task entry %r", entry)
|
||||
|
||||
+ if entry.nsTaskWarning:
|
||||
+ warningCode = int(entry.nsTaskWarning)
|
||||
if entry.nsTaskExitCode:
|
||||
exitCode = int(entry.nsTaskExitCode)
|
||||
done = True
|
||||
@@ -405,7 +419,7 @@ class Tasks(object):
|
||||
time.sleep(1)
|
||||
else:
|
||||
break
|
||||
- return (done, exitCode)
|
||||
+ return (done, exitCode, warningCode)
|
||||
|
||||
def importLDIF(self, suffix=None, benamebase=None, input_file=None,
|
||||
args=None):
|
||||
@@ -461,8 +475,9 @@ class Tasks(object):
|
||||
self.conn.add_s(entry)
|
||||
|
||||
exitCode = 0
|
||||
+ warningCode = 0
|
||||
if args and args.get(TASK_WAIT, False):
|
||||
- (done, exitCode) = self.conn.tasks.checkTask(entry, True)
|
||||
+ (done, exitCode, warningCode) = self.conn.tasks.checkTask(entry, True)
|
||||
|
||||
if exitCode:
|
||||
self.log.error("Error: import task %s for file %s exited with %d",
|
||||
@@ -470,6 +485,8 @@ class Tasks(object):
|
||||
else:
|
||||
self.log.info("Import task %s for file %s completed successfully",
|
||||
cn, input_file)
|
||||
+ if warningCode:
|
||||
+ self.log.info("with warning code %d", warningCode)
|
||||
self.dn = dn
|
||||
self.entry = entry
|
||||
return exitCode
|
||||
--
|
||||
2.26.2
|
||||
|
@ -0,0 +1,149 @@
|
||||
From 61d82ef842e0e4e013937bf05d7f640be2d2fc09 Mon Sep 17 00:00:00 2001
|
||||
From: tbordaz <tbordaz@redhat.com>
|
||||
Date: Wed, 16 Dec 2020 16:30:28 +0100
|
||||
Subject: [PATCH 5/6] Issue 4480 - Unexpected info returned to ldap request
|
||||
(#4491)
|
||||
|
||||
Bug description:
|
||||
If the bind entry does not exist, the bind result info
|
||||
reports that 'No such entry'. It should not give any
|
||||
information if the target entry exists or not
|
||||
|
||||
Fix description:
|
||||
Does not return any additional information during a bind
|
||||
|
||||
relates: https://github.com/389ds/389-ds-base/issues/4480
|
||||
|
||||
Reviewed by: William Brown, Viktor Ashirov, Mark Reynolds (thank you all)
|
||||
|
||||
Platforms tested: F31
|
||||
---
|
||||
dirsrvtests/tests/suites/basic/basic_test.py | 112 +++++++++++++++++++
|
||||
1 file changed, 112 insertions(+)
|
||||
|
||||
diff --git a/dirsrvtests/tests/suites/basic/basic_test.py b/dirsrvtests/tests/suites/basic/basic_test.py
|
||||
index 1ae82dcdd..02b73ee85 100644
|
||||
--- a/dirsrvtests/tests/suites/basic/basic_test.py
|
||||
+++ b/dirsrvtests/tests/suites/basic/basic_test.py
|
||||
@@ -1400,6 +1400,118 @@ def test_dscreate_multiple_dashes_name(dscreate_long_instance):
|
||||
assert not dscreate_long_instance.exists()
|
||||
|
||||
|
||||
+@pytest.fixture(scope="module", params=('c=uk', 'cn=test_user', 'dc=example,dc=com', 'o=south', 'ou=sales', 'wrong=some_value'))
|
||||
+def dscreate_test_rdn_value(request):
|
||||
+ template_file = "/tmp/dssetup.inf"
|
||||
+ template_text = f"""[general]
|
||||
+config_version = 2
|
||||
+# This invalid hostname ...
|
||||
+full_machine_name = localhost.localdomain
|
||||
+# Means we absolutely require this.
|
||||
+strict_host_checking = False
|
||||
+# In tests, we can be run in containers, NEVER trust
|
||||
+# that systemd is there, or functional in any capacity
|
||||
+systemd = False
|
||||
+
|
||||
+[slapd]
|
||||
+instance_name = test_different_rdn
|
||||
+root_dn = cn=directory manager
|
||||
+root_password = someLongPassword_123
|
||||
+# We do not have access to high ports in containers,
|
||||
+# so default to something higher.
|
||||
+port = 38999
|
||||
+secure_port = 63699
|
||||
+
|
||||
+[backend-userroot]
|
||||
+create_suffix_entry = True
|
||||
+suffix = {request.param}
|
||||
+"""
|
||||
+
|
||||
+ with open(template_file, "w") as template_fd:
|
||||
+ template_fd.write(template_text)
|
||||
+
|
||||
+ # Unset PYTHONPATH to avoid mixing old CLI tools and new lib389
|
||||
+ tmp_env = os.environ
|
||||
+ if "PYTHONPATH" in tmp_env:
|
||||
+ del tmp_env["PYTHONPATH"]
|
||||
+
|
||||
+ def fin():
|
||||
+ os.remove(template_file)
|
||||
+ if request.param != "wrong=some_value":
|
||||
+ try:
|
||||
+ subprocess.check_call(['dsctl', 'test_different_rdn', 'remove', '--do-it'])
|
||||
+ except subprocess.CalledProcessError as e:
|
||||
+ log.fatal(f"Failed to remove test instance Error ({e.returncode}) {e.output}")
|
||||
+ else:
|
||||
+ log.info("Wrong RDN is passed, instance not created")
|
||||
+ request.addfinalizer(fin)
|
||||
+ return template_file, tmp_env, request.param,
|
||||
+
|
||||
+
|
||||
+@pytest.mark.skipif(not get_user_is_root() or ds_is_older('1.4.0.0'),
|
||||
+ reason="This test is only required with new admin cli, and requires root.")
|
||||
+@pytest.mark.bz1807419
|
||||
+@pytest.mark.ds50928
|
||||
+def test_dscreate_with_different_rdn(dscreate_test_rdn_value):
|
||||
+ """Test that dscreate works with different RDN attributes as suffix
|
||||
+
|
||||
+ :id: 77ed6300-6a2f-4e79-a862-1f1105f1e3ef
|
||||
+ :parametrized: yes
|
||||
+ :setup: None
|
||||
+ :steps:
|
||||
+ 1. Create template file for dscreate with different RDN attributes as suffix
|
||||
+ 2. Create instance using template file
|
||||
+ 3. Create instance with 'wrong=some_value' as suffix's RDN attribute
|
||||
+ :expectedresults:
|
||||
+ 1. Should succeeds
|
||||
+ 2. Should succeeds
|
||||
+ 3. Should fail
|
||||
+ """
|
||||
+ try:
|
||||
+ subprocess.check_call([
|
||||
+ 'dscreate',
|
||||
+ 'from-file',
|
||||
+ dscreate_test_rdn_value[0]
|
||||
+ ], env=dscreate_test_rdn_value[1])
|
||||
+ except subprocess.CalledProcessError as e:
|
||||
+ log.fatal(f"dscreate failed! Error ({e.returncode}) {e.output}")
|
||||
+ if dscreate_test_rdn_value[2] != "wrong=some_value":
|
||||
+ assert False
|
||||
+ else:
|
||||
+ assert True
|
||||
+
|
||||
+def test_bind_invalid_entry(topology_st):
|
||||
+ """Test the failing bind does not return information about the entry
|
||||
+
|
||||
+ :id: 5cd9b083-eea6-426b-84ca-83c26fc49a6f
|
||||
+
|
||||
+ :setup: Standalone instance
|
||||
+
|
||||
+ :steps:
|
||||
+ 1: bind as non existing entry
|
||||
+ 2: check that bind info does not report 'No such entry'
|
||||
+
|
||||
+ :expectedresults:
|
||||
+ 1: pass
|
||||
+ 2: pass
|
||||
+ """
|
||||
+
|
||||
+ topology_st.standalone.restart()
|
||||
+ INVALID_ENTRY="cn=foooo,%s" % DEFAULT_SUFFIX
|
||||
+ try:
|
||||
+ topology_st.standalone.simple_bind_s(INVALID_ENTRY, PASSWORD)
|
||||
+ except ldap.LDAPError as e:
|
||||
+ log.info('test_bind_invalid_entry: Failed to bind as %s (expected)' % INVALID_ENTRY)
|
||||
+ log.info('exception description: ' + e.args[0]['desc'])
|
||||
+ if 'info' in e.args[0]:
|
||||
+ log.info('exception info: ' + e.args[0]['info'])
|
||||
+ assert e.args[0]['desc'] == 'Invalid credentials'
|
||||
+ assert 'info' not in e.args[0]
|
||||
+ pass
|
||||
+
|
||||
+ log.info('test_bind_invalid_entry: PASSED')
|
||||
+
|
||||
+
|
||||
if __name__ == '__main__':
|
||||
# Run isolated
|
||||
# -s for DEBUG mode
|
||||
--
|
||||
2.26.2
|
||||
|
@ -0,0 +1,99 @@
|
||||
From 3c74f736c657d007770fe866842b08d0a74772ca Mon Sep 17 00:00:00 2001
|
||||
From: Mark Reynolds <mreynolds@redhat.com>
|
||||
Date: Wed, 9 Dec 2020 15:21:11 -0500
|
||||
Subject: [PATCH 6/6] Issue 4414 - disk monitoring - prevent division by zero
|
||||
crash
|
||||
|
||||
Bug Description: If a disk mount has zero total space or zero used
|
||||
space then a division by zero can occur and the
|
||||
server will crash.
|
||||
|
||||
It has also been observed that sometimes a system
|
||||
can return the wrong disk entirely, and when that
|
||||
happens the incorrect disk also has zero available
|
||||
space which triggers the disk monitioring thread to
|
||||
immediately shut the server down.
|
||||
|
||||
Fix Description: Check the total and used space for zero and do not
|
||||
divide, just ignore it. As a preemptive measure
|
||||
ignore disks from /dev, /proc, /sys (except /dev/shm).
|
||||
Yes it's a bit hacky, but the true underlying cause
|
||||
is not known yet. So better to be safe than sorry.
|
||||
|
||||
Relates: https://github.com/389ds/389-ds-base/issues/4414
|
||||
|
||||
Reviewed by: firstyear(Thanks!)
|
||||
---
|
||||
ldap/servers/slapd/daemon.c | 22 +++++++++++++++++++++-
|
||||
ldap/servers/slapd/monitor.c | 13 +++++--------
|
||||
2 files changed, 26 insertions(+), 9 deletions(-)
|
||||
|
||||
diff --git a/ldap/servers/slapd/daemon.c b/ldap/servers/slapd/daemon.c
|
||||
index 691f77570..bfd965263 100644
|
||||
--- a/ldap/servers/slapd/daemon.c
|
||||
+++ b/ldap/servers/slapd/daemon.c
|
||||
@@ -221,7 +221,27 @@ disk_mon_get_mount_point(char *dir)
|
||||
}
|
||||
if (s.st_dev == dev_id) {
|
||||
endmntent(fp);
|
||||
- return (slapi_ch_strdup(mnt->mnt_dir));
|
||||
+
|
||||
+ if ((strncmp(mnt->mnt_dir, "/dev", 4) == 0 && strncmp(mnt->mnt_dir, "/dev/shm", 8) != 0) ||
|
||||
+ strncmp(mnt->mnt_dir, "/proc", 4) == 0 ||
|
||||
+ strncmp(mnt->mnt_dir, "/sys", 4) == 0)
|
||||
+ {
|
||||
+ /*
|
||||
+ * Ignore "mount directories" starting with /dev (except
|
||||
+ * /dev/shm), /proc, /sys For some reason these mounts are
|
||||
+ * occasionally/incorrectly returned. Only seen this at a
|
||||
+ * customer site once. When it happens it causes disk
|
||||
+ * monitoring to think the server has 0 disk space left, and
|
||||
+ * it abruptly/unexpectedly shuts the server down. At this
|
||||
+ * point it looks like a bug in stat(), setmntent(), or
|
||||
+ * getmntent(), but there is no way to prove that since there
|
||||
+ * is no way to reproduce the original issue. For now just
|
||||
+ * return NULL to be safe.
|
||||
+ */
|
||||
+ return NULL;
|
||||
+ } else {
|
||||
+ return (slapi_ch_strdup(mnt->mnt_dir));
|
||||
+ }
|
||||
}
|
||||
}
|
||||
endmntent(fp);
|
||||
diff --git a/ldap/servers/slapd/monitor.c b/ldap/servers/slapd/monitor.c
|
||||
index 562721bed..65f082986 100644
|
||||
--- a/ldap/servers/slapd/monitor.c
|
||||
+++ b/ldap/servers/slapd/monitor.c
|
||||
@@ -131,7 +131,6 @@ monitor_disk_info (Slapi_PBlock *pb __attribute__((unused)),
|
||||
{
|
||||
int32_t rc = LDAP_SUCCESS;
|
||||
char **dirs = NULL;
|
||||
- char buf[BUFSIZ];
|
||||
struct berval val;
|
||||
struct berval *vals[2];
|
||||
uint64_t total_space;
|
||||
@@ -143,15 +142,13 @@ monitor_disk_info (Slapi_PBlock *pb __attribute__((unused)),
|
||||
|
||||
disk_mon_get_dirs(&dirs);
|
||||
|
||||
- for (uint16_t i = 0; dirs && dirs[i]; i++) {
|
||||
+ for (size_t i = 0; dirs && dirs[i]; i++) {
|
||||
+ char buf[BUFSIZ] = {0};
|
||||
rc = disk_get_info(dirs[i], &total_space, &avail_space, &used_space);
|
||||
- if (rc) {
|
||||
- slapi_log_err(SLAPI_LOG_WARNING, "monitor_disk_info",
|
||||
- "Unable to get 'cn=disk space,cn=monitor' stats for %s\n", dirs[i]);
|
||||
- } else {
|
||||
+ if (rc == 0 && total_space > 0 && used_space > 0) {
|
||||
val.bv_len = snprintf(buf, sizeof(buf),
|
||||
- "partition=\"%s\" size=\"%" PRIu64 "\" used=\"%" PRIu64 "\" available=\"%" PRIu64 "\" use%%=\"%" PRIu64 "\"",
|
||||
- dirs[i], total_space, used_space, avail_space, used_space * 100 / total_space);
|
||||
+ "partition=\"%s\" size=\"%" PRIu64 "\" used=\"%" PRIu64 "\" available=\"%" PRIu64 "\" use%%=\"%" PRIu64 "\"",
|
||||
+ dirs[i], total_space, used_space, avail_space, used_space * 100 / total_space);
|
||||
val.bv_val = buf;
|
||||
attrlist_merge(&e->e_attrs, "dsDisk", vals);
|
||||
}
|
||||
--
|
||||
2.26.2
|
||||
|
@ -45,7 +45,7 @@ ExcludeArch: i686
|
||||
Summary: 389 Directory Server (base)
|
||||
Name: 389-ds-base
|
||||
Version: 1.4.3.16
|
||||
Release: %{?relprefix}6%{?prerel}%{?dist}
|
||||
Release: %{?relprefix}7%{?prerel}%{?dist}
|
||||
License: GPLv3+
|
||||
URL: https://www.port389.org
|
||||
Group: System Environment/Daemons
|
||||
@ -193,6 +193,12 @@ Patch16: 0016-Issue-4460-BUG-add-machine-name-to-subject-alt-names.patc
|
||||
Patch17: 0017-Issue-4483-heap-use-after-free-in-slapi_be_getsuffix.patch
|
||||
Patch18: 0018-Issue-4480-Unexpected-info-returned-to-ldap-request-.patch
|
||||
Patch19: 0019-Issue-4504-Fix-pytest-test_dsconf_replication_monito.patch
|
||||
Patch20: 0020-Issue-4418-ldif2db-offline.-Warn-the-user-of-skipped.patch
|
||||
Patch21: 0021-Issue-4418-ldif2db-offline.-Warn-the-user-of-skipped.patch
|
||||
Patch22: 0022-Fix-cherry-pick-erorr.patch
|
||||
Patch23: 0023-Issue-4419-Warn-users-of-skipped-entries-during-ldif.patch
|
||||
Patch24: 0024-Issue-4480-Unexpected-info-returned-to-ldap-request-.patch
|
||||
Patch25: 0025-Issue-4414-disk-monitoring-prevent-division-by-zero-.patch
|
||||
|
||||
%description
|
||||
389 Directory Server is an LDAPv3 compliant server. The base package includes
|
||||
@ -810,6 +816,12 @@ exit 0
|
||||
%doc README.md
|
||||
|
||||
%changelog
|
||||
* Thu Jan 7 2021 Mark Reynolds <mreynolds@redhat.com> - 1.4.3.16-7
|
||||
- Bump version to 1.4.3.16-7
|
||||
- Resolves: Bug 1890118 - SIGFPE crash in rhds disk monitoring routine
|
||||
- Resolves: Bug 1904991 - 389-ds:1.4/389-ds-base: information disclosure during the binding of a DN
|
||||
- Resolves: Bug 1627645 - ldif2db does not change exit code when there are skipped entries
|
||||
|
||||
* Wed Dec 16 2020 Mark Reynolds <mreynolds@redhat.com> - 1.4.3.16-6
|
||||
- Bump version to 1.4.3.16-6
|
||||
- Resolves: Bug 1879386 - cli dsconf replication monitor fails to retrieve database RUV - consumer (Unavailable) State (green) Reason (error (0)
|
||||
|
Loading…
Reference in New Issue
Block a user