rpms/rsyslog
7
0
Fork 0
Code Issues Pull Requests Releases Activity

import rsyslog-8.1911.0-2.el8

Browse Source
This commit is contained in:
CentOS Sources 2020-01-21 18:28:00 -05:00 committed by Stepan Oksanichenko
parent 52bfb17394
commit 3f7674c145
13 changed files with 215 additions and 2281 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
.gitignore vendored
View File

@ -1,2 +1,2 @@
SOURCES/rsyslog-8.37.0.tar.gz SOURCES/rsyslog-8.1911.0.tar.gz
SOURCES/rsyslog-doc-8.37.0.tar.gz SOURCES/rsyslog-doc-8.1911.0.tar.gz

4
.rsyslog.metadata
View File

@ -1,2 +1,2 @@
43076e3010fc3fd5178201a916beb93848b5249c SOURCES/rsyslog-8.37.0.tar.gz 30dfc2b99d73598788e2bd0d0ac45e16e7c3a3d5 SOURCES/rsyslog-8.1911.0.tar.gz
4c75f56e2d55c4c87d07781fb6d9deabf63395fb SOURCES/rsyslog-doc-8.37.0.tar.gz 8bcb23571ab8011b712ccf52acee20f8940b7f03 SOURCES/rsyslog-doc-8.1911.0.tar.gz

34
SOURCES/rsyslog-8.37.0-rhbz1659898-imjournal-default-tag.patch → SOURCES/rsyslog-8.1911.0-rhbz1659898-imjournal-default-tag.patch
View File

@ -2,19 +2,19 @@ diff -up ./plugins/imjournal/imjournal.c.default-tag ./plugins/imjournal/imjourn
--- ./plugins/imjournal/imjournal.c.default-tag 2018-05-17 08:50:11.416418022 -0400 --- ./plugins/imjournal/imjournal.c.default-tag 2018-05-17 08:50:11.416418022 -0400
+++ ./plugins/imjournal/imjournal.c 2018-05-17 08:53:02.884418022 -0400 +++ ./plugins/imjournal/imjournal.c 2018-05-17 08:53:02.884418022 -0400
@@ -78,6 +78,7 @@ static struct configSettings_s { @@ -78,6 +78,7 @@ static struct configSettings_s {
int bUseJnlPID; int bWorkAroundJournalBug; /* deprecated, left for backwards compatibility only */
char *usePid; int bFsync;
int bWorkAroundJournalBug; int bRemote;
+ char *dfltTag; + char *dfltTag;
} cs; } cs;
static rsRetVal facilityHdlr(uchar **pp, void *pVal); static rsRetVal facilityHdlr(uchar **pp, void *pVal);
@@ -93,7 +94,8 @@ static struct cnfparamdescr modpdescr[] @@ -93,7 +94,8 @@ static struct cnfparamdescr modpdescr[]
{ "defaultfacility", eCmdHdlrString, 0 },
{ "usepidfromsystem", eCmdHdlrBinary, 0 },
{ "usepid", eCmdHdlrString, 0 }, { "usepid", eCmdHdlrString, 0 },
- { "workaroundjournalbug", eCmdHdlrBinary, 0 } { "workaroundjournalbug", eCmdHdlrBinary, 0 },
+ { "workaroundjournalbug", eCmdHdlrBinary, 0 }, { "fsync", eCmdHdlrBinary, 0 },
- { "remote", eCmdHdlrBinary, 0 }
+ { "remote", eCmdHdlrBinary, 0 },
+ { "defaulttag", eCmdHdlrGetWord, 0 } + { "defaulttag", eCmdHdlrGetWord, 0 }
}; };
static struct cnfparamblk modpblk = static struct cnfparamblk modpblk =
@ -34,10 +34,10 @@ diff -up ./plugins/imjournal/imjournal.c.default-tag ./plugins/imjournal/imjourn
- char *sys_iden; - char *sys_iden;
+ char *sys_iden = NULL; + char *sys_iden = NULL;
char *sys_iden_help = NULL; char *sys_iden_help = NULL;
char *c = NULL;
const void *get;
@@ -331,7 +334,7 @@ readjournal(void) @@ -331,7 +334,7 @@ readjournal(void)
if (sd_journal_get_data(j, "SYSLOG_IDENTIFIER", &get, &length) >= 0) { if (journalGetData("SYSLOG_IDENTIFIER", &get, &length) >= 0) {
CHKiRet(sanitizeValue(((const char *)get) + 18, length - 18, &sys_iden)); CHKiRet(sanitizeValue(((const char *)get) + 18, length - 18, &sys_iden));
} else { } else {
- CHKmalloc(sys_iden = strdup("journal")); - CHKmalloc(sys_iden = strdup("journal"));
@ -58,9 +58,9 @@ diff -up ./plugins/imjournal/imjournal.c.default-tag ./plugins/imjournal/imjourn
pidFieldName = "_PID"; pidFieldName = "_PID";
bPidFallBack = 0; bPidFallBack = 0;
@@ -732,6 +740,7 @@ CODESTARTbeginCnfLoad @@ -732,6 +740,7 @@ CODESTARTbeginCnfLoad
cs.bUseJnlPID = -1; cs.bWorkAroundJournalBug = 1;
cs.usePid = NULL; cs.bFsync = 0;
cs.bWorkAroundJournalBug = 0; cs.bRemote = 0;
+ cs.dfltTag = NULL; + cs.dfltTag = NULL;
ENDbeginCnfLoad ENDbeginCnfLoad
@ -70,13 +70,13 @@ diff -up ./plugins/imjournal/imjournal.c.default-tag ./plugins/imjournal/imjourn
free(cs.stateFile); free(cs.stateFile);
free(cs.usePid); free(cs.usePid);
+ free(cs.dfltTag); + free(cs.dfltTag);
statsobj.Destruct(&(statsCounter.stats)); free(journalContext.cursor);
statsobj.Destruct(&(statsCounter.stats));
ENDfreeCnf ENDfreeCnf
@@ -832,6 +842,8 @@ CODESTARTsetModCnf @@ -832,6 +842,8 @@ CODESTARTsetModCnf
cs.usePid = (char *)es_str2cstr(pvals[i].val.d.estr, NULL); cs.bFsync = (int) pvals[i].val.d.n;
} else if (!strcmp(modpblk.descr[i].name, "workaroundjournalbug")) { } else if (!strcmp(modpblk.descr[i].name, "remote")) {
cs.bWorkAroundJournalBug = (int) pvals[i].val.d.n; cs.bRemote = (int) pvals[i].val.d.n;
+ } else if (!strcmp(modpblk.descr[i].name, "defaulttag")) { + } else if (!strcmp(modpblk.descr[i].name, "defaulttag")) {
+ cs.dfltTag = (char *)es_str2cstr(pvals[i].val.d.estr, NULL); + cs.dfltTag = (char *)es_str2cstr(pvals[i].val.d.estr, NULL);
} else { } else {

142
SOURCES/rsyslog-8.1911.0-rhbz1763757-imfile-statefiles.patch Normal file
View File

@ -0,0 +1,142 @@
From ac30968b7858d4ca3743d2b4d296eca543864fe2 Mon Sep 17 00:00:00 2001
From: Jiri Vymazal <jvymazal@redhat.com>
Date: Fri, 22 Nov 2019 14:25:59 +0100
Subject: [PATCH] Thorougher state-file renaming and cleaning
Now checking if file-id changes and reanming - cleaning state file
accordingly and always checking and cleaning old inode-only style
state files.
---
plugins/imfile/imfile.c | 66 +++++++++++++++++++++++++++--------------
1 file changed, 43 insertions(+), 23 deletions(-)
diff --git a/plugins/imfile/imfile.c b/plugins/imfile/imfile.c
index d9bf0fbb6d..9db2b47ac9 100644
--- a/plugins/imfile/imfile.c
+++ b/plugins/imfile/imfile.c
@@ -182,6 +182,7 @@ struct act_obj_s {
time_t timeoutBase; /* what time to calculate the timeout against? */
/* file dynamic data */
char file_id[FILE_ID_HASH_SIZE]; /* file id for this entry, once we could obtain it */
+ char file_id_prev[FILE_ID_HASH_SIZE]; /* previous file id for this entry, set if changed */
int in_move; /* workaround for inotify move: if set, state file must not be deleted */
ino_t ino; /* current inode nbr */
int fd; /* fd to file in order to obtain file_id (needs to be preserved across move) */
@@ -711,7 +712,7 @@ act_obj_add(fs_edge_t *const edge, const char *const name, const int is_file,
if (is_file) {
LogError(errno, RS_RET_ERR, "imfile: error accessing file '%s'", name);
} else { /* reporting only in debug for dirs as higher lvl paths are likely blocked by selinux */
- DBGPRINTF("imfile: error accessing file '%s'", name);
+ DBGPRINTF("imfile: error accessing directory '%s'", name);
}
FINALIZE;
}
@@ -727,6 +728,7 @@ act_obj_add(fs_edge_t *const edge, const char *const name, const int is_file,
act->ino = ino;
act->fd = fd;
act->file_id[0] = '\0';
+ act->file_id_prev[0] = '\0';
act->is_symlink = is_symlink;
if (source) { /* we are target of symlink */
CHKmalloc(act->source_name = strdup(source));
@@ -1256,17 +1258,15 @@ get_file_id_hash(const char *data, size_t lendata,
static void ATTR_NONNULL(1)
getFileID(act_obj_t *const act)
{
- if(act->file_id[0] != '\0') {
- return; /* everything already done */
- }
+ /* save the old id for cleaning purposes */
+ strncpy(act->file_id_prev, (const char*)act->file_id, FILE_ID_HASH_SIZE);
+ act->file_id[0] = '\0';
assert(act->fd >= 0); /* fd must have been opened at act_obj_t creation! */
char filedata[FILE_ID_SIZE];
+ lseek(act->fd, 0, SEEK_SET); /* Seek to beginning of file so we have correct id */
const int r = read(act->fd, filedata, FILE_ID_SIZE);
if(r == FILE_ID_SIZE) {
get_file_id_hash(filedata, sizeof(filedata), act->file_id, sizeof(act->file_id));
- dbgprintf("file_id '%s' obtained, closing monitoring file handle\n", act->file_id);
- close(act->fd); /* we will never go here! */
- act->fd = -1;
} else {
DBGPRINTF("getFileID partial or error read, ret %d\n", r);
}
@@ -1378,28 +1378,13 @@ openFileWithStateFile(act_obj_t *const act)
if(fd < 0) {
if(errno == ENOENT) {
if(act->file_id[0] != '\0') {
- const char *pszSFNamHash = strdup((const char*)pszSFNam);
- CHKmalloc(pszSFNamHash);
DBGPRINTF("state file %s for %s does not exist - trying to see if "
"inode-only file exists\n", pszSFNam, act->name);
getFullStateFileName(statefn, "", pszSFNam, sizeof(pszSFNam));
fd = open((char*)pszSFNam, O_CLOEXEC | O_NOCTTY | O_RDONLY, 0600);
if(fd >= 0) {
- dbgprintf("found inode-only state file, renaming it now that we "
- "know the file_id, new name: %s\n", pszSFNamHash);
- /* we now can use identify the file, so let's rename it */
- if(rename((const char*)pszSFNam, pszSFNamHash) != 0) {
- LogError(errno, RS_RET_IO_ERROR,
- "imfile error trying to rename state file for '%s' - "
- "ignoring this error, usually this means a file no "
- "longer file is left over, but this may also cause "
- "some real trouble. Still the best we can do ",
- act->name);
- free((void*) pszSFNamHash);
- ABORT_FINALIZE(RS_RET_IO_ERROR);
- }
+ dbgprintf("found inode-only state file, will be renamed at next persist\n");
}
- free((void*) pszSFNamHash);
}
if(fd < 0) {
DBGPRINTF("state file %s for %s does not exist - trying to see if "
@@ -2609,6 +2594,36 @@ atomicWriteStateFile(const char *fn, const char *content)
RETiRet;
}
+/* This function should be called after any file ID change - that is if
+ * file grown from hash-only statefile, or was truncated, this will ensure
+ * we delete the old file so we do not make garbage in our working dir and
+ * there are no leftover statefiles which can in theory later bind to something
+ * and cause data loss.
+ * jvymazal 2019-11-27
+ */
+static void
+removeOldStatefile(const uchar *statefn, const char *hashToDelete)
+{
+ int ret;
+ uchar statefname[MAXFNAME];
+
+ getFullStateFileName(statefn, hashToDelete, statefname, sizeof(statefname));
+ DBGPRINTF("removing old state file: '%s'\n", statefname);
+ ret = unlink((const char*)statefname);
+ if(ret != 0) {
+ if (errno != ENOENT) {
+ LogError(errno, RS_RET_IO_ERROR,
+ "imfile error trying to delete old state file: '%s' - ignoring this "
+ "error, usually this means a file no longer file is left over, but "
+ "this may also cause some real trouble. Still the best we can do ",
+ statefname);
+ } else {
+ DBGPRINTF("trying to delete no longer valid statefile '%s' which no "
+ "longer exists (probably already deleted)\n", statefname);
+ }
+ }
+}
+
/* This function persists information for a specific file being monitored.
* To do so, it simply persists the stream object. We do NOT abort on error
@@ -2660,6 +2675,11 @@ persistStrmState(act_obj_t *const act)
CHKiRet(atomicWriteStateFile((const char*)statefname, jstr));
json_object_put(json);
+ /* file-id changed remove the old statefile */
+ if (strncmp((const char *)act->file_id_prev, (const char *)act->file_id, FILE_ID_HASH_SIZE)) {
+ removeOldStatefile(statefn, act->file_id_prev);
+ }
+
finalize_it:
if(iRet != RS_RET_OK) {
LogError(0, iRet, "imfile: could not persist state "

2
SOURCES/rsyslog-8.32.0-service.patch → SOURCES/rsyslog-8.1911.0-service.patch
View File

@ -8,7 +8,7 @@ diff -up ./rsyslog.service.in.service ./rsyslog.service.in
+Wants=network.target network-online.target +Wants=network.target network-online.target
+After=network.target network-online.target +After=network.target network-online.target
Documentation=man:rsyslogd(8) Documentation=man:rsyslogd(8)
Documentation=http://www.rsyslog.com/doc/ Documentation=https://www.rsyslog.com/doc/
[Service] [Service]
Type=notify Type=notify

416
SOURCES/rsyslog-8.37.0-rhbz1614179-imfile-symlink-support.patch
View File

@ -1,416 +0,0 @@
From 3822da837e4d531e8a9cd78ae76359a410f8d98d Mon Sep 17 00:00:00 2001
From: Jiri Vymazal <jvymazal@redhat.com>
Date: Thu, 31 May 2018 16:30:08 +0200
Subject: [PATCH] Symlink support for imfile
this introduces symlink detection and following as well
as monitoring changes on them. Also added test for the new
functionality and ensuring the original symlink behavior
stays as well.
---
plugins/imfile/imfile.c | 182 +++++++++++++++++++++++++++----------
1 file changed, 133 insertions(+), 49 deletions(-)
diff --git a/plugins/imfile/imfile.c b/plugins/imfile/imfile.c
index 3c9308bfe..4ca23d2ca 100644
--- a/plugins/imfile/imfile.c
+++ b/plugins/imfile/imfile.c
@@ -152,6 +152,7 @@ struct act_obj_s {
fs_edge_t *edge; /* edge which this object belongs to */
char *name; /* full path name of active object */
char *basename; /* only basename */ //TODO: remove when refactoring rename support
+ char *source_name; /* if this object is target of a symlink, source_name is its name (else NULL) */
//char *statefile; /* base name of state file (for move operations) */
int wd;
#if defined(OS_SOLARIS) && defined (HAVE_PORT_SOURCE_FILE)
@@ -167,6 +168,7 @@ struct act_obj_s {
int nRecords; /**< How many records did we process before persisting the stream? */
ratelimit_t *ratelimiter;
multi_submit_t multiSub;
+ int is_symlink;
};
struct fs_edge_s {
fs_node_t *parent;
@@ -181,7 +182,8 @@ struct act_obj_s {
instanceConf_t **instarr;
};
struct fs_node_s {
- fs_edge_t *edges;
+ fs_edge_t *edges; /* NULL in leaf nodes */
+ fs_node_t *root; /* node one level up (NULL for file system root) */
};
@@ -189,7 +191,7 @@ static rsRetVal persistStrmState(act_obj_t *);
static rsRetVal resetConfigVariables(uchar __attribute__((unused)) *pp, void __attribute__((unused)) *pVal);
static rsRetVal ATTR_NONNULL(1) pollFile(act_obj_t *act);
static int ATTR_NONNULL() getBasename(uchar *const __restrict__ basen, uchar *const __restrict__ path);
-static void ATTR_NONNULL() act_obj_unlink(act_obj_t *const act);
+static void ATTR_NONNULL() act_obj_unlink(act_obj_t *act);
static uchar * ATTR_NONNULL(1, 2) getStateFileName(const act_obj_t *, uchar *, const size_t);
static int ATTR_NONNULL() getFullStateFileName(const uchar *const, uchar *const pszout, const size_t ilenout);
@@ -483,14 +485,17 @@ in_setupWatch(act_obj_t *const act, const int is_file)
goto done;
wd = inotify_add_watch(ino_fd, act->name,
- (is_file) ? IN_MODIFY : IN_CREATE|IN_DELETE|IN_MOVED_FROM|IN_MOVED_TO);
+ (is_file) ? IN_MODIFY|IN_DONT_FOLLOW : IN_CREATE|IN_DELETE|IN_MOVED_FROM|IN_MOVED_TO);
if(wd < 0) {
- LogError(errno, RS_RET_IO_ERROR, "imfile: cannot watch object '%s'",
- act->name);
+ if (errno == EACCES) { /* There is high probability of selinux denial on top-level paths */
+ DBGPRINTF("imfile: permission denied when adding watch for '%s'\n", act->name);
+ } else {
+ LogError(errno, RS_RET_IO_ERROR, "imfile: cannot watch object '%s'", act->name);
+ }
goto done;
}
wdmapAdd(wd, act);
- DBGPRINTF("in_setupDirWatch: watch %d added for dir %s(%p)\n", wd, act->name, act);
+ DBGPRINTF("in_setupWatch: watch %d added for %s(object %p)\n", wd, act->name, act);
done: return wd;
}
@@ -605,7 +610,7 @@ done: return;
static void ATTR_NONNULL()
fen_setupWatch(act_obj_t *const act __attribute__((unused)))
{
- DBGPRINTF("fen_setupWatch: DUMMY CALLED - not on Solaris?");
+ DBGPRINTF("fen_setupWatch: DUMMY CALLED - not on Solaris?\n");
}
#endif /* FEN */
@@ -633,38 +638,48 @@ fs_node_print(const fs_node_t *const node, const int level)
}
}
-
/* add a new file system object if it not yet exists, ignore call
* if it already does.
*/
-static rsRetVal ATTR_NONNULL()
+static rsRetVal ATTR_NONNULL(1,2)
act_obj_add(fs_edge_t *const edge, const char *const name, const int is_file,
- const ino_t ino)
+ const ino_t ino, const int is_symlink, const char *const source)
{
act_obj_t *act;
char basename[MAXFNAME];
DEFiRet;
- DBGPRINTF("act_obj_add: edge %p, name '%s'\n", edge, name);
+ DBGPRINTF("act_obj_add: edge %p, name '%s' (source '%s')\n", edge, name, source? source : "---");
for(act = edge->active ; act != NULL ; act = act->next) {
if(!strcmp(act->name, name)) {
- DBGPRINTF("active object '%s' already exists in '%s' - no need to add\n",
- name, edge->path);
- FINALIZE;
+ if (!source || !act->source_name || !strcmp(act->source_name, source)) {
+ DBGPRINTF("active object '%s' already exists in '%s' - no need to add\n",
+ name, edge->path);
+ FINALIZE;
+ }
}
}
DBGPRINTF("add new active object '%s' in '%s'\n", name, edge->path);
CHKmalloc(act = calloc(sizeof(act_obj_t), 1));
CHKmalloc(act->name = strdup(name));
- getBasename((uchar*)basename, (uchar*)name);
- CHKmalloc(act->basename = strdup(basename));
+ if (-1 == getBasename((uchar*)basename, (uchar*)name)) {
+ CHKmalloc(act->basename = strdup(name)); /* assume basename is same as name */
+ } else {
+ CHKmalloc(act->basename = strdup(basename));
+ }
act->edge = edge;
act->ino = ino;
+ act->is_symlink = is_symlink;
+ if (source) { /* we are target of symlink */
+ CHKmalloc(act->source_name = strdup(source));
+ } else {
+ act->source_name = NULL;
+ }
#ifdef HAVE_INOTIFY_INIT
act->wd = in_setupWatch(act, is_file);
#endif
fen_setupWatch(act);
- if(is_file) {
+ if(is_file && !is_symlink) {
const instanceConf_t *const inst = edge->instarr[0];// TODO: same file, multiple instances?
CHKiRet(ratelimitNew(&act->ratelimiter, "imfile", name));
CHKmalloc(act->multiSub.ppMsgs = MALLOC(inst->nMultiSub * sizeof(smsg_t *)));
@@ -702,27 +717,24 @@ detect_updates(fs_edge_t *const edge)
{
act_obj_t *act;
struct stat fileInfo;
+ int restart = 0;
- for(act = edge->active ; act != NULL ; ) {
+ for(act = edge->active ; act != NULL ; act = act->next) {
DBGPRINTF("detect_updates checking active obj '%s'\n", act->name);
- const int r = stat(act->name, &fileInfo);
+ const int r = lstat(act->name, &fileInfo);
if(r == -1) { /* object gone away? */
DBGPRINTF("object gone away, unlinking: '%s'\n", act->name);
- act_obj_t *toDel = act;
- act = act->next;
- DBGPRINTF("new next act %p\n", act);
- act_obj_unlink(toDel);
- continue;
+ act_obj_unlink(act);
+ restart = 1;
+ break;
}
// TODO: add inode check for change notification!
- /* Note: active nodes may get deleted, so we need to do the
- * pointer advancement at the end of the for loop!
- */
- act = act->next;
}
-
+ if (restart) {
+ detect_updates(edge);
+ }
}
@@ -746,14 +758,52 @@ poll_active_files(fs_edge_t *const edge)
}
}
+static rsRetVal ATTR_NONNULL()
+process_symlink(fs_edge_t *const chld, const char *symlink)
+{
+ DEFiRet;
+ char *target = NULL;
+ CHKmalloc(target = realpath(symlink, target));
+ struct stat fileInfo;
+ if(lstat(target, &fileInfo) != 0) {
+ LogError(errno, RS_RET_ERR, "imfile: process_symlink: cannot stat file '%s' - ignored", target);
+ FINALIZE;
+ }
+ const int is_file = (S_ISREG(fileInfo.st_mode));
+ DBGPRINTF("process_symlink: found '%s', File: %d (config file: %d), symlink: %d\n",
+ target, is_file, chld->is_file, 0);
+ if (act_obj_add(chld, target, is_file, fileInfo.st_ino, 0, symlink) == RS_RET_OK) {
+ /* need to watch parent target as well for proper rotation support */
+ uint idx = ustrlen(chld->active->name) - ustrlen(chld->active->basename);
+ if (idx) { /* basename is different from name */
+ char parent[MAXFNAME];
+ idx--; /* move past trailing slash */
+ memcpy(parent, chld->active->name, idx);
+ parent[idx] = '\0';
+ if(lstat(parent, &fileInfo) != 0) {
+ LogError(errno, RS_RET_ERR,
+ "imfile: process_symlink: cannot stat directory '%s' - ignored", parent);
+ FINALIZE;
+ }
+ if (chld->parent->root->edges) {
+ DBGPRINTF("process_symlink: adding parent '%s' of target '%s'\n", parent, target);
+ act_obj_add(chld->parent->root->edges, parent, 0, fileInfo.st_ino, 0, NULL);
+ }
+ }
+ }
+
+finalize_it:
+ free(target);
+ RETiRet;
+}
-static void ATTR_NONNULL() poll_tree(fs_edge_t *const chld);
static void ATTR_NONNULL()
poll_tree(fs_edge_t *const chld)
{
struct stat fileInfo;
glob_t files;
int need_globfree = 0;
+ int issymlink;
DBGPRINTF("poll_tree: chld %p, name '%s', path: %s\n", chld, chld->name, chld->path);
detect_updates(chld);
const int ret = glob((char*)chld->path, runModConf->sortFiles|GLOB_BRACE, NULL, &files);
@@ -766,18 +803,27 @@ poll_tree(fs_edge_t *const chld)
goto done;
}
char *const file = files.gl_pathv[i];
- if(stat(file, &fileInfo) != 0) {
+ if(lstat(file, &fileInfo) != 0) {
LogError(errno, RS_RET_ERR,
"imfile: poll_tree cannot stat file '%s' - ignored", file);
continue;
}
- const int is_file = S_ISREG(fileInfo.st_mode);
- DBGPRINTF("poll_tree: found '%s', File: %d (config file: %d)\n",
- file, is_file, chld->is_file);
+ if (S_ISLNK(fileInfo.st_mode)) {
+ rsRetVal slink_ret = process_symlink(chld, file);
+ if (slink_ret != RS_RET_OK) {
+ continue;
+ }
+ issymlink = 1;
+ } else {
+ issymlink = 0;
+ }
+ const int is_file = (S_ISREG(fileInfo.st_mode) || issymlink);
+ DBGPRINTF("poll_tree: found '%s', File: %d (config file: %d), symlink: %d\n",
+ file, is_file, chld->is_file, issymlink);
if(!is_file && S_ISREG(fileInfo.st_mode)) {
LogMsg(0, RS_RET_ERR, LOG_WARNING,
- "imfile: '%s' is neither a regular file nor a "
+ "imfile: '%s' is neither a regular file, symlink, nor a "
"directory - ignored", file);
continue;
}
@@ -788,7 +834,7 @@ poll_tree(fs_edge_t *const chld)
(chld->is_file) ? "FILE" : "DIRECTORY");
continue;
}
- act_obj_add(chld, file, is_file, fileInfo.st_ino);
+ act_obj_add(chld, file, is_file, fileInfo.st_ino, issymlink, NULL);
}
}
@@ -829,8 +875,20 @@ act_obj_destroy(act_obj_t *const act, const int is_deleted)
if(act == NULL)
return;
- DBGPRINTF("act_obj_destroy: act %p '%s', wd %d, pStrm %p, is_deleted %d, in_move %d\n",
- act, act->name, act->wd, act->pStrm, is_deleted, act->in_move);
+ DBGPRINTF("act_obj_destroy: act %p '%s' (source '%s'), wd %d, pStrm %p, is_deleted %d, in_move %d\n",
+ act, act->name, act->source_name? act->source_name : "---", act->wd, act->pStrm, is_deleted,
+ act->in_move);
+ if(act->is_symlink && is_deleted) {
+ act_obj_t *target_act;
+ for(target_act = act->edge->active ; target_act != NULL ; target_act = target_act->next) {
+ if(target_act->source_name && !strcmp(target_act->source_name, act->name)) {
+ DBGPRINTF("act_obj_destroy: unlinking slink target %s of %s "
+ "symlink\n", target_act->name, act->name);
+ act_obj_unlink(target_act);
+ break;
+ }
+ }
+ }
if(act->ratelimiter != NULL) {
ratelimitDestruct(act->ratelimiter);
}
@@ -862,6 +920,7 @@ act_obj_destroy(act_obj_t *const act, const int is_deleted)
}
#endif
free(act->basename);
+ free(act->source_name);
//free(act->statefile);
free(act->multiSub.ppMsgs);
#if defined(OS_SOLARIS) && defined (HAVE_PORT_SOURCE_FILE)
@@ -909,7 +968,7 @@ chk_active(const act_obj_t *act, const act_obj_t *const deleted)
* destruct it.
*/
static void //ATTR_NONNULL()
-act_obj_unlink(act_obj_t *const act)
+act_obj_unlink(act_obj_t *act)
{
DBGPRINTF("act_obj_unlink %p: %s\n", act, act->name);
if(act->prev == NULL) {
@@ -921,6 +980,7 @@ act_obj_unlink(act_obj_t *const act)
act->next->prev = act->prev;
}
act_obj_destroy(act, 1);
+ act = NULL;
//dbgprintf("printout of fs tree post unlink\n");
//fs_node_print(runModConf->conf_tree, 0);
//dbg_wdmapPrint("wdmap after");
@@ -1025,6 +1038,7 @@ fs_node_walk(fs_node_t *const node,
*/
static rsRetVal
fs_node_add(fs_node_t *const node,
+ fs_node_t *const source,
const uchar *const toFind,
const size_t pathIdx,
instanceConf_t *const inst)
@@ -1053,6 +1067,7 @@ fs_node_add(fs_node_t *const node,
memcpy(name, toFind+pathIdx, len);
name[len] = '\0';
DBGPRINTF("fs_node_add: name '%s'\n", name);
+ node->root = source;
fs_edge_t *chld;
for(chld = node->edges ; chld != NULL ; chld = chld->next) {
@@ -1064,7 +1079,7 @@ fs_node_add(fs_node_t *const node,
chld->instarr[chld->ninst-1] = inst;
/* recurse */
if(!isFile) {
- CHKiRet(fs_node_add(chld->node, toFind, nextPathIdx, inst));
+ CHKiRet(fs_node_add(chld->node, node, toFind, nextPathIdx, inst));
}
FINALIZE;
}
@@ -1086,7 +1101,7 @@ fs_node_add(fs_node_t *const node,
DBGPRINTF("fs_node_add(%p, '%s') returns %p\n", node, toFind, newchld->node);
if(!isFile) {
- CHKiRet(fs_node_add(newchld->node, toFind, nextPathIdx, inst));
+ CHKiRet(fs_node_add(newchld->node, node, toFind, nextPathIdx, inst));
}
/* link to list */
@@ -1162,7 +1222,11 @@ enqLine(act_obj_t *const act,
msgSetPRI(pMsg, inst->iFacility | inst->iSeverity);
MsgSetRuleset(pMsg, inst->pBindRuleset);
if(inst->addMetadata) {
- metadata_values[0] = (const uchar*)act->name;
+ if (act->source_name) {
+ metadata_values[0] = (const uchar*)act->source_name;
+ } else {
+ metadata_values[0] = (const uchar*)act->name;
+ }
snprintf((char *)file_offset, MAX_OFFSET_REPRESENTATION_NUM_BYTES+1, "%lld", strtOffs);
metadata_values[1] = file_offset;
msgAddMultiMetadata(pMsg, metadata_names, metadata_values, 2);
@@ -1389,13 +1453,16 @@ pollFile(act_obj_t *const act)
{
cstr_t *pCStr = NULL;
DEFiRet;
+ if (act->is_symlink) {
+ FINALIZE; /* no reason to poll symlink file */
+ }
/* Note: we must do pthread_cleanup_push() immediately, because the POSIX macros
* otherwise do not work if I include the _cleanup_pop() inside an if... -- rgerhards, 2008-08-14
*/
pthread_cleanup_push(pollFileCancelCleanup, &pCStr);
iRet = pollFileReal(act, &pCStr);
pthread_cleanup_pop(0);
- RETiRet;
+finalize_it: RETiRet;
}
@@ -1931,7 +1946,7 @@ CODESTARTactivateCnf
"be processed. Reason", inst->pszFileName);
}
}
- fs_node_add(runModConf->conf_tree, inst->pszFileName, 0, inst);
+ fs_node_add(runModConf->conf_tree, NULL, inst->pszFileName, 0, inst);
}
if(Debug) {
@@ -2031,6 +2113,9 @@ flag_in_move(fs_edge_t *const edge, const char *name_moved)
DBGPRINTF("name check fails, '%s' != '%s'\n", act->basename, name_moved);
}
}
+ if (!act && edge->next) {
+ flag_in_move(edge->next, name_moved);
+ }
}
static void ATTR_NONNULL(1)
@@ -2057,7 +2145,7 @@ in_processEvent(struct inotify_event *ev)
}
if(ev->mask & (IN_MOVED_FROM | IN_MOVED_TO)) {
fs_node_walk(etry->act->edge->node, poll_tree);
- } else if(etry->act->edge->is_file) {
+ } else if(etry->act->edge->is_file && !(etry->act->is_symlink)) {
in_handleFileEvent(ev, etry); // esentially poll_file()!
} else {
fs_node_walk(etry->act->edge->node, poll_tree);

286
SOURCES/rsyslog-8.37.0-rhbz1614181-imtcp-imudp-preservecase-option.patch
View File

@ -1,286 +0,0 @@
From 9ac54f0d7d70b8a9879889b4522a1d552fca1100 Mon Sep 17 00:00:00 2001
From: Noriko Hosoi <nhosoi@momo7.localdomain>
Date: Thu, 12 Jul 2018 11:52:04 -0700
Subject: [PATCH] Introducing an option preservecase to imudp and imtcp module
for managing the case of FROMHOST value.
Usage:
module(load="imudp" [preservecase="on"|"off"])
module(load="imtdp" [preservecase="on"|"off"])
If preservecase="on", FROMHOST value is handled in the case sensitive manner.
If preservecase="off", FROMHOST value is handled in the case insensitive manner.
To maintain the current behaviour, the default value of preservecase is
"on" for imtcp and "off" for imudp.
Incremented tcpsrvCURR_IF_VERSION by 1.
References:
https://github.com/rsyslog/rsyslog/pull/2774
https://bugzilla.redhat.com/show_bug.cgi?id=1309698
---
plugins/imtcp/imtcp.c | 14 ++++++++++++--
plugins/imudp/imudp.c | 15 ++++++++++++---
runtime/msg.c | 6 +++++-
runtime/msg.h | 2 ++
runtime/net.c | 2 +-
runtime/tcpsrv.c | 21 +++++++++++++++++++++
runtime/tcpsrv.h | 5 ++++-
7 files changed, 57 insertions(+), 8 deletions(-)
diff --git a/plugins/imtcp/imtcp.c b/plugins/imtcp/imtcp.c
index 8e3dcc0a2..45fa240b5 100644
--- a/plugins/imtcp/imtcp.c
+++ b/plugins/imtcp/imtcp.c
@@ -100,6 +100,7 @@ static struct configSettings_s {
int bDisableLFDelim;
int discardTruncatedMsg;
int bUseFlowControl;
+ int bPreserveCase;
uchar *gnutlsPriorityString;
uchar *pszStrmDrvrAuthMode;
uchar *pszInputName;
@@ -144,6 +145,7 @@ struct modConfData_s {
uchar *pszStrmDrvrAuthMode; /* authentication mode to use */
struct cnfarray *permittedPeers;
sbool configSetViaV2Method;
+ sbool bPreserveCase; /* preserve case of fromhost; true by default */
};
static modConfData_t *loadModConf = NULL;/* modConf ptr to use for the current load process */
@@ -169,7 +171,8 @@ static struct cnfparamdescr modpdescr[] = {
{ "keepalive.probes", eCmdHdlrPositiveInt, 0 },
{ "keepalive.time", eCmdHdlrPositiveInt, 0 },
{ "keepalive.interval", eCmdHdlrPositiveInt, 0 },
- { "gnutlsprioritystring", eCmdHdlrString, 0 }
+ { "gnutlsprioritystring", eCmdHdlrString, 0 },
+ { "preservecase", eCmdHdlrBinary, 0 }
};
static struct cnfparamblk modpblk =
{ CNFPARAMBLK_VERSION,
@@ -375,6 +378,7 @@ addListner(modConfData_t *modConf, instanceConf_t *inst)
if(pPermPeersRoot != NULL) {
CHKiRet(tcpsrv.SetDrvrPermPeers(pOurTcpsrv, pPermPeersRoot));
}
+ CHKiRet(tcpsrv.SetPreserveCase(pOurTcpsrv, modConf->bPreserveCase));
}
/* initialized, now add socket and listener params */
@@ -473,6 +477,7 @@ CODESTARTbeginCnfLoad
loadModConf->pszStrmDrvrAuthMode = NULL;
loadModConf->permittedPeers = NULL;
loadModConf->configSetViaV2Method = 0;
+ loadModConf->bPreserveCase = 1; /* default to true */
bLegacyCnfModGlobalsPermitted = 1;
/* init legacy config variables */
cs.pszStrmDrvrAuthMode = NULL;
@@ -543,6 +548,8 @@ CODESTARTsetModCnf
loadModConf->pszStrmDrvrName = (uchar*)es_str2cstr(pvals[i].val.d.estr, NULL);
} else if(!strcmp(modpblk.descr[i].name, "permittedpeer")) {
loadModConf->permittedPeers = cnfarrayDup(pvals[i].val.d.ar);
+ } else if(!strcmp(modpblk.descr[i].name, "preservecase")) {
+ loadModConf->bPreserveCase = (int) pvals[i].val.d.n;
} else {
dbgprintf("imtcp: program error, non-handled "
"param '%s' in beginCnfLoad\n", modpblk.descr[i].name);
@@ -584,6 +591,7 @@ CODESTARTendCnfLoad
loadModConf->pszStrmDrvrAuthMode = cs.pszStrmDrvrAuthMode;
cs.pszStrmDrvrAuthMode = NULL;
}
+ pModConf->bPreserveCase = cs.bPreserveCase;
}
free(cs.pszStrmDrvrAuthMode);
cs.pszStrmDrvrAuthMode = NULL;
@@ -731,6 +739,7 @@ resetConfigVariables(uchar __attribute__((unused)) *pp, void __attribute__((unus
cs.pszInputName = NULL;
free(cs.pszStrmDrvrAuthMode);
cs.pszStrmDrvrAuthMode = NULL;
+ cs.bPreserveCase = 1;
return RS_RET_OK;
}
@@ -797,7 +806,8 @@ CODEmodInit_QueryRegCFSLineHdlr
NULL, &cs.bEmitMsgOnClose, STD_LOADABLE_MODULE_ID, &bLegacyCnfModGlobalsPermitted));
CHKiRet(regCfSysLineHdlr2(UCHAR_CONSTANT("inputtcpserverstreamdrivermode"), 0, eCmdHdlrInt,
NULL, &cs.iStrmDrvrMode, STD_LOADABLE_MODULE_ID, &bLegacyCnfModGlobalsPermitted));
-
+ CHKiRet(regCfSysLineHdlr2(UCHAR_CONSTANT("inputtcpserverpreservecase"), 1, eCmdHdlrBinary,
+ NULL, &cs.bPreserveCase, STD_LOADABLE_MODULE_ID, &bLegacyCnfModGlobalsPermitted));
CHKiRet(omsdRegCFSLineHdlr(UCHAR_CONSTANT("resetconfigvariables"), 1, eCmdHdlrCustomHandler,
resetConfigVariables, NULL, STD_LOADABLE_MODULE_ID));
ENDmodInit
diff --git a/plugins/imudp/imudp.c b/plugins/imudp/imudp.c
index 51a9d712a..74437781c 100644
--- a/plugins/imudp/imudp.c
+++ b/plugins/imudp/imudp.c
@@ -152,6 +152,7 @@ struct modConfData_s {
int batchSize; /* max nbr of input batch --> also recvmmsg() max count */
int8_t wrkrMax; /* max nbr of worker threads */
sbool configSetViaV2Method;
+ sbool bPreserveCase; /* preserves the case of fromhost; "off" by default */
};
static modConfData_t *loadModConf = NULL;/* modConf ptr to use for the current load process */
static modConfData_t *runModConf = NULL;/* modConf ptr to use for the current load process */
@@ -162,7 +163,8 @@ static struct cnfparamdescr modpdescr[] = {
{ "schedulingpriority", eCmdHdlrInt, 0 },
{ "batchsize", eCmdHdlrInt, 0 },
{ "threads", eCmdHdlrPositiveInt, 0 },
- { "timerequery", eCmdHdlrInt, 0 }
+ { "timerequery", eCmdHdlrInt, 0 },
+ { "preservecase", eCmdHdlrBinary, 0 }
};
static struct cnfparamblk modpblk =
{ CNFPARAMBLK_VERSION,
@@ -447,8 +449,12 @@ processPacket(struct lstn_s *lstn, struct sockaddr_storage *frominetPrev, int *p
if(lstn->dfltTZ != NULL)
MsgSetDfltTZ(pMsg, (char*) lstn->dfltTZ);
pMsg->msgFlags = NEEDS_PARSING | PARSE_HOSTNAME | NEEDS_DNSRESOL;
- if(*pbIsPermitted == 2)
- pMsg->msgFlags |= NEEDS_ACLCHK_U; /* request ACL check after resolution */
+ if(*pbIsPermitted == 2) {
+ pMsg->msgFlags |= NEEDS_ACLCHK_U; /* request ACL check after resolution */
+ }
+ if(runModConf->bPreserveCase) {
+ pMsg->msgFlags |= PRESERVE_CASE; /* preserve case of fromhost */
+ }
CHKiRet(msgSetFromSockinfo(pMsg, frominet));
CHKiRet(ratelimitAddMsg(lstn->ratelimiter, multiSub, pMsg));
STATSCOUNTER_INC(lstn->ctrSubmit, lstn->mutCtrSubmit);
@@ -1030,6 +1036,7 @@ CODESTARTbeginCnfLoad
loadModConf->iTimeRequery = TIME_REQUERY_DFLT;
loadModConf->iSchedPrio = SCHED_PRIO_UNSET;
loadModConf->pszSchedPolicy = NULL;
+ loadModConf->bPreserveCase = 0; /* off */
bLegacyCnfModGlobalsPermitted = 1;
/* init legacy config vars */
cs.pszBindRuleset = NULL;
@@ -1079,6 +1086,8 @@ CODESTARTsetModCnf
} else {
loadModConf->wrkrMax = wrkrMax;
}
+ } else if(!strcmp(modpblk.descr[i].name, "preservecase")) {
+ loadModConf->bPreserveCase = (int) pvals[i].val.d.n;
} else {
dbgprintf("imudp: program error, non-handled "
"param '%s' in beginCnfLoad\n", modpblk.descr[i].name);
diff --git a/runtime/msg.c b/runtime/msg.c
index c43f81314..9ed4eaf84 100644
--- a/runtime/msg.c
+++ b/runtime/msg.c
@@ -506,7 +506,11 @@ resolveDNS(smsg_t * const pMsg) {
MsgLock(pMsg);
CHKiRet(objUse(net, CORE_COMPONENT));
if(pMsg->msgFlags & NEEDS_DNSRESOL) {
- localRet = net.cvthname(pMsg->rcvFrom.pfrominet, &localName, NULL, &ip);
+ if (pMsg->msgFlags & PRESERVE_CASE) {
+ localRet = net.cvthname(pMsg->rcvFrom.pfrominet, NULL, &localName, &ip);
+ } else {
+ localRet = net.cvthname(pMsg->rcvFrom.pfrominet, &localName, NULL, &ip);
+ }
if(localRet == RS_RET_OK) {
/* we pass down the props, so no need for AddRef */
MsgSetRcvFromWithoutAddRef(pMsg, localName);
diff --git a/runtime/msg.h b/runtime/msg.h
index cd530aca3..1287cb7a4 100644
--- a/runtime/msg.h
+++ b/runtime/msg.h
@@ -156,6 +156,8 @@ struct msg {
/* check UDP ACLs after DNS resolution has been done in main queue consumer */
#define NO_PRI_IN_RAW 0x100
/* rawmsg does not include a PRI (Solaris!), but PRI is already set correctly in the msg object */
+#define PRESERVE_CASE 0x200
+/* preserve case in fromhost */
/* (syslog) protocol types */
#define MSG_LEGACY_PROTOCOL 0
diff --git a/runtime/net.c b/runtime/net.c
index d6ff8a3d4..aef906601 100644
--- a/runtime/net.c
+++ b/runtime/net.c
@@ -1152,7 +1152,7 @@ cvthname(struct sockaddr_storage *f, prop_t **localName, prop_t **fqdn, prop_t *
{
DEFiRet;
assert(f != NULL);
- iRet = dnscacheLookup(f, NULL, fqdn, localName, ip);
+ iRet = dnscacheLookup(f, fqdn, NULL, localName, ip);
RETiRet;
}
diff --git a/runtime/tcpsrv.c b/runtime/tcpsrv.c
index 61e9ff4d2..d5993b4f0 100644
--- a/runtime/tcpsrv.c
+++ b/runtime/tcpsrv.c
@@ -495,6 +495,15 @@ SessAccept(tcpsrv_t *pThis, tcpLstnPortList_t *pLstnInfo, tcps_sess_t **ppSess,
/* get the host name */
CHKiRet(netstrm.GetRemoteHName(pNewStrm, &fromHostFQDN));
+ if (!pThis->bPreserveCase) {
+ /* preserve_case = off */
+ uchar *p;
+ for(p = fromHostFQDN; *p; p++) {
+ if (isupper((int) *p)) {
+ *p = tolower((int) *p);
+ }
+ }
+ }
CHKiRet(netstrm.GetRemoteIP(pNewStrm, &fromHostIP));
CHKiRet(netstrm.GetRemAddr(pNewStrm, &addr));
/* TODO: check if we need to strip the domain name here -- rgerhards, 2008-04-24 */
@@ -1001,6 +1010,7 @@ BEGINobjConstruct(tcpsrv) /* be sure to specify the object type also in END macr
pThis->ratelimitBurst = 10000;
pThis->bUseFlowControl = 1;
pThis->pszDrvrName = NULL;
+ pThis->bPreserveCase = 1; /* preserve case in fromhost; default to true. */
ENDobjConstruct(tcpsrv)
@@ -1433,6 +1443,16 @@ SetSessMax(tcpsrv_t *pThis, int iMax)
}
+static rsRetVal
+SetPreserveCase(tcpsrv_t *pThis, int bPreserveCase)
+{
+ DEFiRet;
+ ISOBJ_TYPE_assert(pThis, tcpsrv);
+ pThis-> bPreserveCase = bPreserveCase;
+ RETiRet;
+}
+
+
/* queryInterface function
* rgerhards, 2008-02-29
*/
@@ -1491,6 +1511,7 @@ CODESTARTobjQueryInterface(tcpsrv)
pIf->SetRuleset = SetRuleset;
pIf->SetLinuxLikeRatelimiters = SetLinuxLikeRatelimiters;
pIf->SetNotificationOnRemoteClose = SetNotificationOnRemoteClose;
+ pIf->SetPreserveCase = SetPreserveCase;
finalize_it:
ENDobjQueryInterface(tcpsrv)
diff --git a/runtime/tcpsrv.h b/runtime/tcpsrv.h
index 22a65c20a..f17b1b438 100644
--- a/runtime/tcpsrv.h
+++ b/runtime/tcpsrv.h
@@ -85,6 +85,7 @@ struct tcpsrv_s {
int maxFrameSize; /**< max frame size for octet counted*/
int bDisableLFDelim; /**< if 1, standard LF frame delimiter is disabled (*very dangerous*) */
int discardTruncatedMsg;/**< discard msg part that has been truncated*/
+ sbool bPreserveCase; /**< preserve case in fromhost */
int ratelimitInterval;
int ratelimitBurst;
tcps_sess_t **pSessions;/**< array of all of our sessions */
@@ -177,8 +178,10 @@ BEGINinterface(tcpsrv) /* name must also be changed in ENDinterface macro! */
rsRetVal (*SetbSPFramingFix)(tcpsrv_t*, sbool);
/* added v19 -- PascalWithopf, 2017-08-08 */
rsRetVal (*SetGnutlsPriorityString)(tcpsrv_t*, uchar*);
+ /* added v21 -- Preserve case in fromhost, 2018-08-16 */
+ rsRetVal (*SetPreserveCase)(tcpsrv_t *pThis, int bPreserveCase);
ENDinterface(tcpsrv)
-#define tcpsrvCURR_IF_VERSION 20 /* increment whenever you change the interface structure! */
+#define tcpsrvCURR_IF_VERSION 21 /* increment whenever you change the interface structure! */
/* change for v4:
* - SetAddtlFrameDelim() added -- rgerhards, 2008-12-10
* - SetInputName() added -- rgerhards, 2008-12-10

761
SOURCES/rsyslog-8.37.0-rhbz1622768-kubernetes-404-handling.patch
View File

@ -1,761 +0,0 @@
From 3987cd929d859f900318b393133c3bdde8dfffd5 Mon Sep 17 00:00:00 2001
From: Rich Megginson <rmeggins@redhat.com>
Date: Tue, 28 Aug 2018 12:44:23 -0600
Subject: [PATCH] mmkubertnetes: action fails preparation cycle if kubernetes
API destroys resource during bootup sequence
The plugin was not handling 404 Not Found correctly when looking
up pods and namespaces. In this case, we assume the pod/namespace
was deleted, annotate the record with whatever metadata we have,
and cache the fact that the pod/namespace is missing so we don't
attempt to look it up again.
In addition, the plugin was not handling error 429 Busy correctly.
In this case, it should also annotate the record with whatever
metadata it has, and _not_ cache anything. By default the plugin
will retry every 5 seconds to connect to Kubernetes. This
behavior is controlled by the new config param `busyretryinterval`.
This commit also adds impstats counters so that admins can
view the state of the plugin to see if the lookups are working
or are returning errors. The stats are reported per-instance
or per-action to facilitate using multiple different actions
for different Kubernetes servers.
This commit also adds support for client cert auth to
Kubernetes via the two new config params `tls.mycert` and
`tls.myprivkey`.
---
contrib/mmkubernetes/mmkubernetes.c | 296 ++++++++++++++++++++++++----
7 files changed, 160 insertions(+), 36 deletions(-)
diff --git a/contrib/mmkubernetes/mmkubernetes.c b/contrib/mmkubernetes/mmkubernetes.c
index 422cb2577..5bf5b049d 100644
--- a/contrib/mmkubernetes/mmkubernetes.c
+++ b/contrib/mmkubernetes/mmkubernetes.c
@@ -52,9 +52,12 @@
#include "syslogd-types.h"
#include "module-template.h"
#include "errmsg.h"
+#include "statsobj.h"
#include "regexp.h"
#include "hashtable.h"
#include "srUtils.h"
+#include "unicode-helper.h"
+#include "datetime.h"
/* static data */
MODULE_TYPE_OUTPUT /* this is technically an output plugin */
@@ -62,6 +65,8 @@ MODULE_TYPE_KEEP /* releasing the module would cause a leak through libcurl */
MODULE_CNFNAME("mmkubernetes")
DEF_OMOD_STATIC_DATA
DEFobjCurrIf(regexp)
+DEFobjCurrIf(statsobj)
+DEFobjCurrIf(datetime)
#define HAVE_LOADSAMPLESFROMSTRING 1
#if defined(NO_LOADSAMPLESFROMSTRING)
@@ -95,12 +100,14 @@ DEFobjCurrIf(regexp)
#define DFLT_CONTAINER_NAME "$!CONTAINER_NAME" /* name of variable holding CONTAINER_NAME value */
#define DFLT_CONTAINER_ID_FULL "$!CONTAINER_ID_FULL" /* name of variable holding CONTAINER_ID_FULL value */
#define DFLT_KUBERNETES_URL "https://kubernetes.default.svc.cluster.local:443"
+#define DFLT_BUSY_RETRY_INTERVAL 5 /* retry every 5 seconds */
static struct cache_s {
const uchar *kbUrl;
struct hashtable *mdHt;
struct hashtable *nsHt;
pthread_mutex_t *cacheMtx;
+ int lastBusyTime;
} **caches;
typedef struct {
@@ -116,6 +123,8 @@ struct modConfData_s {
uchar *srcMetadataPath; /* where to get data for kubernetes queries */
uchar *dstMetadataPath; /* where to put metadata obtained from kubernetes */
uchar *caCertFile; /* File holding the CA cert (+optional chain) of CA that issued the Kubernetes server cert */
+ uchar *myCertFile; /* File holding cert corresponding to private key used for client cert auth */
+ uchar *myPrivKeyFile; /* File holding private key corresponding to cert used for client cert auth */
sbool allowUnsignedCerts; /* For testing/debugging - do not check for CA certs (CURLOPT_SSL_VERIFYPEER FALSE) */
uchar *token; /* The token value to use to authenticate to Kubernetes - takes precedence over tokenFile */
uchar *tokenFile; /* The file whose contents is the token value to use to authenticate to Kubernetes */
@@ -127,6 +136,7 @@ struct modConfData_s {
uchar *fnRulebase; /* lognorm rulebase filename for container log filename match */
char *contRules; /* lognorm rules for CONTAINER_NAME value match */
uchar *contRulebase; /* lognorm rulebase filename for CONTAINER_NAME value match */
+ int busyRetryInterval; /* how to handle 429 response - 0 means error, non-zero means retry every N seconds */
};
/* action (instance) configuration data */
@@ -135,6 +145,8 @@ typedef struct _instanceData {
msgPropDescr_t *srcMetadataDescr; /* where to get data for kubernetes queries */
uchar *dstMetadataPath; /* where to put metadata obtained from kubernetes */
uchar *caCertFile; /* File holding the CA cert (+optional chain) of CA that issued the Kubernetes server cert */
+ uchar *myCertFile; /* File holding cert corresponding to private key used for client cert auth */
+ uchar *myPrivKeyFile; /* File holding private key corresponding to cert used for client cert auth */
sbool allowUnsignedCerts; /* For testing/debugging - do not check for CA certs (CURLOPT_SSL_VERIFYPEER FALSE) */
uchar *token; /* The token value to use to authenticate to Kubernetes - takes precedence over tokenFile */
uchar *tokenFile; /* The file whose contents is the token value to use to authenticate to Kubernetes */
@@ -151,6 +163,7 @@ typedef struct _instanceData {
msgPropDescr_t *contNameDescr; /* CONTAINER_NAME field */
msgPropDescr_t *contIdFullDescr; /* CONTAINER_ID_FULL field */
struct cache_s *cache;
+ int busyRetryInterval; /* how to handle 429 response - 0 means error, non-zero means retry every N seconds */
} instanceData;
typedef struct wrkrInstanceData {
@@ -159,6 +172,16 @@ typedef struct wrkrInstanceData {
struct curl_slist *curlHdr;
char *curlRply;
size_t curlRplyLen;
+ statsobj_t *stats; /* stats for this instance */
+ STATSCOUNTER_DEF(k8sRecordSeen, mutK8sRecordSeen)
+ STATSCOUNTER_DEF(namespaceMetadataSuccess, mutNamespaceMetadataSuccess)
+ STATSCOUNTER_DEF(namespaceMetadataNotFound, mutNamespaceMetadataNotFound)
+ STATSCOUNTER_DEF(namespaceMetadataBusy, mutNamespaceMetadataBusy)
+ STATSCOUNTER_DEF(namespaceMetadataError, mutNamespaceMetadataError)
+ STATSCOUNTER_DEF(podMetadataSuccess, mutPodMetadataSuccess)
+ STATSCOUNTER_DEF(podMetadataNotFound, mutPodMetadataNotFound)
+ STATSCOUNTER_DEF(podMetadataBusy, mutPodMetadataBusy)
+ STATSCOUNTER_DEF(podMetadataError, mutPodMetadataError)
} wrkrInstanceData_t;
/* module parameters (v6 config format) */
@@ -167,6 +190,8 @@ static struct cnfparamdescr modpdescr[] = {
{ "srcmetadatapath", eCmdHdlrString, 0 },
{ "dstmetadatapath", eCmdHdlrString, 0 },
{ "tls.cacert", eCmdHdlrString, 0 },
+ { "tls.mycert", eCmdHdlrString, 0 },
+ { "tls.myprivkey", eCmdHdlrString, 0 },
{ "allowunsignedcerts", eCmdHdlrBinary, 0 },
{ "token", eCmdHdlrString, 0 },
{ "tokenfile", eCmdHdlrString, 0 },
@@ -174,7 +199,8 @@ static struct cnfparamdescr modpdescr[] = {
{ "de_dot", eCmdHdlrBinary, 0 },
{ "de_dot_separator", eCmdHdlrString, 0 },
{ "filenamerulebase", eCmdHdlrString, 0 },
- { "containerrulebase", eCmdHdlrString, 0 }
+ { "containerrulebase", eCmdHdlrString, 0 },
+ { "busyretryinterval", eCmdHdlrInt, 0 }
#if HAVE_LOADSAMPLESFROMSTRING == 1
,
{ "filenamerules", eCmdHdlrArray, 0 },
@@ -193,6 +219,8 @@ static struct cnfparamdescr actpdescr[] = {
{ "srcmetadatapath", eCmdHdlrString, 0 },
{ "dstmetadatapath", eCmdHdlrString, 0 },
{ "tls.cacert", eCmdHdlrString, 0 },
+ { "tls.mycert", eCmdHdlrString, 0 },
+ { "tls.myprivkey", eCmdHdlrString, 0 },
{ "allowunsignedcerts", eCmdHdlrBinary, 0 },
{ "token", eCmdHdlrString, 0 },
{ "tokenfile", eCmdHdlrString, 0 },
@@ -200,7 +228,8 @@ static struct cnfparamdescr actpdescr[] = {
{ "de_dot", eCmdHdlrBinary, 0 },
{ "de_dot_separator", eCmdHdlrString, 0 },
{ "filenamerulebase", eCmdHdlrString, 0 },
- { "containerrulebase", eCmdHdlrString, 0 }
+ { "containerrulebase", eCmdHdlrString, 0 },
+ { "busyretryinterval", eCmdHdlrInt, 0 }
#if HAVE_LOADSAMPLESFROMSTRING == 1
,
{ "filenamerules", eCmdHdlrArray, 0 },
@@ -493,8 +522,9 @@ ENDbeginCnfLoad
BEGINsetModCnf
struct cnfparamvals *pvals = NULL;
int i;
- FILE *fp;
+ FILE *fp = NULL;
int ret;
+ char errStr[1024];
CODESTARTsetModCnf
pvals = nvlstGetParams(lst, &modpblk, NULL);
if(pvals == NULL) {
@@ -509,6 +539,7 @@ CODESTARTsetModCnf
}
loadModConf->de_dot = DFLT_DE_DOT;
+ loadModConf->busyRetryInterval = DFLT_BUSY_RETRY_INTERVAL;
for(i = 0 ; i < modpblk.nParams ; ++i) {
if(!pvals[i].bUsed) {
continue;
@@ -528,15 +559,42 @@ CODESTARTsetModCnf
loadModConf->caCertFile = (uchar *) es_str2cstr(pvals[i].val.d.estr, NULL);
fp = fopen((const char*)loadModConf->caCertFile, "r");
if(fp == NULL) {
- char errStr[1024];
rs_strerror_r(errno, errStr, sizeof(errStr));
iRet = RS_RET_NO_FILE_ACCESS;
LogError(0, iRet,
- "error: certificate file %s couldn't be accessed: %s\n",
+ "error: 'tls.cacert' file %s couldn't be accessed: %s\n",
loadModConf->caCertFile, errStr);
ABORT_FINALIZE(iRet);
} else {
fclose(fp);
+ fp = NULL;
+ }
+ } else if(!strcmp(modpblk.descr[i].name, "tls.mycert")) {
+ free(loadModConf->myCertFile);
+ loadModConf->myCertFile = (uchar*)es_str2cstr(pvals[i].val.d.estr, NULL);
+ fp = fopen((const char*)loadModConf->myCertFile, "r");
+ if(fp == NULL) {
+ rs_strerror_r(errno, errStr, sizeof(errStr));
+ iRet = RS_RET_NO_FILE_ACCESS;
+ LogError(0, iRet,
+ "error: 'tls.mycert' file %s couldn't be accessed: %s\n",
+ loadModConf->myCertFile, errStr);
+ } else {
+ fclose(fp);
+ fp = NULL;
+ }
+ } else if(!strcmp(modpblk.descr[i].name, "tls.myprivkey")) {
+ loadModConf->myPrivKeyFile = (uchar*)es_str2cstr(pvals[i].val.d.estr, NULL);
+ fp = fopen((const char*)loadModConf->myPrivKeyFile, "r");
+ if(fp == NULL) {
+ rs_strerror_r(errno, errStr, sizeof(errStr));
+ iRet = RS_RET_NO_FILE_ACCESS;
+ LogError(0, iRet,
+ "error: 'tls.myprivkey' file %s couldn't be accessed: %s\n",
+ loadModConf->myPrivKeyFile, errStr);
+ } else {
+ fclose(fp);
+ fp = NULL;
}
} else if(!strcmp(modpblk.descr[i].name, "allowunsignedcerts")) {
loadModConf->allowUnsignedCerts = pvals[i].val.d.n;
@@ -548,7 +606,6 @@ CODESTARTsetModCnf
loadModConf->tokenFile = (uchar *) es_str2cstr(pvals[i].val.d.estr, NULL);
fp = fopen((const char*)loadModConf->tokenFile, "r");
if(fp == NULL) {
- char errStr[1024];
rs_strerror_r(errno, errStr, sizeof(errStr));
iRet = RS_RET_NO_FILE_ACCESS;
LogError(0, iRet,
@@ -557,6 +614,7 @@ CODESTARTsetModCnf
ABORT_FINALIZE(iRet);
} else {
fclose(fp);
+ fp = NULL;
}
} else if(!strcmp(modpblk.descr[i].name, "annotation_match")) {
free_annotationmatch(&loadModConf->annotation_match);
@@ -577,7 +635,6 @@ CODESTARTsetModCnf
loadModConf->fnRulebase = (uchar *) es_str2cstr(pvals[i].val.d.estr, NULL);
fp = fopen((const char*)loadModConf->fnRulebase, "r");
if(fp == NULL) {
- char errStr[1024];
rs_strerror_r(errno, errStr, sizeof(errStr));
iRet = RS_RET_NO_FILE_ACCESS;
LogError(0, iRet,
@@ -586,6 +643,7 @@ CODESTARTsetModCnf
ABORT_FINALIZE(iRet);
} else {
fclose(fp);
+ fp = NULL;
}
#if HAVE_LOADSAMPLESFROMSTRING == 1
} else if(!strcmp(modpblk.descr[i].name, "containerrules")) {
@@ -597,7 +655,6 @@ CODESTARTsetModCnf
loadModConf->contRulebase = (uchar *) es_str2cstr(pvals[i].val.d.estr, NULL);
fp = fopen((const char*)loadModConf->contRulebase, "r");
if(fp == NULL) {
- char errStr[1024];
rs_strerror_r(errno, errStr, sizeof(errStr));
iRet = RS_RET_NO_FILE_ACCESS;
LogError(0, iRet,
@@ -606,7 +663,10 @@ CODESTARTsetModCnf
ABORT_FINALIZE(iRet);
} else {
fclose(fp);
+ fp = NULL;
}
+ } else if(!strcmp(modpblk.descr[i].name, "busyretryinterval")) {
+ loadModConf->busyRetryInterval = pvals[i].val.d.n;
} else {
dbgprintf("mmkubernetes: program error, non-handled "
"param '%s' in module() block\n", modpblk.descr[i].name);
@@ -650,6 +710,8 @@ CODESTARTsetModCnf
caches = calloc(1, sizeof(struct cache_s *));
finalize_it:
+ if (fp)
+ fclose(fp);
if(pvals != NULL)
cnfparamvalsDestruct(pvals, &modpblk);
ENDsetModCnf
@@ -667,6 +729,8 @@ CODESTARTfreeInstance
free(pData->srcMetadataDescr);
free(pData->dstMetadataPath);
free(pData->caCertFile);
+ free(pData->myCertFile);
+ free(pData->myPrivKeyFile);
free(pData->token);
free(pData->tokenFile);
free(pData->fnRules);
@@ -710,6 +774,45 @@ CODESTARTcreateWrkrInstance
char *tokenHdr = NULL;
FILE *fp = NULL;
char *token = NULL;
+ char *statsName = NULL;
+
+ CHKiRet(statsobj.Construct(&(pWrkrData->stats)));
+ if ((-1 == asprintf(&statsName, "mmkubernetes(%s)", pWrkrData->pData->kubernetesUrl)) ||
+ (!statsName)) {
+ ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY);
+ }
+ CHKiRet(statsobj.SetName(pWrkrData->stats, (uchar *)statsName));
+ free(statsName);
+ statsName = NULL;
+ CHKiRet(statsobj.SetOrigin(pWrkrData->stats, UCHAR_CONSTANT("mmkubernetes")));
+ STATSCOUNTER_INIT(pWrkrData->k8sRecordSeen, pWrkrData->mutK8sRecordSeen);
+ CHKiRet(statsobj.AddCounter(pWrkrData->stats, UCHAR_CONSTANT("recordseen"),
+ ctrType_IntCtr, CTR_FLAG_RESETTABLE, &(pWrkrData->k8sRecordSeen)));
+ STATSCOUNTER_INIT(pWrkrData->namespaceMetadataSuccess, pWrkrData->mutNamespaceMetadataSuccess);
+ CHKiRet(statsobj.AddCounter(pWrkrData->stats, UCHAR_CONSTANT("namespacemetadatasuccess"),
+ ctrType_IntCtr, CTR_FLAG_RESETTABLE, &(pWrkrData->namespaceMetadataSuccess)));
+ STATSCOUNTER_INIT(pWrkrData->namespaceMetadataNotFound, pWrkrData->mutNamespaceMetadataNotFound);
+ CHKiRet(statsobj.AddCounter(pWrkrData->stats, UCHAR_CONSTANT("namespacemetadatanotfound"),
+ ctrType_IntCtr, CTR_FLAG_RESETTABLE, &(pWrkrData->namespaceMetadataNotFound)));
+ STATSCOUNTER_INIT(pWrkrData->namespaceMetadataBusy, pWrkrData->mutNamespaceMetadataBusy);
+ CHKiRet(statsobj.AddCounter(pWrkrData->stats, UCHAR_CONSTANT("namespacemetadatabusy"),
+ ctrType_IntCtr, CTR_FLAG_RESETTABLE, &(pWrkrData->namespaceMetadataBusy)));
+ STATSCOUNTER_INIT(pWrkrData->namespaceMetadataError, pWrkrData->mutNamespaceMetadataError);
+ CHKiRet(statsobj.AddCounter(pWrkrData->stats, UCHAR_CONSTANT("namespacemetadataerror"),
+ ctrType_IntCtr, CTR_FLAG_RESETTABLE, &(pWrkrData->namespaceMetadataError)));
+ STATSCOUNTER_INIT(pWrkrData->podMetadataSuccess, pWrkrData->mutPodMetadataSuccess);
+ CHKiRet(statsobj.AddCounter(pWrkrData->stats, UCHAR_CONSTANT("podmetadatasuccess"),
+ ctrType_IntCtr, CTR_FLAG_RESETTABLE, &(pWrkrData->podMetadataSuccess)));
+ STATSCOUNTER_INIT(pWrkrData->podMetadataNotFound, pWrkrData->mutPodMetadataNotFound);
+ CHKiRet(statsobj.AddCounter(pWrkrData->stats, UCHAR_CONSTANT("podmetadatanotfound"),
+ ctrType_IntCtr, CTR_FLAG_RESETTABLE, &(pWrkrData->podMetadataNotFound)));
+ STATSCOUNTER_INIT(pWrkrData->podMetadataBusy, pWrkrData->mutPodMetadataBusy);
+ CHKiRet(statsobj.AddCounter(pWrkrData->stats, UCHAR_CONSTANT("podmetadatabusy"),
+ ctrType_IntCtr, CTR_FLAG_RESETTABLE, &(pWrkrData->podMetadataBusy)));
+ STATSCOUNTER_INIT(pWrkrData->podMetadataError, pWrkrData->mutPodMetadataError);
+ CHKiRet(statsobj.AddCounter(pWrkrData->stats, UCHAR_CONSTANT("podmetadataerror"),
+ ctrType_IntCtr, CTR_FLAG_RESETTABLE, &(pWrkrData->podMetadataError)));
+ CHKiRet(statsobj.ConstructFinalize(pWrkrData->stats));
hdr = curl_slist_append(hdr, "Content-Type: text/json; charset=utf-8");
if (pWrkrData->pData->token) {
@@ -749,12 +852,20 @@ CODESTARTcreateWrkrInstance
curl_easy_setopt(ctx, CURLOPT_WRITEDATA, pWrkrData);
if(pWrkrData->pData->caCertFile)
curl_easy_setopt(ctx, CURLOPT_CAINFO, pWrkrData->pData->caCertFile);
+ if(pWrkrData->pData->myCertFile)
+ curl_easy_setopt(ctx, CURLOPT_SSLCERT, pWrkrData->pData->myCertFile);
+ if(pWrkrData->pData->myPrivKeyFile)
+ curl_easy_setopt(ctx, CURLOPT_SSLKEY, pWrkrData->pData->myPrivKeyFile);
if(pWrkrData->pData->allowUnsignedCerts)
curl_easy_setopt(ctx, CURLOPT_SSL_VERIFYPEER, 0);
pWrkrData->curlCtx = ctx;
finalize_it:
free(token);
+ free(statsName);
+ if ((iRet != RS_RET_OK) && pWrkrData->stats) {
+ statsobj.Destruct(&(pWrkrData->stats));
+ }
if (fp) {
fclose(fp);
}
@@ -765,6 +876,7 @@ BEGINfreeWrkrInstance
CODESTARTfreeWrkrInstance
curl_easy_cleanup(pWrkrData->curlCtx);
curl_slist_free_all(pWrkrData->curlHdr);
+ statsobj.Destruct(&(pWrkrData->stats));
ENDfreeWrkrInstance
@@ -790,6 +902,8 @@ cacheNew(const uchar *const url)
key_equals_string, hashtable_json_object_put);
cache->nsHt = create_hashtable(100, hash_from_string,
key_equals_string, hashtable_json_object_put);
+ dbgprintf("mmkubernetes: created cache mdht [%p] nsht [%p]\n",
+ cache->mdHt, cache->nsHt);
cache->cacheMtx = malloc(sizeof(pthread_mutex_t));
if (!cache->mdHt || !cache->nsHt || !cache->cacheMtx) {
free (cache);
@@ -797,6 +911,7 @@ cacheNew(const uchar *const url)
FINALIZE;
}
pthread_mutex_init(cache->cacheMtx, NULL);
+ cache->lastBusyTime = 0;
finalize_it:
return cache;
@@ -816,9 +931,10 @@ static void cacheFree(struct cache_s *cache)
BEGINnewActInst
struct cnfparamvals *pvals = NULL;
int i;
- FILE *fp;
+ FILE *fp = NULL;
char *rxstr = NULL;
char *srcMetadataPath = NULL;
+ char errStr[1024];
CODESTARTnewActInst
DBGPRINTF("newActInst (mmkubernetes)\n");
@@ -840,6 +956,7 @@ CODESTARTnewActInst
pData->de_dot = loadModConf->de_dot;
pData->allowUnsignedCerts = loadModConf->allowUnsignedCerts;
+ pData->busyRetryInterval = loadModConf->busyRetryInterval;
for(i = 0 ; i < actpblk.nParams ; ++i) {
if(!pvals[i].bUsed) {
continue;
@@ -863,7 +980,6 @@ CODESTARTnewActInst
pData->caCertFile = (uchar *) es_str2cstr(pvals[i].val.d.estr, NULL);
fp = fopen((const char*)pData->caCertFile, "r");
if(fp == NULL) {
- char errStr[1024];
rs_strerror_r(errno, errStr, sizeof(errStr));
iRet = RS_RET_NO_FILE_ACCESS;
LogError(0, iRet,
@@ -872,6 +988,33 @@ CODESTARTnewActInst
ABORT_FINALIZE(iRet);
} else {
fclose(fp);
+ fp = NULL;
+ }
+ } else if(!strcmp(actpblk.descr[i].name, "tls.mycert")) {
+ pData->myCertFile = (uchar*)es_str2cstr(pvals[i].val.d.estr, NULL);
+ fp = fopen((const char*)pData->myCertFile, "r");
+ if(fp == NULL) {
+ rs_strerror_r(errno, errStr, sizeof(errStr));
+ iRet = RS_RET_NO_FILE_ACCESS;
+ LogError(0, iRet,
+ "error: 'tls.mycert' file %s couldn't be accessed: %s\n",
+ pData->myCertFile, errStr);
+ } else {
+ fclose(fp);
+ fp = NULL;
+ }
+ } else if(!strcmp(actpblk.descr[i].name, "tls.myprivkey")) {
+ pData->myPrivKeyFile = (uchar*)es_str2cstr(pvals[i].val.d.estr, NULL);
+ fp = fopen((const char*)pData->myPrivKeyFile, "r");
+ if(fp == NULL) {
+ rs_strerror_r(errno, errStr, sizeof(errStr));
+ iRet = RS_RET_NO_FILE_ACCESS;
+ LogError(0, iRet,
+ "error: 'tls.myprivkey' file %s couldn't be accessed: %s\n",
+ pData->myPrivKeyFile, errStr);
+ } else {
+ fclose(fp);
+ fp = NULL;
}
} else if(!strcmp(actpblk.descr[i].name, "allowunsignedcerts")) {
pData->allowUnsignedCerts = pvals[i].val.d.n;
@@ -883,7 +1026,6 @@ CODESTARTnewActInst
pData->tokenFile = (uchar *) es_str2cstr(pvals[i].val.d.estr, NULL);
fp = fopen((const char*)pData->tokenFile, "r");
if(fp == NULL) {
- char errStr[1024];
rs_strerror_r(errno, errStr, sizeof(errStr));
iRet = RS_RET_NO_FILE_ACCESS;
LogError(0, iRet,
@@ -892,6 +1034,7 @@ CODESTARTnewActInst
ABORT_FINALIZE(iRet);
} else {
fclose(fp);
+ fp = NULL;
}
} else if(!strcmp(actpblk.descr[i].name, "annotation_match")) {
free_annotationmatch(&pData->annotation_match);
@@ -912,7 +1055,6 @@ CODESTARTnewActInst
pData->fnRulebase = (uchar *) es_str2cstr(pvals[i].val.d.estr, NULL);
fp = fopen((const char*)pData->fnRulebase, "r");
if(fp == NULL) {
- char errStr[1024];
rs_strerror_r(errno, errStr, sizeof(errStr));
iRet = RS_RET_NO_FILE_ACCESS;
LogError(0, iRet,
@@ -921,6 +1063,7 @@ CODESTARTnewActInst
ABORT_FINALIZE(iRet);
} else {
fclose(fp);
+ fp = NULL;
}
#if HAVE_LOADSAMPLESFROMSTRING == 1
} else if(!strcmp(modpblk.descr[i].name, "containerrules")) {
@@ -932,7 +1075,6 @@ CODESTARTnewActInst
pData->contRulebase = (uchar *) es_str2cstr(pvals[i].val.d.estr, NULL);
fp = fopen((const char*)pData->contRulebase, "r");
if(fp == NULL) {
- char errStr[1024];
rs_strerror_r(errno, errStr, sizeof(errStr));
iRet = RS_RET_NO_FILE_ACCESS;
LogError(0, iRet,
@@ -941,7 +1083,10 @@ CODESTARTnewActInst
ABORT_FINALIZE(iRet);
} else {
fclose(fp);
+ fp = NULL;
}
+ } else if(!strcmp(actpblk.descr[i].name, "busyretryinterval")) {
+ pData->busyRetryInterval = pvals[i].val.d.n;
} else {
dbgprintf("mmkubernetes: program error, non-handled "
"param '%s' in action() block\n", actpblk.descr[i].name);
@@ -982,6 +1127,10 @@ CODESTARTnewActInst
pData->dstMetadataPath = (uchar *) strdup((char *) loadModConf->dstMetadataPath);
if(pData->caCertFile == NULL && loadModConf->caCertFile)
pData->caCertFile = (uchar *) strdup((char *) loadModConf->caCertFile);
+ if(pData->myCertFile == NULL && loadModConf->myCertFile)
+ pData->myCertFile = (uchar *) strdup((char *) loadModConf->myCertFile);
+ if(pData->myPrivKeyFile == NULL && loadModConf->myPrivKeyFile)
+ pData->myPrivKeyFile = (uchar *) strdup((char *) loadModConf->myPrivKeyFile);
if(pData->token == NULL && loadModConf->token)
pData->token = (uchar *) strdup((char *) loadModConf->token);
if(pData->tokenFile == NULL && loadModConf->tokenFile)
@@ -1018,6 +1167,8 @@ CODESTARTnewActInst
CODE_STD_FINALIZERnewActInst
if(pvals != NULL)
cnfparamvalsDestruct(pvals, &actpblk);
+ if(fp)
+ fclose(fp);
free(rxstr);
free(srcMetadataPath);
ENDnewActInst
@@ -1061,6 +1212,8 @@ CODESTARTfreeCnf
free(pModConf->srcMetadataPath);
free(pModConf->dstMetadataPath);
free(pModConf->caCertFile);
+ free(pModConf->myCertFile);
+ free(pModConf->myPrivKeyFile);
free(pModConf->token);
free(pModConf->tokenFile);
free(pModConf->de_dot_separator);
@@ -1069,8 +1222,11 @@ CODESTARTfreeCnf
free(pModConf->contRules);
free(pModConf->contRulebase);
free_annotationmatch(&pModConf->annotation_match);
- for(i = 0; caches[i] != NULL; i++)
+ for(i = 0; caches[i] != NULL; i++) {
+ dbgprintf("mmkubernetes: freeing cache [%d] mdht [%p] nsht [%p]\n",
+ i, caches[i]->mdHt, caches[i]->nsHt);
cacheFree(caches[i]);
+ }
free(caches);
ENDfreeCnf
@@ -1082,6 +1238,8 @@ CODESTARTdbgPrintInstInfo
dbgprintf("\tsrcMetadataPath='%s'\n", pData->srcMetadataDescr->name);
dbgprintf("\tdstMetadataPath='%s'\n", pData->dstMetadataPath);
dbgprintf("\ttls.cacert='%s'\n", pData->caCertFile);
+ dbgprintf("\ttls.mycert='%s'\n", pData->myCertFile);
+ dbgprintf("\ttls.myprivkey='%s'\n", pData->myPrivKeyFile);
dbgprintf("\tallowUnsignedCerts='%d'\n", pData->allowUnsignedCerts);
dbgprintf("\ttoken='%s'\n", pData->token);
dbgprintf("\ttokenFile='%s'\n", pData->tokenFile);
@@ -1093,6 +1251,7 @@ CODESTARTdbgPrintInstInfo
dbgprintf("\tfilenamerules='%s'\n", pData->fnRules);
dbgprintf("\tcontainerrules='%s'\n", pData->contRules);
#endif
+ dbgprintf("\tbusyretryinterval='%d'\n", pData->busyRetryInterval);
ENDdbgPrintInstInfo
@@ -1206,6 +1365,24 @@ queryKB(wrkrInstanceData_t *pWrkrData, char *url, struct json_object **rply)
struct json_object *jo;
long resp_code = 400;
+ if (pWrkrData->pData->cache->lastBusyTime) {
+ time_t now;
+ datetime.GetTime(&now);
+ now -= pWrkrData->pData->cache->lastBusyTime;
+ if (now < pWrkrData->pData->busyRetryInterval) {
+ LogMsg(0, RS_RET_RETRY, LOG_DEBUG,
+ "mmkubernetes: Waited [%ld] of [%d] seconds for the requested url [%s]\n",
+ now, pWrkrData->pData->busyRetryInterval, url);
+ ABORT_FINALIZE(RS_RET_RETRY);
+ } else {
+ LogMsg(0, RS_RET_OK, LOG_DEBUG,
+ "mmkubernetes: Cleared busy status after [%d] seconds - "
+ "will retry the requested url [%s]\n",
+ pWrkrData->pData->busyRetryInterval, url);
+ pWrkrData->pData->cache->lastBusyTime = 0;
+ }
+ }
+
/* query kubernetes for pod info */
ccode = curl_easy_setopt(pWrkrData->curlCtx, CURLOPT_URL, url);
if(ccode != CURLE_OK)
@@ -1238,17 +1415,23 @@ queryKB(wrkrInstanceData_t *pWrkrData, char *url, struct json_object **rply)
ABORT_FINALIZE(RS_RET_ERR);
}
if(resp_code == 404) {
- LogMsg(0, RS_RET_ERR, LOG_ERR,
+ LogMsg(0, RS_RET_NOT_FOUND, LOG_INFO,
"mmkubernetes: Not Found: the resource does not exist at url [%s]\n",
url);
- ABORT_FINALIZE(RS_RET_ERR);
+ ABORT_FINALIZE(RS_RET_NOT_FOUND);
}
if(resp_code == 429) {
- LogMsg(0, RS_RET_ERR, LOG_ERR,
+ if (pWrkrData->pData->busyRetryInterval) {
+ time_t now;
+ datetime.GetTime(&now);
+ pWrkrData->pData->cache->lastBusyTime = now;
+ }
+
+ LogMsg(0, RS_RET_RETRY, LOG_INFO,
"mmkubernetes: Too Many Requests: the server is too heavily loaded "
"to provide the data for the requested url [%s]\n",
url);
- ABORT_FINALIZE(RS_RET_ERR);
+ ABORT_FINALIZE(RS_RET_RETRY);
}
if(resp_code != 200) {
LogMsg(0, RS_RET_ERR, LOG_ERR,
@@ -1299,12 +1482,14 @@ BEGINdoAction
char *mdKey = NULL;
struct json_object *jMetadata = NULL, *jMetadataCopy = NULL, *jMsgMeta = NULL,
*jo = NULL;
- int add_ns_metadata = 0;
+ int add_pod_metadata = 1;
CODESTARTdoAction
CHKiRet_Hdlr(extractMsgMetadata(pMsg, pWrkrData->pData, &jMsgMeta)) {
ABORT_FINALIZE((iRet == RS_RET_NOT_FOUND) ? RS_RET_OK : iRet);
}
+ STATSCOUNTER_INC(pWrkrData->k8sRecordSeen, pWrkrData->mutK8sRecordSeen);
+
if (fjson_object_object_get_ex(jMsgMeta, "pod_name", &jo))
podName = json_object_get_string(jo);
if (fjson_object_object_get_ex(jMsgMeta, "namespace_name", &jo))
@@ -1347,28 +1532,49 @@ CODESTARTdoAction
}
iRet = queryKB(pWrkrData, url, &jReply);
free(url);
- /* todo: implement support for the .orphaned namespace */
- if (iRet != RS_RET_OK) {
+ if (iRet == RS_RET_NOT_FOUND) {
+ /* negative cache namespace - make a dummy empty namespace metadata object */
+ jNsMeta = json_object_new_object();
+ STATSCOUNTER_INC(pWrkrData->namespaceMetadataNotFound,
+ pWrkrData->mutNamespaceMetadataNotFound);
+ } else if (iRet == RS_RET_RETRY) {
+ /* server is busy - retry or error */
+ STATSCOUNTER_INC(pWrkrData->namespaceMetadataBusy,
+ pWrkrData->mutNamespaceMetadataBusy);
+ if (0 == pWrkrData->pData->busyRetryInterval) {
+ pthread_mutex_unlock(pWrkrData->pData->cache->cacheMtx);
+ ABORT_FINALIZE(RS_RET_ERR);
+ }
+ add_pod_metadata = 0; /* don't cache pod metadata either - retry both */
+ } else if (iRet != RS_RET_OK) {
+ /* hard error - something the admin needs to fix e.g. network, config, auth */
json_object_put(jReply);
jReply = NULL;
+ STATSCOUNTER_INC(pWrkrData->namespaceMetadataError,
+ pWrkrData->mutNamespaceMetadataError);
pthread_mutex_unlock(pWrkrData->pData->cache->cacheMtx);
FINALIZE;
- }
-
- if(fjson_object_object_get_ex(jReply, "metadata", &jNsMeta)) {
+ } else if (fjson_object_object_get_ex(jReply, "metadata", &jNsMeta)) {
jNsMeta = json_object_get(jNsMeta);
parse_labels_annotations(jNsMeta, &pWrkrData->pData->annotation_match,
pWrkrData->pData->de_dot,
(const char *)pWrkrData->pData->de_dot_separator,
pWrkrData->pData->de_dot_separator_len);
- add_ns_metadata = 1;
+ STATSCOUNTER_INC(pWrkrData->namespaceMetadataSuccess,
+ pWrkrData->mutNamespaceMetadataSuccess);
} else {
/* namespace with no metadata??? */
LogMsg(0, RS_RET_ERR, LOG_INFO,
"mmkubernetes: namespace [%s] has no metadata!\n", ns);
- jNsMeta = NULL;
+ /* negative cache namespace - make a dummy empty namespace metadata object */
+ jNsMeta = json_object_new_object();
+ STATSCOUNTER_INC(pWrkrData->namespaceMetadataSuccess,
+ pWrkrData->mutNamespaceMetadataSuccess);
}
+ if(jNsMeta) {
+ hashtable_insert(pWrkrData->pData->cache->nsHt, strdup(ns), jNsMeta);
+ }
json_object_put(jReply);
jReply = NULL;
}
@@ -1381,14 +1587,28 @@ CODESTARTdoAction
}
iRet = queryKB(pWrkrData, url, &jReply);
free(url);
- if(iRet != RS_RET_OK) {
- if(jNsMeta && add_ns_metadata) {
- hashtable_insert(pWrkrData->pData->cache->nsHt, strdup(ns), jNsMeta);
+ if (iRet == RS_RET_NOT_FOUND) {
+ /* negative cache pod - make a dummy empty pod metadata object */
+ iRet = RS_RET_OK;
+ STATSCOUNTER_INC(pWrkrData->podMetadataNotFound, pWrkrData->mutPodMetadataNotFound);
+ } else if (iRet == RS_RET_RETRY) {
+ /* server is busy - retry or error */
+ STATSCOUNTER_INC(pWrkrData->podMetadataBusy, pWrkrData->mutPodMetadataBusy);
+ if (0 == pWrkrData->pData->busyRetryInterval) {
+ pthread_mutex_unlock(pWrkrData->pData->cache->cacheMtx);
+ ABORT_FINALIZE(RS_RET_ERR);
}
+ add_pod_metadata = 0; /* do not cache so that we can retry */
+ iRet = RS_RET_OK;
+ } else if(iRet != RS_RET_OK) {
+ /* hard error - something the admin needs to fix e.g. network, config, auth */
json_object_put(jReply);
jReply = NULL;
+ STATSCOUNTER_INC(pWrkrData->podMetadataError, pWrkrData->mutPodMetadataError);
pthread_mutex_unlock(pWrkrData->pData->cache->cacheMtx);
FINALIZE;
+ } else {
+ STATSCOUNTER_INC(pWrkrData->podMetadataSuccess, pWrkrData->mutPodMetadataSuccess);
}
jo = json_object_new_object();
@@ -1435,11 +1655,9 @@ CODESTARTdoAction
json_object_object_add(jo, "container_id", json_object_get(jo2));
json_object_object_add(jMetadata, "docker", jo);
- hashtable_insert(pWrkrData->pData->cache->mdHt, mdKey, jMetadata);
- mdKey = NULL;
- if(jNsMeta && add_ns_metadata) {
- hashtable_insert(pWrkrData->pData->cache->nsHt, strdup(ns), jNsMeta);
- ns = NULL;
+ if (add_pod_metadata) {
+ hashtable_insert(pWrkrData->pData->cache->mdHt, mdKey, jMetadata);
+ mdKey = NULL;
}
}
@@ -1450,6 +1668,11 @@ CODESTARTdoAction
* outside of the cache lock
*/
jMetadataCopy = json_tokener_parse(json_object_get_string(jMetadata));
+ if (!add_pod_metadata) {
+ /* jMetadata object was created from scratch and not cached */
+ json_object_put(jMetadata);
+ jMetadata = NULL;
+ }
pthread_mutex_unlock(pWrkrData->pData->cache->cacheMtx);
/* the +1 is there to skip the leading '$' */
msgAddJSON(pMsg, (uchar *) pWrkrData->pData->dstMetadataPath + 1, jMetadataCopy, 0, 0);
@@ -1470,7 +1693,9 @@ BEGINmodExit
CODESTARTmodExit
curl_global_cleanup();
+ objRelease(datetime, CORE_COMPONENT);
objRelease(regexp, LM_REGEXP_FILENAME);
+ objRelease(statsobj, CORE_COMPONENT);
ENDmodExit
@@ -1489,8 +1714,9 @@ CODESTARTmodInit
*ipIFVersProvided = CURR_MOD_IF_VERSION; /* we only support the current interface specification */
CODEmodInit_QueryRegCFSLineHdlr
DBGPRINTF("mmkubernetes: module compiled with rsyslog version %s.\n", VERSION);
+ CHKiRet(objUse(statsobj, CORE_COMPONENT));
CHKiRet(objUse(regexp, LM_REGEXP_FILENAME));
-
+ CHKiRet(objUse(datetime, CORE_COMPONENT));
/* CURL_GLOBAL_ALL initializes more than is needed but the
* libcurl documentation discourages use of other values
*/

263
SOURCES/rsyslog-8.37.0-rhbz1627941-imfile-support-for-endmsg.regex.patch
View File

@ -1,263 +0,0 @@
From e8d64cbd15fa84907dc23f8b52d6f2f847b46fec Mon Sep 17 00:00:00 2001
From: Rich Megginson <rmeggins@redhat.com>
Date: Mon, 10 Sep 2018 17:25:38 -0600
Subject: [PATCH] imfile: support for endmsg.regex
This adds support for endmsg.regex. It is similar to
startmsg.regex except that it matches the line that denotes
the end of the message, rather than the start of the next message.
This is primarily for container log file use cases such as this:
date stdout P start of message
date stdout P middle of message
date stdout F end of message
The `F` means this is the line which contains the final part of
the message. The fully assembled message should be
`start of message middle of message end of message`.
`startmsg.regex="^[^ ]+ stdout F "` will match.
(cherry picked from commit c902a0938fe163b5351829d2b72001d024895c16)
(cherry picked from commit dd4a72c4d52d8da98ed6b86114868e1a450ccb41)
---
plugins/imfile/imfile.c | 44 ++++--
plugins/imptcp/imptcp.c | 10 +-
runtime/stream.c | 28 +++-
runtime/stream.h | 2 +-
4 files changed, 62 insertions(+), 20 deletions(-)
diff --git a/plugins/imfile/imfile.c b/plugins/imfile/imfile.c
index 7767c9f02..87706082f 100644
--- a/plugins/imfile/imfile.c
+++ b/plugins/imfile/imfile.c
@@ -126,7 +126,9 @@ struct instanceConf_s {
sbool bRMStateOnDel;
uint8_t readMode;
uchar *startRegex;
- regex_t end_preg; /* compiled version of startRegex */
+ uchar *endRegex;
+ regex_t start_preg; /* compiled version of startRegex */
+ regex_t end_preg; /* compiled version of endRegex */
sbool discardTruncatedMsg;
sbool msgDiscardingError;
sbool escapeLF;
@@ -281,6 +283,7 @@ static struct cnfparamdescr inppdescr[] = {
{ "ruleset", eCmdHdlrString, 0 },
{ "readmode", eCmdHdlrInt, 0 },
{ "startmsg.regex", eCmdHdlrString, 0 },
+ { "endmsg.regex", eCmdHdlrString, 0 },
{ "discardtruncatedmsg", eCmdHdlrBinary, 0 },
{ "msgdiscardingerror", eCmdHdlrBinary, 0 },
{ "escapelf", eCmdHdlrBinary, 0 },
@@ -1421,6 +1424,7 @@ pollFileReal(act_obj_t *act, cstr_t **pCStr)
int64 strtOffs;
DEFiRet;
int nProcessed = 0;
+ regex_t *start_preg = NULL, *end_preg = NULL;
DBGPRINTF("pollFileReal enter, pStrm %p, name '%s'\n", act->pStrm, act->name);
DBGPRINTF("pollFileReal enter, edge %p\n", act->edge);
@@ -1432,15 +1436,18 @@ pollFileReal(act_obj_t *act, cstr_t **pCStr)
CHKiRet(openFile(act)); /* open file */
}
+ start_preg = (inst->startRegex == NULL) ? NULL : &inst->start_preg;
+ end_preg = (inst->endRegex == NULL) ? NULL : &inst->end_preg;
+
/* loop below will be exited when strmReadLine() returns EOF */
while(glbl.GetGlobalInputTermState() == 0) {
if(inst->maxLinesAtOnce != 0 && nProcessed >= inst->maxLinesAtOnce)
break;
- if(inst->startRegex == NULL) {
+ if((start_preg == NULL) && (end_preg == NULL)) {
CHKiRet(strm.ReadLine(act->pStrm, pCStr, inst->readMode, inst->escapeLF,
inst->trimLineOverBytes, &strtOffs));
} else {
- CHKiRet(strmReadMultiLine(act->pStrm, pCStr, &inst->end_preg,
+ CHKiRet(strmReadMultiLine(act->pStrm, pCStr, start_preg, end_preg,
inst->escapeLF, inst->discardTruncatedMsg, inst->msgDiscardingError, &strtOffs));
}
++nProcessed;
@@ -1506,6 +1513,7 @@ createInstance(instanceConf_t **const pinst)
inst->iPersistStateInterval = 0;
inst->readMode = 0;
inst->startRegex = NULL;
+ inst->endRegex = NULL;
inst->discardTruncatedMsg = 0;
inst->msgDiscardingError = 1;
inst->bRMStateOnDel = 1;
@@ -1713,6 +1721,8 @@ CODESTARTnewInpInst
inst->readMode = (sbool) pvals[i].val.d.n;
} else if(!strcmp(inppblk.descr[i].name, "startmsg.regex")) {
inst->startRegex = (uchar*)es_str2cstr(pvals[i].val.d.estr, NULL);
+ } else if(!strcmp(inppblk.descr[i].name, "endmsg.regex")) {
+ inst->endRegex = (uchar*)es_str2cstr(pvals[i].val.d.estr, NULL);
} else if(!strcmp(inppblk.descr[i].name, "discardtruncatedmsg")) {
inst->discardTruncatedMsg = (sbool) pvals[i].val.d.n;
} else if(!strcmp(inppblk.descr[i].name, "msgdiscardingerror")) {
@@ -1753,19 +1763,31 @@ CODESTARTnewInpInst
"param '%s'\n", inppblk.descr[i].name);
}
}
- if(inst->readMode != 0 && inst->startRegex != NULL) {
+ i = (inst->readMode > 0) ? 1 : 0;
+ i = (NULL != inst->startRegex) ? (i+1) : i;
+ i = (NULL != inst->endRegex) ? (i+1) : i;
+ if(i > 1) {
LogError(0, RS_RET_PARAM_NOT_PERMITTED,
- "readMode and startmsg.regex cannot be set "
- "at the same time --- remove one of them");
+ "only one of readMode or startmsg.regex or endmsg.regex can be set "
+ "at the same time");
ABORT_FINALIZE(RS_RET_PARAM_NOT_PERMITTED);
}
if(inst->startRegex != NULL) {
- const int errcode = regcomp(&inst->end_preg, (char*)inst->startRegex, REG_EXTENDED);
+ const int errcode = regcomp(&inst->start_preg, (char*)inst->startRegex, REG_EXTENDED);
+ if(errcode != 0) {
+ char errbuff[512];
+ regerror(errcode, &inst->start_preg, errbuff, sizeof(errbuff));
+ parser_errmsg("imfile: error in startmsg.regex expansion: %s", errbuff);
+ ABORT_FINALIZE(RS_RET_ERR);
+ }
+ }
+ if(inst->endRegex != NULL) {
+ const int errcode = regcomp(&inst->end_preg, (char*)inst->endRegex, REG_EXTENDED);
if(errcode != 0) {
char errbuff[512];
regerror(errcode, &inst->end_preg, errbuff, sizeof(errbuff));
- parser_errmsg("imfile: error in regex expansion: %s", errbuff);
+ parser_errmsg("imfile: error in endmsg.regex expansion: %s", errbuff);
ABORT_FINALIZE(RS_RET_ERR);
}
}
@@ -1970,9 +1992,13 @@ CODESTARTfreeCnf
free(inst->pszStateFile);
free(inst->pszFileName_forOldStateFile);
if(inst->startRegex != NULL) {
- regfree(&inst->end_preg);
+ regfree(&inst->start_preg);
free(inst->startRegex);
}
+ if(inst->endRegex != NULL) {
+ regfree(&inst->end_preg);
+ free(inst->endRegex);
+ }
del = inst;
inst = inst->next;
free(del);
diff --git a/plugins/imptcp/imptcp.c b/plugins/imptcp/imptcp.c
index 9b6be0f40..a94b97f41 100644
--- a/plugins/imptcp/imptcp.c
+++ b/plugins/imptcp/imptcp.c
@@ -162,7 +162,7 @@ struct instanceConf_s {
int ratelimitInterval;
int ratelimitBurst;
uchar *startRegex;
- regex_t end_preg; /* compiled version of startRegex */
+ regex_t start_preg; /* compiled version of startRegex */
struct instanceConf_s *next;
};
@@ -961,7 +961,7 @@ processDataRcvd_regexFraming(ptcpsess_t *const __restrict__ pThis,
if(c == '\n') {
pThis->iCurrLine = pThis->iMsg;
} else {
- const int isMatch = !regexec(&inst->end_preg, (char*)pThis->pMsg+pThis->iCurrLine, 0, NULL, 0);
+ const int isMatch = !regexec(&inst->start_preg, (char*)pThis->pMsg+pThis->iCurrLine, 0, NULL, 0);
if(isMatch) {
DBGPRINTF("regex match (%d), framing line: %s\n", pThis->iCurrLine, pThis->pMsg);
strcpy((char*)pThis->pMsg_save, (char*) pThis->pMsg+pThis->iCurrLine);
@@ -2188,10 +2188,10 @@ CODESTARTnewInpInst
}
if(inst->startRegex != NULL) {
- const int errcode = regcomp(&inst->end_preg, (char*)inst->startRegex, REG_EXTENDED);
+ const int errcode = regcomp(&inst->start_preg, (char*)inst->startRegex, REG_EXTENDED);
if(errcode != 0) {
char errbuff[512];
- regerror(errcode, &inst->end_preg, errbuff, sizeof(errbuff));
+ regerror(errcode, &inst->start_preg, errbuff, sizeof(errbuff));
parser_errmsg("imptcp: error in framing.delimiter.regex expansion: %s", errbuff);
ABORT_FINALIZE(RS_RET_ERR);
}
@@ -2348,7 +2348,7 @@ CODESTARTfreeCnf
free(inst->pszInputName);
free(inst->dfltTZ);
if(inst->startRegex != NULL) {
- regfree(&inst->end_preg);
+ regfree(&inst->start_preg);
free(inst->startRegex);
}
del = inst;
diff --git a/runtime/stream.c b/runtime/stream.c
index 6b7e7028e..0f4197103 100644
--- a/runtime/stream.c
+++ b/runtime/stream.c
@@ -942,12 +942,12 @@ strmReadMultiLine_isTimedOut(const strm_t *const __restrict__ pThis)
/* read a multi-line message from a strm file.
* The multi-line message is terminated based on the user-provided
- * startRegex (Posix ERE). For performance reasons, the regex
+ * startRegex or endRegex (Posix ERE). For performance reasons, the regex
* must already have been compiled by the user.
* added 2015-05-12 rgerhards
*/
rsRetVal
-strmReadMultiLine(strm_t *pThis, cstr_t **ppCStr, regex_t *preg, const sbool bEscapeLF,
+strmReadMultiLine(strm_t *pThis, cstr_t **ppCStr, regex_t *start_preg, regex_t *end_preg, const sbool bEscapeLF,
const sbool discardTruncatedMsg, const sbool msgDiscardingError, int64 *const strtOffs)
{
uchar c;
@@ -979,9 +979,14 @@ strmReadMultiLine(strm_t *pThis, cstr_t **ppCStr, regex_t *preg, const sbool bEs
cstrFinalize(thisLine);
/* we have a line, now let's assemble the message */
- const int isMatch = !regexec(preg, (char*)rsCStrGetSzStrNoNULL(thisLine), 0, NULL, 0);
-
- if(isMatch) {
+ const int isStartMatch = start_preg ?
+ !regexec(start_preg, (char*)rsCStrGetSzStrNoNULL(thisLine), 0, NULL, 0) :
+ 0;
+ const int isEndMatch = end_preg ?
+ !regexec(end_preg, (char*)rsCStrGetSzStrNoNULL(thisLine), 0, NULL, 0) :
+ 0;
+
+ if(isStartMatch) {
/* in this case, the *previous* message is complete and we are
* at the start of a new one.
*/
@@ -1047,6 +1052,19 @@ strmReadMultiLine(strm_t *pThis, cstr_t **ppCStr, regex_t *preg, const sbool bEs
}
}
}
+ if(isEndMatch) {
+ /* in this case, the *current* message is complete and we are
+ * at the end of it.
+ */
+ if(pThis->ignoringMsg == 0) {
+ if(pThis->prevMsgSegment != NULL) {
+ finished = 1;
+ *ppCStr = pThis->prevMsgSegment;
+ pThis->prevMsgSegment= NULL;
+ }
+ }
+ pThis->ignoringMsg = 0;
+ }
cstrDestruct(&thisLine);
} while(finished == 0);
diff --git a/runtime/stream.h b/runtime/stream.h
index 71596879e..7dc597ff5 100644
--- a/runtime/stream.h
+++ b/runtime/stream.h
@@ -225,7 +225,7 @@ ENDinterface(strm)
/* prototypes */
PROTOTYPEObjClassInit(strm);
rsRetVal strmMultiFileSeek(strm_t *pThis, unsigned int fileNum, off64_t offs, off64_t *bytesDel);
-rsRetVal strmReadMultiLine(strm_t *pThis, cstr_t **ppCStr, regex_t *preg,
+rsRetVal strmReadMultiLine(strm_t *pThis, cstr_t **ppCStr, regex_t *start_preg, regex_t *end_preg,
sbool bEscapeLF, sbool discardTruncatedMsg, sbool msgDiscardingError, int64 *const strtOffs);
int strmReadMultiLine_isTimedOut(const strm_t *const __restrict__ pThis);
void strmDebugOutBuf(const strm_t *const pThis);

368
SOURCES/rsyslog-8.37.0-rhbz1674471-imfile-log-rotation.patch
View File

@ -1,368 +0,0 @@
From f85ef7aabcec84497a5eaf9670616b3402c79d9c Mon Sep 17 00:00:00 2001
From: Rainer Gerhards <rgerhards@adiscon.com>
Date: Sun, 23 Sep 2018 13:19:31 +0200
Subject: File rotation with imfile broken
Previously, truncation was only detected at end of file. Especially with
busy files that could cause loss of data and possibly also stall imfile
reading. The new code now also checks during each read. Obviously, there
is some additional overhead associated with that, but this is unavoidable.
It still is highly recommended NOT to turn on "reopenOnTruncate" in imfile.
Note that there are also inherant reliability issues. There is no way to
"fix" these, as they are caused by races between the process(es) who truncate
and rsyslog reading the file. But with the new code, the "problem window"
should be much smaller and, more importantly, imfile should not stall.
A change in the inode was not detected under all circumstances,
most importantly not in some logrotate cases.
Includes new tests made by Andre Lorbach. They now use the
logrotate tool natively to reproduce the issue.
---
runtime/rsyslog.h | 6 ++--
plugins/imfile/imfile.c | 17 +++-
runtime/stream.c | 122 ++++++++----
runtime/stream.h | 7 ++
4 files changed, 126 insertions(+), 36 deletions(-)
diff --git a/runtime/rsyslog.h b/runtime/rsyslog.h
index 61d0af623..22a1c46d1 100644
--- a/runtime/rsyslog.h
+++ b/runtime/rsyslog.h
@@ -221,9 +221,9 @@ enum rsRetVal_ /** return value. All methods return this if not specified oth
/* begin regular error codes */
RS_RET_NOT_IMPLEMENTED = -7, /**< implementation is missing (probably internal error or lazyness ;)) */
RS_RET_OUT_OF_MEMORY = -6, /**< memory allocation failed */
- RS_RET_PROVIDED_BUFFER_TOO_SMALL = -50,
-/*< the caller provided a buffer, but the called function sees the size of this buffer is too small -
-operation not carried out */
+ RS_RET_PROVIDED_BUFFER_TOO_SMALL = -50, /*< the caller provided a buffer, but the called function sees
+ the size of this buffer is too small - operation not carried out */
+ RS_RET_FILE_TRUNCATED = -51, /**< (input) file was truncated, not an error but a status */
RS_RET_TRUE = -3, /**< to indicate a true state (can be used as TRUE, legacy) */
RS_RET_FALSE = -2, /**< to indicate a false state (can be used as FALSE, legacy) */
RS_RET_NO_IRET = -8, /**< This is a trick for the debuging system - it means no iRet is provided */
diff --git a/plugins/imfile/imfile.c b/plugins/imfile/imfile.c
index f4a4ef9b7..6be8b2999 100644
--- a/plugins/imfile/imfile.c
+++ b/plugins/imfile/imfile.c
@@ -740,8 +740,19 @@ detect_updates(fs_edge_t *const edge)
act_obj_unlink(act);
restart = 1;
break;
+ } else if(fileInfo.st_ino != act->ino) {
+ DBGPRINTF("file '%s' inode changed from %llu to %llu, unlinking from "
+ "internal lists\n", act->name, (long long unsigned) act->ino,
+ (long long unsigned) fileInfo.st_ino);
+ if(act->pStrm != NULL) {
+ /* we do no need to re-set later, as act_obj_unlink
+ * will destroy the strm obj */
+ strmSet_checkRotation(act->pStrm, STRM_ROTATION_DO_NOT_CHECK);
+ }
+ act_obj_unlink(act);
+ restart = 1;
+ break;
}
- // TODO: add inode check for change notification!
}
@@ -993,10 +1004,10 @@ chk_active(const act_obj_t *act, const act_obj_t *const deleted)
/* unlink act object from linked list and then
* destruct it.
*/
-static void //ATTR_NONNULL()
+static void ATTR_NONNULL()
act_obj_unlink(act_obj_t *act)
{
- DBGPRINTF("act_obj_unlink %p: %s\n", act, act->name);
+ DBGPRINTF("act_obj_unlink %p: %s, pStrm %p\n", act, act->name, act->pStrm);
if(act->prev == NULL) {
act->edge->active = act->next;
} else {
diff --git a/runtime/stream.c b/runtime/stream.c
index 0f4197103..32a12b256 100644
--- a/runtime/stream.c
+++ b/runtime/stream.c
@@ -400,6 +400,7 @@ static rsRetVal strmOpenFile(strm_t *pThis)
CHKiRet(doPhysOpen(pThis));
pThis->iCurrOffs = 0;
+ pThis->iBufPtrMax = 0;
CHKiRet(getFileSize(pThis->pszCurrFName, &offset));
if(pThis->tOperationsMode == STREAMMODE_WRITE_APPEND) {
pThis->iCurrOffs = offset;
@@ -636,6 +637,78 @@ strmHandleEOF(strm_t *pThis)
RETiRet;
}
+
+/* helper to checkTruncation */
+static rsRetVal ATTR_NONNULL()
+rereadTruncated(strm_t *const pThis, const char *const reason)
+{
+ DEFiRet;
+
+ LogMsg(errno, RS_RET_FILE_TRUNCATED, LOG_WARNING, "file '%s': truncation detected, "
+ "(%s) - re-start reading from beginning",
+ pThis->pszCurrFName, reason);
+ DBGPRINTF("checkTruncation, file %s last buffer CHANGED\n", pThis->pszCurrFName);
+ CHKiRet(strmCloseFile(pThis));
+ CHKiRet(strmOpenFile(pThis));
+ iRet = RS_RET_FILE_TRUNCATED;
+
+finalize_it:
+ RETiRet;
+}
+/* helper to read:
+ * Check if file has been truncated since last read and, if so, re-set reading
+ * to begin of file. To detect truncation, we try to re-read the last block.
+ * If that does not succeed or different data than from the original read is
+ * returned, truncation is assumed.
+ * NOTE: this function must be called only if truncation is enabled AND
+ * when the previous read buffer still is valid (aka "before the next read").
+ * It is ok to call with a 0-size buffer, which we than assume as begin of
+ * reading. In that case, no truncation will be detected.
+ * rgerhards, 2018-09-20
+ */
+static rsRetVal ATTR_NONNULL()
+checkTruncation(strm_t *const pThis)
+{
+ DEFiRet;
+ int ret;
+ off64_t backseek;
+ assert(pThis->bReopenOnTruncate);
+
+ DBGPRINTF("checkTruncation, file %s, iBufPtrMax %zd\n", pThis->pszCurrFName, pThis->iBufPtrMax);
+ if(pThis->iBufPtrMax == 0) {
+ FINALIZE;
+ }
+
+ int currpos = lseek64(pThis->fd, 0, SEEK_CUR);
+ backseek = -1 * (off64_t) pThis->iBufPtrMax;
+ dbgprintf("checkTruncation in actual processing, currpos %d, backseek is %d\n", (int)currpos, (int) backseek);
+ ret = lseek64(pThis->fd, backseek, SEEK_CUR);
+ if(ret < 0) {
+ iRet = rereadTruncated(pThis, "cannot seek backward to begin of last block");
+ FINALIZE;
+ }
+dbgprintf("checkTruncation seek backwrds: %d\n", ret);
+currpos = lseek64(pThis->fd, 0, SEEK_CUR);
+dbgprintf("checkTruncation in actual processing, NEW currpos %d, backseek is %d\n", (int)currpos, (int) backseek);
+
+ const ssize_t lenRead = read(pThis->fd, pThis->pIOBuf_truncation, pThis->iBufPtrMax);
+ dbgprintf("checkTruncation proof-read: %d bytes\n", (int) lenRead);
+ if(lenRead < 0) {
+ iRet = rereadTruncated(pThis, "last block could not be re-read");
+ FINALIZE;
+ }
+
+ if(!memcmp(pThis->pIOBuf_truncation, pThis->pIOBuf, pThis->iBufPtrMax)) {
+ DBGPRINTF("checkTruncation, file %s last buffer unchanged\n", pThis->pszCurrFName);
+ } else {
+ iRet = rereadTruncated(pThis, "last block data different");
+ }
+
+finalize_it:
+ RETiRet;
+}
+
+
/* read the next buffer from disk
* rgerhards, 2008-02-13
*/
@@ -668,6 +741,13 @@ strmReadBuf(strm_t *pThis, int *padBytes)
toRead = (size_t) bytesLeft;
}
}
+ if(pThis->bReopenOnTruncate) {
+ rsRetVal localRet = checkTruncation(pThis);
+ if(localRet == RS_RET_FILE_TRUNCATED) {
+ continue;
+ }
+ CHKiRet(localRet);
+ }
iLenRead = read(pThis->fd, pThis->pIOBuf, toRead);
DBGOPRINT((obj_t*) pThis, "file %d read %ld bytes\n", pThis->fd, iLenRead);
/* end crypto */
@@ -1184,6 +1264,7 @@ static rsRetVal strmConstructFinalize(strm_t *pThis)
} else {
/* we work synchronously, so we need to alloc a fixed pIOBuf */
CHKmalloc(pThis->pIOBuf = (uchar*) MALLOC(pThis->sIOBufSize));
+ CHKmalloc(pThis->pIOBuf_truncation = (char*) MALLOC(pThis->sIOBufSize));
}
finalize_it:
@@ -1231,6 +1312,7 @@ CODESTARTobjDestruct(strm)
}
} else {
free(pThis->pIOBuf);
+ free(pThis->pIOBuf_truncation);
}
/* Finally, we can free the resources.
diff --git a/runtime/stream.c b/runtime/stream.c
index 2d494c612..5b52591ef 100644
--- a/runtime/stream.c
+++ b/runtime/stream.c
@@ -360,8 +360,8 @@ CheckFileChange(strm_t *pThis)
CHKiRet(strmSetCurrFName(pThis));
if(stat((char*) pThis->pszCurrFName, &statName) == -1)
ABORT_FINALIZE(RS_RET_IO_ERROR);
- DBGPRINTF("stream/after deserialize checking for file change on '%s', "
- "inode %u/%u, size/currOffs %llu/%llu\n",
+ DBGPRINTF("CheckFileChange: stream/after deserialize checking for file change "
+ "on '%s', inode %u/%u, size/currOffs %llu/%llu\n",
pThis->pszCurrFName, (unsigned) pThis->inode,
(unsigned) statName.st_ino,
(long long unsigned) statName.st_size,
@@ -574,8 +574,8 @@ strmNextFile(strm_t *pThis)
* circumstances). So starting as of now, we only check the inode number and
* a file change is detected only if the inode changes. -- rgerhards, 2011-01-10
*/
-static rsRetVal
-strmHandleEOFMonitor(strm_t *pThis)
+static rsRetVal ATTR_NONNULL()
+strmHandleEOFMonitor(strm_t *const pThis)
{
DEFiRet;
struct stat statName;
@@ -611,8 +611,8 @@ strmHandleEOFMonitor(strm_t *pThis)
* try to open the next one.
* rgerhards, 2008-02-13
*/
-static rsRetVal
-strmHandleEOF(strm_t *pThis)
+static rsRetVal ATTR_NONNULL()
+strmHandleEOF(strm_t *const pThis)
{
DEFiRet;
@@ -629,7 +629,13 @@ strmHandleEOF(strm_t *pThis)
CHKiRet(strmNextFile(pThis));
break;
case STREAMTYPE_FILE_MONITOR:
- CHKiRet(strmHandleEOFMonitor(pThis));
+ DBGOPRINT((obj_t*) pThis, "file '%s' (%d) EOF, rotationCheck %d\n",
+ pThis->pszCurrFName, pThis->fd, pThis->rotationCheck);
+ if(pThis->rotationCheck == STRM_ROTATION_DO_CHECK) {
+ CHKiRet(strmHandleEOFMonitor(pThis));
+ } else {
+ ABORT_FINALIZE(RS_RET_EOF);
+ }
break;
}
@@ -687,9 +693,6 @@ checkTruncation(strm_t *const pThis)
iRet = rereadTruncated(pThis, "cannot seek backward to begin of last block");
FINALIZE;
}
-dbgprintf("checkTruncation seek backwrds: %d\n", ret);
-currpos = lseek64(pThis->fd, 0, SEEK_CUR);
-dbgprintf("checkTruncation in actual processing, NEW currpos %d, backseek is %d\n", (int)currpos, (int) backseek);
const ssize_t lenRead = read(pThis->fd, pThis->pIOBuf_truncation, pThis->iBufPtrMax);
dbgprintf("checkTruncation proof-read: %d bytes\n", (int) lenRead);
@@ -861,7 +864,7 @@ static rsRetVal strmUnreadChar(strm_t *pThis, uchar c)
* a line, but following lines that are indented are part of the same log entry
*/
static rsRetVal
-strmReadLine(strm_t *pThis, cstr_t **ppCStr, uint8_t mode, sbool bEscapeLF,
+strmReadLine(strm_t *const pThis, cstr_t **ppCStr, uint8_t mode, sbool bEscapeLF,
uint32_t trimLineOverBytes, int64 *const strtOffs)
{
uchar c;
@@ -2147,14 +2150,25 @@ DEFpropSetMeth(strm, cryprov, cryprov_if_t*)
DEFpropSetMeth(strm, cryprovData, void*)
/* sets timeout in seconds */
-void
+void ATTR_NONNULL()
strmSetReadTimeout(strm_t *const __restrict__ pThis, const int val)
{
+ ISOBJ_TYPE_assert(pThis, strm);
pThis->readTimeout = val;
}
-static rsRetVal strmSetbDeleteOnClose(strm_t *pThis, int val)
+void ATTR_NONNULL()
+strmSet_checkRotation(strm_t *const pThis, const int val) {
+ ISOBJ_TYPE_assert(pThis, strm);
+ assert(val == STRM_ROTATION_DO_CHECK || val == STRM_ROTATION_DO_NOT_CHECK);
+ pThis->rotationCheck = val;
+}
+
+
+static rsRetVal ATTR_NONNULL()
+strmSetbDeleteOnClose(strm_t *const pThis, const int val)
{
+ ISOBJ_TYPE_assert(pThis, strm);
pThis->bDeleteOnClose = val;
if(pThis->cryprov != NULL) {
pThis->cryprov->SetDeleteOnClose(pThis->cryprovFileData, pThis->bDeleteOnClose);
@@ -2162,15 +2176,19 @@ static rsRetVal strmSetbDeleteOnClose(strm_t *pThis, int val)
return RS_RET_OK;
}
-static rsRetVal strmSetiMaxFiles(strm_t *pThis, int iNewVal)
+static rsRetVal ATTR_NONNULL()
+strmSetiMaxFiles(strm_t *const pThis, const int iNewVal)
{
+ ISOBJ_TYPE_assert(pThis, strm);
pThis->iMaxFiles = iNewVal;
pThis->iFileNumDigits = getNumberDigits(iNewVal);
return RS_RET_OK;
}
-static rsRetVal strmSetFileNotFoundError(strm_t *pThis, int pFileNotFoundError)
+static rsRetVal ATTR_NONNULL()
+strmSetFileNotFoundError(strm_t *const pThis, const int pFileNotFoundError)
{
+ ISOBJ_TYPE_assert(pThis, strm);
pThis->fileNotFoundError = pFileNotFoundError;
return RS_RET_OK;
}
diff --git a/runtime/stream.h b/runtime/stream.h
index 7dc597ff5..e3d6c2372 100644
--- a/runtime/stream.h
+++ b/runtime/stream.h
@@ -124,6 +124,7 @@ typedef struct strm_s {
ino_t inode; /* current inode for files being monitored (undefined else) */
uchar *pszCurrFName; /* name of current file (if open) */
uchar *pIOBuf; /* the iobuffer currently in use to gather data */
+ char *pIOBuf_truncation; /* iobuffer used during trucation detection block re-reads */
size_t iBufPtrMax; /* current max Ptr in Buffer (if partial read!) */
size_t iBufPtr; /* pointer into current buffer */
int iUngetC; /* char set via UngetChar() call or -1 if none set */
diff --git a/runtime/stream.h b/runtime/stream.h
index e3d6c2372..f6f48378a 100644
--- a/runtime/stream.h
+++ b/runtime/stream.h
@@ -91,6 +91,10 @@ typedef enum { /* when extending, do NOT change existing modes! */
STREAMMODE_WRITE_APPEND = 4
} strmMode_t;
+/* settings for stream rotation (applies not to all processing modes!) */
+#define STRM_ROTATION_DO_CHECK 0
+#define STRM_ROTATION_DO_NOT_CHECK 1
+
#define STREAM_ASYNC_NUMBUFS 2 /* must be a power of 2 -- TODO: make configurable */
/* The strm_t data structure */
typedef struct strm_s {
@@ -114,6 +118,7 @@ typedef struct strm_s {
sbool bDisabled; /* should file no longer be written to? (currently set only if omfile file size limit fails) */
sbool bSync; /* sync this file after every write? */
sbool bReopenOnTruncate;
+ int rotationCheck; /* rotation check mode */
size_t sIOBufSize;/* size of IO buffer */
uchar *pszDir; /* Directory */
int lenDir;
@@ -234,5 +239,6 @@ void strmSetReadTimeout(strm_t *const __restrict__ pThis, const int val);
const uchar * ATTR_NONNULL() strmGetPrevLineSegment(strm_t *const pThis);
const uchar * ATTR_NONNULL() strmGetPrevMsgSegment(strm_t *const pThis);
int ATTR_NONNULL() strmGetPrevWasNL(const strm_t *const pThis);
+void ATTR_NONNULL() strmSet_checkRotation(strm_t *const pThis, const int val);
#endif /* #ifndef STREAM_H_INCLUDED */

86
SOURCES/rsyslog-8.37.0-rhbz1677037-short-offMsg-overrun-crash.patch
View File

@ -1,86 +0,0 @@
From 1255a67fdec2fc44cd49b6ea8c463f4319910812 Mon Sep 17 00:00:00 2001
From: Jiri Vymazal <jvymazal@redhat.com>
Date: Wed, 27 Feb 2019 11:57:49 +0100
Subject: [PATCH] Enlarged msg offset types for bigger structured messages
using a large enough (dozens of kBs) structured message
it is possible to overflow the signed short type which leads
to rsyslog crash.
---
runtime/msg.c | 12 ++++++------
runtime/msg.h | 8 ++++----
2 files changed, 10 insertions(+), 10 deletions(-)
diff --git a/runtime/msg.c b/runtime/msg.c
index b82c38b9ee..96306bbeab 100644
--- a/runtime/msg.c
+++ b/runtime/msg.c
@@ -839,7 +839,7 @@ msgBaseConstruct(smsg_t **ppThis)
pM->iFacility = LOG_INVLD;
pM->iLenPROGNAME = -1;
pM->offAfterPRI = 0;
- pM->offMSG = -1;
+ pM->offMSG = 0;
pM->iProtocolVersion = 0;
pM->msgFlags = 0;
pM->iLenRawMsg = 0;
@@ -2167,7 +2167,7 @@ MsgSetFlowControlType(smsg_t * const pMsg, flowControl_t eFlowCtl)
* rgerhards, 2009-06-16
*/
rsRetVal
-MsgSetAfterPRIOffs(smsg_t * const pMsg, short offs)
+MsgSetAfterPRIOffs(smsg_t * const pMsg, uint32_t offs)
{
assert(pMsg != NULL);
pMsg->offAfterPRI = offs;
@@ -2819,12 +2819,12 @@ void MsgSetHOSTNAME(smsg_t *pThis, const uchar* pszHOSTNAME, const int lenHOSTNA
* (exactly by one). This can happen if we have a message that does not
* contain any MSG part.
*/
-void MsgSetMSGoffs(smsg_t * const pMsg, short offs)
+void MsgSetMSGoffs(smsg_t * const pMsg, uint32_t offs)
{
ISOBJ_TYPE_assert(pMsg, msg);
pMsg->offMSG = offs;
- if(offs > pMsg->iLenRawMsg) {
- assert(offs - 1 == pMsg->iLenRawMsg);
+ if(offs > (uint32_t)pMsg->iLenRawMsg) {
+ assert((int)offs - 1 == pMsg->iLenRawMsg);
pMsg->iLenMSG = 0;
} else {
pMsg->iLenMSG = pMsg->iLenRawMsg - offs;
@@ -2920,7 +2920,7 @@ MsgSetRawMsg(smsg_t *const pThis, const char*const pszRawMsg, const size_t lenMs
memcpy(pThis->pszRawMsg, pszRawMsg, pThis->iLenRawMsg);
pThis->pszRawMsg[pThis->iLenRawMsg] = '\0'; /* this also works with truncation! */
/* correct other information */
- if(pThis->iLenRawMsg > pThis->offMSG)
+ if((uint32_t)pThis->iLenRawMsg > pThis->offMSG)
pThis->iLenMSG += deltaSize;
else
pThis->iLenMSG = 0;
diff --git a/runtime/msg.h b/runtime/msg.h
index 74439275b1..722cca6e8a 100644
--- a/runtime/msg.h
+++ b/runtime/msg.h
@@ -67,8 +67,8 @@ struct msg {
sbool bParseSuccess; /* set to reflect state of last executed higher level parser */
unsigned short iSeverity;/* the severity */
unsigned short iFacility;/* Facility code */
- short offAfterPRI; /* offset, at which raw message WITHOUT PRI part starts in pszRawMsg */
- short offMSG; /* offset at which the MSG part starts in pszRawMsg */
+ uint32_t offAfterPRI; /* offset, at which raw message WITHOUT PRI part starts in pszRawMsg */
+ uint32_t offMSG; /* offset at which the MSG part starts in pszRawMsg */
short iProtocolVersion;/* protocol version of message received 0 - legacy, 1 syslog-protocol) */
int msgFlags; /* flags associated with this message */
int iLenRawMsg; /* length of raw message */
@@ -194,8 +194,8 @@ void MsgSetRcvFromStr(smsg_t *const pMsg, const uchar* pszRcvFrom, const int, pr
rsRetVal MsgSetRcvFromIP(smsg_t *pMsg, prop_t*);
rsRetVal MsgSetRcvFromIPStr(smsg_t *const pThis, const uchar *psz, const int len, prop_t **ppProp);
void MsgSetHOSTNAME(smsg_t *pMsg, const uchar* pszHOSTNAME, const int lenHOSTNAME);
-rsRetVal MsgSetAfterPRIOffs(smsg_t *pMsg, short offs);
-void MsgSetMSGoffs(smsg_t *pMsg, short offs);
+rsRetVal MsgSetAfterPRIOffs(smsg_t *pMsg, uint32_t offs);
+void MsgSetMSGoffs(smsg_t *pMsg, uint32_t offs);
void MsgSetRawMsgWOSize(smsg_t *pMsg, char* pszRawMsg);
void ATTR_NONNULL() MsgSetRawMsg(smsg_t *const pThis, const char*const pszRawMsg, const size_t lenMsg);
rsRetVal MsgReplaceMSG(smsg_t *pThis, const uchar* pszMSG, int lenMSG);

58
SOURCES/rsyslog-8.37.0-rhbz1716867-imjournal-memleak.patch
View File

@ -1,58 +0,0 @@
From 920c28ff705aac74f389b4613815b14b9482e497 Mon Sep 17 00:00:00 2001
From: Jiri Vymazal <jvymazal@redhat.com>
Date: Mon, 21 Jan 2019 10:58:03 +0100
Subject: [PATCH] Added missing free() calls of received journal cursor
---
plugins/imjournal/imjournal.c | 5 +++++
1 file changed, 5 insertions(+)
diff --git a/plugins/imjournal/imjournal.c b/plugins/imjournal/imjournal.c
index a85e521003..f5c2be4b6e 100644
--- a/plugins/imjournal/imjournal.c
+++ b/plugins/imjournal/imjournal.c
@@ -449,6 +449,7 @@ readjournal(void)
finalize_it:
free(sys_iden_help);
free(message);
+ free(c);
RETiRet;
}
@@ -470,6 +471,7 @@ persistJournalState(void)
}
} else {
int ret;
+ free(last_cursor);
if ((ret = sd_journal_get_cursor(j, &last_cursor))) {
LogError(-ret, RS_RET_ERR, "imjournal: sd_journal_get_cursor() failed");
ABORT_FINALIZE(RS_RET_ERR);
@@ -630,6 +632,7 @@ loadJournalState(void)
iRet = RS_RET_ERR;
}
}
+ free(tmp_cursor);
}
} else {
LogError(0, RS_RET_IO_ERROR, "imjournal: "
@@ -843,6 +846,7 @@ BEGINfreeCnf
free(cs.stateFile);
free(cs.usePid);
free(cs.dfltTag);
+ free(last_cursor);
statsobj.Destruct(&(statsCounter.stats));
ENDfreeCnf
diff --git a/plugins/imjournal/imjournal.c b/plugins/imjournal/imjournal.c
index f5c2be4b6e..7225fae1ab 100644
--- a/plugins/imjournal/imjournal.c
+++ b/plugins/imjournal/imjournal.c
@@ -474,6 +474,7 @@ persistJournalState(void)
free(last_cursor);
if ((ret = sd_journal_get_cursor(j, &last_cursor))) {
LogError(-ret, RS_RET_ERR, "imjournal: sd_journal_get_cursor() failed");
+ last_cursor = NULL;
ABORT_FINALIZE(RS_RET_ERR);
}
}

72
SPECS/rsyslog.spec
View File

@ -5,8 +5,8 @@
Summary: Enhanced system logging and kernel message trapping daemon Summary: Enhanced system logging and kernel message trapping daemon
Name: rsyslog Name: rsyslog
Version: 8.37.0 Version: 8.1911.0
Release: 11%{?dist} Release: 2%{?dist}
License: (GPLv3+ and ASL 2.0) License: (GPLv3+ and ASL 2.0)
Group: System Environment/Daemons Group: System Environment/Daemons
ExcludeArch: i686 ExcludeArch: i686
@ -44,16 +44,10 @@ Provides: syslog
Obsoletes: sysklogd < 1.5-11 Obsoletes: sysklogd < 1.5-11
# tweak the upstream service file to honour configuration from /etc/sysconfig/rsyslog # tweak the upstream service file to honour configuration from /etc/sysconfig/rsyslog
Patch0: rsyslog-8.32.0-service.patch Patch0: rsyslog-8.1911.0-service.patch
# imjournal: adds "journal" when tag/process name is missing # imjournal: adds "journal" when tag/process name is missing
Patch1: rsyslog-8.37.0-rhbz1659898-imjournal-default-tag.patch Patch1: rsyslog-8.1911.0-rhbz1659898-imjournal-default-tag.patch
Patch2: rsyslog-8.37.0-rhbz1614179-imfile-symlink-support.patch Patch2: rsyslog-8.1911.0-rhbz1763757-imfile-statefiles.patch
Patch3: rsyslog-8.37.0-rhbz1622768-kubernetes-404-handling.patch
Patch4: rsyslog-8.37.0-rhbz1627941-imfile-support-for-endmsg.regex.patch
Patch5: rsyslog-8.37.0-rhbz1674471-imfile-log-rotation.patch
Patch6: rsyslog-8.37.0-rhbz1677037-short-offMsg-overrun-crash.patch
Patch7: rsyslog-8.37.0-rhbz1614181-imtcp-imudp-preservecase-option.patch
Patch8: rsyslog-8.37.0-rhbz1716867-imjournal-memleak.patch
%package crypto %package crypto
Summary: Encryption support Summary: Encryption support
@ -161,8 +155,8 @@ Elasticsearch.
%description gnutls %description gnutls
The rsyslog-gnutls package contains the rsyslog plugins that provide the The rsyslog-gnutls package contains the rsyslog plugins that provide the
ability to receive syslog messages via upcoming syslog-transport-tls ability to send and receive syslog messages via TCP or RELP using TLS
IETF standard protocol. encryption. For details refer to rsyslog doc on imtcp and omfwd modules.
%description gssapi %description gssapi
The rsyslog-gssapi package contains the rsyslog plugins which support GSSAPI The rsyslog-gssapi package contains the rsyslog plugins which support GSSAPI
@ -227,13 +221,7 @@ mv build doc
%patch0 -p1 -b .service %patch0 -p1 -b .service
%patch1 -p1 -b .default-tag %patch1 -p1 -b .default-tag
%patch2 -p1 -b .imfile-symlink %patch2 -p1 -b .imfile-statefiles
%patch3 -p1 -b .mmkubernetes-404
%patch4 -p1 -b .endmsg-regex
%patch5 -p1 -b .rotation-detection
%patch6 -p1 -b .short-offmsg-crash
%patch7 -p1 -b .preservecase-option
%patch8 -p1 -b .imjournal-memleak
%build %build
%ifarch sparc64 %ifarch sparc64
@ -270,6 +258,7 @@ autoreconf -if
--enable-mmsnmptrapd \ --enable-mmsnmptrapd \
--enable-mmutf8fix \ --enable-mmutf8fix \
--enable-mysql \ --enable-mysql \
--enable-omhttp \
--enable-omjournal \ --enable-omjournal \
--enable-omkafka \ --enable-omkafka \
--enable-omprog \ --enable-omprog \
@ -361,7 +350,6 @@ done
%{_libdir}/rsyslog/lmnetstrms.so %{_libdir}/rsyslog/lmnetstrms.so
%{_libdir}/rsyslog/lmnsd_ptcp.so %{_libdir}/rsyslog/lmnsd_ptcp.so
%{_libdir}/rsyslog/lmregexp.so %{_libdir}/rsyslog/lmregexp.so
%{_libdir}/rsyslog/lmstrmsrv.so
%{_libdir}/rsyslog/lmtcpclt.so %{_libdir}/rsyslog/lmtcpclt.so
%{_libdir}/rsyslog/lmtcpsrv.so %{_libdir}/rsyslog/lmtcpsrv.so
%{_libdir}/rsyslog/lmzlibw.so %{_libdir}/rsyslog/lmzlibw.so
@ -369,6 +357,7 @@ done
%{_libdir}/rsyslog/mmcount.so %{_libdir}/rsyslog/mmcount.so
%{_libdir}/rsyslog/mmexternal.so %{_libdir}/rsyslog/mmexternal.so
%{_libdir}/rsyslog/mmutf8fix.so %{_libdir}/rsyslog/mmutf8fix.so
%{_libdir}/rsyslog/omhttp.so
%{_libdir}/rsyslog/omjournal.so %{_libdir}/rsyslog/omjournal.so
%{_libdir}/rsyslog/ommail.so %{_libdir}/rsyslog/ommail.so
%{_libdir}/rsyslog/omprog.so %{_libdir}/rsyslog/omprog.so
@ -437,6 +426,47 @@ done
%changelog %changelog
* Tue Dec 03 2019 Jiri Vymazal <jvymazal@redhat.com> - 8.1911.0-2
RHEL 8.2.0 ERRATUM
- cleaned old patches, fixed patch names
resolves: rhbz#1740683
* Mon Dec 02 2019 Jiri Vymazal <jvymazal@redhat.com> - 8.1911.0-1
RHEL 8.2.0 ERRATUM
- rebased to 8.1911.0 upstream version, removed, previously
upstreamed patches
resolves: rhbz#1740683
resolves: rhbz#1659383
resolves: rhbz#1746876
resolves: rhbz#1676559
resolves: rhbz#1692072
resolves: rhbz#1692073
resolves: rhbz#1692074
resolves: rhbz#1699242
resolves: rhbz#1738213
resolves: rhbz#1744691
resolves: rhbz#1755218
resolves: rhbz#1768321
resolves: rhbz#1768324
- added patch fixing imfile stefiles naming
resolves: rhbz#1763757
* Fri Aug 30 2019 Jiri Vymazal <jvymazal@redhat.com> - 8.37.0-13
RHEL 8.1.0 ERRATUM
- added patch enabling stricter TLS certs checking conforming to
common criteria requirements
resolves: rhbz#1733244
* Mon Jul 22 2019 Jiri Vymazal <jvymazal@redhat.com> - 8.37.0-12
RHEL 8.1.0 ERRATUM
- edited imjournal memleak patch to not cause double-free crash
resolves: rhbz#1729995
- added patch calling journald API only when there are no
preceeding errors
resolves: rhbz#1722165
- added patch fixing imrelp module when invoked with old syntax
resolves: rhbz#1724218
* Wed Jun 05 2019 Jiri Vymazal <jvymazal@redhat.com> - 8.37.0-11 * Wed Jun 05 2019 Jiri Vymazal <jvymazal@redhat.com> - 8.37.0-11
RHEL 8.1.0 ERRATUM RHEL 8.1.0 ERRATUM
- fixed memory leak in imjournal by proper cursor releasing - fixed memory leak in imjournal by proper cursor releasing