diff --git a/openscap-1.3.6-memory-limit-pr-1827.patch b/openscap-1.3.6-memory-limit-pr-1827.patch new file mode 100644 index 0000000..32e0280 --- /dev/null +++ b/openscap-1.3.6-memory-limit-pr-1827.patch @@ -0,0 +1,242 @@ +From a7a0c4a3f528594bb3181174b6986e9c50a684b4 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Jan=20=C4=8Cern=C3=BD?= +Date: Mon, 30 Aug 2021 15:44:37 +0200 +Subject: [PATCH 1/3] Lower memory limits and improve their checking + +This patch attempts to mitigate problems caused by a large amount of +collected objects such as rhbz#1932833. + +Specifically, these changes are made: +- Lower the threshold so that the amount of used memory is checked when + only 1000 items are collected for the given OVAL object. That's + because 32768 items (the original value) is already a large amount which + occupies a lot of memory during further processing. +- Lower the memory usage ratio limit for the probe to 10 %. We have + found experimentally that giving the probe 15 % or more will cause the + oscap process to be killed when processing the collected data and + generating results. +- In the calling function probe_item_collect, distinguish between return + codes which means different behavior when there is insufficient memory + than when the memory consumption can't be checked. +- Improve the warning message to show greater details about memory + consumption to the user. +- Remove the check for the absolute amount of remaining free memory. As + we can see on the example of rhbz#1932833, on systems with large + amount of memory the remaining memory of 512 MB isn't enough memory for + openscap to process the collected data. At the same time, if we lowered + the usage ratio, we don't need this anymore. +- Remove useless message "spt:" from the verbose log because it's + produced many times and pollutes the log extremely. +--- + src/OVAL/probes/probe/icache.c | 23 +++++++++++------------ + src/common/memusage.c | 2 -- + 2 files changed, 11 insertions(+), 14 deletions(-) + +diff --git a/src/OVAL/probes/probe/icache.c b/src/OVAL/probes/probe/icache.c +index 7e16daa334..af7b528372 100644 +--- a/src/OVAL/probes/probe/icache.c ++++ b/src/OVAL/probes/probe/icache.c +@@ -487,9 +487,8 @@ int probe_icache_nop(probe_icache_t *cache) + return (0); + } + +-#define PROBE_RESULT_MEMCHECK_CTRESHOLD 32768 /* item count */ +-#define PROBE_RESULT_MEMCHECK_MINFREEMEM 512 /* MiB */ +-#define PROBE_RESULT_MEMCHECK_MAXRATIO 0.8 /* max. memory usage ratio - used/total */ ++#define PROBE_RESULT_MEMCHECK_CTRESHOLD 1000 /* item count */ ++#define PROBE_RESULT_MEMCHECK_MAXRATIO 0.1 /* max. memory usage ratio - used/total */ + + /** + * Returns 0 if the memory constraints are not reached. Otherwise, 1 is returned. +@@ -511,18 +510,12 @@ static int probe_cobj_memcheck(size_t item_cnt) + c_ratio = (double)mu_proc.mu_rss/(double)(mu_sys.mu_total); + + if (c_ratio > PROBE_RESULT_MEMCHECK_MAXRATIO) { +- dW("Memory usage ratio limit reached! limit=%f, current=%f", +- PROBE_RESULT_MEMCHECK_MAXRATIO, c_ratio); ++ dW("Memory usage ratio limit reached! limit=%f, current=%f, used=%ld MB, free=%ld MB, total=%ld MB, count of items=%ld", ++ PROBE_RESULT_MEMCHECK_MAXRATIO, c_ratio, mu_proc.mu_rss / 1024, mu_sys.mu_realfree / 1024, mu_sys.mu_total / 1024, item_cnt); + errno = ENOMEM; + return (1); + } + +- if ((mu_sys.mu_realfree / 1024) < PROBE_RESULT_MEMCHECK_MINFREEMEM) { +- dW("Minimum free memory limit reached! limit=%zu, current=%zu", +- PROBE_RESULT_MEMCHECK_MINFREEMEM, mu_sys.mu_realfree / 1024); +- errno = ENOMEM; +- return (1); +- } + } + + return (0); +@@ -547,6 +540,7 @@ int probe_item_collect(struct probe_ctx *ctx, SEXP_t *item) + { + SEXP_t *cobj_content; + size_t cobj_itemcnt; ++ int memcheck_ret; + + if (ctx == NULL || ctx->probe_out == NULL || item == NULL) { + return -1; +@@ -556,7 +550,12 @@ int probe_item_collect(struct probe_ctx *ctx, SEXP_t *item) + cobj_itemcnt = SEXP_list_length(cobj_content); + SEXP_free(cobj_content); + +- if (probe_cobj_memcheck(cobj_itemcnt) != 0) { ++ memcheck_ret = probe_cobj_memcheck(cobj_itemcnt); ++ if (memcheck_ret == -1) { ++ dE("Failed to check available memory"); ++ return -1; ++ } ++ if (memcheck_ret == 1) { + + /* + * Don't set the message again if the collected object is +diff --git a/src/common/memusage.c b/src/common/memusage.c +index fc6909e6fb..c6755f21f1 100644 +--- a/src/common/memusage.c ++++ b/src/common/memusage.c +@@ -137,8 +137,6 @@ static int read_status(const char *source, void *base, struct stat_parser *spt, + sp = oscap_bfind(spt, spt_size, sizeof(struct stat_parser), + linebuf, (int(*)(void *, void *))&cmpkey); + +- dD("spt: %s", linebuf); +- + if (sp == NULL) { + /* drop end of unread line */ + while (strchr(strval, '\n') == NULL) { + +From ded3d58cd62259b217a9ab35030827ac3cb8dd45 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Jan=20=C4=8Cern=C3=BD?= +Date: Tue, 7 Sep 2021 13:52:50 +0200 +Subject: [PATCH 2/3] Allow to set memory ratio by environment variable + +If the probe memory usage ratio limit will be too small or too big +in some situation, the user will be able to modify the limit easily +by setting the environment variable OSCAP_PROBE_MEMORY_USAGE_RATIO +to a different value. This can also help users when debugging memory +problems. +--- + docs/manual/manual.adoc | 1 + + src/OVAL/probes/probe/icache.c | 9 ++++----- + src/OVAL/probes/probe/probe.h | 1 + + src/OVAL/probes/probe/worker.c | 12 ++++++++++++ + 4 files changed, 18 insertions(+), 5 deletions(-) + +diff --git a/docs/manual/manual.adoc b/docs/manual/manual.adoc +index 90e2cc2c63..825844bc71 100644 +--- a/docs/manual/manual.adoc ++++ b/docs/manual/manual.adoc +@@ -1613,6 +1613,7 @@ not considered local by the scanner: + * `OSCAP_PROBE_ROOT` - Path to a directory which contains mounted filesystem to be evaluated. Used for offline scanning. + * `SEXP_VALIDATE_DISABLE` - If set, `oscap` will not validate SEXP expressions during its execution. + * `SOURCE_DATE_EPOCH` - Timestamp in seconds since epoch. This timestamp will be used instead of the current time to populate `timestamp` attributes in SCAP source data streams created by `oscap ds sds-compose` sub-module. This is used for reproducible builds of data streams. ++* `OSCAP_PROBE_MEMORY_USAGE_RATIO` - maximum memory usage ratio (used/total) for OpenSCAP probes, default: 0.1 + + Also, OpenSCAP uses `libcurl` library which also can be configured using environment variables. See https://curl.se/libcurl/c/libcurl-env.html[the list of libcurl environment variables]. + +diff --git a/src/OVAL/probes/probe/icache.c b/src/OVAL/probes/probe/icache.c +index af7b528372..a397d35ec2 100644 +--- a/src/OVAL/probes/probe/icache.c ++++ b/src/OVAL/probes/probe/icache.c +@@ -488,13 +488,12 @@ int probe_icache_nop(probe_icache_t *cache) + } + + #define PROBE_RESULT_MEMCHECK_CTRESHOLD 1000 /* item count */ +-#define PROBE_RESULT_MEMCHECK_MAXRATIO 0.1 /* max. memory usage ratio - used/total */ + + /** + * Returns 0 if the memory constraints are not reached. Otherwise, 1 is returned. + * In case of an error, -1 is returned. + */ +-static int probe_cobj_memcheck(size_t item_cnt) ++static int probe_cobj_memcheck(size_t item_cnt, double max_ratio) + { + if (item_cnt > PROBE_RESULT_MEMCHECK_CTRESHOLD) { + struct proc_memusage mu_proc; +@@ -509,9 +508,9 @@ static int probe_cobj_memcheck(size_t item_cnt) + + c_ratio = (double)mu_proc.mu_rss/(double)(mu_sys.mu_total); + +- if (c_ratio > PROBE_RESULT_MEMCHECK_MAXRATIO) { ++ if (c_ratio > max_ratio) { + dW("Memory usage ratio limit reached! limit=%f, current=%f, used=%ld MB, free=%ld MB, total=%ld MB, count of items=%ld", +- PROBE_RESULT_MEMCHECK_MAXRATIO, c_ratio, mu_proc.mu_rss / 1024, mu_sys.mu_realfree / 1024, mu_sys.mu_total / 1024, item_cnt); ++ max_ratio, c_ratio, mu_proc.mu_rss / 1024, mu_sys.mu_realfree / 1024, mu_sys.mu_total / 1024, item_cnt); + errno = ENOMEM; + return (1); + } +@@ -550,7 +549,7 @@ int probe_item_collect(struct probe_ctx *ctx, SEXP_t *item) + cobj_itemcnt = SEXP_list_length(cobj_content); + SEXP_free(cobj_content); + +- memcheck_ret = probe_cobj_memcheck(cobj_itemcnt); ++ memcheck_ret = probe_cobj_memcheck(cobj_itemcnt, ctx->max_mem_ratio); + if (memcheck_ret == -1) { + dE("Failed to check available memory"); + return -1; +diff --git a/src/OVAL/probes/probe/probe.h b/src/OVAL/probes/probe/probe.h +index 1c7a3b1b00..d3a488c4d5 100644 +--- a/src/OVAL/probes/probe/probe.h ++++ b/src/OVAL/probes/probe/probe.h +@@ -83,6 +83,7 @@ struct probe_ctx { + SEXP_t *filters; /**< object filters (OVAL 5.8 and higher) */ + probe_icache_t *icache; /**< item cache */ + int offline_mode; ++ double max_mem_ratio; + }; + + typedef enum { +diff --git a/src/OVAL/probes/probe/worker.c b/src/OVAL/probes/probe/worker.c +index 94fe5c2037..3ef489b40d 100644 +--- a/src/OVAL/probes/probe/worker.c ++++ b/src/OVAL/probes/probe/worker.c +@@ -52,6 +52,10 @@ extern int chroot(const char *); + #include "probe-table.h" + #include "probe.h" + ++/* default max. memory usage ratio - used/total */ ++/* can be overridden by environment variable OSCAP_PROBE_MEMORY_USAGE_RATIO */ ++#define OSCAP_PROBE_MEMORY_USAGE_RATIO_DEFAULT 0.1 ++ + extern bool OSCAP_GSYM(varref_handling); + extern void *OSCAP_GSYM(probe_arg); + +@@ -1064,6 +1068,14 @@ SEXP_t *probe_worker(probe_t *probe, SEAP_msg_t *msg_in, int *ret) + + pctx.offline_mode = probe->selected_offline_mode; + ++ pctx.max_mem_ratio = OSCAP_PROBE_MEMORY_USAGE_RATIO_DEFAULT; ++ char *max_ratio_str = getenv("OSCAP_PROBE_MEMORY_USAGE_RATIO"); ++ if (max_ratio_str != NULL) { ++ double max_ratio = strtod(max_ratio_str, NULL); ++ if (max_ratio != 0) ++ pctx.max_mem_ratio = max_ratio; ++ } ++ + /* simple object */ + pctx.icache = probe->icache; + pctx.filters = probe_prepare_filters(probe, probe_in); + +From 0f5cf5b09f469920616a2037d0f9c81cf0868a58 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Jan=20=C4=8Cern=C3=BD?= +Date: Wed, 15 Sep 2021 14:41:30 +0200 +Subject: [PATCH 3/3] Update src/OVAL/probes/probe/worker.c + +Co-authored-by: Evgeny Kolesnikov +--- + src/OVAL/probes/probe/worker.c | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/src/OVAL/probes/probe/worker.c b/src/OVAL/probes/probe/worker.c +index 3ef489b40d..1183ed06bf 100644 +--- a/src/OVAL/probes/probe/worker.c ++++ b/src/OVAL/probes/probe/worker.c +@@ -1072,7 +1072,7 @@ SEXP_t *probe_worker(probe_t *probe, SEAP_msg_t *msg_in, int *ret) + char *max_ratio_str = getenv("OSCAP_PROBE_MEMORY_USAGE_RATIO"); + if (max_ratio_str != NULL) { + double max_ratio = strtod(max_ratio_str, NULL); +- if (max_ratio != 0) ++ if (max_ratio > 0) + pctx.max_mem_ratio = max_ratio; + } + diff --git a/openscap.spec b/openscap.spec index 1d2e8e4..13e9cd2 100644 --- a/openscap.spec +++ b/openscap.spec @@ -25,6 +25,7 @@ Patch16: openscap-1.3.6-blueprint-toml-pr-1810.patch Patch17: openscap-1.3.6-local-files-pr-1769.patch Patch18: openscap-1.3.6-oscap-ssh-local-files-pr-1786.patch Patch19: openscap-1.3.6-alternative-hostname-pr-1806.patch +Patch20: openscap-1.3.6-memory-limit-pr-1827.patch BuildRequires: make BuildRequires: cmake >= 2.6 BuildRequires: gcc @@ -218,6 +219,7 @@ pathfix.py -i %{__python3} -p -n $RPM_BUILD_ROOT%{_bindir}/scap-as-rpm * Tue Nov 09 2021 Jan Černý - 1:1.3.5-12 - Allow using local files instead of remote resources (rhbz#2015518) - Add an alternative source of hostname (rhbz#2021509) +- Lower memory limits and improve their checking (rhbz#2022362) * Thu Nov 04 2021 Jan Černý - 1:1.3.5-11 - Initialize crypto API only once (rhbz#2020044)