import CS git pacemaker-2.1.7-5.6.el8_10
This commit is contained in:
parent
00e0349c74
commit
9a878be200
303
SOURCES/015-ipc-disconnect.patch
Normal file
303
SOURCES/015-ipc-disconnect.patch
Normal file
@ -0,0 +1,303 @@
|
||||
From 581ef435f7b6b0fde76663069ec63b3b4fb4b067 Mon Sep 17 00:00:00 2001
|
||||
From: Chris Lumens <clumens@redhat.com>
|
||||
Date: Wed, 27 Aug 2025 10:41:13 -0400
|
||||
Subject: [PATCH 1/5] Refactor: libcrmcommon: Rearrange the queue_len check.
|
||||
|
||||
Check if the queue length is 0 first and return, which allows everything
|
||||
else to be un-indented one level.
|
||||
---
|
||||
lib/common/ipc_server.c | 47 ++++++++++++++++++++---------------------
|
||||
1 file changed, 23 insertions(+), 24 deletions(-)
|
||||
|
||||
diff --git a/lib/common/ipc_server.c b/lib/common/ipc_server.c
|
||||
index 5cd7e70..ee8ae9e 100644
|
||||
--- a/lib/common/ipc_server.c
|
||||
+++ b/lib/common/ipc_server.c
|
||||
@@ -535,34 +535,33 @@ crm_ipcs_flush_events(pcmk__client_t *c)
|
||||
pcmk_rc_str(rc), (long long) qb_rc);
|
||||
}
|
||||
|
||||
- if (queue_len) {
|
||||
-
|
||||
- /* Allow clients to briefly fall behind on processing incoming messages,
|
||||
- * but drop completely unresponsive clients so the connection doesn't
|
||||
- * consume resources indefinitely.
|
||||
- */
|
||||
- if (queue_len > QB_MAX(c->queue_max, PCMK_IPC_DEFAULT_QUEUE_MAX)) {
|
||||
- if ((c->queue_backlog <= 1) || (queue_len < c->queue_backlog)) {
|
||||
- /* Don't evict for a new or shrinking backlog */
|
||||
- crm_warn("Client with process ID %u has a backlog of %u messages "
|
||||
- CRM_XS " %p", c->pid, queue_len, c->ipcs);
|
||||
- } else {
|
||||
- crm_err("Evicting client with process ID %u due to backlog of %u messages "
|
||||
- CRM_XS " %p", c->pid, queue_len, c->ipcs);
|
||||
- c->queue_backlog = 0;
|
||||
- qb_ipcs_disconnect(c->ipcs);
|
||||
- return rc;
|
||||
- }
|
||||
- }
|
||||
-
|
||||
- c->queue_backlog = queue_len;
|
||||
- delay_next_flush(c, queue_len);
|
||||
-
|
||||
- } else {
|
||||
+ if (queue_len == 0) {
|
||||
/* Event queue is empty, there is no backlog */
|
||||
c->queue_backlog = 0;
|
||||
+ return rc;
|
||||
+ }
|
||||
+
|
||||
+ /* Allow clients to briefly fall behind on processing incoming messages,
|
||||
+ * but drop completely unresponsive clients so the connection doesn't
|
||||
+ * consume resources indefinitely.
|
||||
+ */
|
||||
+ if (queue_len > QB_MAX(c->queue_max, PCMK_IPC_DEFAULT_QUEUE_MAX)) {
|
||||
+ if ((c->queue_backlog <= 1) || (queue_len < c->queue_backlog)) {
|
||||
+ /* Don't evict for a new or shrinking backlog */
|
||||
+ crm_warn("Client with process ID %u has a backlog of %u messages "
|
||||
+ CRM_XS " %p", c->pid, queue_len, c->ipcs);
|
||||
+ } else {
|
||||
+ crm_err("Evicting client with process ID %u due to backlog of %u messages "
|
||||
+ CRM_XS " %p", c->pid, queue_len, c->ipcs);
|
||||
+ c->queue_backlog = 0;
|
||||
+ qb_ipcs_disconnect(c->ipcs);
|
||||
+ return rc;
|
||||
+ }
|
||||
}
|
||||
|
||||
+ c->queue_backlog = queue_len;
|
||||
+ delay_next_flush(c, queue_len);
|
||||
+
|
||||
return rc;
|
||||
}
|
||||
|
||||
--
|
||||
2.43.0
|
||||
|
||||
From 54fbc6bea137d0642308d49506f13bd84cd2084e Mon Sep 17 00:00:00 2001
|
||||
From: Chris Lumens <clumens@redhat.com>
|
||||
Date: Wed, 27 Aug 2025 11:31:37 -0400
|
||||
Subject: [PATCH 2/5] Refactor: libcrmcommon: Simplify an empty event queue
|
||||
check.
|
||||
|
||||
I find this just a little bit more straightforward to follow.
|
||||
---
|
||||
lib/common/ipc_server.c | 9 ++++++---
|
||||
1 file changed, 6 insertions(+), 3 deletions(-)
|
||||
|
||||
diff --git a/lib/common/ipc_server.c b/lib/common/ipc_server.c
|
||||
index ee8ae9e..d24db59 100644
|
||||
--- a/lib/common/ipc_server.c
|
||||
+++ b/lib/common/ipc_server.c
|
||||
@@ -500,10 +500,13 @@ crm_ipcs_flush_events(pcmk__client_t *c)
|
||||
pcmk__ipc_header_t *header = NULL;
|
||||
struct iovec *event = NULL;
|
||||
|
||||
- if (c->event_queue) {
|
||||
- // We don't pop unless send is successful
|
||||
- event = g_queue_peek_head(c->event_queue);
|
||||
+ if ((c->event_queue == NULL) || g_queue_is_empty(c->event_queue)) {
|
||||
+ break;
|
||||
}
|
||||
+
|
||||
+ // We don't pop unless send is successful
|
||||
+ event = g_queue_peek_head(c->event_queue);
|
||||
+
|
||||
if (event == NULL) { // Queue is empty
|
||||
break;
|
||||
}
|
||||
--
|
||||
2.43.0
|
||||
|
||||
From 6446aa1d917be090989860c3a5cc00ea6a311d67 Mon Sep 17 00:00:00 2001
|
||||
From: Chris Lumens <clumens@redhat.com>
|
||||
Date: Wed, 27 Aug 2025 11:35:38 -0400
|
||||
Subject: [PATCH 3/5] Refactor: libcrmcommon: Rearrange a few tests in
|
||||
crm_ipcs_flush_events.
|
||||
|
||||
Again, no important code changes here. I just find these a little
|
||||
easier to follow.
|
||||
---
|
||||
lib/common/ipc_server.c | 6 ++++--
|
||||
1 file changed, 4 insertions(+), 2 deletions(-)
|
||||
|
||||
diff --git a/lib/common/ipc_server.c b/lib/common/ipc_server.c
|
||||
index d24db59..c305dfc 100644
|
||||
--- a/lib/common/ipc_server.c
|
||||
+++ b/lib/common/ipc_server.c
|
||||
@@ -486,16 +486,18 @@ crm_ipcs_flush_events(pcmk__client_t *c)
|
||||
|
||||
if (c == NULL) {
|
||||
return rc;
|
||||
+ }
|
||||
|
||||
- } else if (c->event_timer) {
|
||||
+ if (c->event_timer != 0) {
|
||||
/* There is already a timer, wait until it goes off */
|
||||
crm_trace("Timer active for %p - %d", c->ipcs, c->event_timer);
|
||||
return rc;
|
||||
}
|
||||
|
||||
- if (c->event_queue) {
|
||||
+ if (c->event_queue != NULL) {
|
||||
queue_len = g_queue_get_length(c->event_queue);
|
||||
}
|
||||
+
|
||||
while (sent < 100) {
|
||||
pcmk__ipc_header_t *header = NULL;
|
||||
struct iovec *event = NULL;
|
||||
--
|
||||
2.43.0
|
||||
|
||||
From d7576ecb3f51050a21057d86257bf8b8c273e4db Mon Sep 17 00:00:00 2001
|
||||
From: Chris Lumens <clumens@redhat.com>
|
||||
Date: Wed, 27 Aug 2025 13:14:54 -0400
|
||||
Subject: [PATCH 4/5] Feature: libcrmcommon: Be more lenient in evicting IPC
|
||||
clients.
|
||||
|
||||
Each IPC connection has a message queue. If the client is unable to
|
||||
process messages faster than the server is sending them, that queue
|
||||
start to back up. pacemaker enforces a cap on the queue size, and
|
||||
that's adjustable with the cluster-ipc-limit parameter. Once the queue
|
||||
grows beyond that size, the client is assumed to be dead and is evicted
|
||||
so it can be restarted and the queue resources freed.
|
||||
|
||||
However, it's possible that the client is not dead. On clusters with
|
||||
very large numbers of resources (I've tried with 300, but fewer might
|
||||
also cause problems), certain actions can happen that cause a spike in
|
||||
IPC messages. In RHEL-76276, the action that causes this is moving
|
||||
nodes in and out of standby. This spike in messages causes the server
|
||||
to overwhelm the client, which is then evicted.
|
||||
|
||||
My multi-part IPC patches made this even worse, as now if the CIB is so
|
||||
large that it needs to split an IPC message up, there will be more
|
||||
messages than before.
|
||||
|
||||
What this fix does is get rid of the cap on the queue size for pacemaker
|
||||
daemons. As long as the server has been able to send messages to the
|
||||
client, the client is still doing work and shouldn't be evicted. It may
|
||||
just be processing messages slower than the server is sending them.
|
||||
Note that this could lead the queue to grow without bound, eventually
|
||||
crashing the server. For this reason, we're only allowing pacemaker
|
||||
daemons to ignore the queue size limit.
|
||||
|
||||
Potential problems with this approach:
|
||||
|
||||
* If the client is so busy that it can't receive even a single message
|
||||
that crm_ipcs_flush_events tries to send, it will still be evicted.
|
||||
However, the flush operation does retry with a delay several times
|
||||
giving the client time to finish up what it's doing.
|
||||
|
||||
* We have timers all over the place with daemons waiting on replies.
|
||||
It's possible that because we are no longer just evicting the clients,
|
||||
we will now see those timers expire which will just lead to different
|
||||
problems. If so, these fixes would probably need to take place in the
|
||||
client code.
|
||||
|
||||
Fixes T38
|
||||
---
|
||||
lib/common/ipc_server.c | 14 ++++++++++++--
|
||||
1 file changed, 12 insertions(+), 2 deletions(-)
|
||||
|
||||
diff --git a/lib/common/ipc_server.c b/lib/common/ipc_server.c
|
||||
index c305dfc..16a2986 100644
|
||||
--- a/lib/common/ipc_server.c
|
||||
+++ b/lib/common/ipc_server.c
|
||||
@@ -551,10 +551,20 @@ crm_ipcs_flush_events(pcmk__client_t *c)
|
||||
* consume resources indefinitely.
|
||||
*/
|
||||
if (queue_len > QB_MAX(c->queue_max, PCMK_IPC_DEFAULT_QUEUE_MAX)) {
|
||||
- if ((c->queue_backlog <= 1) || (queue_len < c->queue_backlog)) {
|
||||
- /* Don't evict for a new or shrinking backlog */
|
||||
+ /* Don't evict:
|
||||
+ * - Clients with a new backlog.
|
||||
+ * - Clients with a shrinking backlog (the client is processing
|
||||
+ * messages faster than the server is sending them).
|
||||
+ * - Clients that are pacemaker daemons and have had any messages sent
|
||||
+ * to them in this flush call (the server is sending messages faster
|
||||
+ * than the client is processing them, but the client is not dead).
|
||||
+ */
|
||||
+ if ((c->queue_backlog <= 1)
|
||||
+ || (queue_len < c->queue_backlog)
|
||||
+ || ((sent > 0) && crm_is_daemon_name(c->name))) {
|
||||
crm_warn("Client with process ID %u has a backlog of %u messages "
|
||||
CRM_XS " %p", c->pid, queue_len, c->ipcs);
|
||||
+
|
||||
} else {
|
||||
crm_err("Evicting client with process ID %u due to backlog of %u messages "
|
||||
CRM_XS " %p", c->pid, queue_len, c->ipcs);
|
||||
--
|
||||
2.43.0
|
||||
|
||||
From 6b5e50b272c26c95e9fae1c3270c77a8d72446e8 Mon Sep 17 00:00:00 2001
|
||||
From: Chris Lumens <clumens@redhat.com>
|
||||
Date: Tue, 30 Sep 2025 13:50:53 -0400
|
||||
Subject: [PATCH 5/5] Feature: libcrmcommon: Update documentation for
|
||||
cluster-ipc-limit.
|
||||
|
||||
Clarify that this no longer applies to pacemaker daemons.
|
||||
---
|
||||
cts/cli/regression.daemons.exp | 4 ++--
|
||||
doc/sphinx/Pacemaker_Explained/cluster-options.rst | 12 +++++++-----
|
||||
lib/common/options.c | 6 +++---
|
||||
3 files changed, 12 insertions(+), 10 deletions(-)
|
||||
|
||||
diff --git a/cts/cli/regression.daemons.exp b/cts/cli/regression.daemons.exp
|
||||
index 678cb62..dffbe6a 100644
|
||||
--- a/cts/cli/regression.daemons.exp
|
||||
+++ b/cts/cli/regression.daemons.exp
|
||||
@@ -11,8 +11,8 @@
|
||||
<content type="boolean" default=""/>
|
||||
</parameter>
|
||||
<parameter name="cluster-ipc-limit">
|
||||
- <longdesc lang="en">Raise this if log has "Evicting client" messages for cluster daemon PIDs (a good value is the number of resources in the cluster multiplied by the number of nodes).</longdesc>
|
||||
- <shortdesc lang="en">Maximum IPC message backlog before disconnecting a cluster daemon</shortdesc>
|
||||
+ <longdesc lang="en">Raise this if log has "Evicting client" messages for cluster PIDs (a good value is the number of resources in the cluster multiplied by the number of nodes).</longdesc>
|
||||
+ <shortdesc lang="en">Maximum IPC message backlog before disconnecting a client</shortdesc>
|
||||
<content type="integer" default=""/>
|
||||
</parameter>
|
||||
</parameters>
|
||||
diff --git a/doc/sphinx/Pacemaker_Explained/cluster-options.rst b/doc/sphinx/Pacemaker_Explained/cluster-options.rst
|
||||
index 77bd7e6..fe2d4f1 100644
|
||||
--- a/doc/sphinx/Pacemaker_Explained/cluster-options.rst
|
||||
+++ b/doc/sphinx/Pacemaker_Explained/cluster-options.rst
|
||||
@@ -675,11 +675,13 @@ values, by running the ``man pacemaker-schedulerd`` and
|
||||
cluster-ipc-limit
|
||||
- :ref:`nonnegative integer <nonnegative_integer>`
|
||||
- 500
|
||||
- - The maximum IPC message backlog before one cluster daemon will
|
||||
- disconnect another. This is of use in large clusters, for which a good
|
||||
- value is the number of resources in the cluster multiplied by the number
|
||||
- of nodes. The default of 500 is also the minimum. Raise this if you see
|
||||
- "Evicting client" log messages for cluster daemon process IDs.
|
||||
+ - The maximum IPC message backlog before a cluster daemon will disconnect
|
||||
+ a client. Other cluster daemons are not subject to this limit as long as
|
||||
+ they are still processing messages. This is of use in large clusters,
|
||||
+ for which a good value is the number of resources in the cluster
|
||||
+ multiplied by the number of nodes. The default of 500 is also the
|
||||
+ minimum. Raise this if you see "Evicting client" log messages for
|
||||
+ cluster process IDs.
|
||||
* - .. _pe_error_series_max:
|
||||
|
||||
.. index::
|
||||
diff --git a/lib/common/options.c b/lib/common/options.c
|
||||
index 96f059c..d3fc684 100644
|
||||
--- a/lib/common/options.c
|
||||
+++ b/lib/common/options.c
|
||||
@@ -422,10 +422,10 @@ static pcmk__cluster_option_t cluster_options[] = {
|
||||
"cluster-ipc-limit", NULL, "integer", NULL,
|
||||
"500", pcmk__valid_positive_number,
|
||||
pcmk__opt_context_based,
|
||||
- N_("Maximum IPC message backlog before disconnecting a cluster daemon"),
|
||||
+ N_("Maximum IPC message backlog before disconnecting a client"),
|
||||
N_("Raise this if log has \"Evicting client\" messages for cluster "
|
||||
- "daemon PIDs (a good value is the number of resources in the "
|
||||
- "cluster multiplied by the number of nodes)."),
|
||||
+ "PIDs (a good value is the number of resources in the cluster "
|
||||
+ "multiplied by the number of nodes)."),
|
||||
},
|
||||
|
||||
// Orphans and stopping
|
||||
--
|
||||
2.43.0
|
||||
|
||||
88
SOURCES/016-fewer-messages.patch
Normal file
88
SOURCES/016-fewer-messages.patch
Normal file
@ -0,0 +1,88 @@
|
||||
From 7be9cdca98217d002497714aaafcfc292c02555b Mon Sep 17 00:00:00 2001
|
||||
From: Chris Lumens <clumens@redhat.com>
|
||||
Date: Fri, 31 Oct 2025 11:24:14 -0400
|
||||
Subject: [PATCH] Med: daemons: Don't add repeated I_PE_CALC messages to the
|
||||
fsa queue.
|
||||
|
||||
Let's say you have a two node cluster, node1 and node2. For purposes of
|
||||
testing, it's easiest if you use fence_dummy instead of a real fencing
|
||||
agent as this will fake fencing happen but without rebooting the node so
|
||||
you can see all the log files.
|
||||
|
||||
Assume the DC is node1. Now do the following:
|
||||
|
||||
- pcs node standby node1
|
||||
- pcs resource defaults update resource-stickiness=1
|
||||
- for i in $(seq 1 300); do echo $i; pcs resource create dummy$i ocf:heartbeat:Dummy --group dummy-group; done
|
||||
- pcs node unstandby node1
|
||||
|
||||
It will take a long time to create that many resources. After node1
|
||||
comes out of standby, it'll take a minute or two but eventually you'll
|
||||
see that node1 was fenced. On node1, you'll see a lot of transition
|
||||
abort messages happen. Each of these transition aborts causes an
|
||||
I_PE_CALC message to be generated and added to the fsa queue. In my
|
||||
testing, I've seen the queue grow to ~ 600 messages, all of which are
|
||||
exactly the same thing.
|
||||
|
||||
The FSA is triggered at G_PRIORITY_HIGH, and once it is triggered, it
|
||||
will run until its queue is empty. With so many messages being added so
|
||||
quickly, we've basically ensured it won't be empty any time soon. While
|
||||
controld is processing the FSA messages, it will be unable to read
|
||||
anything out of the IPC backlog.
|
||||
|
||||
based continues to attempt to send IPC events to controld but is unable
|
||||
to do so, so the backlog continues to grow. Eventually, the backlog
|
||||
reaches that 500 message threshold without anything having been read by
|
||||
controld, which triggers the eviction process.
|
||||
|
||||
There doesn't seem to be any reason for all these I_PE_CALC messages to
|
||||
be generated. They're all exactly the same, and they don't appear to be
|
||||
tagged with any unique data tying them to a specific query, and their
|
||||
presence just slows everything down.
|
||||
|
||||
Thus, the fix here is very simple: if the latest message in the queue is
|
||||
an I_PE_CALC message, just don't add another one. We could also make
|
||||
sure there's only ever one I_PE_CALC message in the queue, but there
|
||||
could potentially be valid reasons for there to be multiple interleaved
|
||||
with other message types. I am erring on the side of caution with this
|
||||
minimal fix.
|
||||
|
||||
Related: RHEL-76276
|
||||
---
|
||||
daemons/controld/controld_messages.c | 20 ++++++++++++++++++++
|
||||
1 file changed, 20 insertions(+)
|
||||
|
||||
diff --git a/daemons/controld/controld_messages.c b/daemons/controld/controld_messages.c
|
||||
index 0b0f25b..30af707 100644
|
||||
--- a/daemons/controld/controld_messages.c
|
||||
+++ b/daemons/controld/controld_messages.c
|
||||
@@ -73,6 +73,26 @@ register_fsa_input_adv(enum crmd_fsa_cause cause, enum crmd_fsa_input input,
|
||||
return;
|
||||
}
|
||||
|
||||
+ if (input == I_PE_CALC) {
|
||||
+ GList *ele = NULL;
|
||||
+
|
||||
+ if (prepend) {
|
||||
+ ele = g_list_first(controld_globals.fsa_message_queue);
|
||||
+ } else {
|
||||
+ ele = g_list_last(controld_globals.fsa_message_queue);
|
||||
+ }
|
||||
+
|
||||
+ if (ele != NULL) {
|
||||
+ fsa_data_t *message = (fsa_data_t *) ele->data;
|
||||
+
|
||||
+ if (message->fsa_input == I_PE_CALC) {
|
||||
+ crm_debug("%s item in fsa queue is I_PE_CALC, not adding another",
|
||||
+ (prepend ? "First" : "Last"));
|
||||
+ return;
|
||||
+ }
|
||||
+ }
|
||||
+ }
|
||||
+
|
||||
if (input == I_WAIT_FOR_EVENT) {
|
||||
controld_set_global_flags(controld_fsa_is_stalled);
|
||||
crm_debug("Stalling the FSA pending further input: source=%s cause=%s data=%p queue=%d",
|
||||
--
|
||||
2.43.0
|
||||
|
||||
386
SOURCES/017-remote-overflow.patch
Normal file
386
SOURCES/017-remote-overflow.patch
Normal file
@ -0,0 +1,386 @@
|
||||
From 2d9c031078a83796dcbe4faa0bad3579c4fcd1c3 Mon Sep 17 00:00:00 2001
|
||||
From: Chris Lumens <clumens@redhat.com>
|
||||
Date: Thu, 14 May 2026 12:57:17 -0400
|
||||
Subject: [PATCH 1/5] Med: libcrmcommon: Add sanity checks to
|
||||
localized_remote_header.
|
||||
|
||||
These checks are present in the main and 3.0 branches, and they make
|
||||
sense here as well.
|
||||
---
|
||||
lib/common/remote.c | 52 ++++++++++++++++++++++++++++++++++++++++-----
|
||||
1 file changed, 47 insertions(+), 5 deletions(-)
|
||||
|
||||
diff --git a/lib/common/remote.c b/lib/common/remote.c
|
||||
index fe19296..f771f46 100644
|
||||
--- a/lib/common/remote.c
|
||||
+++ b/lib/common/remote.c
|
||||
@@ -1,5 +1,5 @@
|
||||
/*
|
||||
- * Copyright 2008-2023 the Pacemaker project contributors
|
||||
+ * Copyright 2008-2026 the Pacemaker project contributors
|
||||
*
|
||||
* The version control history for this file may have further details.
|
||||
*
|
||||
@@ -23,7 +23,7 @@
|
||||
#include <netdb.h>
|
||||
#include <stdlib.h>
|
||||
#include <errno.h>
|
||||
-#include <inttypes.h> // PRIx32
|
||||
+#include <inttypes.h> // PRIu32, PRIx32
|
||||
|
||||
#include <glib.h>
|
||||
#include <bzlib.h>
|
||||
@@ -97,11 +97,18 @@ struct remote_header_v0 {
|
||||
static struct remote_header_v0 *
|
||||
localized_remote_header(pcmk__remote_t *remote)
|
||||
{
|
||||
- struct remote_header_v0 *header = (struct remote_header_v0 *)remote->buffer;
|
||||
- if(remote->buffer_offset < sizeof(struct remote_header_v0)) {
|
||||
+ struct remote_header_v0 *header = NULL;
|
||||
+ size_t expected_size = 0;
|
||||
+
|
||||
+ if ((remote == NULL) || (remote->buffer == NULL)
|
||||
+ || (remote->buffer_offset < sizeof(struct remote_header_v0))) {
|
||||
+
|
||||
+ // Caller error or we haven't received the full header yet
|
||||
return NULL;
|
||||
+ }
|
||||
|
||||
- } else if(header->endian != ENDIAN_LOCAL) {
|
||||
+ header = (struct remote_header_v0 *) remote->buffer;
|
||||
+ if (header->endian != ENDIAN_LOCAL) {
|
||||
uint32_t endian = __swab32(header->endian);
|
||||
|
||||
CRM_LOG_ASSERT(endian == ENDIAN_LOCAL);
|
||||
@@ -123,6 +130,41 @@ localized_remote_header(pcmk__remote_t *remote)
|
||||
header->payload_uncompressed = __swab32(header->payload_uncompressed);
|
||||
}
|
||||
|
||||
+ // Sanity checks
|
||||
+ if (header->payload_offset != sizeof(struct remote_header_v0)) {
|
||||
+ crm_err("Header payload offset %" PRIu32 " does not have expected "
|
||||
+ "size %zu", header->payload_offset,
|
||||
+ sizeof(struct remote_header_v0));
|
||||
+ return NULL;
|
||||
+ }
|
||||
+
|
||||
+ if (header->payload_compressed != 0) {
|
||||
+ if (header->payload_compressed > (SIZE_MAX - header->payload_offset)) {
|
||||
+ crm_err("Header compressed size %" PRIu32 " is too large",
|
||||
+ header->payload_compressed);
|
||||
+ return NULL;
|
||||
+ }
|
||||
+
|
||||
+ expected_size = (size_t) header->payload_offset
|
||||
+ + header->payload_compressed;
|
||||
+
|
||||
+ } else {
|
||||
+ if (header->payload_uncompressed > (SIZE_MAX - header->payload_offset)) {
|
||||
+ crm_err("Header uncompressed size %" PRIu32 " is too large",
|
||||
+ header->payload_uncompressed);
|
||||
+ return NULL;
|
||||
+ }
|
||||
+
|
||||
+ expected_size = (size_t) header->payload_offset
|
||||
+ + header->payload_uncompressed;
|
||||
+ }
|
||||
+
|
||||
+ if (expected_size != header->size_total) {
|
||||
+ crm_err("Header total size %" PRIu32 " does not match calculated "
|
||||
+ "size %zu", header->size_total, expected_size);
|
||||
+ return NULL;
|
||||
+ }
|
||||
+
|
||||
return header;
|
||||
}
|
||||
|
||||
--
|
||||
2.53.0
|
||||
|
||||
From 43cadbc4ce424a881ec0b3258d65e4f94bda3175 Mon Sep 17 00:00:00 2001
|
||||
From: Chris Lumens <clumens@redhat.com>
|
||||
Date: Thu, 14 May 2026 13:06:57 -0400
|
||||
Subject: [PATCH 2/5] High: libcrmcommon: Fix integer overflow in remote
|
||||
message code.
|
||||
|
||||
It's possible for a client to send a specifically constructed header
|
||||
packet that will cause our size checks in pcmk__remote_message_xml to
|
||||
overflow, which could result in crashes (potentially leading to a node
|
||||
being fenced) or memory corruption problems.
|
||||
|
||||
Such a header would have to indicate that the payload is compressed,
|
||||
which nothing in pacemaker is doing. In fact, we haven't used any of
|
||||
the remote message compression code since it was introduced in 2013
|
||||
which explains why we haven't seen any problems with it. Thus, it would
|
||||
require a malicious client to cause this to happen.
|
||||
|
||||
It would also require remote CIB administration to be enabled via
|
||||
remote-clear-port or remote-tls-port, which we do not do by default.
|
||||
Additionally, if remote-tls-port is in use, it requires the client to
|
||||
complete the gnutls handshake procedure (pcmk__remote_message_xml is
|
||||
only called in cib_remote_msg after the handshake but before the client
|
||||
has authenticated).
|
||||
|
||||
Co-Authored-By: Reid Wahl <nrwahl@protonmail.com>
|
||||
---
|
||||
lib/common/remote.c | 50 +++++++++++++++++++++++++++++++++++++++------
|
||||
1 file changed, 44 insertions(+), 6 deletions(-)
|
||||
|
||||
diff --git a/lib/common/remote.c b/lib/common/remote.c
|
||||
index f771f46..54620c2 100644
|
||||
--- a/lib/common/remote.c
|
||||
+++ b/lib/common/remote.c
|
||||
@@ -593,15 +593,53 @@ pcmk__remote_message_xml(pcmk__remote_t *remote)
|
||||
}
|
||||
|
||||
/* Support compression on the receiving end now, in case we ever want to add it later */
|
||||
- if (header->payload_compressed) {
|
||||
+ if (header->payload_compressed != 0) {
|
||||
int rc = 0;
|
||||
- unsigned int size_u = 1 + header->payload_uncompressed;
|
||||
- char *uncompressed = calloc(1, header->payload_offset + size_u);
|
||||
+ unsigned int size_u = 0;
|
||||
+ char *uncompressed = NULL;
|
||||
+
|
||||
+#if (UINT32_MAX < UINT_MAX)
|
||||
+ if (header->payload_uncompressed >= UINT_MAX) {
|
||||
+ crm_err("Couldn't decompress message because uncompressed "
|
||||
+ "payload size (%" PRIu32 ") is greater than UINT_MAX "
|
||||
+ "(%u)", header->payload_uncompressed, UINT_MAX);
|
||||
+ return NULL;
|
||||
+ }
|
||||
+#endif
|
||||
|
||||
- crm_trace("Decompressing message data %d bytes into %d bytes",
|
||||
- header->payload_compressed, size_u);
|
||||
+ /* @TODO Is the extra byte for the null terminator?
|
||||
+ * pcmk__remote_send_xml() also adds one byte to the iov length.
|
||||
+ * (However, we do need to account for the possibility of receiving a
|
||||
+ * message from an untrusted sender.)
|
||||
+ */
|
||||
+ size_u = 1 + header->payload_uncompressed;
|
||||
+
|
||||
+ /* Header and uncompressed payload must fit in the destination buffer.
|
||||
+ * We do not need to separately check the header size here since
|
||||
+ * localized_remote_header will return NULL if it's incorrect.
|
||||
+ */
|
||||
+#if (UINT_MAX >= SIZE_MAX)
|
||||
+ if ((size_u >= SIZE_MAX)
|
||||
+ || (header->payload_offset > (SIZE_MAX - size_u))) {
|
||||
+#else
|
||||
+ if (header->payload_offset > (SIZE_MAX - size_u)) {
|
||||
+#endif
|
||||
+ crm_err("Couldn't decompress message because the required buffer "
|
||||
+ "size (%" PRIu32 " + %u) is greater than SIZE_MAX (%zu)",
|
||||
+ header->payload_offset, size_u, SIZE_MAX);
|
||||
+ return NULL;
|
||||
+ }
|
||||
+
|
||||
+ crm_trace("Decompressing message data %" PRIu32 " bytes into %u "
|
||||
+ "bytes", header->payload_compressed, size_u);
|
||||
+
|
||||
+ uncompressed = calloc(1, header->payload_offset + size_u);
|
||||
+ if (uncompressed == NULL) {
|
||||
+ return NULL;
|
||||
+ }
|
||||
|
||||
- rc = BZ2_bzBuffToBuffDecompress(uncompressed + header->payload_offset, &size_u,
|
||||
+ rc = BZ2_bzBuffToBuffDecompress(uncompressed + header->payload_offset,
|
||||
+ &size_u,
|
||||
remote->buffer + header->payload_offset,
|
||||
header->payload_compressed, 1, 0);
|
||||
rc = pcmk__bzlib2rc(rc);
|
||||
--
|
||||
2.53.0
|
||||
|
||||
From 07f44f9db114ea7f609029410712835c1256cd55 Mon Sep 17 00:00:00 2001
|
||||
From: Chris Lumens <clumens@redhat.com>
|
||||
Date: Thu, 14 May 2026 13:26:07 -0400
|
||||
Subject: [PATCH 3/5] High: libcrmcommon: Limit the max size of a remote
|
||||
message.
|
||||
|
||||
Previously, we were just taking the sizes straight from the message
|
||||
header and allocating however much memory was asked for without
|
||||
checking. It's possible for a malicious client to ask for a very large
|
||||
amount of memory (say, 4 GB since that'll fit in the uint32_t value for
|
||||
header->size_total).
|
||||
|
||||
In that case, due to the use of calloc, we'll crash due to an out of
|
||||
memory error and the node could potentially be fenced.
|
||||
|
||||
The fix is to simply check that the client isn't asking for too much
|
||||
memory. This does unfortunately impose a size restriction on the
|
||||
message. It's doubly unfortunate given how much work we've recently put
|
||||
into removing restrictions on IPC messages allowing for much larger
|
||||
clusters.
|
||||
|
||||
I think the biggest thing these messages will ever contain is a CIB, so
|
||||
for now I'm going with a limit of 20 MB. It is not my intention that
|
||||
this become a tuneable parameter that is exposed to the user.
|
||||
---
|
||||
include/crm/common/remote_internal.h | 5 ++++-
|
||||
lib/common/remote.c | 16 +++++++++++++++-
|
||||
2 files changed, 19 insertions(+), 2 deletions(-)
|
||||
|
||||
diff --git a/include/crm/common/remote_internal.h b/include/crm/common/remote_internal.h
|
||||
index 030c7a4..c39fcef 100644
|
||||
--- a/include/crm/common/remote_internal.h
|
||||
+++ b/include/crm/common/remote_internal.h
|
||||
@@ -1,5 +1,5 @@
|
||||
/*
|
||||
- * Copyright 2008-2023 the Pacemaker project contributors
|
||||
+ * Copyright 2008-2026 the Pacemaker project contributors
|
||||
*
|
||||
* The version control history for this file may have further details.
|
||||
*
|
||||
@@ -10,6 +10,9 @@
|
||||
#ifndef PCMK__REMOTE_INTERNAL__H
|
||||
# define PCMK__REMOTE_INTERNAL__H
|
||||
|
||||
+// The maximum payload size for a remote message (in bytes)
|
||||
+#define PCMK__REMOTE_MSG_MAX_SIZE (20 * 1024 * 1024)
|
||||
+
|
||||
// internal functions from remote.c
|
||||
|
||||
typedef struct pcmk__remote_s pcmk__remote_t;
|
||||
diff --git a/lib/common/remote.c b/lib/common/remote.c
|
||||
index 54620c2..256ff11 100644
|
||||
--- a/lib/common/remote.c
|
||||
+++ b/lib/common/remote.c
|
||||
@@ -597,6 +597,7 @@ pcmk__remote_message_xml(pcmk__remote_t *remote)
|
||||
int rc = 0;
|
||||
unsigned int size_u = 0;
|
||||
char *uncompressed = NULL;
|
||||
+ size_t buffer_size = 0;
|
||||
|
||||
#if (UINT32_MAX < UINT_MAX)
|
||||
if (header->payload_uncompressed >= UINT_MAX) {
|
||||
@@ -630,10 +631,17 @@ pcmk__remote_message_xml(pcmk__remote_t *remote)
|
||||
return NULL;
|
||||
}
|
||||
|
||||
+ buffer_size = (size_t) header->payload_offset + size_u;
|
||||
+ if (buffer_size > PCMK__REMOTE_MSG_MAX_SIZE) {
|
||||
+ crm_err("Message size %zu is larger than max allowed %u bytes",
|
||||
+ buffer_size, PCMK__REMOTE_MSG_MAX_SIZE);
|
||||
+ return NULL;
|
||||
+ }
|
||||
+
|
||||
crm_trace("Decompressing message data %" PRIu32 " bytes into %u "
|
||||
"bytes", header->payload_compressed, size_u);
|
||||
|
||||
- uncompressed = calloc(1, header->payload_offset + size_u);
|
||||
+ uncompressed = calloc(1, buffer_size);
|
||||
if (uncompressed == NULL) {
|
||||
return NULL;
|
||||
}
|
||||
@@ -780,6 +788,12 @@ read_available_remote_data(pcmk__remote_t *remote)
|
||||
read_len = header->size_total;
|
||||
}
|
||||
|
||||
+ if (read_len > PCMK__REMOTE_MSG_MAX_SIZE) {
|
||||
+ crm_err("Message size %zu is larger than max allowed %u bytes",
|
||||
+ read_len, PCMK__REMOTE_MSG_MAX_SIZE);
|
||||
+ return EINVAL;
|
||||
+ }
|
||||
+
|
||||
/* automatically grow the buffer when needed */
|
||||
if(remote->buffer_size < read_len) {
|
||||
remote->buffer_size = 2 * read_len;
|
||||
--
|
||||
2.53.0
|
||||
|
||||
From 3d9bf582dc28b94d59037ac32a3b4ee6498a3e35 Mon Sep 17 00:00:00 2001
|
||||
From: Chris Lumens <clumens@redhat.com>
|
||||
Date: Fri, 15 May 2026 10:19:33 -0400
|
||||
Subject: [PATCH 4/5] High: libcrmcommon: Fix an integer overflow in
|
||||
pcmk__remote_send_xml.
|
||||
|
||||
iov[0].iov_len is a size_t. iov[1].iov_len is also a size_t (which
|
||||
comes from the length of a string, which is a size_t). We're adding
|
||||
those together and storing them in a uint32_t, so make sure the result
|
||||
will fit.
|
||||
|
||||
Fixes RHEL-181155
|
||||
---
|
||||
lib/common/remote.c | 8 ++++++++
|
||||
1 file changed, 8 insertions(+)
|
||||
|
||||
diff --git a/lib/common/remote.c b/lib/common/remote.c
|
||||
index 256ff11..26f1917 100644
|
||||
--- a/lib/common/remote.c
|
||||
+++ b/lib/common/remote.c
|
||||
@@ -560,6 +560,13 @@ pcmk__remote_send_xml(pcmk__remote_t *remote, const xmlNode *msg)
|
||||
header->version = REMOTE_MSG_VERSION;
|
||||
header->payload_offset = iov[0].iov_len;
|
||||
header->payload_uncompressed = iov[1].iov_len;
|
||||
+
|
||||
+ if ((UINT32_MAX - iov[0].iov_len) < iov[1].iov_len) {
|
||||
+ crm_err("Remote message size %zu + %zu exceeds maximum of %" PRIu32,
|
||||
+ iov[0].iov_len, iov[1].iov_len, UINT32_MAX);
|
||||
+ goto done;
|
||||
+ }
|
||||
+
|
||||
header->size_total = iov[0].iov_len + iov[1].iov_len;
|
||||
|
||||
rc = remote_send_iovs(remote, iov, 2);
|
||||
@@ -568,6 +575,7 @@ pcmk__remote_send_xml(pcmk__remote_t *remote, const xmlNode *msg)
|
||||
pcmk_rc_str(rc), rc);
|
||||
}
|
||||
|
||||
+done:
|
||||
free(iov[0].iov_base);
|
||||
free(iov[1].iov_base);
|
||||
return rc;
|
||||
--
|
||||
2.53.0
|
||||
|
||||
From d022a58012629a5d4ecc84dc5d2f325f01738adc Mon Sep 17 00:00:00 2001
|
||||
From: Chris Lumens <clumens@redhat.com>
|
||||
Date: Wed, 17 Jun 2026 15:55:28 -0400
|
||||
Subject: [PATCH 5/5] Med: libcrmcommon: Add additional checks to
|
||||
pcmk__remote_message_xml.
|
||||
|
||||
coverity flags the existing code for using a tainted expression as an
|
||||
index to a pointer. While we're fixing other range errors and overflows
|
||||
in this code, we should fix this one as well by porting the existing
|
||||
code from the main branch.
|
||||
---
|
||||
lib/common/remote.c | 14 +++++++++++++-
|
||||
1 file changed, 13 insertions(+), 1 deletion(-)
|
||||
|
||||
diff --git a/lib/common/remote.c b/lib/common/remote.c
|
||||
index 26f1917..fbcf0a0 100644
|
||||
--- a/lib/common/remote.c
|
||||
+++ b/lib/common/remote.c
|
||||
@@ -594,6 +594,7 @@ xmlNode *
|
||||
pcmk__remote_message_xml(pcmk__remote_t *remote)
|
||||
{
|
||||
xmlNode *xml = NULL;
|
||||
+ size_t data_size = 0;
|
||||
struct remote_header_v0 *header = localized_remote_header(remote);
|
||||
|
||||
if (header == NULL) {
|
||||
@@ -686,7 +687,18 @@ pcmk__remote_message_xml(pcmk__remote_t *remote)
|
||||
/* take ownership of the buffer */
|
||||
remote->buffer_offset = 0;
|
||||
|
||||
- CRM_LOG_ASSERT(remote->buffer[sizeof(struct remote_header_v0) + header->payload_uncompressed - 1] == 0);
|
||||
+ data_size = (size_t) header->payload_offset + header->payload_uncompressed;
|
||||
+
|
||||
+ // Ensure the buffer is as big as it should be
|
||||
+ CRM_CHECK(remote->buffer_size >= data_size, return NULL);
|
||||
+
|
||||
+ /* Ensure the buffer is null-terminated (see
|
||||
+ * pcmk__read_available_remote_data()).
|
||||
+ *
|
||||
+ * Note that payload_uncompressed contains the payload size including the
|
||||
+ * null byte (see pcmk__remote_send_xml()).
|
||||
+ */
|
||||
+ CRM_CHECK(remote->buffer[data_size] == '\0', return NULL);
|
||||
|
||||
xml = string2xml(remote->buffer + header->payload_offset);
|
||||
if (xml == NULL && header->version > REMOTE_MSG_VERSION) {
|
||||
--
|
||||
2.53.0
|
||||
|
||||
@ -244,7 +244,7 @@
|
||||
Name: pacemaker
|
||||
Summary: Scalable High-Availability cluster resource manager
|
||||
Version: %{pcmkversion}
|
||||
Release: %{pcmk_release}.3%{?dist}
|
||||
Release: %{pcmk_release}.6%{?dist}
|
||||
%if %{defined _unitdir}
|
||||
License: GPL-2.0-or-later AND LGPL-2.1-or-later
|
||||
%else
|
||||
@ -279,6 +279,9 @@ Patch011: 011-attrd-memory-leak.patch
|
||||
Patch012: 012-dont-set-as-xml-id.patch
|
||||
Patch013: 013-crm_node-i-initialize.patch
|
||||
Patch014: 014-remote-fencing.patch
|
||||
Patch015: 015-ipc-disconnect.patch
|
||||
Patch016: 016-fewer-messages.patch
|
||||
Patch017: 017-remote-overflow.patch
|
||||
|
||||
Requires: resource-agents
|
||||
Requires: %{pkgname_pcmk_libs}%{?_isa} = %{version}-%{release}
|
||||
@ -1030,6 +1033,18 @@ exit 0
|
||||
%license %{nagios_name}-%{nagios_hash}/COPYING
|
||||
|
||||
%changelog
|
||||
* Tue Jun 30 2026 Chris Lumens <clumens@redhat.com> - 2.1.7-5.6
|
||||
- Fix integer overflows in remote message decompression code (CVE-2026-10649)
|
||||
- Resolves: RHEL-181157
|
||||
|
||||
* Mon Nov 17 2025 Chris Lumens <clumens@redhat.com> - 2.1.7-5.5
|
||||
- Don't overwhelm the FSA queue with repeated CIB queries
|
||||
- Related: RHEL-76276
|
||||
|
||||
* Tue Sep 30 2025 Chris Lumens <clumens@redhat.com> - 2.1.7-5.4
|
||||
- Be more lenient in evicting IPC clients
|
||||
- Resolves: RHEL-76276
|
||||
|
||||
* Thu Jul 10 2025 Chris Lumens <clumens@redhat.com> - 2.1.7-5.3
|
||||
- Add option for controlling remote node fencing behavior
|
||||
- Resolves: RHEL-93220
|
||||
|
||||
Loading…
Reference in New Issue
Block a user