diff --git a/.gitignore b/.gitignore index 2a1c896..2802d2a 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,2 @@ SOURCES/nagios-agents-metadata-105ab8a7b2c16b9a29cf1c1596b80136eeef332b.tar.gz -SOURCES/pacemaker-6fdc9deea.tar.gz +SOURCES/pacemaker-0f7f88312.tar.gz diff --git a/.pacemaker.metadata b/.pacemaker.metadata index d1dbf02..8db4d95 100644 --- a/.pacemaker.metadata +++ b/.pacemaker.metadata @@ -1,2 +1,2 @@ 2cbec94ad67dfbeba75e38d2c3c5c44961b3cd16 SOURCES/nagios-agents-metadata-105ab8a7b2c16b9a29cf1c1596b80136eeef332b.tar.gz -fbf71fb3fb42c76f9f1e98497505eb8521cab55e SOURCES/pacemaker-6fdc9deea.tar.gz +88946a460e3be18852861269f8837aaaf339328c SOURCES/pacemaker-0f7f88312.tar.gz diff --git a/SOURCES/001-remote-start-state.patch b/SOURCES/001-remote-start-state.patch deleted file mode 100644 index e66bf16..0000000 --- a/SOURCES/001-remote-start-state.patch +++ /dev/null @@ -1,402 +0,0 @@ -From cf53f523e691295879cd75cff1a86bc15664fa51 Mon Sep 17 00:00:00 2001 -From: Chris Lumens -Date: Tue, 2 May 2023 09:59:13 -0400 -Subject: [PATCH 1/7] Feature: daemons: Add start state to LRMD handshake XML - -This gets read out of /etc/sysconfig/pacemaker and set into the -environment. The remote node executor will then add that to the XML -that it sends to the controller upon startup. - -Ref T183 ---- - daemons/execd/execd_commands.c | 5 +++++ - include/crm_internal.h | 1 + - 2 files changed, 6 insertions(+) - -diff --git a/daemons/execd/execd_commands.c b/daemons/execd/execd_commands.c -index fa2761e..9a783a5 100644 ---- a/daemons/execd/execd_commands.c -+++ b/daemons/execd/execd_commands.c -@@ -1474,6 +1474,7 @@ process_lrmd_signon(pcmk__client_t *client, xmlNode *request, int call_id, - int rc = pcmk_ok; - time_t now = time(NULL); - const char *protocol_version = crm_element_value(request, F_LRMD_PROTOCOL_VERSION); -+ const char *start_state = pcmk__env_option(PCMK__ENV_NODE_START_STATE); - - if (compare_version(protocol_version, LRMD_MIN_PROTOCOL_VERSION) < 0) { - crm_err("Cluster API version must be greater than or equal to %s, not %s", -@@ -1503,6 +1504,10 @@ process_lrmd_signon(pcmk__client_t *client, xmlNode *request, int call_id, - crm_xml_add(*reply, F_LRMD_PROTOCOL_VERSION, LRMD_PROTOCOL_VERSION); - crm_xml_add_ll(*reply, PCMK__XA_UPTIME, now - start_time); - -+ if (start_state) { -+ crm_xml_add(*reply, PCMK__XA_NODE_START_STATE, start_state); -+ } -+ - return rc; - } - -diff --git a/include/crm_internal.h b/include/crm_internal.h -index 5f6531f..771bd26 100644 ---- a/include/crm_internal.h -+++ b/include/crm_internal.h -@@ -84,6 +84,7 @@ - #define PCMK__XA_GRAPH_ERRORS "graph-errors" - #define PCMK__XA_GRAPH_WARNINGS "graph-warnings" - #define PCMK__XA_MODE "mode" -+#define PCMK__XA_NODE_START_STATE "node_start_state" - #define PCMK__XA_TASK "task" - #define PCMK__XA_UPTIME "uptime" - #define PCMK__XA_CONN_HOST "connection_host" --- -2.31.1 - -From c950291742711b5c4c8986adc8e938fe6fef861c Mon Sep 17 00:00:00 2001 -From: Chris Lumens -Date: Tue, 2 May 2023 10:04:32 -0400 -Subject: [PATCH 2/7] Feature: liblrmd: Save a remote node's requested start - state - -Ref T183 ---- - include/crm/common/ipc_internal.h | 1 + - lib/lrmd/lrmd_client.c | 7 +++++++ - 2 files changed, 8 insertions(+) - -diff --git a/include/crm/common/ipc_internal.h b/include/crm/common/ipc_internal.h -index 5099dda..d203924 100644 ---- a/include/crm/common/ipc_internal.h -+++ b/include/crm/common/ipc_internal.h -@@ -112,6 +112,7 @@ struct pcmk__remote_s { - int tcp_socket; - mainloop_io_t *source; - time_t uptime; -+ char *start_state; - - /* CIB-only */ - char *token; -diff --git a/lib/lrmd/lrmd_client.c b/lib/lrmd/lrmd_client.c -index c565728..4239105 100644 ---- a/lib/lrmd/lrmd_client.c -+++ b/lib/lrmd/lrmd_client.c -@@ -588,7 +588,9 @@ lrmd_tls_connection_destroy(gpointer userdata) - } - - free(native->remote->buffer); -+ free(native->remote->start_state); - native->remote->buffer = NULL; -+ native->remote->start_state = NULL; - native->source = 0; - native->sock = 0; - native->psk_cred_c = NULL; -@@ -980,6 +982,7 @@ lrmd_handshake(lrmd_t * lrmd, const char *name) - const char *version = crm_element_value(reply, F_LRMD_PROTOCOL_VERSION); - const char *msg_type = crm_element_value(reply, F_LRMD_OPERATION); - const char *tmp_ticket = crm_element_value(reply, F_LRMD_CLIENTID); -+ const char *start_state = crm_element_value(reply, PCMK__XA_NODE_START_STATE); - long long uptime = -1; - - crm_element_value_int(reply, F_LRMD_RC, &rc); -@@ -992,6 +995,10 @@ lrmd_handshake(lrmd_t * lrmd, const char *name) - crm_element_value_ll(reply, PCMK__XA_UPTIME, &uptime); - native->remote->uptime = uptime; - -+ if (start_state) { -+ native->remote->start_state = strdup(start_state); -+ } -+ - if (rc == -EPROTO) { - crm_err("Executor protocol version mismatch between client (%s) and server (%s)", - LRMD_PROTOCOL_VERSION, version); --- -2.31.1 - -From 7302014c7b7296be31b1f542b3f107d55b1fb2a0 Mon Sep 17 00:00:00 2001 -From: Chris Lumens -Date: Tue, 2 May 2023 10:05:13 -0400 -Subject: [PATCH 3/7] Feature: liblrmd: Add lrmd__node_start_state. - -This function is used to get the start state out of an lrmd_private_t -structure. - -Ref T183 ---- - include/crm/lrmd_internal.h | 1 + - lib/lrmd/lrmd_client.c | 12 ++++++++++++ - 2 files changed, 13 insertions(+) - -diff --git a/include/crm/lrmd_internal.h b/include/crm/lrmd_internal.h -index 5810554..d1cd25d 100644 ---- a/include/crm/lrmd_internal.h -+++ b/include/crm/lrmd_internal.h -@@ -47,6 +47,7 @@ void lrmd__set_result(lrmd_event_data_t *event, enum ocf_exitcode rc, - void lrmd__reset_result(lrmd_event_data_t *event); - - time_t lrmd__uptime(lrmd_t *lrmd); -+const char *lrmd__node_start_state(lrmd_t *lrmd); - - /* Shared functions for IPC proxy back end */ - -diff --git a/lib/lrmd/lrmd_client.c b/lib/lrmd/lrmd_client.c -index 4239105..82434b9 100644 ---- a/lib/lrmd/lrmd_client.c -+++ b/lib/lrmd/lrmd_client.c -@@ -2538,3 +2538,15 @@ lrmd__uptime(lrmd_t *lrmd) - return native->remote->uptime; - } - } -+ -+const char * -+lrmd__node_start_state(lrmd_t *lrmd) -+{ -+ lrmd_private_t *native = lrmd->lrmd_private; -+ -+ if (native->remote == NULL) { -+ return NULL; -+ } else { -+ return native->remote->start_state; -+ } -+} --- -2.31.1 - -From e5e4d43f847da0930bae12f63c7e9d9c44c07cdf Mon Sep 17 00:00:00 2001 -From: Chris Lumens -Date: Tue, 2 May 2023 10:07:58 -0400 -Subject: [PATCH 4/7] Refactor: controller: Make set_join_state a public - function. - -This already does all the work of setting a node's start state. It just -needs to be made public and given arguments for what node to set instead -of reading globals. - -Ref T183 ---- - daemons/controld/controld_join_client.c | 20 ++++++++++---------- - daemons/controld/pacemaker-controld.h | 3 +++ - 2 files changed, 13 insertions(+), 10 deletions(-) - -diff --git a/daemons/controld/controld_join_client.c b/daemons/controld/controld_join_client.c -index da6a9d6..07e2a27 100644 ---- a/daemons/controld/controld_join_client.c -+++ b/daemons/controld/controld_join_client.c -@@ -195,32 +195,31 @@ join_query_callback(xmlNode * msg, int call_id, int rc, xmlNode * output, void * - free_xml(generation); - } - --static void --set_join_state(const char * start_state) -+void -+set_join_state(const char *start_state, const char *node_name, const char *node_uuid) - { - if (pcmk__str_eq(start_state, "standby", pcmk__str_casei)) { - crm_notice("Forcing node %s to join in %s state per configured " -- "environment", controld_globals.our_nodename, start_state); -+ "environment", node_name, start_state); - cib__update_node_attr(controld_globals.logger_out, - controld_globals.cib_conn, cib_sync_call, -- XML_CIB_TAG_NODES, controld_globals.our_uuid, -+ XML_CIB_TAG_NODES, node_uuid, - NULL, NULL, NULL, "standby", "on", NULL, NULL); - - } else if (pcmk__str_eq(start_state, "online", pcmk__str_casei)) { - crm_notice("Forcing node %s to join in %s state per configured " -- "environment", controld_globals.our_nodename, start_state); -+ "environment", node_name, start_state); - cib__update_node_attr(controld_globals.logger_out, - controld_globals.cib_conn, cib_sync_call, -- XML_CIB_TAG_NODES, controld_globals.our_uuid, -+ XML_CIB_TAG_NODES, node_uuid, - NULL, NULL, NULL, "standby", "off", NULL, NULL); - - } else if (pcmk__str_eq(start_state, "default", pcmk__str_casei)) { -- crm_debug("Not forcing a starting state on node %s", -- controld_globals.our_nodename); -+ crm_debug("Not forcing a starting state on node %s", node_name); - - } else { - crm_warn("Unrecognized start state '%s', using 'default' (%s)", -- start_state, controld_globals.our_nodename); -+ start_state, node_name); - } - } - -@@ -335,7 +334,8 @@ do_cl_join_finalize_respond(long long action, - - first_join = FALSE; - if (start_state) { -- set_join_state(start_state); -+ set_join_state(start_state, controld_globals.our_nodename, -+ controld_globals.our_uuid); - } - } - -diff --git a/daemons/controld/pacemaker-controld.h b/daemons/controld/pacemaker-controld.h -index 1484a00..d8c2ddd 100644 ---- a/daemons/controld/pacemaker-controld.h -+++ b/daemons/controld/pacemaker-controld.h -@@ -36,4 +36,7 @@ void controld_remove_voter(const char *uname); - void controld_election_fini(void); - void controld_stop_current_election_timeout(void); - -+void set_join_state(const char *start_state, const char *node_name, -+ const char *node_uuid); -+ - #endif --- -2.31.1 - -From 63d069adb344bba2c982013226f87dfd95afaff3 Mon Sep 17 00:00:00 2001 -From: Chris Lumens -Date: Tue, 2 May 2023 13:38:03 -0400 -Subject: [PATCH 5/7] Refactor: controller: set_join_state needs to take a - remote parameter. - -Without this parameter, we won't know what to pass to as node_type to -cib__update_node_attr. And without that, that function will not know to -update a remote node - it'll try to update a regular node by the same -name, which either doesn't exist or is not what we were hoping would -happen. - -Ref T138 ---- - daemons/controld/controld_join_client.c | 11 +++++++---- - daemons/controld/pacemaker-controld.h | 2 +- - 2 files changed, 8 insertions(+), 5 deletions(-) - -diff --git a/daemons/controld/controld_join_client.c b/daemons/controld/controld_join_client.c -index 07e2a27..799d1b4 100644 ---- a/daemons/controld/controld_join_client.c -+++ b/daemons/controld/controld_join_client.c -@@ -196,7 +196,8 @@ join_query_callback(xmlNode * msg, int call_id, int rc, xmlNode * output, void * - } - - void --set_join_state(const char *start_state, const char *node_name, const char *node_uuid) -+set_join_state(const char *start_state, const char *node_name, const char *node_uuid, -+ bool remote) - { - if (pcmk__str_eq(start_state, "standby", pcmk__str_casei)) { - crm_notice("Forcing node %s to join in %s state per configured " -@@ -204,7 +205,8 @@ set_join_state(const char *start_state, const char *node_name, const char *node_ - cib__update_node_attr(controld_globals.logger_out, - controld_globals.cib_conn, cib_sync_call, - XML_CIB_TAG_NODES, node_uuid, -- NULL, NULL, NULL, "standby", "on", NULL, NULL); -+ NULL, NULL, NULL, "standby", "on", NULL, -+ remote ? "remote" : NULL); - - } else if (pcmk__str_eq(start_state, "online", pcmk__str_casei)) { - crm_notice("Forcing node %s to join in %s state per configured " -@@ -212,7 +214,8 @@ set_join_state(const char *start_state, const char *node_name, const char *node_ - cib__update_node_attr(controld_globals.logger_out, - controld_globals.cib_conn, cib_sync_call, - XML_CIB_TAG_NODES, node_uuid, -- NULL, NULL, NULL, "standby", "off", NULL, NULL); -+ NULL, NULL, NULL, "standby", "off", NULL, -+ remote ? "remote" : NULL); - - } else if (pcmk__str_eq(start_state, "default", pcmk__str_casei)) { - crm_debug("Not forcing a starting state on node %s", node_name); -@@ -335,7 +338,7 @@ do_cl_join_finalize_respond(long long action, - first_join = FALSE; - if (start_state) { - set_join_state(start_state, controld_globals.our_nodename, -- controld_globals.our_uuid); -+ controld_globals.our_uuid, false); - } - } - -diff --git a/daemons/controld/pacemaker-controld.h b/daemons/controld/pacemaker-controld.h -index d8c2ddd..2334cce 100644 ---- a/daemons/controld/pacemaker-controld.h -+++ b/daemons/controld/pacemaker-controld.h -@@ -37,6 +37,6 @@ void controld_election_fini(void); - void controld_stop_current_election_timeout(void); - - void set_join_state(const char *start_state, const char *node_name, -- const char *node_uuid); -+ const char *node_uuid, bool remote); - - #endif --- -2.31.1 - -From 67274787898355065315f8c06d62458e2c2b0afe Mon Sep 17 00:00:00 2001 -From: Chris Lumens -Date: Tue, 2 May 2023 10:09:02 -0400 -Subject: [PATCH 6/7] Feature: controller: When a remote node starts, apply any - start state. - -If we were given a start state in the handshake XML, that is now stored -in the remote node cache's private data. Extract it and set the state -on the node with set_node_state. - -Fixes T183 ---- - daemons/controld/controld_remote_ra.c | 15 +++++++++++++++ - 1 file changed, 15 insertions(+) - -diff --git a/daemons/controld/controld_remote_ra.c b/daemons/controld/controld_remote_ra.c -index f24b755..8ab1e46 100644 ---- a/daemons/controld/controld_remote_ra.c -+++ b/daemons/controld/controld_remote_ra.c -@@ -280,6 +280,7 @@ remote_node_up(const char *node_name) - int call_opt; - xmlNode *update, *state; - crm_node_t *node; -+ lrm_state_t *connection_rsc = NULL; - - CRM_CHECK(node_name != NULL, return); - crm_info("Announcing Pacemaker Remote node %s", node_name); -@@ -301,6 +302,20 @@ remote_node_up(const char *node_name) - purge_remote_node_attrs(call_opt, node); - pcmk__update_peer_state(__func__, node, CRM_NODE_MEMBER, 0); - -+ /* Apply any start state that we were given from the environment on the -+ * remote node. -+ */ -+ connection_rsc = lrm_state_find(node->uname); -+ -+ if (connection_rsc != NULL) { -+ lrmd_t *lrm = connection_rsc->conn; -+ const char *start_state = lrmd__node_start_state(lrm); -+ -+ if (start_state) { -+ set_join_state(start_state, node->uname, node->uuid, true); -+ } -+ } -+ - /* pacemaker_remote nodes don't participate in the membership layer, - * so cluster nodes don't automatically get notified when they come and go. - * We send a cluster message to the DC, and update the CIB node state entry, --- -2.31.1 - -From 91cdda7056c9b9254a0d7e7a016b30f788e3e3ff Mon Sep 17 00:00:00 2001 -From: Chris Lumens -Date: Tue, 2 May 2023 10:16:30 -0400 -Subject: [PATCH 7/7] Doc: sysconfig: Remote nodes now respect start state. - -Ref T183 ---- - etc/sysconfig/pacemaker.in | 3 +-- - 1 file changed, 1 insertion(+), 2 deletions(-) - -diff --git a/etc/sysconfig/pacemaker.in b/etc/sysconfig/pacemaker.in -index 3b03ad6..041da71 100644 ---- a/etc/sysconfig/pacemaker.in -+++ b/etc/sysconfig/pacemaker.in -@@ -144,8 +144,7 @@ - # By default, the local host will join the cluster in an online or standby - # state when Pacemaker first starts depending on whether it was previously put - # into standby mode. If this variable is set to "standby" or "online", it will --# force the local host to join in the specified state. This has no effect on --# Pacemaker Remote nodes. -+# force the local host to join in the specified state. - # - # Default: PCMK_node_start_state="default" - --- -2.31.1 - diff --git a/SOURCES/001-schema-glib.patch b/SOURCES/001-schema-glib.patch new file mode 100644 index 0000000..c38d1d8 --- /dev/null +++ b/SOURCES/001-schema-glib.patch @@ -0,0 +1,2334 @@ +From a59d703de97a49a27564f572dac52b455b356ba9 Mon Sep 17 00:00:00 2001 +From: Chris Lumens +Date: Wed, 25 Oct 2023 12:14:57 -0400 +Subject: [PATCH 01/20] Refactor: libcrmcommon: Remove prototypes for internal + functions. + +These are only here to give a place to put the G_GNUC_PRINTF attribute, +but that can go in the function definition itself. +--- + lib/common/schemas.c | 12 ++---------- + 1 file changed, 2 insertions(+), 10 deletions(-) + +diff --git a/lib/common/schemas.c b/lib/common/schemas.c +index b3c09eb..a85438c 100644 +--- a/lib/common/schemas.c ++++ b/lib/common/schemas.c +@@ -64,11 +64,7 @@ static struct schema_s *known_schemas = NULL; + static int xml_schema_max = 0; + static bool silent_logging = FALSE; + +-static void +-xml_log(int priority, const char *fmt, ...) +-G_GNUC_PRINTF(2, 3); +- +-static void ++static void G_GNUC_PRINTF(2, 3) + xml_log(int priority, const char *fmt, ...) + { + va_list ap; +@@ -716,10 +712,6 @@ pcmk__validate_xml(xmlNode *xml_blob, const char *validation, xmlRelaxNGValidity + return FALSE; + } + +-static void +-cib_upgrade_err(void *ctx, const char *fmt, ...) +-G_GNUC_PRINTF(2, 3); +- + /* With this arrangement, an attempt to identify the message severity + as explicitly signalled directly from XSLT is performed in rather + a smart way (no reliance on formatting string + arguments being +@@ -743,7 +735,7 @@ G_GNUC_PRINTF(2, 3); + (suspicious, likely internal errors or some runaways) is + LOG_WARNING. + */ +-static void ++static void G_GNUC_PRINTF(2, 3) + cib_upgrade_err(void *ctx, const char *fmt, ...) + { + va_list ap, aq; +-- +2.31.1 + +From 3d50aeebce74e520606036ec7db8b5c70fe327b5 Mon Sep 17 00:00:00 2001 +From: Chris Lumens +Date: Wed, 25 Oct 2023 12:54:37 -0400 +Subject: [PATCH 02/20] Refactor: libcrmcommon: validate_with should take a + schema as argument. + +...instead of taking an index, and then finding that in the +known_schemas array. +--- + lib/common/schemas.c | 25 ++++++++++++------------- + 1 file changed, 12 insertions(+), 13 deletions(-) + +diff --git a/lib/common/schemas.c b/lib/common/schemas.c +index a85438c..f1f86f4 100644 +--- a/lib/common/schemas.c ++++ b/lib/common/schemas.c +@@ -555,18 +555,16 @@ crm_schema_cleanup(void) + } + + static gboolean +-validate_with(xmlNode *xml, int method, xmlRelaxNGValidityErrorFunc error_handler, void* error_handler_context) ++validate_with(xmlNode *xml, struct schema_s *schema, xmlRelaxNGValidityErrorFunc error_handler, void* error_handler_context) + { + gboolean valid = FALSE; + char *file = NULL; +- struct schema_s *schema = NULL; + relaxng_ctx_cache_t **cache = NULL; + +- if (method < 0) { ++ if (schema == NULL) { + return FALSE; + } + +- schema = &(known_schemas[method]); + if (schema->validator == schema_validator_none) { + return TRUE; + } +@@ -587,8 +585,7 @@ validate_with(xmlNode *xml, int method, xmlRelaxNGValidityErrorFunc error_handle + valid = validate_with_relaxng(xml->doc, error_handler, error_handler_context, file, cache); + break; + default: +- crm_err("Unknown validator type: %d", +- known_schemas[method].validator); ++ crm_err("Unknown validator type: %d", schema->validator); + break; + } + +@@ -597,11 +594,11 @@ validate_with(xmlNode *xml, int method, xmlRelaxNGValidityErrorFunc error_handle + } + + static bool +-validate_with_silent(xmlNode *xml, int method) ++validate_with_silent(xmlNode *xml, struct schema_s *schema) + { + bool rc, sl_backup = silent_logging; + silent_logging = TRUE; +- rc = validate_with(xml, method, (xmlRelaxNGValidityErrorFunc) xml_log, GUINT_TO_POINTER(LOG_ERR)); ++ rc = validate_with(xml, schema, (xmlRelaxNGValidityErrorFunc) xml_log, GUINT_TO_POINTER(LOG_ERR)); + silent_logging = sl_backup; + return rc; + } +@@ -687,7 +684,7 @@ pcmk__validate_xml(xmlNode *xml_blob, const char *validation, xmlRelaxNGValidity + bool valid = FALSE; + + for (lpc = 0; lpc < xml_schema_max; lpc++) { +- if (validate_with(xml_blob, lpc, NULL, NULL)) { ++ if (validate_with(xml_blob, &known_schemas[lpc], NULL, NULL)) { + valid = TRUE; + crm_xml_add(xml_blob, XML_ATTR_VALIDATION, + known_schemas[lpc].name); +@@ -705,7 +702,8 @@ pcmk__validate_xml(xmlNode *xml_blob, const char *validation, xmlRelaxNGValidity + if (strcmp(validation, PCMK__VALUE_NONE) == 0) { + return TRUE; + } else if (version < xml_schema_max) { +- return validate_with(xml_blob, version, error_handler, error_handler_context); ++ return validate_with(xml_blob, version >= 0 ? &known_schemas[version] : NULL, ++ error_handler, error_handler_context); + } + + crm_err("Unknown validator: %s", validation); +@@ -1019,7 +1017,7 @@ update_validation(xmlNode **xml_blob, int *best, int max, gboolean transform, + known_schemas[lpc].name ? known_schemas[lpc].name : "", + lpc, max_stable_schemas); + +- if (validate_with(xml, lpc, error_handler, GUINT_TO_POINTER(LOG_ERR)) == FALSE) { ++ if (validate_with(xml, &known_schemas[lpc], error_handler, GUINT_TO_POINTER(LOG_ERR)) == FALSE) { + if (next != -1) { + crm_info("Configuration not valid for schema: %s", + known_schemas[lpc].name); +@@ -1067,7 +1065,7 @@ update_validation(xmlNode **xml_blob, int *best, int max, gboolean transform, + version boundary, as X.0 "transitional" version is + expected to be more strict than it's successors that + may re-allow constructs from previous major line) */ +- || validate_with_silent(xml, next)) { ++ || validate_with_silent(xml, next >= 0 ? &known_schemas[next] : NULL)) { + crm_debug("%s-style configuration is also valid for %s", + known_schemas[lpc].name, known_schemas[next].name); + +@@ -1084,7 +1082,8 @@ update_validation(xmlNode **xml_blob, int *best, int max, gboolean transform, + known_schemas[lpc].transform); + rc = -pcmk_err_transform_failed; + +- } else if (validate_with(upgrade, next, error_handler, GUINT_TO_POINTER(LOG_ERR))) { ++ } else if (validate_with(upgrade, next >= 0 ? &known_schemas[next] : NULL, ++ error_handler, GUINT_TO_POINTER(LOG_ERR))) { + crm_info("Transformation %s.xsl successful", + known_schemas[lpc].transform); + lpc = next; +-- +2.31.1 + +From 26a17af650a842659f57c9e58185c290c30a3fb3 Mon Sep 17 00:00:00 2001 +From: Chris Lumens +Date: Wed, 25 Oct 2023 14:33:11 -0400 +Subject: [PATCH 03/20] Refactor: libcrmcommon: Break schema freeing out into a + function. + +--- + lib/common/schemas.c | 69 +++++++++++++++++++++++++------------------- + 1 file changed, 40 insertions(+), 29 deletions(-) + +diff --git a/lib/common/schemas.c b/lib/common/schemas.c +index f1f86f4..c21b9ae 100644 +--- a/lib/common/schemas.c ++++ b/lib/common/schemas.c +@@ -511,6 +511,43 @@ validate_with_relaxng(xmlDocPtr doc, xmlRelaxNGValidityErrorFunc error_handler, + return valid; + } + ++static void ++free_schema(struct schema_s *schema) ++{ ++ relaxng_ctx_cache_t *ctx = NULL; ++ ++ switch (schema->validator) { ++ case schema_validator_none: // not cached ++ break; ++ ++ case schema_validator_rng: // cached ++ ctx = (relaxng_ctx_cache_t *) schema->cache; ++ if (ctx == NULL) { ++ break; ++ } ++ ++ if (ctx->parser != NULL) { ++ xmlRelaxNGFreeParserCtxt(ctx->parser); ++ } ++ ++ if (ctx->valid != NULL) { ++ xmlRelaxNGFreeValidCtxt(ctx->valid); ++ } ++ ++ if (ctx->rng != NULL) { ++ xmlRelaxNGFree(ctx->rng); ++ } ++ ++ free(ctx); ++ schema->cache = NULL; ++ break; ++ } ++ ++ free(schema->name); ++ free(schema->transform); ++ free(schema->transform_enter); ++} ++ + /*! + * \internal + * \brief Clean up global memory associated with XML schemas +@@ -518,36 +555,10 @@ validate_with_relaxng(xmlDocPtr doc, xmlRelaxNGValidityErrorFunc error_handler, + void + crm_schema_cleanup(void) + { +- int lpc; +- relaxng_ctx_cache_t *ctx = NULL; +- +- for (lpc = 0; lpc < xml_schema_max; lpc++) { +- +- switch (known_schemas[lpc].validator) { +- case schema_validator_none: // not cached +- break; +- case schema_validator_rng: // cached +- ctx = (relaxng_ctx_cache_t *) known_schemas[lpc].cache; +- if (ctx == NULL) { +- break; +- } +- if (ctx->parser != NULL) { +- xmlRelaxNGFreeParserCtxt(ctx->parser); +- } +- if (ctx->valid != NULL) { +- xmlRelaxNGFreeValidCtxt(ctx->valid); +- } +- if (ctx->rng != NULL) { +- xmlRelaxNGFree(ctx->rng); +- } +- free(ctx); +- known_schemas[lpc].cache = NULL; +- break; +- } +- free(known_schemas[lpc].name); +- free(known_schemas[lpc].transform); +- free(known_schemas[lpc].transform_enter); ++ for (int lpc = 0; lpc < xml_schema_max; lpc++) { ++ free_schema(&known_schemas[lpc]); + } ++ + free(known_schemas); + known_schemas = NULL; + +-- +2.31.1 + +From fdf66811c23d93715fcd34e16eb58fce5f4294d7 Mon Sep 17 00:00:00 2001 +From: Chris Lumens +Date: Wed, 25 Oct 2023 14:36:07 -0400 +Subject: [PATCH 04/20] Refactor: libcrmcommon: Clean up add_schema a bit. + +* Use true/false instead of TRUE/FALSE. + +* Call CRM_ASSERT after all the strdups. + +* There's no need to have a for loop over a two element list. If the + version number struct ever changes, plenty of other places will have + to be changed as well so this isn't saving us much. +--- + lib/common/schemas.c | 20 +++++++++++++------- + 1 file changed, 13 insertions(+), 7 deletions(-) + +diff --git a/lib/common/schemas.c b/lib/common/schemas.c +index c21b9ae..17cd21f 100644 +--- a/lib/common/schemas.c ++++ b/lib/common/schemas.c +@@ -178,7 +178,7 @@ add_schema(enum schema_validator_e validator, const schema_version_t *version, + int after_transform) + { + int last = xml_schema_max; +- bool have_version = FALSE; ++ bool have_version = false; + + xml_schema_max++; + known_schemas = pcmk__realloc(known_schemas, +@@ -188,26 +188,32 @@ add_schema(enum schema_validator_e validator, const schema_version_t *version, + known_schemas[last].validator = validator; + known_schemas[last].after_transform = after_transform; + +- for (int i = 0; i < 2; ++i) { +- known_schemas[last].version.v[i] = version->v[i]; +- if (version->v[i]) { +- have_version = TRUE; +- } ++ known_schemas[last].version.v[0] = version->v[0]; ++ known_schemas[last].version.v[1] = version->v[1]; ++ ++ if (version->v[0] || version->v[1]) { ++ have_version = true; + } ++ + if (have_version) { + known_schemas[last].name = schema_strdup_printf("pacemaker-", *version, ""); + } else { +- CRM_ASSERT(name); ++ CRM_ASSERT(name != NULL); + schema_scanf(name, "%*[^-]-", known_schemas[last].version, ""); + known_schemas[last].name = strdup(name); ++ CRM_ASSERT(known_schemas[last].name != NULL); + } + + if (transform) { + known_schemas[last].transform = strdup(transform); ++ CRM_ASSERT(known_schemas[last].transform != NULL); + } ++ + if (transform_enter) { + known_schemas[last].transform_enter = strdup(transform_enter); ++ CRM_ASSERT(known_schemas[last].transform_enter != NULL); + } ++ + known_schemas[last].transform_onleave = transform_onleave; + if (after_transform == 0) { + after_transform = xml_schema_max; /* upgrade is a one-way */ +-- +2.31.1 + +From ef89e0536fae09036a657cf651da1eed75356054 Mon Sep 17 00:00:00 2001 +From: Chris Lumens +Date: Wed, 25 Oct 2023 15:16:11 -0400 +Subject: [PATCH 05/20] Refactor: libcrmcommon: Use pcmk__s in schemas.c where + possible. + +--- + lib/common/schemas.c | 8 ++++---- + 1 file changed, 4 insertions(+), 4 deletions(-) + +diff --git a/lib/common/schemas.c b/lib/common/schemas.c +index 17cd21f..fca81e4 100644 +--- a/lib/common/schemas.c ++++ b/lib/common/schemas.c +@@ -1031,7 +1031,7 @@ update_validation(xmlNode **xml_blob, int *best, int max, gboolean transform, + + while (lpc <= max_stable_schemas) { + crm_debug("Testing '%s' validation (%d of %d)", +- known_schemas[lpc].name ? known_schemas[lpc].name : "", ++ pcmk__s(known_schemas[lpc].name, ""), + lpc, max_stable_schemas); + + if (validate_with(xml, &known_schemas[lpc], error_handler, GUINT_TO_POINTER(LOG_ERR)) == FALSE) { +@@ -1041,7 +1041,7 @@ update_validation(xmlNode **xml_blob, int *best, int max, gboolean transform, + next = -1; + } else { + crm_trace("%s validation failed", +- known_schemas[lpc].name ? known_schemas[lpc].name : ""); ++ pcmk__s(known_schemas[lpc].name, "")); + } + if (*best) { + /* we've satisfied the validation, no need to check further */ +@@ -1128,8 +1128,8 @@ update_validation(xmlNode **xml_blob, int *best, int max, gboolean transform, + + if (*best > match && *best) { + crm_info("%s the configuration from %s to %s", +- transform?"Transformed":"Upgraded", +- value ? value : "", known_schemas[*best].name); ++ transform?"Transformed":"Upgraded", pcmk__s(value, ""), ++ known_schemas[*best].name); + crm_xml_add(xml, XML_ATTR_VALIDATION, known_schemas[*best].name); + } + +-- +2.31.1 + +From 574c3c1f5ca00514eff77b927821b695c980a683 Mon Sep 17 00:00:00 2001 +From: Chris Lumens +Date: Wed, 25 Oct 2023 15:20:04 -0400 +Subject: [PATCH 06/20] Refactor: libcrmcommon: Use a schema variable in + update_validation. + +This just gets rid of a ton of references to the known_schemas array, +making it easier to replace that array with something else in a future +commit. +--- + lib/common/schemas.c | 38 +++++++++++++++++--------------------- + 1 file changed, 17 insertions(+), 21 deletions(-) + +diff --git a/lib/common/schemas.c b/lib/common/schemas.c +index fca81e4..888a473 100644 +--- a/lib/common/schemas.c ++++ b/lib/common/schemas.c +@@ -1030,18 +1030,18 @@ update_validation(xmlNode **xml_blob, int *best, int max, gboolean transform, + } + + while (lpc <= max_stable_schemas) { ++ struct schema_s *schema = &known_schemas[lpc]; ++ + crm_debug("Testing '%s' validation (%d of %d)", +- pcmk__s(known_schemas[lpc].name, ""), +- lpc, max_stable_schemas); ++ pcmk__s(schema->name, ""), lpc, max_stable_schemas); + +- if (validate_with(xml, &known_schemas[lpc], error_handler, GUINT_TO_POINTER(LOG_ERR)) == FALSE) { ++ if (validate_with(xml, schema, error_handler, GUINT_TO_POINTER(LOG_ERR)) == FALSE) { + if (next != -1) { + crm_info("Configuration not valid for schema: %s", +- known_schemas[lpc].name); ++ schema->name); + next = -1; + } else { +- crm_trace("%s validation failed", +- pcmk__s(known_schemas[lpc].name, "")); ++ crm_trace("%s validation failed", pcmk__s(schema->name, "")); + } + if (*best) { + /* we've satisfied the validation, no need to check further */ +@@ -1051,8 +1051,7 @@ update_validation(xmlNode **xml_blob, int *best, int max, gboolean transform, + + } else { + if (next != -1) { +- crm_debug("Configuration valid for schema: %s", +- known_schemas[next].name); ++ crm_debug("Configuration valid for schema: %s", schema->name); + next = -1; + } + rc = pcmk_ok; +@@ -1064,19 +1063,19 @@ update_validation(xmlNode **xml_blob, int *best, int max, gboolean transform, + + if (rc == pcmk_ok && transform) { + xmlNode *upgrade = NULL; +- next = known_schemas[lpc].after_transform; ++ next = schema->after_transform; + + if (next <= lpc) { + /* There is no next version, or next would regress */ +- crm_trace("Stopping at %s", known_schemas[lpc].name); ++ crm_trace("Stopping at %s", schema->name); + break; + + } else if (max > 0 && (lpc == max || next > max)) { + crm_trace("Upgrade limit reached at %s (lpc=%d, next=%d, max=%d)", +- known_schemas[lpc].name, lpc, next, max); ++ schema->name, lpc, next, max); + break; + +- } else if (known_schemas[lpc].transform == NULL ++ } else if (schema->transform == NULL + /* possibly avoid transforming when readily valid + (in general more restricted when crossing the major + version boundary, as X.0 "transitional" version is +@@ -1084,25 +1083,22 @@ update_validation(xmlNode **xml_blob, int *best, int max, gboolean transform, + may re-allow constructs from previous major line) */ + || validate_with_silent(xml, next >= 0 ? &known_schemas[next] : NULL)) { + crm_debug("%s-style configuration is also valid for %s", +- known_schemas[lpc].name, known_schemas[next].name); ++ schema->name, known_schemas[next].name); + + lpc = next; + + } else { + crm_debug("Upgrading %s-style configuration to %s with %s.xsl", +- known_schemas[lpc].name, known_schemas[next].name, +- known_schemas[lpc].transform); ++ schema->name, known_schemas[next].name, schema->transform); + +- upgrade = apply_upgrade(xml, &known_schemas[lpc], to_logs); ++ upgrade = apply_upgrade(xml, schema, to_logs); + if (upgrade == NULL) { +- crm_err("Transformation %s.xsl failed", +- known_schemas[lpc].transform); ++ crm_err("Transformation %s.xsl failed", schema->transform); + rc = -pcmk_err_transform_failed; + + } else if (validate_with(upgrade, next >= 0 ? &known_schemas[next] : NULL, + error_handler, GUINT_TO_POINTER(LOG_ERR))) { +- crm_info("Transformation %s.xsl successful", +- known_schemas[lpc].transform); ++ crm_info("Transformation %s.xsl successful", schema->transform); + lpc = next; + *best = next; + free_xml(xml); +@@ -1111,7 +1107,7 @@ update_validation(xmlNode **xml_blob, int *best, int max, gboolean transform, + + } else { + crm_err("Transformation %s.xsl did not produce a valid configuration", +- known_schemas[lpc].transform); ++ schema->transform); + crm_log_xml_info(upgrade, "transform:bad"); + free_xml(upgrade); + rc = -pcmk_err_schema_validation; +-- +2.31.1 + +From dc724c940014fbe60aa506d8acb652b2dd5dce90 Mon Sep 17 00:00:00 2001 +From: Chris Lumens +Date: Wed, 25 Oct 2023 15:31:27 -0400 +Subject: [PATCH 07/20] Refactor: libcrmcommon: Use a variable for the next + schema, too. + +This gets rid of further references to known_schemas update_validation, +also with the purpose of making it easier to change the implementation +of that array. +--- + lib/common/schemas.c | 20 +++++++++++++------- + 1 file changed, 13 insertions(+), 7 deletions(-) + +diff --git a/lib/common/schemas.c b/lib/common/schemas.c +index 888a473..8e5c22e 100644 +--- a/lib/common/schemas.c ++++ b/lib/common/schemas.c +@@ -1063,41 +1063,47 @@ update_validation(xmlNode **xml_blob, int *best, int max, gboolean transform, + + if (rc == pcmk_ok && transform) { + xmlNode *upgrade = NULL; ++ struct schema_s *next_schema = NULL; + next = schema->after_transform; + + if (next <= lpc) { + /* There is no next version, or next would regress */ + crm_trace("Stopping at %s", schema->name); + break; ++ } + +- } else if (max > 0 && (lpc == max || next > max)) { ++ if (max > 0 && (lpc == max || next > max)) { + crm_trace("Upgrade limit reached at %s (lpc=%d, next=%d, max=%d)", + schema->name, lpc, next, max); + break; ++ } ++ ++ next_schema = &known_schemas[next]; ++ CRM_ASSERT(next_schema != NULL); + +- } else if (schema->transform == NULL ++ if (schema->transform == NULL + /* possibly avoid transforming when readily valid + (in general more restricted when crossing the major + version boundary, as X.0 "transitional" version is + expected to be more strict than it's successors that + may re-allow constructs from previous major line) */ +- || validate_with_silent(xml, next >= 0 ? &known_schemas[next] : NULL)) { ++ || validate_with_silent(xml, next_schema)) { + crm_debug("%s-style configuration is also valid for %s", +- schema->name, known_schemas[next].name); ++ schema->name, next_schema->name); + + lpc = next; + + } else { + crm_debug("Upgrading %s-style configuration to %s with %s.xsl", +- schema->name, known_schemas[next].name, schema->transform); ++ schema->name, next_schema->name, schema->transform); + + upgrade = apply_upgrade(xml, schema, to_logs); + if (upgrade == NULL) { + crm_err("Transformation %s.xsl failed", schema->transform); + rc = -pcmk_err_transform_failed; + +- } else if (validate_with(upgrade, next >= 0 ? &known_schemas[next] : NULL, +- error_handler, GUINT_TO_POINTER(LOG_ERR))) { ++ } else if (validate_with(upgrade, next_schema, error_handler, ++ GUINT_TO_POINTER(LOG_ERR))) { + crm_info("Transformation %s.xsl successful", schema->transform); + lpc = next; + *best = next; +-- +2.31.1 + +From 0664f7e327c612d5602515ecf4bb32fb7c3503f6 Mon Sep 17 00:00:00 2001 +From: Chris Lumens +Date: Wed, 25 Oct 2023 16:09:30 -0400 +Subject: [PATCH 08/20] Refactor: libcrmcommon: Add pcmk__dump_known_schemas. + +The debug logging in add_schema isn't necessarily all that useful. +Typically, schema adding happens in crm_log_preinit which means it +happens before logging is set up, so nothing that we log actually goes +anywhere. + +This function does the same thing but can be called where needed. +--- + include/crm/common/xml_internal.h | 3 +++ + lib/common/schemas.c | 21 +++++++++++++++++++++ + 2 files changed, 24 insertions(+) + +diff --git a/include/crm/common/xml_internal.h b/include/crm/common/xml_internal.h +index ddb4384..f319856 100644 +--- a/include/crm/common/xml_internal.h ++++ b/include/crm/common/xml_internal.h +@@ -441,8 +441,11 @@ pcmk__xml_attr_value(const xmlAttr *attr) + : (const char *) attr->children->content; + } + ++ + gboolean pcmk__validate_xml(xmlNode *xml_blob, const char *validation, + xmlRelaxNGValidityErrorFunc error_handler, + void *error_handler_context); + ++void pcmk__log_known_schemas(void); ++ + #endif // PCMK__XML_INTERNAL__H +diff --git a/lib/common/schemas.c b/lib/common/schemas.c +index 8e5c22e..41ca138 100644 +--- a/lib/common/schemas.c ++++ b/lib/common/schemas.c +@@ -1248,3 +1248,24 @@ cli_config_update(xmlNode **xml, int *best_version, gboolean to_logs) + free(orig_value); + return rc; + } ++ ++void ++pcmk__log_known_schemas(void) ++{ ++ for (int lpc = 0; lpc < xml_schema_max; lpc++) { ++ if (known_schemas[lpc].after_transform < 0) { ++ crm_debug("known_schemas[%d] => %s", lpc, known_schemas[lpc].name); ++ ++ } else if (known_schemas[lpc].transform != NULL) { ++ crm_debug("known_schemas[%d] => %s (upgrades to %d with %s.xsl)", ++ lpc, known_schemas[lpc].name, ++ known_schemas[lpc].after_transform, ++ known_schemas[lpc].transform); ++ ++ } else { ++ crm_debug("known_schemas[%d] => %s (upgrades to %d)", ++ lpc, known_schemas[lpc].name, ++ known_schemas[lpc].after_transform); ++ } ++ } ++} +-- +2.31.1 + +From 423a28fd5c2b71945d75b68b168a607279d795f7 Mon Sep 17 00:00:00 2001 +From: Chris Lumens +Date: Wed, 25 Oct 2023 17:22:53 -0400 +Subject: [PATCH 09/20] Refactor: libcrmcommon: Store known_schemas as a GList. + +Instead of managing our own array with realloc, use GList and the +various glib list functions. + +In many places, this makes the code easier to follow - we can simply +iterate over the list and do something on each node. In other places, +we're still relying on list indices too much to help. Those spots can +probably be cleaned up in future commits. +--- + lib/common/schemas.c | 181 ++++++++++++++++++++++++++----------------- + 1 file changed, 108 insertions(+), 73 deletions(-) + +diff --git a/lib/common/schemas.c b/lib/common/schemas.c +index 41ca138..6e6f32e 100644 +--- a/lib/common/schemas.c ++++ b/lib/common/schemas.c +@@ -60,7 +60,7 @@ struct schema_s { + bool transform_onleave; + }; + +-static struct schema_s *known_schemas = NULL; ++static GList *known_schemas = NULL; + static int xml_schema_max = 0; + static bool silent_logging = FALSE; + +@@ -81,27 +81,45 @@ static int + xml_latest_schema_index(void) + { + // @COMPAT: pacemaker-next is deprecated since 2.1.5 +- return xml_schema_max - 3; // index from 0, ignore "pacemaker-next"/"none" ++ // FIXME: This function assumes at least three schemas have been added ++ // before it has been called for the first time. ++ return g_list_length(known_schemas) - 3; // index from 0, ignore "pacemaker-next"/"none" + } + + static int + xml_minimum_schema_index(void) + { + static int best = 0; +- if (best == 0) { +- int lpc = 0; +- +- best = xml_latest_schema_index(); +- for (lpc = best; lpc > 0; lpc--) { +- if (known_schemas[lpc].version.v[0] +- < known_schemas[best].version.v[0]) { +- return best; +- } else { +- best = lpc; +- } ++ struct schema_s *best_schema = NULL; ++ GList *last_real_ele = NULL; ++ ++ if (best != 0) { ++ return best; ++ } ++ ++ best = xml_latest_schema_index(); ++ ++ /* We can't just use g_list_last here because "pacemaker-next" and "none" ++ * are stored at the end of the list. We need to start several elements ++ * back, at the last real schema. ++ */ ++ last_real_ele = g_list_nth(known_schemas, best); ++ best_schema = last_real_ele->data; ++ ++ for (GList *iter = last_real_ele; iter != NULL; iter = iter->prev) { ++ struct schema_s *schema = iter->data; ++ ++ if (schema->version.v[0] < best_schema->version.v[0]) { ++ return best; ++ } else { ++ best--; + } +- best = xml_latest_schema_index(); + } ++ ++ /* If we never found a schema that meets the above criteria, default to ++ * the last one. ++ */ ++ best = xml_latest_schema_index(); + return best; + } + +@@ -177,63 +195,61 @@ add_schema(enum schema_validator_e validator, const schema_version_t *version, + const char *transform_enter, bool transform_onleave, + int after_transform) + { ++ struct schema_s *schema = NULL; + int last = xml_schema_max; + bool have_version = false; + + xml_schema_max++; +- known_schemas = pcmk__realloc(known_schemas, +- xml_schema_max * sizeof(struct schema_s)); +- CRM_ASSERT(known_schemas != NULL); +- memset(known_schemas+last, 0, sizeof(struct schema_s)); +- known_schemas[last].validator = validator; +- known_schemas[last].after_transform = after_transform; + +- known_schemas[last].version.v[0] = version->v[0]; +- known_schemas[last].version.v[1] = version->v[1]; ++ schema = calloc(1, sizeof(struct schema_s)); ++ CRM_ASSERT(schema != NULL); ++ ++ schema->validator = validator; ++ schema->after_transform = after_transform; ++ schema->version.v[0] = version->v[0]; ++ schema->version.v[1] = version->v[1]; + + if (version->v[0] || version->v[1]) { + have_version = true; + } + + if (have_version) { +- known_schemas[last].name = schema_strdup_printf("pacemaker-", *version, ""); ++ schema->name = schema_strdup_printf("pacemaker-", *version, ""); + } else { + CRM_ASSERT(name != NULL); +- schema_scanf(name, "%*[^-]-", known_schemas[last].version, ""); +- known_schemas[last].name = strdup(name); +- CRM_ASSERT(known_schemas[last].name != NULL); ++ schema_scanf(name, "%*[^-]-", schema->version, ""); ++ schema->name = strdup(name); ++ CRM_ASSERT(schema->name != NULL); + } + + if (transform) { +- known_schemas[last].transform = strdup(transform); +- CRM_ASSERT(known_schemas[last].transform != NULL); ++ schema->transform = strdup(transform); ++ CRM_ASSERT(schema->transform != NULL); + } + + if (transform_enter) { +- known_schemas[last].transform_enter = strdup(transform_enter); +- CRM_ASSERT(known_schemas[last].transform_enter != NULL); ++ schema->transform_enter = strdup(transform_enter); ++ CRM_ASSERT(schema->transform_enter != NULL); + } + +- known_schemas[last].transform_onleave = transform_onleave; ++ schema->transform_onleave = transform_onleave; + if (after_transform == 0) { + after_transform = xml_schema_max; /* upgrade is a one-way */ + } +- known_schemas[last].after_transform = after_transform; ++ schema->after_transform = after_transform; + +- if (known_schemas[last].after_transform < 0) { +- crm_debug("Added supported schema %d: %s", +- last, known_schemas[last].name); ++ known_schemas = g_list_append(known_schemas, schema); + +- } else if (known_schemas[last].transform) { ++ if (schema->after_transform < 0) { ++ crm_debug("Added supported schema %d: %s", last, schema->name); ++ ++ } else if (schema->transform != NULL) { + crm_debug("Added supported schema %d: %s (upgrades to %d with %s.xsl)", +- last, known_schemas[last].name, +- known_schemas[last].after_transform, +- known_schemas[last].transform); ++ last, schema->name, schema->after_transform, schema->transform); + + } else { + crm_debug("Added supported schema %d: %s (upgrades to %d)", +- last, known_schemas[last].name, +- known_schemas[last].after_transform); ++ last, schema->name, schema->after_transform); + } + } + +@@ -518,8 +534,9 @@ validate_with_relaxng(xmlDocPtr doc, xmlRelaxNGValidityErrorFunc error_handler, + } + + static void +-free_schema(struct schema_s *schema) ++free_schema(gpointer data) + { ++ struct schema_s *schema = data; + relaxng_ctx_cache_t *ctx = NULL; + + switch (schema->validator) { +@@ -561,11 +578,7 @@ free_schema(struct schema_s *schema) + void + crm_schema_cleanup(void) + { +- for (int lpc = 0; lpc < xml_schema_max; lpc++) { +- free_schema(&known_schemas[lpc]); +- } +- +- free(known_schemas); ++ g_list_free_full(known_schemas, free_schema); + known_schemas = NULL; + + wrap_libxslt(true); +@@ -697,16 +710,17 @@ pcmk__validate_xml(xmlNode *xml_blob, const char *validation, xmlRelaxNGValidity + } + + if (validation == NULL) { +- int lpc = 0; + bool valid = FALSE; + +- for (lpc = 0; lpc < xml_schema_max; lpc++) { +- if (validate_with(xml_blob, &known_schemas[lpc], NULL, NULL)) { ++ for (GList *iter = known_schemas; iter != NULL; iter = iter->next) { ++ struct schema_s *schema = iter->data; ++ ++ if (validate_with(xml_blob, schema, NULL, NULL)) { + valid = TRUE; +- crm_xml_add(xml_blob, XML_ATTR_VALIDATION, +- known_schemas[lpc].name); +- crm_info("XML validated against %s", known_schemas[lpc].name); +- if(known_schemas[lpc].after_transform == 0) { ++ crm_xml_add(xml_blob, XML_ATTR_VALIDATION, schema->name); ++ crm_info("XML validated against %s", schema->name); ++ ++ if (schema->after_transform == 0) { + break; + } + } +@@ -719,8 +733,9 @@ pcmk__validate_xml(xmlNode *xml_blob, const char *validation, xmlRelaxNGValidity + if (strcmp(validation, PCMK__VALUE_NONE) == 0) { + return TRUE; + } else if (version < xml_schema_max) { +- return validate_with(xml_blob, version >= 0 ? &known_schemas[version] : NULL, +- error_handler, error_handler_context); ++ struct schema_s *schema = g_list_nth_data(known_schemas, version); ++ return validate_with(xml_blob, schema, error_handler, ++ error_handler_context); + } + + crm_err("Unknown validator: %s", validation); +@@ -964,10 +979,13 @@ apply_upgrade(xmlNode *xml, const struct schema_s *schema, gboolean to_logs) + const char * + get_schema_name(int version) + { +- if (version < 0 || version >= xml_schema_max) { ++ struct schema_s *schema = g_list_nth_data(known_schemas, version); ++ ++ if (schema == NULL) { + return "unknown"; + } +- return known_schemas[version].name; ++ ++ return schema->name; + } + + int +@@ -978,11 +996,17 @@ get_schema_version(const char *name) + if (name == NULL) { + name = PCMK__VALUE_NONE; + } +- for (; lpc < xml_schema_max; lpc++) { +- if (pcmk__str_eq(name, known_schemas[lpc].name, pcmk__str_casei)) { ++ ++ for (GList *iter = known_schemas; iter != NULL; iter = iter->next) { ++ struct schema_s *schema = iter->data; ++ ++ if (pcmk__str_eq(name, schema->name, pcmk__str_casei)) { + return lpc; + } ++ ++ lpc++; + } ++ + return -1; + } + +@@ -1030,7 +1054,12 @@ update_validation(xmlNode **xml_blob, int *best, int max, gboolean transform, + } + + while (lpc <= max_stable_schemas) { +- struct schema_s *schema = &known_schemas[lpc]; ++ /* FIXME: This will cause us to walk the known_schemas list every time ++ * this loop iterates, which is not ideal. However, for now it's a lot ++ * easier than trying to get all the loop indices we're using here ++ * sorted out and working correctly. ++ */ ++ struct schema_s *schema = g_list_nth_data(known_schemas, lpc); + + crm_debug("Testing '%s' validation (%d of %d)", + pcmk__s(schema->name, ""), lpc, max_stable_schemas); +@@ -1078,7 +1107,7 @@ update_validation(xmlNode **xml_blob, int *best, int max, gboolean transform, + break; + } + +- next_schema = &known_schemas[next]; ++ next_schema = g_list_nth_data(known_schemas, next); + CRM_ASSERT(next_schema != NULL); + + if (schema->transform == NULL +@@ -1129,10 +1158,12 @@ update_validation(xmlNode **xml_blob, int *best, int max, gboolean transform, + } + + if (*best > match && *best) { ++ struct schema_s *best_schema = g_list_nth_data(known_schemas, *best); ++ + crm_info("%s the configuration from %s to %s", + transform?"Transformed":"Upgraded", pcmk__s(value, ""), +- known_schemas[*best].name); +- crm_xml_add(xml, XML_ATTR_VALIDATION, known_schemas[*best].name); ++ best_schema->name); ++ crm_xml_add(xml, XML_ATTR_VALIDATION, best_schema->name); + } + + *xml_blob = xml; +@@ -1252,20 +1283,24 @@ cli_config_update(xmlNode **xml, int *best_version, gboolean to_logs) + void + pcmk__log_known_schemas(void) + { +- for (int lpc = 0; lpc < xml_schema_max; lpc++) { +- if (known_schemas[lpc].after_transform < 0) { +- crm_debug("known_schemas[%d] => %s", lpc, known_schemas[lpc].name); ++ int lpc = 0; + +- } else if (known_schemas[lpc].transform != NULL) { ++ for (GList *iter = known_schemas; iter != NULL; iter = iter->next) { ++ struct schema_s *schema = iter->data; ++ ++ if (schema->after_transform < 0) { ++ crm_debug("known_schemas[%d] => %s", lpc, schema->name); ++ ++ } else if (schema->transform != NULL) { + crm_debug("known_schemas[%d] => %s (upgrades to %d with %s.xsl)", +- lpc, known_schemas[lpc].name, +- known_schemas[lpc].after_transform, +- known_schemas[lpc].transform); ++ lpc, schema->name, schema->after_transform, ++ schema->transform); + + } else { + crm_debug("known_schemas[%d] => %s (upgrades to %d)", +- lpc, known_schemas[lpc].name, +- known_schemas[lpc].after_transform); ++ lpc, schema->name, schema->after_transform); + } ++ ++ lpc++; + } + } +-- +2.31.1 + +From 33cfcc0d98603e04dde15d69acd46823679405f0 Mon Sep 17 00:00:00 2001 +From: Chris Lumens +Date: Wed, 25 Oct 2023 17:34:25 -0400 +Subject: [PATCH 10/20] Refactor: libcrmcommon: Get rid of xml_schema_max. + +This is just the length of the known_schemas list. +--- + lib/common/schemas.c | 9 +++------ + 1 file changed, 3 insertions(+), 6 deletions(-) + +diff --git a/lib/common/schemas.c b/lib/common/schemas.c +index 6e6f32e..d4ce68e 100644 +--- a/lib/common/schemas.c ++++ b/lib/common/schemas.c +@@ -61,7 +61,6 @@ struct schema_s { + }; + + static GList *known_schemas = NULL; +-static int xml_schema_max = 0; + static bool silent_logging = FALSE; + + static void G_GNUC_PRINTF(2, 3) +@@ -196,11 +195,9 @@ add_schema(enum schema_validator_e validator, const schema_version_t *version, + int after_transform) + { + struct schema_s *schema = NULL; +- int last = xml_schema_max; ++ int last = g_list_length(known_schemas); + bool have_version = false; + +- xml_schema_max++; +- + schema = calloc(1, sizeof(struct schema_s)); + CRM_ASSERT(schema != NULL); + +@@ -234,7 +231,7 @@ add_schema(enum schema_validator_e validator, const schema_version_t *version, + + schema->transform_onleave = transform_onleave; + if (after_transform == 0) { +- after_transform = xml_schema_max; /* upgrade is a one-way */ ++ after_transform = last + 1; /* upgrade is a one-way */ + } + schema->after_transform = after_transform; + +@@ -732,7 +729,7 @@ pcmk__validate_xml(xmlNode *xml_blob, const char *validation, xmlRelaxNGValidity + version = get_schema_version(validation); + if (strcmp(validation, PCMK__VALUE_NONE) == 0) { + return TRUE; +- } else if (version < xml_schema_max) { ++ } else if (version < g_list_length(known_schemas)) { + struct schema_s *schema = g_list_nth_data(known_schemas, version); + return validate_with(xml_blob, schema, error_handler, + error_handler_context); +-- +2.31.1 + +From 6ace4c912d34f495ab5f52500628c82fb1533256 Mon Sep 17 00:00:00 2001 +From: Chris Lumens +Date: Thu, 26 Oct 2023 12:52:14 -0400 +Subject: [PATCH 11/20] Refactor: libcrmcommon: Rename + xml_minimum_schema_index. + +This function's name is unclear. It actually returns the most recent +X.0 schema index. The new name is pretty bad, but I think it's at least +clear. + +And then while I'm at it, rewrite it to make it more clear. Just +iterate the known_schemas list, looking for the right .0 one. This code +does not get run very often, and it caches its result, so there's no +need to do the reverse traversal with a lagging index. +--- + lib/common/schemas.c | 41 +++++++++++++++++++++++++---------------- + 1 file changed, 25 insertions(+), 16 deletions(-) + +diff --git a/lib/common/schemas.c b/lib/common/schemas.c +index d4ce68e..55519e8 100644 +--- a/lib/common/schemas.c ++++ b/lib/common/schemas.c +@@ -85,39 +85,48 @@ xml_latest_schema_index(void) + return g_list_length(known_schemas) - 3; // index from 0, ignore "pacemaker-next"/"none" + } + ++/* Return the index of the most recent X.0 schema. */ + static int +-xml_minimum_schema_index(void) ++xml_find_x_0_schema_index(void) + { + static int best = 0; ++ int i = 0; + struct schema_s *best_schema = NULL; +- GList *last_real_ele = NULL; + + if (best != 0) { + return best; + } + ++ /* Get the most recent schema so we can look at its version number. */ + best = xml_latest_schema_index(); ++ best_schema = g_list_nth(known_schemas, best)->data; + +- /* We can't just use g_list_last here because "pacemaker-next" and "none" +- * are stored at the end of the list. We need to start several elements +- * back, at the last real schema. ++ /* Iterate over the schema list until we find a schema with the same major ++ * version as best, and with a minor version number of 0. ++ * ++ * This assumes that the first schema in a major series is always X.0, ++ * which seems like a safe assumption. + */ +- last_real_ele = g_list_nth(known_schemas, best); +- best_schema = last_real_ele->data; +- +- for (GList *iter = last_real_ele; iter != NULL; iter = iter->prev) { ++ for (GList *iter = known_schemas; iter != NULL; iter = iter->next) { + struct schema_s *schema = iter->data; + +- if (schema->version.v[0] < best_schema->version.v[0]) { ++ /* If we hit the initial best schema, the only things left in the list ++ * are "pacemaker-next" and "none" which aren't worth checking. ++ */ ++ if (schema == best_schema) { ++ break; ++ } ++ ++ if (schema->version.v[0] == best_schema->version.v[0] && ++ schema->version.v[1] == 0) { ++ best = i; + return best; +- } else { +- best--; + } ++ ++ i++; + } + +- /* If we never found a schema that meets the above criteria, default to +- * the last one. +- */ ++ /* If we got here, we never found a match. Just return the latest. */ + best = xml_latest_schema_index(); + return best; + } +@@ -1177,7 +1186,7 @@ cli_config_update(xmlNode **xml, int *best_version, gboolean to_logs) + + int version = get_schema_version(value); + int orig_version = version; +- int min_version = xml_minimum_schema_index(); ++ int min_version = xml_find_x_0_schema_index(); + + if (version < min_version) { + // Current configuration schema is not acceptable, try to update +-- +2.31.1 + +From 7df1d2df9e8710183735b19947a885bb129e523e Mon Sep 17 00:00:00 2001 +From: Chris Lumens +Date: Thu, 26 Oct 2023 14:14:02 -0400 +Subject: [PATCH 12/20] Refactor: libcrmcommon: Remove an unnecessary check in + validate_xml. + +I believe that add_schema ensures after_transform is never 0 - it's +either negative, or some positive non-zero value. So this check should +be pointless. +--- + lib/common/schemas.c | 4 ---- + 1 file changed, 4 deletions(-) + +diff --git a/lib/common/schemas.c b/lib/common/schemas.c +index 55519e8..cfb83dd 100644 +--- a/lib/common/schemas.c ++++ b/lib/common/schemas.c +@@ -725,10 +725,6 @@ pcmk__validate_xml(xmlNode *xml_blob, const char *validation, xmlRelaxNGValidity + valid = TRUE; + crm_xml_add(xml_blob, XML_ATTR_VALIDATION, schema->name); + crm_info("XML validated against %s", schema->name); +- +- if (schema->after_transform == 0) { +- break; +- } + } + } + +-- +2.31.1 + +From dbf94f5a3c146992fb60c231a7eda21271b62b99 Mon Sep 17 00:00:00 2001 +From: Chris Lumens +Date: Fri, 27 Oct 2023 10:49:49 -0400 +Subject: [PATCH 13/20] Refactor: libcrmcommon: Change how schema upgrade + versions are handled + +...in update_validation. Schemas always either upgrade to the next one +in the list, or do not upgrade. The latter only happens when we get to +the last real version and the next one is pacemaker-next/none. + +With that change made, we also need to change the conditional. There's +no need to check that the upgrade will regress. We only need to check +that we've run off the end of the list of real schema versions. + +A future commit will remove the after_transform variable entirely, but +since this is its most visible and complicated use, splitting this into +a separate commit seems worth it. +--- + lib/common/schemas.c | 6 +++--- + 1 file changed, 3 insertions(+), 3 deletions(-) + +diff --git a/lib/common/schemas.c b/lib/common/schemas.c +index cfb83dd..3ebdf1c 100644 +--- a/lib/common/schemas.c ++++ b/lib/common/schemas.c +@@ -1095,10 +1095,10 @@ update_validation(xmlNode **xml_blob, int *best, int max, gboolean transform, + if (rc == pcmk_ok && transform) { + xmlNode *upgrade = NULL; + struct schema_s *next_schema = NULL; +- next = schema->after_transform; ++ next = lpc+1; + +- if (next <= lpc) { +- /* There is no next version, or next would regress */ ++ if (next > max_stable_schemas) { ++ /* There is no next version */ + crm_trace("Stopping at %s", schema->name); + break; + } +-- +2.31.1 + +From b2da7aaba5a1afe9d4f56989c0b81dd55abaf1b8 Mon Sep 17 00:00:00 2001 +From: Chris Lumens +Date: Fri, 27 Oct 2023 11:00:12 -0400 +Subject: [PATCH 14/20] Refactor: libcrmcommon: Get rid of after_transform. + +As stated in the previous commit, schemas always just upgrade to the +next one in the list. There's no need to keep track of that fact, so +get rid of the variable that held it. This then allows us to get rid of +all the places that value was being set and passed around. +--- + lib/common/schemas.c | 54 +++++++++++++------------------------------- + 1 file changed, 16 insertions(+), 38 deletions(-) + +diff --git a/lib/common/schemas.c b/lib/common/schemas.c +index 3ebdf1c..e33d3c7 100644 +--- a/lib/common/schemas.c ++++ b/lib/common/schemas.c +@@ -54,7 +54,6 @@ struct schema_s { + char *transform; + void *cache; + enum schema_validator_e validator; +- int after_transform; + schema_version_t version; + char *transform_enter; + bool transform_onleave; +@@ -200,8 +199,7 @@ schema_sort(const struct dirent **a, const struct dirent **b) + static void + add_schema(enum schema_validator_e validator, const schema_version_t *version, + const char *name, const char *transform, +- const char *transform_enter, bool transform_onleave, +- int after_transform) ++ const char *transform_enter, bool transform_onleave) + { + struct schema_s *schema = NULL; + int last = g_list_length(known_schemas); +@@ -211,9 +209,9 @@ add_schema(enum schema_validator_e validator, const schema_version_t *version, + CRM_ASSERT(schema != NULL); + + schema->validator = validator; +- schema->after_transform = after_transform; + schema->version.v[0] = version->v[0]; + schema->version.v[1] = version->v[1]; ++ schema->transform_onleave = transform_onleave; + + if (version->v[0] || version->v[1]) { + have_version = true; +@@ -238,24 +236,14 @@ add_schema(enum schema_validator_e validator, const schema_version_t *version, + CRM_ASSERT(schema->transform_enter != NULL); + } + +- schema->transform_onleave = transform_onleave; +- if (after_transform == 0) { +- after_transform = last + 1; /* upgrade is a one-way */ +- } +- schema->after_transform = after_transform; +- + known_schemas = g_list_append(known_schemas, schema); + +- if (schema->after_transform < 0) { +- crm_debug("Added supported schema %d: %s", last, schema->name); +- +- } else if (schema->transform != NULL) { +- crm_debug("Added supported schema %d: %s (upgrades to %d with %s.xsl)", +- last, schema->name, schema->after_transform, schema->transform); ++ if (schema->transform != NULL) { ++ crm_debug("Added supported schema %d: %s (upgrades with %s.xsl)", ++ last, schema->name, schema->transform); + + } else { +- crm_debug("Added supported schema %d: %s (upgrades to %d)", +- last, schema->name, schema->after_transform); ++ crm_debug("Added supported schema %d: %s", last, schema->name); + } + } + +@@ -288,8 +276,7 @@ add_schema(enum schema_validator_e validator, const schema_version_t *version, + * . name convention: (see "upgrade-enter") + */ + static int +-add_schema_by_version(const schema_version_t *version, int next, +- bool transform_expected) ++add_schema_by_version(const schema_version_t *version, bool transform_expected) + { + bool transform_onleave = FALSE; + int rc = pcmk_rc_ok; +@@ -345,12 +332,11 @@ add_schema_by_version(const schema_version_t *version, int next, + free(xslt); + free(transform_upgrade); + transform_upgrade = NULL; +- next = -1; + rc = ENOENT; + } + + add_schema(schema_validator_rng, version, NULL, +- transform_upgrade, transform_enter, transform_onleave, next); ++ transform_upgrade, transform_enter, transform_onleave); + + free(transform_upgrade); + free(transform_enter); +@@ -416,7 +402,6 @@ crm_schema_init(void) + free(base); + for (lpc = 0; lpc < max; lpc++) { + bool transform_expected = FALSE; +- int next = 0; + schema_version_t version = SCHEMA_ZERO; + + if (!version_from_filename(namelist[lpc]->d_name, &version)) { +@@ -432,11 +417,9 @@ crm_schema_init(void) + && (version.v[0] < next_version.v[0])) { + transform_expected = TRUE; + } +- +- } else { +- next = -1; + } +- if (add_schema_by_version(&version, next, transform_expected) ++ ++ if (add_schema_by_version(&version, transform_expected) + == ENOENT) { + break; + } +@@ -450,10 +433,10 @@ crm_schema_init(void) + + // @COMPAT: Deprecated since 2.1.5 + add_schema(schema_validator_rng, &zero, "pacemaker-next", +- NULL, NULL, FALSE, -1); ++ NULL, NULL, FALSE); + + add_schema(schema_validator_none, &zero, PCMK__VALUE_NONE, +- NULL, NULL, FALSE, -1); ++ NULL, NULL, FALSE); + } + + static gboolean +@@ -1290,17 +1273,12 @@ pcmk__log_known_schemas(void) + for (GList *iter = known_schemas; iter != NULL; iter = iter->next) { + struct schema_s *schema = iter->data; + +- if (schema->after_transform < 0) { +- crm_debug("known_schemas[%d] => %s", lpc, schema->name); +- +- } else if (schema->transform != NULL) { +- crm_debug("known_schemas[%d] => %s (upgrades to %d with %s.xsl)", +- lpc, schema->name, schema->after_transform, +- schema->transform); ++ if (schema->transform != NULL) { ++ crm_debug("known_schemas[%d] => %s (upgrades with %s.xsl)", ++ lpc, schema->name, schema->transform); + + } else { +- crm_debug("known_schemas[%d] => %s (upgrades to %d)", +- lpc, schema->name, schema->after_transform); ++ crm_debug("known_schemas[%d] => %s", lpc, schema->name); + } + + lpc++; +-- +2.31.1 + +From 645bb233e52b9f5f559ffcd354b2f4ef0bcdee90 Mon Sep 17 00:00:00 2001 +From: Chris Lumens +Date: Mon, 6 Nov 2023 08:22:04 -0500 +Subject: [PATCH 15/20] Refactor: libcrmcommon: Remove unnecessary schema code. + +The block that sets the version if we didn't previously do so is no +longer necessary. This block only executes if the version parameter is +all zeros, which at the moment is only "pacemaker-next" and "none". We +could probably guarantee this will continue to be the case. + +Additionally, I don't see that this would even do anything useful +anymore. Scanning the name for a version number is going to fail for +"pacemaker-next" and "none". So really, this block was just handling +the possibility that we passed in no version number but that the name +contained a number. + +And with that done, there's only one more spot using schema_scanf so we +can just replace that with a call to sscanf. +--- + lib/common/schemas.c | 14 +------------- + 1 file changed, 1 insertion(+), 13 deletions(-) + +diff --git a/lib/common/schemas.c b/lib/common/schemas.c +index e33d3c7..7b91f71 100644 +--- a/lib/common/schemas.c ++++ b/lib/common/schemas.c +@@ -32,9 +32,6 @@ typedef struct { + + #define SCHEMA_ZERO { .v = { 0, 0 } } + +-#define schema_scanf(s, prefix, version, suffix) \ +- sscanf((s), prefix "%hhu.%hhu" suffix, &((version).v[0]), &((version).v[1])) +- + #define schema_strdup_printf(prefix, version, suffix) \ + crm_strdup_printf(prefix "%u.%u" suffix, (version).v[0], (version).v[1]) + +@@ -139,9 +136,7 @@ xml_latest_schema(void) + static inline bool + version_from_filename(const char *filename, schema_version_t *version) + { +- int rc = schema_scanf(filename, "pacemaker-", *version, ".rng"); +- +- return (rc == 2); ++ return sscanf(filename, "pacemaker-%hhu.%hhu.rng", &(version->v[0]), &(version->v[1])) == 2; + } + + static int +@@ -203,7 +198,6 @@ add_schema(enum schema_validator_e validator, const schema_version_t *version, + { + struct schema_s *schema = NULL; + int last = g_list_length(known_schemas); +- bool have_version = false; + + schema = calloc(1, sizeof(struct schema_s)); + CRM_ASSERT(schema != NULL); +@@ -214,14 +208,8 @@ add_schema(enum schema_validator_e validator, const schema_version_t *version, + schema->transform_onleave = transform_onleave; + + if (version->v[0] || version->v[1]) { +- have_version = true; +- } +- +- if (have_version) { + schema->name = schema_strdup_printf("pacemaker-", *version, ""); + } else { +- CRM_ASSERT(name != NULL); +- schema_scanf(name, "%*[^-]-", schema->version, ""); + schema->name = strdup(name); + CRM_ASSERT(schema->name != NULL); + } +-- +2.31.1 + +From 0943f1ff0a9e72e88c5a234a32bb83d0f2e02c84 Mon Sep 17 00:00:00 2001 +From: Chris Lumens +Date: Fri, 17 Nov 2023 09:42:55 -0500 +Subject: [PATCH 16/20] Refactor: libcrmcommon: Improve + xml_find_x_0_schema_index. + +* Lots of comments to explain how it works. + +* Walk the list backwards, stopping on the first one in the major + version series. This means the first one no longer has to be X.0. + +* Require that known_schemas be non-NULL. + +* Don't use the returned index to also mean we've found something since + that means if the index we actually want to return is 0, the function + will have to run every time. +--- + lib/common/schemas.c | 62 +++++++++++++++++++++++++++++--------------- + 1 file changed, 41 insertions(+), 21 deletions(-) + +diff --git a/lib/common/schemas.c b/lib/common/schemas.c +index 7b91f71..466ad5a 100644 +--- a/lib/common/schemas.c ++++ b/lib/common/schemas.c +@@ -85,45 +85,65 @@ xml_latest_schema_index(void) + static int + xml_find_x_0_schema_index(void) + { ++ /* We can't just use best to determine whether we've found the index ++ * or not. What if we have a very long list of schemas all in the ++ * same major version series? We'd return 0 for that, which means ++ * we would still run this function every time. ++ */ ++ static bool found = false; + static int best = 0; +- int i = 0; ++ int i; ++ GList *best_node = NULL; + struct schema_s *best_schema = NULL; + +- if (best != 0) { ++ if (found) { + return best; + } + ++ CRM_ASSERT(known_schemas != NULL); ++ + /* Get the most recent schema so we can look at its version number. */ + best = xml_latest_schema_index(); +- best_schema = g_list_nth(known_schemas, best)->data; ++ best_node = g_list_nth(known_schemas, best); ++ best_schema = best_node->data; ++ ++ /* If this is a singleton list, we're done. */ ++ if (pcmk__list_of_1(known_schemas)) { ++ goto done; ++ } + +- /* Iterate over the schema list until we find a schema with the same major +- * version as best, and with a minor version number of 0. +- * +- * This assumes that the first schema in a major series is always X.0, +- * which seems like a safe assumption. ++ /* Start comparing the list from the node before the best schema (there's ++ * no point in comparing something to itself). Then, 'i' is an index ++ * starting at the best schema and will always point at the node after ++ * 'iter'. This makes it the value we want to return when we find what ++ * we're looking for. + */ +- for (GList *iter = known_schemas; iter != NULL; iter = iter->next) { ++ i = best; ++ ++ for (GList *iter = best_node->prev; iter != NULL; iter = iter->prev) { + struct schema_s *schema = iter->data; + +- /* If we hit the initial best schema, the only things left in the list +- * are "pacemaker-next" and "none" which aren't worth checking. ++ /* We've found a schema in an older major version series. Return ++ * the index of the first one in the same major version series as ++ * the best schema. + */ +- if (schema == best_schema) { +- break; +- } +- +- if (schema->version.v[0] == best_schema->version.v[0] && +- schema->version.v[1] == 0) { ++ if (schema->version.v[0] < best_schema->version.v[0]) { + best = i; +- return best; ++ goto done; ++ ++ /* We're out of list to examine. This probably means there was only ++ * one major version series, so return index 0. ++ */ ++ } else if (iter->prev == NULL) { ++ best = 0; ++ goto done; + } + +- i++; ++ i--; + } + +- /* If we got here, we never found a match. Just return the latest. */ +- best = xml_latest_schema_index(); ++done: ++ found = true; + return best; + } + +-- +2.31.1 + +From eeeb36338f48d40f9f15a51c18aeca533b6c260d Mon Sep 17 00:00:00 2001 +From: Chris Lumens +Date: Fri, 17 Nov 2023 11:18:34 -0500 +Subject: [PATCH 17/20] Refactor: libcrmcommon: Add a parameter to a couple + schema functions. + +Instead of assuming known_schemas, pass the list to use as a parameter. +--- + lib/common/schemas.c | 22 +++++++++++----------- + 1 file changed, 11 insertions(+), 11 deletions(-) + +diff --git a/lib/common/schemas.c b/lib/common/schemas.c +index 466ad5a..9d98695 100644 +--- a/lib/common/schemas.c ++++ b/lib/common/schemas.c +@@ -73,17 +73,17 @@ xml_log(int priority, const char *fmt, ...) + } + + static int +-xml_latest_schema_index(void) ++xml_latest_schema_index(GList *schemas) + { + // @COMPAT: pacemaker-next is deprecated since 2.1.5 + // FIXME: This function assumes at least three schemas have been added + // before it has been called for the first time. +- return g_list_length(known_schemas) - 3; // index from 0, ignore "pacemaker-next"/"none" ++ return g_list_length(schemas) - 3; // index from 0, ignore "pacemaker-next"/"none" + } + + /* Return the index of the most recent X.0 schema. */ + static int +-xml_find_x_0_schema_index(void) ++xml_find_x_0_schema_index(GList *schemas) + { + /* We can't just use best to determine whether we've found the index + * or not. What if we have a very long list of schemas all in the +@@ -100,15 +100,15 @@ xml_find_x_0_schema_index(void) + return best; + } + +- CRM_ASSERT(known_schemas != NULL); ++ CRM_ASSERT(schemas != NULL); + + /* Get the most recent schema so we can look at its version number. */ +- best = xml_latest_schema_index(); +- best_node = g_list_nth(known_schemas, best); ++ best = xml_latest_schema_index(schemas); ++ best_node = g_list_nth(schemas, best); + best_schema = best_node->data; + + /* If this is a singleton list, we're done. */ +- if (pcmk__list_of_1(known_schemas)) { ++ if (pcmk__list_of_1(schemas)) { + goto done; + } + +@@ -150,7 +150,7 @@ done: + const char * + xml_latest_schema(void) + { +- return get_schema_name(xml_latest_schema_index()); ++ return get_schema_name(xml_latest_schema_index(known_schemas)); + } + + static inline bool +@@ -1010,7 +1010,7 @@ update_validation(xmlNode **xml_blob, int *best, int max, gboolean transform, + { + xmlNode *xml = NULL; + char *value = NULL; +- int max_stable_schemas = xml_latest_schema_index(); ++ int max_stable_schemas = xml_latest_schema_index(known_schemas); + int lpc = 0, match = -1, rc = pcmk_ok; + int next = -1; /* -1 denotes "inactive" value */ + xmlRelaxNGValidityErrorFunc error_handler = +@@ -1173,7 +1173,7 @@ cli_config_update(xmlNode **xml, int *best_version, gboolean to_logs) + + int version = get_schema_version(value); + int orig_version = version; +- int min_version = xml_find_x_0_schema_index(); ++ int min_version = xml_find_x_0_schema_index(known_schemas); + + if (version < min_version) { + // Current configuration schema is not acceptable, try to update +@@ -1235,7 +1235,7 @@ cli_config_update(xmlNode **xml, int *best_version, gboolean to_logs) + free_xml(*xml); + *xml = converted; + +- if (version < xml_latest_schema_index()) { ++ if (version < xml_latest_schema_index(known_schemas)) { + if (to_logs) { + pcmk__config_warn("Configuration with schema %s was " + "internally upgraded to acceptable (but " +-- +2.31.1 + +From 12e7b982da61c6cc6cf01164d45bb8f7b0255a8a Mon Sep 17 00:00:00 2001 +From: Chris Lumens +Date: Fri, 17 Nov 2023 11:30:18 -0500 +Subject: [PATCH 18/20] Refactor: libcrmcommon: Rename several schema-related + types. + +Give them pcmk__ names indicating they are private. This is in +preparation for moving them out into a header file. +--- + lib/common/schemas.c | 80 ++++++++++++++++++++++---------------------- + 1 file changed, 40 insertions(+), 40 deletions(-) + +diff --git a/lib/common/schemas.c b/lib/common/schemas.c +index 9d98695..cf8f325 100644 +--- a/lib/common/schemas.c ++++ b/lib/common/schemas.c +@@ -28,7 +28,7 @@ + + typedef struct { + unsigned char v[2]; +-} schema_version_t; ++} pcmk__schema_version_t; + + #define SCHEMA_ZERO { .v = { 0, 0 } } + +@@ -41,20 +41,20 @@ typedef struct { + xmlRelaxNGParserCtxtPtr parser; + } relaxng_ctx_cache_t; + +-enum schema_validator_e { +- schema_validator_none, +- schema_validator_rng ++enum pcmk__schema_validator { ++ pcmk__schema_validator_none, ++ pcmk__schema_validator_rng + }; + +-struct schema_s { ++typedef struct { + char *name; + char *transform; + void *cache; +- enum schema_validator_e validator; +- schema_version_t version; ++ enum pcmk__schema_validator validator; ++ pcmk__schema_version_t version; + char *transform_enter; + bool transform_onleave; +-}; ++} pcmk__schema_t; + + static GList *known_schemas = NULL; + static bool silent_logging = FALSE; +@@ -94,7 +94,7 @@ xml_find_x_0_schema_index(GList *schemas) + static int best = 0; + int i; + GList *best_node = NULL; +- struct schema_s *best_schema = NULL; ++ pcmk__schema_t *best_schema = NULL; + + if (found) { + return best; +@@ -121,7 +121,7 @@ xml_find_x_0_schema_index(GList *schemas) + i = best; + + for (GList *iter = best_node->prev; iter != NULL; iter = iter->prev) { +- struct schema_s *schema = iter->data; ++ pcmk__schema_t *schema = iter->data; + + /* We've found a schema in an older major version series. Return + * the index of the first one in the same major version series as +@@ -154,7 +154,7 @@ xml_latest_schema(void) + } + + static inline bool +-version_from_filename(const char *filename, schema_version_t *version) ++version_from_filename(const char *filename, pcmk__schema_version_t *version) + { + return sscanf(filename, "pacemaker-%hhu.%hhu.rng", &(version->v[0]), &(version->v[1])) == 2; + } +@@ -163,7 +163,7 @@ static int + schema_filter(const struct dirent *a) + { + int rc = 0; +- schema_version_t version = SCHEMA_ZERO; ++ pcmk__schema_version_t version = SCHEMA_ZERO; + + if (strstr(a->d_name, "pacemaker-") != a->d_name) { + /* crm_trace("%s - wrong prefix", a->d_name); */ +@@ -185,8 +185,8 @@ schema_filter(const struct dirent *a) + static int + schema_sort(const struct dirent **a, const struct dirent **b) + { +- schema_version_t a_version = SCHEMA_ZERO; +- schema_version_t b_version = SCHEMA_ZERO; ++ pcmk__schema_version_t a_version = SCHEMA_ZERO; ++ pcmk__schema_version_t b_version = SCHEMA_ZERO; + + if (!version_from_filename(a[0]->d_name, &a_version) + || !version_from_filename(b[0]->d_name, &b_version)) { +@@ -212,14 +212,14 @@ schema_sort(const struct dirent **a, const struct dirent **b) + * through \c add_schema_by_version. + */ + static void +-add_schema(enum schema_validator_e validator, const schema_version_t *version, ++add_schema(enum pcmk__schema_validator validator, const pcmk__schema_version_t *version, + const char *name, const char *transform, + const char *transform_enter, bool transform_onleave) + { +- struct schema_s *schema = NULL; ++ pcmk__schema_t *schema = NULL; + int last = g_list_length(known_schemas); + +- schema = calloc(1, sizeof(struct schema_s)); ++ schema = calloc(1, sizeof(pcmk__schema_t)); + CRM_ASSERT(schema != NULL); + + schema->validator = validator; +@@ -284,7 +284,7 @@ add_schema(enum schema_validator_e validator, const schema_version_t *version, + * . name convention: (see "upgrade-enter") + */ + static int +-add_schema_by_version(const schema_version_t *version, bool transform_expected) ++add_schema_by_version(const pcmk__schema_version_t *version, bool transform_expected) + { + bool transform_onleave = FALSE; + int rc = pcmk_rc_ok; +@@ -343,7 +343,7 @@ add_schema_by_version(const schema_version_t *version, bool transform_expected) + rc = ENOENT; + } + +- add_schema(schema_validator_rng, version, NULL, ++ add_schema(pcmk__schema_validator_rng, version, NULL, + transform_upgrade, transform_enter, transform_onleave); + + free(transform_upgrade); +@@ -397,7 +397,7 @@ crm_schema_init(void) + int lpc, max; + char *base = pcmk__xml_artefact_root(pcmk__xml_artefact_ns_legacy_rng); + struct dirent **namelist = NULL; +- const schema_version_t zero = SCHEMA_ZERO; ++ const pcmk__schema_version_t zero = SCHEMA_ZERO; + + wrap_libxslt(false); + +@@ -410,7 +410,7 @@ crm_schema_init(void) + free(base); + for (lpc = 0; lpc < max; lpc++) { + bool transform_expected = FALSE; +- schema_version_t version = SCHEMA_ZERO; ++ pcmk__schema_version_t version = SCHEMA_ZERO; + + if (!version_from_filename(namelist[lpc]->d_name, &version)) { + // Shouldn't be possible, but makes static analysis happy +@@ -419,7 +419,7 @@ crm_schema_init(void) + continue; + } + if ((lpc + 1) < max) { +- schema_version_t next_version = SCHEMA_ZERO; ++ pcmk__schema_version_t next_version = SCHEMA_ZERO; + + if (version_from_filename(namelist[lpc+1]->d_name, &next_version) + && (version.v[0] < next_version.v[0])) { +@@ -440,10 +440,10 @@ crm_schema_init(void) + } + + // @COMPAT: Deprecated since 2.1.5 +- add_schema(schema_validator_rng, &zero, "pacemaker-next", ++ add_schema(pcmk__schema_validator_rng, &zero, "pacemaker-next", + NULL, NULL, FALSE); + +- add_schema(schema_validator_none, &zero, PCMK__VALUE_NONE, ++ add_schema(pcmk__schema_validator_none, &zero, PCMK__VALUE_NONE, + NULL, NULL, FALSE); + } + +@@ -533,14 +533,14 @@ validate_with_relaxng(xmlDocPtr doc, xmlRelaxNGValidityErrorFunc error_handler, + static void + free_schema(gpointer data) + { +- struct schema_s *schema = data; ++ pcmk__schema_t *schema = data; + relaxng_ctx_cache_t *ctx = NULL; + + switch (schema->validator) { +- case schema_validator_none: // not cached ++ case pcmk__schema_validator_none: // not cached + break; + +- case schema_validator_rng: // cached ++ case pcmk__schema_validator_rng: // cached + ctx = (relaxng_ctx_cache_t *) schema->cache; + if (ctx == NULL) { + break; +@@ -582,7 +582,7 @@ crm_schema_cleanup(void) + } + + static gboolean +-validate_with(xmlNode *xml, struct schema_s *schema, xmlRelaxNGValidityErrorFunc error_handler, void* error_handler_context) ++validate_with(xmlNode *xml, pcmk__schema_t *schema, xmlRelaxNGValidityErrorFunc error_handler, void* error_handler_context) + { + gboolean valid = FALSE; + char *file = NULL; +@@ -592,7 +592,7 @@ validate_with(xmlNode *xml, struct schema_s *schema, xmlRelaxNGValidityErrorFunc + return FALSE; + } + +- if (schema->validator == schema_validator_none) { ++ if (schema->validator == pcmk__schema_validator_none) { + return TRUE; + } + +@@ -607,7 +607,7 @@ validate_with(xmlNode *xml, struct schema_s *schema, xmlRelaxNGValidityErrorFunc + crm_trace("Validating with %s (type=%d)", + pcmk__s(file, "missing schema"), schema->validator); + switch (schema->validator) { +- case schema_validator_rng: ++ case pcmk__schema_validator_rng: + cache = (relaxng_ctx_cache_t **) &(schema->cache); + valid = validate_with_relaxng(xml->doc, error_handler, error_handler_context, file, cache); + break; +@@ -621,7 +621,7 @@ validate_with(xmlNode *xml, struct schema_s *schema, xmlRelaxNGValidityErrorFunc + } + + static bool +-validate_with_silent(xmlNode *xml, struct schema_s *schema) ++validate_with_silent(xmlNode *xml, pcmk__schema_t *schema) + { + bool rc, sl_backup = silent_logging; + silent_logging = TRUE; +@@ -710,7 +710,7 @@ pcmk__validate_xml(xmlNode *xml_blob, const char *validation, xmlRelaxNGValidity + bool valid = FALSE; + + for (GList *iter = known_schemas; iter != NULL; iter = iter->next) { +- struct schema_s *schema = iter->data; ++ pcmk__schema_t *schema = iter->data; + + if (validate_with(xml_blob, schema, NULL, NULL)) { + valid = TRUE; +@@ -726,7 +726,7 @@ pcmk__validate_xml(xmlNode *xml_blob, const char *validation, xmlRelaxNGValidity + if (strcmp(validation, PCMK__VALUE_NONE) == 0) { + return TRUE; + } else if (version < g_list_length(known_schemas)) { +- struct schema_s *schema = g_list_nth_data(known_schemas, version); ++ pcmk__schema_t *schema = g_list_nth_data(known_schemas, version); + return validate_with(xml_blob, schema, error_handler, + error_handler_context); + } +@@ -918,7 +918,7 @@ apply_transformation(xmlNode *xml, const char *transform, gboolean to_logs) + * \note Only emits warnings about enter/leave phases in case of issues. + */ + static xmlNode * +-apply_upgrade(xmlNode *xml, const struct schema_s *schema, gboolean to_logs) ++apply_upgrade(xmlNode *xml, const pcmk__schema_t *schema, gboolean to_logs) + { + bool transform_onleave = schema->transform_onleave; + char *transform_leave; +@@ -972,7 +972,7 @@ apply_upgrade(xmlNode *xml, const struct schema_s *schema, gboolean to_logs) + const char * + get_schema_name(int version) + { +- struct schema_s *schema = g_list_nth_data(known_schemas, version); ++ pcmk__schema_t *schema = g_list_nth_data(known_schemas, version); + + if (schema == NULL) { + return "unknown"; +@@ -991,7 +991,7 @@ get_schema_version(const char *name) + } + + for (GList *iter = known_schemas; iter != NULL; iter = iter->next) { +- struct schema_s *schema = iter->data; ++ pcmk__schema_t *schema = iter->data; + + if (pcmk__str_eq(name, schema->name, pcmk__str_casei)) { + return lpc; +@@ -1052,7 +1052,7 @@ update_validation(xmlNode **xml_blob, int *best, int max, gboolean transform, + * easier than trying to get all the loop indices we're using here + * sorted out and working correctly. + */ +- struct schema_s *schema = g_list_nth_data(known_schemas, lpc); ++ pcmk__schema_t *schema = g_list_nth_data(known_schemas, lpc); + + crm_debug("Testing '%s' validation (%d of %d)", + pcmk__s(schema->name, ""), lpc, max_stable_schemas); +@@ -1085,7 +1085,7 @@ update_validation(xmlNode **xml_blob, int *best, int max, gboolean transform, + + if (rc == pcmk_ok && transform) { + xmlNode *upgrade = NULL; +- struct schema_s *next_schema = NULL; ++ pcmk__schema_t *next_schema = NULL; + next = lpc+1; + + if (next > max_stable_schemas) { +@@ -1151,7 +1151,7 @@ update_validation(xmlNode **xml_blob, int *best, int max, gboolean transform, + } + + if (*best > match && *best) { +- struct schema_s *best_schema = g_list_nth_data(known_schemas, *best); ++ pcmk__schema_t *best_schema = g_list_nth_data(known_schemas, *best); + + crm_info("%s the configuration from %s to %s", + transform?"Transformed":"Upgraded", pcmk__s(value, ""), +@@ -1279,7 +1279,7 @@ pcmk__log_known_schemas(void) + int lpc = 0; + + for (GList *iter = known_schemas; iter != NULL; iter = iter->next) { +- struct schema_s *schema = iter->data; ++ pcmk__schema_t *schema = iter->data; + + if (schema->transform != NULL) { + crm_debug("known_schemas[%d] => %s (upgrades with %s.xsl)", +-- +2.31.1 + +From 97b3fa3462039d4d7bdad6c6ff328a5124977e5f Mon Sep 17 00:00:00 2001 +From: Chris Lumens +Date: Fri, 17 Nov 2023 11:32:55 -0500 +Subject: [PATCH 19/20] Refactor: libcrmcommon: Make various schema stuff + non-static. + +This is the minimum amount necessary to make the function unit testable. +None of this is intended to ever become public. +--- + lib/common/crmcommon_private.h | 26 ++++++++++++++++++++++++++ + lib/common/schemas.c | 25 ++++--------------------- + 2 files changed, 30 insertions(+), 21 deletions(-) + +diff --git a/lib/common/crmcommon_private.h b/lib/common/crmcommon_private.h +index 121d663..6ab9de1 100644 +--- a/lib/common/crmcommon_private.h ++++ b/lib/common/crmcommon_private.h +@@ -283,4 +283,30 @@ void pcmk__register_patchset_messages(pcmk__output_t *out); + #define PCMK__PW_BUFFER_LEN 500 + + ++/* ++ * Schemas ++ */ ++typedef struct { ++ unsigned char v[2]; ++} pcmk__schema_version_t; ++ ++enum pcmk__schema_validator { ++ pcmk__schema_validator_none, ++ pcmk__schema_validator_rng ++}; ++ ++typedef struct { ++ char *name; ++ char *transform; ++ void *cache; ++ enum pcmk__schema_validator validator; ++ pcmk__schema_version_t version; ++ char *transform_enter; ++ bool transform_onleave; ++} pcmk__schema_t; ++ ++G_GNUC_INTERNAL ++int pcmk__find_x_0_schema_index(GList *schemas); ++ ++ + #endif // CRMCOMMON_PRIVATE__H +diff --git a/lib/common/schemas.c b/lib/common/schemas.c +index cf8f325..83334b4 100644 +--- a/lib/common/schemas.c ++++ b/lib/common/schemas.c +@@ -26,9 +26,7 @@ + #include + #include /* PCMK__XML_LOG_BASE */ + +-typedef struct { +- unsigned char v[2]; +-} pcmk__schema_version_t; ++#include "crmcommon_private.h" + + #define SCHEMA_ZERO { .v = { 0, 0 } } + +@@ -41,21 +39,6 @@ typedef struct { + xmlRelaxNGParserCtxtPtr parser; + } relaxng_ctx_cache_t; + +-enum pcmk__schema_validator { +- pcmk__schema_validator_none, +- pcmk__schema_validator_rng +-}; +- +-typedef struct { +- char *name; +- char *transform; +- void *cache; +- enum pcmk__schema_validator validator; +- pcmk__schema_version_t version; +- char *transform_enter; +- bool transform_onleave; +-} pcmk__schema_t; +- + static GList *known_schemas = NULL; + static bool silent_logging = FALSE; + +@@ -82,8 +65,8 @@ xml_latest_schema_index(GList *schemas) + } + + /* Return the index of the most recent X.0 schema. */ +-static int +-xml_find_x_0_schema_index(GList *schemas) ++int ++pcmk__find_x_0_schema_index(GList *schemas) + { + /* We can't just use best to determine whether we've found the index + * or not. What if we have a very long list of schemas all in the +@@ -1173,7 +1156,7 @@ cli_config_update(xmlNode **xml, int *best_version, gboolean to_logs) + + int version = get_schema_version(value); + int orig_version = version; +- int min_version = xml_find_x_0_schema_index(known_schemas); ++ int min_version = pcmk__find_x_0_schema_index(known_schemas); + + if (version < min_version) { + // Current configuration schema is not acceptable, try to update +-- +2.31.1 + +From c4c093c0785e06ca3371556b19ede1d5b44d090a Mon Sep 17 00:00:00 2001 +From: Chris Lumens +Date: Fri, 17 Nov 2023 12:39:53 -0500 +Subject: [PATCH 20/20] Test: libcrmcommon: Add unit tests for + pcmk__xml_find_x_0_schema_index. + +This requires making various things in the function conditional, which I +kind of hate. But, it all comes down to the fact that when we are +running for real, we're adding the pacemaker-next/none schemas with +crm_schema_init. + +When we are unit testing, we aren't doing any of that. The only +"schemas" we have are the ones we are adding directly. So, the list has +two items fewer than the real function expects. I think this is okay +and doesn't totally invalidating the testing. +--- + configure.ac | 1 + + lib/common/schemas.c | 33 +++++- + lib/common/tests/Makefile.am | 1 + + lib/common/tests/schemas/Makefile.am | 16 +++ + .../pcmk__xml_find_x_0_schema_index_test.c | 112 ++++++++++++++++++ + 5 files changed, 161 insertions(+), 2 deletions(-) + create mode 100644 lib/common/tests/schemas/Makefile.am + create mode 100644 lib/common/tests/schemas/pcmk__xml_find_x_0_schema_index_test.c + +diff --git a/configure.ac b/configure.ac +index 6bff02e..9eb7539 100644 +--- a/configure.ac ++++ b/configure.ac +@@ -2153,6 +2153,7 @@ AC_CONFIG_FILES(Makefile \ + lib/common/tests/output/Makefile \ + lib/common/tests/procfs/Makefile \ + lib/common/tests/results/Makefile \ ++ lib/common/tests/schemas/Makefile \ + lib/common/tests/scores/Makefile \ + lib/common/tests/strings/Makefile \ + lib/common/tests/utils/Makefile \ +diff --git a/lib/common/schemas.c b/lib/common/schemas.c +index 83334b4..372e872 100644 +--- a/lib/common/schemas.c ++++ b/lib/common/schemas.c +@@ -60,8 +60,13 @@ xml_latest_schema_index(GList *schemas) + { + // @COMPAT: pacemaker-next is deprecated since 2.1.5 + // FIXME: This function assumes at least three schemas have been added +- // before it has been called for the first time. ++ // before it has been called for the first time, which is only the case ++ // if we are not unit testing. ++#if defined(PCMK__UNIT_TESTING) ++ return g_list_length(schemas) - 1; // index from 0 ++#else + return g_list_length(schemas) - 3; // index from 0, ignore "pacemaker-next"/"none" ++#endif + } + + /* Return the index of the most recent X.0 schema. */ +@@ -73,8 +78,17 @@ pcmk__find_x_0_schema_index(GList *schemas) + * same major version series? We'd return 0 for that, which means + * we would still run this function every time. + */ ++#if defined(PCMK__UNIT_TESTING) ++ /* If we're unit testing, these can't be static because they'll stick ++ * around from one test run to the next. They need to be cleared out ++ * every time. ++ */ ++ bool found = false; ++ int best = 0; ++#else + static bool found = false; + static int best = 0; ++#endif + int i; + GList *best_node = NULL; + pcmk__schema_t *best_schema = NULL; +@@ -90,10 +104,25 @@ pcmk__find_x_0_schema_index(GList *schemas) + best_node = g_list_nth(schemas, best); + best_schema = best_node->data; + +- /* If this is a singleton list, we're done. */ ++ /* If we are unit testing, we don't add the pacemaker-next/none schemas ++ * to the list because we're not using the standard schema adding ++ * functions. Thus, a singleton list means we're done. ++ * ++ * On the other hand, if we are running as usually, we have those two ++ * schemas added to the list. A list of length three actually only has ++ * one useful schema. So we're still done. ++ * ++ * @COMPAT Change this when we stop adding those schemas. ++ */ ++#if defined(PCMK__UNIT_TESTING) + if (pcmk__list_of_1(schemas)) { + goto done; + } ++#else ++ if (g_list_length(schemas) == 3) { ++ goto done; ++ } ++#endif + + /* Start comparing the list from the node before the best schema (there's + * no point in comparing something to itself). Then, 'i' is an index +diff --git a/lib/common/tests/Makefile.am b/lib/common/tests/Makefile.am +index c0407e5..22fb32e 100644 +--- a/lib/common/tests/Makefile.am ++++ b/lib/common/tests/Makefile.am +@@ -21,6 +21,7 @@ SUBDIRS = \ + options \ + output \ + results \ ++ schemas \ + scores \ + strings \ + utils \ +diff --git a/lib/common/tests/schemas/Makefile.am b/lib/common/tests/schemas/Makefile.am +new file mode 100644 +index 0000000..5f485b3 +--- /dev/null ++++ b/lib/common/tests/schemas/Makefile.am +@@ -0,0 +1,16 @@ ++# ++# Copyright 2023 the Pacemaker project contributors ++# ++# The version control history for this file may have further details. ++# ++# This source code is licensed under the GNU General Public License version 2 ++# or later (GPLv2+) WITHOUT ANY WARRANTY. ++# ++ ++include $(top_srcdir)/mk/tap.mk ++include $(top_srcdir)/mk/unittest.mk ++ ++# Add "_test" to the end of all test program names to simplify .gitignore. ++check_PROGRAMS = pcmk__xml_find_x_0_schema_index_test ++ ++TESTS = $(check_PROGRAMS) +diff --git a/lib/common/tests/schemas/pcmk__xml_find_x_0_schema_index_test.c b/lib/common/tests/schemas/pcmk__xml_find_x_0_schema_index_test.c +new file mode 100644 +index 0000000..9f16ba1 +--- /dev/null ++++ b/lib/common/tests/schemas/pcmk__xml_find_x_0_schema_index_test.c +@@ -0,0 +1,112 @@ ++/* ++ * Copyright 2023 the Pacemaker project contributors ++ * ++ * The version control history for this file may have further details. ++ * ++ * This source code is licensed under the GNU General Public License version 2 ++ * or later (GPLv2+) WITHOUT ANY WARRANTY. ++ */ ++ ++#include ++#include ++ ++#include ++ ++#include "crmcommon_private.h" ++ ++static pcmk__schema_t * ++mk_schema(const char *name, unsigned char x, unsigned char y) ++{ ++ pcmk__schema_t *schema = malloc(sizeof(pcmk__schema_t)); ++ ++ schema->name = strdup(name); ++ schema->version.v[0] = x; ++ schema->version.v[1] = y; ++ return schema; ++} ++ ++static void ++free_schema(void *data) ++{ ++ pcmk__schema_t *schema = data; ++ free(schema->name); ++ free(schema); ++} ++ ++static void ++empty_schema_list(void **state) ++{ ++ pcmk__assert_asserts(pcmk__find_x_0_schema_index(NULL)); ++} ++ ++static void ++singleton_schema_list(void **state) ++{ ++ GList *schemas = NULL; ++ ++ schemas = g_list_append(schemas, mk_schema("pacemaker-1.0", 1, 0)); ++ assert_int_equal(0, pcmk__find_x_0_schema_index(schemas)); ++ g_list_free_full(schemas, free_schema); ++} ++ ++static void ++one_major_version(void **state) ++{ ++ GList *schemas = NULL; ++ ++ schemas = g_list_append(schemas, mk_schema("pacemaker-1.0", 1, 0)); ++ schemas = g_list_append(schemas, mk_schema("pacemaker-1.2", 1, 2)); ++ schemas = g_list_append(schemas, mk_schema("pacemaker-1.3", 1, 3)); ++ assert_int_equal(0, pcmk__find_x_0_schema_index(schemas)); ++ g_list_free_full(schemas, free_schema); ++} ++ ++static void ++first_version_is_not_0(void **state) ++{ ++ GList *schemas = NULL; ++ ++ schemas = g_list_append(schemas, mk_schema("pacemaker-1.1", 1, 1)); ++ schemas = g_list_append(schemas, mk_schema("pacemaker-1.2", 1, 2)); ++ schemas = g_list_append(schemas, mk_schema("pacemaker-1.3", 1, 3)); ++ assert_int_equal(0, pcmk__find_x_0_schema_index(schemas)); ++ g_list_free_full(schemas, free_schema); ++} ++ ++static void ++multiple_major_versions(void **state) ++{ ++ GList *schemas = NULL; ++ ++ schemas = g_list_append(schemas, mk_schema("pacemaker-1.0", 1, 0)); ++ schemas = g_list_append(schemas, mk_schema("pacemaker-1.1", 1, 1)); ++ schemas = g_list_append(schemas, mk_schema("pacemaker-2.0", 2, 0)); ++ assert_int_equal(2, pcmk__find_x_0_schema_index(schemas)); ++ g_list_free_full(schemas, free_schema); ++} ++ ++static void ++many_versions(void **state) ++{ ++ GList *schemas = NULL; ++ ++ schemas = g_list_append(schemas, mk_schema("pacemaker-1.0", 1, 0)); ++ schemas = g_list_append(schemas, mk_schema("pacemaker-1.1", 1, 1)); ++ schemas = g_list_append(schemas, mk_schema("pacemaker-1.2", 1, 2)); ++ schemas = g_list_append(schemas, mk_schema("pacemaker-2.0", 2, 0)); ++ schemas = g_list_append(schemas, mk_schema("pacemaker-2.1", 2, 1)); ++ schemas = g_list_append(schemas, mk_schema("pacemaker-2.2", 2, 2)); ++ schemas = g_list_append(schemas, mk_schema("pacemaker-3.0", 3, 0)); ++ schemas = g_list_append(schemas, mk_schema("pacemaker-3.1", 3, 1)); ++ schemas = g_list_append(schemas, mk_schema("pacemaker-3.2", 3, 2)); ++ assert_int_equal(6, pcmk__find_x_0_schema_index(schemas)); ++ g_list_free_full(schemas, free_schema); ++} ++ ++PCMK__UNIT_TEST(NULL, NULL, ++ cmocka_unit_test(empty_schema_list), ++ cmocka_unit_test(singleton_schema_list), ++ cmocka_unit_test(one_major_version), ++ cmocka_unit_test(first_version_is_not_0), ++ cmocka_unit_test(multiple_major_versions), ++ cmocka_unit_test(many_versions)) +-- +2.31.1 + diff --git a/SOURCES/002-group-colocation-constraint.patch b/SOURCES/002-group-colocation-constraint.patch deleted file mode 100644 index 4cd58c0..0000000 --- a/SOURCES/002-group-colocation-constraint.patch +++ /dev/null @@ -1,2661 +0,0 @@ -From 6d438daa021eaef4ca41b84009b9d6fc11173826 Mon Sep 17 00:00:00 2001 -From: Ken Gaillot -Date: Thu, 20 Apr 2023 11:01:41 -0500 -Subject: [PATCH 01/17] Refactor: scheduler: drop redundant argument from - pcmk__new_colocation() - ---- - lib/pacemaker/libpacemaker_private.h | 2 +- - lib/pacemaker/pcmk_sched_bundle.c | 5 ++--- - lib/pacemaker/pcmk_sched_colocation.c | 27 +++++++++++---------------- - lib/pacemaker/pcmk_sched_group.c | 3 +-- - lib/pacemaker/pcmk_sched_primitive.c | 3 +-- - 5 files changed, 16 insertions(+), 24 deletions(-) - -diff --git a/lib/pacemaker/libpacemaker_private.h b/lib/pacemaker/libpacemaker_private.h -index 192d5a703ff..a6c13220e1d 100644 ---- a/lib/pacemaker/libpacemaker_private.h -+++ b/lib/pacemaker/libpacemaker_private.h -@@ -483,7 +483,7 @@ G_GNUC_INTERNAL - void pcmk__new_colocation(const char *id, const char *node_attr, int score, - pe_resource_t *dependent, pe_resource_t *primary, - const char *dependent_role, const char *primary_role, -- bool influence, pe_working_set_t *data_set); -+ bool influence); - - G_GNUC_INTERNAL - void pcmk__block_colocation_dependents(pe_action_t *action, -diff --git a/lib/pacemaker/pcmk_sched_bundle.c b/lib/pacemaker/pcmk_sched_bundle.c -index 5682744395a..6024da68fb7 100644 ---- a/lib/pacemaker/pcmk_sched_bundle.c -+++ b/lib/pacemaker/pcmk_sched_bundle.c -@@ -83,7 +83,7 @@ pcmk__bundle_allocate(pe_resource_t *rsc, const pe_node_t *prefer) - pcmk__new_colocation("child-remote-with-docker-remote", NULL, - INFINITY, replica->remote, - container_host->details->remote_rsc, NULL, -- NULL, true, rsc->cluster); -+ NULL, true); - } - - if (replica->remote) { -@@ -252,8 +252,7 @@ pcmk__bundle_internal_constraints(pe_resource_t *rsc) - pe_order_implies_first|pe_order_preserve); - - pcmk__new_colocation("ip-with-docker", NULL, INFINITY, replica->ip, -- replica->container, NULL, NULL, true, -- rsc->cluster); -+ replica->container, NULL, NULL, true); - } - - if (replica->remote) { -diff --git a/lib/pacemaker/pcmk_sched_colocation.c b/lib/pacemaker/pcmk_sched_colocation.c -index eeef4f1ca55..7d41f4d03e5 100644 ---- a/lib/pacemaker/pcmk_sched_colocation.c -+++ b/lib/pacemaker/pcmk_sched_colocation.c -@@ -297,13 +297,12 @@ anti_colocation_order(pe_resource_t *first_rsc, int first_role, - * \param[in] dependent_role Current role of \p dependent - * \param[in] primary_role Current role of \p primary - * \param[in] influence Whether colocation constraint has influence -- * \param[in,out] data_set Cluster working set to add constraint to - */ - void - pcmk__new_colocation(const char *id, const char *node_attr, int score, - pe_resource_t *dependent, pe_resource_t *primary, - const char *dependent_role, const char *primary_role, -- bool influence, pe_working_set_t *data_set) -+ bool influence) - { - pcmk__colocation_t *new_con = NULL; - -@@ -351,8 +350,8 @@ pcmk__new_colocation(const char *id, const char *node_attr, int score, - pcmk__add_this_with(&(dependent->rsc_cons), new_con); - pcmk__add_with_this(&(primary->rsc_cons_lhs), new_con); - -- data_set->colocation_constraints = g_list_append(data_set->colocation_constraints, -- new_con); -+ dependent->cluster->colocation_constraints = g_list_append( -+ dependent->cluster->colocation_constraints, new_con); - - if (score <= -INFINITY) { - anti_colocation_order(dependent, new_con->dependent_role, primary, -@@ -433,7 +432,7 @@ unpack_colocation_set(xmlNode *set, int score, const char *coloc_id, - pcmk__new_colocation(set_id, NULL, local_score, resource, - with, role, role, - unpack_influence(coloc_id, resource, -- influence_s), data_set); -+ influence_s)); - } - with = resource; - } -@@ -451,7 +450,7 @@ unpack_colocation_set(xmlNode *set, int score, const char *coloc_id, - pcmk__new_colocation(set_id, NULL, local_score, last, - resource, role, role, - unpack_influence(coloc_id, last, -- influence_s), data_set); -+ influence_s)); - } - - last = resource; -@@ -484,8 +483,7 @@ unpack_colocation_set(xmlNode *set, int score, const char *coloc_id, - pe_rsc_trace(resource, "Anti-Colocating %s with %s", resource->id, - with->id); - pcmk__new_colocation(set_id, NULL, local_score, -- resource, with, role, role, -- influence, data_set); -+ resource, with, role, role, influence); - } - } - } -@@ -535,8 +533,7 @@ colocate_rsc_sets(const char *id, xmlNode *set1, xmlNode *set2, int score, - - if ((rsc_1 != NULL) && (rsc_2 != NULL)) { - pcmk__new_colocation(id, NULL, score, rsc_1, rsc_2, role_1, role_2, -- unpack_influence(id, rsc_1, influence_s), -- data_set); -+ unpack_influence(id, rsc_1, influence_s)); - - } else if (rsc_1 != NULL) { - bool influence = unpack_influence(id, rsc_1, influence_s); -@@ -546,7 +543,7 @@ colocate_rsc_sets(const char *id, xmlNode *set1, xmlNode *set2, int score, - - EXPAND_CONSTRAINT_IDREF(id, rsc_2, ID(xml_rsc)); - pcmk__new_colocation(id, NULL, score, rsc_1, rsc_2, role_1, -- role_2, influence, data_set); -+ role_2, influence); - } - - } else if (rsc_2 != NULL) { -@@ -556,8 +553,7 @@ colocate_rsc_sets(const char *id, xmlNode *set1, xmlNode *set2, int score, - EXPAND_CONSTRAINT_IDREF(id, rsc_1, ID(xml_rsc)); - pcmk__new_colocation(id, NULL, score, rsc_1, rsc_2, role_1, - role_2, -- unpack_influence(id, rsc_1, influence_s), -- data_set); -+ unpack_influence(id, rsc_1, influence_s)); - } - - } else { -@@ -576,8 +572,7 @@ colocate_rsc_sets(const char *id, xmlNode *set1, xmlNode *set2, int score, - - EXPAND_CONSTRAINT_IDREF(id, rsc_2, ID(xml_rsc_2)); - pcmk__new_colocation(id, NULL, score, rsc_1, rsc_2, -- role_1, role_2, influence, -- data_set); -+ role_1, role_2, influence); - } - } - } -@@ -678,7 +673,7 @@ unpack_simple_colocation(xmlNode *xml_obj, const char *id, - - pcmk__new_colocation(id, attr, score_i, dependent, primary, - dependent_role, primary_role, -- unpack_influence(id, dependent, influence_s), data_set); -+ unpack_influence(id, dependent, influence_s)); - } - - // \return Standard Pacemaker return code -diff --git a/lib/pacemaker/pcmk_sched_group.c b/lib/pacemaker/pcmk_sched_group.c -index cb139f7ddf9..c1392e07a4c 100644 ---- a/lib/pacemaker/pcmk_sched_group.c -+++ b/lib/pacemaker/pcmk_sched_group.c -@@ -171,8 +171,7 @@ member_internal_constraints(gpointer data, gpointer user_data) - // Colocate this member with the previous one - pcmk__new_colocation("group:internal_colocation", NULL, INFINITY, - member, member_data->previous_member, NULL, NULL, -- pcmk_is_set(member->flags, pe_rsc_critical), -- member->cluster); -+ pcmk_is_set(member->flags, pe_rsc_critical)); - } - - if (member_data->promotable) { -diff --git a/lib/pacemaker/pcmk_sched_primitive.c b/lib/pacemaker/pcmk_sched_primitive.c -index aefbf9aa140..4e3eca3e18a 100644 ---- a/lib/pacemaker/pcmk_sched_primitive.c -+++ b/lib/pacemaker/pcmk_sched_primitive.c -@@ -999,8 +999,7 @@ pcmk__primitive_internal_constraints(pe_resource_t *rsc) - score = INFINITY; /* Force them to run on the same host */ - } - pcmk__new_colocation("resource-with-container", NULL, score, rsc, -- rsc->container, NULL, NULL, true, -- rsc->cluster); -+ rsc->container, NULL, NULL, true); - } - } - - -From c6efbe4bc45795f6991b600fc0a70b6a46c10fc3 Mon Sep 17 00:00:00 2001 -From: Ken Gaillot -Date: Mon, 26 Jun 2023 11:50:57 -0500 -Subject: [PATCH 02/17] Low: scheduler: improve error-checking when creating - colocations - ---- - lib/pacemaker/pcmk_sched_colocation.c | 20 ++++++++++++-------- - 1 file changed, 12 insertions(+), 8 deletions(-) - -diff --git a/lib/pacemaker/pcmk_sched_colocation.c b/lib/pacemaker/pcmk_sched_colocation.c -index 7d41f4d03e5..d591550fb97 100644 ---- a/lib/pacemaker/pcmk_sched_colocation.c -+++ b/lib/pacemaker/pcmk_sched_colocation.c -@@ -306,21 +306,24 @@ pcmk__new_colocation(const char *id, const char *node_attr, int score, - { - pcmk__colocation_t *new_con = NULL; - -- if (score == 0) { -- crm_trace("Ignoring colocation '%s' because score is 0", id); -- return; -- } -+ CRM_CHECK(id != NULL, return); -+ - if ((dependent == NULL) || (primary == NULL)) { - pcmk__config_err("Ignoring colocation '%s' because resource " - "does not exist", id); - return; - } - -- new_con = calloc(1, sizeof(pcmk__colocation_t)); -- if (new_con == NULL) { -+ if (score == 0) { -+ pe_rsc_trace(dependent, -+ "Ignoring colocation '%s' (%s with %s) because score is 0", -+ id, dependent->id, primary->id); - return; - } - -+ new_con = calloc(1, sizeof(pcmk__colocation_t)); -+ CRM_ASSERT(new_con != NULL); -+ - if (pcmk__str_eq(dependent_role, RSC_ROLE_STARTED_S, - pcmk__str_null_matches|pcmk__str_casei)) { - dependent_role = RSC_ROLE_UNKNOWN_S; -@@ -344,8 +347,9 @@ pcmk__new_colocation(const char *id, const char *node_attr, int score, - node_attr = CRM_ATTR_UNAME; - } - -- pe_rsc_trace(dependent, "%s ==> %s (%s %d)", -- dependent->id, primary->id, node_attr, score); -+ pe_rsc_trace(dependent, "Added colocation %s (%s with %s @%s using %s)", -+ new_con->id, dependent->id, primary->id, -+ pcmk_readable_score(score), node_attr); - - pcmk__add_this_with(&(dependent->rsc_cons), new_con); - pcmk__add_with_this(&(primary->rsc_cons_lhs), new_con); - -From 589403f548459eeddfd5188ba70723ecf9987d2b Mon Sep 17 00:00:00 2001 -From: Ken Gaillot -Date: Mon, 26 Jun 2023 12:19:44 -0500 -Subject: [PATCH 03/17] Refactor: scheduler: use flag group instead of bool for - colocation influence - -... so we can add more flags ---- - include/pcmki/pcmki_scheduler.h | 2 +- - lib/pacemaker/libpacemaker_private.h | 13 +++++- - lib/pacemaker/pcmk_sched_bundle.c | 5 ++- - lib/pacemaker/pcmk_sched_colocation.c | 61 ++++++++++++++------------- - lib/pacemaker/pcmk_sched_group.c | 8 +++- - lib/pacemaker/pcmk_sched_primitive.c | 3 +- - 6 files changed, 55 insertions(+), 37 deletions(-) - -diff --git a/include/pcmki/pcmki_scheduler.h b/include/pcmki/pcmki_scheduler.h -index dde50a57e32..53de7e1f52e 100644 ---- a/include/pcmki/pcmki_scheduler.h -+++ b/include/pcmki/pcmki_scheduler.h -@@ -29,7 +29,7 @@ typedef struct { - int primary_role; // Colocation applies only if primary has this role - - int score; -- bool influence; // Whether dependent influences active primary placement -+ uint32_t flags; // Group of enum pcmk__coloc_flags - } pcmk__colocation_t; - - void pcmk__unpack_constraints(pe_working_set_t *data_set); -diff --git a/lib/pacemaker/libpacemaker_private.h b/lib/pacemaker/libpacemaker_private.h -index a6c13220e1d..51de9d3e9a9 100644 ---- a/lib/pacemaker/libpacemaker_private.h -+++ b/lib/pacemaker/libpacemaker_private.h -@@ -16,6 +16,14 @@ - - #include // pe_action_t, pe_node_t, pe_working_set_t - -+// Colocation flags -+enum pcmk__coloc_flags { -+ pcmk__coloc_none = 0U, -+ -+ // Primary is affected even if already active -+ pcmk__coloc_influence = (1U << 0), -+}; -+ - // Flags to modify the behavior of add_colocated_node_scores() - enum pcmk__coloc_select { - // With no other flags, apply all "with this" colocations -@@ -483,7 +491,7 @@ G_GNUC_INTERNAL - void pcmk__new_colocation(const char *id, const char *node_attr, int score, - pe_resource_t *dependent, pe_resource_t *primary, - const char *dependent_role, const char *primary_role, -- bool influence); -+ uint32_t flags); - - G_GNUC_INTERNAL - void pcmk__block_colocation_dependents(pe_action_t *action, -@@ -530,7 +538,8 @@ pcmk__colocation_has_influence(const pcmk__colocation_t *colocation, - /* The dependent in a colocation influences the primary's location - * if the influence option is true or the primary is not yet active. - */ -- return colocation->influence || (rsc->running_on == NULL); -+ return pcmk_is_set(colocation->flags, pcmk__coloc_influence) -+ || (rsc->running_on == NULL); - } - - -diff --git a/lib/pacemaker/pcmk_sched_bundle.c b/lib/pacemaker/pcmk_sched_bundle.c -index 6024da68fb7..ca3c21a9977 100644 ---- a/lib/pacemaker/pcmk_sched_bundle.c -+++ b/lib/pacemaker/pcmk_sched_bundle.c -@@ -83,7 +83,7 @@ pcmk__bundle_allocate(pe_resource_t *rsc, const pe_node_t *prefer) - pcmk__new_colocation("child-remote-with-docker-remote", NULL, - INFINITY, replica->remote, - container_host->details->remote_rsc, NULL, -- NULL, true); -+ NULL, pcmk__coloc_influence); - } - - if (replica->remote) { -@@ -252,7 +252,8 @@ pcmk__bundle_internal_constraints(pe_resource_t *rsc) - pe_order_implies_first|pe_order_preserve); - - pcmk__new_colocation("ip-with-docker", NULL, INFINITY, replica->ip, -- replica->container, NULL, NULL, true); -+ replica->container, NULL, NULL, -+ pcmk__coloc_influence); - } - - if (replica->remote) { -diff --git a/lib/pacemaker/pcmk_sched_colocation.c b/lib/pacemaker/pcmk_sched_colocation.c -index d591550fb97..dbdefadfd10 100644 ---- a/lib/pacemaker/pcmk_sched_colocation.c -+++ b/lib/pacemaker/pcmk_sched_colocation.c -@@ -296,13 +296,13 @@ anti_colocation_order(pe_resource_t *first_rsc, int first_role, - * \param[in,out] primary Resource to colocate \p dependent with - * \param[in] dependent_role Current role of \p dependent - * \param[in] primary_role Current role of \p primary -- * \param[in] influence Whether colocation constraint has influence -+ * \param[in] flags Group of enum pcmk__coloc_flags - */ - void - pcmk__new_colocation(const char *id, const char *node_attr, int score, - pe_resource_t *dependent, pe_resource_t *primary, - const char *dependent_role, const char *primary_role, -- bool influence) -+ uint32_t flags) - { - pcmk__colocation_t *new_con = NULL; - -@@ -341,7 +341,7 @@ pcmk__new_colocation(const char *id, const char *node_attr, int score, - new_con->dependent_role = text2role(dependent_role); - new_con->primary_role = text2role(primary_role); - new_con->node_attribute = node_attr; -- new_con->influence = influence; -+ new_con->flags = flags; - - if (node_attr == NULL) { - node_attr = CRM_ATTR_UNAME; -@@ -373,10 +373,11 @@ pcmk__new_colocation(const char *id, const char *node_attr, int score, - * \param[in] rsc Resource involved in constraint (for default) - * \param[in] influence_s String value of influence option - * -- * \return true if string evaluates true, false if string evaluates false, -- * or value of resource's critical option if string is NULL or invalid -+ * \return pcmk__coloc_influence if string evaluates true, or string is NULL or -+ * invalid and resource's critical option evaluates true, otherwise -+ * pcmk__coloc_none - */ --static bool -+static uint32_t - unpack_influence(const char *coloc_id, const pe_resource_t *rsc, - const char *influence_s) - { -@@ -388,10 +389,13 @@ unpack_influence(const char *coloc_id, const pe_resource_t *rsc, - XML_COLOC_ATTR_INFLUENCE " (using default)", - coloc_id); - } else { -- return (influence_i != 0); -+ return (influence_i == 0)? pcmk__coloc_none : pcmk__coloc_influence; - } - } -- return pcmk_is_set(rsc->flags, pe_rsc_critical); -+ if (pcmk_is_set(rsc->flags, pe_rsc_critical)) { -+ return pcmk__coloc_influence; -+ } -+ return pcmk__coloc_none; - } - - static void -@@ -406,7 +410,7 @@ unpack_colocation_set(xmlNode *set, int score, const char *coloc_id, - const char *ordering = crm_element_value(set, "ordering"); - int local_score = score; - bool sequential = false; -- -+ uint32_t flags = pcmk__coloc_none; - const char *score_s = crm_element_value(set, XML_RULE_ATTR_SCORE); - - if (score_s) { -@@ -433,10 +437,9 @@ unpack_colocation_set(xmlNode *set, int score, const char *coloc_id, - EXPAND_CONSTRAINT_IDREF(set_id, resource, ID(xml_rsc)); - if (with != NULL) { - pe_rsc_trace(resource, "Colocating %s with %s", resource->id, with->id); -+ flags = unpack_influence(coloc_id, resource, influence_s); - pcmk__new_colocation(set_id, NULL, local_score, resource, -- with, role, role, -- unpack_influence(coloc_id, resource, -- influence_s)); -+ with, role, role, flags); - } - with = resource; - } -@@ -451,12 +454,10 @@ unpack_colocation_set(xmlNode *set, int score, const char *coloc_id, - if (last != NULL) { - pe_rsc_trace(resource, "Colocating %s with %s", - last->id, resource->id); -+ flags = unpack_influence(coloc_id, resource, influence_s); - pcmk__new_colocation(set_id, NULL, local_score, last, -- resource, role, role, -- unpack_influence(coloc_id, last, -- influence_s)); -+ resource, role, role, flags); - } -- - last = resource; - } - -@@ -470,11 +471,10 @@ unpack_colocation_set(xmlNode *set, int score, const char *coloc_id, - xml_rsc != NULL; xml_rsc = crm_next_same_xml(xml_rsc)) { - - xmlNode *xml_rsc_with = NULL; -- bool influence = true; - - EXPAND_CONSTRAINT_IDREF(set_id, resource, ID(xml_rsc)); -- influence = unpack_influence(coloc_id, resource, influence_s); - -+ flags = unpack_influence(coloc_id, resource, influence_s); - for (xml_rsc_with = first_named_child(set, XML_TAG_RESOURCE_REF); - xml_rsc_with != NULL; - xml_rsc_with = crm_next_same_xml(xml_rsc_with)) { -@@ -487,7 +487,7 @@ unpack_colocation_set(xmlNode *set, int score, const char *coloc_id, - pe_rsc_trace(resource, "Anti-Colocating %s with %s", resource->id, - with->id); - pcmk__new_colocation(set_id, NULL, local_score, -- resource, with, role, role, influence); -+ resource, with, role, role, flags); - } - } - } -@@ -506,6 +506,7 @@ colocate_rsc_sets(const char *id, xmlNode *set1, xmlNode *set2, int score, - - int rc = pcmk_rc_ok; - bool sequential = false; -+ uint32_t flags = pcmk__coloc_none; - - if (score == 0) { - crm_trace("Ignoring colocation '%s' between sets because score is 0", -@@ -536,18 +537,18 @@ colocate_rsc_sets(const char *id, xmlNode *set1, xmlNode *set2, int score, - } - - if ((rsc_1 != NULL) && (rsc_2 != NULL)) { -+ flags = unpack_influence(id, rsc_1, influence_s); - pcmk__new_colocation(id, NULL, score, rsc_1, rsc_2, role_1, role_2, -- unpack_influence(id, rsc_1, influence_s)); -+ flags); - - } else if (rsc_1 != NULL) { -- bool influence = unpack_influence(id, rsc_1, influence_s); -- -+ flags = unpack_influence(id, rsc_1, influence_s); - for (xml_rsc = first_named_child(set2, XML_TAG_RESOURCE_REF); - xml_rsc != NULL; xml_rsc = crm_next_same_xml(xml_rsc)) { - - EXPAND_CONSTRAINT_IDREF(id, rsc_2, ID(xml_rsc)); - pcmk__new_colocation(id, NULL, score, rsc_1, rsc_2, role_1, -- role_2, influence); -+ role_2, flags); - } - - } else if (rsc_2 != NULL) { -@@ -555,9 +556,9 @@ colocate_rsc_sets(const char *id, xmlNode *set1, xmlNode *set2, int score, - xml_rsc != NULL; xml_rsc = crm_next_same_xml(xml_rsc)) { - - EXPAND_CONSTRAINT_IDREF(id, rsc_1, ID(xml_rsc)); -+ flags = unpack_influence(id, rsc_1, influence_s); - pcmk__new_colocation(id, NULL, score, rsc_1, rsc_2, role_1, -- role_2, -- unpack_influence(id, rsc_1, influence_s)); -+ role_2, flags); - } - - } else { -@@ -565,18 +566,17 @@ colocate_rsc_sets(const char *id, xmlNode *set1, xmlNode *set2, int score, - xml_rsc != NULL; xml_rsc = crm_next_same_xml(xml_rsc)) { - - xmlNode *xml_rsc_2 = NULL; -- bool influence = true; - - EXPAND_CONSTRAINT_IDREF(id, rsc_1, ID(xml_rsc)); -- influence = unpack_influence(id, rsc_1, influence_s); - -+ flags = unpack_influence(id, rsc_1, influence_s); - for (xml_rsc_2 = first_named_child(set2, XML_TAG_RESOURCE_REF); - xml_rsc_2 != NULL; - xml_rsc_2 = crm_next_same_xml(xml_rsc_2)) { - - EXPAND_CONSTRAINT_IDREF(id, rsc_2, ID(xml_rsc_2)); - pcmk__new_colocation(id, NULL, score, rsc_1, rsc_2, -- role_1, role_2, influence); -+ role_1, role_2, flags); - } - } - } -@@ -587,6 +587,7 @@ unpack_simple_colocation(xmlNode *xml_obj, const char *id, - const char *influence_s, pe_working_set_t *data_set) - { - int score_i = 0; -+ uint32_t flags = pcmk__coloc_none; - - const char *score = crm_element_value(xml_obj, XML_RULE_ATTR_SCORE); - const char *dependent_id = crm_element_value(xml_obj, -@@ -675,9 +676,9 @@ unpack_simple_colocation(xmlNode *xml_obj, const char *id, - score_i = char2score(score); - } - -+ flags = unpack_influence(id, dependent, influence_s); - pcmk__new_colocation(id, attr, score_i, dependent, primary, -- dependent_role, primary_role, -- unpack_influence(id, dependent, influence_s)); -+ dependent_role, primary_role, flags); - } - - // \return Standard Pacemaker return code -diff --git a/lib/pacemaker/pcmk_sched_group.c b/lib/pacemaker/pcmk_sched_group.c -index c1392e07a4c..72f088a2709 100644 ---- a/lib/pacemaker/pcmk_sched_group.c -+++ b/lib/pacemaker/pcmk_sched_group.c -@@ -168,10 +168,16 @@ member_internal_constraints(gpointer data, gpointer user_data) - } - - } else if (member_data->colocated) { -+ uint32_t flags = pcmk__coloc_none; -+ -+ if (pcmk_is_set(member->flags, pe_rsc_critical)) { -+ flags |= pcmk__coloc_influence; -+ } -+ - // Colocate this member with the previous one - pcmk__new_colocation("group:internal_colocation", NULL, INFINITY, - member, member_data->previous_member, NULL, NULL, -- pcmk_is_set(member->flags, pe_rsc_critical)); -+ flags); - } - - if (member_data->promotable) { -diff --git a/lib/pacemaker/pcmk_sched_primitive.c b/lib/pacemaker/pcmk_sched_primitive.c -index 4e3eca3e18a..ff7052f6c79 100644 ---- a/lib/pacemaker/pcmk_sched_primitive.c -+++ b/lib/pacemaker/pcmk_sched_primitive.c -@@ -999,7 +999,8 @@ pcmk__primitive_internal_constraints(pe_resource_t *rsc) - score = INFINITY; /* Force them to run on the same host */ - } - pcmk__new_colocation("resource-with-container", NULL, score, rsc, -- rsc->container, NULL, NULL, true); -+ rsc->container, NULL, NULL, -+ pcmk__coloc_influence); - } - } - - -From 2f8d4186e16fb026176f1ddb774eb38940c90390 Mon Sep 17 00:00:00 2001 -From: Ken Gaillot -Date: Mon, 26 Jun 2023 12:33:49 -0500 -Subject: [PATCH 04/17] Refactor: scheduler: prefix all internal colocation IDs - with "#" - -... to ensure they're easily distinguished from user-configured colocations -in log messages. ---- - lib/pacemaker/pcmk_sched_bundle.c | 6 +++--- - lib/pacemaker/pcmk_sched_group.c | 5 ++--- - lib/pacemaker/pcmk_sched_primitive.c | 2 +- - 3 files changed, 6 insertions(+), 7 deletions(-) - -diff --git a/lib/pacemaker/pcmk_sched_bundle.c b/lib/pacemaker/pcmk_sched_bundle.c -index ca3c21a9977..b4beb0d488f 100644 ---- a/lib/pacemaker/pcmk_sched_bundle.c -+++ b/lib/pacemaker/pcmk_sched_bundle.c -@@ -80,7 +80,7 @@ pcmk__bundle_allocate(pe_resource_t *rsc, const pe_node_t *prefer) - * host because pacemaker-remoted only supports a single - * active connection - */ -- pcmk__new_colocation("child-remote-with-docker-remote", NULL, -+ pcmk__new_colocation("#replica-remote-with-host-remote", NULL, - INFINITY, replica->remote, - container_host->details->remote_rsc, NULL, - NULL, pcmk__coloc_influence); -@@ -251,14 +251,14 @@ pcmk__bundle_internal_constraints(pe_resource_t *rsc) - pcmk__order_stops(replica->container, replica->ip, - pe_order_implies_first|pe_order_preserve); - -- pcmk__new_colocation("ip-with-docker", NULL, INFINITY, replica->ip, -+ pcmk__new_colocation("#ip-with-container", NULL, INFINITY, replica->ip, - replica->container, NULL, NULL, - pcmk__coloc_influence); - } - - if (replica->remote) { - /* This handles ordering and colocating remote relative to container -- * (via "resource-with-container"). Since IP is also ordered and -+ * (via "#resource-with-container"). Since IP is also ordered and - * colocated relative to the container, we don't need to do anything - * explicit here with IP. - */ -diff --git a/lib/pacemaker/pcmk_sched_group.c b/lib/pacemaker/pcmk_sched_group.c -index 72f088a2709..1b6c5c416ab 100644 ---- a/lib/pacemaker/pcmk_sched_group.c -+++ b/lib/pacemaker/pcmk_sched_group.c -@@ -175,9 +175,8 @@ member_internal_constraints(gpointer data, gpointer user_data) - } - - // Colocate this member with the previous one -- pcmk__new_colocation("group:internal_colocation", NULL, INFINITY, -- member, member_data->previous_member, NULL, NULL, -- flags); -+ pcmk__new_colocation("#group-members", NULL, INFINITY, member, -+ member_data->previous_member, NULL, NULL, flags); - } - - if (member_data->promotable) { -diff --git a/lib/pacemaker/pcmk_sched_primitive.c b/lib/pacemaker/pcmk_sched_primitive.c -index ff7052f6c79..d6b39e38c5f 100644 ---- a/lib/pacemaker/pcmk_sched_primitive.c -+++ b/lib/pacemaker/pcmk_sched_primitive.c -@@ -998,7 +998,7 @@ pcmk__primitive_internal_constraints(pe_resource_t *rsc) - } else { - score = INFINITY; /* Force them to run on the same host */ - } -- pcmk__new_colocation("resource-with-container", NULL, score, rsc, -+ pcmk__new_colocation("#resource-with-container", NULL, score, rsc, - rsc->container, NULL, NULL, - pcmk__coloc_influence); - } - -From 93230be27fb4c156a1cc15daf161e2206961421e Mon Sep 17 00:00:00 2001 -From: Ken Gaillot -Date: Mon, 26 Jun 2023 16:25:02 -0500 -Subject: [PATCH 05/17] Refactor: scheduler: don't use macro for finding - constraint resource - -It obscured what was happening ---- - lib/pacemaker/pcmk_sched_colocation.c | 105 ++++++++++++++++++++------ - 1 file changed, 81 insertions(+), 24 deletions(-) - -diff --git a/lib/pacemaker/pcmk_sched_colocation.c b/lib/pacemaker/pcmk_sched_colocation.c -index dbdefadfd10..4d8fe74c206 100644 ---- a/lib/pacemaker/pcmk_sched_colocation.c -+++ b/lib/pacemaker/pcmk_sched_colocation.c -@@ -21,14 +21,6 @@ - #include "crm/msg_xml.h" - #include "libpacemaker_private.h" - --#define EXPAND_CONSTRAINT_IDREF(__set, __rsc, __name) do { \ -- __rsc = pcmk__find_constraint_resource(data_set->resources, __name); \ -- if (__rsc == NULL) { \ -- pcmk__config_err("%s: No resource found for %s", __set, __name); \ -- return; \ -- } \ -- } while(0) -- - // Used to temporarily mark a node as unusable - #define INFINITY_HACK (INFINITY * -100) - -@@ -411,6 +403,7 @@ unpack_colocation_set(xmlNode *set, int score, const char *coloc_id, - int local_score = score; - bool sequential = false; - uint32_t flags = pcmk__coloc_none; -+ const char *xml_rsc_id = NULL; - const char *score_s = crm_element_value(set, XML_RULE_ATTR_SCORE); - - if (score_s) { -@@ -434,7 +427,14 @@ unpack_colocation_set(xmlNode *set, int score, const char *coloc_id, - for (xml_rsc = first_named_child(set, XML_TAG_RESOURCE_REF); - xml_rsc != NULL; xml_rsc = crm_next_same_xml(xml_rsc)) { - -- EXPAND_CONSTRAINT_IDREF(set_id, resource, ID(xml_rsc)); -+ xml_rsc_id = ID(xml_rsc); -+ resource = pcmk__find_constraint_resource(data_set->resources, -+ xml_rsc_id); -+ if (resource == NULL) { -+ pcmk__config_err("%s: No resource found for %s", -+ set_id, xml_rsc_id); -+ return; -+ } - if (with != NULL) { - pe_rsc_trace(resource, "Colocating %s with %s", resource->id, with->id); - flags = unpack_influence(coloc_id, resource, influence_s); -@@ -450,7 +450,14 @@ unpack_colocation_set(xmlNode *set, int score, const char *coloc_id, - for (xml_rsc = first_named_child(set, XML_TAG_RESOURCE_REF); - xml_rsc != NULL; xml_rsc = crm_next_same_xml(xml_rsc)) { - -- EXPAND_CONSTRAINT_IDREF(set_id, resource, ID(xml_rsc)); -+ xml_rsc_id = ID(xml_rsc); -+ resource = pcmk__find_constraint_resource(data_set->resources, -+ xml_rsc_id); -+ if (resource == NULL) { -+ pcmk__config_err("%s: No resource found for %s", -+ set_id, xml_rsc_id); -+ return; -+ } - if (last != NULL) { - pe_rsc_trace(resource, "Colocating %s with %s", - last->id, resource->id); -@@ -472,18 +479,30 @@ unpack_colocation_set(xmlNode *set, int score, const char *coloc_id, - - xmlNode *xml_rsc_with = NULL; - -- EXPAND_CONSTRAINT_IDREF(set_id, resource, ID(xml_rsc)); -- -+ xml_rsc_id = ID(xml_rsc); -+ resource = pcmk__find_constraint_resource(data_set->resources, -+ xml_rsc_id); -+ if (resource == NULL) { -+ pcmk__config_err("%s: No resource found for %s", -+ set_id, xml_rsc_id); -+ return; -+ } - flags = unpack_influence(coloc_id, resource, influence_s); - for (xml_rsc_with = first_named_child(set, XML_TAG_RESOURCE_REF); - xml_rsc_with != NULL; - xml_rsc_with = crm_next_same_xml(xml_rsc_with)) { - -- if (pcmk__str_eq(resource->id, ID(xml_rsc_with), -- pcmk__str_casei)) { -+ xml_rsc_id = ID(xml_rsc_with); -+ if (pcmk__str_eq(resource->id, xml_rsc_id, pcmk__str_none)) { - break; - } -- EXPAND_CONSTRAINT_IDREF(set_id, with, ID(xml_rsc_with)); -+ with = pcmk__find_constraint_resource(data_set->resources, -+ xml_rsc_id); -+ if (with == NULL) { -+ pcmk__config_err("%s: No resource found for %s", -+ set_id, xml_rsc_id); -+ return; -+ } - pe_rsc_trace(resource, "Anti-Colocating %s with %s", resource->id, - with->id); - pcmk__new_colocation(set_id, NULL, local_score, -@@ -501,6 +520,7 @@ colocate_rsc_sets(const char *id, xmlNode *set1, xmlNode *set2, int score, - pe_resource_t *rsc_1 = NULL; - pe_resource_t *rsc_2 = NULL; - -+ const char *xml_rsc_id = NULL; - const char *role_1 = crm_element_value(set1, "role"); - const char *role_2 = crm_element_value(set2, "role"); - -@@ -519,21 +539,30 @@ colocate_rsc_sets(const char *id, xmlNode *set1, xmlNode *set2, int score, - // Get the first one - xml_rsc = first_named_child(set1, XML_TAG_RESOURCE_REF); - if (xml_rsc != NULL) { -- EXPAND_CONSTRAINT_IDREF(id, rsc_1, ID(xml_rsc)); -+ xml_rsc_id = ID(xml_rsc); -+ rsc_1 = pcmk__find_constraint_resource(data_set->resources, -+ xml_rsc_id); -+ if (rsc_1 == NULL) { -+ pcmk__config_err("%s: No resource found for %s", -+ id, xml_rsc_id); -+ return; -+ } - } - } - - rc = pcmk__xe_get_bool_attr(set2, "sequential", &sequential); - if (rc != pcmk_rc_ok || sequential) { - // Get the last one -- const char *rid = NULL; -- - for (xml_rsc = first_named_child(set2, XML_TAG_RESOURCE_REF); - xml_rsc != NULL; xml_rsc = crm_next_same_xml(xml_rsc)) { - -- rid = ID(xml_rsc); -+ xml_rsc_id = ID(xml_rsc); -+ } -+ rsc_2 = pcmk__find_constraint_resource(data_set->resources, xml_rsc_id); -+ if (rsc_2 == NULL) { -+ pcmk__config_err("%s: No resource found for %s", id, xml_rsc_id); -+ return; - } -- EXPAND_CONSTRAINT_IDREF(id, rsc_2, rid); - } - - if ((rsc_1 != NULL) && (rsc_2 != NULL)) { -@@ -546,7 +575,14 @@ colocate_rsc_sets(const char *id, xmlNode *set1, xmlNode *set2, int score, - for (xml_rsc = first_named_child(set2, XML_TAG_RESOURCE_REF); - xml_rsc != NULL; xml_rsc = crm_next_same_xml(xml_rsc)) { - -- EXPAND_CONSTRAINT_IDREF(id, rsc_2, ID(xml_rsc)); -+ xml_rsc_id = ID(xml_rsc); -+ rsc_2 = pcmk__find_constraint_resource(data_set->resources, -+ xml_rsc_id); -+ if (rsc_2 == NULL) { -+ pcmk__config_err("%s: No resource found for %s", -+ id, xml_rsc_id); -+ return; -+ } - pcmk__new_colocation(id, NULL, score, rsc_1, rsc_2, role_1, - role_2, flags); - } -@@ -555,7 +591,14 @@ colocate_rsc_sets(const char *id, xmlNode *set1, xmlNode *set2, int score, - for (xml_rsc = first_named_child(set1, XML_TAG_RESOURCE_REF); - xml_rsc != NULL; xml_rsc = crm_next_same_xml(xml_rsc)) { - -- EXPAND_CONSTRAINT_IDREF(id, rsc_1, ID(xml_rsc)); -+ xml_rsc_id = ID(xml_rsc); -+ rsc_1 = pcmk__find_constraint_resource(data_set->resources, -+ xml_rsc_id); -+ if (rsc_1 == NULL) { -+ pcmk__config_err("%s: No resource found for %s", -+ id, xml_rsc_id); -+ return; -+ } - flags = unpack_influence(id, rsc_1, influence_s); - pcmk__new_colocation(id, NULL, score, rsc_1, rsc_2, role_1, - role_2, flags); -@@ -567,14 +610,28 @@ colocate_rsc_sets(const char *id, xmlNode *set1, xmlNode *set2, int score, - - xmlNode *xml_rsc_2 = NULL; - -- EXPAND_CONSTRAINT_IDREF(id, rsc_1, ID(xml_rsc)); -+ xml_rsc_id = ID(xml_rsc); -+ rsc_1 = pcmk__find_constraint_resource(data_set->resources, -+ xml_rsc_id); -+ if (rsc_1 == NULL) { -+ pcmk__config_err("%s: No resource found for %s", -+ id, xml_rsc_id); -+ return; -+ } - - flags = unpack_influence(id, rsc_1, influence_s); - for (xml_rsc_2 = first_named_child(set2, XML_TAG_RESOURCE_REF); - xml_rsc_2 != NULL; - xml_rsc_2 = crm_next_same_xml(xml_rsc_2)) { - -- EXPAND_CONSTRAINT_IDREF(id, rsc_2, ID(xml_rsc_2)); -+ xml_rsc_id = ID(xml_rsc_2); -+ rsc_2 = pcmk__find_constraint_resource(data_set->resources, -+ xml_rsc_id); -+ if (rsc_2 == NULL) { -+ pcmk__config_err("%s: No resource found for %s", -+ id, xml_rsc_id); -+ return; -+ } - pcmk__new_colocation(id, NULL, score, rsc_1, rsc_2, - role_1, role_2, flags); - } - -From 23393992a75905f6bd4636f71263c15338c1556f Mon Sep 17 00:00:00 2001 -From: Ken Gaillot -Date: Tue, 27 Jun 2023 10:15:19 -0500 -Subject: [PATCH 06/17] Refactor: scheduler: use bool for "group ordering" in - colocation sets - -... for readability ---- - lib/pacemaker/pcmk_sched_colocation.c | 13 ++++++++----- - 1 file changed, 8 insertions(+), 5 deletions(-) - -diff --git a/lib/pacemaker/pcmk_sched_colocation.c b/lib/pacemaker/pcmk_sched_colocation.c -index 4d8fe74c206..4c8bca56e86 100644 ---- a/lib/pacemaker/pcmk_sched_colocation.c -+++ b/lib/pacemaker/pcmk_sched_colocation.c -@@ -399,7 +399,7 @@ unpack_colocation_set(xmlNode *set, int score, const char *coloc_id, - pe_resource_t *resource = NULL; - const char *set_id = ID(set); - const char *role = crm_element_value(set, "role"); -- const char *ordering = crm_element_value(set, "ordering"); -+ bool with_previous = false; - int local_score = score; - bool sequential = false; - uint32_t flags = pcmk__coloc_none; -@@ -415,15 +415,18 @@ unpack_colocation_set(xmlNode *set, int score, const char *coloc_id, - return; - } - -- if (ordering == NULL) { -- ordering = "group"; -+ /* The "ordering" attribute specifies whether resources in a positive-score -+ * set are colocated with the previous or next resource. -+ */ -+ if (pcmk__str_eq(crm_element_value(set, "ordering"), "group", -+ pcmk__str_null_matches|pcmk__str_casei)) { -+ with_previous = true; - } - - if (pcmk__xe_get_bool_attr(set, "sequential", &sequential) == pcmk_rc_ok && !sequential) { - return; - -- } else if ((local_score > 0) -- && pcmk__str_eq(ordering, "group", pcmk__str_casei)) { -+ } else if ((local_score > 0) && with_previous) { - for (xml_rsc = first_named_child(set, XML_TAG_RESOURCE_REF); - xml_rsc != NULL; xml_rsc = crm_next_same_xml(xml_rsc)) { - - -From e42ec03e0fe488a80172e79b319a3084854332de Mon Sep 17 00:00:00 2001 -From: Ken Gaillot -Date: Tue, 27 Jun 2023 10:18:22 -0500 -Subject: [PATCH 07/17] Refactor: scheduler: simplify unpacking a colocation - set (slightly) - ---- - lib/pacemaker/pcmk_sched_colocation.c | 56 ++++++++++----------------- - 1 file changed, 20 insertions(+), 36 deletions(-) - -diff --git a/lib/pacemaker/pcmk_sched_colocation.c b/lib/pacemaker/pcmk_sched_colocation.c -index 4c8bca56e86..e8f01e49a27 100644 ---- a/lib/pacemaker/pcmk_sched_colocation.c -+++ b/lib/pacemaker/pcmk_sched_colocation.c -@@ -395,7 +395,7 @@ unpack_colocation_set(xmlNode *set, int score, const char *coloc_id, - const char *influence_s, pe_working_set_t *data_set) - { - xmlNode *xml_rsc = NULL; -- pe_resource_t *with = NULL; -+ pe_resource_t *other = NULL; - pe_resource_t *resource = NULL; - const char *set_id = ID(set); - const char *role = crm_element_value(set, "role"); -@@ -426,30 +426,7 @@ unpack_colocation_set(xmlNode *set, int score, const char *coloc_id, - if (pcmk__xe_get_bool_attr(set, "sequential", &sequential) == pcmk_rc_ok && !sequential) { - return; - -- } else if ((local_score > 0) && with_previous) { -- for (xml_rsc = first_named_child(set, XML_TAG_RESOURCE_REF); -- xml_rsc != NULL; xml_rsc = crm_next_same_xml(xml_rsc)) { -- -- xml_rsc_id = ID(xml_rsc); -- resource = pcmk__find_constraint_resource(data_set->resources, -- xml_rsc_id); -- if (resource == NULL) { -- pcmk__config_err("%s: No resource found for %s", -- set_id, xml_rsc_id); -- return; -- } -- if (with != NULL) { -- pe_rsc_trace(resource, "Colocating %s with %s", resource->id, with->id); -- flags = unpack_influence(coloc_id, resource, influence_s); -- pcmk__new_colocation(set_id, NULL, local_score, resource, -- with, role, role, flags); -- } -- with = resource; -- } -- - } else if (local_score > 0) { -- pe_resource_t *last = NULL; -- - for (xml_rsc = first_named_child(set, XML_TAG_RESOURCE_REF); - xml_rsc != NULL; xml_rsc = crm_next_same_xml(xml_rsc)) { - -@@ -461,14 +438,21 @@ unpack_colocation_set(xmlNode *set, int score, const char *coloc_id, - set_id, xml_rsc_id); - return; - } -- if (last != NULL) { -- pe_rsc_trace(resource, "Colocating %s with %s", -- last->id, resource->id); -+ if (other != NULL) { - flags = unpack_influence(coloc_id, resource, influence_s); -- pcmk__new_colocation(set_id, NULL, local_score, last, -- resource, role, role, flags); -+ if (with_previous) { -+ pe_rsc_trace(resource, "Colocating %s with %s in set %s", -+ resource->id, other->id, set_id); -+ pcmk__new_colocation(set_id, NULL, local_score, resource, -+ other, role, role, flags); -+ } else { -+ pe_rsc_trace(resource, "Colocating %s with %s in set %s", -+ other->id, resource->id, set_id); -+ pcmk__new_colocation(set_id, NULL, local_score, other, -+ resource, role, role, flags); -+ } - } -- last = resource; -+ other = resource; - } - - } else { -@@ -499,17 +483,17 @@ unpack_colocation_set(xmlNode *set, int score, const char *coloc_id, - if (pcmk__str_eq(resource->id, xml_rsc_id, pcmk__str_none)) { - break; - } -- with = pcmk__find_constraint_resource(data_set->resources, -- xml_rsc_id); -- if (with == NULL) { -+ other = pcmk__find_constraint_resource(data_set->resources, -+ xml_rsc_id); -+ if (other == NULL) { - pcmk__config_err("%s: No resource found for %s", - set_id, xml_rsc_id); - return; - } -- pe_rsc_trace(resource, "Anti-Colocating %s with %s", resource->id, -- with->id); -+ pe_rsc_trace(resource, "Anti-Colocating %s with %s", -+ resource->id, other->id); - pcmk__new_colocation(set_id, NULL, local_score, -- resource, with, role, role, flags); -+ resource, other, role, role, flags); - } - } - } - -From a26ebb380b4bcf1f4fb8a2d69d4b8c8af306dfec Mon Sep 17 00:00:00 2001 -From: Ken Gaillot -Date: Mon, 26 Jun 2023 14:56:53 -0500 -Subject: [PATCH 08/17] Feature: CIB: deprecate "ordering" attribute of - "resource_set" - -It's undocumented, and makes sets even more confusing than they already are, -especially since it only applies when the score is positive. ---- - include/crm/pengine/internal.h | 1 + - lib/pacemaker/pcmk_sched_colocation.c | 9 +++++++-- - 2 files changed, 8 insertions(+), 2 deletions(-) - -diff --git a/include/crm/pengine/internal.h b/include/crm/pengine/internal.h -index 1b5f6f1d8d9..53cbb54de5e 100644 ---- a/include/crm/pengine/internal.h -+++ b/include/crm/pengine/internal.h -@@ -170,6 +170,7 @@ enum pe_warn_once_e { - pe_wo_group_coloc = (1 << 12), - pe_wo_upstart = (1 << 13), - pe_wo_nagios = (1 << 14), -+ pe_wo_set_ordering = (1 << 15), - }; - - extern uint32_t pe_wo; -diff --git a/lib/pacemaker/pcmk_sched_colocation.c b/lib/pacemaker/pcmk_sched_colocation.c -index e8f01e49a27..36558f38c4e 100644 ---- a/lib/pacemaker/pcmk_sched_colocation.c -+++ b/lib/pacemaker/pcmk_sched_colocation.c -@@ -415,12 +415,17 @@ unpack_colocation_set(xmlNode *set, int score, const char *coloc_id, - return; - } - -- /* The "ordering" attribute specifies whether resources in a positive-score -- * set are colocated with the previous or next resource. -+ /* @COMPAT The deprecated "ordering" attribute specifies whether resources -+ * in a positive-score set are colocated with the previous or next resource. - */ - if (pcmk__str_eq(crm_element_value(set, "ordering"), "group", - pcmk__str_null_matches|pcmk__str_casei)) { - with_previous = true; -+ } else { -+ pe_warn_once(pe_wo_set_ordering, -+ "Support for 'ordering' other than 'group' in " -+ XML_CONS_TAG_RSC_SET " (such as %s) is deprecated and " -+ "will be removed in a future release", set_id); - } - - if (pcmk__xe_get_bool_attr(set, "sequential", &sequential) == pcmk_rc_ok && !sequential) { - -From f18f365c0995df68599ec2c241f81bae54d2bd38 Mon Sep 17 00:00:00 2001 -From: Ken Gaillot -Date: Mon, 26 Jun 2023 15:05:21 -0500 -Subject: [PATCH 09/17] Log: scheduler: improve logs when unpacking colocation - sets - ---- - lib/pacemaker/pcmk_sched_colocation.c | 54 +++++++++++++++++---------- - 1 file changed, 34 insertions(+), 20 deletions(-) - -diff --git a/lib/pacemaker/pcmk_sched_colocation.c b/lib/pacemaker/pcmk_sched_colocation.c -index 36558f38c4e..7555afbc522 100644 ---- a/lib/pacemaker/pcmk_sched_colocation.c -+++ b/lib/pacemaker/pcmk_sched_colocation.c -@@ -439,8 +439,9 @@ unpack_colocation_set(xmlNode *set, int score, const char *coloc_id, - resource = pcmk__find_constraint_resource(data_set->resources, - xml_rsc_id); - if (resource == NULL) { -- pcmk__config_err("%s: No resource found for %s", -- set_id, xml_rsc_id); -+ // Should be possible only with validation disabled -+ pcmk__config_err("Ignoring %s and later resources in set %s: " -+ "No such resource", xml_rsc_id, set_id); - return; - } - if (other != NULL) { -@@ -475,8 +476,9 @@ unpack_colocation_set(xmlNode *set, int score, const char *coloc_id, - resource = pcmk__find_constraint_resource(data_set->resources, - xml_rsc_id); - if (resource == NULL) { -- pcmk__config_err("%s: No resource found for %s", -- set_id, xml_rsc_id); -+ // Should be possible only with validation disabled -+ pcmk__config_err("Ignoring %s and later resources in set %s: " -+ "No such resource", xml_rsc_id, set_id); - return; - } - flags = unpack_influence(coloc_id, resource, influence_s); -@@ -490,11 +492,7 @@ unpack_colocation_set(xmlNode *set, int score, const char *coloc_id, - } - other = pcmk__find_constraint_resource(data_set->resources, - xml_rsc_id); -- if (other == NULL) { -- pcmk__config_err("%s: No resource found for %s", -- set_id, xml_rsc_id); -- return; -- } -+ CRM_ASSERT(other != NULL); // We already processed it - pe_rsc_trace(resource, "Anti-Colocating %s with %s", - resource->id, other->id); - pcmk__new_colocation(set_id, NULL, local_score, -@@ -527,7 +525,7 @@ colocate_rsc_sets(const char *id, xmlNode *set1, xmlNode *set2, int score, - } - - rc = pcmk__xe_get_bool_attr(set1, "sequential", &sequential); -- if (rc != pcmk_rc_ok || sequential) { -+ if ((rc != pcmk_rc_ok) || sequential) { - // Get the first one - xml_rsc = first_named_child(set1, XML_TAG_RESOURCE_REF); - if (xml_rsc != NULL) { -@@ -535,15 +533,17 @@ colocate_rsc_sets(const char *id, xmlNode *set1, xmlNode *set2, int score, - rsc_1 = pcmk__find_constraint_resource(data_set->resources, - xml_rsc_id); - if (rsc_1 == NULL) { -- pcmk__config_err("%s: No resource found for %s", -- id, xml_rsc_id); -+ // Should be possible only with validation disabled -+ pcmk__config_err("Ignoring colocation of set %s with set %s " -+ "because first resource %s not found", -+ ID(set1), ID(set2), xml_rsc_id); - return; - } - } - } - - rc = pcmk__xe_get_bool_attr(set2, "sequential", &sequential); -- if (rc != pcmk_rc_ok || sequential) { -+ if ((rc != pcmk_rc_ok) || sequential) { - // Get the last one - for (xml_rsc = first_named_child(set2, XML_TAG_RESOURCE_REF); - xml_rsc != NULL; xml_rsc = crm_next_same_xml(xml_rsc)) { -@@ -552,7 +552,10 @@ colocate_rsc_sets(const char *id, xmlNode *set1, xmlNode *set2, int score, - } - rsc_2 = pcmk__find_constraint_resource(data_set->resources, xml_rsc_id); - if (rsc_2 == NULL) { -- pcmk__config_err("%s: No resource found for %s", id, xml_rsc_id); -+ // Should be possible only with validation disabled -+ pcmk__config_err("Ignoring colocation of set %s with set %s " -+ "because last resource %s not found", -+ ID(set1), ID(set2), xml_rsc_id); - return; - } - } -@@ -573,6 +576,10 @@ colocate_rsc_sets(const char *id, xmlNode *set1, xmlNode *set2, int score, - if (rsc_2 == NULL) { - pcmk__config_err("%s: No resource found for %s", - id, xml_rsc_id); -+ // Should be possible only with validation disabled -+ pcmk__config_err("Ignoring resource %s and later in set %s " -+ "for colocation with set %s: No such resource", -+ xml_rsc_id, set2, set1); - return; - } - pcmk__new_colocation(id, NULL, score, rsc_1, rsc_2, role_1, -@@ -587,8 +594,10 @@ colocate_rsc_sets(const char *id, xmlNode *set1, xmlNode *set2, int score, - rsc_1 = pcmk__find_constraint_resource(data_set->resources, - xml_rsc_id); - if (rsc_1 == NULL) { -- pcmk__config_err("%s: No resource found for %s", -- id, xml_rsc_id); -+ // Should be possible only with validation disabled -+ pcmk__config_err("Ignoring resource %s and later in set %s " -+ "for colocation with set %s: No such resource", -+ xml_rsc_id, set1, set2); - return; - } - flags = unpack_influence(id, rsc_1, influence_s); -@@ -606,8 +615,10 @@ colocate_rsc_sets(const char *id, xmlNode *set1, xmlNode *set2, int score, - rsc_1 = pcmk__find_constraint_resource(data_set->resources, - xml_rsc_id); - if (rsc_1 == NULL) { -- pcmk__config_err("%s: No resource found for %s", -- id, xml_rsc_id); -+ // Should be possible only with validation disabled -+ pcmk__config_err("Ignoring resource %s and later in set %s " -+ "for colocation with set %s: No such resource", -+ xml_rsc_id, set1, set2); - return; - } - -@@ -620,8 +631,11 @@ colocate_rsc_sets(const char *id, xmlNode *set1, xmlNode *set2, int score, - rsc_2 = pcmk__find_constraint_resource(data_set->resources, - xml_rsc_id); - if (rsc_2 == NULL) { -- pcmk__config_err("%s: No resource found for %s", -- id, xml_rsc_id); -+ // Should be possible only with validation disabled -+ pcmk__config_err("Ignoring resource %s and later in set %s " -+ "for colocation with %s in set %s: " -+ "No such resource", -+ xml_rsc_id, set2, ID(xml_rsc), set1); - return; - } - pcmk__new_colocation(id, NULL, score, rsc_1, rsc_2, - -From 19e9a9d3b30e857f98459b7f5c4f4938e48e4261 Mon Sep 17 00:00:00 2001 -From: Ken Gaillot -Date: Mon, 26 Jun 2023 16:25:17 -0500 -Subject: [PATCH 10/17] Refactor: scheduler: mark explicitly configured - colocations - ---- - lib/pacemaker/libpacemaker_private.h | 3 +++ - lib/pacemaker/pcmk_sched_colocation.c | 18 +++++++++++------- - 2 files changed, 14 insertions(+), 7 deletions(-) - -diff --git a/lib/pacemaker/libpacemaker_private.h b/lib/pacemaker/libpacemaker_private.h -index 51de9d3e9a9..a49d55d3c41 100644 ---- a/lib/pacemaker/libpacemaker_private.h -+++ b/lib/pacemaker/libpacemaker_private.h -@@ -22,6 +22,9 @@ enum pcmk__coloc_flags { - - // Primary is affected even if already active - pcmk__coloc_influence = (1U << 0), -+ -+ // Colocation was explicitly configured in CIB -+ pcmk__coloc_explicit = (1U << 1), - }; - - // Flags to modify the behavior of add_colocated_node_scores() -diff --git a/lib/pacemaker/pcmk_sched_colocation.c b/lib/pacemaker/pcmk_sched_colocation.c -index 7555afbc522..e0b39b59e81 100644 ---- a/lib/pacemaker/pcmk_sched_colocation.c -+++ b/lib/pacemaker/pcmk_sched_colocation.c -@@ -445,7 +445,8 @@ unpack_colocation_set(xmlNode *set, int score, const char *coloc_id, - return; - } - if (other != NULL) { -- flags = unpack_influence(coloc_id, resource, influence_s); -+ flags = pcmk__coloc_explicit -+ | unpack_influence(coloc_id, resource, influence_s); - if (with_previous) { - pe_rsc_trace(resource, "Colocating %s with %s in set %s", - resource->id, other->id, set_id); -@@ -481,7 +482,8 @@ unpack_colocation_set(xmlNode *set, int score, const char *coloc_id, - "No such resource", xml_rsc_id, set_id); - return; - } -- flags = unpack_influence(coloc_id, resource, influence_s); -+ flags = pcmk__coloc_explicit -+ | unpack_influence(coloc_id, resource, influence_s); - for (xml_rsc_with = first_named_child(set, XML_TAG_RESOURCE_REF); - xml_rsc_with != NULL; - xml_rsc_with = crm_next_same_xml(xml_rsc_with)) { -@@ -561,12 +563,12 @@ colocate_rsc_sets(const char *id, xmlNode *set1, xmlNode *set2, int score, - } - - if ((rsc_1 != NULL) && (rsc_2 != NULL)) { -- flags = unpack_influence(id, rsc_1, influence_s); -+ flags = pcmk__coloc_explicit | unpack_influence(id, rsc_1, influence_s); - pcmk__new_colocation(id, NULL, score, rsc_1, rsc_2, role_1, role_2, - flags); - - } else if (rsc_1 != NULL) { -- flags = unpack_influence(id, rsc_1, influence_s); -+ flags = pcmk__coloc_explicit | unpack_influence(id, rsc_1, influence_s); - for (xml_rsc = first_named_child(set2, XML_TAG_RESOURCE_REF); - xml_rsc != NULL; xml_rsc = crm_next_same_xml(xml_rsc)) { - -@@ -600,7 +602,8 @@ colocate_rsc_sets(const char *id, xmlNode *set1, xmlNode *set2, int score, - xml_rsc_id, set1, set2); - return; - } -- flags = unpack_influence(id, rsc_1, influence_s); -+ flags = pcmk__coloc_explicit -+ | unpack_influence(id, rsc_1, influence_s); - pcmk__new_colocation(id, NULL, score, rsc_1, rsc_2, role_1, - role_2, flags); - } -@@ -622,7 +625,8 @@ colocate_rsc_sets(const char *id, xmlNode *set1, xmlNode *set2, int score, - return; - } - -- flags = unpack_influence(id, rsc_1, influence_s); -+ flags = pcmk__coloc_explicit -+ | unpack_influence(id, rsc_1, influence_s); - for (xml_rsc_2 = first_named_child(set2, XML_TAG_RESOURCE_REF); - xml_rsc_2 != NULL; - xml_rsc_2 = crm_next_same_xml(xml_rsc_2)) { -@@ -739,7 +743,7 @@ unpack_simple_colocation(xmlNode *xml_obj, const char *id, - score_i = char2score(score); - } - -- flags = unpack_influence(id, dependent, influence_s); -+ flags = pcmk__coloc_explicit | unpack_influence(id, dependent, influence_s); - pcmk__new_colocation(id, attr, score_i, dependent, primary, - dependent_role, primary_role, flags); - } - -From 4f9e2bc6fb1dd78d5784d918a85bb2028f01d265 Mon Sep 17 00:00:00 2001 -From: Ken Gaillot -Date: Tue, 27 Jun 2023 10:24:58 -0500 -Subject: [PATCH 11/17] Test: scheduler: add regression test for colocation - with an inner group member - -As of this commit, the behavior is incorrect. ---- - cts/cts-scheduler.in | 4 + - .../dot/coloc-with-inner-group-member.dot | 8 + - .../exp/coloc-with-inner-group-member.exp | 38 +++ - .../coloc-with-inner-group-member.scores | 46 ++++ - .../coloc-with-inner-group-member.summary | 33 +++ - .../xml/coloc-with-inner-group-member.xml | 258 ++++++++++++++++++ - 6 files changed, 387 insertions(+) - create mode 100644 cts/scheduler/dot/coloc-with-inner-group-member.dot - create mode 100644 cts/scheduler/exp/coloc-with-inner-group-member.exp - create mode 100644 cts/scheduler/scores/coloc-with-inner-group-member.scores - create mode 100644 cts/scheduler/summary/coloc-with-inner-group-member.summary - create mode 100644 cts/scheduler/xml/coloc-with-inner-group-member.xml - -diff --git a/cts/cts-scheduler.in b/cts/cts-scheduler.in -index ee0cb7b4722..de455105985 100644 ---- a/cts/cts-scheduler.in -+++ b/cts/cts-scheduler.in -@@ -80,6 +80,10 @@ TESTS = [ - [ "group-dependents", "Account for the location preferences of things colocated with a group" ], - [ "group-stop-ordering", "Ensure blocked group member stop does not force other member stops" ], - [ "colocate-unmanaged-group", "Respect mandatory colocations even if earlier group member is unmanaged" ], -+ [ -+ "coloc-with-inner-group-member", -+ "Consider explicit colocations with inner group members" -+ ], - ], - [ - [ "rsc_dep1", "Must not" ], -diff --git a/cts/scheduler/dot/coloc-with-inner-group-member.dot b/cts/scheduler/dot/coloc-with-inner-group-member.dot -new file mode 100644 -index 00000000000..77e1a8e6e40 ---- /dev/null -+++ b/cts/scheduler/dot/coloc-with-inner-group-member.dot -@@ -0,0 +1,8 @@ -+ digraph "g" { -+"grp_stop_0" -> "grp_stopped_0" [ style = bold] -+"grp_stop_0" -> "vip_stop_0 rhel8-3" [ style = bold] -+"grp_stop_0" [ style=bold color="green" fontcolor="orange"] -+"grp_stopped_0" [ style=bold color="green" fontcolor="orange"] -+"vip_stop_0 rhel8-3" -> "grp_stopped_0" [ style = bold] -+"vip_stop_0 rhel8-3" [ style=bold color="green" fontcolor="black"] -+} -diff --git a/cts/scheduler/exp/coloc-with-inner-group-member.exp b/cts/scheduler/exp/coloc-with-inner-group-member.exp -new file mode 100644 -index 00000000000..e6d94d5fe7f ---- /dev/null -+++ b/cts/scheduler/exp/coloc-with-inner-group-member.exp -@@ -0,0 +1,38 @@ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -diff --git a/cts/scheduler/scores/coloc-with-inner-group-member.scores b/cts/scheduler/scores/coloc-with-inner-group-member.scores -new file mode 100644 -index 00000000000..10fe944cb42 ---- /dev/null -+++ b/cts/scheduler/scores/coloc-with-inner-group-member.scores -@@ -0,0 +1,46 @@ -+ -+pcmk__group_assign: bar allocation score on rhel8-1: 0 -+pcmk__group_assign: bar allocation score on rhel8-2: 0 -+pcmk__group_assign: bar allocation score on rhel8-3: 0 -+pcmk__group_assign: bar allocation score on rhel8-4: 0 -+pcmk__group_assign: bar allocation score on rhel8-5: 0 -+pcmk__group_assign: foo allocation score on rhel8-1: 0 -+pcmk__group_assign: foo allocation score on rhel8-2: 0 -+pcmk__group_assign: foo allocation score on rhel8-3: 0 -+pcmk__group_assign: foo allocation score on rhel8-4: 0 -+pcmk__group_assign: foo allocation score on rhel8-5: 0 -+pcmk__group_assign: grp allocation score on rhel8-1: 0 -+pcmk__group_assign: grp allocation score on rhel8-2: 0 -+pcmk__group_assign: grp allocation score on rhel8-3: 0 -+pcmk__group_assign: grp allocation score on rhel8-4: 0 -+pcmk__group_assign: grp allocation score on rhel8-5: 0 -+pcmk__group_assign: vip allocation score on rhel8-1: 0 -+pcmk__group_assign: vip allocation score on rhel8-2: 0 -+pcmk__group_assign: vip allocation score on rhel8-3: 0 -+pcmk__group_assign: vip allocation score on rhel8-4: 0 -+pcmk__group_assign: vip allocation score on rhel8-5: 0 -+pcmk__primitive_assign: Fencing allocation score on rhel8-1: 0 -+pcmk__primitive_assign: Fencing allocation score on rhel8-2: 0 -+pcmk__primitive_assign: Fencing allocation score on rhel8-3: 0 -+pcmk__primitive_assign: Fencing allocation score on rhel8-4: 0 -+pcmk__primitive_assign: Fencing allocation score on rhel8-5: 0 -+pcmk__primitive_assign: bar allocation score on rhel8-1: -INFINITY -+pcmk__primitive_assign: bar allocation score on rhel8-2: -INFINITY -+pcmk__primitive_assign: bar allocation score on rhel8-3: -INFINITY -+pcmk__primitive_assign: bar allocation score on rhel8-4: 0 -+pcmk__primitive_assign: bar allocation score on rhel8-5: -INFINITY -+pcmk__primitive_assign: foo allocation score on rhel8-1: 0 -+pcmk__primitive_assign: foo allocation score on rhel8-2: 0 -+pcmk__primitive_assign: foo allocation score on rhel8-3: 0 -+pcmk__primitive_assign: foo allocation score on rhel8-4: 0 -+pcmk__primitive_assign: foo allocation score on rhel8-5: 0 -+pcmk__primitive_assign: vip allocation score on rhel8-1: -INFINITY -+pcmk__primitive_assign: vip allocation score on rhel8-2: -INFINITY -+pcmk__primitive_assign: vip allocation score on rhel8-3: -INFINITY -+pcmk__primitive_assign: vip allocation score on rhel8-4: -INFINITY -+pcmk__primitive_assign: vip allocation score on rhel8-5: -INFINITY -+pcmk__primitive_assign: vip-dep allocation score on rhel8-1: 0 -+pcmk__primitive_assign: vip-dep allocation score on rhel8-2: 0 -+pcmk__primitive_assign: vip-dep allocation score on rhel8-3: 0 -+pcmk__primitive_assign: vip-dep allocation score on rhel8-4: 0 -+pcmk__primitive_assign: vip-dep allocation score on rhel8-5: 0 -diff --git a/cts/scheduler/summary/coloc-with-inner-group-member.summary b/cts/scheduler/summary/coloc-with-inner-group-member.summary -new file mode 100644 -index 00000000000..3e87f0867ef ---- /dev/null -+++ b/cts/scheduler/summary/coloc-with-inner-group-member.summary -@@ -0,0 +1,33 @@ -+Using the original execution date of: 2023-06-20 20:45:06Z -+Current cluster status: -+ * Node List: -+ * Online: [ rhel8-1 rhel8-2 rhel8-3 rhel8-4 rhel8-5 ] -+ -+ * Full List of Resources: -+ * Fencing (stonith:fence_xvm): Started rhel8-1 -+ * vip-dep (ocf:pacemaker:Dummy): Started rhel8-3 -+ * Resource Group: grp: -+ * foo (ocf:pacemaker:Dummy): Started rhel8-4 -+ * bar (ocf:pacemaker:Dummy): Started rhel8-4 -+ * vip (ocf:pacemaker:Dummy): Started rhel8-3 -+ -+Transition Summary: -+ * Stop vip ( rhel8-3 ) due to node availability -+ -+Executing Cluster Transition: -+ * Pseudo action: grp_stop_0 -+ * Resource action: vip stop on rhel8-3 -+ * Pseudo action: grp_stopped_0 -+Using the original execution date of: 2023-06-20 20:45:06Z -+ -+Revised Cluster Status: -+ * Node List: -+ * Online: [ rhel8-1 rhel8-2 rhel8-3 rhel8-4 rhel8-5 ] -+ -+ * Full List of Resources: -+ * Fencing (stonith:fence_xvm): Started rhel8-1 -+ * vip-dep (ocf:pacemaker:Dummy): Started rhel8-3 -+ * Resource Group: grp: -+ * foo (ocf:pacemaker:Dummy): Started rhel8-4 -+ * bar (ocf:pacemaker:Dummy): Started rhel8-4 -+ * vip (ocf:pacemaker:Dummy): Stopped -diff --git a/cts/scheduler/xml/coloc-with-inner-group-member.xml b/cts/scheduler/xml/coloc-with-inner-group-member.xml -new file mode 100644 -index 00000000000..c07edecb81a ---- /dev/null -+++ b/cts/scheduler/xml/coloc-with-inner-group-member.xml -@@ -0,0 +1,258 @@ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ - -From 7fa4999f96d7541ee0dad248477c3e7d4affff00 Mon Sep 17 00:00:00 2001 -From: Ken Gaillot -Date: Tue, 20 Jun 2023 19:23:18 -0500 -Subject: [PATCH 12/17] Fix: scheduler: consider explicit colocations with - group members - -Previously, a group's colocations would include only colocations explicitly -with the group itself, and with its first member (for "group with" colocations) -or last member (for "with group" colocations). Explicit colocations with a -different group member could cause incorrect node assignment. - -Fixes T679 ---- - lib/pacemaker/pcmk_sched_group.c | 70 +++++++++++++++++++++------- - lib/pacemaker/pcmk_sched_primitive.c | 52 ++++++++++++++------- - 2 files changed, 90 insertions(+), 32 deletions(-) - -diff --git a/lib/pacemaker/pcmk_sched_group.c b/lib/pacemaker/pcmk_sched_group.c -index 1b6c5c416ab..95e2d77aa5f 100644 ---- a/lib/pacemaker/pcmk_sched_group.c -+++ b/lib/pacemaker/pcmk_sched_group.c -@@ -674,16 +674,36 @@ pcmk__with_group_colocations(const pe_resource_t *rsc, - } - - /* "With this" colocations are needed only for the group itself and for its -- * last member. Add the group's colocations plus any relevant -- * parent colocations if cloned. -+ * last member. (Previous members will chain via the group internal -+ * colocations.) - */ -- if ((rsc == orig_rsc) || (orig_rsc == pe__last_group_member(rsc))) { -- crm_trace("Adding 'with %s' colocations to list for %s", -- rsc->id, orig_rsc->id); -- pcmk__add_with_this_list(list, rsc->rsc_cons_lhs); -- if (rsc->parent != NULL) { // Cloned group -- rsc->parent->cmds->with_this_colocations(rsc->parent, orig_rsc, -- list); -+ if ((orig_rsc != rsc) && (orig_rsc != pe__last_group_member(rsc))) { -+ return; -+ } -+ -+ pe_rsc_trace(rsc, "Adding 'with %s' colocations to list for %s", -+ rsc->id, orig_rsc->id); -+ -+ // Add the group's own colocations -+ pcmk__add_with_this_list(list, rsc->rsc_cons_lhs); -+ -+ // If cloned, add any relevant colocations with the clone -+ if (rsc->parent != NULL) { -+ rsc->parent->cmds->with_this_colocations(rsc->parent, orig_rsc, -+ list); -+ } -+ -+ if (!pe__group_flag_is_set(rsc, pe__group_colocated)) { -+ // @COMPAT Non-colocated groups are deprecated -+ return; -+ } -+ -+ // Add explicit colocations with the group's (other) children -+ for (GList *iter = rsc->children; iter != NULL; iter = iter->next) { -+ pe_resource_t *member = iter->data; -+ -+ if (member != orig_rsc) { -+ member->cmds->with_this_colocations(member, orig_rsc, list); - } - } - } -@@ -693,6 +713,8 @@ void - pcmk__group_with_colocations(const pe_resource_t *rsc, - const pe_resource_t *orig_rsc, GList **list) - { -+ const pe_resource_t *member = NULL; -+ - CRM_CHECK((rsc != NULL) && (rsc->variant == pe_group) - && (orig_rsc != NULL) && (list != NULL), - return); -@@ -702,18 +724,35 @@ pcmk__group_with_colocations(const pe_resource_t *rsc, - return; - } - -- /* Colocations for the group itself, or for its first member, consist of the -- * group's colocations plus any relevant parent colocations if cloned. -+ /* "This with" colocations are normally needed only for the group itself and -+ * for its first member. - */ - if ((rsc == orig_rsc) - || (orig_rsc == (const pe_resource_t *) rsc->children->data)) { -- crm_trace("Adding '%s with' colocations to list for %s", -- rsc->id, orig_rsc->id); -+ pe_rsc_trace(rsc, "Adding '%s with' colocations to list for %s", -+ rsc->id, orig_rsc->id); -+ -+ // Add the group's own colocations - pcmk__add_this_with_list(list, rsc->rsc_cons); -- if (rsc->parent != NULL) { // Cloned group -+ -+ // If cloned, add any relevant colocations involving the clone -+ if (rsc->parent != NULL) { - rsc->parent->cmds->this_with_colocations(rsc->parent, orig_rsc, - list); - } -+ -+ if (!pe__group_flag_is_set(rsc, pe__group_colocated)) { -+ // @COMPAT Non-colocated groups are deprecated -+ return; -+ } -+ -+ // Add explicit colocations involving the group's (other) children -+ for (GList *iter = rsc->children; iter != NULL; iter = iter->next) { -+ member = iter->data; -+ if (member != orig_rsc) { -+ member->cmds->this_with_colocations(member, orig_rsc, list); -+ } -+ } - return; - } - -@@ -723,8 +762,7 @@ pcmk__group_with_colocations(const pe_resource_t *rsc, - * happen, so the group's mandatory colocations must be explicitly added. - */ - for (GList *iter = rsc->children; iter != NULL; iter = iter->next) { -- const pe_resource_t *member = (const pe_resource_t *) iter->data; -- -+ member = iter->data; - if (orig_rsc == member) { - break; // We've seen all earlier members, and none are unmanaged - } -diff --git a/lib/pacemaker/pcmk_sched_primitive.c b/lib/pacemaker/pcmk_sched_primitive.c -index d6b39e38c5f..bfc6fc7fedd 100644 ---- a/lib/pacemaker/pcmk_sched_primitive.c -+++ b/lib/pacemaker/pcmk_sched_primitive.c -@@ -1069,15 +1069,25 @@ void - pcmk__with_primitive_colocations(const pe_resource_t *rsc, - const pe_resource_t *orig_rsc, GList **list) - { -- // Primitives don't have children, so rsc should also be orig_rsc -- CRM_CHECK((rsc != NULL) && (rsc->variant == pe_native) -- && (rsc == orig_rsc) && (list != NULL), -- return); -+ CRM_ASSERT((rsc != NULL) && (rsc->variant == pe_native) && (list != NULL)); - -- // Add primitive's own colocations plus any relevant ones from parent -- pcmk__add_with_this_list(list, rsc->rsc_cons_lhs); -- if (rsc->parent != NULL) { -- rsc->parent->cmds->with_this_colocations(rsc->parent, rsc, list); -+ if (rsc == orig_rsc) { -+ /* For the resource itself, add all of its own colocations and relevant -+ * colocations from its parent (if any). -+ */ -+ pcmk__add_with_this_list(list, rsc->rsc_cons_lhs); -+ if (rsc->parent != NULL) { -+ rsc->parent->cmds->with_this_colocations(rsc->parent, rsc, list); -+ } -+ } else { -+ // For an ancestor, add only explicitly configured constraints -+ for (GList *iter = rsc->rsc_cons_lhs; iter != NULL; iter = iter->next) { -+ pcmk__colocation_t *colocation = iter->data; -+ -+ if (pcmk_is_set(colocation->flags, pcmk__coloc_explicit)) { -+ pcmk__add_with_this(list, colocation); -+ } -+ } - } - } - -@@ -1088,15 +1098,25 @@ void - pcmk__primitive_with_colocations(const pe_resource_t *rsc, - const pe_resource_t *orig_rsc, GList **list) - { -- // Primitives don't have children, so rsc should also be orig_rsc -- CRM_CHECK((rsc != NULL) && (rsc->variant == pe_native) -- && (rsc == orig_rsc) && (list != NULL), -- return); -+ CRM_ASSERT((rsc != NULL) && (rsc->variant == pe_native) && (list != NULL)); - -- // Add primitive's own colocations plus any relevant ones from parent -- pcmk__add_this_with_list(list, rsc->rsc_cons); -- if (rsc->parent != NULL) { -- rsc->parent->cmds->this_with_colocations(rsc->parent, rsc, list); -+ if (rsc == orig_rsc) { -+ /* For the resource itself, add all of its own colocations and relevant -+ * colocations from its parent (if any). -+ */ -+ pcmk__add_this_with_list(list, rsc->rsc_cons); -+ if (rsc->parent != NULL) { -+ rsc->parent->cmds->this_with_colocations(rsc->parent, rsc, list); -+ } -+ } else { -+ // For an ancestor, add only explicitly configured constraints -+ for (GList *iter = rsc->rsc_cons; iter != NULL; iter = iter->next) { -+ pcmk__colocation_t *colocation = iter->data; -+ -+ if (pcmk_is_set(colocation->flags, pcmk__coloc_explicit)) { -+ pcmk__add_this_with(list, colocation); -+ } -+ } - } - } - - -From e9e734eabf147a827c8bc6731da4c54b2a4d8658 Mon Sep 17 00:00:00 2001 -From: Ken Gaillot -Date: Tue, 27 Jun 2023 10:31:18 -0500 -Subject: [PATCH 13/17] Test: scheduler: update test output for group - colocation fix - ---- - .../dot/coloc-with-inner-group-member.dot | 32 ++++ - .../exp/coloc-with-inner-group-member.exp | 176 +++++++++++++++++- - .../coloc-with-inner-group-member.scores | 14 +- - .../coloc-with-inner-group-member.summary | 20 +- - 4 files changed, 225 insertions(+), 17 deletions(-) - -diff --git a/cts/scheduler/dot/coloc-with-inner-group-member.dot b/cts/scheduler/dot/coloc-with-inner-group-member.dot -index 77e1a8e6e40..a3bad7aab12 100644 ---- a/cts/scheduler/dot/coloc-with-inner-group-member.dot -+++ b/cts/scheduler/dot/coloc-with-inner-group-member.dot -@@ -1,8 +1,40 @@ - digraph "g" { -+"bar_monitor_10000 rhel8-3" [ style=bold color="green" fontcolor="black"] -+"bar_start_0 rhel8-3" -> "bar_monitor_10000 rhel8-3" [ style = bold] -+"bar_start_0 rhel8-3" -> "grp_running_0" [ style = bold] -+"bar_start_0 rhel8-3" -> "vip_start_0 rhel8-3" [ style = bold] -+"bar_start_0 rhel8-3" [ style=bold color="green" fontcolor="black"] -+"bar_stop_0 rhel8-4" -> "bar_start_0 rhel8-3" [ style = bold] -+"bar_stop_0 rhel8-4" -> "foo_stop_0 rhel8-4" [ style = bold] -+"bar_stop_0 rhel8-4" -> "grp_stopped_0" [ style = bold] -+"bar_stop_0 rhel8-4" [ style=bold color="green" fontcolor="black"] -+"foo_monitor_10000 rhel8-3" [ style=bold color="green" fontcolor="black"] -+"foo_start_0 rhel8-3" -> "bar_start_0 rhel8-3" [ style = bold] -+"foo_start_0 rhel8-3" -> "foo_monitor_10000 rhel8-3" [ style = bold] -+"foo_start_0 rhel8-3" -> "grp_running_0" [ style = bold] -+"foo_start_0 rhel8-3" [ style=bold color="green" fontcolor="black"] -+"foo_stop_0 rhel8-4" -> "foo_start_0 rhel8-3" [ style = bold] -+"foo_stop_0 rhel8-4" -> "grp_stopped_0" [ style = bold] -+"foo_stop_0 rhel8-4" [ style=bold color="green" fontcolor="black"] -+"grp_running_0" [ style=bold color="green" fontcolor="orange"] -+"grp_start_0" -> "bar_start_0 rhel8-3" [ style = bold] -+"grp_start_0" -> "foo_start_0 rhel8-3" [ style = bold] -+"grp_start_0" -> "grp_running_0" [ style = bold] -+"grp_start_0" -> "vip_start_0 rhel8-3" [ style = bold] -+"grp_start_0" [ style=bold color="green" fontcolor="orange"] -+"grp_stop_0" -> "bar_stop_0 rhel8-4" [ style = bold] -+"grp_stop_0" -> "foo_stop_0 rhel8-4" [ style = bold] - "grp_stop_0" -> "grp_stopped_0" [ style = bold] - "grp_stop_0" -> "vip_stop_0 rhel8-3" [ style = bold] - "grp_stop_0" [ style=bold color="green" fontcolor="orange"] -+"grp_stopped_0" -> "grp_start_0" [ style = bold] - "grp_stopped_0" [ style=bold color="green" fontcolor="orange"] -+"vip_monitor_10000 rhel8-3" [ style=bold color="green" fontcolor="black"] -+"vip_start_0 rhel8-3" -> "grp_running_0" [ style = bold] -+"vip_start_0 rhel8-3" -> "vip_monitor_10000 rhel8-3" [ style = bold] -+"vip_start_0 rhel8-3" [ style=bold color="green" fontcolor="black"] -+"vip_stop_0 rhel8-3" -> "bar_stop_0 rhel8-4" [ style = bold] - "vip_stop_0 rhel8-3" -> "grp_stopped_0" [ style = bold] -+"vip_stop_0 rhel8-3" -> "vip_start_0 rhel8-3" [ style = bold] - "vip_stop_0 rhel8-3" [ style=bold color="green" fontcolor="black"] - } -diff --git a/cts/scheduler/exp/coloc-with-inner-group-member.exp b/cts/scheduler/exp/coloc-with-inner-group-member.exp -index e6d94d5fe7f..bb8f779feb1 100644 ---- a/cts/scheduler/exp/coloc-with-inner-group-member.exp -+++ b/cts/scheduler/exp/coloc-with-inner-group-member.exp -@@ -1,22 +1,28 @@ - - - -- -+ - - - - - -- -+ - - -- -+ -+ -+ -+ -+ -+ -+ - - - - - -- -+ - - - -@@ -24,14 +30,172 @@ - - - -- -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ - - - - - - -- -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ - - - -diff --git a/cts/scheduler/scores/coloc-with-inner-group-member.scores b/cts/scheduler/scores/coloc-with-inner-group-member.scores -index 10fe944cb42..8d1c6f621c1 100644 ---- a/cts/scheduler/scores/coloc-with-inner-group-member.scores -+++ b/cts/scheduler/scores/coloc-with-inner-group-member.scores -@@ -26,17 +26,17 @@ pcmk__primitive_assign: Fencing allocation score on rhel8-4: 0 - pcmk__primitive_assign: Fencing allocation score on rhel8-5: 0 - pcmk__primitive_assign: bar allocation score on rhel8-1: -INFINITY - pcmk__primitive_assign: bar allocation score on rhel8-2: -INFINITY --pcmk__primitive_assign: bar allocation score on rhel8-3: -INFINITY --pcmk__primitive_assign: bar allocation score on rhel8-4: 0 -+pcmk__primitive_assign: bar allocation score on rhel8-3: 0 -+pcmk__primitive_assign: bar allocation score on rhel8-4: -INFINITY - pcmk__primitive_assign: bar allocation score on rhel8-5: -INFINITY --pcmk__primitive_assign: foo allocation score on rhel8-1: 0 --pcmk__primitive_assign: foo allocation score on rhel8-2: 0 -+pcmk__primitive_assign: foo allocation score on rhel8-1: -INFINITY -+pcmk__primitive_assign: foo allocation score on rhel8-2: -INFINITY - pcmk__primitive_assign: foo allocation score on rhel8-3: 0 --pcmk__primitive_assign: foo allocation score on rhel8-4: 0 --pcmk__primitive_assign: foo allocation score on rhel8-5: 0 -+pcmk__primitive_assign: foo allocation score on rhel8-4: -INFINITY -+pcmk__primitive_assign: foo allocation score on rhel8-5: -INFINITY - pcmk__primitive_assign: vip allocation score on rhel8-1: -INFINITY - pcmk__primitive_assign: vip allocation score on rhel8-2: -INFINITY --pcmk__primitive_assign: vip allocation score on rhel8-3: -INFINITY -+pcmk__primitive_assign: vip allocation score on rhel8-3: 0 - pcmk__primitive_assign: vip allocation score on rhel8-4: -INFINITY - pcmk__primitive_assign: vip allocation score on rhel8-5: -INFINITY - pcmk__primitive_assign: vip-dep allocation score on rhel8-1: 0 -diff --git a/cts/scheduler/summary/coloc-with-inner-group-member.summary b/cts/scheduler/summary/coloc-with-inner-group-member.summary -index 3e87f0867ef..6659721a79c 100644 ---- a/cts/scheduler/summary/coloc-with-inner-group-member.summary -+++ b/cts/scheduler/summary/coloc-with-inner-group-member.summary -@@ -12,12 +12,24 @@ Current cluster status: - * vip (ocf:pacemaker:Dummy): Started rhel8-3 - - Transition Summary: -- * Stop vip ( rhel8-3 ) due to node availability -+ * Move foo ( rhel8-4 -> rhel8-3 ) -+ * Move bar ( rhel8-4 -> rhel8-3 ) -+ * Restart vip ( rhel8-3 ) due to required bar start - - Executing Cluster Transition: - * Pseudo action: grp_stop_0 - * Resource action: vip stop on rhel8-3 -+ * Resource action: bar stop on rhel8-4 -+ * Resource action: foo stop on rhel8-4 - * Pseudo action: grp_stopped_0 -+ * Pseudo action: grp_start_0 -+ * Resource action: foo start on rhel8-3 -+ * Resource action: bar start on rhel8-3 -+ * Resource action: vip start on rhel8-3 -+ * Resource action: vip monitor=10000 on rhel8-3 -+ * Pseudo action: grp_running_0 -+ * Resource action: foo monitor=10000 on rhel8-3 -+ * Resource action: bar monitor=10000 on rhel8-3 - Using the original execution date of: 2023-06-20 20:45:06Z - - Revised Cluster Status: -@@ -28,6 +40,6 @@ Revised Cluster Status: - * Fencing (stonith:fence_xvm): Started rhel8-1 - * vip-dep (ocf:pacemaker:Dummy): Started rhel8-3 - * Resource Group: grp: -- * foo (ocf:pacemaker:Dummy): Started rhel8-4 -- * bar (ocf:pacemaker:Dummy): Started rhel8-4 -- * vip (ocf:pacemaker:Dummy): Stopped -+ * foo (ocf:pacemaker:Dummy): Started rhel8-3 -+ * bar (ocf:pacemaker:Dummy): Started rhel8-3 -+ * vip (ocf:pacemaker:Dummy): Started rhel8-3 - -From 9ada709b568cf5050f768b83e4682a8b93d1b361 Mon Sep 17 00:00:00 2001 -From: Ken Gaillot -Date: Thu, 29 Jun 2023 09:01:41 -0500 -Subject: [PATCH 14/17] Fix: CIB: be more strict about ignoring colocation - elements without an ID - -Callers of pcmk__unpack_colocation() have more context about the element being -unpacked, so the checks are done there. ---- - lib/pacemaker/pcmk_sched_colocation.c | 24 ++++++++++++++++++------ - 1 file changed, 18 insertions(+), 6 deletions(-) - -diff --git a/lib/pacemaker/pcmk_sched_colocation.c b/lib/pacemaker/pcmk_sched_colocation.c -index e0b39b59e81..a2baddbbb5c 100644 ---- a/lib/pacemaker/pcmk_sched_colocation.c -+++ b/lib/pacemaker/pcmk_sched_colocation.c -@@ -886,23 +886,30 @@ pcmk__unpack_colocation(xmlNode *xml_obj, pe_working_set_t *data_set) - xmlNode *expanded_xml = NULL; - - const char *id = crm_element_value(xml_obj, XML_ATTR_ID); -- const char *score = crm_element_value(xml_obj, XML_RULE_ATTR_SCORE); -- const char *influence_s = crm_element_value(xml_obj, -- XML_COLOC_ATTR_INFLUENCE); -+ const char *score = NULL; -+ const char *influence_s = NULL; - -- if (score) { -- score_i = char2score(score); -+ if (pcmk__str_empty(id)) { -+ pcmk__config_err("Ignoring " XML_CONS_TAG_RSC_DEPEND -+ " without " CRM_ATTR_ID); -+ return; - } - - if (unpack_colocation_tags(xml_obj, &expanded_xml, - data_set) != pcmk_rc_ok) { - return; - } -- if (expanded_xml) { -+ if (expanded_xml != NULL) { - orig_xml = xml_obj; - xml_obj = expanded_xml; - } - -+ score = crm_element_value(xml_obj, XML_RULE_ATTR_SCORE); -+ if (score != NULL) { -+ score_i = char2score(score); -+ } -+ influence_s = crm_element_value(xml_obj, XML_COLOC_ATTR_INFLUENCE); -+ - for (set = first_named_child(xml_obj, XML_CONS_TAG_RSC_SET); set != NULL; - set = crm_next_same_xml(set)) { - -@@ -914,6 +921,11 @@ pcmk__unpack_colocation(xmlNode *xml_obj, pe_working_set_t *data_set) - return; - } - -+ if (pcmk__str_empty(ID(set))) { -+ pcmk__config_err("Ignoring " XML_CONS_TAG_RSC_SET -+ " without " CRM_ATTR_ID); -+ continue; -+ } - unpack_colocation_set(set, score_i, id, influence_s, data_set); - - if (last != NULL) { - -From e830a9663c80ea348eff694a8e71a1e07d380690 Mon Sep 17 00:00:00 2001 -From: Ken Gaillot -Date: Thu, 29 Jun 2023 09:40:57 -0500 -Subject: [PATCH 15/17] Log: scheduler: improve colocation unpacking messages - (and comments) - ---- - lib/pacemaker/pcmk_sched_colocation.c | 60 ++++++++++++++------------- - 1 file changed, 32 insertions(+), 28 deletions(-) - -diff --git a/lib/pacemaker/pcmk_sched_colocation.c b/lib/pacemaker/pcmk_sched_colocation.c -index a2baddbbb5c..9c9195ed02c 100644 ---- a/lib/pacemaker/pcmk_sched_colocation.c -+++ b/lib/pacemaker/pcmk_sched_colocation.c -@@ -136,13 +136,13 @@ pcmk__add_this_with(GList **list, const pcmk__colocation_t *colocation) - { - CRM_ASSERT((list != NULL) && (colocation != NULL)); - -- crm_trace("Adding colocation %s (%s with %s%s%s @%d) " -+ crm_trace("Adding colocation %s (%s with %s%s%s @%s) " - "to 'this with' list", - colocation->id, colocation->dependent->id, - colocation->primary->id, - (colocation->node_attribute == NULL)? "" : " using ", - pcmk__s(colocation->node_attribute, ""), -- colocation->score); -+ pcmk_readable_score(colocation->score)); - *list = g_list_insert_sorted(*list, (gpointer) colocation, - cmp_primary_priority); - } -@@ -187,13 +187,13 @@ pcmk__add_with_this(GList **list, const pcmk__colocation_t *colocation) - { - CRM_ASSERT((list != NULL) && (colocation != NULL)); - -- crm_trace("Adding colocation %s (%s with %s%s%s @%d) " -+ crm_trace("Adding colocation %s (%s with %s%s%s @%s) " - "to 'with this' list", - colocation->id, colocation->dependent->id, - colocation->primary->id, - (colocation->node_attribute == NULL)? "" : " using ", - pcmk__s(colocation->node_attribute, ""), -- colocation->score); -+ pcmk_readable_score(colocation->score)); - *list = g_list_insert_sorted(*list, (gpointer) colocation, - cmp_dependent_priority); - } -@@ -339,10 +339,6 @@ pcmk__new_colocation(const char *id, const char *node_attr, int score, - node_attr = CRM_ATTR_UNAME; - } - -- pe_rsc_trace(dependent, "Added colocation %s (%s with %s @%s using %s)", -- new_con->id, dependent->id, primary->id, -- pcmk_readable_score(score), node_attr); -- - pcmk__add_this_with(&(dependent->rsc_cons), new_con); - pcmk__add_with_this(&(primary->rsc_cons_lhs), new_con); - -@@ -495,8 +491,6 @@ unpack_colocation_set(xmlNode *set, int score, const char *coloc_id, - other = pcmk__find_constraint_resource(data_set->resources, - xml_rsc_id); - CRM_ASSERT(other != NULL); // We already processed it -- pe_rsc_trace(resource, "Anti-Colocating %s with %s", -- resource->id, other->id); - pcmk__new_colocation(set_id, NULL, local_score, - resource, other, role, role, flags); - } -@@ -504,9 +498,21 @@ unpack_colocation_set(xmlNode *set, int score, const char *coloc_id, - } - } - -+/*! -+ * \internal -+ * \brief Colocate two resource sets relative to each other -+ * -+ * \param[in] id Colocation XML ID -+ * \param[in] set1 Dependent set -+ * \param[in] set2 Primary set -+ * \param[in] score Colocation score -+ * \param[in] influence_s Value of colocation's "influence" attribute -+ * \param[in,out] data_set Cluster working set -+ */ - static void --colocate_rsc_sets(const char *id, xmlNode *set1, xmlNode *set2, int score, -- const char *influence_s, pe_working_set_t *data_set) -+colocate_rsc_sets(const char *id, const xmlNode *set1, const xmlNode *set2, -+ int score, const char *influence_s, -+ pe_working_set_t *data_set) - { - xmlNode *xml_rsc = NULL; - pe_resource_t *rsc_1 = NULL; -@@ -521,8 +527,8 @@ colocate_rsc_sets(const char *id, xmlNode *set1, xmlNode *set2, int score, - uint32_t flags = pcmk__coloc_none; - - if (score == 0) { -- crm_trace("Ignoring colocation '%s' between sets because score is 0", -- id); -+ crm_trace("Ignoring colocation '%s' between sets %s and %s " -+ "because score is 0", id, ID(set1), ID(set2)); - return; - } - -@@ -562,12 +568,12 @@ colocate_rsc_sets(const char *id, xmlNode *set1, xmlNode *set2, int score, - } - } - -- if ((rsc_1 != NULL) && (rsc_2 != NULL)) { -+ if ((rsc_1 != NULL) && (rsc_2 != NULL)) { // Both sets are sequential - flags = pcmk__coloc_explicit | unpack_influence(id, rsc_1, influence_s); - pcmk__new_colocation(id, NULL, score, rsc_1, rsc_2, role_1, role_2, - flags); - -- } else if (rsc_1 != NULL) { -+ } else if (rsc_1 != NULL) { // Only set1 is sequential - flags = pcmk__coloc_explicit | unpack_influence(id, rsc_1, influence_s); - for (xml_rsc = first_named_child(set2, XML_TAG_RESOURCE_REF); - xml_rsc != NULL; xml_rsc = crm_next_same_xml(xml_rsc)) { -@@ -576,19 +582,17 @@ colocate_rsc_sets(const char *id, xmlNode *set1, xmlNode *set2, int score, - rsc_2 = pcmk__find_constraint_resource(data_set->resources, - xml_rsc_id); - if (rsc_2 == NULL) { -- pcmk__config_err("%s: No resource found for %s", -- id, xml_rsc_id); - // Should be possible only with validation disabled -- pcmk__config_err("Ignoring resource %s and later in set %s " -- "for colocation with set %s: No such resource", -- xml_rsc_id, set2, set1); -+ pcmk__config_err("Ignoring set %s colocation with resource %s " -+ "and later in set %s: No such resource", -+ ID(set1), xml_rsc_id, ID(set2)); - return; - } - pcmk__new_colocation(id, NULL, score, rsc_1, rsc_2, role_1, - role_2, flags); - } - -- } else if (rsc_2 != NULL) { -+ } else if (rsc_2 != NULL) { // Only set2 is sequential - for (xml_rsc = first_named_child(set1, XML_TAG_RESOURCE_REF); - xml_rsc != NULL; xml_rsc = crm_next_same_xml(xml_rsc)) { - -@@ -599,7 +603,7 @@ colocate_rsc_sets(const char *id, xmlNode *set1, xmlNode *set2, int score, - // Should be possible only with validation disabled - pcmk__config_err("Ignoring resource %s and later in set %s " - "for colocation with set %s: No such resource", -- xml_rsc_id, set1, set2); -+ xml_rsc_id, ID(set1), ID(set2)); - return; - } - flags = pcmk__coloc_explicit -@@ -608,7 +612,7 @@ colocate_rsc_sets(const char *id, xmlNode *set1, xmlNode *set2, int score, - role_2, flags); - } - -- } else { -+ } else { // Neither set is sequential - for (xml_rsc = first_named_child(set1, XML_TAG_RESOURCE_REF); - xml_rsc != NULL; xml_rsc = crm_next_same_xml(xml_rsc)) { - -@@ -621,7 +625,7 @@ colocate_rsc_sets(const char *id, xmlNode *set1, xmlNode *set2, int score, - // Should be possible only with validation disabled - pcmk__config_err("Ignoring resource %s and later in set %s " - "for colocation with set %s: No such resource", -- xml_rsc_id, set1, set2); -+ xml_rsc_id, ID(set1), ID(set2)); - return; - } - -@@ -636,10 +640,10 @@ colocate_rsc_sets(const char *id, xmlNode *set1, xmlNode *set2, int score, - xml_rsc_id); - if (rsc_2 == NULL) { - // Should be possible only with validation disabled -- pcmk__config_err("Ignoring resource %s and later in set %s " -- "for colocation with %s in set %s: " -+ pcmk__config_err("Ignoring set %s resource %s colocation with " -+ "resource %s and later in set %s: " - "No such resource", -- xml_rsc_id, set2, ID(xml_rsc), set1); -+ ID(set1), ID(xml_rsc), xml_rsc_id, ID(set2)); - return; - } - pcmk__new_colocation(id, NULL, score, rsc_1, rsc_2, - -From 737d74b656cad7b5514397bb461b8a18fb5590df Mon Sep 17 00:00:00 2001 -From: Ken Gaillot -Date: Thu, 29 Jun 2023 09:49:13 -0500 -Subject: [PATCH 16/17] Low: scheduler: continue with non-sequential set - members after error - ---- - lib/pacemaker/pcmk_sched_colocation.c | 30 +++++++++++++-------------- - 1 file changed, 15 insertions(+), 15 deletions(-) - -diff --git a/lib/pacemaker/pcmk_sched_colocation.c b/lib/pacemaker/pcmk_sched_colocation.c -index 9c9195ed02c..3e094a4b87b 100644 ---- a/lib/pacemaker/pcmk_sched_colocation.c -+++ b/lib/pacemaker/pcmk_sched_colocation.c -@@ -584,9 +584,9 @@ colocate_rsc_sets(const char *id, const xmlNode *set1, const xmlNode *set2, - if (rsc_2 == NULL) { - // Should be possible only with validation disabled - pcmk__config_err("Ignoring set %s colocation with resource %s " -- "and later in set %s: No such resource", -+ "in set %s: No such resource", - ID(set1), xml_rsc_id, ID(set2)); -- return; -+ continue; - } - pcmk__new_colocation(id, NULL, score, rsc_1, rsc_2, role_1, - role_2, flags); -@@ -601,10 +601,10 @@ colocate_rsc_sets(const char *id, const xmlNode *set1, const xmlNode *set2, - xml_rsc_id); - if (rsc_1 == NULL) { - // Should be possible only with validation disabled -- pcmk__config_err("Ignoring resource %s and later in set %s " -- "for colocation with set %s: No such resource", -- xml_rsc_id, ID(set1), ID(set2)); -- return; -+ pcmk__config_err("Ignoring colocation of set %s resource %s " -+ "with set %s: No such resource", -+ ID(set1), xml_rsc_id, ID(set2)); -+ continue; - } - flags = pcmk__coloc_explicit - | unpack_influence(id, rsc_1, influence_s); -@@ -623,10 +623,10 @@ colocate_rsc_sets(const char *id, const xmlNode *set1, const xmlNode *set2, - xml_rsc_id); - if (rsc_1 == NULL) { - // Should be possible only with validation disabled -- pcmk__config_err("Ignoring resource %s and later in set %s " -- "for colocation with set %s: No such resource", -- xml_rsc_id, ID(set1), ID(set2)); -- return; -+ pcmk__config_err("Ignoring colocation of set %s resource %s " -+ "with set %s: No such resource", -+ ID(set1), xml_rsc_id, ID(set2)); -+ continue; - } - - flags = pcmk__coloc_explicit -@@ -640,11 +640,11 @@ colocate_rsc_sets(const char *id, const xmlNode *set1, const xmlNode *set2, - xml_rsc_id); - if (rsc_2 == NULL) { - // Should be possible only with validation disabled -- pcmk__config_err("Ignoring set %s resource %s colocation with " -- "resource %s and later in set %s: " -- "No such resource", -- ID(set1), ID(xml_rsc), xml_rsc_id, ID(set2)); -- return; -+ pcmk__config_err("Ignoring colocation of set %s resource " -+ "%s with set %s resource %s: No such " -+ "resource", ID(set1), ID(xml_rsc), -+ ID(set2), xml_rsc_id); -+ continue; - } - pcmk__new_colocation(id, NULL, score, rsc_1, rsc_2, - role_1, role_2, flags); - -From d9c8593f17975371e64e0c187bc8234e901349a9 Mon Sep 17 00:00:00 2001 -From: Ken Gaillot -Date: Thu, 29 Jun 2023 09:49:55 -0500 -Subject: [PATCH 17/17] Refactor: scheduler: make some variables const that can - be - ---- - lib/pacemaker/pcmk_sched_group.c | 9 +++++---- - 1 file changed, 5 insertions(+), 4 deletions(-) - -diff --git a/lib/pacemaker/pcmk_sched_group.c b/lib/pacemaker/pcmk_sched_group.c -index 95e2d77aa5f..a2bf5f6dcd4 100644 ---- a/lib/pacemaker/pcmk_sched_group.c -+++ b/lib/pacemaker/pcmk_sched_group.c -@@ -699,8 +699,8 @@ pcmk__with_group_colocations(const pe_resource_t *rsc, - } - - // Add explicit colocations with the group's (other) children -- for (GList *iter = rsc->children; iter != NULL; iter = iter->next) { -- pe_resource_t *member = iter->data; -+ for (const GList *iter = rsc->children; iter != NULL; iter = iter->next) { -+ const pe_resource_t *member = iter->data; - - if (member != orig_rsc) { - member->cmds->with_this_colocations(member, orig_rsc, list); -@@ -747,7 +747,8 @@ pcmk__group_with_colocations(const pe_resource_t *rsc, - } - - // Add explicit colocations involving the group's (other) children -- for (GList *iter = rsc->children; iter != NULL; iter = iter->next) { -+ for (const GList *iter = rsc->children; -+ iter != NULL; iter = iter->next) { - member = iter->data; - if (member != orig_rsc) { - member->cmds->this_with_colocations(member, orig_rsc, list); -@@ -761,7 +762,7 @@ pcmk__group_with_colocations(const pe_resource_t *rsc, - * However, if an earlier group member is unmanaged, this chaining will not - * happen, so the group's mandatory colocations must be explicitly added. - */ -- for (GList *iter = rsc->children; iter != NULL; iter = iter->next) { -+ for (const GList *iter = rsc->children; iter != NULL; iter = iter->next) { - member = iter->data; - if (orig_rsc == member) { - break; // We've seen all earlier members, and none are unmanaged diff --git a/SOURCES/002-schema-transfer.patch b/SOURCES/002-schema-transfer.patch new file mode 100644 index 0000000..9c1c05b --- /dev/null +++ b/SOURCES/002-schema-transfer.patch @@ -0,0 +1,1986 @@ +From 9e0c58dc3b949e4eadfa43364e60677478b7aa0f Mon Sep 17 00:00:00 2001 +From: Chris Lumens +Date: Thu, 7 Sep 2023 16:48:52 -0400 +Subject: [PATCH 01/15] Refactor: libcrmcommon: Break schema version comparison + out... + +...into its own function. Then, wrap this function with +schema_sort_directory so existing code doesn't need to be changed. This +allows us to use the version comparison elsewhere. +--- + lib/common/schemas.c | 26 ++++++++++++++++---------- + 1 file changed, 16 insertions(+), 10 deletions(-) + +diff --git a/lib/common/schemas.c b/lib/common/schemas.c +index 372e87223..b3ff05917 100644 +--- a/lib/common/schemas.c ++++ b/lib/common/schemas.c +@@ -195,7 +195,20 @@ schema_filter(const struct dirent *a) + } + + static int +-schema_sort(const struct dirent **a, const struct dirent **b) ++schema_cmp(pcmk__schema_version_t a_version, pcmk__schema_version_t b_version) ++{ ++ for (int i = 0; i < 2; ++i) { ++ if (a_version.v[i] < b_version.v[i]) { ++ return -1; ++ } else if (a_version.v[i] > b_version.v[i]) { ++ return 1; ++ } ++ } ++ return 0; ++} ++ ++static int ++schema_cmp_directory(const struct dirent **a, const struct dirent **b) + { + pcmk__schema_version_t a_version = SCHEMA_ZERO; + pcmk__schema_version_t b_version = SCHEMA_ZERO; +@@ -206,14 +219,7 @@ schema_sort(const struct dirent **a, const struct dirent **b) + return 0; + } + +- for (int i = 0; i < 2; ++i) { +- if (a_version.v[i] < b_version.v[i]) { +- return -1; +- } else if (a_version.v[i] > b_version.v[i]) { +- return 1; +- } +- } +- return 0; ++ return schema_cmp(a_version, b_version); + } + + /*! +@@ -413,7 +419,7 @@ crm_schema_init(void) + + wrap_libxslt(false); + +- max = scandir(base, &namelist, schema_filter, schema_sort); ++ max = scandir(base, &namelist, schema_filter, schema_cmp_directory); + if (max < 0) { + crm_notice("scandir(%s) failed: %s (%d)", base, strerror(errno), errno); + free(base); +-- +2.41.0 + +From e7d7c33eb8329c3c50fe648133cbb7651c1ecb9d Mon Sep 17 00:00:00 2001 +From: Chris Lumens +Date: Wed, 29 Nov 2023 09:37:31 -0500 +Subject: [PATCH 02/15] Feature: libcrmcommon: Add + pcmk__schema_files_later_than. + +This function takes in a schema version and returns a list of all RNG +and XSLT files from all schema versions more recent than that one. + +Also, add unit tests for the new function. +--- + include/crm/common/internal.h | 1 + + lib/common/schemas.c | 62 ++++++++++- + lib/common/tests/schemas/Makefile.am | 42 ++++++- + .../pcmk__schema_files_later_than_test.c | 104 ++++++++++++++++++ + 4 files changed, 207 insertions(+), 2 deletions(-) + create mode 100644 lib/common/tests/schemas/pcmk__schema_files_later_than_test.c + +diff --git a/include/crm/common/internal.h b/include/crm/common/internal.h +index 307860636..a3cd455dc 100644 +--- a/include/crm/common/internal.h ++++ b/include/crm/common/internal.h +@@ -134,6 +134,7 @@ bool pcmk__procfs_has_pids(void); + void crm_schema_init(void); + void crm_schema_cleanup(void); + ++GList *pcmk__schema_files_later_than(const char *name); + + /* internal functions related to process IDs (from pid.c) */ + +diff --git a/lib/common/schemas.c b/lib/common/schemas.c +index b3ff05917..1c60738ec 100644 +--- a/lib/common/schemas.c ++++ b/lib/common/schemas.c +@@ -168,7 +168,11 @@ xml_latest_schema(void) + static inline bool + version_from_filename(const char *filename, pcmk__schema_version_t *version) + { +- return sscanf(filename, "pacemaker-%hhu.%hhu.rng", &(version->v[0]), &(version->v[1])) == 2; ++ if (pcmk__ends_with(filename, ".rng")) { ++ return sscanf(filename, "pacemaker-%hhu.%hhu.rng", &(version->v[0]), &(version->v[1])) == 2; ++ } else { ++ return sscanf(filename, "pacemaker-%hhu.%hhu", &(version->v[0]), &(version->v[1])) == 2; ++ } + } + + static int +@@ -1291,6 +1295,62 @@ cli_config_update(xmlNode **xml, int *best_version, gboolean to_logs) + return rc; + } + ++/*! ++ * \internal ++ * \brief Return a list of all schema files and any associated XSLT files ++ * later than the given one ++ * \brief Return a list of all schema versions later than the given one ++ * ++ * \param[in] schema The schema to compare against (for example, ++ * "pacemaker-3.1.rng" or "pacemaker-3.1") ++ * ++ * \note The caller is responsible for freeing both the returned list and ++ * the elements of the list ++ */ ++GList * ++pcmk__schema_files_later_than(const char *name) ++{ ++ GList *lst = NULL; ++ pcmk__schema_version_t ver; ++ ++ if (!version_from_filename(name, &ver)) { ++ return lst; ++ } ++ ++ for (GList *iter = g_list_nth(known_schemas, xml_latest_schema_index(known_schemas)); ++ iter != NULL; iter = iter->prev) { ++ pcmk__schema_t *schema = iter->data; ++ char *s = NULL; ++ ++ if (schema_cmp(ver, schema->version) != -1) { ++ continue; ++ } ++ ++ s = crm_strdup_printf("%s.rng", schema->name); ++ lst = g_list_prepend(lst, s); ++ ++ if (schema->transform != NULL) { ++ char *xform = crm_strdup_printf("%s.xsl", schema->transform); ++ lst = g_list_prepend(lst, xform); ++ } ++ ++ if (schema->transform_enter != NULL) { ++ char *enter = crm_strdup_printf("%s.xsl", schema->transform_enter); ++ ++ lst = g_list_prepend(lst, enter); ++ ++ if (schema->transform_onleave) { ++ int last_dash = strrchr(enter, '-') - enter; ++ char *leave = crm_strdup_printf("%.*s-leave.xsl", last_dash, enter); ++ ++ lst = g_list_prepend(lst, leave); ++ } ++ } ++ } ++ ++ return lst; ++} ++ + void + pcmk__log_known_schemas(void) + { +diff --git a/lib/common/tests/schemas/Makefile.am b/lib/common/tests/schemas/Makefile.am +index 5f485b3e9..b5c5e7f3c 100644 +--- a/lib/common/tests/schemas/Makefile.am ++++ b/lib/common/tests/schemas/Makefile.am +@@ -10,7 +10,47 @@ + include $(top_srcdir)/mk/tap.mk + include $(top_srcdir)/mk/unittest.mk + ++CFLAGS += -DPCMK__TEST_SCHEMA_DIR='"$(abs_builddir)/schemas"' ++ + # Add "_test" to the end of all test program names to simplify .gitignore. +-check_PROGRAMS = pcmk__xml_find_x_0_schema_index_test ++check_PROGRAMS = pcmk__schema_files_later_than_test \ ++ pcmk__xml_find_x_0_schema_index_test + + TESTS = $(check_PROGRAMS) ++ ++$(TESTS): setup-schema-dir ++ ++# Set up a temporary schemas/ directory containing only some of the full set of ++# pacemaker schema files. This lets us know exactly how many schemas are present, ++# allowing us to write tests without having to make changes when new schemas are ++# added. ++# ++# This directory contains the following: ++# ++# * pacemaker-next.rng - Used to verify that this sorts before all versions ++# * upgrade-*.xsl - Required by various schema versions ++# * pacemaker-[0-9]*.rng - We're only pulling in 15 schemas, which is enough ++# to get everything through pacemaker-3.0.rng. This ++# includes 2.10, needed so we can check that versions ++# are compared as numbers instead of strings. ++# * other RNG files - This catches everything except the pacemaker-*rng ++# files. These files are included by the top-level ++# pacemaker-*rng files, so we need them for tests. ++# This will glob more than we need, but the extra ones ++# won't get in the way. ++.PHONY: setup-schema-dir ++setup-schema-dir: ++ $(MKDIR_P) schemas ++ ( cd schemas ; \ ++ ln -sf $(abs_top_builddir)/xml/pacemaker-next.rng . ; \ ++ ln -sf $(abs_top_builddir)/xml/upgrade-*.xsl . ; \ ++ for f in $(shell ls -1v $(abs_top_builddir)/xml/pacemaker-[0-9]*.rng | head -15); do \ ++ ln -sf $$f $$(basename $$f); \ ++ done ; \ ++ for f in $(shell ls -1 $(top_srcdir)/xml/*.rng | grep -v pacemaker); do \ ++ ln -sf ../$$f $$(basename $$f); \ ++ done ) ++ ++.PHONY: clean-local ++clean-local: ++ -rm -rf schemas +diff --git a/lib/common/tests/schemas/pcmk__schema_files_later_than_test.c b/lib/common/tests/schemas/pcmk__schema_files_later_than_test.c +new file mode 100644 +index 000000000..50b5f4334 +--- /dev/null ++++ b/lib/common/tests/schemas/pcmk__schema_files_later_than_test.c +@@ -0,0 +1,104 @@ ++/* ++ * Copyright 2023 the Pacemaker project contributors ++ * ++ * The version control history for this file may have further details. ++ * ++ * This source code is licensed under the GNU General Public License version 2 ++ * or later (GPLv2+) WITHOUT ANY WARRANTY. ++ */ ++ ++#include ++ ++#include ++#include ++ ++#include ++ ++static int ++setup(void **state) { ++ setenv("PCMK_schema_directory", PCMK__TEST_SCHEMA_DIR, 1); ++ crm_schema_init(); ++ return 0; ++} ++ ++static int ++teardown(void **state) { ++ crm_schema_cleanup(); ++ unsetenv("PCMK_schema_directory"); ++ return 0; ++} ++ ++static void ++invalid_name(void **state) ++{ ++ assert_null(pcmk__schema_files_later_than("xyz")); ++ assert_null(pcmk__schema_files_later_than("pacemaker-")); ++} ++ ++static void ++valid_name(void **state) ++{ ++ GList *schemas = NULL; ++ ++ schemas = pcmk__schema_files_later_than("pacemaker-1.0"); ++ assert_int_equal(g_list_length(schemas), 18); ++ /* There is no "pacemaker-1.1". */ ++ assert_string_equal("pacemaker-1.2.rng", g_list_nth_data(schemas, 0)); ++ assert_string_equal("upgrade-1.3.xsl", g_list_nth_data(schemas, 1)); ++ assert_string_equal("pacemaker-1.3.rng", g_list_nth_data(schemas, 2)); ++ assert_string_equal("pacemaker-2.0.rng", g_list_nth_data(schemas, 3)); ++ assert_string_equal("pacemaker-2.1.rng", g_list_nth_data(schemas, 4)); ++ assert_string_equal("pacemaker-2.2.rng", g_list_nth_data(schemas, 5)); ++ assert_string_equal("pacemaker-2.3.rng", g_list_nth_data(schemas, 6)); ++ assert_string_equal("pacemaker-2.4.rng", g_list_nth_data(schemas, 7)); ++ assert_string_equal("pacemaker-2.5.rng", g_list_nth_data(schemas, 8)); ++ assert_string_equal("pacemaker-2.6.rng", g_list_nth_data(schemas, 9)); ++ assert_string_equal("pacemaker-2.7.rng", g_list_nth_data(schemas, 10)); ++ assert_string_equal("pacemaker-2.8.rng", g_list_nth_data(schemas, 11)); ++ assert_string_equal("pacemaker-2.9.rng", g_list_nth_data(schemas, 12)); ++ assert_string_equal("upgrade-2.10-leave.xsl", g_list_nth_data(schemas, 13)); ++ assert_string_equal("upgrade-2.10-enter.xsl", g_list_nth_data(schemas, 14)); ++ assert_string_equal("upgrade-2.10.xsl", g_list_nth_data(schemas, 15)); ++ assert_string_equal("pacemaker-2.10.rng", g_list_nth_data(schemas, 16)); ++ assert_string_equal("pacemaker-3.0.rng", g_list_nth_data(schemas, 17)); ++ g_list_free_full(schemas, free); ++ ++ /* Adding .rng to the end of the schema we're requesting is also valid. */ ++ schemas = pcmk__schema_files_later_than("pacemaker-2.0.rng"); ++ assert_int_equal(g_list_length(schemas), 14); ++ assert_string_equal("pacemaker-2.1.rng", g_list_nth_data(schemas, 0)); ++ assert_string_equal("pacemaker-2.2.rng", g_list_nth_data(schemas, 1)); ++ assert_string_equal("pacemaker-2.3.rng", g_list_nth_data(schemas, 2)); ++ assert_string_equal("pacemaker-2.4.rng", g_list_nth_data(schemas, 3)); ++ assert_string_equal("pacemaker-2.5.rng", g_list_nth_data(schemas, 4)); ++ assert_string_equal("pacemaker-2.6.rng", g_list_nth_data(schemas, 5)); ++ assert_string_equal("pacemaker-2.7.rng", g_list_nth_data(schemas, 6)); ++ assert_string_equal("pacemaker-2.8.rng", g_list_nth_data(schemas, 7)); ++ assert_string_equal("pacemaker-2.9.rng", g_list_nth_data(schemas, 8)); ++ assert_string_equal("upgrade-2.10-leave.xsl", g_list_nth_data(schemas, 9)); ++ assert_string_equal("upgrade-2.10-enter.xsl", g_list_nth_data(schemas, 10)); ++ assert_string_equal("upgrade-2.10.xsl", g_list_nth_data(schemas, 11)); ++ assert_string_equal("pacemaker-2.10.rng", g_list_nth_data(schemas, 12)); ++ assert_string_equal("pacemaker-3.0.rng", g_list_nth_data(schemas, 13)); ++ g_list_free_full(schemas, free); ++ ++ /* Check that "pacemaker-2.10" counts as later than "pacemaker-2.9". */ ++ schemas = pcmk__schema_files_later_than("pacemaker-2.9"); ++ assert_int_equal(g_list_length(schemas), 5); ++ assert_string_equal("upgrade-2.10-leave.xsl", g_list_nth_data(schemas, 0)); ++ assert_string_equal("upgrade-2.10-enter.xsl", g_list_nth_data(schemas, 1)); ++ assert_string_equal("upgrade-2.10.xsl", g_list_nth_data(schemas, 2)); ++ assert_string_equal("pacemaker-2.10.rng", g_list_nth_data(schemas, 3)); ++ assert_string_equal("pacemaker-3.0.rng", g_list_nth_data(schemas, 4)); ++ g_list_free_full(schemas, free); ++ ++ /* And then something way in the future that will never apply due to our ++ * special schema directory. ++ */ ++ schemas = pcmk__schema_files_later_than("pacemaker-9.0"); ++ assert_null(schemas); ++} ++ ++PCMK__UNIT_TEST(setup, teardown, ++ cmocka_unit_test(invalid_name), ++ cmocka_unit_test(valid_name)) +-- +2.41.0 + +From 76859d61f4d35e1f8b7c35d8766d3d0c123d4552 Mon Sep 17 00:00:00 2001 +From: Chris Lumens +Date: Mon, 11 Sep 2023 08:56:22 -0400 +Subject: [PATCH 03/15] Refactor: libcrmcommon: Allow more specs in + pcmk__xml_artefact_path. + +If the given filespec already has a .rng or .xsl extension, don't add an +additional one. This allows reusing this function to grab files given +in another schema file's externalRef links without modification. +--- + lib/common/xml.c | 12 ++++++++++-- + 1 file changed, 10 insertions(+), 2 deletions(-) + +diff --git a/lib/common/xml.c b/lib/common/xml.c +index 53ebff770..142b501df 100644 +--- a/lib/common/xml.c ++++ b/lib/common/xml.c +@@ -2618,11 +2618,19 @@ pcmk__xml_artefact_path(enum pcmk__xml_artefact_ns ns, const char *filespec) + switch (ns) { + case pcmk__xml_artefact_ns_legacy_rng: + case pcmk__xml_artefact_ns_base_rng: +- ret = crm_strdup_printf("%s/%s.rng", base, filespec); ++ if (pcmk__ends_with(filespec, ".rng")) { ++ ret = crm_strdup_printf("%s/%s", base, filespec); ++ } else { ++ ret = crm_strdup_printf("%s/%s.rng", base, filespec); ++ } + break; + case pcmk__xml_artefact_ns_legacy_xslt: + case pcmk__xml_artefact_ns_base_xslt: +- ret = crm_strdup_printf("%s/%s.xsl", base, filespec); ++ if (pcmk__ends_with(filespec, ".xsl")) { ++ ret = crm_strdup_printf("%s/%s", base, filespec); ++ } else { ++ ret = crm_strdup_printf("%s/%s.xsl", base, filespec); ++ } + break; + default: + crm_err("XML artefact family specified as %u not recognized", ns); +-- +2.41.0 + +From 0d702bba5b50e1eede201853c7680a2517fade9f Mon Sep 17 00:00:00 2001 +From: Chris Lumens +Date: Mon, 18 Sep 2023 12:37:46 -0400 +Subject: [PATCH 04/15] Feature: build: Add PCMK__REMOTE_SCHEMA_DIR to + configure.ac. + +This is a new subdirectory where schema files downloaded to a remote +executor can be stored. + +Also, add support for an environment variable that can override this +compile-time setting and explain it in the documentation. +--- + configure.ac | 5 +++++ + doc/sphinx/Pacemaker_Explained/local-options.rst | 12 ++++++++++++ + etc/sysconfig/pacemaker.in | 7 +++++++ + include/crm/common/options_internal.h | 1 + + 4 files changed, 25 insertions(+) + +diff --git a/configure.ac b/configure.ac +index 17cee41e9..bd548200c 100644 +--- a/configure.ac ++++ b/configure.ac +@@ -838,6 +838,11 @@ AC_DEFINE_UNQUOTED([CRM_SCHEMA_DIRECTORY], ["$CRM_SCHEMA_DIRECTORY"], + [Location for the Pacemaker Relax-NG Schema]) + AC_SUBST(CRM_SCHEMA_DIRECTORY) + ++PCMK__REMOTE_SCHEMA_DIR="${localstatedir}/lib/pacemaker/schemas" ++AC_DEFINE_UNQUOTED([PCMK__REMOTE_SCHEMA_DIR], ["$PCMK__REMOTE_SCHEMA_DIR"], ++ [Location to store Relax-NG Schema files on remote nodes]) ++AC_SUBST(PCMK__REMOTE_SCHEMA_DIR) ++ + CRM_CORE_DIR="${localstatedir}/lib/pacemaker/cores" + AC_DEFINE_UNQUOTED([CRM_CORE_DIR], ["$CRM_CORE_DIR"], + [Directory Pacemaker daemons should change to (without systemd, core files will go here)]) +diff --git a/doc/sphinx/Pacemaker_Explained/local-options.rst b/doc/sphinx/Pacemaker_Explained/local-options.rst +index 91eda6632..118256247 100644 +--- a/doc/sphinx/Pacemaker_Explained/local-options.rst ++++ b/doc/sphinx/Pacemaker_Explained/local-options.rst +@@ -478,6 +478,18 @@ environment variables when Pacemaker daemons start up. + - *Advanced Use Only:* Specify an alternate location for RNG schemas and + XSL transforms. + ++ * - .. _pcmk_remote_schema_directory: ++ ++ .. index:: ++ pair:: node option; PCMK_remote_schema_directory ++ ++ PCMK_remote_schema_directory ++ - :ref:`text ` ++ - |PCMK__REMOTE_SCHEMA_DIR| ++ - *Advanced Use Only:* Specify an alternate location on Pacemaker Remote ++ nodes for storing newer RNG schemas and XSL transforms fetched from ++ the cluster. ++ + * - .. _pcmk_valgrind_enabled: + + .. index:: +diff --git a/etc/sysconfig/pacemaker.in b/etc/sysconfig/pacemaker.in +index 0c3609d8e..487a104a7 100644 +--- a/etc/sysconfig/pacemaker.in ++++ b/etc/sysconfig/pacemaker.in +@@ -339,6 +339,13 @@ + # + # Default: PCMK_schema_directory="@CRM_SCHEMA_DIRECTORY@" + ++# PCMK_remote_schema_directory (Advanced Use Only) ++# ++# Specify an alternate location on Pacemaker Remote nodes for storing newer ++# RNG schemas and XSL transforms fetched from the cluster. ++# ++# Default: PCMK_remote_schema_directory="@PCMK__REMOTE_SCHEMA_DIR@" ++ + # G_SLICE (Advanced Use Only) + # + # Affect the behavior of glib's memory allocator. Setting to "always-malloc" +diff --git a/include/crm/common/options_internal.h b/include/crm/common/options_internal.h +index 5c561fd1f..a9316ca33 100644 +--- a/include/crm/common/options_internal.h ++++ b/include/crm/common/options_internal.h +@@ -95,6 +95,7 @@ bool pcmk__valid_sbd_timeout(const char *value); + #define PCMK__ENV_PANIC_ACTION "panic_action" + #define PCMK__ENV_PHYSICAL_HOST "physical_host" + #define PCMK__ENV_REMOTE_ADDRESS "remote_address" ++#define PCMK__ENV_REMOTE_SCHEMA_DIR "remote_schema_directory" + #define PCMK__ENV_REMOTE_PID1 "remote_pid1" + #define PCMK__ENV_REMOTE_PORT "remote_port" + #define PCMK__ENV_RESPAWNED "respawned" +-- +2.41.0 + +From 16d46de389f33a8b29cbd74ee2c9077f28029446 Mon Sep 17 00:00:00 2001 +From: Chris Lumens +Date: Wed, 6 Dec 2023 12:38:12 -0500 +Subject: [PATCH 05/15] Feature: libcrmcommon: Add pcmk__remote_schema_dir. + +This function checks both the environment (thus, +/etc/sysconfig/pacemaker as well) and ./configure options for the +location where any additional schema files should be stored. +--- + include/crm/common/xml_internal.h | 1 + + lib/common/schemas.c | 17 +++++++++++++++++ + 2 files changed, 18 insertions(+) + +diff --git a/include/crm/common/xml_internal.h b/include/crm/common/xml_internal.h +index f319856c8..cb27ec6b2 100644 +--- a/include/crm/common/xml_internal.h ++++ b/include/crm/common/xml_internal.h +@@ -447,5 +447,6 @@ gboolean pcmk__validate_xml(xmlNode *xml_blob, const char *validation, + void *error_handler_context); + + void pcmk__log_known_schemas(void); ++const char *pcmk__remote_schema_dir(void); + + #endif // PCMK__XML_INTERNAL__H +diff --git a/lib/common/schemas.c b/lib/common/schemas.c +index 1c60738ec..c03d80036 100644 +--- a/lib/common/schemas.c ++++ b/lib/common/schemas.c +@@ -1351,6 +1351,23 @@ pcmk__schema_files_later_than(const char *name) + return lst; + } + ++/*! ++ * \internal ++ * \brief Return the directory containing any extra schema files that a ++ * Pacemaker Remote node fetched from the cluster ++ */ ++const char * ++pcmk__remote_schema_dir(void) ++{ ++ const char *dir = pcmk__env_option(PCMK__ENV_REMOTE_SCHEMA_DIR); ++ ++ if (pcmk__str_empty(dir)) { ++ return PCMK__REMOTE_SCHEMA_DIR; ++ } ++ ++ return dir; ++} ++ + void + pcmk__log_known_schemas(void) + { +-- +2.41.0 + +From d2ed95651ea7c44153a34ba009494c70928319d7 Mon Sep 17 00:00:00 2001 +From: Chris Lumens +Date: Tue, 19 Sep 2023 11:40:59 -0400 +Subject: [PATCH 06/15] Refactor: libcrmcommon: Look in more dirs in + pcmk__xml_artefact_path. + +If the directory returned by pcmk__xml_artefact_root doesn't contain the +file we were looking for, fall back on PCMK__REMOTE_SCHEMA_DIR. If that +still doesn't contain the file, return NULL. +--- + lib/common/xml.c | 32 +++++++++++++++++++++++++------- + 1 file changed, 25 insertions(+), 7 deletions(-) + +diff --git a/lib/common/xml.c b/lib/common/xml.c +index 142b501df..a1b1291f9 100644 +--- a/lib/common/xml.c ++++ b/lib/common/xml.c +@@ -10,6 +10,7 @@ + #include + + #include ++#include + #include + #include + #include +@@ -2610,33 +2611,50 @@ pcmk__xml_artefact_root(enum pcmk__xml_artefact_ns ns) + return ret; + } + +-char * +-pcmk__xml_artefact_path(enum pcmk__xml_artefact_ns ns, const char *filespec) ++static char * ++find_artefact(enum pcmk__xml_artefact_ns ns, const char *path, const char *filespec) + { +- char *base = pcmk__xml_artefact_root(ns), *ret = NULL; ++ char *ret = NULL; + + switch (ns) { + case pcmk__xml_artefact_ns_legacy_rng: + case pcmk__xml_artefact_ns_base_rng: + if (pcmk__ends_with(filespec, ".rng")) { +- ret = crm_strdup_printf("%s/%s", base, filespec); ++ ret = crm_strdup_printf("%s/%s", path, filespec); + } else { +- ret = crm_strdup_printf("%s/%s.rng", base, filespec); ++ ret = crm_strdup_printf("%s/%s.rng", path, filespec); + } + break; + case pcmk__xml_artefact_ns_legacy_xslt: + case pcmk__xml_artefact_ns_base_xslt: + if (pcmk__ends_with(filespec, ".xsl")) { +- ret = crm_strdup_printf("%s/%s", base, filespec); ++ ret = crm_strdup_printf("%s/%s", path, filespec); + } else { +- ret = crm_strdup_printf("%s/%s.xsl", base, filespec); ++ ret = crm_strdup_printf("%s/%s.xsl", path, filespec); + } + break; + default: + crm_err("XML artefact family specified as %u not recognized", ns); + } ++ ++ return ret; ++} ++ ++char * ++pcmk__xml_artefact_path(enum pcmk__xml_artefact_ns ns, const char *filespec) ++{ ++ struct stat sb; ++ char *base = pcmk__xml_artefact_root(ns); ++ char *ret = NULL; ++ ++ ret = find_artefact(ns, base, filespec); + free(base); + ++ if (stat(ret, &sb) != 0 || !S_ISREG(sb.st_mode)) { ++ const char *remote_schema_dir = pcmk__remote_schema_dir(); ++ ret = find_artefact(ns, remote_schema_dir, filespec); ++ } ++ + return ret; + } + +-- +2.41.0 + +From 9f4da4102a40f7dbfe0d78c7735c2801ba852f4b Mon Sep 17 00:00:00 2001 +From: Chris Lumens +Date: Mon, 11 Sep 2023 09:17:18 -0400 +Subject: [PATCH 07/15] Feature: libcrmcommon: Add XML attrs needed for schema + file transfer. + +Also, move PCMK__XA_CONN_HOST to be alphabetized. +--- + include/crm_internal.h | 6 +++++- + 1 file changed, 5 insertions(+), 1 deletion(-) + +diff --git a/include/crm_internal.h b/include/crm_internal.h +index 71a0f7efa..3bc8d096a 100644 +--- a/include/crm_internal.h ++++ b/include/crm_internal.h +@@ -81,17 +81,21 @@ + #define PCMK__XA_CONFIG_ERRORS "config-errors" + #define PCMK__XA_CONFIG_WARNINGS "config-warnings" + #define PCMK__XA_CONFIRM "confirm" ++#define PCMK__XA_CONN_HOST "connection_host" + #define PCMK__XA_CRMD "crmd" + #define PCMK__XA_EXPECTED "expected" ++#define PCMK__XA_FILE "file" + #define PCMK__XA_GRAPH_ERRORS "graph-errors" + #define PCMK__XA_GRAPH_WARNINGS "graph-warnings" + #define PCMK__XA_IN_CCM "in_ccm" + #define PCMK__XA_JOIN "join" + #define PCMK__XA_MODE "mode" + #define PCMK__XA_NODE_START_STATE "node_start_state" ++#define PCMK__XA_PATH "path" ++#define PCMK__XA_SCHEMA "schema" ++#define PCMK__XA_SCHEMAS "schemas" + #define PCMK__XA_TASK "task" + #define PCMK__XA_UPTIME "uptime" +-#define PCMK__XA_CONN_HOST "connection_host" + + + /* +-- +2.41.0 + +From 3384643bd4572ad03e183b16d5cc84fe69599380 Mon Sep 17 00:00:00 2001 +From: Chris Lumens +Date: Mon, 11 Sep 2023 10:46:26 -0400 +Subject: [PATCH 08/15] Feature: libcrmcommon: Add pcmk__build_schema_xml_node. + +This function adds a given RNG or XSLT file and all of the files they +refer to as children of a given XML node. + +Also, add unit tests for the new function. +--- + include/crm/common/internal.h | 3 + + lib/common/schemas.c | 144 +++++++++++++++++ + lib/common/tests/schemas/Makefile.am | 3 +- + .../pcmk__build_schema_xml_node_test.c | 149 ++++++++++++++++++ + 4 files changed, 298 insertions(+), 1 deletion(-) + create mode 100644 lib/common/tests/schemas/pcmk__build_schema_xml_node_test.c + +diff --git a/include/crm/common/internal.h b/include/crm/common/internal.h +index a3cd455dc..318003efe 100644 +--- a/include/crm/common/internal.h ++++ b/include/crm/common/internal.h +@@ -135,6 +135,9 @@ void crm_schema_init(void); + void crm_schema_cleanup(void); + + GList *pcmk__schema_files_later_than(const char *name); ++void pcmk__build_schema_xml_node(xmlNode *parent, const char *name, ++ GList **already_included); ++ + + /* internal functions related to process IDs (from pid.c) */ + +diff --git a/lib/common/schemas.c b/lib/common/schemas.c +index c03d80036..1bcdff031 100644 +--- a/lib/common/schemas.c ++++ b/lib/common/schemas.c +@@ -1351,6 +1351,150 @@ pcmk__schema_files_later_than(const char *name) + return lst; + } + ++static void ++append_href(xmlNode *xml, void *user_data) ++{ ++ GList **list = user_data; ++ const char *href = crm_element_value(xml, "href"); ++ char *s = NULL; ++ ++ if (href == NULL) { ++ return; ++ } ++ ++ s = strdup(href); ++ CRM_ASSERT(s != NULL); ++ *list = g_list_prepend(*list, s); ++} ++ ++static void ++external_refs_in_schema(GList **list, const char *contents) ++{ ++ /* local-name()= is needed to ignore the xmlns= setting at the top of ++ * the XML file. Otherwise, the xpath query will always return nothing. ++ */ ++ const char *search = "//*[local-name()='externalRef'] | //*[local-name()='include']"; ++ xmlNode *xml = string2xml(contents); ++ ++ crm_foreach_xpath_result(xml, search, append_href, list); ++ free_xml(xml); ++} ++ ++static int ++read_file_contents(const char *file, char **contents) ++{ ++ int rc = pcmk_rc_ok; ++ char *path = NULL; ++ ++ if (pcmk__ends_with(file, ".rng")) { ++ path = pcmk__xml_artefact_path(pcmk__xml_artefact_ns_legacy_rng, file); ++ } else { ++ path = pcmk__xml_artefact_path(pcmk__xml_artefact_ns_legacy_xslt, file); ++ } ++ ++ rc = pcmk__file_contents(path, contents); ++ ++ free(path); ++ return rc; ++} ++ ++static void ++add_schema_file_to_xml(xmlNode *parent, const char *file, GList **already_included) ++{ ++ char *contents = NULL; ++ char *path = NULL; ++ xmlNode *file_node = NULL; ++ GList *includes = NULL; ++ int rc = pcmk_rc_ok; ++ ++ /* If we already included this file, don't do so again. */ ++ if (g_list_find_custom(*already_included, file, (GCompareFunc) strcmp) != NULL) { ++ return; ++ } ++ ++ /* Ensure whatever file we were given has a suffix we know about. If not, ++ * just assume it's an RNG file. ++ */ ++ if (!pcmk__ends_with(file, ".rng") && !pcmk__ends_with(file, ".xsl")) { ++ path = crm_strdup_printf("%s.rng", file); ++ } else { ++ path = strdup(file); ++ CRM_ASSERT(path != NULL); ++ } ++ ++ rc = read_file_contents(path, &contents); ++ if (rc != pcmk_rc_ok || contents == NULL) { ++ crm_warn("Could not read schema file %s: %s", file, pcmk_rc_str(rc)); ++ free(path); ++ return; ++ } ++ ++ /* Create a new node with the contents of the file ++ * as a CDATA block underneath it. ++ */ ++ file_node = create_xml_node(parent, PCMK__XA_FILE); ++ if (file_node == NULL) { ++ free(contents); ++ free(path); ++ return; ++ } ++ ++ crm_xml_add(file_node, PCMK__XA_PATH, path); ++ *already_included = g_list_prepend(*already_included, path); ++ ++ xmlAddChild(file_node, xmlNewCDataBlock(parent->doc, (pcmkXmlStr) contents, ++ strlen(contents))); ++ ++ /* Scan the file for any or nodes and build up ++ * a list of the files they reference. ++ */ ++ external_refs_in_schema(&includes, contents); ++ ++ /* For each referenced file, recurse to add it (and potentially anything it ++ * references, ...) to the XML. ++ */ ++ for (GList *iter = includes; iter != NULL; iter = iter->next) { ++ add_schema_file_to_xml(parent, iter->data, already_included); ++ } ++ ++ free(contents); ++ g_list_free_full(includes, free); ++} ++ ++/*! ++ * \internal ++ * \brief Add an XML schema file and all the files it references as children ++ * of a given XML node ++ * ++ * \param[in,out] parent The parent XML node ++ * \param[in] name The schema version to compare against ++ * (for example, "pacemaker-3.1" or "pacemaker-3.1.rng") ++ * \param[in,out] already_included A list of names that have already been added ++ * to the parent node. ++ * ++ * \note The caller is responsible for freeing both the returned list and ++ * the elements of the list ++ */ ++void ++pcmk__build_schema_xml_node(xmlNode *parent, const char *name, GList **already_included) ++{ ++ /* First, create an unattached node to add all the schema files to as children. */ ++ xmlNode *schema_node = create_xml_node(NULL, PCMK__XA_SCHEMA); ++ ++ crm_xml_add(schema_node, XML_ATTR_VERSION, name); ++ add_schema_file_to_xml(schema_node, name, already_included); ++ ++ /* Then, if we actually added any children, attach the node to parent. If ++ * we did not add any children (for instance, name was invalid), this prevents ++ * us from returning a document with additional empty children. ++ */ ++ if (schema_node->children != NULL) { ++ xmlAddChild(parent, schema_node); ++ } else { ++ free_xml(schema_node); ++ } ++} ++ + /*! + * \internal + * \brief Return the directory containing any extra schema files that a +diff --git a/lib/common/tests/schemas/Makefile.am b/lib/common/tests/schemas/Makefile.am +index b5c5e7f3c..8854eb264 100644 +--- a/lib/common/tests/schemas/Makefile.am ++++ b/lib/common/tests/schemas/Makefile.am +@@ -13,7 +13,8 @@ include $(top_srcdir)/mk/unittest.mk + CFLAGS += -DPCMK__TEST_SCHEMA_DIR='"$(abs_builddir)/schemas"' + + # Add "_test" to the end of all test program names to simplify .gitignore. +-check_PROGRAMS = pcmk__schema_files_later_than_test \ ++check_PROGRAMS = pcmk__build_schema_xml_node_test \ ++ pcmk__schema_files_later_than_test \ + pcmk__xml_find_x_0_schema_index_test + + TESTS = $(check_PROGRAMS) +diff --git a/lib/common/tests/schemas/pcmk__build_schema_xml_node_test.c b/lib/common/tests/schemas/pcmk__build_schema_xml_node_test.c +new file mode 100644 +index 000000000..1f5cb6ce1 +--- /dev/null ++++ b/lib/common/tests/schemas/pcmk__build_schema_xml_node_test.c +@@ -0,0 +1,149 @@ ++/* ++ * Copyright 2023 the Pacemaker project contributors ++ * ++ * The version control history for this file may have further details. ++ * ++ * This source code is licensed under the GNU General Public License version 2 ++ * or later (GPLv2+) WITHOUT ANY WARRANTY. ++ */ ++ ++#include ++ ++#include ++#include ++#include ++ ++#include ++ ++const char *rngs1[] = { "pacemaker-3.0.rng", "status-1.0.rng", "alerts-2.10.rng", ++ "nvset-2.9.rng", "score.rng", "rule-2.9.rng", ++ "tags-1.3.rng", "acls-2.0.rng", "fencing-2.4.rng", ++ "constraints-3.0.rng", "resources-3.0.rng", "nvset-3.0.rng", ++ "nodes-3.0.rng", "options-3.0.rng", NULL }; ++ ++const char *rngs2[] = { "pacemaker-2.0.rng", "status-1.0.rng", "tags-1.3.rng", ++ "acls-2.0.rng", "fencing-1.2.rng", "constraints-1.2.rng", ++ "rule.rng", "score.rng", "resources-1.3.rng", ++ "nvset-1.3.rng", "nodes-1.3.rng", "options-1.0.rng", ++ "nvset.rng", "cib-1.2.rng", NULL }; ++ ++const char *rngs3[] = { "pacemaker-2.1.rng", "constraints-2.1.rng", NULL }; ++ ++static int ++setup(void **state) { ++ setenv("PCMK_schema_directory", PCMK__TEST_SCHEMA_DIR, 1); ++ crm_schema_init(); ++ return 0; ++} ++ ++static int ++teardown(void **state) { ++ crm_schema_cleanup(); ++ unsetenv("PCMK_schema_directory"); ++ return 0; ++} ++ ++static void ++invalid_name(void **state) ++{ ++ GList *already_included = NULL; ++ xmlNode *parent = create_xml_node(NULL, PCMK__XA_SCHEMAS); ++ ++ pcmk__build_schema_xml_node(parent, "pacemaker-9.0", &already_included); ++ assert_null(parent->children); ++ assert_null(already_included); ++ free_xml(parent); ++} ++ ++static void ++single_schema(void **state) ++{ ++ GList *already_included = NULL; ++ xmlNode *parent = create_xml_node(NULL, PCMK__XA_SCHEMAS); ++ xmlNode *schema_node = NULL; ++ xmlNode *file_node = NULL; ++ int i = 0; ++ ++ pcmk__build_schema_xml_node(parent, "pacemaker-3.0", &already_included); ++ ++ assert_non_null(already_included); ++ assert_non_null(parent->children); ++ ++ /* Test that the result looks like this: ++ * ++ * ++ * ++ * CDATA ++ * CDATA ++ * ... ++ * ++ * ++ */ ++ schema_node = pcmk__xml_first_child(parent); ++ assert_string_equal("pacemaker-3.0", crm_element_value(schema_node, XML_ATTR_VERSION)); ++ ++ file_node = pcmk__xml_first_child(schema_node); ++ while (file_node != NULL && rngs1[i] != NULL) { ++ assert_string_equal(rngs1[i], crm_element_value(file_node, PCMK__XA_PATH)); ++ assert_int_equal(pcmk__xml_first_child(file_node)->type, XML_CDATA_SECTION_NODE); ++ ++ file_node = pcmk__xml_next(file_node); ++ i++; ++ } ++ ++ g_list_free_full(already_included, free); ++ free_xml(parent); ++} ++ ++static void ++multiple_schemas(void **state) ++{ ++ GList *already_included = NULL; ++ xmlNode *parent = create_xml_node(NULL, PCMK__XA_SCHEMAS); ++ xmlNode *schema_node = NULL; ++ xmlNode *file_node = NULL; ++ int i = 0; ++ ++ pcmk__build_schema_xml_node(parent, "pacemaker-2.0", &already_included); ++ pcmk__build_schema_xml_node(parent, "pacemaker-2.1", &already_included); ++ ++ assert_non_null(already_included); ++ assert_non_null(parent->children); ++ ++ /* Like single_schema, but make sure files aren't included multiple times ++ * when the function is called repeatedly. ++ */ ++ schema_node = pcmk__xml_first_child(parent); ++ assert_string_equal("pacemaker-2.0", crm_element_value(schema_node, XML_ATTR_VERSION)); ++ ++ file_node = pcmk__xml_first_child(schema_node); ++ while (file_node != NULL && rngs2[i] != NULL) { ++ assert_string_equal(rngs2[i], crm_element_value(file_node, PCMK__XA_PATH)); ++ assert_int_equal(pcmk__xml_first_child(file_node)->type, XML_CDATA_SECTION_NODE); ++ ++ file_node = pcmk__xml_next(file_node); ++ i++; ++ } ++ ++ schema_node = pcmk__xml_next(schema_node); ++ assert_string_equal("pacemaker-2.1", crm_element_value(schema_node, XML_ATTR_VERSION)); ++ ++ file_node = pcmk__xml_first_child(schema_node); ++ i = 0; ++ ++ while (file_node != NULL && rngs3[i] != NULL) { ++ assert_string_equal(rngs3[i], crm_element_value(file_node, PCMK__XA_PATH)); ++ assert_int_equal(pcmk__xml_first_child(file_node)->type, XML_CDATA_SECTION_NODE); ++ ++ file_node = pcmk__xml_next(file_node); ++ i++; ++ } ++ ++ g_list_free_full(already_included, free); ++ free_xml(parent); ++} ++ ++PCMK__UNIT_TEST(setup, teardown, ++ cmocka_unit_test(invalid_name), ++ cmocka_unit_test(single_schema), ++ cmocka_unit_test(multiple_schemas)) +-- +2.41.0 + +From 036eb9f59326962ed2d1f2f4af88b20755a046d5 Mon Sep 17 00:00:00 2001 +From: Chris Lumens +Date: Thu, 14 Sep 2023 10:02:16 -0400 +Subject: [PATCH 09/15] Feature: daemons: Add a new fetch_schemas CIB command. + +This command wraps pcmk__schema_files_later_than and +pcmk__build_schema_xml_node to produce a new CIB command that takes in a +minimum schema version and returns a big XML message containing all the +schema files after that version. +--- + daemons/based/based_messages.c | 43 +++++++++++++++++++++++++++++++++ + daemons/based/based_operation.c | 1 + + daemons/based/pacemaker-based.h | 4 +++ + include/crm/cib/cib_types.h | 3 +++ + include/crm/cib/internal.h | 2 ++ + lib/cib/cib_client.c | 15 ++++++++++++ + lib/cib/cib_ops.c | 3 +++ + 7 files changed, 71 insertions(+) + +diff --git a/daemons/based/based_messages.c b/daemons/based/based_messages.c +index 35d639a89..a87d9ac2e 100644 +--- a/daemons/based/based_messages.c ++++ b/daemons/based/based_messages.c +@@ -478,3 +478,46 @@ cib_process_commit_transaction(const char *op, int options, const char *section, + } + return pcmk_rc2legacy(rc); + } ++ ++int ++cib_process_schemas(const char *op, int options, const char *section, xmlNode *req, ++ xmlNode *input, xmlNode *existing_cib, xmlNode **result_cib, ++ xmlNode **answer) ++{ ++ xmlNode *data = NULL; ++ const char *after_ver = NULL; ++ GList *schemas = NULL; ++ GList *already_included = NULL; ++ ++ *answer = create_xml_node(NULL, PCMK__XA_SCHEMAS); ++ ++ data = get_message_xml(req, F_CIB_CALLDATA); ++ if (data == NULL) { ++ crm_warn("No data specified in request"); ++ return -EPROTO; ++ } ++ ++ after_ver = crm_element_value(data, XML_ATTR_VERSION); ++ if (after_ver == NULL) { ++ crm_warn("No version specified in request"); ++ return -EPROTO; ++ } ++ ++ /* The client requested all schemas after the latest one we know about, which ++ * means the client is fully up-to-date. Return a properly formatted reply ++ * with no schemas. ++ */ ++ if (pcmk__str_eq(after_ver, xml_latest_schema(), pcmk__str_none)) { ++ return pcmk_ok; ++ } ++ ++ schemas = pcmk__schema_files_later_than(after_ver); ++ ++ for (GList *iter = schemas; iter != NULL; iter = iter->next) { ++ pcmk__build_schema_xml_node(*answer, iter->data, &already_included); ++ } ++ ++ g_list_free_full(schemas, free); ++ g_list_free_full(already_included, free); ++ return pcmk_ok; ++} +diff --git a/daemons/based/based_operation.c b/daemons/based/based_operation.c +index 736d425e3..8dd07af93 100644 +--- a/daemons/based/based_operation.c ++++ b/daemons/based/based_operation.c +@@ -35,6 +35,7 @@ static const cib__op_fn_t cib_op_functions[] = { + [cib__op_sync_all] = cib_process_sync, + [cib__op_sync_one] = cib_process_sync_one, + [cib__op_upgrade] = cib_process_upgrade_server, ++ [cib__op_schemas] = cib_process_schemas, + }; + + /*! +diff --git a/daemons/based/pacemaker-based.h b/daemons/based/pacemaker-based.h +index 33c7642c5..de24779ac 100644 +--- a/daemons/based/pacemaker-based.h ++++ b/daemons/based/pacemaker-based.h +@@ -122,6 +122,10 @@ int cib_process_commit_transaction(const char *op, int options, + const char *section, xmlNode *req, + xmlNode *input, xmlNode *existing_cib, + xmlNode **result_cib, xmlNode **answer); ++int cib_process_schemas(const char *op, int options, const char *section, ++ xmlNode *req, xmlNode *input, xmlNode *existing_cib, ++ xmlNode **result_cib, xmlNode **answer); ++ + void send_sync_request(const char *host); + int sync_our_cib(xmlNode *request, gboolean all); + +diff --git a/include/crm/cib/cib_types.h b/include/crm/cib/cib_types.h +index a803311c2..bebe770ed 100644 +--- a/include/crm/cib/cib_types.h ++++ b/include/crm/cib/cib_types.h +@@ -324,6 +324,9 @@ typedef struct cib_api_operations_s { + * processing requests + */ + void (*set_user)(cib_t *cib, const char *user); ++ ++ int (*fetch_schemas)(cib_t *cib, xmlNode **output_data, const char *after_ver, ++ int call_options); + } cib_api_operations_t; + + struct cib_s { +diff --git a/include/crm/cib/internal.h b/include/crm/cib/internal.h +index 20059ec7e..9d54d52b7 100644 +--- a/include/crm/cib/internal.h ++++ b/include/crm/cib/internal.h +@@ -32,6 +32,7 @@ + #define PCMK__CIB_REQUEST_NOOP "noop" + #define PCMK__CIB_REQUEST_SHUTDOWN "cib_shutdown_req" + #define PCMK__CIB_REQUEST_COMMIT_TRANSACT "cib_commit_transact" ++#define PCMK__CIB_REQUEST_SCHEMAS "cib_schemas" + + # define F_CIB_CLIENTID "cib_clientid" + # define F_CIB_CALLOPTS "cib_callopt" +@@ -110,6 +111,7 @@ enum cib__op_type { + cib__op_sync_all, + cib__op_sync_one, + cib__op_upgrade, ++ cib__op_schemas, + }; + + gboolean cib_diff_version_details(xmlNode * diff, int *admin_epoch, int *epoch, int *updates, +diff --git a/lib/cib/cib_client.c b/lib/cib/cib_client.c +index 32e1f83c5..a2fcabbca 100644 +--- a/lib/cib/cib_client.c ++++ b/lib/cib/cib_client.c +@@ -451,6 +451,19 @@ cib_client_end_transaction(cib_t *cib, bool commit, int call_options) + return rc; + } + ++static int ++cib_client_fetch_schemas(cib_t *cib, xmlNode **output_data, const char *after_ver, ++ int call_options) ++{ ++ xmlNode *data = create_xml_node(NULL, PCMK__XA_SCHEMA); ++ ++ crm_xml_add(data, XML_ATTR_VERSION, after_ver); ++ ++ return cib_internal_op(cib, PCMK__CIB_REQUEST_SCHEMAS, NULL, NULL, data, ++ output_data, call_options, NULL); ++ ++} ++ + static void + cib_client_set_user(cib_t *cib, const char *user) + { +@@ -736,6 +749,8 @@ cib_new_variant(void) + + new_cib->cmds->set_user = cib_client_set_user; + ++ new_cib->cmds->fetch_schemas = cib_client_fetch_schemas; ++ + return new_cib; + } + +diff --git a/lib/cib/cib_ops.c b/lib/cib/cib_ops.c +index c324304b9..2165d8af3 100644 +--- a/lib/cib/cib_ops.c ++++ b/lib/cib/cib_ops.c +@@ -127,6 +127,9 @@ static const cib__operation_t cib_ops[] = { + |cib__op_attr_writes_through + |cib__op_attr_transaction + }, ++ { ++ PCMK__CIB_REQUEST_SCHEMAS, cib__op_schemas, cib__op_attr_local ++ } + }; + + /*! +-- +2.41.0 + +From e8076b4a387ee758508f0683739b3e880f79db47 Mon Sep 17 00:00:00 2001 +From: Chris Lumens +Date: Thu, 14 Sep 2023 15:48:31 -0400 +Subject: [PATCH 10/15] Refactor: libcrmcommon: Add + pcmk__load_schemas_from_dir. + +This just moves the bulk of crm_schema_init out into its own function, +allowing us to call it multiple times as other schema directories +appear. + +There is no similar function for unloading schemas from a directory. If +you want to do that, you'll have to use crm_schema_cleanup to unload +everything and start over. + +This function has not been tested for the possibility that the same +schema files exist in multiple directories. It is assumed that it won't +be used like that. +--- + include/crm/common/internal.h | 1 + + lib/common/schemas.c | 83 +++++++++++++++++++---------------- + 2 files changed, 45 insertions(+), 39 deletions(-) + +diff --git a/include/crm/common/internal.h b/include/crm/common/internal.h +index 318003efe..542d0a67c 100644 +--- a/include/crm/common/internal.h ++++ b/include/crm/common/internal.h +@@ -134,6 +134,7 @@ bool pcmk__procfs_has_pids(void); + void crm_schema_init(void); + void crm_schema_cleanup(void); + ++void pcmk__load_schemas_from_dir(const char *dir); + GList *pcmk__schema_files_later_than(const char *name); + void pcmk__build_schema_xml_node(xmlNode *parent, const char *name, + GList **already_included); +diff --git a/lib/common/schemas.c b/lib/common/schemas.c +index 1bcdff031..a0c844131 100644 +--- a/lib/common/schemas.c ++++ b/lib/common/schemas.c +@@ -406,6 +406,49 @@ wrap_libxslt(bool finalize) + } + } + ++void ++pcmk__load_schemas_from_dir(const char *dir) ++{ ++ int lpc, max; ++ struct dirent **namelist = NULL; ++ ++ max = scandir(dir, &namelist, schema_filter, schema_cmp_directory); ++ if (max < 0) { ++ crm_warn("Could not load schemas from %s: %s", dir, strerror(errno)); ++ return; ++ } ++ ++ for (lpc = 0; lpc < max; lpc++) { ++ bool transform_expected = false; ++ pcmk__schema_version_t version = SCHEMA_ZERO; ++ ++ if (!version_from_filename(namelist[lpc]->d_name, &version)) { ++ // Shouldn't be possible, but makes static analysis happy ++ crm_warn("Skipping schema '%s': could not parse version", ++ namelist[lpc]->d_name); ++ continue; ++ } ++ if ((lpc + 1) < max) { ++ pcmk__schema_version_t next_version = SCHEMA_ZERO; ++ ++ if (version_from_filename(namelist[lpc+1]->d_name, &next_version) ++ && (version.v[0] < next_version.v[0])) { ++ transform_expected = true; ++ } ++ } ++ ++ if (add_schema_by_version(&version, transform_expected) != pcmk_rc_ok) { ++ break; ++ } ++ } ++ ++ for (lpc = 0; lpc < max; lpc++) { ++ free(namelist[lpc]); ++ } ++ ++ free(namelist); ++} ++ + /*! + * \internal + * \brief Load pacemaker schemas into cache +@@ -416,50 +459,12 @@ wrap_libxslt(bool finalize) + void + crm_schema_init(void) + { +- int lpc, max; + char *base = pcmk__xml_artefact_root(pcmk__xml_artefact_ns_legacy_rng); +- struct dirent **namelist = NULL; + const pcmk__schema_version_t zero = SCHEMA_ZERO; + + wrap_libxslt(false); + +- max = scandir(base, &namelist, schema_filter, schema_cmp_directory); +- if (max < 0) { +- crm_notice("scandir(%s) failed: %s (%d)", base, strerror(errno), errno); +- free(base); +- +- } else { +- free(base); +- for (lpc = 0; lpc < max; lpc++) { +- bool transform_expected = FALSE; +- pcmk__schema_version_t version = SCHEMA_ZERO; +- +- if (!version_from_filename(namelist[lpc]->d_name, &version)) { +- // Shouldn't be possible, but makes static analysis happy +- crm_err("Skipping schema '%s': could not parse version", +- namelist[lpc]->d_name); +- continue; +- } +- if ((lpc + 1) < max) { +- pcmk__schema_version_t next_version = SCHEMA_ZERO; +- +- if (version_from_filename(namelist[lpc+1]->d_name, &next_version) +- && (version.v[0] < next_version.v[0])) { +- transform_expected = TRUE; +- } +- } +- +- if (add_schema_by_version(&version, transform_expected) +- == ENOENT) { +- break; +- } +- } +- +- for (lpc = 0; lpc < max; lpc++) { +- free(namelist[lpc]); +- } +- free(namelist); +- } ++ pcmk__load_schemas_from_dir(base); + + // @COMPAT: Deprecated since 2.1.5 + add_schema(pcmk__schema_validator_rng, &zero, "pacemaker-next", +-- +2.41.0 + +From 5b40f0227b33edd6be65515dcc1af4eb656b78e7 Mon Sep 17 00:00:00 2001 +From: Chris Lumens +Date: Thu, 14 Sep 2023 11:24:36 -0400 +Subject: [PATCH 11/15] Feature: daemons: Download newer schema files to a + remote executor. + +If the remote executor supports an older version of the CIB schema than +the rest of the cluster, various operations could fail due to the schema +not validating on the remote node. + +Instead, ask the CIB manager for the updated schema files, store them in +a separate directory on the remote node, and add them to the list of +known schema files. These can then be used just like the packaged +schema files. + +Note that while this is a fairly large patch, it's really just a lot of +boring IO and file management code, which requires a lot of error +handling. + +Fixes T182 +--- + daemons/execd/Makefile.am | 4 +- + daemons/execd/execd_commands.c | 6 + + daemons/execd/pacemaker-execd.h | 1 + + daemons/execd/remoted_schemas.c | 282 ++++++++++++++++++++++++++++++++ + 4 files changed, 292 insertions(+), 1 deletion(-) + create mode 100644 daemons/execd/remoted_schemas.c + +diff --git a/daemons/execd/Makefile.am b/daemons/execd/Makefile.am +index ab8544f9d..ce0e16126 100644 +--- a/daemons/execd/Makefile.am ++++ b/daemons/execd/Makefile.am +@@ -44,12 +44,14 @@ pacemaker_remoted_LDFLAGS = $(LDFLAGS_HARDENED_EXE) + + pacemaker_remoted_LDADD = $(top_builddir)/lib/fencing/libstonithd.la + pacemaker_remoted_LDADD += $(top_builddir)/lib/services/libcrmservice.la ++pacemaker_remoted_LDADD += $(top_builddir)/lib/cib/libcib.la + pacemaker_remoted_LDADD += $(top_builddir)/lib/lrmd/liblrmd.la + pacemaker_remoted_LDADD += $(top_builddir)/lib/common/libcrmcommon.la + pacemaker_remoted_SOURCES = $(pacemaker_execd_SOURCES) \ + remoted_tls.c \ + remoted_pidone.c \ +- remoted_proxy.c ++ remoted_proxy.c \ ++ remoted_schemas.c + endif + + cts_exec_helper_LDADD = $(top_builddir)/lib/pengine/libpe_status.la +diff --git a/daemons/execd/execd_commands.c b/daemons/execd/execd_commands.c +index cf4503a25..1601efb0b 100644 +--- a/daemons/execd/execd_commands.c ++++ b/daemons/execd/execd_commands.c +@@ -1493,9 +1493,15 @@ process_lrmd_signon(pcmk__client_t *client, xmlNode *request, int call_id, + if ((client->remote != NULL) + && pcmk_is_set(client->flags, + pcmk__client_tls_handshake_complete)) { ++ const char *op = crm_element_value(request, F_LRMD_OPERATION); + + // This is a remote connection from a cluster node's controller + ipc_proxy_add_provider(client); ++ ++ /* If this was a register operation, also ask for new schema files. */ ++ if (pcmk__str_eq(op, CRM_OP_REGISTER, pcmk__str_none)) { ++ remoted_request_cib_schema_files(); ++ } + } else { + rc = -EACCES; + } +diff --git a/daemons/execd/pacemaker-execd.h b/daemons/execd/pacemaker-execd.h +index 9c1d173f5..6fb8ef440 100644 +--- a/daemons/execd/pacemaker-execd.h ++++ b/daemons/execd/pacemaker-execd.h +@@ -101,6 +101,7 @@ void ipc_proxy_forward_client(pcmk__client_t *client, xmlNode *xml); + pcmk__client_t *ipc_proxy_get_provider(void); + int ipc_proxy_shutdown_req(pcmk__client_t *ipc_proxy); + void remoted_spawn_pidone(int argc, char **argv, char **envp); ++void remoted_request_cib_schema_files(void); + #endif + + int process_lrmd_alert_exec(pcmk__client_t *client, uint32_t id, +diff --git a/daemons/execd/remoted_schemas.c b/daemons/execd/remoted_schemas.c +new file mode 100644 +index 000000000..d501fa495 +--- /dev/null ++++ b/daemons/execd/remoted_schemas.c +@@ -0,0 +1,282 @@ ++/* ++ * Copyright 2023 the Pacemaker project contributors ++ * ++ * The version control history for this file may have further details. ++ * ++ * This source code is licensed under the GNU Lesser General Public License ++ * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY. ++ */ ++ ++#include ++ ++#include ++#include ++#include ++ ++#include ++#include ++#include ++#include ++#include ++#include ++ ++#include "pacemaker-execd.h" ++ ++static pid_t schema_fetch_pid = 0; ++ ++static int ++rm_files(const char *pathname, const struct stat *sbuf, int type, struct FTW *ftwb) ++{ ++ /* Don't delete PCMK__REMOTE_SCHEMA_DIR . */ ++ if (ftwb->level == 0) { ++ return 0; ++ } ++ ++ if (remove(pathname) != 0) { ++ int rc = errno; ++ crm_err("Could not remove %s: %s", pathname, pcmk_rc_str(rc)); ++ return -1; ++ } ++ ++ return 0; ++} ++ ++static void ++clean_up_extra_schema_files(void) ++{ ++ const char *remote_schema_dir = pcmk__remote_schema_dir(); ++ struct stat sb; ++ int rc; ++ ++ rc = stat(remote_schema_dir, &sb); ++ ++ if (rc == -1) { ++ if (errno == ENOENT) { ++ /* If the directory doesn't exist, try to make it first. */ ++ if (mkdir(remote_schema_dir, 0755) != 0) { ++ rc = errno; ++ crm_err("Could not create directory for schemas: %s", ++ pcmk_rc_str(rc)); ++ } ++ ++ } else { ++ rc = errno; ++ crm_err("Could not create directory for schemas: %s", ++ pcmk_rc_str(rc)); ++ } ++ ++ } else if (!S_ISDIR(sb.st_mode)) { ++ /* If something exists with the same name that's not a directory, that's ++ * an error. ++ */ ++ crm_err("%s already exists but is not a directory", remote_schema_dir); ++ ++ } else { ++ /* It's a directory - clear it out so we can download potentially new ++ * schema files. ++ */ ++ rc = nftw(remote_schema_dir, rm_files, 10, FTW_DEPTH|FTW_MOUNT|FTW_PHYS); ++ ++ if (rc != 0) { ++ crm_err("Could not remove %s: %s", remote_schema_dir, pcmk_rc_str(rc)); ++ } ++ } ++} ++ ++static void ++write_extra_schema_file(xmlNode *xml, void *user_data) ++{ ++ const char *remote_schema_dir = pcmk__remote_schema_dir(); ++ const char *file = NULL; ++ char *path = NULL; ++ int rc; ++ ++ file = crm_element_value(xml, PCMK__XA_PATH); ++ if (file == NULL) { ++ crm_warn("No destination path given in schema request"); ++ return; ++ } ++ ++ path = crm_strdup_printf("%s/%s", remote_schema_dir, file); ++ ++ /* The schema is a CDATA node, which is a child of the node. Traverse ++ * all children and look for the first CDATA child. There can't be more than ++ * one because we only have one file attribute on the parent. ++ */ ++ for (xmlNode *child = xml->children; child != NULL; child = child->next) { ++ FILE *stream = NULL; ++ ++ if (child->type != XML_CDATA_SECTION_NODE) { ++ continue; ++ } ++ ++ stream = fopen(path, "w+"); ++ if (stream == NULL) { ++ crm_warn("Could not write schema file %s: %s", path, strerror(errno)); ++ } else { ++ rc = fprintf(stream, "%s", child->content); ++ ++ if (rc < 0) { ++ crm_warn("Could not write schema file %s: %s", path, strerror(errno)); ++ } ++ ++ fclose(stream); ++ } ++ ++ break; ++ } ++ ++ free(path); ++} ++ ++static void ++get_schema_files(void) ++{ ++ int rc = pcmk_rc_ok; ++ cib_t *cib = NULL; ++ xmlNode *reply; ++ ++ cib = cib_new(); ++ if (cib == NULL) { ++ _exit(ENOTCONN); ++ } ++ ++ rc = cib->cmds->signon(cib, crm_system_name, cib_query); ++ if (rc != pcmk_ok) { ++ crm_err("Could not connect to the CIB manager: %s", pcmk_strerror(rc)); ++ _exit(pcmk_rc2exitc(rc)); ++ } ++ ++ rc = cib->cmds->fetch_schemas(cib, &reply, xml_latest_schema(), cib_sync_call); ++ if (rc != pcmk_ok) { ++ crm_err("Could not get schema files: %s", pcmk_strerror(rc)); ++ rc = pcmk_legacy2rc(rc); ++ ++ } else if (reply->children != NULL) { ++ /* The returned document looks something like this: ++ * ++ * ++ * ++ * ++ * ++ * ++ * ++ * ++ * ... ++ * ++ * ++ * ++ * ... ++ * ++ * ++ * ++ * ++ * All the and tags are really just there for organizing ++ * the XML a little better. What we really care about are the nodes, ++ * and specifically the path attributes and the CDATA children (not shown) ++ * of each. We can use an xpath query to reach down and get all the ++ * nodes at once. ++ * ++ * If we already have the latest schema version, or we asked for one later ++ * than what the cluster supports, we'll get back an empty node, ++ * so all this will continue to work. It just won't do anything. ++ */ ++ crm_foreach_xpath_result(reply, "//" PCMK__XA_FILE, write_extra_schema_file, NULL); ++ } ++ ++ cib__clean_up_connection(&cib); ++ _exit(pcmk_rc2exitc(rc)); ++} ++ ++/* Load any additional schema files when the child is finished fetching and ++ * saving them to disk. ++ */ ++static void ++get_schema_files_complete(mainloop_child_t *p, pid_t pid, int core, int signo, int exitcode) ++{ ++ const char *errmsg = "Could not load additional schema files"; ++ ++ if ((signo == 0) && (exitcode == 0)) { ++ const char *remote_schema_dir = pcmk__remote_schema_dir(); ++ ++ /* Don't just crm_schema_init here because that will load the base ++ * schemas again too. Instead just load the things we fetched. ++ */ ++ pcmk__load_schemas_from_dir(remote_schema_dir); ++ crm_info("Fetching extra schema files completed successfully"); ++ ++ } else { ++ if (signo == 0) { ++ crm_err("%s: process %d exited %d", errmsg, (int) pid, exitcode); ++ ++ } else { ++ crm_err("%s: process %d terminated with signal %d (%s)%s", ++ errmsg, (int) pid, signo, strsignal(signo), ++ (core? " and dumped core" : "")); ++ } ++ ++ /* Clean up any incomplete schema data we might have been downloading when ++ * the process timed out or crashed. We don't need to do any extra cleanup ++ * because we never loaded the extra schemas, and we don't need to call ++ * crm_schema_init because that was called in remoted_request_cib_schema_files ++ * before this function. ++ */ ++ clean_up_extra_schema_files(); ++ } ++} ++ ++void ++remoted_request_cib_schema_files(void) ++{ ++ pid_t pid; ++ int rc; ++ ++ /* If a previous schema-fetch process is still running when we're called ++ * again, it's hung. Attempt to kill it before cleaning up the extra ++ * directory. ++ */ ++ if (schema_fetch_pid != 0) { ++ if (mainloop_child_kill(schema_fetch_pid) == FALSE) { ++ crm_warn("Unable to kill pre-existing schema-fetch process"); ++ return; ++ } ++ ++ schema_fetch_pid = 0; ++ } ++ ++ /* Clean up any extra schema files we downloaded from a previous cluster ++ * connection. After the files are gone, we need to wipe them from ++ * known_schemas, but there's no opposite operation for add_schema(). ++ * ++ * Instead, unload all the schemas. This means we'll also forget about all ++ * the installed schemas as well, which means that xml_latest_schema() will ++ * fail. So we need to load the base schemas right now. ++ */ ++ clean_up_extra_schema_files(); ++ crm_schema_cleanup(); ++ crm_schema_init(); ++ ++ crm_info("Fetching extra schema files from cluster"); ++ pid = fork(); ++ ++ switch (pid) { ++ case -1: { ++ rc = errno; ++ crm_warn("Could not spawn process to get schema files: %s", pcmk_rc_str(rc)); ++ break; ++ } ++ ++ case 0: ++ /* child */ ++ get_schema_files(); ++ break; ++ ++ default: ++ /* parent */ ++ schema_fetch_pid = pid; ++ mainloop_child_add_with_flags(pid, 5 * 60 * 1000, "schema-fetch", NULL, ++ mainloop_leave_pid_group, ++ get_schema_files_complete); ++ break; ++ } ++} +-- +2.41.0 + +From b05fef32cf1f9063e01db5108f95386be329d778 Mon Sep 17 00:00:00 2001 +From: Chris Lumens +Date: Wed, 29 Nov 2023 10:04:57 -0500 +Subject: [PATCH 12/15] Feature: libcrmcommon: Load additional schema files in + crm_schema_init. + +If the /var/lib/pacemaker/schemas directory exists, load any extra +schemas from it when we init. This makes them available for command +line programs to use. +--- + lib/common/schemas.c | 2 ++ + 1 file changed, 2 insertions(+) + +diff --git a/lib/common/schemas.c b/lib/common/schemas.c +index a0c844131..68d79cfc7 100644 +--- a/lib/common/schemas.c ++++ b/lib/common/schemas.c +@@ -459,12 +459,14 @@ pcmk__load_schemas_from_dir(const char *dir) + void + crm_schema_init(void) + { ++ const char *remote_schema_dir = pcmk__remote_schema_dir(); + char *base = pcmk__xml_artefact_root(pcmk__xml_artefact_ns_legacy_rng); + const pcmk__schema_version_t zero = SCHEMA_ZERO; + + wrap_libxslt(false); + + pcmk__load_schemas_from_dir(base); ++ pcmk__load_schemas_from_dir(remote_schema_dir); + + // @COMPAT: Deprecated since 2.1.5 + add_schema(pcmk__schema_validator_rng, &zero, "pacemaker-next", +-- +2.41.0 + +From 7454a8400238301848cc6694f5413fdb19d5834e Mon Sep 17 00:00:00 2001 +From: Chris Lumens +Date: Fri, 29 Sep 2023 10:41:51 -0400 +Subject: [PATCH 13/15] Refactor: daemons: Remove redundant includes from + remoted_tls.c. + +--- + daemons/execd/remoted_tls.c | 2 -- + 1 file changed, 2 deletions(-) + +diff --git a/daemons/execd/remoted_tls.c b/daemons/execd/remoted_tls.c +index 23a2dcf45..978edfdd3 100644 +--- a/daemons/execd/remoted_tls.c ++++ b/daemons/execd/remoted_tls.c +@@ -12,8 +12,6 @@ + #include + #include + +-#include +-#include + #include + #include + #include +-- +2.41.0 + +From c6b5f73a013515d6acd818d348d011e38f3d8e0e Mon Sep 17 00:00:00 2001 +From: Chris Lumens +Date: Fri, 27 Oct 2023 15:23:41 -0400 +Subject: [PATCH 14/15] Refactor: daemons: Keep XML schemas sorted. + +Now that we can add schemas multiple times (including after next/none +have been added in the initial set), we've broken the assumption that +schemas are always sorted by scandir. + +This introduces a function that can be called wherever to keep them +sorted. It should be called pretty much whenever +pcmk__load_schemas_from_dir is called. +--- + daemons/execd/remoted_schemas.c | 1 + + include/crm/common/xml_internal.h | 1 + + lib/common/schemas.c | 40 +++++++++++++++++++++++++++++++ + 3 files changed, 42 insertions(+) + +diff --git a/daemons/execd/remoted_schemas.c b/daemons/execd/remoted_schemas.c +index d501fa495..eed43dfa9 100644 +--- a/daemons/execd/remoted_schemas.c ++++ b/daemons/execd/remoted_schemas.c +@@ -203,6 +203,7 @@ get_schema_files_complete(mainloop_child_t *p, pid_t pid, int core, int signo, i + * schemas again too. Instead just load the things we fetched. + */ + pcmk__load_schemas_from_dir(remote_schema_dir); ++ pcmk__sort_schemas(); + crm_info("Fetching extra schema files completed successfully"); + + } else { +diff --git a/include/crm/common/xml_internal.h b/include/crm/common/xml_internal.h +index cb27ec6b2..9b50f4e72 100644 +--- a/include/crm/common/xml_internal.h ++++ b/include/crm/common/xml_internal.h +@@ -448,5 +448,6 @@ gboolean pcmk__validate_xml(xmlNode *xml_blob, const char *validation, + + void pcmk__log_known_schemas(void); + const char *pcmk__remote_schema_dir(void); ++void pcmk__sort_schemas(void); + + #endif // PCMK__XML_INTERNAL__H +diff --git a/lib/common/schemas.c b/lib/common/schemas.c +index 68d79cfc7..fd6202f62 100644 +--- a/lib/common/schemas.c ++++ b/lib/common/schemas.c +@@ -449,6 +449,41 @@ pcmk__load_schemas_from_dir(const char *dir) + free(namelist); + } + ++static gint ++schema_sort_GCompareFunc(gconstpointer a, gconstpointer b) ++{ ++ const pcmk__schema_t *schema_a = a; ++ const pcmk__schema_t *schema_b = b; ++ ++ if (pcmk__str_eq(schema_a->name, "pacemaker-next", pcmk__str_none)) { ++ if (pcmk__str_eq(schema_b->name, "none", pcmk__str_none)) { ++ return -1; ++ } else { ++ return 1; ++ } ++ } else if (pcmk__str_eq(schema_a->name, "none", pcmk__str_none)) { ++ return 1; ++ } else if (pcmk__str_eq(schema_b->name, "pacemaker-next", pcmk__str_none)) { ++ return -1; ++ } else { ++ return schema_cmp(schema_a->version, schema_b->version); ++ } ++} ++ ++/*! ++ * \internal ++ * \brief Sort the list of known schemas such that all pacemaker-X.Y are in ++ * version order, then pacemaker-next, then none ++ * ++ * This function should be called whenever additional schemas are loaded using ++ * pcmk__load_schemas_from_dir(), after the initial sets in crm_schema_init(). ++ */ ++void ++pcmk__sort_schemas(void) ++{ ++ known_schemas = g_list_sort(known_schemas, schema_sort_GCompareFunc); ++} ++ + /*! + * \internal + * \brief Load pacemaker schemas into cache +@@ -474,6 +509,11 @@ crm_schema_init(void) + + add_schema(pcmk__schema_validator_none, &zero, PCMK__VALUE_NONE, + NULL, NULL, FALSE); ++ ++ /* This shouldn't be strictly necessary, but we'll do it here just in case ++ * there's anything in PCMK__REMOTE_SCHEMA_DIR that messes up the order. ++ */ ++ pcmk__sort_schemas(); + } + + static gboolean +-- +2.41.0 + +From d6a535ca43db202bd01a2e085b58722ed1abcdb0 Mon Sep 17 00:00:00 2001 +From: Chris Lumens +Date: Mon, 30 Oct 2023 12:59:13 -0400 +Subject: [PATCH 15/15] Feature: daemons: Only ask for schemas if supported by + the server + +We can use LRMD_PROTOCOL_VERSION in the handshake to determine what the +server supports, similar to what is being done in attrd. Add a macro to +compare the version we were given with the known minimum version that +supports the schema transfer command. + +Additionally, introduce LRMD_COMPATIBLE_PROTOCOL which is just the major +version number required for the connection to succeed. This gets rid of +the need for LRMD_MIN_PROTOCOL_VERSION, which can now be deprecated. + +And then since I wasn't sure compare_version would work if you give it a +full version number and just a major version, add a unit test for that. +--- + daemons/execd/execd_commands.c | 11 +++++---- + include/crm/lrmd.h | 23 +++++++++++++++---- + lib/common/tests/utils/compare_version_test.c | 5 +++- + 3 files changed, 30 insertions(+), 9 deletions(-) + +diff --git a/daemons/execd/execd_commands.c b/daemons/execd/execd_commands.c +index 1601efb0b..4ec4d03d6 100644 +--- a/daemons/execd/execd_commands.c ++++ b/daemons/execd/execd_commands.c +@@ -1482,9 +1482,9 @@ process_lrmd_signon(pcmk__client_t *client, xmlNode *request, int call_id, + const char *protocol_version = crm_element_value(request, F_LRMD_PROTOCOL_VERSION); + const char *start_state = pcmk__env_option(PCMK__ENV_NODE_START_STATE); + +- if (compare_version(protocol_version, LRMD_MIN_PROTOCOL_VERSION) < 0) { ++ if (compare_version(protocol_version, LRMD_COMPATIBLE_PROTOCOL) < 0) { + crm_err("Cluster API version must be greater than or equal to %s, not %s", +- LRMD_MIN_PROTOCOL_VERSION, protocol_version); ++ LRMD_COMPATIBLE_PROTOCOL, protocol_version); + rc = -EPROTO; + } + +@@ -1498,8 +1498,11 @@ process_lrmd_signon(pcmk__client_t *client, xmlNode *request, int call_id, + // This is a remote connection from a cluster node's controller + ipc_proxy_add_provider(client); + +- /* If this was a register operation, also ask for new schema files. */ +- if (pcmk__str_eq(op, CRM_OP_REGISTER, pcmk__str_none)) { ++ /* If this was a register operation, also ask for new schema files but ++ * only if it's supported by the protocol version. ++ */ ++ if (pcmk__str_eq(op, CRM_OP_REGISTER, pcmk__str_none) && ++ LRMD_SUPPORTS_SCHEMA_XFER(protocol_version)) { + remoted_request_cib_schema_files(); + } + } else { +diff --git a/include/crm/lrmd.h b/include/crm/lrmd.h +index 0c5a40b62..948a3b1fc 100644 +--- a/include/crm/lrmd.h ++++ b/include/crm/lrmd.h +@@ -33,12 +33,27 @@ typedef struct lrmd_key_value_s { + struct lrmd_key_value_s *next; + } lrmd_key_value_t; + +-/* This should be bumped every time there is an incompatible change that +- * prevents older clients from connecting to this version of the server. ++/* The major version should be bumped every time there is an incompatible ++ * change that prevents older clients from connecting to this version of ++ * the server. The minor version indicates feature support. ++ * ++ * Protocol Pacemaker Significant changes ++ * -------- --------- ------------------- ++ * 1.2 2.1.7 PCMK__CIB_REQUEST_SCHEMAS + */ +-#define LRMD_PROTOCOL_VERSION "1.1" ++#define LRMD_PROTOCOL_VERSION "1.2" ++ ++#define LRMD_SUPPORTS_SCHEMA_XFER(x) (compare_version((x), "1.2") >= 0) + +-/* This is the version that the client version will actually be compared ++/* The major protocol version the client and server both need to support for ++ * the connection to be successful. This should only ever be the major ++ * version - not a complete version number. ++ */ ++#define LRMD_COMPATIBLE_PROTOCOL "1" ++ ++/* \deprecated Do not use (will be removed in a future release) ++ * ++ * This is the version that the client version will actually be compared + * against. This should be identical to LRMD_PROTOCOL_VERSION. However, we + * accidentally bumped LRMD_PROTOCOL_VERSION in 6424a647 (1.1.15) when we didn't + * need to, so for now it's different. If we ever have a truly incompatible +diff --git a/lib/common/tests/utils/compare_version_test.c b/lib/common/tests/utils/compare_version_test.c +index 35ebb63c6..d191f4abb 100644 +--- a/lib/common/tests/utils/compare_version_test.c ++++ b/lib/common/tests/utils/compare_version_test.c +@@ -1,5 +1,5 @@ + /* +- * Copyright 2022 the Pacemaker project contributors ++ * Copyright 2022-2023 the Pacemaker project contributors + * + * The version control history for this file may have further details. + * +@@ -46,6 +46,9 @@ shorter_versions(void **state) + { + assert_int_equal(compare_version("1.0", "1.0.1"), -1); + assert_int_equal(compare_version("1.0.1", "1.0"), 1); ++ assert_int_equal(compare_version("1.0", "1"), 0); ++ assert_int_equal(compare_version("1", "1.2"), -1); ++ assert_int_equal(compare_version("1.2", "1"), 1); + } + + PCMK__UNIT_TEST(NULL, NULL, +-- +2.41.0 + diff --git a/SOURCES/003-clone-shuffle.patch b/SOURCES/003-clone-shuffle.patch deleted file mode 100644 index e615705..0000000 --- a/SOURCES/003-clone-shuffle.patch +++ /dev/null @@ -1,15630 +0,0 @@ -From eae355ca4c869f7ccf1ad3d1f5ce488375a6f353 Mon Sep 17 00:00:00 2001 -From: Reid Wahl -Date: Tue, 20 Apr 2021 12:55:45 -0700 -Subject: [PATCH 01/19] Refactor: libpe_status: Add pe__rsc_node flag enum - -This commit adds a new pe__rsc_node flag enum containing values for -assigned, current, and pending. This indicates the criterion used to -look up a resource's location. - -After a compatibility break, native_location() could use these flags -instead of an int. - -Signed-off-by: Reid Wahl ---- - include/crm/pengine/internal.h | 14 ++++++++++++++ - lib/pengine/native.c | 1 + - 2 files changed, 15 insertions(+) - -diff --git a/include/crm/pengine/internal.h b/include/crm/pengine/internal.h -index 1b5f6f1d8d9..8fcb9c991f3 100644 ---- a/include/crm/pengine/internal.h -+++ b/include/crm/pengine/internal.h -@@ -235,6 +235,19 @@ bool pe_can_fence(const pe_working_set_t *data_set, const pe_node_t *node); - - void add_hash_param(GHashTable * hash, const char *name, const char *value); - -+/*! -+ * \internal -+ * \enum pe__rsc_node -+ * \brief Type of resource location lookup to perform -+ */ -+enum pe__rsc_node { -+ pe__rsc_node_assigned = 0, //!< Where resource is assigned -+ pe__rsc_node_current = 1, //!< Where resource is running -+ -+ // @COMPAT: Use in native_location() at a compatibility break -+ pe__rsc_node_pending = 2, //!< Where resource is pending -+}; -+ - char *native_parameter(pe_resource_t * rsc, pe_node_t * node, gboolean create, const char *name, - pe_working_set_t * data_set); - pe_node_t *native_location(const pe_resource_t *rsc, GList **list, int current); -@@ -576,6 +589,7 @@ bool pe__bundle_needs_remote_name(pe_resource_t *rsc); - const char *pe__add_bundle_remote_name(pe_resource_t *rsc, - pe_working_set_t *data_set, - xmlNode *xml, const char *field); -+ - const char *pe_node_attribute_calculated(const pe_node_t *node, - const char *name, - const pe_resource_t *rsc); -diff --git a/lib/pengine/native.c b/lib/pengine/native.c -index 5e92ddcefdf..44d4805ac56 100644 ---- a/lib/pengine/native.c -+++ b/lib/pengine/native.c -@@ -1092,6 +1092,7 @@ native_resource_state(const pe_resource_t * rsc, gboolean current) - pe_node_t * - native_location(const pe_resource_t *rsc, GList **list, int current) - { -+ // @COMPAT: Accept a pe__rsc_node argument instead of int current - pe_node_t *one = NULL; - GList *result = NULL; - - -From 809b9c2ea13e5f32bfa6eecf3482eb257802b92d Mon Sep 17 00:00:00 2001 -From: Reid Wahl -Date: Sun, 11 Sep 2022 19:36:07 -0700 -Subject: [PATCH 02/19] Refactor: libpe_status: pe_node_attribute_calculated() - accepts node type - -Use enum pe__rsc_node in pe_node_attribute_calculated() to determine -which container host (assigned or current) to get the attribute value -from. For now, there's no use case for pending. - -Pass pe__rsc_node_current for existing calls, since that maintains the -existing behavior. - -Signed-off-by: Reid Wahl ---- - include/crm/pengine/internal.h | 3 +- - lib/pacemaker/pcmk_sched_location.c | 5 ++- - lib/pacemaker/pcmk_sched_promotable.c | 3 +- - lib/pengine/common.c | 60 ++++++++++++++++++++++----- - 4 files changed, 57 insertions(+), 14 deletions(-) - -diff --git a/include/crm/pengine/internal.h b/include/crm/pengine/internal.h -index 8fcb9c991f3..ef8c382f62b 100644 ---- a/include/crm/pengine/internal.h -+++ b/include/crm/pengine/internal.h -@@ -592,7 +592,8 @@ const char *pe__add_bundle_remote_name(pe_resource_t *rsc, - - const char *pe_node_attribute_calculated(const pe_node_t *node, - const char *name, -- const pe_resource_t *rsc); -+ const pe_resource_t *rsc, -+ enum pe__rsc_node node_type); - const char *pe_node_attribute_raw(const pe_node_t *node, const char *name); - bool pe__is_universal_clone(const pe_resource_t *rsc, - const pe_working_set_t *data_set); -diff --git a/lib/pacemaker/pcmk_sched_location.c b/lib/pacemaker/pcmk_sched_location.c -index b4ce4ff07dc..5f42ec0fc8c 100644 ---- a/lib/pacemaker/pcmk_sched_location.c -+++ b/lib/pacemaker/pcmk_sched_location.c -@@ -31,7 +31,10 @@ get_node_score(const char *rule, const char *score, bool raw, - score_f = char2score(score); - - } else { -- const char *attr_score = pe_node_attribute_calculated(node, score, rsc); -+ const char *attr_score = NULL; -+ -+ attr_score = pe_node_attribute_calculated(node, score, rsc, -+ pe__rsc_node_current); - - if (attr_score == NULL) { - crm_debug("Rule %s: %s did not have a value for %s", -diff --git a/lib/pacemaker/pcmk_sched_promotable.c b/lib/pacemaker/pcmk_sched_promotable.c -index d12d017bab2..2bad1d0c487 100644 ---- a/lib/pacemaker/pcmk_sched_promotable.c -+++ b/lib/pacemaker/pcmk_sched_promotable.c -@@ -649,7 +649,8 @@ promotion_attr_value(const pe_resource_t *rsc, const pe_node_t *node, - CRM_CHECK((rsc != NULL) && (node != NULL) && (name != NULL), return NULL); - - attr_name = pcmk_promotion_score_name(name); -- attr_value = pe_node_attribute_calculated(node, attr_name, rsc); -+ attr_value = pe_node_attribute_calculated(node, attr_name, rsc, -+ pe__rsc_node_current); - free(attr_name); - return attr_value; - } -diff --git a/lib/pengine/common.c b/lib/pengine/common.c -index 6c69bfcb41a..af41c1f6e89 100644 ---- a/lib/pengine/common.c -+++ b/lib/pengine/common.c -@@ -516,9 +516,15 @@ add_hash_param(GHashTable * hash, const char *name, const char *value) - - const char * - pe_node_attribute_calculated(const pe_node_t *node, const char *name, -- const pe_resource_t *rsc) -+ const pe_resource_t *rsc, -+ enum pe__rsc_node node_type) - { -- const char *source; -+ const char *source = NULL; -+ const char *node_type_s = NULL; -+ const char *reason = NULL; -+ -+ const pe_resource_t *container = NULL; -+ const pe_node_t *host = NULL; - - if(node == NULL) { - return NULL; -@@ -539,18 +545,50 @@ pe_node_attribute_calculated(const pe_node_t *node, const char *name, - * storage - */ - -- CRM_ASSERT(node->details->remote_rsc); -- CRM_ASSERT(node->details->remote_rsc->container); -+ CRM_ASSERT(node->details->remote_rsc != NULL); -+ -+ container = node->details->remote_rsc->container; -+ CRM_ASSERT(container != NULL); -+ -+ switch (node_type) { -+ case pe__rsc_node_assigned: -+ node_type_s = "assigned"; -+ host = container->allocated_to; -+ if (host == NULL) { -+ reason = "not assigned"; -+ } -+ break; -+ -+ case pe__rsc_node_current: -+ node_type_s = "current"; - -- if(node->details->remote_rsc->container->running_on) { -- pe_node_t *host = node->details->remote_rsc->container->running_on->data; -- pe_rsc_trace(rsc, "%s: Looking for %s on the container host %s", -- rsc->id, name, pe__node_name(host)); -- return g_hash_table_lookup(host->details->attrs, name); -+ if (container->running_on != NULL) { -+ host = container->running_on->data; -+ } -+ if (host == NULL) { -+ reason = "inactive"; -+ } -+ break; -+ -+ default: -+ // Add support for other enum pe__rsc_node values if needed -+ CRM_ASSERT(false); -+ break; - } - -- pe_rsc_trace(rsc, "%s: Not looking for %s on the container host: %s is inactive", -- rsc->id, name, node->details->remote_rsc->container->id); -+ if (host != NULL) { -+ const char *value = g_hash_table_lookup(host->details->attrs, name); -+ -+ pe_rsc_trace(rsc, -+ "%s: Value lookup for %s on %s container host %s %s%s", -+ rsc->id, name, node_type_s, pe__node_name(host), -+ ((value != NULL)? "succeeded: " : "failed"), -+ pcmk__s(value, "")); -+ return value; -+ } -+ pe_rsc_trace(rsc, -+ "%s: Not looking for %s on %s container host: %s is %s", -+ rsc->id, name, node_type_s, container->id, reason); - return NULL; - } - - -From d5a56afd2ecd861e0cf0d1049157e82a034f3f7a Mon Sep 17 00:00:00 2001 -From: Reid Wahl -Date: Thu, 15 Jun 2023 00:34:39 -0700 -Subject: [PATCH 03/19] Fix: libpacemaker: Get container attr from assigned - node, if any - -promotion_attr_value() should get a container's promotion score from -the host to which it's assigned (if it's been assigned), rather than the -host on which it's running. - -Ref T489 - -Signed-off-by: Reid Wahl ---- - lib/pacemaker/pcmk_sched_promotable.c | 10 ++++++---- - 1 file changed, 6 insertions(+), 4 deletions(-) - -diff --git a/lib/pacemaker/pcmk_sched_promotable.c b/lib/pacemaker/pcmk_sched_promotable.c -index 2bad1d0c487..8612c25a51d 100644 ---- a/lib/pacemaker/pcmk_sched_promotable.c -+++ b/lib/pacemaker/pcmk_sched_promotable.c -@@ -645,12 +645,14 @@ promotion_attr_value(const pe_resource_t *rsc, const pe_node_t *node, - { - char *attr_name = NULL; - const char *attr_value = NULL; -+ enum pe__rsc_node node_type = pe__rsc_node_assigned; - -- CRM_CHECK((rsc != NULL) && (node != NULL) && (name != NULL), return NULL); -- -+ if (pcmk_is_set(rsc->flags, pe_rsc_provisional)) { -+ // Not assigned yet -+ node_type = pe__rsc_node_current; -+ } - attr_name = pcmk_promotion_score_name(name); -- attr_value = pe_node_attribute_calculated(node, attr_name, rsc, -- pe__rsc_node_current); -+ attr_value = pe_node_attribute_calculated(node, attr_name, rsc, node_type); - free(attr_name); - return attr_value; - } - -From cfc2cd20e15c0f1c6b6ed8517c310acd756c1533 Mon Sep 17 00:00:00 2001 -From: Reid Wahl -Date: Mon, 10 Jul 2023 02:26:26 -0700 -Subject: [PATCH 04/19] Test: scheduler: Update outputs for - promotion_attr_value() fix - -Update outputs after previous commit (get container's promotion score -from assigned host). - -There are a few changes to scores, as well as dot and exp files. The -behavior in the bundle-interleave-start test appears to be an -improvement. - -Ref T489 - -Signed-off-by: Reid Wahl ---- - cts/scheduler/dot/bundle-interleave-start.dot | 44 +- - cts/scheduler/exp/bundle-interleave-start.exp | 556 +++++++++++------- - .../exp/no-promote-on-unrunnable-guest.exp | 14 +- - .../scores/bundle-interleave-start.scores | 12 +- - .../scores/cancel-behind-moving-remote.scores | 2 +- - .../scores/guest-host-not-fenceable.scores | 2 +- - .../no-promote-on-unrunnable-guest.scores | 2 +- - .../summary/bundle-interleave-start.summary | 54 +- - cts/scheduler/xml/bundle-interleave-start.xml | 3 +- - 9 files changed, 445 insertions(+), 244 deletions(-) - -diff --git a/cts/scheduler/dot/bundle-interleave-start.dot b/cts/scheduler/dot/bundle-interleave-start.dot -index bf6ed7f9edb..a513ac5806b 100644 ---- a/cts/scheduler/dot/bundle-interleave-start.dot -+++ b/cts/scheduler/dot/bundle-interleave-start.dot -@@ -41,9 +41,15 @@ - "app-bundle-2_monitor_0 node5" [ style=bold color="green" fontcolor="black"] - "app-bundle-2_monitor_30000 node4" [ style=bold color="green" fontcolor="black"] - "app-bundle-2_start_0 node4" -> "app-bundle-2_monitor_30000 node4" [ style = bold] --"app-bundle-2_start_0 node4" -> "app:2_monitor_16000 app-bundle-2" [ style = bold] -+"app-bundle-2_start_0 node4" -> "app:2_monitor_15000 app-bundle-2" [ style = bold] -+"app-bundle-2_start_0 node4" -> "app:2_promote_0 app-bundle-2" [ style = bold] - "app-bundle-2_start_0 node4" -> "app:2_start_0 app-bundle-2" [ style = bold] - "app-bundle-2_start_0 node4" [ style=bold color="green" fontcolor="black"] -+"app-bundle-clone_promote_0" -> "app:2_promote_0 app-bundle-2" [ style = bold] -+"app-bundle-clone_promote_0" [ style=bold color="green" fontcolor="orange"] -+"app-bundle-clone_promoted_0" -> "app-bundle_promoted_0" [ style = bold] -+"app-bundle-clone_promoted_0" [ style=bold color="green" fontcolor="orange"] -+"app-bundle-clone_running_0" -> "app-bundle-clone_promote_0" [ style = bold] - "app-bundle-clone_running_0" -> "app-bundle_running_0" [ style = bold] - "app-bundle-clone_running_0" [ style=bold color="green" fontcolor="orange"] - "app-bundle-clone_start_0" -> "app-bundle-clone_running_0" [ style = bold] -@@ -133,8 +139,13 @@ - "app-bundle-podman-2_start_0 node4" -> "app-bundle-2_start_0 node4" [ style = bold] - "app-bundle-podman-2_start_0 node4" -> "app-bundle-podman-2_monitor_60000 node4" [ style = bold] - "app-bundle-podman-2_start_0 node4" -> "app-bundle_running_0" [ style = bold] -+"app-bundle-podman-2_start_0 node4" -> "app:2_promote_0 app-bundle-2" [ style = bold] - "app-bundle-podman-2_start_0 node4" -> "app:2_start_0 app-bundle-2" [ style = bold] - "app-bundle-podman-2_start_0 node4" [ style=bold color="green" fontcolor="black"] -+"app-bundle_promote_0" -> "app-bundle-clone_promote_0" [ style = bold] -+"app-bundle_promote_0" [ style=bold color="green" fontcolor="orange"] -+"app-bundle_promoted_0" [ style=bold color="green" fontcolor="orange"] -+"app-bundle_running_0" -> "app-bundle_promote_0" [ style = bold] - "app-bundle_running_0" [ style=bold color="green" fontcolor="orange"] - "app-bundle_start_0" -> "app-bundle-clone_start_0" [ style = bold] - "app-bundle_start_0" -> "app-bundle-podman-0_start_0 node2" [ style = bold] -@@ -151,9 +162,13 @@ - "app:1_start_0 app-bundle-1" -> "app:1_monitor_16000 app-bundle-1" [ style = bold] - "app:1_start_0 app-bundle-1" -> "app:2_start_0 app-bundle-2" [ style = bold] - "app:1_start_0 app-bundle-1" [ style=bold color="green" fontcolor="black"] --"app:2_monitor_16000 app-bundle-2" [ style=bold color="green" fontcolor="black"] -+"app:2_monitor_15000 app-bundle-2" [ style=bold color="green" fontcolor="black"] -+"app:2_promote_0 app-bundle-2" -> "app-bundle-clone_promoted_0" [ style = bold] -+"app:2_promote_0 app-bundle-2" -> "app:2_monitor_15000 app-bundle-2" [ style = bold] -+"app:2_promote_0 app-bundle-2" [ style=bold color="green" fontcolor="black"] - "app:2_start_0 app-bundle-2" -> "app-bundle-clone_running_0" [ style = bold] --"app:2_start_0 app-bundle-2" -> "app:2_monitor_16000 app-bundle-2" [ style = bold] -+"app:2_start_0 app-bundle-2" -> "app:2_monitor_15000 app-bundle-2" [ style = bold] -+"app:2_start_0 app-bundle-2" -> "app:2_promote_0 app-bundle-2" [ style = bold] - "app:2_start_0 app-bundle-2" [ style=bold color="green" fontcolor="black"] - "base-bundle-0_monitor_0 node1" -> "base-bundle-0_start_0 node2" [ style = bold] - "base-bundle-0_monitor_0 node1" [ style=bold color="green" fontcolor="black"] -@@ -197,9 +212,15 @@ - "base-bundle-2_monitor_0 node5" [ style=bold color="green" fontcolor="black"] - "base-bundle-2_monitor_30000 node4" [ style=bold color="green" fontcolor="black"] - "base-bundle-2_start_0 node4" -> "base-bundle-2_monitor_30000 node4" [ style = bold] --"base-bundle-2_start_0 node4" -> "base:2_monitor_16000 base-bundle-2" [ style = bold] -+"base-bundle-2_start_0 node4" -> "base:2_monitor_15000 base-bundle-2" [ style = bold] -+"base-bundle-2_start_0 node4" -> "base:2_promote_0 base-bundle-2" [ style = bold] - "base-bundle-2_start_0 node4" -> "base:2_start_0 base-bundle-2" [ style = bold] - "base-bundle-2_start_0 node4" [ style=bold color="green" fontcolor="black"] -+"base-bundle-clone_promote_0" -> "base:2_promote_0 base-bundle-2" [ style = bold] -+"base-bundle-clone_promote_0" [ style=bold color="green" fontcolor="orange"] -+"base-bundle-clone_promoted_0" -> "base-bundle_promoted_0" [ style = bold] -+"base-bundle-clone_promoted_0" [ style=bold color="green" fontcolor="orange"] -+"base-bundle-clone_running_0" -> "base-bundle-clone_promote_0" [ style = bold] - "base-bundle-clone_running_0" -> "base-bundle_running_0" [ style = bold] - "base-bundle-clone_running_0" [ style=bold color="green" fontcolor="orange"] - "base-bundle-clone_start_0" -> "base-bundle-clone_running_0" [ style = bold] -@@ -289,9 +310,15 @@ - "base-bundle-podman-2_start_0 node4" -> "base-bundle-2_start_0 node4" [ style = bold] - "base-bundle-podman-2_start_0 node4" -> "base-bundle-podman-2_monitor_60000 node4" [ style = bold] - "base-bundle-podman-2_start_0 node4" -> "base-bundle_running_0" [ style = bold] -+"base-bundle-podman-2_start_0 node4" -> "base:2_promote_0 base-bundle-2" [ style = bold] - "base-bundle-podman-2_start_0 node4" -> "base:2_start_0 base-bundle-2" [ style = bold] - "base-bundle-podman-2_start_0 node4" [ style=bold color="green" fontcolor="black"] -+"base-bundle_promote_0" -> "base-bundle-clone_promote_0" [ style = bold] -+"base-bundle_promote_0" [ style=bold color="green" fontcolor="orange"] -+"base-bundle_promoted_0" -> "app-bundle_promote_0" [ style = bold] -+"base-bundle_promoted_0" [ style=bold color="green" fontcolor="orange"] - "base-bundle_running_0" -> "app-bundle_start_0" [ style = bold] -+"base-bundle_running_0" -> "base-bundle_promote_0" [ style = bold] - "base-bundle_running_0" [ style=bold color="green" fontcolor="orange"] - "base-bundle_start_0" -> "base-bundle-clone_start_0" [ style = bold] - "base-bundle_start_0" -> "base-bundle-podman-0_start_0 node2" [ style = bold] -@@ -310,9 +337,14 @@ - "base:1_start_0 base-bundle-1" -> "base:1_monitor_16000 base-bundle-1" [ style = bold] - "base:1_start_0 base-bundle-1" -> "base:2_start_0 base-bundle-2" [ style = bold] - "base:1_start_0 base-bundle-1" [ style=bold color="green" fontcolor="black"] --"base:2_monitor_16000 base-bundle-2" [ style=bold color="green" fontcolor="black"] -+"base:2_monitor_15000 base-bundle-2" [ style=bold color="green" fontcolor="black"] -+"base:2_promote_0 base-bundle-2" -> "app:2_promote_0 app-bundle-2" [ style = bold] -+"base:2_promote_0 base-bundle-2" -> "base-bundle-clone_promoted_0" [ style = bold] -+"base:2_promote_0 base-bundle-2" -> "base:2_monitor_15000 base-bundle-2" [ style = bold] -+"base:2_promote_0 base-bundle-2" [ style=bold color="green" fontcolor="black"] - "base:2_start_0 base-bundle-2" -> "app-bundle-podman-2_start_0 node4" [ style = bold] - "base:2_start_0 base-bundle-2" -> "base-bundle-clone_running_0" [ style = bold] --"base:2_start_0 base-bundle-2" -> "base:2_monitor_16000 base-bundle-2" [ style = bold] -+"base:2_start_0 base-bundle-2" -> "base:2_monitor_15000 base-bundle-2" [ style = bold] -+"base:2_start_0 base-bundle-2" -> "base:2_promote_0 base-bundle-2" [ style = bold] - "base:2_start_0 base-bundle-2" [ style=bold color="green" fontcolor="black"] - } -diff --git a/cts/scheduler/exp/bundle-interleave-start.exp b/cts/scheduler/exp/bundle-interleave-start.exp -index e676b1bfba9..57e551c487e 100644 ---- a/cts/scheduler/exp/bundle-interleave-start.exp -+++ b/cts/scheduler/exp/bundle-interleave-start.exp -@@ -1,7 +1,7 @@ - - - -- -+ - - - -@@ -11,13 +11,13 @@ - - - -- -+ - - - - - -- -+ - - - -@@ -30,13 +30,13 @@ - - - -- -+ - - - - - -- -+ - - - -@@ -46,13 +46,13 @@ - - - -- -+ - - - - - -- -+ - - - -@@ -65,18 +65,18 @@ - - - -- -+ - - -- -+ - - - - - -- -+ - -- -+ - - - -@@ -84,13 +84,38 @@ - - - -- -+ -+ -+ -+ - - - - - -- -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ - - - -@@ -103,37 +128,64 @@ - - - -- -+ - - -- -+ - - - -- -+ - -- -+ - - - - - -- -+ - -+ -+ -+ -+ -+ -+ -+ -+ -+ - -- -+ - - -- -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ - - -- -+ -+ -+ -+ -+ -+ -+ - - - -- -+ - -- -+ - - - -@@ -188,7 +240,7 @@ - - - -- -+ - - - -@@ -201,7 +253,7 @@ - - - -- -+ - - - -@@ -235,7 +287,7 @@ - - - -- -+ - - - -@@ -244,7 +296,7 @@ - - - -- -+ - - - -@@ -253,7 +305,7 @@ - - - -- -+ - - - -@@ -262,7 +314,7 @@ - - - -- -+ - - - -@@ -271,7 +323,7 @@ - - - -- -+ - - - -@@ -280,7 +332,7 @@ - - - -- -+ - - - -@@ -293,7 +345,7 @@ - - - -- -+ - - - -@@ -321,7 +373,7 @@ - - - -- -+ - - - -@@ -334,7 +386,7 @@ - - - -- -+ - - - -@@ -347,7 +399,7 @@ - - - -- -+ - - - -@@ -360,7 +412,7 @@ - - - -- -+ - - - -@@ -373,7 +425,7 @@ - - - -- -+ - - - -@@ -386,7 +438,7 @@ - - - -- -+ - - - -@@ -399,7 +451,7 @@ - - - -- -+ - - - -@@ -433,7 +485,7 @@ - - - -- -+ - - - -@@ -442,7 +494,7 @@ - - - -- -+ - - - -@@ -451,7 +503,7 @@ - - - -- -+ - - - -@@ -460,7 +512,7 @@ - - - -- -+ - - - -@@ -469,7 +521,7 @@ - - - -- -+ - - - -@@ -478,7 +530,7 @@ - - - -- -+ - - - -@@ -491,7 +543,7 @@ - - - -- -+ - - - -@@ -519,7 +571,7 @@ - - - -- -+ - - - -@@ -532,7 +584,7 @@ - - - -- -+ - - - -@@ -545,7 +597,7 @@ - - - -- -+ - - - -@@ -558,7 +610,7 @@ - - - -- -+ - - - -@@ -571,7 +623,7 @@ - - - -- -+ - - - -@@ -584,7 +636,7 @@ - - - -- -+ - - - -@@ -597,7 +649,7 @@ - - - -- -+ - - - -@@ -631,7 +683,7 @@ - - - -- -+ - - - -@@ -640,7 +692,7 @@ - - - -- -+ - - - -@@ -649,7 +701,7 @@ - - - -- -+ - - - -@@ -658,7 +710,7 @@ - - - -- -+ - - - -@@ -667,7 +719,7 @@ - - - -- -+ - - - -@@ -676,7 +728,7 @@ - - - -- -+ - - - -@@ -689,7 +741,7 @@ - - - -- -+ - - - -@@ -717,7 +769,7 @@ - - - -- -+ - - - -@@ -730,7 +782,7 @@ - - - -- -+ - - - -@@ -743,7 +795,7 @@ - - - -- -+ - - - -@@ -756,7 +808,7 @@ - - - -- -+ - - - -@@ -769,7 +821,7 @@ - - - -- -+ - - - -@@ -782,141 +834,196 @@ - - - -- -+ - -- -+ - - - - - - -- -+ - - -- -+ - - - -- -+ - -- -+ - - - - - - -- -+ - - -- -+ - - -- -+ - - - -- -+ - -- -+ - - - - - - -- -+ - - -- -+ - - - -- -+ - -- -+ - - - - - - -- -+ - - -- -+ - - -- -+ - - -- -+ - - - -- -+ - -- -+ - -- -+ - - - - -- -+ - - -- -+ -+ -+ -+ - - - -- -+ - -- -+ - - - - - - -- -+ -+ -+ -+ - - -- -+ - - -- -+ - - -- -+ - - - -- -+ - -- -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ - - - - - -- -+ - - -- -+ - - -- -+ - - -- -+ - - - -- -+ - -- -+ - - - -@@ -967,26 +1074,26 @@ - - - -- -+ - - - -- -+ - -- -+ - - - - - - -- -+ - - - -- -+ - -- -+ - - - -@@ -1014,14 +1121,14 @@ - - - -- -+ - - -- -+ - - - -- -+ - - - -@@ -1030,7 +1137,7 @@ - - - -- -+ - - - -@@ -1039,7 +1146,7 @@ - - - -- -+ - - - -@@ -1048,7 +1155,7 @@ - - - -- -+ - - - -@@ -1057,7 +1164,7 @@ - - - -- -+ - - - -@@ -1066,22 +1173,22 @@ - - - -- -+ - -- -+ - - - - - - -- -+ - - - -- -+ - -- -+ - - - -@@ -1103,11 +1210,11 @@ - - - -- -+ - - - -- -+ - - - -@@ -1116,11 +1223,11 @@ - - - -- -+ - - - -- -+ - - - -@@ -1129,11 +1236,11 @@ - - - -- -+ - - - -- -+ - - - -@@ -1142,11 +1249,11 @@ - - - -- -+ - - - -- -+ - - - -@@ -1155,11 +1262,11 @@ - - - -- -+ - - - -- -+ - - - -@@ -1168,26 +1275,26 @@ - - - -- -+ - - - -- -+ - -- -+ - - - - - - -- -+ - - - -- -+ - -- -+ - - - -@@ -1215,14 +1322,14 @@ - - - -- -+ - - -- -+ - - - -- -+ - - - -@@ -1231,7 +1338,7 @@ - - - -- -+ - - - -@@ -1240,7 +1347,7 @@ - - - -- -+ - - - -@@ -1249,7 +1356,7 @@ - - - -- -+ - - - -@@ -1258,7 +1365,7 @@ - - - -- -+ - - - -@@ -1267,22 +1374,22 @@ - - - -- -+ - -- -+ - - - - - - -- -+ - - - -- -+ - -- -+ - - - -@@ -1304,11 +1411,11 @@ - - - -- -+ - - - -- -+ - - - -@@ -1317,11 +1424,11 @@ - - - -- -+ - - - -- -+ - - - -@@ -1330,11 +1437,11 @@ - - - -- -+ - - - -- -+ - - - -@@ -1343,11 +1450,11 @@ - - - -- -+ - - - -- -+ - - - -@@ -1356,11 +1463,11 @@ - - - -- -+ - - - -- -+ - - - -@@ -1369,26 +1476,26 @@ - - - -- -+ - - - -- -+ - -- -+ - - - - - - -- -+ - - - -- -+ - -- -+ - - - -@@ -1416,14 +1523,14 @@ - - - -- -+ - - -- -+ - - - -- -+ - - - -@@ -1432,7 +1539,7 @@ - - - -- -+ - - - -@@ -1441,7 +1548,7 @@ - - - -- -+ - - - -@@ -1450,7 +1557,7 @@ - - - -- -+ - - - -@@ -1459,7 +1566,7 @@ - - - -- -+ - - - -@@ -1468,22 +1575,22 @@ - - - -- -+ - -- -+ - - - - - - -- -+ - - - -- -+ - -- -+ - - - -@@ -1505,11 +1612,11 @@ - - - -- -+ - - - -- -+ - - - -@@ -1518,11 +1625,11 @@ - - - -- -+ - - - -- -+ - - - -@@ -1531,11 +1638,11 @@ - - - -- -+ - - - -- -+ - - - -@@ -1544,11 +1651,11 @@ - - - -- -+ - - - -- -+ - - - -@@ -1557,11 +1664,11 @@ - - - -- -+ - - - -- -+ - - - -@@ -1570,34 +1677,61 @@ - - - -- -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ - - - -- -+ - -- -+ - - - - - -- -+ - - -- -+ - -+ -+ -+ -+ -+ -+ -+ -+ -+ - -- -+ - - -- -+ -+ -+ -+ -+ -+ -+ - - - -- -+ - -- -+ - - - -@@ -1607,7 +1741,31 @@ - - - -- -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ - - - -@@ -1624,11 +1782,11 @@ - - - -- -+ - - - -- -+ - - - -diff --git a/cts/scheduler/exp/no-promote-on-unrunnable-guest.exp b/cts/scheduler/exp/no-promote-on-unrunnable-guest.exp -index 351aec11df0..350495f4a6f 100644 ---- a/cts/scheduler/exp/no-promote-on-unrunnable-guest.exp -+++ b/cts/scheduler/exp/no-promote-on-unrunnable-guest.exp -@@ -14,7 +14,7 @@ - - - -- -+ - - - -@@ -82,14 +82,14 @@ - - - -- -+ - - - - - - -- -+ - - - -@@ -101,7 +101,7 @@ - - - -- -+ - - - -@@ -250,7 +250,7 @@ - - - -- -+ - - - -@@ -396,7 +396,7 @@ - - - -- -+ - - - -@@ -473,7 +473,7 @@ - - - -- -+ - - - -diff --git a/cts/scheduler/scores/bundle-interleave-start.scores b/cts/scheduler/scores/bundle-interleave-start.scores -index 7f4a370474d..b3aa9b571e8 100644 ---- a/cts/scheduler/scores/bundle-interleave-start.scores -+++ b/cts/scheduler/scores/bundle-interleave-start.scores -@@ -1,10 +1,10 @@ - --app:0 promotion score on app-bundle-0: -1 --app:1 promotion score on app-bundle-1: -1 --app:2 promotion score on app-bundle-2: -1 --base:0 promotion score on base-bundle-0: -1 --base:1 promotion score on base-bundle-1: -1 --base:2 promotion score on base-bundle-2: -1 -+app:0 promotion score on app-bundle-0: 12 -+app:1 promotion score on app-bundle-1: 13 -+app:2 promotion score on app-bundle-2: 14 -+base:0 promotion score on base-bundle-0: 12 -+base:1 promotion score on base-bundle-1: 13 -+base:2 promotion score on base-bundle-2: 14 - pcmk__bundle_allocate: app-bundle allocation score on node1: 0 - pcmk__bundle_allocate: app-bundle allocation score on node2: 0 - pcmk__bundle_allocate: app-bundle allocation score on node3: 0 -diff --git a/cts/scheduler/scores/cancel-behind-moving-remote.scores b/cts/scheduler/scores/cancel-behind-moving-remote.scores -index 0dfd78caa92..0e11b225aea 100644 ---- a/cts/scheduler/scores/cancel-behind-moving-remote.scores -+++ b/cts/scheduler/scores/cancel-behind-moving-remote.scores -@@ -2,7 +2,7 @@ - galera:0 promotion score on galera-bundle-0: 100 - galera:1 promotion score on galera-bundle-1: 100 - galera:2 promotion score on galera-bundle-2: 100 --ovndb_servers:0 promotion score on ovn-dbs-bundle-0: -1 -+ovndb_servers:0 promotion score on ovn-dbs-bundle-0: 5 - ovndb_servers:1 promotion score on ovn-dbs-bundle-1: 5 - ovndb_servers:2 promotion score on ovn-dbs-bundle-2: 5 - pcmk__bundle_allocate: galera-bundle allocation score on compute-0: -INFINITY -diff --git a/cts/scheduler/scores/guest-host-not-fenceable.scores b/cts/scheduler/scores/guest-host-not-fenceable.scores -index e4c7fc2033d..5f43bcb0812 100644 ---- a/cts/scheduler/scores/guest-host-not-fenceable.scores -+++ b/cts/scheduler/scores/guest-host-not-fenceable.scores -@@ -1,6 +1,6 @@ - - galera:0 promotion score on galera-bundle-0: 100 --galera:1 promotion score on galera-bundle-1: 100 -+galera:1 promotion score on galera-bundle-1: -1 - galera:2 promotion score on galera-bundle-2: -1 - pcmk__bundle_allocate: galera-bundle allocation score on node1: 0 - pcmk__bundle_allocate: galera-bundle allocation score on node2: 0 -diff --git a/cts/scheduler/scores/no-promote-on-unrunnable-guest.scores b/cts/scheduler/scores/no-promote-on-unrunnable-guest.scores -index 7923cdc2320..9362dc0e1f2 100644 ---- a/cts/scheduler/scores/no-promote-on-unrunnable-guest.scores -+++ b/cts/scheduler/scores/no-promote-on-unrunnable-guest.scores -@@ -2,7 +2,7 @@ - galera:0 promotion score on galera-bundle-0: 100 - galera:1 promotion score on galera-bundle-1: 100 - galera:2 promotion score on galera-bundle-2: 100 --ovndb_servers:0 promotion score on ovn-dbs-bundle-0: 5 -+ovndb_servers:0 promotion score on ovn-dbs-bundle-0: -1 - ovndb_servers:1 promotion score on ovn-dbs-bundle-1: 5 - ovndb_servers:2 promotion score on ovn-dbs-bundle-2: 5 - pcmk__bundle_allocate: galera-bundle allocation score on controller-0: 0 -diff --git a/cts/scheduler/summary/bundle-interleave-start.summary b/cts/scheduler/summary/bundle-interleave-start.summary -index 1648e929bf7..07ff7561968 100644 ---- a/cts/scheduler/summary/bundle-interleave-start.summary -+++ b/cts/scheduler/summary/bundle-interleave-start.summary -@@ -14,24 +14,24 @@ Current cluster status: - * app-bundle-2 (ocf:pacemaker:Stateful): Stopped - - Transition Summary: -- * Start base-bundle-podman-0 ( node2 ) -- * Start base-bundle-0 ( node2 ) -- * Start base:0 ( base-bundle-0 ) -- * Start base-bundle-podman-1 ( node3 ) -- * Start base-bundle-1 ( node3 ) -- * Start base:1 ( base-bundle-1 ) -- * Start base-bundle-podman-2 ( node4 ) -- * Start base-bundle-2 ( node4 ) -- * Start base:2 ( base-bundle-2 ) -- * Start app-bundle-podman-0 ( node2 ) -- * Start app-bundle-0 ( node2 ) -- * Start app:0 ( app-bundle-0 ) -- * Start app-bundle-podman-1 ( node3 ) -- * Start app-bundle-1 ( node3 ) -- * Start app:1 ( app-bundle-1 ) -- * Start app-bundle-podman-2 ( node4 ) -- * Start app-bundle-2 ( node4 ) -- * Start app:2 ( app-bundle-2 ) -+ * Start base-bundle-podman-0 ( node2 ) -+ * Start base-bundle-0 ( node2 ) -+ * Start base:0 ( base-bundle-0 ) -+ * Start base-bundle-podman-1 ( node3 ) -+ * Start base-bundle-1 ( node3 ) -+ * Start base:1 ( base-bundle-1 ) -+ * Start base-bundle-podman-2 ( node4 ) -+ * Start base-bundle-2 ( node4 ) -+ * Promote base:2 ( Stopped -> Promoted base-bundle-2 ) -+ * Start app-bundle-podman-0 ( node2 ) -+ * Start app-bundle-0 ( node2 ) -+ * Start app:0 ( app-bundle-0 ) -+ * Start app-bundle-podman-1 ( node3 ) -+ * Start app-bundle-1 ( node3 ) -+ * Start app:1 ( app-bundle-1 ) -+ * Start app-bundle-podman-2 ( node4 ) -+ * Start app-bundle-2 ( node4 ) -+ * Promote app:2 ( Stopped -> Promoted app-bundle-2 ) - - Executing Cluster Transition: - * Resource action: base-bundle-podman-0 monitor on node5 -@@ -100,8 +100,9 @@ Executing Cluster Transition: - * Pseudo action: base-bundle_running_0 - * Resource action: base:0 monitor=16000 on base-bundle-0 - * Resource action: base:1 monitor=16000 on base-bundle-1 -- * Resource action: base:2 monitor=16000 on base-bundle-2 - * Pseudo action: app-bundle_start_0 -+ * Pseudo action: base-bundle_promote_0 -+ * Pseudo action: base-bundle-clone_promote_0 - * Pseudo action: app-bundle-clone_start_0 - * Resource action: app-bundle-podman-0 start on node2 - * Resource action: app-bundle-0 monitor on node5 -@@ -121,12 +122,16 @@ Executing Cluster Transition: - * Resource action: app-bundle-2 monitor on node3 - * Resource action: app-bundle-2 monitor on node2 - * Resource action: app-bundle-2 monitor on node1 -+ * Resource action: base:2 promote on base-bundle-2 -+ * Pseudo action: base-bundle-clone_promoted_0 - * Resource action: app-bundle-podman-0 monitor=60000 on node2 - * Resource action: app-bundle-0 start on node2 - * Resource action: app-bundle-podman-1 monitor=60000 on node3 - * Resource action: app-bundle-1 start on node3 - * Resource action: app-bundle-podman-2 monitor=60000 on node4 - * Resource action: app-bundle-2 start on node4 -+ * Pseudo action: base-bundle_promoted_0 -+ * Resource action: base:2 monitor=15000 on base-bundle-2 - * Resource action: app:0 start on app-bundle-0 - * Resource action: app:1 start on app-bundle-1 - * Resource action: app:2 start on app-bundle-2 -@@ -137,7 +142,12 @@ Executing Cluster Transition: - * Pseudo action: app-bundle_running_0 - * Resource action: app:0 monitor=16000 on app-bundle-0 - * Resource action: app:1 monitor=16000 on app-bundle-1 -- * Resource action: app:2 monitor=16000 on app-bundle-2 -+ * Pseudo action: app-bundle_promote_0 -+ * Pseudo action: app-bundle-clone_promote_0 -+ * Resource action: app:2 promote on app-bundle-2 -+ * Pseudo action: app-bundle-clone_promoted_0 -+ * Pseudo action: app-bundle_promoted_0 -+ * Resource action: app:2 monitor=15000 on app-bundle-2 - - Revised Cluster Status: - * Node List: -@@ -149,8 +159,8 @@ Revised Cluster Status: - * Container bundle set: base-bundle [localhost/pcmktest:base]: - * base-bundle-0 (ocf:pacemaker:Stateful): Unpromoted node2 - * base-bundle-1 (ocf:pacemaker:Stateful): Unpromoted node3 -- * base-bundle-2 (ocf:pacemaker:Stateful): Unpromoted node4 -+ * base-bundle-2 (ocf:pacemaker:Stateful): Promoted node4 - * Container bundle set: app-bundle [localhost/pcmktest:app]: - * app-bundle-0 (ocf:pacemaker:Stateful): Unpromoted node2 - * app-bundle-1 (ocf:pacemaker:Stateful): Unpromoted node3 -- * app-bundle-2 (ocf:pacemaker:Stateful): Unpromoted node4 -+ * app-bundle-2 (ocf:pacemaker:Stateful): Promoted node4 -diff --git a/cts/scheduler/xml/bundle-interleave-start.xml b/cts/scheduler/xml/bundle-interleave-start.xml -index e8630cdf054..facb181b2a4 100644 ---- a/cts/scheduler/xml/bundle-interleave-start.xml -+++ b/cts/scheduler/xml/bundle-interleave-start.xml -@@ -6,7 +6,8 @@ - and its promoted role is colocated with base's. App's starts and - promotes are ordered after base's. - -- In this test, all are stopped and must be started. -+ In this test, all are stopped and must be started. One replica of each -+ bundle must be promoted. - --> - - - -From 6e5bc0d119c1609a3228763a5116a68829870948 Mon Sep 17 00:00:00 2001 -From: Reid Wahl -Date: Mon, 26 Jun 2023 12:42:10 -0700 -Subject: [PATCH 05/19] Refactor: libpacemaker: De-functionize - pcmk__finalize_assignment() - -Move it into pcmk__assign_resource(). Also correct the "was assignment -changed" logic, and allocate rc_stopped only once. - -Signed-off-by: Reid Wahl ---- - lib/pacemaker/libpacemaker_private.h | 4 - - lib/pacemaker/pcmk_sched_primitive.c | 9 +- - lib/pacemaker/pcmk_sched_resource.c | 156 ++++++++++++--------------- - 3 files changed, 74 insertions(+), 95 deletions(-) - -diff --git a/lib/pacemaker/libpacemaker_private.h b/lib/pacemaker/libpacemaker_private.h -index 192d5a703ff..614d695f83f 100644 ---- a/lib/pacemaker/libpacemaker_private.h -+++ b/lib/pacemaker/libpacemaker_private.h -@@ -908,10 +908,6 @@ void pcmk__noop_add_graph_meta(const pe_resource_t *rsc, xmlNode *xml); - G_GNUC_INTERNAL - void pcmk__output_resource_actions(pe_resource_t *rsc); - --G_GNUC_INTERNAL --bool pcmk__finalize_assignment(pe_resource_t *rsc, pe_node_t *chosen, -- bool force); -- - G_GNUC_INTERNAL - bool pcmk__assign_resource(pe_resource_t *rsc, pe_node_t *node, bool force); - -diff --git a/lib/pacemaker/pcmk_sched_primitive.c b/lib/pacemaker/pcmk_sched_primitive.c -index aefbf9aa140..2470b08ed69 100644 ---- a/lib/pacemaker/pcmk_sched_primitive.c -+++ b/lib/pacemaker/pcmk_sched_primitive.c -@@ -152,7 +152,6 @@ assign_best_node(pe_resource_t *rsc, const pe_node_t *prefer) - GList *nodes = NULL; - pe_node_t *chosen = NULL; - pe_node_t *best = NULL; -- bool result = false; - const pe_node_t *most_free_node = pcmk__ban_insufficient_capacity(rsc); - - if (prefer == NULL) { -@@ -260,9 +259,9 @@ assign_best_node(pe_resource_t *rsc, const pe_node_t *prefer) - pe__node_name(chosen), rsc->id, g_list_length(nodes)); - } - -- result = pcmk__finalize_assignment(rsc, chosen, false); -+ pcmk__assign_resource(rsc, chosen, false); - g_list_free(nodes); -- return result; -+ return rsc->allocated_to != NULL; - } - - /*! -@@ -475,11 +474,11 @@ pcmk__primitive_assign(pe_resource_t *rsc, const pe_node_t *prefer) - } - pe_rsc_info(rsc, "Unmanaged resource %s assigned to %s: %s", rsc->id, - (assign_to? assign_to->details->uname : "no node"), reason); -- pcmk__finalize_assignment(rsc, assign_to, true); -+ pcmk__assign_resource(rsc, assign_to, true); - - } else if (pcmk_is_set(rsc->cluster->flags, pe_flag_stop_everything)) { - pe_rsc_debug(rsc, "Forcing %s to stop: stop-all-resources", rsc->id); -- pcmk__finalize_assignment(rsc, NULL, true); -+ pcmk__assign_resource(rsc, NULL, true); - - } else if (pcmk_is_set(rsc->flags, pe_rsc_provisional) - && assign_best_node(rsc, prefer)) { -diff --git a/lib/pacemaker/pcmk_sched_resource.c b/lib/pacemaker/pcmk_sched_resource.c -index b8554998197..dd9939a42a6 100644 ---- a/lib/pacemaker/pcmk_sched_resource.c -+++ b/lib/pacemaker/pcmk_sched_resource.c -@@ -331,140 +331,124 @@ pcmk__output_resource_actions(pe_resource_t *rsc) - - /*! - * \internal -- * \brief Assign a specified primitive resource to a node -+ * \brief Assign a specified resource (of any variant) to a node - * -- * Assign a specified primitive resource to a specified node, if the node can -- * run the resource (or unconditionally, if \p force is true). Mark the resource -- * as no longer provisional. If the primitive can't be assigned (or \p chosen is -- * NULL), unassign any previous assignment for it, set its next role to stopped, -- * and update any existing actions scheduled for it. This is not done -- * recursively for children, so it should be called only for primitives. -+ * Assign a specified resource and its children (if any) to a specified node, if -+ * the node can run the resource (or unconditionally, if \p force is true). Mark -+ * the resources as no longer provisional. If a resource can't be assigned (or -+ * \p node is \c NULL), unassign any previous assignment, set next role to -+ * stopped, and update any existing actions scheduled for it. - * -- * \param[in,out] rsc Resource to assign -- * \param[in,out] chosen Node to assign \p rsc to -- * \param[in] force If true, assign to \p chosen even if unavailable -+ * \param[in,out] rsc Resource to assign -+ * \param[in,out] node Node to assign \p rsc to -+ * \param[in] force If true, assign to \p node even if unavailable - * -- * \return true if \p rsc could be assigned, otherwise false -+ * \return \c true if the assignment of \p rsc changed, or \c false otherwise - * - * \note Assigning a resource to the NULL node using this function is different - * from calling pcmk__unassign_resource(), in that it will also update any - * actions created for the resource. -+ * \note The \c resource_alloc_functions_t:assign() method is preferred, unless -+ * a resource should be assigned to the \c NULL node or every resource in -+ * a tree should be assigned to the same node. - */ - bool --pcmk__finalize_assignment(pe_resource_t *rsc, pe_node_t *chosen, bool force) -+pcmk__assign_resource(pe_resource_t *rsc, pe_node_t *node, bool force) - { -- pcmk__output_t *out = rsc->cluster->priv; -+ bool changed = false; -+ -+ CRM_ASSERT(rsc != NULL); - -- CRM_ASSERT(rsc->variant == pe_native); -- -- if (!force && (chosen != NULL)) { -- if ((chosen->weight < 0) -- // Allow the graph to assume that guest node connections will come up -- || (!pcmk__node_available(chosen, true, false) -- && !pe__is_guest_node(chosen))) { -- -- crm_debug("All nodes for resource %s are unavailable, unclean or " -- "shutting down (%s can%s run resources, with weight %d)", -- rsc->id, pe__node_name(chosen), -- (pcmk__node_available(chosen, true, false)? "" : "not"), -- chosen->weight); -- pe__set_next_role(rsc, RSC_ROLE_STOPPED, "node availability"); -- chosen = NULL; -+ if (rsc->children != NULL) { -+ for (GList *iter = rsc->children; iter != NULL; iter = iter->next) { -+ pe_resource_t *child_rsc = iter->data; -+ -+ changed |= pcmk__assign_resource(child_rsc, node, force); - } -+ return changed; - } - -+ // Assigning a primitive -+ -+ if (!force && (node != NULL) -+ && ((node->weight < 0) -+ // Allow graph to assume that guest node connections will come up -+ || (!pcmk__node_available(node, true, false) -+ && !pe__is_guest_node(node)))) { -+ -+ pe_rsc_debug(rsc, -+ "All nodes for resource %s are unavailable, unclean or " -+ "shutting down (%s can%s run resources, with score %s)", -+ rsc->id, pe__node_name(node), -+ (pcmk__node_available(node, true, false)? "" : "not"), -+ pcmk_readable_score(node->weight)); -+ pe__set_next_role(rsc, RSC_ROLE_STOPPED, "node availability"); -+ node = NULL; -+ } -+ -+ if (rsc->allocated_to != NULL) { -+ changed = !pe__same_node(rsc->allocated_to, node); -+ } else { -+ changed = (node != NULL); -+ } - pcmk__unassign_resource(rsc); - pe__clear_resource_flags(rsc, pe_rsc_provisional); - -- if (chosen == NULL) { -- crm_debug("Could not allocate a node for %s", rsc->id); -- pe__set_next_role(rsc, RSC_ROLE_STOPPED, "unable to allocate"); -+ if (node == NULL) { -+ char *rc_stopped = NULL; -+ -+ pe_rsc_debug(rsc, "Could not assign %s to a node", rsc->id); -+ pe__set_next_role(rsc, RSC_ROLE_STOPPED, "unable to assign"); - - for (GList *iter = rsc->actions; iter != NULL; iter = iter->next) { - pe_action_t *op = (pe_action_t *) iter->data; - -- crm_debug("Updating %s for allocation failure", op->uuid); -+ pe_rsc_debug(rsc, "Updating %s for %s assignment failure", -+ op->uuid, rsc->id); - - if (pcmk__str_eq(op->task, RSC_STOP, pcmk__str_casei)) { - pe__clear_action_flags(op, pe_action_optional); - - } else if (pcmk__str_eq(op->task, RSC_START, pcmk__str_casei)) { - pe__clear_action_flags(op, pe_action_runnable); -- //pe__set_resource_flags(rsc, pe_rsc_block); - - } else { - // Cancel recurring actions, unless for stopped state - const char *interval_ms_s = NULL; - const char *target_rc_s = NULL; -- char *rc_stopped = pcmk__itoa(PCMK_OCF_NOT_RUNNING); - - interval_ms_s = g_hash_table_lookup(op->meta, - XML_LRM_ATTR_INTERVAL_MS); - target_rc_s = g_hash_table_lookup(op->meta, - XML_ATTR_TE_TARGET_RC); -- if ((interval_ms_s != NULL) -- && !pcmk__str_eq(interval_ms_s, "0", pcmk__str_none) -+ if (rc_stopped == NULL) { -+ rc_stopped = pcmk__itoa(PCMK_OCF_NOT_RUNNING); -+ } -+ -+ if (!pcmk__str_eq(interval_ms_s, "0", pcmk__str_null_matches) - && !pcmk__str_eq(rc_stopped, target_rc_s, pcmk__str_none)) { -+ - pe__clear_action_flags(op, pe_action_runnable); - } -- free(rc_stopped); - } - } -- return false; -+ free(rc_stopped); -+ return changed; - } - -- crm_debug("Assigning %s to %s", rsc->id, pe__node_name(chosen)); -- rsc->allocated_to = pe__copy_node(chosen); -+ pe_rsc_debug(rsc, "Assigning %s to %s", rsc->id, pe__node_name(node)); -+ rsc->allocated_to = pe__copy_node(node); - -- chosen->details->allocated_rsc = g_list_prepend(chosen->details->allocated_rsc, -- rsc); -- chosen->details->num_resources++; -- chosen->count++; -- pcmk__consume_node_capacity(chosen->details->utilization, rsc); -+ node->details->allocated_rsc = g_list_prepend(node->details->allocated_rsc, -+ rsc); -+ node->details->num_resources++; -+ node->count++; -+ pcmk__consume_node_capacity(node->details->utilization, rsc); - - if (pcmk_is_set(rsc->cluster->flags, pe_flag_show_utilization)) { -- out->message(out, "resource-util", rsc, chosen, __func__); -- } -- return true; --} -- --/*! -- * \internal -- * \brief Assign a specified resource (of any variant) to a node -- * -- * Assign a specified resource and its children (if any) to a specified node, if -- * the node can run the resource (or unconditionally, if \p force is true). Mark -- * the resources as no longer provisional. If the resources can't be assigned -- * (or \p chosen is NULL), unassign any previous assignments, set next role to -- * stopped, and update any existing actions scheduled for them. -- * -- * \param[in,out] rsc Resource to assign -- * \param[in,out] chosen Node to assign \p rsc to -- * \param[in] force If true, assign to \p chosen even if unavailable -- * -- * \return true if \p rsc could be assigned, otherwise false -- * -- * \note Assigning a resource to the NULL node using this function is different -- * from calling pcmk__unassign_resource(), in that it will also update any -- * actions created for the resource. -- */ --bool --pcmk__assign_resource(pe_resource_t *rsc, pe_node_t *node, bool force) --{ -- bool changed = false; -- -- if (rsc->children == NULL) { -- if (rsc->allocated_to != NULL) { -- changed = true; -- } -- pcmk__finalize_assignment(rsc, node, force); -- -- } else { -- for (GList *iter = rsc->children; iter != NULL; iter = iter->next) { -- pe_resource_t *child_rsc = (pe_resource_t *) iter->data; -+ pcmk__output_t *out = rsc->cluster->priv; - -- changed |= pcmk__assign_resource(child_rsc, node, force); -- } -+ out->message(out, "resource-util", rsc, node, __func__); - } - return changed; - } - -From b01ecf9444e856227cd61c53f1c0106936eccd74 Mon Sep 17 00:00:00 2001 -From: Reid Wahl -Date: Mon, 10 Jul 2023 02:28:54 -0700 -Subject: [PATCH 06/19] Test: cts-cli: Update tests after defunctionization - -pcmk__finalize_assignment() -> pcmk__assign_resource() - -Signed-off-by: Reid Wahl ---- - cts/cli/regression.tools.exp | 42 ++++++++++++++++++------------------ - 1 file changed, 21 insertions(+), 21 deletions(-) - -diff --git a/cts/cli/regression.tools.exp b/cts/cli/regression.tools.exp -index a8e2236063c..506e9ba01b6 100644 ---- a/cts/cli/regression.tools.exp -+++ b/cts/cli/regression.tools.exp -@@ -5711,26 +5711,26 @@ Original: cluster02 capacity: - Original: httpd-bundle-0 capacity: - Original: httpd-bundle-1 capacity: - Original: httpd-bundle-2 capacity: --pcmk__finalize_assignment: ping:0 utilization on cluster02: --pcmk__finalize_assignment: ping:1 utilization on cluster01: --pcmk__finalize_assignment: Fencing utilization on cluster01: --pcmk__finalize_assignment: dummy utilization on cluster02: --pcmk__finalize_assignment: httpd-bundle-docker-0 utilization on cluster01: --pcmk__finalize_assignment: httpd-bundle-docker-1 utilization on cluster02: --pcmk__finalize_assignment: httpd-bundle-ip-192.168.122.131 utilization on cluster01: --pcmk__finalize_assignment: httpd-bundle-0 utilization on cluster01: --pcmk__finalize_assignment: httpd:0 utilization on httpd-bundle-0: --pcmk__finalize_assignment: httpd-bundle-ip-192.168.122.132 utilization on cluster02: --pcmk__finalize_assignment: httpd-bundle-1 utilization on cluster02: --pcmk__finalize_assignment: httpd:1 utilization on httpd-bundle-1: --pcmk__finalize_assignment: httpd-bundle-2 utilization on cluster01: --pcmk__finalize_assignment: httpd:2 utilization on httpd-bundle-2: --pcmk__finalize_assignment: Public-IP utilization on cluster02: --pcmk__finalize_assignment: Email utilization on cluster02: --pcmk__finalize_assignment: mysql-proxy:0 utilization on cluster02: --pcmk__finalize_assignment: mysql-proxy:1 utilization on cluster01: --pcmk__finalize_assignment: promotable-rsc:0 utilization on cluster02: --pcmk__finalize_assignment: promotable-rsc:1 utilization on cluster01: -+pcmk__assign_resource: ping:0 utilization on cluster02: -+pcmk__assign_resource: ping:1 utilization on cluster01: -+pcmk__assign_resource: Fencing utilization on cluster01: -+pcmk__assign_resource: dummy utilization on cluster02: -+pcmk__assign_resource: httpd-bundle-docker-0 utilization on cluster01: -+pcmk__assign_resource: httpd-bundle-docker-1 utilization on cluster02: -+pcmk__assign_resource: httpd-bundle-ip-192.168.122.131 utilization on cluster01: -+pcmk__assign_resource: httpd-bundle-0 utilization on cluster01: -+pcmk__assign_resource: httpd:0 utilization on httpd-bundle-0: -+pcmk__assign_resource: httpd-bundle-ip-192.168.122.132 utilization on cluster02: -+pcmk__assign_resource: httpd-bundle-1 utilization on cluster02: -+pcmk__assign_resource: httpd:1 utilization on httpd-bundle-1: -+pcmk__assign_resource: httpd-bundle-2 utilization on cluster01: -+pcmk__assign_resource: httpd:2 utilization on httpd-bundle-2: -+pcmk__assign_resource: Public-IP utilization on cluster02: -+pcmk__assign_resource: Email utilization on cluster02: -+pcmk__assign_resource: mysql-proxy:0 utilization on cluster02: -+pcmk__assign_resource: mysql-proxy:1 utilization on cluster01: -+pcmk__assign_resource: promotable-rsc:0 utilization on cluster02: -+pcmk__assign_resource: promotable-rsc:1 utilization on cluster01: - Remaining: cluster01 capacity: - Remaining: cluster02 capacity: - Remaining: httpd-bundle-0 capacity: -@@ -5961,7 +5961,7 @@ Transition Summary: - * Move Public-IP ( cluster02 -> cluster01 ) - * Move Email ( cluster02 -> cluster01 ) - * Stop mysql-proxy:0 ( cluster02 ) due to node availability -- * Stop promotable-rsc:0 ( Promoted cluster02 ) due to node availability -+ * Stop promotable-rsc:0 ( Promoted cluster02 ) due to node availability - - Executing Cluster Transition: - * Pseudo action: httpd-bundle-1_stop_0 - -From 0ad4a3c8404d57e2026e41a234a9b8a0a237b2bd Mon Sep 17 00:00:00 2001 -From: Reid Wahl -Date: Tue, 20 Jun 2023 23:22:54 -0700 -Subject: [PATCH 07/19] Test: scheduler: Clone instances should not shuffle - unnecessarily - -In some cases, clone instances may be shuffled when a new instance is -scheduled to start or promote. This can cause instances to be stopped -and started unnecessarily. - -Here we add tests for three types of clones: -* "Bare" clones of primitives -* Clones of groups -* Clones of primitives within bundles (clone is bundle child resource) - -For each clone type, we add four tests. In each test, no clone instance -is running on node 1, and a new instance should be started and possibly -promoted there. -* No constraints or stickiness -* Location constraint preferring node 1 -* Promotable clone where node 1 has the highest promotion score -* Promotable clone where node 1 does not have the highest promotion - score - -The following tests are currently incorrect: -* clone-no-recover-shuffle-4 (shuffling) -* clone-no-recover-shuffle-5 (all instances of an anonymous clone move - to one node) -* clone-no-recover-shuffle-6 (shuffling) -* clone-no-recover-shuffle-7 (shuffling) - -Ref T489 - -Signed-off-by: Reid Wahl ---- - cts/cts-scheduler.in | 33 ++ - .../dot/clone-recover-no-shuffle-1.dot | 10 + - .../dot/clone-recover-no-shuffle-10.dot | 10 + - .../dot/clone-recover-no-shuffle-11.dot | 21 + - .../dot/clone-recover-no-shuffle-12.dot | 35 ++ - .../dot/clone-recover-no-shuffle-2.dot | 21 + - .../dot/clone-recover-no-shuffle-3.dot | 32 ++ - .../dot/clone-recover-no-shuffle-4.dot | 23 + - .../dot/clone-recover-no-shuffle-5.dot | 80 +++ - .../dot/clone-recover-no-shuffle-6.dot | 97 ++++ - .../dot/clone-recover-no-shuffle-7.dot | 45 ++ - .../dot/clone-recover-no-shuffle-8.dot | 63 +++ - .../dot/clone-recover-no-shuffle-9.dot | 69 +++ - .../exp/clone-recover-no-shuffle-1.exp | 51 ++ - .../exp/clone-recover-no-shuffle-10.exp | 51 ++ - .../exp/clone-recover-no-shuffle-11.exp | 110 ++++ - .../exp/clone-recover-no-shuffle-12.exp | 187 +++++++ - .../exp/clone-recover-no-shuffle-2.exp | 110 ++++ - .../exp/clone-recover-no-shuffle-3.exp | 171 ++++++ - .../exp/clone-recover-no-shuffle-4.exp | 123 +++++ - .../exp/clone-recover-no-shuffle-5.exp | 452 ++++++++++++++++ - .../exp/clone-recover-no-shuffle-6.exp | 507 ++++++++++++++++++ - .../exp/clone-recover-no-shuffle-7.exp | 240 +++++++++ - .../exp/clone-recover-no-shuffle-8.exp | 338 ++++++++++++ - .../exp/clone-recover-no-shuffle-9.exp | 364 +++++++++++++ - .../scores/clone-recover-no-shuffle-1.scores | 25 + - .../scores/clone-recover-no-shuffle-10.scores | 31 ++ - .../scores/clone-recover-no-shuffle-11.scores | 82 +++ - .../scores/clone-recover-no-shuffle-12.scores | 67 +++ - .../scores/clone-recover-no-shuffle-2.scores | 79 +++ - .../scores/clone-recover-no-shuffle-3.scores | 64 +++ - .../scores/clone-recover-no-shuffle-4.scores | 31 ++ - .../scores/clone-recover-no-shuffle-5.scores | 79 +++ - .../scores/clone-recover-no-shuffle-6.scores | 70 +++ - .../scores/clone-recover-no-shuffle-7.scores | 34 ++ - .../scores/clone-recover-no-shuffle-8.scores | 82 +++ - .../scores/clone-recover-no-shuffle-9.scores | 67 +++ - .../clone-recover-no-shuffle-1.summary | 29 + - .../clone-recover-no-shuffle-10.summary | 29 + - .../clone-recover-no-shuffle-11.summary | 34 ++ - .../clone-recover-no-shuffle-12.summary | 43 ++ - .../clone-recover-no-shuffle-2.summary | 32 ++ - .../clone-recover-no-shuffle-3.summary | 42 ++ - .../clone-recover-no-shuffle-4.summary | 35 ++ - .../clone-recover-no-shuffle-5.summary | 59 ++ - .../clone-recover-no-shuffle-6.summary | 68 +++ - .../clone-recover-no-shuffle-7.summary | 44 ++ - .../clone-recover-no-shuffle-8.summary | 52 ++ - .../clone-recover-no-shuffle-9.summary | 56 ++ - .../xml/clone-recover-no-shuffle-1.xml | 113 ++++ - .../xml/clone-recover-no-shuffle-10.xml | 120 +++++ - .../xml/clone-recover-no-shuffle-11.xml | 153 ++++++ - .../xml/clone-recover-no-shuffle-12.xml | 186 +++++++ - .../xml/clone-recover-no-shuffle-2.xml | 141 +++++ - .../xml/clone-recover-no-shuffle-3.xml | 180 +++++++ - .../xml/clone-recover-no-shuffle-4.xml | 120 +++++ - .../xml/clone-recover-no-shuffle-5.xml | 148 +++++ - .../xml/clone-recover-no-shuffle-6.xml | 187 +++++++ - .../xml/clone-recover-no-shuffle-7.xml | 125 +++++ - .../xml/clone-recover-no-shuffle-8.xml | 153 ++++++ - .../xml/clone-recover-no-shuffle-9.xml | 186 +++++++ - 61 files changed, 6289 insertions(+) - create mode 100644 cts/scheduler/dot/clone-recover-no-shuffle-1.dot - create mode 100644 cts/scheduler/dot/clone-recover-no-shuffle-10.dot - create mode 100644 cts/scheduler/dot/clone-recover-no-shuffle-11.dot - create mode 100644 cts/scheduler/dot/clone-recover-no-shuffle-12.dot - create mode 100644 cts/scheduler/dot/clone-recover-no-shuffle-2.dot - create mode 100644 cts/scheduler/dot/clone-recover-no-shuffle-3.dot - create mode 100644 cts/scheduler/dot/clone-recover-no-shuffle-4.dot - create mode 100644 cts/scheduler/dot/clone-recover-no-shuffle-5.dot - create mode 100644 cts/scheduler/dot/clone-recover-no-shuffle-6.dot - create mode 100644 cts/scheduler/dot/clone-recover-no-shuffle-7.dot - create mode 100644 cts/scheduler/dot/clone-recover-no-shuffle-8.dot - create mode 100644 cts/scheduler/dot/clone-recover-no-shuffle-9.dot - create mode 100644 cts/scheduler/exp/clone-recover-no-shuffle-1.exp - create mode 100644 cts/scheduler/exp/clone-recover-no-shuffle-10.exp - create mode 100644 cts/scheduler/exp/clone-recover-no-shuffle-11.exp - create mode 100644 cts/scheduler/exp/clone-recover-no-shuffle-12.exp - create mode 100644 cts/scheduler/exp/clone-recover-no-shuffle-2.exp - create mode 100644 cts/scheduler/exp/clone-recover-no-shuffle-3.exp - create mode 100644 cts/scheduler/exp/clone-recover-no-shuffle-4.exp - create mode 100644 cts/scheduler/exp/clone-recover-no-shuffle-5.exp - create mode 100644 cts/scheduler/exp/clone-recover-no-shuffle-6.exp - create mode 100644 cts/scheduler/exp/clone-recover-no-shuffle-7.exp - create mode 100644 cts/scheduler/exp/clone-recover-no-shuffle-8.exp - create mode 100644 cts/scheduler/exp/clone-recover-no-shuffle-9.exp - create mode 100644 cts/scheduler/scores/clone-recover-no-shuffle-1.scores - create mode 100644 cts/scheduler/scores/clone-recover-no-shuffle-10.scores - create mode 100644 cts/scheduler/scores/clone-recover-no-shuffle-11.scores - create mode 100644 cts/scheduler/scores/clone-recover-no-shuffle-12.scores - create mode 100644 cts/scheduler/scores/clone-recover-no-shuffle-2.scores - create mode 100644 cts/scheduler/scores/clone-recover-no-shuffle-3.scores - create mode 100644 cts/scheduler/scores/clone-recover-no-shuffle-4.scores - create mode 100644 cts/scheduler/scores/clone-recover-no-shuffle-5.scores - create mode 100644 cts/scheduler/scores/clone-recover-no-shuffle-6.scores - create mode 100644 cts/scheduler/scores/clone-recover-no-shuffle-7.scores - create mode 100644 cts/scheduler/scores/clone-recover-no-shuffle-8.scores - create mode 100644 cts/scheduler/scores/clone-recover-no-shuffle-9.scores - create mode 100644 cts/scheduler/summary/clone-recover-no-shuffle-1.summary - create mode 100644 cts/scheduler/summary/clone-recover-no-shuffle-10.summary - create mode 100644 cts/scheduler/summary/clone-recover-no-shuffle-11.summary - create mode 100644 cts/scheduler/summary/clone-recover-no-shuffle-12.summary - create mode 100644 cts/scheduler/summary/clone-recover-no-shuffle-2.summary - create mode 100644 cts/scheduler/summary/clone-recover-no-shuffle-3.summary - create mode 100644 cts/scheduler/summary/clone-recover-no-shuffle-4.summary - create mode 100644 cts/scheduler/summary/clone-recover-no-shuffle-5.summary - create mode 100644 cts/scheduler/summary/clone-recover-no-shuffle-6.summary - create mode 100644 cts/scheduler/summary/clone-recover-no-shuffle-7.summary - create mode 100644 cts/scheduler/summary/clone-recover-no-shuffle-8.summary - create mode 100644 cts/scheduler/summary/clone-recover-no-shuffle-9.summary - create mode 100644 cts/scheduler/xml/clone-recover-no-shuffle-1.xml - create mode 100644 cts/scheduler/xml/clone-recover-no-shuffle-10.xml - create mode 100644 cts/scheduler/xml/clone-recover-no-shuffle-11.xml - create mode 100644 cts/scheduler/xml/clone-recover-no-shuffle-12.xml - create mode 100644 cts/scheduler/xml/clone-recover-no-shuffle-2.xml - create mode 100644 cts/scheduler/xml/clone-recover-no-shuffle-3.xml - create mode 100644 cts/scheduler/xml/clone-recover-no-shuffle-4.xml - create mode 100644 cts/scheduler/xml/clone-recover-no-shuffle-5.xml - create mode 100644 cts/scheduler/xml/clone-recover-no-shuffle-6.xml - create mode 100644 cts/scheduler/xml/clone-recover-no-shuffle-7.xml - create mode 100644 cts/scheduler/xml/clone-recover-no-shuffle-8.xml - create mode 100644 cts/scheduler/xml/clone-recover-no-shuffle-9.xml - -diff --git a/cts/cts-scheduler.in b/cts/cts-scheduler.in -index ee0cb7b4722..4ff035c23a3 100644 ---- a/cts/cts-scheduler.in -+++ b/cts/cts-scheduler.in -@@ -441,6 +441,39 @@ TESTS = [ - [ "cloned-group", "Make sure only the correct number of cloned groups are started" ], - [ "cloned-group-stop", "Ensure stopping qpidd also stops glance and cinder" ], - [ "clone-no-shuffle", "Don't prioritize allocation of instances that must be moved" ], -+ [ "clone-recover-no-shuffle-1", -+ "Don't shuffle instances when starting a new primitive instance" ], -+ [ "clone-recover-no-shuffle-2", -+ "Don't shuffle instances when starting a new group instance" ], -+ [ "clone-recover-no-shuffle-3", -+ "Don't shuffle instances when starting a new bundle instance" ], -+ [ "clone-recover-no-shuffle-4", -+ "Don't shuffle instances when starting a new primitive instance with " -+ "location preference "], -+ [ "clone-recover-no-shuffle-5", -+ "Don't shuffle instances when starting a new group instance with " -+ "location preference" ], -+ [ "clone-recover-no-shuffle-6", -+ "Don't shuffle instances when starting a new bundle instance with " -+ "location preference" ], -+ [ "clone-recover-no-shuffle-7", -+ "Don't shuffle instances when starting a new primitive instance that " -+ "will be promoted" ], -+ [ "clone-recover-no-shuffle-8", -+ "Don't shuffle instances when starting a new group instance that " -+ "will be promoted " ], -+ [ "clone-recover-no-shuffle-9", -+ "Don't shuffle instances when starting a new bundle instance that " -+ "will be promoted " ], -+ [ "clone-recover-no-shuffle-10", -+ "Don't shuffle instances when starting a new primitive instance that " -+ "won't be promoted" ], -+ [ "clone-recover-no-shuffle-11", -+ "Don't shuffle instances when starting a new group instance that " -+ "won't be promoted " ], -+ [ "clone-recover-no-shuffle-12", -+ "Don't shuffle instances when starting a new bundle instance that " -+ "won't be promoted " ], - [ "clone-max-zero", "Orphan processing with clone-max=0" ], - [ "clone-anon-dup", - "Bug LF#2087 - Correctly parse the state of anonymous clones that are active more than once per node" ], -diff --git a/cts/scheduler/dot/clone-recover-no-shuffle-1.dot b/cts/scheduler/dot/clone-recover-no-shuffle-1.dot -new file mode 100644 -index 00000000000..287d82d3806 ---- /dev/null -+++ b/cts/scheduler/dot/clone-recover-no-shuffle-1.dot -@@ -0,0 +1,10 @@ -+ digraph "g" { -+"dummy-clone_running_0" [ style=bold color="green" fontcolor="orange"] -+"dummy-clone_start_0" -> "dummy-clone_running_0" [ style = bold] -+"dummy-clone_start_0" -> "dummy:2_start_0 node1" [ style = bold] -+"dummy-clone_start_0" [ style=bold color="green" fontcolor="orange"] -+"dummy:2_monitor_10000 node1" [ style=bold color="green" fontcolor="black"] -+"dummy:2_start_0 node1" -> "dummy-clone_running_0" [ style = bold] -+"dummy:2_start_0 node1" -> "dummy:2_monitor_10000 node1" [ style = bold] -+"dummy:2_start_0 node1" [ style=bold color="green" fontcolor="black"] -+} -diff --git a/cts/scheduler/dot/clone-recover-no-shuffle-10.dot b/cts/scheduler/dot/clone-recover-no-shuffle-10.dot -new file mode 100644 -index 00000000000..1e1840966fa ---- /dev/null -+++ b/cts/scheduler/dot/clone-recover-no-shuffle-10.dot -@@ -0,0 +1,10 @@ -+ digraph "g" { -+"dummy-clone_running_0" [ style=bold color="green" fontcolor="orange"] -+"dummy-clone_start_0" -> "dummy-clone_running_0" [ style = bold] -+"dummy-clone_start_0" -> "dummy:2_start_0 node1" [ style = bold] -+"dummy-clone_start_0" [ style=bold color="green" fontcolor="orange"] -+"dummy:2_monitor_11000 node1" [ style=bold color="green" fontcolor="black"] -+"dummy:2_start_0 node1" -> "dummy-clone_running_0" [ style = bold] -+"dummy:2_start_0 node1" -> "dummy:2_monitor_11000 node1" [ style = bold] -+"dummy:2_start_0 node1" [ style=bold color="green" fontcolor="black"] -+} -diff --git a/cts/scheduler/dot/clone-recover-no-shuffle-11.dot b/cts/scheduler/dot/clone-recover-no-shuffle-11.dot -new file mode 100644 -index 00000000000..2b08a594561 ---- /dev/null -+++ b/cts/scheduler/dot/clone-recover-no-shuffle-11.dot -@@ -0,0 +1,21 @@ -+ digraph "g" { -+"grp-clone_running_0" [ style=bold color="green" fontcolor="orange"] -+"grp-clone_start_0" -> "grp-clone_running_0" [ style = bold] -+"grp-clone_start_0" -> "grp:2_start_0" [ style = bold] -+"grp-clone_start_0" [ style=bold color="green" fontcolor="orange"] -+"grp:2_running_0" -> "grp-clone_running_0" [ style = bold] -+"grp:2_running_0" [ style=bold color="green" fontcolor="orange"] -+"grp:2_start_0" -> "grp:2_running_0" [ style = bold] -+"grp:2_start_0" -> "rsc1:2_start_0 node1" [ style = bold] -+"grp:2_start_0" -> "rsc2:2_start_0 node1" [ style = bold] -+"grp:2_start_0" [ style=bold color="green" fontcolor="orange"] -+"rsc1:2_monitor_11000 node1" [ style=bold color="green" fontcolor="black"] -+"rsc1:2_start_0 node1" -> "grp:2_running_0" [ style = bold] -+"rsc1:2_start_0 node1" -> "rsc1:2_monitor_11000 node1" [ style = bold] -+"rsc1:2_start_0 node1" -> "rsc2:2_start_0 node1" [ style = bold] -+"rsc1:2_start_0 node1" [ style=bold color="green" fontcolor="black"] -+"rsc2:2_monitor_11000 node1" [ style=bold color="green" fontcolor="black"] -+"rsc2:2_start_0 node1" -> "grp:2_running_0" [ style = bold] -+"rsc2:2_start_0 node1" -> "rsc2:2_monitor_11000 node1" [ style = bold] -+"rsc2:2_start_0 node1" [ style=bold color="green" fontcolor="black"] -+} -diff --git a/cts/scheduler/dot/clone-recover-no-shuffle-12.dot b/cts/scheduler/dot/clone-recover-no-shuffle-12.dot -new file mode 100644 -index 00000000000..ebc1dc6a815 ---- /dev/null -+++ b/cts/scheduler/dot/clone-recover-no-shuffle-12.dot -@@ -0,0 +1,35 @@ -+ digraph "g" { -+"base-bundle-2_monitor_0 node1" -> "base-bundle-2_start_0 node1" [ style = bold] -+"base-bundle-2_monitor_0 node1" [ style=bold color="green" fontcolor="black"] -+"base-bundle-2_monitor_0 node2" -> "base-bundle-2_start_0 node1" [ style = bold] -+"base-bundle-2_monitor_0 node2" [ style=bold color="green" fontcolor="black"] -+"base-bundle-2_monitor_0 node3" -> "base-bundle-2_start_0 node1" [ style = bold] -+"base-bundle-2_monitor_0 node3" [ style=bold color="green" fontcolor="black"] -+"base-bundle-2_monitor_30000 node1" [ style=bold color="green" fontcolor="black"] -+"base-bundle-2_start_0 node1" -> "base-bundle-2_monitor_30000 node1" [ style = bold] -+"base-bundle-2_start_0 node1" -> "base:2_monitor_16000 base-bundle-2" [ style = bold] -+"base-bundle-2_start_0 node1" -> "base:2_start_0 base-bundle-2" [ style = bold] -+"base-bundle-2_start_0 node1" [ style=bold color="green" fontcolor="black"] -+"base-bundle-clone_running_0" -> "base-bundle_running_0" [ style = bold] -+"base-bundle-clone_running_0" [ style=bold color="green" fontcolor="orange"] -+"base-bundle-clone_start_0" -> "base-bundle-clone_running_0" [ style = bold] -+"base-bundle-clone_start_0" -> "base:2_start_0 base-bundle-2" [ style = bold] -+"base-bundle-clone_start_0" [ style=bold color="green" fontcolor="orange"] -+"base-bundle-podman-2_monitor_60000 node1" [ style=bold color="green" fontcolor="black"] -+"base-bundle-podman-2_start_0 node1" -> "base-bundle-2_monitor_0 node1" [ style = bold] -+"base-bundle-podman-2_start_0 node1" -> "base-bundle-2_monitor_0 node2" [ style = bold] -+"base-bundle-podman-2_start_0 node1" -> "base-bundle-2_monitor_0 node3" [ style = bold] -+"base-bundle-podman-2_start_0 node1" -> "base-bundle-2_start_0 node1" [ style = bold] -+"base-bundle-podman-2_start_0 node1" -> "base-bundle-podman-2_monitor_60000 node1" [ style = bold] -+"base-bundle-podman-2_start_0 node1" -> "base-bundle_running_0" [ style = bold] -+"base-bundle-podman-2_start_0 node1" -> "base:2_start_0 base-bundle-2" [ style = bold] -+"base-bundle-podman-2_start_0 node1" [ style=bold color="green" fontcolor="black"] -+"base-bundle_running_0" [ style=bold color="green" fontcolor="orange"] -+"base-bundle_start_0" -> "base-bundle-clone_start_0" [ style = bold] -+"base-bundle_start_0" -> "base-bundle-podman-2_start_0 node1" [ style = bold] -+"base-bundle_start_0" [ style=bold color="green" fontcolor="orange"] -+"base:2_monitor_16000 base-bundle-2" [ style=bold color="green" fontcolor="black"] -+"base:2_start_0 base-bundle-2" -> "base-bundle-clone_running_0" [ style = bold] -+"base:2_start_0 base-bundle-2" -> "base:2_monitor_16000 base-bundle-2" [ style = bold] -+"base:2_start_0 base-bundle-2" [ style=bold color="green" fontcolor="black"] -+} -diff --git a/cts/scheduler/dot/clone-recover-no-shuffle-2.dot b/cts/scheduler/dot/clone-recover-no-shuffle-2.dot -new file mode 100644 -index 00000000000..d3bdf04baa9 ---- /dev/null -+++ b/cts/scheduler/dot/clone-recover-no-shuffle-2.dot -@@ -0,0 +1,21 @@ -+ digraph "g" { -+"grp-clone_running_0" [ style=bold color="green" fontcolor="orange"] -+"grp-clone_start_0" -> "grp-clone_running_0" [ style = bold] -+"grp-clone_start_0" -> "grp:2_start_0" [ style = bold] -+"grp-clone_start_0" [ style=bold color="green" fontcolor="orange"] -+"grp:2_running_0" -> "grp-clone_running_0" [ style = bold] -+"grp:2_running_0" [ style=bold color="green" fontcolor="orange"] -+"grp:2_start_0" -> "grp:2_running_0" [ style = bold] -+"grp:2_start_0" -> "rsc1:2_start_0 node1" [ style = bold] -+"grp:2_start_0" -> "rsc2:2_start_0 node1" [ style = bold] -+"grp:2_start_0" [ style=bold color="green" fontcolor="orange"] -+"rsc1:2_monitor_10000 node1" [ style=bold color="green" fontcolor="black"] -+"rsc1:2_start_0 node1" -> "grp:2_running_0" [ style = bold] -+"rsc1:2_start_0 node1" -> "rsc1:2_monitor_10000 node1" [ style = bold] -+"rsc1:2_start_0 node1" -> "rsc2:2_start_0 node1" [ style = bold] -+"rsc1:2_start_0 node1" [ style=bold color="green" fontcolor="black"] -+"rsc2:2_monitor_10000 node1" [ style=bold color="green" fontcolor="black"] -+"rsc2:2_start_0 node1" -> "grp:2_running_0" [ style = bold] -+"rsc2:2_start_0 node1" -> "rsc2:2_monitor_10000 node1" [ style = bold] -+"rsc2:2_start_0 node1" [ style=bold color="green" fontcolor="black"] -+} -diff --git a/cts/scheduler/dot/clone-recover-no-shuffle-3.dot b/cts/scheduler/dot/clone-recover-no-shuffle-3.dot -new file mode 100644 -index 00000000000..f60fd2cc04e ---- /dev/null -+++ b/cts/scheduler/dot/clone-recover-no-shuffle-3.dot -@@ -0,0 +1,32 @@ -+ digraph "g" { -+"base-bundle-2_monitor_0 node1" -> "base-bundle-2_start_0 node1" [ style = bold] -+"base-bundle-2_monitor_0 node1" [ style=bold color="green" fontcolor="black"] -+"base-bundle-2_monitor_0 node2" -> "base-bundle-2_start_0 node1" [ style = bold] -+"base-bundle-2_monitor_0 node2" [ style=bold color="green" fontcolor="black"] -+"base-bundle-2_monitor_0 node3" -> "base-bundle-2_start_0 node1" [ style = bold] -+"base-bundle-2_monitor_0 node3" [ style=bold color="green" fontcolor="black"] -+"base-bundle-2_monitor_30000 node1" [ style=bold color="green" fontcolor="black"] -+"base-bundle-2_start_0 node1" -> "base-bundle-2_monitor_30000 node1" [ style = bold] -+"base-bundle-2_start_0 node1" -> "base:2_start_0 base-bundle-2" [ style = bold] -+"base-bundle-2_start_0 node1" [ style=bold color="green" fontcolor="black"] -+"base-bundle-clone_running_0" -> "base-bundle_running_0" [ style = bold] -+"base-bundle-clone_running_0" [ style=bold color="green" fontcolor="orange"] -+"base-bundle-clone_start_0" -> "base-bundle-clone_running_0" [ style = bold] -+"base-bundle-clone_start_0" -> "base:2_start_0 base-bundle-2" [ style = bold] -+"base-bundle-clone_start_0" [ style=bold color="green" fontcolor="orange"] -+"base-bundle-podman-2_monitor_60000 node1" [ style=bold color="green" fontcolor="black"] -+"base-bundle-podman-2_start_0 node1" -> "base-bundle-2_monitor_0 node1" [ style = bold] -+"base-bundle-podman-2_start_0 node1" -> "base-bundle-2_monitor_0 node2" [ style = bold] -+"base-bundle-podman-2_start_0 node1" -> "base-bundle-2_monitor_0 node3" [ style = bold] -+"base-bundle-podman-2_start_0 node1" -> "base-bundle-2_start_0 node1" [ style = bold] -+"base-bundle-podman-2_start_0 node1" -> "base-bundle-podman-2_monitor_60000 node1" [ style = bold] -+"base-bundle-podman-2_start_0 node1" -> "base-bundle_running_0" [ style = bold] -+"base-bundle-podman-2_start_0 node1" -> "base:2_start_0 base-bundle-2" [ style = bold] -+"base-bundle-podman-2_start_0 node1" [ style=bold color="green" fontcolor="black"] -+"base-bundle_running_0" [ style=bold color="green" fontcolor="orange"] -+"base-bundle_start_0" -> "base-bundle-clone_start_0" [ style = bold] -+"base-bundle_start_0" -> "base-bundle-podman-2_start_0 node1" [ style = bold] -+"base-bundle_start_0" [ style=bold color="green" fontcolor="orange"] -+"base:2_start_0 base-bundle-2" -> "base-bundle-clone_running_0" [ style = bold] -+"base:2_start_0 base-bundle-2" [ style=bold color="green" fontcolor="black"] -+} -diff --git a/cts/scheduler/dot/clone-recover-no-shuffle-4.dot b/cts/scheduler/dot/clone-recover-no-shuffle-4.dot -new file mode 100644 -index 00000000000..fd002f28fcf ---- /dev/null -+++ b/cts/scheduler/dot/clone-recover-no-shuffle-4.dot -@@ -0,0 +1,23 @@ -+ digraph "g" { -+"dummy-clone_running_0" [ style=bold color="green" fontcolor="orange"] -+"dummy-clone_start_0" -> "dummy-clone_running_0" [ style = bold] -+"dummy-clone_start_0" -> "dummy:2_start_0 node2" [ style = bold] -+"dummy-clone_start_0" -> "dummy_start_0 node1" [ style = bold] -+"dummy-clone_start_0" [ style=bold color="green" fontcolor="orange"] -+"dummy-clone_stop_0" -> "dummy-clone_stopped_0" [ style = bold] -+"dummy-clone_stop_0" -> "dummy_stop_0 node2" [ style = bold] -+"dummy-clone_stop_0" [ style=bold color="green" fontcolor="orange"] -+"dummy-clone_stopped_0" -> "dummy-clone_start_0" [ style = bold] -+"dummy-clone_stopped_0" [ style=bold color="green" fontcolor="orange"] -+"dummy:2_monitor_10000 node2" [ style=bold color="green" fontcolor="black"] -+"dummy:2_start_0 node2" -> "dummy-clone_running_0" [ style = bold] -+"dummy:2_start_0 node2" -> "dummy:2_monitor_10000 node2" [ style = bold] -+"dummy:2_start_0 node2" [ style=bold color="green" fontcolor="black"] -+"dummy_monitor_10000 node1" [ style=bold color="green" fontcolor="black"] -+"dummy_start_0 node1" -> "dummy-clone_running_0" [ style = bold] -+"dummy_start_0 node1" -> "dummy_monitor_10000 node1" [ style = bold] -+"dummy_start_0 node1" [ style=bold color="green" fontcolor="black"] -+"dummy_stop_0 node2" -> "dummy-clone_stopped_0" [ style = bold] -+"dummy_stop_0 node2" -> "dummy_start_0 node1" [ style = bold] -+"dummy_stop_0 node2" [ style=bold color="green" fontcolor="black"] -+} -diff --git a/cts/scheduler/dot/clone-recover-no-shuffle-5.dot b/cts/scheduler/dot/clone-recover-no-shuffle-5.dot -new file mode 100644 -index 00000000000..7219ee5a6d3 ---- /dev/null -+++ b/cts/scheduler/dot/clone-recover-no-shuffle-5.dot -@@ -0,0 +1,80 @@ -+ digraph "g" { -+"grp-clone_running_0" [ style=bold color="green" fontcolor="orange"] -+"grp-clone_start_0" -> "grp-clone_running_0" [ style = bold] -+"grp-clone_start_0" -> "grp:0_start_0" [ style = bold] -+"grp-clone_start_0" -> "grp:1_start_0" [ style = bold] -+"grp-clone_start_0" -> "grp:2_start_0" [ style = bold] -+"grp-clone_start_0" [ style=bold color="green" fontcolor="orange"] -+"grp-clone_stop_0" -> "grp-clone_stopped_0" [ style = bold] -+"grp-clone_stop_0" -> "grp:0_stop_0" [ style = bold] -+"grp-clone_stop_0" -> "grp:1_stop_0" [ style = bold] -+"grp-clone_stop_0" [ style=bold color="green" fontcolor="orange"] -+"grp-clone_stopped_0" -> "grp-clone_start_0" [ style = bold] -+"grp-clone_stopped_0" [ style=bold color="green" fontcolor="orange"] -+"grp:0_running_0" -> "grp-clone_running_0" [ style = bold] -+"grp:0_running_0" [ style=bold color="green" fontcolor="orange"] -+"grp:0_start_0" -> "grp:0_running_0" [ style = bold] -+"grp:0_start_0" -> "rsc1_start_0 node1" [ style = bold] -+"grp:0_start_0" -> "rsc2_start_0 node1" [ style = bold] -+"grp:0_start_0" [ style=bold color="green" fontcolor="orange"] -+"grp:0_stop_0" -> "grp:0_stopped_0" [ style = bold] -+"grp:0_stop_0" -> "rsc1_stop_0 node2" [ style = bold] -+"grp:0_stop_0" -> "rsc2_stop_0 node2" [ style = bold] -+"grp:0_stop_0" [ style=bold color="green" fontcolor="orange"] -+"grp:0_stopped_0" -> "grp-clone_stopped_0" [ style = bold] -+"grp:0_stopped_0" -> "grp:0_start_0" [ style = bold] -+"grp:0_stopped_0" [ style=bold color="green" fontcolor="orange"] -+"grp:1_running_0" -> "grp-clone_running_0" [ style = bold] -+"grp:1_running_0" [ style=bold color="green" fontcolor="orange"] -+"grp:1_start_0" -> "grp:1_running_0" [ style = bold] -+"grp:1_start_0" -> "rsc1_start_0 node1" [ style = bold] -+"grp:1_start_0" -> "rsc2_start_0 node1" [ style = bold] -+"grp:1_start_0" [ style=bold color="green" fontcolor="orange"] -+"grp:1_stop_0" -> "grp:1_stopped_0" [ style = bold] -+"grp:1_stop_0" -> "rsc1_stop_0 node3" [ style = bold] -+"grp:1_stop_0" -> "rsc2_stop_0 node3" [ style = bold] -+"grp:1_stop_0" [ style=bold color="green" fontcolor="orange"] -+"grp:1_stopped_0" -> "grp-clone_stopped_0" [ style = bold] -+"grp:1_stopped_0" -> "grp:1_start_0" [ style = bold] -+"grp:1_stopped_0" [ style=bold color="green" fontcolor="orange"] -+"grp:2_running_0" -> "grp-clone_running_0" [ style = bold] -+"grp:2_running_0" [ style=bold color="green" fontcolor="orange"] -+"grp:2_start_0" -> "grp:2_running_0" [ style = bold] -+"grp:2_start_0" -> "rsc1:2_start_0 node1" [ style = bold] -+"grp:2_start_0" -> "rsc2:2_start_0 node1" [ style = bold] -+"grp:2_start_0" [ style=bold color="green" fontcolor="orange"] -+"rsc1:2_monitor_10000 node1" [ style=bold color="green" fontcolor="black"] -+"rsc1:2_start_0 node1" -> "grp:2_running_0" [ style = bold] -+"rsc1:2_start_0 node1" -> "rsc1:2_monitor_10000 node1" [ style = bold] -+"rsc1:2_start_0 node1" -> "rsc2:2_start_0 node1" [ style = bold] -+"rsc1:2_start_0 node1" [ style=bold color="green" fontcolor="black"] -+"rsc1_monitor_10000 node1" [ style=bold color="green" fontcolor="black"] -+"rsc1_start_0 node1" -> "grp:0_running_0" [ style = bold] -+"rsc1_start_0 node1" -> "grp:1_running_0" [ style = bold] -+"rsc1_start_0 node1" -> "rsc1_monitor_10000 node1" [ style = bold] -+"rsc1_start_0 node1" -> "rsc2_start_0 node1" [ style = bold] -+"rsc1_start_0 node1" [ style=bold color="green" fontcolor="black"] -+"rsc1_stop_0 node2" -> "grp:0_stopped_0" [ style = bold] -+"rsc1_stop_0 node2" -> "rsc1_start_0 node1" [ style = bold] -+"rsc1_stop_0 node2" [ style=bold color="green" fontcolor="black"] -+"rsc1_stop_0 node3" -> "grp:1_stopped_0" [ style = bold] -+"rsc1_stop_0 node3" -> "rsc1_start_0 node1" [ style = bold] -+"rsc1_stop_0 node3" [ style=bold color="green" fontcolor="black"] -+"rsc2:2_monitor_10000 node1" [ style=bold color="green" fontcolor="black"] -+"rsc2:2_start_0 node1" -> "grp:2_running_0" [ style = bold] -+"rsc2:2_start_0 node1" -> "rsc2:2_monitor_10000 node1" [ style = bold] -+"rsc2:2_start_0 node1" [ style=bold color="green" fontcolor="black"] -+"rsc2_monitor_10000 node1" [ style=bold color="green" fontcolor="black"] -+"rsc2_start_0 node1" -> "grp:0_running_0" [ style = bold] -+"rsc2_start_0 node1" -> "grp:1_running_0" [ style = bold] -+"rsc2_start_0 node1" -> "rsc2_monitor_10000 node1" [ style = bold] -+"rsc2_start_0 node1" [ style=bold color="green" fontcolor="black"] -+"rsc2_stop_0 node2" -> "grp:0_stopped_0" [ style = bold] -+"rsc2_stop_0 node2" -> "rsc1_stop_0 node2" [ style = bold] -+"rsc2_stop_0 node2" -> "rsc2_start_0 node1" [ style = bold] -+"rsc2_stop_0 node2" [ style=bold color="green" fontcolor="black"] -+"rsc2_stop_0 node3" -> "grp:1_stopped_0" [ style = bold] -+"rsc2_stop_0 node3" -> "rsc1_stop_0 node3" [ style = bold] -+"rsc2_stop_0 node3" -> "rsc2_start_0 node1" [ style = bold] -+"rsc2_stop_0 node3" [ style=bold color="green" fontcolor="black"] -+} -diff --git a/cts/scheduler/dot/clone-recover-no-shuffle-6.dot b/cts/scheduler/dot/clone-recover-no-shuffle-6.dot -new file mode 100644 -index 00000000000..f8cfe9252d2 ---- /dev/null -+++ b/cts/scheduler/dot/clone-recover-no-shuffle-6.dot -@@ -0,0 +1,97 @@ -+ digraph "g" { -+"base-bundle-0_monitor_30000 node1" [ style=bold color="green" fontcolor="black"] -+"base-bundle-0_start_0 node1" -> "base-bundle-0_monitor_30000 node1" [ style = bold] -+"base-bundle-0_start_0 node1" -> "base_start_0 base-bundle-0" [ style = bold] -+"base-bundle-0_start_0 node1" [ style=bold color="green" fontcolor="black"] -+"base-bundle-0_stop_0 node3" -> "base-bundle-0_start_0 node1" [ style = bold] -+"base-bundle-0_stop_0 node3" -> "base-bundle-podman-0_stop_0 node3" [ style = bold] -+"base-bundle-0_stop_0 node3" [ style=bold color="green" fontcolor="black"] -+"base-bundle-1_monitor_30000 node3" [ style=bold color="green" fontcolor="black"] -+"base-bundle-1_start_0 node3" -> "base-bundle-1_monitor_30000 node3" [ style = bold] -+"base-bundle-1_start_0 node3" -> "base_start_0 base-bundle-1" [ style = bold] -+"base-bundle-1_start_0 node3" [ style=bold color="green" fontcolor="black"] -+"base-bundle-1_stop_0 node2" -> "base-bundle-1_start_0 node3" [ style = bold] -+"base-bundle-1_stop_0 node2" -> "base-bundle-podman-1_stop_0 node2" [ style = bold] -+"base-bundle-1_stop_0 node2" [ style=bold color="green" fontcolor="black"] -+"base-bundle-2_monitor_0 node1" -> "base-bundle-2_start_0 node2" [ style = bold] -+"base-bundle-2_monitor_0 node1" [ style=bold color="green" fontcolor="black"] -+"base-bundle-2_monitor_0 node2" -> "base-bundle-2_start_0 node2" [ style = bold] -+"base-bundle-2_monitor_0 node2" [ style=bold color="green" fontcolor="black"] -+"base-bundle-2_monitor_0 node3" -> "base-bundle-2_start_0 node2" [ style = bold] -+"base-bundle-2_monitor_0 node3" [ style=bold color="green" fontcolor="black"] -+"base-bundle-2_monitor_30000 node2" [ style=bold color="green" fontcolor="black"] -+"base-bundle-2_start_0 node2" -> "base-bundle-2_monitor_30000 node2" [ style = bold] -+"base-bundle-2_start_0 node2" -> "base:2_start_0 base-bundle-2" [ style = bold] -+"base-bundle-2_start_0 node2" [ style=bold color="green" fontcolor="black"] -+"base-bundle-clone_running_0" -> "base-bundle_running_0" [ style = bold] -+"base-bundle-clone_running_0" [ style=bold color="green" fontcolor="orange"] -+"base-bundle-clone_start_0" -> "base-bundle-clone_running_0" [ style = bold] -+"base-bundle-clone_start_0" -> "base:2_start_0 base-bundle-2" [ style = bold] -+"base-bundle-clone_start_0" -> "base_start_0 base-bundle-0" [ style = bold] -+"base-bundle-clone_start_0" -> "base_start_0 base-bundle-1" [ style = bold] -+"base-bundle-clone_start_0" [ style=bold color="green" fontcolor="orange"] -+"base-bundle-clone_stop_0" -> "base-bundle-clone_stopped_0" [ style = bold] -+"base-bundle-clone_stop_0" -> "base_stop_0 base-bundle-0" [ style = bold] -+"base-bundle-clone_stop_0" -> "base_stop_0 base-bundle-1" [ style = bold] -+"base-bundle-clone_stop_0" [ style=bold color="green" fontcolor="orange"] -+"base-bundle-clone_stopped_0" -> "base-bundle-clone_start_0" [ style = bold] -+"base-bundle-clone_stopped_0" -> "base-bundle_stopped_0" [ style = bold] -+"base-bundle-clone_stopped_0" [ style=bold color="green" fontcolor="orange"] -+"base-bundle-podman-0_monitor_60000 node1" [ style=bold color="green" fontcolor="black"] -+"base-bundle-podman-0_start_0 node1" -> "base-bundle-0_start_0 node1" [ style = bold] -+"base-bundle-podman-0_start_0 node1" -> "base-bundle-podman-0_monitor_60000 node1" [ style = bold] -+"base-bundle-podman-0_start_0 node1" -> "base-bundle_running_0" [ style = bold] -+"base-bundle-podman-0_start_0 node1" -> "base_start_0 base-bundle-0" [ style = bold] -+"base-bundle-podman-0_start_0 node1" [ style=bold color="green" fontcolor="black"] -+"base-bundle-podman-0_stop_0 node3" -> "base-bundle-podman-0_start_0 node1" [ style = bold] -+"base-bundle-podman-0_stop_0 node3" -> "base-bundle_stopped_0" [ style = bold] -+"base-bundle-podman-0_stop_0 node3" [ style=bold color="green" fontcolor="black"] -+"base-bundle-podman-1_monitor_60000 node3" [ style=bold color="green" fontcolor="black"] -+"base-bundle-podman-1_start_0 node3" -> "base-bundle-1_start_0 node3" [ style = bold] -+"base-bundle-podman-1_start_0 node3" -> "base-bundle-podman-1_monitor_60000 node3" [ style = bold] -+"base-bundle-podman-1_start_0 node3" -> "base-bundle_running_0" [ style = bold] -+"base-bundle-podman-1_start_0 node3" -> "base_start_0 base-bundle-1" [ style = bold] -+"base-bundle-podman-1_start_0 node3" [ style=bold color="green" fontcolor="black"] -+"base-bundle-podman-1_stop_0 node2" -> "base-bundle-podman-1_start_0 node3" [ style = bold] -+"base-bundle-podman-1_stop_0 node2" -> "base-bundle_stopped_0" [ style = bold] -+"base-bundle-podman-1_stop_0 node2" [ style=bold color="green" fontcolor="black"] -+"base-bundle-podman-2_monitor_60000 node2" [ style=bold color="green" fontcolor="black"] -+"base-bundle-podman-2_start_0 node2" -> "base-bundle-2_monitor_0 node1" [ style = bold] -+"base-bundle-podman-2_start_0 node2" -> "base-bundle-2_monitor_0 node2" [ style = bold] -+"base-bundle-podman-2_start_0 node2" -> "base-bundle-2_monitor_0 node3" [ style = bold] -+"base-bundle-podman-2_start_0 node2" -> "base-bundle-2_start_0 node2" [ style = bold] -+"base-bundle-podman-2_start_0 node2" -> "base-bundle-podman-2_monitor_60000 node2" [ style = bold] -+"base-bundle-podman-2_start_0 node2" -> "base-bundle_running_0" [ style = bold] -+"base-bundle-podman-2_start_0 node2" -> "base:2_start_0 base-bundle-2" [ style = bold] -+"base-bundle-podman-2_start_0 node2" [ style=bold color="green" fontcolor="black"] -+"base-bundle_running_0" [ style=bold color="green" fontcolor="orange"] -+"base-bundle_start_0" -> "base-bundle-clone_start_0" [ style = bold] -+"base-bundle_start_0" -> "base-bundle-podman-0_start_0 node1" [ style = bold] -+"base-bundle_start_0" -> "base-bundle-podman-1_start_0 node3" [ style = bold] -+"base-bundle_start_0" -> "base-bundle-podman-2_start_0 node2" [ style = bold] -+"base-bundle_start_0" [ style=bold color="green" fontcolor="orange"] -+"base-bundle_stop_0" -> "base-bundle-clone_stop_0" [ style = bold] -+"base-bundle_stop_0" -> "base-bundle-podman-0_stop_0 node3" [ style = bold] -+"base-bundle_stop_0" -> "base-bundle-podman-1_stop_0 node2" [ style = bold] -+"base-bundle_stop_0" -> "base_stop_0 base-bundle-0" [ style = bold] -+"base-bundle_stop_0" -> "base_stop_0 base-bundle-1" [ style = bold] -+"base-bundle_stop_0" [ style=bold color="green" fontcolor="orange"] -+"base-bundle_stopped_0" [ style=bold color="green" fontcolor="orange"] -+"base:2_start_0 base-bundle-2" -> "base-bundle-clone_running_0" [ style = bold] -+"base:2_start_0 base-bundle-2" [ style=bold color="green" fontcolor="black"] -+"base_start_0 base-bundle-0" -> "base-bundle-clone_running_0" [ style = bold] -+"base_start_0 base-bundle-0" -> "base_start_0 base-bundle-1" [ style = bold] -+"base_start_0 base-bundle-0" [ style=bold color="green" fontcolor="black"] -+"base_start_0 base-bundle-1" -> "base-bundle-clone_running_0" [ style = bold] -+"base_start_0 base-bundle-1" -> "base:2_start_0 base-bundle-2" [ style = bold] -+"base_start_0 base-bundle-1" [ style=bold color="green" fontcolor="black"] -+"base_stop_0 base-bundle-0" -> "base-bundle-0_stop_0 node3" [ style = bold] -+"base_stop_0 base-bundle-0" -> "base-bundle-clone_stopped_0" [ style = bold] -+"base_stop_0 base-bundle-0" -> "base_start_0 base-bundle-0" [ style = bold] -+"base_stop_0 base-bundle-0" [ style=bold color="green" fontcolor="black"] -+"base_stop_0 base-bundle-1" -> "base-bundle-1_stop_0 node2" [ style = bold] -+"base_stop_0 base-bundle-1" -> "base-bundle-clone_stopped_0" [ style = bold] -+"base_stop_0 base-bundle-1" -> "base_start_0 base-bundle-1" [ style = bold] -+"base_stop_0 base-bundle-1" -> "base_stop_0 base-bundle-0" [ style = bold] -+"base_stop_0 base-bundle-1" [ style=bold color="green" fontcolor="black"] -+} -diff --git a/cts/scheduler/dot/clone-recover-no-shuffle-7.dot b/cts/scheduler/dot/clone-recover-no-shuffle-7.dot -new file mode 100644 -index 00000000000..8bff7da01db ---- /dev/null -+++ b/cts/scheduler/dot/clone-recover-no-shuffle-7.dot -@@ -0,0 +1,45 @@ -+ digraph "g" { -+"Cancel dummy_monitor_10000 node2" -> "dummy_demote_0 node2" [ style = bold] -+"Cancel dummy_monitor_10000 node2" [ style=bold color="green" fontcolor="black"] -+"dummy-clone_demote_0" -> "dummy-clone_demoted_0" [ style = bold] -+"dummy-clone_demote_0" -> "dummy_demote_0 node2" [ style = bold] -+"dummy-clone_demote_0" [ style=bold color="green" fontcolor="orange"] -+"dummy-clone_demoted_0" -> "dummy-clone_promote_0" [ style = bold] -+"dummy-clone_demoted_0" -> "dummy-clone_start_0" [ style = bold] -+"dummy-clone_demoted_0" -> "dummy-clone_stop_0" [ style = bold] -+"dummy-clone_demoted_0" [ style=bold color="green" fontcolor="orange"] -+"dummy-clone_promote_0" -> "dummy_promote_0 node1" [ style = bold] -+"dummy-clone_promote_0" [ style=bold color="green" fontcolor="orange"] -+"dummy-clone_promoted_0" [ style=bold color="green" fontcolor="orange"] -+"dummy-clone_running_0" -> "dummy-clone_promote_0" [ style = bold] -+"dummy-clone_running_0" [ style=bold color="green" fontcolor="orange"] -+"dummy-clone_start_0" -> "dummy-clone_running_0" [ style = bold] -+"dummy-clone_start_0" -> "dummy:2_start_0 node3" [ style = bold] -+"dummy-clone_start_0" -> "dummy_start_0 node1" [ style = bold] -+"dummy-clone_start_0" [ style=bold color="green" fontcolor="orange"] -+"dummy-clone_stop_0" -> "dummy-clone_stopped_0" [ style = bold] -+"dummy-clone_stop_0" -> "dummy_stop_0 node3" [ style = bold] -+"dummy-clone_stop_0" [ style=bold color="green" fontcolor="orange"] -+"dummy-clone_stopped_0" -> "dummy-clone_promote_0" [ style = bold] -+"dummy-clone_stopped_0" -> "dummy-clone_start_0" [ style = bold] -+"dummy-clone_stopped_0" [ style=bold color="green" fontcolor="orange"] -+"dummy:2_monitor_11000 node3" [ style=bold color="green" fontcolor="black"] -+"dummy:2_start_0 node3" -> "dummy-clone_running_0" [ style = bold] -+"dummy:2_start_0 node3" -> "dummy:2_monitor_11000 node3" [ style = bold] -+"dummy:2_start_0 node3" [ style=bold color="green" fontcolor="black"] -+"dummy_demote_0 node2" -> "dummy-clone_demoted_0" [ style = bold] -+"dummy_demote_0 node2" -> "dummy_monitor_11000 node2" [ style = bold] -+"dummy_demote_0 node2" [ style=bold color="green" fontcolor="black"] -+"dummy_monitor_10000 node1" [ style=bold color="green" fontcolor="black"] -+"dummy_monitor_11000 node2" [ style=bold color="green" fontcolor="black"] -+"dummy_promote_0 node1" -> "dummy-clone_promoted_0" [ style = bold] -+"dummy_promote_0 node1" -> "dummy_monitor_10000 node1" [ style = bold] -+"dummy_promote_0 node1" [ style=bold color="green" fontcolor="black"] -+"dummy_start_0 node1" -> "dummy-clone_running_0" [ style = bold] -+"dummy_start_0 node1" -> "dummy_monitor_10000 node1" [ style = bold] -+"dummy_start_0 node1" -> "dummy_promote_0 node1" [ style = bold] -+"dummy_start_0 node1" [ style=bold color="green" fontcolor="black"] -+"dummy_stop_0 node3" -> "dummy-clone_stopped_0" [ style = bold] -+"dummy_stop_0 node3" -> "dummy_start_0 node1" [ style = bold] -+"dummy_stop_0 node3" [ style=bold color="green" fontcolor="black"] -+} -diff --git a/cts/scheduler/dot/clone-recover-no-shuffle-8.dot b/cts/scheduler/dot/clone-recover-no-shuffle-8.dot -new file mode 100644 -index 00000000000..d9c311a67cb ---- /dev/null -+++ b/cts/scheduler/dot/clone-recover-no-shuffle-8.dot -@@ -0,0 +1,63 @@ -+ digraph "g" { -+"Cancel rsc1_monitor_10000 node2" -> "rsc1_demote_0 node2" [ style = bold] -+"Cancel rsc1_monitor_10000 node2" [ style=bold color="green" fontcolor="black"] -+"Cancel rsc2_monitor_10000 node2" -> "rsc2_demote_0 node2" [ style = bold] -+"Cancel rsc2_monitor_10000 node2" [ style=bold color="green" fontcolor="black"] -+"grp-clone_demote_0" -> "grp-clone_demoted_0" [ style = bold] -+"grp-clone_demote_0" -> "grp:1_demote_0" [ style = bold] -+"grp-clone_demote_0" [ style=bold color="green" fontcolor="orange"] -+"grp-clone_demoted_0" -> "grp-clone_promote_0" [ style = bold] -+"grp-clone_demoted_0" -> "grp-clone_start_0" [ style = bold] -+"grp-clone_demoted_0" [ style=bold color="green" fontcolor="orange"] -+"grp-clone_promote_0" -> "grp:2_promote_0" [ style = bold] -+"grp-clone_promote_0" [ style=bold color="green" fontcolor="orange"] -+"grp-clone_promoted_0" [ style=bold color="green" fontcolor="orange"] -+"grp-clone_running_0" -> "grp-clone_promote_0" [ style = bold] -+"grp-clone_running_0" [ style=bold color="green" fontcolor="orange"] -+"grp-clone_start_0" -> "grp-clone_running_0" [ style = bold] -+"grp-clone_start_0" -> "grp:2_start_0" [ style = bold] -+"grp-clone_start_0" [ style=bold color="green" fontcolor="orange"] -+"grp:1_demote_0" -> "rsc1_demote_0 node2" [ style = bold] -+"grp:1_demote_0" -> "rsc2_demote_0 node2" [ style = bold] -+"grp:1_demote_0" [ style=bold color="green" fontcolor="orange"] -+"grp:1_demoted_0" -> "grp-clone_demoted_0" [ style = bold] -+"grp:1_demoted_0" [ style=bold color="green" fontcolor="orange"] -+"grp:2_promote_0" -> "rsc1:2_promote_0 node1" [ style = bold] -+"grp:2_promote_0" -> "rsc2:2_promote_0 node1" [ style = bold] -+"grp:2_promote_0" [ style=bold color="green" fontcolor="orange"] -+"grp:2_promoted_0" -> "grp-clone_promoted_0" [ style = bold] -+"grp:2_promoted_0" [ style=bold color="green" fontcolor="orange"] -+"grp:2_running_0" -> "grp-clone_running_0" [ style = bold] -+"grp:2_running_0" [ style=bold color="green" fontcolor="orange"] -+"grp:2_start_0" -> "grp:2_running_0" [ style = bold] -+"grp:2_start_0" -> "rsc1:2_start_0 node1" [ style = bold] -+"grp:2_start_0" -> "rsc2:2_start_0 node1" [ style = bold] -+"grp:2_start_0" [ style=bold color="green" fontcolor="orange"] -+"rsc1:2_monitor_10000 node1" [ style=bold color="green" fontcolor="black"] -+"rsc1:2_promote_0 node1" -> "grp:2_promoted_0" [ style = bold] -+"rsc1:2_promote_0 node1" -> "rsc1:2_monitor_10000 node1" [ style = bold] -+"rsc1:2_promote_0 node1" -> "rsc2:2_promote_0 node1" [ style = bold] -+"rsc1:2_promote_0 node1" [ style=bold color="green" fontcolor="black"] -+"rsc1:2_start_0 node1" -> "grp:2_running_0" [ style = bold] -+"rsc1:2_start_0 node1" -> "rsc1:2_monitor_10000 node1" [ style = bold] -+"rsc1:2_start_0 node1" -> "rsc1:2_promote_0 node1" [ style = bold] -+"rsc1:2_start_0 node1" -> "rsc2:2_start_0 node1" [ style = bold] -+"rsc1:2_start_0 node1" [ style=bold color="green" fontcolor="black"] -+"rsc1_demote_0 node2" -> "grp:1_demoted_0" [ style = bold] -+"rsc1_demote_0 node2" -> "rsc1_monitor_11000 node2" [ style = bold] -+"rsc1_demote_0 node2" [ style=bold color="green" fontcolor="black"] -+"rsc1_monitor_11000 node2" [ style=bold color="green" fontcolor="black"] -+"rsc2:2_monitor_10000 node1" [ style=bold color="green" fontcolor="black"] -+"rsc2:2_promote_0 node1" -> "grp:2_promoted_0" [ style = bold] -+"rsc2:2_promote_0 node1" -> "rsc2:2_monitor_10000 node1" [ style = bold] -+"rsc2:2_promote_0 node1" [ style=bold color="green" fontcolor="black"] -+"rsc2:2_start_0 node1" -> "grp:2_running_0" [ style = bold] -+"rsc2:2_start_0 node1" -> "rsc2:2_monitor_10000 node1" [ style = bold] -+"rsc2:2_start_0 node1" -> "rsc2:2_promote_0 node1" [ style = bold] -+"rsc2:2_start_0 node1" [ style=bold color="green" fontcolor="black"] -+"rsc2_demote_0 node2" -> "grp:1_demoted_0" [ style = bold] -+"rsc2_demote_0 node2" -> "rsc1_demote_0 node2" [ style = bold] -+"rsc2_demote_0 node2" -> "rsc2_monitor_11000 node2" [ style = bold] -+"rsc2_demote_0 node2" [ style=bold color="green" fontcolor="black"] -+"rsc2_monitor_11000 node2" [ style=bold color="green" fontcolor="black"] -+} -diff --git a/cts/scheduler/dot/clone-recover-no-shuffle-9.dot b/cts/scheduler/dot/clone-recover-no-shuffle-9.dot -new file mode 100644 -index 00000000000..45dbac47e2b ---- /dev/null -+++ b/cts/scheduler/dot/clone-recover-no-shuffle-9.dot -@@ -0,0 +1,69 @@ -+ digraph "g" { -+"Cancel base_monitor_15000 base-bundle-1" -> "base_demote_0 base-bundle-1" [ style = bold] -+"Cancel base_monitor_15000 base-bundle-1" [ style=bold color="green" fontcolor="black"] -+"base-bundle-2_monitor_0 node1" -> "base-bundle-2_start_0 node1" [ style = bold] -+"base-bundle-2_monitor_0 node1" [ style=bold color="green" fontcolor="black"] -+"base-bundle-2_monitor_0 node2" -> "base-bundle-2_start_0 node1" [ style = bold] -+"base-bundle-2_monitor_0 node2" [ style=bold color="green" fontcolor="black"] -+"base-bundle-2_monitor_0 node3" -> "base-bundle-2_start_0 node1" [ style = bold] -+"base-bundle-2_monitor_0 node3" [ style=bold color="green" fontcolor="black"] -+"base-bundle-2_monitor_30000 node1" [ style=bold color="green" fontcolor="black"] -+"base-bundle-2_start_0 node1" -> "base-bundle-2_monitor_30000 node1" [ style = bold] -+"base-bundle-2_start_0 node1" -> "base:2_monitor_15000 base-bundle-2" [ style = bold] -+"base-bundle-2_start_0 node1" -> "base:2_promote_0 base-bundle-2" [ style = bold] -+"base-bundle-2_start_0 node1" -> "base:2_start_0 base-bundle-2" [ style = bold] -+"base-bundle-2_start_0 node1" [ style=bold color="green" fontcolor="black"] -+"base-bundle-clone_demote_0" -> "base-bundle-clone_demoted_0" [ style = bold] -+"base-bundle-clone_demote_0" -> "base_demote_0 base-bundle-1" [ style = bold] -+"base-bundle-clone_demote_0" [ style=bold color="green" fontcolor="orange"] -+"base-bundle-clone_demoted_0" -> "base-bundle-clone_promote_0" [ style = bold] -+"base-bundle-clone_demoted_0" -> "base-bundle-clone_start_0" [ style = bold] -+"base-bundle-clone_demoted_0" -> "base-bundle_demoted_0" [ style = bold] -+"base-bundle-clone_demoted_0" [ style=bold color="green" fontcolor="orange"] -+"base-bundle-clone_promote_0" -> "base:2_promote_0 base-bundle-2" [ style = bold] -+"base-bundle-clone_promote_0" [ style=bold color="green" fontcolor="orange"] -+"base-bundle-clone_promoted_0" -> "base-bundle_promoted_0" [ style = bold] -+"base-bundle-clone_promoted_0" [ style=bold color="green" fontcolor="orange"] -+"base-bundle-clone_running_0" -> "base-bundle-clone_promote_0" [ style = bold] -+"base-bundle-clone_running_0" -> "base-bundle_running_0" [ style = bold] -+"base-bundle-clone_running_0" [ style=bold color="green" fontcolor="orange"] -+"base-bundle-clone_start_0" -> "base-bundle-clone_running_0" [ style = bold] -+"base-bundle-clone_start_0" -> "base:2_start_0 base-bundle-2" [ style = bold] -+"base-bundle-clone_start_0" [ style=bold color="green" fontcolor="orange"] -+"base-bundle-podman-2_monitor_60000 node1" [ style=bold color="green" fontcolor="black"] -+"base-bundle-podman-2_start_0 node1" -> "base-bundle-2_monitor_0 node1" [ style = bold] -+"base-bundle-podman-2_start_0 node1" -> "base-bundle-2_monitor_0 node2" [ style = bold] -+"base-bundle-podman-2_start_0 node1" -> "base-bundle-2_monitor_0 node3" [ style = bold] -+"base-bundle-podman-2_start_0 node1" -> "base-bundle-2_start_0 node1" [ style = bold] -+"base-bundle-podman-2_start_0 node1" -> "base-bundle-podman-2_monitor_60000 node1" [ style = bold] -+"base-bundle-podman-2_start_0 node1" -> "base-bundle_running_0" [ style = bold] -+"base-bundle-podman-2_start_0 node1" -> "base:2_promote_0 base-bundle-2" [ style = bold] -+"base-bundle-podman-2_start_0 node1" -> "base:2_start_0 base-bundle-2" [ style = bold] -+"base-bundle-podman-2_start_0 node1" [ style=bold color="green" fontcolor="black"] -+"base-bundle_demote_0" -> "base-bundle-clone_demote_0" [ style = bold] -+"base-bundle_demote_0" -> "base-bundle_demoted_0" [ style = bold] -+"base-bundle_demote_0" [ style=bold color="green" fontcolor="orange"] -+"base-bundle_demoted_0" -> "base-bundle_promote_0" [ style = bold] -+"base-bundle_demoted_0" -> "base-bundle_start_0" [ style = bold] -+"base-bundle_demoted_0" [ style=bold color="green" fontcolor="orange"] -+"base-bundle_promote_0" -> "base-bundle-clone_promote_0" [ style = bold] -+"base-bundle_promote_0" [ style=bold color="green" fontcolor="orange"] -+"base-bundle_promoted_0" [ style=bold color="green" fontcolor="orange"] -+"base-bundle_running_0" -> "base-bundle_promote_0" [ style = bold] -+"base-bundle_running_0" [ style=bold color="green" fontcolor="orange"] -+"base-bundle_start_0" -> "base-bundle-clone_start_0" [ style = bold] -+"base-bundle_start_0" -> "base-bundle-podman-2_start_0 node1" [ style = bold] -+"base-bundle_start_0" [ style=bold color="green" fontcolor="orange"] -+"base:2_monitor_15000 base-bundle-2" [ style=bold color="green" fontcolor="black"] -+"base:2_promote_0 base-bundle-2" -> "base-bundle-clone_promoted_0" [ style = bold] -+"base:2_promote_0 base-bundle-2" -> "base:2_monitor_15000 base-bundle-2" [ style = bold] -+"base:2_promote_0 base-bundle-2" [ style=bold color="green" fontcolor="black"] -+"base:2_start_0 base-bundle-2" -> "base-bundle-clone_running_0" [ style = bold] -+"base:2_start_0 base-bundle-2" -> "base:2_monitor_15000 base-bundle-2" [ style = bold] -+"base:2_start_0 base-bundle-2" -> "base:2_promote_0 base-bundle-2" [ style = bold] -+"base:2_start_0 base-bundle-2" [ style=bold color="green" fontcolor="black"] -+"base_demote_0 base-bundle-1" -> "base-bundle-clone_demoted_0" [ style = bold] -+"base_demote_0 base-bundle-1" -> "base_monitor_16000 base-bundle-1" [ style = bold] -+"base_demote_0 base-bundle-1" [ style=bold color="green" fontcolor="black"] -+"base_monitor_16000 base-bundle-1" [ style=bold color="green" fontcolor="black"] -+} -diff --git a/cts/scheduler/exp/clone-recover-no-shuffle-1.exp b/cts/scheduler/exp/clone-recover-no-shuffle-1.exp -new file mode 100644 -index 00000000000..670a823dac9 ---- /dev/null -+++ b/cts/scheduler/exp/clone-recover-no-shuffle-1.exp -@@ -0,0 +1,51 @@ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -diff --git a/cts/scheduler/exp/clone-recover-no-shuffle-10.exp b/cts/scheduler/exp/clone-recover-no-shuffle-10.exp -new file mode 100644 -index 00000000000..27b8b7037c3 ---- /dev/null -+++ b/cts/scheduler/exp/clone-recover-no-shuffle-10.exp -@@ -0,0 +1,51 @@ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -diff --git a/cts/scheduler/exp/clone-recover-no-shuffle-11.exp b/cts/scheduler/exp/clone-recover-no-shuffle-11.exp -new file mode 100644 -index 00000000000..40cf1f69c11 ---- /dev/null -+++ b/cts/scheduler/exp/clone-recover-no-shuffle-11.exp -@@ -0,0 +1,110 @@ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -diff --git a/cts/scheduler/exp/clone-recover-no-shuffle-12.exp b/cts/scheduler/exp/clone-recover-no-shuffle-12.exp -new file mode 100644 -index 00000000000..919e6b291c0 ---- /dev/null -+++ b/cts/scheduler/exp/clone-recover-no-shuffle-12.exp -@@ -0,0 +1,187 @@ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -diff --git a/cts/scheduler/exp/clone-recover-no-shuffle-2.exp b/cts/scheduler/exp/clone-recover-no-shuffle-2.exp -new file mode 100644 -index 00000000000..84b1e1bc98c ---- /dev/null -+++ b/cts/scheduler/exp/clone-recover-no-shuffle-2.exp -@@ -0,0 +1,110 @@ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -diff --git a/cts/scheduler/exp/clone-recover-no-shuffle-3.exp b/cts/scheduler/exp/clone-recover-no-shuffle-3.exp -new file mode 100644 -index 00000000000..6b6ed075f57 ---- /dev/null -+++ b/cts/scheduler/exp/clone-recover-no-shuffle-3.exp -@@ -0,0 +1,171 @@ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -diff --git a/cts/scheduler/exp/clone-recover-no-shuffle-4.exp b/cts/scheduler/exp/clone-recover-no-shuffle-4.exp -new file mode 100644 -index 00000000000..4596c685d0a ---- /dev/null -+++ b/cts/scheduler/exp/clone-recover-no-shuffle-4.exp -@@ -0,0 +1,123 @@ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -diff --git a/cts/scheduler/exp/clone-recover-no-shuffle-5.exp b/cts/scheduler/exp/clone-recover-no-shuffle-5.exp -new file mode 100644 -index 00000000000..8a8e799793e ---- /dev/null -+++ b/cts/scheduler/exp/clone-recover-no-shuffle-5.exp -@@ -0,0 +1,452 @@ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -diff --git a/cts/scheduler/exp/clone-recover-no-shuffle-6.exp b/cts/scheduler/exp/clone-recover-no-shuffle-6.exp -new file mode 100644 -index 00000000000..e6704c9e254 ---- /dev/null -+++ b/cts/scheduler/exp/clone-recover-no-shuffle-6.exp -@@ -0,0 +1,507 @@ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -diff --git a/cts/scheduler/exp/clone-recover-no-shuffle-7.exp b/cts/scheduler/exp/clone-recover-no-shuffle-7.exp -new file mode 100644 -index 00000000000..950de9e0312 ---- /dev/null -+++ b/cts/scheduler/exp/clone-recover-no-shuffle-7.exp -@@ -0,0 +1,240 @@ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -diff --git a/cts/scheduler/exp/clone-recover-no-shuffle-8.exp b/cts/scheduler/exp/clone-recover-no-shuffle-8.exp -new file mode 100644 -index 00000000000..763a2f02fb0 ---- /dev/null -+++ b/cts/scheduler/exp/clone-recover-no-shuffle-8.exp -@@ -0,0 +1,338 @@ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -diff --git a/cts/scheduler/exp/clone-recover-no-shuffle-9.exp b/cts/scheduler/exp/clone-recover-no-shuffle-9.exp -new file mode 100644 -index 00000000000..7bfe3c47281 ---- /dev/null -+++ b/cts/scheduler/exp/clone-recover-no-shuffle-9.exp -@@ -0,0 +1,364 @@ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -diff --git a/cts/scheduler/scores/clone-recover-no-shuffle-1.scores b/cts/scheduler/scores/clone-recover-no-shuffle-1.scores -new file mode 100644 -index 00000000000..c1d60b2f39a ---- /dev/null -+++ b/cts/scheduler/scores/clone-recover-no-shuffle-1.scores -@@ -0,0 +1,25 @@ -+ -+pcmk__clone_assign: dummy-clone allocation score on node1: 0 -+pcmk__clone_assign: dummy-clone allocation score on node2: 0 -+pcmk__clone_assign: dummy-clone allocation score on node3: 0 -+pcmk__clone_assign: dummy:0 allocation score on node1: 0 -+pcmk__clone_assign: dummy:0 allocation score on node2: 1 -+pcmk__clone_assign: dummy:0 allocation score on node3: 0 -+pcmk__clone_assign: dummy:1 allocation score on node1: 0 -+pcmk__clone_assign: dummy:1 allocation score on node2: 0 -+pcmk__clone_assign: dummy:1 allocation score on node3: 1 -+pcmk__clone_assign: dummy:2 allocation score on node1: 0 -+pcmk__clone_assign: dummy:2 allocation score on node2: 0 -+pcmk__clone_assign: dummy:2 allocation score on node3: 0 -+pcmk__primitive_assign: Fencing allocation score on node1: 0 -+pcmk__primitive_assign: Fencing allocation score on node2: 0 -+pcmk__primitive_assign: Fencing allocation score on node3: 0 -+pcmk__primitive_assign: dummy:0 allocation score on node1: 0 -+pcmk__primitive_assign: dummy:0 allocation score on node2: 1 -+pcmk__primitive_assign: dummy:0 allocation score on node3: 0 -+pcmk__primitive_assign: dummy:1 allocation score on node1: 0 -+pcmk__primitive_assign: dummy:1 allocation score on node2: -INFINITY -+pcmk__primitive_assign: dummy:1 allocation score on node3: 1 -+pcmk__primitive_assign: dummy:2 allocation score on node1: 0 -+pcmk__primitive_assign: dummy:2 allocation score on node2: -INFINITY -+pcmk__primitive_assign: dummy:2 allocation score on node3: -INFINITY -diff --git a/cts/scheduler/scores/clone-recover-no-shuffle-10.scores b/cts/scheduler/scores/clone-recover-no-shuffle-10.scores -new file mode 100644 -index 00000000000..4ac63e37058 ---- /dev/null -+++ b/cts/scheduler/scores/clone-recover-no-shuffle-10.scores -@@ -0,0 +1,31 @@ -+ -+dummy:0 promotion score on node3: 5 -+dummy:1 promotion score on node2: 15 -+dummy:2 promotion score on node1: 10 -+pcmk__clone_assign: dummy-clone allocation score on node1: 0 -+pcmk__clone_assign: dummy-clone allocation score on node2: 0 -+pcmk__clone_assign: dummy-clone allocation score on node3: 0 -+pcmk__clone_assign: dummy:0 allocation score on node1: 10 -+pcmk__clone_assign: dummy:0 allocation score on node2: 0 -+pcmk__clone_assign: dummy:0 allocation score on node3: 6 -+pcmk__clone_assign: dummy:1 allocation score on node1: 10 -+pcmk__clone_assign: dummy:1 allocation score on node2: 16 -+pcmk__clone_assign: dummy:1 allocation score on node3: 0 -+pcmk__clone_assign: dummy:2 allocation score on node1: 10 -+pcmk__clone_assign: dummy:2 allocation score on node2: 15 -+pcmk__clone_assign: dummy:2 allocation score on node3: 5 -+pcmk__primitive_assign: Fencing allocation score on node1: 0 -+pcmk__primitive_assign: Fencing allocation score on node2: 0 -+pcmk__primitive_assign: Fencing allocation score on node3: 0 -+pcmk__primitive_assign: dummy:0 allocation score on node1: -INFINITY -+pcmk__primitive_assign: dummy:0 allocation score on node1: 10 -+pcmk__primitive_assign: dummy:0 allocation score on node2: -INFINITY -+pcmk__primitive_assign: dummy:0 allocation score on node2: -INFINITY -+pcmk__primitive_assign: dummy:0 allocation score on node3: 6 -+pcmk__primitive_assign: dummy:0 allocation score on node3: 6 -+pcmk__primitive_assign: dummy:1 allocation score on node1: 10 -+pcmk__primitive_assign: dummy:1 allocation score on node2: 16 -+pcmk__primitive_assign: dummy:1 allocation score on node3: 0 -+pcmk__primitive_assign: dummy:2 allocation score on node1: 10 -+pcmk__primitive_assign: dummy:2 allocation score on node2: -INFINITY -+pcmk__primitive_assign: dummy:2 allocation score on node3: 5 -diff --git a/cts/scheduler/scores/clone-recover-no-shuffle-11.scores b/cts/scheduler/scores/clone-recover-no-shuffle-11.scores -new file mode 100644 -index 00000000000..1216dba711a ---- /dev/null -+++ b/cts/scheduler/scores/clone-recover-no-shuffle-11.scores -@@ -0,0 +1,82 @@ -+ -+grp:0 promotion score on node3: 10 -+grp:1 promotion score on node2: 30 -+grp:2 promotion score on node1: 20 -+pcmk__clone_assign: grp-clone allocation score on node1: 0 -+pcmk__clone_assign: grp-clone allocation score on node2: 0 -+pcmk__clone_assign: grp-clone allocation score on node3: 0 -+pcmk__clone_assign: grp:0 allocation score on node1: 20 -+pcmk__clone_assign: grp:0 allocation score on node2: 0 -+pcmk__clone_assign: grp:0 allocation score on node3: 10 -+pcmk__clone_assign: grp:1 allocation score on node1: 20 -+pcmk__clone_assign: grp:1 allocation score on node2: 30 -+pcmk__clone_assign: grp:1 allocation score on node3: 0 -+pcmk__clone_assign: grp:2 allocation score on node1: 20 -+pcmk__clone_assign: grp:2 allocation score on node2: 30 -+pcmk__clone_assign: grp:2 allocation score on node3: 10 -+pcmk__clone_assign: rsc1:0 allocation score on node1: 0 -+pcmk__clone_assign: rsc1:0 allocation score on node2: 0 -+pcmk__clone_assign: rsc1:0 allocation score on node3: 1 -+pcmk__clone_assign: rsc1:1 allocation score on node1: 0 -+pcmk__clone_assign: rsc1:1 allocation score on node2: 1 -+pcmk__clone_assign: rsc1:1 allocation score on node3: 0 -+pcmk__clone_assign: rsc1:2 allocation score on node1: 0 -+pcmk__clone_assign: rsc1:2 allocation score on node2: 0 -+pcmk__clone_assign: rsc1:2 allocation score on node3: 0 -+pcmk__clone_assign: rsc2:0 allocation score on node1: 0 -+pcmk__clone_assign: rsc2:0 allocation score on node2: 0 -+pcmk__clone_assign: rsc2:0 allocation score on node3: 1 -+pcmk__clone_assign: rsc2:1 allocation score on node1: 0 -+pcmk__clone_assign: rsc2:1 allocation score on node2: 1 -+pcmk__clone_assign: rsc2:1 allocation score on node3: 0 -+pcmk__clone_assign: rsc2:2 allocation score on node1: 0 -+pcmk__clone_assign: rsc2:2 allocation score on node2: 0 -+pcmk__clone_assign: rsc2:2 allocation score on node3: 0 -+pcmk__group_assign: grp:0 allocation score on node1: 20 -+pcmk__group_assign: grp:0 allocation score on node2: -INFINITY -+pcmk__group_assign: grp:0 allocation score on node3: 10 -+pcmk__group_assign: grp:1 allocation score on node1: 20 -+pcmk__group_assign: grp:1 allocation score on node2: 30 -+pcmk__group_assign: grp:1 allocation score on node3: 0 -+pcmk__group_assign: grp:2 allocation score on node1: 20 -+pcmk__group_assign: grp:2 allocation score on node2: -INFINITY -+pcmk__group_assign: grp:2 allocation score on node3: -INFINITY -+pcmk__group_assign: rsc1:0 allocation score on node1: 0 -+pcmk__group_assign: rsc1:0 allocation score on node2: -INFINITY -+pcmk__group_assign: rsc1:0 allocation score on node3: 1 -+pcmk__group_assign: rsc1:1 allocation score on node1: 0 -+pcmk__group_assign: rsc1:1 allocation score on node2: 1 -+pcmk__group_assign: rsc1:1 allocation score on node3: 0 -+pcmk__group_assign: rsc1:2 allocation score on node1: 0 -+pcmk__group_assign: rsc1:2 allocation score on node2: -INFINITY -+pcmk__group_assign: rsc1:2 allocation score on node3: -INFINITY -+pcmk__group_assign: rsc2:0 allocation score on node1: 0 -+pcmk__group_assign: rsc2:0 allocation score on node2: -INFINITY -+pcmk__group_assign: rsc2:0 allocation score on node3: 1 -+pcmk__group_assign: rsc2:1 allocation score on node1: 0 -+pcmk__group_assign: rsc2:1 allocation score on node2: 1 -+pcmk__group_assign: rsc2:1 allocation score on node3: 0 -+pcmk__group_assign: rsc2:2 allocation score on node1: 0 -+pcmk__group_assign: rsc2:2 allocation score on node2: -INFINITY -+pcmk__group_assign: rsc2:2 allocation score on node3: -INFINITY -+pcmk__primitive_assign: Fencing allocation score on node1: 0 -+pcmk__primitive_assign: Fencing allocation score on node2: 0 -+pcmk__primitive_assign: Fencing allocation score on node3: 0 -+pcmk__primitive_assign: rsc1:0 allocation score on node1: 0 -+pcmk__primitive_assign: rsc1:0 allocation score on node2: -INFINITY -+pcmk__primitive_assign: rsc1:0 allocation score on node3: 2 -+pcmk__primitive_assign: rsc1:1 allocation score on node1: 0 -+pcmk__primitive_assign: rsc1:1 allocation score on node2: 2 -+pcmk__primitive_assign: rsc1:1 allocation score on node3: 0 -+pcmk__primitive_assign: rsc1:2 allocation score on node1: 0 -+pcmk__primitive_assign: rsc1:2 allocation score on node2: -INFINITY -+pcmk__primitive_assign: rsc1:2 allocation score on node3: -INFINITY -+pcmk__primitive_assign: rsc2:0 allocation score on node1: -INFINITY -+pcmk__primitive_assign: rsc2:0 allocation score on node2: -INFINITY -+pcmk__primitive_assign: rsc2:0 allocation score on node3: 1 -+pcmk__primitive_assign: rsc2:1 allocation score on node1: -INFINITY -+pcmk__primitive_assign: rsc2:1 allocation score on node2: 1 -+pcmk__primitive_assign: rsc2:1 allocation score on node3: -INFINITY -+pcmk__primitive_assign: rsc2:2 allocation score on node1: 0 -+pcmk__primitive_assign: rsc2:2 allocation score on node2: -INFINITY -+pcmk__primitive_assign: rsc2:2 allocation score on node3: -INFINITY -diff --git a/cts/scheduler/scores/clone-recover-no-shuffle-12.scores b/cts/scheduler/scores/clone-recover-no-shuffle-12.scores -new file mode 100644 -index 00000000000..24cf3148c4c ---- /dev/null -+++ b/cts/scheduler/scores/clone-recover-no-shuffle-12.scores -@@ -0,0 +1,67 @@ -+ -+base:0 promotion score on base-bundle-0: 5 -+base:1 promotion score on base-bundle-1: 15 -+base:2 promotion score on base-bundle-2: 10 -+pcmk__bundle_allocate: base-bundle allocation score on node1: 0 -+pcmk__bundle_allocate: base-bundle allocation score on node2: 0 -+pcmk__bundle_allocate: base-bundle allocation score on node3: 0 -+pcmk__bundle_allocate: base-bundle-0 allocation score on node1: 0 -+pcmk__bundle_allocate: base-bundle-0 allocation score on node2: 0 -+pcmk__bundle_allocate: base-bundle-0 allocation score on node3: 0 -+pcmk__bundle_allocate: base-bundle-1 allocation score on node1: 0 -+pcmk__bundle_allocate: base-bundle-1 allocation score on node2: 0 -+pcmk__bundle_allocate: base-bundle-1 allocation score on node3: 0 -+pcmk__bundle_allocate: base-bundle-2 allocation score on node1: 0 -+pcmk__bundle_allocate: base-bundle-2 allocation score on node2: 0 -+pcmk__bundle_allocate: base-bundle-2 allocation score on node3: 0 -+pcmk__bundle_allocate: base-bundle-clone allocation score on base-bundle-0: -INFINITY -+pcmk__bundle_allocate: base-bundle-clone allocation score on base-bundle-1: -INFINITY -+pcmk__bundle_allocate: base-bundle-clone allocation score on base-bundle-2: -INFINITY -+pcmk__bundle_allocate: base-bundle-clone allocation score on node1: 0 -+pcmk__bundle_allocate: base-bundle-clone allocation score on node2: 0 -+pcmk__bundle_allocate: base-bundle-clone allocation score on node3: 0 -+pcmk__bundle_allocate: base-bundle-podman-0 allocation score on node1: 0 -+pcmk__bundle_allocate: base-bundle-podman-0 allocation score on node2: 0 -+pcmk__bundle_allocate: base-bundle-podman-0 allocation score on node3: 0 -+pcmk__bundle_allocate: base-bundle-podman-1 allocation score on node1: 0 -+pcmk__bundle_allocate: base-bundle-podman-1 allocation score on node2: 0 -+pcmk__bundle_allocate: base-bundle-podman-1 allocation score on node3: 0 -+pcmk__bundle_allocate: base-bundle-podman-2 allocation score on node1: 0 -+pcmk__bundle_allocate: base-bundle-podman-2 allocation score on node2: 0 -+pcmk__bundle_allocate: base-bundle-podman-2 allocation score on node3: 0 -+pcmk__bundle_allocate: base:0 allocation score on base-bundle-0: 501 -+pcmk__bundle_allocate: base:1 allocation score on base-bundle-1: 501 -+pcmk__bundle_allocate: base:2 allocation score on base-bundle-2: 500 -+pcmk__clone_assign: base-bundle-clone allocation score on base-bundle-0: 0 -+pcmk__clone_assign: base-bundle-clone allocation score on base-bundle-1: 0 -+pcmk__clone_assign: base-bundle-clone allocation score on base-bundle-2: 0 -+pcmk__clone_assign: base-bundle-clone allocation score on node1: -INFINITY -+pcmk__clone_assign: base-bundle-clone allocation score on node2: -INFINITY -+pcmk__clone_assign: base-bundle-clone allocation score on node3: -INFINITY -+pcmk__clone_assign: base:0 allocation score on base-bundle-0: INFINITY -+pcmk__clone_assign: base:1 allocation score on base-bundle-1: INFINITY -+pcmk__clone_assign: base:2 allocation score on base-bundle-2: INFINITY -+pcmk__primitive_assign: Fencing allocation score on node1: 0 -+pcmk__primitive_assign: Fencing allocation score on node2: 0 -+pcmk__primitive_assign: Fencing allocation score on node3: 0 -+pcmk__primitive_assign: base-bundle-0 allocation score on node1: 0 -+pcmk__primitive_assign: base-bundle-0 allocation score on node2: 0 -+pcmk__primitive_assign: base-bundle-0 allocation score on node3: 10000 -+pcmk__primitive_assign: base-bundle-1 allocation score on node1: 0 -+pcmk__primitive_assign: base-bundle-1 allocation score on node2: 10000 -+pcmk__primitive_assign: base-bundle-1 allocation score on node3: 0 -+pcmk__primitive_assign: base-bundle-2 allocation score on node1: 10000 -+pcmk__primitive_assign: base-bundle-2 allocation score on node2: 0 -+pcmk__primitive_assign: base-bundle-2 allocation score on node3: 0 -+pcmk__primitive_assign: base-bundle-podman-0 allocation score on node1: 0 -+pcmk__primitive_assign: base-bundle-podman-0 allocation score on node2: 0 -+pcmk__primitive_assign: base-bundle-podman-0 allocation score on node3: 0 -+pcmk__primitive_assign: base-bundle-podman-1 allocation score on node1: 0 -+pcmk__primitive_assign: base-bundle-podman-1 allocation score on node2: 0 -+pcmk__primitive_assign: base-bundle-podman-1 allocation score on node3: -INFINITY -+pcmk__primitive_assign: base-bundle-podman-2 allocation score on node1: 0 -+pcmk__primitive_assign: base-bundle-podman-2 allocation score on node2: -INFINITY -+pcmk__primitive_assign: base-bundle-podman-2 allocation score on node3: -INFINITY -+pcmk__primitive_assign: base:0 allocation score on base-bundle-0: INFINITY -+pcmk__primitive_assign: base:1 allocation score on base-bundle-1: INFINITY -+pcmk__primitive_assign: base:2 allocation score on base-bundle-2: INFINITY -diff --git a/cts/scheduler/scores/clone-recover-no-shuffle-2.scores b/cts/scheduler/scores/clone-recover-no-shuffle-2.scores -new file mode 100644 -index 00000000000..cfbd5bf5337 ---- /dev/null -+++ b/cts/scheduler/scores/clone-recover-no-shuffle-2.scores -@@ -0,0 +1,79 @@ -+ -+pcmk__clone_assign: grp-clone allocation score on node1: 0 -+pcmk__clone_assign: grp-clone allocation score on node2: 0 -+pcmk__clone_assign: grp-clone allocation score on node3: 0 -+pcmk__clone_assign: grp:0 allocation score on node1: 0 -+pcmk__clone_assign: grp:0 allocation score on node2: 0 -+pcmk__clone_assign: grp:0 allocation score on node3: 0 -+pcmk__clone_assign: grp:1 allocation score on node1: 0 -+pcmk__clone_assign: grp:1 allocation score on node2: 0 -+pcmk__clone_assign: grp:1 allocation score on node3: 0 -+pcmk__clone_assign: grp:2 allocation score on node1: 0 -+pcmk__clone_assign: grp:2 allocation score on node2: 0 -+pcmk__clone_assign: grp:2 allocation score on node3: 0 -+pcmk__clone_assign: rsc1:0 allocation score on node1: 0 -+pcmk__clone_assign: rsc1:0 allocation score on node2: 1 -+pcmk__clone_assign: rsc1:0 allocation score on node3: 0 -+pcmk__clone_assign: rsc1:1 allocation score on node1: 0 -+pcmk__clone_assign: rsc1:1 allocation score on node2: 0 -+pcmk__clone_assign: rsc1:1 allocation score on node3: 1 -+pcmk__clone_assign: rsc1:2 allocation score on node1: 0 -+pcmk__clone_assign: rsc1:2 allocation score on node2: 0 -+pcmk__clone_assign: rsc1:2 allocation score on node3: 0 -+pcmk__clone_assign: rsc2:0 allocation score on node1: 0 -+pcmk__clone_assign: rsc2:0 allocation score on node2: 1 -+pcmk__clone_assign: rsc2:0 allocation score on node3: 0 -+pcmk__clone_assign: rsc2:1 allocation score on node1: 0 -+pcmk__clone_assign: rsc2:1 allocation score on node2: 0 -+pcmk__clone_assign: rsc2:1 allocation score on node3: 1 -+pcmk__clone_assign: rsc2:2 allocation score on node1: 0 -+pcmk__clone_assign: rsc2:2 allocation score on node2: 0 -+pcmk__clone_assign: rsc2:2 allocation score on node3: 0 -+pcmk__group_assign: grp:0 allocation score on node1: 0 -+pcmk__group_assign: grp:0 allocation score on node2: 0 -+pcmk__group_assign: grp:0 allocation score on node3: 0 -+pcmk__group_assign: grp:1 allocation score on node1: 0 -+pcmk__group_assign: grp:1 allocation score on node2: -INFINITY -+pcmk__group_assign: grp:1 allocation score on node3: 0 -+pcmk__group_assign: grp:2 allocation score on node1: 0 -+pcmk__group_assign: grp:2 allocation score on node2: -INFINITY -+pcmk__group_assign: grp:2 allocation score on node3: -INFINITY -+pcmk__group_assign: rsc1:0 allocation score on node1: 0 -+pcmk__group_assign: rsc1:0 allocation score on node2: 1 -+pcmk__group_assign: rsc1:0 allocation score on node3: 0 -+pcmk__group_assign: rsc1:1 allocation score on node1: 0 -+pcmk__group_assign: rsc1:1 allocation score on node2: -INFINITY -+pcmk__group_assign: rsc1:1 allocation score on node3: 1 -+pcmk__group_assign: rsc1:2 allocation score on node1: 0 -+pcmk__group_assign: rsc1:2 allocation score on node2: -INFINITY -+pcmk__group_assign: rsc1:2 allocation score on node3: -INFINITY -+pcmk__group_assign: rsc2:0 allocation score on node1: 0 -+pcmk__group_assign: rsc2:0 allocation score on node2: 1 -+pcmk__group_assign: rsc2:0 allocation score on node3: 0 -+pcmk__group_assign: rsc2:1 allocation score on node1: 0 -+pcmk__group_assign: rsc2:1 allocation score on node2: -INFINITY -+pcmk__group_assign: rsc2:1 allocation score on node3: 1 -+pcmk__group_assign: rsc2:2 allocation score on node1: 0 -+pcmk__group_assign: rsc2:2 allocation score on node2: -INFINITY -+pcmk__group_assign: rsc2:2 allocation score on node3: -INFINITY -+pcmk__primitive_assign: Fencing allocation score on node1: 0 -+pcmk__primitive_assign: Fencing allocation score on node2: 0 -+pcmk__primitive_assign: Fencing allocation score on node3: 0 -+pcmk__primitive_assign: rsc1:0 allocation score on node1: 0 -+pcmk__primitive_assign: rsc1:0 allocation score on node2: 2 -+pcmk__primitive_assign: rsc1:0 allocation score on node3: 0 -+pcmk__primitive_assign: rsc1:1 allocation score on node1: 0 -+pcmk__primitive_assign: rsc1:1 allocation score on node2: -INFINITY -+pcmk__primitive_assign: rsc1:1 allocation score on node3: 2 -+pcmk__primitive_assign: rsc1:2 allocation score on node1: 0 -+pcmk__primitive_assign: rsc1:2 allocation score on node2: -INFINITY -+pcmk__primitive_assign: rsc1:2 allocation score on node3: -INFINITY -+pcmk__primitive_assign: rsc2:0 allocation score on node1: -INFINITY -+pcmk__primitive_assign: rsc2:0 allocation score on node2: 1 -+pcmk__primitive_assign: rsc2:0 allocation score on node3: -INFINITY -+pcmk__primitive_assign: rsc2:1 allocation score on node1: -INFINITY -+pcmk__primitive_assign: rsc2:1 allocation score on node2: -INFINITY -+pcmk__primitive_assign: rsc2:1 allocation score on node3: 1 -+pcmk__primitive_assign: rsc2:2 allocation score on node1: 0 -+pcmk__primitive_assign: rsc2:2 allocation score on node2: -INFINITY -+pcmk__primitive_assign: rsc2:2 allocation score on node3: -INFINITY -diff --git a/cts/scheduler/scores/clone-recover-no-shuffle-3.scores b/cts/scheduler/scores/clone-recover-no-shuffle-3.scores -new file mode 100644 -index 00000000000..461c11633b1 ---- /dev/null -+++ b/cts/scheduler/scores/clone-recover-no-shuffle-3.scores -@@ -0,0 +1,64 @@ -+ -+pcmk__bundle_allocate: base-bundle allocation score on node1: 0 -+pcmk__bundle_allocate: base-bundle allocation score on node2: 0 -+pcmk__bundle_allocate: base-bundle allocation score on node3: 0 -+pcmk__bundle_allocate: base-bundle-0 allocation score on node1: 0 -+pcmk__bundle_allocate: base-bundle-0 allocation score on node2: 0 -+pcmk__bundle_allocate: base-bundle-0 allocation score on node3: 0 -+pcmk__bundle_allocate: base-bundle-1 allocation score on node1: 0 -+pcmk__bundle_allocate: base-bundle-1 allocation score on node2: 0 -+pcmk__bundle_allocate: base-bundle-1 allocation score on node3: 0 -+pcmk__bundle_allocate: base-bundle-2 allocation score on node1: 0 -+pcmk__bundle_allocate: base-bundle-2 allocation score on node2: 0 -+pcmk__bundle_allocate: base-bundle-2 allocation score on node3: 0 -+pcmk__bundle_allocate: base-bundle-clone allocation score on base-bundle-0: -INFINITY -+pcmk__bundle_allocate: base-bundle-clone allocation score on base-bundle-1: -INFINITY -+pcmk__bundle_allocate: base-bundle-clone allocation score on base-bundle-2: -INFINITY -+pcmk__bundle_allocate: base-bundle-clone allocation score on node1: 0 -+pcmk__bundle_allocate: base-bundle-clone allocation score on node2: 0 -+pcmk__bundle_allocate: base-bundle-clone allocation score on node3: 0 -+pcmk__bundle_allocate: base-bundle-podman-0 allocation score on node1: 0 -+pcmk__bundle_allocate: base-bundle-podman-0 allocation score on node2: 0 -+pcmk__bundle_allocate: base-bundle-podman-0 allocation score on node3: 0 -+pcmk__bundle_allocate: base-bundle-podman-1 allocation score on node1: 0 -+pcmk__bundle_allocate: base-bundle-podman-1 allocation score on node2: 0 -+pcmk__bundle_allocate: base-bundle-podman-1 allocation score on node3: 0 -+pcmk__bundle_allocate: base-bundle-podman-2 allocation score on node1: 0 -+pcmk__bundle_allocate: base-bundle-podman-2 allocation score on node2: 0 -+pcmk__bundle_allocate: base-bundle-podman-2 allocation score on node3: 0 -+pcmk__bundle_allocate: base:0 allocation score on base-bundle-0: 501 -+pcmk__bundle_allocate: base:1 allocation score on base-bundle-1: 501 -+pcmk__bundle_allocate: base:2 allocation score on base-bundle-2: 500 -+pcmk__clone_assign: base-bundle-clone allocation score on base-bundle-0: 0 -+pcmk__clone_assign: base-bundle-clone allocation score on base-bundle-1: 0 -+pcmk__clone_assign: base-bundle-clone allocation score on base-bundle-2: 0 -+pcmk__clone_assign: base-bundle-clone allocation score on node1: -INFINITY -+pcmk__clone_assign: base-bundle-clone allocation score on node2: -INFINITY -+pcmk__clone_assign: base-bundle-clone allocation score on node3: -INFINITY -+pcmk__clone_assign: base:0 allocation score on base-bundle-0: INFINITY -+pcmk__clone_assign: base:1 allocation score on base-bundle-1: INFINITY -+pcmk__clone_assign: base:2 allocation score on base-bundle-2: INFINITY -+pcmk__primitive_assign: Fencing allocation score on node1: 0 -+pcmk__primitive_assign: Fencing allocation score on node2: 0 -+pcmk__primitive_assign: Fencing allocation score on node3: 0 -+pcmk__primitive_assign: base-bundle-0 allocation score on node1: 0 -+pcmk__primitive_assign: base-bundle-0 allocation score on node2: 0 -+pcmk__primitive_assign: base-bundle-0 allocation score on node3: 10000 -+pcmk__primitive_assign: base-bundle-1 allocation score on node1: 0 -+pcmk__primitive_assign: base-bundle-1 allocation score on node2: 10000 -+pcmk__primitive_assign: base-bundle-1 allocation score on node3: 0 -+pcmk__primitive_assign: base-bundle-2 allocation score on node1: 10000 -+pcmk__primitive_assign: base-bundle-2 allocation score on node2: 0 -+pcmk__primitive_assign: base-bundle-2 allocation score on node3: 0 -+pcmk__primitive_assign: base-bundle-podman-0 allocation score on node1: 0 -+pcmk__primitive_assign: base-bundle-podman-0 allocation score on node2: 0 -+pcmk__primitive_assign: base-bundle-podman-0 allocation score on node3: 0 -+pcmk__primitive_assign: base-bundle-podman-1 allocation score on node1: 0 -+pcmk__primitive_assign: base-bundle-podman-1 allocation score on node2: 0 -+pcmk__primitive_assign: base-bundle-podman-1 allocation score on node3: -INFINITY -+pcmk__primitive_assign: base-bundle-podman-2 allocation score on node1: 0 -+pcmk__primitive_assign: base-bundle-podman-2 allocation score on node2: -INFINITY -+pcmk__primitive_assign: base-bundle-podman-2 allocation score on node3: -INFINITY -+pcmk__primitive_assign: base:0 allocation score on base-bundle-0: INFINITY -+pcmk__primitive_assign: base:1 allocation score on base-bundle-1: INFINITY -+pcmk__primitive_assign: base:2 allocation score on base-bundle-2: INFINITY -diff --git a/cts/scheduler/scores/clone-recover-no-shuffle-4.scores b/cts/scheduler/scores/clone-recover-no-shuffle-4.scores -new file mode 100644 -index 00000000000..492dad1baa4 ---- /dev/null -+++ b/cts/scheduler/scores/clone-recover-no-shuffle-4.scores -@@ -0,0 +1,31 @@ -+ -+pcmk__clone_assign: dummy-clone allocation score on node1: 100 -+pcmk__clone_assign: dummy-clone allocation score on node2: 0 -+pcmk__clone_assign: dummy-clone allocation score on node3: 0 -+pcmk__clone_assign: dummy:0 allocation score on node1: 100 -+pcmk__clone_assign: dummy:0 allocation score on node2: 1 -+pcmk__clone_assign: dummy:0 allocation score on node3: 0 -+pcmk__clone_assign: dummy:1 allocation score on node1: 100 -+pcmk__clone_assign: dummy:1 allocation score on node2: 0 -+pcmk__clone_assign: dummy:1 allocation score on node3: 1 -+pcmk__clone_assign: dummy:2 allocation score on node1: 100 -+pcmk__clone_assign: dummy:2 allocation score on node2: 0 -+pcmk__clone_assign: dummy:2 allocation score on node3: 0 -+pcmk__primitive_assign: Fencing allocation score on node1: 0 -+pcmk__primitive_assign: Fencing allocation score on node2: 0 -+pcmk__primitive_assign: Fencing allocation score on node3: 0 -+pcmk__primitive_assign: dummy:0 allocation score on node1: 100 -+pcmk__primitive_assign: dummy:0 allocation score on node1: 100 -+pcmk__primitive_assign: dummy:0 allocation score on node2: 1 -+pcmk__primitive_assign: dummy:0 allocation score on node2: 1 -+pcmk__primitive_assign: dummy:0 allocation score on node3: 0 -+pcmk__primitive_assign: dummy:0 allocation score on node3: 0 -+pcmk__primitive_assign: dummy:1 allocation score on node1: -INFINITY -+pcmk__primitive_assign: dummy:1 allocation score on node1: 100 -+pcmk__primitive_assign: dummy:1 allocation score on node2: 0 -+pcmk__primitive_assign: dummy:1 allocation score on node2: 0 -+pcmk__primitive_assign: dummy:1 allocation score on node3: 1 -+pcmk__primitive_assign: dummy:1 allocation score on node3: 1 -+pcmk__primitive_assign: dummy:2 allocation score on node1: -INFINITY -+pcmk__primitive_assign: dummy:2 allocation score on node2: 0 -+pcmk__primitive_assign: dummy:2 allocation score on node3: -INFINITY -diff --git a/cts/scheduler/scores/clone-recover-no-shuffle-5.scores b/cts/scheduler/scores/clone-recover-no-shuffle-5.scores -new file mode 100644 -index 00000000000..eecba43fae0 ---- /dev/null -+++ b/cts/scheduler/scores/clone-recover-no-shuffle-5.scores -@@ -0,0 +1,79 @@ -+ -+pcmk__clone_assign: grp-clone allocation score on node1: 100 -+pcmk__clone_assign: grp-clone allocation score on node2: 0 -+pcmk__clone_assign: grp-clone allocation score on node3: 0 -+pcmk__clone_assign: grp:0 allocation score on node1: 100 -+pcmk__clone_assign: grp:0 allocation score on node2: 0 -+pcmk__clone_assign: grp:0 allocation score on node3: 0 -+pcmk__clone_assign: grp:1 allocation score on node1: 100 -+pcmk__clone_assign: grp:1 allocation score on node2: 0 -+pcmk__clone_assign: grp:1 allocation score on node3: 0 -+pcmk__clone_assign: grp:2 allocation score on node1: 100 -+pcmk__clone_assign: grp:2 allocation score on node2: 0 -+pcmk__clone_assign: grp:2 allocation score on node3: 0 -+pcmk__clone_assign: rsc1:0 allocation score on node1: 100 -+pcmk__clone_assign: rsc1:0 allocation score on node2: 1 -+pcmk__clone_assign: rsc1:0 allocation score on node3: 0 -+pcmk__clone_assign: rsc1:1 allocation score on node1: 100 -+pcmk__clone_assign: rsc1:1 allocation score on node2: 0 -+pcmk__clone_assign: rsc1:1 allocation score on node3: 1 -+pcmk__clone_assign: rsc1:2 allocation score on node1: 100 -+pcmk__clone_assign: rsc1:2 allocation score on node2: 0 -+pcmk__clone_assign: rsc1:2 allocation score on node3: 0 -+pcmk__clone_assign: rsc2:0 allocation score on node1: 0 -+pcmk__clone_assign: rsc2:0 allocation score on node2: 1 -+pcmk__clone_assign: rsc2:0 allocation score on node3: 0 -+pcmk__clone_assign: rsc2:1 allocation score on node1: 0 -+pcmk__clone_assign: rsc2:1 allocation score on node2: 0 -+pcmk__clone_assign: rsc2:1 allocation score on node3: 1 -+pcmk__clone_assign: rsc2:2 allocation score on node1: 0 -+pcmk__clone_assign: rsc2:2 allocation score on node2: 0 -+pcmk__clone_assign: rsc2:2 allocation score on node3: 0 -+pcmk__group_assign: grp:0 allocation score on node1: 100 -+pcmk__group_assign: grp:0 allocation score on node2: 0 -+pcmk__group_assign: grp:0 allocation score on node3: 0 -+pcmk__group_assign: grp:1 allocation score on node1: 100 -+pcmk__group_assign: grp:1 allocation score on node2: 0 -+pcmk__group_assign: grp:1 allocation score on node3: 0 -+pcmk__group_assign: grp:2 allocation score on node1: 100 -+pcmk__group_assign: grp:2 allocation score on node2: 0 -+pcmk__group_assign: grp:2 allocation score on node3: 0 -+pcmk__group_assign: rsc1:0 allocation score on node1: 100 -+pcmk__group_assign: rsc1:0 allocation score on node2: 1 -+pcmk__group_assign: rsc1:0 allocation score on node3: 0 -+pcmk__group_assign: rsc1:1 allocation score on node1: 100 -+pcmk__group_assign: rsc1:1 allocation score on node2: 0 -+pcmk__group_assign: rsc1:1 allocation score on node3: 1 -+pcmk__group_assign: rsc1:2 allocation score on node1: 100 -+pcmk__group_assign: rsc1:2 allocation score on node2: 0 -+pcmk__group_assign: rsc1:2 allocation score on node3: 0 -+pcmk__group_assign: rsc2:0 allocation score on node1: 0 -+pcmk__group_assign: rsc2:0 allocation score on node2: 1 -+pcmk__group_assign: rsc2:0 allocation score on node3: 0 -+pcmk__group_assign: rsc2:1 allocation score on node1: 0 -+pcmk__group_assign: rsc2:1 allocation score on node2: 0 -+pcmk__group_assign: rsc2:1 allocation score on node3: 1 -+pcmk__group_assign: rsc2:2 allocation score on node1: 0 -+pcmk__group_assign: rsc2:2 allocation score on node2: 0 -+pcmk__group_assign: rsc2:2 allocation score on node3: 0 -+pcmk__primitive_assign: Fencing allocation score on node1: 0 -+pcmk__primitive_assign: Fencing allocation score on node2: 0 -+pcmk__primitive_assign: Fencing allocation score on node3: 0 -+pcmk__primitive_assign: rsc1:0 allocation score on node1: 100 -+pcmk__primitive_assign: rsc1:0 allocation score on node2: 2 -+pcmk__primitive_assign: rsc1:0 allocation score on node3: 0 -+pcmk__primitive_assign: rsc1:1 allocation score on node1: 100 -+pcmk__primitive_assign: rsc1:1 allocation score on node2: 0 -+pcmk__primitive_assign: rsc1:1 allocation score on node3: 2 -+pcmk__primitive_assign: rsc1:2 allocation score on node1: 100 -+pcmk__primitive_assign: rsc1:2 allocation score on node2: 0 -+pcmk__primitive_assign: rsc1:2 allocation score on node3: 0 -+pcmk__primitive_assign: rsc2:0 allocation score on node1: 0 -+pcmk__primitive_assign: rsc2:0 allocation score on node2: -INFINITY -+pcmk__primitive_assign: rsc2:0 allocation score on node3: -INFINITY -+pcmk__primitive_assign: rsc2:1 allocation score on node1: 0 -+pcmk__primitive_assign: rsc2:1 allocation score on node2: -INFINITY -+pcmk__primitive_assign: rsc2:1 allocation score on node3: -INFINITY -+pcmk__primitive_assign: rsc2:2 allocation score on node1: 0 -+pcmk__primitive_assign: rsc2:2 allocation score on node2: -INFINITY -+pcmk__primitive_assign: rsc2:2 allocation score on node3: -INFINITY -diff --git a/cts/scheduler/scores/clone-recover-no-shuffle-6.scores b/cts/scheduler/scores/clone-recover-no-shuffle-6.scores -new file mode 100644 -index 00000000000..643e30f9d18 ---- /dev/null -+++ b/cts/scheduler/scores/clone-recover-no-shuffle-6.scores -@@ -0,0 +1,70 @@ -+ -+pcmk__bundle_allocate: base-bundle allocation score on node1: 100 -+pcmk__bundle_allocate: base-bundle allocation score on node2: 0 -+pcmk__bundle_allocate: base-bundle allocation score on node3: 0 -+pcmk__bundle_allocate: base-bundle-0 allocation score on node1: 0 -+pcmk__bundle_allocate: base-bundle-0 allocation score on node2: 0 -+pcmk__bundle_allocate: base-bundle-0 allocation score on node3: 0 -+pcmk__bundle_allocate: base-bundle-1 allocation score on node1: 0 -+pcmk__bundle_allocate: base-bundle-1 allocation score on node2: 0 -+pcmk__bundle_allocate: base-bundle-1 allocation score on node3: 0 -+pcmk__bundle_allocate: base-bundle-2 allocation score on node1: 0 -+pcmk__bundle_allocate: base-bundle-2 allocation score on node2: 0 -+pcmk__bundle_allocate: base-bundle-2 allocation score on node3: 0 -+pcmk__bundle_allocate: base-bundle-clone allocation score on base-bundle-0: -INFINITY -+pcmk__bundle_allocate: base-bundle-clone allocation score on base-bundle-1: -INFINITY -+pcmk__bundle_allocate: base-bundle-clone allocation score on base-bundle-2: -INFINITY -+pcmk__bundle_allocate: base-bundle-clone allocation score on node1: 0 -+pcmk__bundle_allocate: base-bundle-clone allocation score on node2: 0 -+pcmk__bundle_allocate: base-bundle-clone allocation score on node3: 0 -+pcmk__bundle_allocate: base-bundle-podman-0 allocation score on node1: 100 -+pcmk__bundle_allocate: base-bundle-podman-0 allocation score on node2: 0 -+pcmk__bundle_allocate: base-bundle-podman-0 allocation score on node3: 0 -+pcmk__bundle_allocate: base-bundle-podman-1 allocation score on node1: 100 -+pcmk__bundle_allocate: base-bundle-podman-1 allocation score on node2: 0 -+pcmk__bundle_allocate: base-bundle-podman-1 allocation score on node3: 0 -+pcmk__bundle_allocate: base-bundle-podman-2 allocation score on node1: 100 -+pcmk__bundle_allocate: base-bundle-podman-2 allocation score on node2: 0 -+pcmk__bundle_allocate: base-bundle-podman-2 allocation score on node3: 0 -+pcmk__bundle_allocate: base:0 allocation score on base-bundle-0: 501 -+pcmk__bundle_allocate: base:1 allocation score on base-bundle-1: 501 -+pcmk__bundle_allocate: base:2 allocation score on base-bundle-2: 500 -+pcmk__clone_assign: base-bundle-clone allocation score on base-bundle-0: 0 -+pcmk__clone_assign: base-bundle-clone allocation score on base-bundle-1: 0 -+pcmk__clone_assign: base-bundle-clone allocation score on base-bundle-2: 0 -+pcmk__clone_assign: base-bundle-clone allocation score on node1: -INFINITY -+pcmk__clone_assign: base-bundle-clone allocation score on node2: -INFINITY -+pcmk__clone_assign: base-bundle-clone allocation score on node3: -INFINITY -+pcmk__clone_assign: base:0 allocation score on base-bundle-0: INFINITY -+pcmk__clone_assign: base:1 allocation score on base-bundle-1: INFINITY -+pcmk__clone_assign: base:2 allocation score on base-bundle-2: INFINITY -+pcmk__primitive_assign: Fencing allocation score on node1: 0 -+pcmk__primitive_assign: Fencing allocation score on node2: 0 -+pcmk__primitive_assign: Fencing allocation score on node3: 0 -+pcmk__primitive_assign: base-bundle-0 allocation score on node1: 10000 -+pcmk__primitive_assign: base-bundle-0 allocation score on node2: 0 -+pcmk__primitive_assign: base-bundle-0 allocation score on node3: 0 -+pcmk__primitive_assign: base-bundle-1 allocation score on node1: 0 -+pcmk__primitive_assign: base-bundle-1 allocation score on node2: 0 -+pcmk__primitive_assign: base-bundle-1 allocation score on node3: 10000 -+pcmk__primitive_assign: base-bundle-2 allocation score on node1: 0 -+pcmk__primitive_assign: base-bundle-2 allocation score on node2: 10000 -+pcmk__primitive_assign: base-bundle-2 allocation score on node3: 0 -+pcmk__primitive_assign: base-bundle-podman-0 allocation score on node1: 100 -+pcmk__primitive_assign: base-bundle-podman-0 allocation score on node1: 100 -+pcmk__primitive_assign: base-bundle-podman-0 allocation score on node2: 0 -+pcmk__primitive_assign: base-bundle-podman-0 allocation score on node2: 0 -+pcmk__primitive_assign: base-bundle-podman-0 allocation score on node3: 0 -+pcmk__primitive_assign: base-bundle-podman-0 allocation score on node3: 0 -+pcmk__primitive_assign: base-bundle-podman-1 allocation score on node1: -INFINITY -+pcmk__primitive_assign: base-bundle-podman-1 allocation score on node1: 100 -+pcmk__primitive_assign: base-bundle-podman-1 allocation score on node2: 0 -+pcmk__primitive_assign: base-bundle-podman-1 allocation score on node2: 0 -+pcmk__primitive_assign: base-bundle-podman-1 allocation score on node3: 0 -+pcmk__primitive_assign: base-bundle-podman-1 allocation score on node3: 0 -+pcmk__primitive_assign: base-bundle-podman-2 allocation score on node1: -INFINITY -+pcmk__primitive_assign: base-bundle-podman-2 allocation score on node2: 0 -+pcmk__primitive_assign: base-bundle-podman-2 allocation score on node3: -INFINITY -+pcmk__primitive_assign: base:0 allocation score on base-bundle-0: INFINITY -+pcmk__primitive_assign: base:1 allocation score on base-bundle-1: INFINITY -+pcmk__primitive_assign: base:2 allocation score on base-bundle-2: INFINITY -diff --git a/cts/scheduler/scores/clone-recover-no-shuffle-7.scores b/cts/scheduler/scores/clone-recover-no-shuffle-7.scores -new file mode 100644 -index 00000000000..fc45bf740fd ---- /dev/null -+++ b/cts/scheduler/scores/clone-recover-no-shuffle-7.scores -@@ -0,0 +1,34 @@ -+ -+dummy:0 promotion score on node1: 15 -+dummy:1 promotion score on node2: 10 -+dummy:2 promotion score on node3: 5 -+pcmk__clone_assign: dummy-clone allocation score on node1: 0 -+pcmk__clone_assign: dummy-clone allocation score on node2: 0 -+pcmk__clone_assign: dummy-clone allocation score on node3: 0 -+pcmk__clone_assign: dummy:0 allocation score on node1: 15 -+pcmk__clone_assign: dummy:0 allocation score on node2: 0 -+pcmk__clone_assign: dummy:0 allocation score on node3: 6 -+pcmk__clone_assign: dummy:1 allocation score on node1: 15 -+pcmk__clone_assign: dummy:1 allocation score on node2: 11 -+pcmk__clone_assign: dummy:1 allocation score on node3: 0 -+pcmk__clone_assign: dummy:2 allocation score on node1: 15 -+pcmk__clone_assign: dummy:2 allocation score on node2: 10 -+pcmk__clone_assign: dummy:2 allocation score on node3: 5 -+pcmk__primitive_assign: Fencing allocation score on node1: 0 -+pcmk__primitive_assign: Fencing allocation score on node2: 0 -+pcmk__primitive_assign: Fencing allocation score on node3: 0 -+pcmk__primitive_assign: dummy:0 allocation score on node1: 15 -+pcmk__primitive_assign: dummy:0 allocation score on node1: 15 -+pcmk__primitive_assign: dummy:0 allocation score on node2: 0 -+pcmk__primitive_assign: dummy:0 allocation score on node2: 0 -+pcmk__primitive_assign: dummy:0 allocation score on node3: 6 -+pcmk__primitive_assign: dummy:0 allocation score on node3: 6 -+pcmk__primitive_assign: dummy:1 allocation score on node1: -INFINITY -+pcmk__primitive_assign: dummy:1 allocation score on node1: 15 -+pcmk__primitive_assign: dummy:1 allocation score on node2: 11 -+pcmk__primitive_assign: dummy:1 allocation score on node2: 11 -+pcmk__primitive_assign: dummy:1 allocation score on node3: 0 -+pcmk__primitive_assign: dummy:1 allocation score on node3: 0 -+pcmk__primitive_assign: dummy:2 allocation score on node1: -INFINITY -+pcmk__primitive_assign: dummy:2 allocation score on node2: -INFINITY -+pcmk__primitive_assign: dummy:2 allocation score on node3: 5 -diff --git a/cts/scheduler/scores/clone-recover-no-shuffle-8.scores b/cts/scheduler/scores/clone-recover-no-shuffle-8.scores -new file mode 100644 -index 00000000000..56d4cc8395a ---- /dev/null -+++ b/cts/scheduler/scores/clone-recover-no-shuffle-8.scores -@@ -0,0 +1,82 @@ -+ -+grp:0 promotion score on node3: 10 -+grp:1 promotion score on node2: 20 -+grp:2 promotion score on node1: 30 -+pcmk__clone_assign: grp-clone allocation score on node1: 0 -+pcmk__clone_assign: grp-clone allocation score on node2: 0 -+pcmk__clone_assign: grp-clone allocation score on node3: 0 -+pcmk__clone_assign: grp:0 allocation score on node1: 30 -+pcmk__clone_assign: grp:0 allocation score on node2: 0 -+pcmk__clone_assign: grp:0 allocation score on node3: 10 -+pcmk__clone_assign: grp:1 allocation score on node1: 30 -+pcmk__clone_assign: grp:1 allocation score on node2: 20 -+pcmk__clone_assign: grp:1 allocation score on node3: 0 -+pcmk__clone_assign: grp:2 allocation score on node1: 30 -+pcmk__clone_assign: grp:2 allocation score on node2: 20 -+pcmk__clone_assign: grp:2 allocation score on node3: 10 -+pcmk__clone_assign: rsc1:0 allocation score on node1: 0 -+pcmk__clone_assign: rsc1:0 allocation score on node2: 0 -+pcmk__clone_assign: rsc1:0 allocation score on node3: 1 -+pcmk__clone_assign: rsc1:1 allocation score on node1: 0 -+pcmk__clone_assign: rsc1:1 allocation score on node2: 1 -+pcmk__clone_assign: rsc1:1 allocation score on node3: 0 -+pcmk__clone_assign: rsc1:2 allocation score on node1: 0 -+pcmk__clone_assign: rsc1:2 allocation score on node2: 0 -+pcmk__clone_assign: rsc1:2 allocation score on node3: 0 -+pcmk__clone_assign: rsc2:0 allocation score on node1: 0 -+pcmk__clone_assign: rsc2:0 allocation score on node2: 0 -+pcmk__clone_assign: rsc2:0 allocation score on node3: 1 -+pcmk__clone_assign: rsc2:1 allocation score on node1: 0 -+pcmk__clone_assign: rsc2:1 allocation score on node2: 1 -+pcmk__clone_assign: rsc2:1 allocation score on node3: 0 -+pcmk__clone_assign: rsc2:2 allocation score on node1: 0 -+pcmk__clone_assign: rsc2:2 allocation score on node2: 0 -+pcmk__clone_assign: rsc2:2 allocation score on node3: 0 -+pcmk__group_assign: grp:0 allocation score on node1: 30 -+pcmk__group_assign: grp:0 allocation score on node2: 0 -+pcmk__group_assign: grp:0 allocation score on node3: 10 -+pcmk__group_assign: grp:1 allocation score on node1: 30 -+pcmk__group_assign: grp:1 allocation score on node2: 20 -+pcmk__group_assign: grp:1 allocation score on node3: -INFINITY -+pcmk__group_assign: grp:2 allocation score on node1: 30 -+pcmk__group_assign: grp:2 allocation score on node2: -INFINITY -+pcmk__group_assign: grp:2 allocation score on node3: -INFINITY -+pcmk__group_assign: rsc1:0 allocation score on node1: 0 -+pcmk__group_assign: rsc1:0 allocation score on node2: 0 -+pcmk__group_assign: rsc1:0 allocation score on node3: 1 -+pcmk__group_assign: rsc1:1 allocation score on node1: 0 -+pcmk__group_assign: rsc1:1 allocation score on node2: 1 -+pcmk__group_assign: rsc1:1 allocation score on node3: -INFINITY -+pcmk__group_assign: rsc1:2 allocation score on node1: 0 -+pcmk__group_assign: rsc1:2 allocation score on node2: -INFINITY -+pcmk__group_assign: rsc1:2 allocation score on node3: -INFINITY -+pcmk__group_assign: rsc2:0 allocation score on node1: 0 -+pcmk__group_assign: rsc2:0 allocation score on node2: 0 -+pcmk__group_assign: rsc2:0 allocation score on node3: 1 -+pcmk__group_assign: rsc2:1 allocation score on node1: 0 -+pcmk__group_assign: rsc2:1 allocation score on node2: 1 -+pcmk__group_assign: rsc2:1 allocation score on node3: -INFINITY -+pcmk__group_assign: rsc2:2 allocation score on node1: 0 -+pcmk__group_assign: rsc2:2 allocation score on node2: -INFINITY -+pcmk__group_assign: rsc2:2 allocation score on node3: -INFINITY -+pcmk__primitive_assign: Fencing allocation score on node1: 0 -+pcmk__primitive_assign: Fencing allocation score on node2: 0 -+pcmk__primitive_assign: Fencing allocation score on node3: 0 -+pcmk__primitive_assign: rsc1:0 allocation score on node1: 0 -+pcmk__primitive_assign: rsc1:0 allocation score on node2: 0 -+pcmk__primitive_assign: rsc1:0 allocation score on node3: 2 -+pcmk__primitive_assign: rsc1:1 allocation score on node1: 0 -+pcmk__primitive_assign: rsc1:1 allocation score on node2: 2 -+pcmk__primitive_assign: rsc1:1 allocation score on node3: -INFINITY -+pcmk__primitive_assign: rsc1:2 allocation score on node1: 0 -+pcmk__primitive_assign: rsc1:2 allocation score on node2: -INFINITY -+pcmk__primitive_assign: rsc1:2 allocation score on node3: -INFINITY -+pcmk__primitive_assign: rsc2:0 allocation score on node1: -INFINITY -+pcmk__primitive_assign: rsc2:0 allocation score on node2: -INFINITY -+pcmk__primitive_assign: rsc2:0 allocation score on node3: 1 -+pcmk__primitive_assign: rsc2:1 allocation score on node1: -INFINITY -+pcmk__primitive_assign: rsc2:1 allocation score on node2: 1 -+pcmk__primitive_assign: rsc2:1 allocation score on node3: -INFINITY -+pcmk__primitive_assign: rsc2:2 allocation score on node1: 0 -+pcmk__primitive_assign: rsc2:2 allocation score on node2: -INFINITY -+pcmk__primitive_assign: rsc2:2 allocation score on node3: -INFINITY -diff --git a/cts/scheduler/scores/clone-recover-no-shuffle-9.scores b/cts/scheduler/scores/clone-recover-no-shuffle-9.scores -new file mode 100644 -index 00000000000..947c86b262c ---- /dev/null -+++ b/cts/scheduler/scores/clone-recover-no-shuffle-9.scores -@@ -0,0 +1,67 @@ -+ -+base:0 promotion score on base-bundle-0: 5 -+base:1 promotion score on base-bundle-1: 10 -+base:2 promotion score on base-bundle-2: 15 -+pcmk__bundle_allocate: base-bundle allocation score on node1: 0 -+pcmk__bundle_allocate: base-bundle allocation score on node2: 0 -+pcmk__bundle_allocate: base-bundle allocation score on node3: 0 -+pcmk__bundle_allocate: base-bundle-0 allocation score on node1: 0 -+pcmk__bundle_allocate: base-bundle-0 allocation score on node2: 0 -+pcmk__bundle_allocate: base-bundle-0 allocation score on node3: 0 -+pcmk__bundle_allocate: base-bundle-1 allocation score on node1: 0 -+pcmk__bundle_allocate: base-bundle-1 allocation score on node2: 0 -+pcmk__bundle_allocate: base-bundle-1 allocation score on node3: 0 -+pcmk__bundle_allocate: base-bundle-2 allocation score on node1: 0 -+pcmk__bundle_allocate: base-bundle-2 allocation score on node2: 0 -+pcmk__bundle_allocate: base-bundle-2 allocation score on node3: 0 -+pcmk__bundle_allocate: base-bundle-clone allocation score on base-bundle-0: -INFINITY -+pcmk__bundle_allocate: base-bundle-clone allocation score on base-bundle-1: -INFINITY -+pcmk__bundle_allocate: base-bundle-clone allocation score on base-bundle-2: -INFINITY -+pcmk__bundle_allocate: base-bundle-clone allocation score on node1: 0 -+pcmk__bundle_allocate: base-bundle-clone allocation score on node2: 0 -+pcmk__bundle_allocate: base-bundle-clone allocation score on node3: 0 -+pcmk__bundle_allocate: base-bundle-podman-0 allocation score on node1: 0 -+pcmk__bundle_allocate: base-bundle-podman-0 allocation score on node2: 0 -+pcmk__bundle_allocate: base-bundle-podman-0 allocation score on node3: 0 -+pcmk__bundle_allocate: base-bundle-podman-1 allocation score on node1: 0 -+pcmk__bundle_allocate: base-bundle-podman-1 allocation score on node2: 0 -+pcmk__bundle_allocate: base-bundle-podman-1 allocation score on node3: 0 -+pcmk__bundle_allocate: base-bundle-podman-2 allocation score on node1: 0 -+pcmk__bundle_allocate: base-bundle-podman-2 allocation score on node2: 0 -+pcmk__bundle_allocate: base-bundle-podman-2 allocation score on node3: 0 -+pcmk__bundle_allocate: base:0 allocation score on base-bundle-0: 501 -+pcmk__bundle_allocate: base:1 allocation score on base-bundle-1: 501 -+pcmk__bundle_allocate: base:2 allocation score on base-bundle-2: 500 -+pcmk__clone_assign: base-bundle-clone allocation score on base-bundle-0: 0 -+pcmk__clone_assign: base-bundle-clone allocation score on base-bundle-1: 0 -+pcmk__clone_assign: base-bundle-clone allocation score on base-bundle-2: 0 -+pcmk__clone_assign: base-bundle-clone allocation score on node1: -INFINITY -+pcmk__clone_assign: base-bundle-clone allocation score on node2: -INFINITY -+pcmk__clone_assign: base-bundle-clone allocation score on node3: -INFINITY -+pcmk__clone_assign: base:0 allocation score on base-bundle-0: INFINITY -+pcmk__clone_assign: base:1 allocation score on base-bundle-1: INFINITY -+pcmk__clone_assign: base:2 allocation score on base-bundle-2: INFINITY -+pcmk__primitive_assign: Fencing allocation score on node1: 0 -+pcmk__primitive_assign: Fencing allocation score on node2: 0 -+pcmk__primitive_assign: Fencing allocation score on node3: 0 -+pcmk__primitive_assign: base-bundle-0 allocation score on node1: 0 -+pcmk__primitive_assign: base-bundle-0 allocation score on node2: 0 -+pcmk__primitive_assign: base-bundle-0 allocation score on node3: 10000 -+pcmk__primitive_assign: base-bundle-1 allocation score on node1: 0 -+pcmk__primitive_assign: base-bundle-1 allocation score on node2: 10000 -+pcmk__primitive_assign: base-bundle-1 allocation score on node3: 0 -+pcmk__primitive_assign: base-bundle-2 allocation score on node1: 10000 -+pcmk__primitive_assign: base-bundle-2 allocation score on node2: 0 -+pcmk__primitive_assign: base-bundle-2 allocation score on node3: 0 -+pcmk__primitive_assign: base-bundle-podman-0 allocation score on node1: 0 -+pcmk__primitive_assign: base-bundle-podman-0 allocation score on node2: 0 -+pcmk__primitive_assign: base-bundle-podman-0 allocation score on node3: 0 -+pcmk__primitive_assign: base-bundle-podman-1 allocation score on node1: 0 -+pcmk__primitive_assign: base-bundle-podman-1 allocation score on node2: 0 -+pcmk__primitive_assign: base-bundle-podman-1 allocation score on node3: -INFINITY -+pcmk__primitive_assign: base-bundle-podman-2 allocation score on node1: 0 -+pcmk__primitive_assign: base-bundle-podman-2 allocation score on node2: -INFINITY -+pcmk__primitive_assign: base-bundle-podman-2 allocation score on node3: -INFINITY -+pcmk__primitive_assign: base:0 allocation score on base-bundle-0: INFINITY -+pcmk__primitive_assign: base:1 allocation score on base-bundle-1: INFINITY -+pcmk__primitive_assign: base:2 allocation score on base-bundle-2: INFINITY -diff --git a/cts/scheduler/summary/clone-recover-no-shuffle-1.summary b/cts/scheduler/summary/clone-recover-no-shuffle-1.summary -new file mode 100644 -index 00000000000..0b6866ec16c ---- /dev/null -+++ b/cts/scheduler/summary/clone-recover-no-shuffle-1.summary -@@ -0,0 +1,29 @@ -+Using the original execution date of: 2023-06-21 00:59:59Z -+Current cluster status: -+ * Node List: -+ * Online: [ node1 node2 node3 ] -+ -+ * Full List of Resources: -+ * Fencing (stonith:fence_xvm): Started node2 -+ * Clone Set: dummy-clone [dummy]: -+ * Started: [ node2 node3 ] -+ * Stopped: [ node1 ] -+ -+Transition Summary: -+ * Start dummy:2 ( node1 ) -+ -+Executing Cluster Transition: -+ * Pseudo action: dummy-clone_start_0 -+ * Resource action: dummy start on node1 -+ * Pseudo action: dummy-clone_running_0 -+ * Resource action: dummy monitor=10000 on node1 -+Using the original execution date of: 2023-06-21 00:59:59Z -+ -+Revised Cluster Status: -+ * Node List: -+ * Online: [ node1 node2 node3 ] -+ -+ * Full List of Resources: -+ * Fencing (stonith:fence_xvm): Started node2 -+ * Clone Set: dummy-clone [dummy]: -+ * Started: [ node1 node2 node3 ] -diff --git a/cts/scheduler/summary/clone-recover-no-shuffle-10.summary b/cts/scheduler/summary/clone-recover-no-shuffle-10.summary -new file mode 100644 -index 00000000000..5b0f9b6d685 ---- /dev/null -+++ b/cts/scheduler/summary/clone-recover-no-shuffle-10.summary -@@ -0,0 +1,29 @@ -+Current cluster status: -+ * Node List: -+ * Online: [ node1 node2 node3 ] -+ -+ * Full List of Resources: -+ * Fencing (stonith:fence_xvm): Started node2 -+ * Clone Set: dummy-clone [dummy] (promotable): -+ * Promoted: [ node2 ] -+ * Unpromoted: [ node3 ] -+ * Stopped: [ node1 ] -+ -+Transition Summary: -+ * Start dummy:2 ( node1 ) -+ -+Executing Cluster Transition: -+ * Pseudo action: dummy-clone_start_0 -+ * Resource action: dummy start on node1 -+ * Pseudo action: dummy-clone_running_0 -+ * Resource action: dummy monitor=11000 on node1 -+ -+Revised Cluster Status: -+ * Node List: -+ * Online: [ node1 node2 node3 ] -+ -+ * Full List of Resources: -+ * Fencing (stonith:fence_xvm): Started node2 -+ * Clone Set: dummy-clone [dummy] (promotable): -+ * Promoted: [ node2 ] -+ * Unpromoted: [ node1 node3 ] -diff --git a/cts/scheduler/summary/clone-recover-no-shuffle-11.summary b/cts/scheduler/summary/clone-recover-no-shuffle-11.summary -new file mode 100644 -index 00000000000..e0bdb61d605 ---- /dev/null -+++ b/cts/scheduler/summary/clone-recover-no-shuffle-11.summary -@@ -0,0 +1,34 @@ -+Current cluster status: -+ * Node List: -+ * Online: [ node1 node2 node3 ] -+ -+ * Full List of Resources: -+ * Fencing (stonith:fence_xvm): Started node2 -+ * Clone Set: grp-clone [grp] (promotable): -+ * Promoted: [ node2 ] -+ * Unpromoted: [ node3 ] -+ * Stopped: [ node1 ] -+ -+Transition Summary: -+ * Start rsc1:2 ( node1 ) -+ * Start rsc2:2 ( node1 ) -+ -+Executing Cluster Transition: -+ * Pseudo action: grp-clone_start_0 -+ * Pseudo action: grp:2_start_0 -+ * Resource action: rsc1 start on node1 -+ * Resource action: rsc2 start on node1 -+ * Pseudo action: grp:2_running_0 -+ * Resource action: rsc1 monitor=11000 on node1 -+ * Resource action: rsc2 monitor=11000 on node1 -+ * Pseudo action: grp-clone_running_0 -+ -+Revised Cluster Status: -+ * Node List: -+ * Online: [ node1 node2 node3 ] -+ -+ * Full List of Resources: -+ * Fencing (stonith:fence_xvm): Started node2 -+ * Clone Set: grp-clone [grp] (promotable): -+ * Promoted: [ node2 ] -+ * Unpromoted: [ node1 node3 ] -diff --git a/cts/scheduler/summary/clone-recover-no-shuffle-12.summary b/cts/scheduler/summary/clone-recover-no-shuffle-12.summary -new file mode 100644 -index 00000000000..6e55a0b7f2f ---- /dev/null -+++ b/cts/scheduler/summary/clone-recover-no-shuffle-12.summary -@@ -0,0 +1,43 @@ -+Current cluster status: -+ * Node List: -+ * Online: [ node1 node2 node3 ] -+ * GuestOnline: [ base-bundle-0 base-bundle-1 ] -+ -+ * Full List of Resources: -+ * Fencing (stonith:fence_xvm): Started node2 -+ * Container bundle set: base-bundle [localhost/pcmktest]: -+ * base-bundle-0 (ocf:pacemaker:Stateful): Unpromoted node3 -+ * base-bundle-1 (ocf:pacemaker:Stateful): Promoted node2 -+ * base-bundle-2 (ocf:pacemaker:Stateful): Stopped -+ -+Transition Summary: -+ * Start base-bundle-podman-2 ( node1 ) -+ * Start base-bundle-2 ( node1 ) -+ * Start base:2 ( base-bundle-2 ) -+ -+Executing Cluster Transition: -+ * Pseudo action: base-bundle_start_0 -+ * Pseudo action: base-bundle-clone_start_0 -+ * Resource action: base-bundle-podman-2 start on node1 -+ * Resource action: base-bundle-2 monitor on node3 -+ * Resource action: base-bundle-2 monitor on node2 -+ * Resource action: base-bundle-2 monitor on node1 -+ * Resource action: base-bundle-podman-2 monitor=60000 on node1 -+ * Resource action: base-bundle-2 start on node1 -+ * Resource action: base start on base-bundle-2 -+ * Pseudo action: base-bundle-clone_running_0 -+ * Resource action: base-bundle-2 monitor=30000 on node1 -+ * Pseudo action: base-bundle_running_0 -+ * Resource action: base monitor=16000 on base-bundle-2 -+ -+Revised Cluster Status: -+ * Node List: -+ * Online: [ node1 node2 node3 ] -+ * GuestOnline: [ base-bundle-0 base-bundle-1 base-bundle-2 ] -+ -+ * Full List of Resources: -+ * Fencing (stonith:fence_xvm): Started node2 -+ * Container bundle set: base-bundle [localhost/pcmktest]: -+ * base-bundle-0 (ocf:pacemaker:Stateful): Unpromoted node3 -+ * base-bundle-1 (ocf:pacemaker:Stateful): Promoted node2 -+ * base-bundle-2 (ocf:pacemaker:Stateful): Unpromoted node1 -diff --git a/cts/scheduler/summary/clone-recover-no-shuffle-2.summary b/cts/scheduler/summary/clone-recover-no-shuffle-2.summary -new file mode 100644 -index 00000000000..8b18120ad8d ---- /dev/null -+++ b/cts/scheduler/summary/clone-recover-no-shuffle-2.summary -@@ -0,0 +1,32 @@ -+Current cluster status: -+ * Node List: -+ * Online: [ node1 node2 node3 ] -+ -+ * Full List of Resources: -+ * Fencing (stonith:fence_xvm): Started node2 -+ * Clone Set: grp-clone [grp]: -+ * Started: [ node2 node3 ] -+ * Stopped: [ node1 ] -+ -+Transition Summary: -+ * Start rsc1:2 ( node1 ) -+ * Start rsc2:2 ( node1 ) -+ -+Executing Cluster Transition: -+ * Pseudo action: grp-clone_start_0 -+ * Pseudo action: grp:2_start_0 -+ * Resource action: rsc1 start on node1 -+ * Resource action: rsc2 start on node1 -+ * Pseudo action: grp:2_running_0 -+ * Resource action: rsc1 monitor=10000 on node1 -+ * Resource action: rsc2 monitor=10000 on node1 -+ * Pseudo action: grp-clone_running_0 -+ -+Revised Cluster Status: -+ * Node List: -+ * Online: [ node1 node2 node3 ] -+ -+ * Full List of Resources: -+ * Fencing (stonith:fence_xvm): Started node2 -+ * Clone Set: grp-clone [grp]: -+ * Started: [ node1 node2 node3 ] -diff --git a/cts/scheduler/summary/clone-recover-no-shuffle-3.summary b/cts/scheduler/summary/clone-recover-no-shuffle-3.summary -new file mode 100644 -index 00000000000..5702177e33d ---- /dev/null -+++ b/cts/scheduler/summary/clone-recover-no-shuffle-3.summary -@@ -0,0 +1,42 @@ -+Current cluster status: -+ * Node List: -+ * Online: [ node1 node2 node3 ] -+ * GuestOnline: [ base-bundle-0 base-bundle-1 ] -+ -+ * Full List of Resources: -+ * Fencing (stonith:fence_xvm): Started node2 -+ * Container bundle set: base-bundle [localhost/pcmktest]: -+ * base-bundle-0 (ocf:pacemaker:Stateful): Started node3 -+ * base-bundle-1 (ocf:pacemaker:Stateful): Started node2 -+ * base-bundle-2 (ocf:pacemaker:Stateful): Stopped -+ -+Transition Summary: -+ * Start base-bundle-podman-2 ( node1 ) -+ * Start base-bundle-2 ( node1 ) -+ * Start base:2 ( base-bundle-2 ) -+ -+Executing Cluster Transition: -+ * Pseudo action: base-bundle_start_0 -+ * Pseudo action: base-bundle-clone_start_0 -+ * Resource action: base-bundle-podman-2 start on node1 -+ * Resource action: base-bundle-2 monitor on node3 -+ * Resource action: base-bundle-2 monitor on node2 -+ * Resource action: base-bundle-2 monitor on node1 -+ * Resource action: base-bundle-podman-2 monitor=60000 on node1 -+ * Resource action: base-bundle-2 start on node1 -+ * Resource action: base start on base-bundle-2 -+ * Pseudo action: base-bundle-clone_running_0 -+ * Resource action: base-bundle-2 monitor=30000 on node1 -+ * Pseudo action: base-bundle_running_0 -+ -+Revised Cluster Status: -+ * Node List: -+ * Online: [ node1 node2 node3 ] -+ * GuestOnline: [ base-bundle-0 base-bundle-1 base-bundle-2 ] -+ -+ * Full List of Resources: -+ * Fencing (stonith:fence_xvm): Started node2 -+ * Container bundle set: base-bundle [localhost/pcmktest]: -+ * base-bundle-0 (ocf:pacemaker:Stateful): Started node3 -+ * base-bundle-1 (ocf:pacemaker:Stateful): Started node2 -+ * base-bundle-2 (ocf:pacemaker:Stateful): Started node1 -diff --git a/cts/scheduler/summary/clone-recover-no-shuffle-4.summary b/cts/scheduler/summary/clone-recover-no-shuffle-4.summary -new file mode 100644 -index 00000000000..944bcb834b3 ---- /dev/null -+++ b/cts/scheduler/summary/clone-recover-no-shuffle-4.summary -@@ -0,0 +1,35 @@ -+Using the original execution date of: 2023-06-21 00:59:59Z -+Current cluster status: -+ * Node List: -+ * Online: [ node1 node2 node3 ] -+ -+ * Full List of Resources: -+ * Fencing (stonith:fence_xvm): Started node2 -+ * Clone Set: dummy-clone [dummy]: -+ * Started: [ node2 node3 ] -+ * Stopped: [ node1 ] -+ -+Transition Summary: -+ * Move dummy:0 ( node2 -> node1 ) -+ * Start dummy:2 ( node2 ) -+ -+Executing Cluster Transition: -+ * Pseudo action: dummy-clone_stop_0 -+ * Resource action: dummy stop on node2 -+ * Pseudo action: dummy-clone_stopped_0 -+ * Pseudo action: dummy-clone_start_0 -+ * Resource action: dummy start on node1 -+ * Resource action: dummy start on node2 -+ * Pseudo action: dummy-clone_running_0 -+ * Resource action: dummy monitor=10000 on node1 -+ * Resource action: dummy monitor=10000 on node2 -+Using the original execution date of: 2023-06-21 00:59:59Z -+ -+Revised Cluster Status: -+ * Node List: -+ * Online: [ node1 node2 node3 ] -+ -+ * Full List of Resources: -+ * Fencing (stonith:fence_xvm): Started node2 -+ * Clone Set: dummy-clone [dummy]: -+ * Started: [ node1 node2 node3 ] -diff --git a/cts/scheduler/summary/clone-recover-no-shuffle-5.summary b/cts/scheduler/summary/clone-recover-no-shuffle-5.summary -new file mode 100644 -index 00000000000..e84d0a574de ---- /dev/null -+++ b/cts/scheduler/summary/clone-recover-no-shuffle-5.summary -@@ -0,0 +1,59 @@ -+Current cluster status: -+ * Node List: -+ * Online: [ node1 node2 node3 ] -+ -+ * Full List of Resources: -+ * Fencing (stonith:fence_xvm): Started node2 -+ * Clone Set: grp-clone [grp]: -+ * Started: [ node2 node3 ] -+ * Stopped: [ node1 ] -+ -+Transition Summary: -+ * Move rsc1:0 ( node2 -> node1 ) -+ * Move rsc2:0 ( node2 -> node1 ) -+ * Move rsc1:1 ( node3 -> node1 ) -+ * Move rsc2:1 ( node3 -> node1 ) -+ * Start rsc1:2 ( node1 ) -+ * Start rsc2:2 ( node1 ) -+ -+Executing Cluster Transition: -+ * Pseudo action: grp-clone_stop_0 -+ * Pseudo action: grp:0_stop_0 -+ * Resource action: rsc2 stop on node2 -+ * Pseudo action: grp:1_stop_0 -+ * Resource action: rsc2 stop on node3 -+ * Resource action: rsc1 stop on node2 -+ * Resource action: rsc1 stop on node3 -+ * Pseudo action: grp:0_stopped_0 -+ * Pseudo action: grp:1_stopped_0 -+ * Pseudo action: grp-clone_stopped_0 -+ * Pseudo action: grp-clone_start_0 -+ * Pseudo action: grp:0_start_0 -+ * Resource action: rsc1 start on node1 -+ * Resource action: rsc2 start on node1 -+ * Pseudo action: grp:1_start_0 -+ * Resource action: rsc1 start on node1 -+ * Resource action: rsc2 start on node1 -+ * Pseudo action: grp:2_start_0 -+ * Resource action: rsc1 start on node1 -+ * Resource action: rsc2 start on node1 -+ * Pseudo action: grp:0_running_0 -+ * Resource action: rsc1 monitor=10000 on node1 -+ * Resource action: rsc2 monitor=10000 on node1 -+ * Pseudo action: grp:1_running_0 -+ * Resource action: rsc1 monitor=10000 on node1 -+ * Resource action: rsc2 monitor=10000 on node1 -+ * Pseudo action: grp:2_running_0 -+ * Resource action: rsc1 monitor=10000 on node1 -+ * Resource action: rsc2 monitor=10000 on node1 -+ * Pseudo action: grp-clone_running_0 -+ -+Revised Cluster Status: -+ * Node List: -+ * Online: [ node1 node2 node3 ] -+ -+ * Full List of Resources: -+ * Fencing (stonith:fence_xvm): Started node2 -+ * Clone Set: grp-clone [grp]: -+ * Started: [ node1 ] -+ * Stopped: [ node2 node3 ] -diff --git a/cts/scheduler/summary/clone-recover-no-shuffle-6.summary b/cts/scheduler/summary/clone-recover-no-shuffle-6.summary -new file mode 100644 -index 00000000000..19a957e15fb ---- /dev/null -+++ b/cts/scheduler/summary/clone-recover-no-shuffle-6.summary -@@ -0,0 +1,68 @@ -+Current cluster status: -+ * Node List: -+ * Online: [ node1 node2 node3 ] -+ * GuestOnline: [ base-bundle-0 base-bundle-1 ] -+ -+ * Full List of Resources: -+ * Fencing (stonith:fence_xvm): Started node2 -+ * Container bundle set: base-bundle [localhost/pcmktest]: -+ * base-bundle-0 (ocf:pacemaker:Stateful): Started node3 -+ * base-bundle-1 (ocf:pacemaker:Stateful): Started node2 -+ * base-bundle-2 (ocf:pacemaker:Stateful): Stopped -+ -+Transition Summary: -+ * Move base-bundle-podman-0 ( node3 -> node1 ) -+ * Move base-bundle-0 ( node3 -> node1 ) -+ * Restart base:0 ( base-bundle-0 ) due to required base-bundle-podman-0 start -+ * Move base-bundle-podman-1 ( node2 -> node3 ) -+ * Move base-bundle-1 ( node2 -> node3 ) -+ * Restart base:1 ( base-bundle-1 ) due to required base-bundle-podman-1 start -+ * Start base-bundle-podman-2 ( node2 ) -+ * Start base-bundle-2 ( node2 ) -+ * Start base:2 ( base-bundle-2 ) -+ -+Executing Cluster Transition: -+ * Pseudo action: base-bundle_stop_0 -+ * Pseudo action: base-bundle_start_0 -+ * Pseudo action: base-bundle-clone_stop_0 -+ * Resource action: base-bundle-podman-2 start on node2 -+ * Resource action: base-bundle-2 monitor on node3 -+ * Resource action: base-bundle-2 monitor on node2 -+ * Resource action: base-bundle-2 monitor on node1 -+ * Resource action: base stop on base-bundle-1 -+ * Resource action: base-bundle-1 stop on node2 -+ * Resource action: base-bundle-podman-2 monitor=60000 on node2 -+ * Resource action: base-bundle-2 start on node2 -+ * Resource action: base stop on base-bundle-0 -+ * Pseudo action: base-bundle-clone_stopped_0 -+ * Pseudo action: base-bundle-clone_start_0 -+ * Resource action: base-bundle-0 stop on node3 -+ * Resource action: base-bundle-podman-1 stop on node2 -+ * Resource action: base-bundle-2 monitor=30000 on node2 -+ * Resource action: base-bundle-podman-0 stop on node3 -+ * Resource action: base-bundle-podman-1 start on node3 -+ * Resource action: base-bundle-1 start on node3 -+ * Pseudo action: base-bundle_stopped_0 -+ * Resource action: base-bundle-podman-0 start on node1 -+ * Resource action: base-bundle-0 start on node1 -+ * Resource action: base-bundle-podman-1 monitor=60000 on node3 -+ * Resource action: base-bundle-1 monitor=30000 on node3 -+ * Resource action: base start on base-bundle-0 -+ * Resource action: base start on base-bundle-1 -+ * Resource action: base start on base-bundle-2 -+ * Pseudo action: base-bundle-clone_running_0 -+ * Resource action: base-bundle-podman-0 monitor=60000 on node1 -+ * Resource action: base-bundle-0 monitor=30000 on node1 -+ * Pseudo action: base-bundle_running_0 -+ -+Revised Cluster Status: -+ * Node List: -+ * Online: [ node1 node2 node3 ] -+ * GuestOnline: [ base-bundle-0 base-bundle-1 base-bundle-2 ] -+ -+ * Full List of Resources: -+ * Fencing (stonith:fence_xvm): Started node2 -+ * Container bundle set: base-bundle [localhost/pcmktest]: -+ * base-bundle-0 (ocf:pacemaker:Stateful): Started node1 -+ * base-bundle-1 (ocf:pacemaker:Stateful): Started node3 -+ * base-bundle-2 (ocf:pacemaker:Stateful): Started node2 -diff --git a/cts/scheduler/summary/clone-recover-no-shuffle-7.summary b/cts/scheduler/summary/clone-recover-no-shuffle-7.summary -new file mode 100644 -index 00000000000..e6c9baed0db ---- /dev/null -+++ b/cts/scheduler/summary/clone-recover-no-shuffle-7.summary -@@ -0,0 +1,44 @@ -+Current cluster status: -+ * Node List: -+ * Online: [ node1 node2 node3 ] -+ -+ * Full List of Resources: -+ * Fencing (stonith:fence_xvm): Started node2 -+ * Clone Set: dummy-clone [dummy] (promotable): -+ * Promoted: [ node2 ] -+ * Unpromoted: [ node3 ] -+ * Stopped: [ node1 ] -+ -+Transition Summary: -+ * Move dummy:0 ( Unpromoted node3 -> Promoted node1 ) -+ * Demote dummy:1 ( Promoted -> Unpromoted node2 ) -+ * Start dummy:2 ( node3 ) -+ -+Executing Cluster Transition: -+ * Resource action: dummy cancel=10000 on node2 -+ * Pseudo action: dummy-clone_demote_0 -+ * Resource action: dummy demote on node2 -+ * Pseudo action: dummy-clone_demoted_0 -+ * Pseudo action: dummy-clone_stop_0 -+ * Resource action: dummy stop on node3 -+ * Resource action: dummy monitor=11000 on node2 -+ * Pseudo action: dummy-clone_stopped_0 -+ * Pseudo action: dummy-clone_start_0 -+ * Resource action: dummy start on node1 -+ * Resource action: dummy start on node3 -+ * Pseudo action: dummy-clone_running_0 -+ * Resource action: dummy monitor=11000 on node3 -+ * Pseudo action: dummy-clone_promote_0 -+ * Resource action: dummy promote on node1 -+ * Pseudo action: dummy-clone_promoted_0 -+ * Resource action: dummy monitor=10000 on node1 -+ -+Revised Cluster Status: -+ * Node List: -+ * Online: [ node1 node2 node3 ] -+ -+ * Full List of Resources: -+ * Fencing (stonith:fence_xvm): Started node2 -+ * Clone Set: dummy-clone [dummy] (promotable): -+ * Promoted: [ node1 ] -+ * Unpromoted: [ node2 node3 ] -diff --git a/cts/scheduler/summary/clone-recover-no-shuffle-8.summary b/cts/scheduler/summary/clone-recover-no-shuffle-8.summary -new file mode 100644 -index 00000000000..878f24801dd ---- /dev/null -+++ b/cts/scheduler/summary/clone-recover-no-shuffle-8.summary -@@ -0,0 +1,52 @@ -+Current cluster status: -+ * Node List: -+ * Online: [ node1 node2 node3 ] -+ -+ * Full List of Resources: -+ * Fencing (stonith:fence_xvm): Started node2 -+ * Clone Set: grp-clone [grp] (promotable): -+ * Promoted: [ node2 ] -+ * Unpromoted: [ node3 ] -+ * Stopped: [ node1 ] -+ -+Transition Summary: -+ * Demote rsc1:1 ( Promoted -> Unpromoted node2 ) -+ * Demote rsc2:1 ( Promoted -> Unpromoted node2 ) -+ * Promote rsc1:2 ( Stopped -> Promoted node1 ) -+ * Promote rsc2:2 ( Stopped -> Promoted node1 ) -+ -+Executing Cluster Transition: -+ * Resource action: rsc1 cancel=10000 on node2 -+ * Resource action: rsc2 cancel=10000 on node2 -+ * Pseudo action: grp-clone_demote_0 -+ * Pseudo action: grp:1_demote_0 -+ * Resource action: rsc2 demote on node2 -+ * Resource action: rsc1 demote on node2 -+ * Resource action: rsc2 monitor=11000 on node2 -+ * Pseudo action: grp:1_demoted_0 -+ * Resource action: rsc1 monitor=11000 on node2 -+ * Pseudo action: grp-clone_demoted_0 -+ * Pseudo action: grp-clone_start_0 -+ * Pseudo action: grp:2_start_0 -+ * Resource action: rsc1 start on node1 -+ * Resource action: rsc2 start on node1 -+ * Pseudo action: grp:2_running_0 -+ * Pseudo action: grp-clone_running_0 -+ * Pseudo action: grp-clone_promote_0 -+ * Pseudo action: grp:2_promote_0 -+ * Resource action: rsc1 promote on node1 -+ * Resource action: rsc2 promote on node1 -+ * Pseudo action: grp:2_promoted_0 -+ * Resource action: rsc1 monitor=10000 on node1 -+ * Resource action: rsc2 monitor=10000 on node1 -+ * Pseudo action: grp-clone_promoted_0 -+ -+Revised Cluster Status: -+ * Node List: -+ * Online: [ node1 node2 node3 ] -+ -+ * Full List of Resources: -+ * Fencing (stonith:fence_xvm): Started node2 -+ * Clone Set: grp-clone [grp] (promotable): -+ * Promoted: [ node1 ] -+ * Unpromoted: [ node2 node3 ] -diff --git a/cts/scheduler/summary/clone-recover-no-shuffle-9.summary b/cts/scheduler/summary/clone-recover-no-shuffle-9.summary -new file mode 100644 -index 00000000000..7ede39a6e58 ---- /dev/null -+++ b/cts/scheduler/summary/clone-recover-no-shuffle-9.summary -@@ -0,0 +1,56 @@ -+Current cluster status: -+ * Node List: -+ * Online: [ node1 node2 node3 ] -+ * GuestOnline: [ base-bundle-0 base-bundle-1 ] -+ -+ * Full List of Resources: -+ * Fencing (stonith:fence_xvm): Started node2 -+ * Container bundle set: base-bundle [localhost/pcmktest]: -+ * base-bundle-0 (ocf:pacemaker:Stateful): Unpromoted node3 -+ * base-bundle-1 (ocf:pacemaker:Stateful): Promoted node2 -+ * base-bundle-2 (ocf:pacemaker:Stateful): Stopped -+ -+Transition Summary: -+ * Demote base:1 ( Promoted -> Unpromoted base-bundle-1 ) -+ * Start base-bundle-podman-2 ( node1 ) -+ * Start base-bundle-2 ( node1 ) -+ * Promote base:2 ( Stopped -> Promoted base-bundle-2 ) -+ -+Executing Cluster Transition: -+ * Resource action: base cancel=15000 on base-bundle-1 -+ * Pseudo action: base-bundle_demote_0 -+ * Pseudo action: base-bundle-clone_demote_0 -+ * Resource action: base demote on base-bundle-1 -+ * Pseudo action: base-bundle-clone_demoted_0 -+ * Pseudo action: base-bundle_demoted_0 -+ * Pseudo action: base-bundle_start_0 -+ * Resource action: base monitor=16000 on base-bundle-1 -+ * Pseudo action: base-bundle-clone_start_0 -+ * Resource action: base-bundle-podman-2 start on node1 -+ * Resource action: base-bundle-2 monitor on node3 -+ * Resource action: base-bundle-2 monitor on node2 -+ * Resource action: base-bundle-2 monitor on node1 -+ * Resource action: base-bundle-podman-2 monitor=60000 on node1 -+ * Resource action: base-bundle-2 start on node1 -+ * Resource action: base start on base-bundle-2 -+ * Pseudo action: base-bundle-clone_running_0 -+ * Resource action: base-bundle-2 monitor=30000 on node1 -+ * Pseudo action: base-bundle_running_0 -+ * Pseudo action: base-bundle_promote_0 -+ * Pseudo action: base-bundle-clone_promote_0 -+ * Resource action: base promote on base-bundle-2 -+ * Pseudo action: base-bundle-clone_promoted_0 -+ * Pseudo action: base-bundle_promoted_0 -+ * Resource action: base monitor=15000 on base-bundle-2 -+ -+Revised Cluster Status: -+ * Node List: -+ * Online: [ node1 node2 node3 ] -+ * GuestOnline: [ base-bundle-0 base-bundle-1 base-bundle-2 ] -+ -+ * Full List of Resources: -+ * Fencing (stonith:fence_xvm): Started node2 -+ * Container bundle set: base-bundle [localhost/pcmktest]: -+ * base-bundle-0 (ocf:pacemaker:Stateful): Unpromoted node3 -+ * base-bundle-1 (ocf:pacemaker:Stateful): Unpromoted node2 -+ * base-bundle-2 (ocf:pacemaker:Stateful): Promoted node1 -diff --git a/cts/scheduler/xml/clone-recover-no-shuffle-1.xml b/cts/scheduler/xml/clone-recover-no-shuffle-1.xml -new file mode 100644 -index 00000000000..a634ff352cd ---- /dev/null -+++ b/cts/scheduler/xml/clone-recover-no-shuffle-1.xml -@@ -0,0 +1,113 @@ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -diff --git a/cts/scheduler/xml/clone-recover-no-shuffle-10.xml b/cts/scheduler/xml/clone-recover-no-shuffle-10.xml -new file mode 100644 -index 00000000000..faa202a0ae0 ---- /dev/null -+++ b/cts/scheduler/xml/clone-recover-no-shuffle-10.xml -@@ -0,0 +1,120 @@ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -diff --git a/cts/scheduler/xml/clone-recover-no-shuffle-11.xml b/cts/scheduler/xml/clone-recover-no-shuffle-11.xml -new file mode 100644 -index 00000000000..43d6d749525 ---- /dev/null -+++ b/cts/scheduler/xml/clone-recover-no-shuffle-11.xml -@@ -0,0 +1,153 @@ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -diff --git a/cts/scheduler/xml/clone-recover-no-shuffle-12.xml b/cts/scheduler/xml/clone-recover-no-shuffle-12.xml -new file mode 100644 -index 00000000000..e3026903533 ---- /dev/null -+++ b/cts/scheduler/xml/clone-recover-no-shuffle-12.xml -@@ -0,0 +1,186 @@ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -diff --git a/cts/scheduler/xml/clone-recover-no-shuffle-2.xml b/cts/scheduler/xml/clone-recover-no-shuffle-2.xml -new file mode 100644 -index 00000000000..486666c1f26 ---- /dev/null -+++ b/cts/scheduler/xml/clone-recover-no-shuffle-2.xml -@@ -0,0 +1,141 @@ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -diff --git a/cts/scheduler/xml/clone-recover-no-shuffle-3.xml b/cts/scheduler/xml/clone-recover-no-shuffle-3.xml -new file mode 100644 -index 00000000000..ddafb741dce ---- /dev/null -+++ b/cts/scheduler/xml/clone-recover-no-shuffle-3.xml -@@ -0,0 +1,180 @@ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -diff --git a/cts/scheduler/xml/clone-recover-no-shuffle-4.xml b/cts/scheduler/xml/clone-recover-no-shuffle-4.xml -new file mode 100644 -index 00000000000..40e6520c6d0 ---- /dev/null -+++ b/cts/scheduler/xml/clone-recover-no-shuffle-4.xml -@@ -0,0 +1,120 @@ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -diff --git a/cts/scheduler/xml/clone-recover-no-shuffle-5.xml b/cts/scheduler/xml/clone-recover-no-shuffle-5.xml -new file mode 100644 -index 00000000000..67176dc1a03 ---- /dev/null -+++ b/cts/scheduler/xml/clone-recover-no-shuffle-5.xml -@@ -0,0 +1,148 @@ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -diff --git a/cts/scheduler/xml/clone-recover-no-shuffle-6.xml b/cts/scheduler/xml/clone-recover-no-shuffle-6.xml -new file mode 100644 -index 00000000000..3de42f581d4 ---- /dev/null -+++ b/cts/scheduler/xml/clone-recover-no-shuffle-6.xml -@@ -0,0 +1,187 @@ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -diff --git a/cts/scheduler/xml/clone-recover-no-shuffle-7.xml b/cts/scheduler/xml/clone-recover-no-shuffle-7.xml -new file mode 100644 -index 00000000000..6e9dad50db4 ---- /dev/null -+++ b/cts/scheduler/xml/clone-recover-no-shuffle-7.xml -@@ -0,0 +1,125 @@ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -diff --git a/cts/scheduler/xml/clone-recover-no-shuffle-8.xml b/cts/scheduler/xml/clone-recover-no-shuffle-8.xml -new file mode 100644 -index 00000000000..6f882b80785 ---- /dev/null -+++ b/cts/scheduler/xml/clone-recover-no-shuffle-8.xml -@@ -0,0 +1,153 @@ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -diff --git a/cts/scheduler/xml/clone-recover-no-shuffle-9.xml b/cts/scheduler/xml/clone-recover-no-shuffle-9.xml -new file mode 100644 -index 00000000000..104331d6c00 ---- /dev/null -+++ b/cts/scheduler/xml/clone-recover-no-shuffle-9.xml -@@ -0,0 +1,186 @@ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ - -From 44dfe36a316bddc562c07f7e1adbbaa57b9adf77 Mon Sep 17 00:00:00 2001 -From: Reid Wahl -Date: Wed, 28 Jun 2023 02:04:45 -0700 -Subject: [PATCH 08/19] Refactor: libpacemaker: Recursively copy and restore - allowed node tables - -Given a resource, these two new functions create copies of the allowed -nodes tables of its entire tree of descendants, or restore from such a -backup copy. - -Ref T678 - -Signed-off-by: Reid Wahl ---- - lib/pacemaker/libpacemaker_private.h | 6 +++ - lib/pacemaker/pcmk_sched_nodes.c | 76 ++++++++++++++++++++++++++++ - 2 files changed, 82 insertions(+) - -diff --git a/lib/pacemaker/libpacemaker_private.h b/lib/pacemaker/libpacemaker_private.h -index 614d695f83f..8cdd13f7304 100644 ---- a/lib/pacemaker/libpacemaker_private.h -+++ b/lib/pacemaker/libpacemaker_private.h -@@ -874,6 +874,12 @@ bool pcmk__any_node_available(GHashTable *nodes); - G_GNUC_INTERNAL - GHashTable *pcmk__copy_node_table(GHashTable *nodes); - -+G_GNUC_INTERNAL -+void pcmk__copy_node_tables(const pe_resource_t *rsc, GHashTable **copy); -+ -+G_GNUC_INTERNAL -+void pcmk__restore_node_tables(pe_resource_t *rsc, GHashTable *backup); -+ - G_GNUC_INTERNAL - GList *pcmk__sort_nodes(GList *nodes, pe_node_t *active_node); - -diff --git a/lib/pacemaker/pcmk_sched_nodes.c b/lib/pacemaker/pcmk_sched_nodes.c -index d7d5ba46169..eb0b2a41e39 100644 ---- a/lib/pacemaker/pcmk_sched_nodes.c -+++ b/lib/pacemaker/pcmk_sched_nodes.c -@@ -82,6 +82,82 @@ pcmk__copy_node_table(GHashTable *nodes) - return new_table; - } - -+/*! -+ * \internal -+ * \brief Free a table of node tables -+ * -+ * \param[in,out] data Table to free -+ * -+ * \note This is a \c GDestroyNotify wrapper for \c g_hash_table_destroy(). -+ */ -+static void -+destroy_node_tables(gpointer data) -+{ -+ g_hash_table_destroy((GHashTable *) data); -+} -+ -+/*! -+ * \internal -+ * \brief Recursively copy the node tables of a resource -+ * -+ * Build a hash table containing copies of the allowed nodes tables of \p rsc -+ * and its entire tree of descendants. The key is the resource ID, and the value -+ * is a copy of the resource's node table. -+ * -+ * \param[in] rsc Resource whose node table to copy -+ * \param[in,out] copy Where to store the copied node tables -+ * -+ * \note \p *copy should be \c NULL for the top-level call. -+ * \note The caller is responsible for freeing \p copy using -+ * \c g_hash_table_destroy(). -+ */ -+void -+pcmk__copy_node_tables(const pe_resource_t *rsc, GHashTable **copy) -+{ -+ CRM_ASSERT((rsc != NULL) && (copy != NULL)); -+ -+ if (*copy == NULL) { -+ *copy = pcmk__strkey_table(NULL, destroy_node_tables); -+ } -+ -+ g_hash_table_insert(*copy, rsc->id, -+ pcmk__copy_node_table(rsc->allowed_nodes)); -+ -+ for (const GList *iter = rsc->children; iter != NULL; iter = iter->next) { -+ pcmk__copy_node_tables((const pe_resource_t *) iter->data, copy); -+ } -+} -+ -+/*! -+ * \internal -+ * \brief Recursively restore the node tables of a resource from backup -+ * -+ * Given a hash table containing backup copies of the allowed nodes tables of -+ * \p rsc and its entire tree of descendants, replace the resources' current -+ * node tables with the backed-up copies. -+ * -+ * \param[in,out] rsc Resource whose node tables to restore -+ * \param[in] backup Table of backup node tables (created by -+ * \c pcmk__copy_node_tables()) -+ * -+ * \note This function frees the resources' current node tables. -+ */ -+void -+pcmk__restore_node_tables(pe_resource_t *rsc, GHashTable *backup) -+{ -+ CRM_ASSERT((rsc != NULL) && (backup != NULL)); -+ -+ g_hash_table_destroy(rsc->allowed_nodes); -+ -+ // Copy to avoid danger with multiple restores -+ rsc->allowed_nodes = g_hash_table_lookup(backup, rsc->id); -+ rsc->allowed_nodes = pcmk__copy_node_table(rsc->allowed_nodes); -+ -+ for (GList *iter = rsc->children; iter != NULL; iter = iter->next) { -+ pcmk__restore_node_tables((pe_resource_t *) iter->data, backup); -+ } -+} -+ - /*! - * \internal - * \brief Copy a list of node objects - -From a3c120c4c0aeb48efd55bac6de68423be099831d Mon Sep 17 00:00:00 2001 -From: Reid Wahl -Date: Wed, 28 Jun 2023 02:09:28 -0700 -Subject: [PATCH 09/19] Refactor: libpacemaker: Restore node tables if cloned - group assign fails - -Currently, when assigning an instance of a cloned group (that is, one of -the groups), we make a copy only of the group's allowed nodes table. We -restore only that table if an early assignment attempt fails. - -Here, we make a recursive copy containing the allowed nodes tables of -the group itself and of all the resources in the group. Then we restore -all of them from backup if the assignment fails. - -This doesn't visibly fix anything yet, but it's a necessary part of the -fix for T678. And it was obviously wrong before :) - -Ref T678 - -Signed-off-by: Reid Wahl ---- - lib/pacemaker/pcmk_sched_instances.c | 9 ++++----- - 1 file changed, 4 insertions(+), 5 deletions(-) - -diff --git a/lib/pacemaker/pcmk_sched_instances.c b/lib/pacemaker/pcmk_sched_instances.c -index c880196f70f..783820bbf69 100644 ---- a/lib/pacemaker/pcmk_sched_instances.c -+++ b/lib/pacemaker/pcmk_sched_instances.c -@@ -600,8 +600,9 @@ assign_instance(pe_resource_t *instance, const pe_node_t *prefer, - chosen = instance->cmds->assign(instance, NULL); - - } else { // Possible early assignment to preferred node -- GHashTable *backup = pcmk__copy_node_table(instance->allowed_nodes); -+ GHashTable *backup = NULL; - -+ pcmk__copy_node_tables(instance, &backup); - chosen = instance->cmds->assign(instance, prefer); - - // Revert nodes if preferred node won't be assigned -@@ -609,13 +610,11 @@ assign_instance(pe_resource_t *instance, const pe_node_t *prefer, - crm_info("Not assigning %s to preferred node %s: %s is better", - instance->id, pe__node_name(prefer), - pe__node_name(chosen)); -- g_hash_table_destroy(instance->allowed_nodes); -- instance->allowed_nodes = backup; -+ pcmk__restore_node_tables(instance, backup); - pcmk__unassign_resource(instance); - chosen = NULL; -- } else if (backup != NULL) { -- g_hash_table_destroy(backup); - } -+ g_hash_table_destroy(backup); - } - - // The parent tracks how many instances have been assigned to each node - -From a5a5c76333365be87f5d3d62f354b45376894506 Mon Sep 17 00:00:00 2001 -From: Reid Wahl -Date: Wed, 28 Jun 2023 02:08:44 -0700 -Subject: [PATCH 10/19] Fix: libpacemaker: Respect clone-node-max for cloned - groups - -Currently, cloned groups may have more than clone-node-max instances -assigned to a given node. This can happen when a location constraint -exists for the clone. - -For example, consider the case of the clone-recover-no-shuffle-5 test. -The cloned group prefers node1 with a score of 100. The location score -is applied only to a group's first member. - -So in the early assignment attempt (within pcmk__assign_instances()), we -try to assign each instance (group) to its current node. However, the -first member prefers a different node (node1) and gets assigned there -instead. The second member has to follow the first due to the group's -internal colocation. - -However, node1 wasn't the preferred node. So assign_instance() tries to -revert the assignment by calling pcmk__unassign_resource() on the -instance (the group). But this leaves the group members assigned, -because pcmk__unassign_resource() doesn't act recursively. - -With this commit, pcmk__unassign_resource() acts recursively. We can now -unassign a resource and all its children recursively. - -Fixes T678 - -Signed-off-by: Reid Wahl ---- - lib/pacemaker/pcmk_sched_resource.c | 43 +++++++++++++++++++---------- - 1 file changed, 28 insertions(+), 15 deletions(-) - -diff --git a/lib/pacemaker/pcmk_sched_resource.c b/lib/pacemaker/pcmk_sched_resource.c -index dd9939a42a6..8f703789b20 100644 ---- a/lib/pacemaker/pcmk_sched_resource.c -+++ b/lib/pacemaker/pcmk_sched_resource.c -@@ -455,13 +455,14 @@ pcmk__assign_resource(pe_resource_t *rsc, pe_node_t *node, bool force) - - /*! - * \internal -- * \brief Remove any assignment of a specified resource to a node -+ * \brief Remove any node assignment from a specified resource and its children - * - * If a specified resource has been assigned to a node, remove that assignment -- * and mark the resource as provisional again. This is not done recursively for -- * children, so it should be called only for primitives. -+ * and mark the resource as provisional again. - * - * \param[in,out] rsc Resource to unassign -+ * -+ * \note This function is called recursively on \p rsc and its children. - */ - void - pcmk__unassign_resource(pe_resource_t *rsc) -@@ -469,21 +470,33 @@ pcmk__unassign_resource(pe_resource_t *rsc) - pe_node_t *old = rsc->allocated_to; - - if (old == NULL) { -- return; -+ crm_info("Unassigning %s", rsc->id); -+ } else { -+ crm_info("Unassigning %s from %s", rsc->id, pe__node_name(old)); - } - -- crm_info("Unassigning %s from %s", rsc->id, pe__node_name(old)); - pe__set_resource_flags(rsc, pe_rsc_provisional); -- rsc->allocated_to = NULL; -- -- /* We're going to free the pe_node_t, but its details member is shared and -- * will remain, so update that appropriately first. -- */ -- old->details->allocated_rsc = g_list_remove(old->details->allocated_rsc, -- rsc); -- old->details->num_resources--; -- pcmk__release_node_capacity(old->details->utilization, rsc); -- free(old); -+ -+ if (rsc->children == NULL) { -+ if (old == NULL) { -+ return; -+ } -+ rsc->allocated_to = NULL; -+ -+ /* We're going to free the pe_node_t, but its details member is shared -+ * and will remain, so update that appropriately first. -+ */ -+ old->details->allocated_rsc = g_list_remove(old->details->allocated_rsc, -+ rsc); -+ old->details->num_resources--; -+ pcmk__release_node_capacity(old->details->utilization, rsc); -+ free(old); -+ return; -+ } -+ -+ for (GList *iter = rsc->children; iter != NULL; iter = iter->next) { -+ pcmk__unassign_resource((pe_resource_t *) iter->data); -+ } - } - - /*! - -From edd9b4ef2094e776530ff540047848aa6d2a1b42 Mon Sep 17 00:00:00 2001 -From: Reid Wahl -Date: Wed, 28 Jun 2023 02:39:39 -0700 -Subject: [PATCH 11/19] Test: scheduler: Update tests for cloned group - clone-node-max fix - -Ref T678 - -Signed-off-by: Reid Wahl ---- - .../dot/clone-recover-no-shuffle-5.dot | 46 +--- - .../exp/clone-recover-no-shuffle-5.exp | 231 +++--------------- - .../scores/clone-recover-no-shuffle-5.scores | 50 +++- - .../clone-recover-no-shuffle-5.summary | 27 +- - .../xml/clone-recover-no-shuffle-5.xml | 6 +- - 5 files changed, 97 insertions(+), 263 deletions(-) - -diff --git a/cts/scheduler/dot/clone-recover-no-shuffle-5.dot b/cts/scheduler/dot/clone-recover-no-shuffle-5.dot -index 7219ee5a6d3..a2356f2280b 100644 ---- a/cts/scheduler/dot/clone-recover-no-shuffle-5.dot -+++ b/cts/scheduler/dot/clone-recover-no-shuffle-5.dot -@@ -2,12 +2,10 @@ - "grp-clone_running_0" [ style=bold color="green" fontcolor="orange"] - "grp-clone_start_0" -> "grp-clone_running_0" [ style = bold] - "grp-clone_start_0" -> "grp:0_start_0" [ style = bold] --"grp-clone_start_0" -> "grp:1_start_0" [ style = bold] - "grp-clone_start_0" -> "grp:2_start_0" [ style = bold] - "grp-clone_start_0" [ style=bold color="green" fontcolor="orange"] - "grp-clone_stop_0" -> "grp-clone_stopped_0" [ style = bold] - "grp-clone_stop_0" -> "grp:0_stop_0" [ style = bold] --"grp-clone_stop_0" -> "grp:1_stop_0" [ style = bold] - "grp-clone_stop_0" [ style=bold color="green" fontcolor="orange"] - "grp-clone_stopped_0" -> "grp-clone_start_0" [ style = bold] - "grp-clone_stopped_0" [ style=bold color="green" fontcolor="orange"] -@@ -24,57 +22,35 @@ - "grp:0_stopped_0" -> "grp-clone_stopped_0" [ style = bold] - "grp:0_stopped_0" -> "grp:0_start_0" [ style = bold] - "grp:0_stopped_0" [ style=bold color="green" fontcolor="orange"] --"grp:1_running_0" -> "grp-clone_running_0" [ style = bold] --"grp:1_running_0" [ style=bold color="green" fontcolor="orange"] --"grp:1_start_0" -> "grp:1_running_0" [ style = bold] --"grp:1_start_0" -> "rsc1_start_0 node1" [ style = bold] --"grp:1_start_0" -> "rsc2_start_0 node1" [ style = bold] --"grp:1_start_0" [ style=bold color="green" fontcolor="orange"] --"grp:1_stop_0" -> "grp:1_stopped_0" [ style = bold] --"grp:1_stop_0" -> "rsc1_stop_0 node3" [ style = bold] --"grp:1_stop_0" -> "rsc2_stop_0 node3" [ style = bold] --"grp:1_stop_0" [ style=bold color="green" fontcolor="orange"] --"grp:1_stopped_0" -> "grp-clone_stopped_0" [ style = bold] --"grp:1_stopped_0" -> "grp:1_start_0" [ style = bold] --"grp:1_stopped_0" [ style=bold color="green" fontcolor="orange"] - "grp:2_running_0" -> "grp-clone_running_0" [ style = bold] - "grp:2_running_0" [ style=bold color="green" fontcolor="orange"] - "grp:2_start_0" -> "grp:2_running_0" [ style = bold] --"grp:2_start_0" -> "rsc1:2_start_0 node1" [ style = bold] --"grp:2_start_0" -> "rsc2:2_start_0 node1" [ style = bold] -+"grp:2_start_0" -> "rsc1:2_start_0 node2" [ style = bold] -+"grp:2_start_0" -> "rsc2:2_start_0 node2" [ style = bold] - "grp:2_start_0" [ style=bold color="green" fontcolor="orange"] --"rsc1:2_monitor_10000 node1" [ style=bold color="green" fontcolor="black"] --"rsc1:2_start_0 node1" -> "grp:2_running_0" [ style = bold] --"rsc1:2_start_0 node1" -> "rsc1:2_monitor_10000 node1" [ style = bold] --"rsc1:2_start_0 node1" -> "rsc2:2_start_0 node1" [ style = bold] --"rsc1:2_start_0 node1" [ style=bold color="green" fontcolor="black"] -+"rsc1:2_monitor_10000 node2" [ style=bold color="green" fontcolor="black"] -+"rsc1:2_start_0 node2" -> "grp:2_running_0" [ style = bold] -+"rsc1:2_start_0 node2" -> "rsc1:2_monitor_10000 node2" [ style = bold] -+"rsc1:2_start_0 node2" -> "rsc2:2_start_0 node2" [ style = bold] -+"rsc1:2_start_0 node2" [ style=bold color="green" fontcolor="black"] - "rsc1_monitor_10000 node1" [ style=bold color="green" fontcolor="black"] - "rsc1_start_0 node1" -> "grp:0_running_0" [ style = bold] --"rsc1_start_0 node1" -> "grp:1_running_0" [ style = bold] - "rsc1_start_0 node1" -> "rsc1_monitor_10000 node1" [ style = bold] - "rsc1_start_0 node1" -> "rsc2_start_0 node1" [ style = bold] - "rsc1_start_0 node1" [ style=bold color="green" fontcolor="black"] - "rsc1_stop_0 node2" -> "grp:0_stopped_0" [ style = bold] - "rsc1_stop_0 node2" -> "rsc1_start_0 node1" [ style = bold] - "rsc1_stop_0 node2" [ style=bold color="green" fontcolor="black"] --"rsc1_stop_0 node3" -> "grp:1_stopped_0" [ style = bold] --"rsc1_stop_0 node3" -> "rsc1_start_0 node1" [ style = bold] --"rsc1_stop_0 node3" [ style=bold color="green" fontcolor="black"] --"rsc2:2_monitor_10000 node1" [ style=bold color="green" fontcolor="black"] --"rsc2:2_start_0 node1" -> "grp:2_running_0" [ style = bold] --"rsc2:2_start_0 node1" -> "rsc2:2_monitor_10000 node1" [ style = bold] --"rsc2:2_start_0 node1" [ style=bold color="green" fontcolor="black"] -+"rsc2:2_monitor_10000 node2" [ style=bold color="green" fontcolor="black"] -+"rsc2:2_start_0 node2" -> "grp:2_running_0" [ style = bold] -+"rsc2:2_start_0 node2" -> "rsc2:2_monitor_10000 node2" [ style = bold] -+"rsc2:2_start_0 node2" [ style=bold color="green" fontcolor="black"] - "rsc2_monitor_10000 node1" [ style=bold color="green" fontcolor="black"] - "rsc2_start_0 node1" -> "grp:0_running_0" [ style = bold] --"rsc2_start_0 node1" -> "grp:1_running_0" [ style = bold] - "rsc2_start_0 node1" -> "rsc2_monitor_10000 node1" [ style = bold] - "rsc2_start_0 node1" [ style=bold color="green" fontcolor="black"] - "rsc2_stop_0 node2" -> "grp:0_stopped_0" [ style = bold] - "rsc2_stop_0 node2" -> "rsc1_stop_0 node2" [ style = bold] - "rsc2_stop_0 node2" -> "rsc2_start_0 node1" [ style = bold] - "rsc2_stop_0 node2" [ style=bold color="green" fontcolor="black"] --"rsc2_stop_0 node3" -> "grp:1_stopped_0" [ style = bold] --"rsc2_stop_0 node3" -> "rsc1_stop_0 node3" [ style = bold] --"rsc2_stop_0 node3" -> "rsc2_start_0 node1" [ style = bold] --"rsc2_stop_0 node3" [ style=bold color="green" fontcolor="black"] - } -diff --git a/cts/scheduler/exp/clone-recover-no-shuffle-5.exp b/cts/scheduler/exp/clone-recover-no-shuffle-5.exp -index 8a8e799793e..c1cee43b12f 100644 ---- a/cts/scheduler/exp/clone-recover-no-shuffle-5.exp -+++ b/cts/scheduler/exp/clone-recover-no-shuffle-5.exp -@@ -25,7 +25,7 @@ - - - -- -+ - - - -@@ -58,7 +58,7 @@ - - - -- -+ - - - -@@ -154,245 +154,92 @@ - - - -- -+ - - - - - -- -+ - - -- -+ - - -- -+ - - - - - -- -+ - - - - - -- -+ - - - - - -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -+ - -- -+ - - - - -- -+ - - - -- -+ - -- -+ - -- -+ - - - - -- -+ - - - -- -+ - -- -+ - -- -+ - - - - -- -+ - - - -- -+ - -- -+ - -- -+ - - - - -- -+ - - -- -+ - - - -- -+ - -- -+ - - - -@@ -401,24 +248,21 @@ - - - -- -- -- -- -+ - - - -- -+ - -- -+ - - - - - -- -+ - -- -+ - - - -@@ -427,25 +271,22 @@ - - - -- -+ - - -- -- -- -- -+ - - - -- -+ - -- -+ - - - - - -- -+ - - - -diff --git a/cts/scheduler/scores/clone-recover-no-shuffle-5.scores b/cts/scheduler/scores/clone-recover-no-shuffle-5.scores -index eecba43fae0..0dd9728830c 100644 ---- a/cts/scheduler/scores/clone-recover-no-shuffle-5.scores -+++ b/cts/scheduler/scores/clone-recover-no-shuffle-5.scores -@@ -30,50 +30,80 @@ pcmk__clone_assign: rsc2:2 allocation score on node1: 0 - pcmk__clone_assign: rsc2:2 allocation score on node2: 0 - pcmk__clone_assign: rsc2:2 allocation score on node3: 0 - pcmk__group_assign: grp:0 allocation score on node1: 100 -+pcmk__group_assign: grp:0 allocation score on node1: 100 -+pcmk__group_assign: grp:0 allocation score on node2: 0 - pcmk__group_assign: grp:0 allocation score on node2: 0 - pcmk__group_assign: grp:0 allocation score on node3: 0 -+pcmk__group_assign: grp:0 allocation score on node3: 0 -+pcmk__group_assign: grp:1 allocation score on node1: -INFINITY - pcmk__group_assign: grp:1 allocation score on node1: 100 - pcmk__group_assign: grp:1 allocation score on node2: 0 -+pcmk__group_assign: grp:1 allocation score on node2: 0 -+pcmk__group_assign: grp:1 allocation score on node3: 0 - pcmk__group_assign: grp:1 allocation score on node3: 0 --pcmk__group_assign: grp:2 allocation score on node1: 100 -+pcmk__group_assign: grp:2 allocation score on node1: -INFINITY - pcmk__group_assign: grp:2 allocation score on node2: 0 --pcmk__group_assign: grp:2 allocation score on node3: 0 -+pcmk__group_assign: grp:2 allocation score on node3: -INFINITY -+pcmk__group_assign: rsc1:0 allocation score on node1: 100 - pcmk__group_assign: rsc1:0 allocation score on node1: 100 - pcmk__group_assign: rsc1:0 allocation score on node2: 1 -+pcmk__group_assign: rsc1:0 allocation score on node2: 1 -+pcmk__group_assign: rsc1:0 allocation score on node3: 0 - pcmk__group_assign: rsc1:0 allocation score on node3: 0 -+pcmk__group_assign: rsc1:1 allocation score on node1: -INFINITY - pcmk__group_assign: rsc1:1 allocation score on node1: 100 - pcmk__group_assign: rsc1:1 allocation score on node2: 0 -+pcmk__group_assign: rsc1:1 allocation score on node2: 0 - pcmk__group_assign: rsc1:1 allocation score on node3: 1 --pcmk__group_assign: rsc1:2 allocation score on node1: 100 -+pcmk__group_assign: rsc1:1 allocation score on node3: 1 -+pcmk__group_assign: rsc1:2 allocation score on node1: -INFINITY - pcmk__group_assign: rsc1:2 allocation score on node2: 0 --pcmk__group_assign: rsc1:2 allocation score on node3: 0 -+pcmk__group_assign: rsc1:2 allocation score on node3: -INFINITY -+pcmk__group_assign: rsc2:0 allocation score on node1: 0 - pcmk__group_assign: rsc2:0 allocation score on node1: 0 - pcmk__group_assign: rsc2:0 allocation score on node2: 1 -+pcmk__group_assign: rsc2:0 allocation score on node2: 1 - pcmk__group_assign: rsc2:0 allocation score on node3: 0 -+pcmk__group_assign: rsc2:0 allocation score on node3: 0 -+pcmk__group_assign: rsc2:1 allocation score on node1: -INFINITY - pcmk__group_assign: rsc2:1 allocation score on node1: 0 - pcmk__group_assign: rsc2:1 allocation score on node2: 0 -+pcmk__group_assign: rsc2:1 allocation score on node2: 0 -+pcmk__group_assign: rsc2:1 allocation score on node3: 1 - pcmk__group_assign: rsc2:1 allocation score on node3: 1 --pcmk__group_assign: rsc2:2 allocation score on node1: 0 -+pcmk__group_assign: rsc2:2 allocation score on node1: -INFINITY - pcmk__group_assign: rsc2:2 allocation score on node2: 0 --pcmk__group_assign: rsc2:2 allocation score on node3: 0 -+pcmk__group_assign: rsc2:2 allocation score on node3: -INFINITY - pcmk__primitive_assign: Fencing allocation score on node1: 0 - pcmk__primitive_assign: Fencing allocation score on node2: 0 - pcmk__primitive_assign: Fencing allocation score on node3: 0 - pcmk__primitive_assign: rsc1:0 allocation score on node1: 100 -+pcmk__primitive_assign: rsc1:0 allocation score on node1: 100 -+pcmk__primitive_assign: rsc1:0 allocation score on node2: 2 - pcmk__primitive_assign: rsc1:0 allocation score on node2: 2 - pcmk__primitive_assign: rsc1:0 allocation score on node3: 0 -+pcmk__primitive_assign: rsc1:0 allocation score on node3: 0 -+pcmk__primitive_assign: rsc1:1 allocation score on node1: -INFINITY - pcmk__primitive_assign: rsc1:1 allocation score on node1: 100 - pcmk__primitive_assign: rsc1:1 allocation score on node2: 0 -+pcmk__primitive_assign: rsc1:1 allocation score on node2: 0 -+pcmk__primitive_assign: rsc1:1 allocation score on node3: 2 - pcmk__primitive_assign: rsc1:1 allocation score on node3: 2 --pcmk__primitive_assign: rsc1:2 allocation score on node1: 100 -+pcmk__primitive_assign: rsc1:2 allocation score on node1: -INFINITY - pcmk__primitive_assign: rsc1:2 allocation score on node2: 0 --pcmk__primitive_assign: rsc1:2 allocation score on node3: 0 -+pcmk__primitive_assign: rsc1:2 allocation score on node3: -INFINITY -+pcmk__primitive_assign: rsc2:0 allocation score on node1: 0 - pcmk__primitive_assign: rsc2:0 allocation score on node1: 0 - pcmk__primitive_assign: rsc2:0 allocation score on node2: -INFINITY -+pcmk__primitive_assign: rsc2:0 allocation score on node2: -INFINITY -+pcmk__primitive_assign: rsc2:0 allocation score on node3: -INFINITY - pcmk__primitive_assign: rsc2:0 allocation score on node3: -INFINITY -+pcmk__primitive_assign: rsc2:1 allocation score on node1: -INFINITY - pcmk__primitive_assign: rsc2:1 allocation score on node1: 0 - pcmk__primitive_assign: rsc2:1 allocation score on node2: -INFINITY -+pcmk__primitive_assign: rsc2:1 allocation score on node2: -INFINITY - pcmk__primitive_assign: rsc2:1 allocation score on node3: -INFINITY --pcmk__primitive_assign: rsc2:2 allocation score on node1: 0 --pcmk__primitive_assign: rsc2:2 allocation score on node2: -INFINITY -+pcmk__primitive_assign: rsc2:1 allocation score on node3: 1 -+pcmk__primitive_assign: rsc2:2 allocation score on node1: -INFINITY -+pcmk__primitive_assign: rsc2:2 allocation score on node2: 0 - pcmk__primitive_assign: rsc2:2 allocation score on node3: -INFINITY -diff --git a/cts/scheduler/summary/clone-recover-no-shuffle-5.summary b/cts/scheduler/summary/clone-recover-no-shuffle-5.summary -index e84d0a574de..121214c42ab 100644 ---- a/cts/scheduler/summary/clone-recover-no-shuffle-5.summary -+++ b/cts/scheduler/summary/clone-recover-no-shuffle-5.summary -@@ -11,41 +11,29 @@ Current cluster status: - Transition Summary: - * Move rsc1:0 ( node2 -> node1 ) - * Move rsc2:0 ( node2 -> node1 ) -- * Move rsc1:1 ( node3 -> node1 ) -- * Move rsc2:1 ( node3 -> node1 ) -- * Start rsc1:2 ( node1 ) -- * Start rsc2:2 ( node1 ) -+ * Start rsc1:2 ( node2 ) -+ * Start rsc2:2 ( node2 ) - - Executing Cluster Transition: - * Pseudo action: grp-clone_stop_0 - * Pseudo action: grp:0_stop_0 - * Resource action: rsc2 stop on node2 -- * Pseudo action: grp:1_stop_0 -- * Resource action: rsc2 stop on node3 - * Resource action: rsc1 stop on node2 -- * Resource action: rsc1 stop on node3 - * Pseudo action: grp:0_stopped_0 -- * Pseudo action: grp:1_stopped_0 - * Pseudo action: grp-clone_stopped_0 - * Pseudo action: grp-clone_start_0 - * Pseudo action: grp:0_start_0 - * Resource action: rsc1 start on node1 - * Resource action: rsc2 start on node1 -- * Pseudo action: grp:1_start_0 -- * Resource action: rsc1 start on node1 -- * Resource action: rsc2 start on node1 - * Pseudo action: grp:2_start_0 -- * Resource action: rsc1 start on node1 -- * Resource action: rsc2 start on node1 -+ * Resource action: rsc1 start on node2 -+ * Resource action: rsc2 start on node2 - * Pseudo action: grp:0_running_0 - * Resource action: rsc1 monitor=10000 on node1 - * Resource action: rsc2 monitor=10000 on node1 -- * Pseudo action: grp:1_running_0 -- * Resource action: rsc1 monitor=10000 on node1 -- * Resource action: rsc2 monitor=10000 on node1 - * Pseudo action: grp:2_running_0 -- * Resource action: rsc1 monitor=10000 on node1 -- * Resource action: rsc2 monitor=10000 on node1 -+ * Resource action: rsc1 monitor=10000 on node2 -+ * Resource action: rsc2 monitor=10000 on node2 - * Pseudo action: grp-clone_running_0 - - Revised Cluster Status: -@@ -55,5 +43,4 @@ Revised Cluster Status: - * Full List of Resources: - * Fencing (stonith:fence_xvm): Started node2 - * Clone Set: grp-clone [grp]: -- * Started: [ node1 ] -- * Stopped: [ node2 node3 ] -+ * Started: [ node1 node2 node3 ] -diff --git a/cts/scheduler/xml/clone-recover-no-shuffle-5.xml b/cts/scheduler/xml/clone-recover-no-shuffle-5.xml -index 67176dc1a03..45f3b5a9f3a 100644 ---- a/cts/scheduler/xml/clone-recover-no-shuffle-5.xml -+++ b/cts/scheduler/xml/clone-recover-no-shuffle-5.xml -@@ -14,9 +14,9 @@ - * Instance grp:2 should start on node1 - - This test output is incorrect: -- * Instance grp:0 moves from node2 to node1 -- * Instance grp:1 moves from node3 to node1 -- * Instance grp:2 starts on node1 (correct) -+ * Instance grp:0 moves to node1 -+ * Instance grp:1 remains started on node3 (correct) -+ * Instance grp:2 starts on node2 - --> - - - -From ff60c47e89c6434819dbe5e5e9a87d01122e165e Mon Sep 17 00:00:00 2001 -From: Reid Wahl -Date: Thu, 6 Jul 2023 13:52:59 -0700 -Subject: [PATCH 12/19] Refactor: libpacemaker: Move instance provisional check - to loop body - -Avoid calling preferred_node() this way. Since assign_instance() is -static and has only two callers, we don't have to worry about a sanity -provisional check inside the function. - -Signed-off-by: Reid Wahl ---- - lib/pacemaker/pcmk_sched_instances.c | 9 ++++----- - 1 file changed, 4 insertions(+), 5 deletions(-) - -diff --git a/lib/pacemaker/pcmk_sched_instances.c b/lib/pacemaker/pcmk_sched_instances.c -index 783820bbf69..58fad741729 100644 ---- a/lib/pacemaker/pcmk_sched_instances.c -+++ b/lib/pacemaker/pcmk_sched_instances.c -@@ -568,11 +568,6 @@ assign_instance(pe_resource_t *instance, const pe_node_t *prefer, - pe_rsc_trace(instance, "Assigning %s (preferring %s)", instance->id, - ((prefer == NULL)? "no node" : prefer->details->uname)); - -- if (!pcmk_is_set(instance->flags, pe_rsc_provisional)) { -- // Instance is already assigned -- return instance->fns->location(instance, NULL, FALSE) != NULL; -- } -- - if (pcmk_is_set(instance->flags, pe_rsc_allocating)) { - pe_rsc_debug(instance, - "Assignment loop detected involving %s colocations", -@@ -745,6 +740,10 @@ pcmk__assign_instances(pe_resource_t *collective, GList *instances, - iter = iter->next) { - instance = (pe_resource_t *) iter->data; - -+ if (!pcmk_is_set(instance->flags, pe_rsc_provisional)) { -+ continue; // Already assigned -+ } -+ - current = preferred_node(collective, instance, optimal_per_node); - if ((current != NULL) - && assign_instance(instance, current, max_per_node)) { - -From 0f9e84238a4778da71488ff67ea9f1772e797d80 Mon Sep 17 00:00:00 2001 -From: Reid Wahl -Date: Fri, 23 Jun 2023 15:16:57 -0700 -Subject: [PATCH 13/19] Refactor: libpacemaker: Functionize updating parent - allowed node count - -...in pcmk_sched_instances.c:assign_instance(). We'll use this elsewhere -in an upcoming commit. - -Ref T489 - -Signed-off-by: Reid Wahl ---- - lib/pacemaker/pcmk_sched_instances.c | 54 ++++++++++++++++++---------- - 1 file changed, 36 insertions(+), 18 deletions(-) - -diff --git a/lib/pacemaker/pcmk_sched_instances.c b/lib/pacemaker/pcmk_sched_instances.c -index 58fad741729..1b051cb2ed9 100644 ---- a/lib/pacemaker/pcmk_sched_instances.c -+++ b/lib/pacemaker/pcmk_sched_instances.c -@@ -545,6 +545,39 @@ pcmk__cmp_instance(gconstpointer a, gconstpointer b) - return rc; - } - -+/*! -+ * \internal -+ * \brief Increment the parent's instance count after assigning an instance -+ * -+ * An instance's parent tracks how many instances have been assigned to each -+ * node via its pe_node_t:count member. After assigning an instance to a node, -+ * find the corresponding node in the parent's allowed table and increment it. -+ * -+ * \param[in,out] instance Instance whose parent to update -+ * \param[in] assigned_to Node to which the instance was assigned -+ */ -+static void -+increment_parent_count(pe_resource_t *instance, const pe_node_t *assigned_to) -+{ -+ pe_node_t *allowed = NULL; -+ -+ if (assigned_to == NULL) { -+ return; -+ } -+ allowed = pcmk__top_allowed_node(instance, assigned_to); -+ -+ if (allowed == NULL) { -+ /* The instance is allowed on the node, but its parent isn't. This -+ * shouldn't be possible if the resource is managed, and we won't be -+ * able to limit the number of instances assigned to the node. -+ */ -+ CRM_LOG_ASSERT(!pcmk_is_set(instance->flags, pe_rsc_managed)); -+ -+ } else { -+ allowed->count++; -+ } -+} -+ - /*! - * \internal - * \brief Choose a node for an instance -@@ -562,9 +595,7 @@ assign_instance(pe_resource_t *instance, const pe_node_t *prefer, - int max_per_node) - { - pe_node_t *chosen = NULL; -- pe_node_t *allowed = NULL; - -- CRM_ASSERT(instance != NULL); - pe_rsc_trace(instance, "Assigning %s (preferring %s)", instance->id, - ((prefer == NULL)? "no node" : prefer->details->uname)); - -@@ -578,8 +609,8 @@ assign_instance(pe_resource_t *instance, const pe_node_t *prefer, - if (prefer != NULL) { // Possible early assignment to preferred node - - // Get preferred node with instance's scores -- allowed = g_hash_table_lookup(instance->allowed_nodes, -- prefer->details->id); -+ pe_node_t *allowed = g_hash_table_lookup(instance->allowed_nodes, -+ prefer->details->id); - - if ((allowed == NULL) || (allowed->weight < 0)) { - pe_rsc_trace(instance, -@@ -612,20 +643,7 @@ assign_instance(pe_resource_t *instance, const pe_node_t *prefer, - g_hash_table_destroy(backup); - } - -- // The parent tracks how many instances have been assigned to each node -- if (chosen != NULL) { -- allowed = pcmk__top_allowed_node(instance, chosen); -- if (allowed == NULL) { -- /* The instance is allowed on the node, but its parent isn't. This -- * shouldn't be possible if the resource is managed, and we won't be -- * able to limit the number of instances assigned to the node. -- */ -- CRM_LOG_ASSERT(!pcmk_is_set(instance->flags, pe_rsc_managed)); -- -- } else { -- allowed->count++; -- } -- } -+ increment_parent_count(instance, chosen); - return chosen != NULL; - } - - -From 6cddfe269531661112537eb3ef7c90975feb73ea Mon Sep 17 00:00:00 2001 -From: Reid Wahl -Date: Thu, 22 Jun 2023 13:49:42 -0700 -Subject: [PATCH 14/19] Refactor: libpe_status: Copy count in pe__copy_node() - -pe__copy_node() is supposed to make a shallow copy of a pe_node_t -object. That should include the count member. The caller is free to -reset it to 0 if desired. - -Signed-off-by: Reid Wahl ---- - lib/pengine/utils.c | 1 + - 1 file changed, 1 insertion(+) - -diff --git a/lib/pengine/utils.c b/lib/pengine/utils.c -index ef0a092dc16..199ce87e61f 100644 ---- a/lib/pengine/utils.c -+++ b/lib/pengine/utils.c -@@ -98,6 +98,7 @@ pe__copy_node(const pe_node_t *this_node) - new_node->rsc_discover_mode = this_node->rsc_discover_mode; - new_node->weight = this_node->weight; - new_node->fixed = this_node->fixed; // @COMPAT deprecated and unused -+ new_node->count = this_node->count; - new_node->details = this_node->details; - - return new_node; - -From 30385bedeb5177b703b3b68d9579d55356187f26 Mon Sep 17 00:00:00 2001 -From: Reid Wahl -Date: Fri, 23 Jun 2023 15:29:17 -0700 -Subject: [PATCH 15/19] Refactor: libpacemaker: Return chosen node from - assign_instance() - -The return type was changed to bool by commit 97f67da8. However, an -upcoming commit will need the assigned-to node. - -Ref T489 - -Signed-off-by: Reid Wahl ---- - lib/pacemaker/pcmk_sched_instances.c | 28 ++++++++++++++++------------ - 1 file changed, 16 insertions(+), 12 deletions(-) - -diff --git a/lib/pacemaker/pcmk_sched_instances.c b/lib/pacemaker/pcmk_sched_instances.c -index 1b051cb2ed9..64c027b20b1 100644 ---- a/lib/pacemaker/pcmk_sched_instances.c -+++ b/lib/pacemaker/pcmk_sched_instances.c -@@ -580,7 +580,7 @@ increment_parent_count(pe_resource_t *instance, const pe_node_t *assigned_to) - - /*! - * \internal -- * \brief Choose a node for an instance -+ * \brief Assign an instance to a node - * - * \param[in,out] instance Clone instance or bundle replica container - * \param[in] prefer If not NULL, attempt early assignment to this -@@ -588,9 +588,9 @@ increment_parent_count(pe_resource_t *instance, const pe_node_t *assigned_to) - * perform final assignment - * \param[in] max_per_node Assign at most this many instances to one node - * -- * \return true if \p instance could be assigned to a node, otherwise false -+ * \return Node to which \p instance is assigned - */ --static bool -+static const pe_node_t * - assign_instance(pe_resource_t *instance, const pe_node_t *prefer, - int max_per_node) - { -@@ -603,7 +603,7 @@ assign_instance(pe_resource_t *instance, const pe_node_t *prefer, - pe_rsc_debug(instance, - "Assignment loop detected involving %s colocations", - instance->id); -- return false; -+ return NULL; - } - - if (prefer != NULL) { // Possible early assignment to preferred node -@@ -616,7 +616,7 @@ assign_instance(pe_resource_t *instance, const pe_node_t *prefer, - pe_rsc_trace(instance, - "Not assigning %s to preferred node %s: unavailable", - instance->id, pe__node_name(prefer)); -- return false; -+ return NULL; - } - } - -@@ -644,7 +644,7 @@ assign_instance(pe_resource_t *instance, const pe_node_t *prefer, - } - - increment_parent_count(instance, chosen); -- return chosen != NULL; -+ return chosen; - } - - /*! -@@ -763,11 +763,15 @@ pcmk__assign_instances(pe_resource_t *collective, GList *instances, - } - - current = preferred_node(collective, instance, optimal_per_node); -- if ((current != NULL) -- && assign_instance(instance, current, max_per_node)) { -- pe_rsc_trace(collective, "Assigned %s to current node %s", -- instance->id, pe__node_name(current)); -- assigned++; -+ if (current != NULL) { -+ const pe_node_t *chosen = assign_instance(instance, current, -+ max_per_node); -+ -+ if (pe__same_node(chosen, current)) { -+ pe_rsc_trace(collective, "Assigned %s to current node %s", -+ instance->id, pe__node_name(current)); -+ assigned++; -+ } - } - } - -@@ -802,7 +806,7 @@ pcmk__assign_instances(pe_resource_t *collective, GList *instances, - resource_location(instance, NULL, -INFINITY, - "collective_limit_reached", collective->cluster); - -- } else if (assign_instance(instance, NULL, max_per_node)) { -+ } else if (assign_instance(instance, NULL, max_per_node) != NULL) { - assigned++; - } - } - -From 010649ef135ee0d4aca916d2d61c79bcba446951 Mon Sep 17 00:00:00 2001 -From: Reid Wahl -Date: Fri, 23 Jun 2023 21:30:47 -0700 -Subject: [PATCH 16/19] Refactor: libpacemaker: New stop_if_fail argument for - assign() method - -...of resource_alloc_functions_t. This will allow us to do a fully -reversible assignment. - -Currently pcmk__unassign_resource() undoes everything assignment-related -but can't undo changes to roles and actions. - -Now, if stop_if_fail is true, the assign() method and -pcmk__assign_resource() behave as before. - -If stop_if_fail is false and assignment succeeds, we can safely either -consider the assignment final or revert it via -pcmk__unassign_resource(). If assignment fails, the effect is as if we -had called pcmk__unassign_resource(); there are no side effects on next -role or actions. - -Ref T489 - -Signed-off-by: Reid Wahl ---- - include/pcmki/pcmki_sched_allocate.h | 3 +- - lib/pacemaker/libpacemaker_private.h | 30 ++++++++++++---- - lib/pacemaker/pcmk_sched_bundle.c | 30 +++++++++++----- - lib/pacemaker/pcmk_sched_clone.c | 22 +++++++++--- - lib/pacemaker/pcmk_sched_group.c | 18 +++++++--- - lib/pacemaker/pcmk_sched_instances.c | 24 +++++++------ - lib/pacemaker/pcmk_sched_primitive.c | 52 ++++++++++++++++++++-------- - lib/pacemaker/pcmk_sched_resource.c | 41 ++++++++++++++++------ - lib/pacemaker/pcmk_scheduler.c | 4 +-- - 9 files changed, 163 insertions(+), 61 deletions(-) - -diff --git a/include/pcmki/pcmki_sched_allocate.h b/include/pcmki/pcmki_sched_allocate.h -index 32044ea96d4..f027d1211f0 100644 ---- a/include/pcmki/pcmki_sched_allocate.h -+++ b/include/pcmki/pcmki_sched_allocate.h -@@ -19,7 +19,8 @@ - # include - # include - --pe_node_t *pcmk__bundle_allocate(pe_resource_t *rsc, const pe_node_t *prefer); -+pe_node_t *pcmk__bundle_allocate(pe_resource_t *rsc, const pe_node_t *prefer, -+ bool stop_if_fail); - void pcmk__bundle_create_actions(pe_resource_t *rsc); - bool pcmk__bundle_create_probe(pe_resource_t *rsc, pe_node_t *node); - void pcmk__bundle_internal_constraints(pe_resource_t *rsc); -diff --git a/lib/pacemaker/libpacemaker_private.h b/lib/pacemaker/libpacemaker_private.h -index 8cdd13f7304..642176aafcd 100644 ---- a/lib/pacemaker/libpacemaker_private.h -+++ b/lib/pacemaker/libpacemaker_private.h -@@ -58,12 +58,24 @@ struct resource_alloc_functions_s { - * \internal - * \brief Assign a resource to a node - * -- * \param[in,out] rsc Resource to assign to a node -- * \param[in] prefer Node to prefer, if all else is equal -+ * \param[in,out] rsc Resource to assign to a node -+ * \param[in] prefer Node to prefer, if all else is equal -+ * \param[in] stop_if_fail If \c true and \p rsc can't be assigned to a -+ * node, set next role to stopped and update -+ * existing actions (if \p rsc is not a -+ * primitive, this applies to its primitive -+ * descendants instead) - * - * \return Node that \p rsc is assigned to, if assigned entirely to one node -+ * -+ * \note If \p stop_if_fail is \c false, then \c pcmk__unassign_resource() -+ * can completely undo the assignment. A successful assignment can be -+ * either undone or left alone as final. A failed assignment has the -+ * same effect as calling pcmk__unassign_resource(); there are no side -+ * effects on roles or actions. - */ -- pe_node_t *(*assign)(pe_resource_t *rsc, const pe_node_t *prefer); -+ pe_node_t *(*assign)(pe_resource_t *rsc, const pe_node_t *prefer, -+ bool stop_if_fail); - - /*! - * \internal -@@ -649,7 +661,8 @@ void pcmk__add_bundle_meta_to_xml(xmlNode *args_xml, const pe_action_t *action); - // Primitives (pcmk_sched_primitive.c) - - G_GNUC_INTERNAL --pe_node_t *pcmk__primitive_assign(pe_resource_t *rsc, const pe_node_t *prefer); -+pe_node_t *pcmk__primitive_assign(pe_resource_t *rsc, const pe_node_t *prefer, -+ bool stop_if_fail); - - G_GNUC_INTERNAL - void pcmk__primitive_create_actions(pe_resource_t *rsc); -@@ -696,7 +709,8 @@ void pcmk__primitive_shutdown_lock(pe_resource_t *rsc); - // Groups (pcmk_sched_group.c) - - G_GNUC_INTERNAL --pe_node_t *pcmk__group_assign(pe_resource_t *rsc, const pe_node_t *prefer); -+pe_node_t *pcmk__group_assign(pe_resource_t *rsc, const pe_node_t *prefer, -+ bool stop_if_fail); - - G_GNUC_INTERNAL - void pcmk__group_create_actions(pe_resource_t *rsc); -@@ -756,7 +770,8 @@ void pcmk__group_shutdown_lock(pe_resource_t *rsc); - // Clones (pcmk_sched_clone.c) - - G_GNUC_INTERNAL --pe_node_t *pcmk__clone_assign(pe_resource_t *rsc, const pe_node_t *prefer); -+pe_node_t *pcmk__clone_assign(pe_resource_t *rsc, const pe_node_t *prefer, -+ bool stop_if_fail); - - G_GNUC_INTERNAL - void pcmk__clone_apply_coloc_score(pe_resource_t *dependent, -@@ -915,7 +930,8 @@ G_GNUC_INTERNAL - void pcmk__output_resource_actions(pe_resource_t *rsc); - - G_GNUC_INTERNAL --bool pcmk__assign_resource(pe_resource_t *rsc, pe_node_t *node, bool force); -+bool pcmk__assign_resource(pe_resource_t *rsc, pe_node_t *node, bool force, -+ bool stop_if_fail); - - G_GNUC_INTERNAL - void pcmk__unassign_resource(pe_resource_t *rsc); -diff --git a/lib/pacemaker/pcmk_sched_bundle.c b/lib/pacemaker/pcmk_sched_bundle.c -index 5682744395a..05a8626c889 100644 ---- a/lib/pacemaker/pcmk_sched_bundle.c -+++ b/lib/pacemaker/pcmk_sched_bundle.c -@@ -36,13 +36,24 @@ is_bundle_node(pe__bundle_variant_data_t *data, pe_node_t *node) - * \internal - * \brief Assign a bundle resource to a node - * -- * \param[in,out] rsc Resource to assign to a node -- * \param[in] prefer Node to prefer, if all else is equal -+ * \param[in,out] rsc Resource to assign to a node -+ * \param[in] prefer Node to prefer, if all else is equal -+ * \param[in] stop_if_fail If \c true and a primitive descendant of \p rsc -+ * can't be assigned to a node, set the -+ * descendant's next role to stopped and update -+ * existing actions - * - * \return Node that \p rsc is assigned to, if assigned entirely to one node -+ * -+ * \note If \p stop_if_fail is \c false, then \c pcmk__unassign_resource() can -+ * completely undo the assignment. A successful assignment can be either -+ * undone or left alone as final. A failed assignment has the same effect -+ * as calling pcmk__unassign_resource(); there are no side effects on -+ * roles or actions. - */ - pe_node_t * --pcmk__bundle_allocate(pe_resource_t *rsc, const pe_node_t *prefer) -+pcmk__bundle_allocate(pe_resource_t *rsc, const pe_node_t *prefer, -+ bool stop_if_fail) - { - GList *containers = NULL; - pe__bundle_variant_data_t *bundle_data = NULL; -@@ -71,7 +82,7 @@ pcmk__bundle_allocate(pe_resource_t *rsc, const pe_node_t *prefer) - if (replica->ip) { - pe_rsc_trace(rsc, "Allocating bundle %s IP %s", - rsc->id, replica->ip->id); -- replica->ip->cmds->assign(replica->ip, prefer); -+ replica->ip->cmds->assign(replica->ip, prefer, stop_if_fail); - } - - container_host = replica->container->allocated_to; -@@ -89,7 +100,8 @@ pcmk__bundle_allocate(pe_resource_t *rsc, const pe_node_t *prefer) - if (replica->remote) { - pe_rsc_trace(rsc, "Allocating bundle %s connection %s", - rsc->id, replica->remote->id); -- replica->remote->cmds->assign(replica->remote, prefer); -+ replica->remote->cmds->assign(replica->remote, prefer, -+ stop_if_fail); - } - - // Explicitly allocate replicas' children before bundle child -@@ -110,7 +122,8 @@ pcmk__bundle_allocate(pe_resource_t *rsc, const pe_node_t *prefer) - pe__set_resource_flags(replica->child->parent, pe_rsc_allocating); - pe_rsc_trace(rsc, "Allocating bundle %s replica child %s", - rsc->id, replica->child->id); -- replica->child->cmds->assign(replica->child, replica->node); -+ replica->child->cmds->assign(replica->child, replica->node, -+ stop_if_fail); - pe__clear_resource_flags(replica->child->parent, - pe_rsc_allocating); - } -@@ -129,7 +142,8 @@ pcmk__bundle_allocate(pe_resource_t *rsc, const pe_node_t *prefer) - } - pe_rsc_trace(rsc, "Allocating bundle %s child %s", - rsc->id, bundle_data->child->id); -- bundle_data->child->cmds->assign(bundle_data->child, prefer); -+ bundle_data->child->cmds->assign(bundle_data->child, prefer, -+ stop_if_fail); - } - - pe__clear_resource_flags(rsc, pe_rsc_allocating|pe_rsc_provisional); -@@ -457,7 +471,7 @@ pcmk__bundle_apply_coloc_score(pe_resource_t *dependent, - } else if (colocation->score >= INFINITY) { - crm_notice("Cannot pair %s with instance of %s", - dependent->id, primary->id); -- pcmk__assign_resource(dependent, NULL, true); -+ pcmk__assign_resource(dependent, NULL, true, true); - - } else { - pe_rsc_debug(primary, "Cannot pair %s with instance of %s", -diff --git a/lib/pacemaker/pcmk_sched_clone.c b/lib/pacemaker/pcmk_sched_clone.c -index 934f512d549..229257fd2be 100644 ---- a/lib/pacemaker/pcmk_sched_clone.c -+++ b/lib/pacemaker/pcmk_sched_clone.c -@@ -18,13 +18,24 @@ - * \internal - * \brief Assign a clone resource's instances to nodes - * -- * \param[in,out] rsc Clone resource to assign -- * \param[in] prefer Node to prefer, if all else is equal -+ * \param[in,out] rsc Clone resource to assign -+ * \param[in] prefer Node to prefer, if all else is equal -+ * \param[in] stop_if_fail If \c true and a primitive descendant of \p rsc -+ * can't be assigned to a node, set the -+ * descendant's next role to stopped and update -+ * existing actions - * - * \return NULL (clones are not assigned to a single node) -+ * -+ * \note If \p stop_if_fail is \c false, then \c pcmk__unassign_resource() can -+ * completely undo the assignment. A successful assignment can be either -+ * undone or left alone as final. A failed assignment has the same effect -+ * as calling pcmk__unassign_resource(); there are no side effects on -+ * roles or actions. - */ - pe_node_t * --pcmk__clone_assign(pe_resource_t *rsc, const pe_node_t *prefer) -+pcmk__clone_assign(pe_resource_t *rsc, const pe_node_t *prefer, -+ bool stop_if_fail) - { - CRM_ASSERT(pe_rsc_is_clone(rsc)); - -@@ -53,7 +64,8 @@ pcmk__clone_assign(pe_resource_t *rsc, const pe_node_t *prefer) - - pe_rsc_trace(rsc, "%s: Assigning colocation %s primary %s first", - rsc->id, constraint->id, constraint->primary->id); -- constraint->primary->cmds->assign(constraint->primary, prefer); -+ constraint->primary->cmds->assign(constraint->primary, prefer, -+ stop_if_fail); - } - - /* If any resources are colocated with this one, consider their preferences. -@@ -305,7 +317,7 @@ pcmk__clone_apply_coloc_score(pe_resource_t *dependent, - } else if (colocation->score >= INFINITY) { - crm_notice("Cannot pair %s with instance of %s", - dependent->id, primary->id); -- pcmk__assign_resource(dependent, NULL, true); -+ pcmk__assign_resource(dependent, NULL, true, true); - - } else { - pe_rsc_debug(primary, "Cannot pair %s with instance of %s", -diff --git a/lib/pacemaker/pcmk_sched_group.c b/lib/pacemaker/pcmk_sched_group.c -index cb139f7ddf9..55d890a5c4f 100644 ---- a/lib/pacemaker/pcmk_sched_group.c -+++ b/lib/pacemaker/pcmk_sched_group.c -@@ -20,13 +20,23 @@ - * \internal - * \brief Assign a group resource to a node - * -- * \param[in,out] rsc Group resource to assign to a node -- * \param[in] prefer Node to prefer, if all else is equal -+ * \param[in,out] rsc Group resource to assign to a node -+ * \param[in] prefer Node to prefer, if all else is equal -+ * \param[in] stop_if_fail If \c true and a child of \p rsc can't be -+ * assigned to a node, set the child's next role to -+ * stopped and update existing actions - * - * \return Node that \p rsc is assigned to, if assigned entirely to one node -+ * -+ * \note If \p stop_if_fail is \c false, then \c pcmk__unassign_resource() can -+ * completely undo the assignment. A successful assignment can be either -+ * undone or left alone as final. A failed assignment has the same effect -+ * as calling pcmk__unassign_resource(); there are no side effects on -+ * roles or actions. - */ - pe_node_t * --pcmk__group_assign(pe_resource_t *rsc, const pe_node_t *prefer) -+pcmk__group_assign(pe_resource_t *rsc, const pe_node_t *prefer, -+ bool stop_if_fail) - { - pe_node_t *first_assigned_node = NULL; - pe_resource_t *first_member = NULL; -@@ -61,7 +71,7 @@ pcmk__group_assign(pe_resource_t *rsc, const pe_node_t *prefer) - - pe_rsc_trace(rsc, "Assigning group %s member %s", - rsc->id, member->id); -- node = member->cmds->assign(member, prefer); -+ node = member->cmds->assign(member, prefer, stop_if_fail); - if (first_assigned_node == NULL) { - first_assigned_node = node; - } -diff --git a/lib/pacemaker/pcmk_sched_instances.c b/lib/pacemaker/pcmk_sched_instances.c -index 64c027b20b1..b551f3bee61 100644 ---- a/lib/pacemaker/pcmk_sched_instances.c -+++ b/lib/pacemaker/pcmk_sched_instances.c -@@ -623,22 +623,26 @@ assign_instance(pe_resource_t *instance, const pe_node_t *prefer, - ban_unavailable_allowed_nodes(instance, max_per_node); - - if (prefer == NULL) { // Final assignment -- chosen = instance->cmds->assign(instance, NULL); -+ chosen = instance->cmds->assign(instance, NULL, true); - - } else { // Possible early assignment to preferred node - GHashTable *backup = NULL; - - pcmk__copy_node_tables(instance, &backup); -- chosen = instance->cmds->assign(instance, prefer); -- -- // Revert nodes if preferred node won't be assigned -- if ((chosen != NULL) && (chosen->details != prefer->details)) { -- crm_info("Not assigning %s to preferred node %s: %s is better", -- instance->id, pe__node_name(prefer), -- pe__node_name(chosen)); -+ chosen = instance->cmds->assign(instance, prefer, false); -+ -+ if (!pe__same_node(chosen, prefer)) { -+ // Revert nodes if preferred node won't be assigned -+ if (chosen != NULL) { -+ pe_rsc_info(instance, -+ "Not assigning %s to preferred node %s: " -+ "%s is better", -+ instance->id, pe__node_name(prefer), -+ pe__node_name(chosen)); -+ chosen = NULL; -+ } - pcmk__restore_node_tables(instance, backup); - pcmk__unassign_resource(instance); -- chosen = NULL; - } - g_hash_table_destroy(backup); - } -@@ -1181,7 +1185,7 @@ unassign_if_mandatory(const pe_action_t *first, const pe_action_t *then, - "Inhibiting %s from being active " - "because there is no %s instance to interleave", - then_instance->id, first->rsc->id); -- return pcmk__assign_resource(then_instance, NULL, true); -+ return pcmk__assign_resource(then_instance, NULL, true, true); - } - return false; - } -diff --git a/lib/pacemaker/pcmk_sched_primitive.c b/lib/pacemaker/pcmk_sched_primitive.c -index 2470b08ed69..50f11138f23 100644 ---- a/lib/pacemaker/pcmk_sched_primitive.c -+++ b/lib/pacemaker/pcmk_sched_primitive.c -@@ -141,13 +141,23 @@ sorted_allowed_nodes(const pe_resource_t *rsc) - * \internal - * \brief Assign a resource to its best allowed node, if possible - * -- * \param[in,out] rsc Resource to choose a node for -- * \param[in] prefer If not NULL, prefer this node when all else equal -+ * \param[in,out] rsc Resource to choose a node for -+ * \param[in] prefer If not \c NULL, prefer this node when all else -+ * equal -+ * \param[in] stop_if_fail If \c true and \p rsc can't be assigned to a -+ * node, set next role to stopped and update -+ * existing actions - * - * \return true if \p rsc could be assigned to a node, otherwise false -+ * -+ * \note If \p stop_if_fail is \c false, then \c pcmk__unassign_resource() can -+ * completely undo the assignment. A successful assignment can be either -+ * undone or left alone as final. A failed assignment has the same effect -+ * as calling pcmk__unassign_resource(); there are no side effects on -+ * roles or actions. - */ - static bool --assign_best_node(pe_resource_t *rsc, const pe_node_t *prefer) -+assign_best_node(pe_resource_t *rsc, const pe_node_t *prefer, bool stop_if_fail) - { - GList *nodes = NULL; - pe_node_t *chosen = NULL; -@@ -259,7 +269,7 @@ assign_best_node(pe_resource_t *rsc, const pe_node_t *prefer) - pe__node_name(chosen), rsc->id, g_list_length(nodes)); - } - -- pcmk__assign_resource(rsc, chosen, false); -+ pcmk__assign_resource(rsc, chosen, false, stop_if_fail); - g_list_free(nodes); - return rsc->allocated_to != NULL; - } -@@ -292,7 +302,7 @@ apply_this_with(gpointer data, gpointer user_data) - "(score=%d role=%s)", - rsc->id, colocation->id, other->id, - colocation->score, role2text(colocation->dependent_role)); -- other->cmds->assign(other, NULL); -+ other->cmds->assign(other, NULL, true); - } - - // Apply the colocation score to this resource's allowed node scores -@@ -351,13 +361,23 @@ remote_connection_assigned(const pe_resource_t *connection) - * \internal - * \brief Assign a primitive resource to a node - * -- * \param[in,out] rsc Resource to assign to a node -- * \param[in] prefer Node to prefer, if all else is equal -+ * \param[in,out] rsc Resource to assign to a node -+ * \param[in] prefer Node to prefer, if all else is equal -+ * \param[in] stop_if_fail If \c true and \p rsc can't be assigned to a -+ * node, set next role to stopped and update -+ * existing actions - * - * \return Node that \p rsc is assigned to, if assigned entirely to one node -+ * -+ * \note If \p stop_if_fail is \c false, then \c pcmk__unassign_resource() can -+ * completely undo the assignment. A successful assignment can be either -+ * undone or left alone as final. A failed assignment has the same effect -+ * as calling pcmk__unassign_resource(); there are no side effects on -+ * roles or actions. - */ - pe_node_t * --pcmk__primitive_assign(pe_resource_t *rsc, const pe_node_t *prefer) -+pcmk__primitive_assign(pe_resource_t *rsc, const pe_node_t *prefer, -+ bool stop_if_fail) - { - GList *this_with_colocations = NULL; - GList *with_this_colocations = NULL; -@@ -371,7 +391,7 @@ pcmk__primitive_assign(pe_resource_t *rsc, const pe_node_t *prefer) - && !pcmk_is_set(rsc->parent->flags, pe_rsc_allocating)) { - pe_rsc_debug(rsc, "%s: Assigning parent %s first", - rsc->id, rsc->parent->id); -- rsc->parent->cmds->assign(rsc->parent, prefer); -+ rsc->parent->cmds->assign(rsc->parent, prefer, stop_if_fail); - } - - if (!pcmk_is_set(rsc->flags, pe_rsc_provisional)) { -@@ -474,20 +494,24 @@ pcmk__primitive_assign(pe_resource_t *rsc, const pe_node_t *prefer) - } - pe_rsc_info(rsc, "Unmanaged resource %s assigned to %s: %s", rsc->id, - (assign_to? assign_to->details->uname : "no node"), reason); -- pcmk__assign_resource(rsc, assign_to, true); -+ pcmk__assign_resource(rsc, assign_to, true, stop_if_fail); - - } else if (pcmk_is_set(rsc->cluster->flags, pe_flag_stop_everything)) { -- pe_rsc_debug(rsc, "Forcing %s to stop: stop-all-resources", rsc->id); -- pcmk__assign_resource(rsc, NULL, true); -+ // Must stop at some point, but be consistent with stop_if_fail -+ if (stop_if_fail) { -+ pe_rsc_debug(rsc, "Forcing %s to stop: stop-all-resources", -+ rsc->id); -+ } -+ pcmk__assign_resource(rsc, NULL, true, stop_if_fail); - - } else if (pcmk_is_set(rsc->flags, pe_rsc_provisional) -- && assign_best_node(rsc, prefer)) { -+ && assign_best_node(rsc, prefer, stop_if_fail)) { - // Assignment successful - - } else if (rsc->allocated_to == NULL) { - if (!pcmk_is_set(rsc->flags, pe_rsc_orphan)) { - pe_rsc_info(rsc, "Resource %s cannot run anywhere", rsc->id); -- } else if (rsc->running_on != NULL) { -+ } else if ((rsc->running_on != NULL) && stop_if_fail) { - pe_rsc_info(rsc, "Stopping orphan resource %s", rsc->id); - } - -diff --git a/lib/pacemaker/pcmk_sched_resource.c b/lib/pacemaker/pcmk_sched_resource.c -index 8f703789b20..36f49dc49b9 100644 ---- a/lib/pacemaker/pcmk_sched_resource.c -+++ b/lib/pacemaker/pcmk_sched_resource.c -@@ -335,25 +335,38 @@ pcmk__output_resource_actions(pe_resource_t *rsc) - * - * Assign a specified resource and its children (if any) to a specified node, if - * the node can run the resource (or unconditionally, if \p force is true). Mark -- * the resources as no longer provisional. If a resource can't be assigned (or -- * \p node is \c NULL), unassign any previous assignment, set next role to -- * stopped, and update any existing actions scheduled for it. -+ * the resources as no longer provisional. - * -- * \param[in,out] rsc Resource to assign -- * \param[in,out] node Node to assign \p rsc to -- * \param[in] force If true, assign to \p node even if unavailable -+ * If a resource can't be assigned (or \p node is \c NULL), unassign any -+ * previous assignment. If \p stop_if_fail is \c true, set next role to stopped -+ * and update any existing actions scheduled for the resource. -+ * -+ * \param[in,out] rsc Resource to assign -+ * \param[in,out] node Node to assign \p rsc to -+ * \param[in] force If true, assign to \p node even if unavailable -+ * \param[in] stop_if_fail If \c true and either \p rsc can't be assigned -+ * or \p chosen is \c NULL, set next role to -+ * stopped and update existing actions (if \p rsc -+ * is not a primitive, this applies to its -+ * primitive descendants instead) - * - * \return \c true if the assignment of \p rsc changed, or \c false otherwise - * - * \note Assigning a resource to the NULL node using this function is different -- * from calling pcmk__unassign_resource(), in that it will also update any -+ * from calling pcmk__unassign_resource(), in that it may also update any - * actions created for the resource. - * \note The \c resource_alloc_functions_t:assign() method is preferred, unless - * a resource should be assigned to the \c NULL node or every resource in - * a tree should be assigned to the same node. -+ * \note If \p stop_if_fail is \c false, then \c pcmk__unassign_resource() can -+ * completely undo the assignment. A successful assignment can be either -+ * undone or left alone as final. A failed assignment has the same effect -+ * as calling pcmk__unassign_resource(); there are no side effects on -+ * roles or actions. - */ - bool --pcmk__assign_resource(pe_resource_t *rsc, pe_node_t *node, bool force) -+pcmk__assign_resource(pe_resource_t *rsc, pe_node_t *node, bool force, -+ bool stop_if_fail) - { - bool changed = false; - -@@ -363,7 +376,8 @@ pcmk__assign_resource(pe_resource_t *rsc, pe_node_t *node, bool force) - for (GList *iter = rsc->children; iter != NULL; iter = iter->next) { - pe_resource_t *child_rsc = iter->data; - -- changed |= pcmk__assign_resource(child_rsc, node, force); -+ changed |= pcmk__assign_resource(child_rsc, node, force, -+ stop_if_fail); - } - return changed; - } -@@ -382,7 +396,10 @@ pcmk__assign_resource(pe_resource_t *rsc, pe_node_t *node, bool force) - rsc->id, pe__node_name(node), - (pcmk__node_available(node, true, false)? "" : "not"), - pcmk_readable_score(node->weight)); -- pe__set_next_role(rsc, RSC_ROLE_STOPPED, "node availability"); -+ -+ if (stop_if_fail) { -+ pe__set_next_role(rsc, RSC_ROLE_STOPPED, "node availability"); -+ } - node = NULL; - } - -@@ -398,6 +415,10 @@ pcmk__assign_resource(pe_resource_t *rsc, pe_node_t *node, bool force) - char *rc_stopped = NULL; - - pe_rsc_debug(rsc, "Could not assign %s to a node", rsc->id); -+ -+ if (!stop_if_fail) { -+ return changed; -+ } - pe__set_next_role(rsc, RSC_ROLE_STOPPED, "unable to assign"); - - for (GList *iter = rsc->actions; iter != NULL; iter = iter->next) { -diff --git a/lib/pacemaker/pcmk_scheduler.c b/lib/pacemaker/pcmk_scheduler.c -index b4e670d865c..508cd5721c4 100644 ---- a/lib/pacemaker/pcmk_scheduler.c -+++ b/lib/pacemaker/pcmk_scheduler.c -@@ -318,7 +318,7 @@ allocate_resources(pe_working_set_t *data_set) - if (rsc->is_remote_node) { - pe_rsc_trace(rsc, "Allocating remote connection resource '%s'", - rsc->id); -- rsc->cmds->assign(rsc, rsc->partial_migration_target); -+ rsc->cmds->assign(rsc, rsc->partial_migration_target, true); - } - } - } -@@ -330,7 +330,7 @@ allocate_resources(pe_working_set_t *data_set) - if (!rsc->is_remote_node) { - pe_rsc_trace(rsc, "Allocating %s resource '%s'", - crm_element_name(rsc->xml), rsc->id); -- rsc->cmds->assign(rsc, NULL); -+ rsc->cmds->assign(rsc, NULL, true); - } - } - - -From a698dd1e17f184977f87c4ef44c2eb5b9bd933f6 Mon Sep 17 00:00:00 2001 -From: Reid Wahl -Date: Mon, 10 Jul 2023 02:44:46 -0700 -Subject: [PATCH 17/19] Test: scheduler: Update tests after new stop_if_fail - argument - -Some scores are repeated since we're able to back out of a failed early -assignment now. - -Only one test changes otherwise. bug-1822 has a score change from --INFINITY to 49. However, the partially active group is still not -allowed to promote, which is the purpose of the test. - -Ref T489 - -Signed-off-by: Reid Wahl ---- - cts/scheduler/scores/594.scores | 3 +++ - cts/scheduler/scores/bug-1822.scores | 2 +- - .../bug-5014-CLONE-A-stop-B-started.scores | 1 + - cts/scheduler/scores/bug-lf-2171.scores | 4 ++++ - cts/scheduler/scores/bug-lf-2422.scores | 16 ++++++++++++++++ - cts/scheduler/scores/bug-lf-2453.scores | 4 ++++ - cts/scheduler/scores/bug-lf-2574.scores | 3 +++ - .../scores/bundle-order-stop-clone.scores | 4 ++++ - cts/scheduler/scores/clone-max-zero.scores | 8 ++++++++ - cts/scheduler/scores/cloned-group-stop.scores | 4 ++++ - cts/scheduler/scores/complex_enforce_colo.scores | 9 +++++++++ - cts/scheduler/scores/enforce-colo1.scores | 9 +++++++++ - .../scores/promoted-asymmetrical-order.scores | 4 ++++ - .../scores/promoted-failed-demote-2.scores | 10 ++++++++++ - .../scores/promoted-failed-demote.scores | 10 ++++++++++ - 15 files changed, 90 insertions(+), 1 deletion(-) - -diff --git a/cts/scheduler/scores/594.scores b/cts/scheduler/scores/594.scores -index 5e99750df21..96c8f441b98 100644 ---- a/cts/scheduler/scores/594.scores -+++ b/cts/scheduler/scores/594.scores -@@ -21,8 +21,11 @@ pcmk__primitive_assign: child_DoFencing:1 allocation score on hadev1: 1 - pcmk__primitive_assign: child_DoFencing:1 allocation score on hadev2: -INFINITY - pcmk__primitive_assign: child_DoFencing:1 allocation score on hadev3: -INFINITY - pcmk__primitive_assign: child_DoFencing:2 allocation score on hadev1: -INFINITY -+pcmk__primitive_assign: child_DoFencing:2 allocation score on hadev1: -INFINITY -+pcmk__primitive_assign: child_DoFencing:2 allocation score on hadev2: -INFINITY - pcmk__primitive_assign: child_DoFencing:2 allocation score on hadev2: -INFINITY - pcmk__primitive_assign: child_DoFencing:2 allocation score on hadev3: -INFINITY -+pcmk__primitive_assign: child_DoFencing:2 allocation score on hadev3: -INFINITY - pcmk__primitive_assign: rsc_hadev1 allocation score on hadev1: 100 - pcmk__primitive_assign: rsc_hadev1 allocation score on hadev2: 0 - pcmk__primitive_assign: rsc_hadev1 allocation score on hadev3: 0 -diff --git a/cts/scheduler/scores/bug-1822.scores b/cts/scheduler/scores/bug-1822.scores -index 82191d1e74b..0a9056bbf3e 100644 ---- a/cts/scheduler/scores/bug-1822.scores -+++ b/cts/scheduler/scores/bug-1822.scores -@@ -1,5 +1,5 @@ - --ms-sf_group:0 promotion score on process2b: -INFINITY -+ms-sf_group:0 promotion score on process2b: 49 - ms-sf_group:1 promotion score on none: 0 - pcmk__clone_assign: ms-sf allocation score on process1a: 0 - pcmk__clone_assign: ms-sf allocation score on process2b: 0 -diff --git a/cts/scheduler/scores/bug-5014-CLONE-A-stop-B-started.scores b/cts/scheduler/scores/bug-5014-CLONE-A-stop-B-started.scores -index e698b145274..d79208c7336 100644 ---- a/cts/scheduler/scores/bug-5014-CLONE-A-stop-B-started.scores -+++ b/cts/scheduler/scores/bug-5014-CLONE-A-stop-B-started.scores -@@ -5,3 +5,4 @@ pcmk__clone_assign: clone1 allocation score on fc16-builder: 0 - pcmk__clone_assign: clone2 allocation score on fc16-builder: 0 - pcmk__primitive_assign: ClusterIP2:0 allocation score on fc16-builder: 1 - pcmk__primitive_assign: ClusterIP:0 allocation score on fc16-builder: -INFINITY -+pcmk__primitive_assign: ClusterIP:0 allocation score on fc16-builder: -INFINITY -diff --git a/cts/scheduler/scores/bug-lf-2171.scores b/cts/scheduler/scores/bug-lf-2171.scores -index 7d2bdd45307..14cc28a88c5 100644 ---- a/cts/scheduler/scores/bug-lf-2171.scores -+++ b/cts/scheduler/scores/bug-lf-2171.scores -@@ -12,8 +12,12 @@ pcmk__group_assign: res_Dummy2 allocation score on xenserver2: 0 - pcmk__group_assign: res_Dummy3 allocation score on xenserver1: 200 - pcmk__group_assign: res_Dummy3 allocation score on xenserver2: 0 - pcmk__primitive_assign: res_Dummy1:0 allocation score on xenserver1: -INFINITY -+pcmk__primitive_assign: res_Dummy1:0 allocation score on xenserver1: -INFINITY -+pcmk__primitive_assign: res_Dummy1:0 allocation score on xenserver2: -INFINITY - pcmk__primitive_assign: res_Dummy1:0 allocation score on xenserver2: -INFINITY - pcmk__primitive_assign: res_Dummy1:1 allocation score on xenserver1: -INFINITY -+pcmk__primitive_assign: res_Dummy1:1 allocation score on xenserver1: -INFINITY -+pcmk__primitive_assign: res_Dummy1:1 allocation score on xenserver2: -INFINITY - pcmk__primitive_assign: res_Dummy1:1 allocation score on xenserver2: -INFINITY - pcmk__primitive_assign: res_Dummy2 allocation score on xenserver1: 200 - pcmk__primitive_assign: res_Dummy2 allocation score on xenserver2: 0 -diff --git a/cts/scheduler/scores/bug-lf-2422.scores b/cts/scheduler/scores/bug-lf-2422.scores -index 99ff12e3bb6..77a284da9ce 100644 ---- a/cts/scheduler/scores/bug-lf-2422.scores -+++ b/cts/scheduler/scores/bug-lf-2422.scores -@@ -248,20 +248,36 @@ pcmk__primitive_assign: o2cb:3 allocation score on qa-suse-2: -INFINITY - pcmk__primitive_assign: o2cb:3 allocation score on qa-suse-3: -INFINITY - pcmk__primitive_assign: o2cb:3 allocation score on qa-suse-4: -INFINITY - pcmk__primitive_assign: ocfs:0 allocation score on qa-suse-1: -INFINITY -+pcmk__primitive_assign: ocfs:0 allocation score on qa-suse-1: -INFINITY -+pcmk__primitive_assign: ocfs:0 allocation score on qa-suse-2: -INFINITY - pcmk__primitive_assign: ocfs:0 allocation score on qa-suse-2: -INFINITY - pcmk__primitive_assign: ocfs:0 allocation score on qa-suse-3: -INFINITY -+pcmk__primitive_assign: ocfs:0 allocation score on qa-suse-3: -INFINITY -+pcmk__primitive_assign: ocfs:0 allocation score on qa-suse-4: -INFINITY - pcmk__primitive_assign: ocfs:0 allocation score on qa-suse-4: -INFINITY - pcmk__primitive_assign: ocfs:1 allocation score on qa-suse-1: -INFINITY -+pcmk__primitive_assign: ocfs:1 allocation score on qa-suse-1: -INFINITY -+pcmk__primitive_assign: ocfs:1 allocation score on qa-suse-2: -INFINITY - pcmk__primitive_assign: ocfs:1 allocation score on qa-suse-2: -INFINITY - pcmk__primitive_assign: ocfs:1 allocation score on qa-suse-3: -INFINITY -+pcmk__primitive_assign: ocfs:1 allocation score on qa-suse-3: -INFINITY -+pcmk__primitive_assign: ocfs:1 allocation score on qa-suse-4: -INFINITY - pcmk__primitive_assign: ocfs:1 allocation score on qa-suse-4: -INFINITY - pcmk__primitive_assign: ocfs:2 allocation score on qa-suse-1: -INFINITY -+pcmk__primitive_assign: ocfs:2 allocation score on qa-suse-1: -INFINITY -+pcmk__primitive_assign: ocfs:2 allocation score on qa-suse-2: -INFINITY - pcmk__primitive_assign: ocfs:2 allocation score on qa-suse-2: -INFINITY - pcmk__primitive_assign: ocfs:2 allocation score on qa-suse-3: -INFINITY -+pcmk__primitive_assign: ocfs:2 allocation score on qa-suse-3: -INFINITY -+pcmk__primitive_assign: ocfs:2 allocation score on qa-suse-4: -INFINITY - pcmk__primitive_assign: ocfs:2 allocation score on qa-suse-4: -INFINITY - pcmk__primitive_assign: ocfs:3 allocation score on qa-suse-1: -INFINITY -+pcmk__primitive_assign: ocfs:3 allocation score on qa-suse-1: -INFINITY -+pcmk__primitive_assign: ocfs:3 allocation score on qa-suse-2: -INFINITY - pcmk__primitive_assign: ocfs:3 allocation score on qa-suse-2: -INFINITY - pcmk__primitive_assign: ocfs:3 allocation score on qa-suse-3: -INFINITY -+pcmk__primitive_assign: ocfs:3 allocation score on qa-suse-3: -INFINITY -+pcmk__primitive_assign: ocfs:3 allocation score on qa-suse-4: -INFINITY - pcmk__primitive_assign: ocfs:3 allocation score on qa-suse-4: -INFINITY - pcmk__primitive_assign: sbd_stonith allocation score on qa-suse-1: 0 - pcmk__primitive_assign: sbd_stonith allocation score on qa-suse-2: 0 -diff --git a/cts/scheduler/scores/bug-lf-2453.scores b/cts/scheduler/scores/bug-lf-2453.scores -index eaee72d2002..3ef0f6dc375 100644 ---- a/cts/scheduler/scores/bug-lf-2453.scores -+++ b/cts/scheduler/scores/bug-lf-2453.scores -@@ -17,6 +17,10 @@ pcmk__primitive_assign: DummyResource:1 allocation score on domu1: -INFINITY - pcmk__primitive_assign: DummyResource:1 allocation score on domu2: INFINITY - pcmk__primitive_assign: PrimitiveResource1 allocation score on domu1: INFINITY - pcmk__primitive_assign: apache:0 allocation score on domu1: -INFINITY -+pcmk__primitive_assign: apache:0 allocation score on domu1: -INFINITY -+pcmk__primitive_assign: apache:0 allocation score on domu2: -INFINITY - pcmk__primitive_assign: apache:0 allocation score on domu2: -INFINITY - pcmk__primitive_assign: apache:1 allocation score on domu1: -INFINITY -+pcmk__primitive_assign: apache:1 allocation score on domu1: -INFINITY -+pcmk__primitive_assign: apache:1 allocation score on domu2: -INFINITY - pcmk__primitive_assign: apache:1 allocation score on domu2: -INFINITY -diff --git a/cts/scheduler/scores/bug-lf-2574.scores b/cts/scheduler/scores/bug-lf-2574.scores -index 0f5cf60a7e0..b4a1bd95841 100644 ---- a/cts/scheduler/scores/bug-lf-2574.scores -+++ b/cts/scheduler/scores/bug-lf-2574.scores -@@ -39,8 +39,11 @@ pcmk__primitive_assign: prmDummy1:2 allocation score on srv01: -INFINITY - pcmk__primitive_assign: prmDummy1:2 allocation score on srv02: -INFINITY - pcmk__primitive_assign: prmDummy1:2 allocation score on srv03: -INFINITY - pcmk__primitive_assign: prmPingd:0 allocation score on srv01: -INFINITY -+pcmk__primitive_assign: prmPingd:0 allocation score on srv01: -INFINITY -+pcmk__primitive_assign: prmPingd:0 allocation score on srv02: -INFINITY - pcmk__primitive_assign: prmPingd:0 allocation score on srv02: -INFINITY - pcmk__primitive_assign: prmPingd:0 allocation score on srv03: -INFINITY -+pcmk__primitive_assign: prmPingd:0 allocation score on srv03: -INFINITY - pcmk__primitive_assign: prmPingd:1 allocation score on srv01: -INFINITY - pcmk__primitive_assign: prmPingd:1 allocation score on srv02: -INFINITY - pcmk__primitive_assign: prmPingd:1 allocation score on srv03: INFINITY -diff --git a/cts/scheduler/scores/bundle-order-stop-clone.scores b/cts/scheduler/scores/bundle-order-stop-clone.scores -index 707260b80a9..06596e86a24 100644 ---- a/cts/scheduler/scores/bundle-order-stop-clone.scores -+++ b/cts/scheduler/scores/bundle-order-stop-clone.scores -@@ -147,8 +147,12 @@ pcmk__primitive_assign: galera-bundle-2 allocation score on metal-2: 0 - pcmk__primitive_assign: galera-bundle-2 allocation score on metal-3: INFINITY - pcmk__primitive_assign: galera-bundle-2 allocation score on rabbitmq-bundle-0: -INFINITY - pcmk__primitive_assign: galera-bundle-docker-0 allocation score on metal-1: -INFINITY -+pcmk__primitive_assign: galera-bundle-docker-0 allocation score on metal-1: -INFINITY -+pcmk__primitive_assign: galera-bundle-docker-0 allocation score on metal-2: -INFINITY - pcmk__primitive_assign: galera-bundle-docker-0 allocation score on metal-2: -INFINITY - pcmk__primitive_assign: galera-bundle-docker-0 allocation score on metal-3: -INFINITY -+pcmk__primitive_assign: galera-bundle-docker-0 allocation score on metal-3: -INFINITY -+pcmk__primitive_assign: galera-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY - pcmk__primitive_assign: galera-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY - pcmk__primitive_assign: galera-bundle-docker-1 allocation score on metal-1: -INFINITY - pcmk__primitive_assign: galera-bundle-docker-1 allocation score on metal-2: INFINITY -diff --git a/cts/scheduler/scores/clone-max-zero.scores b/cts/scheduler/scores/clone-max-zero.scores -index f1711b7885e..bd116a2764c 100644 ---- a/cts/scheduler/scores/clone-max-zero.scores -+++ b/cts/scheduler/scores/clone-max-zero.scores -@@ -26,10 +26,18 @@ pcmk__primitive_assign: drbd0:1 allocation score on c001n12: -INFINITY - pcmk__primitive_assign: fencing allocation score on c001n11: 0 - pcmk__primitive_assign: fencing allocation score on c001n12: 0 - pcmk__primitive_assign: o2cb:0 allocation score on c001n11: -INFINITY -+pcmk__primitive_assign: o2cb:0 allocation score on c001n11: -INFINITY -+pcmk__primitive_assign: o2cb:0 allocation score on c001n12: -INFINITY - pcmk__primitive_assign: o2cb:0 allocation score on c001n12: -INFINITY - pcmk__primitive_assign: o2cb:1 allocation score on c001n11: -INFINITY -+pcmk__primitive_assign: o2cb:1 allocation score on c001n11: -INFINITY -+pcmk__primitive_assign: o2cb:1 allocation score on c001n12: -INFINITY - pcmk__primitive_assign: o2cb:1 allocation score on c001n12: -INFINITY - pcmk__primitive_assign: ocfs2-1:0 allocation score on c001n11: -INFINITY -+pcmk__primitive_assign: ocfs2-1:0 allocation score on c001n11: -INFINITY -+pcmk__primitive_assign: ocfs2-1:0 allocation score on c001n12: -INFINITY - pcmk__primitive_assign: ocfs2-1:0 allocation score on c001n12: -INFINITY - pcmk__primitive_assign: ocfs2-1:1 allocation score on c001n11: -INFINITY -+pcmk__primitive_assign: ocfs2-1:1 allocation score on c001n11: -INFINITY -+pcmk__primitive_assign: ocfs2-1:1 allocation score on c001n12: -INFINITY - pcmk__primitive_assign: ocfs2-1:1 allocation score on c001n12: -INFINITY -diff --git a/cts/scheduler/scores/cloned-group-stop.scores b/cts/scheduler/scores/cloned-group-stop.scores -index be835fa5371..7e406c6ddc2 100644 ---- a/cts/scheduler/scores/cloned-group-stop.scores -+++ b/cts/scheduler/scores/cloned-group-stop.scores -@@ -122,8 +122,12 @@ pcmk__primitive_assign: mysql-fs allocation score on rhos4-node4: -INFINITY - pcmk__primitive_assign: mysql-vip allocation score on rhos4-node3: 300 - pcmk__primitive_assign: mysql-vip allocation score on rhos4-node4: -INFINITY - pcmk__primitive_assign: qpidd:0 allocation score on rhos4-node3: -INFINITY -+pcmk__primitive_assign: qpidd:0 allocation score on rhos4-node3: -INFINITY -+pcmk__primitive_assign: qpidd:0 allocation score on rhos4-node4: -INFINITY - pcmk__primitive_assign: qpidd:0 allocation score on rhos4-node4: -INFINITY - pcmk__primitive_assign: qpidd:1 allocation score on rhos4-node3: -INFINITY -+pcmk__primitive_assign: qpidd:1 allocation score on rhos4-node3: -INFINITY -+pcmk__primitive_assign: qpidd:1 allocation score on rhos4-node4: -INFINITY - pcmk__primitive_assign: qpidd:1 allocation score on rhos4-node4: -INFINITY - pcmk__primitive_assign: virt-fencing allocation score on rhos4-node3: 100 - pcmk__primitive_assign: virt-fencing allocation score on rhos4-node4: 0 -diff --git a/cts/scheduler/scores/complex_enforce_colo.scores b/cts/scheduler/scores/complex_enforce_colo.scores -index 9968e1097ef..a5d0b2b4125 100644 ---- a/cts/scheduler/scores/complex_enforce_colo.scores -+++ b/cts/scheduler/scores/complex_enforce_colo.scores -@@ -588,13 +588,22 @@ pcmk__primitive_assign: horizon:2 allocation score on rhos6-node1: -INFINITY - pcmk__primitive_assign: horizon:2 allocation score on rhos6-node2: -INFINITY - pcmk__primitive_assign: horizon:2 allocation score on rhos6-node3: 1 - pcmk__primitive_assign: keystone:0 allocation score on rhos6-node1: -INFINITY -+pcmk__primitive_assign: keystone:0 allocation score on rhos6-node1: -INFINITY -+pcmk__primitive_assign: keystone:0 allocation score on rhos6-node2: -INFINITY - pcmk__primitive_assign: keystone:0 allocation score on rhos6-node2: -INFINITY - pcmk__primitive_assign: keystone:0 allocation score on rhos6-node3: -INFINITY -+pcmk__primitive_assign: keystone:0 allocation score on rhos6-node3: -INFINITY - pcmk__primitive_assign: keystone:1 allocation score on rhos6-node1: -INFINITY -+pcmk__primitive_assign: keystone:1 allocation score on rhos6-node1: -INFINITY -+pcmk__primitive_assign: keystone:1 allocation score on rhos6-node2: -INFINITY - pcmk__primitive_assign: keystone:1 allocation score on rhos6-node2: -INFINITY - pcmk__primitive_assign: keystone:1 allocation score on rhos6-node3: -INFINITY -+pcmk__primitive_assign: keystone:1 allocation score on rhos6-node3: -INFINITY -+pcmk__primitive_assign: keystone:2 allocation score on rhos6-node1: -INFINITY - pcmk__primitive_assign: keystone:2 allocation score on rhos6-node1: -INFINITY - pcmk__primitive_assign: keystone:2 allocation score on rhos6-node2: -INFINITY -+pcmk__primitive_assign: keystone:2 allocation score on rhos6-node2: -INFINITY -+pcmk__primitive_assign: keystone:2 allocation score on rhos6-node3: -INFINITY - pcmk__primitive_assign: keystone:2 allocation score on rhos6-node3: -INFINITY - pcmk__primitive_assign: lb-haproxy:0 allocation score on rhos6-node1: 1 - pcmk__primitive_assign: lb-haproxy:0 allocation score on rhos6-node2: 0 -diff --git a/cts/scheduler/scores/enforce-colo1.scores b/cts/scheduler/scores/enforce-colo1.scores -index 8194789648a..262cbd94a30 100644 ---- a/cts/scheduler/scores/enforce-colo1.scores -+++ b/cts/scheduler/scores/enforce-colo1.scores -@@ -18,13 +18,22 @@ pcmk__primitive_assign: engine allocation score on rhel7-auto1: -INFINITY - pcmk__primitive_assign: engine allocation score on rhel7-auto2: -INFINITY - pcmk__primitive_assign: engine allocation score on rhel7-auto3: 0 - pcmk__primitive_assign: keystone:0 allocation score on rhel7-auto1: -INFINITY -+pcmk__primitive_assign: keystone:0 allocation score on rhel7-auto1: -INFINITY -+pcmk__primitive_assign: keystone:0 allocation score on rhel7-auto2: -INFINITY - pcmk__primitive_assign: keystone:0 allocation score on rhel7-auto2: -INFINITY - pcmk__primitive_assign: keystone:0 allocation score on rhel7-auto3: -INFINITY -+pcmk__primitive_assign: keystone:0 allocation score on rhel7-auto3: -INFINITY - pcmk__primitive_assign: keystone:1 allocation score on rhel7-auto1: -INFINITY -+pcmk__primitive_assign: keystone:1 allocation score on rhel7-auto1: -INFINITY -+pcmk__primitive_assign: keystone:1 allocation score on rhel7-auto2: -INFINITY - pcmk__primitive_assign: keystone:1 allocation score on rhel7-auto2: -INFINITY - pcmk__primitive_assign: keystone:1 allocation score on rhel7-auto3: -INFINITY -+pcmk__primitive_assign: keystone:1 allocation score on rhel7-auto3: -INFINITY -+pcmk__primitive_assign: keystone:2 allocation score on rhel7-auto1: -INFINITY - pcmk__primitive_assign: keystone:2 allocation score on rhel7-auto1: -INFINITY - pcmk__primitive_assign: keystone:2 allocation score on rhel7-auto2: -INFINITY -+pcmk__primitive_assign: keystone:2 allocation score on rhel7-auto2: -INFINITY -+pcmk__primitive_assign: keystone:2 allocation score on rhel7-auto3: -INFINITY - pcmk__primitive_assign: keystone:2 allocation score on rhel7-auto3: -INFINITY - pcmk__primitive_assign: shooter allocation score on rhel7-auto1: 0 - pcmk__primitive_assign: shooter allocation score on rhel7-auto2: 0 -diff --git a/cts/scheduler/scores/promoted-asymmetrical-order.scores b/cts/scheduler/scores/promoted-asymmetrical-order.scores -index 382e0ebe285..18bc704551e 100644 ---- a/cts/scheduler/scores/promoted-asymmetrical-order.scores -+++ b/cts/scheduler/scores/promoted-asymmetrical-order.scores -@@ -12,8 +12,12 @@ pcmk__clone_assign: rsc2:0 allocation score on node2: 0 - pcmk__clone_assign: rsc2:1 allocation score on node1: 0 - pcmk__clone_assign: rsc2:1 allocation score on node2: 1 - pcmk__primitive_assign: rsc1:0 allocation score on node1: -INFINITY -+pcmk__primitive_assign: rsc1:0 allocation score on node1: -INFINITY -+pcmk__primitive_assign: rsc1:0 allocation score on node2: -INFINITY - pcmk__primitive_assign: rsc1:0 allocation score on node2: -INFINITY - pcmk__primitive_assign: rsc1:1 allocation score on node1: -INFINITY -+pcmk__primitive_assign: rsc1:1 allocation score on node1: -INFINITY -+pcmk__primitive_assign: rsc1:1 allocation score on node2: -INFINITY - pcmk__primitive_assign: rsc1:1 allocation score on node2: -INFINITY - pcmk__primitive_assign: rsc2:0 allocation score on node1: 1 - pcmk__primitive_assign: rsc2:0 allocation score on node2: 0 -diff --git a/cts/scheduler/scores/promoted-failed-demote-2.scores b/cts/scheduler/scores/promoted-failed-demote-2.scores -index 2a85ae6060e..e457d8c6057 100644 ---- a/cts/scheduler/scores/promoted-failed-demote-2.scores -+++ b/cts/scheduler/scores/promoted-failed-demote-2.scores -@@ -16,22 +16,32 @@ pcmk__clone_assign: stateful-2:0 allocation score on dl380g5b: -INFINITY - pcmk__clone_assign: stateful-2:1 allocation score on dl380g5a: INFINITY - pcmk__clone_assign: stateful-2:1 allocation score on dl380g5b: 0 - pcmk__group_assign: group:0 allocation score on dl380g5a: -INFINITY -+pcmk__group_assign: group:0 allocation score on dl380g5a: -INFINITY -+pcmk__group_assign: group:0 allocation score on dl380g5b: 0 - pcmk__group_assign: group:0 allocation score on dl380g5b: 0 - pcmk__group_assign: group:1 allocation score on dl380g5a: 0 - pcmk__group_assign: group:1 allocation score on dl380g5b: 0 - pcmk__group_assign: stateful-1:0 allocation score on dl380g5a: -INFINITY -+pcmk__group_assign: stateful-1:0 allocation score on dl380g5a: -INFINITY -+pcmk__group_assign: stateful-1:0 allocation score on dl380g5b: -INFINITY - pcmk__group_assign: stateful-1:0 allocation score on dl380g5b: -INFINITY - pcmk__group_assign: stateful-1:1 allocation score on dl380g5a: INFINITY - pcmk__group_assign: stateful-1:1 allocation score on dl380g5b: 0 - pcmk__group_assign: stateful-2:0 allocation score on dl380g5a: -INFINITY -+pcmk__group_assign: stateful-2:0 allocation score on dl380g5a: -INFINITY -+pcmk__group_assign: stateful-2:0 allocation score on dl380g5b: -INFINITY - pcmk__group_assign: stateful-2:0 allocation score on dl380g5b: -INFINITY - pcmk__group_assign: stateful-2:1 allocation score on dl380g5a: INFINITY - pcmk__group_assign: stateful-2:1 allocation score on dl380g5b: 0 - pcmk__primitive_assign: stateful-1:0 allocation score on dl380g5a: -INFINITY -+pcmk__primitive_assign: stateful-1:0 allocation score on dl380g5a: -INFINITY -+pcmk__primitive_assign: stateful-1:0 allocation score on dl380g5b: -INFINITY - pcmk__primitive_assign: stateful-1:0 allocation score on dl380g5b: -INFINITY - pcmk__primitive_assign: stateful-1:1 allocation score on dl380g5a: INFINITY - pcmk__primitive_assign: stateful-1:1 allocation score on dl380g5b: 0 - pcmk__primitive_assign: stateful-2:0 allocation score on dl380g5a: -INFINITY -+pcmk__primitive_assign: stateful-2:0 allocation score on dl380g5a: -INFINITY -+pcmk__primitive_assign: stateful-2:0 allocation score on dl380g5b: -INFINITY - pcmk__primitive_assign: stateful-2:0 allocation score on dl380g5b: -INFINITY - pcmk__primitive_assign: stateful-2:1 allocation score on dl380g5a: INFINITY - pcmk__primitive_assign: stateful-2:1 allocation score on dl380g5b: -INFINITY -diff --git a/cts/scheduler/scores/promoted-failed-demote.scores b/cts/scheduler/scores/promoted-failed-demote.scores -index 2a85ae6060e..e457d8c6057 100644 ---- a/cts/scheduler/scores/promoted-failed-demote.scores -+++ b/cts/scheduler/scores/promoted-failed-demote.scores -@@ -16,22 +16,32 @@ pcmk__clone_assign: stateful-2:0 allocation score on dl380g5b: -INFINITY - pcmk__clone_assign: stateful-2:1 allocation score on dl380g5a: INFINITY - pcmk__clone_assign: stateful-2:1 allocation score on dl380g5b: 0 - pcmk__group_assign: group:0 allocation score on dl380g5a: -INFINITY -+pcmk__group_assign: group:0 allocation score on dl380g5a: -INFINITY -+pcmk__group_assign: group:0 allocation score on dl380g5b: 0 - pcmk__group_assign: group:0 allocation score on dl380g5b: 0 - pcmk__group_assign: group:1 allocation score on dl380g5a: 0 - pcmk__group_assign: group:1 allocation score on dl380g5b: 0 - pcmk__group_assign: stateful-1:0 allocation score on dl380g5a: -INFINITY -+pcmk__group_assign: stateful-1:0 allocation score on dl380g5a: -INFINITY -+pcmk__group_assign: stateful-1:0 allocation score on dl380g5b: -INFINITY - pcmk__group_assign: stateful-1:0 allocation score on dl380g5b: -INFINITY - pcmk__group_assign: stateful-1:1 allocation score on dl380g5a: INFINITY - pcmk__group_assign: stateful-1:1 allocation score on dl380g5b: 0 - pcmk__group_assign: stateful-2:0 allocation score on dl380g5a: -INFINITY -+pcmk__group_assign: stateful-2:0 allocation score on dl380g5a: -INFINITY -+pcmk__group_assign: stateful-2:0 allocation score on dl380g5b: -INFINITY - pcmk__group_assign: stateful-2:0 allocation score on dl380g5b: -INFINITY - pcmk__group_assign: stateful-2:1 allocation score on dl380g5a: INFINITY - pcmk__group_assign: stateful-2:1 allocation score on dl380g5b: 0 - pcmk__primitive_assign: stateful-1:0 allocation score on dl380g5a: -INFINITY -+pcmk__primitive_assign: stateful-1:0 allocation score on dl380g5a: -INFINITY -+pcmk__primitive_assign: stateful-1:0 allocation score on dl380g5b: -INFINITY - pcmk__primitive_assign: stateful-1:0 allocation score on dl380g5b: -INFINITY - pcmk__primitive_assign: stateful-1:1 allocation score on dl380g5a: INFINITY - pcmk__primitive_assign: stateful-1:1 allocation score on dl380g5b: 0 - pcmk__primitive_assign: stateful-2:0 allocation score on dl380g5a: -INFINITY -+pcmk__primitive_assign: stateful-2:0 allocation score on dl380g5a: -INFINITY -+pcmk__primitive_assign: stateful-2:0 allocation score on dl380g5b: -INFINITY - pcmk__primitive_assign: stateful-2:0 allocation score on dl380g5b: -INFINITY - pcmk__primitive_assign: stateful-2:1 allocation score on dl380g5a: INFINITY - pcmk__primitive_assign: stateful-2:1 allocation score on dl380g5b: -INFINITY - -From 4abb93e5c779cf058861a25c5eac456ac1087fd6 Mon Sep 17 00:00:00 2001 -From: Reid Wahl -Date: Wed, 21 Jun 2023 22:40:20 -0700 -Subject: [PATCH 18/19] Fix: libpacemaker: Don't shuffle clone instances - unnecessarily - -Currently, clone instances may be shuffled under certain conditions, -causing an unnecessary resource downtime when an instance is moved -away from its current running node. - -For example, this can happen when a stopped promotable instance is -scheduled to promote and the stickiness is lower than the promotion -score (see the clone-recover-no-shuffle-7 test). Instance 0 gets -assigned first and goes to the node that will be promoted. If instance 0 -is already running on some node, it must stop there before it can start -on the new node. Another instance may start in its place after it stops. - -The fix is to assign an instance to its current node during the early -assignment phase, if that node is going to receive any instance at all. -If the node will receive an instance, it should receive its current -instance. - -The approach is described in detail in comments. - -Previously, if instance 0 was running on node1 and got assigned to node2 -during the early assignment phase (due to node2 having a higher score), -we backed out and immediately gave up on assigning instance 0 early. - -Now, we increment a "number of instances reserved" counter, as well as -the parent's counter of instances assigned to node2. We then try again -to assign instance 0 to node1. If node2 already has the max allowed -number of instances, then it will be marked unavailable for this round. - -Fixes T489 -Fixes RHBZ#1931023 - -Signed-off-by: Reid Wahl ---- - lib/pacemaker/pcmk_sched_instances.c | 163 ++++++++++++++++++++------- - 1 file changed, 122 insertions(+), 41 deletions(-) - -diff --git a/lib/pacemaker/pcmk_sched_instances.c b/lib/pacemaker/pcmk_sched_instances.c -index b551f3bee61..b010d460dbc 100644 ---- a/lib/pacemaker/pcmk_sched_instances.c -+++ b/lib/pacemaker/pcmk_sched_instances.c -@@ -605,50 +605,135 @@ assign_instance(pe_resource_t *instance, const pe_node_t *prefer, - instance->id); - return NULL; - } -+ ban_unavailable_allowed_nodes(instance, max_per_node); -+ -+ // Failed early assignments are reversible (stop_if_fail=false) -+ chosen = instance->cmds->assign(instance, prefer, (prefer == NULL)); -+ increment_parent_count(instance, chosen); -+ return chosen; -+} -+ -+/*! -+ * \internal -+ * \brief Try to assign an instance to its current node early -+ * -+ * \param[in] rsc Clone or bundle being assigned (for logs only) -+ * \param[in] instance Clone instance or bundle replica container -+ * \param[in] current Instance's current node -+ * \param[in] max_per_node Maximum number of instances per node -+ * \param[in] available Number of instances still available for assignment -+ * -+ * \return \c true if \p instance was successfully assigned to its current node, -+ * or \c false otherwise -+ */ -+static bool -+assign_instance_early(const pe_resource_t *rsc, pe_resource_t *instance, -+ const pe_node_t *current, int max_per_node, int available) -+{ -+ const pe_node_t *chosen = NULL; -+ int reserved = 0; - -- if (prefer != NULL) { // Possible early assignment to preferred node -+ pe_resource_t *parent = instance->parent; -+ GHashTable *allowed_orig = NULL; -+ GHashTable *allowed_orig_parent = parent->allowed_nodes; - -- // Get preferred node with instance's scores -- pe_node_t *allowed = g_hash_table_lookup(instance->allowed_nodes, -- prefer->details->id); -+ const pe_node_t *allowed_node = g_hash_table_lookup(instance->allowed_nodes, -+ current->details->id); - -- if ((allowed == NULL) || (allowed->weight < 0)) { -- pe_rsc_trace(instance, -- "Not assigning %s to preferred node %s: unavailable", -- instance->id, pe__node_name(prefer)); -- return NULL; -- } -+ pe_rsc_trace(instance, "Trying to assign %s to its current node %s", -+ instance->id, pe__node_name(current)); -+ -+ if (!pcmk__node_available(allowed_node, true, false)) { -+ pe_rsc_info(instance, -+ "Not assigning %s to current node %s: unavailable", -+ instance->id, pe__node_name(current)); -+ return false; - } - -- ban_unavailable_allowed_nodes(instance, max_per_node); -+ /* On each iteration, if instance gets assigned to a node other than its -+ * current one, we reserve one instance for the chosen node, unassign -+ * instance, restore instance's original node tables, and try again. This -+ * way, instances are proportionally assigned to nodes based on preferences, -+ * but shuffling of specific instances is minimized. If a node will be -+ * assigned instances at all, it preferentially receives instances that are -+ * currently active there. -+ * -+ * parent->allowed_nodes tracks the number of instances assigned to each -+ * node. If a node already has max_per_node instances assigned, -+ * ban_unavailable_allowed_nodes() marks it as unavailable. -+ * -+ * In the end, we restore the original parent->allowed_nodes to undo the -+ * changes to counts during tentative assignments. If we successfully -+ * assigned instance to its current node, we increment that node's counter. -+ */ - -- if (prefer == NULL) { // Final assignment -- chosen = instance->cmds->assign(instance, NULL, true); -+ // Back up the allowed node tables of instance and its children recursively -+ pcmk__copy_node_tables(instance, &allowed_orig); - -- } else { // Possible early assignment to preferred node -- GHashTable *backup = NULL; -+ // Update instances-per-node counts in a scratch table -+ parent->allowed_nodes = pcmk__copy_node_table(parent->allowed_nodes); - -- pcmk__copy_node_tables(instance, &backup); -- chosen = instance->cmds->assign(instance, prefer, false); -+ while (reserved < available) { -+ chosen = assign_instance(instance, current, max_per_node); - -- if (!pe__same_node(chosen, prefer)) { -- // Revert nodes if preferred node won't be assigned -- if (chosen != NULL) { -- pe_rsc_info(instance, -- "Not assigning %s to preferred node %s: " -- "%s is better", -- instance->id, pe__node_name(prefer), -- pe__node_name(chosen)); -- chosen = NULL; -- } -- pcmk__restore_node_tables(instance, backup); -- pcmk__unassign_resource(instance); -+ if (pe__same_node(chosen, current)) { -+ // Successfully assigned to current node -+ break; -+ } -+ -+ // Assignment updates scores, so restore to original state -+ pe_rsc_debug(instance, "Rolling back node scores for %s", instance->id); -+ pcmk__restore_node_tables(instance, allowed_orig); -+ -+ if (chosen == NULL) { -+ // Assignment failed, so give up -+ pe_rsc_info(instance, -+ "Not assigning %s to current node %s: unavailable", -+ instance->id, pe__node_name(current)); -+ pe__set_resource_flags(instance, pe_rsc_provisional); -+ break; -+ } -+ -+ // We prefer more strongly to assign an instance to the chosen node -+ pe_rsc_debug(instance, -+ "Not assigning %s to current node %s: %s is better", -+ instance->id, pe__node_name(current), -+ pe__node_name(chosen)); -+ -+ // Reserve one instance for the chosen node and try again -+ if (++reserved >= available) { -+ pe_rsc_info(instance, -+ "Not assigning %s to current node %s: " -+ "other assignments are more important", -+ instance->id, pe__node_name(current)); -+ -+ } else { -+ pe_rsc_debug(instance, -+ "Reserved an instance of %s for %s. Retrying " -+ "assignment of %s to %s", -+ rsc->id, pe__node_name(chosen), instance->id, -+ pe__node_name(current)); - } -- g_hash_table_destroy(backup); -+ -+ // Clear this assignment (frees chosen); leave instance counts in parent -+ pcmk__unassign_resource(instance); -+ chosen = NULL; - } - -+ g_hash_table_destroy(allowed_orig); -+ -+ // Restore original instances-per-node counts -+ g_hash_table_destroy(parent->allowed_nodes); -+ parent->allowed_nodes = allowed_orig_parent; -+ -+ if (chosen == NULL) { -+ // Couldn't assign instance to current node -+ return false; -+ } -+ pe_rsc_trace(instance, "Assigned %s to current node %s", -+ instance->id, pe__node_name(current)); - increment_parent_count(instance, chosen); -- return chosen; -+ return true; - } - - /*! -@@ -760,22 +845,18 @@ pcmk__assign_instances(pe_resource_t *collective, GList *instances, - // Assign as many instances as possible to their current location - for (iter = instances; (iter != NULL) && (assigned < max_total); - iter = iter->next) { -- instance = (pe_resource_t *) iter->data; -+ int available = max_total - assigned; - -+ instance = iter->data; - if (!pcmk_is_set(instance->flags, pe_rsc_provisional)) { - continue; // Already assigned - } - - current = preferred_node(collective, instance, optimal_per_node); -- if (current != NULL) { -- const pe_node_t *chosen = assign_instance(instance, current, -- max_per_node); -- -- if (pe__same_node(chosen, current)) { -- pe_rsc_trace(collective, "Assigned %s to current node %s", -- instance->id, pe__node_name(current)); -- assigned++; -- } -+ if ((current != NULL) -+ && assign_instance_early(collective, instance, current, -+ max_per_node, available)) { -+ assigned++; - } - } - - -From 59e9950212506a9034db8e90a17033734a1d18a1 Mon Sep 17 00:00:00 2001 -From: Reid Wahl -Date: Mon, 10 Jul 2023 02:50:28 -0700 -Subject: [PATCH 19/19] Test: scheduler: Update test outputs after clone - instance shuffling fix - -The following tests are now correct: -* clone-recover-no-shuffle-4 -* clone-recover-no-shuffle-5 -* clone-recover-no-shuffle-6 -* clone-recover-no-shuffle-7 - -Scores for several other tests are changed in ways (usually duplicates -from additional tentative assignments) that don't impact the resulting -transition. - -One test (cancel-behind-moving-remote) technically breaks. Previously, -due to shuffling, ovn-dbs-bundle-1 moved to controller-0. Since -ovndb_servers:1 gets promoted on ovn-dbs-bundle-1, controller-0 held the -promoted instance of ovn-dbs-bundle. - -Now, since instances correctly prefer their current nodes, -ovn-dbs-bundle-1 remains on controller-2. However, ovndb_servers:1 still -gets promoted on ovn-dbs-bundle-1, so controller-2 holds the promoted -instance of ovn-dbs-bundle. - -ip-172.17.1.87 is colocated with ovn-dbs-bundle's promoted role and is -banned from controller-2. As a result, ip-172.17.1.87 is now stopped. - -This test is believed to have worked properly in the past due only to -luck. At this point (see T672 and the bundle-promoted-*colocation-* -tests), it's well-established that colocations involving promotable -bundles don't work correctly. - -Ref T489 -Ref RHBZ#1931023 - -Signed-off-by: Reid Wahl ---- - .../dot/cancel-behind-moving-remote.dot | 99 +-- - .../dot/clone-recover-no-shuffle-4.dot | 23 +- - .../dot/clone-recover-no-shuffle-5.dot | 57 +- - .../dot/clone-recover-no-shuffle-6.dot | 99 +-- - .../dot/clone-recover-no-shuffle-7.dot | 35 +- - .../exp/cancel-behind-moving-remote.exp | 724 +++++------------- - .../exp/clone-recover-no-shuffle-4.exp | 98 +-- - .../exp/clone-recover-no-shuffle-5.exp | 239 +----- - .../exp/clone-recover-no-shuffle-6.exp | 434 ++--------- - .../exp/clone-recover-no-shuffle-7.exp | 174 ++--- - cts/scheduler/scores/bug-cl-5168.scores | 2 +- - .../scores/cancel-behind-moving-remote.scores | 27 +- - .../scores/clone-recover-no-shuffle-10.scores | 2 +- - .../scores/clone-recover-no-shuffle-4.scores | 10 +- - .../scores/clone-recover-no-shuffle-5.scores | 48 +- - .../scores/clone-recover-no-shuffle-6.scores | 22 +- - .../scores/clone-recover-no-shuffle-7.scores | 14 +- - .../scores/promoted-failed-demote-2.scores | 4 - - .../scores/promoted-failed-demote.scores | 4 - - .../scores/utilization-complex.scores | 24 + - .../scores/utilization-order2.scores | 2 + - .../cancel-behind-moving-remote.summary | 61 +- - .../clone-recover-no-shuffle-4.summary | 8 +- - .../clone-recover-no-shuffle-5.summary | 22 +- - .../clone-recover-no-shuffle-6.summary | 48 +- - .../clone-recover-no-shuffle-7.summary | 12 +- - .../xml/cancel-behind-moving-remote.xml | 14 + - .../xml/clone-recover-no-shuffle-4.xml | 5 - - .../xml/clone-recover-no-shuffle-5.xml | 5 - - .../xml/clone-recover-no-shuffle-6.xml | 5 - - .../xml/clone-recover-no-shuffle-7.xml | 5 - - 31 files changed, 526 insertions(+), 1800 deletions(-) - -diff --git a/cts/scheduler/dot/cancel-behind-moving-remote.dot b/cts/scheduler/dot/cancel-behind-moving-remote.dot -index 1a0dfc8c889..de803a7e299 100644 ---- a/cts/scheduler/dot/cancel-behind-moving-remote.dot -+++ b/cts/scheduler/dot/cancel-behind-moving-remote.dot -@@ -1,28 +1,12 @@ - digraph "g" { - "Cancel ovndb_servers_monitor_30000 ovn-dbs-bundle-1" -> "ovndb_servers_promote_0 ovn-dbs-bundle-1" [ style = bold] - "Cancel ovndb_servers_monitor_30000 ovn-dbs-bundle-1" [ style=bold color="green" fontcolor="black"] --"ip-172.17.1.87_monitor_10000 controller-0" [ style=bold color="green" fontcolor="black"] --"ip-172.17.1.87_start_0 controller-0" -> "ip-172.17.1.87_monitor_10000 controller-0" [ style = bold] --"ip-172.17.1.87_start_0 controller-0" [ style=bold color="green" fontcolor="black"] - "nova-evacuate_clear_failcount_0 messaging-0" [ style=bold color="green" fontcolor="black"] --"ovn-dbs-bundle-0_clear_failcount_0 controller-0" -> "ovn-dbs-bundle-0_start_0 controller-2" [ style = bold] --"ovn-dbs-bundle-0_clear_failcount_0 controller-0" [ style=bold color="green" fontcolor="black"] --"ovn-dbs-bundle-0_monitor_30000 controller-2" [ style=bold color="green" fontcolor="black"] --"ovn-dbs-bundle-0_start_0 controller-2" -> "ovn-dbs-bundle-0_monitor_30000 controller-2" [ style = bold] --"ovn-dbs-bundle-0_start_0 controller-2" -> "ovndb_servers:0_monitor_30000 ovn-dbs-bundle-0" [ style = bold] --"ovn-dbs-bundle-0_start_0 controller-2" -> "ovndb_servers:0_start_0 ovn-dbs-bundle-0" [ style = bold] --"ovn-dbs-bundle-0_start_0 controller-2" [ style=bold color="green" fontcolor="black"] --"ovn-dbs-bundle-1_clear_failcount_0 controller-2" -> "ovn-dbs-bundle-1_start_0 controller-0" [ style = bold] --"ovn-dbs-bundle-1_clear_failcount_0 controller-2" [ style=bold color="green" fontcolor="black"] --"ovn-dbs-bundle-1_monitor_30000 controller-0" [ style=bold color="green" fontcolor="black"] --"ovn-dbs-bundle-1_start_0 controller-0" -> "ovn-dbs-bundle-1_monitor_30000 controller-0" [ style = bold] --"ovn-dbs-bundle-1_start_0 controller-0" -> "ovndb_servers_monitor_10000 ovn-dbs-bundle-1" [ style = bold] --"ovn-dbs-bundle-1_start_0 controller-0" -> "ovndb_servers_promote_0 ovn-dbs-bundle-1" [ style = bold] --"ovn-dbs-bundle-1_start_0 controller-0" -> "ovndb_servers_start_0 ovn-dbs-bundle-1" [ style = bold] --"ovn-dbs-bundle-1_start_0 controller-0" [ style=bold color="green" fontcolor="black"] --"ovn-dbs-bundle-1_stop_0 controller-2" -> "ovn-dbs-bundle-1_start_0 controller-0" [ style = bold] --"ovn-dbs-bundle-1_stop_0 controller-2" -> "ovn-dbs-bundle-podman-1_stop_0 controller-2" [ style = bold] --"ovn-dbs-bundle-1_stop_0 controller-2" [ style=bold color="green" fontcolor="black"] -+"ovn-dbs-bundle-0_monitor_30000 controller-0" [ style=bold color="green" fontcolor="black"] -+"ovn-dbs-bundle-0_start_0 controller-0" -> "ovn-dbs-bundle-0_monitor_30000 controller-0" [ style = bold] -+"ovn-dbs-bundle-0_start_0 controller-0" -> "ovndb_servers:0_monitor_30000 ovn-dbs-bundle-0" [ style = bold] -+"ovn-dbs-bundle-0_start_0 controller-0" -> "ovndb_servers:0_start_0 ovn-dbs-bundle-0" [ style = bold] -+"ovn-dbs-bundle-0_start_0 controller-0" [ style=bold color="green" fontcolor="black"] - "ovn-dbs-bundle-master_confirmed-post_notify_promoted_0" -> "ovn-dbs-bundle_promoted_0" [ style = bold] - "ovn-dbs-bundle-master_confirmed-post_notify_promoted_0" -> "ovndb_servers:0_monitor_30000 ovn-dbs-bundle-0" [ style = bold] - "ovn-dbs-bundle-master_confirmed-post_notify_promoted_0" -> "ovndb_servers_monitor_10000 ovn-dbs-bundle-1" [ style = bold] -@@ -32,19 +16,12 @@ - "ovn-dbs-bundle-master_confirmed-post_notify_running_0" -> "ovndb_servers:0_monitor_30000 ovn-dbs-bundle-0" [ style = bold] - "ovn-dbs-bundle-master_confirmed-post_notify_running_0" -> "ovndb_servers_monitor_10000 ovn-dbs-bundle-1" [ style = bold] - "ovn-dbs-bundle-master_confirmed-post_notify_running_0" [ style=bold color="green" fontcolor="orange"] --"ovn-dbs-bundle-master_confirmed-post_notify_stopped_0" -> "ovn-dbs-bundle-master_pre_notify_promote_0" [ style = bold] --"ovn-dbs-bundle-master_confirmed-post_notify_stopped_0" -> "ovn-dbs-bundle-master_pre_notify_start_0" [ style = bold] --"ovn-dbs-bundle-master_confirmed-post_notify_stopped_0" -> "ovn-dbs-bundle_stopped_0" [ style = bold] --"ovn-dbs-bundle-master_confirmed-post_notify_stopped_0" [ style=bold color="green" fontcolor="orange"] - "ovn-dbs-bundle-master_confirmed-pre_notify_promote_0" -> "ovn-dbs-bundle-master_post_notify_promoted_0" [ style = bold] - "ovn-dbs-bundle-master_confirmed-pre_notify_promote_0" -> "ovn-dbs-bundle-master_promote_0" [ style = bold] - "ovn-dbs-bundle-master_confirmed-pre_notify_promote_0" [ style=bold color="green" fontcolor="orange"] - "ovn-dbs-bundle-master_confirmed-pre_notify_start_0" -> "ovn-dbs-bundle-master_post_notify_running_0" [ style = bold] - "ovn-dbs-bundle-master_confirmed-pre_notify_start_0" -> "ovn-dbs-bundle-master_start_0" [ style = bold] - "ovn-dbs-bundle-master_confirmed-pre_notify_start_0" [ style=bold color="green" fontcolor="orange"] --"ovn-dbs-bundle-master_confirmed-pre_notify_stop_0" -> "ovn-dbs-bundle-master_post_notify_stopped_0" [ style = bold] --"ovn-dbs-bundle-master_confirmed-pre_notify_stop_0" -> "ovn-dbs-bundle-master_stop_0" [ style = bold] --"ovn-dbs-bundle-master_confirmed-pre_notify_stop_0" [ style=bold color="green" fontcolor="orange"] - "ovn-dbs-bundle-master_post_notify_promoted_0" -> "ovn-dbs-bundle-master_confirmed-post_notify_promoted_0" [ style = bold] - "ovn-dbs-bundle-master_post_notify_promoted_0" -> "ovndb_servers:0_post_notify_promote_0 ovn-dbs-bundle-0" [ style = bold] - "ovn-dbs-bundle-master_post_notify_promoted_0" -> "ovndb_servers_post_notify_promoted_0 ovn-dbs-bundle-1" [ style = bold] -@@ -55,21 +32,15 @@ - "ovn-dbs-bundle-master_post_notify_running_0" -> "ovndb_servers_post_notify_running_0 ovn-dbs-bundle-1" [ style = bold] - "ovn-dbs-bundle-master_post_notify_running_0" -> "ovndb_servers_post_notify_running_0 ovn-dbs-bundle-2" [ style = bold] - "ovn-dbs-bundle-master_post_notify_running_0" [ style=bold color="green" fontcolor="orange"] --"ovn-dbs-bundle-master_post_notify_stopped_0" -> "ovn-dbs-bundle-master_confirmed-post_notify_stopped_0" [ style = bold] --"ovn-dbs-bundle-master_post_notify_stopped_0" -> "ovndb_servers_post_notify_stopped_0 ovn-dbs-bundle-2" [ style = bold] --"ovn-dbs-bundle-master_post_notify_stopped_0" [ style=bold color="green" fontcolor="orange"] - "ovn-dbs-bundle-master_pre_notify_promote_0" -> "ovn-dbs-bundle-master_confirmed-pre_notify_promote_0" [ style = bold] - "ovn-dbs-bundle-master_pre_notify_promote_0" -> "ovndb_servers:0_pre_notify_promote_0 ovn-dbs-bundle-0" [ style = bold] - "ovn-dbs-bundle-master_pre_notify_promote_0" -> "ovndb_servers_pre_notify_promote_0 ovn-dbs-bundle-1" [ style = bold] - "ovn-dbs-bundle-master_pre_notify_promote_0" -> "ovndb_servers_pre_notify_promote_0 ovn-dbs-bundle-2" [ style = bold] - "ovn-dbs-bundle-master_pre_notify_promote_0" [ style=bold color="green" fontcolor="orange"] - "ovn-dbs-bundle-master_pre_notify_start_0" -> "ovn-dbs-bundle-master_confirmed-pre_notify_start_0" [ style = bold] -+"ovn-dbs-bundle-master_pre_notify_start_0" -> "ovndb_servers_pre_notify_start_0 ovn-dbs-bundle-1" [ style = bold] - "ovn-dbs-bundle-master_pre_notify_start_0" -> "ovndb_servers_pre_notify_start_0 ovn-dbs-bundle-2" [ style = bold] - "ovn-dbs-bundle-master_pre_notify_start_0" [ style=bold color="green" fontcolor="orange"] --"ovn-dbs-bundle-master_pre_notify_stop_0" -> "ovn-dbs-bundle-master_confirmed-pre_notify_stop_0" [ style = bold] --"ovn-dbs-bundle-master_pre_notify_stop_0" -> "ovndb_servers_pre_notify_stop_0 ovn-dbs-bundle-1" [ style = bold] --"ovn-dbs-bundle-master_pre_notify_stop_0" -> "ovndb_servers_pre_notify_stop_0 ovn-dbs-bundle-2" [ style = bold] --"ovn-dbs-bundle-master_pre_notify_stop_0" [ style=bold color="green" fontcolor="orange"] - "ovn-dbs-bundle-master_promote_0" -> "ovndb_servers_promote_0 ovn-dbs-bundle-1" [ style = bold] - "ovn-dbs-bundle-master_promote_0" [ style=bold color="green" fontcolor="orange"] - "ovn-dbs-bundle-master_promoted_0" -> "ovn-dbs-bundle-master_post_notify_promoted_0" [ style = bold] -@@ -79,48 +50,21 @@ - "ovn-dbs-bundle-master_running_0" [ style=bold color="green" fontcolor="orange"] - "ovn-dbs-bundle-master_start_0" -> "ovn-dbs-bundle-master_running_0" [ style = bold] - "ovn-dbs-bundle-master_start_0" -> "ovndb_servers:0_start_0 ovn-dbs-bundle-0" [ style = bold] --"ovn-dbs-bundle-master_start_0" -> "ovndb_servers_start_0 ovn-dbs-bundle-1" [ style = bold] - "ovn-dbs-bundle-master_start_0" [ style=bold color="green" fontcolor="orange"] --"ovn-dbs-bundle-master_stop_0" -> "ovn-dbs-bundle-master_stopped_0" [ style = bold] --"ovn-dbs-bundle-master_stop_0" -> "ovndb_servers_stop_0 ovn-dbs-bundle-1" [ style = bold] --"ovn-dbs-bundle-master_stop_0" [ style=bold color="green" fontcolor="orange"] --"ovn-dbs-bundle-master_stopped_0" -> "ovn-dbs-bundle-master_post_notify_stopped_0" [ style = bold] --"ovn-dbs-bundle-master_stopped_0" -> "ovn-dbs-bundle-master_promote_0" [ style = bold] --"ovn-dbs-bundle-master_stopped_0" -> "ovn-dbs-bundle-master_start_0" [ style = bold] --"ovn-dbs-bundle-master_stopped_0" [ style=bold color="green" fontcolor="orange"] --"ovn-dbs-bundle-podman-0_monitor_60000 controller-2" [ style=bold color="green" fontcolor="black"] --"ovn-dbs-bundle-podman-0_start_0 controller-2" -> "ovn-dbs-bundle-0_start_0 controller-2" [ style = bold] --"ovn-dbs-bundle-podman-0_start_0 controller-2" -> "ovn-dbs-bundle-podman-0_monitor_60000 controller-2" [ style = bold] --"ovn-dbs-bundle-podman-0_start_0 controller-2" -> "ovn-dbs-bundle_running_0" [ style = bold] --"ovn-dbs-bundle-podman-0_start_0 controller-2" -> "ovndb_servers:0_start_0 ovn-dbs-bundle-0" [ style = bold] --"ovn-dbs-bundle-podman-0_start_0 controller-2" [ style=bold color="green" fontcolor="black"] --"ovn-dbs-bundle-podman-1_monitor_60000 controller-0" [ style=bold color="green" fontcolor="black"] --"ovn-dbs-bundle-podman-1_start_0 controller-0" -> "ovn-dbs-bundle-1_start_0 controller-0" [ style = bold] --"ovn-dbs-bundle-podman-1_start_0 controller-0" -> "ovn-dbs-bundle-podman-1_monitor_60000 controller-0" [ style = bold] --"ovn-dbs-bundle-podman-1_start_0 controller-0" -> "ovn-dbs-bundle_running_0" [ style = bold] --"ovn-dbs-bundle-podman-1_start_0 controller-0" -> "ovndb_servers_promote_0 ovn-dbs-bundle-1" [ style = bold] --"ovn-dbs-bundle-podman-1_start_0 controller-0" -> "ovndb_servers_start_0 ovn-dbs-bundle-1" [ style = bold] --"ovn-dbs-bundle-podman-1_start_0 controller-0" [ style=bold color="green" fontcolor="black"] --"ovn-dbs-bundle-podman-1_stop_0 controller-2" -> "ovn-dbs-bundle-podman-1_start_0 controller-0" [ style = bold] --"ovn-dbs-bundle-podman-1_stop_0 controller-2" -> "ovn-dbs-bundle_stopped_0" [ style = bold] --"ovn-dbs-bundle-podman-1_stop_0 controller-2" [ style=bold color="green" fontcolor="black"] --"ovn-dbs-bundle_promote_0" -> "ip-172.17.1.87_start_0 controller-0" [ style = bold] -+"ovn-dbs-bundle-podman-0_monitor_60000 controller-0" [ style=bold color="green" fontcolor="black"] -+"ovn-dbs-bundle-podman-0_start_0 controller-0" -> "ovn-dbs-bundle-0_start_0 controller-0" [ style = bold] -+"ovn-dbs-bundle-podman-0_start_0 controller-0" -> "ovn-dbs-bundle-podman-0_monitor_60000 controller-0" [ style = bold] -+"ovn-dbs-bundle-podman-0_start_0 controller-0" -> "ovn-dbs-bundle_running_0" [ style = bold] -+"ovn-dbs-bundle-podman-0_start_0 controller-0" -> "ovndb_servers:0_start_0 ovn-dbs-bundle-0" [ style = bold] -+"ovn-dbs-bundle-podman-0_start_0 controller-0" [ style=bold color="green" fontcolor="black"] - "ovn-dbs-bundle_promote_0" -> "ovn-dbs-bundle-master_promote_0" [ style = bold] - "ovn-dbs-bundle_promote_0" [ style=bold color="green" fontcolor="orange"] - "ovn-dbs-bundle_promoted_0" [ style=bold color="green" fontcolor="orange"] - "ovn-dbs-bundle_running_0" -> "ovn-dbs-bundle_promote_0" [ style = bold] - "ovn-dbs-bundle_running_0" [ style=bold color="green" fontcolor="orange"] - "ovn-dbs-bundle_start_0" -> "ovn-dbs-bundle-master_start_0" [ style = bold] --"ovn-dbs-bundle_start_0" -> "ovn-dbs-bundle-podman-0_start_0 controller-2" [ style = bold] --"ovn-dbs-bundle_start_0" -> "ovn-dbs-bundle-podman-1_start_0 controller-0" [ style = bold] -+"ovn-dbs-bundle_start_0" -> "ovn-dbs-bundle-podman-0_start_0 controller-0" [ style = bold] - "ovn-dbs-bundle_start_0" [ style=bold color="green" fontcolor="orange"] --"ovn-dbs-bundle_stop_0" -> "ovn-dbs-bundle-master_stop_0" [ style = bold] --"ovn-dbs-bundle_stop_0" -> "ovn-dbs-bundle-podman-1_stop_0 controller-2" [ style = bold] --"ovn-dbs-bundle_stop_0" -> "ovndb_servers_stop_0 ovn-dbs-bundle-1" [ style = bold] --"ovn-dbs-bundle_stop_0" [ style=bold color="green" fontcolor="orange"] --"ovn-dbs-bundle_stopped_0" -> "ovn-dbs-bundle_promote_0" [ style = bold] --"ovn-dbs-bundle_stopped_0" -> "ovn-dbs-bundle_start_0" [ style = bold] --"ovn-dbs-bundle_stopped_0" [ style=bold color="green" fontcolor="orange"] - "ovndb_servers:0_monitor_30000 ovn-dbs-bundle-0" [ style=bold color="green" fontcolor="black"] - "ovndb_servers:0_post_notify_promote_0 ovn-dbs-bundle-0" -> "ovn-dbs-bundle-master_confirmed-post_notify_promoted_0" [ style = bold] - "ovndb_servers:0_post_notify_promote_0 ovn-dbs-bundle-0" [ style=bold color="green" fontcolor="black"] -@@ -130,7 +74,6 @@ - "ovndb_servers:0_pre_notify_promote_0 ovn-dbs-bundle-0" [ style=bold color="green" fontcolor="black"] - "ovndb_servers:0_start_0 ovn-dbs-bundle-0" -> "ovn-dbs-bundle-master_running_0" [ style = bold] - "ovndb_servers:0_start_0 ovn-dbs-bundle-0" -> "ovndb_servers:0_monitor_30000 ovn-dbs-bundle-0" [ style = bold] --"ovndb_servers:0_start_0 ovn-dbs-bundle-0" -> "ovndb_servers_start_0 ovn-dbs-bundle-1" [ style = bold] - "ovndb_servers:0_start_0 ovn-dbs-bundle-0" [ style=bold color="green" fontcolor="black"] - "ovndb_servers_monitor_10000 ovn-dbs-bundle-1" [ style=bold color="green" fontcolor="black"] - "ovndb_servers_post_notify_promoted_0 ovn-dbs-bundle-1" -> "ovn-dbs-bundle-master_confirmed-post_notify_promoted_0" [ style = bold] -@@ -141,29 +84,17 @@ - "ovndb_servers_post_notify_running_0 ovn-dbs-bundle-1" [ style=bold color="green" fontcolor="black"] - "ovndb_servers_post_notify_running_0 ovn-dbs-bundle-2" -> "ovn-dbs-bundle-master_confirmed-post_notify_running_0" [ style = bold] - "ovndb_servers_post_notify_running_0 ovn-dbs-bundle-2" [ style=bold color="green" fontcolor="black"] --"ovndb_servers_post_notify_stopped_0 ovn-dbs-bundle-2" -> "ovn-dbs-bundle-master_confirmed-post_notify_stopped_0" [ style = bold] --"ovndb_servers_post_notify_stopped_0 ovn-dbs-bundle-2" [ style=bold color="green" fontcolor="black"] - "ovndb_servers_pre_notify_promote_0 ovn-dbs-bundle-1" -> "ovn-dbs-bundle-master_confirmed-pre_notify_promote_0" [ style = bold] - "ovndb_servers_pre_notify_promote_0 ovn-dbs-bundle-1" [ style=bold color="green" fontcolor="black"] - "ovndb_servers_pre_notify_promote_0 ovn-dbs-bundle-2" -> "ovn-dbs-bundle-master_confirmed-pre_notify_promote_0" [ style = bold] - "ovndb_servers_pre_notify_promote_0 ovn-dbs-bundle-2" [ style=bold color="green" fontcolor="black"] -+"ovndb_servers_pre_notify_start_0 ovn-dbs-bundle-1" -> "ovn-dbs-bundle-master_confirmed-pre_notify_start_0" [ style = bold] -+"ovndb_servers_pre_notify_start_0 ovn-dbs-bundle-1" [ style=bold color="green" fontcolor="black"] - "ovndb_servers_pre_notify_start_0 ovn-dbs-bundle-2" -> "ovn-dbs-bundle-master_confirmed-pre_notify_start_0" [ style = bold] - "ovndb_servers_pre_notify_start_0 ovn-dbs-bundle-2" [ style=bold color="green" fontcolor="black"] --"ovndb_servers_pre_notify_stop_0 ovn-dbs-bundle-1" -> "ovn-dbs-bundle-master_confirmed-pre_notify_stop_0" [ style = bold] --"ovndb_servers_pre_notify_stop_0 ovn-dbs-bundle-1" [ style=bold color="green" fontcolor="black"] --"ovndb_servers_pre_notify_stop_0 ovn-dbs-bundle-2" -> "ovn-dbs-bundle-master_confirmed-pre_notify_stop_0" [ style = bold] --"ovndb_servers_pre_notify_stop_0 ovn-dbs-bundle-2" [ style=bold color="green" fontcolor="black"] - "ovndb_servers_promote_0 ovn-dbs-bundle-1" -> "ovn-dbs-bundle-master_promoted_0" [ style = bold] - "ovndb_servers_promote_0 ovn-dbs-bundle-1" -> "ovndb_servers_monitor_10000 ovn-dbs-bundle-1" [ style = bold] - "ovndb_servers_promote_0 ovn-dbs-bundle-1" [ style=bold color="green" fontcolor="black"] --"ovndb_servers_start_0 ovn-dbs-bundle-1" -> "ovn-dbs-bundle-master_running_0" [ style = bold] --"ovndb_servers_start_0 ovn-dbs-bundle-1" -> "ovndb_servers_monitor_10000 ovn-dbs-bundle-1" [ style = bold] --"ovndb_servers_start_0 ovn-dbs-bundle-1" -> "ovndb_servers_promote_0 ovn-dbs-bundle-1" [ style = bold] --"ovndb_servers_start_0 ovn-dbs-bundle-1" [ style=bold color="green" fontcolor="black"] --"ovndb_servers_stop_0 ovn-dbs-bundle-1" -> "ovn-dbs-bundle-1_stop_0 controller-2" [ style = bold] --"ovndb_servers_stop_0 ovn-dbs-bundle-1" -> "ovn-dbs-bundle-master_stopped_0" [ style = bold] --"ovndb_servers_stop_0 ovn-dbs-bundle-1" -> "ovndb_servers_start_0 ovn-dbs-bundle-1" [ style = bold] --"ovndb_servers_stop_0 ovn-dbs-bundle-1" [ style=bold color="green" fontcolor="black"] - "rabbitmq-bundle-1_monitor_30000 controller-0" [ style=dashed color="red" fontcolor="black"] - "rabbitmq-bundle-1_start_0 controller-0" -> "rabbitmq-bundle-1_monitor_30000 controller-0" [ style = dashed] - "rabbitmq-bundle-1_start_0 controller-0" -> "rabbitmq:1_monitor_10000 rabbitmq-bundle-1" [ style = dashed] -diff --git a/cts/scheduler/dot/clone-recover-no-shuffle-4.dot b/cts/scheduler/dot/clone-recover-no-shuffle-4.dot -index fd002f28fcf..287d82d3806 100644 ---- a/cts/scheduler/dot/clone-recover-no-shuffle-4.dot -+++ b/cts/scheduler/dot/clone-recover-no-shuffle-4.dot -@@ -1,23 +1,10 @@ - digraph "g" { - "dummy-clone_running_0" [ style=bold color="green" fontcolor="orange"] - "dummy-clone_start_0" -> "dummy-clone_running_0" [ style = bold] --"dummy-clone_start_0" -> "dummy:2_start_0 node2" [ style = bold] --"dummy-clone_start_0" -> "dummy_start_0 node1" [ style = bold] -+"dummy-clone_start_0" -> "dummy:2_start_0 node1" [ style = bold] - "dummy-clone_start_0" [ style=bold color="green" fontcolor="orange"] --"dummy-clone_stop_0" -> "dummy-clone_stopped_0" [ style = bold] --"dummy-clone_stop_0" -> "dummy_stop_0 node2" [ style = bold] --"dummy-clone_stop_0" [ style=bold color="green" fontcolor="orange"] --"dummy-clone_stopped_0" -> "dummy-clone_start_0" [ style = bold] --"dummy-clone_stopped_0" [ style=bold color="green" fontcolor="orange"] --"dummy:2_monitor_10000 node2" [ style=bold color="green" fontcolor="black"] --"dummy:2_start_0 node2" -> "dummy-clone_running_0" [ style = bold] --"dummy:2_start_0 node2" -> "dummy:2_monitor_10000 node2" [ style = bold] --"dummy:2_start_0 node2" [ style=bold color="green" fontcolor="black"] --"dummy_monitor_10000 node1" [ style=bold color="green" fontcolor="black"] --"dummy_start_0 node1" -> "dummy-clone_running_0" [ style = bold] --"dummy_start_0 node1" -> "dummy_monitor_10000 node1" [ style = bold] --"dummy_start_0 node1" [ style=bold color="green" fontcolor="black"] --"dummy_stop_0 node2" -> "dummy-clone_stopped_0" [ style = bold] --"dummy_stop_0 node2" -> "dummy_start_0 node1" [ style = bold] --"dummy_stop_0 node2" [ style=bold color="green" fontcolor="black"] -+"dummy:2_monitor_10000 node1" [ style=bold color="green" fontcolor="black"] -+"dummy:2_start_0 node1" -> "dummy-clone_running_0" [ style = bold] -+"dummy:2_start_0 node1" -> "dummy:2_monitor_10000 node1" [ style = bold] -+"dummy:2_start_0 node1" [ style=bold color="green" fontcolor="black"] - } -diff --git a/cts/scheduler/dot/clone-recover-no-shuffle-5.dot b/cts/scheduler/dot/clone-recover-no-shuffle-5.dot -index a2356f2280b..d3bdf04baa9 100644 ---- a/cts/scheduler/dot/clone-recover-no-shuffle-5.dot -+++ b/cts/scheduler/dot/clone-recover-no-shuffle-5.dot -@@ -1,56 +1,21 @@ - digraph "g" { - "grp-clone_running_0" [ style=bold color="green" fontcolor="orange"] - "grp-clone_start_0" -> "grp-clone_running_0" [ style = bold] --"grp-clone_start_0" -> "grp:0_start_0" [ style = bold] - "grp-clone_start_0" -> "grp:2_start_0" [ style = bold] - "grp-clone_start_0" [ style=bold color="green" fontcolor="orange"] --"grp-clone_stop_0" -> "grp-clone_stopped_0" [ style = bold] --"grp-clone_stop_0" -> "grp:0_stop_0" [ style = bold] --"grp-clone_stop_0" [ style=bold color="green" fontcolor="orange"] --"grp-clone_stopped_0" -> "grp-clone_start_0" [ style = bold] --"grp-clone_stopped_0" [ style=bold color="green" fontcolor="orange"] --"grp:0_running_0" -> "grp-clone_running_0" [ style = bold] --"grp:0_running_0" [ style=bold color="green" fontcolor="orange"] --"grp:0_start_0" -> "grp:0_running_0" [ style = bold] --"grp:0_start_0" -> "rsc1_start_0 node1" [ style = bold] --"grp:0_start_0" -> "rsc2_start_0 node1" [ style = bold] --"grp:0_start_0" [ style=bold color="green" fontcolor="orange"] --"grp:0_stop_0" -> "grp:0_stopped_0" [ style = bold] --"grp:0_stop_0" -> "rsc1_stop_0 node2" [ style = bold] --"grp:0_stop_0" -> "rsc2_stop_0 node2" [ style = bold] --"grp:0_stop_0" [ style=bold color="green" fontcolor="orange"] --"grp:0_stopped_0" -> "grp-clone_stopped_0" [ style = bold] --"grp:0_stopped_0" -> "grp:0_start_0" [ style = bold] --"grp:0_stopped_0" [ style=bold color="green" fontcolor="orange"] - "grp:2_running_0" -> "grp-clone_running_0" [ style = bold] - "grp:2_running_0" [ style=bold color="green" fontcolor="orange"] - "grp:2_start_0" -> "grp:2_running_0" [ style = bold] --"grp:2_start_0" -> "rsc1:2_start_0 node2" [ style = bold] --"grp:2_start_0" -> "rsc2:2_start_0 node2" [ style = bold] -+"grp:2_start_0" -> "rsc1:2_start_0 node1" [ style = bold] -+"grp:2_start_0" -> "rsc2:2_start_0 node1" [ style = bold] - "grp:2_start_0" [ style=bold color="green" fontcolor="orange"] --"rsc1:2_monitor_10000 node2" [ style=bold color="green" fontcolor="black"] --"rsc1:2_start_0 node2" -> "grp:2_running_0" [ style = bold] --"rsc1:2_start_0 node2" -> "rsc1:2_monitor_10000 node2" [ style = bold] --"rsc1:2_start_0 node2" -> "rsc2:2_start_0 node2" [ style = bold] --"rsc1:2_start_0 node2" [ style=bold color="green" fontcolor="black"] --"rsc1_monitor_10000 node1" [ style=bold color="green" fontcolor="black"] --"rsc1_start_0 node1" -> "grp:0_running_0" [ style = bold] --"rsc1_start_0 node1" -> "rsc1_monitor_10000 node1" [ style = bold] --"rsc1_start_0 node1" -> "rsc2_start_0 node1" [ style = bold] --"rsc1_start_0 node1" [ style=bold color="green" fontcolor="black"] --"rsc1_stop_0 node2" -> "grp:0_stopped_0" [ style = bold] --"rsc1_stop_0 node2" -> "rsc1_start_0 node1" [ style = bold] --"rsc1_stop_0 node2" [ style=bold color="green" fontcolor="black"] --"rsc2:2_monitor_10000 node2" [ style=bold color="green" fontcolor="black"] --"rsc2:2_start_0 node2" -> "grp:2_running_0" [ style = bold] --"rsc2:2_start_0 node2" -> "rsc2:2_monitor_10000 node2" [ style = bold] --"rsc2:2_start_0 node2" [ style=bold color="green" fontcolor="black"] --"rsc2_monitor_10000 node1" [ style=bold color="green" fontcolor="black"] --"rsc2_start_0 node1" -> "grp:0_running_0" [ style = bold] --"rsc2_start_0 node1" -> "rsc2_monitor_10000 node1" [ style = bold] --"rsc2_start_0 node1" [ style=bold color="green" fontcolor="black"] --"rsc2_stop_0 node2" -> "grp:0_stopped_0" [ style = bold] --"rsc2_stop_0 node2" -> "rsc1_stop_0 node2" [ style = bold] --"rsc2_stop_0 node2" -> "rsc2_start_0 node1" [ style = bold] --"rsc2_stop_0 node2" [ style=bold color="green" fontcolor="black"] -+"rsc1:2_monitor_10000 node1" [ style=bold color="green" fontcolor="black"] -+"rsc1:2_start_0 node1" -> "grp:2_running_0" [ style = bold] -+"rsc1:2_start_0 node1" -> "rsc1:2_monitor_10000 node1" [ style = bold] -+"rsc1:2_start_0 node1" -> "rsc2:2_start_0 node1" [ style = bold] -+"rsc1:2_start_0 node1" [ style=bold color="green" fontcolor="black"] -+"rsc2:2_monitor_10000 node1" [ style=bold color="green" fontcolor="black"] -+"rsc2:2_start_0 node1" -> "grp:2_running_0" [ style = bold] -+"rsc2:2_start_0 node1" -> "rsc2:2_monitor_10000 node1" [ style = bold] -+"rsc2:2_start_0 node1" [ style=bold color="green" fontcolor="black"] - } -diff --git a/cts/scheduler/dot/clone-recover-no-shuffle-6.dot b/cts/scheduler/dot/clone-recover-no-shuffle-6.dot -index f8cfe9252d2..f60fd2cc04e 100644 ---- a/cts/scheduler/dot/clone-recover-no-shuffle-6.dot -+++ b/cts/scheduler/dot/clone-recover-no-shuffle-6.dot -@@ -1,97 +1,32 @@ - digraph "g" { --"base-bundle-0_monitor_30000 node1" [ style=bold color="green" fontcolor="black"] --"base-bundle-0_start_0 node1" -> "base-bundle-0_monitor_30000 node1" [ style = bold] --"base-bundle-0_start_0 node1" -> "base_start_0 base-bundle-0" [ style = bold] --"base-bundle-0_start_0 node1" [ style=bold color="green" fontcolor="black"] --"base-bundle-0_stop_0 node3" -> "base-bundle-0_start_0 node1" [ style = bold] --"base-bundle-0_stop_0 node3" -> "base-bundle-podman-0_stop_0 node3" [ style = bold] --"base-bundle-0_stop_0 node3" [ style=bold color="green" fontcolor="black"] --"base-bundle-1_monitor_30000 node3" [ style=bold color="green" fontcolor="black"] --"base-bundle-1_start_0 node3" -> "base-bundle-1_monitor_30000 node3" [ style = bold] --"base-bundle-1_start_0 node3" -> "base_start_0 base-bundle-1" [ style = bold] --"base-bundle-1_start_0 node3" [ style=bold color="green" fontcolor="black"] --"base-bundle-1_stop_0 node2" -> "base-bundle-1_start_0 node3" [ style = bold] --"base-bundle-1_stop_0 node2" -> "base-bundle-podman-1_stop_0 node2" [ style = bold] --"base-bundle-1_stop_0 node2" [ style=bold color="green" fontcolor="black"] --"base-bundle-2_monitor_0 node1" -> "base-bundle-2_start_0 node2" [ style = bold] -+"base-bundle-2_monitor_0 node1" -> "base-bundle-2_start_0 node1" [ style = bold] - "base-bundle-2_monitor_0 node1" [ style=bold color="green" fontcolor="black"] --"base-bundle-2_monitor_0 node2" -> "base-bundle-2_start_0 node2" [ style = bold] -+"base-bundle-2_monitor_0 node2" -> "base-bundle-2_start_0 node1" [ style = bold] - "base-bundle-2_monitor_0 node2" [ style=bold color="green" fontcolor="black"] --"base-bundle-2_monitor_0 node3" -> "base-bundle-2_start_0 node2" [ style = bold] -+"base-bundle-2_monitor_0 node3" -> "base-bundle-2_start_0 node1" [ style = bold] - "base-bundle-2_monitor_0 node3" [ style=bold color="green" fontcolor="black"] --"base-bundle-2_monitor_30000 node2" [ style=bold color="green" fontcolor="black"] --"base-bundle-2_start_0 node2" -> "base-bundle-2_monitor_30000 node2" [ style = bold] --"base-bundle-2_start_0 node2" -> "base:2_start_0 base-bundle-2" [ style = bold] --"base-bundle-2_start_0 node2" [ style=bold color="green" fontcolor="black"] -+"base-bundle-2_monitor_30000 node1" [ style=bold color="green" fontcolor="black"] -+"base-bundle-2_start_0 node1" -> "base-bundle-2_monitor_30000 node1" [ style = bold] -+"base-bundle-2_start_0 node1" -> "base:2_start_0 base-bundle-2" [ style = bold] -+"base-bundle-2_start_0 node1" [ style=bold color="green" fontcolor="black"] - "base-bundle-clone_running_0" -> "base-bundle_running_0" [ style = bold] - "base-bundle-clone_running_0" [ style=bold color="green" fontcolor="orange"] - "base-bundle-clone_start_0" -> "base-bundle-clone_running_0" [ style = bold] - "base-bundle-clone_start_0" -> "base:2_start_0 base-bundle-2" [ style = bold] --"base-bundle-clone_start_0" -> "base_start_0 base-bundle-0" [ style = bold] --"base-bundle-clone_start_0" -> "base_start_0 base-bundle-1" [ style = bold] - "base-bundle-clone_start_0" [ style=bold color="green" fontcolor="orange"] --"base-bundle-clone_stop_0" -> "base-bundle-clone_stopped_0" [ style = bold] --"base-bundle-clone_stop_0" -> "base_stop_0 base-bundle-0" [ style = bold] --"base-bundle-clone_stop_0" -> "base_stop_0 base-bundle-1" [ style = bold] --"base-bundle-clone_stop_0" [ style=bold color="green" fontcolor="orange"] --"base-bundle-clone_stopped_0" -> "base-bundle-clone_start_0" [ style = bold] --"base-bundle-clone_stopped_0" -> "base-bundle_stopped_0" [ style = bold] --"base-bundle-clone_stopped_0" [ style=bold color="green" fontcolor="orange"] --"base-bundle-podman-0_monitor_60000 node1" [ style=bold color="green" fontcolor="black"] --"base-bundle-podman-0_start_0 node1" -> "base-bundle-0_start_0 node1" [ style = bold] --"base-bundle-podman-0_start_0 node1" -> "base-bundle-podman-0_monitor_60000 node1" [ style = bold] --"base-bundle-podman-0_start_0 node1" -> "base-bundle_running_0" [ style = bold] --"base-bundle-podman-0_start_0 node1" -> "base_start_0 base-bundle-0" [ style = bold] --"base-bundle-podman-0_start_0 node1" [ style=bold color="green" fontcolor="black"] --"base-bundle-podman-0_stop_0 node3" -> "base-bundle-podman-0_start_0 node1" [ style = bold] --"base-bundle-podman-0_stop_0 node3" -> "base-bundle_stopped_0" [ style = bold] --"base-bundle-podman-0_stop_0 node3" [ style=bold color="green" fontcolor="black"] --"base-bundle-podman-1_monitor_60000 node3" [ style=bold color="green" fontcolor="black"] --"base-bundle-podman-1_start_0 node3" -> "base-bundle-1_start_0 node3" [ style = bold] --"base-bundle-podman-1_start_0 node3" -> "base-bundle-podman-1_monitor_60000 node3" [ style = bold] --"base-bundle-podman-1_start_0 node3" -> "base-bundle_running_0" [ style = bold] --"base-bundle-podman-1_start_0 node3" -> "base_start_0 base-bundle-1" [ style = bold] --"base-bundle-podman-1_start_0 node3" [ style=bold color="green" fontcolor="black"] --"base-bundle-podman-1_stop_0 node2" -> "base-bundle-podman-1_start_0 node3" [ style = bold] --"base-bundle-podman-1_stop_0 node2" -> "base-bundle_stopped_0" [ style = bold] --"base-bundle-podman-1_stop_0 node2" [ style=bold color="green" fontcolor="black"] --"base-bundle-podman-2_monitor_60000 node2" [ style=bold color="green" fontcolor="black"] --"base-bundle-podman-2_start_0 node2" -> "base-bundle-2_monitor_0 node1" [ style = bold] --"base-bundle-podman-2_start_0 node2" -> "base-bundle-2_monitor_0 node2" [ style = bold] --"base-bundle-podman-2_start_0 node2" -> "base-bundle-2_monitor_0 node3" [ style = bold] --"base-bundle-podman-2_start_0 node2" -> "base-bundle-2_start_0 node2" [ style = bold] --"base-bundle-podman-2_start_0 node2" -> "base-bundle-podman-2_monitor_60000 node2" [ style = bold] --"base-bundle-podman-2_start_0 node2" -> "base-bundle_running_0" [ style = bold] --"base-bundle-podman-2_start_0 node2" -> "base:2_start_0 base-bundle-2" [ style = bold] --"base-bundle-podman-2_start_0 node2" [ style=bold color="green" fontcolor="black"] -+"base-bundle-podman-2_monitor_60000 node1" [ style=bold color="green" fontcolor="black"] -+"base-bundle-podman-2_start_0 node1" -> "base-bundle-2_monitor_0 node1" [ style = bold] -+"base-bundle-podman-2_start_0 node1" -> "base-bundle-2_monitor_0 node2" [ style = bold] -+"base-bundle-podman-2_start_0 node1" -> "base-bundle-2_monitor_0 node3" [ style = bold] -+"base-bundle-podman-2_start_0 node1" -> "base-bundle-2_start_0 node1" [ style = bold] -+"base-bundle-podman-2_start_0 node1" -> "base-bundle-podman-2_monitor_60000 node1" [ style = bold] -+"base-bundle-podman-2_start_0 node1" -> "base-bundle_running_0" [ style = bold] -+"base-bundle-podman-2_start_0 node1" -> "base:2_start_0 base-bundle-2" [ style = bold] -+"base-bundle-podman-2_start_0 node1" [ style=bold color="green" fontcolor="black"] - "base-bundle_running_0" [ style=bold color="green" fontcolor="orange"] - "base-bundle_start_0" -> "base-bundle-clone_start_0" [ style = bold] --"base-bundle_start_0" -> "base-bundle-podman-0_start_0 node1" [ style = bold] --"base-bundle_start_0" -> "base-bundle-podman-1_start_0 node3" [ style = bold] --"base-bundle_start_0" -> "base-bundle-podman-2_start_0 node2" [ style = bold] -+"base-bundle_start_0" -> "base-bundle-podman-2_start_0 node1" [ style = bold] - "base-bundle_start_0" [ style=bold color="green" fontcolor="orange"] --"base-bundle_stop_0" -> "base-bundle-clone_stop_0" [ style = bold] --"base-bundle_stop_0" -> "base-bundle-podman-0_stop_0 node3" [ style = bold] --"base-bundle_stop_0" -> "base-bundle-podman-1_stop_0 node2" [ style = bold] --"base-bundle_stop_0" -> "base_stop_0 base-bundle-0" [ style = bold] --"base-bundle_stop_0" -> "base_stop_0 base-bundle-1" [ style = bold] --"base-bundle_stop_0" [ style=bold color="green" fontcolor="orange"] --"base-bundle_stopped_0" [ style=bold color="green" fontcolor="orange"] - "base:2_start_0 base-bundle-2" -> "base-bundle-clone_running_0" [ style = bold] - "base:2_start_0 base-bundle-2" [ style=bold color="green" fontcolor="black"] --"base_start_0 base-bundle-0" -> "base-bundle-clone_running_0" [ style = bold] --"base_start_0 base-bundle-0" -> "base_start_0 base-bundle-1" [ style = bold] --"base_start_0 base-bundle-0" [ style=bold color="green" fontcolor="black"] --"base_start_0 base-bundle-1" -> "base-bundle-clone_running_0" [ style = bold] --"base_start_0 base-bundle-1" -> "base:2_start_0 base-bundle-2" [ style = bold] --"base_start_0 base-bundle-1" [ style=bold color="green" fontcolor="black"] --"base_stop_0 base-bundle-0" -> "base-bundle-0_stop_0 node3" [ style = bold] --"base_stop_0 base-bundle-0" -> "base-bundle-clone_stopped_0" [ style = bold] --"base_stop_0 base-bundle-0" -> "base_start_0 base-bundle-0" [ style = bold] --"base_stop_0 base-bundle-0" [ style=bold color="green" fontcolor="black"] --"base_stop_0 base-bundle-1" -> "base-bundle-1_stop_0 node2" [ style = bold] --"base_stop_0 base-bundle-1" -> "base-bundle-clone_stopped_0" [ style = bold] --"base_stop_0 base-bundle-1" -> "base_start_0 base-bundle-1" [ style = bold] --"base_stop_0 base-bundle-1" -> "base_stop_0 base-bundle-0" [ style = bold] --"base_stop_0 base-bundle-1" [ style=bold color="green" fontcolor="black"] - } -diff --git a/cts/scheduler/dot/clone-recover-no-shuffle-7.dot b/cts/scheduler/dot/clone-recover-no-shuffle-7.dot -index 8bff7da01db..f61bf0d7acf 100644 ---- a/cts/scheduler/dot/clone-recover-no-shuffle-7.dot -+++ b/cts/scheduler/dot/clone-recover-no-shuffle-7.dot -@@ -6,40 +6,25 @@ - "dummy-clone_demote_0" [ style=bold color="green" fontcolor="orange"] - "dummy-clone_demoted_0" -> "dummy-clone_promote_0" [ style = bold] - "dummy-clone_demoted_0" -> "dummy-clone_start_0" [ style = bold] --"dummy-clone_demoted_0" -> "dummy-clone_stop_0" [ style = bold] - "dummy-clone_demoted_0" [ style=bold color="green" fontcolor="orange"] --"dummy-clone_promote_0" -> "dummy_promote_0 node1" [ style = bold] -+"dummy-clone_promote_0" -> "dummy:2_promote_0 node1" [ style = bold] - "dummy-clone_promote_0" [ style=bold color="green" fontcolor="orange"] - "dummy-clone_promoted_0" [ style=bold color="green" fontcolor="orange"] - "dummy-clone_running_0" -> "dummy-clone_promote_0" [ style = bold] - "dummy-clone_running_0" [ style=bold color="green" fontcolor="orange"] - "dummy-clone_start_0" -> "dummy-clone_running_0" [ style = bold] --"dummy-clone_start_0" -> "dummy:2_start_0 node3" [ style = bold] --"dummy-clone_start_0" -> "dummy_start_0 node1" [ style = bold] -+"dummy-clone_start_0" -> "dummy:2_start_0 node1" [ style = bold] - "dummy-clone_start_0" [ style=bold color="green" fontcolor="orange"] --"dummy-clone_stop_0" -> "dummy-clone_stopped_0" [ style = bold] --"dummy-clone_stop_0" -> "dummy_stop_0 node3" [ style = bold] --"dummy-clone_stop_0" [ style=bold color="green" fontcolor="orange"] --"dummy-clone_stopped_0" -> "dummy-clone_promote_0" [ style = bold] --"dummy-clone_stopped_0" -> "dummy-clone_start_0" [ style = bold] --"dummy-clone_stopped_0" [ style=bold color="green" fontcolor="orange"] --"dummy:2_monitor_11000 node3" [ style=bold color="green" fontcolor="black"] --"dummy:2_start_0 node3" -> "dummy-clone_running_0" [ style = bold] --"dummy:2_start_0 node3" -> "dummy:2_monitor_11000 node3" [ style = bold] --"dummy:2_start_0 node3" [ style=bold color="green" fontcolor="black"] -+"dummy:2_monitor_10000 node1" [ style=bold color="green" fontcolor="black"] -+"dummy:2_promote_0 node1" -> "dummy-clone_promoted_0" [ style = bold] -+"dummy:2_promote_0 node1" -> "dummy:2_monitor_10000 node1" [ style = bold] -+"dummy:2_promote_0 node1" [ style=bold color="green" fontcolor="black"] -+"dummy:2_start_0 node1" -> "dummy-clone_running_0" [ style = bold] -+"dummy:2_start_0 node1" -> "dummy:2_monitor_10000 node1" [ style = bold] -+"dummy:2_start_0 node1" -> "dummy:2_promote_0 node1" [ style = bold] -+"dummy:2_start_0 node1" [ style=bold color="green" fontcolor="black"] - "dummy_demote_0 node2" -> "dummy-clone_demoted_0" [ style = bold] - "dummy_demote_0 node2" -> "dummy_monitor_11000 node2" [ style = bold] - "dummy_demote_0 node2" [ style=bold color="green" fontcolor="black"] --"dummy_monitor_10000 node1" [ style=bold color="green" fontcolor="black"] - "dummy_monitor_11000 node2" [ style=bold color="green" fontcolor="black"] --"dummy_promote_0 node1" -> "dummy-clone_promoted_0" [ style = bold] --"dummy_promote_0 node1" -> "dummy_monitor_10000 node1" [ style = bold] --"dummy_promote_0 node1" [ style=bold color="green" fontcolor="black"] --"dummy_start_0 node1" -> "dummy-clone_running_0" [ style = bold] --"dummy_start_0 node1" -> "dummy_monitor_10000 node1" [ style = bold] --"dummy_start_0 node1" -> "dummy_promote_0 node1" [ style = bold] --"dummy_start_0 node1" [ style=bold color="green" fontcolor="black"] --"dummy_stop_0 node3" -> "dummy-clone_stopped_0" [ style = bold] --"dummy_stop_0 node3" -> "dummy_start_0 node1" [ style = bold] --"dummy_stop_0 node3" [ style=bold color="green" fontcolor="black"] - } -diff --git a/cts/scheduler/exp/cancel-behind-moving-remote.exp b/cts/scheduler/exp/cancel-behind-moving-remote.exp -index 17759cb8c57..68cdf4d5370 100644 ---- a/cts/scheduler/exp/cancel-behind-moving-remote.exp -+++ b/cts/scheduler/exp/cancel-behind-moving-remote.exp -@@ -1,46 +1,46 @@ - - - -- -+ - - - - - -- -+ - - - - - -- -+ - - - - - -- -+ - - -- -+ - - - - - -- -+ - - - - - -- -+ - - - - - -- -+ - - - -@@ -48,193 +48,187 @@ - - - -- -+ - - - - - -- -+ - - - - - -- -+ - - - - - -- -+ - - -- -+ - - - - - -- -+ - -- -+ - - - - -- -+ - - - - - -- -+ - -- -+ - - - - -- -+ - - - - - -- -+ - -- -+ - - - - -- -+ - - - - - -- -+ - -- -+ - - - - -- -+ - - -- -+ - - -- -+ - - -- -+ - - - - - -- -+ - -- -+ - - - - -- -+ - - -- -+ - - -- -+ - - - - - -- -+ - -- -+ - - - - -- -+ - - - - - -- -+ - -- -+ - - - - -- -+ - - - -- -+ - -- -+ - -- -+ - - - - -- -+ - - - -- -+ - -- -+ - -- -+ - - - - -- -+ - - - - - -- -+ - -- -+ - - - - -- -+ - - -- -+ - - -- -- -- -- -- -- -- -+ - - - - - -- -+ - -- -+ - - - -@@ -242,61 +236,11 @@ - - - -- -- -- -- -- -- -- -- -- -- -+ - - - - -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- - - - -@@ -305,567 +249,302 @@ - - - -- -+ - -- -+ - -- -+ - - - - -- -+ - - - -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -+ - -- -+ - -- -+ - - - - -- -+ - - - -- -+ - -- -+ - -- -+ - - - - -- -+ - - - -- -+ - -- -+ - -- -+ - - - - -- -+ - - - -- -+ - -- -+ - - - - - -- -+ - - -- -+ - - -- -+ - - -- -+ - - - -- -+ - -- -+ - - - - - -- -+ - - -- -+ - - - -- -+ - -- -+ - - - - - -- -+ - - -- -+ - - -- -+ - - -- -+ - - - -- -+ - -- -+ - - - - - -- -- -- -- -+ - - - -- -+ - -- -+ - - - - - -- -+ - - - -- -+ - -- -+ - - - - - -- -- -- -- -+ - - -- -+ - - -- -+ - - - -- -+ - -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -+ - - - - - -- -+ - - -- -+ - - -- -+ - - -- -+ - - - -- -+ - -- -+ - - - - - -- -+ - - -- -+ - - - -- -+ - -- -+ - - - - - -- -+ - - -- -+ - -- -- -- -- -- -- -- -- -- - -- -+ - - - -- -+ - -- -- -+ -+ - - -- -- -- -- -- -- -- -- -+ - -- -+ - -- -+ - - - - - -- -+ - - -- -+ - - - -- -+ - -- -+ - - - - - -- -+ - - -- -- -- -- -+ - - - -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -+ - -- -+ - -- -+ - - - - -- -+ - - - -- -+ - -- -+ - -- -+ - - - - -- -+ - - - -- -+ - -- -+ - -- -+ - - - - -- -+ - - - -- -+ - -- -+ - -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -+ - - - - -- -- -- -- -- -- -- -+ - - - -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -+ - - - -@@ -874,7 +553,7 @@ - - - -- -+ - - - -@@ -883,7 +562,7 @@ - - - -- -+ - - - -@@ -892,7 +571,7 @@ - - - -- -+ - - - -@@ -901,42 +580,42 @@ - - - -- -+ - -- -+ - - - - - - -- -+ - - - -- -+ - -- -+ - - - - - - -- -+ - - - -- -+ - -- -+ - - - - - - -- -+ - - - -@@ -945,7 +624,7 @@ - - - -- -+ - - - -@@ -954,7 +633,7 @@ - - - -- -+ - - - -@@ -963,42 +642,42 @@ - - - -- -+ - -- -+ - - - - - - -- -+ - - - -- -+ - -- -+ - - - - - - -- -+ - - - -- -+ - -- -+ - - - - - - -- -+ - - - -@@ -1007,7 +686,7 @@ - - - -- -+ - - - -@@ -1016,7 +695,7 @@ - - - -- -+ - - - -@@ -1025,7 +704,7 @@ - - - -- -+ - - - -@@ -1034,101 +713,68 @@ - - - -- -+ - -- -+ - - - - - -- -+ - - - -- -+ - -- -+ - - - - - -- -- -- -- -+ - - - -- -+ - -- -+ - - - - - -- -+ - - -- -+ - - - -- -+ - -- -+ - - - - - -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -+ - -- -+ - - - - - -- -+ - - - -- -+ - -- -+ - - - -diff --git a/cts/scheduler/exp/clone-recover-no-shuffle-4.exp b/cts/scheduler/exp/clone-recover-no-shuffle-4.exp -index 4596c685d0a..670a823dac9 100644 ---- a/cts/scheduler/exp/clone-recover-no-shuffle-4.exp -+++ b/cts/scheduler/exp/clone-recover-no-shuffle-4.exp -@@ -1,123 +1,51 @@ - - - -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -+ - -- -+ - - - - -- -+ - - - -- -+ - -- -+ - -- -+ - - - - -- -+ - - - -- -+ - -- -+ - - - - - -- -+ - - -- -+ - - - -- -+ - -- -+ - - - - - -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- - -diff --git a/cts/scheduler/exp/clone-recover-no-shuffle-5.exp b/cts/scheduler/exp/clone-recover-no-shuffle-5.exp -index c1cee43b12f..84b1e1bc98c 100644 ---- a/cts/scheduler/exp/clone-recover-no-shuffle-5.exp -+++ b/cts/scheduler/exp/clone-recover-no-shuffle-5.exp -@@ -1,293 +1,110 @@ - - - -- -+ - - - - - -- -+ - - -- -+ - - -- -+ - - - - - -- -+ - - - - - -- -+ - - - - - -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -+ - -- -+ - - - - -- -+ - - - -- -+ - -- -+ - -- -+ - - - - -- -+ - - - -- -+ - -- -+ - -- -+ - - - - -- -+ - - - -- -+ - -- -+ - -- -+ - - - - -- -+ - - -- -+ - - - -- -+ - -- -+ - - - - - -- -+ - - -- -+ - - - -- -+ - -- -+ - - - - - -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- - -diff --git a/cts/scheduler/exp/clone-recover-no-shuffle-6.exp b/cts/scheduler/exp/clone-recover-no-shuffle-6.exp -index e6704c9e254..6b6ed075f57 100644 ---- a/cts/scheduler/exp/clone-recover-no-shuffle-6.exp -+++ b/cts/scheduler/exp/clone-recover-no-shuffle-6.exp -@@ -1,504 +1,168 @@ - - - -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -+ - -- -+ - - - - -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -+ - - -- -+ - - -- -+ - - - -- -+ - -- -+ - - - - - -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -+ - - -- -- -- -- -+ - - - -- -+ - -- -+ - - - - - -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -+ - - - -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -+ - -- -+ - -- -+ - - - - -- -+ - - - -- -+ - -- -+ - -- -+ - - - - -- -+ - - - -- -+ - -- -+ - -- -+ - - - - -- -+ - - - -- -+ - -- -+ - -- -+ - - - - -- -+ - - -- -+ - - -- -+ - - -- -+ - - - -- -+ - -- -+ - -- -+ - - - - -- -+ - - - -- -+ - -- -+ - -- -+ - - - - -- -+ - - - -- -+ - -- -+ - -- -+ - - - - -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -+ - - - -- -- -- -- -- -- -- -- -- -+ - -- -+ - - - - - -- -- -- -- -+ - - -- -- -- -- -+ - - - -- -+ - -- -+ - - - -diff --git a/cts/scheduler/exp/clone-recover-no-shuffle-7.exp b/cts/scheduler/exp/clone-recover-no-shuffle-7.exp -index 950de9e0312..870ed54e9c2 100644 ---- a/cts/scheduler/exp/clone-recover-no-shuffle-7.exp -+++ b/cts/scheduler/exp/clone-recover-no-shuffle-7.exp -@@ -1,239 +1,161 @@ - - - -- -- -- -+ -+ -+ - - - - -- -- -- -- -+ - - - - - -- -- -- -+ -+ -+ - - - - -- -+ - - -- -+ - - - - - -- -- -- -+ -+ -+ - - -- -- -- -- -- -- -- -- -+ - - - -- -- -- -+ -+ -+ - - - - -- -+ - -- -- -- -- -- -- -- -- -- -- - -- -+ - - - -- -+ - -- -- -- -+ -+ -+ - - - - -- -+ - - -- -+ - - - -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -+ - -- -+ - -- -+ - - - - -- -+ - - - -- -+ - -- -+ - - - - - -- -+ - - -- -+ - - - -- -+ - -- -+ - - - - - -- -+ - -- -+ - - - - - -- -+ - - - -- -+ - -- -+ - - - - - -- -- -- -- -+ - - -- -+ - - - -- -+ - -- -+ - - - - - -- -+ - - -- -+ - - - -- -+ - -- -+ - - - - - -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -+ - - - -diff --git a/cts/scheduler/scores/bug-cl-5168.scores b/cts/scheduler/scores/bug-cl-5168.scores -index 916fecb195f..59dee5d39b3 100644 ---- a/cts/scheduler/scores/bug-cl-5168.scores -+++ b/cts/scheduler/scores/bug-cl-5168.scores -@@ -200,7 +200,7 @@ pcmk__primitive_assign: drbd-r1:0 allocation score on hex-2: 1001 - pcmk__primitive_assign: drbd-r1:0 allocation score on hex-3: -INFINITY - pcmk__primitive_assign: drbd-r1:0 allocation score on hex-3: INFINITY - pcmk__primitive_assign: drbd-r1:1 allocation score on hex-1: -INFINITY --pcmk__primitive_assign: drbd-r1:1 allocation score on hex-2: 0 -+pcmk__primitive_assign: drbd-r1:1 allocation score on hex-2: -INFINITY - pcmk__primitive_assign: drbd-r1:1 allocation score on hex-3: INFINITY - pcmk__primitive_assign: dummy1 allocation score on hex-1: -INFINITY - pcmk__primitive_assign: dummy1 allocation score on hex-2: -INFINITY -diff --git a/cts/scheduler/scores/cancel-behind-moving-remote.scores b/cts/scheduler/scores/cancel-behind-moving-remote.scores -index 0e11b225aea..09f0175b9e2 100644 ---- a/cts/scheduler/scores/cancel-behind-moving-remote.scores -+++ b/cts/scheduler/scores/cancel-behind-moving-remote.scores -@@ -1799,7 +1799,7 @@ pcmk__primitive_assign: ip-172.17.1.151 allocation score on messaging-1: -INFINI - pcmk__primitive_assign: ip-172.17.1.151 allocation score on messaging-2: -INFINITY - pcmk__primitive_assign: ip-172.17.1.87 allocation score on compute-0: -INFINITY - pcmk__primitive_assign: ip-172.17.1.87 allocation score on compute-1: -INFINITY --pcmk__primitive_assign: ip-172.17.1.87 allocation score on controller-0: 0 -+pcmk__primitive_assign: ip-172.17.1.87 allocation score on controller-0: -INFINITY - pcmk__primitive_assign: ip-172.17.1.87 allocation score on controller-1: -INFINITY - pcmk__primitive_assign: ip-172.17.1.87 allocation score on controller-2: -INFINITY - pcmk__primitive_assign: ip-172.17.1.87 allocation score on database-0: -INFINITY -@@ -1865,9 +1865,9 @@ pcmk__primitive_assign: openstack-cinder-volume-podman-0 allocation score on mes - pcmk__primitive_assign: openstack-cinder-volume-podman-0 allocation score on messaging-2: -INFINITY - pcmk__primitive_assign: ovn-dbs-bundle-0 allocation score on compute-0: -INFINITY - pcmk__primitive_assign: ovn-dbs-bundle-0 allocation score on compute-1: -INFINITY --pcmk__primitive_assign: ovn-dbs-bundle-0 allocation score on controller-0: 0 -+pcmk__primitive_assign: ovn-dbs-bundle-0 allocation score on controller-0: 10000 - pcmk__primitive_assign: ovn-dbs-bundle-0 allocation score on controller-1: 0 --pcmk__primitive_assign: ovn-dbs-bundle-0 allocation score on controller-2: 10000 -+pcmk__primitive_assign: ovn-dbs-bundle-0 allocation score on controller-2: 0 - pcmk__primitive_assign: ovn-dbs-bundle-0 allocation score on database-0: 0 - pcmk__primitive_assign: ovn-dbs-bundle-0 allocation score on database-1: 0 - pcmk__primitive_assign: ovn-dbs-bundle-0 allocation score on database-2: 0 -@@ -1876,9 +1876,9 @@ pcmk__primitive_assign: ovn-dbs-bundle-0 allocation score on messaging-1: 0 - pcmk__primitive_assign: ovn-dbs-bundle-0 allocation score on messaging-2: 0 - pcmk__primitive_assign: ovn-dbs-bundle-1 allocation score on compute-0: -INFINITY - pcmk__primitive_assign: ovn-dbs-bundle-1 allocation score on compute-1: -INFINITY --pcmk__primitive_assign: ovn-dbs-bundle-1 allocation score on controller-0: 10000 -+pcmk__primitive_assign: ovn-dbs-bundle-1 allocation score on controller-0: 0 - pcmk__primitive_assign: ovn-dbs-bundle-1 allocation score on controller-1: 0 --pcmk__primitive_assign: ovn-dbs-bundle-1 allocation score on controller-2: 0 -+pcmk__primitive_assign: ovn-dbs-bundle-1 allocation score on controller-2: 10000 - pcmk__primitive_assign: ovn-dbs-bundle-1 allocation score on database-0: 0 - pcmk__primitive_assign: ovn-dbs-bundle-1 allocation score on database-1: 0 - pcmk__primitive_assign: ovn-dbs-bundle-1 allocation score on database-2: 0 -@@ -1898,9 +1898,9 @@ pcmk__primitive_assign: ovn-dbs-bundle-2 allocation score on messaging-1: 0 - pcmk__primitive_assign: ovn-dbs-bundle-2 allocation score on messaging-2: 0 - pcmk__primitive_assign: ovn-dbs-bundle-podman-0 allocation score on compute-0: -INFINITY - pcmk__primitive_assign: ovn-dbs-bundle-podman-0 allocation score on compute-1: -INFINITY --pcmk__primitive_assign: ovn-dbs-bundle-podman-0 allocation score on controller-0: -INFINITY -+pcmk__primitive_assign: ovn-dbs-bundle-podman-0 allocation score on controller-0: 0 - pcmk__primitive_assign: ovn-dbs-bundle-podman-0 allocation score on controller-1: -INFINITY --pcmk__primitive_assign: ovn-dbs-bundle-podman-0 allocation score on controller-2: 0 -+pcmk__primitive_assign: ovn-dbs-bundle-podman-0 allocation score on controller-2: -INFINITY - pcmk__primitive_assign: ovn-dbs-bundle-podman-0 allocation score on database-0: -INFINITY - pcmk__primitive_assign: ovn-dbs-bundle-podman-0 allocation score on database-1: -INFINITY - pcmk__primitive_assign: ovn-dbs-bundle-podman-0 allocation score on database-2: -INFINITY -@@ -1909,24 +1909,35 @@ pcmk__primitive_assign: ovn-dbs-bundle-podman-0 allocation score on messaging-1: - pcmk__primitive_assign: ovn-dbs-bundle-podman-0 allocation score on messaging-2: -INFINITY - pcmk__primitive_assign: ovn-dbs-bundle-podman-1 allocation score on compute-0: -INFINITY - pcmk__primitive_assign: ovn-dbs-bundle-podman-1 allocation score on compute-0: -INFINITY -+pcmk__primitive_assign: ovn-dbs-bundle-podman-1 allocation score on compute-0: -INFINITY - pcmk__primitive_assign: ovn-dbs-bundle-podman-1 allocation score on compute-1: -INFINITY - pcmk__primitive_assign: ovn-dbs-bundle-podman-1 allocation score on compute-1: -INFINITY --pcmk__primitive_assign: ovn-dbs-bundle-podman-1 allocation score on controller-0: 0 -+pcmk__primitive_assign: ovn-dbs-bundle-podman-1 allocation score on compute-1: -INFINITY -+pcmk__primitive_assign: ovn-dbs-bundle-podman-1 allocation score on controller-0: -INFINITY -+pcmk__primitive_assign: ovn-dbs-bundle-podman-1 allocation score on controller-0: -INFINITY - pcmk__primitive_assign: ovn-dbs-bundle-podman-1 allocation score on controller-0: 0 - pcmk__primitive_assign: ovn-dbs-bundle-podman-1 allocation score on controller-1: -INFINITY - pcmk__primitive_assign: ovn-dbs-bundle-podman-1 allocation score on controller-1: 0 -+pcmk__primitive_assign: ovn-dbs-bundle-podman-1 allocation score on controller-1: 0 - pcmk__primitive_assign: ovn-dbs-bundle-podman-1 allocation score on controller-2: -INFINITY - pcmk__primitive_assign: ovn-dbs-bundle-podman-1 allocation score on controller-2: -INFINITY -+pcmk__primitive_assign: ovn-dbs-bundle-podman-1 allocation score on controller-2: 0 -+pcmk__primitive_assign: ovn-dbs-bundle-podman-1 allocation score on database-0: -INFINITY - pcmk__primitive_assign: ovn-dbs-bundle-podman-1 allocation score on database-0: -INFINITY - pcmk__primitive_assign: ovn-dbs-bundle-podman-1 allocation score on database-0: -INFINITY - pcmk__primitive_assign: ovn-dbs-bundle-podman-1 allocation score on database-1: -INFINITY - pcmk__primitive_assign: ovn-dbs-bundle-podman-1 allocation score on database-1: -INFINITY -+pcmk__primitive_assign: ovn-dbs-bundle-podman-1 allocation score on database-1: -INFINITY -+pcmk__primitive_assign: ovn-dbs-bundle-podman-1 allocation score on database-2: -INFINITY - pcmk__primitive_assign: ovn-dbs-bundle-podman-1 allocation score on database-2: -INFINITY - pcmk__primitive_assign: ovn-dbs-bundle-podman-1 allocation score on database-2: -INFINITY - pcmk__primitive_assign: ovn-dbs-bundle-podman-1 allocation score on messaging-0: -INFINITY - pcmk__primitive_assign: ovn-dbs-bundle-podman-1 allocation score on messaging-0: -INFINITY -+pcmk__primitive_assign: ovn-dbs-bundle-podman-1 allocation score on messaging-0: -INFINITY - pcmk__primitive_assign: ovn-dbs-bundle-podman-1 allocation score on messaging-1: -INFINITY - pcmk__primitive_assign: ovn-dbs-bundle-podman-1 allocation score on messaging-1: -INFINITY -+pcmk__primitive_assign: ovn-dbs-bundle-podman-1 allocation score on messaging-1: -INFINITY -+pcmk__primitive_assign: ovn-dbs-bundle-podman-1 allocation score on messaging-2: -INFINITY - pcmk__primitive_assign: ovn-dbs-bundle-podman-1 allocation score on messaging-2: -INFINITY - pcmk__primitive_assign: ovn-dbs-bundle-podman-1 allocation score on messaging-2: -INFINITY - pcmk__primitive_assign: ovn-dbs-bundle-podman-2 allocation score on compute-0: -INFINITY -diff --git a/cts/scheduler/scores/clone-recover-no-shuffle-10.scores b/cts/scheduler/scores/clone-recover-no-shuffle-10.scores -index 4ac63e37058..4f4c29ed7f1 100644 ---- a/cts/scheduler/scores/clone-recover-no-shuffle-10.scores -+++ b/cts/scheduler/scores/clone-recover-no-shuffle-10.scores -@@ -28,4 +28,4 @@ pcmk__primitive_assign: dummy:1 allocation score on node2: 16 - pcmk__primitive_assign: dummy:1 allocation score on node3: 0 - pcmk__primitive_assign: dummy:2 allocation score on node1: 10 - pcmk__primitive_assign: dummy:2 allocation score on node2: -INFINITY --pcmk__primitive_assign: dummy:2 allocation score on node3: 5 -+pcmk__primitive_assign: dummy:2 allocation score on node3: -INFINITY -diff --git a/cts/scheduler/scores/clone-recover-no-shuffle-4.scores b/cts/scheduler/scores/clone-recover-no-shuffle-4.scores -index 492dad1baa4..2a52c8185b2 100644 ---- a/cts/scheduler/scores/clone-recover-no-shuffle-4.scores -+++ b/cts/scheduler/scores/clone-recover-no-shuffle-4.scores -@@ -14,7 +14,7 @@ pcmk__clone_assign: dummy:2 allocation score on node3: 0 - pcmk__primitive_assign: Fencing allocation score on node1: 0 - pcmk__primitive_assign: Fencing allocation score on node2: 0 - pcmk__primitive_assign: Fencing allocation score on node3: 0 --pcmk__primitive_assign: dummy:0 allocation score on node1: 100 -+pcmk__primitive_assign: dummy:0 allocation score on node1: -INFINITY - pcmk__primitive_assign: dummy:0 allocation score on node1: 100 - pcmk__primitive_assign: dummy:0 allocation score on node2: 1 - pcmk__primitive_assign: dummy:0 allocation score on node2: 1 -@@ -22,10 +22,10 @@ pcmk__primitive_assign: dummy:0 allocation score on node3: 0 - pcmk__primitive_assign: dummy:0 allocation score on node3: 0 - pcmk__primitive_assign: dummy:1 allocation score on node1: -INFINITY - pcmk__primitive_assign: dummy:1 allocation score on node1: 100 --pcmk__primitive_assign: dummy:1 allocation score on node2: 0 --pcmk__primitive_assign: dummy:1 allocation score on node2: 0 -+pcmk__primitive_assign: dummy:1 allocation score on node2: -INFINITY -+pcmk__primitive_assign: dummy:1 allocation score on node2: -INFINITY - pcmk__primitive_assign: dummy:1 allocation score on node3: 1 - pcmk__primitive_assign: dummy:1 allocation score on node3: 1 --pcmk__primitive_assign: dummy:2 allocation score on node1: -INFINITY --pcmk__primitive_assign: dummy:2 allocation score on node2: 0 -+pcmk__primitive_assign: dummy:2 allocation score on node1: 100 -+pcmk__primitive_assign: dummy:2 allocation score on node2: -INFINITY - pcmk__primitive_assign: dummy:2 allocation score on node3: -INFINITY -diff --git a/cts/scheduler/scores/clone-recover-no-shuffle-5.scores b/cts/scheduler/scores/clone-recover-no-shuffle-5.scores -index 0dd9728830c..c6c8072db82 100644 ---- a/cts/scheduler/scores/clone-recover-no-shuffle-5.scores -+++ b/cts/scheduler/scores/clone-recover-no-shuffle-5.scores -@@ -29,7 +29,7 @@ pcmk__clone_assign: rsc2:1 allocation score on node3: 1 - pcmk__clone_assign: rsc2:2 allocation score on node1: 0 - pcmk__clone_assign: rsc2:2 allocation score on node2: 0 - pcmk__clone_assign: rsc2:2 allocation score on node3: 0 --pcmk__group_assign: grp:0 allocation score on node1: 100 -+pcmk__group_assign: grp:0 allocation score on node1: -INFINITY - pcmk__group_assign: grp:0 allocation score on node1: 100 - pcmk__group_assign: grp:0 allocation score on node2: 0 - pcmk__group_assign: grp:0 allocation score on node2: 0 -@@ -37,14 +37,14 @@ pcmk__group_assign: grp:0 allocation score on node3: 0 - pcmk__group_assign: grp:0 allocation score on node3: 0 - pcmk__group_assign: grp:1 allocation score on node1: -INFINITY - pcmk__group_assign: grp:1 allocation score on node1: 100 --pcmk__group_assign: grp:1 allocation score on node2: 0 --pcmk__group_assign: grp:1 allocation score on node2: 0 -+pcmk__group_assign: grp:1 allocation score on node2: -INFINITY -+pcmk__group_assign: grp:1 allocation score on node2: -INFINITY - pcmk__group_assign: grp:1 allocation score on node3: 0 - pcmk__group_assign: grp:1 allocation score on node3: 0 --pcmk__group_assign: grp:2 allocation score on node1: -INFINITY --pcmk__group_assign: grp:2 allocation score on node2: 0 -+pcmk__group_assign: grp:2 allocation score on node1: 100 -+pcmk__group_assign: grp:2 allocation score on node2: -INFINITY - pcmk__group_assign: grp:2 allocation score on node3: -INFINITY --pcmk__group_assign: rsc1:0 allocation score on node1: 100 -+pcmk__group_assign: rsc1:0 allocation score on node1: -INFINITY - pcmk__group_assign: rsc1:0 allocation score on node1: 100 - pcmk__group_assign: rsc1:0 allocation score on node2: 1 - pcmk__group_assign: rsc1:0 allocation score on node2: 1 -@@ -52,14 +52,14 @@ pcmk__group_assign: rsc1:0 allocation score on node3: 0 - pcmk__group_assign: rsc1:0 allocation score on node3: 0 - pcmk__group_assign: rsc1:1 allocation score on node1: -INFINITY - pcmk__group_assign: rsc1:1 allocation score on node1: 100 --pcmk__group_assign: rsc1:1 allocation score on node2: 0 --pcmk__group_assign: rsc1:1 allocation score on node2: 0 -+pcmk__group_assign: rsc1:1 allocation score on node2: -INFINITY -+pcmk__group_assign: rsc1:1 allocation score on node2: -INFINITY - pcmk__group_assign: rsc1:1 allocation score on node3: 1 - pcmk__group_assign: rsc1:1 allocation score on node3: 1 --pcmk__group_assign: rsc1:2 allocation score on node1: -INFINITY --pcmk__group_assign: rsc1:2 allocation score on node2: 0 -+pcmk__group_assign: rsc1:2 allocation score on node1: 100 -+pcmk__group_assign: rsc1:2 allocation score on node2: -INFINITY - pcmk__group_assign: rsc1:2 allocation score on node3: -INFINITY --pcmk__group_assign: rsc2:0 allocation score on node1: 0 -+pcmk__group_assign: rsc2:0 allocation score on node1: -INFINITY - pcmk__group_assign: rsc2:0 allocation score on node1: 0 - pcmk__group_assign: rsc2:0 allocation score on node2: 1 - pcmk__group_assign: rsc2:0 allocation score on node2: 1 -@@ -67,17 +67,17 @@ pcmk__group_assign: rsc2:0 allocation score on node3: 0 - pcmk__group_assign: rsc2:0 allocation score on node3: 0 - pcmk__group_assign: rsc2:1 allocation score on node1: -INFINITY - pcmk__group_assign: rsc2:1 allocation score on node1: 0 --pcmk__group_assign: rsc2:1 allocation score on node2: 0 --pcmk__group_assign: rsc2:1 allocation score on node2: 0 -+pcmk__group_assign: rsc2:1 allocation score on node2: -INFINITY -+pcmk__group_assign: rsc2:1 allocation score on node2: -INFINITY - pcmk__group_assign: rsc2:1 allocation score on node3: 1 - pcmk__group_assign: rsc2:1 allocation score on node3: 1 --pcmk__group_assign: rsc2:2 allocation score on node1: -INFINITY --pcmk__group_assign: rsc2:2 allocation score on node2: 0 -+pcmk__group_assign: rsc2:2 allocation score on node1: 0 -+pcmk__group_assign: rsc2:2 allocation score on node2: -INFINITY - pcmk__group_assign: rsc2:2 allocation score on node3: -INFINITY - pcmk__primitive_assign: Fencing allocation score on node1: 0 - pcmk__primitive_assign: Fencing allocation score on node2: 0 - pcmk__primitive_assign: Fencing allocation score on node3: 0 --pcmk__primitive_assign: rsc1:0 allocation score on node1: 100 -+pcmk__primitive_assign: rsc1:0 allocation score on node1: -INFINITY - pcmk__primitive_assign: rsc1:0 allocation score on node1: 100 - pcmk__primitive_assign: rsc1:0 allocation score on node2: 2 - pcmk__primitive_assign: rsc1:0 allocation score on node2: 2 -@@ -85,17 +85,17 @@ pcmk__primitive_assign: rsc1:0 allocation score on node3: 0 - pcmk__primitive_assign: rsc1:0 allocation score on node3: 0 - pcmk__primitive_assign: rsc1:1 allocation score on node1: -INFINITY - pcmk__primitive_assign: rsc1:1 allocation score on node1: 100 --pcmk__primitive_assign: rsc1:1 allocation score on node2: 0 --pcmk__primitive_assign: rsc1:1 allocation score on node2: 0 -+pcmk__primitive_assign: rsc1:1 allocation score on node2: -INFINITY -+pcmk__primitive_assign: rsc1:1 allocation score on node2: -INFINITY - pcmk__primitive_assign: rsc1:1 allocation score on node3: 2 - pcmk__primitive_assign: rsc1:1 allocation score on node3: 2 --pcmk__primitive_assign: rsc1:2 allocation score on node1: -INFINITY --pcmk__primitive_assign: rsc1:2 allocation score on node2: 0 -+pcmk__primitive_assign: rsc1:2 allocation score on node1: 100 -+pcmk__primitive_assign: rsc1:2 allocation score on node2: -INFINITY - pcmk__primitive_assign: rsc1:2 allocation score on node3: -INFINITY -+pcmk__primitive_assign: rsc2:0 allocation score on node1: -INFINITY - pcmk__primitive_assign: rsc2:0 allocation score on node1: 0 --pcmk__primitive_assign: rsc2:0 allocation score on node1: 0 --pcmk__primitive_assign: rsc2:0 allocation score on node2: -INFINITY - pcmk__primitive_assign: rsc2:0 allocation score on node2: -INFINITY -+pcmk__primitive_assign: rsc2:0 allocation score on node2: 1 - pcmk__primitive_assign: rsc2:0 allocation score on node3: -INFINITY - pcmk__primitive_assign: rsc2:0 allocation score on node3: -INFINITY - pcmk__primitive_assign: rsc2:1 allocation score on node1: -INFINITY -@@ -104,6 +104,6 @@ pcmk__primitive_assign: rsc2:1 allocation score on node2: -INFINITY - pcmk__primitive_assign: rsc2:1 allocation score on node2: -INFINITY - pcmk__primitive_assign: rsc2:1 allocation score on node3: -INFINITY - pcmk__primitive_assign: rsc2:1 allocation score on node3: 1 --pcmk__primitive_assign: rsc2:2 allocation score on node1: -INFINITY --pcmk__primitive_assign: rsc2:2 allocation score on node2: 0 -+pcmk__primitive_assign: rsc2:2 allocation score on node1: 0 -+pcmk__primitive_assign: rsc2:2 allocation score on node2: -INFINITY - pcmk__primitive_assign: rsc2:2 allocation score on node3: -INFINITY -diff --git a/cts/scheduler/scores/clone-recover-no-shuffle-6.scores b/cts/scheduler/scores/clone-recover-no-shuffle-6.scores -index 643e30f9d18..f1f300cbd66 100644 ---- a/cts/scheduler/scores/clone-recover-no-shuffle-6.scores -+++ b/cts/scheduler/scores/clone-recover-no-shuffle-6.scores -@@ -41,16 +41,16 @@ pcmk__clone_assign: base:2 allocation score on base-bundle-2: INFINITY - pcmk__primitive_assign: Fencing allocation score on node1: 0 - pcmk__primitive_assign: Fencing allocation score on node2: 0 - pcmk__primitive_assign: Fencing allocation score on node3: 0 --pcmk__primitive_assign: base-bundle-0 allocation score on node1: 10000 -+pcmk__primitive_assign: base-bundle-0 allocation score on node1: 0 - pcmk__primitive_assign: base-bundle-0 allocation score on node2: 0 --pcmk__primitive_assign: base-bundle-0 allocation score on node3: 0 -+pcmk__primitive_assign: base-bundle-0 allocation score on node3: 10000 - pcmk__primitive_assign: base-bundle-1 allocation score on node1: 0 --pcmk__primitive_assign: base-bundle-1 allocation score on node2: 0 --pcmk__primitive_assign: base-bundle-1 allocation score on node3: 10000 --pcmk__primitive_assign: base-bundle-2 allocation score on node1: 0 --pcmk__primitive_assign: base-bundle-2 allocation score on node2: 10000 -+pcmk__primitive_assign: base-bundle-1 allocation score on node2: 10000 -+pcmk__primitive_assign: base-bundle-1 allocation score on node3: 0 -+pcmk__primitive_assign: base-bundle-2 allocation score on node1: 10000 -+pcmk__primitive_assign: base-bundle-2 allocation score on node2: 0 - pcmk__primitive_assign: base-bundle-2 allocation score on node3: 0 --pcmk__primitive_assign: base-bundle-podman-0 allocation score on node1: 100 -+pcmk__primitive_assign: base-bundle-podman-0 allocation score on node1: -INFINITY - pcmk__primitive_assign: base-bundle-podman-0 allocation score on node1: 100 - pcmk__primitive_assign: base-bundle-podman-0 allocation score on node2: 0 - pcmk__primitive_assign: base-bundle-podman-0 allocation score on node2: 0 -@@ -60,10 +60,10 @@ pcmk__primitive_assign: base-bundle-podman-1 allocation score on node1: -INFINIT - pcmk__primitive_assign: base-bundle-podman-1 allocation score on node1: 100 - pcmk__primitive_assign: base-bundle-podman-1 allocation score on node2: 0 - pcmk__primitive_assign: base-bundle-podman-1 allocation score on node2: 0 --pcmk__primitive_assign: base-bundle-podman-1 allocation score on node3: 0 --pcmk__primitive_assign: base-bundle-podman-1 allocation score on node3: 0 --pcmk__primitive_assign: base-bundle-podman-2 allocation score on node1: -INFINITY --pcmk__primitive_assign: base-bundle-podman-2 allocation score on node2: 0 -+pcmk__primitive_assign: base-bundle-podman-1 allocation score on node3: -INFINITY -+pcmk__primitive_assign: base-bundle-podman-1 allocation score on node3: -INFINITY -+pcmk__primitive_assign: base-bundle-podman-2 allocation score on node1: 100 -+pcmk__primitive_assign: base-bundle-podman-2 allocation score on node2: -INFINITY - pcmk__primitive_assign: base-bundle-podman-2 allocation score on node3: -INFINITY - pcmk__primitive_assign: base:0 allocation score on base-bundle-0: INFINITY - pcmk__primitive_assign: base:1 allocation score on base-bundle-1: INFINITY -diff --git a/cts/scheduler/scores/clone-recover-no-shuffle-7.scores b/cts/scheduler/scores/clone-recover-no-shuffle-7.scores -index fc45bf740fd..503cbb3addf 100644 ---- a/cts/scheduler/scores/clone-recover-no-shuffle-7.scores -+++ b/cts/scheduler/scores/clone-recover-no-shuffle-7.scores -@@ -1,7 +1,7 @@ - --dummy:0 promotion score on node1: 15 -+dummy:0 promotion score on node3: 5 - dummy:1 promotion score on node2: 10 --dummy:2 promotion score on node3: 5 -+dummy:2 promotion score on node1: 15 - pcmk__clone_assign: dummy-clone allocation score on node1: 0 - pcmk__clone_assign: dummy-clone allocation score on node2: 0 - pcmk__clone_assign: dummy-clone allocation score on node3: 0 -@@ -17,7 +17,7 @@ pcmk__clone_assign: dummy:2 allocation score on node3: 5 - pcmk__primitive_assign: Fencing allocation score on node1: 0 - pcmk__primitive_assign: Fencing allocation score on node2: 0 - pcmk__primitive_assign: Fencing allocation score on node3: 0 --pcmk__primitive_assign: dummy:0 allocation score on node1: 15 -+pcmk__primitive_assign: dummy:0 allocation score on node1: -INFINITY - pcmk__primitive_assign: dummy:0 allocation score on node1: 15 - pcmk__primitive_assign: dummy:0 allocation score on node2: 0 - pcmk__primitive_assign: dummy:0 allocation score on node2: 0 -@@ -27,8 +27,8 @@ pcmk__primitive_assign: dummy:1 allocation score on node1: -INFINITY - pcmk__primitive_assign: dummy:1 allocation score on node1: 15 - pcmk__primitive_assign: dummy:1 allocation score on node2: 11 - pcmk__primitive_assign: dummy:1 allocation score on node2: 11 --pcmk__primitive_assign: dummy:1 allocation score on node3: 0 --pcmk__primitive_assign: dummy:1 allocation score on node3: 0 --pcmk__primitive_assign: dummy:2 allocation score on node1: -INFINITY -+pcmk__primitive_assign: dummy:1 allocation score on node3: -INFINITY -+pcmk__primitive_assign: dummy:1 allocation score on node3: -INFINITY -+pcmk__primitive_assign: dummy:2 allocation score on node1: 15 - pcmk__primitive_assign: dummy:2 allocation score on node2: -INFINITY --pcmk__primitive_assign: dummy:2 allocation score on node3: 5 -+pcmk__primitive_assign: dummy:2 allocation score on node3: -INFINITY -diff --git a/cts/scheduler/scores/promoted-failed-demote-2.scores b/cts/scheduler/scores/promoted-failed-demote-2.scores -index e457d8c6057..39399d9eac4 100644 ---- a/cts/scheduler/scores/promoted-failed-demote-2.scores -+++ b/cts/scheduler/scores/promoted-failed-demote-2.scores -@@ -34,14 +34,10 @@ pcmk__group_assign: stateful-2:0 allocation score on dl380g5b: -INFINITY - pcmk__group_assign: stateful-2:1 allocation score on dl380g5a: INFINITY - pcmk__group_assign: stateful-2:1 allocation score on dl380g5b: 0 - pcmk__primitive_assign: stateful-1:0 allocation score on dl380g5a: -INFINITY --pcmk__primitive_assign: stateful-1:0 allocation score on dl380g5a: -INFINITY --pcmk__primitive_assign: stateful-1:0 allocation score on dl380g5b: -INFINITY - pcmk__primitive_assign: stateful-1:0 allocation score on dl380g5b: -INFINITY - pcmk__primitive_assign: stateful-1:1 allocation score on dl380g5a: INFINITY - pcmk__primitive_assign: stateful-1:1 allocation score on dl380g5b: 0 - pcmk__primitive_assign: stateful-2:0 allocation score on dl380g5a: -INFINITY --pcmk__primitive_assign: stateful-2:0 allocation score on dl380g5a: -INFINITY --pcmk__primitive_assign: stateful-2:0 allocation score on dl380g5b: -INFINITY - pcmk__primitive_assign: stateful-2:0 allocation score on dl380g5b: -INFINITY - pcmk__primitive_assign: stateful-2:1 allocation score on dl380g5a: INFINITY - pcmk__primitive_assign: stateful-2:1 allocation score on dl380g5b: -INFINITY -diff --git a/cts/scheduler/scores/promoted-failed-demote.scores b/cts/scheduler/scores/promoted-failed-demote.scores -index e457d8c6057..39399d9eac4 100644 ---- a/cts/scheduler/scores/promoted-failed-demote.scores -+++ b/cts/scheduler/scores/promoted-failed-demote.scores -@@ -34,14 +34,10 @@ pcmk__group_assign: stateful-2:0 allocation score on dl380g5b: -INFINITY - pcmk__group_assign: stateful-2:1 allocation score on dl380g5a: INFINITY - pcmk__group_assign: stateful-2:1 allocation score on dl380g5b: 0 - pcmk__primitive_assign: stateful-1:0 allocation score on dl380g5a: -INFINITY --pcmk__primitive_assign: stateful-1:0 allocation score on dl380g5a: -INFINITY --pcmk__primitive_assign: stateful-1:0 allocation score on dl380g5b: -INFINITY - pcmk__primitive_assign: stateful-1:0 allocation score on dl380g5b: -INFINITY - pcmk__primitive_assign: stateful-1:1 allocation score on dl380g5a: INFINITY - pcmk__primitive_assign: stateful-1:1 allocation score on dl380g5b: 0 - pcmk__primitive_assign: stateful-2:0 allocation score on dl380g5a: -INFINITY --pcmk__primitive_assign: stateful-2:0 allocation score on dl380g5a: -INFINITY --pcmk__primitive_assign: stateful-2:0 allocation score on dl380g5b: -INFINITY - pcmk__primitive_assign: stateful-2:0 allocation score on dl380g5b: -INFINITY - pcmk__primitive_assign: stateful-2:1 allocation score on dl380g5a: INFINITY - pcmk__primitive_assign: stateful-2:1 allocation score on dl380g5b: -INFINITY -diff --git a/cts/scheduler/scores/utilization-complex.scores b/cts/scheduler/scores/utilization-complex.scores -index 29bc92c193f..b9dd80c4b6a 100644 ---- a/cts/scheduler/scores/utilization-complex.scores -+++ b/cts/scheduler/scores/utilization-complex.scores -@@ -312,18 +312,26 @@ pcmk__primitive_assign: clone1:2 allocation score on rhel8-4: 1 - pcmk__primitive_assign: clone1:2 allocation score on rhel8-5: 0 - pcmk__primitive_assign: clone1:3 allocation score on httpd-bundle-0: -INFINITY - pcmk__primitive_assign: clone1:3 allocation score on httpd-bundle-0: -INFINITY -+pcmk__primitive_assign: clone1:3 allocation score on httpd-bundle-0: -INFINITY -+pcmk__primitive_assign: clone1:3 allocation score on httpd-bundle-1: -INFINITY - pcmk__primitive_assign: clone1:3 allocation score on httpd-bundle-1: -INFINITY - pcmk__primitive_assign: clone1:3 allocation score on httpd-bundle-1: -INFINITY - pcmk__primitive_assign: clone1:3 allocation score on httpd-bundle-2: -INFINITY - pcmk__primitive_assign: clone1:3 allocation score on httpd-bundle-2: -INFINITY -+pcmk__primitive_assign: clone1:3 allocation score on httpd-bundle-2: -INFINITY -+pcmk__primitive_assign: clone1:3 allocation score on rhel8-1: -INFINITY - pcmk__primitive_assign: clone1:3 allocation score on rhel8-1: -INFINITY - pcmk__primitive_assign: clone1:3 allocation score on rhel8-1: 0 - pcmk__primitive_assign: clone1:3 allocation score on rhel8-2: -INFINITY - pcmk__primitive_assign: clone1:3 allocation score on rhel8-2: -INFINITY -+pcmk__primitive_assign: clone1:3 allocation score on rhel8-2: -INFINITY -+pcmk__primitive_assign: clone1:3 allocation score on rhel8-3: -INFINITY - pcmk__primitive_assign: clone1:3 allocation score on rhel8-3: -INFINITY - pcmk__primitive_assign: clone1:3 allocation score on rhel8-3: -INFINITY - pcmk__primitive_assign: clone1:3 allocation score on rhel8-4: -INFINITY - pcmk__primitive_assign: clone1:3 allocation score on rhel8-4: -INFINITY -+pcmk__primitive_assign: clone1:3 allocation score on rhel8-4: -INFINITY -+pcmk__primitive_assign: clone1:3 allocation score on rhel8-5: 1 - pcmk__primitive_assign: clone1:3 allocation score on rhel8-5: 1 - pcmk__primitive_assign: clone1:3 allocation score on rhel8-5: 1 - pcmk__primitive_assign: clone1:4 allocation score on httpd-bundle-0: -INFINITY -@@ -384,18 +392,26 @@ pcmk__primitive_assign: clone2:2 allocation score on rhel8-4: 1 - pcmk__primitive_assign: clone2:2 allocation score on rhel8-5: -INFINITY - pcmk__primitive_assign: clone2:3 allocation score on httpd-bundle-0: -INFINITY - pcmk__primitive_assign: clone2:3 allocation score on httpd-bundle-0: -INFINITY -+pcmk__primitive_assign: clone2:3 allocation score on httpd-bundle-0: -INFINITY - pcmk__primitive_assign: clone2:3 allocation score on httpd-bundle-1: -INFINITY - pcmk__primitive_assign: clone2:3 allocation score on httpd-bundle-1: -INFINITY -+pcmk__primitive_assign: clone2:3 allocation score on httpd-bundle-1: -INFINITY -+pcmk__primitive_assign: clone2:3 allocation score on httpd-bundle-2: -INFINITY - pcmk__primitive_assign: clone2:3 allocation score on httpd-bundle-2: -INFINITY - pcmk__primitive_assign: clone2:3 allocation score on httpd-bundle-2: -INFINITY - pcmk__primitive_assign: clone2:3 allocation score on rhel8-1: -INFINITY -+pcmk__primitive_assign: clone2:3 allocation score on rhel8-1: -INFINITY - pcmk__primitive_assign: clone2:3 allocation score on rhel8-1: 0 - pcmk__primitive_assign: clone2:3 allocation score on rhel8-2: -INFINITY - pcmk__primitive_assign: clone2:3 allocation score on rhel8-2: -INFINITY -+pcmk__primitive_assign: clone2:3 allocation score on rhel8-2: -INFINITY -+pcmk__primitive_assign: clone2:3 allocation score on rhel8-3: -INFINITY - pcmk__primitive_assign: clone2:3 allocation score on rhel8-3: -INFINITY - pcmk__primitive_assign: clone2:3 allocation score on rhel8-3: -INFINITY - pcmk__primitive_assign: clone2:3 allocation score on rhel8-4: -INFINITY - pcmk__primitive_assign: clone2:3 allocation score on rhel8-4: -INFINITY -+pcmk__primitive_assign: clone2:3 allocation score on rhel8-4: -INFINITY -+pcmk__primitive_assign: clone2:3 allocation score on rhel8-5: -INFINITY - pcmk__primitive_assign: clone2:3 allocation score on rhel8-5: -INFINITY - pcmk__primitive_assign: clone2:3 allocation score on rhel8-5: -INFINITY - pcmk__primitive_assign: clone2:4 allocation score on httpd-bundle-0: -INFINITY -@@ -535,18 +551,26 @@ pcmk__primitive_assign: httpd-bundle-ip-192.168.122.133 allocation score on rhel - pcmk__primitive_assign: httpd-bundle-ip-192.168.122.133 allocation score on rhel8-5: -INFINITY - pcmk__primitive_assign: httpd-bundle-podman-0 allocation score on httpd-bundle-0: -INFINITY - pcmk__primitive_assign: httpd-bundle-podman-0 allocation score on httpd-bundle-0: -INFINITY -+pcmk__primitive_assign: httpd-bundle-podman-0 allocation score on httpd-bundle-0: -INFINITY -+pcmk__primitive_assign: httpd-bundle-podman-0 allocation score on httpd-bundle-1: -INFINITY - pcmk__primitive_assign: httpd-bundle-podman-0 allocation score on httpd-bundle-1: -INFINITY - pcmk__primitive_assign: httpd-bundle-podman-0 allocation score on httpd-bundle-1: -INFINITY - pcmk__primitive_assign: httpd-bundle-podman-0 allocation score on httpd-bundle-2: -INFINITY - pcmk__primitive_assign: httpd-bundle-podman-0 allocation score on httpd-bundle-2: -INFINITY -+pcmk__primitive_assign: httpd-bundle-podman-0 allocation score on httpd-bundle-2: -INFINITY -+pcmk__primitive_assign: httpd-bundle-podman-0 allocation score on rhel8-1: -INFINITY - pcmk__primitive_assign: httpd-bundle-podman-0 allocation score on rhel8-1: -INFINITY - pcmk__primitive_assign: httpd-bundle-podman-0 allocation score on rhel8-1: -INFINITY - pcmk__primitive_assign: httpd-bundle-podman-0 allocation score on rhel8-2: -INFINITY - pcmk__primitive_assign: httpd-bundle-podman-0 allocation score on rhel8-2: -INFINITY -+pcmk__primitive_assign: httpd-bundle-podman-0 allocation score on rhel8-2: -INFINITY - pcmk__primitive_assign: httpd-bundle-podman-0 allocation score on rhel8-3: -INFINITY - pcmk__primitive_assign: httpd-bundle-podman-0 allocation score on rhel8-3: -INFINITY -+pcmk__primitive_assign: httpd-bundle-podman-0 allocation score on rhel8-3: -INFINITY -+pcmk__primitive_assign: httpd-bundle-podman-0 allocation score on rhel8-4: -INFINITY - pcmk__primitive_assign: httpd-bundle-podman-0 allocation score on rhel8-4: -INFINITY - pcmk__primitive_assign: httpd-bundle-podman-0 allocation score on rhel8-4: -INFINITY -+pcmk__primitive_assign: httpd-bundle-podman-0 allocation score on rhel8-5: -INFINITY - pcmk__primitive_assign: httpd-bundle-podman-0 allocation score on rhel8-5: 0 - pcmk__primitive_assign: httpd-bundle-podman-0 allocation score on rhel8-5: 0 - pcmk__primitive_assign: httpd-bundle-podman-1 allocation score on httpd-bundle-1: -INFINITY -diff --git a/cts/scheduler/scores/utilization-order2.scores b/cts/scheduler/scores/utilization-order2.scores -index c4b49d9b366..4476b60ee21 100644 ---- a/cts/scheduler/scores/utilization-order2.scores -+++ b/cts/scheduler/scores/utilization-order2.scores -@@ -9,6 +9,8 @@ pcmk__primitive_assign: rsc1 allocation score on node1: 0 - pcmk__primitive_assign: rsc1 allocation score on node2: 0 - pcmk__primitive_assign: rsc2:0 allocation score on node1: 1 - pcmk__primitive_assign: rsc2:0 allocation score on node1: 1 -+pcmk__primitive_assign: rsc2:0 allocation score on node1: 1 -+pcmk__primitive_assign: rsc2:0 allocation score on node2: -INFINITY - pcmk__primitive_assign: rsc2:0 allocation score on node2: -INFINITY - pcmk__primitive_assign: rsc2:0 allocation score on node2: 0 - pcmk__primitive_assign: rsc2:1 allocation score on node1: 0 -diff --git a/cts/scheduler/summary/cancel-behind-moving-remote.summary b/cts/scheduler/summary/cancel-behind-moving-remote.summary -index 7726876f949..58de340318b 100644 ---- a/cts/scheduler/summary/cancel-behind-moving-remote.summary -+++ b/cts/scheduler/summary/cancel-behind-moving-remote.summary -@@ -58,22 +58,17 @@ Current cluster status: - Transition Summary: - * Start rabbitmq-bundle-1 ( controller-0 ) due to unrunnable rabbitmq-bundle-podman-1 start (blocked) - * Start rabbitmq:1 ( rabbitmq-bundle-1 ) due to unrunnable rabbitmq-bundle-podman-1 start (blocked) -- * Start ovn-dbs-bundle-podman-0 ( controller-2 ) -- * Start ovn-dbs-bundle-0 ( controller-2 ) -+ * Start ovn-dbs-bundle-podman-0 ( controller-0 ) -+ * Start ovn-dbs-bundle-0 ( controller-0 ) - * Start ovndb_servers:0 ( ovn-dbs-bundle-0 ) -- * Move ovn-dbs-bundle-podman-1 ( controller-2 -> controller-0 ) -- * Move ovn-dbs-bundle-1 ( controller-2 -> controller-0 ) -- * Restart ovndb_servers:1 ( Unpromoted -> Promoted ovn-dbs-bundle-1 ) due to required ovn-dbs-bundle-podman-1 start -- * Start ip-172.17.1.87 ( controller-0 ) -+ * Promote ovndb_servers:1 ( Unpromoted -> Promoted ovn-dbs-bundle-1 ) - * Move stonith-fence_ipmilan-52540040bb56 ( messaging-2 -> database-0 ) - * Move stonith-fence_ipmilan-525400e1534e ( database-1 -> messaging-2 ) - - Executing Cluster Transition: - * Pseudo action: rabbitmq-bundle-clone_pre_notify_start_0 - * Resource action: ovndb_servers cancel=30000 on ovn-dbs-bundle-1 -- * Pseudo action: ovn-dbs-bundle-master_pre_notify_stop_0 -- * Cluster action: clear_failcount for ovn-dbs-bundle-0 on controller-0 -- * Cluster action: clear_failcount for ovn-dbs-bundle-1 on controller-2 -+ * Pseudo action: ovn-dbs-bundle-master_pre_notify_start_0 - * Cluster action: clear_failcount for stonith-fence_compute-fence-nova on messaging-0 - * Cluster action: clear_failcount for nova-evacuate on messaging-0 - * Cluster action: clear_failcount for stonith-fence_ipmilan-525400aa1373 on database-0 -@@ -87,52 +82,34 @@ Executing Cluster Transition: - * Cluster action: clear_failcount for stonith-fence_ipmilan-52540060dbba on messaging-0 - * Cluster action: clear_failcount for stonith-fence_ipmilan-525400e018b6 on database-0 - * Cluster action: clear_failcount for stonith-fence_ipmilan-525400c87cdb on database-2 -- * Pseudo action: ovn-dbs-bundle_stop_0 -+ * Pseudo action: ovn-dbs-bundle_start_0 - * Pseudo action: rabbitmq-bundle_start_0 - * Pseudo action: rabbitmq-bundle-clone_confirmed-pre_notify_start_0 - * Pseudo action: rabbitmq-bundle-clone_start_0 - * Resource action: ovndb_servers notify on ovn-dbs-bundle-1 - * Resource action: ovndb_servers notify on ovn-dbs-bundle-2 -- * Pseudo action: ovn-dbs-bundle-master_confirmed-pre_notify_stop_0 -- * Pseudo action: ovn-dbs-bundle-master_stop_0 -+ * Pseudo action: ovn-dbs-bundle-master_confirmed-pre_notify_start_0 -+ * Pseudo action: ovn-dbs-bundle-master_start_0 -+ * Resource action: ovn-dbs-bundle-podman-0 start on controller-0 -+ * Resource action: ovn-dbs-bundle-0 start on controller-0 - * Resource action: stonith-fence_ipmilan-52540040bb56 start on database-0 - * Resource action: stonith-fence_ipmilan-525400e1534e start on messaging-2 - * Pseudo action: rabbitmq-bundle-clone_running_0 -- * Resource action: ovndb_servers stop on ovn-dbs-bundle-1 -- * Pseudo action: ovn-dbs-bundle-master_stopped_0 -- * Resource action: ovn-dbs-bundle-1 stop on controller-2 -+ * Resource action: ovndb_servers start on ovn-dbs-bundle-0 -+ * Pseudo action: ovn-dbs-bundle-master_running_0 -+ * Resource action: ovn-dbs-bundle-podman-0 monitor=60000 on controller-0 -+ * Resource action: ovn-dbs-bundle-0 monitor=30000 on controller-0 - * Resource action: stonith-fence_ipmilan-52540040bb56 monitor=60000 on database-0 - * Resource action: stonith-fence_ipmilan-525400e1534e monitor=60000 on messaging-2 - * Pseudo action: rabbitmq-bundle-clone_post_notify_running_0 -- * Pseudo action: ovn-dbs-bundle-master_post_notify_stopped_0 -- * Resource action: ovn-dbs-bundle-podman-1 stop on controller-2 -- * Pseudo action: rabbitmq-bundle-clone_confirmed-post_notify_running_0 -- * Resource action: ovndb_servers notify on ovn-dbs-bundle-2 -- * Pseudo action: ovn-dbs-bundle-master_confirmed-post_notify_stopped_0 -- * Pseudo action: ovn-dbs-bundle-master_pre_notify_start_0 -- * Pseudo action: ovn-dbs-bundle_stopped_0 -- * Pseudo action: ovn-dbs-bundle_start_0 -- * Pseudo action: rabbitmq-bundle_running_0 -- * Resource action: ovndb_servers notify on ovn-dbs-bundle-2 -- * Pseudo action: ovn-dbs-bundle-master_confirmed-pre_notify_start_0 -- * Pseudo action: ovn-dbs-bundle-master_start_0 -- * Resource action: ovn-dbs-bundle-podman-0 start on controller-2 -- * Resource action: ovn-dbs-bundle-0 start on controller-2 -- * Resource action: ovn-dbs-bundle-podman-1 start on controller-0 -- * Resource action: ovn-dbs-bundle-1 start on controller-0 -- * Resource action: ovndb_servers start on ovn-dbs-bundle-0 -- * Resource action: ovndb_servers start on ovn-dbs-bundle-1 -- * Pseudo action: ovn-dbs-bundle-master_running_0 -- * Resource action: ovn-dbs-bundle-podman-0 monitor=60000 on controller-2 -- * Resource action: ovn-dbs-bundle-0 monitor=30000 on controller-2 -- * Resource action: ovn-dbs-bundle-podman-1 monitor=60000 on controller-0 -- * Resource action: ovn-dbs-bundle-1 monitor=30000 on controller-0 - * Pseudo action: ovn-dbs-bundle-master_post_notify_running_0 -+ * Pseudo action: rabbitmq-bundle-clone_confirmed-post_notify_running_0 - * Resource action: ovndb_servers notify on ovn-dbs-bundle-0 - * Resource action: ovndb_servers notify on ovn-dbs-bundle-1 - * Resource action: ovndb_servers notify on ovn-dbs-bundle-2 - * Pseudo action: ovn-dbs-bundle-master_confirmed-post_notify_running_0 - * Pseudo action: ovn-dbs-bundle_running_0 -+ * Pseudo action: rabbitmq-bundle_running_0 - * Pseudo action: ovn-dbs-bundle-master_pre_notify_promote_0 - * Pseudo action: ovn-dbs-bundle_promote_0 - * Resource action: ovndb_servers notify on ovn-dbs-bundle-0 -@@ -140,10 +117,8 @@ Executing Cluster Transition: - * Resource action: ovndb_servers notify on ovn-dbs-bundle-2 - * Pseudo action: ovn-dbs-bundle-master_confirmed-pre_notify_promote_0 - * Pseudo action: ovn-dbs-bundle-master_promote_0 -- * Resource action: ip-172.17.1.87 start on controller-0 - * Resource action: ovndb_servers promote on ovn-dbs-bundle-1 - * Pseudo action: ovn-dbs-bundle-master_promoted_0 -- * Resource action: ip-172.17.1.87 monitor=10000 on controller-0 - * Pseudo action: ovn-dbs-bundle-master_post_notify_promoted_0 - * Resource action: ovndb_servers notify on ovn-dbs-bundle-0 - * Resource action: ovndb_servers notify on ovn-dbs-bundle-1 -@@ -187,10 +162,10 @@ Revised Cluster Status: - * haproxy-bundle-podman-1 (ocf:heartbeat:podman): Started controller-0 - * haproxy-bundle-podman-2 (ocf:heartbeat:podman): Started controller-1 - * Container bundle set: ovn-dbs-bundle [cluster.common.tag/rhosp16-openstack-ovn-northd:pcmklatest]: -- * ovn-dbs-bundle-0 (ocf:ovn:ovndb-servers): Unpromoted controller-2 -- * ovn-dbs-bundle-1 (ocf:ovn:ovndb-servers): Promoted controller-0 -+ * ovn-dbs-bundle-0 (ocf:ovn:ovndb-servers): Unpromoted controller-0 -+ * ovn-dbs-bundle-1 (ocf:ovn:ovndb-servers): Promoted controller-2 - * ovn-dbs-bundle-2 (ocf:ovn:ovndb-servers): Unpromoted controller-1 -- * ip-172.17.1.87 (ocf:heartbeat:IPaddr2): Started controller-0 -+ * ip-172.17.1.87 (ocf:heartbeat:IPaddr2): Stopped - * stonith-fence_compute-fence-nova (stonith:fence_compute): Started database-1 - * Clone Set: compute-unfence-trigger-clone [compute-unfence-trigger]: - * Started: [ compute-0 compute-1 ] -diff --git a/cts/scheduler/summary/clone-recover-no-shuffle-4.summary b/cts/scheduler/summary/clone-recover-no-shuffle-4.summary -index 944bcb834b3..0b6866ec16c 100644 ---- a/cts/scheduler/summary/clone-recover-no-shuffle-4.summary -+++ b/cts/scheduler/summary/clone-recover-no-shuffle-4.summary -@@ -10,19 +10,13 @@ Current cluster status: - * Stopped: [ node1 ] - - Transition Summary: -- * Move dummy:0 ( node2 -> node1 ) -- * Start dummy:2 ( node2 ) -+ * Start dummy:2 ( node1 ) - - Executing Cluster Transition: -- * Pseudo action: dummy-clone_stop_0 -- * Resource action: dummy stop on node2 -- * Pseudo action: dummy-clone_stopped_0 - * Pseudo action: dummy-clone_start_0 - * Resource action: dummy start on node1 -- * Resource action: dummy start on node2 - * Pseudo action: dummy-clone_running_0 - * Resource action: dummy monitor=10000 on node1 -- * Resource action: dummy monitor=10000 on node2 - Using the original execution date of: 2023-06-21 00:59:59Z - - Revised Cluster Status: -diff --git a/cts/scheduler/summary/clone-recover-no-shuffle-5.summary b/cts/scheduler/summary/clone-recover-no-shuffle-5.summary -index 121214c42ab..8b18120ad8d 100644 ---- a/cts/scheduler/summary/clone-recover-no-shuffle-5.summary -+++ b/cts/scheduler/summary/clone-recover-no-shuffle-5.summary -@@ -9,31 +9,17 @@ Current cluster status: - * Stopped: [ node1 ] - - Transition Summary: -- * Move rsc1:0 ( node2 -> node1 ) -- * Move rsc2:0 ( node2 -> node1 ) -- * Start rsc1:2 ( node2 ) -- * Start rsc2:2 ( node2 ) -+ * Start rsc1:2 ( node1 ) -+ * Start rsc2:2 ( node1 ) - - Executing Cluster Transition: -- * Pseudo action: grp-clone_stop_0 -- * Pseudo action: grp:0_stop_0 -- * Resource action: rsc2 stop on node2 -- * Resource action: rsc1 stop on node2 -- * Pseudo action: grp:0_stopped_0 -- * Pseudo action: grp-clone_stopped_0 - * Pseudo action: grp-clone_start_0 -- * Pseudo action: grp:0_start_0 -+ * Pseudo action: grp:2_start_0 - * Resource action: rsc1 start on node1 - * Resource action: rsc2 start on node1 -- * Pseudo action: grp:2_start_0 -- * Resource action: rsc1 start on node2 -- * Resource action: rsc2 start on node2 -- * Pseudo action: grp:0_running_0 -+ * Pseudo action: grp:2_running_0 - * Resource action: rsc1 monitor=10000 on node1 - * Resource action: rsc2 monitor=10000 on node1 -- * Pseudo action: grp:2_running_0 -- * Resource action: rsc1 monitor=10000 on node2 -- * Resource action: rsc2 monitor=10000 on node2 - * Pseudo action: grp-clone_running_0 - - Revised Cluster Status: -diff --git a/cts/scheduler/summary/clone-recover-no-shuffle-6.summary b/cts/scheduler/summary/clone-recover-no-shuffle-6.summary -index 19a957e15fb..5702177e33d 100644 ---- a/cts/scheduler/summary/clone-recover-no-shuffle-6.summary -+++ b/cts/scheduler/summary/clone-recover-no-shuffle-6.summary -@@ -11,48 +11,22 @@ Current cluster status: - * base-bundle-2 (ocf:pacemaker:Stateful): Stopped - - Transition Summary: -- * Move base-bundle-podman-0 ( node3 -> node1 ) -- * Move base-bundle-0 ( node3 -> node1 ) -- * Restart base:0 ( base-bundle-0 ) due to required base-bundle-podman-0 start -- * Move base-bundle-podman-1 ( node2 -> node3 ) -- * Move base-bundle-1 ( node2 -> node3 ) -- * Restart base:1 ( base-bundle-1 ) due to required base-bundle-podman-1 start -- * Start base-bundle-podman-2 ( node2 ) -- * Start base-bundle-2 ( node2 ) -- * Start base:2 ( base-bundle-2 ) -+ * Start base-bundle-podman-2 ( node1 ) -+ * Start base-bundle-2 ( node1 ) -+ * Start base:2 ( base-bundle-2 ) - - Executing Cluster Transition: -- * Pseudo action: base-bundle_stop_0 - * Pseudo action: base-bundle_start_0 -- * Pseudo action: base-bundle-clone_stop_0 -- * Resource action: base-bundle-podman-2 start on node2 -+ * Pseudo action: base-bundle-clone_start_0 -+ * Resource action: base-bundle-podman-2 start on node1 - * Resource action: base-bundle-2 monitor on node3 - * Resource action: base-bundle-2 monitor on node2 - * Resource action: base-bundle-2 monitor on node1 -- * Resource action: base stop on base-bundle-1 -- * Resource action: base-bundle-1 stop on node2 -- * Resource action: base-bundle-podman-2 monitor=60000 on node2 -- * Resource action: base-bundle-2 start on node2 -- * Resource action: base stop on base-bundle-0 -- * Pseudo action: base-bundle-clone_stopped_0 -- * Pseudo action: base-bundle-clone_start_0 -- * Resource action: base-bundle-0 stop on node3 -- * Resource action: base-bundle-podman-1 stop on node2 -- * Resource action: base-bundle-2 monitor=30000 on node2 -- * Resource action: base-bundle-podman-0 stop on node3 -- * Resource action: base-bundle-podman-1 start on node3 -- * Resource action: base-bundle-1 start on node3 -- * Pseudo action: base-bundle_stopped_0 -- * Resource action: base-bundle-podman-0 start on node1 -- * Resource action: base-bundle-0 start on node1 -- * Resource action: base-bundle-podman-1 monitor=60000 on node3 -- * Resource action: base-bundle-1 monitor=30000 on node3 -- * Resource action: base start on base-bundle-0 -- * Resource action: base start on base-bundle-1 -+ * Resource action: base-bundle-podman-2 monitor=60000 on node1 -+ * Resource action: base-bundle-2 start on node1 - * Resource action: base start on base-bundle-2 - * Pseudo action: base-bundle-clone_running_0 -- * Resource action: base-bundle-podman-0 monitor=60000 on node1 -- * Resource action: base-bundle-0 monitor=30000 on node1 -+ * Resource action: base-bundle-2 monitor=30000 on node1 - * Pseudo action: base-bundle_running_0 - - Revised Cluster Status: -@@ -63,6 +37,6 @@ Revised Cluster Status: - * Full List of Resources: - * Fencing (stonith:fence_xvm): Started node2 - * Container bundle set: base-bundle [localhost/pcmktest]: -- * base-bundle-0 (ocf:pacemaker:Stateful): Started node1 -- * base-bundle-1 (ocf:pacemaker:Stateful): Started node3 -- * base-bundle-2 (ocf:pacemaker:Stateful): Started node2 -+ * base-bundle-0 (ocf:pacemaker:Stateful): Started node3 -+ * base-bundle-1 (ocf:pacemaker:Stateful): Started node2 -+ * base-bundle-2 (ocf:pacemaker:Stateful): Started node1 -diff --git a/cts/scheduler/summary/clone-recover-no-shuffle-7.summary b/cts/scheduler/summary/clone-recover-no-shuffle-7.summary -index e6c9baed0db..77445700f04 100644 ---- a/cts/scheduler/summary/clone-recover-no-shuffle-7.summary -+++ b/cts/scheduler/summary/clone-recover-no-shuffle-7.summary -@@ -10,24 +10,18 @@ Current cluster status: - * Stopped: [ node1 ] - - Transition Summary: -- * Move dummy:0 ( Unpromoted node3 -> Promoted node1 ) -- * Demote dummy:1 ( Promoted -> Unpromoted node2 ) -- * Start dummy:2 ( node3 ) -+ * Demote dummy:1 ( Promoted -> Unpromoted node2 ) -+ * Promote dummy:2 ( Stopped -> Promoted node1 ) - - Executing Cluster Transition: - * Resource action: dummy cancel=10000 on node2 - * Pseudo action: dummy-clone_demote_0 - * Resource action: dummy demote on node2 - * Pseudo action: dummy-clone_demoted_0 -- * Pseudo action: dummy-clone_stop_0 -- * Resource action: dummy stop on node3 -- * Resource action: dummy monitor=11000 on node2 -- * Pseudo action: dummy-clone_stopped_0 - * Pseudo action: dummy-clone_start_0 -+ * Resource action: dummy monitor=11000 on node2 - * Resource action: dummy start on node1 -- * Resource action: dummy start on node3 - * Pseudo action: dummy-clone_running_0 -- * Resource action: dummy monitor=11000 on node3 - * Pseudo action: dummy-clone_promote_0 - * Resource action: dummy promote on node1 - * Pseudo action: dummy-clone_promoted_0 -diff --git a/cts/scheduler/xml/cancel-behind-moving-remote.xml b/cts/scheduler/xml/cancel-behind-moving-remote.xml -index 67e14300ba8..7b880602b1b 100644 ---- a/cts/scheduler/xml/cancel-behind-moving-remote.xml -+++ b/cts/scheduler/xml/cancel-behind-moving-remote.xml -@@ -1,5 +1,19 @@ - - -+ - - - -diff --git a/cts/scheduler/xml/clone-recover-no-shuffle-4.xml b/cts/scheduler/xml/clone-recover-no-shuffle-4.xml -index 40e6520c6d0..f0a5feb8c2f 100644 ---- a/cts/scheduler/xml/clone-recover-no-shuffle-4.xml -+++ b/cts/scheduler/xml/clone-recover-no-shuffle-4.xml -@@ -11,11 +11,6 @@ - * Instance dummy:0 should remain started on node2 - * Instance dummy:1 should remain started on node3 - * Instance dummy:2 should start on node1 -- -- This test output is incorrect: -- * Instance dummy:0 moves from node2 to node1 -- * Instance dummy:1 remains started on node3 (correct) -- * Instance dummy:2 starts on node2 - --> - - -diff --git a/cts/scheduler/xml/clone-recover-no-shuffle-5.xml b/cts/scheduler/xml/clone-recover-no-shuffle-5.xml -index 45f3b5a9f3a..95e5eca9c9d 100644 ---- a/cts/scheduler/xml/clone-recover-no-shuffle-5.xml -+++ b/cts/scheduler/xml/clone-recover-no-shuffle-5.xml -@@ -12,11 +12,6 @@ - * Instance grp:0 should remain started on node2 - * Instance grp:1 should remain started on node3 - * Instance grp:2 should start on node1 -- -- This test output is incorrect: -- * Instance grp:0 moves to node1 -- * Instance grp:1 remains started on node3 (correct) -- * Instance grp:2 starts on node2 - --> - - -diff --git a/cts/scheduler/xml/clone-recover-no-shuffle-6.xml b/cts/scheduler/xml/clone-recover-no-shuffle-6.xml -index 3de42f581d4..64bb4d90179 100644 ---- a/cts/scheduler/xml/clone-recover-no-shuffle-6.xml -+++ b/cts/scheduler/xml/clone-recover-no-shuffle-6.xml -@@ -12,11 +12,6 @@ - * Instance base:0 should remain started on node3 - * Instance base:1 should remain started on node2 - * Instance base:2 should start on node1 -- -- This test output is incorrect: -- * Instance base:0 moves from node3 to node1 -- * Instance base:1 moves from node2 to node3 -- * Instance base:2 starts on node2 - --> - - -diff --git a/cts/scheduler/xml/clone-recover-no-shuffle-7.xml b/cts/scheduler/xml/clone-recover-no-shuffle-7.xml -index 6e9dad50db4..e588b811d77 100644 ---- a/cts/scheduler/xml/clone-recover-no-shuffle-7.xml -+++ b/cts/scheduler/xml/clone-recover-no-shuffle-7.xml -@@ -11,11 +11,6 @@ - * Instance dummy:0 should remain started (unpromoted) on node3 - * Instance dummy:1 should demote on node2 - * Instance dummy:2 should promote on node1 -- -- This test output is incorrect: -- * Instance dummy:0 moves from unpromoted on node3 to promoted on node1 -- * Instance dummy:1 demotes on node2 -- * Instance dummy:2 starts on node3 - --> - - diff --git a/SOURCES/003-schema-doc.patch b/SOURCES/003-schema-doc.patch new file mode 100644 index 0000000..293e50f --- /dev/null +++ b/SOURCES/003-schema-doc.patch @@ -0,0 +1,42 @@ +From a3bffc7c66bf6f796f977cffd44f223635b008c5 Mon Sep 17 00:00:00 2001 +From: Reid Wahl +Date: Wed, 20 Dec 2023 13:33:47 -0800 +Subject: [PATCH] Doc: Pacemaker Explained: Add replace for + PCMK__REMOTE_SCHEMA_DIR + +So that the existing use in local-options.rst expands correctly. + +Signed-off-by: Reid Wahl +--- + doc/sphinx/Makefile.am | 1 + + doc/sphinx/conf.py.in | 1 + + 3 files changed, 2 insertions(+) + create mode 100644 doc/sphinx/conf.py.in.rej + +diff --git a/doc/sphinx/Makefile.am b/doc/sphinx/Makefile.am +index e48e19a..d0309ff 100644 +--- a/doc/sphinx/Makefile.am ++++ b/doc/sphinx/Makefile.am +@@ -134,6 +134,7 @@ $(BOOKS:%=%/conf.py): conf.py.in + -e 's#%CRM_SCHEMA_DIRECTORY%#@CRM_SCHEMA_DIRECTORY@#g' \ + -e 's#%PACEMAKER_CONFIG_DIR%#@PACEMAKER_CONFIG_DIR@#g' \ + -e 's#%PCMK_GNUTLS_PRIORITIES%#@PCMK_GNUTLS_PRIORITIES@#g' \ ++ -e 's#%PCMK__REMOTE_SCHEMA_DIR%#@PCMK__REMOTE_SCHEMA_DIR@#g' \ + $(<) > "$@" + + $(BOOK)/_build: $(STATIC_FILES) $(BOOK)/conf.py $(DEPS_$(BOOK)) $(wildcard $(srcdir)/$(BOOK)/*.rst) +diff --git a/doc/sphinx/conf.py.in b/doc/sphinx/conf.py.in +index 556eb72..511f029 100644 +--- a/doc/sphinx/conf.py.in ++++ b/doc/sphinx/conf.py.in +@@ -40,6 +40,7 @@ rst_prolog=""" + .. |PCMK_INIT_ENV_FILE| replace:: ``%PACEMAKER_CONFIG_DIR%/pcmk-init.env`` + .. |PCMK_LOG_FILE| replace:: %CRM_LOG_DIR%/pacemaker.log + .. |PCMK_GNUTLS_PRIORITIES| replace:: %PCMK_GNUTLS_PRIORITIES% ++.. |PCMK__REMOTE_SCHEMA_DIR| replace:: %PCMK__REMOTE_SCHEMA_DIR% + .. |REMOTE_DISTRO| replace:: AlmaLinux + .. |REMOTE_DISTRO_VER| replace:: 9 + """ +-- +2.31.1 + diff --git a/SOURCES/004-attrd-cache-1.patch b/SOURCES/004-attrd-cache-1.patch new file mode 100644 index 0000000..dd617c4 --- /dev/null +++ b/SOURCES/004-attrd-cache-1.patch @@ -0,0 +1,1443 @@ +From 543a1e9b6f22f13956a8ef22b20c8fe93dad7ae9 Mon Sep 17 00:00:00 2001 +From: Ken Gaillot +Date: Tue, 12 Dec 2023 16:08:44 -0600 +Subject: [PATCH 01/12] Refactor: libcrmcommon: support attrd purge requests + without clearing cache + +Nothing uses the new capability yet +--- + daemons/attrd/attrd_corosync.c | 4 +++- + daemons/attrd/attrd_messages.c | 8 +++++++- + daemons/attrd/pacemaker-attrd.h | 3 ++- + daemons/controld/controld_attrd.c | 2 +- + include/crm/common/ipc_attrd_internal.h | 7 ++++--- + include/crm_internal.h | 1 + + lib/common/ipc_attrd.c | 3 ++- + lib/common/ipc_client.c | 1 + + 8 files changed, 21 insertions(+), 8 deletions(-) + +diff --git a/daemons/attrd/attrd_corosync.c b/daemons/attrd/attrd_corosync.c +index 86dc67b04..e6cd07f65 100644 +--- a/daemons/attrd/attrd_corosync.c ++++ b/daemons/attrd/attrd_corosync.c +@@ -540,7 +540,9 @@ attrd_peer_remove(const char *host, bool uncache, const char *source) + GHashTableIter aIter; + + CRM_CHECK(host != NULL, return); +- crm_notice("Removing all %s attributes for peer %s", host, source); ++ crm_notice("Removing all %s attributes for node %s " ++ CRM_XS " %s reaping node from cache", ++ host, source, (uncache? "and" : "without")); + + g_hash_table_iter_init(&aIter, attributes); + while (g_hash_table_iter_next(&aIter, NULL, (gpointer *) & a)) { +diff --git a/daemons/attrd/attrd_messages.c b/daemons/attrd/attrd_messages.c +index 89da6d894..ac32e18af 100644 +--- a/daemons/attrd/attrd_messages.c ++++ b/daemons/attrd/attrd_messages.c +@@ -148,7 +148,13 @@ handle_remove_request(pcmk__request_t *request) + { + if (request->peer != NULL) { + const char *host = crm_element_value(request->xml, PCMK__XA_ATTR_NODE_NAME); +- attrd_peer_remove(host, true, request->peer); ++ bool reap = false; ++ ++ if (pcmk__xe_get_bool_attr(request->xml, PCMK__XA_REAP, ++ &reap) != pcmk_rc_ok) { ++ reap = true; // Default to true for backward compatibility ++ } ++ attrd_peer_remove(host, reap, request->peer); + pcmk__set_result(&request->result, CRM_EX_OK, PCMK_EXEC_DONE, NULL); + return NULL; + } else { +diff --git a/daemons/attrd/pacemaker-attrd.h b/daemons/attrd/pacemaker-attrd.h +index b8929a7f7..70e2cb41b 100644 +--- a/daemons/attrd/pacemaker-attrd.h ++++ b/daemons/attrd/pacemaker-attrd.h +@@ -42,8 +42,9 @@ + * 4 2.1.5 Multiple attributes can be updated in a single IPC + * message + * 5 2.1.5 Peers can request confirmation of a sent message ++ * 6 2.1.7 PCMK__ATTRD_CMD_PEER_REMOVE supports PCMK__XA_REAP + */ +-#define ATTRD_PROTOCOL_VERSION "5" ++#define ATTRD_PROTOCOL_VERSION "6" + + #define ATTRD_SUPPORTS_MULTI_MESSAGE(x) ((x) >= 4) + #define ATTRD_SUPPORTS_CONFIRMATION(x) ((x) >= 5) +diff --git a/daemons/controld/controld_attrd.c b/daemons/controld/controld_attrd.c +index 923abb92d..958dc2f14 100644 +--- a/daemons/controld/controld_attrd.c ++++ b/daemons/controld/controld_attrd.c +@@ -117,7 +117,7 @@ update_attrd_remote_node_removed(const char *host, const char *user_name) + if (rc == pcmk_rc_ok) { + crm_trace("Asking attribute manager to purge Pacemaker Remote node %s", + host); +- rc = pcmk__attrd_api_purge(attrd_api, host); ++ rc = pcmk__attrd_api_purge(attrd_api, host, true); + } + if (rc != pcmk_rc_ok) { + crm_err("Could not purge Pacemaker Remote node %s " +diff --git a/include/crm/common/ipc_attrd_internal.h b/include/crm/common/ipc_attrd_internal.h +index b1b7584bd..39a55ad1d 100644 +--- a/include/crm/common/ipc_attrd_internal.h ++++ b/include/crm/common/ipc_attrd_internal.h +@@ -89,10 +89,11 @@ int pcmk__attrd_api_delete(pcmk_ipc_api_t *api, const char *node, const char *na + + /*! + * \internal +- * \brief Purge a node from pacemaker-attrd ++ * \brief Request removal of a node's transient attributes + * + * \param[in,out] api pacemaker-attrd IPC object +- * \param[in] node Node to remove ++ * \param[in] node Node whose attributes should be purged ++ * \param[in] reap If true, also request removal from node caches + * + * \note If \p api is NULL, a new temporary connection will be created + * just for this operation and destroyed afterwards. If \p api is +@@ -102,7 +103,7 @@ int pcmk__attrd_api_delete(pcmk_ipc_api_t *api, const char *node, const char *na + * + * \return Standard Pacemaker return code + */ +-int pcmk__attrd_api_purge(pcmk_ipc_api_t *api, const char *node); ++int pcmk__attrd_api_purge(pcmk_ipc_api_t *api, const char *node, bool reap); + + /*! + * \internal +diff --git a/include/crm_internal.h b/include/crm_internal.h +index 3bc8d096a..f800ab0cc 100644 +--- a/include/crm_internal.h ++++ b/include/crm_internal.h +@@ -92,6 +92,7 @@ + #define PCMK__XA_MODE "mode" + #define PCMK__XA_NODE_START_STATE "node_start_state" + #define PCMK__XA_PATH "path" ++#define PCMK__XA_REAP "reap" + #define PCMK__XA_SCHEMA "schema" + #define PCMK__XA_SCHEMAS "schemas" + #define PCMK__XA_TASK "task" +diff --git a/lib/common/ipc_attrd.c b/lib/common/ipc_attrd.c +index 9caaabec0..56cdb5aba 100644 +--- a/lib/common/ipc_attrd.c ++++ b/lib/common/ipc_attrd.c +@@ -277,7 +277,7 @@ pcmk__attrd_api_delete(pcmk_ipc_api_t *api, const char *node, const char *name, + } + + int +-pcmk__attrd_api_purge(pcmk_ipc_api_t *api, const char *node) ++pcmk__attrd_api_purge(pcmk_ipc_api_t *api, const char *node, bool reap) + { + int rc = pcmk_rc_ok; + xmlNode *request = NULL; +@@ -291,6 +291,7 @@ pcmk__attrd_api_purge(pcmk_ipc_api_t *api, const char *node) + request = create_attrd_op(NULL); + + crm_xml_add(request, PCMK__XA_TASK, PCMK__ATTRD_CMD_PEER_REMOVE); ++ pcmk__xe_set_bool_attr(request, PCMK__XA_REAP, reap); + pcmk__xe_add_node(request, node, 0); + + if (api == NULL) { +diff --git a/lib/common/ipc_client.c b/lib/common/ipc_client.c +index 0d3865095..5e64e2324 100644 +--- a/lib/common/ipc_client.c ++++ b/lib/common/ipc_client.c +@@ -759,6 +759,7 @@ create_purge_node_request(const pcmk_ipc_api_t *api, const char *node_name, + crm_xml_add(request, F_TYPE, T_ATTRD); + crm_xml_add(request, F_ORIG, crm_system_name); + crm_xml_add(request, PCMK__XA_TASK, PCMK__ATTRD_CMD_PEER_REMOVE); ++ pcmk__xe_set_bool_attr(request, PCMK__XA_REAP, true); + pcmk__xe_add_node(request, node_name, nodeid); + break; + +-- +2.41.0 + +From adc1d8ef587913e5505494e0205bd77a8e0a878e Mon Sep 17 00:00:00 2001 +From: Ken Gaillot +Date: Wed, 13 Dec 2023 09:24:28 -0600 +Subject: [PATCH 02/12] Log: attrd: improve messages for CIB wipe + +Also, expose attrd_erase_attrs() as attrd_cib_erase_transient_attrs() and make +it take the node name as an argument, for future reuse. +--- + daemons/attrd/attrd_cib.c | 60 ++++++++++++++++++++------------- + daemons/attrd/pacemaker-attrd.h | 1 + + 2 files changed, 37 insertions(+), 24 deletions(-) + +diff --git a/daemons/attrd/attrd_cib.c b/daemons/attrd/attrd_cib.c +index 80e5580d9..ca1c5b9e0 100644 +--- a/daemons/attrd/attrd_cib.c ++++ b/daemons/attrd/attrd_cib.c +@@ -153,41 +153,44 @@ static void + attrd_erase_cb(xmlNode *msg, int call_id, int rc, xmlNode *output, + void *user_data) + { +- do_crm_log_unlikely(((rc != pcmk_ok)? LOG_NOTICE : LOG_DEBUG), +- "Cleared transient attributes: %s " +- CRM_XS " xpath=%s rc=%d", +- pcmk_strerror(rc), (char *) user_data, rc); ++ const char *node = pcmk__s((const char *) user_data, "a node"); ++ ++ if (rc == pcmk_ok) { ++ crm_info("Cleared transient node attributes for %s from CIB", node); ++ } else { ++ crm_err("Unable to clear transient node attributes for %s from CIB: %s", ++ node, pcmk_strerror(rc)); ++ } + } + + #define XPATH_TRANSIENT "//node_state[@uname='%s']/" XML_TAG_TRANSIENT_NODEATTRS + + /*! + * \internal +- * \brief Wipe all transient attributes for this node from the CIB ++ * \brief Wipe all transient node attributes for a node from the CIB + * +- * Clear any previous transient node attributes from the CIB. This is +- * normally done by the DC's controller when this node leaves the cluster, but +- * this handles the case where the node restarted so quickly that the +- * cluster layer didn't notice. +- * +- * \todo If pacemaker-attrd respawns after crashing (see PCMK_ENV_RESPAWNED), +- * ideally we'd skip this and sync our attributes from the writer. +- * However, currently we reject any values for us that the writer has, in +- * attrd_peer_update(). ++ * \param[in] node Node to clear attributes for + */ +-static void +-attrd_erase_attrs(void) ++void ++attrd_cib_erase_transient_attrs(const char *node) + { + int call_id = 0; +- char *xpath = crm_strdup_printf(XPATH_TRANSIENT, attrd_cluster->uname); ++ char *xpath = NULL; ++ ++ CRM_CHECK(node != NULL, return); ++ ++ xpath = crm_strdup_printf(XPATH_TRANSIENT, node); + +- crm_info("Clearing transient attributes from CIB " CRM_XS " xpath=%s", +- xpath); ++ crm_debug("Clearing transient node attributes for %s from CIB using %s", ++ node, xpath); + + call_id = the_cib->cmds->remove(the_cib, xpath, NULL, cib_xpath); +- the_cib->cmds->register_callback_full(the_cib, call_id, 120, FALSE, xpath, +- "attrd_erase_cb", attrd_erase_cb, +- free); ++ free(xpath); ++ ++ // strdup() is just for logging here, so ignore failure ++ the_cib->cmds->register_callback_full(the_cib, call_id, 120, FALSE, ++ strdup(node), "attrd_erase_cb", ++ attrd_erase_cb, free); + } + + /*! +@@ -197,8 +200,17 @@ attrd_erase_attrs(void) + void + attrd_cib_init(void) + { +- // We have no attribute values in memory, wipe the CIB to match +- attrd_erase_attrs(); ++ /* We have no attribute values in memory, so wipe the CIB to match. This is ++ * normally done by the DC's controller when this node leaves the cluster, but ++ * this handles the case where the node restarted so quickly that the ++ * cluster layer didn't notice. ++ * ++ * \todo If pacemaker-attrd respawns after crashing (see PCMK_ENV_RESPAWNED), ++ * ideally we'd skip this and sync our attributes from the writer. ++ * However, currently we reject any values for us that the writer has, in ++ * attrd_peer_update(). ++ */ ++ attrd_cib_erase_transient_attrs(attrd_cluster->uname); + + // Set a trigger for reading the CIB (for the alerts section) + attrd_config_read = mainloop_add_trigger(G_PRIORITY_HIGH, attrd_read_options, NULL); +diff --git a/daemons/attrd/pacemaker-attrd.h b/daemons/attrd/pacemaker-attrd.h +index 70e2cb41b..62637d1d7 100644 +--- a/daemons/attrd/pacemaker-attrd.h ++++ b/daemons/attrd/pacemaker-attrd.h +@@ -66,6 +66,7 @@ void attrd_ipc_fini(void); + int attrd_cib_connect(int max_retry); + void attrd_cib_disconnect(void); + void attrd_cib_init(void); ++void attrd_cib_erase_transient_attrs(const char *node); + + bool attrd_value_needs_expansion(const char *value); + int attrd_expand_value(const char *value, const char *old_value); +-- +2.41.0 + +From 9be38897eaa683ad7920503d9c9fd7db7a20a8ec Mon Sep 17 00:00:00 2001 +From: Ken Gaillot +Date: Wed, 13 Dec 2023 11:20:07 -0600 +Subject: [PATCH 03/12] Refactor: attrd: convert value booleans to flags + +--- + daemons/attrd/attrd_attributes.c | 7 +++--- + daemons/attrd/attrd_corosync.c | 38 +++++++++++++++++--------------- + daemons/attrd/pacemaker-attrd.h | 21 ++++++++++++++++-- + 3 files changed, 42 insertions(+), 24 deletions(-) + +diff --git a/daemons/attrd/attrd_attributes.c b/daemons/attrd/attrd_attributes.c +index 388c181d7..8f32988be 100644 +--- a/daemons/attrd/attrd_attributes.c ++++ b/daemons/attrd/attrd_attributes.c +@@ -1,5 +1,5 @@ + /* +- * Copyright 2013-2022 the Pacemaker project contributors ++ * Copyright 2013-2023 the Pacemaker project contributors + * + * The version control history for this file may have further details. + * +@@ -143,7 +143,7 @@ attrd_add_value_xml(xmlNode *parent, const attribute_t *a, + crm_xml_add(xml, PCMK__XA_ATTR_UUID, a->uuid); + crm_xml_add(xml, PCMK__XA_ATTR_USER, a->user); + pcmk__xe_add_node(xml, v->nodename, v->nodeid); +- if (v->is_remote != 0) { ++ if (pcmk_is_set(v->flags, attrd_value_remote)) { + crm_xml_add_int(xml, PCMK__XA_ATTR_IS_REMOTE, 1); + } + crm_xml_add(xml, PCMK__XA_ATTR_VALUE, v->current); +@@ -166,8 +166,7 @@ attrd_clear_value_seen(void) + while (g_hash_table_iter_next(&aIter, NULL, (gpointer *) & a)) { + g_hash_table_iter_init(&vIter, a->values); + while (g_hash_table_iter_next(&vIter, NULL, (gpointer *) & v)) { +- v->seen = FALSE; +- crm_trace("Clear seen flag %s[%s] = %s.", a->id, v->nodename, v->current); ++ attrd_clear_value_flags(v, attrd_value_from_peer); + } + } + } +diff --git a/daemons/attrd/attrd_corosync.c b/daemons/attrd/attrd_corosync.c +index e6cd07f65..ca20bdc0f 100644 +--- a/daemons/attrd/attrd_corosync.c ++++ b/daemons/attrd/attrd_corosync.c +@@ -192,34 +192,35 @@ cache_remote_node(const char *node_name) + + /*! + * \internal +- * \brief Return host's hash table entry (creating one if needed) ++ * \brief Return a node's value from hash table (creating one if needed) + * +- * \param[in,out] values Hash table of values +- * \param[in] host Name of peer to look up +- * \param[in] xml XML describing the attribute ++ * \param[in,out] values Hash table of values ++ * \param[in] node_name Name of node to look up ++ * \param[in] xml XML describing the attribute + * + * \return Pointer to new or existing hash table entry + */ + static attribute_value_t * +-attrd_lookup_or_create_value(GHashTable *values, const char *host, ++attrd_lookup_or_create_value(GHashTable *values, const char *node_name, + const xmlNode *xml) + { +- attribute_value_t *v = g_hash_table_lookup(values, host); ++ attribute_value_t *v = g_hash_table_lookup(values, node_name); + int is_remote = 0; + +- crm_element_value_int(xml, PCMK__XA_ATTR_IS_REMOTE, &is_remote); +- if (is_remote) { +- cache_remote_node(host); +- } +- + if (v == NULL) { + v = calloc(1, sizeof(attribute_value_t)); + CRM_ASSERT(v != NULL); + +- pcmk__str_update(&v->nodename, host); +- v->is_remote = is_remote; ++ pcmk__str_update(&v->nodename, node_name); + g_hash_table_replace(values, v->nodename, v); + } ++ ++ crm_element_value_int(xml, PCMK__XA_ATTR_IS_REMOTE, &is_remote); ++ if (is_remote) { ++ attrd_set_value_flags(v, attrd_value_remote); ++ cache_remote_node(node_name); ++ } ++ + return(v); + } + +@@ -344,11 +345,11 @@ update_attr_on_host(attribute_t *a, const crm_node_t *peer, const xmlNode *xml, + } + } + +- /* Set the seen flag for attribute processing held only in the own node. */ +- v->seen = TRUE; ++ // This allows us to later detect local values that peer doesn't know about ++ attrd_set_value_flags(v, attrd_value_from_peer); + + /* If this is a cluster node whose node ID we are learning, remember it */ +- if ((v->nodeid == 0) && (v->is_remote == FALSE) ++ if ((v->nodeid == 0) && !pcmk_is_set(v->flags, attrd_value_remote) + && (crm_element_value_int(xml, PCMK__XA_ATTR_NODE_ID, + (int*)&v->nodeid) == 0) && (v->nodeid > 0)) { + record_peer_nodeid(v, host); +@@ -414,8 +415,9 @@ broadcast_unseen_local_values(void) + while (g_hash_table_iter_next(&aIter, NULL, (gpointer *) & a)) { + g_hash_table_iter_init(&vIter, a->values); + while (g_hash_table_iter_next(&vIter, NULL, (gpointer *) & v)) { +- if (!(v->seen) && pcmk__str_eq(v->nodename, attrd_cluster->uname, +- pcmk__str_casei)) { ++ if (!pcmk_is_set(v->flags, attrd_value_from_peer) ++ && pcmk__str_eq(v->nodename, attrd_cluster->uname, ++ pcmk__str_casei)) { + if (sync == NULL) { + sync = create_xml_node(NULL, __func__); + crm_xml_add(sync, PCMK__XA_TASK, PCMK__ATTRD_CMD_SYNC_RESPONSE); +diff --git a/daemons/attrd/pacemaker-attrd.h b/daemons/attrd/pacemaker-attrd.h +index 62637d1d7..738418857 100644 +--- a/daemons/attrd/pacemaker-attrd.h ++++ b/daemons/attrd/pacemaker-attrd.h +@@ -140,15 +140,32 @@ typedef struct attribute_s { + + } attribute_t; + ++enum attrd_value_flags { ++ attrd_value_none = 0U, ++ attrd_value_remote = (1U << 0), // Value is for Pacemaker Remote node ++ attrd_value_from_peer = (1U << 1), // Value is from peer sync response ++}; ++ + typedef struct attribute_value_s { + uint32_t nodeid; +- gboolean is_remote; + char *nodename; + char *current; + char *requested; +- gboolean seen; ++ uint32_t flags; // Group of attrd_value_flags + } attribute_value_t; + ++#define attrd_set_value_flags(attr_value, flags_to_set) do { \ ++ (attr_value)->flags = pcmk__set_flags_as(__func__, __LINE__, \ ++ LOG_TRACE, "Value for node", (attr_value)->nodename, \ ++ (attr_value)->flags, (flags_to_set), #flags_to_set); \ ++ } while (0) ++ ++#define attrd_clear_value_flags(attr_value, flags_to_clear) do { \ ++ (attr_value)->flags = pcmk__clear_flags_as(__func__, __LINE__, \ ++ LOG_TRACE, "Value for node", (attr_value)->nodename, \ ++ (attr_value)->flags, (flags_to_clear), #flags_to_clear); \ ++ } while (0) ++ + extern crm_cluster_t *attrd_cluster; + extern GHashTable *attributes; + extern GHashTable *peer_protocol_vers; +-- +2.41.0 + +From 922c79f4e39dc9501ff7c0136df8043081b771cb Mon Sep 17 00:00:00 2001 +From: Ken Gaillot +Date: Wed, 13 Dec 2023 16:51:39 -0600 +Subject: [PATCH 04/12] Log: attrd: improve logging of CIB write result + +When attrd requests a write-out of a changed attribute value, it saves the new +value in attribute_value_t:requested so it can be used in a log when the write +completes (which may occur after the value has already changed again, so we +can't log the current value at that time). + +Previously, the log call relied on libqb mapping a NULL pointer to "(null)". +To be safer, do that explicitly. + +Also, it previously erased "requested" after the write completed, even if the +write failed and would be reattempted. Leave the value alone in this case so +the result of the reattempt can be logged correctly. +--- + daemons/attrd/attrd_cib.c | 18 ++++++++---------- + 1 file changed, 8 insertions(+), 10 deletions(-) + +diff --git a/daemons/attrd/attrd_cib.c b/daemons/attrd/attrd_cib.c +index ca1c5b9e0..ae6564856 100644 +--- a/daemons/attrd/attrd_cib.c ++++ b/daemons/attrd/attrd_cib.c +@@ -274,11 +274,12 @@ attrd_cib_callback(xmlNode *msg, int call_id, int rc, xmlNode *output, void *use + + g_hash_table_iter_init(&iter, a->values); + while (g_hash_table_iter_next(&iter, (gpointer *) & peer, (gpointer *) & v)) { +- do_crm_log(level, "* %s[%s]=%s", a->id, peer, v->requested); +- free(v->requested); +- v->requested = NULL; +- if (rc != pcmk_ok) { +- a->changed = true; /* Attempt write out again */ ++ do_crm_log(level, "* %s[%s]=%s", ++ a->id, peer, pcmk__s(v->requested, "(null)")); ++ if (rc == pcmk_ok) { ++ pcmk__str_update(&(v->requested), NULL); ++ } else { ++ a->changed = true; // Reattempt write below if we are still writer + } + } + +@@ -605,11 +606,8 @@ write_attribute(attribute_t *a, bool ignore_delay) + /* Preservation of the attribute to transmit alert */ + set_alert_attribute_value(alert_attribute_value, v); + +- free(v->requested); +- v->requested = NULL; +- if (v->current) { +- v->requested = strdup(v->current); +- } ++ // Save this value so we can log it when write completes ++ pcmk__str_update(&(v->requested), v->current); + } + + if (private_updates) { +-- +2.41.0 + +From fa2830b1c4acf061faa40490620eb63c48a56a2b Mon Sep 17 00:00:00 2001 +From: Ken Gaillot +Date: Wed, 13 Dec 2023 17:01:01 -0600 +Subject: [PATCH 05/12] Low: libcrmcluster: avoid use-after-free in trace log + +--- + lib/cluster/membership.c | 16 ++++++++++++++-- + 1 file changed, 14 insertions(+), 2 deletions(-) + +diff --git a/lib/cluster/membership.c b/lib/cluster/membership.c +index f856ccaca..6958e65f2 100644 +--- a/lib/cluster/membership.c ++++ b/lib/cluster/membership.c +@@ -143,11 +143,23 @@ crm_remote_peer_get(const char *node_name) + return node; + } + ++/*! ++ * \brief Remove a node from the Pacemaker Remote node cache ++ * ++ * \param[in] node_name Name of node to remove from cache ++ * ++ * \note The caller must be careful not to use \p node_name after calling this ++ * function if it might be a pointer into the cache entry being removed. ++ */ + void + crm_remote_peer_cache_remove(const char *node_name) + { +- if (g_hash_table_remove(crm_remote_peer_cache, node_name)) { +- crm_trace("removed %s from remote peer cache", node_name); ++ /* Do a lookup first, because node_name could be a pointer within the entry ++ * being removed -- we can't log it *after* removing it. ++ */ ++ if (g_hash_table_lookup(crm_remote_peer_cache, node_name) != NULL) { ++ crm_trace("Removing %s from Pacemaker Remote node cache", node_name); ++ g_hash_table_remove(crm_remote_peer_cache, node_name); + } + } + +-- +2.41.0 + +From 14a7449a413f3f10eb80634c607386007d264475 Mon Sep 17 00:00:00 2001 +From: Ken Gaillot +Date: Thu, 14 Dec 2023 09:24:38 -0600 +Subject: [PATCH 06/12] Refactor: libcrmcluster,attrd: functionize removing + node from both caches + +This future-proofs against a potential use-after-free (not possible with +current code) and isolates cache management better. +--- + daemons/attrd/attrd_corosync.c | 3 +-- + include/crm/cluster/internal.h | 9 +++---- + lib/cluster/membership.c | 44 ++++++++++++++++++++++++++++++++++ + 3 files changed, 50 insertions(+), 6 deletions(-) + +diff --git a/daemons/attrd/attrd_corosync.c b/daemons/attrd/attrd_corosync.c +index ca20bdc0f..aa94a078e 100644 +--- a/daemons/attrd/attrd_corosync.c ++++ b/daemons/attrd/attrd_corosync.c +@@ -554,8 +554,7 @@ attrd_peer_remove(const char *host, bool uncache, const char *source) + } + + if (uncache) { +- crm_remote_peer_cache_remove(host); +- reap_crm_member(0, host); ++ pcmk__purge_node_from_cache(host, 0); + } + } + +diff --git a/include/crm/cluster/internal.h b/include/crm/cluster/internal.h +index e20ee4c59..c71069be2 100644 +--- a/include/crm/cluster/internal.h ++++ b/include/crm/cluster/internal.h +@@ -1,5 +1,5 @@ + /* +- * Copyright 2004-2021 the Pacemaker project contributors ++ * Copyright 2004-2023 the Pacemaker project contributors + * + * The version control history for this file may have further details. + * +@@ -7,8 +7,8 @@ + * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY. + */ + +-#ifndef CRM_CLUSTER_INTERNAL__H +-# define CRM_CLUSTER_INTERNAL__H ++#ifndef PCMK__CRM_CLUSTER_INTERNAL__H ++# define PCMK__CRM_CLUSTER_INTERNAL__H + + # include // uint32_t, uint64_t + # include +@@ -126,6 +126,7 @@ crm_node_t *pcmk__search_node_caches(unsigned int id, const char *uname, + uint32_t flags); + crm_node_t *pcmk__search_cluster_node_cache(unsigned int id, const char *uname, + const char *uuid); ++void pcmk__purge_node_from_cache(const char *node_name, uint32_t node_id); + + void pcmk__refresh_node_caches_from_cib(xmlNode *cib); + crm_node_t *pcmk__search_known_node_cache(unsigned int id, const char *uname, +@@ -136,4 +137,4 @@ crm_node_t *pcmk__get_peer(unsigned int id, const char *uname, + crm_node_t *pcmk__get_peer_full(unsigned int id, const char *uname, + const char *uuid, int flags); + +-#endif ++#endif // PCMK__CRM_CLUSTER_INTERNAL__H +diff --git a/lib/cluster/membership.c b/lib/cluster/membership.c +index 6958e65f2..173aaaa17 100644 +--- a/lib/cluster/membership.c ++++ b/lib/cluster/membership.c +@@ -341,6 +341,9 @@ crm_reap_dead_member(gpointer key, gpointer value, gpointer user_data) + * \param[in] name Uname of node to remove (or NULL to ignore) + * + * \return Number of cache entries removed ++ * ++ * \note The caller must be careful not to use \p name after calling this ++ * function if it might be a pointer into the cache entry being removed. + */ + guint + reap_crm_member(uint32_t id, const char *name) +@@ -564,6 +567,47 @@ pcmk__get_peer_full(unsigned int id, const char *uname, const char *uuid, + return node; + } + ++/*! ++ * \internal ++ * \brief Purge a node from cache (both cluster and Pacemaker Remote) ++ * ++ * \param[in] node_name If not NULL, purge only nodes with this name ++ * \param[in] node_id If not 0, purge cluster nodes only if they have this ID ++ * ++ * \note If \p node_name is NULL and \p node_id is 0, no nodes will be purged. ++ * If \p node_name is not NULL and \p node_id is not 0, Pacemaker Remote ++ * nodes that match \p node_name will be purged, and cluster nodes that ++ * match both \p node_name and \p node_id will be purged. ++ * \note The caller must be careful not to use \p node_name after calling this ++ * function if it might be a pointer into a cache entry being removed. ++ */ ++void ++pcmk__purge_node_from_cache(const char *node_name, uint32_t node_id) ++{ ++ char *node_name_copy = NULL; ++ ++ if ((node_name == NULL) && (node_id == 0U)) { ++ return; ++ } ++ ++ // Purge from Pacemaker Remote node cache ++ if ((node_name != NULL) ++ && (g_hash_table_lookup(crm_remote_peer_cache, node_name) != NULL)) { ++ /* node_name could be a pointer into the cache entry being purged, ++ * so reassign it to a copy before the original gets freed ++ */ ++ node_name_copy = strdup(node_name); ++ CRM_ASSERT(node_name_copy != NULL); ++ node_name = node_name_copy; ++ ++ crm_trace("Purging %s from Pacemaker Remote node cache", node_name); ++ g_hash_table_remove(crm_remote_peer_cache, node_name); ++ } ++ ++ reap_crm_member(node_id, node_name); ++ free(node_name_copy); ++} ++ + /*! + * \brief Get a node cache entry (cluster or Pacemaker Remote) + * +-- +2.41.0 + +From 8d552c1b582a95f9879b15e2dd991a7f995e7eca Mon Sep 17 00:00:00 2001 +From: Ken Gaillot +Date: Thu, 14 Dec 2023 09:51:37 -0600 +Subject: [PATCH 07/12] Fix: pacemaker-attrd,libcrmcluster: avoid + use-after-free when remote node in cluster node cache + +Previously, pacemaker-attrd removed any conflicting entry from the cluster node +cache before adding a node to the remote node cache. However, if the name used +was a pointer into the cluster node cache entry being freed, it would be reused +to create the remote node cache entry. + +This avoids that and also moves the functionality into libcrmcluster for better +isolation of cache management. It also corrects mistakenly setting errno to a +negative value. +--- + daemons/attrd/attrd_corosync.c | 26 ++------------------------ + lib/cluster/membership.c | 30 ++++++++++++++++++++++++++++-- + 2 files changed, 30 insertions(+), 26 deletions(-) + +diff --git a/daemons/attrd/attrd_corosync.c b/daemons/attrd/attrd_corosync.c +index aa94a078e..1d0f87f04 100644 +--- a/daemons/attrd/attrd_corosync.c ++++ b/daemons/attrd/attrd_corosync.c +@@ -166,28 +166,6 @@ broadcast_local_value(const attribute_t *a) + return v; + } + +-/*! +- * \internal +- * \brief Ensure a Pacemaker Remote node is in the correct peer cache +- * +- * \param[in] node_name Name of Pacemaker Remote node to check +- */ +-static void +-cache_remote_node(const char *node_name) +-{ +- /* If we previously assumed this node was an unseen cluster node, +- * remove its entry from the cluster peer cache. +- */ +- crm_node_t *dup = pcmk__search_cluster_node_cache(0, node_name, NULL); +- +- if (dup && (dup->uuid == NULL)) { +- reap_crm_member(0, node_name); +- } +- +- // Ensure node is in the remote peer cache +- CRM_ASSERT(crm_remote_peer_get(node_name) != NULL); +-} +- + #define state_text(state) pcmk__s((state), "in unknown state") + + /*! +@@ -218,7 +196,7 @@ attrd_lookup_or_create_value(GHashTable *values, const char *node_name, + crm_element_value_int(xml, PCMK__XA_ATTR_IS_REMOTE, &is_remote); + if (is_remote) { + attrd_set_value_flags(v, attrd_value_remote); +- cache_remote_node(node_name); ++ CRM_ASSERT(crm_remote_peer_get(node_name) != NULL); + } + + return(v); +@@ -273,7 +251,7 @@ attrd_peer_change_cb(enum crm_status_type kind, crm_node_t *peer, const void *da + + // Ensure remote nodes that come up are in the remote node cache + } else if (!gone && is_remote) { +- cache_remote_node(peer->uname); ++ CRM_ASSERT(crm_remote_peer_get(peer->uname) != NULL); + } + } + +diff --git a/lib/cluster/membership.c b/lib/cluster/membership.c +index 173aaaa17..a653617fa 100644 +--- a/lib/cluster/membership.c ++++ b/lib/cluster/membership.c +@@ -102,26 +102,50 @@ crm_remote_peer_cache_size(void) + * \note When creating a new entry, this will leave the node state undetermined, + * so the caller should also call pcmk__update_peer_state() if the state + * is known. ++ * \note Because this can add and remove cache entries, callers should not ++ * assume any previously obtained cache entry pointers remain valid. + */ + crm_node_t * + crm_remote_peer_get(const char *node_name) + { + crm_node_t *node; ++ char *node_name_copy = NULL; + + if (node_name == NULL) { +- errno = -EINVAL; ++ errno = EINVAL; + return NULL; + } + ++ /* It's theoretically possible that the node was added to the cluster peer ++ * cache before it was known to be a Pacemaker Remote node. Remove that ++ * entry unless it has a node ID, which means the name actually is ++ * associated with a cluster node. (@TODO return an error in that case?) ++ */ ++ node = pcmk__search_cluster_node_cache(0, node_name, NULL); ++ if ((node != NULL) && (node->uuid == NULL)) { ++ /* node_name could be a pointer into the cache entry being removed, so ++ * reassign it to a copy before the original gets freed ++ */ ++ node_name_copy = strdup(node_name); ++ if (node_name_copy == NULL) { ++ errno = ENOMEM; ++ return NULL; ++ } ++ node_name = node_name_copy; ++ reap_crm_member(0, node_name); ++ } ++ + /* Return existing cache entry if one exists */ + node = g_hash_table_lookup(crm_remote_peer_cache, node_name); + if (node) { ++ free(node_name_copy); + return node; + } + + /* Allocate a new entry */ + node = calloc(1, sizeof(crm_node_t)); + if (node == NULL) { ++ free(node_name_copy); + return NULL; + } + +@@ -130,7 +154,8 @@ crm_remote_peer_get(const char *node_name) + node->uuid = strdup(node_name); + if (node->uuid == NULL) { + free(node); +- errno = -ENOMEM; ++ errno = ENOMEM; ++ free(node_name_copy); + return NULL; + } + +@@ -140,6 +165,7 @@ crm_remote_peer_get(const char *node_name) + + /* Update the entry's uname, ensuring peer status callbacks are called */ + update_peer_uname(node, node_name); ++ free(node_name_copy); + return node; + } + +-- +2.41.0 + +From 17ac8f0409021cbcd3e03a1b70518ab7abd9b259 Mon Sep 17 00:00:00 2001 +From: Ken Gaillot +Date: Thu, 14 Dec 2023 10:03:05 -0600 +Subject: [PATCH 08/12] Refactor: attrd: remove dead code + +The peer change callback can't be called for a Pacemaker Remote node unless the +node is already in the remote node cache, so don't bother trying to add it. +Modifying the peer caches is forbidden in peer change callbacks anyway since it +could lead to use-after-free issues in libcrmcluster. +--- + daemons/attrd/attrd_corosync.c | 4 ---- + 1 file changed, 4 deletions(-) + +diff --git a/daemons/attrd/attrd_corosync.c b/daemons/attrd/attrd_corosync.c +index 1d0f87f04..eba734c3a 100644 +--- a/daemons/attrd/attrd_corosync.c ++++ b/daemons/attrd/attrd_corosync.c +@@ -248,10 +248,6 @@ attrd_peer_change_cb(enum crm_status_type kind, crm_node_t *peer, const void *da + attrd_remove_voter(peer); + attrd_remove_peer_protocol_ver(peer->uname); + attrd_do_not_expect_from_peer(peer->uname); +- +- // Ensure remote nodes that come up are in the remote node cache +- } else if (!gone && is_remote) { +- CRM_ASSERT(crm_remote_peer_get(peer->uname) != NULL); + } + } + +-- +2.41.0 + +From 221c4d697edc0481817c206ce8fdd878afd98ca1 Mon Sep 17 00:00:00 2001 +From: Ken Gaillot +Date: Thu, 14 Dec 2023 17:17:32 -0600 +Subject: [PATCH 09/12] Low: libcrmcommon: handle disconnected attrd API + connections consistently + +Drop send_attrd_request() in favor of using connect_and_send_attrd_request(), +since pcmk__connect_ipc() will return pcmk_rc_ok immediately if the API is +already connected. + +All the attribute manager IPC APIs attempted the connection if not already +connected except for pcmk__attrd_api_query(). Now that it uses +connect_and_send_attrd_request(), they are all consistent. +--- + lib/common/ipc_attrd.c | 28 +++++----------------------- + 1 file changed, 5 insertions(+), 23 deletions(-) + +diff --git a/lib/common/ipc_attrd.c b/lib/common/ipc_attrd.c +index 56cdb5aba..e36b42cbc 100644 +--- a/lib/common/ipc_attrd.c ++++ b/lib/common/ipc_attrd.c +@@ -190,12 +190,6 @@ connect_and_send_attrd_request(pcmk_ipc_api_t *api, const xmlNode *request) + return pcmk_rc_ok; + } + +-static int +-send_attrd_request(pcmk_ipc_api_t *api, const xmlNode *request) +-{ +- return pcmk__send_ipc_request(api, request); +-} +- + int + pcmk__attrd_api_clear_failures(pcmk_ipc_api_t *api, const char *node, + const char *resource, const char *operation, +@@ -229,11 +223,8 @@ pcmk__attrd_api_clear_failures(pcmk_ipc_api_t *api, const char *node, + rc = connect_and_send_attrd_request(api, request); + destroy_api(api); + +- } else if (!pcmk_ipc_is_connected(api)) { +- rc = connect_and_send_attrd_request(api, request); +- + } else { +- rc = send_attrd_request(api, request); ++ rc = connect_and_send_attrd_request(api, request); + } + + free_xml(request); +@@ -303,11 +294,8 @@ pcmk__attrd_api_purge(pcmk_ipc_api_t *api, const char *node, bool reap) + rc = connect_and_send_attrd_request(api, request); + destroy_api(api); + +- } else if (!pcmk_ipc_is_connected(api)) { +- rc = connect_and_send_attrd_request(api, request); +- + } else { +- rc = send_attrd_request(api, request); ++ rc = connect_and_send_attrd_request(api, request); + } + + free_xml(request); +@@ -346,7 +334,7 @@ pcmk__attrd_api_query(pcmk_ipc_api_t *api, const char *node, const char *name, + crm_xml_add(request, PCMK__XA_TASK, PCMK__ATTRD_CMD_QUERY); + pcmk__xe_add_node(request, node, 0); + +- rc = send_attrd_request(api, request); ++ rc = connect_and_send_attrd_request(api, request); + free_xml(request); + + if (node) { +@@ -386,11 +374,8 @@ pcmk__attrd_api_refresh(pcmk_ipc_api_t *api, const char *node) + rc = connect_and_send_attrd_request(api, request); + destroy_api(api); + +- } else if (!pcmk_ipc_is_connected(api)) { +- rc = connect_and_send_attrd_request(api, request); +- + } else { +- rc = send_attrd_request(api, request); ++ rc = connect_and_send_attrd_request(api, request); + } + + free_xml(request); +@@ -479,11 +464,8 @@ pcmk__attrd_api_update(pcmk_ipc_api_t *api, const char *node, const char *name, + rc = connect_and_send_attrd_request(api, request); + destroy_api(api); + +- } else if (!pcmk_ipc_is_connected(api)) { +- rc = connect_and_send_attrd_request(api, request); +- + } else { +- rc = send_attrd_request(api, request); ++ rc = connect_and_send_attrd_request(api, request); + } + + free_xml(request); +-- +2.41.0 + +From 85502a405c384fdf0331e43ec161910ee1d14973 Mon Sep 17 00:00:00 2001 +From: Ken Gaillot +Date: Thu, 14 Dec 2023 17:29:11 -0600 +Subject: [PATCH 10/12] Low: libcrmcommon: handle NULL attribute manager IPC + API connections consistently + +Previously, all attribute manager IPC APIs except pcmk__attrd_api_query() would +create a temporary connection if passed a NULL argument for one. Now, +connect_and_send_attrd_request() does this itself, reducing code duplication and +making the handling consistent across all APIs. +--- + lib/common/ipc_attrd.c | 116 +++++++++-------------------------------- + 1 file changed, 25 insertions(+), 91 deletions(-) + +diff --git a/lib/common/ipc_attrd.c b/lib/common/ipc_attrd.c +index e36b42cbc..68975c7b6 100644 +--- a/lib/common/ipc_attrd.c ++++ b/lib/common/ipc_attrd.c +@@ -148,46 +148,39 @@ create_attrd_op(const char *user_name) + return attrd_op; + } + +-static int +-create_api(pcmk_ipc_api_t **api) +-{ +- int rc = pcmk_new_ipc_api(api, pcmk_ipc_attrd); +- +- if (rc != pcmk_rc_ok) { +- crm_err("Could not connect to attrd: %s", pcmk_rc_str(rc)); +- } +- +- return rc; +-} +- +-static void +-destroy_api(pcmk_ipc_api_t *api) +-{ +- pcmk_disconnect_ipc(api); +- pcmk_free_ipc_api(api); +- api = NULL; +-} +- + static int + connect_and_send_attrd_request(pcmk_ipc_api_t *api, const xmlNode *request) + { + int rc = pcmk_rc_ok; ++ bool created_api = false; ++ ++ if (api == NULL) { ++ rc = pcmk_new_ipc_api(&api, pcmk_ipc_attrd); ++ if (rc != pcmk_rc_ok) { ++ crm_err("Could not connect to attribute manager: %s", ++ pcmk_rc_str(rc)); ++ return rc; ++ } ++ created_api = true; ++ } + + rc = pcmk__connect_ipc(api, pcmk_ipc_dispatch_sync, 5); + if (rc != pcmk_rc_ok) { + crm_err("Could not connect to %s: %s", + pcmk_ipc_name(api, true), pcmk_rc_str(rc)); +- return rc; +- } + +- rc = pcmk__send_ipc_request(api, request); +- if (rc != pcmk_rc_ok) { +- crm_err("Could not send request to %s: %s", +- pcmk_ipc_name(api, true), pcmk_rc_str(rc)); +- return rc; ++ } else { ++ rc = pcmk__send_ipc_request(api, request); ++ if (rc != pcmk_rc_ok) { ++ crm_err("Could not send request to %s: %s", ++ pcmk_ipc_name(api, true), pcmk_rc_str(rc)); ++ } + } + +- return pcmk_rc_ok; ++ if (created_api) { ++ pcmk_free_ipc_api(api); ++ } ++ return rc; + } + + int +@@ -214,18 +207,7 @@ pcmk__attrd_api_clear_failures(pcmk_ipc_api_t *api, const char *node, + crm_xml_add_int(request, PCMK__XA_ATTR_IS_REMOTE, + pcmk_is_set(options, pcmk__node_attr_remote)); + +- if (api == NULL) { +- rc = create_api(&api); +- if (rc != pcmk_rc_ok) { +- return rc; +- } +- +- rc = connect_and_send_attrd_request(api, request); +- destroy_api(api); +- +- } else { +- rc = connect_and_send_attrd_request(api, request); +- } ++ rc = connect_and_send_attrd_request(api, request); + + free_xml(request); + +@@ -285,18 +267,7 @@ pcmk__attrd_api_purge(pcmk_ipc_api_t *api, const char *node, bool reap) + pcmk__xe_set_bool_attr(request, PCMK__XA_REAP, reap); + pcmk__xe_add_node(request, node, 0); + +- if (api == NULL) { +- rc = create_api(&api); +- if (rc != pcmk_rc_ok) { +- return rc; +- } +- +- rc = connect_and_send_attrd_request(api, request); +- destroy_api(api); +- +- } else { +- rc = connect_and_send_attrd_request(api, request); +- } ++ rc = connect_and_send_attrd_request(api, request); + + free_xml(request); + +@@ -365,18 +336,7 @@ pcmk__attrd_api_refresh(pcmk_ipc_api_t *api, const char *node) + crm_xml_add(request, PCMK__XA_TASK, PCMK__ATTRD_CMD_REFRESH); + pcmk__xe_add_node(request, node, 0); + +- if (api == NULL) { +- rc = create_api(&api); +- if (rc != pcmk_rc_ok) { +- return rc; +- } +- +- rc = connect_and_send_attrd_request(api, request); +- destroy_api(api); +- +- } else { +- rc = connect_and_send_attrd_request(api, request); +- } ++ rc = connect_and_send_attrd_request(api, request); + + free_xml(request); + +@@ -455,18 +415,7 @@ pcmk__attrd_api_update(pcmk_ipc_api_t *api, const char *node, const char *name, + request = create_attrd_op(user_name); + populate_update_op(request, node, name, value, dampen, set, options); + +- if (api == NULL) { +- rc = create_api(&api); +- if (rc != pcmk_rc_ok) { +- return rc; +- } +- +- rc = connect_and_send_attrd_request(api, request); +- destroy_api(api); +- +- } else { +- rc = connect_and_send_attrd_request(api, request); +- } ++ rc = connect_and_send_attrd_request(api, request); + + free_xml(request); + +@@ -547,23 +496,8 @@ pcmk__attrd_api_update_list(pcmk_ipc_api_t *api, GList *attrs, const char *dampe + * request. Do that now, creating and destroying the API object if needed. + */ + if (pcmk__is_daemon) { +- bool created_api = false; +- +- if (api == NULL) { +- rc = create_api(&api); +- if (rc != pcmk_rc_ok) { +- return rc; +- } +- +- created_api = true; +- } +- + rc = connect_and_send_attrd_request(api, request); + free_xml(request); +- +- if (created_api) { +- destroy_api(api); +- } + } + + return rc; +-- +2.41.0 + +From 4b25e2e2cf52e6c772805309e1f3dd6bb7ce8fab Mon Sep 17 00:00:00 2001 +From: Ken Gaillot +Date: Thu, 14 Dec 2023 18:11:14 -0600 +Subject: [PATCH 11/12] Log: controld,libcrmcommon: improve attrd IPC API + messages + +Previously, connect_and_send_attrd_request() would log error messages for +failures, attrd IPC APIs would log debug messages with the result whether +success or failure, and then callers would log or output failures again. + +Now, connect_and_send_attrd_request() does not log, the attrd IPC APIs log a +debug message before sending the request, and the callers log or output +failures. +--- + daemons/controld/controld_attrd.c | 22 ++++----- + lib/common/ipc_attrd.c | 76 ++++++++++++------------------- + 2 files changed, 38 insertions(+), 60 deletions(-) + +diff --git a/daemons/controld/controld_attrd.c b/daemons/controld/controld_attrd.c +index 958dc2f14..24c1e7068 100644 +--- a/daemons/controld/controld_attrd.c ++++ b/daemons/controld/controld_attrd.c +@@ -1,5 +1,5 @@ + /* +- * Copyright 2006-2022 the Pacemaker project contributors ++ * Copyright 2006-2023 the Pacemaker project contributors + * + * The version control history for this file may have further details. + * +@@ -136,25 +136,23 @@ update_attrd_clear_failures(const char *host, const char *rsc, const char *op, + rc = pcmk_new_ipc_api(&attrd_api, pcmk_ipc_attrd); + } + if (rc == pcmk_rc_ok) { +- const char *op_desc = pcmk__s(op, "operations"); +- const char *interval_desc = "all"; + uint32_t attrd_opts = pcmk__node_attr_none; + +- if (op != NULL) { +- interval_desc = pcmk__s(interval_spec, "nonrecurring"); +- } + if (is_remote_node) { + pcmk__set_node_attr_flags(attrd_opts, pcmk__node_attr_remote); + } +- crm_info("Asking attribute manager to clear failure of %s %s for %s " +- "on %s node %s", interval_desc, op_desc, rsc, +- node_type(is_remote_node), host); + rc = pcmk__attrd_api_clear_failures(attrd_api, host, rsc, op, + interval_spec, NULL, attrd_opts); + } + if (rc != pcmk_rc_ok) { +- crm_err("Could not clear failure attributes for %s on %s node %s%s: %s " +- CRM_XS " rc=%d", pcmk__s(rsc, "all resources"), +- node_type(is_remote_node), host, when(), pcmk_rc_str(rc), rc); ++ const char *interval_desc = "all"; ++ ++ if (op != NULL) { ++ interval_desc = pcmk__s(interval_spec, "nonrecurring"); ++ } ++ crm_err("Could not clear failure of %s %s for %s on %s node %s%s: %s " ++ CRM_XS " rc=%d", interval_desc, pcmk__s(op, "operations"), ++ pcmk__s(rsc, "all resources"), node_type(is_remote_node), host, ++ when(), pcmk_rc_str(rc), rc); + } + } +diff --git a/lib/common/ipc_attrd.c b/lib/common/ipc_attrd.c +index 68975c7b6..3951bd3df 100644 +--- a/lib/common/ipc_attrd.c ++++ b/lib/common/ipc_attrd.c +@@ -157,24 +157,14 @@ connect_and_send_attrd_request(pcmk_ipc_api_t *api, const xmlNode *request) + if (api == NULL) { + rc = pcmk_new_ipc_api(&api, pcmk_ipc_attrd); + if (rc != pcmk_rc_ok) { +- crm_err("Could not connect to attribute manager: %s", +- pcmk_rc_str(rc)); + return rc; + } + created_api = true; + } + + rc = pcmk__connect_ipc(api, pcmk_ipc_dispatch_sync, 5); +- if (rc != pcmk_rc_ok) { +- crm_err("Could not connect to %s: %s", +- pcmk_ipc_name(api, true), pcmk_rc_str(rc)); +- +- } else { ++ if (rc == pcmk_rc_ok) { + rc = pcmk__send_ipc_request(api, request); +- if (rc != pcmk_rc_ok) { +- crm_err("Could not send request to %s: %s", +- pcmk_ipc_name(api, true), pcmk_rc_str(rc)); +- } + } + + if (created_api) { +@@ -199,6 +189,17 @@ pcmk__attrd_api_clear_failures(pcmk_ipc_api_t *api, const char *node, + node = target; + } + ++ if (operation) { ++ interval_desc = pcmk__s(interval_spec, "nonrecurring"); ++ op_desc = operation; ++ } else { ++ interval_desc = "all"; ++ op_desc = "operations"; ++ } ++ crm_debug("Asking %s to clear failure of %s %s for %s on %s", ++ pcmk_ipc_name(api, true), interval_desc, op_desc, ++ pcmk__s(resource, "all resources"), pcmk__s(node, "all nodes")); ++ + crm_xml_add(request, PCMK__XA_TASK, PCMK__ATTRD_CMD_CLEAR_FAILURE); + pcmk__xe_add_node(request, node, 0); + crm_xml_add(request, PCMK__XA_ATTR_RESOURCE, resource); +@@ -210,19 +211,6 @@ pcmk__attrd_api_clear_failures(pcmk_ipc_api_t *api, const char *node, + rc = connect_and_send_attrd_request(api, request); + + free_xml(request); +- +- if (operation) { +- interval_desc = interval_spec? interval_spec : "nonrecurring"; +- op_desc = operation; +- } else { +- interval_desc = "all"; +- op_desc = "operations"; +- } +- +- crm_debug("Asked pacemaker-attrd to clear failure of %s %s for %s on %s: %s (%d)", +- interval_desc, op_desc, (resource? resource : "all resources"), +- (node? node : "all nodes"), pcmk_rc_str(rc), rc); +- + return rc; + } + +@@ -254,13 +242,17 @@ pcmk__attrd_api_purge(pcmk_ipc_api_t *api, const char *node, bool reap) + { + int rc = pcmk_rc_ok; + xmlNode *request = NULL; +- const char *display_host = (node ? node : "localhost"); + const char *target = pcmk__node_attr_target(node); + + if (target != NULL) { + node = target; + } + ++ crm_debug("Asking %s to purge transient attributes%s for %s", ++ pcmk_ipc_name(api, true), ++ (reap? " and node cache entries" : ""), ++ pcmk__s(node, "local node")); ++ + request = create_attrd_op(NULL); + + crm_xml_add(request, PCMK__XA_TASK, PCMK__ATTRD_CMD_PEER_REMOVE); +@@ -270,10 +262,6 @@ pcmk__attrd_api_purge(pcmk_ipc_api_t *api, const char *node, bool reap) + rc = connect_and_send_attrd_request(api, request); + + free_xml(request); +- +- crm_debug("Asked pacemaker-attrd to purge %s: %s (%d)", +- display_host, pcmk_rc_str(rc), rc); +- + return rc; + } + +@@ -299,6 +287,10 @@ pcmk__attrd_api_query(pcmk_ipc_api_t *api, const char *node, const char *name, + } + } + ++ crm_debug("Querying %s for value of '%s'%s%s", ++ pcmk_ipc_name(api, true), name, ++ ((node == NULL)? "" : " on "), pcmk__s(node, "")); ++ + request = create_attrd_op(NULL); + + crm_xml_add(request, PCMK__XA_ATTR_NAME, name); +@@ -307,15 +299,6 @@ pcmk__attrd_api_query(pcmk_ipc_api_t *api, const char *node, const char *name, + + rc = connect_and_send_attrd_request(api, request); + free_xml(request); +- +- if (node) { +- crm_debug("Queried pacemaker-attrd for %s on %s: %s (%d)", +- name, node, pcmk_rc_str(rc), rc); +- } else { +- crm_debug("Queried pacemaker-attrd for %s: %s (%d)", +- name, pcmk_rc_str(rc), rc); +- } +- + return rc; + } + +@@ -324,13 +307,15 @@ pcmk__attrd_api_refresh(pcmk_ipc_api_t *api, const char *node) + { + int rc = pcmk_rc_ok; + xmlNode *request = NULL; +- const char *display_host = (node ? node : "localhost"); + const char *target = pcmk__node_attr_target(node); + + if (target != NULL) { + node = target; + } + ++ crm_debug("Asking %s to write all transient attributes for %s to CIB", ++ pcmk_ipc_name(api, true), pcmk__s(node, "local node")); ++ + request = create_attrd_op(NULL); + + crm_xml_add(request, PCMK__XA_TASK, PCMK__ATTRD_CMD_REFRESH); +@@ -339,10 +324,6 @@ pcmk__attrd_api_refresh(pcmk_ipc_api_t *api, const char *node) + rc = connect_and_send_attrd_request(api, request); + + free_xml(request); +- +- crm_debug("Asked pacemaker-attrd to refresh %s: %s (%d)", +- display_host, pcmk_rc_str(rc), rc); +- + return rc; + } + +@@ -399,7 +380,6 @@ pcmk__attrd_api_update(pcmk_ipc_api_t *api, const char *node, const char *name, + { + int rc = pcmk_rc_ok; + xmlNode *request = NULL; +- const char *display_host = (node ? node : "localhost"); + const char *target = NULL; + + if (name == NULL) { +@@ -412,16 +392,16 @@ pcmk__attrd_api_update(pcmk_ipc_api_t *api, const char *node, const char *name, + node = target; + } + ++ crm_debug("Asking %s to update '%s' to '%s' for %s", ++ pcmk_ipc_name(api, true), name, pcmk__s(value, "(null)"), ++ pcmk__s(node, "local node")); ++ + request = create_attrd_op(user_name); + populate_update_op(request, node, name, value, dampen, set, options); + + rc = connect_and_send_attrd_request(api, request); + + free_xml(request); +- +- crm_debug("Asked pacemaker-attrd to update %s on %s: %s (%d)", +- name, display_host, pcmk_rc_str(rc), rc); +- + return rc; + } + +-- +2.41.0 + +From e5d22ef2a6b130768bd59ab5b7d8cd1155bb02a5 Mon Sep 17 00:00:00 2001 +From: Ken Gaillot +Date: Thu, 14 Dec 2023 17:54:01 -0600 +Subject: [PATCH 12/12] Log: libcrmcommon: use log-friendly name in pacemakerd + IPC logs + +--- + lib/common/ipc_pacemakerd.c | 15 ++++++++------- + 1 file changed, 8 insertions(+), 7 deletions(-) + +diff --git a/lib/common/ipc_pacemakerd.c b/lib/common/ipc_pacemakerd.c +index 2f0370974..6d6f6d6bf 100644 +--- a/lib/common/ipc_pacemakerd.c ++++ b/lib/common/ipc_pacemakerd.c +@@ -210,15 +210,16 @@ dispatch(pcmk_ipc_api_t *api, xmlNode *reply) + value = crm_element_value(reply, F_CRM_MSG_TYPE); + if (pcmk__str_empty(value) + || !pcmk__str_eq(value, XML_ATTR_RESPONSE, pcmk__str_none)) { +- crm_info("Unrecognizable message from pacemakerd: " ++ crm_info("Unrecognizable message from %s: " + "message type '%s' not '" XML_ATTR_RESPONSE "'", +- pcmk__s(value, "")); ++ pcmk_ipc_name(api, true), pcmk__s(value, "")); + status = CRM_EX_PROTOCOL; + goto done; + } + + if (pcmk__str_empty(crm_element_value(reply, XML_ATTR_REFERENCE))) { +- crm_info("Unrecognizable message from pacemakerd: no reference"); ++ crm_info("Unrecognizable message from %s: no reference", ++ pcmk_ipc_name(api, true)); + status = CRM_EX_PROTOCOL; + goto done; + } +@@ -244,8 +245,8 @@ dispatch(pcmk_ipc_api_t *api, xmlNode *reply) + reply_data.reply_type = pcmk_pacemakerd_reply_shutdown; + reply_data.data.shutdown.status = atoi(crm_element_value(msg_data, XML_LRM_ATTR_OPSTATUS)); + } else { +- crm_info("Unrecognizable message from pacemakerd: " +- "unknown command '%s'", pcmk__s(value, "")); ++ crm_info("Unrecognizable message from %s: unknown command '%s'", ++ pcmk_ipc_name(api, true), pcmk__s(value, "")); + status = CRM_EX_PROTOCOL; + goto done; + } +@@ -292,8 +293,8 @@ do_pacemakerd_api_call(pcmk_ipc_api_t *api, const char *ipc_name, const char *ta + if (cmd) { + rc = pcmk__send_ipc_request(api, cmd); + if (rc != pcmk_rc_ok) { +- crm_debug("Couldn't send request to pacemakerd: %s rc=%d", +- pcmk_rc_str(rc), rc); ++ crm_debug("Couldn't send request to %s: %s rc=%d", ++ pcmk_ipc_name(api, true), pcmk_rc_str(rc), rc); + } + free_xml(cmd); + } else { +-- +2.41.0 + diff --git a/SOURCES/004-clone-rsc-display.patch b/SOURCES/004-clone-rsc-display.patch deleted file mode 100644 index b09a53a..0000000 --- a/SOURCES/004-clone-rsc-display.patch +++ /dev/null @@ -1,35 +0,0 @@ -From 770d417e28dc9527fec8b8a00caaba8825995454 Mon Sep 17 00:00:00 2001 -From: Grace Chin -Date: Wed, 19 Jul 2023 10:25:55 -0400 -Subject: [PATCH] Fix: tools: Fix a bug in clone resource description display - -Previously, descriptions of resources running on multiple -nodes were displayed despite --full not being used (with pcs -status) or --show-detail not being used (with crm_mon). - -For example, clone resources running on multiple nodes were -affected. - -Now, --full and --show-detail must be used in order for resource -descriptions to be displayed, regardless of the number of nodes -the resource is run on. - -see bz: 2106642 ---- - lib/pengine/pe_output.c | 3 +-- - 1 file changed, 1 insertion(+), 2 deletions(-) - -diff --git a/lib/pengine/pe_output.c b/lib/pengine/pe_output.c -index e0b43d997a..d1c9f6e226 100644 ---- a/lib/pengine/pe_output.c -+++ b/lib/pengine/pe_output.c -@@ -20,8 +20,7 @@ pe__resource_description(const pe_resource_t *rsc, uint32_t show_opts) - { - const char * desc = NULL; - // User-supplied description -- if (pcmk_any_flags_set(show_opts, pcmk_show_rsc_only|pcmk_show_description) -- || pcmk__list_of_multiple(rsc->running_on)) { -+ if (pcmk_any_flags_set(show_opts, pcmk_show_rsc_only|pcmk_show_description)) { - desc = crm_element_value(rsc->xml, XML_ATTR_DESC); - } - return desc; diff --git a/SOURCES/005-attrd-cache-2.patch b/SOURCES/005-attrd-cache-2.patch new file mode 100644 index 0000000..c1174e1 --- /dev/null +++ b/SOURCES/005-attrd-cache-2.patch @@ -0,0 +1,2786 @@ +From 843ef27542aac43ed7789b15255dd4f30004f0d1 Mon Sep 17 00:00:00 2001 +From: Ken Gaillot +Date: Mon, 18 Dec 2023 10:08:30 -0600 +Subject: [PATCH 1/9] Fix: attrd: write Pacemaker Remote node attributes even + if not in cache + +Previously, we required a node to be in one of the node caches in order to +write out its attributes. However for Pacemaker Remote nodes, we only need the +node name to do the write, and we already have that even if it's not cached. +--- + daemons/attrd/attrd_cib.c | 55 +++++++++++++++++++++------------------ + 1 file changed, 30 insertions(+), 25 deletions(-) + +diff --git a/daemons/attrd/attrd_cib.c b/daemons/attrd/attrd_cib.c +index ae65648..b22137a 100644 +--- a/daemons/attrd/attrd_cib.c ++++ b/daemons/attrd/attrd_cib.c +@@ -20,6 +20,7 @@ + #include + #include + #include ++#include // pcmk__get_peer_full() + + #include "pacemaker-attrd.h" + +@@ -556,20 +557,26 @@ write_attribute(attribute_t *a, bool ignore_delay) + /* Iterate over each peer value of this attribute */ + g_hash_table_iter_init(&iter, a->values); + while (g_hash_table_iter_next(&iter, NULL, (gpointer *) &v)) { +- crm_node_t *peer = crm_get_peer_full(v->nodeid, v->nodename, +- CRM_GET_PEER_ANY); ++ const char *uuid = NULL; + +- /* If the value's peer info does not correspond to a peer, ignore it */ +- if (peer == NULL) { +- crm_notice("Cannot update %s[%s]=%s because peer not known", +- a->id, v->nodename, v->current); +- continue; +- } ++ if (pcmk_is_set(v->flags, attrd_value_remote)) { ++ /* If this is a Pacemaker Remote node, the node's UUID is the same ++ * as its name, which we already have. ++ */ ++ uuid = v->nodename; + +- /* If we're just learning the peer's node id, remember it */ +- if (peer->id && (v->nodeid == 0)) { +- crm_trace("Learned ID %u for node %s", peer->id, v->nodename); +- v->nodeid = peer->id; ++ } else { ++ // This will create a cluster node cache entry if none exists ++ crm_node_t *peer = pcmk__get_peer_full(v->nodeid, v->nodename, NULL, ++ CRM_GET_PEER_ANY); ++ ++ uuid = peer->uuid; ++ ++ // Remember peer's node ID if we're just now learning it ++ if ((peer->id != 0) && (v->nodeid == 0)) { ++ crm_trace("Learned ID %u for node %s", peer->id, v->nodename); ++ v->nodeid = peer->id; ++ } + } + + /* If this is a private attribute, no update needs to be sent */ +@@ -578,29 +585,27 @@ write_attribute(attribute_t *a, bool ignore_delay) + continue; + } + +- /* If the peer is found, but its uuid is unknown, defer write */ +- if (peer->uuid == NULL) { ++ // Defer write if this is a cluster node that's never been seen ++ if (uuid == NULL) { + a->unknown_peer_uuids = true; +- crm_notice("Cannot update %s[%s]=%s because peer UUID not known " +- "(will retry if learned)", ++ crm_notice("Cannot update %s[%s]='%s' now because node's UUID is " ++ "unknown (will retry if learned)", + a->id, v->nodename, v->current); + continue; + } + + // Update this value as part of the CIB transaction we're building +- rc = add_attr_update(a, v->current, peer->uuid); ++ rc = add_attr_update(a, v->current, uuid); + if (rc != pcmk_rc_ok) { +- crm_err("Failed to update %s[%s]=%s (peer known as %s, UUID %s, " +- "ID %" PRIu32 "/%" PRIu32 "): %s", +- a->id, v->nodename, v->current, peer->uname, peer->uuid, +- peer->id, v->nodeid, pcmk_rc_str(rc)); ++ crm_err("Failed to update %s[%s]='%s': %s " ++ CRM_XS " node uuid=%s id=%" PRIu32, ++ a->id, v->nodename, v->current, pcmk_rc_str(rc), ++ uuid, v->nodeid); + continue; + } + +- crm_debug("Updating %s[%s]=%s (peer known as %s, UUID %s, ID " +- "%" PRIu32 "/%" PRIu32 ")", +- a->id, v->nodename, v->current, +- peer->uname, peer->uuid, peer->id, v->nodeid); ++ crm_debug("Updating %s[%s]=%s (node uuid=%s id=%" PRIu32 ")", ++ a->id, v->nodename, v->current, uuid, v->nodeid); + cib_updates++; + + /* Preservation of the attribute to transmit alert */ +-- +2.31.1 + +From 724e6db9830475e212381430a30014ccda43c901 Mon Sep 17 00:00:00 2001 +From: Ken Gaillot +Date: Tue, 19 Dec 2023 14:59:54 -0600 +Subject: [PATCH 2/9] API: libcrmcluster: deprecate crm_get_peer_full() + +--- + daemons/attrd/attrd_messages.c | 1 + + daemons/controld/controld_execd.c | 2 +- + include/crm/cluster.h | 5 +---- + include/crm/cluster/compat.h | 5 ++++- + lib/cluster/membership.c | 21 ++++++--------------- + 5 files changed, 13 insertions(+), 21 deletions(-) + +diff --git a/daemons/attrd/attrd_messages.c b/daemons/attrd/attrd_messages.c +index ac32e18..53c70bd 100644 +--- a/daemons/attrd/attrd_messages.c ++++ b/daemons/attrd/attrd_messages.c +@@ -12,6 +12,7 @@ + #include + + #include ++#include // pcmk__get_peer() + #include + + #include "pacemaker-attrd.h" +diff --git a/daemons/controld/controld_execd.c b/daemons/controld/controld_execd.c +index 480d37d..381b0be 100644 +--- a/daemons/controld/controld_execd.c ++++ b/daemons/controld/controld_execd.c +@@ -581,7 +581,7 @@ controld_query_executor_state(void) + return NULL; + } + +- peer = crm_get_peer_full(0, lrm_state->node_name, CRM_GET_PEER_ANY); ++ peer = pcmk__get_peer_full(0, lrm_state->node_name, NULL, CRM_GET_PEER_ANY); + CRM_CHECK(peer != NULL, return NULL); + + xml_state = create_node_state_update(peer, +diff --git a/include/crm/cluster.h b/include/crm/cluster.h +index b61fd70..137684d 100644 +--- a/include/crm/cluster.h ++++ b/include/crm/cluster.h +@@ -1,5 +1,5 @@ + /* +- * Copyright 2004-2023 the Pacemaker project contributors ++ * Copyright 2004-2024 the Pacemaker project contributors + * + * The version control history for this file may have further details. + * +@@ -146,9 +146,6 @@ void crm_remote_peer_cache_refresh(xmlNode *cib); + crm_node_t *crm_remote_peer_get(const char *node_name); + void crm_remote_peer_cache_remove(const char *node_name); + +-/* allows filtering of remote and cluster nodes using crm_get_peer_flags */ +-crm_node_t *crm_get_peer_full(unsigned int id, const char *uname, int flags); +- + /* only searches cluster nodes */ + crm_node_t *crm_get_peer(unsigned int id, const char *uname); + +diff --git a/include/crm/cluster/compat.h b/include/crm/cluster/compat.h +index 89a03fd..fc68f27 100644 +--- a/include/crm/cluster/compat.h ++++ b/include/crm/cluster/compat.h +@@ -1,5 +1,5 @@ + /* +- * Copyright 2004-2023 the Pacemaker project contributors ++ * Copyright 2004-2024 the Pacemaker project contributors + * + * The version control history for this file may have further details. + * +@@ -26,6 +26,9 @@ extern "C" { + * release. + */ + ++// \deprecated Do not use Pacemaker for cluster node cacheing ++crm_node_t *crm_get_peer_full(unsigned int id, const char *uname, int flags); ++ + // \deprecated Use stonith_api_kick() from libstonithd instead + int crm_terminate_member(int nodeid, const char *uname, void *unused); + +diff --git a/lib/cluster/membership.c b/lib/cluster/membership.c +index a653617..52db840 100644 +--- a/lib/cluster/membership.c ++++ b/lib/cluster/membership.c +@@ -634,21 +634,6 @@ pcmk__purge_node_from_cache(const char *node_name, uint32_t node_id) + free(node_name_copy); + } + +-/*! +- * \brief Get a node cache entry (cluster or Pacemaker Remote) +- * +- * \param[in] id If not 0, cluster node ID to search for +- * \param[in] uname If not NULL, node name to search for +- * \param[in] flags Bitmask of enum crm_get_peer_flags +- * +- * \return (Possibly newly created) node cache entry +- */ +-crm_node_t * +-crm_get_peer_full(unsigned int id, const char *uname, int flags) +-{ +- return pcmk__get_peer_full(id, uname, NULL, flags); +-} +- + /*! + * \internal + * \brief Search cluster node cache +@@ -1444,5 +1429,11 @@ crm_terminate_member_no_mainloop(int nodeid, const char *uname, int *connection) + return stonith_api_kick(nodeid, uname, 120, TRUE); + } + ++crm_node_t * ++crm_get_peer_full(unsigned int id, const char *uname, int flags) ++{ ++ return pcmk__get_peer_full(id, uname, NULL, flags); ++} ++ + // LCOV_EXCL_STOP + // End deprecated API +-- +2.31.1 + +From 8a263fa254a62b07f3b591844e7eacd5cdd0538f Mon Sep 17 00:00:00 2001 +From: Ken Gaillot +Date: Tue, 19 Dec 2023 15:07:47 -0600 +Subject: [PATCH 3/9] API: libcrmcluster: deprecate crm_get_peer() + +Use pcmk__get_peer() internally +--- + daemons/attrd/attrd_corosync.c | 8 +++----- + daemons/attrd/attrd_messages.c | 6 +++--- + daemons/based/based_callbacks.c | 5 +++-- + daemons/based/based_messages.c | 7 ++++--- + daemons/controld/controld_corosync.c | 2 +- + daemons/controld/controld_election.c | 3 ++- + daemons/controld/controld_execd.c | 2 +- + daemons/controld/controld_fencing.c | 2 +- + daemons/controld/controld_join_client.c | 6 +++--- + daemons/controld/controld_join_dc.c | 10 +++++----- + daemons/controld/controld_messages.c | 2 +- + daemons/controld/controld_remote_ra.c | 2 +- + daemons/controld/controld_te_actions.c | 8 +++++--- + daemons/controld/controld_te_events.c | 3 ++- + daemons/controld/controld_utils.c | 2 +- + daemons/fenced/fenced_commands.c | 8 ++++---- + daemons/fenced/fenced_remote.c | 8 +++++--- + include/crm/cluster.h | 3 --- + include/crm/cluster/compat.h | 3 +++ + lib/cluster/corosync.c | 8 ++++---- + lib/cluster/cpg.c | 8 ++++---- + lib/cluster/election.c | 6 +++--- + lib/cluster/membership.c | 21 ++++++--------------- + 23 files changed, 65 insertions(+), 68 deletions(-) + +diff --git a/daemons/attrd/attrd_corosync.c b/daemons/attrd/attrd_corosync.c +index eba734c..3b2880b 100644 +--- a/daemons/attrd/attrd_corosync.c ++++ b/daemons/attrd/attrd_corosync.c +@@ -119,9 +119,7 @@ attrd_cpg_dispatch(cpg_handle_t handle, + if (xml == NULL) { + crm_err("Bad message of class %d received from %s[%u]: '%.120s'", kind, from, nodeid, data); + } else { +- crm_node_t *peer = crm_get_peer(nodeid, from); +- +- attrd_peer_message(peer, xml); ++ attrd_peer_message(pcmk__get_peer(nodeid, from, NULL), xml); + } + + free_xml(xml); +@@ -254,7 +252,7 @@ attrd_peer_change_cb(enum crm_status_type kind, crm_node_t *peer, const void *da + static void + record_peer_nodeid(attribute_value_t *v, const char *host) + { +- crm_node_t *known_peer = crm_get_peer(v->nodeid, host); ++ crm_node_t *known_peer = pcmk__get_peer(v->nodeid, host, NULL); + + crm_trace("Learned %s has node id %s", known_peer->uname, known_peer->uuid); + if (attrd_election_won()) { +@@ -439,7 +437,7 @@ attrd_peer_clear_failure(pcmk__request_t *request) + GHashTableIter iter; + regex_t regex; + +- crm_node_t *peer = crm_get_peer(0, request->peer); ++ crm_node_t *peer = pcmk__get_peer(0, request->peer, NULL); + + if (attrd_failure_regex(®ex, rsc, op, interval_ms) != pcmk_ok) { + crm_info("Ignoring invalid request to clear failures for %s", +diff --git a/daemons/attrd/attrd_messages.c b/daemons/attrd/attrd_messages.c +index 53c70bd..5536207 100644 +--- a/daemons/attrd/attrd_messages.c ++++ b/daemons/attrd/attrd_messages.c +@@ -177,7 +177,7 @@ static xmlNode * + handle_sync_request(pcmk__request_t *request) + { + if (request->peer != NULL) { +- crm_node_t *peer = crm_get_peer(0, request->peer); ++ crm_node_t *peer = pcmk__get_peer(0, request->peer, NULL); + + attrd_peer_sync(peer, request->xml); + pcmk__set_result(&request->result, CRM_EX_OK, PCMK_EXEC_DONE, NULL); +@@ -194,7 +194,7 @@ handle_sync_response_request(pcmk__request_t *request) + return handle_unknown_request(request); + } else { + if (request->peer != NULL) { +- crm_node_t *peer = crm_get_peer(0, request->peer); ++ crm_node_t *peer = pcmk__get_peer(0, request->peer, NULL); + bool peer_won = attrd_check_for_new_writer(peer, request->xml); + + if (!pcmk__str_eq(peer->uname, attrd_cluster->uname, pcmk__str_casei)) { +@@ -212,7 +212,7 @@ handle_update_request(pcmk__request_t *request) + { + if (request->peer != NULL) { + const char *host = crm_element_value(request->xml, PCMK__XA_ATTR_NODE_NAME); +- crm_node_t *peer = crm_get_peer(0, request->peer); ++ crm_node_t *peer = pcmk__get_peer(0, request->peer, NULL); + + attrd_peer_update(peer, request->xml, host, false); + pcmk__set_result(&request->result, CRM_EX_OK, PCMK_EXEC_DONE, NULL); +diff --git a/daemons/based/based_callbacks.c b/daemons/based/based_callbacks.c +index 4fac222..02f3425 100644 +--- a/daemons/based/based_callbacks.c ++++ b/daemons/based/based_callbacks.c +@@ -928,7 +928,7 @@ forward_request(xmlNode *request) + + crm_xml_add(request, F_CIB_DELEGATED, OUR_NODENAME); + +- send_cluster_message(((host != NULL)? crm_get_peer(0, host) : NULL), ++ send_cluster_message(((host != NULL)? pcmk__get_peer(0, host, NULL) : NULL), + crm_msg_cib, request, FALSE); + + // Return the request to its original state +@@ -986,7 +986,8 @@ send_peer_reply(xmlNode * msg, xmlNode * result_diff, const char *originator, gb + /* send reply via HA to originating node */ + crm_trace("Sending request result to %s only", originator); + crm_xml_add(msg, F_CIB_ISREPLY, originator); +- return send_cluster_message(crm_get_peer(0, originator), crm_msg_cib, msg, FALSE); ++ return send_cluster_message(pcmk__get_peer(0, originator, NULL), ++ crm_msg_cib, msg, FALSE); + } + + return FALSE; +diff --git a/daemons/based/based_messages.c b/daemons/based/based_messages.c +index a87d9ac..08521e4 100644 +--- a/daemons/based/based_messages.c ++++ b/daemons/based/based_messages.c +@@ -127,7 +127,8 @@ send_sync_request(const char *host) + crm_xml_add(sync_me, F_CIB_DELEGATED, + stand_alone? "localhost" : crm_cluster->uname); + +- send_cluster_message(host ? crm_get_peer(0, host) : NULL, crm_msg_cib, sync_me, FALSE); ++ send_cluster_message((host == NULL)? NULL : pcmk__get_peer(0, host, NULL), ++ crm_msg_cib, sync_me, FALSE); + free_xml(sync_me); + } + +@@ -443,8 +444,8 @@ sync_our_cib(xmlNode * request, gboolean all) + + add_message_xml(replace_request, F_CIB_CALLDATA, the_cib); + +- if (send_cluster_message +- (all ? NULL : crm_get_peer(0, host), crm_msg_cib, replace_request, FALSE) == FALSE) { ++ if (!send_cluster_message(all? NULL : pcmk__get_peer(0, host, NULL), ++ crm_msg_cib, replace_request, FALSE)) { + result = -ENOTCONN; + } + free_xml(replace_request); +diff --git a/daemons/controld/controld_corosync.c b/daemons/controld/controld_corosync.c +index b69e821..c2953b5 100644 +--- a/daemons/controld/controld_corosync.c ++++ b/daemons/controld/controld_corosync.c +@@ -49,7 +49,7 @@ crmd_cs_dispatch(cpg_handle_t handle, const struct cpg_name *groupName, + crm_xml_add(xml, F_ORIG, from); + /* crm_xml_add_int(xml, F_SEQ, wrapper->id); Fake? */ + +- peer = crm_get_peer(0, from); ++ peer = pcmk__get_peer(0, from, NULL); + if (!pcmk_is_set(peer->processes, crm_proc_cpg)) { + /* If we can still talk to our peer process on that node, + * then it must be part of the corosync membership +diff --git a/daemons/controld/controld_election.c b/daemons/controld/controld_election.c +index 70ffecc..6e22067 100644 +--- a/daemons/controld/controld_election.c ++++ b/daemons/controld/controld_election.c +@@ -265,7 +265,8 @@ do_dc_release(long long action, + crm_info("DC role released"); + if (pcmk_is_set(controld_globals.fsa_input_register, R_SHUTDOWN)) { + xmlNode *update = NULL; +- crm_node_t *node = crm_get_peer(0, controld_globals.our_nodename); ++ crm_node_t *node = pcmk__get_peer(0, controld_globals.our_nodename, ++ NULL); + + pcmk__update_peer_expected(__func__, node, CRMD_JOINSTATE_DOWN); + update = create_node_state_update(node, node_update_expected, NULL, +diff --git a/daemons/controld/controld_execd.c b/daemons/controld/controld_execd.c +index 381b0be..45b6b58 100644 +--- a/daemons/controld/controld_execd.c ++++ b/daemons/controld/controld_execd.c +@@ -1752,7 +1752,7 @@ controld_ack_event_directly(const char *to_host, const char *to_sys, + to_sys = CRM_SYSTEM_TENGINE; + } + +- peer = crm_get_peer(0, controld_globals.our_nodename); ++ peer = pcmk__get_peer(0, controld_globals.our_nodename, NULL); + update = create_node_state_update(peer, node_update_none, NULL, + __func__); + +diff --git a/daemons/controld/controld_fencing.c b/daemons/controld/controld_fencing.c +index 9557d9e..6c0ee09 100644 +--- a/daemons/controld/controld_fencing.c ++++ b/daemons/controld/controld_fencing.c +@@ -374,7 +374,7 @@ execute_stonith_cleanup(void) + + for (iter = stonith_cleanup_list; iter != NULL; iter = iter->next) { + char *target = iter->data; +- crm_node_t *target_node = crm_get_peer(0, target); ++ crm_node_t *target_node = pcmk__get_peer(0, target, NULL); + const char *uuid = crm_peer_uuid(target_node); + + crm_notice("Marking %s, target of a previous stonith action, as clean", target); +diff --git a/daemons/controld/controld_join_client.c b/daemons/controld/controld_join_client.c +index 805ecbd..2b5267d 100644 +--- a/daemons/controld/controld_join_client.c ++++ b/daemons/controld/controld_join_client.c +@@ -35,7 +35,7 @@ update_dc_expected(const xmlNode *msg) + { + if ((controld_globals.dc_name != NULL) + && pcmk__xe_attr_is_true(msg, F_CRM_DC_LEAVING)) { +- crm_node_t *dc_node = crm_get_peer(0, controld_globals.dc_name); ++ crm_node_t *dc_node = pcmk__get_peer(0, controld_globals.dc_name, NULL); + + pcmk__update_peer_expected(__func__, dc_node, CRMD_JOINSTATE_DOWN); + } +@@ -177,7 +177,7 @@ join_query_callback(xmlNode * msg, int call_id, int rc, xmlNode * output, void * + + crm_xml_add(reply, F_CRM_JOIN_ID, join_id); + crm_xml_add(reply, XML_ATTR_CRM_VERSION, CRM_FEATURE_SET); +- send_cluster_message(crm_get_peer(0, controld_globals.dc_name), ++ send_cluster_message(pcmk__get_peer(0, controld_globals.dc_name, NULL), + crm_msg_crmd, reply, TRUE); + free_xml(reply); + } +@@ -333,7 +333,7 @@ do_cl_join_finalize_respond(long long action, + } + } + +- send_cluster_message(crm_get_peer(0, controld_globals.dc_name), ++ send_cluster_message(pcmk__get_peer(0, controld_globals.dc_name, NULL), + crm_msg_crmd, reply, TRUE); + free_xml(reply); + +diff --git a/daemons/controld/controld_join_dc.c b/daemons/controld/controld_join_dc.c +index 2fe6710..45e1eba 100644 +--- a/daemons/controld/controld_join_dc.c ++++ b/daemons/controld/controld_join_dc.c +@@ -318,7 +318,7 @@ do_dc_join_offer_one(long long action, + crm_err("Can't make join-%d offer to unknown node", current_join_id); + return; + } +- member = crm_get_peer(0, join_to); ++ member = pcmk__get_peer(0, join_to, NULL); + + /* It is possible that a node will have been sick or starting up when the + * original offer was made. However, it will either re-announce itself in +@@ -332,7 +332,7 @@ do_dc_join_offer_one(long long action, + * well, to ensure the correct value for max_generation_from. + */ + if (strcasecmp(join_to, controld_globals.our_nodename) != 0) { +- member = crm_get_peer(0, controld_globals.our_nodename); ++ member = pcmk__get_peer(0, controld_globals.our_nodename, NULL); + join_make_offer(NULL, member, NULL); + } + +@@ -396,7 +396,7 @@ do_dc_join_filter_offer(long long action, + crm_err("Ignoring invalid join request without node name"); + return; + } +- join_node = crm_get_peer(0, join_from); ++ join_node = pcmk__get_peer(0, join_from, NULL); + + crm_element_value_int(join_ack->msg, F_CRM_JOIN_ID, &join_id); + if (join_id != current_join_id) { +@@ -732,7 +732,7 @@ do_dc_join_ack(long long action, + goto done; + } + +- peer = crm_get_peer(0, join_from); ++ peer = pcmk__get_peer(0, join_from, NULL); + if (peer->join != crm_join_finalized) { + crm_info("Ignoring out-of-sequence join-%d confirmation from %s " + "(currently %s not %s)", +@@ -866,7 +866,7 @@ finalize_join_for(gpointer key, gpointer value, gpointer user_data) + return; + } + +- join_node = crm_get_peer(0, join_to); ++ join_node = pcmk__get_peer(0, join_to, NULL); + if (!crm_is_peer_active(join_node)) { + /* + * NACK'ing nodes that the membership layer doesn't know about yet +diff --git a/daemons/controld/controld_messages.c b/daemons/controld/controld_messages.c +index 39f3c7a..8d3cef7 100644 +--- a/daemons/controld/controld_messages.c ++++ b/daemons/controld/controld_messages.c +@@ -458,7 +458,7 @@ relay_message(xmlNode * msg, gboolean originated_locally) + ref, pcmk__s(host_to, "broadcast")); + crm_log_xml_trace(msg, "relayed"); + if (!broadcast) { +- node_to = crm_get_peer(0, host_to); ++ node_to = pcmk__get_peer(0, host_to, NULL); + } + send_cluster_message(node_to, dest, msg, TRUE); + return TRUE; +diff --git a/daemons/controld/controld_remote_ra.c b/daemons/controld/controld_remote_ra.c +index d692ef6..a9c398d 100644 +--- a/daemons/controld/controld_remote_ra.c ++++ b/daemons/controld/controld_remote_ra.c +@@ -206,7 +206,7 @@ should_purge_attributes(crm_node_t *node) + /* Get the node that was hosting the remote connection resource from the + * peer cache. That's the one we really care about here. + */ +- conn_node = crm_get_peer(0, node->conn_host); ++ conn_node = pcmk__get_peer(0, node->conn_host, NULL); + if (conn_node == NULL) { + return purge; + } +diff --git a/daemons/controld/controld_te_actions.c b/daemons/controld/controld_te_actions.c +index fe6b744..e76174b 100644 +--- a/daemons/controld/controld_te_actions.c ++++ b/daemons/controld/controld_te_actions.c +@@ -158,7 +158,7 @@ execute_cluster_action(pcmk__graph_t *graph, pcmk__graph_action_t *action) + return pcmk_rc_ok; + + } else if (pcmk__str_eq(task, PCMK_ACTION_DO_SHUTDOWN, pcmk__str_none)) { +- crm_node_t *peer = crm_get_peer(0, router_node); ++ crm_node_t *peer = pcmk__get_peer(0, router_node, NULL); + + pcmk__update_peer_expected(__func__, peer, CRMD_JOINSTATE_DOWN); + } +@@ -170,7 +170,8 @@ execute_cluster_action(pcmk__graph_t *graph, pcmk__graph_action_t *action) + controld_globals.te_uuid); + crm_xml_add(cmd, XML_ATTR_TRANSITION_KEY, counter); + +- rc = send_cluster_message(crm_get_peer(0, router_node), crm_msg_crmd, cmd, TRUE); ++ rc = send_cluster_message(pcmk__get_peer(0, router_node, NULL), ++ crm_msg_crmd, cmd, TRUE); + free(counter); + free_xml(cmd); + +@@ -421,7 +422,8 @@ execute_rsc_action(pcmk__graph_t *graph, pcmk__graph_action_t *action) + I_NULL, &msg); + + } else { +- rc = send_cluster_message(crm_get_peer(0, router_node), crm_msg_lrmd, cmd, TRUE); ++ rc = send_cluster_message(pcmk__get_peer(0, router_node, NULL), ++ crm_msg_lrmd, cmd, TRUE); + } + + free(counter); +diff --git a/daemons/controld/controld_te_events.c b/daemons/controld/controld_te_events.c +index 28977c0..c8cceed 100644 +--- a/daemons/controld/controld_te_events.c ++++ b/daemons/controld/controld_te_events.c +@@ -119,7 +119,8 @@ fail_incompletable_actions(pcmk__graph_t *graph, const char *down_node) + target_uuid = crm_element_value(action->xml, XML_LRM_ATTR_TARGET_UUID); + router = crm_element_value(action->xml, XML_LRM_ATTR_ROUTER_NODE); + if (router) { +- crm_node_t *node = crm_get_peer(0, router); ++ crm_node_t *node = pcmk__get_peer(0, router, NULL); ++ + if (node) { + router_uuid = node->uuid; + } +diff --git a/daemons/controld/controld_utils.c b/daemons/controld/controld_utils.c +index 9b306ee..55790c0 100644 +--- a/daemons/controld/controld_utils.c ++++ b/daemons/controld/controld_utils.c +@@ -734,7 +734,7 @@ update_dc(xmlNode * msg) + /* do nothing */ + + } else if (controld_globals.dc_name != NULL) { +- crm_node_t *dc_node = crm_get_peer(0, controld_globals.dc_name); ++ crm_node_t *dc_node = pcmk__get_peer(0, controld_globals.dc_name, NULL); + + crm_info("Set DC to %s (%s)", + controld_globals.dc_name, +diff --git a/daemons/fenced/fenced_commands.c b/daemons/fenced/fenced_commands.c +index 7a62ed6..28f08dd 100644 +--- a/daemons/fenced/fenced_commands.c ++++ b/daemons/fenced/fenced_commands.c +@@ -645,7 +645,7 @@ schedule_stonith_command(async_command_t * cmd, stonith_device_t * device) + } + + if (device->include_nodeid && (cmd->target != NULL)) { +- crm_node_t *node = crm_get_peer(0, cmd->target); ++ crm_node_t *node = pcmk__get_peer(0, cmd->target, NULL); + + cmd->target_nodeid = node->id; + } +@@ -2402,8 +2402,8 @@ stonith_send_reply(const xmlNode *reply, int call_options, + if (remote_peer == NULL) { + do_local_reply(reply, client, call_options); + } else { +- send_cluster_message(crm_get_peer(0, remote_peer), crm_msg_stonith_ng, +- reply, FALSE); ++ send_cluster_message(pcmk__get_peer(0, remote_peer, NULL), ++ crm_msg_stonith_ng, reply, FALSE); + } + } + +@@ -3371,7 +3371,7 @@ handle_fence_request(pcmk__request_t *request) + crm_xml_add(request->xml, F_STONITH_CLIENTID, + request->ipc_client->id); + crm_xml_add(request->xml, F_STONITH_REMOTE_OP_ID, op->id); +- send_cluster_message(crm_get_peer(0, alternate_host), ++ send_cluster_message(pcmk__get_peer(0, alternate_host, NULL), + crm_msg_stonith_ng, request->xml, FALSE); + pcmk__set_result(&request->result, CRM_EX_OK, PCMK_EXEC_PENDING, + NULL); +diff --git a/daemons/fenced/fenced_remote.c b/daemons/fenced/fenced_remote.c +index 843b3d4..3c176c8 100644 +--- a/daemons/fenced/fenced_remote.c ++++ b/daemons/fenced/fenced_remote.c +@@ -1030,7 +1030,7 @@ merge_duplicates(remote_fencing_op_t *op) + op->id, other->id, other->target); + continue; + } +- if (!fencing_peer_active(crm_get_peer(0, other->originator))) { ++ if (!fencing_peer_active(pcmk__get_peer(0, other->originator, NULL))) { + crm_notice("Failing action '%s' targeting %s originating from " + "client %s@%s: Originator is dead " CRM_XS " id=%.8s", + other->action, other->target, other->client_name, +@@ -1663,7 +1663,8 @@ report_timeout_period(remote_fencing_op_t * op, int op_timeout) + crm_xml_add(update, F_STONITH_CALLID, call_id); + crm_xml_add_int(update, F_STONITH_TIMEOUT, op_timeout); + +- send_cluster_message(crm_get_peer(0, client_node), crm_msg_stonith_ng, update, FALSE); ++ send_cluster_message(pcmk__get_peer(0, client_node, NULL), ++ crm_msg_stonith_ng, update, FALSE); + + free_xml(update); + +@@ -1916,7 +1917,8 @@ request_peer_fencing(remote_fencing_op_t *op, peer_device_info_t *peer) + op->op_timer_one = g_timeout_add((1000 * timeout_one), remote_op_timeout_one, op); + } + +- send_cluster_message(crm_get_peer(0, peer->host), crm_msg_stonith_ng, remote_op, FALSE); ++ send_cluster_message(pcmk__get_peer(0, peer->host, NULL), ++ crm_msg_stonith_ng, remote_op, FALSE); + peer->tried = TRUE; + free_xml(remote_op); + return; +diff --git a/include/crm/cluster.h b/include/crm/cluster.h +index 137684d..302b807 100644 +--- a/include/crm/cluster.h ++++ b/include/crm/cluster.h +@@ -146,9 +146,6 @@ void crm_remote_peer_cache_refresh(xmlNode *cib); + crm_node_t *crm_remote_peer_get(const char *node_name); + void crm_remote_peer_cache_remove(const char *node_name); + +-/* only searches cluster nodes */ +-crm_node_t *crm_get_peer(unsigned int id, const char *uname); +- + guint crm_active_peers(void); + gboolean crm_is_peer_active(const crm_node_t * node); + guint reap_crm_member(uint32_t id, const char *name); +diff --git a/include/crm/cluster/compat.h b/include/crm/cluster/compat.h +index fc68f27..e853fd8 100644 +--- a/include/crm/cluster/compat.h ++++ b/include/crm/cluster/compat.h +@@ -26,6 +26,9 @@ extern "C" { + * release. + */ + ++// \deprecated Do not use Pacemaker for cluster node cacheing ++crm_node_t *crm_get_peer(unsigned int id, const char *uname); ++ + // \deprecated Do not use Pacemaker for cluster node cacheing + crm_node_t *crm_get_peer_full(unsigned int id, const char *uname, int flags); + +diff --git a/lib/cluster/corosync.c b/lib/cluster/corosync.c +index 08280ce..34a31fb 100644 +--- a/lib/cluster/corosync.c ++++ b/lib/cluster/corosync.c +@@ -309,12 +309,12 @@ quorum_notification_cb(quorum_handle_t handle, uint32_t quorate, + crm_debug("Member[%d] %u ", i, id); + + /* Get this node's peer cache entry (adding one if not already there) */ +- node = crm_get_peer(id, NULL); ++ node = pcmk__get_peer(id, NULL, NULL); + if (node->uname == NULL) { + char *name = pcmk__corosync_name(0, id); + + crm_info("Obtaining name for new node %u", id); +- node = crm_get_peer(id, name); ++ node = pcmk__get_peer(id, name, NULL); + free(name); + } + +@@ -480,7 +480,7 @@ pcmk__corosync_connect(crm_cluster_t *cluster) + } + + // Ensure local node always exists in peer cache +- peer = crm_get_peer(cluster->nodeid, cluster->uname); ++ peer = pcmk__get_peer(cluster->nodeid, cluster->uname, NULL); + cluster->uuid = pcmk__corosync_uuid(peer); + + return TRUE; +@@ -640,7 +640,7 @@ pcmk__corosync_add_nodes(xmlNode *xml_parent) + + if (nodeid > 0 || name != NULL) { + crm_trace("Initializing node[%d] %u = %s", lpc, nodeid, name); +- crm_get_peer(nodeid, name); ++ pcmk__get_peer(nodeid, name, NULL); + } + + if (nodeid > 0 && name != NULL) { +diff --git a/lib/cluster/cpg.c b/lib/cluster/cpg.c +index d1decc6..778368f 100644 +--- a/lib/cluster/cpg.c ++++ b/lib/cluster/cpg.c +@@ -465,7 +465,7 @@ pcmk_message_common_cs(cpg_handle_t handle, uint32_t nodeid, uint32_t pid, void + + msg->sender.id = nodeid; + if (msg->sender.size == 0) { +- crm_node_t *peer = crm_get_peer(nodeid, NULL); ++ crm_node_t *peer = pcmk__get_peer(nodeid, NULL, NULL); + + if (peer == NULL) { + crm_err("Peer with nodeid=%u is unknown", nodeid); +@@ -526,7 +526,7 @@ pcmk_message_common_cs(cpg_handle_t handle, uint32_t nodeid, uint32_t pid, void + } + + // Is this necessary? +- crm_get_peer(msg->sender.id, msg->sender.uname); ++ pcmk__get_peer(msg->sender.id, msg->sender.uname, NULL); + + crm_trace("Payload: %.200s", data); + return data; +@@ -720,7 +720,7 @@ pcmk_cpg_membership(cpg_handle_t handle, + } + + for (i = 0; i < member_list_entries; i++) { +- crm_node_t *peer = crm_get_peer(member_list[i].nodeid, NULL); ++ crm_node_t *peer = pcmk__get_peer(member_list[i].nodeid, NULL, NULL); + + if (member_list[i].nodeid == local_nodeid + && member_list[i].pid != getpid()) { +@@ -873,7 +873,7 @@ cluster_connect_cpg(crm_cluster_t *cluster) + return FALSE; + } + +- peer = crm_get_peer(id, NULL); ++ peer = pcmk__get_peer(id, NULL, NULL); + crm_update_peer_proc(__func__, peer, crm_proc_cpg, ONLINESTATUS); + return TRUE; + } +diff --git a/lib/cluster/election.c b/lib/cluster/election.c +index ebbae72..31867f2 100644 +--- a/lib/cluster/election.c ++++ b/lib/cluster/election.c +@@ -298,7 +298,7 @@ election_vote(election_t *e) + return; + } + +- our_node = crm_get_peer(0, e->uname); ++ our_node = pcmk__get_peer(0, e->uname, NULL); + if ((our_node == NULL) || (crm_is_peer_active(our_node) == FALSE)) { + crm_trace("Cannot vote in %s yet: local node not connected to cluster", + e->name); +@@ -547,8 +547,8 @@ election_count_vote(election_t *e, const xmlNode *message, bool can_win) + return election_error; + } + +- your_node = crm_get_peer(0, vote.from); +- our_node = crm_get_peer(0, e->uname); ++ your_node = pcmk__get_peer(0, vote.from, NULL); ++ our_node = pcmk__get_peer(0, e->uname, NULL); + we_are_owner = (our_node != NULL) + && pcmk__str_eq(our_node->uuid, vote.election_owner, + pcmk__str_none); +diff --git a/lib/cluster/membership.c b/lib/cluster/membership.c +index 52db840..41e0fa3 100644 +--- a/lib/cluster/membership.c ++++ b/lib/cluster/membership.c +@@ -868,21 +868,6 @@ pcmk__get_peer(unsigned int id, const char *uname, const char *uuid) + return node; + } + +-/*! +- * \brief Get a cluster node cache entry +- * +- * \param[in] id If not 0, cluster node ID to search for +- * \param[in] uname If not NULL, node name to search for +- * +- * \return (Possibly newly created) cluster node cache entry +- */ +-/* coverity[-alloc] Memory is referenced in one or both hashtables */ +-crm_node_t * +-crm_get_peer(unsigned int id, const char *uname) +-{ +- return pcmk__get_peer(id, uname, NULL); +-} +- + /*! + * \internal + * \brief Update a node's uname +@@ -1429,6 +1414,12 @@ crm_terminate_member_no_mainloop(int nodeid, const char *uname, int *connection) + return stonith_api_kick(nodeid, uname, 120, TRUE); + } + ++crm_node_t * ++crm_get_peer(unsigned int id, const char *uname) ++{ ++ return pcmk__get_peer(id, uname, NULL); ++} ++ + crm_node_t * + crm_get_peer_full(unsigned int id, const char *uname, int flags) + { +-- +2.31.1 + +From 39e949a698afb5b0177b05e7d81b403cbb27a57a Mon Sep 17 00:00:00 2001 +From: Ken Gaillot +Date: Tue, 19 Dec 2023 15:23:59 -0600 +Subject: [PATCH 4/9] Refactor: libcrmcluster: consolidate pcmk__get_peer() and + pcmk__get_peer_full() + +... into a new function pcmk__get_node() (since it can handle Pacemaker Remote +nodes, which aren't peers) +--- + daemons/attrd/attrd_cib.c | 6 +-- + daemons/attrd/attrd_corosync.c | 10 +++-- + daemons/attrd/attrd_messages.c | 11 ++++-- + daemons/based/based_callbacks.c | 10 +++-- + daemons/based/based_messages.c | 14 +++++-- + daemons/controld/controld_corosync.c | 2 +- + daemons/controld/controld_election.c | 4 +- + daemons/controld/controld_execd.c | 5 ++- + daemons/controld/controld_fencing.c | 5 ++- + daemons/controld/controld_join_client.c | 9 +++-- + daemons/controld/controld_join_dc.c | 11 +++--- + daemons/controld/controld_messages.c | 3 +- + daemons/controld/controld_remote_ra.c | 2 +- + daemons/controld/controld_te_actions.c | 9 +++-- + daemons/controld/controld_te_events.c | 3 +- + daemons/controld/controld_utils.c | 3 +- + daemons/fenced/fenced_commands.c | 9 +++-- + daemons/fenced/fenced_remote.c | 9 +++-- + include/crm/cluster/internal.h | 8 ++-- + lib/cluster/corosync.c | 9 +++-- + lib/cluster/cpg.c | 13 ++++--- + lib/cluster/election.c | 6 +-- + lib/cluster/membership.c | 52 ++++++++----------------- + 23 files changed, 116 insertions(+), 97 deletions(-) + +diff --git a/daemons/attrd/attrd_cib.c b/daemons/attrd/attrd_cib.c +index b22137a..7018a32 100644 +--- a/daemons/attrd/attrd_cib.c ++++ b/daemons/attrd/attrd_cib.c +@@ -20,7 +20,7 @@ + #include + #include + #include +-#include // pcmk__get_peer_full() ++#include // pcmk__get_node() + + #include "pacemaker-attrd.h" + +@@ -567,8 +567,8 @@ write_attribute(attribute_t *a, bool ignore_delay) + + } else { + // This will create a cluster node cache entry if none exists +- crm_node_t *peer = pcmk__get_peer_full(v->nodeid, v->nodename, NULL, +- CRM_GET_PEER_ANY); ++ crm_node_t *peer = pcmk__get_node(v->nodeid, v->nodename, NULL, ++ CRM_GET_PEER_ANY); + + uuid = peer->uuid; + +diff --git a/daemons/attrd/attrd_corosync.c b/daemons/attrd/attrd_corosync.c +index 3b2880b..c9e11e6 100644 +--- a/daemons/attrd/attrd_corosync.c ++++ b/daemons/attrd/attrd_corosync.c +@@ -119,7 +119,9 @@ attrd_cpg_dispatch(cpg_handle_t handle, + if (xml == NULL) { + crm_err("Bad message of class %d received from %s[%u]: '%.120s'", kind, from, nodeid, data); + } else { +- attrd_peer_message(pcmk__get_peer(nodeid, from, NULL), xml); ++ attrd_peer_message(pcmk__get_node(nodeid, from, NULL, ++ CRM_GET_PEER_CLUSTER), ++ xml); + } + + free_xml(xml); +@@ -252,7 +254,8 @@ attrd_peer_change_cb(enum crm_status_type kind, crm_node_t *peer, const void *da + static void + record_peer_nodeid(attribute_value_t *v, const char *host) + { +- crm_node_t *known_peer = pcmk__get_peer(v->nodeid, host, NULL); ++ crm_node_t *known_peer = pcmk__get_node(v->nodeid, host, NULL, ++ CRM_GET_PEER_CLUSTER); + + crm_trace("Learned %s has node id %s", known_peer->uname, known_peer->uuid); + if (attrd_election_won()) { +@@ -437,7 +440,8 @@ attrd_peer_clear_failure(pcmk__request_t *request) + GHashTableIter iter; + regex_t regex; + +- crm_node_t *peer = pcmk__get_peer(0, request->peer, NULL); ++ crm_node_t *peer = pcmk__get_node(0, request->peer, NULL, ++ CRM_GET_PEER_CLUSTER); + + if (attrd_failure_regex(®ex, rsc, op, interval_ms) != pcmk_ok) { + crm_info("Ignoring invalid request to clear failures for %s", +diff --git a/daemons/attrd/attrd_messages.c b/daemons/attrd/attrd_messages.c +index 5536207..c6c1b9a 100644 +--- a/daemons/attrd/attrd_messages.c ++++ b/daemons/attrd/attrd_messages.c +@@ -12,7 +12,7 @@ + #include + + #include +-#include // pcmk__get_peer() ++#include // pcmk__get_node() + #include + + #include "pacemaker-attrd.h" +@@ -177,7 +177,8 @@ static xmlNode * + handle_sync_request(pcmk__request_t *request) + { + if (request->peer != NULL) { +- crm_node_t *peer = pcmk__get_peer(0, request->peer, NULL); ++ crm_node_t *peer = pcmk__get_node(0, request->peer, NULL, ++ CRM_GET_PEER_CLUSTER); + + attrd_peer_sync(peer, request->xml); + pcmk__set_result(&request->result, CRM_EX_OK, PCMK_EXEC_DONE, NULL); +@@ -194,7 +195,8 @@ handle_sync_response_request(pcmk__request_t *request) + return handle_unknown_request(request); + } else { + if (request->peer != NULL) { +- crm_node_t *peer = pcmk__get_peer(0, request->peer, NULL); ++ crm_node_t *peer = pcmk__get_node(0, request->peer, NULL, ++ CRM_GET_PEER_CLUSTER); + bool peer_won = attrd_check_for_new_writer(peer, request->xml); + + if (!pcmk__str_eq(peer->uname, attrd_cluster->uname, pcmk__str_casei)) { +@@ -212,7 +214,8 @@ handle_update_request(pcmk__request_t *request) + { + if (request->peer != NULL) { + const char *host = crm_element_value(request->xml, PCMK__XA_ATTR_NODE_NAME); +- crm_node_t *peer = pcmk__get_peer(0, request->peer, NULL); ++ crm_node_t *peer = pcmk__get_node(0, request->peer, NULL, ++ CRM_GET_PEER_CLUSTER); + + attrd_peer_update(peer, request->xml, host, false); + pcmk__set_result(&request->result, CRM_EX_OK, PCMK_EXEC_DONE, NULL); +diff --git a/daemons/based/based_callbacks.c b/daemons/based/based_callbacks.c +index 02f3425..b1f3b4b 100644 +--- a/daemons/based/based_callbacks.c ++++ b/daemons/based/based_callbacks.c +@@ -910,6 +910,7 @@ forward_request(xmlNode *request) + const char *originator = crm_element_value(request, F_ORIG); + const char *client_name = crm_element_value(request, F_CIB_CLIENTNAME); + const char *call_id = crm_element_value(request, F_CIB_CALLID); ++ crm_node_t *peer = NULL; + + int log_level = LOG_INFO; + +@@ -928,8 +929,10 @@ forward_request(xmlNode *request) + + crm_xml_add(request, F_CIB_DELEGATED, OUR_NODENAME); + +- send_cluster_message(((host != NULL)? pcmk__get_peer(0, host, NULL) : NULL), +- crm_msg_cib, request, FALSE); ++ if (host != NULL) { ++ peer = pcmk__get_node(0, host, NULL, CRM_GET_PEER_CLUSTER); ++ } ++ send_cluster_message(peer, crm_msg_cib, request, FALSE); + + // Return the request to its original state + xml_remove_prop(request, F_CIB_DELEGATED); +@@ -986,7 +989,8 @@ send_peer_reply(xmlNode * msg, xmlNode * result_diff, const char *originator, gb + /* send reply via HA to originating node */ + crm_trace("Sending request result to %s only", originator); + crm_xml_add(msg, F_CIB_ISREPLY, originator); +- return send_cluster_message(pcmk__get_peer(0, originator, NULL), ++ return send_cluster_message(pcmk__get_node(0, originator, NULL, ++ CRM_GET_PEER_CLUSTER), + crm_msg_cib, msg, FALSE); + } + +diff --git a/daemons/based/based_messages.c b/daemons/based/based_messages.c +index 08521e4..ff1a6aa 100644 +--- a/daemons/based/based_messages.c ++++ b/daemons/based/based_messages.c +@@ -118,6 +118,7 @@ void + send_sync_request(const char *host) + { + xmlNode *sync_me = create_xml_node(NULL, "sync-me"); ++ crm_node_t *peer = NULL; + + crm_info("Requesting re-sync from %s", (host? host : "all peers")); + sync_in_progress = 1; +@@ -127,8 +128,10 @@ send_sync_request(const char *host) + crm_xml_add(sync_me, F_CIB_DELEGATED, + stand_alone? "localhost" : crm_cluster->uname); + +- send_cluster_message((host == NULL)? NULL : pcmk__get_peer(0, host, NULL), +- crm_msg_cib, sync_me, FALSE); ++ if (host != NULL) { ++ peer = pcmk__get_node(0, host, NULL, CRM_GET_PEER_CLUSTER); ++ } ++ send_cluster_message(peer, crm_msg_cib, sync_me, FALSE); + free_xml(sync_me); + } + +@@ -418,6 +421,7 @@ sync_our_cib(xmlNode * request, gboolean all) + const char *host = crm_element_value(request, F_ORIG); + const char *op = crm_element_value(request, F_CIB_OPERATION); + ++ crm_node_t *peer = NULL; + xmlNode *replace_request = NULL; + + CRM_CHECK(the_cib != NULL, return -EINVAL); +@@ -444,8 +448,10 @@ sync_our_cib(xmlNode * request, gboolean all) + + add_message_xml(replace_request, F_CIB_CALLDATA, the_cib); + +- if (!send_cluster_message(all? NULL : pcmk__get_peer(0, host, NULL), +- crm_msg_cib, replace_request, FALSE)) { ++ if (!all) { ++ peer = pcmk__get_node(0, host, NULL, CRM_GET_PEER_CLUSTER); ++ } ++ if (!send_cluster_message(peer, crm_msg_cib, replace_request, FALSE)) { + result = -ENOTCONN; + } + free_xml(replace_request); +diff --git a/daemons/controld/controld_corosync.c b/daemons/controld/controld_corosync.c +index c2953b5..fa1df6f 100644 +--- a/daemons/controld/controld_corosync.c ++++ b/daemons/controld/controld_corosync.c +@@ -49,7 +49,7 @@ crmd_cs_dispatch(cpg_handle_t handle, const struct cpg_name *groupName, + crm_xml_add(xml, F_ORIG, from); + /* crm_xml_add_int(xml, F_SEQ, wrapper->id); Fake? */ + +- peer = pcmk__get_peer(0, from, NULL); ++ peer = pcmk__get_node(0, from, NULL, CRM_GET_PEER_CLUSTER); + if (!pcmk_is_set(peer->processes, crm_proc_cpg)) { + /* If we can still talk to our peer process on that node, + * then it must be part of the corosync membership +diff --git a/daemons/controld/controld_election.c b/daemons/controld/controld_election.c +index 6e22067..734064d 100644 +--- a/daemons/controld/controld_election.c ++++ b/daemons/controld/controld_election.c +@@ -265,8 +265,8 @@ do_dc_release(long long action, + crm_info("DC role released"); + if (pcmk_is_set(controld_globals.fsa_input_register, R_SHUTDOWN)) { + xmlNode *update = NULL; +- crm_node_t *node = pcmk__get_peer(0, controld_globals.our_nodename, +- NULL); ++ crm_node_t *node = pcmk__get_node(0, controld_globals.our_nodename, ++ NULL, CRM_GET_PEER_CLUSTER); + + pcmk__update_peer_expected(__func__, node, CRMD_JOINSTATE_DOWN); + update = create_node_state_update(node, node_update_expected, NULL, +diff --git a/daemons/controld/controld_execd.c b/daemons/controld/controld_execd.c +index 45b6b58..df715aa 100644 +--- a/daemons/controld/controld_execd.c ++++ b/daemons/controld/controld_execd.c +@@ -581,7 +581,7 @@ controld_query_executor_state(void) + return NULL; + } + +- peer = pcmk__get_peer_full(0, lrm_state->node_name, NULL, CRM_GET_PEER_ANY); ++ peer = pcmk__get_node(0, lrm_state->node_name, NULL, CRM_GET_PEER_ANY); + CRM_CHECK(peer != NULL, return NULL); + + xml_state = create_node_state_update(peer, +@@ -1752,7 +1752,8 @@ controld_ack_event_directly(const char *to_host, const char *to_sys, + to_sys = CRM_SYSTEM_TENGINE; + } + +- peer = pcmk__get_peer(0, controld_globals.our_nodename, NULL); ++ peer = pcmk__get_node(0, controld_globals.our_nodename, NULL, ++ CRM_GET_PEER_CLUSTER); + update = create_node_state_update(peer, node_update_none, NULL, + __func__); + +diff --git a/daemons/controld/controld_fencing.c b/daemons/controld/controld_fencing.c +index 6c0ee09..60a7f9f 100644 +--- a/daemons/controld/controld_fencing.c ++++ b/daemons/controld/controld_fencing.c +@@ -222,7 +222,7 @@ send_stonith_update(pcmk__graph_action_t *action, const char *target, + * Try getting any existing node cache entry also by node uuid in case it + * doesn't have an uname yet. + */ +- peer = pcmk__get_peer_full(0, target, uuid, CRM_GET_PEER_ANY); ++ peer = pcmk__get_node(0, target, uuid, CRM_GET_PEER_ANY); + + CRM_CHECK(peer != NULL, return); + +@@ -374,7 +374,8 @@ execute_stonith_cleanup(void) + + for (iter = stonith_cleanup_list; iter != NULL; iter = iter->next) { + char *target = iter->data; +- crm_node_t *target_node = pcmk__get_peer(0, target, NULL); ++ crm_node_t *target_node = pcmk__get_node(0, target, NULL, ++ CRM_GET_PEER_CLUSTER); + const char *uuid = crm_peer_uuid(target_node); + + crm_notice("Marking %s, target of a previous stonith action, as clean", target); +diff --git a/daemons/controld/controld_join_client.c b/daemons/controld/controld_join_client.c +index 2b5267d..6f20ef2 100644 +--- a/daemons/controld/controld_join_client.c ++++ b/daemons/controld/controld_join_client.c +@@ -35,7 +35,8 @@ update_dc_expected(const xmlNode *msg) + { + if ((controld_globals.dc_name != NULL) + && pcmk__xe_attr_is_true(msg, F_CRM_DC_LEAVING)) { +- crm_node_t *dc_node = pcmk__get_peer(0, controld_globals.dc_name, NULL); ++ crm_node_t *dc_node = pcmk__get_node(0, controld_globals.dc_name, NULL, ++ CRM_GET_PEER_CLUSTER); + + pcmk__update_peer_expected(__func__, dc_node, CRMD_JOINSTATE_DOWN); + } +@@ -177,7 +178,8 @@ join_query_callback(xmlNode * msg, int call_id, int rc, xmlNode * output, void * + + crm_xml_add(reply, F_CRM_JOIN_ID, join_id); + crm_xml_add(reply, XML_ATTR_CRM_VERSION, CRM_FEATURE_SET); +- send_cluster_message(pcmk__get_peer(0, controld_globals.dc_name, NULL), ++ send_cluster_message(pcmk__get_node(0, controld_globals.dc_name, NULL, ++ CRM_GET_PEER_CLUSTER), + crm_msg_crmd, reply, TRUE); + free_xml(reply); + } +@@ -333,7 +335,8 @@ do_cl_join_finalize_respond(long long action, + } + } + +- send_cluster_message(pcmk__get_peer(0, controld_globals.dc_name, NULL), ++ send_cluster_message(pcmk__get_node(0, controld_globals.dc_name, NULL, ++ CRM_GET_PEER_CLUSTER), + crm_msg_crmd, reply, TRUE); + free_xml(reply); + +diff --git a/daemons/controld/controld_join_dc.c b/daemons/controld/controld_join_dc.c +index 45e1eba..064649f 100644 +--- a/daemons/controld/controld_join_dc.c ++++ b/daemons/controld/controld_join_dc.c +@@ -318,7 +318,7 @@ do_dc_join_offer_one(long long action, + crm_err("Can't make join-%d offer to unknown node", current_join_id); + return; + } +- member = pcmk__get_peer(0, join_to, NULL); ++ member = pcmk__get_node(0, join_to, NULL, CRM_GET_PEER_CLUSTER); + + /* It is possible that a node will have been sick or starting up when the + * original offer was made. However, it will either re-announce itself in +@@ -332,7 +332,8 @@ do_dc_join_offer_one(long long action, + * well, to ensure the correct value for max_generation_from. + */ + if (strcasecmp(join_to, controld_globals.our_nodename) != 0) { +- member = pcmk__get_peer(0, controld_globals.our_nodename, NULL); ++ member = pcmk__get_node(0, controld_globals.our_nodename, NULL, ++ CRM_GET_PEER_CLUSTER); + join_make_offer(NULL, member, NULL); + } + +@@ -396,7 +397,7 @@ do_dc_join_filter_offer(long long action, + crm_err("Ignoring invalid join request without node name"); + return; + } +- join_node = pcmk__get_peer(0, join_from, NULL); ++ join_node = pcmk__get_node(0, join_from, NULL, CRM_GET_PEER_CLUSTER); + + crm_element_value_int(join_ack->msg, F_CRM_JOIN_ID, &join_id); + if (join_id != current_join_id) { +@@ -732,7 +733,7 @@ do_dc_join_ack(long long action, + goto done; + } + +- peer = pcmk__get_peer(0, join_from, NULL); ++ peer = pcmk__get_node(0, join_from, NULL, CRM_GET_PEER_CLUSTER); + if (peer->join != crm_join_finalized) { + crm_info("Ignoring out-of-sequence join-%d confirmation from %s " + "(currently %s not %s)", +@@ -866,7 +867,7 @@ finalize_join_for(gpointer key, gpointer value, gpointer user_data) + return; + } + +- join_node = pcmk__get_peer(0, join_to, NULL); ++ join_node = pcmk__get_node(0, join_to, NULL, CRM_GET_PEER_CLUSTER); + if (!crm_is_peer_active(join_node)) { + /* + * NACK'ing nodes that the membership layer doesn't know about yet +diff --git a/daemons/controld/controld_messages.c b/daemons/controld/controld_messages.c +index 8d3cef7..71f5680 100644 +--- a/daemons/controld/controld_messages.c ++++ b/daemons/controld/controld_messages.c +@@ -458,7 +458,8 @@ relay_message(xmlNode * msg, gboolean originated_locally) + ref, pcmk__s(host_to, "broadcast")); + crm_log_xml_trace(msg, "relayed"); + if (!broadcast) { +- node_to = pcmk__get_peer(0, host_to, NULL); ++ node_to = pcmk__get_node(0, host_to, NULL, ++ CRM_GET_PEER_CLUSTER); + } + send_cluster_message(node_to, dest, msg, TRUE); + return TRUE; +diff --git a/daemons/controld/controld_remote_ra.c b/daemons/controld/controld_remote_ra.c +index a9c398d..9c4bb58 100644 +--- a/daemons/controld/controld_remote_ra.c ++++ b/daemons/controld/controld_remote_ra.c +@@ -206,7 +206,7 @@ should_purge_attributes(crm_node_t *node) + /* Get the node that was hosting the remote connection resource from the + * peer cache. That's the one we really care about here. + */ +- conn_node = pcmk__get_peer(0, node->conn_host, NULL); ++ conn_node = pcmk__get_node(0, node->conn_host, NULL, CRM_GET_PEER_CLUSTER); + if (conn_node == NULL) { + return purge; + } +diff --git a/daemons/controld/controld_te_actions.c b/daemons/controld/controld_te_actions.c +index e76174b..01ba4a0 100644 +--- a/daemons/controld/controld_te_actions.c ++++ b/daemons/controld/controld_te_actions.c +@@ -158,7 +158,8 @@ execute_cluster_action(pcmk__graph_t *graph, pcmk__graph_action_t *action) + return pcmk_rc_ok; + + } else if (pcmk__str_eq(task, PCMK_ACTION_DO_SHUTDOWN, pcmk__str_none)) { +- crm_node_t *peer = pcmk__get_peer(0, router_node, NULL); ++ crm_node_t *peer = pcmk__get_node(0, router_node, NULL, ++ CRM_GET_PEER_CLUSTER); + + pcmk__update_peer_expected(__func__, peer, CRMD_JOINSTATE_DOWN); + } +@@ -170,7 +171,8 @@ execute_cluster_action(pcmk__graph_t *graph, pcmk__graph_action_t *action) + controld_globals.te_uuid); + crm_xml_add(cmd, XML_ATTR_TRANSITION_KEY, counter); + +- rc = send_cluster_message(pcmk__get_peer(0, router_node, NULL), ++ rc = send_cluster_message(pcmk__get_node(0, router_node, NULL, ++ CRM_GET_PEER_CLUSTER), + crm_msg_crmd, cmd, TRUE); + free(counter); + free_xml(cmd); +@@ -422,7 +424,8 @@ execute_rsc_action(pcmk__graph_t *graph, pcmk__graph_action_t *action) + I_NULL, &msg); + + } else { +- rc = send_cluster_message(pcmk__get_peer(0, router_node, NULL), ++ rc = send_cluster_message(pcmk__get_node(0, router_node, NULL, ++ CRM_GET_PEER_CLUSTER), + crm_msg_lrmd, cmd, TRUE); + } + +diff --git a/daemons/controld/controld_te_events.c b/daemons/controld/controld_te_events.c +index c8cceed..84bef5b 100644 +--- a/daemons/controld/controld_te_events.c ++++ b/daemons/controld/controld_te_events.c +@@ -119,7 +119,8 @@ fail_incompletable_actions(pcmk__graph_t *graph, const char *down_node) + target_uuid = crm_element_value(action->xml, XML_LRM_ATTR_TARGET_UUID); + router = crm_element_value(action->xml, XML_LRM_ATTR_ROUTER_NODE); + if (router) { +- crm_node_t *node = pcmk__get_peer(0, router, NULL); ++ crm_node_t *node = pcmk__get_node(0, router, NULL, ++ CRM_GET_PEER_CLUSTER); + + if (node) { + router_uuid = node->uuid; +diff --git a/daemons/controld/controld_utils.c b/daemons/controld/controld_utils.c +index 55790c0..0e92416 100644 +--- a/daemons/controld/controld_utils.c ++++ b/daemons/controld/controld_utils.c +@@ -734,7 +734,8 @@ update_dc(xmlNode * msg) + /* do nothing */ + + } else if (controld_globals.dc_name != NULL) { +- crm_node_t *dc_node = pcmk__get_peer(0, controld_globals.dc_name, NULL); ++ crm_node_t *dc_node = pcmk__get_node(0, controld_globals.dc_name, NULL, ++ CRM_GET_PEER_CLUSTER); + + crm_info("Set DC to %s (%s)", + controld_globals.dc_name, +diff --git a/daemons/fenced/fenced_commands.c b/daemons/fenced/fenced_commands.c +index 28f08dd..c519607 100644 +--- a/daemons/fenced/fenced_commands.c ++++ b/daemons/fenced/fenced_commands.c +@@ -645,7 +645,8 @@ schedule_stonith_command(async_command_t * cmd, stonith_device_t * device) + } + + if (device->include_nodeid && (cmd->target != NULL)) { +- crm_node_t *node = pcmk__get_peer(0, cmd->target, NULL); ++ crm_node_t *node = pcmk__get_node(0, cmd->target, NULL, ++ CRM_GET_PEER_CLUSTER); + + cmd->target_nodeid = node->id; + } +@@ -2402,7 +2403,8 @@ stonith_send_reply(const xmlNode *reply, int call_options, + if (remote_peer == NULL) { + do_local_reply(reply, client, call_options); + } else { +- send_cluster_message(pcmk__get_peer(0, remote_peer, NULL), ++ send_cluster_message(pcmk__get_node(0, remote_peer, NULL, ++ CRM_GET_PEER_CLUSTER), + crm_msg_stonith_ng, reply, FALSE); + } + } +@@ -3371,7 +3373,8 @@ handle_fence_request(pcmk__request_t *request) + crm_xml_add(request->xml, F_STONITH_CLIENTID, + request->ipc_client->id); + crm_xml_add(request->xml, F_STONITH_REMOTE_OP_ID, op->id); +- send_cluster_message(pcmk__get_peer(0, alternate_host, NULL), ++ send_cluster_message(pcmk__get_node(0, alternate_host, NULL, ++ CRM_GET_PEER_CLUSTER), + crm_msg_stonith_ng, request->xml, FALSE); + pcmk__set_result(&request->result, CRM_EX_OK, PCMK_EXEC_PENDING, + NULL); +diff --git a/daemons/fenced/fenced_remote.c b/daemons/fenced/fenced_remote.c +index 3c176c8..96b518a 100644 +--- a/daemons/fenced/fenced_remote.c ++++ b/daemons/fenced/fenced_remote.c +@@ -1030,7 +1030,8 @@ merge_duplicates(remote_fencing_op_t *op) + op->id, other->id, other->target); + continue; + } +- if (!fencing_peer_active(pcmk__get_peer(0, other->originator, NULL))) { ++ if (!fencing_peer_active(pcmk__get_node(0, other->originator, NULL, ++ CRM_GET_PEER_CLUSTER))) { + crm_notice("Failing action '%s' targeting %s originating from " + "client %s@%s: Originator is dead " CRM_XS " id=%.8s", + other->action, other->target, other->client_name, +@@ -1663,7 +1664,8 @@ report_timeout_period(remote_fencing_op_t * op, int op_timeout) + crm_xml_add(update, F_STONITH_CALLID, call_id); + crm_xml_add_int(update, F_STONITH_TIMEOUT, op_timeout); + +- send_cluster_message(pcmk__get_peer(0, client_node, NULL), ++ send_cluster_message(pcmk__get_node(0, client_node, NULL, ++ CRM_GET_PEER_CLUSTER), + crm_msg_stonith_ng, update, FALSE); + + free_xml(update); +@@ -1917,7 +1919,8 @@ request_peer_fencing(remote_fencing_op_t *op, peer_device_info_t *peer) + op->op_timer_one = g_timeout_add((1000 * timeout_one), remote_op_timeout_one, op); + } + +- send_cluster_message(pcmk__get_peer(0, peer->host, NULL), ++ send_cluster_message(pcmk__get_node(0, peer->host, NULL, ++ CRM_GET_PEER_CLUSTER), + crm_msg_stonith_ng, remote_op, FALSE); + peer->tried = TRUE; + free_xml(remote_op); +diff --git a/include/crm/cluster/internal.h b/include/crm/cluster/internal.h +index c71069b..bea4086 100644 +--- a/include/crm/cluster/internal.h ++++ b/include/crm/cluster/internal.h +@@ -1,5 +1,5 @@ + /* +- * Copyright 2004-2023 the Pacemaker project contributors ++ * Copyright 2004-2024 the Pacemaker project contributors + * + * The version control history for this file may have further details. + * +@@ -132,9 +132,7 @@ void pcmk__refresh_node_caches_from_cib(xmlNode *cib); + crm_node_t *pcmk__search_known_node_cache(unsigned int id, const char *uname, + uint32_t flags); + +-crm_node_t *pcmk__get_peer(unsigned int id, const char *uname, +- const char *uuid); +-crm_node_t *pcmk__get_peer_full(unsigned int id, const char *uname, +- const char *uuid, int flags); ++crm_node_t *pcmk__get_node(unsigned int id, const char *uname, ++ const char *uuid, uint32_t flags); + + #endif // PCMK__CRM_CLUSTER_INTERNAL__H +diff --git a/lib/cluster/corosync.c b/lib/cluster/corosync.c +index 34a31fb..47a3321 100644 +--- a/lib/cluster/corosync.c ++++ b/lib/cluster/corosync.c +@@ -309,12 +309,12 @@ quorum_notification_cb(quorum_handle_t handle, uint32_t quorate, + crm_debug("Member[%d] %u ", i, id); + + /* Get this node's peer cache entry (adding one if not already there) */ +- node = pcmk__get_peer(id, NULL, NULL); ++ node = pcmk__get_node(id, NULL, NULL, CRM_GET_PEER_CLUSTER); + if (node->uname == NULL) { + char *name = pcmk__corosync_name(0, id); + + crm_info("Obtaining name for new node %u", id); +- node = pcmk__get_peer(id, name, NULL); ++ node = pcmk__get_node(id, name, NULL, CRM_GET_PEER_CLUSTER); + free(name); + } + +@@ -480,7 +480,8 @@ pcmk__corosync_connect(crm_cluster_t *cluster) + } + + // Ensure local node always exists in peer cache +- peer = pcmk__get_peer(cluster->nodeid, cluster->uname, NULL); ++ peer = pcmk__get_node(cluster->nodeid, cluster->uname, NULL, ++ CRM_GET_PEER_CLUSTER); + cluster->uuid = pcmk__corosync_uuid(peer); + + return TRUE; +@@ -640,7 +641,7 @@ pcmk__corosync_add_nodes(xmlNode *xml_parent) + + if (nodeid > 0 || name != NULL) { + crm_trace("Initializing node[%d] %u = %s", lpc, nodeid, name); +- pcmk__get_peer(nodeid, name, NULL); ++ pcmk__get_node(nodeid, name, NULL, CRM_GET_PEER_CLUSTER); + } + + if (nodeid > 0 && name != NULL) { +diff --git a/lib/cluster/cpg.c b/lib/cluster/cpg.c +index 778368f..bc251da 100644 +--- a/lib/cluster/cpg.c ++++ b/lib/cluster/cpg.c +@@ -1,5 +1,5 @@ + /* +- * Copyright 2004-2023 the Pacemaker project contributors ++ * Copyright 2004-2024 the Pacemaker project contributors + * + * The version control history for this file may have further details. + * +@@ -465,7 +465,8 @@ pcmk_message_common_cs(cpg_handle_t handle, uint32_t nodeid, uint32_t pid, void + + msg->sender.id = nodeid; + if (msg->sender.size == 0) { +- crm_node_t *peer = pcmk__get_peer(nodeid, NULL, NULL); ++ crm_node_t *peer = pcmk__get_node(nodeid, NULL, NULL, ++ CRM_GET_PEER_CLUSTER); + + if (peer == NULL) { + crm_err("Peer with nodeid=%u is unknown", nodeid); +@@ -526,7 +527,8 @@ pcmk_message_common_cs(cpg_handle_t handle, uint32_t nodeid, uint32_t pid, void + } + + // Is this necessary? +- pcmk__get_peer(msg->sender.id, msg->sender.uname, NULL); ++ pcmk__get_node(msg->sender.id, msg->sender.uname, NULL, ++ CRM_GET_PEER_CLUSTER); + + crm_trace("Payload: %.200s", data); + return data; +@@ -720,7 +722,8 @@ pcmk_cpg_membership(cpg_handle_t handle, + } + + for (i = 0; i < member_list_entries; i++) { +- crm_node_t *peer = pcmk__get_peer(member_list[i].nodeid, NULL, NULL); ++ crm_node_t *peer = pcmk__get_node(member_list[i].nodeid, NULL, NULL, ++ CRM_GET_PEER_CLUSTER); + + if (member_list[i].nodeid == local_nodeid + && member_list[i].pid != getpid()) { +@@ -873,7 +876,7 @@ cluster_connect_cpg(crm_cluster_t *cluster) + return FALSE; + } + +- peer = pcmk__get_peer(id, NULL, NULL); ++ peer = pcmk__get_node(id, NULL, NULL, CRM_GET_PEER_CLUSTER); + crm_update_peer_proc(__func__, peer, crm_proc_cpg, ONLINESTATUS); + return TRUE; + } +diff --git a/lib/cluster/election.c b/lib/cluster/election.c +index 31867f2..576c0aa 100644 +--- a/lib/cluster/election.c ++++ b/lib/cluster/election.c +@@ -298,7 +298,7 @@ election_vote(election_t *e) + return; + } + +- our_node = pcmk__get_peer(0, e->uname, NULL); ++ our_node = pcmk__get_node(0, e->uname, NULL, CRM_GET_PEER_CLUSTER); + if ((our_node == NULL) || (crm_is_peer_active(our_node) == FALSE)) { + crm_trace("Cannot vote in %s yet: local node not connected to cluster", + e->name); +@@ -547,8 +547,8 @@ election_count_vote(election_t *e, const xmlNode *message, bool can_win) + return election_error; + } + +- your_node = pcmk__get_peer(0, vote.from, NULL); +- our_node = pcmk__get_peer(0, e->uname, NULL); ++ your_node = pcmk__get_node(0, vote.from, NULL, CRM_GET_PEER_CLUSTER); ++ our_node = pcmk__get_node(0, e->uname, NULL, CRM_GET_PEER_CLUSTER); + we_are_owner = (our_node != NULL) + && pcmk__str_eq(our_node->uuid, vote.election_owner, + pcmk__str_none); +diff --git a/lib/cluster/membership.c b/lib/cluster/membership.c +index 41e0fa3..4c89a7c 100644 +--- a/lib/cluster/membership.c ++++ b/lib/cluster/membership.c +@@ -562,37 +562,6 @@ pcmk__search_node_caches(unsigned int id, const char *uname, uint32_t flags) + return node; + } + +-/*! +- * \brief Get a node cache entry (cluster or Pacemaker Remote) +- * +- * \param[in] id If not 0, cluster node ID to search for +- * \param[in] uname If not NULL, node name to search for +- * \param[in] uuid If not NULL while id is 0, node UUID instead of cluster +- * node ID to search for +- * \param[in] flags Bitmask of enum crm_get_peer_flags +- * +- * \return (Possibly newly created) node cache entry +- */ +-crm_node_t * +-pcmk__get_peer_full(unsigned int id, const char *uname, const char *uuid, +- int flags) +-{ +- crm_node_t *node = NULL; +- +- CRM_ASSERT(id > 0 || uname != NULL); +- +- crm_peer_init(); +- +- if (pcmk_is_set(flags, CRM_GET_PEER_REMOTE)) { +- node = g_hash_table_lookup(crm_remote_peer_cache, uname); +- } +- +- if ((node == NULL) && pcmk_is_set(flags, CRM_GET_PEER_CLUSTER)) { +- node = pcmk__get_peer(id, uname, uuid); +- } +- return node; +-} +- + /*! + * \internal + * \brief Purge a node from cache (both cluster and Pacemaker Remote) +@@ -794,12 +763,14 @@ remove_conflicting_peer(crm_node_t *node) + * \param[in] uname If not NULL, node name to search for + * \param[in] uuid If not NULL while id is 0, node UUID instead of cluster + * node ID to search for ++ * \param[in] flags Group of enum crm_get_peer_flags + * + * \return (Possibly newly created) cluster node cache entry + */ + /* coverity[-alloc] Memory is referenced in one or both hashtables */ + crm_node_t * +-pcmk__get_peer(unsigned int id, const char *uname, const char *uuid) ++pcmk__get_node(unsigned int id, const char *uname, const char *uuid, ++ uint32_t flags) + { + crm_node_t *node = NULL; + char *uname_lookup = NULL; +@@ -808,6 +779,18 @@ pcmk__get_peer(unsigned int id, const char *uname, const char *uuid) + + crm_peer_init(); + ++ // Check the Pacemaker Remote node cache first ++ if (pcmk_is_set(flags, CRM_GET_PEER_REMOTE)) { ++ node = g_hash_table_lookup(crm_remote_peer_cache, uname); ++ if (node != NULL) { ++ return node; ++ } ++ } ++ ++ if (!pcmk_is_set(flags, CRM_GET_PEER_CLUSTER)) { ++ return NULL; ++ } ++ + node = pcmk__search_cluster_node_cache(id, uname, uuid); + + /* if uname wasn't provided, and find_peer did not turn up a uname based on id. +@@ -826,7 +809,6 @@ pcmk__get_peer(unsigned int id, const char *uname, const char *uuid) + } + } + +- + if (node == NULL) { + char *uniqueid = crm_generate_uuid(); + +@@ -1417,13 +1399,13 @@ crm_terminate_member_no_mainloop(int nodeid, const char *uname, int *connection) + crm_node_t * + crm_get_peer(unsigned int id, const char *uname) + { +- return pcmk__get_peer(id, uname, NULL); ++ return pcmk__get_node(id, uname, NULL, CRM_GET_PEER_CLUSTER); + } + + crm_node_t * + crm_get_peer_full(unsigned int id, const char *uname, int flags) + { +- return pcmk__get_peer_full(id, uname, NULL, flags); ++ return pcmk__get_node(id, uname, NULL, flags); + } + + // LCOV_EXCL_STOP +-- +2.31.1 + +From 8a33a98c48475790a033f59aeb3e026f2bb68e4f Mon Sep 17 00:00:00 2001 +From: Ken Gaillot +Date: Tue, 19 Dec 2023 16:18:45 -0600 +Subject: [PATCH 5/9] API: libcrmcluster: deprecate enum crm_get_peer_flags and + all its values + +Replace it internally with a new enum pcmk__node_search_flags +--- + daemons/attrd/attrd_cib.c | 2 +- + daemons/attrd/attrd_corosync.c | 6 +++--- + daemons/attrd/attrd_messages.c | 6 +++--- + daemons/based/based_callbacks.c | 4 ++-- + daemons/based/based_messages.c | 4 ++-- + daemons/controld/controld_corosync.c | 2 +- + daemons/controld/controld_election.c | 2 +- + daemons/controld/controld_execd.c | 4 ++-- + daemons/controld/controld_fencing.c | 6 +++--- + daemons/controld/controld_join_client.c | 6 +++--- + daemons/controld/controld_join_dc.c | 10 +++++----- + daemons/controld/controld_messages.c | 4 ++-- + daemons/controld/controld_remote_ra.c | 3 ++- + daemons/controld/controld_te_actions.c | 6 +++--- + daemons/controld/controld_te_events.c | 2 +- + daemons/controld/controld_utils.c | 2 +- + daemons/fenced/fenced_commands.c | 9 +++++---- + daemons/fenced/fenced_history.c | 3 ++- + daemons/fenced/fenced_remote.c | 9 +++++---- + include/crm/cluster.h | 7 ------- + include/crm/cluster/compat.h | 7 +++++++ + include/crm/cluster/internal.h | 13 +++++++++++++ + lib/cluster/corosync.c | 8 ++++---- + lib/cluster/cpg.c | 8 ++++---- + lib/cluster/election.c | 6 +++--- + lib/cluster/membership.c | 18 +++++++++--------- + 26 files changed, 87 insertions(+), 70 deletions(-) + +diff --git a/daemons/attrd/attrd_cib.c b/daemons/attrd/attrd_cib.c +index 7018a32..bdc0a10 100644 +--- a/daemons/attrd/attrd_cib.c ++++ b/daemons/attrd/attrd_cib.c +@@ -568,7 +568,7 @@ write_attribute(attribute_t *a, bool ignore_delay) + } else { + // This will create a cluster node cache entry if none exists + crm_node_t *peer = pcmk__get_node(v->nodeid, v->nodename, NULL, +- CRM_GET_PEER_ANY); ++ pcmk__node_search_any); + + uuid = peer->uuid; + +diff --git a/daemons/attrd/attrd_corosync.c b/daemons/attrd/attrd_corosync.c +index c9e11e6..158d82f 100644 +--- a/daemons/attrd/attrd_corosync.c ++++ b/daemons/attrd/attrd_corosync.c +@@ -120,7 +120,7 @@ attrd_cpg_dispatch(cpg_handle_t handle, + crm_err("Bad message of class %d received from %s[%u]: '%.120s'", kind, from, nodeid, data); + } else { + attrd_peer_message(pcmk__get_node(nodeid, from, NULL, +- CRM_GET_PEER_CLUSTER), ++ pcmk__node_search_cluster), + xml); + } + +@@ -255,7 +255,7 @@ static void + record_peer_nodeid(attribute_value_t *v, const char *host) + { + crm_node_t *known_peer = pcmk__get_node(v->nodeid, host, NULL, +- CRM_GET_PEER_CLUSTER); ++ pcmk__node_search_cluster); + + crm_trace("Learned %s has node id %s", known_peer->uname, known_peer->uuid); + if (attrd_election_won()) { +@@ -441,7 +441,7 @@ attrd_peer_clear_failure(pcmk__request_t *request) + regex_t regex; + + crm_node_t *peer = pcmk__get_node(0, request->peer, NULL, +- CRM_GET_PEER_CLUSTER); ++ pcmk__node_search_cluster); + + if (attrd_failure_regex(®ex, rsc, op, interval_ms) != pcmk_ok) { + crm_info("Ignoring invalid request to clear failures for %s", +diff --git a/daemons/attrd/attrd_messages.c b/daemons/attrd/attrd_messages.c +index c6c1b9a..5525d4b 100644 +--- a/daemons/attrd/attrd_messages.c ++++ b/daemons/attrd/attrd_messages.c +@@ -178,7 +178,7 @@ handle_sync_request(pcmk__request_t *request) + { + if (request->peer != NULL) { + crm_node_t *peer = pcmk__get_node(0, request->peer, NULL, +- CRM_GET_PEER_CLUSTER); ++ pcmk__node_search_cluster); + + attrd_peer_sync(peer, request->xml); + pcmk__set_result(&request->result, CRM_EX_OK, PCMK_EXEC_DONE, NULL); +@@ -196,7 +196,7 @@ handle_sync_response_request(pcmk__request_t *request) + } else { + if (request->peer != NULL) { + crm_node_t *peer = pcmk__get_node(0, request->peer, NULL, +- CRM_GET_PEER_CLUSTER); ++ pcmk__node_search_cluster); + bool peer_won = attrd_check_for_new_writer(peer, request->xml); + + if (!pcmk__str_eq(peer->uname, attrd_cluster->uname, pcmk__str_casei)) { +@@ -215,7 +215,7 @@ handle_update_request(pcmk__request_t *request) + if (request->peer != NULL) { + const char *host = crm_element_value(request->xml, PCMK__XA_ATTR_NODE_NAME); + crm_node_t *peer = pcmk__get_node(0, request->peer, NULL, +- CRM_GET_PEER_CLUSTER); ++ pcmk__node_search_cluster); + + attrd_peer_update(peer, request->xml, host, false); + pcmk__set_result(&request->result, CRM_EX_OK, PCMK_EXEC_DONE, NULL); +diff --git a/daemons/based/based_callbacks.c b/daemons/based/based_callbacks.c +index b1f3b4b..5f3dc62 100644 +--- a/daemons/based/based_callbacks.c ++++ b/daemons/based/based_callbacks.c +@@ -930,7 +930,7 @@ forward_request(xmlNode *request) + crm_xml_add(request, F_CIB_DELEGATED, OUR_NODENAME); + + if (host != NULL) { +- peer = pcmk__get_node(0, host, NULL, CRM_GET_PEER_CLUSTER); ++ peer = pcmk__get_node(0, host, NULL, pcmk__node_search_cluster); + } + send_cluster_message(peer, crm_msg_cib, request, FALSE); + +@@ -990,7 +990,7 @@ send_peer_reply(xmlNode * msg, xmlNode * result_diff, const char *originator, gb + crm_trace("Sending request result to %s only", originator); + crm_xml_add(msg, F_CIB_ISREPLY, originator); + return send_cluster_message(pcmk__get_node(0, originator, NULL, +- CRM_GET_PEER_CLUSTER), ++ pcmk__node_search_cluster), + crm_msg_cib, msg, FALSE); + } + +diff --git a/daemons/based/based_messages.c b/daemons/based/based_messages.c +index ff1a6aa..7f503b2 100644 +--- a/daemons/based/based_messages.c ++++ b/daemons/based/based_messages.c +@@ -129,7 +129,7 @@ send_sync_request(const char *host) + stand_alone? "localhost" : crm_cluster->uname); + + if (host != NULL) { +- peer = pcmk__get_node(0, host, NULL, CRM_GET_PEER_CLUSTER); ++ peer = pcmk__get_node(0, host, NULL, pcmk__node_search_cluster); + } + send_cluster_message(peer, crm_msg_cib, sync_me, FALSE); + free_xml(sync_me); +@@ -449,7 +449,7 @@ sync_our_cib(xmlNode * request, gboolean all) + add_message_xml(replace_request, F_CIB_CALLDATA, the_cib); + + if (!all) { +- peer = pcmk__get_node(0, host, NULL, CRM_GET_PEER_CLUSTER); ++ peer = pcmk__get_node(0, host, NULL, pcmk__node_search_cluster); + } + if (!send_cluster_message(peer, crm_msg_cib, replace_request, FALSE)) { + result = -ENOTCONN; +diff --git a/daemons/controld/controld_corosync.c b/daemons/controld/controld_corosync.c +index fa1df6f..0f3ea32 100644 +--- a/daemons/controld/controld_corosync.c ++++ b/daemons/controld/controld_corosync.c +@@ -49,7 +49,7 @@ crmd_cs_dispatch(cpg_handle_t handle, const struct cpg_name *groupName, + crm_xml_add(xml, F_ORIG, from); + /* crm_xml_add_int(xml, F_SEQ, wrapper->id); Fake? */ + +- peer = pcmk__get_node(0, from, NULL, CRM_GET_PEER_CLUSTER); ++ peer = pcmk__get_node(0, from, NULL, pcmk__node_search_cluster); + if (!pcmk_is_set(peer->processes, crm_proc_cpg)) { + /* If we can still talk to our peer process on that node, + * then it must be part of the corosync membership +diff --git a/daemons/controld/controld_election.c b/daemons/controld/controld_election.c +index 734064d..adad168 100644 +--- a/daemons/controld/controld_election.c ++++ b/daemons/controld/controld_election.c +@@ -266,7 +266,7 @@ do_dc_release(long long action, + if (pcmk_is_set(controld_globals.fsa_input_register, R_SHUTDOWN)) { + xmlNode *update = NULL; + crm_node_t *node = pcmk__get_node(0, controld_globals.our_nodename, +- NULL, CRM_GET_PEER_CLUSTER); ++ NULL, pcmk__node_search_cluster); + + pcmk__update_peer_expected(__func__, node, CRMD_JOINSTATE_DOWN); + update = create_node_state_update(node, node_update_expected, NULL, +diff --git a/daemons/controld/controld_execd.c b/daemons/controld/controld_execd.c +index df715aa..fe2313c 100644 +--- a/daemons/controld/controld_execd.c ++++ b/daemons/controld/controld_execd.c +@@ -581,7 +581,7 @@ controld_query_executor_state(void) + return NULL; + } + +- peer = pcmk__get_node(0, lrm_state->node_name, NULL, CRM_GET_PEER_ANY); ++ peer = pcmk__get_node(0, lrm_state->node_name, NULL, pcmk__node_search_any); + CRM_CHECK(peer != NULL, return NULL); + + xml_state = create_node_state_update(peer, +@@ -1753,7 +1753,7 @@ controld_ack_event_directly(const char *to_host, const char *to_sys, + } + + peer = pcmk__get_node(0, controld_globals.our_nodename, NULL, +- CRM_GET_PEER_CLUSTER); ++ pcmk__node_search_cluster); + update = create_node_state_update(peer, node_update_none, NULL, + __func__); + +diff --git a/daemons/controld/controld_fencing.c b/daemons/controld/controld_fencing.c +index 60a7f9f..79a52be 100644 +--- a/daemons/controld/controld_fencing.c ++++ b/daemons/controld/controld_fencing.c +@@ -222,7 +222,7 @@ send_stonith_update(pcmk__graph_action_t *action, const char *target, + * Try getting any existing node cache entry also by node uuid in case it + * doesn't have an uname yet. + */ +- peer = pcmk__get_node(0, target, uuid, CRM_GET_PEER_ANY); ++ peer = pcmk__get_node(0, target, uuid, pcmk__node_search_any); + + CRM_CHECK(peer != NULL, return); + +@@ -375,7 +375,7 @@ execute_stonith_cleanup(void) + for (iter = stonith_cleanup_list; iter != NULL; iter = iter->next) { + char *target = iter->data; + crm_node_t *target_node = pcmk__get_node(0, target, NULL, +- CRM_GET_PEER_CLUSTER); ++ pcmk__node_search_cluster); + const char *uuid = crm_peer_uuid(target_node); + + crm_notice("Marking %s, target of a previous stonith action, as clean", target); +@@ -582,7 +582,7 @@ handle_fence_notification(stonith_t *st, stonith_event_t *event) + + if (succeeded) { + crm_node_t *peer = pcmk__search_known_node_cache(0, event->target, +- CRM_GET_PEER_ANY); ++ pcmk__node_search_any); + const char *uuid = NULL; + + if (peer == NULL) { +diff --git a/daemons/controld/controld_join_client.c b/daemons/controld/controld_join_client.c +index 6f20ef2..101c73d 100644 +--- a/daemons/controld/controld_join_client.c ++++ b/daemons/controld/controld_join_client.c +@@ -36,7 +36,7 @@ update_dc_expected(const xmlNode *msg) + if ((controld_globals.dc_name != NULL) + && pcmk__xe_attr_is_true(msg, F_CRM_DC_LEAVING)) { + crm_node_t *dc_node = pcmk__get_node(0, controld_globals.dc_name, NULL, +- CRM_GET_PEER_CLUSTER); ++ pcmk__node_search_cluster); + + pcmk__update_peer_expected(__func__, dc_node, CRMD_JOINSTATE_DOWN); + } +@@ -179,7 +179,7 @@ join_query_callback(xmlNode * msg, int call_id, int rc, xmlNode * output, void * + crm_xml_add(reply, F_CRM_JOIN_ID, join_id); + crm_xml_add(reply, XML_ATTR_CRM_VERSION, CRM_FEATURE_SET); + send_cluster_message(pcmk__get_node(0, controld_globals.dc_name, NULL, +- CRM_GET_PEER_CLUSTER), ++ pcmk__node_search_cluster), + crm_msg_crmd, reply, TRUE); + free_xml(reply); + } +@@ -336,7 +336,7 @@ do_cl_join_finalize_respond(long long action, + } + + send_cluster_message(pcmk__get_node(0, controld_globals.dc_name, NULL, +- CRM_GET_PEER_CLUSTER), ++ pcmk__node_search_cluster), + crm_msg_crmd, reply, TRUE); + free_xml(reply); + +diff --git a/daemons/controld/controld_join_dc.c b/daemons/controld/controld_join_dc.c +index 064649f..e9fc698 100644 +--- a/daemons/controld/controld_join_dc.c ++++ b/daemons/controld/controld_join_dc.c +@@ -318,7 +318,7 @@ do_dc_join_offer_one(long long action, + crm_err("Can't make join-%d offer to unknown node", current_join_id); + return; + } +- member = pcmk__get_node(0, join_to, NULL, CRM_GET_PEER_CLUSTER); ++ member = pcmk__get_node(0, join_to, NULL, pcmk__node_search_cluster); + + /* It is possible that a node will have been sick or starting up when the + * original offer was made. However, it will either re-announce itself in +@@ -333,7 +333,7 @@ do_dc_join_offer_one(long long action, + */ + if (strcasecmp(join_to, controld_globals.our_nodename) != 0) { + member = pcmk__get_node(0, controld_globals.our_nodename, NULL, +- CRM_GET_PEER_CLUSTER); ++ pcmk__node_search_cluster); + join_make_offer(NULL, member, NULL); + } + +@@ -397,7 +397,7 @@ do_dc_join_filter_offer(long long action, + crm_err("Ignoring invalid join request without node name"); + return; + } +- join_node = pcmk__get_node(0, join_from, NULL, CRM_GET_PEER_CLUSTER); ++ join_node = pcmk__get_node(0, join_from, NULL, pcmk__node_search_cluster); + + crm_element_value_int(join_ack->msg, F_CRM_JOIN_ID, &join_id); + if (join_id != current_join_id) { +@@ -733,7 +733,7 @@ do_dc_join_ack(long long action, + goto done; + } + +- peer = pcmk__get_node(0, join_from, NULL, CRM_GET_PEER_CLUSTER); ++ peer = pcmk__get_node(0, join_from, NULL, pcmk__node_search_cluster); + if (peer->join != crm_join_finalized) { + crm_info("Ignoring out-of-sequence join-%d confirmation from %s " + "(currently %s not %s)", +@@ -867,7 +867,7 @@ finalize_join_for(gpointer key, gpointer value, gpointer user_data) + return; + } + +- join_node = pcmk__get_node(0, join_to, NULL, CRM_GET_PEER_CLUSTER); ++ join_node = pcmk__get_node(0, join_to, NULL, pcmk__node_search_cluster); + if (!crm_is_peer_active(join_node)) { + /* + * NACK'ing nodes that the membership layer doesn't know about yet +diff --git a/daemons/controld/controld_messages.c b/daemons/controld/controld_messages.c +index 71f5680..999dd13 100644 +--- a/daemons/controld/controld_messages.c ++++ b/daemons/controld/controld_messages.c +@@ -459,7 +459,7 @@ relay_message(xmlNode * msg, gboolean originated_locally) + crm_log_xml_trace(msg, "relayed"); + if (!broadcast) { + node_to = pcmk__get_node(0, host_to, NULL, +- CRM_GET_PEER_CLUSTER); ++ pcmk__node_search_cluster); + } + send_cluster_message(node_to, dest, msg, TRUE); + return TRUE; +@@ -904,7 +904,7 @@ handle_node_info_request(const xmlNode *msg) + value = controld_globals.our_nodename; + } + +- node = pcmk__search_node_caches(node_id, value, CRM_GET_PEER_ANY); ++ node = pcmk__search_node_caches(node_id, value, pcmk__node_search_any); + if (node) { + crm_xml_add(reply_data, XML_ATTR_ID, node->uuid); + crm_xml_add(reply_data, XML_ATTR_UNAME, node->uname); +diff --git a/daemons/controld/controld_remote_ra.c b/daemons/controld/controld_remote_ra.c +index 9c4bb58..662643c 100644 +--- a/daemons/controld/controld_remote_ra.c ++++ b/daemons/controld/controld_remote_ra.c +@@ -206,7 +206,8 @@ should_purge_attributes(crm_node_t *node) + /* Get the node that was hosting the remote connection resource from the + * peer cache. That's the one we really care about here. + */ +- conn_node = pcmk__get_node(0, node->conn_host, NULL, CRM_GET_PEER_CLUSTER); ++ conn_node = pcmk__get_node(0, node->conn_host, NULL, ++ pcmk__node_search_cluster); + if (conn_node == NULL) { + return purge; + } +diff --git a/daemons/controld/controld_te_actions.c b/daemons/controld/controld_te_actions.c +index 01ba4a0..fbd9955 100644 +--- a/daemons/controld/controld_te_actions.c ++++ b/daemons/controld/controld_te_actions.c +@@ -159,7 +159,7 @@ execute_cluster_action(pcmk__graph_t *graph, pcmk__graph_action_t *action) + + } else if (pcmk__str_eq(task, PCMK_ACTION_DO_SHUTDOWN, pcmk__str_none)) { + crm_node_t *peer = pcmk__get_node(0, router_node, NULL, +- CRM_GET_PEER_CLUSTER); ++ pcmk__node_search_cluster); + + pcmk__update_peer_expected(__func__, peer, CRMD_JOINSTATE_DOWN); + } +@@ -172,7 +172,7 @@ execute_cluster_action(pcmk__graph_t *graph, pcmk__graph_action_t *action) + crm_xml_add(cmd, XML_ATTR_TRANSITION_KEY, counter); + + rc = send_cluster_message(pcmk__get_node(0, router_node, NULL, +- CRM_GET_PEER_CLUSTER), ++ pcmk__node_search_cluster), + crm_msg_crmd, cmd, TRUE); + free(counter); + free_xml(cmd); +@@ -425,7 +425,7 @@ execute_rsc_action(pcmk__graph_t *graph, pcmk__graph_action_t *action) + + } else { + rc = send_cluster_message(pcmk__get_node(0, router_node, NULL, +- CRM_GET_PEER_CLUSTER), ++ pcmk__node_search_cluster), + crm_msg_lrmd, cmd, TRUE); + } + +diff --git a/daemons/controld/controld_te_events.c b/daemons/controld/controld_te_events.c +index 84bef5b..a54304b 100644 +--- a/daemons/controld/controld_te_events.c ++++ b/daemons/controld/controld_te_events.c +@@ -120,7 +120,7 @@ fail_incompletable_actions(pcmk__graph_t *graph, const char *down_node) + router = crm_element_value(action->xml, XML_LRM_ATTR_ROUTER_NODE); + if (router) { + crm_node_t *node = pcmk__get_node(0, router, NULL, +- CRM_GET_PEER_CLUSTER); ++ pcmk__node_search_cluster); + + if (node) { + router_uuid = node->uuid; +diff --git a/daemons/controld/controld_utils.c b/daemons/controld/controld_utils.c +index 0e92416..1143e88 100644 +--- a/daemons/controld/controld_utils.c ++++ b/daemons/controld/controld_utils.c +@@ -735,7 +735,7 @@ update_dc(xmlNode * msg) + + } else if (controld_globals.dc_name != NULL) { + crm_node_t *dc_node = pcmk__get_node(0, controld_globals.dc_name, NULL, +- CRM_GET_PEER_CLUSTER); ++ pcmk__node_search_cluster); + + crm_info("Set DC to %s (%s)", + controld_globals.dc_name, +diff --git a/daemons/fenced/fenced_commands.c b/daemons/fenced/fenced_commands.c +index c519607..d2a556f 100644 +--- a/daemons/fenced/fenced_commands.c ++++ b/daemons/fenced/fenced_commands.c +@@ -646,7 +646,7 @@ schedule_stonith_command(async_command_t * cmd, stonith_device_t * device) + + if (device->include_nodeid && (cmd->target != NULL)) { + crm_node_t *node = pcmk__get_node(0, cmd->target, NULL, +- CRM_GET_PEER_CLUSTER); ++ pcmk__node_search_cluster); + + cmd->target_nodeid = node->id; + } +@@ -2404,7 +2404,7 @@ stonith_send_reply(const xmlNode *reply, int call_options, + do_local_reply(reply, client, call_options); + } else { + send_cluster_message(pcmk__get_node(0, remote_peer, NULL, +- CRM_GET_PEER_CLUSTER), ++ pcmk__node_search_cluster), + crm_msg_stonith_ng, reply, FALSE); + } + } +@@ -2920,7 +2920,8 @@ fence_locally(xmlNode *msg, pcmk__action_result_t *result) + crm_node_t *node = NULL; + + pcmk__scan_min_int(host, &nodeid, 0); +- node = pcmk__search_known_node_cache(nodeid, NULL, CRM_GET_PEER_ANY); ++ node = pcmk__search_known_node_cache(nodeid, NULL, ++ pcmk__node_search_any); + if (node != NULL) { + host = node->uname; + } +@@ -3374,7 +3375,7 @@ handle_fence_request(pcmk__request_t *request) + request->ipc_client->id); + crm_xml_add(request->xml, F_STONITH_REMOTE_OP_ID, op->id); + send_cluster_message(pcmk__get_node(0, alternate_host, NULL, +- CRM_GET_PEER_CLUSTER), ++ pcmk__node_search_cluster), + crm_msg_stonith_ng, request->xml, FALSE); + pcmk__set_result(&request->result, CRM_EX_OK, PCMK_EXEC_PENDING, + NULL); +diff --git a/daemons/fenced/fenced_history.c b/daemons/fenced/fenced_history.c +index a766477..4fa2215 100644 +--- a/daemons/fenced/fenced_history.c ++++ b/daemons/fenced/fenced_history.c +@@ -469,7 +469,8 @@ stonith_fence_history(xmlNode *msg, xmlNode **output, + crm_node_t *node; + + pcmk__scan_min_int(target, &nodeid, 0); +- node = pcmk__search_known_node_cache(nodeid, NULL, CRM_GET_PEER_ANY); ++ node = pcmk__search_known_node_cache(nodeid, NULL, ++ pcmk__node_search_any); + if (node) { + target = node->uname; + } +diff --git a/daemons/fenced/fenced_remote.c b/daemons/fenced/fenced_remote.c +index 96b518a..482efb9 100644 +--- a/daemons/fenced/fenced_remote.c ++++ b/daemons/fenced/fenced_remote.c +@@ -1031,7 +1031,7 @@ merge_duplicates(remote_fencing_op_t *op) + continue; + } + if (!fencing_peer_active(pcmk__get_node(0, other->originator, NULL, +- CRM_GET_PEER_CLUSTER))) { ++ pcmk__node_search_cluster))) { + crm_notice("Failing action '%s' targeting %s originating from " + "client %s@%s: Originator is dead " CRM_XS " id=%.8s", + other->action, other->target, other->client_name, +@@ -1221,7 +1221,8 @@ create_remote_stonith_op(const char *client, xmlNode *request, gboolean peer) + crm_node_t *node; + + pcmk__scan_min_int(op->target, &nodeid, 0); +- node = pcmk__search_known_node_cache(nodeid, NULL, CRM_GET_PEER_ANY); ++ node = pcmk__search_known_node_cache(nodeid, NULL, ++ pcmk__node_search_any); + + /* Ensure the conversion only happens once */ + stonith__clear_call_options(op->call_options, op->id, st_opt_cs_nodeid); +@@ -1665,7 +1666,7 @@ report_timeout_period(remote_fencing_op_t * op, int op_timeout) + crm_xml_add_int(update, F_STONITH_TIMEOUT, op_timeout); + + send_cluster_message(pcmk__get_node(0, client_node, NULL, +- CRM_GET_PEER_CLUSTER), ++ pcmk__node_search_cluster), + crm_msg_stonith_ng, update, FALSE); + + free_xml(update); +@@ -1920,7 +1921,7 @@ request_peer_fencing(remote_fencing_op_t *op, peer_device_info_t *peer) + } + + send_cluster_message(pcmk__get_node(0, peer->host, NULL, +- CRM_GET_PEER_CLUSTER), ++ pcmk__node_search_cluster), + crm_msg_stonith_ng, remote_op, FALSE); + peer->tried = TRUE; + free_xml(remote_op); +diff --git a/include/crm/cluster.h b/include/crm/cluster.h +index 302b807..decb8e8 100644 +--- a/include/crm/cluster.h ++++ b/include/crm/cluster.h +@@ -128,13 +128,6 @@ enum crm_ais_msg_types { + crm_msg_stonith_ng = 9, + }; + +-/* used with crm_get_peer_full */ +-enum crm_get_peer_flags { +- CRM_GET_PEER_CLUSTER = 0x0001, +- CRM_GET_PEER_REMOTE = 0x0002, +- CRM_GET_PEER_ANY = CRM_GET_PEER_CLUSTER|CRM_GET_PEER_REMOTE, +-}; +- + gboolean send_cluster_message(const crm_node_t *node, + enum crm_ais_msg_types service, + const xmlNode *data, gboolean ordered); +diff --git a/include/crm/cluster/compat.h b/include/crm/cluster/compat.h +index e853fd8..14c4504 100644 +--- a/include/crm/cluster/compat.h ++++ b/include/crm/cluster/compat.h +@@ -26,6 +26,13 @@ extern "C" { + * release. + */ + ++// \deprecated Do not use ++enum crm_get_peer_flags { ++ CRM_GET_PEER_CLUSTER = 0x0001, ++ CRM_GET_PEER_REMOTE = 0x0002, ++ CRM_GET_PEER_ANY = CRM_GET_PEER_CLUSTER|CRM_GET_PEER_REMOTE, ++}; ++ + // \deprecated Do not use Pacemaker for cluster node cacheing + crm_node_t *crm_get_peer(unsigned int id, const char *uname); + +diff --git a/include/crm/cluster/internal.h b/include/crm/cluster/internal.h +index bea4086..9513254 100644 +--- a/include/crm/cluster/internal.h ++++ b/include/crm/cluster/internal.h +@@ -30,6 +30,19 @@ enum crm_proc_flag { + }; + /* *INDENT-ON* */ + ++// Used with node cache search functions ++enum pcmk__node_search_flags { ++ pcmk__node_search_none = 0, ++ pcmk__node_search_cluster = (1 << 0), // Search for cluster nodes ++ pcmk__node_search_remote = (1 << 1), // Search for remote nodes ++ pcmk__node_search_any = pcmk__node_search_cluster ++ |pcmk__node_search_remote, ++ ++ /* @COMPAT The values before this must stay the same until we can drop ++ * support for enum crm_get_peer_flags ++ */ ++}; ++ + /*! + * \internal + * \brief Return the process bit corresponding to the current cluster stack +diff --git a/lib/cluster/corosync.c b/lib/cluster/corosync.c +index 47a3321..374250f 100644 +--- a/lib/cluster/corosync.c ++++ b/lib/cluster/corosync.c +@@ -309,12 +309,12 @@ quorum_notification_cb(quorum_handle_t handle, uint32_t quorate, + crm_debug("Member[%d] %u ", i, id); + + /* Get this node's peer cache entry (adding one if not already there) */ +- node = pcmk__get_node(id, NULL, NULL, CRM_GET_PEER_CLUSTER); ++ node = pcmk__get_node(id, NULL, NULL, pcmk__node_search_cluster); + if (node->uname == NULL) { + char *name = pcmk__corosync_name(0, id); + + crm_info("Obtaining name for new node %u", id); +- node = pcmk__get_node(id, name, NULL, CRM_GET_PEER_CLUSTER); ++ node = pcmk__get_node(id, name, NULL, pcmk__node_search_cluster); + free(name); + } + +@@ -481,7 +481,7 @@ pcmk__corosync_connect(crm_cluster_t *cluster) + + // Ensure local node always exists in peer cache + peer = pcmk__get_node(cluster->nodeid, cluster->uname, NULL, +- CRM_GET_PEER_CLUSTER); ++ pcmk__node_search_cluster); + cluster->uuid = pcmk__corosync_uuid(peer); + + return TRUE; +@@ -641,7 +641,7 @@ pcmk__corosync_add_nodes(xmlNode *xml_parent) + + if (nodeid > 0 || name != NULL) { + crm_trace("Initializing node[%d] %u = %s", lpc, nodeid, name); +- pcmk__get_node(nodeid, name, NULL, CRM_GET_PEER_CLUSTER); ++ pcmk__get_node(nodeid, name, NULL, pcmk__node_search_cluster); + } + + if (nodeid > 0 && name != NULL) { +diff --git a/lib/cluster/cpg.c b/lib/cluster/cpg.c +index bc251da..b5f2884 100644 +--- a/lib/cluster/cpg.c ++++ b/lib/cluster/cpg.c +@@ -466,7 +466,7 @@ pcmk_message_common_cs(cpg_handle_t handle, uint32_t nodeid, uint32_t pid, void + msg->sender.id = nodeid; + if (msg->sender.size == 0) { + crm_node_t *peer = pcmk__get_node(nodeid, NULL, NULL, +- CRM_GET_PEER_CLUSTER); ++ pcmk__node_search_cluster); + + if (peer == NULL) { + crm_err("Peer with nodeid=%u is unknown", nodeid); +@@ -528,7 +528,7 @@ pcmk_message_common_cs(cpg_handle_t handle, uint32_t nodeid, uint32_t pid, void + + // Is this necessary? + pcmk__get_node(msg->sender.id, msg->sender.uname, NULL, +- CRM_GET_PEER_CLUSTER); ++ pcmk__node_search_cluster); + + crm_trace("Payload: %.200s", data); + return data; +@@ -723,7 +723,7 @@ pcmk_cpg_membership(cpg_handle_t handle, + + for (i = 0; i < member_list_entries; i++) { + crm_node_t *peer = pcmk__get_node(member_list[i].nodeid, NULL, NULL, +- CRM_GET_PEER_CLUSTER); ++ pcmk__node_search_cluster); + + if (member_list[i].nodeid == local_nodeid + && member_list[i].pid != getpid()) { +@@ -876,7 +876,7 @@ cluster_connect_cpg(crm_cluster_t *cluster) + return FALSE; + } + +- peer = pcmk__get_node(id, NULL, NULL, CRM_GET_PEER_CLUSTER); ++ peer = pcmk__get_node(id, NULL, NULL, pcmk__node_search_cluster); + crm_update_peer_proc(__func__, peer, crm_proc_cpg, ONLINESTATUS); + return TRUE; + } +diff --git a/lib/cluster/election.c b/lib/cluster/election.c +index 576c0aa..7276a2d 100644 +--- a/lib/cluster/election.c ++++ b/lib/cluster/election.c +@@ -298,7 +298,7 @@ election_vote(election_t *e) + return; + } + +- our_node = pcmk__get_node(0, e->uname, NULL, CRM_GET_PEER_CLUSTER); ++ our_node = pcmk__get_node(0, e->uname, NULL, pcmk__node_search_cluster); + if ((our_node == NULL) || (crm_is_peer_active(our_node) == FALSE)) { + crm_trace("Cannot vote in %s yet: local node not connected to cluster", + e->name); +@@ -547,8 +547,8 @@ election_count_vote(election_t *e, const xmlNode *message, bool can_win) + return election_error; + } + +- your_node = pcmk__get_node(0, vote.from, NULL, CRM_GET_PEER_CLUSTER); +- our_node = pcmk__get_node(0, e->uname, NULL, CRM_GET_PEER_CLUSTER); ++ your_node = pcmk__get_node(0, vote.from, NULL, pcmk__node_search_cluster); ++ our_node = pcmk__get_node(0, e->uname, NULL, pcmk__node_search_cluster); + we_are_owner = (our_node != NULL) + && pcmk__str_eq(our_node->uuid, vote.election_owner, + pcmk__str_none); +diff --git a/lib/cluster/membership.c b/lib/cluster/membership.c +index 4c89a7c..705b70c 100644 +--- a/lib/cluster/membership.c ++++ b/lib/cluster/membership.c +@@ -539,7 +539,7 @@ hash_find_by_data(gpointer key, gpointer value, gpointer user_data) + * + * \param[in] id If not 0, cluster node ID to search for + * \param[in] uname If not NULL, node name to search for +- * \param[in] flags Bitmask of enum crm_get_peer_flags ++ * \param[in] flags Group of enum pcmk__node_search_flags + * + * \return Node cache entry if found, otherwise NULL + */ +@@ -552,11 +552,11 @@ pcmk__search_node_caches(unsigned int id, const char *uname, uint32_t flags) + + crm_peer_init(); + +- if ((uname != NULL) && pcmk_is_set(flags, CRM_GET_PEER_REMOTE)) { ++ if ((uname != NULL) && pcmk_is_set(flags, pcmk__node_search_remote)) { + node = g_hash_table_lookup(crm_remote_peer_cache, uname); + } + +- if ((node == NULL) && pcmk_is_set(flags, CRM_GET_PEER_CLUSTER)) { ++ if ((node == NULL) && pcmk_is_set(flags, pcmk__node_search_cluster)) { + node = pcmk__search_cluster_node_cache(id, uname, NULL); + } + return node; +@@ -763,7 +763,7 @@ remove_conflicting_peer(crm_node_t *node) + * \param[in] uname If not NULL, node name to search for + * \param[in] uuid If not NULL while id is 0, node UUID instead of cluster + * node ID to search for +- * \param[in] flags Group of enum crm_get_peer_flags ++ * \param[in] flags Group of enum pcmk__node_search_flags + * + * \return (Possibly newly created) cluster node cache entry + */ +@@ -780,14 +780,14 @@ pcmk__get_node(unsigned int id, const char *uname, const char *uuid, + crm_peer_init(); + + // Check the Pacemaker Remote node cache first +- if (pcmk_is_set(flags, CRM_GET_PEER_REMOTE)) { ++ if (pcmk_is_set(flags, pcmk__node_search_remote)) { + node = g_hash_table_lookup(crm_remote_peer_cache, uname); + if (node != NULL) { + return node; + } + } + +- if (!pcmk_is_set(flags, CRM_GET_PEER_CLUSTER)) { ++ if (!pcmk_is_set(flags, pcmk__node_search_cluster)) { + return NULL; + } + +@@ -1349,7 +1349,7 @@ pcmk__refresh_node_caches_from_cib(xmlNode *cib) + * + * \param[in] id If not 0, cluster node ID to search for + * \param[in] uname If not NULL, node name to search for +- * \param[in] flags Bitmask of enum crm_get_peer_flags ++ * \param[in] flags Group of enum pcmk__node_search_flags + * + * \return Known node cache entry if found, otherwise NULL + */ +@@ -1364,7 +1364,7 @@ pcmk__search_known_node_cache(unsigned int id, const char *uname, + + node = pcmk__search_node_caches(id, uname, flags); + +- if (node || !(flags & CRM_GET_PEER_CLUSTER)) { ++ if (node || !(flags & pcmk__node_search_cluster)) { + return node; + } + +@@ -1399,7 +1399,7 @@ crm_terminate_member_no_mainloop(int nodeid, const char *uname, int *connection) + crm_node_t * + crm_get_peer(unsigned int id, const char *uname) + { +- return pcmk__get_node(id, uname, NULL, CRM_GET_PEER_CLUSTER); ++ return pcmk__get_node(id, uname, NULL, pcmk__node_search_cluster); + } + + crm_node_t * +-- +2.31.1 + +From aef8f5016b2de67ab12f896b2bfa7a0f1954b5b1 Mon Sep 17 00:00:00 2001 +From: Ken Gaillot +Date: Tue, 19 Dec 2023 16:27:24 -0600 +Subject: [PATCH 6/9] Refactor: libcrmcluster: replace + pcmk__search_known_node_cache() + +... with new flag in enum pcmk__node_search_flags +--- + daemons/controld/controld_fencing.c | 5 ++-- + daemons/fenced/fenced_commands.c | 5 ++-- + daemons/fenced/fenced_history.c | 5 ++-- + daemons/fenced/fenced_remote.c | 5 ++-- + include/crm/cluster/internal.h | 4 +-- + lib/cluster/membership.c | 45 ++++++----------------------- + 6 files changed, 23 insertions(+), 46 deletions(-) + +diff --git a/daemons/controld/controld_fencing.c b/daemons/controld/controld_fencing.c +index 79a52be..ede2c27 100644 +--- a/daemons/controld/controld_fencing.c ++++ b/daemons/controld/controld_fencing.c +@@ -581,8 +581,9 @@ handle_fence_notification(stonith_t *st, stonith_event_t *event) + event->id); + + if (succeeded) { +- crm_node_t *peer = pcmk__search_known_node_cache(0, event->target, +- pcmk__node_search_any); ++ crm_node_t *peer = pcmk__search_node_caches(0, event->target, ++ pcmk__node_search_any ++ |pcmk__node_search_known); + const char *uuid = NULL; + + if (peer == NULL) { +diff --git a/daemons/fenced/fenced_commands.c b/daemons/fenced/fenced_commands.c +index d2a556f..4f21858 100644 +--- a/daemons/fenced/fenced_commands.c ++++ b/daemons/fenced/fenced_commands.c +@@ -2920,8 +2920,9 @@ fence_locally(xmlNode *msg, pcmk__action_result_t *result) + crm_node_t *node = NULL; + + pcmk__scan_min_int(host, &nodeid, 0); +- node = pcmk__search_known_node_cache(nodeid, NULL, +- pcmk__node_search_any); ++ node = pcmk__search_node_caches(nodeid, NULL, ++ pcmk__node_search_any ++ |pcmk__node_search_known); + if (node != NULL) { + host = node->uname; + } +diff --git a/daemons/fenced/fenced_history.c b/daemons/fenced/fenced_history.c +index 4fa2215..fb709ff 100644 +--- a/daemons/fenced/fenced_history.c ++++ b/daemons/fenced/fenced_history.c +@@ -469,8 +469,9 @@ stonith_fence_history(xmlNode *msg, xmlNode **output, + crm_node_t *node; + + pcmk__scan_min_int(target, &nodeid, 0); +- node = pcmk__search_known_node_cache(nodeid, NULL, +- pcmk__node_search_any); ++ node = pcmk__search_node_caches(nodeid, NULL, ++ pcmk__node_search_any ++ |pcmk__node_search_known); + if (node) { + target = node->uname; + } +diff --git a/daemons/fenced/fenced_remote.c b/daemons/fenced/fenced_remote.c +index 482efb9..ba70c57 100644 +--- a/daemons/fenced/fenced_remote.c ++++ b/daemons/fenced/fenced_remote.c +@@ -1221,8 +1221,9 @@ create_remote_stonith_op(const char *client, xmlNode *request, gboolean peer) + crm_node_t *node; + + pcmk__scan_min_int(op->target, &nodeid, 0); +- node = pcmk__search_known_node_cache(nodeid, NULL, +- pcmk__node_search_any); ++ node = pcmk__search_node_caches(nodeid, NULL, ++ pcmk__node_search_any ++ |pcmk__node_search_known); + + /* Ensure the conversion only happens once */ + stonith__clear_call_options(op->call_options, op->id, st_opt_cs_nodeid); +diff --git a/include/crm/cluster/internal.h b/include/crm/cluster/internal.h +index 9513254..b75784c 100644 +--- a/include/crm/cluster/internal.h ++++ b/include/crm/cluster/internal.h +@@ -41,6 +41,8 @@ enum pcmk__node_search_flags { + /* @COMPAT The values before this must stay the same until we can drop + * support for enum crm_get_peer_flags + */ ++ ++ pcmk__node_search_known = (1 << 2), // Search previously known nodes + }; + + /*! +@@ -142,8 +144,6 @@ crm_node_t *pcmk__search_cluster_node_cache(unsigned int id, const char *uname, + void pcmk__purge_node_from_cache(const char *node_name, uint32_t node_id); + + void pcmk__refresh_node_caches_from_cib(xmlNode *cib); +-crm_node_t *pcmk__search_known_node_cache(unsigned int id, const char *uname, +- uint32_t flags); + + crm_node_t *pcmk__get_node(unsigned int id, const char *uname, + const char *uuid, uint32_t flags); +diff --git a/lib/cluster/membership.c b/lib/cluster/membership.c +index 705b70c..ef4aaac 100644 +--- a/lib/cluster/membership.c ++++ b/lib/cluster/membership.c +@@ -82,6 +82,7 @@ static gboolean crm_autoreap = TRUE; + } while (0) + + static void update_peer_uname(crm_node_t *node, const char *uname); ++static crm_node_t *find_known_node(const char *id, const char *uname); + + int + crm_remote_peer_cache_size(void) +@@ -559,6 +560,14 @@ pcmk__search_node_caches(unsigned int id, const char *uname, uint32_t flags) + if ((node == NULL) && pcmk_is_set(flags, pcmk__node_search_cluster)) { + node = pcmk__search_cluster_node_cache(id, uname, NULL); + } ++ ++ if ((node == NULL) && pcmk_is_set(flags, pcmk__node_search_known)) { ++ char *id_str = (id == 0)? NULL : crm_strdup_printf("%u", id); ++ ++ node = find_known_node(id_str, uname); ++ free(id_str); ++ } ++ + return node; + } + +@@ -1343,42 +1352,6 @@ pcmk__refresh_node_caches_from_cib(xmlNode *cib) + refresh_known_node_cache(cib); + } + +-/*! +- * \internal +- * \brief Search known node cache +- * +- * \param[in] id If not 0, cluster node ID to search for +- * \param[in] uname If not NULL, node name to search for +- * \param[in] flags Group of enum pcmk__node_search_flags +- * +- * \return Known node cache entry if found, otherwise NULL +- */ +-crm_node_t * +-pcmk__search_known_node_cache(unsigned int id, const char *uname, +- uint32_t flags) +-{ +- crm_node_t *node = NULL; +- char *id_str = NULL; +- +- CRM_ASSERT(id > 0 || uname != NULL); +- +- node = pcmk__search_node_caches(id, uname, flags); +- +- if (node || !(flags & pcmk__node_search_cluster)) { +- return node; +- } +- +- if (id > 0) { +- id_str = crm_strdup_printf("%u", id); +- } +- +- node = find_known_node(id_str, uname); +- +- free(id_str); +- return node; +-} +- +- + // Deprecated functions kept only for backward API compatibility + // LCOV_EXCL_START + +-- +2.31.1 + +From 5b64c943bd8ba82b06e803fa97737fb7b574ec04 Mon Sep 17 00:00:00 2001 +From: Ken Gaillot +Date: Tue, 19 Dec 2023 16:38:10 -0600 +Subject: [PATCH 7/9] Refactor: libcrmcluster: replace + pcmk__search_cluster_node_cache() + +... with calls to pcmk__search_node_caches() using pcmk__node_search_cluster +where possible +--- + daemons/attrd/attrd_ipc.c | 5 +++-- + daemons/based/based_messages.c | 5 ++++- + daemons/controld/controld_corosync.c | 4 ++-- + daemons/controld/controld_messages.c | 6 ++++-- + lib/cluster/cluster.c | 3 ++- + lib/cluster/cpg.c | 4 ++-- + lib/cluster/membership.c | 2 +- + 7 files changed, 18 insertions(+), 11 deletions(-) + +diff --git a/daemons/attrd/attrd_ipc.c b/daemons/attrd/attrd_ipc.c +index 05c4a69..b08963d 100644 +--- a/daemons/attrd/attrd_ipc.c ++++ b/daemons/attrd/attrd_ipc.c +@@ -162,10 +162,11 @@ attrd_client_peer_remove(pcmk__request_t *request) + + crm_element_value_int(xml, PCMK__XA_ATTR_NODE_ID, &nodeid); + if (nodeid > 0) { +- crm_node_t *node = pcmk__search_cluster_node_cache(nodeid, NULL, +- NULL); ++ crm_node_t *node = NULL; + char *host_alloc = NULL; + ++ node = pcmk__search_node_caches(nodeid, NULL, ++ pcmk__node_search_cluster); + if (node && node->uname) { + // Use cached name if available + host = node->uname; +diff --git a/daemons/based/based_messages.c b/daemons/based/based_messages.c +index 7f503b2..efad9a7 100644 +--- a/daemons/based/based_messages.c ++++ b/daemons/based/based_messages.c +@@ -247,7 +247,10 @@ cib_process_upgrade_server(const char *op, int options, const char *section, xml + + if (rc != pcmk_ok) { + // Notify originating peer so it can notify its local clients +- crm_node_t *origin = pcmk__search_cluster_node_cache(0, host, NULL); ++ crm_node_t *origin = NULL; ++ ++ origin = pcmk__search_node_caches(0, host, ++ pcmk__node_search_cluster); + + crm_info("Rejecting upgrade request from %s: %s " + CRM_XS " rc=%d peer=%s", host, pcmk_strerror(rc), rc, +diff --git a/daemons/controld/controld_corosync.c b/daemons/controld/controld_corosync.c +index 0f3ea32..63184d2 100644 +--- a/daemons/controld/controld_corosync.c ++++ b/daemons/controld/controld_corosync.c +@@ -119,8 +119,8 @@ cpg_membership_callback(cpg_handle_t handle, const struct cpg_name *cpg_name, + if (controld_globals.dc_name != NULL) { + crm_node_t *peer = NULL; + +- peer = pcmk__search_cluster_node_cache(0, controld_globals.dc_name, +- NULL); ++ peer = pcmk__search_node_caches(0, controld_globals.dc_name, ++ pcmk__node_search_cluster); + if (peer != NULL) { + for (int i = 0; i < left_list_entries; ++i) { + if (left_list[i].nodeid == peer->id) { +diff --git a/daemons/controld/controld_messages.c b/daemons/controld/controld_messages.c +index 999dd13..bd5237e 100644 +--- a/daemons/controld/controld_messages.c ++++ b/daemons/controld/controld_messages.c +@@ -485,7 +485,8 @@ relay_message(xmlNode * msg, gboolean originated_locally) + } + + if (!broadcast) { +- node_to = pcmk__search_cluster_node_cache(0, host_to, NULL); ++ node_to = pcmk__search_node_caches(0, host_to, ++ pcmk__node_search_cluster); + if (node_to == NULL) { + crm_warn("Ignoring message %s because node %s is unknown", + ref, host_to); +@@ -1029,7 +1030,8 @@ handle_request(xmlNode *stored_msg, enum crmd_fsa_cause cause) + + if (strcmp(op, CRM_OP_SHUTDOWN_REQ) == 0) { + const char *from = crm_element_value(stored_msg, F_CRM_HOST_FROM); +- crm_node_t *node = pcmk__search_cluster_node_cache(0, from, NULL); ++ crm_node_t *node = pcmk__search_node_caches(0, from, ++ pcmk__node_search_cluster); + + pcmk__update_peer_expected(__func__, node, CRMD_JOINSTATE_DOWN); + if(AM_I_DC == FALSE) { +diff --git a/lib/cluster/cluster.c b/lib/cluster/cluster.c +index f2cd428..1cdc204 100644 +--- a/lib/cluster/cluster.c ++++ b/lib/cluster/cluster.c +@@ -280,7 +280,8 @@ crm_peer_uname(const char *uuid) + return NULL; + } + +- node = pcmk__search_cluster_node_cache((uint32_t) id, NULL, NULL); ++ node = pcmk__search_node_caches((uint32_t) id, NULL, ++ pcmk__node_search_cluster); + if (node != NULL) { + crm_info("Setting uuid for node %s[%u] to %s", + node->uname, node->id, uuid); +diff --git a/lib/cluster/cpg.c b/lib/cluster/cpg.c +index b5f2884..4f3e81c 100644 +--- a/lib/cluster/cpg.c ++++ b/lib/cluster/cpg.c +@@ -629,8 +629,8 @@ node_left(const char *cpg_group_name, int event_counter, + const struct cpg_address **sorted_member_list, + size_t member_list_entries) + { +- crm_node_t *peer = pcmk__search_cluster_node_cache(cpg_peer->nodeid, +- NULL, NULL); ++ crm_node_t *peer = pcmk__search_node_caches(cpg_peer->nodeid, NULL, ++ pcmk__node_search_cluster); + const struct cpg_address **rival = NULL; + + /* Most CPG-related Pacemaker code assumes that only one process on a node +diff --git a/lib/cluster/membership.c b/lib/cluster/membership.c +index ef4aaac..73ea1e3 100644 +--- a/lib/cluster/membership.c ++++ b/lib/cluster/membership.c +@@ -122,7 +122,7 @@ crm_remote_peer_get(const char *node_name) + * entry unless it has a node ID, which means the name actually is + * associated with a cluster node. (@TODO return an error in that case?) + */ +- node = pcmk__search_cluster_node_cache(0, node_name, NULL); ++ node = pcmk__search_node_caches(0, node_name, pcmk__node_search_cluster); + if ((node != NULL) && (node->uuid == NULL)) { + /* node_name could be a pointer into the cache entry being removed, so + * reassign it to a copy before the original gets freed +-- +2.31.1 + +From cbeb9eb516d3bf29df7850dcf2a8515f6a0dfb2c Mon Sep 17 00:00:00 2001 +From: Ken Gaillot +Date: Mon, 18 Dec 2023 17:09:12 -0600 +Subject: [PATCH 8/9] Test: cts-cli: strip feature set out of reference output + +--- + cts/cli/regression.tools.exp | 4 ++-- + cts/cts-cli.in | 2 ++ + 2 files changed, 4 insertions(+), 2 deletions(-) + +diff --git a/cts/cli/regression.tools.exp b/cts/cli/regression.tools.exp +index accf781..417b5cd 100644 +--- a/cts/cli/regression.tools.exp ++++ b/cts/cli/regression.tools.exp +@@ -7667,7 +7667,7 @@ Diff: +++ 0.1.0 (null) + -- /cib/status/node_state[@id='1'] + -- /cib/status/node_state[@id='httpd-bundle-0'] + -- /cib/status/node_state[@id='httpd-bundle-1'] +-+ /cib: @crm_feature_set=3.19.0, @num_updates=0, @admin_epoch=0 +++ /cib: @num_updates=0, @admin_epoch=0 + -- /cib: @cib-last-written, @update-origin, @update-client, @update-user, @have-quorum, @dc-uuid + =#=#=#= End test: Get active shadow instance's diff (empty CIB) - Error occurred (1) =#=#=#= + * Passed: crm_shadow - Get active shadow instance's diff (empty CIB) +@@ -7701,7 +7701,7 @@ Diff: +++ 0.1.0 (null) + + + +- ++ + + + +diff --git a/cts/cts-cli.in b/cts/cts-cli.in +index f4cb7c3..40ada49 100755 +--- a/cts/cts-cli.in ++++ b/cts/cts-cli.in +@@ -3357,7 +3357,9 @@ for t in $tests; do + -e 's/ version="[^"]*"/ version=""/' \ + -e 's/request=\".*\(crm_[a-zA-Z0-9]*\)/request=\"\1/' \ + -e 's/crm_feature_set="[^"]*" //'\ ++ -e 's/@crm_feature_set=[0-9.]*, //'\ + -e 's/validate-with="[^"]*" //'\ ++ -e 's/\( +Date: Tue, 2 Jan 2024 14:48:14 -0600 +Subject: [PATCH 9/9] Test: cts-lab: ignore all transition calculation log + messages + +9e28f3b6d means these are now possible for more ignorable errors +--- + python/pacemaker/_cts/patterns.py | 26 +++++++------------------- + 1 file changed, 7 insertions(+), 19 deletions(-) + +diff --git a/python/pacemaker/_cts/patterns.py b/python/pacemaker/_cts/patterns.py +index 0fb1c2b..d05ff5f 100644 +--- a/python/pacemaker/_cts/patterns.py ++++ b/python/pacemaker/_cts/patterns.py +@@ -1,7 +1,7 @@ + """ Pattern-holding classes for Pacemaker's Cluster Test Suite (CTS) """ + + __all__ = ["PatternSelector"] +-__copyright__ = "Copyright 2008-2023 the Pacemaker project contributors" ++__copyright__ = "Copyright 2008-2024 the Pacemaker project contributors" + __license__ = "GNU General Public License version 2 or later (GPLv2+)" + + import argparse +@@ -32,6 +32,12 @@ class BasePatterns: + # pcs can log this when node is fenced, but fencing is OK in some + # tests (and we will catch it in pacemaker logs when not OK) + r"pcs.daemon:No response from: .* request: get_configs, error:", ++ ++ # This is overbroad, but there's no way to say that only certain ++ # transition errors are acceptable. We have to rely on causes of a ++ # transition error logging their own error message, which should ++ # always be the case. ++ r"pacemaker-schedulerd.* Calculated transition .*/pe-error", + ] + + self._commands = { +@@ -239,12 +245,6 @@ class Corosync2Patterns(BasePatterns): + r"error:.*cib_(shm|rw) IPC provider disconnected while waiting", + r"error:.*Connection to (fencer|stonith-ng).* (closed|failed|lost)", + r"error: Lost fencer connection", +- # This is overbroad, but we don't have a way to say that only +- # certain transition errors are acceptable (if the fencer respawns, +- # fence devices may appear multiply active). We have to rely on +- # other causes of a transition error logging their own error +- # message, which is the usual practice. +- r"pacemaker-schedulerd.* Calculated transition .*/pe-error", + ] + + self._components["corosync"] = [ +@@ -281,12 +281,6 @@ class Corosync2Patterns(BasePatterns): + r"pacemaker-execd.*Connection to (fencer|stonith-ng).* (closed|failed|lost)", + r"pacemaker-controld.*:\s+Result of .* operation for Fencing.*Error \(Lost connection to fencer\)", + r"pacemaker-controld.*:Could not connect to attrd: Connection refused", +- # This is overbroad, but we don't have a way to say that only +- # certain transition errors are acceptable (if the fencer respawns, +- # fence devices may appear multiply active). We have to rely on +- # other causes of a transition error logging their own error +- # message, which is the usual practice. +- r"pacemaker-schedulerd.* Calculated transition .*/pe-error", + ] + + self._components["pacemaker-execd"] = [ +@@ -338,12 +332,6 @@ class Corosync2Patterns(BasePatterns): + r"error:.*Lost fencer connection", + r"error:.*Fencer connection failed \(will retry\)", + r"pacemaker-controld.*:\s+Result of .* operation for Fencing.*Error \(Lost connection to fencer\)", +- # This is overbroad, but we don't have a way to say that only +- # certain transition errors are acceptable (if the fencer respawns, +- # fence devices may appear multiply active). We have to rely on +- # other causes of a transition error logging their own error +- # message, which is the usual practice. +- r"pacemaker-schedulerd.* Calculated transition .*/pe-error", + ] + + self._components["pacemaker-fenced-ignore"].extend(self._components["common-ignore"]) +-- +2.31.1 + diff --git a/SOURCES/005-attrd-dampen.patch b/SOURCES/005-attrd-dampen.patch deleted file mode 100644 index 80c8a67..0000000 --- a/SOURCES/005-attrd-dampen.patch +++ /dev/null @@ -1,26 +0,0 @@ -From ebac530c815a62f7c3a1c24f64e9a530d9753dbe Mon Sep 17 00:00:00 2001 -From: Hideo Yamauchi -Date: Wed, 19 Jul 2023 18:21:07 +0900 -Subject: [PATCH] High: tools: The dampen parameter is disabled when setting - values with attrd_updater. - ---- - tools/attrd_updater.c | 2 +- - 1 file changed, 1 insertion(+), 1 deletion(-) - -diff --git a/tools/attrd_updater.c b/tools/attrd_updater.c -index b615a3575..4688b9ff6 100644 ---- a/tools/attrd_updater.c -+++ b/tools/attrd_updater.c -@@ -501,7 +501,7 @@ send_attrd_update(char command, const char *attr_node, const char *attr_name, - - case 'U': - rc = pcmk__attrd_api_update(NULL, attr_node, attr_name, attr_value, -- NULL, attr_set, NULL, -+ attr_dampen, attr_set, NULL, - attr_options | pcmk__node_attr_value); - break; - --- -2.41.0 - diff --git a/SOURCES/006-cib-file-feature-set.patch b/SOURCES/006-cib-file-feature-set.patch new file mode 100644 index 0000000..a7ce74d --- /dev/null +++ b/SOURCES/006-cib-file-feature-set.patch @@ -0,0 +1,276 @@ +From d50bbafc32428e873c0052a9defcf93d2e52667e Mon Sep 17 00:00:00 2001 +From: Chris Lumens +Date: Wed, 10 Jan 2024 11:35:11 -0500 +Subject: [PATCH 1/3] Refactor: libcrmcommon: Split feature set check into its + own function. + +--- + include/crm/common/cib_internal.h | 4 +++- + lib/cib/cib_utils.c | 12 ++++++------ + lib/common/cib.c | 18 +++++++++++++++++- + 3 files changed, 26 insertions(+), 8 deletions(-) + +diff --git a/include/crm/common/cib_internal.h b/include/crm/common/cib_internal.h +index c41c12e..fa65e58 100644 +--- a/include/crm/common/cib_internal.h ++++ b/include/crm/common/cib_internal.h +@@ -1,5 +1,5 @@ + /* +- * Copyright 2023 the Pacemaker project contributors ++ * Copyright 2023-2024 the Pacemaker project contributors + * + * The version control history for this file may have further details. + * +@@ -16,6 +16,8 @@ extern "C" { + + const char *pcmk__cib_abs_xpath_for(const char *element); + ++int pcmk__check_feature_set(const char *cib_version); ++ + #ifdef __cplusplus + } + #endif +diff --git a/lib/cib/cib_utils.c b/lib/cib/cib_utils.c +index 0082eef..bf2982c 100644 +--- a/lib/cib/cib_utils.c ++++ b/lib/cib/cib_utils.c +@@ -353,7 +353,6 @@ cib_perform_op(const char *op, int call_options, cib__op_fn_t fn, bool is_query, + xmlNode *patchset_cib = NULL; + xmlNode *local_diff = NULL; + +- const char *new_version = NULL; + const char *user = crm_element_value(req, F_CIB_USER); + bool with_digest = false; + +@@ -470,12 +469,13 @@ cib_perform_op(const char *op, int call_options, cib__op_fn_t fn, bool is_query, + } + + if (scratch) { +- new_version = crm_element_value(scratch, XML_ATTR_CRM_VERSION); ++ const char *new_version = crm_element_value(scratch, XML_ATTR_CRM_VERSION); + +- if (new_version && compare_version(new_version, CRM_FEATURE_SET) > 0) { +- crm_err("Discarding update with feature set '%s' greater than our own '%s'", +- new_version, CRM_FEATURE_SET); +- rc = -EPROTONOSUPPORT; ++ rc = pcmk__check_feature_set(new_version); ++ if (rc != pcmk_rc_ok) { ++ pcmk__config_err("Discarding update with feature set '%s' greater than our own '%s'", ++ new_version, CRM_FEATURE_SET); ++ rc = pcmk_rc2legacy(rc); + goto done; + } + } +diff --git a/lib/common/cib.c b/lib/common/cib.c +index fee7881..cbebc2e 100644 +--- a/lib/common/cib.c ++++ b/lib/common/cib.c +@@ -1,6 +1,6 @@ + /* + * Original copyright 2004 International Business Machines +- * Later changes copyright 2008-2023 the Pacemaker project contributors ++ * Later changes copyright 2008-2024 the Pacemaker project contributors + * + * The version control history for this file may have further details. + * +@@ -173,3 +173,19 @@ pcmk_find_cib_element(xmlNode *cib, const char *element_name) + { + return get_xpath_object(pcmk_cib_xpath_for(element_name), cib, LOG_TRACE); + } ++ ++/*! ++ * \internal ++ * \brief Check that the feature set in the CIB is supported on this node ++ * ++ * \param[in] new_version XML_ATTR_CRM_VERSION attribute from the CIB ++ */ ++int ++pcmk__check_feature_set(const char *cib_version) ++{ ++ if (cib_version && compare_version(cib_version, CRM_FEATURE_SET) > 0) { ++ return EPROTONOSUPPORT; ++ } ++ ++ return pcmk_rc_ok; ++} +-- +2.31.1 + +From d89fd8336ae47d892201513c99773705d57f15f0 Mon Sep 17 00:00:00 2001 +From: Chris Lumens +Date: Wed, 10 Jan 2024 13:46:42 -0500 +Subject: [PATCH 2/3] Feature: scheduler: Check the CIB feature set in + cluster_status. + +This adds the check that was previously only in cib_perform_op to the +scheduler code, ensuring that any daemon or tool that calls the +scheduler will check that the feature set in the CIB is supported. +--- + lib/pengine/status.c | 10 ++++++++++ + 1 file changed, 10 insertions(+) + +diff --git a/lib/pengine/status.c b/lib/pengine/status.c +index e6ec237..1294803 100644 +--- a/lib/pengine/status.c ++++ b/lib/pengine/status.c +@@ -14,6 +14,7 @@ + #include + #include + #include ++#include + + #include + +@@ -70,12 +71,21 @@ pe_free_working_set(pcmk_scheduler_t *scheduler) + gboolean + cluster_status(pcmk_scheduler_t * scheduler) + { ++ const char *new_version = NULL; + xmlNode *section = NULL; + + if ((scheduler == NULL) || (scheduler->input == NULL)) { + return FALSE; + } + ++ new_version = crm_element_value(scheduler->input, XML_ATTR_CRM_VERSION); ++ ++ if (pcmk__check_feature_set(new_version) != pcmk_rc_ok) { ++ pcmk__config_err("Can't process CIB with feature set '%s' greater than our own '%s'", ++ new_version, CRM_FEATURE_SET); ++ return FALSE; ++ } ++ + crm_trace("Beginning unpack"); + + if (scheduler->failed != NULL) { +-- +2.31.1 + +From a3428926d37af506014a6b462d1308d8541c5932 Mon Sep 17 00:00:00 2001 +From: Chris Lumens +Date: Wed, 10 Jan 2024 14:56:36 -0500 +Subject: [PATCH 3/3] Low: libcib: Do not check CIB feature set for files in + cib_perform_op. + +This is related to the previous feature for transferring schema files to +older remote nodes. In that case, the newer schema files may also have +a newer feature set than the node supports, so the transferred files are +still not usable. + +However, the feature set only matters for the scheduler, not for most +command line tools (obviously, crm_simulate would still care). So in +those cases, we can just disable the feature set check if the CIB was +read in from a file. For the scheduler, the check is still performed as +part of cluster_status. +--- + cts/cli/regression.tools.exp | 2 +- + daemons/based/based_callbacks.c | 4 ++-- + include/crm/cib/internal.h | 4 ++-- + lib/cib/cib_file.c | 2 +- + lib/cib/cib_utils.c | 15 +++++++++------ + 5 files changed, 15 insertions(+), 12 deletions(-) + +diff --git a/cts/cli/regression.tools.exp b/cts/cli/regression.tools.exp +index 417b5cd..c81c420 100644 +--- a/cts/cli/regression.tools.exp ++++ b/cts/cli/regression.tools.exp +@@ -7939,7 +7939,7 @@ unpack_config warning: Blind faith: not fencing unseen nodes + =#=#=#= End test: Verbosely verify a file-specified invalid configuration, outputting as xml - Invalid configuration (78) =#=#=#= + * Passed: crm_verify - Verbosely verify a file-specified invalid configuration, outputting as xml + =#=#=#= Begin test: Verbosely verify another file-specified invalid configuration, outputting as xml =#=#=#= +-(cluster_status@status.c:113) warning: Fencing and resource management disabled due to lack of quorum ++(cluster_status@status.c:123) warning: Fencing and resource management disabled due to lack of quorum + + + +diff --git a/daemons/based/based_callbacks.c b/daemons/based/based_callbacks.c +index 5f3dc62..f16e4d9 100644 +--- a/daemons/based/based_callbacks.c ++++ b/daemons/based/based_callbacks.c +@@ -1362,7 +1362,7 @@ cib_process_command(xmlNode *request, const cib__operation_t *operation, + input = prepare_input(request, operation->type, §ion); + + if (!pcmk_is_set(operation->flags, cib__op_attr_modifies)) { +- rc = cib_perform_op(op, call_options, op_function, true, section, ++ rc = cib_perform_op(NULL, op, call_options, op_function, true, section, + request, input, false, &config_changed, &the_cib, + &result_cib, NULL, &output); + +@@ -1395,7 +1395,7 @@ cib_process_command(xmlNode *request, const cib__operation_t *operation, + } + + // result_cib must not be modified after cib_perform_op() returns +- rc = cib_perform_op(op, call_options, op_function, false, section, ++ rc = cib_perform_op(NULL, op, call_options, op_function, false, section, + request, input, manage_counters, &config_changed, + &the_cib, &result_cib, cib_diff, &output); + +diff --git a/include/crm/cib/internal.h b/include/crm/cib/internal.h +index 9d54d52..b6d6871 100644 +--- a/include/crm/cib/internal.h ++++ b/include/crm/cib/internal.h +@@ -1,5 +1,5 @@ + /* +- * Copyright 2004-2023 the Pacemaker project contributors ++ * Copyright 2004-2024 the Pacemaker project contributors + * + * The version control history for this file may have further details. + * +@@ -206,7 +206,7 @@ int cib__get_notify_patchset(const xmlNode *msg, const xmlNode **patchset); + + bool cib__element_in_patchset(const xmlNode *patchset, const char *element); + +-int cib_perform_op(const char *op, int call_options, cib__op_fn_t fn, ++int cib_perform_op(cib_t *cib, const char *op, int call_options, cib__op_fn_t fn, + bool is_query, const char *section, xmlNode *req, + xmlNode *input, bool manage_counters, bool *config_changed, + xmlNode **current_cib, xmlNode **result_cib, xmlNode **diff, +diff --git a/lib/cib/cib_file.c b/lib/cib/cib_file.c +index a279823..9dd952c 100644 +--- a/lib/cib/cib_file.c ++++ b/lib/cib/cib_file.c +@@ -245,7 +245,7 @@ cib_file_process_request(cib_t *cib, xmlNode *request, xmlNode **output) + data = pcmk_find_cib_element(data, section); + } + +- rc = cib_perform_op(op, call_options, op_function, read_only, section, ++ rc = cib_perform_op(cib, op, call_options, op_function, read_only, section, + request, data, true, &changed, &private->cib_xml, + &result_cib, &cib_diff, output); + +diff --git a/lib/cib/cib_utils.c b/lib/cib/cib_utils.c +index bf2982c..9c3f9f1 100644 +--- a/lib/cib/cib_utils.c ++++ b/lib/cib/cib_utils.c +@@ -339,11 +339,10 @@ should_copy_cib(const char *op, const char *section, int call_options) + } + + int +-cib_perform_op(const char *op, int call_options, cib__op_fn_t fn, bool is_query, +- const char *section, xmlNode *req, xmlNode *input, +- bool manage_counters, bool *config_changed, +- xmlNode **current_cib, xmlNode **result_cib, xmlNode **diff, +- xmlNode **output) ++cib_perform_op(cib_t *cib, const char *op, int call_options, cib__op_fn_t fn, ++ bool is_query, const char *section, xmlNode *req, xmlNode *input, ++ bool manage_counters, bool *config_changed, xmlNode **current_cib, ++ xmlNode **result_cib, xmlNode **diff, xmlNode **output) + { + int rc = pcmk_ok; + bool check_schema = true; +@@ -468,7 +467,11 @@ cib_perform_op(const char *op, int call_options, cib__op_fn_t fn, bool is_query, + goto done; + } + +- if (scratch) { ++ /* If the CIB is from a file, we don't need to check that the feature set is ++ * supported. All we care about in that case is the schema version, which ++ * is checked elsewhere. ++ */ ++ if (scratch && (cib == NULL || cib->variant != cib_file)) { + const char *new_version = crm_element_value(scratch, XML_ATTR_CRM_VERSION); + + rc = pcmk__check_feature_set(new_version); +-- +2.31.1 + diff --git a/SOURCES/006-controller-reply.patch b/SOURCES/006-controller-reply.patch deleted file mode 100644 index efd4f9c..0000000 --- a/SOURCES/006-controller-reply.patch +++ /dev/null @@ -1,109 +0,0 @@ -From 3e31da0016795397bfeacb2f3d76ecfe35cc1f67 Mon Sep 17 00:00:00 2001 -From: Ken Gaillot -Date: Mon, 17 Jul 2023 14:52:42 -0500 -Subject: [PATCH] Fix: libcrmcommon: wait for reply from appropriate controller - commands - -ipc_controld.c:reply_expected() wrongly omitted PCMK__CONTROLD_CMD_NODES (which -hasn't been a problem because crm_node uses a mainloop instead of sync dispatch -for that) and CRM_OP_RM_NODE_CACHE (which can be sent via -ipc_client.c:pcmk_ipc_purge_node()). - -Because CRM_OP_RM_NODE_CACHE gets only an ack and no further replies, we now -have to be careful not to return true from the controller's dispatch() -function, otherwise crm_node -R would wait forever for more data. That means -we have to check for whether any replies are expected, which means we have to -increment expected replies *before* sending a request (in case it's sync). - -Regression introduced in 2.0.5 by ae14fa4a - -Fixes T681 ---- - lib/common/ipc_controld.c | 49 ++++++++++++++------------------------- - 1 file changed, 17 insertions(+), 32 deletions(-) - -diff --git a/lib/common/ipc_controld.c b/lib/common/ipc_controld.c -index 3c3a98964..405fd0518 100644 ---- a/lib/common/ipc_controld.c -+++ b/lib/common/ipc_controld.c -@@ -177,18 +177,16 @@ set_nodes_data(pcmk_controld_api_reply_t *data, xmlNode *msg_data) - static bool - reply_expected(pcmk_ipc_api_t *api, xmlNode *request) - { -- const char *command = crm_element_value(request, F_CRM_TASK); -- -- if (command == NULL) { -- return false; -- } -- -- // We only need to handle commands that functions in this file can send -- return !strcmp(command, CRM_OP_REPROBE) -- || !strcmp(command, CRM_OP_NODE_INFO) -- || !strcmp(command, CRM_OP_PING) -- || !strcmp(command, CRM_OP_LRM_FAIL) -- || !strcmp(command, CRM_OP_LRM_DELETE); -+ // We only need to handle commands that API functions can send -+ return pcmk__str_any_of(crm_element_value(request, F_CRM_TASK), -+ PCMK__CONTROLD_CMD_NODES, -+ CRM_OP_LRM_DELETE, -+ CRM_OP_LRM_FAIL, -+ CRM_OP_NODE_INFO, -+ CRM_OP_PING, -+ CRM_OP_REPROBE, -+ CRM_OP_RM_NODE_CACHE, -+ NULL); - } - - static bool -@@ -202,22 +200,12 @@ dispatch(pcmk_ipc_api_t *api, xmlNode *reply) - pcmk_controld_reply_unknown, NULL, NULL, - }; - -- /* If we got an ACK, return true so the caller knows to expect more responses -- * from the IPC server. We do this before decrementing replies_expected because -- * ACKs are not going to be included in that value. -- * -- * Note that we cannot do the same kind of status checking here that we do in -- * ipc_pacemakerd.c. The ACK message we receive does not necessarily contain -- * a status attribute. That is, we may receive this: -- * -- * -- * -- * Instead of this: -- * -- * -- */ - if (pcmk__str_eq(crm_element_name(reply), "ack", pcmk__str_none)) { -- return true; // More replies needed -+ /* ACKs are trivial responses that do not count toward expected replies, -+ * and do not have all the fields that validation requires, so skip that -+ * processing. -+ */ -+ return private->replies_expected > 0; - } - - if (private->replies_expected > 0) { -@@ -344,18 +332,15 @@ static int - send_controller_request(pcmk_ipc_api_t *api, xmlNode *request, - bool reply_is_expected) - { -- int rc; -- - if (crm_element_value(request, XML_ATTR_REFERENCE) == NULL) { - return EINVAL; - } -- rc = pcmk__send_ipc_request(api, request); -- if ((rc == pcmk_rc_ok) && reply_is_expected) { -+ if (reply_is_expected) { - struct controld_api_private_s *private = api->api_data; - - private->replies_expected++; - } -- return rc; -+ return pcmk__send_ipc_request(api, request); - } - - static xmlNode * --- -2.41.0 - diff --git a/SOURCES/007-glib-assertions.patch b/SOURCES/007-glib-assertions.patch deleted file mode 100644 index 5679ee6..0000000 --- a/SOURCES/007-glib-assertions.patch +++ /dev/null @@ -1,163 +0,0 @@ -From 63f4bd4d5a324e6eb279340a42c7c36c8902ada7 Mon Sep 17 00:00:00 2001 -From: Ken Gaillot -Date: Wed, 2 Aug 2023 15:55:26 -0500 -Subject: [PATCH 1/4] Fix: controller: don't try to execute agent action at - shutdown - -Normally, agent execution is not possible at shutdown. However, when metadata -is needed for some action, the agent can be called asynchronously, and when the -metadata action returns, the original action is performed. If the metadata is -initiated before shutdown, but completes after shutdown has begun, do not try -to attempt the original action, so we avoid unnecessary error logs. ---- - daemons/controld/controld_execd.c | 4 +++- - 1 file changed, 3 insertions(+), 1 deletion(-) - -diff --git a/daemons/controld/controld_execd.c b/daemons/controld/controld_execd.c -index 530e4346c8..a90e8d833e 100644 ---- a/daemons/controld/controld_execd.c -+++ b/daemons/controld/controld_execd.c -@@ -1400,7 +1400,9 @@ metadata_complete(int pid, const pcmk__action_result_t *result, void *user_data) - md = controld_cache_metadata(lrm_state->metadata_cache, data->rsc, - result->action_stdout); - } -- do_lrm_rsc_op(lrm_state, data->rsc, data->input_xml, md); -+ if (!pcmk_is_set(controld_globals.fsa_input_register, R_HA_DISCONNECTED)) { -+ do_lrm_rsc_op(lrm_state, data->rsc, data->input_xml, md); -+ } - free_metadata_cb_data(data); - } - - -From 247d9534f36f690c1474e36cedaadb3934022a05 Mon Sep 17 00:00:00 2001 -From: Ken Gaillot -Date: Wed, 2 Aug 2023 16:16:31 -0500 -Subject: [PATCH 2/4] Refactor: controller: de-functionize lrm_state_destroy() - -It was a one-liner called once ---- - daemons/controld/controld_execd_state.c | 8 +------- - daemons/controld/controld_lrm.h | 5 ----- - 2 files changed, 1 insertion(+), 12 deletions(-) - -diff --git a/daemons/controld/controld_execd_state.c b/daemons/controld/controld_execd_state.c -index 8c68bfca08..4a87a9b332 100644 ---- a/daemons/controld/controld_execd_state.c -+++ b/daemons/controld/controld_execd_state.c -@@ -132,12 +132,6 @@ lrm_state_create(const char *node_name) - return state; - } - --void --lrm_state_destroy(const char *node_name) --{ -- g_hash_table_remove(lrm_state_table, node_name); --} -- - static gboolean - remote_proxy_remove_by_node(gpointer key, gpointer value, gpointer user_data) - { -@@ -799,7 +793,7 @@ lrm_state_unregister_rsc(lrm_state_t * lrm_state, - } - - if (is_remote_lrmd_ra(NULL, NULL, rsc_id)) { -- lrm_state_destroy(rsc_id); -+ g_hash_table_remove(lrm_state_table, rsc_id); - return pcmk_ok; - } - -diff --git a/daemons/controld/controld_lrm.h b/daemons/controld/controld_lrm.h -index 25f3db3316..c3113e49c3 100644 ---- a/daemons/controld/controld_lrm.h -+++ b/daemons/controld/controld_lrm.h -@@ -108,11 +108,6 @@ gboolean lrm_state_init_local(void); - */ - void lrm_state_destroy_all(void); - --/*! -- * \brief Destroy executor connection by node name -- */ --void lrm_state_destroy(const char *node_name); -- - /*! - * \brief Find lrm_state data by node name - */ - -From 1b915f1ce38756431f7faa142565e3e07aade194 Mon Sep 17 00:00:00 2001 -From: Ken Gaillot -Date: Wed, 2 Aug 2023 15:58:09 -0500 -Subject: [PATCH 3/4] Low: controller: guard lrm_state_table usage with NULL - check - -It is NULL while draining the mainloop during the shutdown sequence. ---- - daemons/controld/controld_execd_state.c | 7 ++++++- - 1 file changed, 6 insertions(+), 1 deletion(-) - -diff --git a/daemons/controld/controld_execd_state.c b/daemons/controld/controld_execd_state.c -index 4a87a9b332..b90cc5e635 100644 ---- a/daemons/controld/controld_execd_state.c -+++ b/daemons/controld/controld_execd_state.c -@@ -301,7 +301,7 @@ lrm_state_destroy_all(void) - lrm_state_t * - lrm_state_find(const char *node_name) - { -- if (!node_name) { -+ if ((node_name == NULL) || (lrm_state_table == NULL)) { - return NULL; - } - return g_hash_table_lookup(lrm_state_table, node_name); -@@ -312,6 +312,8 @@ lrm_state_find_or_create(const char *node_name) - { - lrm_state_t *lrm_state; - -+ CRM_CHECK(lrm_state_table != NULL, return NULL); -+ - lrm_state = g_hash_table_lookup(lrm_state_table, node_name); - if (!lrm_state) { - lrm_state = lrm_state_create(node_name); -@@ -323,6 +325,9 @@ lrm_state_find_or_create(const char *node_name) - GList * - lrm_state_get_list(void) - { -+ if (lrm_state_table == NULL) { -+ return NULL; -+ } - return g_hash_table_get_values(lrm_state_table); - } - - -From 78581213ed3bf4183b0ec1f391b720d5d91f3f68 Mon Sep 17 00:00:00 2001 -From: Ken Gaillot -Date: Wed, 2 Aug 2023 15:48:36 -0500 -Subject: [PATCH 4/4] Log: controller: improve messages for resource history - updates - ---- - daemons/controld/controld_cib.c | 11 +++++++++-- - 1 file changed, 9 insertions(+), 2 deletions(-) - -diff --git a/daemons/controld/controld_cib.c b/daemons/controld/controld_cib.c -index 22ac42486f..c9dde0b748 100644 ---- a/daemons/controld/controld_cib.c -+++ b/daemons/controld/controld_cib.c -@@ -861,10 +861,17 @@ cib_rsc_callback(xmlNode * msg, int call_id, int rc, xmlNode * output, void *use - case pcmk_ok: - case -pcmk_err_diff_failed: - case -pcmk_err_diff_resync: -- crm_trace("Resource update %d complete: rc=%d", call_id, rc); -+ crm_trace("Resource history update completed (call=%d rc=%d)", -+ call_id, rc); - break; - default: -- crm_warn("Resource update %d failed: (rc=%d) %s", call_id, rc, pcmk_strerror(rc)); -+ if (call_id > 0) { -+ crm_warn("Resource history update %d failed: %s " -+ CRM_XS " rc=%d", call_id, pcmk_strerror(rc), rc); -+ } else { -+ crm_warn("Resource history update failed: %s " CRM_XS " rc=%d", -+ pcmk_strerror(rc), rc); -+ } - } - - if (call_id == pending_rsc_update) { diff --git a/SOURCES/007-option-metadata.patch b/SOURCES/007-option-metadata.patch new file mode 100644 index 0000000..d948042 --- /dev/null +++ b/SOURCES/007-option-metadata.patch @@ -0,0 +1,3689 @@ +From 4401064f409921caed9444d18a74713250213c44 Mon Sep 17 00:00:00 2001 +From: Reid Wahl +Date: Tue, 2 Jan 2024 19:52:41 -0800 +Subject: [PATCH 01/24] Test: cts-cli: Update for added spaces + +Signed-off-by: Reid Wahl +--- + cts/cli/regression.daemons.exp | 4 ++-- + 1 file changed, 2 insertions(+), 2 deletions(-) + +diff --git a/cts/cli/regression.daemons.exp b/cts/cli/regression.daemons.exp +index b34fba8..1cd049f 100644 +--- a/cts/cli/regression.daemons.exp ++++ b/cts/cli/regression.daemons.exp +@@ -47,7 +47,7 @@ + + + +- Pacemaker is primarily event-driven, and looks ahead to know when to recheck cluster state for failure timeouts and most time-based rules. However, it will also recheck the cluster after this amount of inactivity, to evaluate rules with date specifications and serve as a fail-safe for certain types of scheduler bugs. Allowed values: Zero disables polling, while positive values are an interval in seconds(unless other units are specified, for example "5min") ++ Pacemaker is primarily event-driven, and looks ahead to know when to recheck cluster state for failure timeouts and most time-based rules. However, it will also recheck the cluster after this amount of inactivity, to evaluate rules with date specifications and serve as a fail-safe for certain types of scheduler bugs. Allowed values: Zero disables polling, while positive values are an interval in seconds (unless other units are specified, for example "5min") + Polling interval to recheck cluster state and evaluate rules with date specifications + + +@@ -345,7 +345,7 @@ + + + +- Setting this to false may lead to a "split-brain" situation,potentially leading to data loss and/or service unavailability. ++ Setting this to false may lead to a "split-brain" situation, potentially leading to data loss and/or service unavailability. + *** Advanced Use Only *** Whether to fence unseen nodes at start-up + + +-- +2.31.1 + +From e3597b061afa62a1f869a3e238ad78d3f7222029 Mon Sep 17 00:00:00 2001 +From: Reid Wahl +Date: Tue, 2 Jan 2024 20:28:21 -0800 +Subject: [PATCH 02/24] Test: cts-cli: Update daemon tests to note that + stonith-timeout is used + +Signed-off-by: Reid Wahl +--- + cts/cli/regression.daemons.exp | 4 ++-- + 1 file changed, 2 insertions(+), 2 deletions(-) + +diff --git a/cts/cli/regression.daemons.exp b/cts/cli/regression.daemons.exp +index 1cd049f..6a24089 100644 +--- a/cts/cli/regression.daemons.exp ++++ b/cts/cli/regression.daemons.exp +@@ -330,8 +330,8 @@ + + + +- This value is not used by Pacemaker, but is kept for backward compatibility, and certain legacy fence agents might use it. +- *** Advanced Use Only *** Unused by Pacemaker ++ How long to wait for on, off, and reboot fence actions to complete by default ++ How long to wait for on, off, and reboot fence actions to complete by default + + + +-- +2.31.1 + +From cb3431c1058dd3cfbcc1cc490db268a95d1731e1 Mon Sep 17 00:00:00 2001 +From: Reid Wahl +Date: Sat, 6 Jan 2024 19:20:26 -0800 +Subject: [PATCH 03/24] Refactor: libcrmcommon: New + pcmk__valid_placement_strategy() + +We'll soon need this function outside the scheduler. + +Ref T746 + +Signed-off-by: Reid Wahl +--- + include/crm/common/options_internal.h | 1 + + lib/common/options.c | 16 ++++++++++++++++ + lib/pengine/common.c | 9 +-------- + 3 files changed, 18 insertions(+), 8 deletions(-) + +diff --git a/include/crm/common/options_internal.h b/include/crm/common/options_internal.h +index a9316ca..1ea27ee 100644 +--- a/include/crm/common/options_internal.h ++++ b/include/crm/common/options_internal.h +@@ -67,6 +67,7 @@ bool pcmk__valid_number(const char *value); + bool pcmk__valid_positive_number(const char *value); + bool pcmk__valid_quorum(const char *value); + bool pcmk__valid_script(const char *value); ++bool pcmk__valid_placement_strategy(const char *value); + bool pcmk__valid_percentage(const char *value); + + // from watchdog.c +diff --git a/lib/common/options.c b/lib/common/options.c +index 2d86ebc..1db41a2 100644 +--- a/lib/common/options.c ++++ b/lib/common/options.c +@@ -246,6 +246,22 @@ pcmk__valid_script(const char *value) + + return true; + } ++ ++/*! ++ * \internal ++ * \brief Check whether a string represents a valid placement strategy ++ * ++ * \param[in] value String to validate ++ * ++ * \return \c true if \p value is a valid placement strategy, or \c false ++ * otherwise ++ */ ++bool ++pcmk__valid_placement_strategy(const char *value) ++{ ++ return pcmk__strcase_any_of(value, "default", "utilization", "minimal", ++ "balanced", NULL); ++} + + bool + pcmk__valid_percentage(const char *value) +diff --git a/lib/pengine/common.c b/lib/pengine/common.c +index 0fdd5a1..6878f4d 100644 +--- a/lib/pengine/common.c ++++ b/lib/pengine/common.c +@@ -21,13 +21,6 @@ + gboolean was_processing_error = FALSE; + gboolean was_processing_warning = FALSE; + +-static bool +-check_placement_strategy(const char *value) +-{ +- return pcmk__strcase_any_of(value, "default", "utilization", "minimal", +- "balanced", NULL); +-} +- + static pcmk__cluster_option_t pe_opts[] = { + /* name, old name, type, allowed values, + * default value, validator, +@@ -285,7 +278,7 @@ static pcmk__cluster_option_t pe_opts[] = { + { + "placement-strategy", NULL, "select", + "default, utilization, minimal, balanced", +- "default", check_placement_strategy, ++ "default", pcmk__valid_placement_strategy, + N_("How the cluster should allocate resources to nodes"), + NULL + }, +-- +2.31.1 + +From 4c877cefcde40da8a2cd776956ade62919a2c926 Mon Sep 17 00:00:00 2001 +From: Reid Wahl +Date: Tue, 9 Jan 2024 22:13:19 -0800 +Subject: [PATCH 04/24] Refactor: controller: Remove stonith-watchdog-timeout + validator function + +...from options array. Instead, call it from the controller after +validating the options array. + +We'll soon be moving the options array to libcrmcommon. There, we don't +have access to controld_verify_stonith_watchdog_timeout() or to the +controller's stonith API connection and node name. New comments and the +following task have more details: https://projects.clusterlabs.org/T749. + +Ref T746 + +Signed-off-by: Reid Wahl +--- + daemons/controld/controld_control.c | 25 ++++++++++++++++++++++++- + lib/common/watchdog.c | 4 ++++ + 2 files changed, 28 insertions(+), 1 deletion(-) + +diff --git a/daemons/controld/controld_control.c b/daemons/controld/controld_control.c +index 644d686..83b802e 100644 +--- a/daemons/controld/controld_control.c ++++ b/daemons/controld/controld_control.c +@@ -621,8 +621,18 @@ static pcmk__cluster_option_t controller_options[] = { + "the order in which ping updates arrive.") + }, + { ++ /* @COMPAT Currently unparsable values default to -1 (auto-calculate), ++ * while missing values default to 0 (disable). All values are accepted ++ * (unless the controller finds that the value conflicts with the ++ * SBD_WATCHDOG_TIMEOUT). ++ * ++ * At a compatibility break: properly validate as a timeout, let ++ * either negative values or a particular string like "auto" mean auto- ++ * calculate, and use 0 as the single default for when the option either ++ * is unset or fails to validate. ++ */ + "stonith-watchdog-timeout", NULL, "time", NULL, +- "0", controld_verify_stonith_watchdog_timeout, ++ "0", NULL, + N_("How long before nodes can be assumed to be safely down when " + "watchdog-based self-fencing via SBD is in use"), + N_("If this is set to a positive value, lost nodes are assumed to " +@@ -747,6 +757,19 @@ config_query_callback(xmlNode * msg, int call_id, int rc, xmlNode * output, void + pcmk__validate_cluster_options(config_hash, controller_options, + PCMK__NELEM(controller_options)); + ++ /* Validate the watchdog timeout in the context of the local node ++ * environment. If invalid, the controller will exit with a fatal error. ++ * ++ * We do this via a wrapper in the controller, so that we call ++ * pcmk__valid_stonith_watchdog_timeout() only if watchdog fencing is ++ * enabled for the local node. Otherwise, we may exit unnecessarily. ++ * ++ * A validator function in libcrmcommon can't act as such a wrapper, because ++ * it doesn't have a stonith API connection or the local node name. ++ */ ++ value = g_hash_table_lookup(config_hash, "stonith-watchdog-timeout"); ++ controld_verify_stonith_watchdog_timeout(value); ++ + value = g_hash_table_lookup(config_hash, "no-quorum-policy"); + if (pcmk__str_eq(value, "suicide", pcmk__str_casei) && pcmk__locate_sbd()) { + controld_set_global_flags(controld_no_quorum_suicide); +diff --git a/lib/common/watchdog.c b/lib/common/watchdog.c +index e569214..7d126af 100644 +--- a/lib/common/watchdog.c ++++ b/lib/common/watchdog.c +@@ -278,6 +278,10 @@ pcmk__valid_sbd_timeout(const char *value) + { + long st_timeout = value? crm_get_msec(value) : 0; + ++ /* @COMPAT At a compatibility break, accept either negative values or a ++ * specific string like "auto" (but not both) to mean "auto-calculate the ++ * timeout." Reject other values that aren't parsable as timeouts. ++ */ + if (st_timeout < 0) { + st_timeout = pcmk__auto_watchdog_timeout(); + crm_debug("Using calculated value %ld for stonith-watchdog-timeout (%s)", +-- +2.31.1 + +From 28d96fc802bca24ed3e52b8ce5946f4b2b971b7d Mon Sep 17 00:00:00 2001 +From: Reid Wahl +Date: Sat, 6 Jan 2024 17:52:18 -0800 +Subject: [PATCH 05/24] Refactor: libcrmcommon: New enum pcmk__opt_context + +At first this will be used only for backward compatibility with metadata +commands for pacemaker-based, pacemaker-controld, and +pacemaker-schedulerd. It may be extended later for meta-attribute +contexts or similar. + +The idea is that we'll consolidate all cluster options into a single +table and use these enum values as filters for getting daemon metadata. + +We won't need a pcmk__opt_context_fenced, because its metadata consists +of stonith instance attributes, not cluster options. Those instance +attributes will be stored in a separate array from the cluster options. + +Ref T746 + +Signed-off-by: Reid Wahl +--- + include/crm/common/options_internal.h | 13 +++++++++++++ + 1 file changed, 13 insertions(+) + +diff --git a/include/crm/common/options_internal.h b/include/crm/common/options_internal.h +index 1ea27ee..0c6c9e8 100644 +--- a/include/crm/common/options_internal.h ++++ b/include/crm/common/options_internal.h +@@ -34,6 +34,19 @@ bool pcmk__env_option_enabled(const char *daemon, const char *option); + * Cluster option handling + */ + ++/*! ++ * \internal ++ * \enum pcmk__opt_context ++ * \brief Context flags for options ++ */ ++enum pcmk__opt_context { ++ // @COMPAT Used only for daemon metadata ++ pcmk__opt_context_none = 0, //!< No additional context ++ pcmk__opt_context_based = (1 << 1), //!< CIB manager metadata ++ pcmk__opt_context_controld = (1 << 2), //!< Controller metadata ++ pcmk__opt_context_schedulerd = (1 << 3), //!< Scheduler metadata ++}; ++ + typedef struct pcmk__cluster_option_s { + const char *name; + const char *alt_name; +-- +2.31.1 + +From 2f8537331e2948b9186555ffbd9c9f2c121587d1 Mon Sep 17 00:00:00 2001 +From: Reid Wahl +Date: Sat, 6 Jan 2024 17:58:25 -0800 +Subject: [PATCH 06/24] Refactor: libcrmcommon: New + pcmk__cluster_option_t:context member + +Arguably makes more sense adjacent to the type member, but this +placement keeps the diffs smaller when updating existing options arrays. + +We will use this soon to ensure that each option occurs in exactly one +daemon's metadata. Several options (for example, +PCMK_OPT_NO_QUORUM_POLICY) currently appear in the metadata of both the +controller and the scheduler, causing issues for external tools that +parse the output. + +Where an option currently appears in the metadata of both the controller +and the scheduler, it will soon appear only in the scheduler's metadata. +We assign context flags accordingly. + +Note that the fencer doesn't have a context flag. The options in its +metadata are actually stonith device instance attributes, not cluster +options. They will continue to reside in a separate table from the +cluster options, so there's no ambiguity about which daemon they "belong +to." + +Ref T746 + +Signed-off-by: Reid Wahl +--- + daemons/controld/controld_control.c | 20 +++++++++++++++++ + daemons/fenced/pacemaker-fenced.c | 26 ++++++++++++++++++++++ + include/crm/common/options_internal.h | 3 +++ + lib/cib/cib_utils.c | 3 +++ + lib/pengine/common.c | 32 +++++++++++++++++++++++++++ + 5 files changed, 84 insertions(+) + +diff --git a/daemons/controld/controld_control.c b/daemons/controld/controld_control.c +index 83b802e..4d7cb14 100644 +--- a/daemons/controld/controld_control.c ++++ b/daemons/controld/controld_control.c +@@ -518,22 +518,26 @@ do_recover(long long action, + static pcmk__cluster_option_t controller_options[] = { + /* name, old name, type, allowed values, + * default value, validator, ++ * context, + * short description, + * long description + */ + { + "dc-version", NULL, "string", NULL, PCMK__VALUE_NONE, NULL, ++ pcmk__opt_context_controld, + N_("Pacemaker version on cluster node elected Designated Controller (DC)"), + N_("Includes a hash which identifies the exact changeset the code was " + "built from. Used for diagnostic purposes.") + }, + { + "cluster-infrastructure", NULL, "string", NULL, "corosync", NULL, ++ pcmk__opt_context_controld, + N_("The messaging stack on which Pacemaker is currently running"), + N_("Used for informational and diagnostic purposes.") + }, + { + "cluster-name", NULL, "string", NULL, NULL, NULL, ++ pcmk__opt_context_controld, + N_("An arbitrary name for the cluster"), + N_("This optional value is mostly for users' convenience as desired " + "in administration, but may also be used in Pacemaker " +@@ -543,6 +547,7 @@ static pcmk__cluster_option_t controller_options[] = { + { + XML_CONFIG_ATTR_DC_DEADTIME, NULL, "time", + NULL, "20s", pcmk__valid_interval_spec, ++ pcmk__opt_context_controld, + N_("How long to wait for a response from other nodes during start-up"), + N_("The optimal value will depend on the speed and load of your network " + "and the type of switches used.") +@@ -552,6 +557,7 @@ static pcmk__cluster_option_t controller_options[] = { + N_("Zero disables polling, while positive values are an interval in seconds" + "(unless other units are specified, for example \"5min\")"), + "15min", pcmk__valid_interval_spec, ++ pcmk__opt_context_controld, + N_("Polling interval to recheck cluster state and evaluate rules " + "with date specifications"), + N_("Pacemaker is primarily event-driven, and looks ahead to know when to " +@@ -563,6 +569,7 @@ static pcmk__cluster_option_t controller_options[] = { + { + "load-threshold", NULL, "percentage", NULL, + "80%", pcmk__valid_percentage, ++ pcmk__opt_context_controld, + N_("Maximum amount of system load that should be used by cluster nodes"), + N_("The cluster will slow down its recovery process when the amount of " + "system resources used (currently CPU) approaches this limit"), +@@ -570,10 +577,12 @@ static pcmk__cluster_option_t controller_options[] = { + { + "node-action-limit", NULL, "integer", NULL, + "0", pcmk__valid_number, ++ pcmk__opt_context_controld, + N_("Maximum number of jobs that can be scheduled per node " + "(defaults to 2x cores)") + }, + { XML_CONFIG_ATTR_FENCE_REACTION, NULL, "string", NULL, "stop", NULL, ++ pcmk__opt_context_controld, + N_("How a cluster node should react if notified of its own fencing"), + N_("A cluster node may receive notification of its own fencing if fencing " + "is misconfigured, or if fabric fencing is in use that doesn't cut " +@@ -584,6 +593,7 @@ static pcmk__cluster_option_t controller_options[] = { + { + XML_CONFIG_ATTR_ELECTION_FAIL, NULL, "time", NULL, + "2min", pcmk__valid_interval_spec, ++ pcmk__opt_context_controld, + "*** Advanced Use Only ***", + N_("Declare an election failed if it is not decided within this much " + "time. If you need to adjust this value, it probably indicates " +@@ -592,6 +602,7 @@ static pcmk__cluster_option_t controller_options[] = { + { + XML_CONFIG_ATTR_FORCE_QUIT, NULL, "time", NULL, + "20min", pcmk__valid_interval_spec, ++ pcmk__opt_context_controld, + "*** Advanced Use Only ***", + N_("Exit immediately if shutdown does not complete within this much " + "time. If you need to adjust this value, it probably indicates " +@@ -600,6 +611,7 @@ static pcmk__cluster_option_t controller_options[] = { + { + "join-integration-timeout", "crmd-integration-timeout", "time", NULL, + "3min", pcmk__valid_interval_spec, ++ pcmk__opt_context_controld, + "*** Advanced Use Only ***", + N_("If you need to adjust this value, it probably indicates " + "the presence of a bug.") +@@ -607,6 +619,7 @@ static pcmk__cluster_option_t controller_options[] = { + { + "join-finalization-timeout", "crmd-finalization-timeout", "time", NULL, + "30min", pcmk__valid_interval_spec, ++ pcmk__opt_context_controld, + "*** Advanced Use Only ***", + N_("If you need to adjust this value, it probably indicates " + "the presence of a bug.") +@@ -614,6 +627,7 @@ static pcmk__cluster_option_t controller_options[] = { + { + "transition-delay", "crmd-transition-delay", "time", NULL, + "0s", pcmk__valid_interval_spec, ++ pcmk__opt_context_controld, + N_("*** Advanced Use Only *** Enabling this option will slow down " + "cluster recovery under all conditions"), + N_("Delay cluster recovery for this much time to allow for additional " +@@ -633,6 +647,7 @@ static pcmk__cluster_option_t controller_options[] = { + */ + "stonith-watchdog-timeout", NULL, "time", NULL, + "0", NULL, ++ pcmk__opt_context_controld, + N_("How long before nodes can be assumed to be safely down when " + "watchdog-based self-fencing via SBD is in use"), + N_("If this is set to a positive value, lost nodes are assumed to " +@@ -654,6 +669,7 @@ static pcmk__cluster_option_t controller_options[] = { + { + "stonith-max-attempts", NULL, "integer", NULL, + "10", pcmk__valid_positive_number, ++ pcmk__opt_context_controld, + N_("How many times fencing can fail before it will no longer be " + "immediately re-attempted on a target") + }, +@@ -662,11 +678,13 @@ static pcmk__cluster_option_t controller_options[] = { + { + "no-quorum-policy", NULL, "select", + "stop, freeze, ignore, demote, suicide", "stop", pcmk__valid_quorum, ++ pcmk__opt_context_controld, + N_("What to do when the cluster does not have quorum"), NULL + }, + { + XML_CONFIG_ATTR_SHUTDOWN_LOCK, NULL, "boolean", NULL, + "false", pcmk__valid_boolean, ++ pcmk__opt_context_controld, + N_("Whether to lock resources to a cleanly shut down node"), + N_("When true, resources active on a node when it is cleanly shut down " + "are kept \"locked\" to that node (not allowed to run elsewhere) " +@@ -680,6 +698,7 @@ static pcmk__cluster_option_t controller_options[] = { + { + XML_CONFIG_ATTR_SHUTDOWN_LOCK_LIMIT, NULL, "time", NULL, + "0", pcmk__valid_interval_spec, ++ pcmk__opt_context_controld, + N_("Do not lock resources to a cleanly shut down node longer than " + "this"), + N_("If shutdown-lock is true and this is set to a nonzero time " +@@ -690,6 +709,7 @@ static pcmk__cluster_option_t controller_options[] = { + { + XML_CONFIG_ATTR_NODE_PENDING_TIMEOUT, NULL, "time", NULL, + "0", pcmk__valid_interval_spec, ++ pcmk__opt_context_controld, + N_("How long to wait for a node that has joined the cluster to join " + "the controller process group"), + N_("Fence nodes that do not join the controller process group within " +diff --git a/daemons/fenced/pacemaker-fenced.c b/daemons/fenced/pacemaker-fenced.c +index 7c69fb8..b2f4742 100644 +--- a/daemons/fenced/pacemaker-fenced.c ++++ b/daemons/fenced/pacemaker-fenced.c +@@ -528,11 +528,13 @@ st_peer_update_callback(enum crm_status_type type, crm_node_t * node, const void + static pcmk__cluster_option_t fencer_options[] = { + /* name, old name, type, allowed values, + * default value, validator, ++ * context, + * short description, + * long description + */ + { + PCMK_STONITH_HOST_ARGUMENT, NULL, "string", NULL, "port", NULL, ++ pcmk__opt_context_none, + N_("Advanced use only: An alternate parameter to supply instead of 'port'"), + N_("some devices do not support the " + "standard 'port' parameter or may provide additional ones. Use " +@@ -543,17 +545,20 @@ static pcmk__cluster_option_t fencer_options[] = { + }, + { + PCMK_STONITH_HOST_MAP,NULL, "string", NULL, "", NULL, ++ pcmk__opt_context_none, + N_("A mapping of host names to ports numbers for devices that do not support host names."), + N_("Eg. node1:1;node2:2,3 would tell the cluster to use port 1 for node1 and ports 2 and 3 for node2") + }, + { + PCMK_STONITH_HOST_LIST,NULL, "string", NULL, "", NULL, ++ pcmk__opt_context_none, + N_("Eg. node1,node2,node3"), + N_("A list of machines controlled by " + "this device (Optional unless pcmk_host_list=static-list)") + }, + { + PCMK_STONITH_HOST_CHECK,NULL, "string", NULL, "dynamic-list", NULL, ++ pcmk__opt_context_none, + N_("How to determine which machines are controlled by the device."), + N_("Allowed values: dynamic-list " + "(query the device via the 'list' command), static-list " +@@ -564,6 +569,7 @@ static pcmk__cluster_option_t fencer_options[] = { + }, + { + PCMK_STONITH_DELAY_MAX,NULL, "time", NULL, "0s", NULL, ++ pcmk__opt_context_none, + N_("Enable a base delay for fencing actions and specify base delay value."), + N_("Enable a delay of no more than the " + "time specified before executing fencing actions. Pacemaker " +@@ -573,6 +579,7 @@ static pcmk__cluster_option_t fencer_options[] = { + }, + { + PCMK_STONITH_DELAY_BASE,NULL, "string", NULL, "0s", NULL, ++ pcmk__opt_context_none, + N_("Enable a base delay for " + "fencing actions and specify base delay value."), + N_("This enables a static delay for " +@@ -587,6 +594,7 @@ static pcmk__cluster_option_t fencer_options[] = { + }, + { + PCMK_STONITH_ACTION_LIMIT,NULL, "integer", NULL, "1", NULL, ++ pcmk__opt_context_none, + N_("The maximum number of actions can be performed in parallel on this device"), + N_("Cluster property concurrent-fencing=true needs to be configured first." + "Then use this to specify the maximum number of actions can be performed in parallel on this device. -1 is unlimited.") +@@ -594,18 +602,21 @@ static pcmk__cluster_option_t fencer_options[] = { + { + "pcmk_reboot_action", NULL, "string", NULL, + PCMK_ACTION_REBOOT, NULL, ++ pcmk__opt_context_none, + N_("Advanced use only: An alternate command to run instead of 'reboot'"), + N_("Some devices do not support the standard commands or may provide additional ones.\n" + "Use this to specify an alternate, device-specific, command that implements the \'reboot\' action.") + }, + { + "pcmk_reboot_timeout",NULL, "time", NULL, "60s", NULL, ++ pcmk__opt_context_none, + N_("Advanced use only: Specify an alternate timeout to use for reboot actions instead of stonith-timeout"), + N_("Some devices need much more/less time to complete than normal." + "Use this to specify an alternate, device-specific, timeout for \'reboot\' actions.") + }, + { + "pcmk_reboot_retries",NULL, "integer", NULL, "2", NULL, ++ pcmk__opt_context_none, + N_("Advanced use only: The maximum number of times to retry the 'reboot' command within the timeout period"), + N_("Some devices do not support multiple connections." + " Operations may 'fail' if the device is busy with another task so Pacemaker will automatically retry the operation, if there is time remaining." +@@ -614,18 +625,21 @@ static pcmk__cluster_option_t fencer_options[] = { + { + "pcmk_off_action", NULL, "string", NULL, + PCMK_ACTION_OFF, NULL, ++ pcmk__opt_context_none, + N_("Advanced use only: An alternate command to run instead of \'off\'"), + N_("Some devices do not support the standard commands or may provide additional ones." + "Use this to specify an alternate, device-specific, command that implements the \'off\' action.") + }, + { + "pcmk_off_timeout",NULL, "time", NULL, "60s", NULL, ++ pcmk__opt_context_none, + N_("Advanced use only: Specify an alternate timeout to use for off actions instead of stonith-timeout"), + N_("Some devices need much more/less time to complete than normal." + "Use this to specify an alternate, device-specific, timeout for \'off\' actions.") + }, + { + "pcmk_off_retries",NULL, "integer", NULL, "2", NULL, ++ pcmk__opt_context_none, + N_("Advanced use only: The maximum number of times to retry the 'off' command within the timeout period"), + N_("Some devices do not support multiple connections." + " Operations may 'fail' if the device is busy with another task so Pacemaker will automatically retry the operation, if there is time remaining." +@@ -634,18 +648,21 @@ static pcmk__cluster_option_t fencer_options[] = { + { + "pcmk_on_action", NULL, "string", NULL, + PCMK_ACTION_ON, NULL, ++ pcmk__opt_context_none, + N_("Advanced use only: An alternate command to run instead of 'on'"), + N_("Some devices do not support the standard commands or may provide additional ones." + "Use this to specify an alternate, device-specific, command that implements the \'on\' action.") + }, + { + "pcmk_on_timeout",NULL, "time", NULL, "60s", NULL, ++ pcmk__opt_context_none, + N_("Advanced use only: Specify an alternate timeout to use for on actions instead of stonith-timeout"), + N_("Some devices need much more/less time to complete than normal." + "Use this to specify an alternate, device-specific, timeout for \'on\' actions.") + }, + { + "pcmk_on_retries",NULL, "integer", NULL, "2", NULL, ++ pcmk__opt_context_none, + N_("Advanced use only: The maximum number of times to retry the 'on' command within the timeout period"), + N_("Some devices do not support multiple connections." + " Operations may 'fail' if the device is busy with another task so Pacemaker will automatically retry the operation, if there is time remaining." +@@ -654,18 +671,21 @@ static pcmk__cluster_option_t fencer_options[] = { + { + "pcmk_list_action",NULL, "string", NULL, + PCMK_ACTION_LIST, NULL, ++ pcmk__opt_context_none, + N_("Advanced use only: An alternate command to run instead of \'list\'"), + N_("Some devices do not support the standard commands or may provide additional ones." + "Use this to specify an alternate, device-specific, command that implements the \'list\' action.") + }, + { + "pcmk_list_timeout",NULL, "time", NULL, "60s", NULL, ++ pcmk__opt_context_none, + N_("Advanced use only: Specify an alternate timeout to use for list actions instead of stonith-timeout"), + N_("Some devices need much more/less time to complete than normal." + "Use this to specify an alternate, device-specific, timeout for \'list\' actions.") + }, + { + "pcmk_list_retries",NULL, "integer", NULL, "2", NULL, ++ pcmk__opt_context_none, + N_("Advanced use only: The maximum number of times to retry the \'list\' command within the timeout period"), + N_("Some devices do not support multiple connections." + " Operations may 'fail' if the device is busy with another task so Pacemaker will automatically retry the operation, if there is time remaining." +@@ -674,18 +694,21 @@ static pcmk__cluster_option_t fencer_options[] = { + { + "pcmk_monitor_action", NULL, "string", NULL, + PCMK_ACTION_MONITOR, NULL, ++ pcmk__opt_context_none, + N_("Advanced use only: An alternate command to run instead of \'monitor\'"), + N_("Some devices do not support the standard commands or may provide additional ones." + "Use this to specify an alternate, device-specific, command that implements the \'monitor\' action.") + }, + { + "pcmk_monitor_timeout",NULL, "time", NULL, "60s", NULL, ++ pcmk__opt_context_none, + N_("Advanced use only: Specify an alternate timeout to use for monitor actions instead of stonith-timeout"), + N_("Some devices need much more/less time to complete than normal.\n" + "Use this to specify an alternate, device-specific, timeout for \'monitor\' actions.") + }, + { + "pcmk_monitor_retries",NULL, "integer", NULL, "2", NULL, ++ pcmk__opt_context_none, + N_("Advanced use only: The maximum number of times to retry the \'monitor\' command within the timeout period"), + N_("Some devices do not support multiple connections." + " Operations may 'fail' if the device is busy with another task so Pacemaker will automatically retry the operation, if there is time remaining." +@@ -694,18 +717,21 @@ static pcmk__cluster_option_t fencer_options[] = { + { + "pcmk_status_action", NULL, "string", NULL, + PCMK_ACTION_STATUS, NULL, ++ pcmk__opt_context_none, + N_("Advanced use only: An alternate command to run instead of \'status\'"), + N_("Some devices do not support the standard commands or may provide additional ones." + "Use this to specify an alternate, device-specific, command that implements the \'status\' action.") + }, + { + "pcmk_status_timeout",NULL, "time", NULL, "60s", NULL, ++ pcmk__opt_context_none, + N_("Advanced use only: Specify an alternate timeout to use for status actions instead of stonith-timeout"), + N_("Some devices need much more/less time to complete than normal." + "Use this to specify an alternate, device-specific, timeout for \'status\' actions.") + }, + { + "pcmk_status_retries",NULL, "integer", NULL, "2", NULL, ++ pcmk__opt_context_none, + N_("Advanced use only: The maximum number of times to retry the \'status\' command within the timeout period"), + N_("Some devices do not support multiple connections." + " Operations may 'fail' if the device is busy with another task so Pacemaker will automatically retry the operation, if there is time remaining." +diff --git a/include/crm/common/options_internal.h b/include/crm/common/options_internal.h +index 0c6c9e8..b2525ef 100644 +--- a/include/crm/common/options_internal.h ++++ b/include/crm/common/options_internal.h +@@ -56,6 +56,9 @@ typedef struct pcmk__cluster_option_s { + + bool (*is_valid)(const char *); + ++ // @COMPAT context is used only for daemon meta-data ++ enum pcmk__opt_context context; ++ + const char *description_short; + const char *description_long; + +diff --git a/lib/cib/cib_utils.c b/lib/cib/cib_utils.c +index 9c3f9f1..9e4060b 100644 +--- a/lib/cib/cib_utils.c ++++ b/lib/cib/cib_utils.c +@@ -869,18 +869,21 @@ cib_native_notify(gpointer data, gpointer user_data) + static pcmk__cluster_option_t cib_opts[] = { + /* name, legacy name, type, allowed values, + * default value, validator, ++ * context, + * short description, + * long description + */ + { + "enable-acl", NULL, "boolean", NULL, + "false", pcmk__valid_boolean, ++ pcmk__opt_context_based, + N_("Enable Access Control Lists (ACLs) for the CIB"), + NULL + }, + { + "cluster-ipc-limit", NULL, "integer", NULL, + "500", pcmk__valid_positive_number, ++ pcmk__opt_context_based, + N_("Maximum IPC message backlog before disconnecting a cluster daemon"), + N_("Raise this if log has \"Evicting client\" messages for cluster daemon" + " PIDs (a good value is the number of resources in the cluster" +diff --git a/lib/pengine/common.c b/lib/pengine/common.c +index 6878f4d..383c4af 100644 +--- a/lib/pengine/common.c ++++ b/lib/pengine/common.c +@@ -24,24 +24,28 @@ gboolean was_processing_warning = FALSE; + static pcmk__cluster_option_t pe_opts[] = { + /* name, old name, type, allowed values, + * default value, validator, ++ * context, + * short description, + * long description + */ + { + "no-quorum-policy", NULL, "select", "stop, freeze, ignore, demote, suicide", + "stop", pcmk__valid_quorum, ++ pcmk__opt_context_schedulerd, + N_("What to do when the cluster does not have quorum"), + NULL + }, + { + "symmetric-cluster", NULL, "boolean", NULL, + "true", pcmk__valid_boolean, ++ pcmk__opt_context_schedulerd, + N_("Whether resources can run on any node by default"), + NULL + }, + { + "maintenance-mode", NULL, "boolean", NULL, + "false", pcmk__valid_boolean, ++ pcmk__opt_context_schedulerd, + N_("Whether the cluster should refrain from monitoring, starting, " + "and stopping resources"), + NULL +@@ -49,6 +53,7 @@ static pcmk__cluster_option_t pe_opts[] = { + { + "start-failure-is-fatal", NULL, "boolean", NULL, + "true", pcmk__valid_boolean, ++ pcmk__opt_context_schedulerd, + N_("Whether a start failure should prevent a resource from being " + "recovered on the same node"), + N_("When true, the cluster will immediately ban a resource from a node " +@@ -58,12 +63,14 @@ static pcmk__cluster_option_t pe_opts[] = { + { + "enable-startup-probes", NULL, "boolean", NULL, + "true", pcmk__valid_boolean, ++ pcmk__opt_context_schedulerd, + N_("Whether the cluster should check for active resources during start-up"), + NULL + }, + { + XML_CONFIG_ATTR_SHUTDOWN_LOCK, NULL, "boolean", NULL, + "false", pcmk__valid_boolean, ++ pcmk__opt_context_schedulerd, + N_("Whether to lock resources to a cleanly shut down node"), + N_("When true, resources active on a node when it is cleanly shut down " + "are kept \"locked\" to that node (not allowed to run elsewhere) " +@@ -77,6 +84,7 @@ static pcmk__cluster_option_t pe_opts[] = { + { + XML_CONFIG_ATTR_SHUTDOWN_LOCK_LIMIT, NULL, "time", NULL, + "0", pcmk__valid_interval_spec, ++ pcmk__opt_context_schedulerd, + N_("Do not lock resources to a cleanly shut down node longer than " + "this"), + N_("If shutdown-lock is true and this is set to a nonzero time " +@@ -89,6 +97,7 @@ static pcmk__cluster_option_t pe_opts[] = { + { + "stonith-enabled", NULL, "boolean", NULL, + "true", pcmk__valid_boolean, ++ pcmk__opt_context_schedulerd, + N_("*** Advanced Use Only *** " + "Whether nodes may be fenced as part of recovery"), + N_("If false, unresponsive nodes are immediately assumed to be harmless, " +@@ -99,6 +108,7 @@ static pcmk__cluster_option_t pe_opts[] = { + { + "stonith-action", NULL, "select", "reboot, off, poweroff", + PCMK_ACTION_REBOOT, pcmk__is_fencing_action, ++ pcmk__opt_context_schedulerd, + N_("Action to send to fence device when a node needs to be fenced " + "(\"poweroff\" is a deprecated alias for \"off\")"), + NULL +@@ -106,6 +116,7 @@ static pcmk__cluster_option_t pe_opts[] = { + { + "stonith-timeout", NULL, "time", NULL, + "60s", pcmk__valid_interval_spec, ++ pcmk__opt_context_schedulerd, + N_("*** Advanced Use Only *** Unused by Pacemaker"), + N_("This value is not used by Pacemaker, but is kept for backward " + "compatibility, and certain legacy fence agents might use it.") +@@ -113,6 +124,7 @@ static pcmk__cluster_option_t pe_opts[] = { + { + XML_ATTR_HAVE_WATCHDOG, NULL, "boolean", NULL, + "false", pcmk__valid_boolean, ++ pcmk__opt_context_schedulerd, + N_("Whether watchdog integration is enabled"), + N_("This is set automatically by the cluster according to whether SBD " + "is detected to be in use. User-configured values are ignored. " +@@ -124,12 +136,14 @@ static pcmk__cluster_option_t pe_opts[] = { + { + "concurrent-fencing", NULL, "boolean", NULL, + PCMK__CONCURRENT_FENCING_DEFAULT, pcmk__valid_boolean, ++ pcmk__opt_context_schedulerd, + N_("Allow performing fencing operations in parallel"), + NULL + }, + { + "startup-fencing", NULL, "boolean", NULL, + "true", pcmk__valid_boolean, ++ pcmk__opt_context_schedulerd, + N_("*** Advanced Use Only *** Whether to fence unseen nodes at start-up"), + N_("Setting this to false may lead to a \"split-brain\" situation," + "potentially leading to data loss and/or service unavailability.") +@@ -137,6 +151,7 @@ static pcmk__cluster_option_t pe_opts[] = { + { + XML_CONFIG_ATTR_PRIORITY_FENCING_DELAY, NULL, "time", NULL, + "0", pcmk__valid_interval_spec, ++ pcmk__opt_context_schedulerd, + N_("Apply fencing delay targeting the lost nodes with the highest total resource priority"), + N_("Apply specified delay for the fencings that are targeting the lost " + "nodes with the highest total resource priority in case we don't " +@@ -154,6 +169,7 @@ static pcmk__cluster_option_t pe_opts[] = { + { + XML_CONFIG_ATTR_NODE_PENDING_TIMEOUT, NULL, "time", NULL, + "0", pcmk__valid_interval_spec, ++ pcmk__opt_context_schedulerd, + N_("How long to wait for a node that has joined the cluster to join " + "the controller process group"), + N_("Fence nodes that do not join the controller process group within " +@@ -165,6 +181,7 @@ static pcmk__cluster_option_t pe_opts[] = { + { + "cluster-delay", NULL, "time", NULL, + "60s", pcmk__valid_interval_spec, ++ pcmk__opt_context_schedulerd, + N_("Maximum time for node-to-node communication"), + N_("The node elected Designated Controller (DC) will consider an action " + "failed if it does not get a response from the node executing the " +@@ -175,6 +192,7 @@ static pcmk__cluster_option_t pe_opts[] = { + { + "batch-limit", NULL, "integer", NULL, + "0", pcmk__valid_number, ++ pcmk__opt_context_schedulerd, + N_("Maximum number of jobs that the cluster may execute in parallel " + "across all nodes"), + N_("The \"correct\" value will depend on the speed and load of your " +@@ -185,6 +203,7 @@ static pcmk__cluster_option_t pe_opts[] = { + { + "migration-limit", NULL, "integer", NULL, + "-1", pcmk__valid_number, ++ pcmk__opt_context_schedulerd, + N_("The number of live migration actions that the cluster is allowed " + "to execute in parallel on a node (-1 means no limit)") + }, +@@ -193,24 +212,28 @@ static pcmk__cluster_option_t pe_opts[] = { + { + "stop-all-resources", NULL, "boolean", NULL, + "false", pcmk__valid_boolean, ++ pcmk__opt_context_schedulerd, + N_("Whether the cluster should stop all active resources"), + NULL + }, + { + "stop-orphan-resources", NULL, "boolean", NULL, + "true", pcmk__valid_boolean, ++ pcmk__opt_context_schedulerd, + N_("Whether to stop resources that were removed from the configuration"), + NULL + }, + { + "stop-orphan-actions", NULL, "boolean", NULL, + "true", pcmk__valid_boolean, ++ pcmk__opt_context_schedulerd, + N_("Whether to cancel recurring actions removed from the configuration"), + NULL + }, + { + "remove-after-stop", NULL, "boolean", NULL, + "false", pcmk__valid_boolean, ++ pcmk__opt_context_schedulerd, + N_("*** Deprecated *** Whether to remove stopped resources from " + "the executor"), + N_("Values other than default are poorly tested and potentially dangerous." +@@ -221,18 +244,21 @@ static pcmk__cluster_option_t pe_opts[] = { + { + "pe-error-series-max", NULL, "integer", NULL, + "-1", pcmk__valid_number, ++ pcmk__opt_context_schedulerd, + N_("The number of scheduler inputs resulting in errors to save"), + N_("Zero to disable, -1 to store unlimited.") + }, + { + "pe-warn-series-max", NULL, "integer", NULL, + "5000", pcmk__valid_number, ++ pcmk__opt_context_schedulerd, + N_("The number of scheduler inputs resulting in warnings to save"), + N_("Zero to disable, -1 to store unlimited.") + }, + { + "pe-input-series-max", NULL, "integer", NULL, + "4000", pcmk__valid_number, ++ pcmk__opt_context_schedulerd, + N_("The number of scheduler inputs without errors or warnings to save"), + N_("Zero to disable, -1 to store unlimited.") + }, +@@ -244,6 +270,7 @@ static pcmk__cluster_option_t pe_opts[] = { + PCMK__VALUE_ONLY_GREEN ", " PCMK__VALUE_PROGRESSIVE ", " + PCMK__VALUE_CUSTOM, + PCMK__VALUE_NONE, pcmk__validate_health_strategy, ++ pcmk__opt_context_schedulerd, + N_("How cluster should react to node health attributes"), + N_("Requires external entities to create node attributes (named with " + "the prefix \"#health\") with values \"red\", " +@@ -252,24 +279,28 @@ static pcmk__cluster_option_t pe_opts[] = { + { + PCMK__OPT_NODE_HEALTH_BASE, NULL, "integer", NULL, + "0", pcmk__valid_number, ++ pcmk__opt_context_schedulerd, + N_("Base health score assigned to a node"), + N_("Only used when \"node-health-strategy\" is set to \"progressive\".") + }, + { + PCMK__OPT_NODE_HEALTH_GREEN, NULL, "integer", NULL, + "0", pcmk__valid_number, ++ pcmk__opt_context_schedulerd, + N_("The score to use for a node health attribute whose value is \"green\""), + N_("Only used when \"node-health-strategy\" is set to \"custom\" or \"progressive\".") + }, + { + PCMK__OPT_NODE_HEALTH_YELLOW, NULL, "integer", NULL, + "0", pcmk__valid_number, ++ pcmk__opt_context_schedulerd, + N_("The score to use for a node health attribute whose value is \"yellow\""), + N_("Only used when \"node-health-strategy\" is set to \"custom\" or \"progressive\".") + }, + { + PCMK__OPT_NODE_HEALTH_RED, NULL, "integer", NULL, + "-INFINITY", pcmk__valid_number, ++ pcmk__opt_context_schedulerd, + N_("The score to use for a node health attribute whose value is \"red\""), + N_("Only used when \"node-health-strategy\" is set to \"custom\" or \"progressive\".") + }, +@@ -279,6 +310,7 @@ static pcmk__cluster_option_t pe_opts[] = { + "placement-strategy", NULL, "select", + "default, utilization, minimal, balanced", + "default", pcmk__valid_placement_strategy, ++ pcmk__opt_context_schedulerd, + N_("How the cluster should allocate resources to nodes"), + NULL + }, +-- +2.31.1 + +From cc7c3c87d333854d0f28abe461dd58d5c94b0888 Mon Sep 17 00:00:00 2001 +From: Reid Wahl +Date: Sat, 6 Jan 2024 18:47:25 -0800 +Subject: [PATCH 07/24] Refactor: libcrmcommon: Consolidate cluster option + metadata + +This isn't plugged in yet. It's also currently defined out, to avoid an +unused variable warning from the compiler. + +Ref T746 + +Signed-off-by: Reid Wahl +--- + lib/common/options.c | 499 +++++++++++++++++++++++++++++++++++++++++++ + 1 file changed, 499 insertions(+) + +diff --git a/lib/common/options.c b/lib/common/options.c +index 1db41a2..ff73dcc 100644 +--- a/lib/common/options.c ++++ b/lib/common/options.c +@@ -20,6 +20,7 @@ + #include + + #include ++#include + + void + pcmk__cli_help(char cmd) +@@ -38,6 +39,504 @@ pcmk__cli_help(char cmd) + } + + ++/* ++ * Option metadata ++ */ ++ ++#if 0 ++static pcmk__cluster_option_t cluster_options[] = { ++ /* name, old name, type, allowed values, ++ * default value, validator, ++ * context, ++ * short description, ++ * long description ++ */ ++ { ++ "dc-version", NULL, "string", NULL, ++ PCMK__VALUE_NONE, NULL, ++ pcmk__opt_context_controld, ++ N_("Pacemaker version on cluster node elected Designated Controller " ++ "(DC)"), ++ N_("Includes a hash which identifies the exact changeset the code was " ++ "built from. Used for diagnostic purposes."), ++ }, ++ { ++ "cluster-infrastructure", NULL, "string", NULL, ++ "corosync", NULL, ++ pcmk__opt_context_controld, ++ N_("The messaging stack on which Pacemaker is currently running"), ++ N_("Used for informational and diagnostic purposes."), ++ }, ++ { ++ "cluster-name", NULL, "string", NULL, ++ NULL, NULL, ++ pcmk__opt_context_controld, ++ N_("An arbitrary name for the cluster"), ++ N_("This optional value is mostly for users' convenience as desired " ++ "in administration, but may also be used in Pacemaker " ++ "configuration rules via the #cluster-name node attribute, and " ++ "by higher-level tools and resource agents."), ++ }, ++ { ++ "dc-deadtime", NULL, "time", NULL, ++ "20s", pcmk__valid_interval_spec, ++ pcmk__opt_context_controld, ++ N_("How long to wait for a response from other nodes during start-up"), ++ N_("The optimal value will depend on the speed and load of your " ++ "network and the type of switches used."), ++ }, ++ { ++ "cluster-recheck-interval", NULL, "time", ++ N_("Zero disables polling, while positive values are an interval in " ++ "seconds (unless other units are specified, for example \"5min\")"), ++ "15min", pcmk__valid_interval_spec, ++ pcmk__opt_context_controld, ++ N_("Polling interval to recheck cluster state and evaluate rules " ++ "with date specifications"), ++ N_("Pacemaker is primarily event-driven, and looks ahead to know when " ++ "to recheck cluster state for failure timeouts and most time-based " ++ "rules. However, it will also recheck the cluster after this " ++ "amount of inactivity, to evaluate rules with date specifications " ++ "and serve as a fail-safe for certain types of scheduler bugs."), ++ }, ++ { ++ "fence-reaction", NULL, "select", "stop, panic", ++ "stop", NULL, ++ pcmk__opt_context_controld, ++ N_("How a cluster node should react if notified of its own fencing"), ++ N_("A cluster node may receive notification of its own fencing if " ++ "fencing is misconfigured, or if fabric fencing is in use that " ++ "doesn't cut cluster communication. Use \"stop\" to attempt to " ++ "immediately stop Pacemaker and stay stopped, or \"panic\" to " ++ "attempt to immediately reboot the local node, falling back to " ++ "stop on failure."), ++ }, ++ { ++ "election-timeout", NULL, "time", NULL, ++ "2min", pcmk__valid_interval_spec, ++ pcmk__opt_context_controld, ++ N_("*** Advanced Use Only ***"), ++ N_("Declare an election failed if it is not decided within this much " ++ "time. If you need to adjust this value, it probably indicates " ++ "the presence of a bug."), ++ }, ++ { ++ "shutdown-escalation", NULL, "time", NULL, ++ "20min", pcmk__valid_interval_spec, ++ pcmk__opt_context_controld, ++ N_("*** Advanced Use Only ***"), ++ N_("Exit immediately if shutdown does not complete within this much " ++ "time. If you need to adjust this value, it probably indicates " ++ "the presence of a bug."), ++ }, ++ { ++ "join-integration-timeout", "crmd-integration-timeout", "time", ++ NULL, ++ "3min", pcmk__valid_interval_spec, ++ pcmk__opt_context_controld, ++ N_("*** Advanced Use Only ***"), ++ N_("If you need to adjust this value, it probably indicates " ++ "the presence of a bug."), ++ }, ++ { ++ "join-finalization-timeout", "crmd-finalization-timeout", ++ "time", NULL, ++ "30min", pcmk__valid_interval_spec, ++ pcmk__opt_context_controld, ++ N_("*** Advanced Use Only ***"), ++ N_("If you need to adjust this value, it probably indicates " ++ "the presence of a bug."), ++ }, ++ { ++ "transition-delay", "crmd-transition-delay", "time", NULL, ++ "0s", pcmk__valid_interval_spec, ++ pcmk__opt_context_controld, ++ N_("*** Advanced Use Only *** " ++ "Enabling this option will slow down cluster recovery under all " ++ "conditions"), ++ N_("Delay cluster recovery for this much time to allow for additional " ++ "events to occur. Useful if your configuration is sensitive to " ++ "the order in which ping updates arrive."), ++ }, ++ { ++ "no-quorum-policy", NULL, "select", ++ "stop, freeze, ignore, demote, suicide", ++ "stop", pcmk__valid_quorum, ++ pcmk__opt_context_schedulerd, ++ N_("What to do when the cluster does not have quorum"), ++ NULL, ++ }, ++ { ++ "shutdown-lock", NULL, "boolean", NULL, ++ XML_BOOLEAN_FALSE, pcmk__valid_boolean, ++ pcmk__opt_context_schedulerd, ++ N_("Whether to lock resources to a cleanly shut down node"), ++ N_("When true, resources active on a node when it is cleanly shut down " ++ "are kept \"locked\" to that node (not allowed to run elsewhere) " ++ "until they start again on that node after it rejoins (or for at " ++ "most shutdown-lock-limit, if set). Stonith resources and " ++ "Pacemaker Remote connections are never locked. Clone and bundle " ++ "instances and the promoted role of promotable clones are " ++ "currently never locked, though support could be added in a future " ++ "release."), ++ }, ++ { ++ "shutdown-lock-limit", NULL, "time", NULL, ++ "0", pcmk__valid_interval_spec, ++ pcmk__opt_context_schedulerd, ++ N_("Do not lock resources to a cleanly shut down node longer than " ++ "this"), ++ N_("If shutdown-lock is true and this is set to a nonzero time " ++ "duration, shutdown locks will expire after this much time has " ++ "passed since the shutdown was initiated, even if the node has not " ++ "rejoined."), ++ }, ++ { ++ "enable-acl", NULL, "boolean", NULL, ++ XML_BOOLEAN_FALSE, pcmk__valid_boolean, ++ pcmk__opt_context_based, ++ N_("Enable Access Control Lists (ACLs) for the CIB"), ++ NULL, ++ }, ++ { ++ "symmetric-cluster", NULL, "boolean", NULL, ++ XML_BOOLEAN_TRUE, pcmk__valid_boolean, ++ pcmk__opt_context_schedulerd, ++ N_("Whether resources can run on any node by default"), ++ NULL, ++ }, ++ { ++ "maintenance-mode", NULL, "boolean", NULL, ++ XML_BOOLEAN_FALSE, pcmk__valid_boolean, ++ pcmk__opt_context_schedulerd, ++ N_("Whether the cluster should refrain from monitoring, starting, and " ++ "stopping resources"), ++ NULL, ++ }, ++ { ++ "start-failure-is-fatal", NULL, "boolean", NULL, ++ XML_BOOLEAN_TRUE, pcmk__valid_boolean, ++ pcmk__opt_context_schedulerd, ++ N_("Whether a start failure should prevent a resource from being " ++ "recovered on the same node"), ++ N_("When true, the cluster will immediately ban a resource from a node " ++ "if it fails to start there. When false, the cluster will instead " ++ "check the resource's fail count against its migration-threshold.") ++ }, ++ { ++ "enable-startup-probes", NULL, "boolean", NULL, ++ XML_BOOLEAN_TRUE, pcmk__valid_boolean, ++ pcmk__opt_context_schedulerd, ++ N_("Whether the cluster should check for active resources during " ++ "start-up"), ++ NULL, ++ }, ++ ++ // Fencing-related options ++ { ++ "stonith-enabled", NULL, "boolean", NULL, ++ XML_BOOLEAN_TRUE, pcmk__valid_boolean, ++ pcmk__opt_context_schedulerd, ++ N_("*** Advanced Use Only *** " ++ "Whether nodes may be fenced as part of recovery"), ++ N_("If false, unresponsive nodes are immediately assumed to be " ++ "harmless, and resources that were active on them may be recovered " ++ "elsewhere. This can result in a \"split-brain\" situation, " ++ "potentially leading to data loss and/or service unavailability."), ++ }, ++ { ++ "stonith-action", NULL, "select", "reboot, off, poweroff", ++ PCMK_ACTION_REBOOT, pcmk__is_fencing_action, ++ pcmk__opt_context_schedulerd, ++ N_("Action to send to fence device when a node needs to be fenced " ++ "(\"poweroff\" is a deprecated alias for \"off\")"), ++ NULL, ++ }, ++ { ++ "stonith-timeout", NULL, "time", NULL, ++ "60s", pcmk__valid_interval_spec, ++ pcmk__opt_context_schedulerd, ++ N_("How long to wait for on, off, and reboot fence actions to complete " ++ "by default"), ++ NULL, ++ }, ++ { ++ "have-watchdog", NULL, "boolean", NULL, ++ XML_BOOLEAN_FALSE, pcmk__valid_boolean, ++ pcmk__opt_context_schedulerd, ++ N_("Whether watchdog integration is enabled"), ++ N_("This is set automatically by the cluster according to whether SBD " ++ "is detected to be in use. User-configured values are ignored. " ++ "The value `true` is meaningful if diskless SBD is used and " ++ "`stonith-watchdog-timeout` is nonzero. In that case, if fencing " ++ "is required, watchdog-based self-fencing will be performed via " ++ "SBD without requiring a fencing resource explicitly configured."), ++ }, ++ { ++ /* @COMPAT Currently, unparsable values default to -1 (auto-calculate), ++ * while missing values default to 0 (disable). All values are accepted ++ * (unless the controller finds that the value conflicts with the ++ * SBD_WATCHDOG_TIMEOUT). ++ * ++ * At a compatibility break: properly validate as a timeout, let ++ * either negative values or a particular string like "auto" mean auto- ++ * calculate, and use 0 as the single default for when the option either ++ * is unset or fails to validate. ++ */ ++ "stonith-watchdog-timeout", NULL, "time", NULL, ++ "0", NULL, ++ pcmk__opt_context_controld, ++ N_("How long before nodes can be assumed to be safely down when " ++ "watchdog-based self-fencing via SBD is in use"), ++ N_("If this is set to a positive value, lost nodes are assumed to " ++ "self-fence using watchdog-based SBD within this much time. This " ++ "does not require a fencing resource to be explicitly configured, " ++ "though a fence_watchdog resource can be configured, to limit use " ++ "to specific nodes. If this is set to 0 (the default), the cluster " ++ "will never assume watchdog-based self-fencing. If this is set to a " ++ "negative value, the cluster will use twice the local value of the " ++ "`SBD_WATCHDOG_TIMEOUT` environment variable if that is positive, " ++ "or otherwise treat this as 0. WARNING: When used, this timeout " ++ "must be larger than `SBD_WATCHDOG_TIMEOUT` on all nodes that use " ++ "watchdog-based SBD, and Pacemaker will refuse to start on any of " ++ "those nodes where this is not true for the local value or SBD is " ++ "not active. When this is set to a negative value, " ++ "`SBD_WATCHDOG_TIMEOUT` must be set to the same value on all nodes " ++ "that use SBD, otherwise data corruption or loss could occur."), ++ }, ++ { ++ "stonith-max-attempts", NULL, "integer", NULL, ++ "10", pcmk__valid_positive_number, ++ pcmk__opt_context_controld, ++ N_("How many times fencing can fail before it will no longer be " ++ "immediately re-attempted on a target"), ++ NULL, ++ }, ++ { ++ "concurrent-fencing", NULL, "boolean", NULL, ++ PCMK__CONCURRENT_FENCING_DEFAULT, pcmk__valid_boolean, ++ pcmk__opt_context_schedulerd, ++ N_("Allow performing fencing operations in parallel"), ++ NULL, ++ }, ++ { ++ "startup-fencing", NULL, "boolean", NULL, ++ XML_BOOLEAN_TRUE, pcmk__valid_boolean, ++ pcmk__opt_context_schedulerd, ++ N_("*** Advanced Use Only *** " ++ "Whether to fence unseen nodes at start-up"), ++ N_("Setting this to false may lead to a \"split-brain\" situation, " ++ "potentially leading to data loss and/or service unavailability."), ++ }, ++ { ++ "priority-fencing-delay", NULL, "time", NULL, ++ "0", pcmk__valid_interval_spec, ++ pcmk__opt_context_schedulerd, ++ N_("Apply fencing delay targeting the lost nodes with the highest " ++ "total resource priority"), ++ N_("Apply specified delay for the fencings that are targeting the lost " ++ "nodes with the highest total resource priority in case we don't " ++ "have the majority of the nodes in our cluster partition, so that " ++ "the more significant nodes potentially win any fencing match, " ++ "which is especially meaningful under split-brain of 2-node " ++ "cluster. A promoted resource instance takes the base priority + 1 " ++ "on calculation if the base priority is not 0. Any static/random " ++ "delays that are introduced by `pcmk_delay_base/max` configured " ++ "for the corresponding fencing resources will be added to this " ++ "delay. This delay should be significantly greater than, safely " ++ "twice, the maximum `pcmk_delay_base/max`. By default, priority " ++ "fencing delay is disabled."), ++ }, ++ { ++ "node-pending-timeout", NULL, "time", NULL, ++ "0", pcmk__valid_interval_spec, ++ pcmk__opt_context_schedulerd, ++ N_("How long to wait for a node that has joined the cluster to join " ++ "the controller process group"), ++ N_("Fence nodes that do not join the controller process group within " ++ "this much time after joining the cluster, to allow the cluster " ++ "to continue managing resources. A value of 0 means never fence " ++ "pending nodes. Setting the value to 2h means fence nodes after " ++ "2 hours."), ++ }, ++ { ++ "cluster-delay", NULL, "time", NULL, ++ "60s", pcmk__valid_interval_spec, ++ pcmk__opt_context_schedulerd, ++ N_("Maximum time for node-to-node communication"), ++ N_("The node elected Designated Controller (DC) will consider an action " ++ "failed if it does not get a response from the node executing the " ++ "action within this time (after considering the action's own " ++ "timeout). The \"correct\" value will depend on the speed and " ++ "load of your network and cluster nodes.") ++ }, ++ ++ // Limits ++ { ++ "load-threshold", NULL, "percentage", NULL, ++ "80%", pcmk__valid_percentage, ++ pcmk__opt_context_controld, ++ N_("Maximum amount of system load that should be used by cluster " ++ "nodes"), ++ N_("The cluster will slow down its recovery process when the amount of " ++ "system resources used (currently CPU) approaches this limit"), ++ }, ++ { ++ "node-action-limit", NULL, "integer", NULL, ++ "0", pcmk__valid_number, ++ pcmk__opt_context_controld, ++ N_("Maximum number of jobs that can be scheduled per node (defaults to " ++ "2x cores)"), ++ NULL, ++ }, ++ { ++ "batch-limit", NULL, "integer", NULL, ++ "0", pcmk__valid_number, ++ pcmk__opt_context_schedulerd, ++ N_("Maximum number of jobs that the cluster may execute in parallel " ++ "across all nodes"), ++ N_("The \"correct\" value will depend on the speed and load of your " ++ "network and cluster nodes. If set to 0, the cluster will " ++ "impose a dynamically calculated limit when any node has a " ++ "high load."), ++ }, ++ { ++ "migration-limit", NULL, "integer", NULL, ++ "-1", pcmk__valid_number, ++ pcmk__opt_context_schedulerd, ++ N_("The number of live migration actions that the cluster is allowed " ++ "to execute in parallel on a node (-1 means no limit)"), ++ NULL, ++ }, ++ { ++ "cluster-ipc-limit", NULL, "integer", NULL, ++ "500", pcmk__valid_positive_number, ++ pcmk__opt_context_based, ++ N_("Maximum IPC message backlog before disconnecting a cluster daemon"), ++ N_("Raise this if log has \"Evicting client\" messages for cluster " ++ "daemon PIDs (a good value is the number of resources in the " ++ "cluster multiplied by the number of nodes)."), ++ }, ++ ++ // Orphans and stopping ++ { ++ "stop-all-resources", NULL, "boolean", NULL, ++ XML_BOOLEAN_FALSE, pcmk__valid_boolean, ++ pcmk__opt_context_schedulerd, ++ N_("Whether the cluster should stop all active resources"), ++ NULL, ++ }, ++ { ++ "stop-orphan-resources", NULL, "boolean", NULL, ++ XML_BOOLEAN_TRUE, pcmk__valid_boolean, ++ pcmk__opt_context_schedulerd, ++ N_("Whether to stop resources that were removed from the " ++ "configuration"), ++ NULL, ++ }, ++ { ++ "stop-orphan-actions", NULL, "boolean", NULL, ++ XML_BOOLEAN_TRUE, pcmk__valid_boolean, ++ pcmk__opt_context_schedulerd, ++ N_("Whether to cancel recurring actions removed from the " ++ "configuration"), ++ NULL, ++ }, ++ { ++ "remove-after-stop", NULL, "boolean", NULL, ++ XML_BOOLEAN_FALSE, pcmk__valid_boolean, ++ pcmk__opt_context_schedulerd, ++ N_("*** Deprecated *** " ++ "Whether to remove stopped resources from the executor"), ++ N_("Values other than default are poorly tested and potentially " ++ "dangerous. This option will be removed in a future release."), ++ }, ++ ++ // Storing inputs ++ { ++ "pe-error-series-max", NULL, "integer", NULL, ++ "-1", pcmk__valid_number, ++ pcmk__opt_context_schedulerd, ++ N_("The number of scheduler inputs resulting in errors to save"), ++ N_("Zero to disable, -1 to store unlimited."), ++ }, ++ { ++ "pe-warn-series-max", NULL, "integer", NULL, ++ "5000", pcmk__valid_number, ++ pcmk__opt_context_schedulerd, ++ N_("The number of scheduler inputs resulting in warnings to save"), ++ N_("Zero to disable, -1 to store unlimited."), ++ }, ++ { ++ "pe-input-series-max", NULL, "integer", NULL, ++ "4000", pcmk__valid_number, ++ pcmk__opt_context_schedulerd, ++ N_("The number of scheduler inputs without errors or warnings to save"), ++ N_("Zero to disable, -1 to store unlimited."), ++ }, ++ ++ // Node health ++ { ++ "node-health-strategy", NULL, "select", ++ PCMK__VALUE_NONE ", " PCMK__VALUE_MIGRATE_ON_RED ", " ++ PCMK__VALUE_ONLY_GREEN ", " PCMK__VALUE_PROGRESSIVE ", " ++ PCMK__VALUE_CUSTOM, ++ PCMK__VALUE_NONE, pcmk__validate_health_strategy, ++ pcmk__opt_context_schedulerd, ++ N_("How cluster should react to node health attributes"), ++ N_("Requires external entities to create node attributes (named with " ++ "the prefix \"#health\") with values \"red\", \"yellow\", or " ++ "\"green\".") ++ }, ++ { ++ "node-health-base", NULL, "integer", NULL, ++ "0", pcmk__valid_number, ++ pcmk__opt_context_schedulerd, ++ N_("Base health score assigned to a node"), ++ N_("Only used when \"node-health-strategy\" is set to " ++ "\"progressive\"."), ++ }, ++ { ++ "node-health-green", NULL, "integer", NULL, ++ "0", pcmk__valid_number, ++ pcmk__opt_context_schedulerd, ++ N_("The score to use for a node health attribute whose value is " ++ "\"green\""), ++ N_("Only used when \"node-health-strategy\" is set to \"custom\" or " ++ "\"progressive\"."), ++ }, ++ { ++ "node-health-yellow", NULL, "integer", NULL, ++ "0", pcmk__valid_number, ++ pcmk__opt_context_schedulerd, ++ N_("The score to use for a node health attribute whose value is " ++ "\"yellow\""), ++ N_("Only used when \"node-health-strategy\" is set to \"custom\" or " ++ "\"progressive\"."), ++ }, ++ { ++ "node-health-red", NULL, "integer", NULL, ++ "-INFINITY", pcmk__valid_number, ++ pcmk__opt_context_schedulerd, ++ N_("The score to use for a node health attribute whose value is " ++ "\"red\""), ++ N_("Only used when \"node-health-strategy\" is set to \"custom\" or " ++ "\"progressive\".") ++ }, ++ ++ // Placement strategy ++ { ++ "placement-strategy", NULL, "select", ++ "default, utilization, minimal, balanced", ++ "default", pcmk__valid_placement_strategy, ++ pcmk__opt_context_schedulerd, ++ N_("How the cluster should allocate resources to nodes"), ++ NULL, ++ }, ++}; ++#endif // 0 ++ ++ + /* + * Environment variable option handling + */ +-- +2.31.1 + +From 96fa08b7adc911cce417f7f9889029510ec1c428 Mon Sep 17 00:00:00 2001 +From: Reid Wahl +Date: Wed, 10 Jan 2024 23:35:22 -0800 +Subject: [PATCH 08/24] Refactor: libcrmcommon: New filter arg to + pcmk__format_option_metadata() + +Now each cluster option is in exactly one daemon's metadata. The four +options that were previously in the metadata of both the controller and +the scheduler are now only in the scheduler's metadata. + +All daemons still have access to all the options they use. + +All function calls in daemons still use the local options arrays rather +than the one in libcrmcommon. That will change in upcoming commits. + +Closes T746 + +Signed-off-by: Reid Wahl +--- + daemons/controld/controld_control.c | 6 ++++-- + daemons/fenced/pacemaker-fenced.c | 17 +++++++++------- + include/crm/common/options_internal.h | 1 + + lib/cib/cib_utils.c | 7 ++++--- + lib/common/options.c | 28 +++++++++++++++++++++++++-- + lib/pengine/common.c | 7 ++++--- + 6 files changed, 49 insertions(+), 17 deletions(-) + +diff --git a/daemons/controld/controld_control.c b/daemons/controld/controld_control.c +index 4d7cb14..8fe09da 100644 +--- a/daemons/controld/controld_control.c ++++ b/daemons/controld/controld_control.c +@@ -723,11 +723,13 @@ static pcmk__cluster_option_t controller_options[] = { + void + crmd_metadata(void) + { ++ const char *name = "pacemaker-controld"; + const char *desc_short = "Pacemaker controller options"; + const char *desc_long = "Cluster options used by Pacemaker's controller"; + +- gchar *s = pcmk__format_option_metadata("pacemaker-controld", desc_short, +- desc_long, controller_options, ++ gchar *s = pcmk__format_option_metadata(name, desc_short, desc_long, ++ pcmk__opt_context_controld, ++ controller_options, + PCMK__NELEM(controller_options)); + printf("%s", s); + g_free(s); +diff --git a/daemons/fenced/pacemaker-fenced.c b/daemons/fenced/pacemaker-fenced.c +index b2f4742..d0b6c31 100644 +--- a/daemons/fenced/pacemaker-fenced.c ++++ b/daemons/fenced/pacemaker-fenced.c +@@ -742,14 +742,17 @@ static pcmk__cluster_option_t fencer_options[] = { + void + fencer_metadata(void) + { ++ const char *name = "pacemaker-fenced"; + const char *desc_short = N_("Instance attributes available for all " +- "\"stonith\"-class resources"); +- const char *desc_long = N_("Instance attributes available for all \"stonith\"-" +- "class resources and used by Pacemaker's fence " +- "daemon, formerly known as stonithd"); +- +- gchar *s = pcmk__format_option_metadata("pacemaker-fenced", desc_short, +- desc_long, fencer_options, ++ "\"stonith\"-class resources"); ++ const char *desc_long = N_("Instance attributes available for all " ++ "\"stonith\"-class resources and used by " ++ "Pacemaker's fence daemon, formerly known as " ++ "stonithd"); ++ ++ gchar *s = pcmk__format_option_metadata(name, desc_short, desc_long, ++ pcmk__opt_context_none, ++ fencer_options, + PCMK__NELEM(fencer_options)); + printf("%s", s); + g_free(s); +diff --git a/include/crm/common/options_internal.h b/include/crm/common/options_internal.h +index b2525ef..89d27d1 100644 +--- a/include/crm/common/options_internal.h ++++ b/include/crm/common/options_internal.h +@@ -70,6 +70,7 @@ const char *pcmk__cluster_option(GHashTable *options, + + gchar *pcmk__format_option_metadata(const char *name, const char *desc_short, + const char *desc_long, ++ enum pcmk__opt_context filter, + pcmk__cluster_option_t *option_list, + int len); + +diff --git a/lib/cib/cib_utils.c b/lib/cib/cib_utils.c +index 9e4060b..2205d15 100644 +--- a/lib/cib/cib_utils.c ++++ b/lib/cib/cib_utils.c +@@ -894,13 +894,14 @@ static pcmk__cluster_option_t cib_opts[] = { + void + cib_metadata(void) + { ++ const char *name = "pacemaker-based"; + const char *desc_short = "Cluster Information Base manager options"; + const char *desc_long = "Cluster options used by Pacemaker's Cluster " + "Information Base manager"; + +- gchar *s = pcmk__format_option_metadata("pacemaker-based", desc_short, +- desc_long, cib_opts, +- PCMK__NELEM(cib_opts)); ++ gchar *s = pcmk__format_option_metadata(name, desc_short, desc_long, ++ pcmk__opt_context_based, ++ cib_opts, PCMK__NELEM(cib_opts)); + printf("%s", s); + g_free(s); + } +diff --git a/lib/common/options.c b/lib/common/options.c +index ff73dcc..d5b6c17 100644 +--- a/lib/common/options.c ++++ b/lib/common/options.c +@@ -937,13 +937,32 @@ add_desc(GString *s, const char *tag, const char *desc, const char *values, + free(escaped_en); + } + ++/*! ++ * \internal ++ * \brief Format option metadata as an OCF-like XML string ++ * ++ * \param[in] name Daemon name ++ * \param[in] desc_short Short description of the daemon ++ * \param[in] desc_long Long description of the daemon ++ * \param[in] filter If not \c pcmk__opt_context_none, include only ++ * those options whose \c context field is equal to ++ * \p filter ++ * \param[in] option_list Options whose metadata to format ++ * \param[in] len Number of items in \p option_list ++ * ++ * \return A string containing OCF-like option metadata XML ++ * ++ * \note The caller is responsible for freeing the return value using ++ * \c g_free(). ++ */ + gchar * + pcmk__format_option_metadata(const char *name, const char *desc_short, + const char *desc_long, ++ enum pcmk__opt_context filter, + pcmk__cluster_option_t *option_list, int len) + { +- /* big enough to hold "pacemaker-schedulerd metadata" output */ +- GString *s = g_string_sized_new(13000); ++ // Large enough to hold current cluster options with room for growth (2^15) ++ GString *s = g_string_sized_new(32768); + + pcmk__g_strcat(s, + "\n" +@@ -964,6 +983,11 @@ pcmk__format_option_metadata(const char *name, const char *desc_short, + const char *opt_desc_short = option_list[lpc].description_short; + const char *opt_desc_long = option_list[lpc].description_long; + ++ if ((filter != pcmk__opt_context_none) ++ && (filter != option_list[lpc].context)) { ++ continue; ++ } ++ + // The standard requires long and short parameter descriptions + CRM_ASSERT((opt_desc_short != NULL) || (opt_desc_long != NULL)); + +diff --git a/lib/pengine/common.c b/lib/pengine/common.c +index 383c4af..e9aa2e2 100644 +--- a/lib/pengine/common.c ++++ b/lib/pengine/common.c +@@ -319,12 +319,13 @@ static pcmk__cluster_option_t pe_opts[] = { + void + pe_metadata(pcmk__output_t *out) + { ++ const char *name = "pacemaker-schedulerd"; + const char *desc_short = "Pacemaker scheduler options"; + const char *desc_long = "Cluster options used by Pacemaker's scheduler"; + +- gchar *s = pcmk__format_option_metadata("pacemaker-schedulerd", desc_short, +- desc_long, pe_opts, +- PCMK__NELEM(pe_opts)); ++ gchar *s = pcmk__format_option_metadata(name, desc_short, desc_long, ++ pcmk__opt_context_schedulerd, ++ pe_opts, PCMK__NELEM(pe_opts)); + out->output_xml(out, "metadata", s); + g_free(s); + } +-- +2.31.1 + +From 96b59bf0c66fccc0656a9195ebe7580d54083eb3 Mon Sep 17 00:00:00 2001 +From: Reid Wahl +Date: Wed, 10 Jan 2024 23:59:39 -0800 +Subject: [PATCH 09/24] Test: cts-cli: Update daemon outputs for option + filtering + +Now each cluster option is in exactly one daemon's metadata. The four +options that were previously in the metadata of both the controller and +the scheduler are now only in the scheduler's metadata. + +All daemons still have access to all the options they use. + +Ref T746 + +Signed-off-by: Reid Wahl +--- + cts/cli/regression.daemons.exp | 26 -------------------------- + 1 file changed, 26 deletions(-) + +diff --git a/cts/cli/regression.daemons.exp b/cts/cli/regression.daemons.exp +index 6a24089..9b2dd96 100644 +--- a/cts/cli/regression.daemons.exp ++++ b/cts/cli/regression.daemons.exp +@@ -101,32 +101,6 @@ + How many times fencing can fail before it will no longer be immediately re-attempted on a target + + +- +- What to do when the cluster does not have quorum Allowed values: stop, freeze, ignore, demote, suicide +- What to do when the cluster does not have quorum +- +- +- +- +- When true, resources active on a node when it is cleanly shut down are kept "locked" to that node (not allowed to run elsewhere) until they start again on that node after it rejoins (or for at most shutdown-lock-limit, if set). Stonith resources and Pacemaker Remote connections are never locked. Clone and bundle instances and the promoted role of promotable clones are currently never locked, though support could be added in a future release. +- Whether to lock resources to a cleanly shut down node +- +- +- +- If shutdown-lock is true and this is set to a nonzero time duration, shutdown locks will expire after this much time has passed since the shutdown was initiated, even if the node has not rejoined. +- Do not lock resources to a cleanly shut down node longer than this +- +- +- +- Fence nodes that do not join the controller process group within this much time after joining the cluster, to allow the cluster to continue managing resources. A value of 0 means never fence pending nodes. Setting the value to 2h means fence nodes after 2 hours. +- How long to wait for a node that has joined the cluster to join the controller process group +- +- + + + =#=#=#= End test: Get controller metadata - OK (0) =#=#=#= +-- +2.31.1 + +From fec945824ed11395a8366882c29315c509de80f0 Mon Sep 17 00:00:00 2001 +From: Reid Wahl +Date: Sat, 6 Jan 2024 19:07:00 -0800 +Subject: [PATCH 10/24] Refactor: libcrmcommon, daemons: New + pcmk__cluster_option_metadata() + +This new function is a wrapper for pcmk__format_option_metadata() that +always uses the shared cluster_options array and its length. + +Daemons can now call this function to get metadata instead of using +their local options arrays for that purpose. + +Soon we'll introduce a command that outputs all cluster option metadata +directly, instead of calling daemon metadata commands. + +Ref T746 + +Signed-off-by: Reid Wahl +--- + daemons/controld/controld_control.c | 7 +++---- + include/crm/common/options_internal.h | 4 ++++ + lib/cib/cib_utils.c | 6 +++--- + lib/common/options.c | 28 +++++++++++++++++++++++++-- + lib/pengine/common.c | 6 +++--- + 5 files changed, 39 insertions(+), 12 deletions(-) + +diff --git a/daemons/controld/controld_control.c b/daemons/controld/controld_control.c +index 8fe09da..82aa143 100644 +--- a/daemons/controld/controld_control.c ++++ b/daemons/controld/controld_control.c +@@ -727,10 +727,9 @@ crmd_metadata(void) + const char *desc_short = "Pacemaker controller options"; + const char *desc_long = "Cluster options used by Pacemaker's controller"; + +- gchar *s = pcmk__format_option_metadata(name, desc_short, desc_long, +- pcmk__opt_context_controld, +- controller_options, +- PCMK__NELEM(controller_options)); ++ gchar *s = pcmk__cluster_option_metadata(name, desc_short, desc_long, ++ pcmk__opt_context_controld); ++ + printf("%s", s); + g_free(s); + } +diff --git a/include/crm/common/options_internal.h b/include/crm/common/options_internal.h +index 89d27d1..a62015f 100644 +--- a/include/crm/common/options_internal.h ++++ b/include/crm/common/options_internal.h +@@ -74,6 +74,10 @@ gchar *pcmk__format_option_metadata(const char *name, const char *desc_short, + pcmk__cluster_option_t *option_list, + int len); + ++gchar *pcmk__cluster_option_metadata(const char *name, const char *desc_short, ++ const char *desc_long, ++ enum pcmk__opt_context filter); ++ + void pcmk__validate_cluster_options(GHashTable *options, + pcmk__cluster_option_t *option_list, + int len); +diff --git a/lib/cib/cib_utils.c b/lib/cib/cib_utils.c +index 2205d15..479a7fb 100644 +--- a/lib/cib/cib_utils.c ++++ b/lib/cib/cib_utils.c +@@ -899,9 +899,9 @@ cib_metadata(void) + const char *desc_long = "Cluster options used by Pacemaker's Cluster " + "Information Base manager"; + +- gchar *s = pcmk__format_option_metadata(name, desc_short, desc_long, +- pcmk__opt_context_based, +- cib_opts, PCMK__NELEM(cib_opts)); ++ gchar *s = pcmk__cluster_option_metadata(name, desc_short, desc_long, ++ pcmk__opt_context_based); ++ + printf("%s", s); + g_free(s); + } +diff --git a/lib/common/options.c b/lib/common/options.c +index d5b6c17..df4a8b4 100644 +--- a/lib/common/options.c ++++ b/lib/common/options.c +@@ -43,7 +43,6 @@ pcmk__cli_help(char cmd) + * Option metadata + */ + +-#if 0 + static pcmk__cluster_option_t cluster_options[] = { + /* name, old name, type, allowed values, + * default value, validator, +@@ -534,7 +533,6 @@ static pcmk__cluster_option_t cluster_options[] = { + NULL, + }, + }; +-#endif // 0 + + + /* +@@ -1036,6 +1034,32 @@ pcmk__format_option_metadata(const char *name, const char *desc_short, + return g_string_free(s, FALSE); + } + ++/*! ++ * \internal ++ * \brief Format cluster option metadata as an OCF-like XML string ++ * ++ * \param[in] name Daemon name ++ * \param[in] desc_short Short description of the daemon ++ * \param[in] desc_long Long description of the daemon ++ * \param[in] filter If not \c pcmk__opt_context_none, include only ++ * those options whose \c context field is equal to ++ * \p filter ++ * ++ * \return A string containing OCF-like cluster option metadata XML ++ * ++ * \note The caller is responsible for freeing the return value using ++ * \c g_free(). ++ */ ++gchar * ++pcmk__cluster_option_metadata(const char *name, const char *desc_short, ++ const char *desc_long, ++ enum pcmk__opt_context filter) ++{ ++ return pcmk__format_option_metadata(name, desc_short, desc_long, filter, ++ cluster_options, ++ PCMK__NELEM(cluster_options)); ++} ++ + void + pcmk__validate_cluster_options(GHashTable *options, + pcmk__cluster_option_t *option_list, int len) +diff --git a/lib/pengine/common.c b/lib/pengine/common.c +index e9aa2e2..c9f1fc1 100644 +--- a/lib/pengine/common.c ++++ b/lib/pengine/common.c +@@ -323,9 +323,9 @@ pe_metadata(pcmk__output_t *out) + const char *desc_short = "Pacemaker scheduler options"; + const char *desc_long = "Cluster options used by Pacemaker's scheduler"; + +- gchar *s = pcmk__format_option_metadata(name, desc_short, desc_long, +- pcmk__opt_context_schedulerd, +- pe_opts, PCMK__NELEM(pe_opts)); ++ gchar *s = pcmk__cluster_option_metadata(name, desc_short, desc_long, ++ pcmk__opt_context_schedulerd); ++ + out->output_xml(out, "metadata", s); + g_free(s); + } +-- +2.31.1 + +From 9a7d33003dffea465e7b452abd0388db4a7d73b0 Mon Sep 17 00:00:00 2001 +From: Reid Wahl +Date: Thu, 11 Jan 2024 00:06:32 -0800 +Subject: [PATCH 11/24] Test: cts-cli: Update daemon outputs for reordering + +In the new libcrmcommon options array, some options have been reordered +to be near other similar options. + +Ref T746 + +Signed-off-by: Reid Wahl +--- + cts/cli/regression.daemons.exp | 40 +++++++++++++++++----------------- + 1 file changed, 20 insertions(+), 20 deletions(-) + +diff --git a/cts/cli/regression.daemons.exp b/cts/cli/regression.daemons.exp +index 9b2dd96..43393df 100644 +--- a/cts/cli/regression.daemons.exp ++++ b/cts/cli/regression.daemons.exp +@@ -51,16 +51,6 @@ + Polling interval to recheck cluster state and evaluate rules with date specifications + + +- +- The cluster will slow down its recovery process when the amount of system resources used (currently CPU) approaches this limit +- Maximum amount of system load that should be used by cluster nodes +- +- +- +- Maximum number of jobs that can be scheduled per node (defaults to 2x cores) +- Maximum number of jobs that can be scheduled per node (defaults to 2x cores) +- +- + + A cluster node may receive notification of its own fencing if fencing is misconfigured, or if fabric fencing is in use that doesn't cut cluster communication. Allowed values are "stop" to attempt to immediately stop Pacemaker and stay stopped, or "panic" to attempt to immediately reboot the local node, falling back to stop on failure. + How a cluster node should react if notified of its own fencing +@@ -101,6 +91,16 @@ + How many times fencing can fail before it will no longer be immediately re-attempted on a target + + ++ ++ The cluster will slow down its recovery process when the amount of system resources used (currently CPU) approaches this limit ++ Maximum amount of system load that should be used by cluster nodes ++ ++ ++ ++ Maximum number of jobs that can be scheduled per node (defaults to 2x cores) ++ Maximum number of jobs that can be scheduled per node (defaults to 2x cores) ++ ++ + + + =#=#=#= End test: Get controller metadata - OK (0) =#=#=#= +@@ -259,6 +259,16 @@ + + ++ ++ When true, resources active on a node when it is cleanly shut down are kept "locked" to that node (not allowed to run elsewhere) until they start again on that node after it rejoins (or for at most shutdown-lock-limit, if set). Stonith resources and Pacemaker Remote connections are never locked. Clone and bundle instances and the promoted role of promotable clones are currently never locked, though support could be added in a future release. ++ Whether to lock resources to a cleanly shut down node ++ ++ ++ ++ If shutdown-lock is true and this is set to a nonzero time duration, shutdown locks will expire after this much time has passed since the shutdown was initiated, even if the node has not rejoined. ++ Do not lock resources to a cleanly shut down node longer than this ++ ++ + + Whether resources can run on any node by default + Whether resources can run on any node by default +@@ -279,16 +289,6 @@ + Whether the cluster should check for active resources during start-up + + +- +- When true, resources active on a node when it is cleanly shut down are kept "locked" to that node (not allowed to run elsewhere) until they start again on that node after it rejoins (or for at most shutdown-lock-limit, if set). Stonith resources and Pacemaker Remote connections are never locked. Clone and bundle instances and the promoted role of promotable clones are currently never locked, though support could be added in a future release. +- Whether to lock resources to a cleanly shut down node +- +- +- +- If shutdown-lock is true and this is set to a nonzero time duration, shutdown locks will expire after this much time has passed since the shutdown was initiated, even if the node has not rejoined. +- Do not lock resources to a cleanly shut down node longer than this +- +- + + If false, unresponsive nodes are immediately assumed to be harmless, and resources that were active on them may be recovered elsewhere. This can result in a "split-brain" situation, potentially leading to data loss and/or service unavailability. + *** Advanced Use Only *** Whether nodes may be fenced as part of recovery +-- +2.31.1 + +From c085ff844deddefe4f00355e2a273f27eb35ce00 Mon Sep 17 00:00:00 2001 +From: Reid Wahl +Date: Thu, 11 Jan 2024 00:23:01 -0800 +Subject: [PATCH 12/24] Refactor: libcrmcommon, daemons: Use cluster_options + array in getters + +Drop the option_list and len arguments from pcmk__cluster_option() and +pcmk__validate_cluster_options(). Use cluster_options in libcrmcommon +instead. + +Now, all daemons fetch and validate the full set of cluster options, +even the ones they don't use. This is only slightly less efficient. It +ensures that there's no problem using the same option with multiple +daemons, and it makes it easy to use new options in a given daemon in +the future. + +Now that nothing is using the local, per-daemon options arrays anymore, +we can drop them in an upcoming commit. + +Ref T746 + +Signed-off-by: Reid Wahl +--- + daemons/controld/controld_control.c | 9 +++--- + include/crm/common/options_internal.h | 8 ++---- + lib/cib/cib_utils.c | 7 +++-- + lib/common/options.c | 40 ++++++++++++--------------- + lib/pengine/common.c | 6 ++-- + 5 files changed, 31 insertions(+), 39 deletions(-) + +diff --git a/daemons/controld/controld_control.c b/daemons/controld/controld_control.c +index 82aa143..4208947 100644 +--- a/daemons/controld/controld_control.c ++++ b/daemons/controld/controld_control.c +@@ -515,6 +515,7 @@ do_recover(long long action, + register_fsa_input(C_FSA_INTERNAL, I_TERMINATE, NULL); + } + ++#if 0 + static pcmk__cluster_option_t controller_options[] = { + /* name, old name, type, allowed values, + * default value, validator, +@@ -719,6 +720,7 @@ static pcmk__cluster_option_t controller_options[] = { + "2 hours.") + }, + }; ++#endif // 0 + + void + crmd_metadata(void) +@@ -775,8 +777,7 @@ config_query_callback(xmlNode * msg, int call_id, int rc, xmlNode * output, void + config_hash, CIB_OPTIONS_FIRST, FALSE, now, NULL); + + // Validate all options, and use defaults if not already present in hash +- pcmk__validate_cluster_options(config_hash, controller_options, +- PCMK__NELEM(controller_options)); ++ pcmk__validate_cluster_options(config_hash); + + /* Validate the watchdog timeout in the context of the local node + * environment. If invalid, the controller will exit with a fatal error. +@@ -900,9 +901,7 @@ crm_shutdown(int nsig) + * config_query_callback() has been run at least once, it doesn't look like + * anything could have changed the timer period since then. + */ +- value = pcmk__cluster_option(NULL, controller_options, +- PCMK__NELEM(controller_options), +- XML_CONFIG_ATTR_FORCE_QUIT); ++ value = pcmk__cluster_option(NULL, XML_CONFIG_ATTR_FORCE_QUIT); + default_period_ms = crm_parse_interval_spec(value); + controld_shutdown_start_countdown(default_period_ms); + } +diff --git a/include/crm/common/options_internal.h b/include/crm/common/options_internal.h +index a62015f..b727a58 100644 +--- a/include/crm/common/options_internal.h ++++ b/include/crm/common/options_internal.h +@@ -64,9 +64,7 @@ typedef struct pcmk__cluster_option_s { + + } pcmk__cluster_option_t; + +-const char *pcmk__cluster_option(GHashTable *options, +- const pcmk__cluster_option_t *option_list, +- int len, const char *name); ++const char *pcmk__cluster_option(GHashTable *options, const char *name); + + gchar *pcmk__format_option_metadata(const char *name, const char *desc_short, + const char *desc_long, +@@ -78,9 +76,7 @@ gchar *pcmk__cluster_option_metadata(const char *name, const char *desc_short, + const char *desc_long, + enum pcmk__opt_context filter); + +-void pcmk__validate_cluster_options(GHashTable *options, +- pcmk__cluster_option_t *option_list, +- int len); ++void pcmk__validate_cluster_options(GHashTable *options); + + bool pcmk__valid_interval_spec(const char *value); + bool pcmk__valid_boolean(const char *value); +diff --git a/lib/cib/cib_utils.c b/lib/cib/cib_utils.c +index 479a7fb..97f62ac 100644 +--- a/lib/cib/cib_utils.c ++++ b/lib/cib/cib_utils.c +@@ -866,6 +866,7 @@ cib_native_notify(gpointer data, gpointer user_data) + crm_trace("Callback invoked..."); + } + ++#if 0 + static pcmk__cluster_option_t cib_opts[] = { + /* name, legacy name, type, allowed values, + * default value, validator, +@@ -890,6 +891,7 @@ static pcmk__cluster_option_t cib_opts[] = { + " multiplied by the number of nodes).") + }, + }; ++#endif // 0 + + void + cib_metadata(void) +@@ -909,14 +911,13 @@ cib_metadata(void) + static void + verify_cib_options(GHashTable *options) + { +- pcmk__validate_cluster_options(options, cib_opts, PCMK__NELEM(cib_opts)); ++ pcmk__validate_cluster_options(options); + } + + const char * + cib_pref(GHashTable * options, const char *name) + { +- return pcmk__cluster_option(options, cib_opts, PCMK__NELEM(cib_opts), +- name); ++ return pcmk__cluster_option(options, name); + } + + gboolean +diff --git a/lib/common/options.c b/lib/common/options.c +index df4a8b4..13d58e3 100644 +--- a/lib/common/options.c ++++ b/lib/common/options.c +@@ -852,27 +852,21 @@ cluster_option_value(GHashTable *options, bool (*validate)(const char *), + * \internal + * \brief Get the value of a cluster option + * +- * \param[in,out] options Name/value pairs for configured options +- * \param[in] option_list Possible cluster options +- * \param[in] len Length of \p option_list +- * \param[in] name (Primary) option name to look for ++ * \param[in,out] options Name/value pairs for configured options ++ * \param[in] name (Primary) option name to look for + * + * \return Option value + */ + const char * +-pcmk__cluster_option(GHashTable *options, +- const pcmk__cluster_option_t *option_list, +- int len, const char *name) ++pcmk__cluster_option(GHashTable *options, const char *name) + { +- const char *value = NULL; +- +- for (int lpc = 0; lpc < len; lpc++) { +- if (pcmk__str_eq(name, option_list[lpc].name, pcmk__str_casei)) { +- value = cluster_option_value(options, option_list[lpc].is_valid, +- option_list[lpc].name, +- option_list[lpc].alt_name, +- option_list[lpc].default_value); +- return value; ++ for (int lpc = 0; lpc < PCMK__NELEM(cluster_options); lpc++) { ++ if (pcmk__str_eq(name, cluster_options[lpc].name, pcmk__str_casei)) { ++ return cluster_option_value(options, ++ cluster_options[lpc].is_valid, ++ cluster_options[lpc].name, ++ cluster_options[lpc].alt_name, ++ cluster_options[lpc].default_value); + } + } + CRM_CHECK(FALSE, crm_err("Bug: looking for unknown option '%s'", name)); +@@ -1061,13 +1055,13 @@ pcmk__cluster_option_metadata(const char *name, const char *desc_short, + } + + void +-pcmk__validate_cluster_options(GHashTable *options, +- pcmk__cluster_option_t *option_list, int len) ++pcmk__validate_cluster_options(GHashTable *options) + { +- for (int lpc = 0; lpc < len; lpc++) { +- cluster_option_value(options, option_list[lpc].is_valid, +- option_list[lpc].name, +- option_list[lpc].alt_name, +- option_list[lpc].default_value); ++ for (int lpc = 0; lpc < PCMK__NELEM(cluster_options); lpc++) { ++ cluster_option_value(options, ++ cluster_options[lpc].is_valid, ++ cluster_options[lpc].name, ++ cluster_options[lpc].alt_name, ++ cluster_options[lpc].default_value); + } + } +diff --git a/lib/pengine/common.c b/lib/pengine/common.c +index c9f1fc1..f99bd1b 100644 +--- a/lib/pengine/common.c ++++ b/lib/pengine/common.c +@@ -21,6 +21,7 @@ + gboolean was_processing_error = FALSE; + gboolean was_processing_warning = FALSE; + ++#if 0 + static pcmk__cluster_option_t pe_opts[] = { + /* name, old name, type, allowed values, + * default value, validator, +@@ -315,6 +316,7 @@ static pcmk__cluster_option_t pe_opts[] = { + NULL + }, + }; ++#endif // 0 + + void + pe_metadata(pcmk__output_t *out) +@@ -333,13 +335,13 @@ pe_metadata(pcmk__output_t *out) + void + verify_pe_options(GHashTable * options) + { +- pcmk__validate_cluster_options(options, pe_opts, PCMK__NELEM(pe_opts)); ++ pcmk__validate_cluster_options(options); + } + + const char * + pe_pref(GHashTable * options, const char *name) + { +- return pcmk__cluster_option(options, pe_opts, PCMK__NELEM(pe_opts), name); ++ return pcmk__cluster_option(options, name); + } + + const char * +-- +2.31.1 + +From de834cee2c5d8f4f796633e66f263ad77b9cd2eb Mon Sep 17 00:00:00 2001 +From: Reid Wahl +Date: Mon, 8 Jan 2024 03:06:27 -0800 +Subject: [PATCH 13/24] Refactor: various: Drop per-daemon cluster opt tables + +Ref T746 + +Signed-off-by: Reid Wahl +--- + daemons/controld/controld_control.c | 207 ------------------- + lib/cib/cib_utils.c | 27 --- + lib/pengine/common.c | 297 ---------------------------- + 3 files changed, 531 deletions(-) + +diff --git a/daemons/controld/controld_control.c b/daemons/controld/controld_control.c +index 4208947..40b90f8 100644 +--- a/daemons/controld/controld_control.c ++++ b/daemons/controld/controld_control.c +@@ -515,213 +515,6 @@ do_recover(long long action, + register_fsa_input(C_FSA_INTERNAL, I_TERMINATE, NULL); + } + +-#if 0 +-static pcmk__cluster_option_t controller_options[] = { +- /* name, old name, type, allowed values, +- * default value, validator, +- * context, +- * short description, +- * long description +- */ +- { +- "dc-version", NULL, "string", NULL, PCMK__VALUE_NONE, NULL, +- pcmk__opt_context_controld, +- N_("Pacemaker version on cluster node elected Designated Controller (DC)"), +- N_("Includes a hash which identifies the exact changeset the code was " +- "built from. Used for diagnostic purposes.") +- }, +- { +- "cluster-infrastructure", NULL, "string", NULL, "corosync", NULL, +- pcmk__opt_context_controld, +- N_("The messaging stack on which Pacemaker is currently running"), +- N_("Used for informational and diagnostic purposes.") +- }, +- { +- "cluster-name", NULL, "string", NULL, NULL, NULL, +- pcmk__opt_context_controld, +- N_("An arbitrary name for the cluster"), +- N_("This optional value is mostly for users' convenience as desired " +- "in administration, but may also be used in Pacemaker " +- "configuration rules via the #cluster-name node attribute, and " +- "by higher-level tools and resource agents.") +- }, +- { +- XML_CONFIG_ATTR_DC_DEADTIME, NULL, "time", +- NULL, "20s", pcmk__valid_interval_spec, +- pcmk__opt_context_controld, +- N_("How long to wait for a response from other nodes during start-up"), +- N_("The optimal value will depend on the speed and load of your network " +- "and the type of switches used.") +- }, +- { +- XML_CONFIG_ATTR_RECHECK, NULL, "time", +- N_("Zero disables polling, while positive values are an interval in seconds" +- "(unless other units are specified, for example \"5min\")"), +- "15min", pcmk__valid_interval_spec, +- pcmk__opt_context_controld, +- N_("Polling interval to recheck cluster state and evaluate rules " +- "with date specifications"), +- N_("Pacemaker is primarily event-driven, and looks ahead to know when to " +- "recheck cluster state for failure timeouts and most time-based " +- "rules. However, it will also recheck the cluster after this " +- "amount of inactivity, to evaluate rules with date specifications " +- "and serve as a fail-safe for certain types of scheduler bugs.") +- }, +- { +- "load-threshold", NULL, "percentage", NULL, +- "80%", pcmk__valid_percentage, +- pcmk__opt_context_controld, +- N_("Maximum amount of system load that should be used by cluster nodes"), +- N_("The cluster will slow down its recovery process when the amount of " +- "system resources used (currently CPU) approaches this limit"), +- }, +- { +- "node-action-limit", NULL, "integer", NULL, +- "0", pcmk__valid_number, +- pcmk__opt_context_controld, +- N_("Maximum number of jobs that can be scheduled per node " +- "(defaults to 2x cores)") +- }, +- { XML_CONFIG_ATTR_FENCE_REACTION, NULL, "string", NULL, "stop", NULL, +- pcmk__opt_context_controld, +- N_("How a cluster node should react if notified of its own fencing"), +- N_("A cluster node may receive notification of its own fencing if fencing " +- "is misconfigured, or if fabric fencing is in use that doesn't cut " +- "cluster communication. Allowed values are \"stop\" to attempt to " +- "immediately stop Pacemaker and stay stopped, or \"panic\" to attempt " +- "to immediately reboot the local node, falling back to stop on failure.") +- }, +- { +- XML_CONFIG_ATTR_ELECTION_FAIL, NULL, "time", NULL, +- "2min", pcmk__valid_interval_spec, +- pcmk__opt_context_controld, +- "*** Advanced Use Only ***", +- N_("Declare an election failed if it is not decided within this much " +- "time. If you need to adjust this value, it probably indicates " +- "the presence of a bug.") +- }, +- { +- XML_CONFIG_ATTR_FORCE_QUIT, NULL, "time", NULL, +- "20min", pcmk__valid_interval_spec, +- pcmk__opt_context_controld, +- "*** Advanced Use Only ***", +- N_("Exit immediately if shutdown does not complete within this much " +- "time. If you need to adjust this value, it probably indicates " +- "the presence of a bug.") +- }, +- { +- "join-integration-timeout", "crmd-integration-timeout", "time", NULL, +- "3min", pcmk__valid_interval_spec, +- pcmk__opt_context_controld, +- "*** Advanced Use Only ***", +- N_("If you need to adjust this value, it probably indicates " +- "the presence of a bug.") +- }, +- { +- "join-finalization-timeout", "crmd-finalization-timeout", "time", NULL, +- "30min", pcmk__valid_interval_spec, +- pcmk__opt_context_controld, +- "*** Advanced Use Only ***", +- N_("If you need to adjust this value, it probably indicates " +- "the presence of a bug.") +- }, +- { +- "transition-delay", "crmd-transition-delay", "time", NULL, +- "0s", pcmk__valid_interval_spec, +- pcmk__opt_context_controld, +- N_("*** Advanced Use Only *** Enabling this option will slow down " +- "cluster recovery under all conditions"), +- N_("Delay cluster recovery for this much time to allow for additional " +- "events to occur. Useful if your configuration is sensitive to " +- "the order in which ping updates arrive.") +- }, +- { +- /* @COMPAT Currently unparsable values default to -1 (auto-calculate), +- * while missing values default to 0 (disable). All values are accepted +- * (unless the controller finds that the value conflicts with the +- * SBD_WATCHDOG_TIMEOUT). +- * +- * At a compatibility break: properly validate as a timeout, let +- * either negative values or a particular string like "auto" mean auto- +- * calculate, and use 0 as the single default for when the option either +- * is unset or fails to validate. +- */ +- "stonith-watchdog-timeout", NULL, "time", NULL, +- "0", NULL, +- pcmk__opt_context_controld, +- N_("How long before nodes can be assumed to be safely down when " +- "watchdog-based self-fencing via SBD is in use"), +- N_("If this is set to a positive value, lost nodes are assumed to " +- "self-fence using watchdog-based SBD within this much time. This " +- "does not require a fencing resource to be explicitly configured, " +- "though a fence_watchdog resource can be configured, to limit use " +- "to specific nodes. If this is set to 0 (the default), the cluster " +- "will never assume watchdog-based self-fencing. If this is set to a " +- "negative value, the cluster will use twice the local value of the " +- "`SBD_WATCHDOG_TIMEOUT` environment variable if that is positive, " +- "or otherwise treat this as 0. WARNING: When used, this timeout " +- "must be larger than `SBD_WATCHDOG_TIMEOUT` on all nodes that use " +- "watchdog-based SBD, and Pacemaker will refuse to start on any of " +- "those nodes where this is not true for the local value or SBD is " +- "not active. When this is set to a negative value, " +- "`SBD_WATCHDOG_TIMEOUT` must be set to the same value on all nodes " +- "that use SBD, otherwise data corruption or loss could occur.") +- }, +- { +- "stonith-max-attempts", NULL, "integer", NULL, +- "10", pcmk__valid_positive_number, +- pcmk__opt_context_controld, +- N_("How many times fencing can fail before it will no longer be " +- "immediately re-attempted on a target") +- }, +- +- // Already documented in libpe_status (other values must be kept identical) +- { +- "no-quorum-policy", NULL, "select", +- "stop, freeze, ignore, demote, suicide", "stop", pcmk__valid_quorum, +- pcmk__opt_context_controld, +- N_("What to do when the cluster does not have quorum"), NULL +- }, +- { +- XML_CONFIG_ATTR_SHUTDOWN_LOCK, NULL, "boolean", NULL, +- "false", pcmk__valid_boolean, +- pcmk__opt_context_controld, +- N_("Whether to lock resources to a cleanly shut down node"), +- N_("When true, resources active on a node when it is cleanly shut down " +- "are kept \"locked\" to that node (not allowed to run elsewhere) " +- "until they start again on that node after it rejoins (or for at " +- "most shutdown-lock-limit, if set). Stonith resources and " +- "Pacemaker Remote connections are never locked. Clone and bundle " +- "instances and the promoted role of promotable clones are " +- "currently never locked, though support could be added in a future " +- "release.") +- }, +- { +- XML_CONFIG_ATTR_SHUTDOWN_LOCK_LIMIT, NULL, "time", NULL, +- "0", pcmk__valid_interval_spec, +- pcmk__opt_context_controld, +- N_("Do not lock resources to a cleanly shut down node longer than " +- "this"), +- N_("If shutdown-lock is true and this is set to a nonzero time " +- "duration, shutdown locks will expire after this much time has " +- "passed since the shutdown was initiated, even if the node has not " +- "rejoined.") +- }, +- { +- XML_CONFIG_ATTR_NODE_PENDING_TIMEOUT, NULL, "time", NULL, +- "0", pcmk__valid_interval_spec, +- pcmk__opt_context_controld, +- N_("How long to wait for a node that has joined the cluster to join " +- "the controller process group"), +- N_("Fence nodes that do not join the controller process group within " +- "this much time after joining the cluster, to allow the cluster " +- "to continue managing resources. A value of 0 means never fence " +- "pending nodes. Setting the value to 2h means fence nodes after " +- "2 hours.") +- }, +-}; +-#endif // 0 +- + void + crmd_metadata(void) + { +diff --git a/lib/cib/cib_utils.c b/lib/cib/cib_utils.c +index 97f62ac..b83158c 100644 +--- a/lib/cib/cib_utils.c ++++ b/lib/cib/cib_utils.c +@@ -866,33 +866,6 @@ cib_native_notify(gpointer data, gpointer user_data) + crm_trace("Callback invoked..."); + } + +-#if 0 +-static pcmk__cluster_option_t cib_opts[] = { +- /* name, legacy name, type, allowed values, +- * default value, validator, +- * context, +- * short description, +- * long description +- */ +- { +- "enable-acl", NULL, "boolean", NULL, +- "false", pcmk__valid_boolean, +- pcmk__opt_context_based, +- N_("Enable Access Control Lists (ACLs) for the CIB"), +- NULL +- }, +- { +- "cluster-ipc-limit", NULL, "integer", NULL, +- "500", pcmk__valid_positive_number, +- pcmk__opt_context_based, +- N_("Maximum IPC message backlog before disconnecting a cluster daemon"), +- N_("Raise this if log has \"Evicting client\" messages for cluster daemon" +- " PIDs (a good value is the number of resources in the cluster" +- " multiplied by the number of nodes).") +- }, +-}; +-#endif // 0 +- + void + cib_metadata(void) + { +diff --git a/lib/pengine/common.c b/lib/pengine/common.c +index f99bd1b..e96f0b5 100644 +--- a/lib/pengine/common.c ++++ b/lib/pengine/common.c +@@ -21,303 +21,6 @@ + gboolean was_processing_error = FALSE; + gboolean was_processing_warning = FALSE; + +-#if 0 +-static pcmk__cluster_option_t pe_opts[] = { +- /* name, old name, type, allowed values, +- * default value, validator, +- * context, +- * short description, +- * long description +- */ +- { +- "no-quorum-policy", NULL, "select", "stop, freeze, ignore, demote, suicide", +- "stop", pcmk__valid_quorum, +- pcmk__opt_context_schedulerd, +- N_("What to do when the cluster does not have quorum"), +- NULL +- }, +- { +- "symmetric-cluster", NULL, "boolean", NULL, +- "true", pcmk__valid_boolean, +- pcmk__opt_context_schedulerd, +- N_("Whether resources can run on any node by default"), +- NULL +- }, +- { +- "maintenance-mode", NULL, "boolean", NULL, +- "false", pcmk__valid_boolean, +- pcmk__opt_context_schedulerd, +- N_("Whether the cluster should refrain from monitoring, starting, " +- "and stopping resources"), +- NULL +- }, +- { +- "start-failure-is-fatal", NULL, "boolean", NULL, +- "true", pcmk__valid_boolean, +- pcmk__opt_context_schedulerd, +- N_("Whether a start failure should prevent a resource from being " +- "recovered on the same node"), +- N_("When true, the cluster will immediately ban a resource from a node " +- "if it fails to start there. When false, the cluster will instead " +- "check the resource's fail count against its migration-threshold.") +- }, +- { +- "enable-startup-probes", NULL, "boolean", NULL, +- "true", pcmk__valid_boolean, +- pcmk__opt_context_schedulerd, +- N_("Whether the cluster should check for active resources during start-up"), +- NULL +- }, +- { +- XML_CONFIG_ATTR_SHUTDOWN_LOCK, NULL, "boolean", NULL, +- "false", pcmk__valid_boolean, +- pcmk__opt_context_schedulerd, +- N_("Whether to lock resources to a cleanly shut down node"), +- N_("When true, resources active on a node when it is cleanly shut down " +- "are kept \"locked\" to that node (not allowed to run elsewhere) " +- "until they start again on that node after it rejoins (or for at " +- "most shutdown-lock-limit, if set). Stonith resources and " +- "Pacemaker Remote connections are never locked. Clone and bundle " +- "instances and the promoted role of promotable clones are " +- "currently never locked, though support could be added in a future " +- "release.") +- }, +- { +- XML_CONFIG_ATTR_SHUTDOWN_LOCK_LIMIT, NULL, "time", NULL, +- "0", pcmk__valid_interval_spec, +- pcmk__opt_context_schedulerd, +- N_("Do not lock resources to a cleanly shut down node longer than " +- "this"), +- N_("If shutdown-lock is true and this is set to a nonzero time " +- "duration, shutdown locks will expire after this much time has " +- "passed since the shutdown was initiated, even if the node has not " +- "rejoined.") +- }, +- +- // Fencing-related options +- { +- "stonith-enabled", NULL, "boolean", NULL, +- "true", pcmk__valid_boolean, +- pcmk__opt_context_schedulerd, +- N_("*** Advanced Use Only *** " +- "Whether nodes may be fenced as part of recovery"), +- N_("If false, unresponsive nodes are immediately assumed to be harmless, " +- "and resources that were active on them may be recovered " +- "elsewhere. This can result in a \"split-brain\" situation, " +- "potentially leading to data loss and/or service unavailability.") +- }, +- { +- "stonith-action", NULL, "select", "reboot, off, poweroff", +- PCMK_ACTION_REBOOT, pcmk__is_fencing_action, +- pcmk__opt_context_schedulerd, +- N_("Action to send to fence device when a node needs to be fenced " +- "(\"poweroff\" is a deprecated alias for \"off\")"), +- NULL +- }, +- { +- "stonith-timeout", NULL, "time", NULL, +- "60s", pcmk__valid_interval_spec, +- pcmk__opt_context_schedulerd, +- N_("*** Advanced Use Only *** Unused by Pacemaker"), +- N_("This value is not used by Pacemaker, but is kept for backward " +- "compatibility, and certain legacy fence agents might use it.") +- }, +- { +- XML_ATTR_HAVE_WATCHDOG, NULL, "boolean", NULL, +- "false", pcmk__valid_boolean, +- pcmk__opt_context_schedulerd, +- N_("Whether watchdog integration is enabled"), +- N_("This is set automatically by the cluster according to whether SBD " +- "is detected to be in use. User-configured values are ignored. " +- "The value `true` is meaningful if diskless SBD is used and " +- "`stonith-watchdog-timeout` is nonzero. In that case, if fencing " +- "is required, watchdog-based self-fencing will be performed via " +- "SBD without requiring a fencing resource explicitly configured.") +- }, +- { +- "concurrent-fencing", NULL, "boolean", NULL, +- PCMK__CONCURRENT_FENCING_DEFAULT, pcmk__valid_boolean, +- pcmk__opt_context_schedulerd, +- N_("Allow performing fencing operations in parallel"), +- NULL +- }, +- { +- "startup-fencing", NULL, "boolean", NULL, +- "true", pcmk__valid_boolean, +- pcmk__opt_context_schedulerd, +- N_("*** Advanced Use Only *** Whether to fence unseen nodes at start-up"), +- N_("Setting this to false may lead to a \"split-brain\" situation," +- "potentially leading to data loss and/or service unavailability.") +- }, +- { +- XML_CONFIG_ATTR_PRIORITY_FENCING_DELAY, NULL, "time", NULL, +- "0", pcmk__valid_interval_spec, +- pcmk__opt_context_schedulerd, +- N_("Apply fencing delay targeting the lost nodes with the highest total resource priority"), +- N_("Apply specified delay for the fencings that are targeting the lost " +- "nodes with the highest total resource priority in case we don't " +- "have the majority of the nodes in our cluster partition, so that " +- "the more significant nodes potentially win any fencing match, " +- "which is especially meaningful under split-brain of 2-node " +- "cluster. A promoted resource instance takes the base priority + 1 " +- "on calculation if the base priority is not 0. Any static/random " +- "delays that are introduced by `pcmk_delay_base/max` configured " +- "for the corresponding fencing resources will be added to this " +- "delay. This delay should be significantly greater than, safely " +- "twice, the maximum `pcmk_delay_base/max`. By default, priority " +- "fencing delay is disabled.") +- }, +- { +- XML_CONFIG_ATTR_NODE_PENDING_TIMEOUT, NULL, "time", NULL, +- "0", pcmk__valid_interval_spec, +- pcmk__opt_context_schedulerd, +- N_("How long to wait for a node that has joined the cluster to join " +- "the controller process group"), +- N_("Fence nodes that do not join the controller process group within " +- "this much time after joining the cluster, to allow the cluster " +- "to continue managing resources. A value of 0 means never fence " +- "pending nodes. Setting the value to 2h means fence nodes after " +- "2 hours.") +- }, +- { +- "cluster-delay", NULL, "time", NULL, +- "60s", pcmk__valid_interval_spec, +- pcmk__opt_context_schedulerd, +- N_("Maximum time for node-to-node communication"), +- N_("The node elected Designated Controller (DC) will consider an action " +- "failed if it does not get a response from the node executing the " +- "action within this time (after considering the action's own " +- "timeout). The \"correct\" value will depend on the speed and " +- "load of your network and cluster nodes.") +- }, +- { +- "batch-limit", NULL, "integer", NULL, +- "0", pcmk__valid_number, +- pcmk__opt_context_schedulerd, +- N_("Maximum number of jobs that the cluster may execute in parallel " +- "across all nodes"), +- N_("The \"correct\" value will depend on the speed and load of your " +- "network and cluster nodes. If set to 0, the cluster will " +- "impose a dynamically calculated limit when any node has a " +- "high load.") +- }, +- { +- "migration-limit", NULL, "integer", NULL, +- "-1", pcmk__valid_number, +- pcmk__opt_context_schedulerd, +- N_("The number of live migration actions that the cluster is allowed " +- "to execute in parallel on a node (-1 means no limit)") +- }, +- +- /* Orphans and stopping */ +- { +- "stop-all-resources", NULL, "boolean", NULL, +- "false", pcmk__valid_boolean, +- pcmk__opt_context_schedulerd, +- N_("Whether the cluster should stop all active resources"), +- NULL +- }, +- { +- "stop-orphan-resources", NULL, "boolean", NULL, +- "true", pcmk__valid_boolean, +- pcmk__opt_context_schedulerd, +- N_("Whether to stop resources that were removed from the configuration"), +- NULL +- }, +- { +- "stop-orphan-actions", NULL, "boolean", NULL, +- "true", pcmk__valid_boolean, +- pcmk__opt_context_schedulerd, +- N_("Whether to cancel recurring actions removed from the configuration"), +- NULL +- }, +- { +- "remove-after-stop", NULL, "boolean", NULL, +- "false", pcmk__valid_boolean, +- pcmk__opt_context_schedulerd, +- N_("*** Deprecated *** Whether to remove stopped resources from " +- "the executor"), +- N_("Values other than default are poorly tested and potentially dangerous." +- " This option will be removed in a future release.") +- }, +- +- /* Storing inputs */ +- { +- "pe-error-series-max", NULL, "integer", NULL, +- "-1", pcmk__valid_number, +- pcmk__opt_context_schedulerd, +- N_("The number of scheduler inputs resulting in errors to save"), +- N_("Zero to disable, -1 to store unlimited.") +- }, +- { +- "pe-warn-series-max", NULL, "integer", NULL, +- "5000", pcmk__valid_number, +- pcmk__opt_context_schedulerd, +- N_("The number of scheduler inputs resulting in warnings to save"), +- N_("Zero to disable, -1 to store unlimited.") +- }, +- { +- "pe-input-series-max", NULL, "integer", NULL, +- "4000", pcmk__valid_number, +- pcmk__opt_context_schedulerd, +- N_("The number of scheduler inputs without errors or warnings to save"), +- N_("Zero to disable, -1 to store unlimited.") +- }, +- +- /* Node health */ +- { +- PCMK__OPT_NODE_HEALTH_STRATEGY, NULL, "select", +- PCMK__VALUE_NONE ", " PCMK__VALUE_MIGRATE_ON_RED ", " +- PCMK__VALUE_ONLY_GREEN ", " PCMK__VALUE_PROGRESSIVE ", " +- PCMK__VALUE_CUSTOM, +- PCMK__VALUE_NONE, pcmk__validate_health_strategy, +- pcmk__opt_context_schedulerd, +- N_("How cluster should react to node health attributes"), +- N_("Requires external entities to create node attributes (named with " +- "the prefix \"#health\") with values \"red\", " +- "\"yellow\", or \"green\".") +- }, +- { +- PCMK__OPT_NODE_HEALTH_BASE, NULL, "integer", NULL, +- "0", pcmk__valid_number, +- pcmk__opt_context_schedulerd, +- N_("Base health score assigned to a node"), +- N_("Only used when \"node-health-strategy\" is set to \"progressive\".") +- }, +- { +- PCMK__OPT_NODE_HEALTH_GREEN, NULL, "integer", NULL, +- "0", pcmk__valid_number, +- pcmk__opt_context_schedulerd, +- N_("The score to use for a node health attribute whose value is \"green\""), +- N_("Only used when \"node-health-strategy\" is set to \"custom\" or \"progressive\".") +- }, +- { +- PCMK__OPT_NODE_HEALTH_YELLOW, NULL, "integer", NULL, +- "0", pcmk__valid_number, +- pcmk__opt_context_schedulerd, +- N_("The score to use for a node health attribute whose value is \"yellow\""), +- N_("Only used when \"node-health-strategy\" is set to \"custom\" or \"progressive\".") +- }, +- { +- PCMK__OPT_NODE_HEALTH_RED, NULL, "integer", NULL, +- "-INFINITY", pcmk__valid_number, +- pcmk__opt_context_schedulerd, +- N_("The score to use for a node health attribute whose value is \"red\""), +- N_("Only used when \"node-health-strategy\" is set to \"custom\" or \"progressive\".") +- }, +- +- /*Placement Strategy*/ +- { +- "placement-strategy", NULL, "select", +- "default, utilization, minimal, balanced", +- "default", pcmk__valid_placement_strategy, +- pcmk__opt_context_schedulerd, +- N_("How the cluster should allocate resources to nodes"), +- NULL +- }, +-}; +-#endif // 0 +- + void + pe_metadata(pcmk__output_t *out) + { +-- +2.31.1 + +From 9a8bb049fcb49204932e96014c3a63e58fd95d23 Mon Sep 17 00:00:00 2001 +From: Reid Wahl +Date: Mon, 8 Jan 2024 20:36:34 -0800 +Subject: [PATCH 14/24] Refactor: libpe_status: Drop verify_pe_opts() + +Signed-off-by: Reid Wahl +--- + include/crm/pengine/internal.h | 1 - + lib/pengine/common.c | 6 ------ + lib/pengine/unpack.c | 2 +- + 3 files changed, 1 insertion(+), 8 deletions(-) + +diff --git a/include/crm/pengine/internal.h b/include/crm/pengine/internal.h +index 9c8068f..5835ef8 100644 +--- a/include/crm/pengine/internal.h ++++ b/include/crm/pengine/internal.h +@@ -209,7 +209,6 @@ pcmk_node_t *native_location(const pcmk_resource_t *rsc, GList **list, + int current); + + void pe_metadata(pcmk__output_t *out); +-void verify_pe_options(GHashTable * options); + + void native_add_running(pcmk_resource_t *rsc, pcmk_node_t *node, + pcmk_scheduler_t *scheduler, gboolean failed); +diff --git a/lib/pengine/common.c b/lib/pengine/common.c +index e96f0b5..402fae9 100644 +--- a/lib/pengine/common.c ++++ b/lib/pengine/common.c +@@ -35,12 +35,6 @@ pe_metadata(pcmk__output_t *out) + g_free(s); + } + +-void +-verify_pe_options(GHashTable * options) +-{ +- pcmk__validate_cluster_options(options); +-} +- + const char * + pe_pref(GHashTable * options, const char *name) + { +diff --git a/lib/pengine/unpack.c b/lib/pengine/unpack.c +index 3429d56..2a9b563 100644 +--- a/lib/pengine/unpack.c ++++ b/lib/pengine/unpack.c +@@ -228,7 +228,7 @@ unpack_config(xmlNode *config, pcmk_scheduler_t *scheduler) + pe__unpack_dataset_nvpairs(config, XML_CIB_TAG_PROPSET, &rule_data, config_hash, + CIB_OPTIONS_FIRST, FALSE, scheduler); + +- verify_pe_options(scheduler->config_hash); ++ pcmk__validate_cluster_options(config_hash); + + set_config_flag(scheduler, "enable-startup-probes", + pcmk_sched_probe_resources); +-- +2.31.1 + +From af79c50b7a5626218bf2a9b34fe631f07b1e2bda Mon Sep 17 00:00:00 2001 +From: Reid Wahl +Date: Mon, 8 Jan 2024 08:40:13 -0800 +Subject: [PATCH 15/24] Refactor: libpe_status: Drop pe_pref() internally + +Signed-off-by: Reid Wahl +--- + daemons/schedulerd/schedulerd_messages.c | 3 ++- + include/crm/pengine/internal.h | 10 +++++++--- + lib/pacemaker/pcmk_graph_producer.c | 9 +++++---- + lib/pacemaker/pcmk_sched_nodes.c | 5 +++-- + lib/pengine/unpack.c | 4 +++- + tools/crm_resource_print.c | 4 ++-- + 6 files changed, 22 insertions(+), 13 deletions(-) + +diff --git a/daemons/schedulerd/schedulerd_messages.c b/daemons/schedulerd/schedulerd_messages.c +index 5a97365..ff31fce 100644 +--- a/daemons/schedulerd/schedulerd_messages.c ++++ b/daemons/schedulerd/schedulerd_messages.c +@@ -112,7 +112,8 @@ handle_pecalc_request(pcmk__request_t *request) + series_id = 2; + } + +- value = pe_pref(scheduler->config_hash, series[series_id].param); ++ value = pcmk__cluster_option(scheduler->config_hash, ++ series[series_id].param); + if ((value == NULL) + || (pcmk__scan_min_int(value, &series_wrap, -1) != pcmk_rc_ok)) { + series_wrap = series[series_id].wrap; +diff --git a/include/crm/pengine/internal.h b/include/crm/pengine/internal.h +index 5835ef8..2b7f2eb 100644 +--- a/include/crm/pengine/internal.h ++++ b/include/crm/pengine/internal.h +@@ -620,14 +620,18 @@ int pe__node_health(pcmk_node_t *node); + static inline enum pcmk__health_strategy + pe__health_strategy(pcmk_scheduler_t *scheduler) + { +- return pcmk__parse_health_strategy(pe_pref(scheduler->config_hash, +- PCMK__OPT_NODE_HEALTH_STRATEGY)); ++ const char *strategy = pcmk__cluster_option(scheduler->config_hash, ++ PCMK__OPT_NODE_HEALTH_STRATEGY); ++ ++ return pcmk__parse_health_strategy(strategy); + } + + static inline int + pe__health_score(const char *option, pcmk_scheduler_t *scheduler) + { +- return char2score(pe_pref(scheduler->config_hash, option)); ++ const char *value = pcmk__cluster_option(scheduler->config_hash, option); ++ ++ return char2score(value); + } + + /*! +diff --git a/lib/pacemaker/pcmk_graph_producer.c b/lib/pacemaker/pcmk_graph_producer.c +index 59b6176..3006775 100644 +--- a/lib/pacemaker/pcmk_graph_producer.c ++++ b/lib/pacemaker/pcmk_graph_producer.c +@@ -1004,16 +1004,17 @@ pcmk__create_graph(pcmk_scheduler_t *scheduler) + GList *iter = NULL; + const char *value = NULL; + long long limit = 0LL; ++ GHashTable *config_hash = scheduler->config_hash; + + transition_id++; + crm_trace("Creating transition graph %d", transition_id); + + scheduler->graph = create_xml_node(NULL, XML_TAG_GRAPH); + +- value = pe_pref(scheduler->config_hash, "cluster-delay"); ++ value = pcmk__cluster_option(config_hash, "cluster-delay"); + crm_xml_add(scheduler->graph, "cluster-delay", value); + +- value = pe_pref(scheduler->config_hash, "stonith-timeout"); ++ value = pcmk__cluster_option(config_hash, "stonith-timeout"); + crm_xml_add(scheduler->graph, "stonith-timeout", value); + + crm_xml_add(scheduler->graph, "failed-stop-offset", "INFINITY"); +@@ -1024,12 +1025,12 @@ pcmk__create_graph(pcmk_scheduler_t *scheduler) + crm_xml_add(scheduler->graph, "failed-start-offset", "1"); + } + +- value = pe_pref(scheduler->config_hash, "batch-limit"); ++ value = pcmk__cluster_option(config_hash, "batch-limit"); + crm_xml_add(scheduler->graph, "batch-limit", value); + + crm_xml_add_int(scheduler->graph, "transition_id", transition_id); + +- value = pe_pref(scheduler->config_hash, "migration-limit"); ++ value = pcmk__cluster_option(config_hash, "migration-limit"); + if ((pcmk__scan_ll(value, &limit, 0LL) == pcmk_rc_ok) && (limit > 0)) { + crm_xml_add(scheduler->graph, "migration-limit", value); + } +diff --git a/lib/pacemaker/pcmk_sched_nodes.c b/lib/pacemaker/pcmk_sched_nodes.c +index 9cf5545..03baa2c 100644 +--- a/lib/pacemaker/pcmk_sched_nodes.c ++++ b/lib/pacemaker/pcmk_sched_nodes.c +@@ -360,8 +360,9 @@ pcmk__apply_node_health(pcmk_scheduler_t *scheduler) + { + int base_health = 0; + enum pcmk__health_strategy strategy; +- const char *strategy_str = pe_pref(scheduler->config_hash, +- PCMK__OPT_NODE_HEALTH_STRATEGY); ++ const char *strategy_str = ++ pcmk__cluster_option(scheduler->config_hash, ++ PCMK__OPT_NODE_HEALTH_STRATEGY); + + strategy = pcmk__parse_health_strategy(strategy_str); + if (strategy == pcmk__health_strategy_none) { +diff --git a/lib/pengine/unpack.c b/lib/pengine/unpack.c +index 2a9b563..49443c6 100644 +--- a/lib/pengine/unpack.c ++++ b/lib/pengine/unpack.c +@@ -50,7 +50,9 @@ struct action_history { + * flag is stringified more readably in log messages. + */ + #define set_config_flag(scheduler, option, flag) do { \ +- const char *scf_value = pe_pref((scheduler)->config_hash, (option)); \ ++ GHashTable *config_hash = (scheduler)->config_hash; \ ++ const char *scf_value = pcmk__cluster_option(config_hash, (option)); \ ++ \ + if (scf_value != NULL) { \ + if (crm_is_true(scf_value)) { \ + (scheduler)->flags = pcmk__set_flags_as(__func__, __LINE__, \ +diff --git a/tools/crm_resource_print.c b/tools/crm_resource_print.c +index bdf3ad9..26761dd 100644 +--- a/tools/crm_resource_print.c ++++ b/tools/crm_resource_print.c +@@ -479,8 +479,8 @@ resource_check_list_default(pcmk__output_t *out, va_list args) { + "'%s' cannot run on unhealthy nodes due to " + PCMK__OPT_NODE_HEALTH_STRATEGY "='%s'", + parent->id, +- pe_pref(checks->rsc->cluster->config_hash, +- PCMK__OPT_NODE_HEALTH_STRATEGY)); ++ pcmk__cluster_option(checks->rsc->cluster->config_hash, ++ PCMK__OPT_NODE_HEALTH_STRATEGY)); + } + + out->end_list(out); +-- +2.31.1 + +From 1e78e617965b1a2e1a5671aa15943ba42487b09a Mon Sep 17 00:00:00 2001 +From: Reid Wahl +Date: Mon, 8 Jan 2024 08:43:55 -0800 +Subject: [PATCH 16/24] API: libpe_status: Deprecate pe_pref() + +Signed-off-by: Reid Wahl +--- + include/crm/pengine/common.h | 4 +--- + include/crm/pengine/common_compat.h | 5 ++++- + lib/pengine/common.c | 20 ++++++++++++++------ + lib/pengine/unpack.c | 28 ++++++++++++++-------------- + 4 files changed, 33 insertions(+), 24 deletions(-) + +diff --git a/include/crm/pengine/common.h b/include/crm/pengine/common.h +index 2feac8a..a935aa7 100644 +--- a/include/crm/pengine/common.h ++++ b/include/crm/pengine/common.h +@@ -1,5 +1,5 @@ + /* +- * Copyright 2004-2023 the Pacemaker project contributors ++ * Copyright 2004-2024 the Pacemaker project contributors + * + * The version control history for this file may have further details. + * +@@ -28,8 +28,6 @@ enum rsc_role_e text2role(const char *role); + const char *role2text(enum rsc_role_e role); + const char *fail2text(enum action_fail_response fail); + +-const char *pe_pref(GHashTable * options, const char *name); +- + /*! + * \brief Get readable description of a recovery type + * +diff --git a/include/crm/pengine/common_compat.h b/include/crm/pengine/common_compat.h +index 4330ccf..52e11f7 100644 +--- a/include/crm/pengine/common_compat.h ++++ b/include/crm/pengine/common_compat.h +@@ -1,5 +1,5 @@ + /* +- * Copyright 2004-2023 the Pacemaker project contributors ++ * Copyright 2004-2024 the Pacemaker project contributors + * + * The version control history for this file may have further details. + * +@@ -54,6 +54,9 @@ extern "C" { + + //! \deprecated Do not use + #define RSC_ROLE_MASTER_S RSC_ROLE_PROMOTED_LEGACY_S ++ ++//! \deprecated Do not use ++const char *pe_pref(GHashTable * options, const char *name); + + #ifdef __cplusplus + } +diff --git a/lib/pengine/common.c b/lib/pengine/common.c +index 402fae9..0a4dfe6 100644 +--- a/lib/pengine/common.c ++++ b/lib/pengine/common.c +@@ -35,12 +35,6 @@ pe_metadata(pcmk__output_t *out) + g_free(s); + } + +-const char * +-pe_pref(GHashTable * options, const char *name) +-{ +- return pcmk__cluster_option(options, name); +-} +- + const char * + fail2text(enum action_fail_response fail) + { +@@ -350,3 +344,17 @@ pe_node_attribute_raw(const pcmk_node_t *node, const char *name) + } + return g_hash_table_lookup(node->details->attrs, name); + } ++ ++// Deprecated functions kept only for backward API compatibility ++// LCOV_EXCL_START ++ ++#include ++ ++const char * ++pe_pref(GHashTable * options, const char *name) ++{ ++ return pcmk__cluster_option(options, name); ++} ++ ++// LCOV_EXCL_STOP ++// End deprecated API +diff --git a/lib/pengine/unpack.c b/lib/pengine/unpack.c +index 49443c6..d484e93 100644 +--- a/lib/pengine/unpack.c ++++ b/lib/pengine/unpack.c +@@ -238,7 +238,7 @@ unpack_config(xmlNode *config, pcmk_scheduler_t *scheduler) + crm_info("Startup probes: disabled (dangerous)"); + } + +- value = pe_pref(scheduler->config_hash, XML_ATTR_HAVE_WATCHDOG); ++ value = pcmk__cluster_option(config_hash, XML_ATTR_HAVE_WATCHDOG); + if (value && crm_is_true(value)) { + crm_info("Watchdog-based self-fencing will be performed via SBD if " + "fencing is required and stonith-watchdog-timeout is nonzero"); +@@ -251,7 +251,7 @@ unpack_config(xmlNode *config, pcmk_scheduler_t *scheduler) + set_if_xpath(pcmk_sched_enable_unfencing, XPATH_ENABLE_UNFENCING, + scheduler); + +- value = pe_pref(scheduler->config_hash, "stonith-timeout"); ++ value = pcmk__cluster_option(config_hash, "stonith-timeout"); + scheduler->stonith_timeout = (int) crm_parse_interval_spec(value); + crm_debug("STONITH timeout: %d", scheduler->stonith_timeout); + +@@ -262,8 +262,8 @@ unpack_config(xmlNode *config, pcmk_scheduler_t *scheduler) + crm_debug("STONITH of failed nodes is disabled"); + } + +- scheduler->stonith_action = pe_pref(scheduler->config_hash, +- "stonith-action"); ++ scheduler->stonith_action = pcmk__cluster_option(config_hash, ++ "stonith-action"); + if (!strcmp(scheduler->stonith_action, "poweroff")) { + pe_warn_once(pcmk__wo_poweroff, + "Support for stonith-action of 'poweroff' is deprecated " +@@ -280,8 +280,8 @@ unpack_config(xmlNode *config, pcmk_scheduler_t *scheduler) + crm_debug("Concurrent fencing is disabled"); + } + +- value = pe_pref(scheduler->config_hash, +- XML_CONFIG_ATTR_PRIORITY_FENCING_DELAY); ++ value = pcmk__cluster_option(config_hash, ++ XML_CONFIG_ATTR_PRIORITY_FENCING_DELAY); + if (value) { + scheduler->priority_fencing_delay = crm_parse_interval_spec(value) + / 1000; +@@ -299,7 +299,7 @@ unpack_config(xmlNode *config, pcmk_scheduler_t *scheduler) + crm_debug("Cluster is symmetric" " - resources can run anywhere by default"); + } + +- value = pe_pref(scheduler->config_hash, "no-quorum-policy"); ++ value = pcmk__cluster_option(config_hash, "no-quorum-policy"); + + if (pcmk__str_eq(value, "ignore", pcmk__str_casei)) { + scheduler->no_quorum_policy = pcmk_no_quorum_ignore; +@@ -367,7 +367,7 @@ unpack_config(xmlNode *config, pcmk_scheduler_t *scheduler) + crm_trace("Orphan resource actions are ignored"); + } + +- value = pe_pref(scheduler->config_hash, "remove-after-stop"); ++ value = pcmk__cluster_option(config_hash, "remove-after-stop"); + if (value != NULL) { + if (crm_is_true(value)) { + pe__set_working_set_flags(scheduler, pcmk_sched_remove_after_stop); +@@ -407,14 +407,14 @@ unpack_config(xmlNode *config, pcmk_scheduler_t *scheduler) + + pe__unpack_node_health_scores(scheduler); + +- scheduler->placement_strategy = pe_pref(scheduler->config_hash, +- "placement-strategy"); ++ scheduler->placement_strategy = ++ pcmk__cluster_option(config_hash, "placement-strategy"); + crm_trace("Placement strategy: %s", scheduler->placement_strategy); + + set_config_flag(scheduler, "shutdown-lock", pcmk_sched_shutdown_lock); + if (pcmk_is_set(scheduler->flags, pcmk_sched_shutdown_lock)) { +- value = pe_pref(scheduler->config_hash, +- XML_CONFIG_ATTR_SHUTDOWN_LOCK_LIMIT); ++ value = pcmk__cluster_option(config_hash, ++ XML_CONFIG_ATTR_SHUTDOWN_LOCK_LIMIT); + scheduler->shutdown_lock = crm_parse_interval_spec(value) / 1000; + crm_trace("Resources will be locked to nodes that were cleanly " + "shut down (locks expire after %s)", +@@ -424,8 +424,8 @@ unpack_config(xmlNode *config, pcmk_scheduler_t *scheduler) + "shut down"); + } + +- value = pe_pref(scheduler->config_hash, +- XML_CONFIG_ATTR_NODE_PENDING_TIMEOUT); ++ value = pcmk__cluster_option(config_hash, ++ XML_CONFIG_ATTR_NODE_PENDING_TIMEOUT); + scheduler->node_pending_timeout = crm_parse_interval_spec(value) / 1000; + if (scheduler->node_pending_timeout == 0) { + crm_trace("Do not fence pending nodes"); +-- +2.31.1 + +From 866877401075e7ea4c3bc278e69ed94ea3a7af99 Mon Sep 17 00:00:00 2001 +From: Reid Wahl +Date: Mon, 8 Jan 2024 20:32:36 -0800 +Subject: [PATCH 17/24] Refactor: libpe_status, scheduler: Drop pe_metadata() + +Replace with static scheduler_metadata() in the scheduler since we don't +rely on a static options array in lib/pengine/common.c anymore. + +Signed-off-by: Reid Wahl +--- + daemons/schedulerd/pacemaker-schedulerd.c | 18 ++++++++++++++++-- + include/crm/pengine/internal.h | 2 -- + lib/pengine/common.c | 14 -------------- + 3 files changed, 16 insertions(+), 18 deletions(-) + +diff --git a/daemons/schedulerd/pacemaker-schedulerd.c b/daemons/schedulerd/pacemaker-schedulerd.c +index 3f2a3e8..27c96da 100644 +--- a/daemons/schedulerd/pacemaker-schedulerd.c ++++ b/daemons/schedulerd/pacemaker-schedulerd.c +@@ -1,5 +1,5 @@ + /* +- * Copyright 2004-2023 the Pacemaker project contributors ++ * Copyright 2004-2024 the Pacemaker project contributors + * + * The version control history for this file may have further details. + * +@@ -46,6 +46,20 @@ pcmk__supported_format_t formats[] = { + + void pengine_shutdown(int nsig); + ++static void ++scheduler_metadata(pcmk__output_t *out) ++{ ++ const char *name = "pacemaker-schedulerd"; ++ const char *desc_short = "Pacemaker scheduler options"; ++ const char *desc_long = "Cluster options used by Pacemaker's scheduler"; ++ ++ gchar *s = pcmk__cluster_option_metadata(name, desc_short, desc_long, ++ pcmk__opt_context_schedulerd); ++ ++ out->output_xml(out, "metadata", s); ++ g_free(s); ++} ++ + static GOptionContext * + build_arg_context(pcmk__common_args_t *args, GOptionGroup **group) { + GOptionContext *context = NULL; +@@ -98,7 +112,7 @@ main(int argc, char **argv) + if (options.remainder) { + if (g_strv_length(options.remainder) == 1 && + pcmk__str_eq("metadata", options.remainder[0], pcmk__str_casei)) { +- pe_metadata(out); ++ scheduler_metadata(out); + goto done; + } else { + exit_code = CRM_EX_USAGE; +diff --git a/include/crm/pengine/internal.h b/include/crm/pengine/internal.h +index 2b7f2eb..5965c1a 100644 +--- a/include/crm/pengine/internal.h ++++ b/include/crm/pengine/internal.h +@@ -208,8 +208,6 @@ char *native_parameter(pcmk_resource_t *rsc, pcmk_node_t *node, gboolean create, + pcmk_node_t *native_location(const pcmk_resource_t *rsc, GList **list, + int current); + +-void pe_metadata(pcmk__output_t *out); +- + void native_add_running(pcmk_resource_t *rsc, pcmk_node_t *node, + pcmk_scheduler_t *scheduler, gboolean failed); + +diff --git a/lib/pengine/common.c b/lib/pengine/common.c +index 0a4dfe6..6551d10 100644 +--- a/lib/pengine/common.c ++++ b/lib/pengine/common.c +@@ -21,20 +21,6 @@ + gboolean was_processing_error = FALSE; + gboolean was_processing_warning = FALSE; + +-void +-pe_metadata(pcmk__output_t *out) +-{ +- const char *name = "pacemaker-schedulerd"; +- const char *desc_short = "Pacemaker scheduler options"; +- const char *desc_long = "Cluster options used by Pacemaker's scheduler"; +- +- gchar *s = pcmk__cluster_option_metadata(name, desc_short, desc_long, +- pcmk__opt_context_schedulerd); +- +- out->output_xml(out, "metadata", s); +- g_free(s); +-} +- + const char * + fail2text(enum action_fail_response fail) + { +-- +2.31.1 + +From 700c906d621887f257c73ddfd7c82c773cb32c8e Mon Sep 17 00:00:00 2001 +From: Reid Wahl +Date: Mon, 8 Jan 2024 20:40:18 -0800 +Subject: [PATCH 18/24] Refactor: libcib: Drop verify_cib_options() + +Signed-off-by: Reid Wahl +--- + lib/cib/cib_utils.c | 8 +------- + 1 file changed, 1 insertion(+), 7 deletions(-) + +diff --git a/lib/cib/cib_utils.c b/lib/cib/cib_utils.c +index b83158c..227a50f 100644 +--- a/lib/cib/cib_utils.c ++++ b/lib/cib/cib_utils.c +@@ -881,12 +881,6 @@ cib_metadata(void) + g_free(s); + } + +-static void +-verify_cib_options(GHashTable *options) +-{ +- pcmk__validate_cluster_options(options); +-} +- + const char * + cib_pref(GHashTable * options, const char *name) + { +@@ -913,7 +907,7 @@ cib_read_config(GHashTable * options, xmlNode * current_cib) + options, CIB_OPTIONS_FIRST, TRUE, now, NULL); + } + +- verify_cib_options(options); ++ pcmk__validate_cluster_options(options); + + crm_time_free(now); + +-- +2.31.1 + +From 6284a3a79b88fd20630bfbfe866a4c2c3686a246 Mon Sep 17 00:00:00 2001 +From: Reid Wahl +Date: Mon, 8 Jan 2024 20:24:19 -0800 +Subject: [PATCH 19/24] Refactor: libcib: Drop cib_pref() internally + +Signed-off-by: Reid Wahl +--- + lib/cib/cib_utils.c | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/lib/cib/cib_utils.c b/lib/cib/cib_utils.c +index 227a50f..5b241ae 100644 +--- a/lib/cib/cib_utils.c ++++ b/lib/cib/cib_utils.c +@@ -281,7 +281,7 @@ cib_acl_enabled(xmlNode *xml, const char *user) + GHashTable *options = pcmk__strkey_table(free, free); + + cib_read_config(options, xml); +- value = cib_pref(options, "enable-acl"); ++ value = pcmk__cluster_option(options, "enable-acl"); + rc = crm_is_true(value); + g_hash_table_destroy(options); + } +-- +2.31.1 + +From 1806822590b0060079b94b7d2867722ef2430bf9 Mon Sep 17 00:00:00 2001 +From: Reid Wahl +Date: Mon, 8 Jan 2024 20:26:29 -0800 +Subject: [PATCH 20/24] API: libcib: Deprecate cib_pref() + +Signed-off-by: Reid Wahl +--- + include/crm/cib/util.h | 1 - + include/crm/cib/util_compat.h | 3 +++ + lib/cib/cib_utils.c | 12 ++++++------ + 3 files changed, 9 insertions(+), 7 deletions(-) + +diff --git a/include/crm/cib/util.h b/include/crm/cib/util.h +index 18726bb..47894cb 100644 +--- a/include/crm/cib/util.h ++++ b/include/crm/cib/util.h +@@ -57,7 +57,6 @@ int set_standby(cib_t * the_cib, const char *uuid, const char *scope, const char + xmlNode *cib_get_generation(cib_t * cib); + + void cib_metadata(void); +-const char *cib_pref(GHashTable * options, const char *name); + + int cib_apply_patch_event(xmlNode *event, xmlNode *input, xmlNode **output, + int level); +diff --git a/include/crm/cib/util_compat.h b/include/crm/cib/util_compat.h +index 20f1e2d..d6ccd4d 100644 +--- a/include/crm/cib/util_compat.h ++++ b/include/crm/cib/util_compat.h +@@ -33,6 +33,9 @@ const char *get_object_parent(const char *object_type); + //! \deprecated Use pcmk_cib_xpath_for() instead + xmlNode *get_object_root(const char *object_type, xmlNode *the_root); + ++//! \deprecated Do not use ++const char *cib_pref(GHashTable * options, const char *name); ++ + #ifdef __cplusplus + } + #endif +diff --git a/lib/cib/cib_utils.c b/lib/cib/cib_utils.c +index 5b241ae..f9c463e 100644 +--- a/lib/cib/cib_utils.c ++++ b/lib/cib/cib_utils.c +@@ -881,12 +881,6 @@ cib_metadata(void) + g_free(s); + } + +-const char * +-cib_pref(GHashTable * options, const char *name) +-{ +- return pcmk__cluster_option(options, name); +-} +- + gboolean + cib_read_config(GHashTable * options, xmlNode * current_cib) + { +@@ -1085,5 +1079,11 @@ get_object_root(const char *object_type, xmlNode *the_root) + return pcmk_find_cib_element(the_root, object_type); + } + ++const char * ++cib_pref(GHashTable * options, const char *name) ++{ ++ return pcmk__cluster_option(options, name); ++} ++ + // LCOV_EXCL_STOP + // End deprecated API +-- +2.31.1 + +From 422fb81250aa733d2601b4d412c3fbcbf5b74420 Mon Sep 17 00:00:00 2001 +From: Reid Wahl +Date: Mon, 8 Jan 2024 20:29:29 -0800 +Subject: [PATCH 21/24] Refactor: based, libcib: Drop cib_metadata() internally + +Replace with a static based_metadata() in pacemaker-based since we don't +depend on a static options array in lib/cib/cib_utils.c anymore. + +Signed-off-by: Reid Wahl +--- + daemons/based/pacemaker-based.c | 17 ++++++++++++++++- + 1 file changed, 16 insertions(+), 1 deletion(-) + +diff --git a/daemons/based/pacemaker-based.c b/daemons/based/pacemaker-based.c +index 5dd7938..78bcd51 100644 +--- a/daemons/based/pacemaker-based.c ++++ b/daemons/based/pacemaker-based.c +@@ -126,6 +126,21 @@ setup_stand_alone(GError **error) + return pcmk_rc_ok; + } + ++static void ++based_metadata(void) ++{ ++ const char *name = "pacemaker-based"; ++ const char *desc_short = "Cluster Information Base manager options"; ++ const char *desc_long = "Cluster options used by Pacemaker's Cluster " ++ "Information Base manager"; ++ ++ gchar *s = pcmk__cluster_option_metadata(name, desc_short, desc_long, ++ pcmk__opt_context_based); ++ ++ printf("%s", s); ++ g_free(s); ++} ++ + static GOptionEntry entries[] = { + { "stand-alone", 's', G_OPTION_FLAG_NONE, G_OPTION_ARG_NONE, &stand_alone, + "(Advanced use only) Run in stand-alone mode", NULL }, +@@ -204,7 +219,7 @@ main(int argc, char **argv) + + if ((g_strv_length(processed_args) >= 2) + && pcmk__str_eq(processed_args[1], "metadata", pcmk__str_none)) { +- cib_metadata(); ++ based_metadata(); + goto done; + } + +-- +2.31.1 + +From 05b3e08de7c515c38cf42bbbeaf18e3346eb360d Mon Sep 17 00:00:00 2001 +From: Reid Wahl +Date: Mon, 8 Jan 2024 20:30:16 -0800 +Subject: [PATCH 22/24] API: libcib: Deprecate cib_metadata() + +Signed-off-by: Reid Wahl +--- + include/crm/cib/util.h | 2 -- + include/crm/cib/util_compat.h | 3 +++ + lib/cib/cib_utils.c | 30 +++++++++++++++--------------- + 3 files changed, 18 insertions(+), 17 deletions(-) + +diff --git a/include/crm/cib/util.h b/include/crm/cib/util.h +index 47894cb..ce6cbeb 100644 +--- a/include/crm/cib/util.h ++++ b/include/crm/cib/util.h +@@ -56,8 +56,6 @@ int set_standby(cib_t * the_cib, const char *uuid, const char *scope, const char + + xmlNode *cib_get_generation(cib_t * cib); + +-void cib_metadata(void); +- + int cib_apply_patch_event(xmlNode *event, xmlNode *input, xmlNode **output, + int level); + +diff --git a/include/crm/cib/util_compat.h b/include/crm/cib/util_compat.h +index d6ccd4d..95e0766 100644 +--- a/include/crm/cib/util_compat.h ++++ b/include/crm/cib/util_compat.h +@@ -36,6 +36,9 @@ xmlNode *get_object_root(const char *object_type, xmlNode *the_root); + //! \deprecated Do not use + const char *cib_pref(GHashTable * options, const char *name); + ++//! \deprecated Do not use ++void cib_metadata(void); ++ + #ifdef __cplusplus + } + #endif +diff --git a/lib/cib/cib_utils.c b/lib/cib/cib_utils.c +index f9c463e..75dda16 100644 +--- a/lib/cib/cib_utils.c ++++ b/lib/cib/cib_utils.c +@@ -866,21 +866,6 @@ cib_native_notify(gpointer data, gpointer user_data) + crm_trace("Callback invoked..."); + } + +-void +-cib_metadata(void) +-{ +- const char *name = "pacemaker-based"; +- const char *desc_short = "Cluster Information Base manager options"; +- const char *desc_long = "Cluster options used by Pacemaker's Cluster " +- "Information Base manager"; +- +- gchar *s = pcmk__cluster_option_metadata(name, desc_short, desc_long, +- pcmk__opt_context_based); +- +- printf("%s", s); +- g_free(s); +-} +- + gboolean + cib_read_config(GHashTable * options, xmlNode * current_cib) + { +@@ -1085,5 +1070,20 @@ cib_pref(GHashTable * options, const char *name) + return pcmk__cluster_option(options, name); + } + ++void ++cib_metadata(void) ++{ ++ const char *name = "pacemaker-based"; ++ const char *desc_short = "Cluster Information Base manager options"; ++ const char *desc_long = "Cluster options used by Pacemaker's Cluster " ++ "Information Base manager"; ++ ++ gchar *s = pcmk__cluster_option_metadata(name, desc_short, desc_long, ++ pcmk__opt_context_based); ++ ++ printf("%s", s); ++ g_free(s); ++} ++ + // LCOV_EXCL_STOP + // End deprecated API +-- +2.31.1 + +From f8ee575a51f6bacf82abb1d1f41eba1092776682 Mon Sep 17 00:00:00 2001 +From: Reid Wahl +Date: Mon, 8 Jan 2024 20:42:37 -0800 +Subject: [PATCH 23/24] Refactor: controller: Replace crmd_metadata() with + controld_metadata() + +Can be static since we don't rely on an options array that lives in +controld_control.c anymore. + +Signed-off-by: Reid Wahl +--- + daemons/controld/controld_control.c | 14 -------------- + daemons/controld/pacemaker-controld.c | 18 ++++++++++++++++-- + daemons/controld/pacemaker-controld.h | 3 +-- + 3 files changed, 17 insertions(+), 18 deletions(-) + +diff --git a/daemons/controld/controld_control.c b/daemons/controld/controld_control.c +index 40b90f8..9b54900 100644 +--- a/daemons/controld/controld_control.c ++++ b/daemons/controld/controld_control.c +@@ -515,20 +515,6 @@ do_recover(long long action, + register_fsa_input(C_FSA_INTERNAL, I_TERMINATE, NULL); + } + +-void +-crmd_metadata(void) +-{ +- const char *name = "pacemaker-controld"; +- const char *desc_short = "Pacemaker controller options"; +- const char *desc_long = "Cluster options used by Pacemaker's controller"; +- +- gchar *s = pcmk__cluster_option_metadata(name, desc_short, desc_long, +- pcmk__opt_context_controld); +- +- printf("%s", s); +- g_free(s); +-} +- + static void + config_query_callback(xmlNode * msg, int call_id, int rc, xmlNode * output, void *user_data) + { +diff --git a/daemons/controld/pacemaker-controld.c b/daemons/controld/pacemaker-controld.c +index e4a72c2..d80644d 100644 +--- a/daemons/controld/pacemaker-controld.c ++++ b/daemons/controld/pacemaker-controld.c +@@ -1,5 +1,5 @@ + /* +- * Copyright 2004-2023 the Pacemaker project contributors ++ * Copyright 2004-2024 the Pacemaker project contributors + * + * The version control history for this file may have further details. + * +@@ -46,6 +46,20 @@ static pcmk__supported_format_t formats[] = { + { NULL, NULL, NULL } + }; + ++static void ++controld_metadata(void) ++{ ++ const char *name = "pacemaker-controld"; ++ const char *desc_short = "Pacemaker controller options"; ++ const char *desc_long = "Cluster options used by Pacemaker's controller"; ++ ++ gchar *s = pcmk__cluster_option_metadata(name, desc_short, desc_long, ++ pcmk__opt_context_controld); ++ ++ printf("%s", s); ++ g_free(s); ++} ++ + static GOptionContext * + build_arg_context(pcmk__common_args_t *args, GOptionGroup **group) + { +@@ -96,7 +110,7 @@ main(int argc, char **argv) + + if ((g_strv_length(processed_args) >= 2) + && pcmk__str_eq(processed_args[1], "metadata", pcmk__str_none)) { +- crmd_metadata(); ++ controld_metadata(); + initialize = false; + goto done; + } +diff --git a/daemons/controld/pacemaker-controld.h b/daemons/controld/pacemaker-controld.h +index 2334cce..ba8dc8f 100644 +--- a/daemons/controld/pacemaker-controld.h ++++ b/daemons/controld/pacemaker-controld.h +@@ -1,5 +1,5 @@ + /* +- * Copyright 2004-2023 the Pacemaker project contributors ++ * Copyright 2004-2024 the Pacemaker project contributors + * + * The version control history for this file may have further details. + * +@@ -28,7 +28,6 @@ + # define controld_trigger_config() \ + controld_trigger_config_as(__func__, __LINE__) + +-void crmd_metadata(void); + void controld_trigger_config_as(const char *fn, int line); + void controld_election_init(const char *uname); + void controld_configure_election(GHashTable *options); +-- +2.31.1 + +From 282e9eb026699abef5a28fc37f54b9330029da1c Mon Sep 17 00:00:00 2001 +From: Reid Wahl +Date: Tue, 2 Jan 2024 19:56:11 -0800 +Subject: [PATCH 24/24] Test: cts-cli: Update daemon tests after fence-reaction + select + +Signed-off-by: Reid Wahl +--- + cts/cli/regression.daemons.exp | 7 +++++-- + 1 file changed, 5 insertions(+), 2 deletions(-) + +diff --git a/cts/cli/regression.daemons.exp b/cts/cli/regression.daemons.exp +index 43393df..543d62f 100644 +--- a/cts/cli/regression.daemons.exp ++++ b/cts/cli/regression.daemons.exp +@@ -52,9 +52,12 @@ + + + +- A cluster node may receive notification of its own fencing if fencing is misconfigured, or if fabric fencing is in use that doesn't cut cluster communication. Allowed values are "stop" to attempt to immediately stop Pacemaker and stay stopped, or "panic" to attempt to immediately reboot the local node, falling back to stop on failure. ++ A cluster node may receive notification of its own fencing if fencing is misconfigured, or if fabric fencing is in use that doesn't cut cluster communication. Use "stop" to attempt to immediately stop Pacemaker and stay stopped, or "panic" to attempt to immediately reboot the local node, falling back to stop on failure. Allowed values: stop, panic + How a cluster node should react if notified of its own fencing +- ++ ++ + + + Declare an election failed if it is not decided within this much time. If you need to adjust this value, it probably indicates the presence of a bug. +-- +2.31.1 + diff --git a/SOURCES/008-attrd-prep.patch b/SOURCES/008-attrd-prep.patch new file mode 100644 index 0000000..acc22d3 --- /dev/null +++ b/SOURCES/008-attrd-prep.patch @@ -0,0 +1,373 @@ +From 4823643bef8801b33688167b159bb531bcdf8911 Mon Sep 17 00:00:00 2001 +From: Ken Gaillot +Date: Thu, 4 Jan 2024 17:10:08 -0600 +Subject: [PATCH 1/5] Refactor: pacemaker-attrd: drop redundant argument from + update_attr_on_host() + +It can check for a force-write via its xml argument, to simplify the caller +--- + daemons/attrd/attrd_corosync.c | 13 +++++++------ + 1 file changed, 7 insertions(+), 6 deletions(-) + +diff --git a/daemons/attrd/attrd_corosync.c b/daemons/attrd/attrd_corosync.c +index 158d82f..1b56923 100644 +--- a/daemons/attrd/attrd_corosync.c ++++ b/daemons/attrd/attrd_corosync.c +@@ -266,7 +266,7 @@ record_peer_nodeid(attribute_value_t *v, const char *host) + static void + update_attr_on_host(attribute_t *a, const crm_node_t *peer, const xmlNode *xml, + const char *attr, const char *value, const char *host, +- bool filter, int is_force_write) ++ bool filter) + { + attribute_value_t *v = NULL; + +@@ -309,6 +309,10 @@ update_attr_on_host(attribute_t *a, const crm_node_t *peer, const xmlNode *xml, + } + + } else { ++ int is_force_write = 0; ++ ++ crm_element_value_int(xml, PCMK__XA_ATTR_FORCE, &is_force_write); ++ + if (is_force_write == 1 && a->timeout_ms && a->timer) { + /* Save forced writing and set change flag. */ + /* The actual attribute is written by Writer after election. */ +@@ -338,15 +342,12 @@ attrd_peer_update_one(const crm_node_t *peer, xmlNode *xml, bool filter) + const char *attr = crm_element_value(xml, PCMK__XA_ATTR_NAME); + const char *value = crm_element_value(xml, PCMK__XA_ATTR_VALUE); + const char *host = crm_element_value(xml, PCMK__XA_ATTR_NODE_NAME); +- int is_force_write = 0; + + if (attr == NULL) { + crm_warn("Could not update attribute: peer did not specify name"); + return; + } + +- crm_element_value_int(xml, PCMK__XA_ATTR_FORCE, &is_force_write); +- + a = attrd_populate_attribute(xml, attr); + if (a == NULL) { + return; +@@ -361,12 +362,12 @@ attrd_peer_update_one(const crm_node_t *peer, xmlNode *xml, bool filter) + g_hash_table_iter_init(&vIter, a->values); + + while (g_hash_table_iter_next(&vIter, (gpointer *) & host, NULL)) { +- update_attr_on_host(a, peer, xml, attr, value, host, filter, is_force_write); ++ update_attr_on_host(a, peer, xml, attr, value, host, filter); + } + + } else { + // Update attribute value for the given host +- update_attr_on_host(a, peer, xml, attr, value, host, filter, is_force_write); ++ update_attr_on_host(a, peer, xml, attr, value, host, filter); + } + + /* If this is a message from some attrd instance broadcasting its protocol +-- +2.31.1 + +From c7a1ab819b25e3225c185c1630a7139a96fb5c71 Mon Sep 17 00:00:00 2001 +From: Ken Gaillot +Date: Tue, 9 Jan 2024 16:48:37 -0600 +Subject: [PATCH 2/5] Refactor: pacemaker-attrd: drop unused argument from + attrd_peer_sync() + +--- + daemons/attrd/attrd_corosync.c | 10 ++++++++-- + daemons/attrd/attrd_elections.c | 2 +- + daemons/attrd/attrd_messages.c | 2 +- + daemons/attrd/pacemaker-attrd.h | 2 +- + 4 files changed, 11 insertions(+), 5 deletions(-) + +diff --git a/daemons/attrd/attrd_corosync.c b/daemons/attrd/attrd_corosync.c +index 1b56923..088f00c 100644 +--- a/daemons/attrd/attrd_corosync.c ++++ b/daemons/attrd/attrd_corosync.c +@@ -233,7 +233,7 @@ attrd_peer_change_cb(enum crm_status_type kind, crm_node_t *peer, const void *da + */ + if (attrd_election_won() + && !pcmk_is_set(peer->flags, crm_remote_node)) { +- attrd_peer_sync(peer, NULL); ++ attrd_peer_sync(peer); + } + } else { + // Remove all attribute values associated with lost nodes +@@ -535,8 +535,14 @@ attrd_peer_remove(const char *host, bool uncache, const char *source) + } + } + ++/*! ++ * \internal ++ * \brief Send all known attributes and values to a peer ++ * ++ * \param[in] peer Peer to send sync to (if NULL, broadcast to all peers) ++ */ + void +-attrd_peer_sync(crm_node_t *peer, xmlNode *xml) ++attrd_peer_sync(crm_node_t *peer) + { + GHashTableIter aIter; + GHashTableIter vIter; +diff --git a/daemons/attrd/attrd_elections.c b/daemons/attrd/attrd_elections.c +index 82fbe8a..9dbf133 100644 +--- a/daemons/attrd/attrd_elections.c ++++ b/daemons/attrd/attrd_elections.c +@@ -23,7 +23,7 @@ attrd_election_cb(gpointer user_data) + attrd_declare_winner(); + + /* Update the peers after an election */ +- attrd_peer_sync(NULL, NULL); ++ attrd_peer_sync(NULL); + + /* After winning an election, update the CIB with the values of all + * attributes as the winner knows them. +diff --git a/daemons/attrd/attrd_messages.c b/daemons/attrd/attrd_messages.c +index 5525d4b..13ac01f 100644 +--- a/daemons/attrd/attrd_messages.c ++++ b/daemons/attrd/attrd_messages.c +@@ -180,7 +180,7 @@ handle_sync_request(pcmk__request_t *request) + crm_node_t *peer = pcmk__get_node(0, request->peer, NULL, + pcmk__node_search_cluster); + +- attrd_peer_sync(peer, request->xml); ++ attrd_peer_sync(peer); + pcmk__set_result(&request->result, CRM_EX_OK, PCMK_EXEC_DONE, NULL); + return NULL; + } else { +diff --git a/daemons/attrd/pacemaker-attrd.h b/daemons/attrd/pacemaker-attrd.h +index 7384188..bacaad6 100644 +--- a/daemons/attrd/pacemaker-attrd.h ++++ b/daemons/attrd/pacemaker-attrd.h +@@ -175,7 +175,7 @@ extern GHashTable *peer_protocol_vers; + int attrd_cluster_connect(void); + void attrd_peer_update(const crm_node_t *peer, xmlNode *xml, const char *host, + bool filter); +-void attrd_peer_sync(crm_node_t *peer, xmlNode *xml); ++void attrd_peer_sync(crm_node_t *peer); + void attrd_peer_remove(const char *host, bool uncache, const char *source); + void attrd_peer_clear_failure(pcmk__request_t *request); + void attrd_peer_sync_response(const crm_node_t *peer, bool peer_won, +-- +2.31.1 + +From abafae0068e10abb135b0496086947728365299a Mon Sep 17 00:00:00 2001 +From: Ken Gaillot +Date: Thu, 11 Jan 2024 17:31:17 -0600 +Subject: [PATCH 3/5] Refactor: pacemaker-attrd: de-functionize + attrd_lookup_or_create_value() + +... to make planned changes easier +--- + daemons/attrd/attrd_corosync.c | 62 +++++++++++++--------------------- + 1 file changed, 24 insertions(+), 38 deletions(-) + +diff --git a/daemons/attrd/attrd_corosync.c b/daemons/attrd/attrd_corosync.c +index 088f00c..59e6a26 100644 +--- a/daemons/attrd/attrd_corosync.c ++++ b/daemons/attrd/attrd_corosync.c +@@ -168,40 +168,6 @@ broadcast_local_value(const attribute_t *a) + + #define state_text(state) pcmk__s((state), "in unknown state") + +-/*! +- * \internal +- * \brief Return a node's value from hash table (creating one if needed) +- * +- * \param[in,out] values Hash table of values +- * \param[in] node_name Name of node to look up +- * \param[in] xml XML describing the attribute +- * +- * \return Pointer to new or existing hash table entry +- */ +-static attribute_value_t * +-attrd_lookup_or_create_value(GHashTable *values, const char *node_name, +- const xmlNode *xml) +-{ +- attribute_value_t *v = g_hash_table_lookup(values, node_name); +- int is_remote = 0; +- +- if (v == NULL) { +- v = calloc(1, sizeof(attribute_value_t)); +- CRM_ASSERT(v != NULL); +- +- pcmk__str_update(&v->nodename, node_name); +- g_hash_table_replace(values, v->nodename, v); +- } +- +- crm_element_value_int(xml, PCMK__XA_ATTR_IS_REMOTE, &is_remote); +- if (is_remote) { +- attrd_set_value_flags(v, attrd_value_remote); +- CRM_ASSERT(crm_remote_peer_get(node_name) != NULL); +- } +- +- return(v); +-} +- + static void + attrd_peer_change_cb(enum crm_status_type kind, crm_node_t *peer, const void *data) + { +@@ -268,18 +234,38 @@ update_attr_on_host(attribute_t *a, const crm_node_t *peer, const xmlNode *xml, + const char *attr, const char *value, const char *host, + bool filter) + { ++ int is_remote = 0; ++ bool changed = false; + attribute_value_t *v = NULL; + +- v = attrd_lookup_or_create_value(a->values, host, xml); ++ // Create entry for value if not already existing ++ v = g_hash_table_lookup(a->values, host); ++ if (v == NULL) { ++ v = calloc(1, sizeof(attribute_value_t)); ++ CRM_ASSERT(v != NULL); ++ ++ pcmk__str_update(&v->nodename, host); ++ g_hash_table_replace(a->values, v->nodename, v); ++ } ++ ++ // If value is for a Pacemaker Remote node, remember that ++ crm_element_value_int(xml, PCMK__XA_ATTR_IS_REMOTE, &is_remote); ++ if (is_remote) { ++ attrd_set_value_flags(v, attrd_value_remote); ++ CRM_ASSERT(crm_remote_peer_get(host) != NULL); ++ } ++ ++ // Check whether the value changed ++ changed = !pcmk__str_eq(v->current, value, pcmk__str_casei); + +- if (filter && !pcmk__str_eq(v->current, value, pcmk__str_casei) +- && pcmk__str_eq(host, attrd_cluster->uname, pcmk__str_casei)) { ++ if (changed && filter && pcmk__str_eq(host, attrd_cluster->uname, ++ pcmk__str_casei)) { + + crm_notice("%s[%s]: local value '%s' takes priority over '%s' from %s", + attr, host, v->current, value, peer->uname); + v = broadcast_local_value(a); + +- } else if (!pcmk__str_eq(v->current, value, pcmk__str_casei)) { ++ } else if (changed) { + crm_notice("Setting %s[%s]%s%s: %s -> %s " + CRM_XS " from %s with %s write delay", + attr, host, a->set_type ? " in " : "", +-- +2.31.1 + +From 72529ec512fb4938bd8dbbd2caf44bbb1a616826 Mon Sep 17 00:00:00 2001 +From: Ken Gaillot +Date: Thu, 11 Jan 2024 18:04:33 -0600 +Subject: [PATCH 4/5] Refactor: pacemaker-attrd: minor shuffling to make + planned changes easier + +--- + daemons/attrd/attrd_cib.c | 19 +++++++++++-------- + 1 file changed, 11 insertions(+), 8 deletions(-) + +diff --git a/daemons/attrd/attrd_cib.c b/daemons/attrd/attrd_cib.c +index bdc0a10..481fea7 100644 +--- a/daemons/attrd/attrd_cib.c ++++ b/daemons/attrd/attrd_cib.c +@@ -51,6 +51,7 @@ attrd_cib_updated_cb(const char *event, xmlNode *msg) + { + const xmlNode *patchset = NULL; + const char *client_name = NULL; ++ bool status_changed = false; + + if (attrd_shutting_down(true)) { + return; +@@ -64,20 +65,22 @@ attrd_cib_updated_cb(const char *event, xmlNode *msg) + mainloop_set_trigger(attrd_config_read); + } + +- if (!attrd_election_won()) { +- // Don't write attributes if we're not the writer +- return; +- } ++ status_changed = cib__element_in_patchset(patchset, XML_CIB_TAG_STATUS); + + client_name = crm_element_value(msg, F_CIB_CLIENTNAME); + if (!cib__client_triggers_refresh(client_name)) { +- // The CIB is still accurate ++ /* This change came from a source that ensured the CIB is consistent ++ * with our attributes table, so we don't need to write anything out. ++ */ + return; + } + +- if (cib__element_in_patchset(patchset, XML_CIB_TAG_NODES) +- || cib__element_in_patchset(patchset, XML_CIB_TAG_STATUS)) { +- ++ if (!attrd_election_won()) { ++ // Don't write attributes if we're not the writer ++ return; ++ } ++ ++ if (status_changed || cib__element_in_patchset(patchset, XML_CIB_TAG_NODES)) { + /* An unsafe client modified the nodes or status section. Write + * transient attributes to ensure they're up-to-date in the CIB. + */ +-- +2.31.1 + +From b83c2567fb450eec5b18882ded16403831d2c3c0 Mon Sep 17 00:00:00 2001 +From: Ken Gaillot +Date: Thu, 11 Jan 2024 17:53:55 -0600 +Subject: [PATCH 5/5] Log: pacemaker-attrd: make sure we don't try to log NULL + +--- + daemons/attrd/attrd_corosync.c | 15 +++++++++++---- + 1 file changed, 11 insertions(+), 4 deletions(-) + +diff --git a/daemons/attrd/attrd_corosync.c b/daemons/attrd/attrd_corosync.c +index 59e6a26..b348d52 100644 +--- a/daemons/attrd/attrd_corosync.c ++++ b/daemons/attrd/attrd_corosync.c +@@ -229,6 +229,11 @@ record_peer_nodeid(attribute_value_t *v, const char *host) + } + } + ++#define readable_value(rv_v) pcmk__s((rv_v)->current, "(unset)") ++ ++#define readable_peer(p) \ ++ (((p) == NULL)? "all peers" : pcmk__s((p)->uname, "unknown peer")) ++ + static void + update_attr_on_host(attribute_t *a, const crm_node_t *peer, const xmlNode *xml, + const char *attr, const char *value, const char *host, +@@ -262,14 +267,14 @@ update_attr_on_host(attribute_t *a, const crm_node_t *peer, const xmlNode *xml, + pcmk__str_casei)) { + + crm_notice("%s[%s]: local value '%s' takes priority over '%s' from %s", +- attr, host, v->current, value, peer->uname); ++ attr, host, readable_value(v), value, peer->uname); + v = broadcast_local_value(a); + + } else if (changed) { + crm_notice("Setting %s[%s]%s%s: %s -> %s " + CRM_XS " from %s with %s write delay", + attr, host, a->set_type ? " in " : "", +- pcmk__s(a->set_type, ""), pcmk__s(v->current, "(unset)"), ++ pcmk__s(a->set_type, ""), readable_value(v), + pcmk__s(value, "(unset)"), peer->uname, + (a->timeout_ms == 0)? "no" : pcmk__readable_interval(a->timeout_ms)); + pcmk__str_update(&v->current, value); +@@ -543,12 +548,14 @@ attrd_peer_sync(crm_node_t *peer) + while (g_hash_table_iter_next(&aIter, NULL, (gpointer *) & a)) { + g_hash_table_iter_init(&vIter, a->values); + while (g_hash_table_iter_next(&vIter, NULL, (gpointer *) & v)) { +- crm_debug("Syncing %s[%s] = %s to %s", a->id, v->nodename, v->current, peer?peer->uname:"everyone"); ++ crm_debug("Syncing %s[%s]='%s' to %s", ++ a->id, v->nodename, readable_value(v), ++ readable_peer(peer)); + attrd_add_value_xml(sync, a, v, false); + } + } + +- crm_debug("Syncing values to %s", peer?peer->uname:"everyone"); ++ crm_debug("Syncing values to %s", readable_peer(peer)); + attrd_send_message(peer, sync, false); + free_xml(sync); + } +-- +2.31.1 + diff --git a/SOURCES/008-attrd-shutdown.patch b/SOURCES/008-attrd-shutdown.patch deleted file mode 100644 index 1d02526..0000000 --- a/SOURCES/008-attrd-shutdown.patch +++ /dev/null @@ -1,45 +0,0 @@ -From f5263c9401c9c38d4e039149deddcc0da0c184ba Mon Sep 17 00:00:00 2001 -From: Ken Gaillot -Date: Thu, 3 Aug 2023 12:17:08 -0500 -Subject: [PATCH] Fix: attrd: avoid race condition when shutting down - -This addresses a race condition that can occur when the DC and the attribute -writer are different nodes, and shutting down at the same time. When the DC -controller leaves its Corosync process group, the remaining nodes erase its -transient node attributes (including "shutdown") from the CIB. However if the -(former) DC's attrd is still up, it can win the attribute writer election -called after the original writer leaves. As the election winner, it writes out -all its attributes to the CIB, including "shutdown". The next time it rejoins -the cluster, it will be immediately shut down. - -Fixes T138 ---- - daemons/attrd/attrd_elections.c | 10 +++++++++- - 1 file changed, 9 insertions(+), 1 deletion(-) - -diff --git a/daemons/attrd/attrd_elections.c b/daemons/attrd/attrd_elections.c -index 3b6b55a0f59..6f4916888a9 100644 ---- a/daemons/attrd/attrd_elections.c -+++ b/daemons/attrd/attrd_elections.c -@@ -22,12 +22,20 @@ attrd_election_cb(gpointer user_data) - { - attrd_declare_winner(); - -+ if (attrd_requesting_shutdown() || attrd_shutting_down()) { -+ /* This node is shutting down or about to, meaning its attributes will -+ * be removed (and may have already been removed from the CIB by a -+ * controller). Don't sync or write its attributes in this case. -+ */ -+ return G_SOURCE_REMOVE; -+ } -+ - /* Update the peers after an election */ - attrd_peer_sync(NULL, NULL); - - /* Update the CIB after an election */ - attrd_write_attributes(true, false); -- return FALSE; -+ return G_SOURCE_REMOVE; - } - - void diff --git a/SOURCES/009-attrd-cache-3.patch b/SOURCES/009-attrd-cache-3.patch new file mode 100644 index 0000000..6150218 --- /dev/null +++ b/SOURCES/009-attrd-cache-3.patch @@ -0,0 +1,385 @@ +From 84d4a0d5f562df91baa0fece45d06ad3732f941c Mon Sep 17 00:00:00 2001 +From: Ken Gaillot +Date: Tue, 16 Jan 2024 11:20:53 -0600 +Subject: [PATCH 1/5] Low: pacemaker-attrd: properly validate attribute set + type + +The sense of the test was accidentally reversed in 26471a52689 +--- + daemons/attrd/attrd_attributes.c | 6 +++--- + 1 file changed, 3 insertions(+), 3 deletions(-) + +diff --git a/daemons/attrd/attrd_attributes.c b/daemons/attrd/attrd_attributes.c +index 8f32988..f059059 100644 +--- a/daemons/attrd/attrd_attributes.c ++++ b/daemons/attrd/attrd_attributes.c +@@ -40,9 +40,9 @@ attrd_create_attribute(xmlNode *xml) + * attributes are not written. + */ + crm_element_value_int(xml, PCMK__XA_ATTR_IS_PRIVATE, &is_private); +- if ((is_private != 0) +- && !pcmk__str_any_of(set_type, XML_TAG_ATTR_SETS, XML_TAG_UTILIZATION, +- NULL)) { ++ if (!is_private && !pcmk__str_any_of(set_type, ++ XML_TAG_ATTR_SETS, ++ XML_TAG_UTILIZATION, NULL)) { + crm_warn("Ignoring attribute %s with invalid set type %s", + pcmk__s(name, "(unidentified)"), set_type); + return NULL; +-- +2.31.1 + +From d0d0511e71fe983a2d89589c39810b79fb48a8ca Mon Sep 17 00:00:00 2001 +From: Ken Gaillot +Date: Tue, 16 Jan 2024 12:13:42 -0600 +Subject: [PATCH 2/5] Fix: pacemaker-attrd: sync utilization attributes to + peers correctly + +Include the set type with attribute syncs. + +Previously, utilization attributes would have the correct set_type on the node +where they were set, but peers would store it as a regular node attribute. If +one of those peers became writer, the attribute would get written to the wrong +set. +--- + daemons/attrd/attrd_attributes.c | 1 + + 1 file changed, 1 insertion(+) + +diff --git a/daemons/attrd/attrd_attributes.c b/daemons/attrd/attrd_attributes.c +index f059059..0ad9630 100644 +--- a/daemons/attrd/attrd_attributes.c ++++ b/daemons/attrd/attrd_attributes.c +@@ -139,6 +139,7 @@ attrd_add_value_xml(xmlNode *parent, const attribute_t *a, + xmlNode *xml = create_xml_node(parent, __func__); + + crm_xml_add(xml, PCMK__XA_ATTR_NAME, a->id); ++ crm_xml_add(xml, PCMK__XA_ATTR_SET_TYPE, a->set_type); + crm_xml_add(xml, PCMK__XA_ATTR_SET, a->set_id); + crm_xml_add(xml, PCMK__XA_ATTR_UUID, a->uuid); + crm_xml_add(xml, PCMK__XA_ATTR_USER, a->user); +-- +2.31.1 + +From 4479ff8507dd69f5946d31cf83c7e47fe15d3bdb Mon Sep 17 00:00:00 2001 +From: Ken Gaillot +Date: Tue, 16 Jan 2024 12:18:40 -0600 +Subject: [PATCH 3/5] Refactor: pacemaker-attrd: functionize getting attribute + set ID + +... for future reuse +--- + daemons/attrd/attrd_attributes.c | 38 ++++++++++++++++++++++++++++++++ + daemons/attrd/attrd_cib.c | 9 +------- + daemons/attrd/pacemaker-attrd.h | 3 ++- + 3 files changed, 41 insertions(+), 9 deletions(-) + +diff --git a/daemons/attrd/attrd_attributes.c b/daemons/attrd/attrd_attributes.c +index 0ad9630..5727ab8 100644 +--- a/daemons/attrd/attrd_attributes.c ++++ b/daemons/attrd/attrd_attributes.c +@@ -210,3 +210,41 @@ attrd_populate_attribute(xmlNode *xml, const char *attr) + + return a; + } ++ ++/*! ++ * \internal ++ * \brief Get the XML ID used to write out an attribute set ++ * ++ * \param[in] attr Attribute to get set ID for ++ * \param[in] node_state_id XML ID of node state that attribute value is for ++ * ++ * \return Newly allocated string with XML ID to use for \p attr set ++ */ ++char * ++attrd_set_id(const attribute_t *attr, const char *node_state_id) ++{ ++ char *set_id = NULL; ++ ++ CRM_ASSERT((attr != NULL) && (node_state_id != NULL)); ++ ++ if (attr->set_id == NULL) { ++ /* @COMPAT This should really take the set type into account. Currently ++ * we use the same XML ID for transient attributes and utilization ++ * attributes. It doesn't cause problems because the status section is ++ * not limited by the schema in any way, but it's still unfortunate. ++ * For backward compatibility reasons, we can't change this. ++ */ ++ set_id = crm_strdup_printf("%s-%s", XML_CIB_TAG_STATUS, node_state_id); ++ } else { ++ /* @COMPAT When the user specifies a set ID for an attribute, it is the ++ * same for every node. That is less than ideal, but again, the schema ++ * doesn't enforce anything for the status section. We couldn't change ++ * it without allowing the set ID to vary per value rather than per ++ * attribute, which would break backward compatibility, pose design ++ * challenges, and potentially cause problems in rolling upgrades. ++ */ ++ pcmk__str_update(&set_id, attr->set_id); ++ } ++ crm_xml_sanitize_id(set_id); ++ return set_id; ++} +diff --git a/daemons/attrd/attrd_cib.c b/daemons/attrd/attrd_cib.c +index 481fea7..08d3425 100644 +--- a/daemons/attrd/attrd_cib.c ++++ b/daemons/attrd/attrd_cib.c +@@ -423,17 +423,10 @@ add_unset_attr_update(const attribute_t *attr, const char *attr_id, + static int + add_attr_update(const attribute_t *attr, const char *value, const char *node_id) + { +- char *set_id = NULL; ++ char *set_id = attrd_set_id(attr, node_id); + char *attr_id = NULL; + int rc = pcmk_rc_ok; + +- if (attr->set_id != NULL) { +- pcmk__str_update(&set_id, attr->set_id); +- } else { +- set_id = crm_strdup_printf("%s-%s", XML_CIB_TAG_STATUS, node_id); +- } +- crm_xml_sanitize_id(set_id); +- + if (attr->uuid != NULL) { + pcmk__str_update(&attr_id, attr->uuid); + } else { +diff --git a/daemons/attrd/pacemaker-attrd.h b/daemons/attrd/pacemaker-attrd.h +index bacaad6..3da7f8d 100644 +--- a/daemons/attrd/pacemaker-attrd.h ++++ b/daemons/attrd/pacemaker-attrd.h +@@ -1,5 +1,5 @@ + /* +- * Copyright 2013-2023 the Pacemaker project contributors ++ * Copyright 2013-2024 the Pacemaker project contributors + * + * The version control history for this file may have further details. + * +@@ -195,6 +195,7 @@ void attrd_clear_value_seen(void); + void attrd_free_attribute(gpointer data); + void attrd_free_attribute_value(gpointer data); + attribute_t *attrd_populate_attribute(xmlNode *xml, const char *attr); ++char *attrd_set_id(const attribute_t *attr, const char *node_state_id); + + enum attrd_write_options { + attrd_write_changed = 0, +-- +2.31.1 + +From eee2169ac348b8ed26ac0b78cb11ddc5cef9384e Mon Sep 17 00:00:00 2001 +From: Ken Gaillot +Date: Tue, 16 Jan 2024 12:25:59 -0600 +Subject: [PATCH 4/5] Refactor: pacemaker-attrd: functionize getting attribute + nvpair ID + +... for future reuse +--- + daemons/attrd/attrd_attributes.c | 28 ++++++++++++++++++++++++++++ + daemons/attrd/attrd_cib.c | 17 +++++------------ + daemons/attrd/pacemaker-attrd.h | 1 + + 3 files changed, 34 insertions(+), 12 deletions(-) + +diff --git a/daemons/attrd/attrd_attributes.c b/daemons/attrd/attrd_attributes.c +index 5727ab8..23de2e2 100644 +--- a/daemons/attrd/attrd_attributes.c ++++ b/daemons/attrd/attrd_attributes.c +@@ -248,3 +248,31 @@ attrd_set_id(const attribute_t *attr, const char *node_state_id) + crm_xml_sanitize_id(set_id); + return set_id; + } ++ ++/*! ++ * \internal ++ * \brief Get the XML ID used to write out an attribute value ++ * ++ * \param[in] attr Attribute to get value XML ID for ++ * \param[in] node_state_id UUID of node that attribute value is for ++ * ++ * \return Newly allocated string with XML ID of \p attr value ++ */ ++char * ++attrd_nvpair_id(const attribute_t *attr, const char *node_state_id) ++{ ++ char *nvpair_id = NULL; ++ ++ if (attr->uuid != NULL) { ++ pcmk__str_update(&nvpair_id, attr->uuid); ++ ++ } else if (attr->set_id != NULL) { ++ nvpair_id = crm_strdup_printf("%s-%s", attr->set_id, attr->id); ++ ++ } else { ++ nvpair_id = crm_strdup_printf(XML_CIB_TAG_STATUS "-%s-%s", ++ node_state_id, attr->id); ++ } ++ crm_xml_sanitize_id(nvpair_id); ++ return nvpair_id; ++} +diff --git a/daemons/attrd/attrd_cib.c b/daemons/attrd/attrd_cib.c +index 08d3425..d42345f 100644 +--- a/daemons/attrd/attrd_cib.c ++++ b/daemons/attrd/attrd_cib.c +@@ -424,23 +424,16 @@ static int + add_attr_update(const attribute_t *attr, const char *value, const char *node_id) + { + char *set_id = attrd_set_id(attr, node_id); +- char *attr_id = NULL; ++ char *nvpair_id = attrd_nvpair_id(attr, node_id); + int rc = pcmk_rc_ok; + +- if (attr->uuid != NULL) { +- pcmk__str_update(&attr_id, attr->uuid); ++ if (value == NULL) { ++ rc = add_unset_attr_update(attr, nvpair_id, node_id, set_id); + } else { +- attr_id = crm_strdup_printf("%s-%s", set_id, attr->id); +- } +- crm_xml_sanitize_id(attr_id); +- +- if (value != NULL) { +- rc = add_set_attr_update(attr, attr_id, node_id, set_id, value); +- } else { +- rc = add_unset_attr_update(attr, attr_id, node_id, set_id); ++ rc = add_set_attr_update(attr, nvpair_id, node_id, set_id, value); + } + free(set_id); +- free(attr_id); ++ free(nvpair_id); + return rc; + } + +diff --git a/daemons/attrd/pacemaker-attrd.h b/daemons/attrd/pacemaker-attrd.h +index 3da7f8d..deec790 100644 +--- a/daemons/attrd/pacemaker-attrd.h ++++ b/daemons/attrd/pacemaker-attrd.h +@@ -196,6 +196,7 @@ void attrd_free_attribute(gpointer data); + void attrd_free_attribute_value(gpointer data); + attribute_t *attrd_populate_attribute(xmlNode *xml, const char *attr); + char *attrd_set_id(const attribute_t *attr, const char *node_state_id); ++char *attrd_nvpair_id(const attribute_t *attr, const char *node_state_id); + + enum attrd_write_options { + attrd_write_changed = 0, +-- +2.31.1 + +From 2abde6cb87d2e3d31a370c74656f6f7c0818c185 Mon Sep 17 00:00:00 2001 +From: Ken Gaillot +Date: Thu, 18 Jan 2024 10:01:56 -0600 +Subject: [PATCH 5/5] Log: pacemaker-attrd: improve some messages for debugging + +--- + daemons/attrd/attrd_attributes.c | 8 +++++--- + daemons/attrd/attrd_cib.c | 13 +++++++++---- + daemons/attrd/attrd_corosync.c | 10 ++++++---- + 3 files changed, 20 insertions(+), 11 deletions(-) + +diff --git a/daemons/attrd/attrd_attributes.c b/daemons/attrd/attrd_attributes.c +index 23de2e2..68b9585 100644 +--- a/daemons/attrd/attrd_attributes.c ++++ b/daemons/attrd/attrd_attributes.c +@@ -60,13 +60,10 @@ attrd_create_attribute(xmlNode *xml) + a->values = pcmk__strikey_table(NULL, attrd_free_attribute_value); + + a->user = crm_element_value_copy(xml, PCMK__XA_ATTR_USER); +- crm_trace("Performing all %s operations as user '%s'", a->id, a->user); + + if (dampen_s != NULL) { + dampen = crm_get_msec(dampen_s); + } +- crm_trace("Created attribute %s with %s write delay", a->id, +- (a->timeout_ms == 0)? "no" : pcmk__readable_interval(a->timeout_ms)); + + if(dampen > 0) { + a->timeout_ms = dampen; +@@ -75,6 +72,11 @@ attrd_create_attribute(xmlNode *xml) + crm_warn("Ignoring invalid delay %s for attribute %s", dampen_s, a->id); + } + ++ crm_trace("Created attribute %s with %s write delay and %s CIB user", ++ a->id, ++ ((dampen > 0)? pcmk__readable_interval(a->timeout_ms) : "no"), ++ pcmk__s(a->user, "default")); ++ + g_hash_table_replace(attributes, a->id, a); + return a; + } +diff --git a/daemons/attrd/attrd_cib.c b/daemons/attrd/attrd_cib.c +index d42345f..cae6846 100644 +--- a/daemons/attrd/attrd_cib.c ++++ b/daemons/attrd/attrd_cib.c +@@ -54,6 +54,7 @@ attrd_cib_updated_cb(const char *event, xmlNode *msg) + bool status_changed = false; + + if (attrd_shutting_down(true)) { ++ crm_debug("Ignoring CIB change during shutdown"); + return; + } + +@@ -278,11 +279,13 @@ attrd_cib_callback(xmlNode *msg, int call_id, int rc, xmlNode *output, void *use + + g_hash_table_iter_init(&iter, a->values); + while (g_hash_table_iter_next(&iter, (gpointer *) & peer, (gpointer *) & v)) { +- do_crm_log(level, "* %s[%s]=%s", +- a->id, peer, pcmk__s(v->requested, "(null)")); + if (rc == pcmk_ok) { ++ crm_info("* Wrote %s[%s]=%s", ++ a->id, peer, pcmk__s(v->requested, "(unset)")); + pcmk__str_update(&(v->requested), NULL); + } else { ++ do_crm_log(level, "* Could not write %s[%s]=%s", ++ a->id, peer, pcmk__s(v->requested, "(unset)")); + a->changed = true; // Reattempt write below if we are still writer + } + } +@@ -292,6 +295,7 @@ attrd_cib_callback(xmlNode *msg, int call_id, int rc, xmlNode *output, void *use + /* We deferred a write of a new update because this update was in + * progress. Write out the new value without additional delay. + */ ++ crm_debug("Pending update for %s can be written now", a->id); + write_attribute(a, false); + + /* We're re-attempting a write because the original failed; delay +@@ -593,8 +597,9 @@ write_attribute(attribute_t *a, bool ignore_delay) + continue; + } + +- crm_debug("Updating %s[%s]=%s (node uuid=%s id=%" PRIu32 ")", +- a->id, v->nodename, v->current, uuid, v->nodeid); ++ crm_debug("Writing %s[%s]=%s (node-state-id=%s node-id=%" PRIu32 ")", ++ a->id, v->nodename, pcmk__s(v->current, "(unset)"), ++ uuid, v->nodeid); + cib_updates++; + + /* Preservation of the attribute to transmit alert */ +diff --git a/daemons/attrd/attrd_corosync.c b/daemons/attrd/attrd_corosync.c +index b348d52..6fb847b 100644 +--- a/daemons/attrd/attrd_corosync.c ++++ b/daemons/attrd/attrd_corosync.c +@@ -293,7 +293,8 @@ update_attr_on_host(attribute_t *a, const crm_node_t *peer, const xmlNode *xml, + + // Write out new value or start dampening timer + if (a->timeout_ms && a->timer) { +- crm_trace("Delayed write out (%dms) for %s", a->timeout_ms, attr); ++ crm_trace("Delaying write of %s %s for dampening", ++ attr, pcmk__readable_interval(a->timeout_ms)); + mainloop_timer_start(a->timer); + } else { + attrd_write_or_elect_attribute(a); +@@ -307,11 +308,12 @@ update_attr_on_host(attribute_t *a, const crm_node_t *peer, const xmlNode *xml, + if (is_force_write == 1 && a->timeout_ms && a->timer) { + /* Save forced writing and set change flag. */ + /* The actual attribute is written by Writer after election. */ +- crm_trace("Unchanged %s[%s] from %s is %s(Set the forced write flag)", +- attr, host, peer->uname, value); ++ crm_trace("%s[%s] from %s is unchanged (%s), forcing write", ++ attr, host, peer->uname, pcmk__s(value, "unset")); + a->force_write = TRUE; + } else { +- crm_trace("Unchanged %s[%s] from %s is %s", attr, host, peer->uname, value); ++ crm_trace("%s[%s] from %s is unchanged (%s)", ++ attr, host, peer->uname, pcmk__s(value, "unset")); + } + } + +-- +2.31.1 + diff --git a/SOURCES/009-attrd-shutdown-2.patch b/SOURCES/009-attrd-shutdown-2.patch deleted file mode 100644 index ba79a62..0000000 --- a/SOURCES/009-attrd-shutdown-2.patch +++ /dev/null @@ -1,210 +0,0 @@ -From 83e547cc64f2586031a007ab58e91fc22cd1a68a Mon Sep 17 00:00:00 2001 -From: Ken Gaillot -Date: Thu, 24 Aug 2023 12:18:23 -0500 -Subject: [PATCH] Refactor: attrd: use enum instead of bools for - attrd_write_attributes() - ---- - daemons/attrd/attrd_cib.c | 24 ++++++++++++++++++------ - daemons/attrd/attrd_corosync.c | 2 +- - daemons/attrd/attrd_elections.c | 2 +- - daemons/attrd/attrd_ipc.c | 2 +- - daemons/attrd/attrd_utils.c | 2 +- - daemons/attrd/pacemaker-attrd.h | 8 +++++++- - 6 files changed, 29 insertions(+), 11 deletions(-) - -diff --git a/daemons/attrd/attrd_cib.c b/daemons/attrd/attrd_cib.c -index 928c0133745..9c787fe1024 100644 ---- a/daemons/attrd/attrd_cib.c -+++ b/daemons/attrd/attrd_cib.c -@@ -343,16 +343,23 @@ attrd_write_attribute(attribute_t *a, bool ignore_delay) - free_xml(xml_top); - } - -+/*! -+ * \internal -+ * \brief Write out attributes -+ * -+ * \param[in] options Group of enum attrd_write_options -+ */ - void --attrd_write_attributes(bool all, bool ignore_delay) -+attrd_write_attributes(uint32_t options) - { - GHashTableIter iter; - attribute_t *a = NULL; - -- crm_debug("Writing out %s attributes", all? "all" : "changed"); -+ crm_debug("Writing out %s attributes", -+ pcmk_is_set(options, attrd_write_all)? "all" : "changed"); - g_hash_table_iter_init(&iter, attributes); - while (g_hash_table_iter_next(&iter, NULL, (gpointer *) & a)) { -- if (!all && a->unknown_peer_uuids) { -+ if (!pcmk_is_set(options, attrd_write_all) && a->unknown_peer_uuids) { - // Try writing this attribute again, in case peer ID was learned - a->changed = true; - } else if (a->force_write) { -@@ -360,9 +367,14 @@ attrd_write_attributes(bool all, bool ignore_delay) - a->changed = true; - } - -- if(all || a->changed) { -- /* When forced write flag is set, ignore delay. */ -- attrd_write_attribute(a, (a->force_write ? true : ignore_delay)); -+ if (pcmk_is_set(options, attrd_write_all) || a->changed) { -+ bool ignore_delay = pcmk_is_set(options, attrd_write_no_delay); -+ -+ if (a->force_write) { -+ // Always ignore delay when forced write flag is set -+ ignore_delay = true; -+ } -+ attrd_write_attribute(a, ignore_delay); - } else { - crm_trace("Skipping unchanged attribute %s", a->id); - } -diff --git a/daemons/attrd/attrd_corosync.c b/daemons/attrd/attrd_corosync.c -index 1aec35a054e..49631df6e44 100644 ---- a/daemons/attrd/attrd_corosync.c -+++ b/daemons/attrd/attrd_corosync.c -@@ -285,7 +285,7 @@ record_peer_nodeid(attribute_value_t *v, const char *host) - - crm_trace("Learned %s has node id %s", known_peer->uname, known_peer->uuid); - if (attrd_election_won()) { -- attrd_write_attributes(false, false); -+ attrd_write_attributes(attrd_write_changed); - } - } - -diff --git a/daemons/attrd/attrd_elections.c b/daemons/attrd/attrd_elections.c -index c25a41a4492..01341db18e4 100644 ---- a/daemons/attrd/attrd_elections.c -+++ b/daemons/attrd/attrd_elections.c -@@ -34,7 +34,7 @@ attrd_election_cb(gpointer user_data) - attrd_peer_sync(NULL, NULL); - - /* Update the CIB after an election */ -- attrd_write_attributes(true, false); -+ attrd_write_attributes(attrd_write_all); - return G_SOURCE_REMOVE; - } - -diff --git a/daemons/attrd/attrd_ipc.c b/daemons/attrd/attrd_ipc.c -index 4be789de7f9..05c4a696a19 100644 ---- a/daemons/attrd/attrd_ipc.c -+++ b/daemons/attrd/attrd_ipc.c -@@ -232,7 +232,7 @@ attrd_client_refresh(pcmk__request_t *request) - crm_info("Updating all attributes"); - - attrd_send_ack(request->ipc_client, request->ipc_id, request->ipc_flags); -- attrd_write_attributes(true, true); -+ attrd_write_attributes(attrd_write_all|attrd_write_no_delay); - - pcmk__set_result(&request->result, CRM_EX_OK, PCMK_EXEC_DONE, NULL); - return NULL; -diff --git a/daemons/attrd/attrd_utils.c b/daemons/attrd/attrd_utils.c -index c43eac1695a..bfd51368890 100644 ---- a/daemons/attrd/attrd_utils.c -+++ b/daemons/attrd/attrd_utils.c -@@ -156,7 +156,7 @@ attrd_cib_replaced_cb(const char *event, xmlNode * msg) - if (attrd_election_won()) { - if (change_section & (cib_change_section_nodes | cib_change_section_status)) { - crm_notice("Updating all attributes after %s event", event); -- attrd_write_attributes(true, false); -+ attrd_write_attributes(attrd_write_all); - } - } - -diff --git a/daemons/attrd/pacemaker-attrd.h b/daemons/attrd/pacemaker-attrd.h -index 41f31d97b3b..2d781d11394 100644 ---- a/daemons/attrd/pacemaker-attrd.h -+++ b/daemons/attrd/pacemaker-attrd.h -@@ -176,8 +176,14 @@ void attrd_free_attribute(gpointer data); - void attrd_free_attribute_value(gpointer data); - attribute_t *attrd_populate_attribute(xmlNode *xml, const char *attr); - -+enum attrd_write_options { -+ attrd_write_changed = 0, -+ attrd_write_all = (1 << 0), -+ attrd_write_no_delay = (1 << 1), -+}; -+ - void attrd_write_attribute(attribute_t *a, bool ignore_delay); --void attrd_write_attributes(bool all, bool ignore_delay); -+void attrd_write_attributes(uint32_t options); - void attrd_write_or_elect_attribute(attribute_t *a); - - extern int minimum_protocol_version; -From 58400e272cfc51f02eec69cdd0ed0d27a30e78a3 Mon Sep 17 00:00:00 2001 -From: Ken Gaillot -Date: Thu, 24 Aug 2023 12:27:53 -0500 -Subject: [PATCH] Fix: attrd: avoid race condition at writer election - -f5263c94 was not a complete fix. The issue may also occur if a remaining node -(not the original DC or writer) wins the attribute writer election after the -original DC's controller has exited but before its attribute manger has exited. - -The long-term solution will be to have the attribute manager (instead of the -controller) be in control of erasing transient attributes from the CIB when a -node leaves. This short-term workaround simply has new attribute writers skip -shutdown attributes when writing out all attributes. - -Fixes T138 ---- - daemons/attrd/attrd_cib.c | 5 +++++ - daemons/attrd/attrd_elections.c | 14 ++++++++++++-- - daemons/attrd/pacemaker-attrd.h | 1 + - 3 files changed, 18 insertions(+), 2 deletions(-) - -diff --git a/daemons/attrd/attrd_cib.c b/daemons/attrd/attrd_cib.c -index 9c787fe102..2c910b4c64 100644 ---- a/daemons/attrd/attrd_cib.c -+++ b/daemons/attrd/attrd_cib.c -@@ -359,6 +359,11 @@ attrd_write_attributes(uint32_t options) - pcmk_is_set(options, attrd_write_all)? "all" : "changed"); - g_hash_table_iter_init(&iter, attributes); - while (g_hash_table_iter_next(&iter, NULL, (gpointer *) & a)) { -+ if (pcmk_is_set(options, attrd_write_skip_shutdown) -+ && pcmk__str_eq(a->id, XML_CIB_ATTR_SHUTDOWN, pcmk__str_none)) { -+ continue; -+ } -+ - if (!pcmk_is_set(options, attrd_write_all) && a->unknown_peer_uuids) { - // Try writing this attribute again, in case peer ID was learned - a->changed = true; -diff --git a/daemons/attrd/attrd_elections.c b/daemons/attrd/attrd_elections.c -index 01341db18e..a95cd44cbd 100644 ---- a/daemons/attrd/attrd_elections.c -+++ b/daemons/attrd/attrd_elections.c -@@ -33,8 +33,18 @@ attrd_election_cb(gpointer user_data) - /* Update the peers after an election */ - attrd_peer_sync(NULL, NULL); - -- /* Update the CIB after an election */ -- attrd_write_attributes(attrd_write_all); -+ /* After winning an election, update the CIB with the values of all -+ * attributes as the winner knows them. -+ * -+ * However, do not write out any "shutdown" attributes. A node that is -+ * shutting down will have all its transient attributes removed from the CIB -+ * when its controller exits, and from the attribute manager's memory (on -+ * remaining nodes) when its attribute manager exits; if an election is won -+ * between when those two things happen, we don't want to write the shutdown -+ * attribute back out, which would cause the node to immediately shut down -+ * the next time it rejoins. -+ */ -+ attrd_write_attributes(attrd_write_all|attrd_write_skip_shutdown); - return G_SOURCE_REMOVE; - } - -diff --git a/daemons/attrd/pacemaker-attrd.h b/daemons/attrd/pacemaker-attrd.h -index 2d781d1139..2e35bd7ec5 100644 ---- a/daemons/attrd/pacemaker-attrd.h -+++ b/daemons/attrd/pacemaker-attrd.h -@@ -180,6 +180,7 @@ enum attrd_write_options { - attrd_write_changed = 0, - attrd_write_all = (1 << 0), - attrd_write_no_delay = (1 << 1), -+ attrd_write_skip_shutdown = (1 << 2), - }; - - void attrd_write_attribute(attribute_t *a, bool ignore_delay); diff --git a/SOURCES/010-crm_attribute-free.patch b/SOURCES/010-crm_attribute-free.patch new file mode 100644 index 0000000..d1e4831 --- /dev/null +++ b/SOURCES/010-crm_attribute-free.patch @@ -0,0 +1,53 @@ +From 9c13ce6fe95812308443c188ace8f897e6bce942 Mon Sep 17 00:00:00 2001 +From: Reid Wahl +Date: Mon, 29 Jan 2024 11:14:25 -0800 +Subject: [PATCH] Fix: tools: crm_attribute emits garbage for --node localhost + or auto + +This happens because pcmk__node_attr_target() returns its argument if +its argument is NULL, "auto", or "localhost" and no relevant environment +variables are found. Then crm_attribute frees the return value, makes a +copy of it, and assigns it back to options.dest_uname. + +The fix is to check whether the return value is equal to the argument. + +Fixes RHEL-23065 + +Signed-off-by: Reid Wahl +--- + tools/crm_attribute.c | 19 +++++++++++++++++-- + 1 file changed, 17 insertions(+), 2 deletions(-) + +diff --git a/tools/crm_attribute.c b/tools/crm_attribute.c +index d221ab85d..636d03dbd 100644 +--- a/tools/crm_attribute.c ++++ b/tools/crm_attribute.c +@@ -766,8 +766,23 @@ main(int argc, char **argv) + const char *target = pcmk__node_attr_target(options.dest_uname); + + if (target != NULL) { +- g_free(options.dest_uname); +- options.dest_uname = g_strdup(target); ++ /* If options.dest_uname is "auto" or "localhost", then ++ * pcmk__node_attr_target() may return it, depending on environment ++ * variables. In that case, attribute lookups will fail for "auto" ++ * (unless there's a node named "auto"). attrd maps "localhost" to ++ * the true local node name for queries. ++ * ++ * @TODO ++ * * Investigate whether "localhost" is mapped to a real node name ++ * for non-query commands. If not, possibly modify it so that it ++ * is. ++ * * Map "auto" to "localhost" (probably). ++ */ ++ if (target != (const char *) options.dest_uname) { ++ g_free(options.dest_uname); ++ options.dest_uname = g_strdup(target); ++ } ++ + } else if (getenv("CIB_file") != NULL && options.dest_uname == NULL) { + get_node_name_from_local(); + } +-- +2.41.0 + diff --git a/SOURCES/pacemaker.sysusers b/SOURCES/pacemaker.sysusers new file mode 100644 index 0000000..daadb96 --- /dev/null +++ b/SOURCES/pacemaker.sysusers @@ -0,0 +1,3 @@ +#Type Name ID GECOS Home directory Shell +g haclient 189 +u hacluster 189:haclient "cluster user" /var/lib/pacemaker /sbin/nologin diff --git a/SPECS/pacemaker.spec b/SPECS/pacemaker.spec index aeec504..3960db8 100644 --- a/SPECS/pacemaker.spec +++ b/SPECS/pacemaker.spec @@ -35,11 +35,11 @@ ## Upstream pacemaker version, and its package version (specversion ## can be incremented to build packages reliably considered "newer" ## than previously built packages with the same pcmkversion) -%global pcmkversion 2.1.6 -%global specversion 9 +%global pcmkversion 2.1.7 +%global specversion 4 ## Upstream commit (full commit ID, abbreviated commit ID, or tag) to build -%global commit 6fdc9deea294bbad629b003c6ae036aaed8e3ee0 +%global commit 0f7f88312f7a1ccedee60bf768aba79ee13d41e0 ## Since git v2.11, the extent of abbreviation is autoscaled by default ## (used to be constant of 7), so we need to convey it for non-tags, too. @@ -153,7 +153,6 @@ %global pkgname_procps procps %global pkgname_glue_libs libglue %global pkgname_pcmk_libs lib%{name}3 -%global hacluster_id 90 %else %global pkgname_libtool_devel libtool-ltdl-devel %global pkgname_libtool_devel_arch libtool-ltdl-devel%{?_isa} @@ -165,7 +164,6 @@ %global pkgname_procps procps-ng %global pkgname_glue_libs cluster-glue-libs %global pkgname_pcmk_libs %{name}-libs -%global hacluster_id 189 %endif ## Distro-specific configuration choices @@ -246,17 +244,19 @@ Url: https://www.clusterlabs.org/ # You can use "spectool -s 0 pacemaker.spec" (rpmdevtools) to show final URL. Source0: https://codeload.github.com/%{github_owner}/%{name}/tar.gz/%{archive_github_url} Source1: https://codeload.github.com/%{github_owner}/%{nagios_name}/tar.gz/%{nagios_archive_github_url} +Source2: pacemaker.sysusers # upstream commits -Patch001: 001-remote-start-state.patch -Patch002: 002-group-colocation-constraint.patch -Patch003: 003-clone-shuffle.patch -Patch004: 004-clone-rsc-display.patch -Patch005: 005-attrd-dampen.patch -Patch006: 006-controller-reply.patch -Patch007: 007-glib-assertions.patch -Patch008: 008-attrd-shutdown.patch -Patch009: 009-attrd-shutdown-2.patch +Patch001: 001-schema-glib.patch +Patch002: 002-schema-transfer.patch +Patch003: 003-schema-doc.patch +Patch004: 004-attrd-cache-1.patch +Patch005: 005-attrd-cache-2.patch +Patch006: 006-cib-file-feature-set.patch +Patch007: 007-option-metadata.patch +Patch008: 008-attrd-prep.patch +Patch009: 009-attrd-cache-3.patch +Patch010: 010-crm_attribute-free.patch Requires: resource-agents Requires: %{pkgname_pcmk_libs}%{?_isa} = %{version}-%{release} @@ -301,7 +301,7 @@ BuildRequires: sed # Required for core functionality BuildRequires: pkgconfig(glib-2.0) >= 2.42 -BuildRequires: libxml2-devel +BuildRequires: libxml2-devel >= 2.6.0 BuildRequires: libxslt-devel BuildRequires: libuuid-devel BuildRequires: %{pkgname_bzip2_devel} @@ -316,7 +316,7 @@ BuildRequires: pam-devel BuildRequires: %{pkgname_gettext} >= 0.18 # Required for "make check" -BuildRequires: libcmocka-devel +BuildRequires: libcmocka-devel >= 1.1.0 BuildRequires: pkgconfig(systemd) @@ -339,6 +339,10 @@ BuildRequires: inkscape BuildRequires: %{python_name}-sphinx %endif +# Creation of Users / Groups +BuildRequires: systemd-rpm-macros +%{?sysusers_requires_compat} + # Booth requires this Provides: pacemaker-ticket-support = 2.0 @@ -464,7 +468,7 @@ Requires: libqb-devel%{?_isa} Requires: %{?pkgname_libtool_devel_arch} %endif Requires: libuuid-devel%{?_isa} -Requires: libxml2-devel%{?_isa} +Requires: libxml2-devel%{?_isa} >= 2.6.0 Requires: libxslt-devel%{?_isa} %description -n %{pkgname_pcmk_libs}-devel @@ -624,6 +628,8 @@ find %{buildroot} -name '*.la' -type f -print0 | xargs -0 rm -f rm -f %{buildroot}/%{_sbindir}/fence_legacy rm -f %{buildroot}/%{_mandir}/man8/fence_legacy.* +install -p -D -m 0644 %{SOURCE2} %{buildroot}%{_sysusersdir}/pacemaker.conf + # For now, don't package the servicelog-related binaries built only for # ppc64le when certain dependencies are installed. If they get more exercise by # advanced users, we can reconsider. @@ -699,10 +705,7 @@ fi %systemd_postun_with_restart crm_mon.service %pre -n %{pkgname_pcmk_libs} -# @TODO Use sysusers.d: -# https://fedoraproject.org/wiki/Changes/Adopting_sysusers.d_format -getent group %{gname} >/dev/null || groupadd -r %{gname} -g %{hacluster_id} -getent passwd %{uname} >/dev/null || useradd -r -g %{gname} -u %{hacluster_id} -s /sbin/nologin -c "cluster user" %{uname} +%sysusers_create_compat %{SOURCE2} exit 0 %ldconfig_scriptlets -n %{pkgname_pcmk_libs} @@ -723,15 +726,20 @@ exit 0 %exclude %{_sbindir}/pacemaker_remoted %{_libexecdir}/pacemaker/* -%{_sbindir}/crm_master +%if %{with stonithd} +%{_sbindir}/fence_legacy +%endif %{_sbindir}/fence_watchdog %doc %{_mandir}/man7/pacemaker-controld.* %doc %{_mandir}/man7/pacemaker-schedulerd.* %doc %{_mandir}/man7/pacemaker-fenced.* %doc %{_mandir}/man7/ocf_pacemaker_controld.* +%doc %{_mandir}/man7/ocf_pacemaker_o2cb.* %doc %{_mandir}/man7/ocf_pacemaker_remote.* -%doc %{_mandir}/man8/crm_master.* +%if %{with stonithd} +%doc %{_mandir}/man8/fence_legacy.* +%endif %doc %{_mandir}/man8/fence_watchdog.* %doc %{_mandir}/man8/pacemakerd.* @@ -744,6 +752,7 @@ exit 0 %dir %attr (750, %{uname}, %{gname}) %{_var}/lib/pacemaker/cib %dir %attr (750, %{uname}, %{gname}) %{_var}/lib/pacemaker/pengine %{ocf_root}/resource.d/pacemaker/controld +%{ocf_root}/resource.d/pacemaker/o2cb %{ocf_root}/resource.d/pacemaker/remote %files cli @@ -762,6 +771,7 @@ exit 0 %{_sbindir}/crm_diff %{_sbindir}/crm_error %{_sbindir}/crm_failcount +%{_sbindir}/crm_master %{_sbindir}/crm_mon %{_sbindir}/crm_node %{_sbindir}/crm_resource @@ -797,7 +807,6 @@ exit 0 %exclude %{_mandir}/man7/ocf_pacemaker_o2cb.* %exclude %{_mandir}/man7/ocf_pacemaker_remote.* %doc %{_mandir}/man8/crm*.8.gz -%exclude %{_mandir}/man8/crm_master.* %doc %{_mandir}/man8/attrd_updater.* %doc %{_mandir}/man8/cibadmin.* %if %{with cibsecrets} @@ -817,6 +826,7 @@ exit 0 %dir %attr (770, %{uname}, %{gname}) %{_var}/log/pacemaker/bundles %files -n %{pkgname_pcmk_libs} %{?with_nls:-f %{name}.lang} +%{_sysusersdir}/pacemaker.conf %{_libdir}/libcib.so.* %{_libdir}/liblrmd.so.* %{_libdir}/libcrmservice.so.* @@ -863,7 +873,6 @@ exit 0 %license licenses/CC-BY-SA-4.0 %files cts -%{python_site}/cts %{python3_sitelib}/pacemaker/_cts/ %{_datadir}/pacemaker/tests @@ -906,6 +915,43 @@ exit 0 %license %{nagios_name}-%{nagios_hash}/COPYING %changelog +* Wed Jan 31 2024 Chris Lumens - 2.1.7-4 +- Properly validate attribute set type in pacemaker-attrd +- Fix `crm_attribute -t nodes --node localhost` +- Resolves: RHEL-13216 +- Resolves: RHEL-17225 +- Resolves: RHEL-23498 + +* Tue Jan 16 2024 Chris Lumens - 2.1.7-3 +- Rebase on upstream 2.1.7 final release +- Fix documentation for Pacemaker Remote schema transfers +- Do not check CIB feature set version when CIB_file is set +- Consolidate attrd cache handling +- Avoid duplicating option metadata across daemons +- Related: RHEL-7665 +- Related: RHEL-13216 +- Resolves: RHEL-7702 + +* Wed Dec 13 2023 Chris Lumens - 2.1.7-2 +- Rebase on upstream 2.1.7-rc4 release +- Use systemd-sysusers to create user/group +- Pacemaker Remote nodes can validate against later schema versions +- Related: RHEL-17225 +- Resolves: RHEL-7665 + +* Wed Nov 22 2023 Chris Lumens - 2.1.7-1 +- Rebase on upstream 2.1.7-rc2 release +- Resolves: RHEL-7682 +- Related: RHEL-17225 + +* Tue Oct 31 2023 Chris Lumens - 2.1.6-10.1 +- Revert the rest of the attrd shutdown race condition fix +- Related: RHEL-14044 + +* Thu Oct 19 2023 Chris Lumens - 2.1.6-10 +- Avoid an error if the elected attrd is on a node that is shutting down +- Resolves: RHEL-14044 + * Mon Aug 28 2023 Chris Lumens - 2.1.6-9 - Fix an additional shutdown race between attrd and the controller - Related: rhbz2228933