diff --git a/SOURCES/RHEL-114489-2-powervs-move-ip-set-bundled-path.patch b/SOURCES/RHEL-114489-2-powervs-move-ip-set-bundled-path.patch index 167bdf2..20c2d65 100644 --- a/SOURCES/RHEL-114489-2-powervs-move-ip-set-bundled-path.patch +++ b/SOURCES/RHEL-114489-2-powervs-move-ip-set-bundled-path.patch @@ -8,7 +8,7 @@ -import requests.adapters -import urllib3.util +try: -+ sys.path.insert(0, '/usr/lib/fence-agents/support/ibm') ++ sys.path.insert(0, '/usr/lib/fence-agents/support/ibm/lib/python#PYTHON3_VERSION#/site-packages') + import requests + import requests.adapters + import urllib3.util diff --git a/SOURCES/RHEL-151828-portblock-check-inverse-action.patch b/SOURCES/RHEL-151828-portblock-check-inverse-action.patch new file mode 100644 index 0000000..bbba366 --- /dev/null +++ b/SOURCES/RHEL-151828-portblock-check-inverse-action.patch @@ -0,0 +1,156 @@ +--- a/heartbeat/portblock 2026-02-27 08:43:50.813925268 +0100 ++++ b/heartbeat/portblock 2026-02-27 08:44:40.481824601 +0100 +@@ -29,12 +29,17 @@ + OCF_RESKEY_direction_default="in" + OCF_RESKEY_action_default="" + OCF_RESKEY_method_default="drop" +-OCF_RESKEY_status_check_default="rule" + OCF_RESKEY_ip_default="0.0.0.0/0" + OCF_RESKEY_reset_local_on_unblock_stop_default="false" + OCF_RESKEY_tickle_dir_default="" + OCF_RESKEY_sync_script_default="" + ++if ocf_is_ms; then ++ OCF_RESKEY_status_check_default="rule" ++else ++ OCF_RESKEY_status_check_default="pseudo" ++fi ++ + : ${OCF_RESKEY_protocol=${OCF_RESKEY_protocol_default}} + : ${OCF_RESKEY_portno=${OCF_RESKEY_portno_default}} + : ${OCF_RESKEY_direction=${OCF_RESKEY_direction_default}} +@@ -401,6 +406,10 @@ + done + } + ++# A long time ago, these messages needed to go to stdout, ++# "running" / "OK" being the trigger string ++# for heartbeat in haresources mode. ++# Now they are still useful for debugging. + SayActive() + { + ocf_log debug "$CMD $method rule [$*] is running (OK)" +@@ -416,6 +425,11 @@ + ocf_log debug "$CMD $method rule [$*] is inactive" + } + ++SayConsideredInactive() ++{ ++ ocf_log debug "$CMD $method rule [$*] considered to be inactive" ++} ++ + #IptablesStatus {udp|tcp} portno,portno ip {in|out|both} {block|unblock} + IptablesStatus() { + local rc +@@ -441,8 +455,17 @@ + fi + ;; + *) +- SayInactive $* +- rc=$OCF_NOT_RUNNING ++ if [ "$OCF_RESKEY_status_check" != "rule" ] \ ++ && test -e "$state_file" && test "$inverse_state_file" -nt "$state_file"; then ++ # rule present, action=unblock, unblock statefile present, ++ # block state file more recent. ++ # apparently an unusual setup: unblock first, block later ++ SayConsideredActive $* ++ rc=$OCF_SUCCESS ++ else ++ SayInactive $* ++ rc=$OCF_NOT_RUNNING ++ fi + ;; + esac + elif [ "$OCF_RESKEY_status_check" = "rule" ]; then +@@ -454,6 +477,7 @@ + *) + SayActive $* + if [ "$__OCF_ACTION" = "monitor" ] && [ "$promotion_score" = "$SCORE_PROMOTED" ]; then ++ save_tcp_connections + rc=$OCF_RUNNING_MASTER + else + rc=$OCF_SUCCESS +@@ -463,7 +487,10 @@ + else + case $5 in + block) +- if ha_pseudo_resource "${OCF_RESOURCE_INSTANCE}" status; then ++ if test -e "$state_file" && test "$inverse_state_file" -nt "$state_file"; then ++ # rule NOT present, action=block, block state file present, ++ # unblock state file more recent. ++ # expected setup: block first, unblock later + SayConsideredActive $* + rc=$OCF_SUCCESS + else +@@ -472,13 +499,15 @@ + fi + ;; + *) +- if ha_pseudo_resource "${OCF_RESOURCE_INSTANCE}" status; then ++ if test -e "$state_file" ; then ++ # rule NOT present, action=unblock, unblock state file present + SayActive $* +- #This is only run on real monitor events. ++ # This is only run on real monitor events (state file present). + save_tcp_connections + rc=$OCF_SUCCESS + else +- SayInactive $* ++ # rule NOT present, action=unblock, unblock state file NOT present ++ SayConsideredInactive $* + rc=$OCF_NOT_RUNNING + fi + ;; +@@ -562,7 +591,7 @@ + #IptablesStart {udp|tcp} portno,portno ip {in|out|both} {block|unblock} + IptablesStart() + { +- ha_pseudo_resource "${OCF_RESOURCE_INSTANCE}" start ++ ha_pseudo_resource "${OCF_RESOURCE_INSTANCE}" start "$state_file" + case $5 in + block) IptablesBLOCK "$@" + rc=$? +@@ -584,7 +613,8 @@ + #IptablesStop {udp|tcp} portno,portno ip {in|out|both} {block|unblock} + IptablesStop() + { +- ha_pseudo_resource "${OCF_RESOURCE_INSTANCE}" stop ++ ha_pseudo_resource "${OCF_RESOURCE_INSTANCE}" stop "$state_file" ++ + case $5 in + block) IptablesUNBLOCK "$@" + rc=$? +@@ -797,6 +827,33 @@ + + IptablesValidateAll + ++# State file name for ha_pseudo_resource ++# ++# The expected usage of this agent is to pair a "block" with an "unblock", ++# and order startup and configuration of some service between these. ++# ++# The established idiom is to have two separate instances with inverse actions. ++# To "reliably" report the status of "block" during a monitor action, ++# it is not sufficient to check the existence of the blocking rule. ++# ++# It is also insufficient to rely on the pseudo resource state file ++# of this instance only. ++# ++# To know our actual expectation, we need to check the state file of the ++# "inverse" instance as well. ++# ++# Because we don't know the OCF_RESOURCE_INSTANCE value of the other instance, ++# we override the state file name for both instances to something derived from ++# our parameters. ++# ++# This should give use the same "global state" view as the "promotion score" ++# does for the promotable clone variant of this agent. ++# ++[ "$action" = block ] && inverse_action=unblock || inverse_action=block ++state_file_base=$(echo "portblock_${protocol}_${portno}_${ip}_${direction}" | tr -c '0-9a-zA-Z._' _) ++state_file=${HA_RSCTMP}/${state_file_base}_${action} ++inverse_state_file=${HA_RSCTMP}/${state_file_base}_${inverse_action} ++ + case $__OCF_ACTION in + start) + IptablesStart "$protocol" "$portno" "$ip" "$direction" "$action" diff --git a/SOURCES/RHEL-156808-podman-etcd-ignore-learners-when-considering-which-node-has-higher-revision.patch b/SOURCES/RHEL-156808-podman-etcd-ignore-learners-when-considering-which-node-has-higher-revision.patch new file mode 100644 index 0000000..280b14f --- /dev/null +++ b/SOURCES/RHEL-156808-podman-etcd-ignore-learners-when-considering-which-node-has-higher-revision.patch @@ -0,0 +1,40 @@ +From 5890f47bc61703130cd27d767118367f03bca95f Mon Sep 17 00:00:00 2001 +From: Carlo Lobrano +Date: Tue, 10 Mar 2026 17:26:04 +0100 +Subject: [PATCH] podman-etcd: Preserve standalone voter identity during + restart + +If the standalone voter restarts before the peer is added to the member +list, the learner_node attribute may not be set yet. Without checking +is_standalone, the voter incorrectly joins as a learner, causing both +nodes to become learners and creating an unrecoverable deadlock. + +Check is_standalone to ensure the voter restarts in the same role it +had before the restart. +--- + heartbeat/podman-etcd | 11 +++++++++-- + 1 file changed, 9 insertions(+), 2 deletions(-) + +diff --git a/heartbeat/podman-etcd b/heartbeat/podman-etcd +index 539ad33b2..2f8aa122f 100755 +--- a/heartbeat/podman-etcd ++++ b/heartbeat/podman-etcd +@@ -2002,9 +2002,16 @@ podman_start() + ocf_log info "found '$active_resources_count' active etcd resources (active: '$OCF_RESKEY_CRM_meta_notify_active_resource', stop: '$OCF_RESKEY_CRM_meta_notify_stop_resource')" + case "$active_resources_count" in + 1) +- if [ "$(attribute_learner_node get)" = "$(get_peer_node_name)" ]; then +- ocf_log info "peer active but in learner mode: start normally" ++ # is_standalone may return true here due to a restart: in the previous run, ++ # this agent was the sole voter and the peer had not yet joined the member ++ # list (learner_node unset). Since standalone_node was not cleared before ++ # the restart, start normally to recover the previous cluster state. ++ if is_standalone; then ++ ocf_log info "peer active but not a voter: start normally to recover" ++ elif [ "$(attribute_learner_node get)" = "$(get_peer_node_name)" ]; then ++ ocf_log info "peer active but in learner mode: start normally to recover" + else ++ # If (A) we must join the peer's new cluster + ocf_log info "peer is active standalone: joining as learner" + JOIN_AS_LEARNER=true + fi diff --git a/SOURCES/RHEL-157145-podman-etcd-handle-existing-peer-URLs-gracefully-during-force_new_cluster-recovery.patch b/SOURCES/RHEL-157145-podman-etcd-handle-existing-peer-URLs-gracefully-during-force_new_cluster-recovery.patch new file mode 100644 index 0000000..89b206e --- /dev/null +++ b/SOURCES/RHEL-157145-podman-etcd-handle-existing-peer-URLs-gracefully-during-force_new_cluster-recovery.patch @@ -0,0 +1,130 @@ +From 83d16b59a354a20bf7679af45a9acfa9f344959a Mon Sep 17 00:00:00 2001 +From: Carlo Lobrano +Date: Wed, 18 Mar 2026 11:30:31 +0100 +Subject: [PATCH] OCPBUGS-78482: podman-etcd: fix "Peer URLs already exists" in + add_member_as_learner (#2136) + +* podman-etcd: handle "Peer URLs already exists" in add_member_as_learner + +When etcdctl member add fails with "Peer URLs already exists", the stale +member entry is removed and the add is retried. + +Without this fix, add_member_as_learner returns early without setting +the learner_node attribute, causing the peer node to time out waiting +for it. +--- + heartbeat/podman-etcd | 85 +++++++++++++++++++++++++++++++++++++------ + 1 file changed, 73 insertions(+), 12 deletions(-) + +diff --git a/heartbeat/podman-etcd b/heartbeat/podman-etcd +index 2f8aa122f..860aca817 100755 +--- a/heartbeat/podman-etcd ++++ b/heartbeat/podman-etcd +@@ -1072,6 +1072,49 @@ attribute_node_member_id() + esac + } + ++# remove an etcd member identified by its IP from the member list ++remove_etcd_member_by_ip() ++{ ++ local rc ++ local out ++ local member_ip ++ local endpoint_url ++ local member_list_json ++ local stale_member_id ++ local stale_member_id_hex ++ ++ member_ip=$1 ++ member_list_json=$(get_member_list_json) ++ rc=$? ++ if [ "$rc" -ne 0 ] ; then ++ ocf_log err "could not remove etcd member. Failed to get member list, error code: $rc" ++ return $OCF_ERR_GENERIC ++ fi ++ ++ stale_member_id=$(printf "%s" "$member_list_json" | jq -r ".members[] | select( .peerURLs | map(test(\"$member_ip\")) | any).ID") ++ if [ -z "$stale_member_id" ]; then ++ ocf_log err "could not remove etcd member. Failed to find member ID" ++ return $OCF_ERR_GENERIC ++ fi ++ ++ # JSON member_id is decimal, while etcdctl command needs the hex version ++ if ! stale_member_id_hex=$(decimal_to_hex "$stale_member_id"); then ++ ocf_log err "could not remove etcd member. Failed to convert member_id '$stale_member_id' into hex format" ++ return $OCF_ERR_GENERIC ++ fi ++ ++ endpoint_url=$(ip_url $(attribute_node_ip get)) ++ out=$(podman exec "${CONTAINER}" etcdctl --endpoints="$endpoint_url:2379" member remove "$stale_member_id_hex" 2>&1) ++ rc=$? ++ if [ "$rc" -ne 0 ] ; then ++ ocf_log err "could not remove etcd member. etcdctl member remove command failed, error code: $rc, output: $out" ++ return $OCF_ERR_GENERIC ++ fi ++ ++ ocf_log info "$out" ++ return $OCF_SUCCESS ++} ++ + add_member_as_learner() + { + local rc +@@ -1079,18 +1122,36 @@ add_member_as_learner() + local member_ip=$2 + local endpoint_url=$(ip_url $(attribute_node_ip get)) + local peer_url=$(ip_url $member_ip) ++ local i ++ local max_retries ++ local out + +- ocf_log info "add $member_name ($member_ip) to the member list as learner" +- out=$(podman exec "${CONTAINER}" etcdctl --endpoints="$endpoint_url:2379" member add "$member_name" --peer-urls="$peer_url:2380" --learner 2>&1) +- rc=$? +- if [ $rc -ne 0 ]; then +- ocf_log err "could not add $member_name as learner, error code $rc, etcdctl output: $out" +- return $rc +- fi +- ocf_log info "$out" ++ i=0 ++ max_retries=3 ++ while [ "$i" -lt "$max_retries" ]; do ++ i=$((i + 1)) ++ ocf_log info "adding $member_name ($member_ip) to the member list as learner (attempt $i of $max_retries)" ++ out=$(podman exec "${CONTAINER}" etcdctl --endpoints="$endpoint_url:2379" \ ++ member add "$member_name" \ ++ --peer-urls="$peer_url:2380" \ ++ --learner 2>&1) ++ rc=$? ++ if [ "$rc" -eq 0 ]; then ++ ocf_log info "$out" ++ attribute_learner_node update "$member_name" ++ return $OCF_SUCCESS ++ fi ++ if echo "$out" | grep -q "Peer URLs already exists"; then ++ # etcd data might have stale membership data ++ ocf_log warn "could not add member: Peer URLs already exists" ++ remove_etcd_member_by_ip "$member_ip" ++ else ++ ocf_log warn "could not add member: $out" ++ fi ++ done + +- attribute_learner_node update "$member_name" +- return $? ++ ocf_log err "could not add $member_name as learner, error code $rc, etcdctl output: $out" ++ return $OCF_ERR_GENERIC + } + + set_force_new_cluster() +@@ -1454,8 +1515,8 @@ manage_peer_membership() + # "https://:2379" + # ] + # } +- # NOTE that the "name" field is present in voting members only, while "isLearner" +- # field in learner members only and the value is always true (not a string) in that case. ++ # NOTE: voting members have a "name" field but no "isLearner" field, ++ # while learner members have "isLearner": true (boolean) but no "name" field, so we search for peerURLs matching. + peer_member_id=$(printf "%s" "$member_list_json" | jq -r ".members[] | select( .peerURLs | map(test(\"$peer_member_ip\")) | any).ID") + if [ -z "$peer_member_id" ]; then + ocf_log info "$peer_member_name is not in the members list" diff --git a/SOURCES/RHEL-157273-db2-set-reintegration-when-promotion-is-successful.patch b/SOURCES/RHEL-157273-db2-set-reintegration-when-promotion-is-successful.patch new file mode 100644 index 0000000..56bc252 --- /dev/null +++ b/SOURCES/RHEL-157273-db2-set-reintegration-when-promotion-is-successful.patch @@ -0,0 +1,46 @@ +From 66885ea0227e847b571608015b150d391a6234d7 Mon Sep 17 00:00:00 2001 +From: Oyvind Albrigtsen +Date: Mon, 23 Feb 2026 13:35:58 +0100 +Subject: [PATCH] db2: set reintegration when promotion is successful + +--- + heartbeat/db2 | 19 +++++++++++++++++++ + 1 file changed, 19 insertions(+) + +diff --git a/heartbeat/db2 b/heartbeat/db2 +index 82f2f82c3..4420b9989 100755 +--- a/heartbeat/db2 ++++ b/heartbeat/db2 +@@ -955,6 +955,16 @@ db2_promote() { + PRIMARY/PEER/*|PRIMARY/REMOTE_CATCHUP/*|PRIMARY/REMOTE_CATCHUP_PENDING/CONNECTED|Primary/Peer) + # nothing to do, only update pacemaker's view + echo MASTER > $STATE_FILE ++ ++ if [ -n "$remote_host" ]; then ++ for db in $dblist ++ do ++ reint_attr="db2hadr-${inst1}_${inst2}_${db}_reint" ++ ocf_log debug "Promotion succeeded, setting $reint_attr = 1" ++ crm_attribute -n "$reint_attr" -N "$remote_host" -v "1" -l forever ++ done ++ fi ++ + return $OCF_SUCCESS + ;; + +@@ -981,6 +991,15 @@ db2_promote() { + # update pacemaker's view + echo MASTER > $STATE_FILE + ++ if [ -n "$remote_host" ]; then ++ for db in $dblist ++ do ++ reint_attr="db2hadr-${inst1}_${inst2}_${db}_reint" ++ ocf_log debug "Promotion succeeded, setting $reint_attr = 1" ++ crm_attribute -n "$reint_attr" -N "$remote_host" -v "1" -l forever ++ done ++ fi ++ + return $OCF_SUCCESS + fi + diff --git a/SOURCES/RHEL-159202-podman-etcd-hardened-monitor-stop-actions.patch b/SOURCES/RHEL-159202-podman-etcd-hardened-monitor-stop-actions.patch new file mode 100644 index 0000000..3ef0d95 --- /dev/null +++ b/SOURCES/RHEL-159202-podman-etcd-hardened-monitor-stop-actions.patch @@ -0,0 +1,265 @@ +From c909003639ef36f995f855f5b954a5ae2132f19c Mon Sep 17 00:00:00 2001 +From: Vincenzo Mauro <43814449+vimauro@users.noreply.github.com> +Date: Mon, 23 Mar 2026 11:54:51 +0100 +Subject: [PATCH] OCPBUGS-76538: podman-etcd: monitor/stop hardening (#2130) + +* monitor/stop hardening +* removed noisy log +* Improved return code handling + PR comments +* reduced wait_timeout_sec in podman_start +* enriched log line on container_running +* Fixed detect_cluster_leadership_loss in case podman exec fails +* Reverted detect_cluster_leadership_loss changes +* Updated return code in attribute_node_revision +* restored initial attribute_node_revision logic and updated comment +* restored original log line in attribute_node_revision +* updated return codes on changed code path that reaches pacemaker +* updated log line in check_peer +--- + heartbeat/podman-etcd | 133 ++++++++++++++++++++++++++---------------- + 1 file changed, 83 insertions(+), 50 deletions(-) + +diff --git a/heartbeat/podman-etcd b/heartbeat/podman-etcd +index 860aca817..4c9bbd4fa 100755 +--- a/heartbeat/podman-etcd ++++ b/heartbeat/podman-etcd +@@ -371,25 +371,34 @@ monitor_cmd_exec() + { + local rc=$OCF_SUCCESS + local out +- +- out=$(podman exec ${CONTAINER} $OCF_RESKEY_monitor_cmd 2>&1) +- rc=$? +- # 125: no container with name or ID ${CONTAINER} found +- # 126: container state improper (not running) +- # 127: any other error +- # 255: podman 2+: container not running +- case "$rc" in +- 125|126|255) +- rc=$OCF_NOT_RUNNING +- ;; +- 0) +- ocf_log debug "monitor cmd passed: exit code = $rc" +- ;; +- *) +- ocf_exit_reason "monitor cmd failed (rc=$rc), output: $out" +- rc=$OCF_ERR_GENERIC +- ;; +- esac ++ local attempt ++ # 3 attempts × 5s = 15s worst case, fits within the 25s monitor timeout. ++ # The health check normally completes in <1s; the 5s per-attempt timeout ++ # is a safety net for when the container's process namespace is slow. ++ local max_attempts=3 ++ local attempt_timeout=5 ++ ++ for attempt in $(seq 1 $max_attempts); do ++ out=$(timeout $attempt_timeout podman exec ${CONTAINER} $OCF_RESKEY_monitor_cmd 2>&1) ++ rc=$? ++ # 125: no container with name or ID ${CONTAINER} found ++ # 126: container state improper (not running) ++ # 127: any other error ++ # 255: podman 2+: container not running ++ case "$rc" in ++ 125|126|255) ++ return $OCF_NOT_RUNNING ++ ;; ++ 0) ++ ocf_log debug "monitor cmd passed: exit code = $rc" ++ return $OCF_SUCCESS ++ ;; ++ *) ++ ocf_log warn "monitor cmd failed (rc=$rc), output: $out" ++ rc=$OCF_ERR_GENERIC ++ ;; ++ esac ++ done + + return $rc + } +@@ -527,8 +536,9 @@ get_env_from_manifest() { + exit "$OCF_ERR_INSTALLED" + fi + +- if ! env_var_value=$(jq -r ".spec.containers[].env[] | select( .name == \"$env_var_name\" ).value" "$OCF_RESKEY_pod_manifest"); then +- rc=$? ++ env_var_value=$(jq -r ".spec.containers[].env[] | select( .name == \"$env_var_name\" ).value" "$OCF_RESKEY_pod_manifest") ++ rc=$? ++ if [ $rc -ne 0 ]; then + ocf_log err "could not find environment variable $env_var_name in etcd pod manifest, error code: $rc" + exit "$OCF_ERR_INSTALLED" + fi +@@ -934,10 +944,14 @@ attribute_node_cluster_id() + { + local action="$1" + local value +- if ! value=$(jq -r ".clusterId" "$ETCD_REVISION_JSON"); then +- rc=$? ++ local rc ++ value=$(jq -r ".clusterId" "$ETCD_REVISION_JSON") ++ rc=$? ++ if [ $rc -ne 0 ]; then ++ # Log the error but return success to avoid monitor failure if the file is not available yet. ++ #This should not block cluster recovery. + ocf_log err "could not get cluster_id, error code: $rc" +- return "$rc" ++ return $OCF_SUCCESS + fi + + case "$action" in +@@ -945,10 +959,12 @@ attribute_node_cluster_id() + echo "$value" + ;; + update) +- if ! crm_attribute --type nodes --node "$NODENAME" --name "cluster_id" --update "$value"; then +- rc=$? ++ crm_attribute --type nodes --node "$NODENAME" --name "cluster_id" --update "$value" ++ rc=$? ++ if [ $rc -ne 0 ]; then ++ # Log the error but return success to avoid monitor failure if we can not update the attribute. + ocf_log err "could not update cluster_id, error code: $rc" +- return "$rc" ++ return $OCF_SUCCESS + fi + ;; + *) +@@ -983,10 +999,12 @@ attribute_node_revision() + echo "$value" + ;; + update) +- if ! crm_attribute --type nodes --node "$NODENAME" --name "$attribute" --update "$value"; then +- rc=$? +- ocf_log err "could not update etcd $revision, error code: $rc" +- return "$rc" ++ crm_attribute --type nodes --node "$NODENAME" --name "$attribute" --update "$value" ++ rc=$? ++ if [ $rc -ne 0 ]; then ++ # Log the error but return success to avoid monitor failure if we can not update the attribute. ++ ocf_log err "could not update etcd $attribute, error code: $rc" ++ return $OCF_SUCCESS + fi + ;; + *) +@@ -1041,25 +1059,31 @@ attribute_node_member_id() + ocf_log info "member list: $member_list_json" + if [ -z "$member_list_json" ] ; then + ocf_log err "could not get $attribute: could not get member list JSON" +- return "$rc" ++ return $OCF_ERR_GENERIC + fi + +- local value value_hex +- if ! value=$(echo -n "$member_list_json" | jq -r ".header.member_id"); then +- rc=$? ++ local value value_hex rc ++ value=$(echo -n "$member_list_json" | jq -r ".header.member_id") ++ rc=$? ++ if [ $rc -ne 0 ]; then ++ # Log the error but return success to avoid monitor failure if the file is not available yet. + ocf_log err "could not get $attribute from member list JSON, error code: $rc" +- return "$rc" ++ return $OCF_SUCCESS + fi + + # JSON member_id is decimal, while etcdctl command needs the hex version +- if ! value_hex=$(decimal_to_hex "$value"); then +- ocf_log err "could not convert decimal member_id '$value' to hex, error code: $?" ++ value_hex=$(decimal_to_hex "$value") ++ rc=$? ++ if [ $rc -ne 0 ]; then ++ ocf_log err "could not convert decimal member_id '$value' to hex, error code: $rc" + return $OCF_ERR_GENERIC + fi +- if ! crm_attribute --type nodes --node "$NODENAME" --name "$attribute" --update "$value_hex"; then +- rc=$? ++ crm_attribute --type nodes --node "$NODENAME" --name "$attribute" --update "$value_hex" ++ rc=$? ++ if [ $rc -ne 0 ]; then ++ # Log the error but return success to avoid monitor failure if we can not update the attribute. + ocf_log err "could not update etcd $attribute, error code: $rc" +- return "$rc" ++ return $OCF_SUCCESS + fi + ;; + clear) +@@ -1446,7 +1470,7 @@ get_endpoint_status_json() + local all_etcd_endpoints + + all_etcd_endpoints=$(get_all_etcd_endpoints) +- podman exec "${CONTAINER}" etcdctl endpoint status --endpoints="$all_etcd_endpoints" -w json ++ podman exec "${CONTAINER}" etcdctl endpoint status --command-timeout="$MONITOR_ETCDCTL_TIMEOUT" --endpoints="$all_etcd_endpoints" -w json + } + + get_member_list_json() { +@@ -1454,7 +1478,7 @@ get_member_list_json() { + local this_node_endpoint + + this_node_endpoint="$(ip_url $(attribute_node_ip get)):2379" +- podman exec "${CONTAINER}" etcdctl member list --endpoints="$this_node_endpoint" -w json ++ podman exec "${CONTAINER}" etcdctl member list --command-timeout="$MONITOR_ETCDCTL_TIMEOUT" --endpoints="$this_node_endpoint" -w json + } + + detect_cluster_leadership_loss() +@@ -1550,7 +1574,7 @@ check_peer() + fi + + if ! member_list_json=$(get_member_list_json); then +- ocf_log info "podman failed to get member list, error code: $?" ++ ocf_log info "podman failed to get member list" + detect_cluster_leadership_loss + return $? + fi +@@ -2145,7 +2169,7 @@ podman_start() + run_opts="$run_opts --oom-score-adj=${OCF_RESKEY_oom}" + + if ocf_is_true "$JOIN_AS_LEARNER"; then +- local wait_timeout_sec=$((10*60)) ++ local wait_timeout_sec=$((2*60)) + local poll_interval_sec=5 + local retries=$(( wait_timeout_sec / poll_interval_sec )) + +@@ -2354,8 +2378,9 @@ leave_etcd_member_list() + ocf_log info "leaving members list as member with ID $member_id" + local endpoint + endpoint="$(ip_url $(attribute_node_ip get)):2379" +- if ! ocf_run podman exec "$CONTAINER" etcdctl member remove "$member_id" --endpoints="$endpoint"; then +- rc=$? ++ ocf_run timeout 30 podman exec "$CONTAINER" etcdctl member remove "$member_id" --endpoints="$endpoint" ++ rc=$? ++ if [ $rc -ne 0 ]; then + ocf_log err "error leaving members list, error code: $rc" + fi + } +@@ -2376,14 +2401,19 @@ podman_stop() + attribute_node_revision update + attribute_node_cluster_id update + +- podman_simple_status +- if [ $? -eq $OCF_NOT_RUNNING ]; then +- ocf_log info "could not leave members list: etcd container not running" ++ # Use podman inspect instead of podman exec (podman_simple_status) to check ++ # container state. podman exec enters the container's process namespace and ++ # hangs when etcd is unresponsive — the typical scenario that triggers a stop. ++ local container_running ++ container_running=$(podman inspect --format '{{.State.Running}}' "$CONTAINER" 2>/dev/null) ++ if [ "$container_running" != "true" ]; then ++ ocf_log info "could not leave members list: $CONTAINER container not running, running state: ${container_running}" + attribute_node_member_id clear + return $OCF_SUCCESS + fi + + leave_etcd_member_list ++ + # clear node_member_id CIB attribute only after leaving the member list + attribute_node_member_id clear + +@@ -2527,6 +2557,9 @@ ETCD_CERTS_HASH_FILE="${OCF_RESKEY_config_location}/certs.hash" + # This is intentional - reboots are controlled stops, not failures requiring detection. + CONTAINER_HEARTBEAT_FILE=${HA_RSCTMP}/podman-container-last-running + DELAY_SECOND_NODE_LEAVE_SEC=10 ++# Shorter etcdctl command-timeout for monitor-path calls to prevent ++# consuming the 25s monitor budget. Non-monitor callers use the default 5s. ++MONITOR_ETCDCTL_TIMEOUT="3s" + + # Note: we currently monitor podman containers by with the "podman exec" + # command, so make sure that invocation is always valid by enforcing the diff --git a/SOURCES/RHEL-166183-1-db2-fix-bashism.patch b/SOURCES/RHEL-166183-1-db2-fix-bashism.patch new file mode 100644 index 0000000..e2167a2 --- /dev/null +++ b/SOURCES/RHEL-166183-1-db2-fix-bashism.patch @@ -0,0 +1,32 @@ +From 3712b1f52bccddc767ad6f16ec67d6c8c29f1f71 Mon Sep 17 00:00:00 2001 +From: Valentin Vidic +Date: Sun, 3 Apr 2022 20:39:01 +0200 +Subject: [PATCH] db2: fix bashism + +dash only allows -a as AND operator. +--- + heartbeat/db2 | 4 ++-- + 1 file changed, 2 insertions(+), 2 deletions(-) + +diff --git a/heartbeat/db2 b/heartbeat/db2 +index ea24d33fc8..4a4b2f477f 100755 +--- a/heartbeat/db2 ++++ b/heartbeat/db2 +@@ -407,7 +407,7 @@ db2_start() { + # partition is explicitly specified, activate without + # partition information. This allows db2 instances without + # partition support to be managed. +- if [ -z "$OCF_RESKEY_dbpartitionnum" ] && ! [ -a "$db2sql/db2nodes.cfg" ]; then ++ if [ -z "$OCF_RESKEY_dbpartitionnum" ] && ! [ -e "$db2sql/db2nodes.cfg" ]; then + start_opts="" + fi + +@@ -511,7 +511,7 @@ db2_stop_bg() { + + rc=$OCF_SUCCESS + +- if [ -z "$OCF_RESKEY_dbpartitionnum" ] && ! [ -a "$db2sql/db2nodes.cfg" ]; then ++ if [ -z "$OCF_RESKEY_dbpartitionnum" ] && ! [ -e "$db2sql/db2nodes.cfg" ]; then + stop_opts="" + fi + diff --git a/SOURCES/RHEL-166183-2-db2-do-not-use-db2stop-to-avoid-divergence-in-the-log.patch b/SOURCES/RHEL-166183-2-db2-do-not-use-db2stop-to-avoid-divergence-in-the-log.patch new file mode 100644 index 0000000..0b0e0fa --- /dev/null +++ b/SOURCES/RHEL-166183-2-db2-do-not-use-db2stop-to-avoid-divergence-in-the-log.patch @@ -0,0 +1,143 @@ +From 26c0d48bc69da1859f1ce5205a8bb6eaf6297b81 Mon Sep 17 00:00:00 2001 +From: Oyvind Albrigtsen +Date: Wed, 25 Mar 2026 10:46:09 +0100 +Subject: [PATCH] db2: do not use db2stop, as it sends truncation messages, + which in some cases are not delivered + +This caused divergence in the log, and the user would have to manually rebuild +the DB to recover from it. +--- + heartbeat/db2 | 104 +++++++------------------------------------------- + 1 file changed, 13 insertions(+), 91 deletions(-) + +diff --git a/heartbeat/db2 b/heartbeat/db2 +index 4420b9989..9de18639d 100755 +--- a/heartbeat/db2 ++++ b/heartbeat/db2 +@@ -596,45 +596,10 @@ db2_start() { + return $OCF_SUCCESS + } + +-# +-# helper function to be spawned +-# so we can detect a hang of the db2stop command +-# +-db2_stop_bg() { +- local rc output +- local stop_opts="dbpartitionnum $db2node" +- +- rc=$OCF_SUCCESS +- +- if [ -z "$OCF_RESKEY_dbpartitionnum" ] && ! [ -e "$db2sql/db2nodes.cfg" ]; then +- stop_opts="" +- fi +- +- if output=$(runasdb2 db2stop force $stop_opts) +- then +- ocf_log info "DB2 instance $instance($db2node) stopped: $output" +- else +- case $output in +- *SQL1032N*) +- #SQL1032N No start database manager command was issued +- ocf_log info "$output" +- ;; +- +- *) +- ocf_log err "DB2 instance $instance($db2node) stop failed: $output" +- rc=$OCF_ERR_GENERIC +- esac +- fi +- +- return $rc +-} +- + # + # Stop the given db2 database instance + # + db2_stop() { +- local stop_timeout grace_timeout stop_bg_pid i must_kill +- + # remove master score + master_score -D -l reboot + +@@ -647,67 +612,24 @@ db2_stop() { + return $OCF_SUCCESS + fi + +- stop_timeout=${OCF_RESKEY_CRM_meta_timeout:-20000} +- +- # grace_time is 4/5 (unit is ms) +- grace_timeout=$((stop_timeout/1250)) +- +- # start db2stop in background as this may hang +- db2_stop_bg & +- stop_bg_pid=$! +- +- # wait for grace_timeout +- i=0 +- while [ $i -lt $grace_timeout ] +- do +- kill -0 $stop_bg_pid 2>/dev/null || break; +- sleep 1 +- i=$((i+1)) +- done +- +- # collect exit status but don't hang +- if kill -0 $stop_bg_pid 2>/dev/null +- then +- stoprc=1 +- kill -9 $stop_bg_pid 2>/dev/null +- else +- wait $stop_bg_pid +- stoprc=$? +- fi +- +- must_kill=0 +- +- if [ $stoprc -ne 0 ] ++ # db2nkill kills *all* partitions on the node ++ if [ -x $db2bin/db2nkill ] + then +- ocf_log warn "DB2 instance $instance($db2node): db2stop failed, using db2nkill" +- must_kill=1 +- elif ! db2_instance_dead ++ logasdb2 $db2bin/db2nkill $db2node ++ elif [ -x $db2bin/db2_kill ] + then +- ocf_log warn "DB2 instance $instance($db2node): db2stop indicated success but there a still processes, using db2nkill" +- must_kill=1 ++ logasdb2 $db2bin/db2_kill + fi + +- if [ $must_kill -eq 1 ] +- then +- # db2nkill kills *all* partitions on the node +- if [ -x $db2bin/db2nkill ] +- then +- logasdb2 $db2bin/db2nkill $db2node +- elif [ -x $db2bin/db2_kill ] +- then +- logasdb2 $db2bin/db2_kill +- fi +- +- # loop forever (or lrmd kills us due to timeout) until the +- # instance is dead +- while ! db2_instance_dead +- do +- ocf_log info "DB2 instance $instance($db2node): waiting for processes to exit" +- sleep 1 +- done ++ # loop forever (or lrmd kills us due to timeout) until the ++ # instance is dead ++ while ! db2_instance_dead ++ do ++ ocf_log info "DB2 instance $instance($db2node): waiting for processes to exit" ++ sleep 1 ++ done + +- ocf_log info "DB2 instance $instance($db2node) is now dead" +- fi ++ ocf_log info "DB2 instance $instance($db2node) is now dead" + + return $OCF_SUCCESS + } diff --git a/SOURCES/RHEL-177838-podman-etcd-fix-machine-deletion-deadlock.patch b/SOURCES/RHEL-177838-podman-etcd-fix-machine-deletion-deadlock.patch new file mode 100644 index 0000000..b1a8fcb --- /dev/null +++ b/SOURCES/RHEL-177838-podman-etcd-fix-machine-deletion-deadlock.patch @@ -0,0 +1,283 @@ +From 9ba19a62543de4d7365fc711b908a2759f811af9 Mon Sep 17 00:00:00 2001 +From: Vincenzo Mauro +Date: Tue, 5 May 2026 14:24:43 +0200 +Subject: [PATCH 1/4] fix: fixed etcd learner deadlock + +--- + heartbeat/podman-etcd | 79 +++++++++++++++++++++++++++++++++++++++---- + 1 file changed, 73 insertions(+), 6 deletions(-) + +diff --git a/heartbeat/podman-etcd b/heartbeat/podman-etcd +index 4c9bbd4fa..5bb3b2897 100755 +--- a/heartbeat/podman-etcd ++++ b/heartbeat/podman-etcd +@@ -50,6 +50,7 @@ OCF_RESKEY_oom_default="-997" + OCF_RESKEY_config_location_default="/var/lib/etcd" + OCF_RESKEY_backup_location_default="/var/lib/etcd" + OCF_RESKEY_max_backup_snapshots_default="3" ++OCF_RESKEY_kubeconfig_default="/etc/kubernetes/static-pod-resources/kube-apiserver-certs/secrets/node-kubeconfigs/localhost.kubeconfig" + + : ${OCF_RESKEY_image=${OCF_RESKEY_image_default}} + : ${OCF_RESKEY_pod_manifest=${OCF_RESKEY_pod_manifest_default}} +@@ -63,6 +64,7 @@ OCF_RESKEY_max_backup_snapshots_default="3" + : ${OCF_RESKEY_config_location=${OCF_RESKEY_config_location_default}} + : ${OCF_RESKEY_backup_location=${OCF_RESKEY_backup_location_default}} + : ${OCF_RESKEY_max_backup_snapshots=${OCF_RESKEY_max_backup_snapshots_default}} ++: ${OCF_RESKEY_kubeconfig=${OCF_RESKEY_kubeconfig_default}} + + + ####################################################################### +@@ -288,6 +290,16 @@ Set max_backup_snapshots=0 to disable backups. + + + ++ ++ ++Path to a kubeconfig file for querying Machine API objects. Used to detect ++whether a peer node's Machine is being deleted, preventing the resource agent ++from re-adding it as an etcd learner during Machine deletion flows. ++ ++Kubeconfig for Machine API queries ++ ++ ++ + + + +@@ -1505,6 +1517,34 @@ detect_cluster_leadership_loss() + } + + ++# Checks whether the Machine object for a given node is being deleted. ++# Returns 0 (true) if the Machine has a deletionTimestamp set, 1 (false) otherwise. ++# Fails open: returns 1 on API errors to preserve current learner-addition behavior. ++is_peer_machine_deleting() ++{ ++ local node_name="$1" ++ local out ++ local deletion_ts ++ ++ out=$(timeout 10 oc --kubeconfig="$OCF_RESKEY_kubeconfig" get machines \ ++ -n openshift-machine-api -o json 2>&1) ++ if [ $? -ne 0 ]; then ++ ocf_log warn "could not query Machine API for node $node_name (fail-open): $out" ++ return 1 ++ fi ++ ++ # Select the Machine object for the given node and extract its deletionTimestamp if present ++ deletion_ts=$(printf "%s" "$out" | jq -r --arg name "$node_name" \ ++ '.items[] | select(.status.nodeRef.name == $name) | .metadata.deletionTimestamp // empty') ++ ++ if [ -n "$deletion_ts" ]; then ++ ocf_log info "Machine for node $node_name is being deleted (deletionTimestamp: $deletion_ts)" ++ return 0 ++ fi ++ ++ return 1 ++} ++ + # Manages etcd peer membership by detecting and handling missing or rejoining peers + # Adds missing peers as learners and reconciles member states when peers rejoin + # Args: $1 - member list JSON from etcdctl +@@ -1542,9 +1582,21 @@ manage_peer_membership() + # NOTE: voting members have a "name" field but no "isLearner" field, + # while learner members have "isLearner": true (boolean) but no "name" field, so we search for peerURLs matching. + peer_member_id=$(printf "%s" "$member_list_json" | jq -r ".members[] | select( .peerURLs | map(test(\"$peer_member_ip\")) | any).ID") ++ # During Machine deletion, CEO's MachineDeletionHooksController ++ # keeps the EtcdQuorumOperator preDrain hook as long as the peer IP appears in the etcd ++ # member list (learners included). If we add or keep a learner for a peer whose Machine ++ # is being deleted, CEO never clears the hook, MAO never drains, and the Machine hangs ++ # in Deleting. Two safeguards cover the race: ++ # A (below): peer is not yet in the member list — skip adding it as a learner if machine is deleting ++ # B (learner exists): a prior monitor cycle added the learner before the Machine ++ # deletion started — remove it so CEO can clear the hook. + if [ -z "$peer_member_id" ]; then + ocf_log info "$peer_member_name is not in the members list" +- add_member_as_learner "$peer_member_name" "$peer_member_ip" ++ if ! is_peer_machine_deleting "$peer_member_name"; then ++ add_member_as_learner "$peer_member_name" "$peer_member_ip" ++ else ++ ocf_log info "peer Machine is being deleted, skipping learner addition for $peer_member_name" ++ fi + set_standalone_node + return + fi +@@ -1552,10 +1604,21 @@ manage_peer_membership() + # Ensure learner_node attribute is always set when we have a learner member + local learner_member_id=$(printf "%s" "$member_list_json" | jq -r ".members[] | select( .isLearner==true ).ID") + local current_learner_node=$(attribute_learner_node get) +- if [ -n "$learner_member_id" ] && [ -z "$current_learner_node" ]; then +- ocf_log debug "$peer_member_name found as learner in member list, but learner_node attribute was not set. Updating" +- attribute_learner_node update "$peer_member_name" +- return ++ ++ if [ -n "$learner_member_id" ]; then ++ # Clean up a learner added before the Machine deletion started ++ if is_peer_machine_deleting "$peer_member_name"; then ++ ocf_log info "peer Machine is being deleted, removing learner $peer_member_name from member list" ++ remove_etcd_member_by_ip "$peer_member_ip" ++ attribute_learner_node clear ++ set_standalone_node ++ return ++ fi ++ if [ -z "$current_learner_node" ]; then ++ ocf_log debug "$peer_member_name found as learner in member list, but learner_node attribute was not set. Updating" ++ attribute_learner_node update "$peer_member_name" ++ return ++ fi + fi + + ocf_log debug "$peer_member_name is in the members list by IP: $peer_member_ip" +@@ -2312,7 +2375,11 @@ podman_start() + peer_node_name="$(get_peer_node_name)" + peer_node_ip="$(attribute_node_ip_peer)" + if [ -n "$peer_node_name" ] && [ -n "$peer_node_ip" ]; then +- add_member_as_learner "$peer_node_name" "$peer_node_ip" ++ if is_peer_machine_deleting "$peer_node_name"; then ++ ocf_log info "peer Machine is being deleted, skipping learner addition for $peer_node_name" ++ else ++ add_member_as_learner "$peer_node_name" "$peer_node_ip" ++ fi + set_standalone_node + else + ocf_log err "could not add peer as learner (peer node name: ${peer_node_name:-unknown}, peer ip: ${peer_node_ip:-unknown})" + +From 56d9754311ab0595dea1c47e26eca85bbcfb049c Mon Sep 17 00:00:00 2001 +From: Vincenzo Mauro +Date: Wed, 6 May 2026 15:15:27 +0200 +Subject: [PATCH 2/4] fix: added support for both MAPI and CAPI + +--- + heartbeat/podman-etcd | 42 +++++++++++++++++++++++++++++++++++------- + 1 file changed, 35 insertions(+), 7 deletions(-) + +diff --git a/heartbeat/podman-etcd b/heartbeat/podman-etcd +index 5bb3b2897..ad9804c1d 100755 +--- a/heartbeat/podman-etcd ++++ b/heartbeat/podman-etcd +@@ -295,6 +295,7 @@ Set max_backup_snapshots=0 to disable backups. + Path to a kubeconfig file for querying Machine API objects. Used to detect + whether a peer node's Machine is being deleted, preventing the resource agent + from re-adding it as an etcd learner during Machine deletion flows. ++Supports both MAPI (machine.openshift.io) and CAPI (cluster.x-k8s.io) Machine resources. + + Kubeconfig for Machine API queries + +@@ -1525,15 +1526,39 @@ is_peer_machine_deleting() + local node_name="$1" + local out + local deletion_ts ++ local oc_rc ++ local item_count + +- out=$(timeout 10 oc --kubeconfig="$OCF_RESKEY_kubeconfig" get machines \ ++ # Try MAPI first (machine.openshift.io), fall back to CAPI (cluster.x-k8s.io) ++ out=$(timeout 10 oc --kubeconfig="$OCF_RESKEY_kubeconfig" get machines.machine.openshift.io \ + -n openshift-machine-api -o json 2>&1) +- if [ $? -ne 0 ]; then +- ocf_log warn "could not query Machine API for node $node_name (fail-open): $out" +- return 1 ++ oc_rc=$? ++ ++ if [ $oc_rc -eq 0 ]; then ++ item_count=$(printf "%s" "$out" | jq '.items | length' 2>/dev/null) ++ fi ++ ++ # MAPI CRD missing, namespace absent, or no Machine objects — try CAPI ++ if [ $oc_rc -ne 0 ] || [ "${item_count:-0}" -eq 0 ]; then ++ ocf_log info "MAPI returned no machines, trying CAPI for node $node_name" ++ out=$(timeout 10 oc --kubeconfig="$OCF_RESKEY_kubeconfig" get machines.cluster.x-k8s.io \ ++ -n openshift-cluster-api -o json 2>&1) ++ if [ $? -ne 0 ]; then ++ ocf_log warn "could not query Machine API (MAPI or CAPI) for node $node_name (fail-open): $out" ++ return 1 ++ fi + fi + + # Select the Machine object for the given node and extract its deletionTimestamp if present ++ local machine_count ++ machine_count=$(printf "%s" "$out" | jq -r --arg name "$node_name" \ ++ '[.items[] | select(.status.nodeRef.name == $name)] | length' 2>/dev/null) ++ ++ if [ "$machine_count" = "0" ] || [ -z "$machine_count" ]; then ++ ocf_log warn "No Machine object found for node $node_name (fail-open): nodeRef may not be populated yet" ++ return 1 ++ fi ++ + deletion_ts=$(printf "%s" "$out" | jq -r --arg name "$node_name" \ + '.items[] | select(.status.nodeRef.name == $name) | .metadata.deletionTimestamp // empty') + +@@ -1609,9 +1634,12 @@ manage_peer_membership() + # Clean up a learner added before the Machine deletion started + if is_peer_machine_deleting "$peer_member_name"; then + ocf_log info "peer Machine is being deleted, removing learner $peer_member_name from member list" +- remove_etcd_member_by_ip "$peer_member_ip" +- attribute_learner_node clear +- set_standalone_node ++ if remove_etcd_member_by_ip "$peer_member_ip"; then ++ attribute_learner_node clear ++ set_standalone_node ++ else ++ ocf_log err "failed to remove learner for deleting Machine $peer_member_name; will retry next monitor cycle" ++ fi + return + fi + if [ -z "$current_learner_node" ]; then + +From beab70c7acd4f6ccc33c4dbcb3d72f94fc560812 Mon Sep 17 00:00:00 2001 +From: Vincenzo Mauro +Date: Tue, 12 May 2026 09:54:33 +0200 +Subject: [PATCH 3/4] reduced timeout to 5 and fixed MCAPI return code + +--- + heartbeat/podman-etcd | 3 ++- + 1 file changed, 2 insertions(+), 1 deletion(-) + +diff --git a/heartbeat/podman-etcd b/heartbeat/podman-etcd +index ad9804c1d..e022869a8 100755 +--- a/heartbeat/podman-etcd ++++ b/heartbeat/podman-etcd +@@ -1530,7 +1530,7 @@ is_peer_machine_deleting() + local item_count + + # Try MAPI first (machine.openshift.io), fall back to CAPI (cluster.x-k8s.io) +- out=$(timeout 10 oc --kubeconfig="$OCF_RESKEY_kubeconfig" get machines.machine.openshift.io \ ++ out=$(timeout 5 oc --kubeconfig="$OCF_RESKEY_kubeconfig" get machines.machine.openshift.io \ + -n openshift-machine-api -o json 2>&1) + oc_rc=$? + +@@ -1607,6 +1607,7 @@ manage_peer_membership() + # NOTE: voting members have a "name" field but no "isLearner" field, + # while learner members have "isLearner": true (boolean) but no "name" field, so we search for peerURLs matching. + peer_member_id=$(printf "%s" "$member_list_json" | jq -r ".members[] | select( .peerURLs | map(test(\"$peer_member_ip\")) | any).ID") ++ + # During Machine deletion, CEO's MachineDeletionHooksController + # keeps the EtcdQuorumOperator preDrain hook as long as the peer IP appears in the etcd + # member list (learners included). If we add or keep a learner for a peer whose Machine + +From 2b06ed31bda015543a365e02e1bc5a47b3fa0439 Mon Sep 17 00:00:00 2001 +From: Vincenzo Mauro +Date: Tue, 12 May 2026 10:30:48 +0200 +Subject: [PATCH 4/4] fixed return code for CAPI + +--- + heartbeat/podman-etcd | 5 +++-- + 1 file changed, 3 insertions(+), 2 deletions(-) + +diff --git a/heartbeat/podman-etcd b/heartbeat/podman-etcd +index e022869a8..2dbaf9991 100755 +--- a/heartbeat/podman-etcd ++++ b/heartbeat/podman-etcd +@@ -1541,9 +1541,10 @@ is_peer_machine_deleting() + # MAPI CRD missing, namespace absent, or no Machine objects — try CAPI + if [ $oc_rc -ne 0 ] || [ "${item_count:-0}" -eq 0 ]; then + ocf_log info "MAPI returned no machines, trying CAPI for node $node_name" +- out=$(timeout 10 oc --kubeconfig="$OCF_RESKEY_kubeconfig" get machines.cluster.x-k8s.io \ ++ out=$(timeout 5 oc --kubeconfig="$OCF_RESKEY_kubeconfig" get machines.cluster.x-k8s.io \ + -n openshift-cluster-api -o json 2>&1) +- if [ $? -ne 0 ]; then ++ oc_rc=$? ++ if [ $oc_rc -ne 0 ]; then + ocf_log warn "could not query Machine API (MAPI or CAPI) for node $node_name (fail-open): $out" + return 1 + fi diff --git a/SOURCES/RHEL-177843-podman-etcd-fix-learner-start-deadlock.patch b/SOURCES/RHEL-177843-podman-etcd-fix-learner-start-deadlock.patch new file mode 100644 index 0000000..7b6cd6f --- /dev/null +++ b/SOURCES/RHEL-177843-podman-etcd-fix-learner-start-deadlock.patch @@ -0,0 +1,130 @@ +From db041869f4b8612e44561f4ba4a46ed09d18e24e Mon Sep 17 00:00:00 2001 +From: Vincenzo Mauro +Date: Thu, 7 May 2026 18:14:04 +0200 +Subject: [PATCH 1/4] fixed OCPBUGS-83333 + +--- + heartbeat/podman-etcd | 22 +++++++++++++++++++++- + 1 file changed, 21 insertions(+), 1 deletion(-) + +diff --git a/heartbeat/podman-etcd b/heartbeat/podman-etcd +index 4c9bbd4fa..d96c055e3 100755 +--- a/heartbeat/podman-etcd ++++ b/heartbeat/podman-etcd +@@ -2519,7 +2519,27 @@ podman_validate() + + podman_notify() + { +- ocf_log info "notify: type=${OCF_RESKEY_CRM_meta_notify_type}, operation=${OCF_RESKEY_CRM_meta_notify_operation}, nodes { active=[${OCF_RESKEY_CRM_meta_notify_active_uname}], start=[${OCF_RESKEY_CRM_meta_notify_start_uname}], stop=[${OCF_RESKEY_CRM_meta_notify_stop_uname}] }, resources { active=[${OCF_RESKEY_CRM_meta_notify_active_resource}], start =[${OCF_RESKEY_CRM_meta_notify_start_resource}], stop=[${OCF_RESKEY_CRM_meta_notify_stop_resource}] }" ++ local notify_type="${OCF_RESKEY_CRM_meta_notify_type}" ++ local notify_operation="${OCF_RESKEY_CRM_meta_notify_operation}" ++ ++ ocf_log info "notify: type=${notify_type}, operation=${notify_operation}, nodes { active=[${OCF_RESKEY_CRM_meta_notify_active_uname}], start=[${OCF_RESKEY_CRM_meta_notify_start_uname}], stop=[${OCF_RESKEY_CRM_meta_notify_stop_uname}] }, resources { active=[${OCF_RESKEY_CRM_meta_notify_active_resource}], start =[${OCF_RESKEY_CRM_meta_notify_start_resource}], stop=[${OCF_RESKEY_CRM_meta_notify_stop_resource}] }" ++ ++ # Pacemaker serializes operations per resource per node. The start sequence ++ # with notifications is: ++ # pre-notify(start) on peer → start on joiner → post-notify(start) on peer ++ # Between pre-notify and post-notify, the peer's recurring monitor is ++ # queued — Pacemaker won't overlap operations for the same resource on the ++ # same node. The monitor path (check_peer → manage_peer_membership → ++ # add_member_as_learner) is the primary way a running peer adds the ++ # starting node to the etcd member list. Without handling it here, the ++ # starting node's podman_start poll loop (waiting for learner_node attribute) ++ # deadlocks: start waits for learner_node, monitor waits for start to finish. ++ # pre_notify_start fires before the start action, giving us the window to ++ # add the learner so the joiner's poll loop finds it immediately. ++ if [ "$notify_type" = "pre" ] && [ "$notify_operation" = "start" ]; then ++ ocf_log info "pre_notify_start: running peer membership check for starting node" ++ check_peer ++ fi + } + + # TODO : + +From d1c817108276ee3019a20164fca0646985d99cde Mon Sep 17 00:00:00 2001 +From: Vincenzo Mauro +Date: Fri, 8 May 2026 10:38:12 +0200 +Subject: [PATCH 2/4] Updated deadlock comment + +--- + heartbeat/podman-etcd | 18 ++++++------------ + 1 file changed, 6 insertions(+), 12 deletions(-) + +diff --git a/heartbeat/podman-etcd b/heartbeat/podman-etcd +index d96c055e3..21a5e01e1 100755 +--- a/heartbeat/podman-etcd ++++ b/heartbeat/podman-etcd +@@ -2524,18 +2524,12 @@ podman_notify() + + ocf_log info "notify: type=${notify_type}, operation=${notify_operation}, nodes { active=[${OCF_RESKEY_CRM_meta_notify_active_uname}], start=[${OCF_RESKEY_CRM_meta_notify_start_uname}], stop=[${OCF_RESKEY_CRM_meta_notify_stop_uname}] }, resources { active=[${OCF_RESKEY_CRM_meta_notify_active_resource}], start =[${OCF_RESKEY_CRM_meta_notify_start_resource}], stop=[${OCF_RESKEY_CRM_meta_notify_stop_resource}] }" + +- # Pacemaker serializes operations per resource per node. The start sequence +- # with notifications is: +- # pre-notify(start) on peer → start on joiner → post-notify(start) on peer +- # Between pre-notify and post-notify, the peer's recurring monitor is +- # queued — Pacemaker won't overlap operations for the same resource on the +- # same node. The monitor path (check_peer → manage_peer_membership → +- # add_member_as_learner) is the primary way a running peer adds the +- # starting node to the etcd member list. Without handling it here, the +- # starting node's podman_start poll loop (waiting for learner_node attribute) +- # deadlocks: start waits for learner_node, monitor waits for start to finish. +- # pre_notify_start fires before the start action, giving us the window to +- # add the learner so the joiner's poll loop finds it immediately. ++ # Pacemaker suppresses the peer's monitor during an ++ # active start/notify cycle. Since monitor is the only path that calls ++ # add_member_as_learner (outside force_new_cluster), the joiner's ++ # podman_start poll loop deadlocks; it waits for learner_node, but ++ # no monitor runs to set it. pre_notify_start fires before start, ++ # so we add the learner here to break the deadlock + if [ "$notify_type" = "pre" ] && [ "$notify_operation" = "start" ]; then + ocf_log info "pre_notify_start: running peer membership check for starting node" + check_peer + +From 0fafab701878ce4b8c7413610e41dce3e69447aa Mon Sep 17 00:00:00 2001 +From: Vincenzo Mauro +Date: Tue, 12 May 2026 10:24:26 +0200 +Subject: [PATCH 3/4] improved logging + +--- + heartbeat/podman-etcd | 4 ++-- + 1 file changed, 2 insertions(+), 2 deletions(-) + +diff --git a/heartbeat/podman-etcd b/heartbeat/podman-etcd +index 21a5e01e1..41ce84ff1 100755 +--- a/heartbeat/podman-etcd ++++ b/heartbeat/podman-etcd +@@ -2531,7 +2531,7 @@ podman_notify() + # no monitor runs to set it. pre_notify_start fires before start, + # so we add the learner here to break the deadlock + if [ "$notify_type" = "pre" ] && [ "$notify_operation" = "start" ]; then +- ocf_log info "pre_notify_start: running peer membership check for starting node" ++ ocf_log info "pre_notify_start: running peer membership check for ${OCF_RESKEY_CRM_meta_notify_start_uname}" + check_peer + fi + } +@@ -2616,4 +2616,4 @@ validate-all) podman_validate;; + esac + rc=$? + ocf_log debug "${OCF_RESOURCE_INSTANCE} $__OCF_ACTION : $rc" +-exit $rc ++exc + +From bdce9048b4fc2c38255d36e73a1f73a7d72b7471 Mon Sep 17 00:00:00 2001 +From: Vincenzo Mauro +Date: Tue, 12 May 2026 10:46:49 +0200 +Subject: [PATCH 4/4] fixed typo + +--- + heartbeat/podman-etcd | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/heartbeat/podman-etcd b/heartbeat/podman-etcd +index 41ce84ff1..740e2edb4 100755 +--- a/heartbeat/podman-etcd ++++ b/heartbeat/podman-etcd +@@ -2616,4 +2616,4 @@ validate-all) podman_validate;; + esac + rc=$? + ocf_log debug "${OCF_RESOURCE_INSTANCE} $__OCF_ACTION : $rc" +-exc ++exit $rc diff --git a/SOURCES/RHEL-177849-podman-etcd-fix-port-2380-binding-race.patch b/SOURCES/RHEL-177849-podman-etcd-fix-port-2380-binding-race.patch new file mode 100644 index 0000000..e7b3d29 --- /dev/null +++ b/SOURCES/RHEL-177849-podman-etcd-fix-port-2380-binding-race.patch @@ -0,0 +1,96 @@ +From 42dfef941ed80d6073022141fa1cad513e8dae4f Mon Sep 17 00:00:00 2001 +From: Pablo Fontanilla +Date: Wed, 22 Apr 2026 12:54:58 +0200 +Subject: [PATCH 1/2] fix(podman-etcd): use -ge 1 in + etcd_pod_container_exists() + +PR #2112 added -a to crictl ps to include exited containers, but +did not update the count check from -eq 1 to -ge 1. During install, +etcd container crashes create exited containers that inflate the +count past 1, causing the guard to report 'pod not found' despite +the pod running. + +Fixes: OCPBUGS-83742 + +Co-Authored-By: Claude Opus 4.6 +--- + heartbeat/podman-etcd | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/heartbeat/podman-etcd b/heartbeat/podman-etcd +index 4c9bbd4fa..52b2a1386 100755 +--- a/heartbeat/podman-etcd ++++ b/heartbeat/podman-etcd +@@ -932,7 +932,7 @@ etcd_pod_container_exists() { + local count_matches + # Check whether the etcd pod exists on the same node (including stopped/exited containers) + count_matches=$(crictl pods --label app=etcd -q | xargs -I {} crictl ps -a --pod {} -o json | jq -r '.containers[].metadata | select ( .name == "etcd" ).name' | wc -l) +- if [ "$count_matches" -eq 1 ]; then ++ if [ "$count_matches" -ge 1 ]; then + # etcd pod found + return 0 + fi + +From 30d20f6b99ae9898bf801c0a5e690b81fc928faa Mon Sep 17 00:00:00 2001 +From: Pablo Fontanilla +Date: Wed, 22 Apr 2026 12:57:46 +0200 +Subject: [PATCH 2/2] fix(podman-etcd): wait for etcd ports before starting + container + +During the static-pod to podman-etcd transition, the old etcd process +may still hold ports 2379/2380 when the RA tries to start its container. +This causes 'bind: address already in use' errors and eventual fallback +to standalone mode. + +Add a 60-second wait loop (modeled on CEO's pod.gotpl.yaml port check) +that blocks until the ports are free before calling podman run/start. + +Fixes: OCPBUGS-83742 + +Co-Authored-By: Claude Opus 4.6 +--- + heartbeat/podman-etcd | 24 ++++++++++++++++++++++++ + 1 file changed, 24 insertions(+) + +diff --git a/heartbeat/podman-etcd b/heartbeat/podman-etcd +index 52b2a1386..9a960914b 100755 +--- a/heartbeat/podman-etcd ++++ b/heartbeat/podman-etcd +@@ -940,6 +940,25 @@ etcd_pod_container_exists() { + return 1 + } + ++wait_for_etcd_ports_release() { ++ local timeout=${1:-60} ++ local elapsed=0 ++ if [ -z "$(ss -Htan '( sport = 2379 or sport = 2380 )')" ]; then ++ return 0 ++ fi ++ ocf_log info "waiting for etcd ports 2379/2380 to be released (timeout: ${timeout}s)" ++ while [ -n "$(ss -Htan '( sport = 2379 or sport = 2380 )')" ]; do ++ if [ "$elapsed" -ge "$timeout" ]; then ++ ocf_log err "etcd ports still in use after ${timeout}s" ++ return 1 ++ fi ++ sleep 1 ++ elapsed=$((elapsed + 1)) ++ done ++ ocf_log info "etcd ports released after ${elapsed}s" ++ return 0 ++} ++ + attribute_node_cluster_id() + { + local action="$1" +@@ -2267,6 +2286,11 @@ podman_start() + ocf_log notice "Pull image not required, ${OCF_RESKEY_image}" + fi + ++ if ! wait_for_etcd_ports_release 60; then ++ ocf_exit_reason "etcd ports 2379/2380 still bound — cannot start container" ++ return $OCF_ERR_GENERIC ++ fi ++ + if ocf_is_true "$OCF_RESKEY_reuse" && container_exists; then + ocf_log info "starting existing container $CONTAINER." + ocf_run podman start "$CONTAINER" diff --git a/SOURCES/RHEL-64949-oracle-improve-monpassword-description.patch b/SOURCES/RHEL-64949-oracle-improve-monpassword-description.patch old mode 100644 new mode 100755 diff --git a/SOURCES/ha-cloud-support-gcloud.patch b/SOURCES/ha-cloud-support-gcloud.patch index 95b0d7a..f998bfc 100644 --- a/SOURCES/ha-cloud-support-gcloud.patch +++ b/SOURCES/ha-cloud-support-gcloud.patch @@ -5,7 +5,7 @@ diff --color -uNr a/heartbeat/gcp-pd-move.in b/heartbeat/gcp-pd-move.in from ocf import logger try: -+ sys.path.insert(0, '/usr/lib/fence-agents/support/google') ++ sys.path.insert(0, '/usr/lib/fence-agents/support/google/lib/python#PYTHON3_VERSION#/site-packages') import googleapiclient.discovery except ImportError: pass @@ -16,7 +16,7 @@ diff --color -uNr a/heartbeat/gcp-vpc-move-route.in b/heartbeat/gcp-vpc-move-rou from ocf import * try: -+ sys.path.insert(0, '/usr/lib/fence-agents/support/google') ++ sys.path.insert(0, '/usr/lib/fence-agents/support/google/lib/python#PYTHON3_VERSION#/site-packages') import googleapiclient.discovery import pyroute2 try: @@ -27,7 +27,7 @@ diff --color -uNr a/heartbeat/gcp-vpc-move-vip.in b/heartbeat/gcp-vpc-move-vip.i from ocf import * try: -+ sys.path.insert(0, '/usr/lib/fence-agents/support/google') ++ sys.path.insert(0, '/usr/lib/fence-agents/support/google/lib/python#PYTHON3_VERSION#/site-packages') import googleapiclient.discovery try: from google.oauth2.service_account import Credentials as ServiceAccountCredentials diff --git a/SOURCES/ha-cloud-support-ibm.patch b/SOURCES/ha-cloud-support-ibm.patch index 7b5ee4a..475dc62 100644 --- a/SOURCES/ha-cloud-support-ibm.patch +++ b/SOURCES/ha-cloud-support-ibm.patch @@ -8,7 +8,7 @@ -import requests.adapters -import urllib3.util +try: -+ sys.path.insert(0, '/usr/lib/fence-agents/support/ibm') ++ sys.path.insert(0, '/usr/lib/fence-agents/support/ibm/lib/python#PYTHON3_VERSION#/site-packages') + import requests + import requests.adapters + import urllib3.util diff --git a/SPECS/resource-agents.spec b/SPECS/resource-agents.spec index 86bc4bf..3203b68 100644 --- a/SPECS/resource-agents.spec +++ b/SPECS/resource-agents.spec @@ -45,7 +45,7 @@ Name: resource-agents Summary: Open Source HA Reusable Cluster Resource Scripts Version: 4.10.0 -Release: 108%{?rcver:%{rcver}}%{?numcomm:.%{numcomm}}%{?alphatag:.%{alphatag}}%{?dirty:.%{dirty}}%{?dist} +Release: 111%{?rcver:%{rcver}}%{?numcomm:.%{numcomm}}%{?alphatag:.%{alphatag}}%{?dirty:.%{dirty}}%{?dist}.4 License: GPLv2+ and LGPLv2+ URL: https://github.com/ClusterLabs/resource-agents Source0: %{upstream_prefix}-%{upstream_version}.tar.gz @@ -199,6 +199,16 @@ Patch146: RHEL-42513-powervs-subnet-wait-for-IP.patch Patch147: RHEL-143527-powervs-move-ip-powervs-subnet-fix-error-logging.patch Patch148: RHEL-145628-podman-etcd-enhance-etcd-data-backup-with-snapshots-and-retention.patch Patch149: RHEL-150700-podman-etcd-set-attributes-if-they-fail-during-force-new-cluster.patch +Patch150: RHEL-151828-portblock-check-inverse-action.patch +Patch151: RHEL-156808-podman-etcd-ignore-learners-when-considering-which-node-has-higher-revision.patch +Patch152: RHEL-157145-podman-etcd-handle-existing-peer-URLs-gracefully-during-force_new_cluster-recovery.patch +Patch153: RHEL-159202-podman-etcd-hardened-monitor-stop-actions.patch +Patch154: RHEL-157273-db2-set-reintegration-when-promotion-is-successful.patch +Patch155: RHEL-166183-1-db2-fix-bashism.patch +Patch156: RHEL-166183-2-db2-do-not-use-db2stop-to-avoid-divergence-in-the-log.patch +Patch157: RHEL-177849-podman-etcd-fix-port-2380-binding-race.patch +Patch158: RHEL-177838-podman-etcd-fix-machine-deletion-deadlock.patch +Patch159: RHEL-177843-podman-etcd-fix-learner-start-deadlock.patch # bundled ha-cloud-support libs Patch500: ha-cloud-support-aliyun.patch @@ -315,7 +325,7 @@ service managers. License: GPLv2+ and LGPLv2+ Summary: Cloud resource agents Requires: %{name} = %{version}-%{release} -Requires: ha-cloud-support >= 4.10.0-63 +Requires: ha-cloud-support >= 4.10.0-110.el9_8.1 Requires: socat Provides: resource-agents-aliyun Obsoletes: resource-agents-aliyun <= %{version} @@ -493,6 +503,16 @@ exit 1 %patch -p1 -P 147 %patch -p1 -P 148 %patch -p1 -P 149 +%patch -p1 -P 150 +%patch -p1 -P 151 +%patch -p1 -P 152 +%patch -p1 -P 153 +%patch -p1 -P 154 +%patch -p1 -P 155 +%patch -p1 -P 156 +%patch -p1 -P 157 +%patch -p1 -P 158 +%patch -p1 -P 159 # bundled ha-cloud-support libs %patch -p1 -P 500 @@ -504,6 +524,8 @@ chmod 755 heartbeat/NovaEvacuate chmod 755 heartbeat/pgsqlms %build +sed -i -e "s/#PYTHON3_VERSION#/%{python3_version}/" heartbeat/*.in + if [ ! -f configure ]; then ./autogen.sh fi @@ -533,10 +555,10 @@ export CFLAGS PYTHON="%{__python3}" \ %endif %ifarch x86_64 - PYTHONPATH="%{_usr}/lib/fence-agents/support/google" \ + PYTHONPATH="%{_usr}/lib/fence-agents/support/google/lib/python%{python3_version}/site-packages" \ %endif %ifarch ppc64le - PYTHONPATH="%{_usr}/lib/fence-agents/support/ibm" \ + PYTHONPATH="%{_usr}/lib/fence-agents/support/ibm/lib/python%{python3_version}/site-packages" \ %endif %{conf_opt_fatal} \ %if %{defined _unitdir} @@ -825,7 +847,35 @@ rm -rf %{buildroot}/usr/share/doc/resource-agents %{_usr}/lib/ocf/lib/heartbeat/OCF_*.pm %changelog -* Wed Feb 18 2026 Oyvind Albrigtsen - 4.10.0-108 +* Wed May 20 2026 Oyvind Albrigtsen - 4.10.0-111.4 +- podman-etcd: fix port 2380 binding race +- podman-etcd: fix machine deletion deadlock +- podman-etcd: fix learner start deadlock + + Resolves: RHEL-177849, RHEL-177838, RHEL-177843 + +* Mon Apr 20 2026 Oyvind Albrigtsen - 4.10.0-111.3 +- Cloud agents: change bundled lib paths to match changes in + ha-cloud-support + + Resolves: RHEL-168565 + +* Fri Apr 10 2026 Oyvind Albrigtsen - 4.10.0-111.2 +- db2: do not use db2stop to avoid divergence in the log + + Resolves: RHEL-166183 + +* Wed Apr 8 2026 Oyvind Albrigtsen - 4.10.0-111.1 +- portblock: check inverse action state file for non-promotable + resources to avoid issues when doing e.g. block followed by unblock +- podman-etcd: ignore learners when considering which node has higher revision +- podman-etcd: handle existing peer URLs gracefully during force_new_cluster recovery +- podman-etcd: hardened monitor/stop actions +- db2: set reintegration when promotion is successful + + Resolves: RHEL-151828, RHEL-156808, RHEL-157145, RHEL-159202, RHEL-157273 + +* Fri Mar 6 2026 Oyvind Albrigtsen - 4.10.0-108 - podman-etcd: set attributes if they fail during force-new-cluster Resolves: RHEL-150700