import pacemaker-2.1.2-2.el9

2022-01-11 13:49:28 -05:00 · 2022-01-11 13:49:28 -05:00 · f997a33b7b
commit f997a33b7b
parent 4dbc733104
31 changed files with 7341 additions and 7598 deletions
--- a/.gitignore
+++ b/.gitignore
@ -1,2 +1,2 @@
 SOURCES/nagios-agents-metadata-105ab8a7b2c16b9a29cf1c1596b80136eeef332b.tar.gz
-SOURCES/pacemaker-7c3f66070.tar.gz
+SOURCES/pacemaker-ada5c3b36.tar.gz
--- a/.pacemaker.metadata
+++ b/.pacemaker.metadata
@ -1,2 +1,2 @@
 2cbec94ad67dfbeba75e38d2c3c5c44961b3cd16 SOURCES/nagios-agents-metadata-105ab8a7b2c16b9a29cf1c1596b80136eeef332b.tar.gz
-55917d293ae0f9b79ac196fae061ba95647382e1 SOURCES/pacemaker-7c3f66070.tar.gz
+1dec5b062ad8e9a89b4953e17a59e4597797a1e6 SOURCES/pacemaker-ada5c3b36.tar.gz
--- a/SOURCES/001-acl-group-schema.patch
+++ b/SOURCES/001-acl-group-schema.patch
@ -0,0 +1,230 @@
 From f5ffbaf1f537d3d5b00e594211cd322f97df51ac Mon Sep 17 00:00:00 2001
 From: Grace Chin <gchin@redhat.com>
 Date: Fri, 5 Nov 2021 11:39:39 -0400
 Subject: [PATCH 1/3] Low: xml: clone acls schema in preparation for changes
 ---
 xml/acls-3.8.rng | 80 ++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 80 insertions(+)
 create mode 100644 xml/acls-3.8.rng
 diff --git a/xml/acls-3.8.rng b/xml/acls-3.8.rng
 new file mode 100644
 index 000000000..0fe6eed96
 --- /dev/null
 +++ b/xml/acls-3.8.rng
@@ -0,0 +1,80 @@
 +<?xml version="1.0" encoding="UTF-8"?>
 +<grammar xmlns="http://relaxng.org/ns/structure/1.0" 
 +         datatypeLibrary="http://www.w3.org/2001/XMLSchema-datatypes">
 +  <start>
 +    <optional>
 +      <ref name="element-acls"/>
 +    </optional>
 +  </start>
 +
 +  <define name="element-acls">
 +    <element name="acls">
 +      <zeroOrMore>
 +        <choice>
 +          <element name="acl_target">
 +            <attribute name="id"><text/></attribute>
 +            <zeroOrMore>
 +              <element name="role">
 +                <attribute name="id"><data type="IDREF"/></attribute>
 +              </element>
 +            </zeroOrMore>
 +          </element>
 +          <element name="acl_group">
 +            <!-- Here 'id' is the name of a unix group -->
 +            <attribute name="id"><data type="ID"/></attribute>
 +            <zeroOrMore>
 +              <element name="role">
 +                <attribute name="id"><data type="IDREF"/></attribute>
 +              </element>
 +            </zeroOrMore>
 +          </element>
 +          <element name="acl_role">
 +            <attribute name="id"><data type="ID"/></attribute>
 +            <optional>
 +              <attribute name="description"><text/></attribute>
 +            </optional>
 +            <zeroOrMore>
 +              <ref name="element-permission"/>
 +            </zeroOrMore>
 +          </element>
 +        </choice>
 +      </zeroOrMore>
 +    </element>
 +  </define>
 +
 +  <define name="element-permission">
 +    <element name="acl_permission">
 +      <attribute name="id"><data type="ID"/></attribute>
 +
 +      <attribute name="kind">
 +        <choice>
 +          <value>read</value>
 +          <value>write</value>
 +          <value>deny</value>
 +        </choice>
 +      </attribute>
 +
 +      <choice>
 +        <attribute name="xpath"><text/></attribute>
 +        <!-- reference is already sufficiently specific without 'object-type' --> 
 +        <attribute name="reference"><data type="IDREF"/></attribute>
 +        <group>
 +          <!-- Use 'object-type' to avoid conflicting with the 'tag' configuration concept -->
 +          <attribute name="object-type"><text/></attribute>
 +          <optional>
 +            <!-- 
 +                 does not make sense with anything other than object-type
 +                 xpath and reference are already sufficiently specific
 +            -->
 +            <attribute name="attribute"><text/></attribute>
 +          </optional>
 +        </group>
 +      </choice>
 +
 +      <optional>
 +        <attribute name="description"><text/></attribute>
 +      </optional>
 +    </element>
 +  </define>
 +
 +</grammar>
 -- 
 2.27.0
 From 7838213fc639236bdedf5f15320152d973f1bdad Mon Sep 17 00:00:00 2001
 From: Grace Chin <gchin@redhat.com>
 Date: Fri, 5 Nov 2021 11:40:48 -0400
 Subject: [PATCH 2/3] Add a 'name' attribute to acl_target and acl_group
 elements
 ---
 xml/acls-3.8.rng | 6 ++++++
 1 file changed, 6 insertions(+)
 diff --git a/xml/acls-3.8.rng b/xml/acls-3.8.rng
 index 0fe6eed96..48bcdffe3 100644
 --- a/xml/acls-3.8.rng
 +++ b/xml/acls-3.8.rng
@@ -13,6 +13,9 @@
         <choice>
           <element name="acl_target">
             <attribute name="id"><text/></attribute>
 +            <optional>
 +              <attribute name="name"><text/></attribute>
 +            </optional>
             <zeroOrMore>
               <element name="role">
                 <attribute name="id"><data type="IDREF"/></attribute>
@@ -22,6 +25,9 @@
           <element name="acl_group">
             <!-- Here 'id' is the name of a unix group -->
             <attribute name="id"><data type="ID"/></attribute>
 +            <optional>
 +              <attribute name="name"><text/></attribute>
 +            </optional>
             <zeroOrMore>
               <element name="role">
                 <attribute name="id"><data type="IDREF"/></attribute>
 -- 
 2.27.0
 From c3c498f4636f57e29670f8e385b625024ed222d7 Mon Sep 17 00:00:00 2001
 From: Grace Chin <gchin@redhat.com>
 Date: Fri, 5 Nov 2021 11:42:48 -0400
 Subject: [PATCH 3/3] Changes made by run of 'cts/cts-cli -s'
 ---
 cts/cli/regression.upgrade.exp  |  7 +++++--
 cts/cli/regression.validity.exp | 22 ++++++++++++++++++----
 2 files changed, 23 insertions(+), 6 deletions(-)
 diff --git a/cts/cli/regression.upgrade.exp b/cts/cli/regression.upgrade.exp
 index e38adebdd..7ce7ec13b 100644
 --- a/cts/cli/regression.upgrade.exp
 +++ b/cts/cli/regression.upgrade.exp
@@ -91,8 +91,11 @@ update_validation 	debug: Configuration valid for schema: pacemaker-3.6
 update_validation 	debug: pacemaker-3.6-style configuration is also valid for pacemaker-3.7
 update_validation 	debug: Testing 'pacemaker-3.7' validation (21 of X)
 update_validation 	debug: Configuration valid for schema: pacemaker-3.7
 -update_validation 	trace: Stopping at pacemaker-3.7
 -update_validation 	info: Transformed the configuration from pacemaker-2.10 to pacemaker-3.7
 +update_validation 	debug: pacemaker-3.7-style configuration is also valid for pacemaker-3.8
 +update_validation 	debug: Testing 'pacemaker-3.8' validation (22 of X)
 +update_validation 	debug: Configuration valid for schema: pacemaker-3.8
 +update_validation 	trace: Stopping at pacemaker-3.8
 +update_validation 	info: Transformed the configuration from pacemaker-2.10 to pacemaker-3.8
 =#=#=#= Current cib after: Upgrade to latest CIB schema (trigger 2.10.xsl + the wrapping) =#=#=#=
 <cib epoch="2" num_updates="0" admin_epoch="1">
   <configuration>
 diff --git a/cts/cli/regression.validity.exp b/cts/cli/regression.validity.exp
 index 5ace430e7..125035a47 100644
 --- a/cts/cli/regression.validity.exp
 +++ b/cts/cli/regression.validity.exp
@@ -121,7 +121,11 @@ update_validation 	debug: Testing 'pacemaker-3.7' validation (21 of X)
 element rsc_order: Relax-NG validity error : Invalid attribute first-action for element rsc_order
 element rsc_order: Relax-NG validity error : Element constraints has extra content: rsc_order
 update_validation 	trace: pacemaker-3.7 validation failed
 -Cannot upgrade configuration (claiming schema pacemaker-1.2) to at least pacemaker-3.0 because it does not validate with any schema from pacemaker-1.2 to pacemaker-3.7
 +update_validation 	debug: Testing 'pacemaker-3.8' validation (22 of X)
 +element rsc_order: Relax-NG validity error : Invalid attribute first-action for element rsc_order
 +element rsc_order: Relax-NG validity error : Element constraints has extra content: rsc_order
 +update_validation 	trace: pacemaker-3.8 validation failed
 +Cannot upgrade configuration (claiming schema pacemaker-1.2) to at least pacemaker-3.0 because it does not validate with any schema from pacemaker-1.2 to pacemaker-3.8
 =#=#=#= End test: Run crm_simulate with invalid CIB (enum violation) - Invalid configuration (78) =#=#=#=
 * Passed: crm_simulate   - Run crm_simulate with invalid CIB (enum violation)
 =#=#=#= Begin test: Try to make resulting CIB invalid (unrecognized validate-with) =#=#=#=
@@ -226,7 +230,10 @@ update_validation 	trace: pacemaker-3.6 validation failed
 update_validation 	debug: Testing 'pacemaker-3.7' validation (21 of X)
 element cib: Relax-NG validity error : Invalid attribute validate-with for element cib
 update_validation 	trace: pacemaker-3.7 validation failed
 -Cannot upgrade configuration (claiming schema pacemaker-9999.0) to at least pacemaker-3.0 because it does not validate with any schema from unknown to pacemaker-3.7
 +update_validation 	debug: Testing 'pacemaker-3.8' validation (22 of X)
 +element cib: Relax-NG validity error : Invalid attribute validate-with for element cib
 +update_validation 	trace: pacemaker-3.8 validation failed
 +Cannot upgrade configuration (claiming schema pacemaker-9999.0) to at least pacemaker-3.0 because it does not validate with any schema from unknown to pacemaker-3.8
 =#=#=#= End test: Run crm_simulate with invalid CIB (unrecognized validate-with) - Invalid configuration (78) =#=#=#=
 * Passed: crm_simulate   - Run crm_simulate with invalid CIB (unrecognized validate-with)
 =#=#=#= Begin test: Try to make resulting CIB invalid, but possibly recoverable (valid with X.Y+1) =#=#=#=
@@ -326,8 +333,11 @@ update_validation 	debug: Configuration valid for schema: pacemaker-3.6
 update_validation 	debug: pacemaker-3.6-style configuration is also valid for pacemaker-3.7
 update_validation 	debug: Testing 'pacemaker-3.7' validation (21 of X)
 update_validation 	debug: Configuration valid for schema: pacemaker-3.7
 -update_validation 	trace: Stopping at pacemaker-3.7
 -update_validation 	info: Transformed the configuration from pacemaker-1.2 to pacemaker-3.7
 +update_validation 	debug: pacemaker-3.7-style configuration is also valid for pacemaker-3.8
 +update_validation 	debug: Testing 'pacemaker-3.8' validation (22 of X)
 +update_validation 	debug: Configuration valid for schema: pacemaker-3.8
 +update_validation 	trace: Stopping at pacemaker-3.8
 +update_validation 	info: Transformed the configuration from pacemaker-1.2 to pacemaker-3.8
 unpack_resources 	error: Resource start-up disabled since no STONITH resources have been defined
 unpack_resources 	error: Either configure some or disable STONITH with the stonith-enabled option
 unpack_resources 	error: NOTE: Clusters with shared data need STONITH to ensure data integrity
@@ -437,6 +447,8 @@ element rsc_order: Relax-NG validity error : Invalid attribute first-action for
 element rsc_order: Relax-NG validity error : Element constraints has extra content: rsc_order
 element rsc_order: Relax-NG validity error : Invalid attribute first-action for element rsc_order
 element rsc_order: Relax-NG validity error : Element constraints has extra content: rsc_order
 +element rsc_order: Relax-NG validity error : Invalid attribute first-action for element rsc_order
 +element rsc_order: Relax-NG validity error : Element constraints has extra content: rsc_order
 =#=#=#= Current cib after: Make resulting CIB invalid, and without validate-with attribute =#=#=#=
 <cib epoch="41" num_updates="0" admin_epoch="0" validate-with="none">
   <configuration>
@@ -502,6 +514,8 @@ validity.bad.xml:10: element rsc_order: Relax-NG validity error : Invalid attrib
 validity.bad.xml:10: element rsc_order: Relax-NG validity error : Element constraints has extra content: rsc_order
 validity.bad.xml:10: element rsc_order: Relax-NG validity error : Invalid attribute first-action for element rsc_order
 validity.bad.xml:10: element rsc_order: Relax-NG validity error : Element constraints has extra content: rsc_order
 +validity.bad.xml:10: element rsc_order: Relax-NG validity error : Invalid attribute first-action for element rsc_order
 +validity.bad.xml:10: element rsc_order: Relax-NG validity error : Element constraints has extra content: rsc_order
 unpack_resources 	error: Resource start-up disabled since no STONITH resources have been defined
 unpack_resources 	error: Either configure some or disable STONITH with the stonith-enabled option
 unpack_resources 	error: NOTE: Clusters with shared data need STONITH to ensure data integrity
 -- 
 2.27.0
--- a/SOURCES/001-ping-agent.patch
+++ b/SOURCES/001-ping-agent.patch
@ -1,225 +0,0 @@
 From c6ee0973522268ed7b3241cf0ec2e06398444114 Mon Sep 17 00:00:00 2001
 From: Grace Chin <gchin@redhat.com>
 Date: Tue, 4 May 2021 12:02:17 -0400
 Subject: [PATCH 1/4] Remove deprecated attrd_options
 ---
 extra/resources/ping | 11 +++--------
 1 file changed, 3 insertions(+), 8 deletions(-)
 diff --git a/extra/resources/ping b/extra/resources/ping
 index 3cf8dfe..2e93f22 100755
 --- a/extra/resources/ping
 +++ b/extra/resources/ping
@@ -178,7 +178,7 @@ ping_stop() {
     rm -f "${OCF_RESKEY_pidfile}"
 -    attrd_updater -D -n "$OCF_RESKEY_name" -d "$OCF_RESKEY_dampen" $attrd_options
 +    attrd_updater -D -n "$OCF_RESKEY_name" -d "$OCF_RESKEY_dampen"
     return $OCF_SUCCESS
 }
@@ -302,9 +302,9 @@ ping_update() {
     score=$(expr $active \* $OCF_RESKEY_multiplier)
     if [ "$__OCF_ACTION" = "start" ] ; then
 -        attrd_updater -n "$OCF_RESKEY_name" -B "$score" -d "$OCF_RESKEY_dampen" $attrd_options
 +        attrd_updater -n "$OCF_RESKEY_name" -B "$score" -d "$OCF_RESKEY_dampen"
     else
 -        attrd_updater -n "$OCF_RESKEY_name" -v "$score" -d "$OCF_RESKEY_dampen" $attrd_options
 +        attrd_updater -n "$OCF_RESKEY_name" -v "$score" -d "$OCF_RESKEY_dampen"
     fi
     rc=$?
     case $rc in
@@ -396,11 +396,6 @@ case "${OCF_RESKEY_debug}" in
         ;;
 esac
 -attrd_options='-q'
 -if [ "${OCF_RESKEY_debug}" = "true" ]; then
 -    attrd_options=''
 -fi
 -
 case "$__OCF_ACTION" in
 meta-data)      meta_data
                 exit $OCF_SUCCESS
 -- 
 1.8.3.1
 From 6d6c4691cf0970059689856c354daf9e098b4451 Mon Sep 17 00:00:00 2001
 From: Grace Chin <gchin@redhat.com>
 Date: Tue, 4 May 2021 14:50:37 -0400
 Subject: [PATCH 2/4] Replace debug values, true and false, with 0 and 1
 ---
 extra/resources/ping | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)
 diff --git a/extra/resources/ping b/extra/resources/ping
 index 2e93f22..fee019b 100755
 --- a/extra/resources/ping
 +++ b/extra/resources/ping
@@ -24,7 +24,7 @@
 : ${OCF_RESKEY_dampen:="5s"}
 : ${OCF_RESKEY_attempts:="3"}
 : ${OCF_RESKEY_multiplier:="1"}
 -: ${OCF_RESKEY_debug:="false"}
 +: ${OCF_RESKEY_debug:="0"}
 : ${OCF_RESKEY_failure_score:="0"}
 : ${OCF_RESKEY_use_fping:="1"}
 : ${OCF_RESKEY_host_list:=""}
@@ -152,7 +152,7 @@ END
 ping_conditional_log() {
     level="$1"; shift
 -    if [ "${OCF_RESKEY_debug}" = "true" ]; then
 +    if [ $OCF_RESKEY_debug -gt 0 ]; then
         ocf_log "$level" "$*"
     fi
 }
@@ -388,8 +388,8 @@ fi
 # Check the debug option
 case "${OCF_RESKEY_debug}" in
 -    true|True|TRUE|1)    OCF_RESKEY_debug=true;;
 -    false|False|FALSE|0) OCF_RESKEY_debug=false;;
 +    true|True|TRUE|1)    OCF_RESKEY_debug=0;;
 +    false|False|FALSE|0) OCF_RESKEY_debug=1;;
     *)
         ocf_log warn "Value for 'debug' is incorrect. Please specify 'true' or 'false' not: ${OCF_RESKEY_debug}"
         OCF_RESKEY_debug=false
 -- 
 1.8.3.1
 From a886a31056b6aca764c6911f5432af2c5ebf51df Mon Sep 17 00:00:00 2001
 From: Grace Chin <gchin@redhat.com>
 Date: Tue, 11 May 2021 11:04:50 -0400
 Subject: [PATCH 3/4] Add verbose debug mode which logs ping and fping output
 when set
 ---
 extra/resources/ping | 19 ++++++++++++++-----
 1 file changed, 14 insertions(+), 5 deletions(-)
 diff --git a/extra/resources/ping b/extra/resources/ping
 index fee019b..cc796af 100755
 --- a/extra/resources/ping
 +++ b/extra/resources/ping
@@ -249,10 +249,13 @@ fping_check() {
     case $rc in
         0)
 +            if [ $OCF_RESKEY_debug -gt 1 ]; then
 +                ping_conditional_log info "$output"
 +            fi
             ;;
         1)
             for h in $(echo "$output" | grep "is unreachable" | awk '{print $1}'); do
 -                ping_conditional_log warn "$h is inactive"
 +                ping_conditional_log warn "$h is inactive: $output"
             done
             ;;
         *)
@@ -282,7 +285,12 @@ ping_check() {
         p_out=$($p_exe $p_args $OCF_RESKEY_options $host 2>&1); rc=$?
         case $rc in
 -            0) active=$(expr $active + 1);;
 +            0)
 +                active=$(expr $active + 1)
 +                if [ $OCF_RESKEY_debug -gt 1 ]; then
 +                    ping_conditional_log info "$p_out"
 +                fi
 +                ;;
             1) ping_conditional_log warn "$host is inactive: $p_out";;
             *) ocf_log err "Unexpected result for '$p_exe $p_args $OCF_RESKEY_options $host' $rc: $p_out";;
         esac
@@ -388,10 +396,11 @@ fi
 # Check the debug option
 case "${OCF_RESKEY_debug}" in
 -    true|True|TRUE|1)    OCF_RESKEY_debug=0;;
 -    false|False|FALSE|0) OCF_RESKEY_debug=1;;
 +    true|True|TRUE|1)    OCF_RESKEY_debug=1;;
 +    false|False|FALSE|0) OCF_RESKEY_debug=0;;
 +    verbose|Verbose|VERBOSE|2) OCF_RESKEY_debug=2;;
     *)
 -        ocf_log warn "Value for 'debug' is incorrect. Please specify 'true' or 'false' not: ${OCF_RESKEY_debug}"
 +        ocf_log warn "Value for 'debug' is incorrect. Please specify 'true', 'false', or 'verbose', not: ${OCF_RESKEY_debug}"
         OCF_RESKEY_debug=false
         ;;
 esac
 -- 
 1.8.3.1
 From 460043f133ced80e923b1290af70502a72deb7f8 Mon Sep 17 00:00:00 2001
 From: Grace Chin <gchin@redhat.com>
 Date: Tue, 11 May 2021 11:07:05 -0400
 Subject: [PATCH 4/4] Improve variable names
 ---
 extra/resources/ping | 20 ++++++++++----------
 1 file changed, 10 insertions(+), 10 deletions(-)
 diff --git a/extra/resources/ping b/extra/resources/ping
 index cc796af..9763b60 100755
 --- a/extra/resources/ping
 +++ b/extra/resources/ping
@@ -244,22 +244,22 @@ fping_check() {
     timeout=$(expr $OCF_RESKEY_timeout \* 1000 / $OCF_RESKEY_attempts)
     cmd="$p_exe -r $OCF_RESKEY_attempts -t $timeout -B 1.0 $OCF_RESKEY_options $OCF_RESKEY_host_list"
 -    output=$($cmd 2>&1); rc=$?
 -    active=$(echo "$output" | grep "is alive" | wc -l)
 +    fping_output=$($cmd 2>&1); rc=$?
 +    active=$(echo "$fping_output" | grep "is alive" | wc -l)
     case $rc in
         0)
             if [ $OCF_RESKEY_debug -gt 1 ]; then
 -                ping_conditional_log info "$output"
 +                ping_conditional_log info "$fping_output"
             fi
             ;;
         1)
 -            for h in $(echo "$output" | grep "is unreachable" | awk '{print $1}'); do
 -                ping_conditional_log warn "$h is inactive: $output"
 +            for h in $(echo "$fping_output" | grep "is unreachable" | awk '{print $1}'); do
 +                ping_conditional_log warn "$h is inactive: $fping_output"
             done
             ;;
         *)
 -            ocf_log err "Unexpected result for '$cmd' $rc: $(echo "$output" | tr '\n' ';')"
 +            ocf_log err "Unexpected result for '$cmd' $rc: $(echo "$fping_output" | tr '\n' ';')"
             ;;
     esac
@@ -282,17 +282,17 @@ ping_check() {
             *:*) p_exe=ping6
         esac
 -        p_out=$($p_exe $p_args $OCF_RESKEY_options $host 2>&1); rc=$?
 +        ping_output=$($p_exe $p_args $OCF_RESKEY_options $host 2>&1); rc=$?
         case $rc in
             0)
                 active=$(expr $active + 1)
                 if [ $OCF_RESKEY_debug -gt 1 ]; then
 -                    ping_conditional_log info "$p_out"
 +                    ping_conditional_log info "$ping_output"
                 fi
                 ;;
 -            1) ping_conditional_log warn "$host is inactive: $p_out";;
 -            *) ocf_log err "Unexpected result for '$p_exe $p_args $OCF_RESKEY_options $host' $rc: $p_out";;
 +            1) ping_conditional_log warn "$host is inactive: $ping_output";;
 +            *) ocf_log err "Unexpected result for '$p_exe $p_args $OCF_RESKEY_options $host' $rc: $ping_output";;
         esac
     done
     return $active
 -- 
 1.8.3.1
--- a/SOURCES/002-fencing-reasons.patch
+++ b/SOURCES/002-fencing-reasons.patch
--- a/SOURCES/002-pacemakerd-options.patch
+++ b/SOURCES/002-pacemakerd-options.patch
@ -1,451 +0,0 @@
 From 0d40ebf10b1794ece2c5c9768ea7222d3834d3b3 Mon Sep 17 00:00:00 2001
 From: Chris Lumens <clumens@redhat.com>
 Date: Thu, 13 May 2021 11:42:18 -0400
 Subject: [PATCH 1/4] Build: Use a different variable to find man page
 includes.
 With other programs outside of the tools directory being converted to
 use glib for command line handling, their includes are not going to be
 in tools/.  So we need to use a different autoconf variable to find
 them.
 ---
 mk/common.mk | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)
 diff --git a/mk/common.mk b/mk/common.mk
 index b247670..aa59feb 100644
 --- a/mk/common.mk
 +++ b/mk/common.mk
@@ -1,5 +1,5 @@
 #
 -# Copyright 2014-2020 the Pacemaker project contributors
 +# Copyright 2014-2021 the Pacemaker project contributors
 #
 # The version control history for this file may have further details.
 #
@@ -68,11 +68,11 @@ HELP2MAN_ARGS = -N --section 8 --name "Part of the Pacemaker cluster resource ma
 #       and all wrappers to C code.
 %.8:	% $(MAN8DEPS)
 	$(AM_V_at)chmod a+x $(abs_builddir)/$<
 -	$(AM_V_MAN)if [ -f $(top_srcdir)/tools/$@.inc ]; then			\
 +	$(AM_V_MAN)if [ -f $(abs_srcdir)/$@.inc ]; then			\
 		PATH=$(abs_builddir):$$PATH $(HELP2MAN) $(HELP2MAN_ARGS)	\
 			-h --help-all 						\
 			--no-discard-stderr 					\
 -			-i $(top_srcdir)/tools/$@.inc $(abs_builddir)/$<		\
 +			-i $(abs_srcdir)/$@.inc $(abs_builddir)/$<		\
 			| sed -f $(top_srcdir)/tools/fix-manpages > $@ ; \
 	else									\
 		PATH=$(abs_builddir):$$PATH $(HELP2MAN) $(HELP2MAN_ARGS)	\
 -- 
 1.8.3.1
 From c7ab1d901bcbbf0137277e783e072777ca2f82d9 Mon Sep 17 00:00:00 2001
 From: Chris Lumens <clumens@redhat.com>
 Date: Thu, 13 May 2021 11:44:16 -0400
 Subject: [PATCH 2/4] Refactor: daemons: Remove the pid_file variable from
 pacemakerd.
 It's never used anywhere.
 ---
 daemons/pacemakerd/pacemakerd.c | 3 ---
 1 file changed, 3 deletions(-)
 diff --git a/daemons/pacemakerd/pacemakerd.c b/daemons/pacemakerd/pacemakerd.c
 index 8ec9708..03d688e 100644
 --- a/daemons/pacemakerd/pacemakerd.c
 +++ b/daemons/pacemakerd/pacemakerd.c
@@ -27,8 +27,7 @@
 static crm_trigger_t *shutdown_trigger = NULL;
 static crm_trigger_t *startup_trigger = NULL;
 -static const char *pid_file = PCMK_RUN_DIR "/pacemaker.pid";
 /* state we report when asked via pacemakerd-api status-ping */
 static const char *pacemakerd_state = XML_PING_ATTR_PACEMAKERDSTATE_INIT;
 static gboolean running_with_sbd = FALSE; /* local copy */
@@ -224,7 +222,6 @@ main(int argc, char **argv)
                 /* Legacy */
                 break;
             case 'p':
 -                pid_file = optarg;
                 break;
             case 's':
                 pcmk__set_env_option("node_start_state", "standby");
 -- 
 1.8.3.1
 From 98990eed9f6a5dbde7c8a5aa0783e93d5479295b Mon Sep 17 00:00:00 2001
 From: Chris Lumens <clumens@redhat.com>
 Date: Thu, 13 May 2021 13:14:38 -0400
 Subject: [PATCH 3/4] Refactor: daemons: Use glib for command line handling in
 pacemakerd.
 ---
 daemons/pacemakerd/Makefile.am      |   2 +
 daemons/pacemakerd/pacemakerd.8.inc |   5 +
 daemons/pacemakerd/pacemakerd.c     | 195 ++++++++++++++++++------------------
 3 files changed, 102 insertions(+), 100 deletions(-)
 create mode 100644 daemons/pacemakerd/pacemakerd.8.inc
 diff --git a/daemons/pacemakerd/Makefile.am b/daemons/pacemakerd/Makefile.am
 index cc657f5..84517a3 100644
 --- a/daemons/pacemakerd/Makefile.am
 +++ b/daemons/pacemakerd/Makefile.am
@@ -15,6 +15,8 @@ if BUILD_SYSTEMD
 systemdsystemunit_DATA	= pacemaker.service
 endif
 +EXTRA_DIST 	= pacemakerd.8.inc
 +
 ## SOURCES
 noinst_HEADERS		= pacemakerd.h
 diff --git a/daemons/pacemakerd/pacemakerd.8.inc b/daemons/pacemakerd/pacemakerd.8.inc
 new file mode 100644
 index 0000000..902af4e
 --- /dev/null
 +++ b/daemons/pacemakerd/pacemakerd.8.inc
@@ -0,0 +1,5 @@
 +[synopsis]
 +pacemakerd [options]
 +
 +/subsidiary Pacemaker daemons/
 +.SH OPTIONS
 diff --git a/daemons/pacemakerd/pacemakerd.c b/daemons/pacemakerd/pacemakerd.c
 index 03d688e..ce194bf 100644
 --- a/daemons/pacemakerd/pacemakerd.c
 +++ b/daemons/pacemakerd/pacemakerd.c
@@ -23,12 +23,54 @@
 #include <crm/msg_xml.h>
 #include <crm/common/ipc_internal.h>
 #include <crm/common/mainloop.h>
 +#include <crm/common/cmdline_internal.h>
 #include <crm/cluster/internal.h>
 #include <crm/cluster.h>
 #include <dirent.h>
 #include <ctype.h>
 +#define SUMMARY "pacemakerd - primary Pacemaker daemon that launches and monitors all subsidiary Pacemaker daemons"
 +
 +struct {
 +    gboolean features;
 +    gboolean foreground;
 +    gboolean shutdown;
 +    gboolean standby;
 +} options;
 +
 +static gboolean
 +pid_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **err) {
 +    return TRUE;
 +}
 +
 +static gboolean
 +standby_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **err) {
 +    options.standby = TRUE;
 +    pcmk__set_env_option("node_start_state", "standby");
 +    return TRUE;
 +}
 +
 +static GOptionEntry entries[] = {
 +    { "features", 'F', 0, G_OPTION_ARG_NONE, &options.features,
 +      "Display full version and list of features Pacemaker was built with",
 +      NULL },
 +    { "foreground", 'f', 0, G_OPTION_ARG_NONE, &options.foreground,
 +      "(Ignored) Pacemaker always runs in the foreground",
 +      NULL },
 +    { "pid-file", 'p', 0, G_OPTION_ARG_CALLBACK, pid_cb,
 +      "(Ignored) Daemon pid file location",
 +      "FILE" },
 +    { "shutdown", 'S', 0, G_OPTION_ARG_NONE, &options.shutdown,
 +      "Instruct Pacemaker to shutdown on this machine",
 +      NULL },
 +    { "standby", 's', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, standby_cb,
 +      "Start node in standby state",
 +      NULL },
 +
 +    { NULL }
 +};
 +
 static gboolean fatal_error = FALSE;
 static GMainLoop *mainloop = NULL;
 static bool global_keep_tracking = false;
@@ -642,49 +685,6 @@ pcmk_sigquit(int nsig)
     .connection_destroyed = pcmk_ipc_destroy
 };
 -static pcmk__cli_option_t long_options[] = {
 -    // long option, argument type, storage, short option, description, flags
 -    {
 -        "help", no_argument, NULL, '?',
 -        "\tThis text", pcmk__option_default
 -    },
 -    {
 -        "version", no_argument, NULL, '$',
 -        "\tVersion information", pcmk__option_default
 -    },
 -    {
 -        "verbose", no_argument, NULL, 'V',
 -        "\tIncrease debug output", pcmk__option_default
 -    },
 -    {
 -        "shutdown", no_argument, NULL, 'S',
 -        "\tInstruct Pacemaker to shutdown on this machine", pcmk__option_default
 -    },
 -    {
 -        "features", no_argument, NULL, 'F',
 -        "\tDisplay full version and list of features Pacemaker was built with",
 -        pcmk__option_default
 -    },
 -    {
 -        "-spacer-", no_argument, NULL, '-',
 -        "\nAdditional Options:", pcmk__option_default
 -    },
 -    {
 -        "foreground", no_argument, NULL, 'f',
 -        "\t(Ignored) Pacemaker always runs in the foreground",
 -        pcmk__option_default
 -    },
 -    {
 -        "pid-file", required_argument, NULL, 'p',
 -        "\t(Ignored) Daemon pid file location", pcmk__option_default
 -    },
 -    {
 -        "standby", no_argument, NULL, 's',
 -        "\tStart node in standby state", pcmk__option_default
 -    },
 -    { 0, 0, 0, 0 }
 -};
 -
 static void
 mcp_chown(const char *path, uid_t uid, gid_t gid)
 {
@@ -1168,83 +1211,66 @@ request_shutdown(crm_ipc_t *ipc)
     return status;
 }
 +static GOptionContext *
 +build_arg_context(pcmk__common_args_t *args) {
 +    GOptionContext *context = NULL;
 +
 +    context = pcmk__build_arg_context(args, NULL, NULL, NULL);
 +    pcmk__add_main_args(context, entries);
 +    return context;
 +}
 +
 int
 main(int argc, char **argv)
 {
 -    int flag;
 -    int argerr = 0;
 +    crm_exit_t exit_code = CRM_EX_OK;
 +
 +    GError *error = NULL;
 +
 +    pcmk__common_args_t *args = pcmk__new_common_args(SUMMARY);
 +    gchar **processed_args = pcmk__cmdline_preproc(argv, "p");
 +    GOptionContext *context = build_arg_context(args);
 -    int option_index = 0;
     bool old_instance_connected = false;
 -    gboolean shutdown = FALSE;
     crm_ipc_t *old_instance = NULL;
     qb_ipcs_service_t *ipcs = NULL;
     crm_log_preinit(NULL, argc, argv);
 -    pcmk__set_cli_options(NULL, "[options]", long_options,
 -                          "primary Pacemaker daemon that launches and "
 -                          "monitors all subsidiary Pacemaker daemons");
     mainloop_add_signal(SIGHUP, pcmk_ignore);
     mainloop_add_signal(SIGQUIT, pcmk_sigquit);
 -    while (1) {
 -        flag = pcmk__next_cli_option(argc, argv, &option_index, NULL);
 -        if (flag == -1)
 -            break;
 -
 -        switch (flag) {
 -            case 'V':
 -                crm_bump_log_level(argc, argv);
 -                break;
 -            case 'f':
 -                /* Legacy */
 -                break;
 -            case 'p':
 -                break;
 -            case 's':
 -                pcmk__set_env_option("node_start_state", "standby");
 -                break;
 -            case '$':
 -            case '?':
 -                pcmk__cli_help(flag, CRM_EX_OK);
 -                break;
 -            case 'S':
 -                shutdown = TRUE;
 -                break;
 -            case 'F':
 -                printf("Pacemaker %s (Build: %s)\n Supporting v%s: %s\n", PACEMAKER_VERSION, BUILD_VERSION,
 -                       CRM_FEATURE_SET, CRM_FEATURES);
 -                crm_exit(CRM_EX_OK);
 -            default:
 -                printf("Argument code 0%o (%c) is not (?yet?) supported\n", flag, flag);
 -                ++argerr;
 -                break;
 -        }
 +    if (!g_option_context_parse_strv(context, &processed_args, &error)) {
 +        exit_code = CRM_EX_USAGE;
 +        goto done;
     }
 -    if (optind < argc) {
 -        printf("non-option ARGV-elements: ");
 -        while (optind < argc)
 -            printf("%s ", argv[optind++]);
 -        printf("\n");
 -    }
 -    if (argerr) {
 -        pcmk__cli_help('?', CRM_EX_USAGE);
 +    if (options.features) {
 +        printf("Pacemaker %s (Build: %s)\n Supporting v%s: %s\n", PACEMAKER_VERSION, BUILD_VERSION,
 +               CRM_FEATURE_SET, CRM_FEATURES);
 +        exit_code = CRM_EX_OK;
 +        goto done;
     }
 +    if (args->version) {
 +        g_strfreev(processed_args);
 +        pcmk__free_arg_context(context);
 +        /* FIXME:  When pacemakerd is converted to use formatted output, this can go. */
 +        pcmk__cli_help('v', CRM_EX_USAGE);
 +    }
     setenv("LC_ALL", "C", 1);
     pcmk__set_env_option("mcp", "true");
 +    pcmk__cli_init_logging("pacemakerd", args->verbosity);
     crm_log_init(NULL, LOG_INFO, TRUE, FALSE, argc, argv, FALSE);
     crm_debug("Checking for existing Pacemaker instance");
     old_instance = crm_ipc_new(CRM_SYSTEM_MCP, 0);
     old_instance_connected = crm_ipc_connect(old_instance);
 -    if (shutdown) {
 +    if (options.shutdown) {
         if (old_instance_connected) {
             crm_exit(request_shutdown(old_instance));
         } else {
@@ -1253,22 +1279,25 @@ main(int argc, char **argv)
                     "Pacemaker instance: %s", strerror(errno));
             crm_ipc_close(old_instance);
             crm_ipc_destroy(old_instance);
 -            crm_exit(CRM_EX_DISCONNECT);
 +            exit_code = CRM_EX_DISCONNECT;
 +            goto done;
         }
     } else if (old_instance_connected) {
         crm_ipc_close(old_instance);
         crm_ipc_destroy(old_instance);
         crm_err("Aborting start-up because active Pacemaker instance found");
 -        crm_exit(CRM_EX_FATAL);
 +        exit_code = CRM_EX_FATAL;
 +        goto done;
     }
     crm_ipc_close(old_instance);
     crm_ipc_destroy(old_instance);
 #ifdef SUPPORT_COROSYNC
     if (mcp_read_config() == FALSE) {
 -        crm_exit(CRM_EX_UNAVAILABLE);
 +        exit_code = CRM_EX_UNAVAILABLE;
 +        goto done;
     }
 #endif
@@ -1292,7 +1321,8 @@ main(int argc, char **argv)
 #ifdef SUPPORT_COROSYNC
     /* Allows us to block shutdown */
     if (!cluster_connect_cfg()) {
 -        crm_exit(CRM_EX_PROTOCOL);
 +        exit_code = CRM_EX_PROTOCOL;
 +        goto done;
     }
 #endif
@@ -1307,9 +1337,11 @@ main(int argc, char **argv)
         case pcmk_rc_ok:
             break;
         case pcmk_rc_ipc_unauthorized:
 -            crm_exit(CRM_EX_CANTCREAT);
 +            exit_code = CRM_EX_CANTCREAT;
 +            goto done;
         default:
 -            crm_exit(CRM_EX_FATAL);
 +            exit_code = CRM_EX_FATAL;
 +            goto done;
     };
     mainloop_add_signal(SIGTERM, pcmk_shutdown);
@@ -1342,5 +1374,11 @@ main(int argc, char **argv)
 #ifdef SUPPORT_COROSYNC
     cluster_disconnect_cfg();
 #endif
 -    crm_exit(CRM_EX_OK);
 +
 +done:
 +    g_strfreev(processed_args);
 +    pcmk__free_arg_context(context);
 +
 +    pcmk__output_and_clear_error(error, NULL);
 +    crm_exit(exit_code);
 }
 -- 
 1.8.3.1
 From 8f7924fbb2a012bedcad59335b7bebc5020b26e3 Mon Sep 17 00:00:00 2001
 From: Chris Lumens <clumens@redhat.com>
 Date: Thu, 13 May 2021 13:27:13 -0400
 Subject: [PATCH 4/4] Low: pacemaker.service: Don't start pacemakerd with -f.
 This option is completely ignored by pacemakerd.
 ---
 daemons/pacemakerd/pacemaker.service.in           | 2 +-
 doc/sphinx/Clusters_from_Scratch/verification.rst | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)
 diff --git a/daemons/pacemakerd/pacemaker.service.in b/daemons/pacemakerd/pacemaker.service.in
 index b128ddc..0363a22 100644
 --- a/daemons/pacemakerd/pacemaker.service.in
 +++ b/daemons/pacemakerd/pacemaker.service.in
@@ -44,7 +44,7 @@ EnvironmentFile=-@CONFIGDIR@/pacemaker
 EnvironmentFile=-@CONFIGDIR@/sbd
 SuccessExitStatus=100
 -ExecStart=@sbindir@/pacemakerd -f
 +ExecStart=@sbindir@/pacemakerd
 # Systemd v227 and above can limit the number of processes spawned by a
 # service. That is a bad idea for an HA cluster resource manager, so disable it
 diff --git a/doc/sphinx/Clusters_from_Scratch/verification.rst b/doc/sphinx/Clusters_from_Scratch/verification.rst
 index 9d647f8..b7fa20e 100644
 --- a/doc/sphinx/Clusters_from_Scratch/verification.rst
 +++ b/doc/sphinx/Clusters_from_Scratch/verification.rst
@@ -103,7 +103,7 @@ the necessary processes are running:
         2 ?        S      0:00 [kthreadd]
     ...lots of processes...
     17121 ?        SLsl   0:01 /usr/sbin/corosync -f
 -    17133 ?        Ss     0:00 /usr/sbin/pacemakerd -f
 +    17133 ?        Ss     0:00 /usr/sbin/pacemakerd
     17134 ?        Ss     0:00  \_ /usr/libexec/pacemaker/pacemaker-based
     17135 ?        Ss     0:00  \_ /usr/libexec/pacemaker/pacemaker-fenced
     17136 ?        Ss     0:00  \_ /usr/libexec/pacemaker/pacemaker-execd
 -- 
 1.8.3.1
--- a/SOURCES/003-fencing-reasons.patch
+++ b/SOURCES/003-fencing-reasons.patch
--- a/SOURCES/003-pacemakerd-output.patch
+++ b/SOURCES/003-pacemakerd-output.patch
@ -1,343 +0,0 @@
 From 7c35387a9896cb968cf4087b5cbed94af44e1ea5 Mon Sep 17 00:00:00 2001
 From: Chris Lumens <clumens@redhat.com>
 Date: Fri, 14 May 2021 12:03:46 -0400
 Subject: [PATCH 1/5] Feature: daemons: Convert pacemakerd to formatted output.
 The main purpose of this is to finish getting pacemakerd moved off the
 existing command line handling code (pcmk__cli_help in particular) so
 that code can eventually be deprecated or removed.  pacemakerd itself
 does fairly little printing.
 ---
 daemons/pacemakerd/pacemakerd.c | 58 ++++++++++++++++++++++++++++++-----------
 1 file changed, 43 insertions(+), 15 deletions(-)
 diff --git a/daemons/pacemakerd/pacemakerd.c b/daemons/pacemakerd/pacemakerd.c
 index ce194bf..bd59729 100644
 --- a/daemons/pacemakerd/pacemakerd.c
 +++ b/daemons/pacemakerd/pacemakerd.c
@@ -25,6 +25,7 @@
 #include <crm/common/ipc_internal.h>
 #include <crm/common/mainloop.h>
 #include <crm/common/cmdline_internal.h>
 +#include <crm/common/output_internal.h>
 #include <crm/cluster/internal.h>
 #include <crm/cluster.h>
@@ -37,6 +38,14 @@ struct {
     gboolean standby;
 } options;
 +static pcmk__output_t *out = NULL;
 +
 +static pcmk__supported_format_t formats[] = {
 +    PCMK__SUPPORTED_FORMAT_NONE,
 +    PCMK__SUPPORTED_FORMAT_TEXT,
 +    { NULL, NULL, NULL }
 +};
 +
 static gboolean
 pid_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **err) {
     return TRUE;
@@ -1167,10 +1176,10 @@ pacemakerd_event_cb(pcmk_ipc_api_t *pacemakerd_api,
 }
 static GOptionContext *
 -build_arg_context(pcmk__common_args_t *args) {
 +build_arg_context(pcmk__common_args_t *args, GOptionGroup **group) {
     GOptionContext *context = NULL;
 -    context = pcmk__build_arg_context(args, NULL, NULL, NULL);
 +    context = pcmk__build_arg_context(args, "text", group, NULL);
     pcmk__add_main_args(context, entries);
     return context;
 }
@@ -1182,9 +1191,11 @@ main(int argc, char **argv)
     GError *error = NULL;
 +    int rc = pcmk_rc_ok;
 +    GOptionGroup *output_group = NULL;
     pcmk__common_args_t *args = pcmk__new_common_args(SUMMARY);
     gchar **processed_args = pcmk__cmdline_preproc(argv, "p");
 -    GOptionContext *context = build_arg_context(args);
 +    GOptionContext *context = build_arg_context(args, &output_group);
     bool old_instance_connected = false;
@@ -1195,23 +1205,30 @@ main(int argc, char **argv)
     mainloop_add_signal(SIGHUP, pcmk_ignore);
     mainloop_add_signal(SIGQUIT, pcmk_sigquit);
 +    pcmk__register_formats(output_group, formats);
     if (!g_option_context_parse_strv(context, &processed_args, &error)) {
         exit_code = CRM_EX_USAGE;
         goto done;
     }
 +    rc = pcmk__output_new(&out, args->output_ty, args->output_dest, argv);
 +    if (rc != pcmk_rc_ok) {
 +        exit_code = CRM_EX_ERROR;
 +        g_set_error(&error, PCMK__EXITC_ERROR, exit_code, "Error creating output format %s: %s",
 +                    args->output_ty, pcmk_rc_str(rc));
 +        goto done;
 +    }
 +
     if (options.features) {
 -        printf("Pacemaker %s (Build: %s)\n Supporting v%s: %s\n", PACEMAKER_VERSION, BUILD_VERSION,
 -               CRM_FEATURE_SET, CRM_FEATURES);
 +        out->info(out, "Pacemaker %s (Build: %s)\n Supporting v%s: %s", PACEMAKER_VERSION,
 +                  BUILD_VERSION, CRM_FEATURE_SET, CRM_FEATURES);
         exit_code = CRM_EX_OK;
         goto done;
     }
     if (args->version) {
 -        g_strfreev(processed_args);
 -        pcmk__free_arg_context(context);
 -        /* FIXME:  When pacemakerd is converted to use formatted output, this can go. */
 -        pcmk__cli_help('v', CRM_EX_USAGE);
 +        out->version(out, false);
 +        goto done;
     }
     setenv("LC_ALL", "C", 1);
@@ -1248,6 +1265,13 @@ main(int argc, char **argv)
     crm_ipc_close(old_instance);
     crm_ipc_destroy(old_instance);
 +    /* Don't allow any accidental output after this point. */
 +    if (out != NULL) {
 +        out->finish(out, exit_code, true, NULL);
 +        pcmk__output_free(out);
 +        out = NULL;
 +    }
 +
 #ifdef SUPPORT_COROSYNC
     if (mcp_read_config() == FALSE) {
         exit_code = CRM_EX_UNAVAILABLE;
@@ -1333,6 +1357,11 @@ done:
     g_strfreev(processed_args);
     pcmk__free_arg_context(context);
 -    pcmk__output_and_clear_error(error, NULL);
 +    pcmk__output_and_clear_error(error, out);
 +
 +    if (out != NULL) {
 +        out->finish(out, exit_code, true, NULL);
 +        pcmk__output_free(out);
 +    }
     crm_exit(exit_code);
 }
 -- 
 1.8.3.1
 From 35e6da64381fcb092d81ce16835cc28670b077cb Mon Sep 17 00:00:00 2001
 From: Chris Lumens <clumens@redhat.com>
 Date: Mon, 17 May 2021 10:04:04 -0400
 Subject: [PATCH 2/5] Features: daemons: Output the pacemakerd feature list in
 XML.
 ---
 daemons/pacemakerd/pacemakerd.c | 45 ++++++++++++++++++++++++++++++++++++++---
 1 file changed, 42 insertions(+), 3 deletions(-)
 diff --git a/daemons/pacemakerd/pacemakerd.c b/daemons/pacemakerd/pacemakerd.c
 index bd59729..93cf743 100644
 --- a/daemons/pacemakerd/pacemakerd.c
 +++ b/daemons/pacemakerd/pacemakerd.c
@@ -43,6 +43,42 @@ static pcmk__output_t *out = NULL;
 static pcmk__supported_format_t formats[] = {
     PCMK__SUPPORTED_FORMAT_NONE,
     PCMK__SUPPORTED_FORMAT_TEXT,
 +    PCMK__SUPPORTED_FORMAT_XML,
 +    { NULL, NULL, NULL }
 +};
 +
 +static int
 +pacemakerd_features(pcmk__output_t *out, va_list args) {
 +    out->info(out, "Pacemaker %s (Build: %s)\n Supporting v%s: %s", PACEMAKER_VERSION,
 +              BUILD_VERSION, CRM_FEATURE_SET, CRM_FEATURES);
 +    return pcmk_rc_ok;
 +}
 +
 +static int
 +pacemakerd_features_xml(pcmk__output_t *out, va_list args) {
 +    gchar **feature_list = g_strsplit(CRM_FEATURES, " ", 0);
 +
 +    pcmk__output_xml_create_parent(out, "pacemakerd",
 +                                   "version", PACEMAKER_VERSION,
 +                                   "build", BUILD_VERSION,
 +                                   "feature_set", CRM_FEATURE_SET,
 +                                   NULL);
 +    out->begin_list(out, NULL, NULL, "features");
 +
 +    for (char **s = feature_list; *s != NULL; s++) {
 +        pcmk__output_create_xml_text_node(out, "feature", *s);
 +    }
 +
 +    out->end_list(out);
 +
 +    g_strfreev(feature_list);
 +    return pcmk_rc_ok;
 +}
 +
 +static pcmk__message_entry_t fmt_functions[] = {
 +    { "features", "default", pacemakerd_features },
 +    { "features", "xml", pacemakerd_features_xml },
 +
     { NULL, NULL, NULL }
 };
@@ -200,7 +236,7 @@ static GOptionContext *
 build_arg_context(pcmk__common_args_t *args, GOptionGroup **group) {
     GOptionContext *context = NULL;
 -    context = pcmk__build_arg_context(args, "text", group, NULL);
 +    context = pcmk__build_arg_context(args, "text (default), xml", group, NULL);
     pcmk__add_main_args(context, entries);
     return context;
 }
@@ -241,9 +277,12 @@ main(int argc, char **argv)
         goto done;
     }
 +    pcmk__force_args(context, &error, "%s --xml-simple-list", g_get_prgname());
 +
 +    pcmk__register_messages(out, fmt_functions);
 +
     if (options.features) {
 -        out->info(out, "Pacemaker %s (Build: %s)\n Supporting v%s: %s", PACEMAKER_VERSION,
 -                  BUILD_VERSION, CRM_FEATURE_SET, CRM_FEATURES);
 +        out->message(out, "features");
         exit_code = CRM_EX_OK;
         goto done;
     }
 -- 
 1.8.3.1
 From 5b7f5eb35b025b59805cf3c7c3dcb6a3cf4b71b3 Mon Sep 17 00:00:00 2001
 From: Chris Lumens <clumens@redhat.com>
 Date: Mon, 17 May 2021 11:09:53 -0400
 Subject: [PATCH 3/5] Low: daemons: Conditionally enable logging in pacemakerd.
 If we're doing an interactive command-line call, use
 pcmk__cli_init_logging.  At the moment, all command line calls except
 for --shutdown do their work before logging would even come up, so we
 really only need to do this for --shutdown.
 If we're doing a daemon call, use crm_log_init.
 ---
 daemons/pacemakerd/pacemakerd.c | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)
 diff --git a/daemons/pacemakerd/pacemakerd.c b/daemons/pacemakerd/pacemakerd.c
 index 93cf743..c20bde7 100644
 --- a/daemons/pacemakerd/pacemakerd.c
 +++ b/daemons/pacemakerd/pacemakerd.c
@@ -296,8 +296,11 @@ main(int argc, char **argv)
     pcmk__set_env_option("mcp", "true");
 -    pcmk__cli_init_logging("pacemakerd", args->verbosity);
 -    crm_log_init(NULL, LOG_INFO, TRUE, FALSE, argc, argv, FALSE);
 +    if (options.shutdown) {
 +        pcmk__cli_init_logging("pacemakerd", args->verbosity);
 +    } else {
 +        crm_log_init(NULL, LOG_INFO, TRUE, FALSE, argc, argv, FALSE);
 +    }
     crm_debug("Checking for existing Pacemaker instance");
     old_instance = crm_ipc_new(CRM_SYSTEM_MCP, 0);
 -- 
 1.8.3.1
 From 2393362bb7489e86d937ed46a1c5cfb93d9bf3ab Mon Sep 17 00:00:00 2001
 From: Chris Lumens <clumens@redhat.com>
 Date: Mon, 17 May 2021 11:58:06 -0400
 Subject: [PATCH 4/5] Fix: include: Bump CRM_FEATURE_SET for new pacemakerd
 args.
 ---
 include/crm/crm.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
 diff --git a/include/crm/crm.h b/include/crm/crm.h
 index fdfc825..92a98fa 100644
 --- a/include/crm/crm.h
 +++ b/include/crm/crm.h
@@ -66,7 +66,7 @@ extern "C" {
  * >=3.0.13: Fail counts include operation name and interval
  * >=3.2.0:  DC supports PCMK_LRM_OP_INVALID and PCMK_LRM_OP_NOT_CONNECTED
  */
 -#  define CRM_FEATURE_SET		"3.10.0"
 +#  define CRM_FEATURE_SET		"3.10.1"
 /* Pacemaker's CPG protocols use fixed-width binary fields for the sender and
  * recipient of a CPG message. This imposes an arbitrary limit on cluster node
 -- 
 1.8.3.1
 From 3ad8edbd91631b87ef5f53fa2d68f0c8bbb9ee2b Mon Sep 17 00:00:00 2001
 From: Chris Lumens <clumens@redhat.com>
 Date: Mon, 17 May 2021 11:57:09 -0400
 Subject: [PATCH 5/5] Feature: xml: Add schema for pacemakerd.
 ---
 xml/Makefile.am             |  1 +
 xml/api/pacemakerd-2.10.rng | 28 ++++++++++++++++++++++++++++
 2 files changed, 29 insertions(+)
 create mode 100644 xml/api/pacemakerd-2.10.rng
 diff --git a/xml/Makefile.am b/xml/Makefile.am
 index 12a51c5..b9448d4 100644
 --- a/xml/Makefile.am
 +++ b/xml/Makefile.am
@@ -56,6 +56,7 @@ API_request_base	= command-output	\
 			  crm_simulate		\
 			  crmadmin		\
 			  digests		\
 +			  pacemakerd 		\
 			  stonith_admin		\
 			  version
 diff --git a/xml/api/pacemakerd-2.10.rng b/xml/api/pacemakerd-2.10.rng
 new file mode 100644
 index 0000000..41a11e7
 --- /dev/null
 +++ b/xml/api/pacemakerd-2.10.rng
@@ -0,0 +1,28 @@
 +<?xml version="1.0" encoding="UTF-8"?>
 +<grammar xmlns="http://relaxng.org/ns/structure/1.0"
 +         datatypeLibrary="http://www.w3.org/2001/XMLSchema-datatypes">
 +
 +    <start>
 +        <ref name="element-pacemakerd"/>
 +    </start>
 +
 +    <define name="element-pacemakerd">
 +        <element name="pacemakerd">
 +            <attribute name="version"> <text /> </attribute>
 +            <attribute name="build"> <text /> </attribute>
 +            <attribute name="feature_set"> <text /> </attribute>
 +
 +            <optional>
 +                <ref name="feature-list" />
 +            </optional>
 +        </element>
 +    </define>
 +
 +    <define name="feature-list">
 +        <element name="features">
 +            <oneOrMore>
 +                <element name="feature"> <text/> </element>
 +            </oneOrMore>
 +        </element>
 +    </define>
 +</grammar>
 -- 
 1.8.3.1
--- a/SOURCES/004-check-level.patch
+++ b/SOURCES/004-check-level.patch
@ -1,199 +0,0 @@
 From 3905e7eac11298fc20efd567a773666f948edf61 Mon Sep 17 00:00:00 2001
 From: Chris Lumens <clumens@redhat.com>
 Date: Mon, 3 May 2021 11:19:04 -0400
 Subject: [PATCH 1/2] Feature: tools: Add OCF_CHECK_LEVEL to crm_resource
 environment.
 If --validate= or --force-check= are given with a level, pass that along
 as OCF_CHECK_LEVEL.  This argument is optional, and if no value is given
 then the environment variable will not be set and whatever's the default
 on the resource agent will be used.
 See: rhbz#1955792.
 ---
 tools/crm_resource.c         | 29 +++++++++++++++++++++--------
 tools/crm_resource.h         |  4 ++--
 tools/crm_resource_runtime.c | 13 ++++++++++---
 3 files changed, 33 insertions(+), 13 deletions(-)
 diff --git a/tools/crm_resource.c b/tools/crm_resource.c
 index 45db2b2..6ca96f8 100644
 --- a/tools/crm_resource.c
 +++ b/tools/crm_resource.c
@@ -100,6 +100,7 @@ struct {
     int timeout_ms;               // Parsed from --timeout value
     char *agent_spec;             // Standard and/or provider and/or agent
     gchar *xml_file;              // Value of (deprecated) --xml-file
 +    int check_level;              // Optional value of --validate or --force-check
     // Resource configuration specified via command-line arguments
     gboolean cmdline_config;      // Resource configuration was via arguments
@@ -113,6 +114,7 @@ struct {
     GHashTable *override_params;  // Resource parameter values that override config
 } options = {
     .attr_set_type = XML_TAG_ATTR_SETS,
 +    .check_level = -1,
     .cib_options = cib_sync_call,
     .require_cib = TRUE,
     .require_dataset = TRUE,
@@ -402,14 +404,15 @@ static GOptionEntry query_entries[] = {
 };
 static GOptionEntry command_entries[] = {
 -    { "validate", 0, G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK,
 +    { "validate", 0, G_OPTION_FLAG_OPTIONAL_ARG, G_OPTION_ARG_CALLBACK,
       validate_or_force_cb,
       "Validate resource configuration by calling agent's validate-all\n"
       INDENT "action. The configuration may be specified either by giving an\n"
       INDENT "existing resource name with -r, or by specifying --class,\n"
       INDENT "--agent, and --provider arguments, along with any number of\n"
 -      INDENT "--option arguments.",
 -      NULL },
 +      INDENT "--option arguments. An optional LEVEL argument can be given\n"
 +      INDENT "to control the level of checking performed.",
 +      "LEVEL" },
     { "cleanup", 'C', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, cleanup_refresh_cb,
       "If resource has any past failures, clear its history and fail\n"
       INDENT "count. Optionally filtered by --resource, --node, --operation\n"
@@ -546,11 +549,12 @@ static GOptionEntry advanced_entries[] = {
       INDENT "the cluster believes the resource is a clone instance already\n"
       INDENT "running on the local node.",
       NULL },
 -    { "force-check", 0, G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK,
 +    { "force-check", 0, G_OPTION_FLAG_OPTIONAL_ARG, G_OPTION_ARG_CALLBACK,
       validate_or_force_cb,
       "(Advanced) Bypass the cluster and check the state of a resource on\n"
 -      INDENT "the local node",
 -      NULL },
 +      INDENT "the local node. An optional LEVEL argument can be given\n"
 +      INDENT "to control the level of checking performed.",
 +      "LEVEL" },
     { NULL }
 };
@@ -910,6 +914,15 @@ validate_or_force_cb(const gchar *option_name, const gchar *optarg,
     if (options.override_params == NULL) {
         options.override_params = pcmk__strkey_table(free, free);
     }
 +
 +    if (optarg != NULL) {
 +        if (pcmk__scan_min_int(optarg, &options.check_level, 0) != pcmk_rc_ok) {
 +            g_set_error(error, G_OPTION_ERROR, CRM_EX_INVALID_PARAM,
 +                        "Invalid check level setting: %s", optarg);
 +            return FALSE;
 +        }
 +    }
 +
     return TRUE;
 }
@@ -1826,12 +1839,12 @@ main(int argc, char **argv)
                     options.v_class, options.v_provider, options.v_agent,
                     "validate-all", options.cmdline_params,
                     options.override_params, options.timeout_ms,
 -                    args->verbosity, options.force);
 +                    args->verbosity, options.force, options.check_level);
             } else {
                 exit_code = cli_resource_execute(rsc, options.rsc_id,
                     options.operation, options.override_params,
                     options.timeout_ms, cib_conn, data_set,
 -                    args->verbosity, options.force);
 +                    args->verbosity, options.force, options.check_level);
             }
             goto done;
 diff --git a/tools/crm_resource.h b/tools/crm_resource.h
 index 3560377..5ab10d6 100644
 --- a/tools/crm_resource.h
 +++ b/tools/crm_resource.h
@@ -88,11 +88,11 @@ crm_exit_t cli_resource_execute_from_params(pcmk__output_t *out, const char *rsc
                                             const char *rsc_type, const char *rsc_action,
                                             GHashTable *params, GHashTable *override_hash,
                                             int timeout_ms, int resource_verbose,
 -                                            gboolean force);
 +                                            gboolean force, int check_level);
 crm_exit_t cli_resource_execute(pe_resource_t *rsc, const char *requested_name,
                                 const char *rsc_action, GHashTable *override_hash,
                                 int timeout_ms, cib_t *cib, pe_working_set_t *data_set,
 -                                int resource_verbose, gboolean force);
 +                                int resource_verbose, gboolean force, int check_level);
 int cli_resource_update_attribute(pe_resource_t *rsc, const char *requested_name,
                                   const char *attr_set, const char *attr_set_type,
 diff --git a/tools/crm_resource_runtime.c b/tools/crm_resource_runtime.c
 index fe0ec98..bde83b6 100644
 --- a/tools/crm_resource_runtime.c
 +++ b/tools/crm_resource_runtime.c
@@ -1679,7 +1679,8 @@ cli_resource_execute_from_params(pcmk__output_t *out, const char *rsc_name,
                                  const char *rsc_class, const char *rsc_prov,
                                  const char *rsc_type, const char *action,
                                  GHashTable *params, GHashTable *override_hash,
 -                                 int timeout_ms, int resource_verbose, gboolean force)
 +                                 int timeout_ms, int resource_verbose, gboolean force,
 +                                 int check_level)
 {
     GHashTable *params_copy = NULL;
     crm_exit_t exit_code = CRM_EX_OK;
@@ -1703,6 +1704,12 @@ cli_resource_execute_from_params(pcmk__output_t *out, const char *rsc_name,
     /* add crm_feature_set env needed by some resource agents */
     g_hash_table_insert(params, strdup(XML_ATTR_CRM_VERSION), strdup(CRM_FEATURE_SET));
 +    if (check_level >= 0) {
 +        char *level = crm_strdup_printf("%d", check_level);
 +        setenv("OCF_CHECK_LEVEL", level, 1);
 +        free(level);
 +    }
 +
     /* resources_action_create frees the params hash table it's passed, but we
      * may need to reuse it in a second call to resources_action_create.  Thus
      * we'll make a copy here so that gets freed and the original remains for
@@ -1790,7 +1797,7 @@ crm_exit_t
 cli_resource_execute(pe_resource_t *rsc, const char *requested_name,
                      const char *rsc_action, GHashTable *override_hash,
                      int timeout_ms, cib_t * cib, pe_working_set_t *data_set,
 -                     int resource_verbose, gboolean force)
 +                     int resource_verbose, gboolean force, int check_level)
 {
     pcmk__output_t *out = data_set->priv;
     crm_exit_t exit_code = CRM_EX_OK;
@@ -1856,7 +1863,7 @@ cli_resource_execute(pe_resource_t *rsc, const char *requested_name,
     exit_code = cli_resource_execute_from_params(out, rid, rclass, rprov, rtype, action,
                                                  params, override_hash, timeout_ms,
 -                                                 resource_verbose, force);
 +                                                 resource_verbose, force, check_level);
     return exit_code;
 }
 -- 
 1.8.3.1
 From d13ba4bd6defe0dd81fdf8ab39ae5b889513c0c0 Mon Sep 17 00:00:00 2001
 From: Chris Lumens <clumens@redhat.com>
 Date: Thu, 20 May 2021 10:59:23 -0400
 Subject: [PATCH 2/2] Fix: include: Bump feature set to 3.10.2.
 This is for the OCF_CHECK_LEVEL environment variable.
 See: rhbz#1955792.
 ---
 include/crm/crm.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
 diff --git a/include/crm/crm.h b/include/crm/crm.h
 index 92a98fa..ee52c36 100644
 --- a/include/crm/crm.h
 +++ b/include/crm/crm.h
@@ -66,7 +66,7 @@ extern "C" {
  * >=3.0.13: Fail counts include operation name and interval
  * >=3.2.0:  DC supports PCMK_LRM_OP_INVALID and PCMK_LRM_OP_NOT_CONNECTED
  */
 -#  define CRM_FEATURE_SET		"3.10.1"
 +#  define CRM_FEATURE_SET		"3.10.2"
 /* Pacemaker's CPG protocols use fixed-width binary fields for the sender and
  * recipient of a CPG message. This imposes an arbitrary limit on cluster node
 -- 
 1.8.3.1
--- a/SOURCES/004-systemd-metadata.patch
+++ b/SOURCES/004-systemd-metadata.patch
@ -0,0 +1,73 @@
 From 09ef95a2eed48b4eb7488788a1b655d67eafe783 Mon Sep 17 00:00:00 2001
 From: Chris Lumens <clumens@redhat.com>
 Date: Tue, 30 Nov 2021 14:47:12 -0500
 Subject: [PATCH] Low: libcrmservice: Handle systemd service templates.
 These unit files (which have an @ sign at the end) expect to be
 parameterized by an instance name.  Not providing an instance name
 causes the dbus lookup to fail, and we fall back to assume this is an
 LSB service.  If the user doesn't provide an instance name, just add a
 fake one.  It doesn't seem to matter what name is given for the lookup.
 See: rhbz#2003151
 ---
 lib/services/systemd.c | 22 ++++++++++++++++------
 1 file changed, 16 insertions(+), 6 deletions(-)
 diff --git a/lib/services/systemd.c b/lib/services/systemd.c
 index 8e9fff484..27a3b376d 100644
 --- a/lib/services/systemd.c
 +++ b/lib/services/systemd.c
@@ -206,17 +206,27 @@ systemd_unit_extension(const char *name)
 }
 static char *
 -systemd_service_name(const char *name)
 +systemd_service_name(const char *name, bool add_instance_name)
 {
 -    if (name == NULL) {
 +    if (pcmk__str_empty(name)) {
         return NULL;
     }
     if (systemd_unit_extension(name)) {
         return strdup(name);
 -    }
 -    return crm_strdup_printf("%s.service", name);
 +    /* Services that end with an @ sign are systemd templates.  They expect an
 +     * instance name to follow the service name.  If no instance name was
 +     * provided, just add "x" to the string as the instance name.  It doesn't
 +     * seem to matter for purposes of looking up whether a service exists or
 +     * not.
 +     */
 +    } else if (add_instance_name && *(name+strlen(name)-1) == '@') {
 +        return crm_strdup_printf("%sx.service", name);
 +
 +    } else {
 +        return crm_strdup_printf("%s.service", name);
 +    }
 }
 static void
@@ -427,7 +437,7 @@ invoke_unit_by_name(const char *arg_name, svc_action_t *op, char **path)
     CRM_ASSERT(msg != NULL);
     // Add the (expanded) unit name as the argument
 -    name = systemd_service_name(arg_name);
 +    name = systemd_service_name(arg_name, op == NULL || pcmk__str_eq(op->action, "meta-data", pcmk__str_none));
     CRM_LOG_ASSERT(dbus_message_append_args(msg, DBUS_TYPE_STRING, &name,
                                             DBUS_TYPE_INVALID));
     free(name);
@@ -944,7 +954,7 @@ invoke_unit_by_path(svc_action_t *op, const char *unit)
     /* (ss) */
     {
         const char *replace_s = "replace";
 -        char *name = systemd_service_name(op->agent);
 +        char *name = systemd_service_name(op->agent, pcmk__str_eq(op->action, "meta-data", pcmk__str_none));
         CRM_LOG_ASSERT(dbus_message_append_args(msg, DBUS_TYPE_STRING, &name, DBUS_TYPE_INVALID));
         CRM_LOG_ASSERT(dbus_message_append_args(msg, DBUS_TYPE_STRING, &replace_s, DBUS_TYPE_INVALID));
 -- 
 2.27.0
--- a/SOURCES/005-crm_resource.patch
+++ b/SOURCES/005-crm_resource.patch
@ -1,866 +0,0 @@
 From a5a507d4e1abf242903472719a19977811e6f164 Mon Sep 17 00:00:00 2001
 From: Chris Lumens <clumens@redhat.com>
 Date: Thu, 20 May 2021 11:59:36 -0400
 Subject: [PATCH 01/10] Feature: libcrmcommon: Add OCF_OUTPUT_FORMAT to
 crm_resource environment.
 See: rhbz#1644628
 ---
 lib/common/output.c | 2 ++
 1 file changed, 2 insertions(+)
 diff --git a/lib/common/output.c b/lib/common/output.c
 index 6cb49b5..58872e0 100644
 --- a/lib/common/output.c
 +++ b/lib/common/output.c
@@ -71,6 +71,8 @@ pcmk__output_new(pcmk__output_t **out, const char *fmt_name, const char *filenam
         return ENOMEM;
     }
 +    setenv("OCF_OUTPUT_FORMAT", (*out)->fmt_name, 1);
 +
     return pcmk_rc_ok;
 }
 -- 
 1.8.3.1
 From acc6ecdbfb797d69794e68f75a734d6252434e01 Mon Sep 17 00:00:00 2001
 From: Chris Lumens <clumens@redhat.com>
 Date: Fri, 21 May 2021 14:20:30 -0400
 Subject: [PATCH 02/10] Feature: schemas: Copy crm_resource schema in
 preparation for changes.
 See: rhbz#1644628
 ---
 xml/api/crm_resource-2.11.rng | 238 ++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 238 insertions(+)
 create mode 100644 xml/api/crm_resource-2.11.rng
 diff --git a/xml/api/crm_resource-2.11.rng b/xml/api/crm_resource-2.11.rng
 new file mode 100644
 index 0000000..8e386db
 --- /dev/null
 +++ b/xml/api/crm_resource-2.11.rng
@@ -0,0 +1,238 @@
 +<?xml version="1.0" encoding="UTF-8"?>
 +<grammar xmlns="http://relaxng.org/ns/structure/1.0"
 +         datatypeLibrary="http://www.w3.org/2001/XMLSchema-datatypes">
 +
 +    <start>
 +        <ref name="element-crm-resource"/>
 +    </start>
 +
 +    <define name="element-crm-resource">
 +        <choice>
 +            <ref name="agents-list" />
 +            <ref name="alternatives-list" />
 +            <ref name="constraints-list" />
 +            <externalRef href="generic-list-2.4.rng"/>
 +            <element name="metadata"> <text/> </element>
 +            <ref name="locate-list" />
 +            <ref name="operations-list" />
 +            <ref name="providers-list" />
 +            <ref name="reasons-list" />
 +            <ref name="resource-check" />
 +            <ref name="resource-config" />
 +            <ref name="resources-list" />
 +        </choice>
 +    </define>
 +
 +    <define name="agents-list">
 +        <element name="agents">
 +            <attribute name="standard"> <text/> </attribute>
 +            <optional>
 +                <attribute name="provider"> <text/> </attribute>
 +            </optional>
 +            <zeroOrMore>
 +                <element name="agent"> <text/> </element>
 +            </zeroOrMore>
 +        </element>
 +    </define>
 +
 +    <define name="alternatives-list">
 +        <element name="providers">
 +            <attribute name="for"> <text/> </attribute>
 +            <zeroOrMore>
 +                <element name="provider"> <text/> </element>
 +            </zeroOrMore>
 +        </element>
 +    </define>
 +
 +    <define name="constraints-list">
 +        <element name="constraints">
 +            <interleave>
 +                <zeroOrMore>
 +                    <ref name="rsc-location" />
 +                </zeroOrMore>
 +                <zeroOrMore>
 +                    <ref name="rsc-colocation" />
 +                </zeroOrMore>
 +            </interleave>
 +        </element>
 +    </define>
 +
 +    <define name="locate-list">
 +        <element name="nodes">
 +            <attribute name="resource"> <text/> </attribute>
 +            <zeroOrMore>
 +                <element name="node">
 +                    <optional>
 +                        <attribute name="state"><value>promoted</value></attribute>
 +                    </optional>
 +                    <text/>
 +                </element>
 +            </zeroOrMore>
 +        </element>
 +    </define>
 +
 +    <define name="rsc-location">
 +        <element name="rsc_location">
 +            <attribute name="node"> <text/> </attribute>
 +            <attribute name="rsc"> <text/> </attribute>
 +            <attribute name="id"> <text/> </attribute>
 +            <externalRef href="../score.rng"/>
 +        </element>
 +    </define>
 +
 +    <define name="operations-list">
 +        <element name="operations">
 +            <oneOrMore>
 +                <ref name="element-operation-list" />
 +            </oneOrMore>
 +        </element>
 +    </define>
 +
 +    <define name="providers-list">
 +        <element name="providers">
 +            <attribute name="standard"> <value>ocf</value> </attribute>
 +            <optional>
 +                <attribute name="agent"> <text/> </attribute>
 +            </optional>
 +            <zeroOrMore>
 +                <element name="provider"> <text/> </element>
 +            </zeroOrMore>
 +        </element>
 +    </define>
 +
 +    <define name="reasons-list">
 +        <choice>
 +            <ref name="no-resource-or-uname"/>
 +            <ref name="resource-and-uname"/>
 +            <ref name="no-resource-but-uname"/>
 +            <ref name="resource-but-no-uname"/>
 +        </choice>
 +    </define>
 +
 +    <define name="no-resource-or-uname">
 +        <element name="reason">
 +            <element name="resources">
 +                <zeroOrMore>
 +                    <element name="resource">
 +                        <attribute name="id"> <text/> </attribute>
 +                        <attribute name="running"> <data type="boolean"/> </attribute>
 +                        <ref name="resource-check"/>
 +                    </element>
 +                </zeroOrMore>
 +            </element>
 +        </element>
 +    </define>
 +
 +    <define name="resource-and-uname">
 +        <element name="reason">
 +            <attribute name="running_on"> <text/> </attribute>
 +            <ref name="resource-check"/>
 +        </element>
 +    </define>
 +
 +    <define name="no-resource-but-uname">
 +        <element name="reason">
 +            <element name="resources">
 +                <zeroOrMore>
 +                    <element name="resource">
 +                        <attribute name="id"> <text/> </attribute>
 +                        <attribute name="running"> <data type="boolean"/> </attribute>
 +                        <attribute name="host"> <text/> </attribute>
 +                        <ref name="resource-check"/>
 +                    </element>
 +                </zeroOrMore>
 +            </element>
 +        </element>
 +    </define>
 +
 +    <define name="resource-but-no-uname">
 +        <element name="reason">
 +            <attribute name="running"> <data type="boolean"/> </attribute>
 +            <ref name="resource-check"/>
 +        </element>
 +    </define>
 +
 +    <define name="resource-config">
 +        <element name="resource_config">
 +            <externalRef href="resources-2.4.rng" />
 +            <element name="xml"> <text/> </element>
 +        </element>
 +    </define>
 +
 +    <define name="resource-check">
 +        <element name="check">
 +            <attribute name="id"> <text/> </attribute>
 +            <optional>
 +                <choice>
 +                    <attribute name="remain_stopped"><value>true</value></attribute>
 +                    <attribute name="promotable"><value>false</value></attribute>
 +                </choice>
 +            </optional>
 +            <optional>
 +                <attribute name="unmanaged"><value>true</value></attribute>
 +            </optional>
 +            <optional>
 +                <attribute name="locked-to"> <text/> </attribute>
 +            </optional>
 +        </element>
 +    </define>
 +
 +    <define name="resources-list">
 +        <element name="resources">
 +            <zeroOrMore>
 +                <externalRef href="resources-2.4.rng" />
 +            </zeroOrMore>
 +        </element>
 +    </define>
 +
 +    <define name="rsc-colocation">
 +        <element name="rsc_colocation">
 +            <attribute name="id"> <text/> </attribute>
 +            <attribute name="rsc"> <text/> </attribute>
 +            <attribute name="with-rsc"> <text/> </attribute>
 +            <externalRef href="../score.rng"/>
 +            <optional>
 +                <attribute name="node-attribute"> <text/> </attribute>
 +            </optional>
 +            <optional>
 +                <attribute name="rsc-role">
 +                    <ref name="attribute-roles"/>
 +                </attribute>
 +            </optional>
 +            <optional>
 +                <attribute name="with-rsc-role">
 +                    <ref name="attribute-roles"/>
 +                </attribute>
 +            </optional>
 +        </element>
 +    </define>
 +
 +    <define name="element-operation-list">
 +        <element name="operation">
 +            <optional>
 +                <group>
 +                    <attribute name="rsc"> <text/> </attribute>
 +                    <attribute name="agent"> <text/> </attribute>
 +                </group>
 +            </optional>
 +            <attribute name="op"> <text/> </attribute>
 +            <attribute name="node"> <text/> </attribute>
 +            <attribute name="call"> <data type="integer" /> </attribute>
 +            <attribute name="rc"> <data type="nonNegativeInteger" /> </attribute>
 +            <optional>
 +                <attribute name="last-rc-change"> <text/> </attribute>
 +                <attribute name="exec-time"> <data type="nonNegativeInteger" /> </attribute>
 +            </optional>
 +            <attribute name="status"> <text/> </attribute>
 +        </element>
 +    </define>
 +
 +    <define name="attribute-roles">
 +        <choice>
 +            <value>Stopped</value>
 +            <value>Started</value>
 +            <value>Master</value>
 +            <value>Slave</value>
 +        </choice>
 +    </define>
 +</grammar>
 -- 
 1.8.3.1
 From 1bbdf2149a111e9e19c388834f82001e0d31c427 Mon Sep 17 00:00:00 2001
 From: Chris Lumens <clumens@redhat.com>
 Date: Mon, 24 May 2021 12:23:55 -0400
 Subject: [PATCH 03/10] Feature: xml: Update the crm_resource schema for XML
 output.
 See: rhbz#1644628
 ---
 xml/api/crm_resource-2.11.rng | 50 +++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 50 insertions(+)
 diff --git a/xml/api/crm_resource-2.11.rng b/xml/api/crm_resource-2.11.rng
 index 8e386db..aaa54d6 100644
 --- a/xml/api/crm_resource-2.11.rng
 +++ b/xml/api/crm_resource-2.11.rng
@@ -20,6 +20,7 @@
             <ref name="resource-check" />
             <ref name="resource-config" />
             <ref name="resources-list" />
 +            <ref name="resource-agent-action" />
         </choice>
     </define>
@@ -227,6 +228,55 @@
         </element>
     </define>
 +    <define name="resource-agent-action">
 +        <element name="resource-agent-action">
 +            <attribute name="action"> <text/> </attribute>
 +            <optional>
 +                <attribute name="rsc"> <text/> </attribute>
 +            </optional>
 +            <attribute name="class"> <text/> </attribute>
 +            <attribute name="type"> <text/> </attribute>
 +            <optional>
 +                <attribute name="provider"> <text/> </attribute>
 +            </optional>
 +            <optional>
 +                <ref name="overrides-list"/>
 +            </optional>
 +            <ref name="agent-status"/>
 +            <optional>
 +                <choice>
 +                    <element name="command">
 +                        <text />
 +                    </element>
 +                    <externalRef href="command-output-1.0.rng"/>
 +                </choice>
 +            </optional>
 +        </element>
 +    </define>
 +
 +    <define name="overrides-list">
 +        <element name="overrides">
 +            <zeroOrMore>
 +                <element name="override">
 +                    <optional>
 +                        <attribute name="rsc"> <text/> </attribute>
 +                    </optional>
 +                    <attribute name="name"> <text/> </attribute>
 +                    <attribute name="value"> <text/> </attribute>
 +                </element>
 +            </zeroOrMore>
 +        </element>
 +    </define>
 +
 +    <define name="agent-status">
 +        <element name="agent-status">
 +            <attribute name="code"> <data type="integer" /> </attribute>
 +            <optional>
 +                <attribute name="message"> <text/> </attribute>
 +            </optional>
 +        </element>
 +    </define>
 +
     <define name="attribute-roles">
         <choice>
             <value>Stopped</value>
 -- 
 1.8.3.1
 From d89f5bc7fec856fdcd32fa14edbd0019507d5d15 Mon Sep 17 00:00:00 2001
 From: Chris Lumens <clumens@redhat.com>
 Date: Tue, 1 Jun 2021 15:26:58 -0400
 Subject: [PATCH 04/10] Low: libcrmcommon: Increase PCMK__API_VERSION for new
 crm_resource output.
 See: rhbz#1644628
 ---
 include/crm/common/output_internal.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
 diff --git a/include/crm/common/output_internal.h b/include/crm/common/output_internal.h
 index 10b315b..0436cde 100644
 --- a/include/crm/common/output_internal.h
 +++ b/include/crm/common/output_internal.h
@@ -27,7 +27,7 @@ extern "C" {
 #  include <glib.h>
 #  include <crm/common/results.h>
 -#  define PCMK__API_VERSION "2.9"
 +#  define PCMK__API_VERSION "2.11"
 #if defined(PCMK__WITH_ATTRIBUTE_OUTPUT_ARGS)
 #  define PCMK__OUTPUT_ARGS(ARGS...) __attribute__((output_args(ARGS)))
 -- 
 1.8.3.1
 From 30bd2ddf43ee2a911681e51f40ed9ba20ec250b0 Mon Sep 17 00:00:00 2001
 From: Chris Lumens <clumens@redhat.com>
 Date: Thu, 27 May 2021 13:57:12 -0400
 Subject: [PATCH 05/10] Low: tools: Pass NULL to
 cli_resource_execute_from_params...
 if no resource name is given.  This happens if we are validating based
 on the --class/--agent/--provider command line options instead.
 ---
 tools/crm_resource.c         | 2 +-
 tools/crm_resource_runtime.c | 8 ++++----
 2 files changed, 5 insertions(+), 5 deletions(-)
 diff --git a/tools/crm_resource.c b/tools/crm_resource.c
 index 24f1121..37a0bb0 100644
 --- a/tools/crm_resource.c
 +++ b/tools/crm_resource.c
@@ -1840,7 +1840,7 @@ main(int argc, char **argv)
         case cmd_execute_agent:
             if (options.cmdline_config) {
 -                exit_code = cli_resource_execute_from_params(out, "test",
 +                exit_code = cli_resource_execute_from_params(out, NULL,
                     options.v_class, options.v_provider, options.v_agent,
                     "validate-all", options.cmdline_params,
                     options.override_params, options.timeout_ms,
 diff --git a/tools/crm_resource_runtime.c b/tools/crm_resource_runtime.c
 index 48a4b40..ebf48bb 100644
 --- a/tools/crm_resource_runtime.c
 +++ b/tools/crm_resource_runtime.c
@@ -1717,14 +1717,14 @@ cli_resource_execute_from_params(pcmk__output_t *out, const char *rsc_name,
      */
     params_copy = pcmk__str_table_dup(params);
 -    op = resources_action_create(rsc_name, rsc_class, rsc_prov, rsc_type, action, 0,
 -                                 timeout_ms, params_copy, 0);
 +    op = resources_action_create(rsc_name ? rsc_name : "test", rsc_class, rsc_prov,
 +                                 rsc_type, action, 0, timeout_ms, params_copy, 0);
     if (op == NULL) {
         /* Re-run with stderr enabled so we can display a sane error message */
         crm_enable_stderr(TRUE);
         params_copy = pcmk__str_table_dup(params);
 -        op = resources_action_create(rsc_name, rsc_class, rsc_prov, rsc_type, action, 0,
 -                                     timeout_ms, params_copy, 0);
 +        op = resources_action_create(rsc_name ? rsc_name : "test", rsc_class, rsc_prov,
 +                                     rsc_type, action, 0, timeout_ms, params_copy, 0);
         /* Callers of cli_resource_execute expect that the params hash table will
          * be freed.  That function uses this one, so for that reason and for
 -- 
 1.8.3.1
 From ee56efd53d14cfc4f902769540b72b3bb6096a73 Mon Sep 17 00:00:00 2001
 From: Chris Lumens <clumens@redhat.com>
 Date: Mon, 24 May 2021 12:08:52 -0400
 Subject: [PATCH 06/10] Feature: tools: Add an agent-status message for
 crm_resource.
 This moves what was previously only done in an out->info call to its own
 output message, which means it will appear in XML output as well.  Also,
 note that if --class/--agent/--provider are given, the resource name
 will be set to "test".  In that case, do not display the resource name
 in the output.
 This message will be used for --validate and the --force-* command line
 options to crm_resource.
 See: rhbz#1644628
 ---
 tools/crm_resource_print.c | 53 ++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 53 insertions(+)
 diff --git a/tools/crm_resource_print.c b/tools/crm_resource_print.c
 index 9d82cf8..88d5878 100644
 --- a/tools/crm_resource_print.c
 +++ b/tools/crm_resource_print.c
@@ -152,6 +152,57 @@ attribute_list_default(pcmk__output_t *out, va_list args) {
     return pcmk_rc_ok;
 }
 +PCMK__OUTPUT_ARGS("agent-status", "int", "const char *", "const char *", "const char *",
 +                  "const char *", "const char *", "int")
 +static int
 +agent_status_default(pcmk__output_t *out, va_list args) {
 +    int status = va_arg(args, int);
 +    const char *action = va_arg(args, const char *);
 +    const char *name = va_arg(args, const char *);
 +    const char *class = va_arg(args, const char *);
 +    const char *provider = va_arg(args, const char *);
 +    const char *type = va_arg(args, const char *);
 +    int rc = va_arg(args, int);
 +
 +    if (status == PCMK_LRM_OP_DONE) {
 +        out->info(out, "Operation %s%s%s (%s%s%s:%s) returned: '%s' (%d)",
 +                  action, name ? " for " : "", name ? name : "",
 +                  class, provider ? ":" : "", provider ? provider : "", type,
 +                  services_ocf_exitcode_str(rc), rc);
 +    } else {
 +        out->err(out, "Operation %s%s%s (%s%s%s:%s) failed: '%s' (%d)",
 +                 action, name ? " for " : "", name ? name : "",
 +                 class, provider ? ":" : "", provider ? provider : "", type,
 +                 services_lrm_status_str(status), status);
 +    }
 +
 +    return pcmk_rc_ok;
 +}
 +
 +PCMK__OUTPUT_ARGS("agent-status", "int", "const char *", "const char *", "const char *",
 +                  "const char *", "const char *", "int")
 +static int
 +agent_status_xml(pcmk__output_t *out, va_list args) {
 +    int status G_GNUC_UNUSED = va_arg(args, int);
 +    const char *action G_GNUC_UNUSED = va_arg(args, const char *);
 +    const char *name G_GNUC_UNUSED = va_arg(args, const char *);
 +    const char *class G_GNUC_UNUSED = va_arg(args, const char *);
 +    const char *provider G_GNUC_UNUSED = va_arg(args, const char *);
 +    const char *type G_GNUC_UNUSED = va_arg(args, const char *);
 +    int rc = va_arg(args, int);
 +
 +    char *status_str = pcmk__itoa(rc);
 +
 +    pcmk__output_create_xml_node(out, "agent-status",
 +                                 "code", status_str,
 +                                 "message", services_ocf_exitcode_str(rc),
 +                                 NULL);
 +
 +    free(status_str);
 +
 +    return pcmk_rc_ok;
 +}
 +
 PCMK__OUTPUT_ARGS("attribute-list", "pe_resource_t *", "char *", "GHashTable *")
 static int
 attribute_list_text(pcmk__output_t *out, va_list args) {
@@ -562,6 +613,8 @@ resource_names(pcmk__output_t *out, va_list args) {
 }
 static pcmk__message_entry_t fmt_functions[] = {
 +    { "agent-status", "default", agent_status_default },
 +    { "agent-status", "xml", agent_status_xml },
     { "attribute-list", "default", attribute_list_default },
     { "attribute-list", "text", attribute_list_text },
     { "property-list", "default", property_list_default },
 -- 
 1.8.3.1
 From 85cb6b6bff96b18c5174d11e4de4d49cbfb20bb7 Mon Sep 17 00:00:00 2001
 From: Chris Lumens <clumens@redhat.com>
 Date: Tue, 1 Jun 2021 14:47:30 -0400
 Subject: [PATCH 07/10] Feature: tools: Add an overridden params output
 message.
 This also replaces what was previously being done in an out->info call
 with an output message.  This means it shows up in XML output as well.
 Also, note that if --class/--agent/--provider are given, the resource
 name will be set to "test".  In that case, do not display the resource
 name in the output.
 See: rhbz#1644628
 ---
 tools/crm_resource_print.c | 39 +++++++++++++++++++++++++++++++++++++++
 1 file changed, 39 insertions(+)
 diff --git a/tools/crm_resource_print.c b/tools/crm_resource_print.c
 index 88d5878..119d83f 100644
 --- a/tools/crm_resource_print.c
 +++ b/tools/crm_resource_print.c
@@ -224,6 +224,43 @@ attribute_list_text(pcmk__output_t *out, va_list args) {
     return pcmk_rc_ok;
 }
 +PCMK__OUTPUT_ARGS("override", "const char *", "const char *", "const char *")
 +static int
 +override_default(pcmk__output_t *out, va_list args) {
 +    const char *rsc_name = va_arg(args, const char *);
 +    const char *name = va_arg(args, const char *);
 +    const char *value = va_arg(args, const char *);
 +
 +    if (rsc_name == NULL) {
 +        out->list_item(out, NULL, "Overriding the cluster configuration with '%s' = '%s'",
 +                       name, value);
 +    } else {
 +        out->list_item(out, NULL, "Overriding the cluster configuration for '%s' with '%s' = '%s'",
 +                       rsc_name, name, value);
 +    }
 +
 +    return pcmk_rc_ok;
 +}
 +
 +PCMK__OUTPUT_ARGS("override", "const char *", "const char *", "const char *")
 +static int
 +override_xml(pcmk__output_t *out, va_list args) {
 +    const char *rsc_name = va_arg(args, const char *);
 +    const char *name = va_arg(args, const char *);
 +    const char *value = va_arg(args, const char *);
 +
 +    xmlNodePtr node = pcmk__output_create_xml_node(out, "override",
 +                                                   "name", name,
 +                                                   "value", value,
 +                                                   NULL);
 +
 +    if (rsc_name != NULL) {
 +        crm_xml_add(node, "rsc", rsc_name);
 +    }
 +
 +    return pcmk_rc_ok;
 +}
 +
 PCMK__OUTPUT_ARGS("property-list", "pe_resource_t *", "char *")
 static int
 property_list_default(pcmk__output_t *out, va_list args) {
@@ -617,6 +654,8 @@ static pcmk__message_entry_t fmt_functions[] = {
     { "agent-status", "xml", agent_status_xml },
     { "attribute-list", "default", attribute_list_default },
     { "attribute-list", "text", attribute_list_text },
 +    { "override", "default", override_default },
 +    { "override", "xml", override_xml },
     { "property-list", "default", property_list_default },
     { "property-list", "text", property_list_text },
     { "resource-check-list", "default", resource_check_list_default },
 -- 
 1.8.3.1
 From e5e24592c7c3231c619fb5253e7925ffbc634a99 Mon Sep 17 00:00:00 2001
 From: Chris Lumens <clumens@redhat.com>
 Date: Fri, 4 Jun 2021 10:24:51 -0400
 Subject: [PATCH 08/10] Low: tools: Use simple XML lists for resource actions
 as well.
 See: rhbz#1644628
 ---
 tools/crm_resource.c | 1 +
 1 file changed, 1 insertion(+)
 diff --git a/tools/crm_resource.c b/tools/crm_resource.c
 index 37a0bb0..e957011 100644
 --- a/tools/crm_resource.c
 +++ b/tools/crm_resource.c
@@ -1643,6 +1643,7 @@ main(int argc, char **argv)
          * saves from having to write custom messages to build the lists around all these things
          */
         switch (options.rsc_cmd) {
 +            case cmd_execute_agent:
             case cmd_list_resources:
             case cmd_query_xml:
             case cmd_query_raw_xml:
 -- 
 1.8.3.1
 From 3e75174d0bc31b261adb1994214a5878b79da85b Mon Sep 17 00:00:00 2001
 From: Chris Lumens <clumens@redhat.com>
 Date: Fri, 4 Jun 2021 10:30:10 -0400
 Subject: [PATCH 09/10] Feature: tools: Add an output message for resource
 actions.
 This wraps up the override and agent-status messages into a single
 message, along with any stdout/stderr from the resource action.  This
 message should be called after taking the action.
 This also implements handling XML output from resource actions.  Check
 to see if the validate-all action returns XML.  If so, output it as a
 CDATA block under a "command" element.  If not, treat it as plain text
 and output it as stdout/stderr from a command.
 See: rhbz#1644628
 ---
 tools/crm_resource_print.c | 122 +++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 122 insertions(+)
 diff --git a/tools/crm_resource_print.c b/tools/crm_resource_print.c
 index 119d83f..19a366d 100644
 --- a/tools/crm_resource_print.c
 +++ b/tools/crm_resource_print.c
@@ -293,6 +293,126 @@ property_list_text(pcmk__output_t *out, va_list args) {
     return pcmk_rc_ok;
 }
 +PCMK__OUTPUT_ARGS("resource-agent-action", "int", "const char *", "const char *",
 +                  "const char *", "const char *", "const char *", "GHashTable *",
 +                  "int", "int", "char *", "char *")
 +static int
 +resource_agent_action_default(pcmk__output_t *out, va_list args) {
 +    int verbose = va_arg(args, int);
 +
 +    const char *class = va_arg(args, const char *);
 +    const char *provider = va_arg(args, const char *);
 +    const char *type = va_arg(args, const char *);
 +    const char *rsc_name = va_arg(args, const char *);
 +    const char *action = va_arg(args, const char *);
 +    GHashTable *overrides = va_arg(args, GHashTable *);
 +    int rc = va_arg(args, int);
 +    int status = va_arg(args, int);
 +    char *stdout_data = va_arg(args, char *);
 +    char *stderr_data = va_arg(args, char *);
 +
 +    if (overrides) {
 +        GHashTableIter iter;
 +        char *name = NULL;
 +        char *value = NULL;
 +
 +        out->begin_list(out, NULL, NULL, "overrides");
 +
 +        g_hash_table_iter_init(&iter, overrides);
 +        while (g_hash_table_iter_next(&iter, (gpointer *) &name, (gpointer *) &value)) {
 +            out->message(out, "override", rsc_name, name, value);
 +        }
 +
 +        out->end_list(out);
 +    }
 +
 +    out->message(out, "agent-status", status, action, rsc_name, class, provider,
 +                 type, rc);
 +
 +    /* hide output for validate-all if not in verbose */
 +    if (verbose == 0 && pcmk__str_eq(action, "validate-all", pcmk__str_casei)) {
 +        return pcmk_rc_ok;
 +    }
 +
 +    if (stdout_data || stderr_data) {
 +        xmlNodePtr doc = string2xml(stdout_data);
 +
 +        if (doc != NULL) {
 +            out->output_xml(out, "command", stdout_data);
 +            xmlFreeNode(doc);
 +        } else {
 +            out->subprocess_output(out, rc, stdout_data, stderr_data);
 +        }
 +    }
 +
 +    return pcmk_rc_ok;
 +}
 +
 +PCMK__OUTPUT_ARGS("resource-agent-action", "int", "const char *", "const char *",
 +                  "const char *", "const char *", "const char *", "GHashTable *",
 +                  "int", "int", "char *", "char *")
 +static int
 +resource_agent_action_xml(pcmk__output_t *out, va_list args) {
 +    int verbose G_GNUC_UNUSED = va_arg(args, int);
 +
 +    const char *class = va_arg(args, const char *);
 +    const char *provider = va_arg(args, const char *);
 +    const char *type = va_arg(args, const char *);
 +    const char *rsc_name = va_arg(args, const char *);
 +    const char *action = va_arg(args, const char *);
 +    GHashTable *overrides = va_arg(args, GHashTable *);
 +    int rc = va_arg(args, int);
 +    int status = va_arg(args, int);
 +    char *stdout_data = va_arg(args, char *);
 +    char *stderr_data = va_arg(args, char *);
 +
 +    xmlNodePtr node = pcmk__output_xml_create_parent(out, "resource-agent-action",
 +                                                     "action", action,
 +                                                     "class", class,
 +                                                     "type", type,
 +                                                     NULL);
 +
 +    if (rsc_name) {
 +        crm_xml_add(node, "rsc", rsc_name);
 +    }
 +
 +    if (provider) {
 +        crm_xml_add(node, "provider", provider);
 +    }
 +
 +    if (overrides) {
 +        GHashTableIter iter;
 +        char *name = NULL;
 +        char *value = NULL;
 +
 +        out->begin_list(out, NULL, NULL, "overrides");
 +
 +        g_hash_table_iter_init(&iter, overrides);
 +        while (g_hash_table_iter_next(&iter, (gpointer *) &name, (gpointer *) &value)) {
 +            out->message(out, "override", rsc_name, name, value);
 +        }
 +
 +        out->end_list(out);
 +    }
 +
 +    out->message(out, "agent-status", status, action, rsc_name, class, provider,
 +                 type, rc);
 +
 +    if (stdout_data || stderr_data) {
 +        xmlNodePtr doc = string2xml(stdout_data);
 +
 +        if (doc != NULL) {
 +            out->output_xml(out, "command", stdout_data);
 +            xmlFreeNode(doc);
 +        } else {
 +            out->subprocess_output(out, rc, stdout_data, stderr_data);
 +        }
 +    }
 +
 +    pcmk__output_xml_pop_parent(out);
 +    return pcmk_rc_ok;
 +}
 +
 PCMK__OUTPUT_ARGS("resource-check-list", "resource_checks_t *")
 static int
 resource_check_list_default(pcmk__output_t *out, va_list args) {
@@ -658,6 +778,8 @@ static pcmk__message_entry_t fmt_functions[] = {
     { "override", "xml", override_xml },
     { "property-list", "default", property_list_default },
     { "property-list", "text", property_list_text },
 +    { "resource-agent-action", "default", resource_agent_action_default },
 +    { "resource-agent-action", "xml", resource_agent_action_xml },
     { "resource-check-list", "default", resource_check_list_default },
     { "resource-check-list", "xml", resource_check_list_xml },
     { "resource-search-list", "default", resource_search_list_default },
 -- 
 1.8.3.1
 From b50b2418e1e997b42f5370b4672a3f105d74634f Mon Sep 17 00:00:00 2001
 From: Chris Lumens <clumens@redhat.com>
 Date: Fri, 4 Jun 2021 10:40:16 -0400
 Subject: [PATCH 10/10] Feature: tools: Use the new resource-agent-action
 message.
 See: rhbz#1644628
 ---
 tools/crm_resource_runtime.c | 21 +++------------------
 1 file changed, 3 insertions(+), 18 deletions(-)
 diff --git a/tools/crm_resource_runtime.c b/tools/crm_resource_runtime.c
 index ebf48bb..755be9f 100644
 --- a/tools/crm_resource_runtime.c
 +++ b/tools/crm_resource_runtime.c
@@ -1765,28 +1765,13 @@ cli_resource_execute_from_params(pcmk__output_t *out, const char *rsc_name,
     if (services_action_sync(op)) {
         exit_code = op->rc;
 -        if (op->status == PCMK_LRM_OP_DONE) {
 -            out->info(out, "Operation %s for %s (%s:%s:%s) returned: '%s' (%d)",
 -                      action, rsc_name, rsc_class, rsc_prov ? rsc_prov : "", rsc_type,
 -                      services_ocf_exitcode_str(op->rc), op->rc);
 -        } else {
 -            out->err(out, "Operation %s for %s (%s:%s:%s) failed: '%s' (%d)",
 -                     action, rsc_name, rsc_class, rsc_prov ? rsc_prov : "", rsc_type,
 -                     services_lrm_status_str(op->status), op->status);
 -        }
 -
 -        /* hide output for validate-all if not in verbose */
 -        if (resource_verbose == 0 && pcmk__str_eq(action, "validate-all", pcmk__str_casei))
 -            goto done;
 -
 -        if (op->stdout_data || op->stderr_data) {
 -            out->subprocess_output(out, op->rc, op->stdout_data, op->stderr_data);
 -        }
 +        out->message(out, "resource-agent-action", resource_verbose, rsc_class,
 +                     rsc_prov, rsc_type, rsc_name, action, override_hash, op->rc,
 +                     op->status, op->stdout_data, op->stderr_data);
     } else {
         exit_code = op->rc == 0 ? CRM_EX_ERROR : op->rc;
     }
 -done:
     services_action_free(op);
     /* See comment above about why we free params here. */
     g_hash_table_destroy(params);
 -- 
 1.8.3.1
--- a/SOURCES/005-fencing-reasons.patch
+++ b/SOURCES/005-fencing-reasons.patch
--- a/SOURCES/006-crm_simulate.patch
+++ b/SOURCES/006-crm_simulate.patch
@ -1,896 +0,0 @@
 From 97571e6ccc9b7fa339a7e27d9b0b9ab782ff3003 Mon Sep 17 00:00:00 2001
 From: Chris Lumens <clumens@redhat.com>
 Date: Wed, 16 Jun 2021 13:54:10 -0400
 Subject: [PATCH 1/5] Low: schemas: Copy crm_mon.rng in preparation for
 changes.
 ---
 xml/api/crm_mon-2.12.rng | 243 +++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 243 insertions(+)
 create mode 100644 xml/api/crm_mon-2.12.rng
 diff --git a/xml/api/crm_mon-2.12.rng b/xml/api/crm_mon-2.12.rng
 new file mode 100644
 index 0000000..ffec923
 --- /dev/null
 +++ b/xml/api/crm_mon-2.12.rng
@@ -0,0 +1,243 @@
 +<?xml version="1.0" encoding="UTF-8"?>
 +<grammar xmlns="http://relaxng.org/ns/structure/1.0"
 +         datatypeLibrary="http://www.w3.org/2001/XMLSchema-datatypes">
 +
 +    <start>
 +        <ref name="element-crm-mon"/>
 +    </start>
 +
 +    <define name="element-crm-mon">
 +        <optional>
 +            <ref name="element-summary" />
 +        </optional>
 +        <optional>
 +            <ref name="nodes-list" />
 +        </optional>
 +        <optional>
 +            <ref name="resources-list" />
 +        </optional>
 +        <optional>
 +            <ref name="node-attributes-list" />
 +        </optional>
 +        <optional>
 +            <ref name="node-history-list" />
 +        </optional>
 +        <optional>
 +            <ref name="failures-list" />
 +        </optional>
 +        <optional>
 +            <ref name="fence-event-list" />
 +        </optional>
 +        <optional>
 +            <ref name="tickets-list" />
 +        </optional>
 +        <optional>
 +            <ref name="bans-list" />
 +        </optional>
 +    </define>
 +
 +    <define name="element-summary">
 +        <element name="summary">
 +            <optional>
 +                <element name="stack">
 +                    <attribute name="type"> <text /> </attribute>
 +                </element>
 +            </optional>
 +            <optional>
 +                <element name="current_dc">
 +                    <attribute name="present"> <data type="boolean" /> </attribute>
 +                    <optional>
 +                        <group>
 +                            <attribute name="version"> <text /> </attribute>
 +                            <attribute name="name"> <text /> </attribute>
 +                            <attribute name="id"> <text /> </attribute>
 +                            <attribute name="with_quorum"> <data type="boolean" /> </attribute>
 +                        </group>
 +                    </optional>
 +                </element>
 +            </optional>
 +            <optional>
 +                <element name="last_update">
 +                    <attribute name="time"> <text /> </attribute>
 +                </element>
 +                <element name="last_change">
 +                    <attribute name="time"> <text /> </attribute>
 +                    <attribute name="user"> <text /> </attribute>
 +                    <attribute name="client"> <text /> </attribute>
 +                    <attribute name="origin"> <text /> </attribute>
 +                </element>
 +            </optional>
 +            <optional>
 +                <element name="nodes_configured">
 +                    <attribute name="number"> <data type="nonNegativeInteger" /> </attribute>
 +                </element>
 +                <element name="resources_configured">
 +                    <attribute name="number"> <data type="nonNegativeInteger" /> </attribute>
 +                    <attribute name="disabled"> <data type="nonNegativeInteger" /> </attribute>
 +                    <attribute name="blocked"> <data type="nonNegativeInteger" /> </attribute>
 +                </element>
 +            </optional>
 +            <optional>
 +                <element name="cluster_options">
 +                    <attribute name="stonith-enabled"> <data type="boolean" /> </attribute>
 +                    <attribute name="symmetric-cluster"> <data type="boolean" /> </attribute>
 +                    <attribute name="no-quorum-policy"> <text /> </attribute>
 +                    <attribute name="maintenance-mode"> <data type="boolean" /> </attribute>
 +                    <attribute name="stop-all-resources"> <data type="boolean" /> </attribute>
 +                </element>
 +            </optional>
 +        </element>
 +    </define>
 +
 +    <define name="resources-list">
 +        <element name="resources">
 +            <zeroOrMore>
 +                <externalRef href="resources-2.4.rng" />
 +            </zeroOrMore>
 +        </element>
 +    </define>
 +
 +    <define name="nodes-list">
 +        <element name="nodes">
 +            <zeroOrMore>
 +                <externalRef href="nodes-2.8.rng" />
 +            </zeroOrMore>
 +        </element>
 +    </define>
 +
 +    <define name="node-attributes-list">
 +        <element name="node_attributes">
 +            <zeroOrMore>
 +                <externalRef href="node-attrs-2.8.rng" />
 +            </zeroOrMore>
 +        </element>
 +    </define>
 +
 +    <define name="node-history-list">
 +        <element name="node_history">
 +            <zeroOrMore>
 +                <ref name="element-node-history" />
 +            </zeroOrMore>
 +        </element>
 +    </define>
 +
 +    <define name="failures-list">
 +        <element name="failures">
 +            <zeroOrMore>
 +                <externalRef href="failure-2.8.rng" />
 +            </zeroOrMore>
 +        </element>
 +    </define>
 +
 +    <define name="fence-event-list">
 +        <element name="fence_history">
 +            <optional>
 +                <attribute name="status"> <data type="integer" /> </attribute>
 +            </optional>
 +            <zeroOrMore>
 +                <externalRef href="fence-event-2.0.rng" />
 +            </zeroOrMore>
 +        </element>
 +    </define>
 +
 +    <define name="tickets-list">
 +        <element name="tickets">
 +            <zeroOrMore>
 +                <ref name="element-ticket" />
 +            </zeroOrMore>
 +        </element>
 +    </define>
 +
 +    <define name="bans-list">
 +        <element name="bans">
 +            <zeroOrMore>
 +                <ref name="element-ban" />
 +            </zeroOrMore>
 +        </element>
 +    </define>
 +
 +    <define name="element-node-history">
 +        <element name="node">
 +            <attribute name="name"> <text /> </attribute>
 +            <zeroOrMore>
 +                <ref name="element-resource-history" />
 +            </zeroOrMore>
 +        </element>
 +    </define>
 +
 +    <define name="element-resource-history">
 +        <element name="resource_history">
 +            <attribute name="id"> <text /> </attribute>
 +            <attribute name="orphan"> <data type="boolean" /> </attribute>
 +            <optional>
 +                <group>
 +                    <attribute name="migration-threshold"> <data type="nonNegativeInteger" /> </attribute>
 +                    <optional>
 +                        <attribute name="fail-count"> <text /> </attribute>
 +                    </optional>
 +                    <optional>
 +                        <attribute name="last-failure"> <text /> </attribute>
 +                    </optional>
 +                </group>
 +            </optional>
 +            <zeroOrMore>
 +                <ref name="element-operation-history" />
 +            </zeroOrMore>
 +        </element>
 +    </define>
 +
 +    <define name="element-operation-history">
 +        <element name="operation_history">
 +            <attribute name="call"> <text /> </attribute>
 +            <attribute name="task"> <text /> </attribute>
 +            <optional>
 +                <attribute name="interval"> <text /> </attribute>
 +            </optional>
 +            <optional>
 +                <attribute name="last-rc-change"> <text /> </attribute>
 +            </optional>
 +            <optional>
 +                <attribute name="last-run"> <text /> </attribute>
 +            </optional>
 +            <optional>
 +                <attribute name="exec-time"> <text /> </attribute>
 +            </optional>
 +            <optional>
 +                <attribute name="queue-time"> <text /> </attribute>
 +            </optional>
 +            <attribute name="rc"> <data type="integer" /> </attribute>
 +            <attribute name="rc_text"> <text /> </attribute>
 +        </element>
 +    </define>
 +
 +    <define name="element-ticket">
 +        <element name="ticket">
 +            <attribute name="id"> <text /> </attribute>
 +            <attribute name="status">
 +                <choice>
 +                    <value>granted</value>
 +                    <value>revoked</value>
 +                </choice>
 +            </attribute>
 +            <attribute name="standby"> <data type="boolean" /> </attribute>
 +            <optional>
 +                <attribute name="last-granted"> <text /> </attribute>
 +            </optional>
 +        </element>
 +    </define>
 +
 +    <define name="element-ban">
 +        <element name="ban">
 +            <attribute name="id"> <text /> </attribute>
 +            <attribute name="resource"> <text /> </attribute>
 +            <attribute name="node"> <text /> </attribute>
 +            <attribute name="weight"> <data type="integer" /> </attribute>
 +            <attribute name="promoted-only"> <data type="boolean" /> </attribute>
 +            <!-- DEPRECATED: master_only is a duplicate of promoted-only that is
 +                 provided solely for API backward compatibility. It will be
 +                 removed in a future release. Check promoted-only instead.
 +              -->
 +            <attribute name="master_only"> <data type="boolean" /> </attribute>
 +        </element>
 +    </define>
 +</grammar>
 -- 
 1.8.3.1
 From da394983f106f974274ddd94675a04c85086010e Mon Sep 17 00:00:00 2001
 From: Chris Lumens <clumens@redhat.com>
 Date: Fri, 18 Jun 2021 15:06:34 -0400
 Subject: [PATCH 2/5] Refactor: Split node history out into its own XML schema.
 This allows for sharing it between crm_mon and crm_simulate.
 ---
 xml/Makefile.am               |  2 +-
 xml/api/crm_mon-2.12.rng      | 64 +--------------------------------------
 xml/api/node-history-2.12.rng | 70 +++++++++++++++++++++++++++++++++++++++++++
 3 files changed, 72 insertions(+), 64 deletions(-)
 create mode 100644 xml/api/node-history-2.12.rng
 diff --git a/xml/Makefile.am b/xml/Makefile.am
 index b9448d4..8e7b6d3 100644
 --- a/xml/Makefile.am
 +++ b/xml/Makefile.am
@@ -64,7 +64,7 @@ API_request_base	= command-output	\
 CIB_cfg_base		= options nodes resources constraints fencing acls tags alerts
 # Names of all schemas (including top level and those included by others)
 -API_base		= $(API_request_base) fence-event failure generic-list item node-attrs nodes resources status
 +API_base		= $(API_request_base) fence-event failure generic-list item node-attrs node-history nodes resources status
 CIB_base		= cib $(CIB_cfg_base) status score rule nvset
 # Static schema files and transforms (only CIB has transforms)
 diff --git a/xml/api/crm_mon-2.12.rng b/xml/api/crm_mon-2.12.rng
 index ffec923..be14412 100644
 --- a/xml/api/crm_mon-2.12.rng
 +++ b/xml/api/crm_mon-2.12.rng
@@ -20,7 +20,7 @@
             <ref name="node-attributes-list" />
         </optional>
         <optional>
 -            <ref name="node-history-list" />
 +            <externalRef href="node-history-2.12.rng"/>
         </optional>
         <optional>
             <ref name="failures-list" />
@@ -113,14 +113,6 @@
         </element>
     </define>
 -    <define name="node-history-list">
 -        <element name="node_history">
 -            <zeroOrMore>
 -                <ref name="element-node-history" />
 -            </zeroOrMore>
 -        </element>
 -    </define>
 -
     <define name="failures-list">
         <element name="failures">
             <zeroOrMore>
@@ -156,60 +148,6 @@
         </element>
     </define>
 -    <define name="element-node-history">
 -        <element name="node">
 -            <attribute name="name"> <text /> </attribute>
 -            <zeroOrMore>
 -                <ref name="element-resource-history" />
 -            </zeroOrMore>
 -        </element>
 -    </define>
 -
 -    <define name="element-resource-history">
 -        <element name="resource_history">
 -            <attribute name="id"> <text /> </attribute>
 -            <attribute name="orphan"> <data type="boolean" /> </attribute>
 -            <optional>
 -                <group>
 -                    <attribute name="migration-threshold"> <data type="nonNegativeInteger" /> </attribute>
 -                    <optional>
 -                        <attribute name="fail-count"> <text /> </attribute>
 -                    </optional>
 -                    <optional>
 -                        <attribute name="last-failure"> <text /> </attribute>
 -                    </optional>
 -                </group>
 -            </optional>
 -            <zeroOrMore>
 -                <ref name="element-operation-history" />
 -            </zeroOrMore>
 -        </element>
 -    </define>
 -
 -    <define name="element-operation-history">
 -        <element name="operation_history">
 -            <attribute name="call"> <text /> </attribute>
 -            <attribute name="task"> <text /> </attribute>
 -            <optional>
 -                <attribute name="interval"> <text /> </attribute>
 -            </optional>
 -            <optional>
 -                <attribute name="last-rc-change"> <text /> </attribute>
 -            </optional>
 -            <optional>
 -                <attribute name="last-run"> <text /> </attribute>
 -            </optional>
 -            <optional>
 -                <attribute name="exec-time"> <text /> </attribute>
 -            </optional>
 -            <optional>
 -                <attribute name="queue-time"> <text /> </attribute>
 -            </optional>
 -            <attribute name="rc"> <data type="integer" /> </attribute>
 -            <attribute name="rc_text"> <text /> </attribute>
 -        </element>
 -    </define>
 -
     <define name="element-ticket">
         <element name="ticket">
             <attribute name="id"> <text /> </attribute>
 diff --git a/xml/api/node-history-2.12.rng b/xml/api/node-history-2.12.rng
 new file mode 100644
 index 0000000..9628000
 --- /dev/null
 +++ b/xml/api/node-history-2.12.rng
@@ -0,0 +1,70 @@
 +<?xml version="1.0" encoding="UTF-8"?>
 +<grammar xmlns="http://relaxng.org/ns/structure/1.0"
 +         datatypeLibrary="http://www.w3.org/2001/XMLSchema-datatypes">
 +
 +    <start>
 +        <ref name="node-history-list" />
 +    </start>
 +
 +    <define name="node-history-list">
 +        <element name="node_history">
 +            <zeroOrMore>
 +                <ref name="element-node-history" />
 +            </zeroOrMore>
 +        </element>
 +    </define>
 +
 +    <define name="element-node-history">
 +        <element name="node">
 +            <attribute name="name"> <text /> </attribute>
 +            <zeroOrMore>
 +                <ref name="element-resource-history" />
 +            </zeroOrMore>
 +        </element>
 +    </define>
 +
 +    <define name="element-resource-history">
 +        <element name="resource_history">
 +            <attribute name="id"> <text /> </attribute>
 +            <attribute name="orphan"> <data type="boolean" /> </attribute>
 +            <optional>
 +                <group>
 +                    <attribute name="migration-threshold"> <data type="nonNegativeInteger" /> </attribute>
 +                    <optional>
 +                        <attribute name="fail-count"> <text /> </attribute>
 +                    </optional>
 +                    <optional>
 +                        <attribute name="last-failure"> <text /> </attribute>
 +                    </optional>
 +                </group>
 +            </optional>
 +            <zeroOrMore>
 +                <ref name="element-operation-history" />
 +            </zeroOrMore>
 +        </element>
 +    </define>
 +
 +    <define name="element-operation-history">
 +        <element name="operation_history">
 +            <attribute name="call"> <text /> </attribute>
 +            <attribute name="task"> <text /> </attribute>
 +            <optional>
 +                <attribute name="interval"> <text /> </attribute>
 +            </optional>
 +            <optional>
 +                <attribute name="last-rc-change"> <text /> </attribute>
 +            </optional>
 +            <optional>
 +                <attribute name="last-run"> <text /> </attribute>
 +            </optional>
 +            <optional>
 +                <attribute name="exec-time"> <text /> </attribute>
 +            </optional>
 +            <optional>
 +                <attribute name="queue-time"> <text /> </attribute>
 +            </optional>
 +            <attribute name="rc"> <data type="integer" /> </attribute>
 +            <attribute name="rc_text"> <text /> </attribute>
 +        </element>
 +    </define>
 +</grammar>
 -- 
 1.8.3.1
 From bf72b2615630eef7876e443d60b34d5a316de847 Mon Sep 17 00:00:00 2001
 From: Chris Lumens <clumens@redhat.com>
 Date: Wed, 16 Jun 2021 14:09:31 -0400
 Subject: [PATCH 3/5] Low: schemas: Copy crm_simulate.rng in preparation for
 changes.
 ---
 xml/api/crm_simulate-2.12.rng | 335 ++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 335 insertions(+)
 create mode 100644 xml/api/crm_simulate-2.12.rng
 diff --git a/xml/api/crm_simulate-2.12.rng b/xml/api/crm_simulate-2.12.rng
 new file mode 100644
 index 0000000..9a7612d
 --- /dev/null
 +++ b/xml/api/crm_simulate-2.12.rng
@@ -0,0 +1,335 @@
 +<?xml version="1.0" encoding="UTF-8"?>
 +<grammar xmlns="http://relaxng.org/ns/structure/1.0"
 +         datatypeLibrary="http://www.w3.org/2001/XMLSchema-datatypes">
 +
 +    <start>
 +        <ref name="element-crm-simulate"/>
 +    </start>
 +
 +    <define name="element-crm-simulate">
 +        <choice>
 +            <ref name="timings-list" />
 +            <group>
 +                <ref name="cluster-status" />
 +                <optional>
 +                    <ref name="modifications-list" />
 +                </optional>
 +                <optional>
 +                    <ref name="allocations-utilizations-list" />
 +                </optional>
 +                <optional>
 +                    <ref name="action-list" />
 +                </optional>
 +                <optional>
 +                    <ref name="cluster-injected-actions-list" />
 +                    <ref name="revised-cluster-status" />
 +                </optional>
 +            </group>
 +        </choice>
 +    </define>
 +
 +    <define name="allocations-utilizations-list">
 +        <choice>
 +            <element name="allocations">
 +                <zeroOrMore>
 +                    <choice>
 +                        <ref name="element-allocation" />
 +                        <ref name="element-promotion" />
 +                    </choice>
 +                </zeroOrMore>
 +            </element>
 +            <element name="utilizations">
 +                <zeroOrMore>
 +                    <choice>
 +                        <ref name="element-capacity" />
 +                        <ref name="element-utilization" />
 +                    </choice>
 +                </zeroOrMore>
 +            </element>
 +            <element name="allocations_utilizations">
 +                <zeroOrMore>
 +                    <choice>
 +                        <ref name="element-allocation" />
 +                        <ref name="element-promotion" />
 +                        <ref name="element-capacity" />
 +                        <ref name="element-utilization" />
 +                    </choice>
 +                </zeroOrMore>
 +            </element>
 +        </choice>
 +    </define>
 +
 +    <define name="cluster-status">
 +        <element name="cluster_status">
 +            <ref name="nodes-list" />
 +            <ref name="resources-list" />
 +            <optional>
 +                <ref name="node-attributes-list" />
 +            </optional>
 +            <optional>
 +                <ref name="failures-list" />
 +            </optional>
 +        </element>
 +    </define>
 +
 +    <define name="modifications-list">
 +        <element name="modifications">
 +            <optional>
 +                <attribute name="quorum"> <text /> </attribute>
 +            </optional>
 +            <optional>
 +                <attribute name="watchdog"> <text /> </attribute>
 +            </optional>
 +            <zeroOrMore>
 +                <ref name="element-inject-modify-node" />
 +            </zeroOrMore>
 +            <zeroOrMore>
 +                <ref name="element-inject-modify-ticket" />
 +            </zeroOrMore>
 +            <zeroOrMore>
 +                <ref name="element-inject-spec" />
 +            </zeroOrMore>
 +            <zeroOrMore>
 +                <ref name="element-inject-attr" />
 +            </zeroOrMore>
 +        </element>
 +    </define>
 +
 +    <define name="revised-cluster-status">
 +        <element name="revised_cluster_status">
 +            <ref name="nodes-list" />
 +            <ref name="resources-list" />
 +            <optional>
 +                <ref name="node-attributes-list" />
 +            </optional>
 +            <optional>
 +                <ref name="failures-list" />
 +            </optional>
 +        </element>
 +    </define>
 +
 +    <define name="element-inject-attr">
 +        <element name="inject_attr">
 +            <attribute name="cib_node"> <text /> </attribute>
 +            <attribute name="name"> <text /> </attribute>
 +            <attribute name="node_path"> <text /> </attribute>
 +            <attribute name="value"> <text /> </attribute>
 +        </element>
 +    </define>
 +
 +    <define name="element-inject-modify-node">
 +        <element name="modify_node">
 +            <attribute name="action"> <text /> </attribute>
 +            <attribute name="node"> <text /> </attribute>
 +        </element>
 +    </define>
 +
 +    <define name="element-inject-spec">
 +        <element name="inject_spec">
 +            <attribute name="spec"> <text /> </attribute>
 +        </element>
 +    </define>
 +
 +    <define name="element-inject-modify-ticket">
 +        <element name="modify_ticket">
 +            <attribute name="action"> <text /> </attribute>
 +            <attribute name="ticket"> <text /> </attribute>
 +        </element>
 +    </define>
 +
 +    <define name="cluster-injected-actions-list">
 +        <element name="transition">
 +            <zeroOrMore>
 +                <ref name="element-injected-actions" />
 +            </zeroOrMore>
 +        </element>
 +    </define>
 +
 +    <define name="node-attributes-list">
 +        <element name="node_attributes">
 +            <zeroOrMore>
 +                <externalRef href="node-attrs-2.8.rng" />
 +            </zeroOrMore>
 +        </element>
 +    </define>
 +
 +    <define name="failures-list">
 +        <element name="failures">
 +            <zeroOrMore>
 +                <externalRef href="failure-2.8.rng" />
 +            </zeroOrMore>
 +        </element>
 +    </define>
 +
 +    <define name="nodes-list">
 +        <element name="nodes">
 +            <zeroOrMore>
 +                <externalRef href="nodes-2.8.rng" />
 +            </zeroOrMore>
 +        </element>
 +    </define>
 +
 +    <define name="resources-list">
 +        <element name="resources">
 +            <zeroOrMore>
 +                <externalRef href="resources-2.4.rng" />
 +            </zeroOrMore>
 +        </element>
 +    </define>
 +
 +    <define name="timings-list">
 +        <element name="timings">
 +            <zeroOrMore>
 +                <ref name="element-timing" />
 +            </zeroOrMore>
 +        </element>
 +    </define>
 +
 +    <define name="action-list">
 +        <element name="actions">
 +            <zeroOrMore>
 +                <ref name="element-node-action" />
 +            </zeroOrMore>
 +            <zeroOrMore>
 +                <ref name="element-rsc-action" />
 +            </zeroOrMore>
 +        </element>
 +    </define>
 +
 +    <define name="element-allocation">
 +        <element name="node_weight">
 +            <attribute name="function"> <text /> </attribute>
 +            <attribute name="node"> <text /> </attribute>
 +            <externalRef href="../score.rng" />
 +            <optional>
 +                <attribute name="id"> <text /> </attribute>
 +            </optional>
 +        </element>
 +    </define>
 +
 +    <define name="element-capacity">
 +        <element name="capacity">
 +            <attribute name="comment"> <text /> </attribute>
 +            <attribute name="node"> <text /> </attribute>
 +            <zeroOrMore>
 +                <element>
 +                    <anyName />
 +                    <text />
 +                </element>
 +            </zeroOrMore>
 +        </element>
 +    </define>
 +
 +    <define name="element-inject-cluster-action">
 +        <element name="cluster_action">
 +            <attribute name="node"> <text /> </attribute>
 +            <attribute name="task"> <text /> </attribute>
 +            <optional>
 +                <attribute name="id"> <text /> </attribute>
 +            </optional>
 +        </element>
 +    </define>
 +
 +    <define name="element-injected-actions">
 +        <choice>
 +            <ref name="element-inject-cluster-action" />
 +            <ref name="element-inject-fencing-action" />
 +            <ref name="element-inject-pseudo-action" />
 +            <ref name="element-inject-rsc-action" />
 +        </choice>
 +    </define>
 +
 +    <define name="element-inject-fencing-action">
 +        <element name="fencing_action">
 +            <attribute name="op"> <text /> </attribute>
 +            <attribute name="target"> <text /> </attribute>
 +        </element>
 +    </define>
 +
 +    <define name="element-node-action">
 +        <element name="node_action">
 +            <attribute name="node"> <text /> </attribute>
 +            <attribute name="reason"> <text /> </attribute>
 +            <attribute name="task"> <text /> </attribute>
 +        </element>
 +    </define>
 +
 +    <define name="element-promotion">
 +        <element name="promotion_score">
 +            <attribute name="id"> <text /> </attribute>
 +            <externalRef href="../score.rng" />
 +            <optional>
 +                <attribute name="node"> <text /> </attribute>
 +            </optional>
 +        </element>
 +    </define>
 +
 +    <define name="element-inject-pseudo-action">
 +        <element name="pseudo_action">
 +            <attribute name="task"> <text /> </attribute>
 +            <optional>
 +                <attribute name="node"> <text /> </attribute>
 +            </optional>
 +        </element>
 +    </define>
 +
 +    <define name="element-inject-rsc-action">
 +        <element name="rsc_action">
 +            <attribute name="node"> <text /> </attribute>
 +            <attribute name="op"> <text /> </attribute>
 +            <attribute name="resource"> <text /> </attribute>
 +            <optional>
 +                <attribute name="interval"> <data type="integer" /> </attribute>
 +            </optional>
 +        </element>
 +    </define>
 +
 +    <define name="element-timing">
 +        <element name="timing">
 +            <attribute name="file"> <text /> </attribute>
 +            <attribute name="duration"> <data type="double" /> </attribute>
 +        </element>
 +    </define>
 +
 +    <define name="element-rsc-action">
 +        <element name="rsc_action">
 +            <attribute name="action"> <text /> </attribute>
 +            <attribute name="resource"> <text /> </attribute>
 +            <optional>
 +                <attribute name="blocked"> <data type="boolean" /> </attribute>
 +            </optional>
 +            <optional>
 +                <attribute name="dest"> <text /> </attribute>
 +            </optional>
 +            <optional>
 +                <attribute name="next-role"> <text /> </attribute>
 +            </optional>
 +            <optional>
 +                <attribute name="node"> <text /> </attribute>
 +            </optional>
 +            <optional>
 +                <attribute name="reason"> <text /> </attribute>
 +            </optional>
 +            <optional>
 +                <attribute name="role"> <text /> </attribute>
 +            </optional>
 +            <optional>
 +                <attribute name="source"> <text /> </attribute>
 +            </optional>
 +        </element>
 +    </define>
 +
 +    <define name="element-utilization">
 +        <element name="utilization">
 +            <attribute name="function"> <text /> </attribute>
 +            <attribute name="node"> <text /> </attribute>
 +            <attribute name="resource"> <text /> </attribute>
 +            <zeroOrMore>
 +                <element>
 +                    <anyName />
 +                    <text />
 +                </element>
 +            </zeroOrMore>
 +        </element>
 +    </define>
 +</grammar>
 -- 
 1.8.3.1
 From c46e07788788acf5669e3f89b9344190a91c7331 Mon Sep 17 00:00:00 2001
 From: Chris Lumens <clumens@redhat.com>
 Date: Fri, 18 Jun 2021 15:10:19 -0400
 Subject: [PATCH 4/5] Feature: tools: Add the node-summary to crm_simulate
 output.
 If --show-failcounts is given to crm_simulate, it should also display
 the node-summary message.
 See: rhbz#1686426
 ---
 tools/crm_simulate.c          | 7 +++++--
 xml/api/crm_simulate-2.12.rng | 3 +++
 2 files changed, 8 insertions(+), 2 deletions(-)
 diff --git a/tools/crm_simulate.c b/tools/crm_simulate.c
 index b4aa9d1..2ea292c 100644
 --- a/tools/crm_simulate.c
 +++ b/tools/crm_simulate.c
@@ -409,11 +409,14 @@ print_cluster_status(pe_working_set_t * data_set, unsigned int print_opts)
                       FALSE, FALSE, all, all, FALSE);
     if (options.show_attrs) {
 -        out->message(out, "node-attribute-list", data_set,
 -                     0, rc == pcmk_rc_ok, FALSE, FALSE, FALSE, all, all);
 +        rc = out->message(out, "node-attribute-list", data_set,
 +                          0, rc == pcmk_rc_ok, FALSE, FALSE, FALSE, all, all);
     }
     if (options.show_failcounts) {
 +        rc = out->message(out, "node-summary", data_set, all, all,
 +                          0, print_opts, FALSE, FALSE, FALSE, FALSE, rc == pcmk_rc_ok);
 +
         out->message(out, "failed-action-list", data_set, all, all,
                      rc == pcmk_rc_ok);
     }
 diff --git a/xml/api/crm_simulate-2.12.rng b/xml/api/crm_simulate-2.12.rng
 index 9a7612d..f90bd36 100644
 --- a/xml/api/crm_simulate-2.12.rng
 +++ b/xml/api/crm_simulate-2.12.rng
@@ -67,6 +67,9 @@
                 <ref name="node-attributes-list" />
             </optional>
             <optional>
 +                <externalRef href="node-history-2.12.rng" />
 +            </optional>
 +            <optional>
                 <ref name="failures-list" />
             </optional>
         </element>
 -- 
 1.8.3.1
 From bac50336e0264604716e5997b87ee7e65311b982 Mon Sep 17 00:00:00 2001
 From: Chris Lumens <clumens@redhat.com>
 Date: Fri, 18 Jun 2021 15:21:52 -0400
 Subject: [PATCH 5/5] Low: libcrmcommon: Increase PCMK__API_VERSION for new
 crm_resource output.
 See: rhbz#1686426
 ---
 include/crm/common/output_internal.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
 diff --git a/include/crm/common/output_internal.h b/include/crm/common/output_internal.h
 index 0436cde..ba9c423 100644
 --- a/include/crm/common/output_internal.h
 +++ b/include/crm/common/output_internal.h
@@ -27,7 +27,7 @@ extern "C" {
 #  include <glib.h>
 #  include <crm/common/results.h>
 -#  define PCMK__API_VERSION "2.11"
 +#  define PCMK__API_VERSION "2.12"
 #if defined(PCMK__WITH_ATTRIBUTE_OUTPUT_ARGS)
 #  define PCMK__OUTPUT_ARGS(ARGS...) __attribute__((output_args(ARGS)))
 -- 
 1.8.3.1
--- a/SOURCES/006-stateful-metadata.patch
+++ b/SOURCES/006-stateful-metadata.patch
@ -0,0 +1,143 @@
 From b52fe799c89637e2a761a5725c2376db5c05f2d1 Mon Sep 17 00:00:00 2001
 From: Ken Gaillot <kgaillot@redhat.com>
 Date: Tue, 30 Nov 2021 15:51:54 -0600
 Subject: [PATCH 1/2] Low: resources: remove DOCTYPE from OCF 1.1-compliant
 agents
 OCF 1.1 replaced the DTD schema with RNG, but DOCTYPE still refers to the DTD.
 There's no DOCTYPE for RNG, and DOCTYPE is optional, so just remove it.
 ---
 extra/resources/Dummy        | 3 +--
 extra/resources/HealthIOWait | 3 +--
 extra/resources/Stateful     | 3 +--
 extra/resources/attribute    | 3 +--
 extra/resources/ping         | 3 +--
 extra/resources/remote       | 3 +--
 6 files changed, 6 insertions(+), 12 deletions(-)
 diff --git a/extra/resources/Dummy b/extra/resources/Dummy
 index a344deac0..56584e564 100755
 --- a/extra/resources/Dummy
 +++ b/extra/resources/Dummy
@@ -58,8 +58,7 @@
 meta_data() {
     cat <<END
 <?xml version="1.0"?>
 -<!DOCTYPE resource-agent SYSTEM "ra-api-1.dtd">
 -<resource-agent name="Dummy" version="2.0">
 +<resource-agent name="Dummy" version="2.1">
 <version>1.1</version>
 <longdesc lang="en">
 diff --git a/extra/resources/HealthIOWait b/extra/resources/HealthIOWait
 index 43a8b70c4..5f1483ef7 100755
 --- a/extra/resources/HealthIOWait
 +++ b/extra/resources/HealthIOWait
@@ -25,8 +25,7 @@
 meta_data() {
         cat <<END
 <?xml version="1.0"?>
 -<!DOCTYPE resource-agent SYSTEM "ra-api-1.dtd">
 -<resource-agent name="HealthIOWait" version="1.1">
 +<resource-agent name="HealthIOWait" version="1.2">
 <version>1.1</version>
 <longdesc lang="en">
 diff --git a/extra/resources/Stateful b/extra/resources/Stateful
 index ae3424bbf..0d2062d51 100755
 --- a/extra/resources/Stateful
 +++ b/extra/resources/Stateful
@@ -39,8 +39,7 @@ SCORE_PROMOTED=10
 meta_data() {
     cat <<END
 <?xml version="1.0"?>
 -<!DOCTYPE resource-agent SYSTEM "ra-api-1.dtd">
 -<resource-agent name="Stateful" version="1.1">
 +<resource-agent name="Stateful" version="1.2">
 <version>1.1</version>
 <longdesc lang="en">
 diff --git a/extra/resources/attribute b/extra/resources/attribute
 index 1800dff8f..a2bd353e0 100755
 --- a/extra/resources/attribute
 +++ b/extra/resources/attribute
@@ -57,8 +57,7 @@ END
 meta_data() {
     cat <<END
 <?xml version="1.0"?>
 -<!DOCTYPE resource-agent SYSTEM "ra-api-1.dtd">
 -<resource-agent name="attribute" version="1.1">
 +<resource-agent name="attribute" version="1.2">
   <version>1.1</version>
   <shortdesc lang="en">Manages a node attribute</shortdesc>
   <longdesc lang="en">
 diff --git a/extra/resources/ping b/extra/resources/ping
 index 6e296979f..7cc6b802d 100755
 --- a/extra/resources/ping
 +++ b/extra/resources/ping
@@ -36,8 +36,7 @@
 meta_data() {
      cat <<END
 <?xml version="1.0"?>
 -<!DOCTYPE resource-agent SYSTEM "ra-api-1.dtd">
 -<resource-agent name="ping" version="1.1">
 +<resource-agent name="ping" version="1.2">
 <version>1.1</version>
 <longdesc lang="en">
 diff --git a/extra/resources/remote b/extra/resources/remote
 index a53262bb6..f7e40dc81 100755
 --- a/extra/resources/remote
 +++ b/extra/resources/remote
@@ -24,8 +24,7 @@
 meta_data() {
     cat <<END
 <?xml version="1.0"?>
 -<!DOCTYPE resource-agent SYSTEM "ra-api-1.dtd">
 -<resource-agent name="remote" version="1.0">
 +<resource-agent name="remote" version="1.1">
   <version>1.1</version>
   <shortdesc lang="en">Pacemaker Remote connection</shortdesc>
   <parameters>
 -- 
 2.27.0
 From 70f469120f8db6a024c786466ee74a6c7fbd1f43 Mon Sep 17 00:00:00 2001
 From: Ken Gaillot <kgaillot@redhat.com>
 Date: Tue, 30 Nov 2021 15:53:39 -0600
 Subject: [PATCH 2/2] Fix: resources: use correct syntax in Stateful meta-data
 The OCF standard only allows "0" or "1" for booleans.
 This fixes incorrect ocf:pacemaker:Stateful meta-data syntax introduced by
 7024398 as a regression in the 2.1.0 release.
 ---
 extra/resources/Stateful | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)
 diff --git a/extra/resources/Stateful b/extra/resources/Stateful
 index 0d2062d51..2ebe6725f 100755
 --- a/extra/resources/Stateful
 +++ b/extra/resources/Stateful
@@ -57,7 +57,7 @@ Location to store the resource state in
 <content type="string" default="${HA_VARRUN%%/}/Stateful-${OCF_RESOURCE_INSTANCE}.state" />
 </parameter>
 -<parameter name="envfile" reloadable="true">
 +<parameter name="envfile" reloadable="1">
 <longdesc lang="en">
 If this is set, the environment will be dumped to this file for every call.
 </longdesc>
@@ -65,7 +65,7 @@ If this is set, the environment will be dumped to this file for every call.
 <content type="string" default="" />
 </parameter>
 -<parameter name="notify_delay" reloadable="true">
 +<parameter name="notify_delay" reloadable="1">
 <longdesc lang="en">
 The notify action will sleep for this many seconds before returning,
 to simulate a long-running notify.
 -- 
 2.27.0
--- a/SOURCES/007-memory-leak.patch
+++ b/SOURCES/007-memory-leak.patch
@ -0,0 +1,39 @@
 From f491d9d5a7ed554fed985de356bb085fdec3421c Mon Sep 17 00:00:00 2001
 From: Ken Gaillot <kgaillot@redhat.com>
 Date: Tue, 7 Dec 2021 09:01:00 -0600
 Subject: [PATCH] Fix: fencer: avoid memory leak when broadcasting history
 differences
 Regression introduced in 2.1.0 by dbc27b2
 ---
 daemons/fenced/fenced_history.c | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)
 diff --git a/daemons/fenced/fenced_history.c b/daemons/fenced/fenced_history.c
 index bc159383c..a9c57dc86 100644
 --- a/daemons/fenced/fenced_history.c
 +++ b/daemons/fenced/fenced_history.c
@@ -484,8 +484,6 @@ stonith_fence_history(xmlNode *msg, xmlNode **output,
                    !pcmk__str_eq(remote_peer, stonith_our_uname, pcmk__str_casei)) {
             xmlNode *history = get_xpath_object("//" F_STONITH_HISTORY_LIST,
                                                 msg, LOG_NEVER);
 -            GHashTable *received_history =
 -                history?stonith_xml_history_to_list(history):NULL;
             /* either a broadcast created directly upon stonith-API request
             * or a diff as response to such a thing
@@ -497,6 +495,11 @@ stonith_fence_history(xmlNode *msg, xmlNode **output,
             if (!history ||
                 !crm_is_true(crm_element_value(history,
                                                F_STONITH_DIFFERENTIAL))) {
 +                GHashTable *received_history = NULL;
 +
 +                if (history != NULL) {
 +                    received_history = stonith_xml_history_to_list(history);
 +                }
                 out_history =
                     stonith_local_history_diff_and_merge(received_history, TRUE, NULL);
                 if (out_history) {
 -- 
 2.27.0
--- a/SOURCES/007-unfencing-loop.patch
+++ b/SOURCES/007-unfencing-loop.patch
@ -1,733 +0,0 @@
 From 6dcd6b51d7d3993bc483588d6ed75077518ed600 Mon Sep 17 00:00:00 2001
 From: Ken Gaillot <kgaillot@redhat.com>
 Date: Fri, 4 Jun 2021 16:30:55 -0500
 Subject: [PATCH 01/11] Low: controller: check whether unfenced node was remote
 node
 ... so the controller can indicate the node is remote (if known at that point,
 which is not guaranteed) when setting unfencing-related node attributes.
 ---
 daemons/controld/controld_fencing.c | 21 ++++++++++++++++++---
 1 file changed, 18 insertions(+), 3 deletions(-)
 diff --git a/daemons/controld/controld_fencing.c b/daemons/controld/controld_fencing.c
 index 23dff28..0fba661 100644
 --- a/daemons/controld/controld_fencing.c
 +++ b/daemons/controld/controld_fencing.c
@@ -757,15 +757,30 @@ tengine_stonith_callback(stonith_t *stonith, stonith_callback_data_t *data)
             if (pcmk__str_eq("on", op, pcmk__str_casei)) {
                 const char *value = NULL;
                 char *now = pcmk__ttoa(time(NULL));
 +                gboolean is_remote_node = FALSE;
 +
 +                /* This check is not 100% reliable, since this node is not
 +                 * guaranteed to have the remote node cached. However, it
 +                 * doesn't have to be reliable, since the attribute manager can
 +                 * learn a node's "remoteness" by other means sooner or later.
 +                 * This allows it to learn more quickly if this node does have
 +                 * the information.
 +                 */
 +                if (g_hash_table_lookup(crm_remote_peer_cache, uuid) != NULL) {
 +                    is_remote_node = TRUE;
 +                }
 -                update_attrd(target, CRM_ATTR_UNFENCED, now, NULL, FALSE);
 +                update_attrd(target, CRM_ATTR_UNFENCED, now, NULL,
 +                             is_remote_node);
                 free(now);
                 value = crm_meta_value(action->params, XML_OP_ATTR_DIGESTS_ALL);
 -                update_attrd(target, CRM_ATTR_DIGESTS_ALL, value, NULL, FALSE);
 +                update_attrd(target, CRM_ATTR_DIGESTS_ALL, value, NULL,
 +                             is_remote_node);
                 value = crm_meta_value(action->params, XML_OP_ATTR_DIGESTS_SECURE);
 -                update_attrd(target, CRM_ATTR_DIGESTS_SECURE, value, NULL, FALSE);
 +                update_attrd(target, CRM_ATTR_DIGESTS_SECURE, value, NULL,
 +                             is_remote_node);
             } else if (action->sent_update == FALSE) {
                 send_stonith_update(action, target, uuid);
 -- 
 1.8.3.1
 From 3ef6d9403f68ab8559c45cc99f5a8da05ca6420b Mon Sep 17 00:00:00 2001
 From: Ken Gaillot <kgaillot@redhat.com>
 Date: Mon, 7 Jun 2021 10:50:36 -0500
 Subject: [PATCH 02/11] Refactor: pacemaker-attrd: functionize adding remote
 node to cache
 ... for future reuse
 ---
 daemons/attrd/attrd_commands.c | 34 +++++++++++++++++++++++-----------
 1 file changed, 23 insertions(+), 11 deletions(-)
 diff --git a/daemons/attrd/attrd_commands.c b/daemons/attrd/attrd_commands.c
 index 731c243..93a165b 100644
 --- a/daemons/attrd/attrd_commands.c
 +++ b/daemons/attrd/attrd_commands.c
@@ -102,6 +102,28 @@ free_attribute(gpointer data)
     }
 }
 +/*!
 + * \internal
 + * \brief Ensure a Pacemaker Remote node is in the correct peer cache
 + *
 + * \param[in]
 + */
 +static void
 +cache_remote_node(const char *node_name)
 +{
 +    /* If we previously assumed this node was an unseen cluster node,
 +     * remove its entry from the cluster peer cache.
 +     */
 +    crm_node_t *dup = pcmk__search_cluster_node_cache(0, node_name);
 +
 +    if (dup && (dup->uuid == NULL)) {
 +        reap_crm_member(0, node_name);
 +    }
 +
 +    // Ensure node is in the remote peer cache
 +    CRM_ASSERT(crm_remote_peer_get(node_name) != NULL);
 +}
 +
 static xmlNode *
 build_attribute_xml(
     xmlNode *parent, const char *name, const char *set, const char *uuid, unsigned int timeout_ms, const char *user,
@@ -709,17 +731,7 @@ attrd_lookup_or_create_value(GHashTable *values, const char *host, xmlNode *xml)
     crm_element_value_int(xml, PCMK__XA_ATTR_IS_REMOTE, &is_remote);
     if (is_remote) {
 -        /* If we previously assumed this node was an unseen cluster node,
 -         * remove its entry from the cluster peer cache.
 -         */
 -        crm_node_t *dup = pcmk__search_cluster_node_cache(0, host);
 -
 -        if (dup && (dup->uuid == NULL)) {
 -            reap_crm_member(0, host);
 -        }
 -
 -        /* Ensure this host is in the remote peer cache */
 -        CRM_ASSERT(crm_remote_peer_get(host) != NULL);
 +        cache_remote_node(host);
     }
     if (v == NULL) {
 -- 
 1.8.3.1
 From 6fac2c71bc2c56870ac828d7cd7b7c799279c47e Mon Sep 17 00:00:00 2001
 From: Ken Gaillot <kgaillot@redhat.com>
 Date: Mon, 7 Jun 2021 10:39:34 -0500
 Subject: [PATCH 03/11] Refactor: pacemaker-attrd: don't try to remove votes
 for remote nodes
 Remote nodes never vote.
 This has no effect in practice since the removal would simply do nothing,
 but we might as well not waste time trying.
 ---
 daemons/attrd/attrd_commands.c | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)
 diff --git a/daemons/attrd/attrd_commands.c b/daemons/attrd/attrd_commands.c
 index 93a165b..dbe777e 100644
 --- a/daemons/attrd/attrd_commands.c
 +++ b/daemons/attrd/attrd_commands.c
@@ -976,7 +976,8 @@ attrd_election_cb(gpointer user_data)
 void
 attrd_peer_change_cb(enum crm_status_type kind, crm_node_t *peer, const void *data)
 {
 -    bool remove_voter = FALSE;
 +    bool gone = false;
 +    bool is_remote = pcmk_is_set(peer->flags, crm_remote_node);
     switch (kind) {
         case crm_status_uname:
@@ -984,7 +985,7 @@ attrd_peer_change_cb(enum crm_status_type kind, crm_node_t *peer, const void *da
         case crm_status_processes:
             if (!pcmk_is_set(peer->processes, crm_get_cluster_proc())) {
 -                remove_voter = TRUE;
 +                gone = true;
             }
             break;
@@ -1000,13 +1001,13 @@ attrd_peer_change_cb(enum crm_status_type kind, crm_node_t *peer, const void *da
             } else {
                 // Remove all attribute values associated with lost nodes
                 attrd_peer_remove(peer->uname, FALSE, "loss");
 -                remove_voter = TRUE;
 +                gone = true;
             }
             break;
     }
 -    // In case an election is in progress, remove any vote by the node
 -    if (remove_voter) {
 +    // Remove votes from cluster nodes that leave, in case election in progress
 +    if (gone && !is_remote) {
         attrd_remove_voter(peer);
     }
 }
 -- 
 1.8.3.1
 From 54089fc663d6aaf10ca164c6c94b3b17237788de Mon Sep 17 00:00:00 2001
 From: Ken Gaillot <kgaillot@redhat.com>
 Date: Mon, 7 Jun 2021 10:40:06 -0500
 Subject: [PATCH 04/11] Low: pacemaker-attrd: check for remote nodes in peer
 update callback
 If a remote node was started before the local cluster node joined the cluster,
 the cluster node will assume its node attributes are for a cluster node until
 it learns otherwise. Check for remoteness in the peer update callback, to have
 another way we can learn it.
 ---
 daemons/attrd/attrd_commands.c | 4 ++++
 1 file changed, 4 insertions(+)
 diff --git a/daemons/attrd/attrd_commands.c b/daemons/attrd/attrd_commands.c
 index dbe777e..5f6a754 100644
 --- a/daemons/attrd/attrd_commands.c
 +++ b/daemons/attrd/attrd_commands.c
@@ -1009,6 +1009,10 @@ attrd_peer_change_cb(enum crm_status_type kind, crm_node_t *peer, const void *da
     // Remove votes from cluster nodes that leave, in case election in progress
     if (gone && !is_remote) {
         attrd_remove_voter(peer);
 +
 +    // Ensure remote nodes that come up are in the remote node cache
 +    } else if (!gone && is_remote) {
 +        cache_remote_node(peer->uname);
     }
 }
 -- 
 1.8.3.1
 From 8c048df0312d0d9c857d87b570a352429a710928 Mon Sep 17 00:00:00 2001
 From: Ken Gaillot <kgaillot@redhat.com>
 Date: Mon, 7 Jun 2021 11:29:12 -0500
 Subject: [PATCH 05/11] Log: pacemaker-attrd: log peer status changes
 ---
 daemons/attrd/attrd_commands.c | 9 +++++++++
 1 file changed, 9 insertions(+)
 diff --git a/daemons/attrd/attrd_commands.c b/daemons/attrd/attrd_commands.c
 index 5f6a754..d6d179b 100644
 --- a/daemons/attrd/attrd_commands.c
 +++ b/daemons/attrd/attrd_commands.c
@@ -972,6 +972,7 @@ attrd_election_cb(gpointer user_data)
     return FALSE;
 }
 +#define state_text(state) ((state)? (const char *)(state) : "in unknown state")
 void
 attrd_peer_change_cb(enum crm_status_type kind, crm_node_t *peer, const void *data)
@@ -981,15 +982,23 @@ attrd_peer_change_cb(enum crm_status_type kind, crm_node_t *peer, const void *da
     switch (kind) {
         case crm_status_uname:
 +            crm_debug("%s node %s is now %s",
 +                      (is_remote? "Remote" : "Cluster"),
 +                      peer->uname, state_text(peer->state));
             break;
         case crm_status_processes:
             if (!pcmk_is_set(peer->processes, crm_get_cluster_proc())) {
                 gone = true;
             }
 +            crm_debug("Node %s is %s a peer",
 +                      peer->uname, (gone? "no longer" : "now"));
             break;
         case crm_status_nstate:
 +            crm_debug("%s node %s is now %s (was %s)",
 +                      (is_remote? "Remote" : "Cluster"),
 +                      peer->uname, state_text(peer->state), state_text(data));
             if (pcmk__str_eq(peer->state, CRM_NODE_MEMBER, pcmk__str_casei)) {
                 /* If we're the writer, send new peers a list of all attributes
                  * (unless it's a remote node, which doesn't run its own attrd)
 -- 
 1.8.3.1
 From 1dcc8dee4990cf0dbdec0e14db6d9a3ad67a41d5 Mon Sep 17 00:00:00 2001
 From: Ken Gaillot <kgaillot@redhat.com>
 Date: Mon, 7 Jun 2021 11:13:53 -0500
 Subject: [PATCH 06/11] Low: pacemaker-attrd: ensure node ID is only set for
 attributes when known
 In most cases, attribute updates contained the node ID, and the node ID was
 used by other code, only if known (i.e. positive). However a couple places did
 not check this, so add that.
 I am unsure whether the missing check caused problems in practice, but there
 appears to be the possibility that a remote node would wrongly be added to the
 cluster node cache.
 ---
 daemons/attrd/attrd_commands.c | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)
 diff --git a/daemons/attrd/attrd_commands.c b/daemons/attrd/attrd_commands.c
 index d6d179b..b3f441c 100644
 --- a/daemons/attrd/attrd_commands.c
 +++ b/daemons/attrd/attrd_commands.c
@@ -136,7 +136,9 @@ build_attribute_xml(
     crm_xml_add(xml, PCMK__XA_ATTR_UUID, uuid);
     crm_xml_add(xml, PCMK__XA_ATTR_USER, user);
     crm_xml_add(xml, PCMK__XA_ATTR_NODE_NAME, peer);
 -    crm_xml_add_int(xml, PCMK__XA_ATTR_NODE_ID, peerid);
 +    if (peerid > 0) {
 +        crm_xml_add_int(xml, PCMK__XA_ATTR_NODE_ID, peerid);
 +    }
     crm_xml_add(xml, PCMK__XA_ATTR_VALUE, value);
     crm_xml_add_int(xml, PCMK__XA_ATTR_DAMPENING, timeout_ms/1000);
     crm_xml_add_int(xml, PCMK__XA_ATTR_IS_PRIVATE, is_private);
@@ -937,7 +939,7 @@ attrd_peer_update(crm_node_t *peer, xmlNode *xml, const char *host, bool filter)
     /* If this is a cluster node whose node ID we are learning, remember it */
     if ((v->nodeid == 0) && (v->is_remote == FALSE)
         && (crm_element_value_int(xml, PCMK__XA_ATTR_NODE_ID,
 -                                  (int*)&v->nodeid) == 0)) {
 +                                  (int*)&v->nodeid) == 0) && (v->nodeid > 0)) {
         crm_node_t *known_peer = crm_get_peer(v->nodeid, host);
 -- 
 1.8.3.1
 From 8d12490e88b558d01db37a38f7d35175c6d2d69a Mon Sep 17 00:00:00 2001
 From: Ken Gaillot <kgaillot@redhat.com>
 Date: Thu, 10 Jun 2021 17:25:57 -0500
 Subject: [PATCH 07/11] Refactor: pacemaker-attrd: functionize processing a
 sync response
 ... for code isolation, and because we need to add more to it
 ---
 daemons/attrd/attrd_commands.c | 59 ++++++++++++++++++++++++++++--------------
 1 file changed, 39 insertions(+), 20 deletions(-)
 diff --git a/daemons/attrd/attrd_commands.c b/daemons/attrd/attrd_commands.c
 index b3f441c..d02d3e6 100644
 --- a/daemons/attrd/attrd_commands.c
 +++ b/daemons/attrd/attrd_commands.c
@@ -572,6 +572,43 @@ attrd_peer_clear_failure(crm_node_t *peer, xmlNode *xml)
 }
 /*!
 + * \internal
 + * \brief Load attributes from a peer sync response
 + *
 + * \param[in] peer      Peer that sent clear request
 + * \param[in] peer_won  Whether peer is the attribute writer
 + * \param[in] xml       Request XML
 + */
 +static void
 +process_peer_sync_response(crm_node_t *peer, bool peer_won, xmlNode *xml)
 +{
 +    crm_info("Processing " PCMK__ATTRD_CMD_SYNC_RESPONSE " from %s",
 +             peer->uname);
 +
 +    if (peer_won) {
 +        /* Initialize the "seen" flag for all attributes to cleared, so we can
 +         * detect attributes that local node has but the writer doesn't.
 +         */
 +        clear_attribute_value_seen();
 +    }
 +
 +    // Process each attribute update in the sync response
 +    for (xmlNode *child = pcmk__xml_first_child(xml); child != NULL;
 +         child = pcmk__xml_next(child)) {
 +        attrd_peer_update(peer, child,
 +                          crm_element_value(child, PCMK__XA_ATTR_NODE_NAME),
 +                          TRUE);
 +    }
 +
 +    if (peer_won) {
 +        /* If any attributes are still not marked as seen, the writer doesn't
 +         * know about them, so send all peers an update with them.
 +         */
 +        attrd_current_only_attribute_update(peer, xml);
 +    }
 +}
 +
 +/*!
     \internal
     \brief Broadcast private attribute for local node with protocol version
 */
@@ -596,7 +633,7 @@ attrd_peer_message(crm_node_t *peer, xmlNode *xml)
     const char *op = crm_element_value(xml, PCMK__XA_TASK);
     const char *election_op = crm_element_value(xml, F_CRM_TASK);
     const char *host = crm_element_value(xml, PCMK__XA_ATTR_NODE_NAME);
 -    bool peer_won = FALSE;
 +    bool peer_won = false;
     if (election_op) {
         attrd_handle_election_op(peer, xml);
@@ -631,25 +668,7 @@ attrd_peer_message(crm_node_t *peer, xmlNode *xml)
     } else if (pcmk__str_eq(op, PCMK__ATTRD_CMD_SYNC_RESPONSE, pcmk__str_casei)
                && !pcmk__str_eq(peer->uname, attrd_cluster->uname, pcmk__str_casei)) {
 -        xmlNode *child = NULL;
 -
 -        crm_info("Processing %s from %s", op, peer->uname);
 -
 -        /* Clear the seen flag for attribute processing held only in the own node. */
 -        if (peer_won) {
 -            clear_attribute_value_seen();
 -        }
 -
 -        for (child = pcmk__xml_first_child(xml); child != NULL;
 -             child = pcmk__xml_next(child)) {
 -            host = crm_element_value(child, PCMK__XA_ATTR_NODE_NAME);
 -            attrd_peer_update(peer, child, host, TRUE);
 -        }
 -
 -        if (peer_won) {
 -            /* Synchronize if there is an attribute held only by own node that Writer does not have. */
 -            attrd_current_only_attribute_update(peer, xml);
 -        }
 +        process_peer_sync_response(peer, peer_won, xml);
     } else if (pcmk__str_eq(op, PCMK__ATTRD_CMD_FLUSH, pcmk__str_casei)) {
         /* Ignore. The flush command was removed in 2.0.0 but may be
 -- 
 1.8.3.1
 From a890a0e5bbbcabf907f51ed0460868035f72464d Mon Sep 17 00:00:00 2001
 From: Ken Gaillot <kgaillot@redhat.com>
 Date: Fri, 11 Jun 2021 14:40:39 -0500
 Subject: [PATCH 08/11] Refactor: pacemaker-attrd: functionize broadcasting
 local override
 ... for code isolation
 ---
 daemons/attrd/attrd_commands.c | 42 +++++++++++++++++++++++++++++-------------
 1 file changed, 29 insertions(+), 13 deletions(-)
 diff --git a/daemons/attrd/attrd_commands.c b/daemons/attrd/attrd_commands.c
 index d02d3e6..4783427 100644
 --- a/daemons/attrd/attrd_commands.c
 +++ b/daemons/attrd/attrd_commands.c
@@ -804,6 +804,34 @@ attrd_current_only_attribute_update(crm_node_t *peer, xmlNode *xml)
     free_xml(sync);
 }
 +/*!
 + * \internal
 + * \brief Override an attribute sync with a local value
 + *
 + * Broadcast the local node's value for an attribute that's different from the
 + * value provided in a peer's attribute synchronization response. This ensures a
 + * node's values for itself take precedence and all peers are kept in sync.
 + *
 + * \param[in] a          Attribute entry to override
 + *
 + * \return Local instance of attribute value
 + */
 +static attribute_value_t *
 +broadcast_local_value(attribute_t *a)
 +{
 +    attribute_value_t *v = g_hash_table_lookup(a->values, attrd_cluster->uname);
 +    xmlNode *sync = create_xml_node(NULL, __func__);
 +
 +    crm_xml_add(sync, PCMK__XA_TASK, PCMK__ATTRD_CMD_SYNC_RESPONSE);
 +    build_attribute_xml(sync, a->id, a->set, a->uuid, a->timeout_ms,
 +                        a->user, a->is_private, v->nodename, v->nodeid,
 +                        v->current, FALSE);
 +    attrd_xml_add_writer(sync);
 +    send_attrd_message(NULL, sync);
 +    free_xml(sync);
 +    return v;
 +}
 +
 void
 attrd_peer_update(crm_node_t *peer, xmlNode *xml, const char *host, bool filter)
 {
@@ -899,21 +927,9 @@ attrd_peer_update(crm_node_t *peer, xmlNode *xml, const char *host, bool filter)
     if (filter && !pcmk__str_eq(v->current, value, pcmk__str_casei)
         && pcmk__str_eq(host, attrd_cluster->uname, pcmk__str_casei)) {
 -        xmlNode *sync = create_xml_node(NULL, __func__);
 -
         crm_notice("%s[%s]: local value '%s' takes priority over '%s' from %s",
                    attr, host, v->current, value, peer->uname);
 -
 -        crm_xml_add(sync, PCMK__XA_TASK, PCMK__ATTRD_CMD_SYNC_RESPONSE);
 -        v = g_hash_table_lookup(a->values, host);
 -        build_attribute_xml(sync, attr, a->set, a->uuid, a->timeout_ms, a->user,
 -                            a->is_private, v->nodename, v->nodeid, v->current, FALSE);
 -
 -        attrd_xml_add_writer(sync);
 -
 -        /* Broadcast in case any other nodes had the inconsistent value */
 -        send_attrd_message(NULL, sync);
 -        free_xml(sync);
 +        v = broadcast_local_value(a);
     } else if (!pcmk__str_eq(v->current, value, pcmk__str_casei)) {
         crm_notice("Setting %s[%s]: %s -> %s " CRM_XS " from %s",
 -- 
 1.8.3.1
 From f6f65e3dab070f1bbdf6d1383f4d6173a8840bc9 Mon Sep 17 00:00:00 2001
 From: Ken Gaillot <kgaillot@redhat.com>
 Date: Fri, 11 Jun 2021 14:50:29 -0500
 Subject: [PATCH 09/11] Log: pacemaker-attrd: improve messages when
 broadcasting local-only values
 The traces aren't necessary since build_attribute_xml() already logs the same
 info at debug. Also, rename function for clarity, and make static.
 ---
 daemons/attrd/attrd_commands.c | 35 ++++++++++++++++-------------------
 1 file changed, 16 insertions(+), 19 deletions(-)
 diff --git a/daemons/attrd/attrd_commands.c b/daemons/attrd/attrd_commands.c
 index 4783427..356defb 100644
 --- a/daemons/attrd/attrd_commands.c
 +++ b/daemons/attrd/attrd_commands.c
@@ -51,11 +51,12 @@ GHashTable *attributes = NULL;
 void write_attribute(attribute_t *a, bool ignore_delay);
 void write_or_elect_attribute(attribute_t *a);
 -void attrd_current_only_attribute_update(crm_node_t *peer, xmlNode *xml);
 void attrd_peer_update(crm_node_t *peer, xmlNode *xml, const char *host, bool filter);
 void attrd_peer_sync(crm_node_t *peer, xmlNode *xml);
 void attrd_peer_remove(const char *host, gboolean uncache, const char *source);
 +static void broadcast_unseen_local_values(crm_node_t *peer, xmlNode *xml);
 +
 static gboolean
 send_attrd_message(crm_node_t * node, xmlNode * data)
 {
@@ -604,7 +605,7 @@ process_peer_sync_response(crm_node_t *peer, bool peer_won, xmlNode *xml)
         /* If any attributes are still not marked as seen, the writer doesn't
          * know about them, so send all peers an update with them.
          */
 -        attrd_current_only_attribute_update(peer, xml);
 +        broadcast_unseen_local_values(peer, xml);
     }
 }
@@ -768,40 +769,36 @@ attrd_lookup_or_create_value(GHashTable *values, const char *host, xmlNode *xml)
     return(v);
 }
 -void 
 -attrd_current_only_attribute_update(crm_node_t *peer, xmlNode *xml)
 +void
 +broadcast_unseen_local_values(crm_node_t *peer, xmlNode *xml)
 {
     GHashTableIter aIter;
     GHashTableIter vIter;
 -    attribute_t *a;
 +    attribute_t *a = NULL;
     attribute_value_t *v = NULL;
 -    xmlNode *sync = create_xml_node(NULL, __func__);
 -    gboolean build = FALSE;    
 -
 -    crm_xml_add(sync, PCMK__XA_TASK, PCMK__ATTRD_CMD_SYNC_RESPONSE);
 +    xmlNode *sync = NULL;
     g_hash_table_iter_init(&aIter, attributes);
     while (g_hash_table_iter_next(&aIter, NULL, (gpointer *) & a)) {
         g_hash_table_iter_init(&vIter, a->values);
         while (g_hash_table_iter_next(&vIter, NULL, (gpointer *) & v)) {
 -            if (pcmk__str_eq(v->nodename, attrd_cluster->uname, pcmk__str_casei) && v->seen == FALSE) {
 -                crm_trace("Syncing %s[%s] = %s to everyone.(from local only attributes)", a->id, v->nodename, v->current);
 -
 -                build = TRUE;
 +            if (!(v->seen) && pcmk__str_eq(v->nodename, attrd_cluster->uname,
 +                                           pcmk__str_casei)) {
 +                if (sync == NULL) {
 +                    sync = create_xml_node(NULL, __func__);
 +                    crm_xml_add(sync, PCMK__XA_TASK, PCMK__ATTRD_CMD_SYNC_RESPONSE);
 +                }
                 build_attribute_xml(sync, a->id, a->set, a->uuid, a->timeout_ms, a->user, a->is_private,
                             v->nodename, v->nodeid, v->current,  (a->timeout_ms && a->timer ? TRUE : FALSE));
 -            } else {
 -                crm_trace("Local attribute(%s[%s] = %s) was ignore.(another host) : [%s]", a->id, v->nodename, v->current, attrd_cluster->uname);
 -                continue;
             }
         }
     }
 -    if (build) {
 -        crm_debug("Syncing values to everyone.(from local only attributes)");
 +    if (sync != NULL) {
 +        crm_debug("Broadcasting local-only values");
         send_attrd_message(NULL, sync);
 +        free_xml(sync);
     }
 -    free_xml(sync);
 }
 /*!
 -- 
 1.8.3.1
 From ab90ffb785ea018556f216b8f540f8c3429a3947 Mon Sep 17 00:00:00 2001
 From: Ken Gaillot <kgaillot@redhat.com>
 Date: Fri, 11 Jun 2021 15:04:20 -0500
 Subject: [PATCH 10/11] Refactor: pacemaker-attrd: simplify attribute XML
 creation function
 ... and rename for clarity
 ---
 daemons/attrd/attrd_commands.c | 48 ++++++++++++++++++++++++------------------
 1 file changed, 27 insertions(+), 21 deletions(-)
 diff --git a/daemons/attrd/attrd_commands.c b/daemons/attrd/attrd_commands.c
 index 356defb..5b32a77 100644
 --- a/daemons/attrd/attrd_commands.c
 +++ b/daemons/attrd/attrd_commands.c
@@ -125,25 +125,35 @@ cache_remote_node(const char *node_name)
     CRM_ASSERT(crm_remote_peer_get(node_name) != NULL);
 }
 +/*!
 + * \internal
 + * \brief Create an XML representation of an attribute for use in peer messages
 + *
 + * \param[in] parent       Create attribute XML as child element of this element
 + * \param[in] a            Attribute to represent
 + * \param[in] v            Attribute value to represent
 + * \param[in] force_write  If true, value should be written even if unchanged
 + *
 + * \return XML representation of attribute
 + */
 static xmlNode *
 -build_attribute_xml(
 -    xmlNode *parent, const char *name, const char *set, const char *uuid, unsigned int timeout_ms, const char *user,
 -    gboolean is_private, const char *peer, uint32_t peerid, const char *value, gboolean is_force_write)
 +add_attribute_value_xml(xmlNode *parent, attribute_t *a, attribute_value_t *v,
 +                        bool force_write)
 {
     xmlNode *xml = create_xml_node(parent, __func__);
 -    crm_xml_add(xml, PCMK__XA_ATTR_NAME, name);
 -    crm_xml_add(xml, PCMK__XA_ATTR_SET, set);
 -    crm_xml_add(xml, PCMK__XA_ATTR_UUID, uuid);
 -    crm_xml_add(xml, PCMK__XA_ATTR_USER, user);
 -    crm_xml_add(xml, PCMK__XA_ATTR_NODE_NAME, peer);
 -    if (peerid > 0) {
 -        crm_xml_add_int(xml, PCMK__XA_ATTR_NODE_ID, peerid);
 +    crm_xml_add(xml, PCMK__XA_ATTR_NAME, a->id);
 +    crm_xml_add(xml, PCMK__XA_ATTR_SET, a->set);
 +    crm_xml_add(xml, PCMK__XA_ATTR_UUID, a->uuid);
 +    crm_xml_add(xml, PCMK__XA_ATTR_USER, a->user);
 +    crm_xml_add(xml, PCMK__XA_ATTR_NODE_NAME, v->nodename);
 +    if (v->nodeid > 0) {
 +        crm_xml_add_int(xml, PCMK__XA_ATTR_NODE_ID, v->nodeid);
     }
 -    crm_xml_add(xml, PCMK__XA_ATTR_VALUE, value);
 -    crm_xml_add_int(xml, PCMK__XA_ATTR_DAMPENING, timeout_ms/1000);
 -    crm_xml_add_int(xml, PCMK__XA_ATTR_IS_PRIVATE, is_private);
 -    crm_xml_add_int(xml, PCMK__XA_ATTR_FORCE, is_force_write);
 +    crm_xml_add(xml, PCMK__XA_ATTR_VALUE, v->current);
 +    crm_xml_add_int(xml, PCMK__XA_ATTR_DAMPENING, a->timeout_ms / 1000);
 +    crm_xml_add_int(xml, PCMK__XA_ATTR_IS_PRIVATE, a->is_private);
 +    crm_xml_add_int(xml, PCMK__XA_ATTR_FORCE, force_write);
     return xml;
 }
@@ -695,8 +705,7 @@ attrd_peer_sync(crm_node_t *peer, xmlNode *xml)
         g_hash_table_iter_init(&vIter, a->values);
         while (g_hash_table_iter_next(&vIter, NULL, (gpointer *) & v)) {
             crm_debug("Syncing %s[%s] = %s to %s", a->id, v->nodename, v->current, peer?peer->uname:"everyone");
 -            build_attribute_xml(sync, a->id, a->set, a->uuid, a->timeout_ms, a->user, a->is_private,
 -                                v->nodename, v->nodeid, v->current, FALSE);
 +            add_attribute_value_xml(sync, a, v, false);
         }
     }
@@ -788,8 +797,7 @@ broadcast_unseen_local_values(crm_node_t *peer, xmlNode *xml)
                     sync = create_xml_node(NULL, __func__);
                     crm_xml_add(sync, PCMK__XA_TASK, PCMK__ATTRD_CMD_SYNC_RESPONSE);
                 }
 -                build_attribute_xml(sync, a->id, a->set, a->uuid, a->timeout_ms, a->user, a->is_private,
 -                            v->nodename, v->nodeid, v->current,  (a->timeout_ms && a->timer ? TRUE : FALSE));
 +                add_attribute_value_xml(sync, a, v, a->timeout_ms && a->timer);
             }
         }
     }
@@ -820,9 +828,7 @@ broadcast_local_value(attribute_t *a)
     xmlNode *sync = create_xml_node(NULL, __func__);
     crm_xml_add(sync, PCMK__XA_TASK, PCMK__ATTRD_CMD_SYNC_RESPONSE);
 -    build_attribute_xml(sync, a->id, a->set, a->uuid, a->timeout_ms,
 -                        a->user, a->is_private, v->nodename, v->nodeid,
 -                        v->current, FALSE);
 +    add_attribute_value_xml(sync, a, v, false);
     attrd_xml_add_writer(sync);
     send_attrd_message(NULL, sync);
     free_xml(sync);
 -- 
 1.8.3.1
 From 540d74130c5c8d9c626d6c50475e4dc4f64234e7 Mon Sep 17 00:00:00 2001
 From: Ken Gaillot <kgaillot@redhat.com>
 Date: Fri, 4 Jun 2021 16:34:26 -0500
 Subject: [PATCH 11/11] Fix: pacemaker-attrd: avoid repeated unfencing of
 remote nodes
 The attribute manager can't record a remote node's attributes to the CIB until
 it knows the node is remote. Normally, this is learned when the remote node
 starts, because the controller clears the CRM_OP_PROBED attribute and indicates
 that it is for a remote node.
 However, if a cluster node is down when a remote node starts, and later comes
 up, it learns the remote node's existing attributes as part of the attribute
 sync. Previously, this did not include whether each value is for a cluster or
 remote node, so the newly joined attribute manager couldn't write out remote
 nodes' attributes until it learned that via some other event -- which might not
 happen before the node becomes DC, in which case its scheduler will not see any
 unfencing-related node attributes and may wrongly schedule unfencing.
 The sync response handling already calls attrd_lookup_or_create_value(), which
 checks PCMK__XA_ATTR_IS_REMOTE, so all we need to do is add that to the sync
 response.
 ---
 daemons/attrd/attrd_commands.c | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)
 diff --git a/daemons/attrd/attrd_commands.c b/daemons/attrd/attrd_commands.c
 index 5b32a77..0142383 100644
 --- a/daemons/attrd/attrd_commands.c
 +++ b/daemons/attrd/attrd_commands.c
@@ -43,8 +43,9 @@
  *     1       1.1.15   PCMK__ATTRD_CMD_UPDATE_BOTH,
  *                      PCMK__ATTRD_CMD_UPDATE_DELAY
  *     2       1.1.17   PCMK__ATTRD_CMD_CLEAR_FAILURE
 + *     3       2.1.1    PCMK__ATTRD_CMD_SYNC_RESPONSE indicates remote nodes
  */
 -#define ATTRD_PROTOCOL_VERSION "2"
 +#define ATTRD_PROTOCOL_VERSION "3"
 int last_cib_op_done = 0;
 GHashTable *attributes = NULL;
@@ -150,6 +151,9 @@ add_attribute_value_xml(xmlNode *parent, attribute_t *a, attribute_value_t *v,
     if (v->nodeid > 0) {
         crm_xml_add_int(xml, PCMK__XA_ATTR_NODE_ID, v->nodeid);
     }
 +    if (v->is_remote != 0) {
 +        crm_xml_add_int(xml, PCMK__XA_ATTR_IS_REMOTE, 1);
 +    }
     crm_xml_add(xml, PCMK__XA_ATTR_VALUE, v->current);
     crm_xml_add_int(xml, PCMK__XA_ATTR_DAMPENING, a->timeout_ms / 1000);
     crm_xml_add_int(xml, PCMK__XA_ATTR_IS_PRIVATE, a->is_private);
 -- 
 1.8.3.1
--- a/SOURCES/008-dynamic-list-fencing.patch
+++ b/SOURCES/008-dynamic-list-fencing.patch
@ -1,140 +0,0 @@
 From 2d15fb37525f88ec8d5acb689b698044c4bb69b1 Mon Sep 17 00:00:00 2001
 From: Hideo Yamauchi <renayama19661014@ybb.ne.jp>
 Date: Thu, 17 Jun 2021 22:39:12 +0900
 Subject: [PATCH 1/2] Low: fenced: Low: fenced: Remove unnecessary release.
 ---
 daemons/fenced/fenced_commands.c | 3 ---
 1 file changed, 3 deletions(-)
 diff --git a/daemons/fenced/fenced_commands.c b/daemons/fenced/fenced_commands.c
 index fee55a7..35aec06 100644
 --- a/daemons/fenced/fenced_commands.c
 +++ b/daemons/fenced/fenced_commands.c
@@ -1104,9 +1104,6 @@ dynamic_list_search_cb(GPid pid, int rc, const char *output, gpointer user_data)
         /* Fall back to status */
         g_hash_table_replace(dev->params,
                              strdup(PCMK_STONITH_HOST_CHECK), strdup("status"));
 -
 -        g_list_free_full(dev->targets, free);
 -        dev->targets = NULL;
     } else if (!rc) {
         crm_info("Refreshing port list for %s", dev->id);
         g_list_free_full(dev->targets, free);
 -- 
 1.8.3.1
 From a29f88f6020aac5f1ac32072942eb5713d7be50d Mon Sep 17 00:00:00 2001
 From: Hideo Yamauchi <renayama19661014@ybb.ne.jp>
 Date: Thu, 17 Jun 2021 22:40:40 +0900
 Subject: [PATCH 2/2] High: fenced: Wrong device may be selected when
 "dynamic-list" is specified.
 ---
 daemons/fenced/fenced_commands.c | 67 +++++++++++++++++++++++-----------------
 1 file changed, 38 insertions(+), 29 deletions(-)
 diff --git a/daemons/fenced/fenced_commands.c b/daemons/fenced/fenced_commands.c
 index 35aec06..da076fb 100644
 --- a/daemons/fenced/fenced_commands.c
 +++ b/daemons/fenced/fenced_commands.c
@@ -904,6 +904,31 @@ xml2device_params(const char *name, xmlNode *dev)
     return params;
 }
 +static const char *
 +target_list_type(stonith_device_t * dev)
 +{
 +    const char *check_type = NULL;
 +
 +    check_type = g_hash_table_lookup(dev->params, PCMK_STONITH_HOST_CHECK);
 +
 +    if (check_type == NULL) {
 +
 +        if (g_hash_table_lookup(dev->params, PCMK_STONITH_HOST_LIST)) {
 +            check_type = "static-list";
 +        } else if (g_hash_table_lookup(dev->params, PCMK_STONITH_HOST_MAP)) {
 +            check_type = "static-list";
 +        } else if (pcmk_is_set(dev->flags, st_device_supports_list)) {
 +            check_type = "dynamic-list";
 +        } else if (pcmk_is_set(dev->flags, st_device_supports_status)) {
 +            check_type = "status";
 +        } else {
 +            check_type = "none";
 +        }
 +    }
 +
 +    return check_type;
 +}
 +
 static stonith_device_t *
 build_device_from_xml(xmlNode * msg)
 {
@@ -931,6 +956,12 @@ build_device_from_xml(xmlNode * msg)
     value = g_hash_table_lookup(device->params, PCMK_STONITH_HOST_MAP);
     device->aliases = build_port_aliases(value, &(device->targets));
 +    value = target_list_type(device);
 +    if (!pcmk__str_eq(value, "static-list", pcmk__str_casei) && device->targets) {
 +        /* Other than "static-list", dev-> targets is unnecessary. */
 +        g_list_free_full(device->targets, free);
 +        device->targets = NULL;
 +    }
     device->agent_metadata = get_agent_metadata(device->agent);
     if (device->agent_metadata) {
         read_action_metadata(device);
@@ -971,31 +1002,6 @@ build_device_from_xml(xmlNode * msg)
     return device;
 }
 -static const char *
 -target_list_type(stonith_device_t * dev)
 -{
 -    const char *check_type = NULL;
 -
 -    check_type = g_hash_table_lookup(dev->params, PCMK_STONITH_HOST_CHECK);
 -
 -    if (check_type == NULL) {
 -
 -        if (g_hash_table_lookup(dev->params, PCMK_STONITH_HOST_LIST)) {
 -            check_type = "static-list";
 -        } else if (g_hash_table_lookup(dev->params, PCMK_STONITH_HOST_MAP)) {
 -            check_type = "static-list";
 -        } else if (pcmk_is_set(dev->flags, st_device_supports_list)) {
 -            check_type = "dynamic-list";
 -        } else if (pcmk_is_set(dev->flags, st_device_supports_status)) {
 -            check_type = "status";
 -        } else {
 -            check_type = "none";
 -        }
 -    }
 -
 -    return check_type;
 -}
 -
 static void
 schedule_internal_command(const char *origin,
                           stonith_device_t * device,
@@ -1099,11 +1105,14 @@ dynamic_list_search_cb(GPid pid, int rc, const char *output, gpointer user_data)
     /* If we successfully got the targets earlier, don't disable. */
     if (rc != 0 && !dev->targets) {
 -        crm_notice("Disabling port list queries for %s: %s "
 -                   CRM_XS " rc=%d", dev->id, output, rc);
 -        /* Fall back to status */
 -        g_hash_table_replace(dev->params,
 +        if (g_hash_table_lookup(dev->params, PCMK_STONITH_HOST_CHECK) == NULL) {
 +            /*
 +                If the operation fails if the user does not explicitly specify "dynamic-list", it will fall back to "status".
 +            */
 +            crm_notice("Disabling port list queries for %s (%d): %s", dev->id, rc, output);
 +            g_hash_table_replace(dev->params,
                              strdup(PCMK_STONITH_HOST_CHECK), strdup("status"));
 +        }
     } else if (!rc) {
         crm_info("Refreshing port list for %s", dev->id);
         g_list_free_full(dev->targets, free);
 -- 
 1.8.3.1
--- a/SOURCES/008-fencing-history.patch
+++ b/SOURCES/008-fencing-history.patch
@ -0,0 +1,43 @@
 From 0339e89f3238b31df78b864dae8684b82c370741 Mon Sep 17 00:00:00 2001
 From: Ken Gaillot <kgaillot@redhat.com>
 Date: Mon, 13 Dec 2021 15:22:40 -0600
 Subject: [PATCH] Fix: fencer: get current time correctly
 f52bc8e1ce (2.1.2) introduced a regression by using clock_gettime() with
 CLOCK_MONOTONIC to get the current time. Use qb_util_timespec_from_epoch_get()
 instead (which as of this writing uses clock_gettime() with CLOCK_REALTIME if
 available, and falls back to gettimeofday() if not).
 ---
 daemons/fenced/fenced_commands.c | 11 +++--------
 1 file changed, 3 insertions(+), 8 deletions(-)
 diff --git a/daemons/fenced/fenced_commands.c b/daemons/fenced/fenced_commands.c
 index f34cb4f13..7685cb8c3 100644
 --- a/daemons/fenced/fenced_commands.c
 +++ b/daemons/fenced/fenced_commands.c
@@ -2746,19 +2746,14 @@ bool fencing_peer_active(crm_node_t *peer)
     return FALSE;
 }
 -void set_fencing_completed(remote_fencing_op_t * op)
 +void
 +set_fencing_completed(remote_fencing_op_t *op)
 {
 -#ifdef CLOCK_MONOTONIC
     struct timespec tv;
 -    clock_gettime(CLOCK_MONOTONIC, &tv);
 -
 +    qb_util_timespec_from_epoch_get(&tv);
     op->completed = tv.tv_sec;
     op->completed_nsec = tv.tv_nsec;
 -#else
 -    op->completed = time(NULL);
 -    op->completed_nsec = 0L;
 -#endif
 }
 /*!
 -- 
 2.27.0
--- a/SOURCES/009-crm_resource-messages.patch
+++ b/SOURCES/009-crm_resource-messages.patch
@ -1,229 +0,0 @@
 From 5bcab230ad4c647ca78b18bd4a66e30a4bb4417f Mon Sep 17 00:00:00 2001
 From: Oyvind Albrigtsen <oalbrigt@redhat.com>
 Date: Wed, 16 Jun 2021 11:19:03 +0200
 Subject: [PATCH 1/2] Feature: crm_resource: report not supported for --force-*
 w/systemd, upstart, nagios and bundled resources
 ---
 tools/crm_resource.c         | 21 ++++----------
 tools/crm_resource_runtime.c | 67 +++++++++++++++++++++++++++++---------------
 2 files changed, 51 insertions(+), 37 deletions(-)
 diff --git a/tools/crm_resource.c b/tools/crm_resource.c
 index 4abdd03..fa7902c 100644
 --- a/tools/crm_resource.c
 +++ b/tools/crm_resource.c
@@ -660,21 +660,12 @@ attr_set_type_cb(const gchar *option_name, const gchar *optarg, gpointer data, G
 gboolean
 class_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **error) {
 -    if (!(pcmk_get_ra_caps(optarg) & pcmk_ra_cap_params)) {
 -        if (!args->quiet) {
 -            g_set_error(error, G_OPTION_ERROR, CRM_EX_INVALID_PARAM,
 -                        "Standard %s does not support parameters\n", optarg);
 -        }
 -        return FALSE;
 -
 -    } else {
 -        if (options.v_class != NULL) {
 -            free(options.v_class);
 -        }
 -
 -        options.v_class = strdup(optarg);
 +    if (options.v_class != NULL) {
 +        free(options.v_class);
     }
 +    options.v_class = strdup(optarg);
 +
     options.cmdline_config = TRUE;
     options.require_resource = FALSE;
     return TRUE;
@@ -1422,7 +1413,7 @@ validate_cmdline_config(void)
     } else if (options.rsc_cmd != cmd_execute_agent) {
         g_set_error(&error, PCMK__EXITC_ERROR, CRM_EX_USAGE,
                     "--class, --agent, and --provider can only be used with "
 -                    "--validate");
 +                    "--validate and --force-*");
     // Not all of --class, --agent, and --provider need to be given.  Not all
     // classes support the concept of a provider.  Check that what we were given
@@ -1841,7 +1832,7 @@ main(int argc, char **argv)
             if (options.cmdline_config) {
                 exit_code = cli_resource_execute_from_params(out, NULL,
                     options.v_class, options.v_provider, options.v_agent,
 -                    "validate-all", options.cmdline_params,
 +                    options.operation, options.cmdline_params,
                     options.override_params, options.timeout_ms,
                     args->verbosity, options.force, options.check_level);
             } else {
 diff --git a/tools/crm_resource_runtime.c b/tools/crm_resource_runtime.c
 index fe42e60..59e6df5 100644
 --- a/tools/crm_resource_runtime.c
 +++ b/tools/crm_resource_runtime.c
@@ -1674,24 +1674,59 @@ wait_till_stable(pcmk__output_t *out, int timeout_ms, cib_t * cib)
     return rc;
 }
 +static const char *
 +get_action(const char *rsc_action) {
 +    const char *action = NULL;
 +
 +    if (pcmk__str_eq(rsc_action, "validate", pcmk__str_casei)) {
 +        action = "validate-all";
 +
 +    } else if (pcmk__str_eq(rsc_action, "force-check", pcmk__str_casei)) {
 +        action = "monitor";
 +
 +    } else if (pcmk__strcase_any_of(rsc_action, "force-start", "force-stop",
 +                                    "force-demote", "force-promote", NULL)) {
 +        action = rsc_action+6;
 +    } else {
 +        action = rsc_action;
 +    }
 +
 +    return action;
 +}
 +
 crm_exit_t
 cli_resource_execute_from_params(pcmk__output_t *out, const char *rsc_name,
                                  const char *rsc_class, const char *rsc_prov,
 -                                 const char *rsc_type, const char *action,
 +                                 const char *rsc_type, const char *rsc_action,
                                  GHashTable *params, GHashTable *override_hash,
                                  int timeout_ms, int resource_verbose, gboolean force,
                                  int check_level)
 {
 +    const char *action = NULL;
     GHashTable *params_copy = NULL;
     crm_exit_t exit_code = CRM_EX_OK;
     svc_action_t *op = NULL;
     if (pcmk__str_eq(rsc_class, PCMK_RESOURCE_CLASS_STONITH, pcmk__str_casei)) {
         out->err(out, "Sorry, the %s option doesn't support %s resources yet",
 -                 action, rsc_class);
 +                 rsc_action, rsc_class);
 +        crm_exit(CRM_EX_UNIMPLEMENT_FEATURE);
 +    } else if (pcmk__strcase_any_of(rsc_class, PCMK_RESOURCE_CLASS_SYSTEMD,
 +                PCMK_RESOURCE_CLASS_UPSTART, PCMK_RESOURCE_CLASS_NAGIOS, NULL)) {
 +        out->err(out, "Sorry, the %s option doesn't support %s resources",
 +                 rsc_action, rsc_class);
 +        crm_exit(CRM_EX_UNIMPLEMENT_FEATURE);
 +    } else if (pcmk__str_eq(rsc_class, PCMK_RESOURCE_CLASS_SERVICE,
 +                pcmk__str_casei) && !pcmk__str_eq(
 +                resources_find_service_class(rsc_name), PCMK_RESOURCE_CLASS_LSB,
 +                pcmk__str_casei)) {
 +        out->err(out, "Sorry, the %s option doesn't support %s resources",
 +                 rsc_action, resources_find_service_class(rsc_name));
         crm_exit(CRM_EX_UNIMPLEMENT_FEATURE);
     }
 +    action = get_action(rsc_action);
 +
     /* If no timeout was provided, grab the default. */
     if (timeout_ms == 0) {
         timeout_ms = crm_get_msec(CRM_DEFAULT_OP_TIMEOUT_S);
@@ -1766,7 +1801,7 @@ cli_resource_execute_from_params(pcmk__output_t *out, const char *rsc_name,
         exit_code = op->rc;
         out->message(out, "resource-agent-action", resource_verbose, rsc_class,
 -                     rsc_prov, rsc_type, rsc_name, action, override_hash, op->rc,
 +                     rsc_prov, rsc_type, rsc_name, rsc_action, override_hash, op->rc,
                      op->status, op->stdout_data, op->stderr_data);
     } else {
         exit_code = op->rc == 0 ? CRM_EX_ERROR : op->rc;
@@ -1790,27 +1825,15 @@ cli_resource_execute(pe_resource_t *rsc, const char *requested_name,
     const char *rtype = NULL;
     const char *rprov = NULL;
     const char *rclass = NULL;
 -    const char *action = NULL;
     GHashTable *params = NULL;
 -    if (pcmk__str_eq(rsc_action, "validate", pcmk__str_casei)) {
 -        action = "validate-all";
 -
 -    } else if (pcmk__str_eq(rsc_action, "force-check", pcmk__str_casei)) {
 -        action = "monitor";
 -
 -    } else if (pcmk__str_eq(rsc_action, "force-stop", pcmk__str_casei)) {
 -        action = rsc_action+6;
 -
 -    } else if (pcmk__strcase_any_of(rsc_action, "force-start", "force-demote",
 +    if (pcmk__strcase_any_of(rsc_action, "force-start", "force-demote",
                                     "force-promote", NULL)) {
 -        action = rsc_action+6;
 -
         if(pe_rsc_is_clone(rsc)) {
             GList *nodes = cli_resource_search(rsc, requested_name, data_set);
             if(nodes != NULL && force == FALSE) {
                 out->err(out, "It is not safe to %s %s here: the cluster claims it is already active",
 -                         action, rsc->id);
 +                         rsc_action, rsc->id);
                 out->err(out, "Try setting target-role=Stopped first or specifying "
                          "the force option");
                 return CRM_EX_UNSAFE;
@@ -1818,9 +1841,6 @@ cli_resource_execute(pe_resource_t *rsc, const char *requested_name,
             g_list_free_full(nodes, free);
         }
 -
 -    } else {
 -        action = rsc_action;
     }
     if(pe_rsc_is_clone(rsc)) {
@@ -1831,6 +1851,9 @@ cli_resource_execute(pe_resource_t *rsc, const char *requested_name,
     if(rsc->variant == pe_group) {
         out->err(out, "Sorry, the %s option doesn't support group resources", rsc_action);
         return CRM_EX_UNIMPLEMENT_FEATURE;
 +    } else if (rsc->variant == pe_container || pe_rsc_is_bundled(rsc)) {
 +        out->err(out, "Sorry, the %s option doesn't support bundled resources", rsc_action);
 +        return CRM_EX_UNIMPLEMENT_FEATURE;
     }
     rclass = crm_element_value(rsc->xml, XML_AGENT_ATTR_CLASS);
@@ -1841,12 +1864,12 @@ cli_resource_execute(pe_resource_t *rsc, const char *requested_name,
                                       data_set);
     if (timeout_ms == 0) {
 -        timeout_ms = pe_get_configured_timeout(rsc, action, data_set);
 +        timeout_ms = pe_get_configured_timeout(rsc, get_action(rsc_action), data_set);
     }
     rid = pe_rsc_is_anon_clone(rsc->parent)? requested_name : rsc->id;
 -    exit_code = cli_resource_execute_from_params(out, rid, rclass, rprov, rtype, action,
 +    exit_code = cli_resource_execute_from_params(out, rid, rclass, rprov, rtype, rsc_action,
                                                  params, override_hash, timeout_ms,
                                                  resource_verbose, force, check_level);
     return exit_code;
 -- 
 1.8.3.1
 From 289cd231186755d99c1262eb9f968dc852409588 Mon Sep 17 00:00:00 2001
 From: Oyvind Albrigtsen <oalbrigt@redhat.com>
 Date: Fri, 16 Jul 2021 13:20:55 +0200
 Subject: [PATCH 2/2] Refactor: crm_resource: remove duplicate Overriding
 message that's handled elsewhere
 ---
 tools/crm_resource_runtime.c | 2 --
 1 file changed, 2 deletions(-)
 diff --git a/tools/crm_resource_runtime.c b/tools/crm_resource_runtime.c
 index 59e6df5..ce037c5 100644
 --- a/tools/crm_resource_runtime.c
 +++ b/tools/crm_resource_runtime.c
@@ -1791,8 +1791,6 @@ cli_resource_execute_from_params(pcmk__output_t *out, const char *rsc_name,
         g_hash_table_iter_init(&iter, override_hash);
         while (g_hash_table_iter_next(&iter, (gpointer *) & name, (gpointer *) & value)) {
 -            out->info(out, "Overriding the cluster configuration for '%s' with '%s' = '%s'",
 -                      rsc_name, name, value);
             g_hash_table_replace(op->params, strdup(name), strdup(value));
         }
     }
 -- 
 1.8.3.1
--- a/SOURCES/010-probe-pending.patch
+++ b/SOURCES/010-probe-pending.patch
@ -1,715 +0,0 @@
 From b0347f7b8e609420a7055d5fe537cc40ac0d1bb2 Mon Sep 17 00:00:00 2001
 From: Ken Gaillot <kgaillot@redhat.com>
 Date: Fri, 16 Jul 2021 11:08:05 -0500
 Subject: [PATCH 1/3] Fix: scheduler: don't schedule probes of unmanaged
 resources on pending nodes
 Previously, custom_action() would set an action's optional or runnable flag in
 the same, exclusive if-else sequence. This means that if an action should be
 optional *and* runnable, only one would be set. In particular, this meant that
 if a resource is unmanaged *and* its allocated node is pending, any probe would
 be set to optional, but not unrunnable, and the controller could wrongly
 attempt the probe before the join completed.
 Now, optional is checked separately.
 ---
 lib/pengine/utils.c | 22 ++++++++++++++--------
 1 file changed, 14 insertions(+), 8 deletions(-)
 diff --git a/lib/pengine/utils.c b/lib/pengine/utils.c
 index 5ef742e..965824b 100644
 --- a/lib/pengine/utils.c
 +++ b/lib/pengine/utils.c
@@ -541,6 +541,20 @@ custom_action(pe_resource_t * rsc, char *key, const char *task,
                                        FALSE, data_set);
         }
 +        // Make the action optional if its resource is unmanaged
 +        if (!pcmk_is_set(action->flags, pe_action_pseudo)
 +            && (action->node != NULL)
 +            && !pcmk_is_set(action->rsc->flags, pe_rsc_managed)
 +            && (g_hash_table_lookup(action->meta,
 +                                    XML_LRM_ATTR_INTERVAL_MS) == NULL)) {
 +                pe_rsc_debug(rsc, "%s on %s is optional (%s is unmanaged)",
 +                             action->uuid, action->node->details->uname,
 +                             action->rsc->id);
 +                pe__set_action_flags(action, pe_action_optional);
 +                // We shouldn't clear runnable here because ... something
 +        }
 +
 +        // Make the action runnable or unrunnable as appropriate
         if (pcmk_is_set(action->flags, pe_action_pseudo)) {
             /* leave untouched */
@@ -549,14 +563,6 @@ custom_action(pe_resource_t * rsc, char *key, const char *task,
                          action->uuid);
             pe__clear_action_flags(action, pe_action_runnable);
 -        } else if (!pcmk_is_set(rsc->flags, pe_rsc_managed)
 -                   && g_hash_table_lookup(action->meta,
 -                                          XML_LRM_ATTR_INTERVAL_MS) == NULL) {
 -            pe_rsc_debug(rsc, "%s on %s is optional (%s is unmanaged)",
 -                         action->uuid, action->node->details->uname, rsc->id);
 -            pe__set_action_flags(action, pe_action_optional);
 -            //pe__clear_action_flags(action, pe_action_runnable);
 -
         } else if (!pcmk_is_set(action->flags, pe_action_dc)
                    && !(action->node->details->online)
                    && (!pe__is_guest_node(action->node)
 -- 
 1.8.3.1
 From 520303b90eb707f5b7a9afa9b106e4a38b90f0f9 Mon Sep 17 00:00:00 2001
 From: Ken Gaillot <kgaillot@redhat.com>
 Date: Wed, 14 Jul 2021 17:18:44 -0500
 Subject: [PATCH 2/3] Test: scheduler: update existing tests for probe
 scheduling change
 This is an improvement. Looking at bundle-probe-order-2 for example,
 the bundle's first instance has this status to start:
    * Replica[0]
      * galera	(ocf::heartbeat:galera):	 Stopped (unmanaged)
      * galera-bundle-docker-0	(ocf::heartbeat:docker):	 Started centos2 (unmanaged)
      * galera-bundle-0	(ocf::pacemaker:remote):	 Started centos2 (unmanaged)
 After the changes, we now schedule recurring monitors for
 galera-bundle-docker-0 and galera-bundle-0 on centos2, and a probe of galera:0
 on galera-bundle-0, all of which are possible.
 ---
 cts/scheduler/dot/bundle-probe-order-2.dot         |  3 ++
 cts/scheduler/dot/bundle-probe-order-3.dot         |  1 +
 cts/scheduler/exp/bundle-probe-order-2.exp         | 33 ++++++++++++++++++++--
 cts/scheduler/exp/bundle-probe-order-3.exp         | 21 ++++++++++----
 cts/scheduler/summary/bundle-probe-order-2.summary |  3 ++
 cts/scheduler/summary/bundle-probe-order-3.summary |  1 +
 6 files changed, 53 insertions(+), 9 deletions(-)
 diff --git a/cts/scheduler/dot/bundle-probe-order-2.dot b/cts/scheduler/dot/bundle-probe-order-2.dot
 index 0cce3fd..7706195 100644
 --- a/cts/scheduler/dot/bundle-probe-order-2.dot
 +++ b/cts/scheduler/dot/bundle-probe-order-2.dot
@@ -1,6 +1,9 @@
  digraph "g" {
 +"galera-bundle-0_monitor_30000 centos2" [ style=bold color="green" fontcolor="black"]
 +"galera-bundle-docker-0_monitor_60000 centos2" [ style=bold color="green" fontcolor="black"]
 "galera-bundle-docker-1_monitor_0 centos2" [ style=bold color="green" fontcolor="black"]
 "galera-bundle-docker-2_monitor_0 centos1" [ style=bold color="green" fontcolor="black"]
 "galera-bundle-docker-2_monitor_0 centos2" [ style=bold color="green" fontcolor="black"]
 "galera-bundle-docker-2_monitor_0 centos3" [ style=bold color="green" fontcolor="black"]
 +"galera:0_monitor_0 galera-bundle-0" [ style=bold color="green" fontcolor="black"]
 }
 diff --git a/cts/scheduler/dot/bundle-probe-order-3.dot b/cts/scheduler/dot/bundle-probe-order-3.dot
 index a4b109f..53a384b 100644
 --- a/cts/scheduler/dot/bundle-probe-order-3.dot
 +++ b/cts/scheduler/dot/bundle-probe-order-3.dot
@@ -2,6 +2,7 @@
 "galera-bundle-0_monitor_0 centos1" [ style=bold color="green" fontcolor="black"]
 "galera-bundle-0_monitor_0 centos2" [ style=bold color="green" fontcolor="black"]
 "galera-bundle-0_monitor_0 centos3" [ style=bold color="green" fontcolor="black"]
 +"galera-bundle-docker-0_monitor_60000 centos2" [ style=bold color="green" fontcolor="black"]
 "galera-bundle-docker-1_monitor_0 centos2" [ style=bold color="green" fontcolor="black"]
 "galera-bundle-docker-2_monitor_0 centos1" [ style=bold color="green" fontcolor="black"]
 "galera-bundle-docker-2_monitor_0 centos2" [ style=bold color="green" fontcolor="black"]
 diff --git a/cts/scheduler/exp/bundle-probe-order-2.exp b/cts/scheduler/exp/bundle-probe-order-2.exp
 index d6174e7..5b28050 100644
 --- a/cts/scheduler/exp/bundle-probe-order-2.exp
 +++ b/cts/scheduler/exp/bundle-probe-order-2.exp
@@ -1,6 +1,33 @@
 <transition_graph cluster-delay="60s" stonith-timeout="60s" failed-stop-offset="INFINITY" failed-start-offset="INFINITY"  transition_id="0">
   <synapse id="0">
     <action_set>
 +      <rsc_op id="14" operation="monitor" operation_key="galera:0_monitor_0" on_node="galera-bundle-0" on_node_uuid="galera-bundle-0" router_node="centos2">
 +        <primitive id="galera" long-id="galera:0" class="ocf" provider="heartbeat" type="galera"/>
 +        <attributes CRM_meta_clone="0" CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_container_attribute_target="host" CRM_meta_globally_unique="false" CRM_meta_master_max="3" CRM_meta_master_node_max="1" CRM_meta_notify="false" CRM_meta_on_node="galera-bundle-0" CRM_meta_on_node_uuid="galera-bundle-0" CRM_meta_op_target_rc="7" CRM_meta_physical_host="centos2" CRM_meta_promoted_max="3" CRM_meta_promoted_node_max="1" CRM_meta_timeout="30000" cluster_host_map="centos1:centos1;centos2:centos2;centos3:centos3"  enable_creation="true" wsrep_cluster_address="gcomm://centos1,centos2,centos3"/>
 +      </rsc_op>
 +    </action_set>
 +    <inputs/>
 +  </synapse>
 +  <synapse id="1">
 +    <action_set>
 +      <rsc_op id="16" operation="monitor" operation_key="galera-bundle-docker-0_monitor_60000" on_node="centos2" on_node_uuid="2">
 +        <primitive id="galera-bundle-docker-0" class="ocf" provider="heartbeat" type="docker"/>
 +        <attributes CRM_meta_interval="60000" CRM_meta_name="monitor" CRM_meta_on_node="centos2" CRM_meta_on_node_uuid="2" CRM_meta_timeout="20000" allow_pull="true"  force_kill="false" image="docker.io/tripleoupstream/centos-binary-mariadb:latest" monitor_cmd="/bin/true" mount_points="/var/log/pacemaker/bundles/galera-bundle-0" reuse="false" run_cmd="/usr/sbin/pacemaker_remoted" run_opts=" --restart=no -e PCMK_stderr=1 --net=host -e PCMK_remote_port=3123 -v /foo:/etc/libqb/force-filesystem-sockets:ro -v /etc/my.cnf.d/galera.cnf:/etc/my.cnf.d/galera.cnf:ro -v /var/lib/mysql:/var/lib/mysql:rw -v /etc/pacemaker/authkey:/etc/pacemaker/authkey -v /var/log/pacemaker/bundles/galera-bundle-0:/var/log --user=root --log-driver=journald "/>
 +      </rsc_op>
 +    </action_set>
 +    <inputs/>
 +  </synapse>
 +  <synapse id="2">
 +    <action_set>
 +      <rsc_op id="18" operation="monitor" operation_key="galera-bundle-0_monitor_30000" on_node="centos2" on_node_uuid="2">
 +        <primitive id="galera-bundle-0" class="ocf" provider="pacemaker" type="remote"/>
 +        <attributes CRM_meta_container="galera-bundle-docker-0" CRM_meta_interval="30000" CRM_meta_name="monitor" CRM_meta_on_node="centos2" CRM_meta_on_node_uuid="2" CRM_meta_timeout="30000" addr="centos2"  port="3123"/>
 +      </rsc_op>
 +    </action_set>
 +    <inputs/>
 +  </synapse>
 +  <synapse id="3">
 +    <action_set>
       <rsc_op id="7" operation="monitor" operation_key="galera-bundle-docker-1_monitor_0" on_node="centos2" on_node_uuid="2">
         <primitive id="galera-bundle-docker-1" class="ocf" provider="heartbeat" type="docker"/>
         <attributes CRM_meta_on_node="centos2" CRM_meta_on_node_uuid="2" CRM_meta_op_target_rc="7" CRM_meta_timeout="20000" allow_pull="true"  force_kill="false" image="docker.io/tripleoupstream/centos-binary-mariadb:latest" monitor_cmd="/bin/true" mount_points="/var/log/pacemaker/bundles/galera-bundle-1" reuse="false" run_cmd="/usr/sbin/pacemaker_remoted" run_opts=" --restart=no -e PCMK_stderr=1 --net=host -e PCMK_remote_port=3123 -v /foo:/etc/libqb/force-filesystem-sockets:ro -v /etc/my.cnf.d/galera.cnf:/etc/my.cnf.d/galera.cnf:ro -v /var/lib/mysql:/var/lib/mysql:rw -v /etc/pacemaker/authkey:/etc/pacemaker/authkey -v /var/log/pacemaker/bundles/galera-bundle-1:/var/log --user=root --log-driver=journald "/>
@@ -8,7 +35,7 @@
     </action_set>
     <inputs/>
   </synapse>
 -  <synapse id="1">
 +  <synapse id="4">
     <action_set>
       <rsc_op id="12" operation="monitor" operation_key="galera-bundle-docker-2_monitor_0" on_node="centos3" on_node_uuid="3">
         <primitive id="galera-bundle-docker-2" class="ocf" provider="heartbeat" type="docker"/>
@@ -17,7 +44,7 @@
     </action_set>
     <inputs/>
   </synapse>
 -  <synapse id="2">
 +  <synapse id="5">
     <action_set>
       <rsc_op id="9" operation="monitor" operation_key="galera-bundle-docker-2_monitor_0" on_node="centos2" on_node_uuid="2">
         <primitive id="galera-bundle-docker-2" class="ocf" provider="heartbeat" type="docker"/>
@@ -26,7 +53,7 @@
     </action_set>
     <inputs/>
   </synapse>
 -  <synapse id="3">
 +  <synapse id="6">
     <action_set>
       <rsc_op id="5" operation="monitor" operation_key="galera-bundle-docker-2_monitor_0" on_node="centos1" on_node_uuid="1">
         <primitive id="galera-bundle-docker-2" class="ocf" provider="heartbeat" type="docker"/>
 diff --git a/cts/scheduler/exp/bundle-probe-order-3.exp b/cts/scheduler/exp/bundle-probe-order-3.exp
 index e1f60e7..69140a4 100644
 --- a/cts/scheduler/exp/bundle-probe-order-3.exp
 +++ b/cts/scheduler/exp/bundle-probe-order-3.exp
@@ -1,6 +1,15 @@
 <transition_graph cluster-delay="60s" stonith-timeout="60s" failed-stop-offset="INFINITY" failed-start-offset="INFINITY"  transition_id="0">
   <synapse id="0">
     <action_set>
 +      <rsc_op id="16" operation="monitor" operation_key="galera-bundle-docker-0_monitor_60000" on_node="centos2" on_node_uuid="2">
 +        <primitive id="galera-bundle-docker-0" class="ocf" provider="heartbeat" type="docker"/>
 +        <attributes CRM_meta_interval="60000" CRM_meta_name="monitor" CRM_meta_on_node="centos2" CRM_meta_on_node_uuid="2" CRM_meta_timeout="20000" allow_pull="true"  force_kill="false" image="docker.io/tripleoupstream/centos-binary-mariadb:latest" monitor_cmd="/bin/true" mount_points="/var/log/pacemaker/bundles/galera-bundle-0" reuse="false" run_cmd="/usr/sbin/pacemaker_remoted" run_opts=" --restart=no -e PCMK_stderr=1 --net=host -e PCMK_remote_port=3123 -v /foo:/etc/libqb/force-filesystem-sockets:ro -v /etc/my.cnf.d/galera.cnf:/etc/my.cnf.d/galera.cnf:ro -v /var/lib/mysql:/var/lib/mysql:rw -v /etc/pacemaker/authkey:/etc/pacemaker/authkey -v /var/log/pacemaker/bundles/galera-bundle-0:/var/log --user=root --log-driver=journald "/>
 +      </rsc_op>
 +    </action_set>
 +    <inputs/>
 +  </synapse>
 +  <synapse id="1">
 +    <action_set>
       <rsc_op id="11" operation="monitor" operation_key="galera-bundle-0_monitor_0" on_node="centos3" on_node_uuid="3">
         <primitive id="galera-bundle-0" class="ocf" provider="pacemaker" type="remote"/>
         <attributes CRM_meta_container="galera-bundle-docker-0" CRM_meta_on_node="centos3" CRM_meta_on_node_uuid="3" CRM_meta_op_target_rc="7" CRM_meta_timeout="30000" addr="centos2"  port="3123"/>
@@ -8,7 +17,7 @@
     </action_set>
     <inputs/>
   </synapse>
 -  <synapse id="1">
 +  <synapse id="2">
     <action_set>
       <rsc_op id="6" operation="monitor" operation_key="galera-bundle-0_monitor_0" on_node="centos2" on_node_uuid="2">
         <primitive id="galera-bundle-0" class="ocf" provider="pacemaker" type="remote"/>
@@ -17,7 +26,7 @@
     </action_set>
     <inputs/>
   </synapse>
 -  <synapse id="2">
 +  <synapse id="3">
     <action_set>
       <rsc_op id="3" operation="monitor" operation_key="galera-bundle-0_monitor_0" on_node="centos1" on_node_uuid="1">
         <primitive id="galera-bundle-0" class="ocf" provider="pacemaker" type="remote"/>
@@ -26,7 +35,7 @@
     </action_set>
     <inputs/>
   </synapse>
 -  <synapse id="3">
 +  <synapse id="4">
     <action_set>
       <rsc_op id="7" operation="monitor" operation_key="galera-bundle-docker-1_monitor_0" on_node="centos2" on_node_uuid="2">
         <primitive id="galera-bundle-docker-1" class="ocf" provider="heartbeat" type="docker"/>
@@ -35,7 +44,7 @@
     </action_set>
     <inputs/>
   </synapse>
 -  <synapse id="4">
 +  <synapse id="5">
     <action_set>
       <rsc_op id="13" operation="monitor" operation_key="galera-bundle-docker-2_monitor_0" on_node="centos3" on_node_uuid="3">
         <primitive id="galera-bundle-docker-2" class="ocf" provider="heartbeat" type="docker"/>
@@ -44,7 +53,7 @@
     </action_set>
     <inputs/>
   </synapse>
 -  <synapse id="5">
 +  <synapse id="6">
     <action_set>
       <rsc_op id="9" operation="monitor" operation_key="galera-bundle-docker-2_monitor_0" on_node="centos2" on_node_uuid="2">
         <primitive id="galera-bundle-docker-2" class="ocf" provider="heartbeat" type="docker"/>
@@ -53,7 +62,7 @@
     </action_set>
     <inputs/>
   </synapse>
 -  <synapse id="6">
 +  <synapse id="7">
     <action_set>
       <rsc_op id="4" operation="monitor" operation_key="galera-bundle-docker-2_monitor_0" on_node="centos1" on_node_uuid="1">
         <primitive id="galera-bundle-docker-2" class="ocf" provider="heartbeat" type="docker"/>
 diff --git a/cts/scheduler/summary/bundle-probe-order-2.summary b/cts/scheduler/summary/bundle-probe-order-2.summary
 index 681d607..024c472 100644
 --- a/cts/scheduler/summary/bundle-probe-order-2.summary
 +++ b/cts/scheduler/summary/bundle-probe-order-2.summary
@@ -13,6 +13,9 @@ Current cluster status:
 Transition Summary:
 Executing Cluster Transition:
 +  * Resource action: galera:0        monitor on galera-bundle-0
 +  * Resource action: galera-bundle-docker-0 monitor=60000 on centos2
 +  * Resource action: galera-bundle-0 monitor=30000 on centos2
   * Resource action: galera-bundle-docker-1 monitor on centos2
   * Resource action: galera-bundle-docker-2 monitor on centos3
   * Resource action: galera-bundle-docker-2 monitor on centos2
 diff --git a/cts/scheduler/summary/bundle-probe-order-3.summary b/cts/scheduler/summary/bundle-probe-order-3.summary
 index f089618..331bd87 100644
 --- a/cts/scheduler/summary/bundle-probe-order-3.summary
 +++ b/cts/scheduler/summary/bundle-probe-order-3.summary
@@ -12,6 +12,7 @@ Current cluster status:
 Transition Summary:
 Executing Cluster Transition:
 +  * Resource action: galera-bundle-docker-0 monitor=60000 on centos2
   * Resource action: galera-bundle-0 monitor on centos3
   * Resource action: galera-bundle-0 monitor on centos2
   * Resource action: galera-bundle-0 monitor on centos1
 -- 
 1.8.3.1
 From cb9c294a7ef22916866e0e42e51e88c2b1a61c2e Mon Sep 17 00:00:00 2001
 From: Ken Gaillot <kgaillot@redhat.com>
 Date: Wed, 14 Jul 2021 17:23:11 -0500
 Subject: [PATCH 3/3] Test: scheduler: add test for probe of unmanaged resource
 on pending node
 No probes should be scheduled in this case
 ---
 cts/cts-scheduler.in                             |   1 +
 cts/scheduler/dot/probe-pending-node.dot         |   2 +
 cts/scheduler/exp/probe-pending-node.exp         |   1 +
 cts/scheduler/scores/probe-pending-node.scores   |  61 ++++++
 cts/scheduler/summary/probe-pending-node.summary |  55 +++++
 cts/scheduler/xml/probe-pending-node.xml         | 247 +++++++++++++++++++++++
 6 files changed, 367 insertions(+)
 create mode 100644 cts/scheduler/dot/probe-pending-node.dot
 create mode 100644 cts/scheduler/exp/probe-pending-node.exp
 create mode 100644 cts/scheduler/scores/probe-pending-node.scores
 create mode 100644 cts/scheduler/summary/probe-pending-node.summary
 create mode 100644 cts/scheduler/xml/probe-pending-node.xml
 diff --git a/cts/cts-scheduler.in b/cts/cts-scheduler.in
 index fc9790b..7ba2415 100644
 --- a/cts/cts-scheduler.in
 +++ b/cts/cts-scheduler.in
@@ -110,6 +110,7 @@ TESTS = [
         [ "probe-2", "Correctly re-probe cloned groups" ],
         [ "probe-3", "Probe (pending node)" ],
         [ "probe-4", "Probe (pending node + stopped resource)" ],
 +        [ "probe-pending-node", "Probe (pending node + unmanaged resource)" ],
         [ "standby", "Standby" ],
         [ "comments", "Comments" ],
     ],
 diff --git a/cts/scheduler/dot/probe-pending-node.dot b/cts/scheduler/dot/probe-pending-node.dot
 new file mode 100644
 index 0000000..d8f1c9f
 --- /dev/null
 +++ b/cts/scheduler/dot/probe-pending-node.dot
@@ -0,0 +1,2 @@
 + digraph "g" {
 +}
 diff --git a/cts/scheduler/exp/probe-pending-node.exp b/cts/scheduler/exp/probe-pending-node.exp
 new file mode 100644
 index 0000000..56e315f
 --- /dev/null
 +++ b/cts/scheduler/exp/probe-pending-node.exp
@@ -0,0 +1 @@
 +<transition_graph cluster-delay="60s" stonith-timeout="60s" failed-stop-offset="INFINITY" failed-start-offset="INFINITY"  transition_id="0"/>
 diff --git a/cts/scheduler/scores/probe-pending-node.scores b/cts/scheduler/scores/probe-pending-node.scores
 new file mode 100644
 index 0000000..020a1a0
 --- /dev/null
 +++ b/cts/scheduler/scores/probe-pending-node.scores
@@ -0,0 +1,61 @@
 +
 +pcmk__clone_allocate: fs_UC5_SAPMNT-clone allocation score on gcdoubwap01: 0
 +pcmk__clone_allocate: fs_UC5_SAPMNT-clone allocation score on gcdoubwap02: 0
 +pcmk__clone_allocate: fs_UC5_SAPMNT:0 allocation score on gcdoubwap01: 0
 +pcmk__clone_allocate: fs_UC5_SAPMNT:0 allocation score on gcdoubwap02: 0
 +pcmk__clone_allocate: fs_UC5_SAPMNT:1 allocation score on gcdoubwap01: 0
 +pcmk__clone_allocate: fs_UC5_SAPMNT:1 allocation score on gcdoubwap02: 0
 +pcmk__clone_allocate: fs_UC5_SYS-clone allocation score on gcdoubwap01: 0
 +pcmk__clone_allocate: fs_UC5_SYS-clone allocation score on gcdoubwap02: 0
 +pcmk__clone_allocate: fs_UC5_SYS:0 allocation score on gcdoubwap01: 0
 +pcmk__clone_allocate: fs_UC5_SYS:0 allocation score on gcdoubwap02: 0
 +pcmk__clone_allocate: fs_UC5_SYS:1 allocation score on gcdoubwap01: 0
 +pcmk__clone_allocate: fs_UC5_SYS:1 allocation score on gcdoubwap02: 0
 +pcmk__group_allocate: fs_UC5_ascs allocation score on gcdoubwap01: 0
 +pcmk__group_allocate: fs_UC5_ascs allocation score on gcdoubwap02: 0
 +pcmk__group_allocate: fs_UC5_ers allocation score on gcdoubwap01: 0
 +pcmk__group_allocate: fs_UC5_ers allocation score on gcdoubwap02: 0
 +pcmk__group_allocate: grp_UC5_ascs allocation score on gcdoubwap01: 0
 +pcmk__group_allocate: grp_UC5_ascs allocation score on gcdoubwap02: 0
 +pcmk__group_allocate: grp_UC5_ers allocation score on gcdoubwap01: 0
 +pcmk__group_allocate: grp_UC5_ers allocation score on gcdoubwap02: 0
 +pcmk__group_allocate: rsc_sap_UC5_ASCS11 allocation score on gcdoubwap01: 0
 +pcmk__group_allocate: rsc_sap_UC5_ASCS11 allocation score on gcdoubwap02: 0
 +pcmk__group_allocate: rsc_sap_UC5_ERS12 allocation score on gcdoubwap01: 0
 +pcmk__group_allocate: rsc_sap_UC5_ERS12 allocation score on gcdoubwap02: 0
 +pcmk__group_allocate: rsc_vip_gcp_ascs allocation score on gcdoubwap01: INFINITY
 +pcmk__group_allocate: rsc_vip_gcp_ascs allocation score on gcdoubwap02: 0
 +pcmk__group_allocate: rsc_vip_gcp_ers allocation score on gcdoubwap01: 0
 +pcmk__group_allocate: rsc_vip_gcp_ers allocation score on gcdoubwap02: 0
 +pcmk__group_allocate: rsc_vip_init_ers allocation score on gcdoubwap01: 0
 +pcmk__group_allocate: rsc_vip_init_ers allocation score on gcdoubwap02: 0
 +pcmk__group_allocate: rsc_vip_int_ascs allocation score on gcdoubwap01: 0
 +pcmk__group_allocate: rsc_vip_int_ascs allocation score on gcdoubwap02: 0
 +pcmk__native_allocate: fs_UC5_SAPMNT:0 allocation score on gcdoubwap01: 0
 +pcmk__native_allocate: fs_UC5_SAPMNT:0 allocation score on gcdoubwap02: -INFINITY
 +pcmk__native_allocate: fs_UC5_SAPMNT:1 allocation score on gcdoubwap01: 0
 +pcmk__native_allocate: fs_UC5_SAPMNT:1 allocation score on gcdoubwap02: -INFINITY
 +pcmk__native_allocate: fs_UC5_SYS:0 allocation score on gcdoubwap01: 0
 +pcmk__native_allocate: fs_UC5_SYS:0 allocation score on gcdoubwap02: -INFINITY
 +pcmk__native_allocate: fs_UC5_SYS:1 allocation score on gcdoubwap01: 0
 +pcmk__native_allocate: fs_UC5_SYS:1 allocation score on gcdoubwap02: -INFINITY
 +pcmk__native_allocate: fs_UC5_ascs allocation score on gcdoubwap01: 0
 +pcmk__native_allocate: fs_UC5_ascs allocation score on gcdoubwap02: -INFINITY
 +pcmk__native_allocate: fs_UC5_ers allocation score on gcdoubwap01: -INFINITY
 +pcmk__native_allocate: fs_UC5_ers allocation score on gcdoubwap02: -INFINITY
 +pcmk__native_allocate: rsc_sap_UC5_ASCS11 allocation score on gcdoubwap01: -INFINITY
 +pcmk__native_allocate: rsc_sap_UC5_ASCS11 allocation score on gcdoubwap02: -INFINITY
 +pcmk__native_allocate: rsc_sap_UC5_ERS12 allocation score on gcdoubwap01: -INFINITY
 +pcmk__native_allocate: rsc_sap_UC5_ERS12 allocation score on gcdoubwap02: -INFINITY
 +pcmk__native_allocate: rsc_vip_gcp_ascs allocation score on gcdoubwap01: -INFINITY
 +pcmk__native_allocate: rsc_vip_gcp_ascs allocation score on gcdoubwap02: -INFINITY
 +pcmk__native_allocate: rsc_vip_gcp_ers allocation score on gcdoubwap01: -INFINITY
 +pcmk__native_allocate: rsc_vip_gcp_ers allocation score on gcdoubwap02: -INFINITY
 +pcmk__native_allocate: rsc_vip_init_ers allocation score on gcdoubwap01: 0
 +pcmk__native_allocate: rsc_vip_init_ers allocation score on gcdoubwap02: -INFINITY
 +pcmk__native_allocate: rsc_vip_int_ascs allocation score on gcdoubwap01: INFINITY
 +pcmk__native_allocate: rsc_vip_int_ascs allocation score on gcdoubwap02: -INFINITY
 +pcmk__native_allocate: stonith_gcdoubwap01 allocation score on gcdoubwap01: -INFINITY
 +pcmk__native_allocate: stonith_gcdoubwap01 allocation score on gcdoubwap02: 0
 +pcmk__native_allocate: stonith_gcdoubwap02 allocation score on gcdoubwap01: 0
 +pcmk__native_allocate: stonith_gcdoubwap02 allocation score on gcdoubwap02: -INFINITY
 diff --git a/cts/scheduler/summary/probe-pending-node.summary b/cts/scheduler/summary/probe-pending-node.summary
 new file mode 100644
 index 0000000..208186b
 --- /dev/null
 +++ b/cts/scheduler/summary/probe-pending-node.summary
@@ -0,0 +1,55 @@
 +Using the original execution date of: 2021-06-11 13:55:24Z
 +
 +              *** Resource management is DISABLED ***
 +  The cluster will not attempt to start, stop or recover services
 +
 +Current cluster status:
 +  * Node List:
 +    * Node gcdoubwap02: pending
 +    * Online: [ gcdoubwap01 ]
 +
 +  * Full List of Resources:
 +    * stonith_gcdoubwap01	(stonith:fence_gce):	 Stopped (unmanaged)
 +    * stonith_gcdoubwap02	(stonith:fence_gce):	 Stopped (unmanaged)
 +    * Clone Set: fs_UC5_SAPMNT-clone [fs_UC5_SAPMNT] (unmanaged):
 +      * Stopped: [ gcdoubwap01 gcdoubwap02 ]
 +    * Clone Set: fs_UC5_SYS-clone [fs_UC5_SYS] (unmanaged):
 +      * Stopped: [ gcdoubwap01 gcdoubwap02 ]
 +    * Resource Group: grp_UC5_ascs (unmanaged):
 +      * rsc_vip_int_ascs	(ocf:heartbeat:IPaddr2):	 Stopped (unmanaged)
 +      * rsc_vip_gcp_ascs	(ocf:heartbeat:gcp-vpc-move-vip):	 Started gcdoubwap01 (unmanaged)
 +      * fs_UC5_ascs	(ocf:heartbeat:Filesystem):	 Stopped (unmanaged)
 +      * rsc_sap_UC5_ASCS11	(ocf:heartbeat:SAPInstance):	 Stopped (unmanaged)
 +    * Resource Group: grp_UC5_ers (unmanaged):
 +      * rsc_vip_init_ers	(ocf:heartbeat:IPaddr2):	 Stopped (unmanaged)
 +      * rsc_vip_gcp_ers	(ocf:heartbeat:gcp-vpc-move-vip):	 Stopped (unmanaged)
 +      * fs_UC5_ers	(ocf:heartbeat:Filesystem):	 Stopped (unmanaged)
 +      * rsc_sap_UC5_ERS12	(ocf:heartbeat:SAPInstance):	 Stopped (unmanaged)
 +
 +Transition Summary:
 +
 +Executing Cluster Transition:
 +Using the original execution date of: 2021-06-11 13:55:24Z
 +
 +Revised Cluster Status:
 +  * Node List:
 +    * Node gcdoubwap02: pending
 +    * Online: [ gcdoubwap01 ]
 +
 +  * Full List of Resources:
 +    * stonith_gcdoubwap01	(stonith:fence_gce):	 Stopped (unmanaged)
 +    * stonith_gcdoubwap02	(stonith:fence_gce):	 Stopped (unmanaged)
 +    * Clone Set: fs_UC5_SAPMNT-clone [fs_UC5_SAPMNT] (unmanaged):
 +      * Stopped: [ gcdoubwap01 gcdoubwap02 ]
 +    * Clone Set: fs_UC5_SYS-clone [fs_UC5_SYS] (unmanaged):
 +      * Stopped: [ gcdoubwap01 gcdoubwap02 ]
 +    * Resource Group: grp_UC5_ascs (unmanaged):
 +      * rsc_vip_int_ascs	(ocf:heartbeat:IPaddr2):	 Stopped (unmanaged)
 +      * rsc_vip_gcp_ascs	(ocf:heartbeat:gcp-vpc-move-vip):	 Started gcdoubwap01 (unmanaged)
 +      * fs_UC5_ascs	(ocf:heartbeat:Filesystem):	 Stopped (unmanaged)
 +      * rsc_sap_UC5_ASCS11	(ocf:heartbeat:SAPInstance):	 Stopped (unmanaged)
 +    * Resource Group: grp_UC5_ers (unmanaged):
 +      * rsc_vip_init_ers	(ocf:heartbeat:IPaddr2):	 Stopped (unmanaged)
 +      * rsc_vip_gcp_ers	(ocf:heartbeat:gcp-vpc-move-vip):	 Stopped (unmanaged)
 +      * fs_UC5_ers	(ocf:heartbeat:Filesystem):	 Stopped (unmanaged)
 +      * rsc_sap_UC5_ERS12	(ocf:heartbeat:SAPInstance):	 Stopped (unmanaged)
 diff --git a/cts/scheduler/xml/probe-pending-node.xml b/cts/scheduler/xml/probe-pending-node.xml
 new file mode 100644
 index 0000000..9f55c92
 --- /dev/null
 +++ b/cts/scheduler/xml/probe-pending-node.xml
@@ -0,0 +1,247 @@
 +<cib crm_feature_set="3.0.14" validate-with="pacemaker-2.10" epoch="395" num_updates="30" admin_epoch="0" cib-last-written="Thu Jun 10 18:01:13 2021" update-origin="gcdoubwap01" update-client="cibadmin" update-user="root" have-quorum="1" dc-uuid="1" execution-date="1623419724">
 +  <configuration>
 +    <crm_config>
 +      <cluster_property_set id="cib-bootstrap-options">
 +        <nvpair id="cib-bootstrap-options-have-watchdog" name="have-watchdog" value="false"/>
 +        <nvpair id="cib-bootstrap-options-dc-version" name="dc-version" value="1.1.19-8.el7_6.5-c3c624ea3d"/>
 +        <nvpair id="cib-bootstrap-options-cluster-infrastructure" name="cluster-infrastructure" value="corosync"/>
 +        <nvpair id="cib-bootstrap-options-cluster-name" name="cluster-name" value="ascscluster"/>
 +        <nvpair id="cib-bootstrap-options-maintenance-mode" name="maintenance-mode" value="true"/>
 +        <nvpair id="cib-bootstrap-options-last-lrm-refresh" name="last-lrm-refresh" value="1622815036"/>
 +      </cluster_property_set>
 +    </crm_config>
 +    <nodes>
 +      <node id="1" uname="gcdoubwap01"/>
 +      <node id="2" uname="gcdoubwap02"/>
 +    </nodes>
 +    <resources>
 +      <primitive class="stonith" id="stonith_gcdoubwap01" type="fence_gce">
 +        <instance_attributes id="stonith_gcdoubwap01-instance_attributes">
 +          <nvpair id="stonith_gcdoubwap01-instance_attributes-project" name="project" value="pj-uat-do-nane1-01"/>
 +          <nvpair id="stonith_gcdoubwap01-instance_attributes-zone" name="zone" value="northamerica-northeast1-b"/>
 +        </instance_attributes>
 +        <operations>
 +          <op id="stonith_gcdoubwap01-monitor-interval-60s" interval="60s" name="monitor"/>
 +        </operations>
 +      </primitive>
 +      <primitive class="stonith" id="stonith_gcdoubwap02" type="fence_gce">
 +        <instance_attributes id="stonith_gcdoubwap02-instance_attributes">
 +          <nvpair id="stonith_gcdoubwap02-instance_attributes-project" name="project" value="pj-uat-do-nane1-01"/>
 +          <nvpair id="stonith_gcdoubwap02-instance_attributes-zone" name="zone" value="northamerica-northeast1-c"/>
 +        </instance_attributes>
 +        <operations>
 +          <op id="stonith_gcdoubwap02-monitor-interval-60s" interval="60s" name="monitor"/>
 +        </operations>
 +      </primitive>
 +      <clone id="fs_UC5_SAPMNT-clone">
 +        <primitive class="ocf" id="fs_UC5_SAPMNT" provider="heartbeat" type="Filesystem">
 +          <instance_attributes id="fs_UC5_SAPMNT-instance_attributes">
 +            <nvpair id="fs_UC5_SAPMNT-instance_attributes-device" name="device" value="uatdoelfs.igmfinancial.net:UC5_sapmnt/root"/>
 +            <nvpair id="fs_UC5_SAPMNT-instance_attributes-directory" name="directory" value="/sapmnt/UC5"/>
 +            <nvpair id="fs_UC5_SAPMNT-instance_attributes-fstype" name="fstype" value="nfs"/>
 +          </instance_attributes>
 +          <operations>
 +            <op id="fs_UC5_SAPMNT-monitor-interval-20s" interval="20s" name="monitor" timeout="40s"/>
 +            <op id="fs_UC5_SAPMNT-notify-interval-0s" interval="0s" name="notify" timeout="60s"/>
 +            <op id="fs_UC5_SAPMNT-start-interval-0s" interval="0s" name="start" timeout="60s"/>
 +            <op id="fs_UC5_SAPMNT-stop-interval-0s" interval="0s" name="stop" timeout="60s"/>
 +          </operations>
 +        </primitive>
 +        <meta_attributes id="fs_UC5_SAPMNT-clone-meta_attributes">
 +          <nvpair id="fs_UC5_SAPMNT-clone-meta_attributes-interleave" name="interleave" value="true"/>
 +        </meta_attributes>
 +      </clone>
 +      <clone id="fs_UC5_SYS-clone">
 +        <primitive class="ocf" id="fs_UC5_SYS" provider="heartbeat" type="Filesystem">
 +          <instance_attributes id="fs_UC5_SYS-instance_attributes">
 +            <nvpair id="fs_UC5_SYS-instance_attributes-device" name="device" value="uatdoelfs.igmfinancial.net:UC5_SYS/root"/>
 +            <nvpair id="fs_UC5_SYS-instance_attributes-directory" name="directory" value="/usr/sap/UC5/SYS"/>
 +            <nvpair id="fs_UC5_SYS-instance_attributes-fstype" name="fstype" value="nfs"/>
 +          </instance_attributes>
 +          <operations>
 +            <op id="fs_UC5_SYS-monitor-interval-20s" interval="20s" name="monitor" timeout="40s"/>
 +            <op id="fs_UC5_SYS-notify-interval-0s" interval="0s" name="notify" timeout="60s"/>
 +            <op id="fs_UC5_SYS-start-interval-0s" interval="0s" name="start" timeout="60s"/>
 +            <op id="fs_UC5_SYS-stop-interval-0s" interval="0s" name="stop" timeout="60s"/>
 +          </operations>
 +        </primitive>
 +        <meta_attributes id="fs_UC5_SYS-clone-meta_attributes">
 +          <nvpair id="fs_UC5_SYS-clone-meta_attributes-interleave" name="interleave" value="true"/>
 +        </meta_attributes>
 +      </clone>
 +      <group id="grp_UC5_ascs">
 +        <primitive class="ocf" id="rsc_vip_int_ascs" provider="heartbeat" type="IPaddr2">
 +          <instance_attributes id="rsc_vip_int_ascs-instance_attributes">
 +            <nvpair id="rsc_vip_int_ascs-instance_attributes-cidr_netmask" name="cidr_netmask" value="32"/>
 +            <nvpair id="rsc_vip_int_ascs-instance_attributes-ip" name="ip" value="10.4.130.38"/>
 +            <nvpair id="rsc_vip_int_ascs-instance_attributes-nic" name="nic" value="eth0"/>
 +          </instance_attributes>
 +          <operations>
 +            <op id="rsc_vip_int_ascs-monitor-interval-10s" interval="10s" name="monitor" timeout="20s"/>
 +            <op id="rsc_vip_int_ascs-start-interval-0s" interval="0s" name="start" timeout="20s"/>
 +            <op id="rsc_vip_int_ascs-stop-interval-0s" interval="0s" name="stop" timeout="20s"/>
 +          </operations>
 +        </primitive>
 +        <primitive class="ocf" id="rsc_vip_gcp_ascs" provider="heartbeat" type="gcp-vpc-move-vip">
 +          <instance_attributes id="rsc_vip_gcp_ascs-instance_attributes">
 +            <nvpair id="rsc_vip_gcp_ascs-instance_attributes-alias_ip" name="alias_ip" value="10.4.130.38/32"/>
 +            <nvpair id="rsc_vip_gcp_ascs-instance_attributes-hostlist" name="hostlist" value="gcdoubwap01 gcdoubwap02"/>
 +          </instance_attributes>
 +          <operations>
 +            <op id="rsc_vip_gcp_ascs-monitor-interval-60s" interval="60s" name="monitor" on-fail="ignore"/>
 +            <op id="rsc_vip_gcp_ascs-start-interval-0s" interval="0s" name="start" timeout="300s"/>
 +            <op id="rsc_vip_gcp_ascs-stop-interval-0s" interval="0s" name="stop" timeout="15s"/>
 +          </operations>
 +        </primitive>
 +        <primitive class="ocf" id="fs_UC5_ascs" provider="heartbeat" type="Filesystem">
 +          <instance_attributes id="fs_UC5_ascs-instance_attributes">
 +            <nvpair id="fs_UC5_ascs-instance_attributes-device" name="device" value="uatdoelfs.igmfinancial.net:UC5_ASCS/root"/>
 +            <nvpair id="fs_UC5_ascs-instance_attributes-directory" name="directory" value="/usr/sap/UC5/ASCS11"/>
 +            <nvpair id="fs_UC5_ascs-instance_attributes-force_unmount" name="force_unmount" value="safe"/>
 +            <nvpair id="fs_UC5_ascs-instance_attributes-fstype" name="fstype" value="nfs"/>
 +          </instance_attributes>
 +          <operations>
 +            <op id="fs_UC5_ascs-monitor-interval-200" interval="200" name="monitor" timeout="40"/>
 +            <op id="fs_UC5_ascs-notify-interval-0s" interval="0s" name="notify" timeout="60s"/>
 +            <op id="fs_UC5_ascs-start-interval-0" interval="0" name="start" timeout="60"/>
 +            <op id="fs_UC5_ascs-stop-interval-0" interval="0" name="stop" timeout="120"/>
 +          </operations>
 +        </primitive>
 +        <primitive class="ocf" id="rsc_sap_UC5_ASCS11" provider="heartbeat" type="SAPInstance">
 +          <instance_attributes id="rsc_sap_UC5_ASCS11-instance_attributes">
 +            <nvpair id="rsc_sap_UC5_ASCS11-instance_attributes-AUTOMATIC_RECOVER" name="AUTOMATIC_RECOVER" value="false"/>
 +            <nvpair id="rsc_sap_UC5_ASCS11-instance_attributes-InstanceName" name="InstanceName" value="UC5_ASCS11_uatdobwscs"/>
 +            <nvpair id="rsc_sap_UC5_ASCS11-instance_attributes-START_PROFILE" name="START_PROFILE" value="/sapmnt/UC5/profile/UC5_ASCS11_uatdobwscs"/>
 +          </instance_attributes>
 +          <meta_attributes id="rsc_sap_UC5_ASCS11-meta_attributes">
 +            <nvpair id="rsc_sap_UC5_ASCS11-meta_attributes-failure-timeout" name="failure-timeout" value="60"/>
 +            <nvpair id="rsc_sap_UC5_ASCS11-meta_attributes-migration-threshold" name="migration-threshold" value="1"/>
 +            <nvpair id="rsc_sap_UC5_ASCS11-meta_attributes-resource-stickiness" name="resource-stickiness" value="5000"/>
 +          </meta_attributes>
 +          <operations>
 +            <op id="rsc_sap_UC5_ASCS11-demote-interval-0s" interval="0s" name="demote" timeout="320s"/>
 +            <op id="rsc_sap_UC5_ASCS11-methods-interval-0s" interval="0s" name="methods" timeout="5s"/>
 +            <op id="rsc_sap_UC5_ASCS11-monitor-interval-20" interval="20" name="monitor" on-fail="restart" timeout="60"/>
 +            <op id="rsc_sap_UC5_ASCS11-promote-interval-0s" interval="0s" name="promote" timeout="320s"/>
 +            <op id="rsc_sap_UC5_ASCS11-start-interval-0" interval="0" name="start" timeout="600"/>
 +            <op id="rsc_sap_UC5_ASCS11-stop-interval-0" interval="0" name="stop" timeout="600"/>
 +          </operations>
 +        </primitive>
 +      </group>
 +      <group id="grp_UC5_ers">
 +        <primitive class="ocf" id="rsc_vip_init_ers" provider="heartbeat" type="IPaddr2">
 +          <instance_attributes id="rsc_vip_init_ers-instance_attributes">
 +            <nvpair id="rsc_vip_init_ers-instance_attributes-cidr_netmask" name="cidr_netmask" value="32"/>
 +            <nvpair id="rsc_vip_init_ers-instance_attributes-ip" name="ip" value="10.4.130.39"/>
 +            <nvpair id="rsc_vip_init_ers-instance_attributes-nic" name="nic" value="eth0"/>
 +          </instance_attributes>
 +          <operations>
 +            <op id="rsc_vip_init_ers-monitor-interval-10s" interval="10s" name="monitor" timeout="20s"/>
 +            <op id="rsc_vip_init_ers-start-interval-0s" interval="0s" name="start" timeout="20s"/>
 +            <op id="rsc_vip_init_ers-stop-interval-0s" interval="0s" name="stop" timeout="20s"/>
 +          </operations>
 +        </primitive>
 +        <primitive class="ocf" id="rsc_vip_gcp_ers" provider="heartbeat" type="gcp-vpc-move-vip">
 +          <instance_attributes id="rsc_vip_gcp_ers-instance_attributes">
 +            <nvpair id="rsc_vip_gcp_ers-instance_attributes-alias_ip" name="alias_ip" value="10.4.130.39/32"/>
 +            <nvpair id="rsc_vip_gcp_ers-instance_attributes-hostlist" name="hostlist" value="gcdoubwap01 gcdoubwap02"/>
 +          </instance_attributes>
 +          <operations>
 +            <op id="rsc_vip_gcp_ers-monitor-interval-60s" interval="60s" name="monitor" on-fail="ignore"/>
 +            <op id="rsc_vip_gcp_ers-start-interval-0s" interval="0s" name="start" timeout="300s"/>
 +            <op id="rsc_vip_gcp_ers-stop-interval-0s" interval="0s" name="stop" timeout="180s"/>
 +          </operations>
 +        </primitive>
 +        <primitive class="ocf" id="fs_UC5_ers" provider="heartbeat" type="Filesystem">
 +          <instance_attributes id="fs_UC5_ers-instance_attributes">
 +            <nvpair id="fs_UC5_ers-instance_attributes-device" name="device" value="uatdoelfs.igmfinancial.net:UC5_ERS/root"/>
 +            <nvpair id="fs_UC5_ers-instance_attributes-directory" name="directory" value="/usr/sap/UC5/ERS12"/>
 +            <nvpair id="fs_UC5_ers-instance_attributes-force_unmount" name="force_unmount" value="safe"/>
 +            <nvpair id="fs_UC5_ers-instance_attributes-fstype" name="fstype" value="nfs"/>
 +          </instance_attributes>
 +          <operations>
 +            <op id="fs_UC5_ers-monitor-interval-200" interval="200" name="monitor" timeout="40"/>
 +            <op id="fs_UC5_ers-notify-interval-0s" interval="0s" name="notify" timeout="60s"/>
 +            <op id="fs_UC5_ers-start-interval-0" interval="0" name="start" timeout="60"/>
 +            <op id="fs_UC5_ers-stop-interval-0" interval="0" name="stop" timeout="120"/>
 +          </operations>
 +        </primitive>
 +        <primitive class="ocf" id="rsc_sap_UC5_ERS12" provider="heartbeat" type="SAPInstance">
 +          <instance_attributes id="rsc_sap_UC5_ERS12-instance_attributes">
 +            <nvpair id="rsc_sap_UC5_ERS12-instance_attributes-AUTOMATIC_RECOVER" name="AUTOMATIC_RECOVER" value="false"/>
 +            <nvpair id="rsc_sap_UC5_ERS12-instance_attributes-IS_ERS" name="IS_ERS" value="true"/>
 +            <nvpair id="rsc_sap_UC5_ERS12-instance_attributes-InstanceName" name="InstanceName" value="UC5_ERS12_uatdobwers"/>
 +            <nvpair id="rsc_sap_UC5_ERS12-instance_attributes-START_PROFILE" name="START_PROFILE" value="/sapmnt/UC5/profile/UC5_ERS12_uatdobwers"/>
 +          </instance_attributes>
 +          <operations>
 +            <op id="rsc_sap_UC5_ERS12-demote-interval-0s" interval="0s" name="demote" timeout="320s"/>
 +            <op id="rsc_sap_UC5_ERS12-methods-interval-0s" interval="0s" name="methods" timeout="5s"/>
 +            <op id="rsc_sap_UC5_ERS12-monitor-interval-20" interval="20" name="monitor" on-fail="restart" timeout="60"/>
 +            <op id="rsc_sap_UC5_ERS12-promote-interval-0s" interval="0s" name="promote" timeout="320s"/>
 +            <op id="rsc_sap_UC5_ERS12-start-interval-0" interval="0" name="start" timeout="600"/>
 +            <op id="rsc_sap_UC5_ERS12-stop-interval-0" interval="0" name="stop" timeout="600"/>
 +          </operations>
 +        </primitive>
 +      </group>
 +    </resources>
 +    <constraints>
 +      <rsc_colocation id="colocation-grp_UC5_ers-grp_UC5_ascs--5000" rsc="grp_UC5_ers" score="-5000" with-rsc="grp_UC5_ascs"/>
 +      <rsc_order first="grp_UC5_ascs" first-action="start" id="order-grp_UC5_ascs-grp_UC5_ers-mandatory" symmetrical="false" then="grp_UC5_ers" then-action="stop"/>
 +      <rsc_location id="location-rsc_sap_UC5_ASCS11" rsc="rsc_sap_UC5_ASCS11">
 +        <rule id="location-rsc_sap_UC5_ASCS11-rule" score="2000">
 +          <expression attribute="runs_ERS_UC5" id="location-rsc_sap_UC5_ASCS11-rule-expr" operation="eq" value="1"/>
 +        </rule>
 +      </rsc_location>
 +      <rsc_location id="location-stonith_gcdoubwap01-gcdoubwap02" node="gcdoubwap01" rsc="stonith_gcdoubwap01" score="-INFINITY"/>
 +      <rsc_location id="location-stonith_gcdoubwap02-gcdoubwap01" node="gcdoubwap02" rsc="stonith_gcdoubwap02" score="-INFINITY"/>
 +      <rsc_order first="fs_UC5_SAPMNT-clone" first-action="start" id="order-fs_UC5_SAPMNT-clone-grp_UC5_ascs-mandatory" then="grp_UC5_ascs" then-action="start"/>
 +      <rsc_order first="fs_UC5_SAPMNT-clone" first-action="start" id="order-fs_UC5_SAPMNT-clone-grp_UC5_ers-mandatory" then="grp_UC5_ers" then-action="start"/>
 +    </constraints>
 +  </configuration>
 +  <status>
 +    <node_state id="1" uname="gcdoubwap01" in_ccm="true" crmd="online" crm-debug-origin="post_cache_update" join="member" expected="member">
 +      <lrm id="1">
 +        <lrm_resources>
 +          <lrm_resource id="stonith_gcdoubwap01" type="fence_gce" class="stonith">
 +            <lrm_rsc_op id="stonith_gcdoubwap01_last_0" operation_key="stonith_gcdoubwap01_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.14" transition-key="1:0:7:5e41afa8-15bd-443a-88fb-ec276232a804" transition-magic="0:7;1:0:7:5e41afa8-15bd-443a-88fb-ec276232a804" exit-reason="" on_node="gcdoubwap01" call-id="5" rc-code="7" op-status="0" interval="0" last-run="1623419700" last-rc-change="1623419700" exec-time="27" queue-time="0" op-digest="e6935031dfde569ad30fb442953d3d91"/>
 +          </lrm_resource>
 +          <lrm_resource id="stonith_gcdoubwap02" type="fence_gce" class="stonith">
 +            <lrm_rsc_op id="stonith_gcdoubwap02_last_0" operation_key="stonith_gcdoubwap02_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.14" transition-key="2:0:7:5e41afa8-15bd-443a-88fb-ec276232a804" transition-magic="0:7;2:0:7:5e41afa8-15bd-443a-88fb-ec276232a804" exit-reason="" on_node="gcdoubwap01" call-id="9" rc-code="7" op-status="0" interval="0" last-run="1623419700" last-rc-change="1623419700" exec-time="0" queue-time="0" op-digest="064645c51d6d3a802eb6928f6116222c"/>
 +          </lrm_resource>
 +          <lrm_resource id="fs_UC5_SAPMNT" type="Filesystem" class="ocf" provider="heartbeat">
 +            <lrm_rsc_op id="fs_UC5_SAPMNT_last_0" operation_key="fs_UC5_SAPMNT_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.14" transition-key="3:0:7:5e41afa8-15bd-443a-88fb-ec276232a804" transition-magic="0:7;3:0:7:5e41afa8-15bd-443a-88fb-ec276232a804" exit-reason="" on_node="gcdoubwap01" call-id="14" rc-code="7" op-status="0" interval="0" last-run="1623419700" last-rc-change="1623419700" exec-time="126" queue-time="1" op-digest="02c74f325691f1af3c3dd9c2ab702b01"/>
 +          </lrm_resource>
 +          <lrm_resource id="fs_UC5_SYS" type="Filesystem" class="ocf" provider="heartbeat">
 +            <lrm_rsc_op id="fs_UC5_SYS_last_0" operation_key="fs_UC5_SYS_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.14" transition-key="4:0:7:5e41afa8-15bd-443a-88fb-ec276232a804" transition-magic="0:7;4:0:7:5e41afa8-15bd-443a-88fb-ec276232a804" exit-reason="" on_node="gcdoubwap01" call-id="19" rc-code="7" op-status="0" interval="0" last-run="1623419700" last-rc-change="1623419700" exec-time="130" queue-time="0" op-digest="f1f67b01fc16ed22d8fa1fe030d9c06b"/>
 +          </lrm_resource>
 +          <lrm_resource id="rsc_vip_int_ascs" type="IPaddr2" class="ocf" provider="heartbeat">
 +            <lrm_rsc_op id="rsc_vip_int_ascs_last_0" operation_key="rsc_vip_int_ascs_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.14" transition-key="5:0:7:5e41afa8-15bd-443a-88fb-ec276232a804" transition-magic="0:7;5:0:7:5e41afa8-15bd-443a-88fb-ec276232a804" exit-reason="" on_node="gcdoubwap01" call-id="23" rc-code="7" op-status="0" interval="0" last-run="1623419700" last-rc-change="1623419700" exec-time="105" queue-time="0" op-digest="da0b35400002727d7281b8f7051fe400"/>
 +          </lrm_resource>
 +          <lrm_resource id="rsc_vip_gcp_ascs" type="gcp-vpc-move-vip" class="ocf" provider="heartbeat">
 +            <lrm_rsc_op id="rsc_vip_gcp_ascs_last_0" operation_key="rsc_vip_gcp_ascs_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.14" transition-key="6:0:7:5e41afa8-15bd-443a-88fb-ec276232a804" transition-magic="0:0;6:0:7:5e41afa8-15bd-443a-88fb-ec276232a804" exit-reason="" on_node="gcdoubwap01" call-id="27" rc-code="0" op-status="0" interval="0" last-run="1623419705" last-rc-change="1623419705" exec-time="1842" queue-time="0" op-digest="face88a40d76658d0caa541eefc02ca8"/>
 +            <lrm_rsc_op id="rsc_vip_gcp_ascs_last_failure_0" operation_key="rsc_vip_gcp_ascs_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.14" transition-key="6:0:7:5e41afa8-15bd-443a-88fb-ec276232a804" transition-magic="0:0;6:0:7:5e41afa8-15bd-443a-88fb-ec276232a804" exit-reason="" on_node="gcdoubwap01" call-id="27" rc-code="0" op-status="0" interval="0" last-run="1623419705" last-rc-change="1623419705" exec-time="1842" queue-time="0" op-digest="face88a40d76658d0caa541eefc02ca8"/>
 +          </lrm_resource>
 +          <lrm_resource id="fs_UC5_ascs" type="Filesystem" class="ocf" provider="heartbeat">
 +            <lrm_rsc_op id="fs_UC5_ascs_last_0" operation_key="fs_UC5_ascs_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.14" transition-key="7:0:7:5e41afa8-15bd-443a-88fb-ec276232a804" transition-magic="0:7;7:0:7:5e41afa8-15bd-443a-88fb-ec276232a804" exit-reason="" on_node="gcdoubwap01" call-id="31" rc-code="7" op-status="0" interval="0" last-run="1623419705" last-rc-change="1623419705" exec-time="241" queue-time="0" op-digest="acac63abd6c034d7dad4aae73e2ca95d"/>
 +          </lrm_resource>
 +          <lrm_resource id="rsc_sap_UC5_ASCS11" type="SAPInstance" class="ocf" provider="heartbeat">
 +            <lrm_rsc_op id="rsc_sap_UC5_ASCS11_last_0" operation_key="rsc_sap_UC5_ASCS11_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.14" transition-key="8:0:7:5e41afa8-15bd-443a-88fb-ec276232a804" transition-magic="0:7;8:0:7:5e41afa8-15bd-443a-88fb-ec276232a804" exit-reason="" on_node="gcdoubwap01" call-id="35" rc-code="7" op-status="0" interval="0" last-run="1623419705" last-rc-change="1623419705" exec-time="969" queue-time="0" op-digest="08c114a33aa3c16b3204ff09cb983107" op-force-restart=" ERS_START_PROFILE  ERS_InstanceName  START_PROFILE  InstanceName " op-restart-digest="315a463141e0ef59afedf7a62a8d6362"/>
 +          </lrm_resource>
 +          <lrm_resource id="rsc_vip_init_ers" type="IPaddr2" class="ocf" provider="heartbeat">
 +            <lrm_rsc_op id="rsc_vip_init_ers_last_0" operation_key="rsc_vip_init_ers_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.14" transition-key="9:0:7:5e41afa8-15bd-443a-88fb-ec276232a804" transition-magic="0:7;9:0:7:5e41afa8-15bd-443a-88fb-ec276232a804" exit-reason="" on_node="gcdoubwap01" call-id="39" rc-code="7" op-status="0" interval="0" last-run="1623419705" last-rc-change="1623419705" exec-time="1033" queue-time="0" op-digest="7b29d7af6a7baa6015d1eeac471a9b42"/>
 +          </lrm_resource>
 +          <lrm_resource id="rsc_vip_gcp_ers" type="gcp-vpc-move-vip" class="ocf" provider="heartbeat">
 +            <lrm_rsc_op id="rsc_vip_gcp_ers_last_0" operation_key="rsc_vip_gcp_ers_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.14" transition-key="10:0:7:5e41afa8-15bd-443a-88fb-ec276232a804" transition-magic="0:7;10:0:7:5e41afa8-15bd-443a-88fb-ec276232a804" exit-reason="" on_node="gcdoubwap01" call-id="43" rc-code="7" op-status="0" interval="0" last-run="1623419705" last-rc-change="1623419705" exec-time="1702" queue-time="0" op-digest="10365a97fe5a5864a3679c314bf65bfd"/>
 +          </lrm_resource>
 +          <lrm_resource id="fs_UC5_ers" type="Filesystem" class="ocf" provider="heartbeat">
 +            <lrm_rsc_op id="fs_UC5_ers_last_0" operation_key="fs_UC5_ers_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.14" transition-key="11:0:7:5e41afa8-15bd-443a-88fb-ec276232a804" transition-magic="0:7;11:0:7:5e41afa8-15bd-443a-88fb-ec276232a804" exit-reason="" on_node="gcdoubwap01" call-id="47" rc-code="7" op-status="0" interval="0" last-run="1623419706" last-rc-change="1623419706" exec-time="709" queue-time="0" op-digest="61e45529b2da32c1e53055238a00ca99"/>
 +          </lrm_resource>
 +          <lrm_resource id="rsc_sap_UC5_ERS12" type="SAPInstance" class="ocf" provider="heartbeat">
 +            <lrm_rsc_op id="rsc_sap_UC5_ERS12_last_0" operation_key="rsc_sap_UC5_ERS12_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.14" transition-key="12:0:7:5e41afa8-15bd-443a-88fb-ec276232a804" transition-magic="0:7;12:0:7:5e41afa8-15bd-443a-88fb-ec276232a804" exit-reason="" on_node="gcdoubwap01" call-id="51" rc-code="7" op-status="0" interval="0" last-run="1623419706" last-rc-change="1623419706" exec-time="914" queue-time="1" op-digest="b550e70bd4203af88473e4d914b11f87" op-force-restart=" ERS_START_PROFILE  ERS_InstanceName  START_PROFILE  InstanceName " op-restart-digest="2fb6ec6eb77e25302c8dc0dad84dc46f"/>
 +          </lrm_resource>
 +        </lrm_resources>
 +      </lrm>
 +    </node_state>
 +    <node_state id="2" uname="gcdoubwap02" crmd="offline" crm-debug-origin="post_cache_update" in_ccm="true"/>
 +  </status>
 +</cib>
 -- 
 1.8.3.1
--- a/SOURCES/011-crm_attribute-regression.patch
+++ b/SOURCES/011-crm_attribute-regression.patch
@ -1,150 +0,0 @@
 From ea5510dd979bb6d375324cda26925d9e7c4362f5 Mon Sep 17 00:00:00 2001
 From: Chris Lumens <clumens@redhat.com>
 Date: Mon, 19 Jul 2021 10:04:16 -0400
 Subject: [PATCH 1/2] Low: tools: The --get-value option does not require an
 arg.
 Regression in 2.1.0 introduced by 15f5c2901.
 ---
 tools/crm_attribute.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
 diff --git a/tools/crm_attribute.c b/tools/crm_attribute.c
 index 2cc8d26..8a5b4e4 100644
 --- a/tools/crm_attribute.c
 +++ b/tools/crm_attribute.c
@@ -242,7 +242,7 @@ static GOptionEntry deprecated_entries[] = {
       NULL, NULL
     },
 -    { "get-value", 0, G_OPTION_FLAG_HIDDEN, G_OPTION_ARG_CALLBACK, value_cb,
 +    { "get-value", 0, G_OPTION_FLAG_HIDDEN|G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, value_cb,
       NULL, NULL
     },
 -- 
 1.8.3.1
 From ef054d943afe8e60017f6adc4e25f88a59ac91a4 Mon Sep 17 00:00:00 2001
 From: Chris Lumens <clumens@redhat.com>
 Date: Mon, 19 Jul 2021 11:37:04 -0400
 Subject: [PATCH 2/2] Low: libcrmcommon: Allow negative numbers as cmdline
 options.
 The bug here is that negative numbers (for instance, negative scores)
 are not supported as command line arguments.  Because we break up a
 string that starts with a single dash into multiple arguments, "-1000"
 becomes "-1", "-0", "-0", and "-0".
 Because we don't have enough information about what is happening on the
 command line, the best we can do here is recognize something as a
 negative number and pass it on.  Any errors will have to be detected at
 a later step.
 Also note that we only recognize negative numbers if they start with
 1-9.  Starting with 0 will be recognized as some sort of string.
 Regression in 2.1.0 caused by a long-standing bug in
 pcmk__cmdline_preproc_test.
 ---
 lib/common/cmdline.c                               | 29 ++++++++++++++++++++++
 .../tests/cmdline/pcmk__cmdline_preproc_test.c     | 24 +++++++++++++++++-
 2 files changed, 52 insertions(+), 1 deletion(-)
 diff --git a/lib/common/cmdline.c b/lib/common/cmdline.c
 index 7c95d02..9c1b810 100644
 --- a/lib/common/cmdline.c
 +++ b/lib/common/cmdline.c
@@ -9,6 +9,7 @@
 #include <crm_internal.h>
 +#include <ctype.h>
 #include <glib.h>
 #include <crm/crm.h>
@@ -189,6 +190,34 @@ pcmk__cmdline_preproc(char **argv, const char *special) {
             /* Skip over leading dash */
             char *ch = argv[i]+1;
 +            /* This looks like the start of a number, which means it is a negative
 +             * number.  It's probably the argument to the preceeding option, but
 +             * we can't know that here.  Copy it over and let whatever handles
 +             * arguments next figure it out.
 +             */
 +            if (*ch != '\0' && *ch >= '1' && *ch <= '9') {
 +                bool is_numeric = true;
 +
 +                while (*ch != '\0') {
 +                    if (!isdigit(*ch)) {
 +                        is_numeric = false;
 +                        break;
 +                    }
 +
 +                    ch++;
 +                }
 +
 +                if (is_numeric) {
 +                    g_ptr_array_add(arr, g_strdup_printf("%s", argv[i]));
 +                    continue;
 +                } else {
 +                    /* This argument wasn't entirely numeric.  Reset ch to the
 +                     * beginning so we can process it one character at a time.
 +                     */
 +                    ch = argv[i]+1;
 +                }
 +            }
 +
             while (*ch != '\0') {
                 /* This is a special short argument that takes an option.  getopt
                  * allows values to be interspersed with a list of arguments, but
 diff --git a/lib/common/tests/cmdline/pcmk__cmdline_preproc_test.c b/lib/common/tests/cmdline/pcmk__cmdline_preproc_test.c
 index b8506c6..9a752ef 100644
 --- a/lib/common/tests/cmdline/pcmk__cmdline_preproc_test.c
 +++ b/lib/common/tests/cmdline/pcmk__cmdline_preproc_test.c
@@ -1,5 +1,5 @@
 /*
 - * Copyright 2020 the Pacemaker project contributors
 + * Copyright 2020-2021 the Pacemaker project contributors
  *
  * The version control history for this file may have further details.
  *
@@ -86,6 +86,26 @@ long_arg(void) {
     g_strfreev(processed);
 }
 +static void
 +negative_score(void) {
 +    const char *argv[] = { "-v", "-1000", NULL };
 +    const gchar *expected[] = { "-v", "-1000", NULL };
 +
 +    gchar **processed = pcmk__cmdline_preproc((char **) argv, "v");
 +    LISTS_EQ(processed, expected);
 +    g_strfreev(processed);
 +}
 +
 +static void
 +negative_score_2(void) {
 +    const char *argv[] = { "-1i3", NULL };
 +    const gchar *expected[] = { "-1", "-i", "-3", NULL };
 +
 +    gchar **processed = pcmk__cmdline_preproc((char **) argv, NULL);
 +    LISTS_EQ(processed, expected);
 +    g_strfreev(processed);
 +}
 +
 int
 main(int argc, char **argv)
 {
@@ -98,5 +118,7 @@ main(int argc, char **argv)
     g_test_add_func("/common/cmdline/preproc/special_args", special_args);
     g_test_add_func("/common/cmdline/preproc/special_arg_at_end", special_arg_at_end);
     g_test_add_func("/common/cmdline/preproc/long_arg", long_arg);
 +    g_test_add_func("/common/cmdline/preproc/negative_score", negative_score);
 +    g_test_add_func("/common/cmdline/preproc/negative_score_2", negative_score_2);
     return g_test_run();
 }
 -- 
 1.8.3.1
--- a/SOURCES/012-string-arguments.patch
+++ b/SOURCES/012-string-arguments.patch
@ -1,221 +0,0 @@
 From 2eee93e8f9ea2daa81769bc69843d63ced1a7112 Mon Sep 17 00:00:00 2001
 From: Chris Lumens <clumens@redhat.com>
 Date: Tue, 20 Jul 2021 16:39:07 -0400
 Subject: [PATCH 1/2] Low: tools: Audit command line options.
 This just goes through and makes sure the command line options that take
 arguments are in the special parameter to pcmk__cmdline_preproc, and
 that options that do not take arguments are not.
 ---
 tools/crm_attribute.c | 2 +-
 tools/crm_error.c     | 2 +-
 tools/crm_resource.c  | 2 +-
 tools/crm_rule.c      | 2 +-
 tools/crm_simulate.c  | 2 +-
 tools/crmadmin.c      | 2 +-
 tools/stonith_admin.c | 2 +-
 7 files changed, 7 insertions(+), 7 deletions(-)
 diff --git a/tools/crm_attribute.c b/tools/crm_attribute.c
 index 8a5b4e4..6bd4e2a 100644
 --- a/tools/crm_attribute.c
 +++ b/tools/crm_attribute.c
@@ -312,7 +312,7 @@ main(int argc, char **argv)
     GOptionGroup *output_group = NULL;
     pcmk__common_args_t *args = pcmk__new_common_args(SUMMARY);
 -    gchar **processed_args = pcmk__cmdline_preproc(argv, "DGNPdilnpstv");
 +    gchar **processed_args = pcmk__cmdline_preproc(argv, "NPUdilnpstv");
     GOptionContext *context = build_arg_context(args, &output_group);
     if (!g_option_context_parse_strv(context, &processed_args, &error)) {
 diff --git a/tools/crm_error.c b/tools/crm_error.c
 index b4328ce..923f393 100644
 --- a/tools/crm_error.c
 +++ b/tools/crm_error.c
@@ -79,7 +79,7 @@ main(int argc, char **argv)
     GOptionGroup *output_group = NULL;
     pcmk__common_args_t *args = pcmk__new_common_args(SUMMARY);
 -    gchar **processed_args = pcmk__cmdline_preproc(argv, "lrnX");
 +    gchar **processed_args = pcmk__cmdline_preproc(argv, NULL);
     GOptionContext *context = build_arg_context(args, &output_group);
     if (!g_option_context_parse_strv(context, &processed_args, &error)) {
 diff --git a/tools/crm_resource.c b/tools/crm_resource.c
 index fa7902c..d8e140f 100644
 --- a/tools/crm_resource.c
 +++ b/tools/crm_resource.c
@@ -1530,7 +1530,7 @@ main(int argc, char **argv)
      */
     args = pcmk__new_common_args(SUMMARY);
 -    processed_args = pcmk__cmdline_preproc(argv, "GINSTdginpstuv");
 +    processed_args = pcmk__cmdline_preproc(argv, "GHINSTdginpstuvx");
     context = build_arg_context(args, &output_group);
     pcmk__register_formats(output_group, formats);
 diff --git a/tools/crm_rule.c b/tools/crm_rule.c
 index 8b19bcd..30c5155 100644
 --- a/tools/crm_rule.c
 +++ b/tools/crm_rule.c
@@ -239,7 +239,7 @@ main(int argc, char **argv)
     pcmk__common_args_t *args = pcmk__new_common_args(SUMMARY);
     GOptionContext *context = build_arg_context(args);
 -    gchar **processed_args = pcmk__cmdline_preproc(argv, "nopNO");
 +    gchar **processed_args = pcmk__cmdline_preproc(argv, "drX");
     if (!g_option_context_parse_strv(context, &processed_args, &error)) {
         exit_code = CRM_EX_USAGE;
 diff --git a/tools/crm_simulate.c b/tools/crm_simulate.c
 index 0406bff..c83b1b1 100644
 --- a/tools/crm_simulate.c
 +++ b/tools/crm_simulate.c
@@ -865,7 +865,7 @@ main(int argc, char **argv)
     GOptionGroup *output_group = NULL;
     pcmk__common_args_t *args = pcmk__new_common_args(SUMMARY);
 -    gchar **processed_args = pcmk__cmdline_preproc(argv, "bdefgiqrtuwxDFGINO");
 +    gchar **processed_args = pcmk__cmdline_preproc(argv, "bdefgiqrtuwxDFGINOP");
     GOptionContext *context = build_arg_context(args, &output_group);
     /* This must come before g_option_context_parse_strv. */
 diff --git a/tools/crmadmin.c b/tools/crmadmin.c
 index 5cbde1b..b98f282 100644
 --- a/tools/crmadmin.c
 +++ b/tools/crmadmin.c
@@ -188,7 +188,7 @@ main(int argc, char **argv)
     GOptionGroup *output_group = NULL;
     pcmk__common_args_t *args = pcmk__new_common_args(SUMMARY);
 -    gchar **processed_args = pcmk__cmdline_preproc(argv, "itBDEHKNPS");
 +    gchar **processed_args = pcmk__cmdline_preproc(argv, "itKNS");
     GOptionContext *context = build_arg_context(args, &output_group);
     pcmk__register_formats(output_group, formats);
 diff --git a/tools/stonith_admin.c b/tools/stonith_admin.c
 index 6773cea..2d48326 100644
 --- a/tools/stonith_admin.c
 +++ b/tools/stonith_admin.c
@@ -349,7 +349,7 @@ main(int argc, char **argv)
     GOptionGroup *output_group = NULL;
     pcmk__common_args_t *args = pcmk__new_common_args(SUMMARY);
 -    gchar **processed_args = pcmk__cmdline_preproc(argv, "adehilorstvBCDFHQRTU");
 +    gchar **processed_args = pcmk__cmdline_preproc(argv, "adehilorstvyBCDFHQRTU");
     GOptionContext *context = build_arg_context(args, &output_group);
     pcmk__register_formats(output_group, formats);
 -- 
 1.8.3.1
 From 8301678ad1162450814d2fea5288aefe47a67a74 Mon Sep 17 00:00:00 2001
 From: Chris Lumens <clumens@redhat.com>
 Date: Tue, 20 Jul 2021 16:40:58 -0400
 Subject: [PATCH 2/2] Low: libcrmcommon: Also allow string arguments that start
 with a dash.
 There's various places where an option to a command line argument could
 itself be a valid command line argument.  For instance:
    crm_attribute -n crm_mon_options -v "-1i3"
 The previous patching to pcmk__cmdline_preproc did not take this into
 account.  With this patch, options that are last in a string (or by
 themselves) and take an argument will have the next command line option
 grabbed and copied straight through without processing.
 Regression in 2.1.0 caused by a long-standing bug in pcmk__cmdline_preproc.
 ---
 lib/common/cmdline.c                               |  8 ++++++
 .../tests/cmdline/pcmk__cmdline_preproc_test.c     | 33 ++++++++++++++++++++++
 2 files changed, 41 insertions(+)
 diff --git a/lib/common/cmdline.c b/lib/common/cmdline.c
 index 9c1b810..1ca6147 100644
 --- a/lib/common/cmdline.c
 +++ b/lib/common/cmdline.c
@@ -146,6 +146,7 @@ gchar **
 pcmk__cmdline_preproc(char **argv, const char *special) {
     GPtrArray *arr = NULL;
     bool saw_dash_dash = false;
 +    bool copy_option = false;
     if (argv == NULL) {
         return NULL;
@@ -175,6 +176,12 @@ pcmk__cmdline_preproc(char **argv, const char *special) {
             continue;
         }
 +        if (copy_option == true) {
 +            g_ptr_array_add(arr, g_strdup(argv[i]));
 +            copy_option = false;
 +            continue;
 +        }
 +
         /* This is just a dash by itself.  That could indicate stdin/stdout, or
          * it could be user error.  Copy it over and let glib figure it out.
          */
@@ -239,6 +246,7 @@ pcmk__cmdline_preproc(char **argv, const char *special) {
                      */
                     } else {
                         g_ptr_array_add(arr, g_strdup_printf("-%c", *ch));
 +                        copy_option = true;
                         ch++;
                     }
 diff --git a/lib/common/tests/cmdline/pcmk__cmdline_preproc_test.c b/lib/common/tests/cmdline/pcmk__cmdline_preproc_test.c
 index 9a752ef..edc5640 100644
 --- a/lib/common/tests/cmdline/pcmk__cmdline_preproc_test.c
 +++ b/lib/common/tests/cmdline/pcmk__cmdline_preproc_test.c
@@ -106,6 +106,36 @@ negative_score_2(void) {
     g_strfreev(processed);
 }
 +static void
 +string_arg_with_dash(void) {
 +    const char *argv[] = { "-n", "crm_mon_options", "-v", "--opt1 --opt2", NULL };
 +    const gchar *expected[] = { "-n", "crm_mon_options", "-v", "--opt1 --opt2", NULL };
 +
 +    gchar **processed = pcmk__cmdline_preproc((char **) argv, "v");
 +    LISTS_EQ(processed, expected);
 +    g_strfreev(processed);
 +}
 +
 +static void
 +string_arg_with_dash_2(void) {
 +    const char *argv[] = { "-n", "crm_mon_options", "-v", "-1i3", NULL };
 +    const gchar *expected[] = { "-n", "crm_mon_options", "-v", "-1i3", NULL };
 +
 +    gchar **processed = pcmk__cmdline_preproc((char **) argv, "v");
 +    LISTS_EQ(processed, expected);
 +    g_strfreev(processed);
 +}
 +
 +static void
 +string_arg_with_dash_3(void) {
 +    const char *argv[] = { "-abc", "-1i3", NULL };
 +    const gchar *expected[] = { "-a", "-b", "-c", "-1i3", NULL };
 +
 +    gchar **processed = pcmk__cmdline_preproc((char **) argv, "c");
 +    LISTS_EQ(processed, expected);
 +    g_strfreev(processed);
 +}
 +
 int
 main(int argc, char **argv)
 {
@@ -120,5 +150,8 @@ main(int argc, char **argv)
     g_test_add_func("/common/cmdline/preproc/long_arg", long_arg);
     g_test_add_func("/common/cmdline/preproc/negative_score", negative_score);
     g_test_add_func("/common/cmdline/preproc/negative_score_2", negative_score_2);
 +    g_test_add_func("/common/cmdline/preproc/string_arg_with_dash", string_arg_with_dash);
 +    g_test_add_func("/common/cmdline/preproc/string_arg_with_dash_2", string_arg_with_dash_2);
 +    g_test_add_func("/common/cmdline/preproc/string_arg_with_dash_3", string_arg_with_dash_3);
     return g_test_run();
 }
 -- 
 1.8.3.1
--- a/SOURCES/013-leaks.patch
+++ b/SOURCES/013-leaks.patch
@ -1,241 +0,0 @@
 From bee54eba4d9c28d3a7907a3e13a5deeee6bc0916 Mon Sep 17 00:00:00 2001
 From: Ken Gaillot <kgaillot@redhat.com>
 Date: Tue, 27 Jul 2021 11:01:04 -0500
 Subject: [PATCH 1/2] Low: tools: avoid (insignificant) memory leaks
 detected by valgrind
 ---
 lib/pacemaker/pcmk_cluster_queries.c |  2 ++
 tools/crm_diff.c                     |  2 +-
 tools/crm_resource.c                 | 33 ++++++++++++++++++++-------------
 tools/crm_resource_ban.c             |  2 +-
 4 files changed, 24 insertions(+), 15 deletions(-)
 diff --git a/lib/pacemaker/pcmk_cluster_queries.c b/lib/pacemaker/pcmk_cluster_queries.c
 index c68cf9d..46e5538 100644
 --- a/lib/pacemaker/pcmk_cluster_queries.c
 +++ b/lib/pacemaker/pcmk_cluster_queries.c
@@ -440,6 +440,7 @@ pcmk__list_nodes(pcmk__output_t *out, char *node_types, gboolean BASH_EXPORT)
     }
     rc = the_cib->cmds->signon(the_cib, crm_system_name, cib_command);
     if (rc != pcmk_ok) {
 +        cib_delete(the_cib);
         return pcmk_legacy2rc(rc);
     }
@@ -488,6 +489,7 @@ pcmk__list_nodes(pcmk__output_t *out, char *node_types, gboolean BASH_EXPORT)
         free_xml(xml_node);
     }
     the_cib->cmds->signoff(the_cib);
 +    cib_delete(the_cib);
     return pcmk_legacy2rc(rc);
 }
 diff --git a/tools/crm_diff.c b/tools/crm_diff.c
 index b37f0ea..9890c10 100644
 --- a/tools/crm_diff.c
 +++ b/tools/crm_diff.c
@@ -383,5 +383,5 @@ done:
     free_xml(object_2);
     pcmk__output_and_clear_error(error, NULL);
 -    return exit_code;
 +    crm_exit(exit_code);
 }
 diff --git a/tools/crm_resource.c b/tools/crm_resource.c
 index d8e140f..8ca90cb 100644
 --- a/tools/crm_resource.c
 +++ b/tools/crm_resource.c
@@ -1081,6 +1081,8 @@ clear_constraints(pcmk__output_t *out, xmlNodePtr *cib_xml_copy)
             g_set_error(&error, PCMK__RC_ERROR, rc,
                         "Could not get modified CIB: %s\n", pcmk_strerror(rc));
             g_list_free(before);
 +            free_xml(*cib_xml_copy);
 +            *cib_xml_copy = NULL;
             return rc;
         }
@@ -1232,29 +1234,34 @@ populate_working_set(xmlNodePtr *cib_xml_copy)
     if (options.xml_file != NULL) {
         *cib_xml_copy = filename2xml(options.xml_file);
 +        if (*cib_xml_copy == NULL) {
 +            rc = pcmk_rc_cib_corrupt;
 +        }
     } else {
         rc = cib_conn->cmds->query(cib_conn, NULL, cib_xml_copy, cib_scope_local | cib_sync_call);
         rc = pcmk_legacy2rc(rc);
     }
 -    if(rc != pcmk_rc_ok) {
 -        return rc;
 +    if (rc == pcmk_rc_ok) {
 +        data_set = pe_new_working_set();
 +        if (data_set == NULL) {
 +            rc = ENOMEM;
 +        } else {
 +            pe__set_working_set_flags(data_set,
 +                                      pe_flag_no_counts|pe_flag_no_compat);
 +            data_set->priv = out;
 +            rc = update_working_set_xml(data_set, cib_xml_copy);
 +        }
     }
 -    /* Populate the working set instance */
 -    data_set = pe_new_working_set();
 -    if (data_set == NULL) {
 -        rc = ENOMEM;
 +    if (rc != pcmk_rc_ok) {
 +        free_xml(*cib_xml_copy);
 +        *cib_xml_copy = NULL;
         return rc;
     }
 -    pe__set_working_set_flags(data_set, pe_flag_no_counts|pe_flag_no_compat);
 -    data_set->priv = out;
 -    rc = update_working_set_xml(data_set, cib_xml_copy);
 -    if (rc == pcmk_rc_ok) {
 -        cluster_status(data_set);
 -    }
 -    return rc;
 +    cluster_status(data_set);
 +    return pcmk_rc_ok;
 }
 static int
 diff --git a/tools/crm_resource_ban.c b/tools/crm_resource_ban.c
 index a297d49..2c4f48d 100644
 --- a/tools/crm_resource_ban.c
 +++ b/tools/crm_resource_ban.c
@@ -292,7 +292,7 @@ resource_clear_node_in_location(const char *rsc_id, const char *host, cib_t * ci
         rc = pcmk_legacy2rc(rc);
     }
 -    free(fragment);
 +    free_xml(fragment);
     return rc;
 }
 -- 
 1.8.3.1
 From a30ff4a87f291a0c9e03c4efb9c9046d2ac594f1 Mon Sep 17 00:00:00 2001
 From: Ken Gaillot <kgaillot@redhat.com>
 Date: Tue, 27 Jul 2021 11:26:59 -0500
 Subject: [PATCH 2/2] Fix: tools: avoid memory leaks in crm_mon
 could be significant in an interactive session
 regressions introduced in 2.0.4 and 2.0.5
 ---
 lib/pengine/bundle.c    | 3 ++-
 lib/pengine/clone.c     | 5 ++---
 lib/pengine/pe_output.c | 3 +++
 3 files changed, 7 insertions(+), 4 deletions(-)
 diff --git a/lib/pengine/bundle.c b/lib/pengine/bundle.c
 index 6ba786a..7e1d428 100644
 --- a/lib/pengine/bundle.c
 +++ b/lib/pengine/bundle.c
@@ -1497,7 +1497,7 @@ pe__bundle_xml(pcmk__output_t *out, va_list args)
     for (GList *gIter = bundle_data->replicas; gIter != NULL;
          gIter = gIter->next) {
         pe__bundle_replica_t *replica = gIter->data;
 -        char *id = pcmk__itoa(replica->offset);
 +        char *id = NULL;
         gboolean print_ip, print_child, print_ctnr, print_remote;
         CRM_ASSERT(replica);
@@ -1531,6 +1531,7 @@ pe__bundle_xml(pcmk__output_t *out, va_list args)
             CRM_ASSERT(rc == pcmk_rc_ok);
         }
 +        id = pcmk__itoa(replica->offset);
         rc = pe__name_and_nvpairs_xml(out, true, "replica", 1, "id", id);
         free(id);
         CRM_ASSERT(rc == pcmk_rc_ok);
 diff --git a/lib/pengine/clone.c b/lib/pengine/clone.c
 index 6323692..ab91fd1 100644
 --- a/lib/pengine/clone.c
 +++ b/lib/pengine/clone.c
@@ -807,10 +807,10 @@ pe__clone_html(pcmk__output_t *out, va_list args)
         pcmk__add_word(&list_text, &list_text_len, host->details->uname);
         active_instances++;
     }
 +    g_list_free(promoted_list);
     if (list_text != NULL) {
         out->list_item(out, NULL, PROMOTED_INSTANCES ": [ %s ]", list_text);
 -        g_list_free(promoted_list);
         free(list_text);
         list_text = NULL;
         list_text_len = 0;
@@ -828,6 +828,7 @@ pe__clone_html(pcmk__output_t *out, va_list args)
         pcmk__add_word(&list_text, &list_text_len, host->details->uname);
         active_instances++;
     }
 +    g_list_free(started_list);
     if (list_text != NULL) {
         if (pcmk_is_set(rsc->flags, pe_rsc_promotable)) {
@@ -847,7 +848,6 @@ pe__clone_html(pcmk__output_t *out, va_list args)
             out->list_item(out, NULL, "Started: [ %s ]", list_text);
         }
 -        g_list_free(started_list);
         free(list_text);
         list_text = NULL;
         list_text_len = 0;
@@ -1048,10 +1048,10 @@ pe__clone_text(pcmk__output_t *out, va_list args)
         pcmk__add_word(&list_text, &list_text_len, host->details->uname);
         active_instances++;
     }
 +    g_list_free(promoted_list);
     if (list_text != NULL) {
         out->list_item(out, PROMOTED_INSTANCES, "[ %s ]", list_text);
 -        g_list_free(promoted_list);
         free(list_text);
         list_text = NULL;
         list_text_len = 0;
@@ -1069,6 +1069,7 @@ pe__clone_text(pcmk__output_t *out, va_list args)
         pcmk__add_word(&list_text, &list_text_len, host->details->uname);
         active_instances++;
     }
 +    g_list_free(started_list);
     if (list_text != NULL) {
         if (pcmk_is_set(rsc->flags, pe_rsc_promotable)) {
@@ -1084,7 +1085,6 @@ pe__clone_text(pcmk__output_t *out, va_list args)
             out->list_item(out, "Started", "[ %s ]", list_text);
         }
 -        g_list_free(started_list);
         free(list_text);
         list_text = NULL;
     }
 diff --git a/lib/pengine/pe_output.c b/lib/pengine/pe_output.c
 index b8997c4..20bd1a9 100644
 --- a/lib/pengine/pe_output.c
 +++ b/lib/pengine/pe_output.c
@@ -1410,6 +1410,8 @@ node_text(pcmk__output_t *out, va_list args) {
                     out->end_list(out);
                     out->end_list(out);
 +
 +                    g_list_free(rscs);
                 }
             } else {
@@ -1739,6 +1741,7 @@ node_attribute_list(pcmk__output_t *out, va_list args) {
         }
         if (!pcmk__str_in_list(only_node, node->details->uname)) {
 +            g_list_free(attr_list);
             continue;
         }
 -- 
 1.8.3.1
--- a/SOURCES/014-str-list.patch
+++ b/SOURCES/014-str-list.patch
@ -1,465 +0,0 @@
 From 45813df3eb4c8ad8b1744fa5dd56af86ad0fb3dd Mon Sep 17 00:00:00 2001
 From: Chris Lumens <clumens@redhat.com>
 Date: Thu, 17 Jun 2021 16:07:55 -0400
 Subject: [PATCH] Refactor: libs: pcmk__str_in_list should support pcmk__str_*
 flags.
 ---
 include/crm/common/strings_internal.h |  2 +-
 lib/common/strings.c                  | 34 +++++++++++++++++++++++----
 lib/fencing/st_output.c               | 10 ++++----
 lib/pengine/bundle.c                  |  8 +++----
 lib/pengine/clone.c                   | 28 +++++++++++-----------
 lib/pengine/group.c                   | 18 +++++++-------
 lib/pengine/native.c                  |  4 ++--
 lib/pengine/pe_output.c               | 22 ++++++++---------
 lib/pengine/utils.c                   |  6 ++---
 9 files changed, 79 insertions(+), 53 deletions(-)
 diff --git a/include/crm/common/strings_internal.h b/include/crm/common/strings_internal.h
 index 94982cb4e..687079814 100644
 --- a/include/crm/common/strings_internal.h
 +++ b/include/crm/common/strings_internal.h
@@ -117,7 +117,7 @@ pcmk__intkey_table_remove(GHashTable *hash_table, int key)
     return g_hash_table_remove(hash_table, GINT_TO_POINTER(key));
 }
 -gboolean pcmk__str_in_list(GList *lst, const gchar *s);
 +gboolean pcmk__str_in_list(GList *lst, const gchar *s, uint32_t flags);
 bool pcmk__strcase_any_of(const char *s, ...) G_GNUC_NULL_TERMINATED;
 bool pcmk__str_any_of(const char *s, ...) G_GNUC_NULL_TERMINATED;
 diff --git a/lib/common/strings.c b/lib/common/strings.c
 index 3264db5b6..e1e98803b 100644
 --- a/lib/common/strings.c
 +++ b/lib/common/strings.c
@@ -872,14 +872,30 @@ pcmk__parse_ll_range(const char *srcstring, long long *start, long long *end)
  * Search \p lst for \p s, taking case into account.  As a special case,
  * if "*" is the only element of \p lst, the search is successful.
  *
 - * \param[in]  lst  List to search
 - * \param[in]  s    String to search for
 + * Behavior can be changed with various flags:
 + *
 + * - pcmk__str_casei - By default, comparisons are done taking case into
 + *                     account.  This flag makes comparisons case-insensitive.
 + * - pcmk__str_null_matches - If the input string is NULL, return TRUE.
 + *
 + * \note The special "*" matching rule takes precedence over flags.  In
 + *       particular, "*" will match a NULL input string even without
 + *       pcmk__str_null_matches being specified.
 + *
 + * \note No matter what input string or flags are provided, an empty
 + *       list will always return FALSE.
 + *
 + * \param[in]  lst   List to search
 + * \param[in]  s     String to search for
 + * \param[in]  flags A bitfield of pcmk__str_flags to modify operation
  *
  * \return \c TRUE if \p s is in \p lst, or \c FALSE otherwise
  */
 gboolean
 -pcmk__str_in_list(GList *lst, const gchar *s)
 +pcmk__str_in_list(GList *lst, const gchar *s, uint32_t flags)
 {
 +    GCompareFunc fn;
 +
     if (lst == NULL) {
         return FALSE;
     }
@@ -888,7 +904,17 @@ pcmk__str_in_list(GList *lst, const gchar *s)
         return TRUE;
     }
 -    return g_list_find_custom(lst, s, (GCompareFunc) strcmp) != NULL;
 +    if (s == NULL) {
 +        return pcmk_is_set(flags, pcmk__str_null_matches);
 +    }
 +
 +    if (pcmk_is_set(flags, pcmk__str_casei)) {
 +        fn = (GCompareFunc) strcasecmp;
 +    } else {
 +        fn = (GCompareFunc) strcmp;
 +    }
 +
 +    return g_list_find_custom(lst, s, fn) != NULL;
 }
 static bool
 diff --git a/lib/fencing/st_output.c b/lib/fencing/st_output.c
 index 568ae46a8..e1ae8ac87 100644
 --- a/lib/fencing/st_output.c
 +++ b/lib/fencing/st_output.c
@@ -47,7 +47,7 @@ stonith__failed_history(pcmk__output_t *out, va_list args) {
             continue;
         }
 -        if (!pcmk__str_in_list(only_node, hp->target)) {
 +        if (!pcmk__str_in_list(only_node, hp->target, pcmk__str_none)) {
             continue;
         }
@@ -72,7 +72,7 @@ stonith__history(pcmk__output_t *out, va_list args) {
     int rc = pcmk_rc_no_output;
     for (stonith_history_t *hp = history; hp; hp = hp->next) {
 -        if (!pcmk__str_in_list(only_node, hp->target)) {
 +        if (!pcmk__str_in_list(only_node, hp->target, pcmk__str_none)) {
             continue;
         }
@@ -101,7 +101,7 @@ stonith__full_history(pcmk__output_t *out, va_list args) {
     int rc = pcmk_rc_no_output;
     for (stonith_history_t *hp = history; hp; hp = hp->next) {
 -        if (!pcmk__str_in_list(only_node, hp->target)) {
 +        if (!pcmk__str_in_list(only_node, hp->target, pcmk__str_none)) {
             continue;
         }
@@ -129,7 +129,7 @@ full_history_xml(pcmk__output_t *out, va_list args) {
     if (history_rc == 0) {
         for (stonith_history_t *hp = history; hp; hp = hp->next) {
 -            if (!pcmk__str_in_list(only_node, hp->target)) {
 +            if (!pcmk__str_in_list(only_node, hp->target, pcmk__str_none)) {
                 continue;
             }
@@ -218,7 +218,7 @@ stonith__pending_actions(pcmk__output_t *out, va_list args) {
     int rc = pcmk_rc_no_output;
     for (stonith_history_t *hp = history; hp; hp = hp->next) {
 -        if (!pcmk__str_in_list(only_node, hp->target)) {
 +        if (!pcmk__str_in_list(only_node, hp->target, pcmk__str_none)) {
             continue;
         }
 diff --git a/lib/pengine/bundle.c b/lib/pengine/bundle.c
 index 9237392e4..6ba786ae6 100644
 --- a/lib/pengine/bundle.c
 +++ b/lib/pengine/bundle.c
@@ -1492,7 +1492,7 @@ pe__bundle_xml(pcmk__output_t *out, va_list args)
         return rc;
     }
 -    print_everything = pcmk__str_in_list(only_rsc, rsc->id);
 +    print_everything = pcmk__str_in_list(only_rsc, rsc->id, pcmk__str_none);
     for (GList *gIter = bundle_data->replicas; gIter != NULL;
          gIter = gIter->next) {
@@ -1614,7 +1614,7 @@ pe__bundle_html(pcmk__output_t *out, va_list args)
         return rc;
     }
 -    print_everything = pcmk__str_in_list(only_rsc, rsc->id);
 +    print_everything = pcmk__str_in_list(only_rsc, rsc->id, pcmk__str_none);
     for (GList *gIter = bundle_data->replicas; gIter != NULL;
          gIter = gIter->next) {
@@ -1742,7 +1742,7 @@ pe__bundle_text(pcmk__output_t *out, va_list args)
         return rc;
     }
 -    print_everything = pcmk__str_in_list(only_rsc, rsc->id);
 +    print_everything = pcmk__str_in_list(only_rsc, rsc->id, pcmk__str_none);
     for (GList *gIter = bundle_data->replicas; gIter != NULL;
          gIter = gIter->next) {
@@ -2044,7 +2044,7 @@ pe__bundle_is_filtered(pe_resource_t *rsc, GList *only_rsc, gboolean check_paren
     gboolean passes = FALSE;
     pe__bundle_variant_data_t *bundle_data = NULL;
 -    if (pcmk__str_in_list(only_rsc, rsc_printable_id(rsc))) {
 +    if (pcmk__str_in_list(only_rsc, rsc_printable_id(rsc), pcmk__str_none)) {
         passes = TRUE;
     } else {
         get_bundle_variant_data(bundle_data, rsc);
 diff --git a/lib/pengine/clone.c b/lib/pengine/clone.c
 index 5662338f3..5a6bfa61f 100644
 --- a/lib/pengine/clone.c
 +++ b/lib/pengine/clone.c
@@ -624,8 +624,8 @@ pe__clone_xml(pcmk__output_t *out, va_list args)
         return rc;
     }
 -    print_everything = pcmk__str_in_list(only_rsc, rsc_printable_id(rsc)) ||
 -                       (strstr(rsc->id, ":") != NULL && pcmk__str_in_list(only_rsc, rsc->id));
 +    print_everything = pcmk__str_in_list(only_rsc, rsc_printable_id(rsc), pcmk__str_none) ||
 +                       (strstr(rsc->id, ":") != NULL && pcmk__str_in_list(only_rsc, rsc->id, pcmk__str_none));
     for (; gIter != NULL; gIter = gIter->next) {
         pe_resource_t *child_rsc = (pe_resource_t *) gIter->data;
@@ -693,8 +693,8 @@ pe__clone_html(pcmk__output_t *out, va_list args)
         return rc;
     }
 -    print_everything = pcmk__str_in_list(only_rsc, rsc_printable_id(rsc)) ||
 -                       (strstr(rsc->id, ":") != NULL && pcmk__str_in_list(only_rsc, rsc->id));
 +    print_everything = pcmk__str_in_list(only_rsc, rsc_printable_id(rsc), pcmk__str_none) ||
 +                       (strstr(rsc->id, ":") != NULL && pcmk__str_in_list(only_rsc, rsc->id, pcmk__str_none));
     out->begin_list(out, NULL, NULL, "Clone Set: %s [%s]%s%s%s%s",
                     rsc->id, ID(clone_data->xml_obj_child),
@@ -801,7 +801,7 @@ pe__clone_html(pcmk__output_t *out, va_list args)
     for (gIter = promoted_list; gIter; gIter = gIter->next) {
         pe_node_t *host = gIter->data;
 -        if (!pcmk__str_in_list(only_node, host->details->uname)) {
 +        if (!pcmk__str_in_list(only_node, host->details->uname, pcmk__str_none)) {
             continue;
         }
@@ -822,7 +822,7 @@ pe__clone_html(pcmk__output_t *out, va_list args)
     for (gIter = started_list; gIter; gIter = gIter->next) {
         pe_node_t *host = gIter->data;
 -        if (!pcmk__str_in_list(only_node, host->details->uname)) {
 +        if (!pcmk__str_in_list(only_node, host->details->uname, pcmk__str_none)) {
             continue;
         }
@@ -884,7 +884,7 @@ pe__clone_html(pcmk__output_t *out, va_list args)
                 pe_node_t *node = (pe_node_t *)nIter->data;
                 if (pe_find_node(rsc->running_on, node->details->uname) == NULL &&
 -                    pcmk__str_in_list(only_node, node->details->uname)) {
 +                    pcmk__str_in_list(only_node, node->details->uname, pcmk__str_none)) {
                     pcmk__add_word(&stopped_list, &stopped_list_len,
                                    node->details->uname);
                 }
@@ -933,8 +933,8 @@ pe__clone_text(pcmk__output_t *out, va_list args)
         return rc;
     }
 -    print_everything = pcmk__str_in_list(only_rsc, rsc_printable_id(rsc)) ||
 -                       (strstr(rsc->id, ":") != NULL && pcmk__str_in_list(only_rsc, rsc->id));
 +    print_everything = pcmk__str_in_list(only_rsc, rsc_printable_id(rsc), pcmk__str_none) ||
 +                       (strstr(rsc->id, ":") != NULL && pcmk__str_in_list(only_rsc, rsc->id, pcmk__str_none));
     out->begin_list(out, NULL, NULL, "Clone Set: %s [%s]%s%s%s%s",
                     rsc->id, ID(clone_data->xml_obj_child),
@@ -1041,7 +1041,7 @@ pe__clone_text(pcmk__output_t *out, va_list args)
     for (gIter = promoted_list; gIter; gIter = gIter->next) {
         pe_node_t *host = gIter->data;
 -        if (!pcmk__str_in_list(only_node, host->details->uname)) {
 +        if (!pcmk__str_in_list(only_node, host->details->uname, pcmk__str_none)) {
             continue;
         }
@@ -1062,7 +1062,7 @@ pe__clone_text(pcmk__output_t *out, va_list args)
     for (gIter = started_list; gIter; gIter = gIter->next) {
         pe_node_t *host = gIter->data;
 -        if (!pcmk__str_in_list(only_node, host->details->uname)) {
 +        if (!pcmk__str_in_list(only_node, host->details->uname, pcmk__str_none)) {
             continue;
         }
@@ -1120,7 +1120,7 @@ pe__clone_text(pcmk__output_t *out, va_list args)
                 pe_node_t *node = (pe_node_t *)nIter->data;
                 if (pe_find_node(rsc->running_on, node->details->uname) == NULL &&
 -                    pcmk__str_in_list(only_node, node->details->uname)) {
 +                    pcmk__str_in_list(only_node, node->details->uname, pcmk__str_none)) {
                     pcmk__add_word(&stopped_list, &stopped_list_len,
                                    node->details->uname);
                 }
@@ -1220,11 +1220,11 @@ pe__clone_is_filtered(pe_resource_t *rsc, GList *only_rsc, gboolean check_parent
     gboolean passes = FALSE;
     clone_variant_data_t *clone_data = NULL;
 -    if (pcmk__str_in_list(only_rsc, rsc_printable_id(rsc))) {
 +    if (pcmk__str_in_list(only_rsc, rsc_printable_id(rsc), pcmk__str_none)) {
         passes = TRUE;
     } else {
         get_clone_variant_data(clone_data, rsc);
 -        passes = pcmk__str_in_list(only_rsc, ID(clone_data->xml_obj_child));
 +        passes = pcmk__str_in_list(only_rsc, ID(clone_data->xml_obj_child), pcmk__str_none);
         if (!passes) {
             for (GList *gIter = rsc->children; gIter != NULL; gIter = gIter->next) {
 diff --git a/lib/pengine/group.c b/lib/pengine/group.c
 index 23a72cff7..5f9aa83ce 100644
 --- a/lib/pengine/group.c
 +++ b/lib/pengine/group.c
@@ -201,8 +201,8 @@ pe__group_xml(pcmk__output_t *out, va_list args)
         return rc;
     }
 -    print_everything = pcmk__str_in_list(only_rsc, rsc_printable_id(rsc)) ||
 -                       (strstr(rsc->id, ":") != NULL && pcmk__str_in_list(only_rsc, rsc->id));
 +    print_everything = pcmk__str_in_list(only_rsc, rsc_printable_id(rsc), pcmk__str_none) ||
 +                       (strstr(rsc->id, ":") != NULL && pcmk__str_in_list(only_rsc, rsc->id, pcmk__str_none));
     for (; gIter != NULL; gIter = gIter->next) {
         pe_resource_t *child_rsc = (pe_resource_t *) gIter->data;
@@ -248,8 +248,8 @@ pe__group_html(pcmk__output_t *out, va_list args)
         return rc;
     }
 -    print_everything = pcmk__str_in_list(only_rsc, rsc_printable_id(rsc)) ||
 -                       (strstr(rsc->id, ":") != NULL && pcmk__str_in_list(only_rsc, rsc->id));
 +    print_everything = pcmk__str_in_list(only_rsc, rsc_printable_id(rsc), pcmk__str_none) ||
 +                       (strstr(rsc->id, ":") != NULL && pcmk__str_in_list(only_rsc, rsc->id, pcmk__str_none));
     if (options & pe_print_brief) {
         GList *rscs = pe__filter_rsc_list(rsc->children, only_rsc);
@@ -303,8 +303,8 @@ pe__group_text(pcmk__output_t *out, va_list args)
         return rc;
     }
 -    print_everything = pcmk__str_in_list(only_rsc, rsc_printable_id(rsc)) ||
 -                       (strstr(rsc->id, ":") != NULL && pcmk__str_in_list(only_rsc, rsc->id));
 +    print_everything = pcmk__str_in_list(only_rsc, rsc_printable_id(rsc), pcmk__str_none) ||
 +                       (strstr(rsc->id, ":") != NULL && pcmk__str_in_list(only_rsc, rsc->id, pcmk__str_none));
     if (options & pe_print_brief) {
         GList *rscs = pe__filter_rsc_list(rsc->children, only_rsc);
@@ -387,11 +387,11 @@ pe__group_is_filtered(pe_resource_t *rsc, GList *only_rsc, gboolean check_parent
 {
     gboolean passes = FALSE;
 -    if (check_parent && pcmk__str_in_list(only_rsc, rsc_printable_id(uber_parent(rsc)))) {
 +    if (check_parent && pcmk__str_in_list(only_rsc, rsc_printable_id(uber_parent(rsc)), pcmk__str_none)) {
         passes = TRUE;
 -    } else if (pcmk__str_in_list(only_rsc, rsc_printable_id(rsc))) {
 +    } else if (pcmk__str_in_list(only_rsc, rsc_printable_id(rsc), pcmk__str_none)) {
         passes = TRUE;
 -    } else if (strstr(rsc->id, ":") != NULL && pcmk__str_in_list(only_rsc, rsc->id)) {
 +    } else if (strstr(rsc->id, ":") != NULL && pcmk__str_in_list(only_rsc, rsc->id, pcmk__str_none)) {
         passes = TRUE;
     } else {
         for (GList *gIter = rsc->children; gIter != NULL; gIter = gIter->next) {
 diff --git a/lib/pengine/native.c b/lib/pengine/native.c
 index c2333d0d2..56054fc4a 100644
 --- a/lib/pengine/native.c
 +++ b/lib/pengine/native.c
@@ -1338,8 +1338,8 @@ pe__rscs_brief_output(pcmk__output_t *out, GList *rsc_list, unsigned int show_op
 gboolean
 pe__native_is_filtered(pe_resource_t *rsc, GList *only_rsc, gboolean check_parent)
 {
 -    if (pcmk__str_in_list(only_rsc, rsc_printable_id(rsc)) ||
 -        pcmk__str_in_list(only_rsc, rsc->id)) {
 +    if (pcmk__str_in_list(only_rsc, rsc_printable_id(rsc), pcmk__str_none) ||
 +        pcmk__str_in_list(only_rsc, rsc->id, pcmk__str_none)) {
         return FALSE;
     } else if (check_parent) {
         pe_resource_t *up = uber_parent(rsc);
 diff --git a/lib/pengine/pe_output.c b/lib/pengine/pe_output.c
 index 727475735..a6dc4ade8 100644
 --- a/lib/pengine/pe_output.c
 +++ b/lib/pengine/pe_output.c
@@ -670,8 +670,8 @@ ban_list(pcmk__output_t *out, va_list args) {
             continue;
         }
 -        if (!pcmk__str_in_list(only_rsc, rsc_printable_id(location->rsc_lh)) &&
 -            !pcmk__str_in_list(only_rsc, rsc_printable_id(uber_parent(location->rsc_lh)))) {
 +        if (!pcmk__str_in_list(only_rsc, rsc_printable_id(location->rsc_lh), pcmk__str_none) &&
 +            !pcmk__str_in_list(only_rsc, rsc_printable_id(uber_parent(location->rsc_lh)), pcmk__str_none)) {
             continue;
         }
@@ -1254,7 +1254,7 @@ failed_action_list(pcmk__output_t *out, va_list args) {
          xml_op = pcmk__xml_next(xml_op)) {
         char *rsc = NULL;
 -        if (!pcmk__str_in_list(only_node, crm_element_value(xml_op, XML_ATTR_UNAME))) {
 +        if (!pcmk__str_in_list(only_node, crm_element_value(xml_op, XML_ATTR_UNAME), pcmk__str_none)) {
             continue;
         }
@@ -1263,7 +1263,7 @@ failed_action_list(pcmk__output_t *out, va_list args) {
             continue;
         }
 -        if (!pcmk__str_in_list(only_rsc, rsc)) {
 +        if (!pcmk__str_in_list(only_rsc, rsc, pcmk__str_none)) {
             free(rsc);
             continue;
         }
@@ -1738,7 +1738,7 @@ node_attribute_list(pcmk__output_t *out, va_list args) {
             continue;
         }
 -        if (!pcmk__str_in_list(only_node, node->details->uname)) {
 +        if (!pcmk__str_in_list(only_node, node->details->uname, pcmk__str_none)) {
             g_list_free(attr_list);
             continue;
         }
@@ -1835,8 +1835,8 @@ node_history_list(pcmk__output_t *out, va_list args) {
          * For other resource types, is_filtered is okay.
          */
         if (uber_parent(rsc)->variant == pe_group) {
 -            if (!pcmk__str_in_list(only_rsc, rsc_printable_id(rsc)) &&
 -                !pcmk__str_in_list(only_rsc, rsc_printable_id(uber_parent(rsc)))) {
 +            if (!pcmk__str_in_list(only_rsc, rsc_printable_id(rsc), pcmk__str_none) &&
 +                !pcmk__str_in_list(only_rsc, rsc_printable_id(uber_parent(rsc)), pcmk__str_none)) {
                 continue;
             }
         } else {
@@ -1899,7 +1899,7 @@ node_list_html(pcmk__output_t *out, va_list args) {
     for (GList *gIter = nodes; gIter != NULL; gIter = gIter->next) {
         pe_node_t *node = (pe_node_t *) gIter->data;
 -        if (!pcmk__str_in_list(only_node, node->details->uname)) {
 +        if (!pcmk__str_in_list(only_node, node->details->uname, pcmk__str_none)) {
             continue;
         }
@@ -1940,7 +1940,7 @@ pe__node_list_text(pcmk__output_t *out, va_list args) {
         const char *node_mode = NULL;
         char *node_name = pe__node_display_name(node, print_clone_detail);
 -        if (!pcmk__str_in_list(only_node, node->details->uname)) {
 +        if (!pcmk__str_in_list(only_node, node->details->uname, pcmk__str_none)) {
             free(node_name);
             continue;
         }
@@ -2059,7 +2059,7 @@ node_list_xml(pcmk__output_t *out, va_list args) {
     for (GList *gIter = nodes; gIter != NULL; gIter = gIter->next) {
         pe_node_t *node = (pe_node_t *) gIter->data;
 -        if (!pcmk__str_in_list(only_node, node->details->uname)) {
 +        if (!pcmk__str_in_list(only_node, node->details->uname, pcmk__str_none)) {
             continue;
         }
@@ -2097,7 +2097,7 @@ node_summary(pcmk__output_t *out, va_list args) {
             continue;
         }
 -        if (!pcmk__str_in_list(only_node, node->details->uname)) {
 +        if (!pcmk__str_in_list(only_node, node->details->uname, pcmk__str_none)) {
             continue;
         }
 diff --git a/lib/pengine/utils.c b/lib/pengine/utils.c
 index 450d8348c..d1be9e4ca 100644
 --- a/lib/pengine/utils.c
 +++ b/lib/pengine/utils.c
@@ -2394,7 +2394,7 @@ pe__rsc_running_on_any_node_in_list(pe_resource_t *rsc, GList *node_list)
 {
     for (GList *ele = rsc->running_on; ele; ele = ele->next) {
         pe_node_t *node = (pe_node_t *) ele->data;
 -        if (pcmk__str_in_list(node_list, node->details->uname)) {
 +        if (pcmk__str_in_list(node_list, node->details->uname, pcmk__str_none)) {
             return true;
         }
     }
@@ -2419,8 +2419,8 @@ pe__filter_rsc_list(GList *rscs, GList *filter)
         /* I think the second condition is safe here for all callers of this
          * function.  If not, it needs to move into pe__node_text.
          */
 -        if (pcmk__str_in_list(filter, rsc_printable_id(rsc)) ||
 -            (rsc->parent && pcmk__str_in_list(filter, rsc_printable_id(rsc->parent)))) {
 +        if (pcmk__str_in_list(filter, rsc_printable_id(rsc), pcmk__str_none) ||
 +            (rsc->parent && pcmk__str_in_list(filter, rsc_printable_id(rsc->parent), pcmk__str_none))) {
             retval = g_list_prepend(retval, rsc);
         }
     }
 -- 
 2.27.0
--- a/SOURCES/015-sbd.patch
+++ b/SOURCES/015-sbd.patch
--- a/SOURCES/016-cts.patch
+++ b/SOURCES/016-cts.patch
@ -1,59 +0,0 @@
 From b37391fef92548f31822f9df2a9b5fa2a61b4514 Mon Sep 17 00:00:00 2001
 From: Ken Gaillot <kgaillot@redhat.com>
 Date: Wed, 23 Jun 2021 15:17:54 -0500
 Subject: [PATCH] Fix: CTS: handle longer Corosync token timeouts
 Previously, startall() would call cluster_stable() immediately after detecting
 the "controller successfully started" message. If the Corosync token timeout is
 small enough, this will be fine. However with a token timeout of more than
 about 1 second, the controllers will not have formed a membership by this
 point, causing cluster_stable() to think there are multiple partitions, and
 wait for a DC to be elected in each one, when really they will unite into a
 single partition in a short time, and only elect a single DC.
 Now, startall() waits until seeing that each node is a cluster member before
 calling cluster_stable().
 ---
 cts/lab/CTS.py.in   | 3 ++-
 cts/lab/patterns.py | 2 ++
 2 files changed, 4 insertions(+), 1 deletion(-)
 diff --git a/cts/lab/CTS.py.in b/cts/lab/CTS.py.in
 index abcb9d285..d9924437b 100644
 --- a/cts/lab/CTS.py.in
 +++ b/cts/lab/CTS.py.in
@@ -628,9 +628,10 @@ class ClusterManager(UserDict):
         watchpats = [ ]
         watchpats.append(self.templates["Pat:DC_IDLE"])
         for node in nodelist:
 -            watchpats.append(self.templates["Pat:Local_started"] % node)
             watchpats.append(self.templates["Pat:InfraUp"] % node)
             watchpats.append(self.templates["Pat:PacemakerUp"] % node)
 +            watchpats.append(self.templates["Pat:Local_started"] % node)
 +            watchpats.append(self.templates["Pat:They_up"] % (nodelist[0], node))
         #   Start all the nodes - at about the same time...
         watch = LogWatcher(self.Env["LogFileName"], watchpats, "fast-start", self.Env["DeadTime"]+10, hosts=self.Env["nodes"], kind=self.Env["LogWatcher"])
 diff --git a/cts/lab/patterns.py b/cts/lab/patterns.py
 index e21a016ff..400fd3dc8 100644
 --- a/cts/lab/patterns.py
 +++ b/cts/lab/patterns.py
@@ -61,6 +61,7 @@ class BasePatterns(object):
             "Pat:We_stopped"    : "%s\W.*OVERRIDE THIS PATTERN",
             "Pat:They_stopped"  : "%s\W.*LOST:.* %s ",
             "Pat:They_dead"     : "node %s.*: is dead",
 +            "Pat:They_up"       : "%s %s\W.*OVERRIDE THIS PATTERN",
             "Pat:TransitionComplete" : "Transition status: Complete: complete",
             "Pat:Fencing_start"   : r"Requesting peer fencing .* targeting %s",
@@ -130,6 +131,7 @@ class crm_corosync(BasePatterns):
             "Pat:We_stopped"   : "%s\W.*Unloading all Corosync service engines",
             "Pat:They_stopped" : "%s\W.*pacemaker-controld.*Node %s(\[|\s).*state is now lost",
             "Pat:They_dead"    : "pacemaker-controld.*Node %s(\[|\s).*state is now lost",
 +            "Pat:They_up"      : "\W%s\W.*pacemaker-controld.*Node %s state is now member",
             "Pat:ChildExit"    : r"\[[0-9]+\] exited with status [0-9]+ \(",
             # "with signal 9" == pcmk_child_exit(), "$" == check_active_before_startup_processes()
 -- 
 2.27.0
--- a/SOURCES/017-watchdog-fixes.patch
+++ b/SOURCES/017-watchdog-fixes.patch
@ -1,58 +0,0 @@
 From 61eb9c240004d1dbd0b5973e2fecda3686bb4c53 Mon Sep 17 00:00:00 2001
 From: Klaus Wenninger <klaus.wenninger@aon.at>
 Date: Tue, 10 Aug 2021 09:06:55 +0200
 Subject: [PATCH 1/2] Build: rpm: package fence_watchdog in base-package
 ---
 rpm/pacemaker.spec.in | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
 diff --git a/rpm/pacemaker.spec.in b/rpm/pacemaker.spec.in
 index f58357a77..0c569b9ca 100644
 --- a/rpm/pacemaker.spec.in
 +++ b/rpm/pacemaker.spec.in
@@ -734,6 +734,7 @@ exit 0
 %{_sbindir}/crm_attribute
 %{_sbindir}/crm_master
 %{_sbindir}/fence_legacy
 +%{_sbindir}/fence_watchdog
 %doc %{_mandir}/man7/pacemaker-controld.*
 %doc %{_mandir}/man7/pacemaker-schedulerd.*
@@ -797,7 +798,6 @@ exit 0
 %{_sbindir}/crm_simulate
 %{_sbindir}/crm_report
 %{_sbindir}/crm_ticket
 -%{_sbindir}/fence_watchdog
 %{_sbindir}/stonith_admin
 # "dirname" is owned by -schemas, which is a prerequisite
 %{_datadir}/pacemaker/report.collector
 -- 
 2.27.0
 From 88e75d5b98df197fa731e7642434951a24a67095 Mon Sep 17 00:00:00 2001
 From: Klaus Wenninger <klaus.wenninger@aon.at>
 Date: Tue, 10 Aug 2021 09:10:23 +0200
 Subject: [PATCH 2/2] Fix: fence_watchdog: fix version output needed for
 help2man
 ---
 daemons/fenced/fence_watchdog.in | 1 +
 1 file changed, 1 insertion(+)
 diff --git a/daemons/fenced/fence_watchdog.in b/daemons/fenced/fence_watchdog.in
 index c83304f1d..700065e0e 100755
 --- a/daemons/fenced/fence_watchdog.in
 +++ b/daemons/fenced/fence_watchdog.in
@@ -12,6 +12,7 @@ import sys
 import atexit
 import getopt
 +AGENT_VERSION = "1.0.0"
 SHORT_DESC = "Dummy watchdog fence agent"
 LONG_DESC = """fence_watchdog just provides
 meta-data - actual fencing is done by the pacemaker internal watchdog agent."""
 -- 
 2.27.0
--- a/SOURCES/018-controller.patch
+++ b/SOURCES/018-controller.patch
@ -1,122 +0,0 @@
 From ee7eba6a7a05bdf0a12d60ebabb334d8ee021101 Mon Sep 17 00:00:00 2001
 From: Ken Gaillot <kgaillot@redhat.com>
 Date: Mon, 9 Aug 2021 14:48:57 -0500
 Subject: [PATCH] Fix: controller: ensure lost node's transient attributes are
 cleared without DC
 Previously, peer_update_callback() cleared a lost node's transient attributes
 if either the local node is DC, or there is no DC.
 However, that left the possibility of the DC being lost at the same time as
 another node -- the local node would still have fsa_our_dc set while processing
 the leave notifications, so no node would clear the attributes for the non-DC
 node.
 Now, the controller has its own CPG configuration change callback, which sets a
 global boolean before calling the usual one, so that peer_update_callback() can
 know when the DC has been lost.
 ---
 daemons/controld/controld_callbacks.c |  4 +-
 daemons/controld/controld_corosync.c  | 57 ++++++++++++++++++++++++++-
 2 files changed, 59 insertions(+), 2 deletions(-)
 diff --git a/daemons/controld/controld_callbacks.c b/daemons/controld/controld_callbacks.c
 index af24856ae..e564b3dcd 100644
 --- a/daemons/controld/controld_callbacks.c
 +++ b/daemons/controld/controld_callbacks.c
@@ -99,6 +99,8 @@ node_alive(const crm_node_t *node)
 #define state_text(state) ((state)? (const char *)(state) : "in unknown state")
 +bool controld_dc_left = false;
 +
 void
 peer_update_callback(enum crm_status_type type, crm_node_t * node, const void *data)
 {
@@ -217,7 +219,7 @@ peer_update_callback(enum crm_status_type type, crm_node_t * node, const void *d
                                                cib_scope_local);
                 }
 -            } else if (AM_I_DC || (fsa_our_dc == NULL)) {
 +            } else if (AM_I_DC || controld_dc_left || (fsa_our_dc == NULL)) {
                 /* This only needs to be done once, so normally the DC should do
                  * it. However if there is no DC, every node must do it, since
                  * there is no other way to ensure some one node does it.
 diff --git a/daemons/controld/controld_corosync.c b/daemons/controld/controld_corosync.c
 index db99630fb..c5ab6580a 100644
 --- a/daemons/controld/controld_corosync.c
 +++ b/daemons/controld/controld_corosync.c
@@ -87,6 +87,61 @@ crmd_cs_destroy(gpointer user_data)
     }
 }
 +extern bool controld_dc_left;
 +
 +/*!
 + * \brief Handle a Corosync notification of a CPG configuration change
 + *
 + * \param[in] handle               CPG connection
 + * \param[in] cpg_name             CPG group name
 + * \param[in] member_list          List of current CPG members
 + * \param[in] member_list_entries  Number of entries in \p member_list
 + * \param[in] left_list            List of CPG members that left
 + * \param[in] left_list_entries    Number of entries in \p left_list
 + * \param[in] joined_list          List of CPG members that joined
 + * \param[in] joined_list_entries  Number of entries in \p joined_list
 + */
 +static void
 +cpg_membership_callback(cpg_handle_t handle, const struct cpg_name *cpg_name,
 +                        const struct cpg_address *member_list,
 +                        size_t member_list_entries,
 +                        const struct cpg_address *left_list,
 +                        size_t left_list_entries,
 +                        const struct cpg_address *joined_list,
 +                        size_t joined_list_entries)
 +{
 +    /* When nodes leave CPG, the DC clears their transient node attributes.
 +     *
 +     * However if there is no DC, or the DC is among the nodes that left, each
 +     * remaining node needs to do the clearing, to ensure it gets done.
 +     * Otherwise, the attributes would persist when the nodes rejoin, which
 +     * could have serious consequences for unfencing, agents that use attributes
 +     * for internal logic, etc.
 +     *
 +     * Here, we set a global boolean if the DC is among the nodes that left, for
 +     * use by the peer callback.
 +     */
 +    if (fsa_our_dc != NULL) {
 +        crm_node_t *peer = pcmk__search_cluster_node_cache(0, fsa_our_dc);
 +
 +        if (peer != NULL) {
 +            for (int i = 0; i < left_list_entries; ++i) {
 +                if (left_list[i].nodeid == peer->id) {
 +                    controld_dc_left = true;
 +                    break;
 +                }
 +            }
 +        }
 +    }
 +
 +    // Process the change normally, which will call the peer callback as needed
 +    pcmk_cpg_membership(handle, cpg_name, member_list, member_list_entries,
 +                        left_list, left_list_entries,
 +                        joined_list, joined_list_entries);
 +
 +    controld_dc_left = false;
 +}
 +
 extern gboolean crm_connect_corosync(crm_cluster_t * cluster);
 gboolean
@@ -95,7 +150,7 @@ crm_connect_corosync(crm_cluster_t * cluster)
     if (is_corosync_cluster()) {
         crm_set_status_callback(&peer_update_callback);
         cluster->cpg.cpg_deliver_fn = crmd_cs_dispatch;
 -        cluster->cpg.cpg_confchg_fn = pcmk_cpg_membership;
 +        cluster->cpg.cpg_confchg_fn = cpg_membership_callback;
         cluster->destroy = crmd_cs_destroy;
         if (crm_cluster_connect(cluster)) {
 -- 
 2.27.0
--- a/SOURCES/019-crm_resource.patch
+++ b/SOURCES/019-crm_resource.patch
@ -1,114 +0,0 @@
 From b4e426a016a4d7c9ade39e60a83644fc537bce26 Mon Sep 17 00:00:00 2001
 From: Oyvind Albrigtsen <oalbrigt@redhat.com>
 Date: Wed, 11 Aug 2021 12:10:32 +0200
 Subject: [PATCH 1/2] Fix: crm_resource: translate LSB rc to exit code and fix
 resources_find_service_class() call
 ---
 tools/crm_resource_runtime.c | 16 ++++++++++++----
 1 file changed, 12 insertions(+), 4 deletions(-)
 diff --git a/tools/crm_resource_runtime.c b/tools/crm_resource_runtime.c
 index ce037c514..e9d8aa687 100644
 --- a/tools/crm_resource_runtime.c
 +++ b/tools/crm_resource_runtime.c
@@ -1718,10 +1718,10 @@ cli_resource_execute_from_params(pcmk__output_t *out, const char *rsc_name,
         crm_exit(CRM_EX_UNIMPLEMENT_FEATURE);
     } else if (pcmk__str_eq(rsc_class, PCMK_RESOURCE_CLASS_SERVICE,
                 pcmk__str_casei) && !pcmk__str_eq(
 -                resources_find_service_class(rsc_name), PCMK_RESOURCE_CLASS_LSB,
 +                resources_find_service_class(rsc_type), PCMK_RESOURCE_CLASS_LSB,
                 pcmk__str_casei)) {
         out->err(out, "Sorry, the %s option doesn't support %s resources",
 -                 rsc_action, resources_find_service_class(rsc_name));
 +                 rsc_action, resources_find_service_class(rsc_type));
         crm_exit(CRM_EX_UNIMPLEMENT_FEATURE);
     }
@@ -1798,9 +1798,17 @@ cli_resource_execute_from_params(pcmk__output_t *out, const char *rsc_name,
     if (services_action_sync(op)) {
         exit_code = op->rc;
 +        /* Lookup exit code based on rc for LSB resources */
 +        if (( pcmk__str_eq(rsc_class, PCMK_RESOURCE_CLASS_LSB, pcmk__str_casei) ||
 +              (pcmk__str_eq(rsc_class, PCMK_RESOURCE_CLASS_SERVICE, pcmk__str_casei) &&
 +              pcmk__str_eq(resources_find_service_class(rsc_type), PCMK_RESOURCE_CLASS_LSB, pcmk__str_casei)) ) &&
 +              pcmk__str_eq(rsc_action, "force-check", pcmk__str_casei)) {
 +            exit_code = services_get_ocf_exitcode(action, exit_code);
 +        }
 +
         out->message(out, "resource-agent-action", resource_verbose, rsc_class,
 -                     rsc_prov, rsc_type, rsc_name, rsc_action, override_hash, op->rc,
 -                     op->status, op->stdout_data, op->stderr_data);
 +                     rsc_prov, rsc_type, rsc_name, rsc_action, override_hash,
 +                     exit_code, op->status, op->stdout_data, op->stderr_data);
     } else {
         exit_code = op->rc == 0 ? CRM_EX_ERROR : op->rc;
     }
 -- 
 2.27.0
 From 9a6beb74adfb4710fb3a4e588bef79a562c101f3 Mon Sep 17 00:00:00 2001
 From: Oyvind Albrigtsen <oalbrigt@redhat.com>
 Date: Thu, 12 Aug 2021 18:54:30 +0200
 Subject: [PATCH 2/2] Refactor: crm_resource: simplify rsc_class logic by
 getting actual class early if it's of class "service"
 ---
 tools/crm_resource_runtime.c | 23 +++++++++--------------
 1 file changed, 9 insertions(+), 14 deletions(-)
 diff --git a/tools/crm_resource_runtime.c b/tools/crm_resource_runtime.c
 index e9d8aa687..13b78b6b9 100644
 --- a/tools/crm_resource_runtime.c
 +++ b/tools/crm_resource_runtime.c
@@ -1702,26 +1702,23 @@ cli_resource_execute_from_params(pcmk__output_t *out, const char *rsc_name,
                                  int timeout_ms, int resource_verbose, gboolean force,
                                  int check_level)
 {
 +    const char *class = NULL;
     const char *action = NULL;
     GHashTable *params_copy = NULL;
     crm_exit_t exit_code = CRM_EX_OK;
     svc_action_t *op = NULL;
 -    if (pcmk__str_eq(rsc_class, PCMK_RESOURCE_CLASS_STONITH, pcmk__str_casei)) {
 +    class = !pcmk__str_eq(rsc_class, PCMK_RESOURCE_CLASS_SERVICE, pcmk__str_casei) ?
 +                rsc_class : resources_find_service_class(rsc_type);
 +
 +    if (pcmk__str_eq(class, PCMK_RESOURCE_CLASS_STONITH, pcmk__str_casei)) {
         out->err(out, "Sorry, the %s option doesn't support %s resources yet",
 -                 rsc_action, rsc_class);
 +                 rsc_action, class);
         crm_exit(CRM_EX_UNIMPLEMENT_FEATURE);
 -    } else if (pcmk__strcase_any_of(rsc_class, PCMK_RESOURCE_CLASS_SYSTEMD,
 +    } else if (pcmk__strcase_any_of(class, PCMK_RESOURCE_CLASS_SYSTEMD,
                 PCMK_RESOURCE_CLASS_UPSTART, PCMK_RESOURCE_CLASS_NAGIOS, NULL)) {
         out->err(out, "Sorry, the %s option doesn't support %s resources",
 -                 rsc_action, rsc_class);
 -        crm_exit(CRM_EX_UNIMPLEMENT_FEATURE);
 -    } else if (pcmk__str_eq(rsc_class, PCMK_RESOURCE_CLASS_SERVICE,
 -                pcmk__str_casei) && !pcmk__str_eq(
 -                resources_find_service_class(rsc_type), PCMK_RESOURCE_CLASS_LSB,
 -                pcmk__str_casei)) {
 -        out->err(out, "Sorry, the %s option doesn't support %s resources",
 -                 rsc_action, resources_find_service_class(rsc_type));
 +                 rsc_action, class);
         crm_exit(CRM_EX_UNIMPLEMENT_FEATURE);
     }
@@ -1799,9 +1796,7 @@ cli_resource_execute_from_params(pcmk__output_t *out, const char *rsc_name,
         exit_code = op->rc;
         /* Lookup exit code based on rc for LSB resources */
 -        if (( pcmk__str_eq(rsc_class, PCMK_RESOURCE_CLASS_LSB, pcmk__str_casei) ||
 -              (pcmk__str_eq(rsc_class, PCMK_RESOURCE_CLASS_SERVICE, pcmk__str_casei) &&
 -              pcmk__str_eq(resources_find_service_class(rsc_type), PCMK_RESOURCE_CLASS_LSB, pcmk__str_casei)) ) &&
 +        if (pcmk__str_eq(class, PCMK_RESOURCE_CLASS_LSB, pcmk__str_casei) &&
               pcmk__str_eq(rsc_action, "force-check", pcmk__str_casei)) {
             exit_code = services_get_ocf_exitcode(action, exit_code);
         }
 -- 
 2.27.0
--- a/SOURCES/020-fence_watchdog.patch
+++ b/SOURCES/020-fence_watchdog.patch
@ -1,25 +0,0 @@
 From 46dd1118cae948649e000b2159e8e92623520ad9 Mon Sep 17 00:00:00 2001
 From: Klaus Wenninger <klaus.wenninger@aon.at>
 Date: Thu, 19 Aug 2021 09:28:54 +0200
 Subject: [PATCH] Fix: fence_watchdog: fix malformed xml in metadata
 ---
 daemons/fenced/fence_watchdog.in | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
 diff --git a/daemons/fenced/fence_watchdog.in b/daemons/fenced/fence_watchdog.in
 index 700065e0e..eefa7395e 100755
 --- a/daemons/fenced/fence_watchdog.in
 +++ b/daemons/fenced/fence_watchdog.in
@@ -124,7 +124,7 @@ def metadata(avail_opt, options):
     for option, dummy in sorted_options(avail_opt):
         if "shortdesc" in ALL_OPT[option]:
             print('    <parameter name="' + option +
 -                  'required="' + ALL_OPT[option]["required"] + '">')
 +                  '" required="' + ALL_OPT[option]["required"] + '">')
             default = ""
             default_name_arg = "-" + ALL_OPT[option]["getopt"][:-1]
 -- 
 2.27.0
--- a/SPECS/pacemaker.spec
+++ b/SPECS/pacemaker.spec
@ -35,11 +35,11 @@
 ## Upstream pacemaker version, and its package version (specversion
 ## can be incremented to build packages reliably considered "newer"
 ## than previously built packages with the same pcmkversion)
-%global pcmkversion 2.1.0
+%global pcmkversion 2.1.2
-%global specversion 11
+%global specversion 2
 ## Upstream commit (full commit ID, abbreviated commit ID, or tag) to build
-%global commit 7c3f660707a495a1331716ad32cd3ac9d9f8ff58
+%global commit ada5c3b36e2adf1703d54d39f40a4b8628eca175
 ## Since git v2.11, the extent of abbreviation is autoscaled by default
 ## (used to be constant of 7), so we need to convey it for non-tags, too.
@ -242,26 +242,14 @@ Source0:       https://codeload.github.com/%{github_owner}/%{name}/tar.gz/%{arch
 Source1:       https://codeload.github.com/%{github_owner}/%{nagios_name}/tar.gz/%{nagios_archive_github_url}
 # upstream commits
-Patch1:        001-ping-agent.patch
+Patch1:        001-acl-group-schema.patch
-Patch2:        002-pacemakerd-options.patch
+Patch2:        002-fencing-reasons.patch
-Patch3:        003-pacemakerd-output.patch
+Patch3:        003-fencing-reasons.patch
-Patch4:        004-check-level.patch
+Patch4:        004-systemd-metadata.patch
-Patch5:        005-crm_resource.patch
+Patch5:        005-fencing-reasons.patch
-Patch6:        006-crm_simulate.patch
+Patch6:        006-stateful-metadata.patch
-Patch7:        007-unfencing-loop.patch
+Patch7:        007-memory-leak.patch
-Patch8:        008-dynamic-list-fencing.patch
+Patch8:        008-fencing-history.patch
 Patch9:        009-crm_resource-messages.patch
 Patch10:       010-probe-pending.patch
 Patch11:       011-crm_attribute-regression.patch
 Patch12:       012-string-arguments.patch
 Patch13:       013-leaks.patch
 Patch14:       014-str-list.patch
 Patch15:       015-sbd.patch
 Patch16:       016-cts.patch
 Patch17:       017-watchdog-fixes.patch
 Patch18:       018-controller.patch
 Patch19:       019-crm_resource.patch
 Patch20:       020-fence_watchdog.patch
 Requires:      resource-agents
 Requires:      %{pkgname_pcmk_libs}%{?_isa} = %{version}-%{release}
@ -313,13 +301,20 @@ BuildRequires: %{pkgname_gnutls_devel}
 BuildRequires: help2man
 BuildRequires: ncurses-devel
 BuildRequires: pam-devel
 # Required for "make check"
 BuildRequires: libcmocka-devel
 BuildRequires: pkgconfig(systemd)
 # RH patches are created by git, so we need git to apply them
 BuildRequires: git
-Requires:      corosync >= 2.0.0
+# The RHEL 9 build root has corosync_cfg_trackstart() available, so
-BuildRequires: corosync-devel >= 2.0.0
+# Pacemaker's configure script will build support for it. Add a hard dependency
 # to ensure users have compatible Corosync libraries if they upgrade Pacemaker.
 Requires:      corosync >= 3.1.1
 BuildRequires: corosync-devel >= 3.1.1
 %if %{with stonithd}
 BuildRequires: %{pkgname_glue_libs}-devel
@ -337,7 +332,7 @@ Provides:      pcmk-cluster-manager%{?_isa} = %{version}-%{release}
 # Bundled bits
 ## Pacemaker uses the crypto/md5-buffer module from gnulib
 %if 0%{?fedora} || 0%{?rhel}
-Provides:      bundled(gnulib)
+Provides:      bundled(gnulib) = 20200404
 %endif
 %description
@ -361,8 +356,8 @@ Requires:      %{pkgname_pcmk_libs}%{?_isa} = %{version}-%{release}
 %if 0%{?supports_recommends}
 Recommends:    pcmk-cluster-manager = %{version}-%{release}
 # For crm_report
-Recommends:    tar
+Requires:      tar
-Recommends:    bzip2
+Requires:      bzip2
 %endif
 Requires:      perl-TimeDate
 Requires:      %{pkgname_procps}
@ -590,10 +585,6 @@ rm -f %{buildroot}/%{_mandir}/man8/fence_legacy.*
 rm -f %{buildroot}/%{_sbindir}/notifyServicelogEvent
 rm -f %{buildroot}/%{_sbindir}/ipmiservicelogd
 # Don't ship init scripts for systemd based platforms
 rm -f %{buildroot}/%{_initrddir}/pacemaker
 rm -f %{buildroot}/%{_initrddir}/pacemaker_remote
 # Byte-compile Python sources where suitable and the distro procedures known
 %if %{defined py_byte_compile}
 %{py_byte_compile %{python_path} %{buildroot}%{_datadir}/pacemaker/tests}
@ -864,6 +855,18 @@ exit 0
 %license %{nagios_name}-%{nagios_hash}/COPYING
 %changelog
 * Thu Dec 16 2021 Ken Gaillot <kgaillot@redhat.com> - 2.1.2-2
 - Correctly get metadata for systemd agent names that end in '@'
 - Use correct OCF 1.1 syntax in ocf:pacemaker:Stateful meta-data
 - Fix regression in displayed times in crm_mon's fence history
 - Resolves: rhbz2032031
 - Resolves: rhbz2032032
 - Resolves: rhbz2031765
 * Tue Nov 30 2021 Ken Gaillot <kgaillot@redhat.com> - 2.1.2-1
 - Rebase on upstream 2.1.2
 - Resolves: rhbz2011974
 * Fri Aug 20 2021 Ken Gaillot <kgaillot@redhat.com> - 2.1.0-11
 - Fix XML issue with fence_watchdog meta-data
 - Resolves: rhbz1988568
`@ -1,2 +1,2 @@`
	`SOURCES/nagios-agents-metadata-105ab8a7b2c16b9a29cf1c1596b80136eeef332b.tar.gz`	`SOURCES/nagios-agents-metadata-105ab8a7b2c16b9a29cf1c1596b80136eeef332b.tar.gz`
	`SOURCES/pacemaker-7c3f66070.tar.gz`	`SOURCES/pacemaker-ada5c3b36.tar.gz`
`@ -1,2 +1,2 @@`
	`2cbec94ad67dfbeba75e38d2c3c5c44961b3cd16 SOURCES/nagios-agents-metadata-105ab8a7b2c16b9a29cf1c1596b80136eeef332b.tar.gz`	`2cbec94ad67dfbeba75e38d2c3c5c44961b3cd16 SOURCES/nagios-agents-metadata-105ab8a7b2c16b9a29cf1c1596b80136eeef332b.tar.gz`
	`55917d293ae0f9b79ac196fae061ba95647382e1 SOURCES/pacemaker-7c3f66070.tar.gz`	`1dec5b062ad8e9a89b4953e17a59e4597797a1e6 SOURCES/pacemaker-ada5c3b36.tar.gz`