rpms/pacemaker
5
0
Fork 0
Code Issues Pull Requests Releases Activity
dff8d9929d
pacemaker/pacemaker-rollup-7-1-3d781d3.patch
Jan Pokorný dff8d9929d
1.1.13-3: Update to Pacemaker-1.1.13 post-release + patches 
Signed-off-by: Jan Pokorný <jpokorny@redhat.com>
2015-10-14 01:54:55 +02:00

7990 lines
481 KiB
Diff
Raw Blame History

diff --git a/cib/io.c b/cib/io.c
index e2873a8..4e2b24a 100644
--- a/cib/io.c
+++ b/cib/io.c
@@ -254,9 +254,7 @@ readCibXmlFile(const char *dir, const char *file, gboolean discard_status)
if (cib_writes_enabled && use_valgrind) {
if (crm_is_true(use_valgrind) || strstr(use_valgrind, "cib")) {
cib_writes_enabled = FALSE;
- crm_err("*********************************************************");
crm_err("*** Disabling disk writes to avoid confusing Valgrind ***");
- crm_err("*********************************************************");
}
}
diff --git a/crmd/crmd_lrm.h b/crmd/crmd_lrm.h
index 81a53c5..78432df 100644
--- a/crmd/crmd_lrm.h
+++ b/crmd/crmd_lrm.h
@@ -37,6 +37,8 @@ typedef struct resource_history_s {
GHashTable *stop_params;
} rsc_history_t;
+void history_free(gpointer data);
+
/* TDOD - Replace this with lrmd_event_data_t */
struct recurring_op_s {
int call_id;
diff --git a/crmd/lrm.c b/crmd/lrm.c
index 062f769..418e7cf 100644
--- a/crmd/lrm.c
+++ b/crmd/lrm.c
@@ -103,6 +103,80 @@ copy_meta_keys(gpointer key, gpointer value, gpointer user_data)
}
}
+/*
+ * \internal
+ * \brief Remove a recurring operation from a resource's history
+ *
+ * \param[in,out] history Resource history to modify
+ * \param[in] op Operation to remove
+ *
+ * \return TRUE if the operation was found and removed, FALSE otherwise
+ */
+static gboolean
+history_remove_recurring_op(rsc_history_t *history, const lrmd_event_data_t *op)
+{
+ GList *iter;
+
+ for (iter = history->recurring_op_list; iter != NULL; iter = iter->next) {
+ lrmd_event_data_t *existing = iter->data;
+
+ if ((op->interval == existing->interval)
+ && crm_str_eq(op->rsc_id, existing->rsc_id, TRUE)
+ && safe_str_eq(op->op_type, existing->op_type)) {
+
+ history->recurring_op_list = g_list_delete_link(history->recurring_op_list, iter);
+ lrmd_free_event(existing);
+ return TRUE;
+ }
+ }
+ return FALSE;
+}
+
+/*
+ * \internal
+ * \brief Free all recurring operations in resource history
+ *
+ * \param[in,out] history Resource history to modify
+ */
+static void
+history_free_recurring_ops(rsc_history_t *history)
+{
+ GList *iter;
+
+ for (iter = history->recurring_op_list; iter != NULL; iter = iter->next) {
+ lrmd_free_event(iter->data);
+ }
+ g_list_free(history->recurring_op_list);
+ history->recurring_op_list = NULL;
+}
+
+/*
+ * \internal
+ * \brief Free resource history
+ *
+ * \param[in,out] history Resource history to free
+ */
+void
+history_free(gpointer data)
+{
+ rsc_history_t *history = (rsc_history_t*)data;
+
+ if (history->stop_params) {
+ g_hash_table_destroy(history->stop_params);
+ }
+
+ /* Don't need to free history->rsc.id because it's set to history->id */
+ free(history->rsc.type);
+ free(history->rsc.class);
+ free(history->rsc.provider);
+
+ lrmd_free_event(history->failed);
+ lrmd_free_event(history->last);
+ free(history->id);
+ history_free_recurring_ops(history);
+ free(history);
+}
+
static void
update_history_cache(lrm_state_t * lrm_state, lrmd_rsc_info_t * rsc, lrmd_event_data_t * op)
{
@@ -145,25 +219,10 @@ update_history_cache(lrm_state_t * lrm_state, lrmd_rsc_info_t * rsc, lrmd_event_
target_rc = rsc_op_expected_rc(op);
if (op->op_status == PCMK_LRM_OP_CANCELLED) {
if (op->interval > 0) {
- GList *gIter, *gIterNext;
-
crm_trace("Removing cancelled recurring op: %s_%s_%d", op->rsc_id, op->op_type,
op->interval);
-
- for (gIter = entry->recurring_op_list; gIter != NULL; gIter = gIterNext) {
- lrmd_event_data_t *existing = gIter->data;
-
- gIterNext = gIter->next;
-
- if (crm_str_eq(op->rsc_id, existing->rsc_id, TRUE)
- && safe_str_eq(op->op_type, existing->op_type)
- && op->interval == existing->interval) {
- lrmd_free_event(existing);
- entry->recurring_op_list = g_list_delete_link(entry->recurring_op_list, gIter);
- }
- }
+ history_remove_recurring_op(entry, op);
return;
-
} else {
crm_trace("Skipping %s_%s_%d rc=%d, status=%d", op->rsc_id, op->op_type, op->interval,
op->rc, op->op_status);
@@ -201,32 +260,17 @@ update_history_cache(lrm_state_t * lrm_state, lrmd_rsc_info_t * rsc, lrmd_event_
}
if (op->interval > 0) {
- GListPtr iter = NULL;
-
- for(iter = entry->recurring_op_list; iter; iter = iter->next) {
- lrmd_event_data_t *o = iter->data;
-
- /* op->rsc_id is implied */
- if(op->interval == o->interval && strcmp(op->op_type, o->op_type) == 0) {
- crm_trace("Removing existing recurring op entry: %s_%s_%d", op->rsc_id, op->op_type, op->interval);
- entry->recurring_op_list = g_list_remove(entry->recurring_op_list, o);
- break;
- }
- }
+ /* Ensure there are no duplicates */
+ history_remove_recurring_op(entry, op);
crm_trace("Adding recurring op: %s_%s_%d", op->rsc_id, op->op_type, op->interval);
entry->recurring_op_list = g_list_prepend(entry->recurring_op_list, lrmd_copy_event(op));
} else if (entry->recurring_op_list && safe_str_eq(op->op_type, RSC_STATUS) == FALSE) {
- GList *gIter = entry->recurring_op_list;
-
crm_trace("Dropping %d recurring ops because of: %s_%s_%d",
- g_list_length(gIter), op->rsc_id, op->op_type, op->interval);
- for (; gIter != NULL; gIter = gIter->next) {
- lrmd_free_event(gIter->data);
- }
- g_list_free(entry->recurring_op_list);
- entry->recurring_op_list = NULL;
+ g_list_length(entry->recurring_op_list), op->rsc_id,
+ op->op_type, op->interval);
+ history_free_recurring_ops(entry);
}
}
diff --git a/crmd/lrm_state.c b/crmd/lrm_state.c
index 374c806..162ad03 100644
--- a/crmd/lrm_state.c
+++ b/crmd/lrm_state.c
@@ -32,24 +32,6 @@ int lrmd_internal_proxy_send(lrmd_t * lrmd, xmlNode *msg);
void lrmd_internal_set_proxy_callback(lrmd_t * lrmd, void *userdata, void (*callback)(lrmd_t *lrmd, void *userdata, xmlNode *msg));
static void
-history_cache_destroy(gpointer data)
-{
- rsc_history_t *entry = data;
-
- if (entry->stop_params) {
- g_hash_table_destroy(entry->stop_params);
- }
-
- free(entry->rsc.type);
- free(entry->rsc.class);
- free(entry->rsc.provider);
-
- lrmd_free_event(entry->failed);
- lrmd_free_event(entry->last);
- free(entry->id);
- free(entry);
-}
-static void
free_rsc_info(gpointer value)
{
lrmd_rsc_info_t *rsc_info = value;
@@ -155,7 +137,7 @@ lrm_state_create(const char *node_name)
g_str_equal, g_hash_destroy_str, free_recurring_op);
state->resource_history = g_hash_table_new_full(crm_str_hash,
- g_str_equal, NULL, history_cache_destroy);
+ g_str_equal, NULL, history_free);
g_hash_table_insert(lrm_state_table, (char *)state->node_name, state);
return state;
diff --git a/cts/CM_ais.py b/cts/CM_ais.py
index 44f91cd..a34f9b1 100644
--- a/cts/CM_ais.py
+++ b/cts/CM_ais.py
@@ -49,42 +49,46 @@ class crm_ais(crm_lha):
def NodeUUID(self, node):
return node
- def ais_components(self):
+ def ais_components(self, extra={}):
complist = []
if not len(self.fullcomplist.keys()):
for c in ["cib", "lrmd", "crmd", "attrd" ]:
- self.fullcomplist[c] = Process(
- self, c,
- pats = self.templates.get_component(self.name, c),
- badnews_ignore = self.templates.get_component(self.name, "%s-ignore"%c),
- common_ignore = self.templates.get_component(self.name, "common-ignore"))
-
- self.fullcomplist["pengine"] = Process(
- self, "pengine",
- dc_pats = self.templates.get_component(self.name, "pengine"),
- badnews_ignore = self.templates.get_component(self.name, "pengine-ignore"),
- common_ignore = self.templates.get_component(self.name, "common-ignore"))
-
- self.fullcomplist["stonith-ng"] = Process(
- self, "stonith-ng", process="stonithd",
- pats = self.templates.get_component(self.name, "stonith"),
- badnews_ignore = self.templates.get_component(self.name, "stonith-ignore"),
- common_ignore = self.templates.get_component(self.name, "common-ignore"))
-
+ self.fullcomplist[c] = Process(
+ self, c,
+ pats = self.templates.get_component(self.name, c),
+ badnews_ignore = self.templates.get_component(self.name, "%s-ignore" % c),
+ common_ignore = self.templates.get_component(self.name, "common-ignore"))
+
+ # pengine uses dc_pats instead of pats
+ self.fullcomplist["pengine"] = Process(
+ self, "pengine",
+ dc_pats = self.templates.get_component(self.name, "pengine"),
+ badnews_ignore = self.templates.get_component(self.name, "pengine-ignore"),
+ common_ignore = self.templates.get_component(self.name, "common-ignore"))
+
+ # stonith-ng's process name is different from its component name
+ self.fullcomplist["stonith-ng"] = Process(
+ self, "stonith-ng", process="stonithd",
+ pats = self.templates.get_component(self.name, "stonith"),
+ badnews_ignore = self.templates.get_component(self.name, "stonith-ignore"),
+ common_ignore = self.templates.get_component(self.name, "common-ignore"))
+
+ # add (or replace) any extra components passed in
+ self.fullcomplist.update(extra)
+
+ # Processes running under valgrind can't be shot with "killall -9 processname",
+ # so don't include them in the returned list
vgrind = self.Env["valgrind-procs"].split()
for key in self.fullcomplist.keys():
if self.Env["valgrind-tests"]:
- if key in vgrind:
- # Processes running under valgrind can't be shot with "killall -9 processname"
+ if key in vgrind:
self.log("Filtering %s from the component list as it is being profiled by valgrind" % key)
continue
if key == "stonith-ng" and not self.Env["DoFencing"]:
continue
-
complist.append(self.fullcomplist[key])
- #self.complist = [ fullcomplist["pengine"] ]
return complist
@@ -100,17 +104,14 @@ class crm_cs_v0(crm_ais):
crm_ais.__init__(self, Environment, randseed=randseed, name=name)
def Components(self):
- self.ais_components()
- c = "corosync"
-
- self.fullcomplist[c] = Process(
- self, c,
- pats = self.templates.get_component(self.name, c),
- badnews_ignore = self.templates.get_component(self.name, "%s-ignore"%c),
+ extra = {}
+ extra["corosync"] = Process(
+ self, "corosync",
+ pats = self.templates.get_component(self.name, "corosync"),
+ badnews_ignore = self.templates.get_component(self.name, "corosync-ignore"),
common_ignore = self.templates.get_component(self.name, "common-ignore")
)
-
- return self.ais_components()
+ return self.ais_components(extra=extra)
class crm_cs_v1(crm_cs_v0):
diff --git a/cts/environment.py b/cts/environment.py
index a3399c3..61d4211 100644
--- a/cts/environment.py
+++ b/cts/environment.py
@@ -59,7 +59,7 @@ class Environment:
self["stonith-params"] = "hostlist=all,livedangerously=yes"
self["loop-minutes"] = 60
self["valgrind-prefix"] = None
- self["valgrind-procs"] = "cib crmd attrd pengine stonith-ng"
+ self["valgrind-procs"] = "attrd cib crmd lrmd pengine stonith-ng"
self["valgrind-opts"] = """--leak-check=full --show-reachable=yes --trace-children=no --num-callers=25 --gen-suppressions=all --suppressions="""+CTSvars.CTS_home+"""/cts.supp"""
self["experimental-tests"] = 0
@@ -578,6 +578,10 @@ class Environment:
elif args[i] == "--valgrind-tests":
self["valgrind-tests"] = 1
+ elif args[i] == "--valgrind-procs":
+ self["valgrind-procs"] = args[i+1]
+ skipthis = 1
+
elif args[i] == "--no-loop-tests":
self["loop-tests"] = 0
diff --git a/cts/patterns.py b/cts/patterns.py
index 1bc05a6..493b690 100644
--- a/cts/patterns.py
+++ b/cts/patterns.py
@@ -7,7 +7,9 @@ class BasePatterns:
def __init__(self, name):
self.name = name
patternvariants[name] = self
- self.ignore = []
+ self.ignore = [
+ "avoid confusing Valgrind",
+ ]
self.BadNews = []
self.components = {}
self.commands = {
@@ -140,7 +142,7 @@ class crm_lha(BasePatterns):
r"Parameters to .* changed",
]
- self.ignore = [
+ self.ignore = self.ignore + [
r"(ERROR|error):.*\s+assert\s+at\s+crm_glib_handler:"
"(ERROR|error): Message hist queue is filling up",
"stonithd.*CRIT: external_hostlist:.*'vmware gethosts' returned an empty hostlist",
@@ -177,7 +179,7 @@ class crm_cs_v0(BasePatterns):
"Pat:PacemakerUp" : "%s\W.*pacemakerd.*Starting Pacemaker",
})
- self.ignore = [
+ self.ignore = self.ignore + [
r"crm_mon:",
r"crmadmin:",
r"update_trace_data",
diff --git a/extra/ansible/docker/group_vars/all b/extra/ansible/docker/group_vars/all
new file mode 100644
index 0000000..935e88a
--- /dev/null
+++ b/extra/ansible/docker/group_vars/all
@@ -0,0 +1,5 @@
+max: 4
+prefix: ansible-pcmk
+base_image: centos:centos7
+subnet: 172.17.200
+pacemaker_authkey: this_is_very_insecure
\ No newline at end of file
diff --git a/extra/ansible/docker/hosts b/extra/ansible/docker/hosts
new file mode 100644
index 0000000..5b0fb71
--- /dev/null
+++ b/extra/ansible/docker/hosts
@@ -0,0 +1,7 @@
+[controllers]
+oss-uk-1.clusterlabs.org
+
+[containers]
+ansible-1
+ansible-2
+ansible-3
diff --git a/extra/ansible/docker/roles/docker-host/files/docker-enter b/extra/ansible/docker/roles/docker-host/files/docker-enter
new file mode 100644
index 0000000..04c4822
--- /dev/null
+++ b/extra/ansible/docker/roles/docker-host/files/docker-enter
@@ -0,0 +1,29 @@
+#! /bin/sh -e
+
+case "$1" in
+ -h|--help)
+ echo "Usage: docker-enter CONTAINER [COMMAND]"
+ exit 0
+ ;;
+esac
+
+if [ $(id -ru) -ne 0 ]; then
+ echo "You have to be root."
+ exit 1
+fi
+
+if [ $# -eq 0 ]; then
+ echo "Usage: docker-enter CONTAINER [COMMAND]"
+ exit 1
+fi
+
+container=$1; shift
+PID=$(docker inspect --format {{.State.Pid}} "$container")
+
+if [ $# -ne 0 ]; then
+ nsenter --target $PID --mount --uts --ipc --net --pid -- $*
+ exit $?
+fi
+
+nsenter --target $PID --mount --uts --ipc --net --pid
+exit 0
diff --git a/extra/ansible/docker/roles/docker-host/files/fence_docker_cts b/extra/ansible/docker/roles/docker-host/files/fence_docker_cts
new file mode 100644
index 0000000..6d6f025
--- /dev/null
+++ b/extra/ansible/docker/roles/docker-host/files/fence_docker_cts
@@ -0,0 +1,202 @@
+#!/bin/bash
+#
+# Copyright (c) 2014 David Vossel <dvossel@redhat.com>
+# All Rights Reserved.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of version 2 of the GNU General Public License as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it would be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+#
+# Further, this software is distributed without any warranty that it is
+# free of the rightful claim of any third person regarding infringement
+# or the like. Any license provided herein, whether implied or
+# otherwise, applies only to this software file. Patent licenses, if
+# any, provided herein do not apply to combinations of this program with
+# other software, or any other product whatsoever.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write the Free Software Foundation,
+# Inc., 59 Temple Place - Suite 330, Boston MA 02111-1307, USA.
+#
+#######################################################################
+
+port=""
+action="list" # Default fence action
+
+function usage()
+{
+cat <<EOF
+`basename $0` - A fencing agent for docker containers for testing purposes
+
+Usage: `basename $0` -o|--action [-n|--port] [options]
+Options:
+ -h, --help This text
+ -V, --version Version information
+
+Commands:
+ -o, --action Action to perform: on|off|reboot|status|monitor
+ -n, --port The name of a container to control/check
+
+EOF
+ exit 0;
+}
+
+function metadata()
+{
+cat <<EOF
+<?xml version="1.0" ?>
+<resource-agent name="fence_docker_cts" shortdesc="docker fencing agent for testing purposes" >
+ <longdesc>
+ fence_docker_cts fences docker containers for testing purposes.
+ </longdesc>
+ <parameters>
+ <parameter name="action" unique="1" required="0">
+ <getopt mixed="-o, --action=[action]" />
+ <content type="string" default="reboot" />
+ <shortdesc lang="en">Fencing Action</shortdesc>
+ </parameter>
+ <parameter name="port" unique="1" required="0">
+ <getopt mixed="-n, --port=[port]" />
+ <content type="string" />
+ <shortdesc lang="en">The name/id of docker container to control/check</shortdesc>
+ </parameter>
+ </parameters>
+ <actions>
+ <action name="on" />
+ <action name="off" />
+ <action name="reboot" />
+ <action name="status" />
+ <action name="list" />
+ <action name="monitor" />
+ <action name="metadata" />
+ </actions>
+</resource-agent>
+EOF
+ exit 0;
+}
+
+function docker_log() {
+ if ! [ "$action" = "list" ]; then
+ printf "$*\n" 1>&2
+ fi
+}
+
+# stdin option processing
+if [ -z $1 ]; then
+ # If there are no command line args, look for options from stdin
+ while read line; do
+ for word in $(echo "$line"); do
+ case $word in
+ option=*|action=*) action=`echo $word | sed s/.*=//`;;
+ port=*) port=`echo $word | sed s/.*=//`;;
+ node=*) port=`echo $word | sed s/.*=//`;;
+ nodename=*) port=`echo $word | sed s/.*=//`;;
+ --);;
+ *) docker_log "Invalid command: $word";;
+ esac
+ done
+ done
+fi
+
+# Command line option processing
+while true ; do
+ if [ -z "$1" ]; then
+ break;
+ fi
+ case "$1" in
+ -o|--action|--option) action=$2; shift; shift;;
+ -n|--port) port=$2; shift; shift;;
+ -V|--version) echo "1.0.0"; exit 0;;
+ --help|-h)
+ usage;
+ exit 0;;
+ --) shift ; break ;;
+ *) docker_log "Unknown option: $1. See --help for details."; exit 1;;
+ esac
+done
+
+action=`echo $action | tr 'A-Z' 'a-z'`
+case $action in
+ hostlist|list) action=list;;
+ stat|status) action=status;;
+ restart|reboot|reset) action=reboot;;
+ poweron|on) action=start;;
+ poweroff|off) action=stop;;
+esac
+
+function fence_done()
+{
+ if [ $1 -eq 0 ]; then
+ docker_log "Operation $action (port=$port) passed"
+ else
+ docker_log "Operation $action (port=$port) failed: $1"
+ fi
+ if [ -z "$returnfile" ]; then
+ rm -f $returnfile
+ fi
+ if [ -z "$helperscript" ]; then
+ rm -f $helperscript
+ fi
+ exit $1
+}
+
+case $action in
+ metadata) metadata;;
+esac
+
+returnfile=$(mktemp /tmp/fence_docker_cts_returnfileXXXX)
+returnstring=""
+helper_script=$(mktemp /tmp/fence_docker_cts_helperXXXX)
+
+exec_action()
+{
+ echo "#!/bin/bash" > $helper_script
+ echo "sleep 10000" >> $helper_script
+ chmod 755 $helper_script
+ src="$(uname -n)"
+
+ $helper_script "$src" "$action" "$returnfile" "$port" > /dev/null 2>&1 &
+ pid=$!
+ docker_log "waiting on pid $pid"
+ wait $pid > /dev/null 2>&1
+ returnstring=$(cat $returnfile)
+
+ if [ -z "$returnstring" ]; then
+ docker_log "fencing daemon did not respond"
+ fence_done 1
+ fi
+
+ if [ "$returnstring" == "fail" ]; then
+ docker_log "fencing daemon failed to execute action [$action on port $port]"
+ fence_done 1
+ fi
+
+ return 0
+}
+
+exec_action
+case $action in
+ list)
+ cat $returnfile
+ fence_done 0
+ ;;
+
+ status)
+ # 0 if container is on
+ # 1 if container can not be contacted or unknown
+ # 2 if container is off
+ if [ "$returnstring" = "true" ]; then
+ fence_done 0
+ else
+ fence_done 2
+ fi
+ ;;
+ monitor|stop|start|reboot) : ;;
+ *) docker_log "Unknown action: $action"; fence_done 1;;
+esac
+
+fence_done $?
diff --git a/extra/ansible/docker/roles/docker-host/files/launch.sh b/extra/ansible/docker/roles/docker-host/files/launch.sh
new file mode 100644
index 0000000..66bebf4
--- /dev/null
+++ b/extra/ansible/docker/roles/docker-host/files/launch.sh
@@ -0,0 +1,4 @@
+#!/bin/bash
+while true; do
+ sleep 1
+done
diff --git a/extra/ansible/docker/roles/docker-host/files/pcmk_remote_start b/extra/ansible/docker/roles/docker-host/files/pcmk_remote_start
new file mode 100644
index 0000000..1bf0320
--- /dev/null
+++ b/extra/ansible/docker/roles/docker-host/files/pcmk_remote_start
@@ -0,0 +1,18 @@
+#!/bin/bash
+/usr/sbin/ip_start
+pid=$(pidof pacemaker_remoted)
+if [ "$?" -ne 0 ]; then
+ mkdir -p /var/run
+
+ export PCMK_debugfile=$pcmklogs
+ (pacemaker_remoted &) & > /dev/null 2>&1
+ sleep 5
+
+ pid=$(pidof pacemaker_remoted)
+ if [ "$?" -ne 0 ]; then
+ echo "startup of pacemaker failed"
+ exit 1
+ fi
+ echo "$pid" > /var/run/pacemaker_remoted.pid
+fi
+exit 0
diff --git a/extra/ansible/docker/roles/docker-host/files/pcmk_remote_stop b/extra/ansible/docker/roles/docker-host/files/pcmk_remote_stop
new file mode 100644
index 0000000..074cd59
--- /dev/null
+++ b/extra/ansible/docker/roles/docker-host/files/pcmk_remote_stop
@@ -0,0 +1,36 @@
+#!/bin/bash
+status()
+{
+ pid=$(pidof $1 2>/dev/null)
+ rtrn=$?
+ if [ $rtrn -ne 0 ]; then
+ echo "$1 is stopped"
+ else
+ echo "$1 (pid $pid) is running..."
+ fi
+ return $rtrn
+}
+stop()
+{
+ desc="Pacemaker Remote"
+ prog=$1
+ shutdown_prog=$prog
+
+ if status $shutdown_prog > /dev/null 2>&1; then
+ kill -TERM $(pidof $prog) > /dev/null 2>&1
+
+ while status $prog > /dev/null 2>&1; do
+ sleep 1
+ echo -n "."
+ done
+ else
+ echo -n "$desc is already stopped"
+ fi
+
+ rm -f /var/lock/subsystem/pacemaker
+ rm -f /var/run/${prog}.pid
+ killall -q -9 'crmd stonithd attrd cib lrmd pacemakerd pacemaker_remoted'
+}
+
+stop "pacemaker_remoted"
+exit 0
diff --git a/extra/ansible/docker/roles/docker-host/files/pcmk_start b/extra/ansible/docker/roles/docker-host/files/pcmk_start
new file mode 100644
index 0000000..d8b2ba8
--- /dev/null
+++ b/extra/ansible/docker/roles/docker-host/files/pcmk_start
@@ -0,0 +1,23 @@
+#!/bin/bash
+
+/usr/sbin/ip_start
+sed -i 's@to_syslog:.*yes@to_logfile: yes\nlogfile: /var/log/pacemaker.log@g' /etc/corosync/corosync.conf
+
+/usr/share/corosync/corosync start > /dev/null 2>&1
+
+pid=$(pidof pacemakerd)
+if [ "$?" -ne 0 ]; then
+ mkdir -p /var/run
+
+ export PCMK_debugfile=$pcmklogs
+ (pacemakerd &) & > /dev/null 2>&1
+ sleep 5
+
+ pid=$(pidof pacemakerd)
+ if [ "$?" -ne 0 ]; then
+ echo "startup of pacemaker failed"
+ exit 1
+ fi
+ echo "$pid" > /var/run/pacemakerd.pid
+fi
+exit 0
diff --git a/extra/ansible/docker/roles/docker-host/files/pcmk_stop b/extra/ansible/docker/roles/docker-host/files/pcmk_stop
new file mode 100644
index 0000000..a8f395a
--- /dev/null
+++ b/extra/ansible/docker/roles/docker-host/files/pcmk_stop
@@ -0,0 +1,45 @@
+#!/bin/bash
+status()
+{
+ pid=$(pidof $1 2>/dev/null)
+ rtrn=$?
+ if [ $rtrn -ne 0 ]; then
+ echo "$1 is stopped"
+ else
+ echo "$1 (pid $pid) is running..."
+ fi
+ return $rtrn
+}
+stop()
+{
+ desc="Pacemaker Cluster Manager"
+ prog=$1
+ shutdown_prog=$prog
+
+ if ! status $prog > /dev/null 2>&1; then
+ shutdown_prog="crmd"
+ fi
+
+ cname=$(crm_node --name)
+ crm_attribute -N $cname -n standby -v true -l reboot
+
+ if status $shutdown_prog > /dev/null 2>&1; then
+ kill -TERM $(pidof $prog) > /dev/null 2>&1
+
+ while status $prog > /dev/null 2>&1; do
+ sleep 1
+ echo -n "."
+ done
+ else
+ echo -n "$desc is already stopped"
+ fi
+
+ rm -f /var/lock/subsystem/pacemaker
+ rm -f /var/run/${prog}.pid
+ killall -q -9 'crmd stonithd attrd cib lrmd pacemakerd pacemaker_remoted'
+}
+
+stop "pacemakerd"
+/usr/share/corosync/corosync stop > /dev/null 2>&1
+killall -q -9 'corosync'
+exit 0
diff --git a/extra/ansible/docker/roles/docker-host/tasks/main.yml b/extra/ansible/docker/roles/docker-host/tasks/main.yml
new file mode 100644
index 0000000..ce69adf
--- /dev/null
+++ b/extra/ansible/docker/roles/docker-host/tasks/main.yml
@@ -0,0 +1,77 @@
+---
+#local_action: command /usr/bin/take_out_of_pool {{ inventory_hostname }}
+- name: Update docker
+ yum: pkg=docker state=latest
+- name: Start docker
+ service: name=docker state=started enabled=yes
+- name: Install helper
+ copy: src=docker-enter dest=/usr/sbin/ mode=0755
+- name: Download image
+ shell: docker pull {{ base_image }}
+- name: Cleanup kill
+ shell: docker kill $(docker ps -a | grep {{ prefix }} | awk '{print $1}') || echo "Nothing to kill"
+- name: Cleanup remove
+ shell: docker rm $(docker ps -a | grep {{ prefix }} | awk '{print $1}') || echo "Nothing to remove"
+- name: Cleanup docker skeleton
+ file: path={{ prefix }} state=absent
+- name: Create docker skeleton
+ file: path={{ prefix }}/{{ item }} state=directory recurse=yes
+ with_items:
+ - rpms
+ - repos
+ - bin_files
+ - launch_scripts
+- name: Create IP helper
+ template: src=ip_start.j2 dest={{ prefix }}/bin_files/ip_start mode=0755
+- name: Copy helper scripts
+ copy: src={{ item }} dest={{ prefix }}/bin_files/{{ item }} mode=0755
+ with_items:
+ - pcmk_stop
+ - pcmk_start
+ - pcmk_remote_stop
+ - pcmk_remote_start
+ - fence_docker_cts
+- name: Copy launch script
+ copy: src=launch.sh dest={{ prefix }}/launch_scripts/launch.sh mode=0755
+- name: Copy authorized keys
+ shell: cp /root/.ssh/authorized_keys {{ prefix }}
+- name: Create docker file
+ template: src=Dockerfile.j2 dest={{ prefix }}/Dockerfile
+- name: Making image
+ shell: docker build -t {{ prefix }} {{ prefix }}
+- name: Launch images
+ shell: docker run -d -i -t -P -h {{ prefix }}-{{ item }} --name={{ prefix }}-{{ item }} -p 2200{{ item }}:22 $(docker images | grep {{ prefix }}.*latest | awk '{print $3}') /bin/bash
+ with_sequence: count={{ max }}
+- name: Calculate IPs
+ shell: for n in $(seq {{ max }} ); do echo {{ subnet }}.${n}; done | tr '\n' ' '
+ register: node_ips
+- name: Start the IP
+ shell: docker-enter {{ prefix }}-{{ item }} ip_start
+ with_sequence: count={{ max }}
+- name: Configure cluster
+ shell: docker-enter {{ prefix }}-{{ item }} pcs cluster setup --local --name {{ prefix }} {{ node_ips.stdout }}
+ with_sequence: count={{ max }}
+- name: Start the cluster
+ shell: docker-enter {{ prefix }}-{{ item }} pcmk_start
+ with_sequence: count={{ max }}
+- name: Set cluster options
+ shell: docker-enter {{ prefix }}-1 pcs property set stonith-enabled=false
+- name: Configure VIP
+ shell: docker-enter {{ prefix }}-1 pcs resource create ClusterIP ocf:heartbeat:IPaddr2 ip={{ subnet }}.100 cidr_netmask=32 op monitor interval=30s
+- name: Configure
+ shell: docker-enter {{ prefix }}-1 pcs resource defaults resource-stickiness=100
+- name: Configure
+ shell: docker-enter {{ prefix }}-1 pcs resource create WebSite apache configfile=/etc/httpd/conf/httpd.conf statusurl="http://localhost/server-status" op monitor interval=1min
+- name: Configure
+ shell: docker-enter {{ prefix }}-1 pcs constraint colocation add WebSite with ClusterIP INFINITY
+- name: Configure
+ shell: docker-enter {{ prefix }}-1 pcs constraint order ClusterIP then WebSite
+- name: Configure
+ shell: docker-enter {{ prefix }}-1 pcs constraint location WebSite prefers {{ prefix }}-1=50
+# TODO: Enable fencing
+# TODO: Make this a full LAMP stack similar to https://github.com/ansible/ansible-examples/tree/master/lamp_simple
+# TODO: Create a Pacemaker module?
+
+# run_once: true
+# delegate_to: web01.example.org
+
diff --git a/extra/ansible/docker/roles/docker-host/templates/Dockerfile.j2 b/extra/ansible/docker/roles/docker-host/templates/Dockerfile.j2
new file mode 100644
index 0000000..1d57175
--- /dev/null
+++ b/extra/ansible/docker/roles/docker-host/templates/Dockerfile.j2
@@ -0,0 +1,16 @@
+FROM {{ base_image }}
+ADD /repos /etc/yum.repos.d/
+#ADD /rpms /root/
+#RUN yum install -y /root/*.rpm
+ADD /launch_scripts /root/
+ADD /bin_files /usr/sbin/
+
+RUN mkdir -p /root/.ssh; chmod 700 /root/.ssh
+ADD authorized_keys /root/.ssh/
+
+RUN yum install -y openssh-server net-tools pacemaker pacemaker-cts resource-agents pcs corosync which fence-agents-common sysvinit-tools
+RUN mkdir -p /etc/pacemaker/
+RUN echo {{ pacemaker_authkey }} > /etc/pacemaker/authkey
+RUN /usr/sbin/sshd
+
+ENTRYPOINT ["/root/launch.sh"]
diff --git a/extra/ansible/docker/roles/docker-host/templates/ip_start.j2 b/extra/ansible/docker/roles/docker-host/templates/ip_start.j2
new file mode 100755
index 0000000..edbd392
--- /dev/null
+++ b/extra/ansible/docker/roles/docker-host/templates/ip_start.j2
@@ -0,0 +1,3 @@
+offset=$(hostname | sed s/.*-//)
+export OCF_ROOT=/usr/lib/ocf/ OCF_RESKEY_ip={{ subnet }}.${offset} OCF_RESKEY_cidr_netmask=32
+/usr/lib/ocf/resource.d/heartbeat/IPaddr2 start
diff --git a/extra/ansible/docker/site.yml b/extra/ansible/docker/site.yml
new file mode 100644
index 0000000..0cc65e4
--- /dev/null
+++ b/extra/ansible/docker/site.yml
@@ -0,0 +1,12 @@
+---
+# See /etc/ansible/hosts or -i hosts
+- hosts: controllers
+ remote_user: root
+ roles:
+ - docker-host
+
+#- hosts: containers
+# gather_facts: no
+# remote_user: root
+# roles:
+# - docker-container
diff --git a/include/crm/msg_xml.h b/include/crm/msg_xml.h
index 42f9003..15f1b3c 100644
--- a/include/crm/msg_xml.h
+++ b/include/crm/msg_xml.h
@@ -194,6 +194,7 @@
# define XML_RSC_ATTR_INTERLEAVE "interleave"
# define XML_RSC_ATTR_INCARNATION "clone"
# define XML_RSC_ATTR_INCARNATION_MAX "clone-max"
+# define XML_RSC_ATTR_INCARNATION_MIN "clone-min"
# define XML_RSC_ATTR_INCARNATION_NODEMAX "clone-node-max"
# define XML_RSC_ATTR_MASTER_MAX "master-max"
# define XML_RSC_ATTR_MASTER_NODEMAX "master-node-max"
diff --git a/include/crm/pengine/status.h b/include/crm/pengine/status.h
index 4214959..b95b1e5 100644
--- a/include/crm/pengine/status.h
+++ b/include/crm/pengine/status.h
@@ -256,7 +256,6 @@ struct resource_s {
int stickiness;
int sort_index;
int failure_timeout;
- int remote_reconnect_interval;
int effective_priority;
int migration_threshold;
@@ -295,6 +294,7 @@ struct resource_s {
const char *isolation_wrapper;
gboolean exclusive_discover;
+ int remote_reconnect_interval;
};
struct pe_action_s {
@@ -324,6 +324,26 @@ struct pe_action_s {
GHashTable *meta;
GHashTable *extra;
+ /*
+ * These two varables are associated with the constraint logic
+ * that involves first having one or more actions runnable before
+ * then allowing this action to execute.
+ *
+ * These varables are used with features such as 'clone-min' which
+ * requires at minimum X number of cloned instances to be running
+ * before an order dependency can run. Another option that uses
+ * this is 'require-all=false' in ordering constrants. This option
+ * says "only required one instance of a resource to start before
+ * allowing dependencies to start" basicall require-all=false is
+ * the same as clone-min=1.
+ */
+
+ /* current number of known runnable actions in the before list. */
+ int runnable_before;
+ /* the number of "before" runnable actions required for this action
+ * to be considered runnable */
+ int required_runnable_before;
+
GListPtr actions_before; /* action_warpper_t* */
GListPtr actions_after; /* action_warpper_t* */
};
diff --git a/lib/cib/Makefile.am b/lib/cib/Makefile.am
index e84f4f7..1e50511 100644
--- a/lib/cib/Makefile.am
+++ b/lib/cib/Makefile.am
@@ -28,7 +28,7 @@ noinst_HEADERS =
libcib_la_SOURCES = cib_ops.c cib_utils.c cib_client.c cib_native.c cib_attrs.c
libcib_la_SOURCES += cib_file.c cib_remote.c
-libcib_la_LDFLAGS = -version-info 4:1:0 -L$(top_builddir)/lib/pengine/.libs
+libcib_la_LDFLAGS = -version-info 4:2:0 -L$(top_builddir)/lib/pengine/.libs
libcib_la_LIBADD = $(CRYPTOLIB) $(top_builddir)/lib/pengine/libpe_rules.la $(top_builddir)/lib/common/libcrmcommon.la
libcib_la_CFLAGS = -I$(top_srcdir)
diff --git a/lib/cluster/Makefile.am b/lib/cluster/Makefile.am
index 29413ba..29daeb2 100644
--- a/lib/cluster/Makefile.am
+++ b/lib/cluster/Makefile.am
@@ -28,7 +28,7 @@ header_HEADERS =
lib_LTLIBRARIES = libcrmcluster.la
libcrmcluster_la_SOURCES = election.c cluster.c membership.c
-libcrmcluster_la_LDFLAGS = -version-info 4:2:0 $(CLUSTERLIBS)
+libcrmcluster_la_LDFLAGS = -version-info 5:0:1 $(CLUSTERLIBS)
libcrmcluster_la_LIBADD = $(top_builddir)/lib/common/libcrmcommon.la $(top_builddir)/lib/fencing/libstonithd.la
libcrmcluster_la_DEPENDENCIES = $(top_builddir)/lib/common/libcrmcommon.la $(top_builddir)/lib/fencing/libstonithd.la
diff --git a/lib/common/Makefile.am b/lib/common/Makefile.am
index a593f40..f5c0766 100644
--- a/lib/common/Makefile.am
+++ b/lib/common/Makefile.am
@@ -37,7 +37,7 @@ if BUILD_CIBSECRETS
libcrmcommon_la_SOURCES += cib_secrets.c
endif
-libcrmcommon_la_LDFLAGS = -version-info 7:0:4
+libcrmcommon_la_LDFLAGS = -version-info 8:0:5
libcrmcommon_la_LIBADD = @LIBADD_DL@ $(GNUTLSLIBS)
libcrmcommon_la_SOURCES += $(top_builddir)/lib/gnu/md5.c
diff --git a/lib/fencing/Makefile.am b/lib/fencing/Makefile.am
index 2bdcfeb..fbe02e4 100644
--- a/lib/fencing/Makefile.am
+++ b/lib/fencing/Makefile.am
@@ -25,7 +25,7 @@ AM_CPPFLAGS = -I$(top_builddir)/include -I$(top_srcdir)/include \
lib_LTLIBRARIES = libstonithd.la
libstonithd_la_SOURCES = st_client.c
-libstonithd_la_LDFLAGS = -version-info 3:2:1
+libstonithd_la_LDFLAGS = -version-info 3:3:1
libstonithd_la_LIBADD = $(top_builddir)/lib/common/libcrmcommon.la
AM_CFLAGS = $(AM_CPPFLAGS)
diff --git a/lib/lrmd/Makefile.am b/lib/lrmd/Makefile.am
index f961ae1..820654c 100644
--- a/lib/lrmd/Makefile.am
+++ b/lib/lrmd/Makefile.am
@@ -25,7 +25,7 @@ AM_CPPFLAGS = -I$(top_builddir)/include -I$(top_srcdir)/include \
lib_LTLIBRARIES = liblrmd.la
liblrmd_la_SOURCES = lrmd_client.c proxy_common.c
-liblrmd_la_LDFLAGS = -version-info 3:0:2
+liblrmd_la_LDFLAGS = -version-info 3:1:2
liblrmd_la_LIBADD = $(top_builddir)/lib/common/libcrmcommon.la \
$(top_builddir)/lib/services/libcrmservice.la \
$(top_builddir)/lib/fencing/libstonithd.la
diff --git a/lib/pengine/Makefile.am b/lib/pengine/Makefile.am
index 78da075..60d1770 100644
--- a/lib/pengine/Makefile.am
+++ b/lib/pengine/Makefile.am
@@ -26,11 +26,11 @@ lib_LTLIBRARIES = libpe_rules.la libpe_status.la
## SOURCES
noinst_HEADERS = unpack.h variant.h
-libpe_rules_la_LDFLAGS = -version-info 2:4:0
+libpe_rules_la_LDFLAGS = -version-info 2:5:0
libpe_rules_la_SOURCES = rules.c common.c
libpe_rules_la_LIBADD = $(top_builddir)/lib/common/libcrmcommon.la
-libpe_status_la_LDFLAGS = -version-info 8:0:4
+libpe_status_la_LDFLAGS = -version-info 9:0:5
libpe_status_la_SOURCES = status.c unpack.c utils.c complex.c native.c group.c clone.c rules.c common.c
libpe_status_la_LIBADD = @CURSESLIBS@ $(top_builddir)/lib/common/libcrmcommon.la
diff --git a/lib/services/dbus.c b/lib/services/dbus.c
index 6341fc5..e2efecb 100644
--- a/lib/services/dbus.c
+++ b/lib/services/dbus.c
@@ -64,11 +64,14 @@ pcmk_dbus_find_error(const char *method, DBusPendingCall* pending, DBusMessage *
} else {
DBusMessageIter args;
int dtype = dbus_message_get_type(reply);
+ char *sig;
switch(dtype) {
case DBUS_MESSAGE_TYPE_METHOD_RETURN:
dbus_message_iter_init(reply, &args);
- crm_trace("Call to %s returned '%s'", method, dbus_message_iter_get_signature(&args));
+ sig = dbus_message_iter_get_signature(&args);
+ crm_trace("Call to %s returned '%s'", method, sig);
+ dbus_free(sig);
break;
case DBUS_MESSAGE_TYPE_INVALID:
error.message = "Invalid reply";
@@ -217,11 +220,14 @@ bool pcmk_dbus_type_check(DBusMessage *msg, DBusMessageIter *field, int expected
if(dtype != expected) {
DBusMessageIter args;
+ char *sig;
dbus_message_iter_init(msg, &args);
+ sig = dbus_message_iter_get_signature(&args);
do_crm_log_alias(LOG_ERR, __FILE__, function, line,
- "Unexepcted DBus type, expected %c in '%s' instead of %c",
- expected, dbus_message_iter_get_signature(&args), dtype);
+ "Unexpected DBus type, expected %c in '%s' instead of %c",
+ expected, sig, dtype);
+ dbus_free(sig);
return FALSE;
}
diff --git a/lib/services/services.c b/lib/services/services.c
index 08bff88..7e2b9f7 100644
--- a/lib/services/services.c
+++ b/lib/services/services.c
@@ -348,6 +348,34 @@ services_action_create_generic(const char *exec, const char *args[])
return op;
}
+#if SUPPORT_DBUS
+/*
+ * \internal
+ * \brief Update operation's pending DBus call, unreferencing old one if needed
+ *
+ * \param[in,out] op Operation to modify
+ * \param[in] pending Pending call to set
+ */
+void
+services_set_op_pending(svc_action_t *op, DBusPendingCall *pending)
+{
+ if (op->opaque->pending && (op->opaque->pending != pending)) {
+ if (pending) {
+ crm_info("Lost pending DBus call (%p)", op->opaque->pending);
+ } else {
+ crm_trace("Done with pending DBus call (%p)", op->opaque->pending);
+ }
+ dbus_pending_call_unref(op->opaque->pending);
+ }
+ op->opaque->pending = pending;
+ if (pending) {
+ crm_trace("Updated pending DBus call (%p)", pending);
+ } else {
+ crm_trace("Cleared pending DBus call");
+ }
+}
+#endif
+
void
services_action_cleanup(svc_action_t * op)
{
diff --git a/lib/services/services_private.h b/lib/services/services_private.h
index 183afb5..a98cd91 100644
--- a/lib/services/services_private.h
+++ b/lib/services/services_private.h
@@ -63,4 +63,8 @@ void handle_blocked_ops(void);
gboolean is_op_blocked(const char *rsc);
+#if SUPPORT_DBUS
+void services_set_op_pending(svc_action_t *op, DBusPendingCall *pending);
+#endif
+
#endif /* __MH_SERVICES_PRIVATE_H__ */
diff --git a/lib/services/systemd.c b/lib/services/systemd.c
index 749d61c..e1e1bc9 100644
--- a/lib/services/systemd.c
+++ b/lib/services/systemd.c
@@ -461,7 +461,12 @@ systemd_async_dispatch(DBusPendingCall *pending, void *user_data)
if(op) {
crm_trace("Got result: %p for %p for %s, %s", reply, pending, op->rsc, op->action);
- op->opaque->pending = NULL;
+ if (pending == op->opaque->pending) {
+ op->opaque->pending = NULL;
+ } else {
+ crm_info("Received unexpected reply for pending DBus call (%p vs %p)",
+ op->opaque->pending, pending);
+ }
systemd_exec_result(reply, op);
} else {
@@ -499,10 +504,7 @@ systemd_unit_check(const char *name, const char *state, void *userdata)
}
if (op->synchronous == FALSE) {
- if (op->opaque->pending) {
- dbus_pending_call_unref(op->opaque->pending);
- }
- op->opaque->pending = NULL;
+ services_set_op_pending(op, NULL);
operation_finalize(op);
}
}
@@ -535,7 +537,7 @@ systemd_unit_exec_with_unit(svc_action_t * op, const char *unit)
return op->rc == PCMK_OCF_OK;
} else if (pending) {
dbus_pending_call_ref(pending);
- op->opaque->pending = pending;
+ services_set_op_pending(op, pending);
return TRUE;
}
@@ -617,8 +619,7 @@ systemd_unit_exec_with_unit(svc_action_t * op, const char *unit)
dbus_message_unref(msg);
if(pending) {
- dbus_pending_call_ref(pending);
- op->opaque->pending = pending;
+ services_set_op_pending(op, pending);
return TRUE;
}
return FALSE;
diff --git a/lib/transition/Makefile.am b/lib/transition/Makefile.am
index 8ce7775..04d18fe 100644
--- a/lib/transition/Makefile.am
+++ b/lib/transition/Makefile.am
@@ -27,7 +27,7 @@ lib_LTLIBRARIES = libtransitioner.la
noinst_HEADERS =
libtransitioner_la_SOURCES = unpack.c graph.c utils.c
-libtransitioner_la_LDFLAGS = -version-info 2:3:0
+libtransitioner_la_LDFLAGS = -version-info 2:4:0
libtransitioner_la_CFLAGS = -I$(top_builddir)
libtransitioner_la_LIBADD = $(top_builddir)/lib/common/libcrmcommon.la
diff --git a/pengine/Makefile.am b/pengine/Makefile.am
index 31532cf..0e12a1f 100644
--- a/pengine/Makefile.am
+++ b/pengine/Makefile.am
@@ -61,7 +61,7 @@ endif
noinst_HEADERS = allocate.h utils.h pengine.h
#utils.h pengine.h
-libpengine_la_LDFLAGS = -version-info 8:0:4
+libpengine_la_LDFLAGS = -version-info 9:0:5
# -L$(top_builddir)/lib/pils -lpils -export-dynamic -module -avoid-version
libpengine_la_SOURCES = pengine.c allocate.c utils.c constraints.c
libpengine_la_SOURCES += native.c group.c clone.c master.c graph.c utilization.c
diff --git a/pengine/allocate.c b/pengine/allocate.c
index 68cafd4..ec5a18d 100644
--- a/pengine/allocate.c
+++ b/pengine/allocate.c
@@ -1962,7 +1962,6 @@ expand_node_list(GListPtr list)
if(node_list) {
existing_len = strlen(node_list);
}
-
crm_trace("Adding %s (%dc) at offset %d", node->details->uname, len - 2, existing_len);
node_list = realloc_safe(node_list, len + existing_len);
sprintf(node_list + existing_len, "%s%s", existing_len == 0 ? "":" ", node->details->uname);
diff --git a/pengine/allocate.h b/pengine/allocate.h
index f6602c6..73f750e 100644
--- a/pengine/allocate.h
+++ b/pengine/allocate.h
@@ -171,5 +171,6 @@ extern enum pe_graph_flags clone_update_actions(action_t * first, action_t * the
enum pe_action_flags filter, enum pe_ordering type);
gboolean update_action_flags(action_t * action, enum pe_action_flags flags);
+gboolean update_action(action_t * action);
#endif
diff --git a/pengine/clone.c b/pengine/clone.c
index 3840a0a..ebf53ed 100644
--- a/pengine/clone.c
+++ b/pengine/clone.c
@@ -21,6 +21,7 @@
#include <crm/msg_xml.h>
#include <allocate.h>
#include <utils.h>
+#include <allocate.h>
#define VARIANT_CLONE 1
#include <lib/pengine/variant.h>
@@ -1338,6 +1339,8 @@ clone_update_actions(action_t * first, action_t * then, node_t * node, enum pe_a
changed |= native_update_actions(first, then, node, flags, filter, type);
for (; gIter != NULL; gIter = gIter->next) {
+ enum pe_graph_flags child_changed = pe_graph_none;
+ GListPtr lpc = NULL;
resource_t *child = (resource_t *) gIter->data;
action_t *child_action = find_first_action(child->actions, NULL, then->task, node);
@@ -1345,9 +1348,17 @@ clone_update_actions(action_t * first, action_t * then, node_t * node, enum pe_a
enum pe_action_flags child_flags = child->cmds->action_flags(child_action, node);
if (is_set(child_flags, pe_action_runnable)) {
- changed |=
+
+ child_changed |=
child->cmds->update_actions(first, child_action, node, flags, filter, type);
}
+ changed |= child_changed;
+ if (child_changed & pe_graph_updated_then) {
+ for (lpc = child_action->actions_after; lpc != NULL; lpc = lpc->next) {
+ action_wrapper_t *other = (action_wrapper_t *) lpc->data;
+ update_action(other->action);
+ }
+ }
}
}
}
diff --git a/pengine/constraints.c b/pengine/constraints.c
index 1f44811..7527aa6 100644
--- a/pengine/constraints.c
+++ b/pengine/constraints.c
@@ -256,7 +256,7 @@ unpack_simple_rsc_order(xmlNode * xml_obj, pe_working_set_t * data_set)
resource_t *rsc_then = NULL;
resource_t *rsc_first = NULL;
gboolean invert_bool = TRUE;
- gboolean require_all = TRUE;
+ int min_required_before = 0;
enum pe_order_kind kind = pe_order_kind_mandatory;
enum pe_ordering cons_weight = pe_order_optional;
@@ -351,7 +351,15 @@ unpack_simple_rsc_order(xmlNode * xml_obj, pe_working_set_t * data_set)
&& crm_is_true(require_all_s) == FALSE
&& rsc_first->variant >= pe_clone) {
- require_all = FALSE;
+ /* require-all=false means only one instance of the clone is required */
+ min_required_before = 1;
+ } else if (rsc_first->variant >= pe_clone) {
+ const char *min_clones_s = g_hash_table_lookup(rsc_first->meta, XML_RSC_ATTR_INCARNATION_MIN);
+ if (min_clones_s) {
+ /* if clone min is set, we require at a minimum X number of instances
+ * to be runnable before allowing dependencies to be runnable. */
+ min_required_before = crm_parse_int(min_clones_s, "0");
+ }
}
cons_weight = pe_order_optional;
@@ -368,22 +376,31 @@ unpack_simple_rsc_order(xmlNode * xml_obj, pe_working_set_t * data_set)
cons_weight |= get_flags(id, kind, action_first, action_then, FALSE);
}
- if (require_all == FALSE) {
+ /* If there is a minimum number of instances that must be runnable before
+ * the 'then' action is runnable, we use a pseudo action as an intermediate step
+ * start min number of clones -> pseudo action is runnable -> dependency runnable. */
+ if (min_required_before) {
GListPtr rIter = NULL;
char *task = crm_concat(CRM_OP_RELAXED_CLONE, id, ':');
action_t *unordered_action = get_pseudo_op(task, data_set);
free(task);
+ /* require the pseudo action to have "min_required_before" number of
+ * actions to be considered runnable before allowing the pseudo action
+ * to be runnable. */
+ unordered_action->required_runnable_before = min_required_before;
update_action_flags(unordered_action, pe_action_requires_any);
for (rIter = rsc_first->children; id && rIter; rIter = rIter->next) {
resource_t *child = rIter->data;
-
+ /* order each clone instance before the pseudo action */
custom_action_order(child, generate_op_key(child->id, action_first, 0), NULL,
NULL, NULL, unordered_action,
pe_order_one_or_more | pe_order_implies_then_printed, data_set);
}
+ /* order the "then" dependency to occur after the pseudo action only if
+ * the pseudo action is runnable */
order_id = custom_action_order(NULL, NULL, unordered_action,
rsc_then, generate_op_key(rsc_then->id, action_then, 0), NULL,
cons_weight | pe_order_runnable_left, data_set);
diff --git a/pengine/graph.c b/pengine/graph.c
index 9cfede6..3d832f0 100644
--- a/pengine/graph.c
+++ b/pengine/graph.c
@@ -29,7 +29,6 @@
#include <allocate.h>
#include <utils.h>
-gboolean update_action(action_t * action);
void update_colo_start_chain(action_t * action);
gboolean rsc_update_action(action_t * first, action_t * then, enum pe_ordering type);
@@ -261,8 +260,16 @@ graph_update_action(action_t * first, action_t * then, node_t * node, enum pe_ac
pe_action_runnable, pe_order_one_or_more);
} else if (is_set(flags, pe_action_runnable)) {
- if (update_action_flags(then, pe_action_runnable)) {
- changed |= pe_graph_updated_then;
+ /* alright. a "first" action is considered runnable, incremente
+ * the 'runnable_before' counter */
+ then->runnable_before++;
+
+ /* if the runnable before count for then exceeds the required number
+ * of "before" runnable actions... mark then as runnable */
+ if (then->runnable_before >= then->required_runnable_before) {
+ if (update_action_flags(then, pe_action_runnable)) {
+ changed |= pe_graph_updated_then;
+ }
}
}
if (changed) {
@@ -456,6 +463,18 @@ update_action(action_t * then)
pe_action_pseudo) ? "pseudo" : then->node ? then->node->details->uname : "");
if (is_set(then->flags, pe_action_requires_any)) {
+ /* initialize current known runnable before actions to 0
+ * from here as graph_update_action is called for each of
+ * then's before actions, this number will increment as
+ * runnable 'first' actions are encountered */
+ then->runnable_before = 0;
+
+ /* for backwards compatibility with previous options that use
+ * the 'requires_any' flag, initalize required to 1 if it is
+ * not set. */
+ if (then->required_runnable_before == 0) {
+ then->required_runnable_before = 1;
+ }
clear_bit(then->flags, pe_action_runnable);
/* We are relying on the pe_order_one_or_more clause of
* graph_update_action(), called as part of the:
diff --git a/pengine/native.c b/pengine/native.c
index b93f8da..7d5f602 100644
--- a/pengine/native.c
+++ b/pengine/native.c
@@ -2817,8 +2817,7 @@ native_create_probe(resource_t * rsc, node_t * node, action_t * complete,
}
static void
-native_start_constraints(resource_t * rsc, action_t * stonith_op, gboolean is_stonith,
- pe_working_set_t * data_set)
+native_start_constraints(resource_t * rsc, action_t * stonith_op, pe_working_set_t * data_set)
{
node_t *target = stonith_op ? stonith_op->node : NULL;
@@ -2893,14 +2892,24 @@ find_fence_target_node_actions(GListPtr search_list, const char *key, node_t *fe
}
static void
-native_stop_constraints(resource_t * rsc, action_t * stonith_op, gboolean is_stonith,
- pe_working_set_t * data_set)
+native_stop_constraints(resource_t * rsc, action_t * stonith_op, pe_working_set_t * data_set)
{
char *key = NULL;
GListPtr gIter = NULL;
GListPtr action_list = NULL;
+
+ action_t *start = NULL;
resource_t *top = uber_parent(rsc);
+ key = start_key(rsc);
+ action_list = find_actions(rsc->actions, key, NULL);
+ if(action_list) {
+ start = action_list->data;
+ }
+
+ g_list_free(action_list);
+ free(key);
+
key = stop_key(rsc);
action_list = find_fence_target_node_actions(rsc->actions, key, stonith_op->node, data_set);
free(key);
@@ -2932,7 +2941,7 @@ native_stop_constraints(resource_t * rsc, action_t * stonith_op, gboolean is_sto
update_action_flags(action, pe_action_runnable);
update_action_flags(action, pe_action_implied_by_stonith);
- {
+ if(start == NULL || start->needs > rsc_req_quorum) {
enum pe_ordering flags = pe_order_optional;
action_t *parent_stop = find_first_action(top->actions, NULL, RSC_STOP, NULL);
@@ -3032,7 +3041,8 @@ native_stop_constraints(resource_t * rsc, action_t * stonith_op, gboolean is_sto
crm_trace("here - 1");
update_action_flags(action, pe_action_pseudo);
update_action_flags(action, pe_action_runnable);
- if (is_stonith == FALSE) {
+
+ if (start == NULL || start->needs > rsc_req_quorum) {
order_actions(stonith_op, action, pe_order_preserve|pe_order_optional);
}
}
@@ -3044,8 +3054,6 @@ native_stop_constraints(resource_t * rsc, action_t * stonith_op, gboolean is_sto
void
rsc_stonith_ordering(resource_t * rsc, action_t * stonith_op, pe_working_set_t * data_set)
{
- gboolean is_stonith = FALSE;
-
if (rsc->children) {
GListPtr gIter = NULL;
@@ -3063,11 +3071,11 @@ rsc_stonith_ordering(resource_t * rsc, action_t * stonith_op, pe_working_set_t *
}
/* Start constraints */
- native_start_constraints(rsc, stonith_op, is_stonith, data_set);
+ native_start_constraints(rsc, stonith_op, data_set);
/* Stop constraints */
if (stonith_op) {
- native_stop_constraints(rsc, stonith_op, is_stonith, data_set);
+ native_stop_constraints(rsc, stonith_op, data_set);
}
}
diff --git a/pengine/regression.sh b/pengine/regression.sh
index d184798..7f73f92 100755
--- a/pengine/regression.sh
+++ b/pengine/regression.sh
@@ -31,6 +31,20 @@ info Performing the following tests from $io_dir
create_mode="false"
echo ""
+do_test cloned_start_one "order first clone then clone... first clone_min=2"
+do_test cloned_start_two "order first clone then clone... first clone_min=2"
+do_test cloned_stop_one "order first clone then clone... first clone_min=2"
+do_test cloned_stop_two "order first clone then clone... first clone_min=2"
+do_test clone_min_interleave_start_one "order first clone then clone... first clone_min=2 and then has interleave=true"
+do_test clone_min_interleave_start_two "order first clone then clone... first clone_min=2 and then has interleave=true"
+do_test clone_min_interleave_stop_one "order first clone then clone... first clone_min=2 and then has interleave=true"
+do_test clone_min_interleave_stop_two "order first clone then clone... first clone_min=2 and then has interleave=true"
+do_test clone_min_start_one "order first clone then primitive... first clone_min=2"
+do_test clone_min_start_two "order first clone then primitive... first clone_min=2"
+do_test clone_min_stop_all "order first clone then primitive... first clone_min=2"
+do_test clone_min_stop_one "order first clone then primitive... first clone_min=2"
+do_test clone_min_stop_two "order first clone then primitive... first clone_min=2"
+
do_test simple1 "Offline "
do_test simple2 "Start "
do_test simple3 "Start 2 "
diff --git a/pengine/test10/bug-5186-partial-migrate.dot b/pengine/test10/bug-5186-partial-migrate.dot
index 033d41d..65f5616 100644
--- a/pengine/test10/bug-5186-partial-migrate.dot
+++ b/pengine/test10/bug-5186-partial-migrate.dot
@@ -66,13 +66,10 @@
"stonith 'reboot' bl460g1n7" -> "clnDiskd1_stop_0" [ style = bold]
"stonith 'reboot' bl460g1n7" -> "clnDiskd2_stop_0" [ style = bold]
"stonith 'reboot' bl460g1n7" -> "clnPing_stop_0" [ style = bold]
-"stonith 'reboot' bl460g1n7" -> "grpStonith8_stop_0" [ style = bold]
"stonith 'reboot' bl460g1n7" -> "prmDiskd1_stop_0 bl460g1n7" [ style = bold]
"stonith 'reboot' bl460g1n7" -> "prmDiskd2_stop_0 bl460g1n7" [ style = bold]
"stonith 'reboot' bl460g1n7" -> "prmDummy_stop_0 bl460g1n7" [ style = bold]
"stonith 'reboot' bl460g1n7" -> "prmPing_stop_0 bl460g1n7" [ style = bold]
-"stonith 'reboot' bl460g1n7" -> "prmStonith8-1_stop_0 bl460g1n7" [ style = bold]
-"stonith 'reboot' bl460g1n7" -> "prmStonith8-2_stop_0 bl460g1n7" [ style = bold]
"stonith 'reboot' bl460g1n7" -> "prmVM2_stop_0 bl460g1n7" [ style = bold]
"stonith 'reboot' bl460g1n7" -> "stonith_complete" [ style = bold]
"stonith 'reboot' bl460g1n7" [ style=bold color="green" fontcolor="black"]
diff --git a/pengine/test10/bug-5186-partial-migrate.exp b/pengine/test10/bug-5186-partial-migrate.exp
index 216d962..bc058ea 100644
--- a/pengine/test10/bug-5186-partial-migrate.exp
+++ b/pengine/test10/bug-5186-partial-migrate.exp
@@ -104,11 +104,7 @@
<attributes CRM_meta_record_pending="true" CRM_meta_timeout="20000" />
</pseudo_event>
</action_set>
- <inputs>
- <trigger>
- <crm_event id="75" operation="stonith" operation_key="stonith-bl460g1n7-reboot" on_node="bl460g1n7" on_node_uuid="3232261593"/>
- </trigger>
- </inputs>
+ <inputs/>
</synapse>
<synapse id="8">
<action_set>
@@ -182,9 +178,6 @@
<trigger>
<pseudo_event id="45" operation="stop" operation_key="grpStonith8_stop_0"/>
</trigger>
- <trigger>
- <crm_event id="75" operation="stonith" operation_key="stonith-bl460g1n7-reboot" on_node="bl460g1n7" on_node_uuid="3232261593"/>
- </trigger>
</inputs>
</synapse>
<synapse id="13">
@@ -229,9 +222,6 @@
<trigger>
<pseudo_event id="45" operation="stop" operation_key="grpStonith8_stop_0"/>
</trigger>
- <trigger>
- <crm_event id="75" operation="stonith" operation_key="stonith-bl460g1n7-reboot" on_node="bl460g1n7" on_node_uuid="3232261593"/>
- </trigger>
</inputs>
</synapse>
<synapse id="16">
diff --git a/pengine/test10/bug-5186-partial-migrate.summary b/pengine/test10/bug-5186-partial-migrate.summary
index f848c97..5e62a23 100644
--- a/pengine/test10/bug-5186-partial-migrate.summary
+++ b/pengine/test10/bug-5186-partial-migrate.summary
@@ -35,18 +35,22 @@ Transition Summary:
Executing cluster transition:
* Resource action: prmVM2 stop on bl460g1n6
+ * Pseudo action: grpStonith8_stop_0
+ * Pseudo action: prmStonith8-2_stop_0
* Fencing bl460g1n7 (reboot)
* Pseudo action: stonith_complete
* Pseudo action: prmDummy_stop_0
* Pseudo action: prmVM2_stop_0
- * Pseudo action: grpStonith8_stop_0
- * Pseudo action: prmStonith8-2_stop_0
+ * Pseudo action: prmStonith8-1_stop_0
* Pseudo action: clnDiskd1_stop_0
* Pseudo action: clnDiskd2_stop_0
* Pseudo action: clnPing_stop_0
* Resource action: prmDummy start on bl460g1n6
* Resource action: prmVM2 start on bl460g1n8
- * Pseudo action: prmStonith8-1_stop_0
+ * Pseudo action: grpStonith8_stopped_0
+ * Pseudo action: grpStonith8_start_0
+ * Resource action: prmStonith8-1 start on bl460g1n6
+ * Resource action: prmStonith8-2 start on bl460g1n6
* Pseudo action: prmDiskd1_stop_0
* Pseudo action: clnDiskd1_stopped_0
* Pseudo action: prmDiskd2_stop_0
@@ -55,10 +59,6 @@ Executing cluster transition:
* Pseudo action: clnPing_stopped_0
* Pseudo action: all_stopped
* Resource action: prmVM2 monitor=10000 on bl460g1n8
- * Pseudo action: grpStonith8_stopped_0
- * Pseudo action: grpStonith8_start_0
- * Resource action: prmStonith8-1 start on bl460g1n6
- * Resource action: prmStonith8-2 start on bl460g1n6
* Pseudo action: grpStonith8_running_0
* Resource action: prmStonith8-1 monitor=10000 on bl460g1n6
* Resource action: prmStonith8-2 monitor=3600000 on bl460g1n6
diff --git a/pengine/test10/bug-lf-2551.dot b/pengine/test10/bug-lf-2551.dot
index ed80e15..18bca44 100644
--- a/pengine/test10/bug-lf-2551.dot
+++ b/pengine/test10/bug-lf-2551.dot
@@ -56,7 +56,6 @@ digraph "g" {
"stonith 'reboot' hex-9" -> "cmirrord:3_stop_0 hex-9" [ style = bold]
"stonith 'reboot' hex-9" -> "dlm:3_stop_0 hex-9" [ style = bold]
"stonith 'reboot' hex-9" -> "dummy1_stop_0 hex-9" [ style = bold]
-"stonith 'reboot' hex-9" -> "fencing-sbd_stop_0 hex-9" [ style = bold]
"stonith 'reboot' hex-9" -> "o2cb:3_stop_0 hex-9" [ style = bold]
"stonith 'reboot' hex-9" -> "ocfs2-1:3_stop_0 hex-9" [ style = bold]
"stonith 'reboot' hex-9" -> "stonith_complete" [ style = bold]
diff --git a/pengine/test10/bug-lf-2551.exp b/pengine/test10/bug-lf-2551.exp
index 0af9010..d6266e1 100644
--- a/pengine/test10/bug-lf-2551.exp
+++ b/pengine/test10/bug-lf-2551.exp
@@ -18,11 +18,7 @@
<attributes CRM_meta_record_pending="false" CRM_meta_timeout="240000" sbd_device="/dev/disk/by-id/dm-uuid-part1-mpath-3600508b400105b5a0001500000250000"/>
</pseudo_event>
</action_set>
- <inputs>
- <trigger>
- <crm_event id="249" operation="stonith" operation_key="stonith-hex-9-reboot" on_node="hex-9" on_node_uuid="hex-9"/>
- </trigger>
- </inputs>
+ <inputs/>
</synapse>
<synapse id="2">
<action_set>
diff --git a/pengine/test10/bug-lf-2551.summary b/pengine/test10/bug-lf-2551.summary
index f8d861c..158eb73 100644
--- a/pengine/test10/bug-lf-2551.summary
+++ b/pengine/test10/bug-lf-2551.summary
@@ -107,6 +107,7 @@ Transition Summary:
* Stop vm-61 (hex-9)
Executing cluster transition:
+ * Pseudo action: fencing-sbd_stop_0
* Resource action: dummy1 monitor=300000 on hex-8
* Resource action: dummy1 monitor=300000 on hex-7
* Fencing hex-9 (reboot)
@@ -114,7 +115,7 @@ Executing cluster transition:
* Pseudo action: load_stopped_hex-8
* Pseudo action: load_stopped_hex-7
* Pseudo action: load_stopped_hex-0
- * Pseudo action: fencing-sbd_stop_0
+ * Resource action: fencing-sbd start on hex-0
* Pseudo action: dummy1_stop_0
* Pseudo action: vm-03_stop_0
* Pseudo action: vm-06_stop_0
@@ -133,7 +134,6 @@ Executing cluster transition:
* Pseudo action: vm-57_stop_0
* Pseudo action: vm-61_stop_0
* Pseudo action: load_stopped_hex-9
- * Resource action: fencing-sbd start on hex-0
* Resource action: dummy1 start on hex-0
* Pseudo action: base-clone_stop_0
* Resource action: dummy1 monitor=30000 on hex-0
diff --git a/pengine/test10/clone_min_interleave_start_one.dot b/pengine/test10/clone_min_interleave_start_one.dot
new file mode 100644
index 0000000..15ac9be
--- /dev/null
+++ b/pengine/test10/clone_min_interleave_start_one.dot
@@ -0,0 +1,50 @@
+ digraph "g" {
+"FAKE1-clone_running_0" [ style=bold color="green" fontcolor="orange"]
+"FAKE1-clone_start_0" -> "FAKE1-clone_running_0" [ style = bold]
+"FAKE1-clone_start_0" -> "FAKE1_start_0 c7auto1" [ style = bold]
+"FAKE1-clone_start_0" [ style=bold color="green" fontcolor="orange"]
+"FAKE1_monitor_10000 c7auto1" [ style=bold color="green" fontcolor="black"]
+"FAKE1_start_0 c7auto1" -> "FAKE1-clone_running_0" [ style = bold]
+"FAKE1_start_0 c7auto1" -> "FAKE1_monitor_10000 c7auto1" [ style = bold]
+"FAKE1_start_0 c7auto1" [ style=bold color="green" fontcolor="black"]
+"FAKE2-clone_running_0" -> "FAKE3-clone_start_0" [ style = dashed]
+"FAKE2-clone_running_0" [ style=dashed color="red" fontcolor="orange"]
+"FAKE2-clone_start_0" -> "FAKE2-clone_running_0" [ style = dashed]
+"FAKE2-clone_start_0" -> "FAKE2:1_start_0 c7auto3" [ style = dashed]
+"FAKE2-clone_start_0" -> "FAKE2:2_start_0 c7auto1" [ style = dashed]
+"FAKE2-clone_start_0" -> "FAKE2_start_0 c7auto2" [ style = dashed]
+"FAKE2-clone_start_0" [ style=dashed color="red" fontcolor="orange"]
+"FAKE2:1_monitor_10000 c7auto3" [ style=dashed color="red" fontcolor="black"]
+"FAKE2:1_start_0 c7auto3" -> "FAKE2-clone_running_0" [ style = dashed]
+"FAKE2:1_start_0 c7auto3" -> "FAKE2:1_monitor_10000 c7auto3" [ style = dashed]
+"FAKE2:1_start_0 c7auto3" -> "FAKE3:1_start_0 c7auto3" [ style = dashed]
+"FAKE2:1_start_0 c7auto3" [ style=dashed color="red" fontcolor="black"]
+"FAKE2:2_monitor_10000 c7auto1" [ style=dashed color="red" fontcolor="black"]
+"FAKE2:2_start_0 c7auto1" -> "FAKE2-clone_running_0" [ style = dashed]
+"FAKE2:2_start_0 c7auto1" -> "FAKE2:2_monitor_10000 c7auto1" [ style = dashed]
+"FAKE2:2_start_0 c7auto1" -> "FAKE3:2_start_0 c7auto1" [ style = dashed]
+"FAKE2:2_start_0 c7auto1" [ style=dashed color="red" fontcolor="black"]
+"FAKE2_monitor_10000 c7auto2" [ style=dashed color="red" fontcolor="black"]
+"FAKE2_start_0 c7auto2" -> "FAKE2-clone_running_0" [ style = dashed]
+"FAKE2_start_0 c7auto2" -> "FAKE2_monitor_10000 c7auto2" [ style = dashed]
+"FAKE2_start_0 c7auto2" -> "FAKE3_start_0 c7auto2" [ style = dashed]
+"FAKE2_start_0 c7auto2" [ style=dashed color="red" fontcolor="black"]
+"FAKE3-clone_running_0" [ style=dashed color="red" fontcolor="orange"]
+"FAKE3-clone_start_0" -> "FAKE3-clone_running_0" [ style = dashed]
+"FAKE3-clone_start_0" -> "FAKE3:1_start_0 c7auto3" [ style = dashed]
+"FAKE3-clone_start_0" -> "FAKE3:2_start_0 c7auto1" [ style = dashed]
+"FAKE3-clone_start_0" -> "FAKE3_start_0 c7auto2" [ style = dashed]
+"FAKE3-clone_start_0" [ style=dashed color="red" fontcolor="orange"]
+"FAKE3:1_monitor_10000 c7auto3" [ style=dashed color="red" fontcolor="black"]
+"FAKE3:1_start_0 c7auto3" -> "FAKE3-clone_running_0" [ style = dashed]
+"FAKE3:1_start_0 c7auto3" -> "FAKE3:1_monitor_10000 c7auto3" [ style = dashed]
+"FAKE3:1_start_0 c7auto3" [ style=dashed color="red" fontcolor="black"]
+"FAKE3:2_monitor_10000 c7auto1" [ style=dashed color="red" fontcolor="black"]
+"FAKE3:2_start_0 c7auto1" -> "FAKE3-clone_running_0" [ style = dashed]
+"FAKE3:2_start_0 c7auto1" -> "FAKE3:2_monitor_10000 c7auto1" [ style = dashed]
+"FAKE3:2_start_0 c7auto1" [ style=dashed color="red" fontcolor="black"]
+"FAKE3_monitor_10000 c7auto2" [ style=dashed color="red" fontcolor="black"]
+"FAKE3_start_0 c7auto2" -> "FAKE3-clone_running_0" [ style = dashed]
+"FAKE3_start_0 c7auto2" -> "FAKE3_monitor_10000 c7auto2" [ style = dashed]
+"FAKE3_start_0 c7auto2" [ style=dashed color="red" fontcolor="black"]
+}
diff --git a/pengine/test10/clone_min_interleave_start_one.exp b/pengine/test10/clone_min_interleave_start_one.exp
new file mode 100644
index 0000000..b6e0c5d
--- /dev/null
+++ b/pengine/test10/clone_min_interleave_start_one.exp
@@ -0,0 +1,51 @@
+<transition_graph cluster-delay="60s" stonith-timeout="60s" failed-stop-offset="INFINITY" failed-start-offset="INFINITY" transition_id="0">
+ <synapse id="0">
+ <action_set>
+ <rsc_op id="11" operation="monitor" operation_key="FAKE1_monitor_10000" internal_operation_key="FAKE1:0_monitor_10000" on_node="c7auto1" on_node_uuid="1">
+ <primitive id="FAKE1" long-id="FAKE1:0" class="ocf" provider="heartbeat" type="Dummy"/>
+ <attributes CRM_meta_clone="0" CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_interval="10000" CRM_meta_name="monitor" CRM_meta_notify="false" CRM_meta_timeout="20000" />
+ </rsc_op>
+ </action_set>
+ <inputs>
+ <trigger>
+ <rsc_op id="10" operation="start" operation_key="FAKE1_start_0" internal_operation_key="FAKE1:0_start_0" on_node="c7auto1" on_node_uuid="1"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="1">
+ <action_set>
+ <rsc_op id="10" operation="start" operation_key="FAKE1_start_0" internal_operation_key="FAKE1:0_start_0" on_node="c7auto1" on_node_uuid="1">
+ <primitive id="FAKE1" long-id="FAKE1:0" class="ocf" provider="heartbeat" type="Dummy"/>
+ <attributes CRM_meta_clone="0" CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_name="start" CRM_meta_notify="false" CRM_meta_timeout="20000" />
+ </rsc_op>
+ </action_set>
+ <inputs>
+ <trigger>
+ <pseudo_event id="12" operation="start" operation_key="FAKE1-clone_start_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="2" priority="1000000">
+ <action_set>
+ <pseudo_event id="13" operation="running" operation_key="FAKE1-clone_running_0">
+ <attributes CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="false" CRM_meta_timeout="20000" />
+ </pseudo_event>
+ </action_set>
+ <inputs>
+ <trigger>
+ <rsc_op id="10" operation="start" operation_key="FAKE1_start_0" internal_operation_key="FAKE1:0_start_0" on_node="c7auto1" on_node_uuid="1"/>
+ </trigger>
+ <trigger>
+ <pseudo_event id="12" operation="start" operation_key="FAKE1-clone_start_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="3">
+ <action_set>
+ <pseudo_event id="12" operation="start" operation_key="FAKE1-clone_start_0">
+ <attributes CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="false" CRM_meta_timeout="20000" />
+ </pseudo_event>
+ </action_set>
+ <inputs/>
+ </synapse>
+</transition_graph>
diff --git a/pengine/test10/clone_min_interleave_start_one.scores b/pengine/test10/clone_min_interleave_start_one.scores
new file mode 100644
index 0000000..03de018
--- /dev/null
+++ b/pengine/test10/clone_min_interleave_start_one.scores
@@ -0,0 +1,67 @@
+Allocation scores:
+clone_color: FAKE1-clone allocation score on c7auto1: 0
+clone_color: FAKE1-clone allocation score on c7auto2: -INFINITY
+clone_color: FAKE1-clone allocation score on c7auto3: -INFINITY
+clone_color: FAKE1:0 allocation score on c7auto1: 0
+clone_color: FAKE1:0 allocation score on c7auto2: -INFINITY
+clone_color: FAKE1:0 allocation score on c7auto3: -INFINITY
+clone_color: FAKE1:1 allocation score on c7auto1: 0
+clone_color: FAKE1:1 allocation score on c7auto2: -INFINITY
+clone_color: FAKE1:1 allocation score on c7auto3: -INFINITY
+clone_color: FAKE1:2 allocation score on c7auto1: 0
+clone_color: FAKE1:2 allocation score on c7auto2: -INFINITY
+clone_color: FAKE1:2 allocation score on c7auto3: -INFINITY
+clone_color: FAKE2-clone allocation score on c7auto1: 0
+clone_color: FAKE2-clone allocation score on c7auto2: 0
+clone_color: FAKE2-clone allocation score on c7auto3: 0
+clone_color: FAKE2:0 allocation score on c7auto1: 0
+clone_color: FAKE2:0 allocation score on c7auto2: 0
+clone_color: FAKE2:0 allocation score on c7auto3: 0
+clone_color: FAKE2:1 allocation score on c7auto1: 0
+clone_color: FAKE2:1 allocation score on c7auto2: 0
+clone_color: FAKE2:1 allocation score on c7auto3: 0
+clone_color: FAKE2:2 allocation score on c7auto1: 0
+clone_color: FAKE2:2 allocation score on c7auto2: 0
+clone_color: FAKE2:2 allocation score on c7auto3: 0
+clone_color: FAKE3-clone allocation score on c7auto1: 0
+clone_color: FAKE3-clone allocation score on c7auto2: 0
+clone_color: FAKE3-clone allocation score on c7auto3: 0
+clone_color: FAKE3:0 allocation score on c7auto1: 0
+clone_color: FAKE3:0 allocation score on c7auto2: 0
+clone_color: FAKE3:0 allocation score on c7auto3: 0
+clone_color: FAKE3:1 allocation score on c7auto1: 0
+clone_color: FAKE3:1 allocation score on c7auto2: 0
+clone_color: FAKE3:1 allocation score on c7auto3: 0
+clone_color: FAKE3:2 allocation score on c7auto1: 0
+clone_color: FAKE3:2 allocation score on c7auto2: 0
+clone_color: FAKE3:2 allocation score on c7auto3: 0
+native_color: FAKE1:0 allocation score on c7auto1: 0
+native_color: FAKE1:0 allocation score on c7auto2: -INFINITY
+native_color: FAKE1:0 allocation score on c7auto3: -INFINITY
+native_color: FAKE1:1 allocation score on c7auto1: -INFINITY
+native_color: FAKE1:1 allocation score on c7auto2: -INFINITY
+native_color: FAKE1:1 allocation score on c7auto3: -INFINITY
+native_color: FAKE1:2 allocation score on c7auto1: -INFINITY
+native_color: FAKE1:2 allocation score on c7auto2: -INFINITY
+native_color: FAKE1:2 allocation score on c7auto3: -INFINITY
+native_color: FAKE2:0 allocation score on c7auto1: 0
+native_color: FAKE2:0 allocation score on c7auto2: 0
+native_color: FAKE2:0 allocation score on c7auto3: 0
+native_color: FAKE2:1 allocation score on c7auto1: 0
+native_color: FAKE2:1 allocation score on c7auto2: -INFINITY
+native_color: FAKE2:1 allocation score on c7auto3: 0
+native_color: FAKE2:2 allocation score on c7auto1: 0
+native_color: FAKE2:2 allocation score on c7auto2: -INFINITY
+native_color: FAKE2:2 allocation score on c7auto3: -INFINITY
+native_color: FAKE3:0 allocation score on c7auto1: 0
+native_color: FAKE3:0 allocation score on c7auto2: 0
+native_color: FAKE3:0 allocation score on c7auto3: 0
+native_color: FAKE3:1 allocation score on c7auto1: 0
+native_color: FAKE3:1 allocation score on c7auto2: -INFINITY
+native_color: FAKE3:1 allocation score on c7auto3: 0
+native_color: FAKE3:2 allocation score on c7auto1: 0
+native_color: FAKE3:2 allocation score on c7auto2: -INFINITY
+native_color: FAKE3:2 allocation score on c7auto3: -INFINITY
+native_color: shooter allocation score on c7auto1: 0
+native_color: shooter allocation score on c7auto2: 0
+native_color: shooter allocation score on c7auto3: 0
diff --git a/pengine/test10/clone_min_interleave_start_one.summary b/pengine/test10/clone_min_interleave_start_one.summary
new file mode 100644
index 0000000..b15f68a
--- /dev/null
+++ b/pengine/test10/clone_min_interleave_start_one.summary
@@ -0,0 +1,39 @@
+
+Current cluster status:
+Online: [ c7auto1 c7auto2 c7auto3 ]
+
+ shooter (stonith:fence_phd_kvm): Started c7auto1
+ Clone Set: FAKE1-clone [FAKE1]
+ Stopped: [ c7auto1 c7auto2 c7auto3 ]
+ Clone Set: FAKE2-clone [FAKE2]
+ Stopped: [ c7auto1 c7auto2 c7auto3 ]
+ Clone Set: FAKE3-clone [FAKE3]
+ Stopped: [ c7auto1 c7auto2 c7auto3 ]
+
+Transition Summary:
+ * Start FAKE1:0 (c7auto1)
+ * Start FAKE2:0 (c7auto2 - blocked)
+ * Start FAKE2:1 (c7auto3 - blocked)
+ * Start FAKE2:2 (c7auto1 - blocked)
+ * Start FAKE3:0 (c7auto2 - blocked)
+ * Start FAKE3:1 (c7auto3 - blocked)
+ * Start FAKE3:2 (c7auto1 - blocked)
+
+Executing cluster transition:
+ * Pseudo action: FAKE1-clone_start_0
+ * Resource action: FAKE1 start on c7auto1
+ * Pseudo action: FAKE1-clone_running_0
+ * Resource action: FAKE1 monitor=10000 on c7auto1
+
+Revised cluster status:
+Online: [ c7auto1 c7auto2 c7auto3 ]
+
+ shooter (stonith:fence_phd_kvm): Started c7auto1
+ Clone Set: FAKE1-clone [FAKE1]
+ Started: [ c7auto1 ]
+ Stopped: [ c7auto2 c7auto3 ]
+ Clone Set: FAKE2-clone [FAKE2]
+ Stopped: [ c7auto1 c7auto2 c7auto3 ]
+ Clone Set: FAKE3-clone [FAKE3]
+ Stopped: [ c7auto1 c7auto2 c7auto3 ]
+
diff --git a/pengine/test10/clone_min_interleave_start_one.xml b/pengine/test10/clone_min_interleave_start_one.xml
new file mode 100644
index 0000000..fbe99de
--- /dev/null
+++ b/pengine/test10/clone_min_interleave_start_one.xml
@@ -0,0 +1,155 @@
+<cib crm_feature_set="3.0.10" validate-with="pacemaker-2.3" epoch="15" num_updates="0" admin_epoch="0" cib-last-written="Fri Jul 17 11:44:12 2015" update-origin="c7auto1" update-client="crm_resource" update-user="root" have-quorum="1" dc-uuid="3">
+ <configuration>
+ <crm_config>
+ <cluster_property_set id="cib-bootstrap-options">
+ <nvpair id="cib-bootstrap-options-have-watchdog" name="have-watchdog" value="false"/>
+ <nvpair id="cib-bootstrap-options-dc-version" name="dc-version" value="1.1.12-ef86b49"/>
+ <nvpair id="cib-bootstrap-options-cluster-infrastructure" name="cluster-infrastructure" value="corosync"/>
+ <nvpair id="cib-bootstrap-options-cluster-name" name="cluster-name" value="phd"/>
+ </cluster_property_set>
+ </crm_config>
+ <nodes>
+ <node id="1" uname="c7auto1"/>
+ <node id="2" uname="c7auto2"/>
+ <node id="3" uname="c7auto3"/>
+ </nodes>
+ <resources>
+ <primitive class="stonith" id="shooter" type="fence_phd_kvm">
+ <instance_attributes id="shooter-instance_attributes"/>
+ <operations>
+ <op id="shooter-monitor-interval-60s" interval="60s" name="monitor"/>
+ </operations>
+ </primitive>
+ <clone id="FAKE1-clone">
+ <primitive class="ocf" id="FAKE1" provider="heartbeat" type="Dummy">
+ <instance_attributes id="FAKE1-instance_attributes"/>
+ <operations>
+ <op id="FAKE1-start-timeout-20" interval="0s" name="start" timeout="20"/>
+ <op id="FAKE1-stop-timeout-20" interval="0s" name="stop" timeout="20"/>
+ <op id="FAKE1-monitor-interval-10" interval="10" name="monitor" timeout="20"/>
+ </operations>
+ </primitive>
+ <meta_attributes id="FAKE1-clone-meta">
+ <nvpair id="FAKE1-clone-min" name="clone-min" value="2"/>
+ </meta_attributes>
+ <meta_attributes id="FAKE1-clone-meta_attributes"/>
+ </clone>
+ <clone id="FAKE2-clone">
+ <primitive class="ocf" id="FAKE2" provider="heartbeat" type="Dummy">
+ <instance_attributes id="FAKE2-instance_attributes"/>
+ <operations>
+ <op id="FAKE2-start-timeout-20" interval="0s" name="start" timeout="20"/>
+ <op id="FAKE2-stop-timeout-20" interval="0s" name="stop" timeout="20"/>
+ <op id="FAKE2-monitor-interval-10" interval="10" name="monitor" timeout="20"/>
+ </operations>
+ </primitive>
+ <meta_attributes id="FAKE2-clone-meta">
+ <nvpair id="FAKE2-interleave" name="interleave" value="true"/>
+ </meta_attributes>
+ </clone>
+ <clone id="FAKE3-clone">
+ <primitive class="ocf" id="FAKE3" provider="heartbeat" type="Dummy">
+ <instance_attributes id="FAKE3-instance_attributes"/>
+ <operations>
+ <op id="FAKE3-start-timeout-20" interval="0s" name="start" timeout="20"/>
+ <op id="FAKE3-stop-timeout-20" interval="0s" name="stop" timeout="20"/>
+ <op id="FAKE3-monitor-interval-10" interval="10" name="monitor" timeout="20"/>
+ </operations>
+ </primitive>
+ <meta_attributes id="FAKE3-clone-meta">
+ <nvpair id="FAKE3-interleave" name="interleave" value="true"/>
+ </meta_attributes>
+ </clone>
+ </resources>
+ <constraints>
+ <rsc_order first="FAKE1-clone" first-action="start" id="order-FAKE1-clone-FAKE2-clone-mandatory" then="FAKE2-clone" then-action="start"/>
+ <rsc_order first="FAKE2-clone" first-action="start" id="order-FAKE2-clone-FAKE3-clone-mandatory" then="FAKE3-clone" then-action="start"/>
+ <rsc_location id="cli-ban-FAKE1-clone-on-c7auto3" rsc="FAKE1-clone" role="Started" node="c7auto3" score="-INFINITY"/>
+ <rsc_location id="cli-ban-FAKE1-clone-on-c7auto2" rsc="FAKE1-clone" role="Started" node="c7auto2" score="-INFINITY"/>
+ </constraints>
+ </configuration>
+ <status>
+ <node_state id="3" uname="c7auto3" in_ccm="true" crmd="online" crm-debug-origin="do_update_resource" join="member" expected="member">
+ <lrm id="3">
+ <lrm_resources>
+ <lrm_resource id="shooter" type="fence_phd_kvm" class="stonith">
+ <lrm_rsc_op id="shooter_last_0" operation_key="shooter_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="8:4:7:9d56c099-7067-4908-9f99-89a103eff57b" transition-magic="0:7;8:4:7:9d56c099-7067-4908-9f99-89a103eff57b" on_node="c7auto3" call-id="5" rc-code="7" op-status="0" interval="0" last-run="1437146780" last-rc-change="1437146780" exec-time="998" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ </lrm_resource>
+ <lrm_resource id="FAKE1" type="Dummy" class="ocf" provider="heartbeat">
+ <lrm_rsc_op id="FAKE1_last_0" operation_key="FAKE1_stop_0" operation="stop" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="18:10:0:9d56c099-7067-4908-9f99-89a103eff57b" transition-magic="0:0;18:10:0:9d56c099-7067-4908-9f99-89a103eff57b" on_node="c7auto3" call-id="32" rc-code="0" op-status="0" interval="0" last-run="1437147852" last-rc-change="1437147852" exec-time="8" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" fake " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ <lrm_rsc_op id="FAKE1_monitor_10000" operation_key="FAKE1_monitor_10000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="15:5:0:9d56c099-7067-4908-9f99-89a103eff57b" transition-magic="0:0;15:5:0:9d56c099-7067-4908-9f99-89a103eff57b" on_node="c7auto3" call-id="12" rc-code="0" op-status="0" interval="10000" last-rc-change="1437147006" exec-time="9" queue-time="0" op-digest="4811cef7f7f94e3a35a70be7916cb2fd"/>
+ </lrm_resource>
+ <lrm_resource id="FAKE2" type="Dummy" class="ocf" provider="heartbeat">
+ <lrm_rsc_op id="FAKE2_last_0" operation_key="FAKE2_stop_0" operation="stop" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="25:10:0:9d56c099-7067-4908-9f99-89a103eff57b" transition-magic="0:0;25:10:0:9d56c099-7067-4908-9f99-89a103eff57b" on_node="c7auto3" call-id="30" rc-code="0" op-status="0" interval="0" last-run="1437147852" last-rc-change="1437147852" exec-time="8" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" fake " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ <lrm_rsc_op id="FAKE2_monitor_10000" operation_key="FAKE2_monitor_10000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="28:6:0:9d56c099-7067-4908-9f99-89a103eff57b" transition-magic="0:0;28:6:0:9d56c099-7067-4908-9f99-89a103eff57b" on_node="c7auto3" call-id="19" rc-code="0" op-status="0" interval="10000" last-rc-change="1437147019" exec-time="8" queue-time="0" op-digest="4811cef7f7f94e3a35a70be7916cb2fd"/>
+ </lrm_resource>
+ <lrm_resource id="FAKE3" type="Dummy" class="ocf" provider="heartbeat">
+ <lrm_rsc_op id="FAKE3_last_0" operation_key="FAKE3_stop_0" operation="stop" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="35:10:0:9d56c099-7067-4908-9f99-89a103eff57b" transition-magic="0:0;35:10:0:9d56c099-7067-4908-9f99-89a103eff57b" on_node="c7auto3" call-id="28" rc-code="0" op-status="0" interval="0" last-run="1437147852" last-rc-change="1437147852" exec-time="10" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" fake " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ <lrm_rsc_op id="FAKE3_monitor_10000" operation_key="FAKE3_monitor_10000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="41:7:0:9d56c099-7067-4908-9f99-89a103eff57b" transition-magic="0:0;41:7:0:9d56c099-7067-4908-9f99-89a103eff57b" on_node="c7auto3" call-id="26" rc-code="0" op-status="0" interval="10000" last-rc-change="1437147024" exec-time="8" queue-time="0" op-digest="4811cef7f7f94e3a35a70be7916cb2fd"/>
+ </lrm_resource>
+ </lrm_resources>
+ </lrm>
+ <transient_attributes id="3">
+ <instance_attributes id="status-3">
+ <nvpair id="status-3-shutdown" name="shutdown" value="0"/>
+ <nvpair id="status-3-probe_complete" name="probe_complete" value="true"/>
+ </instance_attributes>
+ </transient_attributes>
+ </node_state>
+ <node_state id="1" uname="c7auto1" in_ccm="true" crmd="online" crm-debug-origin="do_update_resource" join="member" expected="member">
+ <lrm id="1">
+ <lrm_resources>
+ <lrm_resource id="shooter" type="fence_phd_kvm" class="stonith">
+ <lrm_rsc_op id="shooter_last_0" operation_key="shooter_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="9:4:0:9d56c099-7067-4908-9f99-89a103eff57b" transition-magic="0:0;9:4:0:9d56c099-7067-4908-9f99-89a103eff57b" on_node="c7auto1" call-id="6" rc-code="0" op-status="0" interval="0" last-run="1437146782" last-rc-change="1437146782" exec-time="5" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ <lrm_rsc_op id="shooter_monitor_60000" operation_key="shooter_monitor_60000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="10:4:0:9d56c099-7067-4908-9f99-89a103eff57b" transition-magic="0:0;10:4:0:9d56c099-7067-4908-9f99-89a103eff57b" on_node="c7auto1" call-id="7" rc-code="0" op-status="0" interval="60000" last-rc-change="1437146783" exec-time="5" queue-time="1" op-digest="4811cef7f7f94e3a35a70be7916cb2fd"/>
+ </lrm_resource>
+ <lrm_resource id="FAKE1" type="Dummy" class="ocf" provider="heartbeat">
+ <lrm_rsc_op id="FAKE1_last_0" operation_key="FAKE1_stop_0" operation="stop" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="19:10:0:9d56c099-7067-4908-9f99-89a103eff57b" transition-magic="0:0;19:10:0:9d56c099-7067-4908-9f99-89a103eff57b" on_node="c7auto1" call-id="34" rc-code="0" op-status="0" interval="0" last-run="1437147852" last-rc-change="1437147852" exec-time="7" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" fake " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ <lrm_rsc_op id="FAKE1_monitor_10000" operation_key="FAKE1_monitor_10000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="17:5:0:9d56c099-7067-4908-9f99-89a103eff57b" transition-magic="0:0;17:5:0:9d56c099-7067-4908-9f99-89a103eff57b" on_node="c7auto1" call-id="14" rc-code="0" op-status="0" interval="10000" last-rc-change="1437147007" exec-time="6" queue-time="0" op-digest="4811cef7f7f94e3a35a70be7916cb2fd"/>
+ </lrm_resource>
+ <lrm_resource id="FAKE2" type="Dummy" class="ocf" provider="heartbeat">
+ <lrm_rsc_op id="FAKE2_last_0" operation_key="FAKE2_stop_0" operation="stop" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="27:10:0:9d56c099-7067-4908-9f99-89a103eff57b" transition-magic="0:0;27:10:0:9d56c099-7067-4908-9f99-89a103eff57b" on_node="c7auto1" call-id="32" rc-code="0" op-status="0" interval="0" last-run="1437147852" last-rc-change="1437147852" exec-time="8" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" fake " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ <lrm_rsc_op id="FAKE2_monitor_10000" operation_key="FAKE2_monitor_10000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="30:6:0:9d56c099-7067-4908-9f99-89a103eff57b" transition-magic="0:0;30:6:0:9d56c099-7067-4908-9f99-89a103eff57b" on_node="c7auto1" call-id="21" rc-code="0" op-status="0" interval="10000" last-rc-change="1437147019" exec-time="7" queue-time="0" op-digest="4811cef7f7f94e3a35a70be7916cb2fd"/>
+ </lrm_resource>
+ <lrm_resource id="FAKE3" type="Dummy" class="ocf" provider="heartbeat">
+ <lrm_rsc_op id="FAKE3_last_0" operation_key="FAKE3_stop_0" operation="stop" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="37:10:0:9d56c099-7067-4908-9f99-89a103eff57b" transition-magic="0:0;37:10:0:9d56c099-7067-4908-9f99-89a103eff57b" on_node="c7auto1" call-id="30" rc-code="0" op-status="0" interval="0" last-run="1437147852" last-rc-change="1437147852" exec-time="7" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" fake " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ <lrm_rsc_op id="FAKE3_monitor_10000" operation_key="FAKE3_monitor_10000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="43:7:0:9d56c099-7067-4908-9f99-89a103eff57b" transition-magic="0:0;43:7:0:9d56c099-7067-4908-9f99-89a103eff57b" on_node="c7auto1" call-id="28" rc-code="0" op-status="0" interval="10000" last-rc-change="1437147024" exec-time="6" queue-time="1" op-digest="4811cef7f7f94e3a35a70be7916cb2fd"/>
+ </lrm_resource>
+ </lrm_resources>
+ </lrm>
+ <transient_attributes id="1">
+ <instance_attributes id="status-1">
+ <nvpair id="status-1-shutdown" name="shutdown" value="0"/>
+ <nvpair id="status-1-probe_complete" name="probe_complete" value="true"/>
+ </instance_attributes>
+ </transient_attributes>
+ </node_state>
+ <node_state id="2" uname="c7auto2" in_ccm="true" crmd="online" crm-debug-origin="do_update_resource" join="member" expected="member">
+ <lrm id="2">
+ <lrm_resources>
+ <lrm_resource id="shooter" type="fence_phd_kvm" class="stonith">
+ <lrm_rsc_op id="shooter_last_0" operation_key="shooter_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="6:4:7:9d56c099-7067-4908-9f99-89a103eff57b" transition-magic="0:7;6:4:7:9d56c099-7067-4908-9f99-89a103eff57b" on_node="c7auto2" call-id="5" rc-code="7" op-status="0" interval="0" last-run="1437146780" last-rc-change="1437146780" exec-time="994" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ </lrm_resource>
+ <lrm_resource id="FAKE1" type="Dummy" class="ocf" provider="heartbeat">
+ <lrm_rsc_op id="FAKE1_last_0" operation_key="FAKE1_stop_0" operation="stop" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="20:10:0:9d56c099-7067-4908-9f99-89a103eff57b" transition-magic="0:0;20:10:0:9d56c099-7067-4908-9f99-89a103eff57b" on_node="c7auto2" call-id="32" rc-code="0" op-status="0" interval="0" last-run="1437147852" last-rc-change="1437147852" exec-time="7" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" fake " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ <lrm_rsc_op id="FAKE1_monitor_10000" operation_key="FAKE1_monitor_10000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="13:5:0:9d56c099-7067-4908-9f99-89a103eff57b" transition-magic="0:0;13:5:0:9d56c099-7067-4908-9f99-89a103eff57b" on_node="c7auto2" call-id="12" rc-code="0" op-status="0" interval="10000" last-rc-change="1437147007" exec-time="8" queue-time="0" op-digest="4811cef7f7f94e3a35a70be7916cb2fd"/>
+ </lrm_resource>
+ <lrm_resource id="FAKE2" type="Dummy" class="ocf" provider="heartbeat">
+ <lrm_rsc_op id="FAKE2_last_0" operation_key="FAKE2_stop_0" operation="stop" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="29:10:0:9d56c099-7067-4908-9f99-89a103eff57b" transition-magic="0:0;29:10:0:9d56c099-7067-4908-9f99-89a103eff57b" on_node="c7auto2" call-id="30" rc-code="0" op-status="0" interval="0" last-run="1437147852" last-rc-change="1437147852" exec-time="7" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" fake " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ <lrm_rsc_op id="FAKE2_monitor_10000" operation_key="FAKE2_monitor_10000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="26:6:0:9d56c099-7067-4908-9f99-89a103eff57b" transition-magic="0:0;26:6:0:9d56c099-7067-4908-9f99-89a103eff57b" on_node="c7auto2" call-id="19" rc-code="0" op-status="0" interval="10000" last-rc-change="1437147019" exec-time="6" queue-time="0" op-digest="4811cef7f7f94e3a35a70be7916cb2fd"/>
+ </lrm_resource>
+ <lrm_resource id="FAKE3" type="Dummy" class="ocf" provider="heartbeat">
+ <lrm_rsc_op id="FAKE3_last_0" operation_key="FAKE3_stop_0" operation="stop" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="39:10:0:9d56c099-7067-4908-9f99-89a103eff57b" transition-magic="0:0;39:10:0:9d56c099-7067-4908-9f99-89a103eff57b" on_node="c7auto2" call-id="28" rc-code="0" op-status="0" interval="0" last-run="1437147852" last-rc-change="1437147852" exec-time="9" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" fake " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ <lrm_rsc_op id="FAKE3_monitor_10000" operation_key="FAKE3_monitor_10000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="39:7:0:9d56c099-7067-4908-9f99-89a103eff57b" transition-magic="0:0;39:7:0:9d56c099-7067-4908-9f99-89a103eff57b" on_node="c7auto2" call-id="26" rc-code="0" op-status="0" interval="10000" last-rc-change="1437147024" exec-time="8" queue-time="0" op-digest="4811cef7f7f94e3a35a70be7916cb2fd"/>
+ </lrm_resource>
+ </lrm_resources>
+ </lrm>
+ <transient_attributes id="2">
+ <instance_attributes id="status-2">
+ <nvpair id="status-2-shutdown" name="shutdown" value="0"/>
+ <nvpair id="status-2-probe_complete" name="probe_complete" value="true"/>
+ </instance_attributes>
+ </transient_attributes>
+ </node_state>
+ </status>
+</cib>
diff --git a/pengine/test10/clone_min_interleave_start_two.dot b/pengine/test10/clone_min_interleave_start_two.dot
new file mode 100644
index 0000000..f99ce32
--- /dev/null
+++ b/pengine/test10/clone_min_interleave_start_two.dot
@@ -0,0 +1,59 @@
+ digraph "g" {
+"FAKE1-clone_running_0" [ style=bold color="green" fontcolor="orange"]
+"FAKE1-clone_start_0" -> "FAKE1-clone_running_0" [ style = bold]
+"FAKE1-clone_start_0" -> "FAKE1:1_start_0 c7auto1" [ style = bold]
+"FAKE1-clone_start_0" -> "FAKE1_start_0 c7auto2" [ style = bold]
+"FAKE1-clone_start_0" [ style=bold color="green" fontcolor="orange"]
+"FAKE1:1_monitor_10000 c7auto1" [ style=bold color="green" fontcolor="black"]
+"FAKE1:1_start_0 c7auto1" -> "FAKE1-clone_running_0" [ style = bold]
+"FAKE1:1_start_0 c7auto1" -> "FAKE1:1_monitor_10000 c7auto1" [ style = bold]
+"FAKE1:1_start_0 c7auto1" -> "clone-one-or-more:order-FAKE1-clone-FAKE2-clone-mandatory" [ style = bold]
+"FAKE1:1_start_0 c7auto1" [ style=bold color="green" fontcolor="black"]
+"FAKE1_monitor_10000 c7auto2" [ style=bold color="green" fontcolor="black"]
+"FAKE1_start_0 c7auto2" -> "FAKE1-clone_running_0" [ style = bold]
+"FAKE1_start_0 c7auto2" -> "FAKE1_monitor_10000 c7auto2" [ style = bold]
+"FAKE1_start_0 c7auto2" -> "clone-one-or-more:order-FAKE1-clone-FAKE2-clone-mandatory" [ style = bold]
+"FAKE1_start_0 c7auto2" [ style=bold color="green" fontcolor="black"]
+"FAKE2-clone_running_0" -> "FAKE3-clone_start_0" [ style = bold]
+"FAKE2-clone_running_0" [ style=bold color="green" fontcolor="orange"]
+"FAKE2-clone_start_0" -> "FAKE2-clone_running_0" [ style = bold]
+"FAKE2-clone_start_0" -> "FAKE2:1_start_0 c7auto2" [ style = bold]
+"FAKE2-clone_start_0" -> "FAKE2:2_start_0 c7auto1" [ style = bold]
+"FAKE2-clone_start_0" -> "FAKE2_start_0 c7auto3" [ style = bold]
+"FAKE2-clone_start_0" [ style=bold color="green" fontcolor="orange"]
+"FAKE2:1_monitor_10000 c7auto2" [ style=bold color="green" fontcolor="black"]
+"FAKE2:1_start_0 c7auto2" -> "FAKE2-clone_running_0" [ style = bold]
+"FAKE2:1_start_0 c7auto2" -> "FAKE2:1_monitor_10000 c7auto2" [ style = bold]
+"FAKE2:1_start_0 c7auto2" -> "FAKE3:1_start_0 c7auto2" [ style = bold]
+"FAKE2:1_start_0 c7auto2" [ style=bold color="green" fontcolor="black"]
+"FAKE2:2_monitor_10000 c7auto1" [ style=bold color="green" fontcolor="black"]
+"FAKE2:2_start_0 c7auto1" -> "FAKE2-clone_running_0" [ style = bold]
+"FAKE2:2_start_0 c7auto1" -> "FAKE2:2_monitor_10000 c7auto1" [ style = bold]
+"FAKE2:2_start_0 c7auto1" -> "FAKE3:2_start_0 c7auto1" [ style = bold]
+"FAKE2:2_start_0 c7auto1" [ style=bold color="green" fontcolor="black"]
+"FAKE2_monitor_10000 c7auto3" [ style=bold color="green" fontcolor="black"]
+"FAKE2_start_0 c7auto3" -> "FAKE2-clone_running_0" [ style = bold]
+"FAKE2_start_0 c7auto3" -> "FAKE2_monitor_10000 c7auto3" [ style = bold]
+"FAKE2_start_0 c7auto3" -> "FAKE3_start_0 c7auto3" [ style = bold]
+"FAKE2_start_0 c7auto3" [ style=bold color="green" fontcolor="black"]
+"FAKE3-clone_running_0" [ style=bold color="green" fontcolor="orange"]
+"FAKE3-clone_start_0" -> "FAKE3-clone_running_0" [ style = bold]
+"FAKE3-clone_start_0" -> "FAKE3:1_start_0 c7auto2" [ style = bold]
+"FAKE3-clone_start_0" -> "FAKE3:2_start_0 c7auto1" [ style = bold]
+"FAKE3-clone_start_0" -> "FAKE3_start_0 c7auto3" [ style = bold]
+"FAKE3-clone_start_0" [ style=bold color="green" fontcolor="orange"]
+"FAKE3:1_monitor_10000 c7auto2" [ style=bold color="green" fontcolor="black"]
+"FAKE3:1_start_0 c7auto2" -> "FAKE3-clone_running_0" [ style = bold]
+"FAKE3:1_start_0 c7auto2" -> "FAKE3:1_monitor_10000 c7auto2" [ style = bold]
+"FAKE3:1_start_0 c7auto2" [ style=bold color="green" fontcolor="black"]
+"FAKE3:2_monitor_10000 c7auto1" [ style=bold color="green" fontcolor="black"]
+"FAKE3:2_start_0 c7auto1" -> "FAKE3-clone_running_0" [ style = bold]
+"FAKE3:2_start_0 c7auto1" -> "FAKE3:2_monitor_10000 c7auto1" [ style = bold]
+"FAKE3:2_start_0 c7auto1" [ style=bold color="green" fontcolor="black"]
+"FAKE3_monitor_10000 c7auto3" [ style=bold color="green" fontcolor="black"]
+"FAKE3_start_0 c7auto3" -> "FAKE3-clone_running_0" [ style = bold]
+"FAKE3_start_0 c7auto3" -> "FAKE3_monitor_10000 c7auto3" [ style = bold]
+"FAKE3_start_0 c7auto3" [ style=bold color="green" fontcolor="black"]
+"clone-one-or-more:order-FAKE1-clone-FAKE2-clone-mandatory" -> "FAKE2-clone_start_0" [ style = bold]
+"clone-one-or-more:order-FAKE1-clone-FAKE2-clone-mandatory" [ style=bold color="green" fontcolor="orange"]
+}
diff --git a/pengine/test10/clone_min_interleave_start_two.exp b/pengine/test10/clone_min_interleave_start_two.exp
new file mode 100644
index 0000000..9846072
--- /dev/null
+++ b/pengine/test10/clone_min_interleave_start_two.exp
@@ -0,0 +1,326 @@
+<transition_graph cluster-delay="60s" stonith-timeout="60s" failed-stop-offset="INFINITY" failed-start-offset="INFINITY" transition_id="0">
+ <synapse id="0">
+ <action_set>
+ <rsc_op id="11" operation="monitor" operation_key="FAKE1_monitor_10000" internal_operation_key="FAKE1:0_monitor_10000" on_node="c7auto2" on_node_uuid="2">
+ <primitive id="FAKE1" long-id="FAKE1:0" class="ocf" provider="heartbeat" type="Dummy"/>
+ <attributes CRM_meta_clone="0" CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_interval="10000" CRM_meta_name="monitor" CRM_meta_notify="false" CRM_meta_timeout="20000" />
+ </rsc_op>
+ </action_set>
+ <inputs>
+ <trigger>
+ <rsc_op id="10" operation="start" operation_key="FAKE1_start_0" internal_operation_key="FAKE1:0_start_0" on_node="c7auto2" on_node_uuid="2"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="1">
+ <action_set>
+ <rsc_op id="10" operation="start" operation_key="FAKE1_start_0" internal_operation_key="FAKE1:0_start_0" on_node="c7auto2" on_node_uuid="2">
+ <primitive id="FAKE1" long-id="FAKE1:0" class="ocf" provider="heartbeat" type="Dummy"/>
+ <attributes CRM_meta_clone="0" CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_name="start" CRM_meta_notify="false" CRM_meta_timeout="20000" />
+ </rsc_op>
+ </action_set>
+ <inputs>
+ <trigger>
+ <pseudo_event id="14" operation="start" operation_key="FAKE1-clone_start_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="2">
+ <action_set>
+ <rsc_op id="13" operation="monitor" operation_key="FAKE1:1_monitor_10000" on_node="c7auto1" on_node_uuid="1">
+ <primitive id="FAKE1" long-id="FAKE1:1" class="ocf" provider="heartbeat" type="Dummy"/>
+ <attributes CRM_meta_clone="1" CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_interval="10000" CRM_meta_name="monitor" CRM_meta_notify="false" CRM_meta_timeout="20000" />
+ </rsc_op>
+ </action_set>
+ <inputs>
+ <trigger>
+ <rsc_op id="12" operation="start" operation_key="FAKE1:1_start_0" on_node="c7auto1" on_node_uuid="1"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="3">
+ <action_set>
+ <rsc_op id="12" operation="start" operation_key="FAKE1:1_start_0" on_node="c7auto1" on_node_uuid="1">
+ <primitive id="FAKE1" long-id="FAKE1:1" class="ocf" provider="heartbeat" type="Dummy"/>
+ <attributes CRM_meta_clone="1" CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_name="start" CRM_meta_notify="false" CRM_meta_timeout="20000" />
+ </rsc_op>
+ </action_set>
+ <inputs>
+ <trigger>
+ <pseudo_event id="14" operation="start" operation_key="FAKE1-clone_start_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="4" priority="1000000">
+ <action_set>
+ <pseudo_event id="15" operation="running" operation_key="FAKE1-clone_running_0">
+ <attributes CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="false" CRM_meta_timeout="20000" />
+ </pseudo_event>
+ </action_set>
+ <inputs>
+ <trigger>
+ <rsc_op id="10" operation="start" operation_key="FAKE1_start_0" internal_operation_key="FAKE1:0_start_0" on_node="c7auto2" on_node_uuid="2"/>
+ </trigger>
+ <trigger>
+ <rsc_op id="12" operation="start" operation_key="FAKE1:1_start_0" on_node="c7auto1" on_node_uuid="1"/>
+ </trigger>
+ <trigger>
+ <pseudo_event id="14" operation="start" operation_key="FAKE1-clone_start_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="5">
+ <action_set>
+ <pseudo_event id="14" operation="start" operation_key="FAKE1-clone_start_0">
+ <attributes CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="false" CRM_meta_timeout="20000" />
+ </pseudo_event>
+ </action_set>
+ <inputs/>
+ </synapse>
+ <synapse id="6">
+ <action_set>
+ <rsc_op id="19" operation="monitor" operation_key="FAKE2_monitor_10000" internal_operation_key="FAKE2:0_monitor_10000" on_node="c7auto3" on_node_uuid="3">
+ <primitive id="FAKE2" long-id="FAKE2:0" class="ocf" provider="heartbeat" type="Dummy"/>
+ <attributes CRM_meta_clone="0" CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_interval="10000" CRM_meta_name="monitor" CRM_meta_notify="false" CRM_meta_timeout="20000" />
+ </rsc_op>
+ </action_set>
+ <inputs>
+ <trigger>
+ <rsc_op id="18" operation="start" operation_key="FAKE2_start_0" internal_operation_key="FAKE2:0_start_0" on_node="c7auto3" on_node_uuid="3"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="7">
+ <action_set>
+ <rsc_op id="18" operation="start" operation_key="FAKE2_start_0" internal_operation_key="FAKE2:0_start_0" on_node="c7auto3" on_node_uuid="3">
+ <primitive id="FAKE2" long-id="FAKE2:0" class="ocf" provider="heartbeat" type="Dummy"/>
+ <attributes CRM_meta_clone="0" CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_name="start" CRM_meta_notify="false" CRM_meta_timeout="20000" />
+ </rsc_op>
+ </action_set>
+ <inputs>
+ <trigger>
+ <pseudo_event id="24" operation="start" operation_key="FAKE2-clone_start_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="8">
+ <action_set>
+ <rsc_op id="21" operation="monitor" operation_key="FAKE2:1_monitor_10000" on_node="c7auto2" on_node_uuid="2">
+ <primitive id="FAKE2" long-id="FAKE2:1" class="ocf" provider="heartbeat" type="Dummy"/>
+ <attributes CRM_meta_clone="1" CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_interval="10000" CRM_meta_name="monitor" CRM_meta_notify="false" CRM_meta_timeout="20000" />
+ </rsc_op>
+ </action_set>
+ <inputs>
+ <trigger>
+ <rsc_op id="20" operation="start" operation_key="FAKE2:1_start_0" on_node="c7auto2" on_node_uuid="2"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="9">
+ <action_set>
+ <rsc_op id="20" operation="start" operation_key="FAKE2:1_start_0" on_node="c7auto2" on_node_uuid="2">
+ <primitive id="FAKE2" long-id="FAKE2:1" class="ocf" provider="heartbeat" type="Dummy"/>
+ <attributes CRM_meta_clone="1" CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_name="start" CRM_meta_notify="false" CRM_meta_timeout="20000" />
+ </rsc_op>
+ </action_set>
+ <inputs>
+ <trigger>
+ <pseudo_event id="24" operation="start" operation_key="FAKE2-clone_start_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="10">
+ <action_set>
+ <rsc_op id="23" operation="monitor" operation_key="FAKE2:2_monitor_10000" on_node="c7auto1" on_node_uuid="1">
+ <primitive id="FAKE2" long-id="FAKE2:2" class="ocf" provider="heartbeat" type="Dummy"/>
+ <attributes CRM_meta_clone="2" CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_interval="10000" CRM_meta_name="monitor" CRM_meta_notify="false" CRM_meta_timeout="20000" />
+ </rsc_op>
+ </action_set>
+ <inputs>
+ <trigger>
+ <rsc_op id="22" operation="start" operation_key="FAKE2:2_start_0" on_node="c7auto1" on_node_uuid="1"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="11">
+ <action_set>
+ <rsc_op id="22" operation="start" operation_key="FAKE2:2_start_0" on_node="c7auto1" on_node_uuid="1">
+ <primitive id="FAKE2" long-id="FAKE2:2" class="ocf" provider="heartbeat" type="Dummy"/>
+ <attributes CRM_meta_clone="2" CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_name="start" CRM_meta_notify="false" CRM_meta_timeout="20000" />
+ </rsc_op>
+ </action_set>
+ <inputs>
+ <trigger>
+ <pseudo_event id="24" operation="start" operation_key="FAKE2-clone_start_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="12" priority="1000000">
+ <action_set>
+ <pseudo_event id="25" operation="running" operation_key="FAKE2-clone_running_0">
+ <attributes CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="false" CRM_meta_timeout="20000" />
+ </pseudo_event>
+ </action_set>
+ <inputs>
+ <trigger>
+ <rsc_op id="18" operation="start" operation_key="FAKE2_start_0" internal_operation_key="FAKE2:0_start_0" on_node="c7auto3" on_node_uuid="3"/>
+ </trigger>
+ <trigger>
+ <rsc_op id="20" operation="start" operation_key="FAKE2:1_start_0" on_node="c7auto2" on_node_uuid="2"/>
+ </trigger>
+ <trigger>
+ <rsc_op id="22" operation="start" operation_key="FAKE2:2_start_0" on_node="c7auto1" on_node_uuid="1"/>
+ </trigger>
+ <trigger>
+ <pseudo_event id="24" operation="start" operation_key="FAKE2-clone_start_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="13">
+ <action_set>
+ <pseudo_event id="24" operation="start" operation_key="FAKE2-clone_start_0">
+ <attributes CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="false" CRM_meta_timeout="20000" />
+ </pseudo_event>
+ </action_set>
+ <inputs>
+ <trigger>
+ <pseudo_event id="2" operation="clone-one-or-more:order-FAKE1-clone-FAKE2-clone-mandatory" operation_key="clone-one-or-more:order-FAKE1-clone-FAKE2-clone-mandatory"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="14">
+ <action_set>
+ <rsc_op id="29" operation="monitor" operation_key="FAKE3_monitor_10000" internal_operation_key="FAKE3:0_monitor_10000" on_node="c7auto3" on_node_uuid="3">
+ <primitive id="FAKE3" long-id="FAKE3:0" class="ocf" provider="heartbeat" type="Dummy"/>
+ <attributes CRM_meta_clone="0" CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_interval="10000" CRM_meta_name="monitor" CRM_meta_notify="false" CRM_meta_timeout="20000" />
+ </rsc_op>
+ </action_set>
+ <inputs>
+ <trigger>
+ <rsc_op id="28" operation="start" operation_key="FAKE3_start_0" internal_operation_key="FAKE3:0_start_0" on_node="c7auto3" on_node_uuid="3"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="15">
+ <action_set>
+ <rsc_op id="28" operation="start" operation_key="FAKE3_start_0" internal_operation_key="FAKE3:0_start_0" on_node="c7auto3" on_node_uuid="3">
+ <primitive id="FAKE3" long-id="FAKE3:0" class="ocf" provider="heartbeat" type="Dummy"/>
+ <attributes CRM_meta_clone="0" CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_name="start" CRM_meta_notify="false" CRM_meta_timeout="20000" />
+ </rsc_op>
+ </action_set>
+ <inputs>
+ <trigger>
+ <rsc_op id="18" operation="start" operation_key="FAKE2_start_0" internal_operation_key="FAKE2:0_start_0" on_node="c7auto3" on_node_uuid="3"/>
+ </trigger>
+ <trigger>
+ <pseudo_event id="34" operation="start" operation_key="FAKE3-clone_start_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="16">
+ <action_set>
+ <rsc_op id="31" operation="monitor" operation_key="FAKE3:1_monitor_10000" on_node="c7auto2" on_node_uuid="2">
+ <primitive id="FAKE3" long-id="FAKE3:1" class="ocf" provider="heartbeat" type="Dummy"/>
+ <attributes CRM_meta_clone="1" CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_interval="10000" CRM_meta_name="monitor" CRM_meta_notify="false" CRM_meta_timeout="20000" />
+ </rsc_op>
+ </action_set>
+ <inputs>
+ <trigger>
+ <rsc_op id="30" operation="start" operation_key="FAKE3:1_start_0" on_node="c7auto2" on_node_uuid="2"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="17">
+ <action_set>
+ <rsc_op id="30" operation="start" operation_key="FAKE3:1_start_0" on_node="c7auto2" on_node_uuid="2">
+ <primitive id="FAKE3" long-id="FAKE3:1" class="ocf" provider="heartbeat" type="Dummy"/>
+ <attributes CRM_meta_clone="1" CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_name="start" CRM_meta_notify="false" CRM_meta_timeout="20000" />
+ </rsc_op>
+ </action_set>
+ <inputs>
+ <trigger>
+ <rsc_op id="20" operation="start" operation_key="FAKE2:1_start_0" on_node="c7auto2" on_node_uuid="2"/>
+ </trigger>
+ <trigger>
+ <pseudo_event id="34" operation="start" operation_key="FAKE3-clone_start_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="18">
+ <action_set>
+ <rsc_op id="33" operation="monitor" operation_key="FAKE3:2_monitor_10000" on_node="c7auto1" on_node_uuid="1">
+ <primitive id="FAKE3" long-id="FAKE3:2" class="ocf" provider="heartbeat" type="Dummy"/>
+ <attributes CRM_meta_clone="2" CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_interval="10000" CRM_meta_name="monitor" CRM_meta_notify="false" CRM_meta_timeout="20000" />
+ </rsc_op>
+ </action_set>
+ <inputs>
+ <trigger>
+ <rsc_op id="32" operation="start" operation_key="FAKE3:2_start_0" on_node="c7auto1" on_node_uuid="1"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="19">
+ <action_set>
+ <rsc_op id="32" operation="start" operation_key="FAKE3:2_start_0" on_node="c7auto1" on_node_uuid="1">
+ <primitive id="FAKE3" long-id="FAKE3:2" class="ocf" provider="heartbeat" type="Dummy"/>
+ <attributes CRM_meta_clone="2" CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_name="start" CRM_meta_notify="false" CRM_meta_timeout="20000" />
+ </rsc_op>
+ </action_set>
+ <inputs>
+ <trigger>
+ <rsc_op id="22" operation="start" operation_key="FAKE2:2_start_0" on_node="c7auto1" on_node_uuid="1"/>
+ </trigger>
+ <trigger>
+ <pseudo_event id="34" operation="start" operation_key="FAKE3-clone_start_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="20" priority="1000000">
+ <action_set>
+ <pseudo_event id="35" operation="running" operation_key="FAKE3-clone_running_0">
+ <attributes CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="false" CRM_meta_timeout="20000" />
+ </pseudo_event>
+ </action_set>
+ <inputs>
+ <trigger>
+ <rsc_op id="28" operation="start" operation_key="FAKE3_start_0" internal_operation_key="FAKE3:0_start_0" on_node="c7auto3" on_node_uuid="3"/>
+ </trigger>
+ <trigger>
+ <rsc_op id="30" operation="start" operation_key="FAKE3:1_start_0" on_node="c7auto2" on_node_uuid="2"/>
+ </trigger>
+ <trigger>
+ <rsc_op id="32" operation="start" operation_key="FAKE3:2_start_0" on_node="c7auto1" on_node_uuid="1"/>
+ </trigger>
+ <trigger>
+ <pseudo_event id="34" operation="start" operation_key="FAKE3-clone_start_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="21">
+ <action_set>
+ <pseudo_event id="34" operation="start" operation_key="FAKE3-clone_start_0">
+ <attributes CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="false" CRM_meta_timeout="20000" />
+ </pseudo_event>
+ </action_set>
+ <inputs>
+ <trigger>
+ <pseudo_event id="25" operation="running" operation_key="FAKE2-clone_running_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="22">
+ <action_set>
+ <pseudo_event id="2" operation="clone-one-or-more:order-FAKE1-clone-FAKE2-clone-mandatory" operation_key="clone-one-or-more:order-FAKE1-clone-FAKE2-clone-mandatory">
+ <attributes />
+ </pseudo_event>
+ </action_set>
+ <inputs>
+ <trigger>
+ <rsc_op id="10" operation="start" operation_key="FAKE1_start_0" internal_operation_key="FAKE1:0_start_0" on_node="c7auto2" on_node_uuid="2"/>
+ </trigger>
+ <trigger>
+ <rsc_op id="12" operation="start" operation_key="FAKE1:1_start_0" on_node="c7auto1" on_node_uuid="1"/>
+ </trigger>
+ </inputs>
+ </synapse>
+</transition_graph>
diff --git a/pengine/test10/clone_min_interleave_start_two.scores b/pengine/test10/clone_min_interleave_start_two.scores
new file mode 100644
index 0000000..d443c58
--- /dev/null
+++ b/pengine/test10/clone_min_interleave_start_two.scores
@@ -0,0 +1,67 @@
+Allocation scores:
+clone_color: FAKE1-clone allocation score on c7auto1: 0
+clone_color: FAKE1-clone allocation score on c7auto2: 0
+clone_color: FAKE1-clone allocation score on c7auto3: -INFINITY
+clone_color: FAKE1:0 allocation score on c7auto1: 0
+clone_color: FAKE1:0 allocation score on c7auto2: 0
+clone_color: FAKE1:0 allocation score on c7auto3: -INFINITY
+clone_color: FAKE1:1 allocation score on c7auto1: 0
+clone_color: FAKE1:1 allocation score on c7auto2: 0
+clone_color: FAKE1:1 allocation score on c7auto3: -INFINITY
+clone_color: FAKE1:2 allocation score on c7auto1: 0
+clone_color: FAKE1:2 allocation score on c7auto2: 0
+clone_color: FAKE1:2 allocation score on c7auto3: -INFINITY
+clone_color: FAKE2-clone allocation score on c7auto1: 0
+clone_color: FAKE2-clone allocation score on c7auto2: 0
+clone_color: FAKE2-clone allocation score on c7auto3: 0
+clone_color: FAKE2:0 allocation score on c7auto1: 0
+clone_color: FAKE2:0 allocation score on c7auto2: 0
+clone_color: FAKE2:0 allocation score on c7auto3: 0
+clone_color: FAKE2:1 allocation score on c7auto1: 0
+clone_color: FAKE2:1 allocation score on c7auto2: 0
+clone_color: FAKE2:1 allocation score on c7auto3: 0
+clone_color: FAKE2:2 allocation score on c7auto1: 0
+clone_color: FAKE2:2 allocation score on c7auto2: 0
+clone_color: FAKE2:2 allocation score on c7auto3: 0
+clone_color: FAKE3-clone allocation score on c7auto1: 0
+clone_color: FAKE3-clone allocation score on c7auto2: 0
+clone_color: FAKE3-clone allocation score on c7auto3: 0
+clone_color: FAKE3:0 allocation score on c7auto1: 0
+clone_color: FAKE3:0 allocation score on c7auto2: 0
+clone_color: FAKE3:0 allocation score on c7auto3: 0
+clone_color: FAKE3:1 allocation score on c7auto1: 0
+clone_color: FAKE3:1 allocation score on c7auto2: 0
+clone_color: FAKE3:1 allocation score on c7auto3: 0
+clone_color: FAKE3:2 allocation score on c7auto1: 0
+clone_color: FAKE3:2 allocation score on c7auto2: 0
+clone_color: FAKE3:2 allocation score on c7auto3: 0
+native_color: FAKE1:0 allocation score on c7auto1: 0
+native_color: FAKE1:0 allocation score on c7auto2: 0
+native_color: FAKE1:0 allocation score on c7auto3: -INFINITY
+native_color: FAKE1:1 allocation score on c7auto1: 0
+native_color: FAKE1:1 allocation score on c7auto2: -INFINITY
+native_color: FAKE1:1 allocation score on c7auto3: -INFINITY
+native_color: FAKE1:2 allocation score on c7auto1: -INFINITY
+native_color: FAKE1:2 allocation score on c7auto2: -INFINITY
+native_color: FAKE1:2 allocation score on c7auto3: -INFINITY
+native_color: FAKE2:0 allocation score on c7auto1: 0
+native_color: FAKE2:0 allocation score on c7auto2: 0
+native_color: FAKE2:0 allocation score on c7auto3: 0
+native_color: FAKE2:1 allocation score on c7auto1: 0
+native_color: FAKE2:1 allocation score on c7auto2: 0
+native_color: FAKE2:1 allocation score on c7auto3: -INFINITY
+native_color: FAKE2:2 allocation score on c7auto1: 0
+native_color: FAKE2:2 allocation score on c7auto2: -INFINITY
+native_color: FAKE2:2 allocation score on c7auto3: -INFINITY
+native_color: FAKE3:0 allocation score on c7auto1: 0
+native_color: FAKE3:0 allocation score on c7auto2: 0
+native_color: FAKE3:0 allocation score on c7auto3: 0
+native_color: FAKE3:1 allocation score on c7auto1: 0
+native_color: FAKE3:1 allocation score on c7auto2: 0
+native_color: FAKE3:1 allocation score on c7auto3: -INFINITY
+native_color: FAKE3:2 allocation score on c7auto1: 0
+native_color: FAKE3:2 allocation score on c7auto2: -INFINITY
+native_color: FAKE3:2 allocation score on c7auto3: -INFINITY
+native_color: shooter allocation score on c7auto1: 0
+native_color: shooter allocation score on c7auto2: 0
+native_color: shooter allocation score on c7auto3: 0
diff --git a/pengine/test10/clone_min_interleave_start_two.summary b/pengine/test10/clone_min_interleave_start_two.summary
new file mode 100644
index 0000000..9f928f2
--- /dev/null
+++ b/pengine/test10/clone_min_interleave_start_two.summary
@@ -0,0 +1,59 @@
+
+Current cluster status:
+Online: [ c7auto1 c7auto2 c7auto3 ]
+
+ shooter (stonith:fence_phd_kvm): Started c7auto1
+ Clone Set: FAKE1-clone [FAKE1]
+ Stopped: [ c7auto1 c7auto2 c7auto3 ]
+ Clone Set: FAKE2-clone [FAKE2]
+ Stopped: [ c7auto1 c7auto2 c7auto3 ]
+ Clone Set: FAKE3-clone [FAKE3]
+ Stopped: [ c7auto1 c7auto2 c7auto3 ]
+
+Transition Summary:
+ * Start FAKE1:0 (c7auto2)
+ * Start FAKE1:1 (c7auto1)
+ * Start FAKE2:0 (c7auto3)
+ * Start FAKE2:1 (c7auto2)
+ * Start FAKE2:2 (c7auto1)
+ * Start FAKE3:0 (c7auto3)
+ * Start FAKE3:1 (c7auto2)
+ * Start FAKE3:2 (c7auto1)
+
+Executing cluster transition:
+ * Pseudo action: FAKE1-clone_start_0
+ * Resource action: FAKE1 start on c7auto2
+ * Resource action: FAKE1 start on c7auto1
+ * Pseudo action: FAKE1-clone_running_0
+ * Pseudo action: clone-one-or-more:order-FAKE1-clone-FAKE2-clone-mandatory
+ * Resource action: FAKE1 monitor=10000 on c7auto2
+ * Resource action: FAKE1 monitor=10000 on c7auto1
+ * Pseudo action: FAKE2-clone_start_0
+ * Resource action: FAKE2 start on c7auto3
+ * Resource action: FAKE2 start on c7auto2
+ * Resource action: FAKE2 start on c7auto1
+ * Pseudo action: FAKE2-clone_running_0
+ * Pseudo action: FAKE3-clone_start_0
+ * Resource action: FAKE2 monitor=10000 on c7auto3
+ * Resource action: FAKE2 monitor=10000 on c7auto2
+ * Resource action: FAKE2 monitor=10000 on c7auto1
+ * Resource action: FAKE3 start on c7auto3
+ * Resource action: FAKE3 start on c7auto2
+ * Resource action: FAKE3 start on c7auto1
+ * Pseudo action: FAKE3-clone_running_0
+ * Resource action: FAKE3 monitor=10000 on c7auto3
+ * Resource action: FAKE3 monitor=10000 on c7auto2
+ * Resource action: FAKE3 monitor=10000 on c7auto1
+
+Revised cluster status:
+Online: [ c7auto1 c7auto2 c7auto3 ]
+
+ shooter (stonith:fence_phd_kvm): Started c7auto1
+ Clone Set: FAKE1-clone [FAKE1]
+ Started: [ c7auto1 c7auto2 ]
+ Stopped: [ c7auto3 ]
+ Clone Set: FAKE2-clone [FAKE2]
+ Started: [ c7auto1 c7auto2 c7auto3 ]
+ Clone Set: FAKE3-clone [FAKE3]
+ Started: [ c7auto1 c7auto2 c7auto3 ]
+
diff --git a/pengine/test10/clone_min_interleave_start_two.xml b/pengine/test10/clone_min_interleave_start_two.xml
new file mode 100644
index 0000000..2507018
--- /dev/null
+++ b/pengine/test10/clone_min_interleave_start_two.xml
@@ -0,0 +1,154 @@
+<cib crm_feature_set="3.0.10" validate-with="pacemaker-2.3" epoch="15" num_updates="0" admin_epoch="0" cib-last-written="Fri Jul 17 11:44:12 2015" update-origin="c7auto1" update-client="crm_resource" update-user="root" have-quorum="1" dc-uuid="3">
+ <configuration>
+ <crm_config>
+ <cluster_property_set id="cib-bootstrap-options">
+ <nvpair id="cib-bootstrap-options-have-watchdog" name="have-watchdog" value="false"/>
+ <nvpair id="cib-bootstrap-options-dc-version" name="dc-version" value="1.1.12-ef86b49"/>
+ <nvpair id="cib-bootstrap-options-cluster-infrastructure" name="cluster-infrastructure" value="corosync"/>
+ <nvpair id="cib-bootstrap-options-cluster-name" name="cluster-name" value="phd"/>
+ </cluster_property_set>
+ </crm_config>
+ <nodes>
+ <node id="1" uname="c7auto1"/>
+ <node id="2" uname="c7auto2"/>
+ <node id="3" uname="c7auto3"/>
+ </nodes>
+ <resources>
+ <primitive class="stonith" id="shooter" type="fence_phd_kvm">
+ <instance_attributes id="shooter-instance_attributes"/>
+ <operations>
+ <op id="shooter-monitor-interval-60s" interval="60s" name="monitor"/>
+ </operations>
+ </primitive>
+ <clone id="FAKE1-clone">
+ <primitive class="ocf" id="FAKE1" provider="heartbeat" type="Dummy">
+ <instance_attributes id="FAKE1-instance_attributes"/>
+ <operations>
+ <op id="FAKE1-start-timeout-20" interval="0s" name="start" timeout="20"/>
+ <op id="FAKE1-stop-timeout-20" interval="0s" name="stop" timeout="20"/>
+ <op id="FAKE1-monitor-interval-10" interval="10" name="monitor" timeout="20"/>
+ </operations>
+ </primitive>
+ <meta_attributes id="FAKE1-clone-meta">
+ <nvpair id="FAKE1-clone-min" name="clone-min" value="2"/>
+ </meta_attributes>
+ <meta_attributes id="FAKE1-clone-meta_attributes"/>
+ </clone>
+ <clone id="FAKE2-clone">
+ <primitive class="ocf" id="FAKE2" provider="heartbeat" type="Dummy">
+ <instance_attributes id="FAKE2-instance_attributes"/>
+ <operations>
+ <op id="FAKE2-start-timeout-20" interval="0s" name="start" timeout="20"/>
+ <op id="FAKE2-stop-timeout-20" interval="0s" name="stop" timeout="20"/>
+ <op id="FAKE2-monitor-interval-10" interval="10" name="monitor" timeout="20"/>
+ </operations>
+ </primitive>
+ <meta_attributes id="FAKE2-clone-meta">
+ <nvpair id="FAKE2-interleave" name="interleave" value="true"/>
+ </meta_attributes>
+ </clone>
+ <clone id="FAKE3-clone">
+ <primitive class="ocf" id="FAKE3" provider="heartbeat" type="Dummy">
+ <instance_attributes id="FAKE3-instance_attributes"/>
+ <operations>
+ <op id="FAKE3-start-timeout-20" interval="0s" name="start" timeout="20"/>
+ <op id="FAKE3-stop-timeout-20" interval="0s" name="stop" timeout="20"/>
+ <op id="FAKE3-monitor-interval-10" interval="10" name="monitor" timeout="20"/>
+ </operations>
+ </primitive>
+ <meta_attributes id="FAKE3-clone-meta">
+ <nvpair id="FAKE3-interleave" name="interleave" value="true"/>
+ </meta_attributes>
+ </clone>
+ </resources>
+ <constraints>
+ <rsc_order first="FAKE1-clone" first-action="start" id="order-FAKE1-clone-FAKE2-clone-mandatory" then="FAKE2-clone" then-action="start"/>
+ <rsc_order first="FAKE2-clone" first-action="start" id="order-FAKE2-clone-FAKE3-clone-mandatory" then="FAKE3-clone" then-action="start"/>
+ <rsc_location id="cli-ban-FAKE1-clone-on-c7auto3" rsc="FAKE1-clone" role="Started" node="c7auto3" score="-INFINITY"/>
+ </constraints>
+ </configuration>
+ <status>
+ <node_state id="3" uname="c7auto3" in_ccm="true" crmd="online" crm-debug-origin="do_update_resource" join="member" expected="member">
+ <lrm id="3">
+ <lrm_resources>
+ <lrm_resource id="shooter" type="fence_phd_kvm" class="stonith">
+ <lrm_rsc_op id="shooter_last_0" operation_key="shooter_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="8:4:7:9d56c099-7067-4908-9f99-89a103eff57b" transition-magic="0:7;8:4:7:9d56c099-7067-4908-9f99-89a103eff57b" on_node="c7auto3" call-id="5" rc-code="7" op-status="0" interval="0" last-run="1437146780" last-rc-change="1437146780" exec-time="998" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ </lrm_resource>
+ <lrm_resource id="FAKE1" type="Dummy" class="ocf" provider="heartbeat">
+ <lrm_rsc_op id="FAKE1_last_0" operation_key="FAKE1_stop_0" operation="stop" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="18:10:0:9d56c099-7067-4908-9f99-89a103eff57b" transition-magic="0:0;18:10:0:9d56c099-7067-4908-9f99-89a103eff57b" on_node="c7auto3" call-id="32" rc-code="0" op-status="0" interval="0" last-run="1437147852" last-rc-change="1437147852" exec-time="8" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" fake " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ <lrm_rsc_op id="FAKE1_monitor_10000" operation_key="FAKE1_monitor_10000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="15:5:0:9d56c099-7067-4908-9f99-89a103eff57b" transition-magic="0:0;15:5:0:9d56c099-7067-4908-9f99-89a103eff57b" on_node="c7auto3" call-id="12" rc-code="0" op-status="0" interval="10000" last-rc-change="1437147006" exec-time="9" queue-time="0" op-digest="4811cef7f7f94e3a35a70be7916cb2fd"/>
+ </lrm_resource>
+ <lrm_resource id="FAKE2" type="Dummy" class="ocf" provider="heartbeat">
+ <lrm_rsc_op id="FAKE2_last_0" operation_key="FAKE2_stop_0" operation="stop" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="25:10:0:9d56c099-7067-4908-9f99-89a103eff57b" transition-magic="0:0;25:10:0:9d56c099-7067-4908-9f99-89a103eff57b" on_node="c7auto3" call-id="30" rc-code="0" op-status="0" interval="0" last-run="1437147852" last-rc-change="1437147852" exec-time="8" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" fake " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ <lrm_rsc_op id="FAKE2_monitor_10000" operation_key="FAKE2_monitor_10000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="28:6:0:9d56c099-7067-4908-9f99-89a103eff57b" transition-magic="0:0;28:6:0:9d56c099-7067-4908-9f99-89a103eff57b" on_node="c7auto3" call-id="19" rc-code="0" op-status="0" interval="10000" last-rc-change="1437147019" exec-time="8" queue-time="0" op-digest="4811cef7f7f94e3a35a70be7916cb2fd"/>
+ </lrm_resource>
+ <lrm_resource id="FAKE3" type="Dummy" class="ocf" provider="heartbeat">
+ <lrm_rsc_op id="FAKE3_last_0" operation_key="FAKE3_stop_0" operation="stop" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="35:10:0:9d56c099-7067-4908-9f99-89a103eff57b" transition-magic="0:0;35:10:0:9d56c099-7067-4908-9f99-89a103eff57b" on_node="c7auto3" call-id="28" rc-code="0" op-status="0" interval="0" last-run="1437147852" last-rc-change="1437147852" exec-time="10" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" fake " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ <lrm_rsc_op id="FAKE3_monitor_10000" operation_key="FAKE3_monitor_10000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="41:7:0:9d56c099-7067-4908-9f99-89a103eff57b" transition-magic="0:0;41:7:0:9d56c099-7067-4908-9f99-89a103eff57b" on_node="c7auto3" call-id="26" rc-code="0" op-status="0" interval="10000" last-rc-change="1437147024" exec-time="8" queue-time="0" op-digest="4811cef7f7f94e3a35a70be7916cb2fd"/>
+ </lrm_resource>
+ </lrm_resources>
+ </lrm>
+ <transient_attributes id="3">
+ <instance_attributes id="status-3">
+ <nvpair id="status-3-shutdown" name="shutdown" value="0"/>
+ <nvpair id="status-3-probe_complete" name="probe_complete" value="true"/>
+ </instance_attributes>
+ </transient_attributes>
+ </node_state>
+ <node_state id="1" uname="c7auto1" in_ccm="true" crmd="online" crm-debug-origin="do_update_resource" join="member" expected="member">
+ <lrm id="1">
+ <lrm_resources>
+ <lrm_resource id="shooter" type="fence_phd_kvm" class="stonith">
+ <lrm_rsc_op id="shooter_last_0" operation_key="shooter_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="9:4:0:9d56c099-7067-4908-9f99-89a103eff57b" transition-magic="0:0;9:4:0:9d56c099-7067-4908-9f99-89a103eff57b" on_node="c7auto1" call-id="6" rc-code="0" op-status="0" interval="0" last-run="1437146782" last-rc-change="1437146782" exec-time="5" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ <lrm_rsc_op id="shooter_monitor_60000" operation_key="shooter_monitor_60000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="10:4:0:9d56c099-7067-4908-9f99-89a103eff57b" transition-magic="0:0;10:4:0:9d56c099-7067-4908-9f99-89a103eff57b" on_node="c7auto1" call-id="7" rc-code="0" op-status="0" interval="60000" last-rc-change="1437146783" exec-time="5" queue-time="1" op-digest="4811cef7f7f94e3a35a70be7916cb2fd"/>
+ </lrm_resource>
+ <lrm_resource id="FAKE1" type="Dummy" class="ocf" provider="heartbeat">
+ <lrm_rsc_op id="FAKE1_last_0" operation_key="FAKE1_stop_0" operation="stop" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="19:10:0:9d56c099-7067-4908-9f99-89a103eff57b" transition-magic="0:0;19:10:0:9d56c099-7067-4908-9f99-89a103eff57b" on_node="c7auto1" call-id="34" rc-code="0" op-status="0" interval="0" last-run="1437147852" last-rc-change="1437147852" exec-time="7" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" fake " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ <lrm_rsc_op id="FAKE1_monitor_10000" operation_key="FAKE1_monitor_10000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="17:5:0:9d56c099-7067-4908-9f99-89a103eff57b" transition-magic="0:0;17:5:0:9d56c099-7067-4908-9f99-89a103eff57b" on_node="c7auto1" call-id="14" rc-code="0" op-status="0" interval="10000" last-rc-change="1437147007" exec-time="6" queue-time="0" op-digest="4811cef7f7f94e3a35a70be7916cb2fd"/>
+ </lrm_resource>
+ <lrm_resource id="FAKE2" type="Dummy" class="ocf" provider="heartbeat">
+ <lrm_rsc_op id="FAKE2_last_0" operation_key="FAKE2_stop_0" operation="stop" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="27:10:0:9d56c099-7067-4908-9f99-89a103eff57b" transition-magic="0:0;27:10:0:9d56c099-7067-4908-9f99-89a103eff57b" on_node="c7auto1" call-id="32" rc-code="0" op-status="0" interval="0" last-run="1437147852" last-rc-change="1437147852" exec-time="8" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" fake " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ <lrm_rsc_op id="FAKE2_monitor_10000" operation_key="FAKE2_monitor_10000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="30:6:0:9d56c099-7067-4908-9f99-89a103eff57b" transition-magic="0:0;30:6:0:9d56c099-7067-4908-9f99-89a103eff57b" on_node="c7auto1" call-id="21" rc-code="0" op-status="0" interval="10000" last-rc-change="1437147019" exec-time="7" queue-time="0" op-digest="4811cef7f7f94e3a35a70be7916cb2fd"/>
+ </lrm_resource>
+ <lrm_resource id="FAKE3" type="Dummy" class="ocf" provider="heartbeat">
+ <lrm_rsc_op id="FAKE3_last_0" operation_key="FAKE3_stop_0" operation="stop" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="37:10:0:9d56c099-7067-4908-9f99-89a103eff57b" transition-magic="0:0;37:10:0:9d56c099-7067-4908-9f99-89a103eff57b" on_node="c7auto1" call-id="30" rc-code="0" op-status="0" interval="0" last-run="1437147852" last-rc-change="1437147852" exec-time="7" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" fake " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ <lrm_rsc_op id="FAKE3_monitor_10000" operation_key="FAKE3_monitor_10000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="43:7:0:9d56c099-7067-4908-9f99-89a103eff57b" transition-magic="0:0;43:7:0:9d56c099-7067-4908-9f99-89a103eff57b" on_node="c7auto1" call-id="28" rc-code="0" op-status="0" interval="10000" last-rc-change="1437147024" exec-time="6" queue-time="1" op-digest="4811cef7f7f94e3a35a70be7916cb2fd"/>
+ </lrm_resource>
+ </lrm_resources>
+ </lrm>
+ <transient_attributes id="1">
+ <instance_attributes id="status-1">
+ <nvpair id="status-1-shutdown" name="shutdown" value="0"/>
+ <nvpair id="status-1-probe_complete" name="probe_complete" value="true"/>
+ </instance_attributes>
+ </transient_attributes>
+ </node_state>
+ <node_state id="2" uname="c7auto2" in_ccm="true" crmd="online" crm-debug-origin="do_update_resource" join="member" expected="member">
+ <lrm id="2">
+ <lrm_resources>
+ <lrm_resource id="shooter" type="fence_phd_kvm" class="stonith">
+ <lrm_rsc_op id="shooter_last_0" operation_key="shooter_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="6:4:7:9d56c099-7067-4908-9f99-89a103eff57b" transition-magic="0:7;6:4:7:9d56c099-7067-4908-9f99-89a103eff57b" on_node="c7auto2" call-id="5" rc-code="7" op-status="0" interval="0" last-run="1437146780" last-rc-change="1437146780" exec-time="994" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ </lrm_resource>
+ <lrm_resource id="FAKE1" type="Dummy" class="ocf" provider="heartbeat">
+ <lrm_rsc_op id="FAKE1_last_0" operation_key="FAKE1_stop_0" operation="stop" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="20:10:0:9d56c099-7067-4908-9f99-89a103eff57b" transition-magic="0:0;20:10:0:9d56c099-7067-4908-9f99-89a103eff57b" on_node="c7auto2" call-id="32" rc-code="0" op-status="0" interval="0" last-run="1437147852" last-rc-change="1437147852" exec-time="7" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" fake " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ <lrm_rsc_op id="FAKE1_monitor_10000" operation_key="FAKE1_monitor_10000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="13:5:0:9d56c099-7067-4908-9f99-89a103eff57b" transition-magic="0:0;13:5:0:9d56c099-7067-4908-9f99-89a103eff57b" on_node="c7auto2" call-id="12" rc-code="0" op-status="0" interval="10000" last-rc-change="1437147007" exec-time="8" queue-time="0" op-digest="4811cef7f7f94e3a35a70be7916cb2fd"/>
+ </lrm_resource>
+ <lrm_resource id="FAKE2" type="Dummy" class="ocf" provider="heartbeat">
+ <lrm_rsc_op id="FAKE2_last_0" operation_key="FAKE2_stop_0" operation="stop" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="29:10:0:9d56c099-7067-4908-9f99-89a103eff57b" transition-magic="0:0;29:10:0:9d56c099-7067-4908-9f99-89a103eff57b" on_node="c7auto2" call-id="30" rc-code="0" op-status="0" interval="0" last-run="1437147852" last-rc-change="1437147852" exec-time="7" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" fake " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ <lrm_rsc_op id="FAKE2_monitor_10000" operation_key="FAKE2_monitor_10000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="26:6:0:9d56c099-7067-4908-9f99-89a103eff57b" transition-magic="0:0;26:6:0:9d56c099-7067-4908-9f99-89a103eff57b" on_node="c7auto2" call-id="19" rc-code="0" op-status="0" interval="10000" last-rc-change="1437147019" exec-time="6" queue-time="0" op-digest="4811cef7f7f94e3a35a70be7916cb2fd"/>
+ </lrm_resource>
+ <lrm_resource id="FAKE3" type="Dummy" class="ocf" provider="heartbeat">
+ <lrm_rsc_op id="FAKE3_last_0" operation_key="FAKE3_stop_0" operation="stop" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="39:10:0:9d56c099-7067-4908-9f99-89a103eff57b" transition-magic="0:0;39:10:0:9d56c099-7067-4908-9f99-89a103eff57b" on_node="c7auto2" call-id="28" rc-code="0" op-status="0" interval="0" last-run="1437147852" last-rc-change="1437147852" exec-time="9" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" fake " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ <lrm_rsc_op id="FAKE3_monitor_10000" operation_key="FAKE3_monitor_10000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="39:7:0:9d56c099-7067-4908-9f99-89a103eff57b" transition-magic="0:0;39:7:0:9d56c099-7067-4908-9f99-89a103eff57b" on_node="c7auto2" call-id="26" rc-code="0" op-status="0" interval="10000" last-rc-change="1437147024" exec-time="8" queue-time="0" op-digest="4811cef7f7f94e3a35a70be7916cb2fd"/>
+ </lrm_resource>
+ </lrm_resources>
+ </lrm>
+ <transient_attributes id="2">
+ <instance_attributes id="status-2">
+ <nvpair id="status-2-shutdown" name="shutdown" value="0"/>
+ <nvpair id="status-2-probe_complete" name="probe_complete" value="true"/>
+ </instance_attributes>
+ </transient_attributes>
+ </node_state>
+ </status>
+</cib>
diff --git a/pengine/test10/clone_min_interleave_stop_one.dot b/pengine/test10/clone_min_interleave_stop_one.dot
new file mode 100644
index 0000000..a66ceb6
--- /dev/null
+++ b/pengine/test10/clone_min_interleave_stop_one.dot
@@ -0,0 +1,18 @@
+ digraph "g" {
+"FAKE1-clone_running_0" [ style=bold color="green" fontcolor="orange"]
+"FAKE1-clone_start_0" -> "FAKE1-clone_running_0" [ style = bold]
+"FAKE1-clone_start_0" -> "FAKE1_start_0 <none>" [ style = dashed]
+"FAKE1-clone_start_0" [ style=bold color="green" fontcolor="orange"]
+"FAKE1-clone_stop_0" -> "FAKE1-clone_stopped_0" [ style = bold]
+"FAKE1-clone_stop_0" -> "FAKE1_stop_0 c7auto3" [ style = bold]
+"FAKE1-clone_stop_0" [ style=bold color="green" fontcolor="orange"]
+"FAKE1-clone_stopped_0" -> "FAKE1-clone_start_0" [ style = bold]
+"FAKE1-clone_stopped_0" [ style=bold color="green" fontcolor="orange"]
+"FAKE1_start_0 <none>" -> "FAKE1-clone_running_0" [ style = dashed]
+"FAKE1_start_0 <none>" [ style=dashed color="red" fontcolor="black"]
+"FAKE1_stop_0 c7auto3" -> "FAKE1-clone_stopped_0" [ style = bold]
+"FAKE1_stop_0 c7auto3" -> "FAKE1_start_0 <none>" [ style = dashed]
+"FAKE1_stop_0 c7auto3" -> "all_stopped" [ style = bold]
+"FAKE1_stop_0 c7auto3" [ style=bold color="green" fontcolor="black"]
+"all_stopped" [ style=bold color="green" fontcolor="orange"]
+}
diff --git a/pengine/test10/clone_min_interleave_stop_one.exp b/pengine/test10/clone_min_interleave_stop_one.exp
new file mode 100644
index 0000000..31a15da
--- /dev/null
+++ b/pengine/test10/clone_min_interleave_stop_one.exp
@@ -0,0 +1,74 @@
+<transition_graph cluster-delay="60s" stonith-timeout="60s" failed-stop-offset="INFINITY" failed-start-offset="INFINITY" transition_id="0">
+ <synapse id="0">
+ <action_set>
+ <rsc_op id="19" operation="stop" operation_key="FAKE1_stop_0" internal_operation_key="FAKE1:0_stop_0" on_node="c7auto3" on_node_uuid="3">
+ <primitive id="FAKE1" long-id="FAKE1:0" class="ocf" provider="heartbeat" type="Dummy"/>
+ <attributes CRM_meta_clone="0" CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_name="stop" CRM_meta_notify="false" CRM_meta_timeout="20000" />
+ </rsc_op>
+ </action_set>
+ <inputs>
+ <trigger>
+ <pseudo_event id="26" operation="stop" operation_key="FAKE1-clone_stop_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="1" priority="1000000">
+ <action_set>
+ <pseudo_event id="27" operation="stopped" operation_key="FAKE1-clone_stopped_0">
+ <attributes CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="false" CRM_meta_timeout="20000" />
+ </pseudo_event>
+ </action_set>
+ <inputs>
+ <trigger>
+ <rsc_op id="19" operation="stop" operation_key="FAKE1_stop_0" internal_operation_key="FAKE1:0_stop_0" on_node="c7auto3" on_node_uuid="3"/>
+ </trigger>
+ <trigger>
+ <pseudo_event id="26" operation="stop" operation_key="FAKE1-clone_stop_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="2">
+ <action_set>
+ <pseudo_event id="26" operation="stop" operation_key="FAKE1-clone_stop_0">
+ <attributes CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="false" CRM_meta_timeout="20000" />
+ </pseudo_event>
+ </action_set>
+ <inputs/>
+ </synapse>
+ <synapse id="3" priority="1000000">
+ <action_set>
+ <pseudo_event id="25" operation="running" operation_key="FAKE1-clone_running_0">
+ <attributes CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="false" CRM_meta_timeout="20000" />
+ </pseudo_event>
+ </action_set>
+ <inputs>
+ <trigger>
+ <pseudo_event id="24" operation="start" operation_key="FAKE1-clone_start_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="4">
+ <action_set>
+ <pseudo_event id="24" operation="start" operation_key="FAKE1-clone_start_0">
+ <attributes CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="false" CRM_meta_timeout="20000" />
+ </pseudo_event>
+ </action_set>
+ <inputs>
+ <trigger>
+ <pseudo_event id="27" operation="stopped" operation_key="FAKE1-clone_stopped_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="5">
+ <action_set>
+ <pseudo_event id="12" operation="all_stopped" operation_key="all_stopped">
+ <attributes />
+ </pseudo_event>
+ </action_set>
+ <inputs>
+ <trigger>
+ <rsc_op id="19" operation="stop" operation_key="FAKE1_stop_0" internal_operation_key="FAKE1:0_stop_0" on_node="c7auto3" on_node_uuid="3"/>
+ </trigger>
+ </inputs>
+ </synapse>
+</transition_graph>
diff --git a/pengine/test10/clone_min_interleave_stop_one.scores b/pengine/test10/clone_min_interleave_stop_one.scores
new file mode 100644
index 0000000..1a98230
--- /dev/null
+++ b/pengine/test10/clone_min_interleave_stop_one.scores
@@ -0,0 +1,67 @@
+Allocation scores:
+clone_color: FAKE1-clone allocation score on c7auto1: 0
+clone_color: FAKE1-clone allocation score on c7auto2: 0
+clone_color: FAKE1-clone allocation score on c7auto3: -INFINITY
+clone_color: FAKE1:0 allocation score on c7auto1: 0
+clone_color: FAKE1:0 allocation score on c7auto2: 0
+clone_color: FAKE1:0 allocation score on c7auto3: -INFINITY
+clone_color: FAKE1:1 allocation score on c7auto1: 1
+clone_color: FAKE1:1 allocation score on c7auto2: 0
+clone_color: FAKE1:1 allocation score on c7auto3: -INFINITY
+clone_color: FAKE1:2 allocation score on c7auto1: 0
+clone_color: FAKE1:2 allocation score on c7auto2: 1
+clone_color: FAKE1:2 allocation score on c7auto3: -INFINITY
+clone_color: FAKE2-clone allocation score on c7auto1: 0
+clone_color: FAKE2-clone allocation score on c7auto2: 0
+clone_color: FAKE2-clone allocation score on c7auto3: 0
+clone_color: FAKE2:0 allocation score on c7auto1: 0
+clone_color: FAKE2:0 allocation score on c7auto2: 0
+clone_color: FAKE2:0 allocation score on c7auto3: 1
+clone_color: FAKE2:1 allocation score on c7auto1: 1
+clone_color: FAKE2:1 allocation score on c7auto2: 0
+clone_color: FAKE2:1 allocation score on c7auto3: 0
+clone_color: FAKE2:2 allocation score on c7auto1: 0
+clone_color: FAKE2:2 allocation score on c7auto2: 1
+clone_color: FAKE2:2 allocation score on c7auto3: 0
+clone_color: FAKE3-clone allocation score on c7auto1: 0
+clone_color: FAKE3-clone allocation score on c7auto2: 0
+clone_color: FAKE3-clone allocation score on c7auto3: 0
+clone_color: FAKE3:0 allocation score on c7auto1: 0
+clone_color: FAKE3:0 allocation score on c7auto2: 0
+clone_color: FAKE3:0 allocation score on c7auto3: 1
+clone_color: FAKE3:1 allocation score on c7auto1: 1
+clone_color: FAKE3:1 allocation score on c7auto2: 0
+clone_color: FAKE3:1 allocation score on c7auto3: 0
+clone_color: FAKE3:2 allocation score on c7auto1: 0
+clone_color: FAKE3:2 allocation score on c7auto2: 1
+clone_color: FAKE3:2 allocation score on c7auto3: 0
+native_color: FAKE1:0 allocation score on c7auto1: -INFINITY
+native_color: FAKE1:0 allocation score on c7auto2: -INFINITY
+native_color: FAKE1:0 allocation score on c7auto3: -INFINITY
+native_color: FAKE1:1 allocation score on c7auto1: 1
+native_color: FAKE1:1 allocation score on c7auto2: -INFINITY
+native_color: FAKE1:1 allocation score on c7auto3: -INFINITY
+native_color: FAKE1:2 allocation score on c7auto1: 0
+native_color: FAKE1:2 allocation score on c7auto2: 1
+native_color: FAKE1:2 allocation score on c7auto3: -INFINITY
+native_color: FAKE2:0 allocation score on c7auto1: 0
+native_color: FAKE2:0 allocation score on c7auto2: 0
+native_color: FAKE2:0 allocation score on c7auto3: 1
+native_color: FAKE2:1 allocation score on c7auto1: 1
+native_color: FAKE2:1 allocation score on c7auto2: -INFINITY
+native_color: FAKE2:1 allocation score on c7auto3: -INFINITY
+native_color: FAKE2:2 allocation score on c7auto1: 0
+native_color: FAKE2:2 allocation score on c7auto2: 1
+native_color: FAKE2:2 allocation score on c7auto3: -INFINITY
+native_color: FAKE3:0 allocation score on c7auto1: 0
+native_color: FAKE3:0 allocation score on c7auto2: 0
+native_color: FAKE3:0 allocation score on c7auto3: 1
+native_color: FAKE3:1 allocation score on c7auto1: 1
+native_color: FAKE3:1 allocation score on c7auto2: -INFINITY
+native_color: FAKE3:1 allocation score on c7auto3: -INFINITY
+native_color: FAKE3:2 allocation score on c7auto1: 0
+native_color: FAKE3:2 allocation score on c7auto2: 1
+native_color: FAKE3:2 allocation score on c7auto3: -INFINITY
+native_color: shooter allocation score on c7auto1: 0
+native_color: shooter allocation score on c7auto2: 0
+native_color: shooter allocation score on c7auto3: 0
diff --git a/pengine/test10/clone_min_interleave_stop_one.summary b/pengine/test10/clone_min_interleave_stop_one.summary
new file mode 100644
index 0000000..9280b7e
--- /dev/null
+++ b/pengine/test10/clone_min_interleave_stop_one.summary
@@ -0,0 +1,35 @@
+
+Current cluster status:
+Online: [ c7auto1 c7auto2 c7auto3 ]
+
+ shooter (stonith:fence_phd_kvm): Started c7auto1
+ Clone Set: FAKE1-clone [FAKE1]
+ Started: [ c7auto1 c7auto2 c7auto3 ]
+ Clone Set: FAKE2-clone [FAKE2]
+ Started: [ c7auto1 c7auto2 c7auto3 ]
+ Clone Set: FAKE3-clone [FAKE3]
+ Started: [ c7auto1 c7auto2 c7auto3 ]
+
+Transition Summary:
+ * Stop FAKE1:0 (c7auto3)
+
+Executing cluster transition:
+ * Pseudo action: FAKE1-clone_stop_0
+ * Resource action: FAKE1 stop on c7auto3
+ * Pseudo action: FAKE1-clone_stopped_0
+ * Pseudo action: FAKE1-clone_start_0
+ * Pseudo action: all_stopped
+ * Pseudo action: FAKE1-clone_running_0
+
+Revised cluster status:
+Online: [ c7auto1 c7auto2 c7auto3 ]
+
+ shooter (stonith:fence_phd_kvm): Started c7auto1
+ Clone Set: FAKE1-clone [FAKE1]
+ Started: [ c7auto1 c7auto2 ]
+ Stopped: [ c7auto3 ]
+ Clone Set: FAKE2-clone [FAKE2]
+ Started: [ c7auto1 c7auto2 c7auto3 ]
+ Clone Set: FAKE3-clone [FAKE3]
+ Started: [ c7auto1 c7auto2 c7auto3 ]
+
diff --git a/pengine/test10/clone_min_interleave_stop_one.xml b/pengine/test10/clone_min_interleave_stop_one.xml
new file mode 100644
index 0000000..31db5f3
--- /dev/null
+++ b/pengine/test10/clone_min_interleave_stop_one.xml
@@ -0,0 +1,153 @@
+<cib crm_feature_set="3.0.10" validate-with="pacemaker-2.3" epoch="12" num_updates="0" admin_epoch="0" cib-last-written="Fri Jul 17 11:30:45 2015" update-origin="c7auto1" update-client="cibadmin" update-user="root" have-quorum="1" dc-uuid="3">
+ <configuration>
+ <crm_config>
+ <cluster_property_set id="cib-bootstrap-options">
+ <nvpair id="cib-bootstrap-options-have-watchdog" name="have-watchdog" value="false"/>
+ <nvpair id="cib-bootstrap-options-dc-version" name="dc-version" value="1.1.12-ef86b49"/>
+ <nvpair id="cib-bootstrap-options-cluster-infrastructure" name="cluster-infrastructure" value="corosync"/>
+ <nvpair id="cib-bootstrap-options-cluster-name" name="cluster-name" value="phd"/>
+ </cluster_property_set>
+ </crm_config>
+ <nodes>
+ <node id="1" uname="c7auto1"/>
+ <node id="2" uname="c7auto2"/>
+ <node id="3" uname="c7auto3"/>
+ </nodes>
+ <resources>
+ <primitive class="stonith" id="shooter" type="fence_phd_kvm">
+ <instance_attributes id="shooter-instance_attributes"/>
+ <operations>
+ <op id="shooter-monitor-interval-60s" interval="60s" name="monitor"/>
+ </operations>
+ </primitive>
+ <clone id="FAKE1-clone">
+ <primitive class="ocf" id="FAKE1" provider="heartbeat" type="Dummy">
+ <instance_attributes id="FAKE1-instance_attributes"/>
+ <operations>
+ <op id="FAKE1-start-timeout-20" interval="0s" name="start" timeout="20"/>
+ <op id="FAKE1-stop-timeout-20" interval="0s" name="stop" timeout="20"/>
+ <op id="FAKE1-monitor-interval-10" interval="10" name="monitor" timeout="20"/>
+ </operations>
+ </primitive>
+ <meta_attributes id="FAKE1-clone-meta">
+ <nvpair id="FAKE1-clone-min" name="clone-min" value="2"/>
+ </meta_attributes>
+ </clone>
+ <clone id="FAKE2-clone">
+ <primitive class="ocf" id="FAKE2" provider="heartbeat" type="Dummy">
+ <instance_attributes id="FAKE2-instance_attributes"/>
+ <operations>
+ <op id="FAKE2-start-timeout-20" interval="0s" name="start" timeout="20"/>
+ <op id="FAKE2-stop-timeout-20" interval="0s" name="stop" timeout="20"/>
+ <op id="FAKE2-monitor-interval-10" interval="10" name="monitor" timeout="20"/>
+ </operations>
+ </primitive>
+ <meta_attributes id="FAKE2-clone-meta">
+ <nvpair id="FAKE2-interleave" name="interleave" value="true"/>
+ </meta_attributes>
+ </clone>
+ <clone id="FAKE3-clone">
+ <primitive class="ocf" id="FAKE3" provider="heartbeat" type="Dummy">
+ <instance_attributes id="FAKE3-instance_attributes"/>
+ <operations>
+ <op id="FAKE3-start-timeout-20" interval="0s" name="start" timeout="20"/>
+ <op id="FAKE3-stop-timeout-20" interval="0s" name="stop" timeout="20"/>
+ <op id="FAKE3-monitor-interval-10" interval="10" name="monitor" timeout="20"/>
+ </operations>
+ </primitive>
+ <meta_attributes id="FAKE3-clone-meta">
+ <nvpair id="FAKE3-interleave" name="interleave" value="true"/>
+ </meta_attributes>
+ </clone>
+ </resources>
+ <constraints>
+ <rsc_order first="FAKE1-clone" first-action="start" id="order-FAKE1-clone-FAKE2-clone-mandatory" then="FAKE2-clone" then-action="start"/>
+ <rsc_order first="FAKE2-clone" first-action="start" id="order-FAKE2-clone-FAKE3-clone-mandatory" then="FAKE3-clone" then-action="start"/>
+ <rsc_location id="cli-ban-FAKE1-clone-on-c7auto3" rsc="FAKE1-clone" role="Started" node="c7auto3" score="-INFINITY"/>
+ </constraints>
+ </configuration>
+ <status>
+ <node_state id="3" uname="c7auto3" in_ccm="true" crmd="online" crm-debug-origin="do_update_resource" join="member" expected="member">
+ <lrm id="3">
+ <lrm_resources>
+ <lrm_resource id="shooter" type="fence_phd_kvm" class="stonith">
+ <lrm_rsc_op id="shooter_last_0" operation_key="shooter_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="8:4:7:9d56c099-7067-4908-9f99-89a103eff57b" transition-magic="0:7;8:4:7:9d56c099-7067-4908-9f99-89a103eff57b" on_node="c7auto3" call-id="5" rc-code="7" op-status="0" interval="0" last-run="1437146780" last-rc-change="1437146780" exec-time="998" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ </lrm_resource>
+ <lrm_resource id="FAKE1" type="Dummy" class="ocf" provider="heartbeat">
+ <lrm_rsc_op id="FAKE1_last_0" operation_key="FAKE1_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="14:5:0:9d56c099-7067-4908-9f99-89a103eff57b" transition-magic="0:0;14:5:0:9d56c099-7067-4908-9f99-89a103eff57b" on_node="c7auto3" call-id="11" rc-code="0" op-status="0" interval="0" last-run="1437147006" last-rc-change="1437147006" exec-time="8" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" fake " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ <lrm_rsc_op id="FAKE1_monitor_10000" operation_key="FAKE1_monitor_10000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="15:5:0:9d56c099-7067-4908-9f99-89a103eff57b" transition-magic="0:0;15:5:0:9d56c099-7067-4908-9f99-89a103eff57b" on_node="c7auto3" call-id="12" rc-code="0" op-status="0" interval="10000" last-rc-change="1437147006" exec-time="9" queue-time="0" op-digest="4811cef7f7f94e3a35a70be7916cb2fd"/>
+ </lrm_resource>
+ <lrm_resource id="FAKE2" type="Dummy" class="ocf" provider="heartbeat">
+ <lrm_rsc_op id="FAKE2_last_0" operation_key="FAKE2_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="27:6:0:9d56c099-7067-4908-9f99-89a103eff57b" transition-magic="0:0;27:6:0:9d56c099-7067-4908-9f99-89a103eff57b" on_node="c7auto3" call-id="18" rc-code="0" op-status="0" interval="0" last-run="1437147019" last-rc-change="1437147019" exec-time="8" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" fake " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ <lrm_rsc_op id="FAKE2_monitor_10000" operation_key="FAKE2_monitor_10000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="28:6:0:9d56c099-7067-4908-9f99-89a103eff57b" transition-magic="0:0;28:6:0:9d56c099-7067-4908-9f99-89a103eff57b" on_node="c7auto3" call-id="19" rc-code="0" op-status="0" interval="10000" last-rc-change="1437147019" exec-time="8" queue-time="0" op-digest="4811cef7f7f94e3a35a70be7916cb2fd"/>
+ </lrm_resource>
+ <lrm_resource id="FAKE3" type="Dummy" class="ocf" provider="heartbeat">
+ <lrm_rsc_op id="FAKE3_last_0" operation_key="FAKE3_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="40:7:0:9d56c099-7067-4908-9f99-89a103eff57b" transition-magic="0:0;40:7:0:9d56c099-7067-4908-9f99-89a103eff57b" on_node="c7auto3" call-id="25" rc-code="0" op-status="0" interval="0" last-run="1437147024" last-rc-change="1437147024" exec-time="9" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" fake " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ <lrm_rsc_op id="FAKE3_monitor_10000" operation_key="FAKE3_monitor_10000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="41:7:0:9d56c099-7067-4908-9f99-89a103eff57b" transition-magic="0:0;41:7:0:9d56c099-7067-4908-9f99-89a103eff57b" on_node="c7auto3" call-id="26" rc-code="0" op-status="0" interval="10000" last-rc-change="1437147024" exec-time="8" queue-time="0" op-digest="4811cef7f7f94e3a35a70be7916cb2fd"/>
+ </lrm_resource>
+ </lrm_resources>
+ </lrm>
+ <transient_attributes id="3">
+ <instance_attributes id="status-3">
+ <nvpair id="status-3-shutdown" name="shutdown" value="0"/>
+ <nvpair id="status-3-probe_complete" name="probe_complete" value="true"/>
+ </instance_attributes>
+ </transient_attributes>
+ </node_state>
+ <node_state id="1" uname="c7auto1" in_ccm="true" crmd="online" crm-debug-origin="do_update_resource" join="member" expected="member">
+ <lrm id="1">
+ <lrm_resources>
+ <lrm_resource id="shooter" type="fence_phd_kvm" class="stonith">
+ <lrm_rsc_op id="shooter_last_0" operation_key="shooter_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="9:4:0:9d56c099-7067-4908-9f99-89a103eff57b" transition-magic="0:0;9:4:0:9d56c099-7067-4908-9f99-89a103eff57b" on_node="c7auto1" call-id="6" rc-code="0" op-status="0" interval="0" last-run="1437146782" last-rc-change="1437146782" exec-time="5" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ <lrm_rsc_op id="shooter_monitor_60000" operation_key="shooter_monitor_60000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="10:4:0:9d56c099-7067-4908-9f99-89a103eff57b" transition-magic="0:0;10:4:0:9d56c099-7067-4908-9f99-89a103eff57b" on_node="c7auto1" call-id="7" rc-code="0" op-status="0" interval="60000" last-rc-change="1437146783" exec-time="5" queue-time="1" op-digest="4811cef7f7f94e3a35a70be7916cb2fd"/>
+ </lrm_resource>
+ <lrm_resource id="FAKE1" type="Dummy" class="ocf" provider="heartbeat">
+ <lrm_rsc_op id="FAKE1_last_0" operation_key="FAKE1_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="16:5:0:9d56c099-7067-4908-9f99-89a103eff57b" transition-magic="0:0;16:5:0:9d56c099-7067-4908-9f99-89a103eff57b" on_node="c7auto1" call-id="13" rc-code="0" op-status="0" interval="0" last-run="1437147007" last-rc-change="1437147007" exec-time="8" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" fake " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ <lrm_rsc_op id="FAKE1_monitor_10000" operation_key="FAKE1_monitor_10000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="17:5:0:9d56c099-7067-4908-9f99-89a103eff57b" transition-magic="0:0;17:5:0:9d56c099-7067-4908-9f99-89a103eff57b" on_node="c7auto1" call-id="14" rc-code="0" op-status="0" interval="10000" last-rc-change="1437147007" exec-time="6" queue-time="0" op-digest="4811cef7f7f94e3a35a70be7916cb2fd"/>
+ </lrm_resource>
+ <lrm_resource id="FAKE2" type="Dummy" class="ocf" provider="heartbeat">
+ <lrm_rsc_op id="FAKE2_last_0" operation_key="FAKE2_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="29:6:0:9d56c099-7067-4908-9f99-89a103eff57b" transition-magic="0:0;29:6:0:9d56c099-7067-4908-9f99-89a103eff57b" on_node="c7auto1" call-id="20" rc-code="0" op-status="0" interval="0" last-run="1437147019" last-rc-change="1437147019" exec-time="7" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" fake " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ <lrm_rsc_op id="FAKE2_monitor_10000" operation_key="FAKE2_monitor_10000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="30:6:0:9d56c099-7067-4908-9f99-89a103eff57b" transition-magic="0:0;30:6:0:9d56c099-7067-4908-9f99-89a103eff57b" on_node="c7auto1" call-id="21" rc-code="0" op-status="0" interval="10000" last-rc-change="1437147019" exec-time="7" queue-time="0" op-digest="4811cef7f7f94e3a35a70be7916cb2fd"/>
+ </lrm_resource>
+ <lrm_resource id="FAKE3" type="Dummy" class="ocf" provider="heartbeat">
+ <lrm_rsc_op id="FAKE3_last_0" operation_key="FAKE3_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="42:7:0:9d56c099-7067-4908-9f99-89a103eff57b" transition-magic="0:0;42:7:0:9d56c099-7067-4908-9f99-89a103eff57b" on_node="c7auto1" call-id="27" rc-code="0" op-status="0" interval="0" last-run="1437147024" last-rc-change="1437147024" exec-time="8" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" fake " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ <lrm_rsc_op id="FAKE3_monitor_10000" operation_key="FAKE3_monitor_10000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="43:7:0:9d56c099-7067-4908-9f99-89a103eff57b" transition-magic="0:0;43:7:0:9d56c099-7067-4908-9f99-89a103eff57b" on_node="c7auto1" call-id="28" rc-code="0" op-status="0" interval="10000" last-rc-change="1437147024" exec-time="6" queue-time="1" op-digest="4811cef7f7f94e3a35a70be7916cb2fd"/>
+ </lrm_resource>
+ </lrm_resources>
+ </lrm>
+ <transient_attributes id="1">
+ <instance_attributes id="status-1">
+ <nvpair id="status-1-shutdown" name="shutdown" value="0"/>
+ <nvpair id="status-1-probe_complete" name="probe_complete" value="true"/>
+ </instance_attributes>
+ </transient_attributes>
+ </node_state>
+ <node_state id="2" uname="c7auto2" in_ccm="true" crmd="online" crm-debug-origin="do_update_resource" join="member" expected="member">
+ <lrm id="2">
+ <lrm_resources>
+ <lrm_resource id="shooter" type="fence_phd_kvm" class="stonith">
+ <lrm_rsc_op id="shooter_last_0" operation_key="shooter_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="6:4:7:9d56c099-7067-4908-9f99-89a103eff57b" transition-magic="0:7;6:4:7:9d56c099-7067-4908-9f99-89a103eff57b" on_node="c7auto2" call-id="5" rc-code="7" op-status="0" interval="0" last-run="1437146780" last-rc-change="1437146780" exec-time="994" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ </lrm_resource>
+ <lrm_resource id="FAKE1" type="Dummy" class="ocf" provider="heartbeat">
+ <lrm_rsc_op id="FAKE1_last_0" operation_key="FAKE1_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="12:5:0:9d56c099-7067-4908-9f99-89a103eff57b" transition-magic="0:0;12:5:0:9d56c099-7067-4908-9f99-89a103eff57b" on_node="c7auto2" call-id="11" rc-code="0" op-status="0" interval="0" last-run="1437147007" last-rc-change="1437147007" exec-time="6" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" fake " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ <lrm_rsc_op id="FAKE1_monitor_10000" operation_key="FAKE1_monitor_10000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="13:5:0:9d56c099-7067-4908-9f99-89a103eff57b" transition-magic="0:0;13:5:0:9d56c099-7067-4908-9f99-89a103eff57b" on_node="c7auto2" call-id="12" rc-code="0" op-status="0" interval="10000" last-rc-change="1437147007" exec-time="8" queue-time="0" op-digest="4811cef7f7f94e3a35a70be7916cb2fd"/>
+ </lrm_resource>
+ <lrm_resource id="FAKE2" type="Dummy" class="ocf" provider="heartbeat">
+ <lrm_rsc_op id="FAKE2_last_0" operation_key="FAKE2_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="25:6:0:9d56c099-7067-4908-9f99-89a103eff57b" transition-magic="0:0;25:6:0:9d56c099-7067-4908-9f99-89a103eff57b" on_node="c7auto2" call-id="18" rc-code="0" op-status="0" interval="0" last-run="1437147019" last-rc-change="1437147019" exec-time="8" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" fake " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ <lrm_rsc_op id="FAKE2_monitor_10000" operation_key="FAKE2_monitor_10000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="26:6:0:9d56c099-7067-4908-9f99-89a103eff57b" transition-magic="0:0;26:6:0:9d56c099-7067-4908-9f99-89a103eff57b" on_node="c7auto2" call-id="19" rc-code="0" op-status="0" interval="10000" last-rc-change="1437147019" exec-time="6" queue-time="0" op-digest="4811cef7f7f94e3a35a70be7916cb2fd"/>
+ </lrm_resource>
+ <lrm_resource id="FAKE3" type="Dummy" class="ocf" provider="heartbeat">
+ <lrm_rsc_op id="FAKE3_last_0" operation_key="FAKE3_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="38:7:0:9d56c099-7067-4908-9f99-89a103eff57b" transition-magic="0:0;38:7:0:9d56c099-7067-4908-9f99-89a103eff57b" on_node="c7auto2" call-id="25" rc-code="0" op-status="0" interval="0" last-run="1437147024" last-rc-change="1437147024" exec-time="7" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" fake " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ <lrm_rsc_op id="FAKE3_monitor_10000" operation_key="FAKE3_monitor_10000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="39:7:0:9d56c099-7067-4908-9f99-89a103eff57b" transition-magic="0:0;39:7:0:9d56c099-7067-4908-9f99-89a103eff57b" on_node="c7auto2" call-id="26" rc-code="0" op-status="0" interval="10000" last-rc-change="1437147024" exec-time="8" queue-time="0" op-digest="4811cef7f7f94e3a35a70be7916cb2fd"/>
+ </lrm_resource>
+ </lrm_resources>
+ </lrm>
+ <transient_attributes id="2">
+ <instance_attributes id="status-2">
+ <nvpair id="status-2-shutdown" name="shutdown" value="0"/>
+ <nvpair id="status-2-probe_complete" name="probe_complete" value="true"/>
+ </instance_attributes>
+ </transient_attributes>
+ </node_state>
+ </status>
+</cib>
diff --git a/pengine/test10/clone_min_interleave_stop_two.dot b/pengine/test10/clone_min_interleave_stop_two.dot
new file mode 100644
index 0000000..73f60dd
--- /dev/null
+++ b/pengine/test10/clone_min_interleave_stop_two.dot
@@ -0,0 +1,108 @@
+ digraph "g" {
+"FAKE1-clone_running_0" [ style=bold color="green" fontcolor="orange"]
+"FAKE1-clone_start_0" -> "FAKE1-clone_running_0" [ style = bold]
+"FAKE1-clone_start_0" -> "FAKE1_start_0 <none>" [ style = dashed]
+"FAKE1-clone_start_0" [ style=bold color="green" fontcolor="orange"]
+"FAKE1-clone_stop_0" -> "FAKE1-clone_stopped_0" [ style = bold]
+"FAKE1-clone_stop_0" -> "FAKE1_stop_0 c7auto2" [ style = bold]
+"FAKE1-clone_stop_0" -> "FAKE1_stop_0 c7auto3" [ style = bold]
+"FAKE1-clone_stop_0" [ style=bold color="green" fontcolor="orange"]
+"FAKE1-clone_stopped_0" -> "FAKE1-clone_start_0" [ style = bold]
+"FAKE1-clone_stopped_0" [ style=bold color="green" fontcolor="orange"]
+"FAKE1_start_0 <none>" -> "FAKE1-clone_running_0" [ style = dashed]
+"FAKE1_start_0 <none>" [ style=dashed color="red" fontcolor="black"]
+"FAKE1_stop_0 c7auto2" -> "FAKE1-clone_stopped_0" [ style = bold]
+"FAKE1_stop_0 c7auto2" -> "FAKE1_start_0 <none>" [ style = dashed]
+"FAKE1_stop_0 c7auto2" -> "all_stopped" [ style = bold]
+"FAKE1_stop_0 c7auto2" [ style=bold color="green" fontcolor="black"]
+"FAKE1_stop_0 c7auto3" -> "FAKE1-clone_stopped_0" [ style = bold]
+"FAKE1_stop_0 c7auto3" -> "FAKE1_start_0 <none>" [ style = dashed]
+"FAKE1_stop_0 c7auto3" -> "all_stopped" [ style = bold]
+"FAKE1_stop_0 c7auto3" [ style=bold color="green" fontcolor="black"]
+"FAKE2-clone_running_0" -> "FAKE3-clone_start_0" [ style = dashed]
+"FAKE2-clone_running_0" [ style=dashed color="red" fontcolor="orange"]
+"FAKE2-clone_start_0" -> "FAKE2-clone_running_0" [ style = dashed]
+"FAKE2-clone_start_0" -> "FAKE2_start_0 c7auto1" [ style = dashed]
+"FAKE2-clone_start_0" -> "FAKE2_start_0 c7auto2" [ style = dashed]
+"FAKE2-clone_start_0" -> "FAKE2_start_0 c7auto3" [ style = dashed]
+"FAKE2-clone_start_0" [ style=dashed color="red" fontcolor="orange"]
+"FAKE2-clone_stop_0" -> "FAKE2-clone_stopped_0" [ style = bold]
+"FAKE2-clone_stop_0" -> "FAKE2_stop_0 c7auto1" [ style = bold]
+"FAKE2-clone_stop_0" -> "FAKE2_stop_0 c7auto2" [ style = bold]
+"FAKE2-clone_stop_0" -> "FAKE2_stop_0 c7auto3" [ style = bold]
+"FAKE2-clone_stop_0" [ style=bold color="green" fontcolor="orange"]
+"FAKE2-clone_stopped_0" -> "FAKE1-clone_stop_0" [ style = bold]
+"FAKE2-clone_stopped_0" -> "FAKE2-clone_start_0" [ style = dashed]
+"FAKE2-clone_stopped_0" [ style=bold color="green" fontcolor="orange"]
+"FAKE2_monitor_10000 c7auto1" [ style=dashed color="red" fontcolor="black"]
+"FAKE2_monitor_10000 c7auto2" [ style=dashed color="red" fontcolor="black"]
+"FAKE2_monitor_10000 c7auto3" [ style=dashed color="red" fontcolor="black"]
+"FAKE2_start_0 c7auto1" -> "FAKE2-clone_running_0" [ style = dashed]
+"FAKE2_start_0 c7auto1" -> "FAKE2_monitor_10000 c7auto1" [ style = dashed]
+"FAKE2_start_0 c7auto1" -> "FAKE3_start_0 c7auto1" [ style = dashed]
+"FAKE2_start_0 c7auto1" [ style=dashed color="red" fontcolor="black"]
+"FAKE2_start_0 c7auto2" -> "FAKE2-clone_running_0" [ style = dashed]
+"FAKE2_start_0 c7auto2" -> "FAKE2_monitor_10000 c7auto2" [ style = dashed]
+"FAKE2_start_0 c7auto2" -> "FAKE3_start_0 c7auto2" [ style = dashed]
+"FAKE2_start_0 c7auto2" [ style=dashed color="red" fontcolor="black"]
+"FAKE2_start_0 c7auto3" -> "FAKE2-clone_running_0" [ style = dashed]
+"FAKE2_start_0 c7auto3" -> "FAKE2_monitor_10000 c7auto3" [ style = dashed]
+"FAKE2_start_0 c7auto3" -> "FAKE3_start_0 c7auto3" [ style = dashed]
+"FAKE2_start_0 c7auto3" [ style=dashed color="red" fontcolor="black"]
+"FAKE2_stop_0 c7auto1" -> "FAKE2-clone_stopped_0" [ style = bold]
+"FAKE2_stop_0 c7auto1" -> "FAKE2_start_0 c7auto1" [ style = dashed]
+"FAKE2_stop_0 c7auto1" -> "all_stopped" [ style = bold]
+"FAKE2_stop_0 c7auto1" [ style=bold color="green" fontcolor="black"]
+"FAKE2_stop_0 c7auto2" -> "FAKE1_stop_0 c7auto2" [ style = bold]
+"FAKE2_stop_0 c7auto2" -> "FAKE2-clone_stopped_0" [ style = bold]
+"FAKE2_stop_0 c7auto2" -> "FAKE2_start_0 c7auto2" [ style = dashed]
+"FAKE2_stop_0 c7auto2" -> "all_stopped" [ style = bold]
+"FAKE2_stop_0 c7auto2" [ style=bold color="green" fontcolor="black"]
+"FAKE2_stop_0 c7auto3" -> "FAKE1_stop_0 c7auto3" [ style = bold]
+"FAKE2_stop_0 c7auto3" -> "FAKE2-clone_stopped_0" [ style = bold]
+"FAKE2_stop_0 c7auto3" -> "FAKE2_start_0 c7auto3" [ style = dashed]
+"FAKE2_stop_0 c7auto3" -> "all_stopped" [ style = bold]
+"FAKE2_stop_0 c7auto3" [ style=bold color="green" fontcolor="black"]
+"FAKE3-clone_running_0" [ style=dashed color="red" fontcolor="orange"]
+"FAKE3-clone_start_0" -> "FAKE3-clone_running_0" [ style = dashed]
+"FAKE3-clone_start_0" -> "FAKE3_start_0 c7auto1" [ style = dashed]
+"FAKE3-clone_start_0" -> "FAKE3_start_0 c7auto2" [ style = dashed]
+"FAKE3-clone_start_0" -> "FAKE3_start_0 c7auto3" [ style = dashed]
+"FAKE3-clone_start_0" [ style=dashed color="red" fontcolor="orange"]
+"FAKE3-clone_stop_0" -> "FAKE3-clone_stopped_0" [ style = bold]
+"FAKE3-clone_stop_0" -> "FAKE3_stop_0 c7auto1" [ style = bold]
+"FAKE3-clone_stop_0" -> "FAKE3_stop_0 c7auto2" [ style = bold]
+"FAKE3-clone_stop_0" -> "FAKE3_stop_0 c7auto3" [ style = bold]
+"FAKE3-clone_stop_0" [ style=bold color="green" fontcolor="orange"]
+"FAKE3-clone_stopped_0" -> "FAKE2-clone_stop_0" [ style = bold]
+"FAKE3-clone_stopped_0" -> "FAKE3-clone_start_0" [ style = dashed]
+"FAKE3-clone_stopped_0" [ style=bold color="green" fontcolor="orange"]
+"FAKE3_monitor_10000 c7auto1" [ style=dashed color="red" fontcolor="black"]
+"FAKE3_monitor_10000 c7auto2" [ style=dashed color="red" fontcolor="black"]
+"FAKE3_monitor_10000 c7auto3" [ style=dashed color="red" fontcolor="black"]
+"FAKE3_start_0 c7auto1" -> "FAKE3-clone_running_0" [ style = dashed]
+"FAKE3_start_0 c7auto1" -> "FAKE3_monitor_10000 c7auto1" [ style = dashed]
+"FAKE3_start_0 c7auto1" [ style=dashed color="red" fontcolor="black"]
+"FAKE3_start_0 c7auto2" -> "FAKE3-clone_running_0" [ style = dashed]
+"FAKE3_start_0 c7auto2" -> "FAKE3_monitor_10000 c7auto2" [ style = dashed]
+"FAKE3_start_0 c7auto2" [ style=dashed color="red" fontcolor="black"]
+"FAKE3_start_0 c7auto3" -> "FAKE3-clone_running_0" [ style = dashed]
+"FAKE3_start_0 c7auto3" -> "FAKE3_monitor_10000 c7auto3" [ style = dashed]
+"FAKE3_start_0 c7auto3" [ style=dashed color="red" fontcolor="black"]
+"FAKE3_stop_0 c7auto1" -> "FAKE2_stop_0 c7auto1" [ style = bold]
+"FAKE3_stop_0 c7auto1" -> "FAKE3-clone_stopped_0" [ style = bold]
+"FAKE3_stop_0 c7auto1" -> "FAKE3_start_0 c7auto1" [ style = dashed]
+"FAKE3_stop_0 c7auto1" -> "all_stopped" [ style = bold]
+"FAKE3_stop_0 c7auto1" [ style=bold color="green" fontcolor="black"]
+"FAKE3_stop_0 c7auto2" -> "FAKE2_stop_0 c7auto2" [ style = bold]
+"FAKE3_stop_0 c7auto2" -> "FAKE3-clone_stopped_0" [ style = bold]
+"FAKE3_stop_0 c7auto2" -> "FAKE3_start_0 c7auto2" [ style = dashed]
+"FAKE3_stop_0 c7auto2" -> "all_stopped" [ style = bold]
+"FAKE3_stop_0 c7auto2" [ style=bold color="green" fontcolor="black"]
+"FAKE3_stop_0 c7auto3" -> "FAKE2_stop_0 c7auto3" [ style = bold]
+"FAKE3_stop_0 c7auto3" -> "FAKE3-clone_stopped_0" [ style = bold]
+"FAKE3_stop_0 c7auto3" -> "FAKE3_start_0 c7auto3" [ style = dashed]
+"FAKE3_stop_0 c7auto3" -> "all_stopped" [ style = bold]
+"FAKE3_stop_0 c7auto3" [ style=bold color="green" fontcolor="black"]
+"all_stopped" [ style=bold color="green" fontcolor="orange"]
+}
diff --git a/pengine/test10/clone_min_interleave_stop_two.exp b/pengine/test10/clone_min_interleave_stop_two.exp
new file mode 100644
index 0000000..62fe1e6
--- /dev/null
+++ b/pengine/test10/clone_min_interleave_stop_two.exp
@@ -0,0 +1,270 @@
+<transition_graph cluster-delay="60s" stonith-timeout="60s" failed-stop-offset="INFINITY" failed-start-offset="INFINITY" transition_id="0">
+ <synapse id="0">
+ <action_set>
+ <rsc_op id="19" operation="stop" operation_key="FAKE1_stop_0" internal_operation_key="FAKE1:0_stop_0" on_node="c7auto3" on_node_uuid="3">
+ <primitive id="FAKE1" long-id="FAKE1:0" class="ocf" provider="heartbeat" type="Dummy"/>
+ <attributes CRM_meta_clone="0" CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_name="stop" CRM_meta_notify="false" CRM_meta_timeout="20000" />
+ </rsc_op>
+ </action_set>
+ <inputs>
+ <trigger>
+ <pseudo_event id="25" operation="stop" operation_key="FAKE1-clone_stop_0"/>
+ </trigger>
+ <trigger>
+ <rsc_op id="27" operation="stop" operation_key="FAKE2_stop_0" internal_operation_key="FAKE2:0_stop_0" on_node="c7auto3" on_node_uuid="3"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="1">
+ <action_set>
+ <rsc_op id="22" operation="stop" operation_key="FAKE1_stop_0" internal_operation_key="FAKE1:2_stop_0" on_node="c7auto2" on_node_uuid="2">
+ <primitive id="FAKE1" long-id="FAKE1:2" class="ocf" provider="heartbeat" type="Dummy"/>
+ <attributes CRM_meta_clone="2" CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_name="stop" CRM_meta_notify="false" CRM_meta_timeout="20000" />
+ </rsc_op>
+ </action_set>
+ <inputs>
+ <trigger>
+ <pseudo_event id="25" operation="stop" operation_key="FAKE1-clone_stop_0"/>
+ </trigger>
+ <trigger>
+ <rsc_op id="31" operation="stop" operation_key="FAKE2_stop_0" internal_operation_key="FAKE2:2_stop_0" on_node="c7auto2" on_node_uuid="2"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="2" priority="1000000">
+ <action_set>
+ <pseudo_event id="26" operation="stopped" operation_key="FAKE1-clone_stopped_0">
+ <attributes CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="false" CRM_meta_timeout="20000" />
+ </pseudo_event>
+ </action_set>
+ <inputs>
+ <trigger>
+ <rsc_op id="19" operation="stop" operation_key="FAKE1_stop_0" internal_operation_key="FAKE1:0_stop_0" on_node="c7auto3" on_node_uuid="3"/>
+ </trigger>
+ <trigger>
+ <rsc_op id="22" operation="stop" operation_key="FAKE1_stop_0" internal_operation_key="FAKE1:2_stop_0" on_node="c7auto2" on_node_uuid="2"/>
+ </trigger>
+ <trigger>
+ <pseudo_event id="25" operation="stop" operation_key="FAKE1-clone_stop_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="3">
+ <action_set>
+ <pseudo_event id="25" operation="stop" operation_key="FAKE1-clone_stop_0">
+ <attributes CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="false" CRM_meta_timeout="20000" />
+ </pseudo_event>
+ </action_set>
+ <inputs>
+ <trigger>
+ <pseudo_event id="36" operation="stopped" operation_key="FAKE2-clone_stopped_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="4" priority="1000000">
+ <action_set>
+ <pseudo_event id="24" operation="running" operation_key="FAKE1-clone_running_0">
+ <attributes CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="false" CRM_meta_timeout="20000" />
+ </pseudo_event>
+ </action_set>
+ <inputs>
+ <trigger>
+ <pseudo_event id="23" operation="start" operation_key="FAKE1-clone_start_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="5">
+ <action_set>
+ <pseudo_event id="23" operation="start" operation_key="FAKE1-clone_start_0">
+ <attributes CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="false" CRM_meta_timeout="20000" />
+ </pseudo_event>
+ </action_set>
+ <inputs>
+ <trigger>
+ <pseudo_event id="26" operation="stopped" operation_key="FAKE1-clone_stopped_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="6">
+ <action_set>
+ <rsc_op id="27" operation="stop" operation_key="FAKE2_stop_0" internal_operation_key="FAKE2:0_stop_0" on_node="c7auto3" on_node_uuid="3">
+ <primitive id="FAKE2" long-id="FAKE2:0" class="ocf" provider="heartbeat" type="Dummy"/>
+ <attributes CRM_meta_clone="0" CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_name="stop" CRM_meta_notify="false" CRM_meta_timeout="20000" />
+ </rsc_op>
+ </action_set>
+ <inputs>
+ <trigger>
+ <pseudo_event id="35" operation="stop" operation_key="FAKE2-clone_stop_0"/>
+ </trigger>
+ <trigger>
+ <rsc_op id="37" operation="stop" operation_key="FAKE3_stop_0" internal_operation_key="FAKE3:0_stop_0" on_node="c7auto3" on_node_uuid="3"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="7">
+ <action_set>
+ <rsc_op id="29" operation="stop" operation_key="FAKE2_stop_0" internal_operation_key="FAKE2:1_stop_0" on_node="c7auto1" on_node_uuid="1">
+ <primitive id="FAKE2" long-id="FAKE2:1" class="ocf" provider="heartbeat" type="Dummy"/>
+ <attributes CRM_meta_clone="1" CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_name="stop" CRM_meta_notify="false" CRM_meta_timeout="20000" />
+ </rsc_op>
+ </action_set>
+ <inputs>
+ <trigger>
+ <pseudo_event id="35" operation="stop" operation_key="FAKE2-clone_stop_0"/>
+ </trigger>
+ <trigger>
+ <rsc_op id="39" operation="stop" operation_key="FAKE3_stop_0" internal_operation_key="FAKE3:1_stop_0" on_node="c7auto1" on_node_uuid="1"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="8">
+ <action_set>
+ <rsc_op id="31" operation="stop" operation_key="FAKE2_stop_0" internal_operation_key="FAKE2:2_stop_0" on_node="c7auto2" on_node_uuid="2">
+ <primitive id="FAKE2" long-id="FAKE2:2" class="ocf" provider="heartbeat" type="Dummy"/>
+ <attributes CRM_meta_clone="2" CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_name="stop" CRM_meta_notify="false" CRM_meta_timeout="20000" />
+ </rsc_op>
+ </action_set>
+ <inputs>
+ <trigger>
+ <pseudo_event id="35" operation="stop" operation_key="FAKE2-clone_stop_0"/>
+ </trigger>
+ <trigger>
+ <rsc_op id="41" operation="stop" operation_key="FAKE3_stop_0" internal_operation_key="FAKE3:2_stop_0" on_node="c7auto2" on_node_uuid="2"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="9" priority="1000000">
+ <action_set>
+ <pseudo_event id="36" operation="stopped" operation_key="FAKE2-clone_stopped_0">
+ <attributes CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="false" CRM_meta_timeout="20000" />
+ </pseudo_event>
+ </action_set>
+ <inputs>
+ <trigger>
+ <rsc_op id="27" operation="stop" operation_key="FAKE2_stop_0" internal_operation_key="FAKE2:0_stop_0" on_node="c7auto3" on_node_uuid="3"/>
+ </trigger>
+ <trigger>
+ <rsc_op id="29" operation="stop" operation_key="FAKE2_stop_0" internal_operation_key="FAKE2:1_stop_0" on_node="c7auto1" on_node_uuid="1"/>
+ </trigger>
+ <trigger>
+ <rsc_op id="31" operation="stop" operation_key="FAKE2_stop_0" internal_operation_key="FAKE2:2_stop_0" on_node="c7auto2" on_node_uuid="2"/>
+ </trigger>
+ <trigger>
+ <pseudo_event id="35" operation="stop" operation_key="FAKE2-clone_stop_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="10">
+ <action_set>
+ <pseudo_event id="35" operation="stop" operation_key="FAKE2-clone_stop_0">
+ <attributes CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="false" CRM_meta_timeout="20000" />
+ </pseudo_event>
+ </action_set>
+ <inputs>
+ <trigger>
+ <pseudo_event id="46" operation="stopped" operation_key="FAKE3-clone_stopped_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="11">
+ <action_set>
+ <rsc_op id="37" operation="stop" operation_key="FAKE3_stop_0" internal_operation_key="FAKE3:0_stop_0" on_node="c7auto3" on_node_uuid="3">
+ <primitive id="FAKE3" long-id="FAKE3:0" class="ocf" provider="heartbeat" type="Dummy"/>
+ <attributes CRM_meta_clone="0" CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_name="stop" CRM_meta_notify="false" CRM_meta_timeout="20000" />
+ </rsc_op>
+ </action_set>
+ <inputs>
+ <trigger>
+ <pseudo_event id="45" operation="stop" operation_key="FAKE3-clone_stop_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="12">
+ <action_set>
+ <rsc_op id="39" operation="stop" operation_key="FAKE3_stop_0" internal_operation_key="FAKE3:1_stop_0" on_node="c7auto1" on_node_uuid="1">
+ <primitive id="FAKE3" long-id="FAKE3:1" class="ocf" provider="heartbeat" type="Dummy"/>
+ <attributes CRM_meta_clone="1" CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_name="stop" CRM_meta_notify="false" CRM_meta_timeout="20000" />
+ </rsc_op>
+ </action_set>
+ <inputs>
+ <trigger>
+ <pseudo_event id="45" operation="stop" operation_key="FAKE3-clone_stop_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="13">
+ <action_set>
+ <rsc_op id="41" operation="stop" operation_key="FAKE3_stop_0" internal_operation_key="FAKE3:2_stop_0" on_node="c7auto2" on_node_uuid="2">
+ <primitive id="FAKE3" long-id="FAKE3:2" class="ocf" provider="heartbeat" type="Dummy"/>
+ <attributes CRM_meta_clone="2" CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_name="stop" CRM_meta_notify="false" CRM_meta_timeout="20000" />
+ </rsc_op>
+ </action_set>
+ <inputs>
+ <trigger>
+ <pseudo_event id="45" operation="stop" operation_key="FAKE3-clone_stop_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="14" priority="1000000">
+ <action_set>
+ <pseudo_event id="46" operation="stopped" operation_key="FAKE3-clone_stopped_0">
+ <attributes CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="false" CRM_meta_timeout="20000" />
+ </pseudo_event>
+ </action_set>
+ <inputs>
+ <trigger>
+ <rsc_op id="37" operation="stop" operation_key="FAKE3_stop_0" internal_operation_key="FAKE3:0_stop_0" on_node="c7auto3" on_node_uuid="3"/>
+ </trigger>
+ <trigger>
+ <rsc_op id="39" operation="stop" operation_key="FAKE3_stop_0" internal_operation_key="FAKE3:1_stop_0" on_node="c7auto1" on_node_uuid="1"/>
+ </trigger>
+ <trigger>
+ <rsc_op id="41" operation="stop" operation_key="FAKE3_stop_0" internal_operation_key="FAKE3:2_stop_0" on_node="c7auto2" on_node_uuid="2"/>
+ </trigger>
+ <trigger>
+ <pseudo_event id="45" operation="stop" operation_key="FAKE3-clone_stop_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="15">
+ <action_set>
+ <pseudo_event id="45" operation="stop" operation_key="FAKE3-clone_stop_0">
+ <attributes CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="false" CRM_meta_timeout="20000" />
+ </pseudo_event>
+ </action_set>
+ <inputs/>
+ </synapse>
+ <synapse id="16">
+ <action_set>
+ <pseudo_event id="12" operation="all_stopped" operation_key="all_stopped">
+ <attributes />
+ </pseudo_event>
+ </action_set>
+ <inputs>
+ <trigger>
+ <rsc_op id="19" operation="stop" operation_key="FAKE1_stop_0" internal_operation_key="FAKE1:0_stop_0" on_node="c7auto3" on_node_uuid="3"/>
+ </trigger>
+ <trigger>
+ <rsc_op id="22" operation="stop" operation_key="FAKE1_stop_0" internal_operation_key="FAKE1:2_stop_0" on_node="c7auto2" on_node_uuid="2"/>
+ </trigger>
+ <trigger>
+ <rsc_op id="27" operation="stop" operation_key="FAKE2_stop_0" internal_operation_key="FAKE2:0_stop_0" on_node="c7auto3" on_node_uuid="3"/>
+ </trigger>
+ <trigger>
+ <rsc_op id="29" operation="stop" operation_key="FAKE2_stop_0" internal_operation_key="FAKE2:1_stop_0" on_node="c7auto1" on_node_uuid="1"/>
+ </trigger>
+ <trigger>
+ <rsc_op id="31" operation="stop" operation_key="FAKE2_stop_0" internal_operation_key="FAKE2:2_stop_0" on_node="c7auto2" on_node_uuid="2"/>
+ </trigger>
+ <trigger>
+ <rsc_op id="37" operation="stop" operation_key="FAKE3_stop_0" internal_operation_key="FAKE3:0_stop_0" on_node="c7auto3" on_node_uuid="3"/>
+ </trigger>
+ <trigger>
+ <rsc_op id="39" operation="stop" operation_key="FAKE3_stop_0" internal_operation_key="FAKE3:1_stop_0" on_node="c7auto1" on_node_uuid="1"/>
+ </trigger>
+ <trigger>
+ <rsc_op id="41" operation="stop" operation_key="FAKE3_stop_0" internal_operation_key="FAKE3:2_stop_0" on_node="c7auto2" on_node_uuid="2"/>
+ </trigger>
+ </inputs>
+ </synapse>
+</transition_graph>
diff --git a/pengine/test10/clone_min_interleave_stop_two.scores b/pengine/test10/clone_min_interleave_stop_two.scores
new file mode 100644
index 0000000..ee7df92
--- /dev/null
+++ b/pengine/test10/clone_min_interleave_stop_two.scores
@@ -0,0 +1,67 @@
+Allocation scores:
+clone_color: FAKE1-clone allocation score on c7auto1: 0
+clone_color: FAKE1-clone allocation score on c7auto2: -INFINITY
+clone_color: FAKE1-clone allocation score on c7auto3: -INFINITY
+clone_color: FAKE1:0 allocation score on c7auto1: 0
+clone_color: FAKE1:0 allocation score on c7auto2: -INFINITY
+clone_color: FAKE1:0 allocation score on c7auto3: -INFINITY
+clone_color: FAKE1:1 allocation score on c7auto1: 1
+clone_color: FAKE1:1 allocation score on c7auto2: -INFINITY
+clone_color: FAKE1:1 allocation score on c7auto3: -INFINITY
+clone_color: FAKE1:2 allocation score on c7auto1: 0
+clone_color: FAKE1:2 allocation score on c7auto2: -INFINITY
+clone_color: FAKE1:2 allocation score on c7auto3: -INFINITY
+clone_color: FAKE2-clone allocation score on c7auto1: 0
+clone_color: FAKE2-clone allocation score on c7auto2: 0
+clone_color: FAKE2-clone allocation score on c7auto3: 0
+clone_color: FAKE2:0 allocation score on c7auto1: 0
+clone_color: FAKE2:0 allocation score on c7auto2: 0
+clone_color: FAKE2:0 allocation score on c7auto3: 1
+clone_color: FAKE2:1 allocation score on c7auto1: 1
+clone_color: FAKE2:1 allocation score on c7auto2: 0
+clone_color: FAKE2:1 allocation score on c7auto3: 0
+clone_color: FAKE2:2 allocation score on c7auto1: 0
+clone_color: FAKE2:2 allocation score on c7auto2: 1
+clone_color: FAKE2:2 allocation score on c7auto3: 0
+clone_color: FAKE3-clone allocation score on c7auto1: 0
+clone_color: FAKE3-clone allocation score on c7auto2: 0
+clone_color: FAKE3-clone allocation score on c7auto3: 0
+clone_color: FAKE3:0 allocation score on c7auto1: 0
+clone_color: FAKE3:0 allocation score on c7auto2: 0
+clone_color: FAKE3:0 allocation score on c7auto3: 1
+clone_color: FAKE3:1 allocation score on c7auto1: 1
+clone_color: FAKE3:1 allocation score on c7auto2: 0
+clone_color: FAKE3:1 allocation score on c7auto3: 0
+clone_color: FAKE3:2 allocation score on c7auto1: 0
+clone_color: FAKE3:2 allocation score on c7auto2: 1
+clone_color: FAKE3:2 allocation score on c7auto3: 0
+native_color: FAKE1:0 allocation score on c7auto1: -INFINITY
+native_color: FAKE1:0 allocation score on c7auto2: -INFINITY
+native_color: FAKE1:0 allocation score on c7auto3: -INFINITY
+native_color: FAKE1:1 allocation score on c7auto1: 1
+native_color: FAKE1:1 allocation score on c7auto2: -INFINITY
+native_color: FAKE1:1 allocation score on c7auto3: -INFINITY
+native_color: FAKE1:2 allocation score on c7auto1: -INFINITY
+native_color: FAKE1:2 allocation score on c7auto2: -INFINITY
+native_color: FAKE1:2 allocation score on c7auto3: -INFINITY
+native_color: FAKE2:0 allocation score on c7auto1: 0
+native_color: FAKE2:0 allocation score on c7auto2: -INFINITY
+native_color: FAKE2:0 allocation score on c7auto3: 1
+native_color: FAKE2:1 allocation score on c7auto1: 1
+native_color: FAKE2:1 allocation score on c7auto2: -INFINITY
+native_color: FAKE2:1 allocation score on c7auto3: -INFINITY
+native_color: FAKE2:2 allocation score on c7auto1: 0
+native_color: FAKE2:2 allocation score on c7auto2: 1
+native_color: FAKE2:2 allocation score on c7auto3: 0
+native_color: FAKE3:0 allocation score on c7auto1: 0
+native_color: FAKE3:0 allocation score on c7auto2: -INFINITY
+native_color: FAKE3:0 allocation score on c7auto3: 1
+native_color: FAKE3:1 allocation score on c7auto1: 1
+native_color: FAKE3:1 allocation score on c7auto2: -INFINITY
+native_color: FAKE3:1 allocation score on c7auto3: -INFINITY
+native_color: FAKE3:2 allocation score on c7auto1: 0
+native_color: FAKE3:2 allocation score on c7auto2: 1
+native_color: FAKE3:2 allocation score on c7auto3: 0
+native_color: shooter allocation score on c7auto1: 0
+native_color: shooter allocation score on c7auto2: 0
+native_color: shooter allocation score on c7auto3: 0
diff --git a/pengine/test10/clone_min_interleave_stop_two.summary b/pengine/test10/clone_min_interleave_stop_two.summary
new file mode 100644
index 0000000..fb28e0d
--- /dev/null
+++ b/pengine/test10/clone_min_interleave_stop_two.summary
@@ -0,0 +1,53 @@
+
+Current cluster status:
+Online: [ c7auto1 c7auto2 c7auto3 ]
+
+ shooter (stonith:fence_phd_kvm): Started c7auto1
+ Clone Set: FAKE1-clone [FAKE1]
+ Started: [ c7auto1 c7auto2 c7auto3 ]
+ Clone Set: FAKE2-clone [FAKE2]
+ Started: [ c7auto1 c7auto2 c7auto3 ]
+ Clone Set: FAKE3-clone [FAKE3]
+ Started: [ c7auto1 c7auto2 c7auto3 ]
+
+Transition Summary:
+ * Stop FAKE1:0 (c7auto3)
+ * Stop FAKE1:2 (c7auto2)
+ * Stop FAKE2:0 (Started c7auto3)
+ * Stop FAKE2:1 (Started c7auto1)
+ * Stop FAKE2:2 (Started c7auto2)
+ * Stop FAKE3:0 (Started c7auto3)
+ * Stop FAKE3:1 (Started c7auto1)
+ * Stop FAKE3:2 (Started c7auto2)
+
+Executing cluster transition:
+ * Pseudo action: FAKE3-clone_stop_0
+ * Resource action: FAKE3 stop on c7auto3
+ * Resource action: FAKE3 stop on c7auto1
+ * Resource action: FAKE3 stop on c7auto2
+ * Pseudo action: FAKE3-clone_stopped_0
+ * Pseudo action: FAKE2-clone_stop_0
+ * Resource action: FAKE2 stop on c7auto3
+ * Resource action: FAKE2 stop on c7auto1
+ * Resource action: FAKE2 stop on c7auto2
+ * Pseudo action: FAKE2-clone_stopped_0
+ * Pseudo action: FAKE1-clone_stop_0
+ * Resource action: FAKE1 stop on c7auto3
+ * Resource action: FAKE1 stop on c7auto2
+ * Pseudo action: FAKE1-clone_stopped_0
+ * Pseudo action: FAKE1-clone_start_0
+ * Pseudo action: all_stopped
+ * Pseudo action: FAKE1-clone_running_0
+
+Revised cluster status:
+Online: [ c7auto1 c7auto2 c7auto3 ]
+
+ shooter (stonith:fence_phd_kvm): Started c7auto1
+ Clone Set: FAKE1-clone [FAKE1]
+ Started: [ c7auto1 ]
+ Stopped: [ c7auto2 c7auto3 ]
+ Clone Set: FAKE2-clone [FAKE2]
+ Stopped: [ c7auto1 c7auto2 c7auto3 ]
+ Clone Set: FAKE3-clone [FAKE3]
+ Stopped: [ c7auto1 c7auto2 c7auto3 ]
+
diff --git a/pengine/test10/clone_min_interleave_stop_two.xml b/pengine/test10/clone_min_interleave_stop_two.xml
new file mode 100644
index 0000000..32c2b3b
--- /dev/null
+++ b/pengine/test10/clone_min_interleave_stop_two.xml
@@ -0,0 +1,154 @@
+<cib crm_feature_set="3.0.10" validate-with="pacemaker-2.3" epoch="13" num_updates="0" admin_epoch="0" cib-last-written="Fri Jul 17 11:30:45 2015" update-origin="c7auto1" update-client="cibadmin" update-user="root" have-quorum="1" dc-uuid="3">
+ <configuration>
+ <crm_config>
+ <cluster_property_set id="cib-bootstrap-options">
+ <nvpair id="cib-bootstrap-options-have-watchdog" name="have-watchdog" value="false"/>
+ <nvpair id="cib-bootstrap-options-dc-version" name="dc-version" value="1.1.12-ef86b49"/>
+ <nvpair id="cib-bootstrap-options-cluster-infrastructure" name="cluster-infrastructure" value="corosync"/>
+ <nvpair id="cib-bootstrap-options-cluster-name" name="cluster-name" value="phd"/>
+ </cluster_property_set>
+ </crm_config>
+ <nodes>
+ <node id="1" uname="c7auto1"/>
+ <node id="2" uname="c7auto2"/>
+ <node id="3" uname="c7auto3"/>
+ </nodes>
+ <resources>
+ <primitive class="stonith" id="shooter" type="fence_phd_kvm">
+ <instance_attributes id="shooter-instance_attributes"/>
+ <operations>
+ <op id="shooter-monitor-interval-60s" interval="60s" name="monitor"/>
+ </operations>
+ </primitive>
+ <clone id="FAKE1-clone">
+ <primitive class="ocf" id="FAKE1" provider="heartbeat" type="Dummy">
+ <instance_attributes id="FAKE1-instance_attributes"/>
+ <operations>
+ <op id="FAKE1-start-timeout-20" interval="0s" name="start" timeout="20"/>
+ <op id="FAKE1-stop-timeout-20" interval="0s" name="stop" timeout="20"/>
+ <op id="FAKE1-monitor-interval-10" interval="10" name="monitor" timeout="20"/>
+ </operations>
+ </primitive>
+ <meta_attributes id="FAKE1-clone-meta">
+ <nvpair id="FAKE1-clone-min" name="clone-min" value="2"/>
+ </meta_attributes>
+ </clone>
+ <clone id="FAKE2-clone">
+ <primitive class="ocf" id="FAKE2" provider="heartbeat" type="Dummy">
+ <instance_attributes id="FAKE2-instance_attributes"/>
+ <operations>
+ <op id="FAKE2-start-timeout-20" interval="0s" name="start" timeout="20"/>
+ <op id="FAKE2-stop-timeout-20" interval="0s" name="stop" timeout="20"/>
+ <op id="FAKE2-monitor-interval-10" interval="10" name="monitor" timeout="20"/>
+ </operations>
+ </primitive>
+ <meta_attributes id="FAKE2-clone-meta">
+ <nvpair id="FAKE2-interleave" name="interleave" value="true"/>
+ </meta_attributes>
+ </clone>
+ <clone id="FAKE3-clone">
+ <primitive class="ocf" id="FAKE3" provider="heartbeat" type="Dummy">
+ <instance_attributes id="FAKE3-instance_attributes"/>
+ <operations>
+ <op id="FAKE3-start-timeout-20" interval="0s" name="start" timeout="20"/>
+ <op id="FAKE3-stop-timeout-20" interval="0s" name="stop" timeout="20"/>
+ <op id="FAKE3-monitor-interval-10" interval="10" name="monitor" timeout="20"/>
+ </operations>
+ </primitive>
+ <meta_attributes id="FAKE3-clone-meta">
+ <nvpair id="FAKE3-interleave" name="interleave" value="true"/>
+ </meta_attributes>
+ </clone>
+ </resources>
+ <constraints>
+ <rsc_order first="FAKE1-clone" first-action="start" id="order-FAKE1-clone-FAKE2-clone-mandatory" then="FAKE2-clone" then-action="start"/>
+ <rsc_order first="FAKE2-clone" first-action="start" id="order-FAKE2-clone-FAKE3-clone-mandatory" then="FAKE3-clone" then-action="start"/>
+ <rsc_location id="cli-ban-FAKE1-clone-on-c7auto3" rsc="FAKE1-clone" role="Started" node="c7auto3" score="-INFINITY"/>
+ <rsc_location id="cli-ban-FAKE1-clone-on-c7auto2" rsc="FAKE1-clone" role="Started" node="c7auto2" score="-INFINITY"/>
+ </constraints>
+ </configuration>
+ <status>
+ <node_state id="3" uname="c7auto3" in_ccm="true" crmd="online" crm-debug-origin="do_update_resource" join="member" expected="member">
+ <lrm id="3">
+ <lrm_resources>
+ <lrm_resource id="shooter" type="fence_phd_kvm" class="stonith">
+ <lrm_rsc_op id="shooter_last_0" operation_key="shooter_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="8:4:7:9d56c099-7067-4908-9f99-89a103eff57b" transition-magic="0:7;8:4:7:9d56c099-7067-4908-9f99-89a103eff57b" on_node="c7auto3" call-id="5" rc-code="7" op-status="0" interval="0" last-run="1437146780" last-rc-change="1437146780" exec-time="998" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ </lrm_resource>
+ <lrm_resource id="FAKE1" type="Dummy" class="ocf" provider="heartbeat">
+ <lrm_rsc_op id="FAKE1_last_0" operation_key="FAKE1_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="14:5:0:9d56c099-7067-4908-9f99-89a103eff57b" transition-magic="0:0;14:5:0:9d56c099-7067-4908-9f99-89a103eff57b" on_node="c7auto3" call-id="11" rc-code="0" op-status="0" interval="0" last-run="1437147006" last-rc-change="1437147006" exec-time="8" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" fake " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ <lrm_rsc_op id="FAKE1_monitor_10000" operation_key="FAKE1_monitor_10000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="15:5:0:9d56c099-7067-4908-9f99-89a103eff57b" transition-magic="0:0;15:5:0:9d56c099-7067-4908-9f99-89a103eff57b" on_node="c7auto3" call-id="12" rc-code="0" op-status="0" interval="10000" last-rc-change="1437147006" exec-time="9" queue-time="0" op-digest="4811cef7f7f94e3a35a70be7916cb2fd"/>
+ </lrm_resource>
+ <lrm_resource id="FAKE2" type="Dummy" class="ocf" provider="heartbeat">
+ <lrm_rsc_op id="FAKE2_last_0" operation_key="FAKE2_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="27:6:0:9d56c099-7067-4908-9f99-89a103eff57b" transition-magic="0:0;27:6:0:9d56c099-7067-4908-9f99-89a103eff57b" on_node="c7auto3" call-id="18" rc-code="0" op-status="0" interval="0" last-run="1437147019" last-rc-change="1437147019" exec-time="8" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" fake " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ <lrm_rsc_op id="FAKE2_monitor_10000" operation_key="FAKE2_monitor_10000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="28:6:0:9d56c099-7067-4908-9f99-89a103eff57b" transition-magic="0:0;28:6:0:9d56c099-7067-4908-9f99-89a103eff57b" on_node="c7auto3" call-id="19" rc-code="0" op-status="0" interval="10000" last-rc-change="1437147019" exec-time="8" queue-time="0" op-digest="4811cef7f7f94e3a35a70be7916cb2fd"/>
+ </lrm_resource>
+ <lrm_resource id="FAKE3" type="Dummy" class="ocf" provider="heartbeat">
+ <lrm_rsc_op id="FAKE3_last_0" operation_key="FAKE3_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="40:7:0:9d56c099-7067-4908-9f99-89a103eff57b" transition-magic="0:0;40:7:0:9d56c099-7067-4908-9f99-89a103eff57b" on_node="c7auto3" call-id="25" rc-code="0" op-status="0" interval="0" last-run="1437147024" last-rc-change="1437147024" exec-time="9" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" fake " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ <lrm_rsc_op id="FAKE3_monitor_10000" operation_key="FAKE3_monitor_10000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="41:7:0:9d56c099-7067-4908-9f99-89a103eff57b" transition-magic="0:0;41:7:0:9d56c099-7067-4908-9f99-89a103eff57b" on_node="c7auto3" call-id="26" rc-code="0" op-status="0" interval="10000" last-rc-change="1437147024" exec-time="8" queue-time="0" op-digest="4811cef7f7f94e3a35a70be7916cb2fd"/>
+ </lrm_resource>
+ </lrm_resources>
+ </lrm>
+ <transient_attributes id="3">
+ <instance_attributes id="status-3">
+ <nvpair id="status-3-shutdown" name="shutdown" value="0"/>
+ <nvpair id="status-3-probe_complete" name="probe_complete" value="true"/>
+ </instance_attributes>
+ </transient_attributes>
+ </node_state>
+ <node_state id="1" uname="c7auto1" in_ccm="true" crmd="online" crm-debug-origin="do_update_resource" join="member" expected="member">
+ <lrm id="1">
+ <lrm_resources>
+ <lrm_resource id="shooter" type="fence_phd_kvm" class="stonith">
+ <lrm_rsc_op id="shooter_last_0" operation_key="shooter_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="9:4:0:9d56c099-7067-4908-9f99-89a103eff57b" transition-magic="0:0;9:4:0:9d56c099-7067-4908-9f99-89a103eff57b" on_node="c7auto1" call-id="6" rc-code="0" op-status="0" interval="0" last-run="1437146782" last-rc-change="1437146782" exec-time="5" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ <lrm_rsc_op id="shooter_monitor_60000" operation_key="shooter_monitor_60000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="10:4:0:9d56c099-7067-4908-9f99-89a103eff57b" transition-magic="0:0;10:4:0:9d56c099-7067-4908-9f99-89a103eff57b" on_node="c7auto1" call-id="7" rc-code="0" op-status="0" interval="60000" last-rc-change="1437146783" exec-time="5" queue-time="1" op-digest="4811cef7f7f94e3a35a70be7916cb2fd"/>
+ </lrm_resource>
+ <lrm_resource id="FAKE1" type="Dummy" class="ocf" provider="heartbeat">
+ <lrm_rsc_op id="FAKE1_last_0" operation_key="FAKE1_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="16:5:0:9d56c099-7067-4908-9f99-89a103eff57b" transition-magic="0:0;16:5:0:9d56c099-7067-4908-9f99-89a103eff57b" on_node="c7auto1" call-id="13" rc-code="0" op-status="0" interval="0" last-run="1437147007" last-rc-change="1437147007" exec-time="8" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" fake " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ <lrm_rsc_op id="FAKE1_monitor_10000" operation_key="FAKE1_monitor_10000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="17:5:0:9d56c099-7067-4908-9f99-89a103eff57b" transition-magic="0:0;17:5:0:9d56c099-7067-4908-9f99-89a103eff57b" on_node="c7auto1" call-id="14" rc-code="0" op-status="0" interval="10000" last-rc-change="1437147007" exec-time="6" queue-time="0" op-digest="4811cef7f7f94e3a35a70be7916cb2fd"/>
+ </lrm_resource>
+ <lrm_resource id="FAKE2" type="Dummy" class="ocf" provider="heartbeat">
+ <lrm_rsc_op id="FAKE2_last_0" operation_key="FAKE2_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="29:6:0:9d56c099-7067-4908-9f99-89a103eff57b" transition-magic="0:0;29:6:0:9d56c099-7067-4908-9f99-89a103eff57b" on_node="c7auto1" call-id="20" rc-code="0" op-status="0" interval="0" last-run="1437147019" last-rc-change="1437147019" exec-time="7" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" fake " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ <lrm_rsc_op id="FAKE2_monitor_10000" operation_key="FAKE2_monitor_10000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="30:6:0:9d56c099-7067-4908-9f99-89a103eff57b" transition-magic="0:0;30:6:0:9d56c099-7067-4908-9f99-89a103eff57b" on_node="c7auto1" call-id="21" rc-code="0" op-status="0" interval="10000" last-rc-change="1437147019" exec-time="7" queue-time="0" op-digest="4811cef7f7f94e3a35a70be7916cb2fd"/>
+ </lrm_resource>
+ <lrm_resource id="FAKE3" type="Dummy" class="ocf" provider="heartbeat">
+ <lrm_rsc_op id="FAKE3_last_0" operation_key="FAKE3_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="42:7:0:9d56c099-7067-4908-9f99-89a103eff57b" transition-magic="0:0;42:7:0:9d56c099-7067-4908-9f99-89a103eff57b" on_node="c7auto1" call-id="27" rc-code="0" op-status="0" interval="0" last-run="1437147024" last-rc-change="1437147024" exec-time="8" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" fake " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ <lrm_rsc_op id="FAKE3_monitor_10000" operation_key="FAKE3_monitor_10000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="43:7:0:9d56c099-7067-4908-9f99-89a103eff57b" transition-magic="0:0;43:7:0:9d56c099-7067-4908-9f99-89a103eff57b" on_node="c7auto1" call-id="28" rc-code="0" op-status="0" interval="10000" last-rc-change="1437147024" exec-time="6" queue-time="1" op-digest="4811cef7f7f94e3a35a70be7916cb2fd"/>
+ </lrm_resource>
+ </lrm_resources>
+ </lrm>
+ <transient_attributes id="1">
+ <instance_attributes id="status-1">
+ <nvpair id="status-1-shutdown" name="shutdown" value="0"/>
+ <nvpair id="status-1-probe_complete" name="probe_complete" value="true"/>
+ </instance_attributes>
+ </transient_attributes>
+ </node_state>
+ <node_state id="2" uname="c7auto2" in_ccm="true" crmd="online" crm-debug-origin="do_update_resource" join="member" expected="member">
+ <lrm id="2">
+ <lrm_resources>
+ <lrm_resource id="shooter" type="fence_phd_kvm" class="stonith">
+ <lrm_rsc_op id="shooter_last_0" operation_key="shooter_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="6:4:7:9d56c099-7067-4908-9f99-89a103eff57b" transition-magic="0:7;6:4:7:9d56c099-7067-4908-9f99-89a103eff57b" on_node="c7auto2" call-id="5" rc-code="7" op-status="0" interval="0" last-run="1437146780" last-rc-change="1437146780" exec-time="994" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ </lrm_resource>
+ <lrm_resource id="FAKE1" type="Dummy" class="ocf" provider="heartbeat">
+ <lrm_rsc_op id="FAKE1_last_0" operation_key="FAKE1_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="12:5:0:9d56c099-7067-4908-9f99-89a103eff57b" transition-magic="0:0;12:5:0:9d56c099-7067-4908-9f99-89a103eff57b" on_node="c7auto2" call-id="11" rc-code="0" op-status="0" interval="0" last-run="1437147007" last-rc-change="1437147007" exec-time="6" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" fake " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ <lrm_rsc_op id="FAKE1_monitor_10000" operation_key="FAKE1_monitor_10000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="13:5:0:9d56c099-7067-4908-9f99-89a103eff57b" transition-magic="0:0;13:5:0:9d56c099-7067-4908-9f99-89a103eff57b" on_node="c7auto2" call-id="12" rc-code="0" op-status="0" interval="10000" last-rc-change="1437147007" exec-time="8" queue-time="0" op-digest="4811cef7f7f94e3a35a70be7916cb2fd"/>
+ </lrm_resource>
+ <lrm_resource id="FAKE2" type="Dummy" class="ocf" provider="heartbeat">
+ <lrm_rsc_op id="FAKE2_last_0" operation_key="FAKE2_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="25:6:0:9d56c099-7067-4908-9f99-89a103eff57b" transition-magic="0:0;25:6:0:9d56c099-7067-4908-9f99-89a103eff57b" on_node="c7auto2" call-id="18" rc-code="0" op-status="0" interval="0" last-run="1437147019" last-rc-change="1437147019" exec-time="8" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" fake " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ <lrm_rsc_op id="FAKE2_monitor_10000" operation_key="FAKE2_monitor_10000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="26:6:0:9d56c099-7067-4908-9f99-89a103eff57b" transition-magic="0:0;26:6:0:9d56c099-7067-4908-9f99-89a103eff57b" on_node="c7auto2" call-id="19" rc-code="0" op-status="0" interval="10000" last-rc-change="1437147019" exec-time="6" queue-time="0" op-digest="4811cef7f7f94e3a35a70be7916cb2fd"/>
+ </lrm_resource>
+ <lrm_resource id="FAKE3" type="Dummy" class="ocf" provider="heartbeat">
+ <lrm_rsc_op id="FAKE3_last_0" operation_key="FAKE3_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="38:7:0:9d56c099-7067-4908-9f99-89a103eff57b" transition-magic="0:0;38:7:0:9d56c099-7067-4908-9f99-89a103eff57b" on_node="c7auto2" call-id="25" rc-code="0" op-status="0" interval="0" last-run="1437147024" last-rc-change="1437147024" exec-time="7" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" fake " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ <lrm_rsc_op id="FAKE3_monitor_10000" operation_key="FAKE3_monitor_10000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="39:7:0:9d56c099-7067-4908-9f99-89a103eff57b" transition-magic="0:0;39:7:0:9d56c099-7067-4908-9f99-89a103eff57b" on_node="c7auto2" call-id="26" rc-code="0" op-status="0" interval="10000" last-rc-change="1437147024" exec-time="8" queue-time="0" op-digest="4811cef7f7f94e3a35a70be7916cb2fd"/>
+ </lrm_resource>
+ </lrm_resources>
+ </lrm>
+ <transient_attributes id="2">
+ <instance_attributes id="status-2">
+ <nvpair id="status-2-shutdown" name="shutdown" value="0"/>
+ <nvpair id="status-2-probe_complete" name="probe_complete" value="true"/>
+ </instance_attributes>
+ </transient_attributes>
+ </node_state>
+ </status>
+</cib>
diff --git a/pengine/test10/clone_min_start_one.dot b/pengine/test10/clone_min_start_one.dot
new file mode 100644
index 0000000..3940361
--- /dev/null
+++ b/pengine/test10/clone_min_start_one.dot
@@ -0,0 +1,20 @@
+ digraph "g" {
+"FAKECLONE-clone_running_0" [ style=bold color="green" fontcolor="orange"]
+"FAKECLONE-clone_start_0" -> "FAKECLONE-clone_running_0" [ style = bold]
+"FAKECLONE-clone_start_0" -> "FAKECLONE_start_0 c7auto3" [ style = bold]
+"FAKECLONE-clone_start_0" [ style=bold color="green" fontcolor="orange"]
+"FAKECLONE_monitor_10000 c7auto3" [ style=bold color="green" fontcolor="black"]
+"FAKECLONE_start_0 c7auto3" -> "FAKECLONE-clone_running_0" [ style = bold]
+"FAKECLONE_start_0 c7auto3" -> "FAKECLONE_monitor_10000 c7auto3" [ style = bold]
+"FAKECLONE_start_0 c7auto3" [ style=bold color="green" fontcolor="black"]
+"FAKE_monitor_10000 c7auto4" [ style=dashed color="red" fontcolor="black"]
+"FAKE_start_0 c7auto4" -> "FAKE_monitor_10000 c7auto4" [ style = dashed]
+"FAKE_start_0 c7auto4" [ style=dashed color="red" fontcolor="black"]
+"all_stopped" [ style=bold color="green" fontcolor="orange"]
+"shooter_monitor_60000 c7auto3" [ style=bold color="green" fontcolor="black"]
+"shooter_start_0 c7auto3" -> "shooter_monitor_60000 c7auto3" [ style = bold]
+"shooter_start_0 c7auto3" [ style=bold color="green" fontcolor="black"]
+"shooter_stop_0 c7auto1" -> "all_stopped" [ style = bold]
+"shooter_stop_0 c7auto1" -> "shooter_start_0 c7auto3" [ style = bold]
+"shooter_stop_0 c7auto1" [ style=bold color="green" fontcolor="black"]
+}
diff --git a/pengine/test10/clone_min_start_one.exp b/pengine/test10/clone_min_start_one.exp
new file mode 100644
index 0000000..a6868f6
--- /dev/null
+++ b/pengine/test10/clone_min_start_one.exp
@@ -0,0 +1,98 @@
+<transition_graph cluster-delay="60s" stonith-timeout="60s" failed-stop-offset="INFINITY" failed-start-offset="INFINITY" transition_id="0">
+ <synapse id="0">
+ <action_set>
+ <rsc_op id="11" operation="monitor" operation_key="shooter_monitor_60000" on_node="c7auto3" on_node_uuid="3">
+ <primitive id="shooter" class="stonith" type="fence_phd_kvm"/>
+ <attributes CRM_meta_interval="60000" CRM_meta_name="monitor" CRM_meta_timeout="20000" />
+ </rsc_op>
+ </action_set>
+ <inputs>
+ <trigger>
+ <rsc_op id="10" operation="start" operation_key="shooter_start_0" on_node="c7auto3" on_node_uuid="3"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="1">
+ <action_set>
+ <rsc_op id="10" operation="start" operation_key="shooter_start_0" on_node="c7auto3" on_node_uuid="3">
+ <primitive id="shooter" class="stonith" type="fence_phd_kvm"/>
+ <attributes CRM_meta_timeout="20000" />
+ </rsc_op>
+ </action_set>
+ <inputs>
+ <trigger>
+ <rsc_op id="9" operation="stop" operation_key="shooter_stop_0" on_node="c7auto1" on_node_uuid="1"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="2">
+ <action_set>
+ <rsc_op id="9" operation="stop" operation_key="shooter_stop_0" on_node="c7auto1" on_node_uuid="1">
+ <primitive id="shooter" class="stonith" type="fence_phd_kvm"/>
+ <attributes CRM_meta_timeout="20000" />
+ </rsc_op>
+ </action_set>
+ <inputs/>
+ </synapse>
+ <synapse id="3">
+ <action_set>
+ <rsc_op id="13" operation="monitor" operation_key="FAKECLONE_monitor_10000" internal_operation_key="FAKECLONE:0_monitor_10000" on_node="c7auto3" on_node_uuid="3">
+ <primitive id="FAKECLONE" long-id="FAKECLONE:0" class="ocf" provider="heartbeat" type="Dummy"/>
+ <attributes CRM_meta_clone="0" CRM_meta_clone_max="4" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_interval="10000" CRM_meta_name="monitor" CRM_meta_notify="false" CRM_meta_timeout="20000" />
+ </rsc_op>
+ </action_set>
+ <inputs>
+ <trigger>
+ <rsc_op id="12" operation="start" operation_key="FAKECLONE_start_0" internal_operation_key="FAKECLONE:0_start_0" on_node="c7auto3" on_node_uuid="3"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="4">
+ <action_set>
+ <rsc_op id="12" operation="start" operation_key="FAKECLONE_start_0" internal_operation_key="FAKECLONE:0_start_0" on_node="c7auto3" on_node_uuid="3">
+ <primitive id="FAKECLONE" long-id="FAKECLONE:0" class="ocf" provider="heartbeat" type="Dummy"/>
+ <attributes CRM_meta_clone="0" CRM_meta_clone_max="4" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_name="start" CRM_meta_notify="false" CRM_meta_timeout="20000" />
+ </rsc_op>
+ </action_set>
+ <inputs>
+ <trigger>
+ <pseudo_event id="14" operation="start" operation_key="FAKECLONE-clone_start_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="5" priority="1000000">
+ <action_set>
+ <pseudo_event id="15" operation="running" operation_key="FAKECLONE-clone_running_0">
+ <attributes CRM_meta_clone_max="4" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="false" CRM_meta_timeout="20000" />
+ </pseudo_event>
+ </action_set>
+ <inputs>
+ <trigger>
+ <rsc_op id="12" operation="start" operation_key="FAKECLONE_start_0" internal_operation_key="FAKECLONE:0_start_0" on_node="c7auto3" on_node_uuid="3"/>
+ </trigger>
+ <trigger>
+ <pseudo_event id="14" operation="start" operation_key="FAKECLONE-clone_start_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="6">
+ <action_set>
+ <pseudo_event id="14" operation="start" operation_key="FAKECLONE-clone_start_0">
+ <attributes CRM_meta_clone_max="4" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="false" CRM_meta_timeout="20000" />
+ </pseudo_event>
+ </action_set>
+ <inputs/>
+ </synapse>
+ <synapse id="7">
+ <action_set>
+ <pseudo_event id="3" operation="all_stopped" operation_key="all_stopped">
+ <attributes />
+ </pseudo_event>
+ </action_set>
+ <inputs>
+ <trigger>
+ <rsc_op id="9" operation="stop" operation_key="shooter_stop_0" on_node="c7auto1" on_node_uuid="1"/>
+ </trigger>
+ </inputs>
+ </synapse>
+</transition_graph>
diff --git a/pengine/test10/clone_min_start_one.scores b/pengine/test10/clone_min_start_one.scores
new file mode 100644
index 0000000..668689e
--- /dev/null
+++ b/pengine/test10/clone_min_start_one.scores
@@ -0,0 +1,45 @@
+Allocation scores:
+clone_color: FAKECLONE-clone allocation score on c7auto1: 0
+clone_color: FAKECLONE-clone allocation score on c7auto2: 0
+clone_color: FAKECLONE-clone allocation score on c7auto3: 0
+clone_color: FAKECLONE-clone allocation score on c7auto4: -INFINITY
+clone_color: FAKECLONE:0 allocation score on c7auto1: 0
+clone_color: FAKECLONE:0 allocation score on c7auto2: 0
+clone_color: FAKECLONE:0 allocation score on c7auto3: 0
+clone_color: FAKECLONE:0 allocation score on c7auto4: -INFINITY
+clone_color: FAKECLONE:1 allocation score on c7auto1: 0
+clone_color: FAKECLONE:1 allocation score on c7auto2: 0
+clone_color: FAKECLONE:1 allocation score on c7auto3: 0
+clone_color: FAKECLONE:1 allocation score on c7auto4: -INFINITY
+clone_color: FAKECLONE:2 allocation score on c7auto1: 0
+clone_color: FAKECLONE:2 allocation score on c7auto2: 0
+clone_color: FAKECLONE:2 allocation score on c7auto3: 0
+clone_color: FAKECLONE:2 allocation score on c7auto4: -INFINITY
+clone_color: FAKECLONE:3 allocation score on c7auto1: 0
+clone_color: FAKECLONE:3 allocation score on c7auto2: 0
+clone_color: FAKECLONE:3 allocation score on c7auto3: 0
+clone_color: FAKECLONE:3 allocation score on c7auto4: -INFINITY
+native_color: FAKE allocation score on c7auto1: -INFINITY
+native_color: FAKE allocation score on c7auto2: -INFINITY
+native_color: FAKE allocation score on c7auto3: -INFINITY
+native_color: FAKE allocation score on c7auto4: 0
+native_color: FAKECLONE:0 allocation score on c7auto1: -INFINITY
+native_color: FAKECLONE:0 allocation score on c7auto2: -INFINITY
+native_color: FAKECLONE:0 allocation score on c7auto3: 0
+native_color: FAKECLONE:0 allocation score on c7auto4: -INFINITY
+native_color: FAKECLONE:1 allocation score on c7auto1: -INFINITY
+native_color: FAKECLONE:1 allocation score on c7auto2: -INFINITY
+native_color: FAKECLONE:1 allocation score on c7auto3: -INFINITY
+native_color: FAKECLONE:1 allocation score on c7auto4: -INFINITY
+native_color: FAKECLONE:2 allocation score on c7auto1: -INFINITY
+native_color: FAKECLONE:2 allocation score on c7auto2: -INFINITY
+native_color: FAKECLONE:2 allocation score on c7auto3: -INFINITY
+native_color: FAKECLONE:2 allocation score on c7auto4: -INFINITY
+native_color: FAKECLONE:3 allocation score on c7auto1: -INFINITY
+native_color: FAKECLONE:3 allocation score on c7auto2: -INFINITY
+native_color: FAKECLONE:3 allocation score on c7auto3: -INFINITY
+native_color: FAKECLONE:3 allocation score on c7auto4: -INFINITY
+native_color: shooter allocation score on c7auto1: 0
+native_color: shooter allocation score on c7auto2: 0
+native_color: shooter allocation score on c7auto3: 0
+native_color: shooter allocation score on c7auto4: 0
diff --git a/pengine/test10/clone_min_start_one.summary b/pengine/test10/clone_min_start_one.summary
new file mode 100644
index 0000000..ee33e01
--- /dev/null
+++ b/pengine/test10/clone_min_start_one.summary
@@ -0,0 +1,37 @@
+
+Current cluster status:
+Node c7auto1 (1): standby
+Node c7auto2 (2): standby
+Online: [ c7auto3 c7auto4 ]
+
+ shooter (stonith:fence_phd_kvm): Started c7auto1
+ Clone Set: FAKECLONE-clone [FAKECLONE]
+ Stopped: [ c7auto1 c7auto2 c7auto3 c7auto4 ]
+ FAKE (ocf::heartbeat:Dummy): Stopped
+
+Transition Summary:
+ * Move shooter (Started c7auto1 -> c7auto3)
+ * Start FAKECLONE:0 (c7auto3)
+ * Start FAKE (c7auto4 - blocked)
+
+Executing cluster transition:
+ * Resource action: shooter stop on c7auto1
+ * Pseudo action: FAKECLONE-clone_start_0
+ * Pseudo action: all_stopped
+ * Resource action: shooter start on c7auto3
+ * Resource action: FAKECLONE start on c7auto3
+ * Pseudo action: FAKECLONE-clone_running_0
+ * Resource action: shooter monitor=60000 on c7auto3
+ * Resource action: FAKECLONE monitor=10000 on c7auto3
+
+Revised cluster status:
+Node c7auto1 (1): standby
+Node c7auto2 (2): standby
+Online: [ c7auto3 c7auto4 ]
+
+ shooter (stonith:fence_phd_kvm): Started c7auto3
+ Clone Set: FAKECLONE-clone [FAKECLONE]
+ Started: [ c7auto3 ]
+ Stopped: [ c7auto1 c7auto2 c7auto4 ]
+ FAKE (ocf::heartbeat:Dummy): Stopped
+
diff --git a/pengine/test10/clone_min_start_one.xml b/pengine/test10/clone_min_start_one.xml
new file mode 100644
index 0000000..dfb9379
--- /dev/null
+++ b/pengine/test10/clone_min_start_one.xml
@@ -0,0 +1,155 @@
+<cib crm_feature_set="3.0.10" validate-with="pacemaker-2.3" epoch="17" num_updates="0" admin_epoch="0" cib-last-written="Thu Jul 16 15:37:54 2015" update-origin="c7auto1" update-client="crm_resource" update-user="root" have-quorum="1" dc-uuid="2">
+ <configuration>
+ <crm_config>
+ <cluster_property_set id="cib-bootstrap-options">
+ <nvpair id="cib-bootstrap-options-have-watchdog" name="have-watchdog" value="false"/>
+ <nvpair id="cib-bootstrap-options-dc-version" name="dc-version" value="1.1.12-ef86b49"/>
+ <nvpair id="cib-bootstrap-options-cluster-infrastructure" name="cluster-infrastructure" value="corosync"/>
+ <nvpair id="cib-bootstrap-options-cluster-name" name="cluster-name" value="phd"/>
+ </cluster_property_set>
+ </crm_config>
+ <nodes>
+ <node id="1" uname="c7auto1">
+ <instance_attributes id="nodes-1">
+ <nvpair id="nodes-1-standby" name="standby" value="on"/>
+ </instance_attributes>
+ </node>
+ <node id="2" uname="c7auto2">
+ <instance_attributes id="nodes-2">
+ <nvpair id="nodes-2-standby" name="standby" value="on"/>
+ </instance_attributes>
+ </node>
+ <node id="3" uname="c7auto3"/>
+ <node id="4" uname="c7auto4"/>
+ </nodes>
+ <resources>
+ <primitive class="stonith" id="shooter" type="fence_phd_kvm">
+ <instance_attributes id="shooter-instance_attributes"/>
+ <operations>
+ <op id="shooter-monitor-interval-60s" interval="60s" name="monitor"/>
+ </operations>
+ </primitive>
+ <clone id="FAKECLONE-clone">
+ <primitive class="ocf" id="FAKECLONE" provider="heartbeat" type="Dummy">
+ <instance_attributes id="FAKECLONE-instance_attributes"/>
+ <operations>
+ <op id="FAKECLONE-start-timeout-20" interval="0s" name="start" timeout="20"/>
+ <op id="FAKECLONE-stop-timeout-20" interval="0s" name="stop" timeout="20"/>
+ <op id="FAKECLONE-monitor-interval-10" interval="10" name="monitor" timeout="20"/>
+ </operations>
+ </primitive>
+ <meta_attributes id="FAKECLONE-clone-meta">
+ <nvpair id="FAKECLONE-clone-min" name="clone-min" value="2"/>
+ </meta_attributes>
+ <meta_attributes id="FAKECLONE-clone-meta_attributes"/>
+ </clone>
+ <primitive class="ocf" id="FAKE" provider="heartbeat" type="Dummy">
+ <instance_attributes id="FAKE-instance_attributes"/>
+ <operations>
+ <op id="FAKE-start-timeout-20" interval="0s" name="start" timeout="20"/>
+ <op id="FAKE-stop-timeout-20" interval="0s" name="stop" timeout="20"/>
+ <op id="FAKE-monitor-interval-10" interval="10" name="monitor" timeout="20"/>
+ </operations>
+ </primitive>
+ </resources>
+ <constraints>
+ <rsc_location id="location-FAKECLONE-clone-c7auto4--INFINITY" node="c7auto4" rsc="FAKECLONE-clone" score="-INFINITY"/>
+ <rsc_location id="location-FAKE-c7auto1--INFINITY" node="c7auto1" rsc="FAKE" score="-INFINITY"/>
+ <rsc_location id="location-FAKE-c7auto2--INFINITY" node="c7auto2" rsc="FAKE" score="-INFINITY"/>
+ <rsc_location id="location-FAKE-c7auto3--INFINITY" node="c7auto3" rsc="FAKE" score="-INFINITY"/>
+ <rsc_order first="FAKECLONE-clone" first-action="start" id="order-FAKECLONE-clone-FAKE-mandatory" then="FAKE" then-action="start"/>
+ </constraints>
+ </configuration>
+ <status>
+ <node_state id="1" uname="c7auto1" in_ccm="true" crmd="online" crm-debug-origin="do_update_resource" join="member" expected="member">
+ <lrm id="1">
+ <lrm_resources>
+ <lrm_resource id="shooter" type="fence_phd_kvm" class="stonith">
+ <lrm_rsc_op id="shooter_last_0" operation_key="shooter_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="11:0:0:b489d50c-a7db-4605-8519-a2d5e1452e29" transition-magic="0:0;11:0:0:b489d50c-a7db-4605-8519-a2d5e1452e29" on_node="c7auto1" call-id="6" rc-code="0" op-status="0" interval="0" last-run="1437073991" last-rc-change="1437073991" exec-time="9" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ <lrm_rsc_op id="shooter_monitor_60000" operation_key="shooter_monitor_60000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="9:1:0:b489d50c-a7db-4605-8519-a2d5e1452e29" transition-magic="0:0;9:1:0:b489d50c-a7db-4605-8519-a2d5e1452e29" on_node="c7auto1" call-id="7" rc-code="0" op-status="0" interval="60000" last-rc-change="1437073992" exec-time="7" queue-time="0" op-digest="4811cef7f7f94e3a35a70be7916cb2fd"/>
+ </lrm_resource>
+ <lrm_resource id="FAKECLONE" type="Dummy" class="ocf" provider="heartbeat">
+ <lrm_rsc_op id="FAKECLONE_last_0" operation_key="FAKECLONE_stop_0" operation="stop" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="14:9:0:b489d50c-a7db-4605-8519-a2d5e1452e29" transition-magic="0:0;14:9:0:b489d50c-a7db-4605-8519-a2d5e1452e29" on_node="c7auto1" call-id="20" rc-code="0" op-status="0" interval="0" last-run="1437075474" last-rc-change="1437075474" exec-time="8" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" fake " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ <lrm_rsc_op id="FAKECLONE_monitor_10000" operation_key="FAKECLONE_monitor_10000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="21:2:0:b489d50c-a7db-4605-8519-a2d5e1452e29" transition-magic="0:0;21:2:0:b489d50c-a7db-4605-8519-a2d5e1452e29" on_node="c7auto1" call-id="14" rc-code="0" op-status="0" interval="10000" last-rc-change="1437074829" exec-time="7" queue-time="0" op-digest="4811cef7f7f94e3a35a70be7916cb2fd"/>
+ </lrm_resource>
+ <lrm_resource id="FAKE" type="Dummy" class="ocf" provider="heartbeat">
+ <lrm_rsc_op id="FAKE_last_0" operation_key="FAKE_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="8:4:7:b489d50c-a7db-4605-8519-a2d5e1452e29" transition-magic="0:7;8:4:7:b489d50c-a7db-4605-8519-a2d5e1452e29" on_node="c7auto1" call-id="18" rc-code="7" op-status="0" interval="0" last-run="1437074871" last-rc-change="1437074871" exec-time="7" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" fake " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ </lrm_resource>
+ </lrm_resources>
+ </lrm>
+ <transient_attributes id="1">
+ <instance_attributes id="status-1">
+ <nvpair id="status-1-shutdown" name="shutdown" value="0"/>
+ <nvpair id="status-1-probe_complete" name="probe_complete" value="true"/>
+ </instance_attributes>
+ </transient_attributes>
+ </node_state>
+ <node_state id="2" uname="c7auto2" in_ccm="true" crmd="online" crm-debug-origin="do_update_resource" join="member" expected="member">
+ <lrm id="2">
+ <lrm_resources>
+ <lrm_resource id="shooter" type="fence_phd_kvm" class="stonith">
+ <lrm_rsc_op id="shooter_last_0" operation_key="shooter_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="6:0:7:b489d50c-a7db-4605-8519-a2d5e1452e29" transition-magic="0:7;6:0:7:b489d50c-a7db-4605-8519-a2d5e1452e29" on_node="c7auto2" call-id="5" rc-code="7" op-status="0" interval="0" last-run="1437073988" last-rc-change="1437073988" exec-time="964" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ </lrm_resource>
+ <lrm_resource id="FAKECLONE" type="Dummy" class="ocf" provider="heartbeat">
+ <lrm_rsc_op id="FAKECLONE_last_0" operation_key="FAKECLONE_stop_0" operation="stop" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="15:9:0:b489d50c-a7db-4605-8519-a2d5e1452e29" transition-magic="0:0;15:9:0:b489d50c-a7db-4605-8519-a2d5e1452e29" on_node="c7auto2" call-id="18" rc-code="0" op-status="0" interval="0" last-run="1437075473" last-rc-change="1437075473" exec-time="13" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" fake " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ <lrm_rsc_op id="FAKECLONE_monitor_10000" operation_key="FAKECLONE_monitor_10000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="15:2:0:b489d50c-a7db-4605-8519-a2d5e1452e29" transition-magic="0:0;15:2:0:b489d50c-a7db-4605-8519-a2d5e1452e29" on_node="c7auto2" call-id="12" rc-code="0" op-status="0" interval="10000" last-rc-change="1437074829" exec-time="10" queue-time="0" op-digest="4811cef7f7f94e3a35a70be7916cb2fd"/>
+ </lrm_resource>
+ <lrm_resource id="FAKE" type="Dummy" class="ocf" provider="heartbeat">
+ <lrm_rsc_op id="FAKE_last_0" operation_key="FAKE_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="10:4:7:b489d50c-a7db-4605-8519-a2d5e1452e29" transition-magic="0:7;10:4:7:b489d50c-a7db-4605-8519-a2d5e1452e29" on_node="c7auto2" call-id="16" rc-code="7" op-status="0" interval="0" last-run="1437074871" last-rc-change="1437074871" exec-time="10" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" fake " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ </lrm_resource>
+ </lrm_resources>
+ </lrm>
+ <transient_attributes id="2">
+ <instance_attributes id="status-2">
+ <nvpair id="status-2-shutdown" name="shutdown" value="0"/>
+ <nvpair id="status-2-probe_complete" name="probe_complete" value="true"/>
+ </instance_attributes>
+ </transient_attributes>
+ </node_state>
+ <node_state id="3" uname="c7auto3" in_ccm="true" crmd="online" crm-debug-origin="do_update_resource" join="member" expected="member">
+ <lrm id="3">
+ <lrm_resources>
+ <lrm_resource id="shooter" type="fence_phd_kvm" class="stonith">
+ <lrm_rsc_op id="shooter_last_0" operation_key="shooter_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="8:0:7:b489d50c-a7db-4605-8519-a2d5e1452e29" transition-magic="0:7;8:0:7:b489d50c-a7db-4605-8519-a2d5e1452e29" on_node="c7auto3" call-id="5" rc-code="7" op-status="0" interval="0" last-run="1437073988" last-rc-change="1437073988" exec-time="963" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ </lrm_resource>
+ <lrm_resource id="FAKECLONE" type="Dummy" class="ocf" provider="heartbeat">
+ <lrm_rsc_op id="FAKECLONE_last_0" operation_key="FAKECLONE_stop_0" operation="stop" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="16:9:0:b489d50c-a7db-4605-8519-a2d5e1452e29" transition-magic="0:0;16:9:0:b489d50c-a7db-4605-8519-a2d5e1452e29" on_node="c7auto3" call-id="18" rc-code="0" op-status="0" interval="0" last-run="1437075473" last-rc-change="1437075473" exec-time="12" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" fake " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ <lrm_rsc_op id="FAKECLONE_monitor_10000" operation_key="FAKECLONE_monitor_10000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="17:2:0:b489d50c-a7db-4605-8519-a2d5e1452e29" transition-magic="0:0;17:2:0:b489d50c-a7db-4605-8519-a2d5e1452e29" on_node="c7auto3" call-id="12" rc-code="0" op-status="0" interval="10000" last-rc-change="1437074828" exec-time="10" queue-time="0" op-digest="4811cef7f7f94e3a35a70be7916cb2fd"/>
+ </lrm_resource>
+ <lrm_resource id="FAKE" type="Dummy" class="ocf" provider="heartbeat">
+ <lrm_rsc_op id="FAKE_last_0" operation_key="FAKE_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="12:4:7:b489d50c-a7db-4605-8519-a2d5e1452e29" transition-magic="0:7;12:4:7:b489d50c-a7db-4605-8519-a2d5e1452e29" on_node="c7auto3" call-id="16" rc-code="7" op-status="0" interval="0" last-run="1437074870" last-rc-change="1437074870" exec-time="8" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" fake " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ </lrm_resource>
+ </lrm_resources>
+ </lrm>
+ <transient_attributes id="3">
+ <instance_attributes id="status-3">
+ <nvpair id="status-3-shutdown" name="shutdown" value="0"/>
+ <nvpair id="status-3-probe_complete" name="probe_complete" value="true"/>
+ </instance_attributes>
+ </transient_attributes>
+ </node_state>
+ <node_state id="4" uname="c7auto4" in_ccm="true" crmd="online" crm-debug-origin="do_update_resource" join="member" expected="member">
+ <lrm id="4">
+ <lrm_resources>
+ <lrm_resource id="shooter" type="fence_phd_kvm" class="stonith">
+ <lrm_rsc_op id="shooter_last_0" operation_key="shooter_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="10:0:7:b489d50c-a7db-4605-8519-a2d5e1452e29" transition-magic="0:7;10:0:7:b489d50c-a7db-4605-8519-a2d5e1452e29" on_node="c7auto4" call-id="5" rc-code="7" op-status="0" interval="0" last-run="1437073988" last-rc-change="1437073988" exec-time="963" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ </lrm_resource>
+ <lrm_resource id="FAKECLONE" type="Dummy" class="ocf" provider="heartbeat">
+ <lrm_rsc_op id="FAKECLONE_last_0" operation_key="FAKECLONE_stop_0" operation="stop" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="20:3:0:b489d50c-a7db-4605-8519-a2d5e1452e29" transition-magic="0:0;20:3:0:b489d50c-a7db-4605-8519-a2d5e1452e29" on_node="c7auto4" call-id="14" rc-code="0" op-status="0" interval="0" last-run="1437074856" last-rc-change="1437074856" exec-time="8" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" fake " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ <lrm_rsc_op id="FAKECLONE_monitor_10000" operation_key="FAKECLONE_monitor_10000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="19:2:0:b489d50c-a7db-4605-8519-a2d5e1452e29" transition-magic="0:0;19:2:0:b489d50c-a7db-4605-8519-a2d5e1452e29" on_node="c7auto4" call-id="12" rc-code="0" op-status="0" interval="10000" last-rc-change="1437074829" exec-time="6" queue-time="0" op-digest="4811cef7f7f94e3a35a70be7916cb2fd"/>
+ </lrm_resource>
+ <lrm_resource id="FAKE" type="Dummy" class="ocf" provider="heartbeat">
+ <lrm_rsc_op id="FAKE_last_0" operation_key="FAKE_stop_0" operation="stop" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="21:9:0:b489d50c-a7db-4605-8519-a2d5e1452e29" transition-magic="0:0;21:9:0:b489d50c-a7db-4605-8519-a2d5e1452e29" on_node="c7auto4" call-id="22" rc-code="0" op-status="0" interval="0" last-run="1437075473" last-rc-change="1437075473" exec-time="6" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" fake " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ <lrm_rsc_op id="FAKE_monitor_10000" operation_key="FAKE_monitor_10000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="28:4:0:b489d50c-a7db-4605-8519-a2d5e1452e29" transition-magic="0:0;28:4:0:b489d50c-a7db-4605-8519-a2d5e1452e29" on_node="c7auto4" call-id="20" rc-code="0" op-status="0" interval="10000" last-rc-change="1437074871" exec-time="7" queue-time="0" op-digest="4811cef7f7f94e3a35a70be7916cb2fd"/>
+ </lrm_resource>
+ </lrm_resources>
+ </lrm>
+ <transient_attributes id="4">
+ <instance_attributes id="status-4">
+ <nvpair id="status-4-shutdown" name="shutdown" value="0"/>
+ <nvpair id="status-4-probe_complete" name="probe_complete" value="true"/>
+ </instance_attributes>
+ </transient_attributes>
+ </node_state>
+ </status>
+</cib>
diff --git a/pengine/test10/clone_min_start_two.dot b/pengine/test10/clone_min_start_two.dot
new file mode 100644
index 0000000..3fe0062
--- /dev/null
+++ b/pengine/test10/clone_min_start_two.dot
@@ -0,0 +1,22 @@
+ digraph "g" {
+"FAKECLONE-clone_running_0" [ style=bold color="green" fontcolor="orange"]
+"FAKECLONE-clone_start_0" -> "FAKECLONE-clone_running_0" [ style = bold]
+"FAKECLONE-clone_start_0" -> "FAKECLONE:1_start_0 c7auto1" [ style = bold]
+"FAKECLONE-clone_start_0" -> "FAKECLONE_start_0 c7auto3" [ style = bold]
+"FAKECLONE-clone_start_0" [ style=bold color="green" fontcolor="orange"]
+"FAKECLONE:1_monitor_10000 c7auto1" [ style=bold color="green" fontcolor="black"]
+"FAKECLONE:1_start_0 c7auto1" -> "FAKECLONE-clone_running_0" [ style = bold]
+"FAKECLONE:1_start_0 c7auto1" -> "FAKECLONE:1_monitor_10000 c7auto1" [ style = bold]
+"FAKECLONE:1_start_0 c7auto1" -> "clone-one-or-more:order-FAKECLONE-clone-FAKE-mandatory" [ style = bold]
+"FAKECLONE:1_start_0 c7auto1" [ style=bold color="green" fontcolor="black"]
+"FAKECLONE_monitor_10000 c7auto3" [ style=bold color="green" fontcolor="black"]
+"FAKECLONE_start_0 c7auto3" -> "FAKECLONE-clone_running_0" [ style = bold]
+"FAKECLONE_start_0 c7auto3" -> "FAKECLONE_monitor_10000 c7auto3" [ style = bold]
+"FAKECLONE_start_0 c7auto3" -> "clone-one-or-more:order-FAKECLONE-clone-FAKE-mandatory" [ style = bold]
+"FAKECLONE_start_0 c7auto3" [ style=bold color="green" fontcolor="black"]
+"FAKE_monitor_10000 c7auto4" [ style=bold color="green" fontcolor="black"]
+"FAKE_start_0 c7auto4" -> "FAKE_monitor_10000 c7auto4" [ style = bold]
+"FAKE_start_0 c7auto4" [ style=bold color="green" fontcolor="black"]
+"clone-one-or-more:order-FAKECLONE-clone-FAKE-mandatory" -> "FAKE_start_0 c7auto4" [ style = bold]
+"clone-one-or-more:order-FAKECLONE-clone-FAKE-mandatory" [ style=bold color="green" fontcolor="orange"]
+}
diff --git a/pengine/test10/clone_min_start_two.exp b/pengine/test10/clone_min_start_two.exp
new file mode 100644
index 0000000..f7a053c
--- /dev/null
+++ b/pengine/test10/clone_min_start_two.exp
@@ -0,0 +1,121 @@
+<transition_graph cluster-delay="60s" stonith-timeout="60s" failed-stop-offset="INFINITY" failed-start-offset="INFINITY" transition_id="0">
+ <synapse id="0">
+ <action_set>
+ <rsc_op id="12" operation="monitor" operation_key="FAKECLONE_monitor_10000" internal_operation_key="FAKECLONE:0_monitor_10000" on_node="c7auto3" on_node_uuid="3">
+ <primitive id="FAKECLONE" long-id="FAKECLONE:0" class="ocf" provider="heartbeat" type="Dummy"/>
+ <attributes CRM_meta_clone="0" CRM_meta_clone_max="4" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_interval="10000" CRM_meta_name="monitor" CRM_meta_notify="false" CRM_meta_timeout="20000" />
+ </rsc_op>
+ </action_set>
+ <inputs>
+ <trigger>
+ <rsc_op id="11" operation="start" operation_key="FAKECLONE_start_0" internal_operation_key="FAKECLONE:0_start_0" on_node="c7auto3" on_node_uuid="3"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="1">
+ <action_set>
+ <rsc_op id="11" operation="start" operation_key="FAKECLONE_start_0" internal_operation_key="FAKECLONE:0_start_0" on_node="c7auto3" on_node_uuid="3">
+ <primitive id="FAKECLONE" long-id="FAKECLONE:0" class="ocf" provider="heartbeat" type="Dummy"/>
+ <attributes CRM_meta_clone="0" CRM_meta_clone_max="4" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_name="start" CRM_meta_notify="false" CRM_meta_timeout="20000" />
+ </rsc_op>
+ </action_set>
+ <inputs>
+ <trigger>
+ <pseudo_event id="15" operation="start" operation_key="FAKECLONE-clone_start_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="2">
+ <action_set>
+ <rsc_op id="14" operation="monitor" operation_key="FAKECLONE:1_monitor_10000" on_node="c7auto1" on_node_uuid="1">
+ <primitive id="FAKECLONE" long-id="FAKECLONE:1" class="ocf" provider="heartbeat" type="Dummy"/>
+ <attributes CRM_meta_clone="1" CRM_meta_clone_max="4" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_interval="10000" CRM_meta_name="monitor" CRM_meta_notify="false" CRM_meta_timeout="20000" />
+ </rsc_op>
+ </action_set>
+ <inputs>
+ <trigger>
+ <rsc_op id="13" operation="start" operation_key="FAKECLONE:1_start_0" on_node="c7auto1" on_node_uuid="1"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="3">
+ <action_set>
+ <rsc_op id="13" operation="start" operation_key="FAKECLONE:1_start_0" on_node="c7auto1" on_node_uuid="1">
+ <primitive id="FAKECLONE" long-id="FAKECLONE:1" class="ocf" provider="heartbeat" type="Dummy"/>
+ <attributes CRM_meta_clone="1" CRM_meta_clone_max="4" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_name="start" CRM_meta_notify="false" CRM_meta_timeout="20000" />
+ </rsc_op>
+ </action_set>
+ <inputs>
+ <trigger>
+ <pseudo_event id="15" operation="start" operation_key="FAKECLONE-clone_start_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="4" priority="1000000">
+ <action_set>
+ <pseudo_event id="16" operation="running" operation_key="FAKECLONE-clone_running_0">
+ <attributes CRM_meta_clone_max="4" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="false" CRM_meta_timeout="20000" />
+ </pseudo_event>
+ </action_set>
+ <inputs>
+ <trigger>
+ <rsc_op id="11" operation="start" operation_key="FAKECLONE_start_0" internal_operation_key="FAKECLONE:0_start_0" on_node="c7auto3" on_node_uuid="3"/>
+ </trigger>
+ <trigger>
+ <rsc_op id="13" operation="start" operation_key="FAKECLONE:1_start_0" on_node="c7auto1" on_node_uuid="1"/>
+ </trigger>
+ <trigger>
+ <pseudo_event id="15" operation="start" operation_key="FAKECLONE-clone_start_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="5">
+ <action_set>
+ <pseudo_event id="15" operation="start" operation_key="FAKECLONE-clone_start_0">
+ <attributes CRM_meta_clone_max="4" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="false" CRM_meta_timeout="20000" />
+ </pseudo_event>
+ </action_set>
+ <inputs/>
+ </synapse>
+ <synapse id="6">
+ <action_set>
+ <rsc_op id="20" operation="monitor" operation_key="FAKE_monitor_10000" on_node="c7auto4" on_node_uuid="4">
+ <primitive id="FAKE" class="ocf" provider="heartbeat" type="Dummy"/>
+ <attributes CRM_meta_interval="10000" CRM_meta_name="monitor" CRM_meta_timeout="20000" />
+ </rsc_op>
+ </action_set>
+ <inputs>
+ <trigger>
+ <rsc_op id="19" operation="start" operation_key="FAKE_start_0" on_node="c7auto4" on_node_uuid="4"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="7">
+ <action_set>
+ <rsc_op id="19" operation="start" operation_key="FAKE_start_0" on_node="c7auto4" on_node_uuid="4">
+ <primitive id="FAKE" class="ocf" provider="heartbeat" type="Dummy"/>
+ <attributes CRM_meta_name="start" CRM_meta_timeout="20000" />
+ </rsc_op>
+ </action_set>
+ <inputs>
+ <trigger>
+ <pseudo_event id="2" operation="clone-one-or-more:order-FAKECLONE-clone-FAKE-mandatory" operation_key="clone-one-or-more:order-FAKECLONE-clone-FAKE-mandatory"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="8">
+ <action_set>
+ <pseudo_event id="2" operation="clone-one-or-more:order-FAKECLONE-clone-FAKE-mandatory" operation_key="clone-one-or-more:order-FAKECLONE-clone-FAKE-mandatory">
+ <attributes />
+ </pseudo_event>
+ </action_set>
+ <inputs>
+ <trigger>
+ <rsc_op id="11" operation="start" operation_key="FAKECLONE_start_0" internal_operation_key="FAKECLONE:0_start_0" on_node="c7auto3" on_node_uuid="3"/>
+ </trigger>
+ <trigger>
+ <rsc_op id="13" operation="start" operation_key="FAKECLONE:1_start_0" on_node="c7auto1" on_node_uuid="1"/>
+ </trigger>
+ </inputs>
+ </synapse>
+</transition_graph>
diff --git a/pengine/test10/clone_min_start_two.scores b/pengine/test10/clone_min_start_two.scores
new file mode 100644
index 0000000..b3bcac0
--- /dev/null
+++ b/pengine/test10/clone_min_start_two.scores
@@ -0,0 +1,45 @@
+Allocation scores:
+clone_color: FAKECLONE-clone allocation score on c7auto1: 0
+clone_color: FAKECLONE-clone allocation score on c7auto2: 0
+clone_color: FAKECLONE-clone allocation score on c7auto3: 0
+clone_color: FAKECLONE-clone allocation score on c7auto4: -INFINITY
+clone_color: FAKECLONE:0 allocation score on c7auto1: 0
+clone_color: FAKECLONE:0 allocation score on c7auto2: 0
+clone_color: FAKECLONE:0 allocation score on c7auto3: 0
+clone_color: FAKECLONE:0 allocation score on c7auto4: -INFINITY
+clone_color: FAKECLONE:1 allocation score on c7auto1: 0
+clone_color: FAKECLONE:1 allocation score on c7auto2: 0
+clone_color: FAKECLONE:1 allocation score on c7auto3: 0
+clone_color: FAKECLONE:1 allocation score on c7auto4: -INFINITY
+clone_color: FAKECLONE:2 allocation score on c7auto1: 0
+clone_color: FAKECLONE:2 allocation score on c7auto2: 0
+clone_color: FAKECLONE:2 allocation score on c7auto3: 0
+clone_color: FAKECLONE:2 allocation score on c7auto4: -INFINITY
+clone_color: FAKECLONE:3 allocation score on c7auto1: 0
+clone_color: FAKECLONE:3 allocation score on c7auto2: 0
+clone_color: FAKECLONE:3 allocation score on c7auto3: 0
+clone_color: FAKECLONE:3 allocation score on c7auto4: -INFINITY
+native_color: FAKE allocation score on c7auto1: -INFINITY
+native_color: FAKE allocation score on c7auto2: -INFINITY
+native_color: FAKE allocation score on c7auto3: -INFINITY
+native_color: FAKE allocation score on c7auto4: 0
+native_color: FAKECLONE:0 allocation score on c7auto1: 0
+native_color: FAKECLONE:0 allocation score on c7auto2: -INFINITY
+native_color: FAKECLONE:0 allocation score on c7auto3: 0
+native_color: FAKECLONE:0 allocation score on c7auto4: -INFINITY
+native_color: FAKECLONE:1 allocation score on c7auto1: 0
+native_color: FAKECLONE:1 allocation score on c7auto2: -INFINITY
+native_color: FAKECLONE:1 allocation score on c7auto3: -INFINITY
+native_color: FAKECLONE:1 allocation score on c7auto4: -INFINITY
+native_color: FAKECLONE:2 allocation score on c7auto1: -INFINITY
+native_color: FAKECLONE:2 allocation score on c7auto2: -INFINITY
+native_color: FAKECLONE:2 allocation score on c7auto3: -INFINITY
+native_color: FAKECLONE:2 allocation score on c7auto4: -INFINITY
+native_color: FAKECLONE:3 allocation score on c7auto1: -INFINITY
+native_color: FAKECLONE:3 allocation score on c7auto2: -INFINITY
+native_color: FAKECLONE:3 allocation score on c7auto3: -INFINITY
+native_color: FAKECLONE:3 allocation score on c7auto4: -INFINITY
+native_color: shooter allocation score on c7auto1: 0
+native_color: shooter allocation score on c7auto2: 0
+native_color: shooter allocation score on c7auto3: 0
+native_color: shooter allocation score on c7auto4: 0
diff --git a/pengine/test10/clone_min_start_two.summary b/pengine/test10/clone_min_start_two.summary
new file mode 100644
index 0000000..f0c649c
--- /dev/null
+++ b/pengine/test10/clone_min_start_two.summary
@@ -0,0 +1,36 @@
+
+Current cluster status:
+Node c7auto2 (2): standby
+Online: [ c7auto1 c7auto3 c7auto4 ]
+
+ shooter (stonith:fence_phd_kvm): Started c7auto1
+ Clone Set: FAKECLONE-clone [FAKECLONE]
+ Stopped: [ c7auto1 c7auto2 c7auto3 c7auto4 ]
+ FAKE (ocf::heartbeat:Dummy): Stopped
+
+Transition Summary:
+ * Start FAKECLONE:0 (c7auto3)
+ * Start FAKECLONE:1 (c7auto1)
+ * Start FAKE (c7auto4)
+
+Executing cluster transition:
+ * Pseudo action: FAKECLONE-clone_start_0
+ * Resource action: FAKECLONE start on c7auto3
+ * Resource action: FAKECLONE start on c7auto1
+ * Pseudo action: FAKECLONE-clone_running_0
+ * Pseudo action: clone-one-or-more:order-FAKECLONE-clone-FAKE-mandatory
+ * Resource action: FAKECLONE monitor=10000 on c7auto3
+ * Resource action: FAKECLONE monitor=10000 on c7auto1
+ * Resource action: FAKE start on c7auto4
+ * Resource action: FAKE monitor=10000 on c7auto4
+
+Revised cluster status:
+Node c7auto2 (2): standby
+Online: [ c7auto1 c7auto3 c7auto4 ]
+
+ shooter (stonith:fence_phd_kvm): Started c7auto1
+ Clone Set: FAKECLONE-clone [FAKECLONE]
+ Started: [ c7auto1 c7auto3 ]
+ Stopped: [ c7auto2 c7auto4 ]
+ FAKE (ocf::heartbeat:Dummy): Started c7auto4
+
diff --git a/pengine/test10/clone_min_start_two.xml b/pengine/test10/clone_min_start_two.xml
new file mode 100644
index 0000000..ae84425
--- /dev/null
+++ b/pengine/test10/clone_min_start_two.xml
@@ -0,0 +1,153 @@
+<cib crm_feature_set="3.0.10" validate-with="pacemaker-2.3" epoch="18" num_updates="0" admin_epoch="0" cib-last-written="Thu Jul 16 15:37:54 2015" update-origin="c7auto1" update-client="crm_resource" update-user="root" have-quorum="1" dc-uuid="2">
+ <configuration>
+ <crm_config>
+ <cluster_property_set id="cib-bootstrap-options">
+ <nvpair id="cib-bootstrap-options-have-watchdog" name="have-watchdog" value="false"/>
+ <nvpair id="cib-bootstrap-options-dc-version" name="dc-version" value="1.1.12-ef86b49"/>
+ <nvpair id="cib-bootstrap-options-cluster-infrastructure" name="cluster-infrastructure" value="corosync"/>
+ <nvpair id="cib-bootstrap-options-cluster-name" name="cluster-name" value="phd"/>
+ </cluster_property_set>
+ </crm_config>
+ <nodes>
+ <node id="1" uname="c7auto1">
+ <instance_attributes id="nodes-1"/>
+ </node>
+ <node id="2" uname="c7auto2">
+ <instance_attributes id="nodes-2">
+ <nvpair id="nodes-2-standby" name="standby" value="on"/>
+ </instance_attributes>
+ </node>
+ <node id="3" uname="c7auto3"/>
+ <node id="4" uname="c7auto4"/>
+ </nodes>
+ <resources>
+ <primitive class="stonith" id="shooter" type="fence_phd_kvm">
+ <instance_attributes id="shooter-instance_attributes"/>
+ <operations>
+ <op id="shooter-monitor-interval-60s" interval="60s" name="monitor"/>
+ </operations>
+ </primitive>
+ <clone id="FAKECLONE-clone">
+ <primitive class="ocf" id="FAKECLONE" provider="heartbeat" type="Dummy">
+ <instance_attributes id="FAKECLONE-instance_attributes"/>
+ <operations>
+ <op id="FAKECLONE-start-timeout-20" interval="0s" name="start" timeout="20"/>
+ <op id="FAKECLONE-stop-timeout-20" interval="0s" name="stop" timeout="20"/>
+ <op id="FAKECLONE-monitor-interval-10" interval="10" name="monitor" timeout="20"/>
+ </operations>
+ </primitive>
+ <meta_attributes id="FAKECLONE-clone-meta">
+ <nvpair id="FAKECLONE-clone-min" name="clone-min" value="2"/>
+ </meta_attributes>
+ <meta_attributes id="FAKECLONE-clone-meta_attributes"/>
+ </clone>
+ <primitive class="ocf" id="FAKE" provider="heartbeat" type="Dummy">
+ <instance_attributes id="FAKE-instance_attributes"/>
+ <operations>
+ <op id="FAKE-start-timeout-20" interval="0s" name="start" timeout="20"/>
+ <op id="FAKE-stop-timeout-20" interval="0s" name="stop" timeout="20"/>
+ <op id="FAKE-monitor-interval-10" interval="10" name="monitor" timeout="20"/>
+ </operations>
+ </primitive>
+ </resources>
+ <constraints>
+ <rsc_location id="location-FAKECLONE-clone-c7auto4--INFINITY" node="c7auto4" rsc="FAKECLONE-clone" score="-INFINITY"/>
+ <rsc_location id="location-FAKE-c7auto1--INFINITY" node="c7auto1" rsc="FAKE" score="-INFINITY"/>
+ <rsc_location id="location-FAKE-c7auto2--INFINITY" node="c7auto2" rsc="FAKE" score="-INFINITY"/>
+ <rsc_location id="location-FAKE-c7auto3--INFINITY" node="c7auto3" rsc="FAKE" score="-INFINITY"/>
+ <rsc_order first="FAKECLONE-clone" first-action="start" id="order-FAKECLONE-clone-FAKE-mandatory" then="FAKE" then-action="start"/>
+ </constraints>
+ </configuration>
+ <status>
+ <node_state id="1" uname="c7auto1" in_ccm="true" crmd="online" crm-debug-origin="do_update_resource" join="member" expected="member">
+ <lrm id="1">
+ <lrm_resources>
+ <lrm_resource id="shooter" type="fence_phd_kvm" class="stonith">
+ <lrm_rsc_op id="shooter_last_0" operation_key="shooter_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="11:0:0:b489d50c-a7db-4605-8519-a2d5e1452e29" transition-magic="0:0;11:0:0:b489d50c-a7db-4605-8519-a2d5e1452e29" on_node="c7auto1" call-id="6" rc-code="0" op-status="0" interval="0" last-run="1437073991" last-rc-change="1437073991" exec-time="9" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ <lrm_rsc_op id="shooter_monitor_60000" operation_key="shooter_monitor_60000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="9:1:0:b489d50c-a7db-4605-8519-a2d5e1452e29" transition-magic="0:0;9:1:0:b489d50c-a7db-4605-8519-a2d5e1452e29" on_node="c7auto1" call-id="7" rc-code="0" op-status="0" interval="60000" last-rc-change="1437073992" exec-time="7" queue-time="0" op-digest="4811cef7f7f94e3a35a70be7916cb2fd"/>
+ </lrm_resource>
+ <lrm_resource id="FAKECLONE" type="Dummy" class="ocf" provider="heartbeat">
+ <lrm_rsc_op id="FAKECLONE_last_0" operation_key="FAKECLONE_stop_0" operation="stop" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="14:9:0:b489d50c-a7db-4605-8519-a2d5e1452e29" transition-magic="0:0;14:9:0:b489d50c-a7db-4605-8519-a2d5e1452e29" on_node="c7auto1" call-id="20" rc-code="0" op-status="0" interval="0" last-run="1437075474" last-rc-change="1437075474" exec-time="8" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" fake " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ <lrm_rsc_op id="FAKECLONE_monitor_10000" operation_key="FAKECLONE_monitor_10000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="21:2:0:b489d50c-a7db-4605-8519-a2d5e1452e29" transition-magic="0:0;21:2:0:b489d50c-a7db-4605-8519-a2d5e1452e29" on_node="c7auto1" call-id="14" rc-code="0" op-status="0" interval="10000" last-rc-change="1437074829" exec-time="7" queue-time="0" op-digest="4811cef7f7f94e3a35a70be7916cb2fd"/>
+ </lrm_resource>
+ <lrm_resource id="FAKE" type="Dummy" class="ocf" provider="heartbeat">
+ <lrm_rsc_op id="FAKE_last_0" operation_key="FAKE_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="8:4:7:b489d50c-a7db-4605-8519-a2d5e1452e29" transition-magic="0:7;8:4:7:b489d50c-a7db-4605-8519-a2d5e1452e29" on_node="c7auto1" call-id="18" rc-code="7" op-status="0" interval="0" last-run="1437074871" last-rc-change="1437074871" exec-time="7" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" fake " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ </lrm_resource>
+ </lrm_resources>
+ </lrm>
+ <transient_attributes id="1">
+ <instance_attributes id="status-1">
+ <nvpair id="status-1-shutdown" name="shutdown" value="0"/>
+ <nvpair id="status-1-probe_complete" name="probe_complete" value="true"/>
+ </instance_attributes>
+ </transient_attributes>
+ </node_state>
+ <node_state id="2" uname="c7auto2" in_ccm="true" crmd="online" crm-debug-origin="do_update_resource" join="member" expected="member">
+ <lrm id="2">
+ <lrm_resources>
+ <lrm_resource id="shooter" type="fence_phd_kvm" class="stonith">
+ <lrm_rsc_op id="shooter_last_0" operation_key="shooter_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="6:0:7:b489d50c-a7db-4605-8519-a2d5e1452e29" transition-magic="0:7;6:0:7:b489d50c-a7db-4605-8519-a2d5e1452e29" on_node="c7auto2" call-id="5" rc-code="7" op-status="0" interval="0" last-run="1437073988" last-rc-change="1437073988" exec-time="964" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ </lrm_resource>
+ <lrm_resource id="FAKECLONE" type="Dummy" class="ocf" provider="heartbeat">
+ <lrm_rsc_op id="FAKECLONE_last_0" operation_key="FAKECLONE_stop_0" operation="stop" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="15:9:0:b489d50c-a7db-4605-8519-a2d5e1452e29" transition-magic="0:0;15:9:0:b489d50c-a7db-4605-8519-a2d5e1452e29" on_node="c7auto2" call-id="18" rc-code="0" op-status="0" interval="0" last-run="1437075473" last-rc-change="1437075473" exec-time="13" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" fake " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ <lrm_rsc_op id="FAKECLONE_monitor_10000" operation_key="FAKECLONE_monitor_10000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="15:2:0:b489d50c-a7db-4605-8519-a2d5e1452e29" transition-magic="0:0;15:2:0:b489d50c-a7db-4605-8519-a2d5e1452e29" on_node="c7auto2" call-id="12" rc-code="0" op-status="0" interval="10000" last-rc-change="1437074829" exec-time="10" queue-time="0" op-digest="4811cef7f7f94e3a35a70be7916cb2fd"/>
+ </lrm_resource>
+ <lrm_resource id="FAKE" type="Dummy" class="ocf" provider="heartbeat">
+ <lrm_rsc_op id="FAKE_last_0" operation_key="FAKE_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="10:4:7:b489d50c-a7db-4605-8519-a2d5e1452e29" transition-magic="0:7;10:4:7:b489d50c-a7db-4605-8519-a2d5e1452e29" on_node="c7auto2" call-id="16" rc-code="7" op-status="0" interval="0" last-run="1437074871" last-rc-change="1437074871" exec-time="10" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" fake " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ </lrm_resource>
+ </lrm_resources>
+ </lrm>
+ <transient_attributes id="2">
+ <instance_attributes id="status-2">
+ <nvpair id="status-2-shutdown" name="shutdown" value="0"/>
+ <nvpair id="status-2-probe_complete" name="probe_complete" value="true"/>
+ </instance_attributes>
+ </transient_attributes>
+ </node_state>
+ <node_state id="3" uname="c7auto3" in_ccm="true" crmd="online" crm-debug-origin="do_update_resource" join="member" expected="member">
+ <lrm id="3">
+ <lrm_resources>
+ <lrm_resource id="shooter" type="fence_phd_kvm" class="stonith">
+ <lrm_rsc_op id="shooter_last_0" operation_key="shooter_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="8:0:7:b489d50c-a7db-4605-8519-a2d5e1452e29" transition-magic="0:7;8:0:7:b489d50c-a7db-4605-8519-a2d5e1452e29" on_node="c7auto3" call-id="5" rc-code="7" op-status="0" interval="0" last-run="1437073988" last-rc-change="1437073988" exec-time="963" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ </lrm_resource>
+ <lrm_resource id="FAKECLONE" type="Dummy" class="ocf" provider="heartbeat">
+ <lrm_rsc_op id="FAKECLONE_last_0" operation_key="FAKECLONE_stop_0" operation="stop" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="16:9:0:b489d50c-a7db-4605-8519-a2d5e1452e29" transition-magic="0:0;16:9:0:b489d50c-a7db-4605-8519-a2d5e1452e29" on_node="c7auto3" call-id="18" rc-code="0" op-status="0" interval="0" last-run="1437075473" last-rc-change="1437075473" exec-time="12" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" fake " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ <lrm_rsc_op id="FAKECLONE_monitor_10000" operation_key="FAKECLONE_monitor_10000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="17:2:0:b489d50c-a7db-4605-8519-a2d5e1452e29" transition-magic="0:0;17:2:0:b489d50c-a7db-4605-8519-a2d5e1452e29" on_node="c7auto3" call-id="12" rc-code="0" op-status="0" interval="10000" last-rc-change="1437074828" exec-time="10" queue-time="0" op-digest="4811cef7f7f94e3a35a70be7916cb2fd"/>
+ </lrm_resource>
+ <lrm_resource id="FAKE" type="Dummy" class="ocf" provider="heartbeat">
+ <lrm_rsc_op id="FAKE_last_0" operation_key="FAKE_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="12:4:7:b489d50c-a7db-4605-8519-a2d5e1452e29" transition-magic="0:7;12:4:7:b489d50c-a7db-4605-8519-a2d5e1452e29" on_node="c7auto3" call-id="16" rc-code="7" op-status="0" interval="0" last-run="1437074870" last-rc-change="1437074870" exec-time="8" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" fake " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ </lrm_resource>
+ </lrm_resources>
+ </lrm>
+ <transient_attributes id="3">
+ <instance_attributes id="status-3">
+ <nvpair id="status-3-shutdown" name="shutdown" value="0"/>
+ <nvpair id="status-3-probe_complete" name="probe_complete" value="true"/>
+ </instance_attributes>
+ </transient_attributes>
+ </node_state>
+ <node_state id="4" uname="c7auto4" in_ccm="true" crmd="online" crm-debug-origin="do_update_resource" join="member" expected="member">
+ <lrm id="4">
+ <lrm_resources>
+ <lrm_resource id="shooter" type="fence_phd_kvm" class="stonith">
+ <lrm_rsc_op id="shooter_last_0" operation_key="shooter_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="10:0:7:b489d50c-a7db-4605-8519-a2d5e1452e29" transition-magic="0:7;10:0:7:b489d50c-a7db-4605-8519-a2d5e1452e29" on_node="c7auto4" call-id="5" rc-code="7" op-status="0" interval="0" last-run="1437073988" last-rc-change="1437073988" exec-time="963" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ </lrm_resource>
+ <lrm_resource id="FAKECLONE" type="Dummy" class="ocf" provider="heartbeat">
+ <lrm_rsc_op id="FAKECLONE_last_0" operation_key="FAKECLONE_stop_0" operation="stop" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="20:3:0:b489d50c-a7db-4605-8519-a2d5e1452e29" transition-magic="0:0;20:3:0:b489d50c-a7db-4605-8519-a2d5e1452e29" on_node="c7auto4" call-id="14" rc-code="0" op-status="0" interval="0" last-run="1437074856" last-rc-change="1437074856" exec-time="8" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" fake " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ <lrm_rsc_op id="FAKECLONE_monitor_10000" operation_key="FAKECLONE_monitor_10000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="19:2:0:b489d50c-a7db-4605-8519-a2d5e1452e29" transition-magic="0:0;19:2:0:b489d50c-a7db-4605-8519-a2d5e1452e29" on_node="c7auto4" call-id="12" rc-code="0" op-status="0" interval="10000" last-rc-change="1437074829" exec-time="6" queue-time="0" op-digest="4811cef7f7f94e3a35a70be7916cb2fd"/>
+ </lrm_resource>
+ <lrm_resource id="FAKE" type="Dummy" class="ocf" provider="heartbeat">
+ <lrm_rsc_op id="FAKE_last_0" operation_key="FAKE_stop_0" operation="stop" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="21:9:0:b489d50c-a7db-4605-8519-a2d5e1452e29" transition-magic="0:0;21:9:0:b489d50c-a7db-4605-8519-a2d5e1452e29" on_node="c7auto4" call-id="22" rc-code="0" op-status="0" interval="0" last-run="1437075473" last-rc-change="1437075473" exec-time="6" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" fake " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ <lrm_rsc_op id="FAKE_monitor_10000" operation_key="FAKE_monitor_10000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="28:4:0:b489d50c-a7db-4605-8519-a2d5e1452e29" transition-magic="0:0;28:4:0:b489d50c-a7db-4605-8519-a2d5e1452e29" on_node="c7auto4" call-id="20" rc-code="0" op-status="0" interval="10000" last-rc-change="1437074871" exec-time="7" queue-time="0" op-digest="4811cef7f7f94e3a35a70be7916cb2fd"/>
+ </lrm_resource>
+ </lrm_resources>
+ </lrm>
+ <transient_attributes id="4">
+ <instance_attributes id="status-4">
+ <nvpair id="status-4-shutdown" name="shutdown" value="0"/>
+ <nvpair id="status-4-probe_complete" name="probe_complete" value="true"/>
+ </instance_attributes>
+ </transient_attributes>
+ </node_state>
+ </status>
+</cib>
diff --git a/pengine/test10/clone_min_stop_all.dot b/pengine/test10/clone_min_stop_all.dot
new file mode 100644
index 0000000..254e889
--- /dev/null
+++ b/pengine/test10/clone_min_stop_all.dot
@@ -0,0 +1,41 @@
+ digraph "g" {
+"FAKECLONE-clone_running_0" [ style=dashed color="red" fontcolor="orange"]
+"FAKECLONE-clone_start_0" -> "FAKECLONE-clone_running_0" [ style = dashed]
+"FAKECLONE-clone_start_0" -> "FAKECLONE_start_0 <none>" [ style = dashed]
+"FAKECLONE-clone_start_0" [ style=dashed color="red" fontcolor="orange"]
+"FAKECLONE-clone_stop_0" -> "FAKECLONE-clone_stopped_0" [ style = bold]
+"FAKECLONE-clone_stop_0" -> "FAKECLONE_stop_0 c7auto1" [ style = bold]
+"FAKECLONE-clone_stop_0" -> "FAKECLONE_stop_0 c7auto2" [ style = bold]
+"FAKECLONE-clone_stop_0" -> "FAKECLONE_stop_0 c7auto3" [ style = bold]
+"FAKECLONE-clone_stop_0" [ style=bold color="green" fontcolor="orange"]
+"FAKECLONE-clone_stopped_0" -> "FAKECLONE-clone_start_0" [ style = dashed]
+"FAKECLONE-clone_stopped_0" [ style=bold color="green" fontcolor="orange"]
+"FAKECLONE_start_0 <none>" -> "FAKECLONE-clone_running_0" [ style = dashed]
+"FAKECLONE_start_0 <none>" [ style=dashed color="red" fontcolor="black"]
+"FAKECLONE_stop_0 c7auto1" -> "FAKECLONE-clone_stopped_0" [ style = bold]
+"FAKECLONE_stop_0 c7auto1" -> "FAKECLONE_start_0 <none>" [ style = dashed]
+"FAKECLONE_stop_0 c7auto1" -> "all_stopped" [ style = bold]
+"FAKECLONE_stop_0 c7auto1" [ style=bold color="green" fontcolor="black"]
+"FAKECLONE_stop_0 c7auto2" -> "FAKECLONE-clone_stopped_0" [ style = bold]
+"FAKECLONE_stop_0 c7auto2" -> "FAKECLONE_start_0 <none>" [ style = dashed]
+"FAKECLONE_stop_0 c7auto2" -> "all_stopped" [ style = bold]
+"FAKECLONE_stop_0 c7auto2" [ style=bold color="green" fontcolor="black"]
+"FAKECLONE_stop_0 c7auto3" -> "FAKECLONE-clone_stopped_0" [ style = bold]
+"FAKECLONE_stop_0 c7auto3" -> "FAKECLONE_start_0 <none>" [ style = dashed]
+"FAKECLONE_stop_0 c7auto3" -> "all_stopped" [ style = bold]
+"FAKECLONE_stop_0 c7auto3" [ style=bold color="green" fontcolor="black"]
+"FAKE_monitor_10000 c7auto4" [ style=dashed color="red" fontcolor="black"]
+"FAKE_start_0 c7auto4" -> "FAKE_monitor_10000 c7auto4" [ style = dashed]
+"FAKE_start_0 c7auto4" [ style=dashed color="red" fontcolor="black"]
+"FAKE_stop_0 c7auto4" -> "FAKECLONE-clone_stop_0" [ style = bold]
+"FAKE_stop_0 c7auto4" -> "FAKE_start_0 c7auto4" [ style = dashed]
+"FAKE_stop_0 c7auto4" -> "all_stopped" [ style = bold]
+"FAKE_stop_0 c7auto4" [ style=bold color="green" fontcolor="black"]
+"all_stopped" [ style=bold color="green" fontcolor="orange"]
+"shooter_monitor_60000 c7auto4" [ style=bold color="green" fontcolor="black"]
+"shooter_start_0 c7auto4" -> "shooter_monitor_60000 c7auto4" [ style = bold]
+"shooter_start_0 c7auto4" [ style=bold color="green" fontcolor="black"]
+"shooter_stop_0 c7auto1" -> "all_stopped" [ style = bold]
+"shooter_stop_0 c7auto1" -> "shooter_start_0 c7auto4" [ style = bold]
+"shooter_stop_0 c7auto1" [ style=bold color="green" fontcolor="black"]
+}
diff --git a/pengine/test10/clone_min_stop_all.exp b/pengine/test10/clone_min_stop_all.exp
new file mode 100644
index 0000000..1b8c9ce
--- /dev/null
+++ b/pengine/test10/clone_min_stop_all.exp
@@ -0,0 +1,142 @@
+<transition_graph cluster-delay="60s" stonith-timeout="60s" failed-stop-offset="INFINITY" failed-start-offset="INFINITY" transition_id="0">
+ <synapse id="0">
+ <action_set>
+ <rsc_op id="15" operation="monitor" operation_key="shooter_monitor_60000" on_node="c7auto4" on_node_uuid="4">
+ <primitive id="shooter" class="stonith" type="fence_phd_kvm"/>
+ <attributes CRM_meta_interval="60000" CRM_meta_name="monitor" CRM_meta_timeout="20000" />
+ </rsc_op>
+ </action_set>
+ <inputs>
+ <trigger>
+ <rsc_op id="14" operation="start" operation_key="shooter_start_0" on_node="c7auto4" on_node_uuid="4"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="1">
+ <action_set>
+ <rsc_op id="14" operation="start" operation_key="shooter_start_0" on_node="c7auto4" on_node_uuid="4">
+ <primitive id="shooter" class="stonith" type="fence_phd_kvm"/>
+ <attributes CRM_meta_timeout="20000" />
+ </rsc_op>
+ </action_set>
+ <inputs>
+ <trigger>
+ <rsc_op id="13" operation="stop" operation_key="shooter_stop_0" on_node="c7auto1" on_node_uuid="1"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="2">
+ <action_set>
+ <rsc_op id="13" operation="stop" operation_key="shooter_stop_0" on_node="c7auto1" on_node_uuid="1">
+ <primitive id="shooter" class="stonith" type="fence_phd_kvm"/>
+ <attributes CRM_meta_timeout="20000" />
+ </rsc_op>
+ </action_set>
+ <inputs/>
+ </synapse>
+ <synapse id="3">
+ <action_set>
+ <rsc_op id="16" operation="stop" operation_key="FAKECLONE_stop_0" internal_operation_key="FAKECLONE:0_stop_0" on_node="c7auto1" on_node_uuid="1">
+ <primitive id="FAKECLONE" long-id="FAKECLONE:0" class="ocf" provider="heartbeat" type="Dummy"/>
+ <attributes CRM_meta_clone="0" CRM_meta_clone_max="4" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_name="stop" CRM_meta_notify="false" CRM_meta_timeout="20000" />
+ </rsc_op>
+ </action_set>
+ <inputs>
+ <trigger>
+ <pseudo_event id="21" operation="stop" operation_key="FAKECLONE-clone_stop_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="4">
+ <action_set>
+ <rsc_op id="17" operation="stop" operation_key="FAKECLONE_stop_0" internal_operation_key="FAKECLONE:1_stop_0" on_node="c7auto2" on_node_uuid="2">
+ <primitive id="FAKECLONE" long-id="FAKECLONE:1" class="ocf" provider="heartbeat" type="Dummy"/>
+ <attributes CRM_meta_clone="1" CRM_meta_clone_max="4" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_name="stop" CRM_meta_notify="false" CRM_meta_timeout="20000" />
+ </rsc_op>
+ </action_set>
+ <inputs>
+ <trigger>
+ <pseudo_event id="21" operation="stop" operation_key="FAKECLONE-clone_stop_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="5">
+ <action_set>
+ <rsc_op id="18" operation="stop" operation_key="FAKECLONE_stop_0" internal_operation_key="FAKECLONE:2_stop_0" on_node="c7auto3" on_node_uuid="3">
+ <primitive id="FAKECLONE" long-id="FAKECLONE:2" class="ocf" provider="heartbeat" type="Dummy"/>
+ <attributes CRM_meta_clone="2" CRM_meta_clone_max="4" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_name="stop" CRM_meta_notify="false" CRM_meta_timeout="20000" />
+ </rsc_op>
+ </action_set>
+ <inputs>
+ <trigger>
+ <pseudo_event id="21" operation="stop" operation_key="FAKECLONE-clone_stop_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="6" priority="1000000">
+ <action_set>
+ <pseudo_event id="22" operation="stopped" operation_key="FAKECLONE-clone_stopped_0">
+ <attributes CRM_meta_clone_max="4" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="false" CRM_meta_timeout="20000" />
+ </pseudo_event>
+ </action_set>
+ <inputs>
+ <trigger>
+ <rsc_op id="16" operation="stop" operation_key="FAKECLONE_stop_0" internal_operation_key="FAKECLONE:0_stop_0" on_node="c7auto1" on_node_uuid="1"/>
+ </trigger>
+ <trigger>
+ <rsc_op id="17" operation="stop" operation_key="FAKECLONE_stop_0" internal_operation_key="FAKECLONE:1_stop_0" on_node="c7auto2" on_node_uuid="2"/>
+ </trigger>
+ <trigger>
+ <rsc_op id="18" operation="stop" operation_key="FAKECLONE_stop_0" internal_operation_key="FAKECLONE:2_stop_0" on_node="c7auto3" on_node_uuid="3"/>
+ </trigger>
+ <trigger>
+ <pseudo_event id="21" operation="stop" operation_key="FAKECLONE-clone_stop_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="7">
+ <action_set>
+ <pseudo_event id="21" operation="stop" operation_key="FAKECLONE-clone_stop_0">
+ <attributes CRM_meta_clone_max="4" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="false" CRM_meta_timeout="20000" />
+ </pseudo_event>
+ </action_set>
+ <inputs>
+ <trigger>
+ <rsc_op id="23" operation="stop" operation_key="FAKE_stop_0" on_node="c7auto4" on_node_uuid="4"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="8">
+ <action_set>
+ <rsc_op id="23" operation="stop" operation_key="FAKE_stop_0" on_node="c7auto4" on_node_uuid="4">
+ <primitive id="FAKE" class="ocf" provider="heartbeat" type="Dummy"/>
+ <attributes CRM_meta_name="stop" CRM_meta_timeout="20000" />
+ </rsc_op>
+ </action_set>
+ <inputs/>
+ </synapse>
+ <synapse id="9">
+ <action_set>
+ <pseudo_event id="7" operation="all_stopped" operation_key="all_stopped">
+ <attributes />
+ </pseudo_event>
+ </action_set>
+ <inputs>
+ <trigger>
+ <rsc_op id="13" operation="stop" operation_key="shooter_stop_0" on_node="c7auto1" on_node_uuid="1"/>
+ </trigger>
+ <trigger>
+ <rsc_op id="16" operation="stop" operation_key="FAKECLONE_stop_0" internal_operation_key="FAKECLONE:0_stop_0" on_node="c7auto1" on_node_uuid="1"/>
+ </trigger>
+ <trigger>
+ <rsc_op id="17" operation="stop" operation_key="FAKECLONE_stop_0" internal_operation_key="FAKECLONE:1_stop_0" on_node="c7auto2" on_node_uuid="2"/>
+ </trigger>
+ <trigger>
+ <rsc_op id="18" operation="stop" operation_key="FAKECLONE_stop_0" internal_operation_key="FAKECLONE:2_stop_0" on_node="c7auto3" on_node_uuid="3"/>
+ </trigger>
+ <trigger>
+ <rsc_op id="23" operation="stop" operation_key="FAKE_stop_0" on_node="c7auto4" on_node_uuid="4"/>
+ </trigger>
+ </inputs>
+ </synapse>
+</transition_graph>
diff --git a/pengine/test10/clone_min_stop_all.scores b/pengine/test10/clone_min_stop_all.scores
new file mode 100644
index 0000000..0bcbb1f
--- /dev/null
+++ b/pengine/test10/clone_min_stop_all.scores
@@ -0,0 +1,45 @@
+Allocation scores:
+clone_color: FAKECLONE-clone allocation score on c7auto1: 0
+clone_color: FAKECLONE-clone allocation score on c7auto2: 0
+clone_color: FAKECLONE-clone allocation score on c7auto3: 0
+clone_color: FAKECLONE-clone allocation score on c7auto4: -INFINITY
+clone_color: FAKECLONE:0 allocation score on c7auto1: 1
+clone_color: FAKECLONE:0 allocation score on c7auto2: 0
+clone_color: FAKECLONE:0 allocation score on c7auto3: 0
+clone_color: FAKECLONE:0 allocation score on c7auto4: -INFINITY
+clone_color: FAKECLONE:1 allocation score on c7auto1: 0
+clone_color: FAKECLONE:1 allocation score on c7auto2: 1
+clone_color: FAKECLONE:1 allocation score on c7auto3: 0
+clone_color: FAKECLONE:1 allocation score on c7auto4: -INFINITY
+clone_color: FAKECLONE:2 allocation score on c7auto1: 0
+clone_color: FAKECLONE:2 allocation score on c7auto2: 0
+clone_color: FAKECLONE:2 allocation score on c7auto3: 1
+clone_color: FAKECLONE:2 allocation score on c7auto4: -INFINITY
+clone_color: FAKECLONE:3 allocation score on c7auto1: 0
+clone_color: FAKECLONE:3 allocation score on c7auto2: 0
+clone_color: FAKECLONE:3 allocation score on c7auto3: 0
+clone_color: FAKECLONE:3 allocation score on c7auto4: -INFINITY
+native_color: FAKE allocation score on c7auto1: -INFINITY
+native_color: FAKE allocation score on c7auto2: -INFINITY
+native_color: FAKE allocation score on c7auto3: -INFINITY
+native_color: FAKE allocation score on c7auto4: 0
+native_color: FAKECLONE:0 allocation score on c7auto1: -INFINITY
+native_color: FAKECLONE:0 allocation score on c7auto2: -INFINITY
+native_color: FAKECLONE:0 allocation score on c7auto3: -INFINITY
+native_color: FAKECLONE:0 allocation score on c7auto4: -INFINITY
+native_color: FAKECLONE:1 allocation score on c7auto1: -INFINITY
+native_color: FAKECLONE:1 allocation score on c7auto2: -INFINITY
+native_color: FAKECLONE:1 allocation score on c7auto3: -INFINITY
+native_color: FAKECLONE:1 allocation score on c7auto4: -INFINITY
+native_color: FAKECLONE:2 allocation score on c7auto1: -INFINITY
+native_color: FAKECLONE:2 allocation score on c7auto2: -INFINITY
+native_color: FAKECLONE:2 allocation score on c7auto3: -INFINITY
+native_color: FAKECLONE:2 allocation score on c7auto4: -INFINITY
+native_color: FAKECLONE:3 allocation score on c7auto1: -INFINITY
+native_color: FAKECLONE:3 allocation score on c7auto2: -INFINITY
+native_color: FAKECLONE:3 allocation score on c7auto3: -INFINITY
+native_color: FAKECLONE:3 allocation score on c7auto4: -INFINITY
+native_color: shooter allocation score on c7auto1: 0
+native_color: shooter allocation score on c7auto2: 0
+native_color: shooter allocation score on c7auto3: 0
+native_color: shooter allocation score on c7auto4: 0
diff --git a/pengine/test10/clone_min_stop_all.summary b/pengine/test10/clone_min_stop_all.summary
new file mode 100644
index 0000000..eb2944f
--- /dev/null
+++ b/pengine/test10/clone_min_stop_all.summary
@@ -0,0 +1,43 @@
+
+Current cluster status:
+Node c7auto1 (1): standby
+Node c7auto2 (2): standby
+Node c7auto3 (3): standby
+Online: [ c7auto4 ]
+
+ shooter (stonith:fence_phd_kvm): Started c7auto1
+ Clone Set: FAKECLONE-clone [FAKECLONE]
+ Started: [ c7auto1 c7auto2 c7auto3 ]
+ Stopped: [ c7auto4 ]
+ FAKE (ocf::heartbeat:Dummy): Started c7auto4
+
+Transition Summary:
+ * Move shooter (Started c7auto1 -> c7auto4)
+ * Stop FAKECLONE:0 (c7auto1)
+ * Stop FAKECLONE:1 (c7auto2)
+ * Stop FAKECLONE:2 (c7auto3)
+ * Stop FAKE (Started c7auto4)
+
+Executing cluster transition:
+ * Resource action: shooter stop on c7auto1
+ * Resource action: FAKE stop on c7auto4
+ * Resource action: shooter start on c7auto4
+ * Pseudo action: FAKECLONE-clone_stop_0
+ * Resource action: shooter monitor=60000 on c7auto4
+ * Resource action: FAKECLONE stop on c7auto1
+ * Resource action: FAKECLONE stop on c7auto2
+ * Resource action: FAKECLONE stop on c7auto3
+ * Pseudo action: FAKECLONE-clone_stopped_0
+ * Pseudo action: all_stopped
+
+Revised cluster status:
+Node c7auto1 (1): standby
+Node c7auto2 (2): standby
+Node c7auto3 (3): standby
+Online: [ c7auto4 ]
+
+ shooter (stonith:fence_phd_kvm): Started c7auto4
+ Clone Set: FAKECLONE-clone [FAKECLONE]
+ Stopped: [ c7auto1 c7auto2 c7auto3 c7auto4 ]
+ FAKE (ocf::heartbeat:Dummy): Stopped
+
diff --git a/pengine/test10/clone_min_stop_all.xml b/pengine/test10/clone_min_stop_all.xml
new file mode 100644
index 0000000..70e8a96
--- /dev/null
+++ b/pengine/test10/clone_min_stop_all.xml
@@ -0,0 +1,158 @@
+<cib crm_feature_set="3.0.10" validate-with="pacemaker-2.3" epoch="18" num_updates="0" admin_epoch="0" cib-last-written="Thu Jul 16 15:28:22 2015" update-origin="c7auto1" update-client="cibadmin" update-user="root" have-quorum="1" dc-uuid="2">
+ <configuration>
+ <crm_config>
+ <cluster_property_set id="cib-bootstrap-options">
+ <nvpair id="cib-bootstrap-options-have-watchdog" name="have-watchdog" value="false"/>
+ <nvpair id="cib-bootstrap-options-dc-version" name="dc-version" value="1.1.12-ef86b49"/>
+ <nvpair id="cib-bootstrap-options-cluster-infrastructure" name="cluster-infrastructure" value="corosync"/>
+ <nvpair id="cib-bootstrap-options-cluster-name" name="cluster-name" value="phd"/>
+ </cluster_property_set>
+ </crm_config>
+ <nodes>
+ <node id="1" uname="c7auto1">
+ <instance_attributes id="nodes-1">
+ <nvpair id="nodes-1-standby" name="standby" value="on"/>
+ </instance_attributes>
+ </node>
+ <node id="2" uname="c7auto2">
+ <instance_attributes id="nodes-2">
+ <nvpair id="nodes-2-standby" name="standby" value="on"/>
+ </instance_attributes>
+ </node>
+ <node id="3" uname="c7auto3">
+ <instance_attributes id="nodes-3">
+ <nvpair id="nodes-3-standby" name="standby" value="on"/>
+ </instance_attributes>
+ </node>
+ <node id="4" uname="c7auto4"/>
+ </nodes>
+ <resources>
+ <primitive class="stonith" id="shooter" type="fence_phd_kvm">
+ <instance_attributes id="shooter-instance_attributes"/>
+ <operations>
+ <op id="shooter-monitor-interval-60s" interval="60s" name="monitor"/>
+ </operations>
+ </primitive>
+ <clone id="FAKECLONE-clone">
+ <primitive class="ocf" id="FAKECLONE" provider="heartbeat" type="Dummy">
+ <instance_attributes id="FAKECLONE-instance_attributes"/>
+ <operations>
+ <op id="FAKECLONE-start-timeout-20" interval="0s" name="start" timeout="20"/>
+ <op id="FAKECLONE-stop-timeout-20" interval="0s" name="stop" timeout="20"/>
+ <op id="FAKECLONE-monitor-interval-10" interval="10" name="monitor" timeout="20"/>
+ </operations>
+ </primitive>
+ <meta_attributes id="FAKECLONE-clone-meta">
+ <nvpair id="FAKECLONE-clone-min" name="clone-min" value="2"/>
+ </meta_attributes>
+ </clone>
+ <primitive class="ocf" id="FAKE" provider="heartbeat" type="Dummy">
+ <instance_attributes id="FAKE-instance_attributes"/>
+ <operations>
+ <op id="FAKE-start-timeout-20" interval="0s" name="start" timeout="20"/>
+ <op id="FAKE-stop-timeout-20" interval="0s" name="stop" timeout="20"/>
+ <op id="FAKE-monitor-interval-10" interval="10" name="monitor" timeout="20"/>
+ </operations>
+ </primitive>
+ </resources>
+ <constraints>
+ <rsc_location id="location-FAKECLONE-clone-c7auto4--INFINITY" node="c7auto4" rsc="FAKECLONE-clone" score="-INFINITY"/>
+ <rsc_location id="location-FAKE-c7auto1--INFINITY" node="c7auto1" rsc="FAKE" score="-INFINITY"/>
+ <rsc_location id="location-FAKE-c7auto2--INFINITY" node="c7auto2" rsc="FAKE" score="-INFINITY"/>
+ <rsc_location id="location-FAKE-c7auto3--INFINITY" node="c7auto3" rsc="FAKE" score="-INFINITY"/>
+ <rsc_order first="FAKECLONE-clone" first-action="start" id="order-FAKECLONE-clone-FAKE-mandatory" then="FAKE" then-action="start"/>
+ </constraints>
+ </configuration>
+ <status>
+ <node_state id="1" uname="c7auto1" in_ccm="true" crmd="online" crm-debug-origin="do_update_resource" join="member" expected="member">
+ <lrm id="1">
+ <lrm_resources>
+ <lrm_resource id="shooter" type="fence_phd_kvm" class="stonith">
+ <lrm_rsc_op id="shooter_last_0" operation_key="shooter_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="11:0:0:b489d50c-a7db-4605-8519-a2d5e1452e29" transition-magic="0:0;11:0:0:b489d50c-a7db-4605-8519-a2d5e1452e29" on_node="c7auto1" call-id="6" rc-code="0" op-status="0" interval="0" last-run="1437073991" last-rc-change="1437073991" exec-time="9" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ <lrm_rsc_op id="shooter_monitor_60000" operation_key="shooter_monitor_60000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="9:1:0:b489d50c-a7db-4605-8519-a2d5e1452e29" transition-magic="0:0;9:1:0:b489d50c-a7db-4605-8519-a2d5e1452e29" on_node="c7auto1" call-id="7" rc-code="0" op-status="0" interval="60000" last-rc-change="1437073992" exec-time="7" queue-time="0" op-digest="4811cef7f7f94e3a35a70be7916cb2fd"/>
+ </lrm_resource>
+ <lrm_resource id="FAKECLONE" type="Dummy" class="ocf" provider="heartbeat">
+ <lrm_rsc_op id="FAKECLONE_last_0" operation_key="FAKECLONE_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="20:2:0:b489d50c-a7db-4605-8519-a2d5e1452e29" transition-magic="0:0;20:2:0:b489d50c-a7db-4605-8519-a2d5e1452e29" on_node="c7auto1" call-id="13" rc-code="0" op-status="0" interval="0" last-run="1437074829" last-rc-change="1437074829" exec-time="7" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" fake " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ <lrm_rsc_op id="FAKECLONE_monitor_10000" operation_key="FAKECLONE_monitor_10000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="21:2:0:b489d50c-a7db-4605-8519-a2d5e1452e29" transition-magic="0:0;21:2:0:b489d50c-a7db-4605-8519-a2d5e1452e29" on_node="c7auto1" call-id="14" rc-code="0" op-status="0" interval="10000" last-rc-change="1437074829" exec-time="7" queue-time="0" op-digest="4811cef7f7f94e3a35a70be7916cb2fd"/>
+ </lrm_resource>
+ <lrm_resource id="FAKE" type="Dummy" class="ocf" provider="heartbeat">
+ <lrm_rsc_op id="FAKE_last_0" operation_key="FAKE_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="8:4:7:b489d50c-a7db-4605-8519-a2d5e1452e29" transition-magic="0:7;8:4:7:b489d50c-a7db-4605-8519-a2d5e1452e29" on_node="c7auto1" call-id="18" rc-code="7" op-status="0" interval="0" last-run="1437074871" last-rc-change="1437074871" exec-time="7" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" fake " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ </lrm_resource>
+ </lrm_resources>
+ </lrm>
+ <transient_attributes id="1">
+ <instance_attributes id="status-1">
+ <nvpair id="status-1-shutdown" name="shutdown" value="0"/>
+ <nvpair id="status-1-probe_complete" name="probe_complete" value="true"/>
+ </instance_attributes>
+ </transient_attributes>
+ </node_state>
+ <node_state id="2" uname="c7auto2" in_ccm="true" crmd="online" crm-debug-origin="do_update_resource" join="member" expected="member">
+ <lrm id="2">
+ <lrm_resources>
+ <lrm_resource id="shooter" type="fence_phd_kvm" class="stonith">
+ <lrm_rsc_op id="shooter_last_0" operation_key="shooter_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="6:0:7:b489d50c-a7db-4605-8519-a2d5e1452e29" transition-magic="0:7;6:0:7:b489d50c-a7db-4605-8519-a2d5e1452e29" on_node="c7auto2" call-id="5" rc-code="7" op-status="0" interval="0" last-run="1437073988" last-rc-change="1437073988" exec-time="964" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ </lrm_resource>
+ <lrm_resource id="FAKECLONE" type="Dummy" class="ocf" provider="heartbeat">
+ <lrm_rsc_op id="FAKECLONE_last_0" operation_key="FAKECLONE_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="14:2:0:b489d50c-a7db-4605-8519-a2d5e1452e29" transition-magic="0:0;14:2:0:b489d50c-a7db-4605-8519-a2d5e1452e29" on_node="c7auto2" call-id="11" rc-code="0" op-status="0" interval="0" last-run="1437074829" last-rc-change="1437074829" exec-time="8" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" fake " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ <lrm_rsc_op id="FAKECLONE_monitor_10000" operation_key="FAKECLONE_monitor_10000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="15:2:0:b489d50c-a7db-4605-8519-a2d5e1452e29" transition-magic="0:0;15:2:0:b489d50c-a7db-4605-8519-a2d5e1452e29" on_node="c7auto2" call-id="12" rc-code="0" op-status="0" interval="10000" last-rc-change="1437074829" exec-time="10" queue-time="0" op-digest="4811cef7f7f94e3a35a70be7916cb2fd"/>
+ </lrm_resource>
+ <lrm_resource id="FAKE" type="Dummy" class="ocf" provider="heartbeat">
+ <lrm_rsc_op id="FAKE_last_0" operation_key="FAKE_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="10:4:7:b489d50c-a7db-4605-8519-a2d5e1452e29" transition-magic="0:7;10:4:7:b489d50c-a7db-4605-8519-a2d5e1452e29" on_node="c7auto2" call-id="16" rc-code="7" op-status="0" interval="0" last-run="1437074871" last-rc-change="1437074871" exec-time="10" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" fake " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ </lrm_resource>
+ </lrm_resources>
+ </lrm>
+ <transient_attributes id="2">
+ <instance_attributes id="status-2">
+ <nvpair id="status-2-shutdown" name="shutdown" value="0"/>
+ <nvpair id="status-2-probe_complete" name="probe_complete" value="true"/>
+ </instance_attributes>
+ </transient_attributes>
+ </node_state>
+ <node_state id="3" uname="c7auto3" in_ccm="true" crmd="online" crm-debug-origin="do_update_resource" join="member" expected="member">
+ <lrm id="3">
+ <lrm_resources>
+ <lrm_resource id="shooter" type="fence_phd_kvm" class="stonith">
+ <lrm_rsc_op id="shooter_last_0" operation_key="shooter_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="8:0:7:b489d50c-a7db-4605-8519-a2d5e1452e29" transition-magic="0:7;8:0:7:b489d50c-a7db-4605-8519-a2d5e1452e29" on_node="c7auto3" call-id="5" rc-code="7" op-status="0" interval="0" last-run="1437073988" last-rc-change="1437073988" exec-time="963" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ </lrm_resource>
+ <lrm_resource id="FAKECLONE" type="Dummy" class="ocf" provider="heartbeat">
+ <lrm_rsc_op id="FAKECLONE_last_0" operation_key="FAKECLONE_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="16:2:0:b489d50c-a7db-4605-8519-a2d5e1452e29" transition-magic="0:0;16:2:0:b489d50c-a7db-4605-8519-a2d5e1452e29" on_node="c7auto3" call-id="11" rc-code="0" op-status="0" interval="0" last-run="1437074828" last-rc-change="1437074828" exec-time="8" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" fake " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ <lrm_rsc_op id="FAKECLONE_monitor_10000" operation_key="FAKECLONE_monitor_10000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="17:2:0:b489d50c-a7db-4605-8519-a2d5e1452e29" transition-magic="0:0;17:2:0:b489d50c-a7db-4605-8519-a2d5e1452e29" on_node="c7auto3" call-id="12" rc-code="0" op-status="0" interval="10000" last-rc-change="1437074828" exec-time="10" queue-time="0" op-digest="4811cef7f7f94e3a35a70be7916cb2fd"/>
+ </lrm_resource>
+ <lrm_resource id="FAKE" type="Dummy" class="ocf" provider="heartbeat">
+ <lrm_rsc_op id="FAKE_last_0" operation_key="FAKE_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="12:4:7:b489d50c-a7db-4605-8519-a2d5e1452e29" transition-magic="0:7;12:4:7:b489d50c-a7db-4605-8519-a2d5e1452e29" on_node="c7auto3" call-id="16" rc-code="7" op-status="0" interval="0" last-run="1437074870" last-rc-change="1437074870" exec-time="8" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" fake " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ </lrm_resource>
+ </lrm_resources>
+ </lrm>
+ <transient_attributes id="3">
+ <instance_attributes id="status-3">
+ <nvpair id="status-3-shutdown" name="shutdown" value="0"/>
+ <nvpair id="status-3-probe_complete" name="probe_complete" value="true"/>
+ </instance_attributes>
+ </transient_attributes>
+ </node_state>
+ <node_state id="4" uname="c7auto4" in_ccm="true" crmd="online" crm-debug-origin="do_update_resource" join="member" expected="member">
+ <lrm id="4">
+ <lrm_resources>
+ <lrm_resource id="shooter" type="fence_phd_kvm" class="stonith">
+ <lrm_rsc_op id="shooter_last_0" operation_key="shooter_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="10:0:7:b489d50c-a7db-4605-8519-a2d5e1452e29" transition-magic="0:7;10:0:7:b489d50c-a7db-4605-8519-a2d5e1452e29" on_node="c7auto4" call-id="5" rc-code="7" op-status="0" interval="0" last-run="1437073988" last-rc-change="1437073988" exec-time="963" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ </lrm_resource>
+ <lrm_resource id="FAKECLONE" type="Dummy" class="ocf" provider="heartbeat">
+ <lrm_rsc_op id="FAKECLONE_last_0" operation_key="FAKECLONE_stop_0" operation="stop" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="20:3:0:b489d50c-a7db-4605-8519-a2d5e1452e29" transition-magic="0:0;20:3:0:b489d50c-a7db-4605-8519-a2d5e1452e29" on_node="c7auto4" call-id="14" rc-code="0" op-status="0" interval="0" last-run="1437074856" last-rc-change="1437074856" exec-time="8" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" fake " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ <lrm_rsc_op id="FAKECLONE_monitor_10000" operation_key="FAKECLONE_monitor_10000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="19:2:0:b489d50c-a7db-4605-8519-a2d5e1452e29" transition-magic="0:0;19:2:0:b489d50c-a7db-4605-8519-a2d5e1452e29" on_node="c7auto4" call-id="12" rc-code="0" op-status="0" interval="10000" last-rc-change="1437074829" exec-time="6" queue-time="0" op-digest="4811cef7f7f94e3a35a70be7916cb2fd"/>
+ </lrm_resource>
+ <lrm_resource id="FAKE" type="Dummy" class="ocf" provider="heartbeat">
+ <lrm_rsc_op id="FAKE_last_0" operation_key="FAKE_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="27:4:0:b489d50c-a7db-4605-8519-a2d5e1452e29" transition-magic="0:0;27:4:0:b489d50c-a7db-4605-8519-a2d5e1452e29" on_node="c7auto4" call-id="19" rc-code="0" op-status="0" interval="0" last-run="1437074871" last-rc-change="1437074871" exec-time="8" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" fake " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ <lrm_rsc_op id="FAKE_monitor_10000" operation_key="FAKE_monitor_10000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="28:4:0:b489d50c-a7db-4605-8519-a2d5e1452e29" transition-magic="0:0;28:4:0:b489d50c-a7db-4605-8519-a2d5e1452e29" on_node="c7auto4" call-id="20" rc-code="0" op-status="0" interval="10000" last-rc-change="1437074871" exec-time="7" queue-time="0" op-digest="4811cef7f7f94e3a35a70be7916cb2fd"/>
+ </lrm_resource>
+ </lrm_resources>
+ </lrm>
+ <transient_attributes id="4">
+ <instance_attributes id="status-4">
+ <nvpair id="status-4-shutdown" name="shutdown" value="0"/>
+ <nvpair id="status-4-probe_complete" name="probe_complete" value="true"/>
+ </instance_attributes>
+ </transient_attributes>
+ </node_state>
+ </status>
+</cib>
diff --git a/pengine/test10/clone_min_stop_one.dot b/pengine/test10/clone_min_stop_one.dot
new file mode 100644
index 0000000..19f84cc
--- /dev/null
+++ b/pengine/test10/clone_min_stop_one.dot
@@ -0,0 +1,18 @@
+ digraph "g" {
+"FAKECLONE-clone_running_0" [ style=bold color="green" fontcolor="orange"]
+"FAKECLONE-clone_start_0" -> "FAKECLONE-clone_running_0" [ style = bold]
+"FAKECLONE-clone_start_0" -> "FAKECLONE_start_0 <none>" [ style = dashed]
+"FAKECLONE-clone_start_0" [ style=bold color="green" fontcolor="orange"]
+"FAKECLONE-clone_stop_0" -> "FAKECLONE-clone_stopped_0" [ style = bold]
+"FAKECLONE-clone_stop_0" -> "FAKECLONE_stop_0 c7auto2" [ style = bold]
+"FAKECLONE-clone_stop_0" [ style=bold color="green" fontcolor="orange"]
+"FAKECLONE-clone_stopped_0" -> "FAKECLONE-clone_start_0" [ style = bold]
+"FAKECLONE-clone_stopped_0" [ style=bold color="green" fontcolor="orange"]
+"FAKECLONE_start_0 <none>" -> "FAKECLONE-clone_running_0" [ style = dashed]
+"FAKECLONE_start_0 <none>" [ style=dashed color="red" fontcolor="black"]
+"FAKECLONE_stop_0 c7auto2" -> "FAKECLONE-clone_stopped_0" [ style = bold]
+"FAKECLONE_stop_0 c7auto2" -> "FAKECLONE_start_0 <none>" [ style = dashed]
+"FAKECLONE_stop_0 c7auto2" -> "all_stopped" [ style = bold]
+"FAKECLONE_stop_0 c7auto2" [ style=bold color="green" fontcolor="black"]
+"all_stopped" [ style=bold color="green" fontcolor="orange"]
+}
diff --git a/pengine/test10/clone_min_stop_one.exp b/pengine/test10/clone_min_stop_one.exp
new file mode 100644
index 0000000..4e6edb8
--- /dev/null
+++ b/pengine/test10/clone_min_stop_one.exp
@@ -0,0 +1,74 @@
+<transition_graph cluster-delay="60s" stonith-timeout="60s" failed-stop-offset="INFINITY" failed-start-offset="INFINITY" transition_id="0">
+ <synapse id="0">
+ <action_set>
+ <rsc_op id="17" operation="stop" operation_key="FAKECLONE_stop_0" internal_operation_key="FAKECLONE:1_stop_0" on_node="c7auto2" on_node_uuid="2">
+ <primitive id="FAKECLONE" long-id="FAKECLONE:1" class="ocf" provider="heartbeat" type="Dummy"/>
+ <attributes CRM_meta_clone="1" CRM_meta_clone_max="4" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_name="stop" CRM_meta_notify="false" CRM_meta_timeout="20000" />
+ </rsc_op>
+ </action_set>
+ <inputs>
+ <trigger>
+ <pseudo_event id="22" operation="stop" operation_key="FAKECLONE-clone_stop_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="1" priority="1000000">
+ <action_set>
+ <pseudo_event id="23" operation="stopped" operation_key="FAKECLONE-clone_stopped_0">
+ <attributes CRM_meta_clone_max="4" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="false" CRM_meta_timeout="20000" />
+ </pseudo_event>
+ </action_set>
+ <inputs>
+ <trigger>
+ <rsc_op id="17" operation="stop" operation_key="FAKECLONE_stop_0" internal_operation_key="FAKECLONE:1_stop_0" on_node="c7auto2" on_node_uuid="2"/>
+ </trigger>
+ <trigger>
+ <pseudo_event id="22" operation="stop" operation_key="FAKECLONE-clone_stop_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="2">
+ <action_set>
+ <pseudo_event id="22" operation="stop" operation_key="FAKECLONE-clone_stop_0">
+ <attributes CRM_meta_clone_max="4" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="false" CRM_meta_timeout="20000" />
+ </pseudo_event>
+ </action_set>
+ <inputs/>
+ </synapse>
+ <synapse id="3" priority="1000000">
+ <action_set>
+ <pseudo_event id="21" operation="running" operation_key="FAKECLONE-clone_running_0">
+ <attributes CRM_meta_clone_max="4" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="false" CRM_meta_timeout="20000" />
+ </pseudo_event>
+ </action_set>
+ <inputs>
+ <trigger>
+ <pseudo_event id="20" operation="start" operation_key="FAKECLONE-clone_start_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="4">
+ <action_set>
+ <pseudo_event id="20" operation="start" operation_key="FAKECLONE-clone_start_0">
+ <attributes CRM_meta_clone_max="4" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="false" CRM_meta_timeout="20000" />
+ </pseudo_event>
+ </action_set>
+ <inputs>
+ <trigger>
+ <pseudo_event id="23" operation="stopped" operation_key="FAKECLONE-clone_stopped_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="5">
+ <action_set>
+ <pseudo_event id="7" operation="all_stopped" operation_key="all_stopped">
+ <attributes />
+ </pseudo_event>
+ </action_set>
+ <inputs>
+ <trigger>
+ <rsc_op id="17" operation="stop" operation_key="FAKECLONE_stop_0" internal_operation_key="FAKECLONE:1_stop_0" on_node="c7auto2" on_node_uuid="2"/>
+ </trigger>
+ </inputs>
+ </synapse>
+</transition_graph>
diff --git a/pengine/test10/clone_min_stop_one.scores b/pengine/test10/clone_min_stop_one.scores
new file mode 100644
index 0000000..1f28932
--- /dev/null
+++ b/pengine/test10/clone_min_stop_one.scores
@@ -0,0 +1,45 @@
+Allocation scores:
+clone_color: FAKECLONE-clone allocation score on c7auto1: 0
+clone_color: FAKECLONE-clone allocation score on c7auto2: 0
+clone_color: FAKECLONE-clone allocation score on c7auto3: 0
+clone_color: FAKECLONE-clone allocation score on c7auto4: -INFINITY
+clone_color: FAKECLONE:0 allocation score on c7auto1: 1
+clone_color: FAKECLONE:0 allocation score on c7auto2: 0
+clone_color: FAKECLONE:0 allocation score on c7auto3: 0
+clone_color: FAKECLONE:0 allocation score on c7auto4: -INFINITY
+clone_color: FAKECLONE:1 allocation score on c7auto1: 0
+clone_color: FAKECLONE:1 allocation score on c7auto2: 1
+clone_color: FAKECLONE:1 allocation score on c7auto3: 0
+clone_color: FAKECLONE:1 allocation score on c7auto4: -INFINITY
+clone_color: FAKECLONE:2 allocation score on c7auto1: 0
+clone_color: FAKECLONE:2 allocation score on c7auto2: 0
+clone_color: FAKECLONE:2 allocation score on c7auto3: 1
+clone_color: FAKECLONE:2 allocation score on c7auto4: -INFINITY
+clone_color: FAKECLONE:3 allocation score on c7auto1: 0
+clone_color: FAKECLONE:3 allocation score on c7auto2: 0
+clone_color: FAKECLONE:3 allocation score on c7auto3: 0
+clone_color: FAKECLONE:3 allocation score on c7auto4: -INFINITY
+native_color: FAKE allocation score on c7auto1: -INFINITY
+native_color: FAKE allocation score on c7auto2: -INFINITY
+native_color: FAKE allocation score on c7auto3: -INFINITY
+native_color: FAKE allocation score on c7auto4: 0
+native_color: FAKECLONE:0 allocation score on c7auto1: 1
+native_color: FAKECLONE:0 allocation score on c7auto2: -INFINITY
+native_color: FAKECLONE:0 allocation score on c7auto3: -INFINITY
+native_color: FAKECLONE:0 allocation score on c7auto4: -INFINITY
+native_color: FAKECLONE:1 allocation score on c7auto1: -INFINITY
+native_color: FAKECLONE:1 allocation score on c7auto2: -INFINITY
+native_color: FAKECLONE:1 allocation score on c7auto3: -INFINITY
+native_color: FAKECLONE:1 allocation score on c7auto4: -INFINITY
+native_color: FAKECLONE:2 allocation score on c7auto1: 0
+native_color: FAKECLONE:2 allocation score on c7auto2: -INFINITY
+native_color: FAKECLONE:2 allocation score on c7auto3: 1
+native_color: FAKECLONE:2 allocation score on c7auto4: -INFINITY
+native_color: FAKECLONE:3 allocation score on c7auto1: -INFINITY
+native_color: FAKECLONE:3 allocation score on c7auto2: -INFINITY
+native_color: FAKECLONE:3 allocation score on c7auto3: -INFINITY
+native_color: FAKECLONE:3 allocation score on c7auto4: -INFINITY
+native_color: shooter allocation score on c7auto1: 0
+native_color: shooter allocation score on c7auto2: 0
+native_color: shooter allocation score on c7auto3: 0
+native_color: shooter allocation score on c7auto4: 0
diff --git a/pengine/test10/clone_min_stop_one.summary b/pengine/test10/clone_min_stop_one.summary
new file mode 100644
index 0000000..9206a0d
--- /dev/null
+++ b/pengine/test10/clone_min_stop_one.summary
@@ -0,0 +1,32 @@
+
+Current cluster status:
+Node c7auto2 (2): standby
+Online: [ c7auto1 c7auto3 c7auto4 ]
+
+ shooter (stonith:fence_phd_kvm): Started c7auto1
+ Clone Set: FAKECLONE-clone [FAKECLONE]
+ Started: [ c7auto1 c7auto2 c7auto3 ]
+ Stopped: [ c7auto4 ]
+ FAKE (ocf::heartbeat:Dummy): Started c7auto4
+
+Transition Summary:
+ * Stop FAKECLONE:1 (c7auto2)
+
+Executing cluster transition:
+ * Pseudo action: FAKECLONE-clone_stop_0
+ * Resource action: FAKECLONE stop on c7auto2
+ * Pseudo action: FAKECLONE-clone_stopped_0
+ * Pseudo action: FAKECLONE-clone_start_0
+ * Pseudo action: all_stopped
+ * Pseudo action: FAKECLONE-clone_running_0
+
+Revised cluster status:
+Node c7auto2 (2): standby
+Online: [ c7auto1 c7auto3 c7auto4 ]
+
+ shooter (stonith:fence_phd_kvm): Started c7auto1
+ Clone Set: FAKECLONE-clone [FAKECLONE]
+ Started: [ c7auto1 c7auto3 ]
+ Stopped: [ c7auto2 c7auto4 ]
+ FAKE (ocf::heartbeat:Dummy): Started c7auto4
+
diff --git a/pengine/test10/clone_min_stop_one.xml b/pengine/test10/clone_min_stop_one.xml
new file mode 100644
index 0000000..eb05803
--- /dev/null
+++ b/pengine/test10/clone_min_stop_one.xml
@@ -0,0 +1,152 @@
+<cib crm_feature_set="3.0.10" validate-with="pacemaker-2.3" epoch="16" num_updates="0" admin_epoch="0" cib-last-written="Thu Jul 16 15:28:22 2015" update-origin="c7auto1" update-client="cibadmin" update-user="root" have-quorum="1" dc-uuid="2">
+ <configuration>
+ <crm_config>
+ <cluster_property_set id="cib-bootstrap-options">
+ <nvpair id="cib-bootstrap-options-have-watchdog" name="have-watchdog" value="false"/>
+ <nvpair id="cib-bootstrap-options-dc-version" name="dc-version" value="1.1.12-ef86b49"/>
+ <nvpair id="cib-bootstrap-options-cluster-infrastructure" name="cluster-infrastructure" value="corosync"/>
+ <nvpair id="cib-bootstrap-options-cluster-name" name="cluster-name" value="phd"/>
+ </cluster_property_set>
+ </crm_config>
+ <nodes>
+ <node id="1" uname="c7auto1">
+ <instance_attributes id="nodes-1"/>
+ </node>
+ <node id="2" uname="c7auto2">
+ <instance_attributes id="nodes-2">
+ <nvpair id="nodes-2-standby" name="standby" value="on"/>
+ </instance_attributes>
+ </node>
+ <node id="3" uname="c7auto3"/>
+ <node id="4" uname="c7auto4"/>
+ </nodes>
+ <resources>
+ <primitive class="stonith" id="shooter" type="fence_phd_kvm">
+ <instance_attributes id="shooter-instance_attributes"/>
+ <operations>
+ <op id="shooter-monitor-interval-60s" interval="60s" name="monitor"/>
+ </operations>
+ </primitive>
+ <clone id="FAKECLONE-clone">
+ <primitive class="ocf" id="FAKECLONE" provider="heartbeat" type="Dummy">
+ <instance_attributes id="FAKECLONE-instance_attributes"/>
+ <operations>
+ <op id="FAKECLONE-start-timeout-20" interval="0s" name="start" timeout="20"/>
+ <op id="FAKECLONE-stop-timeout-20" interval="0s" name="stop" timeout="20"/>
+ <op id="FAKECLONE-monitor-interval-10" interval="10" name="monitor" timeout="20"/>
+ </operations>
+ </primitive>
+ <meta_attributes id="FAKECLONE-clone-meta">
+ <nvpair id="FAKECLONE-clone-min" name="clone-min" value="2"/>
+ </meta_attributes>
+ </clone>
+ <primitive class="ocf" id="FAKE" provider="heartbeat" type="Dummy">
+ <instance_attributes id="FAKE-instance_attributes"/>
+ <operations>
+ <op id="FAKE-start-timeout-20" interval="0s" name="start" timeout="20"/>
+ <op id="FAKE-stop-timeout-20" interval="0s" name="stop" timeout="20"/>
+ <op id="FAKE-monitor-interval-10" interval="10" name="monitor" timeout="20"/>
+ </operations>
+ </primitive>
+ </resources>
+ <constraints>
+ <rsc_location id="location-FAKECLONE-clone-c7auto4--INFINITY" node="c7auto4" rsc="FAKECLONE-clone" score="-INFINITY"/>
+ <rsc_location id="location-FAKE-c7auto1--INFINITY" node="c7auto1" rsc="FAKE" score="-INFINITY"/>
+ <rsc_location id="location-FAKE-c7auto2--INFINITY" node="c7auto2" rsc="FAKE" score="-INFINITY"/>
+ <rsc_location id="location-FAKE-c7auto3--INFINITY" node="c7auto3" rsc="FAKE" score="-INFINITY"/>
+ <rsc_order first="FAKECLONE-clone" first-action="start" id="order-FAKECLONE-clone-FAKE-mandatory" then="FAKE" then-action="start"/>
+ </constraints>
+ </configuration>
+ <status>
+ <node_state id="1" uname="c7auto1" in_ccm="true" crmd="online" crm-debug-origin="do_update_resource" join="member" expected="member">
+ <lrm id="1">
+ <lrm_resources>
+ <lrm_resource id="shooter" type="fence_phd_kvm" class="stonith">
+ <lrm_rsc_op id="shooter_last_0" operation_key="shooter_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="11:0:0:b489d50c-a7db-4605-8519-a2d5e1452e29" transition-magic="0:0;11:0:0:b489d50c-a7db-4605-8519-a2d5e1452e29" on_node="c7auto1" call-id="6" rc-code="0" op-status="0" interval="0" last-run="1437073991" last-rc-change="1437073991" exec-time="9" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ <lrm_rsc_op id="shooter_monitor_60000" operation_key="shooter_monitor_60000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="9:1:0:b489d50c-a7db-4605-8519-a2d5e1452e29" transition-magic="0:0;9:1:0:b489d50c-a7db-4605-8519-a2d5e1452e29" on_node="c7auto1" call-id="7" rc-code="0" op-status="0" interval="60000" last-rc-change="1437073992" exec-time="7" queue-time="0" op-digest="4811cef7f7f94e3a35a70be7916cb2fd"/>
+ </lrm_resource>
+ <lrm_resource id="FAKECLONE" type="Dummy" class="ocf" provider="heartbeat">
+ <lrm_rsc_op id="FAKECLONE_last_0" operation_key="FAKECLONE_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="20:2:0:b489d50c-a7db-4605-8519-a2d5e1452e29" transition-magic="0:0;20:2:0:b489d50c-a7db-4605-8519-a2d5e1452e29" on_node="c7auto1" call-id="13" rc-code="0" op-status="0" interval="0" last-run="1437074829" last-rc-change="1437074829" exec-time="7" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" fake " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ <lrm_rsc_op id="FAKECLONE_monitor_10000" operation_key="FAKECLONE_monitor_10000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="21:2:0:b489d50c-a7db-4605-8519-a2d5e1452e29" transition-magic="0:0;21:2:0:b489d50c-a7db-4605-8519-a2d5e1452e29" on_node="c7auto1" call-id="14" rc-code="0" op-status="0" interval="10000" last-rc-change="1437074829" exec-time="7" queue-time="0" op-digest="4811cef7f7f94e3a35a70be7916cb2fd"/>
+ </lrm_resource>
+ <lrm_resource id="FAKE" type="Dummy" class="ocf" provider="heartbeat">
+ <lrm_rsc_op id="FAKE_last_0" operation_key="FAKE_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="8:4:7:b489d50c-a7db-4605-8519-a2d5e1452e29" transition-magic="0:7;8:4:7:b489d50c-a7db-4605-8519-a2d5e1452e29" on_node="c7auto1" call-id="18" rc-code="7" op-status="0" interval="0" last-run="1437074871" last-rc-change="1437074871" exec-time="7" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" fake " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ </lrm_resource>
+ </lrm_resources>
+ </lrm>
+ <transient_attributes id="1">
+ <instance_attributes id="status-1">
+ <nvpair id="status-1-shutdown" name="shutdown" value="0"/>
+ <nvpair id="status-1-probe_complete" name="probe_complete" value="true"/>
+ </instance_attributes>
+ </transient_attributes>
+ </node_state>
+ <node_state id="2" uname="c7auto2" in_ccm="true" crmd="online" crm-debug-origin="do_update_resource" join="member" expected="member">
+ <lrm id="2">
+ <lrm_resources>
+ <lrm_resource id="shooter" type="fence_phd_kvm" class="stonith">
+ <lrm_rsc_op id="shooter_last_0" operation_key="shooter_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="6:0:7:b489d50c-a7db-4605-8519-a2d5e1452e29" transition-magic="0:7;6:0:7:b489d50c-a7db-4605-8519-a2d5e1452e29" on_node="c7auto2" call-id="5" rc-code="7" op-status="0" interval="0" last-run="1437073988" last-rc-change="1437073988" exec-time="964" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ </lrm_resource>
+ <lrm_resource id="FAKECLONE" type="Dummy" class="ocf" provider="heartbeat">
+ <lrm_rsc_op id="FAKECLONE_last_0" operation_key="FAKECLONE_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="14:2:0:b489d50c-a7db-4605-8519-a2d5e1452e29" transition-magic="0:0;14:2:0:b489d50c-a7db-4605-8519-a2d5e1452e29" on_node="c7auto2" call-id="11" rc-code="0" op-status="0" interval="0" last-run="1437074829" last-rc-change="1437074829" exec-time="8" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" fake " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ <lrm_rsc_op id="FAKECLONE_monitor_10000" operation_key="FAKECLONE_monitor_10000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="15:2:0:b489d50c-a7db-4605-8519-a2d5e1452e29" transition-magic="0:0;15:2:0:b489d50c-a7db-4605-8519-a2d5e1452e29" on_node="c7auto2" call-id="12" rc-code="0" op-status="0" interval="10000" last-rc-change="1437074829" exec-time="10" queue-time="0" op-digest="4811cef7f7f94e3a35a70be7916cb2fd"/>
+ </lrm_resource>
+ <lrm_resource id="FAKE" type="Dummy" class="ocf" provider="heartbeat">
+ <lrm_rsc_op id="FAKE_last_0" operation_key="FAKE_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="10:4:7:b489d50c-a7db-4605-8519-a2d5e1452e29" transition-magic="0:7;10:4:7:b489d50c-a7db-4605-8519-a2d5e1452e29" on_node="c7auto2" call-id="16" rc-code="7" op-status="0" interval="0" last-run="1437074871" last-rc-change="1437074871" exec-time="10" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" fake " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ </lrm_resource>
+ </lrm_resources>
+ </lrm>
+ <transient_attributes id="2">
+ <instance_attributes id="status-2">
+ <nvpair id="status-2-shutdown" name="shutdown" value="0"/>
+ <nvpair id="status-2-probe_complete" name="probe_complete" value="true"/>
+ </instance_attributes>
+ </transient_attributes>
+ </node_state>
+ <node_state id="3" uname="c7auto3" in_ccm="true" crmd="online" crm-debug-origin="do_update_resource" join="member" expected="member">
+ <lrm id="3">
+ <lrm_resources>
+ <lrm_resource id="shooter" type="fence_phd_kvm" class="stonith">
+ <lrm_rsc_op id="shooter_last_0" operation_key="shooter_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="8:0:7:b489d50c-a7db-4605-8519-a2d5e1452e29" transition-magic="0:7;8:0:7:b489d50c-a7db-4605-8519-a2d5e1452e29" on_node="c7auto3" call-id="5" rc-code="7" op-status="0" interval="0" last-run="1437073988" last-rc-change="1437073988" exec-time="963" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ </lrm_resource>
+ <lrm_resource id="FAKECLONE" type="Dummy" class="ocf" provider="heartbeat">
+ <lrm_rsc_op id="FAKECLONE_last_0" operation_key="FAKECLONE_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="16:2:0:b489d50c-a7db-4605-8519-a2d5e1452e29" transition-magic="0:0;16:2:0:b489d50c-a7db-4605-8519-a2d5e1452e29" on_node="c7auto3" call-id="11" rc-code="0" op-status="0" interval="0" last-run="1437074828" last-rc-change="1437074828" exec-time="8" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" fake " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ <lrm_rsc_op id="FAKECLONE_monitor_10000" operation_key="FAKECLONE_monitor_10000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="17:2:0:b489d50c-a7db-4605-8519-a2d5e1452e29" transition-magic="0:0;17:2:0:b489d50c-a7db-4605-8519-a2d5e1452e29" on_node="c7auto3" call-id="12" rc-code="0" op-status="0" interval="10000" last-rc-change="1437074828" exec-time="10" queue-time="0" op-digest="4811cef7f7f94e3a35a70be7916cb2fd"/>
+ </lrm_resource>
+ <lrm_resource id="FAKE" type="Dummy" class="ocf" provider="heartbeat">
+ <lrm_rsc_op id="FAKE_last_0" operation_key="FAKE_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="12:4:7:b489d50c-a7db-4605-8519-a2d5e1452e29" transition-magic="0:7;12:4:7:b489d50c-a7db-4605-8519-a2d5e1452e29" on_node="c7auto3" call-id="16" rc-code="7" op-status="0" interval="0" last-run="1437074870" last-rc-change="1437074870" exec-time="8" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" fake " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ </lrm_resource>
+ </lrm_resources>
+ </lrm>
+ <transient_attributes id="3">
+ <instance_attributes id="status-3">
+ <nvpair id="status-3-shutdown" name="shutdown" value="0"/>
+ <nvpair id="status-3-probe_complete" name="probe_complete" value="true"/>
+ </instance_attributes>
+ </transient_attributes>
+ </node_state>
+ <node_state id="4" uname="c7auto4" in_ccm="true" crmd="online" crm-debug-origin="do_update_resource" join="member" expected="member">
+ <lrm id="4">
+ <lrm_resources>
+ <lrm_resource id="shooter" type="fence_phd_kvm" class="stonith">
+ <lrm_rsc_op id="shooter_last_0" operation_key="shooter_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="10:0:7:b489d50c-a7db-4605-8519-a2d5e1452e29" transition-magic="0:7;10:0:7:b489d50c-a7db-4605-8519-a2d5e1452e29" on_node="c7auto4" call-id="5" rc-code="7" op-status="0" interval="0" last-run="1437073988" last-rc-change="1437073988" exec-time="963" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ </lrm_resource>
+ <lrm_resource id="FAKECLONE" type="Dummy" class="ocf" provider="heartbeat">
+ <lrm_rsc_op id="FAKECLONE_last_0" operation_key="FAKECLONE_stop_0" operation="stop" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="20:3:0:b489d50c-a7db-4605-8519-a2d5e1452e29" transition-magic="0:0;20:3:0:b489d50c-a7db-4605-8519-a2d5e1452e29" on_node="c7auto4" call-id="14" rc-code="0" op-status="0" interval="0" last-run="1437074856" last-rc-change="1437074856" exec-time="8" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" fake " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ <lrm_rsc_op id="FAKECLONE_monitor_10000" operation_key="FAKECLONE_monitor_10000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="19:2:0:b489d50c-a7db-4605-8519-a2d5e1452e29" transition-magic="0:0;19:2:0:b489d50c-a7db-4605-8519-a2d5e1452e29" on_node="c7auto4" call-id="12" rc-code="0" op-status="0" interval="10000" last-rc-change="1437074829" exec-time="6" queue-time="0" op-digest="4811cef7f7f94e3a35a70be7916cb2fd"/>
+ </lrm_resource>
+ <lrm_resource id="FAKE" type="Dummy" class="ocf" provider="heartbeat">
+ <lrm_rsc_op id="FAKE_last_0" operation_key="FAKE_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="27:4:0:b489d50c-a7db-4605-8519-a2d5e1452e29" transition-magic="0:0;27:4:0:b489d50c-a7db-4605-8519-a2d5e1452e29" on_node="c7auto4" call-id="19" rc-code="0" op-status="0" interval="0" last-run="1437074871" last-rc-change="1437074871" exec-time="8" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" fake " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ <lrm_rsc_op id="FAKE_monitor_10000" operation_key="FAKE_monitor_10000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="28:4:0:b489d50c-a7db-4605-8519-a2d5e1452e29" transition-magic="0:0;28:4:0:b489d50c-a7db-4605-8519-a2d5e1452e29" on_node="c7auto4" call-id="20" rc-code="0" op-status="0" interval="10000" last-rc-change="1437074871" exec-time="7" queue-time="0" op-digest="4811cef7f7f94e3a35a70be7916cb2fd"/>
+ </lrm_resource>
+ </lrm_resources>
+ </lrm>
+ <transient_attributes id="4">
+ <instance_attributes id="status-4">
+ <nvpair id="status-4-shutdown" name="shutdown" value="0"/>
+ <nvpair id="status-4-probe_complete" name="probe_complete" value="true"/>
+ </instance_attributes>
+ </transient_attributes>
+ </node_state>
+ </status>
+</cib>
diff --git a/pengine/test10/clone_min_stop_two.dot b/pengine/test10/clone_min_stop_two.dot
new file mode 100644
index 0000000..11640f4
--- /dev/null
+++ b/pengine/test10/clone_min_stop_two.dot
@@ -0,0 +1,36 @@
+ digraph "g" {
+"FAKECLONE-clone_running_0" [ style=bold color="green" fontcolor="orange"]
+"FAKECLONE-clone_start_0" -> "FAKECLONE-clone_running_0" [ style = bold]
+"FAKECLONE-clone_start_0" -> "FAKECLONE_start_0 <none>" [ style = dashed]
+"FAKECLONE-clone_start_0" [ style=bold color="green" fontcolor="orange"]
+"FAKECLONE-clone_stop_0" -> "FAKECLONE-clone_stopped_0" [ style = bold]
+"FAKECLONE-clone_stop_0" -> "FAKECLONE_stop_0 c7auto1" [ style = bold]
+"FAKECLONE-clone_stop_0" -> "FAKECLONE_stop_0 c7auto2" [ style = bold]
+"FAKECLONE-clone_stop_0" [ style=bold color="green" fontcolor="orange"]
+"FAKECLONE-clone_stopped_0" -> "FAKECLONE-clone_start_0" [ style = bold]
+"FAKECLONE-clone_stopped_0" [ style=bold color="green" fontcolor="orange"]
+"FAKECLONE_start_0 <none>" -> "FAKECLONE-clone_running_0" [ style = dashed]
+"FAKECLONE_start_0 <none>" [ style=dashed color="red" fontcolor="black"]
+"FAKECLONE_stop_0 c7auto1" -> "FAKECLONE-clone_stopped_0" [ style = bold]
+"FAKECLONE_stop_0 c7auto1" -> "FAKECLONE_start_0 <none>" [ style = dashed]
+"FAKECLONE_stop_0 c7auto1" -> "all_stopped" [ style = bold]
+"FAKECLONE_stop_0 c7auto1" [ style=bold color="green" fontcolor="black"]
+"FAKECLONE_stop_0 c7auto2" -> "FAKECLONE-clone_stopped_0" [ style = bold]
+"FAKECLONE_stop_0 c7auto2" -> "FAKECLONE_start_0 <none>" [ style = dashed]
+"FAKECLONE_stop_0 c7auto2" -> "all_stopped" [ style = bold]
+"FAKECLONE_stop_0 c7auto2" [ style=bold color="green" fontcolor="black"]
+"FAKE_monitor_10000 c7auto4" [ style=dashed color="red" fontcolor="black"]
+"FAKE_start_0 c7auto4" -> "FAKE_monitor_10000 c7auto4" [ style = dashed]
+"FAKE_start_0 c7auto4" [ style=dashed color="red" fontcolor="black"]
+"FAKE_stop_0 c7auto4" -> "FAKECLONE-clone_stop_0" [ style = bold]
+"FAKE_stop_0 c7auto4" -> "FAKE_start_0 c7auto4" [ style = dashed]
+"FAKE_stop_0 c7auto4" -> "all_stopped" [ style = bold]
+"FAKE_stop_0 c7auto4" [ style=bold color="green" fontcolor="black"]
+"all_stopped" [ style=bold color="green" fontcolor="orange"]
+"shooter_monitor_60000 c7auto3" [ style=bold color="green" fontcolor="black"]
+"shooter_start_0 c7auto3" -> "shooter_monitor_60000 c7auto3" [ style = bold]
+"shooter_start_0 c7auto3" [ style=bold color="green" fontcolor="black"]
+"shooter_stop_0 c7auto1" -> "all_stopped" [ style = bold]
+"shooter_stop_0 c7auto1" -> "shooter_start_0 c7auto3" [ style = bold]
+"shooter_stop_0 c7auto1" [ style=bold color="green" fontcolor="black"]
+}
diff --git a/pengine/test10/clone_min_stop_two.exp b/pengine/test10/clone_min_stop_two.exp
new file mode 100644
index 0000000..5697611
--- /dev/null
+++ b/pengine/test10/clone_min_stop_two.exp
@@ -0,0 +1,147 @@
+<transition_graph cluster-delay="60s" stonith-timeout="60s" failed-stop-offset="INFINITY" failed-start-offset="INFINITY" transition_id="0">
+ <synapse id="0">
+ <action_set>
+ <rsc_op id="15" operation="monitor" operation_key="shooter_monitor_60000" on_node="c7auto3" on_node_uuid="3">
+ <primitive id="shooter" class="stonith" type="fence_phd_kvm"/>
+ <attributes CRM_meta_interval="60000" CRM_meta_name="monitor" CRM_meta_timeout="20000" />
+ </rsc_op>
+ </action_set>
+ <inputs>
+ <trigger>
+ <rsc_op id="14" operation="start" operation_key="shooter_start_0" on_node="c7auto3" on_node_uuid="3"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="1">
+ <action_set>
+ <rsc_op id="14" operation="start" operation_key="shooter_start_0" on_node="c7auto3" on_node_uuid="3">
+ <primitive id="shooter" class="stonith" type="fence_phd_kvm"/>
+ <attributes CRM_meta_timeout="20000" />
+ </rsc_op>
+ </action_set>
+ <inputs>
+ <trigger>
+ <rsc_op id="13" operation="stop" operation_key="shooter_stop_0" on_node="c7auto1" on_node_uuid="1"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="2">
+ <action_set>
+ <rsc_op id="13" operation="stop" operation_key="shooter_stop_0" on_node="c7auto1" on_node_uuid="1">
+ <primitive id="shooter" class="stonith" type="fence_phd_kvm"/>
+ <attributes CRM_meta_timeout="20000" />
+ </rsc_op>
+ </action_set>
+ <inputs/>
+ </synapse>
+ <synapse id="3">
+ <action_set>
+ <rsc_op id="16" operation="stop" operation_key="FAKECLONE_stop_0" internal_operation_key="FAKECLONE:0_stop_0" on_node="c7auto1" on_node_uuid="1">
+ <primitive id="FAKECLONE" long-id="FAKECLONE:0" class="ocf" provider="heartbeat" type="Dummy"/>
+ <attributes CRM_meta_clone="0" CRM_meta_clone_max="4" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_name="stop" CRM_meta_notify="false" CRM_meta_timeout="20000" />
+ </rsc_op>
+ </action_set>
+ <inputs>
+ <trigger>
+ <pseudo_event id="22" operation="stop" operation_key="FAKECLONE-clone_stop_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="4">
+ <action_set>
+ <rsc_op id="17" operation="stop" operation_key="FAKECLONE_stop_0" internal_operation_key="FAKECLONE:1_stop_0" on_node="c7auto2" on_node_uuid="2">
+ <primitive id="FAKECLONE" long-id="FAKECLONE:1" class="ocf" provider="heartbeat" type="Dummy"/>
+ <attributes CRM_meta_clone="1" CRM_meta_clone_max="4" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_name="stop" CRM_meta_notify="false" CRM_meta_timeout="20000" />
+ </rsc_op>
+ </action_set>
+ <inputs>
+ <trigger>
+ <pseudo_event id="22" operation="stop" operation_key="FAKECLONE-clone_stop_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="5" priority="1000000">
+ <action_set>
+ <pseudo_event id="23" operation="stopped" operation_key="FAKECLONE-clone_stopped_0">
+ <attributes CRM_meta_clone_max="4" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="false" CRM_meta_timeout="20000" />
+ </pseudo_event>
+ </action_set>
+ <inputs>
+ <trigger>
+ <rsc_op id="16" operation="stop" operation_key="FAKECLONE_stop_0" internal_operation_key="FAKECLONE:0_stop_0" on_node="c7auto1" on_node_uuid="1"/>
+ </trigger>
+ <trigger>
+ <rsc_op id="17" operation="stop" operation_key="FAKECLONE_stop_0" internal_operation_key="FAKECLONE:1_stop_0" on_node="c7auto2" on_node_uuid="2"/>
+ </trigger>
+ <trigger>
+ <pseudo_event id="22" operation="stop" operation_key="FAKECLONE-clone_stop_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="6">
+ <action_set>
+ <pseudo_event id="22" operation="stop" operation_key="FAKECLONE-clone_stop_0">
+ <attributes CRM_meta_clone_max="4" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="false" CRM_meta_timeout="20000" />
+ </pseudo_event>
+ </action_set>
+ <inputs>
+ <trigger>
+ <rsc_op id="24" operation="stop" operation_key="FAKE_stop_0" on_node="c7auto4" on_node_uuid="4"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="7" priority="1000000">
+ <action_set>
+ <pseudo_event id="21" operation="running" operation_key="FAKECLONE-clone_running_0">
+ <attributes CRM_meta_clone_max="4" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="false" CRM_meta_timeout="20000" />
+ </pseudo_event>
+ </action_set>
+ <inputs>
+ <trigger>
+ <pseudo_event id="20" operation="start" operation_key="FAKECLONE-clone_start_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="8">
+ <action_set>
+ <pseudo_event id="20" operation="start" operation_key="FAKECLONE-clone_start_0">
+ <attributes CRM_meta_clone_max="4" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="false" CRM_meta_timeout="20000" />
+ </pseudo_event>
+ </action_set>
+ <inputs>
+ <trigger>
+ <pseudo_event id="23" operation="stopped" operation_key="FAKECLONE-clone_stopped_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="9">
+ <action_set>
+ <rsc_op id="24" operation="stop" operation_key="FAKE_stop_0" on_node="c7auto4" on_node_uuid="4">
+ <primitive id="FAKE" class="ocf" provider="heartbeat" type="Dummy"/>
+ <attributes CRM_meta_name="stop" CRM_meta_timeout="20000" />
+ </rsc_op>
+ </action_set>
+ <inputs/>
+ </synapse>
+ <synapse id="10">
+ <action_set>
+ <pseudo_event id="7" operation="all_stopped" operation_key="all_stopped">
+ <attributes />
+ </pseudo_event>
+ </action_set>
+ <inputs>
+ <trigger>
+ <rsc_op id="13" operation="stop" operation_key="shooter_stop_0" on_node="c7auto1" on_node_uuid="1"/>
+ </trigger>
+ <trigger>
+ <rsc_op id="16" operation="stop" operation_key="FAKECLONE_stop_0" internal_operation_key="FAKECLONE:0_stop_0" on_node="c7auto1" on_node_uuid="1"/>
+ </trigger>
+ <trigger>
+ <rsc_op id="17" operation="stop" operation_key="FAKECLONE_stop_0" internal_operation_key="FAKECLONE:1_stop_0" on_node="c7auto2" on_node_uuid="2"/>
+ </trigger>
+ <trigger>
+ <rsc_op id="24" operation="stop" operation_key="FAKE_stop_0" on_node="c7auto4" on_node_uuid="4"/>
+ </trigger>
+ </inputs>
+ </synapse>
+</transition_graph>
diff --git a/pengine/test10/clone_min_stop_two.scores b/pengine/test10/clone_min_stop_two.scores
new file mode 100644
index 0000000..ce43eb9
--- /dev/null
+++ b/pengine/test10/clone_min_stop_two.scores
@@ -0,0 +1,45 @@
+Allocation scores:
+clone_color: FAKECLONE-clone allocation score on c7auto1: 0
+clone_color: FAKECLONE-clone allocation score on c7auto2: 0
+clone_color: FAKECLONE-clone allocation score on c7auto3: 0
+clone_color: FAKECLONE-clone allocation score on c7auto4: -INFINITY
+clone_color: FAKECLONE:0 allocation score on c7auto1: 1
+clone_color: FAKECLONE:0 allocation score on c7auto2: 0
+clone_color: FAKECLONE:0 allocation score on c7auto3: 0
+clone_color: FAKECLONE:0 allocation score on c7auto4: -INFINITY
+clone_color: FAKECLONE:1 allocation score on c7auto1: 0
+clone_color: FAKECLONE:1 allocation score on c7auto2: 1
+clone_color: FAKECLONE:1 allocation score on c7auto3: 0
+clone_color: FAKECLONE:1 allocation score on c7auto4: -INFINITY
+clone_color: FAKECLONE:2 allocation score on c7auto1: 0
+clone_color: FAKECLONE:2 allocation score on c7auto2: 0
+clone_color: FAKECLONE:2 allocation score on c7auto3: 1
+clone_color: FAKECLONE:2 allocation score on c7auto4: -INFINITY
+clone_color: FAKECLONE:3 allocation score on c7auto1: 0
+clone_color: FAKECLONE:3 allocation score on c7auto2: 0
+clone_color: FAKECLONE:3 allocation score on c7auto3: 0
+clone_color: FAKECLONE:3 allocation score on c7auto4: -INFINITY
+native_color: FAKE allocation score on c7auto1: -INFINITY
+native_color: FAKE allocation score on c7auto2: -INFINITY
+native_color: FAKE allocation score on c7auto3: -INFINITY
+native_color: FAKE allocation score on c7auto4: 0
+native_color: FAKECLONE:0 allocation score on c7auto1: -INFINITY
+native_color: FAKECLONE:0 allocation score on c7auto2: -INFINITY
+native_color: FAKECLONE:0 allocation score on c7auto3: -INFINITY
+native_color: FAKECLONE:0 allocation score on c7auto4: -INFINITY
+native_color: FAKECLONE:1 allocation score on c7auto1: -INFINITY
+native_color: FAKECLONE:1 allocation score on c7auto2: -INFINITY
+native_color: FAKECLONE:1 allocation score on c7auto3: -INFINITY
+native_color: FAKECLONE:1 allocation score on c7auto4: -INFINITY
+native_color: FAKECLONE:2 allocation score on c7auto1: -INFINITY
+native_color: FAKECLONE:2 allocation score on c7auto2: -INFINITY
+native_color: FAKECLONE:2 allocation score on c7auto3: 1
+native_color: FAKECLONE:2 allocation score on c7auto4: -INFINITY
+native_color: FAKECLONE:3 allocation score on c7auto1: -INFINITY
+native_color: FAKECLONE:3 allocation score on c7auto2: -INFINITY
+native_color: FAKECLONE:3 allocation score on c7auto3: -INFINITY
+native_color: FAKECLONE:3 allocation score on c7auto4: -INFINITY
+native_color: shooter allocation score on c7auto1: 0
+native_color: shooter allocation score on c7auto2: 0
+native_color: shooter allocation score on c7auto3: 0
+native_color: shooter allocation score on c7auto4: 0
diff --git a/pengine/test10/clone_min_stop_two.summary b/pengine/test10/clone_min_stop_two.summary
new file mode 100644
index 0000000..c009d7d
--- /dev/null
+++ b/pengine/test10/clone_min_stop_two.summary
@@ -0,0 +1,42 @@
+
+Current cluster status:
+Node c7auto1 (1): standby
+Node c7auto2 (2): standby
+Online: [ c7auto3 c7auto4 ]
+
+ shooter (stonith:fence_phd_kvm): Started c7auto1
+ Clone Set: FAKECLONE-clone [FAKECLONE]
+ Started: [ c7auto1 c7auto2 c7auto3 ]
+ Stopped: [ c7auto4 ]
+ FAKE (ocf::heartbeat:Dummy): Started c7auto4
+
+Transition Summary:
+ * Move shooter (Started c7auto1 -> c7auto3)
+ * Stop FAKECLONE:0 (c7auto1)
+ * Stop FAKECLONE:1 (c7auto2)
+ * Stop FAKE (Started c7auto4)
+
+Executing cluster transition:
+ * Resource action: shooter stop on c7auto1
+ * Resource action: FAKE stop on c7auto4
+ * Resource action: shooter start on c7auto3
+ * Pseudo action: FAKECLONE-clone_stop_0
+ * Resource action: shooter monitor=60000 on c7auto3
+ * Resource action: FAKECLONE stop on c7auto1
+ * Resource action: FAKECLONE stop on c7auto2
+ * Pseudo action: FAKECLONE-clone_stopped_0
+ * Pseudo action: FAKECLONE-clone_start_0
+ * Pseudo action: all_stopped
+ * Pseudo action: FAKECLONE-clone_running_0
+
+Revised cluster status:
+Node c7auto1 (1): standby
+Node c7auto2 (2): standby
+Online: [ c7auto3 c7auto4 ]
+
+ shooter (stonith:fence_phd_kvm): Started c7auto3
+ Clone Set: FAKECLONE-clone [FAKECLONE]
+ Started: [ c7auto3 ]
+ Stopped: [ c7auto1 c7auto2 c7auto4 ]
+ FAKE (ocf::heartbeat:Dummy): Stopped
+
diff --git a/pengine/test10/clone_min_stop_two.xml b/pengine/test10/clone_min_stop_two.xml
new file mode 100644
index 0000000..8d085ad
--- /dev/null
+++ b/pengine/test10/clone_min_stop_two.xml
@@ -0,0 +1,154 @@
+<cib crm_feature_set="3.0.10" validate-with="pacemaker-2.3" epoch="15" num_updates="0" admin_epoch="0" cib-last-written="Thu Jul 16 15:28:22 2015" update-origin="c7auto1" update-client="cibadmin" update-user="root" have-quorum="1" dc-uuid="2">
+ <configuration>
+ <crm_config>
+ <cluster_property_set id="cib-bootstrap-options">
+ <nvpair id="cib-bootstrap-options-have-watchdog" name="have-watchdog" value="false"/>
+ <nvpair id="cib-bootstrap-options-dc-version" name="dc-version" value="1.1.12-ef86b49"/>
+ <nvpair id="cib-bootstrap-options-cluster-infrastructure" name="cluster-infrastructure" value="corosync"/>
+ <nvpair id="cib-bootstrap-options-cluster-name" name="cluster-name" value="phd"/>
+ </cluster_property_set>
+ </crm_config>
+ <nodes>
+ <node id="1" uname="c7auto1">
+ <instance_attributes id="nodes-1">
+ <nvpair id="nodes-1-standby" name="standby" value="on"/>
+ </instance_attributes>
+ </node>
+ <node id="2" uname="c7auto2">
+ <instance_attributes id="nodes-2">
+ <nvpair id="nodes-2-standby" name="standby" value="on"/>
+ </instance_attributes>
+ </node>
+ <node id="3" uname="c7auto3"/>
+ <node id="4" uname="c7auto4"/>
+ </nodes>
+ <resources>
+ <primitive class="stonith" id="shooter" type="fence_phd_kvm">
+ <instance_attributes id="shooter-instance_attributes"/>
+ <operations>
+ <op id="shooter-monitor-interval-60s" interval="60s" name="monitor"/>
+ </operations>
+ </primitive>
+ <clone id="FAKECLONE-clone">
+ <primitive class="ocf" id="FAKECLONE" provider="heartbeat" type="Dummy">
+ <instance_attributes id="FAKECLONE-instance_attributes"/>
+ <operations>
+ <op id="FAKECLONE-start-timeout-20" interval="0s" name="start" timeout="20"/>
+ <op id="FAKECLONE-stop-timeout-20" interval="0s" name="stop" timeout="20"/>
+ <op id="FAKECLONE-monitor-interval-10" interval="10" name="monitor" timeout="20"/>
+ </operations>
+ </primitive>
+ <meta_attributes id="FAKECLONE-clone-meta">
+ <nvpair id="FAKECLONE-clone-min" name="clone-min" value="2"/>
+ </meta_attributes>
+ </clone>
+ <primitive class="ocf" id="FAKE" provider="heartbeat" type="Dummy">
+ <instance_attributes id="FAKE-instance_attributes"/>
+ <operations>
+ <op id="FAKE-start-timeout-20" interval="0s" name="start" timeout="20"/>
+ <op id="FAKE-stop-timeout-20" interval="0s" name="stop" timeout="20"/>
+ <op id="FAKE-monitor-interval-10" interval="10" name="monitor" timeout="20"/>
+ </operations>
+ </primitive>
+ </resources>
+ <constraints>
+ <rsc_location id="location-FAKECLONE-clone-c7auto4--INFINITY" node="c7auto4" rsc="FAKECLONE-clone" score="-INFINITY"/>
+ <rsc_location id="location-FAKE-c7auto1--INFINITY" node="c7auto1" rsc="FAKE" score="-INFINITY"/>
+ <rsc_location id="location-FAKE-c7auto2--INFINITY" node="c7auto2" rsc="FAKE" score="-INFINITY"/>
+ <rsc_location id="location-FAKE-c7auto3--INFINITY" node="c7auto3" rsc="FAKE" score="-INFINITY"/>
+ <rsc_order first="FAKECLONE-clone" first-action="start" id="order-FAKECLONE-clone-FAKE-mandatory" then="FAKE" then-action="start"/>
+ </constraints>
+ </configuration>
+ <status>
+ <node_state id="1" uname="c7auto1" in_ccm="true" crmd="online" crm-debug-origin="do_update_resource" join="member" expected="member">
+ <lrm id="1">
+ <lrm_resources>
+ <lrm_resource id="shooter" type="fence_phd_kvm" class="stonith">
+ <lrm_rsc_op id="shooter_last_0" operation_key="shooter_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="11:0:0:b489d50c-a7db-4605-8519-a2d5e1452e29" transition-magic="0:0;11:0:0:b489d50c-a7db-4605-8519-a2d5e1452e29" on_node="c7auto1" call-id="6" rc-code="0" op-status="0" interval="0" last-run="1437073991" last-rc-change="1437073991" exec-time="9" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ <lrm_rsc_op id="shooter_monitor_60000" operation_key="shooter_monitor_60000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="9:1:0:b489d50c-a7db-4605-8519-a2d5e1452e29" transition-magic="0:0;9:1:0:b489d50c-a7db-4605-8519-a2d5e1452e29" on_node="c7auto1" call-id="7" rc-code="0" op-status="0" interval="60000" last-rc-change="1437073992" exec-time="7" queue-time="0" op-digest="4811cef7f7f94e3a35a70be7916cb2fd"/>
+ </lrm_resource>
+ <lrm_resource id="FAKECLONE" type="Dummy" class="ocf" provider="heartbeat">
+ <lrm_rsc_op id="FAKECLONE_last_0" operation_key="FAKECLONE_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="20:2:0:b489d50c-a7db-4605-8519-a2d5e1452e29" transition-magic="0:0;20:2:0:b489d50c-a7db-4605-8519-a2d5e1452e29" on_node="c7auto1" call-id="13" rc-code="0" op-status="0" interval="0" last-run="1437074829" last-rc-change="1437074829" exec-time="7" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" fake " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ <lrm_rsc_op id="FAKECLONE_monitor_10000" operation_key="FAKECLONE_monitor_10000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="21:2:0:b489d50c-a7db-4605-8519-a2d5e1452e29" transition-magic="0:0;21:2:0:b489d50c-a7db-4605-8519-a2d5e1452e29" on_node="c7auto1" call-id="14" rc-code="0" op-status="0" interval="10000" last-rc-change="1437074829" exec-time="7" queue-time="0" op-digest="4811cef7f7f94e3a35a70be7916cb2fd"/>
+ </lrm_resource>
+ <lrm_resource id="FAKE" type="Dummy" class="ocf" provider="heartbeat">
+ <lrm_rsc_op id="FAKE_last_0" operation_key="FAKE_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="8:4:7:b489d50c-a7db-4605-8519-a2d5e1452e29" transition-magic="0:7;8:4:7:b489d50c-a7db-4605-8519-a2d5e1452e29" on_node="c7auto1" call-id="18" rc-code="7" op-status="0" interval="0" last-run="1437074871" last-rc-change="1437074871" exec-time="7" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" fake " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ </lrm_resource>
+ </lrm_resources>
+ </lrm>
+ <transient_attributes id="1">
+ <instance_attributes id="status-1">
+ <nvpair id="status-1-shutdown" name="shutdown" value="0"/>
+ <nvpair id="status-1-probe_complete" name="probe_complete" value="true"/>
+ </instance_attributes>
+ </transient_attributes>
+ </node_state>
+ <node_state id="2" uname="c7auto2" in_ccm="true" crmd="online" crm-debug-origin="do_update_resource" join="member" expected="member">
+ <lrm id="2">
+ <lrm_resources>
+ <lrm_resource id="shooter" type="fence_phd_kvm" class="stonith">
+ <lrm_rsc_op id="shooter_last_0" operation_key="shooter_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="6:0:7:b489d50c-a7db-4605-8519-a2d5e1452e29" transition-magic="0:7;6:0:7:b489d50c-a7db-4605-8519-a2d5e1452e29" on_node="c7auto2" call-id="5" rc-code="7" op-status="0" interval="0" last-run="1437073988" last-rc-change="1437073988" exec-time="964" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ </lrm_resource>
+ <lrm_resource id="FAKECLONE" type="Dummy" class="ocf" provider="heartbeat">
+ <lrm_rsc_op id="FAKECLONE_last_0" operation_key="FAKECLONE_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="14:2:0:b489d50c-a7db-4605-8519-a2d5e1452e29" transition-magic="0:0;14:2:0:b489d50c-a7db-4605-8519-a2d5e1452e29" on_node="c7auto2" call-id="11" rc-code="0" op-status="0" interval="0" last-run="1437074829" last-rc-change="1437074829" exec-time="8" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" fake " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ <lrm_rsc_op id="FAKECLONE_monitor_10000" operation_key="FAKECLONE_monitor_10000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="15:2:0:b489d50c-a7db-4605-8519-a2d5e1452e29" transition-magic="0:0;15:2:0:b489d50c-a7db-4605-8519-a2d5e1452e29" on_node="c7auto2" call-id="12" rc-code="0" op-status="0" interval="10000" last-rc-change="1437074829" exec-time="10" queue-time="0" op-digest="4811cef7f7f94e3a35a70be7916cb2fd"/>
+ </lrm_resource>
+ <lrm_resource id="FAKE" type="Dummy" class="ocf" provider="heartbeat">
+ <lrm_rsc_op id="FAKE_last_0" operation_key="FAKE_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="10:4:7:b489d50c-a7db-4605-8519-a2d5e1452e29" transition-magic="0:7;10:4:7:b489d50c-a7db-4605-8519-a2d5e1452e29" on_node="c7auto2" call-id="16" rc-code="7" op-status="0" interval="0" last-run="1437074871" last-rc-change="1437074871" exec-time="10" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" fake " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ </lrm_resource>
+ </lrm_resources>
+ </lrm>
+ <transient_attributes id="2">
+ <instance_attributes id="status-2">
+ <nvpair id="status-2-shutdown" name="shutdown" value="0"/>
+ <nvpair id="status-2-probe_complete" name="probe_complete" value="true"/>
+ </instance_attributes>
+ </transient_attributes>
+ </node_state>
+ <node_state id="3" uname="c7auto3" in_ccm="true" crmd="online" crm-debug-origin="do_update_resource" join="member" expected="member">
+ <lrm id="3">
+ <lrm_resources>
+ <lrm_resource id="shooter" type="fence_phd_kvm" class="stonith">
+ <lrm_rsc_op id="shooter_last_0" operation_key="shooter_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="8:0:7:b489d50c-a7db-4605-8519-a2d5e1452e29" transition-magic="0:7;8:0:7:b489d50c-a7db-4605-8519-a2d5e1452e29" on_node="c7auto3" call-id="5" rc-code="7" op-status="0" interval="0" last-run="1437073988" last-rc-change="1437073988" exec-time="963" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ </lrm_resource>
+ <lrm_resource id="FAKECLONE" type="Dummy" class="ocf" provider="heartbeat">
+ <lrm_rsc_op id="FAKECLONE_last_0" operation_key="FAKECLONE_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="16:2:0:b489d50c-a7db-4605-8519-a2d5e1452e29" transition-magic="0:0;16:2:0:b489d50c-a7db-4605-8519-a2d5e1452e29" on_node="c7auto3" call-id="11" rc-code="0" op-status="0" interval="0" last-run="1437074828" last-rc-change="1437074828" exec-time="8" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" fake " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ <lrm_rsc_op id="FAKECLONE_monitor_10000" operation_key="FAKECLONE_monitor_10000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="17:2:0:b489d50c-a7db-4605-8519-a2d5e1452e29" transition-magic="0:0;17:2:0:b489d50c-a7db-4605-8519-a2d5e1452e29" on_node="c7auto3" call-id="12" rc-code="0" op-status="0" interval="10000" last-rc-change="1437074828" exec-time="10" queue-time="0" op-digest="4811cef7f7f94e3a35a70be7916cb2fd"/>
+ </lrm_resource>
+ <lrm_resource id="FAKE" type="Dummy" class="ocf" provider="heartbeat">
+ <lrm_rsc_op id="FAKE_last_0" operation_key="FAKE_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="12:4:7:b489d50c-a7db-4605-8519-a2d5e1452e29" transition-magic="0:7;12:4:7:b489d50c-a7db-4605-8519-a2d5e1452e29" on_node="c7auto3" call-id="16" rc-code="7" op-status="0" interval="0" last-run="1437074870" last-rc-change="1437074870" exec-time="8" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" fake " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ </lrm_resource>
+ </lrm_resources>
+ </lrm>
+ <transient_attributes id="3">
+ <instance_attributes id="status-3">
+ <nvpair id="status-3-shutdown" name="shutdown" value="0"/>
+ <nvpair id="status-3-probe_complete" name="probe_complete" value="true"/>
+ </instance_attributes>
+ </transient_attributes>
+ </node_state>
+ <node_state id="4" uname="c7auto4" in_ccm="true" crmd="online" crm-debug-origin="do_update_resource" join="member" expected="member">
+ <lrm id="4">
+ <lrm_resources>
+ <lrm_resource id="shooter" type="fence_phd_kvm" class="stonith">
+ <lrm_rsc_op id="shooter_last_0" operation_key="shooter_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="10:0:7:b489d50c-a7db-4605-8519-a2d5e1452e29" transition-magic="0:7;10:0:7:b489d50c-a7db-4605-8519-a2d5e1452e29" on_node="c7auto4" call-id="5" rc-code="7" op-status="0" interval="0" last-run="1437073988" last-rc-change="1437073988" exec-time="963" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ </lrm_resource>
+ <lrm_resource id="FAKECLONE" type="Dummy" class="ocf" provider="heartbeat">
+ <lrm_rsc_op id="FAKECLONE_last_0" operation_key="FAKECLONE_stop_0" operation="stop" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="20:3:0:b489d50c-a7db-4605-8519-a2d5e1452e29" transition-magic="0:0;20:3:0:b489d50c-a7db-4605-8519-a2d5e1452e29" on_node="c7auto4" call-id="14" rc-code="0" op-status="0" interval="0" last-run="1437074856" last-rc-change="1437074856" exec-time="8" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" fake " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ <lrm_rsc_op id="FAKECLONE_monitor_10000" operation_key="FAKECLONE_monitor_10000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="19:2:0:b489d50c-a7db-4605-8519-a2d5e1452e29" transition-magic="0:0;19:2:0:b489d50c-a7db-4605-8519-a2d5e1452e29" on_node="c7auto4" call-id="12" rc-code="0" op-status="0" interval="10000" last-rc-change="1437074829" exec-time="6" queue-time="0" op-digest="4811cef7f7f94e3a35a70be7916cb2fd"/>
+ </lrm_resource>
+ <lrm_resource id="FAKE" type="Dummy" class="ocf" provider="heartbeat">
+ <lrm_rsc_op id="FAKE_last_0" operation_key="FAKE_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="27:4:0:b489d50c-a7db-4605-8519-a2d5e1452e29" transition-magic="0:0;27:4:0:b489d50c-a7db-4605-8519-a2d5e1452e29" on_node="c7auto4" call-id="19" rc-code="0" op-status="0" interval="0" last-run="1437074871" last-rc-change="1437074871" exec-time="8" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" fake " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ <lrm_rsc_op id="FAKE_monitor_10000" operation_key="FAKE_monitor_10000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="28:4:0:b489d50c-a7db-4605-8519-a2d5e1452e29" transition-magic="0:0;28:4:0:b489d50c-a7db-4605-8519-a2d5e1452e29" on_node="c7auto4" call-id="20" rc-code="0" op-status="0" interval="10000" last-rc-change="1437074871" exec-time="7" queue-time="0" op-digest="4811cef7f7f94e3a35a70be7916cb2fd"/>
+ </lrm_resource>
+ </lrm_resources>
+ </lrm>
+ <transient_attributes id="4">
+ <instance_attributes id="status-4">
+ <nvpair id="status-4-shutdown" name="shutdown" value="0"/>
+ <nvpair id="status-4-probe_complete" name="probe_complete" value="true"/>
+ </instance_attributes>
+ </transient_attributes>
+ </node_state>
+ </status>
+</cib>
diff --git a/pengine/test10/cloned_start_one.dot b/pengine/test10/cloned_start_one.dot
new file mode 100644
index 0000000..b3c254c
--- /dev/null
+++ b/pengine/test10/cloned_start_one.dot
@@ -0,0 +1,32 @@
+ digraph "g" {
+"FAKECLONE-clone_running_0" [ style=bold color="green" fontcolor="orange"]
+"FAKECLONE-clone_start_0" -> "FAKECLONE-clone_running_0" [ style = bold]
+"FAKECLONE-clone_start_0" -> "FAKECLONE_start_0 c7auto1" [ style = bold]
+"FAKECLONE-clone_start_0" [ style=bold color="green" fontcolor="orange"]
+"FAKECLONE2-clone_running_0" [ style=dashed color="red" fontcolor="orange"]
+"FAKECLONE2-clone_start_0" -> "FAKECLONE2-clone_running_0" [ style = dashed]
+"FAKECLONE2-clone_start_0" -> "FAKECLONE2_start_0 c7auto4" [ style = dashed]
+"FAKECLONE2-clone_start_0" [ style=dashed color="red" fontcolor="orange"]
+"FAKECLONE2-clone_stop_0" -> "FAKECLONE2-clone_stopped_0" [ style = bold]
+"FAKECLONE2-clone_stop_0" -> "FAKECLONE2_stop_0 c7auto3" [ style = bold]
+"FAKECLONE2-clone_stop_0" -> "FAKECLONE2_stop_0 c7auto4" [ style = bold]
+"FAKECLONE2-clone_stop_0" [ style=bold color="green" fontcolor="orange"]
+"FAKECLONE2-clone_stopped_0" -> "FAKECLONE2-clone_start_0" [ style = dashed]
+"FAKECLONE2-clone_stopped_0" [ style=bold color="green" fontcolor="orange"]
+"FAKECLONE2_monitor_10000 c7auto4" [ style=dashed color="red" fontcolor="black"]
+"FAKECLONE2_start_0 c7auto4" -> "FAKECLONE2-clone_running_0" [ style = dashed]
+"FAKECLONE2_start_0 c7auto4" -> "FAKECLONE2_monitor_10000 c7auto4" [ style = dashed]
+"FAKECLONE2_start_0 c7auto4" [ style=dashed color="red" fontcolor="black"]
+"FAKECLONE2_stop_0 c7auto3" -> "FAKECLONE2-clone_stopped_0" [ style = bold]
+"FAKECLONE2_stop_0 c7auto3" -> "all_stopped" [ style = bold]
+"FAKECLONE2_stop_0 c7auto3" [ style=bold color="green" fontcolor="black"]
+"FAKECLONE2_stop_0 c7auto4" -> "FAKECLONE2-clone_stopped_0" [ style = bold]
+"FAKECLONE2_stop_0 c7auto4" -> "FAKECLONE2_start_0 c7auto4" [ style = dashed]
+"FAKECLONE2_stop_0 c7auto4" -> "all_stopped" [ style = bold]
+"FAKECLONE2_stop_0 c7auto4" [ style=bold color="green" fontcolor="black"]
+"FAKECLONE_monitor_10000 c7auto1" [ style=bold color="green" fontcolor="black"]
+"FAKECLONE_start_0 c7auto1" -> "FAKECLONE-clone_running_0" [ style = bold]
+"FAKECLONE_start_0 c7auto1" -> "FAKECLONE_monitor_10000 c7auto1" [ style = bold]
+"FAKECLONE_start_0 c7auto1" [ style=bold color="green" fontcolor="black"]
+"all_stopped" [ style=bold color="green" fontcolor="orange"]
+}
diff --git a/pengine/test10/cloned_start_one.exp b/pengine/test10/cloned_start_one.exp
new file mode 100644
index 0000000..636ccd8
--- /dev/null
+++ b/pengine/test10/cloned_start_one.exp
@@ -0,0 +1,118 @@
+<transition_graph cluster-delay="60s" stonith-timeout="60s" failed-stop-offset="INFINITY" failed-start-offset="INFINITY" transition_id="0">
+ <synapse id="0">
+ <action_set>
+ <rsc_op id="14" operation="monitor" operation_key="FAKECLONE_monitor_10000" internal_operation_key="FAKECLONE:0_monitor_10000" on_node="c7auto1" on_node_uuid="1">
+ <primitive id="FAKECLONE" long-id="FAKECLONE:0" class="ocf" provider="heartbeat" type="Dummy"/>
+ <attributes CRM_meta_clone="0" CRM_meta_clone_max="4" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_interval="10000" CRM_meta_name="monitor" CRM_meta_notify="false" CRM_meta_timeout="20000" />
+ </rsc_op>
+ </action_set>
+ <inputs>
+ <trigger>
+ <rsc_op id="13" operation="start" operation_key="FAKECLONE_start_0" internal_operation_key="FAKECLONE:0_start_0" on_node="c7auto1" on_node_uuid="1"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="1">
+ <action_set>
+ <rsc_op id="13" operation="start" operation_key="FAKECLONE_start_0" internal_operation_key="FAKECLONE:0_start_0" on_node="c7auto1" on_node_uuid="1">
+ <primitive id="FAKECLONE" long-id="FAKECLONE:0" class="ocf" provider="heartbeat" type="Dummy"/>
+ <attributes CRM_meta_clone="0" CRM_meta_clone_max="4" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_name="start" CRM_meta_notify="false" CRM_meta_timeout="20000" />
+ </rsc_op>
+ </action_set>
+ <inputs>
+ <trigger>
+ <pseudo_event id="15" operation="start" operation_key="FAKECLONE-clone_start_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="2" priority="1000000">
+ <action_set>
+ <pseudo_event id="16" operation="running" operation_key="FAKECLONE-clone_running_0">
+ <attributes CRM_meta_clone_max="4" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="false" CRM_meta_timeout="20000" />
+ </pseudo_event>
+ </action_set>
+ <inputs>
+ <trigger>
+ <rsc_op id="13" operation="start" operation_key="FAKECLONE_start_0" internal_operation_key="FAKECLONE:0_start_0" on_node="c7auto1" on_node_uuid="1"/>
+ </trigger>
+ <trigger>
+ <pseudo_event id="15" operation="start" operation_key="FAKECLONE-clone_start_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="3">
+ <action_set>
+ <pseudo_event id="15" operation="start" operation_key="FAKECLONE-clone_start_0">
+ <attributes CRM_meta_clone_max="4" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="false" CRM_meta_timeout="20000" />
+ </pseudo_event>
+ </action_set>
+ <inputs/>
+ </synapse>
+ <synapse id="4">
+ <action_set>
+ <rsc_op id="19" operation="stop" operation_key="FAKECLONE2_stop_0" internal_operation_key="FAKECLONE2:0_stop_0" on_node="c7auto3" on_node_uuid="3">
+ <primitive id="FAKECLONE2" long-id="FAKECLONE2:0" class="ocf" provider="heartbeat" type="Dummy"/>
+ <attributes CRM_meta_clone="0" CRM_meta_clone_max="4" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_name="stop" CRM_meta_notify="false" CRM_meta_timeout="20000" />
+ </rsc_op>
+ </action_set>
+ <inputs>
+ <trigger>
+ <pseudo_event id="24" operation="stop" operation_key="FAKECLONE2-clone_stop_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="5">
+ <action_set>
+ <rsc_op id="20" operation="stop" operation_key="FAKECLONE2_stop_0" internal_operation_key="FAKECLONE2:1_stop_0" on_node="c7auto4" on_node_uuid="4">
+ <primitive id="FAKECLONE2" long-id="FAKECLONE2:1" class="ocf" provider="heartbeat" type="Dummy"/>
+ <attributes CRM_meta_clone="1" CRM_meta_clone_max="4" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_name="stop" CRM_meta_notify="false" CRM_meta_timeout="20000" />
+ </rsc_op>
+ </action_set>
+ <inputs>
+ <trigger>
+ <pseudo_event id="24" operation="stop" operation_key="FAKECLONE2-clone_stop_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="6" priority="1000000">
+ <action_set>
+ <pseudo_event id="25" operation="stopped" operation_key="FAKECLONE2-clone_stopped_0">
+ <attributes CRM_meta_clone_max="4" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="false" CRM_meta_timeout="20000" />
+ </pseudo_event>
+ </action_set>
+ <inputs>
+ <trigger>
+ <rsc_op id="19" operation="stop" operation_key="FAKECLONE2_stop_0" internal_operation_key="FAKECLONE2:0_stop_0" on_node="c7auto3" on_node_uuid="3"/>
+ </trigger>
+ <trigger>
+ <rsc_op id="20" operation="stop" operation_key="FAKECLONE2_stop_0" internal_operation_key="FAKECLONE2:1_stop_0" on_node="c7auto4" on_node_uuid="4"/>
+ </trigger>
+ <trigger>
+ <pseudo_event id="24" operation="stop" operation_key="FAKECLONE2-clone_stop_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="7">
+ <action_set>
+ <pseudo_event id="24" operation="stop" operation_key="FAKECLONE2-clone_stop_0">
+ <attributes CRM_meta_clone_max="4" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="false" CRM_meta_timeout="20000" />
+ </pseudo_event>
+ </action_set>
+ <inputs/>
+ </synapse>
+ <synapse id="8">
+ <action_set>
+ <pseudo_event id="5" operation="all_stopped" operation_key="all_stopped">
+ <attributes />
+ </pseudo_event>
+ </action_set>
+ <inputs>
+ <trigger>
+ <rsc_op id="19" operation="stop" operation_key="FAKECLONE2_stop_0" internal_operation_key="FAKECLONE2:0_stop_0" on_node="c7auto3" on_node_uuid="3"/>
+ </trigger>
+ <trigger>
+ <rsc_op id="20" operation="stop" operation_key="FAKECLONE2_stop_0" internal_operation_key="FAKECLONE2:1_stop_0" on_node="c7auto4" on_node_uuid="4"/>
+ </trigger>
+ </inputs>
+ </synapse>
+</transition_graph>
diff --git a/pengine/test10/cloned_start_one.scores b/pengine/test10/cloned_start_one.scores
new file mode 100644
index 0000000..3dc6ab8
--- /dev/null
+++ b/pengine/test10/cloned_start_one.scores
@@ -0,0 +1,77 @@
+Allocation scores:
+clone_color: FAKECLONE-clone allocation score on c7auto1: 0
+clone_color: FAKECLONE-clone allocation score on c7auto2: 0
+clone_color: FAKECLONE-clone allocation score on c7auto3: 0
+clone_color: FAKECLONE-clone allocation score on c7auto4: -INFINITY
+clone_color: FAKECLONE2-clone allocation score on c7auto1: -INFINITY
+clone_color: FAKECLONE2-clone allocation score on c7auto2: -INFINITY
+clone_color: FAKECLONE2-clone allocation score on c7auto3: 0
+clone_color: FAKECLONE2-clone allocation score on c7auto4: 0
+clone_color: FAKECLONE2:0 allocation score on c7auto1: -INFINITY
+clone_color: FAKECLONE2:0 allocation score on c7auto2: -INFINITY
+clone_color: FAKECLONE2:0 allocation score on c7auto3: 1
+clone_color: FAKECLONE2:0 allocation score on c7auto4: 0
+clone_color: FAKECLONE2:1 allocation score on c7auto1: -INFINITY
+clone_color: FAKECLONE2:1 allocation score on c7auto2: -INFINITY
+clone_color: FAKECLONE2:1 allocation score on c7auto3: 0
+clone_color: FAKECLONE2:1 allocation score on c7auto4: 1
+clone_color: FAKECLONE2:2 allocation score on c7auto1: -INFINITY
+clone_color: FAKECLONE2:2 allocation score on c7auto2: -INFINITY
+clone_color: FAKECLONE2:2 allocation score on c7auto3: 0
+clone_color: FAKECLONE2:2 allocation score on c7auto4: 0
+clone_color: FAKECLONE2:3 allocation score on c7auto1: -INFINITY
+clone_color: FAKECLONE2:3 allocation score on c7auto2: -INFINITY
+clone_color: FAKECLONE2:3 allocation score on c7auto3: 0
+clone_color: FAKECLONE2:3 allocation score on c7auto4: 0
+clone_color: FAKECLONE:0 allocation score on c7auto1: 0
+clone_color: FAKECLONE:0 allocation score on c7auto2: 0
+clone_color: FAKECLONE:0 allocation score on c7auto3: 0
+clone_color: FAKECLONE:0 allocation score on c7auto4: -INFINITY
+clone_color: FAKECLONE:1 allocation score on c7auto1: 0
+clone_color: FAKECLONE:1 allocation score on c7auto2: 0
+clone_color: FAKECLONE:1 allocation score on c7auto3: 0
+clone_color: FAKECLONE:1 allocation score on c7auto4: -INFINITY
+clone_color: FAKECLONE:2 allocation score on c7auto1: 0
+clone_color: FAKECLONE:2 allocation score on c7auto2: 0
+clone_color: FAKECLONE:2 allocation score on c7auto3: 0
+clone_color: FAKECLONE:2 allocation score on c7auto4: -INFINITY
+clone_color: FAKECLONE:3 allocation score on c7auto1: 0
+clone_color: FAKECLONE:3 allocation score on c7auto2: 0
+clone_color: FAKECLONE:3 allocation score on c7auto3: 0
+clone_color: FAKECLONE:3 allocation score on c7auto4: -INFINITY
+native_color: FAKECLONE2:0 allocation score on c7auto1: -INFINITY
+native_color: FAKECLONE2:0 allocation score on c7auto2: -INFINITY
+native_color: FAKECLONE2:0 allocation score on c7auto3: -INFINITY
+native_color: FAKECLONE2:0 allocation score on c7auto4: -INFINITY
+native_color: FAKECLONE2:1 allocation score on c7auto1: -INFINITY
+native_color: FAKECLONE2:1 allocation score on c7auto2: -INFINITY
+native_color: FAKECLONE2:1 allocation score on c7auto3: -INFINITY
+native_color: FAKECLONE2:1 allocation score on c7auto4: 1
+native_color: FAKECLONE2:2 allocation score on c7auto1: -INFINITY
+native_color: FAKECLONE2:2 allocation score on c7auto2: -INFINITY
+native_color: FAKECLONE2:2 allocation score on c7auto3: -INFINITY
+native_color: FAKECLONE2:2 allocation score on c7auto4: -INFINITY
+native_color: FAKECLONE2:3 allocation score on c7auto1: -INFINITY
+native_color: FAKECLONE2:3 allocation score on c7auto2: -INFINITY
+native_color: FAKECLONE2:3 allocation score on c7auto3: -INFINITY
+native_color: FAKECLONE2:3 allocation score on c7auto4: -INFINITY
+native_color: FAKECLONE:0 allocation score on c7auto1: 0
+native_color: FAKECLONE:0 allocation score on c7auto2: -INFINITY
+native_color: FAKECLONE:0 allocation score on c7auto3: -INFINITY
+native_color: FAKECLONE:0 allocation score on c7auto4: -INFINITY
+native_color: FAKECLONE:1 allocation score on c7auto1: -INFINITY
+native_color: FAKECLONE:1 allocation score on c7auto2: -INFINITY
+native_color: FAKECLONE:1 allocation score on c7auto3: -INFINITY
+native_color: FAKECLONE:1 allocation score on c7auto4: -INFINITY
+native_color: FAKECLONE:2 allocation score on c7auto1: -INFINITY
+native_color: FAKECLONE:2 allocation score on c7auto2: -INFINITY
+native_color: FAKECLONE:2 allocation score on c7auto3: -INFINITY
+native_color: FAKECLONE:2 allocation score on c7auto4: -INFINITY
+native_color: FAKECLONE:3 allocation score on c7auto1: -INFINITY
+native_color: FAKECLONE:3 allocation score on c7auto2: -INFINITY
+native_color: FAKECLONE:3 allocation score on c7auto3: -INFINITY
+native_color: FAKECLONE:3 allocation score on c7auto4: -INFINITY
+native_color: shooter allocation score on c7auto1: 0
+native_color: shooter allocation score on c7auto2: 0
+native_color: shooter allocation score on c7auto3: 0
+native_color: shooter allocation score on c7auto4: 0
diff --git a/pengine/test10/cloned_start_one.summary b/pengine/test10/cloned_start_one.summary
new file mode 100644
index 0000000..20ac58f
--- /dev/null
+++ b/pengine/test10/cloned_start_one.summary
@@ -0,0 +1,41 @@
+
+Current cluster status:
+Node c7auto2 (2): standby
+Node c7auto3 (3): standby
+Online: [ c7auto1 c7auto4 ]
+
+ shooter (stonith:fence_phd_kvm): Started c7auto1
+ Clone Set: FAKECLONE-clone [FAKECLONE]
+ Stopped: [ c7auto1 c7auto2 c7auto3 c7auto4 ]
+ Clone Set: FAKECLONE2-clone [FAKECLONE2]
+ Started: [ c7auto3 c7auto4 ]
+ Stopped: [ c7auto1 c7auto2 ]
+
+Transition Summary:
+ * Start FAKECLONE:0 (c7auto1)
+ * Stop FAKECLONE2:0 (c7auto3)
+ * Stop FAKECLONE2:1 (Started c7auto4)
+
+Executing cluster transition:
+ * Pseudo action: FAKECLONE-clone_start_0
+ * Pseudo action: FAKECLONE2-clone_stop_0
+ * Resource action: FAKECLONE start on c7auto1
+ * Pseudo action: FAKECLONE-clone_running_0
+ * Resource action: FAKECLONE2 stop on c7auto3
+ * Resource action: FAKECLONE2 stop on c7auto4
+ * Pseudo action: FAKECLONE2-clone_stopped_0
+ * Pseudo action: all_stopped
+ * Resource action: FAKECLONE monitor=10000 on c7auto1
+
+Revised cluster status:
+Node c7auto2 (2): standby
+Node c7auto3 (3): standby
+Online: [ c7auto1 c7auto4 ]
+
+ shooter (stonith:fence_phd_kvm): Started c7auto1
+ Clone Set: FAKECLONE-clone [FAKECLONE]
+ Started: [ c7auto1 ]
+ Stopped: [ c7auto2 c7auto3 c7auto4 ]
+ Clone Set: FAKECLONE2-clone [FAKECLONE2]
+ Stopped: [ c7auto1 c7auto2 c7auto3 c7auto4 ]
+
diff --git a/pengine/test10/cloned_start_one.xml b/pengine/test10/cloned_start_one.xml
new file mode 100644
index 0000000..6c2bfe1
--- /dev/null
+++ b/pengine/test10/cloned_start_one.xml
@@ -0,0 +1,154 @@
+<cib crm_feature_set="3.0.10" validate-with="pacemaker-2.3" epoch="30" num_updates="0" admin_epoch="0" cib-last-written="Thu Jul 16 17:55:58 2015" update-origin="c7auto1" update-client="cibadmin" update-user="root" have-quorum="1" dc-uuid="2">
+ <configuration>
+ <crm_config>
+ <cluster_property_set id="cib-bootstrap-options">
+ <nvpair id="cib-bootstrap-options-have-watchdog" name="have-watchdog" value="false"/>
+ <nvpair id="cib-bootstrap-options-dc-version" name="dc-version" value="1.1.12-ef86b49"/>
+ <nvpair id="cib-bootstrap-options-cluster-infrastructure" name="cluster-infrastructure" value="corosync"/>
+ <nvpair id="cib-bootstrap-options-cluster-name" name="cluster-name" value="phd"/>
+ </cluster_property_set>
+ </crm_config>
+ <nodes>
+ <node id="1" uname="c7auto1"/>
+ <node id="2" uname="c7auto2">
+ <instance_attributes id="nodes-2">
+ <nvpair id="nodes-2-standby" name="standby" value="on"/>
+ </instance_attributes>
+ </node>
+ <node id="3" uname="c7auto3">
+ <instance_attributes id="nodes-3">
+ <nvpair id="nodes-3-standby" name="standby" value="on"/>
+ </instance_attributes>
+ </node>
+ <node id="4" uname="c7auto4"/>
+ </nodes>
+ <resources>
+ <primitive class="stonith" id="shooter" type="fence_phd_kvm">
+ <instance_attributes id="shooter-instance_attributes"/>
+ <operations>
+ <op id="shooter-monitor-interval-60s" interval="60s" name="monitor"/>
+ </operations>
+ </primitive>
+ <clone id="FAKECLONE-clone">
+ <primitive class="ocf" id="FAKECLONE" provider="heartbeat" type="Dummy">
+ <instance_attributes id="FAKECLONE-instance_attributes"/>
+ <operations>
+ <op id="FAKECLONE-start-timeout-20" interval="0s" name="start" timeout="20"/>
+ <op id="FAKECLONE-stop-timeout-20" interval="0s" name="stop" timeout="20"/>
+ <op id="FAKECLONE-monitor-interval-10" interval="10" name="monitor" timeout="20"/>
+ </operations>
+ </primitive>
+ <meta_attributes id="FAKECLONE-clone-meta">
+ <nvpair id="FAKECLONE-clone-min" name="clone-min" value="2"/>
+ </meta_attributes>
+ <meta_attributes id="FAKECLONE-clone-meta_attributes"/>
+ </clone>
+ <clone id="FAKECLONE2-clone">
+ <primitive class="ocf" id="FAKECLONE2" provider="heartbeat" type="Dummy">
+ <instance_attributes id="FAKECLONE2-instance_attributes"/>
+ <operations>
+ <op id="FAKECLONE2-start-timeout-20" interval="0s" name="start" timeout="20"/>
+ <op id="FAKECLONE2-stop-timeout-20" interval="0s" name="stop" timeout="20"/>
+ <op id="FAKECLONE2-monitor-interval-10" interval="10" name="monitor" timeout="20"/>
+ </operations>
+ </primitive>
+ <meta_attributes id="FAKECLONE2-clone-meta"/>
+ </clone>
+ </resources>
+ <constraints>
+ <rsc_location id="location-FAKECLONE-clone-c7auto4--INFINITY" node="c7auto4" rsc="FAKECLONE-clone" score="-INFINITY"/>
+ <rsc_location id="location-FAKECLONE2-clone-c7auto1--INFINITY" node="c7auto1" rsc="FAKECLONE2-clone" score="-INFINITY"/>
+ <rsc_location id="location-FAKECLONE2-clone-c7auto2--INFINITY" node="c7auto2" rsc="FAKECLONE2-clone" score="-INFINITY"/>
+ <rsc_order first="FAKECLONE-clone" first-action="start" id="order-FAKECLONE-clone-FAKECLONE2-clone-mandatory" then="FAKECLONE2-clone" then-action="start"/>
+ </constraints>
+ </configuration>
+ <status>
+ <node_state id="1" uname="c7auto1" in_ccm="true" crmd="online" crm-debug-origin="do_state_transition" join="member" expected="member">
+ <transient_attributes id="1">
+ <instance_attributes id="status-1">
+ <nvpair id="status-1-shutdown" name="shutdown" value="0"/>
+ <nvpair id="status-1-probe_complete" name="probe_complete" value="true"/>
+ </instance_attributes>
+ </transient_attributes>
+ <lrm id="1">
+ <lrm_resources>
+ <lrm_resource id="FAKECLONE2" type="Dummy" class="ocf" provider="heartbeat">
+ <lrm_rsc_op id="FAKECLONE2_last_0" operation_key="FAKECLONE2_stop_0" operation="stop" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.10" transition-key="17:28:0:b489d50c-a7db-4605-8519-a2d5e1452e29" transition-magic="0:0;17:28:0:b489d50c-a7db-4605-8519-a2d5e1452e29" on_node="c7auto1" call-id="30" rc-code="0" op-status="0" interval="0" last-run="1437083676" last-rc-change="1437083676" exec-time="12" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ </lrm_resource>
+ <lrm_resource id="shooter" type="fence_phd_kvm" class="stonith">
+ <lrm_rsc_op id="shooter_last_0" operation_key="shooter_start_0" operation="start" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.10" transition-key="11:0:0:b489d50c-a7db-4605-8519-a2d5e1452e29" transition-magic="0:0;11:0:0:b489d50c-a7db-4605-8519-a2d5e1452e29" on_node="c7auto1" call-id="6" rc-code="0" op-status="0" interval="0" last-run="1437073991" last-rc-change="1437073991" exec-time="9" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ <lrm_rsc_op id="shooter_monitor_60000" operation_key="shooter_monitor_60000" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.10" transition-key="9:1:0:b489d50c-a7db-4605-8519-a2d5e1452e29" transition-magic="0:0;9:1:0:b489d50c-a7db-4605-8519-a2d5e1452e29" on_node="c7auto1" call-id="7" rc-code="0" op-status="0" interval="60000" last-rc-change="1437073992" exec-time="7" queue-time="0" op-digest="4811cef7f7f94e3a35a70be7916cb2fd"/>
+ </lrm_resource>
+ <lrm_resource id="FAKECLONE" type="Dummy" class="ocf" provider="heartbeat">
+ <lrm_rsc_op id="FAKECLONE_last_0" operation_key="FAKECLONE_stop_0" operation="stop" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.10" transition-key="14:9:0:b489d50c-a7db-4605-8519-a2d5e1452e29" transition-magic="0:0;14:9:0:b489d50c-a7db-4605-8519-a2d5e1452e29" on_node="c7auto1" call-id="20" rc-code="0" op-status="0" interval="0" last-run="1437075474" last-rc-change="1437075474" exec-time="8" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ </lrm_resource>
+ </lrm_resources>
+ </lrm>
+ </node_state>
+ <node_state id="2" uname="c7auto2" in_ccm="true" crmd="online" crm-debug-origin="do_state_transition" join="member" expected="member">
+ <transient_attributes id="2">
+ <instance_attributes id="status-2">
+ <nvpair id="status-2-shutdown" name="shutdown" value="0"/>
+ <nvpair id="status-2-probe_complete" name="probe_complete" value="true"/>
+ </instance_attributes>
+ </transient_attributes>
+ <lrm id="2">
+ <lrm_resources>
+ <lrm_resource id="FAKECLONE2" type="Dummy" class="ocf" provider="heartbeat">
+ <lrm_rsc_op id="FAKECLONE2_last_0" operation_key="FAKECLONE2_stop_0" operation="stop" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.10" transition-key="16:29:0:b489d50c-a7db-4605-8519-a2d5e1452e29" transition-magic="0:0;16:29:0:b489d50c-a7db-4605-8519-a2d5e1452e29" on_node="c7auto2" call-id="32" rc-code="0" op-status="0" interval="0" last-run="1437083678" last-rc-change="1437083678" exec-time="10" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ </lrm_resource>
+ <lrm_resource id="shooter" type="fence_phd_kvm" class="stonith">
+ <lrm_rsc_op id="shooter_last_0" operation_key="shooter_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.10" transition-key="6:0:7:b489d50c-a7db-4605-8519-a2d5e1452e29" transition-magic="0:7;6:0:7:b489d50c-a7db-4605-8519-a2d5e1452e29" on_node="c7auto2" call-id="5" rc-code="7" op-status="0" interval="0" last-run="1437073988" last-rc-change="1437073988" exec-time="964" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ </lrm_resource>
+ <lrm_resource id="FAKECLONE" type="Dummy" class="ocf" provider="heartbeat">
+ <lrm_rsc_op id="FAKECLONE_last_0" operation_key="FAKECLONE_stop_0" operation="stop" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.10" transition-key="15:9:0:b489d50c-a7db-4605-8519-a2d5e1452e29" transition-magic="0:0;15:9:0:b489d50c-a7db-4605-8519-a2d5e1452e29" on_node="c7auto2" call-id="18" rc-code="0" op-status="0" interval="0" last-run="1437075473" last-rc-change="1437075473" exec-time="13" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ </lrm_resource>
+ </lrm_resources>
+ </lrm>
+ </node_state>
+ <node_state id="3" uname="c7auto3" in_ccm="true" crmd="online" crm-debug-origin="do_state_transition" join="member" expected="member">
+ <transient_attributes id="3">
+ <instance_attributes id="status-3">
+ <nvpair id="status-3-shutdown" name="shutdown" value="0"/>
+ <nvpair id="status-3-probe_complete" name="probe_complete" value="true"/>
+ </instance_attributes>
+ </transient_attributes>
+ <lrm id="3">
+ <lrm_resources>
+ <lrm_resource id="FAKECLONE2" type="Dummy" class="ocf" provider="heartbeat">
+ <lrm_rsc_op id="FAKECLONE2_last_0" operation_key="FAKECLONE2_start_0" operation="start" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.10" transition-key="17:30:0:b489d50c-a7db-4605-8519-a2d5e1452e29" transition-magic="0:0;17:30:0:b489d50c-a7db-4605-8519-a2d5e1452e29" on_node="c7auto3" call-id="29" rc-code="0" op-status="0" interval="0" last-run="1437083710" last-rc-change="1437083710" exec-time="6" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" fake " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ <lrm_rsc_op id="FAKECLONE2_monitor_10000" operation_key="FAKECLONE2_monitor_10000" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.10" transition-key="18:30:0:b489d50c-a7db-4605-8519-a2d5e1452e29" transition-magic="0:0;18:30:0:b489d50c-a7db-4605-8519-a2d5e1452e29" on_node="c7auto3" call-id="30" rc-code="0" op-status="0" interval="10000" last-rc-change="1437083710" exec-time="6" queue-time="0" op-digest="4811cef7f7f94e3a35a70be7916cb2fd"/>
+ </lrm_resource>
+ <lrm_resource id="shooter" type="fence_phd_kvm" class="stonith">
+ <lrm_rsc_op id="shooter_last_0" operation_key="shooter_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.10" transition-key="8:0:7:b489d50c-a7db-4605-8519-a2d5e1452e29" transition-magic="0:7;8:0:7:b489d50c-a7db-4605-8519-a2d5e1452e29" on_node="c7auto3" call-id="5" rc-code="7" op-status="0" interval="0" last-run="1437073988" last-rc-change="1437073988" exec-time="963" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ </lrm_resource>
+ <lrm_resource id="FAKECLONE" type="Dummy" class="ocf" provider="heartbeat">
+ <lrm_rsc_op id="FAKECLONE_last_0" operation_key="FAKECLONE_stop_0" operation="stop" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.10" transition-key="16:9:0:b489d50c-a7db-4605-8519-a2d5e1452e29" transition-magic="0:0;16:9:0:b489d50c-a7db-4605-8519-a2d5e1452e29" on_node="c7auto3" call-id="18" rc-code="0" op-status="0" interval="0" last-run="1437075473" last-rc-change="1437075473" exec-time="12" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ </lrm_resource>
+ </lrm_resources>
+ </lrm>
+ </node_state>
+ <node_state id="4" uname="c7auto4" in_ccm="true" crmd="online" crm-debug-origin="do_state_transition" join="member" expected="member">
+ <transient_attributes id="4">
+ <instance_attributes id="status-4">
+ <nvpair id="status-4-shutdown" name="shutdown" value="0"/>
+ <nvpair id="status-4-probe_complete" name="probe_complete" value="true"/>
+ </instance_attributes>
+ </transient_attributes>
+ <lrm id="4">
+ <lrm_resources>
+ <lrm_resource id="FAKECLONE2" type="Dummy" class="ocf" provider="heartbeat">
+ <lrm_rsc_op id="FAKECLONE2_last_0" operation_key="FAKECLONE2_start_0" operation="start" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.10" transition-key="22:26:0:b489d50c-a7db-4605-8519-a2d5e1452e29" transition-magic="0:0;22:26:0:b489d50c-a7db-4605-8519-a2d5e1452e29" on_node="c7auto4" call-id="29" rc-code="0" op-status="0" interval="0" last-run="1437083649" last-rc-change="1437083649" exec-time="7" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" fake " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ <lrm_rsc_op id="FAKECLONE2_monitor_10000" operation_key="FAKECLONE2_monitor_10000" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.10" transition-key="23:26:0:b489d50c-a7db-4605-8519-a2d5e1452e29" transition-magic="0:0;23:26:0:b489d50c-a7db-4605-8519-a2d5e1452e29" on_node="c7auto4" call-id="30" rc-code="0" op-status="0" interval="10000" last-rc-change="1437083649" exec-time="7" queue-time="0" op-digest="4811cef7f7f94e3a35a70be7916cb2fd"/>
+ </lrm_resource>
+ <lrm_resource id="shooter" type="fence_phd_kvm" class="stonith">
+ <lrm_rsc_op id="shooter_last_0" operation_key="shooter_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.10" transition-key="10:0:7:b489d50c-a7db-4605-8519-a2d5e1452e29" transition-magic="0:7;10:0:7:b489d50c-a7db-4605-8519-a2d5e1452e29" on_node="c7auto4" call-id="5" rc-code="7" op-status="0" interval="0" last-run="1437073988" last-rc-change="1437073988" exec-time="963" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ </lrm_resource>
+ <lrm_resource id="FAKECLONE" type="Dummy" class="ocf" provider="heartbeat">
+ <lrm_rsc_op id="FAKECLONE_last_0" operation_key="FAKECLONE_stop_0" operation="stop" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.10" transition-key="20:3:0:b489d50c-a7db-4605-8519-a2d5e1452e29" transition-magic="0:0;20:3:0:b489d50c-a7db-4605-8519-a2d5e1452e29" on_node="c7auto4" call-id="14" rc-code="0" op-status="0" interval="0" last-run="1437074856" last-rc-change="1437074856" exec-time="8" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ </lrm_resource>
+ </lrm_resources>
+ </lrm>
+ </node_state>
+ </status>
+</cib>
diff --git a/pengine/test10/cloned_start_two.dot b/pengine/test10/cloned_start_two.dot
new file mode 100644
index 0000000..348d435
--- /dev/null
+++ b/pengine/test10/cloned_start_two.dot
@@ -0,0 +1,26 @@
+ digraph "g" {
+"FAKECLONE-clone_running_0" [ style=bold color="green" fontcolor="orange"]
+"FAKECLONE-clone_start_0" -> "FAKECLONE-clone_running_0" [ style = bold]
+"FAKECLONE-clone_start_0" -> "FAKECLONE:1_start_0 c7auto1" [ style = bold]
+"FAKECLONE-clone_start_0" -> "FAKECLONE_start_0 c7auto2" [ style = bold]
+"FAKECLONE-clone_start_0" [ style=bold color="green" fontcolor="orange"]
+"FAKECLONE2-clone_stop_0" -> "FAKECLONE2-clone_stopped_0" [ style = bold]
+"FAKECLONE2-clone_stop_0" -> "FAKECLONE2_stop_0 c7auto3" [ style = bold]
+"FAKECLONE2-clone_stop_0" [ style=bold color="green" fontcolor="orange"]
+"FAKECLONE2-clone_stopped_0" [ style=bold color="green" fontcolor="orange"]
+"FAKECLONE2_stop_0 c7auto3" -> "FAKECLONE2-clone_stopped_0" [ style = bold]
+"FAKECLONE2_stop_0 c7auto3" -> "all_stopped" [ style = bold]
+"FAKECLONE2_stop_0 c7auto3" [ style=bold color="green" fontcolor="black"]
+"FAKECLONE:1_monitor_10000 c7auto1" [ style=bold color="green" fontcolor="black"]
+"FAKECLONE:1_start_0 c7auto1" -> "FAKECLONE-clone_running_0" [ style = bold]
+"FAKECLONE:1_start_0 c7auto1" -> "FAKECLONE:1_monitor_10000 c7auto1" [ style = bold]
+"FAKECLONE:1_start_0 c7auto1" -> "clone-one-or-more:order-FAKECLONE-clone-FAKECLONE2-clone-mandatory" [ style = bold]
+"FAKECLONE:1_start_0 c7auto1" [ style=bold color="green" fontcolor="black"]
+"FAKECLONE_monitor_10000 c7auto2" [ style=bold color="green" fontcolor="black"]
+"FAKECLONE_start_0 c7auto2" -> "FAKECLONE-clone_running_0" [ style = bold]
+"FAKECLONE_start_0 c7auto2" -> "FAKECLONE_monitor_10000 c7auto2" [ style = bold]
+"FAKECLONE_start_0 c7auto2" -> "clone-one-or-more:order-FAKECLONE-clone-FAKECLONE2-clone-mandatory" [ style = bold]
+"FAKECLONE_start_0 c7auto2" [ style=bold color="green" fontcolor="black"]
+"all_stopped" [ style=bold color="green" fontcolor="orange"]
+"clone-one-or-more:order-FAKECLONE-clone-FAKECLONE2-clone-mandatory" [ style=bold color="green" fontcolor="orange"]
+}
diff --git a/pengine/test10/cloned_start_two.exp b/pengine/test10/cloned_start_two.exp
new file mode 100644
index 0000000..ee82324
--- /dev/null
+++ b/pengine/test10/cloned_start_two.exp
@@ -0,0 +1,143 @@
+<transition_graph cluster-delay="60s" stonith-timeout="60s" failed-stop-offset="INFINITY" failed-start-offset="INFINITY" transition_id="0">
+ <synapse id="0">
+ <action_set>
+ <rsc_op id="14" operation="monitor" operation_key="FAKECLONE_monitor_10000" internal_operation_key="FAKECLONE:0_monitor_10000" on_node="c7auto2" on_node_uuid="2">
+ <primitive id="FAKECLONE" long-id="FAKECLONE:0" class="ocf" provider="heartbeat" type="Dummy"/>
+ <attributes CRM_meta_clone="0" CRM_meta_clone_max="4" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_interval="10000" CRM_meta_name="monitor" CRM_meta_notify="false" CRM_meta_timeout="20000" />
+ </rsc_op>
+ </action_set>
+ <inputs>
+ <trigger>
+ <rsc_op id="13" operation="start" operation_key="FAKECLONE_start_0" internal_operation_key="FAKECLONE:0_start_0" on_node="c7auto2" on_node_uuid="2"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="1">
+ <action_set>
+ <rsc_op id="13" operation="start" operation_key="FAKECLONE_start_0" internal_operation_key="FAKECLONE:0_start_0" on_node="c7auto2" on_node_uuid="2">
+ <primitive id="FAKECLONE" long-id="FAKECLONE:0" class="ocf" provider="heartbeat" type="Dummy"/>
+ <attributes CRM_meta_clone="0" CRM_meta_clone_max="4" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_name="start" CRM_meta_notify="false" CRM_meta_timeout="20000" />
+ </rsc_op>
+ </action_set>
+ <inputs>
+ <trigger>
+ <pseudo_event id="17" operation="start" operation_key="FAKECLONE-clone_start_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="2">
+ <action_set>
+ <rsc_op id="16" operation="monitor" operation_key="FAKECLONE:1_monitor_10000" on_node="c7auto1" on_node_uuid="1">
+ <primitive id="FAKECLONE" long-id="FAKECLONE:1" class="ocf" provider="heartbeat" type="Dummy"/>
+ <attributes CRM_meta_clone="1" CRM_meta_clone_max="4" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_interval="10000" CRM_meta_name="monitor" CRM_meta_notify="false" CRM_meta_timeout="20000" />
+ </rsc_op>
+ </action_set>
+ <inputs>
+ <trigger>
+ <rsc_op id="15" operation="start" operation_key="FAKECLONE:1_start_0" on_node="c7auto1" on_node_uuid="1"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="3">
+ <action_set>
+ <rsc_op id="15" operation="start" operation_key="FAKECLONE:1_start_0" on_node="c7auto1" on_node_uuid="1">
+ <primitive id="FAKECLONE" long-id="FAKECLONE:1" class="ocf" provider="heartbeat" type="Dummy"/>
+ <attributes CRM_meta_clone="1" CRM_meta_clone_max="4" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_name="start" CRM_meta_notify="false" CRM_meta_timeout="20000" />
+ </rsc_op>
+ </action_set>
+ <inputs>
+ <trigger>
+ <pseudo_event id="17" operation="start" operation_key="FAKECLONE-clone_start_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="4" priority="1000000">
+ <action_set>
+ <pseudo_event id="18" operation="running" operation_key="FAKECLONE-clone_running_0">
+ <attributes CRM_meta_clone_max="4" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="false" CRM_meta_timeout="20000" />
+ </pseudo_event>
+ </action_set>
+ <inputs>
+ <trigger>
+ <rsc_op id="13" operation="start" operation_key="FAKECLONE_start_0" internal_operation_key="FAKECLONE:0_start_0" on_node="c7auto2" on_node_uuid="2"/>
+ </trigger>
+ <trigger>
+ <rsc_op id="15" operation="start" operation_key="FAKECLONE:1_start_0" on_node="c7auto1" on_node_uuid="1"/>
+ </trigger>
+ <trigger>
+ <pseudo_event id="17" operation="start" operation_key="FAKECLONE-clone_start_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="5">
+ <action_set>
+ <pseudo_event id="17" operation="start" operation_key="FAKECLONE-clone_start_0">
+ <attributes CRM_meta_clone_max="4" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="false" CRM_meta_timeout="20000" />
+ </pseudo_event>
+ </action_set>
+ <inputs/>
+ </synapse>
+ <synapse id="6">
+ <action_set>
+ <rsc_op id="21" operation="stop" operation_key="FAKECLONE2_stop_0" internal_operation_key="FAKECLONE2:0_stop_0" on_node="c7auto3" on_node_uuid="3">
+ <primitive id="FAKECLONE2" long-id="FAKECLONE2:0" class="ocf" provider="heartbeat" type="Dummy"/>
+ <attributes CRM_meta_clone="0" CRM_meta_clone_max="4" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_name="stop" CRM_meta_notify="false" CRM_meta_timeout="20000" />
+ </rsc_op>
+ </action_set>
+ <inputs>
+ <trigger>
+ <pseudo_event id="26" operation="stop" operation_key="FAKECLONE2-clone_stop_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="7" priority="1000000">
+ <action_set>
+ <pseudo_event id="27" operation="stopped" operation_key="FAKECLONE2-clone_stopped_0">
+ <attributes CRM_meta_clone_max="4" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="false" CRM_meta_timeout="20000" />
+ </pseudo_event>
+ </action_set>
+ <inputs>
+ <trigger>
+ <rsc_op id="21" operation="stop" operation_key="FAKECLONE2_stop_0" internal_operation_key="FAKECLONE2:0_stop_0" on_node="c7auto3" on_node_uuid="3"/>
+ </trigger>
+ <trigger>
+ <pseudo_event id="26" operation="stop" operation_key="FAKECLONE2-clone_stop_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="8">
+ <action_set>
+ <pseudo_event id="26" operation="stop" operation_key="FAKECLONE2-clone_stop_0">
+ <attributes CRM_meta_clone_max="4" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="false" CRM_meta_timeout="20000" />
+ </pseudo_event>
+ </action_set>
+ <inputs/>
+ </synapse>
+ <synapse id="9">
+ <action_set>
+ <pseudo_event id="5" operation="all_stopped" operation_key="all_stopped">
+ <attributes />
+ </pseudo_event>
+ </action_set>
+ <inputs>
+ <trigger>
+ <rsc_op id="21" operation="stop" operation_key="FAKECLONE2_stop_0" internal_operation_key="FAKECLONE2:0_stop_0" on_node="c7auto3" on_node_uuid="3"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="10">
+ <action_set>
+ <pseudo_event id="4" operation="clone-one-or-more:order-FAKECLONE-clone-FAKECLONE2-clone-mandatory" operation_key="clone-one-or-more:order-FAKECLONE-clone-FAKECLONE2-clone-mandatory">
+ <attributes />
+ </pseudo_event>
+ </action_set>
+ <inputs>
+ <trigger>
+ <rsc_op id="13" operation="start" operation_key="FAKECLONE_start_0" internal_operation_key="FAKECLONE:0_start_0" on_node="c7auto2" on_node_uuid="2"/>
+ </trigger>
+ <trigger>
+ <rsc_op id="15" operation="start" operation_key="FAKECLONE:1_start_0" on_node="c7auto1" on_node_uuid="1"/>
+ </trigger>
+ </inputs>
+ </synapse>
+</transition_graph>
diff --git a/pengine/test10/cloned_start_two.scores b/pengine/test10/cloned_start_two.scores
new file mode 100644
index 0000000..dae3b5d
--- /dev/null
+++ b/pengine/test10/cloned_start_two.scores
@@ -0,0 +1,77 @@
+Allocation scores:
+clone_color: FAKECLONE-clone allocation score on c7auto1: 0
+clone_color: FAKECLONE-clone allocation score on c7auto2: 0
+clone_color: FAKECLONE-clone allocation score on c7auto3: 0
+clone_color: FAKECLONE-clone allocation score on c7auto4: -INFINITY
+clone_color: FAKECLONE2-clone allocation score on c7auto1: -INFINITY
+clone_color: FAKECLONE2-clone allocation score on c7auto2: -INFINITY
+clone_color: FAKECLONE2-clone allocation score on c7auto3: 0
+clone_color: FAKECLONE2-clone allocation score on c7auto4: 0
+clone_color: FAKECLONE2:0 allocation score on c7auto1: -INFINITY
+clone_color: FAKECLONE2:0 allocation score on c7auto2: -INFINITY
+clone_color: FAKECLONE2:0 allocation score on c7auto3: 1
+clone_color: FAKECLONE2:0 allocation score on c7auto4: 0
+clone_color: FAKECLONE2:1 allocation score on c7auto1: -INFINITY
+clone_color: FAKECLONE2:1 allocation score on c7auto2: -INFINITY
+clone_color: FAKECLONE2:1 allocation score on c7auto3: 0
+clone_color: FAKECLONE2:1 allocation score on c7auto4: 1
+clone_color: FAKECLONE2:2 allocation score on c7auto1: -INFINITY
+clone_color: FAKECLONE2:2 allocation score on c7auto2: -INFINITY
+clone_color: FAKECLONE2:2 allocation score on c7auto3: 0
+clone_color: FAKECLONE2:2 allocation score on c7auto4: 0
+clone_color: FAKECLONE2:3 allocation score on c7auto1: -INFINITY
+clone_color: FAKECLONE2:3 allocation score on c7auto2: -INFINITY
+clone_color: FAKECLONE2:3 allocation score on c7auto3: 0
+clone_color: FAKECLONE2:3 allocation score on c7auto4: 0
+clone_color: FAKECLONE:0 allocation score on c7auto1: 0
+clone_color: FAKECLONE:0 allocation score on c7auto2: 0
+clone_color: FAKECLONE:0 allocation score on c7auto3: 0
+clone_color: FAKECLONE:0 allocation score on c7auto4: -INFINITY
+clone_color: FAKECLONE:1 allocation score on c7auto1: 0
+clone_color: FAKECLONE:1 allocation score on c7auto2: 0
+clone_color: FAKECLONE:1 allocation score on c7auto3: 0
+clone_color: FAKECLONE:1 allocation score on c7auto4: -INFINITY
+clone_color: FAKECLONE:2 allocation score on c7auto1: 0
+clone_color: FAKECLONE:2 allocation score on c7auto2: 0
+clone_color: FAKECLONE:2 allocation score on c7auto3: 0
+clone_color: FAKECLONE:2 allocation score on c7auto4: -INFINITY
+clone_color: FAKECLONE:3 allocation score on c7auto1: 0
+clone_color: FAKECLONE:3 allocation score on c7auto2: 0
+clone_color: FAKECLONE:3 allocation score on c7auto3: 0
+clone_color: FAKECLONE:3 allocation score on c7auto4: -INFINITY
+native_color: FAKECLONE2:0 allocation score on c7auto1: -INFINITY
+native_color: FAKECLONE2:0 allocation score on c7auto2: -INFINITY
+native_color: FAKECLONE2:0 allocation score on c7auto3: -INFINITY
+native_color: FAKECLONE2:0 allocation score on c7auto4: -INFINITY
+native_color: FAKECLONE2:1 allocation score on c7auto1: -INFINITY
+native_color: FAKECLONE2:1 allocation score on c7auto2: -INFINITY
+native_color: FAKECLONE2:1 allocation score on c7auto3: -INFINITY
+native_color: FAKECLONE2:1 allocation score on c7auto4: 1
+native_color: FAKECLONE2:2 allocation score on c7auto1: -INFINITY
+native_color: FAKECLONE2:2 allocation score on c7auto2: -INFINITY
+native_color: FAKECLONE2:2 allocation score on c7auto3: -INFINITY
+native_color: FAKECLONE2:2 allocation score on c7auto4: -INFINITY
+native_color: FAKECLONE2:3 allocation score on c7auto1: -INFINITY
+native_color: FAKECLONE2:3 allocation score on c7auto2: -INFINITY
+native_color: FAKECLONE2:3 allocation score on c7auto3: -INFINITY
+native_color: FAKECLONE2:3 allocation score on c7auto4: -INFINITY
+native_color: FAKECLONE:0 allocation score on c7auto1: 0
+native_color: FAKECLONE:0 allocation score on c7auto2: 0
+native_color: FAKECLONE:0 allocation score on c7auto3: -INFINITY
+native_color: FAKECLONE:0 allocation score on c7auto4: -INFINITY
+native_color: FAKECLONE:1 allocation score on c7auto1: 0
+native_color: FAKECLONE:1 allocation score on c7auto2: -INFINITY
+native_color: FAKECLONE:1 allocation score on c7auto3: -INFINITY
+native_color: FAKECLONE:1 allocation score on c7auto4: -INFINITY
+native_color: FAKECLONE:2 allocation score on c7auto1: -INFINITY
+native_color: FAKECLONE:2 allocation score on c7auto2: -INFINITY
+native_color: FAKECLONE:2 allocation score on c7auto3: -INFINITY
+native_color: FAKECLONE:2 allocation score on c7auto4: -INFINITY
+native_color: FAKECLONE:3 allocation score on c7auto1: -INFINITY
+native_color: FAKECLONE:3 allocation score on c7auto2: -INFINITY
+native_color: FAKECLONE:3 allocation score on c7auto3: -INFINITY
+native_color: FAKECLONE:3 allocation score on c7auto4: -INFINITY
+native_color: shooter allocation score on c7auto1: 0
+native_color: shooter allocation score on c7auto2: 0
+native_color: shooter allocation score on c7auto3: 0
+native_color: shooter allocation score on c7auto4: 0
diff --git a/pengine/test10/cloned_start_two.summary b/pengine/test10/cloned_start_two.summary
new file mode 100644
index 0000000..bea4609
--- /dev/null
+++ b/pengine/test10/cloned_start_two.summary
@@ -0,0 +1,42 @@
+
+Current cluster status:
+Node c7auto3 (3): standby
+Online: [ c7auto1 c7auto2 c7auto4 ]
+
+ shooter (stonith:fence_phd_kvm): Started c7auto1
+ Clone Set: FAKECLONE-clone [FAKECLONE]
+ Stopped: [ c7auto1 c7auto2 c7auto3 c7auto4 ]
+ Clone Set: FAKECLONE2-clone [FAKECLONE2]
+ Started: [ c7auto3 c7auto4 ]
+ Stopped: [ c7auto1 c7auto2 ]
+
+Transition Summary:
+ * Start FAKECLONE:0 (c7auto2)
+ * Start FAKECLONE:1 (c7auto1)
+ * Stop FAKECLONE2:0 (c7auto3)
+
+Executing cluster transition:
+ * Pseudo action: FAKECLONE-clone_start_0
+ * Pseudo action: FAKECLONE2-clone_stop_0
+ * Resource action: FAKECLONE start on c7auto2
+ * Resource action: FAKECLONE start on c7auto1
+ * Pseudo action: FAKECLONE-clone_running_0
+ * Resource action: FAKECLONE2 stop on c7auto3
+ * Pseudo action: FAKECLONE2-clone_stopped_0
+ * Pseudo action: all_stopped
+ * Pseudo action: clone-one-or-more:order-FAKECLONE-clone-FAKECLONE2-clone-mandatory
+ * Resource action: FAKECLONE monitor=10000 on c7auto2
+ * Resource action: FAKECLONE monitor=10000 on c7auto1
+
+Revised cluster status:
+Node c7auto3 (3): standby
+Online: [ c7auto1 c7auto2 c7auto4 ]
+
+ shooter (stonith:fence_phd_kvm): Started c7auto1
+ Clone Set: FAKECLONE-clone [FAKECLONE]
+ Started: [ c7auto1 c7auto2 ]
+ Stopped: [ c7auto3 c7auto4 ]
+ Clone Set: FAKECLONE2-clone [FAKECLONE2]
+ Started: [ c7auto4 ]
+ Stopped: [ c7auto1 c7auto2 c7auto3 ]
+
diff --git a/pengine/test10/cloned_start_two.xml b/pengine/test10/cloned_start_two.xml
new file mode 100644
index 0000000..be78317
--- /dev/null
+++ b/pengine/test10/cloned_start_two.xml
@@ -0,0 +1,152 @@
+<cib crm_feature_set="3.0.10" validate-with="pacemaker-2.3" epoch="31" num_updates="0" admin_epoch="0" cib-last-written="Thu Jul 16 17:55:58 2015" update-origin="c7auto1" update-client="cibadmin" update-user="root" have-quorum="1" dc-uuid="2">
+ <configuration>
+ <crm_config>
+ <cluster_property_set id="cib-bootstrap-options">
+ <nvpair id="cib-bootstrap-options-have-watchdog" name="have-watchdog" value="false"/>
+ <nvpair id="cib-bootstrap-options-dc-version" name="dc-version" value="1.1.12-ef86b49"/>
+ <nvpair id="cib-bootstrap-options-cluster-infrastructure" name="cluster-infrastructure" value="corosync"/>
+ <nvpair id="cib-bootstrap-options-cluster-name" name="cluster-name" value="phd"/>
+ </cluster_property_set>
+ </crm_config>
+ <nodes>
+ <node id="1" uname="c7auto1"/>
+ <node id="2" uname="c7auto2">
+ <instance_attributes id="nodes-2"/>
+ </node>
+ <node id="3" uname="c7auto3">
+ <instance_attributes id="nodes-3">
+ <nvpair id="nodes-3-standby" name="standby" value="on"/>
+ </instance_attributes>
+ </node>
+ <node id="4" uname="c7auto4"/>
+ </nodes>
+ <resources>
+ <primitive class="stonith" id="shooter" type="fence_phd_kvm">
+ <instance_attributes id="shooter-instance_attributes"/>
+ <operations>
+ <op id="shooter-monitor-interval-60s" interval="60s" name="monitor"/>
+ </operations>
+ </primitive>
+ <clone id="FAKECLONE-clone">
+ <primitive class="ocf" id="FAKECLONE" provider="heartbeat" type="Dummy">
+ <instance_attributes id="FAKECLONE-instance_attributes"/>
+ <operations>
+ <op id="FAKECLONE-start-timeout-20" interval="0s" name="start" timeout="20"/>
+ <op id="FAKECLONE-stop-timeout-20" interval="0s" name="stop" timeout="20"/>
+ <op id="FAKECLONE-monitor-interval-10" interval="10" name="monitor" timeout="20"/>
+ </operations>
+ </primitive>
+ <meta_attributes id="FAKECLONE-clone-meta">
+ <nvpair id="FAKECLONE-clone-min" name="clone-min" value="2"/>
+ </meta_attributes>
+ <meta_attributes id="FAKECLONE-clone-meta_attributes"/>
+ </clone>
+ <clone id="FAKECLONE2-clone">
+ <primitive class="ocf" id="FAKECLONE2" provider="heartbeat" type="Dummy">
+ <instance_attributes id="FAKECLONE2-instance_attributes"/>
+ <operations>
+ <op id="FAKECLONE2-start-timeout-20" interval="0s" name="start" timeout="20"/>
+ <op id="FAKECLONE2-stop-timeout-20" interval="0s" name="stop" timeout="20"/>
+ <op id="FAKECLONE2-monitor-interval-10" interval="10" name="monitor" timeout="20"/>
+ </operations>
+ </primitive>
+ <meta_attributes id="FAKECLONE2-clone-meta"/>
+ </clone>
+ </resources>
+ <constraints>
+ <rsc_location id="location-FAKECLONE-clone-c7auto4--INFINITY" node="c7auto4" rsc="FAKECLONE-clone" score="-INFINITY"/>
+ <rsc_location id="location-FAKECLONE2-clone-c7auto1--INFINITY" node="c7auto1" rsc="FAKECLONE2-clone" score="-INFINITY"/>
+ <rsc_location id="location-FAKECLONE2-clone-c7auto2--INFINITY" node="c7auto2" rsc="FAKECLONE2-clone" score="-INFINITY"/>
+ <rsc_order first="FAKECLONE-clone" first-action="start" id="order-FAKECLONE-clone-FAKECLONE2-clone-mandatory" then="FAKECLONE2-clone" then-action="start"/>
+ </constraints>
+ </configuration>
+ <status>
+ <node_state id="1" uname="c7auto1" in_ccm="true" crmd="online" crm-debug-origin="do_state_transition" join="member" expected="member">
+ <transient_attributes id="1">
+ <instance_attributes id="status-1">
+ <nvpair id="status-1-shutdown" name="shutdown" value="0"/>
+ <nvpair id="status-1-probe_complete" name="probe_complete" value="true"/>
+ </instance_attributes>
+ </transient_attributes>
+ <lrm id="1">
+ <lrm_resources>
+ <lrm_resource id="FAKECLONE2" type="Dummy" class="ocf" provider="heartbeat">
+ <lrm_rsc_op id="FAKECLONE2_last_0" operation_key="FAKECLONE2_stop_0" operation="stop" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.10" transition-key="17:28:0:b489d50c-a7db-4605-8519-a2d5e1452e29" transition-magic="0:0;17:28:0:b489d50c-a7db-4605-8519-a2d5e1452e29" on_node="c7auto1" call-id="30" rc-code="0" op-status="0" interval="0" last-run="1437083676" last-rc-change="1437083676" exec-time="12" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ </lrm_resource>
+ <lrm_resource id="shooter" type="fence_phd_kvm" class="stonith">
+ <lrm_rsc_op id="shooter_last_0" operation_key="shooter_start_0" operation="start" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.10" transition-key="11:0:0:b489d50c-a7db-4605-8519-a2d5e1452e29" transition-magic="0:0;11:0:0:b489d50c-a7db-4605-8519-a2d5e1452e29" on_node="c7auto1" call-id="6" rc-code="0" op-status="0" interval="0" last-run="1437073991" last-rc-change="1437073991" exec-time="9" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ <lrm_rsc_op id="shooter_monitor_60000" operation_key="shooter_monitor_60000" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.10" transition-key="9:1:0:b489d50c-a7db-4605-8519-a2d5e1452e29" transition-magic="0:0;9:1:0:b489d50c-a7db-4605-8519-a2d5e1452e29" on_node="c7auto1" call-id="7" rc-code="0" op-status="0" interval="60000" last-rc-change="1437073992" exec-time="7" queue-time="0" op-digest="4811cef7f7f94e3a35a70be7916cb2fd"/>
+ </lrm_resource>
+ <lrm_resource id="FAKECLONE" type="Dummy" class="ocf" provider="heartbeat">
+ <lrm_rsc_op id="FAKECLONE_last_0" operation_key="FAKECLONE_stop_0" operation="stop" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.10" transition-key="14:9:0:b489d50c-a7db-4605-8519-a2d5e1452e29" transition-magic="0:0;14:9:0:b489d50c-a7db-4605-8519-a2d5e1452e29" on_node="c7auto1" call-id="20" rc-code="0" op-status="0" interval="0" last-run="1437075474" last-rc-change="1437075474" exec-time="8" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ </lrm_resource>
+ </lrm_resources>
+ </lrm>
+ </node_state>
+ <node_state id="2" uname="c7auto2" in_ccm="true" crmd="online" crm-debug-origin="do_state_transition" join="member" expected="member">
+ <transient_attributes id="2">
+ <instance_attributes id="status-2">
+ <nvpair id="status-2-shutdown" name="shutdown" value="0"/>
+ <nvpair id="status-2-probe_complete" name="probe_complete" value="true"/>
+ </instance_attributes>
+ </transient_attributes>
+ <lrm id="2">
+ <lrm_resources>
+ <lrm_resource id="FAKECLONE2" type="Dummy" class="ocf" provider="heartbeat">
+ <lrm_rsc_op id="FAKECLONE2_last_0" operation_key="FAKECLONE2_stop_0" operation="stop" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.10" transition-key="16:29:0:b489d50c-a7db-4605-8519-a2d5e1452e29" transition-magic="0:0;16:29:0:b489d50c-a7db-4605-8519-a2d5e1452e29" on_node="c7auto2" call-id="32" rc-code="0" op-status="0" interval="0" last-run="1437083678" last-rc-change="1437083678" exec-time="10" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ </lrm_resource>
+ <lrm_resource id="shooter" type="fence_phd_kvm" class="stonith">
+ <lrm_rsc_op id="shooter_last_0" operation_key="shooter_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.10" transition-key="6:0:7:b489d50c-a7db-4605-8519-a2d5e1452e29" transition-magic="0:7;6:0:7:b489d50c-a7db-4605-8519-a2d5e1452e29" on_node="c7auto2" call-id="5" rc-code="7" op-status="0" interval="0" last-run="1437073988" last-rc-change="1437073988" exec-time="964" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ </lrm_resource>
+ <lrm_resource id="FAKECLONE" type="Dummy" class="ocf" provider="heartbeat">
+ <lrm_rsc_op id="FAKECLONE_last_0" operation_key="FAKECLONE_stop_0" operation="stop" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.10" transition-key="15:9:0:b489d50c-a7db-4605-8519-a2d5e1452e29" transition-magic="0:0;15:9:0:b489d50c-a7db-4605-8519-a2d5e1452e29" on_node="c7auto2" call-id="18" rc-code="0" op-status="0" interval="0" last-run="1437075473" last-rc-change="1437075473" exec-time="13" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ </lrm_resource>
+ </lrm_resources>
+ </lrm>
+ </node_state>
+ <node_state id="3" uname="c7auto3" in_ccm="true" crmd="online" crm-debug-origin="do_state_transition" join="member" expected="member">
+ <transient_attributes id="3">
+ <instance_attributes id="status-3">
+ <nvpair id="status-3-shutdown" name="shutdown" value="0"/>
+ <nvpair id="status-3-probe_complete" name="probe_complete" value="true"/>
+ </instance_attributes>
+ </transient_attributes>
+ <lrm id="3">
+ <lrm_resources>
+ <lrm_resource id="FAKECLONE2" type="Dummy" class="ocf" provider="heartbeat">
+ <lrm_rsc_op id="FAKECLONE2_last_0" operation_key="FAKECLONE2_start_0" operation="start" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.10" transition-key="17:30:0:b489d50c-a7db-4605-8519-a2d5e1452e29" transition-magic="0:0;17:30:0:b489d50c-a7db-4605-8519-a2d5e1452e29" on_node="c7auto3" call-id="29" rc-code="0" op-status="0" interval="0" last-run="1437083710" last-rc-change="1437083710" exec-time="6" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" fake " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ <lrm_rsc_op id="FAKECLONE2_monitor_10000" operation_key="FAKECLONE2_monitor_10000" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.10" transition-key="18:30:0:b489d50c-a7db-4605-8519-a2d5e1452e29" transition-magic="0:0;18:30:0:b489d50c-a7db-4605-8519-a2d5e1452e29" on_node="c7auto3" call-id="30" rc-code="0" op-status="0" interval="10000" last-rc-change="1437083710" exec-time="6" queue-time="0" op-digest="4811cef7f7f94e3a35a70be7916cb2fd"/>
+ </lrm_resource>
+ <lrm_resource id="shooter" type="fence_phd_kvm" class="stonith">
+ <lrm_rsc_op id="shooter_last_0" operation_key="shooter_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.10" transition-key="8:0:7:b489d50c-a7db-4605-8519-a2d5e1452e29" transition-magic="0:7;8:0:7:b489d50c-a7db-4605-8519-a2d5e1452e29" on_node="c7auto3" call-id="5" rc-code="7" op-status="0" interval="0" last-run="1437073988" last-rc-change="1437073988" exec-time="963" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ </lrm_resource>
+ <lrm_resource id="FAKECLONE" type="Dummy" class="ocf" provider="heartbeat">
+ <lrm_rsc_op id="FAKECLONE_last_0" operation_key="FAKECLONE_stop_0" operation="stop" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.10" transition-key="16:9:0:b489d50c-a7db-4605-8519-a2d5e1452e29" transition-magic="0:0;16:9:0:b489d50c-a7db-4605-8519-a2d5e1452e29" on_node="c7auto3" call-id="18" rc-code="0" op-status="0" interval="0" last-run="1437075473" last-rc-change="1437075473" exec-time="12" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ </lrm_resource>
+ </lrm_resources>
+ </lrm>
+ </node_state>
+ <node_state id="4" uname="c7auto4" in_ccm="true" crmd="online" crm-debug-origin="do_state_transition" join="member" expected="member">
+ <transient_attributes id="4">
+ <instance_attributes id="status-4">
+ <nvpair id="status-4-shutdown" name="shutdown" value="0"/>
+ <nvpair id="status-4-probe_complete" name="probe_complete" value="true"/>
+ </instance_attributes>
+ </transient_attributes>
+ <lrm id="4">
+ <lrm_resources>
+ <lrm_resource id="FAKECLONE2" type="Dummy" class="ocf" provider="heartbeat">
+ <lrm_rsc_op id="FAKECLONE2_last_0" operation_key="FAKECLONE2_start_0" operation="start" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.10" transition-key="22:26:0:b489d50c-a7db-4605-8519-a2d5e1452e29" transition-magic="0:0;22:26:0:b489d50c-a7db-4605-8519-a2d5e1452e29" on_node="c7auto4" call-id="29" rc-code="0" op-status="0" interval="0" last-run="1437083649" last-rc-change="1437083649" exec-time="7" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" fake " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ <lrm_rsc_op id="FAKECLONE2_monitor_10000" operation_key="FAKECLONE2_monitor_10000" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.10" transition-key="23:26:0:b489d50c-a7db-4605-8519-a2d5e1452e29" transition-magic="0:0;23:26:0:b489d50c-a7db-4605-8519-a2d5e1452e29" on_node="c7auto4" call-id="30" rc-code="0" op-status="0" interval="10000" last-rc-change="1437083649" exec-time="7" queue-time="0" op-digest="4811cef7f7f94e3a35a70be7916cb2fd"/>
+ </lrm_resource>
+ <lrm_resource id="shooter" type="fence_phd_kvm" class="stonith">
+ <lrm_rsc_op id="shooter_last_0" operation_key="shooter_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.10" transition-key="10:0:7:b489d50c-a7db-4605-8519-a2d5e1452e29" transition-magic="0:7;10:0:7:b489d50c-a7db-4605-8519-a2d5e1452e29" on_node="c7auto4" call-id="5" rc-code="7" op-status="0" interval="0" last-run="1437073988" last-rc-change="1437073988" exec-time="963" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ </lrm_resource>
+ <lrm_resource id="FAKECLONE" type="Dummy" class="ocf" provider="heartbeat">
+ <lrm_rsc_op id="FAKECLONE_last_0" operation_key="FAKECLONE_stop_0" operation="stop" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.10" transition-key="20:3:0:b489d50c-a7db-4605-8519-a2d5e1452e29" transition-magic="0:0;20:3:0:b489d50c-a7db-4605-8519-a2d5e1452e29" on_node="c7auto4" call-id="14" rc-code="0" op-status="0" interval="0" last-run="1437074856" last-rc-change="1437074856" exec-time="8" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ </lrm_resource>
+ </lrm_resources>
+ </lrm>
+ </node_state>
+ </status>
+</cib>
diff --git a/pengine/test10/cloned_stop_one.dot b/pengine/test10/cloned_stop_one.dot
new file mode 100644
index 0000000..d181135
--- /dev/null
+++ b/pengine/test10/cloned_stop_one.dot
@@ -0,0 +1,26 @@
+ digraph "g" {
+"FAKECLONE-clone_running_0" [ style=bold color="green" fontcolor="orange"]
+"FAKECLONE-clone_start_0" -> "FAKECLONE-clone_running_0" [ style = bold]
+"FAKECLONE-clone_start_0" -> "FAKECLONE_start_0 <none>" [ style = dashed]
+"FAKECLONE-clone_start_0" [ style=bold color="green" fontcolor="orange"]
+"FAKECLONE-clone_stop_0" -> "FAKECLONE-clone_stopped_0" [ style = bold]
+"FAKECLONE-clone_stop_0" -> "FAKECLONE_stop_0 c7auto3" [ style = bold]
+"FAKECLONE-clone_stop_0" [ style=bold color="green" fontcolor="orange"]
+"FAKECLONE-clone_stopped_0" -> "FAKECLONE-clone_start_0" [ style = bold]
+"FAKECLONE-clone_stopped_0" [ style=bold color="green" fontcolor="orange"]
+"FAKECLONE2-clone_stop_0" -> "FAKECLONE2-clone_stopped_0" [ style = bold]
+"FAKECLONE2-clone_stop_0" -> "FAKECLONE2_stop_0 c7auto3" [ style = bold]
+"FAKECLONE2-clone_stop_0" [ style=bold color="green" fontcolor="orange"]
+"FAKECLONE2-clone_stopped_0" -> "FAKECLONE-clone_stop_0" [ style = bold]
+"FAKECLONE2-clone_stopped_0" [ style=bold color="green" fontcolor="orange"]
+"FAKECLONE2_stop_0 c7auto3" -> "FAKECLONE2-clone_stopped_0" [ style = bold]
+"FAKECLONE2_stop_0 c7auto3" -> "all_stopped" [ style = bold]
+"FAKECLONE2_stop_0 c7auto3" [ style=bold color="green" fontcolor="black"]
+"FAKECLONE_start_0 <none>" -> "FAKECLONE-clone_running_0" [ style = dashed]
+"FAKECLONE_start_0 <none>" [ style=dashed color="red" fontcolor="black"]
+"FAKECLONE_stop_0 c7auto3" -> "FAKECLONE-clone_stopped_0" [ style = bold]
+"FAKECLONE_stop_0 c7auto3" -> "FAKECLONE_start_0 <none>" [ style = dashed]
+"FAKECLONE_stop_0 c7auto3" -> "all_stopped" [ style = bold]
+"FAKECLONE_stop_0 c7auto3" [ style=bold color="green" fontcolor="black"]
+"all_stopped" [ style=bold color="green" fontcolor="orange"]
+}
diff --git a/pengine/test10/cloned_stop_one.exp b/pengine/test10/cloned_stop_one.exp
new file mode 100644
index 0000000..9613d6f
--- /dev/null
+++ b/pengine/test10/cloned_stop_one.exp
@@ -0,0 +1,117 @@
+<transition_graph cluster-delay="60s" stonith-timeout="60s" failed-stop-offset="INFINITY" failed-start-offset="INFINITY" transition_id="0">
+ <synapse id="0">
+ <action_set>
+ <rsc_op id="20" operation="stop" operation_key="FAKECLONE_stop_0" internal_operation_key="FAKECLONE:2_stop_0" on_node="c7auto3" on_node_uuid="3">
+ <primitive id="FAKECLONE" long-id="FAKECLONE:2" class="ocf" provider="heartbeat" type="Dummy"/>
+ <attributes CRM_meta_clone="2" CRM_meta_clone_max="4" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_name="stop" CRM_meta_notify="false" CRM_meta_timeout="20000" />
+ </rsc_op>
+ </action_set>
+ <inputs>
+ <trigger>
+ <pseudo_event id="23" operation="stop" operation_key="FAKECLONE-clone_stop_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="1" priority="1000000">
+ <action_set>
+ <pseudo_event id="24" operation="stopped" operation_key="FAKECLONE-clone_stopped_0">
+ <attributes CRM_meta_clone_max="4" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="false" CRM_meta_timeout="20000" />
+ </pseudo_event>
+ </action_set>
+ <inputs>
+ <trigger>
+ <rsc_op id="20" operation="stop" operation_key="FAKECLONE_stop_0" internal_operation_key="FAKECLONE:2_stop_0" on_node="c7auto3" on_node_uuid="3"/>
+ </trigger>
+ <trigger>
+ <pseudo_event id="23" operation="stop" operation_key="FAKECLONE-clone_stop_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="2">
+ <action_set>
+ <pseudo_event id="23" operation="stop" operation_key="FAKECLONE-clone_stop_0">
+ <attributes CRM_meta_clone_max="4" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="false" CRM_meta_timeout="20000" />
+ </pseudo_event>
+ </action_set>
+ <inputs>
+ <trigger>
+ <pseudo_event id="31" operation="stopped" operation_key="FAKECLONE2-clone_stopped_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="3" priority="1000000">
+ <action_set>
+ <pseudo_event id="22" operation="running" operation_key="FAKECLONE-clone_running_0">
+ <attributes CRM_meta_clone_max="4" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="false" CRM_meta_timeout="20000" />
+ </pseudo_event>
+ </action_set>
+ <inputs>
+ <trigger>
+ <pseudo_event id="21" operation="start" operation_key="FAKECLONE-clone_start_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="4">
+ <action_set>
+ <pseudo_event id="21" operation="start" operation_key="FAKECLONE-clone_start_0">
+ <attributes CRM_meta_clone_max="4" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="false" CRM_meta_timeout="20000" />
+ </pseudo_event>
+ </action_set>
+ <inputs>
+ <trigger>
+ <pseudo_event id="24" operation="stopped" operation_key="FAKECLONE-clone_stopped_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="5">
+ <action_set>
+ <rsc_op id="25" operation="stop" operation_key="FAKECLONE2_stop_0" internal_operation_key="FAKECLONE2:0_stop_0" on_node="c7auto3" on_node_uuid="3">
+ <primitive id="FAKECLONE2" long-id="FAKECLONE2:0" class="ocf" provider="heartbeat" type="Dummy"/>
+ <attributes CRM_meta_clone="0" CRM_meta_clone_max="4" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_name="stop" CRM_meta_notify="false" CRM_meta_timeout="20000" />
+ </rsc_op>
+ </action_set>
+ <inputs>
+ <trigger>
+ <pseudo_event id="30" operation="stop" operation_key="FAKECLONE2-clone_stop_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="6" priority="1000000">
+ <action_set>
+ <pseudo_event id="31" operation="stopped" operation_key="FAKECLONE2-clone_stopped_0">
+ <attributes CRM_meta_clone_max="4" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="false" CRM_meta_timeout="20000" />
+ </pseudo_event>
+ </action_set>
+ <inputs>
+ <trigger>
+ <rsc_op id="25" operation="stop" operation_key="FAKECLONE2_stop_0" internal_operation_key="FAKECLONE2:0_stop_0" on_node="c7auto3" on_node_uuid="3"/>
+ </trigger>
+ <trigger>
+ <pseudo_event id="30" operation="stop" operation_key="FAKECLONE2-clone_stop_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="7">
+ <action_set>
+ <pseudo_event id="30" operation="stop" operation_key="FAKECLONE2-clone_stop_0">
+ <attributes CRM_meta_clone_max="4" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="false" CRM_meta_timeout="20000" />
+ </pseudo_event>
+ </action_set>
+ <inputs/>
+ </synapse>
+ <synapse id="8">
+ <action_set>
+ <pseudo_event id="8" operation="all_stopped" operation_key="all_stopped">
+ <attributes />
+ </pseudo_event>
+ </action_set>
+ <inputs>
+ <trigger>
+ <rsc_op id="20" operation="stop" operation_key="FAKECLONE_stop_0" internal_operation_key="FAKECLONE:2_stop_0" on_node="c7auto3" on_node_uuid="3"/>
+ </trigger>
+ <trigger>
+ <rsc_op id="25" operation="stop" operation_key="FAKECLONE2_stop_0" internal_operation_key="FAKECLONE2:0_stop_0" on_node="c7auto3" on_node_uuid="3"/>
+ </trigger>
+ </inputs>
+ </synapse>
+</transition_graph>
diff --git a/pengine/test10/cloned_stop_one.scores b/pengine/test10/cloned_stop_one.scores
new file mode 100644
index 0000000..6d66638
--- /dev/null
+++ b/pengine/test10/cloned_stop_one.scores
@@ -0,0 +1,77 @@
+Allocation scores:
+clone_color: FAKECLONE-clone allocation score on c7auto1: 0
+clone_color: FAKECLONE-clone allocation score on c7auto2: 0
+clone_color: FAKECLONE-clone allocation score on c7auto3: 0
+clone_color: FAKECLONE-clone allocation score on c7auto4: -INFINITY
+clone_color: FAKECLONE2-clone allocation score on c7auto1: -INFINITY
+clone_color: FAKECLONE2-clone allocation score on c7auto2: -INFINITY
+clone_color: FAKECLONE2-clone allocation score on c7auto3: 0
+clone_color: FAKECLONE2-clone allocation score on c7auto4: 0
+clone_color: FAKECLONE2:0 allocation score on c7auto1: -INFINITY
+clone_color: FAKECLONE2:0 allocation score on c7auto2: -INFINITY
+clone_color: FAKECLONE2:0 allocation score on c7auto3: 1
+clone_color: FAKECLONE2:0 allocation score on c7auto4: 0
+clone_color: FAKECLONE2:1 allocation score on c7auto1: -INFINITY
+clone_color: FAKECLONE2:1 allocation score on c7auto2: -INFINITY
+clone_color: FAKECLONE2:1 allocation score on c7auto3: 0
+clone_color: FAKECLONE2:1 allocation score on c7auto4: 1
+clone_color: FAKECLONE2:2 allocation score on c7auto1: -INFINITY
+clone_color: FAKECLONE2:2 allocation score on c7auto2: -INFINITY
+clone_color: FAKECLONE2:2 allocation score on c7auto3: 0
+clone_color: FAKECLONE2:2 allocation score on c7auto4: 0
+clone_color: FAKECLONE2:3 allocation score on c7auto1: -INFINITY
+clone_color: FAKECLONE2:3 allocation score on c7auto2: -INFINITY
+clone_color: FAKECLONE2:3 allocation score on c7auto3: 0
+clone_color: FAKECLONE2:3 allocation score on c7auto4: 0
+clone_color: FAKECLONE:0 allocation score on c7auto1: 1
+clone_color: FAKECLONE:0 allocation score on c7auto2: 0
+clone_color: FAKECLONE:0 allocation score on c7auto3: 0
+clone_color: FAKECLONE:0 allocation score on c7auto4: -INFINITY
+clone_color: FAKECLONE:1 allocation score on c7auto1: 0
+clone_color: FAKECLONE:1 allocation score on c7auto2: 1
+clone_color: FAKECLONE:1 allocation score on c7auto3: 0
+clone_color: FAKECLONE:1 allocation score on c7auto4: -INFINITY
+clone_color: FAKECLONE:2 allocation score on c7auto1: 0
+clone_color: FAKECLONE:2 allocation score on c7auto2: 0
+clone_color: FAKECLONE:2 allocation score on c7auto3: 1
+clone_color: FAKECLONE:2 allocation score on c7auto4: -INFINITY
+clone_color: FAKECLONE:3 allocation score on c7auto1: 0
+clone_color: FAKECLONE:3 allocation score on c7auto2: 0
+clone_color: FAKECLONE:3 allocation score on c7auto3: 0
+clone_color: FAKECLONE:3 allocation score on c7auto4: -INFINITY
+native_color: FAKECLONE2:0 allocation score on c7auto1: -INFINITY
+native_color: FAKECLONE2:0 allocation score on c7auto2: -INFINITY
+native_color: FAKECLONE2:0 allocation score on c7auto3: -INFINITY
+native_color: FAKECLONE2:0 allocation score on c7auto4: -INFINITY
+native_color: FAKECLONE2:1 allocation score on c7auto1: -INFINITY
+native_color: FAKECLONE2:1 allocation score on c7auto2: -INFINITY
+native_color: FAKECLONE2:1 allocation score on c7auto3: -INFINITY
+native_color: FAKECLONE2:1 allocation score on c7auto4: 1
+native_color: FAKECLONE2:2 allocation score on c7auto1: -INFINITY
+native_color: FAKECLONE2:2 allocation score on c7auto2: -INFINITY
+native_color: FAKECLONE2:2 allocation score on c7auto3: -INFINITY
+native_color: FAKECLONE2:2 allocation score on c7auto4: -INFINITY
+native_color: FAKECLONE2:3 allocation score on c7auto1: -INFINITY
+native_color: FAKECLONE2:3 allocation score on c7auto2: -INFINITY
+native_color: FAKECLONE2:3 allocation score on c7auto3: -INFINITY
+native_color: FAKECLONE2:3 allocation score on c7auto4: -INFINITY
+native_color: FAKECLONE:0 allocation score on c7auto1: 1
+native_color: FAKECLONE:0 allocation score on c7auto2: -INFINITY
+native_color: FAKECLONE:0 allocation score on c7auto3: -INFINITY
+native_color: FAKECLONE:0 allocation score on c7auto4: -INFINITY
+native_color: FAKECLONE:1 allocation score on c7auto1: 0
+native_color: FAKECLONE:1 allocation score on c7auto2: 1
+native_color: FAKECLONE:1 allocation score on c7auto3: -INFINITY
+native_color: FAKECLONE:1 allocation score on c7auto4: -INFINITY
+native_color: FAKECLONE:2 allocation score on c7auto1: -INFINITY
+native_color: FAKECLONE:2 allocation score on c7auto2: -INFINITY
+native_color: FAKECLONE:2 allocation score on c7auto3: -INFINITY
+native_color: FAKECLONE:2 allocation score on c7auto4: -INFINITY
+native_color: FAKECLONE:3 allocation score on c7auto1: -INFINITY
+native_color: FAKECLONE:3 allocation score on c7auto2: -INFINITY
+native_color: FAKECLONE:3 allocation score on c7auto3: -INFINITY
+native_color: FAKECLONE:3 allocation score on c7auto4: -INFINITY
+native_color: shooter allocation score on c7auto1: 0
+native_color: shooter allocation score on c7auto2: 0
+native_color: shooter allocation score on c7auto3: 0
+native_color: shooter allocation score on c7auto4: 0
diff --git a/pengine/test10/cloned_stop_one.summary b/pengine/test10/cloned_stop_one.summary
new file mode 100644
index 0000000..1a952a2
--- /dev/null
+++ b/pengine/test10/cloned_stop_one.summary
@@ -0,0 +1,40 @@
+
+Current cluster status:
+Node c7auto3 (3): standby
+Online: [ c7auto1 c7auto2 c7auto4 ]
+
+ shooter (stonith:fence_phd_kvm): Started c7auto1
+ Clone Set: FAKECLONE-clone [FAKECLONE]
+ Started: [ c7auto1 c7auto2 c7auto3 ]
+ Stopped: [ c7auto4 ]
+ Clone Set: FAKECLONE2-clone [FAKECLONE2]
+ Started: [ c7auto3 c7auto4 ]
+ Stopped: [ c7auto1 c7auto2 ]
+
+Transition Summary:
+ * Stop FAKECLONE:2 (c7auto3)
+ * Stop FAKECLONE2:0 (c7auto3)
+
+Executing cluster transition:
+ * Pseudo action: FAKECLONE2-clone_stop_0
+ * Resource action: FAKECLONE2 stop on c7auto3
+ * Pseudo action: FAKECLONE2-clone_stopped_0
+ * Pseudo action: FAKECLONE-clone_stop_0
+ * Resource action: FAKECLONE stop on c7auto3
+ * Pseudo action: FAKECLONE-clone_stopped_0
+ * Pseudo action: FAKECLONE-clone_start_0
+ * Pseudo action: all_stopped
+ * Pseudo action: FAKECLONE-clone_running_0
+
+Revised cluster status:
+Node c7auto3 (3): standby
+Online: [ c7auto1 c7auto2 c7auto4 ]
+
+ shooter (stonith:fence_phd_kvm): Started c7auto1
+ Clone Set: FAKECLONE-clone [FAKECLONE]
+ Started: [ c7auto1 c7auto2 ]
+ Stopped: [ c7auto3 c7auto4 ]
+ Clone Set: FAKECLONE2-clone [FAKECLONE2]
+ Started: [ c7auto4 ]
+ Stopped: [ c7auto1 c7auto2 c7auto3 ]
+
diff --git a/pengine/test10/cloned_stop_one.xml b/pengine/test10/cloned_stop_one.xml
new file mode 100644
index 0000000..2e2fdfd
--- /dev/null
+++ b/pengine/test10/cloned_stop_one.xml
@@ -0,0 +1,153 @@
+<cib crm_feature_set="3.0.10" validate-with="pacemaker-2.3" epoch="29" num_updates="0" admin_epoch="0" cib-last-written="Thu Jul 16 18:03:09 2015" update-origin="c7auto1" update-client="cibadmin" update-user="root" have-quorum="1" dc-uuid="2">
+ <configuration>
+ <crm_config>
+ <cluster_property_set id="cib-bootstrap-options">
+ <nvpair id="cib-bootstrap-options-have-watchdog" name="have-watchdog" value="false"/>
+ <nvpair id="cib-bootstrap-options-dc-version" name="dc-version" value="1.1.12-ef86b49"/>
+ <nvpair id="cib-bootstrap-options-cluster-infrastructure" name="cluster-infrastructure" value="corosync"/>
+ <nvpair id="cib-bootstrap-options-cluster-name" name="cluster-name" value="phd"/>
+ </cluster_property_set>
+ </crm_config>
+ <nodes>
+ <node id="1" uname="c7auto1"/>
+ <node id="2" uname="c7auto2"/>
+ <node id="3" uname="c7auto3">
+ <instance_attributes id="nodes-3">
+ <nvpair id="nodes-3-standby" name="standby" value="on"/>
+ </instance_attributes>
+ </node>
+ <node id="4" uname="c7auto4"/>
+ </nodes>
+ <resources>
+ <primitive class="stonith" id="shooter" type="fence_phd_kvm">
+ <instance_attributes id="shooter-instance_attributes"/>
+ <operations>
+ <op id="shooter-monitor-interval-60s" interval="60s" name="monitor"/>
+ </operations>
+ </primitive>
+ <clone id="FAKECLONE-clone">
+ <primitive class="ocf" id="FAKECLONE" provider="heartbeat" type="Dummy">
+ <instance_attributes id="FAKECLONE-instance_attributes"/>
+ <operations>
+ <op id="FAKECLONE-start-timeout-20" interval="0s" name="start" timeout="20"/>
+ <op id="FAKECLONE-stop-timeout-20" interval="0s" name="stop" timeout="20"/>
+ <op id="FAKECLONE-monitor-interval-10" interval="10" name="monitor" timeout="20"/>
+ </operations>
+ </primitive>
+ <meta_attributes id="FAKECLONE-clone-meta">
+ <nvpair id="FAKECLONE-clone-min" name="clone-min" value="2"/>
+ </meta_attributes>
+ <meta_attributes id="FAKECLONE-clone-meta_attributes"/>
+ </clone>
+ <clone id="FAKECLONE2-clone">
+ <primitive class="ocf" id="FAKECLONE2" provider="heartbeat" type="Dummy">
+ <instance_attributes id="FAKECLONE2-instance_attributes"/>
+ <operations>
+ <op id="FAKECLONE2-start-timeout-20" interval="0s" name="start" timeout="20"/>
+ <op id="FAKECLONE2-stop-timeout-20" interval="0s" name="stop" timeout="20"/>
+ <op id="FAKECLONE2-monitor-interval-10" interval="10" name="monitor" timeout="20"/>
+ </operations>
+ </primitive>
+ <meta_attributes id="FAKECLONE2-clone-meta"/>
+ </clone>
+ </resources>
+ <constraints>
+ <rsc_location id="location-FAKECLONE-clone-c7auto4--INFINITY" node="c7auto4" rsc="FAKECLONE-clone" score="-INFINITY"/>
+ <rsc_location id="location-FAKECLONE2-clone-c7auto1--INFINITY" node="c7auto1" rsc="FAKECLONE2-clone" score="-INFINITY"/>
+ <rsc_location id="location-FAKECLONE2-clone-c7auto2--INFINITY" node="c7auto2" rsc="FAKECLONE2-clone" score="-INFINITY"/>
+ <rsc_order first="FAKECLONE-clone" first-action="start" id="order-FAKECLONE-clone-FAKECLONE2-clone-mandatory" then="FAKECLONE2-clone" then-action="start"/>
+ </constraints>
+ </configuration>
+ <status>
+ <node_state id="1" uname="c7auto1" in_ccm="true" crmd="online" crm-debug-origin="do_update_resource" join="member" expected="member">
+ <transient_attributes id="1">
+ <instance_attributes id="status-1">
+ <nvpair id="status-1-shutdown" name="shutdown" value="0"/>
+ <nvpair id="status-1-probe_complete" name="probe_complete" value="true"/>
+ </instance_attributes>
+ </transient_attributes>
+ <lrm id="1">
+ <lrm_resources>
+ <lrm_resource id="FAKECLONE2" type="Dummy" class="ocf" provider="heartbeat">
+ <lrm_rsc_op id="FAKECLONE2_last_0" operation_key="FAKECLONE2_stop_0" operation="stop" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.10" transition-key="17:28:0:b489d50c-a7db-4605-8519-a2d5e1452e29" transition-magic="0:0;17:28:0:b489d50c-a7db-4605-8519-a2d5e1452e29" on_node="c7auto1" call-id="30" rc-code="0" op-status="0" interval="0" last-run="1437083676" last-rc-change="1437083676" exec-time="12" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ </lrm_resource>
+ <lrm_resource id="shooter" type="fence_phd_kvm" class="stonith">
+ <lrm_rsc_op id="shooter_last_0" operation_key="shooter_start_0" operation="start" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.10" transition-key="11:0:0:b489d50c-a7db-4605-8519-a2d5e1452e29" transition-magic="0:0;11:0:0:b489d50c-a7db-4605-8519-a2d5e1452e29" on_node="c7auto1" call-id="6" rc-code="0" op-status="0" interval="0" last-run="1437073991" last-rc-change="1437073991" exec-time="9" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ <lrm_rsc_op id="shooter_monitor_60000" operation_key="shooter_monitor_60000" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.10" transition-key="9:1:0:b489d50c-a7db-4605-8519-a2d5e1452e29" transition-magic="0:0;9:1:0:b489d50c-a7db-4605-8519-a2d5e1452e29" on_node="c7auto1" call-id="7" rc-code="0" op-status="0" interval="60000" last-rc-change="1437073992" exec-time="7" queue-time="0" op-digest="4811cef7f7f94e3a35a70be7916cb2fd"/>
+ </lrm_resource>
+ <lrm_resource id="FAKECLONE" type="Dummy" class="ocf" provider="heartbeat">
+ <lrm_rsc_op id="FAKECLONE_last_0" operation_key="FAKECLONE_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="16:33:0:b489d50c-a7db-4605-8519-a2d5e1452e29" transition-magic="0:0;16:33:0:b489d50c-a7db-4605-8519-a2d5e1452e29" on_node="c7auto1" call-id="31" rc-code="0" op-status="0" interval="0" last-run="1437084158" last-rc-change="1437084158" exec-time="9" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" fake " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ <lrm_rsc_op id="FAKECLONE_monitor_10000" operation_key="FAKECLONE_monitor_10000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="17:33:0:b489d50c-a7db-4605-8519-a2d5e1452e29" transition-magic="0:0;17:33:0:b489d50c-a7db-4605-8519-a2d5e1452e29" on_node="c7auto1" call-id="32" rc-code="0" op-status="0" interval="10000" last-rc-change="1437084158" exec-time="7" queue-time="0" op-digest="4811cef7f7f94e3a35a70be7916cb2fd"/>
+ </lrm_resource>
+ </lrm_resources>
+ </lrm>
+ </node_state>
+ <node_state id="2" uname="c7auto2" in_ccm="true" crmd="online" crm-debug-origin="do_update_resource" join="member" expected="member">
+ <transient_attributes id="2">
+ <instance_attributes id="status-2">
+ <nvpair id="status-2-shutdown" name="shutdown" value="0"/>
+ <nvpair id="status-2-probe_complete" name="probe_complete" value="true"/>
+ </instance_attributes>
+ </transient_attributes>
+ <lrm id="2">
+ <lrm_resources>
+ <lrm_resource id="FAKECLONE2" type="Dummy" class="ocf" provider="heartbeat">
+ <lrm_rsc_op id="FAKECLONE2_last_0" operation_key="FAKECLONE2_stop_0" operation="stop" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.10" transition-key="16:29:0:b489d50c-a7db-4605-8519-a2d5e1452e29" transition-magic="0:0;16:29:0:b489d50c-a7db-4605-8519-a2d5e1452e29" on_node="c7auto2" call-id="32" rc-code="0" op-status="0" interval="0" last-run="1437083678" last-rc-change="1437083678" exec-time="10" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ </lrm_resource>
+ <lrm_resource id="shooter" type="fence_phd_kvm" class="stonith">
+ <lrm_rsc_op id="shooter_last_0" operation_key="shooter_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.10" transition-key="6:0:7:b489d50c-a7db-4605-8519-a2d5e1452e29" transition-magic="0:7;6:0:7:b489d50c-a7db-4605-8519-a2d5e1452e29" on_node="c7auto2" call-id="5" rc-code="7" op-status="0" interval="0" last-run="1437073988" last-rc-change="1437073988" exec-time="964" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ </lrm_resource>
+ <lrm_resource id="FAKECLONE" type="Dummy" class="ocf" provider="heartbeat">
+ <lrm_rsc_op id="FAKECLONE_last_0" operation_key="FAKECLONE_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="12:33:0:b489d50c-a7db-4605-8519-a2d5e1452e29" transition-magic="0:0;12:33:0:b489d50c-a7db-4605-8519-a2d5e1452e29" on_node="c7auto2" call-id="33" rc-code="0" op-status="0" interval="0" last-run="1437084158" last-rc-change="1437084158" exec-time="11" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" fake " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ <lrm_rsc_op id="FAKECLONE_monitor_10000" operation_key="FAKECLONE_monitor_10000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="13:33:0:b489d50c-a7db-4605-8519-a2d5e1452e29" transition-magic="0:0;13:33:0:b489d50c-a7db-4605-8519-a2d5e1452e29" on_node="c7auto2" call-id="34" rc-code="0" op-status="0" interval="10000" last-rc-change="1437084158" exec-time="12" queue-time="0" op-digest="4811cef7f7f94e3a35a70be7916cb2fd"/>
+ </lrm_resource>
+ </lrm_resources>
+ </lrm>
+ </node_state>
+ <node_state id="3" uname="c7auto3" in_ccm="true" crmd="online" crm-debug-origin="do_update_resource" join="member" expected="member">
+ <transient_attributes id="3">
+ <instance_attributes id="status-3">
+ <nvpair id="status-3-shutdown" name="shutdown" value="0"/>
+ <nvpair id="status-3-probe_complete" name="probe_complete" value="true"/>
+ </instance_attributes>
+ </transient_attributes>
+ <lrm id="3">
+ <lrm_resources>
+ <lrm_resource id="FAKECLONE2" type="Dummy" class="ocf" provider="heartbeat">
+ <lrm_rsc_op id="FAKECLONE2_last_0" operation_key="FAKECLONE2_start_0" operation="start" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.10" transition-key="17:30:0:b489d50c-a7db-4605-8519-a2d5e1452e29" transition-magic="0:0;17:30:0:b489d50c-a7db-4605-8519-a2d5e1452e29" on_node="c7auto3" call-id="29" rc-code="0" op-status="0" interval="0" last-run="1437083710" last-rc-change="1437083710" exec-time="6" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" fake " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ <lrm_rsc_op id="FAKECLONE2_monitor_10000" operation_key="FAKECLONE2_monitor_10000" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.10" transition-key="18:30:0:b489d50c-a7db-4605-8519-a2d5e1452e29" transition-magic="0:0;18:30:0:b489d50c-a7db-4605-8519-a2d5e1452e29" on_node="c7auto3" call-id="30" rc-code="0" op-status="0" interval="10000" last-rc-change="1437083710" exec-time="6" queue-time="0" op-digest="4811cef7f7f94e3a35a70be7916cb2fd"/>
+ </lrm_resource>
+ <lrm_resource id="shooter" type="fence_phd_kvm" class="stonith">
+ <lrm_rsc_op id="shooter_last_0" operation_key="shooter_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.10" transition-key="8:0:7:b489d50c-a7db-4605-8519-a2d5e1452e29" transition-magic="0:7;8:0:7:b489d50c-a7db-4605-8519-a2d5e1452e29" on_node="c7auto3" call-id="5" rc-code="7" op-status="0" interval="0" last-run="1437073988" last-rc-change="1437073988" exec-time="963" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ </lrm_resource>
+ <lrm_resource id="FAKECLONE" type="Dummy" class="ocf" provider="heartbeat">
+ <lrm_rsc_op id="FAKECLONE_last_0" operation_key="FAKECLONE_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="14:33:0:b489d50c-a7db-4605-8519-a2d5e1452e29" transition-magic="0:0;14:33:0:b489d50c-a7db-4605-8519-a2d5e1452e29" on_node="c7auto3" call-id="31" rc-code="0" op-status="0" interval="0" last-run="1437084157" last-rc-change="1437084157" exec-time="9" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" fake " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ <lrm_rsc_op id="FAKECLONE_monitor_10000" operation_key="FAKECLONE_monitor_10000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="15:33:0:b489d50c-a7db-4605-8519-a2d5e1452e29" transition-magic="0:0;15:33:0:b489d50c-a7db-4605-8519-a2d5e1452e29" on_node="c7auto3" call-id="32" rc-code="0" op-status="0" interval="10000" last-rc-change="1437084157" exec-time="7" queue-time="0" op-digest="4811cef7f7f94e3a35a70be7916cb2fd"/>
+ </lrm_resource>
+ </lrm_resources>
+ </lrm>
+ </node_state>
+ <node_state id="4" uname="c7auto4" in_ccm="true" crmd="online" crm-debug-origin="do_state_transition" join="member" expected="member">
+ <transient_attributes id="4">
+ <instance_attributes id="status-4">
+ <nvpair id="status-4-shutdown" name="shutdown" value="0"/>
+ <nvpair id="status-4-probe_complete" name="probe_complete" value="true"/>
+ </instance_attributes>
+ </transient_attributes>
+ <lrm id="4">
+ <lrm_resources>
+ <lrm_resource id="FAKECLONE2" type="Dummy" class="ocf" provider="heartbeat">
+ <lrm_rsc_op id="FAKECLONE2_last_0" operation_key="FAKECLONE2_start_0" operation="start" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.10" transition-key="22:26:0:b489d50c-a7db-4605-8519-a2d5e1452e29" transition-magic="0:0;22:26:0:b489d50c-a7db-4605-8519-a2d5e1452e29" on_node="c7auto4" call-id="29" rc-code="0" op-status="0" interval="0" last-run="1437083649" last-rc-change="1437083649" exec-time="7" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" fake " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ <lrm_rsc_op id="FAKECLONE2_monitor_10000" operation_key="FAKECLONE2_monitor_10000" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.10" transition-key="23:26:0:b489d50c-a7db-4605-8519-a2d5e1452e29" transition-magic="0:0;23:26:0:b489d50c-a7db-4605-8519-a2d5e1452e29" on_node="c7auto4" call-id="30" rc-code="0" op-status="0" interval="10000" last-rc-change="1437083649" exec-time="7" queue-time="0" op-digest="4811cef7f7f94e3a35a70be7916cb2fd"/>
+ </lrm_resource>
+ <lrm_resource id="shooter" type="fence_phd_kvm" class="stonith">
+ <lrm_rsc_op id="shooter_last_0" operation_key="shooter_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.10" transition-key="10:0:7:b489d50c-a7db-4605-8519-a2d5e1452e29" transition-magic="0:7;10:0:7:b489d50c-a7db-4605-8519-a2d5e1452e29" on_node="c7auto4" call-id="5" rc-code="7" op-status="0" interval="0" last-run="1437073988" last-rc-change="1437073988" exec-time="963" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ </lrm_resource>
+ <lrm_resource id="FAKECLONE" type="Dummy" class="ocf" provider="heartbeat">
+ <lrm_rsc_op id="FAKECLONE_last_0" operation_key="FAKECLONE_stop_0" operation="stop" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.10" transition-key="20:3:0:b489d50c-a7db-4605-8519-a2d5e1452e29" transition-magic="0:0;20:3:0:b489d50c-a7db-4605-8519-a2d5e1452e29" on_node="c7auto4" call-id="14" rc-code="0" op-status="0" interval="0" last-run="1437074856" last-rc-change="1437074856" exec-time="8" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ </lrm_resource>
+ </lrm_resources>
+ </lrm>
+ </node_state>
+ </status>
+</cib>
diff --git a/pengine/test10/cloned_stop_two.dot b/pengine/test10/cloned_stop_two.dot
new file mode 100644
index 0000000..2c7fd3d
--- /dev/null
+++ b/pengine/test10/cloned_stop_two.dot
@@ -0,0 +1,45 @@
+ digraph "g" {
+"FAKECLONE-clone_running_0" [ style=bold color="green" fontcolor="orange"]
+"FAKECLONE-clone_start_0" -> "FAKECLONE-clone_running_0" [ style = bold]
+"FAKECLONE-clone_start_0" -> "FAKECLONE_start_0 <none>" [ style = dashed]
+"FAKECLONE-clone_start_0" [ style=bold color="green" fontcolor="orange"]
+"FAKECLONE-clone_stop_0" -> "FAKECLONE-clone_stopped_0" [ style = bold]
+"FAKECLONE-clone_stop_0" -> "FAKECLONE_stop_0 c7auto2" [ style = bold]
+"FAKECLONE-clone_stop_0" -> "FAKECLONE_stop_0 c7auto3" [ style = bold]
+"FAKECLONE-clone_stop_0" [ style=bold color="green" fontcolor="orange"]
+"FAKECLONE-clone_stopped_0" -> "FAKECLONE-clone_start_0" [ style = bold]
+"FAKECLONE-clone_stopped_0" [ style=bold color="green" fontcolor="orange"]
+"FAKECLONE2-clone_running_0" [ style=dashed color="red" fontcolor="orange"]
+"FAKECLONE2-clone_start_0" -> "FAKECLONE2-clone_running_0" [ style = dashed]
+"FAKECLONE2-clone_start_0" -> "FAKECLONE2_start_0 c7auto4" [ style = dashed]
+"FAKECLONE2-clone_start_0" [ style=dashed color="red" fontcolor="orange"]
+"FAKECLONE2-clone_stop_0" -> "FAKECLONE2-clone_stopped_0" [ style = bold]
+"FAKECLONE2-clone_stop_0" -> "FAKECLONE2_stop_0 c7auto3" [ style = bold]
+"FAKECLONE2-clone_stop_0" -> "FAKECLONE2_stop_0 c7auto4" [ style = bold]
+"FAKECLONE2-clone_stop_0" [ style=bold color="green" fontcolor="orange"]
+"FAKECLONE2-clone_stopped_0" -> "FAKECLONE-clone_stop_0" [ style = bold]
+"FAKECLONE2-clone_stopped_0" -> "FAKECLONE2-clone_start_0" [ style = dashed]
+"FAKECLONE2-clone_stopped_0" [ style=bold color="green" fontcolor="orange"]
+"FAKECLONE2_monitor_10000 c7auto4" [ style=dashed color="red" fontcolor="black"]
+"FAKECLONE2_start_0 c7auto4" -> "FAKECLONE2-clone_running_0" [ style = dashed]
+"FAKECLONE2_start_0 c7auto4" -> "FAKECLONE2_monitor_10000 c7auto4" [ style = dashed]
+"FAKECLONE2_start_0 c7auto4" [ style=dashed color="red" fontcolor="black"]
+"FAKECLONE2_stop_0 c7auto3" -> "FAKECLONE2-clone_stopped_0" [ style = bold]
+"FAKECLONE2_stop_0 c7auto3" -> "all_stopped" [ style = bold]
+"FAKECLONE2_stop_0 c7auto3" [ style=bold color="green" fontcolor="black"]
+"FAKECLONE2_stop_0 c7auto4" -> "FAKECLONE2-clone_stopped_0" [ style = bold]
+"FAKECLONE2_stop_0 c7auto4" -> "FAKECLONE2_start_0 c7auto4" [ style = dashed]
+"FAKECLONE2_stop_0 c7auto4" -> "all_stopped" [ style = bold]
+"FAKECLONE2_stop_0 c7auto4" [ style=bold color="green" fontcolor="black"]
+"FAKECLONE_start_0 <none>" -> "FAKECLONE-clone_running_0" [ style = dashed]
+"FAKECLONE_start_0 <none>" [ style=dashed color="red" fontcolor="black"]
+"FAKECLONE_stop_0 c7auto2" -> "FAKECLONE-clone_stopped_0" [ style = bold]
+"FAKECLONE_stop_0 c7auto2" -> "FAKECLONE_start_0 <none>" [ style = dashed]
+"FAKECLONE_stop_0 c7auto2" -> "all_stopped" [ style = bold]
+"FAKECLONE_stop_0 c7auto2" [ style=bold color="green" fontcolor="black"]
+"FAKECLONE_stop_0 c7auto3" -> "FAKECLONE-clone_stopped_0" [ style = bold]
+"FAKECLONE_stop_0 c7auto3" -> "FAKECLONE_start_0 <none>" [ style = dashed]
+"FAKECLONE_stop_0 c7auto3" -> "all_stopped" [ style = bold]
+"FAKECLONE_stop_0 c7auto3" [ style=bold color="green" fontcolor="black"]
+"all_stopped" [ style=bold color="green" fontcolor="orange"]
+}
diff --git a/pengine/test10/cloned_stop_two.exp b/pengine/test10/cloned_stop_two.exp
new file mode 100644
index 0000000..4aa0e58
--- /dev/null
+++ b/pengine/test10/cloned_stop_two.exp
@@ -0,0 +1,155 @@
+<transition_graph cluster-delay="60s" stonith-timeout="60s" failed-stop-offset="INFINITY" failed-start-offset="INFINITY" transition_id="0">
+ <synapse id="0">
+ <action_set>
+ <rsc_op id="18" operation="stop" operation_key="FAKECLONE_stop_0" internal_operation_key="FAKECLONE:1_stop_0" on_node="c7auto2" on_node_uuid="2">
+ <primitive id="FAKECLONE" long-id="FAKECLONE:1" class="ocf" provider="heartbeat" type="Dummy"/>
+ <attributes CRM_meta_clone="1" CRM_meta_clone_max="4" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_name="stop" CRM_meta_notify="false" CRM_meta_timeout="20000" />
+ </rsc_op>
+ </action_set>
+ <inputs>
+ <trigger>
+ <pseudo_event id="22" operation="stop" operation_key="FAKECLONE-clone_stop_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="1">
+ <action_set>
+ <rsc_op id="19" operation="stop" operation_key="FAKECLONE_stop_0" internal_operation_key="FAKECLONE:2_stop_0" on_node="c7auto3" on_node_uuid="3">
+ <primitive id="FAKECLONE" long-id="FAKECLONE:2" class="ocf" provider="heartbeat" type="Dummy"/>
+ <attributes CRM_meta_clone="2" CRM_meta_clone_max="4" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_name="stop" CRM_meta_notify="false" CRM_meta_timeout="20000" />
+ </rsc_op>
+ </action_set>
+ <inputs>
+ <trigger>
+ <pseudo_event id="22" operation="stop" operation_key="FAKECLONE-clone_stop_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="2" priority="1000000">
+ <action_set>
+ <pseudo_event id="23" operation="stopped" operation_key="FAKECLONE-clone_stopped_0">
+ <attributes CRM_meta_clone_max="4" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="false" CRM_meta_timeout="20000" />
+ </pseudo_event>
+ </action_set>
+ <inputs>
+ <trigger>
+ <rsc_op id="18" operation="stop" operation_key="FAKECLONE_stop_0" internal_operation_key="FAKECLONE:1_stop_0" on_node="c7auto2" on_node_uuid="2"/>
+ </trigger>
+ <trigger>
+ <rsc_op id="19" operation="stop" operation_key="FAKECLONE_stop_0" internal_operation_key="FAKECLONE:2_stop_0" on_node="c7auto3" on_node_uuid="3"/>
+ </trigger>
+ <trigger>
+ <pseudo_event id="22" operation="stop" operation_key="FAKECLONE-clone_stop_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="3">
+ <action_set>
+ <pseudo_event id="22" operation="stop" operation_key="FAKECLONE-clone_stop_0">
+ <attributes CRM_meta_clone_max="4" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="false" CRM_meta_timeout="20000" />
+ </pseudo_event>
+ </action_set>
+ <inputs>
+ <trigger>
+ <pseudo_event id="30" operation="stopped" operation_key="FAKECLONE2-clone_stopped_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="4" priority="1000000">
+ <action_set>
+ <pseudo_event id="21" operation="running" operation_key="FAKECLONE-clone_running_0">
+ <attributes CRM_meta_clone_max="4" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="false" CRM_meta_timeout="20000" />
+ </pseudo_event>
+ </action_set>
+ <inputs>
+ <trigger>
+ <pseudo_event id="20" operation="start" operation_key="FAKECLONE-clone_start_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="5">
+ <action_set>
+ <pseudo_event id="20" operation="start" operation_key="FAKECLONE-clone_start_0">
+ <attributes CRM_meta_clone_max="4" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="false" CRM_meta_timeout="20000" />
+ </pseudo_event>
+ </action_set>
+ <inputs>
+ <trigger>
+ <pseudo_event id="23" operation="stopped" operation_key="FAKECLONE-clone_stopped_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="6">
+ <action_set>
+ <rsc_op id="24" operation="stop" operation_key="FAKECLONE2_stop_0" internal_operation_key="FAKECLONE2:0_stop_0" on_node="c7auto3" on_node_uuid="3">
+ <primitive id="FAKECLONE2" long-id="FAKECLONE2:0" class="ocf" provider="heartbeat" type="Dummy"/>
+ <attributes CRM_meta_clone="0" CRM_meta_clone_max="4" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_name="stop" CRM_meta_notify="false" CRM_meta_timeout="20000" />
+ </rsc_op>
+ </action_set>
+ <inputs>
+ <trigger>
+ <pseudo_event id="29" operation="stop" operation_key="FAKECLONE2-clone_stop_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="7">
+ <action_set>
+ <rsc_op id="25" operation="stop" operation_key="FAKECLONE2_stop_0" internal_operation_key="FAKECLONE2:1_stop_0" on_node="c7auto4" on_node_uuid="4">
+ <primitive id="FAKECLONE2" long-id="FAKECLONE2:1" class="ocf" provider="heartbeat" type="Dummy"/>
+ <attributes CRM_meta_clone="1" CRM_meta_clone_max="4" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_name="stop" CRM_meta_notify="false" CRM_meta_timeout="20000" />
+ </rsc_op>
+ </action_set>
+ <inputs>
+ <trigger>
+ <pseudo_event id="29" operation="stop" operation_key="FAKECLONE2-clone_stop_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="8" priority="1000000">
+ <action_set>
+ <pseudo_event id="30" operation="stopped" operation_key="FAKECLONE2-clone_stopped_0">
+ <attributes CRM_meta_clone_max="4" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="false" CRM_meta_timeout="20000" />
+ </pseudo_event>
+ </action_set>
+ <inputs>
+ <trigger>
+ <rsc_op id="24" operation="stop" operation_key="FAKECLONE2_stop_0" internal_operation_key="FAKECLONE2:0_stop_0" on_node="c7auto3" on_node_uuid="3"/>
+ </trigger>
+ <trigger>
+ <rsc_op id="25" operation="stop" operation_key="FAKECLONE2_stop_0" internal_operation_key="FAKECLONE2:1_stop_0" on_node="c7auto4" on_node_uuid="4"/>
+ </trigger>
+ <trigger>
+ <pseudo_event id="29" operation="stop" operation_key="FAKECLONE2-clone_stop_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="9">
+ <action_set>
+ <pseudo_event id="29" operation="stop" operation_key="FAKECLONE2-clone_stop_0">
+ <attributes CRM_meta_clone_max="4" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="false" CRM_meta_timeout="20000" />
+ </pseudo_event>
+ </action_set>
+ <inputs/>
+ </synapse>
+ <synapse id="10">
+ <action_set>
+ <pseudo_event id="8" operation="all_stopped" operation_key="all_stopped">
+ <attributes />
+ </pseudo_event>
+ </action_set>
+ <inputs>
+ <trigger>
+ <rsc_op id="18" operation="stop" operation_key="FAKECLONE_stop_0" internal_operation_key="FAKECLONE:1_stop_0" on_node="c7auto2" on_node_uuid="2"/>
+ </trigger>
+ <trigger>
+ <rsc_op id="19" operation="stop" operation_key="FAKECLONE_stop_0" internal_operation_key="FAKECLONE:2_stop_0" on_node="c7auto3" on_node_uuid="3"/>
+ </trigger>
+ <trigger>
+ <rsc_op id="24" operation="stop" operation_key="FAKECLONE2_stop_0" internal_operation_key="FAKECLONE2:0_stop_0" on_node="c7auto3" on_node_uuid="3"/>
+ </trigger>
+ <trigger>
+ <rsc_op id="25" operation="stop" operation_key="FAKECLONE2_stop_0" internal_operation_key="FAKECLONE2:1_stop_0" on_node="c7auto4" on_node_uuid="4"/>
+ </trigger>
+ </inputs>
+ </synapse>
+</transition_graph>
diff --git a/pengine/test10/cloned_stop_two.scores b/pengine/test10/cloned_stop_two.scores
new file mode 100644
index 0000000..f6e9779
--- /dev/null
+++ b/pengine/test10/cloned_stop_two.scores
@@ -0,0 +1,77 @@
+Allocation scores:
+clone_color: FAKECLONE-clone allocation score on c7auto1: 0
+clone_color: FAKECLONE-clone allocation score on c7auto2: 0
+clone_color: FAKECLONE-clone allocation score on c7auto3: 0
+clone_color: FAKECLONE-clone allocation score on c7auto4: -INFINITY
+clone_color: FAKECLONE2-clone allocation score on c7auto1: -INFINITY
+clone_color: FAKECLONE2-clone allocation score on c7auto2: -INFINITY
+clone_color: FAKECLONE2-clone allocation score on c7auto3: 0
+clone_color: FAKECLONE2-clone allocation score on c7auto4: 0
+clone_color: FAKECLONE2:0 allocation score on c7auto1: -INFINITY
+clone_color: FAKECLONE2:0 allocation score on c7auto2: -INFINITY
+clone_color: FAKECLONE2:0 allocation score on c7auto3: 1
+clone_color: FAKECLONE2:0 allocation score on c7auto4: 0
+clone_color: FAKECLONE2:1 allocation score on c7auto1: -INFINITY
+clone_color: FAKECLONE2:1 allocation score on c7auto2: -INFINITY
+clone_color: FAKECLONE2:1 allocation score on c7auto3: 0
+clone_color: FAKECLONE2:1 allocation score on c7auto4: 1
+clone_color: FAKECLONE2:2 allocation score on c7auto1: -INFINITY
+clone_color: FAKECLONE2:2 allocation score on c7auto2: -INFINITY
+clone_color: FAKECLONE2:2 allocation score on c7auto3: 0
+clone_color: FAKECLONE2:2 allocation score on c7auto4: 0
+clone_color: FAKECLONE2:3 allocation score on c7auto1: -INFINITY
+clone_color: FAKECLONE2:3 allocation score on c7auto2: -INFINITY
+clone_color: FAKECLONE2:3 allocation score on c7auto3: 0
+clone_color: FAKECLONE2:3 allocation score on c7auto4: 0
+clone_color: FAKECLONE:0 allocation score on c7auto1: 1
+clone_color: FAKECLONE:0 allocation score on c7auto2: 0
+clone_color: FAKECLONE:0 allocation score on c7auto3: 0
+clone_color: FAKECLONE:0 allocation score on c7auto4: -INFINITY
+clone_color: FAKECLONE:1 allocation score on c7auto1: 0
+clone_color: FAKECLONE:1 allocation score on c7auto2: 1
+clone_color: FAKECLONE:1 allocation score on c7auto3: 0
+clone_color: FAKECLONE:1 allocation score on c7auto4: -INFINITY
+clone_color: FAKECLONE:2 allocation score on c7auto1: 0
+clone_color: FAKECLONE:2 allocation score on c7auto2: 0
+clone_color: FAKECLONE:2 allocation score on c7auto3: 1
+clone_color: FAKECLONE:2 allocation score on c7auto4: -INFINITY
+clone_color: FAKECLONE:3 allocation score on c7auto1: 0
+clone_color: FAKECLONE:3 allocation score on c7auto2: 0
+clone_color: FAKECLONE:3 allocation score on c7auto3: 0
+clone_color: FAKECLONE:3 allocation score on c7auto4: -INFINITY
+native_color: FAKECLONE2:0 allocation score on c7auto1: -INFINITY
+native_color: FAKECLONE2:0 allocation score on c7auto2: -INFINITY
+native_color: FAKECLONE2:0 allocation score on c7auto3: -INFINITY
+native_color: FAKECLONE2:0 allocation score on c7auto4: -INFINITY
+native_color: FAKECLONE2:1 allocation score on c7auto1: -INFINITY
+native_color: FAKECLONE2:1 allocation score on c7auto2: -INFINITY
+native_color: FAKECLONE2:1 allocation score on c7auto3: -INFINITY
+native_color: FAKECLONE2:1 allocation score on c7auto4: 1
+native_color: FAKECLONE2:2 allocation score on c7auto1: -INFINITY
+native_color: FAKECLONE2:2 allocation score on c7auto2: -INFINITY
+native_color: FAKECLONE2:2 allocation score on c7auto3: -INFINITY
+native_color: FAKECLONE2:2 allocation score on c7auto4: -INFINITY
+native_color: FAKECLONE2:3 allocation score on c7auto1: -INFINITY
+native_color: FAKECLONE2:3 allocation score on c7auto2: -INFINITY
+native_color: FAKECLONE2:3 allocation score on c7auto3: -INFINITY
+native_color: FAKECLONE2:3 allocation score on c7auto4: -INFINITY
+native_color: FAKECLONE:0 allocation score on c7auto1: 1
+native_color: FAKECLONE:0 allocation score on c7auto2: -INFINITY
+native_color: FAKECLONE:0 allocation score on c7auto3: -INFINITY
+native_color: FAKECLONE:0 allocation score on c7auto4: -INFINITY
+native_color: FAKECLONE:1 allocation score on c7auto1: -INFINITY
+native_color: FAKECLONE:1 allocation score on c7auto2: -INFINITY
+native_color: FAKECLONE:1 allocation score on c7auto3: -INFINITY
+native_color: FAKECLONE:1 allocation score on c7auto4: -INFINITY
+native_color: FAKECLONE:2 allocation score on c7auto1: -INFINITY
+native_color: FAKECLONE:2 allocation score on c7auto2: -INFINITY
+native_color: FAKECLONE:2 allocation score on c7auto3: -INFINITY
+native_color: FAKECLONE:2 allocation score on c7auto4: -INFINITY
+native_color: FAKECLONE:3 allocation score on c7auto1: -INFINITY
+native_color: FAKECLONE:3 allocation score on c7auto2: -INFINITY
+native_color: FAKECLONE:3 allocation score on c7auto3: -INFINITY
+native_color: FAKECLONE:3 allocation score on c7auto4: -INFINITY
+native_color: shooter allocation score on c7auto1: 0
+native_color: shooter allocation score on c7auto2: 0
+native_color: shooter allocation score on c7auto3: 0
+native_color: shooter allocation score on c7auto4: 0
diff --git a/pengine/test10/cloned_stop_two.summary b/pengine/test10/cloned_stop_two.summary
new file mode 100644
index 0000000..531295f
--- /dev/null
+++ b/pengine/test10/cloned_stop_two.summary
@@ -0,0 +1,45 @@
+
+Current cluster status:
+Node c7auto2 (2): standby
+Node c7auto3 (3): standby
+Online: [ c7auto1 c7auto4 ]
+
+ shooter (stonith:fence_phd_kvm): Started c7auto1
+ Clone Set: FAKECLONE-clone [FAKECLONE]
+ Started: [ c7auto1 c7auto2 c7auto3 ]
+ Stopped: [ c7auto4 ]
+ Clone Set: FAKECLONE2-clone [FAKECLONE2]
+ Started: [ c7auto3 c7auto4 ]
+ Stopped: [ c7auto1 c7auto2 ]
+
+Transition Summary:
+ * Stop FAKECLONE:1 (c7auto2)
+ * Stop FAKECLONE:2 (c7auto3)
+ * Stop FAKECLONE2:0 (c7auto3)
+ * Stop FAKECLONE2:1 (Started c7auto4)
+
+Executing cluster transition:
+ * Pseudo action: FAKECLONE2-clone_stop_0
+ * Resource action: FAKECLONE2 stop on c7auto3
+ * Resource action: FAKECLONE2 stop on c7auto4
+ * Pseudo action: FAKECLONE2-clone_stopped_0
+ * Pseudo action: FAKECLONE-clone_stop_0
+ * Resource action: FAKECLONE stop on c7auto2
+ * Resource action: FAKECLONE stop on c7auto3
+ * Pseudo action: FAKECLONE-clone_stopped_0
+ * Pseudo action: FAKECLONE-clone_start_0
+ * Pseudo action: all_stopped
+ * Pseudo action: FAKECLONE-clone_running_0
+
+Revised cluster status:
+Node c7auto2 (2): standby
+Node c7auto3 (3): standby
+Online: [ c7auto1 c7auto4 ]
+
+ shooter (stonith:fence_phd_kvm): Started c7auto1
+ Clone Set: FAKECLONE-clone [FAKECLONE]
+ Started: [ c7auto1 ]
+ Stopped: [ c7auto2 c7auto3 c7auto4 ]
+ Clone Set: FAKECLONE2-clone [FAKECLONE2]
+ Stopped: [ c7auto1 c7auto2 c7auto3 c7auto4 ]
+
diff --git a/pengine/test10/cloned_stop_two.xml b/pengine/test10/cloned_stop_two.xml
new file mode 100644
index 0000000..220dfc2
--- /dev/null
+++ b/pengine/test10/cloned_stop_two.xml
@@ -0,0 +1,157 @@
+<cib crm_feature_set="3.0.10" validate-with="pacemaker-2.3" epoch="30" num_updates="0" admin_epoch="0" cib-last-written="Thu Jul 16 18:03:09 2015" update-origin="c7auto1" update-client="cibadmin" update-user="root" have-quorum="1" dc-uuid="2">
+ <configuration>
+ <crm_config>
+ <cluster_property_set id="cib-bootstrap-options">
+ <nvpair id="cib-bootstrap-options-have-watchdog" name="have-watchdog" value="false"/>
+ <nvpair id="cib-bootstrap-options-dc-version" name="dc-version" value="1.1.12-ef86b49"/>
+ <nvpair id="cib-bootstrap-options-cluster-infrastructure" name="cluster-infrastructure" value="corosync"/>
+ <nvpair id="cib-bootstrap-options-cluster-name" name="cluster-name" value="phd"/>
+ </cluster_property_set>
+ </crm_config>
+ <nodes>
+ <node id="1" uname="c7auto1"/>
+ <node id="2" uname="c7auto2">
+ <instance_attributes id="nodes-2">
+ <nvpair id="nodes-2-standby" name="standby" value="on"/>
+ </instance_attributes>
+ </node>
+ <node id="3" uname="c7auto3">
+ <instance_attributes id="nodes-3">
+ <nvpair id="nodes-3-standby" name="standby" value="on"/>
+ </instance_attributes>
+ </node>
+ <node id="4" uname="c7auto4"/>
+ </nodes>
+ <resources>
+ <primitive class="stonith" id="shooter" type="fence_phd_kvm">
+ <instance_attributes id="shooter-instance_attributes"/>
+ <operations>
+ <op id="shooter-monitor-interval-60s" interval="60s" name="monitor"/>
+ </operations>
+ </primitive>
+ <clone id="FAKECLONE-clone">
+ <primitive class="ocf" id="FAKECLONE" provider="heartbeat" type="Dummy">
+ <instance_attributes id="FAKECLONE-instance_attributes"/>
+ <operations>
+ <op id="FAKECLONE-start-timeout-20" interval="0s" name="start" timeout="20"/>
+ <op id="FAKECLONE-stop-timeout-20" interval="0s" name="stop" timeout="20"/>
+ <op id="FAKECLONE-monitor-interval-10" interval="10" name="monitor" timeout="20"/>
+ </operations>
+ </primitive>
+ <meta_attributes id="FAKECLONE-clone-meta">
+ <nvpair id="FAKECLONE-clone-min" name="clone-min" value="2"/>
+ </meta_attributes>
+ <meta_attributes id="FAKECLONE-clone-meta_attributes"/>
+ </clone>
+ <clone id="FAKECLONE2-clone">
+ <primitive class="ocf" id="FAKECLONE2" provider="heartbeat" type="Dummy">
+ <instance_attributes id="FAKECLONE2-instance_attributes"/>
+ <operations>
+ <op id="FAKECLONE2-start-timeout-20" interval="0s" name="start" timeout="20"/>
+ <op id="FAKECLONE2-stop-timeout-20" interval="0s" name="stop" timeout="20"/>
+ <op id="FAKECLONE2-monitor-interval-10" interval="10" name="monitor" timeout="20"/>
+ </operations>
+ </primitive>
+ <meta_attributes id="FAKECLONE2-clone-meta"/>
+ </clone>
+ </resources>
+ <constraints>
+ <rsc_location id="location-FAKECLONE-clone-c7auto4--INFINITY" node="c7auto4" rsc="FAKECLONE-clone" score="-INFINITY"/>
+ <rsc_location id="location-FAKECLONE2-clone-c7auto1--INFINITY" node="c7auto1" rsc="FAKECLONE2-clone" score="-INFINITY"/>
+ <rsc_location id="location-FAKECLONE2-clone-c7auto2--INFINITY" node="c7auto2" rsc="FAKECLONE2-clone" score="-INFINITY"/>
+ <rsc_order first="FAKECLONE-clone" first-action="start" id="order-FAKECLONE-clone-FAKECLONE2-clone-mandatory" then="FAKECLONE2-clone" then-action="start"/>
+ </constraints>
+ </configuration>
+ <status>
+ <node_state id="1" uname="c7auto1" in_ccm="true" crmd="online" crm-debug-origin="do_update_resource" join="member" expected="member">
+ <transient_attributes id="1">
+ <instance_attributes id="status-1">
+ <nvpair id="status-1-shutdown" name="shutdown" value="0"/>
+ <nvpair id="status-1-probe_complete" name="probe_complete" value="true"/>
+ </instance_attributes>
+ </transient_attributes>
+ <lrm id="1">
+ <lrm_resources>
+ <lrm_resource id="FAKECLONE2" type="Dummy" class="ocf" provider="heartbeat">
+ <lrm_rsc_op id="FAKECLONE2_last_0" operation_key="FAKECLONE2_stop_0" operation="stop" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.10" transition-key="17:28:0:b489d50c-a7db-4605-8519-a2d5e1452e29" transition-magic="0:0;17:28:0:b489d50c-a7db-4605-8519-a2d5e1452e29" on_node="c7auto1" call-id="30" rc-code="0" op-status="0" interval="0" last-run="1437083676" last-rc-change="1437083676" exec-time="12" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ </lrm_resource>
+ <lrm_resource id="shooter" type="fence_phd_kvm" class="stonith">
+ <lrm_rsc_op id="shooter_last_0" operation_key="shooter_start_0" operation="start" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.10" transition-key="11:0:0:b489d50c-a7db-4605-8519-a2d5e1452e29" transition-magic="0:0;11:0:0:b489d50c-a7db-4605-8519-a2d5e1452e29" on_node="c7auto1" call-id="6" rc-code="0" op-status="0" interval="0" last-run="1437073991" last-rc-change="1437073991" exec-time="9" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ <lrm_rsc_op id="shooter_monitor_60000" operation_key="shooter_monitor_60000" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.10" transition-key="9:1:0:b489d50c-a7db-4605-8519-a2d5e1452e29" transition-magic="0:0;9:1:0:b489d50c-a7db-4605-8519-a2d5e1452e29" on_node="c7auto1" call-id="7" rc-code="0" op-status="0" interval="60000" last-rc-change="1437073992" exec-time="7" queue-time="0" op-digest="4811cef7f7f94e3a35a70be7916cb2fd"/>
+ </lrm_resource>
+ <lrm_resource id="FAKECLONE" type="Dummy" class="ocf" provider="heartbeat">
+ <lrm_rsc_op id="FAKECLONE_last_0" operation_key="FAKECLONE_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="16:33:0:b489d50c-a7db-4605-8519-a2d5e1452e29" transition-magic="0:0;16:33:0:b489d50c-a7db-4605-8519-a2d5e1452e29" on_node="c7auto1" call-id="31" rc-code="0" op-status="0" interval="0" last-run="1437084158" last-rc-change="1437084158" exec-time="9" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" fake " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ <lrm_rsc_op id="FAKECLONE_monitor_10000" operation_key="FAKECLONE_monitor_10000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="17:33:0:b489d50c-a7db-4605-8519-a2d5e1452e29" transition-magic="0:0;17:33:0:b489d50c-a7db-4605-8519-a2d5e1452e29" on_node="c7auto1" call-id="32" rc-code="0" op-status="0" interval="10000" last-rc-change="1437084158" exec-time="7" queue-time="0" op-digest="4811cef7f7f94e3a35a70be7916cb2fd"/>
+ </lrm_resource>
+ </lrm_resources>
+ </lrm>
+ </node_state>
+ <node_state id="2" uname="c7auto2" in_ccm="true" crmd="online" crm-debug-origin="do_update_resource" join="member" expected="member">
+ <transient_attributes id="2">
+ <instance_attributes id="status-2">
+ <nvpair id="status-2-shutdown" name="shutdown" value="0"/>
+ <nvpair id="status-2-probe_complete" name="probe_complete" value="true"/>
+ </instance_attributes>
+ </transient_attributes>
+ <lrm id="2">
+ <lrm_resources>
+ <lrm_resource id="FAKECLONE2" type="Dummy" class="ocf" provider="heartbeat">
+ <lrm_rsc_op id="FAKECLONE2_last_0" operation_key="FAKECLONE2_stop_0" operation="stop" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.10" transition-key="16:29:0:b489d50c-a7db-4605-8519-a2d5e1452e29" transition-magic="0:0;16:29:0:b489d50c-a7db-4605-8519-a2d5e1452e29" on_node="c7auto2" call-id="32" rc-code="0" op-status="0" interval="0" last-run="1437083678" last-rc-change="1437083678" exec-time="10" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ </lrm_resource>
+ <lrm_resource id="shooter" type="fence_phd_kvm" class="stonith">
+ <lrm_rsc_op id="shooter_last_0" operation_key="shooter_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.10" transition-key="6:0:7:b489d50c-a7db-4605-8519-a2d5e1452e29" transition-magic="0:7;6:0:7:b489d50c-a7db-4605-8519-a2d5e1452e29" on_node="c7auto2" call-id="5" rc-code="7" op-status="0" interval="0" last-run="1437073988" last-rc-change="1437073988" exec-time="964" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ </lrm_resource>
+ <lrm_resource id="FAKECLONE" type="Dummy" class="ocf" provider="heartbeat">
+ <lrm_rsc_op id="FAKECLONE_last_0" operation_key="FAKECLONE_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="12:33:0:b489d50c-a7db-4605-8519-a2d5e1452e29" transition-magic="0:0;12:33:0:b489d50c-a7db-4605-8519-a2d5e1452e29" on_node="c7auto2" call-id="33" rc-code="0" op-status="0" interval="0" last-run="1437084158" last-rc-change="1437084158" exec-time="11" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" fake " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ <lrm_rsc_op id="FAKECLONE_monitor_10000" operation_key="FAKECLONE_monitor_10000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="13:33:0:b489d50c-a7db-4605-8519-a2d5e1452e29" transition-magic="0:0;13:33:0:b489d50c-a7db-4605-8519-a2d5e1452e29" on_node="c7auto2" call-id="34" rc-code="0" op-status="0" interval="10000" last-rc-change="1437084158" exec-time="12" queue-time="0" op-digest="4811cef7f7f94e3a35a70be7916cb2fd"/>
+ </lrm_resource>
+ </lrm_resources>
+ </lrm>
+ </node_state>
+ <node_state id="3" uname="c7auto3" in_ccm="true" crmd="online" crm-debug-origin="do_update_resource" join="member" expected="member">
+ <transient_attributes id="3">
+ <instance_attributes id="status-3">
+ <nvpair id="status-3-shutdown" name="shutdown" value="0"/>
+ <nvpair id="status-3-probe_complete" name="probe_complete" value="true"/>
+ </instance_attributes>
+ </transient_attributes>
+ <lrm id="3">
+ <lrm_resources>
+ <lrm_resource id="FAKECLONE2" type="Dummy" class="ocf" provider="heartbeat">
+ <lrm_rsc_op id="FAKECLONE2_last_0" operation_key="FAKECLONE2_start_0" operation="start" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.10" transition-key="17:30:0:b489d50c-a7db-4605-8519-a2d5e1452e29" transition-magic="0:0;17:30:0:b489d50c-a7db-4605-8519-a2d5e1452e29" on_node="c7auto3" call-id="29" rc-code="0" op-status="0" interval="0" last-run="1437083710" last-rc-change="1437083710" exec-time="6" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" fake " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ <lrm_rsc_op id="FAKECLONE2_monitor_10000" operation_key="FAKECLONE2_monitor_10000" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.10" transition-key="18:30:0:b489d50c-a7db-4605-8519-a2d5e1452e29" transition-magic="0:0;18:30:0:b489d50c-a7db-4605-8519-a2d5e1452e29" on_node="c7auto3" call-id="30" rc-code="0" op-status="0" interval="10000" last-rc-change="1437083710" exec-time="6" queue-time="0" op-digest="4811cef7f7f94e3a35a70be7916cb2fd"/>
+ </lrm_resource>
+ <lrm_resource id="shooter" type="fence_phd_kvm" class="stonith">
+ <lrm_rsc_op id="shooter_last_0" operation_key="shooter_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.10" transition-key="8:0:7:b489d50c-a7db-4605-8519-a2d5e1452e29" transition-magic="0:7;8:0:7:b489d50c-a7db-4605-8519-a2d5e1452e29" on_node="c7auto3" call-id="5" rc-code="7" op-status="0" interval="0" last-run="1437073988" last-rc-change="1437073988" exec-time="963" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ </lrm_resource>
+ <lrm_resource id="FAKECLONE" type="Dummy" class="ocf" provider="heartbeat">
+ <lrm_rsc_op id="FAKECLONE_last_0" operation_key="FAKECLONE_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="14:33:0:b489d50c-a7db-4605-8519-a2d5e1452e29" transition-magic="0:0;14:33:0:b489d50c-a7db-4605-8519-a2d5e1452e29" on_node="c7auto3" call-id="31" rc-code="0" op-status="0" interval="0" last-run="1437084157" last-rc-change="1437084157" exec-time="9" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" fake " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ <lrm_rsc_op id="FAKECLONE_monitor_10000" operation_key="FAKECLONE_monitor_10000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="15:33:0:b489d50c-a7db-4605-8519-a2d5e1452e29" transition-magic="0:0;15:33:0:b489d50c-a7db-4605-8519-a2d5e1452e29" on_node="c7auto3" call-id="32" rc-code="0" op-status="0" interval="10000" last-rc-change="1437084157" exec-time="7" queue-time="0" op-digest="4811cef7f7f94e3a35a70be7916cb2fd"/>
+ </lrm_resource>
+ </lrm_resources>
+ </lrm>
+ </node_state>
+ <node_state id="4" uname="c7auto4" in_ccm="true" crmd="online" crm-debug-origin="do_state_transition" join="member" expected="member">
+ <transient_attributes id="4">
+ <instance_attributes id="status-4">
+ <nvpair id="status-4-shutdown" name="shutdown" value="0"/>
+ <nvpair id="status-4-probe_complete" name="probe_complete" value="true"/>
+ </instance_attributes>
+ </transient_attributes>
+ <lrm id="4">
+ <lrm_resources>
+ <lrm_resource id="FAKECLONE2" type="Dummy" class="ocf" provider="heartbeat">
+ <lrm_rsc_op id="FAKECLONE2_last_0" operation_key="FAKECLONE2_start_0" operation="start" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.10" transition-key="22:26:0:b489d50c-a7db-4605-8519-a2d5e1452e29" transition-magic="0:0;22:26:0:b489d50c-a7db-4605-8519-a2d5e1452e29" on_node="c7auto4" call-id="29" rc-code="0" op-status="0" interval="0" last-run="1437083649" last-rc-change="1437083649" exec-time="7" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" fake " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ <lrm_rsc_op id="FAKECLONE2_monitor_10000" operation_key="FAKECLONE2_monitor_10000" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.10" transition-key="23:26:0:b489d50c-a7db-4605-8519-a2d5e1452e29" transition-magic="0:0;23:26:0:b489d50c-a7db-4605-8519-a2d5e1452e29" on_node="c7auto4" call-id="30" rc-code="0" op-status="0" interval="10000" last-rc-change="1437083649" exec-time="7" queue-time="0" op-digest="4811cef7f7f94e3a35a70be7916cb2fd"/>
+ </lrm_resource>
+ <lrm_resource id="shooter" type="fence_phd_kvm" class="stonith">
+ <lrm_rsc_op id="shooter_last_0" operation_key="shooter_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.10" transition-key="10:0:7:b489d50c-a7db-4605-8519-a2d5e1452e29" transition-magic="0:7;10:0:7:b489d50c-a7db-4605-8519-a2d5e1452e29" on_node="c7auto4" call-id="5" rc-code="7" op-status="0" interval="0" last-run="1437073988" last-rc-change="1437073988" exec-time="963" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ </lrm_resource>
+ <lrm_resource id="FAKECLONE" type="Dummy" class="ocf" provider="heartbeat">
+ <lrm_rsc_op id="FAKECLONE_last_0" operation_key="FAKECLONE_stop_0" operation="stop" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.10" transition-key="20:3:0:b489d50c-a7db-4605-8519-a2d5e1452e29" transition-magic="0:0;20:3:0:b489d50c-a7db-4605-8519-a2d5e1452e29" on_node="c7auto4" call-id="14" rc-code="0" op-status="0" interval="0" last-run="1437074856" last-rc-change="1437074856" exec-time="8" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ </lrm_resource>
+ </lrm_resources>
+ </lrm>
+ </node_state>
+ </status>
+</cib>
diff --git a/pengine/test10/stonith-1.dot b/pengine/test10/stonith-1.dot
index d64edcf..3814ac9 100644
--- a/pengine/test10/stonith-1.dot
+++ b/pengine/test10/stonith-1.dot
@@ -88,8 +88,6 @@ digraph "g" {
"rsc_sles-3_stop_0 sles-3" -> "rsc_sles-3_start_0 sles-4" [ style = bold]
"rsc_sles-3_stop_0 sles-3" [ style=bold color="green" fontcolor="orange" ]
"rsc_sles-4_monitor_5000 sles-4" [ style=bold color="green" fontcolor="black" ]
-"stonith 'reboot' sles-3" -> "DoFencing_stop_0" [ style = bold]
-"stonith 'reboot' sles-3" -> "child_DoFencing:2_stop_0 sles-3" [ style = bold]
"stonith 'reboot' sles-3" -> "master_rsc_1_stop_0" [ style = bold]
"stonith 'reboot' sles-3" -> "migrator_stop_0 sles-3" [ style = bold]
"stonith 'reboot' sles-3" -> "ocf_msdummy:2_stop_0 sles-3" [ style = bold]
diff --git a/pengine/test10/stonith-1.exp b/pengine/test10/stonith-1.exp
index 4d58afa..40b22cb 100644
--- a/pengine/test10/stonith-1.exp
+++ b/pengine/test10/stonith-1.exp
@@ -210,9 +210,6 @@
<trigger>
<pseudo_event id="47" operation="stop" operation_key="DoFencing_stop_0"/>
</trigger>
- <trigger>
- <crm_event id="72" operation="stonith" operation_key="stonith-sles-3-reboot" on_node="sles-3" on_node_uuid="2298606a-6a8c-499a-9d25-76242f7006ec"/>
- </trigger>
</inputs>
</synapse>
<synapse id="17" priority="1000000">
@@ -236,11 +233,7 @@
<attributes CRM_meta_clone_max="4" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="false" CRM_meta_timeout="10000" />
</pseudo_event>
</action_set>
- <inputs>
- <trigger>
- <crm_event id="72" operation="stonith" operation_key="stonith-sles-3-reboot" on_node="sles-3" on_node_uuid="2298606a-6a8c-499a-9d25-76242f7006ec"/>
- </trigger>
- </inputs>
+ <inputs/>
</synapse>
<synapse id="19" priority="1000000">
<action_set>
diff --git a/pengine/test10/stonith-1.summary b/pengine/test10/stonith-1.summary
index ef904fe..e99bb5e 100644
--- a/pengine/test10/stonith-1.summary
+++ b/pengine/test10/stonith-1.summary
@@ -45,20 +45,22 @@ Executing cluster transition:
* Resource action: lsb_dummy monitor=5000 on sles-2
* Resource action: rsc_sles-2 monitor=5000 on sles-2
* Resource action: rsc_sles-4 monitor=5000 on sles-4
+ * Pseudo action: DoFencing_stop_0
* Fencing sles-3 (reboot)
* Pseudo action: stonith_complete
* Resource action: r192.168.100.183 start on sles-1
* Pseudo action: migrator_stop_0
* Pseudo action: rsc_sles-3_stop_0
- * Pseudo action: DoFencing_stop_0
+ * Pseudo action: child_DoFencing:2_stop_0
+ * Pseudo action: DoFencing_stopped_0
+ * Pseudo action: DoFencing_start_0
* Pseudo action: master_rsc_1_stop_0
* Pseudo action: group-1_running_0
* Resource action: r192.168.100.183 monitor=5000 on sles-1
* Resource action: migrator start on sles-4
* Resource action: rsc_sles-3 start on sles-4
- * Pseudo action: child_DoFencing:2_stop_0
- * Pseudo action: DoFencing_stopped_0
- * Pseudo action: DoFencing_start_0
+ * Resource action: child_DoFencing:2 start on sles-4
+ * Pseudo action: DoFencing_running_0
* Pseudo action: ocf_msdummy:2_stop_0
* Pseudo action: ocf_msdummy:5_stop_0
* Pseudo action: master_rsc_1_stopped_0
@@ -66,8 +68,7 @@ Executing cluster transition:
* Pseudo action: all_stopped
* Resource action: migrator monitor=10000 on sles-4
* Resource action: rsc_sles-3 monitor=5000 on sles-4
- * Resource action: child_DoFencing:2 start on sles-4
- * Pseudo action: DoFencing_running_0
+ * Resource action: child_DoFencing:2 monitor=60000 on sles-4
* Resource action: ocf_msdummy:0 start on sles-4
* Resource action: ocf_msdummy:1 start on sles-1
* Resource action: ocf_msdummy:2 start on sles-2
@@ -75,7 +76,6 @@ Executing cluster transition:
* Resource action: ocf_msdummy:4 start on sles-1
* Resource action: ocf_msdummy:5 start on sles-2
* Pseudo action: master_rsc_1_running_0
- * Resource action: child_DoFencing:2 monitor=60000 on sles-4
* Resource action: ocf_msdummy:0 monitor=5000 on sles-4
* Resource action: ocf_msdummy:1 monitor=5000 on sles-1
* Resource action: ocf_msdummy:2 monitor=5000 on sles-2
diff --git a/pengine/test10/ticket-master-21.dot b/pengine/test10/ticket-master-21.dot
index 60386a8..3f94948 100644
--- a/pengine/test10/ticket-master-21.dot
+++ b/pengine/test10/ticket-master-21.dot
@@ -23,7 +23,6 @@ digraph "g" {
"stonith 'reboot' node1" -> "ms1_stop_0" [ style = bold]
"stonith 'reboot' node1" -> "rsc1:1_demote_0 node1" [ style = bold]
"stonith 'reboot' node1" -> "rsc1:1_stop_0 node1" [ style = bold]
-"stonith 'reboot' node1" -> "rsc_stonith_stop_0 node1" [ style = bold]
"stonith 'reboot' node1" -> "stonith_complete" [ style = bold]
"stonith 'reboot' node1" [ style=bold color="green" fontcolor="black"]
"stonith_complete" -> "all_stopped" [ style = bold]
diff --git a/pengine/test10/ticket-master-21.exp b/pengine/test10/ticket-master-21.exp
index cc8df2f..c32bac5 100644
--- a/pengine/test10/ticket-master-21.exp
+++ b/pengine/test10/ticket-master-21.exp
@@ -18,11 +18,7 @@
<attributes CRM_meta_timeout="20000" />
</pseudo_event>
</action_set>
- <inputs>
- <trigger>
- <crm_event id="21" operation="stonith" operation_key="stonith-node1-reboot" on_node="node1" on_node_uuid="node1"/>
- </trigger>
- </inputs>
+ <inputs/>
</synapse>
<synapse id="2">
<action_set>
diff --git a/pengine/test10/ticket-master-21.summary b/pengine/test10/ticket-master-21.summary
index 64a9cbe..b228696 100644
--- a/pengine/test10/ticket-master-21.summary
+++ b/pengine/test10/ticket-master-21.summary
@@ -12,14 +12,14 @@ Transition Summary:
* Demote rsc1:0 (Master -> Stopped node1)
Executing cluster transition:
+ * Pseudo action: rsc_stonith_stop_0
* Pseudo action: ms1_demote_0
* Fencing node1 (reboot)
* Pseudo action: stonith_complete
- * Pseudo action: rsc_stonith_stop_0
+ * Resource action: rsc_stonith start on node2
* Pseudo action: rsc1:1_demote_0
* Pseudo action: ms1_demoted_0
* Pseudo action: ms1_stop_0
- * Resource action: rsc_stonith start on node2
* Pseudo action: rsc1:1_stop_0
* Pseudo action: ms1_stopped_0
* Pseudo action: all_stopped
diff --git a/pengine/test10/ticket-master-9.dot b/pengine/test10/ticket-master-9.dot
index 3a29836..c648feb 100644
--- a/pengine/test10/ticket-master-9.dot
+++ b/pengine/test10/ticket-master-9.dot
@@ -23,7 +23,6 @@ digraph "g" {
"stonith 'reboot' node1" -> "ms1_stop_0" [ style = bold]
"stonith 'reboot' node1" -> "rsc1:1_demote_0 node1" [ style = bold]
"stonith 'reboot' node1" -> "rsc1:1_stop_0 node1" [ style = bold]
-"stonith 'reboot' node1" -> "rsc_stonith_stop_0 node1" [ style = bold]
"stonith 'reboot' node1" -> "stonith_complete" [ style = bold]
"stonith 'reboot' node1" [ style=bold color="green" fontcolor="black"]
"stonith_complete" -> "all_stopped" [ style = bold]
diff --git a/pengine/test10/ticket-master-9.exp b/pengine/test10/ticket-master-9.exp
index cc8df2f..c32bac5 100644
--- a/pengine/test10/ticket-master-9.exp
+++ b/pengine/test10/ticket-master-9.exp
@@ -18,11 +18,7 @@
<attributes CRM_meta_timeout="20000" />
</pseudo_event>
</action_set>
- <inputs>
- <trigger>
- <crm_event id="21" operation="stonith" operation_key="stonith-node1-reboot" on_node="node1" on_node_uuid="node1"/>
- </trigger>
- </inputs>
+ <inputs/>
</synapse>
<synapse id="2">
<action_set>
diff --git a/pengine/test10/ticket-master-9.summary b/pengine/test10/ticket-master-9.summary
index 64a9cbe..b228696 100644
--- a/pengine/test10/ticket-master-9.summary
+++ b/pengine/test10/ticket-master-9.summary
@@ -12,14 +12,14 @@ Transition Summary:
* Demote rsc1:0 (Master -> Stopped node1)
Executing cluster transition:
+ * Pseudo action: rsc_stonith_stop_0
* Pseudo action: ms1_demote_0
* Fencing node1 (reboot)
* Pseudo action: stonith_complete
- * Pseudo action: rsc_stonith_stop_0
+ * Resource action: rsc_stonith start on node2
* Pseudo action: rsc1:1_demote_0
* Pseudo action: ms1_demoted_0
* Pseudo action: ms1_stop_0
- * Resource action: rsc_stonith start on node2
* Pseudo action: rsc1:1_stop_0
* Pseudo action: ms1_stopped_0
* Pseudo action: all_stopped
diff --git a/pengine/test10/whitebox-imply-stop-on-fence.dot b/pengine/test10/whitebox-imply-stop-on-fence.dot
index 66700b8..b3fd40b 100644
--- a/pengine/test10/whitebox-imply-stop-on-fence.dot
+++ b/pengine/test10/whitebox-imply-stop-on-fence.dot
@@ -69,7 +69,6 @@
"stonith 'reboot' kiff-01" -> "clvmd_stop_0 kiff-01" [ style = bold]
"stonith 'reboot' kiff-01" -> "dlm-clone_stop_0" [ style = bold]
"stonith 'reboot' kiff-01" -> "dlm_stop_0 kiff-01" [ style = bold]
-"stonith 'reboot' kiff-01" -> "fence-kiff-02_stop_0 kiff-01" [ style = bold]
"stonith 'reboot' kiff-01" -> "lxc-01_kiff-01_stop_0 kiff-01" [ style = bold]
"stonith 'reboot' kiff-01" -> "lxc-02_kiff-01_stop_0 kiff-01" [ style = bold]
"stonith 'reboot' kiff-01" -> "shared0-clone_stop_0" [ style = bold]
diff --git a/pengine/test10/whitebox-imply-stop-on-fence.exp b/pengine/test10/whitebox-imply-stop-on-fence.exp
index d13c25f..4a3e757 100644
--- a/pengine/test10/whitebox-imply-stop-on-fence.exp
+++ b/pengine/test10/whitebox-imply-stop-on-fence.exp
@@ -31,11 +31,7 @@
<attributes CRM_meta_timeout="20000" action="reboot" ipaddr="kiff-02-ilo" login="admin" passwd="admin" pcmk_host_check="static-list" pcmk_host_list="kiff-02"/>
</pseudo_event>
</action_set>
- <inputs>
- <trigger>
- <crm_event id="63" operation="stonith" operation_key="stonith-kiff-01-reboot" on_node="kiff-01" on_node_uuid="1"/>
- </trigger>
- </inputs>
+ <inputs/>
</synapse>
<synapse id="3">
<action_set>
diff --git a/pengine/test10/whitebox-imply-stop-on-fence.summary b/pengine/test10/whitebox-imply-stop-on-fence.summary
index 3bb1572..3ee9570 100644
--- a/pengine/test10/whitebox-imply-stop-on-fence.summary
+++ b/pengine/test10/whitebox-imply-stop-on-fence.summary
@@ -36,16 +36,16 @@ Transition Summary:
* Move lxc-02_kiff-01 (Started kiff-01 -> kiff-02)
Executing cluster transition:
+ * Pseudo action: fence-kiff-02_stop_0
* Fencing kiff-01 (reboot)
* Pseudo action: stonith_complete
- * Pseudo action: fence-kiff-02_stop_0
+ * Resource action: fence-kiff-02 start on kiff-02
* Pseudo action: vm-fs_stop_0
* Pseudo action: lxc-01_kiff-01_stop_0
* Pseudo action: lxc-02_kiff-01_stop_0
- * Resource action: fence-kiff-02 start on kiff-02
+ * Resource action: fence-kiff-02 monitor=60000 on kiff-02
* Pseudo action: R-lxc-01_kiff-01_stop_0
* Pseudo action: R-lxc-02_kiff-01_stop_0
- * Resource action: fence-kiff-02 monitor=60000 on kiff-02
* Pseudo action: shared0-clone_stop_0
* Resource action: R-lxc-01_kiff-01 start on kiff-02
* Resource action: R-lxc-02_kiff-01 start on kiff-02
Reference in New Issue View Git Blame Copy Permalink