From ebea4c3620261c529cad908c0e52064df84b0c61 Mon Sep 17 00:00:00 2001
From: Oyvind Albrigtsen <oalbrigt@redhat.com>
Date: Mon, 11 Jul 2022 10:28:11 +0200
Subject: [PATCH] openstack-agents: warn when openstackcli is slow

---
 heartbeat/openstack-cinder-volume | 19 +++++++++++--------
 heartbeat/openstack-common.sh     | 22 ++++++++++++++++++++++
 heartbeat/openstack-floating-ip   | 17 ++++++++++-------
 heartbeat/openstack-info.in       | 20 ++++++++++----------
 heartbeat/openstack-virtual-ip    | 20 ++++++++++----------
 5 files changed, 63 insertions(+), 35 deletions(-)

diff --git a/heartbeat/openstack-cinder-volume b/heartbeat/openstack-cinder-volume
index 19bf04faf..116442c41 100755
--- a/heartbeat/openstack-cinder-volume
+++ b/heartbeat/openstack-cinder-volume
@@ -113,11 +113,14 @@ _get_node_id() {
 }
 
 osvol_validate() {
+	local result
+
 	check_binary "$OCF_RESKEY_openstackcli"
 	
 	get_config
 
-	if ! $OCF_RESKEY_openstackcli volume list|grep -q $OCF_RESKEY_volume_id ; then
+	result=$(run_openstackcli "volume list")
+	if ! echo "$result" | grep -q $OCF_RESKEY_volume_id; then
 		ocf_exit_reason "volume-id $OCF_RESKEY_volume_id not found"
 		return $OCF_ERR_CONFIGURED
 	fi
@@ -156,17 +159,17 @@ osvol_monitor() {
 	# Is the volue attached?
 	# We use the API
 	#
-	result=$($OCF_RESKEY_openstackcli volume show \
+	result=$(run_openstackcli "volume show \
 		--column status \
 		--column attachments \
 		--format value \
-		$OCF_RESKEY_volume_id)
+		$OCF_RESKEY_volume_id")
 
-	if echo "$result" | grep -q available ; then
+	if echo "$result" | grep -q available; then
 		ocf_log warn "$OCF_RESKEY_volume_id is not attached to any instance"
 		return $OCF_NOT_RUNNING
 	else
-		export attached_server_id=$(echo $result|head -n1|
+		export attached_server_id=$(echo "$result"|head -n1|
 			grep -P -o "'server_id': '[0-9a-f]{8}-([0-9a-f]{4}-){3}[0-9a-f]{12}'"|
 			grep -P -o "[0-9a-f]{8}-([0-9a-f]{4}-){3}[0-9a-f]{12}")
 		ocf_log info "$OCF_RESKEY_volume_id is attached to instance $attached_server_id"
@@ -199,7 +202,7 @@ osvol_stop() {
 	#
 	# Detach the volume
 	#
-	if ! $OCF_RESKEY_openstackcli server remove volume $node_id $OCF_RESKEY_volume_id ; then
+	if ! run_openstackcli "server remove volume $node_id $OCF_RESKEY_volume_id"; then
 		ocf_log error "Couldn't remove volume $OCF_RESKEY_volume_id from instance $node_id"
 		return $OCF_ERR_GENERIC
 	fi
@@ -225,7 +228,7 @@ osvol_start() {
 	# TODO: make it optional in case multi-attachment is allowed by Cinder
 	#
 	if [ ! -z $attached_server_id ] ; then
-		if ! $OCF_RESKEY_openstackcli server remove volume $attached_server_id $OCF_RESKEY_volume_id ; then
+		if ! run_openstackcli "server remove volume $attached_server_id $OCF_RESKEY_volume_id"; then
 			ocf_log error "Couldn't remove volume $OCF_RESKEY_volume_id from instance $attached_server_id"
 			return $OCF_ERR_GENERIC
 		fi
@@ -238,7 +241,7 @@ osvol_start() {
 	#
 	# Attach the volume
 	#
-	$OCF_RESKEY_openstackcli server add volume $node_id $OCF_RESKEY_volume_id
+	run_openstackcli "server add volume $node_id $OCF_RESKEY_volume_id"
 	if [ $? != $OCF_SUCCESS ]; then
 		ocf_log error "Couldn't add volume $OCF_RESKEY_volume_id to instance $node_id"
 		return $OCF_ERR_GENERIC
diff --git a/heartbeat/openstack-common.sh b/heartbeat/openstack-common.sh
index 4763c90db..b6eec09c2 100644
--- a/heartbeat/openstack-common.sh
+++ b/heartbeat/openstack-common.sh
@@ -145,3 +145,25 @@ get_config() {
 		OCF_RESKEY_openstackcli="${OCF_RESKEY_openstackcli} --os-project-domain-name $OCF_RESKEY_project_domain_name"
 	fi
 }
+
+run_openstackcli() {
+	local cmd="${OCF_RESKEY_openstackcli} $1"
+	local result
+	local rc
+	local start_time=$(date +%s)
+	local end_time
+	local elapsed_time
+
+	result=$($cmd)
+	rc=$?
+	end_time=$(date +%s)
+	elapsed_time=$(expr $end_time - $start_time)
+
+	if [ $elapsed_time -gt 20 ]; then
+		ocf_log warn "$cmd took ${elapsed_time}s to complete"
+	fi
+
+	echo "$result"
+
+	return $rc
+}
diff --git a/heartbeat/openstack-floating-ip b/heartbeat/openstack-floating-ip
index 6e2895654..7317f19a8 100755
--- a/heartbeat/openstack-floating-ip
+++ b/heartbeat/openstack-floating-ip
@@ -101,11 +101,14 @@ END
 }
 
 osflip_validate() {
+	local result
+
 	check_binary "$OCF_RESKEY_openstackcli"
 
 	get_config
 
-	if ! $OCF_RESKEY_openstackcli floating ip list|grep -q $OCF_RESKEY_ip_id ; then
+	result=$(run_openstackcli "floating ip list")
+	if ! echo "$result" | grep -q $OCF_RESKEY_ip_id; then
 		ocf_exit_reason "ip-id $OCF_RESKEY_ip_id not found"
 		return $OCF_ERR_CONFIGURED
 	fi
@@ -132,14 +135,14 @@ osflip_monitor() {
 		| awk '{gsub("[^ ]*:", "");print}')
 
 	# Is the IP active and attached?
-	result=$($OCF_RESKEY_openstackcli floating ip show \
+	result=$(run_openstackcli "floating ip show \
 		--column port_id --column floating_ip_address \
 		--format yaml \
-		$OCF_RESKEY_ip_id)
+		$OCF_RESKEY_ip_id")
 
 	for port in $node_port_ids ; do
-		if echo $result | grep -q $port ; then
-			floating_ip=$(echo $result | awk '/floating_ip_address/ {print $2}')
+		if echo "$result" | grep -q $port ; then
+			floating_ip=$(echo "$result" | awk '/floating_ip_address/ {print $2}')
 			${HA_SBIN_DIR}/attrd_updater ${OCF_RESKEY_delay} -S status -n openstack_floating_ip -v $floating_ip
 
 			return $OCF_SUCCESS
@@ -160,7 +163,7 @@ osflip_stop() {
 		return $OCF_SUCCESS
 	fi
 
-	if ! $OCF_RESKEY_openstackcli floating ip unset --port $OCF_RESKEY_ip_id ; then
+	if ! run_openstackcli "floating ip unset --port $OCF_RESKEY_ip_id"; then
 		return $OCF_ERR_GENERIC
 	fi
 
@@ -194,7 +197,7 @@ osflip_start() {
 
 	ocf_log info "Moving IP address $OCF_RESKEY_ip_id to port ID $node_port_id"
 
-	$OCF_RESKEY_openstackcli floating ip set --port $node_port_id $OCF_RESKEY_ip_id
+	run_openstackcli "floating ip set --port $node_port_id $OCF_RESKEY_ip_id"
 	if [ $? != $OCF_SUCCESS ]; then
 		ocf_log error "$OCF_RESKEY_ip_id Cannot be set to port $node_port_id"
 		return $OCF_ERR_GENERIC
diff --git a/heartbeat/openstack-info.in b/heartbeat/openstack-info.in
index f3a59fc7a..6502f1df1 100755
--- a/heartbeat/openstack-info.in
+++ b/heartbeat/openstack-info.in
@@ -119,9 +119,7 @@ END
 #######################################################################
 
 OSInfoStats() {
-	local result
 	local value
-	local node
 	local node_id
 
 	get_config
@@ -141,31 +139,33 @@ OSInfoStats() {
 	${HA_SBIN_DIR}/attrd_updater ${OCF_RESKEY_delay} -n openstack_id -v "$node_id"
 
 	# Nova data: flavor
-	value=$($OCF_RESKEY_openstackcli server show \
+	value=$(run_openstackcli "server show \
 		--format value \
 		--column flavor \
-		$node_id)
+		$node_id")
 
 	${HA_SBIN_DIR}/attrd_updater ${OCF_RESKEY_delay} -n openstack_flavor -v "$value"
 
 	# Nova data: availability zone
-	value=$($OCF_RESKEY_openstackcli server show \
+	value=$(run_openstackcli "server show \
 		--format value \
 		--column OS-EXT-AZ:availability_zone \
-		$node_id)
+		$node_id")
 
 	${HA_SBIN_DIR}/attrd_updater ${OCF_RESKEY_delay} -n openstack_az -v "$value"
 
 	# Network data: ports
 	value=""
-	for port_id in $($OCF_RESKEY_openstackcli port list \
+	for port_id in $(run_openstackcli "port list \
 		--format value \
 		--column id \
-		--server $node_id); do
-		subnet_id=$($OCF_RESKEY_openstackcli port show \
+		--server $node_id"); do
+		subnet_result=$(run_openstackcli "port show \
 			--format json \
 			--column fixed_ips \
-			${port_id} | grep -P '\"subnet_id\": \".*\",$' |
+			${port_id}")
+		subnet_id=$(echo "$subnet_result" |
+			grep -P '\"subnet_id\": \".*\",$' |
 			grep -P -o '[0-9a-f]{8}-([0-9a-f]{4}-){3}[0-9a-f]{12}')
 		value="${value}${subnet_id}:${port_id},"
 	done
diff --git a/heartbeat/openstack-virtual-ip b/heartbeat/openstack-virtual-ip
index c654d980a..361357d55 100755
--- a/heartbeat/openstack-virtual-ip
+++ b/heartbeat/openstack-virtual-ip
@@ -132,11 +132,11 @@ osvip_monitor() {
 
 	node_port_id=$(osvip_port_id)
 
-	result=$($OCF_RESKEY_openstackcli port show \
+	result=$(run_openstackcli "port show \
 		--format value \
 		--column allowed_address_pairs \
-		${node_port_id})
-	if echo $result | grep -q "$OCF_RESKEY_ip"; then
+		${node_port_id}")
+	if echo "$result" | grep -q "$OCF_RESKEY_ip"; then
 		${HA_SBIN_DIR}/attrd_updater ${OCF_RESKEY_delay} -S status -n openstack_virtual_ip -v $OCF_RESKEY_ip
 
 		return $OCF_SUCCESS
@@ -158,20 +158,20 @@ osvip_stop() {
 		return $OCF_SUCCESS
 	fi
 
-	mac_address=$($OCF_RESKEY_openstackcli port show \
+	mac_address=$(run_openstackcli "port show \
 		--format value \
 		--column mac_address \
-		$node_port_id)
-	echo ${mac_address} | grep -q -P "^([0-9a-f]{2}:){5}[0-9a-f]{2}$"
+		$node_port_id")
+	echo "${mac_address}" | grep -q -P "^([0-9a-f]{2}:){5}[0-9a-f]{2}$"
 	if [ $? -ne 0 ]; then
 		ocf_log error "MAC address '${mac_address}' is not valid."
 		return $OCF_ERR_GENERIC
 	fi
 
-	if ! $OCF_RESKEY_openstackcli port unset \
+	if ! run_openstackcli "port unset \
 		--allowed-address \
 		ip-address=$OCF_RESKEY_ip,mac-address=${mac_address} \
-		$node_port_id; then
+		$node_port_id"; then
 		return $OCF_ERR_GENERIC
 	fi
 
@@ -196,9 +196,9 @@ osvip_start() {
 
 	ocf_log info "Moving IP address $OCF_RESKEY_ip to port ID $node_port_id"
 
-	$OCF_RESKEY_openstackcli port set \
+	run_openstackcli "port set \
 		--allowed-address ip-address=$OCF_RESKEY_ip \
-		$node_port_id
+		$node_port_id"
 	if [ $? != $OCF_SUCCESS ]; then
 		ocf_log error "$OCF_RESKEY_ip Cannot be set to port $node_port_id"
 		return $OCF_ERR_GENERIC