283 lines
9.3 KiB
Diff
283 lines
9.3 KiB
Diff
From ebea4c3620261c529cad908c0e52064df84b0c61 Mon Sep 17 00:00:00 2001
|
||
From: Oyvind Albrigtsen <oalbrigt@redhat.com>
|
||
Date: Mon, 11 Jul 2022 10:28:11 +0200
|
||
Subject: [PATCH] openstack-agents: warn when openstackcli is slow
|
||
|
||
---
|
||
heartbeat/openstack-cinder-volume | 19 +++++++++++--------
|
||
heartbeat/openstack-common.sh | 22 ++++++++++++++++++++++
|
||
heartbeat/openstack-floating-ip | 17 ++++++++++-------
|
||
heartbeat/openstack-info.in | 20 ++++++++++----------
|
||
heartbeat/openstack-virtual-ip | 20 ++++++++++----------
|
||
5 files changed, 63 insertions(+), 35 deletions(-)
|
||
|
||
diff --git a/heartbeat/openstack-cinder-volume b/heartbeat/openstack-cinder-volume
|
||
index 19bf04faf..116442c41 100755
|
||
--- a/heartbeat/openstack-cinder-volume
|
||
+++ b/heartbeat/openstack-cinder-volume
|
||
@@ -113,11 +113,14 @@ _get_node_id() {
|
||
}
|
||
|
||
osvol_validate() {
|
||
+ local result
|
||
+
|
||
check_binary "$OCF_RESKEY_openstackcli"
|
||
|
||
get_config
|
||
|
||
- if ! $OCF_RESKEY_openstackcli volume list|grep -q $OCF_RESKEY_volume_id ; then
|
||
+ result=$(run_openstackcli "volume list")
|
||
+ if ! echo "$result" | grep -q $OCF_RESKEY_volume_id; then
|
||
ocf_exit_reason "volume-id $OCF_RESKEY_volume_id not found"
|
||
return $OCF_ERR_CONFIGURED
|
||
fi
|
||
@@ -156,17 +159,17 @@ osvol_monitor() {
|
||
# Is the volue attached?
|
||
# We use the API
|
||
#
|
||
- result=$($OCF_RESKEY_openstackcli volume show \
|
||
+ result=$(run_openstackcli "volume show \
|
||
--column status \
|
||
--column attachments \
|
||
--format value \
|
||
- $OCF_RESKEY_volume_id)
|
||
+ $OCF_RESKEY_volume_id")
|
||
|
||
- if echo "$result" | grep -q available ; then
|
||
+ if echo "$result" | grep -q available; then
|
||
ocf_log warn "$OCF_RESKEY_volume_id is not attached to any instance"
|
||
return $OCF_NOT_RUNNING
|
||
else
|
||
- export attached_server_id=$(echo $result|head -n1|
|
||
+ export attached_server_id=$(echo "$result"|head -n1|
|
||
grep -P -o "'server_id': '[0-9a-f]{8}-([0-9a-f]{4}-){3}[0-9a-f]{12}'"|
|
||
grep -P -o "[0-9a-f]{8}-([0-9a-f]{4}-){3}[0-9a-f]{12}")
|
||
ocf_log info "$OCF_RESKEY_volume_id is attached to instance $attached_server_id"
|
||
@@ -199,7 +202,7 @@ osvol_stop() {
|
||
#
|
||
# Detach the volume
|
||
#
|
||
- if ! $OCF_RESKEY_openstackcli server remove volume $node_id $OCF_RESKEY_volume_id ; then
|
||
+ if ! run_openstackcli "server remove volume $node_id $OCF_RESKEY_volume_id"; then
|
||
ocf_log error "Couldn't remove volume $OCF_RESKEY_volume_id from instance $node_id"
|
||
return $OCF_ERR_GENERIC
|
||
fi
|
||
@@ -225,7 +228,7 @@ osvol_start() {
|
||
# TODO: make it optional in case multi-attachment is allowed by Cinder
|
||
#
|
||
if [ ! -z $attached_server_id ] ; then
|
||
- if ! $OCF_RESKEY_openstackcli server remove volume $attached_server_id $OCF_RESKEY_volume_id ; then
|
||
+ if ! run_openstackcli "server remove volume $attached_server_id $OCF_RESKEY_volume_id"; then
|
||
ocf_log error "Couldn't remove volume $OCF_RESKEY_volume_id from instance $attached_server_id"
|
||
return $OCF_ERR_GENERIC
|
||
fi
|
||
@@ -238,7 +241,7 @@ osvol_start() {
|
||
#
|
||
# Attach the volume
|
||
#
|
||
- $OCF_RESKEY_openstackcli server add volume $node_id $OCF_RESKEY_volume_id
|
||
+ run_openstackcli "server add volume $node_id $OCF_RESKEY_volume_id"
|
||
if [ $? != $OCF_SUCCESS ]; then
|
||
ocf_log error "Couldn't add volume $OCF_RESKEY_volume_id to instance $node_id"
|
||
return $OCF_ERR_GENERIC
|
||
diff --git a/heartbeat/openstack-common.sh b/heartbeat/openstack-common.sh
|
||
index 4763c90db..b6eec09c2 100644
|
||
--- a/heartbeat/openstack-common.sh
|
||
+++ b/heartbeat/openstack-common.sh
|
||
@@ -145,3 +145,25 @@ get_config() {
|
||
OCF_RESKEY_openstackcli="${OCF_RESKEY_openstackcli} --os-project-domain-name $OCF_RESKEY_project_domain_name"
|
||
fi
|
||
}
|
||
+
|
||
+run_openstackcli() {
|
||
+ local cmd="${OCF_RESKEY_openstackcli} $1"
|
||
+ local result
|
||
+ local rc
|
||
+ local start_time=$(date +%s)
|
||
+ local end_time
|
||
+ local elapsed_time
|
||
+
|
||
+ result=$($cmd)
|
||
+ rc=$?
|
||
+ end_time=$(date +%s)
|
||
+ elapsed_time=$(expr $end_time - $start_time)
|
||
+
|
||
+ if [ $elapsed_time -gt 20 ]; then
|
||
+ ocf_log warn "$cmd took ${elapsed_time}s to complete"
|
||
+ fi
|
||
+
|
||
+ echo "$result"
|
||
+
|
||
+ return $rc
|
||
+}
|
||
diff --git a/heartbeat/openstack-floating-ip b/heartbeat/openstack-floating-ip
|
||
index 6e2895654..7317f19a8 100755
|
||
--- a/heartbeat/openstack-floating-ip
|
||
+++ b/heartbeat/openstack-floating-ip
|
||
@@ -101,11 +101,14 @@ END
|
||
}
|
||
|
||
osflip_validate() {
|
||
+ local result
|
||
+
|
||
check_binary "$OCF_RESKEY_openstackcli"
|
||
|
||
get_config
|
||
|
||
- if ! $OCF_RESKEY_openstackcli floating ip list|grep -q $OCF_RESKEY_ip_id ; then
|
||
+ result=$(run_openstackcli "floating ip list")
|
||
+ if ! echo "$result" | grep -q $OCF_RESKEY_ip_id; then
|
||
ocf_exit_reason "ip-id $OCF_RESKEY_ip_id not found"
|
||
return $OCF_ERR_CONFIGURED
|
||
fi
|
||
@@ -132,14 +135,14 @@ osflip_monitor() {
|
||
| awk '{gsub("[^ ]*:", "");print}')
|
||
|
||
# Is the IP active and attached?
|
||
- result=$($OCF_RESKEY_openstackcli floating ip show \
|
||
+ result=$(run_openstackcli "floating ip show \
|
||
--column port_id --column floating_ip_address \
|
||
--format yaml \
|
||
- $OCF_RESKEY_ip_id)
|
||
+ $OCF_RESKEY_ip_id")
|
||
|
||
for port in $node_port_ids ; do
|
||
- if echo $result | grep -q $port ; then
|
||
- floating_ip=$(echo $result | awk '/floating_ip_address/ {print $2}')
|
||
+ if echo "$result" | grep -q $port ; then
|
||
+ floating_ip=$(echo "$result" | awk '/floating_ip_address/ {print $2}')
|
||
${HA_SBIN_DIR}/attrd_updater ${OCF_RESKEY_delay} -S status -n openstack_floating_ip -v $floating_ip
|
||
|
||
return $OCF_SUCCESS
|
||
@@ -160,7 +163,7 @@ osflip_stop() {
|
||
return $OCF_SUCCESS
|
||
fi
|
||
|
||
- if ! $OCF_RESKEY_openstackcli floating ip unset --port $OCF_RESKEY_ip_id ; then
|
||
+ if ! run_openstackcli "floating ip unset --port $OCF_RESKEY_ip_id"; then
|
||
return $OCF_ERR_GENERIC
|
||
fi
|
||
|
||
@@ -194,7 +197,7 @@ osflip_start() {
|
||
|
||
ocf_log info "Moving IP address $OCF_RESKEY_ip_id to port ID $node_port_id"
|
||
|
||
- $OCF_RESKEY_openstackcli floating ip set --port $node_port_id $OCF_RESKEY_ip_id
|
||
+ run_openstackcli "floating ip set --port $node_port_id $OCF_RESKEY_ip_id"
|
||
if [ $? != $OCF_SUCCESS ]; then
|
||
ocf_log error "$OCF_RESKEY_ip_id Cannot be set to port $node_port_id"
|
||
return $OCF_ERR_GENERIC
|
||
diff --git a/heartbeat/openstack-info.in b/heartbeat/openstack-info.in
|
||
index f3a59fc7a..6502f1df1 100755
|
||
--- a/heartbeat/openstack-info.in
|
||
+++ b/heartbeat/openstack-info.in
|
||
@@ -119,9 +119,7 @@ END
|
||
#######################################################################
|
||
|
||
OSInfoStats() {
|
||
- local result
|
||
local value
|
||
- local node
|
||
local node_id
|
||
|
||
get_config
|
||
@@ -141,31 +139,33 @@ OSInfoStats() {
|
||
${HA_SBIN_DIR}/attrd_updater ${OCF_RESKEY_delay} -n openstack_id -v "$node_id"
|
||
|
||
# Nova data: flavor
|
||
- value=$($OCF_RESKEY_openstackcli server show \
|
||
+ value=$(run_openstackcli "server show \
|
||
--format value \
|
||
--column flavor \
|
||
- $node_id)
|
||
+ $node_id")
|
||
|
||
${HA_SBIN_DIR}/attrd_updater ${OCF_RESKEY_delay} -n openstack_flavor -v "$value"
|
||
|
||
# Nova data: availability zone
|
||
- value=$($OCF_RESKEY_openstackcli server show \
|
||
+ value=$(run_openstackcli "server show \
|
||
--format value \
|
||
--column OS-EXT-AZ:availability_zone \
|
||
- $node_id)
|
||
+ $node_id")
|
||
|
||
${HA_SBIN_DIR}/attrd_updater ${OCF_RESKEY_delay} -n openstack_az -v "$value"
|
||
|
||
# Network data: ports
|
||
value=""
|
||
- for port_id in $($OCF_RESKEY_openstackcli port list \
|
||
+ for port_id in $(run_openstackcli "port list \
|
||
--format value \
|
||
--column id \
|
||
- --server $node_id); do
|
||
- subnet_id=$($OCF_RESKEY_openstackcli port show \
|
||
+ --server $node_id"); do
|
||
+ subnet_result=$(run_openstackcli "port show \
|
||
--format json \
|
||
--column fixed_ips \
|
||
- ${port_id} | grep -P '\"subnet_id\": \".*\",$' |
|
||
+ ${port_id}")
|
||
+ subnet_id=$(echo "$subnet_result" |
|
||
+ grep -P '\"subnet_id\": \".*\",$' |
|
||
grep -P -o '[0-9a-f]{8}-([0-9a-f]{4}-){3}[0-9a-f]{12}')
|
||
value="${value}${subnet_id}:${port_id},"
|
||
done
|
||
diff --git a/heartbeat/openstack-virtual-ip b/heartbeat/openstack-virtual-ip
|
||
index c654d980a..361357d55 100755
|
||
--- a/heartbeat/openstack-virtual-ip
|
||
+++ b/heartbeat/openstack-virtual-ip
|
||
@@ -132,11 +132,11 @@ osvip_monitor() {
|
||
|
||
node_port_id=$(osvip_port_id)
|
||
|
||
- result=$($OCF_RESKEY_openstackcli port show \
|
||
+ result=$(run_openstackcli "port show \
|
||
--format value \
|
||
--column allowed_address_pairs \
|
||
- ${node_port_id})
|
||
- if echo $result | grep -q "$OCF_RESKEY_ip"; then
|
||
+ ${node_port_id}")
|
||
+ if echo "$result" | grep -q "$OCF_RESKEY_ip"; then
|
||
${HA_SBIN_DIR}/attrd_updater ${OCF_RESKEY_delay} -S status -n openstack_virtual_ip -v $OCF_RESKEY_ip
|
||
|
||
return $OCF_SUCCESS
|
||
@@ -158,20 +158,20 @@ osvip_stop() {
|
||
return $OCF_SUCCESS
|
||
fi
|
||
|
||
- mac_address=$($OCF_RESKEY_openstackcli port show \
|
||
+ mac_address=$(run_openstackcli "port show \
|
||
--format value \
|
||
--column mac_address \
|
||
- $node_port_id)
|
||
- echo ${mac_address} | grep -q -P "^([0-9a-f]{2}:){5}[0-9a-f]{2}$"
|
||
+ $node_port_id")
|
||
+ echo "${mac_address}" | grep -q -P "^([0-9a-f]{2}:){5}[0-9a-f]{2}$"
|
||
if [ $? -ne 0 ]; then
|
||
ocf_log error "MAC address '${mac_address}' is not valid."
|
||
return $OCF_ERR_GENERIC
|
||
fi
|
||
|
||
- if ! $OCF_RESKEY_openstackcli port unset \
|
||
+ if ! run_openstackcli "port unset \
|
||
--allowed-address \
|
||
ip-address=$OCF_RESKEY_ip,mac-address=${mac_address} \
|
||
- $node_port_id; then
|
||
+ $node_port_id"; then
|
||
return $OCF_ERR_GENERIC
|
||
fi
|
||
|
||
@@ -196,9 +196,9 @@ osvip_start() {
|
||
|
||
ocf_log info "Moving IP address $OCF_RESKEY_ip to port ID $node_port_id"
|
||
|
||
- $OCF_RESKEY_openstackcli port set \
|
||
+ run_openstackcli "port set \
|
||
--allowed-address ip-address=$OCF_RESKEY_ip \
|
||
- $node_port_id
|
||
+ $node_port_id"
|
||
if [ $? != $OCF_SUCCESS ]; then
|
||
ocf_log error "$OCF_RESKEY_ip Cannot be set to port $node_port_id"
|
||
return $OCF_ERR_GENERIC
|