283 lines
9.3 KiB
Diff
283 lines
9.3 KiB
Diff
|
From ebea4c3620261c529cad908c0e52064df84b0c61 Mon Sep 17 00:00:00 2001
|
|||
|
|
From: Oyvind Albrigtsen <oalbrigt@redhat.com>
|
|||
|
|
Date: Mon, 11 Jul 2022 10:28:11 +0200
|
|||
|
|
Subject: [PATCH] openstack-agents: warn when openstackcli is slow
|
|||
|
|
|
|||
|
|
---
|
|||
|
|
heartbeat/openstack-cinder-volume | 19 +++++++++++--------
|
|||
|
|
heartbeat/openstack-common.sh | 22 ++++++++++++++++++++++
|
|||
|
|
heartbeat/openstack-floating-ip | 17 ++++++++++-------
|
|||
|
|
heartbeat/openstack-info.in | 20 ++++++++++----------
|
|||
|
|
heartbeat/openstack-virtual-ip | 20 ++++++++++----------
|
|||
|
|
5 files changed, 63 insertions(+), 35 deletions(-)
|
|||
|
|
|
|||
|
|
diff --git a/heartbeat/openstack-cinder-volume b/heartbeat/openstack-cinder-volume
|
|||
|
|
index 19bf04faf..116442c41 100755
|
|||
|
|
--- a/heartbeat/openstack-cinder-volume
|
|||
|
|
+++ b/heartbeat/openstack-cinder-volume
|
|||
|
|
@@ -113,11 +113,14 @@ _get_node_id() {
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
osvol_validate() {
|
|||
|
|
+ local result
|
|||
|
|
+
|
|||
|
|
check_binary "$OCF_RESKEY_openstackcli"
|
|||
|
|
|
|||
|
|
get_config
|
|||
|
|
|
|||
|
|
- if ! $OCF_RESKEY_openstackcli volume list|grep -q $OCF_RESKEY_volume_id ; then
|
|||
|
|
+ result=$(run_openstackcli "volume list")
|
|||
|
|
+ if ! echo "$result" | grep -q $OCF_RESKEY_volume_id; then
|
|||
|
|
ocf_exit_reason "volume-id $OCF_RESKEY_volume_id not found"
|
|||
|
|
return $OCF_ERR_CONFIGURED
|
|||
|
|
fi
|
|||
|
|
@@ -156,17 +159,17 @@ osvol_monitor() {
|
|||
|
|
# Is the volue attached?
|
|||
|
|
# We use the API
|
|||
|
|
#
|
|||
|
|
- result=$($OCF_RESKEY_openstackcli volume show \
|
|||
|
|
+ result=$(run_openstackcli "volume show \
|
|||
|
|
--column status \
|
|||
|
|
--column attachments \
|
|||
|
|
--format value \
|
|||
|
|
- $OCF_RESKEY_volume_id)
|
|||
|
|
+ $OCF_RESKEY_volume_id")
|
|||
|
|
|
|||
|
|
- if echo "$result" | grep -q available ; then
|
|||
|
|
+ if echo "$result" | grep -q available; then
|
|||
|
|
ocf_log warn "$OCF_RESKEY_volume_id is not attached to any instance"
|
|||
|
|
return $OCF_NOT_RUNNING
|
|||
|
|
else
|
|||
|
|
- export attached_server_id=$(echo $result|head -n1|
|
|||
|
|
+ export attached_server_id=$(echo "$result"|head -n1|
|
|||
|
|
grep -P -o "'server_id': '[0-9a-f]{8}-([0-9a-f]{4}-){3}[0-9a-f]{12}'"|
|
|||
|
|
grep -P -o "[0-9a-f]{8}-([0-9a-f]{4}-){3}[0-9a-f]{12}")
|
|||
|
|
ocf_log info "$OCF_RESKEY_volume_id is attached to instance $attached_server_id"
|
|||
|
|
@@ -199,7 +202,7 @@ osvol_stop() {
|
|||
|
|
#
|
|||
|
|
# Detach the volume
|
|||
|
|
#
|
|||
|
|
- if ! $OCF_RESKEY_openstackcli server remove volume $node_id $OCF_RESKEY_volume_id ; then
|
|||
|
|
+ if ! run_openstackcli "server remove volume $node_id $OCF_RESKEY_volume_id"; then
|
|||
|
|
ocf_log error "Couldn't remove volume $OCF_RESKEY_volume_id from instance $node_id"
|
|||
|
|
return $OCF_ERR_GENERIC
|
|||
|
|
fi
|
|||
|
|
@@ -225,7 +228,7 @@ osvol_start() {
|
|||
|
|
# TODO: make it optional in case multi-attachment is allowed by Cinder
|
|||
|
|
#
|
|||
|
|
if [ ! -z $attached_server_id ] ; then
|
|||
|
|
- if ! $OCF_RESKEY_openstackcli server remove volume $attached_server_id $OCF_RESKEY_volume_id ; then
|
|||
|
|
+ if ! run_openstackcli "server remove volume $attached_server_id $OCF_RESKEY_volume_id"; then
|
|||
|
|
ocf_log error "Couldn't remove volume $OCF_RESKEY_volume_id from instance $attached_server_id"
|
|||
|
|
return $OCF_ERR_GENERIC
|
|||
|
|
fi
|
|||
|
|
@@ -238,7 +241,7 @@ osvol_start() {
|
|||
|
|
#
|
|||
|
|
# Attach the volume
|
|||
|
|
#
|
|||
|
|
- $OCF_RESKEY_openstackcli server add volume $node_id $OCF_RESKEY_volume_id
|
|||
|
|
+ run_openstackcli "server add volume $node_id $OCF_RESKEY_volume_id"
|
|||
|
|
if [ $? != $OCF_SUCCESS ]; then
|
|||
|
|
ocf_log error "Couldn't add volume $OCF_RESKEY_volume_id to instance $node_id"
|
|||
|
|
return $OCF_ERR_GENERIC
|
|||
|
|
diff --git a/heartbeat/openstack-common.sh b/heartbeat/openstack-common.sh
|
|||
|
|
index 4763c90db..b6eec09c2 100644
|
|||
|
|
--- a/heartbeat/openstack-common.sh
|
|||
|
|
+++ b/heartbeat/openstack-common.sh
|
|||
|
|
@@ -145,3 +145,25 @@ get_config() {
|
|||
|
|
OCF_RESKEY_openstackcli="${OCF_RESKEY_openstackcli} --os-project-domain-name $OCF_RESKEY_project_domain_name"
|
|||
|
|
fi
|
|||
|
|
}
|
|||
|
|
+
|
|||
|
|
+run_openstackcli() {
|
|||
|
|
+ local cmd="${OCF_RESKEY_openstackcli} $1"
|
|||
|
|
+ local result
|
|||
|
|
+ local rc
|
|||
|
|
+ local start_time=$(date +%s)
|
|||
|
|
+ local end_time
|
|||
|
|
+ local elapsed_time
|
|||
|
|
+
|
|||
|
|
+ result=$($cmd)
|
|||
|
|
+ rc=$?
|
|||
|
|
+ end_time=$(date +%s)
|
|||
|
|
+ elapsed_time=$(expr $end_time - $start_time)
|
|||
|
|
+
|
|||
|
|
+ if [ $elapsed_time -gt 20 ]; then
|
|||
|
|
+ ocf_log warn "$cmd took ${elapsed_time}s to complete"
|
|||
|
|
+ fi
|
|||
|
|
+
|
|||
|
|
+ echo "$result"
|
|||
|
|
+
|
|||
|
|
+ return $rc
|
|||
|
|
+}
|
|||
|
|
diff --git a/heartbeat/openstack-floating-ip b/heartbeat/openstack-floating-ip
|
|||
|
|
index 6e2895654..7317f19a8 100755
|
|||
|
|
--- a/heartbeat/openstack-floating-ip
|
|||
|
|
+++ b/heartbeat/openstack-floating-ip
|
|||
|
|
@@ -101,11 +101,14 @@ END
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
osflip_validate() {
|
|||
|
|
+ local result
|
|||
|
|
+
|
|||
|
|
check_binary "$OCF_RESKEY_openstackcli"
|
|||
|
|
|
|||
|
|
get_config
|
|||
|
|
|
|||
|
|
- if ! $OCF_RESKEY_openstackcli floating ip list|grep -q $OCF_RESKEY_ip_id ; then
|
|||
|
|
+ result=$(run_openstackcli "floating ip list")
|
|||
|
|
+ if ! echo "$result" | grep -q $OCF_RESKEY_ip_id; then
|
|||
|
|
ocf_exit_reason "ip-id $OCF_RESKEY_ip_id not found"
|
|||
|
|
return $OCF_ERR_CONFIGURED
|
|||
|
|
fi
|
|||
|
|
@@ -132,14 +135,14 @@ osflip_monitor() {
|
|||
|
|
| awk '{gsub("[^ ]*:", "");print}')
|
|||
|
|
|
|||
|
|
# Is the IP active and attached?
|
|||
|
|
- result=$($OCF_RESKEY_openstackcli floating ip show \
|
|||
|
|
+ result=$(run_openstackcli "floating ip show \
|
|||
|
|
--column port_id --column floating_ip_address \
|
|||
|
|
--format yaml \
|
|||
|
|
- $OCF_RESKEY_ip_id)
|
|||
|
|
+ $OCF_RESKEY_ip_id")
|
|||
|
|
|
|||
|
|
for port in $node_port_ids ; do
|
|||
|
|
- if echo $result | grep -q $port ; then
|
|||
|
|
- floating_ip=$(echo $result | awk '/floating_ip_address/ {print $2}')
|
|||
|
|
+ if echo "$result" | grep -q $port ; then
|
|||
|
|
+ floating_ip=$(echo "$result" | awk '/floating_ip_address/ {print $2}')
|
|||
|
|
${HA_SBIN_DIR}/attrd_updater ${OCF_RESKEY_delay} -S status -n openstack_floating_ip -v $floating_ip
|
|||
|
|
|
|||
|
|
return $OCF_SUCCESS
|
|||
|
|
@@ -160,7 +163,7 @@ osflip_stop() {
|
|||
|
|
return $OCF_SUCCESS
|
|||
|
|
fi
|
|||
|
|
|
|||
|
|
- if ! $OCF_RESKEY_openstackcli floating ip unset --port $OCF_RESKEY_ip_id ; then
|
|||
|
|
+ if ! run_openstackcli "floating ip unset --port $OCF_RESKEY_ip_id"; then
|
|||
|
|
return $OCF_ERR_GENERIC
|
|||
|
|
fi
|
|||
|
|
|
|||
|
|
@@ -194,7 +197,7 @@ osflip_start() {
|
|||
|
|
|
|||
|
|
ocf_log info "Moving IP address $OCF_RESKEY_ip_id to port ID $node_port_id"
|
|||
|
|
|
|||
|
|
- $OCF_RESKEY_openstackcli floating ip set --port $node_port_id $OCF_RESKEY_ip_id
|
|||
|
|
+ run_openstackcli "floating ip set --port $node_port_id $OCF_RESKEY_ip_id"
|
|||
|
|
if [ $? != $OCF_SUCCESS ]; then
|
|||
|
|
ocf_log error "$OCF_RESKEY_ip_id Cannot be set to port $node_port_id"
|
|||
|
|
return $OCF_ERR_GENERIC
|
|||
|
|
diff --git a/heartbeat/openstack-info.in b/heartbeat/openstack-info.in
|
|||
|
|
index f3a59fc7a..6502f1df1 100755
|
|||
|
|
--- a/heartbeat/openstack-info.in
|
|||
|
|
+++ b/heartbeat/openstack-info.in
|
|||
|
|
@@ -119,9 +119,7 @@ END
|
|||
|
|
#######################################################################
|
|||
|
|
|
|||
|
|
OSInfoStats() {
|
|||
|
|
- local result
|
|||
|
|
local value
|
|||
|
|
- local node
|
|||
|
|
local node_id
|
|||
|
|
|
|||
|
|
get_config
|
|||
|
|
@@ -141,31 +139,33 @@ OSInfoStats() {
|
|||
|
|
${HA_SBIN_DIR}/attrd_updater ${OCF_RESKEY_delay} -n openstack_id -v "$node_id"
|
|||
|
|
|
|||
|
|
# Nova data: flavor
|
|||
|
|
- value=$($OCF_RESKEY_openstackcli server show \
|
|||
|
|
+ value=$(run_openstackcli "server show \
|
|||
|
|
--format value \
|
|||
|
|
--column flavor \
|
|||
|
|
- $node_id)
|
|||
|
|
+ $node_id")
|
|||
|
|
|
|||
|
|
${HA_SBIN_DIR}/attrd_updater ${OCF_RESKEY_delay} -n openstack_flavor -v "$value"
|
|||
|
|
|
|||
|
|
# Nova data: availability zone
|
|||
|
|
- value=$($OCF_RESKEY_openstackcli server show \
|
|||
|
|
+ value=$(run_openstackcli "server show \
|
|||
|
|
--format value \
|
|||
|
|
--column OS-EXT-AZ:availability_zone \
|
|||
|
|
- $node_id)
|
|||
|
|
+ $node_id")
|
|||
|
|
|
|||
|
|
${HA_SBIN_DIR}/attrd_updater ${OCF_RESKEY_delay} -n openstack_az -v "$value"
|
|||
|
|
|
|||
|
|
# Network data: ports
|
|||
|
|
value=""
|
|||
|
|
- for port_id in $($OCF_RESKEY_openstackcli port list \
|
|||
|
|
+ for port_id in $(run_openstackcli "port list \
|
|||
|
|
--format value \
|
|||
|
|
--column id \
|
|||
|
|
- --server $node_id); do
|
|||
|
|
- subnet_id=$($OCF_RESKEY_openstackcli port show \
|
|||
|
|
+ --server $node_id"); do
|
|||
|
|
+ subnet_result=$(run_openstackcli "port show \
|
|||
|
|
--format json \
|
|||
|
|
--column fixed_ips \
|
|||
|
|
- ${port_id} | grep -P '\"subnet_id\": \".*\",$' |
|
|||
|
|
+ ${port_id}")
|
|||
|
|
+ subnet_id=$(echo "$subnet_result" |
|
|||
|
|
+ grep -P '\"subnet_id\": \".*\",$' |
|
|||
|
|
grep -P -o '[0-9a-f]{8}-([0-9a-f]{4}-){3}[0-9a-f]{12}')
|
|||
|
|
value="${value}${subnet_id}:${port_id},"
|
|||
|
|
done
|
|||
|
|
diff --git a/heartbeat/openstack-virtual-ip b/heartbeat/openstack-virtual-ip
|
|||
|
|
index c654d980a..361357d55 100755
|
|||
|
|
--- a/heartbeat/openstack-virtual-ip
|
|||
|
|
+++ b/heartbeat/openstack-virtual-ip
|
|||
|
|
@@ -132,11 +132,11 @@ osvip_monitor() {
|
|||
|
|
|
|||
|
|
node_port_id=$(osvip_port_id)
|
|||
|
|
|
|||
|
|
- result=$($OCF_RESKEY_openstackcli port show \
|
|||
|
|
+ result=$(run_openstackcli "port show \
|
|||
|
|
--format value \
|
|||
|
|
--column allowed_address_pairs \
|
|||
|
|
- ${node_port_id})
|
|||
|
|
- if echo $result | grep -q "$OCF_RESKEY_ip"; then
|
|||
|
|
+ ${node_port_id}")
|
|||
|
|
+ if echo "$result" | grep -q "$OCF_RESKEY_ip"; then
|
|||
|
|
${HA_SBIN_DIR}/attrd_updater ${OCF_RESKEY_delay} -S status -n openstack_virtual_ip -v $OCF_RESKEY_ip
|
|||
|
|
|
|||
|
|
return $OCF_SUCCESS
|
|||
|
|
@@ -158,20 +158,20 @@ osvip_stop() {
|
|||
|
|
return $OCF_SUCCESS
|
|||
|
|
fi
|
|||
|
|
|
|||
|
|
- mac_address=$($OCF_RESKEY_openstackcli port show \
|
|||
|
|
+ mac_address=$(run_openstackcli "port show \
|
|||
|
|
--format value \
|
|||
|
|
--column mac_address \
|
|||
|
|
- $node_port_id)
|
|||
|
|
- echo ${mac_address} | grep -q -P "^([0-9a-f]{2}:){5}[0-9a-f]{2}$"
|
|||
|
|
+ $node_port_id")
|
|||
|
|
+ echo "${mac_address}" | grep -q -P "^([0-9a-f]{2}:){5}[0-9a-f]{2}$"
|
|||
|
|
if [ $? -ne 0 ]; then
|
|||
|
|
ocf_log error "MAC address '${mac_address}' is not valid."
|
|||
|
|
return $OCF_ERR_GENERIC
|
|||
|
|
fi
|
|||
|
|
|
|||
|
|
- if ! $OCF_RESKEY_openstackcli port unset \
|
|||
|
|
+ if ! run_openstackcli "port unset \
|
|||
|
|
--allowed-address \
|
|||
|
|
ip-address=$OCF_RESKEY_ip,mac-address=${mac_address} \
|
|||
|
|
- $node_port_id; then
|
|||
|
|
+ $node_port_id"; then
|
|||
|
|
return $OCF_ERR_GENERIC
|
|||
|
|
fi
|
|||
|
|
|
|||
|
|
@@ -196,9 +196,9 @@ osvip_start() {
|
|||
|
|
|
|||
|
|
ocf_log info "Moving IP address $OCF_RESKEY_ip to port ID $node_port_id"
|
|||
|
|
|
|||
|
|
- $OCF_RESKEY_openstackcli port set \
|
|||
|
|
+ run_openstackcli "port set \
|
|||
|
|
--allowed-address ip-address=$OCF_RESKEY_ip \
|
|||
|
|
- $node_port_id
|
|||
|
|
+ $node_port_id"
|
|||
|
|
if [ $? != $OCF_SUCCESS ]; then
|
|||
|
|
ocf_log error "$OCF_RESKEY_ip Cannot be set to port $node_port_id"
|
|||
|
|
return $OCF_ERR_GENERIC
|