resource-agents/SOURCES/bz1759115-aws-vpc-route53-1-update.patch

--- ClusterLabs-resource-agents-e711383f/heartbeat/aws-vpc-route53.in	2018-06-29 14:05:02.000000000 +0200
+++ /home/oalbrigt/src/resource-agents/heartbeat/aws-vpc-route53.in	2019-11-07 12:24:18.822111495 +0100
@@ -152,9 +152,15 @@
 END
 }
 
-ec2ip_validate() {
+r53_validate() {
 	ocf_log debug "function: validate"
 
+	# Check for required binaries
+	ocf_log debug "Checking for required binaries"
+	for command in curl dig; do
+		check_binary "$command"
+	done
+
 	# Full name
 	[[ -z "$OCF_RESKEY_fullname" ]] && ocf_log error "Full name parameter not set $OCF_RESKEY_fullname!" && exit $OCF_ERR_CONFIGURED
 
@@ -175,32 +181,111 @@
 	ocf_log debug "ok"
 
 	if [ -n "$OCF_RESKEY_profile" ]; then
-		AWS_PROFILE_OPT="--profile $OCF_RESKEY_profile"
+		AWS_PROFILE_OPT="--profile $OCF_RESKEY_profile --cli-connect-timeout 10"
 	else
-		AWS_PROFILE_OPT="--profile default"
+		AWS_PROFILE_OPT="--profile default --cli-connect-timeout 10"
 	fi
 
 	return $OCF_SUCCESS
 }
 
-ec2ip_monitor() {
-	ec2ip_validate
+r53_monitor() {
+	#
+	# For every start action the agent  will call Route53 API to check for DNS record
+	# otherwise it will try to get results directly bu querying the DNS using "dig".
+	# Due to complexity in some DNS architectures "dig" can fail, and if this happens
+	# the monitor will fallback to the Route53 API call.
+	#
+	# There will be no failure, failover or restart of the agent if the monitor operation fails
+	# hence we only return $OCF_SUCESS in this function
+	#
+	# In case of the monitor operation detects a wrong or non-existent Route53 DNS entry
+	# it will try to fix the existing one, or create it again
+	#
+	#
+	ARECORD=""
+	IPREGEX="^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}$"
+	r53_validate
 	ocf_log debug "Checking Route53 record sets"
-	IPADDRESS="$(ec2metadata aws ip | grep local-ipv4 | /usr/bin/awk '{ print $2 }')"
-	ARECORD="$(aws $AWS_PROFILE_OPT route53 list-resource-record-sets --hosted-zone-id $OCF_RESKEY_hostedzoneid --query "ResourceRecordSets[?Name=='$OCF_RESKEY_fullname']" | grep RESOURCERECORDS | /usr/bin/awk '{ print $2 }' )"
-	ocf_log debug "Found IP address: $ARECORD ."
-	if [ "${ARECORD}" == "${IPADDRESS}" ]; then
-		ocf_log debug "ARECORD $ARECORD found"
+	#
+	IPADDRESS="$(curl -s http://169.254.169.254/latest/meta-data/local-ipv4)"
+	#
+	if [ "$__OCF_ACTION" = "start" ] || ocf_is_probe ; then
+		#
+		cmd="aws $AWS_PROFILE_OPT route53 list-resource-record-sets --hosted-zone-id $OCF_RESKEY_hostedzoneid --query ResourceRecordSets[?Name=='$OCF_RESKEY_fullname']"
+		ocf_log info "Route53 Agent Starting or probing - executing monitoring API call: $cmd"
+		CLIRES="$($cmd 2>&1)"
+		rc=$?
+		ocf_log debug "awscli returned code: $rc"
+		if [ $rc -ne 0 ]; then
+			CLIRES=$(echo $CLIRES | grep -v '^$')
+			ocf_log warn "Route53 API returned an error: $CLIRES"
+			ocf_log warn "Skipping cluster action due to API call error"
+			return $OCF_ERR_GENERIC
+		fi
+		ARECORD=$(echo $CLIRES | grep RESOURCERECORDS | awk '{ print $5 }')
+		#
+		if ocf_is_probe; then
+			#
+			# Prevent R53 record change during probe
+			#
+			if [[ $ARECORD =~ $IPREGEX ]] && [ "$ARECORD" != "$IPADDRESS" ]; then
+				ocf_log debug "Route53 DNS record $ARECORD found at probing, disregarding"
+				return $OCF_NOT_RUNNING
+			fi
+		fi
+	else
+		#
+		cmd="dig +retries=3 +time=5 +short $OCF_RESKEY_fullname 2>/dev/null"
+		ocf_log info "executing monitoring command : $cmd"
+		ARECORD="$($cmd)"
+		rc=$?
+		ocf_log debug "dig return code: $rc"
+		#
+		if  [[ ! $ARECORD =~ $IPREGEX ]] || [ $rc -ne 0 ]; then
+			ocf_log info "Fallback to Route53 API query due to DNS resolution failure"
+			cmd="aws $AWS_PROFILE_OPT route53 list-resource-record-sets --hosted-zone-id $OCF_RESKEY_hostedzoneid --query ResourceRecordSets[?Name=='$OCF_RESKEY_fullname']"
+			ocf_log debug "executing monitoring API call: $cmd"
+			CLIRES="$($cmd 2>&1)"
+			rc=$?
+			ocf_log debug "awscli return code: $rc"
+			if [ $rc -ne 0 ]; then
+				CLIRES=$(echo $CLIRES | grep -v '^$')
+				ocf_log warn "Route53 API returned an error: $CLIRES"
+				ocf_log warn "Monitor skipping cluster action due to API call error"
+				return $OCF_SUCCESS
+			fi
+			ARECORD=$(echo $CLIRES | grep RESOURCERECORDS | awk '{ print $5 }')
+		fi
+		#
+	fi
+	ocf_log info "Route53 DNS record pointing $OCF_RESKEY_fullname to IP address $ARECORD"
+	#
+	if [ "$ARECORD" == "$IPADDRESS" ]; then
+		ocf_log info "Route53 DNS record $ARECORD found"
+		return $OCF_SUCCESS
+	elif [[ $ARECORD =~ $IPREGEX ]] && [ "$ARECORD" != "$IPADDRESS" ]; then
+		ocf_log info "Route53 DNS record points to a different host, setting DNS record on Route53 to this host"
+		_update_record "UPSERT" "$IPADDRESS"
 		return $OCF_SUCCESS
 	else
-		ocf_log debug "No ARECORD found"
-		return $OCF_NOT_RUNNING
+		ocf_log info "No Route53 DNS record found, setting DNS record on Route53 to this host"
+		_update_record "UPSERT" "$IPADDRESS"
+		return $OCF_SUCCESS
 	fi
 
 	return $OCF_SUCCESS
 }
 
 _update_record() {
+	#
+	# This function is the one that will actually execute Route53's API call
+	# and configure the DNS record using the correct API calls and parameters
+	#
+	# It creates a temporary JSON file under /tmp with the required API payload
+	#
+	# Failures in this function are critical and will cause the agent to fail
+	#
 	update_action="$1"
 	IPADDRESS="$2"
 	ocf_log info "Updating Route53 $OCF_RESKEY_hostedzoneid with $IPADDRESS for $OCF_RESKEY_fullname"
@@ -209,19 +294,19 @@
 		ocf_exit_reason "Failed to create temporary file for record update"
 		exit $OCF_ERR_GENERIC
 	fi
-	cat >>"${ROUTE53RECORD}" <<-EOF
+	cat >>"$ROUTE53RECORD" <<-EOF
 	{
 		  "Comment": "Update record to reflect new IP address for a system ",
 		  "Changes": [
 			  {
-				  "Action": "${update_action}",
+				  "Action": "$update_action",
 				  "ResourceRecordSet": {
-					  "Name": "${OCF_RESKEY_fullname}",
+					  "Name": "$OCF_RESKEY_fullname",
 					  "Type": "A",
-					  "TTL": ${OCF_RESKEY_ttl},
+					  "TTL": $OCF_RESKEY_ttl,
 					  "ResourceRecords": [
 						  {
-							  "Value": "${IPADDRESS}"
+							  "Value": "$IPADDRESS"
 						  }
 					  ]
 				  }
@@ -229,46 +314,53 @@
 		  ]
 	}
 	EOF
-	cmd="aws --profile ${OCF_RESKEY_profile} route53 change-resource-record-sets --hosted-zone-id ${OCF_RESKEY_hostedzoneid} \
-	  --change-batch file://${ROUTE53RECORD} "
+	cmd="aws --profile $OCF_RESKEY_profile route53 change-resource-record-sets --hosted-zone-id $OCF_RESKEY_hostedzoneid --change-batch file://$ROUTE53RECORD "
 	ocf_log debug "Executing command: $cmd"
-	CHANGEID=$($cmd | grep CHANGEINFO |	 /usr/bin/awk -F'\t' '{ print $3 }' )
-	ocf_log debug "Change id: ${CHANGEID}"
-	rmtempfile ${ROUTE53RECORD}
-	CHANGEID=$(echo $CHANGEID |cut -d'/' -f 3 |cut -d'"' -f 1 )
-	ocf_log debug "Change id: ${CHANGEID}"
+	CLIRES="$($cmd 2>&1)"
+	rc=$?
+	ocf_log debug "awscli returned code: $rc"
+	if [ $rc -ne 0 ]; then
+		CLIRES=$(echo $CLIRES | grep -v '^$')
+		ocf_log warn "Route53 API returned an error: $CLIRES"
+		ocf_log warn "Skipping cluster action due to API call error"
+		return $OCF_ERR_GENERIC
+	fi
+	CHANGEID=$(echo $CLIRES | awk '{ print $12 }')
+	ocf_log debug "Change id: $CHANGEID"
+	rmtempfile $ROUTE53RECORD
+	CHANGEID=$(echo $CHANGEID | cut -d'/' -f 3 | cut -d'"' -f 1 )
+	ocf_log debug "Change id: $CHANGEID"
 	STATUS="PENDING"
-	MYSECONDS=2
+	MYSECONDS=20
 	while [ "$STATUS" = 'PENDING' ]; do
-		sleep	${MYSECONDS}
-		STATUS="$(aws --profile ${OCF_RESKEY_profile} route53 get-change --id $CHANGEID | grep CHANGEINFO |  /usr/bin/awk -F'\t' '{ print $4 }' |cut -d'"' -f 2 )"
-		ocf_log debug "Waited for ${MYSECONDS} seconds and checked execution of Route 53 update status: ${STATUS} "
+		sleep $MYSECONDS
+		STATUS="$(aws --profile $OCF_RESKEY_profile route53 get-change --id $CHANGEID | grep CHANGEINFO | awk -F'\t' '{ print $4 }' |cut -d'"' -f 2 )"
+		ocf_log debug "Waited for $MYSECONDS seconds and checked execution of Route 53 update status: $STATUS "
 	done
 }
 
-ec2ip_stop() {
-	ocf_log info "Bringing down Route53 agent. (Will remove ARECORD)"
-	IPADDRESS="$(ec2metadata aws ip | grep local-ipv4 | /usr/bin/awk '{ print $2 }')"
-	ARECORD="$(aws $AWS_PROFILE_OPT route53 list-resource-record-sets --hosted-zone-id $OCF_RESKEY_hostedzoneid --query "ResourceRecordSets[?Name=='$OCF_RESKEY_fullname']" | grep RESOURCERECORDS | /usr/bin/awk '{ print $2 }' )"
-	ocf_log debug "Found IP address: $ARECORD ."
-	if [ ${ARECORD} != ${IPADDRESS} ]; then
-		ocf_log debug "No ARECORD found"
-		return $OCF_SUCCESS
-	else
-		# determine IP address
-		IPADDRESS="$(ec2metadata aws ip | grep local-ipv4 | /usr/bin/awk '{ print $2 }')"
-		# Patch file
-		ocf_log debug "Deleting IP address to ${IPADDRESS}"
-		return $OCF_SUCCESS
-	fi
-
-	_update_record "DELETE" "$IPADDRESS"
+r53_stop() {
+	#
+	# Stop operation doesn't perform any API call or try to remove the DNS record
+	# this mostly because this is not necessarily mandatory or desired
+	# the start and monitor functions will take care of changing the DNS record
+	# if the agent starts in a different cluster node
+	#
+	ocf_log info "Bringing down Route53 agent. (Will NOT remove Route53 DNS record)"
 	return $OCF_SUCCESS
 }
 
-ec2ip_start() {
-	IPADDRESS="$(ec2metadata aws ip | grep local-ipv4 | /usr/bin/awk '{ print $2 }')"
-	_update_record "UPSERT" "$IPADDRESS"
+r53_start() {
+	#
+	# Start agent and config DNS in Route53
+	#
+	ocf_log info "Starting Route53 DNS update...."
+	IPADDRESS="$(curl -s http://169.254.169.254/latest/meta-data/local-ipv4)"
+	r53_monitor
+	if [ $? != $OCF_SUCCESS ]; then
+		ocf_log info "Could not start agent - check configurations"
+		return $OCF_ERR_GENERIC
+	fi
 	return $OCF_SUCCESS
 }
 
@@ -284,16 +376,16 @@
 		exit $OCF_SUCCESS
 		;;
 	monitor)
-		ec2ip_monitor
+		r53_monitor
 		;;
 	stop)
-		ec2ip_stop
+		r53_stop
 		;;
 	validate-all)
-		ec2ip_validate
+		r53_validate
 		;;
 	start)
-		ec2ip_start
+		r53_start
 		;;
 	*)
 		usage