fence-agents/bz1816203-fence_aws-1-fix-race-condition.patch

From 1742baf17954c58a84b9c668a617bac78303ce95 Mon Sep 17 00:00:00 2001
From: Guilherme Felix <fguilher@amazon.com>
Date: Tue, 17 Mar 2020 13:18:38 +0000
Subject: [PATCH 1/9] fence_aws: Fix fence race condition by checking local
 instance status

---
 agents/aws/fence_aws.py | 37 ++++++++++++++++++++++++++++++++++++-
 1 file changed, 36 insertions(+), 1 deletion(-)

diff --git a/agents/aws/fence_aws.py b/agents/aws/fence_aws.py
index 4a4d9de2..f37f68d6 100644
--- a/agents/aws/fence_aws.py
+++ b/agents/aws/fence_aws.py
@@ -3,6 +3,7 @@
 import sys, re
 import logging
 import atexit
+import requests
 sys.path.append("@FENCEAGENTSLIBDIR@")
 from fencing import *
 from fencing import fail, fail_usage, run_delay, EC_STATUS
@@ -10,6 +11,17 @@
 import boto3
 from botocore.exceptions import ClientError, EndpointConnectionError, NoRegionError
 
+def get_instance_id():
+	try:
+		r = requests.get('http://169.254.169.254/latest/meta-data/instance-id')
+		return r.content
+	except HTTPError as http_err:
+		logging.error('HTTP error occurred while trying to access EC2 metadata server: %s', http_err)
+	except Exception as err:
+		logging.error('A fatal error occurred while trying to access EC2 metadata server: %s', err)
+	return None
+	
+
 def get_nodes_list(conn, options):
 	result = {}
 	try:
@@ -45,10 +57,33 @@ def get_power_status(conn, options):
 		logging.error("Failed to get power status: %s", e)
 		fail(EC_STATUS)
 
+def get_self_power_status(conn, options):
+	try:
+		instance = conn.instances.filter(Filters=[{"Name": "instance-id", "Values": [instance_id]}])
+		state = list(instance)[0].state["Name"]
+		if state == "running":
+			logging.debug("Captured my (%s) state and it %s - returning OK - Proceeding with fencing",instance_id,state.upper())
+			return "ok"
+		else:
+			logging.debug("Captured my (%s) state it is %s - returning Alert - Unable to fence other nodes",instance_id,state.upper())
+			return "alert"
+	
+	except ClientError:
+		fail_usage("Failed: Incorrect Access Key or Secret Key.")
+	except EndpointConnectionError:
+		fail_usage("Failed: Incorrect Region.")
+	except IndexError:
+		return "fail"
+
 def set_power_status(conn, options):
+	my_instance = get_instance_id()
 	try:
 		if (options["--action"]=="off"):
-			conn.instances.filter(InstanceIds=[options["--plug"]]).stop(Force=True)
+			if (get_self_power_status(conn,myinstance) == "ok"):
+				logging.info("Called StopInstance API call for %s", options["--plug"])
+				conn.instances.filter(InstanceIds=[options["--plug"]]).stop(Force=True)
+			else:
+				logging.info("Skipping fencing as instance is not in running status")
 		elif (options["--action"]=="on"):
 			conn.instances.filter(InstanceIds=[options["--plug"]]).start()
 	except Exception as e:

From 45e429b3132ebc9e78121c3fbb15f0bf46845a59 Mon Sep 17 00:00:00 2001
From: Guilherme Felix <fguilher@amazon.com>
Date: Tue, 17 Mar 2020 13:28:34 +0000
Subject: [PATCH 2/9] fence_aws: Use local logger and improve logging
 experience

---
 agents/aws/fence_aws.py | 34 ++++++++++++++++++++++++++--------
 1 file changed, 26 insertions(+), 8 deletions(-)

diff --git a/agents/aws/fence_aws.py b/agents/aws/fence_aws.py
index f37f68d6..b0b6685a 100644
--- a/agents/aws/fence_aws.py
+++ b/agents/aws/fence_aws.py
@@ -6,7 +6,7 @@
 import requests
 sys.path.append("@FENCEAGENTSLIBDIR@")
 from fencing import *
-from fencing import fail, fail_usage, run_delay, EC_STATUS
+from fencing import fail, fail_usage, run_delay, EC_STATUS, SyslogLibHandler
 
 import boto3
 from botocore.exceptions import ClientError, EndpointConnectionError, NoRegionError
@@ -16,13 +16,14 @@ def get_instance_id():
 		r = requests.get('http://169.254.169.254/latest/meta-data/instance-id')
 		return r.content
 	except HTTPError as http_err:
-		logging.error('HTTP error occurred while trying to access EC2 metadata server: %s', http_err)
+		logger.error('HTTP error occurred while trying to access EC2 metadata server: %s', http_err)
 	except Exception as err:
-		logging.error('A fatal error occurred while trying to access EC2 metadata server: %s', err)
+		logger.error('A fatal error occurred while trying to access EC2 metadata server: %s', err)
 	return None
 	
 
 def get_nodes_list(conn, options):
+	logger.info("Starting monitor operation")
 	result = {}
 	try:
 		for instance in conn.instances.all():
@@ -32,14 +33,16 @@ def get_nodes_list(conn, options):
 	except EndpointConnectionError:
 		fail_usage("Failed: Incorrect Region.")
 	except Exception as e:
-		logging.error("Failed to get node list: %s", e)
-
+		logger.error("Failed to get node list: %s", e)
+	logger.debug("Monitor operation OK: %s",result)
 	return result
 
 def get_power_status(conn, options):
+	logger.debug("Starting status operation")
 	try:
 		instance = conn.instances.filter(Filters=[{"Name": "instance-id", "Values": [options["--plug"]]}])
 		state = list(instance)[0].state["Name"]
+		logger.info("Status operation for EC2 instance %s returned state: %s",options["--plug"],state.upper())
 		if state == "running":
 			return "on"
 		elif state == "stopped":
@@ -80,14 +83,14 @@ def set_power_status(conn, options):
 	try:
 		if (options["--action"]=="off"):
 			if (get_self_power_status(conn,myinstance) == "ok"):
-				logging.info("Called StopInstance API call for %s", options["--plug"])
 				conn.instances.filter(InstanceIds=[options["--plug"]]).stop(Force=True)
+				logger.info("Called StopInstance API call for %s", options["--plug"])
 			else:
-				logging.info("Skipping fencing as instance is not in running status")
+				logger.info("Skipping fencing as instance is not in running status")
 		elif (options["--action"]=="on"):
 			conn.instances.filter(InstanceIds=[options["--plug"]]).start()
 	except Exception as e:
-		logging.error("Failed to power %s %s: %s", \
+		logger.error("Failed to power %s %s: %s", \
 				options["--action"], options["--plug"], e)
 
 def define_new_opts():
@@ -142,6 +145,13 @@ def main():
 
 	run_delay(options)
 
+	if options.get("--verbose") is not None:
+		lh = logging.FileHandler('/var/log/fence_aws_debug.log')
+		logger.addHandler(lh)
+		lhf = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
+		lh.setFormatter(lhf)
+		logger.setLevel(logging.DEBUG)
+
 	region = options.get("--region")
 	access_key = options.get("--access-key")
 	secret_key = options.get("--secret-key")
@@ -157,4 +167,12 @@ def main():
 	sys.exit(result)
 
 if __name__ == "__main__":
+
+	logger = logging.getLogger("fence_aws")
+	logger.propagate = False
+	logger.setLevel(logging.INFO)
+	logger.addHandler(SyslogLibHandler())
+	logger.getLogger('botocore.vendored').propagate = False
+	
+
 	main()

From 00569921597b8007c67296ab8332747baf1e6fae Mon Sep 17 00:00:00 2001
From: Guilherme Felix <fguilher@amazon.com>
Date: Tue, 17 Mar 2020 13:33:02 +0000
Subject: [PATCH 3/9] fence_aws: Decouple boto3 and botocore debug logging from
 local logging

---
 agents/aws/fence_aws.py | 26 +++++++++++++++++++++++++-
 1 file changed, 25 insertions(+), 1 deletion(-)

diff --git a/agents/aws/fence_aws.py b/agents/aws/fence_aws.py
index b0b6685a..11714315 100644
--- a/agents/aws/fence_aws.py
+++ b/agents/aws/fence_aws.py
@@ -118,18 +118,27 @@ def define_new_opts():
 		"required" : "0",
 		"order" : 4
 	}
+	all_opt["boto3_debug"] = {
+		"getopt" : "b:",
+		"longopt" : "boto3_debug",
+		"help" : "-b, --boto3_debug=on|off      Boto3 and Botocore library debug logging",
+		"shortdesc": "Boto Lib debug",
+		"required": "0",
+		"order": 5
+	}
 
 # Main agent method
 def main():
 	conn = None
 
-	device_opt = ["port", "no_password", "region", "access_key", "secret_key"]
+	device_opt = ["port", "no_password", "region", "access_key", "secret_key", "boto3_debug"]
 
 	atexit.register(atexit_handler)
 
 	define_new_opts()
 
 	all_opt["power_timeout"]["default"] = "60"
+	all_opt["boto3_debug"]["default"] = "off"
 
 	options = check_input(device_opt, process_input(device_opt))
 
@@ -151,6 +160,21 @@ def main():
 		lhf = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
 		lh.setFormatter(lhf)
 		logger.setLevel(logging.DEBUG)
+	
+	if options["--boto3_debug"] != "on":
+		boto3.set_stream_logger('boto3',logging.INFO)
+		boto3.set_stream_logger('botocore',logging.INFO)
+		logging.getLogger('botocore').propagate = False
+		logging.getLogger('boto3').propagate = False
+	else:
+		log_format = logging.Formatter('%(asctime)s %(name)-12s %(levelname)-8s %(message)s')
+		logging.getLogger('botocore').propagate = False
+		logging.getLogger('boto3').propagate = False
+		fdh = logging.FileHandler('/var/log/fence_aws_boto3.log')
+		fdh.setFormatter(log_format)
+		logging.getLogger('boto3').addHandler(fdh)
+		logging.getLogger('botocore').addHandler(fdh)
+		logging.debug("Boto debug level is %s and sending debug info to /var/log/fence_aws_boto3.log", options["--boto3_debug"])
 
 	region = options.get("--region")
 	access_key = options.get("--access-key")

From ed309bd51dfd5e0fed30156e7a312d5b5a8f4bd4 Mon Sep 17 00:00:00 2001
From: Guilherme Felix <fguilher@amazon.com>
Date: Thu, 19 Mar 2020 16:02:47 +0000
Subject: [PATCH 4/9] fence_aws: Fix typos and variable names

---
 agents/aws/fence_aws.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/agents/aws/fence_aws.py b/agents/aws/fence_aws.py
index 11714315..207631e8 100644
--- a/agents/aws/fence_aws.py
+++ b/agents/aws/fence_aws.py
@@ -60,7 +60,7 @@ def get_power_status(conn, options):
 		logging.error("Failed to get power status: %s", e)
 		fail(EC_STATUS)
 
-def get_self_power_status(conn, options):
+def get_self_power_status(conn, instance_id):
 	try:
 		instance = conn.instances.filter(Filters=[{"Name": "instance-id", "Values": [instance_id]}])
 		state = list(instance)[0].state["Name"]
@@ -82,7 +82,7 @@ def set_power_status(conn, options):
 	my_instance = get_instance_id()
 	try:
 		if (options["--action"]=="off"):
-			if (get_self_power_status(conn,myinstance) == "ok"):
+			if (get_self_power_status(conn,my_instance) == "ok"):
 				conn.instances.filter(InstanceIds=[options["--plug"]]).stop(Force=True)
 				logger.info("Called StopInstance API call for %s", options["--plug"])
 			else:
@@ -196,7 +196,7 @@ def main():
 	logger.propagate = False
 	logger.setLevel(logging.INFO)
 	logger.addHandler(SyslogLibHandler())
-	logger.getLogger('botocore.vendored').propagate = False
+	logging.getLogger('botocore.vendored').propagate = False
 	
 
 	main()

From 624c652a95a676286af408898186186b7d7fcf55 Mon Sep 17 00:00:00 2001
From: Guilherme Felix <fguilher@amazon.com>
Date: Thu, 19 Mar 2020 16:58:45 +0000
Subject: [PATCH 5/9] fence_aws: Missing brackets on boto3_debug metadata

---
 agents/aws/fence_aws.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/agents/aws/fence_aws.py b/agents/aws/fence_aws.py
index 207631e8..8916f4a0 100644
--- a/agents/aws/fence_aws.py
+++ b/agents/aws/fence_aws.py
@@ -121,7 +121,7 @@ def define_new_opts():
 	all_opt["boto3_debug"] = {
 		"getopt" : "b:",
 		"longopt" : "boto3_debug",
-		"help" : "-b, --boto3_debug=on|off      Boto3 and Botocore library debug logging",
+		"help" : "-b, --boto3_debug=[on|off]      Boto3 and Botocore library debug logging",
 		"shortdesc": "Boto Lib debug",
 		"required": "0",
 		"order": 5

From 7c641a6885c4ab67b7739a43892d92d95a6f566c Mon Sep 17 00:00:00 2001
From: Guilherme Felix <fguilher@amazon.com>
Date: Thu, 19 Mar 2020 17:04:31 +0000
Subject: [PATCH 6/9] fence_aws: Fix travis build #1

---
 agents/aws/fence_aws.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/agents/aws/fence_aws.py b/agents/aws/fence_aws.py
index 8916f4a0..f41a47e4 100644
--- a/agents/aws/fence_aws.py
+++ b/agents/aws/fence_aws.py
@@ -121,7 +121,7 @@ def define_new_opts():
 	all_opt["boto3_debug"] = {
 		"getopt" : "b:",
 		"longopt" : "boto3_debug",
-		"help" : "-b, --boto3_debug=[on|off]      Boto3 and Botocore library debug logging",
+		"help" : "-b, --boto3_debug=[option]      Boto3 and Botocore library debug logging",
 		"shortdesc": "Boto Lib debug",
 		"required": "0",
 		"order": 5

From 257af7ccc9789646adc7abf1e7dbac744b756071 Mon Sep 17 00:00:00 2001
From: Guilherme Felix <fguilher@amazon.com>
Date: Fri, 20 Mar 2020 10:59:56 +0000
Subject: [PATCH 7/9] fence_aws: Updated metadata XML file

---
 tests/data/metadata/fence_aws.xml | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/tests/data/metadata/fence_aws.xml b/tests/data/metadata/fence_aws.xml
index 5e5d5d99..acfebb61 100644
--- a/tests/data/metadata/fence_aws.xml
+++ b/tests/data/metadata/fence_aws.xml
@@ -36,6 +36,11 @@ For instructions see: https://boto3.readthedocs.io/en/latest/guide/quickstart.ht
 		<content type="string"  />
 		<shortdesc lang="en">Secret Key.</shortdesc>
 	</parameter>
+	<parameter name="boto3_debug" unique="0" required="0">
+		<getopt mixed="-b, --boto3_debug=[option]" />
+		<content type="string" default="off"  />
+		<shortdesc lang="en">Boto Lib debug</shortdesc>
+	</parameter>
 	<parameter name="quiet" unique="0" required="0">
 		<getopt mixed="-q, --quiet" />
 		<content type="boolean"  />

From 8f78bc19356b5e07d0021aaf7da3fc4e712e00f0 Mon Sep 17 00:00:00 2001
From: Guilherme Felix <fguilher@amazon.com>
Date: Fri, 20 Mar 2020 12:13:16 +0000
Subject: [PATCH 8/9] fence_aws: Moving logger config next to import statements
 for visibility

---
 agents/aws/fence_aws.py | 13 ++++++-------
 1 file changed, 6 insertions(+), 7 deletions(-)

diff --git a/agents/aws/fence_aws.py b/agents/aws/fence_aws.py
index f41a47e4..72fb8843 100644
--- a/agents/aws/fence_aws.py
+++ b/agents/aws/fence_aws.py
@@ -11,6 +11,12 @@
 import boto3
 from botocore.exceptions import ClientError, EndpointConnectionError, NoRegionError
 
+logger = logging.getLogger("fence_aws")
+logger.propagate = False
+logger.setLevel(logging.INFO)
+logger.addHandler(SyslogLibHandler())
+logging.getLogger('botocore.vendored').propagate = False
+	
 def get_instance_id():
 	try:
 		r = requests.get('http://169.254.169.254/latest/meta-data/instance-id')
@@ -192,11 +198,4 @@ def main():
 
 if __name__ == "__main__":
 
-	logger = logging.getLogger("fence_aws")
-	logger.propagate = False
-	logger.setLevel(logging.INFO)
-	logger.addHandler(SyslogLibHandler())
-	logging.getLogger('botocore.vendored').propagate = False
-	
-
 	main()

From 570a05c425fe55008c8892ebaad8a73d36143909 Mon Sep 17 00:00:00 2001
From: Guilherme Felix <fguilher@amazon.com>
Date: Fri, 20 Mar 2020 14:17:55 +0000
Subject: [PATCH 9/9] fence_aws: Remove empty line

---
 agents/aws/fence_aws.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/agents/aws/fence_aws.py b/agents/aws/fence_aws.py
index 72fb8843..ed55f390 100644
--- a/agents/aws/fence_aws.py
+++ b/agents/aws/fence_aws.py
@@ -197,5 +197,4 @@ def main():
 	sys.exit(result)
 
 if __name__ == "__main__":
-
-	main()
+	main()
\ No newline at end of file