422 lines
14 KiB
Diff
422 lines
14 KiB
Diff
From 1742baf17954c58a84b9c668a617bac78303ce95 Mon Sep 17 00:00:00 2001
|
|
From: Guilherme Felix <fguilher@amazon.com>
|
|
Date: Tue, 17 Mar 2020 13:18:38 +0000
|
|
Subject: [PATCH 1/9] fence_aws: Fix fence race condition by checking local
|
|
instance status
|
|
|
|
---
|
|
agents/aws/fence_aws.py | 37 ++++++++++++++++++++++++++++++++++++-
|
|
1 file changed, 36 insertions(+), 1 deletion(-)
|
|
|
|
diff --git a/agents/aws/fence_aws.py b/agents/aws/fence_aws.py
|
|
index 4a4d9de2..f37f68d6 100644
|
|
--- a/agents/aws/fence_aws.py
|
|
+++ b/agents/aws/fence_aws.py
|
|
@@ -3,6 +3,7 @@
|
|
import sys, re
|
|
import logging
|
|
import atexit
|
|
+import requests
|
|
sys.path.append("@FENCEAGENTSLIBDIR@")
|
|
from fencing import *
|
|
from fencing import fail, fail_usage, run_delay, EC_STATUS
|
|
@@ -10,6 +11,17 @@
|
|
import boto3
|
|
from botocore.exceptions import ClientError, EndpointConnectionError, NoRegionError
|
|
|
|
+def get_instance_id():
|
|
+ try:
|
|
+ r = requests.get('http://169.254.169.254/latest/meta-data/instance-id')
|
|
+ return r.content
|
|
+ except HTTPError as http_err:
|
|
+ logging.error('HTTP error occurred while trying to access EC2 metadata server: %s', http_err)
|
|
+ except Exception as err:
|
|
+ logging.error('A fatal error occurred while trying to access EC2 metadata server: %s', err)
|
|
+ return None
|
|
+
|
|
+
|
|
def get_nodes_list(conn, options):
|
|
result = {}
|
|
try:
|
|
@@ -45,10 +57,33 @@ def get_power_status(conn, options):
|
|
logging.error("Failed to get power status: %s", e)
|
|
fail(EC_STATUS)
|
|
|
|
+def get_self_power_status(conn, options):
|
|
+ try:
|
|
+ instance = conn.instances.filter(Filters=[{"Name": "instance-id", "Values": [instance_id]}])
|
|
+ state = list(instance)[0].state["Name"]
|
|
+ if state == "running":
|
|
+ logging.debug("Captured my (%s) state and it %s - returning OK - Proceeding with fencing",instance_id,state.upper())
|
|
+ return "ok"
|
|
+ else:
|
|
+ logging.debug("Captured my (%s) state it is %s - returning Alert - Unable to fence other nodes",instance_id,state.upper())
|
|
+ return "alert"
|
|
+
|
|
+ except ClientError:
|
|
+ fail_usage("Failed: Incorrect Access Key or Secret Key.")
|
|
+ except EndpointConnectionError:
|
|
+ fail_usage("Failed: Incorrect Region.")
|
|
+ except IndexError:
|
|
+ return "fail"
|
|
+
|
|
def set_power_status(conn, options):
|
|
+ my_instance = get_instance_id()
|
|
try:
|
|
if (options["--action"]=="off"):
|
|
- conn.instances.filter(InstanceIds=[options["--plug"]]).stop(Force=True)
|
|
+ if (get_self_power_status(conn,myinstance) == "ok"):
|
|
+ logging.info("Called StopInstance API call for %s", options["--plug"])
|
|
+ conn.instances.filter(InstanceIds=[options["--plug"]]).stop(Force=True)
|
|
+ else:
|
|
+ logging.info("Skipping fencing as instance is not in running status")
|
|
elif (options["--action"]=="on"):
|
|
conn.instances.filter(InstanceIds=[options["--plug"]]).start()
|
|
except Exception as e:
|
|
|
|
From 45e429b3132ebc9e78121c3fbb15f0bf46845a59 Mon Sep 17 00:00:00 2001
|
|
From: Guilherme Felix <fguilher@amazon.com>
|
|
Date: Tue, 17 Mar 2020 13:28:34 +0000
|
|
Subject: [PATCH 2/9] fence_aws: Use local logger and improve logging
|
|
experience
|
|
|
|
---
|
|
agents/aws/fence_aws.py | 34 ++++++++++++++++++++++++++--------
|
|
1 file changed, 26 insertions(+), 8 deletions(-)
|
|
|
|
diff --git a/agents/aws/fence_aws.py b/agents/aws/fence_aws.py
|
|
index f37f68d6..b0b6685a 100644
|
|
--- a/agents/aws/fence_aws.py
|
|
+++ b/agents/aws/fence_aws.py
|
|
@@ -6,7 +6,7 @@
|
|
import requests
|
|
sys.path.append("@FENCEAGENTSLIBDIR@")
|
|
from fencing import *
|
|
-from fencing import fail, fail_usage, run_delay, EC_STATUS
|
|
+from fencing import fail, fail_usage, run_delay, EC_STATUS, SyslogLibHandler
|
|
|
|
import boto3
|
|
from botocore.exceptions import ClientError, EndpointConnectionError, NoRegionError
|
|
@@ -16,13 +16,14 @@ def get_instance_id():
|
|
r = requests.get('http://169.254.169.254/latest/meta-data/instance-id')
|
|
return r.content
|
|
except HTTPError as http_err:
|
|
- logging.error('HTTP error occurred while trying to access EC2 metadata server: %s', http_err)
|
|
+ logger.error('HTTP error occurred while trying to access EC2 metadata server: %s', http_err)
|
|
except Exception as err:
|
|
- logging.error('A fatal error occurred while trying to access EC2 metadata server: %s', err)
|
|
+ logger.error('A fatal error occurred while trying to access EC2 metadata server: %s', err)
|
|
return None
|
|
|
|
|
|
def get_nodes_list(conn, options):
|
|
+ logger.info("Starting monitor operation")
|
|
result = {}
|
|
try:
|
|
for instance in conn.instances.all():
|
|
@@ -32,14 +33,16 @@ def get_nodes_list(conn, options):
|
|
except EndpointConnectionError:
|
|
fail_usage("Failed: Incorrect Region.")
|
|
except Exception as e:
|
|
- logging.error("Failed to get node list: %s", e)
|
|
-
|
|
+ logger.error("Failed to get node list: %s", e)
|
|
+ logger.debug("Monitor operation OK: %s",result)
|
|
return result
|
|
|
|
def get_power_status(conn, options):
|
|
+ logger.debug("Starting status operation")
|
|
try:
|
|
instance = conn.instances.filter(Filters=[{"Name": "instance-id", "Values": [options["--plug"]]}])
|
|
state = list(instance)[0].state["Name"]
|
|
+ logger.info("Status operation for EC2 instance %s returned state: %s",options["--plug"],state.upper())
|
|
if state == "running":
|
|
return "on"
|
|
elif state == "stopped":
|
|
@@ -80,14 +83,14 @@ def set_power_status(conn, options):
|
|
try:
|
|
if (options["--action"]=="off"):
|
|
if (get_self_power_status(conn,myinstance) == "ok"):
|
|
- logging.info("Called StopInstance API call for %s", options["--plug"])
|
|
conn.instances.filter(InstanceIds=[options["--plug"]]).stop(Force=True)
|
|
+ logger.info("Called StopInstance API call for %s", options["--plug"])
|
|
else:
|
|
- logging.info("Skipping fencing as instance is not in running status")
|
|
+ logger.info("Skipping fencing as instance is not in running status")
|
|
elif (options["--action"]=="on"):
|
|
conn.instances.filter(InstanceIds=[options["--plug"]]).start()
|
|
except Exception as e:
|
|
- logging.error("Failed to power %s %s: %s", \
|
|
+ logger.error("Failed to power %s %s: %s", \
|
|
options["--action"], options["--plug"], e)
|
|
|
|
def define_new_opts():
|
|
@@ -142,6 +145,13 @@ def main():
|
|
|
|
run_delay(options)
|
|
|
|
+ if options.get("--verbose") is not None:
|
|
+ lh = logging.FileHandler('/var/log/fence_aws_debug.log')
|
|
+ logger.addHandler(lh)
|
|
+ lhf = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
|
|
+ lh.setFormatter(lhf)
|
|
+ logger.setLevel(logging.DEBUG)
|
|
+
|
|
region = options.get("--region")
|
|
access_key = options.get("--access-key")
|
|
secret_key = options.get("--secret-key")
|
|
@@ -157,4 +167,12 @@ def main():
|
|
sys.exit(result)
|
|
|
|
if __name__ == "__main__":
|
|
+
|
|
+ logger = logging.getLogger("fence_aws")
|
|
+ logger.propagate = False
|
|
+ logger.setLevel(logging.INFO)
|
|
+ logger.addHandler(SyslogLibHandler())
|
|
+ logger.getLogger('botocore.vendored').propagate = False
|
|
+
|
|
+
|
|
main()
|
|
|
|
From 00569921597b8007c67296ab8332747baf1e6fae Mon Sep 17 00:00:00 2001
|
|
From: Guilherme Felix <fguilher@amazon.com>
|
|
Date: Tue, 17 Mar 2020 13:33:02 +0000
|
|
Subject: [PATCH 3/9] fence_aws: Decouple boto3 and botocore debug logging from
|
|
local logging
|
|
|
|
---
|
|
agents/aws/fence_aws.py | 26 +++++++++++++++++++++++++-
|
|
1 file changed, 25 insertions(+), 1 deletion(-)
|
|
|
|
diff --git a/agents/aws/fence_aws.py b/agents/aws/fence_aws.py
|
|
index b0b6685a..11714315 100644
|
|
--- a/agents/aws/fence_aws.py
|
|
+++ b/agents/aws/fence_aws.py
|
|
@@ -118,18 +118,27 @@ def define_new_opts():
|
|
"required" : "0",
|
|
"order" : 4
|
|
}
|
|
+ all_opt["boto3_debug"] = {
|
|
+ "getopt" : "b:",
|
|
+ "longopt" : "boto3_debug",
|
|
+ "help" : "-b, --boto3_debug=on|off Boto3 and Botocore library debug logging",
|
|
+ "shortdesc": "Boto Lib debug",
|
|
+ "required": "0",
|
|
+ "order": 5
|
|
+ }
|
|
|
|
# Main agent method
|
|
def main():
|
|
conn = None
|
|
|
|
- device_opt = ["port", "no_password", "region", "access_key", "secret_key"]
|
|
+ device_opt = ["port", "no_password", "region", "access_key", "secret_key", "boto3_debug"]
|
|
|
|
atexit.register(atexit_handler)
|
|
|
|
define_new_opts()
|
|
|
|
all_opt["power_timeout"]["default"] = "60"
|
|
+ all_opt["boto3_debug"]["default"] = "off"
|
|
|
|
options = check_input(device_opt, process_input(device_opt))
|
|
|
|
@@ -151,6 +160,21 @@ def main():
|
|
lhf = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
|
|
lh.setFormatter(lhf)
|
|
logger.setLevel(logging.DEBUG)
|
|
+
|
|
+ if options["--boto3_debug"] != "on":
|
|
+ boto3.set_stream_logger('boto3',logging.INFO)
|
|
+ boto3.set_stream_logger('botocore',logging.INFO)
|
|
+ logging.getLogger('botocore').propagate = False
|
|
+ logging.getLogger('boto3').propagate = False
|
|
+ else:
|
|
+ log_format = logging.Formatter('%(asctime)s %(name)-12s %(levelname)-8s %(message)s')
|
|
+ logging.getLogger('botocore').propagate = False
|
|
+ logging.getLogger('boto3').propagate = False
|
|
+ fdh = logging.FileHandler('/var/log/fence_aws_boto3.log')
|
|
+ fdh.setFormatter(log_format)
|
|
+ logging.getLogger('boto3').addHandler(fdh)
|
|
+ logging.getLogger('botocore').addHandler(fdh)
|
|
+ logging.debug("Boto debug level is %s and sending debug info to /var/log/fence_aws_boto3.log", options["--boto3_debug"])
|
|
|
|
region = options.get("--region")
|
|
access_key = options.get("--access-key")
|
|
|
|
From ed309bd51dfd5e0fed30156e7a312d5b5a8f4bd4 Mon Sep 17 00:00:00 2001
|
|
From: Guilherme Felix <fguilher@amazon.com>
|
|
Date: Thu, 19 Mar 2020 16:02:47 +0000
|
|
Subject: [PATCH 4/9] fence_aws: Fix typos and variable names
|
|
|
|
---
|
|
agents/aws/fence_aws.py | 6 +++---
|
|
1 file changed, 3 insertions(+), 3 deletions(-)
|
|
|
|
diff --git a/agents/aws/fence_aws.py b/agents/aws/fence_aws.py
|
|
index 11714315..207631e8 100644
|
|
--- a/agents/aws/fence_aws.py
|
|
+++ b/agents/aws/fence_aws.py
|
|
@@ -60,7 +60,7 @@ def get_power_status(conn, options):
|
|
logging.error("Failed to get power status: %s", e)
|
|
fail(EC_STATUS)
|
|
|
|
-def get_self_power_status(conn, options):
|
|
+def get_self_power_status(conn, instance_id):
|
|
try:
|
|
instance = conn.instances.filter(Filters=[{"Name": "instance-id", "Values": [instance_id]}])
|
|
state = list(instance)[0].state["Name"]
|
|
@@ -82,7 +82,7 @@ def set_power_status(conn, options):
|
|
my_instance = get_instance_id()
|
|
try:
|
|
if (options["--action"]=="off"):
|
|
- if (get_self_power_status(conn,myinstance) == "ok"):
|
|
+ if (get_self_power_status(conn,my_instance) == "ok"):
|
|
conn.instances.filter(InstanceIds=[options["--plug"]]).stop(Force=True)
|
|
logger.info("Called StopInstance API call for %s", options["--plug"])
|
|
else:
|
|
@@ -196,7 +196,7 @@ def main():
|
|
logger.propagate = False
|
|
logger.setLevel(logging.INFO)
|
|
logger.addHandler(SyslogLibHandler())
|
|
- logger.getLogger('botocore.vendored').propagate = False
|
|
+ logging.getLogger('botocore.vendored').propagate = False
|
|
|
|
|
|
main()
|
|
|
|
From 624c652a95a676286af408898186186b7d7fcf55 Mon Sep 17 00:00:00 2001
|
|
From: Guilherme Felix <fguilher@amazon.com>
|
|
Date: Thu, 19 Mar 2020 16:58:45 +0000
|
|
Subject: [PATCH 5/9] fence_aws: Missing brackets on boto3_debug metadata
|
|
|
|
---
|
|
agents/aws/fence_aws.py | 2 +-
|
|
1 file changed, 1 insertion(+), 1 deletion(-)
|
|
|
|
diff --git a/agents/aws/fence_aws.py b/agents/aws/fence_aws.py
|
|
index 207631e8..8916f4a0 100644
|
|
--- a/agents/aws/fence_aws.py
|
|
+++ b/agents/aws/fence_aws.py
|
|
@@ -121,7 +121,7 @@ def define_new_opts():
|
|
all_opt["boto3_debug"] = {
|
|
"getopt" : "b:",
|
|
"longopt" : "boto3_debug",
|
|
- "help" : "-b, --boto3_debug=on|off Boto3 and Botocore library debug logging",
|
|
+ "help" : "-b, --boto3_debug=[on|off] Boto3 and Botocore library debug logging",
|
|
"shortdesc": "Boto Lib debug",
|
|
"required": "0",
|
|
"order": 5
|
|
|
|
From 7c641a6885c4ab67b7739a43892d92d95a6f566c Mon Sep 17 00:00:00 2001
|
|
From: Guilherme Felix <fguilher@amazon.com>
|
|
Date: Thu, 19 Mar 2020 17:04:31 +0000
|
|
Subject: [PATCH 6/9] fence_aws: Fix travis build #1
|
|
|
|
---
|
|
agents/aws/fence_aws.py | 2 +-
|
|
1 file changed, 1 insertion(+), 1 deletion(-)
|
|
|
|
diff --git a/agents/aws/fence_aws.py b/agents/aws/fence_aws.py
|
|
index 8916f4a0..f41a47e4 100644
|
|
--- a/agents/aws/fence_aws.py
|
|
+++ b/agents/aws/fence_aws.py
|
|
@@ -121,7 +121,7 @@ def define_new_opts():
|
|
all_opt["boto3_debug"] = {
|
|
"getopt" : "b:",
|
|
"longopt" : "boto3_debug",
|
|
- "help" : "-b, --boto3_debug=[on|off] Boto3 and Botocore library debug logging",
|
|
+ "help" : "-b, --boto3_debug=[option] Boto3 and Botocore library debug logging",
|
|
"shortdesc": "Boto Lib debug",
|
|
"required": "0",
|
|
"order": 5
|
|
|
|
From 257af7ccc9789646adc7abf1e7dbac744b756071 Mon Sep 17 00:00:00 2001
|
|
From: Guilherme Felix <fguilher@amazon.com>
|
|
Date: Fri, 20 Mar 2020 10:59:56 +0000
|
|
Subject: [PATCH 7/9] fence_aws: Updated metadata XML file
|
|
|
|
---
|
|
tests/data/metadata/fence_aws.xml | 5 +++++
|
|
1 file changed, 5 insertions(+)
|
|
|
|
diff --git a/tests/data/metadata/fence_aws.xml b/tests/data/metadata/fence_aws.xml
|
|
index 5e5d5d99..acfebb61 100644
|
|
--- a/tests/data/metadata/fence_aws.xml
|
|
+++ b/tests/data/metadata/fence_aws.xml
|
|
@@ -36,6 +36,11 @@ For instructions see: https://boto3.readthedocs.io/en/latest/guide/quickstart.ht
|
|
<content type="string" />
|
|
<shortdesc lang="en">Secret Key.</shortdesc>
|
|
</parameter>
|
|
+ <parameter name="boto3_debug" unique="0" required="0">
|
|
+ <getopt mixed="-b, --boto3_debug=[option]" />
|
|
+ <content type="string" default="off" />
|
|
+ <shortdesc lang="en">Boto Lib debug</shortdesc>
|
|
+ </parameter>
|
|
<parameter name="quiet" unique="0" required="0">
|
|
<getopt mixed="-q, --quiet" />
|
|
<content type="boolean" />
|
|
|
|
From 8f78bc19356b5e07d0021aaf7da3fc4e712e00f0 Mon Sep 17 00:00:00 2001
|
|
From: Guilherme Felix <fguilher@amazon.com>
|
|
Date: Fri, 20 Mar 2020 12:13:16 +0000
|
|
Subject: [PATCH 8/9] fence_aws: Moving logger config next to import statements
|
|
for visibility
|
|
|
|
---
|
|
agents/aws/fence_aws.py | 13 ++++++-------
|
|
1 file changed, 6 insertions(+), 7 deletions(-)
|
|
|
|
diff --git a/agents/aws/fence_aws.py b/agents/aws/fence_aws.py
|
|
index f41a47e4..72fb8843 100644
|
|
--- a/agents/aws/fence_aws.py
|
|
+++ b/agents/aws/fence_aws.py
|
|
@@ -11,6 +11,12 @@
|
|
import boto3
|
|
from botocore.exceptions import ClientError, EndpointConnectionError, NoRegionError
|
|
|
|
+logger = logging.getLogger("fence_aws")
|
|
+logger.propagate = False
|
|
+logger.setLevel(logging.INFO)
|
|
+logger.addHandler(SyslogLibHandler())
|
|
+logging.getLogger('botocore.vendored').propagate = False
|
|
+
|
|
def get_instance_id():
|
|
try:
|
|
r = requests.get('http://169.254.169.254/latest/meta-data/instance-id')
|
|
@@ -192,11 +198,4 @@ def main():
|
|
|
|
if __name__ == "__main__":
|
|
|
|
- logger = logging.getLogger("fence_aws")
|
|
- logger.propagate = False
|
|
- logger.setLevel(logging.INFO)
|
|
- logger.addHandler(SyslogLibHandler())
|
|
- logging.getLogger('botocore.vendored').propagate = False
|
|
-
|
|
-
|
|
main()
|
|
|
|
From 570a05c425fe55008c8892ebaad8a73d36143909 Mon Sep 17 00:00:00 2001
|
|
From: Guilherme Felix <fguilher@amazon.com>
|
|
Date: Fri, 20 Mar 2020 14:17:55 +0000
|
|
Subject: [PATCH 9/9] fence_aws: Remove empty line
|
|
|
|
---
|
|
agents/aws/fence_aws.py | 3 +--
|
|
1 file changed, 1 insertion(+), 2 deletions(-)
|
|
|
|
diff --git a/agents/aws/fence_aws.py b/agents/aws/fence_aws.py
|
|
index 72fb8843..ed55f390 100644
|
|
--- a/agents/aws/fence_aws.py
|
|
+++ b/agents/aws/fence_aws.py
|
|
@@ -197,5 +197,4 @@ def main():
|
|
sys.exit(result)
|
|
|
|
if __name__ == "__main__":
|
|
-
|
|
- main()
|
|
+ main()
|
|
\ No newline at end of file
|