--- fence-agents-4.10.0/agents/gce/fence_gce.py 2022-04-29 10:13:50.317888041 +0200 +++ fence-agents-4.10.0/agents/gce/fence_gce.py2 2022-04-29 10:17:54.813248566 +0200 @@ -2,10 +2,10 @@ # # Requires the googleapiclient and oauth2client -# RHEL 7.x: google-api-python-client==1.6.7 python-gflags==2.0 pyasn1==0.4.8 rsa==3.4.2 -# RHEL 8.x: nothing additional needed -# SLES 12.x: python-google-api-python-client python-oauth2client python-oauth2client-gce -# SLES 15.x: python3-google-api-python-client python3-oauth2client python3-oauth2client-gce +# RHEL 7.x: google-api-python-client==1.6.7 python-gflags==2.0 pyasn1==0.4.8 rsa==3.4.2 pysocks==1.7.1 httplib2==0.19.0 +# RHEL 8.x: pysocks==1.7.1 httplib2==0.19.0 +# SLES 12.x: python-google-api-python-client python-oauth2client python-oauth2client-gce pysocks==1.7.1 httplib2==0.19.0 +# SLES 15.x: python3-google-api-python-client python3-oauth2client pysocks==1.7.1 httplib2==0.19.0 # import atexit @@ -27,7 +27,7 @@ import urllib2 as urlrequest sys.path.append("@FENCEAGENTSLIBDIR@") -from fencing import fail_usage, run_delay, all_opt, atexit_handler, check_input, process_input, show_docs, fence_action +from fencing import fail_usage, run_delay, all_opt, atexit_handler, check_input, process_input, show_docs, fence_action, run_command try: sys.path.insert(0, '/usr/lib/fence-agents/support/google') import httplib2 @@ -42,6 +42,19 @@ METADATA_SERVER = 'http://metadata.google.internal/computeMetadata/v1/' METADATA_HEADERS = {'Metadata-Flavor': 'Google'} +INSTANCE_LINK = 'https://www.googleapis.com/compute/v1/projects/{}/zones/{}/instances/{}' + +def run_on_fail(options): + if "--runonfail" in options: + run_command(options, options["--runonfail"]) + +def fail_fence_agent(options, message): + run_on_fail(options) + fail_usage(message) + +def raise_fence_agent(options, message): + run_on_fail(options) + raise Exception(message) # # Will use baremetalsolution setting or the environment variable @@ -66,7 +79,7 @@ { "matchlength": 4, "match": "https://compute.googleapis.com/compute/v1/projects/(.*)/zones/(.*)/instances/(.*)/reset(.*)", - "replace": "https://baremetalsolution.googleapis.com/v1alpha1/projects/\\1/locations/\\2/instances/\\3:resetInstance\\4" + "replace": "https://baremetalsolution.googleapis.com/v1/projects/\\1/locations/\\2/instances/\\3:resetInstance\\4" }) for uri_replacement in uri_replacements: # each uri_replacement should have matchlength, match, and replace @@ -121,14 +134,17 @@ def get_nodes_list(conn, options): result = {} + if "--zone" not in options: + fail_fence_agent(options, "Failed: get_nodes_list: Please specify the --zone in the command") try: - instanceList = retry_api_execute(options, conn.instances().list( - project=options["--project"], - zone=options["--zone"])) - for instance in instanceList["items"]: - result[instance["id"]] = (instance["name"], translate_status(instance["status"])) + for zone in options["--zone"].split(","): + instanceList = retry_api_execute(options, conn.instances().list( + project=options["--project"], + zone=zone)) + for instance in instanceList["items"]: + result[instance["id"]] = (instance["name"], translate_status(instance["status"])) except Exception as err: - fail_usage("Failed: get_nodes_list: {}".format(str(err))) + fail_fence_agent(options, "Failed: get_nodes_list: {}".format(str(err))) return result @@ -142,23 +158,54 @@ return "off" else: return "on" + # If zone is not listed for an entry we attempt to get it automatically + instance = options["--plug"] + zone = get_zone(conn, options, instance) if "--plugzonemap" not in options else options["--plugzonemap"][instance] + instance_status = get_instance_power_status(conn, options, instance, zone) + # If any of the instances do not match the intended status we return the + # the opposite status so that the fence agent can change it. + if instance_status != options.get("--action"): + return instance_status + + return options.get("--action") + + +def get_instance_power_status(conn, options, instance, zone): try: - instance = retry_api_execute(options, conn.instances().get( - project=options["--project"], - zone=options["--zone"], - instance=options["--plug"])) + instance = retry_api_execute( + options, + conn.instances().get(project=options["--project"], zone=zone, instance=instance)) return translate_status(instance["status"]) except Exception as err: - fail_usage("Failed: get_power_status: {}".format(str(err))) + fail_fence_agent(options, "Failed: get_instance_power_status: {}".format(str(err))) -def wait_for_operation(conn, options, operation): +def check_for_existing_operation(conn, options, instance, zone, operation_type): + logging.debug("check_for_existing_operation") + if "--baremetalsolution" in options: + # There is no API for checking in progress operations + return False + + project = options["--project"] + target_link = INSTANCE_LINK.format(project, zone, instance) + query_filter = '(targetLink = "{}") AND (operationType = "{}") AND (status = "RUNNING")'.format(target_link, operation_type) + result = retry_api_execute( + options, + conn.zoneOperations().list(project=project, zone=zone, filter=query_filter, maxResults=1)) + + if "items" in result and result["items"]: + logging.info("Existing %s operation found", operation_type) + return result["items"][0] + + +def wait_for_operation(conn, options, zone, operation): if 'name' not in operation: logging.warning('Cannot wait for operation to complete, the' ' requested operation will continue asynchronously') - return + return False + + wait_time = 0 project = options["--project"] - zone = options["--zone"] while True: result = retry_api_execute(options, conn.zoneOperations().get( project=project, @@ -166,56 +213,93 @@ operation=operation['name'])) if result['status'] == 'DONE': if 'error' in result: - raise Exception(result['error']) - return + raise_fence_agent(options, result['error']) + return True + + if "--errortimeout" in options and wait_time > int(options["--errortimeout"]): + raise_fence_agent(options, "Operation did not complete before the timeout.") + + if "--warntimeout" in options and wait_time > int(options["--warntimeout"]): + logging.warning("Operation did not complete before the timeout.") + if "--runonwarn" in options: + run_command(options, options["--runonwarn"]) + return False + + wait_time = wait_time + 1 time.sleep(1) def set_power_status(conn, options): - logging.debug("set_power_status"); + logging.debug("set_power_status") + instance = options["--plug"] + # If zone is not listed for an entry we attempt to get it automatically + zone = get_zone(conn, options, instance) if "--plugzonemap" not in options else options["--plugzonemap"][instance] + set_instance_power_status(conn, options, instance, zone, options["--action"]) + + +def set_instance_power_status(conn, options, instance, zone, action): + logging.info("Setting power status of %s in zone %s", instance, zone) + project = options["--project"] + try: - if options["--action"] == "off": - logging.info("Issuing poweroff of %s in zone %s" % (options["--plug"], options["--zone"])) - operation = retry_api_execute(options, conn.instances().stop( - project=options["--project"], - zone=options["--zone"], - instance=options["--plug"])) + if action == "off": + logging.info("Issuing poweroff of %s in zone %s", instance, zone) + operation = check_for_existing_operation(conn, options, instance, zone, "stop") + if operation and "--earlyexit" in options: + return + if not operation: + operation = retry_api_execute( + options, + conn.instances().stop(project=project, zone=zone, instance=instance)) logging.info("Poweroff command completed, waiting for the operation to complete") - wait_for_operation(conn, options, operation) - logging.info("Poweroff of %s in zone %s complete" % (options["--plug"], options["--zone"])) - elif options["--action"] == "on": - logging.info("Issuing poweron of %s in zone %s" % (options["--plug"], options["--zone"])) - operation = retry_api_execute(options, conn.instances().start( - project=options["--project"], - zone=options["--zone"], - instance=options["--plug"])) - wait_for_operation(conn, options, operation) - logging.info("Poweron of %s in zone %s complete" % (options["--plug"], options["--zone"])) + if wait_for_operation(conn, options, zone, operation): + logging.info("Poweroff of %s in zone %s complete", instance, zone) + elif action == "on": + logging.info("Issuing poweron of %s in zone %s", instance, zone) + operation = check_for_existing_operation(conn, options, instance, zone, "start") + if operation and "--earlyexit" in options: + return + if not operation: + operation = retry_api_execute( + options, + conn.instances().start(project=project, zone=zone, instance=instance)) + if wait_for_operation(conn, options, zone, operation): + logging.info("Poweron of %s in zone %s complete", instance, zone) except Exception as err: - fail_usage("Failed: set_power_status: {}".format(str(err))) - + fail_fence_agent(options, "Failed: set_instance_power_status: {}".format(str(err))) def power_cycle(conn, options): - logging.debug("power_cycle"); + logging.debug("power_cycle") + instance = options["--plug"] + # If zone is not listed for an entry we attempt to get it automatically + zone = get_zone(conn, options, instance) if "--plugzonemap" not in options else options["--plugzonemap"][instance] + return power_cycle_instance(conn, options, instance, zone) + + +def power_cycle_instance(conn, options, instance, zone): + logging.info("Issuing reset of %s in zone %s", instance, zone) + project = options["--project"] + try: - logging.info('Issuing reset of %s in zone %s' % (options["--plug"], options["--zone"])) - operation = retry_api_execute(options, conn.instances().reset( - project=options["--project"], - zone=options["--zone"], - instance=options["--plug"])) - logging.info("Reset command completed, waiting for the operation to complete") - wait_for_operation(conn, options, operation) - logging.info('Reset of %s in zone %s complete' % (options["--plug"], options["--zone"])) + operation = check_for_existing_operation(conn, options, instance, zone, "reset") + if operation and "--earlyexit" in options: + return True + if not operation: + operation = retry_api_execute( + options, + conn.instances().reset(project=project, zone=zone, instance=instance)) + logging.info("Reset command sent, waiting for the operation to complete") + if wait_for_operation(conn, options, zone, operation): + logging.info("Reset of %s in zone %s complete", instance, zone) return True except Exception as err: - logging.error("Failed: power_cycle: {}".format(str(err))) - return False + logging.exception("Failed: power_cycle") + raise err -def get_zone(conn, options): +def get_zone(conn, options, instance): logging.debug("get_zone"); project = options['--project'] - instance = options['--plug'] fl = 'name="%s"' % instance request = replace_api_uri(options, conn.instances().aggregatedList(project=project, filter=fl)) while request is not None: @@ -227,7 +311,7 @@ return inst['zone'].split("/")[-1] request = replace_api_uri(options, conn.instances().aggregatedList_next( previous_request=request, previous_response=response)) - raise Exception("Unable to find instance %s" % (instance)) + raise_fence_agent(options, "Unable to find instance %s" % (instance)) def get_metadata(metadata_key, params=None, timeout=None): @@ -326,13 +410,21 @@ "required" : "0", "order" : 9 } + all_opt["plugzonemap"] = { + "getopt" : ":", + "longopt" : "plugzonemap", + "help" : "--plugzonemap=[plugzonemap] Comma separated zone map when fencing multiple plugs", + "shortdesc" : "Comma separated zone map when fencing multiple plugs.", + "required" : "0", + "order" : 10 + } all_opt["proxyhost"] = { "getopt" : ":", "longopt" : "proxyhost", "help" : "--proxyhost=[proxy_host] The proxy host to use, if one is needed to access the internet (Example: 10.122.0.33)", "shortdesc" : "If a proxy is used for internet access, the proxy host should be specified.", "required" : "0", - "order" : 10 + "order" : 11 } all_opt["proxyport"] = { "getopt" : ":", @@ -341,7 +433,49 @@ "help" : "--proxyport=[proxy_port] The proxy port to use, if one is needed to access the internet (Example: 3127)", "shortdesc" : "If a proxy is used for internet access, the proxy port should be specified.", "required" : "0", - "order" : 11 + "order" : 12 + } + all_opt["earlyexit"] = { + "getopt" : "", + "longopt" : "earlyexit", + "help" : "--earlyexit Return early if reset is already in progress", + "shortdesc" : "If an existing reset operation is detected, the fence agent will return before the operation completes with a 0 return code.", + "required" : "0", + "order" : 13 + } + all_opt["warntimeout"] = { + "getopt" : ":", + "type" : "second", + "longopt" : "warntimeout", + "help" : "--warntimeout=[warn_timeout] Timeout seconds before logging a warning and returning a 0 status code", + "shortdesc" : "If the operation is not completed within the timeout, the cluster operations are allowed to continue.", + "required" : "0", + "order" : 14 + } + all_opt["errortimeout"] = { + "getopt" : ":", + "type" : "second", + "longopt" : "errortimeout", + "help" : "--errortimeout=[error_timeout] Timeout seconds before failing and returning a non-zero status code", + "shortdesc" : "If the operation is not completed within the timeout, cluster is notified of the operation failure.", + "required" : "0", + "order" : 15 + } + all_opt["runonwarn"] = { + "getopt" : ":", + "longopt" : "runonwarn", + "help" : "--runonwarn=[run_on_warn] If a timeout occurs and warning is generated, run the supplied command", + "shortdesc" : "If a timeout would occur while running the agent, then the supplied command is run.", + "required" : "0", + "order" : 16 + } + all_opt["runonfail"] = { + "getopt" : ":", + "longopt" : "runonfail", + "help" : "--runonfail=[run_on_fail] If a failure occurs, run the supplied command", + "shortdesc" : "If a failure would occur while running the agent, then the supplied command is run.", + "required" : "0", + "order" : 17 } @@ -350,7 +484,8 @@ device_opt = ["port", "no_password", "zone", "project", "stackdriver-logging", "method", "baremetalsolution", "apitimeout", "retries", "retrysleep", - "serviceaccount", "proxyhost", "proxyport"] + "serviceaccount", "plugzonemap", "proxyhost", "proxyport", "earlyexit", + "warntimeout", "errortimeout", "runonwarn", "runonfail"] atexit.register(atexit_handler) @@ -431,22 +566,39 @@ conn = googleapiclient.discovery.build( 'compute', 'v1', credentials=credentials, cache_discovery=False) except Exception as err: - fail_usage("Failed: Create GCE compute v1 connection: {}".format(str(err))) + fail_fence_agent(options, "Failed: Create GCE compute v1 connection: {}".format(str(err))) # Get project and zone if not options.get("--project"): try: options["--project"] = get_metadata('project/project-id') except Exception as err: - fail_usage("Failed retrieving GCE project. Please provide --project option: {}".format(str(err))) + fail_fence_agent(options, "Failed retrieving GCE project. Please provide --project option: {}".format(str(err))) if "--baremetalsolution" in options: options["--zone"] = "none" - if not options.get("--zone"): - try: - options["--zone"] = get_zone(conn, options) - except Exception as err: - fail_usage("Failed retrieving GCE zone. Please provide --zone option: {}".format(str(err))) + + # Populates zone automatically if missing from the command + zones = [] if not "--zone" in options else options["--zone"].split(",") + options["--plugzonemap"] = {} + if "--plug" in options: + for i, instance in enumerate(options["--plug"].split(",")): + if len(zones) == 1: + # If only one zone is specified, use it across all plugs + options["--plugzonemap"][instance] = zones[0] + continue + + if len(zones) - 1 >= i: + # If we have enough zones specified with the --zone flag use the zone at + # the same index as the plug + options["--plugzonemap"][instance] = zones[i] + continue + + try: + # In this case we do not have a zone specified so we attempt to detect it + options["--plugzonemap"][instance] = get_zone(conn, options, instance) + except Exception as err: + fail_fence_agent(options, "Failed retrieving GCE zone. Please provide --zone option: {}".format(str(err))) # Operate the fencing device result = fence_action(conn, options, set_power_status, get_power_status, get_nodes_list, power_cycle)