From 92da4155d881e9ac2dce3a51c6953817349d164a Mon Sep 17 00:00:00 2001 From: Helen Koike Date: Mon, 25 Jun 2018 11:03:51 -0300 Subject: [PATCH 1/4] gcp-vpc-move-vip.in: manage ip alias Add a resource agent to manage ip alias in the cluster. start: Check if any machine in hostlist has the alias_ip assigned and disassociate it. Assign alias_ip to the current machine. stop: Disassociate the alias_ip from the current machine. status/monitor: Check if alias_ip is assigned with the current machine. --- This is a port to the following bash script to python: https://storage.googleapis.com/sapdeploy/pacemaker-gcp/alias The problem with the bash script is the use of gcloud whose command line API is not stable. ocf-tester.in results: > sudo ./tools/ocf-tester.in -o alias_ip='10.128.1.0/32' -o stackdriver_logging=yes -n gcp-vpc-move-vip.in heartbeat/gcp-vpc-move-vip.in Beginning tests for heartbeat/gcp-vpc-move-vip.in... ./tools/ocf-tester.in: line 226: cd: @datadir@/resource-agents: No such file or directory close failed in file object destructor: sys.excepthook is missing lost sys.stderr * rc=1: Your agent produces meta-data which does not conform to ra-api-1.dtd Tests failed: heartbeat/gcp-vpc-move-vip.in failed 1 tests The only test faillig is the meta-data, but all the agents that I tried also fails on this. If this is a concern, could you please point me out to a test which succeeds so I can check what I am doing differently? This commit can also be viewed at: https://github.com/collabora-gce/resource-agents/tree/alias Thanks --- configure.ac | 1 + doc/man/Makefile.am | 1 + heartbeat/Makefile.am | 1 + heartbeat/gcp-vpc-move-vip.in | 299 ++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 302 insertions(+) create mode 100755 heartbeat/gcp-vpc-move-vip.in diff --git a/configure.ac b/configure.ac index bdf057d33..3d8f9ca74 100644 --- a/configure.ac +++ b/configure.ac @@ -959,6 +959,7 @@ AC_CONFIG_FILES([heartbeat/dnsupdate], [chmod +x heartbeat/dnsupdate]) AC_CONFIG_FILES([heartbeat/eDir88], [chmod +x heartbeat/eDir88]) AC_CONFIG_FILES([heartbeat/fio], [chmod +x heartbeat/fio]) AC_CONFIG_FILES([heartbeat/gcp-vpc-move-ip], [chmod +x heartbeat/gcp-vpc-move-ip]) +AC_CONFIG_FILES([heartbeat/gcp-vpc-move-vip], [chmod +x heartbeat/gcp-vpc-move-vip]) AC_CONFIG_FILES([heartbeat/iSCSILogicalUnit], [chmod +x heartbeat/iSCSILogicalUnit]) AC_CONFIG_FILES([heartbeat/iSCSITarget], [chmod +x heartbeat/iSCSITarget]) AC_CONFIG_FILES([heartbeat/jira], [chmod +x heartbeat/jira]) diff --git a/doc/man/Makefile.am b/doc/man/Makefile.am index c59126d13..e9eaf369f 100644 --- a/doc/man/Makefile.am +++ b/doc/man/Makefile.am @@ -114,6 +114,7 @@ man_MANS = ocf_heartbeat_AoEtarget.7 \ ocf_heartbeat_galera.7 \ ocf_heartbeat_garbd.7 \ ocf_heartbeat_gcp-vpc-move-ip.7 \ + ocf_heartbeat_gcp-vpc-move-vip.7 \ ocf_heartbeat_iSCSILogicalUnit.7 \ ocf_heartbeat_iSCSITarget.7 \ ocf_heartbeat_iface-bridge.7 \ diff --git a/heartbeat/Makefile.am b/heartbeat/Makefile.am index 4f5059e27..36b271956 100644 --- a/heartbeat/Makefile.am +++ b/heartbeat/Makefile.am @@ -111,6 +111,7 @@ ocf_SCRIPTS = AoEtarget \ galera \ garbd \ gcp-vpc-move-ip \ + gcp-vpc-move-vip \ iSCSILogicalUnit \ iSCSITarget \ ids \ diff --git a/heartbeat/gcp-vpc-move-vip.in b/heartbeat/gcp-vpc-move-vip.in new file mode 100755 index 000000000..4954e11df --- /dev/null +++ b/heartbeat/gcp-vpc-move-vip.in @@ -0,0 +1,299 @@ +#!/usr/bin/env python +# --------------------------------------------------------------------- +# Copyright 2016 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# --------------------------------------------------------------------- +# Description: Google Cloud Platform - Floating IP Address (Alias) +# --------------------------------------------------------------------- + +import json +import logging +import os +import sys +import time + +import googleapiclient.discovery + +if sys.version_info >= (3, 0): + # Python 3 imports. + import urllib.parse as urlparse + import urllib.request as urlrequest +else: + # Python 2 imports. + import urllib as urlparse + import urllib2 as urlrequest + + +CONN = None +THIS_VM = None +OCF_SUCCESS = 0 +OCF_ERR_GENERIC = 1 +OCF_ERR_CONFIGURED = 6 +OCF_NOT_RUNNING = 7 +METADATA_SERVER = 'http://metadata.google.internal/computeMetadata/v1/' +METADATA_HEADERS = {'Metadata-Flavor': 'Google'} +METADATA = \ +''' + + + 1.0 + Floating IP Address on Google Cloud Platform - Using Alias IP address functionality to attach a secondary IP address to a running instance + Floating IP Address on Google Cloud Platform + + + List of hosts in the cluster + Host list + + + + If enabled (set to true), IP failover logs will be posted to stackdriver logging + Stackdriver-logging support + + + + IP Address to be added including CIDR. E.g 192.168.0.1/32 + IP Address to be added including CIDR. E.g 192.168.0.1/32 + + + + Subnet name for the Alias IP2 + Subnet name for the Alias IP + + + + + + + + + +''' + + +def get_metadata(metadata_key, params=None, timeout=None): + """Performs a GET request with the metadata headers. + + Args: + metadata_key: string, the metadata to perform a GET request on. + params: dictionary, the query parameters in the GET request. + timeout: int, timeout in seconds for metadata requests. + + Returns: + HTTP response from the GET request. + + Raises: + urlerror.HTTPError: raises when the GET request fails. + """ + timeout = timeout or 60 + metadata_url = os.path.join(METADATA_SERVER, metadata_key) + params = urlparse.urlencode(params or {}) + url = '%s?%s' % (metadata_url, params) + request = urlrequest.Request(url, headers=METADATA_HEADERS) + request_opener = urlrequest.build_opener(urlrequest.ProxyHandler({})) + return request_opener.open(request, timeout=timeout * 1.1).read() + + +def get_instance(project, zone, instance): + request = CONN.instances().get( + project=project, zone=zone, instance=instance) + return request.execute() + + +def get_network_ifaces(project, zone, instance): + return get_instance(project, zone, instance)['networkInterfaces'] + + +def wait_for_operation(project, zone, operation): + while True: + result = CONN.zoneOperations().get( + project=project, + zone=zone, + operation=operation['name']).execute() + + if result['status'] == 'DONE': + if 'error' in result: + raise Exception(result['error']) + return + time.sleep(1) + + +def set_alias(project, zone, instance, alias, alias_range_name=None): + fingerprint = get_network_ifaces(project, zone, instance)[0]['fingerprint'] + body = { + 'aliasIpRanges': [], + 'fingerprint': fingerprint + } + if alias: + obj = {'ipCidrRange': alias} + if alias_range_name: + obj['subnetworkRangeName'] = alias_range_name + body['aliasIpRanges'].append(obj) + + request = CONN.instances().updateNetworkInterface( + instance=instance, networkInterface='nic0', project=project, zone=zone, + body=body) + operation = request.execute() + wait_for_operation(project, zone, operation) + + +def get_alias(project, zone, instance): + iface = get_network_ifaces(project, zone, instance) + try: + return iface[0]['aliasIpRanges'][0]['ipCidrRange'] + except KeyError: + return '' + + +def get_localhost_alias(): + net_iface = get_metadata('instance/network-interfaces', {'recursive': True}) + net_iface = json.loads(net_iface.decode('utf-8')) + try: + return net_iface[0]['ipAliases'][0] + except (KeyError, IndexError): + return '' + + +def get_zone(project, instance): + request = CONN.instances().aggregatedList(project=project) + while request is not None: + response = request.execute() + zones = response.get('items', {}) + for zone in zones.values(): + for inst in zone.get('instances', []): + if inst['name'] == instance: + return inst['zone'].split("/")[-1] + request = CONN.instances().aggregatedList_next( + previous_request=request, previous_response=response) + raise Exception("Unable to find instance %s" % (instance)) + + +def gcp_alias_start(alias): + if not alias: + sys.exit(OCF_ERR_CONFIGURED) + my_alias = get_localhost_alias() + my_zone = get_metadata('instance/zone').split('/')[-1] + project = get_metadata('project/project-id') + + # If I already have the IP, exit. If it has an alias IP that isn't the VIP, + # then remove it + if my_alias == alias: + logging.info( + '%s already has %s attached. No action required' % (THIS_VM, alias)) + sys.exit(OCF_SUCCESS) + elif my_alias: + logging.info('Removing %s from %s' % (my_alias, THIS_VM)) + set_alias(project, my_zone, THIS_VM, '') + + # Loops through all hosts & remove the alias IP from the host that has it + hostlist = os.environ.get('OCF_RESKEY_hostlist', '') + hostlist.replace(THIS_VM, '') + for host in hostlist.split(): + host_zone = get_zone(project, host) + host_alias = get_alias(project, host_zone, host) + if alias == host_alias: + logging.info( + '%s is attached to %s - Removing all alias IP addresses from %s' % + (alias, host, host)) + set_alias(project, host_zone, host, '') + break + + # add alias IP to localhost + set_alias( + project, my_zone, THIS_VM, alias, + os.environ.get('OCF_RESKEY_alias_range_name')) + + # Check the IP has been added + my_alias = get_localhost_alias() + if alias == my_alias: + logging.info('Finished adding %s to %s' % (alias, THIS_VM)) + elif my_alias: + logging.error( + 'Failed to add IP. %s has an IP attached but it isn\'t %s' % + (THIS_VM, alias)) + sys.exit(OCF_ERR_GENERIC) + else: + logging.error('Failed to add IP address %s to %s' % (alias, THIS_VM)) + sys.exit(OCF_ERR_GENERIC) + + +def gcp_alias_stop(alias): + if not alias: + sys.exit(OCF_ERR_CONFIGURED) + my_alias = get_localhost_alias() + my_zone = get_metadata('instance/zone').split('/')[-1] + project = get_metadata('project/project-id') + + if my_alias == alias: + logging.info('Removing %s from %s' % (my_alias, THIS_VM)) + set_alias(project, my_zone, THIS_VM, '') + + +def gcp_alias_status(alias): + if not alias: + sys.exit(OCF_ERR_CONFIGURED) + my_alias = get_localhost_alias() + if alias == my_alias: + logging.info('%s has the correct IP address attached' % THIS_VM) + else: + sys.exit(OCF_NOT_RUNNING) + + +def configure(): + global CONN + global THIS_VM + + # Populate global vars + CONN = googleapiclient.discovery.build('compute', 'v1') + THIS_VM = get_metadata('instance/name') + + # Prepare logging + logging.basicConfig( + format='gcp:alias - %(levelname)s - %(message)s', level=logging.INFO) + logging.getLogger('googleapiclient').setLevel(logging.WARN) + logging_env = os.environ.get('OCF_RESKEY_stackdriver_logging') + if logging_env: + logging_env = logging_env.lower() + if any(x in logging_env for x in ['yes', 'true', 'enabled']): + try: + import google.cloud.logging.handlers + client = google.cloud.logging.Client() + handler = google.cloud.logging.handlers.CloudLoggingHandler( + client, name=THIS_VM) + handler.setLevel(logging.INFO) + formatter = logging.Formatter('gcp:alias "%(message)s"') + handler.setFormatter(formatter) + root_logger = logging.getLogger() + root_logger.addHandler(handler) + except ImportError: + logging.error('Couldn\'t import google.cloud.logging, ' + 'disabling Stackdriver-logging support') + + +def main(): + configure() + + alias = os.environ.get('OCF_RESKEY_alias_ip') + if 'start' in sys.argv[1]: + gcp_alias_start(alias) + elif 'stop' in sys.argv[1]: + gcp_alias_stop(alias) + elif 'status' in sys.argv[1] or 'monitor' in sys.argv[1]: + gcp_alias_status(alias) + elif 'meta-data' in sys.argv[1]: + print(METADATA) + else: + logging.error('gcp:alias - no such function %s' % str(sys.argv[1])) + + +if __name__ == "__main__": + main() From 0e6ba4894a748664ac1d8ff5b9e8c271f0b04d93 Mon Sep 17 00:00:00 2001 From: Helen Koike Date: Thu, 12 Jul 2018 09:01:22 -0300 Subject: [PATCH 2/4] gcp-vpc-move-vip.in: minor fixes - Get hostlist from the project if the parameter is not given - Verify if alias is present out of each action function - Don't call configure if 'meta-data' action is given --- heartbeat/gcp-vpc-move-vip.in | 40 ++++++++++++++++++++++++++++------------ 1 file changed, 28 insertions(+), 12 deletions(-) diff --git a/heartbeat/gcp-vpc-move-vip.in b/heartbeat/gcp-vpc-move-vip.in index 4954e11df..f3d117bda 100755 --- a/heartbeat/gcp-vpc-move-vip.in +++ b/heartbeat/gcp-vpc-move-vip.in @@ -50,7 +50,7 @@ METADATA = \ Floating IP Address on Google Cloud Platform - Using Alias IP address functionality to attach a secondary IP address to a running instance Floating IP Address on Google Cloud Platform - + List of hosts in the cluster Host list @@ -177,9 +177,22 @@ def get_zone(project, instance): raise Exception("Unable to find instance %s" % (instance)) +def get_instances_list(project, exclude): + hostlist = [] + request = CONN.instances().aggregatedList(project=project) + while request is not None: + response = request.execute() + zones = response.get('items', {}) + for zone in zones.values(): + for inst in zone.get('instances', []): + if inst['name'] != exclude: + hostlist.append(inst['name']) + request = CONN.instances().aggregatedList_next( + previous_request=request, previous_response=response) + return hostlist + + def gcp_alias_start(alias): - if not alias: - sys.exit(OCF_ERR_CONFIGURED) my_alias = get_localhost_alias() my_zone = get_metadata('instance/zone').split('/')[-1] project = get_metadata('project/project-id') @@ -196,8 +209,11 @@ def gcp_alias_start(alias): # Loops through all hosts & remove the alias IP from the host that has it hostlist = os.environ.get('OCF_RESKEY_hostlist', '') - hostlist.replace(THIS_VM, '') - for host in hostlist.split(): + if hostlist: + hostlist.replace(THIS_VM, '').split() + else: + hostlist = get_instances_list(project, THIS_VM) + for host in hostlist: host_zone = get_zone(project, host) host_alias = get_alias(project, host_zone, host) if alias == host_alias: @@ -227,8 +243,6 @@ def gcp_alias_start(alias): def gcp_alias_stop(alias): - if not alias: - sys.exit(OCF_ERR_CONFIGURED) my_alias = get_localhost_alias() my_zone = get_metadata('instance/zone').split('/')[-1] project = get_metadata('project/project-id') @@ -239,8 +253,6 @@ def gcp_alias_stop(alias): def gcp_alias_status(alias): - if not alias: - sys.exit(OCF_ERR_CONFIGURED) my_alias = get_localhost_alias() if alias == my_alias: logging.info('%s has the correct IP address attached' % THIS_VM) @@ -280,17 +292,21 @@ def configure(): def main(): - configure() + if 'meta-data' in sys.argv[1]: + print(METADATA) + return alias = os.environ.get('OCF_RESKEY_alias_ip') + if not alias: + sys.exit(OCF_ERR_CONFIGURED) + + configure() if 'start' in sys.argv[1]: gcp_alias_start(alias) elif 'stop' in sys.argv[1]: gcp_alias_stop(alias) elif 'status' in sys.argv[1] or 'monitor' in sys.argv[1]: gcp_alias_status(alias) - elif 'meta-data' in sys.argv[1]: - print(METADATA) else: logging.error('gcp:alias - no such function %s' % str(sys.argv[1])) From 1f50c4bc80f23f561a8630c12076707366525899 Mon Sep 17 00:00:00 2001 From: Helen Koike Date: Thu, 12 Jul 2018 13:02:16 -0300 Subject: [PATCH 3/4] gcp-vcp-move-vip.in: implement validate-all Also fix some return errors --- heartbeat/gcp-vpc-move-vip.in | 47 +++++++++++++++++++++++++++++++------------ 1 file changed, 34 insertions(+), 13 deletions(-) diff --git a/heartbeat/gcp-vpc-move-vip.in b/heartbeat/gcp-vpc-move-vip.in index f3d117bda..a90c2de8d 100755 --- a/heartbeat/gcp-vpc-move-vip.in +++ b/heartbeat/gcp-vpc-move-vip.in @@ -22,7 +22,10 @@ import os import sys import time -import googleapiclient.discovery +try: + import googleapiclient.discovery +except ImportError: + pass if sys.version_info >= (3, 0): # Python 3 imports. @@ -36,6 +39,7 @@ else: CONN = None THIS_VM = None +ALIAS = None OCF_SUCCESS = 0 OCF_ERR_GENERIC = 1 OCF_ERR_CONFIGURED = 6 @@ -210,7 +214,7 @@ def gcp_alias_start(alias): # Loops through all hosts & remove the alias IP from the host that has it hostlist = os.environ.get('OCF_RESKEY_hostlist', '') if hostlist: - hostlist.replace(THIS_VM, '').split() + hostlist = hostlist.replace(THIS_VM, '').split() else: hostlist = get_instances_list(project, THIS_VM) for host in hostlist: @@ -260,14 +264,31 @@ def gcp_alias_status(alias): sys.exit(OCF_NOT_RUNNING) -def configure(): +def validate(): + global ALIAS global CONN global THIS_VM # Populate global vars - CONN = googleapiclient.discovery.build('compute', 'v1') - THIS_VM = get_metadata('instance/name') + try: + CONN = googleapiclient.discovery.build('compute', 'v1') + except Exception as e: + logging.error('Couldn\'t connect with google api: ' + str(e)) + sys.exit(OCF_ERR_CONFIGURED) + + try: + THIS_VM = get_metadata('instance/name') + except Exception as e: + logging.error('Couldn\'t get instance name, is this running inside GCE?: ' + str(e)) + sys.exit(OCF_ERR_CONFIGURED) + ALIAS = os.environ.get('OCF_RESKEY_alias_ip') + if not ALIAS: + logging.error('Missing alias_ip parameter') + sys.exit(OCF_ERR_CONFIGURED) + + +def configure_logs(): # Prepare logging logging.basicConfig( format='gcp:alias - %(levelname)s - %(message)s', level=logging.INFO) @@ -296,19 +317,19 @@ def main(): print(METADATA) return - alias = os.environ.get('OCF_RESKEY_alias_ip') - if not alias: - sys.exit(OCF_ERR_CONFIGURED) + validate() + if 'validate-all' in sys.argv[1]: + return - configure() + configure_logs() if 'start' in sys.argv[1]: - gcp_alias_start(alias) + gcp_alias_start(ALIAS) elif 'stop' in sys.argv[1]: - gcp_alias_stop(alias) + gcp_alias_stop(ALIAS) elif 'status' in sys.argv[1] or 'monitor' in sys.argv[1]: - gcp_alias_status(alias) + gcp_alias_status(ALIAS) else: - logging.error('gcp:alias - no such function %s' % str(sys.argv[1])) + logging.error('no such function %s' % str(sys.argv[1])) if __name__ == "__main__": From f11cb236bb348ebee74e962d0ded1cb2fc97bd5f Mon Sep 17 00:00:00 2001 From: Helen Koike Date: Fri, 13 Jul 2018 08:01:02 -0300 Subject: [PATCH 4/4] gcp-vpc-move-vip.in: minor fixes --- heartbeat/gcp-vpc-move-vip.in | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/heartbeat/gcp-vpc-move-vip.in b/heartbeat/gcp-vpc-move-vip.in index a90c2de8d..9fc87242f 100755 --- a/heartbeat/gcp-vpc-move-vip.in +++ b/heartbeat/gcp-vpc-move-vip.in @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!@PYTHON@ -tt # --------------------------------------------------------------------- # Copyright 2016 Google Inc. # @@ -59,7 +59,7 @@ METADATA = \ Host list - + If enabled (set to true), IP failover logs will be posted to stackdriver logging Stackdriver-logging support @@ -80,6 +80,7 @@ METADATA = \ + '''