452 lines
15 KiB
Diff
452 lines
15 KiB
Diff
From db3df55a6f7097e1da7d77eb361e9e7560f13353 Mon Sep 17 00:00:00 2001
|
|
From: Oyvind Albrigtsen <oalbrigt@redhat.com>
|
|
Date: Tue, 24 Jul 2018 13:57:08 +0200
|
|
Subject: [PATCH] aliyun-vpc-move-ip: fixes
|
|
|
|
---
|
|
doc/man/Makefile.am | 1 +
|
|
heartbeat/Makefile.am | 1 +
|
|
heartbeat/aliyun-vpc-move-ip | 336 ++++++++++++++++++++++++-------------------
|
|
3 files changed, 189 insertions(+), 149 deletions(-)
|
|
mode change 100644 => 100755 heartbeat/aliyun-vpc-move-ip
|
|
|
|
diff --git a/doc/man/Makefile.am b/doc/man/Makefile.am
|
|
index 3ac0569de..fc9a67161 100644
|
|
--- a/doc/man/Makefile.am
|
|
+++ b/doc/man/Makefile.am
|
|
@@ -93,6 +93,7 @@ man_MANS = ocf_heartbeat_AoEtarget.7 \
|
|
ocf_heartbeat_WinPopup.7 \
|
|
ocf_heartbeat_Xen.7 \
|
|
ocf_heartbeat_Xinetd.7 \
|
|
+ ocf_heartbeat_aliyun-vpc-move-ip.7 \
|
|
ocf_heartbeat_anything.7 \
|
|
ocf_heartbeat_apache.7 \
|
|
ocf_heartbeat_asterisk.7 \
|
|
diff --git a/heartbeat/Makefile.am b/heartbeat/Makefile.am
|
|
index d4750bf09..6adc6bc3c 100644
|
|
--- a/heartbeat/Makefile.am
|
|
+++ b/heartbeat/Makefile.am
|
|
@@ -90,6 +90,7 @@ ocf_SCRIPTS = AoEtarget \
|
|
Xen \
|
|
Xinetd \
|
|
ZFS \
|
|
+ aliyun-vpc-move-ip \
|
|
anything \
|
|
apache \
|
|
asterisk \
|
|
diff --git a/heartbeat/aliyun-vpc-move-ip b/heartbeat/aliyun-vpc-move-ip
|
|
old mode 100644
|
|
new mode 100755
|
|
index bc97822a8..108feb247
|
|
--- a/heartbeat/aliyun-vpc-move-ip
|
|
+++ b/heartbeat/aliyun-vpc-move-ip
|
|
@@ -1,30 +1,19 @@
|
|
-#!/bin/bash
|
|
+#!/bin/sh
|
|
#
|
|
# OCF resource agent to move an IP address within a VPC in the Aliyun
|
|
# Based on code of Markus Guertler (GitHub AWS-VPC-move-IP)
|
|
# Based on code of Adam Gandelman (GitHub ec2-resource-agents/elasticip)
|
|
#
|
|
|
|
-###############################################################################
|
|
-# For testing purposes delete OCF_ROOT after testing
|
|
-OCF_ROOT=/usr/lib/ocf/
|
|
-#
|
|
-# INIT
|
|
-#: ${OCF_FUNCTIONS_DIR=${OCF_ROOT}/resource.d/heartbeat}
|
|
-#if [ -f ${OCF_FUNCTIONS_DIR}/.ocf-shellfuncs ]; then
|
|
-# . ${OCF_FUNCTIONS_DIR}/.ocf-shellfuncs
|
|
-#fi
|
|
-
|
|
#######################################################################
|
|
# Initialization:
|
|
-
|
|
-: ${OCF_FUNCTIONS=${OCF_ROOT}/resource.d/heartbeat/.ocf-shellfuncs}
|
|
-. ${OCF_FUNCTIONS}
|
|
-: ${__OCF_ACTION=$1}
|
|
-export HOME=/root
|
|
+: ${OCF_FUNCTIONS_DIR=${OCF_ROOT}/lib/heartbeat}
|
|
+. ${OCF_FUNCTIONS_DIR}/ocf-shellfuncs
|
|
#######################################################################
|
|
|
|
-
|
|
+# aliyuncli doesnt work without HOME parameter
|
|
+export HOME="/root"
|
|
+
|
|
USAGE="usage: $0 {start|stop|status|meta-data}";
|
|
###############################################################################
|
|
|
|
@@ -36,8 +25,96 @@ USAGE="usage: $0 {start|stop|status|meta-data}";
|
|
###############################################################################
|
|
|
|
|
|
-metadata() {
|
|
-cat <<END
|
|
+
|
|
+ip_get_and_configure() {
|
|
+ ocf_log debug "function: ip_get_and_configure"
|
|
+
|
|
+ ROUTE_TO_INSTANCE="$($cmd |grep $OCF_RESKEY_address | awk '{ print $3 }')"
|
|
+
|
|
+ if [ "$ECS_INSTANCE_ID" != "$ROUTE_TO_INSTANCE" ]; then
|
|
+ if [ -n "$ROUTE_TO_INSTANCE" ]; then
|
|
+ ip_drop
|
|
+ fi
|
|
+
|
|
+ cmd="aliyuncli vpc CreateRouteEntry --RouteTableId $OCF_RESKEY_routing_table --DestinationCidrBlock ${OCF_RESKEY_address}/32 --NextHopId $ECS_INSTANCE_ID --NextHopType Instance --output text"
|
|
+ ocf_log debug "executing command: $cmd"
|
|
+ $cmd
|
|
+ rc=$?
|
|
+ while [ $rc -ne 0 ]; do
|
|
+ sleep 1
|
|
+ cmd="aliyuncli vpc CreateRouteEntry --RouteTableId $OCF_RESKEY_routing_table --DestinationCidrBlock ${OCF_RESKEY_address}/32 --NextHopId $ECS_INSTANCE_ID --NextHopType Instance --output text"
|
|
+ ocf_log debug "executing command: $cmd"
|
|
+ $cmd
|
|
+ rc=$?
|
|
+ done
|
|
+ wait_for_started
|
|
+ fi
|
|
+
|
|
+
|
|
+ # Reconfigure the local ip address
|
|
+ ip addr add "${OCF_RESKEY_address}/32" dev $OCF_RESKEY_interface
|
|
+ rc=$?
|
|
+ if [ $rc -ne 0 ]; then
|
|
+ ocf_log err "command failed, rc: $rc"
|
|
+ return $OCF_ERR_GENERIC
|
|
+ fi
|
|
+
|
|
+ ocf_log debug "IP added"
|
|
+
|
|
+ return $OCF_SUCCESS
|
|
+}
|
|
+
|
|
+ip_drop() {
|
|
+ ocf_log debug "function: ip_drop"
|
|
+ cmd="ip addr delete ${OCF_RESKEY_address}/32 dev $OCF_RESKEY_interface"
|
|
+ ocf_log debug "executing command: $cmd"
|
|
+ $cmd
|
|
+ rc=$?
|
|
+ if [ $rc -ne 0 ] && [ $rc -ne 2 ]; then
|
|
+ ocf_log err "command failed, rc $rc"
|
|
+ return $OCF_ERR_GENERIC
|
|
+ fi
|
|
+
|
|
+ cmd="aliyuncli vpc DeleteRouteEntry --RouteTableId $OCF_RESKEY_routing_table --DestinationCidrBlock ${OCF_RESKEY_address}/32 --NextHopId $ROUTE_TO_INSTANCE --output text"
|
|
+ ocf_log debug "executing command: $cmd"
|
|
+ $cmd
|
|
+ if [ $? -ne 0 ]; then
|
|
+ ocf_log err "command failed, rc: $rc"
|
|
+ return $OCF_ERR_GENERIC
|
|
+ fi
|
|
+ wait_for_deleted
|
|
+
|
|
+ ocf_log debug "IP dropped"
|
|
+
|
|
+ return $OCF_SUCCESS
|
|
+}
|
|
+
|
|
+wait_for_started() {
|
|
+ cmd="aliyuncli vpc DescribeRouteTables --RouteTableId $OCF_RESKEY_routing_table --output text"
|
|
+ ocf_log debug "executing command: $cmd"
|
|
+ ROUTE_TO_INSTANCE="$($cmd | grep $OCF_RESKEY_address | awk '{ print $3 }')"
|
|
+
|
|
+ while [ "$ECS_INSTANCE_ID" != "$ROUTE_TO_INSTANCE" ]; do
|
|
+ sleep 3
|
|
+ cmd="aliyuncli vpc DescribeRouteTables --RouteTableId $OCF_RESKEY_routing_table --output text"
|
|
+ ocf_log debug "executing command: $cmd"
|
|
+ ROUTE_TO_INSTANCE="$($cmd | grep $OCF_RESKEY_address | awk '{ print $3 }')"
|
|
+ done
|
|
+}
|
|
+
|
|
+wait_for_deleted() {
|
|
+ ROUTE_TO_INSTANCE="$($cmd |grep $OCF_RESKEY_address | awk '{ print $3 }')"
|
|
+
|
|
+ while [ ! -z "$ROUTE_TO_INSTANCE" ]; do
|
|
+ sleep 1
|
|
+ cmd="aliyuncli vpc DescribeRouteTables --RouteTableId $OCF_RESKEY_routing_table --output text"
|
|
+ ocf_log debug "executing command: $cmd"
|
|
+ ROUTE_TO_INSTANCE="$($cmd |grep $OCF_RESKEY_address | awk '{ print $3 }')"
|
|
+ done
|
|
+}
|
|
+
|
|
+ecs_ip_metadata() {
|
|
+ cat <<END
|
|
<?xml version="1.0"?>
|
|
<!DOCTYPE resource-agent SYSTEM "ra-api-1.dtd">
|
|
<resource-agent name="vpc-move-ip">
|
|
@@ -74,8 +151,8 @@ Name of the network interfacen, i.e. eth0
|
|
Valid Aliyun CLI profile name
|
|
</longdesc>
|
|
<shortdesc lang="en">profile name</shortdesc>
|
|
-<content type="string" default="default" />
|
|
-</parameter>
|
|
+<content type="string" default="default" />
|
|
+</parameter>
|
|
</parameters>
|
|
<actions>
|
|
<action name="start" timeout="180" />
|
|
@@ -88,171 +165,132 @@ Valid Aliyun CLI profile name
|
|
END
|
|
}
|
|
|
|
-debugger() {
|
|
- ocf_log info "DEBUG: $1"
|
|
-}
|
|
-
|
|
ecs_ip_validate() {
|
|
- debugger "function: validate"
|
|
-
|
|
+ ocf_log debug "function: validate"
|
|
+
|
|
# IP address
|
|
- [[ -z "$OCF_RESKEY_address" ]] && ocf_log error "IP address parameter not set $OCF_RESKEY_ADDRESS!" && exit $OCF_ERR_CONFIGURED
|
|
-
|
|
+ if [ -z "$OCF_RESKEY_address" ]; then
|
|
+ ocf_log err "IP address parameter not set $OCF_RESKEY_ADDRESS!"
|
|
+ exit $OCF_ERR_CONFIGURED
|
|
+ fi
|
|
+
|
|
# Network Interface
|
|
- [[ -z "$OCF_RESKEY_interface" ]] && ocf_log error "Network interface parameter not set $OCF_RESKEY_INTERFACE!" && exit $OCF_ERR_CONFIGURED
|
|
-
|
|
+ if [ -z "$OCF_RESKEY_interface" ]; then
|
|
+ ocf_log err "Network interface parameter not set $OCF_RESKEY_INTERFACE!"
|
|
+ exit $OCF_ERR_CONFIGURED
|
|
+ fi
|
|
+
|
|
# Routing Table
|
|
- [[ -z "$OCF_RESKEY_routing_table" ]] && ocf_log error "Routing table parameter not set $OCF_RESKEY_ROUTING_TABLE!" && exit $OCF_ERR_CONFIGURED
|
|
-
|
|
- ECS_INSTANCE_ID="$(curl -s http://100.100.100.200/latest/meta-data/instance-id)"
|
|
+ if [ -z "$OCF_RESKEY_routing_table" ]; then
|
|
+ ocf_log err "Routing table parameter not set $OCF_RESKEY_ROUTING_TABLE!"
|
|
+ exit $OCF_ERR_CONFIGURED
|
|
+ fi
|
|
|
|
if [ -z "${ECS_INSTANCE_ID}" ]; then
|
|
ocf_exit_reason "Instance ID not found. Is this a ECS instance?"
|
|
return $OCF_ERR_GENERIC
|
|
fi
|
|
-
|
|
- return $OCF_SUCCESS
|
|
-}
|
|
|
|
-ecs_ip_monitor() {
|
|
- ecs_ip_validate
|
|
- debugger "function: ecsip_monitor: check routing table"
|
|
- cmd="aliyuncli vpc DescribeRouteTables --RouteTableId $OCF_RESKEY_routing_table --output text"
|
|
- debugger "executing command: $cmd"
|
|
- ROUTE_TO_INSTANCE="$($cmd |grep $OCF_RESKEY_address | awk '{ print $3 }')"
|
|
- if [ -z "$ROUTE_TO_INSTANCE" ]; then
|
|
- ROUTE_TO_INSTANCE="<unknown>"
|
|
- fi
|
|
-
|
|
- [[ "$ECS_INSTANCE_ID" != "$ROUTE_TO_INSTANCE" ]] && debugger "not routed to this instance ($ECS_INSTANCE_ID) but to instance $ROUTE_TO_INSTANCE" && return $OCF_NOT_RUNNING
|
|
- cmd="ping -W 1 -c 1 $OCF_RESKEY_address"
|
|
- debugger "executing command: $cmd"
|
|
- $cmd > /dev/null
|
|
- [[ $? -gt 0 ]] && debugger "IP $OCF_RESKEY_address not locally reachable via ping on this system" && return $OCF_NOT_RUNNING
|
|
- debugger "routed in VPC and locally reachable"
|
|
- return $OCF_SUCCESS
|
|
-}
|
|
-
|
|
-
|
|
-ecs_ip_drop() {
|
|
- debugger "function: ecsip_drop"
|
|
- cmd="ip addr delete ${OCF_RESKEY_address}/32 dev $OCF_RESKEY_interface"
|
|
- debugger "executing command: $cmd"
|
|
- $cmd
|
|
- rc=$?
|
|
- [[ $rc -gt 2 ]] && debugger "command failed, rc $rc" && return $OCF_ERR_GENERIC
|
|
- debugger "command succeeded"
|
|
return $OCF_SUCCESS
|
|
}
|
|
|
|
-wait_for_deleted() {
|
|
- while [ ! -z "$ROUTE_TO_INSTANCE" ]; do
|
|
- sleep 1
|
|
- cmd="aliyuncli vpc DescribeRouteTables --RouteTableId $OCF_RESKEY_routing_table --output text"
|
|
- debugger "executing command: $cmd"
|
|
- ROUTE_TO_INSTANCE="$($cmd |grep $OCF_RESKEY_address | awk '{ print $3 }')"
|
|
- done
|
|
- sleep 5
|
|
-}
|
|
+ecs_ip_start() {
|
|
+ ocf_log info "ECS: Moving IP address $OCF_RESKEY_address to this host by adjusting routing table $OCF_RESKEY_routing_table"
|
|
|
|
-wait_for_started() {
|
|
- cmd="aliyuncli vpc DescribeRouteTables --RouteTableId $OCF_RESKEY_routing_table --output text"
|
|
- debugger "executing command: $cmd"
|
|
- ROUTE_TO_INSTANCE="$($cmd |grep $OCF_RESKEY_address | awk '{ print $3 }')"
|
|
-
|
|
- while [ "$ECS_INSTANCE_ID" != "$ROUTE_TO_INSTANCE" ]; do
|
|
- sleep 1
|
|
- cmd="aliyuncli vpc DescribeRouteTables --RouteTableId $OCF_RESKEY_routing_table --output text"
|
|
- debugger "executing command: $cmd"
|
|
- ROUTE_TO_INSTANCE="$($cmd |grep $OCF_RESKEY_address | awk '{ print $3 }')"
|
|
- done
|
|
- sleep 5
|
|
-}
|
|
+ ecs_ip_monitor
|
|
+ if [ $? = $OCF_SUCCESS ]; then
|
|
+ ocf_log info "ECS: $OCF_RESKEY_address already started"
|
|
+ return $OCF_SUCCESS
|
|
+ fi
|
|
|
|
-ecs_ip_get_and_configure() {
|
|
- debugger "function: ecsip_get_and_configure"
|
|
-
|
|
- if [ "$ECS_INSTANCE_ID" != "$ROUTE_TO_INSTANCE" ]; then
|
|
-
|
|
- if [ $ROUTE_TO_INSTANCE != "<unknown>" ]; then
|
|
- # Adjusting the routing table
|
|
- cmd="aliyuncli vpc DeleteRouteEntry --RouteTableId $OCF_RESKEY_routing_table --DestinationCidrBlock ${OCF_RESKEY_address}/32 --NextHopId $ROUTE_TO_INSTANCE --output text"
|
|
- debugger "executing command: $cmd"
|
|
- $cmd
|
|
- rc=$?
|
|
- [[ $rc != 0 ]] && debugger "command failed, rc: $rc" && return $OCF_ERR_GENERIC
|
|
- #wait_for_deleted
|
|
- sleep 3
|
|
- fi
|
|
-
|
|
- cmd="aliyuncli vpc CreateRouteEntry --RouteTableId $OCF_RESKEY_routing_table --DestinationCidrBlock ${OCF_RESKEY_address}/32 --NextHopId $ECS_INSTANCE_ID --NextHopType Instance --output text"
|
|
- debugger "executing command: $cmd"
|
|
- $cmd
|
|
- rc=$?
|
|
- #[[ $rc != 0 ]] && debugger "command failed, rc: $rc" && return $OCF_ERR_GENERIC
|
|
- while [ $rc != 0 ]; do
|
|
- sleep 2
|
|
- cmd="aliyuncli vpc CreateRouteEntry --RouteTableId $OCF_RESKEY_routing_table --DestinationCidrBlock ${OCF_RESKEY_address}/32 --NextHopId $ECS_INSTANCE_ID --NextHopType Instance --output text"
|
|
- debugger "executing command: $cmd"
|
|
- $cmd
|
|
- rc=$?
|
|
- done
|
|
- wait_for_started
|
|
+ ocf_log info "ECS: Adjusting routing table and locally configuring IP address"
|
|
+ ip_get_and_configure
|
|
+ rc=$?
|
|
+ if [ $rc -ne 0 ]; then
|
|
+ ocf_log err "Received $rc from 'aliyun cli'"
|
|
+ return $OCF_ERR_GENERIC
|
|
fi
|
|
-
|
|
-
|
|
- # Reconfigure the local ip address
|
|
- ecs_ip_drop
|
|
- ip addr add "${OCF_RESKEY_address}/32" dev $OCF_RESKEY_interface
|
|
+
|
|
+ ecs_ip_monitor
|
|
rc=$?
|
|
- [[ $rc != 0 ]] && debugger "command failed, rc: $rc" && return $OCF_ERR_GENERIC
|
|
- debugger "-success"
|
|
+ if [ $rc -ne $OCF_SUCCESS ]; then
|
|
+ ocf_log err "IP address couldn't be configured on this host (IP: $OCF_RESKEY_address, Interface: $OCF_RESKEY_interface)"
|
|
+ return $rc
|
|
+ fi
|
|
+
|
|
return $OCF_SUCCESS
|
|
}
|
|
|
|
ecs_ip_stop() {
|
|
ocf_log info "ECS: Bringing down IP address $OCF_RESKEY_address"
|
|
- ecs_ip_validate
|
|
+
|
|
ecs_ip_monitor
|
|
- [[ $? == $OCF_NOT_RUNNING ]] && ocf_log info "ECS: Address $OCF_RESKEY_address already down" && return $OCF_SUCCESS
|
|
- ecs_ip_drop
|
|
- [[ $? != $OCF_SUCCESS ]] && return $OCF_ERR_GENERIC
|
|
+ if [ $? = $OCF_NOT_RUNNING ]; then
|
|
+ ocf_log info "ECS: Address $OCF_RESKEY_address already down"
|
|
+ return $OCF_SUCCESS
|
|
+ fi
|
|
+
|
|
+ ip_drop
|
|
+ if [ $? -ne $OCF_SUCCESS ]; then
|
|
+ ocf_log err "ECS: Couldn't drop IP address $OCF_RESKEY_address on interface $OCF_RESKEY_interface."
|
|
+ return $OCF_ERR_GENERIC
|
|
+ fi
|
|
+
|
|
ecs_ip_monitor
|
|
- [[ $? == $OCF_NOT_RUNNING ]] && ocf_log info "ECS: Successfully brought down $OCF_RESKEY_address" && return $OCF_SUCCESS
|
|
- ocf_log error "ECS: Couldn't bring down IP address $OCF_RESKEY_address on interface $OCF_RESKEY_interface."
|
|
+ if [ $? = $OCF_NOT_RUNNING ]; then
|
|
+ ocf_log info "ECS: Successfully brought down $OCF_RESKEY_address"
|
|
+ return $OCF_SUCCESS
|
|
+ fi
|
|
+
|
|
+ ocf_log err "ECS: Couldn't bring down IP address $OCF_RESKEY_address on interface $OCF_RESKEY_interface."
|
|
return $OCF_ERR_GENERIC
|
|
}
|
|
|
|
-ecs_ip_start() {
|
|
- ocf_log info "ECS: Moving IP address $OCF_RESKEY_address to this host by adjusting routing table $OCF_RESKEY_routing_table"
|
|
- ecs_ip_validate
|
|
- ecs_ip_monitor
|
|
- [[ $? == $OCF_SUCCESS ]] && ocf_log info "ECS: $OCF_RESKEY_address already started" && return $OCF_SUCCESS
|
|
- ocf_log info "ECS: Adjusting routing table and locally configuring IP address"
|
|
- ecs_ip_get_and_configure
|
|
- [[ $? != 0 ]] && ocf_log error "Received $? from 'aliyun cli'" && return $OCF_ERR_GENERIC
|
|
- return $OCF_SUCCESS
|
|
- ecs_ip_monitor
|
|
- [[ $? == $OCF_SUCCESS ]] && return $?
|
|
- ocf_log error "ECS: IP address couldn't be configured on this host (IP: $OCF_RESKEY_address, Interface: $OCF_RESKEY_interface)"
|
|
- return $OCF_ERR_GENERIC
|
|
+ecs_ip_monitor() {
|
|
+ ocf_log debug "function: ecsip_monitor: check routing table"
|
|
+ cmd="aliyuncli vpc DescribeRouteTables --RouteTableId $OCF_RESKEY_routing_table --output text"
|
|
+ ocf_log debug "executing command: $cmd"
|
|
+
|
|
+ ROUTE_TO_INSTANCE="$($cmd |grep $OCF_RESKEY_address | awk '{ print $3 }')"
|
|
+
|
|
+ if [ "$ECS_INSTANCE_ID" != "$ROUTE_TO_INSTANCE" ]; then
|
|
+ ocf_log debug "not routed to this instance ($ECS_INSTANCE_ID) but to instance $ROUTE_TO_INSTANCE"
|
|
+ return $OCF_NOT_RUNNING
|
|
+ fi
|
|
+
|
|
+ cmd="ping -W 1 -c 1 $OCF_RESKEY_address"
|
|
+ ocf_log debug "executing command: $cmd"
|
|
+ $cmd > /dev/null
|
|
+ if [ $? -ne 0 ]; then
|
|
+ ocf_log debug "IP $OCF_RESKEY_address not locally reachable via ping on this system"
|
|
+ return $OCF_NOT_RUNNING
|
|
+ fi
|
|
+ ocf_log debug "routed in VPC and locally reachable"
|
|
+ return $OCF_SUCCESS
|
|
}
|
|
|
|
+
|
|
###############################################################################
|
|
#
|
|
# MAIN
|
|
#
|
|
###############################################################################
|
|
|
|
-case $__OCF_ACTION in
|
|
- meta-data) metadata
|
|
+case $__OCF_ACTION in
|
|
+ meta-data) ecs_ip_metadata
|
|
exit $OCF_SUCCESS;;
|
|
- monitor)
|
|
- ecs_ip_monitor;;
|
|
- stop)
|
|
- ecs_ip_stop;;
|
|
validate-all) ecs_ip_validate;;
|
|
+esac
|
|
+
|
|
+ECS_INSTANCE_ID="$(curl -s http://100.100.100.200/latest/meta-data/instance-id)"
|
|
+
|
|
+case $__OCF_ACTION in
|
|
start)
|
|
+ ecs_ip_validate
|
|
ecs_ip_start;;
|
|
+ stop)
|
|
+ ecs_ip_stop;;
|
|
+ monitor)
|
|
+ ecs_ip_monitor;;
|
|
*) exit $OCF_ERR_UNIMPLEMENTED;;
|
|
-esac
|
|
\ No newline at end of file
|
|
+esac
|