tests: AWS - apply tags when creating resoures and delete by tags

- for objects in S3 - match filenames starting with Composer-Test
- for keys use the new Composer-Test-Key- prefix (for consistency)
- for VM names use Composer-Test-VM- prefix instead of ami id

tests/cleanup/remove_old_objects_aws.sh

@@ -79,7 +79,7 @@ for region in $regions; do
       region: "$region"
       state: absent
     loop: "{{vms_facts.instances}}"
-    when: (item.launch_time < lookup('env','TIMESTAMP')) and (item.tags['keep_me'] is not defined)
+    when: (item.launch_time < lookup('env','TIMESTAMP')) and (item.tags['composer-test'] is defined)
     loop_control:
       label: "{{item.instance_id}}"
 __EOF__
@@ -111,7 +111,7 @@ __EOF__
       state: absent
       delete_snapshot: True
     loop: "{{ami_facts.images}}"
-    when: (item.creation_date < lookup('env','TIMESTAMP')) and (item.tags['keep_me'] is not defined)
+    when: (item.creation_date < lookup('env','TIMESTAMP')) and (item.tags['composer-test'] is defined)
     loop_control:
       label: "{{item.image_id}}"
 __EOF__
@@ -121,8 +121,8 @@ __EOF__
     rlPhaseEnd
 
     rlPhaseStartTest "Delete unused composer key pairs in region $region"
-        # list all key pairs starting with "composer-"
+        # list all key pairs starting with "Composer-Test"
-        keys=`aws ec2 describe-key-pairs --region="$region" --query 'KeyPairs[?starts_with(KeyName, \`composer-\`) == \`true\`].KeyName' --output text`
+        keys=`aws ec2 describe-key-pairs --region="$region" --query 'KeyPairs[?starts_with(KeyName, \`Composer-Test\`) == \`true\`].KeyName' --output text`
         rlLogInfo "Found existing composer keys: $keys"
 
         for key in $keys; do
@@ -139,8 +139,9 @@ __EOF__
     rlPhaseEnd
 
     rlPhaseStartTest "Delete old volumes in region $region"
-        # get a list of unused ("available") volumes older than $TIMESTAMP and not having the tag "keep_me"
+        # get a list of unused ("available") volumes older than $TIMESTAMP having tag "composer-test"
-        volumes_to_delete=$(aws ec2 describe-volumes --region="$region" --query "Volumes[?CreateTime<\`$TIMESTAMP\`] | [?!(Tags[?Key==\`keep_me\`])] | [?State==\`available\`].[VolumeId,CreateTime]" --output text)
+        # note: volume tags inherit from instance tags when instances are created
+        volumes_to_delete=$(aws ec2 describe-volumes --region="$region" --query "Volumes[?CreateTime<\`$TIMESTAMP\`] | [?(Tags[?Key==\`composer-test\`])] | [?State==\`available\`].[VolumeId,CreateTime]" --output text)
 
         while read volume_id creation_time; do
             if [ -n "$volume_id" ]; then
@@ -151,8 +152,8 @@ __EOF__
     rlPhaseEnd
 
     rlPhaseStartTest "Delete old snapshots in region $region"
-        # get a list of snapshots older than $TIMESTAMP and owned by our account and not having the tag "keep_me"
+        # get a list of snapshots older than $TIMESTAMP and owned by our account and having the tag "composer-test"
-        snapshots_to_delete=$(aws ec2 describe-snapshots --region="$region" --owner-ids "$account_id" --query "Snapshots[?StartTime<\`$TIMESTAMP\`] |[?!(Tags[?Key==\`keep_me\`])].[SnapshotId,StartTime]" --output text)
+        snapshots_to_delete=$(aws ec2 describe-snapshots --region="$region" --owner-ids "$account_id" --query "Snapshots[?StartTime<\`$TIMESTAMP\`] |[?(Tags[?Key==\`composer-test\`])].[SnapshotId,StartTime]" --output text)
 
         while read snapshot_id start_time; do
             if [ -n "$snapshot_id" ]; then
@@ -169,14 +170,10 @@ done
         all_objects=`aws s3 ls s3://${AWS_BUCKET} --recursive`
         while read date_f time_f size_f filename_f; do
             creation_date=`date -u -d "$date_f $time_f" '+%FT%T'`
-            if [ "$creation_date" \< "$TIMESTAMP" ]; then
+            if [ "$filename_f" =~ ^Composer-Test ] && [ "$creation_date" \< "$TIMESTAMP" ]; then
-                # find and delete s3 objects without the "keep_me" tag
-                keep=$(aws s3api get-object-tagging --bucket ${AWS_BUCKET} --key ${filename_f} --output text | cut -f2 | grep "^keep_me$")
-                if [ -z "$keep" ]; then
                 rlLogInfo "Removing old file $filename_f created $date_f $time_f"
                 rlRun -t -c "aws s3 rm s3://${AWS_BUCKET}/${filename_f}"
             fi
-            fi
         done <<< "$all_objects"
     rlPhaseEnd
 

tests/cli/playbooks/aws/instance.yml

@@ -7,7 +7,9 @@
 
   - name: Create instance
     ec2_instance:
-      name: "Composer-Test-{{ ami_id }}"
+      name: "{{ vm_name }}"
+      tags:
+        composer-test: true
       image_id: "{{ ami_id }}"
       key_name: "{{ key_name }}"
       instance_type: "{{ instance_type }}"

tests/cli/test_build_and_deploy_aws.sh

@@ -89,8 +89,9 @@ __EOF__
 
     rlPhaseStartTest "Import AMI image in AWS"
         rlRun -t -c "$CLI compose image $UUID"
+        rlRun -t -c "mv $UUID-disk.ami Composer-Test-$UUID-disk.ami"
 
-        AMI="$UUID-disk.ami"
+        AMI="Composer-Test-$UUID-disk.ami"
 
         # upload to S3
         rlRun -t -c "ansible localhost -m aws_s3 -a \
@@ -132,6 +133,8 @@ __EOF__
         else
             rlLogInfo "SNAPSHOT_ID=$SNAPSHOT_ID"
         fi
+        # tag snapshot
+        aws ec2 create-tags --resources $SNAPSHOT_ID --tags Key=composer-test,Value=true
 
         # create an image from the imported selected snapshot
         AMI_ARCH="$(uname -m)"
@@ -148,6 +151,9 @@ __EOF__
         else
             rlLogInfo "AMI_ID=$AMI_ID"
         fi
+
+        # tag AMI
+        aws ec2 create-tags --resources $AMI_ID --tags Key=composer-test,Value=true
     rlPhaseEnd
 
     rlPhaseStartTest "Start EC2 instance"
@@ -156,13 +162,14 @@ __EOF__
             INSTANCE_TYPE="a1.medium"
         fi
         # generate new ssh key
-        KEY_NAME=composer-$UUID
+        KEY_NAME="Composer-Test-Key-$UUID"
         SSH_KEY_DIR=`mktemp -d /tmp/composer-ssh-keys.XXXXXX`
         rlRun -t -c "ssh-keygen -t rsa -N '' -f $SSH_KEY_DIR/id_rsa"
 
         rlRun -t -c "ansible-playbook  --extra-vars \
                        'key_name=$KEY_NAME \
                         ssh_key_dir=$SSH_KEY_DIR \
+                        vm_name=Composer-Test-VM-$AMI_ID \
                         ami_id=$AMI_ID \
                         instance_type=$INSTANCE_TYPE \
                         tmp_dir=$TMP_DIR' \