tests: Split testing the image into a separate script

Nested virt is not reliable enough, especially on other arches, to rely
on for testing the created images. This moves the test code into
test_boot_* scripts to be run from inside the booted images.

It also adds copying the results of the build into
/var/tmp/test-results/, and includes the generated ssh key so that
whatever boots the image can also log in.

The tests/test_image.sh script has been added to handle running the
test_boot_* scripts without any of the extra lorax-composer specific
setup.

Cherry-picked from b8bf258a3c

Related: rhbz#1770193

tests/cli/test_boot_live-iso.sh

@@ -0,0 +1,23 @@
+#!/bin/bash
+# Note: execute this file from the project root directory
+
+#####
+#
+# Test the live-iso image
+#
+#####
+
+set -e
+
+. /usr/share/beakerlib/beakerlib.sh
+. $(dirname $0)/lib/lib.sh
+
+rlJournalStart
+    rlPhaseStartTest "Verify live iso"
+        # Just the fact that this is running means the image can boot and ssh is working
+        rlRun -t -c "passwd --status root | grep -E '^root\s+NP?'" 0 "root account has no password set"
+        rlAssertGrep "liveuser" /etc/passwd
+        rlAssertGrep "custom_cmdline_arg" /proc/cmdline
+    rlPhaseEnd
+rlJournalEnd
+rlJournalPrintText

tests/cli/test_boot_qcow2.sh

@@ -0,0 +1,22 @@
+#!/bin/bash
+# Note: execute this file from the project root directory
+
+#####
+#
+# Test the qcow2 image
+#
+#####
+
+set -e
+
+. /usr/share/beakerlib/beakerlib.sh
+. $(dirname $0)/lib/lib.sh
+
+rlJournalStart
+    rlPhaseStartTest "Verify VM instance"
+        # Just the fact that this is running means the image can boot and ssh is working
+        rlAssertExists "/root/.ssh/authorized_keys"
+        rlAssertGrep "custom_cmdline_arg" /proc/cmdline
+    rlPhaseEnd
+rlJournalEnd
+rlJournalPrintText

tests/cli/test_compose_live-iso.sh

@@ -16,8 +16,6 @@ CLI="${CLI:-./src/bin/composer-cli}"
 
 rlJournalStart
     rlPhaseStartSetup
-        rlAssertExists $QEMU_BIN
-
         OPTIONAL_REPO="/etc/yum.repos.d/rhel7-rel-eng-optional.repo"
 
         if [ ! -f "$OPTIONAL_REPO" ]; then
@@ -36,33 +34,63 @@ __EOF__
     rlPhaseStartTest "compose start"
         rlAssertEquals "SELinux operates in enforcing mode" "$(getenforce)" "Enforcing"
 
+        TMP_DIR=$(mktemp -d /tmp/composer.XXXXX)
+        SSH_KEY_DIR=$(mktemp -d /tmp/composer-ssh-keys.XXXXXX)
+
+        rlRun -t -c "ssh-keygen -t rsa -N '' -f $SSH_KEY_DIR/id_rsa"
+        PUB_KEY=$(cat "$SSH_KEY_DIR/id_rsa.pub")
+
+        cat > "$TMP_DIR/with-ssh.toml" << __EOF__
+name = "with-ssh"
+description = "HTTP image with SSH"
+version = "0.0.1"
+
+[[packages]]
+name = "httpd"
+version = "*"
+
+[[packages]]
+name = "openssh-server"
+version = "*"
+
+[[packages]]
+name = "beakerlib"
+version = "*"
+
+[customizations.services]
+enabled = ["sshd"]
+
+[[customizations.user]]
+name = "root"
+key = "$PUB_KEY"
+
+[customizations.kernel]
+append = "custom_cmdline_arg"
+__EOF__
+
+        rlRun -t -c "$CLI blueprints push $TMP_DIR/with-ssh.toml"
+
         # NOTE: live-iso.ks explicitly disables sshd but test_cli.sh enables it
-        UUID=`$CLI compose start example-http-server live-iso`
+        UUID=$($CLI compose start with-ssh live-iso)
         rlAssertEquals "exit code should be zero" $? 0
 
-        UUID=`echo $UUID | cut -f 2 -d' '`
+        UUID=$(echo "$UUID" | cut -f 2 -d' ')
     rlPhaseEnd
 
     rlPhaseStartTest "compose finished"
-        wait_for_compose $UUID
+        wait_for_compose "$UUID"
 
-        rlRun -t -c "$CLI compose image $UUID"
+        # Save the results for boot test
-        IMAGE="$UUID-live.iso"
+        rlAssertExists "/var/lib/lorax/composer/results/$UUID/live.iso"
-    rlPhaseEnd
+        rlRun -t -c "mkdir -p /var/tmp/test-results/"
-
+        rlRun -t -c "cp /var/lib/lorax/composer/results/$UUID/live.iso /var/tmp/test-results/"
-    rlPhaseStartTest "Start VM instance"
+        # Include the ssh key needed to log into the image
-        boot_image "-boot d -cdrom $IMAGE" 120
+        rlRun -t -c "cp $SSH_KEY_DIR/* /var/tmp/test-results"
-    rlPhaseEnd
-
-    rlPhaseStartTest "Verify VM instance"
-        # run generic tests to verify the instance
-        ROOT_ACCOUNT_LOCKED=0 verify_image liveuser localhost "-p $SSH_PORT"
     rlPhaseEnd
 
     rlPhaseStartCleanup
-        rlRun -t -c "killall -9 $(basename $QEMU_BIN)"
         rlRun -t -c "$CLI compose delete $UUID"
-        rlRun -t -c "rm -rf $IMAGE $OPTIONAL_REPO"
+        rlRun -t -c "rm -rf $OPTIONAL_REPO $TMP_DIR $SSH_KEY_DIR"
     rlPhaseEnd
 
 rlJournalEnd

tests/cli/test_compose_qcow2.sh

@@ -3,7 +3,7 @@
 
 #####
 #
-# Builds qcow2 images and tests them with QEMU-KVM
+# Builds qcow2 images
 #
 #####
 
@@ -15,20 +15,16 @@ set -e
 CLI="${CLI:-./src/bin/composer-cli}"
 
 rlJournalStart
-    rlPhaseStartSetup
-        rlAssertExists $QEMU_BIN
-    rlPhaseEnd
-
     rlPhaseStartTest "compose start"
         rlAssertEquals "SELinux operates in enforcing mode" "$(getenforce)" "Enforcing"
 
-        TMP_DIR=`mktemp -d /tmp/composer.XXXXX`
+        TMP_DIR=$(mktemp -d /tmp/composer.XXXXX)
-        SSH_KEY_DIR=`mktemp -d /tmp/composer-ssh-keys.XXXXXX`
+        SSH_KEY_DIR=$(mktemp -d /tmp/composer-ssh-keys.XXXXXX)
 
         rlRun -t -c "ssh-keygen -t rsa -N '' -f $SSH_KEY_DIR/id_rsa"
-        PUB_KEY=`cat $SSH_KEY_DIR/id_rsa.pub`
+        PUB_KEY=$(cat "$SSH_KEY_DIR/id_rsa.pub")
 
-        cat > $TMP_DIR/with-ssh.toml << __EOF__
+        cat > "$TMP_DIR/with-ssh.toml" << __EOF__
 name = "with-ssh"
 description = "HTTP image with SSH"
 version = "0.0.1"
@@ -41,6 +37,10 @@ version = "*"
 name = "openssh-server"
 version = "*"
 
+[[packages]]
+name = "beakerlib"
+version = "*"
+
 [[customizations.user]]
 name = "root"
 key = "$PUB_KEY"
@@ -51,31 +51,26 @@ __EOF__
 
         rlRun -t -c "$CLI blueprints push $TMP_DIR/with-ssh.toml"
 
-        UUID=`$CLI compose start with-ssh qcow2`
+        UUID=$($CLI compose start with-ssh qcow2)
         rlAssertEquals "exit code should be zero" $? 0
 
-        UUID=`echo $UUID | cut -f 2 -d' '`
+        UUID=$(echo "$UUID" | cut -f 2 -d' ')
     rlPhaseEnd
 
     rlPhaseStartTest "compose finished"
-        wait_for_compose $UUID
+        wait_for_compose "$UUID"
-        rlRun -t -c "$CLI compose image $UUID"
-        IMAGE="$UUID-disk.qcow2"
-    rlPhaseEnd
 
-    rlPhaseStartTest "Start VM instance"
+        # Save the results for boot test
-        boot_image "-boot c -hda $IMAGE" 60
+        rlAssertExists "/var/lib/lorax/composer/results/$UUID/disk.qcow2"
-    rlPhaseEnd
+        rlRun -t -c "mkdir -p /var/tmp/test-results/"
-
+        rlRun -t -c "cp /var/lib/lorax/composer/results/$UUID/disk.qcow2 /var/tmp/test-results/"
-    rlPhaseStartTest "Verify VM instance"
+        # Include the ssh key needed to log into the image
-        # run generic tests to verify the instance
+        rlRun -t -c "cp $SSH_KEY_DIR/* /var/tmp/test-results"
-        verify_image root localhost "-i $SSH_KEY_DIR/id_rsa -p $SSH_PORT"
     rlPhaseEnd
 
     rlPhaseStartCleanup
-        rlRun -t -c "killall -9 $(basename $QEMU_BIN)"
         rlRun -t -c "$CLI compose delete $UUID"
-        rlRun -t -c "rm -rf $IMAGE $TMP_DIR $SSH_KEY_DIR"
+        rlRun -t -c "rm -rf $TMP_DIR $SSH_KEY_DIR"
     rlPhaseEnd
 
 rlJournalEnd

tests/test_cli.sh

@@ -78,6 +78,10 @@ else
     composer_start
 fi
 
+# Clean out the test-results directory
+if [ -e "/var/tmp/test-results" ]; then
+    rm -rf "/var/tmp/test-results"
+fi
 
 export BEAKERLIB_JOURNAL=0
 export PATH="/usr/local/bin:$PATH"

tests/test_image.sh

@@ -0,0 +1,26 @@
+#!/bin/bash
+# Note: execute this file from the project root directory
+
+set -eu
+
+. $(dirname $0)/cli/lib/lib.sh
+
+export BEAKERLIB_DIR=$(mktemp -d /tmp/composer-test.XXXXXX)
+export BEAKERLIB_JOURNAL=0
+if [ -z "$*" ]; then
+    echo "test_image.sh requires a test to execute"
+else
+    # execute tests
+    for TEST in "$@"; do
+        $TEST
+    done
+fi
+
+. $BEAKERLIB_DIR/TestResults
+
+if [ $TESTRESULT_RESULT_ECODE != 0 ]; then
+  echo "Test failed. Leaving log in $BEAKERLIB_DIR"
+  exit $TESTRESULT_RESULT_ECODE
+fi
+
+rm -rf $BEAKERLIB_DIR