Backport changes for Cockpit CI

Related: rhbz#1698366

Makefile

@@ -9,6 +9,12 @@ TAG = lorax-$(VERSION)-$(RELEASE)
 
 IMAGE_RELEASE = $(shell awk -F: '/FROM/ { print $$2}' Dockerfile.test)
 
+ifeq ($(TEST_OS),)
+TEST_OS = rhel-7-7
+endif
+export TEST_OS
+VM_IMAGE=$(CURDIR)/test/images/$(TEST_OS)
+
 default: all
 
 src/composer/version.py:
@@ -43,8 +49,6 @@ test: docs
 
 	coverage report -m
 	[ -f "/usr/bin/coveralls" ] && [ -n "$(COVERALLS_REPO_TOKEN)" ] && coveralls || echo
-	
-	./tests/test_cli.sh
 
 # need `losetup`, which needs Docker to be in privileged mode (--privileged)
 # but even so fails in Travis CI
@@ -80,9 +84,15 @@ docs:
 
 archive: tag
 	@git archive --format=tar --prefix=$(PKGNAME)-$(VERSION)/ $(TAG) > $(PKGNAME)-$(VERSION).tar
-	@gzip $(PKGNAME)-$(VERSION).tar
+	@gzip -f $(PKGNAME)-$(VERSION).tar
 	@echo "The archive is in $(PKGNAME)-$(VERSION).tar.gz"
 
+srpm: archive $(PKGNAME).spec
+	rpmbuild -bs \
+	  --define "_sourcedir $(CURDIR)" \
+	  --define "_srcrpmdir $(CURDIR)" \
+	  lorax-composer.spec
+
 local:
 	@rm -rf $(PKGNAME)-$(VERSION).tar.gz
 	@rm -rf /var/tmp/$(PKGNAME)-$(VERSION)
@@ -106,6 +116,35 @@ docs-in-docker:
 
 ci: check test
 
+$(VM_IMAGE): TAG=HEAD
+$(VM_IMAGE): srpm bots
+	srpm=$(shell rpm --qf '%{Name}-%{Version}-%{Release}.src.rpm\n' -q --specfile lorax-composer.spec | head -n1) ; \
+	bots/image-customize -v \
+		--resize 20G \
+		--upload $$srpm:/var/tmp \
+		--upload $(CURDIR)/test/vm.install:/var/tmp/vm.install \
+		--upload $(realpath tests):/ \
+		--run-command "chmod +x /var/tmp/vm.install" \
+		--run-command "cd /var/tmp; /var/tmp/vm.install $$srpm" \
+		$(TEST_OS)
+	[ -f ~/.config/lorax-test-env ] && bots/image-customize \
+		--upload ~/.config/lorax-test-env:/var/tmp/lorax-test-env \
+		$(TEST_OS) || echo
+
+# convenience target for the above
+vm: $(VM_IMAGE)
+	echo $(VM_IMAGE)
+
+vm-reset:
+	rm -f $(VM_IMAGE) $(VM_IMAGE).qcow2
+
+# checkout Cockpit's bots/ directory for standard test VM images and API to launch them
+# must be from cockpit's master, as only that has current and existing images; but testvm.py API is stable
+bots:
+	git fetch --depth=1 https://github.com/cockpit-project/cockpit.git
+	git checkout --force FETCH_HEAD -- bots/
+	git reset bots
+
 .PHONY: all install check test clean tag docs archive local
 
 .PHONY: ci_after_success

test/README.md

@@ -0,0 +1,60 @@
+# Integration Tests
+
+lorax uses Cockpit's integration test framework and infrastructure. To do this,
+we're checking out Cockpit's `bots/` subdirectory. It contains links to test
+images and tools to manipulate and start virtual machines from them.
+
+Each test is run on a new instance of a virtual machine.
+Branch/test matrix is configured in `bots/tests-scan` inside the
+[cockpit repository](https://github.com/cockpit-project/cockpit).
+
+## Dependencies
+
+These dependencies are needed on Fedora to run tests locally:
+
+    $ sudo dnf install curl expect \
+        libvirt libvirt-client libvirt-daemon libvirt-python \
+        python python-libguestfs python-lxml libguestfs-xfs \
+        python3 libvirt-python3 \
+        libguestfs-tools qemu qemu-kvm rpm-build rsync xz
+
+## Building a test VM
+
+To build a test VM, run
+
+    $ make vm
+
+This downloads a base image from Cockpit's infrastructure. You can control
+which image is downloaded with the `TEST_OS` environment variable. Cockpit's
+[documentation](https://github.com/cockpit-project/cockpit/blob/master/test/README.md#test-configuration)
+lists accepted values. It then creates a new image based on that (a qemu
+snapshot) in `tests/images`, which contain the current `tests/` directory and
+have newly built rpms from the current checkout installed.
+
+To delete the generated image, run
+
+    $ make vm-reset
+
+Base images are stored in `bots/images`. Set `TEST_DATA` to override this
+directory.
+
+## Running tests
+
+After building a test image, run
+
+    $ ./test/check-cli [TESTNAME]
+
+or any of the other `check-*` scripts. To debug a test failure, pass `--sit`.
+This will keep the test machine running after the first failure and print an
+ssh line to connect to it.
+
+Run `make vm` after changing tests or lorax source to recreate the test
+machine. It is usually not necessary to reset the VM.
+
+## Updating images
+
+The `bots/` directory is checked out from Cockpit when `make vm` is first run.
+To get the latest images you need to update it manually (in order not to poll
+GitHub every time):
+
+    $ make -B bots

test/check-api

@@ -0,0 +1,75 @@
+#!/usr/bin/python3
+
+from time import sleep
+
+import composertest
+import requests
+import subprocess
+
+
+class TestApi(composertest.ComposerTestCase):
+    """Test Composer HTTP API"""
+
+    def setUp(self):
+        super(TestApi, self).setUp()
+
+        # Forward /run/weldr/api.socket to a port on the host
+        # Set ExitOnForwardFailure so that ssh blocks until the forward is set
+        # up before going to the background (-f), which it closes stdout. We
+        # wait for that by calling read() on it.
+        self.composer_port = self.network._lock(8080)
+        forwarder_command = self.ssh_command[:]
+        forwarder_command.extend(["-fNT",
+                                  "-o", "ExitOnForwardFailure=yes",
+                                  "-L", "localhost:%d:/run/weldr/api.socket" % self.composer_port])
+        self.forwarder_proc = subprocess.Popen(forwarder_command, stdout=subprocess.PIPE)
+        self.forwarder_proc.stdout.read()
+
+    def tearDown(self):
+        self.forwarder_proc.terminate()
+        sleep(1)  # wait and check for timeout
+        if self.forwarder_proc.poll() is None:
+            self.forwarder_proc.kill()
+        super(TestApi, self).tearDown()
+
+    def request(self, method, path, check=True):
+        self.assertEqual(path[0], "/")
+        url = "http://localhost:%d%s" % (self.composer_port, path)
+        r = requests.request(method, url, timeout=30)
+        if check:
+            r.raise_for_status()
+        return r
+
+    def test_basic(self):
+        """Basic checks for the API"""
+
+        #
+        # API status without depsolve errors
+        #
+        r = self.request("GET", "/api/status")
+        self.assertEqual(r.status_code, 200)
+        status = r.json()
+        self.assertEqual(status.keys(), { "build", "api", "db_version", "schema_version", "db_supported", "backend", "msgs" })
+        self.assertEqual(status["msgs"], [])
+        self.assertEqual(r.headers.keys(), { "Content-Type", "Content-Length", "Date" })
+
+        #
+        # HTTP errors should return json responses
+        #
+        r = self.request("GET", "/marmalade", check=False)
+        self.assertEqual(r.status_code, 404)
+        self.assertEqual(r.json(), {
+            "status": False,
+            "errors": [{ "id": "HTTPError", "code": 404, "msg": "Not Found" }]
+        })
+
+        r = self.request("POST", "/api/status", check=False)
+        self.assertEqual(r.status_code, 405)
+        self.assertEqual(r.json(), {
+            "status": False,
+            "errors": [{ "id": "HTTPError", "code": 405, "msg": "Method Not Allowed" }]
+        })
+
+
+if __name__ == '__main__':
+    composertest.main()

test/check-cli

@@ -0,0 +1,43 @@
+#!/usr/bin/python3
+
+import composertest
+
+
+class TestImages(composertest.ComposerTestCase):
+    """
+        This is the "entry-point" to the test suite when
+        executed in Cockpit CI. If $TEST_SCENARIO=="" or
+        $TEST_SCENARIO="images" we end up here.
+
+        New test methods should be added here first!
+        When this target becomes too slow we split out into
+        separate scenarios!
+    """
+    def test_blueprint_sanity(self):
+        self.runCliTest("/tests/cli/test_blueprints_sanity.sh")
+
+    def test_compose_sanity(self):
+        self.runCliTest("/tests/cli/test_compose_sanity.sh")
+
+    def test_ext4_filesystem(self):
+        self.runCliTest("/tests/cli/test_compose_ext4-filesystem.sh")
+
+    def test_partitioned_disk(self):
+        self.runCliTest("/tests/cli/test_compose_partitioned-disk.sh")
+
+    def test_tar(self):
+        self.runCliTest("/tests/cli/test_compose_tar.sh")
+
+
+class TestQcow2(composertest.ComposerTestCase):
+    def test_qcow2(self):
+        self.runCliTest("/tests/cli/test_compose_qcow2.sh")
+
+
+class TestLiveIso(composertest.ComposerTestCase):
+    def test_live_iso(self):
+        self.runCliTest("/tests/cli/test_compose_live-iso.sh")
+
+
+if __name__ == '__main__':
+    composertest.main()

test/check-cloud

@@ -0,0 +1,21 @@
+#!/usr/bin/python3
+
+import composertest
+
+
+class TestCloud(composertest.ComposerTestCase):
+    def test_aws(self):
+        self.runCliTest("/tests/cli/test_build_and_deploy_aws.sh")
+
+    def test_azure(self):
+        self.runCliTest("/tests/cli/test_build_and_deploy_azure.sh")
+
+    def test_openstack(self):
+        self.runCliTest("/tests/cli/test_build_and_deploy_openstack.sh")
+
+    def test_vmware(self):
+        self.runCliTest("/tests/cli/test_build_and_deploy_vmware.sh")
+
+
+if __name__ == '__main__':
+    composertest.main()

test/composertest.py

@@ -0,0 +1,135 @@
+#!/usr/bin/python3
+
+from __future__ import print_function
+
+import argparse
+import os
+import subprocess
+import sys
+import unittest
+
+# import Cockpit's machinery for test VMs and its browser test API
+sys.path.append(os.path.join(os.path.dirname(__file__), "../bots/machine"))
+import testvm # pylint: disable=import-error
+
+
+def print_exception(etype, value, tb):
+    import traceback
+
+    # only include relevant lines
+    limit = 0
+    while tb and '__unittest' in tb.tb_frame.f_globals:
+        limit += 1
+        tb = tb.tb_next
+
+    traceback.print_exception(etype, value, tb, limit=limit)
+
+
+class ComposerTestCase(unittest.TestCase):
+    image = testvm.DEFAULT_IMAGE
+    sit = False
+
+    def __init__(self, methodName='runTest'):
+        super(ComposerTestCase, self).__init__(methodName=methodName)
+        # by default run() does this and defaultTestResult()
+        # always creates new object which is local for the .run() method
+        self.ci_result = self.defaultTestResult()
+
+    def run(self, result=None):
+        # so we override run() and use an object attribute which we can
+        # reference later in tearDown() and extract the errors from
+        super(ComposerTestCase, self).run(result=self.ci_result)
+
+    def setUp(self):
+        self.network = testvm.VirtNetwork(0)
+        self.machine = testvm.VirtMachine(self.image, networking=self.network.host(), memory_mb=2048)
+
+        print("Starting virtual machine '%s'" % self.image)
+        self.machine.start()
+        self.machine.wait_boot()
+
+        # run a command to force starting the SSH master
+        self.machine.execute("uptime")
+
+        self.ssh_command = ["ssh", "-o", "ControlPath=" + self.machine.ssh_master,
+                                   "-p", self.machine.ssh_port,
+                                   self.machine.ssh_user + "@" + self.machine.ssh_address]
+
+        print("Machine is up. Connect to it via:")
+        print(" ".join(self.ssh_command))
+        print()
+
+        print("Waiting for lorax-composer to become ready...")
+        curl_command = ["curl", "--max-time", "360",
+                                "--silent",
+                                "--unix-socket", "/run/weldr/api.socket",
+                                "http://localhost/api/status"]
+        r = subprocess.call(self.ssh_command + curl_command, stdout=open(os.devnull, 'w'))
+        self.assertEqual(r.returncode, 0)
+
+    def tearDown(self):
+        # `errors` is a list of tuples (method, error)
+        errors = list(e[1] for e in self.ci_result.errors if e[1])
+
+        if errors and self.sit:
+            for e in errors:
+                print_exception(*e)
+
+            print()
+            print(" ".join(self.ssh_command))
+            input("Press RETURN to continue...")
+
+        self.machine.stop()
+
+    def execute(self, command):
+        """Execute a command on the test machine."""
+        return subprocess.call(self.ssh_command + command)
+
+    def runCliTest(self, script):
+        execute_params = ["CLI=/usr/bin/composer-cli",
+                          "TEST=" + self.id(),
+                          "PACKAGE=composer-cli",
+                          "/tests/test_cli.sh", script]
+        if self.sit:
+            execute_params.insert(0, "COMPOSER_TEST_FAIL_FAST=1")
+
+        r = self.execute(execute_params)
+        self.assertEqual(r.returncode, 0)
+
+
+def print_tests(tests):
+    for test in tests:
+        if isinstance(test, unittest.TestSuite):
+            print_tests(test)
+        # I don't know how this is used when running the tests
+        # (maybe not used from what it looks like) so not sure how to refactor it
+        # elif isinstance(test, unittest.loader._FailedTest):
+        #    name = test.id().replace("unittest.loader._FailedTest.", "")
+        #    print("Error: '%s' does not match a test" % name, file=sys.stderr)
+        else:
+            print(test.id().replace("__main__.", ""))
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("tests", nargs="*", help="List of tests modules, classes, and methods")
+    parser.add_argument("-l", "--list", action="store_true", help="Print the list of tests that would be executed")
+    parser.add_argument("-s", "--sit", action="store_true", help="Halt test execution (but keep VM running) when a test fails")
+    args = parser.parse_args()
+
+    ComposerTestCase.sit = args.sit
+
+    module = __import__("__main__")
+    if args.tests:
+        tests = unittest.defaultTestLoader.loadTestsFromNames(args.tests, module)
+    else:
+        tests = unittest.defaultTestLoader.loadTestsFromModule(module)
+
+    if args.list:
+        print_tests(tests)
+        return 0
+
+    runner = unittest.TextTestRunner(verbosity=2, failfast=args.sit)
+    result = runner.run(tests)
+
+    sys.exit(not result.wasSuccessful())

test/run

@@ -0,0 +1,18 @@
+#!/bin/sh -e
+# This is the expected entry point for Cockpit CI; will be called without
+# arguments but with an appropriate $TEST_OS, and optionally $TEST_SCENARIO
+
+make vm
+
+if [ -n "$TEST_SCENARIO" ]; then
+  if [ "$TEST_SCENARIO" == "live-iso" ]; then
+    test/check-cli TestLiveIso
+  elif [ "$TEST_SCENARIO" == "qcow2" ]; then
+    test/check-cli TestQcow2
+  else
+    test/check-cloud TestCloud.test_$TEST_SCENARIO
+  fi
+else
+  test/check-cli TestImages
+  test/check-api
+fi

test/vm.install

@@ -0,0 +1,35 @@
+#!/bin/sh -eux
+
+SRPM="$1"
+
+if ! rpm -q beakerlib; then
+    if [ $(. /etc/os-release && echo $ID) = "rhel" ]; then
+        (cd /etc/yum.repos.d; curl -O -L http://download.devel.redhat.com/beakerrepos/beaker-client-RedHatEnterpriseLinux.repo)
+
+         # The beaker repository doesn't include repos for minor releases
+         VERSION=$(. /etc/os-release && echo ${VERSION_ID%.*})
+         yum install -y --releasever=$VERSION --setopt=sslverify=0 beakerlib
+
+         # prevent yum from trying to sync the cache again later (it fails without sslverify=0)
+         rm /etc/yum.repos.d/beaker-client-RedHatEnterpriseLinux.repo
+     else
+         yum install -y beakerlib
+    fi
+fi
+
+# Grow root partition to make room for images. This only works on Fedora right now.
+parted --script /dev/vda resizepart 2 100%
+partprobe
+pvs --noheadings -opv_name | xargs pvresize
+rootlv=$(findmnt --noheadings -oSOURCE /)
+lvresize $rootlv -l+100%FREE -r
+
+rm -rf build-results
+su builder -c "/usr/bin/mock --no-clean --resultdir build-results --rebuild $SRPM"
+
+packages=$(find build-results -name '*.rpm' -not -name '*.src.rpm')
+rpm -e --verbose $(basename -a ${packages[@]} | sed 's/-[0-9].*.rpm$//') || true
+yum install -y $packages
+
+systemctl enable lorax-composer.socket
+systemctl enable docker.service

tests/cli/lib/lib.sh

@@ -1,5 +1,21 @@
 #!/usr/bin/env bash
 
+# Monkey-patch beakerlib to exit on first failure if COMPOSER_TEST_FAIL_FAST is
+# set. https://github.com/beakerlib/beakerlib/issues/42
+if [ "$COMPOSER_TEST_FAIL_FAST" == "1" ]; then
+  eval "original$(declare -f __INTERNAL_LogAndJournalFail)"
+
+  __INTERNAL_LogAndJournalFail () {
+    original__INTERNAL_LogAndJournalFail
+
+    # end test somewhat cleanly so that beakerlib logs the FAIL correctly
+    rlPhaseEnd
+    rlJournalEnd
+
+    exit 1
+  }
+fi
+
 # a generic helper function unifying the specific checks executed on a running
 # image instance
 verify_image() {
@@ -31,7 +47,7 @@ check_root_account() {
         # ssh returns 255 in case of any ssh error, so it's better to grep the specific error message
         rlRun -t -c "ssh $SSH_OPTS -o PubkeyAuthentication=no root@${SSH_MACHINE} 2>&1 | grep -i 'permission denied ('" \
             0 "Can't ssh to '$SSH_MACHINE' as root using password-based auth"
-        rlRun -t -c "ssh $SSH_OPTS ${SSH_USER}@${SSH_MACHINE} \"sudo grep -E '^root:(\*LOCK\*|!)' /etc/shadow\"" \
+        rlRun -t -c "ssh $SSH_OPTS ${SSH_USER}@${SSH_MACHINE} \"sudo grep -E '^root:(\*LOCK\*|!):' /etc/shadow\"" \
             0 "root account is disabled in /etc/shadow"
         rlRun -t -c "ssh $SSH_OPTS ${SSH_USER}@${SSH_MACHINE} \"sudo grep 'USER_LOGIN.*acct=\\\"root\\\".*terminal=ssh.*res=failed' /var/log/audit/audit.log\"" \
             0 "audit.log contains entry about unsuccessful root login"

tests/cli/test_blueprints_sanity.sh

@@ -1,7 +1,10 @@
 #!/bin/bash
 # Note: execute this file from the project root directory
 
+set -e
+
 . /usr/share/beakerlib/beakerlib.sh
+. $(dirname $0)/lib/lib.sh
 
 CLI="${CLI:-./src/bin/composer-cli}"
 

tests/cli/test_build_and_deploy_aws.sh

@@ -7,8 +7,10 @@
 #
 #####
 
+set -e
+
 . /usr/share/beakerlib/beakerlib.sh
-. ./tests/cli/lib/lib.sh
+. $(dirname 0)/lib/lib.sh
 
 CLI="${CLI:-./src/bin/composer-cli}"
 
@@ -82,7 +84,7 @@ __EOF__
 
     rlPhaseStartTest "compose finished"
         if [ -n "$UUID" ]; then
-            until $CLI compose details $UUID | grep FINISHED; do
+            until $CLI compose details $UUID | grep 'FINISHED\|FAILED'; do
                 rlLogInfo "Waiting for compose to finish ..."
                 sleep 30
             done;

tests/cli/test_build_and_deploy_azure.sh

@@ -7,8 +7,10 @@
 #
 #####
 
+set -e
+
 . /usr/share/beakerlib/beakerlib.sh
-. ./tests/cli/lib/lib.sh
+. $(dirname $0)/lib/lib.sh
 
 CLI="${CLI:-./src/bin/composer-cli}"
 
@@ -70,7 +72,7 @@ rlJournalStart
 
     rlPhaseStartTest "compose finished"
         if [ -n "$UUID" ]; then
-            until $CLI compose details $UUID | grep FINISHED; do
+            until $CLI compose details $UUID | grep 'FINISHED\|FAILED'; do
                 rlLogInfo "Waiting for compose to finish ..."
                 sleep 30
             done;

tests/cli/test_build_and_deploy_openstack.sh

@@ -7,9 +7,10 @@
 #
 #####
 
-. /usr/share/beakerlib/beakerlib.sh
-. ./tests/cli/lib/lib.sh
+set -e
 
+. /usr/share/beakerlib/beakerlib.sh
+. $(dirname $0)/lib/lib.sh
 
 CLI="${CLI:-./src/bin/composer-cli}"
 VENV=`mktemp -d /tmp/ansible.venv.XXX`
@@ -29,8 +30,8 @@ rlJournalStart
             rlLogInfo "OS_USERNAME=$OS_USERNAME"
         fi
 
-        export OS_TENANT_NAME="${OS_TENANT_NAME:-$OS_USERNAME}"
-        rlLogInfo "OS_TENANT_NAME=$OS_TENANT_NAME"
+        export OS_PROJECT_NAME="${OS_PROJECT_NAME:-$OS_USERNAME}"
+        rlLogInfo "OS_PROJECT_NAME=$OS_PROJECT_NAME"
 
         if [ -z "$OS_PASSWORD" ]; then
             rlFail "OS_PASSWORD is empty!"
@@ -81,7 +82,7 @@ __EOF__
 
     rlPhaseStartTest "compose finished"
         if [ -n "$UUID" ]; then
-            until $CLI compose details $UUID | grep FINISHED; do
+            until $CLI compose details $UUID | grep 'FINISHED\|FAILED'; do
                 rlLogInfo "Waiting for compose to finish ..."
                 sleep 30
             done;

tests/cli/test_build_and_deploy_vmware.sh

@@ -1,5 +1,5 @@
 #!/bin/bash
-# Note: Execute this file from the project root directory
+# Note: execute this file from the project root directory
 
 #####
 #
@@ -7,8 +7,10 @@
 #
 #####
 
+set -e
+
 . /usr/share/beakerlib/beakerlib.sh
-. ./tests/cli/lib/lib.sh
+. $(dirname $0)/lib/lib.sh
 
 CLI="${CLI:-./src/bin/composer-cli}"
 
@@ -65,7 +67,7 @@ rlJournalStart
         SAMPLES="$SAMPLES/samples"
     rlPhaseEnd
 
-    rlPhaseStartTest "Compose start"
+    rlPhaseStartTest "compose start"
         rlAssertEquals "SELinux operates in enforcing mode" "$(getenforce)" "Enforcing"
         SSH_KEY_DIR=`mktemp -d /tmp/composer-ssh-keys.XXXXXX`
         rlRun -t -c "ssh-keygen -t rsa -N '' -f $SSH_KEY_DIR/id_rsa"
@@ -96,9 +98,9 @@ __EOF__
         UUID=`echo $UUID | cut -f 2 -d' '`
     rlPhaseEnd
 
-    rlPhaseStartTest "Compose finished"
+    rlPhaseStartTest "compose finished"
         if [ -n "$UUID" ]; then
-            until $CLI compose details $UUID | grep FINISHED; do
+            until $CLI compose details $UUID | grep 'FINISHED\|FAILED'; do
                 rlLogInfo "Waiting for compose to finish ..."
                 sleep 30
             done;
@@ -113,7 +115,7 @@ __EOF__
 
         python $SAMPLES/upload_file_to_datastore.py -S -s $V_HOST -u $V_USERNAME -p $V_PASSWORD \
                 -d $V_DATASTORE -l `readlink -f $IMAGE` -r $IMAGE
-        rlAssert0 "Image upload successful" $?
+        rlAssert0 "Image upload successfull" $?
     rlPhaseEnd
 
     rlPhaseStartTest "Start VM instance"
@@ -129,13 +131,13 @@ __EOF__
             rlLogInfo "INSTANCE_UUID=$INSTANCE_UUID"
         fi
 
-        # Wait for instance to become running and had assigned a public IP
+        # wait for instance to become running and had assigned a public IP
         IP_ADDRESS="None"
         while [ "$IP_ADDRESS" == "None" ]; do
             rlLogInfo "IP_ADDRESS is not assigned yet ..."
             sleep 30
             IP_ADDRESS=`python $SAMPLES/find_by_uuid.py -S -s $V_HOST -u $V_USERNAME -p $V_PASSWORD \
-                            --uuid $INSTANCE_UUID | grep 'ip address' | tr -d ' ' | cut -f2 -d:`
+                            --uuid $INSTANCE_UUID | grep 'ip address' | tr -d ' ' | cut -f2- -d:`
         done
 
         rlLogInfo "Running instance IP_ADDRESS=$IP_ADDRESS"
@@ -150,7 +152,7 @@ __EOF__
     rlPhaseEnd
 
     rlPhaseStartCleanup
-        # note: VMDK disk is removed when destroying the VM
+        # note: vmdk disk is removed when destroying the VM
         python $SAMPLES/destroy_vm.py -S -s $V_HOST -u $V_USERNAME -p $V_PASSWORD --uuid $INSTANCE_UUID
         rlAssert0 "VM destroyed" $?
         rlRun -t -c "$CLI compose delete $UUID"

tests/cli/test_compose_ext4-filesystem.sh

@@ -9,7 +9,10 @@
 #
 #####
 
+set -e
+
 . /usr/share/beakerlib/beakerlib.sh
+. $(dirname $0)/lib/lib.sh
 
 CLI="${CLI:-./src/bin/composer-cli}"
 
@@ -25,7 +28,7 @@ rlJournalStart
 
     rlPhaseStartTest "compose finished"
         if [ -n "$UUID" ]; then
-            until $CLI compose details $UUID | grep FINISHED; do
+            until $CLI compose details $UUID | grep 'FINISHED\|FAILED'; do
                 sleep 60
                 rlLogInfo "Waiting for compose to finish ..."
             done;

tests/cli/test_compose_live-iso.sh

@@ -7,15 +7,18 @@
 #
 #####
 
+set -e
+
 . /usr/share/beakerlib/beakerlib.sh
-. ./tests/cli/lib/lib.sh
+. $(dirname $0)/lib/lib.sh
 
 CLI="${CLI:-./src/bin/composer-cli}"
-QEMU="/usr/libexec/qemu-kvm"
+QEMU_BIN="/usr/bin/qemu-system-$(uname -m)"
+QEMU="$QEMU_BIN -machine accel=kvm:tcg"
 
 rlJournalStart
     rlPhaseStartSetup
-        rlAssertExists $QEMU
+        rlAssertExists $QEMU_BIN
     rlPhaseEnd
 
     rlPhaseStartTest "compose start"
@@ -30,7 +33,7 @@ rlJournalStart
 
     rlPhaseStartTest "compose finished"
         if [ -n "$UUID" ]; then
-            until $CLI compose details $UUID | grep FINISHED; do
+            until $CLI compose details $UUID | grep 'FINISHED\|FAILED'; do
                 sleep 20
                 rlLogInfo "Waiting for compose to finish ..."
             done;
@@ -56,7 +59,7 @@ rlJournalStart
     rlPhaseEnd
 
     rlPhaseStartCleanup
-        rlRun -t -c "killall -9 $(basename $QEMU)"
+        rlRun -t -c "killall -9 $(basename $QEMU_BIN)"
         rlRun -t -c "$CLI compose delete $UUID"
         rlRun -t -c "rm -rf $IMAGE"
     rlPhaseEnd

tests/cli/test_compose_partitioned-disk.sh

@@ -9,7 +9,10 @@
 #
 #####
 
+set -e
+
 . /usr/share/beakerlib/beakerlib.sh
+. $(dirname $0)/lib/lib.sh
 
 CLI="${CLI:-./src/bin/composer-cli}"
 
@@ -25,7 +28,7 @@ rlJournalStart
 
     rlPhaseStartTest "compose finished"
         if [ -n "$UUID" ]; then
-            until $CLI compose details $UUID | grep FINISHED; do
+            until $CLI compose details $UUID | grep 'FINISHED\|FAILED'; do
                 sleep 60
                 rlLogInfo "Waiting for compose to finish ..."
             done;

tests/cli/test_compose_qcow2.sh

@@ -7,15 +7,18 @@
 #
 #####
 
+set -e
+
 . /usr/share/beakerlib/beakerlib.sh
-. ./tests/cli/lib/lib.sh
+. $(dirname $0)/lib/lib.sh
 
 CLI="${CLI:-./src/bin/composer-cli}"
-QEMU="/usr/libexec/qemu-kvm"
+QEMU_BIN="/usr/bin/qemu-system-$(uname -m)"
+QEMU="$QEMU_BIN -machine accel=kvm:tcg"
 
 rlJournalStart
     rlPhaseStartSetup
-        rlAssertExists $QEMU
+        rlAssertExists $QEMU_BIN
     rlPhaseEnd
 
     rlPhaseStartTest "compose start"
@@ -58,7 +61,7 @@ __EOF__
 
     rlPhaseStartTest "compose finished"
         if [ -n "$UUID" ]; then
-            until $CLI compose details $UUID | grep FINISHED; do
+            until $CLI compose details $UUID | grep 'FINISHED\|FAILED'; do
                 sleep 20
                 rlLogInfo "Waiting for compose to finish ..."
             done;
@@ -82,7 +85,7 @@ __EOF__
     rlPhaseEnd
 
     rlPhaseStartCleanup
-        rlRun -t -c "killall -9 $(basename $QEMU)"
+        rlRun -t -c "killall -9 $(basename $QEMU_BIN)"
         rlRun -t -c "$CLI compose delete $UUID"
         rlRun -t -c "rm -rf $IMAGE $TMP_DIR $SSH_KEY_DIR"
     rlPhaseEnd

tests/cli/test_compose_sanity.sh

@@ -1,7 +1,10 @@
 #!/bin/bash
 # Note: execute this file from the project root directory
 
+set -e
+
 . /usr/share/beakerlib/beakerlib.sh
+. $(dirname $0)/lib/lib.sh
 
 CLI="${CLI:-./src/bin/composer-cli}"
 
@@ -29,7 +32,7 @@ rlJournalStart
 
     rlPhaseStartTest "compose image"
         if [ -n "$UUID" ]; then
-            until $CLI compose details $UUID | grep FINISHED; do
+            until $CLI compose details $UUID | grep 'FINISHED\|FAILED'; do
                 sleep 60
                 rlLogInfo "Waiting for compose to finish ..."
             done;

tests/cli/test_compose_tar.sh

@@ -7,7 +7,10 @@
 #
 #####
 
+set -e
+
 . /usr/share/beakerlib/beakerlib.sh
+. $(dirname $0)/lib/lib.sh
 
 CLI="${CLI:-./src/bin/composer-cli}"
 
@@ -28,7 +31,7 @@ rlJournalStart
 
     rlPhaseStartTest "compose finished"
         if [ -n "$UUID" ]; then
-            until $CLI compose details $UUID | grep FINISHED; do
+            until $CLI compose details $UUID | grep 'FINISHED\|FAILED'; do
                 sleep 10
                 rlLogInfo "Waiting for compose to finish ..."
             done;
@@ -36,6 +39,13 @@ rlJournalStart
             rlFail "Compose UUID is empty!"
         fi
 
+        # Running a compose can lead to a different selinux policy in the
+        # kernel, which may break docker. Reload the policy from the host and
+        # restart docker as a workaround.
+        # See https://bugzilla.redhat.com/show_bug.cgi?id=1711813
+        semodule -R
+        systemctl restart docker
+
         rlRun -t -c "$CLI compose image $UUID"
         IMAGE="$UUID-root.tar.xz"
     rlPhaseEnd
@@ -49,8 +59,8 @@ rlJournalStart
 
     rlPhaseStartTest "Verify tar image with systemd-nspawn"
         if [ -f /usr/bin/systemd-nspawn ]; then
-            NSPAWN_DIR=`mktemp -d /tmp/nspawn.XXXX`
-            rlRun -t -c "tar -xJvf $IMAGE -C $NSPAWN_DIR"
+            NSPAWN_DIR=`mktemp -d /var/tmp/nspawn.XXXX`
+            rlRun -t -c "tar -xJf $IMAGE -C $NSPAWN_DIR"
 
             # verify we can run a container with this image
             rlRun -t -c "systemd-nspawn -D $NSPAWN_DIR cat /etc/redhat-release"

tests/test_cli.sh

@@ -1,8 +1,10 @@
 #!/bin/bash
-# Note: Execute this file from the project root directory
+# Note: execute this file from the project root directory
 
-# setup
-rm -rf /var/tmp/beakerlib-*/
+set -eu
+
+export BEAKERLIB_DIR=$(mktemp -d /tmp/composer-test.XXXXXX)
+CLI="${CLI:-}"
 
 function setup_tests {
     local share_dir=$1
@@ -43,6 +45,15 @@ function teardown_tests {
     mv ${blueprints_dir}.orig $blueprints_dir
 }
 
+# cloud credentials
+if [ -f "~/.config/lorax-test-env" ]; then
+    . ~/.config/lorax-test-env
+fi
+
+if [ -f "/var/tmp/lorax-test-env" ]; then
+    . /var/tmp/lorax-test-env
+fi
+
 if [ -z "$CLI" ]; then
     export top_srcdir=`pwd`
     . ./tests/testenv.sh
@@ -55,7 +66,7 @@ if [ -z "$CLI" ]; then
     chmod a+rx -R $SHARE_DIR
 
     setup_tests $SHARE_DIR $BLUEPRINTS_DIR
-    # Start the lorax-composer daemon
+    # start the lorax-composer daemon
     ./src/sbin/lorax-composer --sharedir $SHARE_DIR $BLUEPRINTS_DIR &
 else
     export PACKAGE="composer-cli"
@@ -65,14 +76,14 @@ else
 fi
 
 
-# Wait for the backend to become ready
+# wait for the backend to become ready
 tries=0
 until curl -m 15 --unix-socket /run/weldr/api.socket http://localhost:4000/api/status | grep 'db_supported.*true'; do
     tries=$((tries + 1))
-    if [ $tries -gt 20 ]; then
+    if [ $tries -gt 50 ]; then
         exit 1
     fi
-    sleep 2
+    sleep 5
     echo "DEBUG: Waiting for backend API to become ready before testing ..."
 done;
 
@@ -80,14 +91,14 @@ done;
 export BEAKERLIB_JOURNAL=0
 export PATH="/usr/local/bin:$PATH"
 if [ -z "$*" ]; then
-    # Invoke cli/ tests which can be executed without special preparation
+    # invoke cli/ tests which can be executed without special preparation
     ./tests/cli/test_blueprints_sanity.sh
     ./tests/cli/test_compose_sanity.sh
 else
-    # Execute other cli tests which need more adjustments in the calling environment
+    # execute other cli tests which need more adjustments in the calling environment
     # or can't be executed inside Travis CI
     for TEST in "$@"; do
-        ./$TEST
+        $TEST
     done
 fi
 
@@ -106,8 +117,11 @@ else
     systemctl start lorax-composer
 fi
 
-# Look for failures
-grep RESULT_STRING /var/tmp/beakerlib-*/TestResults | grep -v PASS && exit 1
+. $BEAKERLIB_DIR/TestResults
 
-# Explicit return code for Makefile
-exit 0
+if [ $TESTRESULT_RESULT_ECODE != 0 ]; then
+  echo "Test failed. Leaving log in $BEAKERLIB_DIR"
+  exit $TESTRESULT_RESULT_ECODE
+fi
+
+rm -rf $BEAKERLIB_DIR

tests/testenv.sh

@@ -1,5 +1,9 @@
 #!/bin/sh
 
+top_srcdir="${top_srcdir:-}"
+top_buildir="${top_builddir:-}"
+PYTHONPATH="${PYTHONPATH:-}"
+
 if [ -z "$top_srcdir" ]; then
     echo "*** top_srcdir must be set"
     exit 1