Backport Cockpit CI changes for RHEL 8

Notes:

- also install podman-docker
- on RHEL 8 there's no docker.service to start

Related: rhbz#1724073

Makefile

@@ -9,6 +9,12 @@ TAG = lorax-$(VERSION)-$(RELEASE)
 
 IMAGE_RELEASE = rhel8-latest
 
+ifeq ($(TEST_OS),)
+TEST_OS = rhel-8-1
+endif
+export TEST_OS
+VM_IMAGE=$(CURDIR)/test/images/$(TEST_OS)
+
 default: all
 
 src/composer/version.py: lorax.spec
@@ -43,8 +49,6 @@ test:
 	coverage3 report -m
 	[ -f "/usr/bin/coveralls" ] && [ -n "$(COVERALLS_REPO_TOKEN)" ] && coveralls || echo
 
-	./tests/test_cli.sh
-
 # need `losetup`, which needs Docker to be in privileged mode (--privileged)
 # but even so fails in Travis CI
 test_images:
@@ -78,12 +82,18 @@ docs:
 
 archive:
 	@git archive --format=tar --prefix=$(PKGNAME)-$(VERSION)/ $(TAG) > $(PKGNAME)-$(VERSION).tar
-	@gzip $(PKGNAME)-$(VERSION).tar
+	@gzip -f $(PKGNAME)-$(VERSION).tar
 	@echo "The archive is in $(PKGNAME)-$(VERSION).tar.gz"
 
 dist: tag archive
 	scp $(PKGNAME)-$(VERSION).tar.gz fedorahosted.org:lorax
 
+srpm: archive $(PKGNAME).spec
+	rpmbuild -bs \
+	  --define "_sourcedir $(CURDIR)" \
+	  --define "_srcrpmdir $(CURDIR)" \
+	  lorax.spec
+
 local:
 	@rm -rf $(PKGNAME)-$(VERSION).tar.gz
 	@rm -rf /var/tmp/$(PKGNAME)-$(VERSION)
@@ -107,8 +117,38 @@ docs-in-docker:
 
 ci: check test
 
+$(VM_IMAGE): TAG=HEAD
+$(VM_IMAGE): srpm bots
+	srpm=$(shell rpm --qf '%{Name}-%{Version}-%{Release}.src.rpm\n' -q --specfile lorax.spec | head -n1) ; \
+	bots/image-customize -v \
+		--resize 20G \
+		--upload $$srpm:/var/tmp \
+		--upload $(CURDIR)/test/vm.install:/var/tmp/vm.install \
+		--upload $(realpath tests):/ \
+		--run-command "chmod +x /var/tmp/vm.install" \
+		--run-command "cd /var/tmp; /var/tmp/vm.install $$srpm" \
+		$(TEST_OS)
+	[ -f ~/.config/lorax-test-env ] && bots/image-customize \
+		--upload ~/.config/lorax-test-env:/var/tmp/lorax-test-env \
+		$(TEST_OS) || echo
+
+
+# convenience target for the above
+vm: $(VM_IMAGE)
+	echo $(VM_IMAGE)
+
+vm-reset:
+	rm -f $(VM_IMAGE) $(VM_IMAGE).qcow2
+
+# checkout Cockpit's bots/ directory for standard test VM images and API to launch them
+# must be from cockpit's master, as only that has current and existing images; but testvm.py API is stable
+bots:
+	git fetch --depth=1 https://github.com/cockpit-project/cockpit.git
+	git checkout --force FETCH_HEAD -- bots/
+	git reset bots
+
 .PHONY: ci_after_success
 ci_after_success:
 # nothing to do here, but Jenkins expects this to be present, otherwise fails
 
-.PHONY: docs
+.PHONY: docs check test srpm vm vm-reset

test/README.md

@@ -0,0 +1,60 @@
+# Integration Tests
+
+lorax uses Cockpit's integration test framework and infrastructure. To do this,
+we're checking out Cockpit's `bots/` subdirectory. It contains links to test
+images and tools to manipulate and start virtual machines from them.
+
+Each test is run on a new instance of a virtual machine.
+Branch/test matrix is configured in `bots/tests-scan` inside the
+[cockpit repository](https://github.com/cockpit-project/cockpit).
+
+## Dependencies
+
+These dependencies are needed on Fedora to run tests locally:
+
+    $ sudo dnf install curl expect \
+        libvirt libvirt-client libvirt-daemon libvirt-python \
+        python python-libguestfs python-lxml libguestfs-xfs \
+        python3 libvirt-python3 \
+        libguestfs-tools qemu qemu-kvm rpm-build rsync xz
+
+## Building a test VM
+
+To build a test VM, run
+
+    $ make vm
+
+This downloads a base image from Cockpit's infrastructure. You can control
+which image is downloaded with the `TEST_OS` environment variable. Cockpit's
+[documentation](https://github.com/cockpit-project/cockpit/blob/master/test/README.md#test-configuration)
+lists accepted values. It then creates a new image based on that (a qemu
+snapshot) in `tests/images`, which contain the current `tests/` directory and
+have newly built rpms from the current checkout installed.
+
+To delete the generated image, run
+
+    $ make vm-reset
+
+Base images are stored in `bots/images`. Set `TEST_DATA` to override this
+directory.
+
+## Running tests
+
+After building a test image, run
+
+    $ ./test/check-cli [TESTNAME]
+
+or any of the other `check-*` scripts. To debug a test failure, pass `--sit`.
+This will keep the test machine running after the first failure and print an
+ssh line to connect to it.
+
+Run `make vm` after changing tests or lorax source to recreate the test
+machine. It is usually not necessary to reset the VM.
+
+## Updating images
+
+The `bots/` directory is checked out from Cockpit when `make vm` is first run.
+To get the latest images you need to update it manually (in order not to poll
+GitHub every time):
+
+    $ make -B bots

test/check-cli

@@ -0,0 +1,43 @@
+#!/usr/bin/python3
+
+import composertest
+
+
+class TestImages(composertest.ComposerTestCase):
+    """
+        This is the "entry-point" to the test suite when
+        executed in Cockpit CI. If $TEST_SCENARIO=="" or
+        $TEST_SCENARIO="images" we end up here.
+
+        New test methods should be added here first!
+        When this target becomes too slow we split out into
+        separate scenarios!
+    """
+    def test_blueprint_sanity(self):
+        self.runCliTest("/tests/cli/test_blueprints_sanity.sh")
+
+    def test_compose_sanity(self):
+        self.runCliTest("/tests/cli/test_compose_sanity.sh")
+
+    def test_ext4_filesystem(self):
+        self.runCliTest("/tests/cli/test_compose_ext4-filesystem.sh")
+
+    def test_partitioned_disk(self):
+        self.runCliTest("/tests/cli/test_compose_partitioned-disk.sh")
+
+    def test_tar(self):
+        self.runCliTest("/tests/cli/test_compose_tar.sh")
+
+
+class TestQcow2(composertest.ComposerTestCase):
+    def test_qcow2(self):
+        self.runCliTest("/tests/cli/test_compose_qcow2.sh")
+
+
+class TestLiveIso(composertest.ComposerTestCase):
+    def test_live_iso(self):
+        self.runCliTest("/tests/cli/test_compose_live-iso.sh")
+
+
+if __name__ == '__main__':
+    composertest.main()

test/check-cloud

@@ -0,0 +1,21 @@
+#!/usr/bin/python3
+
+import composertest
+
+
+class TestCloud(composertest.ComposerTestCase):
+    def test_aws(self):
+        self.runCliTest("/tests/cli/test_build_and_deploy_aws.sh")
+
+    def test_azure(self):
+        self.runCliTest("/tests/cli/test_build_and_deploy_azure.sh")
+
+    def test_openstack(self):
+        self.runCliTest("/tests/cli/test_build_and_deploy_openstack.sh")
+
+    def test_vmware(self):
+        self.runCliTest("/tests/cli/test_build_and_deploy_vmware.sh")
+
+
+if __name__ == '__main__':
+    composertest.main()

test/composertest.py

@@ -0,0 +1,125 @@
+#!/usr/bin/python3
+
+import argparse
+import os
+import subprocess
+import sys
+import unittest
+
+# import Cockpit's machinery for test VMs and its browser test API
+sys.path.append(os.path.join(os.path.dirname(__file__), "../bots/machine"))
+import testvm # pylint: disable=import-error
+
+
+def print_exception(etype, value, tb):
+    import traceback
+
+    # only include relevant lines
+    limit = 0
+    while tb and '__unittest' in tb.tb_frame.f_globals:
+        limit += 1
+        tb = tb.tb_next
+
+    traceback.print_exception(etype, value, tb, limit=limit)
+
+
+class ComposerTestCase(unittest.TestCase):
+    image = testvm.DEFAULT_IMAGE
+    sit = False
+
+    def setUp(self):
+        self.network = testvm.VirtNetwork(0)
+        self.machine = testvm.VirtMachine(self.image, networking=self.network.host(), memory_mb=2048)
+
+        print(f"Starting virtual machine '{self.image}'")
+        self.machine.start()
+        self.machine.wait_boot()
+
+        # run a command to force starting the SSH master
+        self.machine.execute("uptime")
+
+        self.ssh_command = ["ssh", "-o", "ControlPath=" + self.machine.ssh_master,
+                                   "-p", self.machine.ssh_port,
+                                   self.machine.ssh_user + "@" + self.machine.ssh_address]
+
+        print("Machine is up. Connect to it via:")
+        print(" ".join(self.ssh_command))
+        print()
+
+        print("Waiting for lorax-composer to become ready...")
+        curl_command = ["curl", "--max-time", "360",
+                                "--silent",
+                                "--unix-socket", "/run/weldr/api.socket",
+                                "http://localhost/api/status"]
+        r = subprocess.run(self.ssh_command + curl_command, stdout=subprocess.DEVNULL)
+        self.assertEqual(r.returncode, 0)
+
+    def tearDown(self):
+        # Peek into internal data structure, because there's no way to get the
+        # TestResult at this point. `errors` is a list of tuples (method, error)
+        errors = list(e[1] for e in self._outcome.errors if e[1])
+
+        if errors and self.sit:
+            for e in errors:
+                print_exception(*e)
+
+            print()
+            print(" ".join(self.ssh_command))
+            input("Press RETURN to continue...")
+
+        self.machine.stop()
+
+    def execute(self, command, **args):
+        """Execute a command on the test machine.
+
+        **args and return value are the same as those for subprocess.run().
+        """
+        return subprocess.run(self.ssh_command + command, **args)
+
+    def runCliTest(self, script):
+        extra_env = []
+        if self.sit:
+            extra_env.append("COMPOSER_TEST_FAIL_FAST=1")
+
+        r = self.execute(["CLI=/usr/bin/composer-cli",
+                          "TEST=" + self.id(),
+                          "PACKAGE=composer-cli",
+                          *extra_env,
+                          "/tests/test_cli.sh", script])
+        self.assertEqual(r.returncode, 0)
+
+
+def print_tests(tests):
+    for test in tests:
+        if isinstance(test, unittest.TestSuite):
+            print_tests(test)
+        elif isinstance(test, unittest.loader._FailedTest):
+            name = test.id().replace("unittest.loader._FailedTest.", "")
+            print(f"Error: '{name}' does not match a test", file=sys.stderr)
+        else:
+            print(test.id().replace("__main__.", ""))
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("tests", nargs="*", help="List of tests modules, classes, and methods")
+    parser.add_argument("-l", "--list", action="store_true", help="Print the list of tests that would be executed")
+    parser.add_argument("-s", "--sit", action="store_true", help="Halt test execution (but keep VM running) when a test fails")
+    args = parser.parse_args()
+
+    ComposerTestCase.sit = args.sit
+
+    module = __import__("__main__")
+    if args.tests:
+        tests = unittest.defaultTestLoader.loadTestsFromNames(args.tests, module)
+    else:
+        tests = unittest.defaultTestLoader.loadTestsFromModule(module)
+
+    if args.list:
+        print_tests(tests)
+        return 0
+
+    runner = unittest.TextTestRunner(verbosity=2, failfast=args.sit)
+    result = runner.run(tests)
+
+    sys.exit(not result.wasSuccessful())

test/run

@@ -0,0 +1,17 @@
+#!/bin/sh -e
+# This is the expected entry point for Cockpit CI; will be called without
+# arguments but with an appropriate $TEST_OS, and optionally $TEST_SCENARIO
+
+make vm
+
+if [ -n "$TEST_SCENARIO" ]; then
+  if [ "$TEST_SCENARIO" == "live-iso" ]; then
+    test/check-cli TestLiveIso
+  elif [ "$TEST_SCENARIO" == "qcow2" ]; then
+    test/check-cli TestQcow2
+  else
+    test/check-cloud TestCloud.test_$TEST_SCENARIO
+  fi
+else
+  test/check-cli TestImages
+fi

test/vm.install

@@ -0,0 +1,46 @@
+#!/bin/sh -eux
+
+SRPM="$1"
+
+if ! rpm -q beakerlib; then
+    if [ $(. /etc/os-release && echo $ID) = "rhel" ]; then
+        (cd /etc/yum.repos.d; curl -O -L http://download.devel.redhat.com/beakerrepos/beaker-client-RedHatEnterpriseLinux.repo)
+
+         # The beaker repository doesn't include repos for minor releases
+         VERSION=$(. /etc/os-release && echo ${VERSION_ID%.*})
+         yum install -y --releasever=$VERSION --setopt=sslverify=0 beakerlib
+
+         # prevent yum from trying to sync the cache again later (it fails without sslverify=0)
+         rm /etc/yum.repos.d/beaker-client-RedHatEnterpriseLinux.repo
+     else
+         yum install -y beakerlib
+    fi
+fi
+
+if ! rpm -q podman-docker; then
+    yum install -y podman-docker
+fi
+
+if ! rpm -q git; then
+    yum install -y git
+fi
+
+if ! rpm -q qemu-kvm; then
+    yum install -y qemu-kvm
+fi
+
+# Grow root partition to make room for images. This only works on Fedora right now.
+parted --script /dev/vda resizepart 2 100%
+partprobe
+pvs --noheadings -opv_name | xargs pvresize
+rootlv=$(findmnt --noheadings -oSOURCE /)
+lvresize $rootlv -l+100%FREE -r
+
+rm -rf build-results
+su builder -c "/usr/bin/mock --no-clean --resultdir build-results --rebuild $SRPM"
+
+packages=$(find build-results -name '*.rpm' -not -name '*.src.rpm')
+rpm -e --verbose $(basename -a ${packages[@]} | sed 's/-[0-9].*.rpm$//') || true
+yum install -y $packages
+
+systemctl enable lorax-composer.socket

tests/cli/lib/lib.sh

@@ -1,5 +1,21 @@
 #!/usr/bin/env bash
 
+# Monkey-patch beakerlib to exit on first failure if COMPOSER_TEST_FAIL_FAST is
+# set. https://github.com/beakerlib/beakerlib/issues/42
+if [ "$COMPOSER_TEST_FAIL_FAST" == "1" ]; then
+  eval "original$(declare -f __INTERNAL_LogAndJournalFail)"
+
+  __INTERNAL_LogAndJournalFail () {
+    original__INTERNAL_LogAndJournalFail
+
+    # end test somewhat cleanly so that beakerlib logs the FAIL correctly
+    rlPhaseEnd
+    rlJournalEnd
+
+    exit 1
+  }
+fi
+
 # a generic helper function unifying the specific checks executed on a running
 # image instance
 verify_image() {

tests/cli/test_blueprints_sanity.sh

@@ -1,7 +1,10 @@
 #!/bin/bash
 # Note: execute this file from the project root directory
 
+set -e
+
 . /usr/share/beakerlib/beakerlib.sh
+. $(dirname $0)/lib/lib.sh
 
 CLI="${CLI:-./src/bin/composer-cli}"
 

tests/cli/test_build_and_deploy_aws.sh

@@ -7,8 +7,10 @@
 #
 #####
 
+set -e
+
 . /usr/share/beakerlib/beakerlib.sh
-. ./tests/cli/lib/lib.sh
+. $(dirname $0)/lib/lib.sh
 
 CLI="${CLI:-./src/bin/composer-cli}"
 
@@ -81,7 +83,7 @@ __EOF__
 
     rlPhaseStartTest "compose finished"
         if [ -n "$UUID" ]; then
-            until $CLI compose info $UUID | grep FINISHED; do
+            until $CLI compose info $UUID | grep 'FINISHED\|FAILED'; do
                 rlLogInfo "Waiting for compose to finish ..."
                 sleep 30
             done;

tests/cli/test_build_and_deploy_azure.sh

@@ -7,8 +7,10 @@
 #
 #####
 
+set -e
+
 . /usr/share/beakerlib/beakerlib.sh
-. ./tests/cli/lib/lib.sh
+. $(dirname $0)/lib/lib.sh
 
 CLI="${CLI:-./src/bin/composer-cli}"
 
@@ -66,7 +68,7 @@ rlJournalStart
 
     rlPhaseStartTest "compose finished"
         if [ -n "$UUID" ]; then
-            until $CLI compose info $UUID | grep FINISHED; do
+            until $CLI compose info $UUID | grep 'FINISHED\|FAILED'; do
                 rlLogInfo "Waiting for compose to finish ..."
                 sleep 30
             done;

tests/cli/test_build_and_deploy_openstack.sh

@@ -7,8 +7,10 @@
 #
 #####
 
+set -e
+
 . /usr/share/beakerlib/beakerlib.sh
-. ./tests/cli/lib/lib.sh
+. $(dirname $0)/lib/lib.sh
 
 CLI="${CLI:-./src/bin/composer-cli}"
 
@@ -27,8 +29,8 @@ rlJournalStart
             rlLogInfo "OS_USERNAME=$OS_USERNAME"
         fi
 
-        export OS_TENANT_NAME="${OS_TENANT_NAME:-$OS_USERNAME}"
+        export OS_PROJECT_NAME="${OS_PROJECT_NAME:-$OS_USERNAME}"
-        rlLogInfo "OS_TENANT_NAME=$OS_TENANT_NAME"
+        rlLogInfo "OS_PROJECT_NAME=$OS_PROJECT_NAME"
 
         if [ -z "$OS_PASSWORD" ]; then
             rlFail "OS_PASSWORD is empty!"
@@ -76,7 +78,7 @@ __EOF__
 
     rlPhaseStartTest "compose finished"
         if [ -n "$UUID" ]; then
-            until $CLI compose info $UUID | grep FINISHED; do
+            until $CLI compose info $UUID | grep 'FINISHED\|FAILED'; do
                 rlLogInfo "Waiting for compose to finish ..."
                 sleep 30
             done;

tests/cli/test_build_and_deploy_vmware.sh

@@ -7,8 +7,10 @@
 #
 #####
 
+set -e
+
 . /usr/share/beakerlib/beakerlib.sh
-. ./tests/cli/lib/lib.sh
+. $(dirname $0)/lib/lib.sh
 
 CLI="${CLI:-./src/bin/composer-cli}"
 
@@ -97,7 +99,7 @@ __EOF__
 
     rlPhaseStartTest "compose finished"
         if [ -n "$UUID" ]; then
-            until $CLI compose info $UUID | grep FINISHED; do
+            until $CLI compose info $UUID | grep 'FINISHED\|FAILED'; do
                 rlLogInfo "Waiting for compose to finish ..."
                 sleep 30
             done;
@@ -134,7 +136,7 @@ __EOF__
             rlLogInfo "IP_ADDRESS is not assigned yet ..."
             sleep 30
             IP_ADDRESS=`python3 $SAMPLES/find_by_uuid.py -S -s $V_HOST -u $V_USERNAME -p $V_PASSWORD \
-                            --uuid $INSTANCE_UUID | grep 'ip address' | tr -d ' ' | cut -f2 -d:`
+                            --uuid $INSTANCE_UUID | grep 'ip address' | tr -d ' ' | cut -f2- -d:`
         done
 
         rlLogInfo "Running instance IP_ADDRESS=$IP_ADDRESS"

tests/cli/test_compose_ext4-filesystem.sh

@@ -9,7 +9,10 @@
 #
 #####
 
+set -e
+
 . /usr/share/beakerlib/beakerlib.sh
+. $(dirname $0)/lib/lib.sh
 
 CLI="${CLI:-./src/bin/composer-cli}"
 
@@ -25,7 +28,7 @@ rlJournalStart
 
     rlPhaseStartTest "compose finished"
         if [ -n "$UUID" ]; then
-            until $CLI compose info $UUID | grep FINISHED; do
+            until $CLI compose info $UUID | grep 'FINISHED\|FAILED'; do
                 sleep 10
                 rlLogInfo "Waiting for compose to finish ..."
             done;

tests/cli/test_compose_google.sh

@@ -6,7 +6,10 @@
 #
 #####
 
+set -e
+
 . /usr/share/beakerlib/beakerlib.sh
+. $(dirname $0)/lib/lib.sh
 
 CLI="${CLI:-./src/bin/composer-cli}"
 
@@ -21,7 +24,7 @@ rlJournalStart
 
     rlPhaseStart "compose finished"
         if [ -n "$UUID" ]; then
-            until $CLI compose info $UUID | grep FINISHED; do
+            until $CLI compose info $UUID | grep 'FINISHED\|FAILED'; do
                 sleep 10
                 rlLogInfo "Waiting for compose to finish..."
             done

tests/cli/test_compose_live-iso.sh

@@ -7,8 +7,10 @@
 #
 #####
 
+set -e
+
 . /usr/share/beakerlib/beakerlib.sh
-. ./tests/cli/lib/lib.sh
+. $(dirname $0)/lib/lib.sh
 
 CLI="${CLI:-./src/bin/composer-cli}"
 QEMU="/usr/libexec/qemu-kvm"
@@ -30,7 +32,7 @@ rlJournalStart
 
     rlPhaseStartTest "compose finished"
         if [ -n "$UUID" ]; then
-            until $CLI compose info $UUID | grep FINISHED; do
+            until $CLI compose info $UUID | grep 'FINISHED\|FAILED'; do
                 sleep 20
                 rlLogInfo "Waiting for compose to finish ..."
             done;
@@ -43,7 +45,7 @@ rlJournalStart
     rlPhaseEnd
 
     rlPhaseStartTest "Start VM instance"
-        rlRun -t -c "$QEMU -m 2048 -boot c -cdrom $IMAGE -nographic \
+        rlRun -t -c "$QEMU -m 2048 -boot d -cdrom $IMAGE -nographic -monitor none \
                            -net user,id=nic0,hostfwd=tcp::2222-:22 -net nic &"
         # 60 seconds timeout at boot menu screen
         # then media check + boot ~ 30 seconds

tests/cli/test_compose_partitioned-disk.sh

@@ -9,7 +9,10 @@
 #
 #####
 
+set -e
+
 . /usr/share/beakerlib/beakerlib.sh
+. $(dirname $0)/lib/lib.sh
 
 CLI="${CLI:-./src/bin/composer-cli}"
 
@@ -25,7 +28,7 @@ rlJournalStart
 
     rlPhaseStartTest "compose finished"
         if [ -n "$UUID" ]; then
-            until $CLI compose info $UUID | grep FINISHED; do
+            until $CLI compose info $UUID | grep 'FINISHED\|FAILED'; do
                 sleep 10
                 rlLogInfo "Waiting for compose to finish ..."
             done;

tests/cli/test_compose_qcow2.sh

@@ -7,8 +7,10 @@
 #
 #####
 
+set -e
+
 . /usr/share/beakerlib/beakerlib.sh
-. ./tests/cli/lib/lib.sh
+. $(dirname $0)/lib/lib.sh
 
 CLI="${CLI:-./src/bin/composer-cli}"
 QEMU="/usr/libexec/qemu-kvm"
@@ -58,7 +60,7 @@ __EOF__
 
     rlPhaseStartTest "compose finished"
         if [ -n "$UUID" ]; then
-            until $CLI compose info $UUID | grep FINISHED; do
+            until $CLI compose info $UUID | grep 'FINISHED\|FAILED'; do
                 sleep 20
                 rlLogInfo "Waiting for compose to finish ..."
             done;
@@ -71,7 +73,7 @@ __EOF__
     rlPhaseEnd
 
     rlPhaseStartTest "Start VM instance"
-        rlRun -t -c "$QEMU -m 2048 -boot c -hda $IMAGE -nographic \
+        rlRun -t -c "$QEMU -m 2048 -boot c -hda $IMAGE -nographic -monitor none \
                            -net user,id=nic0,hostfwd=tcp::2222-:22 -net nic &"
         sleep 60
     rlPhaseEnd

tests/cli/test_compose_sanity.sh

@@ -1,7 +1,10 @@
 #!/bin/bash
 # Note: execute this file from the project root directory
 
+set -e
+
 . /usr/share/beakerlib/beakerlib.sh
+. $(dirname $0)/lib/lib.sh
 
 CLI="${CLI:-./src/bin/composer-cli}"
 
@@ -29,7 +32,7 @@ rlJournalStart
 
     rlPhaseStartTest "compose image"
         if [ -n "$UUID" ]; then
-            until $CLI compose info $UUID | grep FINISHED; do
+            until $CLI compose info $UUID | grep 'FINISHED\|FAILED'; do
                 sleep 5
                 rlLogInfo "Waiting for compose to finish ..."
             done;

tests/cli/test_compose_tar.sh

@@ -7,7 +7,10 @@
 #
 #####
 
+set -e
+
 . /usr/share/beakerlib/beakerlib.sh
+. $(dirname $0)/lib/lib.sh
 
 CLI="${CLI:-./src/bin/composer-cli}"
 
@@ -27,7 +30,7 @@ rlJournalStart
 
     rlPhaseStartTest "compose finished"
         if [ -n "$UUID" ]; then
-            until $CLI compose info $UUID | grep FINISHED; do
+            until $CLI compose info $UUID | grep 'FINISHED\|FAILED'; do
                 sleep 10
                 rlLogInfo "Waiting for compose to finish ..."
             done;
@@ -35,6 +38,13 @@ rlJournalStart
             rlFail "Compose UUID is empty!"
         fi
 
+        # Running a compose can lead to a different selinux policy in the
+        # kernel, which may break docker. Reload the policy from the host and
+        # restart docker as a workaround.
+        # See https://bugzilla.redhat.com/show_bug.cgi?id=1711813
+        semodule -R
+        systemctl restart docker
+
         rlRun -t -c "$CLI compose image $UUID"
         IMAGE="$UUID-root.tar.xz"
     rlPhaseEnd
@@ -43,13 +53,13 @@ rlJournalStart
         rlRun -t -c "docker import $IMAGE composer/$UUID:latest"
 
         # verify we can run a container with this image
-        rlRun -t -c "docker run -it --rm --entrypoint /usr/bin/cat composer/$UUID /etc/redhat-release"
+        rlRun -t -c "docker run --rm --entrypoint /usr/bin/cat composer/$UUID /etc/redhat-release"
     rlPhaseEnd
 
     rlPhaseStartTest "Verify tar image with systemd-nspawn"
         if [ -f /usr/bin/systemd-nspawn ]; then
-            NSPAWN_DIR=`mktemp -d /tmp/nspawn.XXXX`
+            NSPAWN_DIR=`mktemp -d /var/tmp/nspawn.XXXX`
-            rlRun -t -c "tar -xJvf $IMAGE -C $NSPAWN_DIR"
+            rlRun -t -c "tar -xJf $IMAGE -C $NSPAWN_DIR"
 
             # verify we can run a container with this image
             rlRun -t -c "systemd-nspawn -D $NSPAWN_DIR cat /etc/redhat-release"

tests/test_cli.sh

@@ -1,8 +1,10 @@
 #!/bin/bash
 # Note: execute this file from the project root directory
 
-# setup
+set -eu
-rm -rf /var/tmp/beakerlib-*/
+
+export BEAKERLIB_DIR=$(mktemp -d /tmp/composer-test.XXXXXX)
+CLI="${CLI:-}"
 
 function setup_tests {
     local share_dir=$1
@@ -30,7 +32,7 @@ function setup_tests {
     cat >> $blueprints_dir/example-http-server.toml << __EOF__
 
 [customizations.kernel]
-append = "custom_cmdline_arg"
+append = "custom_cmdline_arg console=ttyS0,115200n8"
 __EOF__
 }
 
@@ -43,6 +45,15 @@ function teardown_tests {
     mv ${blueprints_dir}.orig $blueprints_dir
 }
 
+# cloud credentials
+if [ -f "~/.config/lorax-test-env" ]; then
+    . ~/.config/lorax-test-env
+fi
+
+if [ -f "/var/tmp/lorax-test-env" ]; then
+    . /var/tmp/lorax-test-env
+fi
+
 if [ -z "$CLI" ]; then
     export top_srcdir=`pwd`
     . ./tests/testenv.sh
@@ -71,10 +82,10 @@ fi
 tries=0
 until curl -m 15 --unix-socket /run/weldr/api.socket http://localhost:4000/api/status | grep 'db_supported.*true'; do
     tries=$((tries + 1))
-    if [ $tries -gt 20 ]; then
+    if [ $tries -gt 50 ]; then
         exit 1
     fi
-    sleep 2
+    sleep 5
     echo "DEBUG: Waiting for backend API to become ready before testing ..."
 done;
 
@@ -89,7 +100,7 @@ else
     # execute other cli tests which need more adjustments in the calling environment
     # or can't be executed inside Travis CI
     for TEST in "$@"; do
-        ./$TEST
+        $TEST
     done
 fi
 
@@ -108,8 +119,11 @@ else
     systemctl start lorax-composer
 fi
 
-# look for failures
+. $BEAKERLIB_DIR/TestResults
-grep RESULT_STRING /var/tmp/beakerlib-*/TestResults | grep -v PASS && exit 1
 
-# explicit return code for Makefile
+if [ $TESTRESULT_RESULT_ECODE != 0 ]; then
-exit 0
+  echo "Test failed. Leaving log in $BEAKERLIB_DIR"
+  exit $TESTRESULT_RESULT_ECODE
+fi
+
+rm -rf $BEAKERLIB_DIR

tests/testenv.sh

@@ -1,5 +1,9 @@
 #!/bin/sh
 
+top_srcdir="${top_srcdir:-}"
+top_buildir="${top_builddir:-}"
+PYTHONPATH="${PYTHONPATH:-}"
+
 if [ -z "$top_srcdir" ]; then
     echo "*** top_srcdir must be set"
     exit 1