From 8734f662e9f0a2bfae656df552784f6d785c41ac Mon Sep 17 00:00:00 2001 From: Alexander Todorov Date: Mon, 17 Jun 2019 15:35:31 +0300 Subject: [PATCH] Backport changes for Cockpit CI Related: rhbz#1698366 --- Makefile | 45 ++++++- test/README.md | 60 +++++++++ test/check-api | 75 +++++++++++ test/check-cli | 43 ++++++ test/check-cloud | 21 +++ test/composertest.py | 135 +++++++++++++++++++ test/run | 18 +++ test/vm.install | 35 +++++ tests/cli/lib/lib.sh | 18 ++- tests/cli/test_blueprints_sanity.sh | 3 + tests/cli/test_build_and_deploy_aws.sh | 6 +- tests/cli/test_build_and_deploy_azure.sh | 6 +- tests/cli/test_build_and_deploy_openstack.sh | 11 +- tests/cli/test_build_and_deploy_vmware.sh | 20 +-- tests/cli/test_compose_ext4-filesystem.sh | 5 +- tests/cli/test_compose_live-iso.sh | 13 +- tests/cli/test_compose_partitioned-disk.sh | 5 +- tests/cli/test_compose_qcow2.sh | 13 +- tests/cli/test_compose_sanity.sh | 5 +- tests/cli/test_compose_tar.sh | 16 ++- tests/test_cli.sh | 42 ++++-- tests/testenv.sh | 4 + 22 files changed, 547 insertions(+), 52 deletions(-) create mode 100644 test/README.md create mode 100755 test/check-api create mode 100755 test/check-cli create mode 100755 test/check-cloud create mode 100755 test/composertest.py create mode 100755 test/run create mode 100755 test/vm.install diff --git a/Makefile b/Makefile index fbd4b5fc..226ce7e2 100644 --- a/Makefile +++ b/Makefile @@ -9,6 +9,12 @@ TAG = lorax-$(VERSION)-$(RELEASE) IMAGE_RELEASE = $(shell awk -F: '/FROM/ { print $$2}' Dockerfile.test) +ifeq ($(TEST_OS),) +TEST_OS = rhel-7-7 +endif +export TEST_OS +VM_IMAGE=$(CURDIR)/test/images/$(TEST_OS) + default: all src/composer/version.py: @@ -43,8 +49,6 @@ test: docs coverage report -m [ -f "/usr/bin/coveralls" ] && [ -n "$(COVERALLS_REPO_TOKEN)" ] && coveralls || echo - - ./tests/test_cli.sh # need `losetup`, which needs Docker to be in privileged mode (--privileged) # but even so fails in Travis CI @@ -80,9 +84,15 @@ docs: archive: tag @git archive --format=tar --prefix=$(PKGNAME)-$(VERSION)/ $(TAG) > $(PKGNAME)-$(VERSION).tar - @gzip $(PKGNAME)-$(VERSION).tar + @gzip -f $(PKGNAME)-$(VERSION).tar @echo "The archive is in $(PKGNAME)-$(VERSION).tar.gz" +srpm: archive $(PKGNAME).spec + rpmbuild -bs \ + --define "_sourcedir $(CURDIR)" \ + --define "_srcrpmdir $(CURDIR)" \ + lorax-composer.spec + local: @rm -rf $(PKGNAME)-$(VERSION).tar.gz @rm -rf /var/tmp/$(PKGNAME)-$(VERSION) @@ -106,6 +116,35 @@ docs-in-docker: ci: check test +$(VM_IMAGE): TAG=HEAD +$(VM_IMAGE): srpm bots + srpm=$(shell rpm --qf '%{Name}-%{Version}-%{Release}.src.rpm\n' -q --specfile lorax-composer.spec | head -n1) ; \ + bots/image-customize -v \ + --resize 20G \ + --upload $$srpm:/var/tmp \ + --upload $(CURDIR)/test/vm.install:/var/tmp/vm.install \ + --upload $(realpath tests):/ \ + --run-command "chmod +x /var/tmp/vm.install" \ + --run-command "cd /var/tmp; /var/tmp/vm.install $$srpm" \ + $(TEST_OS) + [ -f ~/.config/lorax-test-env ] && bots/image-customize \ + --upload ~/.config/lorax-test-env:/var/tmp/lorax-test-env \ + $(TEST_OS) || echo + +# convenience target for the above +vm: $(VM_IMAGE) + echo $(VM_IMAGE) + +vm-reset: + rm -f $(VM_IMAGE) $(VM_IMAGE).qcow2 + +# checkout Cockpit's bots/ directory for standard test VM images and API to launch them +# must be from cockpit's master, as only that has current and existing images; but testvm.py API is stable +bots: + git fetch --depth=1 https://github.com/cockpit-project/cockpit.git + git checkout --force FETCH_HEAD -- bots/ + git reset bots + .PHONY: all install check test clean tag docs archive local .PHONY: ci_after_success diff --git a/test/README.md b/test/README.md new file mode 100644 index 00000000..abd8e854 --- /dev/null +++ b/test/README.md @@ -0,0 +1,60 @@ +# Integration Tests + +lorax uses Cockpit's integration test framework and infrastructure. To do this, +we're checking out Cockpit's `bots/` subdirectory. It contains links to test +images and tools to manipulate and start virtual machines from them. + +Each test is run on a new instance of a virtual machine. +Branch/test matrix is configured in `bots/tests-scan` inside the +[cockpit repository](https://github.com/cockpit-project/cockpit). + +## Dependencies + +These dependencies are needed on Fedora to run tests locally: + + $ sudo dnf install curl expect \ + libvirt libvirt-client libvirt-daemon libvirt-python \ + python python-libguestfs python-lxml libguestfs-xfs \ + python3 libvirt-python3 \ + libguestfs-tools qemu qemu-kvm rpm-build rsync xz + +## Building a test VM + +To build a test VM, run + + $ make vm + +This downloads a base image from Cockpit's infrastructure. You can control +which image is downloaded with the `TEST_OS` environment variable. Cockpit's +[documentation](https://github.com/cockpit-project/cockpit/blob/master/test/README.md#test-configuration) +lists accepted values. It then creates a new image based on that (a qemu +snapshot) in `tests/images`, which contain the current `tests/` directory and +have newly built rpms from the current checkout installed. + +To delete the generated image, run + + $ make vm-reset + +Base images are stored in `bots/images`. Set `TEST_DATA` to override this +directory. + +## Running tests + +After building a test image, run + + $ ./test/check-cli [TESTNAME] + +or any of the other `check-*` scripts. To debug a test failure, pass `--sit`. +This will keep the test machine running after the first failure and print an +ssh line to connect to it. + +Run `make vm` after changing tests or lorax source to recreate the test +machine. It is usually not necessary to reset the VM. + +## Updating images + +The `bots/` directory is checked out from Cockpit when `make vm` is first run. +To get the latest images you need to update it manually (in order not to poll +GitHub every time): + + $ make -B bots diff --git a/test/check-api b/test/check-api new file mode 100755 index 00000000..d47393cf --- /dev/null +++ b/test/check-api @@ -0,0 +1,75 @@ +#!/usr/bin/python3 + +from time import sleep + +import composertest +import requests +import subprocess + + +class TestApi(composertest.ComposerTestCase): + """Test Composer HTTP API""" + + def setUp(self): + super(TestApi, self).setUp() + + # Forward /run/weldr/api.socket to a port on the host + # Set ExitOnForwardFailure so that ssh blocks until the forward is set + # up before going to the background (-f), which it closes stdout. We + # wait for that by calling read() on it. + self.composer_port = self.network._lock(8080) + forwarder_command = self.ssh_command[:] + forwarder_command.extend(["-fNT", + "-o", "ExitOnForwardFailure=yes", + "-L", "localhost:%d:/run/weldr/api.socket" % self.composer_port]) + self.forwarder_proc = subprocess.Popen(forwarder_command, stdout=subprocess.PIPE) + self.forwarder_proc.stdout.read() + + def tearDown(self): + self.forwarder_proc.terminate() + sleep(1) # wait and check for timeout + if self.forwarder_proc.poll() is None: + self.forwarder_proc.kill() + super(TestApi, self).tearDown() + + def request(self, method, path, check=True): + self.assertEqual(path[0], "/") + url = "http://localhost:%d%s" % (self.composer_port, path) + r = requests.request(method, url, timeout=30) + if check: + r.raise_for_status() + return r + + def test_basic(self): + """Basic checks for the API""" + + # + # API status without depsolve errors + # + r = self.request("GET", "/api/status") + self.assertEqual(r.status_code, 200) + status = r.json() + self.assertEqual(status.keys(), { "build", "api", "db_version", "schema_version", "db_supported", "backend", "msgs" }) + self.assertEqual(status["msgs"], []) + self.assertEqual(r.headers.keys(), { "Content-Type", "Content-Length", "Date" }) + + # + # HTTP errors should return json responses + # + r = self.request("GET", "/marmalade", check=False) + self.assertEqual(r.status_code, 404) + self.assertEqual(r.json(), { + "status": False, + "errors": [{ "id": "HTTPError", "code": 404, "msg": "Not Found" }] + }) + + r = self.request("POST", "/api/status", check=False) + self.assertEqual(r.status_code, 405) + self.assertEqual(r.json(), { + "status": False, + "errors": [{ "id": "HTTPError", "code": 405, "msg": "Method Not Allowed" }] + }) + + +if __name__ == '__main__': + composertest.main() diff --git a/test/check-cli b/test/check-cli new file mode 100755 index 00000000..7efb54fa --- /dev/null +++ b/test/check-cli @@ -0,0 +1,43 @@ +#!/usr/bin/python3 + +import composertest + + +class TestImages(composertest.ComposerTestCase): + """ + This is the "entry-point" to the test suite when + executed in Cockpit CI. If $TEST_SCENARIO=="" or + $TEST_SCENARIO="images" we end up here. + + New test methods should be added here first! + When this target becomes too slow we split out into + separate scenarios! + """ + def test_blueprint_sanity(self): + self.runCliTest("/tests/cli/test_blueprints_sanity.sh") + + def test_compose_sanity(self): + self.runCliTest("/tests/cli/test_compose_sanity.sh") + + def test_ext4_filesystem(self): + self.runCliTest("/tests/cli/test_compose_ext4-filesystem.sh") + + def test_partitioned_disk(self): + self.runCliTest("/tests/cli/test_compose_partitioned-disk.sh") + + def test_tar(self): + self.runCliTest("/tests/cli/test_compose_tar.sh") + + +class TestQcow2(composertest.ComposerTestCase): + def test_qcow2(self): + self.runCliTest("/tests/cli/test_compose_qcow2.sh") + + +class TestLiveIso(composertest.ComposerTestCase): + def test_live_iso(self): + self.runCliTest("/tests/cli/test_compose_live-iso.sh") + + +if __name__ == '__main__': + composertest.main() diff --git a/test/check-cloud b/test/check-cloud new file mode 100755 index 00000000..6877390b --- /dev/null +++ b/test/check-cloud @@ -0,0 +1,21 @@ +#!/usr/bin/python3 + +import composertest + + +class TestCloud(composertest.ComposerTestCase): + def test_aws(self): + self.runCliTest("/tests/cli/test_build_and_deploy_aws.sh") + + def test_azure(self): + self.runCliTest("/tests/cli/test_build_and_deploy_azure.sh") + + def test_openstack(self): + self.runCliTest("/tests/cli/test_build_and_deploy_openstack.sh") + + def test_vmware(self): + self.runCliTest("/tests/cli/test_build_and_deploy_vmware.sh") + + +if __name__ == '__main__': + composertest.main() diff --git a/test/composertest.py b/test/composertest.py new file mode 100755 index 00000000..c837948b --- /dev/null +++ b/test/composertest.py @@ -0,0 +1,135 @@ +#!/usr/bin/python3 + +from __future__ import print_function + +import argparse +import os +import subprocess +import sys +import unittest + +# import Cockpit's machinery for test VMs and its browser test API +sys.path.append(os.path.join(os.path.dirname(__file__), "../bots/machine")) +import testvm # pylint: disable=import-error + + +def print_exception(etype, value, tb): + import traceback + + # only include relevant lines + limit = 0 + while tb and '__unittest' in tb.tb_frame.f_globals: + limit += 1 + tb = tb.tb_next + + traceback.print_exception(etype, value, tb, limit=limit) + + +class ComposerTestCase(unittest.TestCase): + image = testvm.DEFAULT_IMAGE + sit = False + + def __init__(self, methodName='runTest'): + super(ComposerTestCase, self).__init__(methodName=methodName) + # by default run() does this and defaultTestResult() + # always creates new object which is local for the .run() method + self.ci_result = self.defaultTestResult() + + def run(self, result=None): + # so we override run() and use an object attribute which we can + # reference later in tearDown() and extract the errors from + super(ComposerTestCase, self).run(result=self.ci_result) + + def setUp(self): + self.network = testvm.VirtNetwork(0) + self.machine = testvm.VirtMachine(self.image, networking=self.network.host(), memory_mb=2048) + + print("Starting virtual machine '%s'" % self.image) + self.machine.start() + self.machine.wait_boot() + + # run a command to force starting the SSH master + self.machine.execute("uptime") + + self.ssh_command = ["ssh", "-o", "ControlPath=" + self.machine.ssh_master, + "-p", self.machine.ssh_port, + self.machine.ssh_user + "@" + self.machine.ssh_address] + + print("Machine is up. Connect to it via:") + print(" ".join(self.ssh_command)) + print() + + print("Waiting for lorax-composer to become ready...") + curl_command = ["curl", "--max-time", "360", + "--silent", + "--unix-socket", "/run/weldr/api.socket", + "http://localhost/api/status"] + r = subprocess.call(self.ssh_command + curl_command, stdout=open(os.devnull, 'w')) + self.assertEqual(r.returncode, 0) + + def tearDown(self): + # `errors` is a list of tuples (method, error) + errors = list(e[1] for e in self.ci_result.errors if e[1]) + + if errors and self.sit: + for e in errors: + print_exception(*e) + + print() + print(" ".join(self.ssh_command)) + input("Press RETURN to continue...") + + self.machine.stop() + + def execute(self, command): + """Execute a command on the test machine.""" + return subprocess.call(self.ssh_command + command) + + def runCliTest(self, script): + execute_params = ["CLI=/usr/bin/composer-cli", + "TEST=" + self.id(), + "PACKAGE=composer-cli", + "/tests/test_cli.sh", script] + if self.sit: + execute_params.insert(0, "COMPOSER_TEST_FAIL_FAST=1") + + r = self.execute(execute_params) + self.assertEqual(r.returncode, 0) + + +def print_tests(tests): + for test in tests: + if isinstance(test, unittest.TestSuite): + print_tests(test) + # I don't know how this is used when running the tests + # (maybe not used from what it looks like) so not sure how to refactor it + # elif isinstance(test, unittest.loader._FailedTest): + # name = test.id().replace("unittest.loader._FailedTest.", "") + # print("Error: '%s' does not match a test" % name, file=sys.stderr) + else: + print(test.id().replace("__main__.", "")) + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("tests", nargs="*", help="List of tests modules, classes, and methods") + parser.add_argument("-l", "--list", action="store_true", help="Print the list of tests that would be executed") + parser.add_argument("-s", "--sit", action="store_true", help="Halt test execution (but keep VM running) when a test fails") + args = parser.parse_args() + + ComposerTestCase.sit = args.sit + + module = __import__("__main__") + if args.tests: + tests = unittest.defaultTestLoader.loadTestsFromNames(args.tests, module) + else: + tests = unittest.defaultTestLoader.loadTestsFromModule(module) + + if args.list: + print_tests(tests) + return 0 + + runner = unittest.TextTestRunner(verbosity=2, failfast=args.sit) + result = runner.run(tests) + + sys.exit(not result.wasSuccessful()) diff --git a/test/run b/test/run new file mode 100755 index 00000000..4c0556b7 --- /dev/null +++ b/test/run @@ -0,0 +1,18 @@ +#!/bin/sh -e +# This is the expected entry point for Cockpit CI; will be called without +# arguments but with an appropriate $TEST_OS, and optionally $TEST_SCENARIO + +make vm + +if [ -n "$TEST_SCENARIO" ]; then + if [ "$TEST_SCENARIO" == "live-iso" ]; then + test/check-cli TestLiveIso + elif [ "$TEST_SCENARIO" == "qcow2" ]; then + test/check-cli TestQcow2 + else + test/check-cloud TestCloud.test_$TEST_SCENARIO + fi +else + test/check-cli TestImages + test/check-api +fi diff --git a/test/vm.install b/test/vm.install new file mode 100755 index 00000000..c640e640 --- /dev/null +++ b/test/vm.install @@ -0,0 +1,35 @@ +#!/bin/sh -eux + +SRPM="$1" + +if ! rpm -q beakerlib; then + if [ $(. /etc/os-release && echo $ID) = "rhel" ]; then + (cd /etc/yum.repos.d; curl -O -L http://download.devel.redhat.com/beakerrepos/beaker-client-RedHatEnterpriseLinux.repo) + + # The beaker repository doesn't include repos for minor releases + VERSION=$(. /etc/os-release && echo ${VERSION_ID%.*}) + yum install -y --releasever=$VERSION --setopt=sslverify=0 beakerlib + + # prevent yum from trying to sync the cache again later (it fails without sslverify=0) + rm /etc/yum.repos.d/beaker-client-RedHatEnterpriseLinux.repo + else + yum install -y beakerlib + fi +fi + +# Grow root partition to make room for images. This only works on Fedora right now. +parted --script /dev/vda resizepart 2 100% +partprobe +pvs --noheadings -opv_name | xargs pvresize +rootlv=$(findmnt --noheadings -oSOURCE /) +lvresize $rootlv -l+100%FREE -r + +rm -rf build-results +su builder -c "/usr/bin/mock --no-clean --resultdir build-results --rebuild $SRPM" + +packages=$(find build-results -name '*.rpm' -not -name '*.src.rpm') +rpm -e --verbose $(basename -a ${packages[@]} | sed 's/-[0-9].*.rpm$//') || true +yum install -y $packages + +systemctl enable lorax-composer.socket +systemctl enable docker.service diff --git a/tests/cli/lib/lib.sh b/tests/cli/lib/lib.sh index 42f4dadf..98fa1a76 100755 --- a/tests/cli/lib/lib.sh +++ b/tests/cli/lib/lib.sh @@ -1,5 +1,21 @@ #!/usr/bin/env bash +# Monkey-patch beakerlib to exit on first failure if COMPOSER_TEST_FAIL_FAST is +# set. https://github.com/beakerlib/beakerlib/issues/42 +if [ "$COMPOSER_TEST_FAIL_FAST" == "1" ]; then + eval "original$(declare -f __INTERNAL_LogAndJournalFail)" + + __INTERNAL_LogAndJournalFail () { + original__INTERNAL_LogAndJournalFail + + # end test somewhat cleanly so that beakerlib logs the FAIL correctly + rlPhaseEnd + rlJournalEnd + + exit 1 + } +fi + # a generic helper function unifying the specific checks executed on a running # image instance verify_image() { @@ -31,7 +47,7 @@ check_root_account() { # ssh returns 255 in case of any ssh error, so it's better to grep the specific error message rlRun -t -c "ssh $SSH_OPTS -o PubkeyAuthentication=no root@${SSH_MACHINE} 2>&1 | grep -i 'permission denied ('" \ 0 "Can't ssh to '$SSH_MACHINE' as root using password-based auth" - rlRun -t -c "ssh $SSH_OPTS ${SSH_USER}@${SSH_MACHINE} \"sudo grep -E '^root:(\*LOCK\*|!)' /etc/shadow\"" \ + rlRun -t -c "ssh $SSH_OPTS ${SSH_USER}@${SSH_MACHINE} \"sudo grep -E '^root:(\*LOCK\*|!):' /etc/shadow\"" \ 0 "root account is disabled in /etc/shadow" rlRun -t -c "ssh $SSH_OPTS ${SSH_USER}@${SSH_MACHINE} \"sudo grep 'USER_LOGIN.*acct=\\\"root\\\".*terminal=ssh.*res=failed' /var/log/audit/audit.log\"" \ 0 "audit.log contains entry about unsuccessful root login" diff --git a/tests/cli/test_blueprints_sanity.sh b/tests/cli/test_blueprints_sanity.sh index 0db99d6e..1b7d96c6 100755 --- a/tests/cli/test_blueprints_sanity.sh +++ b/tests/cli/test_blueprints_sanity.sh @@ -1,7 +1,10 @@ #!/bin/bash # Note: execute this file from the project root directory +set -e + . /usr/share/beakerlib/beakerlib.sh +. $(dirname $0)/lib/lib.sh CLI="${CLI:-./src/bin/composer-cli}" diff --git a/tests/cli/test_build_and_deploy_aws.sh b/tests/cli/test_build_and_deploy_aws.sh index 7bed9c81..63756b0e 100755 --- a/tests/cli/test_build_and_deploy_aws.sh +++ b/tests/cli/test_build_and_deploy_aws.sh @@ -7,8 +7,10 @@ # ##### +set -e + . /usr/share/beakerlib/beakerlib.sh -. ./tests/cli/lib/lib.sh +. $(dirname 0)/lib/lib.sh CLI="${CLI:-./src/bin/composer-cli}" @@ -82,7 +84,7 @@ __EOF__ rlPhaseStartTest "compose finished" if [ -n "$UUID" ]; then - until $CLI compose details $UUID | grep FINISHED; do + until $CLI compose details $UUID | grep 'FINISHED\|FAILED'; do rlLogInfo "Waiting for compose to finish ..." sleep 30 done; diff --git a/tests/cli/test_build_and_deploy_azure.sh b/tests/cli/test_build_and_deploy_azure.sh index f5a985f2..8745aaa5 100755 --- a/tests/cli/test_build_and_deploy_azure.sh +++ b/tests/cli/test_build_and_deploy_azure.sh @@ -7,8 +7,10 @@ # ##### +set -e + . /usr/share/beakerlib/beakerlib.sh -. ./tests/cli/lib/lib.sh +. $(dirname $0)/lib/lib.sh CLI="${CLI:-./src/bin/composer-cli}" @@ -70,7 +72,7 @@ rlJournalStart rlPhaseStartTest "compose finished" if [ -n "$UUID" ]; then - until $CLI compose details $UUID | grep FINISHED; do + until $CLI compose details $UUID | grep 'FINISHED\|FAILED'; do rlLogInfo "Waiting for compose to finish ..." sleep 30 done; diff --git a/tests/cli/test_build_and_deploy_openstack.sh b/tests/cli/test_build_and_deploy_openstack.sh index 39833772..646b38d8 100755 --- a/tests/cli/test_build_and_deploy_openstack.sh +++ b/tests/cli/test_build_and_deploy_openstack.sh @@ -7,9 +7,10 @@ # ##### -. /usr/share/beakerlib/beakerlib.sh -. ./tests/cli/lib/lib.sh +set -e +. /usr/share/beakerlib/beakerlib.sh +. $(dirname $0)/lib/lib.sh CLI="${CLI:-./src/bin/composer-cli}" VENV=`mktemp -d /tmp/ansible.venv.XXX` @@ -29,8 +30,8 @@ rlJournalStart rlLogInfo "OS_USERNAME=$OS_USERNAME" fi - export OS_TENANT_NAME="${OS_TENANT_NAME:-$OS_USERNAME}" - rlLogInfo "OS_TENANT_NAME=$OS_TENANT_NAME" + export OS_PROJECT_NAME="${OS_PROJECT_NAME:-$OS_USERNAME}" + rlLogInfo "OS_PROJECT_NAME=$OS_PROJECT_NAME" if [ -z "$OS_PASSWORD" ]; then rlFail "OS_PASSWORD is empty!" @@ -81,7 +82,7 @@ __EOF__ rlPhaseStartTest "compose finished" if [ -n "$UUID" ]; then - until $CLI compose details $UUID | grep FINISHED; do + until $CLI compose details $UUID | grep 'FINISHED\|FAILED'; do rlLogInfo "Waiting for compose to finish ..." sleep 30 done; diff --git a/tests/cli/test_build_and_deploy_vmware.sh b/tests/cli/test_build_and_deploy_vmware.sh index c0f5ef55..48a55b29 100755 --- a/tests/cli/test_build_and_deploy_vmware.sh +++ b/tests/cli/test_build_and_deploy_vmware.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Note: Execute this file from the project root directory +# Note: execute this file from the project root directory ##### # @@ -7,8 +7,10 @@ # ##### +set -e + . /usr/share/beakerlib/beakerlib.sh -. ./tests/cli/lib/lib.sh +. $(dirname $0)/lib/lib.sh CLI="${CLI:-./src/bin/composer-cli}" @@ -65,7 +67,7 @@ rlJournalStart SAMPLES="$SAMPLES/samples" rlPhaseEnd - rlPhaseStartTest "Compose start" + rlPhaseStartTest "compose start" rlAssertEquals "SELinux operates in enforcing mode" "$(getenforce)" "Enforcing" SSH_KEY_DIR=`mktemp -d /tmp/composer-ssh-keys.XXXXXX` rlRun -t -c "ssh-keygen -t rsa -N '' -f $SSH_KEY_DIR/id_rsa" @@ -96,9 +98,9 @@ __EOF__ UUID=`echo $UUID | cut -f 2 -d' '` rlPhaseEnd - rlPhaseStartTest "Compose finished" + rlPhaseStartTest "compose finished" if [ -n "$UUID" ]; then - until $CLI compose details $UUID | grep FINISHED; do + until $CLI compose details $UUID | grep 'FINISHED\|FAILED'; do rlLogInfo "Waiting for compose to finish ..." sleep 30 done; @@ -113,7 +115,7 @@ __EOF__ python $SAMPLES/upload_file_to_datastore.py -S -s $V_HOST -u $V_USERNAME -p $V_PASSWORD \ -d $V_DATASTORE -l `readlink -f $IMAGE` -r $IMAGE - rlAssert0 "Image upload successful" $? + rlAssert0 "Image upload successfull" $? rlPhaseEnd rlPhaseStartTest "Start VM instance" @@ -129,13 +131,13 @@ __EOF__ rlLogInfo "INSTANCE_UUID=$INSTANCE_UUID" fi - # Wait for instance to become running and had assigned a public IP + # wait for instance to become running and had assigned a public IP IP_ADDRESS="None" while [ "$IP_ADDRESS" == "None" ]; do rlLogInfo "IP_ADDRESS is not assigned yet ..." sleep 30 IP_ADDRESS=`python $SAMPLES/find_by_uuid.py -S -s $V_HOST -u $V_USERNAME -p $V_PASSWORD \ - --uuid $INSTANCE_UUID | grep 'ip address' | tr -d ' ' | cut -f2 -d:` + --uuid $INSTANCE_UUID | grep 'ip address' | tr -d ' ' | cut -f2- -d:` done rlLogInfo "Running instance IP_ADDRESS=$IP_ADDRESS" @@ -150,7 +152,7 @@ __EOF__ rlPhaseEnd rlPhaseStartCleanup - # note: VMDK disk is removed when destroying the VM + # note: vmdk disk is removed when destroying the VM python $SAMPLES/destroy_vm.py -S -s $V_HOST -u $V_USERNAME -p $V_PASSWORD --uuid $INSTANCE_UUID rlAssert0 "VM destroyed" $? rlRun -t -c "$CLI compose delete $UUID" diff --git a/tests/cli/test_compose_ext4-filesystem.sh b/tests/cli/test_compose_ext4-filesystem.sh index 03de10ae..92192199 100755 --- a/tests/cli/test_compose_ext4-filesystem.sh +++ b/tests/cli/test_compose_ext4-filesystem.sh @@ -9,7 +9,10 @@ # ##### +set -e + . /usr/share/beakerlib/beakerlib.sh +. $(dirname $0)/lib/lib.sh CLI="${CLI:-./src/bin/composer-cli}" @@ -25,7 +28,7 @@ rlJournalStart rlPhaseStartTest "compose finished" if [ -n "$UUID" ]; then - until $CLI compose details $UUID | grep FINISHED; do + until $CLI compose details $UUID | grep 'FINISHED\|FAILED'; do sleep 60 rlLogInfo "Waiting for compose to finish ..." done; diff --git a/tests/cli/test_compose_live-iso.sh b/tests/cli/test_compose_live-iso.sh index 9988b591..b9c037e7 100755 --- a/tests/cli/test_compose_live-iso.sh +++ b/tests/cli/test_compose_live-iso.sh @@ -7,15 +7,18 @@ # ##### +set -e + . /usr/share/beakerlib/beakerlib.sh -. ./tests/cli/lib/lib.sh +. $(dirname $0)/lib/lib.sh CLI="${CLI:-./src/bin/composer-cli}" -QEMU="/usr/libexec/qemu-kvm" +QEMU_BIN="/usr/bin/qemu-system-$(uname -m)" +QEMU="$QEMU_BIN -machine accel=kvm:tcg" rlJournalStart rlPhaseStartSetup - rlAssertExists $QEMU + rlAssertExists $QEMU_BIN rlPhaseEnd rlPhaseStartTest "compose start" @@ -30,7 +33,7 @@ rlJournalStart rlPhaseStartTest "compose finished" if [ -n "$UUID" ]; then - until $CLI compose details $UUID | grep FINISHED; do + until $CLI compose details $UUID | grep 'FINISHED\|FAILED'; do sleep 20 rlLogInfo "Waiting for compose to finish ..." done; @@ -56,7 +59,7 @@ rlJournalStart rlPhaseEnd rlPhaseStartCleanup - rlRun -t -c "killall -9 $(basename $QEMU)" + rlRun -t -c "killall -9 $(basename $QEMU_BIN)" rlRun -t -c "$CLI compose delete $UUID" rlRun -t -c "rm -rf $IMAGE" rlPhaseEnd diff --git a/tests/cli/test_compose_partitioned-disk.sh b/tests/cli/test_compose_partitioned-disk.sh index 1bf32f16..2edd084d 100755 --- a/tests/cli/test_compose_partitioned-disk.sh +++ b/tests/cli/test_compose_partitioned-disk.sh @@ -9,7 +9,10 @@ # ##### +set -e + . /usr/share/beakerlib/beakerlib.sh +. $(dirname $0)/lib/lib.sh CLI="${CLI:-./src/bin/composer-cli}" @@ -25,7 +28,7 @@ rlJournalStart rlPhaseStartTest "compose finished" if [ -n "$UUID" ]; then - until $CLI compose details $UUID | grep FINISHED; do + until $CLI compose details $UUID | grep 'FINISHED\|FAILED'; do sleep 60 rlLogInfo "Waiting for compose to finish ..." done; diff --git a/tests/cli/test_compose_qcow2.sh b/tests/cli/test_compose_qcow2.sh index a7fc0d4b..e61b209b 100755 --- a/tests/cli/test_compose_qcow2.sh +++ b/tests/cli/test_compose_qcow2.sh @@ -7,15 +7,18 @@ # ##### +set -e + . /usr/share/beakerlib/beakerlib.sh -. ./tests/cli/lib/lib.sh +. $(dirname $0)/lib/lib.sh CLI="${CLI:-./src/bin/composer-cli}" -QEMU="/usr/libexec/qemu-kvm" +QEMU_BIN="/usr/bin/qemu-system-$(uname -m)" +QEMU="$QEMU_BIN -machine accel=kvm:tcg" rlJournalStart rlPhaseStartSetup - rlAssertExists $QEMU + rlAssertExists $QEMU_BIN rlPhaseEnd rlPhaseStartTest "compose start" @@ -58,7 +61,7 @@ __EOF__ rlPhaseStartTest "compose finished" if [ -n "$UUID" ]; then - until $CLI compose details $UUID | grep FINISHED; do + until $CLI compose details $UUID | grep 'FINISHED\|FAILED'; do sleep 20 rlLogInfo "Waiting for compose to finish ..." done; @@ -82,7 +85,7 @@ __EOF__ rlPhaseEnd rlPhaseStartCleanup - rlRun -t -c "killall -9 $(basename $QEMU)" + rlRun -t -c "killall -9 $(basename $QEMU_BIN)" rlRun -t -c "$CLI compose delete $UUID" rlRun -t -c "rm -rf $IMAGE $TMP_DIR $SSH_KEY_DIR" rlPhaseEnd diff --git a/tests/cli/test_compose_sanity.sh b/tests/cli/test_compose_sanity.sh index c6de519f..b3d4be03 100755 --- a/tests/cli/test_compose_sanity.sh +++ b/tests/cli/test_compose_sanity.sh @@ -1,7 +1,10 @@ #!/bin/bash # Note: execute this file from the project root directory +set -e + . /usr/share/beakerlib/beakerlib.sh +. $(dirname $0)/lib/lib.sh CLI="${CLI:-./src/bin/composer-cli}" @@ -29,7 +32,7 @@ rlJournalStart rlPhaseStartTest "compose image" if [ -n "$UUID" ]; then - until $CLI compose details $UUID | grep FINISHED; do + until $CLI compose details $UUID | grep 'FINISHED\|FAILED'; do sleep 60 rlLogInfo "Waiting for compose to finish ..." done; diff --git a/tests/cli/test_compose_tar.sh b/tests/cli/test_compose_tar.sh index 2db0b83b..cf1f300a 100755 --- a/tests/cli/test_compose_tar.sh +++ b/tests/cli/test_compose_tar.sh @@ -7,7 +7,10 @@ # ##### +set -e + . /usr/share/beakerlib/beakerlib.sh +. $(dirname $0)/lib/lib.sh CLI="${CLI:-./src/bin/composer-cli}" @@ -28,7 +31,7 @@ rlJournalStart rlPhaseStartTest "compose finished" if [ -n "$UUID" ]; then - until $CLI compose details $UUID | grep FINISHED; do + until $CLI compose details $UUID | grep 'FINISHED\|FAILED'; do sleep 10 rlLogInfo "Waiting for compose to finish ..." done; @@ -36,6 +39,13 @@ rlJournalStart rlFail "Compose UUID is empty!" fi + # Running a compose can lead to a different selinux policy in the + # kernel, which may break docker. Reload the policy from the host and + # restart docker as a workaround. + # See https://bugzilla.redhat.com/show_bug.cgi?id=1711813 + semodule -R + systemctl restart docker + rlRun -t -c "$CLI compose image $UUID" IMAGE="$UUID-root.tar.xz" rlPhaseEnd @@ -49,8 +59,8 @@ rlJournalStart rlPhaseStartTest "Verify tar image with systemd-nspawn" if [ -f /usr/bin/systemd-nspawn ]; then - NSPAWN_DIR=`mktemp -d /tmp/nspawn.XXXX` - rlRun -t -c "tar -xJvf $IMAGE -C $NSPAWN_DIR" + NSPAWN_DIR=`mktemp -d /var/tmp/nspawn.XXXX` + rlRun -t -c "tar -xJf $IMAGE -C $NSPAWN_DIR" # verify we can run a container with this image rlRun -t -c "systemd-nspawn -D $NSPAWN_DIR cat /etc/redhat-release" diff --git a/tests/test_cli.sh b/tests/test_cli.sh index 9879fa9d..8db33611 100755 --- a/tests/test_cli.sh +++ b/tests/test_cli.sh @@ -1,8 +1,10 @@ #!/bin/bash -# Note: Execute this file from the project root directory +# Note: execute this file from the project root directory -# setup -rm -rf /var/tmp/beakerlib-*/ +set -eu + +export BEAKERLIB_DIR=$(mktemp -d /tmp/composer-test.XXXXXX) +CLI="${CLI:-}" function setup_tests { local share_dir=$1 @@ -43,6 +45,15 @@ function teardown_tests { mv ${blueprints_dir}.orig $blueprints_dir } +# cloud credentials +if [ -f "~/.config/lorax-test-env" ]; then + . ~/.config/lorax-test-env +fi + +if [ -f "/var/tmp/lorax-test-env" ]; then + . /var/tmp/lorax-test-env +fi + if [ -z "$CLI" ]; then export top_srcdir=`pwd` . ./tests/testenv.sh @@ -55,7 +66,7 @@ if [ -z "$CLI" ]; then chmod a+rx -R $SHARE_DIR setup_tests $SHARE_DIR $BLUEPRINTS_DIR - # Start the lorax-composer daemon + # start the lorax-composer daemon ./src/sbin/lorax-composer --sharedir $SHARE_DIR $BLUEPRINTS_DIR & else export PACKAGE="composer-cli" @@ -65,14 +76,14 @@ else fi -# Wait for the backend to become ready +# wait for the backend to become ready tries=0 until curl -m 15 --unix-socket /run/weldr/api.socket http://localhost:4000/api/status | grep 'db_supported.*true'; do tries=$((tries + 1)) - if [ $tries -gt 20 ]; then + if [ $tries -gt 50 ]; then exit 1 fi - sleep 2 + sleep 5 echo "DEBUG: Waiting for backend API to become ready before testing ..." done; @@ -80,14 +91,14 @@ done; export BEAKERLIB_JOURNAL=0 export PATH="/usr/local/bin:$PATH" if [ -z "$*" ]; then - # Invoke cli/ tests which can be executed without special preparation + # invoke cli/ tests which can be executed without special preparation ./tests/cli/test_blueprints_sanity.sh ./tests/cli/test_compose_sanity.sh else - # Execute other cli tests which need more adjustments in the calling environment + # execute other cli tests which need more adjustments in the calling environment # or can't be executed inside Travis CI for TEST in "$@"; do - ./$TEST + $TEST done fi @@ -106,8 +117,11 @@ else systemctl start lorax-composer fi -# Look for failures -grep RESULT_STRING /var/tmp/beakerlib-*/TestResults | grep -v PASS && exit 1 +. $BEAKERLIB_DIR/TestResults -# Explicit return code for Makefile -exit 0 +if [ $TESTRESULT_RESULT_ECODE != 0 ]; then + echo "Test failed. Leaving log in $BEAKERLIB_DIR" + exit $TESTRESULT_RESULT_ECODE +fi + +rm -rf $BEAKERLIB_DIR diff --git a/tests/testenv.sh b/tests/testenv.sh index 48074422..f293c1ba 100644 --- a/tests/testenv.sh +++ b/tests/testenv.sh @@ -1,5 +1,9 @@ #!/bin/sh +top_srcdir="${top_srcdir:-}" +top_buildir="${top_builddir:-}" +PYTHONPATH="${PYTHONPATH:-}" + if [ -z "$top_srcdir" ]; then echo "*** top_srcdir must be set" exit 1