From 250f49f78dbb89f72653c185463deefd39ecba13 Mon Sep 17 00:00:00 2001 From: Lars Karlitski Date: Fri, 12 Apr 2019 20:17:28 +0200 Subject: [PATCH] Use Cockpit's test images and infrastructure Allows to run the tests on multiple operating systems and on the infrastructure that the Cockpit team maintains. `make vm` downloads one of Cockpit's test images (override which one with TEST_OS) and installs rpms build from the local checkout of lorax. The resulting image is placed in `test/images/$TEST_OS`. TEST_OS can be set to any of Cockpit's supported images (default: fedora-30). Run `make check-vm` to run the CLI checks in the VM. The bulk of the work is done in `test/check-cli`, which uses Cockpit's `bots` library to start the VM and run the script in it. Also included is a `test/run` script, which is the entrypoint for Cockpit's test infrastructure. --- .gitignore | 5 ++++ Makefile | 42 ++++++++++++++++++++++++++-- test/README.md | 57 +++++++++++++++++++++++++++++++++++++ test/check-cli | 32 +++++++++++++++++++++ test/check-cloud | 21 ++++++++++++++ test/composertest.py | 59 +++++++++++++++++++++++++++++++++++++++ test/run | 11 ++++++++ test/vm.install | 18 ++++++++++++ tests/pylint/runpylint.py | 4 +++ tests/test_cli.sh | 2 +- 10 files changed, 248 insertions(+), 3 deletions(-) create mode 100644 test/README.md create mode 100755 test/check-cli create mode 100755 test/check-cloud create mode 100644 test/composertest.py create mode 100755 test/run create mode 100644 test/vm.install diff --git a/.gitignore b/.gitignore index 0274f244..3c4adf1c 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ *.pyc +*.rpm src/pylorax/version.py* _build/ tests/pylint/.pylint.d/ @@ -7,3 +8,7 @@ __pycache__/ pylint-log .pytest_cache/ .test-results/ +/lorax-*.tar.gz +/bots +/test/images +/tmp diff --git a/Makefile b/Makefile index dbb020ef..b8296149 100644 --- a/Makefile +++ b/Makefile @@ -12,6 +12,12 @@ TAG = lorax-$(VERSION)-$(RELEASE) IMAGE_RELEASE = $(shell awk -F: '/FROM/ { print $$2}' Dockerfile.test) +ifeq ($(TEST_OS),) +TEST_OS = fedora-30 +endif +export TEST_OS +VM_IMAGE=$(CURDIR)/test/images/$(TEST_OS) + default: all src/composer/version.py: lorax.spec @@ -95,12 +101,18 @@ set-docs-owner: archive: @git archive --format=tar --prefix=$(PKGNAME)-$(VERSION)/ $(TAG) > $(PKGNAME)-$(VERSION).tar - @gzip $(PKGNAME)-$(VERSION).tar + @gzip -f $(PKGNAME)-$(VERSION).tar @echo "The archive is in $(PKGNAME)-$(VERSION).tar.gz" dist: tag archive scp $(PKGNAME)-$(VERSION).tar.gz fedorahosted.org:lorax +srpm: archive $(PKGNAME).spec + rpmbuild -bs \ + --define "_sourcedir $(CURDIR)" \ + --define "_srcrpmdir $(CURDIR)" \ + lorax.spec + local: @rm -rf $(PKGNAME)-$(VERSION).tar.gz @rm -rf /var/tmp/$(PKGNAME)-$(VERSION) @@ -129,8 +141,34 @@ docs-in-docker: ci: check test +$(VM_IMAGE): TAG=HEAD +$(VM_IMAGE): srpm bots + srpm=$(shell rpm --qf '%{Name}-%{Version}-%{Release}.src.rpm\n' -q --specfile lorax.spec | head -n1) ; \ + bots/image-customize -v \ + --resize 20G \ + --upload $$srpm:/var/tmp \ + --upload $(CURDIR)/test/vm.install:/var/tmp/vm.install \ + --upload $(realpath tests):/ \ + --run-command "chmod +x /var/tmp/vm.install" \ + --run-command "cd /var/tmp; /var/tmp/vm.install $$srpm" \ + $(TEST_OS) + +# convenience target for the above +vm: $(VM_IMAGE) + echo $(VM_IMAGE) + +vm-reset: + rm -f $(VM_IMAGE) $(VM_IMAGE).qcow2 + +# checkout Cockpit's bots/ directory for standard test VM images and API to launch them +# must be from cockpit's master, as only that has current and existing images; but testvm.py API is stable +bots: + git fetch --depth=1 https://github.com/cockpit-project/cockpit.git + git checkout --force FETCH_HEAD -- bots/ + git reset bots + .PHONY: ci_after_success ci_after_success: # nothing to do here, but Jenkins expects this to be present, otherwise fails -.PHONY: docs +.PHONY: docs check test srpm vm vm-reset diff --git a/test/README.md b/test/README.md new file mode 100644 index 00000000..726d6141 --- /dev/null +++ b/test/README.md @@ -0,0 +1,57 @@ +# Integration Tests + +lorax uses Cockpit's integration test framework and infrastructure. To do this, +we're checking out Cockpit's `bots/` subdirectory. It contains links to test +images and tools to manipulate and start virtual machines from them. + +Each test is run on a new instance of a virtual machine. + +## Dependencies + +These dependencies are needed on Fedora to run tests locally: + + $ sudo dnf install curl expect \ + libvirt libvirt-client libvirt-daemon libvirt-python \ + python python-libguestfs python-lxml libguestfs-xfs \ + python3 libvirt-python3 \ + libguestfs-tools qemu qemu-kvm rpm-build rsync xz + +## Building a test VM + +To build a test VM, run + + $ make vm + +This downloads a base image from Cockpit's infrastructure. You can control +which image is downloaded with the `TEST_OS` environment variable. Cockpit's +[documentation](https://github.com/cockpit-project/cockpit/blob/master/test/README.md#test-configuration) +lists accepted values. It then creates a new image based on that (a qemu +snapshot) in `tests/images`, which contain the current `tests/` directory and +have newly built rpms from the current checkout installed. + +To delete the generated image, run + + $ make vm-reset + +Base images are stored in `bots/images`. Set `TEST_DATA` to override this +directory. + +## Running tests + +After building a test image, run + + $ ./test/check-cli [TESTNAME] + +or any of the other `check-*` scripts. Right after the VM is started, these +scripts print an `ssh` line to connect to it. + +Run `make vm` after changing tests or lorax source to recreate the test +machine. It is usually not necessary to reset the VM. + +## Updating images + +The `bots/` directory is checked out from Cockpit when `make vm` is first run. +To get the latest images you need to update it manually (in order not to poll +GitHub every time): + + $ make -B bots diff --git a/test/check-cli b/test/check-cli new file mode 100755 index 00000000..58eeabcf --- /dev/null +++ b/test/check-cli @@ -0,0 +1,32 @@ +#!/usr/bin/python3 + +import composertest + + +class TestSanity(composertest.ComposerTestCase): + def test_blueprint_sanity(self): + self.runCliTest("/tests/cli/test_blueprints_sanity.sh") + + def test_compose_sanity(self): + self.runCliTest("/tests/cli/test_compose_sanity.sh") + + +class TestImages(composertest.ComposerTestCase): + def test_ext4_filesystem(self): + self.runCliTest("/tests/cli/test_compose_ext4-filesystem.sh") + + def test_partitioned_disk(self): + self.runCliTest("/tests/cli/test_compose_partitioned-disk.sh") + + def test_tar(self): + self.runCliTest("/tests/cli/test_compose_tar.sh") + + def test_qcow2(self): + self.runCliTest("/tests/cli/test_compose_qcow2.sh") + + def test_live_iso(self): + self.runCliTest("/tests/cli/test_compose_live-iso.sh") + + +if __name__ == '__main__': + composertest.main() diff --git a/test/check-cloud b/test/check-cloud new file mode 100755 index 00000000..6877390b --- /dev/null +++ b/test/check-cloud @@ -0,0 +1,21 @@ +#!/usr/bin/python3 + +import composertest + + +class TestCloud(composertest.ComposerTestCase): + def test_aws(self): + self.runCliTest("/tests/cli/test_build_and_deploy_aws.sh") + + def test_azure(self): + self.runCliTest("/tests/cli/test_build_and_deploy_azure.sh") + + def test_openstack(self): + self.runCliTest("/tests/cli/test_build_and_deploy_openstack.sh") + + def test_vmware(self): + self.runCliTest("/tests/cli/test_build_and_deploy_vmware.sh") + + +if __name__ == '__main__': + composertest.main() diff --git a/test/composertest.py b/test/composertest.py new file mode 100644 index 00000000..2d1b27dd --- /dev/null +++ b/test/composertest.py @@ -0,0 +1,59 @@ +#!/usr/bin/python3 + +import os +import subprocess +import sys +import unittest + +# import Cockpit's machinery for test VMs and its browser test API +sys.path.append(os.path.join(os.path.dirname(__file__), "../bots/machine")) +import testvm # pylint: disable=import-error + + +class ComposerTestCase(unittest.TestCase): + image = testvm.DEFAULT_IMAGE + + def setUp(self): + network = testvm.VirtNetwork(0) + self.machine = testvm.VirtMachine(self.image, networking=network.host(), memory_mb=2048) + + print(f"Starting virtual machine '{self.image}'") + self.machine.start() + self.machine.wait_boot() + + # run a command to force starting the SSH master + self.machine.execute("uptime") + + self.ssh_command = ["ssh", "-o", "ControlPath=" + self.machine.ssh_master, + "-p", self.machine.ssh_port, + self.machine.ssh_user + "@" + self.machine.ssh_address] + + print("Machine is up. Connect to it via:") + print(" ".join(self.ssh_command)) + print() + + print("Waiting for lorax-composer to become ready...") + curl_command = ["curl", "--max-time", "360", + "--silent", + "--unix-socket", "/run/weldr/api.socket", + "http://localhost/api/status"] + r = subprocess.run(self.ssh_command + curl_command, stdout=subprocess.DEVNULL) + self.assertEqual(r.returncode, 0) + + def tearDown(self): + self.machine.stop() + + def execute(self, command, **args): + """Execute a command on the test machine. + + **args and return value are the same as those for subprocess.run(). + """ + return subprocess.run(self.ssh_command + command, **args) + + def runCliTest(self, script): + r = self.execute(["CLI=/usr/bin/composer-cli", "TEST=" + self.id(), "/tests/test_cli.sh", script]) + self.assertEqual(r.returncode, 0) + + +def main(): + unittest.main(verbosity=2) diff --git a/test/run b/test/run new file mode 100755 index 00000000..fa8fb404 --- /dev/null +++ b/test/run @@ -0,0 +1,11 @@ +#!/bin/sh -e +# This is the expected entry point for Cockpit CI; will be called without +# arguments but with an appropriate $TEST_OS, and optionally $TEST_SCENARIO + +make vm + +if [ -n "$TEST_SCENARIO" ]; then + test/check-cloud TestCloud.test_$TEST_SCENARIO +else + test/check-cli +fi diff --git a/test/vm.install b/test/vm.install new file mode 100644 index 00000000..4f4bacea --- /dev/null +++ b/test/vm.install @@ -0,0 +1,18 @@ +#!/bin/sh -eux + +SRPM="$1" + +# Grow root partition to make room for images. This only works on Fedora right now. +echo ", +" | sfdisk -N 2 -f /dev/vda +partprobe +pvresize /dev/vda2 +lvresize fedora/root -l+100%FREE -r + +su builder -c "/usr/bin/mock --no-clean --resultdir build-results --rebuild $SRPM" + +packages=$(find build-results -name '*.rpm' -not -name '*.src.rpm') +rpm -e $(basename -a ${packages[@]} | sed 's/-[0-9].*.rpm$//') || true +yum install -y beakerlib $packages + +systemctl enable lorax-composer.socket +systemctl enable docker.service diff --git a/tests/pylint/runpylint.py b/tests/pylint/runpylint.py index d4f76c6f..1a4afc90 100755 --- a/tests/pylint/runpylint.py +++ b/tests/pylint/runpylint.py @@ -22,6 +22,10 @@ class LoraxLintConfig(PocketLintConfig): retval.remove("pocketlint.checkers.markup") return retval + @property + def ignoreNames(self): + return { "bots", "rpmbuild" } + if __name__ == "__main__": conf = LoraxLintConfig() linter = PocketLinter(conf) diff --git a/tests/test_cli.sh b/tests/test_cli.sh index 07bd1d96..c2f36af5 100755 --- a/tests/test_cli.sh +++ b/tests/test_cli.sh @@ -70,7 +70,7 @@ else # execute other cli tests which need more adjustments in the calling environment # or can't be executed inside Travis CI for TEST in "$@"; do - ./$TEST + $TEST done fi