Use Cockpit's test images and infrastructure

Allows to run the tests on multiple operating systems and on the
infrastructure that the Cockpit team maintains.

`make vm` downloads one of Cockpit's test images (override which one
with TEST_OS) and installs rpms build from the local checkout of lorax.
The resulting image is placed in `test/images/$TEST_OS`.

TEST_OS can be set to any of Cockpit's supported images (default:
fedora-30).

Run `make check-vm` to run the CLI checks in the VM. The bulk of the
work is done in `test/check-cli`, which uses Cockpit's `bots` library to
start the VM and run the script in it.

Also included is a `test/run` script, which is the entrypoint for
Cockpit's test infrastructure.
This commit is contained in:
Lars Karlitski 2019-04-12 20:17:28 +02:00 committed by Martin Pitt
parent aa9c2e13b1
commit 250f49f78d
10 changed files with 248 additions and 3 deletions

5
.gitignore vendored
View File

@ -1,4 +1,5 @@
*.pyc *.pyc
*.rpm
src/pylorax/version.py* src/pylorax/version.py*
_build/ _build/
tests/pylint/.pylint.d/ tests/pylint/.pylint.d/
@ -7,3 +8,7 @@ __pycache__/
pylint-log pylint-log
.pytest_cache/ .pytest_cache/
.test-results/ .test-results/
/lorax-*.tar.gz
/bots
/test/images
/tmp

View File

@ -12,6 +12,12 @@ TAG = lorax-$(VERSION)-$(RELEASE)
IMAGE_RELEASE = $(shell awk -F: '/FROM/ { print $$2}' Dockerfile.test) IMAGE_RELEASE = $(shell awk -F: '/FROM/ { print $$2}' Dockerfile.test)
ifeq ($(TEST_OS),)
TEST_OS = fedora-30
endif
export TEST_OS
VM_IMAGE=$(CURDIR)/test/images/$(TEST_OS)
default: all default: all
src/composer/version.py: lorax.spec src/composer/version.py: lorax.spec
@ -95,12 +101,18 @@ set-docs-owner:
archive: archive:
@git archive --format=tar --prefix=$(PKGNAME)-$(VERSION)/ $(TAG) > $(PKGNAME)-$(VERSION).tar @git archive --format=tar --prefix=$(PKGNAME)-$(VERSION)/ $(TAG) > $(PKGNAME)-$(VERSION).tar
@gzip $(PKGNAME)-$(VERSION).tar @gzip -f $(PKGNAME)-$(VERSION).tar
@echo "The archive is in $(PKGNAME)-$(VERSION).tar.gz" @echo "The archive is in $(PKGNAME)-$(VERSION).tar.gz"
dist: tag archive dist: tag archive
scp $(PKGNAME)-$(VERSION).tar.gz fedorahosted.org:lorax scp $(PKGNAME)-$(VERSION).tar.gz fedorahosted.org:lorax
srpm: archive $(PKGNAME).spec
rpmbuild -bs \
--define "_sourcedir $(CURDIR)" \
--define "_srcrpmdir $(CURDIR)" \
lorax.spec
local: local:
@rm -rf $(PKGNAME)-$(VERSION).tar.gz @rm -rf $(PKGNAME)-$(VERSION).tar.gz
@rm -rf /var/tmp/$(PKGNAME)-$(VERSION) @rm -rf /var/tmp/$(PKGNAME)-$(VERSION)
@ -129,8 +141,34 @@ docs-in-docker:
ci: check test ci: check test
$(VM_IMAGE): TAG=HEAD
$(VM_IMAGE): srpm bots
srpm=$(shell rpm --qf '%{Name}-%{Version}-%{Release}.src.rpm\n' -q --specfile lorax.spec | head -n1) ; \
bots/image-customize -v \
--resize 20G \
--upload $$srpm:/var/tmp \
--upload $(CURDIR)/test/vm.install:/var/tmp/vm.install \
--upload $(realpath tests):/ \
--run-command "chmod +x /var/tmp/vm.install" \
--run-command "cd /var/tmp; /var/tmp/vm.install $$srpm" \
$(TEST_OS)
# convenience target for the above
vm: $(VM_IMAGE)
echo $(VM_IMAGE)
vm-reset:
rm -f $(VM_IMAGE) $(VM_IMAGE).qcow2
# checkout Cockpit's bots/ directory for standard test VM images and API to launch them
# must be from cockpit's master, as only that has current and existing images; but testvm.py API is stable
bots:
git fetch --depth=1 https://github.com/cockpit-project/cockpit.git
git checkout --force FETCH_HEAD -- bots/
git reset bots
.PHONY: ci_after_success .PHONY: ci_after_success
ci_after_success: ci_after_success:
# nothing to do here, but Jenkins expects this to be present, otherwise fails # nothing to do here, but Jenkins expects this to be present, otherwise fails
.PHONY: docs .PHONY: docs check test srpm vm vm-reset

57
test/README.md Normal file
View File

@ -0,0 +1,57 @@
# Integration Tests
lorax uses Cockpit's integration test framework and infrastructure. To do this,
we're checking out Cockpit's `bots/` subdirectory. It contains links to test
images and tools to manipulate and start virtual machines from them.
Each test is run on a new instance of a virtual machine.
## Dependencies
These dependencies are needed on Fedora to run tests locally:
$ sudo dnf install curl expect \
libvirt libvirt-client libvirt-daemon libvirt-python \
python python-libguestfs python-lxml libguestfs-xfs \
python3 libvirt-python3 \
libguestfs-tools qemu qemu-kvm rpm-build rsync xz
## Building a test VM
To build a test VM, run
$ make vm
This downloads a base image from Cockpit's infrastructure. You can control
which image is downloaded with the `TEST_OS` environment variable. Cockpit's
[documentation](https://github.com/cockpit-project/cockpit/blob/master/test/README.md#test-configuration)
lists accepted values. It then creates a new image based on that (a qemu
snapshot) in `tests/images`, which contain the current `tests/` directory and
have newly built rpms from the current checkout installed.
To delete the generated image, run
$ make vm-reset
Base images are stored in `bots/images`. Set `TEST_DATA` to override this
directory.
## Running tests
After building a test image, run
$ ./test/check-cli [TESTNAME]
or any of the other `check-*` scripts. Right after the VM is started, these
scripts print an `ssh` line to connect to it.
Run `make vm` after changing tests or lorax source to recreate the test
machine. It is usually not necessary to reset the VM.
## Updating images
The `bots/` directory is checked out from Cockpit when `make vm` is first run.
To get the latest images you need to update it manually (in order not to poll
GitHub every time):
$ make -B bots

32
test/check-cli Executable file
View File

@ -0,0 +1,32 @@
#!/usr/bin/python3
import composertest
class TestSanity(composertest.ComposerTestCase):
def test_blueprint_sanity(self):
self.runCliTest("/tests/cli/test_blueprints_sanity.sh")
def test_compose_sanity(self):
self.runCliTest("/tests/cli/test_compose_sanity.sh")
class TestImages(composertest.ComposerTestCase):
def test_ext4_filesystem(self):
self.runCliTest("/tests/cli/test_compose_ext4-filesystem.sh")
def test_partitioned_disk(self):
self.runCliTest("/tests/cli/test_compose_partitioned-disk.sh")
def test_tar(self):
self.runCliTest("/tests/cli/test_compose_tar.sh")
def test_qcow2(self):
self.runCliTest("/tests/cli/test_compose_qcow2.sh")
def test_live_iso(self):
self.runCliTest("/tests/cli/test_compose_live-iso.sh")
if __name__ == '__main__':
composertest.main()

21
test/check-cloud Executable file
View File

@ -0,0 +1,21 @@
#!/usr/bin/python3
import composertest
class TestCloud(composertest.ComposerTestCase):
def test_aws(self):
self.runCliTest("/tests/cli/test_build_and_deploy_aws.sh")
def test_azure(self):
self.runCliTest("/tests/cli/test_build_and_deploy_azure.sh")
def test_openstack(self):
self.runCliTest("/tests/cli/test_build_and_deploy_openstack.sh")
def test_vmware(self):
self.runCliTest("/tests/cli/test_build_and_deploy_vmware.sh")
if __name__ == '__main__':
composertest.main()

59
test/composertest.py Normal file
View File

@ -0,0 +1,59 @@
#!/usr/bin/python3
import os
import subprocess
import sys
import unittest
# import Cockpit's machinery for test VMs and its browser test API
sys.path.append(os.path.join(os.path.dirname(__file__), "../bots/machine"))
import testvm # pylint: disable=import-error
class ComposerTestCase(unittest.TestCase):
image = testvm.DEFAULT_IMAGE
def setUp(self):
network = testvm.VirtNetwork(0)
self.machine = testvm.VirtMachine(self.image, networking=network.host(), memory_mb=2048)
print(f"Starting virtual machine '{self.image}'")
self.machine.start()
self.machine.wait_boot()
# run a command to force starting the SSH master
self.machine.execute("uptime")
self.ssh_command = ["ssh", "-o", "ControlPath=" + self.machine.ssh_master,
"-p", self.machine.ssh_port,
self.machine.ssh_user + "@" + self.machine.ssh_address]
print("Machine is up. Connect to it via:")
print(" ".join(self.ssh_command))
print()
print("Waiting for lorax-composer to become ready...")
curl_command = ["curl", "--max-time", "360",
"--silent",
"--unix-socket", "/run/weldr/api.socket",
"http://localhost/api/status"]
r = subprocess.run(self.ssh_command + curl_command, stdout=subprocess.DEVNULL)
self.assertEqual(r.returncode, 0)
def tearDown(self):
self.machine.stop()
def execute(self, command, **args):
"""Execute a command on the test machine.
**args and return value are the same as those for subprocess.run().
"""
return subprocess.run(self.ssh_command + command, **args)
def runCliTest(self, script):
r = self.execute(["CLI=/usr/bin/composer-cli", "TEST=" + self.id(), "/tests/test_cli.sh", script])
self.assertEqual(r.returncode, 0)
def main():
unittest.main(verbosity=2)

11
test/run Executable file
View File

@ -0,0 +1,11 @@
#!/bin/sh -e
# This is the expected entry point for Cockpit CI; will be called without
# arguments but with an appropriate $TEST_OS, and optionally $TEST_SCENARIO
make vm
if [ -n "$TEST_SCENARIO" ]; then
test/check-cloud TestCloud.test_$TEST_SCENARIO
else
test/check-cli
fi

18
test/vm.install Normal file
View File

@ -0,0 +1,18 @@
#!/bin/sh -eux
SRPM="$1"
# Grow root partition to make room for images. This only works on Fedora right now.
echo ", +" | sfdisk -N 2 -f /dev/vda
partprobe
pvresize /dev/vda2
lvresize fedora/root -l+100%FREE -r
su builder -c "/usr/bin/mock --no-clean --resultdir build-results --rebuild $SRPM"
packages=$(find build-results -name '*.rpm' -not -name '*.src.rpm')
rpm -e $(basename -a ${packages[@]} | sed 's/-[0-9].*.rpm$//') || true
yum install -y beakerlib $packages
systemctl enable lorax-composer.socket
systemctl enable docker.service

View File

@ -22,6 +22,10 @@ class LoraxLintConfig(PocketLintConfig):
retval.remove("pocketlint.checkers.markup") retval.remove("pocketlint.checkers.markup")
return retval return retval
@property
def ignoreNames(self):
return { "bots", "rpmbuild" }
if __name__ == "__main__": if __name__ == "__main__":
conf = LoraxLintConfig() conf = LoraxLintConfig()
linter = PocketLinter(conf) linter = PocketLinter(conf)

View File

@ -70,7 +70,7 @@ else
# execute other cli tests which need more adjustments in the calling environment # execute other cli tests which need more adjustments in the calling environment
# or can't be executed inside Travis CI # or can't be executed inside Travis CI
for TEST in "$@"; do for TEST in "$@"; do
./$TEST $TEST
done done
fi fi