Use Cockpit's test images and infrastructure

Allows to run the tests on multiple operating systems and on the
infrastructure that the Cockpit team maintains.

`make vm` downloads one of Cockpit's test images (override which one
with TEST_OS) and installs rpms build from the local checkout of lorax.
The resulting image is placed in `test/images/$TEST_OS`.

TEST_OS can be set to any of Cockpit's supported images (default:
fedora-30).

Run `make check-vm` to run the CLI checks in the VM. The bulk of the
work is done in `test/check-cli`, which uses Cockpit's `bots` library to
start the VM and run the script in it.

Also included is a `test/run` script, which is the entrypoint for
Cockpit's test infrastructure.

.gitignore

@@ -1,4 +1,5 @@
 *.pyc
+*.rpm
 src/pylorax/version.py*
 _build/
 tests/pylint/.pylint.d/
@@ -7,3 +8,7 @@ __pycache__/
 pylint-log
 .pytest_cache/
 .test-results/
+/lorax-*.tar.gz
+/bots
+/test/images
+/tmp

Makefile

@@ -12,6 +12,12 @@ TAG = lorax-$(VERSION)-$(RELEASE)
 
 IMAGE_RELEASE = $(shell awk -F: '/FROM/ { print $$2}' Dockerfile.test)
 
+ifeq ($(TEST_OS),)
+TEST_OS = fedora-30
+endif
+export TEST_OS
+VM_IMAGE=$(CURDIR)/test/images/$(TEST_OS)
+
 default: all
 
 src/composer/version.py: lorax.spec
@@ -95,12 +101,18 @@ set-docs-owner:
 
 archive:
 	@git archive --format=tar --prefix=$(PKGNAME)-$(VERSION)/ $(TAG) > $(PKGNAME)-$(VERSION).tar
-	@gzip $(PKGNAME)-$(VERSION).tar
+	@gzip -f $(PKGNAME)-$(VERSION).tar
 	@echo "The archive is in $(PKGNAME)-$(VERSION).tar.gz"
 
 dist: tag archive
 	scp $(PKGNAME)-$(VERSION).tar.gz fedorahosted.org:lorax
 
+srpm: archive $(PKGNAME).spec
+	rpmbuild -bs \
+	  --define "_sourcedir $(CURDIR)" \
+	  --define "_srcrpmdir $(CURDIR)" \
+	  lorax.spec
+
 local:
 	@rm -rf $(PKGNAME)-$(VERSION).tar.gz
 	@rm -rf /var/tmp/$(PKGNAME)-$(VERSION)
@@ -129,8 +141,34 @@ docs-in-docker:
 
 ci: check test
 
+$(VM_IMAGE): TAG=HEAD
+$(VM_IMAGE): srpm bots
+	srpm=$(shell rpm --qf '%{Name}-%{Version}-%{Release}.src.rpm\n' -q --specfile lorax.spec | head -n1) ; \
+	bots/image-customize -v \
+		--resize 20G \
+		--upload $$srpm:/var/tmp \
+		--upload $(CURDIR)/test/vm.install:/var/tmp/vm.install \
+		--upload $(realpath tests):/ \
+		--run-command "chmod +x /var/tmp/vm.install" \
+		--run-command "cd /var/tmp; /var/tmp/vm.install $$srpm" \
+		$(TEST_OS)
+
+# convenience target for the above
+vm: $(VM_IMAGE)
+	echo $(VM_IMAGE)
+
+vm-reset:
+	rm -f $(VM_IMAGE) $(VM_IMAGE).qcow2
+
+# checkout Cockpit's bots/ directory for standard test VM images and API to launch them
+# must be from cockpit's master, as only that has current and existing images; but testvm.py API is stable
+bots:
+	git fetch --depth=1 https://github.com/cockpit-project/cockpit.git
+	git checkout --force FETCH_HEAD -- bots/
+	git reset bots
+
 .PHONY: ci_after_success
 ci_after_success:
 # nothing to do here, but Jenkins expects this to be present, otherwise fails
 
-.PHONY: docs
+.PHONY: docs check test srpm vm vm-reset

test/README.md

@@ -0,0 +1,57 @@
+# Integration Tests
+
+lorax uses Cockpit's integration test framework and infrastructure. To do this,
+we're checking out Cockpit's `bots/` subdirectory. It contains links to test
+images and tools to manipulate and start virtual machines from them.
+
+Each test is run on a new instance of a virtual machine. 
+
+## Dependencies
+
+These dependencies are needed on Fedora to run tests locally:
+
+    $ sudo dnf install curl expect \
+        libvirt libvirt-client libvirt-daemon libvirt-python \
+        python python-libguestfs python-lxml libguestfs-xfs \
+        python3 libvirt-python3 \
+        libguestfs-tools qemu qemu-kvm rpm-build rsync xz
+
+## Building a test VM
+
+To build a test VM, run
+
+    $ make vm
+
+This downloads a base image from Cockpit's infrastructure. You can control
+which image is downloaded with the `TEST_OS` environment variable. Cockpit's
+[documentation](https://github.com/cockpit-project/cockpit/blob/master/test/README.md#test-configuration)
+lists accepted values. It then creates a new image based on that (a qemu
+snapshot) in `tests/images`, which contain the current `tests/` directory and
+have newly built rpms from the current checkout installed.
+
+To delete the generated image, run
+
+    $ make vm-reset
+
+Base images are stored in `bots/images`. Set `TEST_DATA` to override this
+directory.
+
+## Running tests
+
+After building a test image, run
+
+    $ ./test/check-cli [TESTNAME]
+
+or any of the other `check-*` scripts. Right after the VM is started, these
+scripts print an `ssh` line to connect to it.
+
+Run `make vm` after changing tests or lorax source to recreate the test
+machine. It is usually not necessary to reset the VM.
+
+## Updating images
+
+The `bots/` directory is checked out from Cockpit when `make vm` is first run.
+To get the latest images you need to update it manually (in order not to poll
+GitHub every time):
+
+    $ make -B bots

test/check-cli

@@ -0,0 +1,32 @@
+#!/usr/bin/python3
+
+import composertest
+
+
+class TestSanity(composertest.ComposerTestCase):
+    def test_blueprint_sanity(self):
+        self.runCliTest("/tests/cli/test_blueprints_sanity.sh")
+
+    def test_compose_sanity(self):
+        self.runCliTest("/tests/cli/test_compose_sanity.sh")
+
+
+class TestImages(composertest.ComposerTestCase):
+    def test_ext4_filesystem(self):
+        self.runCliTest("/tests/cli/test_compose_ext4-filesystem.sh")
+
+    def test_partitioned_disk(self):
+        self.runCliTest("/tests/cli/test_compose_partitioned-disk.sh")
+
+    def test_tar(self):
+        self.runCliTest("/tests/cli/test_compose_tar.sh")
+
+    def test_qcow2(self):
+        self.runCliTest("/tests/cli/test_compose_qcow2.sh")
+
+    def test_live_iso(self):
+        self.runCliTest("/tests/cli/test_compose_live-iso.sh")
+
+
+if __name__ == '__main__':
+    composertest.main()

test/check-cloud

@@ -0,0 +1,21 @@
+#!/usr/bin/python3
+
+import composertest
+
+
+class TestCloud(composertest.ComposerTestCase):
+    def test_aws(self):
+        self.runCliTest("/tests/cli/test_build_and_deploy_aws.sh")
+
+    def test_azure(self):
+        self.runCliTest("/tests/cli/test_build_and_deploy_azure.sh")
+
+    def test_openstack(self):
+        self.runCliTest("/tests/cli/test_build_and_deploy_openstack.sh")
+
+    def test_vmware(self):
+        self.runCliTest("/tests/cli/test_build_and_deploy_vmware.sh")
+
+
+if __name__ == '__main__':
+    composertest.main()

test/composertest.py

@@ -0,0 +1,59 @@
+#!/usr/bin/python3
+
+import os
+import subprocess
+import sys
+import unittest
+
+# import Cockpit's machinery for test VMs and its browser test API
+sys.path.append(os.path.join(os.path.dirname(__file__), "../bots/machine"))
+import testvm # pylint: disable=import-error
+
+
+class ComposerTestCase(unittest.TestCase):
+    image = testvm.DEFAULT_IMAGE
+
+    def setUp(self):
+        network = testvm.VirtNetwork(0)
+        self.machine = testvm.VirtMachine(self.image, networking=network.host(), memory_mb=2048)
+
+        print(f"Starting virtual machine '{self.image}'")
+        self.machine.start()
+        self.machine.wait_boot()
+
+        # run a command to force starting the SSH master
+        self.machine.execute("uptime")
+
+        self.ssh_command = ["ssh", "-o", "ControlPath=" + self.machine.ssh_master,
+                                   "-p", self.machine.ssh_port,
+                                   self.machine.ssh_user + "@" + self.machine.ssh_address]
+
+        print("Machine is up. Connect to it via:")
+        print(" ".join(self.ssh_command))
+        print()
+
+        print("Waiting for lorax-composer to become ready...")
+        curl_command = ["curl", "--max-time", "360",
+                                "--silent",
+                                "--unix-socket", "/run/weldr/api.socket",
+                                "http://localhost/api/status"]
+        r = subprocess.run(self.ssh_command + curl_command, stdout=subprocess.DEVNULL)
+        self.assertEqual(r.returncode, 0)
+
+    def tearDown(self):
+        self.machine.stop()
+
+    def execute(self, command, **args):
+        """Execute a command on the test machine.
+
+        **args and return value are the same as those for subprocess.run().
+        """
+        return subprocess.run(self.ssh_command + command, **args)
+
+    def runCliTest(self, script):
+        r = self.execute(["CLI=/usr/bin/composer-cli", "TEST=" + self.id(), "/tests/test_cli.sh", script])
+        self.assertEqual(r.returncode, 0)
+
+
+def main():
+    unittest.main(verbosity=2)

test/run

@@ -0,0 +1,11 @@
+#!/bin/sh -e
+# This is the expected entry point for Cockpit CI; will be called without
+# arguments but with an appropriate $TEST_OS, and optionally $TEST_SCENARIO
+
+make vm
+
+if [ -n "$TEST_SCENARIO" ]; then
+  test/check-cloud TestCloud.test_$TEST_SCENARIO
+else
+  test/check-cli
+fi

test/vm.install

@@ -0,0 +1,18 @@
+#!/bin/sh -eux
+
+SRPM="$1"
+
+# Grow root partition to make room for images. This only works on Fedora right now.
+echo ", +" | sfdisk -N 2 -f /dev/vda
+partprobe
+pvresize /dev/vda2
+lvresize fedora/root -l+100%FREE -r
+
+su builder -c "/usr/bin/mock --no-clean --resultdir build-results --rebuild $SRPM"
+
+packages=$(find build-results -name '*.rpm' -not -name '*.src.rpm')
+rpm -e $(basename -a ${packages[@]} | sed 's/-[0-9].*.rpm$//') || true
+yum install -y beakerlib $packages
+
+systemctl enable lorax-composer.socket
+systemctl enable docker.service

tests/pylint/runpylint.py

@@ -22,6 +22,10 @@ class LoraxLintConfig(PocketLintConfig):
         retval.remove("pocketlint.checkers.markup")
         return retval
 
+    @property
+    def ignoreNames(self):
+        return { "bots", "rpmbuild" }
+
 if __name__ == "__main__":
     conf = LoraxLintConfig()
     linter = PocketLinter(conf)

tests/test_cli.sh

@@ -70,7 +70,7 @@ else
     # execute other cli tests which need more adjustments in the calling environment
     # or can't be executed inside Travis CI
     for TEST in "$@"; do
-        ./$TEST
+        $TEST
     done
 fi