From 0fee2830080033ea2be13be30d156b51dcf75b7d Mon Sep 17 00:00:00 2001 From: Martin Pitt Date: Fri, 19 Mar 2021 15:55:58 +0100 Subject: [PATCH] tests: Replace STI tests with upstream FMF tests Upstream now includes FMF metadata and the setup/run scripts to run the nondestructive verify tests in a provided testbed. These are now constantly being verified through packit. Drop the code duplication and just add some minimal FMF metadata to discover the tests. This currently has to hardcode the version number. This eventually must become more flexible in FMF (https://github.com/psss/tmt/issues/585), but until then we can update this in the cockpituous release scripts. --- .fmf/version | 1 + plans/upstream.fmf | 7 +++ tests/run-test.sh | 111 --------------------------------------------- tests/tests.yml | 39 ---------------- tests/verify.sh | 59 ------------------------ 5 files changed, 8 insertions(+), 209 deletions(-) create mode 100644 .fmf/version create mode 100644 plans/upstream.fmf delete mode 100755 tests/run-test.sh delete mode 100644 tests/tests.yml delete mode 100755 tests/verify.sh diff --git a/.fmf/version b/.fmf/version new file mode 100644 index 0000000..d00491f --- /dev/null +++ b/.fmf/version @@ -0,0 +1 @@ +1 diff --git a/plans/upstream.fmf b/plans/upstream.fmf new file mode 100644 index 0000000..e7001d1 --- /dev/null +++ b/plans/upstream.fmf @@ -0,0 +1,7 @@ +discover: + how: fmf + repository: https://github.com/cockpit-project/cockpit + # FIXME: get rid of the hardcoding: https://github.com/psss/tmt/issues/585 + ref: "240" +execute: + how: tmt diff --git a/tests/run-test.sh b/tests/run-test.sh deleted file mode 100755 index 22bc8c5..0000000 --- a/tests/run-test.sh +++ /dev/null @@ -1,111 +0,0 @@ -#!/bin/sh -set -eux - -cd "$SOURCE" - -. /etc/os-release -test_optional= -test_basic= - -if ls ../cockpit-appstream* 1> /dev/null 2>&1; then - test_optional=1 -else - test_basic=1 -fi - -if [ "$ID" = "fedora" ]; then - test_basic=1 - test_optional=1 -fi - -# tests need cockpit's bots/ libraries -git clone --depth=1 https://github.com/cockpit-project/bots - -export TEST_OS="${ID}-${VERSION_ID/./-}" -# HACK: upstream does not yet know about rawhide -if [ "$TEST_OS" = "fedora-34" ] || [ "$TEST_OS" = "fedora-35" ]; then - export TEST_OS=fedora-33 -fi - -# HACK: CI hits this selinux denial. Unrelated to our tests. -export TEST_ALLOW_JOURNAL_MESSAGES=".*Permission denied:.*/var/cache/app-info/xmls.*" - -# select tests -TESTS="" -EXCLUDES="" -RC=0 -if [ -n "$test_optional" ]; then - # pre-download cirros image for Machines tests - bots/image-download cirros - - # triggers SELinux violation - # See journal: SELinux is preventing /usr/libexec/qemu-kvm from open access on the file /var/lib/cockpittest/nfs_pool/nfs-volume-0. - EXCLUDES="$EXCLUDES TestMachinesDisks.testAddDiskNFS" - # not investigated yet - EXCLUDES="$EXCLUDES - TestAutoUpdates.testPrivilegeChange" - - # TestUpdates: we can't run rebooting tests - TESTS="$TESTS - TestAutoUpdates - TestStorage - TestUpdates.testBasic - TestUpdates.testSecurityOnly" - - # Fedora gating tests are running on infra without /dev/kvm; Machines tests are too darn slow there - if [ "$ID" = "fedora" ]; then - TESTS="$TESTS TestMachinesCreate.testCreateImportDisk" - else - TESTS="$TESTS TestMachines" - fi -fi - -if [ -n "$test_basic" ]; then - # still too unstable - EXCLUDES="$EXCLUDES TestFirewall.testNetworkingPage" - - # TODO: fix for CI environment - EXCLUDES="$EXCLUDES TestLogin.testTally" - EXCLUDES="$EXCLUDES TestAccounts.testBasic" - - # PCI devices list is not predictable - EXCLUDES="$EXCLUDES TestSystemInfo.testHardwareInfo" - - # Known issue #1008 - EXCLUDES="$EXCLUDES TestTuned.testBasic" - - TESTS="$TESTS - TestAccounts - TestBonding - TestBridge - TestFirewall - TestKdump - TestLogin - TestNetworking - TestServices - TestSOS - TestSystemInfo - TestTeam - TestTerminal - TestTuned - " -fi - -exclude_options="" -for t in $EXCLUDES; do - exclude_options="$exclude_options --exclude $t" -done - -# execute run-tests -test/common/run-tests --test-dir test/verify --nondestructive $exclude_options \ - --machine localhost:22 --browser localhost:9090 $TESTS || RC=$? - -# check-shell-menu is not @nondestructive yet, keep it last -if [ -n "$test_basic" ]; then - test/verify/check-shell-menu --machine localhost:22 --browser localhost:9090 || RC=$? -fi - -echo $RC > "$LOGS/exitcode" -cp --verbose Test* "$LOGS" || true -# deliver test result via exitcode file -exit 0 diff --git a/tests/tests.yml b/tests/tests.yml deleted file mode 100644 index 5bfd2e9..0000000 --- a/tests/tests.yml +++ /dev/null @@ -1,39 +0,0 @@ ---- -- hosts: localhost - roles: - - role: standard-test-source - tags: - - always - - - role: standard-test-basic - tags: - - atomic - - classic - required_packages: - - cockpit - - cockpit-tests - - cockpit-machines - - cockpit-sosreport - - cockpit-storaged - - createrepo_c - - cryptsetup - - dnf-automatic - - firewalld - - git - - libvirt-daemon-config-network - - libvirt-python3 - - make - - NetworkManager-team - - nodejs - - python3 - - sssd-dbus - - targetcli - - tlog - tests: - - smoke: - dir: ./source/tools/debian/tests/ - run: ./smoke - - verify: - dir: . - run: ./verify.sh - save-files: ["logs/*"] diff --git a/tests/verify.sh b/tests/verify.sh deleted file mode 100755 index a888992..0000000 --- a/tests/verify.sh +++ /dev/null @@ -1,59 +0,0 @@ -#!/bin/sh -set -eux - -TESTS="$(realpath $(dirname "$0"))" -if [ -d source ]; then - # path for standard-test-source - SOURCE="$(pwd)/source" -else - SOURCE="$(realpath $TESTS/..)" -fi -LOGS="$(pwd)/logs" -mkdir -p "$LOGS" -chmod a+w "$LOGS" - -# install browser; on RHEL, use chromium from epel -# HACK: chromium-headless ought to be enough, but version 88 has a crash: https://bugs.chromium.org/p/chromium/issues/detail?id=1170634 -if ! rpm -q chromium; then - if grep -q 'ID=.*rhel' /etc/os-release; then - dnf install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm - dnf config-manager --enable epel - fi - dnf install -y chromium -fi - -# make libpwquality less aggressive, so that our "foobar" password works -printf 'dictcheck = 0\nminlen = 6\n' >> /etc/security/pwquality.conf - -# set root password for logging in -echo root:foobar | chpasswd - -# create user account for logging in -if ! id admin 2>/dev/null; then - useradd -c Administrator -G wheel admin - echo admin:foobar | chpasswd -fi - -# create user account for running the test -if ! id runtest 2>/dev/null; then - useradd -c 'Test runner' runtest - # allow test to set up things on the machine - mkdir -p /root/.ssh - curl https://raw.githubusercontent.com/cockpit-project/bots/master/machine/identity.pub >> /root/.ssh/authorized_keys - chmod 600 /root/.ssh/authorized_keys -fi -chown -R runtest "$SOURCE" - -# disable core dumps, we rather investigate them upstream where test VMs are accessible -echo core > /proc/sys/kernel/core_pattern - -# make sure that we can access cockpit through the firewall -systemctl start firewalld -firewall-cmd --add-service=cockpit --permanent -firewall-cmd --add-service=cockpit - -# Run tests as unprivileged user -su - -c "env SOURCE=$SOURCE LOGS=$LOGS $TESTS/run-test.sh" runtest - -RC=$(cat $LOGS/exitcode) -exit ${RC:-1}