From 1c54526baf0a5b8f9818fb97f3df12925212f73d Mon Sep 17 00:00:00 2001
From: Martin Pitt <martin@piware.de>
Date: Mon, 15 Jun 2020 10:33:34 +0200
Subject: [PATCH] tests: Use run-tests --exclude option

This is simpler, more robust, and fixes the silently skipped tests as
the `{}` did not get expanded by the shell any more.
---
 tests/run-test.sh | 47 +++++++++++++++++++++++++++--------------------
 1 file changed, 27 insertions(+), 20 deletions(-)

diff --git a/tests/run-test.sh b/tests/run-test.sh
index cfce859..0010580 100755
--- a/tests/run-test.sh
+++ b/tests/run-test.sh
@@ -35,51 +35,58 @@ export TEST_ALLOW_JOURNAL_MESSAGES=".*Permission denied: '/var/cache/app-info/xm
 
 # select tests
 TESTS=""
+EXCLUDES=""
 RC=0
 if [ -n "$test_optional" ]; then
     # pre-download cirros image for Machines tests
     bots/image-download cirros
 
-    # some tests are still too unstable: testCreate,testNetworkSettings,testVmNICs
-    # testAddDisk triggers SELinux violation
-    TESTS="$TESTS
-        TestMachines.test{Basic,VCPU,MultipleSettings,BootOrder,Libvirt}
-        TestMachines.testDetachDisk
-        TestMachines.test{InlineConsole,ExternalConsole,SerialConsole}
-        TestMachines.test{StoragePools,StoragePoolsCreate,NICAdd}"
+    # still too unstable
+    EXCLUDES="$EXCLUDES
+    TestMachines.testCreate
+    TestMachines.testNetworkSettings
+    TestMachines.testVmNICs"
 
-    # not all classes are nondestructive, and we can't run rebooting tests
+    # triggers SELinux violation
+    EXCLUDES="$EXCLUDES TestMachines.testAddDisk"
+
+    # TestUpdates: we can't run rebooting tests
     TESTS="$TESTS
-         TestUpdates.test{Basic,SecurityOnly}
          TestAutoUpdates
-         TestStorage"
+         TestMachines
+         TestStorage
+         TestUpdates.testBasic
+         TestUpdates.testSecurityOnly"
 fi
 
 if [ -n "$test_basic" ]; then
-    # TestFirewall.testNetworkingPage is still too unstable
-    TESTS="$TESTS TestFirewall.test{AddCustomServices,AddServices,FirewallPage,MultipleZones}"
+    # still too unstable
+    EXCLUDES="$EXCLUDES TestFirewall.testNetworkingPage"
 
-# testExpired in RHEL 8 is a PAM bug -- it shouldn't pwquality the old password when having to set a new one, that's completely counter-productive and wrong
+    # PAM bug in RHEL 8: it shouldn't pwquality the old password when having to set a new one, that's completely counter-productive and wrong
     if [ "${TEST_OS#rhel}" != "$TEST_OS" ]; then
-        TESTS="$TESTS
-            TestLogin.test{Basic,Conversation,FailingWebsocket,FailingWebsocketSafari,FailingWebsocketSafariNoCA,Logging,Raw,SELinuxRestrictedUser,SessionRecordingShell,UnsupportedBrowser}"
-    else
-        TESTS="$TESTS
-            TestLogin"
+        EXCLUDES="$EXCLUDES TestLogin.testExpired"
     fi
 
     TESTS="$TESTS
         TestAccounts
+        TestFirewall
+        TestLogin
         TestNetworking
         TestSOS"
 fi
 
-# HACK: use fixed run-tests for BlockingIOError, until version 222 gets packaged
+# HACK: use fixed run-tests for BlockingIOError and --exclude, until version 222 gets packaged
 curl https://raw.githubusercontent.com/cockpit-project/cockpit/master/test/common/run-tests > test/common/run-tests
 chmod a+x test/common/run-tests
 
+exclude_options=""
+for t in $EXCLUDES; do
+    exclude_options="$exclude_options --exclude $t"
+done
+
 # execute run-tests
-test/common/run-tests --test-dir test/verify --trace --verbose --nondestructive \
+test/common/run-tests --test-dir test/verify --trace --verbose --nondestructive $exclude_options \
     --machine localhost:22 --browser localhost:9090 $TESTS || RC=$?
 
 # check-menu is not @nondestructive yet, keep it last