From 63b1437d96468d0fc81ea6ed0def628b98cbb285 Mon Sep 17 00:00:00 2001 From: Jan Tulak Date: Tue, 24 Jul 2018 14:11:13 +0200 Subject: [PATCH 6/9] tests: add bash test skipping capabilities Not all tests can be run all the time. If e.g. a required tool for some backend is missing, it is better to skip the test and mark it so, rather than fail, leaving the developer to search why. Signed-off-by: Jan Tulak --- .gitignore | 1 + test.py | 35 ++++++++++++++++++++++++++++---- tests/bashtests/016-multipath.sh | 4 ++-- tests/bashtests/lib/utils.sh | 5 +++++ 4 files changed, 39 insertions(+), 6 deletions(-) diff --git a/.gitignore b/.gitignore index 53af491..d0038e8 100644 --- a/.gitignore +++ b/.gitignore @@ -5,4 +5,5 @@ doc/src/options/ *.pyc *.swp tests/bashtests/*.bad +tests/bashtests/*.skipped tests/bashtests/*.out diff --git a/test.py b/test.py index 5620679..9c2610e 100755 --- a/test.py +++ b/test.py @@ -103,7 +103,7 @@ def check_system_dependencies(): return True -def run_bash_tests(names, want_logs=False): +def run_bash_tests(names, exclude=[], want_logs=False): cur = os.getcwd() os.chdir('./tests/bashtests') command = ['ls', '-m'] @@ -112,6 +112,7 @@ def run_bash_tests(names, want_logs=False): failed = [] passed = [] + skipped = [] count = 0 misc.run('./set.sh', stdout=False) output = misc.run(command, stdout=False)[1] @@ -122,12 +123,20 @@ def run_bash_tests(names, want_logs=False): continue if names and script not in names and script[:3] not in names: continue + if exclude and (script in exclude or script[:3] in exclude): + skipped.append(script) + print("{0:<29} \033[90m[SKIPPED]\033[0m".format(script)) + continue + count += 1 sys.stdout.write("{0:<29}".format(script) + " ") sys.stdout.flush() bad_file = re.sub("\.sh$",".bad", script) + skip_file = re.sub("\.sh$",".skipped", script) if os.access(bad_file, os.R_OK): os.remove(bad_file) + if os.access(skip_file, os.R_OK): + os.remove(skip_file) ret, out, err = misc.run(['./' + script], stdout=False, can_fail=True) if ret: print("\033[91m[FAILED]\033[0m") @@ -140,6 +149,11 @@ def run_bash_tests(names, want_logs=False): print("\033[93m[WARNING]\033[0m") with open(bad_file, 'w') as f: f.write(out) + elif re.search("\[TEST SKIPPED\]", out): + print("\033[90m[SKIPPED]\033[0m") + skipped.append(script) + with open(skip_file, 'w') as f: + f.write(out) else: print("\033[92m[PASSED]\033[0m") passed.append(script) @@ -156,6 +170,10 @@ def run_bash_tests(names, want_logs=False): print("Ran {0} tests in {1} seconds.".format(count, round(t1, 2))) print("{0} tests PASSED: {1}".format(len(passed), ", ".join(passed))) ret = 0 + if len(skipped) > 0: + print("{0} tests SKIPPED: {1}".format(len(skipped), ", ".join(skipped))) + print("See files with \"skipped\" extension for output") + if len(failed) > 0: print("{0} tests FAILED: {1}".format(len(failed), ", ".join(failed))) print("See files with \"bad\" extension for output") @@ -269,7 +287,11 @@ if __name__ == '__main__': parser.add_argument('-l', '--logs', dest='want_logs', action='store_true', help='if a bash test fails, print out it\'s log to stdout') parser.add_argument('-s', '--system', dest='system', action='store_true', - help='Test the installed version of ssm in system. Implies --bash.') + help='test the installed version of ssm in system. implies --bash.') + parser.add_argument('--skip', metavar='TEST', type=str, nargs='+', + help='Bash tests to to be skipped. ' + 'Can\'t be used together with an explicit list of tests to run.' + 'Tests can be specified either with a full name, or by just their number.') parser.add_argument('tests', metavar='TEST', type=str, nargs='*', help='Specific tests to be run. For bash tests, ' 'that means either a full name (001-foo.sh), ' @@ -277,10 +299,15 @@ if __name__ == '__main__': 'For unit tests, it means something like ' 'BtrfsFunctionCheck.test_btrfs_resize for a specific test, ' 'BtrfsFunctionCheck for specific test suite ' - 'and test_btrfs for a whole file of tests.') + 'and test_btrfs for a whole file of tests.' + 'Can\'t be used together with --skip.') args = parser.parse_args() + if args.skip and args.tests: + print("Do not use --skip together with a list of tests to be run.") + sys.exit(1) + check_system_dependencies() if args.system: args.bash = True @@ -306,5 +333,5 @@ if __name__ == '__main__': print("\nRoot privileges required to run more tests!\n") sys.exit(0) print("[+] Running bash tests") - result = run_bash_tests(names=args.tests, want_logs=args.want_logs) + result = run_bash_tests(names=args.tests, exclude=args.skip, want_logs=args.want_logs) sys.exit(result) diff --git a/tests/bashtests/016-multipath.sh b/tests/bashtests/016-multipath.sh index a8e8855..5a43e24 100755 --- a/tests/bashtests/016-multipath.sh +++ b/tests/bashtests/016-multipath.sh @@ -25,11 +25,11 @@ if ! mpath_is_configured; then echo "Multipath is not installed or configured!" echo "If it is installed, then you need to have an empty configuration created with this:" echo "sudo mpathconf --enable --with_multipathd y" - exit 1 + skip_test fi if [ mpath_verify -eq 0]; then echo "This test can't be run, because there already is an existing multipath configuration." - exit 1 + skip_test fi export COLUMNS=1024 diff --git a/tests/bashtests/lib/utils.sh b/tests/bashtests/lib/utils.sh index 5627770..7b87f89 100644 --- a/tests/bashtests/lib/utils.sh +++ b/tests/bashtests/lib/utils.sh @@ -205,3 +205,8 @@ if [ -n "$PREFIX" ]; then eval "mnt$i=$mnt" done fi + +skip_test() { + echo "[TEST SKIPPED]" + exit 0 +} -- 2.21.0