From f67deacfc35be0c0322bdf801ab345a2311bf796 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C4=8Cestm=C3=ADr=20Kalina?= Date: Wed, 24 Jul 2024 18:13:04 +0200 Subject: [PATCH] gating: carry over from c9s MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Čestmír Kalina --- gating.yaml | 7 + tests/README.md | 78 +++++++++++ tests/sanity/Makefile | 89 +++++++++++++ tests/sanity/README.md | 78 +++++++++++ tests/sanity/_env | 9 ++ tests/sanity/common-symbols.sh | 65 ++++++++++ tests/sanity/common-tests.sh | 148 +++++++++++++++++++++ tests/sanity/runtest.sh | 179 ++++++++++++++++++++++++++ tests/sanity/test-ksc-elf.sh | 177 +++++++++++++++++++++++++ tests/sanity/test-ksc-exec.sh | 30 +++++ tests/sanity/test-ksc-help.sh | 56 ++++++++ tests/sanity/test-ksc-invalid-file.sh | 46 +++++++ tests/sanity/test-ksc-man.sh | 55 ++++++++ tests/tests.yml | 18 +++ 14 files changed, 1035 insertions(+) create mode 100644 gating.yaml create mode 100644 tests/README.md create mode 100644 tests/sanity/Makefile create mode 100644 tests/sanity/README.md create mode 100644 tests/sanity/_env create mode 100644 tests/sanity/common-symbols.sh create mode 100644 tests/sanity/common-tests.sh create mode 100755 tests/sanity/runtest.sh create mode 100644 tests/sanity/test-ksc-elf.sh create mode 100644 tests/sanity/test-ksc-exec.sh create mode 100644 tests/sanity/test-ksc-help.sh create mode 100644 tests/sanity/test-ksc-invalid-file.sh create mode 100644 tests/sanity/test-ksc-man.sh create mode 100644 tests/tests.yml diff --git a/gating.yaml b/gating.yaml new file mode 100644 index 0000000..5dfe8b7 --- /dev/null +++ b/gating.yaml @@ -0,0 +1,7 @@ +--- !Policy +product_versions: + - rhel-* +decision_context: osci_compose_gate +rules: + - !PassingTestCaseRule {test_case_name: osci.brew-build.tier0.functional} + diff --git a/tests/README.md b/tests/README.md new file mode 100644 index 0000000..1e87abf --- /dev/null +++ b/tests/README.md @@ -0,0 +1,78 @@ +# README + +The following test cases will be considered as a part of `ksc` gating. + +## Installation issues + +1. Check whether Module.symvers has been installed +1. Check whether kabi-stablelists package has been installed +1. Python Bugzilla interface has been installed +1. Expect error when nm is not found in $PATH. + +## Valid uses of `ksc` + +List of tests: + +1. No external symbol required +1. Whitelisted symbol use only +1. Non-whitelisted symbol use only +1. Mixed whitelisted and non-whitelisted symbol use +1. Mixed whitelisted and non-whitelisted symbol use with multiple -k arguments +1. `ksc` called on 8139, crc32_generic, xor + +Testing will be done as follows: + +1. If applicable, try and build a kernel module usecase. +1. Pass kernel module(s) to ksc, record output. +1. Using the kabi-tools ksc parser [1], parse ksc output. +1. Using the nm tool, uname -i and kabi-whitelist, indepedently compose + the dictionary that ksc parser produces on ksc output. +1. Check for match. + +Note that this series of tests checks both ksc valid functionality as well as +syntax requirements (indeed, should syntax of a ksc report be significantly +changed, ksc parser would produce different results). + +[1] http://git.engineering.redhat.com/git/users/ckalina/kabi-greylists-devel.git/tree/src/greylists/ksc.py + +## Submission tests + +Partner RHBZ is used for this purpose. + +1. All of the 'Valid uses of `ksc`' test cases (called w/ -k) will be re-tested +here as well to test immediate submission after generating. +1. Use any ksc-reports.txt to test submission onto partner RHBZ; in particular + test: + - failure on invalid product + - failure on invalid RHBZ credentials (username, password) + - failure on invalid RHBZ API key + - correct submission (using the gating-embedded bugzilla-cli tool to + download and check details of bug and attachments). + +## Further tests include: + +1. Check whether justification carry-over works within the same ko file. +1. Check whether justification carry-over does not happen when different ko + file is used. + +## Invalid uses of `ksc` + +1. Expect error when argument arity is not expected (e.g., $ ksc -k, w/o a ko + file provided) +1. Expect error when passing non-kernel module as a kernel module (-k) +1. Expect error when passing a file w/o read permissions (e.g., kernel module, + Module.symvers, ...) +1. Expect error when passing a non-existent file (e.g., kernel module, ...) +1. Expect error when passing a malformed kernel module. +1. Expect error when passing a malformed Module.symvers. +1. Expect error when passing a ksc-report.txt as a source of justifications. + +## Compatibility Tests + +1. Verify that no options were removed from -h/--help. +1. Verify that no options were removed from manpage. + +## Test known bugs + +1. EOFError when piping through ksc + diff --git a/tests/sanity/Makefile b/tests/sanity/Makefile new file mode 100644 index 0000000..dd5c4d8 --- /dev/null +++ b/tests/sanity/Makefile @@ -0,0 +1,89 @@ +# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +# +# Makefile of ksc/sanity +# Description: ksc test +# +# 2019-04-01 +# Author: Čestmír Kalina +# +# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +# +# Copyright (c) 2019 Red Hat, Inc. +# +# This copyrighted material is made available to anyone wishing +# to use, modify, copy, or redistribute it subject to the terms +# and conditions of the GNU General Public License version 2. +# +# This program is distributed in the hope that it will be +# useful, but WITHOUT ANY WARRANTY; without even the implied +# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR +# PURPOSE. See the GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public +# License along with this program; if not, write to the Free +# Software Foundation, Inc., 51 Franklin Street, Fifth Floor, +# Boston, MA 02110-1301, USA. +# +# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +TENV=_env +ifeq ($(PKG_TOP_DIR),) + export PKG_TOP_DIR := $(shell p=$$PWD; while :; do \ + [ -e $$p/env.mk -o -z "$$p" ] && { echo $$p; break; }; p=$${p%/*}; done) + export _TOP_DIR := $(shell p=$$PWD; while :; do \ + [ -d $$p/.git -o -z "$$p" ] && { echo $$p; break; }; p=$${p%/*}; done) + -include $(PKG_TOP_DIR)/env.mk +endif +include $(TENV) +ifeq ($(_TOP_DIR),) + _TOP_DIR=/mnt/tests/$(TOPLEVEL_NAMESPACE) +endif + +export TESTVERSION=1.0 + +BUILT_FILES= + +FILES=$(TENV) $(METADATA) Makefile common-symbols.sh common-tests.sh runtest.sh test-ksc-elf.sh test-ksc-exec.sh test-ksc-help.sh test-ksc-invalid-file.sh test-ksc-man.sh + +.PHONY: all install download clean + +run: $(FILES) build + ( set +o posix; . /usr/bin/rhts_environment.sh; \ + . /usr/share/beakerlib/beakerlib.sh; \ + . runtest.sh ) + +build: $(BUILT_FILES) + test -x runtest.sh || chmod a+x runtest.sh + +clean: + rm -fr *~ $(BUILT_FILES) + +include /usr/share/rhts/lib/rhts-make.include + +$(METADATA): Makefile + @echo "Owner: Čestmír Kalina " > $(METADATA) + @echo "Name: $(TEST)" >> $(METADATA) + @echo "TestVersion: $(TESTVERSION)" >> $(METADATA) + @echo "Path: $(TEST_DIR)" >> $(METADATA) + @echo "Description: ksc tests">> $(METADATA) + @echo "Type: Regression" >> $(METADATA) + @echo "TestTime: 1h" >> $(METADATA) + @echo "RunFor: ksc" >> $(METADATA) + @echo "Requires: bash" >> $(METADATA) + @echo "Requires: coreutils" >> $(METADATA) + @echo "Requires: kernel" >> $(METADATA) + @echo "Requires: kmod" >> $(METADATA) + @echo "Requires: kernel-devel" >> $(METADATA) + @echo "Requires: kernel-modules" >> $(METADATA) + @echo "Requires: kernel-abi-stablelists" >> $(METADATA) + @echo "Requires: xz" >> $(METADATA) + @echo "Requires: bzip2" >> $(METADATA) + @echo "Requires: gzip" >> $(METADATA) + @echo "Requires: $(PACKAGE_NAME) rpm wget" >> $(METADATA) + @echo "Priority: Normal" >> $(METADATA) + @echo "License: GPLv2" >> $(METADATA) + @echo "Confidential: no" >> $(METADATA) + @echo "Destructive: no" >> $(METADATA) + rhts-lint $(METADATA) + + diff --git a/tests/sanity/README.md b/tests/sanity/README.md new file mode 100644 index 0000000..e237665 --- /dev/null +++ b/tests/sanity/README.md @@ -0,0 +1,78 @@ +# README + +The following test cases will be considered as a part of `ksc` gating. + +## Installation issues + +1. Check whether Module.symvers has been installed +1. Check whether kabi-stablelists package has been installed +1. Python Bugzilla interface has been installed +1. Expect error when nm is not found in $PATH. + +## Valid uses of `ksc` + +List of tests: + +1. No external symbol required +1. Whitelisted symbol use only +1. Non-stablelisted symbol use only +1. Mixed stablelisted and non-stablelisted symbol use +1. Mixed stablelisted and non-stablelisted symbol use with multiple -k arguments +1. `ksc` called on 8139, crc32_generic, xor + +Testing will be done as follows: + +1. If applicable, try and build a kernel module usecase. +1. Pass kernel module(s) to ksc, record output. +1. Using the kabi-tools ksc parser [1], parse ksc output. +1. Using the nm tool, uname -i and kabi-stablelist, indepedently compose + the dictionary that ksc parser produces on ksc output. +1. Check for match. + +Note that this series of tests checks both ksc valid functionality as well as +syntax requirements (indeed, should syntax of a ksc report be significantly +changed, ksc parser would produce different results). + +[1] http://git.engineering.redhat.com/git/users/ckalina/kabi-greylists-devel.git/tree/src/greylists/ksc.py + +## Submission tests + +Partner RHBZ is used for this purpose. + +1. All of the 'Valid uses of `ksc`' test cases (called w/ -k) will be re-tested +here as well to test immediate submission after generating. +1. Use any ksc-reports.txt to test submission onto partner RHBZ; in particular + test: + - failure on invalid product + - failure on invalid RHBZ credentials (username, password) + - failure on invalid RHBZ API key + - correct submission (using the gating-embedded bugzilla-cli tool to + download and check details of bug and attachments). + +## Further tests include: + +1. Check whether justification carry-over works within the same ko file. +1. Check whether justification carry-over does not happen when different ko + file is used. + +## Invalid uses of `ksc` + +1. Expect error when argument arity is not expected (e.g., $ ksc -k, w/o a ko + file provided) +1. Expect error when passing non-kernel module as a kernel module (-k) +1. Expect error when passing a file w/o read permissions (e.g., kernel module, + Module.symvers, ...) +1. Expect error when passing a non-existent file (e.g., kernel module, ...) +1. Expect error when passing a malformed kernel module. +1. Expect error when passing a malformed Module.symvers. +1. Expect error when passing a ksc-report.txt as a source of justifications. + +## Compatibility Tests + +1. Verify that no options were removed from -h/--help. +1. Verify that no options were removed from manpage. + +## Test known bugs + +1. EOFError when piping through ksc + diff --git a/tests/sanity/_env b/tests/sanity/_env new file mode 100644 index 0000000..917a664 --- /dev/null +++ b/tests/sanity/_env @@ -0,0 +1,9 @@ +#This file was generated automatically,do not manually change it. +export TOPLEVEL_NAMESPACE=kernel +export PKG_NAMESPACE=kernel/general +export RELATIVE_PATH=ksc/sanity +export PACKAGE=general +export PACKAGE_NAME=general +export PKG_LIST= +export TEST=/kernel/general/ksc/sanity + diff --git a/tests/sanity/common-symbols.sh b/tests/sanity/common-symbols.sh new file mode 100644 index 0000000..a8ac42b --- /dev/null +++ b/tests/sanity/common-symbols.sh @@ -0,0 +1,65 @@ +#!/usr/bin/env bash + +# Get all symbols from ksc result's nonstablelisted sections +function ksc_report_symbols_nonstablelisted() { + sed -n 's/^.*(\([^)]*\))$/\1/p' ~/ksc-result.txt \ + | sort \ + | uniq +} + +# Get all symbols from ksc result's stablelisted sections +function ksc_report_symbols_stablelisted() { + sed -n '/^\[WHITELISTUSAGE\]$/,/^\[NONWHITELISTUSAGE\]$/p' \ + ~/ksc-result.txt \ + | grep -v '^\[' \ + | sort \ + | uniq +} + +# Get all symbols from ksc result +function ksc_report_symbols_all() { + { + ksc_report_symbols_stablelisted + ksc_report_symbols_nonstablelisted + } | sort | uniq +} + +# Get undefined symbols in all argument-provided ko files +function ko_get_undefined() { + echo -e ${@/%/\\n} \ + | xargs -I KO nm -u KO \ + | awk '{print $(NF);}' \ + | sort \ + | uniq +} + +# Get defined symbols in all argument-provided ko files +function ko_get_defined() { + echo -e ${@/%/\\n} \ + | xargs -I KO nm --defined-only KO \ + | awk '{print $(NF);}' \ + | sort \ + | uniq +} + +# Get all undefined symbols for the group; i.e., symbols defined in one +# ko and undefined in another will be ignored; this captures ksc behaviour +# when used w/ multiple -k options. +function ko_get_group_undefined() { + comm -23 <(ko_get_undefined "$@") <(ko_get_defined "$@") +} + +# Get all symbols present in kABI stablelist +function kabi_stablelists_symbols() { + grep -h '^[[:space:]]' /lib/modules/kabi-current/* \ + | tr -d '\t' \ + | sort \ + | uniq +} + +# Get all symbols present in Module.symvers +function get_module_symvers_symbols() { + awk '{print $2;}' /usr/src/kernels/$(uname -r)/Module.symvers \ + | sort \ + | uniq +} diff --git a/tests/sanity/common-tests.sh b/tests/sanity/common-tests.sh new file mode 100644 index 0000000..6d2ec7e --- /dev/null +++ b/tests/sanity/common-tests.sh @@ -0,0 +1,148 @@ +if test -z "$MANUAL" +then + export COLOR_NC= + export COLOR_WHITE= + export COLOR_BLACK= + export COLOR_BLUE= + export COLOR_LIGHT_BLUE= + export COLOR_GREEN= + export COLOR_LIGHT_GREEN= + export COLOR_CYAN= + export COLOR_LIGHT_CYAN= + export COLOR_RED= + export COLOR_LIGHT_RED= + export COLOR_PURPLE= + export COLOR_LIGHT_PURPLE= + export COLOR_BROWN= + export COLOR_YELLOW= + export COLOR_GRAY= + export COLOR_LIGHT_GRAY= +else + export COLOR_NC='\e[0m' + export COLOR_WHITE='\e[1;37m' + export COLOR_BLACK='\e[0;30m' + export COLOR_BLUE='\e[0;34m' + export COLOR_LIGHT_BLUE='\e[1;34m' + export COLOR_GREEN='\e[0;32m' + export COLOR_LIGHT_GREEN='\e[1;32m' + export COLOR_CYAN='\e[0;36m' + export COLOR_LIGHT_CYAN='\e[1;36m' + export COLOR_RED='\e[0;31m' + export COLOR_LIGHT_RED='\e[1;31m' + export COLOR_PURPLE='\e[0;35m' + export COLOR_LIGHT_PURPLE='\e[1;35m' + export COLOR_BROWN='\e[0;33m' + export COLOR_YELLOW='\e[1;33m' + export COLOR_GRAY='\e[0;30m' + export COLOR_LIGHT_GRAY='\e[0;37m' +fi + +function pass() +{ + if ! test -z "$MANUAL" + then + echo -en " $COLOR_GRAY$(printf "%0.s-" {1..35})" + echo -en "[ ${COLOR_GREEN}PASS ]" + echo -e "$COLOR_GRAY$(printf "%0.s-" {1..36})$COLOR_NC" + else + rlPass + fi +} + +function fail() +{ + if ! test -z "$MANUAL" + then + echo -en " $COLOR_GRAY$(printf "%0.s-" {1..35})" + echo -en "[ ${COLOR_RED}FAIL ]" + echo -e "$COLOR_GRAY$(printf "%0.s-" {1..36})$COLOR_NC" + else + rlFail + fi +} + +function print_test_case() +{ + if test -z "$MANUAL" + then + return 0 + fi + echo + local msg=" $COLOR_GRAY[${COLOR_BLUE}TEST $1$COLOR_GRAY]" + eval msg="\$msg\$(printf "%0.s-" {1..$[80-${#msg}+${#COLOR_BLUE}+2*${#COLOR_GRAY}]})" + echo -e "$msg$COLOR_NC" +} + +function print_description() +{ + desc=() + eval desc=\(\"\${DESCRIPTION_$1[@]}\"\) + for desc_line in "${desc[@]}" + do + echo " $desc_line" + done + echo +} + +# +# Perform a test +# +# $1 test function +# +function run_test() +{ + rlPhaseStartTest "$1" + + if test $# -eq 0 -o -z $1 + then + rlFail "ERROR: Expected test name to be passed to run_test" \ + "function." >&2 + exit 1 + fi + + local stdout_log="$2" + local stderr_log="$3" + + if test -z $stdout_log -o ! -e $stdout_log + then + rlFail "ERROR: Test inconsistency, STDOUT log file missing or" \ + "non-existent." >&2 + exit 1 + fi + + if test -z $stderr_log -o ! -e $stderr_log + then + rlFail "ERROR: Test inconsistency, STDERR log file missing or" \ + "non-existent." >&2 + exit 1 + fi + + echo > $stdout_log + echo > $stderr_log + + print_test_case "$1" + + $1 "$stdout_log" "$stderr_log" + ret=$? + + print_description "$1" + + if test $ret -eq 0 + then + pass + else + fail + + echo "STDOUT {" + cat $stdout_log + echo "}" + + echo "STDERR {" + cat $stderr_log + echo "}" + + return 1 + fi + + rlPhaseEnd +} diff --git a/tests/sanity/runtest.sh b/tests/sanity/runtest.sh new file mode 100755 index 0000000..4cef9c5 --- /dev/null +++ b/tests/sanity/runtest.sh @@ -0,0 +1,179 @@ +#!/usr/bin/env bash + +function cleanup() +{ + rlPhaseStartCleanup + rm -rf "${TEMPFILES[@]}" + rlPhaseEnd +} +trap cleanup 0 1 9 15 + +# --- Globals ----------------------------------------------------------------- + +SCRIPT_ROOT="$(dirname "$(realpath "${BASH_SOURCE[0]}")")" + +# A list of temporary files; used by cleanup to delete on signals 0 1 9 15. +TEMPFILES=() + +# A list of dependencies to include. +INCLUDES=() + +# A list of files containing test definitions. +# These are auto-discovered using test-*.sh pattern. +TESTS_FILES=() + +# A list of tests to run. +# These are automatically added by test files. +TESTS=() + +# The following can be overriden to force a particular setting. + +# RPM_BIN_DIR is not defined +# RPM_DATA_DIR is not defined +# KSC_BIN is not defined +# MOD_PATH is not defined +# MANUAL is not defined + +# --- Initialization ---------------------------------------------------------- + +echo +echo " :: ksc Gating for RHEL" +echo + +# Requires: restraint-rhts +. /usr/bin/rhts-environment.sh || exit 1 +# Requires: beakerlib +. /usr/share/beakerlib/beakerlib.sh || exit 1 + +INCLUDES+=("$SCRIPT_ROOT/common-tests.sh") +INCLUDES+=("$SCRIPT_ROOT/common-symbols.sh") + +TESTS_FILES+=("$SCRIPT_ROOT/test-"*".sh") + +# --- bkr journal ------------------------------------------------------------- + +rlJournalStart + +rlPhaseStartSetup + +# --- Load dependencies ------------------------------------------------------- + +for path in ${INCLUDES[@]} ${TESTS_FILES[@]} +do + if ! test -r $path + then + rlFail "Path \`$path' does not exist or is not readable" + exit 1 + fi + source $path && { + rlPass "File \`$(basename "$path")' loaded." + } || { + rlFail "Unable to load \`$path'." + exit 1 + } +done + +# --- Temporary files --------------------------------------------------------- + +echo ":: Initialization: Temporary files." + +__stdout_log=$(mktemp -p /tmp ksc-test-stdout.XXXXX) +TEMPFILES+=("$__stdout_log") + +__stderr_log=$(mktemp -p /tmp ksc-test-stderr.XXXXX) +TEMPFILES+=("$__stderr_log") + +# --- Evaluate RPM-specific macros -------------------------------------------- +# This is required not to hardcode ksc install location should %{_bindir} and +# %{_datadir} be changed. + +echo ":: Initialization: Evaluating RPM macros." + +if test -z "$RPM_BIN_DIR" +then + RPM_BIN_DIR="$(rpm --eval '%{_bindir}')" + echo " * RPM %{_bindir} determined as: $RPM_BIN_DIR" +fi + +if test -z "$RPM_DATA_DIR" +then + RPM_DATA_DIR="$(rpm --eval '%{_datadir}')" + echo " * RPM %{_datadir} determined as: $RPM_DATA_DIR" +fi + +# --- Determine ksc location -------------------------------------------------- + +if test -z "$KSC_BIN" +then + KSC_BIN="$RPM_BIN_DIR/ksc" + echo ":: ksc path determined as: $KSC_BIN" +fi + +if test -z "$MOD_PATH" +then + if [[ -L /lib && -d /lib ]]; then + MOD_PATH=/usr/lib/modules/$(uname -r)/ + else + MOD_PATH=/lib/modules/$(uname -r)/ + fi + echo ":: Module path determined as: $MOD_PATH" +fi + +rlPass "Initialization passed." + +rlPhaseEnd + +# --- Run tests --------------------------------------------------------------- + +overall_status=0 +for test in ${TESTS[@]} +do + run_test $test "$__stdout_log" "$__stderr_log" + if test $? -gt 0 + then + overall_status=1 + fi +done + +rlPhaseStartTest + +if test $overall_status -gt 0 +then + rlFail "Some tests failed." + exit 1 +else + rlPass "All tests passed." +fi +echo + +rlPhaseEnd + +rlJournalPrintText + +rlJournalEnd + +exit 0 + +# -- Future tests: pylint/flake8 on the python source itself + +# First, we need to detect whether or not $KSC_BIN is a shell launcher or +# the python script itself. As of this moment, $KSC_BIN is a shell launcher, +# however, currently there is no real reason for that and might be changed +# in the future. To prepare for this alternative, and to be backwards compat +# if the change occurs, detect whether $KSC_BIN is the python executable script +# or a shell launcher and obtain path to the python script in the latter case. + +KSC_PY="" +case $(file "$KSC_BIN" | awk -F'[,:] ' '{print $2;}') in +"POSIX shell script") + # We're dealing with a launcher, get the python script itself + KSC_PY="$(grep -o "[^ ]*ksc.py" "$KSC_BIN")" + ;; +"Python script") + # We're dealing with a python script + KSC_PY="$KSC_BIN" + ;; +esac + +# TODO: Call flake8/pylint here + diff --git a/tests/sanity/test-ksc-elf.sh b/tests/sanity/test-ksc-elf.sh new file mode 100644 index 0000000..06c963d --- /dev/null +++ b/tests/sanity/test-ksc-elf.sh @@ -0,0 +1,177 @@ +#!/usr/bin/env bash +# +# ksc gating +# +# verify that ksc launcher is marked executable +# + +SYMBOL_CHECKS=() + +SYMBOL_CHECKS+=(symbol_check_1) +DESCRIPTION_symbol_check_1=( + "Verify that all symbols in stablelisted and nonstablelisted sections" + "of the just produced ksc-report.txt are marked as undefined by nm," + "discounting symbols defined within the set of modules used." +) +function symbol_check_1() { + test -z "$(comm -13 <(ksc_report_symbols_all) \ + <(ko_get_group_undefined))" + return $? +} + +SYMBOL_CHECKS+=(symbol_check_2) +DESCRIPTION_symbol_check_2=( + "Verify that all symbols in stablelisted and nonstablelisted sections" + "of the just produced ksc-report.txt come from Module.symvers only." +) +function symbol_check_2() { + test -z "$(comm -23 <(ksc_report_symbols_all) \ + <(get_module_symvers_symbols))" + return $? +} + +# +SYMBOL_CHECKS+=(symbol_check_3) +DESCRIPTION_symbol_check_3=( + "Verify that all stablelisted symbols are present on kabi stablelist." +) +function symbol_check_3() { + test -z "$(comm -13 <(kabi_stablelists_symbols) \ + <(ksc_report_symbols_stablelisted))" + return $? +} + +function find_ko() +{ + if test -z $1 + then + # TODO ERRROR + return 1 + fi + + local count=$1 + local matches=() + + if test $count -gt 0 + then + matches+=($( + find "$MOD_PATH" -type f -iname "*.ko*" \ + | head -n $count \ + | xargs -I MATCH bash -c ' + case $(echo MATCH | grep -P "\.ko($|\.[^.]+$)") in + .ko.xz) unxz MATCH;; + .ko.bz2) bzip2 -d MATCH;; + .ko.gz) gunzip MATCH;; + *) echo MATCH; exit 0;; + esac; + echo MATCH | sed "s/\.[^.]*$//g";' + )) + fi + + # make sure some matches were found + if test ${#matches[@]} -eq 0 + then + echo "No modules found." >&2 + return 1 + fi + + # and that they are in fact readable + for match in ${matches[@]} + do + if test -z $match -o ! -r $match + then + # TODO ERRROR + return 1 + fi + done + + echo ${matches[@]} + + return 0 +} + +function common_ksc_elf_ko() +{ + local test_name="$1" + local stdout_log="$2" + local stderr_log="$3" + local count="$4" + shift 4 + + local ko=($(find_ko $count)) + + eval DESCRIPTION_$test_name+=\(\"\"\) + eval DESCRIPTION_$test_name+=\(\"Test used the following modules:\"\) + [ ${#ko[@]} -gt 0 ] || { echo "ERROR: No module found in MOD_PATH=$MOD_PATH."; exit 1; } + for mod in ${ko[@]} + do + mod_rel="$(realpath --relative-to="$MOD_PATH" $ko)" + eval DESCRIPTION_$test_name+=\(\"\${mod_rel/\#/ - }\"\) + done + + rm -f ~/ksc-result.txt + "$KSC_BIN" ${ko[@]/#/-k } >> $stdout_log 2>> $stderr_log + + if test $? -gt 0 + then + echo + echo "ERROR: $KSC_BIN returned w/ non-zero return code when" \ + "presented w/ valid kernel module." + + echo "STDOUT {" + cat $stdout_log + echo "}" + + echo "STDERR {" + cat $stderr_log + echo "}" + + return 1 + fi + + if test ! -s ~/ksc-result.txt + then + echo + echo "ERROR: ~/ksc-result.txt was not created or is empty." + + echo "STDOUT {" + cat $stdout_log + echo "}" + + echo "STDERR {" + cat $stderr_log + echo "}" + + return 1 + fi + + return 0 +} + +TESTS+=(test_ksc_elf_ko_single ${SYMBOL_CHECKS[@]}) + +DESCRIPTION_test_ksc_elf_ko_single=( + "Verify that ksc processes valid ELF kernel module. This amounts to" + "checking that ~/ksc-result.txt is created and that ksc terminates" + "w/ non-zero error code." +) + +function test_ksc_elf_ko_single() +{ + common_ksc_elf_ko test_ksc_elf_ko_single $@ 1 + return $? +} + +TESTS+=(test_ksc_elf_ko_mul) + +DESCRIPTION_test_ksc_elf_ko_mul=( + "Verify that ksc processes multiple valid ELF kernel modules. This" + "amounts to checking that ~/ksc-result.txt is created and that ksc" + "terminates w/ non-zero error code." +) + +function test_ksc_elf_ko_mul() +{ + common_ksc_elf_ko test_ksc_elf_ko_mul $@ 2 + return $? +} \ No newline at end of file diff --git a/tests/sanity/test-ksc-exec.sh b/tests/sanity/test-ksc-exec.sh new file mode 100644 index 0000000..46e186a --- /dev/null +++ b/tests/sanity/test-ksc-exec.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env sh +# +# ksc gating +# +# verify that ksc is executable +# + + +TESTS+=(test_ksc_executable) + +DESCRIPTION_test_ksc_executable=( + "Verify that ksc is executable." +) + +function test_ksc_executable() +{ + local stdout_log="$1" + local stderr_log="$2" + shift 2 + + if ! test -x "$KSC_BIN" + then + echo + echo "ERROR: $KSC_BIN either does not exist or is not" \ + "executable!" >&2 + ls -l "$KSC_BIN" >&2 + return 1 + fi + return 0 +} \ No newline at end of file diff --git a/tests/sanity/test-ksc-help.sh b/tests/sanity/test-ksc-help.sh new file mode 100644 index 0000000..58e112f --- /dev/null +++ b/tests/sanity/test-ksc-help.sh @@ -0,0 +1,56 @@ +#!/usr/bin/env sh +# +# ksc gating +# +# verify that ksc -h (--help) provides usage and terminated w/ non-zero error +# + + +TESTS+=(test_ksc_help) + +DESCRIPTION_test_ksc_help=( + "Verify that ksc -h (--help) provides usage and terminated w/" + "non-zero error." +) + +function test_ksc_help() +{ + local stdout_log="$1" + local stderr_log="$2" + shift 2 + + for arg in -h --help + do + echo > $stdout_log + echo > $stderr_log + + echo "# Calling $KSC_BIN $arg" >> $stdout_log + "$KSC_BIN" $arg >> $stdout_log 2>> $stderr_log + + if test $? -gt 0 + then + echo + echo "ERROR: $KSC_BIN $arg failed with non-zero" \ + "return code. Expected zero." + + echo "STDOUT {" + cat $stdout_log + echo "}" + + echo "STDERR {" + cat $stderr_log + echo "}" + + return 1 + fi + + if test ! -s "$stdout_log" -a ! -s "$stderr_log" + then + echo + echo "ERROR: $KSC_BIN $arg produced no data." >&2 + return 1 + fi + + done + return 0 +} \ No newline at end of file diff --git a/tests/sanity/test-ksc-invalid-file.sh b/tests/sanity/test-ksc-invalid-file.sh new file mode 100644 index 0000000..af89593 --- /dev/null +++ b/tests/sanity/test-ksc-invalid-file.sh @@ -0,0 +1,46 @@ +#!/usr/bin/env sh +# +# ksc gating +# +# verify that ksc fails when presented w/ non-existent path +# + +TESTS+=(test_ksc_invalid_file) + +DESCRIPTION_test_ksc_invalid_file=( + "Verify that ksc fails when presented w/ non-existent path." +) + +function test_ksc_invalid_file() +{ + local stdout_log="$1" + local stderr_log="$2" + shift 2 + + local non_exist_file="/tmp/$RANDOM" + while test -e $non_exist_file + do + non_exist_file=$non_exist_file$RANDOM + done + + "$KSC_BIN" -k "$non_exist_file" >> $stdout_log 2>> $stderr_log + + if test $? -eq 0 -a ! -e "$non_exist_file" + then + echo + echo "ERROR: $KSC_BIN returned w/ zero return code when" \ + "presented w/ non-existent file." + + echo "STDOUT {" + cat $stdout_log + echo "}" + + echo "STDERR {" + cat $stderr_log + echo "}" + + return 1 + fi + + return 0 +} \ No newline at end of file diff --git a/tests/sanity/test-ksc-man.sh b/tests/sanity/test-ksc-man.sh new file mode 100644 index 0000000..0f38da5 --- /dev/null +++ b/tests/sanity/test-ksc-man.sh @@ -0,0 +1,55 @@ +#!/usr/bin/env sh +# +# ksc gating +# +# verify that ksc manpage exists and is nonempty +# + + +TESTS+=(test_ksc_manpage) + +DESCRIPTION_test_ksc_manpage=( + "Verify that ksc manpage exists and is nonempty." +) + +function test_ksc_manpage() +{ + local stdout_log="$1" + local stderr_log="$2" + shift 2 + + LC_ALL=en_US.UTF-8 man -P cat ksc > $stdout_log 2> $stderr_log + + if test $? -gt 0 + then + echo + echo "$? ERROR: man ksc failed with non-zero" \ + "return code. Expected zero." + + echo "STDOUT {" + cat $stdout_log + echo "}" + + echo "STDERR {" + cat $stderr_log + echo "}" + + return 1 + fi + + if test ! -s "$stdout_log" + then + echo + echo "ERROR: man ksc produced no data on stdout." >&2 + return 1 + fi + + if test -s "$stderr_log" + then + echo + echo "ERROR: man ksc produced data on stderr." >&2 + return 1 + fi + + return 0 +} \ No newline at end of file diff --git a/tests/tests.yml b/tests/tests.yml new file mode 100644 index 0000000..41e73a1 --- /dev/null +++ b/tests/tests.yml @@ -0,0 +1,18 @@ +- hosts: localhost + tags: + - classic + roles: + - role: standard-test-beakerlib + tests: + - sanity + required_packages: + - bash + - coreutils + - kernel + - kernel-devel + - kernel-modules + - kernel-abi-stablelists + - xz + - bzip2 + - gzip +