selftest: Add basic test framework

Now, by execute `make test-run` in tests/, kexec-tools sanity tests
will be performed using VMs. There are currently 3 test cases, for local
kdump, nfs kdump and ssh kdump.

For each test VM, the selftest framework will create a snapshot layer,
do setup as required by the test case, this ensure each test runs in a
clean VM.

This framework will install a custom systemd service that starts when
system have finished booting, and the service will do basic routine
(fetch and set boot counter, etc..), then call the test case which is
installed in /kexec-kdump-test/test.sh in VM.

Each VM will have two serial consoles, one for ordinary console usage,
one for the test communication and log. The test script will watch the
second test console to know the test status.

The test cases are located in tests/scripts/testcases, documents about
the test cases structure will be provided in following commits.

Signed-off-by: Kairui Song <kasong@redhat.com>
Acked-by: Dave Young <dyoung@redhat.com>
This commit is contained in:
Kairui Song 2020-07-31 15:34:40 +08:00
parent 2457f22baf
commit a8dbd281f7
15 changed files with 669 additions and 1 deletions

View File

@ -1,3 +1,4 @@
TEST_CASE ?=
BASE_IMAGE ?=
TEST_ROOT := $(shell dirname $(realpath $(firstword $(MAKEFILE_LIST))))
@ -75,6 +76,13 @@ $(TEST_ROOT)/output/test-base-image: $(BUILD_ROOT)/inst-base-image $(KEXEC_TOOLS
$(TEST_ROOT)/scripts/build-scripts/test-base-image.sh \
$(KEXEC_TOOLS_RPM)
test-run: $(TEST_ROOT)/output/test-base-image
ifeq ($(strip $(TEST_CASE)),)
$(TEST_ROOT)/scripts/run-test.sh
else
$(TEST_ROOT)/scripts/run-test.sh --console $(TEST_CASE)
endif
clean:
rm -rf $(TEST_ROOT)/build
rm -rf $(TEST_ROOT)/output

View File

@ -8,4 +8,3 @@ img_inst_pkg grubby\
img_run_cmd "grubby --args systemd.journald.forward_to_console=1 systemd.log_target=console --update-kernel ALL"
img_run_cmd "grubby --args selinux=0 --update-kernel ALL"
img_run_cmd "grubby --args crashkernel=224M --update-kernel ALL"
img_run_cmd "mkdir -p /kexec-kdump-test"

View File

@ -10,6 +10,12 @@ for _rpm in $@; do
fi
done
img_run_cmd "mkdir -p /kexec-kdump-test"
img_inst $TESTDIR/scripts/kexec-kdump-test/init.sh /kexec-kdump-test/init.sh
img_inst $TESTDIR/scripts/kexec-kdump-test/test.sh /kexec-kdump-test/test.sh
img_inst $TESTDIR/scripts/kexec-kdump-test/kexec-kdump-test.service /etc/systemd/system/kexec-kdump-test.service
img_run_cmd "systemctl enable kexec-kdump-test.service"
img_inst_pkg $TEST_RPMS
# Test script should start kdump manually to save time
img_run_cmd "systemctl disable kdump.service"

View File

@ -0,0 +1,21 @@
#!/usr/bin/env bash
. $TESTDIR/scripts/test-lib.sh
TEST_SCRIPT=$1
QEMU_CMD="$DEFAULT_QEMU_CMD \
-serial stdio \
-serial file:$(get_test_output_file $TEST_SCRIPT) \
-monitor none \
-hda $OUTPUT_IMAGE"
img_add_qemu_cmd() {
QEMU_CMD+=" $@"
}
source $TEST_SCRIPT
on_build
img_inst $TEST_SCRIPT /kexec-kdump-test/test.sh
echo $QEMU_CMD > $(get_test_qemu_cmd_file $TEST_SCRIPT)

View File

@ -0,0 +1,112 @@
#!/usr/bin/env sh
BOOT_ARG="test_boot_count"
_YELLOW='\033[1;33m'
_GREEN='\033[0;32m'
_RED='\033[0;31m'
_NC='\033[0m' # No Color
if [ -n "$(cat /proc/cmdline | grep "\bno_test\b")" ]; then
exit 0
fi
get_test_boot_count() {
local boot_count=$(cat /proc/cmdline | sed -n "s/.*$BOOT_ARG=\([0-9]*\).*/\1/p")
if [ -z "$boot_count" ]; then
boot_count=1
fi
echo $boot_count
}
test_output() {
echo $@ > /dev/ttyS1
echo $@ > /dev/ttyS0
sync
}
test_passed() {
echo -e "${_GREEN}TEST PASSED${_NC}" > /dev/ttyS1
echo -e "${_GREEN}kexec-kdump-test: TEST PASSED${_NC}" > /dev/ttyS0
echo $@ > /dev/ttyS1
echo $@ > /dev/ttyS0
sync
shutdown -h 0
exit 0
}
test_failed() {
echo -e "${_RED}TEST FAILED${_NC}" > /dev/ttyS1
echo -e "${_RED}kexec-kdump-test: TEST FAILED${_NC}" > /dev/ttyS0
echo $@ > /dev/ttyS1
echo $@ > /dev/ttyS0
sync
shutdown -h 0
exit 1
}
test_abort() {
echo -e "${_YELLOW}TEST ABORTED${_NC}" > /dev/ttyS1
echo -e "${_YELLOW}kexec-kdump-test: TEST ABORTED${_NC}" > /dev/ttyS0
echo $@ > /dev/ttyS1
echo $@ > /dev/ttyS0
sync
shutdown -h 0
exit 2
}
has_valid_vmcore_dir() {
local path=$1
local vmcore_dir=$path/$(ls -1 $path | tail -n 1)
local vmcore="<invalid>"
# Checking with `crash` is slow and consume a lot of memory/disk,
# just do a sanity check by check if log are available.
if [ -e $vmcore_dir/vmcore ]; then
vmcore=$vmcore_dir/vmcore
makedumpfile --dump-dmesg $vmcore $vmcore_dir/vmcore-dmesg.txt.2 || return 1
elif [ -e $vmcore_dir/vmcore.flat ]; then
vmcore=$vmcore_dir/vmcore.flat
makedumpfile -R $vmcore_dir/vmcore < $vmcore || return 1
makedumpfile --dump-dmesg $vmcore_dir/vmcore $vmcore_dir/vmcore-dmesg.txt.2 || return 1
rm $vmcore_dir/vmcore
else
return 1
fi
if diff $vmcore_dir/vmcore-dmesg.txt.2 $vmcore_dir/vmcore-dmesg.txt; then
return 1
fi
test_output "Found a valid vmcore in \"$vmcore_dir\""
return 0
}
BOOT_COUNT=$(get_test_boot_count)
test_output "Kexec-Kdump-Test Boot #$BOOT_COUNT"
echo 'fedora' | passwd --stdin root
test_output "Updating kernel cmdline"
grubby --update-kernel ALL --args $BOOT_ARG=$(expr $BOOT_COUNT + 1) && sync
test_output "Executing test hook"
source /kexec-kdump-test/test.sh
on_test;
test_output "Test exited, system hang for inspect"

View File

@ -0,0 +1,9 @@
[Unit]
Description=Kexec Kdump Test Service
[Service]
ExecStart=/kexec-kdump-test/init.sh
Type=idle
[Install]
WantedBy=multi-user.target

View File

@ -0,0 +1,15 @@
#!/usr/bin/env sh
# A test example that do nothing
# Executed before VM starts
on_build() {
:
}
# Executed when VM boots
on_test() {
:
# call get_test_boot_count to get boot cound
# call test_passed if test passed
# call test_failed if test passed
}

131
tests/scripts/run-test.sh Executable file
View File

@ -0,0 +1,131 @@
#!/bin/bash
_kill_all_jobs() {
local _jobs=$(jobs -r -p)
[ -n "$_jobs" ] && kill $_jobs
}
trap '
ret=$?;
_kill_all_jobs
exit $ret;
' EXIT
trap 'exit 1;' SIGINT
BASEDIR=$(realpath $(dirname "$0"))
. $BASEDIR/test-lib.sh
TESTCASEDIR="$BASEDIR/testcases"
console=0
testcases=""
while [ $# -gt 0 ]; do
case $1 in
'')
break
;;
--console )
console=1
;;
-*)
echo "Invalid option $1"
;;
*)
testcases+=" $1"
;;
esac
shift;
done
if [ -z "$testcases" ]; then
echo "==== Starting all tests: ===="
testcases=$(ls -1 $TESTCASEDIR)
else
echo "==== Starting specified tests: ===="
fi
echo ${testcases##*/}
echo
declare -A results
ret=0
for test_case in $testcases; do
echo "======== Running Test Case $test_case ========"
results[$test_case]="<Test Skipped>"
testdir=$TESTCASEDIR/$test_case
script_num=$(ls -1 $testdir | wc -l)
scripts=$(ls -r -1 $testdir | tr '\n' ' ')
test_outputs=""
read main_script aux_script <<< "$scripts"
if [ -z "$main_script" ]; then
echo "ERROR: Empty testcase dir $testdir"
continue
fi
for script in $scripts; do
echo "---- Building image for: $script ----"
echo "-------- Output image is: $(get_test_image $testdir/$script)"
echo "-------- Building log is: $(get_test_image $testdir/$script).log"
mkdir -p $(dirname $(get_test_image $testdir/$script))
build_test_image $testdir/$script &> $(get_test_image $testdir/$script).log
if [ $? -ne 0 ]; then
echo "Failing building image!"
continue 2
fi
done
for script in $aux_script; do
echo "---- Starting VM: $script ----"
script="$testdir/$script"
echo "-------- Qemu cmdline: $(get_test_qemu_cmd_file $script)"
echo "-------- Console log: $(get_test_console_file $script)"
echo "-------- Test log: $(get_test_output_file $script)"
test_outputs+="$(get_test_output_file $script) "
rm -f $(get_test_console_file $script)
rm -f $(get_test_output_file $script)
$(run_test_sync $script > $(get_test_console_file $script)) &
sleep 3
done
script="$main_script"
echo "---- Starting test VM: $(basename $script) ----"
script="$testdir/$script"
echo "-------- Qemu cmdline: $(get_test_qemu_cmd_file $script)"
echo "-------- Console log: $(get_test_console_file $script)"
echo "-------- Test log: $(get_test_output_file $script)"
test_outputs+="$(get_test_output_file $script) "
rm -f $(get_test_console_file $script)
rm -f $(get_test_output_file $script)
if [ $console -eq 1 ]; then
run_test_sync $script | tee $(get_test_console_file $script)
[ -n "$(jobs -p)" ] && kill $(jobs -p)
else
$(run_test_sync $script > $(get_test_console_file $script)) &
watch_test_outputs $test_outputs
fi
res="$(gather_test_result $test_outputs)"
[ $? -ne 0 ] && ret=$(expr $ret + 1)
results[$test_case]="$res"
echo -e "-------- Test finished: $test_case $res --------"
done
echo "======== Test results ========"
for i in ${!results[@]}; do
echo "----------------"
echo -e "$i:\t\t${results[$i]}"
done
exit $ret

View File

@ -0,0 +1,16 @@
#!/usr/bin/env bash
BASEDIR=$(realpath $(dirname "$0"))
. $BASEDIR/image-init-lib.sh
# Base image to build from
BOOT_IMAGE=$1
if [[ ! -e $BOOT_IMAGE ]]; then
perror_exit "Image '$BOOT_IMAGE' not found"
else
BOOT_IMAGE=$(realpath "$BOOT_IMAGE")
fi
mount_image $BOOT_IMAGE
shell_in_image $BOOT_IMAGE

177
tests/scripts/test-lib.sh Normal file
View File

@ -0,0 +1,177 @@
#!/usr/bin/env sh
[ -z "$BASEDIR" ] && BASEDIR=$(realpath $(dirname "$0"))
[ -z "$TESTDIR" ] && TESTDIR=$(realpath $BASEDIR/../)
[ -z "$TEST_BASE_IMAGE" ] && TEST_BASE_IMAGE=$TESTDIR/output/test-base-image
[[ ! -e $TEST_BASE_IMAGE ]] && echo "Test base image not found." && exit 1
DEFAULT_QEMU_CMD="-nodefaults \
-nographic \
-smp 2 \
-m 768M \
-monitor none"
_YELLOW='\033[1;33m'
_GREEN='\033[0;32m'
_RED='\033[0;31m'
_NC='\033[0m' # No Color
get_test_path() {
local script=$1
local testname=$(basename $(dirname $script))
local output=$TESTDIR/output/$testname
echo $output
}
get_test_entry_name() {
echo $(basename ${1%.*})
}
get_test_image() {
local script=$1
local testout=$(get_test_path $script)
local entry=$(get_test_entry_name $script)
echo $testout/$entry.img
}
get_test_qemu_cmd_file() {
local script=$1
local testout=$(get_test_path $script)
local entry=$(get_test_entry_name $script)
echo $testout/$entry.qemu_cmd
}
get_test_qemu_cmd() {
cat $(get_test_qemu_cmd_file $1)
}
get_test_output_file() {
local script=$1
local testout=$(get_test_path $script)
local entry=$(get_test_entry_name $script)
echo $testout/$entry.output
}
get_test_console_file() {
local script=$1
local testout=$(get_test_path $script)
local entry=$(get_test_entry_name $script)
echo $testout/$entry.console
}
get_test_output() {
local output=$(get_test_output_file $1)
if [ -e "$output" ]; then
cat $(get_test_output_file $1)
else
echo "<No Output>"
fi
}
build_test_image() {
local script=$1
local test_image=$(get_test_image $script)
mkdir -p $(dirname $test_image)
$BASEDIR/build-image.sh \
$TEST_BASE_IMAGE \
$test_image \
$BASEDIR/build-scripts/test-image.sh \
$script
}
run_test_sync() {
local qemu_cmd=$(get_test_qemu_cmd $1)
if [ -n "$qemu_cmd" ]; then
timeout --foreground 10m qemu-kvm $(get_test_qemu_cmd $1)
else
echo "error: test qemu command line is not configured" > /dev/stderr
return 1
fi
}
_check_test_result() {
grep "TEST PASSED" $1 2>/dev/null
[ $? -eq 0 ] && return 0
grep "TEST FAILED" $1 2>/dev/null
[ $? -eq 0 ] && return 1
grep "TEST ABORTED" $1 2>/dev/null
[ $? -eq 0 ] && return 2
return 255
}
# Print test result and return below value:
# 0: Test passed
# 1: Test failed
# 2: Test aborted, test scripts errored out
# 3: Test exited unexpectely, VM got killed early, or time out
gather_test_result() {
local ret=255
local res=""
for i in $@; do
res=$(_check_test_result $i)
ret=$?
if [ $ret -ne 255 ]; then
echo $res
return $ret
fi
done
echo "${_RED}TEST RESULT NOT FOUND!${_NC}"
return 3
}
# Wait and watch for test result
watch_test_outputs() {
local ret=255
local res=""
# If VMs are still running, check for test result, if
# test finished, kill remaining VMs
while true; do
if [ -n "$(jobs -r)" ]; then
# VMs still running
for i in $@; do
res=$(_check_test_result $i)
ret=$?
if [ $ret -ne 255 ]; then
# Test finished, kill VMs
kill $(jobs -p)
break 2
fi
done
else
# VMs exited
ret=255
for i in $@; do
res=$(_check_test_result $i)
ret=$?
if [ $ret -ne 255 ]; then
break 2
fi
done
if [ $ret -eq 255 ]; then
ret=3
break
fi
fi
sleep 1
done
return $ret
}

View File

@ -0,0 +1,32 @@
on_build() {
:
}
on_test() {
local boot_count=$(get_test_boot_count)
if [ $boot_count -eq 1 ]; then
cat << EOF > /etc/kdump.conf
path /var/crash
core_collector makedumpfile -l --message-level 1 -d 31
EOF
kdumpctl start || test_failed "Failed to start kdump"
sync
echo 1 > /proc/sys/kernel/sysrq
echo c > /proc/sysrq-trigger
elif [ $boot_count -eq 2 ]; then
if has_valid_vmcore_dir /var/crash; then
test_passed
else
test_failed
fi
shutdown -h 0
else
test_failed "Unexpected reboot"
fi
}

View File

@ -0,0 +1,38 @@
#!/usr/bin/env sh
# Executed before VM starts
on_build() {
img_inst_pkg "nfs-utils dnsmasq"
img_run_cmd "mkdir -p /srv/nfs/var/crash"
img_run_cmd "echo /srv/nfs 192.168.77.1/24\(rw,async,insecure,no_root_squash\) > /etc/exports"
img_run_cmd "systemctl enable nfs-server"
img_run_cmd "echo interface=eth0 > /etc/dnsmasq.conf"
img_run_cmd "echo dhcp-authoritative >> /etc/dnsmasq.conf"
img_run_cmd "echo dhcp-range=192.168.77.50,192.168.77.100,255.255.255.0,12h >> /etc/dnsmasq.conf"
img_run_cmd "systemctl enable dnsmasq"
img_run_cmd 'echo DEVICE="eth0" > /etc/sysconfig/network-scripts/ifcfg-eth0'
img_run_cmd 'echo BOOTPROTO="none >> /etc/sysconfig/network-scripts/ifcfg-eth0"'
img_run_cmd 'echo ONBOOT="yes" >> /etc/sysconfig/network-scripts/ifcfg-eth0'
img_run_cmd 'echo PREFIX="24" >> /etc/sysconfig/network-scripts/ifcfg-eth0'
img_run_cmd 'echo IPADDR="192.168.77.1" >> /etc/sysconfig/network-scripts/ifcfg-eth0'
img_run_cmd 'echo TYPE="Ethernet" >> /etc/sysconfig/network-scripts/ifcfg-eth0'
img_add_qemu_cmd "-nic socket,listen=:8010,mac=52:54:00:12:34:56"
}
# Executed when VM boots
on_test() {
while true; do
if has_valid_vmcore_dir /srv/nfs/var/crash; then
# Wait a few seconds so client finish it's work to generate a full log
sleep 5
test_passed
fi
sleep 1
done
}

View File

@ -0,0 +1,30 @@
# Executed before VM starts
on_build() {
img_inst_pkg "nfs-utils"
img_add_qemu_cmd "-nic socket,connect=127.0.0.1:8010,mac=52:54:00:12:34:57"
}
on_test() {
local boot_count=$(get_test_boot_count)
local nfs_server=192.168.77.1
if [ "$boot_count" -eq 1 ]; then
cat << EOF > /etc/kdump.conf
nfs $nfs_server:/srv/nfs
core_collector makedumpfile -l --message-level 1 -d 31
EOF
while ! ping -c 1 $nfs_server -W 1; do
:
done
kdumpctl start || test_failed "Failed to start kdump"
sync
echo 1 > /proc/sys/kernel/sysrq
echo c > /proc/sysrq-trigger
else
shutdown -h 0
fi
}

View File

@ -0,0 +1,34 @@
#!/usr/bin/env sh
# Executed before VM starts
on_build() {
img_add_qemu_cmd "-nic socket,listen=:8010,mac=52:54:00:12:34:56"
img_run_cmd "echo root:fedora | chpasswd"
img_run_cmd 'sed -i "s/^.*PasswordAuthentication .*\$/PasswordAuthentication yes/" /etc/ssh/sshd_config'
img_run_cmd 'sed -i "s/^.*PermitRootLogin .*\$/PermitRootLogin yes/" /etc/ssh/sshd_config'
img_run_cmd "systemctl enable sshd"
img_run_cmd "echo interface=eth0 > /etc/dnsmasq.conf"
img_run_cmd "echo dhcp-authoritative >> /etc/dnsmasq.conf"
img_run_cmd "echo dhcp-range=192.168.77.50,192.168.77.100,255.255.255.0,12h >> /etc/dnsmasq.conf"
img_run_cmd "systemctl enable dnsmasq"
img_run_cmd 'echo DEVICE="eth0" > /etc/sysconfig/network-scripts/ifcfg-eth0'
img_run_cmd 'echo BOOTPROTO="none >> /etc/sysconfig/network-scripts/ifcfg-eth0"'
img_run_cmd 'echo ONBOOT="yes" >> /etc/sysconfig/network-scripts/ifcfg-eth0'
img_run_cmd 'echo PREFIX="24" >> /etc/sysconfig/network-scripts/ifcfg-eth0'
img_run_cmd 'echo IPADDR="192.168.77.1" >> /etc/sysconfig/network-scripts/ifcfg-eth0'
img_run_cmd 'echo TYPE="Ethernet" >> /etc/sysconfig/network-scripts/ifcfg-eth0'
}
# Executed when VM boots
on_test() {
while true; do
if has_valid_vmcore_dir /var/crash; then
test_passed
fi
sleep 1
done
}

View File

@ -0,0 +1,40 @@
# Executed before VM starts
on_build() {
img_inst_pkg "sshpass"
img_add_qemu_cmd "-nic socket,connect=127.0.0.1:8010,mac=52:54:00:12:34:57"
}
on_test() {
local boot_count=$(get_test_boot_count)
local ssh_server=192.168.77.1
if [ "$boot_count" -eq 1 ]; then
cat << EOF > /etc/kdump.conf
ssh root@192.168.77.1
core_collector makedumpfile -l --message-level 1 -d 31 -F
EOF
ssh-keygen -q -t rsa -N '' -f /root/.ssh/id_rsa <<< y &>/dev/ttyS1
while ! ping -c 1 $ssh_server -W 1; do
sleep 1
done
while [ -z "$(cat /root/.ssh/known_hosts)" ]; do
ssh-keyscan -H 192.168.77.1 > /root/.ssh/known_hosts
done
sshpass -p fedora ssh-copy-id root@$ssh_server -f &>/dev/ttyS1
sshpass -p fedora kdumpctl propagate &>/dev/ttyS1
kdumpctl start || test_failed "Failed to start kdump"
sync
echo 1 > /proc/sys/kernel/sysrq
echo c > /proc/sysrq-trigger
else
shutdown -h 0
fi
}