selftest: kill VM reliably by recursively kill children processes

qemu is launched in nested subprocess and can't be killed by simply
killing the job ids,

        PID    Command
    2269634    │  ├─ sshd: root [priv]
    2269637    │  │  └─ sshd: root@pts/0
    2269638    │  │     └─ -bash
    2269744    │  │        └─ make test-run V=1
    2273117    │  │           └─ /bin/bash /root/kexec-tools-300/tests/scripts/run-test.sh
    2273712    │  │              ├─ /bin/bash /root/kexec-tools-300/tests/scripts/run-test.sh
    2273714    │  │              │  └─ /bin/bash /root/kexec-tools-300/tests/scripts/run-test.sh
    2273737    │  │              │     └─ timeout --foreground 10m /root/kexec-tools-300/tests/scripts/run-qemu -nodefaults -nographic -smp 2 -m 768M -monitor no
    2273738    │  │              │        └─ /usr/bin/qemu-system-x86_64 -enable-kvm -cpu host -nodefaults -nographic -smp 2 -m 768M -monitor none -serial stdio
    2273746    │  │              │           ├─ /usr/bin/qemu-system-x86_64 -enable-kvm -cpu host -nodefaults -nographic -smp 2 -m 768M -monitor none -serial std
    2273797    │  │              ├─ /bin/bash /root/kexec-tools-300/tests/scripts/run-test.sh
    2273798    │  │              │  └─ /bin/bash /root/kexec-tools-300/tests/scripts/run-test.sh
    2273831    │  │              │     └─ timeout --foreground 10m /root/kexec-tools-300/tests/scripts/run-qemu -nodefaults -nographic -smp 2 -m 768M -monitor no
    2273832    │  │              │        └─ /usr/bin/qemu-system-x86_64 -enable-kvm -cpu host -nodefaults -nographic -smp 2 -m 768M -monitor none -serial stdio
    2273840    │  │              │           ├─ /usr/bin/qemu-system-x86_64 -enable-kvm -cpu host -nodefaults -nographic -smp 2 -m 768M -monitor none -serial std

This led to the error "qemu-system-x86_64: can't bind ip=0.0.0.0 to
socket: Address already in use".

This patch will kill qemu by killing all the children of the job id.

Signed-off-by: Coiby Xu <coxu@redhat.com>
Acked-by: Kairui Song <kasong@redhat.com>

tests/scripts/run-test.sh

@@ -1,9 +1,38 @@
 #!/bin/bash
 
+_kill_if_valid_pid() {
+	local _pid="$1"
+	if ps -p "$_pid" > /dev/null
+	then
+		kill "$_pid"
+	fi
+}
+
+_recursive_kill() {
+	local _pid="$1"
+	local _children _child
+
+	_children=$(pgrep -P "$_pid")
+	if [ -n "$_children" ]; then
+		for _child in $_children
+		do
+			_recursive_kill "$_child"
+			_kill_if_valid_pid "$_child"
+		done
+	fi
+	_kill_if_valid_pid "$_pid"
+}
+
 _kill_all_jobs() {
 	local _jobs=$(jobs -r -p)
+	local _job
 
-	[ -n "$_jobs" ] && kill $_jobs
+	if [ -n "$_jobs" ]; then
+		for _job in $_jobs
+		do
+			_recursive_kill "$_job"
+		done
+	fi
 }
 
 trap '
@@ -121,7 +150,7 @@ for test_case in $testcases; do
 
 	[ $? -ne 0 ] && ret=$(expr $ret + 1)
 	results[$test_case]="$res"
-
+	_kill_all_jobs
 	echo -e "-------- Test finished: $test_case $res --------"
 	for script in $scripts; do
 		script="$testdir/$script"

tests/scripts/test-lib.sh

@@ -146,8 +146,7 @@ watch_test_outputs() {
 				ret=$?
 
 				if [ $ret -ne 255 ]; then
-					# Test finished, kill VMs
+					# Test finished
-					kill $(jobs -p)
 					break 2
 				fi
 			done