diff --git a/tests/Makefile b/tests/Makefile index a352ab3..6144809 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -1,3 +1,4 @@ +TEST_CASE ?= BASE_IMAGE ?= TEST_ROOT := $(shell dirname $(realpath $(firstword $(MAKEFILE_LIST)))) @@ -75,6 +76,13 @@ $(TEST_ROOT)/output/test-base-image: $(BUILD_ROOT)/inst-base-image $(KEXEC_TOOLS $(TEST_ROOT)/scripts/build-scripts/test-base-image.sh \ $(KEXEC_TOOLS_RPM) +test-run: $(TEST_ROOT)/output/test-base-image +ifeq ($(strip $(TEST_CASE)),) + $(TEST_ROOT)/scripts/run-test.sh +else + $(TEST_ROOT)/scripts/run-test.sh --console $(TEST_CASE) +endif + clean: rm -rf $(TEST_ROOT)/build rm -rf $(TEST_ROOT)/output diff --git a/tests/scripts/build-scripts/base-image.sh b/tests/scripts/build-scripts/base-image.sh index 59f4574..62624e4 100755 --- a/tests/scripts/build-scripts/base-image.sh +++ b/tests/scripts/build-scripts/base-image.sh @@ -8,4 +8,3 @@ img_inst_pkg grubby\ img_run_cmd "grubby --args systemd.journald.forward_to_console=1 systemd.log_target=console --update-kernel ALL" img_run_cmd "grubby --args selinux=0 --update-kernel ALL" img_run_cmd "grubby --args crashkernel=224M --update-kernel ALL" -img_run_cmd "mkdir -p /kexec-kdump-test" diff --git a/tests/scripts/build-scripts/test-base-image.sh b/tests/scripts/build-scripts/test-base-image.sh index a649306..5e2d86a 100755 --- a/tests/scripts/build-scripts/test-base-image.sh +++ b/tests/scripts/build-scripts/test-base-image.sh @@ -10,6 +10,12 @@ for _rpm in $@; do fi done +img_run_cmd "mkdir -p /kexec-kdump-test" +img_inst $TESTDIR/scripts/kexec-kdump-test/init.sh /kexec-kdump-test/init.sh +img_inst $TESTDIR/scripts/kexec-kdump-test/test.sh /kexec-kdump-test/test.sh +img_inst $TESTDIR/scripts/kexec-kdump-test/kexec-kdump-test.service /etc/systemd/system/kexec-kdump-test.service +img_run_cmd "systemctl enable kexec-kdump-test.service" + img_inst_pkg $TEST_RPMS # Test script should start kdump manually to save time img_run_cmd "systemctl disable kdump.service" diff --git a/tests/scripts/build-scripts/test-image.sh b/tests/scripts/build-scripts/test-image.sh new file mode 100755 index 0000000..d8e907e --- /dev/null +++ b/tests/scripts/build-scripts/test-image.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash +. $TESTDIR/scripts/test-lib.sh +TEST_SCRIPT=$1 + +QEMU_CMD="$DEFAULT_QEMU_CMD \ +-serial stdio \ +-serial file:$(get_test_output_file $TEST_SCRIPT) \ +-monitor none \ +-hda $OUTPUT_IMAGE" + +img_add_qemu_cmd() { + QEMU_CMD+=" $@" +} + +source $TEST_SCRIPT + +on_build + +img_inst $TEST_SCRIPT /kexec-kdump-test/test.sh + +echo $QEMU_CMD > $(get_test_qemu_cmd_file $TEST_SCRIPT) diff --git a/tests/scripts/kexec-kdump-test/init.sh b/tests/scripts/kexec-kdump-test/init.sh new file mode 100755 index 0000000..bc2a642 --- /dev/null +++ b/tests/scripts/kexec-kdump-test/init.sh @@ -0,0 +1,112 @@ +#!/usr/bin/env sh +BOOT_ARG="test_boot_count" +_YELLOW='\033[1;33m' +_GREEN='\033[0;32m' +_RED='\033[0;31m' +_NC='\033[0m' # No Color + +if [ -n "$(cat /proc/cmdline | grep "\bno_test\b")" ]; then + exit 0 +fi + +get_test_boot_count() { + local boot_count=$(cat /proc/cmdline | sed -n "s/.*$BOOT_ARG=\([0-9]*\).*/\1/p") + + if [ -z "$boot_count" ]; then + boot_count=1 + fi + + echo $boot_count +} + +test_output() { + echo $@ > /dev/ttyS1 + echo $@ > /dev/ttyS0 + + sync +} + +test_passed() { + echo -e "${_GREEN}TEST PASSED${_NC}" > /dev/ttyS1 + echo -e "${_GREEN}kexec-kdump-test: TEST PASSED${_NC}" > /dev/ttyS0 + + echo $@ > /dev/ttyS1 + echo $@ > /dev/ttyS0 + + sync + + shutdown -h 0 + + exit 0 +} + +test_failed() { + echo -e "${_RED}TEST FAILED${_NC}" > /dev/ttyS1 + echo -e "${_RED}kexec-kdump-test: TEST FAILED${_NC}" > /dev/ttyS0 + + echo $@ > /dev/ttyS1 + echo $@ > /dev/ttyS0 + + sync + + shutdown -h 0 + + exit 1 +} + +test_abort() { + echo -e "${_YELLOW}TEST ABORTED${_NC}" > /dev/ttyS1 + echo -e "${_YELLOW}kexec-kdump-test: TEST ABORTED${_NC}" > /dev/ttyS0 + + echo $@ > /dev/ttyS1 + echo $@ > /dev/ttyS0 + + sync + + shutdown -h 0 + + exit 2 +} + +has_valid_vmcore_dir() { + local path=$1 + local vmcore_dir=$path/$(ls -1 $path | tail -n 1) + local vmcore="" + + # Checking with `crash` is slow and consume a lot of memory/disk, + # just do a sanity check by check if log are available. + if [ -e $vmcore_dir/vmcore ]; then + vmcore=$vmcore_dir/vmcore + makedumpfile --dump-dmesg $vmcore $vmcore_dir/vmcore-dmesg.txt.2 || return 1 + elif [ -e $vmcore_dir/vmcore.flat ]; then + vmcore=$vmcore_dir/vmcore.flat + makedumpfile -R $vmcore_dir/vmcore < $vmcore || return 1 + makedumpfile --dump-dmesg $vmcore_dir/vmcore $vmcore_dir/vmcore-dmesg.txt.2 || return 1 + rm $vmcore_dir/vmcore + else + return 1 + fi + + if diff $vmcore_dir/vmcore-dmesg.txt.2 $vmcore_dir/vmcore-dmesg.txt; then + return 1 + fi + + test_output "Found a valid vmcore in \"$vmcore_dir\"" + + return 0 +} + +BOOT_COUNT=$(get_test_boot_count) +test_output "Kexec-Kdump-Test Boot #$BOOT_COUNT" + +echo 'fedora' | passwd --stdin root + +test_output "Updating kernel cmdline" +grubby --update-kernel ALL --args $BOOT_ARG=$(expr $BOOT_COUNT + 1) && sync + +test_output "Executing test hook" +source /kexec-kdump-test/test.sh + +on_test; + +test_output "Test exited, system hang for inspect" diff --git a/tests/scripts/kexec-kdump-test/kexec-kdump-test.service b/tests/scripts/kexec-kdump-test/kexec-kdump-test.service new file mode 100644 index 0000000..ba7b11e --- /dev/null +++ b/tests/scripts/kexec-kdump-test/kexec-kdump-test.service @@ -0,0 +1,9 @@ +[Unit] +Description=Kexec Kdump Test Service + +[Service] +ExecStart=/kexec-kdump-test/init.sh +Type=idle + +[Install] +WantedBy=multi-user.target diff --git a/tests/scripts/kexec-kdump-test/test.sh b/tests/scripts/kexec-kdump-test/test.sh new file mode 100755 index 0000000..66ac15d --- /dev/null +++ b/tests/scripts/kexec-kdump-test/test.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env sh +# A test example that do nothing + +# Executed before VM starts +on_build() { + : +} + +# Executed when VM boots +on_test() { + : + # call get_test_boot_count to get boot cound + # call test_passed if test passed + # call test_failed if test passed +} diff --git a/tests/scripts/run-test.sh b/tests/scripts/run-test.sh new file mode 100755 index 0000000..cdba7f4 --- /dev/null +++ b/tests/scripts/run-test.sh @@ -0,0 +1,131 @@ +#!/bin/bash + +_kill_all_jobs() { + local _jobs=$(jobs -r -p) + + [ -n "$_jobs" ] && kill $_jobs +} + +trap ' +ret=$?; +_kill_all_jobs +exit $ret; +' EXIT + +trap 'exit 1;' SIGINT + +BASEDIR=$(realpath $(dirname "$0")) +. $BASEDIR/test-lib.sh +TESTCASEDIR="$BASEDIR/testcases" + +console=0 +testcases="" + +while [ $# -gt 0 ]; do + case $1 in + '') + break + ;; + --console ) + console=1 + ;; + -*) + echo "Invalid option $1" + ;; + *) + testcases+=" $1" + ;; + esac + shift; +done + +if [ -z "$testcases" ]; then + echo "==== Starting all tests: ====" + testcases=$(ls -1 $TESTCASEDIR) +else + echo "==== Starting specified tests: ====" +fi +echo ${testcases##*/} +echo + +declare -A results +ret=0 + +for test_case in $testcases; do + echo "======== Running Test Case $test_case ========" + results[$test_case]="" + + testdir=$TESTCASEDIR/$test_case + script_num=$(ls -1 $testdir | wc -l) + scripts=$(ls -r -1 $testdir | tr '\n' ' ') + test_outputs="" + read main_script aux_script <<< "$scripts" + + if [ -z "$main_script" ]; then + echo "ERROR: Empty testcase dir $testdir" + continue + fi + + for script in $scripts; do + echo "---- Building image for: $script ----" + echo "-------- Output image is: $(get_test_image $testdir/$script)" + echo "-------- Building log is: $(get_test_image $testdir/$script).log" + + mkdir -p $(dirname $(get_test_image $testdir/$script)) + build_test_image $testdir/$script &> $(get_test_image $testdir/$script).log + + if [ $? -ne 0 ]; then + echo "Failing building image!" + continue 2 + fi + done + + for script in $aux_script; do + echo "---- Starting VM: $script ----" + + script="$testdir/$script" + echo "-------- Qemu cmdline: $(get_test_qemu_cmd_file $script)" + echo "-------- Console log: $(get_test_console_file $script)" + echo "-------- Test log: $(get_test_output_file $script)" + test_outputs+="$(get_test_output_file $script) " + rm -f $(get_test_console_file $script) + rm -f $(get_test_output_file $script) + + $(run_test_sync $script > $(get_test_console_file $script)) & + + sleep 3 + done + + script="$main_script" + echo "---- Starting test VM: $(basename $script) ----" + script="$testdir/$script" + + echo "-------- Qemu cmdline: $(get_test_qemu_cmd_file $script)" + echo "-------- Console log: $(get_test_console_file $script)" + echo "-------- Test log: $(get_test_output_file $script)" + test_outputs+="$(get_test_output_file $script) " + rm -f $(get_test_console_file $script) + rm -f $(get_test_output_file $script) + + if [ $console -eq 1 ]; then + run_test_sync $script | tee $(get_test_console_file $script) + [ -n "$(jobs -p)" ] && kill $(jobs -p) + else + $(run_test_sync $script > $(get_test_console_file $script)) & + watch_test_outputs $test_outputs + fi + + res="$(gather_test_result $test_outputs)" + [ $? -ne 0 ] && ret=$(expr $ret + 1) + results[$test_case]="$res" + + echo -e "-------- Test finished: $test_case $res --------" +done + +echo "======== Test results ========" +for i in ${!results[@]}; do + echo "----------------" + echo -e "$i:\t\t${results[$i]}" +done + +exit $ret diff --git a/tests/scripts/spawn-image-shell.sh b/tests/scripts/spawn-image-shell.sh new file mode 100755 index 0000000..ccfb655 --- /dev/null +++ b/tests/scripts/spawn-image-shell.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash + +BASEDIR=$(realpath $(dirname "$0")) +. $BASEDIR/image-init-lib.sh + +# Base image to build from +BOOT_IMAGE=$1 +if [[ ! -e $BOOT_IMAGE ]]; then + perror_exit "Image '$BOOT_IMAGE' not found" +else + BOOT_IMAGE=$(realpath "$BOOT_IMAGE") +fi + +mount_image $BOOT_IMAGE + +shell_in_image $BOOT_IMAGE diff --git a/tests/scripts/test-lib.sh b/tests/scripts/test-lib.sh new file mode 100644 index 0000000..aff5cd3 --- /dev/null +++ b/tests/scripts/test-lib.sh @@ -0,0 +1,177 @@ +#!/usr/bin/env sh +[ -z "$BASEDIR" ] && BASEDIR=$(realpath $(dirname "$0")) +[ -z "$TESTDIR" ] && TESTDIR=$(realpath $BASEDIR/../) +[ -z "$TEST_BASE_IMAGE" ] && TEST_BASE_IMAGE=$TESTDIR/output/test-base-image + +[[ ! -e $TEST_BASE_IMAGE ]] && echo "Test base image not found." && exit 1 + +DEFAULT_QEMU_CMD="-nodefaults \ +-nographic \ +-smp 2 \ +-m 768M \ +-monitor none" + +_YELLOW='\033[1;33m' +_GREEN='\033[0;32m' +_RED='\033[0;31m' +_NC='\033[0m' # No Color + +get_test_path() { + local script=$1 + local testname=$(basename $(dirname $script)) + local output=$TESTDIR/output/$testname + + echo $output +} + +get_test_entry_name() { + echo $(basename ${1%.*}) +} + +get_test_image() { + local script=$1 + local testout=$(get_test_path $script) + local entry=$(get_test_entry_name $script) + + echo $testout/$entry.img +} + +get_test_qemu_cmd_file() { + local script=$1 + local testout=$(get_test_path $script) + local entry=$(get_test_entry_name $script) + + echo $testout/$entry.qemu_cmd +} + +get_test_qemu_cmd() { + cat $(get_test_qemu_cmd_file $1) +} + +get_test_output_file() { + local script=$1 + local testout=$(get_test_path $script) + local entry=$(get_test_entry_name $script) + + echo $testout/$entry.output +} + +get_test_console_file() { + local script=$1 + local testout=$(get_test_path $script) + local entry=$(get_test_entry_name $script) + + echo $testout/$entry.console +} + +get_test_output() { + local output=$(get_test_output_file $1) + if [ -e "$output" ]; then + cat $(get_test_output_file $1) + else + echo "" + fi +} + +build_test_image() { + local script=$1 + local test_image=$(get_test_image $script) + mkdir -p $(dirname $test_image) + + $BASEDIR/build-image.sh \ + $TEST_BASE_IMAGE \ + $test_image \ + $BASEDIR/build-scripts/test-image.sh \ + $script +} + +run_test_sync() { + local qemu_cmd=$(get_test_qemu_cmd $1) + + if [ -n "$qemu_cmd" ]; then + timeout --foreground 10m qemu-kvm $(get_test_qemu_cmd $1) + else + echo "error: test qemu command line is not configured" > /dev/stderr + return 1 + fi +} + +_check_test_result() { + grep "TEST PASSED" $1 2>/dev/null + [ $? -eq 0 ] && return 0 + + grep "TEST FAILED" $1 2>/dev/null + [ $? -eq 0 ] && return 1 + + grep "TEST ABORTED" $1 2>/dev/null + [ $? -eq 0 ] && return 2 + + return 255 +} + +# Print test result and return below value: +# 0: Test passed +# 1: Test failed +# 2: Test aborted, test scripts errored out +# 3: Test exited unexpectely, VM got killed early, or time out +gather_test_result() { + local ret=255 + local res="" + + for i in $@; do + res=$(_check_test_result $i) + ret=$? + + if [ $ret -ne 255 ]; then + echo $res + return $ret + fi + done + + echo "${_RED}TEST RESULT NOT FOUND!${_NC}" + return 3 +} + +# Wait and watch for test result +watch_test_outputs() { + local ret=255 + local res="" + # If VMs are still running, check for test result, if + # test finished, kill remaining VMs + while true; do + if [ -n "$(jobs -r)" ]; then + # VMs still running + for i in $@; do + res=$(_check_test_result $i) + ret=$? + + if [ $ret -ne 255 ]; then + # Test finished, kill VMs + kill $(jobs -p) + break 2 + fi + done + else + # VMs exited + ret=255 + + for i in $@; do + res=$(_check_test_result $i) + ret=$? + + if [ $ret -ne 255 ]; then + break 2 + fi + done + + if [ $ret -eq 255 ]; then + ret=3 + break + fi + fi + + sleep 1 + done + + return $ret +} diff --git a/tests/scripts/testcases/local-kdump/0-local.sh b/tests/scripts/testcases/local-kdump/0-local.sh new file mode 100755 index 0000000..d09c6f0 --- /dev/null +++ b/tests/scripts/testcases/local-kdump/0-local.sh @@ -0,0 +1,32 @@ +on_build() { + : +} + +on_test() { + local boot_count=$(get_test_boot_count) + + if [ $boot_count -eq 1 ]; then + cat << EOF > /etc/kdump.conf +path /var/crash +core_collector makedumpfile -l --message-level 1 -d 31 +EOF + kdumpctl start || test_failed "Failed to start kdump" + + sync + + echo 1 > /proc/sys/kernel/sysrq + echo c > /proc/sysrq-trigger + + elif [ $boot_count -eq 2 ]; then + + if has_valid_vmcore_dir /var/crash; then + test_passed + else + test_failed + fi + + shutdown -h 0 + else + test_failed "Unexpected reboot" + fi +} diff --git a/tests/scripts/testcases/nfs-kdump/0-server.sh b/tests/scripts/testcases/nfs-kdump/0-server.sh new file mode 100755 index 0000000..04f102e --- /dev/null +++ b/tests/scripts/testcases/nfs-kdump/0-server.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env sh + +# Executed before VM starts +on_build() { + img_inst_pkg "nfs-utils dnsmasq" + + img_run_cmd "mkdir -p /srv/nfs/var/crash" + img_run_cmd "echo /srv/nfs 192.168.77.1/24\(rw,async,insecure,no_root_squash\) > /etc/exports" + img_run_cmd "systemctl enable nfs-server" + + img_run_cmd "echo interface=eth0 > /etc/dnsmasq.conf" + img_run_cmd "echo dhcp-authoritative >> /etc/dnsmasq.conf" + img_run_cmd "echo dhcp-range=192.168.77.50,192.168.77.100,255.255.255.0,12h >> /etc/dnsmasq.conf" + img_run_cmd "systemctl enable dnsmasq" + + img_run_cmd 'echo DEVICE="eth0" > /etc/sysconfig/network-scripts/ifcfg-eth0' + img_run_cmd 'echo BOOTPROTO="none >> /etc/sysconfig/network-scripts/ifcfg-eth0"' + img_run_cmd 'echo ONBOOT="yes" >> /etc/sysconfig/network-scripts/ifcfg-eth0' + img_run_cmd 'echo PREFIX="24" >> /etc/sysconfig/network-scripts/ifcfg-eth0' + img_run_cmd 'echo IPADDR="192.168.77.1" >> /etc/sysconfig/network-scripts/ifcfg-eth0' + img_run_cmd 'echo TYPE="Ethernet" >> /etc/sysconfig/network-scripts/ifcfg-eth0' + + img_add_qemu_cmd "-nic socket,listen=:8010,mac=52:54:00:12:34:56" +} + +# Executed when VM boots +on_test() { + while true; do + if has_valid_vmcore_dir /srv/nfs/var/crash; then + # Wait a few seconds so client finish it's work to generate a full log + sleep 5 + + test_passed + fi + + sleep 1 + done +} diff --git a/tests/scripts/testcases/nfs-kdump/1-client.sh b/tests/scripts/testcases/nfs-kdump/1-client.sh new file mode 100755 index 0000000..df62463 --- /dev/null +++ b/tests/scripts/testcases/nfs-kdump/1-client.sh @@ -0,0 +1,30 @@ +# Executed before VM starts +on_build() { + img_inst_pkg "nfs-utils" + img_add_qemu_cmd "-nic socket,connect=127.0.0.1:8010,mac=52:54:00:12:34:57" +} + +on_test() { + local boot_count=$(get_test_boot_count) + local nfs_server=192.168.77.1 + + if [ "$boot_count" -eq 1 ]; then + cat << EOF > /etc/kdump.conf +nfs $nfs_server:/srv/nfs +core_collector makedumpfile -l --message-level 1 -d 31 +EOF + + while ! ping -c 1 $nfs_server -W 1; do + : + done + + kdumpctl start || test_failed "Failed to start kdump" + + sync + + echo 1 > /proc/sys/kernel/sysrq + echo c > /proc/sysrq-trigger + else + shutdown -h 0 + fi +} diff --git a/tests/scripts/testcases/ssh-kdump/0-server.sh b/tests/scripts/testcases/ssh-kdump/0-server.sh new file mode 100755 index 0000000..f1e5073 --- /dev/null +++ b/tests/scripts/testcases/ssh-kdump/0-server.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env sh + +# Executed before VM starts +on_build() { + img_add_qemu_cmd "-nic socket,listen=:8010,mac=52:54:00:12:34:56" + + img_run_cmd "echo root:fedora | chpasswd" + img_run_cmd 'sed -i "s/^.*PasswordAuthentication .*\$/PasswordAuthentication yes/" /etc/ssh/sshd_config' + img_run_cmd 'sed -i "s/^.*PermitRootLogin .*\$/PermitRootLogin yes/" /etc/ssh/sshd_config' + img_run_cmd "systemctl enable sshd" + + img_run_cmd "echo interface=eth0 > /etc/dnsmasq.conf" + img_run_cmd "echo dhcp-authoritative >> /etc/dnsmasq.conf" + img_run_cmd "echo dhcp-range=192.168.77.50,192.168.77.100,255.255.255.0,12h >> /etc/dnsmasq.conf" + img_run_cmd "systemctl enable dnsmasq" + + img_run_cmd 'echo DEVICE="eth0" > /etc/sysconfig/network-scripts/ifcfg-eth0' + img_run_cmd 'echo BOOTPROTO="none >> /etc/sysconfig/network-scripts/ifcfg-eth0"' + img_run_cmd 'echo ONBOOT="yes" >> /etc/sysconfig/network-scripts/ifcfg-eth0' + img_run_cmd 'echo PREFIX="24" >> /etc/sysconfig/network-scripts/ifcfg-eth0' + img_run_cmd 'echo IPADDR="192.168.77.1" >> /etc/sysconfig/network-scripts/ifcfg-eth0' + img_run_cmd 'echo TYPE="Ethernet" >> /etc/sysconfig/network-scripts/ifcfg-eth0' +} + +# Executed when VM boots +on_test() { + while true; do + if has_valid_vmcore_dir /var/crash; then + test_passed + fi + + sleep 1 + done +} diff --git a/tests/scripts/testcases/ssh-kdump/1-client.sh b/tests/scripts/testcases/ssh-kdump/1-client.sh new file mode 100755 index 0000000..d79f00f --- /dev/null +++ b/tests/scripts/testcases/ssh-kdump/1-client.sh @@ -0,0 +1,40 @@ +# Executed before VM starts +on_build() { + img_inst_pkg "sshpass" + img_add_qemu_cmd "-nic socket,connect=127.0.0.1:8010,mac=52:54:00:12:34:57" +} + +on_test() { + local boot_count=$(get_test_boot_count) + local ssh_server=192.168.77.1 + + if [ "$boot_count" -eq 1 ]; then +cat << EOF > /etc/kdump.conf +ssh root@192.168.77.1 +core_collector makedumpfile -l --message-level 1 -d 31 -F +EOF + + ssh-keygen -q -t rsa -N '' -f /root/.ssh/id_rsa <<< y &>/dev/ttyS1 + + while ! ping -c 1 $ssh_server -W 1; do + sleep 1 + done + + while [ -z "$(cat /root/.ssh/known_hosts)" ]; do + ssh-keyscan -H 192.168.77.1 > /root/.ssh/known_hosts + done + + sshpass -p fedora ssh-copy-id root@$ssh_server -f &>/dev/ttyS1 + + sshpass -p fedora kdumpctl propagate &>/dev/ttyS1 + + kdumpctl start || test_failed "Failed to start kdump" + + sync + + echo 1 > /proc/sys/kernel/sysrq + echo c > /proc/sysrq-trigger + else + shutdown -h 0 + fi +}