002337c671
Now upon failure kdump script might not be called at all and it might not be able to execute default action. It results in a hang. Because we disable emergency shell and rely on kdump.sh being invoked through dracut-pre-pivot hook. But it might happen that we never call into dracut-pre-pivot hook because certain systemd targets could not reach due to failure in their dependencies. In those cases error handling code does not run and system hangs. For example: sysroot-var-crash.mount --> initrd-root-fs.target --> initrd.target \ --> dracut-pre-pivot.service --> kdump.sh If /sysroot/var/crash mount fails, initrd-root-fs.target will not be reached. And then initrd.target will not be reached, dracut-pre-pivot.service wouldn't run. Finally kdump.sh wouldn't run. To solve this problem, we need to separate the error handling code from dracut-pre-pivot hook, and every time when a failure shows up, the separated code can be called by the emergency service. By default systemd provides an emergency service which will drop us into shell every time upon a critical failure. It's very convenient for us to re-use the framework of systemd emergency, because we don't have to touch the other parts of systemd. We can use our own script instead of the default one. This new scheme will overwrite emergency shell and replace with kdump error handling code. And this code will do the error handling as needed. Now, we will not rely on dracut-pre-pivot hook running always. Instead whenever error happens and it is serious enough that emergency shell needed to run, now kdump error handler will run. dracut-emergency is also replaced by kdump error handler and it's enabled again all the way down. So all the failure (including systemd and dracut) in 2nd kernel could be captured, and trigger kdump error handler. dracut-initqueue is a special case, which calls "systemctl start emergency" directly, not via "OnFailure=emergency". In case of failure, emergency is started, but not in a isolation mode, which means dracut-initqueue is still running. On the other hand, emergency will call dracut-initqueue again when default action is dump_to_rootfs. systemd would block on the last dracut-initqueue, waiting for the first instance to exit, which leaves us hang. It looks like the following: dracut-initqueue (running) --> call dracut-emergency: --> dracut-emergency (running) --> kdump-error-handler.sh (running) --> call dracut-initqueue: --> blocking and waiting for the original instance to exit. To fix this, I'd like to introduce a wrapper emergency service. This emegency service will replace both the systemd and dracut emergency. And this service does nothing but to isolate to real kdump error handler service: dracut-initqueue (running) --> call dracut-emergency: --> dracut-emergency isolate to kdump-error-handler.service --> dracut-emergency and dracut-initqueue will both be stopped and kdump-error-handler.service will run kdump-error-handler.sh. In a normal failure case, this still works: foo.service fails --> trigger emergency.service --> emergency.service isolates to kdump-error-handler.service --> kdump-error-handler.service will run kdump-error-handler.sh Signed-off-by: WANG Chao <chaowang@redhat.com> Acked-by: Vivek Goyal <vgoyal@redhat.com> Acked-by: Dave Young <dyoung@redhat.com>
209 lines
4.7 KiB
Bash
Executable File
209 lines
4.7 KiB
Bash
Executable File
#!/bin/sh
|
|
|
|
# continue here only if we have to save dump.
|
|
if [ -f /etc/fadump.initramfs ] && [ ! -f /proc/device-tree/rtas/ibm,kernel-dump ]; then
|
|
return
|
|
fi
|
|
|
|
exec &> /dev/console
|
|
. /lib/dracut-lib.sh
|
|
. /lib/kdump-lib-initramfs.sh
|
|
|
|
set -o pipefail
|
|
DUMP_RETVAL=0
|
|
|
|
export PATH=$PATH:$KDUMP_SCRIPT_DIR
|
|
|
|
do_dump()
|
|
{
|
|
local _ret
|
|
|
|
eval $DUMP_INSTRUCTION
|
|
_ret=$?
|
|
|
|
if [ $_ret -ne 0 ]; then
|
|
echo "kdump: saving vmcore failed"
|
|
fi
|
|
|
|
return $_ret
|
|
}
|
|
|
|
do_kdump_pre()
|
|
{
|
|
if [ -n "$KDUMP_PRE" ]; then
|
|
"$KDUMP_PRE"
|
|
fi
|
|
}
|
|
|
|
do_kdump_post()
|
|
{
|
|
if [ -n "$KDUMP_POST" ]; then
|
|
"$KDUMP_POST" "$1"
|
|
fi
|
|
}
|
|
|
|
add_dump_code()
|
|
{
|
|
DUMP_INSTRUCTION=$1
|
|
}
|
|
|
|
dump_raw()
|
|
{
|
|
local _raw=$1
|
|
|
|
[ -b "$_raw" ] || return 1
|
|
|
|
echo "kdump: saving to raw disk $_raw"
|
|
|
|
if ! $(echo -n $CORE_COLLECTOR|grep -q makedumpfile); then
|
|
_src_size=`ls -l /proc/vmcore | cut -d' ' -f5`
|
|
_src_size_mb=$(($_src_size / 1048576))
|
|
monitor_dd_progress $_src_size_mb &
|
|
fi
|
|
|
|
echo "kdump: saving vmcore"
|
|
$CORE_COLLECTOR /proc/vmcore | dd of=$_raw bs=$DD_BLKSIZE >> /tmp/dd_progress_file 2>&1 || return 1
|
|
sync
|
|
|
|
echo "kdump: saving vmcore complete"
|
|
return 0
|
|
}
|
|
|
|
dump_ssh()
|
|
{
|
|
local _opt="-i $1 -o BatchMode=yes -o StrictHostKeyChecking=yes"
|
|
local _dir="$KDUMP_PATH/$HOST_IP-$DATEDIR"
|
|
local _host=$2
|
|
|
|
echo "kdump: saving to $_host:$_dir"
|
|
|
|
cat /var/lib/random-seed > /dev/urandom
|
|
ssh -q $_opt $_host mkdir -p $_dir || return 1
|
|
|
|
save_vmcore_dmesg_ssh ${DMESG_COLLECTOR} ${_dir} "${_opt}" $_host
|
|
|
|
echo "kdump: saving vmcore"
|
|
|
|
if [ "${CORE_COLLECTOR%%[[:blank:]]*}" = "scp" ]; then
|
|
scp -q $_opt /proc/vmcore "$_host:$_dir/vmcore-incomplete" || return 1
|
|
ssh $_opt $_host "mv $_dir/vmcore-incomplete $_dir/vmcore" || return 1
|
|
else
|
|
$CORE_COLLECTOR /proc/vmcore | ssh $_opt $_host "dd bs=512 of=$_dir/vmcore-incomplete" || return 1
|
|
ssh $_opt $_host "mv $_dir/vmcore-incomplete $_dir/vmcore.flat" || return 1
|
|
fi
|
|
|
|
echo "kdump: saving vmcore complete"
|
|
return 0
|
|
}
|
|
|
|
save_vmcore_dmesg_ssh() {
|
|
local _dmesg_collector=$1
|
|
local _path=$2
|
|
local _opts="$3"
|
|
local _location=$4
|
|
|
|
echo "kdump: saving vmcore-dmesg.txt"
|
|
$_dmesg_collector /proc/vmcore | ssh $_opts $_location "dd of=$_path/vmcore-dmesg-incomplete.txt"
|
|
_exitcode=$?
|
|
|
|
if [ $_exitcode -eq 0 ]; then
|
|
ssh -q $_opts $_location mv $_path/vmcore-dmesg-incomplete.txt $_path/vmcore-dmesg.txt
|
|
echo "kdump: saving vmcore-dmesg.txt complete"
|
|
else
|
|
echo "kdump: saving vmcore-dmesg.txt failed"
|
|
fi
|
|
}
|
|
|
|
get_host_ip()
|
|
{
|
|
local _host
|
|
if is_nfs_dump_target || is_ssh_dump_target
|
|
then
|
|
kdumpnic=$(getarg kdumpnic=)
|
|
[ -z "$kdumpnic" ] && echo "kdump: failed to get kdumpnic!" && return 1
|
|
_host=`ip addr show dev $kdumpnic|grep 'inet '`
|
|
[ $? -ne 0 ] && echo "kdump: wrong kdumpnic: $kdumpnic" && return 1
|
|
_host="${_host##*inet }"
|
|
_host="${_host%%/*}"
|
|
[ -z "$_host" ] && echo "kdump: wrong kdumpnic: $kdumpnic" && return 1
|
|
HOST_IP=$_host
|
|
fi
|
|
return 0
|
|
}
|
|
|
|
read_kdump_conf()
|
|
{
|
|
if [ ! -f "$KDUMP_CONF" ]; then
|
|
echo "kdump: $KDUMP_CONF not found"
|
|
return
|
|
fi
|
|
|
|
get_kdump_confs
|
|
|
|
# rescan for add code for dump target
|
|
while read config_opt config_val;
|
|
do
|
|
# remove inline comments after the end of a directive.
|
|
config_val=$(strip_comments $config_val)
|
|
case "$config_opt" in
|
|
ext[234]|xfs|btrfs|minix|nfs)
|
|
add_dump_code "dump_fs $config_val"
|
|
;;
|
|
raw)
|
|
add_dump_code "dump_raw $config_val"
|
|
;;
|
|
ssh)
|
|
add_dump_code "dump_ssh $SSH_KEY_LOCATION $config_val"
|
|
;;
|
|
esac
|
|
done < $KDUMP_CONF
|
|
}
|
|
|
|
fence_kdump_notify()
|
|
{
|
|
if [ -n "$FENCE_KDUMP_NODES" ]; then
|
|
$FENCE_KDUMP_SEND $FENCE_KDUMP_ARGS $FENCE_KDUMP_NODES &
|
|
fi
|
|
}
|
|
|
|
read_kdump_conf
|
|
fence_kdump_notify
|
|
|
|
if [ -z "$CORE_COLLECTOR" ];then
|
|
CORE_COLLECTOR=$DEFAULT_CORE_COLLECTOR
|
|
if is_ssh_dump_target || is_raw_dump_target; then
|
|
CORE_COLLECTOR="$CORE_COLLECTOR -F"
|
|
fi
|
|
fi
|
|
|
|
get_host_ip
|
|
if [ $? -ne 0 ]; then
|
|
echo "kdump: get_host_ip exited with non-zero status!"
|
|
do_default_action
|
|
do_final_action
|
|
fi
|
|
|
|
if [ -z "$DUMP_INSTRUCTION" ]; then
|
|
add_dump_code "dump_fs $NEWROOT"
|
|
fi
|
|
|
|
do_kdump_pre
|
|
if [ $? -ne 0 ]; then
|
|
echo "kdump: kdump_pre script exited with non-zero status!"
|
|
do_final_action
|
|
fi
|
|
make_trace_mem "kdump saving vmcore" '1:shortmem' '2+:mem' '3+:slab'
|
|
do_dump
|
|
DUMP_RETVAL=$?
|
|
|
|
do_kdump_post $DUMP_RETVAL
|
|
if [ $? -ne 0 ]; then
|
|
echo "kdump: kdump_post script exited with non-zero status!"
|
|
fi
|
|
|
|
if [ $DUMP_RETVAL -ne 0 ]; then
|
|
do_default_action
|
|
fi
|
|
|
|
do_final_action
|