kexec-tools/kdumpctl
Pingfan Liu c1a06343df kdumpctl: wait a while for network ready if dump target is ssh
If dump target is ipv6 address, a host should have ipv6 address ready
before starting kdump service. Otherwise, kdump service fails to start due
to the failure "ssh dump_server_ip mkdir -p $SAVE_PATH".
And user can see message like:
 "Could not create root@2620:52:0:10da:46a8:42ff:fe23:3272/var/crash"

I observe a long period (about 30s) on some machine before they got ipv6
address dynamiclly, which is never seen on ipv4 host.

Hence kdump service has a dependency on ipv6 address. But there is no good
way to resolve it. One way is asking user to run the cmd "nmcli connection
modify eth0 ipv6.may-fail false". But this will block systemd until ipv6
address is ready. Despite doing so, kdump can try its best (wait 1 minutes
after it starts up) before failure.

How to implement the wait is arguable. It will involve too many technique
details if explicitly waiting on ipv6 address, instead, just lean on 'ssh'
return value to see the availability of network.

Signed-off-by: Pingfan Liu <piliu@redhat.com>
Acked-by: Kairui Song <kasong@redhat.com>
2019-08-12 16:13:08 +08:00

1289 lines
29 KiB
Bash
Executable File

#!/bin/bash
KEXEC=/sbin/kexec
KDUMP_KERNELVER=""
KDUMP_COMMANDLINE=""
KEXEC_ARGS=""
KDUMP_CONFIG_FILE="/etc/kdump.conf"
MKDUMPRD="/sbin/mkdumprd -f"
DRACUT_MODULES_FILE="/usr/lib/dracut/modules.txt"
SAVE_PATH=/var/crash
SSH_KEY_LOCATION="/root/.ssh/kdump_id_rsa"
INITRD_CHECKSUM_LOCATION="/boot/.fadump_initrd_checksum"
DUMP_TARGET=""
DEFAULT_INITRD=""
DEFAULT_INITRD_BAK=""
TARGET_INITRD=""
FADUMP_REGISTER_SYS_NODE="/sys/kernel/fadump_registered"
#kdump shall be the default dump mode
DEFAULT_DUMP_MODE="kdump"
image_time=0
[[ $dracutbasedir ]] || dracutbasedir=/usr/lib/dracut
. $dracutbasedir/dracut-functions.sh
. /lib/kdump/kdump-lib.sh
standard_kexec_args="-p"
# Some default values in case /etc/sysconfig/kdump doesn't include
KDUMP_COMMANDLINE_REMOVE="hugepages hugepagesz slub_debug"
if [ -f /etc/sysconfig/kdump ]; then
. /etc/sysconfig/kdump
fi
single_instance_lock()
{
local rc timeout=5
exec 9>/var/lock/kdump
if [ $? -ne 0 ]; then
echo "Create file lock failed"
exit 1
fi
flock -n 9
rc=$?
while [ $rc -ne 0 ]; do
echo "Another app is currently holding the kdump lock; waiting for it to exit..."
flock -w $timeout 9
rc=$?
done
}
determine_dump_mode()
{
# Check if firmware-assisted dump is enabled
# if yes, set the dump mode as fadump
if is_fadump_capable; then
echo "Dump mode is fadump"
DEFAULT_DUMP_MODE="fadump"
fi
}
save_core()
{
coredir="/var/crash/`date +"%Y-%m-%d-%H:%M"`"
mkdir -p $coredir
cp --sparse=always /proc/vmcore $coredir/vmcore-incomplete
if [ $? == 0 ]; then
mv $coredir/vmcore-incomplete $coredir/vmcore
echo "saved a vmcore to $coredir"
else
echo "failed to save a vmcore to $coredir" >&2
fi
# pass the dmesg to Abrt tool if exists, in order
# to collect the kernel oops message.
# https://fedorahosted.org/abrt/
if [ -x /usr/bin/dumpoops ]; then
makedumpfile --dump-dmesg $coredir/vmcore $coredir/dmesg >/dev/null 2>&1
dumpoops -d $coredir/dmesg >/dev/null 2>&1
if [ $? == 0 ]; then
echo "kernel oops has been collected by abrt tool"
fi
fi
}
rebuild_fadump_initrd()
{
local target_initrd_tmp
# this file tells the initrd is fadump enabled
touch /tmp/fadump.initramfs
target_initrd_tmp="$TARGET_INITRD.tmp"
$MKDUMPRD $target_initrd_tmp --rebuild $DEFAULT_INITRD_BAK --kver $kdump_kver \
-i /tmp/fadump.initramfs /etc/fadump.initramfs
if [ $? != 0 ]; then
echo "mkdumprd: failed to rebuild initrd with fadump support" >&2
rm -f /tmp/fadump.initramfs
return 1
fi
rm -f /tmp/fadump.initramfs
# updating fadump initrd
mv $target_initrd_tmp $TARGET_INITRD
sync
return 0
}
check_earlykdump_is_enabled()
{
grep -q -w "rd.earlykdump" /proc/cmdline
return $?
}
rebuild_kdump_initrd()
{
$MKDUMPRD $TARGET_INITRD $kdump_kver
if [ $? != 0 ]; then
echo "mkdumprd: failed to make kdump initrd" >&2
return 1
fi
if check_earlykdump_is_enabled; then
echo "Tips: If early kdump is enabled, also require rebuilding the system initramfs to make the changes take effect for early kdump."
fi
return 0
}
rebuild_initrd()
{
if [[ ! -w "$KDUMP_BOOTDIR" ]];then
echo "$KDUMP_BOOTDIR does not have write permission. Can not rebuild $TARGET_INITRD"
return 1
fi
if [ $DEFAULT_DUMP_MODE == "fadump" ]; then
rebuild_fadump_initrd
else
rebuild_kdump_initrd
fi
return $?
}
#$1: the files to be checked with IFS=' '
check_exist()
{
for file in $1; do
if [ ! -f "$file" ]; then
echo -n "Error: $file not found."; echo
return 1
fi
done
}
#$1: the files to be checked with IFS=' '
check_executable()
{
for file in $1; do
if [ ! -x "$file" ]; then
echo -n "Error: $file is not executable."; echo
return 1
fi
done
}
backup_default_initrd()
{
if [ ! -f "$DEFAULT_INITRD" ]; then
return
fi
if [ ! -e $DEFAULT_INITRD_BAK ]; then
echo "Backing up $DEFAULT_INITRD before rebuild."
# save checksum to verify before restoring
sha1sum $DEFAULT_INITRD > $INITRD_CHECKSUM_LOCATION
cp $DEFAULT_INITRD $DEFAULT_INITRD_BAK
if [ $? -ne 0 ]; then
echo "WARNING: failed to backup $DEFAULT_INITRD."
rm -f $DEFAULT_INITRD_BAK
fi
fi
}
restore_default_initrd()
{
# If a backup initrd exists, we must be switching back from
# fadump to kdump. Restore the original default initrd.
if [ -f $DEFAULT_INITRD_BAK ] && [ -f $INITRD_CHECKSUM_LOCATION ]; then
# verify checksum before restoring
backup_checksum=`sha1sum $DEFAULT_INITRD_BAK | awk '{ print $1 }'`
default_checksum=`cat $INITRD_CHECKSUM_LOCATION | awk '{ print $1 }'`
if [ "$default_checksum" != "$backup_checksum" ]; then
echo "WARNING: checksum mismatch! Can't restore original initrd.."
else
rm -f $INITRD_CHECKSUM_LOCATION
mv $DEFAULT_INITRD_BAK $DEFAULT_INITRD
if [[ $? -eq 0 ]]; then
echo -n "Restoring original initrd as fadump mode "
echo "is disabled."
sync
fi
fi
fi
}
check_config()
{
local nr
nr=$(awk 'BEGIN{cnt=0} /^raw|^ssh[[:blank:]]|^nfs|^ext[234]|^xfs|^btrfs|^minix|^dracut_args .*\-\-mount/{cnt++} END{print cnt}' $KDUMP_CONFIG_FILE)
[ $nr -gt 1 ] && {
echo "More than one dump targets specified."
return 1
}
# Check if path option is set more than once.
nr=$(awk 'BEGIN{cnt=0} /^path /{cnt++} END{print cnt}' $KDUMP_CONFIG_FILE)
[ $nr -gt 1 ] && {
echo "Mutiple paths specifed in $KDUMP_CONFIG_FILE"
return 1
}
nr=$(grep "^dracut_args .*\-\-mount" $KDUMP_CONFIG_FILE | grep -o "\-\-mount" | wc -l)
[ $nr -gt 1 ] && {
echo "Multiple mount targets specified in one \"dracut_args\"."
return 1
}
# Check if we have any leading spaces (or tabs) before the
# variable name in the kdump conf file
if grep -E -q '^[[:blank:]]+[a-z]' $KDUMP_CONFIG_FILE; then
echo "No whitespaces are allowed before a kdump option name in $KDUMP_CONFIG_FILE"
return 1
fi
while read config_opt config_val; do
case "$config_opt" in
\#* | "")
;;
raw|ext2|ext3|ext4|minix|btrfs|xfs|nfs|ssh|sshkey|path|core_collector|kdump_post|kdump_pre|extra_bins|extra_modules|failure_action|default|final_action|force_rebuild|force_no_rebuild|dracut_args|fence_kdump_args|fence_kdump_nodes)
# remove inline comments after the end of a directive.
[ -z "$config_val" ] && {
echo "Invalid kdump config value for option $config_opt."
return 1;
}
;;
net|options|link_delay|disk_timeout|debug_mem_level|blacklist)
echo "Deprecated kdump config option: $config_opt. Refer to kdump.conf manpage for alternatives."
return 1
;;
*)
echo "Invalid kdump config option $config_opt"
return 1;
;;
esac
done <<< "$(read_strip_comments $KDUMP_CONFIG_FILE)"
check_failure_action_config || return 1
check_final_action_config || return 1
check_fence_kdump_config || return 1
return 0
}
# get_pcs_cluster_modified_files <image timestamp>
# return list of modified file for fence_kdump modified in Pacemaker cluster
get_pcs_cluster_modified_files()
{
local time_stamp
local modified_files
is_generic_fence_kdump && return 1
is_pcs_fence_kdump || return 1
time_stamp=`pcs cluster cib | xmllint --xpath 'string(/cib/@cib-last-written)' - | \
xargs -0 date +%s --date`
if [ -n $time_stamp -a $time_stamp -gt $image_time ]; then
modified_files="cluster-cib"
fi
if [ -f $FENCE_KDUMP_CONFIG_FILE ]; then
time_stamp=`stat -c "%Y" $FENCE_KDUMP_CONFIG_FILE`
if [ "$time_stamp" -gt "$image_time" ]; then
modified_files="$modified_files $FENCE_KDUMP_CONFIG_FILE"
fi
fi
echo $modified_files
}
setup_initrd()
{
KDUMP_BOOTDIR=$(check_boot_dir "${KDUMP_BOOTDIR}")
if [ -z "$KDUMP_KERNELVER" ]; then
kdump_kver=`uname -r`
else
kdump_kver=$KDUMP_KERNELVER
fi
kdump_kernel="${KDUMP_BOOTDIR}/${KDUMP_IMG}-${kdump_kver}${KDUMP_IMG_EXT}"
DEFAULT_INITRD="${KDUMP_BOOTDIR}/initramfs-`uname -r`.img"
DEFAULT_INITRD_BAK="${KDUMP_BOOTDIR}/.initramfs-`uname -r`.img.default"
if [ $DEFAULT_DUMP_MODE == "fadump" ]; then
TARGET_INITRD="$DEFAULT_INITRD"
# backup initrd for reference before replacing it
# with fadump aware initrd
backup_default_initrd
else
TARGET_INITRD="${KDUMP_BOOTDIR}/initramfs-${kdump_kver}kdump.img"
# check if a backup of default initrd exists. If yes,
# it signifies a switch from fadump mode. So, restore
# the backed up default initrd.
restore_default_initrd
fi
}
check_files_modified()
{
local modified_files=""
#also rebuild when Pacemaker cluster conf is changed and fence kdump is enabled.
modified_files=$(get_pcs_cluster_modified_files)
EXTRA_BINS=`grep ^kdump_post $KDUMP_CONFIG_FILE | cut -d\ -f2`
CHECK_FILES=`grep ^kdump_pre $KDUMP_CONFIG_FILE | cut -d\ -f2`
CORE_COLLECTOR=`grep ^core_collector $KDUMP_CONFIG_FILE | cut -d\ -f2`
CORE_COLLECTOR=`type -P $CORE_COLLECTOR`
EXTRA_BINS="$EXTRA_BINS $CHECK_FILES"
CHECK_FILES=`grep ^extra_bins $KDUMP_CONFIG_FILE | cut -d\ -f2-`
EXTRA_BINS="$EXTRA_BINS $CHECK_FILES"
files="$KDUMP_CONFIG_FILE $kdump_kernel $EXTRA_BINS $CORE_COLLECTOR"
[[ -e /etc/fstab ]] && files="$files /etc/fstab"
# Check for any updated extra module
EXTRA_MODULES="$(grep ^extra_modules $KDUMP_CONFIG_FILE | sed 's/^extra_modules\s*//')"
if [ -n "$EXTRA_MODULES" ]; then
if [ -e /lib/modules/$kdump_kver/modules.dep ]; then
files="$files /lib/modules/$kdump_kver/modules.dep"
fi
for _module in $EXTRA_MODULES; do
_module_file="$(modinfo --set-version "$kdump_kver" --filename "$_module" 2>/dev/null)"
if [[ $? -eq 0 ]]; then
files="$files $_module_file"
for _dep_modules in $(modinfo -F depends $_module | tr ',' ' '); do
files="$files $(modinfo --set-version "$kdump_kver" --filename $_dep_modules 2>/dev/null)"
done
else
# If it's not a module nor builtin, give an error
if ! ( modprobe --set-version "$kdump_kver" --dry-run "$_module" &>/dev/null ); then
echo "Module $_module not found"
fi
fi
done
fi
check_exist "$files" && check_executable "$EXTRA_BINS"
[ $? -ne 0 ] && return 2
for file in $files; do
if [ -e "$file" ]; then
time_stamp=`stat -c "%Y" $file`
if [ "$time_stamp" -gt "$image_time" ]; then
modified_files="$modified_files $file"
fi
if [ -L "$file" ]; then
file=$(readlink -m $file)
time_stamp=`stat -c "%Y" $file`
if [ "$time_stamp" -gt "$image_time" ]; then
modified_files="$modified_files $file"
fi
fi
else
echo "$file doesn't exist"
fi
done
if [ -n "$modified_files" ]; then
echo "Detected change(s) in the following file(s):"
echo -n " "; echo "$modified_files" | sed 's/\s/\n /g'
return 1
fi
return 0
}
check_dump_fs_modified()
{
local _old_dev _old_mntpoint _old_fstype
local _new_dev _new_mntpoint _new_fstype
local _target _path _dracut_args
local _target_drivers _module_name
local _old_drivers="$(lsinitrd $TARGET_INITRD -f /usr/lib/dracut/loaded-kernel-modules.txt | tr '\n' ' ')"
# No need to check in case of mount target specified via "dracut_args".
if is_mount_in_dracut_args; then
return 0
fi
# No need to check in case of raw target.
# Currently we do not check also if ssh/nfs target is specified
if is_ssh_dump_target || is_nfs_dump_target || is_raw_dump_target; then
return 0
fi
_target=$(get_user_configured_dump_disk)
if [[ -n "$_target" ]]; then
_target=$(to_dev_name $_target)
_new_fstype=$(blkid $_target | awk -F"TYPE=" '{print $2}' | cut -d '"' -f 2)
else
_path=$(get_save_path)
_target=$(get_target_from_path $_path)
_target=$(to_dev_name $_target)
_new_fstype=$(get_fs_type_from_target $_target)
if [[ -z "$_target" || -z "$_new_fstype" ]];then
echo "Dump path $_path does not exist"
return 2
fi
fi
_record_block_drivers() {
local _drivers
if [[ -b /dev/block/$1 ]]; then
_drivers=$(udevadm info -a "/dev/block/$1" | sed -n 's/\s*DRIVERS=="\(\S\+\)"/\1/p')
fi
if [[ -b $1 ]]; then
_drivers=$(udevadm info -a "$1" | sed -n 's/\s*DRIVERS=="\(\S\+\)"/\1/p')
fi
for _driver in $_drivers; do
if ! [[ " $_target_drivers " == *" $_driver "* ]]; then
_target_drivers="$_target_drivers $_driver"
fi
done
return 1
}
check_block_and_slaves_all _record_block_drivers "$(get_maj_min "$_target")"
for _driver in $_target_drivers; do
# Target is mounted already, if module is not included by current kernel,
# could be a deprecated/invalid driver name or a built-in module
_module_name=$(modinfo --set-version "$kdump_kver" -F name $_driver 2>/dev/null)
if [ $? -ne 0 ] || [ -z "$_module_name" ]; then
continue
fi
if ! [[ " $_old_drivers " == *" $_module_name "* ]]; then
echo "Detected change in block device driver, new loaded module: $_module_name"
return 1
fi
done
if [[ $(expr substr $_new_fstype 1 3) = "nfs" ]];then
_new_dev=$_target
else
_new_dev=$(get_persistent_dev $_target)
if [ -z "$_new_dev" ]; then
echo "Get persistent device name failed"
return 2
fi
fi
if ! findmnt $_target >/dev/null; then
echo "Dump target $_target is probably not mounted."
return 2
fi
if [[ "$_target" = "$(get_root_fs_device)" ]]; then
_new_mntpoint="/sysroot"
else
_new_mntpoint="/kdumproot/$(get_mntpoint_from_target $_target)"
fi
_dracut_args=$(lsinitrd $TARGET_INITRD -f usr/lib/dracut/build-parameter.txt)
if [[ -z "$_dracut_args" ]];then
echo "Warning: No dracut arguments found in initrd"
return 0
fi
# if --mount argument present then match old and new target, mount
# point and file system. If any of them mismatches then rebuild
echo $_dracut_args | grep "\-\-mount" &> /dev/null
if [[ $? -eq 0 ]];then
set -- $(echo $_dracut_args | awk -F "--mount '" '{print $2}' | cut -d' ' -f1,2,3)
_old_dev=$1
_old_mntpoint=$2
_old_fstype=$3
[[ $_new_dev = $_old_dev && $_new_mntpoint = $_old_mntpoint && $_new_fstype = $_old_fstype ]] && return 0
# otherwise rebuild if target device is not a root device
else
[[ "$_target" = "$(get_root_fs_device)" ]] && return 0
fi
echo "Detected change in File System"
return 1
}
check_wdt_modified()
{
local -A _drivers
local _alldrivers _active _wdtdrv _wdtppath _dir
local wd_old wd_new
is_wdt_mod_omitted
[[ $? -eq 0 ]] && return 0
[[ -d /sys/class/watchdog/ ]] || return 0
# Copied logic from dracut 04watchdog/module-setup.sh::installkernel()
for _dir in /sys/class/watchdog/*; do
[[ -d "$_dir" ]] || continue
[[ -f "$_dir/state" ]] || continue
_active=$(< "$_dir/state")
[[ "$_active" = "active" ]] || continue
# device/modalias will return driver of this device
_wdtdrv=$(< "$_dir/device/modalias")
# There can be more than one module represented by same
# modalias. Currently load all of them.
# TODO: Need to find a way to avoid any unwanted module
# represented by modalias
_wdtdrv=$(modprobe --set-version "$kdump_kver" -R $_wdtdrv 2>/dev/null)
if [[ $_wdtdrv ]]; then
for i in $_wdtdrv; do
_drivers[$i]=1
done
fi
# however in some cases, we also need to check that if there is
# a specific driver for the parent bus/device. In such cases
# we also need to enable driver for parent bus/device.
_wdtppath=$(readlink -f "$_dir/device")
while [[ -d "$_wdtppath" ]] && [[ "$_wdtppath" != "/sys" ]]; do
_wdtppath=$(readlink -f "$_wdtppath/..")
[[ -f "$_wdtppath/modalias" ]] || continue
_wdtdrv=$(< "$_wdtppath/modalias")
_wdtdrv=$(modprobe --set-version "$kdump_kver" -R $_wdtdrv 2>/dev/null)
if [[ $_wdtdrv ]]; then
for i in $_wdtdrv; do
_drivers[$i]=1
done
fi
done
done
# ensure that watchdog module is loaded as early as possible
_alldrivers="${!_drivers[*]}"
[[ $_alldrivers ]] && wd_new="rd.driver.pre=${_alldrivers// /,}"
wd_old=$(lsinitrd $TARGET_INITRD -f etc/cmdline.d/00-watchdog.conf)
[[ "$wd_old" = "$wd_new" ]] && return 0
return 1
}
# returns 0 if system is not modified
# returns 1 if system is modified
# returns 2 if system modification is invalid
check_system_modified()
{
local ret
[[ -f $TARGET_INITRD ]] || return 1
check_files_modified
ret=$?
if [ $ret -ne 0 ]; then
return $ret
fi
check_dump_fs_modified
ret=$?
if [ $ret -ne 0 ]; then
return $ret
fi
check_wdt_modified
if [ $? -ne 0 ]; then
echo "Detected change in watchdog state"
return 1
fi
return 0
}
check_rebuild()
{
local capture_capable_initrd="1"
local _force_rebuild force_rebuild="0"
local _force_no_rebuild force_no_rebuild="0"
local ret system_modified="0"
setup_initrd
if [ $? -ne 0 ]; then
return 1
fi
_force_no_rebuild=`grep ^force_no_rebuild $KDUMP_CONFIG_FILE 2>/dev/null`
if [ $? -eq 0 ]; then
force_no_rebuild=`echo $_force_no_rebuild | cut -d' ' -f2`
if [ "$force_no_rebuild" != "0" ] && [ "$force_no_rebuild" != "1" ];then
echo "Error: force_no_rebuild value is invalid"
return 1
fi
fi
_force_rebuild=`grep ^force_rebuild $KDUMP_CONFIG_FILE 2>/dev/null`
if [ $? -eq 0 ]; then
force_rebuild=`echo $_force_rebuild | cut -d' ' -f2`
if [ "$force_rebuild" != "0" ] && [ "$force_rebuild" != "1" ];then
echo "Error: force_rebuild value is invalid"
return 1
fi
fi
if [[ "$force_no_rebuild" == "1" && "$force_rebuild" == "1" ]]; then
echo "Error: force_rebuild and force_no_rebuild are enabled simultaneously in kdump.conf"
return 1
fi
# Will not rebuild kdump initrd
if [ "$force_no_rebuild" == "1" ]; then
return 0
fi
#check to see if dependent files has been modified
#since last build of the image file
if [ -f $TARGET_INITRD ]; then
image_time=`stat -c "%Y" $TARGET_INITRD 2>/dev/null`
#in case of fadump mode, check whether the default/target
#initrd is already built with dump capture capability
if [ "$DEFAULT_DUMP_MODE" == "fadump" ]; then
capture_capable_initrd=$(lsinitrd -f $DRACUT_MODULES_FILE $TARGET_INITRD | grep ^kdumpbase$ | wc -l)
fi
fi
check_system_modified
ret=$?
if [ $ret -eq 2 ]; then
return 1
elif [ $ret -eq 1 ];then
system_modified="1"
fi
if [ $image_time -eq 0 ]; then
echo -n "No kdump initial ramdisk found."; echo
elif [ "$capture_capable_initrd" == "0" ]; then
echo -n "Rebuild $TARGET_INITRD with dump capture support"; echo
elif [ "$force_rebuild" != "0" ]; then
echo -n "Force rebuild $TARGET_INITRD"; echo
elif [ "$system_modified" != "0" ]; then
:
else
return 0
fi
echo "Rebuilding $TARGET_INITRD"
rebuild_initrd
return $?
}
# Load the kdump kernel specified in /etc/sysconfig/kdump
# If none is specified, try to load a kdump kernel with the same version
# as the currently running kernel.
load_kdump()
{
KEXEC_ARGS=$(prepare_kexec_args "${KEXEC_ARGS}")
KDUMP_COMMANDLINE=$(prepare_cmdline "${KDUMP_COMMANDLINE}" "${KDUMP_COMMANDLINE_REMOVE}" "${KDUMP_COMMANDLINE_APPEND}")
# For secureboot enabled machines, use new kexec file based syscall.
# Old syscall will always fail as it does not have capability to
# to kernel signature verification.
if is_secure_boot_enforced; then
echo "Secure Boot is enabled. Using kexec file based syscall."
KEXEC_ARGS="$KEXEC_ARGS -s"
fi
$KEXEC $KEXEC_ARGS $standard_kexec_args \
--command-line="$KDUMP_COMMANDLINE" \
--initrd=$TARGET_INITRD $kdump_kernel
if [ $? == 0 ]; then
echo "kexec: loaded kdump kernel"
return 0
else
echo "kexec: failed to load kdump kernel" >&2
return 1
fi
}
check_ssh_config()
{
while read config_opt config_val; do
case "$config_opt" in
sshkey)
# remove inline comments after the end of a directive.
if [ -f "$config_val" ]; then
# canonicalize the path
SSH_KEY_LOCATION=$(/usr/bin/readlink -m $config_val)
else
echo "WARNING: '$config_val' doesn't exist, using default value '$SSH_KEY_LOCATION'"
fi
;;
path)
SAVE_PATH=$config_val
;;
ssh)
DUMP_TARGET=$config_val
;;
*)
;;
esac
done <<< "$(read_strip_comments $KDUMP_CONFIG_FILE)"
#make sure they've configured kdump.conf for ssh dumps
local SSH_TARGET=`echo -n $DUMP_TARGET | sed -n '/.*@/p'`
if [ -z "$SSH_TARGET" ]; then
return 1
fi
return 0
}
# ipv6 host address may takes a long time to be ready.
# Instead of checking against ipv6 address, we just check the network reachable
# by the return val of 'ssh'
check_and_wait_network_ready()
{
local start_time=$(date +%s)
local cur
local diff
while true; do
ssh -q -i $SSH_KEY_LOCATION -o BatchMode=yes $DUMP_TARGET mkdir -p $SAVE_PATH
# ssh exits with the exit status of the remote command or with 255 if an error occurred
if [ $? -eq 0 ]; then
return 0
elif [ $? -ne 255 ]; then
return 1
fi
cur=$(date +%s)
diff=$( $cur - $start_time )
# 60s time out
if [ $diff -gt 60 ]; then
break;
fi
sleep 1
done
return 1
}
check_ssh_target()
{
check_and_wait_network_ready
if [ $? -ne 0 ]; then
echo "Could not create $DUMP_TARGET:$SAVE_PATH, you probably need to run \"kdumpctl propagate\"" >&2
return 1
fi
return 0
}
propagate_ssh_key()
{
check_ssh_config
if [ $? -ne 0 ]; then
echo "No ssh config specified in $KDUMP_CONFIG_FILE. Can't propagate" >&2
exit 1
fi
local KEYFILE=$SSH_KEY_LOCATION
local errmsg="Failed to propagate ssh key"
#Check to see if we already created key, if not, create it.
if [ -f $KEYFILE ]; then
echo "Using existing keys..."
else
echo -n "Generating new ssh keys... "
/usr/bin/ssh-keygen -t rsa -f $KEYFILE -N "" 2>&1 > /dev/null
echo "done."
fi
#now find the target ssh user and server to contact.
SSH_USER=`echo $DUMP_TARGET | cut -d\ -f2 | cut -d@ -f1`
SSH_SERVER=`echo $DUMP_TARGET | sed -e's/\(.*@\)\(.*$\)/\2/'`
#now send the found key to the found server
ssh-copy-id -i $KEYFILE $SSH_USER@$SSH_SERVER
RET=$?
if [ $RET == 0 ]; then
echo $KEYFILE has been added to ~$SSH_USER/.ssh/authorized_keys on $SSH_SERVER
return 0
else
echo $errmsg, $KEYFILE failed in transfer to $SSH_SERVER >&2
exit 1
fi
}
show_reserved_mem()
{
local mem=$(cat /sys/kernel/kexec_crash_size)
local mem_mb=$(expr $mem / 1024 / 1024)
echo "Reserved "$mem_mb"MB memory for crash kernel"
}
check_current_fadump_status()
{
# Check if firmware-assisted dump has been registered.
rc=`cat $FADUMP_REGISTER_SYS_NODE`
[ $rc -eq 1 ] && return 0
return 1
}
check_current_status()
{
if [ $DEFAULT_DUMP_MODE == "fadump" ]; then
check_current_fadump_status
else
check_current_kdump_status
fi
return $?
}
save_raw()
{
local kdump_dir
local raw_target
raw_target=$(awk '$1 ~ /^raw$/ { print $2; }' $KDUMP_CONFIG_FILE)
[ -z "$raw_target" ] && return 0
[ -b "$raw_target" ] || {
echo "raw partition $raw_target not found"
return 1
}
check_fs=$(lsblk --nodeps -npo FSTYPE $raw_target)
if [[ $(echo $check_fs | wc -w) -ne 0 ]]; then
echo "Warning: Detected '$check_fs' signature on $raw_target, data loss is expected."
return 0
fi
kdump_dir=`grep ^path $KDUMP_CONFIG_FILE | cut -d' ' -f2-`
if [ -z "${kdump_dir}" ]; then
coredir="/var/crash/`date +"%Y-%m-%d-%H:%M"`"
else
coredir="${kdump_dir}/`date +"%Y-%m-%d-%H:%M"`"
fi
mkdir -p "$coredir"
[ -d "$coredir" ] || {
echo "failed to create $coredir"
return 1
}
if makedumpfile -R $coredir/vmcore <$raw_target >/dev/null 2>&1; then
# dump found
echo "Dump saved to $coredir/vmcore"
# wipe makedumpfile header
dd if=/dev/zero of=$raw_target bs=1b count=1 2>/dev/null
else
rm -rf "$coredir"
fi
return 0
}
is_dump_target_configured()
{
local _target
_target=$(egrep "^ext[234]|^xfs|^btrfs|^minix|^raw|^ssh|^nfs" /etc/kdump.conf)
[ -n "$_target" ]
}
local_fs_dump_target()
{
local _target
_target=$(egrep "^ext[234]|^xfs|^btrfs|^minix" /etc/kdump.conf)
if [ $? -eq 0 ]; then
echo $_target|awk '{print $2}'
fi
}
path_to_be_relabeled()
{
local _path _target _mnt="/" _rmnt
if is_dump_target_configured; then
_target=$(local_fs_dump_target)
if [[ -n "$_target" ]]; then
_mnt=$(findmnt -k -f -n -r -o TARGET $_target)
if [ -z "$_mnt" ]; then
return
fi
else
return
fi
fi
if is_mount_in_dracut_args; then
return;
fi
_path=$(get_save_path)
# if $_path is masked by other mount, we will not relabel it.
_rmnt=$(df $_mnt/$_path 2>/dev/null | tail -1 | awk '{ print $NF }')
if [ "$_rmnt" == "$_mnt" ]; then
echo $_mnt/$_path
fi
}
selinux_relabel()
{
local _path _i _attr
_path=$(path_to_be_relabeled)
if [ -z "$_path" ] || ! [ -d "$_path" ] ; then
return
fi
for _i in $(find $_path); do
_attr=$(getfattr -m "security.selinux" $_i 2>/dev/null)
if [ -z "$_attr" ]; then
restorecon $_i;
fi
done
}
check_fence_kdump_config()
{
local hostname=`hostname`
local ipaddrs=`hostname -I`
local nodes=$(get_option_value "fence_kdump_nodes")
for node in $nodes; do
if [ "$node" = "$hostname" ]; then
echo "Option fence_kdump_nodes cannot contain $hostname"
return 1
fi
# node can be ipaddr
echo $ipaddrs | grep $node > /dev/null
if [ $? -eq 0 ]; then
echo "Option fence_kdump_nodes cannot contain $node"
return 1
fi
done
return 0
}
check_dump_feasibility()
{
if [ $DEFAULT_DUMP_MODE == "fadump" ]; then
return 0
fi
check_kdump_feasibility
return $?
}
start_fadump()
{
echo 1 > $FADUMP_REGISTER_SYS_NODE
if ! check_current_fadump_status; then
echo "fadump: failed to register"
return 1
fi
echo "fadump: registered successfully"
return 0
}
start_dump()
{
if [ $DEFAULT_DUMP_MODE == "fadump" ]; then
start_fadump
else
load_kdump
fi
return $?
}
check_failure_action_config()
{
local default_option
local failure_action
local option="failure_action"
default_option=$(awk '$1 ~ /^default$/ {print $2;}' $KDUMP_CONFIG_FILE)
failure_action=$(awk '$1 ~ /^failure_action$/ {print $2;}' $KDUMP_CONFIG_FILE)
if [ -z "$failure_action" -a -z "$default_option" ]; then
return 0
elif [ -n "$failure_action" -a -n "$default_option" ]; then
echo "Cannot specify 'failure_action' and 'default' option together"
return 1
fi
if [ -n "$default_option" ]; then
option="default"
failure_action="$default_option"
fi
case "$failure_action" in
reboot|halt|poweroff|shell|dump_to_rootfs)
return 0
;;
*)
echo $"Usage kdump.conf: $option {reboot|halt|poweroff|shell|dump_to_rootfs}"
return 1
esac
}
check_final_action_config()
{
local final_action
final_action=$(awk '$1 ~ /^final_action$/ {print $2;}' $KDUMP_CONFIG_FILE)
if [ -z "$final_action" ]; then
return 0
else
case "$final_action" in
reboot|halt|poweroff)
return 0
;;
*)
echo $"Usage kdump.conf: final_action {reboot|halt|poweroff}"
return 1
esac
fi
}
start()
{
check_dump_feasibility
if [ $? -ne 0 ]; then
echo "Starting kdump: [FAILED]"
return 1
fi
check_config
if [ $? -ne 0 ]; then
echo "Starting kdump: [FAILED]"
return 1
fi
if sestatus 2>/dev/null | grep -q "SELinux status.*enabled"; then
selinux_relabel
fi
save_raw
if [ $? -ne 0 ]; then
echo "Starting kdump: [FAILED]"
return 1
fi
check_current_status
if [ $? == 0 ]; then
echo "Kdump already running: [WARNING]"
return 0
fi
if check_ssh_config; then
if ! check_ssh_target; then
echo "Starting kdump: [FAILED]"
return 1
fi
fi
check_rebuild
if [ $? != 0 ]; then
echo "Starting kdump: [FAILED]"
return 1
fi
start_dump
if [ $? != 0 ]; then
echo "Starting kdump: [FAILED]"
return 1
fi
echo "Starting kdump: [OK]"
}
reload()
{
check_current_status
if [ $? -ne 0 ]; then
echo "Kdump is not running: [WARNING]"
return 0
fi
if [ $DEFAULT_DUMP_MODE == "fadump" ]; then
reload_fadump
return $?
else
stop_kdump
fi
if [ $? -ne 0 ]; then
echo "Stopping kdump: [FAILED]"
return 1
fi
echo "Stopping kdump: [OK]"
setup_initrd
if [ $? -ne 0 ]; then
echo "Starting kdump: [FAILED]"
return 1
fi
start_dump
if [ $? -ne 0 ]; then
echo "Starting kdump: [FAILED]"
return 1
fi
echo "Starting kdump: [OK]"
}
stop_fadump()
{
echo 0 > $FADUMP_REGISTER_SYS_NODE
if check_current_fadump_status; then
echo "fadump: failed to unregister"
return 1
fi
echo "fadump: unregistered successfully"
return 0
}
stop_kdump()
{
if is_secure_boot_enforced; then
$KEXEC -s -p -u
else
$KEXEC -p -u
fi
if [ $? != 0 ]; then
echo "kexec: failed to unload kdump kernel"
return 1
fi
echo "kexec: unloaded kdump kernel"
return 0
}
reload_fadump()
{
echo 1 > $FADUMP_REGISTER_SYS_NODE
if [ $? == 0 ]; then
echo "fadump: re-registered successfully"
return 0
else
# FADump could fail on older kernel where re-register
# support is not enabled. Try stop/start from userspace
# to handle such scenario.
stop_fadump
if [ $? == 0 ]; then
start_fadump
return $?
fi
fi
return 1
}
stop()
{
if [ $DEFAULT_DUMP_MODE == "fadump" ]; then
stop_fadump
else
stop_kdump
fi
if [ $? != 0 ]; then
echo "Stopping kdump: [FAILED]"
return 1
fi
echo "Stopping kdump: [OK]"
return 0
}
rebuild() {
check_config
if [ $? -ne 0 ]; then
return 1
fi
if check_ssh_config; then
if ! check_ssh_target; then
return 1
fi
fi
setup_initrd
if [ $? -ne 0 ]; then
return 1
fi
echo "Rebuilding $TARGET_INITRD"
rebuild_initrd
return $?
}
if [ ! -f "$KDUMP_CONFIG_FILE" ]; then
echo "Error: No kdump config file found!" >&2
exit 1
fi
main ()
{
# Determine if the dump mode is kdump or fadump
determine_dump_mode
case "$1" in
start)
if [ -s /proc/vmcore ]; then
save_core
reboot
else
start
fi
;;
stop)
stop
;;
status)
EXIT_CODE=0
check_current_status
case "$?" in
0)
echo "Kdump is operational"
EXIT_CODE=0
;;
1)
echo "Kdump is not operational"
EXIT_CODE=3
;;
esac
exit $EXIT_CODE
;;
reload)
reload
;;
restart)
stop
start
;;
rebuild)
rebuild
;;
condrestart)
;;
propagate)
propagate_ssh_key
;;
showmem)
show_reserved_mem
;;
*)
echo $"Usage: $0 {start|stop|status|restart|reload|rebuild|propagate|showmem}"
exit 1
esac
}
# Other kdumpctl instances will block in queue, until this one exits
single_instance_lock
# To avoid fd 9 leaking, we invoke a subshell, close fd 9 and call main.
# So that fd isn't leaking when main is invoking a subshell.
(exec 9<&-; main $1)
exit $?