From 5cac7c3f964b55b95df4dab09f7d529d1fe2abea Mon Sep 17 00:00:00 2001 From: DistroBaker Date: Thu, 5 Nov 2020 05:34:29 +0000 Subject: [PATCH] Merged update from upstream sources This is an automated DistroBaker update from upstream sources. If you do not know what this is about or would like to opt out, contact the OSCI team. Source: https://src.fedoraproject.org/rpms/kexec-tools.git#bfd06661e81465d077bac435c90b4082134adf19 --- dracut-early-kdump-module-setup.sh | 3 + dracut-early-kdump.sh | 23 +- dracut-kdump.sh | 68 ++-- dracut-module-setup.sh | 13 +- kdump-lib-initramfs.sh | 68 +++- kdump-lib.sh | 29 +- kdump-logger.sh | 334 ++++++++++++++++++ kdump.conf | 6 +- kdump.conf.5 | 18 +- kdump.service | 1 + kdump.sysconfig | 16 + kdump.sysconfig.aarch64 | 16 + kdump.sysconfig.i386 | 16 + kdump.sysconfig.ppc64 | 16 + kdump.sysconfig.ppc64le | 16 + kdump.sysconfig.s390x | 18 +- kdump.sysconfig.x86_64 | 16 + kdumpctl | 303 +++++++++------- kexec-kdump-howto.txt | 53 ++- kexec-tools.spec | 26 +- mkdumprd | 25 +- .../scripts/testcases/local-kdump/0-local.sh | 2 +- tests/scripts/testcases/nfs-kdump/1-client.sh | 2 +- tests/scripts/testcases/ssh-kdump/1-client.sh | 2 +- 24 files changed, 855 insertions(+), 235 deletions(-) create mode 100755 kdump-logger.sh diff --git a/dracut-early-kdump-module-setup.sh b/dracut-early-kdump-module-setup.sh index f30bd67..3b23383 100755 --- a/dracut-early-kdump-module-setup.sh +++ b/dracut-early-kdump-module-setup.sh @@ -48,7 +48,10 @@ install() { inst_simple "/etc/sysconfig/kdump" inst_binary "/usr/sbin/kexec" inst_binary "/usr/bin/gawk" "/usr/bin/awk" + inst_binary "/usr/bin/logger" "/usr/bin/logger" + inst_binary "/usr/bin/printf" "/usr/bin/printf" inst_script "/lib/kdump/kdump-lib.sh" "/lib/kdump-lib.sh" + inst_script "/lib/kdump/kdump-logger.sh" "/lib/kdump-logger.sh" inst_hook cmdline 00 "$moddir/early-kdump.sh" inst_binary "$KDUMP_KERNEL" inst_binary "$KDUMP_INITRD" diff --git a/dracut-early-kdump.sh b/dracut-early-kdump.sh index 92913fb..23bb369 100755 --- a/dracut-early-kdump.sh +++ b/dracut-early-kdump.sh @@ -13,6 +13,13 @@ EARLY_KEXEC_ARGS="" . /lib/dracut-lib.sh . /lib/kdump-lib.sh +#initiate the kdump logger +dlog_init +if [ $? -ne 0 ]; then + echo "failed to initiate the kdump logger." + exit 1 +fi + prepare_parameters() { EARLY_KDUMP_CMDLINE=$(prepare_cmdline "${KDUMP_COMMANDLINE}" "${KDUMP_COMMANDLINE_REMOVE}" "${KDUMP_COMMANDLINE_APPEND}") @@ -28,7 +35,7 @@ early_kdump_load() fi if is_fadump_capable; then - echo "WARNING: early kdump doesn't support fadump." + dwarn "WARNING: early kdump doesn't support fadump." return 1 fi @@ -42,18 +49,22 @@ early_kdump_load() EARLY_KEXEC_ARGS=$(prepare_kexec_args "${KEXEC_ARGS}") if is_secure_boot_enforced; then - echo "Secure Boot is enabled. Using kexec file based syscall." + dinfo "Secure Boot is enabled. Using kexec file based syscall." EARLY_KEXEC_ARGS="$EARLY_KEXEC_ARGS -s" fi + ddebug "earlykdump: $KEXEC ${EARLY_KEXEC_ARGS} $standard_kexec_args \ + --command-line=$EARLY_KDUMP_CMDLINE --initrd=$EARLY_KDUMP_INITRD \ + $EARLY_KDUMP_KERNEL" + $KEXEC ${EARLY_KEXEC_ARGS} $standard_kexec_args \ --command-line="$EARLY_KDUMP_CMDLINE" \ --initrd=$EARLY_KDUMP_INITRD $EARLY_KDUMP_KERNEL if [ $? == 0 ]; then - echo "kexec: loaded early-kdump kernel" + dinfo "kexec: loaded early-kdump kernel" return 0 else - echo "kexec: failed to load early-kdump kernel" + derror "kexec: failed to load early-kdump kernel" return 1 fi } @@ -61,10 +72,10 @@ early_kdump_load() set_early_kdump() { if getargbool 0 rd.earlykdump; then - echo "early-kdump is enabled." + dinfo "early-kdump is enabled." early_kdump_load else - echo "early-kdump is disabled." + dinfo "early-kdump is disabled." fi return 0 diff --git a/dracut-kdump.sh b/dracut-kdump.sh index 6f948fc..c2627c2 100755 --- a/dracut-kdump.sh +++ b/dracut-kdump.sh @@ -5,7 +5,6 @@ if [ -f /etc/fadump.initramfs ] && [ ! -f /proc/device-tree/rtas/ibm,kernel-dump exit 0 fi -exec &> /dev/console . /lib/dracut-lib.sh . /lib/kdump-lib-initramfs.sh @@ -22,7 +21,7 @@ do_dump() _ret=$? if [ $_ret -ne 0 ]; then - echo "kdump: saving vmcore failed" + derror "saving vmcore failed" fi return $_ret @@ -36,7 +35,7 @@ do_kdump_pre() "$KDUMP_PRE" _ret=$? if [ $_ret -ne 0 ]; then - echo "kdump: $KDUMP_PRE exited with $_ret status" + derror "$KDUMP_PRE exited with $_ret status" return $_ret fi fi @@ -47,7 +46,7 @@ do_kdump_pre() "$file" _ret=$? if [ $_ret -ne 0 ]; then - echo "kdump: $file exited with $_ret status" + derror "$file exited with $_ret status" fi done fi @@ -63,7 +62,7 @@ do_kdump_post() "$file" "$1" _ret=$? if [ $_ret -ne 0 ]; then - echo "kdump: $file exited with $_ret status" + derror "$file exited with $_ret status" fi done fi @@ -72,7 +71,7 @@ do_kdump_post() "$KDUMP_POST" "$1" _ret=$? if [ $_ret -ne 0 ]; then - echo "kdump: $KDUMP_POST exited with $_ret status" + derror "$KDUMP_POST exited with $_ret status" fi fi } @@ -88,7 +87,7 @@ dump_raw() [ -b "$_raw" ] || return 1 - echo "kdump: saving to raw disk $_raw" + dinfo "saving to raw disk $_raw" if ! $(echo -n $CORE_COLLECTOR|grep -q makedumpfile); then _src_size=`ls -l /proc/vmcore | cut -d' ' -f5` @@ -96,21 +95,22 @@ dump_raw() monitor_dd_progress $_src_size_mb & fi - echo "kdump: saving vmcore" + dinfo "saving vmcore" $CORE_COLLECTOR /proc/vmcore | dd of=$_raw bs=$DD_BLKSIZE >> /tmp/dd_progress_file 2>&1 || return 1 sync - echo "kdump: saving vmcore complete" + dinfo "saving vmcore complete" return 0 } dump_ssh() { + local ret local _opt="-i $1 -o BatchMode=yes -o StrictHostKeyChecking=yes" local _dir="$KDUMP_PATH/$HOST_IP-$DATEDIR" local _host=$2 - echo "kdump: saving to $_host:$_dir" + dinfo "saving to $_host:$_dir" cat /var/lib/random-seed > /dev/urandom ssh -q $_opt $_host mkdir -p $_dir || return 1 @@ -118,17 +118,29 @@ dump_ssh() save_vmcore_dmesg_ssh ${DMESG_COLLECTOR} ${_dir} "${_opt}" $_host save_opalcore_ssh ${_dir} "${_opt}" $_host - echo "kdump: saving vmcore" + dinfo "saving vmcore" if [ "${CORE_COLLECTOR%%[[:blank:]]*}" = "scp" ]; then - scp -q $_opt /proc/vmcore "$_host:$_dir/vmcore-incomplete" || return 1 + scp -q $_opt /proc/vmcore "$_host:$_dir/vmcore-incomplete" + ret=$? + save_log + scp -q $_opt $KDUMP_LOG_FILE "$_host:$_dir/" + if [ $ret -ne 0 ]; then + return 1 + fi ssh $_opt $_host "mv $_dir/vmcore-incomplete $_dir/vmcore" || return 1 else - $CORE_COLLECTOR /proc/vmcore | ssh $_opt $_host "dd bs=512 of=$_dir/vmcore-incomplete" || return 1 + $CORE_COLLECTOR /proc/vmcore | ssh $_opt $_host "dd bs=512 of=$_dir/vmcore-incomplete" + ret=$? + save_log + scp -q $_opt $KDUMP_LOG_FILE "$_host:$_dir/" + if [ $ret -ne 0 ]; then + return 1 + fi ssh $_opt $_host "mv $_dir/vmcore-incomplete $_dir/vmcore.flat" || return 1 fi - echo "kdump: saving vmcore complete" + dinfo "saving vmcore complete" return 0 } @@ -137,6 +149,8 @@ save_opalcore_ssh() { local _opts="$2" local _location=$3 + ddebug "_path=$_path _opts=$_opts _location=$_location" + if [ ! -f $OPALCORE ]; then # Check if we are on an old kernel that uses a different path if [ -f /sys/firmware/opal/core ]; then @@ -146,15 +160,15 @@ save_opalcore_ssh() { fi fi - echo "kdump: saving opalcore" + dinfo "saving opalcore:$OPALCORE to $_location:$_path" scp $_opts $OPALCORE $_location:$_path/opalcore-incomplete if [ $? -ne 0 ]; then - echo "kdump: saving opalcore failed" + derror "saving opalcore failed" return 1 fi ssh $_opts $_location mv $_path/opalcore-incomplete $_path/opalcore - echo "kdump: saving opalcore complete" + dinfo "saving opalcore complete" return 0 } @@ -164,15 +178,15 @@ save_vmcore_dmesg_ssh() { local _opts="$3" local _location=$4 - echo "kdump: saving vmcore-dmesg.txt" + dinfo "saving vmcore-dmesg.txt to $_location:$_path" $_dmesg_collector /proc/vmcore | ssh $_opts $_location "dd of=$_path/vmcore-dmesg-incomplete.txt" _exitcode=$? if [ $_exitcode -eq 0 ]; then ssh -q $_opts $_location mv $_path/vmcore-dmesg-incomplete.txt $_path/vmcore-dmesg.txt - echo "kdump: saving vmcore-dmesg.txt complete" + dinfo "saving vmcore-dmesg.txt complete" else - echo "kdump: saving vmcore-dmesg.txt failed" + derror "saving vmcore-dmesg.txt failed" fi } @@ -182,12 +196,12 @@ get_host_ip() if is_nfs_dump_target || is_ssh_dump_target then kdumpnic=$(getarg kdumpnic=) - [ -z "$kdumpnic" ] && echo "kdump: failed to get kdumpnic!" && return 1 + [ -z "$kdumpnic" ] && derror "failed to get kdumpnic!" && return 1 _host=`ip addr show dev $kdumpnic|grep '[ ]*inet'` - [ $? -ne 0 ] && echo "kdump: wrong kdumpnic: $kdumpnic" && return 1 + [ $? -ne 0 ] && derror "wrong kdumpnic: $kdumpnic" && return 1 _host=`echo $_host | head -n 1 | cut -d' ' -f2` _host="${_host%%/*}" - [ -z "$_host" ] && echo "kdump: wrong kdumpnic: $kdumpnic" && return 1 + [ -z "$_host" ] && derror "wrong kdumpnic: $kdumpnic" && return 1 HOST_IP=$_host fi return 0 @@ -196,7 +210,7 @@ get_host_ip() read_kdump_conf() { if [ ! -f "$KDUMP_CONF" ]; then - echo "kdump: $KDUMP_CONF not found" + derror "$KDUMP_CONF not found" return fi @@ -240,7 +254,7 @@ fence_kdump_notify get_host_ip if [ $? -ne 0 ]; then - echo "kdump: get_host_ip exited with non-zero status!" + derror "get_host_ip exited with non-zero status!" exit 1 fi @@ -250,7 +264,7 @@ fi do_kdump_pre if [ $? -ne 0 ]; then - echo "kdump: kdump_pre script exited with non-zero status!" + derror "kdump_pre script exited with non-zero status!" do_final_action # During systemd service to reboot the machine, stop this shell script running exit 1 @@ -261,7 +275,7 @@ DUMP_RETVAL=$? do_kdump_post $DUMP_RETVAL if [ $? -ne 0 ]; then - echo "kdump: kdump_post script exited with non-zero status!" + derror "kdump_post script exited with non-zero status!" fi if [ $DUMP_RETVAL -ne 0 ]; then diff --git a/dracut-module-setup.sh b/dracut-module-setup.sh index a186570..911a8b8 100755 --- a/dracut-module-setup.sh +++ b/dracut-module-setup.sh @@ -799,13 +799,14 @@ kdump_install_systemd_conf() { echo "DefaultTimeoutStartSec=300s" >> ${initdir}/etc/systemd/system.conf.d/kdump.conf fi - # Forward logs to console directly, this avoids unneccessary memory - # consumption and make console output more useful. + # Forward logs to console directly, and don't read Kmsg, this avoids + # unneccessary memory consumption and make console output more useful. # Only do so for non fadump image. - if ! is_fadump_capable && [ "$failure_action" != "shell" ]; then + if ! is_fadump_capable; then mkdir -p ${initdir}/etc/systemd/journald.conf.d echo "[Journal]" > ${initdir}/etc/systemd/journald.conf.d/kdump.conf - echo "Storage=none" >> ${initdir}/etc/systemd/journald.conf.d/kdump.conf + echo "Storage=volatile" >> ${initdir}/etc/systemd/journald.conf.d/kdump.conf + echo "ReadKMsg=no" >> ${initdir}/etc/systemd/journald.conf.d/kdump.conf echo "ForwardToConsole=yes" >> ${initdir}/etc/systemd/journald.conf.d/kdump.conf fi } @@ -818,6 +819,7 @@ install() { kdump_install_random_seed fi dracut_install -o /etc/adjtime /etc/localtime + inst_simple "/etc/sysconfig/kdump" inst "$moddir/monitor_dd_progress" "/kdumpscripts/monitor_dd_progress" chmod +x ${initdir}/kdumpscripts/monitor_dd_progress inst "/bin/dd" "/bin/dd" @@ -830,8 +832,11 @@ install() { inst "/bin/sed" "/bin/sed" inst "/sbin/makedumpfile" "/sbin/makedumpfile" inst "/sbin/vmcore-dmesg" "/sbin/vmcore-dmesg" + inst "/usr/bin/printf" "/sbin/printf" + inst "/usr/bin/logger" "/sbin/logger" inst "/lib/kdump/kdump-lib.sh" "/lib/kdump-lib.sh" inst "/lib/kdump/kdump-lib-initramfs.sh" "/lib/kdump-lib-initramfs.sh" + inst "/lib/kdump/kdump-logger.sh" "/lib/kdump-logger.sh" inst "$moddir/kdump.sh" "/usr/bin/kdump.sh" inst "$moddir/kdump-capture.service" "$systemdsystemunitdir/kdump-capture.service" mkdir -p "$initdir/$systemdsystemunitdir/initrd.target.wants" diff --git a/kdump-lib-initramfs.sh b/kdump-lib-initramfs.sh index c98a43b..14aac7b 100755 --- a/kdump-lib-initramfs.sh +++ b/kdump-lib-initramfs.sh @@ -1,10 +1,12 @@ # These variables and functions are useful in 2nd kernel +. /etc/sysconfig/kdump . /lib/kdump-lib.sh KDUMP_PATH="/var/crash" +KDUMP_LOG_FILE="/run/initramfs/kexec-dmesg.log" CORE_COLLECTOR="" -DEFAULT_CORE_COLLECTOR="makedumpfile -l --message-level 1 -d 31" +DEFAULT_CORE_COLLECTOR="makedumpfile -l --message-level 7 -d 31" DMESG_COLLECTOR="/sbin/vmcore-dmesg" FAILURE_ACTION="systemctl reboot -f" DATEDIR=`date +%Y-%m-%d-%T` @@ -20,6 +22,13 @@ KDUMP_POST="" NEWROOT="/sysroot" OPALCORE="/sys/firmware/opal/mpipl/core" +#initiate the kdump logger +dlog_init +if [ $? -ne 0 ]; then + echo "failed to initiate the kdump logger." + exit 1 +fi + get_kdump_confs() { local config_opt config_val @@ -94,27 +103,40 @@ get_kdump_confs() fi } +# store the kexec kernel log to a file. +save_log() +{ + dmesg -T > $KDUMP_LOG_FILE + + if command -v journalctl > /dev/null; then + journalctl -ab >> $KDUMP_LOG_FILE + fi +} + # dump_fs dump_fs() { + local ret local _mp=$1 local _dev=$(get_mount_info SOURCE target $_mp -f) local _op=$(get_mount_info OPTIONS target $_mp -f) + ddebug "_mp=$_mp _dev=$_dev _op=$_op" + # If dump path have a corresponding device entry but not mounted, mount it. if [ -n "$_dev" ]; then if ! is_mounted "$_mp"; then - echo "kdump: dump target $_dev is not mounted, trying to mount..." + dinfo "dump target $_dev is not mounted, trying to mount..." mkdir -p $_mp mount -o $_op $_dev $_mp if [ $? -ne 0 ]; then - echo "kdump: mounting failed (mount point: $_mp, option: $_op)" + derror "mounting failed (mount point: $_mp, option: $_op)" return 1 fi fi else - echo "kdump: failed to dump to \"$_mp\", it's not a mount point!" + derror "failed to dump to \"$_mp\", it's not a mount point!" return 1 fi @@ -123,11 +145,11 @@ dump_fs() local _dump_path=$(echo "$_mp/$KDUMP_PATH/$HOST_IP-$DATEDIR/" | tr -s /) - echo "kdump: saving to $_dump_path" + dinfo "saving to $_dump_path" # Only remount to read-write mode if the dump target is mounted read-only. if [[ "$_op" = "ro"* ]]; then - echo "kdump: Mounting Dump target $_dev in rw mode." + dinfo "Mounting Dump target $_dev in rw mode." mount -o remount,rw $_dev $_mp || return 1 fi @@ -136,12 +158,18 @@ dump_fs() save_vmcore_dmesg_fs ${DMESG_COLLECTOR} "$_dump_path" save_opalcore_fs "$_dump_path" - echo "kdump: saving vmcore" - $CORE_COLLECTOR /proc/vmcore $_dump_path/vmcore-incomplete || return 1 + dinfo "saving vmcore" + $CORE_COLLECTOR /proc/vmcore $_dump_path/vmcore-incomplete + ret=$? + save_log + mv $KDUMP_LOG_FILE $_dump_path/ + if [ $ret -ne 0 ]; then + return 1 + fi mv $_dump_path/vmcore-incomplete $_dump_path/vmcore sync - echo "kdump: saving vmcore complete" + dinfo "saving vmcore complete" # improper kernel cmdline can cause the failure of echo, we can ignore this kind of failure return 0 @@ -151,7 +179,7 @@ save_vmcore_dmesg_fs() { local _dmesg_collector=$1 local _path=$2 - echo "kdump: saving vmcore-dmesg.txt" + dinfo "saving vmcore-dmesg.txt to ${_path}" $_dmesg_collector /proc/vmcore > ${_path}/vmcore-dmesg-incomplete.txt _exitcode=$? if [ $_exitcode -eq 0 ]; then @@ -161,9 +189,9 @@ save_vmcore_dmesg_fs() { # saving vmcore failed and system rebooted without sync and there # was no vmcore-dmesg.txt available. sync - echo "kdump: saving vmcore-dmesg.txt complete" + dinfo "saving vmcore-dmesg.txt complete" else - echo "kdump: saving vmcore-dmesg.txt failed" + derror "saving vmcore-dmesg.txt failed" fi } @@ -179,43 +207,47 @@ save_opalcore_fs() { fi fi - echo "kdump: saving opalcore" + dinfo "saving opalcore:$OPALCORE to ${_path}/opalcore" cp $OPALCORE ${_path}/opalcore if [ $? -ne 0 ]; then - echo "kdump: saving opalcore failed" + derror "saving opalcore failed" return 1 fi sync - echo "kdump: saving opalcore complete" + dinfo "saving opalcore complete" return 0 } dump_to_rootfs() { - echo "Kdump: trying to bring up rootfs device" + dinfo "Trying to bring up rootfs device" systemctl start dracut-initqueue - echo "Kdump: waiting for rootfs mount, will timeout after 90 seconds" + dinfo "Waiting for rootfs mount, will timeout after 90 seconds" systemctl start sysroot.mount + ddebug "NEWROOT=$NEWROOT" + dump_fs $NEWROOT } kdump_emergency_shell() { echo "PS1=\"kdump:\\\${PWD}# \"" >/etc/profile + ddebug "Switching to dracut emergency..." /bin/dracut-emergency rm -f /etc/profile } do_failure_action() { - echo "Kdump: Executing failure action $FAILURE_ACTION" + dinfo "Executing failure action $FAILURE_ACTION" eval $FAILURE_ACTION } do_final_action() { + dinfo "Executing final action $FINAL_ACTION" eval $FINAL_ACTION } diff --git a/kdump-lib.sh b/kdump-lib.sh index e18c473..b55d987 100755 --- a/kdump-lib.sh +++ b/kdump-lib.sh @@ -8,6 +8,12 @@ FENCE_KDUMP_CONFIG_FILE="/etc/sysconfig/fence_kdump" FENCE_KDUMP_SEND="/usr/libexec/fence_kdump_send" FADUMP_ENABLED_SYS_NODE="/sys/kernel/fadump_enabled" +if [ -f /lib/kdump/kdump-logger.sh ]; then + . /lib/kdump/kdump-logger.sh +elif [ -f /lib/kdump-logger.sh ]; then + . /lib/kdump-logger.sh +fi + is_fadump_capable() { # Check if firmware-assisted dump is enabled @@ -20,14 +26,10 @@ is_fadump_capable() } perror_exit() { - echo $@ >&2 + derror "$@" exit 1 } -perror() { - echo $@ >&2 -} - is_fs_type_nfs() { [ "$1" = "nfs" ] || [ "$1" = "nfs4" ] @@ -503,7 +505,7 @@ check_crash_mem_reserved() mem_reserved=$(cat /sys/kernel/kexec_crash_size) if [ $mem_reserved -eq 0 ]; then - echo "No memory reserved for crash kernel" + derror "No memory reserved for crash kernel" return 1 fi @@ -513,7 +515,7 @@ check_crash_mem_reserved() check_kdump_feasibility() { if [ ! -e /sys/kernel/kexec_crash_loaded ]; then - echo "Kdump is not supported on this kernel" + derror "Kdump is not supported on this kernel" return 1 fi check_crash_mem_reserved @@ -523,7 +525,7 @@ check_kdump_feasibility() check_current_kdump_status() { if [ ! -f /sys/kernel/kexec_crash_loaded ];then - echo "Perhaps CONFIG_CRASH_DUMP is not enabled in kernel" + derror "Perhaps CONFIG_CRASH_DUMP is not enabled in kernel" return 1 fi @@ -612,6 +614,7 @@ is_secure_boot_enforced() return 0 fi + # Detect secure boot on x86 and arm64 secure_boot_file=$(find /sys/firmware/efi/efivars -name SecureBoot-* 2>/dev/null) setup_mode_file=$(find /sys/firmware/efi/efivars -name SetupMode-* 2>/dev/null) @@ -624,6 +627,11 @@ is_secure_boot_enforced() fi fi + # Detect secure boot on s390x + if [[ -e "/sys/firmware/ipl/secure" && "$(cat /sys/firmware/ipl/secure)" == "1" ]]; then + return 0 + fi + return 1 } @@ -645,8 +653,7 @@ prepare_kexec_args() found_elf_args=`echo $kexec_args | grep elf32-core-headers` if [ -n "$found_elf_args" ] then - echo -n "Warning: elf32-core-headers overrides correct elf64 setting" - echo + dwarn "Warning: elf32-core-headers overrides correct elf64 setting" else kexec_args="$kexec_args --elf64-core-headers" fi @@ -697,7 +704,7 @@ prepare_kdump_bootinfo() done if ! [ -e "$KDUMP_KERNEL" ]; then - echo "Failed to detect kdump kernel location" + derror "Failed to detect kdump kernel location" return 1 fi diff --git a/kdump-logger.sh b/kdump-logger.sh new file mode 100755 index 0000000..c7afa34 --- /dev/null +++ b/kdump-logger.sh @@ -0,0 +1,334 @@ +#!/bin/bash +# +# This comes from the dracut-logger.sh +# +# The logger defined 6 logging levels: +# - dtrace() (6) +# The TRACE Level designates finer-grained informational events than the +# DEBUG. +# - ddebug (5) +# The DEBUG Level designates fine-grained informational events that are most +# useful to debug an application. +# - dinfo (4) +# The INFO level designates informational messages that highlight the +# progress of the application at coarse-grained level. +# - dwarn (3) +# The WARN level designates potentially harmful situations. +# - derror (2) +# The ERROR level designates error events that might still allow the +# application to continue running. +# - dfatal (1) +# The FATAL level designates very severe error events that will presumably +# lead the application to abort. +# +# Logging is controlled by following global variables: +# - @var kdump_stdloglvl - logging level to standard error (console output) +# - @var kdump_sysloglvl - logging level to syslog (by logger command) +# - @var kdump_kmsgloglvl - logging level to /dev/kmsg (only for boot-time) +# +# If any of the variables is not set, this function set it to default: +# - @var kdump_stdloglvl = 4 (info) +# - @var kdump_sysloglvl = 4 (info) +# - @var kdump_kmsgloglvl = 0 (no logging) +# +# First of all you have to start with dlog_init() function which initializes +# required variables. Don't call any other logging function before that one! +# +# @brief Initializes Logger. +# @retval 1 if something has gone wrong +# @retval 0 on success. +# +dlog_init() { + local ret=0; local errmsg + [ -z "$kdump_stdloglvl" ] && kdump_stdloglvl=4 + [ -z "$kdump_sysloglvl" ] && kdump_sysloglvl=4 + [ -z "$kdump_kmsgloglvl" ] && kdump_kmsgloglvl=0 + # Skip initialization if it's already done. + [ -n "$kdump_maxloglvl" ] && return 0 + + if [[ $UID -ne 0 ]]; then + kdump_kmsgloglvl=0 + kdump_sysloglvl=0 + fi + + if [[ $kdump_sysloglvl -gt 0 ]]; then + if [[ -d /run/systemd/journal ]] \ + && type -P systemd-cat &>/dev/null \ + && systemctl --quiet is-active systemd-journald.socket &>/dev/null; then + readonly _systemdcatfile="/var/tmp/systemd-cat" + mkfifo "$_systemdcatfile" &>/dev/null + readonly _dlogfd=15 + systemd-cat -t 'kdump' --level-prefix=true <"$_systemdcatfile" & + exec 15>"$_systemdcatfile" + elif ! [ -S /dev/log -a -w /dev/log ] || ! command -v logger >/dev/null; then + # We cannot log to syslog, so turn this facility off. + kdump_kmsgloglvl=$kdump_sysloglvl + kdump_sysloglvl=0 + ret=1 + errmsg="No '/dev/log' or 'logger' included for syslog logging" + fi + fi + + local lvl; local maxloglvl_l=0 + for lvl in $kdump_stdloglvl $kdump_sysloglvl $kdump_kmsgloglvl; do + [[ $lvl -gt $maxloglvl_l ]] && maxloglvl_l=$lvl + done + readonly kdump_maxloglvl=$maxloglvl_l + export kdump_maxloglvl + + + if [[ $kdump_stdloglvl -lt 6 ]] && [[ $kdump_kmsgloglvl -lt 6 ]] && [[ $kdump_sysloglvl -lt 6 ]]; then + unset dtrace + dtrace() { :; }; + fi + + if [[ $kdump_stdloglvl -lt 5 ]] && [[ $kdump_kmsgloglvl -lt 5 ]] && [[ $kdump_sysloglvl -lt 5 ]]; then + unset ddebug + ddebug() { :; }; + fi + + if [[ $kdump_stdloglvl -lt 4 ]] && [[ $kdump_kmsgloglvl -lt 4 ]] && [[ $kdump_sysloglvl -lt 4 ]]; then + unset dinfo + dinfo() { :; }; + fi + + if [[ $kdump_stdloglvl -lt 3 ]] && [[ $kdump_kmsgloglvl -lt 3 ]] && [[ $kdump_sysloglvl -lt 3 ]]; then + unset dwarn + dwarn() { :; }; + unset dwarning + dwarning() { :; }; + fi + + if [[ $kdump_stdloglvl -lt 2 ]] && [[ $kdump_kmsgloglvl -lt 2 ]] && [[ $kdump_sysloglvl -lt 2 ]]; then + unset derror + derror() { :; }; + fi + + if [[ $kdump_stdloglvl -lt 1 ]] && [[ $kdump_kmsgloglvl -lt 1 ]] && [[ $kdump_sysloglvl -lt 1 ]]; then + unset dfatal + dfatal() { :; }; + fi + + [ -n "$errmsg" ] && derror "$errmsg" + + return $ret +} + +## @brief Converts numeric logging level to the first letter of level name. +# +# @param lvl Numeric logging level in range from 1 to 6. +# @retval 1 if @a lvl is out of range. +# @retval 0 if @a lvl is correct. +# @result Echoes first letter of level name. +_lvl2char() { + case "$1" in + 1) echo F;; + 2) echo E;; + 3) echo W;; + 4) echo I;; + 5) echo D;; + 6) echo T;; + *) return 1;; + esac +} + +## @brief Converts numeric level to logger priority defined by POSIX.2. +# +# @param lvl Numeric logging level in range from 1 to 6. +# @retval 1 if @a lvl is out of range. +# @retval 0 if @a lvl is correct. +# @result Echoes logger priority. +_lvl2syspri() { + case "$1" in + 1) echo crit;; + 2) echo error;; + 3) echo warning;; + 4) echo info;; + 5) echo debug;; + 6) echo debug;; + *) return 1;; + esac +} + +## @brief Converts logger numeric level to syslog log level +# +# @param lvl Numeric logging level in range from 1 to 6. +# @retval 1 if @a lvl is out of range. +# @retval 0 if @a lvl is correct. +# @result Echoes kernel console numeric log level +# +# Conversion is done as follows: +# +# +# none -> LOG_EMERG (0) +# none -> LOG_ALERT (1) +# FATAL(1) -> LOG_CRIT (2) +# ERROR(2) -> LOG_ERR (3) +# WARN(3) -> LOG_WARNING (4) +# none -> LOG_NOTICE (5) +# INFO(4) -> LOG_INFO (6) +# DEBUG(5) -> LOG_DEBUG (7) +# TRACE(6) / +# +# +# @see /usr/include/sys/syslog.h +_dlvl2syslvl() { + local lvl + + case "$1" in + 1) lvl=2;; + 2) lvl=3;; + 3) lvl=4;; + 4) lvl=6;; + 5) lvl=7;; + 6) lvl=7;; + *) return 1;; + esac + + [ -s /proc/vmcore ] && echo $((24+$lvl)) || echo $((8+$lvl)) +} + +## @brief Prints to stderr and/or writes to file, to syslog and/or /dev/kmsg +# given message with given level (priority). +# +# @param lvl Numeric logging level. +# @param msg Message. +# @retval 0 It's always returned, even if logging failed. +# +# @note This function is not supposed to be called manually. Please use +# dtrace(), ddebug(), or others instead which wrap this one. +# +# This is core logging function which logs given message to standard error, file +# and/or syslog (with POSIX shell command logger) and/or to /dev/kmsg. +# The format is following: +# +# X: some message +# +# where @c X is the first letter of logging level. See module description for +# details on that. +# +# Message to syslog is sent with tag @c kdump. Priorities are mapped as +# following: +# - @c FATAL to @c crit +# - @c ERROR to @c error +# - @c WARN to @c warning +# - @c INFO to @c info +# - @c DEBUG and @c TRACE both to @c debug +_do_dlog() { + local lvl="$1"; shift + local lvlc=$(_lvl2char "$lvl") || return 0 + local msg="$*" + local lmsg="$lvlc: $*" + + [[ $lvl -le $kdump_stdloglvl ]] && printf -- 'kdump: %s\n' "$msg" >&2 + + if [[ $lvl -le $kdump_sysloglvl ]]; then + if [[ "$_dlogfd" ]]; then + printf -- "<%s>%s\n" "$(($(_dlvl2syslvl $lvl) & 7))" "$msg" >&$_dlogfd + else + logger -t "kdump[$$]" -p $(_lvl2syspri $lvl) -- "$msg" + fi + fi + + [[ $lvl -le $kdump_kmsgloglvl ]] && \ + echo "<$(_dlvl2syslvl $lvl)>kdump[$$] $msg" >/dev/kmsg +} + +## @brief Internal helper function for _do_dlog() +# +# @param lvl Numeric logging level. +# @param msg Message. +# @retval 0 It's always returned, even if logging failed. +# +# @note This function is not supposed to be called manually. Please use +# dtrace(), ddebug(), or others instead which wrap this one. +# +# This function calls _do_dlog() either with parameter msg, or if +# none is given, it will read standard input and will use every line as +# a message. +# +# This enables: +# dwarn "This is a warning" +# echo "This is a warning" | dwarn +dlog() { + [ -z "$kdump_maxloglvl" ] && return 0 + [[ $1 -le $kdump_maxloglvl ]] || return 0 + + if [[ $# -gt 1 ]]; then + _do_dlog "$@" + else + while read line || [ -n "$line" ]; do + _do_dlog "$1" "$line" + done + fi +} + +## @brief Logs message at TRACE level (6) +# +# @param msg Message. +# @retval 0 It's always returned, even if logging failed. +dtrace() { + set +x + dlog 6 "$@" + [ -n "$debug" ] && set -x || : +} + +## @brief Logs message at DEBUG level (5) +# +# @param msg Message. +# @retval 0 It's always returned, even if logging failed. +ddebug() { + set +x + dlog 5 "$@" + [ -n "$debug" ] && set -x || : +} + +## @brief Logs message at INFO level (4) +# +# @param msg Message. +# @retval 0 It's always returned, even if logging failed. +dinfo() { + set +x + dlog 4 "$@" + [ -n "$debug" ] && set -x || : +} + +## @brief Logs message at WARN level (3) +# +# @param msg Message. +# @retval 0 It's always returned, even if logging failed. +dwarn() { + set +x + dlog 3 "$@" + [ -n "$debug" ] && set -x || : +} + +## @brief It's an alias to dwarn() function. +# +# @param msg Message. +# @retval 0 It's always returned, even if logging failed. +dwarning() { + set +x + dwarn "$@" + [ -n "$debug" ] && set -x || : +} + +## @brief Logs message at ERROR level (2) +# +# @param msg Message. +# @retval 0 It's always returned, even if logging failed. +derror() { + set +x + dlog 2 "$@" + [ -n "$debug" ] && set -x || : +} + +## @brief Logs message at FATAL level (1) +# +# @param msg Message. +# @retval 0 It's always returned, even if logging failed. +dfatal() { + set +x + dlog 1 "$@" + [ -n "$debug" ] && set -x || : +} diff --git a/kdump.conf b/kdump.conf index 9fc0a4f..e4db52e 100644 --- a/kdump.conf +++ b/kdump.conf @@ -62,9 +62,9 @@ # as the initrd will automatically be populated with a # config file appropriate for the running kernel. # The default core_collector for raw/ssh dump is: -# "makedumpfile -F -l --message-level 1 -d 31". +# "makedumpfile -F -l --message-level 7 -d 31". # The default core_collector for other targets is: -# "makedumpfile -l --message-level 1 -d 31". +# "makedumpfile -l --message-level 7 -d 31". # # "makedumpfile -F" will create a flattened vmcore. # You need to use "makedumpfile -R" to rearrange the dump data to @@ -169,7 +169,7 @@ #ssh user@my.server.com #sshkey /root/.ssh/kdump_id_rsa path /var/crash -core_collector makedumpfile -l --message-level 1 -d 31 +core_collector makedumpfile -l --message-level 7 -d 31 #core_collector scp #kdump_post /var/crash/scripts/kdump-post.sh #kdump_pre /var/crash/scripts/kdump-pre.sh diff --git a/kdump.conf.5 b/kdump.conf.5 index 34c996c..ae07466 100644 --- a/kdump.conf.5 +++ b/kdump.conf.5 @@ -41,7 +41,7 @@ mount point. .B ssh .RS -Will scp /proc/vmcore to :/%HOST-%DATE/, +Will save /proc/vmcore through ssh pipe to :/%HOST-%DATE/, supports DNS. NOTE: make sure user has necessary write permissions on server and that a fqdn is used as the server name. .RE @@ -85,9 +85,9 @@ for the running kernel. .PP Note 1: About default core collector: The default core_collector for raw/ssh dump is: -"makedumpfile -F -l --message-level 1 -d 31". +"makedumpfile -F -l --message-level 7 -d 31". The default core_collector for other targets is: -"makedumpfile -l --message-level 1 -d 31". +"makedumpfile -l --message-level 7 -d 31". Even if core_collector option is commented out in kdump.conf, makedumpfile is the default core collector and kdump uses it internally. If one does not want makedumpfile as default core_collector, then they @@ -307,11 +307,11 @@ Above will effectively be translated to: cp --sparse=always /proc/vmcore /vmcore .TP ex2. -core_collector "makedumpfile -l --message-level 1 -d 31" +core_collector "makedumpfile -l --message-level 7 -d 31" Above will effectively be translated to: -makedumpfile -l --message-level 1 -d 31 /proc/vmcore /vmcore +makedumpfile -l --message-level 7 -d 31 /proc/vmcore /vmcore .PP For dump targets like raw and ssh, in general, core collector should expect one argument (source file) and should output the processed core on standard @@ -328,11 +328,11 @@ Above will effectively be translated to. cat /proc/vmcore | dd of= .TP ex4. -core_collector "makedumpfile -F -l --message-level 1 -d 31" +core_collector "makedumpfile -F -l --message-level 7 -d 31" Above will effectively be translated to. -makedumpfile -F -l --message-level 1 -d 31 | dd of= +makedumpfile -F -l --message-level 7 -d 31 | dd of= .PP ssh dumps examples .TP @@ -344,11 +344,11 @@ Above will effectively be translated to. cat /proc/vmcore | ssh "dd of=path/vmcore" .TP ex6. -core_collector "makedumpfile -F -l --message-level 1 -d 31" +core_collector "makedumpfile -F -l --message-level 7 -d 31" Above will effectively be translated to. -makedumpfile -F -l --message-level 1 -d 31 | ssh "dd of=path/vmcore" +makedumpfile -F -l --message-level 7 -d 31 | ssh "dd of=path/vmcore" There is one exception to standard output rule for ssh dumps. And that is scp. As scp can handle ssh destinations for file transfers, one can diff --git a/kdump.service b/kdump.service index f888dd6..99feed8 100644 --- a/kdump.service +++ b/kdump.service @@ -2,6 +2,7 @@ Description=Crash recovery kernel arming After=network.target network-online.target remote-fs.target basic.target DefaultDependencies=no +ConditionKernelCommandLine=crashkernel [Service] Type=oneshot diff --git a/kdump.sysconfig b/kdump.sysconfig index ffc387a..52a0b92 100644 --- a/kdump.sysconfig +++ b/kdump.sysconfig @@ -35,3 +35,19 @@ KEXEC_ARGS="" #What is the image type used for kdump KDUMP_IMG="vmlinuz" + +# Logging is controlled by following global variables: +# - @var kdump_stdloglvl - logging level to standard error (console output) +# - @var kdump_sysloglvl - logging level to syslog (by logger command) +# - @var kdump_kmsgloglvl - logging level to /dev/kmsg (only for boot-time) +# If any of the variables is not set, this function set it to default: +# - @var kdump_stdloglvl=4 (info) +# - @var kdump_sysloglvl=4 (info) +# - @var kdump_kmsgloglvl=0 (no logging) +# +# Logging levels: fatal(1),error(2),warn(3),info(4),debug(5),trace(6) +# +# For example: the following configurations indicate that kdump messages will be printed +# to console and journald for debugging case. +# kdump_sysloglvl=5 +# kdump_stdloglvl=5 diff --git a/kdump.sysconfig.aarch64 b/kdump.sysconfig.aarch64 index 6f55c03..1d75979 100644 --- a/kdump.sysconfig.aarch64 +++ b/kdump.sysconfig.aarch64 @@ -35,3 +35,19 @@ KEXEC_ARGS="" #What is the image type used for kdump KDUMP_IMG="vmlinuz" + +# Logging is controlled by following global variables: +# - @var kdump_stdloglvl - logging level to standard error (console output) +# - @var kdump_sysloglvl - logging level to syslog (by logger command) +# - @var kdump_kmsgloglvl - logging level to /dev/kmsg (only for boot-time) +# If any of the variables is not set, this function set it to default: +# - @var kdump_stdloglvl=4 (info) +# - @var kdump_sysloglvl=4 (info) +# - @var kdump_kmsgloglvl=0 (no logging) +# +# Logging levels: fatal(1),error(2),warn(3),info(4),debug(5),trace(6) +# +# For example: the following configurations indicate that kdump messages will be printed +# to console and journald for debugging case. +# kdump_sysloglvl=5 +# kdump_stdloglvl=5 diff --git a/kdump.sysconfig.i386 b/kdump.sysconfig.i386 index 45f6d50..2eb9705 100644 --- a/kdump.sysconfig.i386 +++ b/kdump.sysconfig.i386 @@ -38,3 +38,19 @@ KDUMP_IMG="vmlinuz" #What is the images extension. Relocatable kernels don't have one KDUMP_IMG_EXT="" + +# Logging is controlled by following global variables: +# - @var kdump_stdloglvl - logging level to standard error (console output) +# - @var kdump_sysloglvl - logging level to syslog (by logger command) +# - @var kdump_kmsgloglvl - logging level to /dev/kmsg (only for boot-time) +# If any of the variables is not set, this function set it to default: +# - @var kdump_stdloglvl=4 (info) +# - @var kdump_sysloglvl=4 (info) +# - @var kdump_kmsgloglvl=0 (no logging) +# +# Logging levels: fatal(1),error(2),warn(3),info(4),debug(5),trace(6) +# +# For example: the following configurations indicate that kdump messages will be printed +# to console and journald for debugging case. +# kdump_sysloglvl=5 +# kdump_stdloglvl=5 diff --git a/kdump.sysconfig.ppc64 b/kdump.sysconfig.ppc64 index 8dcbe60..24279ff 100644 --- a/kdump.sysconfig.ppc64 +++ b/kdump.sysconfig.ppc64 @@ -40,3 +40,19 @@ KDUMP_IMG="vmlinuz" KDUMP_IMG_EXT="" #Specify the action after failure + +# Logging is controlled by following global variables: +# - @var kdump_stdloglvl - logging level to standard error (console output) +# - @var kdump_sysloglvl - logging level to syslog (by logger command) +# - @var kdump_kmsgloglvl - logging level to /dev/kmsg (only for boot-time) +# If any of the variables is not set, this function set it to default: +# - @var kdump_stdloglvl=4 (info) +# - @var kdump_sysloglvl=4 (info) +# - @var kdump_kmsgloglvl=0 (no logging) +# +# Logging levels: fatal(1),error(2),warn(3),info(4),debug(5),trace(6) +# +# For example: the following configurations indicate that kdump messages will be printed +# to console and journald for debugging case. +# kdump_sysloglvl=5 +# kdump_stdloglvl=5 diff --git a/kdump.sysconfig.ppc64le b/kdump.sysconfig.ppc64le index 8dcbe60..24279ff 100644 --- a/kdump.sysconfig.ppc64le +++ b/kdump.sysconfig.ppc64le @@ -40,3 +40,19 @@ KDUMP_IMG="vmlinuz" KDUMP_IMG_EXT="" #Specify the action after failure + +# Logging is controlled by following global variables: +# - @var kdump_stdloglvl - logging level to standard error (console output) +# - @var kdump_sysloglvl - logging level to syslog (by logger command) +# - @var kdump_kmsgloglvl - logging level to /dev/kmsg (only for boot-time) +# If any of the variables is not set, this function set it to default: +# - @var kdump_stdloglvl=4 (info) +# - @var kdump_sysloglvl=4 (info) +# - @var kdump_kmsgloglvl=0 (no logging) +# +# Logging levels: fatal(1),error(2),warn(3),info(4),debug(5),trace(6) +# +# For example: the following configurations indicate that kdump messages will be printed +# to console and journald for debugging case. +# kdump_sysloglvl=5 +# kdump_stdloglvl=5 diff --git a/kdump.sysconfig.s390x b/kdump.sysconfig.s390x index 2a24688..5e1cb5a 100644 --- a/kdump.sysconfig.s390x +++ b/kdump.sysconfig.s390x @@ -31,7 +31,7 @@ MKDUMPRD_ARGS="" # # Example: # KEXEC_ARGS="--elf32-core-headers" -KEXEC_ARGS="" +KEXEC_ARGS="-s" #Where to find the boot image #KDUMP_BOOTDIR="/boot" @@ -41,3 +41,19 @@ KDUMP_IMG="vmlinuz" #What is the images extension. Relocatable kernels don't have one KDUMP_IMG_EXT="" + +# Logging is controlled by following global variables: +# - @var kdump_stdloglvl - logging level to standard error (console output) +# - @var kdump_sysloglvl - logging level to syslog (by logger command) +# - @var kdump_kmsgloglvl - logging level to /dev/kmsg (only for boot-time) +# If any of the variables is not set, this function set it to default: +# - @var kdump_stdloglvl=4 (info) +# - @var kdump_sysloglvl=4 (info) +# - @var kdump_kmsgloglvl=0 (no logging) +# +# Logging levels: fatal(1),error(2),warn(3),info(4),debug(5),trace(6) +# +# For example: the following configurations indicate that kdump messages will be printed +# to console and journald for debugging case. +# kdump_sysloglvl=5 +# kdump_stdloglvl=5 diff --git a/kdump.sysconfig.x86_64 b/kdump.sysconfig.x86_64 index f67d999..eb1c538 100644 --- a/kdump.sysconfig.x86_64 +++ b/kdump.sysconfig.x86_64 @@ -38,3 +38,19 @@ KDUMP_IMG="vmlinuz" #What is the images extension. Relocatable kernels don't have one KDUMP_IMG_EXT="" + +# Logging is controlled by following global variables: +# - @var kdump_stdloglvl - logging level to standard error (console output) +# - @var kdump_sysloglvl - logging level to syslog (by logger command) +# - @var kdump_kmsgloglvl - logging level to /dev/kmsg (only for boot-time) +# If any of the variables is not set, this function set it to default: +# - @var kdump_stdloglvl=4 (info) +# - @var kdump_sysloglvl=4 (info) +# - @var kdump_kmsgloglvl=0 (no logging) +# +# Logging levels: fatal(1),error(2),warn(3),info(4),debug(5),trace(6) +# +# For example: the following configurations indicate that kdump messages will be printed +# to console and journald for debugging case. +# kdump_sysloglvl=5 +# kdump_stdloglvl=5 diff --git a/kdumpctl b/kdumpctl index 2248da4..e816719 100755 --- a/kdumpctl +++ b/kdumpctl @@ -6,6 +6,7 @@ KDUMP_KERNEL="" KDUMP_COMMANDLINE="" KEXEC_ARGS="" KDUMP_CONFIG_FILE="/etc/kdump.conf" +KDUMP_LOG_PATH="/var/log" MKDUMPRD="/sbin/mkdumprd -f" DRACUT_MODULES_FILE="/usr/lib/dracut/modules.txt" SAVE_PATH=/var/crash @@ -21,11 +22,7 @@ FADUMP_REGISTER_SYS_NODE="/sys/kernel/fadump_registered" DEFAULT_DUMP_MODE="kdump" image_time=0 -[[ $dracutbasedir ]] || dracutbasedir=/usr/lib/dracut -. $dracutbasedir/dracut-functions.sh -. /lib/kdump/kdump-lib.sh - -standard_kexec_args="-p" +standard_kexec_args="-d -p" # Some default values in case /etc/sysconfig/kdump doesn't include KDUMP_COMMANDLINE_REMOVE="hugepages hugepagesz slub_debug" @@ -34,13 +31,24 @@ if [ -f /etc/sysconfig/kdump ]; then . /etc/sysconfig/kdump fi +[[ $dracutbasedir ]] || dracutbasedir=/usr/lib/dracut +. $dracutbasedir/dracut-functions.sh +. /lib/kdump/kdump-lib.sh + +#initiate the kdump logger +dlog_init +if [ $? -ne 0 ]; then + echo "failed to initiate the kdump logger." + exit 1 +fi + single_instance_lock() { local rc timeout=5 exec 9>/var/lock/kdump if [ $? -ne 0 ]; then - echo "Create file lock failed" + derror "Create file lock failed" exit 1 fi @@ -48,7 +56,7 @@ single_instance_lock() rc=$? while [ $rc -ne 0 ]; do - echo "Another app is currently holding the kdump lock; waiting for it to exit..." + dinfo "Another app is currently holding the kdump lock; waiting for it to exit..." flock -w $timeout 9 rc=$? done @@ -59,9 +67,10 @@ determine_dump_mode() # Check if firmware-assisted dump is enabled # if yes, set the dump mode as fadump if is_fadump_capable; then - echo "Dump mode is fadump" + dinfo "Dump mode is fadump" DEFAULT_DUMP_MODE="fadump" fi + ddebug "DEFAULT_DUMP_MODE=$DEFAULT_DUMP_MODE" } save_core() @@ -69,22 +78,25 @@ save_core() coredir="/var/crash/`date +"%Y-%m-%d-%H:%M"`" mkdir -p $coredir + ddebug "cp --sparse=always /proc/vmcore $coredir/vmcore-incomplete" cp --sparse=always /proc/vmcore $coredir/vmcore-incomplete if [ $? == 0 ]; then mv $coredir/vmcore-incomplete $coredir/vmcore - echo "saved a vmcore to $coredir" + dinfo "saved a vmcore to $coredir" else - echo "failed to save a vmcore to $coredir" >&2 + derror "failed to save a vmcore to $coredir" fi # pass the dmesg to Abrt tool if exists, in order # to collect the kernel oops message. # https://fedorahosted.org/abrt/ if [ -x /usr/bin/dumpoops ]; then + ddebug "makedumpfile --dump-dmesg $coredir/vmcore $coredir/dmesg" makedumpfile --dump-dmesg $coredir/vmcore $coredir/dmesg >/dev/null 2>&1 + ddebug "dumpoops -d $coredir/dmesg" dumpoops -d $coredir/dmesg >/dev/null 2>&1 if [ $? == 0 ]; then - echo "kernel oops has been collected by abrt tool" + dinfo "kernel oops has been collected by abrt tool" fi fi } @@ -96,16 +108,18 @@ rebuild_fadump_initrd() # this file tells the initrd is fadump enabled touch /tmp/fadump.initramfs target_initrd_tmp="$TARGET_INITRD.tmp" + ddebug "rebuild fadump initrd: $target_initrd_tmp $DEFAULT_INITRD_BAK $KDUMP_KERNELVER" $MKDUMPRD $target_initrd_tmp --rebuild $DEFAULT_INITRD_BAK --kver $KDUMP_KERNELVER \ -i /tmp/fadump.initramfs /etc/fadump.initramfs if [ $? != 0 ]; then - echo "mkdumprd: failed to rebuild initrd with fadump support" >&2 + derror "mkdumprd: failed to rebuild initrd with fadump support" rm -f /tmp/fadump.initramfs return 1 fi rm -f /tmp/fadump.initramfs # updating fadump initrd + ddebug "updating fadump initrd: $target_initrd_tmp $TARGET_INITRD" mv $target_initrd_tmp $TARGET_INITRD sync @@ -120,14 +134,15 @@ check_earlykdump_is_enabled() rebuild_kdump_initrd() { + ddebug "rebuild kdump initrd: $MKDUMPRD $TARGET_INITRD $KDUMP_KERNELVER" $MKDUMPRD $TARGET_INITRD $KDUMP_KERNELVER if [ $? != 0 ]; then - echo "mkdumprd: failed to make kdump initrd" >&2 + derror "mkdumprd: failed to make kdump initrd" return 1 fi if check_earlykdump_is_enabled; then - echo "Tips: If early kdump is enabled, also require rebuilding the system initramfs to make the changes take effect for early kdump." + dwarn "Tips: If early kdump is enabled, also require rebuilding the system initramfs to make the changes take effect for early kdump." fi return 0 @@ -136,7 +151,7 @@ rebuild_kdump_initrd() rebuild_initrd() { if [[ ! -w "$KDUMP_BOOTDIR" ]];then - echo "$KDUMP_BOOTDIR does not have write permission. Can not rebuild $TARGET_INITRD" + derror "$KDUMP_BOOTDIR does not have write permission. Can not rebuild $TARGET_INITRD" return 1 fi @@ -154,7 +169,7 @@ check_exist() { for file in $1; do if [ ! -e "$file" ]; then - echo -n "Error: $file not found."; echo + derror "Error: $file not found." return 1 fi done @@ -165,7 +180,7 @@ check_executable() { for file in $1; do if [ ! -x "$file" ]; then - echo -n "Error: $file is not executable."; echo + derror "Error: $file is not executable." return 1 fi done @@ -173,17 +188,19 @@ check_executable() backup_default_initrd() { + ddebug "backup default initrd: $DEFAULT_INITRD" + if [ ! -f "$DEFAULT_INITRD" ]; then return fi if [ ! -e $DEFAULT_INITRD_BAK ]; then - echo "Backing up $DEFAULT_INITRD before rebuild." + dinfo "Backing up $DEFAULT_INITRD before rebuild." # save checksum to verify before restoring sha1sum $DEFAULT_INITRD > $INITRD_CHECKSUM_LOCATION cp $DEFAULT_INITRD $DEFAULT_INITRD_BAK if [ $? -ne 0 ]; then - echo "WARNING: failed to backup $DEFAULT_INITRD." + dwarn "WARNING: failed to backup $DEFAULT_INITRD." rm -f $DEFAULT_INITRD_BAK fi fi @@ -191,6 +208,8 @@ backup_default_initrd() restore_default_initrd() { + ddebug "restore default initrd: $DEFAULT_INITRD" + if [ ! -f "$DEFAULT_INITRD" ]; then return fi @@ -202,13 +221,12 @@ restore_default_initrd() backup_checksum=`sha1sum $DEFAULT_INITRD_BAK | awk '{ print $1 }'` default_checksum=`cat $INITRD_CHECKSUM_LOCATION | awk '{ print $1 }'` if [ "$default_checksum" != "$backup_checksum" ]; then - echo "WARNING: checksum mismatch! Can't restore original initrd.." + dwarn "WARNING: checksum mismatch! Can't restore original initrd.." else rm -f $INITRD_CHECKSUM_LOCATION mv $DEFAULT_INITRD_BAK $DEFAULT_INITRD if [[ $? -eq 0 ]]; then - echo -n "Restoring original initrd as fadump mode " - echo "is disabled." + derror "Restoring original initrd as fadump mode is disabled." sync fi fi @@ -217,62 +235,57 @@ restore_default_initrd() check_config() { - local nr - - nr=$(awk 'BEGIN{cnt=0} /^raw|^ssh[[:blank:]]|^nfs|^ext[234]|^xfs|^btrfs|^minix|^dracut_args .*\-\-mount/{cnt++} END{print cnt}' $KDUMP_CONFIG_FILE) - [ $nr -gt 1 ] && { - echo "More than one dump targets specified." - return 1 - } - - # Check if path option is set more than once. - nr=$(awk 'BEGIN{cnt=0} /^path /{cnt++} END{print cnt}' $KDUMP_CONFIG_FILE) - [ $nr -gt 1 ] && { - echo "Mutiple paths specifed in $KDUMP_CONFIG_FILE" - return 1 - } - - nr=$(grep "^dracut_args .*\-\-mount" $KDUMP_CONFIG_FILE | grep -o "\-\-mount" | wc -l) - [ $nr -gt 1 ] && { - echo "Multiple mount targets specified in one \"dracut_args\"." - return 1 - } - - # Check if we have any leading spaces (or tabs) before the - # variable name in the kdump conf file - if grep -E -q '^[[:blank:]]+[a-z]' $KDUMP_CONFIG_FILE; then - echo "No whitespaces are allowed before a kdump option name in $KDUMP_CONFIG_FILE" - return 1 - fi - + local -A _opt_rec while read config_opt config_val; do + if [ -z "$config_val" ]; then + derror "Invalid kdump config value for option $config_opt" + return 1 + fi + case "$config_opt" in - \#* | "") - ;; - raw|ext2|ext3|ext4|minix|btrfs|xfs|nfs|ssh|sshkey|path|core_collector|kdump_post|kdump_pre|extra_bins|extra_modules|failure_action|default|final_action|force_rebuild|force_no_rebuild|dracut_args|fence_kdump_args|fence_kdump_nodes) - # remove inline comments after the end of a directive. - [ -z "$config_val" ] && { - echo "Invalid kdump config value for option $config_opt." - return 1; - } - if [ -d "/proc/device-tree/ibm,opal/dump" ] && [ "$config_opt" == "raw" ]; then - echo "WARNING: Won't capture opalcore when 'raw' dump target is used." + dracut_args) + if [[ $config_val == *--mount* ]]; then + if [ $(echo $config_val | grep -o "\-\-mount" | wc -l) -ne 1 ]; then + derror "Multiple mount targets specified in one \"dracut_args\"." + return 1 + fi + config_opt=_target fi ;; + raw) + if [ -d "/proc/device-tree/ibm,opal/dump" ]; then + dwarn "WARNING: Won't capture opalcore when 'raw' dump target is used." + fi + config_opt=_target + ;; + ext[234]|minix|btrfs|xfs|nfs|ssh) + config_opt=_target + ;; + sshkey|path|core_collector|kdump_post|kdump_pre|extra_bins|extra_modules|failure_action|default|final_action|force_rebuild|force_no_rebuild|fence_kdump_args|fence_kdump_nodes) + ;; net|options|link_delay|disk_timeout|debug_mem_level|blacklist) - echo "Deprecated kdump config option: $config_opt. Refer to kdump.conf manpage for alternatives." + derror "Deprecated kdump config option: $config_opt. Refer to kdump.conf manpage for alternatives." return 1 ;; *) - echo "Invalid kdump config option $config_opt" - return 1; + derror "Invalid kdump config option $config_opt" + return 1 ;; esac + + if [ -n "${_opt_rec[$config_opt]}" ]; then + if [ $config_opt == _target ]; then + derror "More than one dump targets specified" + else + derror "Duplicated kdump config value of option $config_opt" + fi + return 1 + fi + _opt_rec[$config_opt]="$config_val" done <<< "$(read_strip_comments $KDUMP_CONFIG_FILE)" check_failure_action_config || return 1 check_final_action_config || return 1 - check_fence_kdump_config || return 1 return 0 @@ -309,6 +322,7 @@ setup_initrd() { prepare_kdump_bootinfo if [ $? -ne 0 ]; then + derror "failed to prepare for kdump bootinfo." return 1 fi @@ -379,7 +393,7 @@ check_files_modified() else # If it's not a module nor builtin, give an error if ! ( modprobe --set-version "$KDUMP_KERNELVER" --dry-run "$_module" &>/dev/null ); then - echo "Module $_module not found" + dwarn "Module $_module not found" fi fi done @@ -404,13 +418,12 @@ check_files_modified() fi fi else - echo "$file doesn't exist" + dwarn "$file doesn't exist" fi done if [ -n "$modified_files" ]; then - echo "Detected change(s) in the following file(s):" - echo -n " "; echo "$modified_files" | sed 's/\s/\n /g' + dinfo "Detected change(s) in the following file(s): $modified_files" return 1 fi @@ -448,13 +461,16 @@ check_dump_fs_modified() _target=$(to_dev_name $_target) _new_fstype=$(get_fs_type_from_target $_target) if [[ -z "$_target" || -z "$_new_fstype" ]];then - echo "Dump path $_path does not exist" + derror "Dump path $_path does not exist" return 2 fi fi + ddebug "_target=$_target _path=$_path _new_fstype=$_new_fstype" + _record_block_drivers() { local _drivers + if [[ -b /dev/block/$1 ]]; then _drivers=$(udevadm info -a "/dev/block/$1" | sed -n 's/\s*DRIVERS=="\(\S\+\)"/\1/p') fi @@ -466,6 +482,7 @@ check_dump_fs_modified() _target_drivers="$_target_drivers $_driver" fi done + ddebug "MAJ:MIN=$1 _drivers=$_drivers _target_drivers=$_targer_drivers" return 1 } @@ -478,7 +495,7 @@ check_dump_fs_modified() continue fi if ! [[ " $_old_drivers " == *" $_module_name "* ]]; then - echo "Detected change in block device driver, new loaded module: $_module_name" + dinfo "Detected change in block device driver, new loaded module: $_module_name" return 1 fi done @@ -488,7 +505,7 @@ check_dump_fs_modified() else _new_dev=$(kdump_get_persistent_dev $_target) if [ -z "$_new_dev" ]; then - echo "Get persistent device name failed" + derror "Get persistent device name failed" return 2 fi fi @@ -496,7 +513,7 @@ check_dump_fs_modified() _new_mntpoint="$(get_kdump_mntpoint_from_target $_target)" _dracut_args=$(lsinitrd $TARGET_INITRD -f usr/lib/dracut/build-parameter.txt) if [[ -z "$_dracut_args" ]];then - echo "Warning: No dracut arguments found in initrd" + dwarn "Warning: No dracut arguments found in initrd" return 0 fi @@ -514,7 +531,7 @@ check_dump_fs_modified() [[ "$_target" = "$(get_root_fs_device)" ]] && return 0 fi - echo "Detected change in File System" + dinfo "Detected change in File System" return 1 } @@ -597,7 +614,7 @@ check_system_modified() check_wdt_modified if [ $? -ne 0 ]; then - echo "Detected change in watchdog state" + dinfo "Detected change in watchdog state" return 1 fi @@ -621,7 +638,7 @@ check_rebuild() if [ $? -eq 0 ]; then force_no_rebuild=`echo $_force_no_rebuild | cut -d' ' -f2` if [ "$force_no_rebuild" != "0" ] && [ "$force_no_rebuild" != "1" ];then - echo "Error: force_no_rebuild value is invalid" + derror "Error: force_no_rebuild value is invalid" return 1 fi fi @@ -630,13 +647,13 @@ check_rebuild() if [ $? -eq 0 ]; then force_rebuild=`echo $_force_rebuild | cut -d' ' -f2` if [ "$force_rebuild" != "0" ] && [ "$force_rebuild" != "1" ];then - echo "Error: force_rebuild value is invalid" + derror "Error: force_rebuild value is invalid" return 1 fi fi if [[ "$force_no_rebuild" == "1" && "$force_rebuild" == "1" ]]; then - echo "Error: force_rebuild and force_no_rebuild are enabled simultaneously in kdump.conf" + derror "Error: force_rebuild and force_no_rebuild are enabled simultaneously in kdump.conf" return 1 fi @@ -666,18 +683,18 @@ check_rebuild() fi if [ $image_time -eq 0 ]; then - echo -n "No kdump initial ramdisk found."; echo + dinfo "No kdump initial ramdisk found." elif [ "$capture_capable_initrd" == "0" ]; then - echo -n "Rebuild $TARGET_INITRD with dump capture support"; echo + dinfo "Rebuild $TARGET_INITRD with dump capture support" elif [ "$force_rebuild" != "0" ]; then - echo -n "Force rebuild $TARGET_INITRD"; echo + dinfo "Force rebuild $TARGET_INITRD" elif [ "$system_modified" != "0" ]; then : else return 0 fi - echo "Rebuilding $TARGET_INITRD" + dinfo "Rebuilding $TARGET_INITRD" rebuild_initrd return $? } @@ -687,6 +704,8 @@ check_rebuild() # as the currently running kernel. load_kdump() { + local ret + KEXEC_ARGS=$(prepare_kexec_args "${KEXEC_ARGS}") KDUMP_COMMANDLINE=$(prepare_cmdline "${KDUMP_COMMANDLINE}" "${KDUMP_COMMANDLINE_REMOVE}" "${KDUMP_COMMANDLINE_APPEND}") @@ -694,18 +713,30 @@ load_kdump() # Old syscall will always fail as it does not have capability to # to kernel signature verification. if is_secure_boot_enforced; then - echo "Secure Boot is enabled. Using kexec file based syscall." + dinfo "Secure Boot is enabled. Using kexec file based syscall." KEXEC_ARGS="$KEXEC_ARGS -s" fi + ddebug "$KEXEC $KEXEC_ARGS $standard_kexec_args --command-line=$KDUMP_COMMANDLINE --initrd=$TARGET_INITRD $KDUMP_KERNEL" + + exec 12>&2 + exec 2>> $KDUMP_LOG_PATH/kdump.log + PS4='+ $(date "+%Y-%m-%d %H:%M:%S") ${BASH_SOURCE}@${LINENO}: ' + set -x + $KEXEC $KEXEC_ARGS $standard_kexec_args \ --command-line="$KDUMP_COMMANDLINE" \ --initrd=$TARGET_INITRD $KDUMP_KERNEL - if [ $? == 0 ]; then - echo "kexec: loaded kdump kernel" + + ret=$? + set +x + exec 2>&12 12>&- + + if [ $ret == 0 ]; then + dinfo "kexec: loaded kdump kernel" return 0 else - echo "kexec: failed to load kdump kernel" >&2 + derror "kexec: failed to load kdump kernel" return 1 fi } @@ -720,7 +751,7 @@ check_ssh_config() # canonicalize the path SSH_KEY_LOCATION=$(/usr/bin/readlink -m $config_val) else - echo "WARNING: '$config_val' doesn't exist, using default value '$SSH_KEY_LOCATION'" + dwarn "WARNING: '$config_val' doesn't exist, using default value '$SSH_KEY_LOCATION'" fi ;; path) @@ -762,22 +793,22 @@ check_and_wait_network_ready() if [ $retval -eq 0 ]; then return 0 elif [ $retval -ne 255 ]; then - echo "Could not create $DUMP_TARGET:$SAVE_PATH, you should check the privilege on server side" >&2 + derror "Could not create $DUMP_TARGET:$SAVE_PATH, you should check the privilege on server side" return 1 fi # if server removes the authorized_keys or, no /root/.ssh/kdump_id_rsa - echo $errmsg | grep -q "Permission denied\|No such file or directory\|Host key verification failed" + ddebug "$errmsg" + echo $errmsg | grep -q "Permission denied\|No such file or directory\|Host key verification failed" &> /dev/null if [ $? -eq 0 ]; then - echo "Could not create $DUMP_TARGET:$SAVE_PATH, you probably need to run \"kdumpctl propagate\"" >&2 + derror "Could not create $DUMP_TARGET:$SAVE_PATH, you probably need to run \"kdumpctl propagate\"" return 1 fi if [ $warn_once -eq 1 ]; then - echo "Network dump target is not usable, waiting for it to be ready" + dwarn "Network dump target is not usable, waiting for it to be ready..." warn_once=0 fi - echo -n . cur=$(date +%s) let "diff = $cur - $start_time" @@ -788,7 +819,7 @@ check_and_wait_network_ready() sleep 1 done - echo "Could not create $DUMP_TARGET:$SAVE_PATH, ipaddr is not ready yet. You should check network connection" >&2 + dinfo "Could not create $DUMP_TARGET:$SAVE_PATH, ipaddr is not ready yet. You should check network connection" return 1 } @@ -805,7 +836,7 @@ propagate_ssh_key() { check_ssh_config if [ $? -ne 0 ]; then - echo "No ssh config specified in $KDUMP_CONFIG_FILE. Can't propagate" >&2 + derror "No ssh config specified in $KDUMP_CONFIG_FILE. Can't propagate" exit 1 fi @@ -814,11 +845,11 @@ propagate_ssh_key() #Check to see if we already created key, if not, create it. if [ -f $KEYFILE ]; then - echo "Using existing keys..." + dinfo "Using existing keys..." else - echo -n "Generating new ssh keys... " + dinfo "Generating new ssh keys... " /usr/bin/ssh-keygen -t rsa -f $KEYFILE -N "" 2>&1 > /dev/null - echo "done." + dinfo "done." fi #now find the target ssh user and server to contact. @@ -829,10 +860,10 @@ propagate_ssh_key() ssh-copy-id -i $KEYFILE $SSH_USER@$SSH_SERVER RET=$? if [ $RET == 0 ]; then - echo $KEYFILE has been added to ~$SSH_USER/.ssh/authorized_keys on $SSH_SERVER + dinfo "$KEYFILE has been added to ~$SSH_USER/.ssh/authorized_keys on $SSH_SERVER" return 0 else - echo $errmsg, $KEYFILE failed in transfer to $SSH_SERVER >&2 + derror "$errmsg, $KEYFILE failed in transfer to $SSH_SERVER" exit 1 fi } @@ -842,7 +873,7 @@ show_reserved_mem() local mem=$(cat /sys/kernel/kexec_crash_size) local mem_mb=$(expr $mem / 1024 / 1024) - echo "Reserved "$mem_mb"MB memory for crash kernel" + dinfo "Reserved "$mem_mb"MB memory for crash kernel" } check_current_fadump_status() @@ -872,12 +903,12 @@ save_raw() raw_target=$(awk '$1 ~ /^raw$/ { print $2; }' $KDUMP_CONFIG_FILE) [ -z "$raw_target" ] && return 0 [ -b "$raw_target" ] || { - echo "raw partition $raw_target not found" + derror "raw partition $raw_target not found" return 1 } check_fs=$(lsblk --nodeps -npo FSTYPE $raw_target) if [[ $(echo $check_fs | wc -w) -ne 0 ]]; then - echo "Warning: Detected '$check_fs' signature on $raw_target, data loss is expected." + dwarn "Warning: Detected '$check_fs' signature on $raw_target, data loss is expected." return 0 fi kdump_dir=`grep ^path $KDUMP_CONFIG_FILE | cut -d' ' -f2-` @@ -889,12 +920,12 @@ save_raw() mkdir -p "$coredir" [ -d "$coredir" ] || { - echo "failed to create $coredir" + derror "failed to create $coredir" return 1 } if makedumpfile -R $coredir/vmcore <$raw_target >/dev/null 2>&1; then # dump found - echo "Dump saved to $coredir/vmcore" + dinfo "Dump saved to $coredir/vmcore" # wipe makedumpfile header dd if=/dev/zero of=$raw_target bs=1b count=1 2>/dev/null else @@ -967,13 +998,13 @@ check_fence_kdump_config() for node in $nodes; do if [ "$node" = "$hostname" ]; then - echo "Option fence_kdump_nodes cannot contain $hostname" + derror "Option fence_kdump_nodes cannot contain $hostname" return 1 fi # node can be ipaddr echo $ipaddrs | grep $node > /dev/null if [ $? -eq 0 ]; then - echo "Option fence_kdump_nodes cannot contain $node" + derror "Option fence_kdump_nodes cannot contain $node" return 1 fi done @@ -995,11 +1026,11 @@ start_fadump() { echo 1 > $FADUMP_REGISTER_SYS_NODE if ! check_current_fadump_status; then - echo "fadump: failed to register" + derror "fadump: failed to register" return 1 fi - echo "fadump: registered successfully" + dinfo "fadump: registered successfully" return 0 } @@ -1026,7 +1057,7 @@ check_failure_action_config() if [ -z "$failure_action" -a -z "$default_option" ]; then return 0 elif [ -n "$failure_action" -a -n "$default_option" ]; then - echo "Cannot specify 'failure_action' and 'default' option together" + derror "Cannot specify 'failure_action' and 'default' option together" return 1 fi @@ -1040,7 +1071,7 @@ check_failure_action_config() return 0 ;; *) - echo $"Usage kdump.conf: $option {reboot|halt|poweroff|shell|dump_to_rootfs}" + dinfo $"Usage kdump.conf: $option {reboot|halt|poweroff|shell|dump_to_rootfs}" return 1 esac } @@ -1058,7 +1089,7 @@ check_final_action_config() return 0 ;; *) - echo $"Usage kdump.conf: final_action {reboot|halt|poweroff}" + dinfo $"Usage kdump.conf: final_action {reboot|halt|poweroff}" return 1 esac fi @@ -1068,13 +1099,13 @@ start() { check_dump_feasibility if [ $? -ne 0 ]; then - echo "Starting kdump: [FAILED]" + derror "Starting kdump: [FAILED]" return 1 fi check_config if [ $? -ne 0 ]; then - echo "Starting kdump: [FAILED]" + derror "Starting kdump: [FAILED]" return 1 fi @@ -1084,43 +1115,43 @@ start() save_raw if [ $? -ne 0 ]; then - echo "Starting kdump: [FAILED]" + derror "Starting kdump: [FAILED]" return 1 fi check_current_status if [ $? == 0 ]; then - echo "Kdump already running: [WARNING]" + dwarn "Kdump already running: [WARNING]" return 0 fi if check_ssh_config; then if ! check_ssh_target; then - echo "Starting kdump: [FAILED]" + derror "Starting kdump: [FAILED]" return 1 fi fi check_rebuild if [ $? != 0 ]; then - echo "Starting kdump: [FAILED]" + derror "Starting kdump: [FAILED]" return 1 fi start_dump if [ $? != 0 ]; then - echo "Starting kdump: [FAILED]" + derror "Starting kdump: [FAILED]" return 1 fi - echo "Starting kdump: [OK]" + dinfo "Starting kdump: [OK]" } reload() { check_current_status if [ $? -ne 0 ]; then - echo "Kdump was not running: [WARNING]" + dwarn "Kdump was not running: [WARNING]" fi if [ $DEFAULT_DUMP_MODE == "fadump" ]; then @@ -1131,36 +1162,36 @@ reload() fi if [ $? -ne 0 ]; then - echo "Stopping kdump: [FAILED]" + derror "Stopping kdump: [FAILED]" return 1 fi - echo "Stopping kdump: [OK]" + dinfo "Stopping kdump: [OK]" setup_initrd if [ $? -ne 0 ]; then - echo "Starting kdump: [FAILED]" + derror "Starting kdump: [FAILED]" return 1 fi start_dump if [ $? -ne 0 ]; then - echo "Starting kdump: [FAILED]" + derror "Starting kdump: [FAILED]" return 1 fi - echo "Starting kdump: [OK]" + dinfo "Starting kdump: [OK]" } stop_fadump() { echo 0 > $FADUMP_REGISTER_SYS_NODE if check_current_fadump_status; then - echo "fadump: failed to unregister" + derror "fadump: failed to unregister" return 1 fi - echo "fadump: unregistered successfully" + dinfo "fadump: unregistered successfully" return 0 } @@ -1173,11 +1204,11 @@ stop_kdump() fi if [ $? != 0 ]; then - echo "kexec: failed to unload kdump kernel" + derror "kexec: failed to unload kdump kernel" return 1 fi - echo "kexec: unloaded kdump kernel" + dinfo "kexec: unloaded kdump kernel" return 0 } @@ -1185,7 +1216,7 @@ reload_fadump() { echo 1 > $FADUMP_REGISTER_SYS_NODE if [ $? == 0 ]; then - echo "fadump: re-registered successfully" + dinfo "fadump: re-registered successfully" return 0 else # FADump could fail on older kernel where re-register @@ -1210,11 +1241,11 @@ stop() fi if [ $? != 0 ]; then - echo "Stopping kdump: [FAILED]" + derror "Stopping kdump: [FAILED]" return 1 fi - echo "Stopping kdump: [OK]" + dinfo "Stopping kdump: [OK]" return 0 } @@ -1235,13 +1266,13 @@ rebuild() { return 1 fi - echo "Rebuilding $TARGET_INITRD" + dinfo "Rebuilding $TARGET_INITRD" rebuild_initrd return $? } if [ ! -f "$KDUMP_CONFIG_FILE" ]; then - echo "Error: No kdump config file found!" >&2 + derror "Error: No kdump config file found!" exit 1 fi @@ -1267,11 +1298,11 @@ main () check_current_status case "$?" in 0) - echo "Kdump is operational" + dinfo "Kdump is operational" EXIT_CODE=0 ;; 1) - echo "Kdump is not operational" + dinfo "Kdump is not operational" EXIT_CODE=3 ;; esac @@ -1296,7 +1327,7 @@ main () show_reserved_mem ;; *) - echo $"Usage: $0 {start|stop|status|restart|reload|rebuild|propagate|showmem}" + dinfo $"Usage: $0 {start|stop|status|restart|reload|rebuild|propagate|showmem}" exit 1 esac } diff --git a/kexec-kdump-howto.txt b/kexec-kdump-howto.txt index 2209e5a..5f57a84 100644 --- a/kexec-kdump-howto.txt +++ b/kexec-kdump-howto.txt @@ -613,7 +613,7 @@ is a dump filtering and compression utility provided with kexec-tools. On some architectures, it can drastically reduce the size of your vmcore files, which becomes very useful on systems with large amounts of memory. -A typical setup is 'core_collector makedumpfile -F -l --message-level 1 -d 31', +A typical setup is 'core_collector makedumpfile -F -l --message-level 7 -d 31', but check the output of '/sbin/makedumpfile --help' for a list of all available options (-i and -g don't need to be specified, they're automatically taken care of). Note that use of makedumpfile requires that the kernel-debuginfo package @@ -633,11 +633,11 @@ First one is source file and second one is target file. For ex. - ex2. - core_collector "makedumpfile -l --message-level 1 -d 31" + core_collector "makedumpfile -l --message-level 7 -d 31" Above will effectively be translated to: - makedumpfile -l --message-level 1 -d 31 /proc/vmcore /vmcore + makedumpfile -l --message-level 7 -d 31 /proc/vmcore /vmcore For dump targets like raw and ssh, in general, core collector should expect one argument (source file) and should output the processed core on standard @@ -656,11 +656,11 @@ raw dumps core_collector examples: - ex4. - core_collector "makedumpfile -F -l --message-level 1 -d 31" + core_collector "makedumpfile -F -l --message-level 7 -d 31" Above will effectively be translated to. - makedumpfile -F -l --message-level 1 -d 31 | dd of= + makedumpfile -F -l --message-level 7 -d 31 | dd of= ssh dumps core_collector examples: @@ -674,11 +674,11 @@ ssh dumps core_collector examples: - ex6. - core_collector "makedumpfile -F -l --message-level 1 -d 31" + core_collector "makedumpfile -F -l --message-level 7 -d 31" Above will effectively be translated to. - makedumpfile -F -l --message-level 1 -d 31 | ssh "dd of=path/vmcore" + makedumpfile -F -l --message-level 7 -d 31 | ssh "dd of=path/vmcore" There is one exception to standard output rule for ssh dumps. And that is scp. As scp can handle ssh destinations for file transfers, one can @@ -696,9 +696,9 @@ About default core collector ---------------------------- Default core_collector for ssh/raw dump is: -"makedumpfile -F -l --message-level 1 -d 31". +"makedumpfile -F -l --message-level 7 -d 31". Default core_collector for other targets is: -"makedumpfile -l --message-level 1 -d 31". +"makedumpfile -l --message-level 7 -d 31". Even if core_collector option is commented out in kdump.conf, makedumpfile is default core collector and kdump uses it internally. @@ -885,3 +885,38 @@ Debugging Tips minicom -C /tmp/console-logs Now minicom should be logging serial console in file console-logs. + +- Using the logger to output kdump log messages + + Currently, kdump messages are printed with the 'echo' command or redirect + to console, and which does not support to output kdump messages according + to the log level. + + That is not convenient to debug kdump issues, we usually need to capture + additional debugging information via the modification of the options or the + scripts like kdumpctl, mkdumprd, etc. Because there is no complete debugging + messages, which could waste valuable time. + + To cope with this challenging, we introduce the logger to output the kdump + messages according to the log level, and provide a chance to save logs to + the journald if the journald service is available, and then dump all logs + to a file, otherwise dump the logs with the dmesg to a file. + + Logging is controlled by following global variables: + - @var kdump_stdloglvl - logging level to standard error (console output) + - @var kdump_sysloglvl - logging level to syslog (by logger command) + - @var kdump_kmsgloglvl - logging level to /dev/kmsg (only for boot-time) + If any of the variables is not set, this function set it to default: + - @var kdump_stdloglvl=4 (info) + - @var kdump_sysloglvl=4 (info) + - @var kdump_kmsgloglvl=0 (no logging) + + Logging levels: fatal(1),error(2),warn(3),info(4),debug(5),trace(6) + + We can easily configure the above variables in the /etc/sysconfig/kdump. For + example: + kdump_sysloglvl=5 + kdump_stdloglvl=5 + + The above configurations indicate that kdump messages will be printed to the + console and journald if the journald service is enabled. diff --git a/kexec-tools.spec b/kexec-tools.spec index 669fd6c..623274c 100644 --- a/kexec-tools.spec +++ b/kexec-tools.spec @@ -4,7 +4,7 @@ Name: kexec-tools Version: 2.0.20 -Release: 18%{?dist} +Release: 19%{?dist} License: GPLv2 Summary: The kexec/kdump userspace component @@ -37,6 +37,7 @@ Source27: early-kdump-howto.txt Source28: kdump-udev-throttler Source29: kdump.sysconfig.aarch64 Source30: 60-kdump.install +Source31: kdump-logger.sh ####################################### # These are sources for mkdumpramfs @@ -190,6 +191,7 @@ install -m 644 %{SOURCE12} $RPM_BUILD_ROOT%{_mandir}/man8/mkdumprd.8 install -m 644 %{SOURCE25} $RPM_BUILD_ROOT%{_mandir}/man8/kdumpctl.8 install -m 755 %{SOURCE20} $RPM_BUILD_ROOT%{_prefix}/lib/kdump/kdump-lib.sh install -m 755 %{SOURCE23} $RPM_BUILD_ROOT%{_prefix}/lib/kdump/kdump-lib-initramfs.sh +install -m 755 %{SOURCE31} $RPM_BUILD_ROOT%{_prefix}/lib/kdump/kdump-logger.sh %ifnarch s390x install -m 755 %{SOURCE28} $RPM_BUILD_ROOT%{_udevrulesdir}/../kdump-udev-throttler %endif @@ -360,6 +362,28 @@ done %endif %changelog +- Fix comment about ssh dump target +- mkdumprd: Ensure kdumpbase is added +- kdump.service: use ConditionKernelCommandLine=crashkernel +- Revert "Revert "s390x: enable the kexec file load by def +- increase makdumpfile default message level to 7 +- Fix error when using raw target with opalcore +- module-setup.sh: Instead of drop journalctl log, just don't read kmsg +- Doc: add a documentation for the usage of logger +- Improve debugging in the kdump kernel +- kdumpctl: add the '-d' option to enable the kexec loading debugging messages +- kdump.sysconfig: add the kdump logger configurations +- enable the logger for kdump +- introduce the kdump logger from the dracut +- Rework check_config and warn on any duplicated option +- (watchdog) kdump-lib.sh: detect secure boot on s390 +- Don't drop journalctl content if failure action is "shell" +- dracut-module-install: Move systemd conf install code to a function +- selftest: Show the path of dumped vmcore on test end +- selftest: Add document for selftests +- selftest: Add basic test framework +- selftest: Add basic infrastructure to build test image + * Thu Aug 27 2020 Kairui Song - 2.0.20-18 - mkdumprd: Improve the warning message when using encrypted target - kdump-lib.sh: Remove is_atomic diff --git a/mkdumprd b/mkdumprd index 36f71b7..1c361d6 100644 --- a/mkdumprd +++ b/mkdumprd @@ -6,18 +6,29 @@ # Written by Cong Wang # +if [ -f /etc/sysconfig/kdump ]; then + . /etc/sysconfig/kdump +fi + [[ $dracutbasedir ]] || dracutbasedir=/usr/lib/dracut . $dracutbasedir/dracut-functions.sh . /lib/kdump/kdump-lib.sh export IN_KDUMP=1 +#initiate the kdump logger +dlog_init +if [ $? -ne 0 ]; then + echo "failed to initiate the kdump logger." + exit 1 +fi + conf_file="/etc/kdump.conf" SSH_KEY_LOCATION="/root/.ssh/kdump_id_rsa" SAVE_PATH=$(get_save_path) OVERRIDE_RESETTABLE=0 extra_modules="" -dracut_args="--quiet --hostonly --hostonly-cmdline --hostonly-i18n --hostonly-mode strict -o \"plymouth dash resume ifcfg earlykdump\"" +dracut_args="--add kdumpbase --quiet --hostonly --hostonly-cmdline --hostonly-i18n --hostonly-mode strict -o \"plymouth dash resume ifcfg earlykdump\"" readonly MKDUMPRD_TMPDIR="$(mktemp -d -t mkdumprd.XXXXXX)" [ -d "$MKDUMPRD_TMPDIR" ] || perror_exit "dracut: mktemp -p -d -t dracut.XXXXXX failed." @@ -180,8 +191,8 @@ check_size() { fi if [ $avail -lt $memtotal ]; then - echo "Warning: There might not be enough space to save a vmcore." - echo " The size of $2 should be greater than $memtotal kilo bytes." + dwarn "Warning: There might not be enough space to save a vmcore." + dwarn " The size of $2 should be greater than $memtotal kilo bytes." fi } @@ -253,7 +264,7 @@ verify_core_collector() { if [ "$_cmd" != "makedumpfile" ]; then if is_raw_dump_target; then - echo "Warning: specifying a non-makedumpfile core collector, you will have to recover the vmcore manually." + dwarn "Warning: specifying a non-makedumpfile core collector, you will have to recover the vmcore manually." fi return fi @@ -343,7 +354,7 @@ is_unresettable() resettable="$(cat $path)" [ $resettable -eq 0 -a "$OVERRIDE_RESETTABLE" -eq 0 ] && { local device=$(udevadm info --query=all --path=/sys/dev/block/$1 | awk -F= '/DEVNAME/{print $2}') - echo "Error: Can not save vmcore because device $device is unresettable" + derror "Error: Can not save vmcore because device $device is unresettable" return 0 } fi @@ -377,7 +388,7 @@ is_crypt() eval "$line" [[ "$ID_FS_TYPE" = "crypto_LUKS" ]] && { dev=$(udevadm info --query=all --path=/sys/dev/block/$majmin | awk -F= '/DEVNAME/{print $2}') - echo "Device $dev is encrypted." + derror "Device $dev is encrypted." return 0 } return 1 @@ -400,7 +411,7 @@ if ! check_resettable; then fi if ! check_crypt; then - echo "Warning: Encrypted device is in dump path, which is not recommended, see kexec-kdump-howto.txt for more details." + dwarn "Warning: Encrypted device is in dump path, which is not recommended, see kexec-kdump-howto.txt for more details." fi # firstly get right SSH_KEY_LOCATION diff --git a/tests/scripts/testcases/local-kdump/0-local.sh b/tests/scripts/testcases/local-kdump/0-local.sh index d09c6f0..948cfa1 100755 --- a/tests/scripts/testcases/local-kdump/0-local.sh +++ b/tests/scripts/testcases/local-kdump/0-local.sh @@ -8,7 +8,7 @@ on_test() { if [ $boot_count -eq 1 ]; then cat << EOF > /etc/kdump.conf path /var/crash -core_collector makedumpfile -l --message-level 1 -d 31 +core_collector makedumpfile -l --message-level 7 -d 31 EOF kdumpctl start || test_failed "Failed to start kdump" diff --git a/tests/scripts/testcases/nfs-kdump/1-client.sh b/tests/scripts/testcases/nfs-kdump/1-client.sh index df62463..4122bb8 100755 --- a/tests/scripts/testcases/nfs-kdump/1-client.sh +++ b/tests/scripts/testcases/nfs-kdump/1-client.sh @@ -11,7 +11,7 @@ on_test() { if [ "$boot_count" -eq 1 ]; then cat << EOF > /etc/kdump.conf nfs $nfs_server:/srv/nfs -core_collector makedumpfile -l --message-level 1 -d 31 +core_collector makedumpfile -l --message-level 7 -d 31 EOF while ! ping -c 1 $nfs_server -W 1; do diff --git a/tests/scripts/testcases/ssh-kdump/1-client.sh b/tests/scripts/testcases/ssh-kdump/1-client.sh index d79f00f..bd03eff 100755 --- a/tests/scripts/testcases/ssh-kdump/1-client.sh +++ b/tests/scripts/testcases/ssh-kdump/1-client.sh @@ -11,7 +11,7 @@ on_test() { if [ "$boot_count" -eq 1 ]; then cat << EOF > /etc/kdump.conf ssh root@192.168.77.1 -core_collector makedumpfile -l --message-level 1 -d 31 -F +core_collector makedumpfile -l --message-level 7 -d 31 -F EOF ssh-keygen -q -t rsa -N '' -f /root/.ssh/id_rsa <<< y &>/dev/ttyS1