diff --git a/dracut-early-kdump-module-setup.sh b/dracut-early-kdump-module-setup.sh index 3b23383..b25d6b5 100755 --- a/dracut-early-kdump-module-setup.sh +++ b/dracut-early-kdump-module-setup.sh @@ -1,7 +1,6 @@ #!/bin/bash . /etc/sysconfig/kdump -. /lib/kdump/kdump-lib.sh KDUMP_KERNEL="" KDUMP_INITRD="" @@ -21,6 +20,8 @@ depends() { } prepare_kernel_initrd() { + . /lib/kdump/kdump-lib.sh + prepare_kdump_bootinfo # $kernel is a variable from dracut diff --git a/dracut-early-kdump.sh b/dracut-early-kdump.sh index a4e14a4..129841e 100755 --- a/dracut-early-kdump.sh +++ b/dracut-early-kdump.sh @@ -12,6 +12,7 @@ EARLY_KEXEC_ARGS="" . /etc/sysconfig/kdump . /lib/dracut-lib.sh . /lib/kdump-lib.sh +. /lib/kdump-logger.sh #initiate the kdump logger dlog_init diff --git a/dracut-kdump.sh b/dracut-kdump.sh index 370d217..3367bc5 100755 --- a/dracut-kdump.sh +++ b/dracut-kdump.sh @@ -111,6 +111,7 @@ dump_ssh() local _dir="$KDUMP_PATH/$HOST_IP-$DATEDIR" local _host=$2 local _vmcore="vmcore" + local _ipv6_addr="" _username="" dinfo "saving to $_host:$_dir" @@ -122,8 +123,17 @@ dump_ssh() dinfo "saving vmcore" + if is_ipv6_address "$_host"; then + _username=${_host%@*} + _ipv6_addr="[${_host#*@}]" + fi + if [ "${CORE_COLLECTOR%%[[:blank:]]*}" = "scp" ]; then - scp -q $_opt /proc/vmcore "$_host:$_dir/vmcore-incomplete" + if [ -n "$_username" ] && [ -n "$_ipv6_addr" ]; then + scp -q $_opt /proc/vmcore "$_username@$_ipv6_addr:$_dir/vmcore-incomplete" + else + scp -q $_opt /proc/vmcore "$_host:$_dir/vmcore-incomplete" + fi _exitcode=$? else $CORE_COLLECTOR /proc/vmcore | ssh $_opt $_host "dd bs=512 of=$_dir/vmcore-incomplete" @@ -143,8 +153,13 @@ dump_ssh() derror "saving vmcore failed, _exitcode:$_exitcode" fi + dinfo "saving the $KDUMP_LOG_FILE to $_host:$_dir/" save_log - scp -q $_opt $KDUMP_LOG_FILE "$_host:$_dir/" + if [ -n "$_username" ] && [ -n "$_ipv6_addr" ]; then + scp -q $_opt $KDUMP_LOG_FILE "$_username@$_ipv6_addr:$_dir/" + else + scp -q $_opt $KDUMP_LOG_FILE "$_host:$_dir/" + fi _ret=$? if [ $_ret -ne 0 ]; then derror "saving log file failed, _exitcode:$_ret" @@ -161,6 +176,7 @@ save_opalcore_ssh() { local _path=$1 local _opts="$2" local _location=$3 + local _user_name="" _ipv6addr="" ddebug "_path=$_path _opts=$_opts _location=$_location" @@ -173,8 +189,18 @@ save_opalcore_ssh() { fi fi + if is_ipv6_address "$_host"; then + _user_name=${_location%@*} + _ipv6addr="[${_location#*@}]" + fi + dinfo "saving opalcore:$OPALCORE to $_location:$_path" - scp $_opts $OPALCORE $_location:$_path/opalcore-incomplete + + if [ -n "$_user_name" ] && [ -n "$_ipv6addr" ]; then + scp $_opts $OPALCORE $_user_name@$_ipv6addr:$_path/opalcore-incomplete + else + scp $_opts $OPALCORE $_location:$_path/opalcore-incomplete + fi if [ $? -ne 0 ]; then derror "saving opalcore failed" return 1 diff --git a/dracut-module-setup.sh b/dracut-module-setup.sh index 21f7105..21143b4 100755 --- a/dracut-module-setup.sh +++ b/dracut-module-setup.sh @@ -1,11 +1,12 @@ #!/bin/bash -. $dracutfunctions -. /lib/kdump/kdump-lib.sh +kdump_module_init() { + if ! [[ -d "${initdir}/tmp" ]]; then + mkdir -p "${initdir}/tmp" + fi -if ! [[ -d "${initdir}/tmp" ]]; then - mkdir -p "${initdir}/tmp" -fi + . /lib/kdump/kdump-lib.sh +} check() { [[ $debug ]] && set -x @@ -20,6 +21,8 @@ check() { depends() { local _dep="base shutdown" + kdump_module_init + add_opt_module() { [[ " $omit_dracutmodules " != *\ $1\ * ]] && _dep="$_dep $1" } @@ -32,6 +35,9 @@ depends() { modprobe -S $KDUMP_KERNELVER --dry-run $kmodule &>/dev/null || return 1 fi done + + # check that the dracut squash module is available + [ -d "$(dracut_module_path squash)" ] || return 1 } if is_squash_available && ! is_fadump_capable; then @@ -813,16 +819,15 @@ kdump_install_systemd_conf() { # Forward logs to console directly, and don't read Kmsg, this avoids # unneccessary memory consumption and make console output more useful. # Only do so for non fadump image. - if ! is_fadump_capable; then - mkdir -p ${initdir}/etc/systemd/journald.conf.d - echo "[Journal]" > ${initdir}/etc/systemd/journald.conf.d/kdump.conf - echo "Storage=volatile" >> ${initdir}/etc/systemd/journald.conf.d/kdump.conf - echo "ReadKMsg=no" >> ${initdir}/etc/systemd/journald.conf.d/kdump.conf - echo "ForwardToConsole=yes" >> ${initdir}/etc/systemd/journald.conf.d/kdump.conf - fi + mkdir -p ${initdir}/etc/systemd/journald.conf.d + echo "[Journal]" > ${initdir}/etc/systemd/journald.conf.d/kdump.conf + echo "Storage=volatile" >> ${initdir}/etc/systemd/journald.conf.d/kdump.conf + echo "ReadKMsg=no" >> ${initdir}/etc/systemd/journald.conf.d/kdump.conf + echo "ForwardToConsole=yes" >> ${initdir}/etc/systemd/journald.conf.d/kdump.conf } install() { + kdump_module_init kdump_install_conf overwrite_sysctl_conf diff --git a/kdump-dep-generator.sh b/kdump-dep-generator.sh index b6fab2d..f48c8f6 100644 --- a/kdump-dep-generator.sh +++ b/kdump-dep-generator.sh @@ -4,6 +4,7 @@ # http://www.freedesktop.org/wiki/Software/systemd/Generators/ . /usr/lib/kdump/kdump-lib.sh +. /usr/lib/kdump/kdump-logger.sh # If invokded with no arguments for testing purpose, output to /tmp to # avoid overriding the existing. diff --git a/kdump-lib-initramfs.sh b/kdump-lib-initramfs.sh index e766f95..86065be 100755 --- a/kdump-lib-initramfs.sh +++ b/kdump-lib-initramfs.sh @@ -1,6 +1,7 @@ # These variables and functions are useful in 2nd kernel . /lib/kdump-lib.sh +. /lib/kdump-logger.sh KDUMP_PATH="/var/crash" KDUMP_LOG_FILE="/run/initramfs/kexec-dmesg.log" @@ -117,26 +118,15 @@ dump_fs() { local _exitcode local _mp=$1 - local _dev=$(get_mount_info SOURCE target $_mp -f) - local _op=$(get_mount_info OPTIONS target $_mp -f) + ddebug "dump_fs _mp=$_mp" - ddebug "_mp=$_mp _dev=$_dev _op=$_op" - - # If dump path have a corresponding device entry but not mounted, mount it. - if [ -n "$_dev" ] && [ "$_dev" != "rootfs" ]; then - if ! is_mounted "$_mp"; then - dinfo "dump target $_dev is not mounted, trying to mount..." - mkdir -p $_mp - mount -o $_op $_dev $_mp - - if [ $? -ne 0 ]; then - derror "mounting failed (mount point: $_mp, option: $_op)" - return 1 - fi + if ! is_mounted "$_mp"; then + dinfo "dump path \"$_mp\" is not mounted, trying to mount..." + mount --target $_mp + if [ $? -ne 0 ]; then + derror "failed to dump to \"$_mp\", it's not a mount point!" + return 1 fi - else - derror "failed to dump to \"$_mp\", it's not a mount point!" - return 1 fi # Remove -F in makedumpfile case. We don't want a flat format dump here. @@ -168,6 +158,7 @@ dump_fs() derror "saving vmcore failed, _exitcode:$_exitcode" fi + dinfo "saving the $KDUMP_LOG_FILE to $_dump_path/" save_log mv $KDUMP_LOG_FILE $_dump_path/ if [ $_exitcode -ne 0 ]; then @@ -226,8 +217,7 @@ dump_to_rootfs() { dinfo "Trying to bring up rootfs device" - systemctl is-failed dracut-initqueue || systemctl start dracut-initqueue - + systemctl start dracut-initqueue dinfo "Waiting for rootfs mount, will timeout after 90 seconds" systemctl start sysroot.mount diff --git a/kdump-lib.sh b/kdump-lib.sh index 0e38580..d2801da 100755 --- a/kdump-lib.sh +++ b/kdump-lib.sh @@ -8,12 +8,6 @@ FENCE_KDUMP_CONFIG_FILE="/etc/sysconfig/fence_kdump" FENCE_KDUMP_SEND="/usr/libexec/fence_kdump_send" FADUMP_ENABLED_SYS_NODE="/sys/kernel/fadump_enabled" -if [ -f /lib/kdump/kdump-logger.sh ]; then - . /lib/kdump/kdump-logger.sh -elif [ -f /lib/kdump-logger.sh ]; then - . /lib/kdump-logger.sh -fi - is_fadump_capable() { # Check if firmware-assisted dump is enabled diff --git a/kdump.conf b/kdump.conf index e4db52e..dea2e94 100644 --- a/kdump.conf +++ b/kdump.conf @@ -21,7 +21,7 @@ # //%HOST-%DATE/, supports DNS. # # ssh -# - Will scp /proc/vmcore to :/%HOST-%DATE/, +# - Will save /proc/vmcore to :/%HOST-%DATE/, # supports DNS. # NOTE: make sure the user has write permissions on the server. # @@ -166,7 +166,9 @@ #ext4 LABEL=/boot #ext4 UUID=03138356-5e61-4ab3-b58e-27507ac41937 #nfs my.server.com:/export/tmp +#nfs [2001:db8::1:2:3:4]:/export/tmp #ssh user@my.server.com +#ssh user@2001:db8::1:2:3:4 #sshkey /root/.ssh/kdump_id_rsa path /var/crash core_collector makedumpfile -l --message-level 7 -d 31 diff --git a/kdump.conf.5 b/kdump.conf.5 index ae07466..2c5a2bc 100644 --- a/kdump.conf.5 +++ b/kdump.conf.5 @@ -98,6 +98,11 @@ vmcore.flat, you will need to use "makedumpfile -R" to rearrange the dump data from standard input to a normal dumpfile (readable with analysis tools). ie. "makedumpfile -R vmcore < vmcore.flat" +.PP +Note 3: If specified core_collector simply copy the vmcore file to the +dump target (eg: cp, scp), the vmcore could be significantly large. +Please make sure the dump target has enough space, at leaset larger +than the system's RAM. .RE diff --git a/kdumpctl b/kdumpctl index 2e7f842..24f5cf7 100755 --- a/kdumpctl +++ b/kdumpctl @@ -34,6 +34,7 @@ fi [[ $dracutbasedir ]] || dracutbasedir=/usr/lib/dracut . $dracutbasedir/dracut-functions.sh . /lib/kdump/kdump-lib.sh +. /lib/kdump/kdump-logger.sh #initiate the kdump logger dlog_init diff --git a/kexec-kdump-howto.txt b/kexec-kdump-howto.txt index 447bc54..88af607 100644 --- a/kexec-kdump-howto.txt +++ b/kexec-kdump-howto.txt @@ -524,17 +524,91 @@ to send over the necessary ssh key file. Restart the kdump service via Advanced Setups =============== -Kdump boot directory --------------------- +About /etc/sysconfig/kdump +------------------------------ + +Currently, there are a few options in /etc/sysconfig/kdump, which are +usually used to control the behavior of kdump kernel. Basically, all of +these options have default values, usually we do not need to change them, +but sometimes, we may modify them in order to better control the behavior +of kdump kernel such as debug, etc. + +-KDUMP_BOOTDIR Usually kdump kernel is the same as 1st kernel. So kdump will try to find kdump kernel under /boot according to /proc/cmdline. E.g we execute below command and get an output: cat /proc/cmdline BOOT_IMAGE=/xxx/vmlinuz-3.yyy.zzz root=xxxx ..... -Then kdump kernel will be /boot/xxx/vmlinuz-3.yyy.zzz. -However a variable KDUMP_BOOTDIR in /etc/sysconfig/kdump is provided to -user if kdump kernel is put in a different directory. + +Then kdump kernel will be /boot/xxx/vmlinuz-3.yyy.zzz. However, this option +is provided to user if kdump kernel is put in a different directory. + +-KDUMP_IMG + +This represents the image type used for kdump. The default value is "vmlinuz". + +-KDUMP_IMG_EXT + +This represents the images extension. Relocatable kernels don't have one. +Currently, it is a null string by default. + +-KEXEC_ARGS + +Any additional kexec arguments required. For example: +KEXEC_ARGS="--elf32-core-headers". + +In most situations, this should be left empty. But, sometimes we hope to get +additional kexec loading debugging information, we can add the '-d' option +for the debugging. + +-KDUMP_KERNELVER + +This is a kernel version string for the kdump kernel. If the version is not +specified, the init script will try to find a kdump kernel with the same +version number as the running kernel. + +-KDUMP_COMMANDLINE + +The value of 'KDUMP_COMMANDLINE' will be passed to kdump kernel as command +line parameters, this will likely match the contents of the grub kernel line. + +In general, if a command line is not specified, which means that it is a null +string such as KDUMP_COMMANDLINE="", the default will be taken automatically +from the '/proc/cmdline'. + +-KDUMP_COMMANDLINE_REMOVE + +This option allows us to remove arguments from the current kdump command line. +If we don't specify any parameters for the KDUMP_COMMANDLINE, it will inherit +all values from the '/proc/cmdline', which is not expected. As you know, some +default kernel parameters could affect kdump, furthermore, that could cause +the failure of kdump kernel boot. + +In addition, the option is also helpful to debug the kdump kernel, we can use +this option to change kdump kernel command line. + +For more kernel parameters, please refer to kernel document. + +-KDUMP_COMMANDLINE_APPEND + +This option allows us to append arguments to the current kdump command line +after processed by the KDUMP_COMMANDLINE_REMOVE. For kdump kernel, some +specific modules require to be disabled like the mce, cgroup, numa, hest_disable, +etc. Those modules may waste memory or kdump kernel doesn't need them, +furthermore, there may affect kdump kernel boot. + +Just like above option, it can be used to disable or enable some kernel +modules so that we can exclude any errors for kdump kernel, this is very +meaningful for debugging. + +-KDUMP_STDLOGLVL | KDUMP_SYSLOGLVL | KDUMP_KMSGLOGLVL + +These variables are used to control the kdump log level in the first kernel. +In the second kernel, kdump will use the rd.kdumploglvl option to set the log +level in the above KDUMP_COMMANDLINE_APPEND. + +Logging levels: no logging(0), error(1), warn(2), info(3), debug(4) Kdump Post-Capture Executable ----------------------------- diff --git a/kexec-tools.spec b/kexec-tools.spec index d7497e0..91bc0f3 100644 --- a/kexec-tools.spec +++ b/kexec-tools.spec @@ -5,7 +5,7 @@ Name: kexec-tools Version: 2.0.21 -Release: 2%{?dist} +Release: 4%{?dist} License: GPLv2 Summary: The kexec/kdump userspace component @@ -61,7 +61,7 @@ Requires(postun): systemd-units Requires(pre): coreutils sed zlib Requires: dracut >= 050 Requires: dracut-network >= 050 -Requires: dracut-squash >= 050 +Recommends: dracut-squash >= 050 Requires: ethtool BuildRequires: make BuildRequires: zlib-devel elfutils-devel glib2-devel bzip2-devel ncurses-devel bison flex lzo-devel snappy-devel @@ -361,6 +361,23 @@ done %endif %changelog +* Fri Jan 22 2021 Kairui Song - 2.0.21-4 +- dracut-module-setup.sh: enable ForwardToConsole=yes in fadump mode +- kdump.conf: add ipv6 example for nfs and ssh dump +- fix kdump failure of saving vmcore with the scp + ipv6 method + +* Wed Jan 20 2021 Kairui Song - 2.0.21-3 +- module-setup.sh: don't polute the namespace unnecessarily +- module-setup.sh: don't source $dracutfunctions +- logger: source the logger file individually +- Fix dump_fs mount point detection and fallback mount +- Revert "Don's try to restart dracut-initqueue if it's already failed" +- Revert "Append both nofail and x-systemd.before to kdump mount target" +- Doc: Improve the kdump sysconfig document +- kdump.conf: Update doc about core_collector for ssh target +- Merge #4 `Make dracut-squash a weak dep` +- Fix a date error in the change log + * Fri Jan 08 2021 Kairui Song - 2.0.21-2 - makedumpfile: make use of 'uts_namespace.name' offset in VMCOREINFO - kdumpctl: fix a variable expansion in check_fence_kdump_config() diff --git a/mkdumprd b/mkdumprd index 0623940..c34b79c 100644 --- a/mkdumprd +++ b/mkdumprd @@ -13,6 +13,7 @@ fi [[ $dracutbasedir ]] || dracutbasedir=/usr/lib/dracut . $dracutbasedir/dracut-functions.sh . /lib/kdump/kdump-lib.sh +. /lib/kdump/kdump-logger.sh export IN_KDUMP=1 #initiate the kdump logger @@ -86,9 +87,6 @@ to_mount() { # drop nofail or nobootwait _options=$(echo $_options | sed 's/\(^\|,\)nofail\($\|,\)/\1/g') _options=$(echo $_options | sed 's/\(^\|,\)nobootwait\($\|,\)/\1/g') - # use both nofail and x-systemd.before to ensure systemd will try best to - # mount it before kdump starts, this is an attempt to improve robustness - _options="$_options,nofail,x-systemd.before=initrd-fs.target" echo "$_pdev $_new_mntpoint $_fstype $_options" }