317 lines
14 KiB
Diff
317 lines
14 KiB
Diff
From 7dd8a2f4d351ce8975c0af122732a2a12697c8cc Mon Sep 17 00:00:00 2001
|
|
From: Kairui Song <kasong@redhat.com>
|
|
Date: Wed, 8 Apr 2020 20:09:38 +0800
|
|
Subject: [PATCH] Add 99memstrack module
|
|
|
|
memstrack is a new tool to track the overall memory usage and
|
|
allocation, which can help off load the improve the builtin module
|
|
memory tracing function in dracut.
|
|
|
|
With this change, the rd.memdebug=4 behavior is similiar with before,
|
|
but the report is defered to pre-pivot, so the memory usage info
|
|
during the whole initramfs run is traced. And the ourput format is
|
|
changed a bit:
|
|
|
|
dracut-pre-pivot[519]: ======== Report format module_summary: ========
|
|
dracut-pre-pivot[519]: Module squashfs using 10.4MB (2658 pages), peak allocation 10.4MB (2671 pages)
|
|
dracut-pre-pivot[519]: Module qxl using 3.4MB (865 pages), peak allocation 3.4MB (880 pages)
|
|
dracut-pre-pivot[519]: Module crc32c_intel using 2.0MB (519 pages), peak allocation 3.0MB (769 pages)
|
|
dracut-pre-pivot[519]: Module serio_raw using 2.0MB (505 pages), peak allocation 3.6MB (918 pages)
|
|
dracut-pre-pivot[519]: Module virtio_console using 1.6MB (416 pages), peak allocation 1.6MB (419 pages)
|
|
... snip ...
|
|
dracut-pre-pivot[519]: ======== Report format module_summary END ========
|
|
|
|
It now contains more detail and also includes the peak usage which could
|
|
be more helpful.
|
|
|
|
And now it have a rd.memdebug=5, which will print more detail about
|
|
the stack trace of the top memory user, also printed on pri-pivot:
|
|
|
|
dracut-pre-pivot[519]: ======== Report format module_top: ========
|
|
dracut-pre-pivot[519]: Top stack usage of module squashfs:
|
|
dracut-pre-pivot[519]: (null) Pages: 2658 (peak: 2671)
|
|
dracut-pre-pivot[519]: (null) Pages: 2658 (peak: 2671)
|
|
dracut-pre-pivot[519]: async_page_fault (0xffffffff81a01149) Pages: 1448 (peak: 1461)
|
|
dracut-pre-pivot[519]: do_async_page_fault (0xffffffff8105c509) Pages: 1448 (peak: 1461)
|
|
dracut-pre-pivot[519]: do_page_fault (0xffffffff8106296a) Pages: 1448 (peak: 1461)
|
|
dracut-pre-pivot[519]: do_user_addr_fault (0xffffffff810626bd) Pages: 1448 (peak: 1461)
|
|
dracut-pre-pivot[519]: handle_mm_fault (0xffffffff812940c4) Pages: 1448 (peak: 1461)
|
|
dracut-pre-pivot[519]: __handle_mm_fault (0xffffffff81293627) Pages: 1195 (peak: 1208)
|
|
dracut-pre-pivot[519]: __do_fault (0xffffffff8128b07e) Pages: 1195 (peak: 1208)
|
|
dracut-pre-pivot[519]: filemap_fault (0xffffffff8124c0b9) Pages: 1195 (peak: 1208)
|
|
dracut-pre-pivot[519]: __do_page_cache_readahead (0xffffffff812585da) Pages: 1063 (peak: 1076)
|
|
dracut-pre-pivot[519]: read_pages (0xffffffff812583c2) Pages: 1063 (peak: 1076)
|
|
dracut-pre-pivot[519]: squashfs_readpage squashfs (0xffffffffc0022073) Pages: 1039 (peak: 1052)
|
|
dracut-pre-pivot[519]: squashfs_readpage_block squashfs (0xffffffffc0024334) Pages: 744 (peak: 744)
|
|
dracut-pre-pivot[519]: squashfs_copy_cache squashfs (0xffffffffc0021a3f) Pages: 744 (peak: 744)
|
|
dracut-pre-pivot[519]: pagecache_get_page (0xffffffff8124abf7) Pages: 744 (peak: 744)
|
|
dracut-pre-pivot[519]: __page_cache_alloc (0xffffffff81247df6) Pages: 744 (peak: 744)
|
|
dracut-pre-pivot[519]: alloc_pages_current (0xffffffff812cdca7) Pages: 744 (peak: 744)
|
|
dracut-pre-pivot[519]: __alloc_pages_nodemask (0xffffffff812b3107) Pages: 744 (peak: 744)
|
|
dracut-pre-pivot[519]: __alloc_pages_nodemask (0xffffffff812b3107) Pages: 1488 (peak: 1488)
|
|
dracut-pre-pivot[519]: Top stack usage of module qxl:
|
|
dracut-pre-pivot[519]: (null) Pages: 865 (peak: 880)
|
|
dracut-pre-pivot[519]: entry_SYSCALL_64_after_hwframe (0xffffffff81a0008c) Pages: 855 (peak: 858)
|
|
dracut-pre-pivot[519]: do_syscall_64 (0xffffffff81002a5a) Pages: 855 (peak: 858)
|
|
dracut-pre-pivot[519]: __x64_sys_finit_module (0xffffffff8117ccea) Pages: 811 (peak: 811)
|
|
dracut-pre-pivot[519]: __do_sys_finit_module (0xffffffff8117cc6e) Pages: 811 (peak: 811)
|
|
dracut-pre-pivot[519]: load_module (0xffffffff8117c6be) Pages: 802 (peak: 802)
|
|
dracut-pre-pivot[519]: do_init_module (0xffffffff81179e72) Pages: 802 (peak: 802)
|
|
dracut-pre-pivot[519]: do_one_initcall (0xffffffff81000d5a) Pages: 802 (peak: 802)
|
|
dracut-pre-pivot[519]: serio_raw_poll serio_raw (0xffffffffc0200054) Pages: 802 (peak: 802)
|
|
dracut-pre-pivot[519]: __pci_register_driver (0xffffffff81557804) Pages: 802 (peak: 802)
|
|
dracut-pre-pivot[519]: driver_register (0xffffffff8167ed24) Pages: 802 (peak: 802)
|
|
dracut-pre-pivot[519]: bus_add_driver (0xffffffff8167cbb2) Pages: 801 (peak: 801)
|
|
dracut-pre-pivot[519]: driver_attach (0xffffffff8167d28e) Pages: 801 (peak: 801)
|
|
dracut-pre-pivot[519]: bus_for_each_dev (0xffffffff8167b62c) Pages: 801 (peak: 801)
|
|
dracut-pre-pivot[519]: __driver_attach (0xffffffff8167e18f) Pages: 801 (peak: 801)
|
|
dracut-pre-pivot[519]: device_driver_attach (0xffffffff8167e0ed) Pages: 801 (peak: 801)
|
|
dracut-pre-pivot[519]: driver_probe_device (0xffffffff8167de6c) Pages: 801 (peak: 801)
|
|
dracut-pre-pivot[519]: really_probe (0xffffffff8167d9c9) Pages: 801 (peak: 801)
|
|
dracut-pre-pivot[519]: pci_device_probe (0xffffffff81559627) Pages: 801 (peak: 801)
|
|
dracut-pre-pivot[519]: local_pci_probe (0xffffffff81557f98) Pages: 801 (peak: 801)
|
|
dracut-pre-pivot[519]: qxl_pci_probe qxl (0xffffffffc01f0387) Pages: 773 (peak: 773)
|
|
dracut-pre-pivot[519]: drm_fbdev_generic_setup drm_kms_helper (0xffffffffc01b30c5) Pages: 773 (peak: 773)
|
|
dracut-pre-pivot[519]: drm_fbdev_client_hotplug drm_kms_helper (0xffffffffc01b2656) Pages: 773 (peak: 773)
|
|
dracut-pre-pivot[519]: __drm_fb_helper_initial_config_and_unlock drm_kms_helper (0xffffffffc01b1a28) Pages: 770 (peak: 770)
|
|
dracut-pre-pivot[519]: drm_fb_helper_generic_probe drm_kms_helper (0xffffffffc01b2fa5) Pages: 770 (peak: 770)
|
|
dracut-pre-pivot[519]: vzalloc (0xffffffff812aa39c) Pages: 770 (peak: 770)
|
|
dracut-pre-pivot[519]: __vmalloc_node_range (0xffffffff812aa200) Pages: 768 (peak: 768)
|
|
... snip ...
|
|
======== Report format module_top END ========
|
|
|
|
This could be very helpful for debuging memory usage issues.
|
|
---
|
|
dracut.cmdline.7.asc | 16 +++++---
|
|
dracut.spec | 2 +
|
|
modules.d/99base/dracut-lib.sh | 2 +-
|
|
modules.d/99memstrack/memstrack-report.sh | 23 +++++++++++
|
|
modules.d/99memstrack/memstrack-start.sh | 68 +++++++++++++++++++++++++++++++
|
|
modules.d/99memstrack/memstrack.service | 13 ++++++
|
|
modules.d/99memstrack/module-setup.sh | 27 ++++++++++++
|
|
7 files changed, 144 insertions(+), 7 deletions(-)
|
|
|
|
diff --git a/dracut.cmdline.7.asc b/dracut.cmdline.7.asc
|
|
index c53601ea..4d4970c8 100644
|
|
--- a/dracut.cmdline.7.asc
|
|
+++ b/dracut.cmdline.7.asc
|
|
@@ -188,8 +188,8 @@ It should be attached to any report about dracut problems.
|
|
_/run/initramfs/init.log_.
|
|
If "quiet" is set, it also logs to the console.
|
|
|
|
-**rd.memdebug=[0-3]**::
|
|
- Print memory usage info at various points, set the verbose level from 0 to 4.
|
|
+**rd.memdebug=[0-5]**::
|
|
+ Print memory usage info at various points, set the verbose level from 0 to 5.
|
|
+
|
|
Higher level means more debugging output:
|
|
+
|
|
@@ -198,10 +198,14 @@ It should be attached to any report about dracut problems.
|
|
1 - partial /proc/meminfo
|
|
2 - /proc/meminfo
|
|
3 - /proc/meminfo + /proc/slabinfo
|
|
- 4 - /proc/meminfo + /proc/slabinfo + tracekomem
|
|
- NOTE: tracekomem is a shell script utilizing kernel trace to track
|
|
- the rough total memory consumption of kernel modules during
|
|
- loading. It may override other trace configurations.
|
|
+ 4 - /proc/meminfo + /proc/slabinfo + memstrack summary
|
|
+ NOTE: memstrack is a memory tracing tool that tracks the total memory
|
|
+ consumption, and peak memory consumption of each kernel modules
|
|
+ and userspace progress during the whole initramfs runtime, report
|
|
+ is genereted and the end of initramsfs run.
|
|
+ 5 - /proc/meminfo + /proc/slabinfo + memstrack (with top memory stacktrace)
|
|
+ NOTE: memstrack (with top memory stacktrace) will print top memory
|
|
+ allocation stack traces during the whole initramfs runtime.
|
|
----
|
|
|
|
**rd.break**::
|
|
diff --git a/dracut.spec b/dracut.spec
|
|
index 0369dde6..1eb47402 100644
|
|
--- a/dracut.spec
|
|
+++ b/dracut.spec
|
|
@@ -77,6 +77,7 @@ Requires: xz
|
|
Requires: gzip
|
|
|
|
%if 0%{?fedora} || 0%{?rhel}
|
|
+Recommends: memstrack
|
|
Recommends: hardlink
|
|
Recommends: pigz
|
|
Recommends: kpartx
|
|
@@ -403,6 +404,7 @@ install -m 0755 51-dracut-rescue-postinst.sh $RPM_BUILD_ROOT%{_sysconfdir}/kerne
|
|
%{dracutlibdir}/modules.d/98syslog
|
|
%{dracutlibdir}/modules.d/98usrmount
|
|
%{dracutlibdir}/modules.d/99base
|
|
+%{dracutlibdir}/modules.d/99memstrack
|
|
%{dracutlibdir}/modules.d/99fs-lib
|
|
%{dracutlibdir}/modules.d/99shutdown
|
|
%attr(0644,root,root) %ghost %config(missingok,noreplace) %{_localstatedir}/log/dracut.log
|
|
diff --git a/modules.d/99base/dracut-lib.sh b/modules.d/99base/dracut-lib.sh
|
|
index b7020892..e602bacb 100755
|
|
--- a/modules.d/99base/dracut-lib.sh
|
|
+++ b/modules.d/99base/dracut-lib.sh
|
|
@@ -1187,7 +1187,7 @@ are_lists_eq() {
|
|
|
|
setmemdebug() {
|
|
if [ -z "$DEBUG_MEM_LEVEL" ]; then
|
|
- export DEBUG_MEM_LEVEL=$(getargnum 0 0 3 rd.memdebug)
|
|
+ export DEBUG_MEM_LEVEL=$(getargnum 0 0 5 rd.memdebug)
|
|
fi
|
|
}
|
|
|
|
diff --git a/modules.d/99memstrack/memstrack-report.sh b/modules.d/99memstrack/memstrack-report.sh
|
|
new file mode 100755
|
|
index 00000000..241e8621
|
|
--- /dev/null
|
|
+++ b/modules.d/99memstrack/memstrack-report.sh
|
|
@@ -0,0 +1,23 @@
|
|
+#!/usr/bin/env bash
|
|
+. /lib/dracut-lib.sh
|
|
+
|
|
+if ! [ "$DEBUG_MEM_LEVEL" -ge 4 ]; then
|
|
+ exit 0
|
|
+fi
|
|
+
|
|
+if type -P systemctl >/dev/null; then
|
|
+ systemctl stop memstrack.service
|
|
+else
|
|
+ get_pid_of_tracer () {
|
|
+ local _user _pid _rest
|
|
+ read _user _pid _rest <<< $(ps aux | grep [m]emstrack | head -1)
|
|
+ echo $_pid
|
|
+ }
|
|
+
|
|
+ kill -s INT $(get_pid_of_tracer)
|
|
+ while [[ -n $(get_pid_of_tracer) ]]; do
|
|
+ sleep 1
|
|
+ done
|
|
+fi
|
|
+
|
|
+cat /.memstrack
|
|
diff --git a/modules.d/99memstrack/memstrack-start.sh b/modules.d/99memstrack/memstrack-start.sh
|
|
new file mode 100755
|
|
index 00000000..5aa73efe
|
|
--- /dev/null
|
|
+++ b/modules.d/99memstrack/memstrack-start.sh
|
|
@@ -0,0 +1,68 @@
|
|
+#!/bin/sh
|
|
+# Mount kernel debug fs so debug tools can work.
|
|
+# memdebug=4 and memdebug=5 requires debug fs to be mounted.
|
|
+# And there is no need to umount it.
|
|
+
|
|
+type getargnum >/dev/null 2>&1 || . /lib/dracut-lib.sh
|
|
+
|
|
+# "sys/kernel/tracing" has the priority if exists.
|
|
+get_trace_base() {
|
|
+ # trace access through debugfs would be obsolete if "/sys/kernel/tracing" is available.
|
|
+ if [ -d "/sys/kernel/tracing" ]; then
|
|
+ echo "/sys/kernel"
|
|
+ else
|
|
+ echo "/sys/kernel/debug"
|
|
+ fi
|
|
+}
|
|
+
|
|
+is_debugfs_ready() {
|
|
+ [ -f "$(get_trace_base)/tracing/trace" ]
|
|
+}
|
|
+
|
|
+prepare_debugfs() {
|
|
+ local trace_base
|
|
+
|
|
+ trace_base=$(get_trace_base)
|
|
+ # old debugfs interface case.
|
|
+ if ! [ -d "$trace_base/tracing" ]; then
|
|
+ mount none -t debugfs $trace_base
|
|
+ # new tracefs interface case.
|
|
+ elif ! [ -f "$trace_base/tracing/trace" ]; then
|
|
+ mount none -t tracefs "$trace_base/tracing"
|
|
+ fi
|
|
+
|
|
+ if ! [ -f "$trace_base/tracing/trace" ]; then
|
|
+ echo "WARN: failed to mount debugfs"
|
|
+ return 1
|
|
+ fi
|
|
+}
|
|
+
|
|
+if ! is_debugfs_ready ; then
|
|
+ prepare_debugfs
|
|
+fi
|
|
+
|
|
+if [ -n "$DEBUG_MEM_LEVEL" ]; then
|
|
+ if [ "$DEBUG_MEM_LEVEL" -ge 5 ]; then
|
|
+ echo "memstrack - will report kernel module memory usage summary and top allocation stack"
|
|
+ memstrack --report module_summary,module_top --notui --throttle 80 -o /.memstrack &
|
|
+ elif [ "$DEBUG_MEM_LEVEL" -ge 4 ]; then
|
|
+ echo "memstrack - will report memory usage summary"
|
|
+ memstrack --report module_summary --notui --throttle 80 -o /.memstrack &
|
|
+ else
|
|
+ exit 0;
|
|
+ fi
|
|
+fi
|
|
+
|
|
+PID=$!
|
|
+RET=$?
|
|
+
|
|
+if [ $RET -ne 0 ]; then
|
|
+ echo "Failed to start memstrack, exit status: $RET"
|
|
+ exit $RET
|
|
+fi
|
|
+
|
|
+# Wait a second for memstrack to setup everything, avoid missing any event
|
|
+sleep 1
|
|
+
|
|
+echo $PID > /run/memstrack.pid
|
|
+disown
|
|
diff --git a/modules.d/99memstrack/memstrack.service b/modules.d/99memstrack/memstrack.service
|
|
new file mode 100644
|
|
index 00000000..6b47adef
|
|
--- /dev/null
|
|
+++ b/modules.d/99memstrack/memstrack.service
|
|
@@ -0,0 +1,13 @@
|
|
+[Unit]
|
|
+Description=Memstrack Anylazing Service
|
|
+DefaultDependencies=no
|
|
+Before=dracut-cmdline.service systemd-udevd.service local-fs-pre.target
|
|
+IgnoreOnIsolate=true
|
|
+
|
|
+[Service]
|
|
+Type=simple
|
|
+ExecStart=/bin/memstrack-start
|
|
+PIDFile=/run/memstrack.pid
|
|
+StandardInput=null
|
|
+StandardOutput=syslog+console
|
|
+StandardError=syslog+console
|
|
diff --git a/modules.d/99memstrack/module-setup.sh b/modules.d/99memstrack/module-setup.sh
|
|
new file mode 100755
|
|
index 00000000..d5bacb4d
|
|
--- /dev/null
|
|
+++ b/modules.d/99memstrack/module-setup.sh
|
|
@@ -0,0 +1,27 @@
|
|
+#!/usr/bin/bash
|
|
+
|
|
+check() {
|
|
+ if type -P memstrack >/dev/null; then
|
|
+ dinfo "memstrack is available"
|
|
+ return 0
|
|
+ fi
|
|
+
|
|
+ dinfo "memstrack is not available"
|
|
+ dinfo "If you need to use rd.memdebug>=4, please install memstrack"
|
|
+
|
|
+ return 1
|
|
+}
|
|
+
|
|
+depends() {
|
|
+ return 0
|
|
+}
|
|
+
|
|
+install() {
|
|
+ inst "/bin/memstrack" "/bin/memstrack"
|
|
+
|
|
+ inst "$moddir/memstrack-start.sh" "/bin/memstrack-start"
|
|
+ inst_hook cleanup 99 "$moddir/memstrack-report.sh"
|
|
+
|
|
+ inst "$moddir/memstrack.service" "$systemdsystemunitdir/memstrack.service"
|
|
+ systemctl -q --root "$initdir" add-wants initrd.target memstrack.service
|
|
+}
|
|
|