123 lines
3.2 KiB
Diff
123 lines
3.2 KiB
Diff
From 3973f65293ce69ddd7c19be9a7a796abfe80e370 Mon Sep 17 00:00:00 2001
|
|
From: Yu Kuai <yukuai3@huawei.com>
|
|
Date: Mon, 29 May 2023 21:28:21 +0800
|
|
Subject: [PATCH 146/165] tests: add a regression test for raid10 deadlock
|
|
|
|
The deadlock is described in [1], it's fixed first by [2], however,
|
|
it turns out this commit will trigger other problems[3], hence this
|
|
commit will be reverted and the deadlock is supposed to be fixed by [1].
|
|
|
|
[1] https://lore.kernel.org/linux-raid/20230322064122.2384589-5-yukuai1@huaweicloud.com/
|
|
[2] https://lore.kernel.org/linux-raid/20220621031129.24778-1-guoqing.jiang@linux.dev/
|
|
[3] https://lore.kernel.org/linux-raid/20230322064122.2384589-2-yukuai1@huaweicloud.com/
|
|
|
|
Signed-off-by: Yu Kuai <yukuai3@huawei.com>
|
|
Signed-off-by: Jes Sorensen <jes@trained-monkey.org>
|
|
---
|
|
tests/24raid10deadlock | 88 +++++++++++++++++++++++++++++
|
|
tests/24raid10deadlock.inject_error | 0
|
|
2 files changed, 88 insertions(+)
|
|
create mode 100644 tests/24raid10deadlock
|
|
create mode 100644 tests/24raid10deadlock.inject_error
|
|
|
|
diff --git a/tests/24raid10deadlock b/tests/24raid10deadlock
|
|
new file mode 100644
|
|
index 00000000..ee330aa9
|
|
--- /dev/null
|
|
+++ b/tests/24raid10deadlock
|
|
@@ -0,0 +1,88 @@
|
|
+devs="$dev0 $dev1 $dev2 $dev3"
|
|
+runtime=120
|
|
+pid=""
|
|
+action_pid=""
|
|
+
|
|
+set_up_injection()
|
|
+{
|
|
+ echo -1 > /sys/kernel/debug/fail_make_request/times
|
|
+ echo 1 > /sys/kernel/debug/fail_make_request/probability
|
|
+ echo 0 > /sys/kernel/debug/fail_make_request/verbose
|
|
+ echo 1 > /sys/block/${1##*/}/make-it-fail
|
|
+}
|
|
+
|
|
+clean_up_injection()
|
|
+{
|
|
+ echo 0 > /sys/block/${1##*/}/make-it-fail
|
|
+ echo 0 > /sys/kernel/debug/fail_make_request/times
|
|
+ echo 0 > /sys/kernel/debug/fail_make_request/probability
|
|
+ echo 2 > /sys/kernel/debug/fail_make_request/verbose
|
|
+}
|
|
+
|
|
+test_rdev()
|
|
+{
|
|
+ while true; do
|
|
+ mdadm -f $md0 $1 &> /dev/null
|
|
+ mdadm -r $md0 $1 &> /dev/null
|
|
+ mdadm --zero-superblock $1 &> /dev/null
|
|
+ mdadm -a $md0 $1 &> /dev/null
|
|
+ sleep $2
|
|
+ done
|
|
+}
|
|
+
|
|
+test_write_action()
|
|
+{
|
|
+ while true; do
|
|
+ echo frozen > /sys/block/md0/md/sync_action
|
|
+ echo idle > /sys/block/md0/md/sync_action
|
|
+ sleep 0.1
|
|
+ done
|
|
+}
|
|
+
|
|
+set_up_test()
|
|
+{
|
|
+ fio -h &> /dev/null || die "fio not found"
|
|
+
|
|
+ # create a simple raid10
|
|
+ mdadm -Cv -R -n 4 -l10 $md0 $devs || die "create raid10 failed"
|
|
+}
|
|
+
|
|
+clean_up_test()
|
|
+{
|
|
+ clean_up_injection $dev0
|
|
+ pkill -9 fio
|
|
+ kill -9 $pid
|
|
+ kill -9 $action_pid
|
|
+
|
|
+ sleep 1
|
|
+
|
|
+ if ps $action_pid | tail -1 | awk '{print $3}' | grep D; then
|
|
+ die "thread that is writing sysfs is stuck in D state, deadlock is triggered"
|
|
+ fi
|
|
+ mdadm -S $md0
|
|
+}
|
|
+
|
|
+cat /sys/kernel/debug/fail_make_request/times || die "fault injection is not enabled"
|
|
+
|
|
+trap 'clean_up_test' EXIT
|
|
+
|
|
+set_up_test || die "set up test failed"
|
|
+
|
|
+# backgroup io pressure
|
|
+fio -filename=$md0 -rw=randwrite -direct=1 -name=test -bs=4k -numjobs=16 -iodepth=16 &
|
|
+
|
|
+# trigger add/remove device by io failure
|
|
+set_up_injection $dev0
|
|
+test_rdev $dev0 2 &
|
|
+pid="$pid $!"
|
|
+
|
|
+# add/remove device directly
|
|
+test_rdev $dev3 10 &
|
|
+pid="$pid $!"
|
|
+
|
|
+test_write_action &
|
|
+action_pid="$!"
|
|
+
|
|
+sleep $runtime
|
|
+
|
|
+exit 0
|
|
diff --git a/tests/24raid10deadlock.inject_error b/tests/24raid10deadlock.inject_error
|
|
new file mode 100644
|
|
index 00000000..e69de29b
|
|
--
|
|
2.40.1
|
|
|