On 03/28/2017 09:30 AM, Jens Axboe wrote:
> On 03/28/2017 10:25 AM, Bart Van Assche wrote:
>> I do not know whether it would be possible to modify the test such that only
>> the dm driver is involved but no SCSI code.
>
> How about the other way around? Just SCSI, but no dm?
Hello Jens,
Sorry but it could take a long time to figure out how to reproduce this
issue if I start modifying the test. BTW, the patch I just posted
("blk-mq: Export queue state through /sys/kernel/debug/block/*/state")
allows me to trigger a blk-mq queue run from user space. If the lockup
occurs and I use that facility to trigger a queue run the test proceeds.
The command I used to trigger a queue run is as follows:
for a in /sys/kernel/debug/block/*/state; do echo 1 >$a; wait
> Thanks for running it again, but it's the wrong state file. I should have
> been more clear. The one I'm interested in is in the mq/<num>/ directories,
> like the 'tags' etc files.
>
> Ala:
>
> for f in "$d"/{dispatch,state,tags*,cpu*/rq_list}; do
Ah, thanks, that makes it clear :-)
> Also, can you include the involved dm devices as well for this state
> dump?
I would like to, but the 02-sq-on-mq test configures the dm device
nodes in single queue mode and there is only information available
under /sys/kernel/debug/block/ for blk-mq devices ...
Anyway, the updated script:
#!/bin/bash
show_state() {
local a dev=$1
for a in device/state queue/scheduler; do
[ -e "$dev/$a" ] && grep -aH '' "$dev/$a"
done
}
cd /sys/class/block || exit $?
for dev in *; do
if [ -e "$dev/mq" ]; then
echo "$dev"
pending=0
for f in "$dev"/mq/*/{pending,*/rq_list}; do
[ -e "$f" ] || continue
if { read -r line1 && read -r line2; } <"$f"; then
echo "$f"
echo "$line1 $line2" >/dev/null
head -n 9 "$f"
((pending++))
fi
done
(
busy=0
cd /sys/kernel/debug/block >&/dev/null &&
for d in "$dev"/mq/*; do
[ ! -d "$d" ] && continue
grep -q '^busy=0$' "$d/tags" && continue
((busy++))
for f in "$d"/{dispatch,state,tags*,cpu*/rq_list}; do
[ -e "$f" ] && grep -aH '' "$f"
done
done
exit $busy
)
pending=$((pending+$?))
if [ "$pending" -gt 0 ]; then
grep -aH '' /sys/kernel/debug/block/"$dev"/state
show_state "$dev"
fi
fi
done
And the output for the test run of today:
sda
sdb
sdd
sdd/mq/0/dispatch:ffff88036437d140 {.cmd_flags=0xca01, .rq_flags=0x2040,
.tag=53, .internal_tag=-1}
sdd/mq/0/state:0x4
sdd/mq/0/tags:nr_tags=62
sdd/mq/0/tags:nr_reserved_tags=0
sdd/mq/0/tags:active_queues=0
sdd/mq/0/tags:
sdd/mq/0/tags:bitmap_tags:
sdd/mq/0/tags:depth=62
sdd/mq/0/tags:busy=31
sdd/mq/0/tags:bits_per_word=8
sdd/mq/0/tags:map_nr=8
sdd/mq/0/tags:alloc_hint={48, 48, 38, 44, 54, 6, 52, 23, 30, 6, 51, 26, 61, 45,
9, 56, 55, 13, 44, 45, 12, 12, 23, 42, 44, 24, 41, 0, 54, 4, 4, 45}
sdd/mq/0/tags:wake_batch=7
sdd/mq/0/tags:wake_index=0
sdd/mq/0/tags:ws={
sdd/mq/0/tags: {.wait_cnt=7, .wait=inactive},
sdd/mq/0/tags: {.wait_cnt=7, .wait=inactive},
sdd/mq/0/tags: {.wait_cnt=7, .wait=inactive},
sdd/mq/0/tags: {.wait_cnt=7, .wait=inactive},
sdd/mq/0/tags: {.wait_cnt=7, .wait=inactive},
sdd/mq/0/tags: {.wait_cnt=7, .wait=inactive},
sdd/mq/0/tags: {.wait_cnt=7, .wait=inactive},
sdd/mq/0/tags: {.wait_cnt=7, .wait=inactive},
sdd/mq/0/tags:}
sdd/mq/0/tags:round_robin=0
sdd/mq/0/tags_bitmap:00000000: ffff 7f00 0000 e01f
sdd/mq/0/cpu7/rq_list:ffff88036437e880 {.cmd_flags=0xca01, .rq_flags=0x2040,
.tag=54, .internal_tag=-1}
sdd/mq/0/cpu7/rq_list:ffff8803f7ef0000 {.cmd_flags=0xca01, .rq_flags=0x2040,
.tag=55, .internal_tag=-1}
sdd/mq/0/cpu7/rq_list:ffff8803f7ef1740 {.cmd_flags=0xca01, .rq_flags=0x2040,
.tag=56, .internal_tag=-1}
sdd/mq/0/cpu7/rq_list:ffff8803f7ef2e80 {.cmd_flags=0xca01, .rq_flags=0x2040,
.tag=57, .internal_tag=-1}
sdd/mq/0/cpu7/rq_list:ffff8803f7ef45c0 {.cmd_flags=0xca01, .rq_flags=0x2040,
.tag=58, .internal_tag=-1}
sdd/mq/0/cpu7/rq_list:ffff8803f7ef5d00 {.cmd_flags=0xca01, .rq_flags=0x2040,
.tag=59, .internal_tag=-1}
sdd/mq/0/cpu7/rq_list:ffff8803f7ef7440 {.cmd_flags=0xca01, .rq_flags=0x2040,
.tag=60, .internal_tag=-1}
sdd/mq/0/cpu7/rq_list:ffff880386760000 {.cmd_flags=0xca01, .rq_flags=0x2040,
.tag=0, .internal_tag=-1}
sdd/mq/0/cpu7/rq_list:ffff880386761740 {.cmd_flags=0xca01, .rq_flags=0x2040,
.tag=1, .internal_tag=-1}
sdd/mq/0/cpu7/rq_list:ffff880386762e80 {.cmd_flags=0xca01, .rq_flags=0x2040,
.tag=2, .internal_tag=-1}
sdd/mq/0/cpu7/rq_list:ffff8803867645c0 {.cmd_flags=0xca01, .rq_flags=0x2040,
.tag=3, .internal_tag=-1}
sdd/mq/0/cpu7/rq_list:ffff880386765d00 {.cmd_flags=0xca01, .rq_flags=0x2040,
.tag=4, .internal_tag=-1}
sdd/mq/0/cpu7/rq_list:ffff880386767440 {.cmd_flags=0xca01, .rq_flags=0x2040,
.tag=5, .internal_tag=-1}
sdd/mq/0/cpu7/rq_list:ffff880386768b80 {.cmd_flags=0xca01, .rq_flags=0x2040,
.tag=6, .internal_tag=-1}
sdd/mq/0/cpu7/rq_list:ffff88038676a2c0 {.cmd_flags=0xca01, .rq_flags=0x2040,
.tag=7, .internal_tag=-1}
sdd/mq/0/cpu7/rq_list:ffff88038676ba00 {.cmd_flags=0xca01, .rq_flags=0x2040,
.tag=8, .internal_tag=-1}
sdd/mq/0/cpu7/rq_list:ffff88038676d140 {.cmd_flags=0xca01, .rq_flags=0x2040,
.tag=9, .internal_tag=-1}
sdd/mq/0/cpu7/rq_list:ffff88038676e880 {.cmd_flags=0xca01, .rq_flags=0x2040,
.tag=10, .internal_tag=-1}
sdd/mq/0/cpu7/rq_list:ffff8803f8650000 {.cmd_flags=0xca01, .rq_flags=0x2040,
.tag=11, .internal_tag=-1}
sdd/mq/0/cpu7/rq_list:ffff8803f8651740 {.cmd_flags=0xca01, .rq_flags=0x2040,
.tag=12, .internal_tag=-1}
sdd/mq/0/cpu7/rq_list:ffff8803f8652e80 {.cmd_flags=0xca01, .rq_flags=0x2040,
.tag=13, .internal_tag=-1}
sdd/mq/0/cpu7/rq_list:ffff8803f86545c0 {.cmd_flags=0xca01, .rq_flags=0x2040,
.tag=14, .internal_tag=-1}
sdd/mq/0/cpu7/rq_list:ffff8803f8655d00 {.cmd_flags=0xca01, .rq_flags=0x2040,
.tag=15, .internal_tag=-1}
sdd/mq/0/cpu7/rq_list:ffff8803f8657440 {.cmd_flags=0xca01, .rq_flags=0x2040,
.tag=16, .internal_tag=-1}
sdd/mq/0/cpu7/rq_list:ffff8803f8658b80 {.cmd_flags=0xca01, .rq_flags=0x2040,
.tag=17, .internal_tag=-1}
sdd/mq/0/cpu7/rq_list:ffff8803f865a2c0 {.cmd_flags=0xca01, .rq_flags=0x2040,
.tag=18, .internal_tag=-1}
sdd/mq/0/cpu7/rq_list:ffff8803f865ba00 {.cmd_flags=0xca01, .rq_flags=0x2040,
.tag=19, .internal_tag=-1}
sdd/mq/0/cpu7/rq_list:ffff8803f865d140 {.cmd_flags=0xca01, .rq_flags=0x2040,
.tag=20, .internal_tag=-1}
sdd/mq/0/cpu7/rq_list:ffff8803f865e880 {.cmd_flags=0xca01, .rq_flags=0x2040,
.tag=21, .internal_tag=-1}
sdd/mq/0/cpu7/rq_list:ffff8803fb630000 {.cmd_flags=0xca01, .rq_flags=0x2040,
.tag=22, .internal_tag=-1}
/sys/kernel/debug/block/sdd/state:SAME_COMP STACKABLE IO_STAT INIT_DONE POLL
sdd/device/state:running
sdd/queue/scheduler:[none]
sde
sde/mq/0/state:0x0
sde/mq/0/tags:nr_tags=62
sde/mq/0/tags:nr_reserved_tags=0
sde/mq/0/tags:active_queues=0
sde/mq/0/tags:
sde/mq/0/tags:bitmap_tags:
sde/mq/0/tags:depth=62
sde/mq/0/tags:busy=31
sde/mq/0/tags:bits_per_word=8
sde/mq/0/tags:map_nr=8
sde/mq/0/tags:alloc_hint={48, 48, 38, 44, 54, 6, 52, 23, 30, 6, 51, 26, 61, 45,
9, 56, 55, 13, 44, 45, 12, 12, 23, 42, 44, 24, 41, 0, 54, 4, 4, 45}
sde/mq/0/tags:wake_batch=7
sde/mq/0/tags:wake_index=0
sde/mq/0/tags:ws={
sde/mq/0/tags: {.wait_cnt=7, .wait=inactive},
sde/mq/0/tags: {.wait_cnt=7, .wait=inactive},
sde/mq/0/tags: {.wait_cnt=7, .wait=inactive},
sde/mq/0/tags: {.wait_cnt=7, .wait=inactive},
sde/mq/0/tags: {.wait_cnt=7, .wait=inactive},
sde/mq/0/tags: {.wait_cnt=7, .wait=inactive},
sde/mq/0/tags: {.wait_cnt=7, .wait=inactive},
sde/mq/0/tags: {.wait_cnt=7, .wait=inactive},
sde/mq/0/tags:}
sde/mq/0/tags:round_robin=0
sde/mq/0/tags_bitmap:00000000: ffff 7f00 0000 e01f
/sys/kernel/debug/block/sde/state:SAME_COMP STACKABLE IO_STAT INIT_DONE POLL
sde/device/state:running
sde/queue/scheduler:[none]
sdf
sdg
sdh
sdi
sdj
sdk
sr0
