Hi
I have been changing the write of buffers in the IO layer, and I
have found that the system gets a lot of contention in
__wait_on_super(). I am using test3-pre1 + kdb patch. I see also the
stalls/vmstat strange output without the kdb patch. I use it to be
able to see the back-traces. You can also look at the vmstat output.
I see that vmstat 1 output when running dbench 48, the results doesn't
make sense that we are almost not doing IO in an IO test :((((
vmstat output:
r b w swpd free buff cache si so bi bo in cs us sy id
0 48 2 5004 3112 1948 112652 0 0 0 151 221 238 2 1 96
0 48 2 5004 3108 1948 112652 0 0 0 133 195 188 0 0 99
0 48 2 5004 3108 1948 112652 0 0 0 141 192 184 0 0 99
^^^^
when things are working properly, that field is between 1000-2000.
Now the traces:
Almost all the dbench processes are stuck with the first back-trace.
Someone find some sense to that output and/or know how to fix that
problem.
Any comment are welcome. If you need more information, let me know.
Later, Juan.
Most common back-trace:
0xc721bdcc 0xc0116fbd schedule+0x389 (0xc6e3bb60, 0x98e01, 0xc721bea0, 0x1234567
, 0xc721a000)
kernel .text 0xc0100000 0xc0116c34 0xc01173c0
0xc01363ac __wait_on_super+0x184 (0xc13f4c00)
kernel .text 0xc0100000 0xc0136228 0xc0136420
0xc01523f5 ext2_alloc_block+0x21 (0xc6e3bb60, 0x98e01, 0xc721bea0)
kernel .text 0xc0100000 0xc01523d4 0xc015246c
0xc721be5c 0xc01528a2 block_getblk+0x15e (0xc6e3bb60, 0xc2f75c20, 0x8d, 0x99, 0x
c721bea0)
kernel .text 0xc0100000 0xc0152744 0xc0152a78
0xc721beac 0xc0152ee0 ext2_get_block+0x468 (0xc6e3bb60, 0x99, 0xc221e440, 0x1)
kernel .text 0xc0100000 0xc0152a78 0xc0152fd0
0xc721bef4 0xc0133af3 __block_prepare_write+0xe7 (0xc6e3bb60, 0xc11c1ee8, 0x0, 0
x1000, 0xc0152a78)
kernel .text 0xc0100000 0xc0133a0c 0xc0133c00
0xc721bf18 0xc0134131 block_prepare_write+0x21 (0xc11c1ee8, 0x0, 0x1000, 0xc0152
a78)
kernel .text 0xc0100000 0xc0134110 0xc013414c
0xc721bf30 0xc01531e1 ext2_prepare_write+0x19 (0xc719b5e0, 0xc11c1ee8, 0x0, 0x10
00)
kernel .text 0xc0100000 0xc01531c8 0xc01531e8
0xc721bf90 0xc0127b8d generic_file_write+0x305 (0xc719b5e0, 0x8054469, 0x6fba, 0
xc719b600)
kernel .text 0xc0100000 0xc0127888 0xc0127cf0
0xc721bfbc 0xc0130ea8 sys_write+0xe8 (0x9, 0x804b460, 0xffc3, 0x28, 0x1092)
kernel .text 0xc0100000 0xc0130dc0 0xc0130ed0
0xc0109874 system_call+0x34
kernel .text 0xc0100000 0xc0109840 0xc0109878
or in:
0xc320de58 0xc0116fbd schedule+0x389 (0xc4c130c0, 0xc13f4c00, 0xc4c138e0, 0x1234
567, 0xc320c000)
kernel .text 0xc0100000 0xc0116c34 0xc01173c0
0xc01363ac __wait_on_super+0x184 (0xc13f4c00)
kernel .text 0xc0100000 0xc0136228 0xc0136420
0xc0151ce9 ext2_new_inode+0x6d (0xc4c130c0, 0x8180, 0xc320df00)
kernel .text 0xc0100000 0xc0151c7c 0xc01521dc
0xc320df04 0xc0154555 ext2_create+0x1d (0xc4c130c0, 0xc5c17b60, 0x8180)
kernel .text 0xc0100000 0xc0154538 0xc01545f4
0xc320df28 0xc013df64 vfs_create+0xdc (0xc4c130c0, 0xc5c17b60, 0x180)
kernel .text 0xc0100000 0xc013de88 0xc013dfcc
0xc320df58 0xc013e198 open_namei+0x1cc (0xc4c12000, 0x243, 0x180, 0xc320df7c)
kernel .text 0xc0100000 0xc013dfcc 0xc013e6cc
0xc320df98 0xc0130376 filp_open+0x3a (0xc4c12000, 0x242, 0x180)
kernel .text 0xc0100000 0xc013033c 0xc0130398
0xc320dfbc 0xc01306dc sys_open+0x68 (0xbffffc4c, 0x242, 0x180, 0x242, 0xbffffc4c
)
kernel .text 0xc0100000 0xc0130674 0xc01307dc
0xc0109874 system_call+0x34
kernel .text 0xc0100000 0xc0109840 0xc0109878
or in:
0xc5a19e50 0xc0116fbd schedule+0x389 (0xc5b58a00, 0x1000, 0xfffffff4, 0xc5a18000
, 0x1234567)
kernel .text 0xc0100000 0xc0116c34 0xc01173c0
0xc0131db8 __wait_on_buffer+0x1d8 (0xc5b58a00)
kernel .text 0xc0100000 0xc0131be0 0xc0131e34
0xc0133773 unmap_buffer+0x33 (0xc5b58a00)
kernel .text 0xc0100000 0xc0133740 0xc01337a0
0xc5a19eb8 0xc01338e1 unmap_underlying_metadata+0x29 (0xc536fc00)
[1]more>
kernel .text 0xc0100000 0xc01338b8 0xc01338f0
0xc5a19ef4 0xc0133b0f __block_prepare_write+0x103 (0xc62c25e0, 0xc110e680, 0x0,
0x1000, 0xc0152a78)
kernel .text 0xc0100000 0xc0133a0c 0xc0133c00
0xc5a19f18 0xc0134131 block_prepare_write+0x21 (0xc110e680, 0x0, 0x1000, 0xc0152
a78)
kernel .text 0xc0100000 0xc0134110 0xc013414c
0xc5a19f30 0xc01531e1 ext2_prepare_write+0x19 (0xc59fd8e0, 0xc110e680, 0x0, 0x10
00)
kernel .text 0xc0100000 0xc01531c8 0xc01531e8
0xc5a19f90 0xc0127b8d generic_file_write+0x305 (0xc59fd8e0, 0x8050462, 0xafc1, 0
xc59fd900)
kernel .text 0xc0100000 0xc0127888 0xc0127cf0
0xc5a19fbc 0xc0130ea8 sys_write+0xe8 (0x6, 0x804b460, 0xffc3, 0x10, 0x1061)
kernel .text 0xc0100000 0xc0130dc0 0xc0130ed0
0xc0109874 system_call+0x34
kernel .text 0xc0100000 0xc0109840 0xc0109878
The super lock _appears_ to be held by a process with:
0xc2ca3e44 0xc0116fbd schedule+0x389 (0xc7d33500, 0xc13f4c00, 0x0, 0xc2ca2000, 0
x1234567)
kernel .text 0xc0100000 0xc0116c34 0xc01173c0
0xc0131db8 __wait_on_buffer+0x1d8 (0xc221ec40)
kernel .text 0xc0100000 0xc0131be0 0xc0131e34
0xc0132fb0 bread+0x44 (0x307, 0x140001, 0x1000)
kernel .text 0xc0100000 0xc0132f6c 0xc0132fcc
0xc2ca3ec0 0xc01518e8 read_inode_bitmap+0x38 (0xc13f4c00, 0x28, 0x0)
kernel .text 0xc0100000 0xc01518b0 0xc0151934
0xc2ca3eec 0xc0151b03 load_inode_bitmap+0x1cf (0xc13f4c00, 0x28)
kernel .text 0xc0100000 0xc0151934 0xc0151b14
0xc2ca3f34 0xc0151e9a ext2_new_inode+0x21e (0xc552e140, 0x4000, 0xc2ca3f58)
kernel .text 0xc0100000 0xc0151c7c 0xc01521dc
0xc2ca3f5c 0xc01546d4 ext2_mkdir+0x30 (0xc552e140, 0xc78cec40, 0x1c0)
kernel .text 0xc0100000 0xc01546a4 0xc0154890
0xc2ca3f7c 0xc013ebd1 vfs_mkdir+0xc5 (0xc552e140, 0xc78cec40, 0x1c0)
kernel .text 0xc0100000 0xc013eb0c 0xc013ec38
0xc2ca3fbc 0xc013eccf sys_mkdir+0x97 (0xbffff863, 0x1c0, 0xbffff863, 0xbffff855,
0x804a217)
kernel .text 0xc0100000 0xc013ec38 0xc013edb8
0xc0109874 system_call+0x34
kernel .text 0xc0100000 0xc0109840 0xc0109878
There are other dbench processes waiting in __wait_on_buffer:
0xc4419e28 0xc0116fbd schedule+0x389 (0xc4419ec8, 0xc4192820, 0xc3c8c760, 0xc441
8000, 0x1234567)
kernel .text 0xc0100000 0xc0116c34 0xc01173c0
0xc0131db8 __wait_on_buffer+0x1d8 (0xc3c8c760)
kernel .text 0xc0100000 0xc0131be0 0xc0131e34
0xc0153ea3 ext2_find_entry+0x153 (0xc4192820, 0xc75e9140, 0xc, 0xc441
9f0c)
kernel .text 0xc0100000 0xc0153d50 0xc0154038
0xc4419f10 0xc015406f ext2_lookup+0x37 (0xc4192820, 0xc75e90e0)
kernel .text 0xc0100000 0xc0154038 0xc01540cc
[1]more>
0xc4419f2c 0xc013dd31 lookup_hash+0x75 (0xc4419f84, 0xc7992b80)
kernel .text 0xc0100000 0xc013dcbc 0xc013dd54
0xc4419f58 0xc013e110 open_namei+0x144 (0xc3ff2000, 0x243, 0x180, 0xc4419f7c)
kernel .text 0xc0100000 0xc013dfcc 0xc013e6cc
0xc4419f98 0xc0130376 filp_open+0x3a (0xc3ff2000, 0x242, 0x180)
kernel .text 0xc0100000 0xc013033c 0xc0130398
0xc4419fbc 0xc01306dc sys_open+0x68 (0xbffffc4c, 0x242, 0x180, 0x242, 0xbffffc4c
)
kernel .text 0xc0100000 0xc0130674 0xc01307dc
0xc0109874 system_call+0x34
kernel .text 0xc0100000 0xc0109840 0xc0109878
--
In theory, practice and theory are the same, but in practice they
are different -- Larry McVoy
