This list has been deprecated. Please subscribe to the new devel list at
lists.nfs-ganesha.org.
After applying https://review.gerrithub.io/c/ffilz/nfs-ganesha/+/441566 our
long-running test is showing continually increasing memory usage. Eventually
ganesha.nfsd consumes all memory in the box and we OOM. Looking at a core, it
appears that the mdcache lru contains far more entries than the high water mark
would normally allow.
We have...
CacheInode {
Dir_Chunk = 500000;
Entries_HWMark = 500000;
}
...and after we run for a while, we observe from a core (obtained at runtime
using gcore)...
(gdb) print lru_state
$8 = {entries_hiwat = 500000, entries_used = 2437134, chunks_hiwat = 100000,
chunks_used = 2002, fds_system_imposed = 400000, fds_hard_limit = 396000,
fds_hiwat = 360000, fds_lowat = 200000, futility = 0, per_lane_work = 50,
biggest_window = 160000, prev_fd_count = 160, prev_time = 1548692973, fd_state
= 0}
So we have 2.4M entries with a high water mark of 500K. The difference appears
to account for the unexpected memory usage.
This seems to be new behavior after applying the above patch, although it's
hard to be certain because earlier we hit the core with entries being freed
twice before running into the high memory usage.
Some more info from the core:
(gdb) print LRU
$2 = {{L1 = {q = {next = 0xe4788210, prev = 0x23a701f0}, id = LRU_ENTRY_L1,
size = 147689}, L2 = {q = {
next = 0x23a8b7b0, prev = 0xe478ef80}, id = LRU_ENTRY_L2, size = 150},
cleanup = {q = {next = 0xcca1ff70,
prev = 0x139e2bf0}, id = LRU_ENTRY_CLEANUP, size = 748}, mtx = {__data
= {__lock = 0, __count = 0,
__owner = 0, __nusers = 0, __kind = 0, __spins = 0, __list = {__prev =
0x0, __next = 0x0}},
__size = '\000' <repeats 39 times>, __align = 0}, iter = {active = false,
glist = 0x0, glistn = 0x0},
__pad0 = '\000' <repeats 63 times>}, {L1 = {q = {next = 0x1dd98360, prev =
0x1100cc30}, id = LRU_ENTRY_L1,
size = 143935}, L2 = {q = {next = 0xfa204e0, prev = 0x1e26a0a0}, id =
LRU_ENTRY_L2, size = 145}, cleanup = {
q = {next = 0xb26f8720, prev = 0x18f1810}, id = LRU_ENTRY_CLEANUP, size =
811}, mtx = {__data = {__lock = 0,
__count = 0, __owner = 0, __nusers = 0, __kind = 0, __spins = 0, __list
= {__prev = 0x0, __next = 0x0}},
__size = '\000' <repeats 39 times>, __align = 0}, iter = {active = false,
glist = 0x0, glistn = 0x0},
__pad0 = '\000' <repeats 63 times>}, {L1 = {q = {next = 0x4903dc0, prev =
0x96cb4320}, id = LRU_ENTRY_L1,
size = 141362}, L2 = {q = {next = 0x2eea11f0, prev = 0x48fd050}, id =
LRU_ENTRY_L2, size = 147}, cleanup = {
q = {next = 0xdc74e5b0, prev = 0xca6f990}, id = LRU_ENTRY_CLEANUP, size =
817}, mtx = {__data = {__lock = 0,
__count = 0, __owner = 0, __nusers = 0, __kind = 0, __spins = 0, __list
= {__prev = 0x0, __next = 0x0}},
__size = '\000' <repeats 39 times>, __align = 0}, iter = {active = false,
glist = 0x0, glistn = 0x0},
__pad0 = '\000' <repeats 63 times>}, {L1 = {q = {next = 0x79dcf850, prev =
0x7d9445d0}, id = LRU_ENTRY_L1,
size = 122171}, L2 = {q = {next = 0x7e823b30, prev = 0x79dcfc90}, id =
LRU_ENTRY_L2, size = 146}, cleanup = {
q = {next = 0x3245ddb0, prev = 0x1121e510}, id = LRU_ENTRY_CLEANUP, size
= 761}, mtx = {__data = {__lock = 0,
__count = 0, __owner = 0, __nusers = 0, __kind = 0, __spins = 0, __list
= {__prev = 0x0, __next = 0x0}},
__size = '\000' <repeats 39 times>, __align = 0}, iter = {active = false,
glist = 0x0, glistn = 0x0},
__pad0 = '\000' <repeats 63 times>}, {L1 = {q = {next = 0x1a94af70, prev =
0xec32de0}, id = LRU_ENTRY_L1,
size = 142951}, L2 = {q = {next = 0xec25300, prev = 0x1a944200}, id =
LRU_ENTRY_L2, size = 150}, cleanup = {
q = {next = 0xcf8eada0, prev = 0xba9afd0}, id = LRU_ENTRY_CLEANUP, size =
759}, mtx = {__data = {__lock = 0,
__count = 0, __owner = 0, __nusers = 0, __kind = 0, __spins = 0, __list
= {__prev = 0x0, __next = 0x0}},
__size = '\000' <repeats 39 times>, __align = 0}, iter = {active = false,
glist = 0x0, glistn = 0x0},
__pad0 = '\000' <repeats 63 times>}, {L1 = {q = {next = 0x49b54c90, prev =
0x1896b90}, id = LRU_ENTRY_L1,
size = 133691}, L2 = {q = {next = 0x413dbca0, prev = 0x66d15ce0}, id =
LRU_ENTRY_L2, size = 142}, cleanup = {
q = {next = 0xe249dcb0, prev = 0x135e9ae0}, id = LRU_ENTRY_CLEANUP, size
= 804}, mtx = {__data = {__lock = 0,
__count = 0, __owner = 0, __nusers = 0, __kind = 0, __spins = 0, __list
= {__prev = 0x0, __next = 0x0}},
__size = '\000' <repeats 39 times>, __align = 0}, iter = {active = false,
glist = 0x0, glistn = 0x0},
__pad0 = '\000' <repeats 63 times>}, {L1 = {q = {next = 0x25eba620, prev =
0x96783780}, id = LRU_ENTRY_L1,
---Type <return> to continue, or q <return> to quit---
size = 146051}, L2 = {q = {next = 0x1d179420, prev = 0x25eb38b0}, id =
LRU_ENTRY_L2, size = 150}, cleanup = {
q = {next = 0xab572d30, prev = 0x11a0d550}, id = LRU_ENTRY_CLEANUP, size
= 765}, mtx = {__data = {__lock = 0,
__count = 0, __owner = 0, __nusers = 0, __kind = 0, __spins = 0, __list
= {__prev = 0x0, __next = 0x0}},
__size = '\000' <repeats 39 times>, __align = 0}, iter = {active = false,
glist = 0x0, glistn = 0x0},
__pad0 = '\000' <repeats 63 times>}, {L1 = {q = {next = 0x15bc6d50, prev =
0x3c292070}, id = LRU_ENTRY_L1,
size = 135767}, L2 = {q = {next = 0x5d1fad10, prev = 0x15bbffe0}, id =
LRU_ENTRY_L2, size = 148}, cleanup = {
q = {next = 0x5b5c1490, prev = 0x88016f0}, id = LRU_ENTRY_CLEANUP, size =
720}, mtx = {__data = {__lock = 0,
__count = 0, __owner = 0, __nusers = 0, __kind = 0, __spins = 0, __list
= {__prev = 0x0, __next = 0x0}},
__size = '\000' <repeats 39 times>, __align = 0}, iter = {active = false,
glist = 0x0, glistn = 0x0},
__pad0 = '\000' <repeats 63 times>}, {L1 = {q = {next = 0x67772f0, prev =
0x964eb1b0}, id = LRU_ENTRY_L1,
size = 140729}, L2 = {q = {next = 0x30d68c80, prev = 0x6770580}, id =
LRU_ENTRY_L2, size = 147}, cleanup = {
q = {next = 0xc4710dd0, prev = 0x874dc10}, id = LRU_ENTRY_CLEANUP, size =
791}, mtx = {__data = {__lock = 0,
__count = 0, __owner = 0, __nusers = 0, __kind = 0, __spins = 0, __list
= {__prev = 0x0, __next = 0x0}},
__size = '\000' <repeats 39 times>, __align = 0}, iter = {active = false,
glist = 0x0, glistn = 0x0},
__pad0 = '\000' <repeats 63 times>}, {L1 = {q = {next = 0x5d1c210, prev =
0x302682d0}, id = LRU_ENTRY_L1,
size = 140670}, L2 = {q = {next = 0x3026f040, prev = 0x5d154a0}, id =
LRU_ENTRY_L2, size = 147}, cleanup = {
q = {next = 0xcb5487f0, prev = 0xc679490}, id = LRU_ENTRY_CLEANUP, size =
794}, mtx = {__data = {__lock = 0,
__count = 0, __owner = 0, __nusers = 0, __kind = 0, __spins = 0, __list
= {__prev = 0x0, __next = 0x0}},
__size = '\000' <repeats 39 times>, __align = 0}, iter = {active = false,
glist = 0x0, glistn = 0x0},
__pad0 = '\000' <repeats 63 times>}, {L1 = {q = {next = 0xe2791600, prev =
0x96252be0}, id = LRU_ENTRY_L1,
size = 148189}, L2 = {q = {next = 0xec06a870, prev = 0xe2798370}, id =
LRU_ENTRY_L2, size = 145}, cleanup = {
q = {next = 0xa4dec380, prev = 0xb29dc50}, id = LRU_ENTRY_CLEANUP, size =
746}, mtx = {__data = {__lock = 0,
__count = 0, __owner = 0, __nusers = 0, __kind = 0, __spins = 0, __list
= {__prev = 0x0, __next = 0x0}},
__size = '\000' <repeats 39 times>, __align = 0}, iter = {active = false,
glist = 0x0, glistn = 0x0},
__pad0 = '\000' <repeats 63 times>}, {L1 = {q = {next = 0xe5dff590, prev =
0x23d43b90}, id = LRU_ENTRY_L1,
size = 147297}, L2 = {q = {next = 0x23de7e10, prev = 0xe5e06300}, id =
LRU_ENTRY_L2, size = 141}, cleanup = {
q = {next = 0xa6b8d390, prev = 0x10229cb0}, id = LRU_ENTRY_CLEANUP, size
= 744}, mtx = {__data = {__lock = 0,
__count = 0, __owner = 0, __nusers = 0, __kind = 0, __spins = 0, __list
= {__prev = 0x0, __next = 0x0}},
__size = '\000' <repeats 39 times>, __align = 0}, iter = {active = false,
glist = 0x0, glistn = 0x0},
__pad0 = '\000' <repeats 63 times>}, {L1 = {q = {next = 0x1cc790a0, prev =
0x4ba049f0}, id = LRU_ENTRY_L1,
size = 143750}, L2 = {q = {next = 0xf39d9d0, prev = 0x1cc72330}, id =
LRU_ENTRY_L2, size = 147}, cleanup = {
---Type <return> to continue, or q <return> to quit---
q = {next = 0xd37ff620, prev = 0x13b47010}, id = LRU_ENTRY_CLEANUP, size
= 780}, mtx = {__data = {__lock = 0,
__count = 0, __owner = 0, __nusers = 0, __kind = 0, __spins = 0, __list
= {__prev = 0x0, __next = 0x0}},
__size = '\000' <repeats 39 times>, __align = 0}, iter = {active = false,
glist = 0x0, glistn = 0x0},
__pad0 = '\000' <repeats 63 times>}, {L1 = {q = {next = 0xeaf4fd00, prev =
0x25ab91d0}, id = LRU_ENTRY_L1,
size = 146505}, L2 = {q = {next = 0x25aa4980, prev = 0xeaf56a70}, id =
LRU_ENTRY_L2, size = 150}, cleanup = {
q = {next = 0x23c6c110, prev = 0x11e02670}, id = LRU_ENTRY_CLEANUP, size
= 788}, mtx = {__data = {__lock = 0,
__count = 0, __owner = 0, __nusers = 0, __kind = 0, __spins = 0, __list
= {__prev = 0x0, __next = 0x0}},
__size = '\000' <repeats 39 times>, __align = 0}, iter = {active = false,
glist = 0x0, glistn = 0x0},
__pad0 = '\000' <repeats 63 times>}, {L1 = {q = {next = 0xe3dea010, prev =
0x2831da50}, id = LRU_ENTRY_L1,
size = 148717}, L2 = {q = {next = 0x283247c0, prev = 0xe3df0d80}, id =
LRU_ENTRY_L2, size = 149}, cleanup = {
q = {next = 0xb84809e0, prev = 0x87cee00}, id = LRU_ENTRY_CLEANUP, size =
850}, mtx = {__data = {__lock = 0,
__count = 0, __owner = 0, __nusers = 0, __kind = 0, __spins = 0, __list
= {__prev = 0x0, __next = 0x0}},
__size = '\000' <repeats 39 times>, __align = 0}, iter = {active = false,
glist = 0x0, glistn = 0x0},
__pad0 = '\000' <repeats 63 times>}, {L1 = {q = {next = 0xce3ced10, prev =
0xd5f2db20}, id = LRU_ENTRY_L1,
size = 153934}, L2 = {q = {next = 0xd5f12560, prev = 0xce3d5a80}, id =
LRU_ENTRY_L2, size = 149}, cleanup = {
q = {next = 0xca840ac0, prev = 0xda6b170}, id = LRU_ENTRY_CLEANUP, size =
779}, mtx = {__data = {__lock = 0,
__count = 0, __owner = 0, __nusers = 0, __kind = 0, __spins = 0, __list
= {__prev = 0x0, __next = 0x0}},
__size = '\000' <repeats 39 times>, __align = 0}, iter = {active = false,
glist = 0x0, glistn = 0x0},
__pad0 = '\000' <repeats 63 times>}, {L1 = {q = {next = 0x2d292f90, prev =
0x17851300}, id = LRU_ENTRY_L1,
size = 138008}, L2 = {q = {next = 0x1786c8c0, prev = 0x2d2854b0}, id =
LRU_ENTRY_L2, size = 147}, cleanup = {
q = {next = 0xc7c1e980, prev = 0xb2a3820}, id = LRU_ENTRY_CLEANUP, size =
761}, mtx = {__data = {__lock = 0,
__count = 0, __owner = 0, __nusers = 0, __kind = 0, __spins = 0, __list
= {__prev = 0x0, __next = 0x0}},
__size = '\000' <repeats 39 times>, __align = 0}, iter = {active = false,
glist = 0x0, glistn = 0x0},
__pad0 = '\000' <repeats 63 times>}}
(gdb)
(gdb) print *(struct mdcache_lru__ *)0xe4788210
$3 = {q = {next = 0xe47814a0, prev = 0x7e0620 <LRU>}, qid = LRU_ENTRY_L1,
refcnt = 1, flags = 0, lane = 0, cf = 0}
(gdb) print *(struct mdcache_lru__ *)0xe47814a0
$5 = {q = {next = 0xe477a730, prev = 0xe4788210}, qid = LRU_ENTRY_L1, refcnt =
1, flags = 0, lane = 0, cf = 0}
(gdb) print *(struct mdcache_lru__ *)0xe477a730
$6 = {q = {next = 0xe47739c0, prev = 0xe47814a0}, qid = LRU_ENTRY_L1, refcnt =
1, flags = 0, lane = 0, cf = 0}
(gdb) print *(struct mdcache_lru__ *)0xe47739c0
$7 = {q = {next = 0xe476cc50, prev = 0xe477a730}, qid = LRU_ENTRY_L1, refcnt =
1, flags = 0, lane = 0, cf = 0}
(gdb) print *(struct mdcache_lru__ *)0xb84809e0
$8 = {q = {next = 0xaa559720, prev = 0x7e12a0 <LRU+3200>}, qid =
LRU_ENTRY_CLEANUP, refcnt = 3, flags = 3,
lane = 14, cf = 0}
(gdb) print *(struct mdcache_lru__ *)0xaa559720
$9 = {q = {next = 0xd9754930, prev = 0xb84809e0}, qid = LRU_ENTRY_CLEANUP,
refcnt = 3, flags = 3, lane = 14, cf = 0}
(gdb) print *(struct mdcache_lru__ *)0xd9754930
$10 = {q = {next = 0xa94e7f40, prev = 0xaa559720}, qid = LRU_ENTRY_CLEANUP,
refcnt = 2, flags = 3, lane = 14, cf = 0}
(gdb) print *(struct mdcache_lru__ *)0xa94e7f40
$11 = {q = {next = 0x185aa7d0, prev = 0xd9754930}, qid = LRU_ENTRY_CLEANUP,
refcnt = 3, flags = 3, lane = 14, cf = 0}
(gdb) print *(struct mdcache_lru__ *)0x185aa7d0
$12 = {q = {next = 0xcb912a30, prev = 0xa94e7f40}, qid = LRU_ENTRY_CLEANUP,
refcnt = 1, flags = 3, lane = 14, cf = 0}
(gdb) print *(struct mdcache_lru__ *)0xcb912a30
$13 = {q = {next = 0xc7727370, prev = 0x185aa7d0}, qid = LRU_ENTRY_CLEANUP,
refcnt = 2, flags = 3, lane = 14, cf = 0}
(gdb)
$14 = {q = {next = 0xc7727370, prev = 0x185aa7d0}, qid = LRU_ENTRY_CLEANUP,
refcnt = 2, flags = 3, lane = 14, cf = 0}
(gdb) print *(struct mdcache_lru__ *)0xf39d9d0
$15 = {q = {next = 0xf3a4740, prev = 0x7e10c0 <LRU+2720>}, qid = LRU_ENTRY_L2,
refcnt = 1, flags = 0, lane = 12,
cf = 0}
(gdb) print *(struct mdcache_lru__ *)0xf3a4740
$16 = {q = {next = 0xf3ab4b0, prev = 0xf39d9d0}, qid = LRU_ENTRY_L2, refcnt =
1, flags = 0, lane = 12, cf = 0}
(gdb) print *(struct mdcache_lru__ *)0xf3ab4b0
$17 = {q = {next = 0xf3b2220, prev = 0xf3a4740}, qid = LRU_ENTRY_L2, refcnt =
1, flags = 0, lane = 12, cf = 0}
_______________________________________________
Nfs-ganesha-devel mailing list
Nfs-ganesha-devel@lists.sourceforge.net
https://lists.sourceforge.net/lists/listinfo/nfs-ganesha-devel
