I haven't seen anything link this. The CLIENT is refcounted, but the
refcounting is unused, and nothing has changed in this respect on HEAD.
So the client lifetime for the CLIENT in NSM appears to just be between
nsm_connect() and nsm_disconnect(). nsm_unmonitor() calls
nsm_connect(), so the issue is that the CLIENT somehow got freed
*without* a call to nsm_disconnect() (which NULLs out nsm_clnt). I'm
not sure how this could happen, but maybe there's some strange error
case in ntirpc somewhere?
Daniel
On 03/13/2017 08:26 AM, Malahal Naineni wrote:
> We got the following stack trace. This is with ganesha 2.3
> (corresponding libntirpc) on a customer system. Did anyone see this in
> the past. The __kind mutex field is -1, implying that the code is
> using freed memory. The pthread_mutex_lock() failed and our
> mutex_lock() macro called abort().
>
> (gdb) bt
> #0 0x00003fff96d20780 in .raise () from /lib64/libc.so.6
> #1 0x00003fff96d22788 in .abort () from /lib64/libc.so.6
> #2 0x00003fff96f3e73c in alloc_rpc_call_ctx (clnt=0x3ffc80020ed0,
> proc=3, xdr_args=@0x102846e0: 0x101049cc <xdr_mon_id>,
> args_ptr=0x3fff0f16b360, xdr_results=@0x102846c0: 0x10104824
> <xdr_sm_stat>, results_ptr=0x3fff0f16b358, timeout=...)
> at
> /usr/src/debug/nfs-ganesha-2.3.2-ibm32-0.1.1-Source/libntirpc/src/rpc_ctx.c:68
> #3 0x00003fff96f313d0 in clnt_vc_call (clnt=0x3ffc80020ed0,
> auth=0x3fff96f95be0 <auth_none_priv>, proc=3,
> xdr_args=@0x102846e0: 0x101049cc <xdr_mon_id>,
> args_ptr=0x3fff0f16b360, xdr_results=@0x102846c0: 0x10104824
> <xdr_sm_stat>,
> results_ptr=0x3fff0f16b358, timeout=...) at
> /usr/src/debug/nfs-ganesha-2.3.2-ibm32-0.1.1-Source/libntirpc/src/clnt_vc.c:390
> #4 0x00000000100e0624 in nsm_unmonitor (host=0x3ffc8001fd70) at
> /usr/src/debug/nfs-ganesha-2.3.2-ibm32-0.1.1-Source/Protocols/NLM/nsm.c:208
> #5 0x0000000010146178 in dec_nsm_client_ref (client=0x3ffc8001fd70)
> at /usr/src/debug/nfs-ganesha-2.3.2-ibm32-0.1.1-Source/SAL/nlm_owner.c:851
> #6 0x0000000010146fb8 in free_nlm_client (client=0x3ffc80020f40) at
> /usr/src/debug/nfs-ganesha-2.3.2-ibm32-0.1.1-Source/SAL/nlm_owner.c:1045
> #7 0x0000000010147440 in dec_nlm_client_ref (client=0x3ffc80020f40)
> at /usr/src/debug/nfs-ganesha-2.3.2-ibm32-0.1.1-Source/SAL/nlm_owner.c:1130
> #8 0x0000000010148028 in free_nlm_owner (owner=0x3ffc8001e7e0) at
> /usr/src/debug/nfs-ganesha-2.3.2-ibm32-0.1.1-Source/SAL/nlm_owner.c:1375
> #9 0x000000001011b328 in free_state_owner (owner=0x3ffc8001e7e0) at
> /usr/src/debug/nfs-ganesha-2.3.2-ibm32-0.1.1-Source/SAL/state_misc.c:911
> #10 0x000000001011ba8c in dec_state_owner_ref (owner=0x3ffc8001e7e0)
> at /usr/src/debug/nfs-ganesha-2.3.2-ibm32-0.1.1-Source/SAL/state_misc.c:1058
> #11 0x00000000101123f4 in state_export_unlock_all () at
> /usr/src/debug/nfs-ganesha-2.3.2-ibm32-0.1.1-Source/SAL/state_lock.c:3651
> #12 0x000000001011cffc in state_release_export (export=0x1003371f058)
> at
> /usr/src/debug/nfs-ganesha-2.3.2-ibm32-0.1.1-Source/SAL/state_misc.c:1454
> #13 0x00000000101a7cf4 in clean_up_export (export=0x1003371f058) at
> /usr/src/debug/nfs-ganesha-2.3.2-ibm32-0.1.1-Source/support/exports.c:1779
> #14 0x00000000101a7dc8 in unexport (export=0x1003371f058) at
> /usr/src/debug/nfs-ganesha-2.3.2-ibm32-0.1.1-Source/support/exports.c:1792
> #15 0x00000000101c2cf8 in remove_all_exports () at
> /usr/src/debug/nfs-ganesha-2.3.2-ibm32-0.1.1-Source/support/export_mgr.c:747
> #16 0x00000000100794e0 in do_shutdown () at
> /usr/src/debug/nfs-ganesha-2.3.2-ibm32-0.1.1-Source/MainNFSD/nfs_admin_thread.c:485
> #17 0x0000000010079ae8 in admin_thread (UnusedArg=0x0) at
> /usr/src/debug/nfs-ganesha-2.3.2-ibm32-0.1.1-Source/MainNFSD/nfs_admin_thread.c:523
> #18 0x00003fff96fdc2bc in .start_thread () from /lib64/libpthread.so.0
> #19 0x00003fff96dfb304 in .__clone () from /lib64/libc.so.6
> (gdb) frame 2
> #2 0x00003fff96f3e73c in alloc_rpc_call_ctx (clnt=0x3ffc80020ed0,
> proc=3, xdr_args=@0x102846e0: 0x101049cc <xdr_mon_id>,
> args_ptr=0x3fff0f16b360, xdr_results=@0x102846c0: 0x10104824
> <xdr_sm_stat>, results_ptr=0x3fff0f16b358, timeout=...)
> at
> /usr/src/debug/nfs-ganesha-2.3.2-ibm32-0.1.1-Source/libntirpc/src/rpc_ctx.c:68
> 68 REC_LOCK(rec);
> (gdb) p *rec
> $2 = {fd_k = 16380, locktrace = {mtx = {__data = {__lock = 16380,
> __count = 2147665216, __owner = 0, __nusers = 0, __kind = -1, __spins
> = 0,
> __list = {__prev = 0x0, __next = 0x0}}, __size =
> "\000\000?�\200\002�@\000\000\000\000\000\000\000\000����", '\000'
> <repeats 19 times>,
> __align = 70353711973696}, func = 0x3fff96f77528 <__func__.8141>
> "free_rpc_call_ctx", line = 245}, node_k = {left = 0x0, right = 0x0,
> parent = 0x0, red = 0, gen = 0}, refcnt = 1, send = {lock = {we =
> {mtx = {__data = {__lock = 0, __count = 0, __owner = 0, __nusers = 0,
> __kind = -1, __spins = 0, __list = {__prev = 0x0, __next = 0x0}},
> __size = '\000' <repeats 16 times>, "����", '\000' <repeats
> 19 times>, __align = 0}, cv = {__data = {__lock = 1, __futex = 0,
> __total_seq = 18446744073709551615, __wakeup_seq = 0,
> __woken_seq = 0, __mutex = 0x0, __nwaiters = 0, __broadcast_seq = 0},
> __size = "\000\000\000\001\000\000\000\000��������", '\000'
> <repeats 31 times>, __align = 4294967296}}, lock_flag_value = 0,
> locktrace = {func = 0x0, line = 0}}}, recv = {lock = {we = {mtx
> = {__data = {__lock = 0, __count = 0, __owner = 0, __nusers = 0,
> __kind = -1, __spins = 0, __list = {__prev = 0x0, __next = 0x0}},
> __size = '\000' <repeats 16 times>, "����", '\000' <repeats
> 19 times>, __align = 0}, cv = {__data = {__lock = 1, __futex = 0,
> __total_seq = 18446744073709551615, __wakeup_seq = 0,
> __woken_seq = 0, __mutex = 0x0, __nwaiters = 0, __broadcast_seq = 0},
> __size = "\000\000\000\001\000\000\000\000��������", '\000'
> <repeats 31 times>, __align = 4294967296}}, lock_flag_value = 0,
> locktrace = {func = 0x0, line = 16380}}}, hdl = {xd =
> 0x3ffc8002e7f0, xprt = 0x0}}
> (gdb)
>
> ------------------------------------------------------------------------------
> Announcing the Oxford Dictionaries API! The API offers world-renowned
> dictionary content that is easy and intuitive to access. Sign up for an
> account today to start using our lexical data to power your apps and
> projects. Get started today and enter our developer competition.
> http://sdm.link/oxford
> _______________________________________________
> Nfs-ganesha-devel mailing list
> Nfs-ganesha-devel@lists.sourceforge.net
> https://lists.sourceforge.net/lists/listinfo/nfs-ganesha-devel
>
------------------------------------------------------------------------------
Announcing the Oxford Dictionaries API! The API offers world-renowned
dictionary content that is easy and intuitive to access. Sign up for an
account today to start using our lexical data to power your apps and
projects. Get started today and enter our developer competition.
http://sdm.link/oxford
_______________________________________________
Nfs-ganesha-devel mailing list
Nfs-ganesha-devel@lists.sourceforge.net
https://lists.sourceforge.net/lists/listinfo/nfs-ganesha-devel
