[ https://issues.apache.org/jira/browse/HBASE-18565?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16122693#comment-16122693 ]
Enis Soztutar commented on HBASE-18565: --------------------------------------- Attaching to the process using gdb reveals these stack traces: A lot of threads are in trying to acquire the mutex for the rpc-connection like this: {code} Thread 10 (Thread 0x7febf1ffb700 (LWP 6210)): #0 0x00007fec0eea81bd in __lll_lock_wait () from /lib64/libpthread.so.0 #1 0x00007fec0eea3d1d in _L_lock_840 () from /lib64/libpthread.so.0 #2 0x00007fec0eea3c3a in pthread_mutex_lock () from /lib64/libpthread.so.0 #3 0x00007fec135fe083 in __gthread_mutex_lock (__mutex=0x7febec001a58) at /opt/rh/devtoolset-6/root/usr/include/c++/6.2.1/x86_64-redhat-linux/bits/gthr-default.h:748 #4 0x00007fec135fe0d3 in __gthread_recursive_mutex_lock (__mutex=0x7febec001a58) at /opt/rh/devtoolset-6/root/usr/include/c++/6.2.1/x86_64-redhat-linux/bits/gthr-default.h:810 #5 0x00007fec135feb56 in std::recursive_mutex::lock (this=0x7febec001a58) at /opt/rh/devtoolset-6/root/usr/include/c++/6.2.1/mutex:105 #6 0x00007fec136005c6 in std::lock_guard<std::recursive_mutex>::lock_guard (this=0x7febf1ff9800, __m=...) at /opt/rh/devtoolset-6/root/usr/include/c++/6.2.1/bits/std_mutex.h:162 #7 0x00007fec135ff9e0 in hbase::RpcConnection::SendRequest (this=0x7febec001a40, req=std::unique_ptr<hbase::Request> containing 0x7feb9c01e300) at ./connection/rpc-connection.h:43 #8 0x00007fec135bfca0 in hbase::RpcClient::SendRequest (this=0xa6f890, remote_id=std::shared_ptr (count 3, weak 0) 0x7feb9c025f40, req=std::unique_ptr<hbase::Request> containing 0x0) at connection/rpc-client.cc:85 #9 0x00007fec135bf85c in hbase::RpcClient::AsyncCall (this=0xa6f890, host="localhost", port=42985, req=std::unique_ptr<hbase::Request> containing 0x0, ticket=std::shared_ptr (count 3, weak 0) 0x7feb9c01e1c0, service_name="ClientService") at connection/rpc-client.cc:71 Python Exception <class 'gdb.error'> There is no member or method named _M_bbegin.: #10 0x00007fec1370e1b1 in hbase::AsyncBatchRpcRetryingCaller::GetMultiResponse (this=0x7feb9c0036c0, actions_by_server=std::unordered_map with 1 elements) at core/async-batch-rpc-retrying-caller.cc:313 Python Exception <class 'gdb.error'> There is no member or method named _M_bbegin.: #11 0x00007fec1370f048 in hbase::AsyncBatchRpcRetryingCaller::Send (this=0x7feb9c0036c0, actions_by_server=std::unordered_map with 1 elements, tries=1) at core/async-batch-rpc-retrying-caller.cc:343 #12 0x00007fec1370d819 in hbase::AsyncBatchRpcRetryingCaller::<lambda(std::vector<folly::Try<std::shared_ptr<hbase::RegionLocation> >, std::allocator<folly::Try<std::shared_ptr<hbase::RegionLocation> > > >&)>::operator()(std::vector<folly::Try<std::shared_ptr<hbase::RegionLocation> >, std::allocator<folly::Try<std::shared_ptr<hbase::RegionLocation> > > > &) const (__closure=0x7feb9c001790, loc=std::vector of length 100, capacity 100 = {...}) at core/async-batch-rpc-retrying-caller.cc:283 #13 0x00007fec137112d0 in folly::detail::CoreCallbackState<folly::Unit, hbase::AsyncBatchRpcRetryingCaller::GroupAndSend(const std::vector<std::shared_ptr<hbase::Action> >&, int32_t)::<lambda(std::vector<folly::Try<std::shared_ptr<hbase::RegionLocation> >, std::allocator<folly::Try<std::shared_ptr<hbase::RegionLocation> > > >&)> >::invoke<std::vector<folly::Try<std::shared_ptr<hbase::RegionLocation> >, std::allocator<folly::Try<std::shared_ptr<hbase::RegionLocation> > > >&>(std::vector<folly::Try<std::shared_ptr<hbase::RegionLocation> >, std::allocator<folly::Try<std::shared_ptr<hbase::RegionLocation> > > > &) (this=0x7feb9c001790, args#0=std::vector of length 100, capacity 100 = {...}) at /usr/local/include/folly/futures/Future-inl.h:85 #14 0x00007fec13711309 in folly::Future<std::vector<folly::Try<std::shared_ptr<hbase::RegionLocation> >, std::allocator<folly::Try<std::shared_ptr<hbase::RegionLocation> > > > >::<lambda(folly::Try<std::vector<folly::Try<std::shared_ptr<hbase::RegionLocation> >, std::allocator<folly::Try<std::shared_ptr<hbase::RegionLocation> > > > >&&)>::<lambda()>::operator()(void) const (__closure=0x7febf1ffa1a0) at /usr/local/include/folly/futures/Future-inl.h:243 #15 0x00007fec13712520 in folly::makeTryWith<folly::Future<T>::thenImplementation(F&&, folly::detail::argResult<isTry, F, Args ...>)::<lambda(folly::Try<T>&&)> mutable [with F = hbase::AsyncBatchRpcRetryingCaller::GroupAndSend(const std::vector<std::shared_ptr<hbase::Action> >&, int32_t)::<lambda(std::vector<folly::Try<std::shared_ptr<hbase::RegionLocation> > >&)>; R = folly::detail::callableResult<std::vector<folly::Try<std::shared_ptr<hbase::RegionLocation> > >, hbase::AsyncBatchRpcRetryingCaller::GroupAndSend(const std::vector<std::shared_ptr<hbase::Action> >&, int32_t)::<lambda(std::vector<folly::Try<std::shared_ptr<hbase::RegionLocation> > >&)> >; bool isTry = false; Args = {std::vector<folly::Try<std::shared_ptr<hbase::RegionLocation> >, std::allocator<folly::Try<std::shared_ptr<hbase::RegionLocation> > > >&}; T = std::vector<folly::Try<std::shared_ptr<hbase::RegionLocation> > >]::<lambda()> >(<unknown type in /usr/local/lib/libHbaseClient_d.so, CU 0xb0d6f8, DIE 0xbd8453>) ( f=<unknown type in /usr/local/lib/libHbaseClient_d.so, CU 0xb0d6f8, DIE 0xbd8453>) at /usr/local/include/folly/Try-inl.h:165 #16 0x00007fec1371137b in folly::Future<std::vector<folly::Try<std::shared_ptr<hbase::RegionLocation> >, std::allocator<folly::Try<std::shared_ptr<hbase::RegionLocation> > > > >::<lambda(folly::Try<std::vector<folly::Try<std::shared_ptr<hbase::RegionLocation> >, std::allocator<folly::Try<std::shared_ptr<hbase::RegionLocation> > > > >&&)>::operator()(<unknown type in /usr/local/lib/libHbaseClient_d.so, CU 0xb0d6f8, DIE 0xba6a18>) (__closure=0x7feb9c001790, t=<unknown type in /usr/local/lib/libHbaseClient_d.so, CU 0xb0d6f8, DIE 0xba6a18>) at /usr/local/include/folly/futures/Future-inl.h:242 #17 0x00007fec13713f57 in folly::detail::function::FunctionTraits<void(folly::Try<std::vector<folly::Try<std::shared_ptr<hbase::RegionLocation> >, std::allocator<folly::Try<std::shared_ptr<hbase::RegionLocation> > > > >&&)>::callBig<folly::Future<T>::thenImplementation(F&&, folly::detail::argResult<isTry, F, Args ...>) [with F = hbase::AsyncBatchRpcRetryingCaller::GroupAndSend(const std::vector<std::shared_ptr<hbase::Action> >&, int32_t)::<lambda(std::vector<folly::Try<std::shared_ptr<hbase::RegionLocation> > >&)>; R = folly::detail::callableResult<std::vector<folly::Try<std::shared_ptr<hbase::RegionLocation> > >, hbase::AsyncBatchRpcRetryingCaller::GroupAndSend(const std::vector<std::shared_ptr<hbase::Action> >&, int32_t)::<lambda(std::vector<folly::Try<std::shared_ptr<hbase::RegionLocation> > >&)> >; bool isTry = false; Args = {std::vector<folly::Try<std::shared_ptr<hbase::RegionLocation> >, std::allocator<folly::Try<std::shared_ptr<hbase::RegionLocation> > > >&}; T = std::vector<folly::Try<std::shared_ptr<hbase::RegionLocation> > >]::<lambda(folly::Try<std::vector<folly::Try<std::shared_ptr<hbase::RegionLocation> > > >&&)> >(folly::detail::function::Data &, <unknown type in /usr/local/lib/libHbaseClient_d.so, CU 0xb0d6f8, DIE 0xba6a18>) (p=..., args#0=<unknown type in /usr/local/lib/libHbaseClient_d.so, CU 0xb0d6f8, DIE 0xba6a18>) at /usr/local/include/folly/Function.h:305 #18 0x00007fec1372a0bc in folly::detail::function::FunctionTraits<void (folly::Try<std::vector<folly::Try<std::shared_ptr<hbase::RegionLocation> >, std::allocator<folly::Try<std::shared_ptr<hbase::RegionLocation> > > > >&&)>::operator()(folly::Try<std::vector<folly::Try<std::shared_ptr<hbase::RegionLocation> >, std::allocator<folly::Try<std::shared_ptr<hbase::RegionLocation> > > > >&&) ( this=0x7feb9c019e90, args#0=<unknown type in /usr/local/lib/libHbaseClient_d.so, CU 0xb0d6f8, DIE 0xba6a18>) at /usr/local/include/folly/Function.h:314 #19 0x00007fec1372759d in folly::detail::Core<std::vector<folly::Try<std::shared_ptr<hbase::RegionLocation> >, std::allocator<folly::Try<std::shared_ptr<hbase::RegionLocation> > > > >::doCallback ( this=0x7feb9c019e90) at /usr/local/include/folly/futures/detail/Core.h:388 #20 0x00007fec13724051 in folly::detail::Core<std::vector<folly::Try<std::shared_ptr<hbase::RegionLocation> >, std::allocator<folly::Try<std::shared_ptr<hbase::RegionLocation> > > > >::maybeCallback()::{lambda()#2}::operator()() const (__closure=0x7febf1ffa4c0) at /usr/local/include/folly/futures/detail/Core.h:317 #21 0x00007fec1372781e in folly::detail::FSM<folly::detail::State>::updateState<folly::detail::Core<std::vector<folly::Try<std::shared_ptr<hbase::RegionLocation> >, std::allocator<folly::Try<std::shared_ptr<hbase::RegionLocation> > > > >::maybeCallback()::{lambda()#1}, folly::detail::Core<std::vector<folly::Try<std::shared_ptr<hbase::RegionLocation> >, std::allocator<folly::Try<std::shared_ptr<hbase::RegionLocation> > > > >::maybeCallback()::{lambda()#2}>(folly::detail::State, folly::detail::State, folly::detail::Core<std::vector<folly::Try<std::shared_ptr<hbase::RegionLocation> >, std::allocator<folly::Try<std::shared_ptr<hbase::RegionLocation> > > > >::maybeCallback()::{lambda()#1} const&, folly::detail::Core<std::vector<folly::Try<std::shared_ptr<hbase::RegionLocation> >, std::allocator<folly::Try<std::shared_ptr<hbase::RegionLocation> > > > >::maybeCallback()::{lambda()#2} const&) (this=0x7feb9c019ef8, A=folly::detail::Armed, B=folly::detail::Done, protectedAction=..., unprotectedAction=...) at /usr/local/include/folly/futures/detail/FSM.h:100 #22 0x00007fec137240cc in folly::detail::Core<std::vector<folly::Try<std::shared_ptr<hbase::RegionLocation> >, std::allocator<folly::Try<std::shared_ptr<hbase::RegionLocation> > > > >::maybeCallback ( this=0x7feb9c019e90) at /usr/local/include/folly/futures/detail/Core.h:317 #23 0x00007fec1371319b in folly::detail::Core<std::vector<folly::Try<std::shared_ptr<hbase::RegionLocation> >, std::allocator<folly::Try<std::shared_ptr<hbase::RegionLocation> > > > >::setCallback<folly::Future<T>::thenImplementation(F&&, folly::detail::argResult<isTry, F, Args ...>) [with F = hbase::AsyncBatchRpcRetryingCaller::GroupAndSend(const std::vector<std::shared_ptr<hbase::Action> >&, int32_t)::<lambda(std::vector<folly::Try<std::shared_ptr<hbase::RegionLocation> > >&)>; R = folly::detail::callableResult<std::vector<folly::Try<std::shared_ptr<hbase::RegionLocation> > >, hbase::AsyncBatchRpcRetryingCaller::GroupAndSend(const std::vector<std::shared_ptr<hbase::Action> >&, int32_t)::<lambda(std::vector<folly::Try<std::shared_ptr<hbase::RegionLocation> > >&)> >; bool isTry = false; Args = {std::vector<folly::Try<std::shared_ptr<hbase::RegionLocation> >, std::allocator<folly::Try<std::shared_ptr<hbase::RegionLocation> > > >&}; T = std::vector<folly::Try<std::shared_ptr<hbase::RegionLocation> > >]::<lambda(folly::Try<std::vector<folly::Try<std::shared_ptr<hbase::RegionLocation> > > >&&)> >(<unknown type in /usr/local/lib/libHbaseClient_d.so, CU 0xb0d6f8, DIE 0xbd000a>) ( this=0x7feb9c019e90, func=<unknown type in /usr/local/lib/libHbaseClient_d.so, CU 0xb0d6f8, DIE 0xbd000a>) at /usr/local/include/folly/futures/detail/Core.h:158 #24 0x00007fec137126d9 in folly::Future<std::vector<folly::Try<std::shared_ptr<hbase::RegionLocation> >, std::allocator<folly::Try<std::shared_ptr<hbase::RegionLocation> > > > >::setCallback_<folly::Future<T>::thenImplementation(F&&, folly::detail::argResult<isTry, F, Args ...>) [with F = hbase::AsyncBatchRpcRetryingCaller::GroupAndSend(const std::vector<std::shared_ptr<hbase::Action> >&, int32_t)::<lambda(std::vector<folly::Try<std::shared_ptr<hbase::RegionLocation> > >&)>; R = folly::detail::callableResult<std::vector<folly::Try<std::shared_ptr<hbase::RegionLocation> > >, hbase::AsyncBatchRpcRetryingCaller::GroupAndSend(const std::vector<std::shared_ptr<hbase::Action> >&, int32_t)::<lambda(std::vector<folly::Try<std::shared_ptr<hbase::RegionLocation> > >&)> >; bool isTry = false; Args = {std::vector<folly::Try<std::shared_ptr<hbase::RegionLocation> >, std::allocator<folly::Try<std::shared_ptr<hbase::RegionLocation> > > >&}; T = std::vector<folly::Try<std::shared_ptr<hbase::RegionLocation> > >]::<lambda(folly::Try<std::vector<folly::Try<std::shared_ptr<hbase::RegionLocation> > > >&&)> >(<unknown type in /usr/local/lib/libHbaseClient_d.so, CU 0xb0d6f8, DIE 0xbd839c>) (this=0x7febf1ffa700, func=<unknown type in /usr/local/lib/libHbaseClient_d.so, CU 0xb0d6f8, DIE 0xbd839c>) at /usr/local/include/folly/futures/Future-inl.h:172 #25 0x00007fec13711509 in folly::Future<std::vector<folly::Try<std::shared_ptr<hbase::RegionLocation> >, std::allocator<folly::Try<std::shared_ptr<hbase::RegionLocation> > > > >::thenImplementation<hbase::AsyncBatchRpcRetryingCaller::GroupAndSend(const std::vector<std::shared_ptr<hbase::Action> >&, int32_t)::<lambda(std::vector<folly::Try<std::shared_ptr<hbase::RegionLocation> > >&)>, folly::detail::callableResult<std::vector<folly::Try<std::shared_ptr<hbase::RegionLocation> > >, hbase::AsyncBatchRpcRetryingCaller::GroupAndSend(const std::vector<std::shared_ptr<hbase::Action> >&, int32_t)::<lambda(sred_ptr<hbase::Action> >&, int32_t)::<lambda(std::vector<folly::Try<std::shared_ptr<hbase::RegionLocation> >, std::allocator<folly::Try<std::shared_ptr<hbase::RegionLocation> > > >&)>, std::vector<folly::Try<std::shared_ptr<hbase::RegionLocation> >, std::allocator<folly::Try<std::shared_ptr<hbase::RegionLocation> > > >&>) (this=0x7febf1ffa700, func=<unknown type in /usr/local/lib/libHbaseClient_d.so, CU 0xb0d6f8, DIE 0xbe0d1b>) at /usr/local/include/folly/futures/Future-inl.h:236 #26 0x00007fec137109de in folly::Future<std::vector<folly::Try<std::shared_ptr<hbase::RegionLocation> >, std::allocator<folly::Try<std::shared_ptr<hbase::RegionLocation> > > > >::then<hbase::AsyncBatchRpcRetryingCaller::GroupAndSend(const std::vector<std::shared_ptr<hbase::Action> >&, int32_t)::<lambda(std::vector<folly::Try<std::shared_ptr<hbase::RegionLocation> > >&)> >(<unknown type in /usr/local/lib/libHbaseClient_d.so, CU 0xb0d6f8, DIE 0xbe840e>) (this=0x7febf1ffa700, func=<unknown type in /usr/local/lib/libHbaseClient_d.so, CU 0xb0d6f8, DIE 0xbe840e>) at /usr/local/include/folly/futures/Future.h:183 #27 0x00007fec1370df37 in hbase::AsyncBatchRpcRetryingCaller::GroupAndSend (this=0x7feb9c0036c0, actions=std::vector of length 100, capacity 128 = {...}, tries=1) at core/async-batch-rpc-retrying-caller.cc:242 #28 0x00007fec1370b328 in hbase::AsyncBatchRpcRetryingCaller::Call (this=0x7feb9c0036c0) at core/async-batch-rpc-retrying-caller.cc:70 #29 0x00007fec13732c33 in hbase::RawAsyncTable::Batch (this=0x7feb9c000980, gets=std::vector of length 100, capacity 128 = {...}) at core/raw-async-table.cc:218 #30 0x00007fec137328a1 in hbase::RawAsyncTable::Get (this=0x7feb9c000980, gets=std::vector of length 100, capacity 128 = {...}) at core/raw-async-table.cc:202 #31 0x00007fec137036a9 in hbase::Table::Get (this=0x7feb9c0008c0, gets=std::vector of length 100, capacity 128 = {...}) at core/table.cc:120 #32 0x000000000042e328 in DoGet (iteration=1, max_row=19999, rows=2000, table=std::unique_ptr<hbase::Table> containing 0x7feb9c0008c0, families=std::vector of length 1, capacity 1 = {...}, batch_num_rows=100) at core/load-client.cc:186 #33 0x000000000042faab in <lambda()>::operator()(void) const (__closure=0xaa34f8) at core/load-client.cc:375 #34 0x0000000000432428 in std::_Bind_simple<main(int, char**)::<lambda()>()>::_M_invoke<>(std::_Index_tuple<>) (this=0xaa34f8) at /opt/rh/devtoolset-6/root/usr/include/c++/6.2.1/functional:1400 #35 0x0000000000432359 in std::_Bind_simple<main(int, char**)::<lambda()>()>::operator()(void) (this=0xaa34f8) at /opt/rh/devtoolset-6/root/usr/include/c++/6.2.1/functional:1389 #36 0x00000000004322d8 in std::thread::_State_impl<std::_Bind_simple<main(int, char**)::<lambda()>()> >::_M_run(void) (this=0xaa34f0) at /opt/rh/devtoolset-6/root/usr/include/c++/6.2.1/thread:196 #37 0x00007fec139c3aff in execute_native_thread_routine () from /usr/local/lib/libHbaseClient_d.so #38 0x00007fec0eea1dc5 in start_thread () from /lib64/libpthread.so.0 #39 0x00007fec0ebd073d in clone () from /lib64/libc.so.6 {code} One thread has the lock, but waits for another lock: {code} Thread 11 (Thread 0x7febf17fa700 (LWP 6209)): #0 0x00007fec0eea56d5 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00007fec0f67f9ec in std::condition_variable::wait(std::unique_lock<std::mutex>&) () from /lib64/libstdc++.so.6 #2 0x00007fec123d056b in wait<folly::EventBase::runInEventBaseThreadAndWait(folly::EventBase::FuncRef)::<lambda()> > (__p=..., __lock=..., this=0x7febf17f7fd0) at /opt/rh/devtoolset-4/root/usr/include/c++/5.2.1/condition_variable:98 #3 folly::EventBase::runInEventBaseThreadAndWait(folly::FunctionRef<void ()>) (this=<optimized out>, fn=...) at io/async/EventBase.cpp:573 #4 0x00007fec135da453 in wangle::EventBaseHandler::write (this=0x7febd0001f20, ctx=0x7febd0001f50, buf=std::unique_ptr<folly::IOBuf> containing 0x7febb001b048) at /usr/local/include/wangle/channel/EventBaseHandler.h:25 #5 0x00007fec135f8eb4 in wangle::OutboundContextImpl<wangle::EventBaseHandler>::write (this=0x7febd0001f50, msg=std::unique_ptr<folly::IOBuf> containing 0x0) at /usr/local/include/wangle/channel/HandlerContext-inl.h:477 #6 0x00007fec1361e18b in wangle::ContextImpl<hbase::SaslHandler>::fireWrite (this=0x7febd00020c0, msg=std::unique_ptr<folly::IOBuf> containing 0x0) at /usr/local/include/wangle/channel/HandlerContext-inl.h:207 #7 0x00007fec13615850 in hbase::SaslHandler::write (this=0x7febd0002040, ctx=0x7febd00020c0, buf=std::unique_ptr<folly::IOBuf> containing 0x0) at connection/sasl-handler.cc:119 #8 0x00007fec1361e89e in wangle::ContextImpl<hbase::SaslHandler>::write (this=0x7febd00020c0, msg=std::unique_ptr<folly::IOBuf> containing 0x0) at /usr/local/include/wangle/channel/HandlerContext-inl.h:290 #9 0x00007fec1361d0d7 in wangle::ContextImpl<hbase::ClientHandler>::fireWrite (this=0x7febd000c2c0, msg=std::unique_ptr<folly::IOBuf> containing 0x0) at /usr/local/include/wangle/channel/HandlerContext-inl.h:207 #10 0x00007fec13605cc8 in hbase::ClientHandler::write (this=0x7febd000c240, ctx=0x7febd000c2c0, r=std::unique_ptr<hbase::Request> containing 0x7febb0012470) at connection/client-handler.cc:138 #11 0x00007fec1361d814 in wangle::ContextImpl<hbase::ClientHandler>::write (this=0x7febd000c2c0, msg=std::unique_ptr<hbase::Request> containing 0x0) at /usr/local/include/wangle/channel/HandlerContext-inl.h:290 #12 0x00007fec1362fb4d in wangle::ContextImpl<wangle::ClientDispatcherBase<wangle::Pipeline<folly::IOBufQueue&, std::unique_ptr<hbase::Request, std::default_delete<hbase::Request> > >, std::unique_ptr<hbase::Request, std::default_delete<hbase::Request> >, std::unique_ptr<hbase::Response, std::default_delete<hbase::Response> > > >::fireWrite (this=0x7febec0010f0, msg=std::unique_ptr<hbase::Request> containing 0x0) at /usr/local/include/wangle/channel/HandlerContext-inl.h:207 #13 0x00007fec13614a1d in wangle::HandlerAdapter<std::unique_ptr<hbase::Response, std::default_delete<hbase::Response> >, std::unique_ptr<hbase::Request, std::default_delete<hbase::Request> > >::write (this=0x7febec001e00, ctx=0x7febec0010f0, msg=std::unique_ptr<hbase::Request> containing 0x0) at /usr/local/include/wangle/channel/Handler.h:162 #14 0x00007fec1363028a in wangle::ContextImpl<wangle::ClientDispatcherBase<wangle::Pipeline<folly::IOBufQueue&, std::unique_ptr<hbase::Request, std::default_delete<hbase::Request> > >, std::unique_ptr<hbase::Request, std::default_delete<hbase::Request> >, std::unique_ptr<hbase::Response, std::default_delete<hbase::Response> > > >::write (this=0x7febec0010f0, msg=std::unique_ptr<hbase::Request> containing 0x0) at /usr/local/include/wangle/channel/HandlerContext-inl.h:290 #15 0x00007fec1360d403 in wangle::Pipeline<folly::IOBufQueue&, std::unique_ptr<hbase::Request, std::default_delete<hbase::Request> > >::write<std::unique_ptr<hbase::Request, std::default_delete<hbase::R#17 0x00007fec135ffb35 in hbase::RpcConnection::SendRequest (this=0x7febec001a40, req=std::unique_ptr<hbase::Request> containing 0x0) at ./connection/rpc-connection.h:48 #18 0x00007fec135bfca0 in hbase::RpcClient::SendRequest (this=0xa6f890, remote_id=std::shared_ptr (count 3, weak 0) 0x7febb001a050, req=std::unique_ptr<hbase::Request> containing 0x0) at connection/rpc-client.cc:85 #19 0x00007fec135bf85c in hbase::RpcClient::AsyncCall (this=0xa6f890, host="localhost", port=42985, req=std::unique_ptr<hbase::Request> containing 0x0, ticket=std::shared_ptr (count 3, weak 0) 0x7febb0012330, service_name="ClientService") at connection/rpc-client.cc:71 Python Exception <class 'gdb.error'> There is no member or method named _M_bbegin.: #20 0x00007fec1370e1b1 in hbase::AsyncBatchRpcRetryingCaller::GetMultiResponse (this=0x7febb0026820, actions_by_server=std::unordered_map with 1 elements) at core/async-batch-rpc-retrying-caller.cc:313 Python Exception <class 'gdb.error'> There is no member or method named _M_bbegin.: #21 0x00007fec1370f048 in hbase::AsyncBatchRpcRetryingCaller::Send (this=0x7febb0026820, actions_by_server=std::unordered_map with 1 elements, tries=1) at core/async-batch-rpc-retrying-caller.cc:343 #22 0x00007fec1370d819 in hbase::AsyncBatchRpcRetryingCaller::<lambda(std::vector<folly::Try<std::shared_ptr<hbase::RegionLocation> >, std::allocator<folly::Try<std::shared_ptr<hbase::RegionLocation> > > >&)>::operator()(std::vector<folly::Try<std::shared_ptr<hbase::RegionLocation> >, std::allocator<folly::Try<std::shared_ptr<hbase::RegionLocation> > > > &) const (__closure=0x7febb00027b0, loc=std::vector of length 100, capacity 100 = {...}) at core/async-batch-rpc-retrying-caller.cc:283 #23 0x00007fec137112d0 in folly::detail::CoreCallbackState<folly::Unit, hbase::AsyncBatchRpcRetryingCaller::GroupAndSend(const std::vector<std::shared_ptr<hbase::Action> >&, int32_t)::<lambda(std::vector<folly::Try<std::shared_ptr<hbase::RegionLocation> >, std::allocator<folly::Try<std::shared_ptr<hbase::RegionLocation> > > >&)> >::invoke<std::vector<folly::Try<std::shared_ptr<hbase::RegionLocation> >, std::allocator<folly::Try<std::shared_ptr<hbase::RegionLocation> > > >&>(std::vector<folly::Try<std::shared_ptr<hbase::RegionLocation> >, std::allocator<folly::Try<std::shared_ptr<hbase::RegionLocation> > > > &) (this=0x7febb00027b0, args#0=std::vector of length 100, capacity 100 = {...}) at /usr/local/include/folly/futures/Future-inl.h:85 #24 0x00007fec13711309 in folly::Future<std::vector<folly::Try<std::shared_ptr<hbase::RegionLocation> >, std::allocator<folly::Try<std::shared_ptr<hbase::RegionLocation> > > > >::<lambda(folly::Try<std::vector<folly::Try<std::shared_ptr<hbase::RegionLocation> >, std::allocator<folly::Try<std::shared_ptr<hbase::RegionLocation> > > > >&&)>::<lambda()>::operator()(void) const (__closure=0x7febf17f91a0) at /usr/local/include/folly/futures/Future-inl.h:243 #25 0x00007fec13712520 in folly::makeTryWith<folly::Future<T>::thenImplementation(F&&, folly::detail::argResult<isTry, F, Args ...>)::<lambda(folly::Try<T>&&)> mutable [with F = hbase::AsyncBatchRpcRetryingCaller::GroupAndSend(const std::vector<std::shared_ptr<hbase::Action> >&, int32_t)::<lambda(std::vector<folly::Try<std::shared_ptr<hbase::RegionLocation> > >&)>; R = folly::detail::callableResult<std::vector<folly::Try<std::shared_ptr<hbase::RegionLocation> > >, hbase::AsyncBatchRpcRetryingCaller::GroupAndSend(const std::vector<std::shared_ptr<hbase::Action> >&, int32_t)::<lambda(std::vector<folly::Try<std::shared_ptr<hbase::RegionLocation> > >&)> >; bool isTry = false; Args = {std::vector<folly::Try<std::shared_ptr<hbase::RegionLocation> >, std::allocator<folly::Try<std::shared_ptr<hbase::RegionLocation> > > >&}; T = std::vector<folly::Try<std::shared_ptr<hbase::RegionLocation> > >]::<lambda()> >(<unknown type in /usr/local/lib/libHbaseClient_d.so, CU 0xb0d6f8, DIE 0xbd8453>) ( f=<unknown type in /usr/local/lib/libHbaseClient_d.so, CU 0xb0d6f8, DIE 0xbd8453>) at /usr/local/include/folly/Try-inl.h:165 #26 0x00007fec1371137b in folly::Future<std::vector<folly::Try<std::shared_ptr<hbase::RegionLocation> >, std::allocator<folly::Try<std::shared_ptr<hbase::RegionLocation> > > > >::<lambda(folly::Try<std::vector<folly::Try<std::shared_ptr<hbase::RegionLocation> >, std::allocator<folly::Try<std::shared_ptr<hbase::RegionLocation> > > > >&&)>::operator()(<unknown type in /usr/local/lib/libHbaseClient_d.so, CU 0xb0d6f8, DIE 0xba6a18>) (__closure=0x7febb00027b0, t=<unknown type in /usr/local/lib/libHbaseClient_d.so, CU 0xb0d6f8, DIE 0xba6a18>) at /usr/local/include/folly/futures/Future-inl.h:242 #27 0x00007fec13713f57 in folly::detail::function::FunctionTraits<void(folly::Try<std::vector<folly::Try<std::shared_ptr<hbase::RegionLocation> >, std::allocator<folly::Try<std::shared_ptr<hbase::RegionLocation> > > > >&&)>::callBig<folly::Future<T>::thenImplementation(F&&, folly::detail::argResult<isTry, F, Args ...>) [with F = hbase::AsyncBatchRpcRetryingCaller::GroupAndSend(const std::vector<std::shared_ptr<hbase::Action> >&, int32_t)::<lambda(std::vector<folly::Try<std::shared_ptr<hbase::RegionLocation> > >&)>; R = folly::detail::callableResult<std::vector<folly::Try<std::shared_ptr<hbase::RegionLocation> > >, hbase::AsyncBatchRpcRetryingCaller::GroupAndSend(const std::vector<std::shared_ptr<hbase::Action> >&, int32_t)::<lambda(std::vector<folly::Try<std::shared_ptr<hbase::RegionLocation> > >&)> >; bool isTry = false; Args = {std::vector<folly::Try<std::shared_ptr<hbase::RegionLocation> >, std::allocator<folly::Try<std::shared_ptr<hbase::RegionLocation> > > >&}; T = std::vector<folly::Try<std::shared_ptr<hbase::RegionLocation> > >]::<lambda(folly::Try<std::vector<folly::Try<std::shared_ptr<hbase::RegionLocation> > > >&&)> >(folly::detail::function::Data &, <unknown type in /usr/local/lib/libHbaseClient_d.so, CU 0xb0d6f8, DIE 0xba6a18>) (p=..., args#0=<unknown type in /usr/local/lib/libHbaseClient_d.so, CU 0xb0d6f8, DIE 0xba6a18>) at /usr/local/include/folly/Function.h:305 #28 0x00007fec1372a0bc in folly::detail::function::FunctionTraits<void (folly::Try<std::vector<folly::Try<std::shared_ptr<hbase::RegionLocation> >, std::allocator<folly::Try<std::shared_ptr<hbase::RegionLocation> > > > >&&)>::operator()(folly::Try<std::vector<folly::Try<std::shared_ptr<hbase::RegionLocation> >, std::allocator<folly::Try<std::shared_ptr<hbase::RegionLocation> > > > >&&) ( this=0x7febb0024af0, args#0=<unknown type in /usr/local/lib/libHbaseClient_d.so, CU 0xb0d6f8, DIE 0xba6a18>) at /usr/local/include/folly/Function.h:314 #29 0x00007fec1372759d in folly::detail::Core<std::vector<folly::Try<std::shared_ptr<hbase::RegionLocation> >, std::allocator<folly::Try<std::shared_ptr<hbase::RegionLocation> > > > >::doCallback ( this=0x7febb0024af0) at /usr/local/include/folly/futures/detail/Core.h:388 #30 0x00007fec13724051 in folly::detail::Core<std::vector<folly::Try<std::shared_ptr<hbase::RegionLocation> >, std::allocator<folly::Try<std::shared_ptr<hbase::RegionLocation> > > > >::maybeCallback()::{lambda()#2}::operator()() const (__closure=0x7febf17f94c0) at /usr/local/include/folly/futures/detail/Core.h:317 #31 0x00007fec1372781e in folly::detail::FSM<folly::detail::State>::updateState<folly::detail::Core<std::vector<folly::Try<std::shared_ptr<hbase::RegionLocation> >, std::allocator<folly::Try<std::shared_ptr<hbase::RegionLocation> > > > >::maybeCallback()::{lambda()#1}, folly::detail::Core<std::vector<folly::Try<std::shared_ptr<hbase::RegionLocation> >, std::allocator<folly::Try<std::shared_ptr<hbase::RegionLocation> > > > >::maybeCallback()::{lambda()#2}>(folly::detail::State, folly::detail::State, folly::detail::Core<std::vector<folly::Try<std::shared_ptr<hbase::RegionLocation> >, std::allocator<folly::Try<std::shared_ptr<hbase::RegionLocation> > > > >::maybeCallback()::{lambda()#1} const&, folly::detail::Core<std::vector<folly::Try<std::shared_ptr<hbase::RegionLocation> >, std::allocator<folly::Try<std::shared_ptr<hbase::RegionLocation> > > > >::maybeCallback()::{lambda()#2} const&) (this=0x7febb0024b58, A=folly::detail::Armed, B=folly::detail::Done, protectedAction=..., unprotectedAction=...) at /usr/local/include/folly/futures/detail/FSM.h:100 #32 0x00007fec137240cc in folly::detail::Core<std::vector<folly::Try<std::shared_ptr<hbase::RegionLocation> >, std::allocator<folly::Try<std::shared_ptr<hbase::RegionLocation> > > > >::maybeCallback ( this=0x7febb0024af0) at /usr/local/include/folly/futures/detail/Core.h:317 #33 0x00007fec1371319b in folly::detail::Core<std::vector<folly::Try<std::shared_ptr<hbase::RegionLocation> >, std::allocator<folly::Try<std::shared_ptr<hbase::RegionLocation> > > > >::setCallback<folly::Future<T>::thenImplementation(F&&, folly::detail::argResult<isTry, F, Args ...>) [with F = hbase::AsyncBatchRpcRetryingCaller::GroupAndSend(const std::vector<std::shared_ptr<hbase::Action> >&, int32_t)::<lambda(std::vector<folly::Try<std::shared_ptr<hbase::RegionLocation> > >&)>; R = folly::detail::callableResult<std::vector<folly::Try<std::shared_ptr<hbase::RegionLocation> > >, hbase::AsyncBatchRpcRetryingCaller::GroupAndSend(const std::vector<std::shared_ptr<hbase::Action> >&, int32_t)::<lambda(std::vector<folly::Try<std::shared_ptr<hbase::RegionLocation> > >&)> >; bool isTry = false; Args = {std::vector<folly::Try<std::shared_ptr<hbase::RegionLocation> >, std::allocator<folly::Try<std::shared_ptr<hbase::RegionLocation> > > >&}; T = std::vector<folly::Try<std::shared_ptr<hbase::RegionLocation>#34 0x00007fec137126d9 in folly::Future<std::vector<folly::Try<std::shared_ptr<hbase::RegionLocation> >, std::allocator<folly::Try<std::shared_ptr<hbase::RegionLocation> > > > >::setCallback_<folly::Future<T>::thenImplementation(F&&, folly::detail::argResult<isTry, F, Args ...>) [with F = hbase::AsyncBatchRpcRetryingCaller::GroupAndSend(const std::vector<std::shared_ptr<hbase::Action> >&, int32_t)::<lambda(std::vector<folly::Try<std::shared_ptr<hbase::RegionLocation> > >&)>; R = folly::detail::callableResult<std::vector<folly::Try<std::shared_ptr<hbase::RegionLocation> > >, hbase::AsyncBatchRpcRetryingCaller::GroupAndSend(const std::vector<std::shared_ptr<hbase::Action> >&, int32_t)::<lambda(std::vector<folly::Try<std::shared_ptr<hbase::RegionLocation> > >&)> >; bool isTry = false; Args = {std::vector<folly::Try<std::shared_ptr<hbase::RegionLocation> >, std::allocator<folly::Try<std::shared_ptr<hbase::RegionLocation> > > >&}; T = std::vector<folly::Try<std::shared_ptr<hbase::RegionLocation> > >]::<lambda(folly::Try<std::vector<folly::Try<std::shared_ptr<hbase::RegionLocation> > > >&&)> >(<unknown type in /usr/local/lib/libHbaseClient_d.so, CU 0xb0d6f8, DIE 0xbd839c>) (this=0x7febf17f9700, func=<unknown type in /usr/local/lib/libHbaseClient_d.so, CU 0xb0d6f8, DIE 0xbd839c>) at /usr/local/include/folly/futures/Future-inl.h:172 #35 0x00007fec13711509 in folly::Future<std::vector<folly::Try<std::shared_ptr<hbase::RegionLocation> >, std::allocator<folly::Try<std::shared_ptr<hbase::RegionLocation> > > > >::thenImplementation<hbase::AsyncBatchRpcRetryingCaller::GroupAndSend(const std::vector<std::shared_ptr<hbase::Action> >&, int32_t)::<lambda(std::vector<folly::Try<std::shared_ptr<hbase::RegionLocation> > >&)>, folly::detail::callableResult<std::vector<folly::Try<std::shared_ptr<hbase::RegionLocation> > >, hbase::AsyncBatchRpcRetryingCaller::GroupAndSend(const std::vector<std::shared_ptr<hbase::Action> >&, int32_t)::<lambda(std::vector<folly::Try<std::shared_ptr<hbase::RegionLocation> > >&)> >, false, std::vector<folly::Try<std::shared_ptr<hbase::RegionLocation> >, std::allocator<folly::Try<std::shared_ptr<hbase::RegionLocation> > > >&>(<unknown type in /usr/local/lib/libHbaseClient_d.so, CU 0xb0d6f8, DIE 0xbe0d1b>, folly::detail::argResult<false, hbase::AsyncBatchRpcRetryingCaller::GroupAndSend(const std::vector<std::shared_ptr<hbase::Action> >&, int32_t)::<lambda(std::vector<folly::Try<std::shared_ptr<hbase::RegionLocation> >, std::allocator<folly::Try<std::shared_ptr<hbase::RegionLocation> > > >&)>, std::vector<folly::Try<std::shared_ptr<hbase::RegionLocation> >, std::allocator<folly::Try<std::shared_ptr<hbase::RegionLocation> > > >&>) (this=0x7febf17f9700, func=<unknown type in /usr/local/lib/libHbaseClient_d.so, CU 0xb0d6f8, DIE 0xbe0d1b>) at /usr/local/include/folly/futures/Future-inl.h:236 #36 0x00007fec137109de in folly::Future<std::vector<folly::Try<std::shared_ptr<hbase::RegionLocation> >, std::allocator<folly::Try<std::shared_ptr<hbase::RegionLocation> > > > >::then<hbase::AsyncBatchRpcRetryingCaller::GroupAndSend(const std::vector<std::shared_ptr<hbase::Action> >&, int32_t)::<lambda(std::vector<folly::Try<std::shared_ptr<hbase::RegionLocation> > >&)> >(<unknown type in /usr/local/lib/libHbaseClient_d.so, CU 0xb0d6f8, DIE 0xbe840e>) (this=0x7febf17f9700, func=<unknown type in /usr/local/lib/libHbaseClient_d.so, CU 0xb0d6f8, DIE 0xbe840e>) at /usr/local/include/folly/futures/Future.h:183 #37 0x00007fec1370df37 in hbase::AsyncBatchRpcRetryingCaller::GroupAndSend (this=0x7febb0026820, actions=std::vector of length 100, capacity 128 = {...}, tries=1) at core/async-batch-rpc-retrying-caller.cc:242 #38 0x00007fec1370b328 in hbase::AsyncBatchRpcRetryingCaller::Call (this=0x7febb0026820) at core/async-batch-rpc-retrying-caller.cc:70 #39 0x00007fec13732c33 in hbase::RawAsyncTable::Batch (this=0x7febb0000980, gets=std::vector of length 100, capacity 128 = {...}) at core/raw-async-table.cc:218 #40 0x00007fec137328a1 in hbase::RawAsyncTable::Get (this=0x7febb0000980, gets=std::vector of length 100, capacity 128 = {...}) at core/raw-async-table.cc:202 #41 0x00007fec137036a9 in hbase::Table::Get (this=0x7febb00008c0, gets=std::vector of length 100, capacity 128 = {...}) at core/table.cc:120 #42 0x000000000042e328 in DoGet (iteration=0, max_row=19999, rows=2000, table=std::unique_ptr<hbase::Table> containing 0x7febb00008c0, families=std::vector of length 1, capacity 1 = {...}, batch_num_rows=100) at core/load-client.cc:186 #43 0x000000000042faab in <lambda()>::operator()(void) const (__closure=0xaa37d8) at core/load-client.cc:375 #44 0x0000000000432428 in std::_Bind_simple<main(int, char**)::<lambda()>()>::_M_invoke<>(std::_Index_tuple<>) (this=0xaa37d8) at /opt/rh/devtoolset-6/root/usr/include/c++/6.2.1/functional:1400 #45 0x0000000000432359 in std::_Bind_simple<main(int, char**)::<lambda()>()>::operator()(void) (this=0xaa37d8) at /opt/rh/devtoolset-6/root/usr/include/c++/6.2.1/functional:1389 #46 0x00000000004322d8 in std::thread::_State_impl<std::_Bind_simple<main(int, char**)::<lambda()>()> >::_M_run(void) (this=0xaa37d0) at /opt/rh/devtoolset-6/root/usr/include/c++/6.2.1/thread:196 #47 0x00007fec139c3aff in execute_native_thread_routine () from /usr/local/lib/libHbaseClient_d.so #48 0x00007fec0eea1dc5 in start_thread () from /lib64/libpthread.so.0 #49 0x00007fec0ebd073d in clone () from /lib64/libc.so.6 {code} and yet some other thread from the scanner has those locks in reverse order: {code} Thread 17 (Thread 0x7fec017fa700 (LWP 6183)): #0 0x00007fec0eea81bd in __lll_lock_wait () from /lib64/libpthread.so.0 #1 0x00007fec0eea3d1d in _L_lock_840 () from /lib64/libpthread.so.0 #2 0x00007fec0eea3c3a in pthread_mutex_lock () from /lib64/libpthread.so.0 #3 0x00007fec135fe083 in __gthread_mutex_lock (__mutex=0x7febec001a58) at /opt/rh/devtoolset-6/root/usr/include/c++/6.2.1/x86_64-redhat-linux/bits/gthr-default.h:748 #4 0x00007fec135fe0d3 in __gthread_recursive_mutex_lock (__mutex=0x7febec001a58) at /opt/rh/devtoolset-6/root/usr/include/c++/6.2.1/x86_64-redhat-linux/bits/gthr-default.h:810 #5 0x00007fec135feb56 in std::recursive_mutex::lock (this=0x7febec001a58) at /opt/rh/devtoolset-6/root/usr/include/c++/6.2.1/mutex:105 #6 0x00007fec136005c6 in std::lock_guard<std::recursive_mutex>::lock_guard (this=0x7fec017f7f10, __m=...) at /opt/rh/devtoolset-6/root/usr/include/c++/6.2.1/bits/std_mutex.h:162 #7 0x00007fec135ff9e0 in hbase::RpcConnection::SendRequest (this=0x7febec001a40, req=std::unique_ptr<hbase::Request> containing 0x7febd16ae450) at ./connection/rpc-connection.h:43 #8 0x00007fec135bfca0 in hbase::RpcClient::SendRequest (this=0xa6f890, remote_id=std::shared_ptr (count 3, weak 0) 0x7febd16ae390, req=std::unique_ptr<hbase::Request> containing 0x0) at connection/rpc-client.cc:85 #9 0x00007fec135bf85c in hbase::RpcClient::AsyncCall (this=0xa6f890, host="localhost", port=42985, req=std::unique_ptr<hbase::Request> containing 0x0, ticket=std::shared_ptr (count 2, weak 0) 0x7febd0af9fc0, service_name="ClientService") at connection/rpc-client.cc:71 #10 0x00007fec136cd6ca in hbase::AsyncScanRpcRetryingCaller::Call (this=0x7febd00011a0) at core/async-scan-rpc-retrying-caller.cc:406 #11 0x00007fec136cd17d in hbase::AsyncScanRpcRetryingCaller::Next (this=0x7febd00011a0) at core/async-scan-rpc-retrying-caller.cc:376 #12 0x00007fec136cba78 in hbase::AsyncScanRpcRetryingCaller::CompleteOrNext (this=0x7febd00011a0, resp=std::shared_ptr (count 5, weak 0) 0x7febd0247930) at core/async-scan-rpc-retrying-caller.cc:241 #13 0x00007fec136cb60f in hbase::AsyncScanRpcRetryingCaller::OnComplete (this=0x7febd00011a0, controller=std::shared_ptr (count 2, weak 0) 0x7febd0214ae0, resp=std::shared_ptr (count 5, weak 0) 0x7febd0247930, cell_scanner=std::shared_ptr (count 2, weak 0) 0x7febd023e690) at core/async-scan-rpc-retrying-caller.cc:222 #14 0x00007fec136cd269 in hbase::AsyncScanRpcRetryingCaller::<lambda(const std::unique_ptr<hbase::Response, std::default_delete<hbase::Response> >&)>::operator()(const std::unique_ptr<hbase::Response, std::default_delete<hbase::Response> > &) const (__closure=0x7febd0245890, resp=std::unique_ptr<hbase::Response> containing 0x7febd0b09650) at core/async-scan-rpc-retrying-caller.cc:411 #15 0x00007fec136cec8e in folly::detail::CoreCallbackState<folly::Unit, hbase::AsyncScanRpcRetryingCaller::Call()::<lambda(const std::unique_ptr<hbase::Response, std::default_delete<hbase::Response> >&)> >::invoke<std::unique_ptr<hbase::Response, std::default_delete<hbase::Response> > >(<unknown type in /usr/local/lib/libHbaseClient_d.so, CU 0x86df39, DIE 0x8d2aaa>) (this=0x7febd0245890, args#0=<unknown type in /usr/local/lib/libHbaseClient_d.so, CU 0x86df39, DIE 0x8d2aaa>) at /usr/local/include/folly/futures/Future-inl.h:85 #16 0x00007fec136cecc7 in folly::Future<std::unique_ptr<hbase::Response, std::default_delete<hbase::Response> > >::<lambda(folly::Try<std::unique_ptr<hbase::Response, std::default_delete<hbase::Response> > >&&)>::<lambda()>::operator()(void) const (__closure=0x7fec017f8730) at /usr/local/include/folly/futures/Future-inl.h:243 #17 0x00007fec136cfa96 in folly::makeTryWith<folly::Future<T>::thenImplementation(F&&, folly::detail::argResult<isTry, F, Args ...>)::<lambda(folly::Try<T>&&)> mutable [with F = hbase::AsyncScanRpcRetryingCaller::Call()::<lambda(const std::unique_ptr<hbase::Response>&)>; R = folly::detail::callableResult<std::unique_ptr<hbase::Response>, hbase::AsyncScanRpcRetryingCaller::Call()::<lambda(const std::unique_ptr<hbase::Response>&)> >; bool isTry = false; Args = {std::unique_ptr<hbase::Response, std::default_delete<hbase::Response> >&&}; T = std::unique_ptr<hbase::Response>]::<lambda()> >(<unknown type in /usr/local/lib/libHbaseClient_d.so, CU 0x86df39, DIE 0x8e66f0>) (f=<unknown type in /usr/local/lib/libHbaseClient_d.so, CU 0x86df39, DIE 0x8e66f0>) at /usr/local/include/folly/Try-inl.h:165 #18 0x00007fec136ced39 in folly::Future<std::unique_ptr<hbase::Response, std::default_delete<hbase::Response> > >::<lambda(folly::Try<std::unique_ptr<hbase::Response, std::default_delete<hbase::Response> > >&&)>::operator()(<unknown type in /usr/local/lib/libHbaseClient_d.so, CU 0x86df39, DIE 0x8d3c22>) (__closure=0x7febd0245890, t=<unknown type in /usr/local/lib/libHbaseClient_d.so, CU 0x86df39, DIE 0x8d3c22>) at /usr/local/include/folly/futures/Future-inl.h:242 #19 0x00007fec136d0bd6 in folly::detail::function::FunctionTraits<void(folly::Try<std::unique_ptr<hbase::Response, std::default_delete<hbase::Response> > >&&)>::callSmall<folly::Future<T>::thenImplementation(F&&, folly::detail::argResult<isTry, F, Args ...>) [with F = hbase::AsyncScanRpcRetryingCaller::Call()::<lambda(const std::unique_ptr<hbase::Response>&)>; R = folly::detail::callableResult<std::unique_ptr<hbase::Response>, hbase::AsyncScanRpcRetryingCaller::Call()::<lambda(const std::unique_ptr<hbase::Response>&)> >; bool isTry = false; Args = {std::unique_ptr<hbase::Response, std::default_delete<hbase::Response> >&&}; T = std::unique_ptr<hbase::Response>]::<lambda(folly::Try<std::unique_ptr<hbase::Response> >&&)> >(folly::detail::function::Data &, <unknown type in /usr/local/lib/libHbaseClient_d.so, CU 0x86df39, DIE 0x8d3c22>) (p=..., args#0=<unknown type in /usr/local/lib/libHbaseClient_d.so, CU 0x86df39, DIE 0x8d3c22>) at /usr/local/include/folly/Function.h:299 #20 0x00007fec135cc402 in folly::detail::function::FunctionTraits<void (folly::Try<std::unique_ptr<hbase::Response, std::default_delete<hbase::Response> > >&&)>::operator()(folly::Try<std::unique_ptr<hbase::Response, std::default_delete<hbase::Response> > >&&) (this=0x7febd0245890, args#0=<unknown type in /usr/local/lib/libHbaseClient_d.so, CU 0x45953, DIE 0xa1de0>) at /usr/local/include/folly/Function.h:314 #21 0x00007fec135cab03 in folly::detail::Core<std::unique_ptr<hbase::Response, std::default_delete<hbase::Response> > >::doCallback (this=0x7febd0245890) at /usr/local/include/folly/futures/detail/Core.h:388 #22 0x00007fec135c898b in folly::detail::Core<std::unique_ptr<hbase::Response, std::default_delete<hbase::Response> > >::maybeCallback()::{lambda()#2}::operator()() const (__closure=0x7fec017f8a50) at /usr/local/include/folly/futures/detail/Core.h:317 #23 0x00007fec135cad84 in folly::detail::FSM<folly::detail::State>::updateState<folly::detail::Core<std::unique_ptr<hbase::Response, std::default_delete<hbase::Response> > >::maybeCallback()::{lambda()#1}, folly::detail::Core<std::unique_ptr<hbase::Response, std::default_delete<hbase::Response> > >::maybeCallback()::{lambda()#2}>(folly::detail::State, folly::detail::State, folly::detail::Core<std::unique_ptr<hbase::Response, std::default_delete<hbase::Response> > >::maybeCallback()::{lambda()#1} const&, folly::detail::Core<std::unique_ptr<hbase::Response, std::default_delete<hbase::Response> > >::maybeCallback()::{lambda()#2} const&) (this=0x7febd02458f8, A=folly::detail::Armed, B=folly::detail::Done, protectedAction=..., unprotectedAction=...) at /usr/local/include/folly/futures/detail/FSM.h:100 #24 0x00007fec135c8a06 in folly::detail::Core<std::unique_ptr<hbase::Response, std::default_delete<hbase::Response> > >::maybeCallback (this=0x7febd0245890) at /usr/local/include/folly/futures/detail/Core.h:317 #25 0x00007fec135c6f44 in folly::detail::Core<std::unique_ptr<hbase::Response, std::default_delete<hbase::Response> > >::setResult(folly::Try<std::unique_ptr<hbase::Response, std::default_delete<hbase::Response> > >&&) (this=0x7febd0245890, t=<unknown type in /usr/local/lib/libHbaseClient_d.so, CU 0x45953, DIE 0xa1de0>) at /usr/local/include/folly/futures/detail/Core.h:183 #26 0x00007fec135c68c0 in folly::Promise<std::unique_ptr<hbase::Response, std::default_delete<hbase::Response> > >::setTry(folly::Try<std::unique_ptr<hbase::Response, std::default_delete<hbase::Response> > >&&) (this=0x7fec017f8af0, t=<unknown type in /usr/local/lib/libHbaseClient_d.so, CU 0x45953, DIE 0xa1de0>) at /usr/local/include/folly/futures/Promise-inl.h:121 #27 0x00007fec135c0c4c in folly::detail::CoreCallbackState<std::unique_ptr<hbase::Response, std::default_delete<hbase::Response> >, hbase::RpcClient::SendRequest(std::shared_ptr<hbase::ConnectionId>, std::unique_ptr<hbase::Request>)::<lambda(const folly::exception_wrapper&)> >::setTry(<unknown type in /usr/local/lib/libHbaseClient_d.so, CU 0x45953, DIE 0xa1de0>) (this=0x7febd0af2a90, t=<unknown type in /usr/local/lib/libHbaseClient_d.so, CU 0x45953, DIE 0xa1de0>) at /usr/local/include/folly/futures/Future-inl.h:94 #28 0x00007fec135c05a0 in folly::Future<std::unique_ptr<hbase::Response, std::default_delete<hbase::Response> > >::<lambda(folly::Try<std::unique_ptr<hbase::Response, std::default_delete<hbase::Response> > >)>::operator()(folly::Try<std::unique_ptr<hbase::Response, std::default_delete<hbase::Response> > >) (__closure=0x7febd0af2a90, t=...) at /usr/local/include/folly/futures/Future-inl.h:424 #29 0x00007fec135c1953 in folly::detail::function::FunctionTraits<void(folly::Try<std::unique_ptr<hbase::Response, std::default_delete<hbase::Response> > >&&)>::callSmall<folly::Future<T>::onError(F&&) [with F = hbase::RpcClient::SendRequest(std::shared_ptr<hbase::ConnectionId>, std::unique_ptr<hbase::Request>)::<lambda(const folly::exception_wrapper&)>; T = std::unique_ptr<hbase::Response>]::<lambda(folly::Try<std::unique_ptr<hbase::Response> >)> >(folly::detail::function::Data &, <unknown type in /usr/local/lib/libHbaseClient_d.so, CU 0x45953, DIE 0xa1de0>) (p=..., args#0=<unknown type in /usr/local/lib/libHbaseClient_d.so, CU 0x45953, DIE 0xa1de0>) at /usr/local/include/folly/Function.h:299 #30 0x00007fec135cc402 in folly::detail::function::FunctionTraits<void (folly::Try<std::unique_ptr<hbase::Response, std::default_delete<hbase::Response> > >&&)>::operator()(folly::Try<std::unique_ptr<hbase::Response, std::default_delete<hbase::Response> > >&&) (this=0x7febd0af2a90, args#0=<unknown type in /usr/local/lib/libHbaseClient_d.so, CU 0x45953, DIE 0xa1de0>) at /usr/local/include/folly/Function.h:314 #31 0x00007fec135cab03 in folly::detail::Core<std::unique_ptr<hbase::Response, std::default_delete<hbase::Response> > >::doCallback (this=0x7febd0af2a90) at /usr/local/include/folly/futures/detail/Core.h:388 #32 0x00007fec135c898b in folly::detail::Core<std::unique_ptr<hbase::Response, std::default_delete<hbase::Response> > >::maybeCallback()::{lambda()#2}::operator()() const (__closure=0x7fec017f8ea0) at /usr/local/include/folly/futures/detail/Core.h:317 #33 0x00007fec135cad84 in folly::detail::FSM<folly::detail::State>::updateState<folly::detail::Core<std::unique_ptr<hbase::Response, std::default_delete<hbase::Response> > >::maybeCallback()::{lambda()#1}, folly::detail::Core<std::unique_ptr<hbase::Response, std::default_delete<hbase::Response> > >::maybeCallback()::{lambda()#2}>(folly::detail::State, folly::detail::State, folly::detail::Core<std::unique_ptr<hbase::Response, std::default_delete<hbase::Response> > >::maybeCallback()::{lambda()#1} const&, folly::detail::Core<std::unique_ptr<hbase::Response, std::default_delete<hbase::Response> > >::maybeCallback()::{lambda()#2} const&) (this=0x7febd0af2af8, A=folly::detail::Armed, B=folly::detail::Done, protectedAction=..., unprotectedAction=...) at /usr/local/include/folly/futures/detail/FSM.h:100 #34 0x00007fec135c8a06 in folly::detail::Core<std::unique_ptr<hbase::Response, std::default_delete<hbase::Response> > >::maybeCallback (this=0x7febd0af2a90) at /usr/local/include/folly/futures/detail/Core.h:317 #35 0x00007fec135c6f44 in folly::detail::Core<std::unique_ptr<hbase::Response, std::default_delete<hbase::Response> > >::setResult(folly::Try<std::unique_ptr<hbase::Response, std::default_delete<hbase::Response> > >&&) (this=0x7febd0af2a90, t=<unknown type in /usr/local/lib/libHbaseClient_d.so, CU 0x45953, DIE 0xa1de0>) at /usr/local/include/folly/futures/detail/Core.h:183 #36 0x00007fec135c68c0 in folly::Promise<std::unique_ptr<hbase::Response, std::default_delete<hbase::Response> > >::setTry(folly::Try<std::unique_ptr<hbase::Response, std::default_delete<hbase::Response> > >&&) (this=0x7fec017f8fa0, t=<unknown type in /usr/local/lib/libHbaseClient_d.so, CU 0x45953, DIE 0xa1de0>) at /usr/local/include/folly/futures/Promise-inl.h:121 #37 0x00007fec1360d150 in folly::Promise<std::unique_ptr<hbase::Response, std::default_delete<hbase::Response> > >::setValue<std::unique_ptr<hbase::Response, std::default_delete<hbase::Response> > >(std::unique_ptr<hbase::Response, std::default_delete<hbase::Response> >&&) (this=0x7fec017f8fa0, v=<unknown type in /usr/local/lib/libHbaseClient_d.so, CU 0x216413, DIE 0x268be9>) at /usr/local/include/folly/futures/Promise-inl.h:130 #38 0x00007fec1360b8b4 in hbase::ClientDispatcher::read (this=0x7febec001e00, ctx=0x7febec0010f0, in=std::unique_ptr<hbase::Response> containing 0x0) at connection/client-dispatcher.cc:49 #39 0x00007fec1362ff72 in wangle::ContextImpl<wangle::ClientDispatcherBase<wangle::Pipeline<folly::IOBufQueue&, std::unique_ptr<hbase::Request, std::default_delete<hbase::Request> > >, std::unique_ptr<hbase::Request, std::default_delete<hbase::Request> >, std::unique_ptr<hbase::Response, std::default_delete<hbase::Response> > > >::read (this=0x7febec0010f0, msg=std::unique_ptr<hbase::Response> containing 0x0) at /usr/local/include/wangle/channel/HandlerContext-inl.h:264 #40 0x00007fec1361ccc1 in wangle::ContextImpl<hbase::ClientHandler>::fireRead (this=0x7febd000c2c0, msg=std::unique_ptr<hbase::Response> containing 0x0) at /usr/local/include/wangle/channel/HandlerContext-inl.h:166 #41 0x00007fec136054c2 in hbase::ClientHandler::read (this=0x7febd000c240, ctx=0x7febd000c2c0, buf=std::unique_ptr<folly::IOBuf> containing 0x0) at connection/client-handler.cc:114 #42 0x00007fec1361d4fc in wangle::ContextImpl<hbase::ClientHandler>::read (this=0x7febd000c2c0, msg=std::unique_ptr<folly::IOBuf> containing 0x0) at /usr/local/include/wangle/channel/HandlerContext-inl.h:264 #43 0x00007fec135f8071 in wangle::InboundContextImpl<wangle::LengthFieldBasedFrameDecoder>::fireRead (this=0x7febd00021e0, msg=std::unique_ptr<folly::IOBuf> containing 0x0) at /usr/local/include/wangle/channel/HandlerContext-inl.h:335 #44 0x0000000000440a94 in wangle::ByteToMessageDecoder<std::unique_ptr<folly::IOBuf, std::default_delete<folly::IOBuf> > >::read(wangle::InboundHandlerContext<std::unique_ptr<folly::IOBuf, std::default_delete<folly::IOBuf> > >*, folly::IOBufQueue&) () #45 0x00007fec135f84b7 in wangle::InboundContextImpl<wangle::LengthFieldBasedFrameDecoder>::read (this=0x7febd00021e0, msg=...) at /usr/local/include/wangle/channel/HandlerContext-inl.h:384 #46 0x00007fec1361dd92 in wangle::ContextImpl<hbase::SaslHandler>::fireRead (this=0x7febd00020c0, msg=...) at /usr/local/include/wangle/channel/HandlerContext-inl.h:166 #47 0x00007fec1361570b in hbase::SaslHandler::read (this=0x7febd0002040, ctx=0x7febd00020c0, buf=...) at connection/sasl-handler.cc:96 #48 0x00007fec1361e59d in wangle::ContextImpl<hbase::SaslHandler>::read (this=0x7febd00020c0, msg=...) at /usr/local/include/wangle/channel/HandlerContext-inl.h:264 #49 0x00007fec135f9432 in wangle::ContextImpl<wangle::AsyncSocketHandler>::fireRead (this=0x7febd0001e80, msg=...) at /usr/local/include/wangle/channel/HandlerContext-inl.h:166 #50 0x00007fec135d9ecd in wangle::AsyncSocketHandler::readDataAvailable (this=0x7febd00017d0, len=841) at /usr/local/include/wangle/channel/AsyncSocketHandler.h:138 #51 0x00007fec123b46d4 in folly::AsyncSocket::handleRead (this=0x7febd0001850) at io/async/AsyncSocket.cpp:1628 #52 0x00007fec123acaee in folly::AsyncSocket::ioReady (this=0x7febd0001850, events=2) at io/async/AsyncSocket.cpp:1458 #53 0x00007fec0d2d7a14 in event_base_loop () from /lib64/libevent-2.0.so.5 #54 0x00007fec123cc3b5 in folly::EventBase::loopBody (this=this@entry=0x7febd00009f0, flags=flags@entry=0) at io/async/EventBase.cpp:297 #55 0x00007fec123ccd97 in folly::EventBase::loop (this=this@entry=0x7febd00009f0) at io/async/EventBase.cpp:236 #56 0x00007fec123ce126 in folly::EventBase::loopForever (this=0x7febd00009f0) at io/async/EventBase.cpp:434 #57 0x000000000044fe5c in wangle::IOThreadPoolExecutor::threadRun(std::shared_ptr<wangle::ThreadPoolExecutor::Thread>) () #58 0x0000000000461ec7 in void std::_Mem_fn_base<void (wangle::ThreadPoolExecutor::*)(std::shared_ptr<wangle::ThreadPoolExecutor::Thread>), true>::operator()<std::shared_ptr<wangle::ThreadPoolExecutor::Thread>&, void>(wangle::ThreadPoolExecutor*, std::shared_ptr<wangle::ThreadPoolExecutor::Thread>&) const () #59 0x00000000004605dd in void std::_Bind<std::_Mem_fn<void (wangle::ThreadPoolExecutor::*)(std::shared_ptr<wangle::ThreadPoolExecutor::Thread>)> (wangle::ThreadPoolExecutor*, std::shared_ptr<wangle::ThreadPoolExecutor::Thread>)>::__call<void, , 0ul, 1ul>(std::tuple<>&&, std::_Index_tuple<0ul, 1ul>) () #60 0x000000000045ef48 in void std::_Bind<std::_Mem_fn<void (wangle::ThreadPoolExecutor::*)(std::shared_ptr<wangle::ThreadPoolExecutor::Thread>)> (wangle::ThreadPoolExecutor*, std::shared_ptr<wangle::ThreadPoolExecutor::Thread>)>::operator()<, void>() () #61 0x000000000045d3e8 in void folly::detail::function::FunctionTraits<void ()>::callBig<std::_Bind<std::_Mem_fn<void (wangle::ThreadPoolExecutor::*)(std::shared_ptr<wangle::ThreadPoolExecutor::Thread>)> (wangle::ThreadPoolExecutor*, std::shared_ptr<wangle::ThreadPoolExecutor::Thread>)> >(folly::detail::function::Data&) () #62 0x000000000044c485 in folly::detail::function::FunctionTraits<void ()>::operator()() () #63 0x000000000044b6c0 in void std::_Bind_simple<folly::Function<void ()> ()>::_M_invoke<>(std::_Index_tuple<>) () #64 0x000000000044af1f in std::_Bind_simple<folly::Function<void ()> ()>::operator()() () #65 0x000000000044a7ea in std::thread::_Impl<std::_Bind_simple<folly::Function<void ()> ()> >::_M_run() () #66 0x00007fec139c3bd0 in execute_native_thread_routine_compat () from /usr/local/lib/libHbaseClient_d.so #67 0x00007fec0eea1dc5 in start_thread () from /lib64/libpthread.so.0 #68 0x00007fec0ebd073d in clone () from /lib64/libc.so.6 {code} > [C++] Fix deadlock in AsyncScanRetryingCaller and other RPCs > ------------------------------------------------------------ > > Key: HBASE-18565 > URL: https://issues.apache.org/jira/browse/HBASE-18565 > Project: HBase > Issue Type: Sub-task > Reporter: Enis Soztutar > Assignee: Enis Soztutar > Fix For: HBASE-14850 > > > When running the load-client test, sometimes we get a deadlock. -- This message was sent by Atlassian JIRA (v6.4.14#64029)