[
https://issues.apache.org/jira/browse/HBASE-18565?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16122693#comment-16122693
]
Enis Soztutar commented on HBASE-18565:
---------------------------------------
Attaching to the process using gdb reveals these stack traces:
A lot of threads are in trying to acquire the mutex for the rpc-connection like
this:
{code}
Thread 10 (Thread 0x7febf1ffb700 (LWP 6210)):
#0 0x00007fec0eea81bd in __lll_lock_wait () from /lib64/libpthread.so.0
#1 0x00007fec0eea3d1d in _L_lock_840 () from /lib64/libpthread.so.0
#2 0x00007fec0eea3c3a in pthread_mutex_lock () from /lib64/libpthread.so.0
#3 0x00007fec135fe083 in __gthread_mutex_lock (__mutex=0x7febec001a58) at
/opt/rh/devtoolset-6/root/usr/include/c++/6.2.1/x86_64-redhat-linux/bits/gthr-default.h:748
#4 0x00007fec135fe0d3 in __gthread_recursive_mutex_lock
(__mutex=0x7febec001a58) at
/opt/rh/devtoolset-6/root/usr/include/c++/6.2.1/x86_64-redhat-linux/bits/gthr-default.h:810
#5 0x00007fec135feb56 in std::recursive_mutex::lock (this=0x7febec001a58) at
/opt/rh/devtoolset-6/root/usr/include/c++/6.2.1/mutex:105
#6 0x00007fec136005c6 in std::lock_guard<std::recursive_mutex>::lock_guard
(this=0x7febf1ff9800, __m=...) at
/opt/rh/devtoolset-6/root/usr/include/c++/6.2.1/bits/std_mutex.h:162
#7 0x00007fec135ff9e0 in hbase::RpcConnection::SendRequest
(this=0x7febec001a40, req=std::unique_ptr<hbase::Request> containing
0x7feb9c01e300) at ./connection/rpc-connection.h:43
#8 0x00007fec135bfca0 in hbase::RpcClient::SendRequest (this=0xa6f890,
remote_id=std::shared_ptr (count 3, weak 0) 0x7feb9c025f40,
req=std::unique_ptr<hbase::Request> containing 0x0)
at connection/rpc-client.cc:85
#9 0x00007fec135bf85c in hbase::RpcClient::AsyncCall (this=0xa6f890,
host="localhost", port=42985, req=std::unique_ptr<hbase::Request> containing
0x0,
ticket=std::shared_ptr (count 3, weak 0) 0x7feb9c01e1c0,
service_name="ClientService") at connection/rpc-client.cc:71
Python Exception <class 'gdb.error'> There is no member or method named
_M_bbegin.:
#10 0x00007fec1370e1b1 in hbase::AsyncBatchRpcRetryingCaller::GetMultiResponse
(this=0x7feb9c0036c0, actions_by_server=std::unordered_map with 1 elements) at
core/async-batch-rpc-retrying-caller.cc:313
Python Exception <class 'gdb.error'> There is no member or method named
_M_bbegin.:
#11 0x00007fec1370f048 in hbase::AsyncBatchRpcRetryingCaller::Send
(this=0x7feb9c0036c0, actions_by_server=std::unordered_map with 1 elements,
tries=1) at core/async-batch-rpc-retrying-caller.cc:343
#12 0x00007fec1370d819 in
hbase::AsyncBatchRpcRetryingCaller::<lambda(std::vector<folly::Try<std::shared_ptr<hbase::RegionLocation>
>, std::allocator<folly::Try<std::shared_ptr<hbase::RegionLocation> > >
>&)>::operator()(std::vector<folly::Try<std::shared_ptr<hbase::RegionLocation>
>, std::allocator<folly::Try<std::shared_ptr<hbase::RegionLocation> > > > &)
const (__closure=0x7feb9c001790,
loc=std::vector of length 100, capacity 100 = {...}) at
core/async-batch-rpc-retrying-caller.cc:283
#13 0x00007fec137112d0 in folly::detail::CoreCallbackState<folly::Unit,
hbase::AsyncBatchRpcRetryingCaller::GroupAndSend(const
std::vector<std::shared_ptr<hbase::Action> >&,
int32_t)::<lambda(std::vector<folly::Try<std::shared_ptr<hbase::RegionLocation>
>, std::allocator<folly::Try<std::shared_ptr<hbase::RegionLocation> > > >&)>
>::invoke<std::vector<folly::Try<std::shared_ptr<hbase::RegionLocation> >,
std::allocator<folly::Try<std::shared_ptr<hbase::RegionLocation> > >
>&>(std::vector<folly::Try<std::shared_ptr<hbase::RegionLocation> >,
std::allocator<folly::Try<std::shared_ptr<hbase::RegionLocation> > > > &)
(this=0x7feb9c001790, args#0=std::vector of length 100, capacity 100 = {...})
at /usr/local/include/folly/futures/Future-inl.h:85
#14 0x00007fec13711309 in
folly::Future<std::vector<folly::Try<std::shared_ptr<hbase::RegionLocation> >,
std::allocator<folly::Try<std::shared_ptr<hbase::RegionLocation> > > >
>::<lambda(folly::Try<std::vector<folly::Try<std::shared_ptr<hbase::RegionLocation>
>, std::allocator<folly::Try<std::shared_ptr<hbase::RegionLocation> > > >
>&&)>::<lambda()>::operator()(void) const (__closure=0x7febf1ffa1a0)
at /usr/local/include/folly/futures/Future-inl.h:243
#15 0x00007fec13712520 in
folly::makeTryWith<folly::Future<T>::thenImplementation(F&&,
folly::detail::argResult<isTry, F, Args ...>)::<lambda(folly::Try<T>&&)>
mutable [with F = hbase::AsyncBatchRpcRetryingCaller::GroupAndSend(const
std::vector<std::shared_ptr<hbase::Action> >&,
int32_t)::<lambda(std::vector<folly::Try<std::shared_ptr<hbase::RegionLocation>
> >&)>; R =
folly::detail::callableResult<std::vector<folly::Try<std::shared_ptr<hbase::RegionLocation>
> >, hbase::AsyncBatchRpcRetryingCaller::GroupAndSend(const
std::vector<std::shared_ptr<hbase::Action> >&,
int32_t)::<lambda(std::vector<folly::Try<std::shared_ptr<hbase::RegionLocation>
> >&)> >; bool isTry = false; Args =
{std::vector<folly::Try<std::shared_ptr<hbase::RegionLocation> >,
std::allocator<folly::Try<std::shared_ptr<hbase::RegionLocation> > > >&}; T =
std::vector<folly::Try<std::shared_ptr<hbase::RegionLocation> > >]::<lambda()>
>(<unknown type in /usr/local/lib/libHbaseClient_d.so, CU 0xb0d6f8, DIE
0xbd8453>) (
f=<unknown type in /usr/local/lib/libHbaseClient_d.so, CU 0xb0d6f8, DIE
0xbd8453>) at /usr/local/include/folly/Try-inl.h:165
#16 0x00007fec1371137b in
folly::Future<std::vector<folly::Try<std::shared_ptr<hbase::RegionLocation> >,
std::allocator<folly::Try<std::shared_ptr<hbase::RegionLocation> > > >
>::<lambda(folly::Try<std::vector<folly::Try<std::shared_ptr<hbase::RegionLocation>
>, std::allocator<folly::Try<std::shared_ptr<hbase::RegionLocation> > > >
>&&)>::operator()(<unknown type in /usr/local/lib/libHbaseClient_d.so, CU
0xb0d6f8, DIE 0xba6a18>) (__closure=0x7feb9c001790, t=<unknown type in
/usr/local/lib/libHbaseClient_d.so, CU 0xb0d6f8, DIE 0xba6a18>) at
/usr/local/include/folly/futures/Future-inl.h:242
#17 0x00007fec13713f57 in
folly::detail::function::FunctionTraits<void(folly::Try<std::vector<folly::Try<std::shared_ptr<hbase::RegionLocation>
>, std::allocator<folly::Try<std::shared_ptr<hbase::RegionLocation> > > >
>&&)>::callBig<folly::Future<T>::thenImplementation(F&&,
folly::detail::argResult<isTry, F, Args ...>) [with F =
hbase::AsyncBatchRpcRetryingCaller::GroupAndSend(const
std::vector<std::shared_ptr<hbase::Action> >&,
int32_t)::<lambda(std::vector<folly::Try<std::shared_ptr<hbase::RegionLocation>
> >&)>; R =
folly::detail::callableResult<std::vector<folly::Try<std::shared_ptr<hbase::RegionLocation>
> >, hbase::AsyncBatchRpcRetryingCaller::GroupAndSend(const
std::vector<std::shared_ptr<hbase::Action> >&,
int32_t)::<lambda(std::vector<folly::Try<std::shared_ptr<hbase::RegionLocation>
> >&)> >; bool isTry = false; Args =
{std::vector<folly::Try<std::shared_ptr<hbase::RegionLocation> >,
std::allocator<folly::Try<std::shared_ptr<hbase::RegionLocation> > > >&}; T =
std::vector<folly::Try<std::shared_ptr<hbase::RegionLocation> >
>]::<lambda(folly::Try<std::vector<folly::Try<std::shared_ptr<hbase::RegionLocation>
> > >&&)> >(folly::detail::function::Data &, <unknown type in
/usr/local/lib/libHbaseClient_d.so, CU 0xb0d6f8, DIE 0xba6a18>) (p=...,
args#0=<unknown type in /usr/local/lib/libHbaseClient_d.so, CU 0xb0d6f8, DIE
0xba6a18>) at /usr/local/include/folly/Function.h:305
#18 0x00007fec1372a0bc in folly::detail::function::FunctionTraits<void
(folly::Try<std::vector<folly::Try<std::shared_ptr<hbase::RegionLocation> >,
std::allocator<folly::Try<std::shared_ptr<hbase::RegionLocation> > > >
>&&)>::operator()(folly::Try<std::vector<folly::Try<std::shared_ptr<hbase::RegionLocation>
>, std::allocator<folly::Try<std::shared_ptr<hbase::RegionLocation> > > > >&&)
(
this=0x7feb9c019e90, args#0=<unknown type in
/usr/local/lib/libHbaseClient_d.so, CU 0xb0d6f8, DIE 0xba6a18>) at
/usr/local/include/folly/Function.h:314
#19 0x00007fec1372759d in
folly::detail::Core<std::vector<folly::Try<std::shared_ptr<hbase::RegionLocation>
>, std::allocator<folly::Try<std::shared_ptr<hbase::RegionLocation> > > >
>::doCallback (
this=0x7feb9c019e90) at /usr/local/include/folly/futures/detail/Core.h:388
#20 0x00007fec13724051 in
folly::detail::Core<std::vector<folly::Try<std::shared_ptr<hbase::RegionLocation>
>, std::allocator<folly::Try<std::shared_ptr<hbase::RegionLocation> > > >
>::maybeCallback()::{lambda()#2}::operator()() const (__closure=0x7febf1ffa4c0)
at /usr/local/include/folly/futures/detail/Core.h:317
#21 0x00007fec1372781e in
folly::detail::FSM<folly::detail::State>::updateState<folly::detail::Core<std::vector<folly::Try<std::shared_ptr<hbase::RegionLocation>
>, std::allocator<folly::Try<std::shared_ptr<hbase::RegionLocation> > > >
>::maybeCallback()::{lambda()#1},
folly::detail::Core<std::vector<folly::Try<std::shared_ptr<hbase::RegionLocation>
>, std::allocator<folly::Try<std::shared_ptr<hbase::RegionLocation> > > >
>::maybeCallback()::{lambda()#2}>(folly::detail::State, folly::detail::State,
folly::detail::Core<std::vector<folly::Try<std::shared_ptr<hbase::RegionLocation>
>, std::allocator<folly::Try<std::shared_ptr<hbase::RegionLocation> > > >
>::maybeCallback()::{lambda()#1} const&,
folly::detail::Core<std::vector<folly::Try<std::shared_ptr<hbase::RegionLocation>
>, std::allocator<folly::Try<std::shared_ptr<hbase::RegionLocation> > > >
>::maybeCallback()::{lambda()#2} const&) (this=0x7feb9c019ef8,
A=folly::detail::Armed, B=folly::detail::Done, protectedAction=...,
unprotectedAction=...) at /usr/local/include/folly/futures/detail/FSM.h:100
#22 0x00007fec137240cc in
folly::detail::Core<std::vector<folly::Try<std::shared_ptr<hbase::RegionLocation>
>, std::allocator<folly::Try<std::shared_ptr<hbase::RegionLocation> > > >
>::maybeCallback (
this=0x7feb9c019e90) at /usr/local/include/folly/futures/detail/Core.h:317
#23 0x00007fec1371319b in
folly::detail::Core<std::vector<folly::Try<std::shared_ptr<hbase::RegionLocation>
>, std::allocator<folly::Try<std::shared_ptr<hbase::RegionLocation> > > >
>::setCallback<folly::Future<T>::thenImplementation(F&&,
folly::detail::argResult<isTry, F, Args ...>) [with F =
hbase::AsyncBatchRpcRetryingCaller::GroupAndSend(const
std::vector<std::shared_ptr<hbase::Action> >&,
int32_t)::<lambda(std::vector<folly::Try<std::shared_ptr<hbase::RegionLocation>
> >&)>; R =
folly::detail::callableResult<std::vector<folly::Try<std::shared_ptr<hbase::RegionLocation>
> >, hbase::AsyncBatchRpcRetryingCaller::GroupAndSend(const
std::vector<std::shared_ptr<hbase::Action> >&,
int32_t)::<lambda(std::vector<folly::Try<std::shared_ptr<hbase::RegionLocation>
> >&)> >; bool isTry = false; Args =
{std::vector<folly::Try<std::shared_ptr<hbase::RegionLocation> >,
std::allocator<folly::Try<std::shared_ptr<hbase::RegionLocation> > > >&}; T =
std::vector<folly::Try<std::shared_ptr<hbase::RegionLocation> >
>]::<lambda(folly::Try<std::vector<folly::Try<std::shared_ptr<hbase::RegionLocation>
> > >&&)> >(<unknown type in /usr/local/lib/libHbaseClient_d.so, CU 0xb0d6f8,
DIE 0xbd000a>) (
this=0x7feb9c019e90, func=<unknown type in
/usr/local/lib/libHbaseClient_d.so, CU 0xb0d6f8, DIE 0xbd000a>) at
/usr/local/include/folly/futures/detail/Core.h:158
#24 0x00007fec137126d9 in
folly::Future<std::vector<folly::Try<std::shared_ptr<hbase::RegionLocation> >,
std::allocator<folly::Try<std::shared_ptr<hbase::RegionLocation> > > >
>::setCallback_<folly::Future<T>::thenImplementation(F&&,
folly::detail::argResult<isTry, F, Args ...>) [with F =
hbase::AsyncBatchRpcRetryingCaller::GroupAndSend(const
std::vector<std::shared_ptr<hbase::Action> >&,
int32_t)::<lambda(std::vector<folly::Try<std::shared_ptr<hbase::RegionLocation>
> >&)>; R =
folly::detail::callableResult<std::vector<folly::Try<std::shared_ptr<hbase::RegionLocation>
> >, hbase::AsyncBatchRpcRetryingCaller::GroupAndSend(const
std::vector<std::shared_ptr<hbase::Action> >&,
int32_t)::<lambda(std::vector<folly::Try<std::shared_ptr<hbase::RegionLocation>
> >&)> >; bool isTry = false; Args =
{std::vector<folly::Try<std::shared_ptr<hbase::RegionLocation> >,
std::allocator<folly::Try<std::shared_ptr<hbase::RegionLocation> > > >&}; T =
std::vector<folly::Try<std::shared_ptr<hbase::RegionLocation> >
>]::<lambda(folly::Try<std::vector<folly::Try<std::shared_ptr<hbase::RegionLocation>
> > >&&)> >(<unknown type in /usr/local/lib/libHbaseClient_d.so, CU 0xb0d6f8,
DIE 0xbd839c>) (this=0x7febf1ffa700,
func=<unknown type in /usr/local/lib/libHbaseClient_d.so, CU 0xb0d6f8, DIE
0xbd839c>) at /usr/local/include/folly/futures/Future-inl.h:172
#25 0x00007fec13711509 in
folly::Future<std::vector<folly::Try<std::shared_ptr<hbase::RegionLocation> >,
std::allocator<folly::Try<std::shared_ptr<hbase::RegionLocation> > > >
>::thenImplementation<hbase::AsyncBatchRpcRetryingCaller::GroupAndSend(const
std::vector<std::shared_ptr<hbase::Action> >&,
int32_t)::<lambda(std::vector<folly::Try<std::shared_ptr<hbase::RegionLocation>
> >&)>,
folly::detail::callableResult<std::vector<folly::Try<std::shared_ptr<hbase::RegionLocation>
> >, hbase::AsyncBatchRpcRetryingCaller::GroupAndSend(const
std::vector<std::shared_ptr<hbase::Action> >&,
int32_t)::<lambda(sred_ptr<hbase::Action> >&,
int32_t)::<lambda(std::vector<folly::Try<std::shared_ptr<hbase::RegionLocation>
>, std::allocator<folly::Try<std::shared_ptr<hbase::RegionLocation> > > >&)>,
std::vector<folly::Try<std::shared_ptr<hbase::RegionLocation> >,
std::allocator<folly::Try<std::shared_ptr<hbase::RegionLocation> > > >&>)
(this=0x7febf1ffa700,
func=<unknown type in /usr/local/lib/libHbaseClient_d.so, CU 0xb0d6f8, DIE
0xbe0d1b>) at /usr/local/include/folly/futures/Future-inl.h:236
#26 0x00007fec137109de in
folly::Future<std::vector<folly::Try<std::shared_ptr<hbase::RegionLocation> >,
std::allocator<folly::Try<std::shared_ptr<hbase::RegionLocation> > > >
>::then<hbase::AsyncBatchRpcRetryingCaller::GroupAndSend(const
std::vector<std::shared_ptr<hbase::Action> >&,
int32_t)::<lambda(std::vector<folly::Try<std::shared_ptr<hbase::RegionLocation>
> >&)> >(<unknown type in /usr/local/lib/libHbaseClient_d.so, CU 0xb0d6f8, DIE
0xbe840e>) (this=0x7febf1ffa700, func=<unknown type in
/usr/local/lib/libHbaseClient_d.so, CU 0xb0d6f8, DIE 0xbe840e>)
at /usr/local/include/folly/futures/Future.h:183
#27 0x00007fec1370df37 in hbase::AsyncBatchRpcRetryingCaller::GroupAndSend
(this=0x7feb9c0036c0, actions=std::vector of length 100, capacity 128 = {...},
tries=1)
at core/async-batch-rpc-retrying-caller.cc:242
#28 0x00007fec1370b328 in hbase::AsyncBatchRpcRetryingCaller::Call
(this=0x7feb9c0036c0) at core/async-batch-rpc-retrying-caller.cc:70
#29 0x00007fec13732c33 in hbase::RawAsyncTable::Batch (this=0x7feb9c000980,
gets=std::vector of length 100, capacity 128 = {...}) at
core/raw-async-table.cc:218
#30 0x00007fec137328a1 in hbase::RawAsyncTable::Get (this=0x7feb9c000980,
gets=std::vector of length 100, capacity 128 = {...}) at
core/raw-async-table.cc:202
#31 0x00007fec137036a9 in hbase::Table::Get (this=0x7feb9c0008c0,
gets=std::vector of length 100, capacity 128 = {...}) at core/table.cc:120
#32 0x000000000042e328 in DoGet (iteration=1, max_row=19999, rows=2000,
table=std::unique_ptr<hbase::Table> containing 0x7feb9c0008c0,
families=std::vector of length 1, capacity 1 = {...},
batch_num_rows=100) at core/load-client.cc:186
#33 0x000000000042faab in <lambda()>::operator()(void) const
(__closure=0xaa34f8) at core/load-client.cc:375
#34 0x0000000000432428 in std::_Bind_simple<main(int,
char**)::<lambda()>()>::_M_invoke<>(std::_Index_tuple<>) (this=0xaa34f8) at
/opt/rh/devtoolset-6/root/usr/include/c++/6.2.1/functional:1400
#35 0x0000000000432359 in std::_Bind_simple<main(int,
char**)::<lambda()>()>::operator()(void) (this=0xaa34f8) at
/opt/rh/devtoolset-6/root/usr/include/c++/6.2.1/functional:1389
#36 0x00000000004322d8 in std::thread::_State_impl<std::_Bind_simple<main(int,
char**)::<lambda()>()> >::_M_run(void) (this=0xaa34f0) at
/opt/rh/devtoolset-6/root/usr/include/c++/6.2.1/thread:196
#37 0x00007fec139c3aff in execute_native_thread_routine () from
/usr/local/lib/libHbaseClient_d.so
#38 0x00007fec0eea1dc5 in start_thread () from /lib64/libpthread.so.0
#39 0x00007fec0ebd073d in clone () from /lib64/libc.so.6
{code}
One thread has the lock, but waits for another lock:
{code}
Thread 11 (Thread 0x7febf17fa700 (LWP 6209)):
#0 0x00007fec0eea56d5 in pthread_cond_wait@@GLIBC_2.3.2 () from
/lib64/libpthread.so.0
#1 0x00007fec0f67f9ec in
std::condition_variable::wait(std::unique_lock<std::mutex>&) () from
/lib64/libstdc++.so.6
#2 0x00007fec123d056b in
wait<folly::EventBase::runInEventBaseThreadAndWait(folly::EventBase::FuncRef)::<lambda()>
> (__p=..., __lock=..., this=0x7febf17f7fd0)
at /opt/rh/devtoolset-4/root/usr/include/c++/5.2.1/condition_variable:98
#3 folly::EventBase::runInEventBaseThreadAndWait(folly::FunctionRef<void ()>)
(this=<optimized out>, fn=...) at io/async/EventBase.cpp:573
#4 0x00007fec135da453 in wangle::EventBaseHandler::write (this=0x7febd0001f20,
ctx=0x7febd0001f50, buf=std::unique_ptr<folly::IOBuf> containing 0x7febb001b048)
at /usr/local/include/wangle/channel/EventBaseHandler.h:25
#5 0x00007fec135f8eb4 in
wangle::OutboundContextImpl<wangle::EventBaseHandler>::write
(this=0x7febd0001f50, msg=std::unique_ptr<folly::IOBuf> containing 0x0)
at /usr/local/include/wangle/channel/HandlerContext-inl.h:477
#6 0x00007fec1361e18b in wangle::ContextImpl<hbase::SaslHandler>::fireWrite
(this=0x7febd00020c0, msg=std::unique_ptr<folly::IOBuf> containing 0x0)
at /usr/local/include/wangle/channel/HandlerContext-inl.h:207
#7 0x00007fec13615850 in hbase::SaslHandler::write (this=0x7febd0002040,
ctx=0x7febd00020c0, buf=std::unique_ptr<folly::IOBuf> containing 0x0) at
connection/sasl-handler.cc:119
#8 0x00007fec1361e89e in wangle::ContextImpl<hbase::SaslHandler>::write
(this=0x7febd00020c0, msg=std::unique_ptr<folly::IOBuf> containing 0x0)
at /usr/local/include/wangle/channel/HandlerContext-inl.h:290
#9 0x00007fec1361d0d7 in wangle::ContextImpl<hbase::ClientHandler>::fireWrite
(this=0x7febd000c2c0, msg=std::unique_ptr<folly::IOBuf> containing 0x0)
at /usr/local/include/wangle/channel/HandlerContext-inl.h:207
#10 0x00007fec13605cc8 in hbase::ClientHandler::write (this=0x7febd000c240,
ctx=0x7febd000c2c0, r=std::unique_ptr<hbase::Request> containing
0x7febb0012470) at connection/client-handler.cc:138
#11 0x00007fec1361d814 in wangle::ContextImpl<hbase::ClientHandler>::write
(this=0x7febd000c2c0, msg=std::unique_ptr<hbase::Request> containing 0x0)
at /usr/local/include/wangle/channel/HandlerContext-inl.h:290
#12 0x00007fec1362fb4d in
wangle::ContextImpl<wangle::ClientDispatcherBase<wangle::Pipeline<folly::IOBufQueue&,
std::unique_ptr<hbase::Request, std::default_delete<hbase::Request> > >,
std::unique_ptr<hbase::Request, std::default_delete<hbase::Request> >,
std::unique_ptr<hbase::Response, std::default_delete<hbase::Response> > >
>::fireWrite (this=0x7febec0010f0,
msg=std::unique_ptr<hbase::Request> containing 0x0) at
/usr/local/include/wangle/channel/HandlerContext-inl.h:207
#13 0x00007fec13614a1d in
wangle::HandlerAdapter<std::unique_ptr<hbase::Response,
std::default_delete<hbase::Response> >, std::unique_ptr<hbase::Request,
std::default_delete<hbase::Request> > >::write
(this=0x7febec001e00, ctx=0x7febec0010f0,
msg=std::unique_ptr<hbase::Request> containing 0x0) at
/usr/local/include/wangle/channel/Handler.h:162
#14 0x00007fec1363028a in
wangle::ContextImpl<wangle::ClientDispatcherBase<wangle::Pipeline<folly::IOBufQueue&,
std::unique_ptr<hbase::Request, std::default_delete<hbase::Request> > >,
std::unique_ptr<hbase::Request, std::default_delete<hbase::Request> >,
std::unique_ptr<hbase::Response, std::default_delete<hbase::Response> > >
>::write (this=0x7febec0010f0,
msg=std::unique_ptr<hbase::Request> containing 0x0) at
/usr/local/include/wangle/channel/HandlerContext-inl.h:290
#15 0x00007fec1360d403 in wangle::Pipeline<folly::IOBufQueue&,
std::unique_ptr<hbase::Request, std::default_delete<hbase::Request> >
>::write<std::unique_ptr<hbase::Request, std::default_delete<hbase::R#17
0x00007fec135ffb35 in hbase::RpcConnection::SendRequest (this=0x7febec001a40,
req=std::unique_ptr<hbase::Request> containing 0x0) at
./connection/rpc-connection.h:48
#18 0x00007fec135bfca0 in hbase::RpcClient::SendRequest (this=0xa6f890,
remote_id=std::shared_ptr (count 3, weak 0) 0x7febb001a050,
req=std::unique_ptr<hbase::Request> containing 0x0)
at connection/rpc-client.cc:85
#19 0x00007fec135bf85c in hbase::RpcClient::AsyncCall (this=0xa6f890,
host="localhost", port=42985, req=std::unique_ptr<hbase::Request> containing
0x0,
ticket=std::shared_ptr (count 3, weak 0) 0x7febb0012330,
service_name="ClientService") at connection/rpc-client.cc:71
Python Exception <class 'gdb.error'> There is no member or method named
_M_bbegin.:
#20 0x00007fec1370e1b1 in hbase::AsyncBatchRpcRetryingCaller::GetMultiResponse
(this=0x7febb0026820, actions_by_server=std::unordered_map with 1 elements) at
core/async-batch-rpc-retrying-caller.cc:313
Python Exception <class 'gdb.error'> There is no member or method named
_M_bbegin.:
#21 0x00007fec1370f048 in hbase::AsyncBatchRpcRetryingCaller::Send
(this=0x7febb0026820, actions_by_server=std::unordered_map with 1 elements,
tries=1) at core/async-batch-rpc-retrying-caller.cc:343
#22 0x00007fec1370d819 in
hbase::AsyncBatchRpcRetryingCaller::<lambda(std::vector<folly::Try<std::shared_ptr<hbase::RegionLocation>
>, std::allocator<folly::Try<std::shared_ptr<hbase::RegionLocation> > >
>&)>::operator()(std::vector<folly::Try<std::shared_ptr<hbase::RegionLocation>
>, std::allocator<folly::Try<std::shared_ptr<hbase::RegionLocation> > > > &)
const (__closure=0x7febb00027b0,
loc=std::vector of length 100, capacity 100 = {...}) at
core/async-batch-rpc-retrying-caller.cc:283
#23 0x00007fec137112d0 in folly::detail::CoreCallbackState<folly::Unit,
hbase::AsyncBatchRpcRetryingCaller::GroupAndSend(const
std::vector<std::shared_ptr<hbase::Action> >&,
int32_t)::<lambda(std::vector<folly::Try<std::shared_ptr<hbase::RegionLocation>
>, std::allocator<folly::Try<std::shared_ptr<hbase::RegionLocation> > > >&)>
>::invoke<std::vector<folly::Try<std::shared_ptr<hbase::RegionLocation> >,
std::allocator<folly::Try<std::shared_ptr<hbase::RegionLocation> > >
>&>(std::vector<folly::Try<std::shared_ptr<hbase::RegionLocation> >,
std::allocator<folly::Try<std::shared_ptr<hbase::RegionLocation> > > > &)
(this=0x7febb00027b0, args#0=std::vector of length 100, capacity 100 = {...})
at /usr/local/include/folly/futures/Future-inl.h:85
#24 0x00007fec13711309 in
folly::Future<std::vector<folly::Try<std::shared_ptr<hbase::RegionLocation> >,
std::allocator<folly::Try<std::shared_ptr<hbase::RegionLocation> > > >
>::<lambda(folly::Try<std::vector<folly::Try<std::shared_ptr<hbase::RegionLocation>
>, std::allocator<folly::Try<std::shared_ptr<hbase::RegionLocation> > > >
>&&)>::<lambda()>::operator()(void) const (__closure=0x7febf17f91a0)
at /usr/local/include/folly/futures/Future-inl.h:243
#25 0x00007fec13712520 in
folly::makeTryWith<folly::Future<T>::thenImplementation(F&&,
folly::detail::argResult<isTry, F, Args ...>)::<lambda(folly::Try<T>&&)>
mutable [with F = hbase::AsyncBatchRpcRetryingCaller::GroupAndSend(const
std::vector<std::shared_ptr<hbase::Action> >&,
int32_t)::<lambda(std::vector<folly::Try<std::shared_ptr<hbase::RegionLocation>
> >&)>; R =
folly::detail::callableResult<std::vector<folly::Try<std::shared_ptr<hbase::RegionLocation>
> >, hbase::AsyncBatchRpcRetryingCaller::GroupAndSend(const
std::vector<std::shared_ptr<hbase::Action> >&,
int32_t)::<lambda(std::vector<folly::Try<std::shared_ptr<hbase::RegionLocation>
> >&)> >; bool isTry = false; Args =
{std::vector<folly::Try<std::shared_ptr<hbase::RegionLocation> >,
std::allocator<folly::Try<std::shared_ptr<hbase::RegionLocation> > > >&}; T =
std::vector<folly::Try<std::shared_ptr<hbase::RegionLocation> > >]::<lambda()>
>(<unknown type in /usr/local/lib/libHbaseClient_d.so, CU 0xb0d6f8, DIE
0xbd8453>) (
f=<unknown type in /usr/local/lib/libHbaseClient_d.so, CU 0xb0d6f8, DIE
0xbd8453>) at /usr/local/include/folly/Try-inl.h:165
#26 0x00007fec1371137b in
folly::Future<std::vector<folly::Try<std::shared_ptr<hbase::RegionLocation> >,
std::allocator<folly::Try<std::shared_ptr<hbase::RegionLocation> > > >
>::<lambda(folly::Try<std::vector<folly::Try<std::shared_ptr<hbase::RegionLocation>
>, std::allocator<folly::Try<std::shared_ptr<hbase::RegionLocation> > > >
>&&)>::operator()(<unknown type in /usr/local/lib/libHbaseClient_d.so, CU
0xb0d6f8, DIE 0xba6a18>) (__closure=0x7febb00027b0, t=<unknown type in
/usr/local/lib/libHbaseClient_d.so, CU 0xb0d6f8, DIE 0xba6a18>) at
/usr/local/include/folly/futures/Future-inl.h:242
#27 0x00007fec13713f57 in
folly::detail::function::FunctionTraits<void(folly::Try<std::vector<folly::Try<std::shared_ptr<hbase::RegionLocation>
>, std::allocator<folly::Try<std::shared_ptr<hbase::RegionLocation> > > >
>&&)>::callBig<folly::Future<T>::thenImplementation(F&&,
folly::detail::argResult<isTry, F, Args ...>) [with F =
hbase::AsyncBatchRpcRetryingCaller::GroupAndSend(const
std::vector<std::shared_ptr<hbase::Action> >&,
int32_t)::<lambda(std::vector<folly::Try<std::shared_ptr<hbase::RegionLocation>
> >&)>; R =
folly::detail::callableResult<std::vector<folly::Try<std::shared_ptr<hbase::RegionLocation>
> >, hbase::AsyncBatchRpcRetryingCaller::GroupAndSend(const
std::vector<std::shared_ptr<hbase::Action> >&,
int32_t)::<lambda(std::vector<folly::Try<std::shared_ptr<hbase::RegionLocation>
> >&)> >; bool isTry = false; Args =
{std::vector<folly::Try<std::shared_ptr<hbase::RegionLocation> >,
std::allocator<folly::Try<std::shared_ptr<hbase::RegionLocation> > > >&}; T =
std::vector<folly::Try<std::shared_ptr<hbase::RegionLocation> >
>]::<lambda(folly::Try<std::vector<folly::Try<std::shared_ptr<hbase::RegionLocation>
> > >&&)> >(folly::detail::function::Data &, <unknown type in
/usr/local/lib/libHbaseClient_d.so, CU 0xb0d6f8, DIE 0xba6a18>) (p=...,
args#0=<unknown type in /usr/local/lib/libHbaseClient_d.so, CU 0xb0d6f8, DIE
0xba6a18>) at /usr/local/include/folly/Function.h:305
#28 0x00007fec1372a0bc in folly::detail::function::FunctionTraits<void
(folly::Try<std::vector<folly::Try<std::shared_ptr<hbase::RegionLocation> >,
std::allocator<folly::Try<std::shared_ptr<hbase::RegionLocation> > > >
>&&)>::operator()(folly::Try<std::vector<folly::Try<std::shared_ptr<hbase::RegionLocation>
>, std::allocator<folly::Try<std::shared_ptr<hbase::RegionLocation> > > > >&&)
(
this=0x7febb0024af0, args#0=<unknown type in
/usr/local/lib/libHbaseClient_d.so, CU 0xb0d6f8, DIE 0xba6a18>) at
/usr/local/include/folly/Function.h:314
#29 0x00007fec1372759d in
folly::detail::Core<std::vector<folly::Try<std::shared_ptr<hbase::RegionLocation>
>, std::allocator<folly::Try<std::shared_ptr<hbase::RegionLocation> > > >
>::doCallback (
this=0x7febb0024af0) at /usr/local/include/folly/futures/detail/Core.h:388
#30 0x00007fec13724051 in
folly::detail::Core<std::vector<folly::Try<std::shared_ptr<hbase::RegionLocation>
>, std::allocator<folly::Try<std::shared_ptr<hbase::RegionLocation> > > >
>::maybeCallback()::{lambda()#2}::operator()() const (__closure=0x7febf17f94c0)
at /usr/local/include/folly/futures/detail/Core.h:317
#31 0x00007fec1372781e in
folly::detail::FSM<folly::detail::State>::updateState<folly::detail::Core<std::vector<folly::Try<std::shared_ptr<hbase::RegionLocation>
>, std::allocator<folly::Try<std::shared_ptr<hbase::RegionLocation> > > >
>::maybeCallback()::{lambda()#1},
folly::detail::Core<std::vector<folly::Try<std::shared_ptr<hbase::RegionLocation>
>, std::allocator<folly::Try<std::shared_ptr<hbase::RegionLocation> > > >
>::maybeCallback()::{lambda()#2}>(folly::detail::State, folly::detail::State,
folly::detail::Core<std::vector<folly::Try<std::shared_ptr<hbase::RegionLocation>
>, std::allocator<folly::Try<std::shared_ptr<hbase::RegionLocation> > > >
>::maybeCallback()::{lambda()#1} const&,
folly::detail::Core<std::vector<folly::Try<std::shared_ptr<hbase::RegionLocation>
>, std::allocator<folly::Try<std::shared_ptr<hbase::RegionLocation> > > >
>::maybeCallback()::{lambda()#2} const&) (this=0x7febb0024b58,
A=folly::detail::Armed, B=folly::detail::Done, protectedAction=...,
unprotectedAction=...) at /usr/local/include/folly/futures/detail/FSM.h:100
#32 0x00007fec137240cc in
folly::detail::Core<std::vector<folly::Try<std::shared_ptr<hbase::RegionLocation>
>, std::allocator<folly::Try<std::shared_ptr<hbase::RegionLocation> > > >
>::maybeCallback (
this=0x7febb0024af0) at /usr/local/include/folly/futures/detail/Core.h:317
#33 0x00007fec1371319b in
folly::detail::Core<std::vector<folly::Try<std::shared_ptr<hbase::RegionLocation>
>, std::allocator<folly::Try<std::shared_ptr<hbase::RegionLocation> > > >
>::setCallback<folly::Future<T>::thenImplementation(F&&,
folly::detail::argResult<isTry, F, Args ...>) [with F =
hbase::AsyncBatchRpcRetryingCaller::GroupAndSend(const
std::vector<std::shared_ptr<hbase::Action> >&,
int32_t)::<lambda(std::vector<folly::Try<std::shared_ptr<hbase::RegionLocation>
> >&)>; R =
folly::detail::callableResult<std::vector<folly::Try<std::shared_ptr<hbase::RegionLocation>
> >, hbase::AsyncBatchRpcRetryingCaller::GroupAndSend(const
std::vector<std::shared_ptr<hbase::Action> >&,
int32_t)::<lambda(std::vector<folly::Try<std::shared_ptr<hbase::RegionLocation>
> >&)> >; bool isTry = false; Args =
{std::vector<folly::Try<std::shared_ptr<hbase::RegionLocation> >,
std::allocator<folly::Try<std::shared_ptr<hbase::RegionLocation> > > >&}; T =
std::vector<folly::Try<std::shared_ptr<hbase::RegionLocation>#34
0x00007fec137126d9 in
folly::Future<std::vector<folly::Try<std::shared_ptr<hbase::RegionLocation> >,
std::allocator<folly::Try<std::shared_ptr<hbase::RegionLocation> > > >
>::setCallback_<folly::Future<T>::thenImplementation(F&&,
folly::detail::argResult<isTry, F, Args ...>) [with F =
hbase::AsyncBatchRpcRetryingCaller::GroupAndSend(const
std::vector<std::shared_ptr<hbase::Action> >&,
int32_t)::<lambda(std::vector<folly::Try<std::shared_ptr<hbase::RegionLocation>
> >&)>; R =
folly::detail::callableResult<std::vector<folly::Try<std::shared_ptr<hbase::RegionLocation>
> >, hbase::AsyncBatchRpcRetryingCaller::GroupAndSend(const
std::vector<std::shared_ptr<hbase::Action> >&,
int32_t)::<lambda(std::vector<folly::Try<std::shared_ptr<hbase::RegionLocation>
> >&)> >; bool isTry = false; Args =
{std::vector<folly::Try<std::shared_ptr<hbase::RegionLocation> >,
std::allocator<folly::Try<std::shared_ptr<hbase::RegionLocation> > > >&}; T =
std::vector<folly::Try<std::shared_ptr<hbase::RegionLocation> >
>]::<lambda(folly::Try<std::vector<folly::Try<std::shared_ptr<hbase::RegionLocation>
> > >&&)> >(<unknown type in /usr/local/lib/libHbaseClient_d.so, CU 0xb0d6f8,
DIE 0xbd839c>) (this=0x7febf17f9700,
func=<unknown type in /usr/local/lib/libHbaseClient_d.so, CU 0xb0d6f8, DIE
0xbd839c>) at /usr/local/include/folly/futures/Future-inl.h:172
#35 0x00007fec13711509 in
folly::Future<std::vector<folly::Try<std::shared_ptr<hbase::RegionLocation> >,
std::allocator<folly::Try<std::shared_ptr<hbase::RegionLocation> > > >
>::thenImplementation<hbase::AsyncBatchRpcRetryingCaller::GroupAndSend(const
std::vector<std::shared_ptr<hbase::Action> >&,
int32_t)::<lambda(std::vector<folly::Try<std::shared_ptr<hbase::RegionLocation>
> >&)>,
folly::detail::callableResult<std::vector<folly::Try<std::shared_ptr<hbase::RegionLocation>
> >, hbase::AsyncBatchRpcRetryingCaller::GroupAndSend(const
std::vector<std::shared_ptr<hbase::Action> >&,
int32_t)::<lambda(std::vector<folly::Try<std::shared_ptr<hbase::RegionLocation>
> >&)> >, false, std::vector<folly::Try<std::shared_ptr<hbase::RegionLocation>
>, std::allocator<folly::Try<std::shared_ptr<hbase::RegionLocation> > >
>&>(<unknown type in /usr/local/lib/libHbaseClient_d.so, CU 0xb0d6f8, DIE
0xbe0d1b>, folly::detail::argResult<false,
hbase::AsyncBatchRpcRetryingCaller::GroupAndSend(const
std::vector<std::shared_ptr<hbase::Action> >&,
int32_t)::<lambda(std::vector<folly::Try<std::shared_ptr<hbase::RegionLocation>
>, std::allocator<folly::Try<std::shared_ptr<hbase::RegionLocation> > > >&)>,
std::vector<folly::Try<std::shared_ptr<hbase::RegionLocation> >,
std::allocator<folly::Try<std::shared_ptr<hbase::RegionLocation> > > >&>)
(this=0x7febf17f9700,
func=<unknown type in /usr/local/lib/libHbaseClient_d.so, CU 0xb0d6f8, DIE
0xbe0d1b>) at /usr/local/include/folly/futures/Future-inl.h:236
#36 0x00007fec137109de in
folly::Future<std::vector<folly::Try<std::shared_ptr<hbase::RegionLocation> >,
std::allocator<folly::Try<std::shared_ptr<hbase::RegionLocation> > > >
>::then<hbase::AsyncBatchRpcRetryingCaller::GroupAndSend(const
std::vector<std::shared_ptr<hbase::Action> >&,
int32_t)::<lambda(std::vector<folly::Try<std::shared_ptr<hbase::RegionLocation>
> >&)> >(<unknown type in /usr/local/lib/libHbaseClient_d.so, CU 0xb0d6f8, DIE
0xbe840e>) (this=0x7febf17f9700, func=<unknown type in
/usr/local/lib/libHbaseClient_d.so, CU 0xb0d6f8, DIE 0xbe840e>)
at /usr/local/include/folly/futures/Future.h:183
#37 0x00007fec1370df37 in hbase::AsyncBatchRpcRetryingCaller::GroupAndSend
(this=0x7febb0026820, actions=std::vector of length 100, capacity 128 = {...},
tries=1)
at core/async-batch-rpc-retrying-caller.cc:242
#38 0x00007fec1370b328 in hbase::AsyncBatchRpcRetryingCaller::Call
(this=0x7febb0026820) at core/async-batch-rpc-retrying-caller.cc:70
#39 0x00007fec13732c33 in hbase::RawAsyncTable::Batch (this=0x7febb0000980,
gets=std::vector of length 100, capacity 128 = {...}) at
core/raw-async-table.cc:218
#40 0x00007fec137328a1 in hbase::RawAsyncTable::Get (this=0x7febb0000980,
gets=std::vector of length 100, capacity 128 = {...}) at
core/raw-async-table.cc:202
#41 0x00007fec137036a9 in hbase::Table::Get (this=0x7febb00008c0,
gets=std::vector of length 100, capacity 128 = {...}) at core/table.cc:120
#42 0x000000000042e328 in DoGet (iteration=0, max_row=19999, rows=2000,
table=std::unique_ptr<hbase::Table> containing 0x7febb00008c0,
families=std::vector of length 1, capacity 1 = {...},
batch_num_rows=100) at core/load-client.cc:186
#43 0x000000000042faab in <lambda()>::operator()(void) const
(__closure=0xaa37d8) at core/load-client.cc:375
#44 0x0000000000432428 in std::_Bind_simple<main(int,
char**)::<lambda()>()>::_M_invoke<>(std::_Index_tuple<>) (this=0xaa37d8) at
/opt/rh/devtoolset-6/root/usr/include/c++/6.2.1/functional:1400
#45 0x0000000000432359 in std::_Bind_simple<main(int,
char**)::<lambda()>()>::operator()(void) (this=0xaa37d8) at
/opt/rh/devtoolset-6/root/usr/include/c++/6.2.1/functional:1389
#46 0x00000000004322d8 in std::thread::_State_impl<std::_Bind_simple<main(int,
char**)::<lambda()>()> >::_M_run(void) (this=0xaa37d0) at
/opt/rh/devtoolset-6/root/usr/include/c++/6.2.1/thread:196
#47 0x00007fec139c3aff in execute_native_thread_routine () from
/usr/local/lib/libHbaseClient_d.so
#48 0x00007fec0eea1dc5 in start_thread () from /lib64/libpthread.so.0
#49 0x00007fec0ebd073d in clone () from /lib64/libc.so.6
{code}
and yet some other thread from the scanner has those locks in reverse order:
{code}
Thread 17 (Thread 0x7fec017fa700 (LWP 6183)):
#0 0x00007fec0eea81bd in __lll_lock_wait () from /lib64/libpthread.so.0
#1 0x00007fec0eea3d1d in _L_lock_840 () from /lib64/libpthread.so.0
#2 0x00007fec0eea3c3a in pthread_mutex_lock () from /lib64/libpthread.so.0
#3 0x00007fec135fe083 in __gthread_mutex_lock (__mutex=0x7febec001a58) at
/opt/rh/devtoolset-6/root/usr/include/c++/6.2.1/x86_64-redhat-linux/bits/gthr-default.h:748
#4 0x00007fec135fe0d3 in __gthread_recursive_mutex_lock
(__mutex=0x7febec001a58) at
/opt/rh/devtoolset-6/root/usr/include/c++/6.2.1/x86_64-redhat-linux/bits/gthr-default.h:810
#5 0x00007fec135feb56 in std::recursive_mutex::lock (this=0x7febec001a58) at
/opt/rh/devtoolset-6/root/usr/include/c++/6.2.1/mutex:105
#6 0x00007fec136005c6 in std::lock_guard<std::recursive_mutex>::lock_guard
(this=0x7fec017f7f10, __m=...) at
/opt/rh/devtoolset-6/root/usr/include/c++/6.2.1/bits/std_mutex.h:162
#7 0x00007fec135ff9e0 in hbase::RpcConnection::SendRequest
(this=0x7febec001a40, req=std::unique_ptr<hbase::Request> containing
0x7febd16ae450) at ./connection/rpc-connection.h:43
#8 0x00007fec135bfca0 in hbase::RpcClient::SendRequest (this=0xa6f890,
remote_id=std::shared_ptr (count 3, weak 0) 0x7febd16ae390,
req=std::unique_ptr<hbase::Request> containing 0x0)
at connection/rpc-client.cc:85
#9 0x00007fec135bf85c in hbase::RpcClient::AsyncCall (this=0xa6f890,
host="localhost", port=42985, req=std::unique_ptr<hbase::Request> containing
0x0,
ticket=std::shared_ptr (count 2, weak 0) 0x7febd0af9fc0,
service_name="ClientService") at connection/rpc-client.cc:71
#10 0x00007fec136cd6ca in hbase::AsyncScanRpcRetryingCaller::Call
(this=0x7febd00011a0) at core/async-scan-rpc-retrying-caller.cc:406
#11 0x00007fec136cd17d in hbase::AsyncScanRpcRetryingCaller::Next
(this=0x7febd00011a0) at core/async-scan-rpc-retrying-caller.cc:376
#12 0x00007fec136cba78 in hbase::AsyncScanRpcRetryingCaller::CompleteOrNext
(this=0x7febd00011a0, resp=std::shared_ptr (count 5, weak 0) 0x7febd0247930) at
core/async-scan-rpc-retrying-caller.cc:241
#13 0x00007fec136cb60f in hbase::AsyncScanRpcRetryingCaller::OnComplete
(this=0x7febd00011a0, controller=std::shared_ptr (count 2, weak 0)
0x7febd0214ae0,
resp=std::shared_ptr (count 5, weak 0) 0x7febd0247930,
cell_scanner=std::shared_ptr (count 2, weak 0) 0x7febd023e690) at
core/async-scan-rpc-retrying-caller.cc:222
#14 0x00007fec136cd269 in hbase::AsyncScanRpcRetryingCaller::<lambda(const
std::unique_ptr<hbase::Response, std::default_delete<hbase::Response>
>&)>::operator()(const std::unique_ptr<hbase::Response,
std::default_delete<hbase::Response> > &) const (__closure=0x7febd0245890,
resp=std::unique_ptr<hbase::Response> containing 0x7febd0b09650) at
core/async-scan-rpc-retrying-caller.cc:411
#15 0x00007fec136cec8e in folly::detail::CoreCallbackState<folly::Unit,
hbase::AsyncScanRpcRetryingCaller::Call()::<lambda(const
std::unique_ptr<hbase::Response, std::default_delete<hbase::Response> >&)>
>::invoke<std::unique_ptr<hbase::Response, std::default_delete<hbase::Response>
> >(<unknown type in /usr/local/lib/libHbaseClient_d.so, CU 0x86df39, DIE
0x8d2aaa>) (this=0x7febd0245890,
args#0=<unknown type in /usr/local/lib/libHbaseClient_d.so, CU 0x86df39,
DIE 0x8d2aaa>) at /usr/local/include/folly/futures/Future-inl.h:85
#16 0x00007fec136cecc7 in folly::Future<std::unique_ptr<hbase::Response,
std::default_delete<hbase::Response> >
>::<lambda(folly::Try<std::unique_ptr<hbase::Response,
std::default_delete<hbase::Response> > >&&)>::<lambda()>::operator()(void)
const (__closure=0x7fec017f8730) at
/usr/local/include/folly/futures/Future-inl.h:243
#17 0x00007fec136cfa96 in
folly::makeTryWith<folly::Future<T>::thenImplementation(F&&,
folly::detail::argResult<isTry, F, Args ...>)::<lambda(folly::Try<T>&&)>
mutable [with F = hbase::AsyncScanRpcRetryingCaller::Call()::<lambda(const
std::unique_ptr<hbase::Response>&)>; R =
folly::detail::callableResult<std::unique_ptr<hbase::Response>,
hbase::AsyncScanRpcRetryingCaller::Call()::<lambda(const
std::unique_ptr<hbase::Response>&)> >; bool isTry = false; Args =
{std::unique_ptr<hbase::Response, std::default_delete<hbase::Response> >&&}; T
= std::unique_ptr<hbase::Response>]::<lambda()> >(<unknown type in
/usr/local/lib/libHbaseClient_d.so, CU 0x86df39, DIE 0x8e66f0>) (f=<unknown
type in /usr/local/lib/libHbaseClient_d.so, CU 0x86df39, DIE 0x8e66f0>) at
/usr/local/include/folly/Try-inl.h:165
#18 0x00007fec136ced39 in folly::Future<std::unique_ptr<hbase::Response,
std::default_delete<hbase::Response> >
>::<lambda(folly::Try<std::unique_ptr<hbase::Response,
std::default_delete<hbase::Response> > >&&)>::operator()(<unknown type in
/usr/local/lib/libHbaseClient_d.so, CU 0x86df39, DIE 0x8d3c22>)
(__closure=0x7febd0245890,
t=<unknown type in /usr/local/lib/libHbaseClient_d.so, CU 0x86df39, DIE
0x8d3c22>) at /usr/local/include/folly/futures/Future-inl.h:242
#19 0x00007fec136d0bd6 in
folly::detail::function::FunctionTraits<void(folly::Try<std::unique_ptr<hbase::Response,
std::default_delete<hbase::Response> >
>&&)>::callSmall<folly::Future<T>::thenImplementation(F&&,
folly::detail::argResult<isTry, F, Args ...>) [with F =
hbase::AsyncScanRpcRetryingCaller::Call()::<lambda(const
std::unique_ptr<hbase::Response>&)>; R =
folly::detail::callableResult<std::unique_ptr<hbase::Response>,
hbase::AsyncScanRpcRetryingCaller::Call()::<lambda(const
std::unique_ptr<hbase::Response>&)> >; bool isTry = false; Args =
{std::unique_ptr<hbase::Response, std::default_delete<hbase::Response> >&&}; T
=
std::unique_ptr<hbase::Response>]::<lambda(folly::Try<std::unique_ptr<hbase::Response>
>&&)> >(folly::detail::function::Data &, <unknown type in
/usr/local/lib/libHbaseClient_d.so, CU 0x86df39, DIE 0x8d3c22>) (p=...,
args#0=<unknown type in /usr/local/lib/libHbaseClient_d.so, CU 0x86df39, DIE
0x8d3c22>) at /usr/local/include/folly/Function.h:299
#20 0x00007fec135cc402 in folly::detail::function::FunctionTraits<void
(folly::Try<std::unique_ptr<hbase::Response,
std::default_delete<hbase::Response> >
>&&)>::operator()(folly::Try<std::unique_ptr<hbase::Response,
std::default_delete<hbase::Response> > >&&) (this=0x7febd0245890,
args#0=<unknown type in /usr/local/lib/libHbaseClient_d.so, CU 0x45953, DIE
0xa1de0>)
at /usr/local/include/folly/Function.h:314
#21 0x00007fec135cab03 in folly::detail::Core<std::unique_ptr<hbase::Response,
std::default_delete<hbase::Response> > >::doCallback (this=0x7febd0245890)
at /usr/local/include/folly/futures/detail/Core.h:388
#22 0x00007fec135c898b in folly::detail::Core<std::unique_ptr<hbase::Response,
std::default_delete<hbase::Response> >
>::maybeCallback()::{lambda()#2}::operator()() const (__closure=0x7fec017f8a50)
at /usr/local/include/folly/futures/detail/Core.h:317
#23 0x00007fec135cad84 in
folly::detail::FSM<folly::detail::State>::updateState<folly::detail::Core<std::unique_ptr<hbase::Response,
std::default_delete<hbase::Response> > >::maybeCallback()::{lambda()#1},
folly::detail::Core<std::unique_ptr<hbase::Response,
std::default_delete<hbase::Response> >
>::maybeCallback()::{lambda()#2}>(folly::detail::State, folly::detail::State,
folly::detail::Core<std::unique_ptr<hbase::Response,
std::default_delete<hbase::Response> > >::maybeCallback()::{lambda()#1} const&,
folly::detail::Core<std::unique_ptr<hbase::Response,
std::default_delete<hbase::Response> > >::maybeCallback()::{lambda()#2} const&)
(this=0x7febd02458f8, A=folly::detail::Armed, B=folly::detail::Done,
protectedAction=..., unprotectedAction=...) at
/usr/local/include/folly/futures/detail/FSM.h:100
#24 0x00007fec135c8a06 in folly::detail::Core<std::unique_ptr<hbase::Response,
std::default_delete<hbase::Response> > >::maybeCallback (this=0x7febd0245890)
at /usr/local/include/folly/futures/detail/Core.h:317
#25 0x00007fec135c6f44 in folly::detail::Core<std::unique_ptr<hbase::Response,
std::default_delete<hbase::Response> >
>::setResult(folly::Try<std::unique_ptr<hbase::Response,
std::default_delete<hbase::Response> > >&&) (this=0x7febd0245890, t=<unknown
type in /usr/local/lib/libHbaseClient_d.so, CU 0x45953, DIE 0xa1de0>) at
/usr/local/include/folly/futures/detail/Core.h:183
#26 0x00007fec135c68c0 in folly::Promise<std::unique_ptr<hbase::Response,
std::default_delete<hbase::Response> >
>::setTry(folly::Try<std::unique_ptr<hbase::Response,
std::default_delete<hbase::Response> > >&&) (this=0x7fec017f8af0, t=<unknown
type in /usr/local/lib/libHbaseClient_d.so, CU 0x45953, DIE 0xa1de0>) at
/usr/local/include/folly/futures/Promise-inl.h:121
#27 0x00007fec135c0c4c in
folly::detail::CoreCallbackState<std::unique_ptr<hbase::Response,
std::default_delete<hbase::Response> >,
hbase::RpcClient::SendRequest(std::shared_ptr<hbase::ConnectionId>,
std::unique_ptr<hbase::Request>)::<lambda(const folly::exception_wrapper&)>
>::setTry(<unknown type in /usr/local/lib/libHbaseClient_d.so, CU 0x45953, DIE
0xa1de0>) (this=0x7febd0af2a90,
t=<unknown type in /usr/local/lib/libHbaseClient_d.so, CU 0x45953, DIE
0xa1de0>) at /usr/local/include/folly/futures/Future-inl.h:94
#28 0x00007fec135c05a0 in folly::Future<std::unique_ptr<hbase::Response,
std::default_delete<hbase::Response> >
>::<lambda(folly::Try<std::unique_ptr<hbase::Response,
std::default_delete<hbase::Response> >
>)>::operator()(folly::Try<std::unique_ptr<hbase::Response,
std::default_delete<hbase::Response> > >) (__closure=0x7febd0af2a90, t=...) at
/usr/local/include/folly/futures/Future-inl.h:424
#29 0x00007fec135c1953 in
folly::detail::function::FunctionTraits<void(folly::Try<std::unique_ptr<hbase::Response,
std::default_delete<hbase::Response> >
>&&)>::callSmall<folly::Future<T>::onError(F&&) [with F =
hbase::RpcClient::SendRequest(std::shared_ptr<hbase::ConnectionId>,
std::unique_ptr<hbase::Request>)::<lambda(const folly::exception_wrapper&)>; T
=
std::unique_ptr<hbase::Response>]::<lambda(folly::Try<std::unique_ptr<hbase::Response>
>)> >(folly::detail::function::Data &, <unknown type in
/usr/local/lib/libHbaseClient_d.so, CU 0x45953, DIE 0xa1de0>) (p=...,
args#0=<unknown type in /usr/local/lib/libHbaseClient_d.so, CU 0x45953, DIE
0xa1de0>) at /usr/local/include/folly/Function.h:299
#30 0x00007fec135cc402 in folly::detail::function::FunctionTraits<void
(folly::Try<std::unique_ptr<hbase::Response,
std::default_delete<hbase::Response> >
>&&)>::operator()(folly::Try<std::unique_ptr<hbase::Response,
std::default_delete<hbase::Response> > >&&) (this=0x7febd0af2a90,
args#0=<unknown type in /usr/local/lib/libHbaseClient_d.so, CU 0x45953, DIE
0xa1de0>)
at /usr/local/include/folly/Function.h:314
#31 0x00007fec135cab03 in folly::detail::Core<std::unique_ptr<hbase::Response,
std::default_delete<hbase::Response> > >::doCallback (this=0x7febd0af2a90)
at /usr/local/include/folly/futures/detail/Core.h:388
#32 0x00007fec135c898b in folly::detail::Core<std::unique_ptr<hbase::Response,
std::default_delete<hbase::Response> >
>::maybeCallback()::{lambda()#2}::operator()() const (__closure=0x7fec017f8ea0)
at /usr/local/include/folly/futures/detail/Core.h:317
#33 0x00007fec135cad84 in
folly::detail::FSM<folly::detail::State>::updateState<folly::detail::Core<std::unique_ptr<hbase::Response,
std::default_delete<hbase::Response> > >::maybeCallback()::{lambda()#1},
folly::detail::Core<std::unique_ptr<hbase::Response,
std::default_delete<hbase::Response> >
>::maybeCallback()::{lambda()#2}>(folly::detail::State, folly::detail::State,
folly::detail::Core<std::unique_ptr<hbase::Response,
std::default_delete<hbase::Response> > >::maybeCallback()::{lambda()#1} const&,
folly::detail::Core<std::unique_ptr<hbase::Response,
std::default_delete<hbase::Response> > >::maybeCallback()::{lambda()#2} const&)
(this=0x7febd0af2af8, A=folly::detail::Armed, B=folly::detail::Done,
protectedAction=..., unprotectedAction=...) at
/usr/local/include/folly/futures/detail/FSM.h:100
#34 0x00007fec135c8a06 in folly::detail::Core<std::unique_ptr<hbase::Response,
std::default_delete<hbase::Response> > >::maybeCallback (this=0x7febd0af2a90)
at /usr/local/include/folly/futures/detail/Core.h:317
#35 0x00007fec135c6f44 in folly::detail::Core<std::unique_ptr<hbase::Response,
std::default_delete<hbase::Response> >
>::setResult(folly::Try<std::unique_ptr<hbase::Response,
std::default_delete<hbase::Response> > >&&) (this=0x7febd0af2a90, t=<unknown
type in /usr/local/lib/libHbaseClient_d.so, CU 0x45953, DIE 0xa1de0>) at
/usr/local/include/folly/futures/detail/Core.h:183
#36 0x00007fec135c68c0 in folly::Promise<std::unique_ptr<hbase::Response,
std::default_delete<hbase::Response> >
>::setTry(folly::Try<std::unique_ptr<hbase::Response,
std::default_delete<hbase::Response> > >&&) (this=0x7fec017f8fa0, t=<unknown
type in /usr/local/lib/libHbaseClient_d.so, CU 0x45953, DIE 0xa1de0>) at
/usr/local/include/folly/futures/Promise-inl.h:121
#37 0x00007fec1360d150 in folly::Promise<std::unique_ptr<hbase::Response,
std::default_delete<hbase::Response> >
>::setValue<std::unique_ptr<hbase::Response,
std::default_delete<hbase::Response> > >(std::unique_ptr<hbase::Response,
std::default_delete<hbase::Response> >&&) (this=0x7fec017f8fa0, v=<unknown type
in /usr/local/lib/libHbaseClient_d.so, CU 0x216413, DIE 0x268be9>)
at /usr/local/include/folly/futures/Promise-inl.h:130
#38 0x00007fec1360b8b4 in hbase::ClientDispatcher::read (this=0x7febec001e00,
ctx=0x7febec0010f0, in=std::unique_ptr<hbase::Response> containing 0x0) at
connection/client-dispatcher.cc:49
#39 0x00007fec1362ff72 in
wangle::ContextImpl<wangle::ClientDispatcherBase<wangle::Pipeline<folly::IOBufQueue&,
std::unique_ptr<hbase::Request, std::default_delete<hbase::Request> > >,
std::unique_ptr<hbase::Request, std::default_delete<hbase::Request> >,
std::unique_ptr<hbase::Response, std::default_delete<hbase::Response> > >
>::read (this=0x7febec0010f0,
msg=std::unique_ptr<hbase::Response> containing 0x0) at
/usr/local/include/wangle/channel/HandlerContext-inl.h:264
#40 0x00007fec1361ccc1 in wangle::ContextImpl<hbase::ClientHandler>::fireRead
(this=0x7febd000c2c0, msg=std::unique_ptr<hbase::Response> containing 0x0)
at /usr/local/include/wangle/channel/HandlerContext-inl.h:166
#41 0x00007fec136054c2 in hbase::ClientHandler::read (this=0x7febd000c240,
ctx=0x7febd000c2c0, buf=std::unique_ptr<folly::IOBuf> containing 0x0) at
connection/client-handler.cc:114
#42 0x00007fec1361d4fc in wangle::ContextImpl<hbase::ClientHandler>::read
(this=0x7febd000c2c0, msg=std::unique_ptr<folly::IOBuf> containing 0x0)
at /usr/local/include/wangle/channel/HandlerContext-inl.h:264
#43 0x00007fec135f8071 in
wangle::InboundContextImpl<wangle::LengthFieldBasedFrameDecoder>::fireRead
(this=0x7febd00021e0, msg=std::unique_ptr<folly::IOBuf> containing 0x0)
at /usr/local/include/wangle/channel/HandlerContext-inl.h:335
#44 0x0000000000440a94 in
wangle::ByteToMessageDecoder<std::unique_ptr<folly::IOBuf,
std::default_delete<folly::IOBuf> >
>::read(wangle::InboundHandlerContext<std::unique_ptr<folly::IOBuf,
std::default_delete<folly::IOBuf> > >*, folly::IOBufQueue&) ()
#45 0x00007fec135f84b7 in
wangle::InboundContextImpl<wangle::LengthFieldBasedFrameDecoder>::read
(this=0x7febd00021e0, msg=...) at
/usr/local/include/wangle/channel/HandlerContext-inl.h:384
#46 0x00007fec1361dd92 in wangle::ContextImpl<hbase::SaslHandler>::fireRead
(this=0x7febd00020c0, msg=...) at
/usr/local/include/wangle/channel/HandlerContext-inl.h:166
#47 0x00007fec1361570b in hbase::SaslHandler::read (this=0x7febd0002040,
ctx=0x7febd00020c0, buf=...) at connection/sasl-handler.cc:96
#48 0x00007fec1361e59d in wangle::ContextImpl<hbase::SaslHandler>::read
(this=0x7febd00020c0, msg=...) at
/usr/local/include/wangle/channel/HandlerContext-inl.h:264
#49 0x00007fec135f9432 in
wangle::ContextImpl<wangle::AsyncSocketHandler>::fireRead (this=0x7febd0001e80,
msg=...) at /usr/local/include/wangle/channel/HandlerContext-inl.h:166
#50 0x00007fec135d9ecd in wangle::AsyncSocketHandler::readDataAvailable
(this=0x7febd00017d0, len=841) at
/usr/local/include/wangle/channel/AsyncSocketHandler.h:138
#51 0x00007fec123b46d4 in folly::AsyncSocket::handleRead (this=0x7febd0001850)
at io/async/AsyncSocket.cpp:1628
#52 0x00007fec123acaee in folly::AsyncSocket::ioReady (this=0x7febd0001850,
events=2) at io/async/AsyncSocket.cpp:1458
#53 0x00007fec0d2d7a14 in event_base_loop () from /lib64/libevent-2.0.so.5
#54 0x00007fec123cc3b5 in folly::EventBase::loopBody
(this=this@entry=0x7febd00009f0, flags=flags@entry=0) at
io/async/EventBase.cpp:297
#55 0x00007fec123ccd97 in folly::EventBase::loop
(this=this@entry=0x7febd00009f0) at io/async/EventBase.cpp:236
#56 0x00007fec123ce126 in folly::EventBase::loopForever (this=0x7febd00009f0)
at io/async/EventBase.cpp:434
#57 0x000000000044fe5c in
wangle::IOThreadPoolExecutor::threadRun(std::shared_ptr<wangle::ThreadPoolExecutor::Thread>)
()
#58 0x0000000000461ec7 in void std::_Mem_fn_base<void
(wangle::ThreadPoolExecutor::*)(std::shared_ptr<wangle::ThreadPoolExecutor::Thread>),
true>::operator()<std::shared_ptr<wangle::ThreadPoolExecutor::Thread>&,
void>(wangle::ThreadPoolExecutor*,
std::shared_ptr<wangle::ThreadPoolExecutor::Thread>&) const ()
#59 0x00000000004605dd in void std::_Bind<std::_Mem_fn<void
(wangle::ThreadPoolExecutor::*)(std::shared_ptr<wangle::ThreadPoolExecutor::Thread>)>
(wangle::ThreadPoolExecutor*,
std::shared_ptr<wangle::ThreadPoolExecutor::Thread>)>::__call<void, , 0ul,
1ul>(std::tuple<>&&, std::_Index_tuple<0ul, 1ul>) ()
#60 0x000000000045ef48 in void std::_Bind<std::_Mem_fn<void
(wangle::ThreadPoolExecutor::*)(std::shared_ptr<wangle::ThreadPoolExecutor::Thread>)>
(wangle::ThreadPoolExecutor*,
std::shared_ptr<wangle::ThreadPoolExecutor::Thread>)>::operator()<, void>() ()
#61 0x000000000045d3e8 in void folly::detail::function::FunctionTraits<void
()>::callBig<std::_Bind<std::_Mem_fn<void
(wangle::ThreadPoolExecutor::*)(std::shared_ptr<wangle::ThreadPoolExecutor::Thread>)>
(wangle::ThreadPoolExecutor*,
std::shared_ptr<wangle::ThreadPoolExecutor::Thread>)>
>(folly::detail::function::Data&) ()
#62 0x000000000044c485 in folly::detail::function::FunctionTraits<void
()>::operator()() ()
#63 0x000000000044b6c0 in void std::_Bind_simple<folly::Function<void ()>
()>::_M_invoke<>(std::_Index_tuple<>) ()
#64 0x000000000044af1f in std::_Bind_simple<folly::Function<void ()>
()>::operator()() ()
#65 0x000000000044a7ea in
std::thread::_Impl<std::_Bind_simple<folly::Function<void ()> ()> >::_M_run() ()
#66 0x00007fec139c3bd0 in execute_native_thread_routine_compat () from
/usr/local/lib/libHbaseClient_d.so
#67 0x00007fec0eea1dc5 in start_thread () from /lib64/libpthread.so.0
#68 0x00007fec0ebd073d in clone () from /lib64/libc.so.6
{code}
> [C++] Fix deadlock in AsyncScanRetryingCaller and other RPCs
> ------------------------------------------------------------
>
> Key: HBASE-18565
> URL: https://issues.apache.org/jira/browse/HBASE-18565
> Project: HBase
> Issue Type: Sub-task
> Reporter: Enis Soztutar
> Assignee: Enis Soztutar
> Fix For: HBASE-14850
>
>
> When running the load-client test, sometimes we get a deadlock.
--
This message was sent by Atlassian JIRA
(v6.4.14#64029)
