[
https://issues.apache.org/jira/browse/KUDU-1745?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=15665902#comment-15665902
]
Matthew Jacobs commented on KUDU-1745:
--------------------------------------
Wow... finally got the right stack:
{code}
Stack: [0x00007f9961c1f000,0x00007f9962620000], sp=0x00007f996261d9e0, free
space=10234k
Native frames: (J=compiled Java code, j=interpreted, Vv=VM code, C=native code)
C [libstdc++.so.6+0xc5018] std::basic_string<char, std::char_traits<char>,
std::allocator<char> >::basic_string(std::string const&)+0x8
C [libkudu_client.so.0+0xf4dc2]
kudu::client::internal::RemoteTablet::MarkReplicaFailed(kudu::client::internal::RemoteTabletServer*,
kudu::Status const&)+0x1d0
C [libkudu_client.so.0+0xf6fcf]
kudu::client::internal::MetaCacheServerPicker::MarkServerFailed(kudu::client::internal::RemoteTabletServer*,
kudu::Status const&)+0x2f
C [libkudu_client.so.0+0xb5a53]
kudu::rpc::RetriableRpc<kudu::client::internal::RemoteTabletServer,
kudu::tserver::WriteRequestPB,
kudu::tserver::WriteResponsePB>::RetryIfNeeded(kudu::rpc::RetriableRpcStatus
const&, kudu::client::internal::RemoteTabletServer*)+0x207
C [libkudu_client.so.0+0xb3ac0]
kudu::rpc::RetriableRpc<kudu::client::internal::RemoteTabletServer,
kudu::tserver::WriteRequestPB,
kudu::tserver::WriteResponsePB>::ReplicaFoundCb(kudu::Status const&,
kudu::client::internal::RemoteTabletServer*)+0x70
C [libkudu_client.so.0+0xb97bc] kudu::internal::RunnableAdapter<void
(kudu::rpc::RetriableRpc<kudu::client::internal::RemoteTabletServer,
kudu::tserver::WriteRequestPB, kudu::tserver::WriteResponsePB>::*)(kudu::Status
const&,
kudu::client::internal::RemoteTabletServer*)>::Run(kudu::rpc::RetriableRpc<kudu::client::internal::RemoteTabletServer,
kudu::tserver::WriteRequestPB, kudu::tserver::WriteResponsePB>*, kudu::Status
const&, kudu::client::internal::RemoteTabletServer* const&)+0x94
C [libkudu_client.so.0+0xb8d85] kudu::internal::InvokeHelper<false, void,
kudu::internal::RunnableAdapter<void
(kudu::rpc::RetriableRpc<kudu::client::internal::RemoteTabletServer,
kudu::tserver::WriteRequestPB, kudu::tserver::WriteResponsePB>::*)(kudu::Status
const&, kudu::client::internal::RemoteTabletServer*)>, void
()(kudu::rpc::RetriableRpc<kudu::client::internal::RemoteTabletServer,
kudu::tserver::WriteRequestPB, kudu::tserver::WriteResponsePB>*, kudu::Status
const&, kudu::client::internal::RemoteTabletServer*
const&)>::MakeItSo(kudu::internal::RunnableAdapter<void
(kudu::rpc::RetriableRpc<kudu::client::internal::RemoteTabletServer,
kudu::tserver::WriteRequestPB, kudu::tserver::WriteResponsePB>::*)(kudu::Status
const&, kudu::client::internal::RemoteTabletServer*)>,
kudu::rpc::RetriableRpc<kudu::client::internal::RemoteTabletServer,
kudu::tserver::WriteRequestPB, kudu::tserver::WriteResponsePB>*, kudu::Status
const&, kudu::client::internal::RemoteTabletServer* const&)+0x6d
C [libkudu_client.so.0+0xb7d5b] kudu::internal::Invoker<1,
kudu::internal::BindState<kudu::internal::RunnableAdapter<void
(kudu::rpc::RetriableRpc<kudu::client::internal::RemoteTabletServer,
kudu::tserver::WriteRequestPB, kudu::tserver::WriteResponsePB>::*)(kudu::Status
const&, kudu::client::internal::RemoteTabletServer*)>, void
()(kudu::rpc::RetriableRpc<kudu::client::internal::RemoteTabletServer,
kudu::tserver::WriteRequestPB, kudu::tserver::WriteResponsePB>*, kudu::Status
const&, kudu::client::internal::RemoteTabletServer*), void
()(kudu::internal::UnretainedWrapper<kudu::rpc::RetriableRpc<kudu::client::internal::RemoteTabletServer,
kudu::tserver::WriteRequestPB, kudu::tserver::WriteResponsePB> >)>, void
()(kudu::rpc::RetriableRpc<kudu::client::internal::RemoteTabletServer,
kudu::tserver::WriteRequestPB, kudu::tserver::WriteResponsePB>*, kudu::Status
const&,
kudu::client::internal::RemoteTabletServer*)>::Run(kudu::internal::BindStateBase*,
kudu::Status const&, kudu::client::internal::RemoteTabletServer* const&)+0x7d
C [libkudu_client.so.0+0xfed1d] kudu::Callback<void ()(kudu::Status const&,
kudu::client::internal::RemoteTabletServer*)>::Run(kudu::Status const&,
kudu::client::internal::RemoteTabletServer* const&) const+0x5f
C [libkudu_client.so.0+0xf70df]
kudu::client::internal::MetaCacheServerPicker::LookUpTabletCb(kudu::Callback<void
()(kudu::Status const&, kudu::client::internal::RemoteTabletServer*)> const&,
kudu::MonoTime const&, kudu::Status const&)+0x7d
C [libkudu_client.so.0+0x1074b1] kudu::internal::RunnableAdapter<void
(kudu::client::internal::MetaCacheServerPicker::*)(kudu::Callback<void
()(kudu::Status const&, kudu::client::internal::RemoteTabletServer*)> const&,
kudu::MonoTime const&, kudu::Status
const&)>::Run(kudu::client::internal::MetaCacheServerPicker*,
kudu::Callback<void ()(kudu::Status const&,
kudu::client::internal::RemoteTabletServer*)> const&, kudu::MonoTime const&,
kudu::Status const&)+0xa9
C [libkudu_client.so.0+0x1057a9] kudu::internal::InvokeHelper<false, void,
kudu::internal::RunnableAdapter<void
(kudu::client::internal::MetaCacheServerPicker::*)(kudu::Callback<void
()(kudu::Status const&, kudu::client::internal::RemoteTabletServer*)> const&,
kudu::MonoTime const&, kudu::Status const&)>, void
()(kudu::client::internal::MetaCacheServerPicker*, kudu::Callback<void
()(kudu::Status const&, kudu::client::internal::RemoteTabletServer*)> const&,
kudu::MonoTime const&, kudu::Status
const&)>::MakeItSo(kudu::internal::RunnableAdapter<void
(kudu::client::internal::MetaCacheServerPicker::*)(kudu::Callback<void
()(kudu::Status const&, kudu::client::internal::RemoteTabletServer*)> const&,
kudu::MonoTime const&, kudu::Status const&)>,
kudu::client::internal::MetaCacheServerPicker*, kudu::Callback<void
()(kudu::Status const&, kudu::client::internal::RemoteTabletServer*)> const&,
kudu::MonoTime const&, kudu::Status const&)+0x85
C [libkudu_client.so.0+0x1035a3] kudu::internal::Invoker<3,
kudu::internal::BindState<kudu::internal::RunnableAdapter<void
(kudu::client::internal::MetaCacheServerPicker::*)(kudu::Callback<void
()(kudu::Status const&, kudu::client::internal::RemoteTabletServer*)> const&,
kudu::MonoTime const&, kudu::Status const&)>, void
()(kudu::client::internal::MetaCacheServerPicker*, kudu::Callback<void
()(kudu::Status const&, kudu::client::internal::RemoteTabletServer*)> const&,
kudu::MonoTime const&, kudu::Status const&), void
()(kudu::internal::UnretainedWrapper<kudu::client::internal::MetaCacheServerPicker>,
kudu::Callback<void ()(kudu::Status const&,
kudu::client::internal::RemoteTabletServer*)>, kudu::MonoTime)>, void
()(kudu::client::internal::MetaCacheServerPicker*, kudu::Callback<void
()(kudu::Status const&, kudu::client::internal::RemoteTabletServer*)> const&,
kudu::MonoTime const&, kudu::Status
const&)>::Run(kudu::internal::BindStateBase*, kudu::Status const&)+0xb5
C [libkudu_client.so.0+0xe6c9f] kudu::Callback<void ()(kudu::Status
const&)>::Run(kudu::Status const&) const+0x47
C [libkudu_client.so.0+0xf9258]
kudu::client::internal::LookupRpc::SendRpcCb(kudu::Status const&)+0x9e0
C [libkudu_client.so.0+0x10a06a] boost::_mfi::mf1<void,
kudu::client::internal::LookupRpc, kudu::Status
const&>::operator()(kudu::client::internal::LookupRpc*, kudu::Status const&)
const+0x70
C [libkudu_client.so.0+0x109027] void
boost::_bi::list2<boost::_bi::value<kudu::client::internal::LookupRpc*>,
boost::_bi::value<kudu::Status> >::operator()<boost::_mfi::mf1<void,
kudu::client::internal::LookupRpc, kudu::Status const&>,
boost::_bi::list0>(boost::_bi::type<void>, boost::_mfi::mf1<void,
kudu::client::internal::LookupRpc, kudu::Status const&>&, boost::_bi::list0&,
int)+0x69
C [libkudu_client.so.0+0x107caf] boost::_bi::bind_t<void,
boost::_mfi::mf1<void, kudu::client::internal::LookupRpc, kudu::Status const&>,
boost::_bi::list2<boost::_bi::value<kudu::client::internal::LookupRpc*>,
boost::_bi::value<kudu::Status> > >::operator()()+0x3b
C [libkudu_client.so.0+0x106214]
boost::detail::function::void_function_obj_invoker0<boost::_bi::bind_t<void,
boost::_mfi::mf1<void, kudu::client::internal::LookupRpc, kudu::Status const&>,
boost::_bi::list2<boost::_bi::value<kudu::client::internal::LookupRpc*>,
boost::_bi::value<kudu::Status> > >,
void>::invoke(boost::detail::function::function_buffer&)+0x23
C [libkudu_client.so.0+0x18fd3a] boost::function0<void>::operator()()
const+0x52
C [libkudu_client.so.0+0x18cc34] kudu::rpc::OutboundCall::CallCallback()+0x40
C [libkudu_client.so.0+0x18d462]
kudu::rpc::OutboundCall::SetFailed(kudu::Status const&,
kudu::rpc::ErrorStatusPB*)+0x1c8
C [libkudu_client.so.0+0x19283e] kudu::rpc::Connection::Shutdown(kudu::Status
const&)+0x3fc
C [libkudu_client.so.0+0x163af9]
kudu::rpc::ReactorThread::DestroyConnection(kudu::rpc::Connection*,
kudu::Status const&)+0xe5
C [libkudu_client.so.0+0x194419] kudu::rpc::Connection::ReadHandler(ev::io&,
int)+0x62f
C [libkudu_client.so.0+0x19a00a] void ev::base<ev_io,
ev::io>::method_thunk<kudu::rpc::Connection,
&(kudu::rpc::Connection::ReadHandler(ev::io&, int))>(ev_loop*, ev_io*, int)+0x2d
C [libkudu_client.so.0+0x3948fb] ev_invoke_pending+0x5b
C [libkudu_client.so.0+0x398004] ev_run+0x9a4
C [libkudu_client.so.0+0x1653e7] ev::loop_ref::run(int)+0x23
C [libkudu_client.so.0+0x162388] kudu::rpc::ReactorThread::RunThread()+0x116
C [libkudu_client.so.0+0x16d64f] boost::_mfi::mf0<void,
kudu::rpc::ReactorThread>::operator()(kudu::rpc::ReactorThread*) const+0x65
C [libkudu_client.so.0+0x16d15c] void
boost::_bi::list1<boost::_bi::value<kudu::rpc::ReactorThread*>
>::operator()<boost::_mfi::mf0<void, kudu::rpc::ReactorThread>,
boost::_bi::list0>(boost::_bi::type<void>, boost::_mfi::mf0<void,
kudu::rpc::ReactorThread>&, boost::_bi::list0&, int)+0x4a
C [libkudu_client.so.0+0x16c823] boost::_bi::bind_t<void,
boost::_mfi::mf0<void, kudu::rpc::ReactorThread>,
boost::_bi::list1<boost::_bi::value<kudu::rpc::ReactorThread*> >
>::operator()()+0x3b
C [libkudu_client.so.0+0x16bd28]
boost::detail::function::void_function_obj_invoker0<boost::_bi::bind_t<void,
boost::_mfi::mf0<void, kudu::rpc::ReactorThread>,
boost::_bi::list1<boost::_bi::value<kudu::rpc::ReactorThread*> > >,
void>::invoke(boost::detail::function::function_buffer&)+0x20
C [libkudu_client.so.0+0x18fd3a] boost::function0<void>::operator()()
const+0x52
C [libkudu_client.so.0+0x28d0a4] kudu::Thread::SuperviseThread(void*)+0x358
{code}
> Kudu causes Impala to crash under stress
> ----------------------------------------
>
> Key: KUDU-1745
> URL: https://issues.apache.org/jira/browse/KUDU-1745
> Project: Kudu
> Issue Type: Bug
> Reporter: Taras Bobrovytsky
> Priority: Critical
> Attachments: hs_err_pid7761.log, stacks.out
>
>
> There were over 200 queries running, about half of which were selects and the
> rest were upsert and delete queries.
> There was a crash after a few minutes with the following stack trace:
> {code}
> Stack: [0x00007f1629c93000,0x00007f162a694000], sp=0x00007f162a6922b0, free
> space=10236k
> Native frames: (J=compiled Java code, j=interpreted, Vv=VM code, C=native
> code)
> C [libstdc++.so.6+0xc5018] std::basic_string<char, std::char_traits<char>,
> std::allocator<char> >::basic_string(std::string const&)+0x8
> C [libkudu_client.so.0+0x6f662]
> _ZNSt6vectorIPN4kudu6client9KuduErrorESaIS3_EE19_M_emplace_back_auxIIS3_EEEvDpOT_+0x1dd2
> C [libkudu_client.so.0+0x4c872] _init+0xd022
> C [libkudu_client.so.0+0x4c9d6] _init+0xd186
> C [libkudu_client.so.0+0x70775]
> _ZNSt6vectorIPN4kudu6client9KuduErrorESaIS3_EE19_M_emplace_back_auxIIS3_EEEvDpOT_+0x2ee5
> C [libkudu_client.so.0+0x754d9]
> _ZNSt6vectorIPN4kudu6client9KuduErrorESaIS3_EE19_M_emplace_back_auxIIS3_EEEvDpOT_+0x7c49
> C [libkudu_client.so.0+0xc9be0]
> kudu::client::KuduUpsert::~KuduUpsert()+0x2ae30
> C [libkudu_client.so.0+0xc9eb2]
> kudu::client::KuduUpsert::~KuduUpsert()+0x2b102
> C [libkudu_client.so.0+0xcc73d]
> _ZNSt6vectorIN4kudu5SliceESaIS1_EE19_M_emplace_back_auxIIS1_EEEvDpOT_+0x123d
> C [libkudu_client.so.0+0xbe405]
> kudu::client::KuduUpsert::~KuduUpsert()+0x1f655
> C [libkudu_client.so.0+0xcdc0c]
> _ZNSt6vectorIN4kudu5SliceESaIS1_EE19_M_emplace_back_auxIIS1_EEEvDpOT_+0x270c
> C [libkudu_client.so.0+0x25ec1b]
> _ZNSt8_Rb_treeISsSt4pairIKSsSsESt10_Select1stIS2_ESt4lessISsESaIS2_EE22_M_emplace_hint_uniqueIJRKSt21piecewise_construct_tSt5tupleIJRS1_EESD_IJEEEEESt17_Rb_tree_iteratorIS2_ESt23_Rb_tree_const_iteratorIS2_EDpOT_+0x311b
> C [libkudu_client.so.0+0x262324]
> _ZNSt8_Rb_treeISsSt4pairIKSsSsESt10_Select1stIS2_ESt4lessISsESaIS2_EE22_M_emplace_hint_uniqueIJRKSt21piecewise_construct_tSt5tupleIJRS1_EESD_IJEEEEESt17_Rb_tree_iteratorIS2_ESt23_Rb_tree_const_iteratorIS2_EDpOT_+0x6824
> T_+0x558a
> {code}
--
This message was sent by Atlassian JIRA
(v6.3.4#6332)
