[
https://issues.apache.org/jira/browse/KUDU-1773?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
Todd Lipcon resolved KUDU-1773.
-------------------------------
Resolution: Fixed
Assignee: Adar Dembo
Fix Version/s: 1.2.0
> Kudu client DCHECK fails
> ------------------------
>
> Key: KUDU-1773
> URL: https://issues.apache.org/jira/browse/KUDU-1773
> Project: Kudu
> Issue Type: Bug
> Components: client
> Affects Versions: 1.1.0
> Reporter: Matthew Jacobs
> Assignee: Adar Dembo
> Priority: Critical
> Labels: impala
> Fix For: 1.2.0
>
>
> Impalad running stress tests w/ debug libkudu_client.so gets the following
> DCHECK failure:
> F1129 19:32:42.075459 185566 meta_cache.cc:267] Check failed: found Tablet
> e0eea4076a6843749389956abfed3172: Specified server not found:
> 520ffc6d07ad42baaf49e59f7129b080 (ve1121.halxg.cloudera.com:7050). Replicas:
> 275ece6d98a14be9b7dfcee3bec8d7a8 (FOLLOWER, OK),
> 2d52a82cb62c4064842cf628d01083fe (FOLLOWER, OK),
> c101883d3e82496989a5f9f667c30e38 (FOLLOWER, OK)
> stack from gdb dump:
> {code}
> #0 0x0000003fdd232625 in raise () from /lib64/libc.so.6
> #1 0x0000003fdd233e05 in abort () from /lib64/libc.so.6
> #2 0x000000000281f3b4 in ?? ()
> #3 0x000000000281881d in google::LogMessage::Fail() ()
> #4 0x000000000281b146 in google::LogMessage::SendToLog() ()
> #5 0x000000000281833d in google::LogMessage::Flush() ()
> #6 0x0000000002818619 in google::LogMessage::~LogMessage() ()
> #7 0x00000000011c902f in impala::LogKuduMessage(void*,
> kudu::client::KuduLogSeverity, char const*, int, tm const*, char const*,
> unsigned long) ()
> #8 0x00000000011c9423 in
> kudu::client::KuduLoggingFunctionCallback<void*>::Run(kudu::client::KuduLogSeverity,
> char const*, int, tm const*, char const*, unsigned long) ()
> #9 0x00007eff1b24073e in
> kudu::client::LoggingAdapterCB(kudu::client::KuduLoggingCallback*,
> kudu::LogSeverity, char const*, int, tm const*, char const*, unsigned long) ()
> at
> /data/jenkins/workspace/verify-impala-toolchain-package-build/label/ec2-package-centos-6/toolchain/source/kudu/kudu-f2aeba6c059ea61f9cf8984d3e84d6c27b64d463/src/kudu/client/client.cc:163
> #10 0x00007eff1b25863c in kudu::internal::RunnableAdapter<void
> (*)(kudu::client::KuduLoggingCallback*, kudu::LogSeverity, char const*, int,
> tm const*, char const*, unsigned
> long)>::Run(kudu::client::KuduLoggingCallback* const&, kudu::LogSeverity
> const&, char const* const&, int const&, tm const* const&, char const* const&,
> unsigned long const&) ()
> at
> /data/jenkins/workspace/verify-impala-toolchain-package-build/label/ec2-package-centos-6/toolchain/source/kudu/kudu-f2aeba6c059ea61f9cf8984d3e84d6c27b64d463/src/kudu/gutil/bind_internal.h:584
> #11 0x00007eff1b2568fe in kudu::internal::InvokeHelper<false, void,
> kudu::internal::RunnableAdapter<void (*)(kudu::client::KuduLoggingCallback*,
> kudu::LogSeverity, char const*, int, tm const*, char const*, unsigned long)>,
> void ()(kudu::client::KuduLoggingCallback*, kudu::LogSeverity const&, char
> const* const&, int const&, tm const* const&, char const* const&, unsigned
> long const&)>::MakeItSo(kudu::internal::RunnableAdapter<void
> (*)(kudu::client::KuduLoggingCallback*, kudu::LogSeverity, char const*, int,
> tm const*, char const*, unsigned long)>, kudu::client::KuduLoggingCallback*,
> kudu::LogSeverity const&, char const* const&, int const&, tm const* const&,
> char const* const&, unsigned long const&) ()
> at
> /data/jenkins/workspace/verify-impala-toolchain-package-build/label/ec2-package-centos-6/toolchain/source/kudu/kudu-f2aeba6c059ea61f9cf8984d3e84d6c27b64d463/src/kudu/gutil/bind_internal.h:990
> #12 0x00007eff1b2540e3 in kudu::internal::Invoker<1,
> kudu::internal::BindState<kudu::internal::RunnableAdapter<void
> (*)(kudu::client::KuduLoggingCallback*, kudu::LogSeverity, char const*, int,
> tm const*, char const*, unsigned long)>, void
> ()(kudu::client::KuduLoggingCallback*, kudu::LogSeverity, char const*, int,
> tm const*, char const*, unsigned long), void
> ()(kudu::internal::UnretainedWrapper<kudu::client::KuduLoggingCallback>)>,
> void ()(kudu::client::KuduLoggingCallback*, kudu::LogSeverity, char const*,
> int, tm const*, char const*, unsigned
> long)>::Run(kudu::internal::BindStateBase*, kudu::LogSeverity const&, char
> const* const&, int const&, tm const* const&, char const* const&, unsigned
> long const&) ()
> at
> /data/jenkins/workspace/verify-impala-toolchain-package-build/label/ec2-package-centos-6/toolchain/source/kudu/kudu-f2aeba6c059ea61f9cf8984d3e84d6c27b64d463/src/kudu/gutil/bind_internal.h:2114
> #13 0x00007eff1b3d1f53 in kudu::Callback<void ()(kudu::LogSeverity, char
> const*, int, tm const*, char const*, unsigned long)>::Run(kudu::LogSeverity
> const&, char const* const&, int const&, tm const* const&, char const* const&,
> unsigned long const&) const ()
> at
> /data/jenkins/workspace/verify-impala-toolchain-package-build/label/ec2-package-centos-6/toolchain/source/kudu/kudu-f2aeba6c059ea61f9cf8984d3e84d6c27b64d463/src/kudu/gutil/callback.h:683
> #14 0x00007eff1b3d08bc in kudu::(anonymous namespace)::SimpleSink::send(int,
> char const*, char const*, int, tm const*, char const*, unsigned long) ()
> at
> /data/jenkins/workspace/verify-impala-toolchain-package-build/label/ec2-package-centos-6/toolchain/source/kudu/kudu-f2aeba6c059ea61f9cf8984d3e84d6c27b64d463/src/kudu/util/logging.cc:81
> #15 0x00007eff1b4ab4a7 in google::LogMessage::SendToLog() ()
> at
> /data/jenkins/workspace/verify-impala-toolchain-package-build/label/ec2-package-centos-6/toolchain/source/kudu/kudu-f2aeba6c059ea61f9cf8984d3e84d6c27b64d463/thirdparty/src/glog-0.3.4/src/logging.cc:784
> #16 0x00007eff1b4a8df9 in google::LogMessage::Flush() ()
> at
> /data/jenkins/workspace/verify-impala-toolchain-package-build/label/ec2-package-centos-6/toolchain/source/kudu/kudu-f2aeba6c059ea61f9cf8984d3e84d6c27b64d463/thirdparty/src/glog-0.3.4/src/logging.cc:1301
> #17 0x00007eff1b4ac3ef in google::LogMessageFatal::~LogMessageFatal() ()
> at
> /data/jenkins/workspace/verify-impala-toolchain-package-build/label/ec2-package-centos-6/toolchain/source/kudu/kudu-f2aeba6c059ea61f9cf8984d3e84d6c27b64d463/thirdparty/src/glog-0.3.4/src/logging.cc:2013
> #18 0x00007eff1b27b3bf in
> kudu::client::internal::RemoteTablet::MarkTServerAsLeader(kudu::client::internal::RemoteTabletServer
> const*) ()
> at
> /data/jenkins/workspace/verify-impala-toolchain-package-build/label/ec2-package-centos-6/toolchain/source/kudu/kudu-f2aeba6c059ea61f9cf8984d3e84d6c27b64d463/src/kudu/client/meta_cache.cc:267
> #19 0x00007eff1b27c9f9 in
> kudu::client::internal::MetaCacheServerPicker::PickLeader(kudu::Callback<void
> ()(kudu::Status const&, kudu::client::internal::RemoteTabletServer*)> const&,
> kudu::MonoTime const&) ()
> at
> /data/jenkins/workspace/verify-impala-toolchain-package-build/label/ec2-package-centos-6/toolchain/source/kudu/kudu-f2aeba6c059ea61f9cf8984d3e84d6c27b64d463/src/kudu/client/meta_cache.cc:413
> #20 0x00007eff1b237aba in
> kudu::rpc::RetriableRpc<kudu::client::internal::RemoteTabletServer,
> kudu::tserver::WriteRequestPB, kudu::tserver::WriteResponsePB>::SendRpc() ()
> at
> /data/jenkins/workspace/verify-impala-toolchain-package-build/label/ec2-package-centos-6/toolchain/source/kudu/kudu-f2aeba6c059ea61f9cf8984d3e84d6c27b64d463/src/kudu/rpc/retriable_rpc.h:130
> #21 0x00007eff1b2f5531 in
> kudu::rpc::RpcRetrier::DelayedRetryCb(kudu::rpc::Rpc*, kudu::Status const&) ()
> at
> /data/jenkins/workspace/verify-impala-toolchain-package-build/label/ec2-package-centos-6/toolchain/source/kudu/kudu-f2aeba6c059ea61f9cf8984d3e84d6c27b64d463/src/kudu/rpc/rpc.cc:86
> #22 0x00007eff1b2f5d68 in boost::_mfi::mf2<void, kudu::rpc::RpcRetrier,
> kudu::rpc::Rpc*, kudu::Status const&>::operator()(kudu::rpc::RpcRetrier*,
> kudu::rpc::Rpc*, kudu::Status const&) const ()
> at
> /data/jenkins/workspace/verify-impala-toolchain-package-build/label/ec2-package-centos-6/toolchain/source/kudu/kudu-f2aeba6c059ea61f9cf8984d3e84d6c27b64d463/thirdparty/installed/common/include/boost/bind/mem_fn_template.hpp:280
> #23 0x00007eff1b2f5c38 in void
> boost::_bi::list3<boost::_bi::value<kudu::rpc::RpcRetrier*>,
> boost::_bi::value<kudu::rpc::Rpc*>, boost::arg<1>
> >::operator()<boost::_mfi::mf2<void, kudu::rpc::RpcRetrier, kudu::rpc::Rpc*,
> kudu::Status const&>, boost::_bi::rrlist1<kudu::Status const&>
> >(boost::_bi::type<void>, boost::_mfi::mf2<void, kudu::rpc::RpcRetrier,
> kudu::rpc::Rpc*, kudu::Status const&>&, boost::_bi::rrlist1<kudu::Status
> const&>&, int) ()
> at
> /data/jenkins/workspace/verify-impala-toolchain-package-build/label/ec2-package-centos-6/toolchain/source/kudu/kudu-f2aeba6c059ea61f9cf8984d3e84d6c27b64d463/thirdparty/installed/common/include/boost/bind/bind.hpp:398
> #24 0x00007eff1b2f59f2 in void boost::_bi::bind_t<void,
> boost::_mfi::mf2<void, kudu::rpc::RpcRetrier, kudu::rpc::Rpc*, kudu::Status
> const&>, boost::_bi::list3<boost::_bi::value<kudu::rpc::RpcRetrier*>,
> boost::_bi::value<kudu::rpc::Rpc*>, boost::arg<1> >
> >::operator()<kudu::Status const&>(kudu::Status const&&&) ()
> at
> /data/jenkins/workspace/verify-impala-toolchain-package-build/label/ec2-package-centos-6/toolchain/source/kudu/kudu-f2aeba6c059ea61f9cf8984d3e84d6c27b64d463/thirdparty/installed/common/include/boost/bind/bind.hpp:1234
> #25 0x00007eff1b2f5922 in
> boost::detail::function::void_function_obj_invoker1<boost::_bi::bind_t<void,
> boost::_mfi::mf2<void, kudu::rpc::RpcRetrier, kudu::rpc::Rpc*, kudu::Status
> const&>, boost::_bi::list3<boost::_bi::value<kudu::rpc::RpcRetrier*>,
> boost::_bi::value<kudu::rpc::Rpc*>, boost::arg<1> > >, void, kudu::Status
> const&>::invoke(boost::detail::function::function_buffer&, kudu::Status
> const&) ()
> at
> /data/jenkins/workspace/verify-impala-toolchain-package-build/label/ec2-package-centos-6/toolchain/source/kudu/kudu-f2aeba6c059ea61f9cf8984d3e84d6c27b64d463/thirdparty/installed/common/include/boost/function/function_template.hpp:159
> #26 0x00007eff1b2ed176 in boost::function1<void, kudu::Status
> const&>::operator()(kudu::Status const&) const ()
> at
> /data/jenkins/workspace/verify-impala-toolchain-package-build/label/ec2-package-centos-6/toolchain/source/kudu/kudu-f2aeba6c059ea61f9cf8984d3e84d6c27b64d463/thirdparty/installed/common/include/boost/function/function_template.hpp:771
> #27 0x00007eff1b2ea18d in kudu::rpc::DelayedTask::TimerHandler(ev::timer&,
> int) ()
> at
> /data/jenkins/workspace/verify-impala-toolchain-package-build/label/ec2-package-centos-6/toolchain/source/kudu/kudu-f2aeba6c059ea61f9cf8984d3e84d6c27b64d463/src/kudu/rpc/reactor.cc:490
> #28 0x00007eff1b2ee554 in void ev::base<ev_timer,
> ev::timer>::method_thunk<kudu::rpc::DelayedTask,
> &(kudu::rpc::DelayedTask::TimerHandler(ev::timer&, int))>(ev_loop*,
> ev_timer*, int) ()
> at
> /data/jenkins/workspace/verify-impala-toolchain-package-build/label/ec2-package-centos-6/toolchain/source/kudu/kudu-f2aeba6c059ea61f9cf8984d3e84d6c27b64d463/thirdparty/installed/uninstrumented/include/ev++.h:479
> #29 0x00007eff1b51a4cb in ev_invoke_pending ()
> at
> /data/jenkins/workspace/verify-impala-toolchain-package-build/label/ec2-package-centos-6/toolchain/source/kudu/kudu-f2aeba6c059ea61f9cf8984d3e84d6c27b64d463/thirdparty/src/libev-4.20/ev.c:3155
> #30 0x00007eff1b51dbd4 in ev_run ()
> at
> /data/jenkins/workspace/verify-impala-toolchain-package-build/label/ec2-package-centos-6/toolchain/source/kudu/kudu-f2aeba6c059ea61f9cf8984d3e84d6c27b64d463/thirdparty/src/libev-4.20/ev.c:3555
> #31 0x00007eff1b2eb2df in ev::loop_ref::run(int) ()
> at
> /data/jenkins/workspace/verify-impala-toolchain-package-build/label/ec2-package-centos-6/toolchain/source/kudu/kudu-f2aeba6c059ea61f9cf8984d3e84d6c27b64d463/thirdparty/installed/uninstrumented/include/ev++.h:211
> #32 0x00007eff1b2e8280 in kudu::rpc::ReactorThread::RunThread() ()
> at
> /data/jenkins/workspace/verify-impala-toolchain-package-build/label/ec2-package-centos-6/toolchain/source/kudu/kudu-f2aeba6c059ea61f9cf8984d3e84d6c27b64d463/src/kudu/rpc/reactor.cc:317
> #33 0x00007eff1b2f3547 in boost::_mfi::mf0<void,
> kudu::rpc::ReactorThread>::operator()(kudu::rpc::ReactorThread*) const ()
> at
> /data/jenkins/workspace/verify-impala-toolchain-package-build/label/ec2-package-centos-6/toolchain/source/kudu/kudu-f2aeba6c059ea61f9cf8984d3e84d6c27b64d463/thirdparty/installed/common/include/boost/bind/mem_fn_template.hpp:49
> #34 0x00007eff1b2f3054 in void
> boost::_bi::list1<boost::_bi::value<kudu::rpc::ReactorThread*>
> >::operator()<boost::_mfi::mf0<void, kudu::rpc::ReactorThread>,
> boost::_bi::list0>(boost::_bi::type<void>, boost::_mfi::mf0<void,
> kudu::rpc::ReactorThread>&, boost::_bi::list0&, int) ()
> at
> /data/jenkins/workspace/verify-impala-toolchain-package-build/label/ec2-package-centos-6/toolchain/source/kudu/kudu-f2aeba6c059ea61f9cf8984d3e84d6c27b64d463/thirdparty/installed/common/include/boost/bind/bind.hpp:259
> #35 0x00007eff1b2f271b in boost::_bi::bind_t<void, boost::_mfi::mf0<void,
> kudu::rpc::ReactorThread>,
> boost::_bi::list1<boost::_bi::value<kudu::rpc::ReactorThread*> >
> >::operator()() ()
> at
> /data/jenkins/workspace/verify-impala-toolchain-package-build/label/ec2-package-centos-6/toolchain/source/kudu/kudu-f2aeba6c059ea61f9cf8984d3e84d6c27b64d463/thirdparty/installed/common/include/boost/bind/bind.hpp:1222
> #36 0x00007eff1b2f1c20 in
> boost::detail::function::void_function_obj_invoker0<boost::_bi::bind_t<void,
> boost::_mfi::mf0<void, kudu::rpc::ReactorThread>,
> boost::_bi::list1<boost::_bi::value<kudu::rpc::ReactorThread*> > >,
> void>::invoke(boost::detail::function::function_buffer&) ()
> at
> /data/jenkins/workspace/verify-impala-toolchain-package-build/label/ec2-package-centos-6/toolchain/source/kudu/kudu-f2aeba6c059ea61f9cf8984d3e84d6c27b64d463/thirdparty/installed/common/include/boost/function/function_template.hpp:159
> #37 0x00007eff1b315f3e in boost::function0<void>::operator()() const ()
> at
> /data/jenkins/workspace/verify-impala-toolchain-package-build/label/ec2-package-centos-6/toolchain/source/kudu/kudu-f2aeba6c059ea61f9cf8984d3e84d6c27b64d463/thirdparty/installed/common/include/boost/function/function_template.hpp:771
> #38 0x00007eff1b413434 in kudu::Thread::SuperviseThread(void*) ()
> at
> /data/jenkins/workspace/verify-impala-toolchain-package-build/label/ec2-package-centos-6/toolchain/source/kudu/kudu-f2aeba6c059ea61f9cf8984d3e84d6c27b64d463/src/kudu/util/thread.cc:588
> #39 0x0000003fdd607aa1 in start_thread () from /lib64/libpthread.so.0
> #40 0x0000003fdd2e893d in clone () from /lib64/libc.so.6
> {code}
> Kudu client built at git hash f2aeba
> Kudu tserver/master is even more recent (1.2 snapshot), git hash
> 2f1a2a06d4fe719ef9f35338bc7403915517718a
--
This message was sent by Atlassian JIRA
(v6.3.4#6332)
