[
https://issues.apache.org/jira/browse/IMPALA-9315?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
Sahil Takiar resolved IMPALA-9315.
----------------------------------
Resolution: Not A Problem
Fixed after adding the {{-DDYNAMIC_ANNOTATIONS_ENABLED}} to the
{{CMAKE_CXX_FLAGS}} in {{be/CMakeLists.txt}}. Credit to Tim for helping me with
this.
> TSAN data race in kudu::ScopedWatchKernelStack / KernelStackWatchdog
> --------------------------------------------------------------------
>
> Key: IMPALA-9315
> URL: https://issues.apache.org/jira/browse/IMPALA-9315
> Project: IMPALA
> Issue Type: Sub-task
> Reporter: Sahil Takiar
> Priority: Major
>
> Several backend tests fail with this data race:
> {code}
> WARNING: ThreadSanitizer: data race (pid=8072)
> Atomic write of size 4 at 0x7b340006f5c4 by main thread:
> #0 __tsan_atomic32_store
> /mnt/source/llvm/llvm-5.0.1.src-p1/projects/compiler-rt/lib/tsan/rtl/tsan_interface_atomic.cc:562
> (llvm-codegen-test+0x18c004e)
> #1 base::subtle::Acquire_Store(int volatile*, int)
> /home/systest/Impala/be/src/gutil/atomicops-internals-tsan.h:104:3
> (llvm-codegen-test+0x19cd822)
> #2 kudu::ScopedWatchKernelStack::ScopedWatchKernelStack(char const*, int)
> /home/systest/Impala/be/src/kudu/util/kernel_stack_watchdog.h:259:5
> (llvm-codegen-test+0x19c9920)
> #3 kudu::Thread::StartThread(std::string const&, std::string const&,
> boost::function<void ()> const&, unsigned long, scoped_refptr<kudu::Thread>*)
> /home/systest/Impala/be/src/kudu/util/thread.cc:555:28
> (llvm-codegen-test+0x19c7129)
> #4 kudu::Status kudu::Thread::Create<void
> (kudu::rpc::ReactorThread::*)(), kudu::rpc::ReactorThread*>(std::string
> const&, std::string const&, void (kudu::rpc::ReactorThread::* const&)(),
> kudu::rpc::ReactorThread* const&, scoped_refptr<kudu::Thread>*)
> /home/systest/I
> mpala/be/src/kudu/util/thread.h:164:12 (llvm-codegen-test+0x26e1875)
> #5 kudu::rpc::ReactorThread::Init()
> /home/systest/Impala/be/src/kudu/rpc/reactor.cc:185:10
> (llvm-codegen-test+0x26dc126)
> #6 kudu::rpc::Reactor::Init()
> /home/systest/Impala/be/src/kudu/rpc/reactor.cc:759:18
> (llvm-codegen-test+0x26e06d1)
> #7 kudu::rpc::Messenger::Init()
> /home/systest/Impala/be/src/kudu/rpc/messenger.cc:446:41
> (llvm-codegen-test+0x26c9a8d)
> #8
> kudu::rpc::MessengerBuilder::Build(std::shared_ptr<kudu::rpc::Messenger>*)
> /home/systest/Impala/be/src/kudu/rpc/messenger.cc:205:46
> (llvm-codegen-test+0x26c95fe)
> #9 impala::RpcMgr::Init(impala::TNetworkAddress const&)
> /home/systest/Impala/be/src/rpc/rpc-mgr.cc:151:38
> (llvm-codegen-test+0x2722a62)
> #10 impala::TestEnv::Init()
> /home/systest/Impala/be/src/runtime/test-env.cc:82:64
> (llvm-codegen-test+0x1b3df91)
> #11 impala::LlvmCodeGenTest::SetUp()
> /home/systest/Impala/be/src/codegen/llvm-codegen-test.cc:49:46
> (llvm-codegen-test+0x1913e02)
> #12 void
> testing::internal::HandleExceptionsInMethodIfSupported<testing::Test,
> void>(testing::Test*, void (testing::Test::*)(), char const*) <null>
> (llvm-codegen-test+0x412afd2)
> #13 __libc_start_main
> /build/glibc-LK5gWL/glibc-2.23/csu/../csu/libc-start.c:291 (libc.so.6+0x2082f)
> Previous read of size 8 at 0x7b340006f5c0 by thread T188 (mutexes: write
> M3832):
> #0 memcpy
> /mnt/source/llvm/llvm-5.0.1.src-p1/projects/compiler-rt/lib/tsan/../sanitizer_common/sanitizer_common_interceptors.inc:738
> (llvm-codegen-test+0x18a3ad7)
> #1
> kudu::KernelStackWatchdog::TLS::Data::SnapshotCopy(kudu::KernelStackWatchdog::TLS::Data*)
> const /home/systest/Impala/be/src/kudu/util/kernel_stack_watchdog.cc:246:5
> (llvm-codegen-test+0x19ed2f3)
> #2 kudu::KernelStackWatchdog::RunThread()
> /home/systest/Impala/be/src/kudu/util/kernel_stack_watchdog.cc:172:12
> (llvm-codegen-test+0x19ec7c3)
> #3 boost::_mfi::mf0<void,
> kudu::KernelStackWatchdog>::operator()(kudu::KernelStackWatchdog*) const
> /home/systest/Impala/toolchain/boost-1.57.0-p3/include/boost/bind/mem_fn_template.hpp:49:16
> (llvm-codegen-test+0x19f0676)
> #4 void boost::_bi::list1<boost::_bi::value<kudu::KernelStackWatchdog*>
> >::operator()<boost::_mfi::mf0<void, kudu::KernelStackWatchdog>,
> boost::_bi::list0>(boost::_bi::type<void>, boost::_mfi::mf0<void,
> kudu::KernelStackWatchdog>&, boost::_bi::list0&, int) /home/sys
> test/Impala/toolchain/boost-1.57.0-p3/include/boost/bind/bind.hpp:253:9
> (llvm-codegen-test+0x19f05ca)
> #5 boost::_bi::bind_t<void, boost::_mfi::mf0<void,
> kudu::KernelStackWatchdog>,
> boost::_bi::list1<boost::_bi::value<kudu::KernelStackWatchdog*> >
> >::operator()()
> /home/systest/Impala/toolchain/boost-1.57.0-p3/include/boost/bind/bind_template.hpp:20:16
> (llvm-codegen-t
> est+0x19f0533)
> #6
> boost::detail::function::void_function_obj_invoker0<boost::_bi::bind_t<void,
> boost::_mfi::mf0<void, kudu::KernelStackWatchdog>,
> boost::_bi::list1<boost::_bi::value<kudu::KernelStackWatchdog*> > >,
> void>::invoke(boost::detail::function::function_buffer&) /home/sys
> test/Impala/toolchain/boost-1.57.0-p3/include/boost/function/function_template.hpp:153:11
> (llvm-codegen-test+0x19f0329)
> #7 boost::function0<void>::operator()() const
> /home/systest/Impala/toolchain/boost-1.57.0-p3/include/boost/function/function_template.hpp:766:14
> (llvm-codegen-test+0x19c9e41)
> #8 kudu::Thread::SuperviseThread(void*)
> /home/systest/Impala/be/src/kudu/util/thread.cc:603:3
> (llvm-codegen-test+0x19c76e9)
> Location is heap block of size 200 at 0x7b340006f500 allocated by main
> thread:
> #0 operator new(unsigned long)
> /mnt/source/llvm/llvm-5.0.1.src-p1/projects/compiler-rt/lib/tsan/rtl/tsan_new_delete.cc:44
> (llvm-codegen-test+0x1907103)
> #1 kudu::KernelStackWatchdog::CreateAndRegisterTLS()
> /home/systest/Impala/be/src/kudu/util/kernel_stack_watchdog.cc:218:15
> (llvm-codegen-test+0x19ed3a7)
> #2 kudu::KernelStackWatchdog::GetTLS()
> /home/systest/Impala/be/src/kudu/util/kernel_stack_watchdog.h:170:7
> (llvm-codegen-test+0x19cd7e1)
> #3 kudu::ScopedWatchKernelStack::ScopedWatchKernelStack(char const*, int)
> /home/systest/Impala/be/src/kudu/util/kernel_stack_watchdog.h:248:13
> (llvm-codegen-test+0x19c99f8)
> #4 kudu::Thread::StartThread(std::string const&, std::string const&,
> boost::function<void ()> const&, unsigned long, scoped_refptr<kudu::Thread>*)
> /home/systest/Impala/be/src/kudu/util/thread.cc:555:28
> (llvm-codegen-test+0x19c7129)
> #5 kudu::Status kudu::Thread::Create<void
> (kudu::rpc::ReactorThread::*)(), kudu::rpc::ReactorThread*>(std::string
> const&, std::string const&, void (kudu::rpc::ReactorThread::* const&)(),
> kudu::rpc::ReactorThread* const&, scoped_refptr<kudu::Thread>*)
> /home/systest/I
> mpala/be/src/kudu/util/thread.h:164:12 (llvm-codegen-test+0x26e1875)
> #6 kudu::rpc::ReactorThread::Init()
> /home/systest/Impala/be/src/kudu/rpc/reactor.cc:185:10
> (llvm-codegen-test+0x26dc126)
> #7 kudu::rpc::Reactor::Init()
> /home/systest/Impala/be/src/kudu/rpc/reactor.cc:759:18
> (llvm-codegen-test+0x26e06d1)
> #8 kudu::rpc::Messenger::Init()
> /home/systest/Impala/be/src/kudu/rpc/messenger.cc:446:41
> (llvm-codegen-test+0x26c9a8d)
> #9
> kudu::rpc::MessengerBuilder::Build(std::shared_ptr<kudu::rpc::Messenger>*)
> /home/systest/Impala/be/src/kudu/rpc/messenger.cc:205:46
> (llvm-codegen-test+0x26c95fe)
> #10 impala::RpcMgr::Init(impala::TNetworkAddress const&)
> /home/systest/Impala/be/src/rpc/rpc-mgr.cc:151:38
> (llvm-codegen-test+0x2722a62)
> #11 impala::TestEnv::Init()
> /home/systest/Impala/be/src/runtime/test-env.cc:82:64
> (llvm-codegen-test+0x1b3df91)
> #12 impala::LlvmCodeGenTest::SetUp()
> /home/systest/Impala/be/src/codegen/llvm-codegen-test.cc:49:46
> (llvm-codegen-test+0x1913e02)
> #13 void
> testing::internal::HandleExceptionsInMethodIfSupported<testing::Test,
> void>(testing::Test*, void (testing::Test::*)(), char const*) <null>
> (llvm-codegen-test+0x412afd2)
> #14 __libc_start_main
> /build/glibc-LK5gWL/glibc-2.23/csu/../csu/libc-start.c:291 (libc.so.6+0x2082f)
> Mutex M3832 (0x7b4400100460) created at:
> #0 pthread_mutex_init
> /mnt/source/llvm/llvm-5.0.1.src-p1/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors.cc:1105
> (llvm-codegen-test+0x189e564)
> #1 kudu::Mutex::Mutex()
> /home/systest/Impala/be/src/kudu/util/mutex.cc:65:8
> (llvm-codegen-test+0x1a15e1f)
> #2 kudu::KernelStackWatchdog::KernelStackWatchdog()
> /home/systest/Impala/be/src/kudu/util/kernel_stack_watchdog.cc:64:22
> (llvm-codegen-test+0x19ec3c9)
> #3 Singleton<kudu::KernelStackWatchdog>::CreateInstance()
> /home/systest/Impala/be/src/kudu/gutil/singleton.h:124:18
> (llvm-codegen-test+0x19ef744)
> #4 Singleton<kudu::KernelStackWatchdog>::Init()
> /home/systest/Impala/be/src/kudu/gutil/singleton.h:117:17
> (llvm-codegen-test+0x19ef6e4)
> #5 GoogleOnceInternalInit(int*, void (*)(), void (*)(void*), void*)
> /home/systest/Impala/be/src/gutil/once.cc:38:7 (llvm-codegen-test+0x4005393)
> #6 GoogleOnceInit(GoogleOnceType*, void (*)())
> /home/systest/Impala/be/src/kudu/gutil/once.h:55:5
> (llvm-codegen-test+0x19c8ecd)
> #7 Singleton<kudu::KernelStackWatchdog>::get()
> /home/systest/Impala/be/src/kudu/gutil/singleton.h:79:5
> (llvm-codegen-test+0x19ef691)
> #8 kudu::KernelStackWatchdog::GetInstance()
> /home/systest/Impala/be/src/kudu/util/kernel_stack_watchdog.h:87:12
> (llvm-codegen-test+0x19ee073)
> #9 kudu::KernelStackWatchdog::CreateAndRegisterTLS()
> /home/systest/Impala/be/src/kudu/util/kernel_stack_watchdog.cc:219:3
> (llvm-codegen-test+0x19ed3b7)
> #10 kudu::KernelStackWatchdog::GetTLS()
> /home/systest/Impala/be/src/kudu/util/kernel_stack_watchdog.h:170:7
> (llvm-codegen-test+0x19cd7e1)
> #11 kudu::ScopedWatchKernelStack::ScopedWatchKernelStack(char const*,
> int) /home/systest/Impala/be/src/kudu/util/kernel_stack_watchdog.h:248:13
> (llvm-codegen-test+0x19c99f8)
> #12 kudu::Thread::StartThread(std::string const&, std::string const&,
> boost::function<void ()> const&, unsigned long, scoped_refptr<kudu::Thread>*)
> /home/systest/Impala/be/src/kudu/util/thread.cc:555:28
> (llvm-codegen-test+0x19c7129)
> #13 kudu::Status kudu::Thread::Create<void
> (kudu::rpc::ReactorThread::*)(), kudu::rpc::ReactorThread*>(std::string
> const&, std::string const&, void (kudu::rpc::ReactorThread::* const&)(),
> kudu::rpc::ReactorThread* const&, scoped_refptr<kudu::Thread>*) /home/systest/
> Impala/be/src/kudu/util/thread.h:164:12 (llvm-codegen-test+0x26e1875)
> #14 kudu::rpc::ReactorThread::Init()
> /home/systest/Impala/be/src/kudu/rpc/reactor.cc:185:10
> (llvm-codegen-test+0x26dc126)
> #15 kudu::rpc::Reactor::Init()
> /home/systest/Impala/be/src/kudu/rpc/reactor.cc:759:18
> (llvm-codegen-test+0x26e06d1)
> #16 kudu::rpc::Messenger::Init()
> /home/systest/Impala/be/src/kudu/rpc/messenger.cc:446:41
> (llvm-codegen-test+0x26c9a8d)
> #17
> kudu::rpc::MessengerBuilder::Build(std::shared_ptr<kudu::rpc::Messenger>*)
> /home/systest/Impala/be/src/kudu/rpc/messenger.cc:205:46
> (llvm-codegen-test+0x26c95fe)
> #18 impala::RpcMgr::Init(impala::TNetworkAddress const&)
> /home/systest/Impala/be/src/rpc/rpc-mgr.cc:151:38
> (llvm-codegen-test+0x2722a62)
> #19 impala::TestEnv::Init()
> /home/systest/Impala/be/src/runtime/test-env.cc:82:64
> (llvm-codegen-test+0x1b3df91)
> #20 impala::LlvmCodeGenTest::SetUp()
> /home/systest/Impala/be/src/codegen/llvm-codegen-test.cc:49:46
> (llvm-codegen-test+0x1913e02)
> #21 void
> testing::internal::HandleExceptionsInMethodIfSupported<testing::Test,
> void>(testing::Test*, void (testing::Test::*)(), char const*) <null>
> (llvm-codegen-test+0x412afd2)
> #22 __libc_start_main
> /build/glibc-LK5gWL/glibc-2.23/csu/../csu/libc-start.c:291 (libc.so.6+0x2082f)
> Thread T188 'kernel-watcher-' (tid=8413, running) created by main thread at:
> #0 pthread_create
> /mnt/source/llvm/llvm-5.0.1.src-p1/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors.cc:889
> (llvm-codegen-test+0x189ea9b)
> #1 kudu::Thread::StartThread(std::string const&, std::string const&,
> boost::function<void ()> const&, unsigned long, scoped_refptr<kudu::Thread>*)
> /home/systest/Impala/be/src/kudu/util/thread.cc:556:15
> (llvm-codegen-test+0x19c7155)
> #2 kudu::Status kudu::Thread::CreateWithFlags<boost::_bi::bind_t<void,
> boost::_mfi::mf0<void, kudu::KernelStackWatchdog>,
> boost::_bi::list1<boost::_bi::value<kudu::KernelStackWatchdog*> > >
> >(std::string const&, std::string const&, boost::_bi::bind_t<void, boost::_m
> fi::mf0<void, kudu::KernelStackWatchdog>,
> boost::_bi::list1<boost::_bi::value<kudu::KernelStackWatchdog*> > > const&,
> unsigned long, scoped_refptr<kudu::Thread>*)
> /home/systest/Impala/be/src/kudu/util/thread.h:152:12
> (llvm-codegen-test+0x19ed771)
> #3 kudu::KernelStackWatchdog::KernelStackWatchdog()
> /home/systest/Impala/be/src/kudu/util/kernel_stack_watchdog.cc:71:36
> (llvm-codegen-test+0x19ec45d)
> #4 Singleton<kudu::KernelStackWatchdog>::CreateInstance()
> /home/systest/Impala/be/src/kudu/gutil/singleton.h:124:18
> (llvm-codegen-test+0x19ef744)
> #5 Singleton<kudu::KernelStackWatchdog>::Init()
> /home/systest/Impala/be/src/kudu/gutil/singleton.h:117:17
> (llvm-codegen-test+0x19ef6e4)
> #6 GoogleOnceInternalInit(int*, void (*)(), void (*)(void*), void*)
> /home/systest/Impala/be/src/gutil/once.cc:38:7 (llvm-codegen-test+0x4005393)
> #7 GoogleOnceInit(GoogleOnceType*, void (*)())
> /home/systest/Impala/be/src/kudu/gutil/once.h:55:5
> (llvm-codegen-test+0x19c8ecd)
> #8 Singleton<kudu::KernelStackWatchdog>::get()
> /home/systest/Impala/be/src/kudu/gutil/singleton.h:79:5
> (llvm-codegen-test+0x19ef691)
> #9 kudu::KernelStackWatchdog::GetInstance()
> /home/systest/Impala/be/src/kudu/util/kernel_stack_watchdog.h:87:12
> (llvm-codegen-test+0x19ee073)
> #10 kudu::KernelStackWatchdog::CreateAndRegisterTLS()
> /home/systest/Impala/be/src/kudu/util/kernel_stack_watchdog.cc:219:3
> (llvm-codegen-test+0x19ed3b7)
> #11 kudu::KernelStackWatchdog::GetTLS()
> /home/systest/Impala/be/src/kudu/util/kernel_stack_watchdog.h:170:7
> (llvm-codegen-test+0x19cd7e1)
> #12 kudu::ScopedWatchKernelStack::ScopedWatchKernelStack(char const*,
> int) /home/systest/Impala/be/src/kudu/util/kernel_stack_watchdog.h:248:13
> (llvm-codegen-test+0x19c99f8)
> #13 kudu::Thread::StartThread(std::string const&, std::string const&,
> boost::function<void ()> const&, unsigned long, scoped_refptr<kudu::Thread>*)
> /home/systest/Impala/be/src/kudu/util/thread.cc:555:28
> (llvm-codegen-test+0x19c7129)
> #14 kudu::Status kudu::Thread::Create<void
> (kudu::rpc::ReactorThread::*)(), kudu::rpc::ReactorThread*>(std::string
> const&, std::string const&, void (kudu::rpc::ReactorThread::* const&)(),
> kudu::rpc::ReactorThread* const&, scoped_refptr<kudu::Thread>*) /home/systest/
> Impala/be/src/kudu/util/thread.h:164:12 (llvm-codegen-test+0x26e1875)
> #15 kudu::rpc::ReactorThread::Init()
> /home/systest/Impala/be/src/kudu/rpc/reactor.cc:185:10
> (llvm-codegen-test+0x26dc126)
> #16 kudu::rpc::Reactor::Init()
> /home/systest/Impala/be/src/kudu/rpc/reactor.cc:759:18
> (llvm-codegen-test+0x26e06d1)
> #17 kudu::rpc::Messenger::Init()
> /home/systest/Impala/be/src/kudu/rpc/messenger.cc:446:41
> (llvm-codegen-test+0x26c9a8d)
> #18
> kudu::rpc::MessengerBuilder::Build(std::shared_ptr<kudu::rpc::Messenger>*)
> /home/systest/Impala/be/src/kudu/rpc/messenger.cc:205:46
> (llvm-codegen-test+0x26c95fe)
> #19 impala::RpcMgr::Init(impala::TNetworkAddress const&)
> /home/systest/Impala/be/src/rpc/rpc-mgr.cc:151:38
> (llvm-codegen-test+0x2722a62)
> #20 impala::TestEnv::Init()
> /home/systest/Impala/be/src/runtime/test-env.cc:82:64
> (llvm-codegen-test+0x1b3df91)
> #21 impala::LlvmCodeGenTest::SetUp()
> /home/systest/Impala/be/src/codegen/llvm-codegen-test.cc:49:46
> (llvm-codegen-test+0x1913e02)
> #22 void
> testing::internal::HandleExceptionsInMethodIfSupported<testing::Test,
> void>(testing::Test*, void (testing::Test::*)(), char const*) <null>
> (llvm-codegen-test+0x412afd2)
> #23 __libc_start_main
> /build/glibc-LK5gWL/glibc-2.23/csu/../csu/libc-start.c:291 (libc.so.6+0x2082f)
> SUMMARY: ThreadSanitizer: data race
> /home/systest/Impala/be/src/gutil/atomicops-internals-tsan.h:104:3 in
> base::subtle::Acquire_Store(int volatile*, int)
> {code}
--
This message was sent by Atlassian Jira
(v8.3.4#803005)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
