Hao Hao created KUDU-2916:
-----------------------------
Summary: Admin.TestDumpMemTrackers is flaky in tsan
Key: KUDU-2916
URL: https://issues.apache.org/jira/browse/KUDU-2916
Project: Kudu
Issue Type: Bug
Affects Versions: 1.10.0
Reporter: Hao Hao
I saw a tsan failure for AdminCliTest.TestDumpMemTrackers with the following
log:
{noformat}
/data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/tools/kudu-admin-test.cc:2162:
Failure
Value of: s.ok()
Actual: false
Expected: true
Runtime error: /tmp/dist-test-taskUWtx7r/build/tsan/bin/kudu: process exited
with non-zero status 66
stdout:
{"id":"root","limit":-1,"current_consumption":481,"peak_consumption":481,"child_trackers":[{"id":"server","parent_id":"root","limit":-1,"current_consumption":313,"peak_consumption":313,"child_trackers":[{"id":"result-tracker","parent_id":"server","limit":-1,"current_consumption":0,"peak_consumption":0},{"id":"log_block_manager","parent_id":"server","limit":-1,"current_consumption":48,"peak_consumption":48},{"id":"tablet-9276b163452b4b0399ff2cae579f7251","parent_id":"server","limit":-1,"current_consumption":265,"peak_consumption":265,"child_trackers":[{"id":"DeltaMemStores","parent_id":"tablet-9276b163452b4b0399ff2cae579f7251","limit":-1,"current_consumption":0,"peak_consumption":0},{"id":"MemRowSet-0","parent_id":"tablet-9276b163452b4b0399ff2cae579f7251","limit":-1,"current_consumption":265,"peak_consumption":265},{"id":"txn_tracker","parent_id":"tablet-9276b163452b4b0399ff2cae579f7251","limit":67108864,"current_consumption":0,"peak_consumption":0}]}]},{"id":"ttl-cache-sharded_fifo_cache","parent_id":"root","limit":-1,"current_consumption":0,"peak_consumption":0},{"id":"code_cache-sharded_lru_cache","parent_id":"root","limit":-1,"current_consumption":0,"peak_consumption":0},{"id":"block_cache-sharded_lru_cache","parent_id":"root","limit":-1,"current_consumption":0,"peak_consumption":0},{"id":"lbm-sharded_lru_cache","parent_id":"root","limit":-1,"current_consumption":0,"peak_consumption":0},{"id":"log_cache","parent_id":"root","limit":1073741824,"current_consumption":168,"peak_consumption":168,"child_trackers":[{"id":"log_cache:457a3168758d4f4f8f4c59e8dd179cd3:9276b163452b4b0399ff2cae579f7251","parent_id":"log_cache","limit":10485760,"current_consumption":168,"peak_consumption":168}]}]}
stderr: W0803 14:03:00.206982 8443 flags.cc:404] Enabled unsafe flag:
--never_fsync=true
W0803 14:03:00.849385 8443 thread.cc:599] rpc reactor (reactor) Time spent
creating pthread: real 0.590s user 0.230s sys 0.360s
W0803 14:03:00.849658 8443 thread.cc:566] rpc reactor (reactor) Time spent
starting thread: real 0.591s user 0.230s sys 0.360s
==================
WARNING: ThreadSanitizer: destroy of a locked mutex (pid=8443)
#0 pthread_rwlock_destroy
/data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/llvm-6.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors.cc:1313
(kudu+0x4bbb24)
#1 glog_internal_namespace_::Mutex::~Mutex()
/data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/glog-0.3.5/src/base/mutex.h:249:30
(libglog.so.0+0x16488)
#2 cxa_at_exit_wrapper(void*)
/data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/llvm-6.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors.cc:386
(kudu+0x48beb3)
and:
#0 operator new(unsigned long)
/data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/llvm-6.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_new_delete.cc:57
(kudu+0x52ae83)
#1 __allocate
/data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/llvm-6.0.0.src/projects/libcxx/include/new:228:10
(libc++.so.1+0xd63f3)
#2 allocate
/data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/llvm-6.0.0.src/projects/libcxx/include/memory:1793
(libc++.so.1+0xd63f3)
#3 allocate
/data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/llvm-6.0.0.src/projects/libcxx/include/memory:1547
(libc++.so.1+0xd63f3)
#4 __init
/data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/llvm-6.0.0.src/projects/libcxx/include/string:1591
(libc++.so.1+0xd63f3)
#5 std::__1::basic_string<char, std::__1::char_traits<char>,
std::__1::allocator<char> >::basic_string(std::__1::basic_string<char,
std::__1::char_traits<char>, std::__1::allocator<char> > const&)
/data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/llvm-6.0.0.src/projects/libcxx/include/string:1653
(libc++.so.1+0xd63f3)
#6 std::__1::pair<std::__1::basic_string<char, std::__1::char_traits<char>,
std::__1::allocator<char> > const, std::__1::pair<void const*, int>
>::pair(std::__1::pair<std::__1::basic_string<char,
std::__1::char_traits<char>, std::__1::allocator<char> > const,
std::__1::pair<void const*, int> > const&)
/data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/c++/v1/utility:324:5
(libprotobuf.so.14+0x188711)
#7 void
std::__1::allocator<std::__1::__tree_node<std::__1::__value_type<std::__1::basic_string<char,
std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::pair<void
const*, int> >, void*> >::construct<std::__1::pair<std::__1::basic_string<char,
std::__1::char_traits<char>, std::__1::allocator<char> > const,
std::__1::pair<void const*, int> >, std::__1::pair<std::__1::basic_string<char,
std::__1::char_traits<char>, std::__1::allocator<char> > const,
std::__1::pair<void const*, int> >
const&>(std::__1::pair<std::__1::basic_string<char,
std::__1::char_traits<char>, std::__1::allocator<char> > const,
std::__1::pair<void const*, int> >*,
std::__1::pair<std::__1::basic_string<char, std::__1::char_traits<char>,
std::__1::allocator<char> > const, std::__1::pair<void const*, int> > const&)
/data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/c++/v1/memory:1805
(libprotobuf.so.14+0x188711)
#8 void
std::__1::allocator_traits<std::__1::allocator<std::__1::__tree_node<std::__1::__value_type<std::__1::basic_string<char,
std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::pair<void
const*, int> >, void*> >
>::__construct<std::__1::pair<std::__1::basic_string<char,
std::__1::char_traits<char>, std::__1::allocator<char> > const,
std::__1::pair<void const*, int> >, std::__1::pair<std::__1::basic_string<char,
std::__1::char_traits<char>, std::__1::allocator<char> > const,
std::__1::pair<void const*, int> > const&>(std::__1::integral_constant<bool,
true>,
std::__1::allocator<std::__1::__tree_node<std::__1::__value_type<std::__1::basic_string<char,
std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::pair<void
const*, int> >, void*> >&, std::__1::pair<std::__1::basic_string<char,
std::__1::char_traits<char>, std::__1::allocator<char> > const,
std::__1::pair<void const*, int> >*,
std::__1::pair<std::__1::basic_string<char, std::__1::char_traits<char>,
std::__1::allocator<char> > const, std::__1::pair<void const*, int> > const&)
/data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/c++/v1/memory:1715
(libprotobuf.so.14+0x188711)
#9 void
std::__1::allocator_traits<std::__1::allocator<std::__1::__tree_node<std::__1::__value_type<std::__1::basic_string<char,
std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::pair<void
const*, int> >, void*> >
>::construct<std::__1::pair<std::__1::basic_string<char,
std::__1::char_traits<char>, std::__1::allocator<char> > const,
std::__1::pair<void const*, int> >, std::__1::pair<std::__1::basic_string<char,
std::__1::char_traits<char>, std::__1::allocator<char> > const,
std::__1::pair<void const*, int> >
const&>(std::__1::allocator<std::__1::__tree_node<std::__1::__value_type<std::__1::basic_string<char,
std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::pair<void
const*, int> >, void*> >&, std::__1::pair<std::__1::basic_string<char,
std::__1::char_traits<char>, std::__1::allocator<char> > const,
std::__1::pair<void const*, int> >*,
std::__1::pair<std::__1::basic_string<char, std::__1::char_traits<char>,
std::__1::allocator<char> > const, std::__1::pair<void const*, int> > const&)
/data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/c++/v1/memory:1561
(libprotobuf.so.14+0x188711)
#10
std::__1::unique_ptr<std::__1::__tree_node<std::__1::__value_type<std::__1::basic_string<char,
std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::pair<void
const*, int> >, void*>,
std::__1::__tree_node_destructor<std::__1::allocator<std::__1::__tree_node<std::__1::__value_type<std::__1::basic_string<char,
std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::pair<void
const*, int> >, void*> > > >
std::__1::__tree<std::__1::__value_type<std::__1::basic_string<char,
std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::pair<void
const*, int> >, std::__1::__map_value_compare<std::__1::basic_string<char,
std::__1::char_traits<char>, std::__1::allocator<char> >,
std::__1::__value_type<std::__1::basic_string<char,
std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::pair<void
const*, int> >, std::__1::less<std::__1::basic_string<char,
std::__1::char_traits<char>, std::__1::allocator<char> > >, true>,
std::__1::allocator<std::__1::__value_type<std::__1::basic_string<char,
std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::pair<void
const*, int> > >
>::__construct_node<std::__1::pair<std::__1::basic_string<char,
std::__1::char_traits<char>, std::__1::allocator<char> > const,
std::__1::pair<void const*, int> >
const&>(std::__1::pair<std::__1::basic_string<char,
std::__1::char_traits<char>, std::__1::allocator<char> > const,
std::__1::pair<void const*, int> > const&)
/data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/c++/v1/__tree:2192
(libprotobuf.so.14+0x188711)
#11
std::__1::pair<std::__1::__tree_iterator<std::__1::__value_type<std::__1::basic_string<char,
std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::pair<void
const*, int> >,
std::__1::__tree_node<std::__1::__value_type<std::__1::basic_string<char,
std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::pair<void
const*, int> >, void*>*, long>, bool>
std::__1::__tree<std::__1::__value_type<std::__1::basic_string<char,
std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::pair<void
const*, int> >, std::__1::__map_value_compare<std::__1::basic_string<char,
std::__1::char_traits<char>, std::__1::allocator<char> >,
std::__1::__value_type<std::__1::basic_string<char,
std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::pair<void
const*, int> >, std::__1::less<std::__1::basic_string<char,
std::__1::char_traits<char>, std::__1::allocator<char> > >, true>,
std::__1::allocator<std::__1::__value_type<std::__1::basic_string<char,
std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::pair<void
const*, int> > > >::__emplace_unique_key_args<std::__1::basic_string<char,
std::__1::char_traits<char>, std::__1::allocator<char> >,
std::__1::pair<std::__1::basic_string<char, std::__1::char_traits<char>,
std::__1::allocator<char> > const, std::__1::pair<void const*, int> >
const&>(std::__1::basic_string<char, std::__1::char_traits<char>,
std::__1::allocator<char> > const&, std::__1::pair<std::__1::basic_string<char,
std::__1::char_traits<char>, std::__1::allocator<char> > const,
std::__1::pair<void const*, int> > const&)
/data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/c++/v1/__tree:2137
(libprotobuf.so.14+0x188711)
#12 bool
google::protobuf::InsertIfNotPresent<std::__1::map<std::__1::basic_string<char,
std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::pair<void
const*, int>, std::__1::less<std::__1::basic_string<char,
std::__1::char_traits<char>, std::__1::allocator<char> > >,
std::__1::allocator<std::__1::pair<std::__1::basic_string<char,
std::__1::char_traits<char>, std::__1::allocator<char> > const,
std::__1::pair<void const*, int> > > >
>(std::__1::map<std::__1::basic_string<char, std::__1::char_traits<char>,
std::__1::allocator<char> >, std::__1::pair<void const*, int>,
std::__1::less<std::__1::basic_string<char, std::__1::char_traits<char>,
std::__1::allocator<char> > >,
std::__1::allocator<std::__1::pair<std::__1::basic_string<char,
std::__1::char_traits<char>, std::__1::allocator<char> > const,
std::__1::pair<void const*, int> > > >*,
std::__1::map<std::__1::basic_string<char, std::__1::char_traits<char>,
std::__1::allocator<char> >, std::__1::pair<void const*, int>,
std::__1::less<std::__1::basic_string<char, std::__1::char_traits<char>,
std::__1::allocator<char> > >,
std::__1::allocator<std::__1::pair<std::__1::basic_string<char,
std::__1::char_traits<char>, std::__1::allocator<char> > const,
std::__1::pair<void const*, int> > > >::value_type::first_type const&,
std::__1::map<std::__1::basic_string<char, std::__1::char_traits<char>,
std::__1::allocator<char> >, std::__1::pair<void const*, int>,
std::__1::less<std::__1::basic_string<char, std::__1::char_traits<char>,
std::__1::allocator<char> > >,
std::__1::allocator<std::__1::pair<std::__1::basic_string<char,
std::__1::char_traits<char>, std::__1::allocator<char> > const,
std::__1::pair<void const*, int> > > >::value_type::second_type const&)
/data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/c++/v1/__tree:1273:16
(libprotobuf.so.14+0x184c18)
#13
google::protobuf::SimpleDescriptorDatabase::DescriptorIndex<std::__1::pair<void
const*, int> >::AddFile(google::protobuf::FileDescriptorProto const&,
std::__1::pair<void const*, int>)
/data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/protobuf-3.4.1/src/google/protobuf/descriptor_database.cc:57
(libprotobuf.so.14+0x184c18)
#14 google::protobuf::EncodedDescriptorDatabase::Add(void const*, int)
/data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/protobuf-3.4.1/src/google/protobuf/descriptor_database.cc:316:19
(libprotobuf.so.14+0x182d7c)
#15 google::protobuf::DescriptorPool::InternalAddGeneratedFile(void const*,
int)
/data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/protobuf-3.4.1/src/google/protobuf/descriptor.cc:1315:3
(libprotobuf.so.14+0x13b705)
#16 google::protobuf::protobuf_google_2fprotobuf_2ftype_2eproto::(anonymous
namespace)::AddDescriptorsImpl()
/data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/protobuf-3.4.1/src/google/protobuf/type.pb.cc:240:3
(libprotobuf.so.14+0x237c10)
#17 google::protobuf::internal::FunctionClosure0::Run()
/data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/protobuf-3.4.1/src/google/protobuf/stubs/callback.h:129:5
(libprotobuf.so.14+0xd330b)
#18 google::protobuf::GoogleOnceInitImpl(long*, google::protobuf::Closure*)
/data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/protobuf-3.4.1/src/google/protobuf/stubs/once.cc:83:14
(libprotobuf.so.14+0xd5d6a)
#19 google::protobuf::GoogleOnceInit(long*, void (*)())
/data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/protobuf-3.4.1/src/google/protobuf/stubs/once.h:128:5
(libprotobuf.so.14+0x237b66)
#20
google::protobuf::protobuf_google_2fprotobuf_2ftype_2eproto::AddDescriptors()
/data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/protobuf-3.4.1/src/google/protobuf/type.pb.cc:251
(libprotobuf.so.14+0x237b66)
#21 google::protobuf::protobuf_google_2fprotobuf_2fapi_2eproto::(anonymous
namespace)::AddDescriptorsImpl()
/data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/protobuf-3.4.1/src/google/protobuf/api.pb.cc:179:3
(libprotobuf.so.14+0x12ca1d)
#22 google::protobuf::internal::FunctionClosure0::Run()
/data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/protobuf-3.4.1/src/google/protobuf/stubs/callback.h:129:5
(libprotobuf.so.14+0xd330b)
#23 google::protobuf::GoogleOnceInitImpl(long*, google::protobuf::Closure*)
/data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/protobuf-3.4.1/src/google/protobuf/stubs/once.cc:83:14
(libprotobuf.so.14+0xd5d6a)
#24 google::protobuf::GoogleOnceInit(long*, void (*)())
/data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/protobuf-3.4.1/src/google/protobuf/stubs/once.h:128:5
(libprotobuf.so.14+0xcf0a6)
#25
google::protobuf::protobuf_google_2fprotobuf_2fapi_2eproto::AddDescriptors()
/data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/protobuf-3.4.1/src/google/protobuf/api.pb.cc:185
(libprotobuf.so.14+0xcf0a6)
#26
google::protobuf::protobuf_google_2fprotobuf_2fapi_2eproto::StaticDescriptorInitializer::StaticDescriptorInitializer()
/data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/protobuf-3.4.1/src/google/protobuf/api.pb.cc:190
(libprotobuf.so.14+0xcf0a6)
#27 __cxx_global_var_init
/data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/protobuf-3.4.1/src/google/protobuf/api.pb.cc:192
(libprotobuf.so.14+0xcf0a6)
#28 _GLOBAL__sub_I_api.pb.cc
/data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/protobuf-3.4.1/src/google/protobuf/api.pb.cc
(libprotobuf.so.14+0xcf0a6)
#29 call_init /build/eglibc-SvCtMH/eglibc-2.19/elf/dl-init.c:78
(ld-linux-x86-64.so.2+0x102d9)
Location is global 'google::vmodule_lock' of size 64 at 0x7f67b632f878
(libglog.so.0+0x00000023c878)
Mutex M134 (0x7f67b632f878) created at:
#0 pthread_rwlock_init
/data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/llvm-6.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors.cc:1304
(kudu+0x4bbd94)
#1 glog_internal_namespace_::Mutex::Mutex()
/data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/glog-0.3.5/src/base/mutex.h:247:19
(libglog.so.0+0xcca6)
#2 __cxx_global_var_init.15
/data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/glog-0.3.5/src/vlog_is_on.cc:120
(libglog.so.0+0xcca6)
#3 _GLOBAL__sub_I_vlog_is_on.cc
/data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/glog-0.3.5/src/vlog_is_on.cc
(libglog.so.0+0xcca6)
#4 call_init /build/eglibc-SvCtMH/eglibc-2.19/elf/dl-init.c:78
(ld-linux-x86-64.so.2+0x102d9)
SUMMARY: ThreadSanitizer: destroy of a locked mutex
/data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/glog-0.3.5/src/base/mutex.h:249:30
in glog_internal_namespace_::Mutex::~Mutex()
==================
==================
WARNING: ThreadSanitizer: lock-order-inversion (potential deadlock) (pid=8443)
Cycle in lock order graph: M1860 (0x7b14000167b8) => M37857730294380664
(0x000000000000) => M1860
Mutex M37857730294380664 acquired here while holding mutex M1860 in main
thread:
#0 pthread_rwlock_wrlock
/data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/llvm-6.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors.cc:1352
(kudu+0x4bb154)
#1 glog_internal_namespace_::Mutex::Lock()
/data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/glog-0.3.5/src/base/mutex.h:250:30
(libglog.so.0+0x1abb7)
#2
glog_internal_namespace_::MutexLock::MutexLock(glog_internal_namespace_::Mutex*)
/data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/glog-0.3.5/src/base/mutex.h:290
(libglog.so.0+0x1abb7)
#3 google::InitVLOG3__(int**, int*, char const*, int)
/data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/glog-0.3.5/src/vlog_is_on.cc:199
(libglog.so.0+0x1abb7)
#4
kudu::rpc::Messenger::ShutdownInternal(kudu::rpc::Messenger::ShutdownMode)
/data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/rpc/messenger.cc:283:5
(libkrpc.so+0xa9471)
#5 kudu::rpc::Messenger::AllExternalReferencesDropped()
/data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/rpc/messenger.cc:249:3
(libkrpc.so+0xa9227)
#6 std::__1::mem_fun_t<void,
kudu::rpc::Messenger>::operator()(kudu::rpc::Messenger*) const
/data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/c++/v1/functional:1120:17
(libkrpc.so+0xad755)
#7 std::__1::__shared_ptr_pointer<kudu::rpc::Messenger*,
std::__1::mem_fun_t<void, kudu::rpc::Messenger>,
std::__1::allocator<kudu::rpc::Messenger> >::__on_zero_shared()
/data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/c++/v1/memory:3586
(libkrpc.so+0xad755)
#8 std::__1::__shared_count::__release_shared()
/data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/c++/v1/memory:3490:9
(kudu+0x576f2e)
#9 std::__1::__shared_weak_count::__release_shared()
/data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/c++/v1/memory:3532
(kudu+0x576f2e)
#10 std::__1::shared_ptr<kudu::rpc::Messenger>::~shared_ptr()
/data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/c++/v1/memory:4468
(kudu+0x576f2e)
#11 kudu::rpc::Proxy::~Proxy()
/data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/rpc/proxy.cc:72:1
(libkrpc.so+0xbd0f3)
#12 kudu::server::GenericServiceProxy::~GenericServiceProxy()
/data/somelongdirectorytoavoidrpathissues/src/kudu/build/tsan/src/kudu/server/server_base.proxy.cc:26:1
(libserver_base_proto.so+0x4bd79)
#13
std::__1::default_delete<kudu::server::GenericServiceProxy>::operator()(kudu::server::GenericServiceProxy*)
const
/data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/c++/v1/memory:2285:5
(libkudu_tools_util.so+0x74acf)
#14 std::__1::unique_ptr<kudu::server::GenericServiceProxy,
std::__1::default_delete<kudu::server::GenericServiceProxy>
>::reset(kudu::server::GenericServiceProxy*)
/data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/c++/v1/memory:2598
(libkudu_tools_util.so+0x74acf)
#15 std::__1::unique_ptr<kudu::server::GenericServiceProxy,
std::__1::default_delete<kudu::server::GenericServiceProxy> >::~unique_ptr()
/data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/c++/v1/memory:2552
(libkudu_tools_util.so+0x74acf)
#16 kudu::tools::DumpMemTrackers(std::__1::basic_string<char,
std::__1::char_traits<char>, std::__1::allocator<char> > const&, unsigned
short)
/data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/tools/tool_action_common.cc:629
(libkudu_tools_util.so+0x74acf)
#17 kudu::tools::(anonymous
namespace)::TserverDumpMemTrackers(kudu::tools::RunnerContext const&)
/data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/tools/tool_action_tserver.cc:161:10
(kudu+0x5b8599)
#18
_ZNSt3__18__invokeIRPFN4kudu6StatusERKNS1_5tools13RunnerContextEEJS6_EEEDTclclsr3std3__1E7forwardIT_Efp_Espclsr3std3__1E7forwardIT0_Efp0_EEEOSA_DpOSB_
/data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/c++/v1/type_traits:4482:1
(kudu+0x53956b)
#19 kudu::Status
std::__1::__invoke_void_return_wrapper<kudu::Status>::__call<kudu::Status
(*&)(kudu::tools::RunnerContext const&), kudu::tools::RunnerContext
const&>(kudu::Status (*&)(kudu::tools::RunnerContext const&),
kudu::tools::RunnerContext const&)
/data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/c++/v1/__functional_base:318
(kudu+0x53956b)
#20 std::__1::__function::__func<kudu::Status
(*)(kudu::tools::RunnerContext const&), std::__1::allocator<kudu::Status
(*)(kudu::tools::RunnerContext const&)>, kudu::Status
(kudu::tools::RunnerContext const&)>::operator()(kudu::tools::RunnerContext
const&)
/data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/c++/v1/functional:1562:12
(kudu+0x53949d)
#21 std::__1::function<kudu::Status (kudu::tools::RunnerContext
const&)>::operator()(kudu::tools::RunnerContext const&) const
/data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/c++/v1/functional:1916:12
(libkudu_tools_util.so+0x6b2a4)
#22 kudu::tools::Action::Run(std::__1::vector<kudu::tools::Mode*,
std::__1::allocator<kudu::tools::Mode*> > const&,
std::__1::unordered_map<std::__1::basic_string<char,
std::__1::char_traits<char>, std::__1::allocator<char> >,
std::__1::basic_string<char, std::__1::char_traits<char>,
std::__1::allocator<char> >, std::__1::hash<std::__1::basic_string<char,
std::__1::char_traits<char>, std::__1::allocator<char> > >,
std::__1::equal_to<std::__1::basic_string<char, std::__1::char_traits<char>,
std::__1::allocator<char> > >,
std::__1::allocator<std::__1::pair<std::__1::basic_string<char,
std::__1::char_traits<char>, std::__1::allocator<char> > const,
std::__1::basic_string<char, std::__1::char_traits<char>,
std::__1::allocator<char> > > > > const&,
std::__1::vector<std::__1::basic_string<char, std::__1::char_traits<char>,
std::__1::allocator<char> >, std::__1::allocator<std::__1::basic_string<char,
std::__1::char_traits<char>, std::__1::allocator<char> > > > const&) const
/data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/tools/tool_action.cc:258:10
(libkudu_tools_util.so+0x69974)
#23 kudu::tools::DispatchCommand(std::__1::vector<kudu::tools::Mode*,
std::__1::allocator<kudu::tools::Mode*> > const&, kudu::tools::Action*,
std::__1::deque<std::__1::basic_string<char, std::__1::char_traits<char>,
std::__1::allocator<char> >, std::__1::allocator<std::__1::basic_string<char,
std::__1::char_traits<char>, std::__1::allocator<char> > > > const&)
/data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/tools/tool_main.cc:132:15
(kudu+0x5bbcf6)
#24 kudu::tools::RunTool(int, char**, bool)
/data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/tools/tool_main.cc:204:16
(kudu+0x5bcc51)
#25 main
/data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/tools/tool_main.cc:265:10
(kudu+0x5bcfbe)
Hint: use TSAN_OPTIONS=second_deadlock_stack=1 to get more informative
warning message
Mutex M1860 acquired here while holding mutex M37857730294380664 in thread T7:
#0 AnnotateRWLockAcquired
/data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/llvm-6.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interface_ann.cc:271
(kudu+0x4dbd0c)
#1 kudu::rw_spinlock::lock()
/data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/util/locks.h:112:5
(libkudu_client.so+0x16d842)
#2 kudu::percpu_rwlock::lock()
/data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/util/locks.h:222:22
(libkudu_client.so+0x16d7d2)
#3
std::__1::lock_guard<kudu::percpu_rwlock>::lock_guard(kudu::percpu_rwlock&)
/data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/c++/v1/__mutex_base:104:27
(libkrpc.so+0xaad39)
#4 kudu::rpc::Messenger::~Messenger()
/data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/rpc/messenger.cc:430
(libkrpc.so+0xaad39)
#5
std::__1::default_delete<kudu::rpc::Messenger>::operator()(kudu::rpc::Messenger*)
const
/data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/c++/v1/memory:2285:5
(libkrpc.so+0xb081b)
#6 std::__1::__shared_ptr_pointer<kudu::rpc::Messenger*,
std::__1::default_delete<kudu::rpc::Messenger>,
std::__1::allocator<kudu::rpc::Messenger> >::__on_zero_shared()
/data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/c++/v1/memory:3586
(libkrpc.so+0xb081b)
#7 std::__1::__shared_count::__release_shared()
/data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/c++/v1/memory:3490:9
(kudu+0x576f2e)
#8 std::__1::__shared_weak_count::__release_shared()
/data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/c++/v1/memory:3532
(kudu+0x576f2e)
#9 std::__1::shared_ptr<kudu::rpc::Messenger>::~shared_ptr()
/data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/c++/v1/memory:4468
(kudu+0x576f2e)
#10 std::__1::shared_ptr<kudu::rpc::Messenger>::reset()
/data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/c++/v1/memory:4603:5
(libkrpc.so+0xbf151)
#11 kudu::rpc::ReactorThread::RunThread()
/data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/rpc/reactor.cc:499
(libkrpc.so+0xbf151)
#12 boost::_mfi::mf0<void,
kudu::rpc::ReactorThread>::operator()(kudu::rpc::ReactorThread*) const
/data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/boost/bind/mem_fn_template.hpp:49:29
(libkrpc.so+0xc9539)
#13 void boost::_bi::list1<boost::_bi::value<kudu::rpc::ReactorThread*>
>::operator()<boost::_mfi::mf0<void, kudu::rpc::ReactorThread>,
boost::_bi::list0>(boost::_bi::type<void>, boost::_mfi::mf0<void,
kudu::rpc::ReactorThread>&, boost::_bi::list0&, int)
/data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/boost/bind/bind.hpp:259:9
(libkrpc.so+0xc948a)
#14 boost::_bi::bind_t<void, boost::_mfi::mf0<void,
kudu::rpc::ReactorThread>,
boost::_bi::list1<boost::_bi::value<kudu::rpc::ReactorThread*> >
>::operator()()
/data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/boost/bind/bind.hpp:1222:16
(libkrpc.so+0xc9413)
#15
boost::detail::function::void_function_obj_invoker0<boost::_bi::bind_t<void,
boost::_mfi::mf0<void, kudu::rpc::ReactorThread>,
boost::_bi::list1<boost::_bi::value<kudu::rpc::ReactorThread*> > >,
void>::invoke(boost::detail::function::function_buffer&)
/data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/boost/function/function_template.hpp:159:11
(libkrpc.so+0xc9209)
#16 boost::function0<void>::operator()() const
/data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/boost/function/function_template.hpp:770:14
(libkrpc.so+0xb8a11)
#17 kudu::Thread::SuperviseThread(void*)
/data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/util/thread.cc:657:3
(libkudu_util.so+0x1e1824)
Thread T7 'rpc reactor-845' (tid=8451, running) created by main thread at:
#0 pthread_create
/data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/llvm-6.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors.cc:992
(kudu+0x4bd8eb)
#1 kudu::Thread::StartThread(std::__1::basic_string<char,
std::__1::char_traits<char>, std::__1::allocator<char> > const&,
std::__1::basic_string<char, std::__1::char_traits<char>,
std::__1::allocator<char> > const&, boost::function<void ()> const&, unsigned
long, scoped_refptr<kudu::Thread>*)
/data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/util/thread.cc:601:15
(libkudu_util.so+0x1e100b)
#2 kudu::Status kudu::Thread::Create<void (kudu::rpc::ReactorThread::*)(),
kudu::rpc::ReactorThread*>(std::__1::basic_string<char,
std::__1::char_traits<char>, std::__1::allocator<char> > const&,
std::__1::basic_string<char, std::__1::char_traits<char>,
std::__1::allocator<char> > const&, void (kudu::rpc::ReactorThread::*
const&)(), kudu::rpc::ReactorThread* const&, scoped_refptr<kudu::Thread>*)
/data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/util/thread.h:164:12
(libkrpc.so+0xc48e5)
#3 kudu::rpc::ReactorThread::Init()
/data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/rpc/reactor.cc:185:10
(libkrpc.so+0xbec4e)
#4 kudu::rpc::Reactor::Init()
/data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/rpc/reactor.cc:759:18
(libkrpc.so+0xc32f1)
#5 kudu::rpc::Messenger::Init()
/data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/rpc/messenger.cc:446:5
(libkrpc.so+0xa90e2)
#6
kudu::rpc::MessengerBuilder::Build(std::__1::shared_ptr<kudu::rpc::Messenger>*)
/data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/rpc/messenger.cc:205:3
(libkrpc.so+0xa8b3d)
#7 kudu::Status
kudu::tools::BuildProxy<kudu::server::GenericServiceProxy>(std::__1::basic_string<char,
std::__1::char_traits<char>, std::__1::allocator<char> > const&, unsigned
short, std::__1::unique_ptr<kudu::server::GenericServiceProxy,
std::__1::default_delete<kudu::server::GenericServiceProxy> >*)
/data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/tools/tool_action_common.cc:328:3
(libkudu_tools_util.so+0x7c55e)
#8 kudu::tools::DumpMemTrackers(std::__1::basic_string<char,
std::__1::char_traits<char>, std::__1::allocator<char> > const&, unsigned
short)
/data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/tools/tool_action_common.cc:591:3
(libkudu_tools_util.so+0x747e3)
#9 kudu::tools::(anonymous
namespace)::TserverDumpMemTrackers(kudu::tools::RunnerContext const&)
/data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/tools/tool_action_tserver.cc:161:10
(kudu+0x5b8599)
#10
_ZNSt3__18__invokeIRPFN4kudu6StatusERKNS1_5tools13RunnerContextEEJS6_EEEDTclclsr3std3__1E7forwardIT_Efp_Espclsr3std3__1E7forwardIT0_Efp0_EEEOSA_DpOSB_
/data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/c++/v1/type_traits:4482:1
(kudu+0x53956b)
#11 kudu::Status
std::__1::__invoke_void_return_wrapper<kudu::Status>::__call<kudu::Status
(*&)(kudu::tools::RunnerContext const&), kudu::tools::RunnerContext
const&>(kudu::Status (*&)(kudu::tools::RunnerContext const&),
kudu::tools::RunnerContext const&)
/data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/c++/v1/__functional_base:318
(kudu+0x53956b)
#12 std::__1::__function::__func<kudu::Status
(*)(kudu::tools::RunnerContext const&), std::__1::allocator<kudu::Status
(*)(kudu::tools::RunnerContext const&)>, kudu::Status
(kudu::tools::RunnerContext const&)>::operator()(kudu::tools::RunnerContext
const&)
/data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/c++/v1/functional:1562:12
(kudu+0x53949d)
#13 std::__1::function<kudu::Status (kudu::tools::RunnerContext
const&)>::operator()(kudu::tools::RunnerContext const&) const
/data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/c++/v1/functional:1916:12
(libkudu_tools_util.so+0x6b2a4)
#14 kudu::tools::Action::Run(std::__1::vector<kudu::tools::Mode*,
std::__1::allocator<kudu::tools::Mode*> > const&,
std::__1::unordered_map<std::__1::basic_string<char,
std::__1::char_traits<char>, std::__1::allocator<char> >,
std::__1::basic_string<char, std::__1::char_traits<char>,
std::__1::allocator<char> >, std::__1::hash<std::__1::basic_string<char,
std::__1::char_traits<char>, std::__1::allocator<char> > >,
std::__1::equal_to<std::__1::basic_string<char, std::__1::char_traits<char>,
std::__1::allocator<char> > >,
std::__1::allocator<std::__1::pair<std::__1::basic_string<char,
std::__1::char_traits<char>, std::__1::allocator<char> > const,
std::__1::basic_string<char, std::__1::char_traits<char>,
std::__1::allocator<char> > > > > const&,
std::__1::vector<std::__1::basic_string<char, std::__1::char_traits<char>,
std::__1::allocator<char> >, std::__1::allocator<std::__1::basic_string<char,
std::__1::char_traits<char>, std::__1::allocator<char> > > > const&) const
/data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/tools/tool_action.cc:258:10
(libkudu_tools_util.so+0x69974)
#15 kudu::tools::DispatchCommand(std::__1::vector<kudu::tools::Mode*,
std::__1::allocator<kudu::tools::Mode*> > const&, kudu::tools::Action*,
std::__1::deque<std::__1::basic_string<char, std::__1::char_traits<char>,
std::__1::allocator<char> >, std::__1::allocator<std::__1::basic_string<char,
std::__1::char_traits<char>, std::__1::allocator<char> > > > const&)
/data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/tools/tool_main.cc:132:15
(kudu+0x5bbcf6)
#16 kudu::tools::RunTool(int, char**, bool)
/data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/tools/tool_main.cc:204:16
(kudu+0x5bcc51)
#17 main
/data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/tools/tool_main.cc:265:10
(kudu+0x5bcfbe)
SUMMARY: ThreadSanitizer: lock-order-inversion (potential deadlock)
/data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/glog-0.3.5/src/base/mutex.h:250:30
in glog_internal_namespace_::Mutex::Lock()
==================
ThreadSanitizer: reported 2 warnings
{noformat}
--
This message was sent by Atlassian JIRA
(v7.6.14#76016)
