[
https://issues.apache.org/jira/browse/KUDU-3119?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=17176072#comment-17176072
]
Andrew Wong commented on KUDU-3119:
-----------------------------------
The race isn't quite where I expected, per the following lines in the logs:
{code:java}
Write of size 1 at 0x7f82f790a760 by thread T5 (mutexes: write M1638):
#0 spp::sparsegroup<>::_sizing(unsigned int)
/data/jenkins/workspace/kudu-pre-commit-unittest-TSAN/thirdparty/installed/common/include/sparsepp/spp.h:1103:56
(libkudu_fs.so+0x102d70)
#1 void
spp::sparsegroup<>::_set_aux<>(kudu::MemTrackerAllocator<std::__1::pair<>,
std::__1::allocator<std::__1::pair<> > >&, unsigned char, std::__1::pair<>&,
spp::integral_constant<bool, false>)
/data/jenkins/workspace/kudu-pre-commit-unittest-TSAN/thirdparty/installed/common/include/sparsepp/spp.h:1392:31
(libkudu_fs.so+0x102ac8)
#2 void
spp::sparsegroup<>::_set<>(kudu::MemTrackerAllocator<std::__1::pair<>,
std::__1::allocator<std::__1::pair<> > >&, unsigned char, unsigned char,
std::__1::pair<>&)
/data/jenkins/workspace/kudu-pre-commit-unittest-TSAN/thirdparty/installed/common/include/sparsepp/spp.h:1426:13
(libkudu_fs.so+0x102a56)
#3 std::__1::pair<>* spp::sparsegroup<>::set<std::__1::pair<>
>(kudu::MemTrackerAllocator<std::__1::pair<kudu::BlockId const,
scoped_refptr<kudu::fs::internal::LogBlock> >,
std::__1::allocator<std::__1::pair<kudu::BlockId const,
scoped_refptr<kudu::fs::internal::LogBlock> > > >&, unsigned char,
std::__1::pair<kudu::BlockId const, scoped_refptr<kudu::fs::internal::LogBlock>
>&)
/data/jenkins/workspace/kudu-pre-commit-unittest-TSAN/thirdparty/installed/common/include/sparsepp/spp.h:1444:9
(libkudu_fs.so+0x10295f)
#4 std::__1::pair<>& spp::sparsetable<>::set<std::__1::pair<> >(unsigned
long, std::__1::pair<>&)
/data/jenkins/workspace/kudu-pre-commit-unittest-TSAN/thirdparty/installed/common/include/sparsepp/spp.h:2236:25
(libkudu_fs.so+0x1036ba)
#5 std::__1::pair<>&
spp::sparse_hashtable<>::_insert_at<std::__1::pair<kudu::BlockId const,
scoped_refptr<kudu::fs::internal::LogBlock> > >(std::__1::pair<kudu::BlockId
const, scoped_refptr<kudu::fs::internal::LogBlock> >&, unsigned long, bool)
/data/jenkins/workspace/kudu-pre-commit-unittest-TSAN/thirdparty/installed/common/include/sparsepp/spp.h:3173:22
(libkudu_fs.so+0x101910)
#6 std::__1::pair<>&
spp::sparse_hashtable<>::find_or_insert<spp::sparse_hash_map<kudu::BlockId,
scoped_refptr<kudu::fs::internal::LogBlock>, kudu::BlockIdHash,
kudu::BlockIdEqual, kudu::MemTrackerAllocator<std::__1::pair<kudu::BlockId
const, scoped_refptr<kudu::fs::internal::LogBlock> >,
std::__1::allocator<std::__1::pair<kudu::BlockId const,
scoped_refptr<kudu::fs::internal::LogBlock> > > >
>::DefaultValue>(kudu::BlockId const&)
/data/jenkins/workspace/kudu-pre-commit-unittest-TSAN/thirdparty/installed/common/include/sparsepp/spp.h:3282:28
(libkudu_fs.so+0x1014a1)
#7 spp::sparse_hash_map<>::operator[](kudu::BlockId const&)
/data/jenkins/workspace/kudu-pre-commit-unittest-TSAN/thirdparty/installed/common/include/sparsepp/spp.h:3792:29
(libkudu_fs.so+0xeece0)
#8
kudu::fs::LogBlockManager::AddLogBlock(scoped_refptr<kudu::fs::internal::LogBlock>)
/data/jenkins/workspace/kudu-pre-commit-unittest-TSAN/src/kudu/fs/log_block_manager.cc:2262:32
(libkudu_fs.so+0xe6a27)
...
Previous read of size 1 at 0x7f82f790a760 by thread T6 (mutexes: write M1637):
#0 spp::sparsegroup<>::_sizing(unsigned int)
/data/jenkins/workspace/kudu-pre-commit-unittest-TSAN/thirdparty/installed/common/include/sparsepp/spp.h:1088:14
(libkudu_fs.so+0x102d1c)
#1 void spp::sparsegroup<>::_set_aux<std::__1::pair<kudu::BlockId const,
scoped_refptr<kudu::fs::internal::LogBlock> >
>(kudu::MemTrackerAllocator<std::__1::pair<kudu::BlockId const,
scoped_refptr<kudu::fs::internal::LogBlock> >,
std::__1::allocator<std::__1::pair<kudu::BlockId const,
scoped_refptr<kudu::fs::internal::LogBlock> > > >&, unsigned char,
std::__1::pair<kudu::BlockId const, scoped_refptr<kudu::fs::internal::LogBlock>
>&, spp::integral_constant<bool, false>)
/data/jenkins/workspace/kudu-pre-commit-unittest-TSAN/thirdparty/installed/common/include/sparsepp/spp.h:1392:31
(libkudu_fs.so+0x102ac8)
#2 void
spp::sparsegroup<>::_set<>(kudu::MemTrackerAllocator<std::__1::pair<>,
std::__1::allocator<> >&, unsigned char, unsigned char, std::__1::pair<>&)
/data/jenkins/workspace/kudu-pre-commit-unittest-TSAN/thirdparty/installed/common/include/sparsepp/spp.h:1426:13
(libkudu_fs.so+0x102a56)
#3 std::__1::pair<>* spp::sparsegroup<>::set<std::__1::pair<kudu::BlockId
const, scoped_refptr<kudu::fs::internal::LogBlock> >
>(kudu::MemTrackerAllocator<std::__1::pair<kudu::BlockId const,
scoped_refptr<kudu::fs::internal::LogBlock> >,
std::__1::allocator<std::__1::pair<kudu::BlockId const,
scoped_refptr<kudu::fs::internal::LogBlock> > > >&, unsigned char,
std::__1::pair<kudu::BlockId const, scoped_refptr<kudu::fs::internal::LogBlock>
>&)
/data/jenkins/workspace/kudu-pre-commit-unittest-TSAN/thirdparty/installed/common/include/sparsepp/spp.h:1444:9
(libkudu_fs.so+0x10295f)
#4 std::__1::pair<>& spp::sparsetable<>::set<std::__1::pair<kudu::BlockId
const, scoped_refptr<kudu::fs::internal::LogBlock> > >(unsigned long,
std::__1::pair<kudu::BlockId const, scoped_refptr<kudu::fs::internal::LogBlock>
>&)
/data/jenkins/workspace/kudu-pre-commit-unittest-TSAN/thirdparty/installed/common/include/sparsepp/spp.h:2236:25
(libkudu_fs.so+0x1036ba)
#5 std::__1::pair<>&
spp::sparse_hashtable<>::_insert_at<std::__1::pair<kudu::BlockId const,
scoped_refptr<kudu::fs::internal::LogBlock> > >(std::__1::pair<kudu::BlockId
const, scoped_refptr<kudu::fs::internal::LogBlock> >&, unsigned long, bool)
/data/jenkins/workspace/kudu-pre-commit-unittest-TSAN/thirdparty/installed/common/include/sparsepp/spp.h:3173:22
(libkudu_fs.so+0x101910)
#6 std::__1::pair<>&
spp::sparse_hashtable<>::find_or_insert<spp::sparse_hash_map<kudu::BlockId,
scoped_refptr<kudu::fs::internal::LogBlock>, kudu::BlockIdHash,
kudu::BlockIdEqual, kudu::MemTrackerAllocator<std::__1::pair<kudu::BlockId
const, scoped_refptr<kudu::fs::internal::LogBlock> >,
std::__1::allocator<std::__1::pair<kudu::BlockId const,
scoped_refptr<kudu::fs::internal::LogBlock> > > >
>::DefaultValue>(kudu::BlockId const&)
/data/jenkins/workspace/kudu-pre-commit-unittest-TSAN/thirdparty/installed/common/include/sparsepp/spp.h:3282:28
(libkudu_fs.so+0x1014a1)
#7 spp::sparse_hash_map<>::operator[](kudu::BlockId const&)
/data/jenkins/workspace/kudu-pre-commit-unittest-TSAN/thirdparty/installed/common/include/sparsepp/spp.h:3792:29
(libkudu_fs.so+0xeece0)
#8
kudu::fs::LogBlockManager::AddLogBlock(scoped_refptr<kudu::fs::internal::LogBlock>)
/data/jenkins/workspace/kudu-pre-commit-unittest-TSAN/src/kudu/fs/log_block_manager.cc:2262:32
(libkudu_fs.so+0xe6a27)
...
Location is global 'spp::sparsegroup<std::__1::pair<>,
kudu::MemTrackerAllocator<std::__1::pair<>,
std::__1::allocator<std::__1::pair<> > > >::_sizing(unsigned
int)::s_alloc_batch_sz' of size 32 at 0x7f82f790a760
(libkudu_fs.so+0x000000354760) {code}
To wit, the race appears to be in the statically defined
{{spp::sparsegroup<>::_sizing::s_alloc_batch_size}}. There's some discussion in
this thread[1] about why their code looks the way it does.
It seems that having multiple threads call
{{spp::sparse_hash_map<>::operator[]}} sets the map up for a race to initialize
the map sizing.
[1] [https://github.com/greg7mdp/sparsepp/issues/42]
> ToolTest.TestFsAddRemoveDataDirEndToEnd reports race under TSAN
> ---------------------------------------------------------------
>
> Key: KUDU-3119
> URL: https://issues.apache.org/jira/browse/KUDU-3119
> Project: Kudu
> Issue Type: Bug
> Components: CLI, test
> Reporter: Alexey Serbin
> Priority: Blocker
> Attachments: kudu-tool-test.20200709.txt.xz, kudu-tool-test.3.txt.xz,
> kudu-tool-test.log.xz
>
>
> Sometimes the {{TestFsAddRemoveDataDirEndToEnd}} scenario of the {{ToolTest}}
> reports races for TSAN builds:
> {noformat}
> /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/tools/kudu-tool-test.cc:266:
> Failure
> Failed
> Bad status: Runtime error: /tmp/dist-test-taskIZqSmU/build/tsan/bin/kudu:
> process exited with non-ze
> ro status 66
> Google Test trace:
> /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/tools/kudu-tool-test.cc:265:
> W0506 17:5
> 6:02.744191 4432 flags.cc:404] Enabled unsafe flag: --never_fsync=true
> I0506 17:56:02.780252 4432 fs_manager.cc:263] Metadata directory not provided
> I0506 17:56:02.780442 4432 fs_manager.cc:269] Using write-ahead log
> directory (fs_wal_dir) as metad
> ata directory
> I0506 17:56:02.789638 4432 fs_manager.cc:399] Time spent opening directory
> manager: real 0.007s
> user 0.005s sys 0.002s
> I0506 17:56:02.789986 4432 env_posix.cc:1676] Not raising this process' open
> files per process limi
> t of 1048576; it is already as high as it can go
> I0506 17:56:02.790426 4432 file_cache.cc:465] Constructed file cache lbm
> with capacity 419430
> ==================
> WARNING: ThreadSanitizer: data race (pid=4432)
> ...
> {noformat}
> The log is attached.
--
This message was sent by Atlassian Jira
(v8.3.4#803005)
