kaijchen opened a new pull request, #30933:
URL: https://github.com/apache/doris/pull/30933
## Proposed changes
We should use `phmap::lazy_emplace_l()` to guarantee thread-safety in
`DeltaWriterV2Map::get_or_create()`.
```
==49474==ERROR: AddressSanitizer: heap-use-after-free on address
0x6040018e19f0 at pc 0x556226609305 bp 0x7f5804784cf0 sp 0x7f5804784ce8
READ of size 8 at 0x6040018e19f0 thread T3902 (FragmentMgrThre)
#0 0x556226609304 in std::__uniq_ptr_impl<doris::DeltaWriterV2,
std::default_delete<doris::DeltaWriterV2> >::_M_ptr() const
/var/local/ldb-toolchain/bin/../lib/gcc/x86_64-linux-gnu/11/../../../../include/c++/11/bits/unique_ptr.h:173:42
#1 0x556226609304 in std::unique_ptr<doris::DeltaWriterV2,
std::default_delete<doris::DeltaWriterV2> >::get() const
/var/local/ldb-toolchain/bin/../lib/gcc/x86_64-linux-gnu/11/../../../../include/c++/11/bits/unique_ptr.h:422:21
#2 0x556226609304 in
doris::vectorized::DeltaWriterV2Map::get_or_create(long,
std::function<std::unique_ptr<doris::DeltaWriterV2,
std::default_delete<doris::DeltaWriterV2> > ()>)
/root/doris/be/src/vec/sink/delta_writer_v2_pool.cpp:38:31
#3 0x556226800d62 in
doris::vectorized::VTabletWriterV2::_write_memtable(std::shared_ptr<doris::vectorized::Block>,
long, doris::vectorized::Rows const&,
std::vector<std::shared_ptr<doris::LoadStreamStub>,
std::allocator<std::shared_ptr<doris::LoadStreamStub> > > const&)
/root/doris/be/src/vec/sink/writer/vtablet_writer_v2.cpp:423:61
#4 0x5562267fed9e in
doris::vectorized::VTabletWriterV2::write(doris::vectorized::Block&)
/root/doris/be/src/vec/sink/writer/vtablet_writer_v2.cpp:415:9
#5 0x5562266e583d in
doris::vectorized::AsyncWriterSink<doris::vectorized::VTabletWriterV2,
&(doris::vectorized::VOLAP_TABLE_SINK_V2.<char const at offset
0>)>::send(doris::RuntimeState*, doris::vectorized::Block*, bool)
/root/doris/be/src/vec/sink/async_writer_sink.h:87:30
#6 0x55620066fa1b in
doris::PlanFragmentExecutor::open_vectorized_internal()
/root/doris/be/src/runtime/plan_fragment_executor.cpp:343:29
0x6040018e19f0 is located 32 bytes inside of 40-byte region
[0x6040018e19d0,0x6040018e19f8)
freed by thread T406 (FragmentMgrThre) here:
#0 0x5561fdedb80d in operator delete(void*)
(/mnt/ssd01/pipline/OpenSourceDoris/clusterEnv/P0/Cluster0/be/lib/doris_be+0x12c2580d)
(BuildId: a42de720d2eca269)
#1 0x556226610c42 in
phmap::priv::raw_hash_set<phmap::priv::FlatHashMapPolicy<long,
std::unique_ptr<doris::DeltaWriterV2, std::default_delete<doris::DeltaWriterV2>
> >, std::hash<long>, std::equal_to<long>, std::allocator<std::pair<long const,
std::unique_ptr<doris::DeltaWriterV2, std::default_delete<doris::DeltaWriterV2>
> > > >::prepare_insert(unsigned long)
/var/local/thirdparty/installed/include/parallel_hashmap/phmap.h:2198:13
#2 0x55622661091f in std::pair<unsigned long, bool>
phmap::priv::raw_hash_set<phmap::priv::FlatHashMapPolicy<long,
std::unique_ptr<doris::DeltaWriterV2, std::default_delete<doris::DeltaWriterV2>
> >, std::hash<long>, std::equal_to<long>, std::allocator<std::pair<long const,
std::unique_ptr<doris::DeltaWriterV2, std::default_delete<doris::DeltaWriterV2>
> > > >::find_or_prepare_insert<long>(long const&, unsigned long)
/var/local/thirdparty/installed/include/parallel_hashmap/phmap.h:2186:17
#3 0x556226609100 in
phmap::priv::raw_hash_set<phmap::priv::FlatHashMapPolicy<long,
std::unique_ptr<doris::DeltaWriterV2, std::default_delete<doris::DeltaWriterV2>
> >, std::hash<long>, std::equal_to<long>, std::allocator<std::pair<long const,
std::unique_ptr<doris::DeltaWriterV2, std::default_delete<doris::DeltaWriterV2>
> > > >::iterator
phmap::priv::raw_hash_set<phmap::priv::FlatHashMapPolicy<long,
std::unique_ptr<doris::DeltaWriterV2, std::default_delete<doris::DeltaWriterV2>
> >, std::hash<long>, std::equal_to<long>, std::allocator<std::pair<long const,
std::unique_ptr<doris::DeltaWriterV2, std::default_delete<doris::DeltaWriterV2>
> > > >::lazy_emplace_with_hash<long,
doris::vectorized::DeltaWriterV2Map::get_or_create(long,
std::function<std::unique_ptr<doris::DeltaWriterV2,
std::default_delete<doris::DeltaWriterV2> > ()>)::$_0>(long const&, unsigned
long, doris::vectorized::DeltaWriterV2Map::get_or_create(long,
std::function<std::unique_ptr<doris::DeltaWriterV2, std::de
fault_delete<doris::DeltaWriterV2> > ()>)::$_0&&)
/var/local/thirdparty/installed/include/parallel_hashmap/phmap.h:1534:20
#4 0x556226609100 in phmap::priv::parallel_hash_set<4ul,
phmap::priv::raw_hash_set, std::mutex, phmap::priv::FlatHashMapPolicy<long,
std::unique_ptr<doris::DeltaWriterV2, std::default_delete<doris::DeltaWriterV2>
> >, std::hash<long>, std::equal_to<long>, std::allocator<std::pair<long const,
std::unique_ptr<doris::DeltaWriterV2, std::default_delete<doris::DeltaWriterV2>
> > > >::iterator phmap::priv::parallel_hash_set<4ul,
phmap::priv::raw_hash_set, std::mutex, phmap::priv::FlatHashMapPolicy<long,
std::unique_ptr<doris::DeltaWriterV2, std::default_delete<doris::DeltaWriterV2>
> >, std::hash<long>, std::equal_to<long>, std::allocator<std::pair<long const,
std::unique_ptr<doris::DeltaWriterV2, std::default_delete<doris::DeltaWriterV2>
> > > >::lazy_emplace<long,
doris::vectorized::DeltaWriterV2Map::get_or_create(long,
std::function<std::unique_ptr<doris::DeltaWriterV2,
std::default_delete<doris::DeltaWriterV2> > ()>)::$_0>(long const&,
doris::vectorized::DeltaWriterV2Map::get_or
_create(long, std::function<std::unique_ptr<doris::DeltaWriterV2,
std::default_delete<doris::DeltaWriterV2> > ()>)::$_0&&)
/var/local/thirdparty/installed/include/parallel_hashmap/phmap.h:3200:42
#5 0x556226609100 in
doris::vectorized::DeltaWriterV2Map::get_or_create(long,
std::function<std::unique_ptr<doris::DeltaWriterV2,
std::default_delete<doris::DeltaWriterV2> > ()>)
/root/doris/be/src/vec/sink/delta_writer_v2_pool.cpp:35:10
#6 0x556226800d62 in
doris::vectorized::VTabletWriterV2::_write_memtable(std::shared_ptr<doris::vectorized::Block>,
long, doris::vectorized::Rows const&,
std::vector<std::shared_ptr<doris::LoadStreamStub>,
std::allocator<std::shared_ptr<doris::LoadStreamStub> > > const&)
/root/doris/be/src/vec/sink/writer/vtablet_writer_v2.cpp:423:61
#7 0x5562267fed9e in
doris::vectorized::VTabletWriterV2::write(doris::vectorized::Block&)
/root/doris/be/src/vec/sink/writer/vtablet_writer_v2.cpp:415:9
#8 0x5562266e583d in
doris::vectorized::AsyncWriterSink<doris::vectorized::VTabletWriterV2,
&(doris::vectorized::VOLAP_TABLE_SINK_V2.<char const at offset
0>)>::send(doris::RuntimeState*, doris::vectorized::Block*, bool)
/root/doris/be/src/vec/sink/async_writer_sink.h:87:30
#9 0x55620066fa1b in
doris::PlanFragmentExecutor::open_vectorized_internal()
/root/doris/be/src/runtime/plan_fragment_executor.cpp:343:29
```
## Further comments
If this is a relatively large or complex change, kick off the discussion at
[[email protected]](mailto:[email protected]) by explaining why you
chose the solution you did and what alternatives you considered, etc...
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
