This is an automated email from the ASF dual-hosted git repository.
dataroaring pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 6808e113c85 [fix](memory) Fix free page not tracking correctly (#36768)
6808e113c85 is described below
commit 6808e113c85e35045b3e1c1816d3deb3c9c8e4ff
Author: Xinyi Zou <[email protected]>
AuthorDate: Mon Jun 24 23:36:59 2024 +0800
[fix](memory) Fix free page not tracking correctly (#36768)
## Proposed changes
#36235 introduced, index page is not freed in query thread, so saved
tracker ptr when alloc page, so that page only needs to be freed before
query ends.
```
F20240624 18:44:51.880192 162249 mem_tracker_limiter.cpp:125] mem tracker
label: Query#Id=f0511107829e465e-a87a985234018b77, consumption: 94, peak
consumption: 636540, mem tracker not equal to 0 when mem tracker destruct, this
usually means that memory tracking is inaccurate and SCOPED_ATTACH_TASK and
SCOPED_SWITCH_THREAD_MEM_TRACKER_LIMITER are not used correctly. 1. For query
and load, memory leaks may have occurred, it is expected that the query mem
tracker will be bound to the thr [...]
0x605000160b30, size 46, strack trace:
0# Allocator<false, false, false>::alloc_impl(unsigned long,
unsigned long)
1# doris::PageBase<Allocator<false, false, false>
>::PageBase(unsigned long, bool, doris::segment_v2::PageTypePB)
2#
doris::segment_v2::PageIO::read_and_decompress_page(doris::segment_v2::PageReadOptions
const&, doris::segment_v2::PageHandle*, doris::Slice*,
doris::segment_v2::PageFooterPB*)
3#
doris::segment_v2::IndexedColumnReader::read_page(doris::segment_v2::PagePointer
const&, doris::segment_v2::PageHandle*, doris::Slice*,
doris::segment_v2::PageFooterPB*, doris::segment_v2::PageTypePB,
doris::BlockCompressionCodec*, bool) const
4#
doris::segment_v2::IndexedColumnReader::load_index_page(doris::segment_v2::PagePointerPB
const&, doris::segment_v2::PageHandle*, doris::segment_v2::IndexPageReader*)
5# doris::segment_v2::IndexedColumnReader::load(bool, bool)
6# doris::segment_v2::BitmapIndexReader::_load(bool, bool,
std::unique_ptr<doris::segment_v2::BitmapIndexPB,
std::default_delete<doris::segment_v2::BitmapIndexPB> >)
7# doris::segment_v2::BitmapIndexReader::load(bool, bool)
8# doris::segment_v2::ColumnReader::_load_bitmap_index(bool, bool)
9#
doris::segment_v2::ColumnReader::new_bitmap_index_iterator(doris::segment_v2::BitmapIndexIterator**)
10#
doris::segment_v2::Segment::new_bitmap_index_iterator(doris::TabletColumn
const&, std::unique_ptr<doris::segment_v2::BitmapIndexIterator,
std::default_delete<doris::segment_v2::BitmapIndexIterator> >*)
11#
doris::segment_v2::SegmentIterator::_init_bitmap_index_iterators()
12# doris::segment_v2::SegmentIterator::init_iterators()
13#
doris::segment_v2::SegmentIterator::_init_impl(doris::StorageReadOptions const&)
14#
doris::segment_v2::SegmentIterator::init(doris::StorageReadOptions const&)
15#
doris::segment_v2::Segment::new_iterator(std::shared_ptr<doris::Schema const>,
doris::StorageReadOptions const&, std::unique_ptr<doris::RowwiseIterator,
std::default_delete<doris::RowwiseIterator> >*)
16#
doris::segment_v2::LazyInitSegmentIterator::init(doris::StorageReadOptions
const&)
17# doris::BetaRowsetReader::_init_iterator()
18# doris::BetaRowsetReader::_init_iterator_once()
19# doris::BetaRowsetReader::next_block(doris::vectorized::Block*)
20#
doris::vectorized::VCollectIterator::_topn_next(doris::vectorized::Block*)
21#
doris::vectorized::VCollectIterator::next(doris::vectorized::Block*)
22#
doris::vectorized::BlockReader::_direct_next_block(doris::vectorized::Block*,
bool*)
23#
doris::vectorized::BlockReader::next_block_with_aggregation(doris::vectorized::Block*,
bool*)
24#
doris::vectorized::NewOlapScanner::_get_block_impl(doris::RuntimeState*,
doris::vectorized::Block*, bool*)
25# doris::vectorized::VScanner::get_block(doris::RuntimeState*,
doris::vectorized::Block*, bool*)
26#
doris::vectorized::VScanner::get_block_after_projects(doris::RuntimeState*,
doris::vectorized::Block*, bool*)
27#
doris::vectorized::ScannerScheduler::_scanner_scan(std::shared_ptr<doris::vectorized::ScannerContext>,
std::shared_ptr<doris::vectorized::ScanTask>)
28# std::_Function_handler<void (),
doris::vectorized::ScannerScheduler::submit(std::shared_ptr<doris::vectorized::ScannerContext>,
std::shared_ptr<doris::vectorized::ScanTask>)::$_1::operator()()
const::{lambda()#1}>::_M_invoke(std::_Any_data const&)
29# doris::ThreadPool::dispatch_thread()
30# doris::Thread::supervise_thread(void*)
```
---
be/src/olap/page_cache.cpp | 25 +++++++++----------------
be/src/olap/page_cache.h | 3 +--
2 files changed, 10 insertions(+), 18 deletions(-)
diff --git a/be/src/olap/page_cache.cpp b/be/src/olap/page_cache.cpp
index fe0a99af34f..1f0556f4642 100644
--- a/be/src/olap/page_cache.cpp
+++ b/be/src/olap/page_cache.cpp
@@ -26,16 +26,14 @@
namespace doris {
template <typename TAllocator>
PageBase<TAllocator>::PageBase(size_t b, bool use_cache,
segment_v2::PageTypePB page_type)
- : LRUCacheValueBase(),
- _size(b),
- _capacity(b),
- _use_cache(use_cache),
- _page_type(page_type) {
- if (_use_cache) {
- SCOPED_SWITCH_THREAD_MEM_TRACKER_LIMITER(
- StoragePageCache::instance()->mem_tracker(_page_type));
- _data = reinterpret_cast<char*>(TAllocator::alloc(_capacity,
ALLOCATOR_ALIGNMENT_16));
+ : LRUCacheValueBase(), _size(b), _capacity(b) {
+ if (use_cache) {
+ _mem_tracker_by_allocator =
StoragePageCache::instance()->mem_tracker(page_type);
} else {
+ _mem_tracker_by_allocator =
thread_context()->thread_mem_tracker_mgr->limiter_mem_tracker();
+ }
+ {
+ SCOPED_SWITCH_THREAD_MEM_TRACKER_LIMITER(_mem_tracker_by_allocator);
_data = reinterpret_cast<char*>(TAllocator::alloc(_capacity,
ALLOCATOR_ALIGNMENT_16));
}
}
@@ -44,13 +42,8 @@ template <typename TAllocator>
PageBase<TAllocator>::~PageBase() {
if (_data != nullptr) {
DCHECK(_capacity != 0 && _size != 0);
- if (_use_cache) {
- SCOPED_SWITCH_THREAD_MEM_TRACKER_LIMITER(
- StoragePageCache::instance()->mem_tracker(_page_type));
- TAllocator::free(_data, _capacity);
- } else {
- TAllocator::free(_data, _capacity);
- }
+ SCOPED_SWITCH_THREAD_MEM_TRACKER_LIMITER(_mem_tracker_by_allocator);
+ TAllocator::free(_data, _capacity);
}
}
diff --git a/be/src/olap/page_cache.h b/be/src/olap/page_cache.h
index 23b3574a10a..09fc689959c 100644
--- a/be/src/olap/page_cache.h
+++ b/be/src/olap/page_cache.h
@@ -60,8 +60,7 @@ private:
// Effective size, smaller than capacity, such as data page remove
checksum suffix.
size_t _size = 0;
size_t _capacity = 0;
- bool _use_cache;
- segment_v2::PageTypePB _page_type;
+ std::shared_ptr<MemTrackerLimiter> _mem_tracker_by_allocator;
};
using DataPage = PageBase<Allocator<false>>;
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
