This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch spill_and_reserve
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/spill_and_reserve by this push:
new 58be6c1b0ec [bugfix](scanmemory)modify scan memory profile (#41435)
58be6c1b0ec is described below
commit 58be6c1b0ecf6806e5b1a1152c5af71d89e17666
Author: yiguolei <[email protected]>
AuthorDate: Sat Sep 28 16:04:04 2024 +0800
[bugfix](scanmemory)modify scan memory profile (#41435)
1. memory usage in scan operator is empty.
2. peak memory usage is not in scan operator , it is in scanner.
---------
Co-authored-by: yiguolei <[email protected]>
---
be/src/pipeline/exec/operator.h | 4 ++--
be/src/pipeline/exec/scan_operator.cpp | 8 ++------
be/src/pipeline/exec/scan_operator.h | 1 -
be/src/vec/exec/scan/scanner_context.cpp | 3 +++
be/src/vec/exec/scan/scanner_context.h | 2 ++
5 files changed, 9 insertions(+), 9 deletions(-)
diff --git a/be/src/pipeline/exec/operator.h b/be/src/pipeline/exec/operator.h
index 1d2dbcc3592..6ab17b40c8b 100644
--- a/be/src/pipeline/exec/operator.h
+++ b/be/src/pipeline/exec/operator.h
@@ -226,7 +226,7 @@ protected:
RuntimeProfile::Counter* _projection_timer = nullptr;
RuntimeProfile::Counter* _exec_timer = nullptr;
// Account for peak memory used by this node
- RuntimeProfile::Counter* _peak_memory_usage_counter = nullptr;
+ RuntimeProfile::HighWaterMarkCounter* _peak_memory_usage_counter = nullptr;
RuntimeProfile::Counter* _init_timer = nullptr;
RuntimeProfile::Counter* _open_timer = nullptr;
RuntimeProfile::Counter* _close_timer = nullptr;
@@ -397,7 +397,7 @@ protected:
RuntimeProfile::Counter* _wait_for_finish_dependency_timer = nullptr;
RuntimeProfile::Counter* _exec_timer = nullptr;
RuntimeProfile::Counter* _memory_used_counter = nullptr;
- RuntimeProfile::Counter* _peak_memory_usage_counter = nullptr;
+ RuntimeProfile::HighWaterMarkCounter* _peak_memory_usage_counter = nullptr;
std::shared_ptr<QueryStatistics> _query_statistics = nullptr;
};
diff --git a/be/src/pipeline/exec/scan_operator.cpp
b/be/src/pipeline/exec/scan_operator.cpp
index b69b10461d6..9327ddfc875 100644
--- a/be/src/pipeline/exec/scan_operator.cpp
+++ b/be/src/pipeline/exec/scan_operator.cpp
@@ -1052,17 +1052,13 @@ Status ScanLocalState<Derived>::_init_profile() {
_total_throughput_counter =
profile()->add_rate_counter("TotalReadThroughput",
_rows_read_counter);
_num_scanners = ADD_COUNTER(_runtime_profile, "NumScanners", TUnit::UNIT);
+ _scanner_peak_memory_usage = _peak_memory_usage_counter;
+ //_runtime_profile->AddHighWaterMarkCounter("PeakMemoryUsage",
TUnit::BYTES);
// 2. counters for scanners
_scanner_profile.reset(new RuntimeProfile("VScanner"));
profile()->add_child(_scanner_profile.get(), true, nullptr);
- _memory_usage_counter = ADD_LABEL_COUNTER_WITH_LEVEL(_scanner_profile,
"MemoryUsage", 1);
- _free_blocks_memory_usage =
- _scanner_profile->AddHighWaterMarkCounter("FreeBlocks",
TUnit::BYTES, "MemoryUsage", 1);
- _scanner_peak_memory_usage =
- _scanner_profile->AddHighWaterMarkCounter("PeakMemoryUsage",
TUnit::BYTES);
-
_newly_create_free_blocks_num =
ADD_COUNTER(_scanner_profile, "NewlyCreateFreeBlocksNum",
TUnit::UNIT);
_scale_up_scanners_counter = ADD_COUNTER(_scanner_profile,
"NumScaleUpScanners", TUnit::UNIT);
diff --git a/be/src/pipeline/exec/scan_operator.h
b/be/src/pipeline/exec/scan_operator.h
index e228b76caa9..9af66dba06b 100644
--- a/be/src/pipeline/exec/scan_operator.h
+++ b/be/src/pipeline/exec/scan_operator.h
@@ -119,7 +119,6 @@ protected:
// time of filter output block from scanner
RuntimeProfile::Counter* _filter_timer = nullptr;
RuntimeProfile::Counter* _memory_usage_counter = nullptr;
- RuntimeProfile::HighWaterMarkCounter* _free_blocks_memory_usage = nullptr;
RuntimeProfile::Counter* _scale_up_scanners_counter = nullptr;
// rows read from the scanner (including those discarded by (pre)filters)
RuntimeProfile::Counter* _rows_read_counter = nullptr;
diff --git a/be/src/vec/exec/scan/scanner_context.cpp
b/be/src/vec/exec/scan/scanner_context.cpp
index 801ed1860f3..0362cdb939b 100644
--- a/be/src/vec/exec/scan/scanner_context.cpp
+++ b/be/src/vec/exec/scan/scanner_context.cpp
@@ -130,6 +130,7 @@ Status ScannerContext::init() {
_scanner_wait_batch_timer = _local_state->_scanner_wait_batch_timer;
_scanner_ctx_sched_time = _local_state->_scanner_ctx_sched_time;
_scale_up_scanners_counter = _local_state->_scale_up_scanners_counter;
+ _scanner_memory_used_counter = _local_state->_memory_used_counter;
#ifndef BE_TEST
// 3. get thread token
@@ -169,6 +170,7 @@ vectorized::BlockUPtr ScannerContext::get_free_block(bool
force) {
if (_free_blocks.try_dequeue(block)) {
DCHECK(block->mem_reuse());
_block_memory_usage -= block->allocated_bytes();
+ _scanner_memory_used_counter->set(_block_memory_usage);
// A free block is reused, so the memory usage should be decreased
// The caller of get_free_block will increase the memory usage
update_peak_memory_usage(-block->allocated_bytes());
@@ -184,6 +186,7 @@ void
ScannerContext::return_free_block(vectorized::BlockUPtr block) {
if (block->mem_reuse() && _block_memory_usage < _max_bytes_in_queue) {
size_t block_size_to_reuse = block->allocated_bytes();
_block_memory_usage += block_size_to_reuse;
+ _scanner_memory_used_counter->set(_block_memory_usage);
block->clear_column_data();
if (_free_blocks.enqueue(std::move(block))) {
update_peak_memory_usage(block_size_to_reuse);
diff --git a/be/src/vec/exec/scan/scanner_context.h
b/be/src/vec/exec/scan/scanner_context.h
index 200c36cdc98..ead314843dd 100644
--- a/be/src/vec/exec/scan/scanner_context.h
+++ b/be/src/vec/exec/scan/scanner_context.h
@@ -236,6 +236,8 @@ protected:
const int _num_parallel_instances;
std::shared_ptr<RuntimeProfile> _scanner_profile;
RuntimeProfile::Counter* _scanner_sched_counter = nullptr;
+ // This counter refers to scan operator's local state
+ RuntimeProfile::Counter* _scanner_memory_used_counter = nullptr;
RuntimeProfile::Counter* _newly_create_free_blocks_num = nullptr;
RuntimeProfile::Counter* _scanner_wait_batch_timer = nullptr;
RuntimeProfile::Counter* _scanner_ctx_sched_time = nullptr;
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
