This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 32b0013a35 [fix](memory) Fix query memory tracking #20253
32b0013a35 is described below
commit 32b0013a35f367601e1f2cee0fc756df592a3ecc
Author: Xinyi Zou <[email protected]>
AuthorDate: Mon Jun 5 08:33:38 2023 +0800
[fix](memory) Fix query memory tracking #20253
The memory released by the query end is recorded in the query mem tracker,
main memory in _runtime_state.
fix page no cache memory tracking
Now the main reason for the inaccurate query memory tracking is that the
virtual memory used by the query is sometimes much larger than the actual
memory. And the mem hook counts virtual memory.
---
be/src/olap/page_cache.h | 2 --
be/src/olap/rowset/segment_v2/page_handle.h | 5 ++++-
be/src/pipeline/pipeline_fragment_context.cpp | 9 ++++++++-
be/src/runtime/plan_fragment_executor.cpp | 10 ++++++++--
4 files changed, 20 insertions(+), 6 deletions(-)
diff --git a/be/src/olap/page_cache.h b/be/src/olap/page_cache.h
index cc52f4a252..7e5ca4de6e 100644
--- a/be/src/olap/page_cache.h
+++ b/be/src/olap/page_cache.h
@@ -42,7 +42,6 @@ public:
PageBase(size_t b) : _size(b), _capacity(b) {
_data = reinterpret_cast<char*>(TAllocator::alloc(_capacity,
ALLOCATOR_ALIGNMENT_16));
-
ExecEnv::GetInstance()->page_no_cache_mem_tracker()->consume(_capacity);
}
PageBase(const PageBase&) = delete;
@@ -52,7 +51,6 @@ public:
if (_data != nullptr) {
DCHECK(_capacity != 0 && _size != 0);
TAllocator::free(_data, _capacity);
-
ExecEnv::GetInstance()->page_no_cache_mem_tracker()->release(_capacity);
}
}
diff --git a/be/src/olap/rowset/segment_v2/page_handle.h
b/be/src/olap/rowset/segment_v2/page_handle.h
index 7e4f766524..6494048774 100644
--- a/be/src/olap/rowset/segment_v2/page_handle.h
+++ b/be/src/olap/rowset/segment_v2/page_handle.h
@@ -36,7 +36,9 @@ public:
// This class will take the ownership of input data's memory. It will
// free it when deconstructs.
- PageHandle(DataPage* data) : _is_data_owner(true), _data(data) {}
+ PageHandle(DataPage* data) : _is_data_owner(true), _data(data) {
+
ExecEnv::GetInstance()->page_no_cache_mem_tracker()->consume(_data->capacity());
+ }
// This class will take the content of cache data, and will make input
// cache_data to a invalid cache handle.
@@ -59,6 +61,7 @@ public:
~PageHandle() {
if (_is_data_owner) {
+
ExecEnv::GetInstance()->page_no_cache_mem_tracker()->release(_data->capacity());
delete _data;
} else {
DCHECK(_data == nullptr);
diff --git a/be/src/pipeline/pipeline_fragment_context.cpp
b/be/src/pipeline/pipeline_fragment_context.cpp
index eea8d34b08..32207ad216 100644
--- a/be/src/pipeline/pipeline_fragment_context.cpp
+++ b/be/src/pipeline/pipeline_fragment_context.cpp
@@ -143,7 +143,14 @@ PipelineFragmentContext::PipelineFragmentContext(
}
PipelineFragmentContext::~PipelineFragmentContext() {
- _call_back(_runtime_state.get(), &_exec_status);
+ if (_runtime_state != nullptr) {
+ // The memory released by the query end is recorded in the query mem
tracker, main memory in _runtime_state.
+ SCOPED_ATTACH_TASK(_runtime_state.get());
+ _call_back(_runtime_state.get(), &_exec_status);
+ _runtime_state.reset();
+ } else {
+ _call_back(_runtime_state.get(), &_exec_status);
+ }
DCHECK(!_report_thread_active);
}
diff --git a/be/src/runtime/plan_fragment_executor.cpp
b/be/src/runtime/plan_fragment_executor.cpp
index bd99a9a2e3..079719927b 100644
--- a/be/src/runtime/plan_fragment_executor.cpp
+++ b/be/src/runtime/plan_fragment_executor.cpp
@@ -88,7 +88,14 @@ PlanFragmentExecutor::PlanFragmentExecutor(ExecEnv* exec_env,
}
PlanFragmentExecutor::~PlanFragmentExecutor() {
- close();
+ if (_runtime_state != nullptr) {
+ // The memory released by the query end is recorded in the query mem
tracker, main memory in _runtime_state.
+ SCOPED_ATTACH_TASK(_runtime_state.get());
+ close();
+ _runtime_state.reset();
+ } else {
+ close();
+ }
// at this point, the report thread should have been stopped
DCHECK(!_report_thread_active);
}
@@ -274,7 +281,6 @@ Status PlanFragmentExecutor::open() {
if (_cancel_reason == PPlanFragmentCancelReason::CALL_RPC_ERROR) {
status = Status::RuntimeError(_cancel_msg);
} else if (_cancel_reason ==
PPlanFragmentCancelReason::MEMORY_LIMIT_EXCEED) {
- // status = Status::MemoryAllocFailed(_cancel_msg);
status = Status::MemoryLimitExceeded(_cancel_msg);
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
