This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new b7b8fd4af0f [refactor](profile) Normalize runtime profile naming
conventions and centralize counter constants (#61371)
b7b8fd4af0f is described below
commit b7b8fd4af0fe9f3ab55f16828a9127c7f04e70f9
Author: Mingyu Chen (Rayner) <[email protected]>
AuthorDate: Thu Mar 19 05:50:06 2026 -0700
[refactor](profile) Normalize runtime profile naming conventions and
centralize counter constants (#61371)
### What problem does this PR solve?
1. Centralize counter name constants:
- Create `runtime_profile_counter_names.h` with all counter name
constants
under `doris::profile` namespace.
- Replace hardcoded counter name strings in operator.cpp, operator.h,
pipeline_task.cpp, scan_operator.cpp, spill_utils.h, and dependency.h.
2. Unify profile node naming to [Name(key=value,
key=value)](cci:1://file:///mnt/disk1/yy/git/wt-update-profile/fe/fe-core/src/main/java/org/apache/doris/common/profile/RuntimeProfile.java:174:4-176:5)
format:
- Pipeline: `"Pipeline : X"` -> `"Pipeline(id=X)"`
- Fragment: `"PipelineX X"` -> `"PipelineX(fragment_id=X)"`
- Operators: `"(nereids_id=X)(id=Y)"` -> `"(nereids_id=X, id=Y)"`
- Scan operators: add `id=` field, fix `.` separator to `,`
- ExchangeSink: add `id=` and `nereids_id=` fields
- MultiCastSink: `"(dest_id=a, dest_id=b)(id=X)"` -> `"(id=X,
dest_ids=[a,b])"`
- Remove legacy `operator_name_suffix` and `exchange_sink_name_suffix`
constants.
3. Deduplicate spill counters:
- Create
[SpillWriteCounters](cci:2://file:///mnt/disk1/yy/git/wt-update-profile/be/src/exec/operator/spill_counters.h:25:0-54:1)
and
[SpillReadCounters](cci:2://file:///mnt/disk1/yy/git/wt-update-profile/be/src/exec/operator/spill_counters.h:57:0-92:1)
structs in
[spill_counters.h](cci:7://file:///mnt/disk1/yy/git/wt-update-profile/be/src/exec/operator/spill_counters.h:0:0-0:0)
to eliminate duplicated counter definitions between
[PipelineXSpillLocalState](cci:2://file:///mnt/disk1/yy/git/wt-update-profile/be/src/exec/operator/operator.h:359:0-464:2)
and
[PipelineXSpillSinkLocalState](cci:2://file:///mnt/disk1/yy/git/wt-update-profile/be/src/exec/operator/operator.h:728:0-805:2).
- Use reference aliases for backward compatibility.
4. Enable level filtering in
[pretty_print()](cci:1://file:///mnt/disk1/yy/git/wt-update-profile/be/src/runtime/runtime_profile.cpp:608:0-668:1):
- Add `RuntimeProfileCounterTreeNode::pretty_print()` method.
-
[pretty_print()](cci:1://file:///mnt/disk1/yy/git/wt-update-profile/be/src/runtime/runtime_profile.cpp:608:0-668:1)
now builds counter tree, prunes by `profile_level`,
and prints from pruned tree, consistent with
[to_thrift()](cci:1://file:///mnt/disk1/yy/git/wt-update-profile/be/src/runtime/runtime_profile.cpp:675:0-714:1)/[to_proto()](cci:1://file:///mnt/disk1/yy/git/wt-update-profile/be/src/runtime/runtime_profile.cpp:721:0-761:1).
5. Adapt FE parsing:
- Optimize
[shouldBeIncluded()](cci:1://file:///mnt/disk1/yy/git/wt-update-profile/fe/fe-core/src/main/java/org/apache/doris/common/profile/RuntimeProfile.java:498:4-504:5)
from regex to `startsWith`/`contains`.
---
be/src/exec/operator/exchange_sink_operator.cpp | 7 +-
be/src/exec/operator/file_scan_operator.cpp | 10 +-
be/src/exec/operator/jdbc_scan_operator.cpp | 10 +-
.../exec/operator/multi_cast_data_stream_sink.cpp | 14 +-
be/src/exec/operator/olap_scan_operator.h | 9 +-
be/src/exec/operator/operator.cpp | 53 ++++---
be/src/exec/operator/operator.h | 169 ++++++++++-----------
be/src/exec/operator/scan_operator.cpp | 29 ++--
be/src/exec/operator/spill_counters.h | 95 ++++++++++++
be/src/exec/pipeline/dependency.h | 1 +
be/src/exec/pipeline/pipeline_task.cpp | 33 ++--
be/src/runtime/runtime_profile.cpp | 6 +-
be/src/runtime/runtime_profile_counter_names.h | 125 +++++++++++++++
.../runtime/runtime_profile_counter_tree_node.cpp | 15 ++
be/src/runtime/runtime_profile_counter_tree_node.h | 4 +
be/src/runtime/runtime_state.cpp | 6 +-
.../runtime/runtime_profile_profile_level_test.cpp | 108 +++++++++++++
be/test/util/profile_spec_test.cpp | 4 +-
.../doris/common/profile/RuntimeProfile.java | 8 +-
.../doris/common/profile/AutoProfileTest.java | 8 +-
20 files changed, 538 insertions(+), 176 deletions(-)
diff --git a/be/src/exec/operator/exchange_sink_operator.cpp
b/be/src/exec/operator/exchange_sink_operator.cpp
index d4f3528a532..89e59a7c5d4 100644
--- a/be/src/exec/operator/exchange_sink_operator.cpp
+++ b/be/src/exec/operator/exchange_sink_operator.cpp
@@ -266,7 +266,12 @@ Status ExchangeSinkLocalState::open(RuntimeState* state) {
std::string ExchangeSinkLocalState::name_suffix() {
auto& p = _parent->cast<ExchangeSinkOperatorX>();
- return fmt::format(exchange_sink_name_suffix,
std::to_string(p._dest_node_id));
+ if (_parent->nereids_id() == -1) {
+ return fmt::format("(id={}, dest_id={})", _parent->node_id(),
p._dest_node_id);
+ } else {
+ return fmt::format("(nereids_id={}, id={}, dest_id={})",
_parent->nereids_id(),
+ _parent->node_id(), p._dest_node_id);
+ }
}
segment_v2::CompressionTypePB ExchangeSinkLocalState::compression_type() const
{
diff --git a/be/src/exec/operator/file_scan_operator.cpp
b/be/src/exec/operator/file_scan_operator.cpp
index 34fc77687ee..e27dc73a3c3 100644
--- a/be/src/exec/operator/file_scan_operator.cpp
+++ b/be/src/exec/operator/file_scan_operator.cpp
@@ -120,10 +120,12 @@ Status
FileScanLocalState::_init_scanners(std::list<ScannerSPtr>* scanners) {
}
std::string FileScanLocalState::name_suffix() const {
- return fmt::format("(nereids_id={}. table_name={})" + operator_name_suffix,
- std::to_string(_parent->nereids_id()),
- _parent->cast<FileScanOperatorX>()._table_name,
- std::to_string(_parent->node_id()));
+ if (_parent->nereids_id() == -1) {
+ return fmt::format("(id={}, table_name={})", _parent->node_id(),
+ _parent->cast<FileScanOperatorX>()._table_name);
+ }
+ return fmt::format("(nereids_id={}, id={}, table_name={})",
_parent->nereids_id(),
+ _parent->node_id(),
_parent->cast<FileScanOperatorX>()._table_name);
}
void FileScanLocalState::set_scan_ranges(RuntimeState* state,
diff --git a/be/src/exec/operator/jdbc_scan_operator.cpp
b/be/src/exec/operator/jdbc_scan_operator.cpp
index 311acb37f90..2618ee9ba0b 100644
--- a/be/src/exec/operator/jdbc_scan_operator.cpp
+++ b/be/src/exec/operator/jdbc_scan_operator.cpp
@@ -23,10 +23,12 @@
namespace doris {
#include "common/compile_check_begin.h"
std::string JDBCScanLocalState::name_suffix() const {
- return fmt::format("(nereids_id={}. table_name={})" + operator_name_suffix,
- std::to_string(_parent->nereids_id()),
- _parent->cast<JDBCScanOperatorX>()._table_name,
- std::to_string(_parent->node_id()));
+ if (_parent->nereids_id() == -1) {
+ return fmt::format("(id={}, table_name={})", _parent->node_id(),
+ _parent->cast<JDBCScanOperatorX>()._table_name);
+ }
+ return fmt::format("(nereids_id={}, id={}, table_name={})",
_parent->nereids_id(),
+ _parent->node_id(),
_parent->cast<JDBCScanOperatorX>()._table_name);
}
Status JDBCScanLocalState::_init_scanners(std::list<ScannerSPtr>* scanners) {
diff --git a/be/src/exec/operator/multi_cast_data_stream_sink.cpp
b/be/src/exec/operator/multi_cast_data_stream_sink.cpp
index 437416b1b23..9c73c2b9117 100644
--- a/be/src/exec/operator/multi_cast_data_stream_sink.cpp
+++ b/be/src/exec/operator/multi_cast_data_stream_sink.cpp
@@ -27,15 +27,19 @@ namespace doris {
std::string MultiCastDataStreamSinkLocalState::name_suffix() {
auto* parent = static_cast<MultiCastDataStreamSinkOperatorX*>(_parent);
auto& dest_ids = parent->dests_id();
- std::string result = "(";
+ std::string dest_list;
for (size_t i = 0; i < dest_ids.size(); ++i) {
if (i > 0) {
- result += ", ";
+ dest_list += ",";
}
- result += fmt::format("dest_id={}", dest_ids[i]);
+ dest_list += std::to_string(dest_ids[i]);
+ }
+ if (_parent->nereids_id() == -1) {
+ return fmt::format("(id={}, dest_ids=[{}])", parent->operator_id(),
dest_list);
+ } else {
+ return fmt::format("(nereids_id={}, id={}, dest_ids=[{}])",
_parent->nereids_id(),
+ parent->operator_id(), dest_list);
}
- result += ")";
- return fmt::format(result + operator_name_suffix, parent->operator_id());
}
std::shared_ptr<BasicSharedState>
MultiCastDataStreamSinkOperatorX::create_shared_state() const {
diff --git a/be/src/exec/operator/olap_scan_operator.h
b/be/src/exec/operator/olap_scan_operator.h
index e38d1eaf8f8..97143c6c601 100644
--- a/be/src/exec/operator/olap_scan_operator.h
+++ b/be/src/exec/operator/olap_scan_operator.h
@@ -47,9 +47,12 @@ public:
TOlapScanNode& olap_scan_node() const;
std::string name_suffix() const override {
- return fmt::format("(nereids_id={}. table_name={})" +
operator_name_suffix,
- std::to_string(_parent->nereids_id()),
olap_scan_node().table_name,
- std::to_string(_parent->node_id()));
+ if (_parent->nereids_id() == -1) {
+ return fmt::format("(id={}, table_name={})", _parent->node_id(),
+ olap_scan_node().table_name);
+ }
+ return fmt::format("(nereids_id={}, id={}, table_name={})",
_parent->nereids_id(),
+ _parent->node_id(), olap_scan_node().table_name);
}
std::vector<Dependency*> execution_dependencies() override {
if (!_cloud_tablet_dependency) {
diff --git a/be/src/exec/operator/operator.cpp
b/be/src/exec/operator/operator.cpp
index 2f813e3d00c..51572da3223 100644
--- a/be/src/exec/operator/operator.cpp
+++ b/be/src/exec/operator/operator.cpp
@@ -92,6 +92,7 @@
#include "exprs/vexpr.h"
#include "exprs/vexpr_context.h"
#include "runtime/runtime_profile.h"
+#include "runtime/runtime_profile_counter_names.h"
#include "util/debug_util.h"
#include "util/string_util.h"
@@ -114,22 +115,18 @@ Status OperatorBase::close(RuntimeState* state) {
template <typename SharedStateArg>
std::string PipelineXLocalState<SharedStateArg>::name_suffix() const {
if (_parent->nereids_id() == -1) {
- return fmt::format(operator_name_suffix,
std::to_string(_parent->node_id()));
+ return fmt::format("(id={})", _parent->node_id());
} else {
- return fmt::format("(nereids_id={})" + operator_name_suffix,
- std::to_string(_parent->nereids_id()),
- std::to_string(_parent->node_id()));
+ return fmt::format("(nereids_id={}, id={})", _parent->nereids_id(),
_parent->node_id());
}
}
template <typename SharedStateArg>
std::string PipelineXSinkLocalState<SharedStateArg>::name_suffix() {
if (_parent->nereids_id() == -1) {
- return fmt::format(operator_name_suffix,
std::to_string(_parent->node_id()));
+ return fmt::format("(id={})", _parent->node_id());
} else {
- return fmt::format("(nereids_id={})" + operator_name_suffix,
- std::to_string(_parent->nereids_id()),
- std::to_string(_parent->node_id()));
+ return fmt::format("(nereids_id={}, id={})", _parent->nereids_id(),
_parent->node_id());
}
}
@@ -533,8 +530,8 @@
PipelineXLocalStateBase::PipelineXLocalStateBase(RuntimeState* state, OperatorXB
template <typename SharedStateArg>
Status PipelineXLocalState<SharedStateArg>::init(RuntimeState* state,
LocalStateInfo& info) {
_operator_profile.reset(new RuntimeProfile(_parent->get_name() +
name_suffix()));
- _common_profile.reset(new RuntimeProfile("CommonCounters"));
- _custom_profile.reset(new RuntimeProfile("CustomCounters"));
+ _common_profile.reset(new RuntimeProfile(profile::COMMON_COUNTERS));
+ _custom_profile.reset(new RuntimeProfile(profile::CUSTOM_COUNTERS));
_operator_profile->set_metadata(_parent->node_id());
// indent is false so that source operator will have same
// indentation_level with its parent operator.
@@ -574,16 +571,16 @@ Status
PipelineXLocalState<SharedStateArg>::init(RuntimeState* state, LocalState
}
_rows_returned_counter =
- ADD_COUNTER_WITH_LEVEL(_common_profile, "RowsProduced",
TUnit::UNIT, 1);
+ ADD_COUNTER_WITH_LEVEL(_common_profile, profile::ROWS_PRODUCED,
TUnit::UNIT, 1);
_blocks_returned_counter =
- ADD_COUNTER_WITH_LEVEL(_common_profile, "BlocksProduced",
TUnit::UNIT, 1);
- _projection_timer = ADD_TIMER_WITH_LEVEL(_common_profile,
"ProjectionTime", 2);
- _init_timer = ADD_TIMER_WITH_LEVEL(_common_profile, "InitTime", 2);
- _open_timer = ADD_TIMER_WITH_LEVEL(_common_profile, "OpenTime", 2);
- _close_timer = ADD_TIMER_WITH_LEVEL(_common_profile, "CloseTime", 2);
- _exec_timer = ADD_TIMER_WITH_LEVEL(_common_profile, "ExecTime", 1);
+ ADD_COUNTER_WITH_LEVEL(_common_profile, profile::BLOCKS_PRODUCED,
TUnit::UNIT, 1);
+ _projection_timer = ADD_TIMER_WITH_LEVEL(_common_profile,
profile::PROJECTION_TIME, 2);
+ _init_timer = ADD_TIMER_WITH_LEVEL(_common_profile, profile::INIT_TIME, 2);
+ _open_timer = ADD_TIMER_WITH_LEVEL(_common_profile, profile::OPEN_TIME, 2);
+ _close_timer = ADD_TIMER_WITH_LEVEL(_common_profile, profile::CLOSE_TIME,
2);
+ _exec_timer = ADD_TIMER_WITH_LEVEL(_common_profile, profile::EXEC_TIME, 1);
_memory_used_counter =
- _common_profile->AddHighWaterMarkCounter("MemoryUsage",
TUnit::BYTES, "", 1);
+ _common_profile->AddHighWaterMarkCounter(profile::MEMORY_USAGE,
TUnit::BYTES, "", 1);
_common_profile->add_info_string("IsColocate",
std::to_string(_parent->is_colocated_operator()));
_common_profile->add_info_string("IsShuffled",
std::to_string(_parent->is_shuffled_operator()));
@@ -639,8 +636,8 @@ Status
PipelineXSinkLocalState<SharedState>::init(RuntimeState* state, LocalSink
// create profile
_operator_profile =
state->obj_pool()->add(new RuntimeProfile(_parent->get_name() +
name_suffix()));
- _common_profile = state->obj_pool()->add(new
RuntimeProfile("CommonCounters"));
- _custom_profile = state->obj_pool()->add(new
RuntimeProfile("CustomCounters"));
+ _common_profile = state->obj_pool()->add(new
RuntimeProfile(profile::COMMON_COUNTERS));
+ _custom_profile = state->obj_pool()->add(new
RuntimeProfile(profile::CUSTOM_COUNTERS));
// indentation is true
// The parent profile of sink operator is usually a RuntimeProfile called
PipelineTask.
@@ -650,7 +647,8 @@ Status
PipelineXSinkLocalState<SharedState>::init(RuntimeState* state, LocalSink
_operator_profile->add_child(_custom_profile, true);
_operator_profile->set_metadata(_parent->node_id());
- _wait_for_finish_dependency_timer = ADD_TIMER(_common_profile,
"PendingFinishDependency");
+ _wait_for_finish_dependency_timer =
+ ADD_TIMER(_common_profile, profile::PENDING_FINISH_DEPENDENCY);
constexpr auto is_fake_shared = std::is_same_v<SharedState,
FakeSharedState>;
if constexpr (!is_fake_shared) {
if (info.shared_state_map.find(_parent->dests_id().front()) !=
@@ -680,13 +678,14 @@ Status
PipelineXSinkLocalState<SharedState>::init(RuntimeState* state, LocalSink
return Status::InternalError("must set shared state, in {}",
_parent->get_name());
}
- _rows_input_counter = ADD_COUNTER_WITH_LEVEL(_common_profile, "InputRows",
TUnit::UNIT, 1);
- _init_timer = ADD_TIMER_WITH_LEVEL(_common_profile, "InitTime", 2);
- _open_timer = ADD_TIMER_WITH_LEVEL(_common_profile, "OpenTime", 2);
- _close_timer = ADD_TIMER_WITH_LEVEL(_common_profile, "CloseTime", 2);
- _exec_timer = ADD_TIMER_WITH_LEVEL(_common_profile, "ExecTime", 1);
+ _rows_input_counter =
+ ADD_COUNTER_WITH_LEVEL(_common_profile, profile::INPUT_ROWS,
TUnit::UNIT, 1);
+ _init_timer = ADD_TIMER_WITH_LEVEL(_common_profile, profile::INIT_TIME, 2);
+ _open_timer = ADD_TIMER_WITH_LEVEL(_common_profile, profile::OPEN_TIME, 2);
+ _close_timer = ADD_TIMER_WITH_LEVEL(_common_profile, profile::CLOSE_TIME,
2);
+ _exec_timer = ADD_TIMER_WITH_LEVEL(_common_profile, profile::EXEC_TIME, 1);
_memory_used_counter =
- _common_profile->AddHighWaterMarkCounter("MemoryUsage",
TUnit::BYTES, "", 1);
+ _common_profile->AddHighWaterMarkCounter(profile::MEMORY_USAGE,
TUnit::BYTES, "", 1);
_common_profile->add_info_string("IsColocate",
std::to_string(_parent->is_colocated_operator()));
_common_profile->add_info_string("IsShuffled",
std::to_string(_parent->is_shuffled_operator()));
diff --git a/be/src/exec/operator/operator.h b/be/src/exec/operator/operator.h
index f9ab1006290..a89cd30770e 100644
--- a/be/src/exec/operator/operator.h
+++ b/be/src/exec/operator/operator.h
@@ -36,11 +36,13 @@
#include "exec/exchange/local_exchanger.h"
#include "exec/exchange/vdata_stream_recvr.h"
#include "exec/operator/operator.h"
+#include "exec/operator/spill_counters.h"
#include "exec/operator/spill_utils.h"
#include "exec/pipeline/dependency.h"
#include "runtime/memory/mem_tracker.h"
#include "runtime/query_context.h"
#include "runtime/runtime_profile.h"
+#include "runtime/runtime_profile_counter_names.h"
#include "runtime/runtime_state.h"
#include "runtime/thread_context.h"
@@ -65,12 +67,6 @@ using Operators = std::vector<OperatorPtr>;
using DataSinkOperatorPtr = std::shared_ptr<DataSinkOperatorXBase>;
-// This suffix will be added back to the name of sink operator
-// when we creating runtime profile.
-const std::string exchange_sink_name_suffix = "(dest_id={})";
-
-const std::string operator_name_suffix = "(id={})";
-
// This struct is used only for initializing local state.
struct LocalStateInfo {
RuntimeProfile* parent_profile = nullptr;
@@ -375,75 +371,65 @@ public:
}
void init_spill_write_counters() {
- _spill_write_file_timer =
- ADD_TIMER_WITH_LEVEL(Base::custom_profile(),
"SpillWriteFileTime", 1);
-
- _spill_write_serialize_block_timer =
- ADD_TIMER_WITH_LEVEL(Base::custom_profile(),
"SpillWriteSerializeBlockTime", 1);
- _spill_write_block_count =
ADD_COUNTER_WITH_LEVEL(Base::custom_profile(),
-
"SpillWriteBlockCount", TUnit::UNIT, 1);
- _spill_write_block_data_size = ADD_COUNTER_WITH_LEVEL(
- Base::custom_profile(), "SpillWriteBlockBytes", TUnit::BYTES,
1);
+ _write_counters.init(Base::custom_profile());
+
+ // Source-only extra write counters
_spill_write_file_total_size = ADD_COUNTER_WITH_LEVEL(
- Base::custom_profile(), "SpillWriteFileBytes", TUnit::BYTES,
1);
- _spill_write_rows_count =
- ADD_COUNTER_WITH_LEVEL(Base::custom_profile(),
"SpillWriteRows", TUnit::UNIT, 1);
- _spill_write_file_total_count = ADD_COUNTER_WITH_LEVEL(
- Base::custom_profile(), "SpillWriteFileTotalCount",
TUnit::UNIT, 1);
+ Base::custom_profile(), profile::SPILL_WRITE_FILE_BYTES,
TUnit::BYTES, 1);
+ _spill_file_total_count = ADD_COUNTER_WITH_LEVEL(
+ Base::custom_profile(), profile::SPILL_WRITE_FILE_TOTAL_COUNT,
TUnit::UNIT, 1);
}
void init_spill_read_counters() {
- // Spill read counters
- _spill_read_file_time =
- ADD_TIMER_WITH_LEVEL(Base::custom_profile(),
"SpillReadFileTime", 1);
- _spill_read_deserialize_block_timer =
- ADD_TIMER_WITH_LEVEL(Base::custom_profile(),
"SpillReadDeserializeBlockTime", 1);
-
- _spill_read_block_count =
ADD_COUNTER_WITH_LEVEL(Base::custom_profile(),
-
"SpillReadBlockCount", TUnit::UNIT, 1);
- _spill_read_block_data_size = ADD_COUNTER_WITH_LEVEL(
- Base::custom_profile(), "SpillReadBlockBytes", TUnit::BYTES,
1);
- _spill_read_file_size = ADD_COUNTER_WITH_LEVEL(Base::custom_profile(),
"SpillReadFileBytes",
- TUnit::BYTES, 1);
- _spill_read_rows_count =
- ADD_COUNTER_WITH_LEVEL(Base::custom_profile(),
"SpillReadRows", TUnit::UNIT, 1);
- _spill_read_file_count = ADD_COUNTER_WITH_LEVEL(Base::custom_profile(),
- "SpillReadFileCount",
TUnit::UNIT, 1);
+ _spill_total_timer =
+ ADD_TIMER_WITH_LEVEL(Base::custom_profile(),
profile::SPILL_TOTAL_TIME, 1);
+
+ _read_counters.init(Base::custom_profile());
_spill_file_current_size = ADD_COUNTER_WITH_LEVEL(
- Base::custom_profile(), "SpillWriteFileCurrentBytes",
TUnit::BYTES, 1);
+ Base::custom_profile(),
profile::SPILL_WRITE_FILE_CURRENT_BYTES, TUnit::BYTES, 1);
_spill_file_current_count = ADD_COUNTER_WITH_LEVEL(
- Base::custom_profile(), "SpillWriteFileCurrentCount",
TUnit::UNIT, 1);
+ Base::custom_profile(),
profile::SPILL_WRITE_FILE_CURRENT_COUNT, TUnit::UNIT, 1);
}
- // Spill write counters
- // Total time of writing file
- RuntimeProfile::Counter* _spill_write_file_timer = nullptr;
- RuntimeProfile::Counter* _spill_write_serialize_block_timer = nullptr;
- // Original count of spilled Blocks
- // One Big Block maybe split into multiple small Blocks when actually
written to disk file.
- RuntimeProfile::Counter* _spill_write_block_count = nullptr;
- // Total bytes of spill data in Block format(in memory format)
- RuntimeProfile::Counter* _spill_write_block_data_size = nullptr;
+ // Total time of spill, including spill task scheduling time,
+ // serialize block time, write disk file time,
+ // and read disk file time, deserialize block time etc.
+ RuntimeProfile::Counter* _spill_total_timer = nullptr;
+
+ // Shared spill write counters
+ SpillWriteCounters _write_counters;
+ // Backward-compatible aliases for commonly accessed write counters
+ RuntimeProfile::Counter*& _spill_write_file_timer =
_write_counters.spill_write_file_timer;
+ RuntimeProfile::Counter*& _spill_write_serialize_block_timer =
+ _write_counters.spill_write_serialize_block_timer;
+ RuntimeProfile::Counter*& _spill_write_block_count =
_write_counters.spill_write_block_count;
+ RuntimeProfile::Counter*& _spill_write_block_data_size =
+ _write_counters.spill_write_block_data_size;
+ RuntimeProfile::Counter*& _spill_write_rows_count =
_write_counters.spill_write_rows_count;
+
+ // Source-only write counters (not in SpillWriteCounters)
// Total bytes of spill data written to disk file(after serialized)
RuntimeProfile::Counter* _spill_write_file_total_size = nullptr;
- RuntimeProfile::Counter* _spill_write_rows_count = nullptr;
- RuntimeProfile::Counter* _spill_write_file_total_count = nullptr;
+ RuntimeProfile::Counter* _spill_file_total_count = nullptr;
RuntimeProfile::Counter* _spill_file_current_count = nullptr;
// Spilled file total size
RuntimeProfile::Counter* _spill_file_total_size = nullptr;
// Current spilled file size
RuntimeProfile::Counter* _spill_file_current_size = nullptr;
- RuntimeProfile::Counter* _spill_read_file_time = nullptr;
- RuntimeProfile::Counter* _spill_read_deserialize_block_timer = nullptr;
- RuntimeProfile::Counter* _spill_read_block_count = nullptr;
- // Total bytes of read data in Block format(in memory format)
- RuntimeProfile::Counter* _spill_read_block_data_size = nullptr;
- // Total bytes of spill data read from disk file
- RuntimeProfile::Counter* _spill_read_file_size = nullptr;
- RuntimeProfile::Counter* _spill_read_rows_count = nullptr;
- RuntimeProfile::Counter* _spill_read_file_count = nullptr;
+ // Shared spill read counters
+ SpillReadCounters _read_counters;
+ // Backward-compatible aliases for commonly accessed read counters
+ RuntimeProfile::Counter*& _spill_read_file_time =
_read_counters.spill_read_file_time;
+ RuntimeProfile::Counter*& _spill_read_deserialize_block_timer =
+ _read_counters.spill_read_deserialize_block_timer;
+ RuntimeProfile::Counter*& _spill_read_block_count =
_read_counters.spill_read_block_count;
+ RuntimeProfile::Counter*& _spill_read_block_data_size =
+ _read_counters.spill_read_block_data_size;
+ RuntimeProfile::Counter*& _spill_read_file_size =
_read_counters.spill_read_file_size;
+ RuntimeProfile::Counter*& _spill_read_rows_count =
_read_counters.spill_read_rows_count;
+ RuntimeProfile::Counter*& _spill_read_file_count =
_read_counters.spill_read_file_count;
};
class DataSinkOperatorXBase;
@@ -519,7 +505,7 @@ protected:
//and shared hash table, some counter/timer about build hash table is
useless,
//so we could add those counter/timer in faker profile, and those will not
display in web profile.
std::unique_ptr<RuntimeProfile> _faker_runtime_profile =
- std::make_unique<RuntimeProfile>("faker profile");
+ std::make_unique<RuntimeProfile>(profile::FAKER_PROFILE);
RuntimeProfile::Counter* _rows_input_counter = nullptr;
RuntimeProfile::Counter* _init_timer = nullptr;
@@ -725,26 +711,24 @@ public:
}
void init_spill_counters() {
- _spill_write_file_total_count = ADD_COUNTER_WITH_LEVEL(
- Base::custom_profile(), "SpillWriteFileTotalCount",
TUnit::UNIT, 1);
+ _spill_total_timer =
+ ADD_TIMER_WITH_LEVEL(Base::custom_profile(),
profile::SPILL_TOTAL_TIME, 1);
+
+ _write_counters.init(Base::custom_profile());
+
+ // SpillFileWriter looks up these counters via get_counter() in its
+ // constructor. They must be registered on the CustomCounters profile
+ // before any SpillFileWriter is created, otherwise the lookups return
+ // nullptr and COUNTER_UPDATE will SEGV.
_spill_write_file_total_size = ADD_COUNTER_WITH_LEVEL(
- Base::custom_profile(), "SpillWriteFileBytes", TUnit::BYTES,
1);
- _spill_write_file_timer =
- ADD_TIMER_WITH_LEVEL(Base::custom_profile(),
"SpillWriteFileTime", 1);
-
- _spill_write_serialize_block_timer =
- ADD_TIMER_WITH_LEVEL(Base::custom_profile(),
"SpillWriteSerializeBlockTime", 1);
- _spill_write_block_count =
ADD_COUNTER_WITH_LEVEL(Base::custom_profile(),
-
"SpillWriteBlockCount", TUnit::UNIT, 1);
- _spill_write_block_data_size = ADD_COUNTER_WITH_LEVEL(
- Base::custom_profile(), "SpillWriteBlockBytes", TUnit::BYTES,
1);
- _spill_write_rows_count =
- ADD_COUNTER_WITH_LEVEL(Base::custom_profile(),
"SpillWriteRows", TUnit::UNIT, 1);
+ Base::custom_profile(), profile::SPILL_WRITE_FILE_BYTES,
TUnit::BYTES, 1);
+ _spill_file_total_count = ADD_COUNTER_WITH_LEVEL(
+ Base::custom_profile(), profile::SPILL_WRITE_FILE_TOTAL_COUNT,
TUnit::UNIT, 1);
_spill_max_rows_of_partition = ADD_COUNTER_WITH_LEVEL(
- Base::custom_profile(), "SpillMaxRowsOfPartition",
TUnit::UNIT, 1);
+ Base::custom_profile(), profile::SPILL_MAX_ROWS_OF_PARTITION,
TUnit::UNIT, 1);
_spill_min_rows_of_partition = ADD_COUNTER_WITH_LEVEL(
- Base::custom_profile(), "SpillMinRowsOfPartition",
TUnit::UNIT, 1);
+ Base::custom_profile(), profile::SPILL_MIN_ROWS_OF_PARTITION,
TUnit::UNIT, 1);
}
std::vector<Dependency*> dependencies() const override {
@@ -770,21 +754,30 @@ public:
}
std::vector<int64_t> _rows_in_partitions;
- // Spill write counters
- // Total time of writing file
- RuntimeProfile::Counter* _spill_write_file_total_size = nullptr;
- RuntimeProfile::Counter* _spill_write_file_total_count = nullptr;
- RuntimeProfile::Counter* _spill_write_file_timer = nullptr;
- RuntimeProfile::Counter* _spill_write_serialize_block_timer = nullptr;
- // Original count of spilled Blocks
- // One Big Block maybe split into multiple small Blocks when actually
written to disk file.
- RuntimeProfile::Counter* _spill_write_block_count = nullptr;
- // Total bytes of spill data in Block format(in memory format)
- RuntimeProfile::Counter* _spill_write_block_data_size = nullptr;
- RuntimeProfile::Counter* _spill_write_rows_count = nullptr;
+
+ // Total time of spill, including spill task scheduling time,
+ // serialize block time, write disk file time,
+ // and read disk file time, deserialize block time etc.
+ RuntimeProfile::Counter* _spill_total_timer = nullptr;
+
+ // Shared spill write counters
+ SpillWriteCounters _write_counters;
+ // Backward-compatible aliases for commonly accessed write counters
+ RuntimeProfile::Counter*& _spill_write_file_timer =
_write_counters.spill_write_file_timer;
+ RuntimeProfile::Counter*& _spill_write_serialize_block_timer =
+ _write_counters.spill_write_serialize_block_timer;
+ RuntimeProfile::Counter*& _spill_write_block_count =
_write_counters.spill_write_block_count;
+ RuntimeProfile::Counter*& _spill_write_block_data_size =
+ _write_counters.spill_write_block_data_size;
+ RuntimeProfile::Counter*& _spill_write_rows_count =
_write_counters.spill_write_rows_count;
+
+ // Sink-only counters
// Spilled file total size
RuntimeProfile::Counter* _spill_file_total_size = nullptr;
-
+ // Total bytes written to spill files (required by SpillFileWriter)
+ RuntimeProfile::Counter* _spill_write_file_total_size = nullptr;
+ // Total number of spill files created (required by SpillFileWriter)
+ RuntimeProfile::Counter* _spill_file_total_count = nullptr;
RuntimeProfile::Counter* _spill_max_rows_of_partition = nullptr;
RuntimeProfile::Counter* _spill_min_rows_of_partition = nullptr;
};
diff --git a/be/src/exec/operator/scan_operator.cpp
b/be/src/exec/operator/scan_operator.cpp
index 4282117afc2..0125172458e 100644
--- a/be/src/exec/operator/scan_operator.cpp
+++ b/be/src/exec/operator/scan_operator.cpp
@@ -50,6 +50,7 @@
#include "exprs/vtopn_pred.h"
#include "runtime/descriptors.h"
#include "runtime/runtime_profile.h"
+#include "runtime/runtime_profile_counter_names.h"
#include "storage/predicate/null_predicate.h"
#include "storage/predicate/predicate_creator.h"
@@ -1030,28 +1031,30 @@ int64_t ScanLocalState<Derived>::limit_per_scanner() {
template <typename Derived>
Status ScanLocalState<Derived>::_init_profile() {
// 1. counters for scan node
- _rows_read_counter = ADD_COUNTER(custom_profile(), "RowsRead",
TUnit::UNIT);
- _num_scanners = ADD_COUNTER(custom_profile(), "NumScanners", TUnit::UNIT);
+ _rows_read_counter = ADD_COUNTER(custom_profile(), profile::ROWS_READ,
TUnit::UNIT);
+ _num_scanners = ADD_COUNTER(custom_profile(), profile::NUM_SCANNERS,
TUnit::UNIT);
//custom_profile()->AddHighWaterMarkCounter("PeakMemoryUsage",
TUnit::BYTES);
// 2. counters for scanners
- _scanner_profile.reset(new RuntimeProfile("Scanner"));
+ _scanner_profile.reset(new RuntimeProfile(profile::SCANNER));
custom_profile()->add_child(_scanner_profile.get(), true, nullptr);
_newly_create_free_blocks_num =
- ADD_COUNTER(_scanner_profile, "NewlyCreateFreeBlocksNum",
TUnit::UNIT);
- _scan_timer = ADD_TIMER(_scanner_profile, "ScannerGetBlockTime");
- _scan_cpu_timer = ADD_TIMER(_scanner_profile, "ScannerCpuTime");
- _filter_timer = ADD_TIMER(_scanner_profile, "ScannerFilterTime");
+ ADD_COUNTER(_scanner_profile,
profile::NEWLY_CREATE_FREE_BLOCKS_NUM, TUnit::UNIT);
+ _scan_timer = ADD_TIMER(_scanner_profile, profile::SCANNER_GET_BLOCK_TIME);
+ _scan_cpu_timer = ADD_TIMER(_scanner_profile, profile::SCANNER_CPU_TIME);
+ _filter_timer = ADD_TIMER(_scanner_profile, profile::SCANNER_FILTER_TIME);
// time of scan thread to wait for worker thread of the thread pool
- _scanner_wait_worker_timer = ADD_TIMER(custom_profile(),
"ScannerWorkerWaitTime");
+ _scanner_wait_worker_timer = ADD_TIMER(custom_profile(),
profile::SCANNER_WORKER_WAIT_TIME);
- _max_scan_concurrency = ADD_COUNTER(custom_profile(),
"MaxScanConcurrency", TUnit::UNIT);
- _min_scan_concurrency = ADD_COUNTER(custom_profile(),
"MinScanConcurrency", TUnit::UNIT);
+ _max_scan_concurrency =
+ ADD_COUNTER(custom_profile(), profile::MAX_SCAN_CONCURRENCY,
TUnit::UNIT);
+ _min_scan_concurrency =
+ ADD_COUNTER(custom_profile(), profile::MIN_SCAN_CONCURRENCY,
TUnit::UNIT);
_peak_running_scanner =
- _scanner_profile->AddHighWaterMarkCounter("RunningScanner",
TUnit::UNIT);
+
_scanner_profile->AddHighWaterMarkCounter(profile::RUNNING_SCANNER,
TUnit::UNIT);
_condition_cache_hit_counter = ADD_COUNTER(_scanner_profile,
"ConditionCacheHit", TUnit::UNIT);
_condition_cache_filtered_rows_counter =
@@ -1059,10 +1062,10 @@ Status ScanLocalState<Derived>::_init_profile() {
// Rows read from storage.
// Include the rows read from doris page cache.
- _scan_rows = ADD_COUNTER_WITH_LEVEL(custom_profile(), "ScanRows",
TUnit::UNIT, 1);
+ _scan_rows = ADD_COUNTER_WITH_LEVEL(custom_profile(), profile::SCAN_ROWS,
TUnit::UNIT, 1);
// Size of data that read from storage.
// Does not include rows that are cached by doris page cache.
- _scan_bytes = ADD_COUNTER_WITH_LEVEL(custom_profile(), "ScanBytes",
TUnit::BYTES, 1);
+ _scan_bytes = ADD_COUNTER_WITH_LEVEL(custom_profile(),
profile::SCAN_BYTES, TUnit::BYTES, 1);
return Status::OK();
}
diff --git a/be/src/exec/operator/spill_counters.h
b/be/src/exec/operator/spill_counters.h
new file mode 100644
index 00000000000..13a04271f54
--- /dev/null
+++ b/be/src/exec/operator/spill_counters.h
@@ -0,0 +1,95 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#pragma once
+#include "runtime/runtime_profile.h"
+#include "runtime/runtime_profile_counter_names.h"
+
+namespace doris {
+
+// Shared spill write counters, used by both PipelineXSpillLocalState (source)
+// and PipelineXSpillSinkLocalState (sink) to eliminate duplicated counter
definitions.
+struct SpillWriteCounters {
+ RuntimeProfile::Counter* spill_write_timer = nullptr;
+ RuntimeProfile::Counter* spill_write_wait_in_queue_task_count = nullptr;
+ RuntimeProfile::Counter* spill_writing_task_count = nullptr;
+ RuntimeProfile::Counter* spill_write_wait_in_queue_timer = nullptr;
+ RuntimeProfile::Counter* spill_write_file_timer = nullptr;
+ RuntimeProfile::Counter* spill_write_serialize_block_timer = nullptr;
+ RuntimeProfile::Counter* spill_write_block_count = nullptr;
+ RuntimeProfile::Counter* spill_write_block_data_size = nullptr;
+ RuntimeProfile::Counter* spill_write_rows_count = nullptr;
+
+ void init(RuntimeProfile* profile) {
+ spill_write_timer = ADD_TIMER_WITH_LEVEL(profile,
profile::SPILL_WRITE_TIME, 1);
+ spill_write_wait_in_queue_task_count = ADD_COUNTER_WITH_LEVEL(
+ profile, profile::SPILL_WRITE_TASK_WAIT_IN_QUEUE_COUNT,
TUnit::UNIT, 1);
+ spill_writing_task_count =
+ ADD_COUNTER_WITH_LEVEL(profile,
profile::SPILL_WRITE_TASK_COUNT, TUnit::UNIT, 1);
+ spill_write_wait_in_queue_timer =
+ ADD_TIMER_WITH_LEVEL(profile,
profile::SPILL_WRITE_TASK_WAIT_IN_QUEUE_TIME, 1);
+ spill_write_file_timer = ADD_TIMER_WITH_LEVEL(profile,
profile::SPILL_WRITE_FILE_TIME, 1);
+ spill_write_serialize_block_timer =
+ ADD_TIMER_WITH_LEVEL(profile,
profile::SPILL_WRITE_SERIALIZE_BLOCK_TIME, 1);
+ spill_write_block_count =
+ ADD_COUNTER_WITH_LEVEL(profile,
profile::SPILL_WRITE_BLOCK_COUNT, TUnit::UNIT, 1);
+ spill_write_block_data_size =
+ ADD_COUNTER_WITH_LEVEL(profile,
profile::SPILL_WRITE_BLOCK_BYTES, TUnit::BYTES, 1);
+ spill_write_rows_count =
+ ADD_COUNTER_WITH_LEVEL(profile, profile::SPILL_WRITE_ROWS,
TUnit::UNIT, 1);
+ }
+};
+
+// Shared spill read counters, used only by PipelineXSpillLocalState (source).
+struct SpillReadCounters {
+ RuntimeProfile::Counter* spill_recover_time = nullptr;
+ RuntimeProfile::Counter* spill_read_wait_in_queue_task_count = nullptr;
+ RuntimeProfile::Counter* spill_reading_task_count = nullptr;
+ RuntimeProfile::Counter* spill_read_wait_in_queue_timer = nullptr;
+ RuntimeProfile::Counter* spill_read_file_time = nullptr;
+ RuntimeProfile::Counter* spill_read_deserialize_block_timer = nullptr;
+ RuntimeProfile::Counter* spill_read_block_count = nullptr;
+ RuntimeProfile::Counter* spill_read_block_data_size = nullptr;
+ RuntimeProfile::Counter* spill_read_file_size = nullptr;
+ RuntimeProfile::Counter* spill_read_rows_count = nullptr;
+ RuntimeProfile::Counter* spill_read_file_count = nullptr;
+
+ void init(RuntimeProfile* profile) {
+ spill_recover_time = ADD_TIMER_WITH_LEVEL(profile,
profile::SPILL_RECOVER_TIME, 1);
+ spill_read_wait_in_queue_task_count = ADD_COUNTER_WITH_LEVEL(
+ profile, profile::SPILL_READ_TASK_WAIT_IN_QUEUE_COUNT,
TUnit::UNIT, 1);
+ spill_reading_task_count =
+ ADD_COUNTER_WITH_LEVEL(profile,
profile::SPILL_READ_TASK_COUNT, TUnit::UNIT, 1);
+ spill_read_wait_in_queue_timer =
+ ADD_TIMER_WITH_LEVEL(profile,
profile::SPILL_READ_TASK_WAIT_IN_QUEUE_TIME, 1);
+ spill_read_file_time = ADD_TIMER_WITH_LEVEL(profile,
profile::SPILL_READ_FILE_TIME, 1);
+ spill_read_deserialize_block_timer =
+ ADD_TIMER_WITH_LEVEL(profile,
profile::SPILL_READ_DESERIALIZE_BLOCK_TIME, 1);
+ spill_read_block_count =
+ ADD_COUNTER_WITH_LEVEL(profile,
profile::SPILL_READ_BLOCK_COUNT, TUnit::UNIT, 1);
+ spill_read_block_data_size =
+ ADD_COUNTER_WITH_LEVEL(profile,
profile::SPILL_READ_BLOCK_BYTES, TUnit::BYTES, 1);
+ spill_read_file_size =
+ ADD_COUNTER_WITH_LEVEL(profile,
profile::SPILL_READ_FILE_BYTES, TUnit::BYTES, 1);
+ spill_read_rows_count =
+ ADD_COUNTER_WITH_LEVEL(profile, profile::SPILL_READ_ROWS,
TUnit::UNIT, 1);
+ spill_read_file_count =
+ ADD_COUNTER_WITH_LEVEL(profile,
profile::SPILL_READ_FILE_COUNT, TUnit::UNIT, 1);
+ }
+};
+
+} // namespace doris
diff --git a/be/src/exec/pipeline/dependency.h
b/be/src/exec/pipeline/dependency.h
index 33c2ebfa2f3..4f26839687b 100644
--- a/be/src/exec/pipeline/dependency.h
+++ b/be/src/exec/pipeline/dependency.h
@@ -45,6 +45,7 @@
#include "exec/sort/partition_sorter.h"
#include "exec/sort/sorter.h"
#include "exec/spill/spill_file.h"
+#include "runtime/runtime_profile_counter_names.h"
#include "util/brpc_closure.h"
#include "util/stack_util.h"
diff --git a/be/src/exec/pipeline/pipeline_task.cpp
b/be/src/exec/pipeline/pipeline_task.cpp
index 6056a072d8e..ffd1817683f 100644
--- a/be/src/exec/pipeline/pipeline_task.cpp
+++ b/be/src/exec/pipeline/pipeline_task.cpp
@@ -45,6 +45,7 @@
#include "runtime/exec_env.h"
#include "runtime/query_context.h"
#include "runtime/runtime_profile.h"
+#include "runtime/runtime_profile_counter_names.h"
#include "runtime/thread_context.h"
#include "runtime/workload_group/workload_group_manager.h"
#include "util/defer_op.h"
@@ -233,25 +234,25 @@ bool
PipelineTask::inject_shared_state(std::shared_ptr<BasicSharedState> shared_
void PipelineTask::_init_profile() {
_task_profile =
std::make_unique<RuntimeProfile>(fmt::format("PipelineTask(index={})", _index));
_parent_profile->add_child(_task_profile.get(), true, nullptr);
- _task_cpu_timer = ADD_TIMER(_task_profile, "TaskCpuTime");
+ _task_cpu_timer = ADD_TIMER(_task_profile, profile::TASK_CPU_TIME);
- static const char* exec_time = "ExecuteTime";
+ static const char* exec_time = profile::EXECUTE_TIME;
_exec_timer = ADD_TIMER(_task_profile, exec_time);
- _prepare_timer = ADD_CHILD_TIMER(_task_profile, "PrepareTime", exec_time);
- _open_timer = ADD_CHILD_TIMER(_task_profile, "OpenTime", exec_time);
- _get_block_timer = ADD_CHILD_TIMER(_task_profile, "GetBlockTime",
exec_time);
- _get_block_counter = ADD_COUNTER(_task_profile, "GetBlockCounter",
TUnit::UNIT);
- _sink_timer = ADD_CHILD_TIMER(_task_profile, "SinkTime", exec_time);
- _close_timer = ADD_CHILD_TIMER(_task_profile, "CloseTime", exec_time);
-
- _wait_worker_timer = ADD_TIMER_WITH_LEVEL(_task_profile, "WaitWorkerTime",
1);
-
- _schedule_counts = ADD_COUNTER(_task_profile, "NumScheduleTimes",
TUnit::UNIT);
- _yield_counts = ADD_COUNTER(_task_profile, "NumYieldTimes", TUnit::UNIT);
- _core_change_times = ADD_COUNTER(_task_profile, "CoreChangeTimes",
TUnit::UNIT);
- _memory_reserve_times = ADD_COUNTER(_task_profile, "MemoryReserveTimes",
TUnit::UNIT);
+ _prepare_timer = ADD_CHILD_TIMER(_task_profile, profile::PREPARE_TIME,
exec_time);
+ _open_timer = ADD_CHILD_TIMER(_task_profile, profile::OPEN_TIME,
exec_time);
+ _get_block_timer = ADD_CHILD_TIMER(_task_profile, profile::GET_BLOCK_TIME,
exec_time);
+ _get_block_counter = ADD_COUNTER(_task_profile,
profile::GET_BLOCK_COUNTER, TUnit::UNIT);
+ _sink_timer = ADD_CHILD_TIMER(_task_profile, profile::SINK_TIME,
exec_time);
+ _close_timer = ADD_CHILD_TIMER(_task_profile, profile::CLOSE_TIME,
exec_time);
+
+ _wait_worker_timer = ADD_TIMER_WITH_LEVEL(_task_profile,
profile::WAIT_WORKER_TIME, 1);
+
+ _schedule_counts = ADD_COUNTER(_task_profile, profile::NUM_SCHEDULE_TIMES,
TUnit::UNIT);
+ _yield_counts = ADD_COUNTER(_task_profile, profile::NUM_YIELD_TIMES,
TUnit::UNIT);
+ _core_change_times = ADD_COUNTER(_task_profile,
profile::CORE_CHANGE_TIMES, TUnit::UNIT);
+ _memory_reserve_times = ADD_COUNTER(_task_profile,
profile::MEMORY_RESERVE_TIMES, TUnit::UNIT);
_memory_reserve_failed_times =
- ADD_COUNTER(_task_profile, "MemoryReserveFailedTimes",
TUnit::UNIT);
+ ADD_COUNTER(_task_profile, profile::MEMORY_RESERVE_FAILED_TIMES,
TUnit::UNIT);
}
void PipelineTask::_fresh_profile_counter() {
diff --git a/be/src/runtime/runtime_profile.cpp
b/be/src/runtime/runtime_profile.cpp
index 15e5b472d90..7246558f811 100644
--- a/be/src/runtime/runtime_profile.cpp
+++ b/be/src/runtime/runtime_profile.cpp
@@ -647,7 +647,11 @@ void RuntimeProfile::pretty_print(std::ostream* s, const
std::string& prefix,
}
}
- RuntimeProfile::print_child_counters(prefix, ROOT_COUNTER, counter_map,
child_counter_map, s);
+ // Build counter tree and prune by profile_level before printing
+ RuntimeProfileCounterTreeNode counter_tree =
+ RuntimeProfileCounterTreeNode::from_map(counter_map,
child_counter_map, ROOT_COUNTER);
+ counter_tree = RuntimeProfileCounterTreeNode::prune_the_tree(counter_tree,
profile_level);
+ counter_tree.pretty_print(s, prefix);
// create copy of _children so we don't need to hold lock while we call
// pretty_print() on the children
diff --git a/be/src/runtime/runtime_profile_counter_names.h
b/be/src/runtime/runtime_profile_counter_names.h
new file mode 100644
index 00000000000..5b0bd86f474
--- /dev/null
+++ b/be/src/runtime/runtime_profile_counter_names.h
@@ -0,0 +1,125 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#pragma once
+
+namespace doris::profile {
+
+// ============================================================
+// Profile node names
+// ============================================================
+
+// Sub-profile names used by all operators
+inline constexpr char COMMON_COUNTERS[] = "CommonCounters";
+inline constexpr char CUSTOM_COUNTERS[] = "CustomCounters";
+inline constexpr char SCANNER[] = "Scanner";
+inline constexpr char FAKER_PROFILE[] = "faker profile";
+
+// ============================================================
+// Source operator common counters (PipelineXLocalState::init)
+// ============================================================
+inline constexpr char ROWS_PRODUCED[] = "RowsProduced";
+inline constexpr char BLOCKS_PRODUCED[] = "BlocksProduced";
+
+// ============================================================
+// Sink operator common counters (PipelineXSinkLocalState::init)
+// ============================================================
+inline constexpr char INPUT_ROWS[] = "InputRows";
+inline constexpr char PENDING_FINISH_DEPENDENCY[] = "PendingFinishDependency";
+
+// ============================================================
+// Shared operator common counters
+// ============================================================
+inline constexpr char EXEC_TIME[] = "ExecTime";
+inline constexpr char INIT_TIME[] = "InitTime";
+inline constexpr char OPEN_TIME[] = "OpenTime";
+inline constexpr char CLOSE_TIME[] = "CloseTime";
+inline constexpr char PROJECTION_TIME[] = "ProjectionTime";
+inline constexpr char MEMORY_USAGE[] = "MemoryUsage";
+
+// ============================================================
+// PipelineTask counters (pipeline_task.cpp)
+// ============================================================
+inline constexpr char TASK_CPU_TIME[] = "TaskCpuTime";
+inline constexpr char EXECUTE_TIME[] = "ExecuteTime";
+inline constexpr char PREPARE_TIME[] = "PrepareTime";
+inline constexpr char GET_BLOCK_TIME[] = "GetBlockTime";
+inline constexpr char GET_BLOCK_COUNTER[] = "GetBlockCounter";
+inline constexpr char SINK_TIME[] = "SinkTime";
+inline constexpr char WAIT_WORKER_TIME[] = "WaitWorkerTime";
+inline constexpr char NUM_SCHEDULE_TIMES[] = "NumScheduleTimes";
+inline constexpr char NUM_YIELD_TIMES[] = "NumYieldTimes";
+inline constexpr char CORE_CHANGE_TIMES[] = "CoreChangeTimes";
+inline constexpr char MEMORY_RESERVE_TIMES[] = "MemoryReserveTimes";
+inline constexpr char MEMORY_RESERVE_FAILED_TIMES[] =
"MemoryReserveFailedTimes";
+
+// ============================================================
+// Scan operator counters (scan_operator.cpp)
+// ============================================================
+inline constexpr char ROWS_READ[] = "RowsRead";
+inline constexpr char NUM_SCANNERS[] = "NumScanners";
+inline constexpr char SCAN_ROWS[] = "ScanRows";
+inline constexpr char SCAN_BYTES[] = "ScanBytes";
+inline constexpr char SCANNER_GET_BLOCK_TIME[] = "ScannerGetBlockTime";
+inline constexpr char SCANNER_CPU_TIME[] = "ScannerCpuTime";
+inline constexpr char SCANNER_FILTER_TIME[] = "ScannerFilterTime";
+inline constexpr char SCANNER_WORKER_WAIT_TIME[] = "ScannerWorkerWaitTime";
+inline constexpr char NEWLY_CREATE_FREE_BLOCKS_NUM[] =
"NewlyCreateFreeBlocksNum";
+inline constexpr char MAX_SCAN_CONCURRENCY[] = "MaxScanConcurrency";
+inline constexpr char MIN_SCAN_CONCURRENCY[] = "MinScanConcurrency";
+inline constexpr char RUNNING_SCANNER[] = "RunningScanner";
+
+// ============================================================
+// Spill write counters (shared between Source and Sink)
+// ============================================================
+inline constexpr char SPILL_TOTAL_TIME[] = "SpillTotalTime";
+inline constexpr char SPILL_WRITE_TIME[] = "SpillWriteTime";
+inline constexpr char SPILL_WRITE_TASK_WAIT_IN_QUEUE_COUNT[] =
"SpillWriteTaskWaitInQueueCount";
+inline constexpr char SPILL_WRITE_TASK_COUNT[] = "SpillWriteTaskCount";
+inline constexpr char SPILL_WRITE_TASK_WAIT_IN_QUEUE_TIME[] =
"SpillWriteTaskWaitInQueueTime";
+inline constexpr char SPILL_WRITE_FILE_TIME[] = "SpillWriteFileTime";
+inline constexpr char SPILL_WRITE_SERIALIZE_BLOCK_TIME[] =
"SpillWriteSerializeBlockTime";
+inline constexpr char SPILL_WRITE_BLOCK_COUNT[] = "SpillWriteBlockCount";
+inline constexpr char SPILL_WRITE_BLOCK_BYTES[] = "SpillWriteBlockBytes";
+inline constexpr char SPILL_WRITE_ROWS[] = "SpillWriteRows";
+
+// Spill write file counters (Source-only)
+inline constexpr char SPILL_WRITE_FILE_BYTES[] = "SpillWriteFileBytes";
+inline constexpr char SPILL_WRITE_FILE_TOTAL_COUNT[] =
"SpillWriteFileTotalCount";
+inline constexpr char SPILL_WRITE_FILE_CURRENT_BYTES[] =
"SpillWriteFileCurrentBytes";
+inline constexpr char SPILL_WRITE_FILE_CURRENT_COUNT[] =
"SpillWriteFileCurrentCount";
+
+// ============================================================
+// Spill read counters (Source-only)
+// ============================================================
+inline constexpr char SPILL_RECOVER_TIME[] = "SpillRecoverTime";
+inline constexpr char SPILL_READ_TASK_WAIT_IN_QUEUE_COUNT[] =
"SpillReadTaskWaitInQueueCount";
+inline constexpr char SPILL_READ_TASK_COUNT[] = "SpillReadTaskCount";
+inline constexpr char SPILL_READ_TASK_WAIT_IN_QUEUE_TIME[] =
"SpillReadTaskWaitInQueueTime";
+inline constexpr char SPILL_READ_FILE_TIME[] = "SpillReadFileTime";
+inline constexpr char SPILL_READ_DESERIALIZE_BLOCK_TIME[] =
"SpillReadDeserializeBlockTime";
+inline constexpr char SPILL_READ_BLOCK_COUNT[] = "SpillReadBlockCount";
+inline constexpr char SPILL_READ_BLOCK_BYTES[] = "SpillReadBlockBytes";
+inline constexpr char SPILL_READ_FILE_BYTES[] = "SpillReadFileBytes";
+inline constexpr char SPILL_READ_ROWS[] = "SpillReadRows";
+inline constexpr char SPILL_READ_FILE_COUNT[] = "SpillReadFileCount";
+
+// Spill partition counters (Sink-only)
+inline constexpr char SPILL_MAX_ROWS_OF_PARTITION[] =
"SpillMaxRowsOfPartition";
+inline constexpr char SPILL_MIN_ROWS_OF_PARTITION[] =
"SpillMinRowsOfPartition";
+
+} // namespace doris::profile
diff --git a/be/src/runtime/runtime_profile_counter_tree_node.cpp
b/be/src/runtime/runtime_profile_counter_tree_node.cpp
index b0239865f35..bb0f8fc316f 100644
--- a/be/src/runtime/runtime_profile_counter_tree_node.cpp
+++ b/be/src/runtime/runtime_profile_counter_tree_node.cpp
@@ -157,4 +157,19 @@ PProfileCounter RuntimeProfileCounterTreeNode::to_proto()
const {
return pcounter;
}
+void RuntimeProfileCounterTreeNode::pretty_print(std::ostream* s, const
std::string& prefix) const {
+ // Print this node's counter (skip ROOT_COUNTER which has no counter)
+ if (name != RuntimeProfile::ROOT_COUNTER && counter != nullptr) {
+ counter->pretty_print(s, prefix, name);
+ }
+
+ // ROOT_COUNTER doesn't print itself, so don't add indentation for its
children;
+ // non-root nodes add " " for their children, matching the old
print_child_counters behavior.
+ const std::string& child_prefix =
+ (name == RuntimeProfile::ROOT_COUNTER) ? prefix : prefix + " ";
+ for (const auto& child : children) {
+ child.pretty_print(s, child_prefix);
+ }
+}
+
} // namespace doris
diff --git a/be/src/runtime/runtime_profile_counter_tree_node.h
b/be/src/runtime/runtime_profile_counter_tree_node.h
index f12b65dcb80..54b2963c742 100644
--- a/be/src/runtime/runtime_profile_counter_tree_node.h
+++ b/be/src/runtime/runtime_profile_counter_tree_node.h
@@ -59,6 +59,10 @@ public:
PProfileCounter to_proto() const;
+ // Print the counter tree to the output stream, similar to the old
+ // RuntimeProfile::print_child_counters() but operating on the pruned tree.
+ void pretty_print(std::ostream* s, const std::string& prefix) const;
+
private:
std::string name;
// counter is not owned by this class
diff --git a/be/src/runtime/runtime_state.cpp b/be/src/runtime/runtime_state.cpp
index dc8de7cbe72..dcd00b49996 100644
--- a/be/src/runtime/runtime_state.cpp
+++ b/be/src/runtime/runtime_state.cpp
@@ -105,7 +105,7 @@ RuntimeState::RuntimeState(const TUniqueId& instance_id,
const TUniqueId& query_
RuntimeState::RuntimeState(const TUniqueId& query_id, int32_t fragment_id,
const TQueryOptions& query_options, const
TQueryGlobals& query_globals,
ExecEnv* exec_env, QueryContext* ctx)
- : _profile("PipelineX " + std::to_string(fragment_id)),
+ : _profile(fmt::format("PipelineX(fragment_id={})", fragment_id)),
_load_channel_profile("<unnamed>"),
_obj_pool(new ObjectPool()),
_unreported_error_idx(0),
@@ -130,7 +130,7 @@ RuntimeState::RuntimeState(const TUniqueId& query_id,
int32_t fragment_id,
const TQueryOptions& query_options, const
TQueryGlobals& query_globals,
ExecEnv* exec_env,
const std::shared_ptr<MemTrackerLimiter>&
query_mem_tracker)
- : _profile("PipelineX " + std::to_string(fragment_id)),
+ : _profile(fmt::format("PipelineX(fragment_id={})", fragment_id)),
_load_channel_profile("<unnamed>"),
_obj_pool(new ObjectPool()),
_unreported_error_idx(0),
@@ -517,7 +517,7 @@ std::vector<std::shared_ptr<RuntimeProfile>>
RuntimeState::build_pipeline_profil
size_t pip_idx = 0;
for (auto& pipeline_profile : _pipeline_id_to_profile) {
pipeline_profile =
- std::make_shared<RuntimeProfile>("Pipeline : " +
std::to_string(pip_idx));
+
std::make_shared<RuntimeProfile>(fmt::format("Pipeline(id={})", pip_idx));
pip_idx++;
}
}
diff --git a/be/test/runtime/runtime_profile_profile_level_test.cpp
b/be/test/runtime/runtime_profile_profile_level_test.cpp
index f800625008d..5a3d438d6aa 100644
--- a/be/test/runtime/runtime_profile_profile_level_test.cpp
+++ b/be/test/runtime/runtime_profile_profile_level_test.cpp
@@ -255,4 +255,112 @@ TEST_F(RuntimeProfileProfileLevelTest, ConcurrentTest) {
add_counter_thread.join();
to_thrift_thread.join();
}
+
+// Verify that pretty_print() respects profile_level parameter for flat
counters.
+TEST_F(RuntimeProfileProfileLevelTest, PrettyPrintLevelFilteringFlat) {
+ RuntimeProfile profile("test");
+ auto* counter_l0 = profile.add_counter("CounterLevel0", TUnit::UNIT, "",
0);
+ auto* counter_l1 = profile.add_counter("CounterLevel1", TUnit::UNIT, "",
1);
+ auto* counter_l2 = profile.add_counter("CounterLevel2", TUnit::UNIT, "",
2);
+ counter_l0->set(int64_t(100));
+ counter_l1->set(int64_t(200));
+ counter_l2->set(int64_t(300));
+
+ // Level 0: only level-0 counters should appear
+ {
+ std::stringstream ss;
+ profile.pretty_print(&ss, "", 0);
+ std::string output = ss.str();
+ EXPECT_TRUE(output.find("CounterLevel0") != std::string::npos)
+ << "Level-0 counter should appear at profile_level=0";
+ EXPECT_TRUE(output.find("CounterLevel1") == std::string::npos)
+ << "Level-1 counter should NOT appear at profile_level=0";
+ EXPECT_TRUE(output.find("CounterLevel2") == std::string::npos)
+ << "Level-2 counter should NOT appear at profile_level=0";
+ }
+
+ // Level 1: level-0 and level-1 counters should appear
+ {
+ std::stringstream ss;
+ profile.pretty_print(&ss, "", 1);
+ std::string output = ss.str();
+ EXPECT_TRUE(output.find("CounterLevel0") != std::string::npos);
+ EXPECT_TRUE(output.find("CounterLevel1") != std::string::npos);
+ EXPECT_TRUE(output.find("CounterLevel2") == std::string::npos)
+ << "Level-2 counter should NOT appear at profile_level=1";
+ }
+
+ // Level 2: all counters should appear
+ {
+ std::stringstream ss;
+ profile.pretty_print(&ss, "", 2);
+ std::string output = ss.str();
+ EXPECT_TRUE(output.find("CounterLevel0") != std::string::npos);
+ EXPECT_TRUE(output.find("CounterLevel1") != std::string::npos);
+ EXPECT_TRUE(output.find("CounterLevel2") != std::string::npos);
+ }
+}
+
+// Verify that pretty_print() respects profile_level for nested (parent-child)
counters.
+TEST_F(RuntimeProfileProfileLevelTest, PrettyPrintLevelFilteringNested) {
+ /*
+ * Tree structure:
+ * ROOT_COUNTER
+ * parent_l0 (level 0)
+ * child_l0 (level 0)
+ * child_l1 (level 1)
+ * child_l2 (level 2)
+ */
+ RuntimeProfile profile("nested_test");
+ profile.add_counter("parent_l0", TUnit::UNIT,
RuntimeProfile::ROOT_COUNTER, 0);
+ profile.add_counter("child_l0", TUnit::UNIT, "parent_l0", 0);
+ profile.add_counter("child_l1", TUnit::UNIT, "parent_l0", 1);
+ profile.add_counter("child_l2", TUnit::UNIT, "parent_l0", 2);
+
+ // Level 0: parent_l0 and child_l0 should appear, child_l1/l2 should be
pruned
+ {
+ std::stringstream ss;
+ profile.pretty_print(&ss, "", 0);
+ std::string output = ss.str();
+ EXPECT_TRUE(output.find("parent_l0") != std::string::npos);
+ EXPECT_TRUE(output.find("child_l0") != std::string::npos);
+ EXPECT_TRUE(output.find("child_l1") == std::string::npos);
+ EXPECT_TRUE(output.find("child_l2") == std::string::npos);
+ }
+
+ // Level 1: parent_l0, child_l0, child_l1 should appear
+ {
+ std::stringstream ss;
+ profile.pretty_print(&ss, "", 1);
+ std::string output = ss.str();
+ EXPECT_TRUE(output.find("parent_l0") != std::string::npos);
+ EXPECT_TRUE(output.find("child_l0") != std::string::npos);
+ EXPECT_TRUE(output.find("child_l1") != std::string::npos);
+ EXPECT_TRUE(output.find("child_l2") == std::string::npos);
+ }
+}
+
+// Verify pretty_print indentation: ROOT_COUNTER's direct children should be at
+// the same prefix level, and grandchildren should be indented by 2 more
spaces.
+TEST_F(RuntimeProfileProfileLevelTest, PrettyPrintIndentation) {
+ RuntimeProfile profile("indent_test");
+ profile.add_counter("top_counter", TUnit::UNIT,
RuntimeProfile::ROOT_COUNTER, 0);
+ profile.add_counter("sub_counter", TUnit::UNIT, "top_counter", 0);
+
+ std::stringstream ss;
+ profile.pretty_print(&ss, "PFX", 2);
+ std::string output = ss.str();
+
+ // ROOT_COUNTER's direct children should have prefix "PFX - "
+ // (PFX + " - " from Counter::pretty_print)
+ EXPECT_TRUE(output.find("PFX - top_counter:") != std::string::npos)
+ << "Top-level counter should be at prefix level. Output:\n"
+ << output;
+
+ // Child of top_counter should have prefix "PFX " + " - " = "PFX -
"
+ EXPECT_TRUE(output.find("PFX - sub_counter:") != std::string::npos)
+ << "Sub-counter should be indented by 2 more spaces. Output:\n"
+ << output;
+}
+
} // namespace doris
diff --git a/be/test/util/profile_spec_test.cpp
b/be/test/util/profile_spec_test.cpp
index 1111aaed6af..7c1caf3292e 100644
--- a/be/test/util/profile_spec_test.cpp
+++ b/be/test/util/profile_spec_test.cpp
@@ -125,7 +125,7 @@ TEST_F(ProfileSpecTest, SourceOperatorNameSuffixTest2) {
op._nereids_id = 100;
RuntimeState* runtime_state = nullptr;
auto local_state = std::make_unique<MockLocalState>(runtime_state, &op);
- ASSERT_EQ(local_state->name_suffix(), "(nereids_id=100)(id=1)");
+ ASSERT_EQ(local_state->name_suffix(), "(nereids_id=100, id=1)");
}
TEST_F(ProfileSpecTest, DataStreamSinkOperatorTest) {
@@ -144,7 +144,7 @@ TEST_F(ProfileSpecTest, DataStreamSinkOperatorTest) {
ExchangeSinkLocalState local_state(state.get());
local_state._parent = &sink_op;
- ASSERT_EQ(local_state.name_suffix(), "(dest_id=101)");
+ ASSERT_EQ(local_state.name_suffix(), "(id=101, dest_id=101)");
}
TEST_F(ProfileSpecTest, CommonCountersCustomCounters) {
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/common/profile/RuntimeProfile.java
b/fe/fe-core/src/main/java/org/apache/doris/common/profile/RuntimeProfile.java
index 2f499a8dd4c..6641bc95d6a 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/common/profile/RuntimeProfile.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/common/profile/RuntimeProfile.java
@@ -47,7 +47,6 @@ import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
-import java.util.Objects;
import java.util.Set;
import java.util.TreeSet;
import java.util.concurrent.ConcurrentMap;
@@ -497,12 +496,11 @@ public class RuntimeProfile {
}
boolean shouldBeIncluded() {
- if (Objects.equals(this.name, "CommonCounters") ||
Objects.equals(this.name, "CustomCounters")
- || Objects.equals(this.name, "Scanner")) {
+ if ("CommonCounters".equals(this.name) ||
"CustomCounters".equals(this.name)
+ || "Scanner".equals(this.name)) {
return true;
- } else {
- return this.name.matches(".*Pipeline.*") ||
this.name.matches(".*_OPERATOR.*");
}
+ return this.name.startsWith("Pipeline") ||
this.name.contains("_OPERATOR");
}
private static void collectActualRowCount(RuntimeProfile mergedProfile) {
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/common/profile/AutoProfileTest.java
b/fe/fe-core/src/test/java/org/apache/doris/common/profile/AutoProfileTest.java
index 5475cfc4c98..eae3b2c93c3 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/common/profile/AutoProfileTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/common/profile/AutoProfileTest.java
@@ -64,15 +64,15 @@ public class AutoProfileTest {
result = System.currentTimeMillis();
}
};
- profile.autoProfileDurationMs = 1000;
- Thread.sleep(899);
+ profile.autoProfileDurationMs = 10000;
+ Thread.sleep(100);
profile.updateSummary(summaryInfo, true, null);
Assertions.assertNull(ProfileManager.getInstance().queryIdToProfileMap.get(profile.getId()));
profile = createProfile();
profile.setSummaryProfile(summaryProfile);
- profile.autoProfileDurationMs = 500;
- Thread.sleep(899);
+ profile.autoProfileDurationMs = 50;
+ Thread.sleep(200);
profile.updateSummary(summaryInfo, true, null);
Assertions.assertNotNull(ProfileManager.getInstance().queryIdToProfileMap.get(profile.getId()));
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
