This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 68525fc112c [feature](profile) add RuntimeFilterInfo in merge profile
#27869
68525fc112c is described below
commit 68525fc112c887216553c409541a5a3ac5888885
Author: Mryange <[email protected]>
AuthorDate: Fri Dec 1 21:42:25 2023 +0800
[feature](profile) add RuntimeFilterInfo in merge profile #27869
---
be/src/pipeline/exec/olap_scan_operator.cpp | 11 +++++++++++
be/src/pipeline/exec/olap_scan_operator.h | 2 ++
be/src/util/runtime_profile.h | 2 ++
be/src/vec/exec/scan/new_olap_scan_node.cpp | 11 +++++++++++
be/src/vec/exec/scan/new_olap_scan_node.h | 2 ++
.../main/java/org/apache/doris/common/util/AggCounter.java | 3 +++
.../java/org/apache/doris/common/util/RuntimeProfile.java | 12 ++++++++----
7 files changed, 39 insertions(+), 4 deletions(-)
diff --git a/be/src/pipeline/exec/olap_scan_operator.cpp
b/be/src/pipeline/exec/olap_scan_operator.cpp
index 059f961501d..2f22daa5190 100644
--- a/be/src/pipeline/exec/olap_scan_operator.cpp
+++ b/be/src/pipeline/exec/olap_scan_operator.cpp
@@ -127,6 +127,7 @@ Status OlapScanLocalState::_init_profile() {
_filtered_segment_counter = ADD_COUNTER(_segment_profile,
"NumSegmentFiltered", TUnit::UNIT);
_total_segment_counter = ADD_COUNTER(_segment_profile, "NumSegmentTotal",
TUnit::UNIT);
_tablet_counter = ADD_COUNTER(_runtime_profile, "TabletNum", TUnit::UNIT);
+ _runtime_filter_info = ADD_LABEL_COUNTER_WITH_LEVEL(_runtime_profile,
"RuntimeFilterInfo", 1);
return Status::OK();
}
@@ -477,6 +478,16 @@ void OlapScanLocalState::add_filter_info(int id, const
PredicateFilterInfo& upda
// add info
_segment_profile->add_info_string(filter_name, info_str);
+
+ const std::string rf_name = "filter id = " + std::to_string(id) + " ";
+
+ // add counter
+ auto* input_count = ADD_CHILD_COUNTER_WITH_LEVEL(_runtime_profile, rf_name
+ "input",
+ TUnit::UNIT,
"RuntimeFilterInfo", 1);
+ auto* filtered_count = ADD_CHILD_COUNTER_WITH_LEVEL(_runtime_profile,
rf_name + "filtered",
+ TUnit::UNIT,
"RuntimeFilterInfo", 1);
+ COUNTER_UPDATE(input_count, info.input_row);
+ COUNTER_UPDATE(filtered_count, info.filtered_row);
}
OlapScanOperatorX::OlapScanOperatorX(ObjectPool* pool, const TPlanNode& tnode,
int operator_id,
diff --git a/be/src/pipeline/exec/olap_scan_operator.h
b/be/src/pipeline/exec/olap_scan_operator.h
index 0527fa6f44d..1f0fac55a43 100644
--- a/be/src/pipeline/exec/olap_scan_operator.h
+++ b/be/src/pipeline/exec/olap_scan_operator.h
@@ -175,6 +175,8 @@ private:
// total number of segment related to this scan node
RuntimeProfile::Counter* _total_segment_counter = nullptr;
+ RuntimeProfile::Counter* _runtime_filter_info = nullptr;
+
std::mutex _profile_mtx;
};
diff --git a/be/src/util/runtime_profile.h b/be/src/util/runtime_profile.h
index fbddb9dc4ed..d5233d40f2c 100644
--- a/be/src/util/runtime_profile.h
+++ b/be/src/util/runtime_profile.h
@@ -51,6 +51,8 @@ class TRuntimeProfileTree;
#define MACRO_CONCAT(x, y) CONCAT_IMPL(x, y)
#define ADD_LABEL_COUNTER(profile, name) (profile)->add_counter(name,
TUnit::NONE)
+#define ADD_LABEL_COUNTER_WITH_LEVEL(profile, name, type) \
+ (profile)->add_counter_with_level(name, TUnit::NONE, type)
#define ADD_COUNTER(profile, name, type) (profile)->add_counter(name, type)
#define ADD_COUNTER_WITH_LEVEL(profile, name, type, level) \
(profile)->add_counter_with_level(name, type, level)
diff --git a/be/src/vec/exec/scan/new_olap_scan_node.cpp
b/be/src/vec/exec/scan/new_olap_scan_node.cpp
index be06280d799..8c67acc1916 100644
--- a/be/src/vec/exec/scan/new_olap_scan_node.cpp
+++ b/be/src/vec/exec/scan/new_olap_scan_node.cpp
@@ -193,6 +193,7 @@ Status NewOlapScanNode::_init_profile() {
_filtered_segment_counter = ADD_COUNTER(_segment_profile,
"NumSegmentFiltered", TUnit::UNIT);
_total_segment_counter = ADD_COUNTER(_segment_profile, "NumSegmentTotal",
TUnit::UNIT);
+ _runtime_filter_info = ADD_LABEL_COUNTER_WITH_LEVEL(_runtime_profile,
"RuntimeFilterInfo", 1);
return Status::OK();
}
@@ -718,6 +719,16 @@ void NewOlapScanNode::add_filter_info(int id, const
PredicateFilterInfo& update_
// add info
_segment_profile->add_info_string(filter_name, info_str);
+
+ const std::string rf_name = "filter id = " + std::to_string(id) + " ";
+
+ // add counter
+ auto* input_count = ADD_CHILD_COUNTER_WITH_LEVEL(_runtime_profile, rf_name
+ "input",
+ TUnit::UNIT,
"RuntimeFilterInfo", 1);
+ auto* filtered_count = ADD_CHILD_COUNTER_WITH_LEVEL(_runtime_profile,
rf_name + "filtered",
+ TUnit::UNIT,
"RuntimeFilterInfo", 1);
+ COUNTER_UPDATE(input_count, info.input_row);
+ COUNTER_UPDATE(filtered_count, info.filtered_row);
}
}; // namespace doris::vectorized
diff --git a/be/src/vec/exec/scan/new_olap_scan_node.h
b/be/src/vec/exec/scan/new_olap_scan_node.h
index 06b68497b51..c62e7028cdc 100644
--- a/be/src/vec/exec/scan/new_olap_scan_node.h
+++ b/be/src/vec/exec/scan/new_olap_scan_node.h
@@ -202,6 +202,8 @@ private:
RuntimeProfile::Counter* _output_index_result_column_timer = nullptr;
+ RuntimeProfile::Counter* _runtime_filter_info = nullptr;
+
// number of segment filtered by column stat when creating seg iterator
RuntimeProfile::Counter* _filtered_segment_counter = nullptr;
// total number of segment related to this scan node
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/common/util/AggCounter.java
b/fe/fe-core/src/main/java/org/apache/doris/common/util/AggCounter.java
index d7eb7184954..69380e14543 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/common/util/AggCounter.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/AggCounter.java
@@ -35,6 +35,9 @@ public class AggCounter extends Counter {
}
public void addCounter(Counter counter) {
+ if (counter == null) {
+ return;
+ }
if (number == 0) {
max.setValue(counter.getValue());
sum.setValue(counter.getValue());
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/common/util/RuntimeProfile.java
b/fe/fe-core/src/main/java/org/apache/doris/common/util/RuntimeProfile.java
index 251e6417fb2..080336871d1 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/common/util/RuntimeProfile.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/RuntimeProfile.java
@@ -484,19 +484,18 @@ public class RuntimeProfile {
}
}
- private static void mergeCounters(String counterName, List<RuntimeProfile>
profiles,
+ private static void mergeCounters(String parentCounterName,
List<RuntimeProfile> profiles,
RuntimeProfile simpleProfile) {
if (profiles.size() == 0) {
return;
}
RuntimeProfile templateProfile = profiles.get(0);
- Set<String> childCounterSet =
templateProfile.childCounterMap.get(counterName);
+ Set<String> childCounterSet =
templateProfile.childCounterMap.get(parentCounterName);
if (childCounterSet == null) {
return;
}
for (String childCounterName : childCounterSet) {
Counter counter = templateProfile.counterMap.get(childCounterName);
- mergeCounters(childCounterName, profiles, simpleProfile);
if (counter.getLevel() == 1) {
Counter oldCounter =
profiles.get(0).counterMap.get(childCounterName);
AggCounter aggCounter = new AggCounter(oldCounter.getType());
@@ -504,8 +503,13 @@ public class RuntimeProfile {
Counter orgCounter =
profile.counterMap.get(childCounterName);
aggCounter.addCounter(orgCounter);
}
- simpleProfile.addCounter(childCounterName, aggCounter,
ROOT_COUNTER);
+ if (simpleProfile.counterMap.containsKey(parentCounterName)) {
+ simpleProfile.addCounter(childCounterName, aggCounter,
parentCounterName);
+ } else {
+ simpleProfile.addCounter(childCounterName, aggCounter,
ROOT_COUNTER);
+ }
}
+ mergeCounters(childCounterName, profiles, simpleProfile);
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
