This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.1 by this push:
new 972c37de956 [opt](scanner profile) Rename some filed name to keep
consistent with audit log. #41993 (#42137)
972c37de956 is described below
commit 972c37de9561acbe75522578b67d0d8b53699dcb
Author: zhiqiang <[email protected]>
AuthorDate: Fri Oct 25 01:05:13 2024 +0800
[opt](scanner profile) Rename some filed name to keep consistent with audit
log. #41993 (#42137)
cherry pick from #41993
---
be/src/pipeline/exec/olap_scan_operator.cpp | 2 ++
be/src/pipeline/exec/scan_operator.cpp | 7 +++++++
be/src/pipeline/exec/scan_operator.h | 3 +++
be/src/vec/exec/scan/new_olap_scanner.cpp | 25 +++++++++++++++----------
be/src/vec/exec/scan/new_olap_scanner.h | 4 ++--
be/src/vec/exec/scan/vscan_node.cpp | 3 +++
be/src/vec/exec/scan/vscan_node.h | 3 +++
7 files changed, 35 insertions(+), 12 deletions(-)
diff --git a/be/src/pipeline/exec/olap_scan_operator.cpp
b/be/src/pipeline/exec/olap_scan_operator.cpp
index 1c7d34058b6..b32f9e0457e 100644
--- a/be/src/pipeline/exec/olap_scan_operator.cpp
+++ b/be/src/pipeline/exec/olap_scan_operator.cpp
@@ -53,6 +53,8 @@ Status OlapScanLocalState::_init_profile() {
_read_compressed_counter = ADD_COUNTER(_segment_profile,
"CompressedBytesRead", TUnit::BYTES);
_read_uncompressed_counter =
ADD_COUNTER(_segment_profile, "UncompressedBytesRead",
TUnit::BYTES);
+ _scan_rows = ADD_COUNTER(_runtime_profile, "ScanRows", TUnit::UNIT);
+ _scan_bytes = ADD_COUNTER(_runtime_profile, "ScanBytes", TUnit::BYTES);
_block_load_timer = ADD_TIMER(_segment_profile, "BlockLoadTime");
_block_load_counter = ADD_COUNTER(_segment_profile, "BlocksLoad",
TUnit::UNIT);
_block_fetch_timer = ADD_TIMER(_scanner_profile, "BlockFetchTime");
diff --git a/be/src/pipeline/exec/scan_operator.cpp
b/be/src/pipeline/exec/scan_operator.cpp
index 2c3e85b972b..7e91b501ab6 100644
--- a/be/src/pipeline/exec/scan_operator.cpp
+++ b/be/src/pipeline/exec/scan_operator.cpp
@@ -1119,6 +1119,13 @@ Status ScanLocalState<Derived>::_init_profile() {
_peak_running_scanner =
_scanner_profile->AddHighWaterMarkCounter("PeakRunningScanner",
TUnit::UNIT);
+
+ // Rows read from storage.
+ // Include the rows read from doris page cache.
+ _scan_rows = ADD_COUNTER(_runtime_profile, "ScanRows", TUnit::UNIT);
+ // Size of data that read from storage.
+ // Does not include rows that are cached by doris page cache.
+ _scan_bytes = ADD_COUNTER(_runtime_profile, "ScanBytes", TUnit::BYTES);
return Status::OK();
}
diff --git a/be/src/pipeline/exec/scan_operator.h
b/be/src/pipeline/exec/scan_operator.h
index 126ed529227..2ff5db88ac6 100644
--- a/be/src/pipeline/exec/scan_operator.h
+++ b/be/src/pipeline/exec/scan_operator.h
@@ -130,6 +130,9 @@ protected:
RuntimeProfile::Counter* _num_scanners = nullptr;
RuntimeProfile::Counter* _wait_for_rf_timer = nullptr;
+
+ RuntimeProfile::Counter* _scan_rows = nullptr;
+ RuntimeProfile::Counter* _scan_bytes = nullptr;
};
template <typename LocalStateType>
diff --git a/be/src/vec/exec/scan/new_olap_scanner.cpp
b/be/src/vec/exec/scan/new_olap_scanner.cpp
index 7e2baf2ae76..aaf6fbdf3b4 100644
--- a/be/src/vec/exec/scan/new_olap_scanner.cpp
+++ b/be/src/vec/exec/scan/new_olap_scanner.cpp
@@ -536,17 +536,21 @@ void NewOlapScanner::_update_realtime_counters() {
NewOlapScanNode* olap_parent = static_cast<NewOlapScanNode*>(_parent);
pipeline::OlapScanLocalState* local_state =
static_cast<pipeline::OlapScanLocalState*>(_local_state);
- auto& stats = _tablet_reader->stats();
+ const OlapReaderStatistics& stats = _tablet_reader->stats();
COUNTER_UPDATE(olap_parent ? olap_parent->_read_compressed_counter
: local_state->_read_compressed_counter,
stats.compressed_bytes_read);
- _compressed_bytes_read += stats.compressed_bytes_read;
+ COUNTER_UPDATE(olap_parent ? olap_parent->_scan_bytes :
local_state->_scan_bytes,
+ stats.compressed_bytes_read);
+ _scan_bytes += stats.compressed_bytes_read;
_tablet_reader->mutable_stats()->compressed_bytes_read = 0;
COUNTER_UPDATE(olap_parent ? olap_parent->_raw_rows_counter :
local_state->_raw_rows_counter,
stats.raw_rows_read);
+ COUNTER_UPDATE(olap_parent ? olap_parent->_scan_rows :
local_state->_scan_rows,
+ stats.raw_rows_read);
+ _scan_rows += stats.raw_rows_read;
// if raw_rows_read is reset, scanNode will scan all table rows which may
cause BE crash
- _raw_rows_read += stats.raw_rows_read;
_tablet_reader->mutable_stats()->raw_rows_read = 0;
}
@@ -563,7 +567,8 @@ void NewOlapScanner::_collect_profile_before_close() {
#define INCR_COUNTER(Parent)
\
COUNTER_UPDATE(Parent->_io_timer, stats.io_ns);
\
COUNTER_UPDATE(Parent->_read_compressed_counter,
stats.compressed_bytes_read); \
- _compressed_bytes_read += stats.compressed_bytes_read;
\
+ COUNTER_UPDATE(Parent->_scan_bytes, stats.compressed_bytes_read);
\
+ _scan_bytes += stats.compressed_bytes_read;
\
COUNTER_UPDATE(Parent->_decompressor_timer, stats.decompress_ns);
\
COUNTER_UPDATE(Parent->_read_uncompressed_counter,
stats.uncompressed_bytes_read); \
COUNTER_UPDATE(Parent->_block_load_timer, stats.block_load_ns);
\
@@ -571,8 +576,8 @@ void NewOlapScanner::_collect_profile_before_close() {
COUNTER_UPDATE(Parent->_block_fetch_timer, stats.block_fetch_ns);
\
COUNTER_UPDATE(Parent->_delete_bitmap_get_agg_timer,
stats.delete_bitmap_get_agg_ns); \
COUNTER_UPDATE(Parent->_block_convert_timer, stats.block_convert_ns);
\
- COUNTER_UPDATE(Parent->_raw_rows_counter, stats.raw_rows_read);
\
- _raw_rows_read += _tablet_reader->mutable_stats()->raw_rows_read;
\
+ COUNTER_UPDATE(Parent->_scan_rows, stats.raw_rows_read);
\
+ _scan_rows += _tablet_reader->mutable_stats()->raw_rows_read;
\
COUNTER_UPDATE(Parent->_vec_cond_timer, stats.vec_cond_ns);
\
COUNTER_UPDATE(Parent->_short_cond_timer, stats.short_cond_ns);
\
COUNTER_UPDATE(Parent->_expr_filter_timer, stats.expr_filter_ns);
\
@@ -665,11 +670,11 @@ void NewOlapScanner::_collect_profile_before_close() {
#undef INCR_COUNTER
#endif
// Update metrics
-
DorisMetrics::instance()->query_scan_bytes->increment(_compressed_bytes_read);
- DorisMetrics::instance()->query_scan_rows->increment(_raw_rows_read);
+ DorisMetrics::instance()->query_scan_bytes->increment(_scan_bytes);
+ DorisMetrics::instance()->query_scan_rows->increment(_scan_rows);
auto& tablet = _tablet_reader_params.tablet;
- tablet->query_scan_bytes->increment(_compressed_bytes_read);
- tablet->query_scan_rows->increment(_raw_rows_read);
+ tablet->query_scan_bytes->increment(_scan_bytes);
+ tablet->query_scan_rows->increment(_scan_rows);
tablet->query_scan_count->increment(1);
}
diff --git a/be/src/vec/exec/scan/new_olap_scanner.h
b/be/src/vec/exec/scan/new_olap_scanner.h
index 7070594383c..cdadf8f7f49 100644
--- a/be/src/vec/exec/scan/new_olap_scanner.h
+++ b/be/src/vec/exec/scan/new_olap_scanner.h
@@ -101,8 +101,8 @@ private:
std::unordered_set<uint32_t> _tablet_columns_convert_to_null_set;
// ========= profiles ==========
- int64_t _compressed_bytes_read = 0;
- int64_t _raw_rows_read = 0;
+ int64_t _scan_bytes = 0;
+ int64_t _scan_rows = 0;
bool _profile_updated = false;
};
} // namespace vectorized
diff --git a/be/src/vec/exec/scan/vscan_node.cpp
b/be/src/vec/exec/scan/vscan_node.cpp
index e3d9f916b5d..a7842188feb 100644
--- a/be/src/vec/exec/scan/vscan_node.cpp
+++ b/be/src/vec/exec/scan/vscan_node.cpp
@@ -300,6 +300,9 @@ Status VScanNode::_init_profile() {
_max_scanner_thread_num = ADD_COUNTER(_runtime_profile,
"MaxScannerThreadNum", TUnit::UNIT);
+ _scan_rows = ADD_COUNTER(_runtime_profile, "ScanRows", TUnit::UNIT);
+ _scan_bytes = ADD_COUNTER(_runtime_profile, "ScanBytes", TUnit::BYTES);
+
return Status::OK();
}
diff --git a/be/src/vec/exec/scan/vscan_node.h
b/be/src/vec/exec/scan/vscan_node.h
index 43a7437cafd..7b802bf5894 100644
--- a/be/src/vec/exec/scan/vscan_node.h
+++ b/be/src/vec/exec/scan/vscan_node.h
@@ -379,6 +379,9 @@ protected:
RuntimeProfile::HighWaterMarkCounter* _free_blocks_memory_usage = nullptr;
RuntimeProfile::Counter* _scale_up_scanners_counter = nullptr;
+ RuntimeProfile::Counter* _scan_rows = nullptr;
+ RuntimeProfile::Counter* _scan_bytes = nullptr;
+
std::unordered_map<std::string, int> _colname_to_slot_id;
TPushAggOp::type _push_down_agg_type;
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
