This is an automated email from the ASF dual-hosted git repository.
kxiao pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 6942e163083 [opt](index) add more inverted index profile metrics
(#36696)
6942e163083 is described below
commit 6942e163083b1484eb7fb4e57960a8820e21a0a9
Author: Kang <[email protected]>
AuthorDate: Mon Aug 5 12:39:04 2024 +0800
[opt](index) add more inverted index profile metrics (#36696)
add more inverted index query profile:
- InvertedIndexQueryNullBitmapTime
- InvertedIndexSearcherCacheHit
- InvertedIndexSearcherCacheMiss
---
be/src/olap/olap_common.h | 3 +++
be/src/olap/rowset/segment_v2/inverted_index_reader.cpp | 8 ++++++--
be/src/olap/rowset/segment_v2/inverted_index_reader.h | 5 +++--
be/src/pipeline/exec/olap_scan_operator.cpp | 6 ++++++
be/src/pipeline/exec/olap_scan_operator.h | 3 +++
be/src/vec/exec/scan/new_olap_scanner.cpp | 6 ++++++
6 files changed, 27 insertions(+), 4 deletions(-)
diff --git a/be/src/olap/olap_common.h b/be/src/olap/olap_common.h
index 5d188f40caf..fee58a87501 100644
--- a/be/src/olap/olap_common.h
+++ b/be/src/olap/olap_common.h
@@ -367,10 +367,13 @@ struct OlapReaderStatistics {
int64_t inverted_index_query_timer = 0;
int64_t inverted_index_query_cache_hit = 0;
int64_t inverted_index_query_cache_miss = 0;
+ int64_t inverted_index_query_null_bitmap_timer = 0;
int64_t inverted_index_query_bitmap_copy_timer = 0;
int64_t inverted_index_query_bitmap_op_timer = 0;
int64_t inverted_index_searcher_open_timer = 0;
int64_t inverted_index_searcher_search_timer = 0;
+ int64_t inverted_index_searcher_cache_hit = 0;
+ int64_t inverted_index_searcher_cache_miss = 0;
int64_t output_index_result_column_timer = 0;
// number of segment filtered by column stat when creating seg iterator
diff --git a/be/src/olap/rowset/segment_v2/inverted_index_reader.cpp
b/be/src/olap/rowset/segment_v2/inverted_index_reader.cpp
index 2ac283e6e34..02339c00080 100644
--- a/be/src/olap/rowset/segment_v2/inverted_index_reader.cpp
+++ b/be/src/olap/rowset/segment_v2/inverted_index_reader.cpp
@@ -187,8 +187,10 @@ void
InvertedIndexReader::get_analyse_result(std::vector<std::string>& analyse_r
}
}
-Status InvertedIndexReader::read_null_bitmap(InvertedIndexQueryCacheHandle*
cache_handle,
+Status InvertedIndexReader::read_null_bitmap(OlapReaderStatistics* stats,
+ InvertedIndexQueryCacheHandle*
cache_handle,
lucene::store::Directory* dir) {
+ SCOPED_RAW_TIMER(&stats->inverted_index_query_null_bitmap_timer);
lucene::store::IndexInput* null_bitmap_in = nullptr;
bool owned_dir = false;
try {
@@ -244,9 +246,11 @@ Status InvertedIndexReader::handle_searcher_cache(
InvertedIndexSearcherCache::CacheKey searcher_cache_key(index_file_key);
if (InvertedIndexSearcherCache::instance()->lookup(searcher_cache_key,
inverted_index_cache_handle)) {
+ stats->inverted_index_searcher_cache_hit++;
return Status::OK();
} else {
// searcher cache miss
+ stats->inverted_index_searcher_cache_miss++;
auto mem_tracker =
std::make_unique<MemTracker>("InvertedIndexSearcherCacheWithRead");
SCOPED_RAW_TIMER(&stats->inverted_index_searcher_open_timer);
IndexSearcherPtr searcher;
@@ -256,7 +260,7 @@ Status InvertedIndexReader::handle_searcher_cache(
// to avoid open directory additionally for null_bitmap
// TODO: handle null bitmap procedure in new format.
InvertedIndexQueryCacheHandle null_bitmap_cache_handle;
- static_cast<void>(read_null_bitmap(&null_bitmap_cache_handle,
dir.get()));
+ static_cast<void>(read_null_bitmap(stats, &null_bitmap_cache_handle,
dir.get()));
RETURN_IF_ERROR(create_index_searcher(dir.release(), &searcher,
mem_tracker.get(), type()));
auto* cache_value = new InvertedIndexSearcherCache::CacheValue(
std::move(searcher), mem_tracker->consumption(), UnixMillis());
diff --git a/be/src/olap/rowset/segment_v2/inverted_index_reader.h
b/be/src/olap/rowset/segment_v2/inverted_index_reader.h
index a598ccc9ee7..2377a91845f 100644
--- a/be/src/olap/rowset/segment_v2/inverted_index_reader.h
+++ b/be/src/olap/rowset/segment_v2/inverted_index_reader.h
@@ -93,7 +93,8 @@ public:
const void* query_value, InvertedIndexQueryType
query_type,
uint32_t* count) = 0;
- Status read_null_bitmap(InvertedIndexQueryCacheHandle* cache_handle,
+ Status read_null_bitmap(OlapReaderStatistics* stats,
+ InvertedIndexQueryCacheHandle* cache_handle,
lucene::store::Directory* dir = nullptr);
virtual InvertedIndexReaderType type() = 0;
@@ -372,7 +373,7 @@ public:
Status read_null_bitmap(InvertedIndexQueryCacheHandle* cache_handle,
lucene::store::Directory* dir = nullptr) {
- return _reader->read_null_bitmap(cache_handle, dir);
+ return _reader->read_null_bitmap(_stats, cache_handle, dir);
}
[[nodiscard]] InvertedIndexReaderType get_inverted_index_reader_type()
const;
diff --git a/be/src/pipeline/exec/olap_scan_operator.cpp
b/be/src/pipeline/exec/olap_scan_operator.cpp
index bee550f1db5..95bfd398041 100644
--- a/be/src/pipeline/exec/olap_scan_operator.cpp
+++ b/be/src/pipeline/exec/olap_scan_operator.cpp
@@ -129,6 +129,8 @@ Status OlapScanLocalState::_init_profile() {
_inverted_index_query_cache_miss_counter =
ADD_COUNTER(_segment_profile, "InvertedIndexQueryCacheMiss",
TUnit::UNIT);
_inverted_index_query_timer = ADD_TIMER(_segment_profile,
"InvertedIndexQueryTime");
+ _inverted_index_query_null_bitmap_timer =
+ ADD_TIMER(_segment_profile, "InvertedIndexQueryNullBitmapTime");
_inverted_index_query_bitmap_copy_timer =
ADD_TIMER(_segment_profile, "InvertedIndexQueryBitmapCopyTime");
_inverted_index_query_bitmap_op_timer =
@@ -137,6 +139,10 @@ Status OlapScanLocalState::_init_profile() {
ADD_TIMER(_segment_profile, "InvertedIndexSearcherOpenTime");
_inverted_index_searcher_search_timer =
ADD_TIMER(_segment_profile, "InvertedIndexSearcherSearchTime");
+ _inverted_index_searcher_cache_hit_counter =
+ ADD_COUNTER(_segment_profile, "InvertedIndexSearcherCacheHit",
TUnit::UNIT);
+ _inverted_index_searcher_cache_miss_counter =
+ ADD_COUNTER(_segment_profile, "InvertedIndexSearcherCacheMiss",
TUnit::UNIT);
_output_index_result_column_timer = ADD_TIMER(_segment_profile,
"OutputIndexResultColumnTimer");
diff --git a/be/src/pipeline/exec/olap_scan_operator.h
b/be/src/pipeline/exec/olap_scan_operator.h
index ca98b171189..83f838dd0fc 100644
--- a/be/src/pipeline/exec/olap_scan_operator.h
+++ b/be/src/pipeline/exec/olap_scan_operator.h
@@ -174,6 +174,7 @@ private:
RuntimeProfile::Counter* _inverted_index_filter_counter = nullptr;
RuntimeProfile::Counter* _inverted_index_filter_timer = nullptr;
+ RuntimeProfile::Counter* _inverted_index_query_null_bitmap_timer = nullptr;
RuntimeProfile::Counter* _inverted_index_query_cache_hit_counter = nullptr;
RuntimeProfile::Counter* _inverted_index_query_cache_miss_counter =
nullptr;
RuntimeProfile::Counter* _inverted_index_query_timer = nullptr;
@@ -181,6 +182,8 @@ private:
RuntimeProfile::Counter* _inverted_index_query_bitmap_op_timer = nullptr;
RuntimeProfile::Counter* _inverted_index_searcher_open_timer = nullptr;
RuntimeProfile::Counter* _inverted_index_searcher_search_timer = nullptr;
+ RuntimeProfile::Counter* _inverted_index_searcher_cache_hit_counter =
nullptr;
+ RuntimeProfile::Counter* _inverted_index_searcher_cache_miss_counter =
nullptr;
RuntimeProfile::Counter* _output_index_result_column_timer = nullptr;
diff --git a/be/src/vec/exec/scan/new_olap_scanner.cpp
b/be/src/vec/exec/scan/new_olap_scanner.cpp
index ae785d92b16..0b06b8e7dd4 100644
--- a/be/src/vec/exec/scan/new_olap_scanner.cpp
+++ b/be/src/vec/exec/scan/new_olap_scanner.cpp
@@ -641,6 +641,8 @@ void NewOlapScanner::_collect_profile_before_close() {
COUNTER_UPDATE(Parent->_inverted_index_query_cache_miss_counter,
\
stats.inverted_index_query_cache_miss);
\
COUNTER_UPDATE(Parent->_inverted_index_query_timer,
stats.inverted_index_query_timer); \
+ COUNTER_UPDATE(Parent->_inverted_index_query_null_bitmap_timer,
\
+ stats.inverted_index_query_null_bitmap_timer);
\
COUNTER_UPDATE(Parent->_inverted_index_query_bitmap_copy_timer,
\
stats.inverted_index_query_bitmap_copy_timer);
\
COUNTER_UPDATE(Parent->_inverted_index_query_bitmap_op_timer,
\
@@ -649,6 +651,10 @@ void NewOlapScanner::_collect_profile_before_close() {
stats.inverted_index_searcher_open_timer);
\
COUNTER_UPDATE(Parent->_inverted_index_searcher_search_timer,
\
stats.inverted_index_searcher_search_timer);
\
+ COUNTER_UPDATE(Parent->_inverted_index_searcher_cache_hit_counter,
\
+ stats.inverted_index_searcher_cache_hit);
\
+ COUNTER_UPDATE(Parent->_inverted_index_searcher_cache_miss_counter,
\
+ stats.inverted_index_searcher_cache_miss);
\
if (config::enable_file_cache) {
\
io::FileCacheProfileReporter
cache_profile(Parent->_segment_profile.get()); \
cache_profile.update(&stats.file_cache_stats);
\
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
