This is an automated email from the ASF dual-hosted git repository.
airborne pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 00b1e7762f8 [opt](inverted index) enhance inverted index query
profiling for better performance monitoring (#54154)
00b1e7762f8 is described below
commit 00b1e7762f89caef98b5c79b97fcdb0f28ca0b7a
Author: zzzxl <yangs...@selectdb.com>
AuthorDate: Tue Aug 12 23:25:07 2025 +0800
[opt](inverted index) enhance inverted index query profiling for better
performance monitoring (#54154)
---
be/src/olap/olap_common.h | 2 ++
.../inverted_index/analyzer/analyzer.cpp | 1 +
.../segment_v2/inverted_index/analyzer/analyzer.h | 1 +
.../inverted_index/query/phrase_query.cpp | 9 ++++---
.../segment_v2/inverted_index/query/phrase_query.h | 3 ++-
.../rowset/segment_v2/inverted_index_reader.cpp | 30 +++++++++++++++-------
be/src/pipeline/exec/olap_scan_operator.cpp | 2 ++
be/src/pipeline/exec/olap_scan_operator.h | 2 ++
be/src/vec/exec/scan/olap_scanner.cpp | 3 +++
.../inverted_index/query/phrase_query_test.cpp | 12 ++++++---
10 files changed, 48 insertions(+), 17 deletions(-)
diff --git a/be/src/olap/olap_common.h b/be/src/olap/olap_common.h
index 1d809ad1c13..3f4b7afbd00 100644
--- a/be/src/olap/olap_common.h
+++ b/be/src/olap/olap_common.h
@@ -378,6 +378,8 @@ struct OlapReaderStatistics {
int64_t inverted_index_searcher_cache_hit = 0;
int64_t inverted_index_searcher_cache_miss = 0;
int64_t inverted_index_downgrade_count = 0;
+ int64_t inverted_index_analyzer_timer = 0;
+ int64_t inverted_index_lookup_timer = 0;
InvertedIndexStatistics inverted_index_stats;
int64_t output_index_result_column_timer = 0;
diff --git a/be/src/olap/rowset/segment_v2/inverted_index/analyzer/analyzer.cpp
b/be/src/olap/rowset/segment_v2/inverted_index/analyzer/analyzer.cpp
index 388fe908e1d..818c2f05f13 100644
--- a/be/src/olap/rowset/segment_v2/inverted_index/analyzer/analyzer.cpp
+++ b/be/src/olap/rowset/segment_v2/inverted_index/analyzer/analyzer.cpp
@@ -34,6 +34,7 @@
#include
"olap/rowset/segment_v2/inverted_index/char_filter/char_filter_factory.h"
#include "runtime/exec_env.h"
#include "runtime/index_policy/index_policy_mgr.h"
+#include "util/runtime_profile.h"
namespace doris::segment_v2::inverted_index {
#include "common/compile_check_begin.h"
diff --git a/be/src/olap/rowset/segment_v2/inverted_index/analyzer/analyzer.h
b/be/src/olap/rowset/segment_v2/inverted_index/analyzer/analyzer.h
index 1a52a772a52..682c7cd9b52 100644
--- a/be/src/olap/rowset/segment_v2/inverted_index/analyzer/analyzer.h
+++ b/be/src/olap/rowset/segment_v2/inverted_index/analyzer/analyzer.h
@@ -21,6 +21,7 @@
#include <vector>
#include "olap/inverted_index_parser.h"
+#include "olap/olap_common.h"
#include "olap/rowset/segment_v2/inverted_index/query/query.h"
#include "olap/rowset/segment_v2/inverted_index_query_type.h"
diff --git
a/be/src/olap/rowset/segment_v2/inverted_index/query/phrase_query.cpp
b/be/src/olap/rowset/segment_v2/inverted_index/query/phrase_query.cpp
index 031d066de61..9494a3571d2 100644
--- a/be/src/olap/rowset/segment_v2/inverted_index/query/phrase_query.cpp
+++ b/be/src/olap/rowset/segment_v2/inverted_index/query/phrase_query.cpp
@@ -272,12 +272,15 @@ void PhraseQuery::parser_slop(std::string& query,
InvertedIndexQueryInfo& query_
}
}
-void PhraseQuery::parser_info(std::string& query,
+void PhraseQuery::parser_info(OlapReaderStatistics* stats, std::string& query,
const std::map<std::string, std::string>&
properties,
InvertedIndexQueryInfo& query_info) {
parser_slop(query, query_info);
- query_info.term_infos =
- inverted_index::InvertedIndexAnalyzer::get_analyse_result(query,
properties);
+ {
+ SCOPED_RAW_TIMER(&stats->inverted_index_analyzer_timer);
+ query_info.term_infos =
+
inverted_index::InvertedIndexAnalyzer::get_analyse_result(query, properties);
+ }
}
} // namespace doris::segment_v2
\ No newline at end of file
diff --git a/be/src/olap/rowset/segment_v2/inverted_index/query/phrase_query.h
b/be/src/olap/rowset/segment_v2/inverted_index/query/phrase_query.h
index d7f99e30413..2251fce7740 100644
--- a/be/src/olap/rowset/segment_v2/inverted_index/query/phrase_query.h
+++ b/be/src/olap/rowset/segment_v2/inverted_index/query/phrase_query.h
@@ -17,6 +17,7 @@
#pragma once
+#include "olap/olap_common.h"
#include
"olap/rowset/segment_v2/inverted_index/query/phrase_query/exact_phrase_matcher.h"
#include
"olap/rowset/segment_v2/inverted_index/query/phrase_query/ordered_sloppy_phrase_matcher.h"
#include
"olap/rowset/segment_v2/inverted_index/query/phrase_query/sloppy_phrase_matcher.h"
@@ -60,7 +61,7 @@ private:
public:
static void parser_slop(std::string& query, InvertedIndexQueryInfo&
query_info);
- static void parser_info(std::string& query,
+ static void parser_info(OlapReaderStatistics* stats, std::string& query,
const std::map<std::string, std::string>&
properties,
InvertedIndexQueryInfo& query_info);
diff --git a/be/src/olap/rowset/segment_v2/inverted_index_reader.cpp
b/be/src/olap/rowset/segment_v2/inverted_index_reader.cpp
index e899ca859bb..dbd28190965 100644
--- a/be/src/olap/rowset/segment_v2/inverted_index_reader.cpp
+++ b/be/src/olap/rowset/segment_v2/inverted_index_reader.cpp
@@ -131,11 +131,14 @@ bool InvertedIndexReader::handle_query_cache(const
IndexQueryContextPtr& context
InvertedIndexQueryCacheHandle*
cache_handler,
std::shared_ptr<roaring::Roaring>& bit_map) {
const auto& query_options = context->runtime_state->query_options();
- if (!query_options.enable_inverted_index_query_cache) {
- return false;
+
+ bool cache_hit = false;
+ if (query_options.enable_inverted_index_query_cache) {
+ SCOPED_RAW_TIMER(&context->stats->inverted_index_lookup_timer);
+ cache_hit = cache->lookup(cache_key, cache_handler);
}
- if (cache->lookup(cache_key, cache_handler)) {
+ if (cache_hit) {
DBUG_EXECUTE_IF("InvertedIndexReader.handle_query_cache_hit", {
return Status::Error<ErrorCode::INTERNAL_ERROR>("handle query
cache hit");
});
@@ -158,24 +161,31 @@ Status InvertedIndexReader::handle_searcher_cache(
auto index_file_key =
_index_file_reader->get_index_file_cache_key(&_index_meta);
InvertedIndexSearcherCache::CacheKey searcher_cache_key(index_file_key);
const auto& query_options = context->runtime_state->query_options();
- if (query_options.enable_inverted_index_searcher_cache &&
- InvertedIndexSearcherCache::instance()->lookup(searcher_cache_key,
-
inverted_index_cache_handle)) {
+
+ bool cache_hit = false;
+ if (query_options.enable_inverted_index_searcher_cache) {
+ SCOPED_RAW_TIMER(&context->stats->inverted_index_lookup_timer);
+ cache_hit =
InvertedIndexSearcherCache::instance()->lookup(searcher_cache_key,
+
inverted_index_cache_handle);
+ }
+
+ if (cache_hit) {
DBUG_EXECUTE_IF("InvertedIndexReader.handle_searcher_cache_hit", {
return Status::Error<ErrorCode::INTERNAL_ERROR>("handle searcher
cache hit");
});
context->stats->inverted_index_searcher_cache_hit++;
return Status::OK();
} else {
+ SCOPED_RAW_TIMER(&context->stats->inverted_index_searcher_open_timer);
+
DBUG_EXECUTE_IF("InvertedIndexReader.handle_searcher_cache_miss", {
return Status::Error<ErrorCode::INTERNAL_ERROR>("handle searcher
cache miss");
});
// searcher cache miss
context->stats->inverted_index_searcher_cache_miss++;
auto mem_tracker =
std::make_unique<MemTracker>("InvertedIndexSearcherCacheWithRead");
- SCOPED_RAW_TIMER(&context->stats->inverted_index_searcher_open_timer);
- IndexSearcherPtr searcher;
+ IndexSearcherPtr searcher;
auto st =
_index_file_reader->init(config::inverted_index_read_buffer_size,
context->io_ctx);
if (!st.ok()) {
@@ -298,8 +308,10 @@ Status FullTextIndexReader::query(const
IndexQueryContextPtr& context,
if (query_type == InvertedIndexQueryType::MATCH_REGEXP_QUERY) {
query_info.term_infos.emplace_back(search_str, 0);
} else if (query_type == InvertedIndexQueryType::MATCH_PHRASE_QUERY) {
- PhraseQuery::parser_info(search_str, _index_meta.properties(),
query_info);
+ PhraseQuery::parser_info(context->stats, search_str,
_index_meta.properties(),
+ query_info);
} else {
+ SCOPED_RAW_TIMER(&context->stats->inverted_index_analyzer_timer);
query_info.term_infos =
inverted_index::InvertedIndexAnalyzer::get_analyse_result(
search_str, _index_meta.properties());
}
diff --git a/be/src/pipeline/exec/olap_scan_operator.cpp
b/be/src/pipeline/exec/olap_scan_operator.cpp
index 2a647caed02..c432104e0fe 100644
--- a/be/src/pipeline/exec/olap_scan_operator.cpp
+++ b/be/src/pipeline/exec/olap_scan_operator.cpp
@@ -206,6 +206,8 @@ Status OlapScanLocalState::_init_profile() {
ADD_COUNTER(_segment_profile, "InvertedIndexSearcherCacheMiss",
TUnit::UNIT);
_inverted_index_downgrade_count_counter =
ADD_COUNTER(_segment_profile, "InvertedIndexDowngradeCount",
TUnit::UNIT);
+ _inverted_index_analyzer_timer = ADD_TIMER(_segment_profile,
"InvertedIndexAnalyzerTime");
+ _inverted_index_lookup_timer = ADD_TIMER(_segment_profile,
"InvertedIndexLookupTimer");
_output_index_result_column_timer = ADD_TIMER(_segment_profile,
"OutputIndexResultColumnTime");
_filtered_segment_counter = ADD_COUNTER(_segment_profile,
"NumSegmentFiltered", TUnit::UNIT);
diff --git a/be/src/pipeline/exec/olap_scan_operator.h
b/be/src/pipeline/exec/olap_scan_operator.h
index d77388cf734..42251be7545 100644
--- a/be/src/pipeline/exec/olap_scan_operator.h
+++ b/be/src/pipeline/exec/olap_scan_operator.h
@@ -206,6 +206,8 @@ private:
RuntimeProfile::Counter* _inverted_index_searcher_cache_hit_counter =
nullptr;
RuntimeProfile::Counter* _inverted_index_searcher_cache_miss_counter =
nullptr;
RuntimeProfile::Counter* _inverted_index_downgrade_count_counter = nullptr;
+ RuntimeProfile::Counter* _inverted_index_analyzer_timer = nullptr;
+ RuntimeProfile::Counter* _inverted_index_lookup_timer = nullptr;
RuntimeProfile::Counter* _output_index_result_column_timer = nullptr;
diff --git a/be/src/vec/exec/scan/olap_scanner.cpp
b/be/src/vec/exec/scan/olap_scanner.cpp
index 56e94921a8a..9dd0f5c591f 100644
--- a/be/src/vec/exec/scan/olap_scanner.cpp
+++ b/be/src/vec/exec/scan/olap_scanner.cpp
@@ -722,6 +722,9 @@ void OlapScanner::_collect_profile_before_close() {
stats.inverted_index_searcher_cache_miss);
COUNTER_UPDATE(local_state->_inverted_index_downgrade_count_counter,
stats.inverted_index_downgrade_count);
+ COUNTER_UPDATE(local_state->_inverted_index_analyzer_timer,
+ stats.inverted_index_analyzer_timer);
+ COUNTER_UPDATE(local_state->_inverted_index_lookup_timer,
stats.inverted_index_lookup_timer);
InvertedIndexProfileReporter inverted_index_profile;
inverted_index_profile.update(local_state->_index_filter_profile.get(),
diff --git
a/be/test/olap/rowset/segment_v2/inverted_index/query/phrase_query_test.cpp
b/be/test/olap/rowset/segment_v2/inverted_index/query/phrase_query_test.cpp
index 2ad9f744a16..197d0a69465 100644
--- a/be/test/olap/rowset/segment_v2/inverted_index/query/phrase_query_test.cpp
+++ b/be/test/olap/rowset/segment_v2/inverted_index/query/phrase_query_test.cpp
@@ -556,8 +556,10 @@ TEST_F(PhraseQueryTest, test_parser_info) {
properties.insert({"support_phrase", "true"});
properties.insert({"lower_case", "true"});
- auto parser_info = [&properties](std::string& search_str,
InvertedIndexQueryInfo& query_info) {
- PhraseQuery::parser_info(search_str, properties, query_info);
+ OlapReaderStatistics stats;
+ auto parser_info = [&properties, &stats](std::string& search_str,
+ InvertedIndexQueryInfo&
query_info) {
+ PhraseQuery::parser_info(&stats, search_str, properties, query_info);
};
auto parser = [&parser_info](std::string search_str, std::string res1,
size_t res2,
@@ -591,8 +593,10 @@ TEST_F(PhraseQueryTest, test_parser_info1) {
properties.insert({"support_phrase", "true"});
properties.insert({"lower_case", "true"});
- auto parser_info = [&properties](std::string& search_str,
InvertedIndexQueryInfo& query_info) {
- PhraseQuery::parser_info(search_str, properties, query_info);
+ OlapReaderStatistics stats;
+ auto parser_info = [&properties, &stats](std::string& search_str,
+ InvertedIndexQueryInfo&
query_info) {
+ PhraseQuery::parser_info(&stats, search_str, properties, query_info);
};
{
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
