github-actions[bot] commented on code in PR #69854:
URL: https://github.com/apache/doris/pull/69854#discussion_r4493130261
##########
be/src/storage/index/inverted/reader/collection_statistics.cpp:
##########
@@ -98,6 +100,62 @@ Result<std::shared_ptr<lucene::search::IndexSearcher>>
open_optional_clucene_sta
return std::make_shared<lucene::search::IndexSearcher>(reader, true);
}
+Result<std::unique_ptr<::doris::snii::reader::LogicalIndexReader>>
open_snii_stats_reader(
+ const segment_v2::IndexFileReader& file_reader, const TabletIndex*
index,
+ const io::IOContext* io_ctx) {
+ auto logical_reader = file_reader.open_snii_index(index, io_ctx);
+ if (!logical_reader.has_value()) {
+ auto status = std::move(logical_reader.error());
+ if (status.code() == ErrorCode::INVERTED_INDEX_SNII_NOT_FOUND ||
+ status.code() == ErrorCode::INVERTED_INDEX_FILE_NOT_FOUND ||
+ status.code() == ErrorCode::INVERTED_INDEX_BYPASS) {
+ return
ResultError(Status::Error<ErrorCode::INVERTED_INDEX_NOT_SUPPORTED>(
+ "SNII scoring requires every logical index in the
collection: {}",
+ status.msg()));
+ }
+ return ResultError(std::move(status));
+ }
+ return std::move(logical_reader.value());
+}
+
+// A segment's SNII logical index as the statistics read it, and the norm
length sums kept with it.
+// Outside tests the reader is the one the query path caches in the searcher
cache, so the sums
+// live and are evicted with it, and a query whose segments are cached opens
no index.
+struct SniiStatsReader {
+#ifdef BE_TEST
+ std::unique_ptr<::doris::snii::reader::LogicalIndexReader> owned_reader;
+ segment_v2::NormLengthSums owned_norm_length_sums;
+#else
+ InvertedIndexCacheHandle cache_handle;
+#endif
+ const ::doris::snii::reader::LogicalIndexReader* reader = nullptr;
+ segment_v2::NormLengthSums* norm_length_sums = nullptr;
+};
+
+Status open_snii_stats_reader(const
std::shared_ptr<segment_v2::IndexFileReader>& file_reader,
+ const TabletIndex* index, const io::IOContext*
io_ctx,
+ SniiStatsReader* out) {
+#ifdef BE_TEST
+ out->owned_reader = DORIS_TRY(open_snii_stats_reader(*file_reader, index,
io_ctx));
+ out->reader = out->owned_reader.get();
+ out->norm_length_sums = &out->owned_norm_length_sums;
+#else
+ InvertedIndexSearcherCache::CacheKey
cache_key(file_reader->get_index_file_cache_key(index));
+ if (!InvertedIndexSearcherCache::instance()->lookup(cache_key,
&out->cache_handle)) {
+ auto opened = DORIS_TRY(open_snii_stats_reader(*file_reader, index,
io_ctx));
+ const size_t reader_size = std::max<size_t>(opened->memory_usage(), 1);
Review Comment:
[P2] Enable hot-term caching before publishing the SNII reader
On a cold searcher cache, statistics populate this entry during
`OlapScanner::_prepare_impl`, before the normal query reader opens. Unlike
`SniiIndexReader::_get_logical_reader`, this path never calls
`enable_term_lookup_cache()`, and later searcher-cache hits return without
enabling it. For a nonresident dictionary, `term_lookups_` therefore stays null
for the reader's cached lifetime, so repeated hot-term queries keep reading and
decoding their DICT blocks, including following non-scoring queries. Enable the
lookup cache before `memory_usage()` and insertion here too, so its full
reserved charge is included.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
