This is an automated email from the ASF dual-hosted git repository.
jianliangqi pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.1 by this push:
new 14ece32b873 [Pick 2.1](inverted index) add inverted index reader
memory size into searcher cache (#36160)
14ece32b873 is described below
commit 14ece32b8730b917c2ff09e141d0d688bca370de
Author: airborne12 <[email protected]>
AuthorDate: Wed Jun 12 14:40:20 2024 +0800
[Pick 2.1](inverted index) add inverted index reader memory size into
searcher cache (#36160)
Pick from #35149
---
be/src/clucene | 2 +-
be/src/olap/rowset/segment_v2/inverted_index_reader.cpp | 2 ++
be/src/olap/rowset/segment_v2/inverted_index_searcher.cpp | 1 +
be/src/olap/rowset/segment_v2/inverted_index_searcher.h | 4 ++++
4 files changed, 8 insertions(+), 1 deletion(-)
diff --git a/be/src/clucene b/be/src/clucene
index d3de160871d..25324632bab 160000
--- a/be/src/clucene
+++ b/be/src/clucene
@@ -1 +1 @@
-Subproject commit d3de160871dc1e2e293e5702e5b870e220ed42e4
+Subproject commit 25324632babc0e5da28048ebbe9adcbdfc73c281
diff --git a/be/src/olap/rowset/segment_v2/inverted_index_reader.cpp
b/be/src/olap/rowset/segment_v2/inverted_index_reader.cpp
index 678c63e078c..f453e7cca31 100644
--- a/be/src/olap/rowset/segment_v2/inverted_index_reader.cpp
+++ b/be/src/olap/rowset/segment_v2/inverted_index_reader.cpp
@@ -239,6 +239,8 @@ Status
InvertedIndexReader::create_index_searcher(lucene::store::Directory* dir,
if (std::string(dir->getObjectName()) == "DorisCompoundReader") {
static_cast<DorisCompoundReader*>(dir)->getDorisIndexInput()->setIdxFileCache(false);
}
+ // NOTE: before mem_tracker hook becomes active, we caculate reader memory
size by hand.
+ mem_tracker->consume(index_searcher_builder->get_reader_size());
return Status::OK();
};
diff --git a/be/src/olap/rowset/segment_v2/inverted_index_searcher.cpp
b/be/src/olap/rowset/segment_v2/inverted_index_searcher.cpp
index aff9c6ae0e0..90d455e5754 100644
--- a/be/src/olap/rowset/segment_v2/inverted_index_searcher.cpp
+++ b/be/src/olap/rowset/segment_v2/inverted_index_searcher.cpp
@@ -44,6 +44,7 @@ Status
FulltextIndexSearcherBuilder::build(lucene::store::Directory* directory,
return Status::Error<ErrorCode::INVERTED_INDEX_CLUCENE_ERROR>(
"FulltextIndexSearcherBuilder build index_searcher error.");
}
+ reader_size = reader->getTermInfosRAMUsed();
// NOTE: need to cl_refcount-- here, so that directory will be deleted when
// index_searcher is destroyed
_CLDECDELETE(directory)
diff --git a/be/src/olap/rowset/segment_v2/inverted_index_searcher.h
b/be/src/olap/rowset/segment_v2/inverted_index_searcher.h
index b0466ab2df2..46297fbf6e2 100644
--- a/be/src/olap/rowset/segment_v2/inverted_index_searcher.h
+++ b/be/src/olap/rowset/segment_v2/inverted_index_searcher.h
@@ -61,6 +61,10 @@ public:
virtual Result<IndexSearcherPtr>
get_index_searcher(lucene::store::Directory* directory);
static Result<std::unique_ptr<IndexSearcherBuilder>>
create_index_searcher_builder(
InvertedIndexReaderType reader_type);
+ int64_t get_reader_size() const { return reader_size; }
+
+protected:
+ int64_t reader_size = 0;
};
class FulltextIndexSearcherBuilder : public IndexSearcherBuilder {
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
