This is an automated email from the ASF dual-hosted git repository.
airborne pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.1 by this push:
new 33540ec87bc [Pick 2.1](inverted index) fix inverted index compound
reader memory leak (#36387)
33540ec87bc is described below
commit 33540ec87bca2276e7f8923b75d4028ed991b8ac
Author: airborne12 <[email protected]>
AuthorDate: Tue Jun 18 16:13:21 2024 +0800
[Pick 2.1](inverted index) fix inverted index compound reader memory leak
(#36387)
## Proposed changes
Issue Number: close #xxx
Pick from #36146 #36420
---
.../segment_v2/inverted_index_compound_reader.cpp | 17 ++++++++++++-----
.../rowset/segment_v2/inverted_index_compound_reader.h | 12 ++++--------
.../rowset/segment_v2/inverted_index_file_reader.cpp | 3 +--
3 files changed, 17 insertions(+), 15 deletions(-)
diff --git a/be/src/olap/rowset/segment_v2/inverted_index_compound_reader.cpp
b/be/src/olap/rowset/segment_v2/inverted_index_compound_reader.cpp
index 540053fc9b0..5e6d8747a2d 100644
--- a/be/src/olap/rowset/segment_v2/inverted_index_compound_reader.cpp
+++ b/be/src/olap/rowset/segment_v2/inverted_index_compound_reader.cpp
@@ -116,7 +116,6 @@
DorisCompoundReader::DorisCompoundReader(lucene::store::Directory* d, const char
dir(d),
ram_dir(new lucene::store::RAMDirectory()),
file_name(name),
- stream(nullptr),
entries(_CLNEW EntriesType(true, true)) {
bool success = false;
try {
@@ -197,9 +196,14 @@ void DorisCompoundReader::copyFile(const char* file,
int64_t file_length, uint8_
}
DorisCompoundReader::~DorisCompoundReader() {
- if (_own_index_input) {
- _CLDELETE(entries)
+ if (!_closed) {
+ try {
+ close();
+ } catch (CLuceneError& err) {
+ LOG(ERROR) << "DorisCompoundReader finalize error:" << err.what();
+ }
}
+ _CLDELETE(entries)
}
const char* DorisCompoundReader::getClassName() {
@@ -285,11 +289,13 @@ bool DorisCompoundReader::openInput(const char* name,
lucene::store::IndexInput*
void DorisCompoundReader::close() {
std::lock_guard<std::mutex> wlock(_this_lock);
- if (_own_index_input && stream != nullptr) {
- entries->clear();
+ if (stream != nullptr) {
stream->close();
_CLDELETE(stream)
}
+ if (entries != nullptr) {
+ entries->clear();
+ }
if (ram_dir) {
ram_dir->close();
_CLDELETE(ram_dir)
@@ -298,6 +304,7 @@ void DorisCompoundReader::close() {
dir->close();
_CLDECDELETE(dir)
}
+ _closed = true;
}
bool DorisCompoundReader::doDeleteFile(const char* /*name*/) {
diff --git a/be/src/olap/rowset/segment_v2/inverted_index_compound_reader.h
b/be/src/olap/rowset/segment_v2/inverted_index_compound_reader.h
index 4786b8f35e3..1ca2d6ad371 100644
--- a/be/src/olap/rowset/segment_v2/inverted_index_compound_reader.h
+++ b/be/src/olap/rowset/segment_v2/inverted_index_compound_reader.h
@@ -38,11 +38,9 @@
class CLuceneError;
-namespace lucene {
-namespace store {
+namespace lucene::store {
class RAMDirectory;
-} // namespace store
-} // namespace lucene
+} // namespace lucene::store
namespace doris {
class TabletIndex;
@@ -76,7 +74,7 @@ private:
CL_NS(store)::IndexInput* stream = nullptr;
EntriesType* entries = nullptr;
std::mutex _this_lock;
- bool _own_index_input = true;
+ bool _closed = false;
protected:
/** Removes an existing file in the directory-> */
@@ -85,12 +83,10 @@ protected:
public:
explicit DorisCompoundReader(
CL_NS(store)::IndexInput* stream, EntriesType* entries_clone,
- bool own_index_input = false,
int32_t _readBufferSize =
CL_NS(store)::BufferedIndexInput::BUFFER_SIZE)
: readBufferSize(_readBufferSize),
stream(stream),
- entries(_CLNEW EntriesType(true, true)),
- _own_index_input(own_index_input) {
+ entries(_CLNEW EntriesType(true, true)) {
for (auto& e : *entries_clone) {
auto* origin_entry = e.second;
auto* entry = _CLNEW ReaderFileEntry();
diff --git a/be/src/olap/rowset/segment_v2/inverted_index_file_reader.cpp
b/be/src/olap/rowset/segment_v2/inverted_index_file_reader.cpp
index c21ece4b86c..9a78c327441 100644
--- a/be/src/olap/rowset/segment_v2/inverted_index_file_reader.cpp
+++ b/be/src/olap/rowset/segment_v2/inverted_index_file_reader.cpp
@@ -179,9 +179,8 @@ Result<std::unique_ptr<DorisCompoundReader>>
InvertedIndexFileReader::_open(
(_index_file_dir / _index_file_name).native(),
errMsg.str()));
}
// Need to clone resource here, because index searcher cache need it.
- bool own_index_input = true;
compound_reader = std::make_unique<DorisCompoundReader>(
- _stream->clone(), index_it->second.get(), own_index_input,
_read_buffer_size);
+ _stream->clone(), index_it->second.get(), _read_buffer_size);
}
return compound_reader;
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
