This is an automated email from the ASF dual-hosted git repository.
lide pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.1 by this push:
new 32b13b285c5 [fix](index compaction) fix fd leak and mem leak while
index compaction (#41915) (#42171)
32b13b285c5 is described below
commit 32b13b285c531c4dc90040717bcdcc61024a3129
Author: camby <[email protected]>
AuthorDate: Thu Oct 24 10:40:10 2024 +0800
[fix](index compaction) fix fd leak and mem leak while index compaction
(#41915) (#42171)
pick #41915 to branch-2.1
---
be/src/olap/compaction.cpp | 10 ++++------
.../olap/rowset/segment_v2/inverted_index_compaction.cpp | 15 +++++++--------
be/src/olap/rowset/segment_v2/inverted_index_compaction.h | 4 +++-
3 files changed, 14 insertions(+), 15 deletions(-)
diff --git a/be/src/olap/compaction.cpp b/be/src/olap/compaction.cpp
index 1c5b52dca0e..782063331df 100644
--- a/be/src/olap/compaction.cpp
+++ b/be/src/olap/compaction.cpp
@@ -652,13 +652,12 @@ Status Compaction::do_compaction_impl(int64_t permits) {
}
std::vector<lucene::store::Directory*>
dest_index_dirs(dest_segment_num);
- std::vector<lucene::store::Directory*>
src_index_dirs(src_segment_num);
try {
+ std::vector<std::unique_ptr<DorisCompoundReader>>
src_idx_dirs(src_segment_num);
for (int src_segment_id = 0; src_segment_id <
src_segment_num;
src_segment_id++) {
- auto src_dir = DORIS_TRY(
+ src_idx_dirs[src_segment_id] = DORIS_TRY(
inverted_index_file_readers[src_segment_id]->open(index_meta));
- src_index_dirs[src_segment_id] = src_dir.release();
}
for (int dest_segment_id = 0; dest_segment_id <
dest_segment_num;
dest_segment_id++) {
@@ -666,9 +665,8 @@ Status Compaction::do_compaction_impl(int64_t permits) {
inverted_index_file_writers[dest_segment_id]->open(index_meta));
dest_index_dirs[dest_segment_id] = dest_dir;
}
- auto st =
- compact_column(index_meta->index_id(),
src_index_dirs, dest_index_dirs,
- fs, index_tmp_path, trans_vec,
dest_segment_num_rows);
+ auto st = compact_column(index_meta->index_id(),
src_idx_dirs, dest_index_dirs,
+ fs, index_tmp_path, trans_vec,
dest_segment_num_rows);
if (!st.ok()) {
error_handler(index_meta->index_id(), column_uniq_id);
status =
Status::Error<INVERTED_INDEX_COMPACTION_ERROR>(st.msg());
diff --git a/be/src/olap/rowset/segment_v2/inverted_index_compaction.cpp
b/be/src/olap/rowset/segment_v2/inverted_index_compaction.cpp
index 51aeab0cb45..fc534f3c60b 100644
--- a/be/src/olap/rowset/segment_v2/inverted_index_compaction.cpp
+++ b/be/src/olap/rowset/segment_v2/inverted_index_compaction.cpp
@@ -23,7 +23,8 @@
#include "util/debug_points.h"
namespace doris::segment_v2 {
-Status compact_column(int64_t index_id,
std::vector<lucene::store::Directory*>& src_index_dirs,
+Status compact_column(int64_t index_id,
+ std::vector<std::unique_ptr<DorisCompoundReader>>&
src_index_dirs,
std::vector<lucene::store::Directory*>& dest_index_dirs,
const io::FileSystemSPtr& fs, std::string tmp_path,
std::vector<std::vector<std::pair<uint32_t, uint32_t>>>
trans_vec,
@@ -47,7 +48,11 @@ Status compact_column(int64_t index_id,
std::vector<lucene::store::Directory*>&
true /*
closeDirOnShutdown */);
DCHECK_EQ(src_index_dirs.size(), trans_vec.size());
- index_writer->indexCompaction(src_index_dirs, dest_index_dirs, trans_vec,
+ std::vector<lucene::store::Directory*>
tmp_src_index_dirs(src_index_dirs.size());
+ for (size_t i = 0; i < tmp_src_index_dirs.size(); ++i) {
+ tmp_src_index_dirs[i] = src_index_dirs[i].get();
+ }
+ index_writer->indexCompaction(tmp_src_index_dirs, dest_index_dirs,
trans_vec,
dest_segment_num_rows);
index_writer->close();
@@ -56,12 +61,6 @@ Status compact_column(int64_t index_id,
std::vector<lucene::store::Directory*>&
// when index_writer is destroyed, if closeDir is set, dir will be close
// _CLDECDELETE(dir) will try to ref_cnt--, when it decreases to 1, dir
will be destroyed.
_CLDECDELETE(dir)
- for (auto* d : src_index_dirs) {
- if (d != nullptr) {
- d->close();
- _CLDELETE(d);
- }
- }
for (auto* d : dest_index_dirs) {
if (d != nullptr) {
// NOTE: DO NOT close dest dir here, because it will be closed
when dest index writer finalize.
diff --git a/be/src/olap/rowset/segment_v2/inverted_index_compaction.h
b/be/src/olap/rowset/segment_v2/inverted_index_compaction.h
index 3dadd7f4766..722ff16ab87 100644
--- a/be/src/olap/rowset/segment_v2/inverted_index_compaction.h
+++ b/be/src/olap/rowset/segment_v2/inverted_index_compaction.h
@@ -22,6 +22,7 @@
#include <string>
#include <vector>
+#include "inverted_index_compound_reader.h"
#include "io/fs/file_system.h"
namespace doris {
@@ -30,7 +31,8 @@ namespace segment_v2 {
class InvertedIndexFileWriter;
class InvertedIndexFileReader;
-Status compact_column(int64_t index_id,
std::vector<lucene::store::Directory*>& src_index_dirs,
+Status compact_column(int64_t index_id,
+ std::vector<std::unique_ptr<DorisCompoundReader>>&
src_index_dirs,
std::vector<lucene::store::Directory*>& dest_index_dirs,
const io::FileSystemSPtr& fs, std::string tmp_path,
std::vector<std::vector<std::pair<uint32_t, uint32_t>>>
trans_vec,
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
