This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.1 by this push:
new 9bc547578cf [opt](scan) Release instances of Segment to avoid
consuming a large amount of memory in ParallelScannerBuilder (#44189) (#44608)
9bc547578cf is described below
commit 9bc547578cf2fa8793b6d9954f7f770ed496989b
Author: Jerry Hu <[email protected]>
AuthorDate: Tue Nov 26 19:25:58 2024 +0800
[opt](scan) Release instances of Segment to avoid consuming a large amount
of memory in ParallelScannerBuilder (#44189) (#44608)
---
be/src/olap/parallel_scanner_builder.cpp | 20 +++++++++++---------
be/src/olap/parallel_scanner_builder.h | 2 +-
2 files changed, 12 insertions(+), 10 deletions(-)
diff --git a/be/src/olap/parallel_scanner_builder.cpp
b/be/src/olap/parallel_scanner_builder.cpp
index f960519d29a..c0a1b75ec75 100644
--- a/be/src/olap/parallel_scanner_builder.cpp
+++ b/be/src/olap/parallel_scanner_builder.cpp
@@ -20,6 +20,7 @@
#include <shared_mutex>
#include "olap/rowset/beta_rowset.h"
+#include "olap/segment_loader.h"
#include "pipeline/exec/olap_scan_operator.h"
#include "vec/exec/scan/new_olap_scanner.h"
@@ -56,21 +57,18 @@ Status
ParallelScannerBuilder<ParentType>::_build_scanners_by_rowid(
auto rowset = reader->rowset();
const auto rowset_id = rowset->rowset_id();
- DCHECK(_segment_cache_handles.contains(rowset_id));
- auto& segment_cache_handle = _segment_cache_handles[rowset_id];
+ const auto& segments_rows = _all_segments_rows[rowset_id];
if (rowset->num_rows() == 0) {
continue;
}
- const auto& segments = segment_cache_handle.get_segments();
int segment_start = 0;
auto split = RowSetSplits(reader->clone());
- for (size_t i = 0; i != segments.size(); ++i) {
- const auto& segment = segments[i];
+ for (size_t i = 0; i != segments_rows.size(); ++i) {
+ const size_t rows_of_segment = segments_rows[i];
RowRanges row_ranges;
- const size_t rows_of_segment = segment->num_rows();
int64_t offset_in_segment = 0;
// try to split large segments into RowRanges
@@ -118,7 +116,7 @@ Status
ParallelScannerBuilder<ParentType>::_build_scanners_by_rowid(
// The non-empty `row_ranges` means there are some rows left
in this segment not added into `split`.
if (!row_ranges.is_empty()) {
DCHECK_GT(rows_collected, 0);
- DCHECK_EQ(row_ranges.to(), segment->num_rows());
+ DCHECK_EQ(row_ranges.to(), rows_of_segment);
split.segment_row_ranges.emplace_back(std::move(row_ranges));
}
}
@@ -126,7 +124,7 @@ Status
ParallelScannerBuilder<ParentType>::_build_scanners_by_rowid(
DCHECK_LE(rows_collected, _rows_per_scanner);
if (rows_collected > 0) {
split.segment_offsets.first = segment_start;
- split.segment_offsets.second = segments.size();
+ split.segment_offsets.second = segments_rows.size();
DCHECK_GT(split.segment_offsets.second,
split.segment_offsets.first);
DCHECK_EQ(split.segment_row_ranges.size(),
split.segment_offsets.second -
split.segment_offsets.first);
@@ -176,10 +174,14 @@ Status ParallelScannerBuilder<ParentType>::_load() {
auto rowset = rs_split.rs_reader->rowset();
RETURN_IF_ERROR(rowset->load());
const auto rowset_id = rowset->rowset_id();
- auto& segment_cache_handle = _segment_cache_handles[rowset_id];
+ SegmentCacheHandle segment_cache_handle;
RETURN_IF_ERROR(SegmentLoader::instance()->load_segments(
std::dynamic_pointer_cast<BetaRowset>(rowset),
&segment_cache_handle, true));
+
+ for (const auto& segment : segment_cache_handle.get_segments()) {
+
_all_segments_rows[rowset_id].emplace_back(segment->num_rows());
+ }
_total_rows += rowset->num_rows();
}
}
diff --git a/be/src/olap/parallel_scanner_builder.h
b/be/src/olap/parallel_scanner_builder.h
index 62ef4302571..b68da8ac7a5 100644
--- a/be/src/olap/parallel_scanner_builder.h
+++ b/be/src/olap/parallel_scanner_builder.h
@@ -85,7 +85,7 @@ private:
size_t _rows_per_scanner {_min_rows_per_scanner};
- std::map<RowsetId, SegmentCacheHandle> _segment_cache_handles;
+ std::map<RowsetId, std::vector<size_t>> _all_segments_rows;
std::shared_ptr<RuntimeProfile> _scanner_profile;
RuntimeState* _state;
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]