This is an automated email from the ASF dual-hosted git repository.

yiguolei pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/doris.git


The following commit(s) were added to refs/heads/branch-2.1 by this push:
     new 9bc547578cf [opt](scan) Release instances of Segment to avoid 
consuming a large amount of memory in ParallelScannerBuilder (#44189) (#44608)
9bc547578cf is described below

commit 9bc547578cf2fa8793b6d9954f7f770ed496989b
Author: Jerry Hu <[email protected]>
AuthorDate: Tue Nov 26 19:25:58 2024 +0800

    [opt](scan) Release instances of Segment to avoid consuming a large amount 
of memory in ParallelScannerBuilder (#44189) (#44608)
---
 be/src/olap/parallel_scanner_builder.cpp | 20 +++++++++++---------
 be/src/olap/parallel_scanner_builder.h   |  2 +-
 2 files changed, 12 insertions(+), 10 deletions(-)

diff --git a/be/src/olap/parallel_scanner_builder.cpp 
b/be/src/olap/parallel_scanner_builder.cpp
index f960519d29a..c0a1b75ec75 100644
--- a/be/src/olap/parallel_scanner_builder.cpp
+++ b/be/src/olap/parallel_scanner_builder.cpp
@@ -20,6 +20,7 @@
 #include <shared_mutex>
 
 #include "olap/rowset/beta_rowset.h"
+#include "olap/segment_loader.h"
 #include "pipeline/exec/olap_scan_operator.h"
 #include "vec/exec/scan/new_olap_scanner.h"
 
@@ -56,21 +57,18 @@ Status 
ParallelScannerBuilder<ParentType>::_build_scanners_by_rowid(
             auto rowset = reader->rowset();
             const auto rowset_id = rowset->rowset_id();
 
-            DCHECK(_segment_cache_handles.contains(rowset_id));
-            auto& segment_cache_handle = _segment_cache_handles[rowset_id];
+            const auto& segments_rows = _all_segments_rows[rowset_id];
 
             if (rowset->num_rows() == 0) {
                 continue;
             }
 
-            const auto& segments = segment_cache_handle.get_segments();
             int segment_start = 0;
             auto split = RowSetSplits(reader->clone());
 
-            for (size_t i = 0; i != segments.size(); ++i) {
-                const auto& segment = segments[i];
+            for (size_t i = 0; i != segments_rows.size(); ++i) {
+                const size_t rows_of_segment = segments_rows[i];
                 RowRanges row_ranges;
-                const size_t rows_of_segment = segment->num_rows();
                 int64_t offset_in_segment = 0;
 
                 // try to split large segments into RowRanges
@@ -118,7 +116,7 @@ Status 
ParallelScannerBuilder<ParentType>::_build_scanners_by_rowid(
                 // The non-empty `row_ranges` means there are some rows left 
in this segment not added into `split`.
                 if (!row_ranges.is_empty()) {
                     DCHECK_GT(rows_collected, 0);
-                    DCHECK_EQ(row_ranges.to(), segment->num_rows());
+                    DCHECK_EQ(row_ranges.to(), rows_of_segment);
                     
split.segment_row_ranges.emplace_back(std::move(row_ranges));
                 }
             }
@@ -126,7 +124,7 @@ Status 
ParallelScannerBuilder<ParentType>::_build_scanners_by_rowid(
             DCHECK_LE(rows_collected, _rows_per_scanner);
             if (rows_collected > 0) {
                 split.segment_offsets.first = segment_start;
-                split.segment_offsets.second = segments.size();
+                split.segment_offsets.second = segments_rows.size();
                 DCHECK_GT(split.segment_offsets.second, 
split.segment_offsets.first);
                 DCHECK_EQ(split.segment_row_ranges.size(),
                           split.segment_offsets.second - 
split.segment_offsets.first);
@@ -176,10 +174,14 @@ Status ParallelScannerBuilder<ParentType>::_load() {
             auto rowset = rs_split.rs_reader->rowset();
             RETURN_IF_ERROR(rowset->load());
             const auto rowset_id = rowset->rowset_id();
-            auto& segment_cache_handle = _segment_cache_handles[rowset_id];
+            SegmentCacheHandle segment_cache_handle;
 
             RETURN_IF_ERROR(SegmentLoader::instance()->load_segments(
                     std::dynamic_pointer_cast<BetaRowset>(rowset), 
&segment_cache_handle, true));
+
+            for (const auto& segment : segment_cache_handle.get_segments()) {
+                
_all_segments_rows[rowset_id].emplace_back(segment->num_rows());
+            }
             _total_rows += rowset->num_rows();
         }
     }
diff --git a/be/src/olap/parallel_scanner_builder.h 
b/be/src/olap/parallel_scanner_builder.h
index 62ef4302571..b68da8ac7a5 100644
--- a/be/src/olap/parallel_scanner_builder.h
+++ b/be/src/olap/parallel_scanner_builder.h
@@ -85,7 +85,7 @@ private:
 
     size_t _rows_per_scanner {_min_rows_per_scanner};
 
-    std::map<RowsetId, SegmentCacheHandle> _segment_cache_handles;
+    std::map<RowsetId, std::vector<size_t>> _all_segments_rows;
 
     std::shared_ptr<RuntimeProfile> _scanner_profile;
     RuntimeState* _state;


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to