This is an automated email from the ASF dual-hosted git repository.

eldenmoon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git


The following commit(s) were added to refs/heads/master by this push:
     new a02ea2b6572 [fix](storage) Retry Variant external metadata reads after 
memory limit errors (#70723)
a02ea2b6572 is described below

commit a02ea2b6572e51fc19988f268c1a6fe1274d7286
Author: lihangyu <[email protected]>
AuthorDate: Thu Nov 19 10:44:43 2026 +0800

    [fix](storage) Retry Variant external metadata reads after memory limit 
errors (#70723)
---
 .../segment/variant/variant_column_reader.cpp      | 19 +++++---
 .../segment/variant/variant_column_reader.h        |  5 +-
 .../variant/variant_external_meta_reader.cpp       | 15 ------
 .../segment/variant/variant_external_meta_reader.h | 14 +-----
 .../segment/external_col_meta_util_test.cpp        | 57 ++++++++++++++++------
 5 files changed, 60 insertions(+), 50 deletions(-)

diff --git a/be/src/storage/segment/variant/variant_column_reader.cpp 
b/be/src/storage/segment/variant/variant_column_reader.cpp
index cec8aefdb16..05a5c06132f 100644
--- a/be/src/storage/segment/variant/variant_column_reader.cpp
+++ b/be/src/storage/segment/variant/variant_column_reader.cpp
@@ -1474,12 +1474,19 @@ Status 
VariantColumnReader::load_external_meta_once(OlapReaderStatistics* stats,
     // Ensure only one writer can populate `_subcolumns_meta_info` / 
`_statistics`
     // while readers of these structures hold shared locks.
     std::unique_lock<std::shared_mutex> lock(_subcolumns_meta_mutex);
-    VariantStatistics* variant_stats = 
variant_util::should_record_variant_path_stats(
-                                               
_tablet_schema->column_by_uid(_root_unique_id))
-                                               ? _statistics.get()
-                                               : nullptr;
-    return _ext_meta_reader->load_all_once(_subcolumns_meta_info.get(), 
variant_stats, stats,
-                                           source_io_ctx);
+    Status status = _external_meta_load_once->call([&] {
+        VariantStatistics* variant_stats = 
variant_util::should_record_variant_path_stats(
+                                                   
_tablet_schema->column_by_uid(_root_unique_id))
+                                                   ? _statistics.get()
+                                                   : nullptr;
+        return _ext_meta_reader->load_all(_subcolumns_meta_info.get(), 
variant_stats, stats,
+                                          source_io_ctx);
+    });
+    if (status.is<ErrorCode::MEM_LIMIT_EXCEEDED>()) {
+        // A retry skips paths already merged by load_all.
+        _external_meta_load_once.emplace();
+    }
+    return status;
 }
 
 TabletIndexes VariantColumnReader::find_subcolumn_tablet_indexes(const 
TabletColumn& column,
diff --git a/be/src/storage/segment/variant/variant_column_reader.h 
b/be/src/storage/segment/variant/variant_column_reader.h
index 34bb2d3cc7a..c530eef48e4 100644
--- a/be/src/storage/segment/variant/variant_column_reader.h
+++ b/be/src/storage/segment/variant/variant_column_reader.h
@@ -420,8 +420,9 @@ private:
             PathToBinaryColumnCache* binary_column_cache_ptr, const 
std::string& path,
             std::shared_ptr<ColumnReader> binary_column_reader);
 
-    // Protect `_subcolumns_meta_info` and `_statistics` when loading external 
meta.
+    // Protect the external meta load guard, subcolumn metadata and statistics.
     mutable std::shared_mutex _subcolumns_meta_mutex;
+    std::optional<DorisCallOnce<Status>> _external_meta_load_once 
{std::in_place};
     std::unique_ptr<SubcolumnColumnMetaInfo> _subcolumns_meta_info;
     // Sparse column readers (single or bucketized)
     std::shared_ptr<BinaryColumnReader> _binary_column_reader;
@@ -440,8 +441,6 @@ private:
     uint64_t _num_rows {0};
     uint32_t _root_unique_id {0};
 
-    // call-once guard moved into VariantExternalMetaReader
-
     // NestedGroup readers for array<object> paths
     NestedGroupReaders _nested_group_readers;
     std::unique_ptr<NestedGroupReadProvider> _nested_group_read_provider;
diff --git a/be/src/storage/segment/variant/variant_external_meta_reader.cpp 
b/be/src/storage/segment/variant/variant_external_meta_reader.cpp
index 611425b9d66..1ba205434bb 100644
--- a/be/src/storage/segment/variant/variant_external_meta_reader.cpp
+++ b/be/src/storage/segment/variant/variant_external_meta_reader.cpp
@@ -228,19 +228,4 @@ Status VariantExternalMetaReader::has_prefix(const 
std::string& prefix, bool* ou
     return Status::OK();
 }
 
-Status VariantExternalMetaReader::load_all_once(SubcolumnColumnMetaInfo* 
out_meta_tree,
-                                                VariantStatistics* out_stats,
-                                                OlapReaderStatistics* stats,
-                                                const io::IOContext* 
source_io_ctx) {
-    DCHECK(available());
-    return _load_once_call.call([this, out_meta_tree, out_stats, stats, 
source_io_ctx]() -> Status {
-        if (_loaded) {
-            return Status::OK();
-        }
-        RETURN_IF_ERROR(load_all(out_meta_tree, out_stats, stats, 
source_io_ctx));
-        _loaded = true;
-        return Status::OK();
-    });
-}
-
 } // namespace doris::segment_v2
diff --git a/be/src/storage/segment/variant/variant_external_meta_reader.h 
b/be/src/storage/segment/variant/variant_external_meta_reader.h
index 2641e8d2dd1..ccc57ce5a3f 100644
--- a/be/src/storage/segment/variant/variant_external_meta_reader.h
+++ b/be/src/storage/segment/variant/variant_external_meta_reader.h
@@ -27,7 +27,6 @@
 #include "storage/segment/external_col_meta_util.h"
 #include "storage/segment/stream_reader.h"
 #include "storage/segment/variant/variant_statistics.h"
-#include "util/once.h"
 
 namespace doris::segment_v2 {
 
@@ -35,7 +34,7 @@ namespace doris::segment_v2 {
 // It discovers key/value indexed-columns from SegmentFooterPB, supports:
 // - availability check
 // - lookup ColumnMetaPB by relative path
-// - one-time bulk load of all metas into SubcolumnColumnMetaInfo and 
VariantStatistics
+// - bulk load of all metas into SubcolumnColumnMetaInfo and VariantStatistics
 class VariantExternalMetaReader {
 public:
     VariantExternalMetaReader() = default;
@@ -64,12 +63,7 @@ public:
     Status has_prefix(const std::string& prefix, bool* out, 
OlapReaderStatistics* stats = nullptr,
                       const io::IOContext* source_io_ctx = nullptr) const;
 
-    // Ensure external metas are loaded exactly once and merged into provided 
structures.
-    Status load_all_once(SubcolumnColumnMetaInfo* out_meta_tree, 
VariantStatistics* out_stats,
-                         OlapReaderStatistics* stats = nullptr,
-                         const io::IOContext* source_io_ctx = nullptr);
-
-    // Load and merge all external metas without call-once guard.
+    // Load all external metas, skipping paths already present in the output 
tree.
     Status load_all(SubcolumnColumnMetaInfo* out_meta_tree, VariantStatistics* 
out_stats,
                     OlapReaderStatistics* stats = nullptr,
                     const io::IOContext* source_io_ctx = nullptr);
@@ -91,10 +85,6 @@ private:
     // For its non-sparse subcolumns, their column_id is derived as:
     //   root_col_id + 1 + key_ordinal
     uint32_t _root_col_id = 0;
-
-    // call-once guard for bulk loading
-    DorisCallOnce<Status> _load_once_call;
-    bool _loaded = false;
 };
 
 } // namespace doris::segment_v2
diff --git a/be/test/storage/segment/external_col_meta_util_test.cpp 
b/be/test/storage/segment/external_col_meta_util_test.cpp
index 2298cce96a6..df37c026a64 100644
--- a/be/test/storage/segment/external_col_meta_util_test.cpp
+++ b/be/test/storage/segment/external_col_meta_util_test.cpp
@@ -26,12 +26,15 @@
 #include <utility>
 #include <vector>
 
+#include "common/exception.h"
 #include "core/field.h"
 #include "gtest/gtest.h"
 #include "io/cache/remote_scan_cache_write_limiter.h"
 #include "io/fs/local_file_system.h"
 #include "io/io_common.h"
+#include "storage/segment/column_meta_accessor.h"
 #include "storage/segment/segment.h"
+#include "storage/segment/variant/variant_column_reader.h"
 #include "storage/segment/variant/variant_ext_meta_writer.h"
 #include "storage/segment/variant/variant_external_meta_reader.h"
 #include "storage/segment/vertical_segment_writer.h"
@@ -71,6 +74,7 @@ public:
 
     const std::vector<RecordedRead>& records() const { return _records; }
     void clear_records() { _records.clear(); }
+    void fail_next_read_with_allocation_error() { _fail_next_read = true; }
     void set_cache_key_suffix(std::string_view suffix) {
         _path = io::Path(_delegate->path().native() + std::string(suffix));
     }
@@ -87,6 +91,9 @@ protected:
             record.is_index_data = io_ctx->is_index_data;
         }
         _records.push_back(record);
+        if (std::exchange(_fail_next_read, false)) {
+            throw Exception(ErrorCode::MEM_ALLOC_FAILED, "injected page 
allocation failure");
+        }
         return _delegate->read_at(offset, result, bytes_read, io_ctx);
     }
 
@@ -94,6 +101,7 @@ private:
     io::FileReaderSPtr _delegate;
     io::Path _path;
     std::vector<RecordedRead> _records;
+    bool _fail_next_read = false;
 };
 
 // Helper to create a clean local test directory and return absolute path + 
filename.
@@ -412,20 +420,16 @@ TEST(ExternalColMetaUtilTest, 
VariantExtMetaWriterAndReaderInteropAndCompatibili
     EXPECT_EQ(loaded_meta_legacy.none_null_size(), 123);
     
EXPECT_EQ(loaded_meta_legacy.column_path_info().parrent_column_unique_id(), 
root_uid);
 
-    // 8. Bulk load all external metas once and verify subcolumn tree and 
statistics.
+    // 8. Bulk load all external metas and verify subcolumn tree and 
statistics.
     SubcolumnColumnMetaInfo meta_tree;
     VariantStatistics stats;
-    st = reader_new.load_all_once(&meta_tree, &stats);
+    st = reader_new.load_all(&meta_tree, &stats);
     ASSERT_TRUE(st.ok()) << st.to_json();
     EXPECT_FALSE(meta_tree.empty());
     // Non-sparse subcolumn v1.key0 should be recorded as relative path "key0".
     ASSERT_EQ(stats.subcolumns_non_null_size.size(), 1);
     EXPECT_EQ(stats.subcolumns_non_null_size.at("key0"), 123);
 
-    // load_all_once should be idempotent.
-    st = reader_new.load_all_once(&meta_tree, &stats);
-    ASSERT_TRUE(st.ok()) << st.to_json();
-
     // 9. has_prefix should detect existing keys and non-existing prefixes.
     bool has = false;
     st = reader_new.has_prefix("key", &has);
@@ -442,12 +446,14 @@ TEST(ExternalColMetaUtilTest, 
VariantExtMetaWriterAndReaderInteropAndCompatibili
     EXPECT_TRUE(has);
 }
 
-TEST(ExternalColMetaUtilTest, 
VariantExternalMetaReaderPassesCacheLimiterToIndexedColumnReads) {
+TEST(ExternalColMetaUtilTest, 
VariantColumnReaderRetriesExternalMetaAfterPageAllocationFailure) {
     SegmentFooterPB footer;
+    footer.set_version(SEGMENT_FOOTER_VERSION_V3_EXT_COL_META);
     const int32_t root_uid = 1;
     ColumnMetaPB* root = footer.add_columns();
     root->set_unique_id(root_uid);
     root->set_type(int(FieldType::OLAP_FIELD_TYPE_VARIANT));
+    root->set_encoding(PLAIN_ENCODING);
     {
         PathInData root_path("v1");
         root_path.to_protobuf(root->mutable_column_path_info(),
@@ -537,13 +543,36 @@ TEST(ExternalColMetaUtilTest, 
VariantExternalMetaReaderPassesCacheLimiterToIndex
     expect_recorded_reads_with_limiter("lookup_meta_by_path");
     EXPECT_EQ(loaded_meta.none_null_size(), 123);
 
-    SubcolumnColumnMetaInfo meta_tree;
-    VariantStatistics stats;
-    reset_recording("#load_all_once");
-    st = reader.load_all_once(&meta_tree, &stats, &reader_stats, &io_ctx);
-    ASSERT_TRUE(st.ok()) << st.to_json();
-    expect_recorded_reads_with_limiter("load_all_once");
-    EXPECT_EQ(stats.subcolumns_non_null_size.at("key0"), 123);
+    TabletColumn tablet_column;
+    tablet_column.set_name("v1");
+    tablet_column.set_unique_id(root_uid);
+    tablet_column.set_type(FieldType::OLAP_FIELD_TYPE_VARIANT);
+    ColumnReaderOptions column_opts;
+    column_opts.tablet_schema = std::make_shared<TabletSchema>();
+    column_opts.tablet_schema->append_column(std::move(tablet_column));
+    ColumnMetaAccessor accessor;
+    ASSERT_TRUE(accessor.init(*footer_sp, recording_fr).ok());
+    VariantColumnReader variant_reader;
+    st = variant_reader.init(column_opts, &accessor, footer_sp, root_uid, 
footer.num_rows(),
+                             recording_fr, &reader_stats, &io_ctx);
+    ASSERT_TRUE(st.ok()) << st;
+
+    reset_recording("#load_external_meta_once");
+    const PathInData key_path("key0");
+    recording_fr->fail_next_read_with_allocation_error();
+    st = variant_reader.load_external_meta_once(&reader_stats, &io_ctx);
+    ASSERT_TRUE(st.is<ErrorCode::MEM_LIMIT_EXCEEDED>()) << st;
+    expect_recorded_reads_with_limiter("failed load_external_meta_once");
+    EXPECT_EQ(variant_reader.get_subcolumn_meta_by_path(key_path), nullptr);
+
+    st = variant_reader.load_external_meta_once(&reader_stats, &io_ctx);
+    ASSERT_TRUE(st.ok()) << st.to_json();
+    expect_recorded_reads_with_limiter("load_external_meta_once");
+    EXPECT_NE(variant_reader.get_subcolumn_meta_by_path(key_path), nullptr);
+    EXPECT_EQ(variant_reader.get_stats()->subcolumns_non_null_size.at("key0"), 
123);
+
+    ASSERT_TRUE(variant_reader.load_external_meta_once(&reader_stats, 
&io_ctx).ok());
+    EXPECT_TRUE(recording_fr->records().empty());
 
     bool has = false;
     reset_recording("#has_prefix");


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to