This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-4.2
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-4.2 by this push:
new d98664c554d branch-4.2: [fix](storage) Retry Variant external metadata
reads after memory limit errors (#70757)
d98664c554d is described below
commit d98664c554dbd1d157463acfd6acff64a7a2a10c
Author: lihangyu <[email protected]>
AuthorDate: Fri Nov 20 15:13:52 2026 +0800
branch-4.2: [fix](storage) Retry Variant external metadata reads after
memory limit errors (#70757)
cherry-pick #70723
Conflicts were resolved by removing load_all_once to match the
retry-on-memory-limit change, while keeping branch-4.2
compile_check_end.h and test includes.
---
.../segment/variant/variant_column_reader.cpp | 19 +++++---
.../segment/variant/variant_column_reader.h | 5 +-
.../variant/variant_external_meta_reader.cpp | 15 ------
.../segment/variant/variant_external_meta_reader.h | 14 +-----
.../segment/external_col_meta_util_test.cpp | 57 ++++++++++++++++------
5 files changed, 60 insertions(+), 50 deletions(-)
diff --git a/be/src/storage/segment/variant/variant_column_reader.cpp
b/be/src/storage/segment/variant/variant_column_reader.cpp
index 1ca727486a2..36899b020ed 100644
--- a/be/src/storage/segment/variant/variant_column_reader.cpp
+++ b/be/src/storage/segment/variant/variant_column_reader.cpp
@@ -1586,12 +1586,19 @@ Status
VariantColumnReader::load_external_meta_once(OlapReaderStatistics* stats,
// Ensure only one writer can populate `_subcolumns_meta_info` /
`_statistics`
// while readers of these structures hold shared locks.
std::unique_lock<std::shared_mutex> lock(_subcolumns_meta_mutex);
- VariantStatistics* variant_stats =
variant_util::should_record_variant_path_stats(
-
_tablet_schema->column_by_uid(_root_unique_id))
- ? _statistics.get()
- : nullptr;
- return _ext_meta_reader->load_all_once(_subcolumns_meta_info.get(),
variant_stats, stats,
- source_io_ctx);
+ Status status = _external_meta_load_once->call([&] {
+ VariantStatistics* variant_stats =
variant_util::should_record_variant_path_stats(
+
_tablet_schema->column_by_uid(_root_unique_id))
+ ? _statistics.get()
+ : nullptr;
+ return _ext_meta_reader->load_all(_subcolumns_meta_info.get(),
variant_stats, stats,
+ source_io_ctx);
+ });
+ if (status.is<ErrorCode::MEM_LIMIT_EXCEEDED>()) {
+ // A retry skips paths already merged by load_all.
+ _external_meta_load_once.emplace();
+ }
+ return status;
}
TabletIndexes VariantColumnReader::find_subcolumn_tablet_indexes(const
TabletColumn& column,
diff --git a/be/src/storage/segment/variant/variant_column_reader.h
b/be/src/storage/segment/variant/variant_column_reader.h
index 3419512d222..1f8779ffc34 100644
--- a/be/src/storage/segment/variant/variant_column_reader.h
+++ b/be/src/storage/segment/variant/variant_column_reader.h
@@ -425,8 +425,9 @@ private:
PathToBinaryColumnCache* binary_column_cache_ptr, const
std::string& path,
std::shared_ptr<ColumnReader> binary_column_reader);
- // Protect `_subcolumns_meta_info` and `_statistics` when loading external
meta.
+ // Protect the external meta load guard, subcolumn metadata and statistics.
mutable std::shared_mutex _subcolumns_meta_mutex;
+ std::optional<DorisCallOnce<Status>> _external_meta_load_once
{std::in_place};
std::unique_ptr<SubcolumnColumnMetaInfo> _subcolumns_meta_info;
// Sparse column readers (single or bucketized)
std::shared_ptr<BinaryColumnReader> _binary_column_reader;
@@ -445,8 +446,6 @@ private:
uint64_t _num_rows {0};
uint32_t _root_unique_id {0};
- // call-once guard moved into VariantExternalMetaReader
-
// NestedGroup readers for array<object> paths
NestedGroupReaders _nested_group_readers;
std::unique_ptr<NestedGroupReadProvider> _nested_group_read_provider;
diff --git a/be/src/storage/segment/variant/variant_external_meta_reader.cpp
b/be/src/storage/segment/variant/variant_external_meta_reader.cpp
index 72cf1f620bf..92d74c54610 100644
--- a/be/src/storage/segment/variant/variant_external_meta_reader.cpp
+++ b/be/src/storage/segment/variant/variant_external_meta_reader.cpp
@@ -230,21 +230,6 @@ Status VariantExternalMetaReader::has_prefix(const
std::string& prefix, bool* ou
return Status::OK();
}
-Status VariantExternalMetaReader::load_all_once(SubcolumnColumnMetaInfo*
out_meta_tree,
- VariantStatistics* out_stats,
- OlapReaderStatistics* stats,
- const io::IOContext*
source_io_ctx) {
- DCHECK(available());
- return _load_once_call.call([this, out_meta_tree, out_stats, stats,
source_io_ctx]() -> Status {
- if (_loaded) {
- return Status::OK();
- }
- RETURN_IF_ERROR(load_all(out_meta_tree, out_stats, stats,
source_io_ctx));
- _loaded = true;
- return Status::OK();
- });
-}
-
#include "common/compile_check_end.h"
} // namespace doris::segment_v2
diff --git a/be/src/storage/segment/variant/variant_external_meta_reader.h
b/be/src/storage/segment/variant/variant_external_meta_reader.h
index d006bdec2bb..394f66c988f 100644
--- a/be/src/storage/segment/variant/variant_external_meta_reader.h
+++ b/be/src/storage/segment/variant/variant_external_meta_reader.h
@@ -27,7 +27,6 @@
#include "storage/segment/external_col_meta_util.h"
#include "storage/segment/stream_reader.h"
#include "storage/segment/variant/variant_statistics.h"
-#include "util/once.h"
namespace doris::segment_v2 {
@@ -37,7 +36,7 @@ namespace doris::segment_v2 {
// It discovers key/value indexed-columns from SegmentFooterPB, supports:
// - availability check
// - lookup ColumnMetaPB by relative path
-// - one-time bulk load of all metas into SubcolumnColumnMetaInfo and
VariantStatistics
+// - bulk load of all metas into SubcolumnColumnMetaInfo and VariantStatistics
class VariantExternalMetaReader {
public:
VariantExternalMetaReader() = default;
@@ -66,12 +65,7 @@ public:
Status has_prefix(const std::string& prefix, bool* out,
OlapReaderStatistics* stats = nullptr,
const io::IOContext* source_io_ctx = nullptr) const;
- // Ensure external metas are loaded exactly once and merged into provided
structures.
- Status load_all_once(SubcolumnColumnMetaInfo* out_meta_tree,
VariantStatistics* out_stats,
- OlapReaderStatistics* stats = nullptr,
- const io::IOContext* source_io_ctx = nullptr);
-
- // Load and merge all external metas without call-once guard.
+ // Load all external metas, skipping paths already present in the output
tree.
Status load_all(SubcolumnColumnMetaInfo* out_meta_tree, VariantStatistics*
out_stats,
OlapReaderStatistics* stats = nullptr,
const io::IOContext* source_io_ctx = nullptr);
@@ -93,10 +87,6 @@ private:
// For its non-sparse subcolumns, their column_id is derived as:
// root_col_id + 1 + key_ordinal
uint32_t _root_col_id = 0;
-
- // call-once guard for bulk loading
- DorisCallOnce<Status> _load_once_call;
- bool _loaded = false;
};
#include "common/compile_check_end.h"
diff --git a/be/test/storage/segment/external_col_meta_util_test.cpp
b/be/test/storage/segment/external_col_meta_util_test.cpp
index ff22c2368a4..6c1d51bd9bf 100644
--- a/be/test/storage/segment/external_col_meta_util_test.cpp
+++ b/be/test/storage/segment/external_col_meta_util_test.cpp
@@ -26,13 +26,16 @@
#include <utility>
#include <vector>
+#include "common/exception.h"
#include "core/field.h"
#include "gtest/gtest.h"
#include "io/cache/remote_scan_cache_write_limiter.h"
#include "io/fs/local_file_system.h"
#include "io/io_common.h"
+#include "storage/segment/column_meta_accessor.h"
#include "storage/segment/segment.h"
#include "storage/segment/segment_writer.h"
+#include "storage/segment/variant/variant_column_reader.h"
#include "storage/segment/variant/variant_ext_meta_writer.h"
#include "storage/segment/variant/variant_external_meta_reader.h"
#include "storage/tablet/tablet_schema_helper.h"
@@ -71,6 +74,7 @@ public:
const std::vector<RecordedRead>& records() const { return _records; }
void clear_records() { _records.clear(); }
+ void fail_next_read_with_allocation_error() { _fail_next_read = true; }
void set_cache_key_suffix(std::string_view suffix) {
_path = io::Path(_delegate->path().native() + std::string(suffix));
}
@@ -87,6 +91,9 @@ protected:
record.is_index_data = io_ctx->is_index_data;
}
_records.push_back(record);
+ if (std::exchange(_fail_next_read, false)) {
+ throw Exception(ErrorCode::MEM_ALLOC_FAILED, "injected page
allocation failure");
+ }
return _delegate->read_at(offset, result, bytes_read, io_ctx);
}
@@ -94,6 +101,7 @@ private:
io::FileReaderSPtr _delegate;
io::Path _path;
std::vector<RecordedRead> _records;
+ bool _fail_next_read = false;
};
// Helper to create a clean local test directory and return absolute path +
filename.
@@ -411,20 +419,16 @@ TEST(ExternalColMetaUtilTest,
VariantExtMetaWriterAndReaderInteropAndCompatibili
EXPECT_EQ(loaded_meta_legacy.none_null_size(), 123);
EXPECT_EQ(loaded_meta_legacy.column_path_info().parrent_column_unique_id(),
root_uid);
- // 8. Bulk load all external metas once and verify subcolumn tree and
statistics.
+ // 8. Bulk load all external metas and verify subcolumn tree and
statistics.
SubcolumnColumnMetaInfo meta_tree;
VariantStatistics stats;
- st = reader_new.load_all_once(&meta_tree, &stats);
+ st = reader_new.load_all(&meta_tree, &stats);
ASSERT_TRUE(st.ok()) << st.to_json();
EXPECT_FALSE(meta_tree.empty());
// Non-sparse subcolumn v1.key0 should be recorded as relative path "key0".
ASSERT_EQ(stats.subcolumns_non_null_size.size(), 1);
EXPECT_EQ(stats.subcolumns_non_null_size.at("key0"), 123);
- // load_all_once should be idempotent.
- st = reader_new.load_all_once(&meta_tree, &stats);
- ASSERT_TRUE(st.ok()) << st.to_json();
-
// 9. has_prefix should detect existing keys and non-existing prefixes.
bool has = false;
st = reader_new.has_prefix("key", &has);
@@ -441,12 +445,14 @@ TEST(ExternalColMetaUtilTest,
VariantExtMetaWriterAndReaderInteropAndCompatibili
EXPECT_TRUE(has);
}
-TEST(ExternalColMetaUtilTest,
VariantExternalMetaReaderPassesCacheLimiterToIndexedColumnReads) {
+TEST(ExternalColMetaUtilTest,
VariantColumnReaderRetriesExternalMetaAfterPageAllocationFailure) {
SegmentFooterPB footer;
+ footer.set_version(SEGMENT_FOOTER_VERSION_V3_EXT_COL_META);
const int32_t root_uid = 1;
ColumnMetaPB* root = footer.add_columns();
root->set_unique_id(root_uid);
root->set_type(int(FieldType::OLAP_FIELD_TYPE_VARIANT));
+ root->set_encoding(PLAIN_ENCODING);
{
PathInData root_path("v1");
root_path.to_protobuf(root->mutable_column_path_info(),
@@ -536,13 +542,36 @@ TEST(ExternalColMetaUtilTest,
VariantExternalMetaReaderPassesCacheLimiterToIndex
expect_recorded_reads_with_limiter("lookup_meta_by_path");
EXPECT_EQ(loaded_meta.none_null_size(), 123);
- SubcolumnColumnMetaInfo meta_tree;
- VariantStatistics stats;
- reset_recording("#load_all_once");
- st = reader.load_all_once(&meta_tree, &stats, &reader_stats, &io_ctx);
- ASSERT_TRUE(st.ok()) << st.to_json();
- expect_recorded_reads_with_limiter("load_all_once");
- EXPECT_EQ(stats.subcolumns_non_null_size.at("key0"), 123);
+ TabletColumn tablet_column;
+ tablet_column.set_name("v1");
+ tablet_column.set_unique_id(root_uid);
+ tablet_column.set_type(FieldType::OLAP_FIELD_TYPE_VARIANT);
+ ColumnReaderOptions column_opts;
+ column_opts.tablet_schema = std::make_shared<TabletSchema>();
+ column_opts.tablet_schema->append_column(std::move(tablet_column));
+ ColumnMetaAccessor accessor;
+ ASSERT_TRUE(accessor.init(*footer_sp, recording_fr).ok());
+ VariantColumnReader variant_reader;
+ st = variant_reader.init(column_opts, &accessor, footer_sp, root_uid,
footer.num_rows(),
+ recording_fr, &reader_stats, &io_ctx);
+ ASSERT_TRUE(st.ok()) << st;
+
+ reset_recording("#load_external_meta_once");
+ const PathInData key_path("key0");
+ recording_fr->fail_next_read_with_allocation_error();
+ st = variant_reader.load_external_meta_once(&reader_stats, &io_ctx);
+ ASSERT_TRUE(st.is<ErrorCode::MEM_LIMIT_EXCEEDED>()) << st;
+ expect_recorded_reads_with_limiter("failed load_external_meta_once");
+ EXPECT_EQ(variant_reader.get_subcolumn_meta_by_path(key_path), nullptr);
+
+ st = variant_reader.load_external_meta_once(&reader_stats, &io_ctx);
+ ASSERT_TRUE(st.ok()) << st.to_json();
+ expect_recorded_reads_with_limiter("load_external_meta_once");
+ EXPECT_NE(variant_reader.get_subcolumn_meta_by_path(key_path), nullptr);
+ EXPECT_EQ(variant_reader.get_stats()->subcolumns_non_null_size.at("key0"),
123);
+
+ ASSERT_TRUE(variant_reader.load_external_meta_once(&reader_stats,
&io_ctx).ok());
+ EXPECT_TRUE(recording_fr->records().empty());
bool has = false;
reset_recording("#has_prefix");
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]