This is an automated email from the ASF dual-hosted git repository.
hello-stephen pushed a commit to branch branch-4.2
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-4.2 by this push:
new 71fc28ca349 branch-4.2: [improvement](variant) Assert Variant sparse
byte order in debug builds (#70590)
71fc28ca349 is described below
commit 71fc28ca3491b902f8f42db444e9ae74c262f0d7
Author: lihangyu <[email protected]>
AuthorDate: Tue Nov 17 18:42:29 2026 +0800
branch-4.2: [improvement](variant) Assert Variant sparse byte order in
debug builds (#70590)
cherry-pick #70416
Conflict in
`be/src/storage/segment/variant/binary_column_extract_iterator.cpp`
only.
branch-4.2 already uses a `previous_end` / `push_back` loop in
`_fill_variant_v2_path` (master still uses pre-sized `start`/`end`
vectors). Kept the branch loop and added:
`DCHECK(is_variant_sparse_row_sorted(*paths, previous_end, end));`
#70001 is already on branch-4.2 via #70083, so the writer-side
byte-order sort this assertion depends on is present.
---
be/src/core/column/variant_column_utils.cpp | 9 +++
be/src/core/column/variant_column_utils.h | 5 ++
.../variant/binary_column_extract_iterator.cpp | 2 +
.../variant/sparse_column_merge_iterator.cpp | 10 +++
.../segment/variant/v2/variant_assembler.cpp | 6 ++
.../binary_column_extract_iterator_test.cpp | 85 ++++++++++++++++++++++
6 files changed, 117 insertions(+)
diff --git a/be/src/core/column/variant_column_utils.cpp
b/be/src/core/column/variant_column_utils.cpp
index 920d439a168..9aba8432947 100644
--- a/be/src/core/column/variant_column_utils.cpp
+++ b/be/src/core/column/variant_column_utils.cpp
@@ -35,6 +35,15 @@ size_t find_variant_sparse_path_lower_bound(StringRef path,
const ColumnString&
return start;
}
+bool is_variant_sparse_row_sorted(const ColumnString& sparse_paths, size_t
start, size_t end) {
+ for (size_t i = start + 1; i < end; ++i) {
+ if (sparse_paths.get_data_at(i) < sparse_paths.get_data_at(i - 1)) {
+ return false;
+ }
+ }
+ return true;
+}
+
bool is_variant_jsonb_value_semantically_empty(const JsonbValue* value) {
if (value == nullptr || value->isNull()) {
return true;
diff --git a/be/src/core/column/variant_column_utils.h
b/be/src/core/column/variant_column_utils.h
index 146ae51df4d..ec6fd99cb3e 100644
--- a/be/src/core/column/variant_column_utils.h
+++ b/be/src/core/column/variant_column_utils.h
@@ -30,6 +30,11 @@ struct JsonbValue;
size_t find_variant_sparse_path_lower_bound(StringRef path, const
ColumnString& sparse_paths,
size_t start, size_t end);
+// Readers merge and binary-search sparse keys by raw path bytes, so every
segment must persist a
+// row in that order: the writer sorts the sparse plan (#70001) and compaction
copies cells verbatim.
+// Only debug assertions call this, so release reads and compaction pay
nothing for the check.
+bool is_variant_sparse_row_sorted(const ColumnString& sparse_paths, size_t
start, size_t end);
+
// True when a JSONB value recursively contains no visible scalar payload.
Variant V1 and V2
// readers share this rule when deciding whether a materialized nested value
contributes a path.
bool is_variant_jsonb_value_semantically_empty(const JsonbValue* value);
diff --git a/be/src/storage/segment/variant/binary_column_extract_iterator.cpp
b/be/src/storage/segment/variant/binary_column_extract_iterator.cpp
index 4ef7cb7e384..cbe03b1f1e2 100644
--- a/be/src/storage/segment/variant/binary_column_extract_iterator.cpp
+++ b/be/src/storage/segment/variant/binary_column_extract_iterator.cpp
@@ -132,6 +132,8 @@ Status
BinaryColumnExtractIterator::_fill_variant_v2_path(MutableColumnPtr& dst,
if (end < previous_end || end > paths->size()) {
return Status::Corruption("Variant sparse row {} has invalid
offset {}", row, end);
}
+ // Readers binary-search by raw path bytes; a producer that violates
the order is a bug.
+ DCHECK(is_variant_sparse_row_sorted(*paths, previous_end, end));
const size_t lower =
find_variant_sparse_path_lower_bound(requested, *paths,
previous_end, end);
if (lower < end && paths->get_data_at(lower) == requested) {
diff --git a/be/src/storage/segment/variant/sparse_column_merge_iterator.cpp
b/be/src/storage/segment/variant/sparse_column_merge_iterator.cpp
index 67974585923..05ee1084c50 100644
--- a/be/src/storage/segment/variant/sparse_column_merge_iterator.cpp
+++ b/be/src/storage/segment/variant/sparse_column_merge_iterator.cpp
@@ -27,6 +27,7 @@
#include "core/column/column_nullable.h"
#include "core/column/column_variant.h"
#include "core/column/subcolumn_tree.h"
+#include "core/column/variant_column_utils.h"
#include "core/data_type/data_type.h"
#include "core/data_type/data_type_nullable.h"
#include "storage/segment/column_reader.h"
@@ -126,6 +127,9 @@ void SparseColumnMergeIterator::_merge_to(MutableColumnPtr&
dst) {
size_t offset = src_serialized_sparse_column_offsets[row - 1];
size_t end = src_serialized_sparse_column_offsets[row];
+ // Compaction only copies cells, and every reader binary-searches by
raw path bytes, so the
+ // source row must already be byte-ordered. Debug-only: release
compaction pays nothing.
+ DCHECK(is_variant_sparse_row_sorted(src_sparse_column_paths, offset,
end));
// Iterator over [path, binary value]
for (size_t i = offset; i != end; ++i) {
const StringRef src_sparse_path_string =
src_sparse_column_paths.get_data_at(i);
@@ -160,6 +164,12 @@ void
SparseColumnMergeIterator::_merge_to(MutableColumnPtr& dst) {
dst_sparse_column_values,
src_path, row);
}
+ // The interleaving above assumes the source sparse cells and the
materialized subcolumns
+ // are byte-ordered. Assert the merged row keeps that invariant for
downstream readers;
+ // debug-only, so release compaction pays nothing.
+ DCHECK(is_variant_sparse_row_sorted(dst_sparse_column_paths,
+
dst_sparse_column_offsets[ssize_t(row) - 1],
+ dst_sparse_column_paths.size()));
// All the sparse columns in this row are null.
dst_sparse_column_offsets.push_back(dst_sparse_column_paths.size());
}
diff --git a/be/src/storage/segment/variant/v2/variant_assembler.cpp
b/be/src/storage/segment/variant/v2/variant_assembler.cpp
index 304641be813..2ec7d450671 100644
--- a/be/src/storage/segment/variant/v2/variant_assembler.cpp
+++ b/be/src/storage/segment/variant/v2/variant_assembler.cpp
@@ -90,6 +90,12 @@ struct StorageMapRowCursor {
DORIS_CHECK(values != nullptr);
DORIS_CHECK_EQ(paths->size(), values->size());
DCHECK(rows == 0 || storage_map->offset_at(static_cast<ssize_t>(rows))
== paths->size());
+ // Readers binary-search sparse keys by raw path bytes. The writer
(#70001) and vertical
+ // compaction canonicalize every row, so a non-conforming row reaching
this cursor is a
+ // producer bug; keep the check debug-only so release scans pay
nothing.
+ for (size_t row = 0; row < rows; ++row) {
+ DCHECK(is_variant_sparse_row_sorted(*paths, begin(row),
row_end(row)));
+ }
}
size_t begin(size_t row) const noexcept { return
source->offset_at(static_cast<ssize_t>(row)); }
diff --git a/be/test/storage/segment/binary_column_extract_iterator_test.cpp
b/be/test/storage/segment/binary_column_extract_iterator_test.cpp
index 1a85c4fb15a..e722da00bae 100644
--- a/be/test/storage/segment/binary_column_extract_iterator_test.cpp
+++ b/be/test/storage/segment/binary_column_extract_iterator_test.cpp
@@ -17,6 +17,7 @@
#include "storage/segment/variant/binary_column_extract_iterator.h"
+#include <glog/logging.h>
#include <gtest/gtest.h>
#include <algorithm>
@@ -31,6 +32,7 @@
#include "core/column/column_string.h"
#include "core/column/column_variant.h"
#include "core/column/column_vector.h"
+#include "core/column/variant_column_utils.h"
#include "core/column/variant_v2/column_variant_v2.h"
#include "core/data_type/data_type_nullable.h"
#include "core/data_type/data_type_string.h"
@@ -40,6 +42,8 @@
#include "core/string_buffer.hpp"
#include "core/value/jsonb_value.h"
#include "storage/iterators.h"
+#include "storage/segment/variant/sparse_column_merge_iterator.h"
+#include "storage/segment/variant/v2/variant_assembler.h"
namespace doris::segment_v2 {
namespace {
@@ -848,5 +852,86 @@ TEST(BinaryColumnExtractIteratorV2Test,
DoesNotRescanFrozenPathOrdering) {
EXPECT_EQ(assert_cast<const
ColumnInt64&>(typed.get_nested_column()).get_element(0), 7);
}
+// Debug builds must reject a sparse row that violates the byte-order
invariant; the check is a
+// DCHECK so release scans and compaction pay nothing.
+// Assert the helper directly: EXPECT_DEATH(assemble(...)) forks, which is
unsafe in the full
+// threaded ASAN BEUT process and aborted the runner with exit 134.
+#if DCHECK_IS_ON()
+TEST(VariantAssemblerTest, DebugCheckRejectsUnsortedSparseRows) {
+ auto sparse = ColumnVariant::create_binary_column_fn();
+ auto& map = assert_cast<ColumnMap&>(*sparse);
+ DataTypeInt64 int_type;
+ auto ints = ColumnInt64::create();
+ ints->insert_value(1);
+ ints->insert_value(2);
+ append_storage_cell(map, "a.c", int_type, *ints, 0);
+ append_storage_cell(map, "a-b", int_type, *ints, 1);
+ map.get_offsets().push_back(2);
+
+ const auto& paths = assert_cast<const ColumnString&>(map.get_keys());
+ ASSERT_EQ(paths.size(), 2);
+ EXPECT_FALSE(is_variant_sparse_row_sorted(paths, 0, paths.size()));
+}
+#endif
+
+// Vertical compaction merges the source sparse row with materialized
subcolumns. The interleaving
+// assumes both sides are byte-ordered, and the merged row is what later
readers see, so it must
+// keep that invariant.
+TEST(SparseColumnMergeIteratorTest,
KeepsByteOrderWhenMergingMaterializedSubcolumn) {
+ auto sparse = ColumnVariant::create_binary_column_fn();
+ auto& map = assert_cast<ColumnMap&>(*sparse);
+ DataTypeInt64 int_type;
+ auto ints = ColumnInt64::create();
+ ints->insert_value(1);
+ ints->insert_value(2);
+ append_storage_cell(map, "a-b", int_type, *ints, 0);
+ append_storage_cell(map, "a.c", int_type, *ints, 1);
+ map.get_offsets().push_back(2);
+
+ auto counters = std::make_shared<SparseReadCounters>();
+ auto cache = std::make_shared<BinaryColumnCache>(
+ std::make_unique<FixedSparseIterator>(std::move(sparse), counters),
+ ColumnVariant::create_binary_column_fn());
+
+ // Materialized path a.d; the dense iterator yields one nullable bigint
row: 3.
+ auto dense_type =
std::make_shared<DataTypeNullable>(std::make_shared<DataTypeInt64>());
+ auto dense_values = ColumnNullable::create(ColumnInt64::create(),
ColumnUInt8::create());
+
assert_cast<ColumnInt64&>(dense_values->get_nested_column()).insert_value(3);
+ dense_values->get_null_map_data().push_back(0);
+ SubstreamReaderTree src_subcolumns;
+ ASSERT_TRUE(src_subcolumns.add(
+ PathInData("a.d"),
+ SubstreamIterator(
+ ColumnNullable::create(ColumnInt64::create(),
ColumnUInt8::create()),
+
std::make_unique<FixedSparseIterator>(std::move(dense_values),
+
std::make_shared<SparseReadCounters>()),
+ dense_type)));
+
+ TabletSchema::PathsSetInfo path_set_info;
+ OlapReaderStatistics stats;
+ StorageReadOptions read_options;
+ read_options.stats = &stats;
+ SparseColumnMergeIterator iterator(path_set_info, std::move(cache),
std::move(src_subcolumns),
+ &read_options);
+ ColumnIteratorOptions iterator_options;
+ iterator_options.stats = &stats;
+ ASSERT_TRUE(iterator.init(iterator_options).ok());
+ ASSERT_TRUE(iterator.seek_to_ordinal(0).ok());
+ auto dst = ColumnVariant::create_binary_column_fn();
+ size_t rows = 1;
+ bool has_null = false;
+ ASSERT_TRUE(iterator.next_batch(&rows, dst, &has_null).ok());
+ ASSERT_EQ(dst->size(), 1);
+ const auto& merged = assert_cast<const ColumnMap&>(*dst);
+ const auto& paths = assert_cast<const ColumnString&>(merged.get_keys());
+ std::vector<std::string> actual;
+ const size_t begin = merged.offset_at(0);
+ const size_t end = begin + merged.size_at(0);
+ for (size_t cell = begin; cell < end; ++cell) {
+ actual.push_back(paths.get_data_at(cell).to_string());
+ }
+ EXPECT_EQ(actual, (std::vector<std::string> {"a-b", "a.c", "a.d"}));
+}
+
} // namespace
} // namespace doris::segment_v2
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]