This is an automated email from the ASF dual-hosted git repository.

hello-stephen pushed a commit to branch branch-4.2
in repository https://gitbox.apache.org/repos/asf/doris.git


The following commit(s) were added to refs/heads/branch-4.2 by this push:
     new 71fc28ca349 branch-4.2: [improvement](variant) Assert Variant sparse 
byte order in debug builds (#70590)
71fc28ca349 is described below

commit 71fc28ca3491b902f8f42db444e9ae74c262f0d7
Author: lihangyu <[email protected]>
AuthorDate: Tue Nov 17 18:42:29 2026 +0800

    branch-4.2: [improvement](variant) Assert Variant sparse byte order in 
debug builds (#70590)
    
    cherry-pick #70416
    
    Conflict in
    `be/src/storage/segment/variant/binary_column_extract_iterator.cpp`
    only.
    
    branch-4.2 already uses a `previous_end` / `push_back` loop in
    `_fill_variant_v2_path` (master still uses pre-sized `start`/`end`
    vectors). Kept the branch loop and added:
    
    `DCHECK(is_variant_sparse_row_sorted(*paths, previous_end, end));`
    
    #70001 is already on branch-4.2 via #70083, so the writer-side
    byte-order sort this assertion depends on is present.
---
 be/src/core/column/variant_column_utils.cpp        |  9 +++
 be/src/core/column/variant_column_utils.h          |  5 ++
 .../variant/binary_column_extract_iterator.cpp     |  2 +
 .../variant/sparse_column_merge_iterator.cpp       | 10 +++
 .../segment/variant/v2/variant_assembler.cpp       |  6 ++
 .../binary_column_extract_iterator_test.cpp        | 85 ++++++++++++++++++++++
 6 files changed, 117 insertions(+)

diff --git a/be/src/core/column/variant_column_utils.cpp 
b/be/src/core/column/variant_column_utils.cpp
index 920d439a168..9aba8432947 100644
--- a/be/src/core/column/variant_column_utils.cpp
+++ b/be/src/core/column/variant_column_utils.cpp
@@ -35,6 +35,15 @@ size_t find_variant_sparse_path_lower_bound(StringRef path, 
const ColumnString&
     return start;
 }
 
+bool is_variant_sparse_row_sorted(const ColumnString& sparse_paths, size_t 
start, size_t end) {
+    for (size_t i = start + 1; i < end; ++i) {
+        if (sparse_paths.get_data_at(i) < sparse_paths.get_data_at(i - 1)) {
+            return false;
+        }
+    }
+    return true;
+}
+
 bool is_variant_jsonb_value_semantically_empty(const JsonbValue* value) {
     if (value == nullptr || value->isNull()) {
         return true;
diff --git a/be/src/core/column/variant_column_utils.h 
b/be/src/core/column/variant_column_utils.h
index 146ae51df4d..ec6fd99cb3e 100644
--- a/be/src/core/column/variant_column_utils.h
+++ b/be/src/core/column/variant_column_utils.h
@@ -30,6 +30,11 @@ struct JsonbValue;
 size_t find_variant_sparse_path_lower_bound(StringRef path, const 
ColumnString& sparse_paths,
                                             size_t start, size_t end);
 
+// Readers merge and binary-search sparse keys by raw path bytes, so every 
segment must persist a
+// row in that order: the writer sorts the sparse plan (#70001) and compaction 
copies cells verbatim.
+// Only debug assertions call this, so release reads and compaction pay 
nothing for the check.
+bool is_variant_sparse_row_sorted(const ColumnString& sparse_paths, size_t 
start, size_t end);
+
 // True when a JSONB value recursively contains no visible scalar payload. 
Variant V1 and V2
 // readers share this rule when deciding whether a materialized nested value 
contributes a path.
 bool is_variant_jsonb_value_semantically_empty(const JsonbValue* value);
diff --git a/be/src/storage/segment/variant/binary_column_extract_iterator.cpp 
b/be/src/storage/segment/variant/binary_column_extract_iterator.cpp
index 4ef7cb7e384..cbe03b1f1e2 100644
--- a/be/src/storage/segment/variant/binary_column_extract_iterator.cpp
+++ b/be/src/storage/segment/variant/binary_column_extract_iterator.cpp
@@ -132,6 +132,8 @@ Status 
BinaryColumnExtractIterator::_fill_variant_v2_path(MutableColumnPtr& dst,
         if (end < previous_end || end > paths->size()) {
             return Status::Corruption("Variant sparse row {} has invalid 
offset {}", row, end);
         }
+        // Readers binary-search by raw path bytes; a producer that violates 
the order is a bug.
+        DCHECK(is_variant_sparse_row_sorted(*paths, previous_end, end));
         const size_t lower =
                 find_variant_sparse_path_lower_bound(requested, *paths, 
previous_end, end);
         if (lower < end && paths->get_data_at(lower) == requested) {
diff --git a/be/src/storage/segment/variant/sparse_column_merge_iterator.cpp 
b/be/src/storage/segment/variant/sparse_column_merge_iterator.cpp
index 67974585923..05ee1084c50 100644
--- a/be/src/storage/segment/variant/sparse_column_merge_iterator.cpp
+++ b/be/src/storage/segment/variant/sparse_column_merge_iterator.cpp
@@ -27,6 +27,7 @@
 #include "core/column/column_nullable.h"
 #include "core/column/column_variant.h"
 #include "core/column/subcolumn_tree.h"
+#include "core/column/variant_column_utils.h"
 #include "core/data_type/data_type.h"
 #include "core/data_type/data_type_nullable.h"
 #include "storage/segment/column_reader.h"
@@ -126,6 +127,9 @@ void SparseColumnMergeIterator::_merge_to(MutableColumnPtr& 
dst) {
 
         size_t offset = src_serialized_sparse_column_offsets[row - 1];
         size_t end = src_serialized_sparse_column_offsets[row];
+        // Compaction only copies cells, and every reader binary-searches by 
raw path bytes, so the
+        // source row must already be byte-ordered. Debug-only: release 
compaction pays nothing.
+        DCHECK(is_variant_sparse_row_sorted(src_sparse_column_paths, offset, 
end));
         // Iterator over [path, binary value]
         for (size_t i = offset; i != end; ++i) {
             const StringRef src_sparse_path_string = 
src_sparse_column_paths.get_data_at(i);
@@ -160,6 +164,12 @@ void 
SparseColumnMergeIterator::_merge_to(MutableColumnPtr& dst) {
                                                  dst_sparse_column_values, 
src_path, row);
         }
 
+        // The interleaving above assumes the source sparse cells and the 
materialized subcolumns
+        // are byte-ordered. Assert the merged row keeps that invariant for 
downstream readers;
+        // debug-only, so release compaction pays nothing.
+        DCHECK(is_variant_sparse_row_sorted(dst_sparse_column_paths,
+                                            
dst_sparse_column_offsets[ssize_t(row) - 1],
+                                            dst_sparse_column_paths.size()));
         // All the sparse columns in this row are null.
         dst_sparse_column_offsets.push_back(dst_sparse_column_paths.size());
     }
diff --git a/be/src/storage/segment/variant/v2/variant_assembler.cpp 
b/be/src/storage/segment/variant/v2/variant_assembler.cpp
index 304641be813..2ec7d450671 100644
--- a/be/src/storage/segment/variant/v2/variant_assembler.cpp
+++ b/be/src/storage/segment/variant/v2/variant_assembler.cpp
@@ -90,6 +90,12 @@ struct StorageMapRowCursor {
         DORIS_CHECK(values != nullptr);
         DORIS_CHECK_EQ(paths->size(), values->size());
         DCHECK(rows == 0 || storage_map->offset_at(static_cast<ssize_t>(rows)) 
== paths->size());
+        // Readers binary-search sparse keys by raw path bytes. The writer 
(#70001) and vertical
+        // compaction canonicalize every row, so a non-conforming row reaching 
this cursor is a
+        // producer bug; keep the check debug-only so release scans pay 
nothing.
+        for (size_t row = 0; row < rows; ++row) {
+            DCHECK(is_variant_sparse_row_sorted(*paths, begin(row), 
row_end(row)));
+        }
     }
 
     size_t begin(size_t row) const noexcept { return 
source->offset_at(static_cast<ssize_t>(row)); }
diff --git a/be/test/storage/segment/binary_column_extract_iterator_test.cpp 
b/be/test/storage/segment/binary_column_extract_iterator_test.cpp
index 1a85c4fb15a..e722da00bae 100644
--- a/be/test/storage/segment/binary_column_extract_iterator_test.cpp
+++ b/be/test/storage/segment/binary_column_extract_iterator_test.cpp
@@ -17,6 +17,7 @@
 
 #include "storage/segment/variant/binary_column_extract_iterator.h"
 
+#include <glog/logging.h>
 #include <gtest/gtest.h>
 
 #include <algorithm>
@@ -31,6 +32,7 @@
 #include "core/column/column_string.h"
 #include "core/column/column_variant.h"
 #include "core/column/column_vector.h"
+#include "core/column/variant_column_utils.h"
 #include "core/column/variant_v2/column_variant_v2.h"
 #include "core/data_type/data_type_nullable.h"
 #include "core/data_type/data_type_string.h"
@@ -40,6 +42,8 @@
 #include "core/string_buffer.hpp"
 #include "core/value/jsonb_value.h"
 #include "storage/iterators.h"
+#include "storage/segment/variant/sparse_column_merge_iterator.h"
+#include "storage/segment/variant/v2/variant_assembler.h"
 
 namespace doris::segment_v2 {
 namespace {
@@ -848,5 +852,86 @@ TEST(BinaryColumnExtractIteratorV2Test, 
DoesNotRescanFrozenPathOrdering) {
     EXPECT_EQ(assert_cast<const 
ColumnInt64&>(typed.get_nested_column()).get_element(0), 7);
 }
 
+// Debug builds must reject a sparse row that violates the byte-order 
invariant; the check is a
+// DCHECK so release scans and compaction pay nothing.
+// Assert the helper directly: EXPECT_DEATH(assemble(...)) forks, which is 
unsafe in the full
+// threaded ASAN BEUT process and aborted the runner with exit 134.
+#if DCHECK_IS_ON()
+TEST(VariantAssemblerTest, DebugCheckRejectsUnsortedSparseRows) {
+    auto sparse = ColumnVariant::create_binary_column_fn();
+    auto& map = assert_cast<ColumnMap&>(*sparse);
+    DataTypeInt64 int_type;
+    auto ints = ColumnInt64::create();
+    ints->insert_value(1);
+    ints->insert_value(2);
+    append_storage_cell(map, "a.c", int_type, *ints, 0);
+    append_storage_cell(map, "a-b", int_type, *ints, 1);
+    map.get_offsets().push_back(2);
+
+    const auto& paths = assert_cast<const ColumnString&>(map.get_keys());
+    ASSERT_EQ(paths.size(), 2);
+    EXPECT_FALSE(is_variant_sparse_row_sorted(paths, 0, paths.size()));
+}
+#endif
+
+// Vertical compaction merges the source sparse row with materialized 
subcolumns. The interleaving
+// assumes both sides are byte-ordered, and the merged row is what later 
readers see, so it must
+// keep that invariant.
+TEST(SparseColumnMergeIteratorTest, 
KeepsByteOrderWhenMergingMaterializedSubcolumn) {
+    auto sparse = ColumnVariant::create_binary_column_fn();
+    auto& map = assert_cast<ColumnMap&>(*sparse);
+    DataTypeInt64 int_type;
+    auto ints = ColumnInt64::create();
+    ints->insert_value(1);
+    ints->insert_value(2);
+    append_storage_cell(map, "a-b", int_type, *ints, 0);
+    append_storage_cell(map, "a.c", int_type, *ints, 1);
+    map.get_offsets().push_back(2);
+
+    auto counters = std::make_shared<SparseReadCounters>();
+    auto cache = std::make_shared<BinaryColumnCache>(
+            std::make_unique<FixedSparseIterator>(std::move(sparse), counters),
+            ColumnVariant::create_binary_column_fn());
+
+    // Materialized path a.d; the dense iterator yields one nullable bigint 
row: 3.
+    auto dense_type = 
std::make_shared<DataTypeNullable>(std::make_shared<DataTypeInt64>());
+    auto dense_values = ColumnNullable::create(ColumnInt64::create(), 
ColumnUInt8::create());
+    
assert_cast<ColumnInt64&>(dense_values->get_nested_column()).insert_value(3);
+    dense_values->get_null_map_data().push_back(0);
+    SubstreamReaderTree src_subcolumns;
+    ASSERT_TRUE(src_subcolumns.add(
+            PathInData("a.d"),
+            SubstreamIterator(
+                    ColumnNullable::create(ColumnInt64::create(), 
ColumnUInt8::create()),
+                    
std::make_unique<FixedSparseIterator>(std::move(dense_values),
+                                                          
std::make_shared<SparseReadCounters>()),
+                    dense_type)));
+
+    TabletSchema::PathsSetInfo path_set_info;
+    OlapReaderStatistics stats;
+    StorageReadOptions read_options;
+    read_options.stats = &stats;
+    SparseColumnMergeIterator iterator(path_set_info, std::move(cache), 
std::move(src_subcolumns),
+                                       &read_options);
+    ColumnIteratorOptions iterator_options;
+    iterator_options.stats = &stats;
+    ASSERT_TRUE(iterator.init(iterator_options).ok());
+    ASSERT_TRUE(iterator.seek_to_ordinal(0).ok());
+    auto dst = ColumnVariant::create_binary_column_fn();
+    size_t rows = 1;
+    bool has_null = false;
+    ASSERT_TRUE(iterator.next_batch(&rows, dst, &has_null).ok());
+    ASSERT_EQ(dst->size(), 1);
+    const auto& merged = assert_cast<const ColumnMap&>(*dst);
+    const auto& paths = assert_cast<const ColumnString&>(merged.get_keys());
+    std::vector<std::string> actual;
+    const size_t begin = merged.offset_at(0);
+    const size_t end = begin + merged.size_at(0);
+    for (size_t cell = begin; cell < end; ++cell) {
+        actual.push_back(paths.get_data_at(cell).to_string());
+    }
+    EXPECT_EQ(actual, (std::vector<std::string> {"a-b", "a.c", "a.d"}));
+}
+
 } // namespace
 } // namespace doris::segment_v2


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to