This is an automated email from the ASF dual-hosted git repository.

hello-stephen pushed a commit to branch branch-4.2
in repository https://gitbox.apache.org/repos/asf/doris.git


The following commit(s) were added to refs/heads/branch-4.2 by this push:
     new a9dfbb7292a branch-4.2: [fix](be) Reject legacy Variant blocks before 
deserialization (#70473)
a9dfbb7292a is described below

commit a9dfbb7292a93cce564ae3ef66d3fc84e3fe0d07
Author: lihangyu <[email protected]>
AuthorDate: Mon Nov 16 00:59:54 2026 -0800

    branch-4.2: [fix](be) Reject legacy Variant blocks before deserialization 
(#70473)
    
    cherry-pick #69434
---
 be/src/core/data_type/data_type_factory.cpp        |   3 +-
 be/src/core/data_type/data_type_variant.cpp        |  15 +++-
 be/src/core/data_type/data_type_variant.h          |   4 +-
 .../core/block/block_variant_wire_format_test.cpp  | 100 +++++++++++++++++++++
 4 files changed, 118 insertions(+), 4 deletions(-)

diff --git a/be/src/core/data_type/data_type_factory.cpp 
b/be/src/core/data_type/data_type_factory.cpp
index 4988b317c2d..1b2a1ec589b 100644
--- a/be/src/core/data_type/data_type_factory.cpp
+++ b/be/src/core/data_type/data_type_factory.cpp
@@ -260,7 +260,8 @@ DataTypePtr DataTypeFactory::create_data_type(const 
PColumnMeta& pcolumn) {
                                                          
pcolumn.variant_enable_doc_mode());
         } else {
             nested = 
std::make_shared<DataTypeVariant>(pcolumn.variant_max_subcolumns_count(),
-                                                       
pcolumn.variant_enable_doc_mode());
+                                                       
pcolumn.variant_enable_doc_mode(),
+                                                       
pcolumn.has_variant_enable_doc_mode());
         }
         break;
     case PGenericType::JSONB:
diff --git a/be/src/core/data_type/data_type_variant.cpp 
b/be/src/core/data_type/data_type_variant.cpp
index c83efe4cc45..2d03237cfdf 100644
--- a/be/src/core/data_type/data_type_variant.cpp
+++ b/be/src/core/data_type/data_type_variant.cpp
@@ -29,6 +29,7 @@
 #include <vector>
 
 #include "agent/be_exec_version_manager.h"
+#include "common/exception.h"
 #include "core/assert_cast.h"
 #include "core/column/column.h"
 #include "core/column/column_variant.h"
@@ -49,8 +50,11 @@ namespace doris {
 DataTypeVariant::DataTypeVariant(int32_t max_subcolumns_count)
         : DataTypeVariant(max_subcolumns_count, false) {}
 
-DataTypeVariant::DataTypeVariant(int32_t max_subcolumns_count, bool 
enable_doc_mode)
-        : _max_subcolumns_count(max_subcolumns_count), 
_enable_doc_mode(enable_doc_mode) {
+DataTypeVariant::DataTypeVariant(int32_t max_subcolumns_count, bool 
enable_doc_mode,
+                                 bool serialized_with_doc_value)
+        : _max_subcolumns_count(max_subcolumns_count),
+          _enable_doc_mode(enable_doc_mode),
+          _serialized_with_doc_value(serialized_with_doc_value) {
     name = fmt::format("Variant(max subcolumns count = {}, enable doc mode = 
{})",
                        max_subcolumns_count, enable_doc_mode);
 }
@@ -179,6 +183,13 @@ Field DataTypeVariant::get_field(const TExprNode& node) 
const {
 
 const char* DataTypeVariant::deserialize(const char* buf, MutableColumnPtr* 
column,
                                          int be_exec_version) const {
+    // Legacy Variant V1 blocks omit both the doc-mode marker and the trailing 
doc-value map.
+    // Current V1 writers set the marker even when doc mode is disabled.
+    if (!_serialized_with_doc_value) {
+        throw Exception(Status::NotSupported(
+                "Unsupported VARIANT block format: legacy Variant V1 block 
lacks doc-value data; "
+                "use compatible backend versions for Variant block transfer"));
+    }
     auto column_variant = assert_cast<ColumnVariant*>(column->get());
 
     // 1. deserialize num of subcolumns
diff --git a/be/src/core/data_type/data_type_variant.h 
b/be/src/core/data_type/data_type_variant.h
index 3f4e08a0ada..55ebb0aa877 100644
--- a/be/src/core/data_type/data_type_variant.h
+++ b/be/src/core/data_type/data_type_variant.h
@@ -49,6 +49,7 @@ class DataTypeVariant : public IDataType {
 private:
     int32_t _max_subcolumns_count = 0;
     bool _enable_doc_mode = false;
+    bool _serialized_with_doc_value = true;
     std::string name = "Variant";
 
 public:
@@ -56,7 +57,8 @@ public:
     PrimitiveType get_primitive_type() const override { return 
PrimitiveType::TYPE_VARIANT; }
     DataTypeVariant() = default;
     DataTypeVariant(int32_t max_subcolumns_count);
-    DataTypeVariant(int32_t max_subcolumns_count, bool enable_doc_mode);
+    DataTypeVariant(int32_t max_subcolumns_count, bool enable_doc_mode,
+                    bool serialized_with_doc_value = true);
     String do_get_name() const override { return name; }
     const std::string get_family_name() const override { return "Variant"; }
 
diff --git a/be/test/core/block/block_variant_wire_format_test.cpp 
b/be/test/core/block/block_variant_wire_format_test.cpp
new file mode 100644
index 00000000000..bf8b7a193c4
--- /dev/null
+++ b/be/test/core/block/block_variant_wire_format_test.cpp
@@ -0,0 +1,100 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include <gen_cpp/data.pb.h>
+#include <gtest/gtest.h>
+
+#include <memory>
+#include <string>
+
+#include "agent/be_exec_version_manager.h"
+#include "core/block/block.h"
+#include "core/data_type/data_type_nullable.h"
+#include "core/data_type/data_type_struct.h"
+#include "core/data_type/data_type_variant.h"
+#include "core/data_type/data_type_variant_v2.h"
+
+namespace doris {
+namespace {
+
+Status serialize_variant_block(const DataTypePtr& type, int exec_version,
+                               segment_v2::CompressionTypePB compression, 
PBlock* wire) {
+    auto column = type->create_column();
+    column->insert_many_defaults(64);
+    Block block({{std::move(column), type, "v"}});
+    size_t uncompressed_bytes = 0;
+    size_t compressed_bytes = 0;
+    int64_t compression_time = 0;
+    return block.serialize(exec_version, wire, &uncompressed_bytes, 
&compressed_bytes,
+                           &compression_time, compression);
+}
+
+Status deserialize_variant_block(const PBlock& wire, Block* block) {
+    size_t uncompressed_bytes = 0;
+    int64_t decompression_time = 0;
+    return block->deserialize(wire, &uncompressed_bytes, &decompression_time);
+}
+
+} // namespace
+
+TEST(BlockVariantWireFormatTest, RejectsLegacyV1MarkerInsideNullableStruct) {
+    const auto type = std::make_shared<DataTypeStruct>(
+            DataTypes {make_nullable(std::make_shared<DataTypeVariant>())}, 
Strings {"nested"});
+    for (auto compression : {segment_v2::NO_COMPRESSION, segment_v2::SNAPPY}) {
+        PBlock wire;
+        // Keep a decodable payload so the pre-fix test fails without an 
out-of-bounds read.
+        // 4.0.8 sends exec version 8 and omits the V1 doc-mode metadata field.
+        ASSERT_TRUE(serialize_variant_block(type, USE_CONST_SERDE, 
compression, &wire).ok());
+        ASSERT_EQ(wire.compressed(), compression == segment_v2::SNAPPY);
+        auto* variant = wire.mutable_column_metas(0)->mutable_children(0);
+        ASSERT_TRUE(variant->has_variant_enable_doc_mode());
+        ASSERT_FALSE(variant->variant_enable_doc_mode());
+        variant->clear_variant_enable_doc_mode();
+
+        Block decoded;
+        const Status status = deserialize_variant_block(wire, &decoded);
+        EXPECT_TRUE(status.is<ErrorCode::NOT_IMPLEMENTED_ERROR>()) << status;
+        EXPECT_NE(status.to_string().find("Unsupported VARIANT block format"), 
std::string::npos)
+                << status;
+        EXPECT_EQ(decoded.columns(), 0);
+    }
+}
+
+TEST(BlockVariantWireFormatTest, RoundTripsCurrentVariantFormats) {
+    const DataTypes types {std::make_shared<DataTypeVariant>(0, false),
+                           std::make_shared<DataTypeVariant>(0, true),
+                           std::make_shared<DataTypeVariantV2>()};
+    for (const auto& type : types) {
+        SCOPED_TRACE(type->get_name());
+        PBlock wire;
+        ASSERT_TRUE(serialize_variant_block(type, 
BeExecVersionManager::get_newest_version(),
+                                            segment_v2::NO_COMPRESSION, &wire)
+                            .ok());
+        const auto& meta = wire.column_metas(0);
+        ASSERT_TRUE(meta.has_variant_enable_doc_mode());
+        EXPECT_EQ(meta.variant_is_v2(),
+                  dynamic_cast<const DataTypeVariantV2*>(type.get()) != 
nullptr);
+
+        Block decoded;
+        const Status status = deserialize_variant_block(wire, &decoded);
+        ASSERT_TRUE(status.ok()) << status;
+        EXPECT_EQ(decoded.rows(), 64);
+        EXPECT_TRUE(decoded.get_by_position(0).type->equals(*type));
+    }
+}
+
+} // namespace doris


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to