This is an automated email from the ASF dual-hosted git repository.

eldenmoon pushed a commit to branch variant-sparse
in repository https://gitbox.apache.org/repos/asf/doris.git


The following commit(s) were added to refs/heads/variant-sparse by this push:
     new d89ac056382 [test](variant) add ut (#45947)
d89ac056382 is described below

commit d89ac0563823e1fc00b49e947b69bdc916d4043f
Author: Sun Chenyang <[email protected]>
AuthorDate: Mon Jan 6 21:56:10 2025 +0800

    [test](variant) add ut (#45947)
---
 .../rowset/segment_v2/hierarchical_data_reader.cpp |  24 +-
 .../segment_v2/variant_column_writer_impl.cpp      |  39 +-
 be/src/vec/columns/column_object.cpp               | 196 +++--
 be/src/vec/columns/column_object.h                 |  19 +-
 be/src/vec/common/schema_util.cpp                  |   5 +-
 be/src/vec/functions/function_variant_element.cpp  |   9 +-
 be/test/vec/columns/column_object_test.cpp         | 795 +++++++++++++++++++++
 .../data/variant_p0/variant_with_rowstore.out      |   2 +-
 8 files changed, 987 insertions(+), 102 deletions(-)

diff --git a/be/src/olap/rowset/segment_v2/hierarchical_data_reader.cpp 
b/be/src/olap/rowset/segment_v2/hierarchical_data_reader.cpp
index a584cda2e3d..8df56dc05e8 100644
--- a/be/src/olap/rowset/segment_v2/hierarchical_data_reader.cpp
+++ b/be/src/olap/rowset/segment_v2/hierarchical_data_reader.cpp
@@ -29,6 +29,7 @@
 #include "vec/common/assert_cast.h"
 #include "vec/common/schema_util.h"
 #include "vec/data_types/data_type.h"
+#include "vec/data_types/data_type_factory.hpp"
 #include "vec/data_types/data_type_nullable.h"
 #include "vec/json/path_in_data.h"
 
@@ -193,7 +194,7 @@ Status HierarchicalDataReader::_process_nested_columns(
     // into a new object column and wrap it with array column using the first 
element offsets.The wrapped array column
     // will type the type of ColumnObject::NESTED_TYPE, whih is 
Nullable<ColumnArray<NULLABLE(ColumnObject)>>.
     for (const auto& entry : nested_subcolumns) {
-        MutableColumnPtr nested_object = ColumnObject::create(true, false);
+        MutableColumnPtr nested_object = ColumnObject::create(true);
         const auto* base_array =
                 
check_and_get_column<ColumnArray>(remove_nullable(entry.second[0].column));
         MutableColumnPtr offset = 
base_array->get_offsets_ptr()->assume_mutable();
@@ -245,10 +246,8 @@ Status HierarchicalDataReader::_process_nested_columns(
 Status HierarchicalDataReader::_init_container(vectorized::MutableColumnPtr& 
container,
                                                size_t nrows) {
     using namespace vectorized;
-    // build variant as container
-    container = ColumnObject::create(true, false);
-    auto& container_variant = assert_cast<ColumnObject&>(*container);
 
+    // build variant as container
     // add root first
     if (_path.get_parts().empty() && _root_reader) {
         // auto& root_var =
@@ -259,16 +258,28 @@ Status 
HierarchicalDataReader::_init_container(vectorized::MutableColumnPtr& con
         //                 : 
assert_cast<vectorized::ColumnObject&>(*_root_reader->column);
         // auto column = root_var.get_root();
         // auto type = root_var.get_root_type();
+
         MutableColumnPtr column = _root_reader->column->get_ptr();
-        container_variant.add_sub_column({}, std::move(column), 
_root_reader->type);
+        // container_variant.add_sub_column({}, std::move(column), 
_root_reader->type);
+        DCHECK(column->size() == nrows);
+        container = ColumnObject::create(_root_reader->type, 
std::move(column));
+    } else {
+        auto root_type =
+                
vectorized::DataTypeFactory::instance().create_data_type(TypeIndex::Nothing, 
true);
+        MutableColumnPtr column = root_type->create_column();
+        column->insert_many_defaults(nrows);
+        container = ColumnObject::create(root_type, std::move(column));
     }
+
+    auto& container_variant = assert_cast<ColumnObject&>(*container);
+
     // parent path -> subcolumns
     std::map<PathInData, PathsWithColumnAndType> nested_subcolumns;
     PathsWithColumnAndType non_nested_subcolumns;
     RETURN_IF_ERROR(tranverse([&](SubstreamReaderTree::Node& node) {
         MutableColumnPtr column = node.data.column->get_ptr();
         PathInData relative_path = 
node.path.copy_pop_nfront(_path.get_parts().size());
-
+        DCHECK(column->size() == nrows);
         if (node.path.has_nested_part()) {
             
CHECK_EQ(getTypeName(remove_nullable(node.data.type)->get_type_id()),
                      getTypeName(TypeIndex::Array));
@@ -301,6 +312,7 @@ static std::string_view get_sub_path(const 
std::string_view& path, const std::st
 Status 
HierarchicalDataReader::_process_sparse_column(vectorized::ColumnObject& 
container_variant,
                                                       size_t nrows) {
     using namespace vectorized;
+    container_variant.clear_sparse_column();
     if (!_sparse_column_reader) {
         
container_variant.get_sparse_column()->assume_mutable()->insert_many_defaults(nrows);
         return Status::OK();
diff --git a/be/src/olap/rowset/segment_v2/variant_column_writer_impl.cpp 
b/be/src/olap/rowset/segment_v2/variant_column_writer_impl.cpp
index 7f35364b172..a26d3d4eab8 100644
--- a/be/src/olap/rowset/segment_v2/variant_column_writer_impl.cpp
+++ b/be/src/olap/rowset/segment_v2/variant_column_writer_impl.cpp
@@ -47,16 +47,12 @@ Status VariantColumnWriterImpl::init() {
     // caculate stats info
     std::set<std::string> subcolumn_paths;
     RETURN_IF_ERROR(_get_subcolumn_paths_from_stats(subcolumn_paths));
-    if (subcolumn_paths.empty()) {
-        _column = vectorized::ColumnObject::create(true, false);
-    } else {
-        // create root
-        auto col = vectorized::ColumnObject::create(true, true);
-        for (const auto& str_path : subcolumn_paths) {
-            DCHECK(col->add_sub_column(vectorized::PathInData(str_path), 0));
-        }
-        _column = std::move(col);
+
+    auto col = vectorized::ColumnObject::create(true);
+    for (const auto& str_path : subcolumn_paths) {
+        DCHECK(col->add_sub_column(vectorized::PathInData(str_path), 0));
     }
+    _column = std::move(col);
     if (_tablet_column->is_nullable()) {
         _null_column = vectorized::ColumnUInt8::create(0);
     }
@@ -91,6 +87,9 @@ Status 
VariantColumnWriterImpl::_get_subcolumn_paths_from_stats(std::set<std::st
                 it->second += size;
             }
             for (const auto& [path, size] : 
source_statistics->sparse_column_non_null_size) {
+                if (path.empty()) {
+                    CHECK(false);
+                }
                 auto it = path_to_total_number_of_non_null_values.find(path);
                 if (it == path_to_total_number_of_non_null_values.end()) {
                     it = path_to_total_number_of_non_null_values.emplace(path, 
0).first;
@@ -112,7 +111,7 @@ Status 
VariantColumnWriterImpl::_get_subcolumn_paths_from_stats(std::set<std::st
         // Fill subcolumn_paths with first subcolumn paths in sorted list.
         // reserve 1 for root column
         for (const auto& [size, path] : paths_with_sizes) {
-            if (paths.size() < config::variant_max_subcolumns_count - 1) {
+            if (paths.size() < config::variant_max_subcolumns_count) {
                 VLOG_DEBUG << "pick " << path << " as subcolumn";
                 paths.emplace(path);
             }
@@ -215,6 +214,11 @@ Status 
VariantColumnWriterImpl::_process_subcolumns(vectorized::ColumnObject* pt
     // convert sub column data from engine format to storage layer format
     for (const auto& entry :
          
vectorized::schema_util::get_sorted_subcolumns(ptr->get_subcolumns())) {
+        const auto& least_common_type = entry->data.get_least_common_type();
+        if (is_nothing(remove_nullable(
+                    
vectorized::schema_util::get_base_type_of_array(least_common_type)))) {
+            continue;
+        }
         if (entry->path.empty()) {
             // already handled
             continue;
@@ -328,13 +332,14 @@ Status VariantColumnWriterImpl::finalize() {
 
     DCHECK(ptr->is_finalized());
 
-    if (ptr->is_null_root()) {
-        auto root_type = vectorized::make_nullable(
-                std::make_shared<vectorized::ColumnObject::MostCommonType>());
-        auto root_col = root_type->create_column();
-        root_col->insert_many_defaults(ptr->rows());
-        ptr->create_root(root_type, std::move(root_col));
-    }
+    // if (ptr->is_null_root()) {
+    //     CHECK(false);
+    //     // auto root_type = vectorized::make_nullable(
+    //     //         
std::make_shared<vectorized::ColumnObject::MostCommonType>());
+    //     // auto root_col = root_type->create_column();
+    //     // root_col->insert_many_defaults(ptr->rows());
+    //     // ptr->create_root(root_type, std::move(root_col));
+    // }
 
 #ifndef NDEBUG
     ptr->check_consistency();
diff --git a/be/src/vec/columns/column_object.cpp 
b/be/src/vec/columns/column_object.cpp
index d6d5b5c44cc..9582f57aa41 100644
--- a/be/src/vec/columns/column_object.cpp
+++ b/be/src/vec/columns/column_object.cpp
@@ -352,7 +352,7 @@ void get_field_info(const Field& field, FieldInfo* info) {
 }
 
 #ifdef NDEBUG
-#define ENABLE_CHECK_CONSISTENCY (void) /* Nothing */
+#define ENABLE_CHECK_CONSISTENCY (void)/* Nothing */
 #else
 #define ENABLE_CHECK_CONSISTENCY(this) (this)->check_consistency()
 #endif
@@ -360,12 +360,21 @@ void get_field_info(const Field& field, FieldInfo* info) {
 // current nested level is 2, inside column object
 constexpr int CURRENT_SERIALIZE_NESTING_LEVEL = 2;
 
+DataTypeSerDeSPtr ColumnObject::Subcolumn::generate_data_serdes(DataTypePtr 
type, bool is_root) {
+    // For the root column, there is no path, so there is no need to add extra 
'"'
+    if (is_root) {
+        return type->get_serde(CURRENT_SERIALIZE_NESTING_LEVEL - 1);
+    } else {
+        return type->get_serde(CURRENT_SERIALIZE_NESTING_LEVEL);
+    }
+}
+
 ColumnObject::Subcolumn::Subcolumn(MutableColumnPtr&& data_, DataTypePtr type, 
bool is_nullable_,
                                    bool is_root_)
         : least_common_type(type), is_nullable(is_nullable_), 
is_root(is_root_) {
     data.push_back(std::move(data_));
     data_types.push_back(type);
-    data_serdes.push_back(type->get_serde(CURRENT_SERIALIZE_NESTING_LEVEL));
+    data_serdes.push_back(generate_data_serdes(type, is_root));
 }
 
 ColumnObject::Subcolumn::Subcolumn(size_t size_, bool is_nullable_, bool 
is_root_)
@@ -406,9 +415,9 @@ void ColumnObject::Subcolumn::insert(Field field) {
 
 void ColumnObject::Subcolumn::add_new_column_part(DataTypePtr type) {
     data.push_back(type->create_column());
-    least_common_type = LeastCommonType {type};
+    least_common_type = LeastCommonType {type, is_root};
     data_types.push_back(type);
-    data_serdes.push_back(type->get_serde(CURRENT_SERIALIZE_NESTING_LEVEL));
+    data_serdes.push_back(generate_data_serdes(type, is_root));
 }
 
 void ColumnObject::Subcolumn::insert(Field field, FieldInfo info) {
@@ -512,8 +521,8 @@ static ColumnPtr recreate_column_with_default_values(const 
ColumnPtr& column, Ty
 ColumnObject::Subcolumn ColumnObject::Subcolumn::clone_with_default_values(
         const FieldInfo& field_info) const {
     Subcolumn new_subcolumn(*this);
-    new_subcolumn.least_common_type =
-            LeastCommonType {create_array(field_info.scalar_type_id, 
field_info.num_dimensions)};
+    new_subcolumn.least_common_type = LeastCommonType {
+            create_array(field_info.scalar_type_id, 
field_info.num_dimensions), is_root};
 
     for (int i = 0; i < new_subcolumn.data.size(); ++i) {
         new_subcolumn.data[i] = recreate_column_with_default_values(
@@ -633,8 +642,12 @@ MutableColumnPtr ColumnObject::apply_for_columns(Func&& 
func) const {
         auto& finalized_object = assert_cast<ColumnObject&>(*finalized);
         return finalized_object.apply_for_columns(std::forward<Func>(func));
     }
-    auto res = ColumnObject::create(is_nullable, false);
+    auto new_root = func(get_root())->assume_mutable();
+    auto res = ColumnObject::create(get_root_type(), std::move(new_root));
     for (const auto& subcolumn : subcolumns) {
+        if (subcolumn->data.is_root) {
+            continue;
+        }
         auto new_subcolumn = func(subcolumn->data.get_finalized_column_ptr());
         res->add_sub_column(subcolumn->path, new_subcolumn->assume_mutable(),
                             subcolumn->data.get_least_common_type());
@@ -663,10 +676,7 @@ void ColumnObject::resize(size_t n) {
 }
 
 void ColumnObject::Subcolumn::finalize(FinalizeMode mode) {
-    if (is_finalized()) {
-        return;
-    }
-    if (data.size() == 1 && num_of_defaults_in_prefix == 0) {
+    if (!is_root && data.size() == 1 && num_of_defaults_in_prefix == 0) {
         data[0] = data[0]->convert_to_full_column_if_const();
         return;
     }
@@ -675,7 +685,7 @@ void ColumnObject::Subcolumn::finalize(FinalizeMode mode) {
         // Root always JSONB type in write mode
         to_type = is_nullable ? 
make_nullable(std::make_shared<MostCommonType>())
                               : std::make_shared<MostCommonType>();
-        least_common_type = LeastCommonType {to_type};
+        least_common_type = LeastCommonType {to_type, is_root};
     }
     auto result_column = to_type->create_column();
     if (num_of_defaults_in_prefix) {
@@ -698,7 +708,8 @@ void ColumnObject::Subcolumn::finalize(FinalizeMode mode) {
     }
     data = {std::move(result_column)};
     data_types = {std::move(to_type)};
-    data_serdes = {data_types[0]->get_serde(CURRENT_SERIALIZE_NESTING_LEVEL)};
+    data_serdes = {(generate_data_serdes(data_types[0], is_root))};
+
     num_of_defaults_in_prefix = 0;
 }
 
@@ -779,11 +790,12 @@ void ColumnObject::Subcolumn::remove_nullable() {
     least_common_type.remove_nullable();
 }
 
-ColumnObject::Subcolumn::LeastCommonType::LeastCommonType(DataTypePtr type_)
+ColumnObject::Subcolumn::LeastCommonType::LeastCommonType(DataTypePtr type_, 
bool is_root)
         : type(std::move(type_)),
           base_type(get_base_type_of_array(type)),
           num_dimensions(get_number_of_dimensions(*type)) {
-    least_common_type_serder = 
type->get_serde(CURRENT_SERIALIZE_NESTING_LEVEL);
+    least_common_type_serder = Subcolumn::generate_data_serdes(type, is_root);
+
     type_id = type->is_nullable() ? assert_cast<const 
DataTypeNullable*>(type.get())
                                             ->get_nested_type()
                                             ->get_type_id()
@@ -794,34 +806,35 @@ 
ColumnObject::Subcolumn::LeastCommonType::LeastCommonType(DataTypePtr type_)
                                             : base_type->get_type_id();
 }
 
-ColumnObject::ColumnObject(bool is_nullable_, bool create_root_)
-        : is_nullable(is_nullable_), num_rows(0) {
-    if (create_root_) {
-        subcolumns.create_root(Subcolumn(0, is_nullable, true /*root*/));
-    }
+ColumnObject::ColumnObject(bool is_nullable_) : is_nullable(is_nullable_), 
num_rows(0) {
+    subcolumns.create_root(Subcolumn(0, is_nullable, true /*root*/));
     ENABLE_CHECK_CONSISTENCY(this);
 }
 
-ColumnObject::ColumnObject(bool is_nullable_, DataTypePtr type, 
MutableColumnPtr&& column)
-        : is_nullable(is_nullable_), num_rows(0) {
-    add_sub_column({}, std::move(column), type);
+ColumnObject::ColumnObject(DataTypePtr root_type, MutableColumnPtr&& 
root_column)
+        : is_nullable(true), num_rows(root_column->size()) {
+    subcolumns.create_root(
+            Subcolumn(std::move(root_column), root_type, is_nullable, true 
/*root*/));
     serialized_sparse_column->insert_many_defaults(num_rows);
     ENABLE_CHECK_CONSISTENCY(this);
 }
 
-ColumnObject::ColumnObject(Subcolumns&& subcolumns_, bool is_nullable_)
-        : is_nullable(is_nullable_),
+ColumnObject::ColumnObject(Subcolumns&& subcolumns_)
+        : is_nullable(true),
           subcolumns(std::move(subcolumns_)),
           num_rows(subcolumns.empty() ? 0 : 
(*subcolumns.begin())->data.size()) {
+    serialized_sparse_column->insert_many_defaults(num_rows);
     ENABLE_CHECK_CONSISTENCY(this);
 }
 
 ColumnObject::ColumnObject(size_t size) : is_nullable(true), num_rows(0) {
+    subcolumns.create_root(Subcolumn(0, is_nullable, true /*root*/));
     insert_many_defaults(size);
     ENABLE_CHECK_CONSISTENCY(this);
 }
 
 void ColumnObject::check_consistency() const {
+    CHECK(subcolumns.get_root() != nullptr);
     for (const auto& leaf : subcolumns) {
         if (num_rows != leaf->data.size()) {
             throw doris::Exception(doris::ErrorCode::INTERNAL_ERROR,
@@ -834,6 +847,32 @@ void ColumnObject::check_consistency() const {
                                "unmatched sparse column:, expeted rows: {}, 
but meet: {}", num_rows,
                                serialized_sparse_column->size());
     }
+
+#ifdef NDEBUG
+    bool error = false;
+    auto [path, value] = get_sparse_data_paths_and_values();
+
+    auto& offsets = serialized_sparse_column_offsets();
+    for (size_t row = 0; row != num_rows; ++row) {
+        size_t offset = offsets[row - 1];
+        size_t end = offsets[row];
+        // Iterator over [path, binary value]
+        for (size_t i = offset; i != end; ++i) {
+            const StringRef sparse_path_string = path->get_data_at(i);
+            const std::string_view sparse_path(sparse_path_string);
+
+            const PathInData column_path(sparse_path);
+            if (auto* subcolumn = get_subcolumn(column_path); subcolumn != 
nullptr) {
+                LOG(WARNING) << "err path: " << sparse_path;
+                error = true;
+            }
+        }
+    }
+    if (error) {
+        throw doris::Exception(doris::ErrorCode::INTERNAL_ERROR,
+                               "path {} both exists in subcolumn and sparse 
columns");
+    }
+#endif
 }
 
 size_t ColumnObject::size() const {
@@ -895,6 +934,7 @@ void ColumnObject::insert_from(const IColumn& src, size_t 
n) {
 }
 
 void ColumnObject::try_insert(const Field& field) {
+    size_t old_size = size();
     if (field.get_type() != Field::Types::VariantMap) {
         if (field.is_null()) {
             insert_default();
@@ -902,20 +942,12 @@ void ColumnObject::try_insert(const Field& field) {
             return;
         }
         auto* root = get_subcolumn({});
-        // Insert to an emtpy ColumnObject may result root null,
-        // so create a root column of Variant is expected.
         if (root == nullptr) {
-            bool succ = add_sub_column({}, num_rows);
-            if (!succ) {
-                throw doris::Exception(doris::ErrorCode::INVALID_ARGUMENT,
-                                       "Failed to add root sub column {}");
-            }
-            root = get_subcolumn({});
+            CHECK(false) << "column object has no root column";
         }
         root->insert(field);
     } else {
         const auto& object = field.get<const VariantMap&>();
-        size_t old_size = size();
         for (const auto& [key_str, value] : object) {
             PathInData key;
             if (!key_str.empty()) {
@@ -935,12 +967,12 @@ void ColumnObject::try_insert(const Field& field) {
             }
             subcolumn->insert(value);
         }
-        for (auto& entry : subcolumns) {
-            if (old_size == entry->data.size()) {
-                bool inserted = try_insert_default_from_nested(entry);
-                if (!inserted) {
-                    entry->data.insert_default();
-                }
+    }
+    for (auto& entry : subcolumns) {
+        if (old_size == entry->data.size()) {
+            bool inserted = try_insert_default_from_nested(entry);
+            if (!inserted) {
+                entry->data.insert_default();
             }
         }
     }
@@ -1053,8 +1085,8 @@ void 
ColumnObject::Subcolumn::serialize_to_sparse_column(ColumnString* key, std:
                 key->insert_data(path.data(), path.size());
 
                 // every subcolumn is always Nullable
-                auto nullable_serde = 
std::static_pointer_cast<DataTypeNullableSerDe>(
-                        
data_types[i]->get_serde(CURRENT_SERIALIZE_NESTING_LEVEL));
+                auto nullable_serde =
+                        
std::static_pointer_cast<DataTypeNullableSerDe>(data_serdes[i]);
                 auto& nullable_col =
                         assert_cast<const ColumnNullable&, 
TypeCheckOnRelease::DISABLE>(*part);
 
@@ -1262,10 +1294,19 @@ void ColumnObject::add_nested_subcolumn(const 
PathInData& key, const FieldInfo&
     ENABLE_CHECK_CONSISTENCY(this);
 }
 
+void ColumnObject::set_num_rows(size_t n) {
+    num_rows = n;
+}
+
 bool ColumnObject::try_add_new_subcolumn(const PathInData& path) {
-    if (subcolumns.size() == config::variant_max_subcolumns_count) return 
false;
+    if (subcolumns.get_root() == nullptr || path.empty()) {
+        throw Exception(ErrorCode::INTERNAL_ERROR, "column object has no root 
or path is empty");
+    }
+    if (subcolumns.size() < config::variant_max_subcolumns_count + 1) {
+        return add_sub_column(path, num_rows);
+    }
 
-    return add_sub_column(path, num_rows);
+    return false;
 }
 
 void ColumnObject::insert_range_from(const IColumn& src, size_t start, size_t 
length) {
@@ -1296,7 +1337,6 @@ void ColumnObject::insert_range_from(const IColumn& src, 
size_t start, size_t le
     auto it = src_path_and_subcoumn_for_sparse_column.begin();
     auto end = src_path_and_subcoumn_for_sparse_column.end();
     while (it != end) {
-        VLOG_DEBUG << "pick " << it->first << " as sparse column";
         sorted_src_subcolumn_for_sparse_column.emplace_back(it->first, 
it->second);
         ++it;
     }
@@ -1524,7 +1564,8 @@ bool ColumnObject::add_sub_column(const PathInData& key, 
MutableColumnPtr&& subc
     }
     if (key.empty() &&
         (!subcolumns.get_root()->is_scalar() ||
-         (is_null_root() || 
is_nothing(subcolumns.get_root()->data.get_least_common_type())))) {
+         (is_null_root() ||
+          
is_nothing(remove_nullable(subcolumns.get_root()->data.get_least_common_type())))))
 {
         bool root_it_scalar = subcolumns.get_root()->is_scalar();
         // update root to scalar
         subcolumns.get_mutable_root()->modify_to_scalar(
@@ -1600,7 +1641,7 @@ void ColumnObject::Subcolumn::wrapp_array_nullable() {
         }
         result_column = ColumnNullable::create(std::move(result_column), 
std::move(new_null_map));
         data_types[0] = make_nullable(data_types[0]);
-        least_common_type = LeastCommonType {data_types[0]};
+        least_common_type = LeastCommonType {data_types[0], is_root};
     }
 }
 
@@ -1609,7 +1650,6 @@ Status ColumnObject::serialize_one_row_to_string(int64_t 
row, std::string* outpu
     VectorBufferWriter write_buffer(*tmp_col.get());
     if (is_scalar_variant()) {
         subcolumns.get_root()->data.serialize_text_json(row, write_buffer);
-        return Status::OK();
     } else {
         // TODO preallocate memory
         RETURN_IF_ERROR(serialize_one_row_to_json_format(row, write_buffer, 
nullptr));
@@ -1894,24 +1934,41 @@ void ColumnObject::unnest(Subcolumns::NodePtr& entry, 
Subcolumns& subcolumns) co
     }
 }
 
+void ColumnObject::clear_sparse_column() {
+#ifndef NDEBUG
+    auto& sparse_column_offsets = serialized_sparse_column_offsets();
+    if (sparse_column_offsets[num_rows - 1] != 0) {
+        throw Exception(ErrorCode::INTERNAL_ERROR, "sprase column has nonnull 
value");
+    }
+#endif
+
+    serialized_sparse_column->clear();
+}
+
 Status ColumnObject::finalize(FinalizeMode mode) {
     Subcolumns new_subcolumns;
 
-    // finalize root first
-    if (!is_null_root()) {
-        new_subcolumns.create_root(subcolumns.get_root()->data);
-        new_subcolumns.get_mutable_root()->data.finalize(mode);
-    } else if (mode == FinalizeMode::WRITE_MODE) {
-        new_subcolumns.create_root(Subcolumn(num_rows, is_nullable, true));
-        new_subcolumns.get_mutable_root()->data.finalize(mode);
+    if (auto root = subcolumns.get_mutable_root(); root == nullptr) {
+        CHECK(false);
+    } else {
+        root->data.finalize(mode);
+        new_subcolumns.create_root(root->data);
     }
 
-    const bool need_pick_subcolumn_to_sparse_column =
+    // 1. only write mode need to pick subcolumns to sparse column
+    // 2. root column must be exsit in subcolumns
+    bool need_pick_subcolumn_to_sparse_column =
             mode == FinalizeMode::WRITE_MODE &&
-            subcolumns.size() > config::variant_max_subcolumns_count;
+            subcolumns.size() > config::variant_max_subcolumns_count + 1;
+
     // finalize all subcolumns
     for (auto&& entry : subcolumns) {
+        if (entry->data.is_root) {
+            continue;
+        }
+
         const auto& least_common_type = entry->data.get_least_common_type();
+
         // unnest all nested columns, add them to new_subcolumns
         if (mode == FinalizeMode::WRITE_MODE &&
             least_common_type->equals(*ColumnObject::NESTED_TYPE)) {
@@ -1919,9 +1976,6 @@ Status ColumnObject::finalize(FinalizeMode mode) {
             continue;
         }
 
-        if (entry->data.is_root) {
-            continue;
-        }
         entry->data.finalize(mode);
         entry->data.wrapp_array_nullable();
 
@@ -1938,10 +1992,14 @@ Status ColumnObject::finalize(FinalizeMode mode) {
         std::unordered_map<std::string_view, size_t> none_null_value_sizes;
         // 1. get the none null value sizes
         for (auto&& entry : subcolumns) {
+            // root column is already in the subcolumns
             if (entry->data.is_root) {
                 continue;
             }
             size_t size = entry->data.get_non_null_value_size();
+            if (size == 0) {
+                continue;
+            }
             none_null_value_sizes[entry->path.get_path()] = size;
         }
         // 2. sort by the size
@@ -1965,12 +2023,16 @@ Status ColumnObject::finalize(FinalizeMode mode) {
         for (auto&& entry : subcolumns) {
             if (selected_path.find(entry->path.get_path()) != 
selected_path.end()) {
                 new_subcolumns.add(entry->path, entry->data);
-            } else {
+            } else if (none_null_value_sizes.find(entry->path.get_path()) !=
+                       none_null_value_sizes.end()) {
                 VLOG_DEBUG << "pick " << entry->path.get_path() << " as sparse 
column";
                 remaing_subcolumns.emplace(entry->path.get_path(), 
entry->data);
             }
         }
-        serialized_sparse_column->clear();
+
+        ENABLE_CHECK_CONSISTENCY(this);
+        clear_sparse_column();
+
         
RETURN_IF_ERROR(serialize_sparse_columns(std::move(remaing_subcolumns)));
     }
 
@@ -1996,7 +2058,7 @@ void ColumnObject::ensure_root_node_type(const 
DataTypePtr& expected_root_type)
                                          expected_root_type, &casted_column));
         root.data[0] = casted_column;
         root.data_types[0] = expected_root_type;
-        root.least_common_type = Subcolumn::LeastCommonType 
{expected_root_type};
+        root.least_common_type = Subcolumn::LeastCommonType 
{expected_root_type, true};
     }
 }
 
@@ -2022,8 +2084,12 @@ ColumnPtr ColumnObject::filter(const Filter& filter, 
ssize_t count) const {
         ENABLE_CHECK_CONSISTENCY(res.get());
         return res;
     }
-    auto new_column = ColumnObject::create(true, false);
+    auto new_root = get_root()->filter(filter, count)->assume_mutable();
+    auto new_column = ColumnObject::create(get_root_type(), 
std::move(new_root));
     for (auto& entry : subcolumns) {
+        if (entry->data.is_root) {
+            continue;
+        }
         auto subcolumn = entry->data.get_finalized_column().filter(filter, -1);
         new_column->add_sub_column(entry->path, subcolumn->assume_mutable(),
                                    entry->data.get_least_common_type());
@@ -2083,6 +2149,8 @@ void ColumnObject::clear_column_data() {
 
 void ColumnObject::clear() {
     Subcolumns empty;
+    // we must keep root column exist
+    empty.create_root(Subcolumn(0, is_nullable, true));
     std::swap(empty, subcolumns);
     serialized_sparse_column->clear();
     num_rows = 0;
diff --git a/be/src/vec/columns/column_object.h 
b/be/src/vec/columns/column_object.h
index b3f2679dbe6..05e2693ed71 100644
--- a/be/src/vec/columns/column_object.h
+++ b/be/src/vec/columns/column_object.h
@@ -188,6 +188,8 @@ public:
         void serialize_to_sparse_column(ColumnString* key, std::string_view 
path,
                                         ColumnString* value, size_t row);
 
+        static DataTypeSerDeSPtr generate_data_serdes(DataTypePtr type, bool 
is_root = false);
+
         friend class ColumnObject;
 
     private:
@@ -195,7 +197,7 @@ public:
         public:
             LeastCommonType() = default;
 
-            explicit LeastCommonType(DataTypePtr type_);
+            explicit LeastCommonType(DataTypePtr type_, bool is_root = false);
 
             const DataTypePtr& get() const { return type; }
 
@@ -269,14 +271,15 @@ private:
 public:
     static constexpr auto COLUMN_NAME_DUMMY = "_dummy";
 
-    explicit ColumnObject(bool is_nullable_, bool create_root = true);
-
-    explicit ColumnObject(bool is_nullable_, DataTypePtr type, 
MutableColumnPtr&& column);
+    // always create root: data type nothing
+    explicit ColumnObject(bool is_nullable_);
 
-    // create without root, num_rows = size
+    // always create root: data type nothing
     explicit ColumnObject(size_t size = 0);
 
-    ColumnObject(Subcolumns&& subcolumns_, bool is_nullable_);
+    explicit ColumnObject(DataTypePtr root_type, MutableColumnPtr&& 
root_column);
+
+    explicit ColumnObject(Subcolumns&& subcolumns_);
 
     ~ColumnObject() override = default;
 
@@ -309,6 +312,8 @@ public:
 
     static const DataTypePtr& get_most_common_type();
 
+    void clear_sparse_column();
+
     // root is null or type nothing
     bool is_null_root() const;
 
@@ -337,7 +342,7 @@ public:
 
     void incr_num_rows(size_t n) { num_rows += n; }
 
-    void set_num_rows(size_t n) { num_rows = n; }
+    void set_num_rows(size_t n);
 
     size_t rows() const { return num_rows; }
 
diff --git a/be/src/vec/common/schema_util.cpp 
b/be/src/vec/common/schema_util.cpp
index 74298d2c838..fdae135cc81 100644
--- a/be/src/vec/common/schema_util.cpp
+++ b/be/src/vec/common/schema_util.cpp
@@ -231,8 +231,9 @@ void get_column_by_type(const vectorized::DataTypePtr& 
data_type, const std::str
         return;
     }
     // TODO handle more types like struct/date/datetime/decimal...
-    LOG(FATAL) << "__builtin_unreachable";
-    __builtin_unreachable();
+    throw doris::Exception(doris::ErrorCode::INTERNAL_ERROR,
+                           "unexcepted data column type: {}, column name is: 
{}",
+                           data_type->get_name(), name);
 }
 
 TabletColumn get_column_by_type(const vectorized::DataTypePtr& data_type, 
const std::string& name,
diff --git a/be/src/vec/functions/function_variant_element.cpp 
b/be/src/vec/functions/function_variant_element.cpp
index e04ff54dbe9..d549c36595f 100644
--- a/be/src/vec/functions/function_variant_element.cpp
+++ b/be/src/vec/functions/function_variant_element.cpp
@@ -151,7 +151,7 @@ private:
                     result_column->insert_default();
                 }
             }
-            *result = ColumnObject::create(true, type, 
std::move(result_column));
+            *result = ColumnObject::create(type, std::move(result_column));
             (*result)->assume_mutable()->finalize();
             return Status::OK();
         } else {
@@ -162,8 +162,7 @@ private:
             const auto* node = subcolumns.find_exact(path);
             MutableColumnPtr result_col;
             if (node != nullptr) {
-                // Create without root, since root will be added
-                result_col = ColumnObject::create(true, false /*should not 
create root*/);
+                result_col = ColumnObject::create(true);
                 std::vector<decltype(node)> nodes;
                 PathsInData paths;
                 ColumnObject::Subcolumns::get_leaves_of_node(node, nodes, 
paths);
@@ -186,10 +185,10 @@ private:
                             
nodes[0]->data.get_finalized_column_ptr()->assume_mutable(),
                             nodes[0]->data.get_least_common_type(), true, 
true});
                 }
-                auto container = 
ColumnObject::create(std::move(new_subcolumns), true);
+                auto container = 
ColumnObject::create(std::move(new_subcolumns));
                 result_col->insert_range_from(*container, 0, 
container->size());
             } else {
-                // Create with root, otherwise the root type maybe type Nothing
+                // Create with root, otherwise the root type maybe type 
Nothing ?
                 result_col = ColumnObject::create(true);
                 result_col->insert_many_defaults(src.size());
             }
diff --git a/be/test/vec/columns/column_object_test.cpp 
b/be/test/vec/columns/column_object_test.cpp
new file mode 100644
index 00000000000..e59219827db
--- /dev/null
+++ b/be/test/vec/columns/column_object_test.cpp
@@ -0,0 +1,795 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "vec/columns/column_object.h"
+
+#include <gmock/gmock-more-matchers.h>
+#include <gtest/gtest.h>
+#include <rapidjson/prettywriter.h>
+#include <stdio.h>
+
+#include "runtime/jsonb_value.h"
+#include "vec/common/string_ref.h"
+#include "vec/data_types/data_type_array.h"
+#include "vec/data_types/data_type_factory.hpp"
+
+using namespace doris::vectorized;
+
+using namespace doris;
+// #define ADD_SUB_COLUMN(key) \
+//    varaint->add_sub_column(PathInData(std::string_view(key)), 0);
+
+doris::vectorized::Field construct_variant_map(
+        const std::vector<std::pair<std::string, doris::vectorized::Field>>& 
key_and_values) {
+    doris::vectorized::Field res = VariantMap();
+    auto& object = res.get<VariantMap&>();
+    for (const auto& [k, v] : key_and_values) {
+        object.try_emplace(k, v);
+    }
+    return res;
+}
+
+auto construct_basic_varint_column() {
+    // 1. create an empty variant column
+    auto variant = ColumnObject::create();
+
+    std::vector<std::pair<std::string, doris::vectorized::Field>> data;
+
+    // 2. subcolumn path
+    data.emplace_back("v.a", 20);
+    data.emplace_back("v.b", "20");
+    data.emplace_back("v.c", 20);
+    data.emplace_back("v.f", 20);
+    data.emplace_back("v.e", "50");
+    for (int i = 0; i < 5; ++i) {
+        auto field = construct_variant_map(data);
+        variant->try_insert(field);
+    }
+
+    // 3. sparse column path
+    data.emplace_back("v.d.d", "50");
+    data.emplace_back("v.c.d", 30);
+    data.emplace_back("v.b.d", 30);
+    for (int i = 0; i < 5; ++i) {
+        auto field = construct_variant_map(data);
+        variant->try_insert(field);
+    }
+    return variant;
+}
+
+auto construct_dst_varint_column() {
+    // 1. create an empty variant column
+    vectorized::ColumnObject::Subcolumns dynamic_subcolumns;
+    dynamic_subcolumns.create_root(vectorized::ColumnObject::Subcolumn(0, 
true, true /*root*/));
+    dynamic_subcolumns.add(vectorized::PathInData("v.f"),
+                           vectorized::ColumnObject::Subcolumn {0, true});
+    dynamic_subcolumns.add(vectorized::PathInData("v.e"),
+                           vectorized::ColumnObject::Subcolumn {0, true});
+    dynamic_subcolumns.add(vectorized::PathInData("v.b"),
+                           vectorized::ColumnObject::Subcolumn {0, true});
+    dynamic_subcolumns.add(vectorized::PathInData("v.b.d"),
+                           vectorized::ColumnObject::Subcolumn {0, true});
+    dynamic_subcolumns.add(vectorized::PathInData("v.c.d"),
+                           vectorized::ColumnObject::Subcolumn {0, true});
+    return ColumnObject::create(std::move(dynamic_subcolumns));
+}
+
+TEST(ColumnVariantTest, basic_finalize) {
+    auto variant = construct_basic_varint_column();
+    // 4. finalize
+    
EXPECT_TRUE(variant->finalize(ColumnObject::FinalizeMode::WRITE_MODE).ok());
+    EXPECT_EQ(variant->size(), 10);
+
+    // check finalized subcolumn
+    // 5 subcolumn + 1 root
+    EXPECT_EQ(variant->subcolumns.size(), 6);
+    for (const auto& column : variant->subcolumns) {
+        if (column->data.is_root) {
+            continue;
+        }
+        EXPECT_EQ(column->data.data.size(), 1);
+    }
+
+    // check sparse column
+    const auto& offsets = variant->serialized_sparse_column_offsets();
+    for (int row = 0; row < 5; ++row) {
+        EXPECT_EQ(offsets[row], 0);
+    }
+    for (int row = 5; row < 10; ++row) {
+        EXPECT_EQ(offsets[row] - offsets[row - 1], 3);
+    }
+}
+
+TEST(ColumnVariantTest, basic_deserialize) {
+    auto variant = construct_basic_varint_column();
+
+    // 4. finalize
+    
EXPECT_TRUE(variant->finalize(ColumnObject::FinalizeMode::WRITE_MODE).ok());
+    EXPECT_EQ(variant->size(), 10);
+
+    const auto& [path, value] = variant->get_sparse_data_paths_and_values();
+    const auto& offsets = variant->serialized_sparse_column_offsets();
+    for (size_t row = 5; row < 10; ++row) {
+        size_t start = offsets[row - 1];
+        size_t end = offsets[row];
+
+        auto data = path->get_data_at(start);
+        EXPECT_EQ(data, StringRef("v.b.d", 5));
+        auto pair = variant->deserialize_from_sparse_column(value, start++);
+        EXPECT_EQ(pair.first.get<Int64>(), 30);
+
+        auto data2 = path->get_data_at(start);
+        auto pair2 = variant->deserialize_from_sparse_column(value, start++);
+        EXPECT_EQ(data2, StringRef("v.c.d", 5));
+        EXPECT_EQ(pair2.first.get<Int64>(), 30);
+
+        auto data3 = path->get_data_at(start);
+        auto pair3 = variant->deserialize_from_sparse_column(value, start++);
+        EXPECT_EQ(data3, StringRef("v.d.d", 5));
+        EXPECT_EQ(pair3.first.get<String>(), "50");
+        EXPECT_EQ(start, end);
+    }
+}
+
+TEST(ColumnVariantTest, basic_inset_range_from) {
+    auto src = construct_basic_varint_column();
+    EXPECT_TRUE(src->finalize(ColumnObject::FinalizeMode::WRITE_MODE).ok());
+    EXPECT_EQ(src->size(), 10);
+
+    // dst is an empty column, has 5 subcolumn + 1 root
+    auto dst = construct_dst_varint_column();
+
+    // subcolumn->subcolumn          v.b v.f v.e
+    // subcolumn->sparse_column      v.a v.c
+    // sparse_column->subcolumn      v.b.d v.c.d
+    // sparse_column->sparse_column  v.d.d
+    dst->insert_range_from(*src, 0, 10);
+    EXPECT_EQ(dst->size(), 10);
+
+    // 5 subcolumn
+    EXPECT_EQ(dst->subcolumns.size(), 6);
+    ColumnObject::Subcolumns dst_subcolumns = dst->subcolumns;
+    std::sort(
+            dst_subcolumns.begin(), dst_subcolumns.end(),
+            [](const auto& lhsItem, const auto& rhsItem) { return 
lhsItem->path < rhsItem->path; });
+
+    for (const auto& column : dst_subcolumns) {
+        if (column->data.is_root) {
+            continue;
+        }
+        EXPECT_EQ(column->data.data.size(), 1);
+        EXPECT_EQ(column->data.data[0]->size(), 10);
+        if (column->path.get_path().size() == 3) {
+            EXPECT_EQ(column->data.get_non_null_value_size(), 10);
+        } else {
+            EXPECT_EQ(column->path.get_path().size(), 5);
+            EXPECT_EQ(column->data.get_non_null_value_size(), 5);
+            for (size_t row = 0; row != 5; ++row) {
+                EXPECT_TRUE(column->data.data[0]->is_null_at(row));
+            }
+            for (size_t row = 5; row != 10; ++row) {
+                EXPECT_EQ((*column->data.data[0])[row].get<Int64>(), 30);
+            }
+        }
+    }
+
+    // check sparse column
+    const auto& [path, value] = dst->get_sparse_data_paths_and_values();
+    const auto& offsets = dst->serialized_sparse_column_offsets();
+
+    // v.a v.c
+    for (int row = 0; row < 5; ++row) {
+        size_t start = offsets[row - 1];
+        size_t end = offsets[row];
+
+        auto data = path->get_data_at(start);
+        EXPECT_EQ(data, StringRef("v.a", 3));
+        auto pair = dst->deserialize_from_sparse_column(value, start++);
+        EXPECT_EQ(pair.first.get<Int64>(), 20);
+
+        auto data2 = path->get_data_at(start);
+        EXPECT_EQ(data2, StringRef("v.c", 3));
+        auto pair2 = dst->deserialize_from_sparse_column(value, start++);
+        EXPECT_EQ(pair2.first.get<Int64>(), 20);
+
+        EXPECT_EQ(start, end);
+    }
+
+    // v.a v.c v.d.d
+    for (int row = 5; row < 10; ++row) {
+        size_t start = offsets[row - 1];
+        size_t end = offsets[row];
+
+        auto data = path->get_data_at(start);
+        EXPECT_EQ(data, StringRef("v.a", 3));
+        auto pair = dst->deserialize_from_sparse_column(value, start++);
+        EXPECT_EQ(pair.first.get<Int64>(), 20);
+
+        auto data2 = path->get_data_at(start);
+        EXPECT_EQ(data2, StringRef("v.c", 3));
+        auto pair2 = dst->deserialize_from_sparse_column(value, start++);
+        EXPECT_EQ(pair2.first.get<Int64>(), 20);
+
+        auto data3 = path->get_data_at(start);
+        EXPECT_EQ(data3, StringRef("v.d.d", 5));
+        auto pair3 = dst->deserialize_from_sparse_column(value, start++);
+        EXPECT_EQ(pair3.first.get<String>(), "50");
+
+        EXPECT_EQ(start, end);
+    }
+}
+
+doris::vectorized::Field get_field(std::string_view type) {
+    static std::unordered_map<std::string_view, doris::vectorized::Field> 
field_map;
+    if (field_map.empty()) {
+        doris::vectorized::Field int_field = 20;
+        doris::vectorized::Field str_field(String("str", 3));
+        doris::vectorized::Field arr_int_field = Array();
+        auto& array1 = arr_int_field.get<Array>();
+        array1.emplace_back(int_field);
+        array1.emplace_back(int_field);
+        doris::vectorized::Field arr_str_field = Array();
+        auto& array2 = arr_str_field.get<Array>();
+        array2.emplace_back(str_field);
+        array2.emplace_back(str_field);
+        field_map["int"] = int_field;
+        field_map["string"] = str_field;
+        field_map["array_int"] = arr_int_field;
+        field_map["array_str"] = arr_str_field;
+    }
+    return field_map[type];
+}
+
+auto convert_to_jsonb_field(auto serde, auto& column) {
+    vectorized::DataTypeSerDe::FormatOptions options;
+    options.escape_char = '\\';
+    auto tmp_col = ColumnString::create();
+    VectorBufferWriter write_buffer(*tmp_col.get());
+    EXPECT_TRUE(serde->serialize_column_to_json(column, 0, 1, write_buffer, 
options).ok());
+
+    write_buffer.commit();
+    auto str_ref = tmp_col->get_data_at(0);
+    Slice data((char*)(str_ref.data), str_ref.size);
+
+    auto jsonb_type = 
doris::vectorized::DataTypeFactory::instance().create_data_type(
+            TypeIndex::JSONB, false);
+    auto jsonb_serde = jsonb_type->get_serde();
+    auto jsonb_column = jsonb_type->create_column();
+
+    DataTypeSerDe::FormatOptions format_options;
+    format_options.converted_from_string = true;
+    EXPECT_TRUE(
+            jsonb_serde->deserialize_one_cell_from_json(*jsonb_column, data, 
format_options).ok());
+    auto res = jsonb_column->get_data_at(0);
+    return JsonbField(res.data, res.size);
+}
+
+auto convert_string_to_jsonb_field(auto& column) {
+    auto str_ref = column.get_data_at(0);
+    Slice data((char*)(str_ref.data), str_ref.size);
+
+    auto jsonb_type = 
doris::vectorized::DataTypeFactory::instance().create_data_type(
+            TypeIndex::JSONB, false);
+    auto jsonb_serde = jsonb_type->get_serde();
+    auto jsonb_column = jsonb_type->create_column();
+    DataTypeSerDe::FormatOptions format_options;
+    format_options.converted_from_string = true;
+    format_options.escape_char = '\\';
+
+    EXPECT_TRUE(
+            jsonb_serde->deserialize_one_cell_from_json(*jsonb_column, data, 
format_options).ok());
+    auto res = jsonb_column->get_data_at(0);
+    return JsonbField(res.data, res.size);
+}
+
+doris::vectorized::Field get_jsonb_field(std::string_view type) {
+    static std::unordered_map<std::string_view, doris::vectorized::Field> 
field_map;
+    if (field_map.empty()) {
+        DataTypePtr data_type_int = 
doris::vectorized::DataTypeFactory::instance().create_data_type(
+                TypeIndex::Int8, false);
+        DataTypePtr data_type_array_int =
+                
std::make_shared<doris::vectorized::DataTypeArray>(data_type_int);
+        auto array_column_int = data_type_array_int->create_column();
+        array_column_int->insert(get_field("array_int"));
+        auto array_serde_int = data_type_array_int->get_serde();
+        field_map["array_int"] = convert_to_jsonb_field(array_serde_int, 
*array_column_int);
+
+        DataTypePtr data_type_str = 
doris::vectorized::DataTypeFactory::instance().create_data_type(
+                TypeIndex::String, false);
+        DataTypePtr data_type_array_str =
+                
std::make_shared<doris::vectorized::DataTypeArray>(data_type_str);
+        auto array_column_str = data_type_array_str->create_column();
+        array_column_str->insert(get_field("array_str"));
+        auto array_serde_str = data_type_array_str->get_serde();
+        field_map["array_str"] = convert_to_jsonb_field(array_serde_str, 
*array_column_str);
+
+        auto column_int = data_type_int->create_column();
+        column_int->insert(get_field("int"));
+        auto serde_int = data_type_int->get_serde();
+        field_map["int"] = convert_to_jsonb_field(serde_int, *column_int);
+
+        // auto column_str = data_type_str->create_column();
+        // column_str->insert(get_field("string"));
+        // field_map["string"] = convert_string_to_jsonb_field(*column_str);
+    }
+    return field_map[type];
+}
+
+auto construct_advanced_varint_column() {
+    // 1. create an empty variant column
+    auto variant = ColumnObject::create();
+
+    std::vector<std::pair<std::string, doris::vectorized::Field>> data;
+
+    // 2. subcolumn path
+    data.emplace_back("v.a", get_field("int"));
+    data.emplace_back("v.b", get_field("string"));
+    data.emplace_back("v.c", get_field("array_int"));
+    data.emplace_back("v.f", get_field("array_str"));
+    data.emplace_back("v.e", get_field("string"));
+
+    for (int i = 0; i < 5; ++i) {
+        auto field = construct_variant_map(data);
+        variant->try_insert(field);
+    }
+
+    // 3. sparse column path
+    data.emplace_back("v.d.d", get_field("array_int"));
+    data.emplace_back("v.c.d", get_field("string"));
+    data.emplace_back("v.b.d", get_field("array_int"));
+    for (int i = 0; i < 5; ++i) {
+        auto field = construct_variant_map(data);
+        variant->try_insert(field);
+    }
+
+    data.clear();
+    data.emplace_back("v.a", get_field("int"));
+    data.emplace_back("v.b", get_field("int"));
+    data.emplace_back("v.c", get_field("array_int"));
+    data.emplace_back("v.f", get_field("array_str"));
+    data.emplace_back("v.e", get_field("string"));
+    data.emplace_back("v.d.d", get_field("array_str"));
+    data.emplace_back("v.c.d", get_field("int"));
+    data.emplace_back("v.b.d", get_field("array_str"));
+    for (int i = 0; i < 5; ++i) {
+        auto field = construct_variant_map(data);
+        variant->try_insert(field);
+    }
+    return variant;
+}
+
+// std::string convert_jsonb_field_to_string(doris::vectorized::Field jsonb) {
+//     const auto& val = jsonb.get<JsonbField>();
+//     const JsonbValue* json_val = 
JsonbDocument::createValue(val.get_value(), val.get_size());
+
+//     rapidjson::Document doc;
+//     doc.SetObject();
+//     rapidjson::Document::AllocatorType& allocator = doc.GetAllocator();
+//     rapidjson::Value json_value;
+//     convert_jsonb_to_rapidjson(*json_val, json_value, allocator);
+//     doc.AddMember("value", json_value, allocator);
+//     rapidjson::StringBuffer buffer;
+//     rapidjson::PrettyWriter<rapidjson::StringBuffer> writer(buffer);
+//     doc.Accept(writer);
+//     return std::string(buffer.GetString());
+// }
+
+std::string convert_field_to_string(doris::vectorized::Field array) {
+    rapidjson::Document doc;
+    doc.SetObject();
+    rapidjson::Document::AllocatorType& allocator = doc.GetAllocator();
+    rapidjson::Value json_value;
+    DataTypeSerDe::convert_field_to_rapidjson(array, json_value, allocator);
+    doc.AddMember("value", json_value, allocator);
+    rapidjson::StringBuffer buffer;
+    rapidjson::PrettyWriter<rapidjson::StringBuffer> writer(buffer);
+    doc.Accept(writer);
+    return std::string(buffer.GetString());
+}
+
+TEST(ColumnVariantTest, advanced_finalize) {
+    auto variant = construct_advanced_varint_column();
+
+    // 4. finalize
+    
EXPECT_TRUE(variant->finalize(ColumnObject::FinalizeMode::WRITE_MODE).ok());
+    EXPECT_EQ(variant->size(), 15);
+
+    // check finalized subcolumn
+    // 5 subcolumn + 1 root
+    EXPECT_EQ(variant->subcolumns.size(), 6);
+    for (const auto& column : variant->subcolumns) {
+        if (column->data.is_root) {
+            continue;
+        }
+        EXPECT_EQ(column->data.data.size(), 1);
+    }
+
+    // check sparse column
+    const auto& offsets = variant->serialized_sparse_column_offsets();
+    for (int row = 0; row < 5; ++row) {
+        EXPECT_EQ(offsets[row] - offsets[row - 1], 0);
+    }
+    for (int row = 5; row < 15; ++row) {
+        EXPECT_EQ(offsets[row] - offsets[row - 1], 3);
+    }
+}
+
+TEST(ColumnVariantTest, advanced_deserialize) {
+    auto variant = construct_advanced_varint_column();
+
+    // 4. finalize
+    
EXPECT_TRUE(variant->finalize(ColumnObject::FinalizeMode::WRITE_MODE).ok());
+    EXPECT_EQ(variant->size(), 15);
+
+    const auto& [path, value] = variant->get_sparse_data_paths_and_values();
+    const auto& offsets = variant->serialized_sparse_column_offsets();
+    for (size_t row = 5; row < 10; ++row) {
+        size_t start = offsets[row - 1];
+        size_t end = offsets[row];
+
+        auto data = path->get_data_at(start);
+        auto pair = variant->deserialize_from_sparse_column(value, start++);
+        EXPECT_EQ(data, StringRef("v.b.d", 5));
+        EXPECT_EQ(convert_field_to_string(pair.first),
+                  convert_field_to_string(get_jsonb_field("array_int")));
+
+        auto data2 = path->get_data_at(start);
+        auto pair2 = variant->deserialize_from_sparse_column(value, start++);
+        EXPECT_EQ(data2, StringRef("v.c.d", 5));
+        EXPECT_EQ(convert_field_to_string(pair2.first),
+                  convert_field_to_string(get_field("string")));
+
+        auto data3 = path->get_data_at(start);
+        auto pair3 = variant->deserialize_from_sparse_column(value, start++);
+        EXPECT_EQ(data3, StringRef("v.d.d", 5));
+        EXPECT_EQ(convert_field_to_string(pair3.first),
+                  convert_field_to_string(get_jsonb_field("array_int")));
+        EXPECT_EQ(start, end);
+    }
+
+    for (size_t row = 10; row < 15; ++row) {
+        size_t start = offsets[row - 1];
+        size_t end = offsets[row];
+
+        auto data = path->get_data_at(start);
+        auto pair = variant->deserialize_from_sparse_column(value, start++);
+        EXPECT_EQ(data, StringRef("v.b.d", 5));
+        EXPECT_EQ(convert_field_to_string(pair.first),
+                  convert_field_to_string(get_jsonb_field("array_str")));
+
+        auto data2 = path->get_data_at(start);
+        auto pair2 = variant->deserialize_from_sparse_column(value, start++);
+        EXPECT_EQ(data2, StringRef("v.c.d", 5));
+        EXPECT_EQ(convert_field_to_string(pair2.first),
+                  convert_field_to_string(get_jsonb_field("int")));
+
+        auto data3 = path->get_data_at(start);
+        auto pair3 = variant->deserialize_from_sparse_column(value, start++);
+        EXPECT_EQ(data3, StringRef("v.d.d", 5));
+        EXPECT_EQ(convert_field_to_string(pair3.first),
+                  convert_field_to_string(get_jsonb_field("array_str")));
+        EXPECT_EQ(start, end);
+    }
+}
+
+TEST(ColumnVariantTest, advanced_insert_range_from) {
+    auto src = construct_advanced_varint_column();
+    EXPECT_TRUE(src->finalize(ColumnObject::FinalizeMode::WRITE_MODE).ok());
+    EXPECT_EQ(src->size(), 15);
+
+    auto dst = construct_dst_varint_column();
+
+    // subcolumn->subcolumn          v.b v.f v.e
+    // subcolumn->sparse_column      v.a v.c
+    // sparse_column->subcolumn      v.b.d v.c.d
+    // sparse_column->sparse_column  v.d.d
+    dst->insert_range_from(*src, 0, src->size());
+    EXPECT_EQ(dst->size(), 15);
+
+    EXPECT_EQ(dst->subcolumns.size(), 6);
+    ColumnObject::Subcolumns dst_subcolumns = dst->subcolumns;
+
+    std::sort(
+            dst_subcolumns.begin(), dst_subcolumns.end(),
+            [](const auto& lhsItem, const auto& rhsItem) { return 
lhsItem->path < rhsItem->path; });
+
+    // subcolumns
+    for (const auto& column : dst_subcolumns) {
+        if (column->data.is_root) {
+            continue;
+        }
+        EXPECT_EQ(column->data.data.size(), 1);
+        EXPECT_EQ(column->data.data[0]->size(), 15);
+
+        if (column->path.get_path().size() == 3) {
+            EXPECT_EQ(column->data.get_non_null_value_size(), 15);
+            if (column->path.get_path() == "v.b") {
+                EXPECT_EQ(assert_cast<const 
DataTypeNullable*>(column->data.data_types[0].get())
+                                  ->get_nested_type()
+                                  ->get_type_id(),
+                          TypeIndex::JSONB);
+            }
+        } else if (column->path.get_path().size() == 5) {
+            EXPECT_EQ(column->data.get_non_null_value_size(), 10);
+            EXPECT_EQ(assert_cast<const 
DataTypeNullable*>(column->data.data_types[0].get())
+                              ->get_nested_type()
+                              ->get_type_id(),
+                      TypeIndex::JSONB);
+            for (size_t row = 0; row < 5; ++row) {
+                EXPECT_TRUE(column->data.data[0]->is_null_at(row));
+            }
+        }
+    }
+
+    // sparse columns
+    const auto& [path, value] = dst->get_sparse_data_paths_and_values();
+    const auto& offsets = dst->serialized_sparse_column_offsets();
+
+    // v.a v.c
+    for (int row = 0; row < 5; ++row) {
+        size_t start = offsets[row - 1];
+        size_t end = offsets[row];
+
+        auto data = path->get_data_at(start);
+        EXPECT_EQ(data, StringRef("v.a", 3));
+        auto pair = dst->deserialize_from_sparse_column(value, start++);
+        EXPECT_EQ(pair.first.get<Int64>(), 20);
+
+        auto data2 = path->get_data_at(start);
+        EXPECT_EQ(data2, StringRef("v.c", 3));
+        auto pair2 = dst->deserialize_from_sparse_column(value, start++);
+        EXPECT_EQ(convert_field_to_string(pair2.first),
+                  convert_field_to_string(get_field("array_int")));
+
+        EXPECT_EQ(start, end);
+    }
+
+    for (int row = 5; row < 10; ++row) {
+        size_t start = offsets[row - 1];
+        size_t end = offsets[row];
+
+        auto data = path->get_data_at(start);
+        auto pair = dst->deserialize_from_sparse_column(value, start++);
+        EXPECT_EQ(data, StringRef("v.a", 3));
+        EXPECT_EQ(pair.first.get<Int64>(), 20);
+
+        auto data2 = path->get_data_at(start);
+        auto pair2 = dst->deserialize_from_sparse_column(value, start++);
+        EXPECT_EQ(data2, StringRef("v.c", 3));
+        EXPECT_EQ(convert_field_to_string(pair2.first),
+                  convert_field_to_string(get_field("array_int")));
+
+        auto data3 = path->get_data_at(start);
+        auto pair3 = dst->deserialize_from_sparse_column(value, start++);
+        EXPECT_EQ(data3, StringRef("v.d.d", 5));
+        EXPECT_EQ(convert_field_to_string(pair3.first),
+                  convert_field_to_string(get_jsonb_field("array_int")));
+
+        EXPECT_EQ(start, end);
+    }
+
+    for (int row = 10; row < 15; ++row) {
+        size_t start = offsets[row - 1];
+        size_t end = offsets[row];
+
+        auto data = path->get_data_at(start);
+        auto pair = dst->deserialize_from_sparse_column(value, start++);
+        EXPECT_EQ(data, StringRef("v.a", 3));
+        EXPECT_EQ(pair.first.get<Int64>(), 20);
+
+        auto data2 = path->get_data_at(start);
+        auto pair2 = dst->deserialize_from_sparse_column(value, start++);
+        EXPECT_EQ(data2, StringRef("v.c", 3));
+        EXPECT_EQ(convert_field_to_string(pair2.first),
+                  convert_field_to_string(get_field("array_int")));
+
+        auto data3 = path->get_data_at(start);
+        auto pair3 = dst->deserialize_from_sparse_column(value, start++);
+        EXPECT_EQ(data3, StringRef("v.d.d", 5));
+        EXPECT_EQ(convert_field_to_string(pair3.first),
+                  convert_field_to_string(get_jsonb_field("array_str")));
+
+        EXPECT_EQ(start, end);
+    }
+}
+
+auto construct_varint_column_only_subcolumns() {
+    // 1. create an empty variant column
+    auto variant = ColumnObject::create();
+
+    std::vector<std::pair<std::string, doris::vectorized::Field>> data;
+
+    // 2. subcolumn path
+    data.emplace_back("v.a", 20);
+    data.emplace_back("v.b", "20");
+    data.emplace_back("v.c", 20);
+    data.emplace_back("v.f", 20);
+    data.emplace_back("v.e", "50");
+    for (int i = 0; i < 5; ++i) {
+        auto field = construct_variant_map(data);
+        variant->try_insert(field);
+    }
+
+    // 3. root
+    variant->try_insert(doris::vectorized::Field(20));
+    return variant;
+}
+
+auto construct_varint_column_more_subcolumns() {
+    // 1. create an empty variant column
+    auto variant = ColumnObject::create();
+
+    std::vector<std::pair<std::string, doris::vectorized::Field>> data;
+
+    // 2. subcolumn path
+    data.emplace_back("v.a", 20);
+    data.emplace_back("v.b", "20");
+    data.emplace_back("v.c", 20);
+    data.emplace_back("v.f", 20);
+    data.emplace_back("v.e", "50");
+    data.emplace_back("v.x", "str");
+    data.emplace_back("v.y", 20);
+    for (int i = 0; i < 5; ++i) {
+        auto field = construct_variant_map(data);
+        variant->try_insert(field);
+    }
+
+    return variant;
+}
+
+TEST(ColumnVariantTest, empty_inset_range_from) {
+    auto src = construct_varint_column_only_subcolumns();
+    EXPECT_TRUE(src->finalize(ColumnObject::FinalizeMode::WRITE_MODE).ok());
+    EXPECT_EQ(src->size(), 6);
+
+    // dst is an empty column
+    auto dst = ColumnObject::create();
+
+    // subcolumn->subcolumn          v.a v.b v.c v.f v.e
+    dst->insert_range_from(*src, 0, 6);
+    EXPECT_EQ(dst->size(), 6);
+
+    // 5 subcolumn
+    EXPECT_EQ(dst->subcolumns.size(), 6);
+
+    for (const auto& column : dst->subcolumns) {
+        if (column->data.is_root) {
+            EXPECT_EQ(column->data.data.size(), 1);
+            EXPECT_EQ(column->data.data[0]->size(), 6);
+            EXPECT_EQ(column->data.get_non_null_value_size(), 1);
+            continue;
+        }
+        EXPECT_EQ(column->data.data.size(), 1);
+        EXPECT_EQ(column->data.data[0]->size(), 6);
+        EXPECT_EQ(column->data.get_non_null_value_size(), 5);
+    }
+
+    // empty sparse column
+    const auto& [path, value] = dst->get_sparse_data_paths_and_values();
+    const auto& offsets = dst->serialized_sparse_column_offsets();
+    EXPECT_EQ(offsets[4], offsets[-1]);
+    EXPECT_EQ(path->size(), value->size());
+
+    auto src_contains_seven_subcolumns = 
construct_varint_column_more_subcolumns();
+
+    EXPECT_TRUE(
+            
src_contains_seven_subcolumns->finalize(ColumnObject::FinalizeMode::WRITE_MODE).ok());
+    EXPECT_EQ(src_contains_seven_subcolumns->size(), 5);
+
+    // subcolumn->subcolumn          v.a v.b v.c v.f v.e
+    // add sprase columns            v.x v.y
+    dst->insert_range_from(*src_contains_seven_subcolumns, 0, 5);
+    EXPECT_EQ(dst->size(), 11);
+
+    // 5 subcolumn
+    EXPECT_EQ(dst->subcolumns.size(), 6);
+
+    for (int row = 0; row < 6; ++row) {
+        size_t start = offsets[row - 1];
+        size_t end = offsets[row];
+
+        EXPECT_EQ(start, end);
+    }
+
+    // v.x v.y
+    for (int row = 6; row < 11; ++row) {
+        size_t start = offsets[row - 1];
+        size_t end = offsets[row];
+
+        auto data = path->get_data_at(start);
+        EXPECT_EQ(data, StringRef("v.x", 3));
+        auto pair = dst->deserialize_from_sparse_column(value, start++);
+        EXPECT_EQ(convert_field_to_string(pair.first),
+                  convert_field_to_string(get_field("string")));
+
+        auto data2 = path->get_data_at(start);
+        EXPECT_EQ(data2, StringRef("v.y", 3));
+        auto pair2 = dst->deserialize_from_sparse_column(value, start++);
+        EXPECT_EQ(pair2.first.get<Int64>(), 20);
+
+        EXPECT_EQ(start, end);
+    }
+
+    auto src_contains_subcoumns_and_sparse_columns = 
construct_basic_varint_column();
+    EXPECT_TRUE(src_contains_subcoumns_and_sparse_columns
+                        ->finalize(ColumnObject::FinalizeMode::WRITE_MODE)
+                        .ok());
+    EXPECT_EQ(src_contains_subcoumns_and_sparse_columns->size(), 10);
+
+    // subcolumn->subcolumn          v.a v.b v.c v.f v.e
+    // add sprase columns            v.x v.y v.b.d v.c.d v.d.d
+    dst->insert_range_from(*src_contains_subcoumns_and_sparse_columns, 0, 10);
+    EXPECT_EQ(dst->size(), 21);
+
+    // 5 subcolumn
+    EXPECT_EQ(dst->subcolumns.size(), 6);
+
+    for (int row = 0; row < 6; ++row) {
+        size_t start = offsets[row - 1];
+        size_t end = offsets[row];
+
+        EXPECT_EQ(start, end);
+    }
+
+    // v.x v.y
+    for (int row = 6; row < 11; ++row) {
+        size_t start = offsets[row - 1];
+        size_t end = offsets[row];
+
+        auto data = path->get_data_at(start);
+        EXPECT_EQ(data, StringRef("v.x", 3));
+        auto pair = dst->deserialize_from_sparse_column(value, start++);
+        EXPECT_EQ(convert_field_to_string(pair.first),
+                  convert_field_to_string(get_field("string")));
+
+        auto data2 = path->get_data_at(start);
+        EXPECT_EQ(data2, StringRef("v.y", 3));
+        auto pair2 = dst->deserialize_from_sparse_column(value, start++);
+        EXPECT_EQ(pair2.first.get<Int64>(), 20);
+
+        EXPECT_EQ(start, end);
+    }
+
+    for (int row = 11; row < 16; ++row) {
+        size_t start = offsets[row - 1];
+        size_t end = offsets[row];
+
+        EXPECT_EQ(start, end);
+    }
+
+    //v.b.d v.c.d v.d.d
+    for (int row = 16; row < 21; ++row) {
+        size_t start = offsets[row - 1];
+        size_t end = offsets[row];
+
+        auto data = path->get_data_at(start);
+        EXPECT_EQ(data, StringRef("v.b.d", 5));
+        auto pair = dst->deserialize_from_sparse_column(value, start++);
+        EXPECT_EQ(pair.first.get<Int64>(), 30);
+
+        auto data2 = path->get_data_at(start);
+        auto pair2 = dst->deserialize_from_sparse_column(value, start++);
+        EXPECT_EQ(data2, StringRef("v.c.d", 5));
+        EXPECT_EQ(pair2.first.get<Int64>(), 30);
+
+        auto data3 = path->get_data_at(start);
+        auto pair3 = dst->deserialize_from_sparse_column(value, start++);
+        EXPECT_EQ(data3, StringRef("v.d.d", 5));
+        EXPECT_EQ(pair3.first.get<String>(), "50");
+        EXPECT_EQ(start, end);
+    }
+}
diff --git a/regression-test/data/variant_p0/variant_with_rowstore.out 
b/regression-test/data/variant_p0/variant_with_rowstore.out
index fbad7c4e44d..ca5e7393e9d 100644
--- a/regression-test/data/variant_p0/variant_with_rowstore.out
+++ b/regression-test/data/variant_p0/variant_with_rowstore.out
@@ -33,5 +33,5 @@
 -1     {"a":1123}      {"a":1123}
 
 -- !point_select --
-1      1|[""]
+1      1|[""]
 


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to