This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-4.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-4.1 by this push:
new be8b2c9abc3 branch-4.1: [fix](variant) Prefer sparse variant fields
over root value #65660 (#65950)
be8b2c9abc3 is described below
commit be8b2c9abc3ab1a8c123170361771c4f2d855999
Author: github-actions[bot]
<41898282+github-actions[bot]@users.noreply.github.com>
AuthorDate: Fri Jul 24 10:43:18 2026 +0800
branch-4.1: [fix](variant) Prefer sparse variant fields over root value
#65660 (#65950)
Cherry-picked from #65660
---------
Co-authored-by: lihangyu <[email protected]>
---
be/src/core/column/column_variant.cpp | 5 ++
be/test/core/column/column_variant_test.cpp | 27 +++++++
.../test_topn_lazy_materialize_sparse_variant.out | 17 +++++
...est_topn_lazy_materialize_sparse_variant.groovy | 85 ++++++++++++++++++++++
4 files changed, 134 insertions(+)
diff --git a/be/src/core/column/column_variant.cpp
b/be/src/core/column/column_variant.cpp
index 694562ac553..7382c2cad4f 100644
--- a/be/src/core/column/column_variant.cpp
+++ b/be/src/core/column/column_variant.cpp
@@ -1830,6 +1830,11 @@ bool ColumnVariant::is_visible_root_value(size_t nrow)
const {
}
}
+ const auto& sparse_offsets = serialized_sparse_column_offsets();
+ if (sparse_offsets[nrow - 1] != sparse_offsets[nrow]) {
+ return false;
+ }
+
const auto& doc_value_column_map = assert_cast<const
ColumnMap&>(*serialized_doc_value_column);
// doc snapshot column is not empty
if (doc_value_column_map.get_offsets()[nrow - 1] !=
doc_value_column_map.get_offsets()[nrow]) {
diff --git a/be/test/core/column/column_variant_test.cpp
b/be/test/core/column/column_variant_test.cpp
index b01a5a28535..0ab96111075 100644
--- a/be/test/core/column/column_variant_test.cpp
+++ b/be/test/core/column/column_variant_test.cpp
@@ -1082,6 +1082,33 @@ TEST_F(ColumnVariantTest, test_insert_indices_from) {
}
}
+TEST_F(ColumnVariantTest, visible_root_does_not_hide_sparse_fields) {
+ auto source = ColumnVariant::create(0, false);
+ VariantMap mixed;
+ mixed.try_emplace(PathInData(), FieldWithDataType {.field =
get_jsonb_field("array_int")});
+ mixed.try_emplace(PathInData("n"), FieldWithDataType {.field =
VariantUtil::get_field("int")});
+ mixed.try_emplace(PathInData("word"),
+ FieldWithDataType {.field =
VariantUtil::get_field("string")});
+ source->try_insert(Field::create_field<TYPE_VARIANT>(std::move(mixed)));
+ source->finalize();
+
+ auto destination = ColumnVariant::create(1, false);
+ destination->try_insert(
+ VariantUtil::construct_variant_map({{"k",
VariantUtil::get_field("int")}}));
+ destination->insert_range_from(*source, 0, 1);
+ destination->finalize();
+
+
EXPECT_EQ(destination->get_subcolumns().get_root()->data.get_least_common_base_type_id(),
+ PrimitiveType::TYPE_JSONB);
+ const auto& sparse_offsets =
destination->serialized_sparse_column_offsets();
+ EXPECT_LT(sparse_offsets[0], sparse_offsets[1]);
+
+ DataTypeSerDe::FormatOptions options;
+ std::string json;
+ destination->serialize_one_row_to_string(1, &json, options);
+ EXPECT_EQ(json, R"({"n":20,"word":"str"})");
+}
+
TEST_F(ColumnVariantTest, is_variable_length) {
EXPECT_TRUE(column_variant->is_variable_length());
}
diff --git
a/regression-test/data/variant_p0/test_topn_lazy_materialize_sparse_variant.out
b/regression-test/data/variant_p0/test_topn_lazy_materialize_sparse_variant.out
new file mode 100644
index 00000000000..505edfbb546
--- /dev/null
+++
b/regression-test/data/variant_p0/test_topn_lazy_materialize_sparse_variant.out
@@ -0,0 +1,17 @@
+-- This file is automatically generated. You should know what you did if you
want to edit this
+-- !topn_lazy_sparse_variant --
+0 1 {} {}
+1 -1 {} [1,2,3]
+3 42 {"k":1} {}
+4 -9223372036854775808 [1,2,3] {"k":1}
+6 1 [1,2,3] {}
+7 0 [1,2,3] [1,2,3]
+9 \N {} []
+10 1 {} {"n":7,"word":"hot"}
+12 1 {"word":"hot","n":7} \N
+13 5777142737451291289 {} {"n":7,"word":"hot"}
+15 -1 [1,2,3] []
+16 -906638365906932436 {"word":"hot","n":7} {"k":1}
+18 -9223372036854775808 {"word":"hot","n":7} {"n":7,"word":"hot"}
+19 -9223372036854775808 {"word":"hot","n":7} [1,2,3]
+
diff --git
a/regression-test/suites/variant_p0/test_topn_lazy_materialize_sparse_variant.groovy
b/regression-test/suites/variant_p0/test_topn_lazy_materialize_sparse_variant.groovy
new file mode 100644
index 00000000000..041c704feaa
--- /dev/null
+++
b/regression-test/suites/variant_p0/test_topn_lazy_materialize_sparse_variant.groovy
@@ -0,0 +1,85 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_topn_lazy_materialize_sparse_variant", "p0") {
+ sql "set default_variant_enable_doc_mode = false"
+ sql "set default_variant_max_subcolumns_count = 1"
+ sql "set default_variant_sparse_hash_shard_count = 2"
+ sql "set use_v3_storage_format = true"
+ sql "set enable_file_cache = true"
+ sql "set topn_lazy_materialization_threshold = 1024"
+
+ sql "DROP TABLE IF EXISTS test_topn_lazy_materialize_sparse_variant"
+ sql """
+ CREATE TABLE test_topn_lazy_materialize_sparse_variant (
+ pk INT,
+ col_bigint BIGINT NULL,
+ col_json JSON NOT NULL,
+ col_variant VARIANT NULL
+ ) ENGINE = OLAP
+ DUPLICATE KEY(pk)
+ PARTITION BY RANGE(pk) (
+ PARTITION p0 VALUES LESS THAN ('4'),
+ PARTITION p1 VALUES LESS THAN ('64'),
+ PARTITION p2 VALUES LESS THAN ('256'),
+ PARTITION pmax VALUES LESS THAN ('2147483647')
+ )
+ DISTRIBUTED BY HASH(pk) BUCKETS 4
+ PROPERTIES (
+ "replication_num" = "1",
+ "disable_auto_compaction" = "true",
+ "storage_format" = "V3"
+ )
+ """
+
+ sql """
+ INSERT INTO test_topn_lazy_materialize_sparse_variant VALUES
+ (19, -9223372036854775808, '{"word":"hot","n":7}', '[1,2,3]'),
+ (18, -9223372036854775808, '{"word":"hot","n":7}',
'{"word":"hot","n":7}'),
+ (17, -1, '{"k":1}', '{}'),
+ (16, -906638365906932436, '{"word":"hot","n":7}', '{"k":1}'),
+ (15, -1, '[1,2,3]', '{}'),
+ (14, -9223372036854775808, '{"word":"hot","n":7}', '[1,2,3]'),
+ (13, 5777142737451291289, '{}', '{"word":"hot","n":7}'),
+ (12, 1, '{"word":"hot","n":7}', NULL),
+ (11, 42, '[1,2,3]', NULL),
+ (10, 1, '{}', '{"word":"hot","n":7}'),
+ (9, NULL, '{}', '{}'),
+ (8, NULL, '{"k":1}', '{"k":1}'),
+ (7, 0, '[1,2,3]', '[1,2,3]'),
+ (6, 1, '[1,2,3]', '{}'),
+ (5, -3610716764638269764, '{"k":1}', '[1,2,3]'),
+ (4, -9223372036854775808, '[1,2,3]', '{"k":1}'),
+ (3, 42, '{"k":1}', '{}'),
+ (2, -9223372036854775808, '{}', '{}'),
+ (1, -1, '{}', '[1,2,3]'),
+ (0, 1, '{}', '{}')
+ """
+
+ def topnQuery = """
+ SELECT pk, col_bigint, col_json, col_variant
+ FROM test_topn_lazy_materialize_sparse_variant
+ WHERE ABS(pk % 3) IN (0, 1, 3)
+ ORDER BY pk ASC
+ LIMIT 128
+ """
+ explain {
+ sql topnQuery
+ contains("MaterializeNode")
+ }
+ qt_topn_lazy_sparse_variant topnQuery
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
