This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-4.2
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-4.2 by this push:
new 1654ee418a6 [branch-4-2](function) Backport nested MAP cast
deduplication to branch-4-2 (#69110)
1654ee418a6 is described below
commit 1654ee418a627360ae04a8d47c8c2451d226fca3
Author: Chenyang Sun <[email protected]>
AuthorDate: Thu Oct 15 23:33:34 2026 +0800
[branch-4-2](function) Backport nested MAP cast deduplication to branch-4-2
(#69110)
## What problem does this PR solve?
Backport the nested MAP duplicate-key handling from Apache Doris PR
#68639 to branch-4.2.
String casts to MAP, ARRAY, and STRUCT now remove duplicate keys from
MAPs nested in arrays, structs, and MAP values, retaining the last
value. The backport removes the old top-level-only MAP wrapper and
normalizes the shared string-to-complex cast path.
## Scope
- Backport the three functional commits from #68639.
- Resolve the branch-4.2 conflicts while preserving branch-specific cast
code.
- Add nested MAP regression and unit coverage for strict and non-strict
modes.
## Validation
- clang-format 16 check passed.
- git diff --check passed.
- Full BE build and unit test were not run in this worktree because no
branch-4.2 build artifact was available.
Related upstream PR: #68639
---------
Co-authored-by: Claude Opus 5.5 <[email protected]>
---
be/src/exprs/function/cast/cast_base.cpp | 28 +++++++++
be/src/exprs/function/cast/cast_to_map.h | 49 ++-------------
be/test/exprs/function/cast/cast_to_map_test.cpp | 50 ++++++++++++++++
.../test_cast_string_to_nested_map_dedup.out | 43 +++++++++++++
.../test_cast_string_to_nested_map_dedup.groovy | 70 ++++++++++++++++++++++
5 files changed, 196 insertions(+), 44 deletions(-)
diff --git a/be/src/exprs/function/cast/cast_base.cpp
b/be/src/exprs/function/cast/cast_base.cpp
index 04ea2cab562..e5a548502a8 100644
--- a/be/src/exprs/function/cast/cast_base.cpp
+++ b/be/src/exprs/function/cast/cast_base.cpp
@@ -19,9 +19,35 @@
#include <cstdint>
+#include "core/column/column_array.h"
+#include "core/column/column_map.h"
+#include "core/column/column_struct.h"
#include "util/jsonb_writer.h"
namespace doris::CastWrapper {
+// Remove the duplicated keys of every map in the column, also the maps inside
arrays, structs
+// and map values. The last value of a key wins, as in map().
+static Status deduplicate_map_keys(IColumn& column) {
+ if (auto* nullable = check_and_get_column<ColumnNullable>(column)) {
+ return deduplicate_map_keys(nullable->get_nested_column());
+ }
+ if (auto* array = check_and_get_column<ColumnArray>(column)) {
+ return deduplicate_map_keys(array->get_data());
+ }
+ if (auto* st = check_and_get_column<ColumnStruct>(column)) {
+ for (size_t i = 0; i < st->tuple_size(); ++i) {
+ RETURN_IF_ERROR(deduplicate_map_keys(st->get_column(i)));
+ }
+ return Status::OK();
+ }
+ if (auto* map = check_and_get_column<ColumnMap>(column)) {
+ // Values last: the ones dropped with a duplicated key are not visited.
+ RETURN_IF_ERROR(map->deduplicate_keys());
+ return deduplicate_map_keys(map->get_values());
+ }
+ return Status::OK();
+}
+
Status cast_from_generic_to_jsonb(FunctionContext* context, Block& block,
const ColumnNumbers& arguments, uint32_t
result,
size_t input_rows_count, const
NullMap::value_type* null_map) {
@@ -201,6 +227,7 @@ Status cast_from_string_to_complex_type(FunctionContext*
context, Block& block,
}
}
+ RETURN_IF_ERROR(deduplicate_map_keys(nested_column));
block.get_by_position(result).column = std::move(to_column);
return Status::OK();
}
@@ -235,6 +262,7 @@ Status
cast_from_string_to_complex_type_strict_mode(FunctionContext* context, Bl
nullable_col_to.get_null_map_data().push_back(0);
}
}
+ RETURN_IF_ERROR(deduplicate_map_keys(nested_column));
block.get_by_position(result).column = std::move(to_column);
return Status::OK();
}
diff --git a/be/src/exprs/function/cast/cast_to_map.h
b/be/src/exprs/function/cast/cast_to_map.h
index ca6b8e5e90b..c47a8ecf710 100644
--- a/be/src/exprs/function/cast/cast_to_map.h
+++ b/be/src/exprs/function/cast/cast_to_map.h
@@ -21,54 +21,16 @@
#include "exprs/function/cast/cast_base.h"
namespace doris::CastWrapper {
-#include "common/compile_check_begin.h"
-
-inline Status deduplicate_map_keys_in_result(Block& block, uint32_t result) {
- auto result_column_name = block.get_by_position(result).column->get_name();
- auto mutable_result_column =
IColumn::mutate(std::move(block.get_by_position(result).column));
-
- if (auto* nullable_column =
check_and_get_column<ColumnNullable>(*mutable_result_column)) {
- auto nested_column =
IColumn::mutate(nullable_column->get_nested_column_ptr());
- auto* map_column = check_and_get_column<ColumnMap>(*nested_column);
- if (!map_column) {
- return Status::RuntimeError("Illegal column {} for function CAST
AS MAP",
- result_column_name);
- }
-
- RETURN_IF_ERROR(map_column->deduplicate_keys(true));
- ColumnPtr nested_column_ptr = std::move(nested_column);
- nullable_column->change_nested_column(nested_column_ptr);
- } else {
- auto* map_column =
check_and_get_column<ColumnMap>(*mutable_result_column);
- if (!map_column) {
- return Status::RuntimeError("Illegal column {} for function CAST
AS MAP",
- result_column_name);
- }
-
- RETURN_IF_ERROR(map_column->deduplicate_keys(true));
- }
-
- block.get_by_position(result).column = std::move(mutable_result_column);
- return Status::OK();
-}
-
-inline WrapperType wrap_string_to_map_wrapper(WrapperType wrapper) {
- return [wrapper = std::move(wrapper)](FunctionContext* context, Block&
block,
- const ColumnNumbers& arguments,
uint32_t result,
- size_t input_rows_count,
- const NullMap::value_type* null_map
= nullptr) {
- RETURN_IF_ERROR(wrapper(context, block, arguments, result,
input_rows_count, null_map));
- return deduplicate_map_keys_in_result(block, result);
- };
-}
//TODO(Amory) . Need support more cast for key , value for map
WrapperType create_map_wrapper(FunctionContext* context, const DataTypePtr&
from_type,
const DataTypeMap& to_type) {
if (is_string_type(from_type->get_primitive_type())) {
- auto wrapper = context->enable_strict_mode() ?
cast_from_string_to_complex_type_strict_mode
- :
cast_from_string_to_complex_type;
- return wrap_string_to_map_wrapper(wrapper);
+ if (context->enable_strict_mode()) {
+ return cast_from_string_to_complex_type_strict_mode;
+ } else {
+ return cast_from_string_to_complex_type;
+ }
}
const auto* from = check_and_get_data_type<DataTypeMap>(from_type.get());
if (!from) {
@@ -120,5 +82,4 @@ WrapperType create_map_wrapper(FunctionContext* context,
const DataTypePtr& from
return Status::OK();
};
}
-#include "common/compile_check_end.h"
} // namespace doris::CastWrapper
diff --git a/be/test/exprs/function/cast/cast_to_map_test.cpp
b/be/test/exprs/function/cast/cast_to_map_test.cpp
index f3020d94f69..f6c882c8570 100644
--- a/be/test/exprs/function/cast/cast_to_map_test.cpp
+++ b/be/test/exprs/function/cast/cast_to_map_test.cpp
@@ -25,10 +25,12 @@
#include "core/column/column.h"
#include "core/column/column_map.h"
+#include "core/data_type/data_type_array.h"
#include "core/data_type/data_type_map.h"
#include "core/data_type/data_type_nullable.h"
#include "core/data_type/data_type_number.h"
#include "core/data_type/data_type_string.h"
+#include "core/data_type/data_type_struct.h"
#include "core/data_type/primitive_type.h"
#include "core/field.h"
#include "exprs/function/cast/cast_base.h"
@@ -179,4 +181,52 @@ TEST_F(FunctionCastTest,
test_from_string_to_map_string_string) {
builder.build(), false);
}
+// A duplicated key keeps its last value in every map of the result, also in
the maps inside
+// arrays, structs and map values, as it does in the top level map.
+TEST_F(FunctionCastTest, test_from_string_deduplicates_nested_map_keys) {
+ auto map_type =
std::make_shared<DataTypeNullable>(std::make_shared<DataTypeMap>(
+
std::make_shared<DataTypeNullable>(std::make_shared<DataTypeString>()),
+
std::make_shared<DataTypeNullable>(std::make_shared<DataTypeInt32>())));
+ auto string_type =
std::make_shared<DataTypeNullable>(std::make_shared<DataTypeString>());
+ auto array_of_map_type =
+
std::make_shared<DataTypeNullable>(std::make_shared<DataTypeArray>(map_type));
+
+ struct Case {
+ DataTypePtr to_type;
+ std::string from;
+ std::string expected;
+ };
+ std::vector<Case> cases = {
+ {map_type, R"({"a":1,"a":2})", R"({"a":2})"},
+ {array_of_map_type, R"([{"a":1,"a":2}])", R"([{"a":2}])"},
+ {std::make_shared<DataTypeNullable>(
+ std::make_shared<DataTypeStruct>(DataTypes {map_type},
Strings {"m"})),
+ R"({"m":{"a":1,"a":2}})", R"({"m":{"a":2}})"},
+ {std::make_shared<DataTypeNullable>(
+ std::make_shared<DataTypeMap>(string_type,
array_of_map_type)),
+ R"({"outer":[{"a":1,"a":2}]})", R"({"outer":[{"a":2}]})"},
+ {std::make_shared<DataTypeNullable>(
+ std::make_shared<DataTypeMap>(string_type,
array_of_map_type)),
+ R"({"outer":[{"a":1,"a":2}],"outer":[{"b":1,"b":2}]})",
R"({"outer":[{"b":2}]})"},
+ };
+
+ for (bool is_strict_mode : {false, true}) {
+ for (const auto& c : cases) {
+ auto ctx = create_context(is_strict_mode);
+ DataTypePtr from_type = std::make_shared<DataTypeString>();
+ auto fn = get_cast_wrapper(ctx.get(), from_type, c.to_type);
+ ASSERT_TRUE(fn != nullptr);
+
+ Block block = {
+ {ColumnHelper::create_column<DataTypeString>({c.from}),
from_type, "from"},
+ {nullptr, c.to_type, "to"},
+ };
+ ASSERT_TRUE(fn(ctx.get(), block, {0}, 1, block.rows(), nullptr));
+ EXPECT_EQ(c.to_type->to_string(*block.get_by_position(1).column,
0), c.expected)
+ << "from: " << c.from << ", to: " << c.to_type->get_name()
+ << ", strict mode: " << is_strict_mode;
+ }
+ }
+}
+
} // namespace doris
diff --git
a/regression-test/data/query_p0/sql_functions/cast_function/test_cast_string_to_nested_map_dedup.out
b/regression-test/data/query_p0/sql_functions/cast_function/test_cast_string_to_nested_map_dedup.out
new file mode 100644
index 00000000000..6beac6d6c48
--- /dev/null
+++
b/regression-test/data/query_p0/sql_functions/cast_function/test_cast_string_to_nested_map_dedup.out
@@ -0,0 +1,43 @@
+-- This file is automatically generated. You should know what you did if you
want to edit this
+-- !top --
+{"a":2}
+
+-- !map_in_map --
+{"outer":{"a":2}}
+
+-- !array --
+[{"a":2}] 1 2
+
+-- !struct --
+{"m":{"a":2}} 1
+
+-- !array_in_map --
+{"outer":[{"a":2}]} 2
+
+-- !struct_in_map --
+{"outer":{"m":{"a":2}}}
+
+-- !top --
+{"a":2}
+
+-- !map_in_map --
+{"outer":{"a":2}}
+
+-- !array --
+[{"a":2}] 1 2
+
+-- !struct --
+{"m":{"a":2}} 1
+
+-- !array_in_map --
+{"outer":[{"a":2}]} 2
+
+-- !struct_in_map --
+{"outer":{"m":{"a":2}}}
+
+-- !column --
+1 [{"a":2}]
+2 [{"a":1}, {"b":3}]
+3 [{"a":1, "b":2}]
+4 []
+5 \N
diff --git
a/regression-test/suites/query_p0/sql_functions/cast_function/test_cast_string_to_nested_map_dedup.groovy
b/regression-test/suites/query_p0/sql_functions/cast_function/test_cast_string_to_nested_map_dedup.groovy
new file mode 100644
index 00000000000..cfc791e2a09
--- /dev/null
+++
b/regression-test/suites/query_p0/sql_functions/cast_function/test_cast_string_to_nested_map_dedup.groovy
@@ -0,0 +1,70 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_cast_string_to_nested_map_dedup") {
+ // A duplicated key keeps its last value in every map of a string cast,
also in the maps
+ // inside arrays, structs and map values, as it does in the top level map.
+ def check = {
+ qt_top """ SELECT CAST('{"a":1,"a":2}' AS MAP<STRING, INT>) """
+ qt_map_in_map """ SELECT CAST('{"outer":{"a":1,"a":2}}' AS MAP<STRING,
MAP<STRING, INT>>) """
+ qt_array """
+ SELECT CAST('[{"a":1,"a":2}]' AS ARRAY<MAP<STRING, INT>>) AS c,
+ size(element_at(CAST('[{"a":1,"a":2}]' AS ARRAY<MAP<STRING,
INT>>), 1)),
+ element_at(element_at(CAST('[{"a":1,"a":2}]' AS
ARRAY<MAP<STRING, INT>>), 1), 'a')
+ """
+ qt_struct """
+ SELECT CAST('{"m":{"a":1,"a":2}}' AS STRUCT<m:MAP<STRING, INT>>)
AS c,
+ size(struct_element(CAST('{"m":{"a":1,"a":2}}' AS
STRUCT<m:MAP<STRING, INT>>), 'm'))
+ """
+ qt_array_in_map """
+ SELECT CAST('{"outer":[{"a":1,"a":2}]}' AS MAP<STRING,
ARRAY<MAP<STRING, INT>>>) AS c,
+ element_at(element_at(element_at(
+ CAST('{"outer":[{"a":1,"a":2}]}' AS MAP<STRING,
ARRAY<MAP<STRING, INT>>>),
+ 'outer'), 1), 'a')
+ """
+ qt_struct_in_map """
+ SELECT CAST('{"outer":{"m":{"a":1,"a":2}}}' AS MAP<STRING,
STRUCT<m:MAP<STRING, INT>>>)
+ """
+ }
+
+ sql """ set enable_strict_cast = false """
+ check()
+ sql """ set enable_strict_cast = true """
+ check()
+ sql """ set enable_strict_cast = false """
+
+ sql """ DROP TABLE IF EXISTS test_cast_string_to_nested_map_dedup """
+ sql """
+ CREATE TABLE test_cast_string_to_nested_map_dedup (
+ id INT,
+ s STRING
+ ) DUPLICATE KEY(id)
+ DISTRIBUTED BY HASH(id) BUCKETS 1
+ PROPERTIES ("replication_num" = "1")
+ """
+ sql """
+ INSERT INTO test_cast_string_to_nested_map_dedup VALUES
+ (1, '[{"a":1,"a":2}]'),
+ (2, '[{"a":1}, {"b":1,"b":2,"b":3}]'),
+ (3, '[{"a":1,"b":2}]'),
+ (4, '[]'),
+ (5, NULL)
+ """
+ order_qt_column """
+ SELECT id, CAST(s AS ARRAY<MAP<STRING, INT>>) FROM
test_cast_string_to_nested_map_dedup
+ """
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
