This is an automated email from the ASF dual-hosted git repository.
HappenLee pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new ace58e00a85 [fix](be) Fix nullable handling in group array set
operations (#66284)
ace58e00a85 is described below
commit ace58e00a8516ee22bd9c3f8184b423e6dc787ae
Author: Jerry Hu <[email protected]>
AuthorDate: Mon Aug 3 11:43:04 2026 +0800
[fix](be) Fix nullable handling in group array set operations (#66284)
Problem Summary: `group_array_intersect` and `group_array_union`
discarded the outer nullable type before aggregate function creation. As
a result, the nullable aggregate wrapper was not installed and masked
payloads beneath NULL array rows could participate in set operations,
making semantically equivalent `CASE` and `IF` expressions return
different results.
The raw aggregate also accepted outer `ColumnNullable` input even though
row-level NULL filtering belongs to the nullable wrapper. In addition,
the string intersection path could read the masked nested payload of a
NULL array element and incorrectly match it as a real string, for
example preserving an empty string for `[""]` intersect `[NULL]`.
This change preserves the original argument types for aggregate wrapper
creation and unwraps nullable types only for element type dispatch. The
raw aggregate now accepts only `ColumnArray`, outer NULL array rows are
skipped by the nullable wrapper, and nullable array elements remain the
responsibility of the numeric/string data implementations. The string
intersection path handles NULL elements without reading their masked
nested payload.
### Compatibility
The serialized state layout is unchanged. States produced before this
fix remain readable, but historical states that already captured the old
outer-NULL payload semantics cannot be distinguished from valid states
and require refresh or rebuild to correct historical results.
### Release note
Fix `group_array_intersect` and `group_array_union` to ignore outer NULL
array rows consistently, and prevent NULL string elements in
`group_array_intersect` from matching masked nested values.
### Check List (For Author)
- Test:
- Build: `./build.sh --be --fe` (ASAN)
- Unit Test: `./run-be-ut.sh --run
--filter='AggGroupArrayIntersectTest.*'` (7 tests passed)
- Regression test: `query_p0/aggregate/group_array_intersect` (generated
output and rerun passed)
- Regression test:
`query_p0/sql_functions/aggregate_functions/test_aggregate_window_functions`
(generated output and rerun passed)
- Regression test: `nereids_syntax_p0/mv/load,agg_sync_mv` (generated
output and rerun passed)
- Regression test: the affected P1 MV assertions completed and generated
the expected results; the suite later hit the unrelated existing
`Duplicate column name '__stddev_1'` failure
- Format: `./build-support/check-format.sh`
- Static analysis: changed-line clang-tidy completed with no diagnostics
after local-only workarounds for the toolchain resource directory and an
unrelated baseline `NOLINTEND`
- Behavior changed: Yes (outer NULL array rows are wrapper-owned and
ignored, and NULL string elements no longer match masked nested values)
- Does this need documentation: No
---
.../aggregate_function_group_array_intersect.cpp | 2 +-
.../aggregate_function_group_array_set_op.h | 25 ++---
.../aggregate_function_group_array_set_op_impl.h | 4 +-
.../aggregate_function_group_array_union.cpp | 2 +-
.../aggregate/agg_group_array_intersect_test.cpp | 117 +++++++++++++++++++++
.../nereids_syntax_p0/mv/aggregate/agg_sync_mv.out | 4 +-
.../nereids_syntax_p1/mv/aggregate/agg_sync_mv.out | 4 +-
.../query_p0/aggregate/group_array_intersect.out | 14 ++-
.../test_aggregate_window_functions.out | 8 +-
.../aggregate/group_array_intersect.groovy | 32 ++++++
10 files changed, 179 insertions(+), 33 deletions(-)
diff --git
a/be/src/exprs/aggregate/aggregate_function_group_array_intersect.cpp
b/be/src/exprs/aggregate/aggregate_function_group_array_intersect.cpp
index 05a9fb53c2c..ca5211f8d85 100644
--- a/be/src/exprs/aggregate/aggregate_function_group_array_intersect.cpp
+++ b/be/src/exprs/aggregate/aggregate_function_group_array_intersect.cpp
@@ -37,7 +37,7 @@ AggregateFunctionPtr
create_aggregate_function_group_array_intersect(
}
return
create_aggregate_function_group_array_impl<GroupArrayNumericIntersectData,
GroupArrayStringIntersectData>(
- {argument_type}, result_is_nullable, attr);
+ argument_types, result_is_nullable, attr);
}
} // namespace doris
diff --git a/be/src/exprs/aggregate/aggregate_function_group_array_set_op.h
b/be/src/exprs/aggregate/aggregate_function_group_array_set_op.h
index 385503c514e..380cb4f6d2e 100644
--- a/be/src/exprs/aggregate/aggregate_function_group_array_set_op.h
+++ b/be/src/exprs/aggregate/aggregate_function_group_array_set_op.h
@@ -334,8 +334,10 @@ struct GroupArrayStringIntersectData : public
GroupArraySetOpStringBaseData {
} else if (!this->set->empty()) {
Set new_set = std::make_unique<NullableStringSet>();
for (size_t i = 0; i < arr_size; ++i) {
- if (col_null->is_null_at(offset + i) &&
this->set->contain_null()) {
- new_set->insert(nullptr);
+ if (col_null->is_null_at(offset + i)) {
+ if (this->set->contain_null()) {
+ new_set->insert(nullptr);
+ }
} else {
auto src = nested_column_data.get_data_at(offset + i);
if (this->set->find((void*)src.data, src.size)) {
@@ -443,16 +445,11 @@ public:
void reset(AggregateDataPtr __restrict place) const override {
this->data(place).reset(); }
void check_input_columns_type(const IColumn** columns) const override {
- const IColumn* column = columns[0];
- if (const auto* nullable_column =
check_and_get_column<ColumnNullable>(*column)) {
- column = &nullable_column->get_nested_column();
- }
-
- const auto* array_column = check_and_get_column<ColumnArray>(*column);
+ const auto* array_column =
check_and_get_column<ColumnArray>(*columns[0]);
if (UNLIKELY(array_column == nullptr)) {
throw doris::Exception(Status::InternalError(
"Aggregate function {} argument 0 type check failed:
Column type {} ({}) is "
- "not ColumnArray or Nullable(ColumnArray)",
+ "not ColumnArray",
get_name(), columns[0]->get_name(),
typeid(*columns[0]).name()));
}
@@ -507,14 +504,8 @@ public:
void add(AggregateDataPtr __restrict place, const IColumn** columns,
ssize_t row_num,
Arena& arena) const override {
- const bool col_is_nullable = is_column_nullable(*columns[0]);
- const ColumnArray& column =
- col_is_nullable
- ? assert_cast<const ColumnArray&,
TypeCheckOnRelease::DISABLE>(
- assert_cast<const ColumnNullable&,
TypeCheckOnRelease::DISABLE>(
- *columns[0])
- .get_nested_column())
- : assert_cast<const ColumnArray&,
TypeCheckOnRelease::DISABLE>(*columns[0]);
+ const auto& column =
+ assert_cast<const ColumnArray&,
TypeCheckOnRelease::DISABLE>(*columns[0]);
const auto& offsets = column.get_offsets();
const auto offset = offsets[row_num - 1];
diff --git
a/be/src/exprs/aggregate/aggregate_function_group_array_set_op_impl.h
b/be/src/exprs/aggregate/aggregate_function_group_array_set_op_impl.h
index b9b9ad8220c..167a29fca2d 100644
--- a/be/src/exprs/aggregate/aggregate_function_group_array_set_op_impl.h
+++ b/be/src/exprs/aggregate/aggregate_function_group_array_set_op_impl.h
@@ -33,8 +33,8 @@ template <template <PrimitiveType> class ImplNumericData,
typename ImplStringDat
inline AggregateFunctionPtr create_aggregate_function_group_array_impl(
const DataTypes& argument_types, const bool result_is_nullable,
const AggregateFunctionAttr& attr) {
- const auto& nested_type = remove_nullable(
- assert_cast<const
DataTypeArray&>(*(argument_types[0])).get_nested_type());
+ const auto& array_type = assert_cast<const
DataTypeArray&>(*remove_nullable(argument_types[0]));
+ const auto& nested_type = remove_nullable(array_type.get_nested_type());
auto pt = nested_type->get_primitive_type();
AggregateFunctionPtr result;
diff --git a/be/src/exprs/aggregate/aggregate_function_group_array_union.cpp
b/be/src/exprs/aggregate/aggregate_function_group_array_union.cpp
index 2f0303ca199..48861ae6552 100644
--- a/be/src/exprs/aggregate/aggregate_function_group_array_union.cpp
+++ b/be/src/exprs/aggregate/aggregate_function_group_array_union.cpp
@@ -36,7 +36,7 @@ AggregateFunctionPtr
create_aggregate_function_group_array_union(
}
return
create_aggregate_function_group_array_impl<GroupArrayNumericUnionData,
GroupArrayStringUnionData>(
- {argument_type}, result_is_nullable, attr);
+ argument_types, result_is_nullable, attr);
}
} // namespace doris
diff --git a/be/test/exprs/aggregate/agg_group_array_intersect_test.cpp
b/be/test/exprs/aggregate/agg_group_array_intersect_test.cpp
index 0be84b45890..00bcd45cc0a 100644
--- a/be/test/exprs/aggregate/agg_group_array_intersect_test.cpp
+++ b/be/test/exprs/aggregate/agg_group_array_intersect_test.cpp
@@ -19,13 +19,16 @@
#include <memory>
#include <string>
+#include "common/exception.h"
#include "core/column/column.h"
#include "core/column/column_array.h"
+#include "core/column/column_nullable.h"
#include "core/column/column_string.h"
#include "core/data_type/data_type_array.h"
#include "core/data_type/data_type_date.h"
#include "core/data_type/data_type_date_or_datetime_v2.h"
#include "core/data_type/data_type_date_time.h"
+#include "core/data_type/data_type_nullable.h"
#include "core/data_type/data_type_number.h"
#include "core/data_type/data_type_string.h"
#include "core/field.h"
@@ -526,4 +529,118 @@ TEST(AggGroupArrayIntersectTest, string_nullable_test) {
agg_function->destroy(place);
}
+TEST(AggGroupArrayIntersectTest,
string_null_element_does_not_match_empty_string) {
+ DataTypePtr array_type =
std::make_shared<DataTypeArray>(std::make_shared<DataTypeString>());
+ DataTypes data_types = {array_type};
+ auto test_column = array_type->create_column();
+ test_column->insert(
+
Field::create_field<TYPE_ARRAY>({Field::create_field<TYPE_STRING>(std::string())}));
+ test_column->insert(Field::create_field<TYPE_ARRAY>({Field()}));
+
+ AggregateFunctionSimpleFactory factory;
+ register_aggregate_function_group_array_set_op(factory);
+ auto agg_function = factory.get("group_array_intersect", data_types,
nullptr, false, -1);
+ ASSERT_NE(agg_function, nullptr);
+ std::unique_ptr<char[]> memory(new char[agg_function->size_of_data()]);
+ AggregateDataPtr place = memory.get();
+ agg_function->create(place);
+
+ Arena arena;
+ ColumnRawPtrs columns(data_types.size(), test_column.get());
+ agg_function->check_input_columns_type(columns.data());
+ agg_function->add_batch_single_place(test_column->size(), place,
columns.data(), arena);
+
+ auto result_column = array_type->create_column();
+ agg_function->insert_result_into(place, *result_column);
+ Field actual_field;
+ result_column->get(0, actual_field);
+ EXPECT_TRUE(actual_field.get<TYPE_ARRAY>().empty());
+
+ agg_function->destroy(place);
+}
+
+TEST(AggGroupArrayIntersectTest, raw_aggregate_rejects_outer_nullable_column) {
+ DataTypePtr array_type =
std::make_shared<DataTypeArray>(std::make_shared<DataTypeInt32>());
+ DataTypes data_types = {array_type};
+
+ AggregateFunctionSimpleFactory factory;
+ register_aggregate_function_group_array_set_op(factory);
+ auto agg_function = factory.get("group_array_intersect", data_types,
nullptr, false, -1);
+ ASSERT_NE(agg_function, nullptr);
+
+ auto outer_nullable_column =
std::make_shared<DataTypeNullable>(array_type)->create_column();
+ ColumnRawPtrs columns(data_types.size(), outer_nullable_column.get());
+ EXPECT_THROW(agg_function->check_input_columns_type(columns.data()),
Exception);
+}
+
+void validate_outer_nullable_array(const std::string& function_name, const
DataTypes& data_types,
+ const DataTypePtr& result_type, const
IColumn& input_column,
+ const Array& expected_result) {
+ for (bool enable_null_v2 : {false, true}) {
+ SCOPED_TRACE("function=" + function_name +
+ ", enable_aggregate_function_null_v2=" +
std::to_string(enable_null_v2));
+ AggregateFunctionSimpleFactory factory;
+ register_aggregate_function_group_array_set_op(factory);
+ AggregateFunctionAttr attr;
+ attr.enable_aggregate_function_null_v2 = enable_null_v2;
+ auto agg_function =
+ factory.get(function_name, data_types, nullptr, false, -1,
std::move(attr));
+ ASSERT_NE(agg_function, nullptr);
+ std::unique_ptr<char[]> memory(new char[agg_function->size_of_data()]);
+ AggregateDataPtr place = memory.get();
+ agg_function->create(place);
+
+ Arena arena;
+ ColumnRawPtrs columns(data_types.size(), &input_column);
+ agg_function->check_input_columns_type(columns.data());
+ agg_function->add_batch_single_place(input_column.size(), place,
columns.data(), arena);
+
+ auto result_column = result_type->create_column();
+ agg_function->insert_result_into(place, *result_column);
+ Field actual_field;
+ result_column->get(0, actual_field);
+ auto actual_result = actual_field.get<TYPE_ARRAY>();
+ auto sorted_expected_result = expected_result;
+ sort_numeric_array<TYPE_INT>(actual_result);
+ sort_numeric_array<TYPE_INT>(sorted_expected_result);
+ EXPECT_EQ(actual_result, sorted_expected_result);
+
+ agg_function->destroy(place);
+ }
+}
+
+TEST(AggGroupArrayIntersectTest, outer_nullable_array_test) {
+ auto nested_type =
std::make_shared<DataTypeNullable>(std::make_shared<DataTypeInt32>());
+ DataTypePtr array_type = std::make_shared<DataTypeArray>(nested_type);
+ DataTypes data_types = {std::make_shared<DataTypeNullable>(array_type)};
+ auto test_column = data_types[0]->create_column();
+ test_column->insert(Field::create_field<TYPE_ARRAY>(
+ {Field::create_field<TYPE_INT>(1),
Field::create_field<TYPE_INT>(2)}));
+ test_column->insert(Field());
+ test_column->insert(Field::create_field<TYPE_ARRAY>(
+ {Field::create_field<TYPE_INT>(2),
Field::create_field<TYPE_INT>(3)}));
+
+ validate_outer_nullable_array("group_array_intersect", data_types,
array_type, *test_column,
+ {Field::create_field<TYPE_INT>(2)});
+}
+
+TEST(AggGroupArrayIntersectTest, group_array_union_skips_outer_null_payload) {
+ auto nested_type =
std::make_shared<DataTypeNullable>(std::make_shared<DataTypeInt32>());
+ DataTypePtr array_type = std::make_shared<DataTypeArray>(nested_type);
+ DataTypes data_types = {std::make_shared<DataTypeNullable>(array_type)};
+ auto test_column = data_types[0]->create_column();
+ test_column->insert(Field::create_field<TYPE_ARRAY>(
+ {Field::create_field<TYPE_INT>(1),
Field::create_field<TYPE_INT>(2)}));
+
test_column->insert(Field::create_field<TYPE_ARRAY>({Field::create_field<TYPE_INT>(99)}));
+ test_column->insert(Field::create_field<TYPE_ARRAY>(
+ {Field::create_field<TYPE_INT>(2),
Field::create_field<TYPE_INT>(3)}));
+ auto& nullable_column = static_cast<ColumnNullable&>(*test_column);
+ nullable_column.get_null_map_data()[1] = 1;
+
+ validate_outer_nullable_array(
+ "group_array_union", data_types, array_type, *test_column,
+ {Field::create_field<TYPE_INT>(1),
Field::create_field<TYPE_INT>(2),
+ Field::create_field<TYPE_INT>(3)});
+}
+
} // namespace doris
diff --git
a/regression-test/data/nereids_syntax_p0/mv/aggregate/agg_sync_mv.out
b/regression-test/data/nereids_syntax_p0/mv/aggregate/agg_sync_mv.out
index 18af62dbfa7..f8243b031b1 100644
--- a/regression-test/data/nereids_syntax_p0/mv/aggregate/agg_sync_mv.out
+++ b/regression-test/data/nereids_syntax_p0/mv/aggregate/agg_sync_mv.out
@@ -420,7 +420,7 @@
11 1
-- !select_group_array_intersect --
-\N []
+\N [1]
0 [1]
1 [2]
2 [3]
@@ -435,7 +435,7 @@
11 [12]
-- !select_group_array_intersect_mv --
-\N []
+\N [1]
0 [1]
1 [2]
2 [3]
diff --git
a/regression-test/data/nereids_syntax_p1/mv/aggregate/agg_sync_mv.out
b/regression-test/data/nereids_syntax_p1/mv/aggregate/agg_sync_mv.out
index 848d637c916..33058f7d084 100644
--- a/regression-test/data/nereids_syntax_p1/mv/aggregate/agg_sync_mv.out
+++ b/regression-test/data/nereids_syntax_p1/mv/aggregate/agg_sync_mv.out
@@ -420,7 +420,7 @@
11 1
-- !select_group_array_intersect --
-\N []
+\N [1]
0 [1]
1 [2]
2 [3]
@@ -435,7 +435,7 @@
11 [12]
-- !select_group_array_intersect_mv --
-\N []
+\N [1]
0 [1]
1 [2]
2 [3]
diff --git a/regression-test/data/query_p0/aggregate/group_array_intersect.out
b/regression-test/data/query_p0/aggregate/group_array_intersect.out
index 07b64d8de3e..e4d691e58f0 100644
--- a/regression-test/data/query_p0/aggregate/group_array_intersect.out
+++ b/regression-test/data/query_p0/aggregate/group_array_intersect.out
@@ -1,4 +1,10 @@
-- This file is automatically generated. You should know what you did if you
want to edit this
+-- !outer_nullable_case_if --
+[2] [2]
+
+-- !string_null_element_does_not_match_empty_string --
+[]
+
-- !int_1 --
[null]
@@ -21,10 +27,10 @@
[null, 12, 13]
-- !int_8 --
-[]
+[null]
-- !int_9 --
-[]
+[null, 12, 13]
-- !float_1 --
[6.3, 7.3]
@@ -36,7 +42,7 @@
[7.3]
-- !datetimev2_1 --
-[]
+["2024-03-24 00:00:00.000"]
-- !datetimev2_2 --
["2024-03-24 00:00:00.000"]
@@ -48,7 +54,7 @@
["2024-05-23"]
-- !string_1 --
-[]
+[null, "a"]
-- !string_2 --
["a"]
diff --git
a/regression-test/data/query_p0/sql_functions/aggregate_functions/test_aggregate_window_functions.out
b/regression-test/data/query_p0/sql_functions/aggregate_functions/test_aggregate_window_functions.out
index 3c3f674ce5f..251162cd3ff 100644
---
a/regression-test/data/query_p0/sql_functions/aggregate_functions/test_aggregate_window_functions.out
+++
b/regression-test/data/query_p0/sql_functions/aggregate_functions/test_aggregate_window_functions.out
@@ -641,10 +641,10 @@ sichuan [{"cbe":{},"notnull":0,"null":1,"all":1}]
3 25
-- !agg_window_group_array_intersect --
-1 []
-1 []
-1 []
-1 []
+1 ["1"]
+1 ["1"]
+1 ["1"]
+1 ["1"]
2 ["2"]
2 ["2"]
2 ["2"]
diff --git
a/regression-test/suites/query_p0/aggregate/group_array_intersect.groovy
b/regression-test/suites/query_p0/aggregate/group_array_intersect.groovy
index 7492e533455..317cb13cbc7 100644
--- a/regression-test/suites/query_p0/aggregate/group_array_intersect.groovy
+++ b/regression-test/suites/query_p0/aggregate/group_array_intersect.groovy
@@ -48,6 +48,38 @@ suite("group_array_intersect") {
sql """INSERT INTO `group_array_intersect_test`(id, c_array_bigint) VALUES
(23, [1234567890123456]), (24, [1234567890123456, 2333333333333333]);"""
sql """INSERT INTO `group_array_intersect_test`(id, c_array_decimal)
VALUES (25, [1.34,2.00188888888888888]), (26, [1.34,2.00123344444455555]);"""
+ qt_outer_nullable_case_if """
+ SELECT
+ array_sort(group_array_intersect(
+ CASE id
+ WHEN 0 THEN [1, 2]
+ WHEN 1 THEN CAST(NULL AS ARRAY<INT>)
+ ELSE [2, 3]
+ END
+ )),
+ array_sort(group_array_intersect(
+ IF(id = 1, CAST(NULL AS ARRAY<INT>),
+ IF(id = 0, [1, 2], [2, 3]))
+ ))
+ FROM group_array_intersect_test
+ WHERE id BETWEEN 0 AND 2
+ """
+
+ qt_string_null_element_does_not_match_empty_string """
+ SELECT result
+ FROM (
+ SELECT id, array_sort(group_array_intersect(c_array_string) OVER (
+ ORDER BY id ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW
+ )) AS result
+ FROM (
+ SELECT 1 AS id, CAST([''] AS ARRAY<STRING>) AS c_array_string
+ UNION ALL
+ SELECT 2 AS id, CAST([NULL] AS ARRAY<STRING>) AS c_array_string
+ ) t
+ ) w
+ WHERE id = 2
+ """
+
qt_int_1 """select array_sort(group_array_intersect(c_array_int)) from
group_array_intersect_test where id in (6, 12);"""
qt_int_2 """select array_sort(group_array_intersect(c_array_int)) from
group_array_intersect_test where id in (14, 12);"""
qt_int_3 """select array_sort(group_array_intersect(c_array_int)) from
group_array_intersect_test where id in (0, 6);"""
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
