This is an automated email from the ASF dual-hosted git repository. yiguolei pushed a commit to branch branch-4.2 in repository https://gitbox.apache.org/repos/asf/doris.git
commit 0673e0daa1682bb68f1d20af681e20d656240b87 Author: HappenLee <[email protected]> AuthorDate: Tue Sep 29 09:27:59 2026 +0800 [fix](be) Backport Boolean top-N support to branch-4.1 (#68409) ### What problem does this PR solve? Related PR: #68254 Backport #68254 (dbea64bcec932fdbd7aa43576953eae1a167b7c0) to `branch-4.1`. The FE already accepts BOOLEAN inputs for `topn_array` and `topn_weighted`, but the BE factory lists omit BOOLEAN, so queries such as `SELECT topn_weighted(TRUE, 1, 2)` fail during aggregate creation. Add `TYPE_BOOLEAN` to both existing creator lists, covering both arities. Resolve the conflicting lists against branch-4.1 by retaining its existing supported types; do not import master's `TYPE_TIMESTAMP_NS`. Reuse branch-4.1's numeric aggregation, nullable handling, serialization and merge behavior. Preserve the BOOLEAN array result type and existing FE signatures. Retain the upstream BE unit tests, SQL regression suite and generated expected output unchanged. ### Release note `TOPN_WEIGHTED` and `TOPN_ARRAY` now execute on BOOLEAN inputs and preserve BOOLEAN array results. ### Check List (For Author) - Test: - C++ syntax and template-instantiation checks passed for both modified factories and the new BE unit test, using this worktree's ASAN unit-test compilation database and PCH. - Attempted `./run-be-ut.sh --run -j48 --filter='AggregateFunctionTopNBooleanTest.*:AggTest.topn*'`. After building the missing simdutf dependency with the existing thirdparty script, the build remained blocked by the environment's older AWS SDK: `common/cpp/aws_common.cpp` cannot find `aws/core/auth/GeneralHTTPCredentialsProvider.h`. The tests did not run. - SQL regression tests were not rerun for this backport. - `build-support/clang-format.sh` and `build-support/check-format.sh` passed with clang-format 16.0.6. - All three test files are byte-identical to merged PR #68254; the existing generated output was preserved, not handwritten. - Behavior changed: Yes. BOOLEAN top-N array aggregates execute instead of failing BE function creation. - Does this need documentation: No. --- .../aggregate/aggregate_function_topn_array.cpp | 9 +- .../aggregate/aggregate_function_topn_weighted.cpp | 9 +- be/test/exprs/aggregate/agg_topn_boolean_test.cpp | 117 +++++++++++++++++++++ .../agg_function/topn/topn_boolean.out | 27 +++++ .../agg_function/topn/topn_boolean.groovy | 78 ++++++++++++++ 5 files changed, 230 insertions(+), 10 deletions(-) diff --git a/be/src/exprs/aggregate/aggregate_function_topn_array.cpp b/be/src/exprs/aggregate/aggregate_function_topn_array.cpp index a63239e9b2b..4e61d4eaf5a 100644 --- a/be/src/exprs/aggregate/aggregate_function_topn_array.cpp +++ b/be/src/exprs/aggregate/aggregate_function_topn_array.cpp @@ -28,11 +28,10 @@ using ImplArray = AggregateFunctionTopNImplArray<T, false>; template <PrimitiveType T> using ImplArrayWithDefault = AggregateFunctionTopNImplArray<T, true>; -using topn_array_creator = - creator_with_type_list<TYPE_TINYINT, TYPE_SMALLINT, TYPE_INT, TYPE_BIGINT, TYPE_LARGEINT, - TYPE_FLOAT, TYPE_DOUBLE, TYPE_DECIMAL32, TYPE_DECIMAL64, - TYPE_DECIMAL128I, TYPE_DECIMAL256, TYPE_VARCHAR, TYPE_DATEV2, - TYPE_DATETIMEV2, TYPE_TIMESTAMPTZ, TYPE_IPV4, TYPE_IPV6>; +using topn_array_creator = creator_with_type_list< + TYPE_BOOLEAN, TYPE_TINYINT, TYPE_SMALLINT, TYPE_INT, TYPE_BIGINT, TYPE_LARGEINT, TYPE_FLOAT, + TYPE_DOUBLE, TYPE_DECIMAL32, TYPE_DECIMAL64, TYPE_DECIMAL128I, TYPE_DECIMAL256, + TYPE_VARCHAR, TYPE_DATEV2, TYPE_DATETIMEV2, TYPE_TIMESTAMPTZ, TYPE_IPV4, TYPE_IPV6>; AggregateFunctionPtr create_aggregate_function_topn_array(const std::string& name, const DataTypes& argument_types, diff --git a/be/src/exprs/aggregate/aggregate_function_topn_weighted.cpp b/be/src/exprs/aggregate/aggregate_function_topn_weighted.cpp index 8e93406e6d7..92dd6ac54a5 100644 --- a/be/src/exprs/aggregate/aggregate_function_topn_weighted.cpp +++ b/be/src/exprs/aggregate/aggregate_function_topn_weighted.cpp @@ -28,11 +28,10 @@ using ImplWeight = AggregateFunctionTopNImplWeight<T, false>; template <PrimitiveType T> using ImplWeightWithDefault = AggregateFunctionTopNImplWeight<T, true>; -using topn_weighted_creator = - creator_with_type_list<TYPE_TINYINT, TYPE_SMALLINT, TYPE_INT, TYPE_BIGINT, TYPE_LARGEINT, - TYPE_FLOAT, TYPE_DOUBLE, TYPE_DECIMAL32, TYPE_DECIMAL64, - TYPE_DECIMAL128I, TYPE_DECIMAL256, TYPE_VARCHAR, TYPE_DATEV2, - TYPE_DATETIMEV2, TYPE_TIMESTAMPTZ, TYPE_IPV4, TYPE_IPV6>; +using topn_weighted_creator = creator_with_type_list< + TYPE_BOOLEAN, TYPE_TINYINT, TYPE_SMALLINT, TYPE_INT, TYPE_BIGINT, TYPE_LARGEINT, TYPE_FLOAT, + TYPE_DOUBLE, TYPE_DECIMAL32, TYPE_DECIMAL64, TYPE_DECIMAL128I, TYPE_DECIMAL256, + TYPE_VARCHAR, TYPE_DATEV2, TYPE_DATETIMEV2, TYPE_TIMESTAMPTZ, TYPE_IPV4, TYPE_IPV6>; AggregateFunctionPtr create_aggregate_function_topn_weighted(const std::string& name, const DataTypes& argument_types, diff --git a/be/test/exprs/aggregate/agg_topn_boolean_test.cpp b/be/test/exprs/aggregate/agg_topn_boolean_test.cpp new file mode 100644 index 00000000000..ea811ac2f29 --- /dev/null +++ b/be/test/exprs/aggregate/agg_topn_boolean_test.cpp @@ -0,0 +1,117 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include <gtest/gtest.h> + +#include "agent/be_exec_version_manager.h" +#include "core/data_type/data_type_array.h" +#include "core/data_type/data_type_nullable.h" +#include "core/data_type/data_type_number.h" +#include "exprs/aggregate/agg_function_test.h" +#include "exprs/aggregate/aggregate_function_simple_factory.h" + +namespace doris { + +struct AggregateFunctionTopNBooleanTest : public AggregateFunctiontest { + void check_topn(ColumnWithTypeAndName values, const std::vector<Int64>& weights, int top_num, + const Field& expected_array, bool weighted, bool expanded) { + const std::string name = weighted ? "topn_weighted" : "topn_array"; + SCOPED_TRACE(name + (expanded ? " with expansion rate" : " without expansion rate")); + SCOPED_TRACE(top_num); + Block block({values}); + DataTypes argument_types {values.type}; + if (weighted) { + auto weight_column = ColumnHelper::create_column_with_name<DataTypeInt64>(weights); + block.insert(weight_column); + argument_types.push_back(weight_column.type); + } + auto top_column = ColumnHelper::create_column_with_name<DataTypeInt32>( + std::vector<Int32>(values.column->size(), top_num)); + block.insert(top_column); + argument_types.push_back(top_column.type); + if (expanded) { + auto rate_column = ColumnHelper::create_column_with_name<DataTypeInt32>( + std::vector<Int32>(values.column->size(), 50)); + block.insert(rate_column); + argument_types.push_back(rate_column.type); + } + + const bool nullable = values.type->is_nullable(); + DataTypePtr result_type = + std::make_shared<DataTypeArray>(make_nullable(std::make_shared<DataTypeBool>())); + if (nullable) { + result_type = make_nullable(result_type); + } + auto function = AggregateFunctionSimpleFactory::instance().get( + name, argument_types, result_type, nullable, + BeExecVersionManager::get_newest_version()); + ASSERT_NE(function, nullptr); + EXPECT_TRUE(function->get_return_type()->equals(*result_type)); + + auto expected_column = result_type->create_column(); + expected_column->insert(expected_array); + create_agg(name, nullable, argument_types, result_type); + execute(block, ColumnWithTypeAndName(std::move(expected_column), result_type, "expected")); + } + + static Field boolean_array(std::initializer_list<UInt8> values) { + Array array; + for (auto value : values) { + array.push_back(Field::create_field<TYPE_BOOLEAN>(value)); + } + return Field::create_field<TYPE_ARRAY>(std::move(array)); + } +}; + +TEST_F(AggregateFunctionTopNBooleanTest, ValuesAndLimits) { + for (bool weighted : {false, true}) { + for (bool expanded : {false, true}) { + auto values = ColumnHelper::create_column_with_name<DataTypeBool>({false, false, true}); + check_topn(values, {1, 1, 5}, 1, boolean_array({weighted}), weighted, expanded); + for (int top_num : {2, 3}) { + check_topn(values, {1, 1, 5}, top_num, boolean_array({weighted, !weighted}), + weighted, expanded); + } + } + } +} + +TEST_F(AggregateFunctionTopNBooleanTest, NullableValues) { + for (bool weighted : {false, true}) { + for (bool expanded : {false, true}) { + auto values = ColumnHelper::create_nullable_column_with_name<DataTypeBool>( + {false, false, true, true}, {0, 0, 0, 1}); + check_topn(values, {1, 1, 5, 100}, 2, boolean_array({weighted, !weighted}), weighted, + expanded); + + auto nulls = ColumnHelper::create_nullable_column_with_name<DataTypeBool>({false, true}, + {1, 1}); + check_topn(nulls, {1, 5}, 2, Field(), weighted, expanded); + } + } +} + +TEST_F(AggregateFunctionTopNBooleanTest, Ties) { + for (bool weighted : {false, true}) { + for (bool expanded : {false, true}) { + auto values = ColumnHelper::create_column_with_name<DataTypeBool>({false, true}); + check_topn(values, {3, 3}, 2, boolean_array({true, false}), weighted, expanded); + } + } +} + +} // namespace doris diff --git a/regression-test/data/nereids_function_p0/agg_function/topn/topn_boolean.out b/regression-test/data/nereids_function_p0/agg_function/topn/topn_boolean.out new file mode 100644 index 00000000000..a83bb17bdb2 --- /dev/null +++ b/regression-test/data/nereids_function_p0/agg_function/topn/topn_boolean.out @@ -0,0 +1,27 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !literals -- +[1] [0] [1] [0] + +-- !null_literals -- +\N \N \N \N + +-- !weighted -- +[1] [1] [1, 0] [1, 0] [1, 0] [1, 0] + +-- !array -- +[0] [0] [0, 1] [0, 1] [0, 1] [0, 1] + +-- !grouped -- +\N \N \N \N \N +false [0] [0] [0] [0] +true [1] [1] [1] [1] + +-- !all_null -- +\N \N \N \N + +-- !empty -- +\N \N \N \N + +-- !ties -- +[1, 0] [1, 0] [1, 0] [1, 0] + diff --git a/regression-test/suites/nereids_function_p0/agg_function/topn/topn_boolean.groovy b/regression-test/suites/nereids_function_p0/agg_function/topn/topn_boolean.groovy new file mode 100644 index 00000000000..72b78245638 --- /dev/null +++ b/regression-test/suites/nereids_function_p0/agg_function/topn/topn_boolean.groovy @@ -0,0 +1,78 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("topn_boolean") { + order_qt_literals """ + SELECT topn_weighted(TRUE, 1, 2), topn_weighted(FALSE, 1, 2, 50), + topn_array(TRUE, 2), topn_array(FALSE, 2, 50) + """ + order_qt_null_literals """ + SELECT topn_weighted(CAST(NULL AS BOOLEAN), 1, 2), + topn_weighted(CAST(NULL AS BOOLEAN), 1, 2, 50), + topn_array(CAST(NULL AS BOOLEAN), 2), + topn_array(CAST(NULL AS BOOLEAN), 2, 50) + """ + + sql "DROP TABLE IF EXISTS test_topn_boolean" + sql """ + CREATE TABLE test_topn_boolean ( + id INT, + flag BOOLEAN, + weight BIGINT + ) DUPLICATE KEY(id) + DISTRIBUTED BY HASH(id) BUCKETS 4 + PROPERTIES ("replication_num" = "1") + """ + sql """ + INSERT INTO test_topn_boolean VALUES + (1, FALSE, 1), (2, FALSE, 1), (3, TRUE, 5), + (4, NULL, 100), (5, TRUE, NULL) + """ + + order_qt_weighted """ + SELECT topn_weighted(flag, weight, 1), topn_weighted(flag, weight, 1, 50), + topn_weighted(flag, weight, 2), topn_weighted(flag, weight, 2, 50), + topn_weighted(flag, weight, 3), topn_weighted(flag, weight, 3, 50) + FROM test_topn_boolean + """ + order_qt_array """ + SELECT topn_array(flag, 1), topn_array(flag, 1, 50), + topn_array(flag, 2), topn_array(flag, 2, 50), + topn_array(flag, 3), topn_array(flag, 3, 50) + FROM test_topn_boolean WHERE weight IS NOT NULL + """ + order_qt_grouped """ + SELECT flag, topn_weighted(flag, weight, 2), topn_weighted(flag, weight, 2, 50), + topn_array(flag, 2), topn_array(flag, 2, 50) + FROM test_topn_boolean GROUP BY flag + """ + order_qt_all_null """ + SELECT topn_weighted(flag, weight, 2), topn_weighted(flag, weight, 2, 50), + topn_array(flag, 2), topn_array(flag, 2, 50) + FROM test_topn_boolean WHERE flag IS NULL + """ + order_qt_empty """ + SELECT topn_weighted(flag, weight, 2), topn_weighted(flag, weight, 2, 50), + topn_array(flag, 2), topn_array(flag, 2, 50) + FROM test_topn_boolean WHERE id < 0 + """ + order_qt_ties """ + SELECT topn_weighted(flag, 1, 2), topn_weighted(flag, 1, 2, 50), + topn_array(flag, 2), topn_array(flag, 2, 50) + FROM test_topn_boolean WHERE id IN (1, 3) + """ +} --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
