This is an automated email from the ASF dual-hosted git repository.
gavinchou pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new e6128536d78 [Feature](array) Support order by array column (#52361)
e6128536d78 is described below
commit e6128536d787801efbd21567bd6e4537df0c292d
Author: Gabriel <[email protected]>
AuthorDate: Thu Jun 26 23:50:16 2025 +0800
[Feature](array) Support order by array column (#52361)
---
be/src/vec/columns/column_array.cpp | 44 +++++++++++++++++++++
be/src/vec/columns/column_array.h | 2 +
be/test/vec/columns/column_array_test.cpp | 11 ------
.../java/org/apache/doris/analysis/QueryStmt.java | 2 +-
.../nereids/rules/analysis/CheckAfterRewrite.java | 6 ++-
.../nereids/rules/rewrite/EliminateOrderByKey.java | 3 +-
.../data/correctness_p0/test_array_order_by.out | Bin 0 -> 845 bytes
.../correctness_p0/test_array_order_by.groovy | 39 ++++++++++++++++++
.../one_level_nestedtypes_with_s3data.groovy | 3 --
.../aggregate_group_by_metric_type.groovy | 9 -----
10 files changed, 92 insertions(+), 27 deletions(-)
diff --git a/be/src/vec/columns/column_array.cpp
b/be/src/vec/columns/column_array.cpp
index 9b0d2a14902..f8c5a7f8531 100644
--- a/be/src/vec/columns/column_array.cpp
+++ b/be/src/vec/columns/column_array.cpp
@@ -177,6 +177,50 @@ StringRef ColumnArray::serialize_value_into_arena(size_t
n, Arena& arena,
return res;
}
+template <bool positive>
+struct less {
+ const ColumnArray& parent;
+ const int nan_direction_hint;
+ explicit less(const ColumnArray& parent_, int nan_direction_hint_)
+ : parent(parent_), nan_direction_hint(nan_direction_hint_) {}
+ bool operator()(size_t lhs, size_t rhs) const {
+ size_t lhs_size = parent.size_at(lhs);
+ size_t rhs_size = parent.size_at(rhs);
+ size_t min_size = std::min(lhs_size, rhs_size);
+ int res = 0;
+ for (size_t i = 0; i < min_size; ++i) {
+ if (res = parent.get_data().compare_at(
+ parent.offset_at(lhs) + i, parent.offset_at(rhs) + i,
+ *parent.get_data_ptr().get(), nan_direction_hint);
+ res) {
+ // if res != 0 , here is something different ,just return
+ break;
+ }
+ }
+ if (res == 0) {
+ // then we check size of array
+ res = lhs_size < rhs_size ? -1 : (lhs_size == rhs_size ? 0 : 1);
+ }
+
+ return positive ? (res < 0) : (res > 0);
+ }
+};
+
+void ColumnArray::get_permutation(bool reverse, size_t limit, int
nan_direction_hint,
+ IColumn::Permutation& res) const {
+ size_t s = size();
+ res.resize(s);
+ for (size_t i = 0; i < s; ++i) {
+ res[i] = i;
+ }
+
+ if (reverse) {
+ pdqsort(res.begin(), res.end(), less<false>(*this,
nan_direction_hint));
+ } else {
+ pdqsort(res.begin(), res.end(), less<true>(*this, nan_direction_hint));
+ }
+}
+
int ColumnArray::compare_at(size_t n, size_t m, const IColumn& rhs_, int
nan_direction_hint) const {
// since column type is complex, we can't use this function
const auto& rhs = assert_cast<const ColumnArray&,
TypeCheckOnRelease::DISABLE>(rhs_);
diff --git a/be/src/vec/columns/column_array.h
b/be/src/vec/columns/column_array.h
index 072c4098771..3d80304d2dd 100644
--- a/be/src/vec/columns/column_array.h
+++ b/be/src/vec/columns/column_array.h
@@ -166,6 +166,8 @@ public:
bool has_enough_capacity(const IColumn& src) const override;
ColumnPtr replicate(const IColumn::Offsets& replicate_offsets) const
override;
void insert_many_from(const IColumn& src, size_t position, size_t length)
override;
+ void get_permutation(bool reverse, size_t limit, int nan_direction_hint,
+ IColumn::Permutation& res) const override;
/** More efficient methods of manipulation */
IColumn& get_data() { return *data; }
diff --git a/be/test/vec/columns/column_array_test.cpp
b/be/test/vec/columns/column_array_test.cpp
index 3512da899d5..10eec203c5a 100644
--- a/be/test/vec/columns/column_array_test.cpp
+++ b/be/test/vec/columns/column_array_test.cpp
@@ -556,17 +556,6 @@ TEST_F(ColumnArrayTest, AppendDataBySelectorTest) {
assert_append_data_by_selector_callback(array_columns, serdes);
}
-TEST_F(ColumnArrayTest, PermutationAndSortTest) {
- for (int i = 0; i < array_columns.size(); i++) {
- auto& column = array_columns[i];
- auto& type = array_types[i];
- auto column_type = type->get_name();
- LOG(INFO) << "column_type: " << column_type;
- // permutation
-
EXPECT_ANY_THROW(assert_column_permutations(column->assume_mutable_ref(),
type));
- }
-}
-
TEST_F(ColumnArrayTest, FilterInplaceTest) {
// The filter method implemented by column_array does not achieve the
memory reuse acceleration effect like other basic types,
// and still returns a new ptr, which can be make a todo task
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/QueryStmt.java
b/fe/fe-core/src/main/java/org/apache/doris/analysis/QueryStmt.java
index ece3694e7ff..8f3c7209b72 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/QueryStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/QueryStmt.java
@@ -340,7 +340,7 @@ public abstract class QueryStmt extends StatementBase
implements Queriable, NotF
}
for (Expr expr : orderingExprs) {
- if (expr.getType().isOnlyMetricType()) {
+ if (expr.getType().isOnlyMetricType() &&
!expr.getType().isArrayType()) {
throw new AnalysisException(Type.OnlyMetricTypeErrorMsg);
}
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CheckAfterRewrite.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CheckAfterRewrite.java
index 06fb3b94829..d8b0ce0e23f 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CheckAfterRewrite.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CheckAfterRewrite.java
@@ -158,14 +158,16 @@ public class CheckAfterRewrite extends
OneAnalysisRuleFactory {
} else if (plan instanceof LogicalSort) {
LogicalSort<?> sort = (LogicalSort<?>) plan;
for (OrderKey orderKey : sort.getOrderKeys()) {
- if (orderKey.getExpr().getDataType().isOnlyMetricType()) {
+ if (orderKey.getExpr().getDataType().isOnlyMetricType()
+ && !orderKey.getExpr().getDataType().isArrayType()) {
throw new AnalysisException(Type.OnlyMetricTypeErrorMsg);
}
}
} else if (plan instanceof LogicalTopN) {
LogicalTopN<?> topN = (LogicalTopN<?>) plan;
for (OrderKey orderKey : topN.getOrderKeys()) {
- if (orderKey.getExpr().getDataType().isOnlyMetricType()) {
+ if (orderKey.getExpr().getDataType().isOnlyMetricType()
+ && !orderKey.getExpr().getDataType().isArrayType()) {
throw new AnalysisException(Type.OnlyMetricTypeErrorMsg);
}
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/EliminateOrderByKey.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/EliminateOrderByKey.java
index d4ede4deafe..5cf27b000cb 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/EliminateOrderByKey.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/EliminateOrderByKey.java
@@ -82,7 +82,8 @@ public class EliminateOrderByKey implements
RewriteRuleFactory {
WindowExpression windowExpression = (WindowExpression)
alias.child();
List<OrderExpression> orderExpressions =
windowExpression.getOrderKeys();
if (orderExpressions.stream().anyMatch((
- orderKey -> orderKey.getDataType().isOnlyMetricType()))) {
+ orderKey -> orderKey.getDataType().isOnlyMetricType()
+ && !orderKey.getDataType().isArrayType()))) {
throw new AnalysisException(Type.OnlyMetricTypeErrorMsg);
}
List<OrderKey> orderKeys = new ArrayList<>();
diff --git a/regression-test/data/correctness_p0/test_array_order_by.out
b/regression-test/data/correctness_p0/test_array_order_by.out
new file mode 100644
index 00000000000..137d4698955
Binary files /dev/null and
b/regression-test/data/correctness_p0/test_array_order_by.out differ
diff --git a/regression-test/suites/correctness_p0/test_array_order_by.groovy
b/regression-test/suites/correctness_p0/test_array_order_by.groovy
new file mode 100644
index 00000000000..b104ec3e78f
--- /dev/null
+++ b/regression-test/suites/correctness_p0/test_array_order_by.groovy
@@ -0,0 +1,39 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+// The cases is copied from https://github.com/trinodb/trino/tree/master
+//
/testing/trino-product-tests/src/main/resources/sql-tests/testcases/aggregate
+// and modified by Doris.
+
+suite("test_array_order_by") {
+ sql """drop table if exists test_array_order_by;"""
+ sql """CREATE TABLE test_array_order_by(
+ typ_id BIGINT NOT NULL COMMENT "ID",
+ name VARCHAR(20) NULL COMMENT "名称",
+ arr ARRAY<int(10)> NULL COMMENT "数组"
+ )
+ DUPLICATE KEY(typ_id)
+ DISTRIBUTED BY HASH(typ_id) BUCKETS 10
+ PROPERTIES ("replication_allocation" =
"tag.location.default: 1");"""
+ sql """insert into test_array_order_by values(1,'name1',NULL),
(1,'name2',[1,2,3,4,5]), (1,'name3',[-1,2,-2]), (1,'name4',[6]),
(1,'name2',[1,2,3,4,5]), (1,'name2',[1,2,3]);"""
+ qt_select1 """ select * from test_array_order_by order by arr ASC; """
+ qt_select2 """ select * from test_array_order_by order by arr DESC; """
+ qt_select3 """ select * from test_array_order_by order by name,arr ASC;
"""
+ qt_select4 """ select * from test_array_order_by order by name,arr DESC;
"""
+ qt_select5 """ select * from test_array_order_by order by typ_id,arr ASC;
"""
+ qt_select6 """ select * from test_array_order_by order by typ_id,arr DESC;
"""
+}
diff --git
a/regression-test/suites/datatype_p0/nested_types/base_cases/one_level_nestedtypes_with_s3data.groovy
b/regression-test/suites/datatype_p0/nested_types/base_cases/one_level_nestedtypes_with_s3data.groovy
index 13891e1380e..c56215c2f9c 100644
---
a/regression-test/suites/datatype_p0/nested_types/base_cases/one_level_nestedtypes_with_s3data.groovy
+++
b/regression-test/suites/datatype_p0/nested_types/base_cases/one_level_nestedtypes_with_s3data.groovy
@@ -180,7 +180,6 @@ suite("one_level_nestedtypes_with_s3data") {
// select * from table where groupby|orderby column will meet exception
for (String col : colNameArr) {
groupby_or_orderby_exception(true, table_names[0], col)
- groupby_or_orderby_exception(false, table_names[0], col)
}
// select * from table where groupby|orderby element_at(column)
for (String col : colNameArr) {
@@ -222,7 +221,6 @@ suite("one_level_nestedtypes_with_s3data") {
// select * from table where groupby|orderby column will meet exception
for (String col : colNameArr) {
groupby_or_orderby_exception(true, table_names[1], col)
- groupby_or_orderby_exception(false, table_names[1], col)
}
// select * from table where groupby|orderby element_at(column)
for (String col : colNameArr) {
@@ -274,7 +272,6 @@ suite("one_level_nestedtypes_with_s3data") {
// select * from table where groupby|orderby column will meet exception
groupby_or_orderby_exception(true, table_names[2], colNameArr[0])
- groupby_or_orderby_exception(false, table_names[2], colNameArr[0])
// select * from table where groupby|orderby element_at(column)
String agg_expr = "struct_element(${colNameArr[0]}, 1)"
diff --git
a/regression-test/suites/query_p0/aggregate/aggregate_group_by_metric_type.groovy
b/regression-test/suites/query_p0/aggregate/aggregate_group_by_metric_type.groovy
index c62a6f0132a..e480663de8e 100644
---
a/regression-test/suites/query_p0/aggregate/aggregate_group_by_metric_type.groovy
+++
b/regression-test/suites/query_p0/aggregate/aggregate_group_by_metric_type.groovy
@@ -36,11 +36,6 @@ suite("aggregate_group_by_metric_type") {
exception "${error_msg}"
}
- test {
- sql "select user_ids from test_group_by_hll_and_bitmap order by
user_ids"
- exception "${error_msg}"
- }
-
test {
sql "select hll_set from test_group_by_hll_and_bitmap order by hll_set"
exception "${error_msg}"
@@ -81,10 +76,6 @@ suite("aggregate_group_by_metric_type") {
sql "select distinct c_array from test_group_by_array"
exception "${error_msg}"
}
- test {
- sql "select c_array from test_group_by_array order by c_array"
- exception "${error_msg}"
- }
test {
sql "select c_array,count(*) from test_group_by_array group by c_array"
exception "${error_msg}"
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
