This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 76c1d9fafed [fix](topn) Fix key topn block reverse is missed in some
cases (#31199)
76c1d9fafed is described below
commit 76c1d9fafed9f69313c51ee434f511a94318c0a8
Author: Kang <[email protected]>
AuthorDate: Wed Feb 21 17:42:58 2024 +0800
[fix](topn) Fix key topn block reverse is missed in some cases (#31199)
* move reverse block row order operation after _next_batch_internal
* add testcase
---
be/src/olap/rowset/segment_v2/segment_iterator.cpp | 40 ++++++++++-------
.../data/query_p0/topn/test_key_topn.out | 28 ++++++++++++
.../suites/query_p0/topn/test_key_topn.groovy | 51 ++++++++++++++++++++++
3 files changed, 103 insertions(+), 16 deletions(-)
diff --git a/be/src/olap/rowset/segment_v2/segment_iterator.cpp
b/be/src/olap/rowset/segment_v2/segment_iterator.cpp
index 98e1fe0d82e..8cc25d39898 100644
--- a/be/src/olap/rowset/segment_v2/segment_iterator.cpp
+++ b/be/src/olap/rowset/segment_v2/segment_iterator.cpp
@@ -2062,7 +2062,30 @@ Status
SegmentIterator::_read_columns_by_rowids(std::vector<ColumnId>& read_colu
}
Status SegmentIterator::next_batch(vectorized::Block* block) {
- auto status = [&]() { RETURN_IF_CATCH_EXCEPTION({ return
_next_batch_internal(block); }); }();
+ auto status = [&]() {
+ RETURN_IF_CATCH_EXCEPTION({
+ RETURN_IF_ERROR(_next_batch_internal(block));
+
+ // reverse block row order if read_orderby_key_reverse is true for
key topn
+ // it should be processed for all success _next_batch_internal
+ if (_opts.read_orderby_key_reverse) {
+ size_t num_rows = block->rows();
+ if (num_rows == 0) {
+ return Status::OK();
+ }
+ size_t num_columns = block->columns();
+ vectorized::IColumn::Permutation permutation;
+ for (size_t i = 0; i < num_rows; ++i)
permutation.emplace_back(num_rows - 1 - i);
+
+ for (size_t i = 0; i < num_columns; ++i)
+ block->get_by_position(i).column =
+
block->get_by_position(i).column->permute(permutation, num_rows);
+ }
+
+ return Status::OK();
+ });
+ }();
+
// if rows read by batch is 0, will return end of file, we should not
remove segment cache in this situation.
if (!status.ok() && !status.is<END_OF_FILE>()) {
_segment->remove_from_segment_cache();
@@ -2364,21 +2387,6 @@ Status
SegmentIterator::_next_batch_internal(vectorized::Block* block) {
#endif
VLOG_DEBUG << "dump block " << block->dump_data(0, block->rows());
- // reverse block row order
- if (_opts.read_orderby_key_reverse) {
- size_t num_rows = block->rows();
- if (num_rows == 0) {
- return Status::OK();
- }
- size_t num_columns = block->columns();
- vectorized::IColumn::Permutation permutation;
- for (size_t i = 0; i < num_rows; ++i)
permutation.emplace_back(num_rows - 1 - i);
-
- for (size_t i = 0; i < num_columns; ++i)
- block->get_by_position(i).column =
- block->get_by_position(i).column->permute(permutation,
num_rows);
- }
-
return Status::OK();
}
diff --git a/regression-test/data/query_p0/topn/test_key_topn.out
b/regression-test/data/query_p0/topn/test_key_topn.out
new file mode 100644
index 00000000000..a180cfb1148
--- /dev/null
+++ b/regression-test/data/query_p0/topn/test_key_topn.out
@@ -0,0 +1,28 @@
+-- This file is automatically generated. You should know what you did if you
want to edit this
+-- !test1 --
+1000
+
+-- !test2 --
+8703592868269757830
+8690150785744825388
+
+-- !test3 --
+8703592868269757830
+8690150785744825388
+
+-- !test4 --
+8703592868269757830
+8690150785744825388
+
+-- !test5 --
+-8597110388149732177
+-8591867649781947679
+
+-- !test6 --
+-8597110388149732177
+-8591867649781947679
+
+-- !test7 --
+-8597110388149732177
+-8591867649781947679
+
diff --git a/regression-test/suites/query_p0/topn/test_key_topn.groovy
b/regression-test/suites/query_p0/topn/test_key_topn.groovy
new file mode 100644
index 00000000000..e612ac74d68
--- /dev/null
+++ b/regression-test/suites/query_p0/topn/test_key_topn.groovy
@@ -0,0 +1,51 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+suite("test_key_topn", "p0") {
+
+ sql """ set topn_opt_limit_threshold = 1024; """
+ sql """ set enable_two_phase_read_opt = 1; """
+
+ sql """
+ drop table if exists `test_key_topn`;
+ """
+
+ sql """
+ CREATE TABLE `test_key_topn` (
+ `c1` BIGINT NULL,
+ `c2` BIGINT NULL,
+ `c3` INT NULL
+ ) ENGINE=OLAP
+ DUPLICATE KEY(`c1`, `c2`, `c3`)
+ COMMENT 'OLAP'
+ DISTRIBUTED BY HASH(`c3`) BUCKETS 10
+ PROPERTIES("replication_num" = "1");
+ """
+
+ sql """
+ INSERT INTO test_key_topn(c3,c1,c2) VALUES
(0,-8,-140632),(1,6400896,null),(2,-17129,19576),(3,-114,-8367860),(4,18583,-30),(5,null,24605),(6,-16288,5457699124954153990),(7,null,-2324180),(8,97,7223589),(9,-5257668201870192722,-9315),(10,-5500234776311337555,889748246483125914),(11,2936629,null),(12,5209683942987364396,-11834),(13,-85,null),(14,6630530972987800192,1126696919469584977),(15,7870120482580890039,-3952227040791679072),(16,12488,-3244574625629861581),(17,-4637167646579
[...]
+ """
+
+ qt_test1 """ SELECT count() FROM test_key_topn; """
+
+ qt_test2 """ SELECT c1 FROM test_key_topn AS T1 ORDER BY c1 DESC LIMIT 9,
2; """
+ qt_test3 """ SELECT c1 FROM test_key_topn AS T1 WHERE (T1.c1 > 4) ORDER BY
c1 DESC LIMIT 9, 2; """
+ qt_test4 """ SELECT c1 FROM test_key_topn AS T1 WHERE (T1.c1 <> 4) ORDER
BY c1 DESC LIMIT 9, 2; """
+
+ qt_test5 """ SELECT c1 FROM test_key_topn AS T1 WHERE T1.c1 IS NOT NULL
ORDER BY c1 ASC LIMIT 9, 2; """
+ qt_test6 """ SELECT c1 FROM test_key_topn AS T1 WHERE (T1.c1 < 4) ORDER BY
c1 ASC LIMIT 9, 2; """
+ qt_test7 """ SELECT c1 FROM test_key_topn AS T1 WHERE (T1.c1 <> 4) ORDER
BY c1 ASC LIMIT 9, 2; """
+}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
