This is an automated email from the ASF dual-hosted git repository.
kxiao pushed a commit to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.0 by this push:
new b35b7922a06 [Pick 2.0](inverted index) fix wrong no need read data
when need_remaining_after_evaluate #36637 (#36685)
b35b7922a06 is described below
commit b35b7922a06e41555d99996266f450d089fa701b
Author: airborne12 <[email protected]>
AuthorDate: Sun Jun 23 19:04:40 2024 +0800
[Pick 2.0](inverted index) fix wrong no need read data when
need_remaining_after_evaluate #36637 (#36685)
---
be/src/olap/rowset/segment_v2/segment_iterator.cpp | 1 +
.../data/inverted_index_p0/test_need_read_data.out | 21 ++++
.../inverted_index_p0/test_need_read_data.groovy | 113 +++++++++++++++++++++
3 files changed, 135 insertions(+)
diff --git a/be/src/olap/rowset/segment_v2/segment_iterator.cpp
b/be/src/olap/rowset/segment_v2/segment_iterator.cpp
index d57f24ef5ca..2fabf724c9c 100644
--- a/be/src/olap/rowset/segment_v2/segment_iterator.cpp
+++ b/be/src/olap/rowset/segment_v2/segment_iterator.cpp
@@ -1009,6 +1009,7 @@ Status
SegmentIterator::_apply_inverted_index_on_column_predicate(
if (need_remaining_after_evaluate) {
remaining_predicates.emplace_back(pred);
+ _need_read_data_indices[pred->column_id()] = true;
return Status::OK();
}
diff --git a/regression-test/data/inverted_index_p0/test_need_read_data.out
b/regression-test/data/inverted_index_p0/test_need_read_data.out
new file mode 100644
index 00000000000..fdfe790afdb
--- /dev/null
+++ b/regression-test/data/inverted_index_p0/test_need_read_data.out
@@ -0,0 +1,21 @@
+-- This file is automatically generated. You should know what you did if you
want to edit this
+-- !sql --
+8 \N
+4 -10
+13 -4
+2 1
+3 2
+3 3
+5 4
+5 5
+1 6
+1 7
+4 8
+1 9
+
+-- !sql1 --
+1
+
+-- !sql2 --
+2024-06-17T15:16:49 tengxun2
+
diff --git
a/regression-test/suites/inverted_index_p0/test_need_read_data.groovy
b/regression-test/suites/inverted_index_p0/test_need_read_data.groovy
new file mode 100644
index 00000000000..694b7856be7
--- /dev/null
+++ b/regression-test/suites/inverted_index_p0/test_need_read_data.groovy
@@ -0,0 +1,113 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+
+suite("test_need_read_data", "p0"){
+ def indexTbName1 = "test_need_read_data"
+
+ sql "DROP TABLE IF EXISTS ${indexTbName1}"
+
+ sql """
+ create table ${indexTbName1} (
+ col_int_undef_signed_not_null int not null ,
+ col_bigint_undef_signed_not_null_index_inverted bigint not null ,
+ col_bigint_undef_signed_not_null bigint not null ,
+ col_int_undef_signed int null ,
+ col_int_undef_signed_index_inverted int null ,
+ col_int_undef_signed_not_null_index_inverted int not null ,
+ col_bigint_undef_signed bigint null ,
+ col_bigint_undef_signed_index_inverted bigint null ,
+ col_date_undef_signed date null ,
+ col_date_undef_signed_index_inverted date null ,
+ col_date_undef_signed_not_null date not null ,
+ col_date_undef_signed_not_null_index_inverted date not null ,
+ col_varchar_10__undef_signed varchar(10) null ,
+ col_varchar_10__undef_signed_index_inverted varchar(10) null ,
+ col_varchar_10__undef_signed_not_null varchar(10) not null ,
+ col_varchar_10__undef_signed_not_null_index_inverted varchar(10) not
null ,
+ col_varchar_1024__undef_signed varchar(1024) null ,
+ col_varchar_1024__undef_signed_index_inverted varchar(1024) null ,
+ col_varchar_1024__undef_signed_not_null varchar(1024) not null ,
+ col_varchar_1024__undef_signed_not_null_index_inverted varchar(1024)
not null ,
+ pk int,
+ INDEX col_int_undef_signed_index_inverted_idx
(`col_int_undef_signed_index_inverted`) USING INVERTED,
+ INDEX col_int_undef_signed_not_null_index_inverted_idx
(`col_int_undef_signed_not_null_index_inverted`) USING INVERTED,
+ INDEX col_bigint_undef_signed_index_inverted_idx
(`col_bigint_undef_signed_index_inverted`) USING INVERTED,
+ INDEX col_bigint_undef_signed_not_null_index_inverted_idx
(`col_bigint_undef_signed_not_null_index_inverted`) USING INVERTED,
+ INDEX col_date_undef_signed_index_inverted_idx
(`col_date_undef_signed_index_inverted`) USING INVERTED,
+ INDEX col_date_undef_signed_not_null_index_inverted_idx
(`col_date_undef_signed_not_null_index_inverted`) USING INVERTED,
+ INDEX col_varchar_10__undef_signed_index_inverted_idx
(`col_varchar_10__undef_signed_index_inverted`) USING INVERTED,
+ INDEX col_varchar_10__undef_signed_not_null_index_inverted_idx
(`col_varchar_10__undef_signed_not_null_index_inverted`) USING INVERTED,
+ INDEX col_varchar_1024__undef_signed_index_inverted_idx
(`col_varchar_1024__undef_signed_index_inverted`) USING INVERTED,
+ INDEX col_varchar_1024__undef_signed_not_null_index_inverted_idx
(`col_varchar_1024__undef_signed_not_null_index_inverted`) USING INVERTED
+ ) engine=olap
+ UNIQUE KEY(col_int_undef_signed_not_null,
col_bigint_undef_signed_not_null_index_inverted,
col_bigint_undef_signed_not_null)
+ PARTITION BY RANGE(col_int_undef_signed_not_null) (
+ PARTITION p0 VALUES LESS THAN ('4'),
+ PARTITION p1 VALUES LESS THAN ('6'),
+ PARTITION p2 VALUES LESS THAN ('7'),
+ PARTITION p3 VALUES LESS THAN ('8'),
+ PARTITION p4 VALUES LESS THAN ('10'),
+ PARTITION p5 VALUES LESS THAN ('1147483647'),
+ PARTITION p100 VALUES LESS THAN ('2147483647')
+ )
+ distributed by hash(col_bigint_undef_signed_not_null)
+ properties("enable_unique_key_merge_on_write" = "true",
"replication_num" = "1");
+ """
+
+ try {
+ sql """ insert into
${indexTbName1}(pk,col_int_undef_signed,col_int_undef_signed_index_inverted,col_int_undef_signed_not_null,col_int_undef_signed_not_null_index_inverted,col_bigint_undef_signed,col_bigint_undef_signed_index_inverted,col_bigint_undef_signed_not_null,col_bigint_undef_signed_not_null_index_inverted,col_date_undef_signed,col_date_undef_signed_index_inverted,col_date_undef_signed_not_null,col_date_undef_signed_not_null_index_inverted,col_varchar_10__undef_signed,col_
[...]
+
+ sql "sync"
+
+ qt_sql """ SELECT COUNT( *) AS field1, col_int_undef_signed AS field2
FROM ${indexTbName1} WHERE( col_date_undef_signed_not_null_index_inverted ==
'2024-01-01' OR day( col_date_undef_signed_not_null_index_inverted ) !=0 )
GROUP BY field2 ORDER BY field2; """
+
+ } finally {
+ //try_sql("DROP TABLE IF EXISTS ${testTable}")
+ }
+
+ def indexTbName2 = "test_need_read_data_2"
+
+ sql "DROP TABLE IF EXISTS ${indexTbName2}"
+
+ sql """
+ create table ${indexTbName2} (
+ a datetime not null,
+ b varchar not null,
+ INDEX idx_inverted_b (`b`) USING INVERTED PROPERTIES("parser" =
"unicode", "support_phrase" = "true") COMMENT ''
+ ) ENGINE=OLAP
+ DUPLICATE KEY(`a`)
+ COMMENT ''
+ DISTRIBUTED BY HASH(`a`) BUCKETS 1
+ PROPERTIES (
+ "replication_allocation" = "tag.location.default: 1"
+ );
+ """
+ sql """insert into ${indexTbName2} values
+ ('2024-06-03 15:16:49.266678','shanghai'),
+ ('2024-06-02 15:16:49.266678','shenzhen'),
+ ('2024-06-01 15:16:49.266678','beijing'),
+ ('2024-06-13 15:16:49.266678','beijing'),
+ ('2024-06-14 15:16:49.266678','beijing'),
+ ('2024-06-15 15:16:49.266678','shanghai'),
+ ('2024-06-16 15:16:49.266678','tengxun'),
+ ('2024-06-17 15:16:49.266678','tengxun2')
+ """
+
+ qt_sql1 """ select COUNT(1) from ${indexTbName2} WHERE a >=
'2024-06-15 00:00:00' AND b = 'tengxun2' and `b` match 'tengxun2' ;
"""
+ qt_sql2 """ select * from ${indexTbName2} WHERE a >= '2024-06-15
00:00:00' AND b = 'tengxun2' and `b` match 'tengxun2' ; """
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
