This is an automated email from the ASF dual-hosted git repository. yiguolei pushed a commit to branch branch-2.1 in repository https://gitbox.apache.org/repos/asf/doris.git
commit 8f40882701294b35d382b018f4496fc10c88eeea Author: minghong <[email protected]> AuthorDate: Sun Apr 28 15:33:22 2024 +0800 [fix](nereids) disable PROJECT_OTHER_JOIN_CONDITION rule if bitmap filter is enabled. #34189 this pr is a quick solution, but not complete. runtime filter on NestLoopJoin suffers this bug even without PROJECT_OTHER_JOIN_CONDITION rule. for example, when enable Min_Max Runtime filter, the target Expression is n_regionkey, but it should be "n_regionkey - 28" explain select n_nationkey, nrkey from (select n_regionkey -28 nrkey, n_nationkey from nation) T join region on nrkey > r_regionkey; we will refactor RuntimeFilterGenerator to completely solve this issue in following pr. --- ...ProjectOtherJoinConditionForNestedLoopJoin.java | 14 ++-- .../java/org/apache/doris/qe/SessionVariable.java | 4 + .../join/test_bitmap_filter_nereids.groovy | 87 ++++++++++------------ 3 files changed, 54 insertions(+), 51 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/ProjectOtherJoinConditionForNestedLoopJoin.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/ProjectOtherJoinConditionForNestedLoopJoin.java index 7a7146e451a..7d28269005c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/ProjectOtherJoinConditionForNestedLoopJoin.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/ProjectOtherJoinConditionForNestedLoopJoin.java @@ -26,6 +26,8 @@ import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.expressions.visitor.DefaultExpressionRewriter; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.logical.LogicalProject; +import org.apache.doris.qe.ConnectContext; +import org.apache.doris.thrift.TRuntimeFilterType; import com.google.common.collect.ImmutableList; import com.google.common.collect.Lists; @@ -49,12 +51,14 @@ import java.util.Set; public class ProjectOtherJoinConditionForNestedLoopJoin extends OneRewriteRuleFactory { @Override public Rule build() { - return logicalJoin().when( - join -> join.getHashJoinConjuncts().isEmpty() + return logicalJoin() + .when(join -> join.getHashJoinConjuncts().isEmpty() && !join.isMarkJoin() - && !join.getOtherJoinConjuncts().isEmpty() - ).then( - join -> { + && !join.getOtherJoinConjuncts().isEmpty()) + .whenNot(join -> ConnectContext.get() != null + && ConnectContext.get().getSessionVariable() + .allowedRuntimeFilterType(TRuntimeFilterType.BITMAP)) + .then(join -> { List<Expression> otherConjuncts = join.getOtherJoinConjuncts(); List<Expression> newOtherConjuncts = new ArrayList<>(); Set<Slot> leftSlots = join.child(0).getOutputSet(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java index 7ad72283d44..480d153ba5a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java @@ -2561,6 +2561,10 @@ public class SessionVariable implements Serializable, Writable { return runtimeFilterType; } + public boolean allowedRuntimeFilterType(TRuntimeFilterType type) { + return (runtimeFilterType & type.getValue()) != 0; + } + public boolean isRuntimeFilterTypeEnabled(TRuntimeFilterType type) { return (runtimeFilterType & type.getValue()) == type.getValue(); } diff --git a/regression-test/suites/query_p0/join/test_bitmap_filter_nereids.groovy b/regression-test/suites/query_p0/join/test_bitmap_filter_nereids.groovy index e219ce03f27..9557aada40d 100644 --- a/regression-test/suites/query_p0/join/test_bitmap_filter_nereids.groovy +++ b/regression-test/suites/query_p0/join/test_bitmap_filter_nereids.groovy @@ -16,84 +16,79 @@ // under the License. suite("test_bitmap_filter_nereids") { - def tbl1 = "test_query_db.bigtable" - def tbl2 = "bitmap_table_nereids" - def tbl3 = "test_query_db.baseall" - - sql "set runtime_filter_type = 16" - - sql "DROP TABLE IF EXISTS ${tbl2}" - sql """ - CREATE TABLE ${tbl2} ( - `k1` int(11) NULL, - `k2` bitmap BITMAP_UNION NULL, - `k3` bitmap BITMAP_UNION NULL - ) ENGINE=OLAP - AGGREGATE KEY(`k1`) - COMMENT 'OLAP' - DISTRIBUTED BY HASH(`k1`) BUCKETS 2 - PROPERTIES ( - "replication_allocation" = "tag.location.default: 1" - ); - """ - sql """ - insert into ${tbl2} values - (1, bitmap_from_string('1, 3, 5, 7, 9, 11, 13, 99, 19910811, 20150402'), - bitmap_from_string('32767, 1985, 255, 789, 1991')), - (2, bitmap_from_string('10, 11, 12, 13, 14'), bitmap_empty());""" - - sql "set enable_nereids_planner=true;" - sql "set enable_fallback_to_original_planner=false;" + multi_sql """ + set runtime_filter_type = 16; + DROP TABLE IF EXISTS bitmap_table_nereids; + + CREATE TABLE bitmap_table_nereids ( + `k1` int(11) NULL, + `k2` bitmap BITMAP_UNION NULL, + `k3` bitmap BITMAP_UNION NULL + ) ENGINE=OLAP + AGGREGATE KEY(`k1`) + COMMENT 'OLAP' + DISTRIBUTED BY HASH(`k1`) BUCKETS 2 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1" + ); + insert into bitmap_table_nereids values + (1, bitmap_from_string('1, 3, 5, 7, 9, 11, 13, 99, 19910811, 20150402'), + bitmap_from_string('32767, 1985, 255, 789, 1991')), + (2, bitmap_from_string('10, 11, 12, 13, 14'), bitmap_empty()); - qt_sql1 "select k1, k2 from ${tbl1} where k1 in (select k2 from ${tbl2}) order by k1;" + set enable_nereids_planner=true; + set enable_fallback_to_original_planner=false; + """ + + qt_sql1 "select k1, k2 from test_query_db.bigtable where k1 in (select k2 from bitmap_table_nereids) order by k1;" - qt_sql2 "select k1, k2 from ${tbl1} where k1 + 1 in (select k2 from ${tbl2}) order by k1;" + qt_sql2 "select k1, k2 from test_query_db.bigtable where k1 + 1 in (select k2 from bitmap_table_nereids) order by k1;" - qt_sql3 "select k1, k2 from ${tbl1} where k1 not in (select k2 from ${tbl2} where k1 = 1) order by k1;" + qt_sql3 "select k1, k2 from test_query_db.bigtable where k1 not in (select k2 from bitmap_table_nereids where k1 = 1) order by k1;" - qt_sql4 "select t1.k1, t1.k2 from ${tbl1} t1 join ${tbl3} t3 on t1.k1 = t3.k1 where t1.k1 in (select k2 from ${tbl2} where k1 = 1) order by t1.k1;" + qt_sql4 "select t1.k1, t1.k2 from test_query_db.bigtable t1 join test_query_db.baseall t3 on t1.k1 = t3.k1 where t1.k1 in (select k2 from bitmap_table_nereids where k1 = 1) order by t1.k1;" - qt_sql5 "select k1, k2 from ${tbl1} where k1 in (select k2 from ${tbl2}) and k2 not in (select k3 from ${tbl2}) order by k1;" + qt_sql5 "select k1, k2 from test_query_db.bigtable where k1 in (select k2 from bitmap_table_nereids) and k2 not in (select k3 from bitmap_table_nereids) order by k1;" - qt_sql6 "select k2, count(k2) from ${tbl1} where k1 in (select k2 from ${tbl2}) group by k2 order by k2;" + qt_sql6 "select k2, count(k2) from test_query_db.bigtable where k1 in (select k2 from bitmap_table_nereids) group by k2 order by k2;" - qt_sql7 "select k1, k2 from (select 2 k1, 2 k2) t where k1 in (select k2 from ${tbl2}) order by 1, 2;" + qt_sql7 "select k1, k2 from (select 2 k1, 2 k2) t where k1 in (select k2 from bitmap_table_nereids) order by 1, 2;" - qt_sql8 "select k1, k2 from (select 11 k1, 11 k2) t where k1 in (select k2 from ${tbl2}) order by 1, 2;" + qt_sql8 "select k1, k2 from (select 11 k1, 11 k2) t where k1 in (select k2 from bitmap_table_nereids) order by 1, 2;" - qt_sql9 "select k1, k2 from (select 2 k1, 11 k2) t where k1 not in (select k2 from ${tbl2}) order by 1, 2;" + qt_sql9 "select k1, k2 from (select 2 k1, 11 k2) t where k1 not in (select k2 from bitmap_table_nereids) order by 1, 2;" - qt_sql10 "select k1, k2 from (select 1 k1, 11 k2) t where k1 not in (select k2 from ${tbl2}) order by 1, 2;" + qt_sql10 "select k1, k2 from (select 1 k1, 11 k2) t where k1 not in (select k2 from bitmap_table_nereids) order by 1, 2;" - qt_sql11 "select k10 from ${tbl1} where cast(k10 as bigint) in (select bitmap_or(k2, to_bitmap(20120314)) from ${tbl2} b) order by 1;" + qt_sql11 "select k10 from test_query_db.bigtable where cast(k10 as bigint) in (select bitmap_or(k2, to_bitmap(20120314)) from bitmap_table_nereids b) order by 1;" qt_sql12 """ - with w1 as (select k1 from ${tbl1} where k1 in (select k2 from ${tbl2})), w2 as (select k2 from ${tbl1} where k2 in (select k3 from ${tbl2})) + with w1 as (select k1 from test_query_db.bigtable where k1 in (select k2 from bitmap_table_nereids)), w2 as (select k2 from test_query_db.bigtable where k2 in (select k3 from bitmap_table_nereids)) select * from (select * from w1 union select * from w2) tmp order by 1; """ - qt_sql13 "select k1, k2 from ${tbl1} where k1 in (select to_bitmap(10)) order by 1, 2" + qt_sql13 "select k1, k2 from test_query_db.bigtable where k1 in (select to_bitmap(10)) order by 1, 2" - qt_sql14 "select k1, k2 from ${tbl1} where k1 in (select bitmap_from_string('1,10')) order by 1, 2" + qt_sql14 "select k1, k2 from test_query_db.bigtable where k1 in (select bitmap_from_string('1,10')) order by 1, 2" test { - sql "select k1, count(*) from ${tbl1} b1 group by k1 having k1 in (select k2 from ${tbl2} b2) order by k1;" + sql "select k1, count(*) from test_query_db.bigtable b1 group by k1 having k1 in (select k2 from bitmap_table_nereids b2) order by k1;" exception "Doris hll, bitmap, array, map, struct, jsonb, variant column must use with specific function, and don't support filter" } sql "set ignore_storage_data_distribution=false" explain{ - sql "select k1, k2 from ${tbl1} where k1 in (select k2 from ${tbl2}) order by k1;" + sql "select k1, k2 from test_query_db.bigtable where k1 in (select k2 from bitmap_table_nereids) order by k1;" contains "RF000[bitmap]" } explain{ - sql "select k1, k2 from ${tbl1} where k1 not in (select k2 from ${tbl2} where k1 = 1)" + sql "select k1, k2 from test_query_db.bigtable where k1 not in (select k2 from bitmap_table_nereids where k1 = 1)" contains "RF000[bitmap]" } explain{ - sql " select k1, k2 from (select 2 k1, 2 k2) t where k1 in (select k2 from ${tbl2})" + sql " select k1, k2 from (select 2 k1, 2 k2) t where k1 in (select k2 from bitmap_table_nereids)" notContains "RF000[bitmap]" } } --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
