This is an automated email from the ASF dual-hosted git repository. kxiao pushed a commit to branch branch-2.0 in repository https://gitbox.apache.org/repos/asf/doris.git
commit 1c65098e97687faddf59041a6e3bb6659d9432bb Author: Pxl <[email protected]> AuthorDate: Mon Aug 28 18:01:08 2023 +0800 [Bug](materialized-view) fix divide double can not match mv (#23504) * fix divide double can not match mv * fix * fix --- .../mv/AbstractSelectMaterializedIndexRule.java | 9 ++-- .../trees/expressions/literal/DoubleLiteral.java | 9 ++++ .../trees/plans/logical/LogicalProject.java | 1 - .../expression/SimplifyArithmeticRuleTest.java | 18 ++++---- .../data/mv_p0/test_dup_mv_div/test_dup_mv_div.out | 13 ++++++ .../nereids_tpcds_shape_sf100_p0/shape/query11.out | 2 +- .../nereids_tpcds_shape_sf100_p0/shape/query39.out | 2 +- .../nereids_tpcds_shape_sf100_p0/shape/query73.out | 2 +- .../nereids_tpcds_shape_sf100_p0/shape/query74.out | 6 +-- .../mv_p0/test_dup_mv_div/test_dup_mv_div.groovy | 52 ++++++++++++++++++++++ 10 files changed, 95 insertions(+), 19 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/mv/AbstractSelectMaterializedIndexRule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/mv/AbstractSelectMaterializedIndexRule.java index e91c685fc2..58b3d10574 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/mv/AbstractSelectMaterializedIndexRule.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/mv/AbstractSelectMaterializedIndexRule.java @@ -148,7 +148,7 @@ public abstract class AbstractSelectMaterializedIndexRule { } protected static boolean containsAllColumn(Expression expression, Set<String> mvColumnNames) { - if (mvColumnNames.contains(Column.getNameWithoutMvPrefix(expression.toSql()))) { + if (mvColumnNames.contains(expression.toSql())) { return true; } if (expression.children().isEmpty()) { @@ -400,7 +400,7 @@ public abstract class AbstractSelectMaterializedIndexRule { for (Slot mvSlot : mvPlan.getOutputByIndex(mvPlan.getSelectedIndexId())) { boolean isPushed = false; for (Slot baseSlot : mvPlan.getOutput()) { - if (org.apache.doris.analysis.CreateMaterializedViewStmt.isMVColumnAggregate(mvSlot.getName())) { + if (org.apache.doris.analysis.CreateMaterializedViewStmt.isMVColumn(mvSlot.getName())) { continue; } if (baseSlot.toSql().equalsIgnoreCase( @@ -412,7 +412,7 @@ public abstract class AbstractSelectMaterializedIndexRule { } } if (!isPushed) { - if (org.apache.doris.analysis.CreateMaterializedViewStmt.isMVColumnAggregate(mvSlot.getName())) { + if (org.apache.doris.analysis.CreateMaterializedViewStmt.isMVColumn(mvSlot.getName())) { mvNameToMvSlot.put(normalizeName( org.apache.doris.analysis.CreateMaterializedViewStmt.mvColumnBreaker(mvSlot.getName())), mvSlot); @@ -570,6 +570,9 @@ public abstract class AbstractSelectMaterializedIndexRule { if (baseSlotToMvSlot.containsKey(slotReference)) { return baseSlotToMvSlot.get(slotReference); } + if (mvNameToMvSlot.containsKey(slotReference.toSql())) { + return mvNameToMvSlot.get(slotReference.toSql()); + } return slotReference; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DoubleLiteral.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DoubleLiteral.java index 34b8ca36fb..bdd26460c0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DoubleLiteral.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DoubleLiteral.java @@ -23,6 +23,8 @@ import org.apache.doris.catalog.Type; import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; import org.apache.doris.nereids.types.DoubleType; +import java.text.NumberFormat; + /** * Double literal */ @@ -49,4 +51,11 @@ public class DoubleLiteral extends Literal { public LiteralExpr toLegacyLiteral() { return new FloatLiteral(value, Type.DOUBLE); } + + @Override + public String toString() { + NumberFormat nf = NumberFormat.getInstance(); + nf.setGroupingUsed(false); + return nf.format(value); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalProject.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalProject.java index 2d1e2db3cc..6e5ecce26e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalProject.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalProject.java @@ -232,7 +232,6 @@ public class LogicalProject<CHILD_TYPE extends Plan> extends LogicalUnary<CHILD_ } public LogicalProject<Plan> readFromJson(JSONObject logicalProject) { - return new LogicalProject<>(ImmutableList.of(new UnboundStar(ImmutableList.of())), null, null, isDistinct); } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/SimplifyArithmeticRuleTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/SimplifyArithmeticRuleTest.java index 8b9f48010a..567073254a 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/SimplifyArithmeticRuleTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/SimplifyArithmeticRuleTest.java @@ -36,7 +36,7 @@ public class SimplifyArithmeticRuleTest extends ExpressionRewriteTestHelper { assertRewriteAfterTypeCoercion("IA", "IA"); assertRewriteAfterTypeCoercion("IA + 1", "IA + 1"); assertRewriteAfterTypeCoercion("IA + IB", "IA + IB"); - assertRewriteAfterTypeCoercion("1 * 3 / IA", "(3.0 / cast(IA as DOUBLE))"); + assertRewriteAfterTypeCoercion("1 * 3 / IA", "(3 / cast(IA as DOUBLE))"); assertRewriteAfterTypeCoercion("1 - IA", "1 - IA"); assertRewriteAfterTypeCoercion("1 + 1", "2"); assertRewriteAfterTypeCoercion("IA + 2 - 1", "IA + 1"); @@ -44,12 +44,12 @@ public class SimplifyArithmeticRuleTest extends ExpressionRewriteTestHelper { assertRewriteAfterTypeCoercion("IA + 2 - ((1 - IB) - (3 + IC))", "IA + IB + IC + 4"); assertRewriteAfterTypeCoercion("IA * IB + 2 - IC * 2", "(IA * IB) - (IC * 2) + 2"); assertRewriteAfterTypeCoercion("IA * IB", "IA * IB"); - assertRewriteAfterTypeCoercion("IA * IB / 2 * 2", "cast((IA * IB) as DOUBLE) / 1.0"); - assertRewriteAfterTypeCoercion("IA * IB / (2 * 2)", "cast((IA * IB) as DOUBLE) / 4.0"); - assertRewriteAfterTypeCoercion("IA * IB / (2 * 2)", "cast((IA * IB) as DOUBLE) / 4.0"); - assertRewriteAfterTypeCoercion("IA * (IB / 2) * 2)", "cast(IA as DOUBLE) * cast(IB as DOUBLE) / 1.0"); - assertRewriteAfterTypeCoercion("IA * (IB / 2) * (IC + 1))", "cast(IA as DOUBLE) * cast(IB as DOUBLE) * cast((IC + 1) as DOUBLE) / 2.0"); - assertRewriteAfterTypeCoercion("IA * IB / 2 / IC * 2 * ID / 4", "(((cast((IA * IB) as DOUBLE) / cast(IC as DOUBLE)) * cast(ID as DOUBLE)) / 4.0)"); + assertRewriteAfterTypeCoercion("IA * IB / 2 * 2", "cast((IA * IB) as DOUBLE) / 1"); + assertRewriteAfterTypeCoercion("IA * IB / (2 * 2)", "cast((IA * IB) as DOUBLE) / 4"); + assertRewriteAfterTypeCoercion("IA * IB / (2 * 2)", "cast((IA * IB) as DOUBLE) / 4"); + assertRewriteAfterTypeCoercion("IA * (IB / 2) * 2)", "cast(IA as DOUBLE) * cast(IB as DOUBLE) / 1"); + assertRewriteAfterTypeCoercion("IA * (IB / 2) * (IC + 1))", "cast(IA as DOUBLE) * cast(IB as DOUBLE) * cast((IC + 1) as DOUBLE) / 2"); + assertRewriteAfterTypeCoercion("IA * IB / 2 / IC * 2 * ID / 4", "(((cast((IA * IB) as DOUBLE) / cast(IC as DOUBLE)) * cast(ID as DOUBLE)) / 4)"); } @Test @@ -86,8 +86,8 @@ public class SimplifyArithmeticRuleTest extends ExpressionRewriteTestHelper { assertRewriteAfterTypeCoercion("IA + 1 > IB", "cast(IA as BIGINT) > (cast(IB as BIGINT) - 1)"); assertRewriteAfterTypeCoercion("IA + 1 > IB * IC", "cast(IA as BIGINT) > ((IB * IC) - 1)"); assertRewriteAfterTypeCoercion("IA * ID > IB * IC", "IA * ID > IB * IC"); - assertRewriteAfterTypeCoercion("IA * ID / 2 > IB * IC", "cast((IA * ID) as DOUBLE) > cast((IB * IC) as DOUBLE) * 2.0"); - assertRewriteAfterTypeCoercion("IA * ID / -2 > IB * IC", "cast((IB * IC) as DOUBLE) * -2.0 > cast((IA * ID) as DOUBLE)"); + assertRewriteAfterTypeCoercion("IA * ID / 2 > IB * IC", "cast((IA * ID) as DOUBLE) > cast((IB * IC) as DOUBLE) * 2"); + assertRewriteAfterTypeCoercion("IA * ID / -2 > IB * IC", "cast((IB * IC) as DOUBLE) * -2 > cast((IA * ID) as DOUBLE)"); } } diff --git a/regression-test/data/mv_p0/test_dup_mv_div/test_dup_mv_div.out b/regression-test/data/mv_p0/test_dup_mv_div/test_dup_mv_div.out new file mode 100644 index 0000000000..64c881ed10 --- /dev/null +++ b/regression-test/data/mv_p0/test_dup_mv_div/test_dup_mv_div.out @@ -0,0 +1,13 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !select_star -- +-4 -4 -4 4 +1 1 1 1 +2 2 2 b +3 -3 \N c + +-- !select_mv -- +-4 -4.0 +1 1.0 +2 2.0 +3 -3.0 + diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query11.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query11.out index ddb622b554..345dcf525d 100644 --- a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query11.out +++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query11.out @@ -45,7 +45,7 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) ------PhysicalDistribute --------PhysicalTopN ----------PhysicalProject -------------hashJoin[INNER_JOIN](t_s_firstyear.customer_id = t_w_secyear.customer_id)(CASE WHEN (year_total > 0.00) THEN (cast(year_total as DOUBLE) / cast(year_total as DOUBLE)) ELSE 0.0 END > CASE WHEN (year_total > 0.00) THEN (cast(year_total as DOUBLE) / cast(year_total as DOUBLE)) ELSE 0.0 END) +------------hashJoin[INNER_JOIN](t_s_firstyear.customer_id = t_w_secyear.customer_id)(CASE WHEN (year_total > 0.00) THEN (cast(year_total as DOUBLE) / cast(year_total as DOUBLE)) ELSE 0 END > CASE WHEN (year_total > 0.00) THEN (cast(year_total as DOUBLE) / cast(year_total as DOUBLE)) ELSE 0 END) --------------hashJoin[INNER_JOIN](t_s_secyear.customer_id = t_s_firstyear.customer_id) ----------------hashJoin[INNER_JOIN](t_s_firstyear.customer_id = t_w_firstyear.customer_id) ------------------PhysicalDistribute diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query39.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query39.out index a35d2c59bb..2bac250532 100644 --- a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query39.out +++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query39.out @@ -3,7 +3,7 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) --PhysicalCteProducer ( cteId=CTEId#0 ) ----PhysicalProject -------filter((CASE WHEN (mean = 0.0) THEN 0.0 ELSE (stdev / mean) END > 1.0)) +------filter((CASE WHEN (mean = 0) THEN 0 ELSE (stdev / mean) END > 1)) --------hashAgg[GLOBAL] ----------PhysicalDistribute ------------hashAgg[LOCAL] diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query73.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query73.out index 61342aa658..34a39fb42f 100644 --- a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query73.out +++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query73.out @@ -25,7 +25,7 @@ PhysicalResultSink ------------------------------------PhysicalOlapScan[date_dim] ----------------------------PhysicalDistribute ------------------------------PhysicalProject ---------------------------------filter(((cast(hd_buy_potential as VARCHAR(*)) = '501-1000') OR (cast(hd_buy_potential as VARCHAR(*)) = 'Unknown'))(household_demographics.hd_vehicle_count > 0)(CASE WHEN (hd_vehicle_count > 0) THEN (cast(hd_dep_count as DOUBLE) / cast(hd_vehicle_count as DOUBLE)) ELSE NULL END > 1.0)) +--------------------------------filter(((cast(hd_buy_potential as VARCHAR(*)) = '501-1000') OR (cast(hd_buy_potential as VARCHAR(*)) = 'Unknown'))(household_demographics.hd_vehicle_count > 0)(CASE WHEN (hd_vehicle_count > 0) THEN (cast(hd_dep_count as DOUBLE) / cast(hd_vehicle_count as DOUBLE)) ELSE NULL END > 1)) ----------------------------------PhysicalOlapScan[household_demographics] --------------------------PhysicalDistribute ----------------------------PhysicalProject diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query74.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query74.out index 0ec828207d..cb6953fa83 100644 --- a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query74.out +++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query74.out @@ -45,12 +45,12 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) ------PhysicalDistribute --------PhysicalTopN ----------PhysicalProject -------------hashJoin[INNER_JOIN](t_s_firstyear.customer_id = t_w_firstyear.customer_id)(CASE WHEN (year_total > 0.0) THEN (year_total / year_total) ELSE NULL END > CASE WHEN (year_total > 0.0) THEN (year_total / year_total) ELSE NULL END) +------------hashJoin[INNER_JOIN](t_s_firstyear.customer_id = t_w_firstyear.customer_id)(CASE WHEN (year_total > 0) THEN (year_total / year_total) ELSE NULL END > CASE WHEN (year_total > 0) THEN (year_total / year_total) ELSE NULL END) --------------hashJoin[INNER_JOIN](t_s_firstyear.customer_id = t_w_secyear.customer_id) ----------------hashJoin[INNER_JOIN](t_s_secyear.customer_id = t_s_firstyear.customer_id) ------------------PhysicalDistribute --------------------PhysicalProject -----------------------filter((t_s_firstyear.year = 1999)(t_s_firstyear.sale_type = 's')(t_s_firstyear.year_total > 0.0)) +----------------------filter((t_s_firstyear.year = 1999)(t_s_firstyear.sale_type = 's')(t_s_firstyear.year_total > 0)) ------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) ------------------PhysicalDistribute --------------------PhysicalProject @@ -62,6 +62,6 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) ----------------------PhysicalCteConsumer ( cteId=CTEId#0 ) --------------PhysicalDistribute ----------------PhysicalProject -------------------filter((t_w_firstyear.year = 1999)(t_w_firstyear.year_total > 0.0)(t_w_firstyear.sale_type = 'w')) +------------------filter((t_w_firstyear.year = 1999)(t_w_firstyear.year_total > 0)(t_w_firstyear.sale_type = 'w')) --------------------PhysicalCteConsumer ( cteId=CTEId#0 ) diff --git a/regression-test/suites/mv_p0/test_dup_mv_div/test_dup_mv_div.groovy b/regression-test/suites/mv_p0/test_dup_mv_div/test_dup_mv_div.groovy new file mode 100644 index 0000000000..17d2fbf2e5 --- /dev/null +++ b/regression-test/suites/mv_p0/test_dup_mv_div/test_dup_mv_div.groovy @@ -0,0 +1,52 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.codehaus.groovy.runtime.IOGroovyMethods + +suite ("test_dup_mv_div") { + sql """set enable_nereids_planner=true""" + sql """SET enable_fallback_to_original_planner=false""" + sql """ DROP TABLE IF EXISTS d_table; """ + + sql """ + create table d_table( + k1 int null, + k2 int not null, + k3 bigint null, + k4 varchar(100) null + ) + duplicate key (k1,k2,k3) + distributed BY hash(k1) buckets 3 + properties("replication_num" = "1"); + """ + + sql "insert into d_table select 1,1,1,'1';" + sql "insert into d_table select 2,2,2,'b';" + sql "insert into d_table select 3,-3,null,'c';" + + createMV ("create materialized view kdiv as select k1,k2/1 from d_table;") + + sql "insert into d_table select -4,-4,-4,'4';" + + qt_select_star "select * from d_table order by k1;" + + explain { + sql("select k1,k2/1 from d_table order by k1;") + contains "(kdiv)" + } + qt_select_mv "select k1,k2/1 from d_table order by k1;" +} --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
