This is an automated email from the ASF dual-hosted git repository.
morrysnow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new dd869077f8 [fix](nereids) do not generate compare between Date to Date
(#16061)
dd869077f8 is described below
commit dd869077f8ce3c2b19f29b3048c4e7e62ac31050
Author: minghong <[email protected]>
AuthorDate: Thu Jan 19 15:56:51 2023 +0800
[fix](nereids) do not generate compare between Date to Date (#16061)
BE storage Engine has some bug in Date comparison, and hence if we push
down predicates like Date'x' < Date 'y', we get error results.
This pr just convert expr like ’Date'x' < Date 'y',‘ to DateTime'x' <
DateTime 'y'
TODO:
do storage engine support date slot compare with datetime?
if it support, we could avoid add cast on the slot
and then, this expression could push down to storage engine.
---
.../rewrite/rules/SimplifyComparisonPredicate.java | 45 +++++++++++++++++++++
.../expression/rewrite/ExpressionRewriteTest.java | 14 ++++---
.../data/nereids_syntax_p0/test_cast_datetime.out | 4 ++
.../nereids_syntax_p0/test_cast_datetime.groovy | 46 ++++++++++++++++++++++
4 files changed, 103 insertions(+), 6 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rewrite/rules/SimplifyComparisonPredicate.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rewrite/rules/SimplifyComparisonPredicate.java
index 0eb957515b..5308e08fb9 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rewrite/rules/SimplifyComparisonPredicate.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rewrite/rules/SimplifyComparisonPredicate.java
@@ -17,6 +17,7 @@
package org.apache.doris.nereids.rules.expression.rewrite.rules;
+import org.apache.doris.nereids.exceptions.AnalysisException;
import
org.apache.doris.nereids.rules.expression.rewrite.AbstractExpressionRewriteRule;
import
org.apache.doris.nereids.rules.expression.rewrite.ExpressionRewriteContext;
import org.apache.doris.nereids.trees.expressions.Cast;
@@ -68,6 +69,7 @@ public class SimplifyComparisonPredicate extends
AbstractExpressionRewriteRule {
}
private Expression processDateLikeTypeCoercion(ComparisonPredicate cp,
Expression left, Expression right) {
+ Expression originalRight = right;
if (left instanceof DateLiteral) {
cp = cp.commute();
Expression temp = left;
@@ -111,6 +113,15 @@ public class SimplifyComparisonPredicate extends
AbstractExpressionRewriteRule {
}
}
+ if (left.getDataType() == DateType.INSTANCE && right.getDataType() ==
DateType.INSTANCE) {
+ //Date cp Date is not supported in BE storage engine. So cast to
DateTime
+ left = new Cast(left, DateTimeType.INSTANCE);
+ if (right instanceof DateLiteral) {
+ right = migrateLiteralToDateTime((DateLiteral) originalRight);
+ } else {
+ right = new Cast(originalRight, DateTimeType.INSTANCE);
+ }
+ }
if (left != cp.left() || right != cp.right()) {
return cp.withChildren(left, right);
} else {
@@ -118,6 +129,40 @@ public class SimplifyComparisonPredicate extends
AbstractExpressionRewriteRule {
}
}
+ private Expression migrateCastToDateTime(Cast cast) {
+ //cast( cast(v as date) as datetime) if v is datetime, set left = v
+ if (cast.child() instanceof Cast
+ && cast.child().child(0).getDataType() instanceof
DateTimeType) {
+ return cast.child().child(0);
+ } else {
+ return new Cast(cast.child(), DateTimeType.INSTANCE);
+ }
+ }
+
+ /*
+ derive tree:
+ DateLiteral
+ |
+ +--->DateTimeLiteral
+ | |
+ | +----->DateTimeV2Literal
+ +--->DateV2Literal
+ */
+ private Expression migrateLiteralToDateTime(DateLiteral l) {
+ if (l instanceof DateV2Literal) {
+ return new DateTimeLiteral(l.getYear(), l.getMonth(), l.getDay(),
0, 0, 0);
+ } else if (l instanceof DateTimeV2Literal) {
+ DateTimeV2Literal dtv2 = (DateTimeV2Literal) l;
+ return new DateTimeLiteral(dtv2.getYear(), dtv2.getMonth(),
dtv2.getDay(),
+ dtv2.getHour(), dtv2.getMinute(), dtv2.getSecond());
+ } else if (l instanceof DateTimeLiteral) {
+ return l;
+ } else if (l instanceof DateLiteral) {
+ return new DateTimeLiteral(l.getYear(), l.getMonth(), l.getDay(),
0, 0, 0);
+ }
+ throw new AnalysisException("cannot convert" + l.toSql() + " to
DateTime");
+ }
+
private Expression migrateToDateTime(DateTimeV2Literal l) {
return new DateTimeLiteral(l.getYear(), l.getMonth(), l.getDay(),
l.getHour(), l.getMinute(), l.getSecond());
}
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/rewrite/ExpressionRewriteTest.java
b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/rewrite/ExpressionRewriteTest.java
index 2e55f76e05..32e7e0b394 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/rewrite/ExpressionRewriteTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/rewrite/ExpressionRewriteTest.java
@@ -42,6 +42,7 @@ import
org.apache.doris.nereids.trees.expressions.literal.SmallIntLiteral;
import org.apache.doris.nereids.trees.expressions.literal.StringLiteral;
import org.apache.doris.nereids.trees.expressions.literal.TinyIntLiteral;
import org.apache.doris.nereids.trees.expressions.literal.VarcharLiteral;
+import org.apache.doris.nereids.types.DateTimeType;
import org.apache.doris.nereids.types.DateTimeV2Type;
import org.apache.doris.nereids.types.DecimalV2Type;
import org.apache.doris.nereids.types.StringType;
@@ -229,10 +230,11 @@ public class ExpressionRewriteTest extends
ExpressionRewriteTestHelper {
Expression dtv2 = new DateTimeV2Literal(1, 1, 1, 1, 1, 1);
Expression dt = new DateTimeLiteral(1, 1, 1, 1, 1, 1);
+ Expression dtNoTime = new DateTimeLiteral(1, 1, 1, 0, 0, 0);
Expression dv2 = new DateV2Literal(1, 1, 1);
Expression dv2PlusOne = new DateV2Literal(1, 1, 2);
Expression d = new DateLiteral(1, 1, 1);
- Expression dPlusOne = new DateLiteral(1, 1, 2);
+ //Expression dPlusOne = new DateLiteral(1, 1, 2);
// DateTimeV2 -> DateTime
assertRewrite(
@@ -264,10 +266,10 @@ public class ExpressionRewriteTest extends
ExpressionRewriteTestHelper {
// DateTimeV2 -> Date
assertRewrite(
new GreaterThan(new Cast(d, DateTimeV2Type.SYSTEM_DEFAULT),
dtv2),
- new GreaterThan(d, d));
+ new GreaterThan(new Cast(d, DateTimeType.INSTANCE), dt));
assertRewrite(
new LessThan(new Cast(d, DateTimeV2Type.SYSTEM_DEFAULT), dtv2),
- new LessThan(d, dPlusOne));
+ new LessThan(new Cast(d, DateTimeType.INSTANCE), dt));
assertRewrite(
new EqualTo(new Cast(d, DateTimeV2Type.SYSTEM_DEFAULT), dtv2),
new EqualTo(new Cast(d, DateTimeV2Type.SYSTEM_DEFAULT), dtv2));
@@ -275,10 +277,10 @@ public class ExpressionRewriteTest extends
ExpressionRewriteTestHelper {
// DateTime -> Date
assertRewrite(
new GreaterThan(new Cast(d, DateTimeV2Type.SYSTEM_DEFAULT),
dt),
- new GreaterThan(d, d));
+ new GreaterThan(new Cast(d, DateTimeType.INSTANCE), dt));
assertRewrite(
new LessThan(new Cast(d, DateTimeV2Type.SYSTEM_DEFAULT), dt),
- new LessThan(d, dPlusOne));
+ new LessThan(new Cast(d, DateTimeType.INSTANCE), dt));
assertRewrite(
new EqualTo(new Cast(d, DateTimeV2Type.SYSTEM_DEFAULT), dt),
new EqualTo(new Cast(d, DateTimeV2Type.SYSTEM_DEFAULT), dt));
@@ -286,7 +288,7 @@ public class ExpressionRewriteTest extends
ExpressionRewriteTestHelper {
// DateV2 -> Date
assertRewrite(
new GreaterThan(new Cast(d, DateTimeV2Type.SYSTEM_DEFAULT),
dv2),
- new GreaterThan(d, d));
+ new GreaterThan(new Cast(d, DateTimeType.INSTANCE), dtNoTime));
// test hour, minute and second all zero
Expression dtv2AtZeroClock = new DateTimeV2Literal(1, 1, 1, 0, 0, 0);
diff --git a/regression-test/data/nereids_syntax_p0/test_cast_datetime.out
b/regression-test/data/nereids_syntax_p0/test_cast_datetime.out
new file mode 100644
index 0000000000..4038c435d0
--- /dev/null
+++ b/regression-test/data/nereids_syntax_p0/test_cast_datetime.out
@@ -0,0 +1,4 @@
+-- This file is automatically generated. You should know what you did if you
want to edit this
+-- !1 --
+1
+
diff --git a/regression-test/suites/nereids_syntax_p0/test_cast_datetime.groovy
b/regression-test/suites/nereids_syntax_p0/test_cast_datetime.groovy
new file mode 100644
index 0000000000..924938c136
--- /dev/null
+++ b/regression-test/suites/nereids_syntax_p0/test_cast_datetime.groovy
@@ -0,0 +1,46 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_cast_datetime") {
+
+ sql "drop table if exists casttbl"
+ sql """CREATE TABLE casttbl (
+ mydate date NULL,
+ mydatev2 DATEV2 null,
+ mydatetime datetime null,
+ mydatetimev2 datetimev2 null
+ ) ENGINE=OLAP
+ DUPLICATE KEY(`mydate`)
+ COMMENT 'OLAP'
+ DISTRIBUTED BY HASH(`mydate`) BUCKETS 1
+ PROPERTIES (
+ "replication_allocation" = "tag.location.default: 1",
+ "in_memory" = "false",
+ "storage_format" = "V2",
+ "light_schema_change" = "true",
+ "disable_auto_compaction" = "false"
+ );
+ """
+
+ sql "insert into casttbl values ('2000-01-01', '2000-01-01 12:12:12',
'2000-01-01', '2000-01-01 12:12:12');"
+
+ sql "set enable_nereids_planner=true;"
+
+//when BE storage support 'Date < Date', we should remove this case
+//currently, if we rewrite expr to CAST(mydatetime AS DATE) < date
'2019-06-01', BE returns 0 tuple.
+ qt_1 "select count(1) from casttbl where CAST(CAST(mydatetime AS DATE) AS
DATETIME) < date '2019-06-01';"
+}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
