This is an automated email from the ASF dual-hosted git repository.
jakevin pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new d4756d3118 [feature](Nereids): fold Cast(s as date/datetime) on FE
(#24353)
d4756d3118 is described below
commit d4756d3118f8baabd1ba0d315cf9fb7a3bdcab54
Author: jakevin <[email protected]>
AuthorDate: Thu Sep 14 22:08:26 2023 +0800
[feature](Nereids): fold Cast(s as date/datetime) on FE (#24353)
cast("20210101" as Date) -> DateLiteral(2021, 1, 1)
---
.../expression/rules/FoldConstantRuleOnFE.java | 12 ++++-
.../doris/nereids/types/coercion/DateLikeType.java | 27 ++++++++++++
.../apache/doris/statistics/ColumnStatistic.java | 7 ---
.../apache/doris/statistics/StatsDeriveResult.java | 7 ---
.../nereids/rules/expression/FoldConstantTest.java | 51 +++++++++++++---------
.../data/correctness/test_cast_as_time.out | 2 +-
.../cast_function/test_cast_function.out | 2 +-
.../cast_function/test_cast_function.out | 2 +-
8 files changed, 71 insertions(+), 39 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/FoldConstantRuleOnFE.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/FoldConstantRuleOnFE.java
index cef81acd04..e1778fe521 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/FoldConstantRuleOnFE.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/FoldConstantRuleOnFE.java
@@ -73,6 +73,7 @@ import
org.apache.doris.nereids.trees.expressions.literal.StringLiteral;
import org.apache.doris.nereids.trees.expressions.literal.VarcharLiteral;
import org.apache.doris.nereids.types.BooleanType;
import org.apache.doris.nereids.types.DataType;
+import org.apache.doris.nereids.types.coercion.DateLikeType;
import org.apache.doris.nereids.util.ExpressionUtils;
import org.apache.doris.qe.ConnectContext;
import org.apache.doris.qe.GlobalVariable;
@@ -343,12 +344,19 @@ public class FoldConstantRuleOnFE extends
AbstractExpressionRewriteRule {
return checkedExpr.get();
}
Expression child = cast.child();
+ DataType dataType = cast.getDataType();
// todo: process other null case
if (child.isNullLiteral()) {
- return new NullLiteral(cast.getDataType());
+ return new NullLiteral(dataType);
+ } else if (child instanceof StringLikeLiteral && dataType instanceof
DateLikeType) {
+ try {
+ return ((DateLikeType)
dataType).fromString(((StringLikeLiteral) child).getStringValue());
+ } catch (AnalysisException t) {
+ return new NullLiteral(dataType);
+ }
}
try {
- Expression castResult = child.checkedCastTo(cast.getDataType());
+ Expression castResult = child.checkedCastTo(dataType);
if (!Objects.equals(castResult, cast) &&
!Objects.equals(castResult, child)) {
castResult = rewrite(castResult, context);
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/types/coercion/DateLikeType.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/types/coercion/DateLikeType.java
index 18b207fdaf..ff728a73ac 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/types/coercion/DateLikeType.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/types/coercion/DateLikeType.java
@@ -17,6 +17,16 @@
package org.apache.doris.nereids.types.coercion;
+import org.apache.doris.nereids.exceptions.AnalysisException;
+import org.apache.doris.nereids.trees.expressions.literal.DateLiteral;
+import org.apache.doris.nereids.trees.expressions.literal.DateTimeLiteral;
+import org.apache.doris.nereids.trees.expressions.literal.DateTimeV2Literal;
+import org.apache.doris.nereids.trees.expressions.literal.DateV2Literal;
+import org.apache.doris.nereids.types.DateTimeType;
+import org.apache.doris.nereids.types.DateTimeV2Type;
+import org.apache.doris.nereids.types.DateType;
+import org.apache.doris.nereids.types.DateV2Type;
+
import java.time.temporal.ChronoUnit;
import java.util.Calendar;
@@ -43,4 +53,21 @@ public abstract class DateLikeType extends PrimitiveType {
Calendar from = toCalendar(low);
return ChronoUnit.DAYS.between(from.toInstant(), to.toInstant());
}
+
+ /**
+ * parse string to date like literal.
+ */
+ public DateLiteral fromString(String s) {
+ if (this instanceof DateType) {
+ return new DateLiteral(s);
+ } else if (this instanceof DateV2Type) {
+ return new DateV2Literal(s);
+ } else if (this instanceof DateTimeType) {
+ return new DateTimeLiteral(s);
+ } else if (this instanceof DateTimeV2Type) {
+ return new DateTimeV2Literal((DateTimeV2Type) this, s);
+ } else {
+ throw new AnalysisException("unknown date like type");
+ }
+ }
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/statistics/ColumnStatistic.java
b/fe/fe-core/src/main/java/org/apache/doris/statistics/ColumnStatistic.java
index 11f924b5d1..7baf39d27a 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/ColumnStatistic.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/ColumnStatistic.java
@@ -204,13 +204,6 @@ public class ColumnStatistic {
return rowCount * 0.9 < ndv && ndv < rowCount * 1.1;
}
- public ColumnStatistic copy() {
- return new
ColumnStatisticBuilder().setCount(count).setNdv(ndv).setAvgSizeByte(avgSizeByte)
-
.setNumNulls(numNulls).setDataSize(dataSize).setMinValue(minValue)
- .setMaxValue(maxValue).setMinExpr(minExpr).setMaxExpr(maxExpr)
- .setIsUnknown(isUnKnown).build();
- }
-
public ColumnStatistic updateByLimit(long limit, double rowCount) {
double ratio = 0;
if (rowCount != 0) {
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatsDeriveResult.java
b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatsDeriveResult.java
index 6010daa6db..8c301f911b 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatsDeriveResult.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatsDeriveResult.java
@@ -144,13 +144,6 @@ public class StatsDeriveResult {
return statsDeriveResult;
}
- public StatsDeriveResult merge(StatsDeriveResult other) {
- for (Entry<Id, ColumnStatistic> entry :
other.getSlotIdToColumnStats().entrySet()) {
- this.slotIdToColumnStats.put(entry.getKey(),
entry.getValue().copy());
- }
- return this;
- }
-
public StatsDeriveResult copy() {
return new StatsDeriveResult(this);
}
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/FoldConstantTest.java
b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/FoldConstantTest.java
index 4f89f77937..9a213c4970 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/FoldConstantTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/FoldConstantTest.java
@@ -53,10 +53,10 @@ import org.junit.jupiter.api.Test;
import java.util.Locale;
-public class FoldConstantTest extends ExpressionRewriteTestHelper {
+class FoldConstantTest extends ExpressionRewriteTestHelper {
@Test
- public void testCaseWhenFold() {
+ void testCaseWhenFold() {
executor = new
ExpressionRuleExecutor(ImmutableList.of(FoldConstantRuleOnFE.INSTANCE));
// assertRewriteAfterTypeCoercion("case when 1 = 2 then 1 when '1' < 2
then 2 else 3 end", "2");
// assertRewriteAfterTypeCoercion("case when 1 = 2 then 1 when '1' > 2
then 2 end", "null");
@@ -73,7 +73,7 @@ public class FoldConstantTest extends
ExpressionRewriteTestHelper {
}
@Test
- public void testInFold() {
+ void testInFold() {
executor = new
ExpressionRuleExecutor(ImmutableList.of(FoldConstantRuleOnFE.INSTANCE));
assertRewriteAfterTypeCoercion("1 in (1,2,3,4)", "true");
// Type Coercion trans all to string.
@@ -86,7 +86,7 @@ public class FoldConstantTest extends
ExpressionRewriteTestHelper {
}
@Test
- public void testLogicalFold() {
+ void testLogicalFold() {
executor = new
ExpressionRuleExecutor(ImmutableList.of(FoldConstantRuleOnFE.INSTANCE));
assertRewriteAfterTypeCoercion("10 + 1 > 1 and 1 > 2", "false");
assertRewriteAfterTypeCoercion("10 + 1 > 1 and 1 < 2", "true");
@@ -124,7 +124,7 @@ public class FoldConstantTest extends
ExpressionRewriteTestHelper {
}
@Test
- public void testIsNullFold() {
+ void testIsNullFold() {
executor = new
ExpressionRuleExecutor(ImmutableList.of(FoldConstantRuleOnFE.INSTANCE));
assertRewriteAfterTypeCoercion("100 is null", "false");
assertRewriteAfterTypeCoercion("null is null", "true");
@@ -135,7 +135,7 @@ public class FoldConstantTest extends
ExpressionRewriteTestHelper {
}
@Test
- public void testNotPredicateFold() {
+ void testNotPredicateFold() {
executor = new
ExpressionRuleExecutor(ImmutableList.of(FoldConstantRuleOnFE.INSTANCE));
assertRewriteAfterTypeCoercion("not 1 > 2", "true");
assertRewriteAfterTypeCoercion("not null + 1 > 2", "null");
@@ -143,7 +143,7 @@ public class FoldConstantTest extends
ExpressionRewriteTestHelper {
}
@Test
- public void testCastFold() {
+ void testCastFold() {
executor = new
ExpressionRuleExecutor(ImmutableList.of(FoldConstantRuleOnFE.INSTANCE));
// cast '1' as tinyint
@@ -154,7 +154,7 @@ public class FoldConstantTest extends
ExpressionRewriteTestHelper {
}
@Test
- public void testCompareFold() {
+ void testCompareFold() {
executor = new
ExpressionRuleExecutor(ImmutableList.of(FoldConstantRuleOnFE.INSTANCE));
assertRewriteAfterTypeCoercion("'1' = 2", "false");
assertRewriteAfterTypeCoercion("1 = 2", "false");
@@ -171,7 +171,7 @@ public class FoldConstantTest extends
ExpressionRewriteTestHelper {
}
@Test
- public void testArithmeticFold() {
+ void testArithmeticFold() {
executor = new
ExpressionRuleExecutor(ImmutableList.of(FoldConstantRuleOnFE.INSTANCE));
assertRewrite("1 + 1", Literal.of((short) 2));
assertRewrite("1 - 1", Literal.of((short) 0));
@@ -204,7 +204,7 @@ public class FoldConstantTest extends
ExpressionRewriteTestHelper {
}
@Test
- public void testTimestampFold() {
+ void testTimestampFold() {
executor = new
ExpressionRuleExecutor(ImmutableList.of(FoldConstantRuleOnFE.INSTANCE));
String interval = "'1991-05-01' - interval 1 day";
Expression e7 = process((TimestampArithmetic)
PARSER.parseExpression(interval));
@@ -290,7 +290,7 @@ public class FoldConstantTest extends
ExpressionRewriteTestHelper {
assertRewrite(process, process);
}
- public Expression process(TimestampArithmetic arithmetic) {
+ Expression process(TimestampArithmetic arithmetic) {
String funcOpName;
if (arithmetic.getFuncName() == null) {
funcOpName = String.format("%sS_%s", arithmetic.getTimeUnit(),
@@ -302,7 +302,7 @@ public class FoldConstantTest extends
ExpressionRewriteTestHelper {
}
@Test
- public void testDateTypeDateTimeArithmeticFunctions() {
+ void testDateTypeDateTimeArithmeticFunctions() {
DateLiteral dateLiteral = new DateLiteral("1999-12-31");
IntegerLiteral integerLiteral = new IntegerLiteral(30);
VarcharLiteral format = new VarcharLiteral("%Y-%m-%d");
@@ -339,7 +339,7 @@ public class FoldConstantTest extends
ExpressionRewriteTestHelper {
}
@Test
- public void testDateTimeTypeDateTimeArithmeticFunctions() {
+ void testDateTimeTypeDateTimeArithmeticFunctions() {
DateTimeLiteral dateLiteral = new DateTimeLiteral("1999-12-31
23:59:59");
IntegerLiteral integerLiteral = new IntegerLiteral(30);
VarcharLiteral format = new VarcharLiteral("%Y-%m-%d");
@@ -394,7 +394,7 @@ public class FoldConstantTest extends
ExpressionRewriteTestHelper {
}
@Test
- public void testDateV2TypeDateTimeArithmeticFunctions() {
+ void testDateV2TypeDateTimeArithmeticFunctions() {
DateV2Literal dateLiteral = new DateV2Literal("1999-12-31");
IntegerLiteral integerLiteral = new IntegerLiteral(30);
VarcharLiteral format = new VarcharLiteral("%Y-%m-%d");
@@ -431,7 +431,7 @@ public class FoldConstantTest extends
ExpressionRewriteTestHelper {
}
@Test
- public void testDateTimeV2TypeDateTimeArithmeticFunctions() {
+ void testDateTimeV2TypeDateTimeArithmeticFunctions() {
DateTimeV2Literal dateLiteral = new
DateTimeV2Literal(DateTimeV2Type.SYSTEM_DEFAULT, "1999-12-31 23:59:59");
IntegerLiteral integerLiteral = new IntegerLiteral(30);
VarcharLiteral format = new VarcharLiteral("%Y-%m-%d");
@@ -496,7 +496,7 @@ public class FoldConstantTest extends
ExpressionRewriteTestHelper {
}
@Test
- public void testDateDiff() {
+ void testDateDiff() {
DateTimeLiteral dateTimeLiteral = new DateTimeLiteral("2001-12-31
00:00:01");
DateV2Literal dateV2Literal = new DateV2Literal("2001-12-31");
DateTimeV2Literal dateTimeV2Literal = new
DateTimeV2Literal("2001-12-31 00:00:01");
@@ -518,7 +518,7 @@ public class FoldConstantTest extends
ExpressionRewriteTestHelper {
}
@Test
- public void testDateTrunc() {
+ void testDateTrunc() {
DateTimeLiteral dateTimeLiteral = new DateTimeLiteral("2001-12-31
01:01:01");
DateTimeV2Literal dateTimeV2Literal = new
DateTimeV2Literal("2001-12-31 01:01:01");
@@ -556,7 +556,7 @@ public class FoldConstantTest extends
ExpressionRewriteTestHelper {
}
@Test
- public void testDateConstructFunction() {
+ void testDateConstructFunction() {
String[] answer = {
"2001-07-19", "6411-08-17", "0000-01-01", "'1977-06-03
17:57:24'",
"'1977-06-03'", "1008909293", "1008864000"
@@ -590,7 +590,7 @@ public class FoldConstantTest extends
ExpressionRewriteTestHelper {
}
@Test
- public void testFoldNestedExpression() {
+ void testFoldNestedExpression() {
assertRewriteExpression("makedate(year('2010-04-10'),
dayofyear('2010-04-11'))", "2010-04-11");
assertRewriteExpression("null in ('d', null)", "NULL");
assertRewriteExpression("null not in ('d', null)", "NULL");
@@ -604,7 +604,18 @@ public class FoldConstantTest extends
ExpressionRewriteTestHelper {
}
@Test
- public void testFoldTypeOfNullLiteral() {
+ void testFoldCastStringToDate() {
+ assertRewriteExpression("cast('2021-01-01' as date)", "2021-01-01");
+ assertRewriteExpression("cast('20210101' as date)", "2021-01-01");
+ assertRewriteExpression("cast('2021-01-01T00:00:00' as date)",
"2021-01-01");
+ assertRewriteExpression("cast('2021-01-01' as datetime)", "2021-01-01
00:00:00");
+ assertRewriteExpression("cast('20210101' as datetime)", "2021-01-01
00:00:00");
+ assertRewriteExpression("cast('2021-01-01T00:00:00' as datetime)",
"2021-01-01 00:00:00");
+ assertRewriteExpression("cast ('2022-12-02 22:23:24.999999' as
datetimev2(3))", "2022-12-02 22:23:24.999");
+ }
+
+ @Test
+ void testFoldTypeOfNullLiteral() {
String actualExpression =
"append_trailing_char_if_absent(cast(version() as varchar), cast(null as
varchar))";
ExpressionRewriteContext context = new ExpressionRewriteContext(
MemoTestUtils.createCascadesContext(new UnboundRelation(new
RelationId(1), ImmutableList.of("test_table"))));
diff --git a/regression-test/data/correctness/test_cast_as_time.out
b/regression-test/data/correctness/test_cast_as_time.out
index 50a6af259e..cacead8658 100644
--- a/regression-test/data/correctness/test_cast_as_time.out
+++ b/regression-test/data/correctness/test_cast_as_time.out
@@ -16,5 +16,5 @@
10:10:10
-- !select5 --
-2010-10-10T00:00
+\N
diff --git
a/regression-test/data/nereids_p0/sql_functions/cast_function/test_cast_function.out
b/regression-test/data/nereids_p0/sql_functions/cast_function/test_cast_function.out
index 1f9c69b366..d6acc36140 100644
---
a/regression-test/data/nereids_p0/sql_functions/cast_function/test_cast_function.out
+++
b/regression-test/data/nereids_p0/sql_functions/cast_function/test_cast_function.out
@@ -6,7 +6,7 @@
11
-- !sql --
-2000-01-01T03:14:17
+\N
-- !sql --
\N
diff --git
a/regression-test/data/query_p0/sql_functions/cast_function/test_cast_function.out
b/regression-test/data/query_p0/sql_functions/cast_function/test_cast_function.out
index 1120e79912..2958bce0bf 100644
---
a/regression-test/data/query_p0/sql_functions/cast_function/test_cast_function.out
+++
b/regression-test/data/query_p0/sql_functions/cast_function/test_cast_function.out
@@ -6,7 +6,7 @@
11
-- !sql --
-2000-01-01T03:14:17
+\N
-- !sql --
\N
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
