This is an automated email from the ASF dual-hosted git repository.
jakevin pushed a commit to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.0 by this push:
new 35289ee2449 [enhance](Nereids): support more DateLiteral (#30876)
35289ee2449 is described below
commit 35289ee2449a10d01b528d071c1821753db86da3
Author: jakevin <[email protected]>
AuthorDate: Mon Feb 5 23:26:33 2024 +0800
[enhance](Nereids): support more DateLiteral (#30876)
* [enhance](Nereids): support DateLiteral with suffix space (#30583)
(cherry picked from commit ab5bddba39c1e8fc0b07255b85dfbbc492d9ce80)
* [enhancement](Nereids): datetime support microsecond overflow (#30744)
(cherry picked from commit 3c6206355b79084927f1fe0117770988312c0683)
---
.../trees/expressions/literal/DateLiteral.java | 17 +++++++----
.../trees/expressions/literal/DateTimeLiteral.java | 16 +++++++++-
.../expressions/literal/DateTimeV2Literal.java | 9 ++++--
.../apache/doris/nereids/types/DateTimeV2Type.java | 5 +++-
.../doris/nereids/util/DateTimeFormatterUtils.java | 5 ++--
.../trees/expressions/literal/DateLiteralTest.java | 16 +++++-----
.../expressions/literal/DateTimeLiteralTest.java | 34 ++++++++++++++++++----
.../nereids/util/DateTimeFormatterUtilsTest.java | 4 +--
8 files changed, 79 insertions(+), 27 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DateLiteral.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DateLiteral.java
index 9fbece96756..58c252b4b98 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DateLiteral.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DateLiteral.java
@@ -36,6 +36,7 @@ import java.time.Year;
import java.time.temporal.ChronoField;
import java.time.temporal.TemporalAccessor;
import java.util.Set;
+import java.util.function.UnaryOperator;
/**
* Date literal in Nereids.
@@ -100,7 +101,7 @@ public class DateLiteral extends Literal {
// normalize yymmdd -> yyyymmdd
static String normalizeBasic(String s) {
- java.util.function.UnaryOperator<String> normalizeTwoDigit = (input)
-> {
+ UnaryOperator<String> normalizeTwoDigit = (input) -> {
String yy = input.substring(0, 2);
int year = Integer.parseInt(yy);
if (year >= 0 && year <= 69) {
@@ -182,9 +183,9 @@ public class DateLiteral extends Literal {
// normalize leading 0 for date and time
// date and time contains 6 number part at most, so we just need
normal 6 number part
int partNumber = 0;
- while (i < s.length()) {
+ while (i < s.length() && partNumber < 6) {
char c = s.charAt(i);
- if (Character.isDigit(c) && partNumber < 6) {
+ if (Character.isDigit(c)) {
// find consecutive digit
int j = i + 1;
while (j < s.length() && Character.isDigit(s.charAt(j))) {
@@ -233,11 +234,14 @@ public class DateLiteral extends Literal {
}
// parse MicroSecond
+ // Keep up to 7 digits at most, 7th digit is use for overflow.
if (partNumber == 6 && i < s.length() && s.charAt(i) == '.') {
sb.append(s.charAt(i));
i += 1;
while (i < s.length() && Character.isDigit(s.charAt(i))) {
- sb.append(s.charAt(i));
+ if (i - 19 <= 7) {
+ sb.append(s.charAt(i));
+ }
i += 1;
}
}
@@ -265,9 +269,12 @@ public class DateLiteral extends Literal {
try {
TemporalAccessor dateTime;
+ // remove suffix/prefix ' '
+ s = s.trim();
// parse condition without '-' and ':'
boolean containsPunctuation = false;
- for (int i = 0; i < s.length(); i++) {
+ int len = Math.min(s.length(), 11);
+ for (int i = 0; i < len; i++) {
if (isPunctuation(s.charAt(i))) {
containsPunctuation = true;
break;
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DateTimeLiteral.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DateTimeLiteral.java
index 36131db238f..3f96ef52e68 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DateTimeLiteral.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DateTimeLiteral.java
@@ -136,7 +136,6 @@ public class DateTimeLiteral extends DateLiteral {
hour = DateUtils.getOrDefault(temporal, ChronoField.HOUR_OF_DAY);
minute = DateUtils.getOrDefault(temporal, ChronoField.MINUTE_OF_HOUR);
second = DateUtils.getOrDefault(temporal,
ChronoField.SECOND_OF_MINUTE);
- microSecond = DateUtils.getOrDefault(temporal,
ChronoField.MICRO_OF_SECOND);
ZoneId zoneId = temporal.query(TemporalQueries.zone());
if (zoneId != null) {
@@ -153,6 +152,21 @@ public class DateTimeLiteral extends DateLiteral {
}
}
+ microSecond = DateUtils.getOrDefault(temporal,
ChronoField.NANO_OF_SECOND) / 100L;
+ // Microseconds have 7 digits.
+ long sevenDigit = microSecond % 10;
+ microSecond = microSecond / 10;
+ if (sevenDigit >= 5 && this instanceof DateTimeV2Literal) {
+ DateTimeV2Literal result = (DateTimeV2Literal)
((DateTimeV2Literal) this).plusMicroSeconds(1);
+ this.second = result.second;
+ this.minute = result.minute;
+ this.hour = result.hour;
+ this.day = result.day;
+ this.month = result.month;
+ this.year = result.year;
+ this.microSecond = result.microSecond;
+ }
+
if (checkRange() || checkDate()) {
throw new AnalysisException("datetime literal [" + s + "] is out
of range");
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DateTimeV2Literal.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DateTimeV2Literal.java
index 7471fc829fc..a9447affddd 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DateTimeV2Literal.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DateTimeV2Literal.java
@@ -66,7 +66,7 @@ public class DateTimeV2Literal extends DateTimeLiteral {
if (this.microSecond >= 1000000) {
LocalDateTime localDateTime =
DateUtils.getTime(StandardDateFormat.DATE_TIME_FORMATTER_TO_MICRO_SECOND,
- getStringValue()).plusSeconds(1);
+ getStringValue()).plusSeconds(1);
this.year = localDateTime.getYear();
this.month = localDateTime.getMonthValue();
this.day = localDateTime.getDayOfMonth();
@@ -77,6 +77,11 @@ public class DateTimeV2Literal extends DateTimeLiteral {
}
}
+ public String getFullMicroSecondValue() {
+ return String.format("%04d-%02d-%02d %02d:%02d:%02d.%06d",
+ year, month, day, hour, minute, second, microSecond);
+ }
+
@Override
public DateTimeV2Type getDataType() throws UnboundException {
return (DateTimeV2Type) super.getDataType();
@@ -157,7 +162,7 @@ public class DateTimeV2Literal extends DateTimeLiteral {
public Expression plusMicroSeconds(long microSeconds) {
return fromJavaDateType(
-
DateUtils.getTime(StandardDateFormat.DATE_TIME_FORMATTER_TO_MICRO_SECOND,
getStringValue())
+
DateUtils.getTime(StandardDateFormat.DATE_TIME_FORMATTER_TO_MICRO_SECOND,
getFullMicroSecondValue())
.plusNanos(microSeconds * 1000L),
getDataType().getScale());
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/types/DateTimeV2Type.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/types/DateTimeV2Type.java
index 77891ed3486..be2dc165886 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/types/DateTimeV2Type.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/types/DateTimeV2Type.java
@@ -85,10 +85,13 @@ public class DateTimeV2Type extends DateLikeType {
/**
* return proper type of datetimev2 for String
- * may be we need to check for validity?
+ * maybe we need to check for validity?
*/
public static DateTimeV2Type forTypeFromString(String s) {
int scale = DateTimeLiteral.determineScale(s);
+ if (scale > MAX_SCALE) {
+ scale = MAX_SCALE;
+ }
return DateTimeV2Type.of(scale);
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/DateTimeFormatterUtils.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/DateTimeFormatterUtils.java
index e88aabc018b..50d0f7169a5 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/DateTimeFormatterUtils.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/DateTimeFormatterUtils.java
@@ -56,8 +56,9 @@ public class DateTimeFormatterUtils {
.appendValue(ChronoField.HOUR_OF_DAY, 2)
.appendLiteral(':').appendValue(ChronoField.MINUTE_OF_HOUR, 2)
.appendLiteral(':').appendValue(ChronoField.SECOND_OF_MINUTE, 2)
+ // microsecond maxWidth is 7, we may need 7th digit to judge
overflow
.appendOptional(new DateTimeFormatterBuilder()
- .appendFraction(ChronoField.MICRO_OF_SECOND, 1, 6,
true).toFormatter())
+ .appendFraction(ChronoField.NANO_OF_SECOND, 1, 7,
true).toFormatter())
.toFormatter().withResolverStyle(ResolverStyle.STRICT);
// Time without delimiter: HHmmss[.microsecond]
private static final DateTimeFormatter BASIC_TIME_FORMATTER = new
DateTimeFormatterBuilder()
@@ -65,7 +66,7 @@ public class DateTimeFormatterUtils {
.appendValue(ChronoField.MINUTE_OF_HOUR, 2)
.appendValue(ChronoField.SECOND_OF_MINUTE, 2)
.appendOptional(new DateTimeFormatterBuilder()
- .appendFraction(ChronoField.MICRO_OF_SECOND, 1, 6,
true).toFormatter())
+ .appendFraction(ChronoField.NANO_OF_SECOND, 1, 7,
true).toFormatter())
.toFormatter().withResolverStyle(ResolverStyle.STRICT);
// yyyymmdd
private static final DateTimeFormatter BASIC_DATE_FORMATTER = new
DateTimeFormatterBuilder()
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/literal/DateLiteralTest.java
b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/literal/DateLiteralTest.java
index a87a177a1b3..3430676b14d 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/literal/DateLiteralTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/literal/DateLiteralTest.java
@@ -28,14 +28,6 @@ import java.util.function.Consumer;
class DateLiteralTest {
@Test
void reject() {
- // TODO: reject them.
- // Now parse them as date + offset.
- // PG parse them as date + offset, MySQL parse them as date + time
(rubbish behavior!)
- // So strange! reject these strange case.
- // Assertions.assertThrows(AnalysisException.class, () -> new
DateLiteral("2022-01-01-01"));
- // Assertions.assertThrows(AnalysisException.class, () -> new
DateLiteral("2022-01-01-1"));
- // Assertions.assertThrows(AnalysisException.class, () -> new
DateLiteral("2022-01-01+01"));
- // Assertions.assertThrows(AnalysisException.class, () -> new
DateLiteral("2022-01-01+1"));
Assertions.assertThrows(AnalysisException.class, () -> new
DateLiteral("2022-01-01 01:00:00.000000"));
Assertions.assertThrows(AnalysisException.class, () -> new
DateLiteral("2022-01-01 00:01:00.000000"));
Assertions.assertThrows(AnalysisException.class, () -> new
DateLiteral("2022-01-01 00:00:01.000000"));
@@ -212,6 +204,12 @@ class DateLiteralTest {
new DateLiteral("2020.02.01 00.00.00");
new DateTimeV2Literal("2020.02.01 00.00.00.1");
new DateTimeV2Literal("2020.02.01 00.00.00.000001");
- Assertions.assertThrows(AnalysisException.class, () -> new
DateTimeV2Literal("2020.02.01 00.00.00.0000001"));
+ new DateTimeV2Literal("2020.02.01 00.00.00.0000001");
+ }
+
+ @Test
+ void testSuffixSpace() {
+ new DateLiteral("2016-07-02 ");
+ new DateLiteral("2016-07-02 00:00:00 ");
}
}
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/literal/DateTimeLiteralTest.java
b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/literal/DateTimeLiteralTest.java
index 3cfaf485bf6..8c636e9eca4 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/literal/DateTimeLiteralTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/literal/DateTimeLiteralTest.java
@@ -41,13 +41,14 @@ class DateTimeLiteralTest {
Assertions.assertEquals(1, datetime.day);
Assertions.assertEquals(1, datetime.hour);
Assertions.assertEquals(1, datetime.minute);
- Assertions.assertEquals(1, datetime.second);
+ Assertions.assertEquals(2, datetime.second);
};
- assertFunc.accept(new DateTimeV2Literal("20220801010101"));
- assertFunc.accept(new DateTimeV2Literal("20220801T010101"));
- assertFunc.accept(new DateTimeV2Literal("220801010101"));
- assertFunc.accept(new DateTimeV2Literal("220801T010101"));
+ assertFunc.accept(new DateTimeV2Literal("20220801010102"));
+ assertFunc.accept(new DateTimeV2Literal("20220801T010102"));
+ assertFunc.accept(new DateTimeV2Literal("220801010102"));
+ assertFunc.accept(new DateTimeV2Literal("220801T010102"));
+ assertFunc.accept(new DateTimeV2Literal("20220801010101.9999999"));
}
@Test
@@ -386,6 +387,29 @@ class DateTimeLiteralTest {
// Testing with microsecond of length 6
new DateTimeV2Literal("2016-07-02 01:01:01.123456");
new DateTimeV2Literal("2016-7-02 01:01:01.123456");
+
+ // Testing with microsecond of length 7
+ DateTimeV2Literal literal = new DateTimeV2Literal("2016-07-02
01:01:01.12345678");
+ Assertions.assertEquals(123457, literal.microSecond);
+
+ literal = new DateTimeV2Literal("2016-07-02 01:01:01.44444444");
+ Assertions.assertEquals(444444, literal.microSecond);
+
+ literal = new DateTimeV2Literal("2016-07-02 01:01:01.44444445");
+ Assertions.assertEquals(444444, literal.microSecond);
+
+ literal = new DateTimeV2Literal("2016-07-02 01:01:01.4444445");
+ Assertions.assertEquals(444445, literal.microSecond);
+
+ literal = new DateTimeV2Literal("2016-07-02 01:01:01.9999995");
+ Assertions.assertEquals(0, literal.microSecond);
+ Assertions.assertEquals(2, literal.second);
+
+ literal = new DateTimeV2Literal("2021-01-01 23:59:59.9999995");
+ Assertions.assertEquals(0, literal.microSecond);
+ Assertions.assertEquals(0, literal.second);
+ Assertions.assertEquals(0, literal.minute);
+ Assertions.assertEquals(0, literal.hour);
}
@Test
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/nereids/util/DateTimeFormatterUtilsTest.java
b/fe/fe-core/src/test/java/org/apache/doris/nereids/util/DateTimeFormatterUtilsTest.java
index 853c8e11b41..8437fdb0092 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/nereids/util/DateTimeFormatterUtilsTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/nereids/util/DateTimeFormatterUtilsTest.java
@@ -70,6 +70,8 @@ class DateTimeFormatterUtilsTest {
assertDatePart(dateTime);
dateTime = formatter.parse("20200219T010101.1");
assertDatePart(dateTime);
+ dateTime = formatter.parse("20200219T010101.0000001");
+ assertDatePart(dateTime);
// failed case
DateTimeFormatter withT =
DateTimeFormatterUtils.BASIC_DATE_TIME_FORMATTER;
@@ -77,11 +79,9 @@ class DateTimeFormatterUtilsTest {
Assertions.assertThrows(DateTimeParseException.class, () ->
withT.parse("20200219010101."));
Assertions.assertThrows(DateTimeParseException.class, () ->
withT.parse("20200219010101.0000001"));
Assertions.assertThrows(DateTimeParseException.class, () ->
withT.parse("20200219T010101."));
- Assertions.assertThrows(DateTimeParseException.class, () ->
withT.parse("20200219T010101.0000001"));
DateTimeFormatter withoutT =
DateTimeFormatterUtils.BASIC_FORMATTER_WITHOUT_T;
Assertions.assertThrows(DateTimeParseException.class, () ->
withoutT.parse("20200219 010101"));
Assertions.assertThrows(DateTimeParseException.class, () ->
withoutT.parse("20200219010101."));
- Assertions.assertThrows(DateTimeParseException.class, () ->
withoutT.parse("20200219010101.0000001"));
Assertions.assertThrows(DateTimeParseException.class, () ->
withoutT.parse("20200219T010101."));
Assertions.assertThrows(DateTimeParseException.class, () ->
withoutT.parse("20200219T010101.0000001"));
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
