This is an automated email from the ASF dual-hosted git repository. yiguolei pushed a commit to branch branch-2.1 in repository https://gitbox.apache.org/repos/asf/doris.git
commit 8a0ea4b6515c06e54816e2b58f866dae6cffab6d Author: jakevin <[email protected]> AuthorDate: Fri Feb 2 23:05:52 2024 +0800 [enhancement](Nereids): datetime support microsecond overflow (#30744) --- .../trees/expressions/literal/DateLiteral.java | 14 +++++---- .../trees/expressions/literal/DateTimeLiteral.java | 16 +++++++++- .../expressions/literal/DateTimeV2Literal.java | 9 ++++-- .../apache/doris/nereids/types/DateTimeV2Type.java | 5 +++- .../doris/nereids/util/DateTimeFormatterUtils.java | 5 ++-- .../rules/SimplifyComparisonPredicateSqlTest.java | 9 ------ .../trees/expressions/literal/DateLiteralTest.java | 10 +------ .../expressions/literal/DateTimeLiteralTest.java | 34 ++++++++++++++++++---- .../nereids/util/DateTimeFormatterUtilsTest.java | 4 +-- .../data/nereids_syntax_p0/test_literal.out | 22 ++++++++++++++ .../suites/nereids_syntax_p0/test_literal.groovy | 28 ++++++++++++++++++ 11 files changed, 120 insertions(+), 36 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DateLiteral.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DateLiteral.java index 955522c9797..c0c1b18488f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DateLiteral.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DateLiteral.java @@ -183,9 +183,9 @@ public class DateLiteral extends Literal { // normalize leading 0 for date and time // date and time contains 6 number part at most, so we just need normal 6 number part int partNumber = 0; - while (i < s.length()) { + while (i < s.length() && partNumber < 6) { char c = s.charAt(i); - if (Character.isDigit(c) && partNumber < 6) { + if (Character.isDigit(c)) { // find consecutive digit int j = i + 1; while (j < s.length() && Character.isDigit(s.charAt(j))) { @@ -234,11 +234,14 @@ public class DateLiteral extends Literal { } // parse MicroSecond + // Keep up to 7 digits at most, 7th digit is use for overflow. if (partNumber == 6 && i < s.length() && s.charAt(i) == '.') { sb.append(s.charAt(i)); i += 1; while (i < s.length() && Character.isDigit(s.charAt(i))) { - sb.append(s.charAt(i)); + if (i - 19 <= 7) { + sb.append(s.charAt(i)); + } i += 1; } } @@ -266,11 +269,12 @@ public class DateLiteral extends Literal { try { TemporalAccessor dateTime; - // remove suffix ' ' + // remove suffix/prefix ' ' s = s.trim(); // parse condition without '-' and ':' boolean containsPunctuation = false; - for (int i = 0; i < s.length(); i++) { + int len = Math.min(s.length(), 11); + for (int i = 0; i < len; i++) { if (isPunctuation(s.charAt(i))) { containsPunctuation = true; break; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DateTimeLiteral.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DateTimeLiteral.java index 36131db238f..3f96ef52e68 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DateTimeLiteral.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DateTimeLiteral.java @@ -136,7 +136,6 @@ public class DateTimeLiteral extends DateLiteral { hour = DateUtils.getOrDefault(temporal, ChronoField.HOUR_OF_DAY); minute = DateUtils.getOrDefault(temporal, ChronoField.MINUTE_OF_HOUR); second = DateUtils.getOrDefault(temporal, ChronoField.SECOND_OF_MINUTE); - microSecond = DateUtils.getOrDefault(temporal, ChronoField.MICRO_OF_SECOND); ZoneId zoneId = temporal.query(TemporalQueries.zone()); if (zoneId != null) { @@ -153,6 +152,21 @@ public class DateTimeLiteral extends DateLiteral { } } + microSecond = DateUtils.getOrDefault(temporal, ChronoField.NANO_OF_SECOND) / 100L; + // Microseconds have 7 digits. + long sevenDigit = microSecond % 10; + microSecond = microSecond / 10; + if (sevenDigit >= 5 && this instanceof DateTimeV2Literal) { + DateTimeV2Literal result = (DateTimeV2Literal) ((DateTimeV2Literal) this).plusMicroSeconds(1); + this.second = result.second; + this.minute = result.minute; + this.hour = result.hour; + this.day = result.day; + this.month = result.month; + this.year = result.year; + this.microSecond = result.microSecond; + } + if (checkRange() || checkDate()) { throw new AnalysisException("datetime literal [" + s + "] is out of range"); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DateTimeV2Literal.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DateTimeV2Literal.java index 069e53f1416..061666c681f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DateTimeV2Literal.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DateTimeV2Literal.java @@ -66,7 +66,7 @@ public class DateTimeV2Literal extends DateTimeLiteral { if (this.microSecond >= 1000000) { LocalDateTime localDateTime = DateUtils.getTime(StandardDateFormat.DATE_TIME_FORMATTER_TO_MICRO_SECOND, - getStringValue()).plusSeconds(1); + getStringValue()).plusSeconds(1); this.year = localDateTime.getYear(); this.month = localDateTime.getMonthValue(); this.day = localDateTime.getDayOfMonth(); @@ -77,6 +77,11 @@ public class DateTimeV2Literal extends DateTimeLiteral { } } + public String getFullMicroSecondValue() { + return String.format("%04d-%02d-%02d %02d:%02d:%02d.%06d", + year, month, day, hour, minute, second, microSecond); + } + @Override public DateTimeV2Type getDataType() throws UnboundException { return (DateTimeV2Type) super.getDataType(); @@ -165,7 +170,7 @@ public class DateTimeV2Literal extends DateTimeLiteral { public Expression plusMicroSeconds(long microSeconds) { return fromJavaDateType( - DateUtils.getTime(StandardDateFormat.DATE_TIME_FORMATTER_TO_MICRO_SECOND, getStringValue()) + DateUtils.getTime(StandardDateFormat.DATE_TIME_FORMATTER_TO_MICRO_SECOND, getFullMicroSecondValue()) .plusNanos(microSeconds * 1000L), getDataType().getScale()); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/types/DateTimeV2Type.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/types/DateTimeV2Type.java index b804815eb6c..94de55aea61 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/types/DateTimeV2Type.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/types/DateTimeV2Type.java @@ -84,10 +84,13 @@ public class DateTimeV2Type extends DateLikeType { /** * return proper type of datetimev2 for String - * may be we need to check for validity? + * maybe we need to check for validity? */ public static DateTimeV2Type forTypeFromString(String s) { int scale = DateTimeLiteral.determineScale(s); + if (scale > MAX_SCALE) { + scale = MAX_SCALE; + } return DateTimeV2Type.of(scale); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/DateTimeFormatterUtils.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/DateTimeFormatterUtils.java index e88aabc018b..50d0f7169a5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/DateTimeFormatterUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/DateTimeFormatterUtils.java @@ -56,8 +56,9 @@ public class DateTimeFormatterUtils { .appendValue(ChronoField.HOUR_OF_DAY, 2) .appendLiteral(':').appendValue(ChronoField.MINUTE_OF_HOUR, 2) .appendLiteral(':').appendValue(ChronoField.SECOND_OF_MINUTE, 2) + // microsecond maxWidth is 7, we may need 7th digit to judge overflow .appendOptional(new DateTimeFormatterBuilder() - .appendFraction(ChronoField.MICRO_OF_SECOND, 1, 6, true).toFormatter()) + .appendFraction(ChronoField.NANO_OF_SECOND, 1, 7, true).toFormatter()) .toFormatter().withResolverStyle(ResolverStyle.STRICT); // Time without delimiter: HHmmss[.microsecond] private static final DateTimeFormatter BASIC_TIME_FORMATTER = new DateTimeFormatterBuilder() @@ -65,7 +66,7 @@ public class DateTimeFormatterUtils { .appendValue(ChronoField.MINUTE_OF_HOUR, 2) .appendValue(ChronoField.SECOND_OF_MINUTE, 2) .appendOptional(new DateTimeFormatterBuilder() - .appendFraction(ChronoField.MICRO_OF_SECOND, 1, 6, true).toFormatter()) + .appendFraction(ChronoField.NANO_OF_SECOND, 1, 7, true).toFormatter()) .toFormatter().withResolverStyle(ResolverStyle.STRICT); // yyyymmdd private static final DateTimeFormatter BASIC_DATE_FORMATTER = new DateTimeFormatterBuilder() diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/rules/SimplifyComparisonPredicateSqlTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/rules/SimplifyComparisonPredicateSqlTest.java index ad384942a1e..517b4828798 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/rules/SimplifyComparisonPredicateSqlTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/rules/SimplifyComparisonPredicateSqlTest.java @@ -116,15 +116,6 @@ class SimplifyComparisonPredicateSqlTest extends TestWithFeService implements Me ) ); - PlanChecker.from(connectContext) - .analyze("select CONVERT('2021-01-30 00:00:00.0000001', DATETIME(6))") - .rewrite() - .matches( - logicalResultSink( - logicalOneRowRelation().when(p -> p.getProjects().get(0).child(0).equals(new NullLiteral(DateTimeV2Type.of(6)))) - ) - ); - PlanChecker.from(connectContext) .analyze("select CONVERT_TZ('2021-01-32 00:00:00', '+08:00', 'America/London') = '2021-01-30'") .rewrite() diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/literal/DateLiteralTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/literal/DateLiteralTest.java index 4212d892760..3430676b14d 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/literal/DateLiteralTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/literal/DateLiteralTest.java @@ -28,14 +28,6 @@ import java.util.function.Consumer; class DateLiteralTest { @Test void reject() { - // TODO: reject them. - // Now parse them as date + offset. - // PG parse them as date + offset, MySQL parse them as date + time (rubbish behavior!) - // So strange! reject these strange case. - // Assertions.assertThrows(AnalysisException.class, () -> new DateLiteral("2022-01-01-01")); - // Assertions.assertThrows(AnalysisException.class, () -> new DateLiteral("2022-01-01-1")); - // Assertions.assertThrows(AnalysisException.class, () -> new DateLiteral("2022-01-01+01")); - // Assertions.assertThrows(AnalysisException.class, () -> new DateLiteral("2022-01-01+1")); Assertions.assertThrows(AnalysisException.class, () -> new DateLiteral("2022-01-01 01:00:00.000000")); Assertions.assertThrows(AnalysisException.class, () -> new DateLiteral("2022-01-01 00:01:00.000000")); Assertions.assertThrows(AnalysisException.class, () -> new DateLiteral("2022-01-01 00:00:01.000000")); @@ -212,7 +204,7 @@ class DateLiteralTest { new DateLiteral("2020.02.01 00.00.00"); new DateTimeV2Literal("2020.02.01 00.00.00.1"); new DateTimeV2Literal("2020.02.01 00.00.00.000001"); - Assertions.assertThrows(AnalysisException.class, () -> new DateTimeV2Literal("2020.02.01 00.00.00.0000001")); + new DateTimeV2Literal("2020.02.01 00.00.00.0000001"); } @Test diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/literal/DateTimeLiteralTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/literal/DateTimeLiteralTest.java index 3cfaf485bf6..8c636e9eca4 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/literal/DateTimeLiteralTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/literal/DateTimeLiteralTest.java @@ -41,13 +41,14 @@ class DateTimeLiteralTest { Assertions.assertEquals(1, datetime.day); Assertions.assertEquals(1, datetime.hour); Assertions.assertEquals(1, datetime.minute); - Assertions.assertEquals(1, datetime.second); + Assertions.assertEquals(2, datetime.second); }; - assertFunc.accept(new DateTimeV2Literal("20220801010101")); - assertFunc.accept(new DateTimeV2Literal("20220801T010101")); - assertFunc.accept(new DateTimeV2Literal("220801010101")); - assertFunc.accept(new DateTimeV2Literal("220801T010101")); + assertFunc.accept(new DateTimeV2Literal("20220801010102")); + assertFunc.accept(new DateTimeV2Literal("20220801T010102")); + assertFunc.accept(new DateTimeV2Literal("220801010102")); + assertFunc.accept(new DateTimeV2Literal("220801T010102")); + assertFunc.accept(new DateTimeV2Literal("20220801010101.9999999")); } @Test @@ -386,6 +387,29 @@ class DateTimeLiteralTest { // Testing with microsecond of length 6 new DateTimeV2Literal("2016-07-02 01:01:01.123456"); new DateTimeV2Literal("2016-7-02 01:01:01.123456"); + + // Testing with microsecond of length 7 + DateTimeV2Literal literal = new DateTimeV2Literal("2016-07-02 01:01:01.12345678"); + Assertions.assertEquals(123457, literal.microSecond); + + literal = new DateTimeV2Literal("2016-07-02 01:01:01.44444444"); + Assertions.assertEquals(444444, literal.microSecond); + + literal = new DateTimeV2Literal("2016-07-02 01:01:01.44444445"); + Assertions.assertEquals(444444, literal.microSecond); + + literal = new DateTimeV2Literal("2016-07-02 01:01:01.4444445"); + Assertions.assertEquals(444445, literal.microSecond); + + literal = new DateTimeV2Literal("2016-07-02 01:01:01.9999995"); + Assertions.assertEquals(0, literal.microSecond); + Assertions.assertEquals(2, literal.second); + + literal = new DateTimeV2Literal("2021-01-01 23:59:59.9999995"); + Assertions.assertEquals(0, literal.microSecond); + Assertions.assertEquals(0, literal.second); + Assertions.assertEquals(0, literal.minute); + Assertions.assertEquals(0, literal.hour); } @Test diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/util/DateTimeFormatterUtilsTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/util/DateTimeFormatterUtilsTest.java index 853c8e11b41..8437fdb0092 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/util/DateTimeFormatterUtilsTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/util/DateTimeFormatterUtilsTest.java @@ -70,6 +70,8 @@ class DateTimeFormatterUtilsTest { assertDatePart(dateTime); dateTime = formatter.parse("20200219T010101.1"); assertDatePart(dateTime); + dateTime = formatter.parse("20200219T010101.0000001"); + assertDatePart(dateTime); // failed case DateTimeFormatter withT = DateTimeFormatterUtils.BASIC_DATE_TIME_FORMATTER; @@ -77,11 +79,9 @@ class DateTimeFormatterUtilsTest { Assertions.assertThrows(DateTimeParseException.class, () -> withT.parse("20200219010101.")); Assertions.assertThrows(DateTimeParseException.class, () -> withT.parse("20200219010101.0000001")); Assertions.assertThrows(DateTimeParseException.class, () -> withT.parse("20200219T010101.")); - Assertions.assertThrows(DateTimeParseException.class, () -> withT.parse("20200219T010101.0000001")); DateTimeFormatter withoutT = DateTimeFormatterUtils.BASIC_FORMATTER_WITHOUT_T; Assertions.assertThrows(DateTimeParseException.class, () -> withoutT.parse("20200219 010101")); Assertions.assertThrows(DateTimeParseException.class, () -> withoutT.parse("20200219010101.")); - Assertions.assertThrows(DateTimeParseException.class, () -> withoutT.parse("20200219010101.0000001")); Assertions.assertThrows(DateTimeParseException.class, () -> withoutT.parse("20200219T010101.")); Assertions.assertThrows(DateTimeParseException.class, () -> withoutT.parse("20200219T010101.0000001")); } diff --git a/regression-test/data/nereids_syntax_p0/test_literal.out b/regression-test/data/nereids_syntax_p0/test_literal.out new file mode 100644 index 00000000000..eb56075f6b2 --- /dev/null +++ b/regression-test/data/nereids_syntax_p0/test_literal.out @@ -0,0 +1,22 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !date1 -- +2016-07-03 + +-- !date2 -- +2016-07-03 + +-- !datetime1 -- +2016-07-02T23:59:59.990 + +-- !datetime2 -- +2016-07-03T00:00 + +-- !datetime3 -- +2016-07-03T00:00 + +-- !datetime4 -- +2020-02-19T01:01:01 + +-- !datetime5 -- +2021-01-30T00:00 + diff --git a/regression-test/suites/nereids_syntax_p0/test_literal.groovy b/regression-test/suites/nereids_syntax_p0/test_literal.groovy index d3ed7d44098..340c254fdc8 100644 --- a/regression-test/suites/nereids_syntax_p0/test_literal.groovy +++ b/regression-test/suites/nereids_syntax_p0/test_literal.groovy @@ -24,4 +24,32 @@ suite("test_literal") { sql """ select {}, [], [[[null]], [[1, 2, 3]]], {1:[null], 3:[3]}; """ + + qt_date1 """ + select cast('20160702235959.9999999' as date); + """ + + qt_date2 """ + select cast('2016-07-02 23:59:59.9999999' as date); + """ + + qt_datetime1 """ + select timestamp'2016-07-02 23:59:59.99'; + """ + + qt_datetime2 """ + select cast('20160702235959.9999999' as datetime); + """ + + qt_datetime3 """ + select cast('2016-07-02 23:59:59.9999999' as datetime); + """ + + qt_datetime4 """ + select timestamp'20200219010101.0000001'; + """ + + qt_datetime5 """ + select CONVERT('2021-01-30 00:00:00.0000001', DATETIME(6)) + """ } --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
