This is an automated email from the ASF dual-hosted git repository. jakevin pushed a commit to branch branch-2.0 in repository https://gitbox.apache.org/repos/asf/doris.git
commit 40ae8f9cd6bd778c7c18d2eff4be6a73c13e8c35 Author: jakevin <[email protected]> AuthorDate: Thu Sep 14 11:25:00 2023 +0800 [feature](Nereids): normalize two-digit basic date/datetime (#24333) normalize two-digit basic date/datetime 220201 -> 20220201 220201T010101 -> 20220201T010101 ...... (cherry picked from commit 354cb3970bfd6631b01ddb16a3138a617332d363) --- .../trees/expressions/literal/DateLiteral.java | 49 +++++++++++++++++++--- .../doris/nereids/util/DateTimeFormatterUtils.java | 13 +----- .../expressions/literal/DateTimeLiteralTest.java | 19 +++++++++ 3 files changed, 64 insertions(+), 17 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DateLiteral.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DateLiteral.java index 79d0128f84f..eef80b6020c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DateLiteral.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DateLiteral.java @@ -96,6 +96,48 @@ public class DateLiteral extends Literal { this.day = other.day; } + // normalize yymmdd -> yyyymmdd + static String normalizeBasic(String s) { + java.util.function.UnaryOperator<String> normalizeTwoDigit = (input) -> { + String yy = input.substring(0, 2); + int year = Integer.parseInt(yy); + if (year >= 0 && year <= 69) { + input = "20" + input; + } else if (year >= 70 && year <= 99) { + input = "19" + input; + } + return input; + }; + + // s.len == 6, assume it is "yymmdd" + // 'T' exists, assume it is "yymmddT......" + if (s.length() == 6 + || (s.length() > 6 && s.charAt(6) == 'T')) { + // check s index 0 - 6 all is digit char + for (int i = 0; i < 6; i++) { + if (!Character.isDigit(s.charAt(i))) { + return s; + } + } + return normalizeTwoDigit.apply(s); + } + + // handle yymmddHHMMSS + if (s.length() >= 12) { + // check s index 0 - 11 all is digit char + for (int i = 0; i < 12; i++) { + if (!Character.isDigit(s.charAt(i))) { + return s; + } + } + if (s.length() == 12 || !Character.isDigit(s.charAt(12))) { + return normalizeTwoDigit.apply(s); + } + } + + return s; + } + static String normalize(String s) { StringBuilder sb = new StringBuilder(); @@ -187,13 +229,10 @@ public class DateLiteral extends Literal { // parse condition without '-' and ':' if (!s.contains("-") && !s.contains(":")) { + s = normalizeBasic(s); // mysql reject "20200219 010101" "200219 010101", can't use ' ' spilt basic date time. if (!s.contains("T")) { - if (s.length() == 6) { - dateTime = DateTimeFormatterUtils.BASIC_TWO_DIGIT_DATE_FORMATTER.parse(s); - } else { - dateTime = DateTimeFormatterUtils.BASIC_FORMATTER_WITHOUT_T.parse(s); - } + dateTime = DateTimeFormatterUtils.BASIC_FORMATTER_WITHOUT_T.parse(s); } else { dateTime = DateTimeFormatterUtils.BASIC_DATE_TIME_FORMATTER.parse(s); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/DateTimeFormatterUtils.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/DateTimeFormatterUtils.java index edacfed333d..e88aabc018b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/DateTimeFormatterUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/DateTimeFormatterUtils.java @@ -46,13 +46,6 @@ public class DateTimeFormatterUtils { .optionalEnd() .toFormatter() .withResolverStyle(ResolverStyle.STRICT); - // yymmdd - public static final DateTimeFormatter BASIC_TWO_DIGIT_DATE_FORMATTER = new DateTimeFormatterBuilder() - .appendValueReduced(ChronoField.YEAR, 2, 2, 1970) - .appendValue(ChronoField.MONTH_OF_YEAR, 2) - .appendValue(ChronoField.DAY_OF_MONTH, 2) - .toFormatter().withResolverStyle(ResolverStyle.STRICT); - // yyyy-mm-dd public static final DateTimeFormatter DATE_FORMATTER = new DateTimeFormatterBuilder() .appendValue(ChronoField.YEAR, 4) .appendLiteral('-').appendValue(ChronoField.MONTH_OF_YEAR, 2) @@ -82,8 +75,7 @@ public class DateTimeFormatterUtils { .toFormatter().withResolverStyle(ResolverStyle.STRICT); // Date without delimiter public static final DateTimeFormatter BASIC_DATE_TIME_FORMATTER = new DateTimeFormatterBuilder() - .appendOptional(BASIC_DATE_FORMATTER) - .appendOptional(BASIC_TWO_DIGIT_DATE_FORMATTER) + .append(BASIC_DATE_FORMATTER) .appendLiteral('T') .append(BASIC_TIME_FORMATTER) .toFormatter().withResolverStyle(ResolverStyle.STRICT); @@ -103,9 +95,6 @@ public class DateTimeFormatterUtils { .appendValue(ChronoField.YEAR, 4) .appendLiteral('-').appendValue(ChronoField.MONTH_OF_YEAR, 2) .appendLiteral('-').appendValue(ChronoField.DAY_OF_MONTH, 2) - // .optionalStart() - // .appendZoneOrOffsetId() - // .optionalEnd() .append(ZONE_FORMATTER) .toFormatter().withResolverStyle(ResolverStyle.STRICT); public static final DateTimeFormatter ZONE_DATE_TIME_FORMATTER = new DateTimeFormatterBuilder() diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/literal/DateTimeLiteralTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/literal/DateTimeLiteralTest.java index ab1f1d59bca..ed5883f4d0a 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/literal/DateTimeLiteralTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/literal/DateTimeLiteralTest.java @@ -20,6 +20,8 @@ package org.apache.doris.nereids.trees.expressions.literal; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; +import java.util.function.Consumer; + class DateTimeLiteralTest { @Test void reject() { @@ -28,6 +30,23 @@ class DateTimeLiteralTest { // }); } + @Test + void testBasic() { + Consumer<DateTimeV2Literal> assertFunc = (datetime) -> { + Assertions.assertEquals(2022, datetime.year); + Assertions.assertEquals(8, datetime.month); + Assertions.assertEquals(1, datetime.day); + Assertions.assertEquals(1, datetime.hour); + Assertions.assertEquals(1, datetime.minute); + Assertions.assertEquals(1, datetime.second); + }; + + assertFunc.accept(new DateTimeV2Literal("20220801010101")); + assertFunc.accept(new DateTimeV2Literal("20220801T010101")); + assertFunc.accept(new DateTimeV2Literal("220801010101")); + assertFunc.accept(new DateTimeV2Literal("220801T010101")); + } + @Test void testWithoutZoneOrOffset() { new DateTimeV2Literal("2022-08-01"); --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
