This is an automated email from the ASF dual-hosted git repository. jakevin pushed a commit to branch branch-2.0 in repository https://gitbox.apache.org/repos/asf/doris.git
commit 5bf00bb20c9b74d4afbe975009704a361dc64a23 Author: jakevin <[email protected]> AuthorDate: Mon Sep 11 14:28:03 2023 +0800 [fix](Nereids): add DateTimeFormatterUtils and fix bug (#24171) bug - should reject 20200219 010101 - datetime should be compatible with date (cherry picked from commit e847091dfe9c225b7133f2182049124261f3b881) --- .../trees/expressions/literal/DateLiteral.java | 10 ++- .../trees/expressions/literal/DateTimeLiteral.java | 42 +----------- .../doris/nereids/util/DateTimeFormatterUtils.java | 80 ++++++++++++++++++++++ .../org/apache/doris/nereids/util/DateUtils.java | 4 +- .../nereids/util/DateTimeFormatterUtilsTest.java | 61 +++++++++++++++++ 5 files changed, 151 insertions(+), 46 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DateLiteral.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DateLiteral.java index f75e2d81dad..e4447e18477 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DateLiteral.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DateLiteral.java @@ -25,6 +25,7 @@ import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; import org.apache.doris.nereids.types.DataType; import org.apache.doris.nereids.types.DateType; import org.apache.doris.nereids.types.coercion.DateLikeType; +import org.apache.doris.nereids.util.DateTimeFormatterUtils; import org.apache.doris.nereids.util.DateUtils; import org.apache.logging.log4j.LogManager; @@ -45,7 +46,6 @@ public class DateLiteral extends Literal { protected static DateTimeFormatter DATE_FORMATTER = null; protected static DateTimeFormatter DATE_FORMATTER_TWO_DIGIT = null; - protected static DateTimeFormatter DATEKEY_FORMATTER = null; // for cast datetime type to date type. protected static DateTimeFormatter DATE_TIME_FORMATTER = null; private static final LocalDateTime startOfAD = LocalDateTime.of(0, 1, 1, 0, 0, 0); @@ -65,8 +65,6 @@ public class DateLiteral extends Literal { try { DATE_FORMATTER = DateUtils.formatBuilder("%Y-%m-%d").toFormatter() .withResolverStyle(ResolverStyle.STRICT); - DATEKEY_FORMATTER = DateUtils.formatBuilder("%Y%m%d").toFormatter() - .withResolverStyle(ResolverStyle.STRICT); DATE_FORMATTER_TWO_DIGIT = DateUtils.formatBuilder("%y-%m-%d").toFormatter() .withResolverStyle(ResolverStyle.STRICT); DATE_TIME_FORMATTER = DateUtils.formatBuilder("%Y-%m-%d %H:%i:%s").toFormatter() @@ -120,10 +118,10 @@ public class DateLiteral extends Literal { protected void init(String s) throws AnalysisException { try { TemporalAccessor dateTime; - if (s.split("-")[0].length() == 2) { + if (!s.contains("-") && !s.contains(":")) { + dateTime = DateTimeFormatterUtils.BASIC_DATE_TIME_FORMATTER.parse(s); + } else if (s.split("-")[0].length() == 2) { dateTime = DATE_FORMATTER_TWO_DIGIT.parse(s); - } else if (s.length() == DATEKEY_LENGTH && !s.contains("-")) { - dateTime = DATEKEY_FORMATTER.parse(s); } else if (s.length() == 19) { dateTime = DATE_TIME_FORMATTER.parse(s); } else { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DateTimeLiteral.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DateTimeLiteral.java index 2db3dacb49b..3be28c2513b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DateTimeLiteral.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DateTimeLiteral.java @@ -24,10 +24,10 @@ import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; import org.apache.doris.nereids.types.DateTimeType; import org.apache.doris.nereids.types.coercion.DateLikeType; +import org.apache.doris.nereids.util.DateTimeFormatterUtils; import org.apache.doris.nereids.util.DateUtils; import com.google.common.base.Preconditions; -import com.google.common.collect.Lists; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -35,12 +35,10 @@ import java.time.LocalDateTime; import java.time.ZoneId; import java.time.format.DateTimeFormatter; import java.time.format.DateTimeFormatterBuilder; -import java.time.format.DateTimeParseException; import java.time.format.ResolverStyle; import java.time.temporal.ChronoField; import java.time.temporal.TemporalAccessor; import java.util.Collections; -import java.util.List; import java.util.Objects; import java.util.regex.Pattern; @@ -51,9 +49,7 @@ public class DateTimeLiteral extends DateLiteral { protected static DateTimeFormatter DATE_TIME_FORMATTER_TO_HOUR = null; protected static DateTimeFormatter DATE_TIME_FORMATTER_TO_MINUTE = null; protected static DateTimeFormatter DATE_TIME_FORMATTER_TWO_DIGIT = null; - protected static DateTimeFormatter DATETIMEKEY_FORMATTER = null; protected static DateTimeFormatter DATE_TIME_FORMATTER_TO_MICRO_SECOND = null; - protected static List<DateTimeFormatter> formatterList = null; protected static final int MAX_MICROSECOND = 999999; private static final DateTimeLiteral MIN_DATETIME = new DateTimeLiteral(0000, 1, 1, 0, 0, 0); @@ -79,29 +75,11 @@ public class DateTimeLiteral extends DateLiteral { DATE_TIME_FORMATTER_TWO_DIGIT = DateUtils.formatBuilder("%y-%m-%d %H:%i:%s") .toFormatter().withResolverStyle(ResolverStyle.STRICT); - DATETIMEKEY_FORMATTER = DateUtils.formatBuilder("%Y%m%d%H%i%s") - .toFormatter().withResolverStyle(ResolverStyle.STRICT); - DATE_TIME_FORMATTER_TO_MICRO_SECOND = new DateTimeFormatterBuilder() .appendPattern("uuuu-MM-dd HH:mm:ss") .appendFraction(ChronoField.MICRO_OF_SECOND, 0, 6, true) .toFormatter() .withResolverStyle(ResolverStyle.STRICT); - - formatterList = Lists.newArrayList( - DateUtils.formatBuilder("%Y%m%d").appendLiteral('T').appendPattern("HHmmss") - .appendFraction(ChronoField.MICRO_OF_SECOND, 0, 6, true) - .toFormatter().withResolverStyle(ResolverStyle.STRICT), - DateUtils.formatBuilder("%Y%m%d").appendLiteral('T').appendPattern("HHmmss") - .appendFraction(ChronoField.MICRO_OF_SECOND, 0, 6, false) - .toFormatter().withResolverStyle(ResolverStyle.STRICT), - DateUtils.formatBuilder("%Y%m%d%H%i%s") - .appendFraction(ChronoField.MICRO_OF_SECOND, 0, 6, true) - .toFormatter().withResolverStyle(ResolverStyle.STRICT), - DateUtils.formatBuilder("%Y%m%d%H%i%s") - .appendFraction(ChronoField.MICRO_OF_SECOND, 0, 6, false) - .toFormatter().withResolverStyle(ResolverStyle.STRICT), - DATETIMEKEY_FORMATTER, DATEKEY_FORMATTER); } catch (AnalysisException e) { LOG.error("invalid date format", e); System.exit(-1); @@ -170,22 +148,8 @@ public class DateTimeLiteral extends DateLiteral { offset = dorisZone.getRules().getOffset(java.time.Instant.now()).getTotalSeconds() - zone.getRules().getOffset(java.time.Instant.now()).getTotalSeconds(); } - - if (!s.contains("-")) { - // handle format like 20210106, but should not handle 2021-1-6 - boolean parsed = false; - for (DateTimeFormatter formatter : formatterList) { - try { - dateTime = formatter.parse(s); - parsed = true; - break; - } catch (DateTimeParseException ex) { - // ignore - } - } - if (!parsed) { - throw new AnalysisException("datetime literal [" + s + "] is invalid"); - } + if (!s.contains("-") && !s.contains(":")) { + dateTime = DateTimeFormatterUtils.BASIC_DATE_TIME_FORMATTER.parse(s); } else { String[] datePart = s.contains(" ") ? s.split(" ")[0].split("-") : s.split("-"); DateTimeFormatterBuilder builder = new DateTimeFormatterBuilder(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/DateTimeFormatterUtils.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/DateTimeFormatterUtils.java new file mode 100644 index 00000000000..244d672229f --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/DateTimeFormatterUtils.java @@ -0,0 +1,80 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.util; + +import java.time.format.DateTimeFormatter; +import java.time.format.DateTimeFormatterBuilder; +import java.time.format.ResolverStyle; +import java.time.temporal.ChronoField; + +/** + * Composition of MySQL Date: + * 1. Without any delimiter, e.g.: '20220801', '20220801010101' + * MySQL supports '20220801T010101' but doesn't support '20220801 010101' + * In this scenario, MySQL does not support zone/offset + * 2. With delimiters (':'/'-'): + * The composition is Date + ' '/'T' + Time + Zone + Offset + * Both Zone and Offset are Optional + * Date needs to be cautious about the two-digit year https://dev.mysql.com/doc/refman/8.0/en/datetime.html + * Dates containing 2-digit year values are ambiguous as the century is unknown. + * MySQL interprets 2-digit year values using these rules: + * Year values in the range 00-69 become 2000-2069. + * Year values in the range 70-99 become 1970-1999. + * Time needs to be cautious about microseconds: + * Note incomplete times 'hh:mm:ss', 'hh:mm', 'D hh:mm', 'D hh', or 'ss' + */ +public class DateTimeFormatterUtils { + // Date: %Y-%m-%d + public static DateTimeFormatter DATE_FORMATTER = new DateTimeFormatterBuilder() + .appendOptional(new DateTimeFormatterBuilder().appendValue(ChronoField.YEAR, 4).toFormatter()) + .appendOptional(new DateTimeFormatterBuilder().appendValue(ChronoField.YEAR, 2).toFormatter()) + .appendLiteral('-').appendValue(ChronoField.MONTH_OF_YEAR, 2) + .appendLiteral('-').appendValue(ChronoField.DAY_OF_MONTH, 2) + // if year isn't present, use -1 as default, so that it can be rejected by check + .parseDefaulting(ChronoField.YEAR, -1) + .toFormatter().withResolverStyle(ResolverStyle.STRICT); + // Date without delimiter: %Y%m%d + public static DateTimeFormatter BASIC_DATE_FORMATTER = new DateTimeFormatterBuilder() + .appendValue(ChronoField.YEAR, 4) + .appendValue(ChronoField.MONTH_OF_YEAR, 2) + .appendValue(ChronoField.DAY_OF_MONTH, 2) + .toFormatter().withResolverStyle(ResolverStyle.STRICT); + // Time: %H:%i:%s + public static DateTimeFormatter TIME_FORMATTER = new DateTimeFormatterBuilder() + .appendValue(ChronoField.HOUR_OF_DAY, 2) + .appendLiteral(':').appendValue(ChronoField.MINUTE_OF_HOUR, 2) + .appendLiteral(':').appendValue(ChronoField.SECOND_OF_MINUTE, 2) + .toFormatter().withResolverStyle(ResolverStyle.STRICT); + // Time without delimiter: HHmmss[microsecond] + public static DateTimeFormatter BASIC_TIME_FORMATTER = new DateTimeFormatterBuilder() + .appendValue(ChronoField.HOUR_OF_DAY, 2) + .appendValue(ChronoField.MINUTE_OF_HOUR, 2) + .appendValue(ChronoField.SECOND_OF_MINUTE, 2) + .appendOptional(new DateTimeFormatterBuilder() + .appendFraction(ChronoField.MICRO_OF_SECOND, 1, 6, true).toFormatter()) + .toFormatter().withResolverStyle(ResolverStyle.STRICT); + + // Date without delimiter + public static DateTimeFormatter BASIC_DATE_TIME_FORMATTER = new DateTimeFormatterBuilder() + .append(BASIC_DATE_FORMATTER) + .optionalStart() + .appendOptional(new DateTimeFormatterBuilder().appendLiteral('T').toFormatter()) + .append(BASIC_TIME_FORMATTER) + .optionalEnd() + .toFormatter().withResolverStyle(ResolverStyle.STRICT); +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/DateUtils.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/DateUtils.java index 1ae262a172f..bd09bfa3429 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/DateUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/DateUtils.java @@ -89,7 +89,9 @@ public class DateUtils { break; case 'r': // %r Time, 12-hour (hh:mm:ss followed by AM or PM) builder.appendValue(ChronoField.HOUR_OF_AMPM, 2) - .appendPattern(":mm:ss ") + .appendValue(ChronoField.MINUTE_OF_HOUR, 2) + .appendValue(ChronoField.SECOND_OF_MINUTE, 2) + .appendLiteral(' ') .appendText(ChronoField.AMPM_OF_DAY, TextStyle.FULL) .toFormatter(); break; diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/util/DateTimeFormatterUtilsTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/util/DateTimeFormatterUtilsTest.java new file mode 100644 index 00000000000..3e7ba62fc96 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/util/DateTimeFormatterUtilsTest.java @@ -0,0 +1,61 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.util; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.time.format.DateTimeFormatter; +import java.time.format.DateTimeParseException; +import java.time.temporal.ChronoField; +import java.time.temporal.TemporalAccessor; + +class DateTimeFormatterUtilsTest { + private void assertDatePart(TemporalAccessor dateTime) { + Assertions.assertEquals(2020, dateTime.get(ChronoField.YEAR)); + Assertions.assertEquals(2, dateTime.get(ChronoField.MONTH_OF_YEAR)); + Assertions.assertEquals(19, dateTime.get(ChronoField.DAY_OF_MONTH)); + } + + @Test + void testBasicDateTimeFormatter() { + DateTimeFormatter formatter = DateTimeFormatterUtils.BASIC_DATE_TIME_FORMATTER; + TemporalAccessor dateTime = formatter.parse("20200219"); + assertDatePart(dateTime); + dateTime = formatter.parse("20200219010101"); + assertDatePart(dateTime); + dateTime = formatter.parse("20200219T010101"); + assertDatePart(dateTime); + // failed case + Assertions.assertThrows(DateTimeParseException.class, () -> formatter.parse("20200219 010101")); + + // microsecond + dateTime = formatter.parse("20200219010101.000001"); + assertDatePart(dateTime); + dateTime = formatter.parse("20200219T010101.000001"); + assertDatePart(dateTime); + dateTime = formatter.parse("20200219010101.1"); + assertDatePart(dateTime); + dateTime = formatter.parse("20200219T010101.1"); + assertDatePart(dateTime); + Assertions.assertThrows(DateTimeParseException.class, () -> formatter.parse("20200219010101.")); + Assertions.assertThrows(DateTimeParseException.class, () -> formatter.parse("20200219010101.0000001")); + Assertions.assertThrows(DateTimeParseException.class, () -> formatter.parse("20200219T010101.")); + Assertions.assertThrows(DateTimeParseException.class, () -> formatter.parse("20200219T010101.0000001")); + } +} --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
