This is an automated email from the ASF dual-hosted git repository. jakevin pushed a commit to branch branch-2.0 in repository https://gitbox.apache.org/repos/asf/doris.git
commit 36fc350bbd1017a9d8c7c5abe583cdca2994eeb4 Author: jakevin <[email protected]> AuthorDate: Wed Sep 13 11:20:27 2023 +0800 [refactor](Nereids): new Date/Datetime parser to support more condition (#24224) * unify all Date/Datetime use one string-parser * support microsecond & ZoneOffset both exist * add many UT case * add determineScale() to get scale of datetime, original code just get length of part after . * reject more bad condition like 2022-01-01 00:00:00., we don't allow . without microsecond. * ..... (cherry picked from commit 7025293e176f6e6ea32a863cb64f6bb7fe24e929) --- be/src/apache-orc | 2 +- .../trees/expressions/literal/DateLiteral.java | 104 +++--- .../trees/expressions/literal/DateTimeLiteral.java | 217 ++++-------- .../expressions/literal/DateTimeV2Literal.java | 50 +-- .../trees/expressions/literal/DateV2Literal.java | 15 +- .../apache/doris/nereids/types/DateTimeV2Type.java | 6 +- .../doris/nereids/util/DateTimeFormatterUtils.java | 98 ++++-- .../doris/nereids/util/StandardDateFormat.java | 53 +++ .../trees/expressions/LiteralEqualTest.java | 45 +++ .../trees/expressions/literal/DateLiteralTest.java | 83 +++++ .../expressions/literal/DateTimeLiteralTest.java | 369 +++++++++++++++++++++ .../nereids/util/DateTimeFormatterUtilsTest.java | 81 ++++- 12 files changed, 868 insertions(+), 255 deletions(-) diff --git a/be/src/apache-orc b/be/src/apache-orc index a7c0af50f8c..78bbe2e41f2 160000 --- a/be/src/apache-orc +++ b/be/src/apache-orc @@ -1 +1 @@ -Subproject commit a7c0af50f8ca8ff7cddaf8675473a037f8b13143 +Subproject commit 78bbe2e41f2140b803855d683fae5e1a4b734a37 diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DateLiteral.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DateLiteral.java index e4447e18477..025fb0936f9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DateLiteral.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DateLiteral.java @@ -27,14 +27,13 @@ import org.apache.doris.nereids.types.DateType; import org.apache.doris.nereids.types.coercion.DateLikeType; import org.apache.doris.nereids.util.DateTimeFormatterUtils; import org.apache.doris.nereids.util.DateUtils; +import org.apache.doris.nereids.util.StandardDateFormat; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import java.time.LocalDateTime; import java.time.Year; -import java.time.format.DateTimeFormatter; -import java.time.format.ResolverStyle; import java.time.temporal.ChronoField; import java.time.temporal.TemporalAccessor; @@ -44,10 +43,7 @@ import java.time.temporal.TemporalAccessor; public class DateLiteral extends Literal { public static final String JAVA_DATE_FORMAT = "yyyy-MM-dd"; - protected static DateTimeFormatter DATE_FORMATTER = null; - protected static DateTimeFormatter DATE_FORMATTER_TWO_DIGIT = null; // for cast datetime type to date type. - protected static DateTimeFormatter DATE_TIME_FORMATTER = null; private static final LocalDateTime startOfAD = LocalDateTime.of(0, 1, 1, 0, 0, 0); private static final LocalDateTime endOfAD = LocalDateTime.of(9999, 12, 31, 23, 59, 59); private static final Logger LOG = LogManager.getLogger(DateLiteral.class); @@ -55,26 +51,11 @@ public class DateLiteral extends Literal { private static final DateLiteral MIN_DATE = new DateLiteral(0000, 1, 1); private static final DateLiteral MAX_DATE = new DateLiteral(9999, 12, 31); private static final int[] DAYS_IN_MONTH = new int[] {0, 31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31}; - private static final int DATEKEY_LENGTH = 8; protected long year; protected long month; protected long day; - static { - try { - DATE_FORMATTER = DateUtils.formatBuilder("%Y-%m-%d").toFormatter() - .withResolverStyle(ResolverStyle.STRICT); - DATE_FORMATTER_TWO_DIGIT = DateUtils.formatBuilder("%y-%m-%d").toFormatter() - .withResolverStyle(ResolverStyle.STRICT); - DATE_TIME_FORMATTER = DateUtils.formatBuilder("%Y-%m-%d %H:%i:%s").toFormatter() - .withResolverStyle(ResolverStyle.STRICT); - } catch (AnalysisException e) { - LOG.error("invalid date format", e); - System.exit(-1); - } - } - public DateLiteral(String s) throws AnalysisException { this(DateType.INSTANCE, s); } @@ -115,27 +96,72 @@ public class DateLiteral extends Literal { this.day = other.day; } - protected void init(String s) throws AnalysisException { + // replace 'T' with ' ' + private static String replaceDelimiterT(String s) { + // Matcher matcher = Pattern.compile("^(\\d{2,4}-\\d{1,2}-\\d{1,2})T").matcher(s); + // if (matcher.find()) { + // return matcher.group(1) + " " + s.substring(matcher.end()); + // } + // return s; + if (s.length() <= 10) { + return s; + } + if (s.charAt(10) == 'T') { + return s.substring(0, 10) + " " + s.substring(11); + } else if (s.charAt(8) == 'T') { + return s.substring(0, 8) + " " + s.substring(9); + } else { + return s; + } + } + + protected static TemporalAccessor parse(String s) { + String originalString = s; try { TemporalAccessor dateTime; + + // parse condition without '-' and ':' if (!s.contains("-") && !s.contains(":")) { - dateTime = DateTimeFormatterUtils.BASIC_DATE_TIME_FORMATTER.parse(s); - } else if (s.split("-")[0].length() == 2) { - dateTime = DATE_FORMATTER_TWO_DIGIT.parse(s); - } else if (s.length() == 19) { - dateTime = DATE_TIME_FORMATTER.parse(s); + // mysql reject "20200219 010101" "200219 010101", can't use ' ' spilt basic date time. + if (!s.contains("T")) { + if (s.length() == 6) { + dateTime = DateTimeFormatterUtils.BASIC_TWO_DIGIT_DATE_FORMATTER.parse(s); + } else { + dateTime = DateTimeFormatterUtils.BASIC_FORMATTER_WITHOUT_T.parse(s); + } + } else { + dateTime = DateTimeFormatterUtils.BASIC_DATE_TIME_FORMATTER.parse(s); + } + return dateTime; + } + + // replace first 'T' with ' ' + s = replaceDelimiterT(s); + if (!s.contains(" ")) { + dateTime = DateTimeFormatterUtils.ZONE_DATE_FORMATTER.parse(s); } else { - dateTime = DATE_FORMATTER.parse(s); + dateTime = DateTimeFormatterUtils.ZONE_DATE_TIME_FORMATTER.parse(s); + } + + // if Year is not present, throw exception + if (!dateTime.isSupported(ChronoField.YEAR)) { + throw new AnalysisException("datetime literal [" + originalString + "] is invalid"); } - year = DateUtils.getOrDefault(dateTime, ChronoField.YEAR); - month = DateUtils.getOrDefault(dateTime, ChronoField.MONTH_OF_YEAR); - day = DateUtils.getOrDefault(dateTime, ChronoField.DAY_OF_MONTH); + + return dateTime; } catch (Exception ex) { - throw new AnalysisException("date literal [" + s + "] is invalid"); + throw new AnalysisException("datetime literal [" + originalString + "] is invalid"); } + } + + protected void init(String s) throws AnalysisException { + TemporalAccessor dateTime = parse(s); + year = DateUtils.getOrDefault(dateTime, ChronoField.YEAR); + month = DateUtils.getOrDefault(dateTime, ChronoField.MONTH_OF_YEAR); + day = DateUtils.getOrDefault(dateTime, ChronoField.DAY_OF_MONTH); if (checkRange() || checkDate()) { - throw new AnalysisException("date literal [" + s + "] is out of range"); + throw new AnalysisException("datetime literal [" + s + "] is out of range"); } } @@ -204,16 +230,18 @@ public class DateLiteral extends Literal { return day; } - public Expression plusDays(int days) { - return fromJavaDateType(DateUtils.getTime(DATE_FORMATTER, getStringValue()).plusDays(days)); + public Expression plusDays(long days) { + return fromJavaDateType(DateUtils.getTime(StandardDateFormat.DATE_FORMATTER, getStringValue()).plusDays(days)); } - public Expression plusMonths(int months) { - return fromJavaDateType(DateUtils.getTime(DATE_FORMATTER, getStringValue()).plusMonths(months)); + public Expression plusMonths(long months) { + return fromJavaDateType( + DateUtils.getTime(StandardDateFormat.DATE_FORMATTER, getStringValue()).plusMonths(months)); } - public Expression plusYears(int years) { - return fromJavaDateType(DateUtils.getTime(DATE_FORMATTER, getStringValue()).plusYears(years)); + public Expression plusYears(long years) { + return fromJavaDateType( + DateUtils.getTime(StandardDateFormat.DATE_FORMATTER, getStringValue()).plusYears(years)); } public LocalDateTime toJavaDateType() { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DateTimeLiteral.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DateTimeLiteral.java index 90d6ef4f9b7..3318951c959 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DateTimeLiteral.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DateTimeLiteral.java @@ -24,31 +24,24 @@ import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; import org.apache.doris.nereids.types.DateTimeType; import org.apache.doris.nereids.types.coercion.DateLikeType; -import org.apache.doris.nereids.util.DateTimeFormatterUtils; import org.apache.doris.nereids.util.DateUtils; +import org.apache.doris.nereids.util.StandardDateFormat; -import com.google.common.base.Preconditions; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import java.time.Instant; import java.time.LocalDateTime; -import java.time.format.DateTimeFormatter; -import java.time.format.DateTimeFormatterBuilder; -import java.time.format.ResolverStyle; +import java.time.ZoneId; import java.time.temporal.ChronoField; import java.time.temporal.TemporalAccessor; -import java.util.Collections; +import java.time.temporal.TemporalQueries; import java.util.Objects; -import java.util.regex.Pattern; /** * date time literal. */ public class DateTimeLiteral extends DateLiteral { - protected static DateTimeFormatter DATE_TIME_FORMATTER_TO_HOUR = null; - protected static DateTimeFormatter DATE_TIME_FORMATTER_TO_MINUTE = null; - protected static DateTimeFormatter DATE_TIME_FORMATTER_TWO_DIGIT = null; - protected static DateTimeFormatter DATE_TIME_FORMATTER_TO_MICRO_SECOND = null; protected static final int MAX_MICROSECOND = 999999; private static final DateTimeLiteral MIN_DATETIME = new DateTimeLiteral(0000, 1, 1, 0, 0, 0); @@ -56,35 +49,11 @@ public class DateTimeLiteral extends DateLiteral { private static final Logger LOG = LogManager.getLogger(DateTimeLiteral.class); - private static final Pattern HAS_OFFSET_PART = Pattern.compile("[\\+\\-]\\d{2}:\\d{2}"); - protected long hour; protected long minute; protected long second; protected long microSecond; - static { - try { - DATE_TIME_FORMATTER = DateUtils.formatBuilder("%Y-%m-%d %H:%i:%s") - .toFormatter().withResolverStyle(ResolverStyle.STRICT); - DATE_TIME_FORMATTER_TO_HOUR = DateUtils.formatBuilder("%Y-%m-%d %H") - .toFormatter().withResolverStyle(ResolverStyle.STRICT); - DATE_TIME_FORMATTER_TO_MINUTE = DateUtils.formatBuilder("%Y-%m-%d %H:%i") - .toFormatter().withResolverStyle(ResolverStyle.STRICT); - DATE_TIME_FORMATTER_TWO_DIGIT = DateUtils.formatBuilder("%y-%m-%d %H:%i:%s") - .toFormatter().withResolverStyle(ResolverStyle.STRICT); - - DATE_TIME_FORMATTER_TO_MICRO_SECOND = new DateTimeFormatterBuilder() - .appendPattern("uuuu-MM-dd HH:mm:ss") - .appendFraction(ChronoField.MICRO_OF_SECOND, 0, 6, true) - .toFormatter() - .withResolverStyle(ResolverStyle.STRICT); - } catch (AnalysisException e) { - LOG.error("invalid date format", e); - System.exit(-1); - } - } - public DateTimeLiteral(String s) { this(DateTimeType.INSTANCE, s); } @@ -124,95 +93,50 @@ public class DateTimeLiteral extends DateLiteral { this.day = day; } + /** + * determine scale by datetime string + */ + public static int determineScale(String s) { + TemporalAccessor dateTime = parse(s); + int microSecond = DateUtils.getOrDefault(dateTime, ChronoField.MICRO_OF_SECOND); + + if (microSecond == 0) { + return 0; + } + + int scale = 6; + while (microSecond % 10 == 0) { + scale--; + microSecond /= 10; + } + return scale; + } + @Override protected void init(String s) throws AnalysisException { - try { - TemporalAccessor dateTime = null; - // parse timezone - if (haveTimeZoneOffset(s) || haveTimeZoneName(s)) { - if (haveTimeZoneName(s)) { // GMT, UTC+8, Z[, CN, Asia/Shanghai] - int split = getTimeZoneSplitPos(s); - Preconditions.checkArgument(split > 0); - s = s.substring(0, split); - } else { // +04:30 - Preconditions.checkArgument(s.charAt(s.length() - 6) == '-' || s.charAt(s.length() - 6) == '+'); - s = s.substring(0, s.length() - 6); - } + TemporalAccessor temporal = parse(s); + + year = DateUtils.getOrDefault(temporal, ChronoField.YEAR); + month = DateUtils.getOrDefault(temporal, ChronoField.MONTH_OF_YEAR); + day = DateUtils.getOrDefault(temporal, ChronoField.DAY_OF_MONTH); + hour = DateUtils.getOrDefault(temporal, ChronoField.HOUR_OF_DAY); + minute = DateUtils.getOrDefault(temporal, ChronoField.MINUTE_OF_HOUR); + second = DateUtils.getOrDefault(temporal, ChronoField.SECOND_OF_MINUTE); + microSecond = DateUtils.getOrDefault(temporal, ChronoField.MICRO_OF_SECOND); + + ZoneId zoneId = temporal.query(TemporalQueries.zone()); + if (zoneId != null) { + int offset = DateUtils.getTimeZone().getRules().getOffset(Instant.now()).getTotalSeconds() + - zoneId.getRules().getOffset(Instant.now()).getTotalSeconds(); + if (offset != 0) { + DateTimeLiteral result = (DateTimeLiteral) this.plusSeconds(offset); + this.second = result.second; + this.minute = result.minute; + this.hour = result.hour; + this.day = result.day; + this.month = result.month; + this.year = result.year; } - if (!s.contains("-") && !s.contains(":")) { - dateTime = DateTimeFormatterUtils.BASIC_DATE_TIME_FORMATTER.parse(s); - } else { - String[] datePart = s.contains(" ") ? s.split(" ")[0].split("-") : s.split("-"); - DateTimeFormatterBuilder builder = new DateTimeFormatterBuilder(); - if (datePart.length != 3) { - throw new AnalysisException("datetime literal [" + s + "] is invalid"); - } - for (int i = 0; i < datePart.length; i++) { - switch (i) { - case 0: - if (datePart[i].length() == 2) { - // If year is represented by two digits, number bigger than 70 will be prefixed - // with 19 otherwise 20. e.g. 69 -> 2069, 70 -> 1970. - builder.appendValueReduced(ChronoField.YEAR, 2, 2, 1970); - } else { - builder.appendPattern(String.join("", Collections.nCopies(datePart[i].length(), "u"))); - } - break; - case 1: - builder.appendPattern(String.join("", Collections.nCopies(datePart[i].length(), "M"))); - break; - case 2: - builder.appendPattern(String.join("", Collections.nCopies(datePart[i].length(), "d"))); - break; - default: - throw new AnalysisException("two many parts in date format " + s); - } - if (i < datePart.length - 1) { - builder.appendLiteral("-"); - } - } - if (s.contains(" ")) { - builder.appendLiteral(" "); - } - String[] timePart = s.contains(" ") ? s.split(" ")[1].split(":") : new String[]{}; - for (int i = 0; i < timePart.length; i++) { - switch (i) { - case 0: - builder.appendPattern(String.join("", Collections.nCopies(timePart[i].length(), "H"))); - break; - case 1: - builder.appendPattern(String.join("", Collections.nCopies(timePart[i].length(), "m"))); - break; - case 2: - builder.appendPattern(String.join("", Collections.nCopies(timePart[i].contains(".") - ? timePart[i].split("\\.")[0].length() : timePart[i].length(), "s"))); - if (timePart[i].contains(".")) { - builder.appendFraction(ChronoField.MICRO_OF_SECOND, 0, 6, true); - } - break; - default: - throw new AnalysisException("too many parts in time format " + s); - } - if (i < timePart.length - 1) { - builder.appendLiteral(":"); - } - } - // The default resolver style is 'SMART', which parses "2022-06-31" as "2022-06-30" - // and does not throw an exception. 'STRICT' is used here. - DateTimeFormatter formatter = builder.toFormatter().withResolverStyle(ResolverStyle.STRICT); - dateTime = formatter.parse(s); - } - - year = DateUtils.getOrDefault(dateTime, ChronoField.YEAR); - month = DateUtils.getOrDefault(dateTime, ChronoField.MONTH_OF_YEAR); - day = DateUtils.getOrDefault(dateTime, ChronoField.DAY_OF_MONTH); - hour = DateUtils.getOrDefault(dateTime, ChronoField.HOUR_OF_DAY); - minute = DateUtils.getOrDefault(dateTime, ChronoField.MINUTE_OF_HOUR); - second = DateUtils.getOrDefault(dateTime, ChronoField.SECOND_OF_MINUTE); - microSecond = DateUtils.getOrDefault(dateTime, ChronoField.MICRO_OF_SECOND); - - } catch (Exception ex) { - throw new AnalysisException("datetime literal [" + s + "] is invalid"); } if (checkRange() || checkDate()) { @@ -261,28 +185,34 @@ public class DateTimeLiteral extends DateLiteral { return new org.apache.doris.analysis.DateLiteral(year, month, day, hour, minute, second, Type.DATETIME); } - public Expression plusYears(int years) { - return fromJavaDateType(DateUtils.getTime(DATE_TIME_FORMATTER, getStringValue()).plusYears(years)); + public Expression plusYears(long years) { + return fromJavaDateType( + DateUtils.getTime(StandardDateFormat.DATE_TIME_FORMATTER, getStringValue()).plusYears(years)); } - public Expression plusMonths(int months) { - return fromJavaDateType(DateUtils.getTime(DATE_TIME_FORMATTER, getStringValue()).plusMonths(months)); + public Expression plusMonths(long months) { + return fromJavaDateType( + DateUtils.getTime(StandardDateFormat.DATE_TIME_FORMATTER, getStringValue()).plusMonths(months)); } - public Expression plusDays(int days) { - return fromJavaDateType(DateUtils.getTime(DATE_TIME_FORMATTER, getStringValue()).plusDays(days)); + public Expression plusDays(long days) { + return fromJavaDateType( + DateUtils.getTime(StandardDateFormat.DATE_TIME_FORMATTER, getStringValue()).plusDays(days)); } - public Expression plusHours(int hours) { - return fromJavaDateType(DateUtils.getTime(DATE_TIME_FORMATTER, getStringValue()).plusHours(hours)); + public Expression plusHours(long hours) { + return fromJavaDateType( + DateUtils.getTime(StandardDateFormat.DATE_TIME_FORMATTER, getStringValue()).plusHours(hours)); } - public Expression plusMinutes(int minutes) { - return fromJavaDateType(DateUtils.getTime(DATE_TIME_FORMATTER, getStringValue()).plusMinutes(minutes)); + public Expression plusMinutes(long minutes) { + return fromJavaDateType( + DateUtils.getTime(StandardDateFormat.DATE_TIME_FORMATTER, getStringValue()).plusMinutes(minutes)); } public Expression plusSeconds(long seconds) { - return fromJavaDateType(DateUtils.getTime(DATE_TIME_FORMATTER, getStringValue()).plusSeconds(seconds)); + return fromJavaDateType( + DateUtils.getTime(StandardDateFormat.DATE_TIME_FORMATTER, getStringValue()).plusSeconds(seconds)); } public long getHour() { @@ -324,27 +254,4 @@ public class DateTimeLiteral extends DateLiteral { : new DateTimeLiteral(dateTime.getYear(), dateTime.getMonthValue(), dateTime.getDayOfMonth(), dateTime.getHour(), dateTime.getMinute(), dateTime.getSecond()); } - - private static boolean haveTimeZoneOffset(String arg) { - Preconditions.checkArgument(arg.length() > 6); - return HAS_OFFSET_PART.matcher(arg.substring(arg.length() - 6)).matches(); - } - - private static boolean haveTimeZoneName(String arg) { - for (char ch : arg.toCharArray()) { - if (Character.isUpperCase(ch) && ch != 'T') { - return true; - } - } - return false; - } - - private static int getTimeZoneSplitPos(String arg) { - int split = arg.length() - 1; - for (; !Character.isAlphabetic(arg.charAt(split)); split--) { - } // skip +8 of UTC+8 - for (; split >= 0 && (Character.isUpperCase(arg.charAt(split)) || arg.charAt(split) == '/'); split--) { - } - return split + 1; - } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DateTimeV2Literal.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DateTimeV2Literal.java index ef09399a5aa..2b520b08ee7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DateTimeV2Literal.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DateTimeV2Literal.java @@ -23,6 +23,7 @@ import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; import org.apache.doris.nereids.types.DateTimeV2Type; import org.apache.doris.nereids.util.DateUtils; +import org.apache.doris.nereids.util.StandardDateFormat; import java.time.LocalDateTime; @@ -76,50 +77,57 @@ public class DateTimeV2Literal extends DateTimeLiteral { @Override public String getStringValue() { return String.format("%04d-%02d-%02d %02d:%02d:%02d" - + (getDataType().getScale() > 0 ? ".%0" + getDataType().getScale() + "d" : ""), + + (getDataType().getScale() > 0 ? ".%0" + getDataType().getScale() + "d" : ""), year, month, day, hour, minute, second, (int) (microSecond / Math.pow(10, DateTimeV2Type.MAX_SCALE - getDataType().getScale()))); } @Override - public Expression plusYears(int years) { - return fromJavaDateType(DateUtils.getTime(DATE_TIME_FORMATTER_TO_MICRO_SECOND, getStringValue()) - .plusYears(years), getDataType().getScale()); + public Expression plusYears(long years) { + return fromJavaDateType( + DateUtils.getTime(StandardDateFormat.DATE_TIME_FORMATTER_TO_MICRO_SECOND, getStringValue()) + .plusYears(years), getDataType().getScale()); } @Override - public Expression plusMonths(int months) { - return fromJavaDateType(DateUtils.getTime(DATE_TIME_FORMATTER_TO_MICRO_SECOND, getStringValue()) - .plusMonths(months), getDataType().getScale()); + public Expression plusMonths(long months) { + return fromJavaDateType( + DateUtils.getTime(StandardDateFormat.DATE_TIME_FORMATTER_TO_MICRO_SECOND, getStringValue()) + .plusMonths(months), getDataType().getScale()); } @Override - public Expression plusDays(int days) { - return fromJavaDateType(DateUtils.getTime(DATE_TIME_FORMATTER_TO_MICRO_SECOND, getStringValue()) - .plusDays(days), getDataType().getScale()); + public Expression plusDays(long days) { + return fromJavaDateType( + DateUtils.getTime(StandardDateFormat.DATE_TIME_FORMATTER_TO_MICRO_SECOND, getStringValue()) + .plusDays(days), getDataType().getScale()); } @Override - public Expression plusHours(int hours) { - return fromJavaDateType(DateUtils.getTime(DATE_TIME_FORMATTER_TO_MICRO_SECOND, getStringValue()) - .plusHours(hours), getDataType().getScale()); + public Expression plusHours(long hours) { + return fromJavaDateType( + DateUtils.getTime(StandardDateFormat.DATE_TIME_FORMATTER_TO_MICRO_SECOND, getStringValue()) + .plusHours(hours), getDataType().getScale()); } @Override - public Expression plusMinutes(int minutes) { - return fromJavaDateType(DateUtils.getTime(DATE_TIME_FORMATTER_TO_MICRO_SECOND, getStringValue()) - .plusMinutes(minutes), getDataType().getScale()); + public Expression plusMinutes(long minutes) { + return fromJavaDateType( + DateUtils.getTime(StandardDateFormat.DATE_TIME_FORMATTER_TO_MICRO_SECOND, getStringValue()) + .plusMinutes(minutes), getDataType().getScale()); } @Override public Expression plusSeconds(long seconds) { - return fromJavaDateType(DateUtils.getTime(DATE_TIME_FORMATTER_TO_MICRO_SECOND, getStringValue()) - .plusSeconds(seconds), getDataType().getScale()); + return fromJavaDateType( + DateUtils.getTime(StandardDateFormat.DATE_TIME_FORMATTER_TO_MICRO_SECOND, getStringValue()) + .plusSeconds(seconds), getDataType().getScale()); } - public Expression plusMicroSeconds(int microSeconds) { - return fromJavaDateType(DateUtils.getTime(DATE_TIME_FORMATTER_TO_MICRO_SECOND, getStringValue()) - .plusNanos(microSeconds * 1000L), getDataType().getScale()); + public Expression plusMicroSeconds(long microSeconds) { + return fromJavaDateType( + DateUtils.getTime(StandardDateFormat.DATE_TIME_FORMATTER_TO_MICRO_SECOND, getStringValue()) + .plusNanos(microSeconds * 1000L), getDataType().getScale()); } /** diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DateV2Literal.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DateV2Literal.java index 928b7ca0f24..c57607892e3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DateV2Literal.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DateV2Literal.java @@ -24,6 +24,7 @@ import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; import org.apache.doris.nereids.types.DateV2Type; import org.apache.doris.nereids.util.DateUtils; +import org.apache.doris.nereids.util.StandardDateFormat; import java.time.LocalDateTime; @@ -50,16 +51,18 @@ public class DateV2Literal extends DateLiteral { return visitor.visitDateV2Literal(this, context); } - public Expression plusDays(int days) { - return fromJavaDateType(DateUtils.getTime(DATE_FORMATTER, getStringValue()).plusDays(days)); + public Expression plusDays(long days) { + return fromJavaDateType(DateUtils.getTime(StandardDateFormat.DATE_FORMATTER, getStringValue()).plusDays(days)); } - public Expression plusMonths(int months) { - return fromJavaDateType(DateUtils.getTime(DATE_FORMATTER, getStringValue()).plusMonths(months)); + public Expression plusMonths(long months) { + return fromJavaDateType( + DateUtils.getTime(StandardDateFormat.DATE_FORMATTER, getStringValue()).plusMonths(months)); } - public Expression plusYears(int years) { - return fromJavaDateType(DateUtils.getTime(DATE_FORMATTER, getStringValue()).plusYears(years)); + public Expression plusYears(long years) { + return fromJavaDateType( + DateUtils.getTime(StandardDateFormat.DATE_FORMATTER, getStringValue()).plusYears(years)); } public static Expression fromJavaDateType(LocalDateTime dateTime) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/types/DateTimeV2Type.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/types/DateTimeV2Type.java index 6a57c7a218c..4980bc789c8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/types/DateTimeV2Type.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/types/DateTimeV2Type.java @@ -20,6 +20,7 @@ package org.apache.doris.nereids.types; import org.apache.doris.catalog.ScalarType; import org.apache.doris.catalog.Type; import org.apache.doris.nereids.exceptions.AnalysisException; +import org.apache.doris.nereids.trees.expressions.literal.DateTimeLiteral; import org.apache.doris.nereids.types.coercion.AbstractDataType; import org.apache.doris.nereids.types.coercion.DateLikeType; import org.apache.doris.nereids.types.coercion.IntegralType; @@ -83,10 +84,11 @@ public class DateTimeV2Type extends DateLikeType { * may be we need to check for validity? */ public static DateTimeV2Type forTypeFromString(String s) { - if (!s.contains(String.valueOf("."))) { + if (!s.contains(".")) { return DateTimeV2Type.SYSTEM_DEFAULT; } - return DateTimeV2Type.of(s.length() - s.lastIndexOf(".") - 1); + int scale = DateTimeLiteral.determineScale(s); + return DateTimeV2Type.of(scale); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/DateTimeFormatterUtils.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/DateTimeFormatterUtils.java index 1925e100aca..6e03ea5b37c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/DateTimeFormatterUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/DateTimeFormatterUtils.java @@ -39,8 +39,27 @@ import java.time.temporal.ChronoField; * Note incomplete times 'hh:mm:ss', 'hh:mm', 'D hh:mm', 'D hh', or 'ss' */ public class DateTimeFormatterUtils { - // Date: %Y-%m-%d - public static DateTimeFormatter DATE_FORMATTER = new DateTimeFormatterBuilder() + public static final DateTimeFormatter ZONE_FORMATTER = new DateTimeFormatterBuilder() + .optionalStart() + // .appendZoneText(TextStyle.FULL) + .appendZoneOrOffsetId() + .optionalEnd() + // .appendOptional( + // new DateTimeFormatterBuilder().appendOffset("+HH", "").toFormatter()) + // .appendOptional( + // new DateTimeFormatterBuilder().appendOffset("+HH:MM", "").toFormatter()) + // .appendOptional( + // new DateTimeFormatterBuilder().appendOffset("+HH:MM:SS", "").toFormatter()) + .toFormatter() + .withResolverStyle(ResolverStyle.STRICT); + // yymmdd + public static final DateTimeFormatter BASIC_TWO_DIGIT_DATE_FORMATTER = new DateTimeFormatterBuilder() + .appendValueReduced(ChronoField.YEAR, 2, 2, 1970) + .appendValue(ChronoField.MONTH_OF_YEAR, 2) + .appendValue(ChronoField.DAY_OF_MONTH, 2) + .toFormatter().withResolverStyle(ResolverStyle.STRICT); + // yyyy-mm-dd + public static final DateTimeFormatter DATE_FORMATTER = new DateTimeFormatterBuilder() .appendOptional( new DateTimeFormatterBuilder().appendValue(ChronoField.YEAR, 4).toFormatter()) .appendOptional( @@ -48,33 +67,72 @@ public class DateTimeFormatterUtils { .appendLiteral('-').appendValue(ChronoField.MONTH_OF_YEAR, 2) .appendLiteral('-').appendValue(ChronoField.DAY_OF_MONTH, 2) .toFormatter().withResolverStyle(ResolverStyle.STRICT); - // Date without delimiter: %Y%m%d - public static DateTimeFormatter BASIC_DATE_FORMATTER = new DateTimeFormatterBuilder() - .appendValue(ChronoField.YEAR, 4) - .appendValue(ChronoField.MONTH_OF_YEAR, 2) - .appendValue(ChronoField.DAY_OF_MONTH, 2) - .toFormatter().withResolverStyle(ResolverStyle.STRICT); - // Time: %H:%i:%s - public static DateTimeFormatter TIME_FORMATTER = new DateTimeFormatterBuilder() + // HH[:mm][:ss][.microsecond] + public static final DateTimeFormatter TIME_FORMATTER = new DateTimeFormatterBuilder() .appendValue(ChronoField.HOUR_OF_DAY, 2) - .appendLiteral(':').appendValue(ChronoField.MINUTE_OF_HOUR, 2) - .appendLiteral(':').appendValue(ChronoField.SECOND_OF_MINUTE, 2) + .appendOptional( + new DateTimeFormatterBuilder() + .appendLiteral(':').appendValue(ChronoField.MINUTE_OF_HOUR, 2) + .appendOptional( + new DateTimeFormatterBuilder() + .appendLiteral(':').appendValue(ChronoField.SECOND_OF_MINUTE, 2) + .appendOptional(new DateTimeFormatterBuilder() + .appendFraction(ChronoField.MICRO_OF_SECOND, 1, 6, true) + .toFormatter()) + .toFormatter() + ) + .toFormatter() + ) .toFormatter().withResolverStyle(ResolverStyle.STRICT); - // Time without delimiter: HHmmss[microsecond] - public static DateTimeFormatter BASIC_TIME_FORMATTER = new DateTimeFormatterBuilder() + // Time without delimiter: HHmmss[.microsecond] + private static final DateTimeFormatter BASIC_TIME_FORMATTER = new DateTimeFormatterBuilder() .appendValue(ChronoField.HOUR_OF_DAY, 2) .appendValue(ChronoField.MINUTE_OF_HOUR, 2) .appendValue(ChronoField.SECOND_OF_MINUTE, 2) .appendOptional(new DateTimeFormatterBuilder() .appendFraction(ChronoField.MICRO_OF_SECOND, 1, 6, true).toFormatter()) .toFormatter().withResolverStyle(ResolverStyle.STRICT); - + // yyyymmdd + private static final DateTimeFormatter BASIC_DATE_FORMATTER = new DateTimeFormatterBuilder() + .appendValue(ChronoField.YEAR, 4) + .appendValue(ChronoField.MONTH_OF_YEAR, 2) + .appendValue(ChronoField.DAY_OF_MONTH, 2) + .toFormatter().withResolverStyle(ResolverStyle.STRICT); // Date without delimiter - public static DateTimeFormatter BASIC_DATE_TIME_FORMATTER = new DateTimeFormatterBuilder() - .append(BASIC_DATE_FORMATTER) - .optionalStart() - .appendOptional(new DateTimeFormatterBuilder().appendLiteral('T').toFormatter()) + public static final DateTimeFormatter BASIC_DATE_TIME_FORMATTER = new DateTimeFormatterBuilder() + .appendOptional(BASIC_DATE_FORMATTER) + .appendOptional(BASIC_TWO_DIGIT_DATE_FORMATTER) + .appendLiteral('T') .append(BASIC_TIME_FORMATTER) - .optionalEnd() + .toFormatter().withResolverStyle(ResolverStyle.STRICT); + // Date without delimiter + public static final DateTimeFormatter BASIC_FORMATTER_WITHOUT_T = new DateTimeFormatterBuilder() + .append(BASIC_DATE_FORMATTER) + .appendOptional(BASIC_TIME_FORMATTER) + .toFormatter().withResolverStyle(ResolverStyle.STRICT); + + // Datetime + public static final DateTimeFormatter DATE_TIME_FORMATTER = new DateTimeFormatterBuilder() + .append(DATE_FORMATTER) + .appendLiteral(' ') + .append(TIME_FORMATTER) + .toFormatter().withResolverStyle(ResolverStyle.STRICT); + public static final DateTimeFormatter ZONE_DATE_FORMATTER = new DateTimeFormatterBuilder() + .appendOptional( + new DateTimeFormatterBuilder().appendValue(ChronoField.YEAR, 4).toFormatter()) + .appendOptional( + new DateTimeFormatterBuilder().appendValueReduced(ChronoField.YEAR, 2, 2, 1970).toFormatter()) + .appendLiteral('-').appendValue(ChronoField.MONTH_OF_YEAR, 2) + .appendLiteral('-').appendValue(ChronoField.DAY_OF_MONTH, 2) + // .optionalStart() + // .appendZoneOrOffsetId() + // .optionalEnd() + .append(ZONE_FORMATTER) + .toFormatter().withResolverStyle(ResolverStyle.STRICT); + public static final DateTimeFormatter ZONE_DATE_TIME_FORMATTER = new DateTimeFormatterBuilder() + .append(DATE_FORMATTER) + .appendLiteral(' ') + .append(TIME_FORMATTER) + .append(ZONE_FORMATTER) .toFormatter().withResolverStyle(ResolverStyle.STRICT); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/StandardDateFormat.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/StandardDateFormat.java new file mode 100644 index 00000000000..aa421eeab43 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/StandardDateFormat.java @@ -0,0 +1,53 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.util; + +import java.time.format.DateTimeFormatter; +import java.time.format.DateTimeFormatterBuilder; +import java.time.format.ResolverStyle; +import java.time.temporal.ChronoField; + +/** + * Following format is *standard* format of date/datetime. It will keep format of memory/output/printf is standard. + */ +public class StandardDateFormat { + // Date: %Y-%m-%d + public static DateTimeFormatter DATE_FORMATTER = new DateTimeFormatterBuilder() + .appendValue(ChronoField.YEAR, 4) + .appendLiteral('-').appendValue(ChronoField.MONTH_OF_YEAR, 2) + .appendLiteral('-').appendValue(ChronoField.DAY_OF_MONTH, 2) + .toFormatter().withResolverStyle(ResolverStyle.STRICT); + // %H:%i:%s + public static DateTimeFormatter TIME_FORMATTER = new DateTimeFormatterBuilder() + .appendValue(ChronoField.HOUR_OF_DAY, 2) + .appendLiteral(':').appendValue(ChronoField.MINUTE_OF_HOUR, 2) + .appendLiteral(':').appendValue(ChronoField.SECOND_OF_MINUTE, 2) + .toFormatter().withResolverStyle(ResolverStyle.STRICT); + public static DateTimeFormatter DATE_TIME_FORMATTER = new DateTimeFormatterBuilder() + .append(DATE_FORMATTER) + .appendLiteral(' ') + .append(TIME_FORMATTER) + .toFormatter().withResolverStyle(ResolverStyle.STRICT); + // "%Y-%m-%d %H:%i:%s.%f" + public static DateTimeFormatter DATE_TIME_FORMATTER_TO_MICRO_SECOND = new DateTimeFormatterBuilder() + .append(DATE_FORMATTER) + .appendLiteral(' ') + .append(TIME_FORMATTER) + .appendFraction(ChronoField.MICRO_OF_SECOND, 0, 6, true) // Notice: min size is 0 + .toFormatter().withResolverStyle(ResolverStyle.STRICT); +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/LiteralEqualTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/LiteralEqualTest.java new file mode 100644 index 00000000000..84422510fd2 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/LiteralEqualTest.java @@ -0,0 +1,45 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.trees.expressions; + +import org.apache.doris.nereids.trees.expressions.literal.BooleanLiteral; +import org.apache.doris.nereids.trees.expressions.literal.IntegerLiteral; +import org.apache.doris.nereids.trees.expressions.literal.Literal; +import org.apache.doris.nereids.trees.expressions.literal.NullLiteral; +import org.apache.doris.nereids.trees.expressions.literal.StringLiteral; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +class LiteralEqualTest { + + @Test + void testEqual() { + IntegerLiteral one = new IntegerLiteral(1); + IntegerLiteral anotherOne = new IntegerLiteral(1); + IntegerLiteral two = new IntegerLiteral(2); + Assertions.assertNotEquals(one, two); + Assertions.assertEquals(one, anotherOne); + StringLiteral str1 = new StringLiteral("hello"); + Assertions.assertNotEquals(str1, one); + Assertions.assertTrue(Literal.of("world") instanceof StringLiteral); + Assertions.assertTrue(Literal.of(null) instanceof NullLiteral); + Assertions.assertTrue(Literal.of(1) instanceof IntegerLiteral); + Assertions.assertTrue(Literal.of(false) instanceof BooleanLiteral); + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/literal/DateLiteralTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/literal/DateLiteralTest.java new file mode 100644 index 00000000000..307addf5244 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/literal/DateLiteralTest.java @@ -0,0 +1,83 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.trees.expressions.literal; + +import org.apache.doris.nereids.exceptions.AnalysisException; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Test; + +class DateLiteralTest { + @Test + void testDate() { + new DateLiteral("220101"); + new DateLiteral("22-01-01"); + + new DateLiteral("2022-01-01"); + new DateLiteral("20220101"); + + Assertions.assertThrows(AnalysisException.class, () -> new DateLiteral("-01-01")); + } + + @Test + void testZone() { + new DateLiteral("2022-01-01Z"); + new DateLiteral("2022-01-01UTC"); + new DateLiteral("2022-01-01GMT"); + // new DateLiteral("2022-01-01UTC+08"); + // new DateLiteral("2022-01-01UTC-06"); + new DateLiteral("2022-01-01UTC+08:00"); + new DateLiteral("2022-01-01UTC-06:00"); + new DateLiteral("2022-01-01Europe/London"); + } + + @Test + @Disabled + void testOffset() { + new DateLiteral("2022-01-01+01:00:00"); + new DateLiteral("2022-01-01+01:00"); + new DateLiteral("2022-01-01+01"); + new DateLiteral("2022-01-01+1:0:0"); + new DateLiteral("2022-01-01+1:0"); + new DateLiteral("2022-01-01+1"); + + new DateLiteral("2022-01-01-01:00:00"); + new DateLiteral("2022-01-01-01:00"); + new DateLiteral("2022-01-01-1:0:0"); + new DateLiteral("2022-01-01-1:0"); + + Assertions.assertThrows(AnalysisException.class, () -> new DateLiteral("2022-01-01-01")); + Assertions.assertThrows(AnalysisException.class, () -> new DateLiteral("2022-01-01-1")); + } + + @Disabled + @Test + void testIrregularDate() { + new DateLiteral("2016-07-02"); + + new DateLiteral("2016-7-02"); + new DateLiteral("2016-07-2"); + new DateLiteral("2016-7-2"); + + new DateLiteral("2016-07-02"); + new DateLiteral("2016-07-2"); + new DateLiteral("2016-7-02"); + new DateLiteral("2016-7-2"); + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/literal/DateTimeLiteralTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/literal/DateTimeLiteralTest.java new file mode 100644 index 00000000000..780483398fe --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/literal/DateTimeLiteralTest.java @@ -0,0 +1,369 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.trees.expressions.literal; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Test; + +class DateTimeLiteralTest { + @Test + void testWithoutZoneOrOffset() { + new DateTimeV2Literal("2022-08-01"); + + new DateTimeV2Literal("2022-08-01 01:01:01"); + new DateTimeV2Literal("2022-08-01 01:01"); + new DateTimeV2Literal("2022-08-01 01"); + + new DateTimeV2Literal("2022-08-01T01:01:01"); + new DateTimeV2Literal("2022-08-01T01:01"); + new DateTimeV2Literal("2022-08-01T01"); + + new DateTimeV2Literal("22-08-01T01:01:01"); + new DateTimeV2Literal("22-08-01T01:01"); + new DateTimeV2Literal("22-08-01T01"); + } + + @Test + void testDetermineScale() { + int scale = DateTimeLiteral.determineScale("2022-08-01T01:01:01.0"); + Assertions.assertEquals(0, scale); + scale = DateTimeLiteral.determineScale("2022-08-01T01:01:01.00000"); + Assertions.assertEquals(0, scale); + scale = DateTimeLiteral.determineScale("2022-08-01T01:01:01.000001"); + Assertions.assertEquals(6, scale); + scale = DateTimeLiteral.determineScale("2022-08-01T01:01:01.123456"); + Assertions.assertEquals(6, scale); + scale = DateTimeLiteral.determineScale("2022-08-01T01:01:01.0001"); + Assertions.assertEquals(4, scale); + scale = DateTimeLiteral.determineScale("2022-08-01T01:01:01.00010"); + Assertions.assertEquals(4, scale); + scale = DateTimeLiteral.determineScale("2022-08-01T01:01:01.12010"); + Assertions.assertEquals(4, scale); + scale = DateTimeLiteral.determineScale("2022-08-01T01:01:01.02010"); + Assertions.assertEquals(4, scale); + } + + @Test + void testTwoDigitYear() { + new DateTimeV2Literal("22-08-01T01"); + new DateTimeV2Literal("22-08-01 01"); + new DateTimeV2Literal("22-08-01T01:01"); + new DateTimeV2Literal("22-08-01 01:01"); + new DateTimeV2Literal("22-08-01T01:01:01"); + new DateTimeV2Literal("22-08-01 01:01:01"); + new DateTimeV2Literal("22-08-01T01"); + new DateTimeV2Literal("22-08-01 01"); + new DateTimeV2Literal("22-08-01T01:01"); + new DateTimeV2Literal("22-08-01 01:01"); + new DateTimeV2Literal("22-08-01T01:01:01"); + new DateTimeV2Literal("22-08-01 01:01:01"); + } + + @Test + void testZone() { + new DateTimeV2Literal("2022-08-01 01:01:01UTC"); + new DateTimeV2Literal("2022-08-01 01:01:01UT"); + new DateTimeV2Literal("2022-08-01 01:01:01GMT"); + new DateTimeV2Literal("2022-08-01 01:01:01Z"); + new DateTimeV2Literal("2022-08-01 01:01:01Europe/London"); + new DateTimeV2Literal("2022-08-01 01:01:01America/New_York"); + new DateTimeV2Literal("2022-08-01 01:01:01Z"); + new DateTimeV2Literal("2022-08-01 01:01:01Europe/Berlin"); + new DateTimeV2Literal("2022-08-01 01:01:01Europe/London"); + } + + @Test + void testZoneOrOffsetRight() { + java.util.function.BiConsumer<DateTimeV2Literal, Long> assertHour = (dateTimeV2Literal, expectHour) -> { + Assertions.assertSame(dateTimeV2Literal.hour, expectHour); + }; + DateTimeV2Literal dateTimeV2Literal; + dateTimeV2Literal = new DateTimeV2Literal("2022-08-01 00:00:00Europe/London"); // +01:00 + assertHour.accept(dateTimeV2Literal, 7L); + dateTimeV2Literal = new DateTimeV2Literal("2022-08-01 00:00:00America/New_York"); // -04:00 + assertHour.accept(dateTimeV2Literal, 12L); + dateTimeV2Literal = new DateTimeV2Literal("2022-08-01 00:00:00Asia/Shanghai"); + assertHour.accept(dateTimeV2Literal, 0L); + dateTimeV2Literal = new DateTimeV2Literal("2022-08-01 00:00:00+01:00"); + assertHour.accept(dateTimeV2Literal, 7L); + dateTimeV2Literal = new DateTimeV2Literal("2022-08-01 00:00:00-01:00"); + assertHour.accept(dateTimeV2Literal, 9L); + } + + @Test + void testTwoDigitalYearZone() { + new DateTimeV2Literal("22-08-01 01:01:01UTC"); + new DateTimeV2Literal("22-08-01 01:01:01UT"); + new DateTimeV2Literal("22-08-01 01:01:01GMT"); + new DateTimeV2Literal("22-08-01 01:01:01Z"); + new DateTimeV2Literal("22-08-01 01:01:01Europe/London"); + new DateTimeV2Literal("22-08-01 01:01:01UTC"); + new DateTimeV2Literal("22-08-01 01:01:01America/New_York"); + new DateTimeV2Literal("22-08-01 01:01:01Z"); + new DateTimeV2Literal("22-08-01 01:01:01Europe/Berlin"); + new DateTimeV2Literal("22-08-01 01:01:01Europe/London"); + } + + @Test + void testZoneOffset() { + new DateTimeV2Literal("2022-08-01 01:01:01UTC+01:01:01"); + // new DateTimeV2Literal("2022-08-01 01:01:01UTC+1:1:1"); + + new DateTimeV2Literal("2022-08-01 01:01:01UTC+01:01"); + + // new DateTimeV2Literal("2022-08-01 01:01:01UTC+01"); + // new DateTimeV2Literal("2022-08-01 01:01:01UTC+1"); + } + + @Test + void testTwoDigitalYearZoneOffset() { + new DateTimeV2Literal("22-08-01 01:01:01UTC+01:01:01"); + // new DateTimeV2Literal("22-08-01 01:01:01UTC+1:1:1"); + + new DateTimeV2Literal("22-08-01 01:01:01UTC+01:01"); + + // new DateTimeV2Literal("22-08-01 01:01:01UTC+01"); + // new DateTimeV2Literal("22-08-01 01:01:01UTC+1"); + } + + @Test + void testOffset() { + new DateTimeV2Literal("2022-08-01 01:01:01+01:01:01"); + new DateTimeV2Literal("2022-08-01 01:01:01+01:01"); + // new DateTimeV2Literal("2022-08-01 01:01:01+01"); + // new DateTimeV2Literal("2022-08-01 01:01:01+01:1:01"); + // new DateTimeV2Literal("2022-08-01 01:01:01+01:1"); + // new DateTimeV2Literal("2022-08-01 01:01:01+01:01:1"); + // new DateTimeV2Literal("2022-08-01 01:01:01+1:1:1"); + // new DateTimeV2Literal("2022-08-01 01:01:01+1:1"); + // new DateTimeV2Literal("2022-08-01 01:01:01+1"); + + new DateTimeV2Literal("2022-05-01 01:02:55+02:30"); + new DateTimeV2Literal("2022-05-01 01:02:55.123-02:30"); + new DateTimeV2Literal("2022-06-01T01:02:55+04:30"); + new DateTimeV2Literal("2022-06-01 01:02:55.123-07:30"); + // new DateTimeV2Literal("20220701010255+07:00"); + // new DateTimeV2Literal("20220701010255-05:00"); + new DateTimeV2Literal("2022-05-01 01:02:55+02:30"); + + new DateTimeV2Literal("2022-05-01 01:02:55.123-02:30"); + new DateTimeV2Literal("2022-06-01T01:02:55+04:30"); + new DateTimeV2Literal("2022-06-01 01:02:55.123-07:30"); + // new DateTimeV2Literal("20220701010255+07:00"); + // new DateTimeV2Literal("20220701010255-05:00"); + } + + @Test + void testDateTime() { + // new DateTimeV2Literal("2022-08-01 01:01:01UTC+1:1:1"); + // new DateTimeV2Literal("2022-08-01 01:01:01UTC+1:1"); + // new DateTimeV2Literal("2022-08-01 01:01:01UTC+1"); + + new DateTimeV2Literal("0001-01-01 00:01:01"); + new DateTimeV2Literal("0001-01-01 00:01:01.001"); + new DateTimeV2Literal("0001-01-01 00:01:01.00305"); + + new DateTimeV2Literal("2022-01-01 01:02:55"); + new DateTimeV2Literal("2022-01-01 01:02:55.123"); + new DateTimeV2Literal("2022-02-01 01:02:55Z"); + new DateTimeV2Literal("2022-02-01 01:02:55.123Z"); + // new DateTimeV2Literal("2022-03-01 01:02:55UTC+8"); + new DateTimeV2Literal("2022-03-01 01:02:55.123UTC"); + // new DateTimeV2Literal("2022-04-01 01:02:55UTC-6"); + // new DateTimeV2Literal("2022-04-01T01:02:55UTC-6"); + // new DateTimeV2Literal("2022-04-01T01:02:55.123UTC+6"); + + new DateTimeV2Literal("2022-01-01 01:02:55"); + new DateTimeV2Literal("2022-01-01 01:02:55.123"); + new DateTimeV2Literal("2022-02-01 01:02:55Z"); + new DateTimeV2Literal("2022-02-01 01:02:55.123Z"); + // new DateTimeV2Literal("2022-03-01 01:02:55UTC+8"); + new DateTimeV2Literal("2022-03-01 01:02:55.123UTC"); + // new DateTimeV2Literal("2022-04-01T01:02:55UTC-6"); + // new DateTimeV2Literal("2022-04-01T01:02:55.123UTC+6"); + + new DateTimeV2Literal("0001-01-01"); + // new DateTimeV2Literal("20220801GMT+5"); + // new DateTimeV2Literal("20220801GMT-3"); + } + + @Disabled + @Test + void testIrregularDateTime() { + new DateLiteral("2016-07-02 01:01:00"); + + new DateLiteral("2016-7-02 01:01:00"); + new DateLiteral("2016-07-2 01:01:00"); + new DateLiteral("2016-7-2 01:01:00"); + + new DateLiteral("2016-07-02 1:01:00"); + new DateLiteral("2016-07-02 01:1:00"); + new DateLiteral("2016-07-02 01:01:0"); + new DateLiteral("2016-07-02 1:1:00"); + new DateLiteral("2016-07-02 1:01:0"); + new DateLiteral("2016-07-02 10:1:0"); + new DateLiteral("2016-07-02 1:1:0"); + + new DateLiteral("2016-7-2 1:1:0"); + new DateLiteral("2016-7-02 1:01:0"); + new DateLiteral("2016-07-2 1:1:0"); + new DateLiteral("2016-7-02 01:01:0"); + new DateLiteral("2016-7-2 01:1:0"); + } + + @Disabled + @Test + void testIrregularDateTimeHour() { + new DateTimeV2Literal("2016-07-02 01"); + new DateTimeV2Literal("2016-07-02 1"); + + new DateTimeV2Literal("2016-7-02 1"); + new DateTimeV2Literal("2016-7-02 01"); + + new DateTimeV2Literal("2016-07-2 1"); + new DateTimeV2Literal("2016-07-2 01"); + + new DateTimeV2Literal("2016-7-2 1"); + new DateTimeV2Literal("2016-7-2 01"); + } + + @Disabled + @Test + void testIrregularDateTimeHourMinute() { + new DateTimeV2Literal("2016-07-02 01:01"); + new DateTimeV2Literal("2016-07-02 1:01"); + new DateTimeV2Literal("2016-07-02 01:1"); + new DateTimeV2Literal("2016-07-02 1:1"); + + new DateTimeV2Literal("2016-7-02 01:01"); + new DateTimeV2Literal("2016-7-02 1:01"); + new DateTimeV2Literal("2016-7-02 01:1"); + new DateTimeV2Literal("2016-7-02 1:1"); + + new DateTimeV2Literal("2016-07-2 01:01"); + new DateTimeV2Literal("2016-07-2 1:01"); + new DateTimeV2Literal("2016-07-2 01:1"); + new DateTimeV2Literal("2016-07-2 1:1"); + + new DateTimeV2Literal("2016-7-2 01:01"); + new DateTimeV2Literal("2016-7-2 1:01"); + new DateTimeV2Literal("2016-7-2 01:1"); + new DateTimeV2Literal("2016-7-2 1:1"); + } + + @Disabled + @Test + void testIrregularDateTimeHourMinuteSecond() { + new DateTimeV2Literal("2016-07-02 01:01:01"); + new DateTimeV2Literal("2016-07-02 1:01:01"); + new DateTimeV2Literal("2016-07-02 01:1:01"); + new DateTimeV2Literal("2016-07-02 1:1:01"); + new DateTimeV2Literal("2016-07-02 01:01:1"); + new DateTimeV2Literal("2016-07-02 1:01:1"); + new DateTimeV2Literal("2016-07-02 01:1:1"); + new DateTimeV2Literal("2016-07-02 1:1:1"); + + new DateTimeV2Literal("2016-7-02 01:01:01"); + new DateTimeV2Literal("2016-7-02 1:01:01"); + new DateTimeV2Literal("2016-7-02 01:1:01"); + new DateTimeV2Literal("2016-7-02 1:1:01"); + new DateTimeV2Literal("2016-7-02 01:01:1"); + new DateTimeV2Literal("2016-7-02 1:01:1"); + new DateTimeV2Literal("2016-7-02 01:1:1"); + new DateTimeV2Literal("2016-7-02 1:1:1"); + + new DateTimeV2Literal("2016-07-2 01:01:01"); + new DateTimeV2Literal("2016-07-2 1:01:01"); + new DateTimeV2Literal("2016-07-2 01:1:01"); + new DateTimeV2Literal("2016-07-2 1:1:01"); + new DateTimeV2Literal("2016-07-2 01:01:1"); + new DateTimeV2Literal("2016-07-2 1:01:1"); + new DateTimeV2Literal("2016-07-2 01:1:1"); + new DateTimeV2Literal("2016-07-2 1:1:1"); + + new DateTimeV2Literal("2016-7-2 01:01:01"); + new DateTimeV2Literal("2016-7-2 1:01:01"); + new DateTimeV2Literal("2016-7-2 01:1:01"); + new DateTimeV2Literal("2016-7-2 1:1:01"); + new DateTimeV2Literal("2016-7-2 01:01:1"); + new DateTimeV2Literal("2016-7-2 1:01:1"); + new DateTimeV2Literal("2016-7-2 01:1:1"); + new DateTimeV2Literal("2016-7-2 1:1:1"); + } + + @Disabled + @Test + void testIrregularDateTimeHourMinuteSecondMicrosecond() { + new DateTimeV2Literal("2016-07-02 01:01:01.1"); + new DateTimeV2Literal("2016-07-02 1:01:01.1"); + new DateTimeV2Literal("2016-07-02 01:1:01.1"); + new DateTimeV2Literal("2016-07-02 1:1:01.1"); + new DateTimeV2Literal("2016-07-02 01:01:1.1"); + new DateTimeV2Literal("2016-07-02 1:01:1.1"); + new DateTimeV2Literal("2016-07-02 01:1:1.1"); + new DateTimeV2Literal("2016-07-02 1:1:1.1"); + + new DateTimeV2Literal("2016-7-02 01:01:01.1"); + new DateTimeV2Literal("2016-7-02 1:01:01.1"); + new DateTimeV2Literal("2016-7-02 01:1:01.1"); + new DateTimeV2Literal("2016-7-02 1:1:01.1"); + new DateTimeV2Literal("2016-7-02 01:01:1.1"); + new DateTimeV2Literal("2016-7-02 1:01:1.1"); + new DateTimeV2Literal("2016-7-02 01:1:1.1"); + new DateTimeV2Literal("2016-7-02 1:1:1.1"); + + new DateTimeV2Literal("2016-07-2 01:01:01.1"); + new DateTimeV2Literal("2016-07-2 1:01:01.1"); + new DateTimeV2Literal("2016-07-2 01:1:01.1"); + new DateTimeV2Literal("2016-07-2 1:1:01.1"); + new DateTimeV2Literal("2016-07-2 01:01:1.1"); + new DateTimeV2Literal("2016-07-2 1:01:1.1"); + new DateTimeV2Literal("2016-07-2 01:1:1.1"); + new DateTimeV2Literal("2016-07-2 1:1:1.1"); + + new DateTimeV2Literal("2016-7-2 01:01:01.1"); + new DateTimeV2Literal("2016-7-2 1:01:01.1"); + new DateTimeV2Literal("2016-7-2 01:1:01.1"); + new DateTimeV2Literal("2016-7-2 1:1:01.1"); + new DateTimeV2Literal("2016-7-2 01:01:1.1"); + new DateTimeV2Literal("2016-7-2 1:01:1.1"); + new DateTimeV2Literal("2016-7-2 01:1:1.1"); + new DateTimeV2Literal("2016-7-2 1:1:1.1"); + + // Testing with microsecond of length 2 + new DateTimeV2Literal("2016-07-02 01:01:01.12"); + new DateTimeV2Literal("2016-7-02 01:01:01.12"); + + // Testing with microsecond of length 3 + new DateTimeV2Literal("2016-07-02 01:01:01.123"); + new DateTimeV2Literal("2016-7-02 01:01:01.123"); + + // Testing with microsecond of length 4 + new DateTimeV2Literal("2016-07-02 01:01:01.1234"); + new DateTimeV2Literal("2016-7-02 01:01:01.1234"); + + // Testing with microsecond of length 5 + new DateTimeV2Literal("2016-07-02 01:01:01.12345"); + new DateTimeV2Literal("2016-7-02 01:01:01.12345"); + + // Testing with microsecond of length 6 + new DateTimeV2Literal("2016-07-02 01:01:01.123456"); + new DateTimeV2Literal("2016-7-02 01:01:01.123456"); + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/util/DateTimeFormatterUtilsTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/util/DateTimeFormatterUtilsTest.java index c40f5c4a85f..76c4fb39e24 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/util/DateTimeFormatterUtilsTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/util/DateTimeFormatterUtilsTest.java @@ -24,8 +24,25 @@ import java.time.format.DateTimeFormatter; import java.time.format.DateTimeParseException; import java.time.temporal.ChronoField; import java.time.temporal.TemporalAccessor; +import java.util.function.Consumer; class DateTimeFormatterUtilsTest { + @Test + void test() { + DateTimeFormatter formatter = DateTimeFormatterUtils.ZONE_FORMATTER; + + formatter.parse(""); + + // formatter.parse("UTC+01"); + formatter.parse("UTC+01:00"); + formatter.parse("UTC+01:00:00"); + + // formatter.parse("GMT+01"); + formatter.parse("GMT+01:00"); + formatter.parse("Asia/Shanghai"); + formatter.parse("Z"); + } + private void assertDatePart(TemporalAccessor dateTime) { Assertions.assertEquals(2020, dateTime.get(ChronoField.YEAR)); Assertions.assertEquals(2, dateTime.get(ChronoField.MONTH_OF_YEAR)); @@ -34,29 +51,41 @@ class DateTimeFormatterUtilsTest { @Test void testBasicDateTimeFormatter() { - DateTimeFormatter formatter = DateTimeFormatterUtils.BASIC_DATE_TIME_FORMATTER; + DateTimeFormatter formatter = DateTimeFormatterUtils.BASIC_FORMATTER_WITHOUT_T; TemporalAccessor dateTime = formatter.parse("20200219"); assertDatePart(dateTime); dateTime = formatter.parse("20200219010101"); assertDatePart(dateTime); - dateTime = formatter.parse("20200219T010101"); + dateTime = formatter.parse("20200219010101.1"); assertDatePart(dateTime); - // failed case - Assertions.assertThrows(DateTimeParseException.class, () -> formatter.parse("20200219 010101")); - - // microsecond dateTime = formatter.parse("20200219010101.000001"); assertDatePart(dateTime); - dateTime = formatter.parse("20200219T010101.000001"); - assertDatePart(dateTime); dateTime = formatter.parse("20200219010101.1"); assertDatePart(dateTime); + + formatter = DateTimeFormatterUtils.BASIC_DATE_TIME_FORMATTER; + dateTime = formatter.parse("20200219T010101"); + assertDatePart(dateTime); dateTime = formatter.parse("20200219T010101.1"); assertDatePart(dateTime); - Assertions.assertThrows(DateTimeParseException.class, () -> formatter.parse("20200219010101.")); - Assertions.assertThrows(DateTimeParseException.class, () -> formatter.parse("20200219010101.0000001")); - Assertions.assertThrows(DateTimeParseException.class, () -> formatter.parse("20200219T010101.")); - Assertions.assertThrows(DateTimeParseException.class, () -> formatter.parse("20200219T010101.0000001")); + dateTime = formatter.parse("20200219T010101.000001"); + assertDatePart(dateTime); + dateTime = formatter.parse("20200219T010101.1"); + assertDatePart(dateTime); + + // failed case + DateTimeFormatter withT = DateTimeFormatterUtils.BASIC_DATE_TIME_FORMATTER; + Assertions.assertThrows(DateTimeParseException.class, () -> withT.parse("20200219 010101")); + Assertions.assertThrows(DateTimeParseException.class, () -> withT.parse("20200219010101.")); + Assertions.assertThrows(DateTimeParseException.class, () -> withT.parse("20200219010101.0000001")); + Assertions.assertThrows(DateTimeParseException.class, () -> withT.parse("20200219T010101.")); + Assertions.assertThrows(DateTimeParseException.class, () -> withT.parse("20200219T010101.0000001")); + DateTimeFormatter withoutT = DateTimeFormatterUtils.BASIC_FORMATTER_WITHOUT_T; + Assertions.assertThrows(DateTimeParseException.class, () -> withoutT.parse("20200219 010101")); + Assertions.assertThrows(DateTimeParseException.class, () -> withoutT.parse("20200219010101.")); + Assertions.assertThrows(DateTimeParseException.class, () -> withoutT.parse("20200219010101.0000001")); + Assertions.assertThrows(DateTimeParseException.class, () -> withoutT.parse("20200219T010101.")); + Assertions.assertThrows(DateTimeParseException.class, () -> withoutT.parse("20200219T010101.0000001")); } @Test @@ -79,4 +108,32 @@ class DateTimeFormatterUtilsTest { } } } + + @Test + void testDateTimeFormatter() { + DateTimeFormatter formatter = DateTimeFormatterUtils.DATE_TIME_FORMATTER; + TemporalAccessor dateTime = formatter.parse("2020-02-19 01:01:01"); + assertDatePart(dateTime); + Assertions.assertThrows(DateTimeParseException.class, () -> formatter.parse("2020-02-19T01:01:01")); + Assertions.assertThrows(DateTimeParseException.class, () -> formatter.parse("2020-02-1901:01:01")); + } + + @Test + void testTimeFormatter() { + // use lambda function to assert time is correct. + Consumer<TemporalAccessor> assertTime = (dateTime) -> + Assertions.assertEquals(1, dateTime.get(ChronoField.HOUR_OF_DAY)); + + DateTimeFormatter timeFormatter = DateTimeFormatterUtils.TIME_FORMATTER; + TemporalAccessor dateTime = timeFormatter.parse("01:01:01.000001"); + assertTime.accept(dateTime); + dateTime = timeFormatter.parse("01:01:01.1"); + assertTime.accept(dateTime); + dateTime = timeFormatter.parse("01:01:01"); + assertTime.accept(dateTime); + dateTime = timeFormatter.parse("01:01"); + assertTime.accept(dateTime); + dateTime = timeFormatter.parse("01"); + assertTime.accept(dateTime); + } } --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
