This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new e769597fd2 [Improvement] (datetime) support microsecond for date
literal (#10917)
e769597fd2 is described below
commit e769597fd292b5be6bd9a31147ae5a8c8f993265
Author: Gabriel <[email protected]>
AuthorDate: Mon Jul 18 21:39:39 2022 +0800
[Improvement] (datetime) support microsecond for date literal (#10917)
* [Improvement] (datetime) support microsecond for date literal
* remove joda dependency
---
fe/fe-core/pom.xml | 5 -
.../org/apache/doris/analysis/DateLiteral.java | 340 ++++++++++++---------
.../java/org/apache/doris/rewrite/FEFunctions.java | 13 +-
.../org/apache/doris/analysis/DateLiteralTest.java | 49 ++-
.../org/apache/doris/rewrite/FEFunctionsTest.java | 15 +-
fe/pom.xml | 7 -
fe/spark-dpp/pom.xml | 6 -
.../apache/doris/load/loadv2/dpp/ColumnParser.java | 20 +-
8 files changed, 270 insertions(+), 185 deletions(-)
diff --git a/fe/fe-core/pom.xml b/fe/fe-core/pom.xml
index 6e3fd28e1f..44eabcce13 100644
--- a/fe/fe-core/pom.xml
+++ b/fe/fe-core/pom.xml
@@ -215,11 +215,6 @@ under the License.
<artifactId>jmockit</artifactId>
<scope>test</scope>
</dependency>
- <!-- https://mvnrepository.com/artifact/joda-time/joda-time -->
- <dependency>
- <groupId>joda-time</groupId>
- <artifactId>joda-time</artifactId>
- </dependency>
<dependency>
<groupId>commons-io</groupId>
<artifactId>commons-io</artifactId>
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/analysis/DateLiteral.java
b/fe/fe-core/src/main/java/org/apache/doris/analysis/DateLiteral.java
index 52a6e58100..cc160c989b 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/DateLiteral.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/DateLiteral.java
@@ -34,17 +34,22 @@ import com.google.common.base.Preconditions;
import com.google.common.collect.Maps;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
-import org.joda.time.DateTime;
-import org.joda.time.DateTimeZone;
-import org.joda.time.LocalDateTime;
-import org.joda.time.format.DateTimeFormatter;
-import org.joda.time.format.DateTimeFormatterBuilder;
import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
import java.nio.ByteBuffer;
+import java.sql.Timestamp;
+import java.time.LocalDateTime;
import java.time.Year;
+import java.time.ZoneId;
+import java.time.ZonedDateTime;
+import java.time.format.DateTimeFormatter;
+import java.time.format.DateTimeFormatterBuilder;
+import java.time.format.TextStyle;
+import java.time.temporal.ChronoField;
+import java.time.temporal.TemporalAccessor;
+import java.util.Collections;
import java.util.Map;
import java.util.Objects;
import java.util.TimeZone;
@@ -57,23 +62,20 @@ public class DateLiteral extends LiteralExpr {
private static final DateLiteral MAX_DATE = new DateLiteral(9999, 12, 31);
private static final DateLiteral MIN_DATETIME = new DateLiteral(0000, 1,
1, 0, 0, 0);
private static final DateLiteral MAX_DATETIME = new DateLiteral(9999, 12,
31, 23, 59, 59);
+
+ private static final DateLiteral MIN_DATETIMEV2
+ = new DateLiteral(0000, 1, 1, 0, 0, 0, 0);
+ private static final DateLiteral MAX_DATETIMEV2
+ = new DateLiteral(9999, 12, 31, 23, 59, 59, 999999L);
private static final int DATEKEY_LENGTH = 8;
private static final int MAX_MICROSECOND = 999999;
private static final int DATETIME_TO_MINUTE_STRING_LENGTH = 16;
private static final int DATETIME_TO_HOUR_STRING_LENGTH = 13;
+ private static final int DATETIME_TO_SECOND_STRING_LENGTH = 19;
private static DateTimeFormatter DATE_TIME_FORMATTER = null;
- private static DateTimeFormatter DATE_TIME_FORMATTER_TO_HOUR = null;
- private static DateTimeFormatter DATE_TIME_FORMATTER_TO_MINUTE = null;
+ private static DateTimeFormatter DATE_TIME_FORMATTER_TO_MICRO_SECOND =
null;
private static DateTimeFormatter DATE_FORMATTER = null;
- /*
- * Dates containing two-digit year values are ambiguous because the
century is unknown.
- * MySQL interprets two-digit year values using these rules:
- * Year values in the range 70-99 are converted to 1970-1999.
- * Year values in the range 00-69 are converted to 2000-2069.
- * */
- private static DateTimeFormatter DATE_TIME_FORMATTER_TWO_DIGIT = null;
- private static DateTimeFormatter DATE_FORMATTER_TWO_DIGIT = null;
/*
* The datekey type is widely used in data warehouses
* For example, 20121229 means '2012-12-29'
@@ -92,12 +94,12 @@ public class DateLiteral extends LiteralExpr {
static {
try {
DATE_TIME_FORMATTER = formatBuilder("%Y-%m-%d
%H:%i:%s").toFormatter();
- DATE_TIME_FORMATTER_TO_HOUR = formatBuilder("%Y-%m-%d
%H").toFormatter();
- DATE_TIME_FORMATTER_TO_MINUTE = formatBuilder("%Y-%m-%d
%H:%i").toFormatter();
DATE_FORMATTER = formatBuilder("%Y-%m-%d").toFormatter();
DATEKEY_FORMATTER = formatBuilder("%Y%m%d").toFormatter();
- DATE_TIME_FORMATTER_TWO_DIGIT = formatBuilder("%y-%m-%d
%H:%i:%s").toFormatter();
- DATE_FORMATTER_TWO_DIGIT = formatBuilder("%y-%m-%d").toFormatter();
+ DATE_TIME_FORMATTER_TO_MICRO_SECOND = new
DateTimeFormatterBuilder()
+ .appendPattern("uuuu-MM-dd HH:mm:ss")
+ .appendFraction(ChronoField.NANO_OF_SECOND, 0, 6, true)
+ .toFormatter();
} catch (AnalysisException e) {
LOG.error("invalid date format", e);
System.exit(-1);
@@ -181,12 +183,18 @@ public class DateLiteral extends LiteralExpr {
} else {
copy(MIN_DATE);
}
- } else {
+ } else if (type.equals(Type.DATETIME)) {
if (isMax) {
copy(MAX_DATETIME);
} else {
copy(MIN_DATETIME);
}
+ } else {
+ if (isMax) {
+ copy(MAX_DATETIMEV2);
+ } else {
+ copy(MIN_DATETIMEV2);
+ }
}
analysisDone();
}
@@ -198,24 +206,30 @@ public class DateLiteral extends LiteralExpr {
}
public DateLiteral(long unixTimestamp, TimeZone timeZone, Type type)
throws AnalysisException {
- DateTime dt = new DateTime(unixTimestamp,
DateTimeZone.forTimeZone(timeZone));
- year = dt.getYear();
- month = dt.getMonthOfYear();
- day = dt.getDayOfMonth();
- hour = dt.getHourOfDay();
- minute = dt.getMinuteOfHour();
- second = dt.getSecondOfMinute();
+ Timestamp timestamp = new Timestamp(unixTimestamp);
+
+ ZonedDateTime zonedDateTime =
ZonedDateTime.ofInstant(timestamp.toInstant(), ZoneId.of(timeZone.getID()));
+ year = zonedDateTime.getYear();
+ month = zonedDateTime.getMonthValue();
+ day = zonedDateTime.getDayOfMonth();
+ hour = zonedDateTime.getHour();
+ minute = zonedDateTime.getMinute();
+ second = zonedDateTime.getSecond();
+ microsecond = zonedDateTime.get(ChronoField.MICRO_OF_SECOND);
if (type.equals(Type.DATE)) {
hour = 0;
minute = 0;
second = 0;
+ microsecond = 0;
this.type = Type.DATE;
} else if (type.equals(Type.DATETIME)) {
this.type = Type.DATETIME;
+ microsecond = 0;
} else if (type.equals(Type.DATEV2)) {
hour = 0;
minute = 0;
second = 0;
+ microsecond = 0;
this.type = Type.DATEV2;
} else if (type.equals(Type.DATETIMEV2)) {
this.type = Type.DATETIMEV2;
@@ -235,9 +249,6 @@ public class DateLiteral extends LiteralExpr {
}
public DateLiteral(long year, long month, long day, Type type) {
- this.hour = hour;
- this.minute = minute;
- this.second = second;
this.year = year;
this.month = month;
this.day = day;
@@ -281,12 +292,12 @@ public class DateLiteral extends LiteralExpr {
public DateLiteral(LocalDateTime dateTime, Type type) {
this.year = dateTime.getYear();
- this.month = dateTime.getMonthOfYear();
+ this.month = dateTime.getMonthValue();
this.day = dateTime.getDayOfMonth();
- this.hour = dateTime.getHourOfDay();
- this.minute = dateTime.getMinuteOfHour();
- this.second = dateTime.getSecondOfMinute();
- this.microsecond = dateTime.getMillisOfSecond() * 1000L;
+ this.hour = dateTime.getHour();
+ this.minute = dateTime.getMinute();
+ this.second = dateTime.getSecond();
+ this.microsecond = dateTime.get(ChronoField.MICRO_OF_SECOND);
this.type = type;
}
@@ -309,40 +320,86 @@ public class DateLiteral extends LiteralExpr {
private void init(String s, Type type) throws AnalysisException {
try {
Preconditions.checkArgument(type.isDateType());
- LocalDateTime dateTime;
- if (type.equals(Type.DATE) || type.equals(Type.DATEV2)) {
- if (s.split("-")[0].length() == 2) {
- dateTime = DATE_FORMATTER_TWO_DIGIT.parseLocalDateTime(s);
- } else if (s.length() == DATEKEY_LENGTH && !s.contains("-")) {
- // handle format like 20210106, but should not handle
2021-1-6
- dateTime = DATEKEY_FORMATTER.parseLocalDateTime(s);
- } else {
- dateTime = DATE_FORMATTER.parseLocalDateTime(s);
- }
+ TemporalAccessor dateTime;
+ if (s.length() == DATEKEY_LENGTH && !s.contains("-")) {
+ // handle format like 20210106, but should not handle 2021-1-6
+ dateTime = DATEKEY_FORMATTER.parse(s);
} else {
- if (s.split("-")[0].length() == 2) {
- dateTime =
DATE_TIME_FORMATTER_TWO_DIGIT.parseLocalDateTime(s);
- } else {
- // parse format '%Y-%m-%d %H:%i' and '%Y-%m-%d %H'
- if (s.length() == DATETIME_TO_MINUTE_STRING_LENGTH) {
- dateTime =
DATE_TIME_FORMATTER_TO_MINUTE.parseLocalDateTime(s);
- } else if (s.length() == DATETIME_TO_HOUR_STRING_LENGTH) {
- dateTime =
DATE_TIME_FORMATTER_TO_HOUR.parseLocalDateTime(s);
- } else {
- dateTime = DATE_TIME_FORMATTER.parseLocalDateTime(s);
+ String[] datePart = s.contains(" ") ? s.split("
")[0].split("-") : s.split("-");
+ DateTimeFormatterBuilder builder = new
DateTimeFormatterBuilder();
+ if (datePart.length != 3) {
+ throw new AnalysisException("Invalid date value: " + s);
+ }
+ for (int i = 0; i < datePart.length; i++) {
+ switch (i) {
+ case 0:
+ if (datePart[i].length() == 2) {
+ // If year is represented by two digits,
number bigger than 70 will be prefixed
+ // with 19 otherwise 20. e.g. 69 -> 2069, 70
-> 1970.
+ builder.appendValueReduced(ChronoField.YEAR,
2, 2, 1970);
+ } else {
+ builder.appendPattern(String.join("",
Collections.nCopies(datePart[i].length(), "u")));
+ }
+ break;
+ case 1:
+ builder.appendPattern(String.join("",
Collections.nCopies(datePart[i].length(), "M")));
+ break;
+ case 2:
+ builder.appendPattern(String.join("",
Collections.nCopies(datePart[i].length(), "d")));
+ break;
+ default:
+ throw new AnalysisException("Two many parts in
date format " + s);
+ }
+ if (i < datePart.length - 1) {
+ builder.appendLiteral("-");
+ }
+ }
+ if (s.contains(" ")) {
+ builder.appendLiteral(" ");
+ }
+ String[] timePart = s.contains(" ") ? s.split("
")[1].split(":") : new String[]{};
+ if (timePart.length > 0 && (type.equals(Type.DATE) ||
type.equals(Type.DATEV2))) {
+ throw new AnalysisException("Invalid date value: " + s);
+ }
+ if (timePart.length == 0 && (type.equals(Type.DATETIME) ||
type.equals(Type.DATETIMEV2))) {
+ throw new AnalysisException("Invalid datetime value: " +
s);
+ }
+ for (int i = 0; i < timePart.length; i++) {
+ switch (i) {
+ case 0:
+ builder.appendPattern(String.join("",
Collections.nCopies(timePart[i].length(), "H")));
+ break;
+ case 1:
+ builder.appendPattern(String.join("",
Collections.nCopies(timePart[i].length(), "m")));
+ break;
+ case 2:
+ builder.appendPattern(String.join("",
Collections.nCopies(timePart[i].contains(".")
+ ? timePart[i].split("\\.")[0].length() :
timePart[i].length(), "s")));
+ if (timePart[i].contains(".")) {
+
builder.appendFraction(ChronoField.NANO_OF_SECOND, 0, 6, true);
+ }
+ break;
+ default:
+ throw new AnalysisException("Two many parts in
time format " + s);
+ }
+ if (i < timePart.length - 1) {
+ builder.appendLiteral(":");
}
}
+ DateTimeFormatter formatter = builder.toFormatter();
+ dateTime = formatter.parse(s);
}
- year = dateTime.getYear();
- month = dateTime.getMonthOfYear();
- day = dateTime.getDayOfMonth();
- hour = dateTime.getHourOfDay();
- minute = dateTime.getMinuteOfHour();
- second = dateTime.getSecondOfMinute();
+ year = getOrDefault(dateTime, ChronoField.YEAR, 0);
+ month = getOrDefault(dateTime, ChronoField.MONTH_OF_YEAR, 0);
+ day = getOrDefault(dateTime, ChronoField.DAY_OF_MONTH, 0);
+ hour = getOrDefault(dateTime, ChronoField.HOUR_OF_DAY, 0);
+ minute = getOrDefault(dateTime, ChronoField.MINUTE_OF_HOUR, 0);
+ second = getOrDefault(dateTime, ChronoField.SECOND_OF_MINUTE, 0);
+ microsecond = getOrDefault(dateTime, ChronoField.MICRO_OF_SECOND,
0);
this.type = type;
} catch (Exception ex) {
- throw new AnalysisException("date literal [" + s + "] is invalid");
+ throw new AnalysisException("date literal [" + s + "] is invalid:
" + ex.getMessage());
}
}
@@ -369,8 +426,9 @@ public class DateLiteral extends LiteralExpr {
case DATEV2:
return
this.getStringValue().compareTo(MIN_DATE.getStringValue()) == 0;
case DATETIME:
- case DATETIMEV2:
return
this.getStringValue().compareTo(MIN_DATETIME.getStringValue()) == 0;
+ case DATETIMEV2:
+ return
this.getStringValue().compareTo(MIN_DATETIMEV2.getStringValue()) == 0;
default:
return false;
}
@@ -605,27 +663,10 @@ public class DateLiteral extends LiteralExpr {
}
public long unixTimestamp(TimeZone timeZone) {
- DateTime dt = new DateTime((int) year, (int) month, (int) day, (int)
hour, (int) minute, (int) second,
- DateTimeZone.forTimeZone(timeZone));
- return dt.getMillis();
- }
-
- public static DateLiteral dateParser(String date, String pattern) throws
AnalysisException {
- DateTimeFormatter formatter = formatBuilder(pattern).toFormatter();
- LocalDateTime dateTime = formatter.parseLocalDateTime(date);
- DateLiteral dateLiteral = new DateLiteral(
- dateTime.getYear(),
- dateTime.getMonthOfYear(),
- dateTime.getDayOfMonth(),
- dateTime.getHourOfDay(),
- dateTime.getMinuteOfHour(),
- dateTime.getSecondOfMinute());
- if (HAS_TIME_PART.matcher(pattern).matches()) {
- dateLiteral.setType(Type.DATETIME);
- } else {
- dateLiteral.setType(Type.DATE);
- }
- return dateLiteral;
+ ZonedDateTime zonedDateTime = ZonedDateTime.of((int) year, (int)
month, (int) day, (int) hour,
+ (int) minute, (int) second, (int) microsecond,
ZoneId.of(timeZone.getID()));
+ Timestamp timestamp = Timestamp.from(zonedDateTime.toInstant());
+ return timestamp.getTime();
}
public static boolean hasTimePart(String format) {
@@ -635,13 +676,16 @@ public class DateLiteral extends LiteralExpr {
//Return the date stored in the dateliteral as pattern format.
//eg : "%Y-%m-%d" or "%Y-%m-%d %H:%i:%s"
public String dateFormat(String pattern) throws AnalysisException {
- if (type.equals(Type.DATE)) {
- return DATE_FORMATTER.parseLocalDateTime(getStringValue())
- .toString(formatBuilder(pattern).toFormatter());
+ TemporalAccessor accessor;
+ if (type.equals(Type.DATE) || type.equals(Type.DATEV2)) {
+ accessor = DATE_FORMATTER.parse(getStringValue());
+ } else if (type.isDatetimeV2()) {
+ accessor =
DATE_TIME_FORMATTER_TO_MICRO_SECOND.parse(getStringValue());
} else {
- return DATE_TIME_FORMATTER.parseLocalDateTime(getStringValue())
- .toString(formatBuilder(pattern).toFormatter());
+ accessor = DATE_TIME_FORMATTER.parse(getStringValue());
}
+ DateTimeFormatter toFormatter = formatBuilder(pattern).toFormatter();
+ return toFormatter.format(accessor);
}
private static DateTimeFormatterBuilder formatBuilder(String pattern)
throws AnalysisException {
@@ -652,84 +696,77 @@ public class DateLiteral extends LiteralExpr {
if (escaped) {
switch (character) {
case 'a': // %a Abbreviated weekday name (Sun..Sat)
- builder.appendDayOfWeekShortText();
+ builder.appendText(ChronoField.DAY_OF_WEEK,
TextStyle.SHORT);
break;
case 'b': // %b Abbreviated month name (Jan..Dec)
- builder.appendMonthOfYearShortText();
+ builder.appendText(ChronoField.MONTH_OF_YEAR,
TextStyle.SHORT);
break;
case 'c': // %c Month, numeric (0..12)
- builder.appendMonthOfYear(1);
+ builder.appendValue(ChronoField.MONTH_OF_YEAR);
break;
case 'd': // %d Day of the month, numeric (00..31)
- builder.appendDayOfMonth(2);
+ builder.appendValue(ChronoField.DAY_OF_MONTH, 2);
break;
case 'e': // %e Day of the month, numeric (0..31)
- builder.appendDayOfMonth(1);
+ builder.appendValue(ChronoField.DAY_OF_MONTH);
break;
case 'H': // %H Hour (00..23)
- builder.appendHourOfDay(2);
+ builder.appendValue(ChronoField.HOUR_OF_DAY, 2);
break;
case 'h': // %h Hour (01..12)
case 'I': // %I Hour (01..12)
- builder.appendClockhourOfHalfday(2);
+ builder.appendValue(ChronoField.HOUR_OF_AMPM, 2);
break;
case 'i': // %i Minutes, numeric (00..59)
- builder.appendMinuteOfHour(2);
+ builder.appendValue(ChronoField.MINUTE_OF_HOUR, 2);
break;
case 'j': // %j Day of year (001..366)
- builder.appendDayOfYear(3);
+ builder.appendValue(ChronoField.DAY_OF_YEAR, 3);
break;
case 'k': // %k Hour (0..23)
- builder.appendHourOfDay(1);
+ builder.appendValue(ChronoField.HOUR_OF_DAY);
break;
case 'l': // %l Hour (1..12)
- builder.appendClockhourOfHalfday(1);
+ builder.appendValue(ChronoField.HOUR_OF_AMPM);
break;
case 'M': // %M Month name (January..December)
- builder.appendMonthOfYearText();
+ builder.appendText(ChronoField.MONTH_OF_YEAR,
TextStyle.FULL);
break;
case 'm': // %m Month, numeric (00..12)
- builder.appendMonthOfYear(2);
+ builder.appendValue(ChronoField.MONTH_OF_YEAR, 2);
break;
case 'p': // %p AM or PM
- builder.appendHalfdayOfDayText();
+ builder.appendText(ChronoField.AMPM_OF_DAY);
break;
case 'r': // %r Time, 12-hour (hh:mm:ss followed by AM or
PM)
- builder.appendClockhourOfHalfday(2)
- .appendLiteral(':')
- .appendMinuteOfHour(2)
- .appendLiteral(':')
- .appendSecondOfMinute(2)
- .appendLiteral(' ')
- .appendHalfdayOfDayText();
+ builder.appendValue(ChronoField.HOUR_OF_AMPM, 2)
+ .appendPattern(":mm:ss ")
+ .appendText(ChronoField.AMPM_OF_DAY,
TextStyle.FULL)
+ .toFormatter();
break;
case 'S': // %S Seconds (00..59)
case 's': // %s Seconds (00..59)
- builder.appendSecondOfMinute(2);
+ builder.appendValue(ChronoField.SECOND_OF_MINUTE, 2);
break;
- case 'T': // %T Time, 24-hour (hh:mm:ss)
- builder.appendHourOfDay(2)
- .appendLiteral(':')
- .appendMinuteOfHour(2)
- .appendLiteral(':')
- .appendSecondOfMinute(2);
+ case 'T': // %T Time, 24-hour (HH:mm:ss)
+ builder.appendPattern("HH:mm:ss");
break;
case 'v': // %v Week (01..53), where Monday is the first
day of the week; used with %x
- builder.appendWeekOfWeekyear(2);
+ builder.appendValue(ChronoField.ALIGNED_WEEK_OF_YEAR,
2);
break;
case 'x':
// %x Year for the week, where Monday is the first day
of the week,
// numeric, four digits; used with %v
- builder.appendWeekyear(4, 4);
+ builder.appendValue(ChronoField.YEAR, 4);
break;
case 'W': // %W Weekday name (Sunday..Saturday)
- builder.appendDayOfWeekText();
+ builder.appendText(ChronoField.DAY_OF_WEEK,
TextStyle.FULL);
break;
case 'Y': // %Y Year, numeric, four digits
- builder.appendYear(4, 4);
+ builder.appendPattern("uuuu");
break;
case 'y': // %y Year, numeric (two digits)
- builder.appendTwoDigitYear(2020);
+ builder.appendValueReduced(ChronoField.YEAR, 2, 2,
1970);
break;
// TODO(Gabriel): support microseconds in date literal
case 'f': // %f Microseconds (000000..999999)
@@ -759,14 +796,29 @@ public class DateLiteral extends LiteralExpr {
return builder;
}
- public LocalDateTime getTimeFormatter() throws AnalysisException {
+ private int getOrDefault(final TemporalAccessor accessor, final
ChronoField field,
+ final int defaultValue) {
+ return accessor.isSupported(field) ? accessor.get(field) :
defaultValue;
+ }
+
+ public LocalDateTime getTimeFormatter() {
+ TemporalAccessor accessor;
if (type.equals(Type.DATE) || type.equals(Type.DATEV2)) {
- return DATE_FORMATTER.parseLocalDateTime(getStringValue());
- } else if (type.equals(Type.DATETIME) || type.equals(Type.DATETIMEV2))
{
- return DATE_TIME_FORMATTER.parseLocalDateTime(getStringValue());
+ accessor = DATE_FORMATTER.parse(getStringValue());
+ } else if (type.isDatetimeV2()) {
+ accessor =
DATE_TIME_FORMATTER_TO_MICRO_SECOND.parse(getStringValue());
} else {
- throw new AnalysisException("Not support date literal type");
+ accessor = DATE_TIME_FORMATTER.parse(getStringValue());
}
+ final int year = accessor.get(ChronoField.YEAR);
+ final int month = accessor.get(ChronoField.MONTH_OF_YEAR);
+ final int dayOfMonth = accessor.get(ChronoField.DAY_OF_MONTH);
+ final int hour = getOrDefault(accessor, ChronoField.HOUR_OF_DAY, 0);
+ final int minute = getOrDefault(accessor, ChronoField.MINUTE_OF_HOUR,
0);
+ final int second = getOrDefault(accessor,
ChronoField.SECOND_OF_MINUTE, 0);
+ final int microSeconds = getOrDefault(accessor,
ChronoField.NANO_OF_SECOND, 0);
+
+ return LocalDateTime.of(year, month, dayOfMonth, hour, minute, second,
microSeconds);
}
public DateLiteral plusYears(int year) throws AnalysisException {
@@ -817,6 +869,10 @@ public class DateLiteral extends LiteralExpr {
return second;
}
+ public long getMicrosecond() {
+ return microsecond;
+ }
+
private long year;
private long month;
private long day;
@@ -843,6 +899,7 @@ public class DateLiteral extends LiteralExpr {
boolean datePartUsed = false;
boolean timePartUsed = false;
+ boolean microSecondPartUsed = false;
int dayPart = 0;
long weekday = -1;
@@ -966,8 +1023,11 @@ public class DateLiteral extends LiteralExpr {
break;
// Micro second
case 'f':
- // micro second is not supported, so just eat it and
go one.
tmp = vp + Math.min(6, vend - vp);
+ intValue = strToLong(value.substring(vp, tmp));
+ this.microsecond = (long) (intValue * Math.pow(10, 6 -
Math.min(6, vend - vp)));
+ timePartUsed = true;
+ microSecondPartUsed = true;
vp = tmp;
break;
// AM/PM
@@ -1162,10 +1222,12 @@ public class DateLiteral extends LiteralExpr {
// TODO(Gabriel): we still use old version datetime/date and change
this to new version when
// we think it's stable enough
if (datePartUsed) {
- if (timePartUsed) {
- this.type = Type.DATETIME;
+ if (microSecondPartUsed) {
+ this.type = Type.DATETIMEV2;
+ } else if (timePartUsed) {
+ this.type = getDefaultDateType(Type.DATETIME);
} else {
- this.type = Type.DATE;
+ this.type = getDefaultDateType(Type.DATE);
}
}
@@ -1421,9 +1483,9 @@ public class DateLiteral extends LiteralExpr {
microsecond = dateVal[6];
if (numField == 3) {
- type = Type.DATE;
+ type = getDefaultDateType(Type.DATE);
} else {
- type = Type.DATETIME;
+ type = getDefaultDateType(Type.DATETIME);
}
if (checkRange() || checkDate()) {
@@ -1431,18 +1493,6 @@ public class DateLiteral extends LiteralExpr {
}
}
- public void fromDateStr(String dateStr, Type type) throws
AnalysisException {
- switch (type.getPrimitiveType()) {
- case DATETIME:
- case DATE:
- fromDateStr(dateStr);
- break;
- default:
- fromDateStr(dateStr);
- convertTypeToV2();
- }
- }
-
public static Type getDefaultDateType(Type type) {
switch (type.getPrimitiveType()) {
case DATE:
diff --git a/fe/fe-core/src/main/java/org/apache/doris/rewrite/FEFunctions.java
b/fe/fe-core/src/main/java/org/apache/doris/rewrite/FEFunctions.java
index e1d2ca5b06..b7f5fe4bd0 100755
--- a/fe/fe-core/src/main/java/org/apache/doris/rewrite/FEFunctions.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/rewrite/FEFunctions.java
@@ -35,11 +35,10 @@ import org.apache.doris.qe.GlobalVariable;
import com.google.common.base.Preconditions;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
-import org.joda.time.DateTimeZone;
-import org.joda.time.LocalDateTime;
import java.math.BigDecimal;
import java.math.BigInteger;
+import java.time.LocalDateTime;
/**
* compute functions in FE.
@@ -380,7 +379,7 @@ public class FEFunctions {
@FEFunction(name = "now", argTypes = {}, returnType = "DATETIME")
public static DateLiteral now() throws AnalysisException {
- return new
DateLiteral(LocalDateTime.now(DateTimeZone.forTimeZone(TimeUtils.getTimeZone())),
+ return new
DateLiteral(LocalDateTime.now(TimeUtils.getTimeZone().toZoneId()),
DateLiteral.getDefaultDateType(Type.DATETIME));
}
@@ -390,8 +389,8 @@ public class FEFunctions {
}
@FEFunction(name = "curdate", argTypes = {}, returnType = "DATE")
- public static DateLiteral curDate() throws AnalysisException {
- return new
DateLiteral(LocalDateTime.now(DateTimeZone.forTimeZone(TimeUtils.getTimeZone())),
+ public static DateLiteral curDate() {
+ return new
DateLiteral(LocalDateTime.now(TimeUtils.getTimeZone().toZoneId()),
DateLiteral.getDefaultDateType(Type.DATE));
}
@@ -408,8 +407,8 @@ public class FEFunctions {
}
@FEFunction(name = "utc_timestamp", argTypes = {}, returnType = "DATETIME")
- public static DateLiteral utcTimestamp() throws AnalysisException {
- return new
DateLiteral(LocalDateTime.now(DateTimeZone.forTimeZone(TimeUtils.getOrSystemTimeZone("+00:00"))),
+ public static DateLiteral utcTimestamp() {
+ return new
DateLiteral(LocalDateTime.now(TimeUtils.getOrSystemTimeZone("+00:00").toZoneId()),
DateLiteral.getDefaultDateType(Type.DATETIME));
}
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/analysis/DateLiteralTest.java
b/fe/fe-core/src/test/java/org/apache/doris/analysis/DateLiteralTest.java
index a6f7bf41dd..60cee42cfe 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/analysis/DateLiteralTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/analysis/DateLiteralTest.java
@@ -23,10 +23,11 @@ import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.InvalidFormatException;
import org.apache.doris.common.jmockit.Deencapsulation;
-import org.joda.time.LocalDateTime;
import org.junit.Assert;
import org.junit.Test;
+import java.time.LocalDateTime;
+
public class DateLiteralTest {
@Test
@@ -108,6 +109,24 @@ public class DateLiteralTest {
Assert.assertTrue(literal.toSql().contains("2020-12-13 12:00:00"));
}
+ @Test
+ public void testParseDateTimeV2ToHourORMinute() throws Exception {
+ String s = "2020-12-13 12:13:14.123";
+ Type type = ScalarType.createDatetimeV2Type(6);
+ DateLiteral literal = new DateLiteral(s, type);
+ Assert.assertTrue(literal.toSql().contains("2020-12-13 12:13:14.123"));
+ s = "2020-12-13 12:13";
+ literal = new DateLiteral(s, type);
+ Assert.assertTrue(literal.toSql().contains("2020-12-13 12:13:00"));
+ s = "2020-12-13 12";
+ literal = new DateLiteral(s, type);
+ Assert.assertTrue(literal.toSql().contains("2020-12-13 12:00:00"));
+
+ String s2 = "2020-12-13 12:13:14.123456";
+ DateLiteral literal2 = new DateLiteral(s2, type);
+ Assert.assertTrue(literal2.toSql().contains("2020-12-13
12:13:14.123456"));
+ }
+
@Test
public void uncheckedCastTo() {
boolean hasException = false;
@@ -231,6 +250,30 @@ public class DateLiteralTest {
Assert.assertFalse(hasException);
}
+ @Test
+ public void testDateFormatForDatetimeV2() {
+ boolean hasException = false;
+ try {
+ DateLiteral literal = new DateLiteral("1997-10-7 00:00:00.123456",
Type.DATETIMEV2);
+ Assert.assertEquals(1997, literal.getYear());
+ Assert.assertEquals(123456, literal.getMicrosecond());
+
+ literal = new DateLiteral("2021-06-1 00:00:00.123456",
Type.DATETIMEV2);
+ Assert.assertEquals(2021, literal.getYear());
+ Assert.assertEquals(6, literal.getMonth());
+ Assert.assertEquals(1, literal.getDay());
+
+ literal = new DateLiteral("2022-6-01 00:00:00.123456",
Type.DATETIMEV2);
+ Assert.assertEquals(2022, literal.getYear());
+ Assert.assertEquals(6, literal.getMonth());
+ Assert.assertEquals(1, literal.getDay());
+ } catch (AnalysisException e) {
+ e.printStackTrace();
+ hasException = true;
+ }
+ Assert.assertFalse(hasException);
+ }
+
@Test
public void testParseDateTimeToHourORMinuteForDateV2() throws Exception {
String s = "2020-12-13 12:13:14";
@@ -308,7 +351,7 @@ public class DateLiteralTest {
}
@Test
- public void testCheckDateForDateV2() throws AnalysisException {
+ public void testCheckDateForDateV2() {
boolean hasException = false;
try {
DateLiteral dateLiteral = new DateLiteral();
@@ -347,7 +390,7 @@ public class DateLiteralTest {
}
@Test
- public void testDateTimeV2Decimal() throws AnalysisException {
+ public void testDateTimeV2Decimal() {
DateLiteral dateLiteral1 = new DateLiteral(LocalDateTime.now(),
DateLiteral.getDefaultDateType(ScalarType.createDatetimeV2Type(3)));
Assert.assertTrue((dateLiteral1.getDecimalNumber() >= 100 &&
dateLiteral1.getDecimalNumber() < 1000)
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/rewrite/FEFunctionsTest.java
b/fe/fe-core/src/test/java/org/apache/doris/rewrite/FEFunctionsTest.java
index 5060fa67f3..b9aa120d7d 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/rewrite/FEFunctionsTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/rewrite/FEFunctionsTest.java
@@ -29,7 +29,6 @@ import org.apache.doris.common.util.TimeUtils;
import mockit.Expectations;
import mockit.Mocked;
-import org.joda.time.DateTime;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Rule;
@@ -37,6 +36,9 @@ import org.junit.Test;
import org.junit.rules.ExpectedException;
import java.time.ZoneId;
+import java.time.ZonedDateTime;
+import java.time.format.DateTimeFormatter;
+import java.time.format.DateTimeFormatterBuilder;
import java.util.Locale;
import java.util.TimeZone;
@@ -740,9 +742,14 @@ public class FEFunctionsTest {
String curTimeString = FEFunctions.curTime().toSqlImpl().replace("'",
"");
String currentTimestampString =
FEFunctions.currentTimestamp().toSqlImpl().replace("'", "");
- String nowTimestampString = new DateTime().toString("yyyy-MM-dd
HH:mm:ss");
- Assert.assertTrue(nowTimestampString.compareTo(currentTimestampString)
>= 0);
- String nowTimeString =
nowTimestampString.substring(nowTimestampString.indexOf(" ") + 1);
+ ZonedDateTime zonedDateTime =
ZonedDateTime.now(TimeUtils.getTimeZone().toZoneId());
+ DateTimeFormatter formatter = new DateTimeFormatterBuilder()
+ .appendPattern("uuuu-MM-dd HH:mm:ss")
+ .toFormatter();
+
+
Assert.assertTrue(formatter.format(zonedDateTime).compareTo(currentTimestampString)
>= 0);
+ String nowTimeString = formatter.format(zonedDateTime).substring(
+ formatter.format(zonedDateTime).indexOf(" ") + 1);
Assert.assertTrue(nowTimeString.compareTo(curTimeString) >= 0);
}
diff --git a/fe/pom.xml b/fe/pom.xml
index 0a78a457d1..0c3be8cca1 100644
--- a/fe/pom.xml
+++ b/fe/pom.xml
@@ -185,7 +185,6 @@ under the License.
<jetty.version>6.1.14</jetty.version>
<jflex.version>1.4.3</jflex.version>
<jmockit.version>1.49</jmockit.version>
- <joda-time.version>2.10.1</joda-time.version>
<commons-io.version>2.6</commons-io.version>
<json-simple.version>1.1.1</json-simple.version>
<junit.version>5.8.2</junit.version>
@@ -431,12 +430,6 @@ under the License.
<version>${jmockit.version}</version>
<scope>test</scope>
</dependency>
- <!-- https://mvnrepository.com/artifact/joda-time/joda-time -->
- <dependency>
- <groupId>joda-time</groupId>
- <artifactId>joda-time</artifactId>
- <version>${joda-time.version}</version>
- </dependency>
<dependency>
<groupId>commons-io</groupId>
<artifactId>commons-io</artifactId>
diff --git a/fe/spark-dpp/pom.xml b/fe/spark-dpp/pom.xml
index 893f4ce353..1d7bff6aae 100644
--- a/fe/spark-dpp/pom.xml
+++ b/fe/spark-dpp/pom.xml
@@ -61,12 +61,6 @@ under the License.
<artifactId>jmockit</artifactId>
<scope>test</scope>
</dependency>
- <!-- https://mvnrepository.com/artifact/joda-time/joda-time -->
- <dependency>
- <groupId>joda-time</groupId>
- <artifactId>joda-time</artifactId>
- <scope>provided</scope>
- </dependency>
<!--
https://mvnrepository.com/artifact/org.junit.jupiter/junit-jupiter-engine -->
<dependency>
<groupId>org.junit.jupiter</groupId>
diff --git
a/fe/spark-dpp/src/main/java/org/apache/doris/load/loadv2/dpp/ColumnParser.java
b/fe/spark-dpp/src/main/java/org/apache/doris/load/loadv2/dpp/ColumnParser.java
index 2d15ab88a3..e72d471a9f 100644
---
a/fe/spark-dpp/src/main/java/org/apache/doris/load/loadv2/dpp/ColumnParser.java
+++
b/fe/spark-dpp/src/main/java/org/apache/doris/load/loadv2/dpp/ColumnParser.java
@@ -20,14 +20,14 @@ package org.apache.doris.load.loadv2.dpp;
import org.apache.doris.common.SparkDppException;
import org.apache.doris.load.loadv2.etl.EtlJobConfig;
-import org.joda.time.format.DateTimeFormat;
-import org.joda.time.format.DateTimeFormatter;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.Serializable;
import java.math.BigDecimal;
import java.math.BigInteger;
+import java.time.format.DateTimeFormatter;
+import java.time.format.DateTimeFormatterBuilder;
// Parser to validate value for different type
public abstract class ColumnParser implements Serializable {
@@ -35,8 +35,12 @@ public abstract class ColumnParser implements Serializable {
protected static final Logger LOG =
LoggerFactory.getLogger(ColumnParser.class);
// thread safe formatter
- public static final DateTimeFormatter DATE_FORMATTER =
DateTimeFormat.forPattern("yyyy-MM-dd");
- public static final DateTimeFormatter DATE_TIME_FORMATTER =
DateTimeFormat.forPattern("yyyy-MM-dd HH:mm:ss");
+ public static final DateTimeFormatter DATE_FORMATTER = new
DateTimeFormatterBuilder()
+ .appendPattern("uuuu-MM-dd")
+ .toFormatter();
+ public static final DateTimeFormatter DATE_TIME_FORMATTER = new
DateTimeFormatterBuilder()
+ .appendPattern("uuuu-MM-dd HH:mm:ss")
+ .toFormatter();
public static ColumnParser create(EtlJobConfig.EtlColumn etlColumn) throws
SparkDppException {
String columnType = etlColumn.columnType;
@@ -166,8 +170,8 @@ class DateParser extends ColumnParser {
@Override
public boolean parse(String value) {
try {
- DATE_FORMATTER.parseDateTime(value);
- } catch (IllegalArgumentException e) {
+ DATE_FORMATTER.parse(value);
+ } catch (Exception e) {
return false;
}
return true;
@@ -178,8 +182,8 @@ class DatetimeParser extends ColumnParser {
@Override
public boolean parse(String value) {
try {
- DATE_TIME_FORMATTER.parseDateTime(value);
- } catch (IllegalArgumentException e) {
+ DATE_TIME_FORMATTER.parse(value);
+ } catch (Exception e) {
return false;
}
return true;
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
