This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-doris.git
The following commit(s) were added to refs/heads/master by this push:
new 065b979 [Bug] behavior of function str_to_date() and date_format() on
BE and FE is inconsistent (#4612)
065b979 is described below
commit 065b979f3503cda3ad4b110b7d44f482a1380815
Author: qiye <[email protected]>
AuthorDate: Thu Sep 17 10:10:19 2020 +0800
[Bug] behavior of function str_to_date() and date_format() on BE and FE is
inconsistent (#4612)
1. add date range check in `DateLiteral` for `FEFunctions`
2. `select str_to_date(202009,'%Y%m')` and `select str_to_date(str,'%Y%m')
from tb where tb.str = '202009'` will return same output `2020-09-00`.
3. add support of zero-date to function `str_to_date()`,`date_format()`
4. fix FE can calculate negative value bug, eg: `select
str_to_date('-2020', '%Y')` will return `NULL` instead of date value.
current behavior is same as MySQL **without** sql_mode `NO_ZERO_IN_DATE`
and `NO_ZERO_DATE`.
**current behavior**
```
mysql> select siteid,str_to_date(siteid,'%Y%m%d') from table2 order by
siteid;
+------------+---------------------------------+
| siteid | str_to_date(`siteid`, '%Y%m%d') |
+------------+---------------------------------+
| 1 | 2001-00-00 |
| 2 | 2002-00-00 |
| 2 | 2002-00-00 |
| 3 | 2003-00-00 |
| 4 | 2004-00-00 |
| 5 | 2005-00-00 |
| 20 | 2020-00-00 |
| 202 | 0202-00-00 |
| 2020 | 2020-00-00 |
| 20209 | 2020-09-00 |
| 202008 | 2020-08-00 |
| 202009 | 2020-09-00 |
| 2020009 | 2020-00-09 |
| 20200009 | 2020-00-09 |
| 20201309 | NULL |
| 2020090909 | 2020-09-09 |
+------------+---------------------------------+
mysql> select
str_to_date('2','%Y%m%d'),str_to_date('20','%Y%m%d'),str_to_date('202','%Y%m%d'),str_to_date('2020','%Y%m%d'),str_to_date('20209','%Y%m%d'),str_to_date('202009','%Y%m%d'),str_to_date('2020099','%Y%m%d'),str_to_date('20200909','%Y%m%d'),str_to_date('2020090909','%Y%m%d'),str_to_date('2020009','%Y%m%d'),str_to_date('20200009','%Y%m%d'),str_to_date('20201309','%Y%m%d');
+----------------------------+-----------------------------+------------------------------+-------------------------------+--------------------------------+---------------------------------+----------------------------------+-----------------------------------+-------------------------------------+----------------------------------+-----------------------------------+-----------------------------------+
| str_to_date('2', '%Y%m%d') | str_to_date('20', '%Y%m%d') |
str_to_date('202', '%Y%m%d') | str_to_date('2020', '%Y%m%d') |
str_to_date('20209', '%Y%m%d') | str_to_date('202009', '%Y%m%d') |
str_to_date('2020099', '%Y%m%d') | str_to_date('20200909', '%Y%m%d') |
str_to_date('2020090909', '%Y%m%d') | str_to_date('2020009', '%Y%m%d') |
str_to_date('20200009', '%Y%m%d') | str_to_date('20201309', '%Y%m%d') |
+----------------------------+-----------------------------+------------------------------+-------------------------------+--------------------------------+---------------------------------+----------------------------------+-----------------------------------+-------------------------------------+----------------------------------+-----------------------------------+-----------------------------------+
| 2002-00-00 | 2020-00-00 | 0202-00-00
| 2020-00-00 | 2020-09-00
| 2020-09-00 | 2020-09-09 |
2020-09-09 | 2020-09-09 |
2020-00-09 | 2020-00-09 | NULL
|
+----------------------------+-----------------------------+------------------------------+-------------------------------+--------------------------------+---------------------------------+----------------------------------+-----------------------------------+-------------------------------------+----------------------------------+-----------------------------------+-----------------------------------+
```
---
be/src/runtime/datetime_value.cpp | 5 +--
be/test/runtime/datetime_value_test.cpp | 10 ++++--
.../org/apache/doris/analysis/DateLiteral.java | 26 +++++++++++++-
.../org/apache/doris/analysis/DateLiteralTest.java | 41 ++++++++++++++++++++++
4 files changed, 74 insertions(+), 8 deletions(-)
diff --git a/be/src/runtime/datetime_value.cpp
b/be/src/runtime/datetime_value.cpp
index 67be081..29e4761 100644
--- a/be/src/runtime/datetime_value.cpp
+++ b/be/src/runtime/datetime_value.cpp
@@ -74,10 +74,7 @@ bool DateTimeValue::check_range() const {
}
bool DateTimeValue::check_date() const {
- if (_month == 0 || _day == 0) {
- return true;
- }
- if (_day > s_days_in_month[_month]) {
+ if (_month != 0 && _day > s_days_in_month[_month]) {
// Feb 29 in leap year is valid.
if (_month == 2 && _day == 29 && is_leap(_year)) {
return false;
diff --git a/be/test/runtime/datetime_value_test.cpp
b/be/test/runtime/datetime_value_test.cpp
index 4d49cc9..e69535e 100644
--- a/be/test/runtime/datetime_value_test.cpp
+++ b/be/test/runtime/datetime_value_test.cpp
@@ -236,11 +236,11 @@ TEST_F(DateTimeValueTest, check_date) {
ASSERT_TRUE(value.from_date_int64(19880201));
value._month = 0;
- ASSERT_TRUE(value.check_date());
+ ASSERT_FALSE(value.check_date());
value._month = 2;
value._day = 0;
- ASSERT_TRUE(value.check_date());
+ ASSERT_FALSE(value.check_date());
value._year = 1987;
value._day = 29;
ASSERT_TRUE(value.check_date());
@@ -1185,6 +1185,7 @@ TEST_F(DateTimeValueTest, from_int_value) {
// Construct from int value invalid
TEST_F(DateTimeValueTest, from_int_value_invalid) {
DateTimeValue value;
+ char str[MAX_DTVALUE_STR_LEN];
// minus value
ASSERT_FALSE(value.from_date_int64(-1231));
// [0, 101)
@@ -1198,7 +1199,10 @@ TEST_F(DateTimeValueTest, from_int_value_invalid) {
// 100-12-31
ASSERT_FALSE(value.from_date_int64(1232));
// 99 00:00:00
- ASSERT_FALSE(value.from_date_int64(99000000));
+ ASSERT_TRUE(value.from_date_int64(99000000));
+ value.to_string(str);
+ ASSERT_STREQ("9900-00-00", str);
+
// 9999-99-99 99:99:99 + 1
ASSERT_FALSE(value.from_date_int64(99999999999999L + 1));
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/analysis/DateLiteral.java
b/fe/fe-core/src/main/java/org/apache/doris/analysis/DateLiteral.java
index 257e6fd..a9cc822 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/DateLiteral.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/DateLiteral.java
@@ -62,6 +62,7 @@ public class DateLiteral extends LiteralExpr {
public static final DateLiteral UNIX_EPOCH_TIME = new DateLiteral(1970,
01, 01, 00, 00, 00);
private static final int DATEKEY_LENGTH = 8;
+ private static final int MAX_MICROSECOND = 999999;
private static DateTimeFormatter DATE_TIME_FORMATTER = null;
private static DateTimeFormatter DATE_FORMATTER = null;
@@ -1035,12 +1036,35 @@ public class DateLiteral extends LiteralExpr {
this.type = Type.DATE;
}
}
+
+ if (checkRange() || checkDate()) {
+ throw new InvalidFormatException("Invalid format");
+ }
return 0;
}
+ private boolean checkRange() {
+ return year > MAX_DATETIME.year || month > MAX_DATETIME.month || day >
MAX_DATETIME.day
+ || hour > MAX_DATETIME.hour || minute > MAX_DATETIME.minute ||
second > MAX_DATETIME.second
+ || microsecond > MAX_MICROSECOND;
+ }
+ private boolean checkDate() {
+ if (month != 0 && day > DAYS_IN_MONTH.get((int)month)){
+ if (month == 2 && day == 29 && Year.isLeap(year)) {
+ return false;
+ }
+ return true;
+ }
+ return false;
+ }
+
private long strToLong(String l) throws InvalidFormatException {
try {
- return Long.valueOf(l);
+ long y = Long.valueOf(l);
+ if (y < 0) {
+ throw new InvalidFormatException("Invalid format: negative
number.");
+ }
+ return y;
} catch (NumberFormatException e) {
throw new InvalidFormatException(e.getMessage());
}
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/analysis/DateLiteralTest.java
b/fe/fe-core/src/test/java/org/apache/doris/analysis/DateLiteralTest.java
index 02b165b..9b94f9d 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/analysis/DateLiteralTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/analysis/DateLiteralTest.java
@@ -19,6 +19,8 @@ package org.apache.doris.analysis;
import org.apache.doris.catalog.Type;
import org.apache.doris.common.AnalysisException;
+import org.apache.doris.common.InvalidFormatException;
+import org.apache.doris.common.jmockit.Deencapsulation;
import org.junit.Assert;
import org.junit.Test;
@@ -74,4 +76,43 @@ public class DateLiteralTest {
}
Assert.assertFalse(hasException);
}
+
+ @Test
+ public void testCheckDate() {
+ boolean hasException = false;
+ try {
+ DateLiteral dateLiteral = new DateLiteral();
+ dateLiteral.fromDateFormatStr("%Y%m%d","19971007", false);
+ Assert.assertFalse(Deencapsulation.invoke(dateLiteral,
"checkDate"));
+
+ dateLiteral.fromDateFormatStr("%Y%m%d","19970007", false);
+ Assert.assertFalse(Deencapsulation.invoke(dateLiteral,
"checkDate"));
+
+ dateLiteral.fromDateFormatStr("%Y%m%d","19971000", false);
+ Assert.assertFalse(Deencapsulation.invoke(dateLiteral,
"checkDate"));
+
+ dateLiteral.fromDateFormatStr("%Y%m%d","20000229", false);
+ Assert.assertFalse(Deencapsulation.invoke(dateLiteral,
"checkDate"));
+
+ } catch (InvalidFormatException e) {
+ e.printStackTrace();
+ hasException = true;
+ }
+ Assert.assertFalse(hasException);
+ }
+
+ @Test
+ public void testCheckRange() {
+ boolean hasException = false;
+ try {
+ DateLiteral dateLiteral = new DateLiteral();
+
dateLiteral.fromDateFormatStr("%Y%m%d%H%i%s%f","20201209123456123456", false);
+ Assert.assertFalse(Deencapsulation.invoke(dateLiteral,
"checkRange"));
+
+ } catch (InvalidFormatException e) {
+ e.printStackTrace();
+ hasException = true;
+ }
+ Assert.assertFalse(hasException);
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
