This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 9d11a81b3f0 [Fix](timezone) fix miss of expected rounding of Date type
with timezone #33553
9d11a81b3f0 is described below
commit 9d11a81b3f0f7716a7ffaa0ebb5ba886d7f4effc
Author: zclllyybb <[email protected]>
AuthorDate: Fri Apr 12 23:22:44 2024 +0800
[Fix](timezone) fix miss of expected rounding of Date type with timezone
#33553
---
be/src/http/action/stream_load.cpp | 3 +++
be/src/http/http_common.h | 1 +
be/src/vec/runtime/vdatetime_value.cpp | 20 ++++++++++++++++++
be/src/vec/runtime/vdatetime_value.h | 13 ++++++++++++
.../data/datatype_p0/datetimev2/only_date.csv | 2 ++
.../datatype_p0/datetimev2/test_tz_streamload.out | 4 ++++
.../datetimev2/test_tz_streamload.groovy | 24 ++++++++++++++++++++++
7 files changed, 67 insertions(+)
diff --git a/be/src/http/action/stream_load.cpp
b/be/src/http/action/stream_load.cpp
index 62c1f3ddf0e..e5b426ade59 100644
--- a/be/src/http/action/stream_load.cpp
+++ b/be/src/http/action/stream_load.cpp
@@ -475,8 +475,11 @@ Status StreamLoadAction::_process_put(HttpRequest*
http_req,
return Status::InvalidArgument("Invalid strict mode format. Must
be bool type");
}
}
+ // timezone first. if not, try time_zone
if (!http_req->header(HTTP_TIMEZONE).empty()) {
request.__set_timezone(http_req->header(HTTP_TIMEZONE));
+ } else if (!http_req->header(HTTP_TIME_ZONE).empty()) {
+ request.__set_timezone(http_req->header(HTTP_TIME_ZONE));
}
if (!http_req->header(HTTP_EXEC_MEM_LIMIT).empty()) {
try {
diff --git a/be/src/http/http_common.h b/be/src/http/http_common.h
index 57bbd0642e6..517ee87c685 100644
--- a/be/src/http/http_common.h
+++ b/be/src/http/http_common.h
@@ -38,6 +38,7 @@ static const std::string HTTP_TEMP_PARTITIONS =
"temporary_partitions";
static const std::string HTTP_NEGATIVE = "negative";
static const std::string HTTP_STRICT_MODE = "strict_mode";
static const std::string HTTP_TIMEZONE = "timezone";
+static const std::string HTTP_TIME_ZONE = "time_zone";
static const std::string HTTP_EXEC_MEM_LIMIT = "exec_mem_limit";
static const std::string HTTP_JSONPATHS = "jsonpaths";
static const std::string HTTP_JSONROOT = "json_root";
diff --git a/be/src/vec/runtime/vdatetime_value.cpp
b/be/src/vec/runtime/vdatetime_value.cpp
index ee384286455..6b769c68f9d 100644
--- a/be/src/vec/runtime/vdatetime_value.cpp
+++ b/be/src/vec/runtime/vdatetime_value.cpp
@@ -2192,6 +2192,26 @@ bool DateV2Value<T>::from_date_str_base(const char*
date_str, int len, int scale
return false;
}
+ // In check_range_and_set_time, for Date type the time part will be
truncated. So if the timezone offset should make
+ // rounding to date part, it would be lost. To avoid this, we use a
Datetime type to do these calc. It will save the
+ // time part and apply the offset. Then convert to Date type back.
+ // see https://github.com/apache/doris/pull/33553 for more details.
+ if constexpr (!is_datetime) {
+ if (sec_offset) {
+ DateV2Value<DateTimeV2ValueType> tmp;
+ if (!tmp.check_range_and_set_time(date_val[0], date_val[1],
date_val[2], date_val[3],
+ date_val[4], date_val[5],
date_val[6])) {
+ return false;
+ }
+ if (!tmp.date_add_interval<TimeUnit::SECOND>(
+ TimeInterval {TimeUnit::SECOND, sec_offset, false})) {
+ return false;
+ }
+ this->assign_from(tmp);
+ return true;
+ }
+ }
+
if (!check_range_and_set_time(date_val[0], date_val[1], date_val[2],
date_val[3], date_val[4],
date_val[5], date_val[6])) {
return false;
diff --git a/be/src/vec/runtime/vdatetime_value.h
b/be/src/vec/runtime/vdatetime_value.h
index 54dc368f620..2031c782267 100644
--- a/be/src/vec/runtime/vdatetime_value.h
+++ b/be/src/vec/runtime/vdatetime_value.h
@@ -824,6 +824,19 @@ public:
return from_date_format_str(format, format_len, value, value_len,
nullptr);
}
+ template <typename U>
+ void assign_from(DateV2Value<U> src) {
+ date_v2_value_.year_ = src.year();
+ date_v2_value_.month_ = src.month();
+ date_v2_value_.day_ = src.day();
+ if constexpr (is_datetime && std::is_same_v<U, DateTimeV2ValueType>) {
+ date_v2_value_.hour_ = src.hour();
+ date_v2_value_.minute_ = src.minute();
+ date_v2_value_.second_ = src.second();
+ date_v2_value_.microsecond_ = src.microsecond();
+ }
+ }
+
// Construct Date/Datetime type value from string.
// At least the following formats are recognised (based on number of
digits)
// 'YYMMDD', 'YYYYMMDD', 'YYMMDDHHMMSS', 'YYYYMMDDHHMMSS'
diff --git a/regression-test/data/datatype_p0/datetimev2/only_date.csv
b/regression-test/data/datatype_p0/datetimev2/only_date.csv
new file mode 100644
index 00000000000..63b49fef9d9
--- /dev/null
+++ b/regression-test/data/datatype_p0/datetimev2/only_date.csv
@@ -0,0 +1,2 @@
+1,2024-04-11T16:00:13+08:00,2024-04-11T16:00:13+08:00
+1,2024-04-11T06:00:13+08:00,2024-04-11T06:00:13+08:00
diff --git a/regression-test/data/datatype_p0/datetimev2/test_tz_streamload.out
b/regression-test/data/datatype_p0/datetimev2/test_tz_streamload.out
index c7fd50e3bb4..ab103c3a306 100644
--- a/regression-test/data/datatype_p0/datetimev2/test_tz_streamload.out
+++ b/regression-test/data/datatype_p0/datetimev2/test_tz_streamload.out
@@ -19,3 +19,7 @@
7 2023-08-17T17:41:18
8 2023-08-17T19:41:18
+-- !table1 --
+1 2024-04-10 2024-04-10T22:00:13
+1 2024-04-11 2024-04-11T08:00:13
+
diff --git
a/regression-test/suites/datatype_p0/datetimev2/test_tz_streamload.groovy
b/regression-test/suites/datatype_p0/datetimev2/test_tz_streamload.groovy
index 6e32facc83e..9ccd48477e1 100644
--- a/regression-test/suites/datatype_p0/datetimev2/test_tz_streamload.groovy
+++ b/regression-test/suites/datatype_p0/datetimev2/test_tz_streamload.groovy
@@ -68,4 +68,28 @@ suite("test_tz_streamload") {
}
sql "sync"
qt_table2 "select * from ${table2} order by id"
+
+ // test rounding for date type. from hour to date.
+ sql "drop table if exists d"
+ sql """
+ CREATE TABLE d (
+ `k1` int,
+ `k2` date,
+ `k3` datetime
+ )
+ DISTRIBUTED BY HASH(k1) BUCKETS 3
+ PROPERTIES (
+ "replication_num" = "1"
+ );
+ """
+
+ streamLoad {
+ table "d"
+ set 'column_separator', ','
+ set 'timezone', 'UTC'
+ file "only_date.csv"
+ time 20000
+ }
+ sql "sync"
+ qt_table1 "select * from d order by k1, k2, k3"
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
