This is an automated email from the ASF dual-hosted git repository.
Gabriel39 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 2c697190ccd [fix](be) Correct Arrow timestamps before Unix epoch in
DATETIMEV2 deserialization (#65689)
2c697190ccd is described below
commit 2c697190ccd5861af8680d07debf94a3eea509ae
Author: zhangstar333 <[email protected]>
AuthorDate: Mon Jul 20 14:21:35 2026 +0800
[fix](be) Correct Arrow timestamps before Unix epoch in DATETIMEV2
deserialization (#65689)
### What problem does this PR solve?
Problem Summary:
brefore: auto utc_epoch = static_cast<**UInt64**>(date_value);
But Arrow timestamps are signed integers. For example, 1969-12-31
23:59:59.123456 UTC is represented as -876544 in timestamp[us]. After
conversion to UInt64, it becomes a very large positive value, causing an
invalid DATETIMEV2 result.
### Release note
None
### Check List (For Author)
- Test <!-- At least one of them must be included. -->
- [ ] Regression test
- [x] Unit Test
- [ ] Manual test (add detailed scripts or steps below)
- [ ] No need to test or manual test. Explain why:
- [ ] This is a refactor/code format and no logic has been changed.
- [ ] Previous test can cover this change.
- [ ] No code files have been changed.
- [ ] Other reason <!-- Add your reason? -->
- Behavior changed:
- [x] No.
- [ ] Yes. <!-- Explain the behavior change -->
- Does this need documentation?
- [x] No.
- [ ] Yes. <!-- Add document PR link here. eg:
https://github.com/apache/doris-website/pull/1214 -->
### Check List (For Reviewer who merge this PR)
- [ ] Confirm the release note
- [ ] Confirm test cases
- [ ] Confirm document
- [ ] Add branch pick label <!-- Add branch pick label that this PR
should merge into -->
---
.../data_type_serde/data_type_datetimev2_serde.cpp | 20 ++++++++++-----
.../data_type_serde_datetime_v2_test.cpp | 29 ++++++++++++++++++++++
.../spark_connector/spark_connector_arrow.out | 15 ++++++-----
3 files changed, 50 insertions(+), 14 deletions(-)
diff --git a/be/src/core/data_type_serde/data_type_datetimev2_serde.cpp
b/be/src/core/data_type_serde/data_type_datetimev2_serde.cpp
index 44402bb5b53..e91e596ab1c 100644
--- a/be/src/core/data_type_serde/data_type_datetimev2_serde.cpp
+++ b/be/src/core/data_type_serde/data_type_datetimev2_serde.cpp
@@ -525,16 +525,24 @@ Status
DataTypeDateTimeV2SerDe::read_column_from_arrow(IColumn& column,
for (auto value_i = start; value_i < end; ++value_i) {
const uint8_t* raw_byte_ptr = base_ptr + value_i * element_size;
auto date_value = unaligned_load<int64_t>(raw_byte_ptr);
- auto utc_epoch = static_cast<UInt64>(date_value);
DateV2Value<DateTimeV2ValueType> v;
- // convert second
- v.from_unixtime(utc_epoch / divisor, ctz);
- // get rest time
+ // C++ integer division truncates toward zero. Normalize the
remainder so a negative
+ // timestamp still has a non-negative fractional part, e.g.
-876544us becomes
+ // -1 second and 123456us.
+ int64_t seconds = date_value / divisor;
+ int64_t remainder = date_value % divisor;
+ if (remainder < 0) {
+ --seconds;
+ remainder += divisor;
+ }
+ v.from_unixtime(seconds, ctz);
+ // Get the fractional part.
// add 0 on the right to make it 6 digits. DateTimeV2Value
microsecond is 6 digits,
// the scale decides to keep the first few digits, so the valid
digits should be kept at the front.
- // "2022-01-01 11:11:11.111", utc_epoch = 1641035471111, divisor =
1000, set_microsecond(111000)
- v.set_microsecond((utc_epoch % divisor) * DIVISOR_FOR_MICRO /
divisor);
+ // "2022-01-01 11:11:11.111", timestamp = 1641035471111, divisor =
1000,
+ // set_microsecond(111000)
+ v.set_microsecond(remainder * DIVISOR_FOR_MICRO / divisor);
col_data.emplace_back(v);
}
} else {
diff --git a/be/test/core/data_type_serde/data_type_serde_datetime_v2_test.cpp
b/be/test/core/data_type_serde/data_type_serde_datetime_v2_test.cpp
index 3769e5f6946..dbe9c875834 100644
--- a/be/test/core/data_type_serde/data_type_serde_datetime_v2_test.cpp
+++ b/be/test/core/data_type_serde/data_type_serde_datetime_v2_test.cpp
@@ -366,4 +366,33 @@ TEST_F(DataTypeDateTimeV2SerDeTest,
ArrowMemNotAlignedDateTime) {
EXPECT_TRUE(st.ok());
}
+TEST_F(DataTypeDateTimeV2SerDeTest, ReadArrowTimestampBeforeEpoch) {
+ auto source_column = ColumnDateTimeV2::create();
+ DateV2Value<DateTimeV2ValueType> source_value;
+ source_value.unchecked_set_time(1969, 12, 31, 23, 59, 59, 123456);
+ source_column->insert_value(source_value);
+ std::string insert_value = "1969-12-31 23:59:59.123456";
+ ASSERT_EQ(insert_value, source_value.to_string(6));
+
+ arrow::TimestampBuilder builder(arrow::timestamp(arrow::TimeUnit::MICRO),
+ arrow::default_memory_pool());
+ ASSERT_TRUE(serde_datetime_v2_6
+ ->write_column_to_arrow(*source_column, nullptr,
&builder, 0,
+ source_column->size(),
cctz::utc_time_zone())
+ .ok());
+
+ std::shared_ptr<arrow::Array> array;
+ ASSERT_TRUE(builder.Finish(&array).ok());
+ const auto* timestamp_array = assert_cast<const
arrow::TimestampArray*>(array.get());
+ ASSERT_EQ(-876544, timestamp_array->Value(0));
+
+ auto dest_column = ColumnDateTimeV2::create();
+ ASSERT_TRUE(serde_datetime_v2_6
+ ->read_column_from_arrow(*dest_column, array.get(), 0,
array->length(),
+ cctz::utc_time_zone())
+ .ok());
+ ASSERT_EQ(1, dest_column->size());
+ EXPECT_EQ(insert_value, dest_column->get_element(0).to_string(6));
+}
+
} // namespace doris
diff --git
a/regression-test/data/connector_p0/spark_connector/spark_connector_arrow.out
b/regression-test/data/connector_p0/spark_connector/spark_connector_arrow.out
index 69d0c6629f0..0c7f078f812 100644
---
a/regression-test/data/connector_p0/spark_connector/spark_connector_arrow.out
+++
b/regression-test/data/connector_p0/spark_connector/spark_connector_arrow.out
@@ -18,13 +18,13 @@
1 [1, 0, 0, 1, 1] [1, 2, 3] [2, 12, 32] [3, 4, 5, 6] [4, 5,
6] [123456789, 987654321, 123789456] [6.6, 6.7, 7.8] [7.7,
8.800000000000001, 8.899999618530273] [3.12000, 1.12345]
["2023-09-08", "2027-10-28"] ["2023-09-08 17:12:34.123456", "2024-09-08
18:12:34.123456"] ["char", "char2"] ["varchar", "varchar2"] ["string",
"string2"]
-- !q03 --
-1 {1:1, 0:1} {1:2, 3:4} {2:4, 5:6} {3:4, 7:8} {4:5,
1:2} {123456789:987654321, 789456123:456789123} {6.6:8.8, 9.9:10.1}
{7.7:1.1, 2.2:3.3} {3.12000:1.23000, 2.34000:5.67000}
{"2023-09-08":"2024-09-08", "1023-09-08":"2023-09-08"} {"":"2023-09-08
17:12:34.123456", "3023-09-08 17:12:34.123456":"4023-09-08 17:12:34.123456"}
{"char":"char2", "char2":"char3"} {"varchar":"varchar2",
"varchar3":"varchar4"} {"string":"string2", "string3":"string4"}
-1 {1:1, 0:1} {1:2, 3:4} {2:4, 5:6} {3:4, 7:8} {4:5,
1:2} {123456789:987654321, 789456123:456789123} {6.6:8.8, 9.9:10.1}
{7.7:1.1, 2.2:3.3} {3.12000:1.23000, 2.34000:5.67000}
{"2023-09-08":"2024-09-08", "1023-09-08":"2023-09-08"} {"":"2023-09-08
17:12:34.123456", "3023-09-08 17:12:34.123456":"4023-09-08 17:12:34.123456"}
{"char":"char2", "char2":"char3"} {"varchar":"varchar2",
"varchar3":"varchar4"} {"string":"string2", "string3":"string4"}
-1 {1:1, 0:1} {1:2, 3:4} {2:4, 5:6} {3:4, 7:8} {4:5,
1:2} {123456789:987654321, 789456123:456789123} {6.6:8.8, 9.9:10.1}
{7.7:1.1, 2.2:3.3} {3.12000:1.23000, 2.34000:5.67000}
{"2023-09-08":"2024-09-08", "1023-09-08":"2023-09-08"} {"":"2023-09-08
17:12:34.123456", "3023-09-08 17:12:34.123456":"4023-09-08 17:12:34.123456"}
{"char":"char2", "char2":"char3"} {"varchar":"varchar2",
"varchar3":"varchar4"} {"string":"string2", "string3":"string4"}
-1 {1:1, 0:1} {1:2, 3:4} {2:4, 5:6} {3:4, 7:8} {4:5,
1:2} {123456789:987654321, 789456123:456789123} {6.6:8.8, 9.9:10.1}
{7.7:1.1, 2.2:3.3} {3.12000:1.23000, 2.34000:5.67000}
{"2023-09-08":"2024-09-08", "1023-09-08":"2023-09-08"} {"":"2023-09-08
17:12:34.123456", "3023-09-08 17:12:34.123456":"4023-09-08 17:12:34.123456"}
{"char":"char2", "char2":"char3"} {"varchar":"varchar2",
"varchar3":"varchar4"} {"string":"string2", "string3":"string4"}
-1 {1:1, 0:1} {1:2, 3:4} {2:4, 5:6} {3:4, 7:8} {4:5,
1:2} {123456789:987654321, 789456123:456789123} {6.6:8.8, 9.9:10.1}
{7.7:1.1, 2.2:3.3} {3.12000:1.23000, 2.34000:5.67000}
{"2023-09-08":"2024-09-08", "1023-09-08":"2023-09-08"} {"":"2023-09-08
17:12:34.123456", "3023-09-08 17:12:34.123456":"4023-09-08 17:12:34.123456"}
{"char":"char2", "char2":"char3"} {"varchar":"varchar2",
"varchar3":"varchar4"} {"string":"string2", "string3":"string4"}
-1 {1:1, 0:1} {1:2, 3:4} {2:4, 5:6} {3:4, 7:8} {4:5,
1:2} {123456789:987654321, 789456123:456789123} {6.6:8.8, 9.9:10.1}
{7.7:1.1, 2.2:3.3} {3.12000:1.23000, 2.34000:5.67000}
{"2023-09-08":"2024-09-08", "1023-09-08":"2023-09-08"} {"":"2023-09-08
17:12:34.123456", "3023-09-08 17:12:34.123456":"4023-09-08 17:12:34.123456"}
{"char":"char2", "char2":"char3"} {"varchar":"varchar2",
"varchar3":"varchar4"} {"string":"string2", "string3":"string4"}
-1 {1:1, 0:1} {1:2, 3:4} {2:4, 5:6} {3:4, 7:8} {4:5,
1:2} {123456789:987654321, 789456123:456789123} {6.6:8.8, 9.9:10.1}
{7.7:1.1, 2.2:3.3} {3.12000:1.23000, 2.34000:5.67000}
{"2023-09-08":"2024-09-08", "1023-09-08":"2023-09-08"} {"":"2023-09-08
17:12:34.123456", "3023-09-08 17:12:34.123456":"4023-09-08 17:12:34.123456"}
{"char":"char2", "char2":"char3"} {"varchar":"varchar2",
"varchar3":"varchar4"} {"string":"string2", "string3":"string4"}
+1 {1:1, 0:1} {1:2, 3:4} {2:4, 5:6} {3:4, 7:8} {4:5,
1:2} {123456789:987654321, 789456123:456789123} {6.6:8.8, 9.9:10.1}
{7.7:1.1, 2.2:3.3} {3.12000:1.23000, 2.34000:5.67000}
{"2023-09-08":"2024-09-08", "1023-09-08":"2023-09-08"} {"1023-09-08
17:06:51.123456":"2023-09-08 17:12:34.123456", "3023-09-08
17:12:34.123456":"4023-09-08 17:12:34.123456"} {"char":"char2",
"char2":"char3"} {"varchar":"varchar2", "varchar3":"varchar4"}
{"string":"string2", "string3":"string4"}
+1 {1:1, 0:1} {1:2, 3:4} {2:4, 5:6} {3:4, 7:8} {4:5,
1:2} {123456789:987654321, 789456123:456789123} {6.6:8.8, 9.9:10.1}
{7.7:1.1, 2.2:3.3} {3.12000:1.23000, 2.34000:5.67000}
{"2023-09-08":"2024-09-08", "1023-09-08":"2023-09-08"} {"1023-09-08
17:06:51.123456":"2023-09-08 17:12:34.123456", "3023-09-08
17:12:34.123456":"4023-09-08 17:12:34.123456"} {"char":"char2",
"char2":"char3"} {"varchar":"varchar2", "varchar3":"varchar4"}
{"string":"string2", "string3":"string4"}
+1 {1:1, 0:1} {1:2, 3:4} {2:4, 5:6} {3:4, 7:8} {4:5,
1:2} {123456789:987654321, 789456123:456789123} {6.6:8.8, 9.9:10.1}
{7.7:1.1, 2.2:3.3} {3.12000:1.23000, 2.34000:5.67000}
{"2023-09-08":"2024-09-08", "1023-09-08":"2023-09-08"} {"1023-09-08
17:06:51.123456":"2023-09-08 17:12:34.123456", "3023-09-08
17:12:34.123456":"4023-09-08 17:12:34.123456"} {"char":"char2",
"char2":"char3"} {"varchar":"varchar2", "varchar3":"varchar4"}
{"string":"string2", "string3":"string4"}
+1 {1:1, 0:1} {1:2, 3:4} {2:4, 5:6} {3:4, 7:8} {4:5,
1:2} {123456789:987654321, 789456123:456789123} {6.6:8.8, 9.9:10.1}
{7.7:1.1, 2.2:3.3} {3.12000:1.23000, 2.34000:5.67000}
{"2023-09-08":"2024-09-08", "1023-09-08":"2023-09-08"} {"1023-09-08
17:06:51.123456":"2023-09-08 17:12:34.123456", "3023-09-08
17:12:34.123456":"4023-09-08 17:12:34.123456"} {"char":"char2",
"char2":"char3"} {"varchar":"varchar2", "varchar3":"varchar4"}
{"string":"string2", "string3":"string4"}
+1 {1:1, 0:1} {1:2, 3:4} {2:4, 5:6} {3:4, 7:8} {4:5,
1:2} {123456789:987654321, 789456123:456789123} {6.6:8.8, 9.9:10.1}
{7.7:1.1, 2.2:3.3} {3.12000:1.23000, 2.34000:5.67000}
{"2023-09-08":"2024-09-08", "1023-09-08":"2023-09-08"} {"1023-09-08
17:06:51.123456":"2023-09-08 17:12:34.123456", "3023-09-08
17:12:34.123456":"4023-09-08 17:12:34.123456"} {"char":"char2",
"char2":"char3"} {"varchar":"varchar2", "varchar3":"varchar4"}
{"string":"string2", "string3":"string4"}
+1 {1:1, 0:1} {1:2, 3:4} {2:4, 5:6} {3:4, 7:8} {4:5,
1:2} {123456789:987654321, 789456123:456789123} {6.6:8.8, 9.9:10.1}
{7.7:1.1, 2.2:3.3} {3.12000:1.23000, 2.34000:5.67000}
{"2023-09-08":"2024-09-08", "1023-09-08":"2023-09-08"} {"1023-09-08
17:06:51.123456":"2023-09-08 17:12:34.123456", "3023-09-08
17:12:34.123456":"4023-09-08 17:12:34.123456"} {"char":"char2",
"char2":"char3"} {"varchar":"varchar2", "varchar3":"varchar4"}
{"string":"string2", "string3":"string4"}
+1 {1:1, 0:1} {1:2, 3:4} {2:4, 5:6} {3:4, 7:8} {4:5,
1:2} {123456789:987654321, 789456123:456789123} {6.6:8.8, 9.9:10.1}
{7.7:1.1, 2.2:3.3} {3.12000:1.23000, 2.34000:5.67000}
{"2023-09-08":"2024-09-08", "1023-09-08":"2023-09-08"} {"1023-09-08
17:06:51.123456":"2023-09-08 17:12:34.123456", "3023-09-08
17:12:34.123456":"4023-09-08 17:12:34.123456"} {"char":"char2",
"char2":"char3"} {"varchar":"varchar2", "varchar3":"varchar4"}
{"string":"string2", "string3":"string4"}
-- !q04 --
1 {"c_bool":1, "c_tinyint":1, "c_smallint":2, "c_int":3, "c_bigint":4,
"c_largeint":123456789, "c_float":6.6, "c_double":7.7, "c_decimal":3.12000,
"c_date":"2023-09-08", "c_datetime":"2023-09-08 17:12:34.123456",
"c_char":"char", "c_varchar":"varchar", "c_string":"string"}
@@ -34,4 +34,3 @@
1 {"c_bool":1, "c_tinyint":1, "c_smallint":2, "c_int":3, "c_bigint":4,
"c_largeint":123456789, "c_float":6.6, "c_double":7.7, "c_decimal":3.12000,
"c_date":"2023-09-08", "c_datetime":"2023-09-08 17:12:34.123456",
"c_char":"char", "c_varchar":"varchar", "c_string":"string"}
1 {"c_bool":1, "c_tinyint":1, "c_smallint":2, "c_int":3, "c_bigint":4,
"c_largeint":123456789, "c_float":6.6, "c_double":7.7, "c_decimal":3.12000,
"c_date":"2023-09-08", "c_datetime":"2023-09-08 17:12:34.123456",
"c_char":"char", "c_varchar":"varchar", "c_string":"string"}
1 {"c_bool":1, "c_tinyint":1, "c_smallint":2, "c_int":3, "c_bigint":4,
"c_largeint":123456789, "c_float":6.6, "c_double":7.7, "c_decimal":3.12000,
"c_date":"2023-09-08", "c_datetime":"2023-09-08 17:12:34.123456",
"c_char":"char", "c_varchar":"varchar", "c_string":"string"}
-
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
