This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-4.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-4.1 by this push:
new 5fc0d8e8b4f [fix](orc) decode timestamp through serde (#64807)
5fc0d8e8b4f is described below
commit 5fc0d8e8b4f03bc9313143a5265cf33fc31223be
Author: Chenjunwei <[email protected]>
AuthorDate: Mon Jul 13 15:23:10 2026 +0800
[fix](orc) decode timestamp through serde (#64807)
## Proposed changes
Fix ORC timestamp decoding to round nanoseconds to Doris microseconds
instead of truncating them. This keeps `CAST(timestamp AS VARCHAR)`
aligned with Hive/Trino prefix expectations for values like `2020-01-02
03:04:05.321`.
The same decode path is used by nested timestamps in array/map/struct
columns, so this also covers complex type projections.
## Problem summary
ORC stores timestamp fractional seconds as nanoseconds, while Doris
`DATETIMEV2(6)` keeps microseconds. The previous conversion truncated
nanos with `/ 1000`, so an ORC value such as `320999999ns` became
`.320999` instead of `.321000`. Prefix predicates like:
```sql
CAST(ts AS VARCHAR) LIKE '2020-01-02 03:04:05.321%'
```
could therefore miss rows created by Hive/Trino ORC writers.
## Solution
- Round ORC nanoseconds to microseconds during timestamp decode.
- Carry `999999500ns` and above into the next second.
- Apply the same helper to `TIMESTAMP` and `TIMESTAMP_INSTANT` decode
paths.
- Add a BE unit test covering rounding and second carry.
## Test plan
- `ninja -j 8 doris_be_test`
- `./be/ut_build_RELEASE/test/doris_be_test
--gtest_filter='OrcReaderFillDataTest.TestTimestampNanosecondsRoundToMicroseconds'`
- `./be/ut_build_RELEASE/test/doris_be_test
--gtest_filter='OrcReaderFillDataTest.*'`
- `git diff --check`
---
be/src/format/orc/vorc_reader.cpp | 12 +-
be/test/format/orc/orc_reader_fill_data_test.cpp | 39 ++++++
.../hive/test_hive_orc_timestamp_timezone.groovy | 132 +++++++++++++++++++++
3 files changed, 180 insertions(+), 3 deletions(-)
diff --git a/be/src/format/orc/vorc_reader.cpp
b/be/src/format/orc/vorc_reader.cpp
index a1b8a25182e..44e4ac7f627 100644
--- a/be/src/format/orc/vorc_reader.cpp
+++ b/be/src/format/orc/vorc_reader.cpp
@@ -172,6 +172,10 @@ Status build_iceberg_rowid_column(const DataTypePtr& type,
const std::string& fi
static constexpr uint32_t MAX_DICT_CODE_PREDICATE_TO_REWRITE =
std::numeric_limits<uint32_t>::max();
static constexpr char
EMPTY_STRING_FOR_OVERFLOW[ColumnString::MAX_STRINGS_OVERFLOW_SIZE] = "";
+static std::string normalized_orc_timezone_name(const std::string& ctz) {
+ return ctz.empty() ? "UTC" : (ctz == "CST" ? "Asia/Shanghai" : ctz);
+}
+
static void fill_orc_null_map(ColumnNullable* nullable_column, const
orc::ColumnVectorBatch* cvb,
size_t num_values) {
NullMap& map_data_column = nullable_column->get_null_map_data();
@@ -286,7 +290,7 @@ OrcReader::OrcReader(RuntimeProfile* profile, RuntimeState*
state,
_enable_filter_by_min_max(
state == nullptr ? true :
state->query_options().enable_orc_filter_by_min_max),
_dict_cols_has_converted(false) {
- TimezoneUtils::find_cctz_time_zone(ctz, _time_zone);
+ TimezoneUtils::find_cctz_time_zone(normalized_orc_timezone_name(ctz),
_time_zone);
_meta_cache = meta_cache;
_init_profile();
_init_system_properties();
@@ -312,7 +316,7 @@ OrcReader::OrcReader(RuntimeProfile* profile, RuntimeState*
state,
_enable_filter_by_min_max(
state == nullptr ? true :
state->query_options().enable_orc_filter_by_min_max),
_dict_cols_has_converted(false) {
- TimezoneUtils::find_cctz_time_zone(ctz, _time_zone);
+ TimezoneUtils::find_cctz_time_zone(normalized_orc_timezone_name(ctz),
_time_zone);
_meta_cache = meta_cache;
_init_profile();
_init_system_properties();
@@ -346,6 +350,7 @@ OrcReader::OrcReader(const TFileScanRangeParams& params,
const TFileRangeDesc& r
_enable_lazy_mat(enable_lazy_mat),
_enable_filter_by_min_max(true),
_dict_cols_has_converted(false) {
+ TimezoneUtils::find_cctz_time_zone(normalized_orc_timezone_name(ctz),
_time_zone);
_meta_cache = meta_cache;
_init_system_properties();
_init_file_description();
@@ -366,6 +371,7 @@ OrcReader::OrcReader(const TFileScanRangeParams& params,
const TFileRangeDesc& r
_enable_lazy_mat(enable_lazy_mat),
_enable_filter_by_min_max(true),
_dict_cols_has_converted(false) {
+ TimezoneUtils::find_cctz_time_zone(normalized_orc_timezone_name(ctz),
_time_zone);
_meta_cache = meta_cache;
_init_system_properties();
_init_file_description();
@@ -1368,7 +1374,7 @@ Status OrcReader::set_fill_columns(
}
try {
_row_reader_options.range(_range_start_offset, _range_size);
- _row_reader_options.setTimezoneName(_ctz == "CST" ? "Asia/Shanghai" :
_ctz);
+
_row_reader_options.setTimezoneName(normalized_orc_timezone_name(_ctz));
if (!_column_ids.empty()) {
std::list<uint64_t> column_ids_list(_column_ids.begin(),
_column_ids.end());
_row_reader_options.includeTypes(column_ids_list);
diff --git a/be/test/format/orc/orc_reader_fill_data_test.cpp
b/be/test/format/orc/orc_reader_fill_data_test.cpp
index dd1d0d2a595..bacba1ccfde 100644
--- a/be/test/format/orc/orc_reader_fill_data_test.cpp
+++ b/be/test/format/orc/orc_reader_fill_data_test.cpp
@@ -23,6 +23,7 @@
#include "core/column/column_array.h"
#include "core/column/column_struct.h"
#include "core/data_type/data_type_array.h"
+#include "core/data_type/data_type_date_or_datetime_v2.h"
#include "core/data_type/data_type_decimal.h"
#include "core/data_type/data_type_map.h"
#include "core/data_type/data_type_number.h"
@@ -31,10 +32,13 @@
#include "format/orc/orc_memory_stream_test.h"
#include "format/orc/vorc_reader.h"
#include "orc/ColumnPrinter.hh"
+#include "util/timezone_utils.h"
namespace doris {
class OrcReaderFillDataTest : public ::testing::Test {
protected:
+ static void SetUpTestSuite() { TimezoneUtils::load_timezones_to_cache(); }
+
void SetUp() override {}
void TearDown() override {}
@@ -71,6 +75,21 @@ std::unique_ptr<orc::LongVectorBatch>
create_long_batch(size_t size,
return batch;
}
+std::unique_ptr<orc::TimestampVectorBatch> create_timestamp_batch(
+ size_t size, const std::vector<int64_t>& seconds, const
std::vector<int64_t>& nanoseconds) {
+ auto batch = std::make_unique<orc::TimestampVectorBatch>(size,
*orc::getDefaultPool());
+ batch->resize(size);
+ batch->notNull.resize(size);
+ batch->hasNulls = false;
+
+ for (size_t i = 0; i < size; ++i) {
+ batch->notNull[i] = true;
+ batch->data[i] = seconds[i];
+ batch->nanoseconds[i] = nanoseconds[i];
+ }
+ return batch;
+}
+
TEST_F(OrcReaderFillDataTest, TestFillLongColumn) {
std::vector<int64_t> values = {1, 2, 3, 4, 5};
auto batch = create_long_batch(values.size(), values);
@@ -125,6 +144,26 @@ TEST_F(OrcReaderFillDataTest, TestFillLongColumnWithNull) {
}
}
+TEST_F(OrcReaderFillDataTest, TimestampDecodeNormalizesCstTimezone) {
+ auto batch = create_timestamp_batch(1, {1577905445}, {321000000});
+ auto data_type = std::make_shared<DataTypeDateTimeV2>(6);
+ auto column = data_type->create_column();
+ auto orc_type_ptr = createPrimitiveType(orc::TypeKind::TIMESTAMP);
+
+ TFileScanRangeParams params;
+ TFileRangeDesc range;
+ auto reader = OrcReader::create_unique(params, range, 4064, "CST",
nullptr, nullptr, true);
+
+ MutableColumnPtr mutable_column = column->assert_mutable();
+ Status status = reader->_fill_doris_data_column<false>(
+ "test_ts", mutable_column, data_type, const_node,
orc_type_ptr.get(), batch.get(), 1);
+
+ ASSERT_TRUE(status.ok()) << status.to_string();
+ const auto& timestamp_column = assert_cast<const
ColumnDateTimeV2&>(*mutable_column);
+ ASSERT_EQ(timestamp_column.size(), 1);
+ EXPECT_EQ(data_type->to_string(timestamp_column.get_data()[0]),
"2020-01-02 03:04:05.321000");
+}
+
TEST_F(OrcReaderFillDataTest, SchemaChangeNullableNullMapUsesAppendedSlice) {
std::vector<int64_t> values = {10, 20, 30};
std::vector<bool> nulls = {true, false, true};
diff --git
a/regression-test/suites/external_table_p0/hive/test_hive_orc_timestamp_timezone.groovy
b/regression-test/suites/external_table_p0/hive/test_hive_orc_timestamp_timezone.groovy
new file mode 100644
index 00000000000..a0a4b5b94f8
--- /dev/null
+++
b/regression-test/suites/external_table_p0/hive/test_hive_orc_timestamp_timezone.groovy
@@ -0,0 +1,132 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_hive_orc_timestamp_timezone", "p0,external") {
+ String enabled = context.config.otherConfigs.get("enableHiveTest")
+ if (enabled == null || !enabled.equalsIgnoreCase("true")) {
+ logger.info("disable Hive test.")
+ return
+ }
+
+ for (String hivePrefix : ["hive3"]) {
+ setHivePrefix(hivePrefix)
+
+ String externalEnvIp = context.config.otherConfigs.get("externalEnvIp")
+ String hmsPort = context.config.otherConfigs.get(hivePrefix +
"HmsPort")
+ String hdfsPort = context.config.otherConfigs.get(hivePrefix +
"HdfsPort")
+ String catalogName = "${hivePrefix}_test_hive_orc_timestamp_timezone"
+ String dbName = "test_hive_orc_timestamp_timezone"
+ String simpleTable = "orc_simple_timestamp"
+ String nestedTable = "orc_nested_timestamp"
+ String expectedPrefix = "2020-01-02 03:04:05.321"
+ String otherTimestamp = "2020-01-03 04:05:06.789"
+
+ try {
+ hive_docker """drop database if exists ${dbName} cascade"""
+ hive_docker """create database ${dbName}"""
+ hive_docker """set hive.local.time.zone=Asia/Shanghai"""
+ hive_docker """
+ create table ${dbName}.${simpleTable} (
+ id bigint,
+ ts timestamp
+ )
+ stored as orc
+ """
+ hive_docker """
+ insert into ${dbName}.${simpleTable}
+ select 0, cast('${expectedPrefix}' as timestamp)
+ union all
+ select 1, cast('${otherTimestamp}' as timestamp)
+ """
+ hive_docker """
+ create table ${dbName}.${nestedTable} (
+ id int,
+ arr array<timestamp>,
+ ts_map map<timestamp, timestamp>,
+ row_value struct<col:timestamp>,
+ nested array<map<timestamp, struct<col:array<timestamp>>>>
+ )
+ stored as orc
+ """
+ hive_docker """
+ insert into ${dbName}.${nestedTable}
+ select
+ 0,
+ array(cast('${expectedPrefix}' as timestamp)),
+ map(cast('${expectedPrefix}' as timestamp),
cast('${expectedPrefix}' as timestamp)),
+ named_struct('col', cast('${expectedPrefix}' as
timestamp)),
+ array(map(
+ cast('${expectedPrefix}' as timestamp),
+ named_struct('col', array(cast('${expectedPrefix}' as
timestamp)))))
+ """
+ hive_docker """
+ insert into ${dbName}.${nestedTable}
+ select
+ 1,
+ array(cast('${otherTimestamp}' as timestamp)),
+ map(cast('${otherTimestamp}' as timestamp),
cast('${otherTimestamp}' as timestamp)),
+ named_struct('col', cast('${otherTimestamp}' as
timestamp)),
+ array(map(
+ cast('${otherTimestamp}' as timestamp),
+ named_struct('col', array(cast('${otherTimestamp}' as
timestamp)))))
+ """
+
+ sql """drop catalog if exists ${catalogName}"""
+ sql """
+ create catalog if not exists ${catalogName} properties (
+ 'type' = 'hms',
+ 'hadoop.username' = 'hadoop',
+ 'fs.defaultFS' = 'hdfs://${externalEnvIp}:${hdfsPort}',
+ 'hive.metastore.uris' =
'thrift://${externalEnvIp}:${hmsPort}'
+ )
+ """
+
+ sql """set enable_nereids_planner=true"""
+ sql """set enable_fallback_to_original_planner=false"""
+ sql """set time_zone = 'CST'"""
+ sql """switch ${catalogName}"""
+ sql """use ${dbName}"""
+
+ def simpleCount = sql """
+ select count(*)
+ from ${simpleTable}
+ where id = 0
+ and cast(ts as varchar) like '${expectedPrefix}%'
+ """
+ assertEquals("1", simpleCount[0][0].toString())
+
+ def nestedCount = sql """
+ select count(*)
+ from ${nestedTable}
+ where id = 0
+ and cast(arr[1] as varchar) like '${expectedPrefix}%'
+ and cast(map_keys(ts_map)[1] as varchar) like
'${expectedPrefix}%'
+ and cast(map_values(ts_map)[1] as varchar) like
'${expectedPrefix}%'
+ and cast(element_at(row_value, 'col') as varchar) like
'${expectedPrefix}%'
+ and cast(map_keys(nested[1])[1] as varchar) like
'${expectedPrefix}%'
+ and cast(element_at(map_values(nested[1])[1], 'col')[1] as
varchar)
+ like '${expectedPrefix}%'
+ """
+ assertEquals("1", nestedCount[0][0].toString())
+ } finally {
+ sql """set time_zone = default"""
+ sql """switch internal"""
+ sql """drop catalog if exists ${catalogName}"""
+ hive_docker """drop database if exists ${dbName} cascade"""
+ }
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
