This is an automated email from the ASF dual-hosted git repository.
Gabriel39 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 66a421b1760 [fix](be) Truncate native ORC timestamps to microseconds
(#68024)
66a421b1760 is described below
commit 66a421b1760ddd0cc44f86074daab3aff95b576f
Author: Oliveira <[email protected]>
AuthorDate: Sun Sep 20 09:41:14 2026 +0800
[fix](be) Truncate native ORC timestamps to microseconds (#68024)
### What problem does this PR solve?
Issue Number: None
Related PR: #67904
Problem Summary:
Paimon ORC tables can store timestamp values with nanosecond precision,
while Doris exposes timestamps at microsecond precision. The Paimon JNI
reader truncates sub-microsecond digits, but the native ORC reader
rounded them. For example, `2024-01-02 10:04:05.123456789` was exposed
as `.123456` by JNI and `.123457` by the native reader, causing equality
predicates to return inconsistent results.
The existing Paimon predicate conversion keeps `TIMESTAMP(7/8/9)`
comparisons as residual predicates. This change makes the native ORC
path use the same truncation semantics as Doris and the JNI reader for
row decoding, stripe statistics, and ORC search argument bounds.
### Release note
Fix inconsistent timestamp values and predicate results between native
ORC and JNI readers for sub-microsecond timestamps.
### Check List (For Author)
- Test
- [x] Regression test
- `./run-regression-test.sh --conf /tmp/yuze-regression-full-conf.groovy
--run -d external_table_p0/paimon -s paimon_timestamp_types`
- [x] Unit Test
- `./run-be-ut.sh --run
--filter=NewOrcReaderTest.*Timestamp*:NewOrcReaderTest.ReadTimestampNanosecondsTruncatesToMicroseconds`
- [ ] Manual test
- [ ] No need to test or manual test
- Behavior changed:
- [x] Yes. Native ORC timestamps now truncate sub-microsecond digits
consistently with Doris precision and the JNI reader.
- [ ] No.
- Does this need documentation?
- [x] No.
- [ ] Yes.
### Check List (For Reviewer who merge this PR)
- [ ] Confirm the release note
- [ ] Confirm test cases
- [ ] Confirm document
- [ ] Add branch pick label
---
.../data_type_serde/data_type_datetimev2_serde.cpp | 12 +-
.../data_type_timestamptz_serde.cpp | 4 +-
be/src/core/data_type_serde/orc_serde_utils.cpp | 21 +--
be/src/core/data_type_serde/orc_serde_utils.h | 7 +-
be/src/format_v2/orc/orc_reader.cpp | 25 ++-
be/src/format_v2/orc/orc_search_argument.cpp | 25 ++-
be/test/format_v2/orc/orc_reader_test.cpp | 175 ++++++++++++++-------
.../paimon/paimon_timestamp_types.out | 47 +++++-
.../external_table_p0/paimon/test_paimon_minio.out | 16 +-
.../tvf/orc_tvf/test_hdfs_orc_group4_orc_files.out | 2 +-
.../tvf/orc_tvf/test_hdfs_orc_group6_orc_files.out | 4 +-
.../paimon/paimon_timestamp_types.groovy | 44 ++++++
12 files changed, 258 insertions(+), 124 deletions(-)
diff --git a/be/src/core/data_type_serde/data_type_datetimev2_serde.cpp
b/be/src/core/data_type_serde/data_type_datetimev2_serde.cpp
index a4b11788986..b695d2ebdfc 100644
--- a/be/src/core/data_type_serde/data_type_datetimev2_serde.cpp
+++ b/be/src/core/data_type_serde/data_type_datetimev2_serde.cpp
@@ -72,8 +72,8 @@ Status decode_timestamp_orc_values(IColumn& nested_column,
const OrcDecodedColum
}
auto& value =
reinterpret_cast<DateV2Value<DateTimeV2ValueType>&>(data[old_data_size + row]);
- orc_serde_utils::RoundedOrcTimestamp timestamp;
- auto status = orc_serde_utils::round_orc_timestamp_to_microseconds(
+ orc_serde_utils::TruncatedOrcTimestamp timestamp;
+ auto status = orc_serde_utils::truncate_orc_timestamp_to_microseconds(
orc_batch->data[source_row],
orc_batch->nanoseconds[source_row], ×tamp);
if (!status.ok()) {
data.resize(old_data_size);
@@ -86,14 +86,6 @@ Status decode_timestamp_orc_values(IColumn& nested_column,
const OrcDecodedColum
"Decoded ORC timestamp is outside the target timezone
range");
}
value.set_microsecond(timestamp.microseconds);
- // Plain ORC TIMESTAMP is a civil value. Carry after timezone
conversion so a fractional
- // round does not jump backward or skip an hour at a daylight-saving
transition.
- if (timestamp.carry &&
- !value.date_add_interval<TimeUnit::SECOND>(TimeInterval
{TimeUnit::SECOND, 1, false})) {
- data.resize(old_data_size);
- return Status::DataQualityError(
- "Decoded ORC timestamp is outside the target timezone
range");
- }
}
return Status::OK();
}
diff --git a/be/src/core/data_type_serde/data_type_timestamptz_serde.cpp
b/be/src/core/data_type_serde/data_type_timestamptz_serde.cpp
index d83bc0745d9..f75755f2f52 100644
--- a/be/src/core/data_type_serde/data_type_timestamptz_serde.cpp
+++ b/be/src/core/data_type_serde/data_type_timestamptz_serde.cpp
@@ -56,8 +56,8 @@ Status decode_timestamp_tz_orc_values(IColumn& nested_column,
continue;
}
auto& value = data[old_data_size + row];
- orc_serde_utils::RoundedOrcTimestamp timestamp;
- auto status = orc_serde_utils::round_orc_timestamp_to_microseconds(
+ orc_serde_utils::TruncatedOrcTimestamp timestamp;
+ auto status = orc_serde_utils::truncate_orc_timestamp_to_microseconds(
orc_batch->data[source_row],
orc_batch->nanoseconds[source_row], ×tamp);
if (!status.ok()) {
data.resize(old_data_size);
diff --git a/be/src/core/data_type_serde/orc_serde_utils.cpp
b/be/src/core/data_type_serde/orc_serde_utils.cpp
index 5b7c88dbd63..74f910798c7 100644
--- a/be/src/core/data_type_serde/orc_serde_utils.cpp
+++ b/be/src/core/data_type_serde/orc_serde_utils.cpp
@@ -41,25 +41,16 @@ bool orc_row_is_null(const ::orc::ColumnVectorBatch& batch,
size_t row) {
return batch.hasNulls && !batch.notNull[row];
}
-Status round_orc_timestamp_to_microseconds(int64_t seconds, int64_t
nanoseconds,
- RoundedOrcTimestamp* result) {
+Status truncate_orc_timestamp_to_microseconds(int64_t seconds, int64_t
nanoseconds,
+ TruncatedOrcTimestamp* result) {
constexpr int64_t NANOS_PER_SECOND = 1000000000;
constexpr int64_t NANOS_PER_MICROSECOND = 1000;
- constexpr int64_t MICROS_PER_SECOND = 1000000;
DORIS_CHECK(result != nullptr);
DORIS_CHECK(nanoseconds >= 0 && nanoseconds < NANOS_PER_SECOND);
- // Doris stores six fractional digits, so use half-up rounding and carry
999999500ns into the
- // next second instead of silently truncating the ORC value.
- const auto rounded_microseconds =
- (nanoseconds + NANOS_PER_MICROSECOND / 2) / NANOS_PER_MICROSECOND;
- // Validate the carry here, but validate Doris' calendar range after
timezone conversion:
- // a valid year-zero local timestamp may have a UTC epoch before year zero.
- if (__builtin_add_overflow(seconds, rounded_microseconds /
MICROS_PER_SECOND,
- &result->seconds)) {
- return Status::DataQualityError("ORC timestamp overflows after
microsecond rounding");
- }
- result->microseconds = cast_set<uint64_t>(rounded_microseconds %
MICROS_PER_SECOND);
- result->carry = rounded_microseconds >= MICROS_PER_SECOND;
+ // Doris exposes ORC timestamps at microsecond precision. Discard
sub-microsecond digits so
+ // native ORC and JNI readers retain the same visible value and predicate
semantics.
+ result->seconds = seconds;
+ result->microseconds = cast_set<uint64_t>(nanoseconds /
NANOS_PER_MICROSECOND);
return Status::OK();
}
diff --git a/be/src/core/data_type_serde/orc_serde_utils.h
b/be/src/core/data_type_serde/orc_serde_utils.h
index 06b9c4406ee..1adcf8faafb 100644
--- a/be/src/core/data_type_serde/orc_serde_utils.h
+++ b/be/src/core/data_type_serde/orc_serde_utils.h
@@ -32,14 +32,13 @@ size_t orc_decode_row_count(size_t rows, const
std::vector<size_t>* selected_row
size_t orc_source_row_at(size_t row, const std::vector<size_t>* selected_rows);
bool orc_row_is_null(const ::orc::ColumnVectorBatch& batch, size_t row);
-struct RoundedOrcTimestamp {
+struct TruncatedOrcTimestamp {
int64_t seconds;
uint64_t microseconds;
- bool carry;
};
-Status round_orc_timestamp_to_microseconds(int64_t seconds, int64_t
nanoseconds,
- RoundedOrcTimestamp* result);
+Status truncate_orc_timestamp_to_microseconds(int64_t seconds, int64_t
nanoseconds,
+ TruncatedOrcTimestamp* result);
DecodedColumnView make_orc_decoded_view(const OrcDecodedColumnView& orc_view,
DecodedValueKind value_kind);
diff --git a/be/src/format_v2/orc/orc_reader.cpp
b/be/src/format_v2/orc/orc_reader.cpp
index b7ffc1c9134..9407e7ef31c 100644
--- a/be/src/format_v2/orc/orc_reader.cpp
+++ b/be/src/format_v2/orc/orc_reader.cpp
@@ -502,22 +502,12 @@ std::optional<DateV2Value<DateTimeV2ValueType>>
datetime_v2_from_orc_millis(
}
const auto extra_nanos = std::max<int32_t>(nanos_tail, 0);
constexpr int64_t NANOS_PER_MICROSECOND = 1000;
- constexpr int64_t MICROS_PER_SECOND = 1000000;
// Stripe statistics split the timestamp into milliseconds and the
remaining nanoseconds. Use
- // the same half-up rule as row decoding so zone-map pruning observes
identical values.
- const auto rounded_extra_microseconds =
- (extra_nanos + NANOS_PER_MICROSECOND / 2) / NANOS_PER_MICROSECOND;
- const auto microseconds_with_carry = millis_remainder * 1000 +
rounded_extra_microseconds;
- // Calendar bounds depend on the target timezone, so only reject
arithmetic overflow here and
- // let the converted value below decide whether the statistic is
representable by Doris.
- int64_t rounded_seconds;
- if (__builtin_add_overflow(seconds, microseconds_with_carry /
MICROS_PER_SECOND,
- &rounded_seconds)) {
- return std::nullopt;
- }
- const auto microseconds = cast_set<uint64_t>(microseconds_with_carry %
MICROS_PER_SECOND);
+ // the same truncation as row decoding so zone-map pruning observes
identical values.
+ const auto microseconds =
+ cast_set<uint64_t>(millis_remainder * 1000 + extra_nanos /
NANOS_PER_MICROSECOND);
DateV2Value<DateTimeV2ValueType> value;
- value.from_unixtime(rounded_seconds, timezone);
+ value.from_unixtime(seconds, timezone);
value.set_microsecond(microseconds);
if (!value.is_valid_date()) {
return std::nullopt;
@@ -543,6 +533,13 @@ bool set_timestamp_zone_map(const ::orc::ColumnStatistics&
statistics,
!timestamp_statistics->hasMaximum()) {
return false;
}
+ // ORC substitutes these conservative tails when ORC-611 nanos fields are
absent. The public
+ // statistics API cannot distinguish those sentinels from equal serialized
values, so fall
+ // back to a row scan rather than expose an inexact bound as an exact
aggregate result.
+ if (timestamp_statistics->getMinimumNanos() == 0 ||
+ timestamp_statistics->getMaximumNanos() == 999999) {
+ return false;
+ }
const auto min_endpoint =
std::pair(timestamp_statistics->getMinimum(),
timestamp_statistics->getMinimumNanos());
const auto max_endpoint =
diff --git a/be/src/format_v2/orc/orc_search_argument.cpp
b/be/src/format_v2/orc/orc_search_argument.cpp
index 5cee9cf46c1..3c7094ec38c 100644
--- a/be/src/format_v2/orc/orc_search_argument.cpp
+++ b/be/src/format_v2/orc/orc_search_argument.cpp
@@ -984,9 +984,8 @@ std::optional<OrcSargComparisonLiteral>
make_comparison_literal_for_sarg(
if (column.predicate_type == ::orc::PredicateDataType::TIMESTAMP) {
const auto timestamp = literal->getTimestamp();
// ORC reconstructs negative timestamp statistics with truncating
division, which can
- // produce a non-canonical nanos component. The exact epoch is unsafe
too because its
- // half-up lower bound is -500ns. Leave these predicates to Doris row
filtering.
- if (timestamp.second < 0 || (timestamp.second == 0 && timestamp.nanos
== 0)) {
+ // produce a non-canonical nanos component. Leave negative timestamps
to row filtering.
+ if (timestamp.second < 0) {
return std::nullopt;
}
}
@@ -1066,7 +1065,7 @@ std::optional<std::vector<::orc::Literal>>
make_in_literals_for_sarg(
}
if (column.predicate_type == ::orc::PredicateDataType::TIMESTAMP) {
const auto timestamp = literal->getTimestamp();
- if (timestamp.second < 0 || (timestamp.second == 0 &&
timestamp.nanos == 0)) {
+ if (timestamp.second < 0) {
return std::nullopt;
}
}
@@ -1248,17 +1247,17 @@ OrcSargNode make_and_node(std::vector<OrcSargNode>
children) {
return shift_timestamp_literal(literal, NANOS_PER_MILLISECOND - remainder);
}
-std::pair<::orc::Literal, ::orc::Literal> timestamp_rounding_bounds(const
::orc::Literal& literal) {
- constexpr int32_t HALF_MICROSECOND_NANOS = 500;
- // All raw ORC values in [lower, upper) round half-up to this Doris
microsecond. Round these
+std::pair<::orc::Literal, ::orc::Literal> timestamp_truncation_bounds(
+ const ::orc::Literal& literal) {
+ constexpr int32_t NANOS_PER_MICROSECOND = 1000;
+ // All raw ORC values in [lower, upper) truncate to this Doris
microsecond. Expand these
// boundaries toward the side that enlarges the SARG match set because ORC
statistics retain
// only millisecond precision.
- return {shift_timestamp_literal(literal, -HALF_MICROSECOND_NANOS),
- shift_timestamp_literal(literal, HALF_MICROSECOND_NANOS)};
+ return {literal, shift_timestamp_literal(literal, NANOS_PER_MICROSECOND)};
}
OrcSargNode make_timestamp_equals_node(const OrcSargColumn& column, const
::orc::Literal& literal) {
- const auto [lower_bound, upper_bound] = timestamp_rounding_bounds(literal);
+ const auto [lower_bound, upper_bound] =
timestamp_truncation_bounds(literal);
std::vector<OrcSargNode> children;
children.push_back(make_not_node(make_literal_node(
OrcSargNodeKind::LESS_THAN, column,
floor_timestamp_literal_to_millis(lower_bound))));
@@ -1268,7 +1267,7 @@ OrcSargNode make_timestamp_equals_node(const
OrcSargColumn& column, const ::orc:
}
std::optional<OrcSargNode> make_timestamp_comparison_node(const
OrcSargComparison& comparison) {
- const auto [lower_bound, upper_bound] =
timestamp_rounding_bounds(comparison.literal);
+ const auto [lower_bound, upper_bound] =
timestamp_truncation_bounds(comparison.literal);
switch (comparison.normalized_op) {
case TExprOpcode::GE:
return make_not_node(make_literal_node(OrcSargNodeKind::LESS_THAN,
comparison.column,
@@ -1310,9 +1309,9 @@ OrcSargNode make_timestamp_in_node(const OrcSargColumn&
column,
}
}
const auto lower_bound =
-
floor_timestamp_literal_to_millis(timestamp_rounding_bounds(min_literal).first);
+
floor_timestamp_literal_to_millis(timestamp_truncation_bounds(min_literal).first);
const auto upper_bound =
-
ceil_timestamp_literal_to_millis(timestamp_rounding_bounds(max_literal).second);
+
ceil_timestamp_literal_to_millis(timestamp_truncation_bounds(max_literal).second);
std::vector<OrcSargNode> children;
children.push_back(
make_not_node(make_literal_node(OrcSargNodeKind::LESS_THAN,
column, lower_bound)));
diff --git a/be/test/format_v2/orc/orc_reader_test.cpp
b/be/test/format_v2/orc/orc_reader_test.cpp
index d9be5a5e6d0..5cad401a06c 100644
--- a/be/test/format_v2/orc/orc_reader_test.cpp
+++ b/be/test/format_v2/orc/orc_reader_test.cpp
@@ -4320,7 +4320,7 @@ void write_multi_stripe_orc_sarg_types_file(const
std::string& file_path) {
void write_multi_stripe_orc_timestamp_instant_sarg_file(
const std::string& file_path, int64_t first_timestamp_second = 0,
- int64_t second_timestamp_second = 1609459200) {
+ int64_t second_timestamp_second = 1609459200, int64_t nanoseconds =
123000000) {
auto type = std::unique_ptr<::orc::Type>(::orc::Type::buildTypeFromString(
"struct<timestamp_instant_col:timestamp with local time
zone,payload:string>"));
@@ -4342,7 +4342,7 @@ void write_multi_stripe_orc_timestamp_instant_sarg_file(
payloads.reserve(ROWS_PER_STRIPE);
for (int64_t row = 0; row < ROWS_PER_STRIPE; ++row) {
timestamp_batch.data[row] = first_timestamp_second + row;
- timestamp_batch.nanoseconds[row] = 123000000;
+ timestamp_batch.nanoseconds[row] = nanoseconds;
payloads.push_back(std::string(2048, static_cast<char>('a' + row %
26)));
set_string_value(payload_batch, row, payloads.back());
}
@@ -4941,7 +4941,7 @@ TEST_F(NewOrcReaderTest,
AggregatePushdownUsesPrunedStripes) {
TEST_F(NewOrcReaderTest,
AggregatePushdownTimestampMinMaxMatchesScanInSessionTimezone) {
const auto multi_stripe_file_path = (_test_dir /
"aggregate_timestamp_timezone.orc").string();
write_two_stripe_constant_timestamp_file(multi_stripe_file_path,
1609430400, 1609434000,
- "Asia/Shanghai");
+ "Asia/Shanghai", 123456789);
RuntimeState state {TQueryOptions(), TQueryGlobals()};
state.set_timezone("Asia/Shanghai");
@@ -4976,8 +4976,8 @@ TEST_F(NewOrcReaderTest,
AggregatePushdownTimestampMinMaxMatchesScanInSessionTim
ASSERT_EQ(scan_rows, 400);
ASSERT_TRUE(scan_min.has_value());
ASSERT_TRUE(scan_max.has_value());
- EXPECT_EQ(*scan_min, make_datetime_v2(2021, 1, 1, 0, 0, 0, 123000));
- EXPECT_EQ(*scan_max, make_datetime_v2(2021, 1, 1, 1, 0, 0, 123000));
+ EXPECT_EQ(*scan_min, make_datetime_v2(2021, 1, 1, 0, 0, 0, 123456));
+ EXPECT_EQ(*scan_max, make_datetime_v2(2021, 1, 1, 1, 0, 0, 123456));
auto reader = create_reader_for_path(multi_stripe_file_path);
ASSERT_TRUE(reader->init(&state).ok());
@@ -5082,8 +5082,8 @@ TEST_F(NewOrcReaderTest,
AggregatePushdownTimestampMinMaxFallsBackForPreOrc135Wr
TEST_F(NewOrcReaderTest,
AggregatePushdownTimestampMinMaxMatchesNegativeRowDecoding) {
const auto file_path = (_test_dir /
"aggregate_negative_timestamp.orc").string();
- // Unlike ORC SARG reconstruction, the aggregate path canonicalizes
negative millisecond
- // remainders before rounding. Its statistics result must match the
decoded row value.
+ // The aggregate path canonicalizes negative millisecond remainders before
truncation. Its
+ // statistics result must match the decoded row value.
write_timestamp_rounding_orc_file(file_path, 1, 3);
auto reader = create_reader_for_path(file_path);
@@ -5106,11 +5106,42 @@ TEST_F(NewOrcReaderTest,
AggregatePushdownTimestampMinMaxMatchesNegativeRowDecod
EXPECT_EQ(aggregate_result.count, 1);
EXPECT_TRUE(aggregate_result.columns[0].has_min);
EXPECT_TRUE(aggregate_result.columns[0].has_max);
- const auto expected = make_datetime_v2(1969, 12, 31, 23, 59, 58, 999999);
+ const auto expected = make_datetime_v2(1969, 12, 31, 23, 59, 58, 999998);
EXPECT_EQ(aggregate_result.columns[0].min_value.get<TYPE_DATETIMEV2>(),
expected);
EXPECT_EQ(aggregate_result.columns[0].max_value.get<TYPE_DATETIMEV2>(),
expected);
}
+TEST_F(NewOrcReaderTest,
AggregatePushdownTimestampMinMaxFallsBackWithoutExactNanosTails) {
+ const auto assert_fallback = [&](std::string_view file_name, bool
timestamp_instant) {
+ const auto file_path = (_test_dir / file_name).string();
+ if (timestamp_instant) {
+ write_multi_stripe_orc_timestamp_instant_sarg_file(file_path);
+ } else {
+ write_two_stripe_constant_timestamp_file(file_path, 1, 2);
+ }
+
+ auto reader = create_reader_for_path(file_path, nullptr, nullptr,
std::nullopt,
+ timestamp_instant);
+ RuntimeState state {TQueryOptions(), TQueryGlobals()};
+ state.set_timezone("+00:00");
+ EXPECT_TRUE(reader->init(&state).ok());
+ auto request = std::make_shared<format::FileScanRequest>();
+ request->non_predicate_columns = {field_projection(0)};
+ EXPECT_TRUE(reader->open(request).ok());
+
+ format::FileAggregateRequest aggregate_request;
+ aggregate_request.agg_type = TPushAggOp::type::MINMAX;
+ aggregate_request.columns.push_back(
+ {.projection =
format::LocalColumnIndex::top_level(format::LocalColumnId(0))});
+ format::FileAggregateResult aggregate_result;
+ const auto status = reader->get_aggregate_result(aggregate_request,
&aggregate_result);
+ EXPECT_TRUE(status.is<ErrorCode::NOT_IMPLEMENTED_ERROR>()) << status;
+ };
+
+ assert_fallback("aggregate_timestamp_no_nanos_tail.orc", false);
+ assert_fallback("aggregate_timestamp_instant_no_nanos_tail.orc", true);
+}
+
TEST_F(NewOrcReaderTest, AggregatePushdownTimestampMinMaxRejectsNamedTimezone)
{
const auto multi_stripe_file_path = (_test_dir /
"aggregate_timestamp_dst_fold.orc").string();
// 2021-11-07 08:30Z and 09:00Z straddle the America/Los_Angeles fall-back
transition.
@@ -5137,7 +5168,8 @@ TEST_F(NewOrcReaderTest,
AggregatePushdownTimestampMinMaxRejectsNamedTimezone) {
TEST_F(NewOrcReaderTest,
AggregatePushdownTimestampInstantMinMaxUsesTimestampTzWhenMapped) {
const auto multi_stripe_file_path = (_test_dir /
"aggregate_timestamp_instant_tz.orc").string();
- write_multi_stripe_orc_timestamp_instant_sarg_file(multi_stripe_file_path);
+ write_multi_stripe_orc_timestamp_instant_sarg_file(multi_stripe_file_path,
0, 1609459200,
+ 123456789);
auto reader =
create_reader_for_path(multi_stripe_file_path, nullptr, nullptr,
std::nullopt, true);
@@ -5169,10 +5201,10 @@ TEST_F(NewOrcReaderTest,
AggregatePushdownTimestampInstantMinMaxUsesTimestampTzW
ASSERT_EQ(aggregate_result.columns[0].max_value.get_type(),
TYPE_TIMESTAMPTZ);
EXPECT_EQ(aggregate_result.columns[0].min_value.get<TYPE_TIMESTAMPTZ>().to_string(
state.timezone_obj(), 6),
- "1970-01-01 08:00:00.123000+08:00");
+ "1970-01-01 08:00:00.123456+08:00");
EXPECT_EQ(aggregate_result.columns[0].max_value.get<TYPE_TIMESTAMPTZ>().to_string(
state.timezone_obj(), 6),
- "2021-01-01 08:03:19.123000+08:00");
+ "2021-01-01 08:03:19.123456+08:00");
}
TEST_F(NewOrcReaderTest, AggregatePushdownCharMinMaxTrimsTrailingSpaces) {
@@ -10371,7 +10403,7 @@ TEST_F(NewOrcReaderTest, ReadPrimitiveTypesWithNulls) {
EXPECT_EQ(schema[15].type->to_string(*block.get_by_position(15).column,
NULL_ROW), "NULL");
}
-TEST_F(NewOrcReaderTest, ReadTimestampNanosecondsRoundsToMicroseconds) {
+TEST_F(NewOrcReaderTest, ReadTimestampNanosecondsTruncatesToMicroseconds) {
const auto file_path = (_test_dir / "timestamp_rounding.orc").string();
write_timestamp_rounding_orc_file(file_path);
auto reader = create_reader_for_path(file_path);
@@ -10395,8 +10427,8 @@ TEST_F(NewOrcReaderTest,
ReadTimestampNanosecondsRoundsToMicroseconds) {
ASSERT_EQ(rows, TIMESTAMP_ROUNDING_ROW_COUNT);
static constexpr std::array<std::string_view,
TIMESTAMP_ROUNDING_ROW_COUNT> expected {
- "1970-01-01 00:00:01.111002", "1970-01-01 00:00:02.345678",
- "2021-01-01 00:00:00.000000", "1969-12-31 23:59:58.999999"};
+ "1970-01-01 00:00:01.111001", "1970-01-01 00:00:02.345678",
+ "2020-12-31 23:59:59.999999", "1969-12-31 23:59:58.999998"};
for (size_t column_id = 0; column_id < schema.size(); ++column_id) {
for (size_t row = 0; row < expected.size(); ++row) {
EXPECT_EQ(schema[column_id].type->to_string(*block.get_by_position(column_id).column,
@@ -10439,48 +10471,48 @@ Status decode_orc_timestamp_boundary(int64_t seconds,
int64_t nanoseconds, bool
return status;
}
-TEST_F(NewOrcReaderTest,
PlainTimestampRoundsCarryInCivilTimeAcrossDstTransitions) {
+TEST_F(NewOrcReaderTest,
PlainTimestampTruncatesSubMicrosecondsAcrossDstTransitions) {
TimezoneUtils::load_timezones_to_cache();
cctz::time_zone los_angeles;
ASSERT_TRUE(TimezoneUtils::find_cctz_time_zone("America/Los_Angeles",
los_angeles));
- constexpr int64_t ROUNDING_CARRY_NANOS = 999999500;
+ constexpr int64_t SUB_MICROSECOND_NANOS = 999999500;
std::string decoded_value;
- ASSERT_TRUE(decode_orc_timestamp_boundary(1636275599,
ROUNDING_CARRY_NANOS, false, los_angeles,
+ ASSERT_TRUE(decode_orc_timestamp_boundary(1636275599,
SUB_MICROSECOND_NANOS, false, los_angeles,
&decoded_value)
.ok());
- EXPECT_EQ(decoded_value, "2021-11-07 02:00:00.000000");
+ EXPECT_EQ(decoded_value, "2021-11-07 01:59:59.999999");
- ASSERT_TRUE(decode_orc_timestamp_boundary(1615715999,
ROUNDING_CARRY_NANOS, false, los_angeles,
+ ASSERT_TRUE(decode_orc_timestamp_boundary(1615715999,
SUB_MICROSECOND_NANOS, false, los_angeles,
&decoded_value)
.ok());
- EXPECT_EQ(decoded_value, "2021-03-14 02:00:00.000000");
+ EXPECT_EQ(decoded_value, "2021-03-14 01:59:59.999999");
}
-TEST_F(NewOrcReaderTest, TimestampInstantKeepsEpochCarryAcrossDstTransitions) {
+TEST_F(NewOrcReaderTest,
TimestampInstantTruncatesSubMicrosecondsAcrossDstTransitions) {
TimezoneUtils::load_timezones_to_cache();
cctz::time_zone los_angeles;
ASSERT_TRUE(TimezoneUtils::find_cctz_time_zone("America/Los_Angeles",
los_angeles));
- constexpr int64_t ROUNDING_CARRY_NANOS = 999999500;
+ constexpr int64_t SUB_MICROSECOND_NANOS = 999999500;
std::string decoded_value;
- ASSERT_TRUE(decode_orc_timestamp_boundary(1636275599,
ROUNDING_CARRY_NANOS, true, los_angeles,
+ ASSERT_TRUE(decode_orc_timestamp_boundary(1636275599,
SUB_MICROSECOND_NANOS, true, los_angeles,
&decoded_value)
.ok());
- EXPECT_EQ(decoded_value, "2021-11-07 09:00:00.000000+00:00");
+ EXPECT_EQ(decoded_value, "2021-11-07 08:59:59.999999+00:00");
- ASSERT_TRUE(decode_orc_timestamp_boundary(1615715999,
ROUNDING_CARRY_NANOS, true, los_angeles,
+ ASSERT_TRUE(decode_orc_timestamp_boundary(1615715999,
SUB_MICROSECOND_NANOS, true, los_angeles,
&decoded_value)
.ok());
- EXPECT_EQ(decoded_value, "2021-03-14 10:00:00.000000+00:00");
+ EXPECT_EQ(decoded_value, "2021-03-14 09:59:59.999999+00:00");
}
-TEST_F(NewOrcReaderTest, PlainTimestampDstCarryMatchesStripeStatistics) {
+TEST_F(NewOrcReaderTest, PlainTimestampTruncationMatchesStripeStatistics) {
const auto file_path = (_test_dir /
"timestamp_dst_carry_statistics.orc").string();
constexpr int64_t BEFORE_DST_ROLLBACK = 1636275599;
- constexpr int64_t ROUNDING_CARRY_NANOS = 999999500;
+ constexpr int64_t SUB_MICROSECOND_NANOS = 999999500;
write_two_stripe_constant_timestamp_file(file_path, BEFORE_DST_ROLLBACK,
BEFORE_DST_ROLLBACK,
- "America/Los_Angeles",
ROUNDING_CARRY_NANOS);
+ "America/Los_Angeles",
SUB_MICROSECOND_NANOS);
RuntimeState state {TQueryOptions(), TQueryGlobals()};
state.set_timezone("America/Los_Angeles");
@@ -10500,7 +10532,7 @@ TEST_F(NewOrcReaderTest,
PlainTimestampDstCarryMatchesStripeStatistics) {
format::FileAggregateResult aggregate_result;
ASSERT_TRUE(reader->get_aggregate_result(aggregate_request,
&aggregate_result).ok());
ASSERT_EQ(aggregate_result.columns.size(), 1);
- const auto expected = make_datetime_v2(2021, 11, 7, 2, 0, 0);
+ const auto expected = make_datetime_v2(2021, 11, 7, 1, 59, 59, 999999);
EXPECT_EQ(aggregate_result.columns[0].min_value.get<TYPE_DATETIMEV2>(),
expected);
EXPECT_EQ(aggregate_result.columns[0].max_value.get<TYPE_DATETIMEV2>(),
expected);
}
@@ -10522,28 +10554,28 @@ TEST_F(NewOrcReaderTest,
PlainTimestampAcceptsDorisBoundariesAcrossTimezones) {
.ok());
}
-TEST_F(NewOrcReaderTest, TimestampRoundingRejectsUpperDorisBoundary) {
- // 253402300799 is 9999-12-31 23:59:59 UTC; rounding the fractional part
carries
- // into year 10000, which neither Doris timestamp representation can store.
+TEST_F(NewOrcReaderTest, TimestampTruncationAcceptsUpperDorisBoundary) {
+ // 253402300799 is 9999-12-31 23:59:59 UTC. Truncation retains the
representable
+ // 999999 microsecond value instead of carrying into year 10000.
constexpr int64_t MAX_DORIS_EPOCH_SECOND = 253402300799LL;
- constexpr int64_t ROUNDING_CARRY_NANOS = 999999500;
- EXPECT_FALSE(decode_orc_timestamp_boundary(MAX_DORIS_EPOCH_SECOND,
ROUNDING_CARRY_NANOS, false)
- .ok());
- EXPECT_FALSE(
- decode_orc_timestamp_boundary(MAX_DORIS_EPOCH_SECOND,
ROUNDING_CARRY_NANOS, true).ok());
+ constexpr int64_t SUB_MICROSECOND_NANOS = 999999500;
+ EXPECT_TRUE(decode_orc_timestamp_boundary(MAX_DORIS_EPOCH_SECOND,
SUB_MICROSECOND_NANOS, false)
+ .ok());
+ EXPECT_TRUE(decode_orc_timestamp_boundary(MAX_DORIS_EPOCH_SECOND,
SUB_MICROSECOND_NANOS, true)
+ .ok());
}
-TEST_F(NewOrcReaderTest, TimestampRoundingRejectsSecondOverflow) {
- constexpr int64_t ROUNDING_CARRY_NANOS = 999999500;
+TEST_F(NewOrcReaderTest, TimestampTruncationRejectsOutOfRangeSecond) {
+ constexpr int64_t SUB_MICROSECOND_NANOS = 999999500;
EXPECT_FALSE(decode_orc_timestamp_boundary(std::numeric_limits<int64_t>::max(),
- ROUNDING_CARRY_NANOS, false)
+ SUB_MICROSECOND_NANOS, false)
.ok());
EXPECT_FALSE(decode_orc_timestamp_boundary(std::numeric_limits<int64_t>::max(),
- ROUNDING_CARRY_NANOS, true)
+ SUB_MICROSECOND_NANOS, true)
.ok());
}
-TEST_F(NewOrcReaderTest, TimestampSargMatchesRoundedMicroseconds) {
+TEST_F(NewOrcReaderTest, TimestampSargMatchesTruncatedMicroseconds) {
const auto file_path = (_test_dir /
"timestamp_rounding_sarg.orc").string();
write_timestamp_rounding_orc_file(file_path, 1);
auto reader = create_reader_for_path(file_path);
@@ -10561,7 +10593,7 @@ TEST_F(NewOrcReaderTest,
TimestampSargMatchesRoundedMicroseconds) {
VExprContext::create_shared(std::make_shared<NullableInExpr<TYPE_DATETIMEV2>>(
0, remove_nullable(schema[0].type),
std::vector<Field> {Field::create_field<TYPE_DATETIMEV2>(
- make_datetime_v2(1970, 1, 1, 0, 0, 1, 111002))},
+ make_datetime_v2(1970, 1, 1, 0, 0, 1, 111001))},
"timestamp_col")));
ASSERT_TRUE(reader->open(request).ok());
@@ -10572,14 +10604,14 @@ TEST_F(NewOrcReaderTest,
TimestampSargMatchesRoundedMicroseconds) {
EXPECT_FALSE(eof);
ASSERT_EQ(rows, 1);
EXPECT_EQ(schema[0].type->to_string(*block.get_by_position(0).column, 0),
- "1970-01-01 00:00:01.111002");
+ "1970-01-01 00:00:01.111001");
}
TEST_F(NewOrcReaderTest, TimestampSargMillisecondBoundsNeverPruneMatches) {
auto scan_rows = [&](std::string_view suffix, TExprOpcode::type
comparison_op,
uint32_t literal_microseconds, bool not_in = false) {
const auto file_path = (_test_dir /
fmt::format("timestamp_sarg_{}.orc", suffix)).string();
- // The first stripe rounds 1.111001900 to 1.111002, while the second
rounds the same
+ // The first stripe truncates 1.111001900 to 1.111001, while the
second truncates the same
// fractional part at second 10. Millisecond statistics must never
make the SARG stricter
// than Doris's exact microsecond row comparison.
write_two_stripe_constant_timestamp_file(file_path, 1, 10, "GMT",
111001900);
@@ -10620,17 +10652,54 @@ TEST_F(NewOrcReaderTest,
TimestampSargMillisecondBoundsNeverPruneMatches) {
return std::pair {result_rows,
reader->reader_statistics().filtered_row_groups};
};
- EXPECT_EQ(scan_rows("gt", TExprOpcode::GT, 111001).first, 400);
+ EXPECT_EQ(scan_rows("gt", TExprOpcode::GT, 111001).first, 200);
EXPECT_EQ(scan_rows("lt", TExprOpcode::LT, 111003).first, 200);
const auto not_equal = scan_rows("ne", TExprOpcode::NE, 111001);
- EXPECT_EQ(not_equal.first, 400);
+ EXPECT_EQ(not_equal.first, 200);
EXPECT_EQ(not_equal.second, 0);
const auto not_in = scan_rows("not_in", TExprOpcode::INVALID_OPCODE,
111001, true);
- EXPECT_EQ(not_in.first, 400);
+ EXPECT_EQ(not_in.first, 200);
EXPECT_EQ(not_in.second, 0);
}
-TEST_F(NewOrcReaderTest, NegativeTimestampRoundingBypassesUnsafeSarg) {
+TEST_F(NewOrcReaderTest, TimestampSargExactEpochPrunesNegativeStripe) {
+ const auto file_path = (_test_dir /
"timestamp_sarg_exact_epoch.orc").string();
+ write_two_stripe_constant_timestamp_file(file_path, -2, 0, "GMT", 789);
+ ASSERT_EQ(get_orc_stripe_count(file_path), 2);
+
+ auto reader = create_reader_for_path(file_path);
+ RuntimeState state {TQueryOptions(), TQueryGlobals()};
+ state.set_timezone("+00:00");
+ ASSERT_TRUE(reader->init(&state).ok());
+ std::vector<format::ColumnDefinition> schema;
+ ASSERT_TRUE(reader->get_schema(&schema).ok());
+ ASSERT_EQ(schema.size(), 2);
+
+ auto request = std::make_shared<format::FileScanRequest>();
+ request->predicate_columns = {field_projection(0)};
+ request->conjuncts.push_back(
+
VExprContext::create_shared(std::make_shared<NullableInExpr<TYPE_DATETIMEV2>>(
+ 0, remove_nullable(schema[0].type),
+ std::vector<Field> {
+
Field::create_field<TYPE_DATETIMEV2>(make_datetime_v2(1970, 1, 1))},
+ "timestamp_col")));
+ ASSERT_TRUE(reader->open(request).ok());
+
+ bool eof = false;
+ size_t result_rows = 0;
+ while (!eof) {
+ Block block = build_file_block({schema[0]});
+ size_t rows = 0;
+ ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok());
+ result_rows += rows;
+ }
+ EXPECT_EQ(result_rows, 200);
+ EXPECT_EQ(reader->reader_statistics().filtered_row_groups, 1);
+ EXPECT_EQ(reader->reader_statistics().filtered_row_groups_by_min_max, 1);
+ EXPECT_EQ(reader->reader_statistics().filtered_group_rows, 200);
+}
+
+TEST_F(NewOrcReaderTest, NegativeTimestampTruncationBypassesUnsafeSarg) {
const auto file_path = (_test_dir /
"negative_timestamp_rounding_sarg.orc").string();
write_timestamp_rounding_orc_file(file_path, 1, 3);
auto reader = create_reader_for_path(file_path);
@@ -10648,7 +10717,7 @@ TEST_F(NewOrcReaderTest,
NegativeTimestampRoundingBypassesUnsafeSarg) {
VExprContext::create_shared(std::make_shared<NullableInExpr<TYPE_DATETIMEV2>>(
0, remove_nullable(schema[0].type),
std::vector<Field> {Field::create_field<TYPE_DATETIMEV2>(
- make_datetime_v2(1969, 12, 31, 23, 59, 58,
999999))},
+ make_datetime_v2(1969, 12, 31, 23, 59, 58,
999998))},
"timestamp_col")));
ASSERT_TRUE(reader->open(request).ok());
@@ -10659,7 +10728,7 @@ TEST_F(NewOrcReaderTest,
NegativeTimestampRoundingBypassesUnsafeSarg) {
EXPECT_FALSE(eof);
ASSERT_EQ(rows, 1);
EXPECT_EQ(schema[0].type->to_string(*block.get_by_position(0).column, 0),
- "1969-12-31 23:59:58.999999");
+ "1969-12-31 23:59:58.999998");
EXPECT_EQ(reader->reader_statistics().filtered_row_groups, 0);
}
@@ -10692,7 +10761,7 @@ TEST_F(NewOrcReaderTest,
NegativeTimestampNotInEpochBypassesUnsafeSarg) {
EXPECT_FALSE(eof);
ASSERT_EQ(rows, 1);
EXPECT_EQ(schema[0].type->to_string(*block.get_by_position(0).column, 0),
- "1969-12-31 23:59:58.999999");
+ "1969-12-31 23:59:58.999998");
EXPECT_EQ(reader->reader_statistics().filtered_row_groups, 0);
}
diff --git
a/regression-test/data/external_table_p0/paimon/paimon_timestamp_types.out
b/regression-test/data/external_table_p0/paimon/paimon_timestamp_types.out
index ddee5b68511..1645aae4969 100644
--- a/regression-test/data/external_table_p0/paimon/paimon_timestamp_types.out
+++ b/regression-test/data/external_table_p0/paimon/paimon_timestamp_types.out
@@ -5,6 +5,28 @@
-- !c2 --
1 2024-01-02T10:04:05.100 2024-01-02T10:04:05.120 2024-01-02T10:04:05.123
2024-01-02T10:04:05.123400 2024-01-02T10:04:05.123450
2024-01-02T10:04:05.123456 2024-01-02T10:04:05.123456
2024-01-02T10:04:05.123456 2024-01-02T10:04:05.123456
2024-01-02T10:04:05.100 2024-01-02T10:04:05.120 2024-01-02T10:04:05.123
2024-01-02T10:04:05.123400 2024-01-02T10:04:05.123450
2024-01-02T10:04:05.123456 2024-01-02T10:04:05.123456
2024-01-02T10:04:05.123456 2024-01-02T10:04:05.123456
+-- !ts9_eq_orc --
+1
+
+-- !ts9_eq_parquet --
+1
+
+-- !ts9_lt_orc --
+
+-- !ts9_le_orc --
+1
+
+-- !ts9_gt_parquet --
+
+-- !ts9_ge_parquet --
+1
+
+-- !ts9_in_orc --
+1
+
+-- !ts9_ne_parquet --
+1
+
-- !ltz_ntz_simple2 --
1 {"2024-01-01 10:12:34.123456":"2024-01-02 10:12:34.123456", "2024-01-03
10:12:34.123456":"2024-01-04 10:12:34.123456"} {"2024-01-03
10:12:34.123456":"2024-01-04 10:12:34.123456", "2024-01-01
10:12:34.123456":"2024-01-02 10:12:34.123456"} ["2024-01-01 10:12:34.123456",
"2024-01-02 10:12:34.123456", "2024-01-03 10:12:34.123456"] ["2024-01-01
10:12:34.123456", "2024-01-02 10:12:34.123456", "2024-01-03 10:12:34.123456"]
{"crow1":"2024-01-01 10:12:34.123456", "crow2":"2024-01-02 10:12:34.123456"}
@@ -46,13 +68,34 @@
-- !ltz_ntz_simple15 --
2024-01-02T10:12:34.123456
-
-- !c1 --
-1 2024-01-02T10:04:05.100 2024-01-02T10:04:05.120 2024-01-02T10:04:05.123
2024-01-02T10:04:05.123400 2024-01-02T10:04:05.123450
2024-01-02T10:04:05.123456 2024-01-02T10:04:05.123457
2024-01-02T10:04:05.123457 2024-01-02T10:04:05.123457
2024-01-02T02:04:05.100 2024-01-02T02:04:05.120 2024-01-02T02:04:05.123
2024-01-02T02:04:05.123400 2024-01-02T02:04:05.123450
2024-01-02T02:04:05.123456 2024-01-02T02:04:05.123457
2024-01-02T02:04:05.123457 2024-01-02T02:04:05.123457
+1 2024-01-02T10:04:05.100 2024-01-02T10:04:05.120 2024-01-02T10:04:05.123
2024-01-02T10:04:05.123400 2024-01-02T10:04:05.123450
2024-01-02T10:04:05.123456 2024-01-02T10:04:05.123456
2024-01-02T10:04:05.123456 2024-01-02T10:04:05.123456
2024-01-02T02:04:05.100 2024-01-02T02:04:05.120 2024-01-02T02:04:05.123
2024-01-02T02:04:05.123400 2024-01-02T02:04:05.123450
2024-01-02T02:04:05.123456 2024-01-02T02:04:05.123456
2024-01-02T02:04:05.123456 2024-01-02T02:04:05.123456
-- !c2 --
1 2024-01-02T10:04:05.100 2024-01-02T10:04:05.120 2024-01-02T10:04:05.123
2024-01-02T10:04:05.123400 2024-01-02T10:04:05.123450
2024-01-02T10:04:05.123456 2024-01-02T10:04:05.123456
2024-01-02T10:04:05.123456 2024-01-02T10:04:05.123456
2024-01-02T10:04:05.100 2024-01-02T10:04:05.120 2024-01-02T10:04:05.123
2024-01-02T10:04:05.123400 2024-01-02T10:04:05.123450
2024-01-02T10:04:05.123456 2024-01-02T10:04:05.123456
2024-01-02T10:04:05.123456 2024-01-02T10:04:05.123456
+-- !ts9_eq_orc --
+1
+
+-- !ts9_eq_parquet --
+1
+
+-- !ts9_lt_orc --
+
+-- !ts9_le_orc --
+1
+
+-- !ts9_gt_parquet --
+
+-- !ts9_ge_parquet --
+1
+
+-- !ts9_in_orc --
+1
+
+-- !ts9_ne_parquet --
+1
+
-- !ltz_ntz_simple2 --
1 {"2024-01-01 10:12:34.123456":"2024-01-02 10:12:34.123456", "2024-01-03
10:12:34.123456":"2024-01-04 10:12:34.123456"} {"2024-01-03
02:12:34.123456":"2024-01-04 02:12:34.123456", "2024-01-01
02:12:34.123456":"2024-01-02 02:12:34.123456"} ["2024-01-01 10:12:34.123456",
"2024-01-02 10:12:34.123456", "2024-01-03 10:12:34.123456"] ["2024-01-01
02:12:34.123456", "2024-01-02 02:12:34.123456", "2024-01-03 02:12:34.123456"]
{"crow1":"2024-01-01 10:12:34.123456", "crow2":"2024-01-02 02:12:34.123456"}
diff --git
a/regression-test/data/external_table_p0/paimon/test_paimon_minio.out
b/regression-test/data/external_table_p0/paimon/test_paimon_minio.out
index 78b3edaca1d..37f9f66619c 100644
--- a/regression-test/data/external_table_p0/paimon/test_paimon_minio.out
+++ b/regression-test/data/external_table_p0/paimon/test_paimon_minio.out
@@ -1,25 +1,25 @@
-- This file is automatically generated. You should know what you did if you
want to edit this
-- !no_region1 --
-2024-01-02T10:04:05.100 2024-01-02T02:04:05.123457
+2024-01-02T10:04:05.100 2024-01-02T02:04:05.123456
-- !no_region2 --
-1 2024-01-02T10:04:05.100 2024-01-02T10:04:05.120 2024-01-02T10:04:05.123
2024-01-02T10:04:05.123400 2024-01-02T10:04:05.123450
2024-01-02T10:04:05.123456 2024-01-02T10:04:05.123457
2024-01-02T10:04:05.123457 2024-01-02T10:04:05.123457
2024-01-02T02:04:05.100 2024-01-02T02:04:05.120 2024-01-02T02:04:05.123
2024-01-02T02:04:05.123400 2024-01-02T02:04:05.123450
2024-01-02T02:04:05.123456 2024-01-02T02:04:05.123457
2024-01-02T02:04:05.123457 2024-01-02T02:04:05.123457
+1 2024-01-02T10:04:05.100 2024-01-02T10:04:05.120 2024-01-02T10:04:05.123
2024-01-02T10:04:05.123400 2024-01-02T10:04:05.123450
2024-01-02T10:04:05.123456 2024-01-02T10:04:05.123456
2024-01-02T10:04:05.123456 2024-01-02T10:04:05.123456
2024-01-02T02:04:05.100 2024-01-02T02:04:05.120 2024-01-02T02:04:05.123
2024-01-02T02:04:05.123400 2024-01-02T02:04:05.123450
2024-01-02T02:04:05.123456 2024-01-02T02:04:05.123456
2024-01-02T02:04:05.123456 2024-01-02T02:04:05.123456
-- !region1 --
-2024-01-02T10:04:05.100 2024-01-02T02:04:05.123457
+2024-01-02T10:04:05.100 2024-01-02T02:04:05.123456
-- !region2 --
-1 2024-01-02T10:04:05.100 2024-01-02T10:04:05.120 2024-01-02T10:04:05.123
2024-01-02T10:04:05.123400 2024-01-02T10:04:05.123450
2024-01-02T10:04:05.123456 2024-01-02T10:04:05.123457
2024-01-02T10:04:05.123457 2024-01-02T10:04:05.123457
2024-01-02T02:04:05.100 2024-01-02T02:04:05.120 2024-01-02T02:04:05.123
2024-01-02T02:04:05.123400 2024-01-02T02:04:05.123450
2024-01-02T02:04:05.123456 2024-01-02T02:04:05.123457
2024-01-02T02:04:05.123457 2024-01-02T02:04:05.123457
+1 2024-01-02T10:04:05.100 2024-01-02T10:04:05.120 2024-01-02T10:04:05.123
2024-01-02T10:04:05.123400 2024-01-02T10:04:05.123450
2024-01-02T10:04:05.123456 2024-01-02T10:04:05.123456
2024-01-02T10:04:05.123456 2024-01-02T10:04:05.123456
2024-01-02T02:04:05.100 2024-01-02T02:04:05.120 2024-01-02T02:04:05.123
2024-01-02T02:04:05.123400 2024-01-02T02:04:05.123450
2024-01-02T02:04:05.123456 2024-01-02T02:04:05.123456
2024-01-02T02:04:05.123456 2024-01-02T02:04:05.123456
-- !no_region1 --
-2024-01-02T10:04:05.100 2024-01-02T02:04:05.123457
+2024-01-02T10:04:05.100 2024-01-02T02:04:05.123456
-- !no_region2 --
-1 2024-01-02T10:04:05.100 2024-01-02T10:04:05.120 2024-01-02T10:04:05.123
2024-01-02T10:04:05.123400 2024-01-02T10:04:05.123450
2024-01-02T10:04:05.123456 2024-01-02T10:04:05.123457
2024-01-02T10:04:05.123457 2024-01-02T10:04:05.123457
2024-01-02T02:04:05.100 2024-01-02T02:04:05.120 2024-01-02T02:04:05.123
2024-01-02T02:04:05.123400 2024-01-02T02:04:05.123450
2024-01-02T02:04:05.123456 2024-01-02T02:04:05.123457
2024-01-02T02:04:05.123457 2024-01-02T02:04:05.123457
+1 2024-01-02T10:04:05.100 2024-01-02T10:04:05.120 2024-01-02T10:04:05.123
2024-01-02T10:04:05.123400 2024-01-02T10:04:05.123450
2024-01-02T10:04:05.123456 2024-01-02T10:04:05.123456
2024-01-02T10:04:05.123456 2024-01-02T10:04:05.123456
2024-01-02T02:04:05.100 2024-01-02T02:04:05.120 2024-01-02T02:04:05.123
2024-01-02T02:04:05.123400 2024-01-02T02:04:05.123450
2024-01-02T02:04:05.123456 2024-01-02T02:04:05.123456
2024-01-02T02:04:05.123456 2024-01-02T02:04:05.123456
-- !region1 --
-2024-01-02T10:04:05.100 2024-01-02T02:04:05.123457
+2024-01-02T10:04:05.100 2024-01-02T02:04:05.123456
-- !region2 --
-1 2024-01-02T10:04:05.100 2024-01-02T10:04:05.120 2024-01-02T10:04:05.123
2024-01-02T10:04:05.123400 2024-01-02T10:04:05.123450
2024-01-02T10:04:05.123456 2024-01-02T10:04:05.123457
2024-01-02T10:04:05.123457 2024-01-02T10:04:05.123457
2024-01-02T02:04:05.100 2024-01-02T02:04:05.120 2024-01-02T02:04:05.123
2024-01-02T02:04:05.123400 2024-01-02T02:04:05.123450
2024-01-02T02:04:05.123456 2024-01-02T02:04:05.123457
2024-01-02T02:04:05.123457 2024-01-02T02:04:05.123457
+1 2024-01-02T10:04:05.100 2024-01-02T10:04:05.120 2024-01-02T10:04:05.123
2024-01-02T10:04:05.123400 2024-01-02T10:04:05.123450
2024-01-02T10:04:05.123456 2024-01-02T10:04:05.123456
2024-01-02T10:04:05.123456 2024-01-02T10:04:05.123456
2024-01-02T02:04:05.100 2024-01-02T02:04:05.120 2024-01-02T02:04:05.123
2024-01-02T02:04:05.123400 2024-01-02T02:04:05.123450
2024-01-02T02:04:05.123456 2024-01-02T02:04:05.123456
2024-01-02T02:04:05.123456 2024-01-02T02:04:05.123456
diff --git
a/regression-test/data/external_table_p0/tvf/orc_tvf/test_hdfs_orc_group4_orc_files.out
b/regression-test/data/external_table_p0/tvf/orc_tvf/test_hdfs_orc_group4_orc_files.out
index 855cae34668..1f5ea974148 100644
---
a/regression-test/data/external_table_p0/tvf/orc_tvf/test_hdfs_orc_group4_orc_files.out
+++
b/regression-test/data/external_table_p0/tvf/orc_tvf/test_hdfs_orc_group4_orc_files.out
@@ -1,6 +1,6 @@
-- This file is automatically generated. You should know what you did if you
want to edit this
-- !test_0 --
-2022-06-10T05:26:22.754 2022-06-09T21:26:22.754
+2022-06-10T05:26:22.753999 2022-06-09T21:26:22.753999
-- !test_1 --
1 John Doe
diff --git
a/regression-test/data/external_table_p0/tvf/orc_tvf/test_hdfs_orc_group6_orc_files.out
b/regression-test/data/external_table_p0/tvf/orc_tvf/test_hdfs_orc_group6_orc_files.out
index dcf3cdb63ef..ea26e348943 100644
---
a/regression-test/data/external_table_p0/tvf/orc_tvf/test_hdfs_orc_group6_orc_files.out
+++
b/regression-test/data/external_table_p0/tvf/orc_tvf/test_hdfs_orc_group6_orc_files.out
@@ -484,7 +484,7 @@ true 127 32767 2147483647
9223372036854775807 12335.6 1234567.8 jack rose titani
9 yuwgyvkrwntnbpheafsj
-- !test_29 --
-2 0 0 2 banana 4.0
+2 0 0 2 banana 4
-- !test_30 --
10
@@ -590,7 +590,7 @@ world world
1 2006-01-02T07:04:05
2 2006-01-02T07:04:05.900
3 2006-01-02T07:04:05.999999
-4 2006-01-02T07:04:06
+4 2006-01-02T07:04:05.999999
-- !test_40 --
1 [0.999999999]
diff --git
a/regression-test/suites/external_table_p0/paimon/paimon_timestamp_types.groovy
b/regression-test/suites/external_table_p0/paimon/paimon_timestamp_types.groovy
index fd0f31d04e1..7adeb3e5677 100644
---
a/regression-test/suites/external_table_p0/paimon/paimon_timestamp_types.groovy
+++
b/regression-test/suites/external_table_p0/paimon/paimon_timestamp_types.groovy
@@ -99,6 +99,50 @@ suite("paimon_timestamp_types", "p0,external") {
qt_c1 ts_scale_orc
qt_c2 ts_scale_parquet
+ // TIMESTAMP(7/8/9) values are exposed by Doris at microsecond
precision. Keep the
+ // predicates below on ts9 so file pruning must retain the row
whose source value has
+ // nanoseconds beyond the Doris-visible literal.
+ order_qt_ts9_eq_orc """
+ select id from ts_scale_orc
+ where ts9 = '2024-01-02 10:04:05.123456'
+ order by id
+ """
+ order_qt_ts9_eq_parquet """
+ select id from ts_scale_parquet
+ where ts9 = '2024-01-02 10:04:05.123456'
+ order by id
+ """
+ order_qt_ts9_lt_orc """
+ select id from ts_scale_orc
+ where ts9 < '2024-01-02 10:04:05.123456'
+ order by id
+ """
+ order_qt_ts9_le_orc """
+ select id from ts_scale_orc
+ where ts9 <= '2024-01-02 10:04:05.123456'
+ order by id
+ """
+ order_qt_ts9_gt_parquet """
+ select id from ts_scale_parquet
+ where ts9 > '2024-01-02 10:04:05.123456'
+ order by id
+ """
+ order_qt_ts9_ge_parquet """
+ select id from ts_scale_parquet
+ where ts9 >= '2024-01-02 10:04:05.123456'
+ order by id
+ """
+ order_qt_ts9_in_orc """
+ select id from ts_scale_orc
+ where ts9 in ('2024-01-02 10:04:05.123457', '2024-01-02
10:04:05.123456')
+ order by id
+ """
+ order_qt_ts9_ne_parquet """
+ select id from ts_scale_parquet
+ where ts9 != '2024-01-02 10:04:05.123457'
+ order by id
+ """
+
}
sql """set force_jni_scanner=true"""
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
