This is an automated email from the ASF dual-hosted git repository.
Gabriel39 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new ecae3c4a4a6 [fix](iceberg) Support fractional timestamps in time
travel (#67705)
ecae3c4a4a6 is described below
commit ecae3c4a4a6c7c6503b43ff3b4679cb22a5b4641
Author: Gabriel <[email protected]>
AuthorDate: Thu Sep 10 11:20:22 2026 +0800
[fix](iceberg) Support fractional timestamps in time travel (#67705)
### What problem does this PR solve?
Iceberg `FOR TIME AS OF` rejected timestamps containing fractional
seconds, even though the snapshots metadata exposes `committed_at` with
fractional precision. Truncating the value to whole seconds can also
select the wrong snapshot when multiple snapshots are committed within
the same second.
### What is changed and how does it work?
- Accept an optional 1-9 digit fractional-second component in Iceberg
time-travel timestamp literals while preserving whole-second
compatibility.
- Keep session time-zone interpretation unchanged.
- Add unit coverage for millisecond and microsecond-formatted literals.
- Update the Iceberg time-travel regression matrix to feed the
fractional `committed_at` value back into `FOR TIME AS OF` directly.
### Check List
- [x] Unit test: `IcebergTimeUtilsTest` (6 tests)
- [x] FE Checkstyle
- [x] `git diff --check`
---
.../doris/connector/iceberg/IcebergTimeUtils.java | 22 ++++++++++++++--------
.../connector/iceberg/IcebergTimeUtilsTest.java | 9 +++++++++
.../test_iceberg_schema_time_travel_matrix.groovy | 4 ++--
3 files changed, 25 insertions(+), 10 deletions(-)
diff --git
a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergTimeUtils.java
b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergTimeUtils.java
index 34f4f8c8d79..fe561834eb1 100644
---
a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergTimeUtils.java
+++
b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergTimeUtils.java
@@ -24,7 +24,9 @@ import java.time.LocalDateTime;
import java.time.ZoneId;
import java.time.ZoneOffset;
import java.time.format.DateTimeFormatter;
+import java.time.format.DateTimeFormatterBuilder;
import java.time.format.DateTimeParseException;
+import java.time.temporal.ChronoField;
import java.util.Collections;
import java.util.Map;
import java.util.TreeMap;
@@ -44,9 +46,14 @@ public final class IcebergTimeUtils {
// Doris overrides (CST/PRC -> Asia/Shanghai, UTC/GMT -> UTC = TimeUtils
DEFAULT/UTC_TIME_ZONE).
private static final Map<String, String> TIME_ZONE_ALIAS_MAP;
- // Byte-parity with legacy TimeUtils.DATETIME_FORMAT (=
DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss")),
- // the formatter TimeUtils.timeStringToLong uses for a non-digital FOR
TIME AS OF datetime string.
- private static final DateTimeFormatter DATETIME_FORMAT =
DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss");
+ // Iceberg's committed_at value may include fractional seconds, so the
time-travel parser must round-trip
+ // that value while preserving compatibility with existing whole-second
literals.
+ private static final DateTimeFormatter DATETIME_FORMAT = new
DateTimeFormatterBuilder()
+ .appendPattern("yyyy-MM-dd HH:mm:ss")
+ .optionalStart()
+ .appendFraction(ChronoField.NANO_OF_SECOND, 1, 9, true)
+ .optionalEnd()
+ .toFormatter();
// Byte-parity with legacy TimeUtils.DATETIME_MS_FORMAT (=
DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss.SSS")),
// the formatter TimeUtils.msTimeStringToLong uses. The
rollback_to_timestamp EXECUTE action parses its
@@ -88,11 +95,10 @@ public final class IcebergTimeUtils {
}
/**
- * Parses a {@code FOR TIME AS OF} datetime string to epoch-millis in
{@code zone}, byte-faithful to legacy
- * {@code TimeUtils.timeStringToLong(value, sessionTZ)} (parse {@code
yyyy-MM-dd HH:mm:ss} as a local
- * date-time, then interpret it in the session zone). Legacy returned
{@code -1} on a parse failure and the
- * caller ({@code IcebergUtils.getQuerySpecSnapshot}) turned that into a
{@code DateTimeException}; we throw
- * it directly (fail loud — a parse error is a user mistake, not a
not-found).
+ * Parses a {@code FOR TIME AS OF} datetime string with optional
fractional seconds to epoch-millis in
+ * {@code zone}. Legacy returned {@code -1} on a parse failure and the
caller
+ * ({@code IcebergUtils.getQuerySpecSnapshot}) turned that into a {@code
DateTimeException}; we throw it
+ * directly (fail loud — a parse error is a user mistake, not a not-found).
*/
static long datetimeToMillis(String value, ZoneId zone) {
try {
diff --git
a/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergTimeUtilsTest.java
b/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergTimeUtilsTest.java
index 86a92fb733a..c5461f85807 100644
---
a/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergTimeUtilsTest.java
+++
b/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergTimeUtilsTest.java
@@ -52,6 +52,15 @@ public class IcebergTimeUtilsTest {
IcebergTimeUtils.datetimeToMillis("2023-06-15 10:30:00",
ZoneOffset.UTC));
}
+ @Test
+ public void datetimeToMillisAcceptsFractionalSeconds() {
+ long expected =
Instant.parse("2023-06-15T10:30:00.526Z").toEpochMilli();
+ Assertions.assertEquals(expected,
+ IcebergTimeUtils.datetimeToMillis("2023-06-15 10:30:00.526",
ZoneOffset.UTC));
+ Assertions.assertEquals(expected,
+ IcebergTimeUtils.datetimeToMillis("2023-06-15
10:30:00.526000", ZoneOffset.UTC));
+ }
+
@Test
public void datetimeToMillisFailsLoudOnMalformedString() {
// WHY: a malformed datetime is a user mistake; legacy returned -1 and
the caller threw
diff --git
a/regression-test/suites/external_table_p0/iceberg/test_iceberg_schema_time_travel_matrix.groovy
b/regression-test/suites/external_table_p0/iceberg/test_iceberg_schema_time_travel_matrix.groovy
index b6fa84dc9bf..dffbc236ea7 100644
---
a/regression-test/suites/external_table_p0/iceberg/test_iceberg_schema_time_travel_matrix.groovy
+++
b/regression-test/suites/external_table_p0/iceberg/test_iceberg_schema_time_travel_matrix.groovy
@@ -457,7 +457,7 @@ suite("test_iceberg_schema_time_travel_matrix",
// Scenario TC02/T03/T04: complex-field time travel uses the
pre-change nested schema.
List<List<Object>> dorisNestedCp0Time = sql("""
- select date_format(date_add(committed_at, interval 1 second),
'%Y-%m-%d %H:%i:%s'),
+ select date_format(committed_at, '%Y-%m-%d %H:%i:%s.%f'),
cast(unix_timestamp(committed_at) * 1000 + 999 as bigint)
from ${dorisNestedTable}\$snapshots
where snapshot_id = ${dorisNestedCp0}
@@ -561,7 +561,7 @@ suite("test_iceberg_schema_time_travel_matrix",
// Scenario T03/T04: string and numeric epoch-millis time travel both
resolve the snapshot.
List<List<Object>> cp0TimeRows = sql("""
- select date_format(date_add(committed_at, interval 1 second),
'%Y-%m-%d %H:%i:%s'),
+ select date_format(committed_at, '%Y-%m-%d %H:%i:%s.%f'),
cast(unix_timestamp(committed_at) * 1000 + 999 as bigint)
from ${topTable}\$snapshots
where snapshot_id = ${topCp0}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
