This is an automated email from the ASF dual-hosted git repository.
ipolyzos pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/fluss.git
The following commit(s) were added to refs/heads/main by this push:
new a956aa557 Fix timestamp precision (#1931)
a956aa557 is described below
commit a956aa5578f12908b8ae5161f6a876b1c0496cf5
Author: buvb <[email protected]>
AuthorDate: Wed Nov 5 15:15:46 2025 +0800
Fix timestamp precision (#1931)
* fix:Enhance the timestamp conversion function, support precision
parameters and add truncation methods
* fix:Precision test file
* [client][hotfix] Add Javadoc for TimestampPojo test class
---
.../fluss/client/converter/PojoToRowConverter.java | 99 +++++++++++--
.../client/converter/PojoToRowConverterTest.java | 158 +++++++++++++++++++++
2 files changed, 244 insertions(+), 13 deletions(-)
diff --git
a/fluss-client/src/main/java/org/apache/fluss/client/converter/PojoToRowConverter.java
b/fluss-client/src/main/java/org/apache/fluss/client/converter/PojoToRowConverter.java
index c2a3dca08..4d75aa91a 100644
---
a/fluss-client/src/main/java/org/apache/fluss/client/converter/PojoToRowConverter.java
+++
b/fluss-client/src/main/java/org/apache/fluss/client/converter/PojoToRowConverter.java
@@ -23,6 +23,7 @@ import org.apache.fluss.row.GenericRow;
import org.apache.fluss.row.TimestampLtz;
import org.apache.fluss.row.TimestampNtz;
import org.apache.fluss.types.DataType;
+import org.apache.fluss.types.DataTypeChecks;
import org.apache.fluss.types.DecimalType;
import org.apache.fluss.types.RowType;
@@ -134,9 +135,15 @@ public final class PojoToRowConverter<T> {
case TIME_WITHOUT_TIME_ZONE:
return (obj) -> convertTimeValue(prop, prop.read(obj));
case TIMESTAMP_WITHOUT_TIME_ZONE:
- return (obj) -> convertTimestampNtzValue(prop, prop.read(obj));
+ {
+ final int precision =
DataTypeChecks.getPrecision(fieldType);
+ return (obj) -> convertTimestampNtzValue(precision, prop,
prop.read(obj));
+ }
case TIMESTAMP_WITH_LOCAL_TIME_ZONE:
- return (obj) -> convertTimestampLtzValue(prop, prop.read(obj));
+ {
+ final int precision =
DataTypeChecks.getPrecision(fieldType);
+ return (obj) -> convertTimestampLtzValue(precision, prop,
prop.read(obj));
+ }
default:
throw new UnsupportedOperationException(
String.format(
@@ -202,9 +209,17 @@ public final class PojoToRowConverter<T> {
return (int) (t.toNanoOfDay() / 1_000_000);
}
- /** Converts a LocalDateTime POJO property to Fluss TimestampNtz. */
+ /**
+ * Converts a LocalDateTime POJO property to Fluss TimestampNtz,
respecting the specified
+ * precision.
+ *
+ * @param precision the timestamp precision (0-9)
+ * @param prop the POJO property metadata
+ * @param v the value to convert
+ * @return TimestampNtz with precision applied, or null if v is null
+ */
private static @Nullable TimestampNtz convertTimestampNtzValue(
- PojoType.Property prop, @Nullable Object v) {
+ int precision, PojoType.Property prop, @Nullable Object v) {
if (v == null) {
return null;
}
@@ -214,24 +229,82 @@ public final class PojoToRowConverter<T> {
"Field %s is not a LocalDateTime. Cannot convert
to TimestampNtz.",
prop.name));
}
- return TimestampNtz.fromLocalDateTime((LocalDateTime) v);
+ LocalDateTime ldt = (LocalDateTime) v;
+ LocalDateTime truncated = truncateToTimestampPrecision(ldt, precision);
+ return TimestampNtz.fromLocalDateTime(truncated);
}
- /** Converts an Instant or OffsetDateTime POJO property to Fluss
TimestampLtz (UTC based). */
+ /**
+ * Converts an Instant or OffsetDateTime POJO property to Fluss
TimestampLtz (UTC based),
+ * respecting the specified precision.
+ *
+ * @param precision the timestamp precision (0-9)
+ * @param prop the POJO property metadata
+ * @param v the value to convert
+ * @return TimestampLtz with precision applied, or null if v is null
+ */
private static @Nullable TimestampLtz convertTimestampLtzValue(
- PojoType.Property prop, @Nullable Object v) {
+ int precision, PojoType.Property prop, @Nullable Object v) {
if (v == null) {
return null;
}
+ Instant instant;
if (v instanceof Instant) {
- return TimestampLtz.fromInstant((Instant) v);
+ instant = (Instant) v;
} else if (v instanceof OffsetDateTime) {
- return TimestampLtz.fromInstant(((OffsetDateTime) v).toInstant());
+ instant = ((OffsetDateTime) v).toInstant();
+ } else {
+ throw new IllegalArgumentException(
+ String.format(
+ "Field %s is not an Instant or OffsetDateTime.
Cannot convert to TimestampLtz.",
+ prop.name));
+ }
+ Instant truncated = truncateToTimestampPrecision(instant, precision);
+ return TimestampLtz.fromInstant(truncated);
+ }
+
+ /**
+ * Truncates a LocalDateTime to the specified timestamp precision.
+ *
+ * @param ldt the LocalDateTime to truncate
+ * @param precision the precision (0-9)
+ * @return truncated LocalDateTime
+ */
+ private static LocalDateTime truncateToTimestampPrecision(LocalDateTime
ldt, int precision) {
+ if (precision >= 9) {
+ return ldt;
}
- throw new IllegalArgumentException(
- String.format(
- "Field %s is not an Instant or OffsetDateTime. Cannot
convert to TimestampLtz.",
- prop.name));
+ int nanos = ldt.getNano();
+ int truncatedNanos = truncateNanos(nanos, precision);
+ return ldt.withNano(truncatedNanos);
+ }
+
+ /**
+ * Truncates an Instant to the specified timestamp precision.
+ *
+ * @param instant the Instant to truncate
+ * @param precision the precision (0-9)
+ * @return truncated Instant
+ */
+ private static Instant truncateToTimestampPrecision(Instant instant, int
precision) {
+ if (precision >= 9) {
+ return instant;
+ }
+ int nanos = instant.getNano();
+ int truncatedNanos = truncateNanos(nanos, precision);
+ return Instant.ofEpochSecond(instant.getEpochSecond(), truncatedNanos);
+ }
+
+ /**
+ * Truncates nanoseconds to the specified precision.
+ *
+ * @param nanos the nanoseconds value (0-999,999,999)
+ * @param precision the precision (0-9)
+ * @return truncated nanoseconds
+ */
+ private static int truncateNanos(int nanos, int precision) {
+ int divisor = (int) Math.pow(10, 9 - precision);
+ return (nanos / divisor) * divisor;
}
private interface FieldToRow {
diff --git
a/fluss-client/src/test/java/org/apache/fluss/client/converter/PojoToRowConverterTest.java
b/fluss-client/src/test/java/org/apache/fluss/client/converter/PojoToRowConverterTest.java
index 11deb35ac..4703c811c 100644
---
a/fluss-client/src/test/java/org/apache/fluss/client/converter/PojoToRowConverterTest.java
+++
b/fluss-client/src/test/java/org/apache/fluss/client/converter/PojoToRowConverterTest.java
@@ -18,11 +18,16 @@
package org.apache.fluss.client.converter;
import org.apache.fluss.row.GenericRow;
+import org.apache.fluss.row.TimestampLtz;
+import org.apache.fluss.row.TimestampNtz;
import org.apache.fluss.types.DataTypes;
import org.apache.fluss.types.RowType;
import org.junit.jupiter.api.Test;
+import java.time.Instant;
+import java.time.LocalDateTime;
+
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
@@ -168,4 +173,157 @@ public class PojoToRowConverterTest {
.hasMessageContaining("MAP")
.hasMessageContaining("mapField");
}
+
+ @Test
+ public void testTimestampPrecision3() {
+ // Test with precision 3 milliseconds
+ RowType table =
+ RowType.builder()
+ .field("timestampNtzField", DataTypes.TIMESTAMP(3))
+ .field("timestampLtzField", DataTypes.TIMESTAMP_LTZ(3))
+ .build();
+
+ PojoToRowConverter<TimestampPojo> writer =
+ PojoToRowConverter.of(TimestampPojo.class, table, table);
+
+ // 123.456789
+ LocalDateTime ldt = LocalDateTime.of(2025, 7, 23, 15, 1, 30,
123456789);
+ Instant instant = Instant.parse("2025-07-23T15:01:30.123456789Z");
+
+ TimestampPojo pojo = new TimestampPojo(ldt, instant);
+ GenericRow row = writer.toRow(pojo);
+
+ // truncate to 123.000000
+ TimestampNtz expectedNtz =
TimestampNtz.fromLocalDateTime(ldt.withNano(123000000));
+ TimestampLtz expectedLtz =
+ TimestampLtz.fromInstant(
+ Instant.ofEpochSecond(instant.getEpochSecond(),
123000000));
+
+ assertThat(row.getTimestampNtz(0, 3)).isEqualTo(expectedNtz);
+ assertThat(row.getTimestampLtz(1, 3)).isEqualTo(expectedLtz);
+ }
+
+ @Test
+ public void testTimestampPrecision6() {
+ // Test with precision 6 microseconds
+ RowType table =
+ RowType.builder()
+ .field("timestampNtzField", DataTypes.TIMESTAMP(6))
+ .field("timestampLtzField", DataTypes.TIMESTAMP_LTZ(6))
+ .build();
+
+ PojoToRowConverter<TimestampPojo> writer =
+ PojoToRowConverter.of(TimestampPojo.class, table, table);
+
+ // 123.456789
+ LocalDateTime ldt = LocalDateTime.of(2025, 7, 23, 15, 1, 30,
123456789);
+ Instant instant = Instant.parse("2025-07-23T15:01:30.123456789Z");
+
+ TimestampPojo pojo = new TimestampPojo(ldt, instant);
+ GenericRow row = writer.toRow(pojo);
+
+ // truncate to 123.456000
+ TimestampNtz expectedNtz =
TimestampNtz.fromLocalDateTime(ldt.withNano(123456000));
+ TimestampLtz expectedLtz =
+ TimestampLtz.fromInstant(
+ Instant.ofEpochSecond(instant.getEpochSecond(),
123456000));
+
+ assertThat(row.getTimestampNtz(0, 6)).isEqualTo(expectedNtz);
+ assertThat(row.getTimestampLtz(1, 6)).isEqualTo(expectedLtz);
+ }
+
+ @Test
+ public void testTimestampPrecision9() {
+ // Test with precision 9 nanoseconds
+ RowType table =
+ RowType.builder()
+ .field("timestampNtzField", DataTypes.TIMESTAMP(9))
+ .field("timestampLtzField", DataTypes.TIMESTAMP_LTZ(9))
+ .build();
+
+ PojoToRowConverter<TimestampPojo> writer =
+ PojoToRowConverter.of(TimestampPojo.class, table, table);
+
+ LocalDateTime ldt = LocalDateTime.of(2025, 7, 23, 15, 1, 30,
123456789);
+ Instant instant = Instant.parse("2025-07-23T15:01:30.123456789Z");
+
+ TimestampPojo pojo = new TimestampPojo(ldt, instant);
+ GenericRow row = writer.toRow(pojo);
+
+ TimestampNtz expectedNtz = TimestampNtz.fromLocalDateTime(ldt);
+ TimestampLtz expectedLtz = TimestampLtz.fromInstant(instant);
+
+ assertThat(row.getTimestampNtz(0, 9)).isEqualTo(expectedNtz);
+ assertThat(row.getTimestampLtz(1, 9)).isEqualTo(expectedLtz);
+ }
+
+ @Test
+ public void testTimestampPrecisionRoundTrip() {
+ testRoundTripWithPrecision(3);
+ testRoundTripWithPrecision(6);
+ testRoundTripWithPrecision(9);
+ }
+
+ private void testRoundTripWithPrecision(int precision) {
+ RowType table =
+ RowType.builder()
+ .field("timestampNtzField",
DataTypes.TIMESTAMP(precision))
+ .field("timestampLtzField",
DataTypes.TIMESTAMP_LTZ(precision))
+ .build();
+
+ PojoToRowConverter<TimestampPojo> writer =
+ PojoToRowConverter.of(TimestampPojo.class, table, table);
+ RowToPojoConverter<TimestampPojo> reader =
+ RowToPojoConverter.of(TimestampPojo.class, table, table);
+
+ LocalDateTime originalLdt = LocalDateTime.of(2025, 7, 23, 15, 1, 30,
123456789);
+ Instant originalInstant =
Instant.parse("2025-07-23T15:01:30.123456789Z");
+
+ TimestampPojo originalPojo = new TimestampPojo(originalLdt,
originalInstant);
+
+ // Convert POJO -> Row -> POJO
+ GenericRow row = writer.toRow(originalPojo);
+ TimestampPojo resultPojo = reader.fromRow(row);
+
+ LocalDateTime expectedLdt = truncateLocalDateTime(originalLdt,
precision);
+ Instant expectedInstant = truncateInstant(originalInstant, precision);
+
+ assertThat(resultPojo.timestampNtzField)
+ .as("Round-trip LocalDateTime with precision %d", precision)
+ .isEqualTo(expectedLdt);
+ assertThat(resultPojo.timestampLtzField)
+ .as("Round-trip Instant with precision %d", precision)
+ .isEqualTo(expectedInstant);
+ }
+
+ private LocalDateTime truncateLocalDateTime(LocalDateTime ldt, int
precision) {
+ if (precision >= 9) {
+ return ldt;
+ }
+ int divisor = (int) Math.pow(10, 9 - precision);
+ int truncatedNanos = (ldt.getNano() / divisor) * divisor;
+ return ldt.withNano(truncatedNanos);
+ }
+
+ private Instant truncateInstant(Instant instant, int precision) {
+ if (precision >= 9) {
+ return instant;
+ }
+ int divisor = (int) Math.pow(10, 9 - precision);
+ int truncatedNanos = (instant.getNano() / divisor) * divisor;
+ return Instant.ofEpochSecond(instant.getEpochSecond(), truncatedNanos);
+ }
+
+ /** POJO for testing timestamp precision. */
+ public static class TimestampPojo {
+ public LocalDateTime timestampNtzField;
+ public Instant timestampLtzField;
+
+ public TimestampPojo() {}
+
+ public TimestampPojo(LocalDateTime timestampNtzField, Instant
timestampLtzField) {
+ this.timestampNtzField = timestampNtzField;
+ this.timestampLtzField = timestampLtzField;
+ }
+ }
}
