voonhous commented on code in PR #8418:
URL: https://github.com/apache/hudi/pull/8418#discussion_r1165174172
##########
hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/io/storage/row/parquet/ParquetRowDataWriter.java:
##########
@@ -283,34 +291,66 @@ public void write(ArrayData array, int ordinal) {
}
}
- /**
- * Timestamp of INT96 bytes, julianDay(4) + nanosOfDay(8). See
- *
https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#timestamp
- * TIMESTAMP_MILLIS and TIMESTAMP_MICROS are the deprecated ConvertedType.
- */
private class Timestamp64Writer implements FieldWriter {
- private Timestamp64Writer() {
+
+ private final int precision;
+ private Timestamp64Writer(int precision) {
+ this.precision = precision;
}
@Override
public void write(RowData row, int ordinal) {
- recordConsumer.addLong(timestampToInt64(row.getTimestamp(ordinal, 3)));
+ TimestampData timestampData = row.getTimestamp(ordinal, precision);
+ recordConsumer.addLong(timestampToInt64(timestampData, precision));
}
@Override
public void write(ArrayData array, int ordinal) {
- recordConsumer.addLong(timestampToInt64(array.getTimestamp(ordinal, 3)));
+ TimestampData timestampData = array.getTimestamp(ordinal, precision);
+ recordConsumer.addLong(timestampToInt64(timestampData, precision));
}
}
- private long timestampToInt64(TimestampData timestampData) {
- return utcTimestamp ? timestampData.getMillisecond() :
timestampData.toTimestamp().getTime();
+ /**
+ * Converts a {@code TimestampData} to its corresponding int64 value. This
function only accepts TimestampData of
+ * precision 3 or 6. Special attention will need to be given to a
TimestampData of precision = 6.
+ * <p>
+ * For example representing `1970-01-01T00:00:03.100001` of precision 6 will
have:
+ * <ul>
+ * <li>millisecond = 3100</li>
+ * <li>nanoOfMillisecond = 1000</li>
+ * </ul>
+ * As such, the int64 value will be:
+ * <p>
+ * millisecond * 1000 + nanoOfMillisecond / 1000
+ *
+ * @param timestampData TimestampData to be converted to int64 format
+ * @param precision the precision of the TimestampData
+ * @return int64 value of the TimestampData
+ */
+ private long timestampToInt64(TimestampData timestampData, int precision) {
+ if (!utcTimestamp) {
+ // toTimestamp is agnostic of precision
+ return timestampData.toTimestamp().getTime();
+ } else if (precision == 3) {
+ return timestampData.getMillisecond();
+ } else if (precision == 6) {
+ // convert timestampDat ato microseconds format
+ return timestampData.getMillisecond() * 1000 +
timestampData.getNanoOfMillisecond() / 1000;
+ }
+
+ throw new IllegalArgumentException(
+ "Unable to convert TimestampData with precision: "
Review Comment:
This is a private method anyways, this `throw` is generally unreachable,
will invoke `ValidationUtils.checkArgument` in the `Timestamp64Writer`.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
