divjotarora commented on code in PR #3610:
URL: https://github.com/apache/parquet-java/pull/3610#discussion_r3586612422
##########
parquet-column/src/main/java/org/apache/parquet/schema/PrimitiveComparator.java:
##########
@@ -354,4 +355,47 @@ public String toString() {
return "BINARY_AS_FLOAT16_IEEE_754_TOTAL_ORDER_COMPARATOR";
}
};
+
+ /**
+ * Comparator for two timestamps encoded as INT96 (12-byte little-endian)
binary.
+ * Layout: first 8 bytes = nanoseconds within the day, last 4 bytes = Julian
day.
+ *
+ * Two-level comparison, matching the INT96 timestamp sort order:
+ * 1. Compare the last 4 bytes (Julian day) as a signed little-endian int32.
+ * 2. If equal, compare the first 8 bytes (nanos) as a signed little-endian
int64.
+ */
+ static final PrimitiveComparator<Binary>
BINARY_AS_INT96_TIMESTAMP_COMPARATOR = new BinaryComparator() {
+ private static final long NANOSECONDS_PER_DAY = 86_400_000_000_000L;
+
+ @Override
+ int compareBinary(Binary b1, Binary b2) {
+ if (b1.length() != 12 || b2.length() != 12) {
+ throw new IllegalArgumentException(
+ "INT96 binary length must be 12, got " + b1.length() + " and " +
b2.length());
+ }
+
+ ByteBuffer bb1 = b1.toByteBuffer().slice();
+ ByteBuffer bb2 = b2.toByteBuffer().slice();
+ bb1.order(ByteOrder.LITTLE_ENDIAN);
+ bb2.order(ByteOrder.LITTLE_ENDIAN);
+
+ int result = Integer.compare(bb1.getInt(8), bb2.getInt(8));
+ if (result != 0) return result;
+
+ long nanos1 = bb1.getLong(0);
+ long nanos2 = bb2.getLong(0);
+ if (nanos1 < 0 || nanos1 > NANOSECONDS_PER_DAY) {
+ throw new IllegalArgumentException("Invalid nanos value: " + nanos1);
Review Comment:
Done
##########
parquet-column/src/test/java/org/apache/parquet/schema/TestPrimitiveComparator.java:
##########
@@ -354,6 +356,51 @@ public void
testBinaryAsSignedIntegerComparatorWithEquals() {
}
}
+ private static Binary int96(int julianDay, long nanosOfDay) {
+ return new NanoTime(julianDay, nanosOfDay).toBinary();
+ }
+
+ @Test
+ public void testInt96TimestampComparator() {
+ Binary[] valuesInAscendingOrder = {
+ int96(Integer.MIN_VALUE, 0), // most negative julian day
+ int96(-1, 86_399_999_999_999L), // negative julian days sort before day 0
+ int96(0, 0), // start of the julian period
+ int96(0, 86_399_999_999_999L), // same day, later time of day
+ int96(2440000, 123L), // 1968-05-23T00:00:00.000000123, pre-epoch but
positive julian day
+ int96(2458850, 43_200_000_000_000L), // 2020-01-01T12:00:00
+ int96(2458881, 39_600_000_000_000L), // 2020-02-01T11:00:00, later day
even though earlier time of day
+ int96(2458881, 39_600_000_000_001L), // 2020-02-01T11:00:00.000000001,
nanos tie-break
+ int96(Integer.MAX_VALUE, 86_399_999_999_999L)
+ };
+
+ for (int i = 0; i < valuesInAscendingOrder.length; ++i) {
+ for (int j = 0; j < valuesInAscendingOrder.length; ++j) {
+ assertEquals(
+ "comparing value " + i + " to value " + j,
+ Integer.signum(Integer.compare(i, j)),
+ Integer.signum(BINARY_AS_INT96_TIMESTAMP_COMPARATOR.compare(
+ valuesInAscendingOrder[i], valuesInAscendingOrder[j])));
+ }
+ }
+ }
+
+ @Test
+ public void testInt96TimestampComparatorRejectsInvalidNanos() {
+ // Same Julian day so the comparator reaches the nanos validation instead
of
+ // returning early on the day comparison.
+ Binary valid = int96(0, 0);
+ for (long invalidNanos : new long[] {-1L, Long.MIN_VALUE,
86_400_000_000_001L, Long.MAX_VALUE}) {
+ Binary invalid = int96(0, invalidNanos);
+ try {
+ BINARY_AS_INT96_TIMESTAMP_COMPARATOR.compare(valid, invalid);
+ fail("Expected IllegalArgumentException for nanos=" + invalidNanos);
Review Comment:
Done
##########
parquet-hadoop/src/test/java/org/apache/parquet/format/converter/TestParquetMetadataConverter.java:
##########
@@ -1413,7 +1434,29 @@ public void testColumnOrders() throws IOException {
assertEquals(
ColumnOrder.typeDefined(),
columns.get(0).getPrimitiveType().columnOrder());
assertEquals(ColumnOrder.undefined(),
columns.get(1).getPrimitiveType().columnOrder());
- assertEquals(ColumnOrder.undefined(),
columns.get(2).getPrimitiveType().columnOrder());
+ assertEquals(ColumnOrder.int96TimestampOrder(),
columns.get(2).getPrimitiveType().columnOrder());
+ }
+
+ @Test
+ public void testLegacyINT96ColumnOrder() throws IOException {
Review Comment:
Sure, split this into `testUndefinedINT96ColumnOrder` and
`testTypeDefinedOrderINT96ColumnOrder` to test the two cases
##########
parquet-hadoop/src/test/java/org/apache/parquet/format/converter/TestParquetMetadataConverter.java:
##########
@@ -1258,10 +1266,12 @@ public void testV2StatsEqualMinMax() {
.named(""),
new BigInteger("-8752832"),
new BigInteger("-8752832"));
+ Binary int96 = new NanoTime(2458850, 43_200_000_000_000L)
+ .toBinary(); // 2020-01-01T12:00:00
Review Comment:
Sure, changed to just pass down `NanoTime` and modified
`createStats`/`createStatsTyped` to accept `NanoTime` rather than `Binary`. At
the bottom layer in `createStatsTyped`, we still call `toBinary()` to serialize
the `NanoTime` values, but the callers are cleaner now
##########
parquet-hadoop/src/main/java/org/apache/parquet/format/converter/ParquetMetadataConverter.java:
##########
@@ -911,8 +916,16 @@ private static byte[] tuncateMax(BinaryTruncator
truncator, int truncateLength,
}
private static boolean isMinMaxStatsSupported(PrimitiveType type) {
- return type.columnOrder().getColumnOrderName() ==
ColumnOrderName.TYPE_DEFINED_ORDER
- || type.columnOrder().getColumnOrderName() ==
ColumnOrderName.IEEE_754_TOTAL_ORDER;
+ switch (type.columnOrder().getColumnOrderName()) {
+ case TYPE_DEFINED_ORDER:
+ case IEEE_754_TOTAL_ORDER:
+ case INT96_TIMESTAMP_ORDER:
+ return true;
+ case UNDEFINED:
+ return false;
+ default:
+ throw new IllegalArgumentException("Unknown column order: " +
type.columnOrder());
Review Comment:
Prefer to leave `IllegalArgumentException` to match `getColumnOrders` above
##########
parquet-hadoop/src/test/java/org/apache/parquet/format/converter/TestParquetMetadataConverter.java:
##########
@@ -1281,6 +1291,8 @@ private static <T> Statistics<?>
createStats(PrimitiveType type, T min, T max) {
return createStatsTyped(type, (Long) min, (Long) max);
} else if (c == BigInteger.class) {
return createStatsTyped(type, (BigInteger) min, (BigInteger) max);
+ } else if (min instanceof Binary) {
+ return createStatsTyped(type, (Binary) min, (Binary) max);
Review Comment:
Changed this and added a `createStatsTyped` override for `NanoTime`
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
