This is an automated email from the ASF dual-hosted git repository.
Fokko pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/parquet-java.git
The following commit(s) were added to refs/heads/master by this push:
new efec55565 GH-3679: Allow TimestampType to annotate FLBA(12) (#3680)
efec55565 is described below
commit efec5556522052b67090f842594bc78fc0a3e330
Author: Divjot Arora <[email protected]>
AuthorDate: Tue Sep 22 09:19:57 2026 +0200
GH-3679: Allow TimestampType to annotate FLBA(12) (#3680)
* Allow TimestampType to annotate FLBA(12)
* address PR comments
* address comments
* Add InterOp read test for FLBA(12) TIMESTAMP file
Co-authored-by: Isaac
* Fix FLBA timestamp tests
* Update timestamp validation assertions
---
.../parquet/arrow/schema/TestSchemaConverter.java | 4 +-
.../apache/parquet/schema/PrimitiveComparator.java | 29 ++++
.../parquet/schema/PrimitiveStringifier.java | 83 +++++++++-
.../org/apache/parquet/schema/PrimitiveType.java | 6 +
.../main/java/org/apache/parquet/schema/Types.java | 22 +--
.../parquet/column/statistics/TestStatistics.java | 28 ++++
.../column/columnindex/TestBinaryTruncator.java | 13 ++
.../column/columnindex/TestColumnIndexBuilder.java | 47 ++++++
.../parquet/schema/TestPrimitiveComparator.java | 42 ++++++
.../parquet/schema/TestPrimitiveStringifier.java | 72 ++++++++-
.../apache/parquet/schema/TestTypeBuilders.java | 46 +++++-
.../schema/TestTypeBuildersWithLogicalTypes.java | 50 +++++-
.../java/org/apache/parquet/bytes/BytesUtils.java | 8 +
.../format/converter/ParquetMetadataConverter.java | 9 +-
.../converter/TestParquetMetadataConverter.java | 25 +++
.../hadoop/TestInterOpReadFlba12Timestamp.java | 167 +++++++++++++++++++++
.../parquet/statistics/TestColumnIndexes.java | 6 +
.../apache/parquet/statistics/TestStatistics.java | 10 +-
18 files changed, 635 insertions(+), 32 deletions(-)
diff --git
a/parquet-arrow/src/test/java/org/apache/parquet/arrow/schema/TestSchemaConverter.java
b/parquet-arrow/src/test/java/org/apache/parquet/arrow/schema/TestSchemaConverter.java
index 04c359d2f..4d8a21a8d 100644
---
a/parquet-arrow/src/test/java/org/apache/parquet/arrow/schema/TestSchemaConverter.java
+++
b/parquet-arrow/src/test/java/org/apache/parquet/arrow/schema/TestSchemaConverter.java
@@ -649,7 +649,7 @@ public class TestSchemaConverter {
.named("a"))
.named("root")))
.isInstanceOf(IllegalStateException.class)
- .hasMessage("TIMESTAMP(MILLIS,false) can only annotate [INT64]");
+ .hasMessage("TIMESTAMP(MILLIS,false) can only annotate [INT64,
FIXED_LEN_BYTE_ARRAY(12)]");
}
@Test
@@ -660,6 +660,6 @@ public class TestSchemaConverter {
.named("a"))
.named("root")))
.isInstanceOf(IllegalStateException.class)
- .hasMessage("TIMESTAMP(MICROS,false) can only annotate [INT64]");
+ .hasMessage("TIMESTAMP(MICROS,false) can only annotate [INT64,
FIXED_LEN_BYTE_ARRAY(12)]");
}
}
diff --git
a/parquet-column/src/main/java/org/apache/parquet/schema/PrimitiveComparator.java
b/parquet-column/src/main/java/org/apache/parquet/schema/PrimitiveComparator.java
index 9d22d2531..8ef548a99 100644
---
a/parquet-column/src/main/java/org/apache/parquet/schema/PrimitiveComparator.java
+++
b/parquet-column/src/main/java/org/apache/parquet/schema/PrimitiveComparator.java
@@ -20,6 +20,7 @@ package org.apache.parquet.schema;
import java.io.Serializable;
import java.nio.ByteBuffer;
+import java.nio.ByteOrder;
import java.util.Comparator;
import org.apache.parquet.io.api.Binary;
@@ -278,6 +279,34 @@ public abstract class PrimitiveComparator<T> implements
Comparator<T>, Serializa
}
};
+ /**
+ * Comparator for timestamps encoded as 12-byte little-endian
two's-complement values.
+ */
+ static final PrimitiveComparator<Binary>
BINARY_AS_SIGNED_TIMESTAMP_COMPARATOR = new BinaryComparator() {
+ @Override
+ int compareBinary(Binary b1, Binary b2) {
+ if (b1.length() != 12 || b2.length() != 12) {
+ throw new IllegalArgumentException(
+ "Timestamp binary length must be 12 bytes, got " + b1.length() + "
and " + b2.length());
+ }
+ ByteBuffer bb1 = b1.toByteBuffer().slice();
+ bb1.order(ByteOrder.LITTLE_ENDIAN);
+ ByteBuffer bb2 = b2.toByteBuffer().slice();
+ bb2.order(ByteOrder.LITTLE_ENDIAN);
+ // Signed comparison of the high 4 bytes followed by unsigned comparison
of the low 8 bytes.
+ int hiResult = Integer.compare(bb1.getInt(8), bb2.getInt(8));
+ if (hiResult != 0) {
+ return hiResult;
+ }
+ return Long.compareUnsigned(bb1.getLong(0), bb2.getLong(0));
+ }
+
+ @Override
+ public String toString() {
+ return "BINARY_AS_SIGNED_TIMESTAMP_COMPARATOR";
+ }
+ };
+
/**
* This comparator is for comparing two float16 values represented in 2
bytes binary.
*/
diff --git
a/parquet-column/src/main/java/org/apache/parquet/schema/PrimitiveStringifier.java
b/parquet-column/src/main/java/org/apache/parquet/schema/PrimitiveStringifier.java
index 800078112..aa5b1eab5 100644
---
a/parquet-column/src/main/java/org/apache/parquet/schema/PrimitiveStringifier.java
+++
b/parquet-column/src/main/java/org/apache/parquet/schema/PrimitiveStringifier.java
@@ -29,10 +29,12 @@ import java.math.BigDecimal;
import java.math.BigInteger;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
+import java.time.DateTimeException;
import java.time.Instant;
import java.time.ZoneOffset;
import java.time.format.DateTimeFormatter;
import java.util.concurrent.TimeUnit;
+import org.apache.parquet.bytes.BytesUtils;
import org.apache.parquet.io.api.Binary;
import org.locationtech.jts.geom.Geometry;
import org.locationtech.jts.io.ParseException;
@@ -249,7 +251,7 @@ public abstract class PrimitiveStringifier {
return toFormattedString(getInstant(value));
}
- private String toFormattedString(Instant instant) {
+ final String toFormattedString(Instant instant) {
return formatter.format(instant);
}
@@ -266,6 +268,33 @@ public abstract class PrimitiveStringifier {
}
}
+ /**
+ * Stringifier implementation for timestamps that handles both int64 and
FLBA(12) carriers.
+ * Values outside of Instant's supported range render as raw integers rather
than human-readable
+ * timestamps.
+ */
+ private abstract static class TimestampStringifier extends DateStringifier {
+ private TimestampStringifier(String name, String format) {
+ super(name, format);
+ }
+
+ @Override
+ public String stringify(Binary value) {
+ if (value == null) {
+ return BINARY_NULL;
+ }
+
+ BigInteger units = new
BigInteger(BytesUtils.reverse(value.getBytesUnsafe()));
+ try {
+ return toFormattedString(getInstant(units));
+ } catch (ArithmeticException | DateTimeException e) {
+ return units.toString();
+ }
+ }
+
+ abstract Instant getInstant(BigInteger value);
+ }
+
static final PrimitiveStringifier DATE_STRINGIFIER = new
DateStringifier("DATE_STRINGIFIER", "yyyy-MM-dd") {
@Override
Instant getInstant(int value) {
@@ -274,56 +303,96 @@ public abstract class PrimitiveStringifier {
;
};
+ // Converts a count of time units since epoch, held as a BigInteger (the
96-bit FLBA(12) carrier),
+ // into an Instant. Throws ArithmeticException if the seconds portion
overflows long. Throws
+ // DateTimeException if the combined seconds/nanos portion does not fit in
an Instant.
+ private static Instant instantFromUnits(BigInteger units, TimeUnit unit) {
+ BigInteger[] secondsAndUnits =
units.divideAndRemainder(BigInteger.valueOf(unit.convert(1, SECONDS)));
+ long seconds = secondsAndUnits[0].longValueExact();
+ long nanos = secondsAndUnits[1].longValue() * unit.toNanos(1);
+ return Instant.ofEpochSecond(seconds, nanos);
+ }
+
static final PrimitiveStringifier TIMESTAMP_MILLIS_STRINGIFIER =
- new DateStringifier("TIMESTAMP_MILLIS_STRINGIFIER",
"yyyy-MM-dd'T'HH:mm:ss.SSS") {
+ new TimestampStringifier("TIMESTAMP_MILLIS_STRINGIFIER",
"yyyy-MM-dd'T'HH:mm:ss.SSS") {
@Override
Instant getInstant(long value) {
return Instant.ofEpochMilli(value);
}
+
+ @Override
+ Instant getInstant(BigInteger value) {
+ return instantFromUnits(value, MILLISECONDS);
+ }
};
static final PrimitiveStringifier TIMESTAMP_MICROS_STRINGIFIER =
- new DateStringifier("TIMESTAMP_MICROS_STRINGIFIER",
"yyyy-MM-dd'T'HH:mm:ss.SSSSSS") {
+ new TimestampStringifier("TIMESTAMP_MICROS_STRINGIFIER",
"yyyy-MM-dd'T'HH:mm:ss.SSSSSS") {
@Override
Instant getInstant(long value) {
return Instant.ofEpochSecond(
MICROSECONDS.toSeconds(value), MICROSECONDS.toNanos(value %
SECONDS.toMicros(1)));
}
+
+ @Override
+ Instant getInstant(BigInteger value) {
+ return instantFromUnits(value, MICROSECONDS);
+ }
};
static final PrimitiveStringifier TIMESTAMP_NANOS_STRINGIFIER =
- new DateStringifier("TIMESTAMP_NANOS_STRINGIFIER",
"yyyy-MM-dd'T'HH:mm:ss.SSSSSSSSS") {
+ new TimestampStringifier("TIMESTAMP_NANOS_STRINGIFIER",
"yyyy-MM-dd'T'HH:mm:ss.SSSSSSSSS") {
@Override
Instant getInstant(long value) {
return Instant.ofEpochSecond(
NANOSECONDS.toSeconds(value), NANOSECONDS.toNanos(value %
SECONDS.toNanos(1)));
}
+
+ @Override
+ Instant getInstant(BigInteger value) {
+ return instantFromUnits(value, NANOSECONDS);
+ }
};
static final PrimitiveStringifier TIMESTAMP_MILLIS_UTC_STRINGIFIER =
- new DateStringifier("TIMESTAMP_MILLIS_UTC_STRINGIFIER",
"yyyy-MM-dd'T'HH:mm:ss.SSSZ") {
+ new TimestampStringifier("TIMESTAMP_MILLIS_UTC_STRINGIFIER",
"yyyy-MM-dd'T'HH:mm:ss.SSSZ") {
@Override
Instant getInstant(long value) {
return Instant.ofEpochMilli(value);
}
+
+ @Override
+ Instant getInstant(BigInteger value) {
+ return instantFromUnits(value, MILLISECONDS);
+ }
};
static final PrimitiveStringifier TIMESTAMP_MICROS_UTC_STRINGIFIER =
- new DateStringifier("TIMESTAMP_MICROS_UTC_STRINGIFIER",
"yyyy-MM-dd'T'HH:mm:ss.SSSSSSZ") {
+ new TimestampStringifier("TIMESTAMP_MICROS_UTC_STRINGIFIER",
"yyyy-MM-dd'T'HH:mm:ss.SSSSSSZ") {
@Override
Instant getInstant(long value) {
return Instant.ofEpochSecond(
MICROSECONDS.toSeconds(value), MICROSECONDS.toNanos(value %
SECONDS.toMicros(1)));
}
+
+ @Override
+ Instant getInstant(BigInteger value) {
+ return instantFromUnits(value, MICROSECONDS);
+ }
};
static final PrimitiveStringifier TIMESTAMP_NANOS_UTC_STRINGIFIER =
- new DateStringifier("TIMESTAMP_NANOS_UTC_STRINGIFIER",
"yyyy-MM-dd'T'HH:mm:ss.SSSSSSSSSZ") {
+ new TimestampStringifier("TIMESTAMP_NANOS_UTC_STRINGIFIER",
"yyyy-MM-dd'T'HH:mm:ss.SSSSSSSSSZ") {
@Override
Instant getInstant(long value) {
return Instant.ofEpochSecond(
NANOSECONDS.toSeconds(value), NANOSECONDS.toNanos(value %
SECONDS.toNanos(1)));
}
+
+ @Override
+ Instant getInstant(BigInteger value) {
+ return instantFromUnits(value, NANOSECONDS);
+ }
};
private abstract static class TimeStringifier extends PrimitiveStringifier {
diff --git
a/parquet-column/src/main/java/org/apache/parquet/schema/PrimitiveType.java
b/parquet-column/src/main/java/org/apache/parquet/schema/PrimitiveType.java
index e04c54a7d..c359c9f41 100644
--- a/parquet-column/src/main/java/org/apache/parquet/schema/PrimitiveType.java
+++ b/parquet-column/src/main/java/org/apache/parquet/schema/PrimitiveType.java
@@ -444,6 +444,12 @@ public final class PrimitiveType extends Type {
: of(PrimitiveComparator.BINARY_AS_FLOAT16_COMPARATOR);
}
+ @Override
+ public Optional<PrimitiveComparator> visit(
+ LogicalTypeAnnotation.TimestampLogicalTypeAnnotation
timestampLogicalType) {
+ return
of(PrimitiveComparator.BINARY_AS_SIGNED_TIMESTAMP_COMPARATOR);
+ }
+
@Override
public Optional<PrimitiveComparator> visit(
LogicalTypeAnnotation.UnknownLogicalTypeAnnotation
unknownLogicalTypeAnnotation) {
diff --git a/parquet-column/src/main/java/org/apache/parquet/schema/Types.java
b/parquet-column/src/main/java/org/apache/parquet/schema/Types.java
index 7b02a6f02..2d6f0cbf8 100644
--- a/parquet-column/src/main/java/org/apache/parquet/schema/Types.java
+++ b/parquet-column/src/main/java/org/apache/parquet/schema/Types.java
@@ -417,7 +417,7 @@ public class Types {
@Override
public Optional<AllowedPhysicalTypes> visit(
LogicalTypeAnnotation.TimestampLogicalTypeAnnotation
timestampLogicalType) {
- return AllowedPhysicalTypes.of(PrimitiveTypeName.INT64);
+ return AllowedPhysicalTypes.fixed(12, PrimitiveTypeName.INT64);
}
@Override
@@ -463,13 +463,16 @@ public class Types {
return Optional.of(new AllowedPhysicalTypes(EnumSet.of(first, rest),
NOT_SET));
}
- private static Optional<AllowedPhysicalTypes> fixed(int requiredLength) {
- return Optional.of(
- new
AllowedPhysicalTypes(EnumSet.of(PrimitiveTypeName.FIXED_LEN_BYTE_ARRAY),
requiredLength));
+ private static Optional<AllowedPhysicalTypes> fixed(int requiredLength,
PrimitiveTypeName... otherTypes) {
+ return Optional.of(new AllowedPhysicalTypes(
+ EnumSet.of(PrimitiveTypeName.FIXED_LEN_BYTE_ARRAY, otherTypes),
requiredLength));
}
private boolean accepts(PrimitiveTypeName type, int length) {
- return types.contains(type) && (requiredLength == NOT_SET || length ==
requiredLength);
+ return types.contains(type)
+ && (type != PrimitiveTypeName.FIXED_LEN_BYTE_ARRAY
+ || requiredLength == NOT_SET
+ || length == requiredLength);
}
private boolean isEmpty() {
@@ -478,10 +481,11 @@ public class Types {
@Override
public String toString() {
- if (requiredLength != NOT_SET) {
- return PrimitiveTypeName.FIXED_LEN_BYTE_ARRAY + "(" + requiredLength
+ ")";
- }
- return types.stream().map(Enum::name).collect(Collectors.joining(",
"));
+ return types.stream()
+ .map(type -> type == PrimitiveTypeName.FIXED_LEN_BYTE_ARRAY &&
requiredLength != NOT_SET
+ ? type + "(" + requiredLength + ")"
+ : type.name())
+ .collect(Collectors.joining(", "));
}
}
diff --git
a/parquet-column/src/test/java/org/apache/parquet/column/statistics/TestStatistics.java
b/parquet-column/src/test/java/org/apache/parquet/column/statistics/TestStatistics.java
index 92c5cfed0..e4783b700 100644
---
a/parquet-column/src/test/java/org/apache/parquet/column/statistics/TestStatistics.java
+++
b/parquet-column/src/test/java/org/apache/parquet/column/statistics/TestStatistics.java
@@ -22,6 +22,8 @@ import static java.lang.Double.doubleToLongBits;
import static java.lang.Float.floatToIntBits;
import static org.apache.parquet.bytes.BytesUtils.intToBytes;
import static org.apache.parquet.bytes.BytesUtils.longToBytes;
+import static org.apache.parquet.schema.LogicalTypeAnnotation.TimeUnit.NANOS;
+import static org.apache.parquet.schema.LogicalTypeAnnotation.timestampType;
import static org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName.BINARY;
import static
org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName.BOOLEAN;
import static org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName.DOUBLE;
@@ -943,6 +945,32 @@ public class TestStatistics {
.hasMessage("isSmallerThan is not supported by
org.apache.parquet.column.statistics.NoopStatistics");
}
+ @Test
+ public void testFlba12TimestampStats() {
+ PrimitiveType type = Types.required(FIXED_LEN_BYTE_ARRAY)
+ .length(12)
+ .as(timestampType(true, NANOS))
+ .named("ts");
+ BinaryStatistics stats = (BinaryStatistics) Statistics.createStats(type);
+
+ // -1 (all 0xFF) — just before epoch, negative value
+ byte[] negOne = new byte[12];
+ for (int i = 0; i < 12; i++) negOne[i] = (byte) 0xFF;
+ // +1 (0x01 followed by zeros) — just after epoch, positive value
+ byte[] posOne = new byte[12];
+ posOne[0] = 1;
+ // epoch (all 0x00)
+ byte[] epoch = new byte[12];
+
+ stats.updateStats(Binary.fromConstantByteArray(posOne));
+ stats.updateStats(Binary.fromConstantByteArray(negOne));
+ stats.updateStats(Binary.fromConstantByteArray(epoch));
+
+ // min must be the negative value (-1), max must be the positive value (+1)
+
assertThat(stats.genericGetMin()).isEqualTo(Binary.fromConstantByteArray(negOne));
+
assertThat(stats.genericGetMax()).isEqualTo(Binary.fromConstantByteArray(posOne));
+ }
+
@Test
public void testBinaryIsSmallerThanNoOverflowForLargeValues() {
BinaryStatistics stats = new BinaryStatistics();
diff --git
a/parquet-column/src/test/java/org/apache/parquet/internal/column/columnindex/TestBinaryTruncator.java
b/parquet-column/src/test/java/org/apache/parquet/internal/column/columnindex/TestBinaryTruncator.java
index 6b7e79305..716c01770 100644
---
a/parquet-column/src/test/java/org/apache/parquet/internal/column/columnindex/TestBinaryTruncator.java
+++
b/parquet-column/src/test/java/org/apache/parquet/internal/column/columnindex/TestBinaryTruncator.java
@@ -36,6 +36,8 @@ import java.nio.charset.StandardCharsets;
import java.util.Comparator;
import java.util.Random;
import org.apache.parquet.io.api.Binary;
+import org.apache.parquet.schema.LogicalTypeAnnotation;
+import org.apache.parquet.schema.LogicalTypeAnnotation.TimeUnit;
import org.apache.parquet.schema.PrimitiveStringifier;
import org.apache.parquet.schema.PrimitiveType;
import org.apache.parquet.schema.Types;
@@ -91,6 +93,17 @@ public class TestBinaryTruncator {
testTruncator(Types.required(INT96).named("test_int96"), false);
}
+ @Test
+ public void testFlba12Timestamp() {
+ BinaryTruncator truncator =
BinaryTruncator.getTruncator(Types.required(FIXED_LEN_BYTE_ARRAY)
+ .length(12)
+ .as(LogicalTypeAnnotation.timestampType(true, TimeUnit.NANOS))
+ .named("test_fixed_timestamp"));
+ Binary value = Binary.fromConstantByteArray(new byte[] {0, 1, 2, 3, 4, 5,
6, 7, 8, 9, 10, 11});
+ assertThat(truncator.truncateMin(value, 4)).isSameAs(value);
+ assertThat(truncator.truncateMax(value, 4)).isSameAs(value);
+ }
+
@Test
public void testStringTruncate() {
BinaryTruncator truncator =
diff --git
a/parquet-column/src/test/java/org/apache/parquet/internal/column/columnindex/TestColumnIndexBuilder.java
b/parquet-column/src/test/java/org/apache/parquet/internal/column/columnindex/TestColumnIndexBuilder.java
index c8061afa4..d94308920 100644
---
a/parquet-column/src/test/java/org/apache/parquet/internal/column/columnindex/TestColumnIndexBuilder.java
+++
b/parquet-column/src/test/java/org/apache/parquet/internal/column/columnindex/TestColumnIndexBuilder.java
@@ -37,12 +37,15 @@ import static
org.apache.parquet.filter2.predicate.FilterApi.notIn;
import static org.apache.parquet.filter2.predicate.FilterApi.or;
import static org.apache.parquet.filter2.predicate.FilterApi.userDefined;
import static org.apache.parquet.filter2.predicate.LogicalInverter.invert;
+import static org.apache.parquet.schema.LogicalTypeAnnotation.TimeUnit.NANOS;
+import static org.apache.parquet.schema.LogicalTypeAnnotation.timestampType;
import static org.apache.parquet.schema.OriginalType.DECIMAL;
import static org.apache.parquet.schema.OriginalType.UINT_8;
import static org.apache.parquet.schema.OriginalType.UTF8;
import static org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName.BINARY;
import static
org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName.BOOLEAN;
import static org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName.DOUBLE;
+import static
org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName.FIXED_LEN_BYTE_ARRAY;
import static org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName.FLOAT;
import static org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName.INT32;
import static org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName.INT64;
@@ -2016,6 +2019,50 @@ public class TestColumnIndexBuilder {
assertCorrectNullCounts(ci6, 0, 1);
}
+ @Test
+ public void testBuildFlba12Timestamp() {
+ // FLBA(12) TIMESTAMP column with negative (pre-1970) and positive
(post-1970) values.
+ // The LE signed comparator must order negatives before zero before
positives.
+ PrimitiveType type = Types.required(FIXED_LEN_BYTE_ARRAY)
+ .length(12)
+ .as(timestampType(true, NANOS))
+ .named("ts");
+ ColumnIndexBuilder builder = ColumnIndexBuilder.getBuilder(type,
Integer.MAX_VALUE);
+ assertThat(builder).isInstanceOf(BinaryColumnIndexBuilder.class);
+
+ // -1 (all 0xFF) encodes a value just before epoch — smallest in the test
+ byte[] negOne = new byte[12];
+ for (int i = 0; i < 12; i++) negOne[i] = (byte) 0xFF;
+ // 0 (all 0x00) — epoch
+ byte[] epoch = new byte[12];
+ // +1 (0x01 followed by zeros) — just after epoch
+ byte[] posOne = new byte[12];
+ posOne[0] = 1;
+
+ StatsBuilder sb = new StatsBuilder();
+ // Page 0: only negative values, min=-1 max=-1
+ builder.add(sb.stats(type, Binary.fromConstantByteArray(negOne)));
+ // Page 1: straddles epoch, min=-1 max=+1
+ builder.add(sb.stats(type, Binary.fromConstantByteArray(negOne),
Binary.fromConstantByteArray(posOne)));
+ // Page 2: only positive values, min=epoch max=+1
+ builder.add(sb.stats(type, Binary.fromConstantByteArray(epoch),
Binary.fromConstantByteArray(posOne)));
+
+ ColumnIndex columnIndex = builder.build();
+ assertThat(columnIndex).isNotNull();
+ // min of page 0 must equal -1
+ assertCorrectValues(
+ columnIndex.getMinValues(),
+ Binary.fromConstantByteArray(negOne),
+ Binary.fromConstantByteArray(negOne),
+ Binary.fromConstantByteArray(epoch));
+ // max of page 0 must equal -1 (all values are negative)
+ assertCorrectValues(
+ columnIndex.getMaxValues(),
+ Binary.fromConstantByteArray(negOne),
+ Binary.fromConstantByteArray(posOne),
+ Binary.fromConstantByteArray(posOne));
+ }
+
@Test
public void testBuildWithoutNullCountsIsNotRejected() {
PrimitiveType type = Types.required(INT32).named("test_col");
diff --git
a/parquet-column/src/test/java/org/apache/parquet/schema/TestPrimitiveComparator.java
b/parquet-column/src/test/java/org/apache/parquet/schema/TestPrimitiveComparator.java
index 35de0c8ca..c07fdb810 100644
---
a/parquet-column/src/test/java/org/apache/parquet/schema/TestPrimitiveComparator.java
+++
b/parquet-column/src/test/java/org/apache/parquet/schema/TestPrimitiveComparator.java
@@ -21,6 +21,7 @@ package org.apache.parquet.schema;
import static
org.apache.parquet.schema.PrimitiveComparator.BINARY_AS_FLOAT16_COMPARATOR;
import static
org.apache.parquet.schema.PrimitiveComparator.BINARY_AS_FLOAT16_IEEE_754_TOTAL_ORDER_COMPARATOR;
import static
org.apache.parquet.schema.PrimitiveComparator.BINARY_AS_SIGNED_INTEGER_COMPARATOR;
+import static
org.apache.parquet.schema.PrimitiveComparator.BINARY_AS_SIGNED_TIMESTAMP_COMPARATOR;
import static org.apache.parquet.schema.PrimitiveComparator.BOOLEAN_COMPARATOR;
import static org.apache.parquet.schema.PrimitiveComparator.DOUBLE_COMPARATOR;
import static
org.apache.parquet.schema.PrimitiveComparator.DOUBLE_IEEE_754_TOTAL_ORDER_COMPARATOR;
@@ -433,6 +434,47 @@ public class TestPrimitiveComparator {
}
}
+ @Test
+ public void testBinaryAsSignedIntegerLE12Comparator() {
+ // 12-byte LE encodings: byte[0]=LSB, byte[11]=MSB/sign
+ // large negative: 0x80 00..00 (most negative 96-bit value)
+ byte[] largeNeg = new byte[12];
+ largeNeg[11] = (byte) 0x80;
+ // -256: 0x00 FF FF..FF (little-endian)
+ byte[] negTwoFiftySix = new byte[12];
+ for (int i = 1; i < 12; i++) negTwoFiftySix[i] = (byte) 0xFF;
+ // -1: all 0xFF
+ byte[] negOne = new byte[12];
+ for (int i = 0; i < 12; i++) negOne[i] = (byte) 0xFF;
+ // 0: all 0x00
+ byte[] zero = new byte[12];
+ // +1: 0x01 followed by 0x00s
+ byte[] posOne = new byte[12];
+ posOne[0] = 1;
+ // +256: byte[0]=0x00, byte[1]=0x01, rest 0x00 — order vs +1 is decided at
byte 1 (high byte);
+ // an LSB-first comparator would wrongly rank +256 < +1 (seeing byte[0]=0
< 1 first)
+ byte[] posTwoFiftySix = new byte[12];
+ posTwoFiftySix[1] = 1;
+ // +2^63: low 8 bytes have bit 63 set (LE: bytes 0..6=0x00, byte 7=0x80),
high 4 bytes=0x00.
+ // Sits between small positives and largePos; fails if the lower 8 bytes
are compared signed.
+ byte[] posTwo63 = new byte[12];
+ posTwo63[7] = (byte) 0x80;
+ // large positive: 0x7F FF..FF (most positive 96-bit value)
+ byte[] largePos = new byte[12];
+ for (int i = 0; i < 12; i++) largePos[i] = (byte) 0xFF;
+ largePos[11] = 0x7F;
+ testObjectComparator(
+ BINARY_AS_SIGNED_TIMESTAMP_COMPARATOR,
+ Binary.fromConstantByteArray(largeNeg),
+ Binary.fromConstantByteArray(negTwoFiftySix),
+ Binary.fromConstantByteArray(negOne),
+ Binary.fromConstantByteArray(zero),
+ Binary.fromConstantByteArray(posOne),
+ Binary.fromConstantByteArray(posTwoFiftySix),
+ Binary.fromConstantByteArray(posTwo63),
+ Binary.fromConstantByteArray(largePos));
+ }
+
private void checkThrowingUnsupportedException(PrimitiveComparator<?>
comparator, Class<?> exclude) {
if (Integer.TYPE != exclude) {
assertThatThrownBy(() -> comparator.compare(0, 0))
diff --git
a/parquet-column/src/test/java/org/apache/parquet/schema/TestPrimitiveStringifier.java
b/parquet-column/src/test/java/org/apache/parquet/schema/TestPrimitiveStringifier.java
index f0da51fc0..bbc6f5a78 100644
---
a/parquet-column/src/test/java/org/apache/parquet/schema/TestPrimitiveStringifier.java
+++
b/parquet-column/src/test/java/org/apache/parquet/schema/TestPrimitiveStringifier.java
@@ -45,12 +45,14 @@ import static
org.assertj.core.api.Assertions.assertThatThrownBy;
import java.math.BigInteger;
import java.nio.ByteBuffer;
+import java.util.Arrays;
import java.util.Calendar;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.TimeZone;
import java.util.concurrent.TimeUnit;
+import org.apache.parquet.bytes.BytesUtils;
import org.apache.parquet.io.api.Binary;
import org.junit.jupiter.api.Test;
@@ -193,7 +195,29 @@ public class TestPrimitiveStringifier {
assertThat(stringifier.stringify(cal.getTimeInMillis()))
.isEqualTo(withZoneString("1948-11-23T20:19:01.009",
timezoneAmendment));
- checkThrowingUnsupportedException(stringifier, Long.TYPE);
+ // The FIXED_LEN_BYTE_ARRAY(12) carrier renders identically to the int64
carrier. Includes a
+ // pre-epoch (negative) value to exercise the signed little-endian
decoding.
+ assertThat(stringifier.stringify(toFlba12Timestamp(0L)))
+ .isEqualTo(withZoneString("1970-01-01T00:00:00.000",
timezoneAmendment));
+ cal.clear();
+ cal.set(2017, Calendar.DECEMBER, 15, 10, 9, 54);
+ cal.set(Calendar.MILLISECOND, 120);
+
assertThat(stringifier.stringify(toFlba12Timestamp(cal.getTimeInMillis())))
+ .isEqualTo(withZoneString("2017-12-15T10:09:54.120",
timezoneAmendment));
+ cal.clear();
+ cal.set(1948, Calendar.NOVEMBER, 23, 20, 19, 1);
+ cal.set(Calendar.MILLISECOND, 9);
+
assertThat(stringifier.stringify(toFlba12Timestamp(cal.getTimeInMillis())))
+ .isEqualTo(withZoneString("1948-11-23T20:19:01.009",
timezoneAmendment));
+
+ // Test FLBA(12) timestamps that overflow the Instant range.
+ BigInteger overflowsLong = BigInteger.valueOf(2).pow(94).negate();
+
assertThat(stringifier.stringify(toFlba12Timestamp(overflowsLong))).isEqualTo(overflowsLong.toString());
+ BigInteger overflowsInstant = BigInteger.valueOf(2).pow(70);
+ assertThat(stringifier.stringify(toFlba12Timestamp(overflowsInstant)))
+ .isEqualTo(overflowsInstant.toString());
+
+ checkThrowingUnsupportedException(stringifier, Long.TYPE, Binary.class);
}
}
@@ -221,7 +245,21 @@ public class TestPrimitiveStringifier {
assertThat(stringifier.stringify(micros))
.isEqualTo(withZoneString("1848-03-15T09:23:59.764999",
timezoneAmendment));
- checkThrowingUnsupportedException(stringifier, Long.TYPE);
+ // The FIXED_LEN_BYTE_ARRAY(12) carrier renders identically to the int64
carrier, including
+ // the pre-epoch (negative) value above which exercises the signed
little-endian decoding.
+ assertThat(stringifier.stringify(toFlba12Timestamp(0L)))
+ .isEqualTo(withZoneString("1970-01-01T00:00:00.000000",
timezoneAmendment));
+ assertThat(stringifier.stringify(toFlba12Timestamp(micros)))
+ .isEqualTo(withZoneString("1848-03-15T09:23:59.764999",
timezoneAmendment));
+
+ // Test FLBA(12) timestamps that overflow the Instant range.
+ BigInteger overflowsLong = BigInteger.valueOf(2).pow(94).negate();
+
assertThat(stringifier.stringify(toFlba12Timestamp(overflowsLong))).isEqualTo(overflowsLong.toString());
+ BigInteger overflowsInstant = BigInteger.valueOf(2).pow(80);
+ assertThat(stringifier.stringify(toFlba12Timestamp(overflowsInstant)))
+ .isEqualTo(overflowsInstant.toString());
+
+ checkThrowingUnsupportedException(stringifier, Long.TYPE, Binary.class);
}
}
@@ -248,7 +286,21 @@ public class TestPrimitiveStringifier {
assertThat(stringifier.stringify(nanos))
.isEqualTo(withZoneString("1848-03-15T09:23:59.764999999",
timezoneAmendment));
- checkThrowingUnsupportedException(stringifier, Long.TYPE);
+ // The FIXED_LEN_BYTE_ARRAY(12) carrier renders identically to the int64
carrier, including
+ // the pre-epoch (negative) value above which exercises the signed
little-endian decoding.
+ assertThat(stringifier.stringify(toFlba12Timestamp(0L)))
+ .isEqualTo(withZoneString("1970-01-01T00:00:00.000000000",
timezoneAmendment));
+ assertThat(stringifier.stringify(toFlba12Timestamp(nanos)))
+ .isEqualTo(withZoneString("1848-03-15T09:23:59.764999999",
timezoneAmendment));
+
+ // Test FLBA(12) timestamps that overflow the Instant range.
+ BigInteger overflowsLong = BigInteger.valueOf(2).pow(94).negate();
+
assertThat(stringifier.stringify(toFlba12Timestamp(overflowsLong))).isEqualTo(overflowsLong.toString());
+ BigInteger overflowsInstant = BigInteger.valueOf(2).pow(90);
+ assertThat(stringifier.stringify(toFlba12Timestamp(overflowsInstant)))
+ .isEqualTo(overflowsInstant.toString());
+
+ checkThrowingUnsupportedException(stringifier, Long.TYPE, Binary.class);
}
}
@@ -417,6 +469,20 @@ public class TestPrimitiveStringifier {
return Binary.fromConstantByteArray(array);
}
+ private Binary toFlba12Timestamp(long units) {
+ return toFlba12Timestamp(BigInteger.valueOf(units));
+ }
+
+ private Binary toFlba12Timestamp(BigInteger units) {
+ byte[] bigEndian = units.toByteArray();
+ assertThat(bigEndian.length).isLessThanOrEqualTo(12);
+
+ byte[] padded = new byte[12];
+ Arrays.fill(padded, 0, 12 - bigEndian.length, (byte) (units.signum() < 0 ?
0xFF : 0x00));
+ System.arraycopy(bigEndian, 0, padded, 12 - bigEndian.length,
bigEndian.length);
+ return Binary.fromConstantByteArray(BytesUtils.reverse(padded));
+ }
+
private void checkThrowingUnsupportedException(PrimitiveStringifier
stringifier, Class<?>... excludes) {
Set<Class<?>> set = new HashSet<>(List.of(excludes));
if (!set.contains(Integer.TYPE)) {
diff --git
a/parquet-column/src/test/java/org/apache/parquet/schema/TestTypeBuilders.java
b/parquet-column/src/test/java/org/apache/parquet/schema/TestTypeBuilders.java
index 81e921b17..1a0a3d686 100644
---
a/parquet-column/src/test/java/org/apache/parquet/schema/TestTypeBuilders.java
+++
b/parquet-column/src/test/java/org/apache/parquet/schema/TestTypeBuilders.java
@@ -531,7 +531,8 @@ public class TestTypeBuilders {
@Test
public void testInt64Annotations() {
- OriginalType[] types = new OriginalType[] {TIME_MICROS, TIMESTAMP_MILLIS,
TIMESTAMP_MICROS, UINT_64, INT_64};
+ // Test the non-timestamp annotations for INT64. Timestamps are tested
separately below.
+ OriginalType[] types = new OriginalType[] {TIME_MICROS, UINT_64, INT_64};
for (OriginalType logicalType : types) {
PrimitiveType expected = new PrimitiveType(REQUIRED, INT64, "col",
logicalType);
PrimitiveType date = Types.required(INT64).as(logicalType).named("col");
@@ -541,7 +542,8 @@ public class TestTypeBuilders {
@Test
public void testInt64AnnotationsRejectNonInt64() {
- OriginalType[] types = new OriginalType[] {TIME_MICROS, TIMESTAMP_MILLIS,
TIMESTAMP_MICROS, UINT_64, INT_64};
+ // Test the non-timestamp annotations for INT64. Timestamps are tested
separately below.
+ OriginalType[] types = new OriginalType[] {TIME_MICROS, UINT_64, INT_64};
for (final OriginalType logicalType : types) {
PrimitiveTypeName[] nonInt64 = new PrimitiveTypeName[] {BOOLEAN, INT32,
INT96, DOUBLE, FLOAT, BINARY};
for (final PrimitiveTypeName type : nonInt64) {
@@ -558,6 +560,46 @@ public class TestTypeBuilders {
}
}
+ @Test
+ public void testTimestampAnnotations() {
+ // Test timestamp annotations for both INT64 and FLBA(12).
+ OriginalType[] types = new OriginalType[] {TIMESTAMP_MILLIS,
TIMESTAMP_MICROS};
+ for (OriginalType logicalType : types) {
+ PrimitiveType expectedInt64 = new PrimitiveType(REQUIRED, INT64, "col",
logicalType);
+ PrimitiveType dateInt64 =
Types.required(INT64).as(logicalType).named("col");
+ assertThat(dateInt64).isEqualTo(expectedInt64);
+
+ PrimitiveType expectedFlba12 = new PrimitiveType(REQUIRED,
FIXED_LEN_BYTE_ARRAY, 12, "col", logicalType);
+ PrimitiveType dateFlba12 = Types.required(FIXED_LEN_BYTE_ARRAY)
+ .length(12)
+ .as(logicalType)
+ .named("col");
+ assertThat(dateFlba12).isEqualTo(expectedFlba12);
+ }
+ }
+
+ @Test
+ public void testTimestampAnnotationsRejectNonTimestamp() {
+ // Test timestamp annotations for both INT64 and FLBA(12).
+ OriginalType[] types = new OriginalType[] {TIMESTAMP_MILLIS,
TIMESTAMP_MICROS};
+ for (OriginalType logicalType : types) {
+ // Invalid primitive types are rejected.
+ PrimitiveTypeName[] nonTimestamp = new PrimitiveTypeName[] {BOOLEAN,
INT32, INT96, DOUBLE, FLOAT, BINARY};
+ for (PrimitiveTypeName type : nonTimestamp) {
+ assertThatThrownBy(() ->
Types.required(type).as(logicalType).named("col"))
+ .isInstanceOf(IllegalStateException.class)
+ .hasMessage(logicalType + " can only annotate [INT64,
FIXED_LEN_BYTE_ARRAY(12)]");
+ }
+ // Invalid FLBA lengths are rejected.
+ assertThatThrownBy(() -> Types.required(FIXED_LEN_BYTE_ARRAY)
+ .length(11)
+ .as(logicalType)
+ .named("col"))
+ .isInstanceOf(IllegalStateException.class)
+ .hasMessage(logicalType + " can only annotate [INT64,
FIXED_LEN_BYTE_ARRAY(12)]");
+ }
+ }
+
@Test
public void testIntervalAnnotation() {
PrimitiveType expected = new PrimitiveType(REQUIRED, FIXED_LEN_BYTE_ARRAY,
12, "interval", INTERVAL);
diff --git
a/parquet-column/src/test/java/org/apache/parquet/schema/TestTypeBuildersWithLogicalTypes.java
b/parquet-column/src/test/java/org/apache/parquet/schema/TestTypeBuildersWithLogicalTypes.java
index 1fc46d7fd..fbc79cc81 100644
---
a/parquet-column/src/test/java/org/apache/parquet/schema/TestTypeBuildersWithLogicalTypes.java
+++
b/parquet-column/src/test/java/org/apache/parquet/schema/TestTypeBuildersWithLogicalTypes.java
@@ -294,9 +294,6 @@ public class TestTypeBuildersWithLogicalTypes {
LogicalTypeAnnotation[] types = new LogicalTypeAnnotation[] {
timeType(true, MICROS), timeType(false, MICROS),
timeType(true, NANOS), timeType(false, NANOS),
- timestampType(true, MILLIS), timestampType(false, MILLIS),
- timestampType(true, MICROS), timestampType(false, MICROS),
- timestampType(true, NANOS), timestampType(false, NANOS),
intType(64, true), intType(64, false)
};
for (LogicalTypeAnnotation logicalType : types) {
@@ -311,9 +308,6 @@ public class TestTypeBuildersWithLogicalTypes {
LogicalTypeAnnotation[] types = new LogicalTypeAnnotation[] {
timeType(true, MICROS), timeType(false, MICROS),
timeType(true, NANOS), timeType(false, NANOS),
- timestampType(true, MILLIS), timestampType(false, MILLIS),
- timestampType(true, MICROS), timestampType(false, MICROS),
- timestampType(true, NANOS), timestampType(false, NANOS),
intType(64, true), intType(64, false)
};
for (final LogicalTypeAnnotation logicalType : types) {
@@ -332,6 +326,50 @@ public class TestTypeBuildersWithLogicalTypes {
}
}
+ @Test
+ public void testTimestampAnnotations() {
+ LogicalTypeAnnotation[] types = new LogicalTypeAnnotation[] {
+ timestampType(true, MILLIS), timestampType(false, MILLIS),
+ timestampType(true, MICROS), timestampType(false, MICROS),
+ timestampType(true, NANOS), timestampType(false, NANOS)
+ };
+ for (LogicalTypeAnnotation logicalType : types) {
+ PrimitiveType expectedInt64 = new PrimitiveType(REQUIRED, INT64, "col",
logicalType);
+
assertThat(Types.required(INT64).as(logicalType).named("col")).isEqualTo(expectedInt64);
+
+ PrimitiveType expectedFlba12 =
+ new PrimitiveType(REQUIRED, FIXED_LEN_BYTE_ARRAY, 12, "col",
logicalType, null);
+ assertThat(Types.required(FIXED_LEN_BYTE_ARRAY)
+ .length(12)
+ .as(logicalType)
+ .named("col"))
+ .isEqualTo(expectedFlba12);
+ }
+ }
+
+ @Test
+ public void testTimestampAnnotationsRejectInvalidTypes() {
+ LogicalTypeAnnotation[] types = new LogicalTypeAnnotation[] {
+ timestampType(true, MILLIS), timestampType(false, MILLIS),
+ timestampType(true, MICROS), timestampType(false, MICROS),
+ timestampType(true, NANOS), timestampType(false, NANOS)
+ };
+ for (LogicalTypeAnnotation logicalType : types) {
+ PrimitiveTypeName[] invalidTypes = new PrimitiveTypeName[] {BOOLEAN,
INT32, INT96, DOUBLE, FLOAT, BINARY};
+ for (PrimitiveTypeName type : invalidTypes) {
+ assertThatThrownBy(() ->
Types.required(type).as(logicalType).named("col"))
+ .isInstanceOf(IllegalStateException.class)
+ .hasMessage(logicalType + " can only annotate [INT64,
FIXED_LEN_BYTE_ARRAY(12)]");
+ }
+ assertThatThrownBy(() -> Types.required(FIXED_LEN_BYTE_ARRAY)
+ .length(1)
+ .as(logicalType)
+ .named("col"))
+ .isInstanceOf(IllegalStateException.class)
+ .hasMessage(logicalType + " can only annotate [INT64,
FIXED_LEN_BYTE_ARRAY(12)]");
+ }
+ }
+
@Test
public void testIntervalAnnotationRejectsNonFixed() {
PrimitiveTypeName[] nonFixed = new PrimitiveTypeName[] {BOOLEAN, INT32,
INT64, INT96, DOUBLE, FLOAT, BINARY};
diff --git
a/parquet-common/src/main/java/org/apache/parquet/bytes/BytesUtils.java
b/parquet-common/src/main/java/org/apache/parquet/bytes/BytesUtils.java
index b8373a898..07bcc2890 100644
--- a/parquet-common/src/main/java/org/apache/parquet/bytes/BytesUtils.java
+++ b/parquet-common/src/main/java/org/apache/parquet/bytes/BytesUtils.java
@@ -345,4 +345,12 @@ public class BytesUtils {
public static boolean bytesToBool(byte[] bytes) {
return ((int) (bytes[0] & 255) != 0);
}
+
+ public static byte[] reverse(byte[] bytes) {
+ byte[] reversed = new byte[bytes.length];
+ for (int i = 0; i < bytes.length; i++) {
+ reversed[i] = bytes[bytes.length - 1 - i];
+ }
+ return reversed;
+ }
}
diff --git
a/parquet-hadoop/src/main/java/org/apache/parquet/format/converter/ParquetMetadataConverter.java
b/parquet-hadoop/src/main/java/org/apache/parquet/format/converter/ParquetMetadataConverter.java
index 8111deb4c..4252523fa 100644
---
a/parquet-hadoop/src/main/java/org/apache/parquet/format/converter/ParquetMetadataConverter.java
+++
b/parquet-hadoop/src/main/java/org/apache/parquet/format/converter/ParquetMetadataConverter.java
@@ -320,7 +320,14 @@ public class ParquetMetadataConverter {
element.setRepetition_type(toParquetRepetition(primitiveType.getRepetition()));
element.setType(getType(primitiveType.getPrimitiveTypeName()));
if (primitiveType.getLogicalTypeAnnotation() != null) {
-
element.setConverted_type(convertToConvertedType(primitiveType.getLogicalTypeAnnotation()));
+ // The TimestampType logical type may have a converted type, but
only for the INT64
+ // physical type.
+ boolean suppressConvertedType =
primitiveType.getLogicalTypeAnnotation()
+ instanceof
LogicalTypeAnnotation.TimestampLogicalTypeAnnotation
+ && primitiveType.getPrimitiveTypeName() !=
PrimitiveTypeName.INT64;
+ if (!suppressConvertedType) {
+
element.setConverted_type(convertToConvertedType(primitiveType.getLogicalTypeAnnotation()));
+ }
element.setLogicalType(convertToLogicalType(primitiveType.getLogicalTypeAnnotation()));
}
if (primitiveType.getDecimalMetadata() != null) {
diff --git
a/parquet-hadoop/src/test/java/org/apache/parquet/format/converter/TestParquetMetadataConverter.java
b/parquet-hadoop/src/test/java/org/apache/parquet/format/converter/TestParquetMetadataConverter.java
index 80760a032..46ffccf7f 100644
---
a/parquet-hadoop/src/test/java/org/apache/parquet/format/converter/TestParquetMetadataConverter.java
+++
b/parquet-hadoop/src/test/java/org/apache/parquet/format/converter/TestParquetMetadataConverter.java
@@ -453,6 +453,18 @@ public class TestParquetMetadataConverter {
.required(PrimitiveTypeName.INT64)
.as(timestampType(true, NANOS))
.named("aTimestampUtcNanos")
+ .required(PrimitiveTypeName.FIXED_LEN_BYTE_ARRAY)
+ .length(12)
+ .as(timestampType(true, MILLIS))
+ .named("aTimestampFlbaMillis")
+ .required(PrimitiveTypeName.FIXED_LEN_BYTE_ARRAY)
+ .length(12)
+ .as(timestampType(true, MICROS))
+ .named("aTimestampFlbaMicros")
+ .required(PrimitiveTypeName.FIXED_LEN_BYTE_ARRAY)
+ .length(12)
+ .as(timestampType(true, NANOS))
+ .named("aTimestampFlbaNanos")
.required(PrimitiveTypeName.INT32)
.as(timeType(false, MILLIS))
.named("aTimeNonUtcMillis")
@@ -473,6 +485,19 @@ public class TestParquetMetadataConverter {
.named("aTimeUtcNanos")
.named("Message");
List<SchemaElement> parquetSchema =
parquetMetadataConverter.toParquetSchema(expected);
+ // FLBA(12) MILLIS/MICROS must not write a legacy converted_type (it is
INT64-only).
+ SchemaElement flbaMillis = parquetSchema.stream()
+ .filter(e -> "aTimestampFlbaMillis".equals(e.getName()))
+ .findFirst()
+ .get();
+ assertThat(flbaMillis.isSetConverted_type()).isFalse();
+ assertThat(flbaMillis.isSetLogicalType()).isTrue();
+ SchemaElement flbaMicros = parquetSchema.stream()
+ .filter(e -> "aTimestampFlbaMicros".equals(e.getName()))
+ .findFirst()
+ .get();
+ assertThat(flbaMicros.isSetConverted_type()).isFalse();
+ assertThat(flbaMicros.isSetLogicalType()).isTrue();
MessageType schema =
parquetMetadataConverter.fromParquetSchema(parquetSchema, null);
assertThat(schema).isEqualTo(expected);
}
diff --git
a/parquet-hadoop/src/test/java/org/apache/parquet/hadoop/TestInterOpReadFlba12Timestamp.java
b/parquet-hadoop/src/test/java/org/apache/parquet/hadoop/TestInterOpReadFlba12Timestamp.java
new file mode 100644
index 000000000..56cfb954a
--- /dev/null
+++
b/parquet-hadoop/src/test/java/org/apache/parquet/hadoop/TestInterOpReadFlba12Timestamp.java
@@ -0,0 +1,167 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.parquet.hadoop;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+import java.io.IOException;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.parquet.example.data.Group;
+import org.apache.parquet.hadoop.example.GroupReadSupport;
+import org.apache.parquet.hadoop.metadata.BlockMetaData;
+import org.apache.parquet.hadoop.metadata.ColumnChunkMetaData;
+import org.apache.parquet.hadoop.util.HadoopInputFile;
+import org.apache.parquet.io.api.Binary;
+import org.apache.parquet.schema.ColumnOrder;
+import org.apache.parquet.schema.LogicalTypeAnnotation;
+import org.apache.parquet.schema.MessageType;
+import org.apache.parquet.schema.PrimitiveStringifier;
+import org.apache.parquet.schema.PrimitiveType;
+import org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName;
+import org.junit.jupiter.api.Test;
+
+public class TestInterOpReadFlba12Timestamp {
+ private static final String FILE = "flba12_timestamp.parquet";
+ private static final String CHANGESET =
"56653c437c8092f704a092d0d1d4e600124cd49f";
+
+ // The six timestamps stored in the file, in row order.
+ private static final String[] EXPECTED_TIMESTAMPS = {
+ "1970-01-01T00:00:00", // row 0: epoch
+ "1970-01-01T00:00:01", // row 1: +1 s
+ "1969-12-31T23:59:59", // row 2: -1 s (pre-1970, negative)
+ "2262-04-11T23:47:16", // row 3: near the INT64-nanos max boundary
+ "9999-12-31T23:59:59", // row 4: far future (NANOS needs > 64 bits)
+ "0001-01-01T00:00:00", // row 5: far past (NANOS is < -2^63)
+ };
+ // Row indices of the minimum (year 0001) and maximum (year 9999) timestamps.
+ private static final int MIN_ROW = 5;
+ private static final int MAX_ROW = 4;
+
+ // Fractional-second digit counts the UTC PrimitiveStringifier renders per
unit.
+ private static final int MILLIS_FRACTION_DIGITS = 3;
+ private static final int MICROS_FRACTION_DIGITS = 6;
+ private static final int NANOS_FRACTION_DIGITS = 9;
+
+ private final InterOpTester interop = new InterOpTester();
+
+ // Stringify the expected timestamp value for the given row with the
required fractional digits.
+ private static String expected(int row, int fractionDigits) {
+ StringBuilder sb = new StringBuilder(EXPECTED_TIMESTAMPS[row]).append('.');
+ for (int i = 0; i < fractionDigits; i++) {
+ sb.append('0');
+ }
+ return sb.append("+0000").toString();
+ }
+
+ @Test
+ public void testInterOpReadFlba12TimestampParquetFile() throws IOException {
+ Path filePath = interop.GetInterOpFile(FILE, CHANGESET);
+
+ final int expectRows = EXPECTED_TIMESTAMPS.length;
+
+ PrimitiveStringifier millisStringifier = null;
+ PrimitiveStringifier microsStringifier = null;
+ PrimitiveStringifier nanosStringifier = null;
+
+ try (ParquetFileReader reader =
+ ParquetFileReader.open(HadoopInputFile.fromPath(filePath, new
Configuration()))) {
+ MessageType schema = reader.getFooter().getFileMetaData().getSchema();
+ BlockMetaData block = reader.getFooter().getBlocks().get(0);
+
+ // timestamp_millis column
+ PrimitiveType millisType =
schema.getType("timestamp_millis").asPrimitiveType();
+
assertThat(millisType.getPrimitiveTypeName()).isEqualTo(PrimitiveTypeName.FIXED_LEN_BYTE_ARRAY);
+ assertThat(millisType.getTypeLength()).isEqualTo(12);
+ assertThat(millisType.getLogicalTypeAnnotation())
+ .isEqualTo(LogicalTypeAnnotation.timestampType(true,
LogicalTypeAnnotation.TimeUnit.MILLIS));
+
assertThat(millisType.columnOrder()).isEqualTo(ColumnOrder.typeDefined());
+ millisStringifier = millisType.stringifier();
+
+ // timestamp_micros column
+ PrimitiveType microsType =
schema.getType("timestamp_micros").asPrimitiveType();
+
assertThat(microsType.getPrimitiveTypeName()).isEqualTo(PrimitiveTypeName.FIXED_LEN_BYTE_ARRAY);
+ assertThat(microsType.getTypeLength()).isEqualTo(12);
+ assertThat(microsType.getLogicalTypeAnnotation())
+ .isEqualTo(LogicalTypeAnnotation.timestampType(true,
LogicalTypeAnnotation.TimeUnit.MICROS));
+
assertThat(microsType.columnOrder()).isEqualTo(ColumnOrder.typeDefined());
+ microsStringifier = microsType.stringifier();
+
+ // timestamp_nanos column
+ PrimitiveType nanosType =
schema.getType("timestamp_nanos").asPrimitiveType();
+
assertThat(nanosType.getPrimitiveTypeName()).isEqualTo(PrimitiveTypeName.FIXED_LEN_BYTE_ARRAY);
+ assertThat(nanosType.getTypeLength()).isEqualTo(12);
+ assertThat(nanosType.getLogicalTypeAnnotation())
+ .isEqualTo(LogicalTypeAnnotation.timestampType(true,
LogicalTypeAnnotation.TimeUnit.NANOS));
+ assertThat(nanosType.columnOrder()).isEqualTo(ColumnOrder.typeDefined());
+ nanosStringifier = nanosType.stringifier();
+
+ // Statistics: min = year 0001, max = year 9999.
+ ColumnChunkMetaData millisCol = block.getColumns().get(0);
+ assertThat(millisStringifier.stringify(Binary.fromConstantByteArray(
+ millisCol.getStatistics().getMinBytes())))
+ .isEqualTo(expected(MIN_ROW, MILLIS_FRACTION_DIGITS));
+ assertThat(millisStringifier.stringify(Binary.fromConstantByteArray(
+ millisCol.getStatistics().getMaxBytes())))
+ .isEqualTo(expected(MAX_ROW, MILLIS_FRACTION_DIGITS));
+
+ ColumnChunkMetaData microsCol = block.getColumns().get(1);
+ assertThat(microsStringifier.stringify(Binary.fromConstantByteArray(
+ microsCol.getStatistics().getMinBytes())))
+ .isEqualTo(expected(MIN_ROW, MICROS_FRACTION_DIGITS));
+ assertThat(microsStringifier.stringify(Binary.fromConstantByteArray(
+ microsCol.getStatistics().getMaxBytes())))
+ .isEqualTo(expected(MAX_ROW, MICROS_FRACTION_DIGITS));
+
+ ColumnChunkMetaData nanosCol = block.getColumns().get(2);
+ assertThat(nanosStringifier.stringify(Binary.fromConstantByteArray(
+ nanosCol.getStatistics().getMinBytes())))
+ .isEqualTo(expected(MIN_ROW, NANOS_FRACTION_DIGITS));
+ assertThat(nanosStringifier.stringify(Binary.fromConstantByteArray(
+ nanosCol.getStatistics().getMaxBytes())))
+ .isEqualTo(expected(MAX_ROW, NANOS_FRACTION_DIGITS));
+ }
+
+ try (ParquetReader<Group> reader =
+ ParquetReader.builder(new GroupReadSupport(), filePath).build()) {
+ for (int i = 0; i < expectRows; ++i) {
+ Group group = reader.read();
+ assertThat(group)
+ .as("Should not reach end of file before " + expectRows + " rows")
+ .isNotNull();
+ assertThat(group.getFieldRepetitionCount(0))
+ .as("timestamp_millis should not be null at row " + i)
+ .isNotEqualTo(0);
+ assertThat(millisStringifier.stringify(group.getBinary(0, 0)))
+ .isEqualTo(expected(i, MILLIS_FRACTION_DIGITS));
+ assertThat(group.getFieldRepetitionCount(1))
+ .as("timestamp_micros should not be null at row " + i)
+ .isNotEqualTo(0);
+ assertThat(microsStringifier.stringify(group.getBinary(1, 0)))
+ .isEqualTo(expected(i, MICROS_FRACTION_DIGITS));
+ assertThat(group.getFieldRepetitionCount(2))
+ .as("timestamp_nanos should not be null at row " + i)
+ .isNotEqualTo(0);
+ assertThat(nanosStringifier.stringify(group.getBinary(2, 0)))
+ .isEqualTo(expected(i, NANOS_FRACTION_DIGITS));
+ }
+ }
+ }
+}
diff --git
a/parquet-hadoop/src/test/java/org/apache/parquet/statistics/TestColumnIndexes.java
b/parquet-hadoop/src/test/java/org/apache/parquet/statistics/TestColumnIndexes.java
index 0b2435f11..35b2f4337 100644
---
a/parquet-hadoop/src/test/java/org/apache/parquet/statistics/TestColumnIndexes.java
+++
b/parquet-hadoop/src/test/java/org/apache/parquet/statistics/TestColumnIndexes.java
@@ -115,6 +115,10 @@ public class TestColumnIndexes {
.as(OriginalType.INTERVAL)
.named("interval"),
Types.optional(FIXED_LEN_BYTE_ARRAY).length(16).as(uuidType()).named("uuid"),
+ Types.optional(FIXED_LEN_BYTE_ARRAY)
+ .length(12)
+ .as(timestampType(true, TimeUnit.NANOS))
+ .named("timestamp-nanos-flba"),
Types.optional(BINARY).as(stringType()).named("always-null"));
private static List<Supplier<?>> buildGenerators(int recordCount, Random
random) {
@@ -219,6 +223,8 @@ public class TestColumnIndexes {
new RandomValues.FixedGenerator(random.nextLong(), 12), random,
recordCount, fieldIndex++),
sortedOrRandom(
new RandomValues.FixedGenerator(random.nextLong(), 16), random,
recordCount, fieldIndex++),
+ sortedOrRandom(
+ new RandomValues.FixedGenerator(random.nextLong(), 12), random,
recordCount, fieldIndex++),
null);
}
diff --git
a/parquet-hadoop/src/test/java/org/apache/parquet/statistics/TestStatistics.java
b/parquet-hadoop/src/test/java/org/apache/parquet/statistics/TestStatistics.java
index d5cb7b953..5dc6c1f80 100644
---
a/parquet-hadoop/src/test/java/org/apache/parquet/statistics/TestStatistics.java
+++
b/parquet-hadoop/src/test/java/org/apache/parquet/statistics/TestStatistics.java
@@ -381,7 +381,8 @@ public class TestStatistics {
new RandomValues.LongGenerator(random.nextLong()),
new RandomValues.LongGenerator(random.nextLong()),
new RandomValues.FixedGenerator(random.nextLong(), 12),
- new RandomValues.FixedGenerator(random.nextLong(), 2));
+ new RandomValues.FixedGenerator(random.nextLong(), 2),
+ new RandomValues.FixedGenerator(random.nextLong(), 12));
}
private static MessageType buildSchema(long seed) {
@@ -451,7 +452,12 @@ public class TestStatistics {
Types.optional(FIXED_LEN_BYTE_ARRAY)
.length(2)
.named("float16")
- .withLogicalTypeAnnotation(LogicalTypeAnnotation.float16Type()));
+ .withLogicalTypeAnnotation(LogicalTypeAnnotation.float16Type()),
+ Types.optional(FIXED_LEN_BYTE_ARRAY)
+ .length(12)
+ .named("timestamp-nanos-flba")
+ .withLogicalTypeAnnotation(
+ LogicalTypeAnnotation.timestampType(true,
LogicalTypeAnnotation.TimeUnit.NANOS)));
}
private static int calculatePrecision(int byteCnt) {
