This is an automated email from the ASF dual-hosted git repository.
Fokko pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/parquet-java.git
The following commit(s) were added to refs/heads/master by this push:
new b03309987 Update inlined parquet.thrift to parquet-format 2.14.0
(#3791)
b03309987 is described below
commit b033099873af82fd65be3094c80adf5ffc691517
Author: Vinoo Ganesh <[email protected]>
AuthorDate: Mon Sep 14 08:17:00 2026 -0400
Update inlined parquet.thrift to parquet-format 2.14.0 (#3791)
Brings the IDL up from 2.13.0, picking up the FILE logical type, INT96
chronological ordering and the ALP encoding.
FILE now exists in the LogicalType union, so the converter would have
thrown on it instead of falling into the unrecognised-type branch that
ignores it. It is ignored explicitly now, keeping the physical type as
before.
ALP is in the format enum but has no Java implementation yet, so
testEnumEquivalence skips it until one lands.
---
.../src/main/thrift/parquet-format.version | 2 +-
.../src/main/thrift/parquet.thrift | 58 +++++++++++++++++++---
.../format/converter/ParquetMetadataConverter.java | 4 ++
.../converter/TestParquetMetadataConverter.java | 15 ++++++
4 files changed, 70 insertions(+), 9 deletions(-)
diff --git a/parquet-format-structures/src/main/thrift/parquet-format.version
b/parquet-format-structures/src/main/thrift/parquet-format.version
index 774d7578d..762e7a62e 100644
--- a/parquet-format-structures/src/main/thrift/parquet-format.version
+++ b/parquet-format-structures/src/main/thrift/parquet-format.version
@@ -1,2 +1,2 @@
# Provenance of the inlined parquet.thrift. Maintained by
dev/update-parquet-thrift.sh.
-parquet-format.commit=c47e2a66e88943fc46fde1b028a9432f14fdf5c0
+parquet-format.commit=04d56f291ff963e98bc37ab8100e2fc133ff583c
diff --git a/parquet-format-structures/src/main/thrift/parquet.thrift
b/parquet-format-structures/src/main/thrift/parquet.thrift
index fe259d61b..bb582017b 100644
--- a/parquet-format-structures/src/main/thrift/parquet.thrift
+++ b/parquet-format-structures/src/main/thrift/parquet.thrift
@@ -468,6 +468,17 @@ struct GeographyType {
2: optional EdgeInterpolationAlgorithm algorithm;
}
+/**
+ * File logical type annotation
+ *
+ * Annotates a group that represents a reference to a file, or to a range of
+ * bytes that may be stored inline or in an external file.
+ *
+ * See LogicalTypes.md for details.
+ */
+struct FileType {
+}
+
/**
* LogicalType annotations to replace ConvertedType.
*
@@ -501,6 +512,7 @@ union LogicalType {
16: VariantType VARIANT // no compatible ConvertedType
17: GeometryType GEOMETRY // no compatible ConvertedType
18: GeographyType GEOGRAPHY // no compatible ConvertedType
+ 19: FileType FILE // no compatible ConvertedType
}
/**
@@ -636,6 +648,14 @@ enum Encoding {
Support for INT32, INT64 and FIXED_LEN_BYTE_ARRAY added in 2.11.
*/
BYTE_STREAM_SPLIT = 9;
+
+ /** Adaptive Lossless floating-Point (ALP) encoding for FLOAT and DOUBLE.
+ Losslessly converts decimal-like floating-point values to integers via
+ decimal scaling, then applies Frame of Reference (FOR) encoding and
+ bit-packing; values that cannot be converted losslessly are stored as
+ exceptions. See Encodings.md for the detailed specification.
+ */
+ ALP = 10;
}
/**
@@ -1061,6 +1081,9 @@ struct TypeDefinedOrder {}
/** Empty struct to signal IEEE 754 total order for floating point types */
struct IEEE754TotalOrder {}
+/** Empty struct to signal chronological ordering of physical type INT96 */
+struct Int96TimestampOrder {}
+
/**
* Union to specify the order used for the min_value and max_value fields for a
* column. This union takes the role of an enhanced enum that allows rich
@@ -1071,6 +1094,8 @@ struct IEEE754TotalOrder {}
* physical type (if there is no logical type).
* * IEEE754TotalOrder - the floating point column uses IEEE 754 total order.
*
+ * * Int96TimestampOrder - the INT96 column uses chronological timestamp order.
+ *
* If the reader does not support the value of this union, min and max stats
* for this column should be ignored.
*/
@@ -1103,25 +1128,29 @@ union ColumnOrder {
* VARIANT - undefined
* GEOMETRY - undefined
* GEOGRAPHY - undefined
+ * FILE - undefined
*
* In the absence of logical types, the sort order is determined by the
physical type:
* BOOLEAN - false, true
* INT32 - signed comparison
* INT64 - signed comparison
- * INT96 (only used for legacy timestamps) - undefined(+)
+ * INT96 (only used for legacy timestamps) - depends on sort order (+)
* FLOAT - signed comparison of the represented value (*)
* DOUBLE - signed comparison of the represented value (*)
* BYTE_ARRAY - unsigned byte-wise comparison
* FIXED_LEN_BYTE_ARRAY - unsigned byte-wise comparison
*
* (+) While the INT96 type has been deprecated, at the time of writing it is
- * still used in many legacy systems. If a Parquet implementation chooses
- * to write statistics for INT96 columns, it is recommended to order them
- * according to the legacy rules:
- * - compare the last 4 bytes (days) as a little-endian 32-bit signed
integer
- * - if equal last 4 bytes, compare the first 8 bytes as a little-endian
- * 64-bit signed integer (nanos)
- * See https://github.com/apache/parquet-format/issues/502 for more
details
+ * still used in many legacy systems. It is optional for writers to emit
+ * statistics for INT96 columns. Writers that emit stats for such columns
+ * should use the INT96_TIMESTAMP_ORDER for this type and order the
values
+ * according to the legacy rules:
+ * - compare the last 4 bytes (days) as a little-endian 32-bit signed
integer
+ * - if equal last 4 bytes, compare the first 8 bytes as a little-endian
+ * 64-bit signed integer (nanos)
+ * If TYPE_ORDER is used for an INT96 column, readers should ignore all
statistics
+ * (`min`/`max` fields in `Statistics` and `min_values`/`max_values`
fields in
+ * `ColumnIndex`) for that column.
*
* (*) Because TYPE_ORDER is ambiguous for floating point types due to
* underspecified handling of NaN and -0/+0, it is recommended that
writers
@@ -1195,6 +1224,12 @@ union ColumnOrder {
* or max_values indicates that all non-null values are NaN.
*/
2: IEEE754TotalOrder IEEE_754_TOTAL_ORDER;
+
+ /*
+ * The INT96 timestamp type is ordered chronologically. Only columns of
+ * physical type INT96 may use this ordering.
+ */
+ 3: Int96TimestampOrder INT96_TIMESTAMP_ORDER;
}
struct PageLocation {
@@ -1278,6 +1313,13 @@ struct ColumnIndex {
* - If the order of this column is IEEE754_TOTAL_ORDER, then min_values[i]
* and max_values[i] of that page must be set to the smallest and largest
* NaN values as defined by IEEE 754 total order.
+ *
+ * For columns of physical type INT96, the writer must do the following:
+ * - If the order of this column is not INT96_TIMESTAMP_ORDER, then a column
+ * index must not be written for this column chunk.
+ * - If the order of this column is INT96_TIMESTAMP_ORDER, the min_values[i]
+ * and max_values[i] of that page must be set to the smallest and largest
+ * values as defined by the INT96 chronological timestamp ordering.
*/
2: required list<binary> min_values
3: required list<binary> max_values
diff --git
a/parquet-hadoop/src/main/java/org/apache/parquet/format/converter/ParquetMetadataConverter.java
b/parquet-hadoop/src/main/java/org/apache/parquet/format/converter/ParquetMetadataConverter.java
index f6ee73bbc..cc7805c31 100644
---
a/parquet-hadoop/src/main/java/org/apache/parquet/format/converter/ParquetMetadataConverter.java
+++
b/parquet-hadoop/src/main/java/org/apache/parquet/format/converter/ParquetMetadataConverter.java
@@ -1409,6 +1409,10 @@ public class ParquetMetadataConverter {
case VARIANT:
VariantType variant = type.getVARIANT();
return
LogicalTypeAnnotation.variantType(variant.getSpecification_version());
+ case FILE:
+ // Present in the format but not mapped to a LogicalTypeAnnotation
yet. Ignore it to
+ // preserve the physical type, as an unrecognised logical type would
be.
+ return null;
default:
throw new RuntimeException("Unknown logical type " + type);
}
diff --git
a/parquet-hadoop/src/test/java/org/apache/parquet/format/converter/TestParquetMetadataConverter.java
b/parquet-hadoop/src/test/java/org/apache/parquet/format/converter/TestParquetMetadataConverter.java
index f5222de82..784a837e3 100644
---
a/parquet-hadoop/src/test/java/org/apache/parquet/format/converter/TestParquetMetadataConverter.java
+++
b/parquet-hadoop/src/test/java/org/apache/parquet/format/converter/TestParquetMetadataConverter.java
@@ -101,6 +101,7 @@ import org.apache.parquet.format.ConvertedType;
import org.apache.parquet.format.DecimalType;
import org.apache.parquet.format.FieldRepetitionType;
import org.apache.parquet.format.FileMetaData;
+import org.apache.parquet.format.FileType;
import org.apache.parquet.format.GeographyType;
import org.apache.parquet.format.GeometryType;
import org.apache.parquet.format.GeospatialStatistics;
@@ -542,6 +543,15 @@ public class TestParquetMetadataConverter {
.isEqualTo(ConvertedType.MAP_KEY_VALUE);
}
+ @Test
+ public void testFileLogicalTypeIsIgnoredRatherThanFailing() {
+ ParquetMetadataConverter converter = new ParquetMetadataConverter();
+ // FILE has no LogicalTypeAnnotation yet, so it must degrade to the
physical type the way an
+ // unrecognised logical type does, rather than throwing.
+ assertThat(converter.getLogicalTypeAnnotation(LogicalType.FILE(new
FileType())))
+ .isNull();
+ }
+
@Test
public void testEnumEquivalence() {
ParquetMetadataConverter parquetMetadataConverter = new
ParquetMetadataConverter();
@@ -550,6 +560,11 @@ public class TestParquetMetadataConverter {
.isEqualTo(encoding);
}
for (org.apache.parquet.format.Encoding encoding :
org.apache.parquet.format.Encoding.values()) {
+ // ALP is in the format spec but is not implemented on the Java side
yet, so it has no
+ // org.apache.parquet.column.Encoding to round trip through. Remove this
once it does.
+ if (encoding == org.apache.parquet.format.Encoding.ALP) {
+ continue;
+ }
assertThat(parquetMetadataConverter.getEncoding(parquetMetadataConverter.getEncoding(encoding)))
.isEqualTo(encoding);
}
