mbutrovich commented on code in PR #2301:
URL: https://github.com/apache/iceberg-rust/pull/2301#discussion_r3017590725
##########
crates/iceberg/src/arrow/reader.rs:
##########
@@ -1232,6 +1253,121 @@ fn add_fallback_field_ids_to_arrow_schema(arrow_schema:
&ArrowSchemaRef) -> Arc<
))
}
+/// Coerce Arrow schema types for INT96 columns to match the Iceberg table
schema.
+///
+/// arrow-rs defaults INT96 to `Timestamp(Nanosecond)`, which overflows i64
for dates outside
+/// ~1677-2262. We use arrow-rs's schema hint mechanism to read INT96 at the
resolution
+/// specified by the Iceberg schema (`timestamp` → microsecond, `timestamp_ns`
→ nanosecond).
+///
+/// Iceberg Java handles this differently: it bypasses parquet-mr with a
custom column reader
+/// (`GenericParquetReaders.TimestampInt96Reader`). We achieve the same result
via schema hints.
+///
+/// References:
+/// - Iceberg spec primitive types:
<https://iceberg.apache.org/spec/#primitive-types>
+/// - arrow-rs schema hint support:
<https://github.com/apache/arrow-rs/pull/7285>
+fn coerce_int96_timestamps(
+ parquet_schema: &SchemaDescriptor,
+ arrow_schema: &ArrowSchemaRef,
+ iceberg_schema: &Schema,
+) -> Option<Arc<ArrowSchema>> {
+ use arrow_schema::{DataType, Field, Fields, TimeUnit};
Review Comment:
Will do, thanks!
##########
crates/iceberg/src/arrow/reader.rs:
##########
@@ -1232,6 +1253,121 @@ fn add_fallback_field_ids_to_arrow_schema(arrow_schema:
&ArrowSchemaRef) -> Arc<
))
}
+/// Coerce Arrow schema types for INT96 columns to match the Iceberg table
schema.
+///
+/// arrow-rs defaults INT96 to `Timestamp(Nanosecond)`, which overflows i64
for dates outside
+/// ~1677-2262. We use arrow-rs's schema hint mechanism to read INT96 at the
resolution
+/// specified by the Iceberg schema (`timestamp` → microsecond, `timestamp_ns`
→ nanosecond).
+///
+/// Iceberg Java handles this differently: it bypasses parquet-mr with a
custom column reader
+/// (`GenericParquetReaders.TimestampInt96Reader`). We achieve the same result
via schema hints.
+///
+/// References:
+/// - Iceberg spec primitive types:
<https://iceberg.apache.org/spec/#primitive-types>
+/// - arrow-rs schema hint support:
<https://github.com/apache/arrow-rs/pull/7285>
+fn coerce_int96_timestamps(
+ parquet_schema: &SchemaDescriptor,
+ arrow_schema: &ArrowSchemaRef,
+ iceberg_schema: &Schema,
+) -> Option<Arc<ArrowSchema>> {
+ use arrow_schema::{DataType, Field, Fields, TimeUnit};
+ use parquet::basic::Type as PhysicalType;
+
+ let int96_paths: HashSet<String> = parquet_schema
+ .columns()
+ .iter()
+ .filter(|col| col.physical_type() == PhysicalType::INT96)
+ .map(|col| col.path().string())
+ .collect();
+
+ if int96_paths.is_empty() {
+ return None;
+ }
+
+ fn coerce_field(
Review Comment:
I'll find a cleaner way to break this us.
##########
crates/iceberg/src/arrow/reader.rs:
##########
@@ -386,6 +386,27 @@ impl ArrowReader {
arrow_metadata
};
+ // Coerce INT96 timestamp columns to the resolution specified by the
Iceberg schema.
+ // This must happen before building the stream reader to avoid i64
overflow in arrow-rs.
+ let arrow_metadata = if let Some(coerced_schema) =
coerce_int96_timestamps(
+ arrow_metadata.metadata().file_metadata().schema_descr(),
+ arrow_metadata.schema(),
+ &task.schema,
+ ) {
+ let options =
ArrowReaderOptions::new().with_schema(coerced_schema);
+
ArrowReaderMetadata::try_new(Arc::clone(arrow_metadata.metadata()),
options).map_err(
+ |e| {
+ Error::new(
+ ErrorKind::Unexpected,
+ "Failed to create ArrowReaderMetadata with
INT96-coerced schema",
Review Comment:
Will do, thanks!
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
