This is an automated email from the ASF dual-hosted git repository.
Jefffrey pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow-rs.git
The following commit(s) were added to refs/heads/main by this push:
new b6a12bb781 GH-10613: Fix IPC FileReader projected schema (#10627)
b6a12bb781 is described below
commit b6a12bb781d010cbb541ae074acc3814feecfc81
Author: Dhruv Vaishnav <[email protected]>
AuthorDate: Wed Aug 12 13:20:58 2026 +0530
GH-10613: Fix IPC FileReader projected schema (#10627)
# Which issue does this PR close?
- Closes #10613.
# Rationale for this change
When `FileReader` uses a projection, the batches contain the projected
schema but `RecordBatchReader::schema()` returns the complete file
schema. This makes the schema reported by the reader inconsistent with
the batches it produces.
# What changes are included in this PR?
- Store the projected output schema on `FileReader` while keeping the
complete schema in `FileDecoder` for decoding.
- Return the projected schema from `FileReader::schema()`.
- Validate projection indices when constructing the reader.
- Add regression tests for projected schemas and invalid projections.
# Are these changes tested?
Yes. I ran:
- `cargo test -p arrow-ipc --all-features`
- `cargo test -p arrow --all-features`
- `cargo clippy -p arrow-ipc --all-targets --all-features -- -D
warnings`
- `cargo +stable fmt --all -- --check`
# Are there any user-facing changes?
Yes. Projected IPC file readers now report the same schema as the record
batches they produce. There are no public API signature changes.
AI assistance: OpenAI Codex assisted with codebase exploration,
implementation support, test preparation, and validation. I reviewed and
validated the changes and remain responsible for understanding and
maintaining them.
---
arrow-ipc/src/reader.rs | 59 ++++++++++++++++++++++++++++++++++++++++++++-----
1 file changed, 53 insertions(+), 6 deletions(-)
diff --git a/arrow-ipc/src/reader.rs b/arrow-ipc/src/reader.rs
index d6aa2d6653..f3856b1d32 100644
--- a/arrow-ipc/src/reader.rs
+++ b/arrow-ipc/src/reader.rs
@@ -1256,7 +1256,12 @@ impl FileReaderBuilder {
));
}
- let schema = crate::convert::fb_to_schema(ipc_schema);
+ let schema = Arc::new(crate::convert::fb_to_schema(ipc_schema));
+
+ let projected_schema = match &self.projection {
+ Some(projection) => Arc::new(schema.project(projection)?),
+ None => schema.clone(),
+ };
let mut custom_metadata = HashMap::new();
if let Some(fb_custom_metadata) = footer.custom_metadata() {
@@ -1268,7 +1273,7 @@ impl FileReaderBuilder {
}
}
- let mut decoder = FileDecoder::new(Arc::new(schema), footer.version());
+ let mut decoder = FileDecoder::new(schema, footer.version());
if let Some(projection) = self.projection {
decoder = decoder.with_projection(projection)
}
@@ -1287,6 +1292,7 @@ impl FileReaderBuilder {
current_block: 0,
total_blocks,
decoder,
+ schema: projected_schema,
custom_metadata,
})
}
@@ -1343,6 +1349,9 @@ pub struct FileReader<R> {
/// The decoder
decoder: FileDecoder,
+ /// Schema of the record batches produced by this reader
+ schema: SchemaRef,
+
/// The blocks in the file
///
/// A block indicates the regions in the file to read to get data
@@ -1387,8 +1396,9 @@ impl<R: Read + Seek> FileReader<R> {
/// # Errors
///
/// An [`Err`] may be returned if:
- /// - the file does not meet the Arrow Format footer requirements, or
- /// - file endianness does not match the target endianness.
+ /// - the file does not meet the Arrow Format footer requirements,
+ /// - file endianness does not match the target endianness, or
+ /// - the projection contains an index outside the file schema.
pub fn try_new(reader: R, projection: Option<Vec<usize>>) -> Result<Self,
ArrowError> {
let builder = FileReaderBuilder {
projection,
@@ -1407,9 +1417,9 @@ impl<R: Read + Seek> FileReader<R> {
self.total_blocks
}
- /// Return the schema of the file
+ /// Return the schema of the record batches produced by this reader
pub fn schema(&self) -> SchemaRef {
- self.decoder.schema.clone()
+ self.schema.clone()
}
/// See to a specific [`RecordBatch`]
@@ -2345,6 +2355,43 @@ mod tests {
}
}
+ #[test]
+ fn test_file_reader_projected_schema_matches_batch_schema() {
+ let schema = create_test_projection_schema();
+ let batch = create_test_projection_batch_data(&schema);
+
+ let mut buf = Vec::new();
+ {
+ let mut writer = crate::writer::FileWriter::try_new(&mut buf,
&schema).unwrap();
+ writer.write(&batch).unwrap();
+ writer.finish().unwrap();
+ }
+
+ let projection = vec![3, 2, 1];
+ let mut reader = FileReader::try_new(Cursor::new(buf),
Some(projection)).unwrap();
+ let reader_schema = RecordBatchReader::schema(&reader);
+ let read_batch = reader.next().unwrap().unwrap();
+
+ assert_eq!(reader_schema, read_batch.schema());
+ }
+
+ #[test]
+ fn test_file_reader_rejects_invalid_projection() {
+ let schema = create_test_projection_schema();
+ let batch = create_test_projection_batch_data(&schema);
+
+ let mut buf = Vec::new();
+ {
+ let mut writer = crate::writer::FileWriter::try_new(&mut buf,
&schema).unwrap();
+ writer.write(&batch).unwrap();
+ writer.finish().unwrap();
+ }
+
+ let result = FileReader::try_new(Cursor::new(buf),
Some(vec![schema.fields().len()]));
+
+ assert!(matches!(result, Err(ArrowError::SchemaError(_))));
+ }
+
#[test]
fn test_projection_duplicate_indices() {
let schema = create_test_projection_schema();
