This is an automated email from the ASF dual-hosted git repository.
alamb pushed a commit to branch 59_maintenance
in repository https://gitbox.apache.org/repos/asf/arrow-rs.git
The following commit(s) were added to refs/heads/59_maintenance by this push:
new 226fc54894 [59_maintenance] fix(parquet): keep virtual columns in the
schema reported with a schema hint (#11447)
226fc54894 is described below
commit 226fc548948b315d3f3ab8ee7c16eed8b678af56
Author: Jeffrey Vo <[email protected]>
AuthorDate: Fri Oct 9 03:19:55 2026 +0900
[59_maintenance] fix(parquet): keep virtual columns in the schema reported
with a schema hint (#11447)
- Backport https://github.com/apache/arrow-rs/pull/11047
- For inclusion in https://github.com/apache/arrow-rs/issues/11352
Cherry-pick of
https://github.com/apache/arrow-rs/commit/5faefa9bf022471856b7a3e0413a743e53980d8c
Co-authored-by: Bharadwaj Pendyala <[email protected]>
---
parquet/src/arrow/arrow_reader/mod.rs | 58 ++++++++++++++++++++++++++++++++++-
1 file changed, 57 insertions(+), 1 deletion(-)
diff --git a/parquet/src/arrow/arrow_reader/mod.rs
b/parquet/src/arrow/arrow_reader/mod.rs
index 5341739d98..ddf8879bf1 100644
--- a/parquet/src/arrow/arrow_reader/mod.rs
+++ b/parquet/src/arrow/arrow_reader/mod.rs
@@ -1031,9 +1031,20 @@ impl ArrowReaderMetadata {
)));
}
+ // `fields` is the supplied fields followed by the virtual columns, so
the reported
+ // schema has to be built from it or the virtual columns go missing.
+ let schema = if virtual_columns.is_empty() {
+ supplied_schema
+ } else {
+ Arc::new(Schema::new_with_metadata(
+ fields,
+ supplied_schema.metadata().clone(),
+ ))
+ };
+
Ok(Self {
metadata,
- schema: supplied_schema,
+ schema,
fields: field_levels.levels.map(Arc::new),
})
}
@@ -6037,4 +6048,49 @@ pub(crate) mod tests {
(Bytes::from(buf), metadata)
}
+
+ #[test]
+ fn test_supplied_schema_keeps_virtual_columns() {
+ let file = write_parquet_from_iter(vec![(
+ "value",
+ Arc::new(Int64Array::from(vec![1, 2, 3])) as ArrayRef,
+ )]);
+ let supplied_fields = Fields::from(vec![Field::new("value",
ArrowDataType::Int64, false)]);
+ let row_number_field = Arc::new(
+ Field::new("row_number", ArrowDataType::Int64,
false).with_extension_type(RowNumber),
+ );
+ let row_group_index_field = Arc::new(
+ Field::new("row_group_index", ArrowDataType::Int64, false)
+ .with_extension_type(RowGroupIndex),
+ );
+ let supplied_metadata = HashMap::from([("k".to_string(),
"v".to_string())]);
+
+ let options = ArrowReaderOptions::new()
+ .with_schema(Arc::new(Schema::new_with_metadata(
+ supplied_fields,
+ supplied_metadata.clone(),
+ )))
+ .with_virtual_columns(vec![
+ row_number_field.clone(),
+ row_group_index_field.clone(),
+ ])
+ .unwrap();
+ let metadata = ArrowReaderMetadata::load(&file, options).unwrap();
+
+ let expected = Fields::from(vec![
+ Arc::new(Field::new("value", ArrowDataType::Int64, false)),
+ row_number_field,
+ row_group_index_field,
+ ]);
+ assert_eq!(metadata.schema().fields(), &expected);
+ assert_eq!(metadata.schema().metadata(), &supplied_metadata);
+
+ let batch = ParquetRecordBatchReaderBuilder::new_with_metadata(file,
metadata.clone())
+ .build()
+ .unwrap()
+ .next()
+ .unwrap()
+ .unwrap();
+ assert_eq!(batch.schema().fields(), metadata.schema().fields());
+ }
}
