tustvold commented on a change in pull request #1622:
URL: https://github.com/apache/arrow-datafusion/pull/1622#discussion_r789897399
##########
File path: datafusion/src/physical_plan/file_format/parquet.rs
##########
@@ -417,15 +446,49 @@ fn read_partition(
);
file_reader.filter_row_groups(&row_group_predicate);
}
+
let mut arrow_reader =
ParquetFileArrowReader::new(Arc::new(file_reader));
- let mut batch_reader = arrow_reader
- .get_record_reader_by_columns(projection.to_owned(), batch_size)?;
+ let mapped_projections =
+ map_projections(&file_schema, &arrow_reader.get_schema()?,
projection);
+
+ let mut batch_reader =
+ arrow_reader.get_record_reader_by_columns(mapped_projections,
batch_size)?;
loop {
match batch_reader.next() {
Some(Ok(batch)) => {
+ let total_cols = &file_schema.fields().len();
+ let batch_rows = batch.num_rows();
total_rows += batch.num_rows();
+
+ let batch_schema = batch.schema();
+
+ let merged_batch = if batch.columns().len() <
projection.len() {
Review comment:
I think this will misbehave if the batch has columns in a different
order?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: github-unsubscr...@arrow.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
