jonvex commented on code in PR #11770:
URL: https://github.com/apache/hudi/pull/11770#discussion_r1715651297
##########
hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/HoodieFileGroupReaderBasedParquetFileFormat.scala:
##########
@@ -127,7 +127,11 @@ class
HoodieFileGroupReaderBasedParquetFileFormat(tableState: HoodieTableState,
new Configuration(hadoopConf))
}
- val requestedAvroSchema =
AvroConversionUtils.convertStructTypeToAvroSchema(requiredSchema,
sanitizedTableName)
+ val (remainingPartitionSchemaArr, fixedPartitionIndexesArr) =
partitionSchema.fields.toSeq.zipWithIndex.filter(p =>
!mandatoryFields.contains(p._1.name)).unzip
+ val remainingPartitionSchema = StructType(remainingPartitionSchemaArr)
+ val fixedPartitionIndexes = fixedPartitionIndexesArr.toSet
+ val requestedSchema = StructType(requiredSchema.fields ++
partitionSchema.fields.filter(f => mandatoryFields.contains(f.name)))
+ val requestedAvroSchema =
AvroConversionUtils.convertStructTypeToAvroSchema(requestedSchema,
sanitizedTableName)
Review Comment:
filter out partition columns that we will read from the parquet. Also get
the indexes in the partition schema so that we can prune the partition column
values to only the remaining partition schema columns
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
