This is an automated email from the ASF dual-hosted git repository.
yihua pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git
The following commit(s) were added to refs/heads/master by this push:
new 380cf7d06f5 [HUDI-7276] Fix schema used for constructing file group
reader on Spark (#12055)
380cf7d06f5 is described below
commit 380cf7d06f510372086e543caa88b50d55061dae
Author: Lin Liu <[email protected]>
AuthorDate: Sat Oct 5 08:43:47 2024 -0700
[HUDI-7276] Fix schema used for constructing file group reader on Spark
(#12055)
---
.../parquet/HoodieFileGroupReaderBasedParquetFileFormat.scala | 6 ++----
1 file changed, 2 insertions(+), 4 deletions(-)
diff --git
a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/HoodieFileGroupReaderBasedParquetFileFormat.scala
b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/HoodieFileGroupReaderBasedParquetFileFormat.scala
index 6422d39e976..84df68058bd 100644
---
a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/HoodieFileGroupReaderBasedParquetFileFormat.scala
+++
b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/HoodieFileGroupReaderBasedParquetFileFormat.scala
@@ -152,8 +152,6 @@ class
HoodieFileGroupReaderBasedParquetFileFormat(tableState: HoodieTableState,
val parquetFileReader =
spark.sparkContext.broadcast(sparkAdapter.createParquetFileReader(supportBatchResult,
spark.sessionState.conf, options, augmentedStorageConf.unwrap()))
val broadcastedStorageConf = spark.sparkContext.broadcast(new
SerializableConfiguration(augmentedStorageConf.unwrap()))
- val broadcastedDataSchema = spark.sparkContext.broadcast(dataAvroSchema)
- val broadcastedRequestedSchema =
spark.sparkContext.broadcast(requestedAvroSchema)
val fileIndexProps: TypedProperties =
HoodieFileIndex.getConfigProperties(spark, options, null)
(file: PartitionedFile) => {
@@ -176,8 +174,8 @@ class
HoodieFileGroupReaderBasedParquetFileFormat(tableState: HoodieTableState,
tableState.tablePath,
tableState.latestCommitTimestamp.get,
fileSlice,
- broadcastedDataSchema.value,
- broadcastedRequestedSchema.value,
+ dataAvroSchema,
+ requestedAvroSchema,
internalSchemaOpt,
metaClient,
props,
