This is an automated email from the ASF dual-hosted git repository. codope pushed a commit to branch release-0.12.0 in repository https://gitbox.apache.org/repos/asf/hudi.git
commit cd14c26ea52017bd3472710f7769f71f2bcc68be Author: Rahil Chertara <[email protected]> AuthorDate: Tue Aug 2 15:27:40 2022 -0700 [HUDI-4470] Remove spark dataPrefetch disabled prop in DefaultSource --- .../src/main/scala/org/apache/hudi/DefaultSource.scala | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/DefaultSource.scala b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/DefaultSource.scala index cbe78b13bd..d5cbf020ed 100644 --- a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/DefaultSource.scala +++ b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/DefaultSource.scala @@ -57,8 +57,7 @@ class DefaultSource extends RelationProvider // Enable "passPartitionByAsOptions" to support "write.partitionBy(...)" spark.conf.set("spark.sql.legacy.sources.write.passPartitionByAsOptions", "true") } - // Revisit EMR Spark and EMRFS incompatibilities, for now disable - spark.conf.set("spark.sql.dataPrefetch.enabled", "false") + // Revisit EMRFS incompatibilities, for now disable spark.sparkContext.hadoopConfiguration.set("fs.s3.metadata.cache.expiration.seconds", "0") }
