This is an automated email from the ASF dual-hosted git repository.
leesf pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git
The following commit(s) were added to refs/heads/master by this push:
new 142adf4ccb [HUDI-4296] Fix the bug that
TestHoodieSparkSqlWriter.testSchemaEvolutionForTableType is flaky (#5973)
142adf4ccb is described below
commit 142adf4ccbf92050dd14d1997c0438565fdfc2cd
Author: xiarixiaoyao <[email protected]>
AuthorDate: Sat Jun 25 21:03:19 2022 +0800
[HUDI-4296] Fix the bug that
TestHoodieSparkSqlWriter.testSchemaEvolutionForTableType is flaky (#5973)
---
.../src/main/scala/org/apache/hudi/BaseFileOnlyRelation.scala | 7 ++++---
1 file changed, 4 insertions(+), 3 deletions(-)
diff --git
a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/BaseFileOnlyRelation.scala
b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/BaseFileOnlyRelation.scala
index 4160c34b0c..d6ec645920 100644
---
a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/BaseFileOnlyRelation.scala
+++
b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/BaseFileOnlyRelation.scala
@@ -166,7 +166,9 @@ class BaseFileOnlyRelation(sqlContext: SQLContext,
DataSource.apply(
sparkSession = sparkSession,
paths = extraReadPaths,
- userSpecifiedSchema = userSchema,
+ // Here we should specify the schema to the latest commit schema since
+ // the table schema evolution.
+ userSpecifiedSchema = userSchema.orElse(Some(tableStructSchema)),
className = formatClassName,
// Since we're reading the table as just collection of files we have
to make sure
// we only read the latest version of every Hudi's file-group, which
might be compacted, clustered, etc.
@@ -175,8 +177,7 @@ class BaseFileOnlyRelation(sqlContext: SQLContext,
// We rely on [[HoodieROTablePathFilter]], to do proper filtering to
assure that
options = optParams ++ Map(
"mapreduce.input.pathFilter.class" ->
classOf[HoodieROTablePathFilter].getName
- ),
- partitionColumns = partitionColumns
+ )
)
.resolveRelation()
.asInstanceOf[HadoopFsRelation]
