This is an automated email from the ASF dual-hosted git repository.
danny0405 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git
The following commit(s) were added to refs/heads/master by this push:
new 3349839f029 [HUDI-8001] Insert overwrite failed due to missing 'path'
property when using Spark 3.5.1 and Hudi 1.0.0 (#11646)
3349839f029 is described below
commit 3349839f0297b6b0b0933a51f9e2a9e793f03642
Author: majian <[email protected]>
AuthorDate: Sat Jul 20 18:27:08 2024 +0800
[HUDI-8001] Insert overwrite failed due to missing 'path' property when
using Spark 3.5.1 and Hudi 1.0.0 (#11646)
---
.../apache/spark/sql/hudi/command/CreateHoodieTableCommand.scala | 4 ++--
.../test/scala/org/apache/spark/sql/hudi/ddl/TestAlterTable.scala | 8 +++++---
.../scala/org/apache/spark/sql/hudi/ddl/TestCreateTable.scala | 2 +-
3 files changed, 8 insertions(+), 6 deletions(-)
diff --git
a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/command/CreateHoodieTableCommand.scala
b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/command/CreateHoodieTableCommand.scala
index a3e94506b2f..07a9b0945c9 100644
---
a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/command/CreateHoodieTableCommand.scala
+++
b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/command/CreateHoodieTableCommand.scala
@@ -145,8 +145,8 @@ object CreateHoodieTableCommand {
var newTblProperties =
hoodieCatalogTable.catalogProperties.--(needFilterProps) ++
HoodieOptionConfig.extractSqlOptions(properties)
- // Add provider -> hudi as a table property
- newTblProperties = newTblProperties + ("provider" -> "hudi")
+ // Add provider -> hudi and path as a table property
+ newTblProperties = newTblProperties + ("provider" -> "hudi") + ("path" ->
path)
val newTable = table.copy(
identifier = newTableIdentifier,
diff --git
a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/ddl/TestAlterTable.scala
b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/ddl/TestAlterTable.scala
index e09bebc2e9f..1a9a028ded1 100644
---
a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/ddl/TestAlterTable.scala
+++
b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/ddl/TestAlterTable.scala
@@ -17,11 +17,11 @@
package org.apache.spark.sql.hudi.ddl
+import org.apache.hadoop.fs.Path
import org.apache.hudi.HoodieSparkUtils
import org.apache.hudi.common.model.HoodieRecord
import org.apache.hudi.common.table.TableSchemaResolver
import org.apache.hudi.testutils.HoodieClientTestUtils.createMetaClient
-
import org.apache.spark.sql.catalyst.TableIdentifier
import org.apache.spark.sql.hudi.HoodieSqlCommonUtils
import org.apache.spark.sql.hudi.common.HoodieSparkSqlTestBase
@@ -282,10 +282,12 @@ class TestAlterTable extends HoodieSparkSqlTestBase {
spark.sql(s"alter table $locTableName rename to $newLocTableName")
val newLocation2 = spark.sessionState.catalog.getTableMetadata(new
TableIdentifier(newLocTableName))
.properties.get("path")
- // only hoodieCatalog will set path to tblp
if (oldLocation2.nonEmpty) {
+ // Remove the impact of the schema.
+ val oldLocation2Path = new
Path(oldLocation2.get.stripPrefix("file:"))
+ val newLocation2Path = new
Path(newLocation2.get.stripPrefix("file:"))
assertResult(true)(
- newLocation2.equals(oldLocation2)
+ newLocation2Path.equals(oldLocation2Path)
)
} else {
assertResult(None) (newLocation2)
diff --git
a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/ddl/TestCreateTable.scala
b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/ddl/TestCreateTable.scala
index c6274c2de6e..169c99d671c 100644
---
a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/ddl/TestCreateTable.scala
+++
b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/ddl/TestCreateTable.scala
@@ -533,7 +533,7 @@ class TestCreateTable extends HoodieSparkSqlTestBase {
)(table.schema.fields)
// Should not include non.hoodie.property
- assertResult(3)(table.properties.size)
+ assertResult(4)(table.properties.size)
assertResult("cow")(table.properties("type"))
assertResult("id,name")(table.properties("primaryKey"))
assertResult("hudi")(table.properties("provider"))
