This is an automated email from the ASF dual-hosted git repository.
xushiyan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git
The following commit(s) were added to refs/heads/master by this push:
new e82c5675554 [HUDI-6411] Make spark SQL options insensitive (#9020)
e82c5675554 is described below
commit e82c5675554f4e94ab7a766558b8ed738d6a0f00
Author: Aditya Goenka <[email protected]>
AuthorDate: Tue Jun 27 21:11:30 2023 +0530
[HUDI-6411] Make spark SQL options insensitive (#9020)
---
.../sql/catalyst/catalog/HoodieCatalogTable.scala | 2 +-
.../apache/spark/sql/hudi/HoodieOptionConfig.scala | 14 +++++++++++-
.../apache/spark/sql/hudi/TestCreateTable.scala | 26 ++++++++++++++++++++++
3 files changed, 40 insertions(+), 2 deletions(-)
diff --git
a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/catalyst/catalog/HoodieCatalogTable.scala
b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/catalyst/catalog/HoodieCatalogTable.scala
index a329a943969..b1ee6924e11 100644
---
a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/catalyst/catalog/HoodieCatalogTable.scala
+++
b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/catalyst/catalog/HoodieCatalogTable.scala
@@ -63,7 +63,7 @@ class HoodieCatalogTable(val spark: SparkSession, var table:
CatalogTable) exten
/**
* properties defined in catalog.
*/
- val catalogProperties: Map[String, String] = table.storage.properties ++
table.properties
+ val catalogProperties: Map[String, String] =
HoodieOptionConfig.makeOptionsCaseInsensitive(table.storage.properties ++
table.properties)
/**
* hoodie table's location.
diff --git
a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/HoodieOptionConfig.scala
b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/HoodieOptionConfig.scala
index a541bbb8e52..d715a108d62 100644
---
a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/HoodieOptionConfig.scala
+++
b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/HoodieOptionConfig.scala
@@ -192,7 +192,6 @@ object HoodieOptionConfig {
// validate primaryKey, preCombineField and type options
def validateTable(spark: SparkSession, schema: StructType, sqlOptions:
Map[String, String]): Unit = {
val resolver = spark.sessionState.conf.resolver
-
// validate primary key
val primaryKeys = sqlOptions.get(SQL_KEY_TABLE_PRIMARY_KEY.sqlKeyName)
.map(_.split(",").filter(_.length > 0))
@@ -222,6 +221,19 @@ object HoodieOptionConfig {
def buildConf[T](): HoodieSQLOptionBuilder[T] = {
new HoodieSQLOptionBuilder[T]
}
+
+ def makeOptionsCaseInsensitive(sqlOptions: Map[String, String]): Map[String,
String] = {
+ // Make Keys Case Insensitive
+ val standardOptions = Seq(SQL_KEY_TABLE_PRIMARY_KEY,
SQL_KEY_PRECOMBINE_FIELD,
+ SQL_KEY_TABLE_TYPE, SQL_PAYLOAD_CLASS, SQL_RECORD_MERGER_STRATEGY).map(key
=> key.sqlKeyName)
+
+ sqlOptions.map(option => {
+ standardOptions.find(x =>
x.toLowerCase().contains(option._1.toLowerCase())) match {
+ case Some(standardKey) => (standardKey, option._2)
+ case None => (option._1, option._2)
+ }
+ })
+ }
}
case class HoodieSQLOption[T](
diff --git
a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/TestCreateTable.scala
b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/TestCreateTable.scala
index 11d21d43417..7cfc143be05 100644
---
a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/TestCreateTable.scala
+++
b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/TestCreateTable.scala
@@ -1106,4 +1106,30 @@ class TestCreateTable extends HoodieSparkSqlTestBase {
val hoodieSchema =
HoodieSqlCommonUtils.getTableSqlSchema(hoodieCatalogTable.metaClient, true)
assertResult(hoodieSchema.get)(table.schema)
}
+
+ test("Test Create Hoodie Table With Options in different case") {
+ val tableName = generateTableName
+ spark.sql(
+ s"""
+ | create table $tableName (
+ | id int,
+ | name string,
+ | price double,
+ | ts long,
+ | dt string
+ | ) using hudi
+ | partitioned by (dt)
+ | options (
+ | hoodie.database.name = "databaseName",
+ | hoodie.table.name = "tableName",
+ | PRIMARYKEY = 'id',
+ | precombineField = 'ts',
+ | hoodie.datasource.write.operation = 'upsert'
+ | )
+ """.stripMargin)
+ val table =
spark.sessionState.catalog.getTableMetadata(TableIdentifier(tableName))
+ assertResult(table.properties("type"))("cow")
+ assertResult(table.properties("primaryKey"))("id")
+ assertResult(table.properties("preCombineField"))("ts")
+ }
}
