This is an automated email from the ASF dual-hosted git repository.
yihua pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git
The following commit(s) were added to refs/heads/master by this push:
new 4ac661a8314c fix: Fix SQL syntax parser for CREATE TABLE on Spark 4.1
(#18779)
4ac661a8314c is described below
commit 4ac661a8314c6dc27ee33b718626c83bd1d4b3e9
Author: Rahil C <[email protected]>
AuthorDate: Tue May 19 16:59:10 2026 -0700
fix: Fix SQL syntax parser for CREATE TABLE on Spark 4.1 (#18779)
---
.../org/apache/spark/sql/hudi/ddl/TestCreateTable.scala | 12 +-----------
.../hudi-spark4.1.x/src/main/antlr4/imports/SqlBase.g4 | 2 +-
.../sql/parser/HoodieSpark4_1ExtendedSqlAstBuilder.scala | 2 +-
3 files changed, 3 insertions(+), 13 deletions(-)
diff --git
a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/ddl/TestCreateTable.scala
b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/ddl/TestCreateTable.scala
index 5062421487f8..ead994342162 100644
---
a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/ddl/TestCreateTable.scala
+++
b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/ddl/TestCreateTable.scala
@@ -17,7 +17,7 @@
package org.apache.spark.sql.hudi.ddl
-import org.apache.hudi.{DataSourceWriteOptions, HoodieSparkUtils}
+import org.apache.hudi.DataSourceWriteOptions
import org.apache.hudi.DataSourceWriteOptions._
import org.apache.hudi.common.model.{HoodieRecord, HoodieTableType,
WriteOperationType}
import org.apache.hudi.common.schema.{HoodieSchema, HoodieSchemaType}
@@ -42,21 +42,11 @@ import
org.apache.spark.sql.hudi.common.HoodieSparkSqlTestBase
import
org.apache.spark.sql.hudi.common.HoodieSparkSqlTestBase.{disableComplexKeygenValidation,
getLastCommitMetadata}
import org.apache.spark.sql.types._
import org.junit.jupiter.api.Assertions.{assertEquals, assertFalse,
assertNull, assertTrue}
-import org.scalatest.{Canceled, Outcome}
import scala.collection.JavaConverters._
class TestCreateTable extends HoodieSparkSqlTestBase {
- // TODO(SPARK-4.1): Re-enable after fixing inline compaction hang on Spark
4.1
- override def withFixture(test: NoArgTest): Outcome = {
- if (HoodieSparkUtils.gteqSpark4_1) {
- Canceled("Disabled on Spark 4.1 due to inline compaction hang during MOR
table creation")
- } else {
- super.withFixture(test)
- }
- }
-
test("Test Create Managed Hoodie Table") {
val databaseName = "hudi_database"
spark.sql(s"create database if not exists $databaseName")
diff --git
a/hudi-spark-datasource/hudi-spark4.1.x/src/main/antlr4/imports/SqlBase.g4
b/hudi-spark-datasource/hudi-spark4.1.x/src/main/antlr4/imports/SqlBase.g4
index 8f15fab16b0a..3572d73af038 100644
--- a/hudi-spark-datasource/hudi-spark4.1.x/src/main/antlr4/imports/SqlBase.g4
+++ b/hudi-spark-datasource/hudi-spark4.1.x/src/main/antlr4/imports/SqlBase.g4
@@ -948,7 +948,7 @@ dataType
| INTERVAL from=(YEAR | MONTH) (TO to=MONTH)?
#yearMonthIntervalDataType
| INTERVAL from=(DAY | HOUR | MINUTE | SECOND)
(TO to=(HOUR | MINUTE | SECOND))?
#dayTimeIntervalDataType
- | identifier ('(' INTEGER_VALUE (',' INTEGER_VALUE)* ')')?
#primitiveDataType
+ | typeName=identifier ('(' (INTEGER_VALUE | identifier) (','
(INTEGER_VALUE | identifier))* ')')? #primitiveDataType
;
qualifiedColTypeWithPositionList
diff --git
a/hudi-spark-datasource/hudi-spark4.1.x/src/main/scala/org/apache/spark/sql/parser/HoodieSpark4_1ExtendedSqlAstBuilder.scala
b/hudi-spark-datasource/hudi-spark4.1.x/src/main/scala/org/apache/spark/sql/parser/HoodieSpark4_1ExtendedSqlAstBuilder.scala
index 2db7472f98a5..bdf78781e699 100644
---
a/hudi-spark-datasource/hudi-spark4.1.x/src/main/scala/org/apache/spark/sql/parser/HoodieSpark4_1ExtendedSqlAstBuilder.scala
+++
b/hudi-spark-datasource/hudi-spark4.1.x/src/main/scala/org/apache/spark/sql/parser/HoodieSpark4_1ExtendedSqlAstBuilder.scala
@@ -2618,7 +2618,7 @@ class HoodieSpark4_1ExtendedSqlAstBuilder(conf: SQLConf,
delegate: ParserInterfa
* Resolve/create a primitive type.
*/
override def visitPrimitiveDataType(ctx: PrimitiveDataTypeContext): DataType
= withOrigin(ctx) {
- val dataType = ctx.identifier.getText.toLowerCase(Locale.ROOT)
+ val dataType = ctx.typeName.getText.toLowerCase(Locale.ROOT)
(dataType, ctx.INTEGER_VALUE().asScala.toList) match {
case ("boolean", Nil) => BooleanType
case ("tinyint" | "byte", Nil) => ByteType
