voonhous commented on code in PR #19408:
URL: https://github.com/apache/hudi/pull/19408#discussion_r3690889328
##########
hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/dml/schema/TestBlobDataType.scala:
##########
@@ -282,4 +284,256 @@ class TestBlobDataType extends HoodieSparkSqlTestBase {
"Expected at least one .clean instant on the timeline after
compaction")
})
}
+
+ // The following cases are parser-coverage only: a BLOB column routes the
whole CREATE TABLE
+ // through the extended AST builder, so its clause visitors run. parsePlan
is purely syntactic
+ // (no catalog, no execution), which lets us exercise clauses Hudi does not
support at execution
+ // time (transform partitioning, STORED AS / ROW FORMAT, interval columns).
The BLOB column type
+ // itself proves routing because the stock Spark parser rejects the BLOB
type name.
+
+ private def parse(sql: String): CreateTable =
+ spark.sessionState.sqlParser.parsePlan(sql).asInstanceOf[CreateTable]
+
+ test("Test parse CREATE TABLE with BLOB column and primitive data types") {
+ // Exercises the primitive-data-type match arms plus NOT NULL and column
COMMENT.
+ val plan = parse(
+ s"""
+ |CREATE TABLE blob_prim_tbl (
+ | c_bool BOOLEAN,
+ | c_tiny TINYINT,
+ | c_small SMALLINT,
+ | c_int INT,
+ | c_big BIGINT,
+ | c_float FLOAT,
+ | c_double DOUBLE,
+ | c_date DATE,
+ | c_str STRING,
+ | c_char CHAR(5),
+ | c_varchar VARCHAR(10),
+ | c_bin BINARY,
+ | c_dec DECIMAL,
+ | c_dec1 DECIMAL(12),
+ | c_dec2 DECIMAL(12, 3),
+ | c_notnull INT NOT NULL,
+ | c_comment INT COMMENT 'a comment',
+ | data BLOB
+ |) USING hudi
+ """.stripMargin)
+ val schema = plan.tableSchema
+ assertResult(BooleanType)(schema("c_bool").dataType)
+ assertResult(ByteType)(schema("c_tiny").dataType)
+ assertResult(ShortType)(schema("c_small").dataType)
+ assertResult(IntegerType)(schema("c_int").dataType)
+ assertResult(LongType)(schema("c_big").dataType)
+ assertResult(FloatType)(schema("c_float").dataType)
+ assertResult(DoubleType)(schema("c_double").dataType)
+ assertResult(DateType)(schema("c_date").dataType)
+ assertResult(StringType)(schema("c_str").dataType)
+ // CHAR/VARCHAR may be preserved or replaced with STRING depending on the
Spark version.
+ assert(Seq[DataType](CharType(5),
StringType).contains(schema("c_char").dataType))
+ assert(Seq[DataType](VarcharType(10),
StringType).contains(schema("c_varchar").dataType))
+ assertResult(BinaryType)(schema("c_bin").dataType)
+ assertResult(DecimalType(10, 0))(schema("c_dec").dataType)
+ assertResult(DecimalType(12, 0))(schema("c_dec1").dataType)
+ assertResult(DecimalType(12, 3))(schema("c_dec2").dataType)
+ assertResult(BlobType())(schema("data").dataType)
+ assert(!schema("c_notnull").nullable)
+ assertResult("a
comment")(schema("c_comment").metadata.getString("comment"))
+ }
+
+ test("Test parse CREATE TABLE with BLOB column and complex data types") {
+ // Exercises the ARRAY / MAP / STRUCT arms and BLOB-in-struct metadata
handling.
+ val plan = parse(
+ s"""
+ |CREATE TABLE blob_complex_tbl (
+ | c_arr ARRAY<INT>,
+ | c_map MAP<STRING, INT>,
+ | c_struct STRUCT<a: INT, b: STRING>,
+ | c_nested_blob STRUCT<x: BLOB>,
+ | data BLOB
+ |) USING hudi
+ """.stripMargin)
+ val schema = plan.tableSchema
+ assertResult(ArrayType(IntegerType))(schema("c_arr").dataType)
+ assertResult(MapType(StringType, IntegerType))(schema("c_map").dataType)
+ val inner = schema("c_struct").dataType.asInstanceOf[StructType]
+ assertResult(IntegerType)(inner("a").dataType)
+ assertResult(StringType)(inner("b").dataType)
+ // A BLOB nested inside a struct still carries the BLOB type descriptor.
+ val nested = schema("c_nested_blob").dataType.asInstanceOf[StructType]("x")
+ assertResult(BlobType())(nested.dataType)
+ assertResult(HoodieSchemaType.BLOB.name())(
+ nested.metadata.getString(HoodieSchema.TYPE_METADATA_FIELD))
+ }
+
+ test("Test parse CREATE TABLE with BLOB column and interval data types") {
+ val plan = parse(
+ s"""
+ |CREATE TABLE blob_ivl_tbl (
+ | i_year INTERVAL YEAR,
+ | i_ym INTERVAL YEAR TO MONTH,
+ | i_day INTERVAL DAY,
+ | i_ds INTERVAL DAY TO SECOND,
+ | data BLOB
+ |) USING hudi
+ """.stripMargin)
+ val schema = plan.tableSchema
+
assertResult(YearMonthIntervalType(YearMonthIntervalType.YEAR))(schema("i_year").dataType)
+ assertResult(YearMonthIntervalType(YearMonthIntervalType.YEAR,
YearMonthIntervalType.MONTH))(
+ schema("i_ym").dataType)
+
assertResult(DayTimeIntervalType(DayTimeIntervalType.DAY))(schema("i_day").dataType)
+ assertResult(DayTimeIntervalType(DayTimeIntervalType.DAY,
DayTimeIntervalType.SECOND))(
+ schema("i_ds").dataType)
+ assertResult(BlobType())(schema("data").dataType)
+
+ // Endpoints where the end field does not follow the start are rejected by
both interval
+ // data-type visitors. The grammar only allows YEAR/MONTH -> MONTH and
DAY/HOUR/MINUTE/SECOND
+ // -> HOUR/MINUTE/SECOND, so these stay grammatical yet still hit the
builder's end <= start guard.
+ checkExceptionContain(
+ "CREATE TABLE blob_bad_ym (id BIGINT, bad INTERVAL MONTH TO MONTH, data
BLOB) USING hudi")(
+ "are not supported")
+ checkExceptionContain(
+ "CREATE TABLE blob_bad_dt (id BIGINT, bad INTERVAL SECOND TO HOUR, data
BLOB) USING hudi")(
+ "are not supported")
+
+ // An unknown primitive type name is rejected.
+ checkExceptionContain(
+ "CREATE TABLE blob_bad_type (id BIGINT, weird sometype, data BLOB) USING
hudi")(
+ "is not supported")
+ }
+
+ test("Test parse CREATE TABLE with BLOB column and partition transforms") {
+ val plan = parse(
+ s"""
+ |CREATE TABLE blob_tf_tbl (
+ | id BIGINT,
+ | ts DATE,
+ | region STRING,
+ | data BLOB
+ |) USING hudi
+ |PARTITIONED BY (region, years(ts), months(ts), days(ts), hours(ts),
myfunc(id))
+ """.stripMargin)
+ assertResult(BlobType())(plan.tableSchema("data").dataType)
+ assertResult(Seq(Seq("region")))(transformFieldRefs(transformByName(plan,
"identity")))
+ assertResult(Seq(Seq("ts")))(transformFieldRefs(transformByName(plan,
"years")))
+ assertResult(Seq(Seq("ts")))(transformFieldRefs(transformByName(plan,
"months")))
+ assertResult(Seq(Seq("ts")))(transformFieldRefs(transformByName(plan,
"days")))
+ assertResult(Seq(Seq("ts")))(transformFieldRefs(transformByName(plan,
"hours")))
+ // an arbitrary function transform falls through to the generic
apply-transform arm
+ assertResult(Seq(Seq("id")))(transformFieldRefs(transformByName(plan,
"myfunc")))
+
+ // bucket(numBuckets, col) with int, long and short number-of-buckets
literals exercises the
+ // three numeric arms of the bucket handling.
+ Seq("4", "4L", "4S").foreach { numLiteral =>
+ val bp = parse(
+ s"CREATE TABLE blob_bkt_tbl (id BIGINT, data BLOB) USING hudi " +
+ s"PARTITIONED BY (bucket($numLiteral, id))")
+ val bkt = transformByName(bp, "bucket")
+ assertResult("4")(firstLiteralArg(bkt).value.toString)
+ assertResult(Seq(Seq("id")))(transformFieldRefs(bkt))
+ }
+ }
+
+ test("Test parse CREATE TABLE with BLOB column and typed transform-argument
literals") {
+ // Constant transform arguments exercise the literal visitors: string,
integer, big-integer and
+ // exponent numerics (the private numeric-literal helper), the typed date
constructor, and both
+ // interval forms (multi-unit and unit-to-unit). A typed timestamp
constructor is not used
+ // because the Spark 4.x extended parser rejects a bare TIMESTAMP token.
Note: a bare
Review Comment:
The root cause here is not the Spark 4.x parser -- it is Hudi's wiring. All
six `HoodieSpark*ExtendedSqlParser` set `parser.SQL_standard_keyword_behavior =
conf.ansiEnabled`, while Spark's own parser sets it from
`conf.enforceReservedKeywords` (`spark.sql.ansi.enforceReservedKeywords`,
default **false** even under ANSI). Spark 4.x defaults `ansi.enabled=true`, so
the fork enforces reserved keywords that stock Spark does not, and `CREATE
TABLE t (ts TIMESTAMP, data BLOB) USING hudi` fails with `no viable alternative
at input 'TIMESTAMP'` while the same statement minus the BLOB column parses
fine. Verified the 3.5 and regenerated 4.1 forked grammars behave identically
once the flag matches; the blast radius also includes common column names
(`user`, `left`, `filter`, `some`) in any blob/vector statement.
Action: please file a GitHub issue for the one-line fix (use
`conf.enforceReservedKeywords` in the six parsers) and reword this comment to
cite it as a tracked bug, not a Spark 4 constraint.
##########
hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/dml/schema/TestBlobDataType.scala:
##########
@@ -282,4 +284,256 @@ class TestBlobDataType extends HoodieSparkSqlTestBase {
"Expected at least one .clean instant on the timeline after
compaction")
})
}
+
+ // The following cases are parser-coverage only: a BLOB column routes the
whole CREATE TABLE
+ // through the extended AST builder, so its clause visitors run. parsePlan
is purely syntactic
+ // (no catalog, no execution), which lets us exercise clauses Hudi does not
support at execution
+ // time (transform partitioning, STORED AS / ROW FORMAT, interval columns).
The BLOB column type
+ // itself proves routing because the stock Spark parser rejects the BLOB
type name.
+
+ private def parse(sql: String): CreateTable =
+ spark.sessionState.sqlParser.parsePlan(sql).asInstanceOf[CreateTable]
+
+ test("Test parse CREATE TABLE with BLOB column and primitive data types") {
+ // Exercises the primitive-data-type match arms plus NOT NULL and column
COMMENT.
+ val plan = parse(
+ s"""
+ |CREATE TABLE blob_prim_tbl (
+ | c_bool BOOLEAN,
+ | c_tiny TINYINT,
+ | c_small SMALLINT,
+ | c_int INT,
+ | c_big BIGINT,
+ | c_float FLOAT,
+ | c_double DOUBLE,
+ | c_date DATE,
+ | c_str STRING,
+ | c_char CHAR(5),
+ | c_varchar VARCHAR(10),
+ | c_bin BINARY,
+ | c_dec DECIMAL,
+ | c_dec1 DECIMAL(12),
+ | c_dec2 DECIMAL(12, 3),
+ | c_notnull INT NOT NULL,
+ | c_comment INT COMMENT 'a comment',
+ | data BLOB
+ |) USING hudi
+ """.stripMargin)
+ val schema = plan.tableSchema
+ assertResult(BooleanType)(schema("c_bool").dataType)
+ assertResult(ByteType)(schema("c_tiny").dataType)
+ assertResult(ShortType)(schema("c_small").dataType)
+ assertResult(IntegerType)(schema("c_int").dataType)
+ assertResult(LongType)(schema("c_big").dataType)
+ assertResult(FloatType)(schema("c_float").dataType)
+ assertResult(DoubleType)(schema("c_double").dataType)
+ assertResult(DateType)(schema("c_date").dataType)
+ assertResult(StringType)(schema("c_str").dataType)
+ // CHAR/VARCHAR may be preserved or replaced with STRING depending on the
Spark version.
+ assert(Seq[DataType](CharType(5),
StringType).contains(schema("c_char").dataType))
+ assert(Seq[DataType](VarcharType(10),
StringType).contains(schema("c_varchar").dataType))
+ assertResult(BinaryType)(schema("c_bin").dataType)
+ assertResult(DecimalType(10, 0))(schema("c_dec").dataType)
+ assertResult(DecimalType(12, 0))(schema("c_dec1").dataType)
+ assertResult(DecimalType(12, 3))(schema("c_dec2").dataType)
+ assertResult(BlobType())(schema("data").dataType)
+ assert(!schema("c_notnull").nullable)
+ assertResult("a
comment")(schema("c_comment").metadata.getString("comment"))
+ }
+
+ test("Test parse CREATE TABLE with BLOB column and complex data types") {
+ // Exercises the ARRAY / MAP / STRUCT arms and BLOB-in-struct metadata
handling.
+ val plan = parse(
+ s"""
+ |CREATE TABLE blob_complex_tbl (
+ | c_arr ARRAY<INT>,
+ | c_map MAP<STRING, INT>,
+ | c_struct STRUCT<a: INT, b: STRING>,
+ | c_nested_blob STRUCT<x: BLOB>,
+ | data BLOB
+ |) USING hudi
+ """.stripMargin)
+ val schema = plan.tableSchema
+ assertResult(ArrayType(IntegerType))(schema("c_arr").dataType)
+ assertResult(MapType(StringType, IntegerType))(schema("c_map").dataType)
+ val inner = schema("c_struct").dataType.asInstanceOf[StructType]
+ assertResult(IntegerType)(inner("a").dataType)
+ assertResult(StringType)(inner("b").dataType)
+ // A BLOB nested inside a struct still carries the BLOB type descriptor.
+ val nested = schema("c_nested_blob").dataType.asInstanceOf[StructType]("x")
+ assertResult(BlobType())(nested.dataType)
+ assertResult(HoodieSchemaType.BLOB.name())(
+ nested.metadata.getString(HoodieSchema.TYPE_METADATA_FIELD))
+ }
+
+ test("Test parse CREATE TABLE with BLOB column and interval data types") {
+ val plan = parse(
+ s"""
+ |CREATE TABLE blob_ivl_tbl (
+ | i_year INTERVAL YEAR,
+ | i_ym INTERVAL YEAR TO MONTH,
+ | i_day INTERVAL DAY,
+ | i_ds INTERVAL DAY TO SECOND,
+ | data BLOB
+ |) USING hudi
+ """.stripMargin)
+ val schema = plan.tableSchema
+
assertResult(YearMonthIntervalType(YearMonthIntervalType.YEAR))(schema("i_year").dataType)
+ assertResult(YearMonthIntervalType(YearMonthIntervalType.YEAR,
YearMonthIntervalType.MONTH))(
+ schema("i_ym").dataType)
+
assertResult(DayTimeIntervalType(DayTimeIntervalType.DAY))(schema("i_day").dataType)
+ assertResult(DayTimeIntervalType(DayTimeIntervalType.DAY,
DayTimeIntervalType.SECOND))(
+ schema("i_ds").dataType)
+ assertResult(BlobType())(schema("data").dataType)
+
+ // Endpoints where the end field does not follow the start are rejected by
both interval
+ // data-type visitors. The grammar only allows YEAR/MONTH -> MONTH and
DAY/HOUR/MINUTE/SECOND
+ // -> HOUR/MINUTE/SECOND, so these stay grammatical yet still hit the
builder's end <= start guard.
+ checkExceptionContain(
+ "CREATE TABLE blob_bad_ym (id BIGINT, bad INTERVAL MONTH TO MONTH, data
BLOB) USING hudi")(
+ "are not supported")
+ checkExceptionContain(
+ "CREATE TABLE blob_bad_dt (id BIGINT, bad INTERVAL SECOND TO HOUR, data
BLOB) USING hudi")(
+ "are not supported")
+
+ // An unknown primitive type name is rejected.
+ checkExceptionContain(
+ "CREATE TABLE blob_bad_type (id BIGINT, weird sometype, data BLOB) USING
hudi")(
Review Comment:
This arm is already covered: a bare `v VECTOR` is `("vector", Nil)` and
falls into the same `case (dt, params) => throw ... "DataType $dtStr is not
supported."` arm, asserted by `test create table with VECTOR without dimension
fails` in `TestCreateTable`. And `"is not supported"` is generic enough to
match almost any error text.
Action: drop these three lines; if you prefer to keep them, tighten the
assertion to `"DataType sometype is not supported"`.
##########
hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/dml/schema/TestBlobDataType.scala:
##########
@@ -282,4 +284,256 @@ class TestBlobDataType extends HoodieSparkSqlTestBase {
"Expected at least one .clean instant on the timeline after
compaction")
})
}
+
+ // The following cases are parser-coverage only: a BLOB column routes the
whole CREATE TABLE
+ // through the extended AST builder, so its clause visitors run. parsePlan
is purely syntactic
+ // (no catalog, no execution), which lets us exercise clauses Hudi does not
support at execution
+ // time (transform partitioning, STORED AS / ROW FORMAT, interval columns).
The BLOB column type
+ // itself proves routing because the stock Spark parser rejects the BLOB
type name.
+
+ private def parse(sql: String): CreateTable =
+ spark.sessionState.sqlParser.parsePlan(sql).asInstanceOf[CreateTable]
+
+ test("Test parse CREATE TABLE with BLOB column and primitive data types") {
+ // Exercises the primitive-data-type match arms plus NOT NULL and column
COMMENT.
+ val plan = parse(
+ s"""
+ |CREATE TABLE blob_prim_tbl (
+ | c_bool BOOLEAN,
+ | c_tiny TINYINT,
+ | c_small SMALLINT,
+ | c_int INT,
+ | c_big BIGINT,
+ | c_float FLOAT,
+ | c_double DOUBLE,
+ | c_date DATE,
+ | c_str STRING,
+ | c_char CHAR(5),
+ | c_varchar VARCHAR(10),
+ | c_bin BINARY,
+ | c_dec DECIMAL,
+ | c_dec1 DECIMAL(12),
+ | c_dec2 DECIMAL(12, 3),
+ | c_notnull INT NOT NULL,
+ | c_comment INT COMMENT 'a comment',
+ | data BLOB
+ |) USING hudi
+ """.stripMargin)
+ val schema = plan.tableSchema
+ assertResult(BooleanType)(schema("c_bool").dataType)
+ assertResult(ByteType)(schema("c_tiny").dataType)
+ assertResult(ShortType)(schema("c_small").dataType)
+ assertResult(IntegerType)(schema("c_int").dataType)
+ assertResult(LongType)(schema("c_big").dataType)
+ assertResult(FloatType)(schema("c_float").dataType)
+ assertResult(DoubleType)(schema("c_double").dataType)
+ assertResult(DateType)(schema("c_date").dataType)
+ assertResult(StringType)(schema("c_str").dataType)
+ // CHAR/VARCHAR may be preserved or replaced with STRING depending on the
Spark version.
+ assert(Seq[DataType](CharType(5),
StringType).contains(schema("c_char").dataType))
+ assert(Seq[DataType](VarcharType(10),
StringType).contains(schema("c_varchar").dataType))
+ assertResult(BinaryType)(schema("c_bin").dataType)
+ assertResult(DecimalType(10, 0))(schema("c_dec").dataType)
+ assertResult(DecimalType(12, 0))(schema("c_dec1").dataType)
+ assertResult(DecimalType(12, 3))(schema("c_dec2").dataType)
+ assertResult(BlobType())(schema("data").dataType)
+ assert(!schema("c_notnull").nullable)
+ assertResult("a
comment")(schema("c_comment").metadata.getString("comment"))
+ }
+
+ test("Test parse CREATE TABLE with BLOB column and complex data types") {
+ // Exercises the ARRAY / MAP / STRUCT arms and BLOB-in-struct metadata
handling.
+ val plan = parse(
+ s"""
+ |CREATE TABLE blob_complex_tbl (
+ | c_arr ARRAY<INT>,
+ | c_map MAP<STRING, INT>,
+ | c_struct STRUCT<a: INT, b: STRING>,
+ | c_nested_blob STRUCT<x: BLOB>,
Review Comment:
nit, feel free to ignore: `c_nested_blob` duplicates existing coverage --
`test BLOB in nested struct` in `TestCreateTable` (~line 2117) already asserts
both `BlobType()` and `TYPE_METADATA_FIELD == BLOB` for a `STRUCT` field
through the same builder path. The ARRAY arm is the genuinely new part of this
test. Consider dropping the field and its two assertions.
##########
hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/dml/schema/TestBlobDataType.scala:
##########
@@ -282,4 +284,256 @@ class TestBlobDataType extends HoodieSparkSqlTestBase {
"Expected at least one .clean instant on the timeline after
compaction")
})
}
+
+ // The following cases are parser-coverage only: a BLOB column routes the
whole CREATE TABLE
Review Comment:
The PR description's "every remaining path is reachable ... no further prune
is warranted" does not hold. Post-#19132 the six builders still carry provably
dead code: `createSchema`, `createStructType`, both `visitPropertyKeys`
overloads (only referenced from scaladoc), `visitBooleanLiteral` (unreachable
in both keyword modes), grammar-guaranteed-unreachable branches
(`getSingleFieldReference` empty-args arm, `visitTransformArgument.getOrElse`,
`visitCreateFileFormat` `case _`, `validateRowFormatFileFormat` `case _`), and
~8 unused imports per builder.
Action: delete these in a small follow-up (or a cleanup commit here) and
adjust the PR description claim.
##########
hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/ddl/TestCreateTable.scala:
##########
@@ -2340,6 +2341,140 @@ class TestCreateTable extends HoodieSparkSqlTestBase {
}
}
+ // The following cases are parser-coverage only: a VECTOR column routes the
whole CREATE TABLE
+ // through the extended AST builder, so its clause visitors run. parsePlan
is purely syntactic
+ // (no catalog, no execution), matching how TestIndexSyntax exercises the
index statements, which
+ // lets us cover clauses that are not supported at execution time (transform
partitioning,
+ // CLUSTERED BY, typed literal arguments). The VECTOR column type proves the
statement routed
+ // here because the stock Spark parser rejects the VECTOR type name.
+
+ private def parseCreateTable(sql: String): CreateTable =
+ spark.sessionState.sqlParser.parsePlan(sql).asInstanceOf[CreateTable]
+
+ test("test create VECTOR table with partition transforms parses (parser
coverage)") {
Review Comment:
These two tests (this one and the typed-literals one below) exercise exactly
the production lines their BLOB twins in `TestBlobDataType` already cover: the
builder branches on blob-vs-vector only in `visitPrimitiveDataType` (3.5
builder lines 572-573); every clause visitor is type-agnostic. The literals
test is a line-for-line copy (same seven literal forms -- the PR description's
claim that the VECTOR one adds long/exponent/interval variants does not hold,
the BLOB copy has them all), and this transforms test is a strict subset:
`truncate` has no dedicated arm (`grep 'case "truncate"'` over the six builders
returns nothing), so it lands in the same generic-apply case as `myfunc`. Same
shape that got `TestExtendedSqlParserCoverage` removed in the #19218 review.
Action: drop both tests (lines 2354-2419) and keep the BLOB copies; if you
want a VECTOR routing smoke check, a single bucket assertion inside the
CLUSTERED BY test below is enough.
##########
hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/ddl/TestCreateTable.scala:
##########
@@ -2340,6 +2341,140 @@ class TestCreateTable extends HoodieSparkSqlTestBase {
}
}
+ // The following cases are parser-coverage only: a VECTOR column routes the
whole CREATE TABLE
+ // through the extended AST builder, so its clause visitors run. parsePlan
is purely syntactic
+ // (no catalog, no execution), matching how TestIndexSyntax exercises the
index statements, which
+ // lets us cover clauses that are not supported at execution time (transform
partitioning,
+ // CLUSTERED BY, typed literal arguments). The VECTOR column type proves the
statement routed
+ // here because the stock Spark parser rejects the VECTOR type name.
+
+ private def parseCreateTable(sql: String): CreateTable =
+ spark.sessionState.sqlParser.parsePlan(sql).asInstanceOf[CreateTable]
+
+ test("test create VECTOR table with partition transforms parses (parser
coverage)") {
+ val plan = parseCreateTable(
+ s"""
+ |CREATE TABLE vec_parse_tbl (
+ | id BIGINT,
+ | ts DATE,
+ | region STRING,
+ | name STRING,
+ | embedding VECTOR(8)
+ |) USING hudi
+ |PARTITIONED BY (region, bucket(4, id), years(ts), truncate(10, name))
+ """.stripMargin)
+
+ assertEquals(ArrayType(FloatType, containsNull = false),
plan.tableSchema("embedding").dataType)
+
+ assertEquals(Seq(Seq("region")), transformFieldRefs(transformByName(plan,
"identity")))
+ val bucket = transformByName(plan, "bucket")
+ assertEquals(IntegerType, firstLiteralArg(bucket).dataType)
+ assertEquals("4", firstLiteralArg(bucket).value.toString)
+ assertEquals(Seq(Seq("id")), transformFieldRefs(bucket))
+ assertEquals(Seq(Seq("ts")), transformFieldRefs(transformByName(plan,
"years")))
+ val truncate = transformByName(plan, "truncate")
+ assertEquals(IntegerType, firstLiteralArg(truncate).dataType)
+ assertEquals("10", firstLiteralArg(truncate).value.toString)
+ assertEquals(Seq(Seq("name")), transformFieldRefs(truncate))
+ }
+
+ test("test create VECTOR table with typed transform-argument literals parses
(parser coverage)") {
+ // Each transform carries a differently-typed constant argument to
exercise the literal
+ // visitors: string, integer, long, exponent/double, the typed date
constructor, and both
+ // interval forms (multi-units and unit-to-unit). A typed timestamp
constructor is not used
+ // because the Spark 4.x extended parser rejects a bare TIMESTAMP token. A
bare true/false/null
+ // in this position is parsed as a column reference (qualifiedName takes
precedence over a
+ // constant in the grammar under the default non-ANSI config), so the
boolean and null literal
+ // visitors are not reachable from a CREATE TABLE statement.
+ val plan = parseCreateTable(
+ s"""
+ |CREATE TABLE vec_lit_tbl (
+ | id BIGINT,
+ | embedding VECTOR(4)
+ |) USING hudi
+ |PARTITIONED BY (
+ | str_t('x', id),
+ | int_t(4, id),
+ | long_t(9000000000L, id),
+ | exp_t(1E3, id),
+ | date_t(DATE '2020-01-01', id),
+ | mu_ivl_t(INTERVAL '1' DAY, id),
+ | uu_ivl_t(INTERVAL '1-2' YEAR TO MONTH, id)
+ |)
+ """.stripMargin)
+
+ assertEquals(StringType, firstLiteralArg(transformByName(plan,
"str_t")).dataType)
+ assertEquals("x", firstLiteralArg(transformByName(plan,
"str_t")).value.toString)
+ assertEquals(IntegerType, firstLiteralArg(transformByName(plan,
"int_t")).dataType)
+ assertEquals(LongType, firstLiteralArg(transformByName(plan,
"long_t")).dataType)
+ assertEquals(DoubleType, firstLiteralArg(transformByName(plan,
"exp_t")).dataType)
+ assertEquals(DateType, firstLiteralArg(transformByName(plan,
"date_t")).dataType)
+ assertEquals(
+ DayTimeIntervalType(DayTimeIntervalType.DAY, DayTimeIntervalType.DAY),
+ firstLiteralArg(transformByName(plan, "mu_ivl_t")).dataType)
+ assertEquals(
+ YearMonthIntervalType(YearMonthIntervalType.YEAR,
YearMonthIntervalType.MONTH),
+ firstLiteralArg(transformByName(plan, "uu_ivl_t")).dataType)
+ }
+
+ test("test create VECTOR table with CLUSTERED BY bucket spec parses (parser
coverage)") {
+ val plan = parseCreateTable(
+ "CREATE TABLE vec_bucket_tbl (id BIGINT, embedding VECTOR(4)) USING hudi
" +
+ "CLUSTERED BY (id) INTO 4 BUCKETS")
+ val bucket = transformByName(plan, "bucket")
+ assertEquals("4", firstLiteralArg(bucket).value.toString)
+ assertEquals(Seq(Seq("id")), transformFieldRefs(bucket))
+
+ // SORTED BY ... ASC is accepted by the bucket-spec visitor.
+ val sortedPlan = parseCreateTable(
+ "CREATE TABLE vec_sbucket_tbl (id BIGINT, ts BIGINT, embedding
VECTOR(4)) USING hudi " +
+ "CLUSTERED BY (id, ts) SORTED BY (id ASC) INTO 8 BUCKETS")
+
assertTrue(sortedPlan.partitioning.exists(_.name.toLowerCase.contains("bucket")))
+
+ // SORTED BY ... DESC is rejected by the bucket-spec visitor.
+ checkExceptionContain(
+ "CREATE TABLE vec_bad_bucket_tbl (id BIGINT, embedding VECTOR(4)) USING
hudi " +
+ "CLUSTERED BY (id) SORTED BY (id DESC) INTO 4 BUCKETS")(
+ "Column ordering must be ASC")
+ }
+
+ test("test create VECTOR table with LOCATION/COMMENT/OPTIONS/TBLPROPERTIES
parses (parser coverage)") {
Review Comment:
Neither positive case asserts any of the four clauses in the test name --
both check only `embedding`'s dataType, so a builder that dropped
LOCATION/COMMENT/OPTIONS/TBLPROPERTIES entirely would pass, and the "path
folded into location" behavior claimed below is never asserted. The fields are
on the plan and portable across all six profiles (3.3/3.4 `TableSpec`, 3.5+
`TableSpecBase`: `location`/`comment`/`properties`; note `.options` is **not**
on `TableSpecBase`, so avoid it).
Action: assert `tableSpec.comment == Some("a vector table")`,
`tableSpec.location == Some("/tmp/vec_clause_tbl")`, the three `prop_*` entries
in `tableSpec.properties`, and `pathPlan.tableSpec.location ==
Some("/tmp/vec_path_tbl")` for the folding case.
##########
hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/dml/schema/TestBlobDataType.scala:
##########
@@ -282,4 +284,256 @@ class TestBlobDataType extends HoodieSparkSqlTestBase {
"Expected at least one .clean instant on the timeline after
compaction")
})
}
+
+ // The following cases are parser-coverage only: a BLOB column routes the
whole CREATE TABLE
+ // through the extended AST builder, so its clause visitors run. parsePlan
is purely syntactic
+ // (no catalog, no execution), which lets us exercise clauses Hudi does not
support at execution
+ // time (transform partitioning, STORED AS / ROW FORMAT, interval columns).
The BLOB column type
+ // itself proves routing because the stock Spark parser rejects the BLOB
type name.
+
+ private def parse(sql: String): CreateTable =
+ spark.sessionState.sqlParser.parsePlan(sql).asInstanceOf[CreateTable]
+
+ test("Test parse CREATE TABLE with BLOB column and primitive data types") {
+ // Exercises the primitive-data-type match arms plus NOT NULL and column
COMMENT.
+ val plan = parse(
+ s"""
+ |CREATE TABLE blob_prim_tbl (
+ | c_bool BOOLEAN,
+ | c_tiny TINYINT,
+ | c_small SMALLINT,
+ | c_int INT,
+ | c_big BIGINT,
+ | c_float FLOAT,
+ | c_double DOUBLE,
+ | c_date DATE,
+ | c_str STRING,
+ | c_char CHAR(5),
+ | c_varchar VARCHAR(10),
+ | c_bin BINARY,
+ | c_dec DECIMAL,
+ | c_dec1 DECIMAL(12),
+ | c_dec2 DECIMAL(12, 3),
+ | c_notnull INT NOT NULL,
+ | c_comment INT COMMENT 'a comment',
+ | data BLOB
+ |) USING hudi
+ """.stripMargin)
+ val schema = plan.tableSchema
+ assertResult(BooleanType)(schema("c_bool").dataType)
+ assertResult(ByteType)(schema("c_tiny").dataType)
+ assertResult(ShortType)(schema("c_small").dataType)
+ assertResult(IntegerType)(schema("c_int").dataType)
+ assertResult(LongType)(schema("c_big").dataType)
+ assertResult(FloatType)(schema("c_float").dataType)
+ assertResult(DoubleType)(schema("c_double").dataType)
+ assertResult(DateType)(schema("c_date").dataType)
+ assertResult(StringType)(schema("c_str").dataType)
+ // CHAR/VARCHAR may be preserved or replaced with STRING depending on the
Spark version.
+ assert(Seq[DataType](CharType(5),
StringType).contains(schema("c_char").dataType))
+ assert(Seq[DataType](VarcharType(10),
StringType).contains(schema("c_varchar").dataType))
+ assertResult(BinaryType)(schema("c_bin").dataType)
+ assertResult(DecimalType(10, 0))(schema("c_dec").dataType)
+ assertResult(DecimalType(12, 0))(schema("c_dec1").dataType)
+ assertResult(DecimalType(12, 3))(schema("c_dec2").dataType)
+ assertResult(BlobType())(schema("data").dataType)
+ assert(!schema("c_notnull").nullable)
+ assertResult("a
comment")(schema("c_comment").metadata.getString("comment"))
+ }
+
+ test("Test parse CREATE TABLE with BLOB column and complex data types") {
+ // Exercises the ARRAY / MAP / STRUCT arms and BLOB-in-struct metadata
handling.
+ val plan = parse(
+ s"""
+ |CREATE TABLE blob_complex_tbl (
+ | c_arr ARRAY<INT>,
+ | c_map MAP<STRING, INT>,
+ | c_struct STRUCT<a: INT, b: STRING>,
+ | c_nested_blob STRUCT<x: BLOB>,
+ | data BLOB
+ |) USING hudi
+ """.stripMargin)
+ val schema = plan.tableSchema
+ assertResult(ArrayType(IntegerType))(schema("c_arr").dataType)
+ assertResult(MapType(StringType, IntegerType))(schema("c_map").dataType)
+ val inner = schema("c_struct").dataType.asInstanceOf[StructType]
+ assertResult(IntegerType)(inner("a").dataType)
+ assertResult(StringType)(inner("b").dataType)
+ // A BLOB nested inside a struct still carries the BLOB type descriptor.
+ val nested = schema("c_nested_blob").dataType.asInstanceOf[StructType]("x")
+ assertResult(BlobType())(nested.dataType)
+ assertResult(HoodieSchemaType.BLOB.name())(
+ nested.metadata.getString(HoodieSchema.TYPE_METADATA_FIELD))
+ }
+
+ test("Test parse CREATE TABLE with BLOB column and interval data types") {
+ val plan = parse(
+ s"""
+ |CREATE TABLE blob_ivl_tbl (
+ | i_year INTERVAL YEAR,
+ | i_ym INTERVAL YEAR TO MONTH,
+ | i_day INTERVAL DAY,
+ | i_ds INTERVAL DAY TO SECOND,
+ | data BLOB
+ |) USING hudi
+ """.stripMargin)
+ val schema = plan.tableSchema
+
assertResult(YearMonthIntervalType(YearMonthIntervalType.YEAR))(schema("i_year").dataType)
+ assertResult(YearMonthIntervalType(YearMonthIntervalType.YEAR,
YearMonthIntervalType.MONTH))(
+ schema("i_ym").dataType)
+
assertResult(DayTimeIntervalType(DayTimeIntervalType.DAY))(schema("i_day").dataType)
+ assertResult(DayTimeIntervalType(DayTimeIntervalType.DAY,
DayTimeIntervalType.SECOND))(
+ schema("i_ds").dataType)
+ assertResult(BlobType())(schema("data").dataType)
+
+ // Endpoints where the end field does not follow the start are rejected by
both interval
+ // data-type visitors. The grammar only allows YEAR/MONTH -> MONTH and
DAY/HOUR/MINUTE/SECOND
+ // -> HOUR/MINUTE/SECOND, so these stay grammatical yet still hit the
builder's end <= start guard.
+ checkExceptionContain(
+ "CREATE TABLE blob_bad_ym (id BIGINT, bad INTERVAL MONTH TO MONTH, data
BLOB) USING hudi")(
+ "are not supported")
+ checkExceptionContain(
+ "CREATE TABLE blob_bad_dt (id BIGINT, bad INTERVAL SECOND TO HOUR, data
BLOB) USING hudi")(
+ "are not supported")
+
+ // An unknown primitive type name is rejected.
+ checkExceptionContain(
+ "CREATE TABLE blob_bad_type (id BIGINT, weird sometype, data BLOB) USING
hudi")(
+ "is not supported")
+ }
+
+ test("Test parse CREATE TABLE with BLOB column and partition transforms") {
+ val plan = parse(
+ s"""
+ |CREATE TABLE blob_tf_tbl (
+ | id BIGINT,
+ | ts DATE,
+ | region STRING,
+ | data BLOB
+ |) USING hudi
+ |PARTITIONED BY (region, years(ts), months(ts), days(ts), hours(ts),
myfunc(id))
+ """.stripMargin)
+ assertResult(BlobType())(plan.tableSchema("data").dataType)
+ assertResult(Seq(Seq("region")))(transformFieldRefs(transformByName(plan,
"identity")))
+ assertResult(Seq(Seq("ts")))(transformFieldRefs(transformByName(plan,
"years")))
+ assertResult(Seq(Seq("ts")))(transformFieldRefs(transformByName(plan,
"months")))
+ assertResult(Seq(Seq("ts")))(transformFieldRefs(transformByName(plan,
"days")))
+ assertResult(Seq(Seq("ts")))(transformFieldRefs(transformByName(plan,
"hours")))
+ // an arbitrary function transform falls through to the generic
apply-transform arm
+ assertResult(Seq(Seq("id")))(transformFieldRefs(transformByName(plan,
"myfunc")))
+
+ // bucket(numBuckets, col) with int, long and short number-of-buckets
literals exercises the
+ // three numeric arms of the bucket handling.
+ Seq("4", "4L", "4S").foreach { numLiteral =>
+ val bp = parse(
+ s"CREATE TABLE blob_bkt_tbl (id BIGINT, data BLOB) USING hudi " +
+ s"PARTITIONED BY (bucket($numLiteral, id))")
+ val bkt = transformByName(bp, "bucket")
+ assertResult("4")(firstLiteralArg(bkt).value.toString)
+ assertResult(Seq(Seq("id")))(transformFieldRefs(bkt))
+ }
+ }
+
+ test("Test parse CREATE TABLE with BLOB column and typed transform-argument
literals") {
+ // Constant transform arguments exercise the literal visitors: string,
integer, big-integer and
+ // exponent numerics (the private numeric-literal helper), the typed date
constructor, and both
+ // interval forms (multi-unit and unit-to-unit). A typed timestamp
constructor is not used
+ // because the Spark 4.x extended parser rejects a bare TIMESTAMP token.
Note: a bare
+ // true/false/null in this position is parsed as a column reference
(qualifiedName takes
+ // precedence over a constant in the grammar under the default non-ANSI
config), so the boolean
+ // and null literal visitors are not reachable from a CREATE TABLE
statement.
Review Comment:
Half of this claim only holds under non-ANSI keyword mode. With the current
wiring (`SQL_standard_keyword_behavior = conf.ansiEnabled`), the Spark 4.x
profiles run with the flag on, and there `f(null, id)` parses as a null
literal, so `visitNullLiteral` **is** reachable on 4.x. Only
`visitBooleanLiteral` is unreachable in both modes (`TRUE`/`FALSE` are in
`ansiNonReserved`, so `qualifiedName` always wins) -- and the `OPTIONS ('k' =
true)` case does not reach it either, since `visitPropertyValue` reads
`booleanValue` via `getText` without visiting the constant.
Action: reword to "`null` is a column reference under non-ANSI keyword mode
and a literal under ANSI (the Spark 4.x default)". `visitBooleanLiteral` is
provably dead in the six builders and can be deleted in a follow-up instead of
documented.
##########
hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/dml/schema/TestBlobDataType.scala:
##########
@@ -282,4 +284,256 @@ class TestBlobDataType extends HoodieSparkSqlTestBase {
"Expected at least one .clean instant on the timeline after
compaction")
})
}
+
+ // The following cases are parser-coverage only: a BLOB column routes the
whole CREATE TABLE
+ // through the extended AST builder, so its clause visitors run. parsePlan
is purely syntactic
+ // (no catalog, no execution), which lets us exercise clauses Hudi does not
support at execution
+ // time (transform partitioning, STORED AS / ROW FORMAT, interval columns).
The BLOB column type
+ // itself proves routing because the stock Spark parser rejects the BLOB
type name.
+
+ private def parse(sql: String): CreateTable =
+ spark.sessionState.sqlParser.parsePlan(sql).asInstanceOf[CreateTable]
+
+ test("Test parse CREATE TABLE with BLOB column and primitive data types") {
+ // Exercises the primitive-data-type match arms plus NOT NULL and column
COMMENT.
+ val plan = parse(
+ s"""
+ |CREATE TABLE blob_prim_tbl (
+ | c_bool BOOLEAN,
+ | c_tiny TINYINT,
+ | c_small SMALLINT,
+ | c_int INT,
+ | c_big BIGINT,
+ | c_float FLOAT,
+ | c_double DOUBLE,
+ | c_date DATE,
+ | c_str STRING,
+ | c_char CHAR(5),
+ | c_varchar VARCHAR(10),
+ | c_bin BINARY,
+ | c_dec DECIMAL,
+ | c_dec1 DECIMAL(12),
+ | c_dec2 DECIMAL(12, 3),
+ | c_notnull INT NOT NULL,
+ | c_comment INT COMMENT 'a comment',
+ | data BLOB
+ |) USING hudi
+ """.stripMargin)
+ val schema = plan.tableSchema
+ assertResult(BooleanType)(schema("c_bool").dataType)
+ assertResult(ByteType)(schema("c_tiny").dataType)
+ assertResult(ShortType)(schema("c_small").dataType)
+ assertResult(IntegerType)(schema("c_int").dataType)
+ assertResult(LongType)(schema("c_big").dataType)
+ assertResult(FloatType)(schema("c_float").dataType)
+ assertResult(DoubleType)(schema("c_double").dataType)
+ assertResult(DateType)(schema("c_date").dataType)
+ assertResult(StringType)(schema("c_str").dataType)
+ // CHAR/VARCHAR may be preserved or replaced with STRING depending on the
Spark version.
+ assert(Seq[DataType](CharType(5),
StringType).contains(schema("c_char").dataType))
+ assert(Seq[DataType](VarcharType(10),
StringType).contains(schema("c_varchar").dataType))
+ assertResult(BinaryType)(schema("c_bin").dataType)
+ assertResult(DecimalType(10, 0))(schema("c_dec").dataType)
+ assertResult(DecimalType(12, 0))(schema("c_dec1").dataType)
+ assertResult(DecimalType(12, 3))(schema("c_dec2").dataType)
+ assertResult(BlobType())(schema("data").dataType)
+ assert(!schema("c_notnull").nullable)
+ assertResult("a
comment")(schema("c_comment").metadata.getString("comment"))
+ }
+
+ test("Test parse CREATE TABLE with BLOB column and complex data types") {
+ // Exercises the ARRAY / MAP / STRUCT arms and BLOB-in-struct metadata
handling.
+ val plan = parse(
+ s"""
+ |CREATE TABLE blob_complex_tbl (
+ | c_arr ARRAY<INT>,
+ | c_map MAP<STRING, INT>,
+ | c_struct STRUCT<a: INT, b: STRING>,
+ | c_nested_blob STRUCT<x: BLOB>,
+ | data BLOB
+ |) USING hudi
+ """.stripMargin)
+ val schema = plan.tableSchema
+ assertResult(ArrayType(IntegerType))(schema("c_arr").dataType)
+ assertResult(MapType(StringType, IntegerType))(schema("c_map").dataType)
+ val inner = schema("c_struct").dataType.asInstanceOf[StructType]
+ assertResult(IntegerType)(inner("a").dataType)
+ assertResult(StringType)(inner("b").dataType)
+ // A BLOB nested inside a struct still carries the BLOB type descriptor.
+ val nested = schema("c_nested_blob").dataType.asInstanceOf[StructType]("x")
+ assertResult(BlobType())(nested.dataType)
+ assertResult(HoodieSchemaType.BLOB.name())(
+ nested.metadata.getString(HoodieSchema.TYPE_METADATA_FIELD))
+ }
+
+ test("Test parse CREATE TABLE with BLOB column and interval data types") {
+ val plan = parse(
+ s"""
+ |CREATE TABLE blob_ivl_tbl (
+ | i_year INTERVAL YEAR,
+ | i_ym INTERVAL YEAR TO MONTH,
+ | i_day INTERVAL DAY,
+ | i_ds INTERVAL DAY TO SECOND,
+ | data BLOB
+ |) USING hudi
+ """.stripMargin)
+ val schema = plan.tableSchema
+
assertResult(YearMonthIntervalType(YearMonthIntervalType.YEAR))(schema("i_year").dataType)
+ assertResult(YearMonthIntervalType(YearMonthIntervalType.YEAR,
YearMonthIntervalType.MONTH))(
+ schema("i_ym").dataType)
+
assertResult(DayTimeIntervalType(DayTimeIntervalType.DAY))(schema("i_day").dataType)
+ assertResult(DayTimeIntervalType(DayTimeIntervalType.DAY,
DayTimeIntervalType.SECOND))(
+ schema("i_ds").dataType)
+ assertResult(BlobType())(schema("data").dataType)
+
+ // Endpoints where the end field does not follow the start are rejected by
both interval
+ // data-type visitors. The grammar only allows YEAR/MONTH -> MONTH and
DAY/HOUR/MINUTE/SECOND
+ // -> HOUR/MINUTE/SECOND, so these stay grammatical yet still hit the
builder's end <= start guard.
+ checkExceptionContain(
+ "CREATE TABLE blob_bad_ym (id BIGINT, bad INTERVAL MONTH TO MONTH, data
BLOB) USING hudi")(
+ "are not supported")
+ checkExceptionContain(
+ "CREATE TABLE blob_bad_dt (id BIGINT, bad INTERVAL SECOND TO HOUR, data
BLOB) USING hudi")(
+ "are not supported")
+
+ // An unknown primitive type name is rejected.
+ checkExceptionContain(
+ "CREATE TABLE blob_bad_type (id BIGINT, weird sometype, data BLOB) USING
hudi")(
+ "is not supported")
+ }
+
+ test("Test parse CREATE TABLE with BLOB column and partition transforms") {
Review Comment:
Coverage gap worth closing while you are here: none of the transform tests
hits the partition-**column** arm (`PARTITIONED BY (p STRING)`), which is the
only untested branch that changes the emitted schema (`visitCreateTable`
appends `partCols` to the columns). Cheap negatives in the same area with zero
coverage today: the mixed form `PARTITIONED BY (p STRING, bucket(4, id))`,
`SKEWED BY`, and `CREATE TEMPORARY TABLE` (all reach the builder's mix-error /
`operationNotAllowed` paths).
Action: add a `PARTITIONED BY (p STRING)` case asserting `p` appears in
`tableSchema`, plus the three negatives.
##########
hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/dml/schema/TestBlobDataType.scala:
##########
@@ -282,4 +284,256 @@ class TestBlobDataType extends HoodieSparkSqlTestBase {
"Expected at least one .clean instant on the timeline after
compaction")
})
}
+
+ // The following cases are parser-coverage only: a BLOB column routes the
whole CREATE TABLE
+ // through the extended AST builder, so its clause visitors run. parsePlan
is purely syntactic
+ // (no catalog, no execution), which lets us exercise clauses Hudi does not
support at execution
+ // time (transform partitioning, STORED AS / ROW FORMAT, interval columns).
The BLOB column type
+ // itself proves routing because the stock Spark parser rejects the BLOB
type name.
+
+ private def parse(sql: String): CreateTable =
+ spark.sessionState.sqlParser.parsePlan(sql).asInstanceOf[CreateTable]
+
+ test("Test parse CREATE TABLE with BLOB column and primitive data types") {
+ // Exercises the primitive-data-type match arms plus NOT NULL and column
COMMENT.
+ val plan = parse(
+ s"""
+ |CREATE TABLE blob_prim_tbl (
+ | c_bool BOOLEAN,
+ | c_tiny TINYINT,
+ | c_small SMALLINT,
+ | c_int INT,
+ | c_big BIGINT,
+ | c_float FLOAT,
+ | c_double DOUBLE,
+ | c_date DATE,
+ | c_str STRING,
+ | c_char CHAR(5),
+ | c_varchar VARCHAR(10),
+ | c_bin BINARY,
+ | c_dec DECIMAL,
+ | c_dec1 DECIMAL(12),
+ | c_dec2 DECIMAL(12, 3),
+ | c_notnull INT NOT NULL,
+ | c_comment INT COMMENT 'a comment',
+ | data BLOB
+ |) USING hudi
+ """.stripMargin)
+ val schema = plan.tableSchema
+ assertResult(BooleanType)(schema("c_bool").dataType)
+ assertResult(ByteType)(schema("c_tiny").dataType)
+ assertResult(ShortType)(schema("c_small").dataType)
+ assertResult(IntegerType)(schema("c_int").dataType)
+ assertResult(LongType)(schema("c_big").dataType)
+ assertResult(FloatType)(schema("c_float").dataType)
+ assertResult(DoubleType)(schema("c_double").dataType)
+ assertResult(DateType)(schema("c_date").dataType)
+ assertResult(StringType)(schema("c_str").dataType)
+ // CHAR/VARCHAR may be preserved or replaced with STRING depending on the
Spark version.
+ assert(Seq[DataType](CharType(5),
StringType).contains(schema("c_char").dataType))
+ assert(Seq[DataType](VarcharType(10),
StringType).contains(schema("c_varchar").dataType))
+ assertResult(BinaryType)(schema("c_bin").dataType)
+ assertResult(DecimalType(10, 0))(schema("c_dec").dataType)
+ assertResult(DecimalType(12, 0))(schema("c_dec1").dataType)
+ assertResult(DecimalType(12, 3))(schema("c_dec2").dataType)
+ assertResult(BlobType())(schema("data").dataType)
+ assert(!schema("c_notnull").nullable)
+ assertResult("a
comment")(schema("c_comment").metadata.getString("comment"))
+ }
+
+ test("Test parse CREATE TABLE with BLOB column and complex data types") {
+ // Exercises the ARRAY / MAP / STRUCT arms and BLOB-in-struct metadata
handling.
+ val plan = parse(
+ s"""
+ |CREATE TABLE blob_complex_tbl (
+ | c_arr ARRAY<INT>,
+ | c_map MAP<STRING, INT>,
+ | c_struct STRUCT<a: INT, b: STRING>,
+ | c_nested_blob STRUCT<x: BLOB>,
+ | data BLOB
+ |) USING hudi
+ """.stripMargin)
+ val schema = plan.tableSchema
+ assertResult(ArrayType(IntegerType))(schema("c_arr").dataType)
+ assertResult(MapType(StringType, IntegerType))(schema("c_map").dataType)
+ val inner = schema("c_struct").dataType.asInstanceOf[StructType]
+ assertResult(IntegerType)(inner("a").dataType)
+ assertResult(StringType)(inner("b").dataType)
+ // A BLOB nested inside a struct still carries the BLOB type descriptor.
+ val nested = schema("c_nested_blob").dataType.asInstanceOf[StructType]("x")
+ assertResult(BlobType())(nested.dataType)
+ assertResult(HoodieSchemaType.BLOB.name())(
+ nested.metadata.getString(HoodieSchema.TYPE_METADATA_FIELD))
+ }
+
+ test("Test parse CREATE TABLE with BLOB column and interval data types") {
+ val plan = parse(
+ s"""
+ |CREATE TABLE blob_ivl_tbl (
+ | i_year INTERVAL YEAR,
+ | i_ym INTERVAL YEAR TO MONTH,
+ | i_day INTERVAL DAY,
+ | i_ds INTERVAL DAY TO SECOND,
+ | data BLOB
+ |) USING hudi
+ """.stripMargin)
+ val schema = plan.tableSchema
+
assertResult(YearMonthIntervalType(YearMonthIntervalType.YEAR))(schema("i_year").dataType)
+ assertResult(YearMonthIntervalType(YearMonthIntervalType.YEAR,
YearMonthIntervalType.MONTH))(
+ schema("i_ym").dataType)
+
assertResult(DayTimeIntervalType(DayTimeIntervalType.DAY))(schema("i_day").dataType)
+ assertResult(DayTimeIntervalType(DayTimeIntervalType.DAY,
DayTimeIntervalType.SECOND))(
+ schema("i_ds").dataType)
+ assertResult(BlobType())(schema("data").dataType)
+
+ // Endpoints where the end field does not follow the start are rejected by
both interval
+ // data-type visitors. The grammar only allows YEAR/MONTH -> MONTH and
DAY/HOUR/MINUTE/SECOND
+ // -> HOUR/MINUTE/SECOND, so these stay grammatical yet still hit the
builder's end <= start guard.
+ checkExceptionContain(
+ "CREATE TABLE blob_bad_ym (id BIGINT, bad INTERVAL MONTH TO MONTH, data
BLOB) USING hudi")(
+ "are not supported")
+ checkExceptionContain(
+ "CREATE TABLE blob_bad_dt (id BIGINT, bad INTERVAL SECOND TO HOUR, data
BLOB) USING hudi")(
+ "are not supported")
+
+ // An unknown primitive type name is rejected.
+ checkExceptionContain(
+ "CREATE TABLE blob_bad_type (id BIGINT, weird sometype, data BLOB) USING
hudi")(
+ "is not supported")
+ }
+
+ test("Test parse CREATE TABLE with BLOB column and partition transforms") {
+ val plan = parse(
+ s"""
+ |CREATE TABLE blob_tf_tbl (
+ | id BIGINT,
+ | ts DATE,
+ | region STRING,
+ | data BLOB
+ |) USING hudi
+ |PARTITIONED BY (region, years(ts), months(ts), days(ts), hours(ts),
myfunc(id))
+ """.stripMargin)
+ assertResult(BlobType())(plan.tableSchema("data").dataType)
+ assertResult(Seq(Seq("region")))(transformFieldRefs(transformByName(plan,
"identity")))
+ assertResult(Seq(Seq("ts")))(transformFieldRefs(transformByName(plan,
"years")))
+ assertResult(Seq(Seq("ts")))(transformFieldRefs(transformByName(plan,
"months")))
+ assertResult(Seq(Seq("ts")))(transformFieldRefs(transformByName(plan,
"days")))
+ assertResult(Seq(Seq("ts")))(transformFieldRefs(transformByName(plan,
"hours")))
+ // an arbitrary function transform falls through to the generic
apply-transform arm
+ assertResult(Seq(Seq("id")))(transformFieldRefs(transformByName(plan,
"myfunc")))
+
+ // bucket(numBuckets, col) with int, long and short number-of-buckets
literals exercises the
+ // three numeric arms of the bucket handling.
+ Seq("4", "4L", "4S").foreach { numLiteral =>
+ val bp = parse(
+ s"CREATE TABLE blob_bkt_tbl (id BIGINT, data BLOB) USING hudi " +
+ s"PARTITIONED BY (bucket($numLiteral, id))")
+ val bkt = transformByName(bp, "bucket")
+ assertResult("4")(firstLiteralArg(bkt).value.toString)
+ assertResult(Seq(Seq("id")))(transformFieldRefs(bkt))
+ }
+ }
+
+ test("Test parse CREATE TABLE with BLOB column and typed transform-argument
literals") {
+ // Constant transform arguments exercise the literal visitors: string,
integer, big-integer and
+ // exponent numerics (the private numeric-literal helper), the typed date
constructor, and both
+ // interval forms (multi-unit and unit-to-unit). A typed timestamp
constructor is not used
+ // because the Spark 4.x extended parser rejects a bare TIMESTAMP token.
Note: a bare
+ // true/false/null in this position is parsed as a column reference
(qualifiedName takes
+ // precedence over a constant in the grammar under the default non-ANSI
config), so the boolean
+ // and null literal visitors are not reachable from a CREATE TABLE
statement.
+ val plan = parse(
+ s"""
+ |CREATE TABLE blob_lit_tbl (
+ | id BIGINT,
+ | data BLOB
+ |) USING hudi
+ |PARTITIONED BY (
+ | str_t('x', id),
+ | int_t(7, id),
+ | long_t(9000000000L, id),
+ | exp_t(1E3, id),
+ | date_t(DATE '2020-01-01', id),
+ | mu_ivl_t(INTERVAL '1' DAY, id),
+ | uu_ivl_t(INTERVAL '1-2' YEAR TO MONTH, id)
+ |)
+ """.stripMargin)
+ assertResult(StringType)(firstLiteralArg(transformByName(plan,
"str_t")).dataType)
+ assertResult("x")(firstLiteralArg(transformByName(plan,
"str_t")).value.toString)
+ assertResult(IntegerType)(firstLiteralArg(transformByName(plan,
"int_t")).dataType)
+ assertResult(LongType)(firstLiteralArg(transformByName(plan,
"long_t")).dataType)
+ assertResult(DoubleType)(firstLiteralArg(transformByName(plan,
"exp_t")).dataType)
+ assertResult(DateType)(firstLiteralArg(transformByName(plan,
"date_t")).dataType)
+ assertResult(DayTimeIntervalType(DayTimeIntervalType.DAY,
DayTimeIntervalType.DAY))(
+ firstLiteralArg(transformByName(plan, "mu_ivl_t")).dataType)
+ assertResult(YearMonthIntervalType(YearMonthIntervalType.YEAR,
YearMonthIntervalType.MONTH))(
+ firstLiteralArg(transformByName(plan, "uu_ivl_t")).dataType)
+ }
+
+ test("Test parse CREATE TABLE with BLOB column and invalid partition
transforms") {
+ // Non-numeric number of buckets.
+ checkExceptionContain(
+ "CREATE TABLE blob_e1 (id BIGINT, data BLOB) USING hudi PARTITIONED BY
(bucket('x', id))")(
+ "Invalid number of buckets")
Review Comment:
These negatives pass for the wrong reason on Spark 3.4+. There, `new
ParseException(String, ctx)` treats the String as an error **class**, so this
case actually throws `SparkException: [INTERNAL_ERROR] Cannot find main error
class 'Invalid number of buckets: 'x''` (verified at runtime on 3.5.5; only the
spark3.3 profile still throws a real `ParseException` with the message).
`checkExceptionContain` substring-matches any `Throwable`, so the assertion
passes against the broken output and cements it. This affects ~10 throw-sites
per builder (3.5 builder lines 597, 608, 623, 918, 954, ...), including a
`$nonRef.describe` interpolation bug that emits `for transform bucket:
5.describe`.
Action: file a follow-up issue to switch those sites to
`ParserUtils.operationNotAllowed(...)` or the `(command, message, start, stop)`
ctor and add the missing `${...}` braces; in this PR, strengthen at least one
of these negatives to also assert the exception type (e.g.
`intercept[ParseException]`) with a TODO referencing the issue, so the fix is
observable.
##########
hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/dml/schema/TestBlobDataType.scala:
##########
@@ -282,4 +284,256 @@ class TestBlobDataType extends HoodieSparkSqlTestBase {
"Expected at least one .clean instant on the timeline after
compaction")
})
}
+
+ // The following cases are parser-coverage only: a BLOB column routes the
whole CREATE TABLE
+ // through the extended AST builder, so its clause visitors run. parsePlan
is purely syntactic
+ // (no catalog, no execution), which lets us exercise clauses Hudi does not
support at execution
+ // time (transform partitioning, STORED AS / ROW FORMAT, interval columns).
The BLOB column type
+ // itself proves routing because the stock Spark parser rejects the BLOB
type name.
+
+ private def parse(sql: String): CreateTable =
+ spark.sessionState.sqlParser.parsePlan(sql).asInstanceOf[CreateTable]
+
+ test("Test parse CREATE TABLE with BLOB column and primitive data types") {
+ // Exercises the primitive-data-type match arms plus NOT NULL and column
COMMENT.
+ val plan = parse(
+ s"""
+ |CREATE TABLE blob_prim_tbl (
+ | c_bool BOOLEAN,
+ | c_tiny TINYINT,
+ | c_small SMALLINT,
+ | c_int INT,
+ | c_big BIGINT,
+ | c_float FLOAT,
+ | c_double DOUBLE,
+ | c_date DATE,
+ | c_str STRING,
+ | c_char CHAR(5),
+ | c_varchar VARCHAR(10),
+ | c_bin BINARY,
+ | c_dec DECIMAL,
+ | c_dec1 DECIMAL(12),
+ | c_dec2 DECIMAL(12, 3),
+ | c_notnull INT NOT NULL,
+ | c_comment INT COMMENT 'a comment',
+ | data BLOB
+ |) USING hudi
+ """.stripMargin)
+ val schema = plan.tableSchema
+ assertResult(BooleanType)(schema("c_bool").dataType)
+ assertResult(ByteType)(schema("c_tiny").dataType)
+ assertResult(ShortType)(schema("c_small").dataType)
+ assertResult(IntegerType)(schema("c_int").dataType)
+ assertResult(LongType)(schema("c_big").dataType)
+ assertResult(FloatType)(schema("c_float").dataType)
+ assertResult(DoubleType)(schema("c_double").dataType)
+ assertResult(DateType)(schema("c_date").dataType)
+ assertResult(StringType)(schema("c_str").dataType)
+ // CHAR/VARCHAR may be preserved or replaced with STRING depending on the
Spark version.
+ assert(Seq[DataType](CharType(5),
StringType).contains(schema("c_char").dataType))
+ assert(Seq[DataType](VarcharType(10),
StringType).contains(schema("c_varchar").dataType))
+ assertResult(BinaryType)(schema("c_bin").dataType)
+ assertResult(DecimalType(10, 0))(schema("c_dec").dataType)
+ assertResult(DecimalType(12, 0))(schema("c_dec1").dataType)
+ assertResult(DecimalType(12, 3))(schema("c_dec2").dataType)
+ assertResult(BlobType())(schema("data").dataType)
+ assert(!schema("c_notnull").nullable)
+ assertResult("a
comment")(schema("c_comment").metadata.getString("comment"))
+ }
+
+ test("Test parse CREATE TABLE with BLOB column and complex data types") {
+ // Exercises the ARRAY / MAP / STRUCT arms and BLOB-in-struct metadata
handling.
+ val plan = parse(
+ s"""
+ |CREATE TABLE blob_complex_tbl (
+ | c_arr ARRAY<INT>,
+ | c_map MAP<STRING, INT>,
+ | c_struct STRUCT<a: INT, b: STRING>,
+ | c_nested_blob STRUCT<x: BLOB>,
+ | data BLOB
+ |) USING hudi
+ """.stripMargin)
+ val schema = plan.tableSchema
+ assertResult(ArrayType(IntegerType))(schema("c_arr").dataType)
+ assertResult(MapType(StringType, IntegerType))(schema("c_map").dataType)
+ val inner = schema("c_struct").dataType.asInstanceOf[StructType]
+ assertResult(IntegerType)(inner("a").dataType)
+ assertResult(StringType)(inner("b").dataType)
+ // A BLOB nested inside a struct still carries the BLOB type descriptor.
+ val nested = schema("c_nested_blob").dataType.asInstanceOf[StructType]("x")
+ assertResult(BlobType())(nested.dataType)
+ assertResult(HoodieSchemaType.BLOB.name())(
+ nested.metadata.getString(HoodieSchema.TYPE_METADATA_FIELD))
+ }
+
+ test("Test parse CREATE TABLE with BLOB column and interval data types") {
+ val plan = parse(
+ s"""
+ |CREATE TABLE blob_ivl_tbl (
+ | i_year INTERVAL YEAR,
+ | i_ym INTERVAL YEAR TO MONTH,
+ | i_day INTERVAL DAY,
+ | i_ds INTERVAL DAY TO SECOND,
+ | data BLOB
+ |) USING hudi
+ """.stripMargin)
+ val schema = plan.tableSchema
+
assertResult(YearMonthIntervalType(YearMonthIntervalType.YEAR))(schema("i_year").dataType)
+ assertResult(YearMonthIntervalType(YearMonthIntervalType.YEAR,
YearMonthIntervalType.MONTH))(
+ schema("i_ym").dataType)
+
assertResult(DayTimeIntervalType(DayTimeIntervalType.DAY))(schema("i_day").dataType)
+ assertResult(DayTimeIntervalType(DayTimeIntervalType.DAY,
DayTimeIntervalType.SECOND))(
+ schema("i_ds").dataType)
+ assertResult(BlobType())(schema("data").dataType)
+
+ // Endpoints where the end field does not follow the start are rejected by
both interval
+ // data-type visitors. The grammar only allows YEAR/MONTH -> MONTH and
DAY/HOUR/MINUTE/SECOND
+ // -> HOUR/MINUTE/SECOND, so these stay grammatical yet still hit the
builder's end <= start guard.
+ checkExceptionContain(
+ "CREATE TABLE blob_bad_ym (id BIGINT, bad INTERVAL MONTH TO MONTH, data
BLOB) USING hudi")(
+ "are not supported")
+ checkExceptionContain(
+ "CREATE TABLE blob_bad_dt (id BIGINT, bad INTERVAL SECOND TO HOUR, data
BLOB) USING hudi")(
+ "are not supported")
+
+ // An unknown primitive type name is rejected.
+ checkExceptionContain(
+ "CREATE TABLE blob_bad_type (id BIGINT, weird sometype, data BLOB) USING
hudi")(
+ "is not supported")
+ }
+
+ test("Test parse CREATE TABLE with BLOB column and partition transforms") {
+ val plan = parse(
+ s"""
+ |CREATE TABLE blob_tf_tbl (
+ | id BIGINT,
+ | ts DATE,
+ | region STRING,
+ | data BLOB
+ |) USING hudi
+ |PARTITIONED BY (region, years(ts), months(ts), days(ts), hours(ts),
myfunc(id))
+ """.stripMargin)
+ assertResult(BlobType())(plan.tableSchema("data").dataType)
+ assertResult(Seq(Seq("region")))(transformFieldRefs(transformByName(plan,
"identity")))
+ assertResult(Seq(Seq("ts")))(transformFieldRefs(transformByName(plan,
"years")))
+ assertResult(Seq(Seq("ts")))(transformFieldRefs(transformByName(plan,
"months")))
+ assertResult(Seq(Seq("ts")))(transformFieldRefs(transformByName(plan,
"days")))
+ assertResult(Seq(Seq("ts")))(transformFieldRefs(transformByName(plan,
"hours")))
+ // an arbitrary function transform falls through to the generic
apply-transform arm
+ assertResult(Seq(Seq("id")))(transformFieldRefs(transformByName(plan,
"myfunc")))
+
+ // bucket(numBuckets, col) with int, long and short number-of-buckets
literals exercises the
+ // three numeric arms of the bucket handling.
+ Seq("4", "4L", "4S").foreach { numLiteral =>
+ val bp = parse(
+ s"CREATE TABLE blob_bkt_tbl (id BIGINT, data BLOB) USING hudi " +
+ s"PARTITIONED BY (bucket($numLiteral, id))")
+ val bkt = transformByName(bp, "bucket")
+ assertResult("4")(firstLiteralArg(bkt).value.toString)
+ assertResult(Seq(Seq("id")))(transformFieldRefs(bkt))
+ }
+ }
+
+ test("Test parse CREATE TABLE with BLOB column and typed transform-argument
literals") {
+ // Constant transform arguments exercise the literal visitors: string,
integer, big-integer and
+ // exponent numerics (the private numeric-literal helper), the typed date
constructor, and both
+ // interval forms (multi-unit and unit-to-unit). A typed timestamp
constructor is not used
+ // because the Spark 4.x extended parser rejects a bare TIMESTAMP token.
Note: a bare
+ // true/false/null in this position is parsed as a column reference
(qualifiedName takes
+ // precedence over a constant in the grammar under the default non-ANSI
config), so the boolean
+ // and null literal visitors are not reachable from a CREATE TABLE
statement.
+ val plan = parse(
+ s"""
+ |CREATE TABLE blob_lit_tbl (
+ | id BIGINT,
+ | data BLOB
+ |) USING hudi
+ |PARTITIONED BY (
+ | str_t('x', id),
+ | int_t(7, id),
+ | long_t(9000000000L, id),
+ | exp_t(1E3, id),
+ | date_t(DATE '2020-01-01', id),
+ | mu_ivl_t(INTERVAL '1' DAY, id),
+ | uu_ivl_t(INTERVAL '1-2' YEAR TO MONTH, id)
+ |)
+ """.stripMargin)
+ assertResult(StringType)(firstLiteralArg(transformByName(plan,
"str_t")).dataType)
+ assertResult("x")(firstLiteralArg(transformByName(plan,
"str_t")).value.toString)
+ assertResult(IntegerType)(firstLiteralArg(transformByName(plan,
"int_t")).dataType)
+ assertResult(LongType)(firstLiteralArg(transformByName(plan,
"long_t")).dataType)
+ assertResult(DoubleType)(firstLiteralArg(transformByName(plan,
"exp_t")).dataType)
+ assertResult(DateType)(firstLiteralArg(transformByName(plan,
"date_t")).dataType)
+ assertResult(DayTimeIntervalType(DayTimeIntervalType.DAY,
DayTimeIntervalType.DAY))(
+ firstLiteralArg(transformByName(plan, "mu_ivl_t")).dataType)
+ assertResult(YearMonthIntervalType(YearMonthIntervalType.YEAR,
YearMonthIntervalType.MONTH))(
+ firstLiteralArg(transformByName(plan, "uu_ivl_t")).dataType)
+ }
+
+ test("Test parse CREATE TABLE with BLOB column and invalid partition
transforms") {
+ // Non-numeric number of buckets.
+ checkExceptionContain(
+ "CREATE TABLE blob_e1 (id BIGINT, data BLOB) USING hudi PARTITIONED BY
(bucket('x', id))")(
+ "Invalid number of buckets")
+ // A non-column-reference where a column is required.
+ checkExceptionContain(
+ "CREATE TABLE blob_e2 (id BIGINT, data BLOB) USING hudi PARTITIONED BY
(bucket(4, 5))")(
+ "Expected a column reference")
+ // A single-field transform given more than one argument.
+ checkExceptionContain(
+ "CREATE TABLE blob_e3 (id BIGINT, ts DATE, data BLOB) USING hudi " +
+ "PARTITIONED BY (years(id, ts))")(
+ "Too many arguments")
+ }
+
+ test("Test parse CREATE TABLE with BLOB column and file-format / row-format
clauses") {
+ // Generic STORED AS format.
+ assertResult(BlobType())(
+ parse("CREATE TABLE blob_ff1 (id BIGINT, data BLOB) STORED AS PARQUET")
Review Comment:
All six positive cases assert only `BlobType()`, which is identical whether
or not the file/row-format visitors produce anything -- a builder that silently
dropped the serde info would still pass. The discriminating values are already
on the plan and portable across 3.3-4.2: `blob_ff1` -> `tableSpec.serde ==
Some(SerdeInfo(storedAs = Some("PARQUET"), ...))`, `blob_ff2` ->
`formatClasses`, `blob_ff3` -> `serde`, `blob_ff4` ->
`serdeProperties("field.delim" -> ",")`.
Action: assert the relevant `plan.tableSpec.serde` field per case instead of
the column type.
##########
hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/ddl/TestCreateTable.scala:
##########
@@ -2340,6 +2341,140 @@ class TestCreateTable extends HoodieSparkSqlTestBase {
}
}
+ // The following cases are parser-coverage only: a VECTOR column routes the
whole CREATE TABLE
+ // through the extended AST builder, so its clause visitors run. parsePlan
is purely syntactic
+ // (no catalog, no execution), matching how TestIndexSyntax exercises the
index statements, which
+ // lets us cover clauses that are not supported at execution time (transform
partitioning,
+ // CLUSTERED BY, typed literal arguments). The VECTOR column type proves the
statement routed
+ // here because the stock Spark parser rejects the VECTOR type name.
+
+ private def parseCreateTable(sql: String): CreateTable =
+ spark.sessionState.sqlParser.parsePlan(sql).asInstanceOf[CreateTable]
+
+ test("test create VECTOR table with partition transforms parses (parser
coverage)") {
+ val plan = parseCreateTable(
+ s"""
+ |CREATE TABLE vec_parse_tbl (
+ | id BIGINT,
+ | ts DATE,
+ | region STRING,
+ | name STRING,
+ | embedding VECTOR(8)
+ |) USING hudi
+ |PARTITIONED BY (region, bucket(4, id), years(ts), truncate(10, name))
+ """.stripMargin)
+
+ assertEquals(ArrayType(FloatType, containsNull = false),
plan.tableSchema("embedding").dataType)
+
+ assertEquals(Seq(Seq("region")), transformFieldRefs(transformByName(plan,
"identity")))
+ val bucket = transformByName(plan, "bucket")
+ assertEquals(IntegerType, firstLiteralArg(bucket).dataType)
+ assertEquals("4", firstLiteralArg(bucket).value.toString)
+ assertEquals(Seq(Seq("id")), transformFieldRefs(bucket))
+ assertEquals(Seq(Seq("ts")), transformFieldRefs(transformByName(plan,
"years")))
+ val truncate = transformByName(plan, "truncate")
+ assertEquals(IntegerType, firstLiteralArg(truncate).dataType)
+ assertEquals("10", firstLiteralArg(truncate).value.toString)
+ assertEquals(Seq(Seq("name")), transformFieldRefs(truncate))
+ }
+
+ test("test create VECTOR table with typed transform-argument literals parses
(parser coverage)") {
+ // Each transform carries a differently-typed constant argument to
exercise the literal
+ // visitors: string, integer, long, exponent/double, the typed date
constructor, and both
+ // interval forms (multi-units and unit-to-unit). A typed timestamp
constructor is not used
+ // because the Spark 4.x extended parser rejects a bare TIMESTAMP token. A
bare true/false/null
+ // in this position is parsed as a column reference (qualifiedName takes
precedence over a
+ // constant in the grammar under the default non-ANSI config), so the
boolean and null literal
+ // visitors are not reachable from a CREATE TABLE statement.
+ val plan = parseCreateTable(
+ s"""
+ |CREATE TABLE vec_lit_tbl (
+ | id BIGINT,
+ | embedding VECTOR(4)
+ |) USING hudi
+ |PARTITIONED BY (
+ | str_t('x', id),
+ | int_t(4, id),
+ | long_t(9000000000L, id),
+ | exp_t(1E3, id),
+ | date_t(DATE '2020-01-01', id),
+ | mu_ivl_t(INTERVAL '1' DAY, id),
+ | uu_ivl_t(INTERVAL '1-2' YEAR TO MONTH, id)
+ |)
+ """.stripMargin)
+
+ assertEquals(StringType, firstLiteralArg(transformByName(plan,
"str_t")).dataType)
+ assertEquals("x", firstLiteralArg(transformByName(plan,
"str_t")).value.toString)
+ assertEquals(IntegerType, firstLiteralArg(transformByName(plan,
"int_t")).dataType)
+ assertEquals(LongType, firstLiteralArg(transformByName(plan,
"long_t")).dataType)
+ assertEquals(DoubleType, firstLiteralArg(transformByName(plan,
"exp_t")).dataType)
+ assertEquals(DateType, firstLiteralArg(transformByName(plan,
"date_t")).dataType)
+ assertEquals(
+ DayTimeIntervalType(DayTimeIntervalType.DAY, DayTimeIntervalType.DAY),
+ firstLiteralArg(transformByName(plan, "mu_ivl_t")).dataType)
+ assertEquals(
+ YearMonthIntervalType(YearMonthIntervalType.YEAR,
YearMonthIntervalType.MONTH),
+ firstLiteralArg(transformByName(plan, "uu_ivl_t")).dataType)
+ }
+
+ test("test create VECTOR table with CLUSTERED BY bucket spec parses (parser
coverage)") {
+ val plan = parseCreateTable(
+ "CREATE TABLE vec_bucket_tbl (id BIGINT, embedding VECTOR(4)) USING hudi
" +
+ "CLUSTERED BY (id) INTO 4 BUCKETS")
+ val bucket = transformByName(plan, "bucket")
+ assertEquals("4", firstLiteralArg(bucket).value.toString)
+ assertEquals(Seq(Seq("id")), transformFieldRefs(bucket))
+
+ // SORTED BY ... ASC is accepted by the bucket-spec visitor.
+ val sortedPlan = parseCreateTable(
+ "CREATE TABLE vec_sbucket_tbl (id BIGINT, ts BIGINT, embedding
VECTOR(4)) USING hudi " +
+ "CLUSTERED BY (id, ts) SORTED BY (id ASC) INTO 8 BUCKETS")
+
assertTrue(sortedPlan.partitioning.exists(_.name.toLowerCase.contains("bucket")))
Review Comment:
This cannot fail for the case it claims to test: plain
`BucketTransform.name` is `"bucket"` and `SortedBucketTransform.name` is
`"sorted_bucket"` (spark-catalyst bytecode, consistent 3.3-4.1), so
`contains("bucket")` passes whether or not SORTED BY was honored.
```suggestion
assertEquals("sorted_bucket", sortedPlan.partitioning.head.name)
```
##########
hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/feature/index/TestIndexSyntax.scala:
##########
@@ -95,6 +95,26 @@ class TestIndexSyntax extends HoodieSparkSqlTestBase {
resolvedLogicalPlan = analyzer.execute(logicalPlan)
assertTableIdentifier(resolvedLogicalPlan.asInstanceOf[RefreshIndexCommand].table,
databaseName, tableName)
assertResult("idx_name")(resolvedLogicalPlan.asInstanceOf[RefreshIndexCommand].indexName)
+
+ // CREATE INDEX without a USING clause: the index type defaults to
empty
+ logicalPlan = sqlParser.parsePlan(s"create index idx_default on
$tableName (name)")
+ resolvedLogicalPlan = analyzer.execute(logicalPlan)
+
assertTableIdentifier(resolvedLogicalPlan.asInstanceOf[CreateIndexCommand].table,
databaseName, tableName)
+
assertResult("idx_default")(resolvedLogicalPlan.asInstanceOf[CreateIndexCommand].indexName)
+
assertResult("")(resolvedLogicalPlan.asInstanceOf[CreateIndexCommand].indexType)
+
assertResult(false)(resolvedLogicalPlan.asInstanceOf[CreateIndexCommand].ignoreIfExists)
Review Comment:
This duplicates the identical assertion at line 77.
```suggestion
```
Also, for the commit message framing: the no-USING create and no-IF-EXISTS
drop *branches* already execute (lines 158-159 in this file, plus ~40 `create
index ... (col)` calls across the SI suites), and `SHOW INDEXES IN` adds no
builder branch (`visitShowIndexes` ignores FROM vs IN). The new plan-field
pinning (`indexType == ""`, `ignoreIfNotExists == false`) is the actual delta
and worth keeping.
##########
hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/dml/schema/TestBlobDataType.scala:
##########
@@ -282,4 +284,256 @@ class TestBlobDataType extends HoodieSparkSqlTestBase {
"Expected at least one .clean instant on the timeline after
compaction")
})
}
+
+ // The following cases are parser-coverage only: a BLOB column routes the
whole CREATE TABLE
+ // through the extended AST builder, so its clause visitors run. parsePlan
is purely syntactic
+ // (no catalog, no execution), which lets us exercise clauses Hudi does not
support at execution
+ // time (transform partitioning, STORED AS / ROW FORMAT, interval columns).
The BLOB column type
+ // itself proves routing because the stock Spark parser rejects the BLOB
type name.
+
+ private def parse(sql: String): CreateTable =
+ spark.sessionState.sqlParser.parsePlan(sql).asInstanceOf[CreateTable]
+
+ test("Test parse CREATE TABLE with BLOB column and primitive data types") {
+ // Exercises the primitive-data-type match arms plus NOT NULL and column
COMMENT.
+ val plan = parse(
+ s"""
+ |CREATE TABLE blob_prim_tbl (
+ | c_bool BOOLEAN,
+ | c_tiny TINYINT,
+ | c_small SMALLINT,
+ | c_int INT,
+ | c_big BIGINT,
+ | c_float FLOAT,
+ | c_double DOUBLE,
+ | c_date DATE,
+ | c_str STRING,
+ | c_char CHAR(5),
+ | c_varchar VARCHAR(10),
+ | c_bin BINARY,
+ | c_dec DECIMAL,
+ | c_dec1 DECIMAL(12),
+ | c_dec2 DECIMAL(12, 3),
+ | c_notnull INT NOT NULL,
+ | c_comment INT COMMENT 'a comment',
+ | data BLOB
+ |) USING hudi
+ """.stripMargin)
+ val schema = plan.tableSchema
+ assertResult(BooleanType)(schema("c_bool").dataType)
+ assertResult(ByteType)(schema("c_tiny").dataType)
+ assertResult(ShortType)(schema("c_small").dataType)
+ assertResult(IntegerType)(schema("c_int").dataType)
+ assertResult(LongType)(schema("c_big").dataType)
+ assertResult(FloatType)(schema("c_float").dataType)
+ assertResult(DoubleType)(schema("c_double").dataType)
+ assertResult(DateType)(schema("c_date").dataType)
+ assertResult(StringType)(schema("c_str").dataType)
+ // CHAR/VARCHAR may be preserved or replaced with STRING depending on the
Spark version.
+ assert(Seq[DataType](CharType(5),
StringType).contains(schema("c_char").dataType))
+ assert(Seq[DataType](VarcharType(10),
StringType).contains(schema("c_varchar").dataType))
+ assertResult(BinaryType)(schema("c_bin").dataType)
+ assertResult(DecimalType(10, 0))(schema("c_dec").dataType)
+ assertResult(DecimalType(12, 0))(schema("c_dec1").dataType)
+ assertResult(DecimalType(12, 3))(schema("c_dec2").dataType)
+ assertResult(BlobType())(schema("data").dataType)
+ assert(!schema("c_notnull").nullable)
+ assertResult("a
comment")(schema("c_comment").metadata.getString("comment"))
+ }
+
+ test("Test parse CREATE TABLE with BLOB column and complex data types") {
+ // Exercises the ARRAY / MAP / STRUCT arms and BLOB-in-struct metadata
handling.
+ val plan = parse(
+ s"""
+ |CREATE TABLE blob_complex_tbl (
+ | c_arr ARRAY<INT>,
+ | c_map MAP<STRING, INT>,
+ | c_struct STRUCT<a: INT, b: STRING>,
+ | c_nested_blob STRUCT<x: BLOB>,
+ | data BLOB
+ |) USING hudi
+ """.stripMargin)
+ val schema = plan.tableSchema
+ assertResult(ArrayType(IntegerType))(schema("c_arr").dataType)
+ assertResult(MapType(StringType, IntegerType))(schema("c_map").dataType)
+ val inner = schema("c_struct").dataType.asInstanceOf[StructType]
+ assertResult(IntegerType)(inner("a").dataType)
+ assertResult(StringType)(inner("b").dataType)
+ // A BLOB nested inside a struct still carries the BLOB type descriptor.
+ val nested = schema("c_nested_blob").dataType.asInstanceOf[StructType]("x")
+ assertResult(BlobType())(nested.dataType)
+ assertResult(HoodieSchemaType.BLOB.name())(
+ nested.metadata.getString(HoodieSchema.TYPE_METADATA_FIELD))
+ }
+
+ test("Test parse CREATE TABLE with BLOB column and interval data types") {
+ val plan = parse(
+ s"""
+ |CREATE TABLE blob_ivl_tbl (
+ | i_year INTERVAL YEAR,
+ | i_ym INTERVAL YEAR TO MONTH,
+ | i_day INTERVAL DAY,
+ | i_ds INTERVAL DAY TO SECOND,
+ | data BLOB
+ |) USING hudi
+ """.stripMargin)
+ val schema = plan.tableSchema
+
assertResult(YearMonthIntervalType(YearMonthIntervalType.YEAR))(schema("i_year").dataType)
+ assertResult(YearMonthIntervalType(YearMonthIntervalType.YEAR,
YearMonthIntervalType.MONTH))(
+ schema("i_ym").dataType)
+
assertResult(DayTimeIntervalType(DayTimeIntervalType.DAY))(schema("i_day").dataType)
+ assertResult(DayTimeIntervalType(DayTimeIntervalType.DAY,
DayTimeIntervalType.SECOND))(
+ schema("i_ds").dataType)
+ assertResult(BlobType())(schema("data").dataType)
+
+ // Endpoints where the end field does not follow the start are rejected by
both interval
+ // data-type visitors. The grammar only allows YEAR/MONTH -> MONTH and
DAY/HOUR/MINUTE/SECOND
+ // -> HOUR/MINUTE/SECOND, so these stay grammatical yet still hit the
builder's end <= start guard.
+ checkExceptionContain(
+ "CREATE TABLE blob_bad_ym (id BIGINT, bad INTERVAL MONTH TO MONTH, data
BLOB) USING hudi")(
+ "are not supported")
+ checkExceptionContain(
+ "CREATE TABLE blob_bad_dt (id BIGINT, bad INTERVAL SECOND TO HOUR, data
BLOB) USING hudi")(
+ "are not supported")
+
+ // An unknown primitive type name is rejected.
+ checkExceptionContain(
+ "CREATE TABLE blob_bad_type (id BIGINT, weird sometype, data BLOB) USING
hudi")(
+ "is not supported")
+ }
+
+ test("Test parse CREATE TABLE with BLOB column and partition transforms") {
+ val plan = parse(
+ s"""
+ |CREATE TABLE blob_tf_tbl (
+ | id BIGINT,
+ | ts DATE,
+ | region STRING,
+ | data BLOB
+ |) USING hudi
+ |PARTITIONED BY (region, years(ts), months(ts), days(ts), hours(ts),
myfunc(id))
+ """.stripMargin)
+ assertResult(BlobType())(plan.tableSchema("data").dataType)
+ assertResult(Seq(Seq("region")))(transformFieldRefs(transformByName(plan,
"identity")))
+ assertResult(Seq(Seq("ts")))(transformFieldRefs(transformByName(plan,
"years")))
+ assertResult(Seq(Seq("ts")))(transformFieldRefs(transformByName(plan,
"months")))
+ assertResult(Seq(Seq("ts")))(transformFieldRefs(transformByName(plan,
"days")))
+ assertResult(Seq(Seq("ts")))(transformFieldRefs(transformByName(plan,
"hours")))
+ // an arbitrary function transform falls through to the generic
apply-transform arm
+ assertResult(Seq(Seq("id")))(transformFieldRefs(transformByName(plan,
"myfunc")))
+
+ // bucket(numBuckets, col) with int, long and short number-of-buckets
literals exercises the
+ // three numeric arms of the bucket handling.
+ Seq("4", "4L", "4S").foreach { numLiteral =>
+ val bp = parse(
+ s"CREATE TABLE blob_bkt_tbl (id BIGINT, data BLOB) USING hudi " +
+ s"PARTITIONED BY (bucket($numLiteral, id))")
+ val bkt = transformByName(bp, "bucket")
+ assertResult("4")(firstLiteralArg(bkt).value.toString)
+ assertResult(Seq(Seq("id")))(transformFieldRefs(bkt))
+ }
+ }
+
+ test("Test parse CREATE TABLE with BLOB column and typed transform-argument
literals") {
+ // Constant transform arguments exercise the literal visitors: string,
integer, big-integer and
+ // exponent numerics (the private numeric-literal helper), the typed date
constructor, and both
+ // interval forms (multi-unit and unit-to-unit). A typed timestamp
constructor is not used
+ // because the Spark 4.x extended parser rejects a bare TIMESTAMP token.
Note: a bare
+ // true/false/null in this position is parsed as a column reference
(qualifiedName takes
+ // precedence over a constant in the grammar under the default non-ANSI
config), so the boolean
+ // and null literal visitors are not reachable from a CREATE TABLE
statement.
+ val plan = parse(
+ s"""
+ |CREATE TABLE blob_lit_tbl (
+ | id BIGINT,
+ | data BLOB
+ |) USING hudi
+ |PARTITIONED BY (
+ | str_t('x', id),
+ | int_t(7, id),
+ | long_t(9000000000L, id),
+ | exp_t(1E3, id),
+ | date_t(DATE '2020-01-01', id),
+ | mu_ivl_t(INTERVAL '1' DAY, id),
+ | uu_ivl_t(INTERVAL '1-2' YEAR TO MONTH, id)
+ |)
+ """.stripMargin)
+ assertResult(StringType)(firstLiteralArg(transformByName(plan,
"str_t")).dataType)
+ assertResult("x")(firstLiteralArg(transformByName(plan,
"str_t")).value.toString)
+ assertResult(IntegerType)(firstLiteralArg(transformByName(plan,
"int_t")).dataType)
+ assertResult(LongType)(firstLiteralArg(transformByName(plan,
"long_t")).dataType)
+ assertResult(DoubleType)(firstLiteralArg(transformByName(plan,
"exp_t")).dataType)
+ assertResult(DateType)(firstLiteralArg(transformByName(plan,
"date_t")).dataType)
+ assertResult(DayTimeIntervalType(DayTimeIntervalType.DAY,
DayTimeIntervalType.DAY))(
+ firstLiteralArg(transformByName(plan, "mu_ivl_t")).dataType)
+ assertResult(YearMonthIntervalType(YearMonthIntervalType.YEAR,
YearMonthIntervalType.MONTH))(
+ firstLiteralArg(transformByName(plan, "uu_ivl_t")).dataType)
+ }
+
+ test("Test parse CREATE TABLE with BLOB column and invalid partition
transforms") {
+ // Non-numeric number of buckets.
+ checkExceptionContain(
+ "CREATE TABLE blob_e1 (id BIGINT, data BLOB) USING hudi PARTITIONED BY
(bucket('x', id))")(
+ "Invalid number of buckets")
+ // A non-column-reference where a column is required.
+ checkExceptionContain(
+ "CREATE TABLE blob_e2 (id BIGINT, data BLOB) USING hudi PARTITIONED BY
(bucket(4, 5))")(
+ "Expected a column reference")
+ // A single-field transform given more than one argument.
+ checkExceptionContain(
+ "CREATE TABLE blob_e3 (id BIGINT, ts DATE, data BLOB) USING hudi " +
+ "PARTITIONED BY (years(id, ts))")(
+ "Too many arguments")
+ }
+
+ test("Test parse CREATE TABLE with BLOB column and file-format / row-format
clauses") {
+ // Generic STORED AS format.
+ assertResult(BlobType())(
+ parse("CREATE TABLE blob_ff1 (id BIGINT, data BLOB) STORED AS PARQUET")
+ .tableSchema("data").dataType)
+ // STORED AS INPUTFORMAT ... OUTPUTFORMAT ... (the table-file-format arm).
+ assertResult(BlobType())(
+ parse("CREATE TABLE blob_ff2 (id BIGINT, data BLOB) " +
+ "STORED AS INPUTFORMAT 'com.example.InFmt' OUTPUTFORMAT
'com.example.OutFmt'")
+ .tableSchema("data").dataType)
+ // ROW FORMAT SERDE on its own.
+ assertResult(BlobType())(
+ parse("CREATE TABLE blob_ff3 (id BIGINT, data BLOB) " +
+ "ROW FORMAT SERDE
'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe'")
+ .tableSchema("data").dataType)
+ // ROW FORMAT DELIMITED on its own.
+ assertResult(BlobType())(
+ parse("CREATE TABLE blob_ff4 (id BIGINT, data BLOB) " +
+ "ROW FORMAT DELIMITED FIELDS TERMINATED BY ','")
+ .tableSchema("data").dataType)
+ // Compatible ROW FORMAT SERDE + STORED AS SEQUENCEFILE.
+ assertResult(BlobType())(
+ parse("CREATE TABLE blob_ff5 (id BIGINT, data BLOB) " +
+ "ROW FORMAT SERDE 'com.example.Serde' STORED AS SEQUENCEFILE")
+ .tableSchema("data").dataType)
+ // Compatible ROW FORMAT DELIMITED + STORED AS TEXTFILE.
+ assertResult(BlobType())(
+ parse("CREATE TABLE blob_ff6 (id BIGINT, data BLOB) " +
+ "ROW FORMAT DELIMITED FIELDS TERMINATED BY ',' STORED AS TEXTFILE")
+ .tableSchema("data").dataType)
+
+ // ROW FORMAT DELIMITED with a non-text file format is rejected.
+ checkExceptionContain(
+ "CREATE TABLE blob_ferr1 (id BIGINT, data BLOB) " +
+ "ROW FORMAT DELIMITED FIELDS TERMINATED BY ',' STORED AS PARQUET")(
+ "only compatible with 'textfile'")
+ // ROW FORMAT SERDE with a format that also specifies a serde is rejected.
+ checkExceptionContain(
+ "CREATE TABLE blob_ferr2 (id BIGINT, data BLOB) " +
+ "ROW FORMAT SERDE 'com.example.Serde' STORED AS PARQUET")(
+ "incompatible with format")
+ // STORED BY (a storage handler) is not allowed.
+ checkExceptionContain(
+ "CREATE TABLE blob_ferr3 (id BIGINT, data BLOB) STORED BY
'com.example.Handler'")(
+ "STORED BY")
Review Comment:
This negative is vacuous: `ParseException.getMessage` appends the offending
statement under `== SQL ==`, so the expected substring `STORED BY` matches no
matter why the parse failed -- even if the statement were routed to the stock
Spark parser. Pin the actual error:
```suggestion
"Operation not allowed: STORED BY")
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
