voonhous commented on code in PR #19408:
URL: https://github.com/apache/hudi/pull/19408#discussion_r3691314100
##########
hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/dml/schema/TestBlobDataType.scala:
##########
@@ -282,4 +284,256 @@ class TestBlobDataType extends HoodieSparkSqlTestBase {
"Expected at least one .clean instant on the timeline after
compaction")
})
}
+
+ // The following cases are parser-coverage only: a BLOB column routes the
whole CREATE TABLE
+ // through the extended AST builder, so its clause visitors run. parsePlan
is purely syntactic
+ // (no catalog, no execution), which lets us exercise clauses Hudi does not
support at execution
+ // time (transform partitioning, STORED AS / ROW FORMAT, interval columns).
The BLOB column type
+ // itself proves routing because the stock Spark parser rejects the BLOB
type name.
+
+ private def parse(sql: String): CreateTable =
+ spark.sessionState.sqlParser.parsePlan(sql).asInstanceOf[CreateTable]
+
+ test("Test parse CREATE TABLE with BLOB column and primitive data types") {
+ // Exercises the primitive-data-type match arms plus NOT NULL and column
COMMENT.
+ val plan = parse(
+ s"""
+ |CREATE TABLE blob_prim_tbl (
+ | c_bool BOOLEAN,
+ | c_tiny TINYINT,
+ | c_small SMALLINT,
+ | c_int INT,
+ | c_big BIGINT,
+ | c_float FLOAT,
+ | c_double DOUBLE,
+ | c_date DATE,
+ | c_str STRING,
+ | c_char CHAR(5),
+ | c_varchar VARCHAR(10),
+ | c_bin BINARY,
+ | c_dec DECIMAL,
+ | c_dec1 DECIMAL(12),
+ | c_dec2 DECIMAL(12, 3),
+ | c_notnull INT NOT NULL,
+ | c_comment INT COMMENT 'a comment',
+ | data BLOB
+ |) USING hudi
+ """.stripMargin)
+ val schema = plan.tableSchema
+ assertResult(BooleanType)(schema("c_bool").dataType)
+ assertResult(ByteType)(schema("c_tiny").dataType)
+ assertResult(ShortType)(schema("c_small").dataType)
+ assertResult(IntegerType)(schema("c_int").dataType)
+ assertResult(LongType)(schema("c_big").dataType)
+ assertResult(FloatType)(schema("c_float").dataType)
+ assertResult(DoubleType)(schema("c_double").dataType)
+ assertResult(DateType)(schema("c_date").dataType)
+ assertResult(StringType)(schema("c_str").dataType)
+ // CHAR/VARCHAR may be preserved or replaced with STRING depending on the
Spark version.
+ assert(Seq[DataType](CharType(5),
StringType).contains(schema("c_char").dataType))
+ assert(Seq[DataType](VarcharType(10),
StringType).contains(schema("c_varchar").dataType))
+ assertResult(BinaryType)(schema("c_bin").dataType)
+ assertResult(DecimalType(10, 0))(schema("c_dec").dataType)
+ assertResult(DecimalType(12, 0))(schema("c_dec1").dataType)
+ assertResult(DecimalType(12, 3))(schema("c_dec2").dataType)
+ assertResult(BlobType())(schema("data").dataType)
+ assert(!schema("c_notnull").nullable)
+ assertResult("a
comment")(schema("c_comment").metadata.getString("comment"))
+ }
+
+ test("Test parse CREATE TABLE with BLOB column and complex data types") {
+ // Exercises the ARRAY / MAP / STRUCT arms and BLOB-in-struct metadata
handling.
+ val plan = parse(
+ s"""
+ |CREATE TABLE blob_complex_tbl (
+ | c_arr ARRAY<INT>,
+ | c_map MAP<STRING, INT>,
+ | c_struct STRUCT<a: INT, b: STRING>,
+ | c_nested_blob STRUCT<x: BLOB>,
+ | data BLOB
+ |) USING hudi
+ """.stripMargin)
+ val schema = plan.tableSchema
+ assertResult(ArrayType(IntegerType))(schema("c_arr").dataType)
+ assertResult(MapType(StringType, IntegerType))(schema("c_map").dataType)
+ val inner = schema("c_struct").dataType.asInstanceOf[StructType]
+ assertResult(IntegerType)(inner("a").dataType)
+ assertResult(StringType)(inner("b").dataType)
+ // A BLOB nested inside a struct still carries the BLOB type descriptor.
+ val nested = schema("c_nested_blob").dataType.asInstanceOf[StructType]("x")
+ assertResult(BlobType())(nested.dataType)
+ assertResult(HoodieSchemaType.BLOB.name())(
+ nested.metadata.getString(HoodieSchema.TYPE_METADATA_FIELD))
+ }
+
+ test("Test parse CREATE TABLE with BLOB column and interval data types") {
+ val plan = parse(
+ s"""
+ |CREATE TABLE blob_ivl_tbl (
+ | i_year INTERVAL YEAR,
+ | i_ym INTERVAL YEAR TO MONTH,
+ | i_day INTERVAL DAY,
+ | i_ds INTERVAL DAY TO SECOND,
+ | data BLOB
+ |) USING hudi
+ """.stripMargin)
+ val schema = plan.tableSchema
+
assertResult(YearMonthIntervalType(YearMonthIntervalType.YEAR))(schema("i_year").dataType)
+ assertResult(YearMonthIntervalType(YearMonthIntervalType.YEAR,
YearMonthIntervalType.MONTH))(
+ schema("i_ym").dataType)
+
assertResult(DayTimeIntervalType(DayTimeIntervalType.DAY))(schema("i_day").dataType)
+ assertResult(DayTimeIntervalType(DayTimeIntervalType.DAY,
DayTimeIntervalType.SECOND))(
+ schema("i_ds").dataType)
+ assertResult(BlobType())(schema("data").dataType)
+
+ // Endpoints where the end field does not follow the start are rejected by
both interval
+ // data-type visitors. The grammar only allows YEAR/MONTH -> MONTH and
DAY/HOUR/MINUTE/SECOND
+ // -> HOUR/MINUTE/SECOND, so these stay grammatical yet still hit the
builder's end <= start guard.
+ checkExceptionContain(
+ "CREATE TABLE blob_bad_ym (id BIGINT, bad INTERVAL MONTH TO MONTH, data
BLOB) USING hudi")(
+ "are not supported")
+ checkExceptionContain(
+ "CREATE TABLE blob_bad_dt (id BIGINT, bad INTERVAL SECOND TO HOUR, data
BLOB) USING hudi")(
+ "are not supported")
+
+ // An unknown primitive type name is rejected.
+ checkExceptionContain(
+ "CREATE TABLE blob_bad_type (id BIGINT, weird sometype, data BLOB) USING
hudi")(
+ "is not supported")
+ }
+
+ test("Test parse CREATE TABLE with BLOB column and partition transforms") {
+ val plan = parse(
+ s"""
+ |CREATE TABLE blob_tf_tbl (
+ | id BIGINT,
+ | ts DATE,
+ | region STRING,
+ | data BLOB
+ |) USING hudi
+ |PARTITIONED BY (region, years(ts), months(ts), days(ts), hours(ts),
myfunc(id))
+ """.stripMargin)
+ assertResult(BlobType())(plan.tableSchema("data").dataType)
+ assertResult(Seq(Seq("region")))(transformFieldRefs(transformByName(plan,
"identity")))
+ assertResult(Seq(Seq("ts")))(transformFieldRefs(transformByName(plan,
"years")))
+ assertResult(Seq(Seq("ts")))(transformFieldRefs(transformByName(plan,
"months")))
+ assertResult(Seq(Seq("ts")))(transformFieldRefs(transformByName(plan,
"days")))
+ assertResult(Seq(Seq("ts")))(transformFieldRefs(transformByName(plan,
"hours")))
+ // an arbitrary function transform falls through to the generic
apply-transform arm
+ assertResult(Seq(Seq("id")))(transformFieldRefs(transformByName(plan,
"myfunc")))
+
+ // bucket(numBuckets, col) with int, long and short number-of-buckets
literals exercises the
+ // three numeric arms of the bucket handling.
+ Seq("4", "4L", "4S").foreach { numLiteral =>
+ val bp = parse(
+ s"CREATE TABLE blob_bkt_tbl (id BIGINT, data BLOB) USING hudi " +
+ s"PARTITIONED BY (bucket($numLiteral, id))")
+ val bkt = transformByName(bp, "bucket")
+ assertResult("4")(firstLiteralArg(bkt).value.toString)
+ assertResult(Seq(Seq("id")))(transformFieldRefs(bkt))
+ }
+ }
+
+ test("Test parse CREATE TABLE with BLOB column and typed transform-argument
literals") {
+ // Constant transform arguments exercise the literal visitors: string,
integer, big-integer and
+ // exponent numerics (the private numeric-literal helper), the typed date
constructor, and both
+ // interval forms (multi-unit and unit-to-unit). A typed timestamp
constructor is not used
+ // because the Spark 4.x extended parser rejects a bare TIMESTAMP token.
Note: a bare
+ // true/false/null in this position is parsed as a column reference
(qualifiedName takes
+ // precedence over a constant in the grammar under the default non-ANSI
config), so the boolean
+ // and null literal visitors are not reachable from a CREATE TABLE
statement.
Review Comment:
Done. The comment now states `null` is a column reference only under
non-ANSI keyword mode and a null literal under ANSI, and `visitBooleanLiteral`
(dead in both modes) is tracked for deletion in #19451.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
