voonhous commented on code in PR #19687:
URL: https://github.com/apache/hudi/pull/19687#discussion_r3850707989
##########
hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/procedure/TestClusteringProcedure.scala:
##########
@@ -867,6 +867,47 @@ class TestClusteringProcedure extends
HoodieSparkProcedureTestBase {
}
}
+ test("Test Call run_clustering rejects unsortable order columns") {
+ // Not gated on any Spark version: BLOB, VECTOR and MAP exist on every
supported Spark, so
+ // this covers the sort-column validation on the lanes where the variant
suite is skipped.
+ withTempDir { tmp =>
+ val tableName = generateTableName
+ val basePath = s"${tmp.getCanonicalPath}/$tableName"
+ spark.sql(
+ s"""
+ |create table $tableName (
+ | id int,
+ | name string,
+ | content blob,
+ | embedding vector(4),
+ | attrs map<string, string>,
+ | ts long
+ |) using hudi
+ | options (
+ | primaryKey = 'id',
+ | orderingFields = 'ts'
+ | )
+ | location '$basePath'
+ """.stripMargin)
+ spark.sql(s"insert into $tableName values (1, 'a1', null, null, null,
1000)")
+
+ // The procedure validates the order columns up front, before any plan
is scheduled.
+ Seq("content", "embedding", "attrs").foreach { col =>
+ checkNestedExceptionContains(s"call run_clustering(table =>
'$tableName', order => '$col')")(
+ s"Sorting by column '$col'")
+ }
+ // Case-insensitive, mirroring Spark's column resolution.
+ checkNestedExceptionContains(s"call run_clustering(table =>
'$tableName', order => 'CONTENT')")(
+ "Sorting by column 'CONTENT'")
+ // The execution-time twin: configured plan-strategy sort columns skip
the procedure
+ // check and are rejected by the execution strategy and partitioner
constructors instead
+ // (SortUtils.validateSortableColumns).
+ checkNestedExceptionContains(
+ s"call run_clustering(table => '$tableName', options =>
'hoodie.clustering.plan.strategy.sort.columns=content')")(
Review Comment:
Added `constructorRejectsColumnsWithoutOrdering` to
`TestRowCustomColumnsSortPartitioner`: a config whose schema has a variant
column, `new RowCustomColumnsSortPartitioner(new String[] {"v"}, config)`
throws naming `'v'`, `"id"` constructs.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
