brijrajk commented on code in PR #57161:
URL: https://github.com/apache/spark/pull/57161#discussion_r3564633045
##########
sql/core/src/test/scala/org/apache/spark/sql/connector/DeleteFromTableSuiteBase.scala:
##########
@@ -1037,6 +1037,59 @@ abstract class DeleteFromTableSuiteBase extends
RowLevelOperationSuiteBase {
}
}
+ protected def checkRowLevelOperationOptions(
+ func: => Unit,
+ expectedOptions: (String, String)*): Unit = {
+ val Seq(qe) = withQueryExecutionsCaptured(spark)(func)
+ val writeRelation = qe.optimizedPlan.collectFirst {
+ case rd: ReplaceData => rd.table
+ case wd: WriteDelta => wd.table
+ }.getOrElse(fail("couldn't find row-level operation in optimized plan"))
+ .asInstanceOf[DataSourceV2Relation]
+ val operation =
writeRelation.table.asInstanceOf[RowLevelOperationTable].operation
+ .asInstanceOf[RowLevelOperationWithOptions]
+ expectedOptions.foreach { case (key, value) =>
+ assert(writeRelation.options.get(key) === value, s"relation option
'$key'")
+ assert(operation.options.get(key) === value, s"row-level operation
option '$key'")
+ assert(table.lastWriteInfo.options().get(key) === value, s"write option
'$key'")
+ }
+ }
+
+ test("SPARK-58008: delete with dynamic options") {
+ createAndInitTable("pk INT NOT NULL, salary INT, dep STRING",
+ """{ "pk": 1, "salary": 100, "dep": "hr" }
+ |{ "pk": 2, "salary": 200, "dep": "software" }
+ |""".stripMargin)
+
+ // Use salary < 200 (LessThan) rather than pk = 1 (EqualTo) so the
optimizer does not
+ // replace the row-level plan with a filter-only deleteWhere call via
+ // OptimizeMetadataOnlyDeleteFromTable (canDeleteWhere returns false for
LessThan).
+ checkRowLevelOperationOptions(
+ sql(s"DELETE FROM $tableNameAsString WITH (`write.split-size` = 10)
WHERE salary < 200"),
+ "write.split-size" -> "10")
+
+ checkAnswer(
+ sql(s"SELECT * FROM $tableNameAsString"),
+ Row(2, 200, "software") :: Nil)
+ }
+
+ test("SPARK-58008: delete with dynamic options and a subquery on the same
table") {
Review Comment:
Thanks @anuragmantri! Both points from your review are now addressed:
**Options-aware APIs** (review body): Added `deleteWhere(Predicate[],
CaseInsensitiveStringMap)` to `SupportsDeleteV2` and
`truncateTable(CaseInsensitiveStringMap)` to both `SupportsDeleteV2` and
`TruncatableTable` with back-compatible defaults. Wired `options` through
`DeleteFromTableWithFilters`, `DeleteFromTableExec`, `TruncateTableExec`, and
all three planning paths in `DataSourceV2Strategy`.
**CTE test** (this comment): Added `SPARK-58008: delete with dynamic options
and a CTE` — uses a `WITH cte AS (...)` in the WHERE clause together with the
`WITH (options)` clause, verifying options flow correctly through row-level
operations when a CTE is involved.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
