This is an automated email from the ASF dual-hosted git repository.

peter-toth pushed a commit to branch branch-4.x
in repository https://gitbox.apache.org/repos/asf/spark.git


The following commit(s) were added to refs/heads/branch-4.x by this push:
     new d0d4b93a81a8 [SPARK-58008][SQL] Support dynamic table options for 
DELETE
d0d4b93a81a8 is described below

commit d0d4b93a81a8d4bb419124a57f476f262f23af65
Author: BRIJ RAJ KISHORE <[email protected]>
AuthorDate: Tue Jul 28 11:06:16 2026 +0200

    [SPARK-58008][SQL] Support dynamic table options for DELETE
    
    ### What changes were proposed in this pull request?
    
    Extends the per-statement `WITH (key = value)` options clause to `DELETE` 
statements, consistent with the support already present for `SELECT` 
(SPARK-36680), `INSERT` (SPARK-49098), and `UPDATE` (SPARK-57681).
    
    Example syntax:
    ```sql
    DELETE FROM catalog.db.table WITH (`write.split-size` = 10) WHERE id < 100
    
    DELETE FROM catalog.db.table AS t WITH (`k` = 'v') WHERE t.id < 100
    ```
    
    The options are forwarded through the `DataSourceV2Relation`, 
`RowLevelOperationInfo`, and `LogicalWriteInfo` so that DataSource V2 
connectors receive them at every layer of the write path.
    
    Options are also wired through the metadata-only DELETE paths:
    - `SupportsDeleteV2.deleteWhere(Predicate[], CaseInsensitiveStringMap)` — 
new back-compatible overload
    - `SupportsDeleteV2.truncateTable(CaseInsensitiveStringMap)` — new 
back-compatible overload
    - `TruncatableTable.truncateTable(CaseInsensitiveStringMap)` — new 
back-compatible overload
    - `DeleteFromTableWithFilters`, `DeleteFromTableExec`, `TruncateTableExec` 
— each carries `options`
    - `DataSourceV2Strategy` — threads `r.options` through all three 
non-row-level DELETE planning paths
    
    ### Why are the changes needed?
    
    `DELETE` was the only DML statement missing this capability. Adding it 
completes the feature set across all row-level write operations and allows 
connectors to accept per-statement tuning options (e.g. split size, isolation 
level) on deletes without requiring separate configuration.
    
    ### Does this PR introduce _any_ user-facing change?
    
    Yes — new SQL syntax. `DELETE FROM tbl WITH (k = v) WHERE ...` is now valid.
    
    ### How was this patch tested?
    
    - `DDLParserSuite`: two new parser tests covering `DELETE ... WITH (...)` 
without and with an alias.
    - `DeleteFromTableSuiteBase`: five new end-to-end tests verifying that 
options reach the `DataSourceV2Relation`, `RowLevelOperationInfo`, and 
`LogicalWriteInfo` layers across the row-level, subquery, CTE, metadata-only 
deleteWhere, and delete-all paths.
    
    ### Was this patch authored or co-authored using generative AI tooling?
    
    Generated-by: Claude Sonnet 4.6
    
    Closes #57161 from brijrajk/SPARK-58008-delete-with-options.
    
    Authored-by: BRIJ RAJ KISHORE <[email protected]>
    Signed-off-by: Peter Toth <[email protected]>
    (cherry picked from commit 5221eedcd64502446d031212e984ea2cd10467bc)
    Signed-off-by: Peter Toth <[email protected]>
---
 .../spark/sql/catalyst/parser/SqlBaseParser.g4     |   2 +-
 .../sql/connector/catalog/SupportsDeleteV2.java    |  19 +++
 .../sql/connector/catalog/TruncatableTable.java    |  15 +++
 .../catalyst/analysis/RewriteDeleteFromTable.scala |   3 +-
 .../spark/sql/catalyst/parser/AstBuilder.scala     |   3 +-
 .../sql/catalyst/plans/logical/v2Commands.scala    |   4 +-
 .../spark/sql/catalyst/parser/DDLParserSuite.scala |  27 ++++
 .../catalog/InMemoryRowLevelOperationTable.scala   |  51 ++++++++
 .../InMemoryRowLevelOperationTableCatalog.scala    |  25 ++++
 .../datasources/v2/DataSourceV2Strategy.scala      |   8 +-
 .../datasources/v2/DeleteFromTableExec.scala       |   4 +-
 .../v2/OptimizeMetadataOnlyDeleteFromTable.scala   |   5 +-
 .../datasources/v2/TruncateTableExec.scala         |   6 +-
 .../sql/connector/DeleteFromTableSuiteBase.scala   | 142 ++++++++++++++++++++-
 14 files changed, 297 insertions(+), 17 deletions(-)

diff --git 
a/sql/api/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBaseParser.g4 
b/sql/api/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBaseParser.g4
index cdeb34c6a0ca..c591dae38da5 100644
--- 
a/sql/api/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBaseParser.g4
+++ 
b/sql/api/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBaseParser.g4
@@ -747,7 +747,7 @@ resource
 dmlStatementNoWith
     : insertInto (query | LEFT_PAREN query RIGHT_PAREN queryAlias=tableAlias)  
    #singleInsertQuery
     | fromClause multiInsertQueryBody+                                         
    #multiInsertQuery
-    | DELETE FROM identifierReference tableAlias whereClause?                  
    #deleteFromTable
+    | DELETE FROM identifierReference tableAlias optionsClause? whereClause?   
    #deleteFromTable
     | UPDATE identifierReference tableAlias optionsClause? setClause 
whereClause?  #updateTable
     | MERGE (WITH SCHEMA EVOLUTION)? INTO target=identifierReference 
targetAlias=tableAlias
         targetOptions=optionsClause?
diff --git 
a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/SupportsDeleteV2.java
 
b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/SupportsDeleteV2.java
index dc5911ad9899..53795dc159e4 100644
--- 
a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/SupportsDeleteV2.java
+++ 
b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/SupportsDeleteV2.java
@@ -20,6 +20,7 @@ package org.apache.spark.sql.connector.catalog;
 import org.apache.spark.annotation.Evolving;
 import org.apache.spark.sql.connector.expressions.filter.AlwaysTrue;
 import org.apache.spark.sql.connector.expressions.filter.Predicate;
+import org.apache.spark.sql.util.CaseInsensitiveStringMap;
 
 /**
  * A mix-in interface for {@link Table} delete support. Data sources can 
implement this
@@ -72,6 +73,24 @@ public interface SupportsDeleteV2 extends TruncatableTable {
    */
   void deleteWhere(Predicate[] predicates);
 
+  /**
+   * Delete data from a data source table that matches filter expressions, 
forwarding the
+   * per-statement options from a {@code DELETE ... WITH (key=value)} clause.
+   * <p>
+   * The default implementation ignores {@code options} and delegates to
+   * {@link #deleteWhere(Predicate[])}, which preserves backward compatibility 
for connectors
+   * that do not need per-statement options.
+   *
+   * @param predicates predicate expressions, used to select rows to delete 
when all expressions
+   *                   match
+   * @param options per-statement options from the {@code WITH} clause; empty 
if none were given
+   * @throws IllegalArgumentException If the delete is rejected due to 
required effort
+   * @since 4.3.0
+   */
+  default void deleteWhere(Predicate[] predicates, CaseInsensitiveStringMap 
options) {
+    deleteWhere(predicates);
+  }
+
   @Override
   default boolean truncateTable() {
     Predicate[] predicates = new Predicate[] { new AlwaysTrue() };
diff --git 
a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/TruncatableTable.java
 
b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/TruncatableTable.java
index 058e105758c7..eccd9809c5e6 100644
--- 
a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/TruncatableTable.java
+++ 
b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/TruncatableTable.java
@@ -20,6 +20,7 @@ package org.apache.spark.sql.connector.catalog;
 import org.apache.spark.annotation.Evolving;
 import org.apache.spark.sql.connector.metric.CustomMetric;
 import org.apache.spark.sql.connector.metric.CustomTaskMetric;
+import org.apache.spark.sql.util.CaseInsensitiveStringMap;
 
 /**
  * Represents a table which can be atomically truncated.
@@ -37,6 +38,20 @@ public interface TruncatableTable extends Table {
    */
   boolean truncateTable();
 
+  /**
+   * Truncate a table with per-statement options from a {@code DELETE ... WITH 
(key=value)} clause.
+   * <p>
+   * The default implementation ignores {@code options} and delegates to 
{@link #truncateTable()},
+   * which preserves backward compatibility for connectors that do not need 
per-statement options.
+   *
+   * @param options per-statement options from the {@code WITH} clause; empty 
if none were given
+   * @return true if the table was truncated successfully, false otherwise
+   * @since 4.3.0
+   */
+  default boolean truncateTable(CaseInsensitiveStringMap options) {
+    return truncateTable();
+  }
+
   /**
    * Returns an array of supported custom metrics with name and description.
    * By default it returns empty array.
diff --git 
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/RewriteDeleteFromTable.scala
 
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/RewriteDeleteFromTable.scala
index f8881e207710..c8795b8ad9b1 100644
--- 
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/RewriteDeleteFromTable.scala
+++ 
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/RewriteDeleteFromTable.scala
@@ -25,7 +25,6 @@ import 
org.apache.spark.sql.connector.catalog.{SupportsDeleteV2, SupportsRowLeve
 import org.apache.spark.sql.connector.write.{RowLevelOperationTable, 
SupportsDelta}
 import org.apache.spark.sql.connector.write.RowLevelOperation.Command.DELETE
 import org.apache.spark.sql.execution.datasources.v2.{DataSourceV2Relation, 
ExtractV2Table}
-import org.apache.spark.sql.util.CaseInsensitiveStringMap
 
 /**
  * A rule that rewrites DELETE operations using plans that operate on 
individual or groups of rows.
@@ -45,7 +44,7 @@ object RewriteDeleteFromTable extends RewriteRowLevelCommand {
           d
 
         case r @ ExtractV2Table(t: SupportsRowLevelOperations) =>
-          val table = buildOperationTable(t, DELETE, 
CaseInsensitiveStringMap.empty())
+          val table = buildOperationTable(t, DELETE, r.options)
           table.operation match {
             case _: SupportsDelta =>
               buildWriteDeltaPlan(r, table, cond)
diff --git 
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala
 
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala
index 8214c67cb8b4..2faf07b5379c 100644
--- 
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala
+++ 
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala
@@ -1244,7 +1244,8 @@ class AstBuilder extends DataTypeAstBuilder
   override def visitDeleteFromTable(
       ctx: DeleteFromTableContext): LogicalPlan = withOrigin(ctx) {
     val table = createUnresolvedRelation(
-      ctx.identifierReference, writePrivileges = 
Set(TableWritePrivilege.DELETE))
+      ctx.identifierReference, Option(ctx.optionsClause()),
+      writePrivileges = Set(TableWritePrivilege.DELETE))
     val tableAlias = getTableAliasWithoutColumnAlias(ctx.tableAlias(), 
"DELETE")
     val aliasedTable = tableAlias.map(SubqueryAlias(_, table)).getOrElse(table)
     val predicate = if (ctx.whereClause() != null) {
diff --git 
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/v2Commands.scala
 
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/v2Commands.scala
index e82a65c8f99e..85f3c726c7ec 100644
--- 
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/v2Commands.scala
+++ 
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/v2Commands.scala
@@ -42,6 +42,7 @@ import org.apache.spark.sql.errors.QueryExecutionErrors
 import org.apache.spark.sql.execution.datasources.v2.{DataSourceV2Relation, 
ExtractV2Table}
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.types.{BooleanType, DataType, IntegerType, 
MapType, MetadataBuilder, StringType, StructType}
+import org.apache.spark.sql.util.CaseInsensitiveStringMap
 import org.apache.spark.util.ArrayImplicits._
 import org.apache.spark.util.Utils
 import org.apache.spark.util.collection.BitSet
@@ -1107,7 +1108,8 @@ case class DeleteFromTable(
  */
 case class DeleteFromTableWithFilters(
     table: LogicalPlan,
-    condition: Seq[Predicate]) extends LeafCommand
+    condition: Seq[Predicate],
+    options: CaseInsensitiveStringMap = CaseInsensitiveStringMap.empty()) 
extends LeafCommand
 
 /**
  * The logical plan of the UPDATE TABLE command.
diff --git 
a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/DDLParserSuite.scala
 
b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/DDLParserSuite.scala
index 5d8ff7d2155b..d85cb0b93133 100644
--- 
a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/DDLParserSuite.scala
+++ 
b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/DDLParserSuite.scala
@@ -2327,6 +2327,33 @@ class DDLParserSuite extends AnalysisTest {
         stop = 56))
   }
 
+  test("SPARK-58008: delete from table: with options") {
+    parseCompare(
+      """
+        |DELETE FROM testcat.ns1.ns2.tbl WITH (`write.split-size` = 10)
+        |WHERE a = 1
+      """.stripMargin,
+      DeleteFromTable(
+        UnresolvedRelation(Seq("testcat", "ns1", "ns2", "tbl"),
+          new CaseInsensitiveStringMap(
+            java.util.Map.of("write.split-size", "10"))),
+        EqualTo(UnresolvedAttribute("a"), Literal(1))))
+  }
+
+  test("SPARK-58008: delete from table: with options and alias") {
+    parseCompare(
+      """
+        |DELETE FROM testcat.ns1.ns2.tbl AS t WITH (`k` = 'v')
+        |WHERE t.a = 2
+      """.stripMargin,
+      DeleteFromTable(
+        SubqueryAlias("t",
+          UnresolvedRelation(Seq("testcat", "ns1", "ns2", "tbl"),
+            new CaseInsensitiveStringMap(
+              java.util.Map.of("k", "v")))),
+        EqualTo(UnresolvedAttribute("t.a"), Literal(2))))
+  }
+
   test("update table: basic") {
     parseCompare(
       """
diff --git 
a/sql/catalyst/src/test/scala/org/apache/spark/sql/connector/catalog/InMemoryRowLevelOperationTable.scala
 
b/sql/catalyst/src/test/scala/org/apache/spark/sql/connector/catalog/InMemoryRowLevelOperationTable.scala
index f62f1b0dce25..fd02f926e814 100644
--- 
a/sql/catalyst/src/test/scala/org/apache/spark/sql/connector/catalog/InMemoryRowLevelOperationTable.scala
+++ 
b/sql/catalyst/src/test/scala/org/apache/spark/sql/connector/catalog/InMemoryRowLevelOperationTable.scala
@@ -24,6 +24,7 @@ import 
org.apache.spark.sql.catalyst.expressions.GenericInternalRow
 import org.apache.spark.sql.connector.catalog.constraints.Constraint
 import org.apache.spark.sql.connector.distributions.{Distribution, 
Distributions}
 import org.apache.spark.sql.connector.expressions.{FieldReference, 
LogicalExpressions, NamedReference, SortDirection, SortOrder, Transform}
+import org.apache.spark.sql.connector.expressions.filter.Predicate
 import org.apache.spark.sql.connector.read.{Scan, ScanBuilder}
 import org.apache.spark.sql.connector.write.{BatchWrite, DeltaBatchWrite, 
DeltaWrite, DeltaWriteBuilder, DeltaWriter, DeltaWriterFactory, 
LogicalWriteInfo, PhysicalWriteInfo, RequiresDistributionAndOrdering, 
RowLevelOperation, RowLevelOperationBuilder, RowLevelOperationInfo, 
SupportsDelta, Write, WriteBuilder, WriterCommitMessage}
 import org.apache.spark.sql.connector.write.RowLevelOperation.Command
@@ -86,6 +87,14 @@ class InMemoryRowLevelOperationTable private (
   // used in row-level operation tests to verify passed records
   // (operation, id, metadata, row)
   var lastWriteLog: Seq[InternalRow] = Seq.empty
+  // used in delete-with-options tests to verify options reach the connector 
on the deleteWhere path
+  var lastDeleteOptions: CaseInsensitiveStringMap = 
CaseInsensitiveStringMap.empty()
+
+  override def deleteWhere(
+      predicates: Array[Predicate], options: CaseInsensitiveStringMap): Unit = 
{
+    lastDeleteOptions = options
+    deleteWhere(predicates)
+  }
 
   override def copy(): Table = {
     val copied = InMemoryRowLevelOperationTable.withColumns(
@@ -323,3 +332,45 @@ object InMemoryRowLevelOperationTable {
       name, columns, partitioning, properties, constraints, tableId)
   }
 }
+
+/**
+ * An in-memory table that implements only [[TruncatableTable]] (NOT 
[[SupportsDeleteV2]]).
+ * Used to exercise the [[TruncateTableExec]] planning path for DELETE 
statements with no WHERE.
+ */
+class InMemoryTruncatableOnlyTable(
+    name: String,
+    columns: Array[Column],
+    partitioning: Array[Transform],
+    properties: util.Map[String, String],
+    constraints: Array[Constraint])
+  extends InMemoryBaseTable(name, columns, partitioning, properties, 
constraints)
+  with TruncatableTable {
+
+  var lastTruncateOptions: CaseInsensitiveStringMap = 
CaseInsensitiveStringMap.empty()
+
+  override def newWriteBuilder(info: LogicalWriteInfo): WriteBuilder = {
+    InMemoryBaseTable.maybeSimulateFailedTableWrite(new 
CaseInsensitiveStringMap(properties))
+    InMemoryBaseTable.maybeSimulateFailedTableWrite(info.options)
+    new InMemoryWriterBuilder(info) {
+      override def truncate(): WriteBuilder = {
+        writer = new TruncateAndAppend(this.info)
+        streamingWriter = new StreamingTruncateAndAppend(this.info)
+        this
+      }
+    }
+  }
+
+  override def truncateTable(): Boolean = {
+    lastTruncateOptions = CaseInsensitiveStringMap.empty()
+    dataMap.synchronized { dataMap.clear() }
+    increaseVersion()
+    true
+  }
+
+  override def truncateTable(options: CaseInsensitiveStringMap): Boolean = {
+    lastTruncateOptions = options
+    dataMap.synchronized { dataMap.clear() }
+    increaseVersion()
+    true
+  }
+}
diff --git 
a/sql/catalyst/src/test/scala/org/apache/spark/sql/connector/catalog/InMemoryRowLevelOperationTableCatalog.scala
 
b/sql/catalyst/src/test/scala/org/apache/spark/sql/connector/catalog/InMemoryRowLevelOperationTableCatalog.scala
index a29f1ed29ee8..95e6659956ca 100644
--- 
a/sql/catalyst/src/test/scala/org/apache/spark/sql/connector/catalog/InMemoryRowLevelOperationTableCatalog.scala
+++ 
b/sql/catalyst/src/test/scala/org/apache/spark/sql/connector/catalog/InMemoryRowLevelOperationTableCatalog.scala
@@ -151,3 +151,28 @@ class PartialSchemaEvolutionCatalog extends 
InMemoryRowLevelOperationTableCatalo
       currentSchema: StructType,
       changes: Seq[TableChange]): StructType = currentSchema
 }
+
+/**
+ * A catalog that creates [[InMemoryTruncatableOnlyTable]] instances - tables 
that implement
+ * [[TruncatableTable]] but NOT [[SupportsDeleteV2]]. Used to test the 
[[TruncateTableExec]]
+ * planning path for DELETE statements with no WHERE clause.
+ */
+class InMemoryTruncatableOnlyTableCatalog extends InMemoryTableCatalog {
+  import CatalogV2Implicits._
+
+  override def loadTable(ident: Identifier): Table = liveTable(ident)
+
+  override def createTable(ident: Identifier, tableInfo: TableInfo): Table = {
+    if (tables.containsKey(ident)) {
+      throw new TableAlreadyExistsException(ident.asMultipartIdentifier)
+    }
+    InMemoryTableCatalog.maybeSimulateFailedTableCreation(tableInfo.properties)
+    val tableName = s"$name.${ident.quoted}"
+    val columns = tableInfo.columns
+    val table = new InMemoryTruncatableOnlyTable(
+      tableName, columns, tableInfo.partitions, tableInfo.properties, 
tableInfo.constraints())
+    tables.put(ident, table)
+    namespaces.putIfAbsent(ident.namespace.toList, Map())
+    table
+  }
+}
diff --git 
a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DataSourceV2Strategy.scala
 
b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DataSourceV2Strategy.scala
index 9c745461b170..bcdbf5fe6ee2 100644
--- 
a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DataSourceV2Strategy.scala
+++ 
b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DataSourceV2Strategy.scala
@@ -527,8 +527,8 @@ class DataSourceV2Strategy(session: SparkSession) extends 
Strategy with Predicat
     case OverwritePartitionsDynamic(r: DataSourceV2Relation, query, _, _, _, 
Some(write)) =>
       OverwritePartitionsDynamicExec(planLater(query), refreshCache(r), write, 
r.name) :: Nil
 
-    case DeleteFromTableWithFilters(r: DataSourceV2Relation, filters) =>
-      DeleteFromTableExec(r.table.asDeletable, filters.toArray, 
refreshCache(r)) :: Nil
+    case DeleteFromTableWithFilters(r: DataSourceV2Relation, filters, options) 
=>
+      DeleteFromTableExec(r.table.asDeletable, filters.toArray, 
refreshCache(r), options) :: Nil
 
     case DeleteFromTable(relation, condition) =>
       relation match {
@@ -547,11 +547,11 @@ class DataSourceV2Strategy(session: SparkSession) extends 
Strategy with Predicat
 
           table match {
             case t: SupportsDeleteV2 if t.canDeleteWhere(filters) =>
-              DeleteFromTableExec(t, filters, refreshCache(r)) :: Nil
+              DeleteFromTableExec(t, filters, refreshCache(r), r.options) :: 
Nil
             case t: SupportsDeleteV2 =>
               throw 
QueryCompilationErrors.cannotDeleteTableWhereFiltersError(t, filters)
             case t: TruncatableTable if condition == TrueLiteral =>
-              TruncateTableExec(t, refreshCache(r)) :: Nil
+              TruncateTableExec(t, refreshCache(r), r.options) :: Nil
             case _ =>
               throw 
QueryCompilationErrors.tableDoesNotSupportDeletesError(table)
           }
diff --git 
a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DeleteFromTableExec.scala
 
b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DeleteFromTableExec.scala
index ea61bca26660..c13cf4e8d402 100644
--- 
a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DeleteFromTableExec.scala
+++ 
b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DeleteFromTableExec.scala
@@ -24,11 +24,13 @@ import 
org.apache.spark.sql.connector.catalog.SupportsDeleteV2
 import org.apache.spark.sql.connector.catalog.transactions.Transaction
 import org.apache.spark.sql.connector.expressions.filter.Predicate
 import org.apache.spark.sql.execution.metric.SQLMetric
+import org.apache.spark.sql.util.CaseInsensitiveStringMap
 
 case class DeleteFromTableExec(
     table: SupportsDeleteV2,
     condition: Array[Predicate],
     refreshCache: () => Unit,
+    options: CaseInsensitiveStringMap = CaseInsensitiveStringMap.empty(),
     transaction: Option[Transaction] = None)
   extends LeafV2CommandExec
   with TransactionalExec
@@ -42,7 +44,7 @@ case class DeleteFromTableExec(
 
   override protected def run(): Seq[InternalRow] = {
     try {
-      table.deleteWhere(condition)
+      table.deleteWhere(condition, options)
     } finally {
       postDriverMetrics(table.reportDriverMetrics())
     }
diff --git 
a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/OptimizeMetadataOnlyDeleteFromTable.scala
 
b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/OptimizeMetadataOnlyDeleteFromTable.scala
index c02fdec1fab6..c052edea53e0 100644
--- 
a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/OptimizeMetadataOnlyDeleteFromTable.scala
+++ 
b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/OptimizeMetadataOnlyDeleteFromTable.scala
@@ -47,7 +47,8 @@ object OptimizeMetadataOnlyDeleteFromTable extends 
Rule[LogicalPlan] with Predic
           if (filtersOpt.exists(table.canDeleteWhere)) {
             logDebug(s"Switching to delete with filters: " +
               s"${filtersOpt.get.mkString("[", ", ", "]")}")
-            DeleteFromTableWithFilters(relation, 
filtersOpt.get.toImmutableArraySeq)
+            DeleteFromTableWithFilters(
+              relation, filtersOpt.get.toImmutableArraySeq, relation.options)
           } else {
             tryDeleteWithPartitionPredicates(table, relation, 
normalizedPredicates)
               .getOrElse {
@@ -90,7 +91,7 @@ object OptimizeMetadataOnlyDeleteFromTable extends 
Rule[LogicalPlan] with Predic
     } yield {
       logDebug(s"Switching to delete with PartitionPredicate filters: " +
         s"${combined.mkString("[", ", ", "]")}")
-      DeleteFromTableWithFilters(relation, combined.toImmutableArraySeq)
+      DeleteFromTableWithFilters(relation, combined.toImmutableArraySeq, 
relation.options)
     }
   }
 
diff --git 
a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/TruncateTableExec.scala
 
b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/TruncateTableExec.scala
index 0bcfc4f36418..6c6c5f5374a9 100644
--- 
a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/TruncateTableExec.scala
+++ 
b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/TruncateTableExec.scala
@@ -21,13 +21,15 @@ import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.expressions.Attribute
 import org.apache.spark.sql.connector.catalog.TruncatableTable
 import org.apache.spark.sql.execution.metric.SQLMetric
+import org.apache.spark.sql.util.CaseInsensitiveStringMap
 
 /**
  * Physical plan node for table truncation.
  */
 case class TruncateTableExec(
     table: TruncatableTable,
-    refreshCache: () => Unit)
+    refreshCache: () => Unit,
+    options: CaseInsensitiveStringMap = CaseInsensitiveStringMap.empty())
   extends LeafV2CommandExec
   with SupportsCustomDriverMetrics {
 
@@ -38,7 +40,7 @@ case class TruncateTableExec(
 
   override protected def run(): Seq[InternalRow] = {
     try {
-      if (table.truncateTable()) refreshCache()
+      if (table.truncateTable(options)) refreshCache()
       Seq.empty
     } finally {
       postDriverMetrics(table.reportDriverMetrics())
diff --git 
a/sql/core/src/test/scala/org/apache/spark/sql/connector/DeleteFromTableSuiteBase.scala
 
b/sql/core/src/test/scala/org/apache/spark/sql/connector/DeleteFromTableSuiteBase.scala
index b894d5d75b3c..b75c0fc1c947 100644
--- 
a/sql/core/src/test/scala/org/apache/spark/sql/connector/DeleteFromTableSuiteBase.scala
+++ 
b/sql/core/src/test/scala/org/apache/spark/sql/connector/DeleteFromTableSuiteBase.scala
@@ -19,12 +19,12 @@ package org.apache.spark.sql.connector
 
 import org.apache.spark.internal.config
 import org.apache.spark.sql.{AnalysisException, Row}
+import org.apache.spark.sql.QueryTest.withQueryExecutionsCaptured
 import org.apache.spark.sql.catalyst.expressions.CheckInvariant
 import org.apache.spark.sql.catalyst.plans.logical.Filter
-import org.apache.spark.sql.connector.catalog.{Aborted, Committed}
-import org.apache.spark.sql.connector.catalog.InMemoryTable
+import org.apache.spark.sql.connector.catalog.{Aborted, Committed, 
InMemoryRowLevelOperationTable, InMemoryTable, InMemoryTruncatableOnlyTable, 
InMemoryTruncatableOnlyTableCatalog, TableCatalog}
 import org.apache.spark.sql.connector.write.DeleteSummary
-import org.apache.spark.sql.execution.datasources.v2.{DeleteFromTableExec, 
ReplaceDataExec, WriteDeltaExec}
+import org.apache.spark.sql.execution.datasources.v2.{DeleteFromTableExec, 
ReplaceDataExec, TruncateTableExec, WriteDeltaExec}
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.sources
 
@@ -1037,6 +1037,142 @@ abstract class DeleteFromTableSuiteBase extends 
RowLevelOperationSuiteBase {
     }
   }
 
+  test("SPARK-58008: delete with dynamic options") {
+    createAndInitTable("pk INT NOT NULL, salary INT, dep STRING",
+      """{ "pk": 1, "salary": 100, "dep": "hr" }
+        |{ "pk": 2, "salary": 200, "dep": "software" }
+        |""".stripMargin)
+
+    // Use salary < 200 (LessThan) rather than pk = 1 (EqualTo) so the 
optimizer does not
+    // replace the row-level plan with a filter-only deleteWhere call via
+    // OptimizeMetadataOnlyDeleteFromTable (canDeleteWhere returns false for 
LessThan).
+    checkRowLevelOperationOptions(
+      sql(s"DELETE FROM $tableNameAsString WITH (`write.split-size` = 10) 
WHERE salary < 200"),
+      "write.split-size" -> "10")
+
+    checkAnswer(
+      sql(s"SELECT * FROM $tableNameAsString"),
+      Row(2, 200, "software") :: Nil)
+  }
+
+  test("SPARK-58008: delete with dynamic options and a subquery on the same 
table") {
+    createAndInitTable("pk INT NOT NULL, salary INT, dep STRING",
+      """{ "pk": 1, "salary": 100, "dep": "hr" }
+        |{ "pk": 2, "salary": 200, "dep": "software" }
+        |{ "pk": 3, "salary": 300, "dep": "hr" }
+        |""".stripMargin)
+
+    checkRowLevelOperationOptions(
+      sql(s"DELETE FROM $tableNameAsString WITH (`write.split-size` = 10) " +
+        s"WHERE pk IN (SELECT pk FROM $tableNameAsString WHERE dep = 'hr')"),
+      "write.split-size" -> "10")
+
+    checkAnswer(
+      sql(s"SELECT * FROM $tableNameAsString"),
+      Row(2, 200, "software") :: Nil)
+  }
+
+  test("SPARK-58008: delete with dynamic options and a CTE") {
+    createAndInitTable("pk INT NOT NULL, salary INT, dep STRING",
+      """{ "pk": 1, "salary": 100, "dep": "hr" }
+        |{ "pk": 2, "salary": 200, "dep": "software" }
+        |{ "pk": 3, "salary": 300, "dep": "hr" }
+        |""".stripMargin)
+
+    checkRowLevelOperationOptions(
+      sql(s"""WITH cte AS (
+             |  SELECT pk FROM $tableNameAsString WHERE dep = 'hr'
+             |)
+             |DELETE FROM $tableNameAsString WITH (`write.split-size` = 10)
+             |WHERE pk IN (SELECT pk FROM cte)""".stripMargin),
+      "write.split-size" -> "10")
+
+    checkAnswer(
+      sql(s"SELECT * FROM $tableNameAsString"),
+      Row(2, 200, "software") :: Nil)
+  }
+
+  test("SPARK-58008: delete with dynamic options on the deleteWhere path") {
+    createAndInitTable("pk INT NOT NULL, salary INT, dep STRING",
+      """{ "pk": 1, "salary": 100, "dep": "hr" }
+        |{ "pk": 2, "salary": 200, "dep": "software" }
+        |""".stripMargin)
+
+    // dep = 'hr' is an EqualTo on the partition column. 
OptimizeMetadataOnlyDeleteFromTable
+    // converts the row-level plan to a deleteWhere call. Verify options flow 
into the exec.
+    val Seq(qe) = withQueryExecutionsCaptured(spark) {
+      sql(s"DELETE FROM $tableNameAsString WITH (`write.split-size` = 10) 
WHERE dep = 'hr'")
+    }
+    val exec = qe.executedPlan.collectFirst {
+      case e: DeleteFromTableExec => e
+    }.getOrElse(fail("expected DeleteFromTableExec for the metadata-only 
deleteWhere path"))
+    assert(exec.options.get("write.split-size") === "10",
+      "options must reach DeleteFromTableExec on the deleteWhere path")
+    
assert(exec.table.asInstanceOf[InMemoryRowLevelOperationTable].lastDeleteOptions
+      .get("write.split-size") === "10",
+      "options must be forwarded to SupportsDeleteV2.deleteWhere(predicates, 
options)")
+
+    checkAnswer(
+      sql(s"SELECT * FROM $tableNameAsString"),
+      Row(2, 200, "software") :: Nil)
+  }
+
+  test("SPARK-58008: delete all rows with dynamic options") {
+    createAndInitTable("pk INT NOT NULL, salary INT, dep STRING",
+      """{ "pk": 1, "salary": 100, "dep": "hr" }
+        |{ "pk": 2, "salary": 200, "dep": "software" }
+        |""".stripMargin)
+
+    // DELETE with no WHERE: RewriteDeleteFromTable leaves the plan 
unrewritten (TruncatableTable
+    // short-circuit). DataSourceV2Strategy then matches 
SupportsDeleteV2.canDeleteWhere(AlwaysTrue)
+    // and produces DeleteFromTableExec, carrying the WITH-clause options 
through r.options.
+    val Seq(qe) = withQueryExecutionsCaptured(spark) {
+      sql(s"DELETE FROM $tableNameAsString WITH (`write.split-size` = 10)")
+    }
+    val exec = qe.executedPlan.collectFirst {
+      case e: DeleteFromTableExec => e
+    }.getOrElse(fail("expected DeleteFromTableExec for the delete-all path"))
+    assert(exec.options.get("write.split-size") === "10",
+      "options must reach DeleteFromTableExec on the delete-all path")
+    
assert(exec.table.asInstanceOf[InMemoryRowLevelOperationTable].lastDeleteOptions
+      .get("write.split-size") === "10",
+      "options must be forwarded to SupportsDeleteV2.deleteWhere(predicates, 
options)")
+
+    checkAnswer(sql(s"SELECT * FROM $tableNameAsString"), Nil)
+  }
+
+  test("SPARK-58008: delete all rows with dynamic options on the truncate 
path") {
+    // Use a TruncatableTable-only fixture (no SupportsDeleteV2) so that 
DELETE with no WHERE
+    // is planned as TruncateTableExec (not DeleteFromTableExec).
+    withSQLConf(
+        "spark.sql.catalog.trunccat" ->
+          classOf[InMemoryTruncatableOnlyTableCatalog].getName) {
+      withTable("trunccat.ns.tbl") {
+        sql("CREATE TABLE trunccat.ns.tbl (pk INT NOT NULL, dep STRING) USING 
foo")
+        sql("INSERT INTO trunccat.ns.tbl VALUES (1, 'hr'), (2, 'software')")
+
+        val Seq(qe) = withQueryExecutionsCaptured(spark) {
+          sql("DELETE FROM trunccat.ns.tbl WITH (`write.split-size` = 10)")
+        }
+        val exec = qe.executedPlan.collectFirst {
+          case e: TruncateTableExec => e
+        }.getOrElse(fail("expected TruncateTableExec for the truncate path"))
+        assert(exec.options.get("write.split-size") === "10",
+          "options must reach TruncateTableExec")
+
+        val truncTable = spark.sessionState.catalogManager
+          .catalog("trunccat").asInstanceOf[TableCatalog]
+          .loadTable(org.apache.spark.sql.connector.catalog.Identifier.of(
+            Array("ns"), "tbl"))
+          .asInstanceOf[InMemoryTruncatableOnlyTable]
+        assert(truncTable.lastTruncateOptions.get("write.split-size") === "10",
+          "options must be forwarded to 
TruncatableTable.truncateTable(options)")
+
+        checkAnswer(spark.table("trunccat.ns.tbl"), Nil)
+      }
+    }
+  }
+
   private def executeDeleteWithRewrite(query: String): Unit = {
     val executedPlan = executeAndKeepPlan {
       sql(query)


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to