This is an automated email from the ASF dual-hosted git repository.
peter-toth pushed a commit to branch branch-4.x
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-4.x by this push:
new d0d4b93a81a8 [SPARK-58008][SQL] Support dynamic table options for
DELETE
d0d4b93a81a8 is described below
commit d0d4b93a81a8d4bb419124a57f476f262f23af65
Author: BRIJ RAJ KISHORE <[email protected]>
AuthorDate: Tue Jul 28 11:06:16 2026 +0200
[SPARK-58008][SQL] Support dynamic table options for DELETE
### What changes were proposed in this pull request?
Extends the per-statement `WITH (key = value)` options clause to `DELETE`
statements, consistent with the support already present for `SELECT`
(SPARK-36680), `INSERT` (SPARK-49098), and `UPDATE` (SPARK-57681).
Example syntax:
```sql
DELETE FROM catalog.db.table WITH (`write.split-size` = 10) WHERE id < 100
DELETE FROM catalog.db.table AS t WITH (`k` = 'v') WHERE t.id < 100
```
The options are forwarded through the `DataSourceV2Relation`,
`RowLevelOperationInfo`, and `LogicalWriteInfo` so that DataSource V2
connectors receive them at every layer of the write path.
Options are also wired through the metadata-only DELETE paths:
- `SupportsDeleteV2.deleteWhere(Predicate[], CaseInsensitiveStringMap)` —
new back-compatible overload
- `SupportsDeleteV2.truncateTable(CaseInsensitiveStringMap)` — new
back-compatible overload
- `TruncatableTable.truncateTable(CaseInsensitiveStringMap)` — new
back-compatible overload
- `DeleteFromTableWithFilters`, `DeleteFromTableExec`, `TruncateTableExec`
— each carries `options`
- `DataSourceV2Strategy` — threads `r.options` through all three
non-row-level DELETE planning paths
### Why are the changes needed?
`DELETE` was the only DML statement missing this capability. Adding it
completes the feature set across all row-level write operations and allows
connectors to accept per-statement tuning options (e.g. split size, isolation
level) on deletes without requiring separate configuration.
### Does this PR introduce _any_ user-facing change?
Yes — new SQL syntax. `DELETE FROM tbl WITH (k = v) WHERE ...` is now valid.
### How was this patch tested?
- `DDLParserSuite`: two new parser tests covering `DELETE ... WITH (...)`
without and with an alias.
- `DeleteFromTableSuiteBase`: five new end-to-end tests verifying that
options reach the `DataSourceV2Relation`, `RowLevelOperationInfo`, and
`LogicalWriteInfo` layers across the row-level, subquery, CTE, metadata-only
deleteWhere, and delete-all paths.
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: Claude Sonnet 4.6
Closes #57161 from brijrajk/SPARK-58008-delete-with-options.
Authored-by: BRIJ RAJ KISHORE <[email protected]>
Signed-off-by: Peter Toth <[email protected]>
(cherry picked from commit 5221eedcd64502446d031212e984ea2cd10467bc)
Signed-off-by: Peter Toth <[email protected]>
---
.../spark/sql/catalyst/parser/SqlBaseParser.g4 | 2 +-
.../sql/connector/catalog/SupportsDeleteV2.java | 19 +++
.../sql/connector/catalog/TruncatableTable.java | 15 +++
.../catalyst/analysis/RewriteDeleteFromTable.scala | 3 +-
.../spark/sql/catalyst/parser/AstBuilder.scala | 3 +-
.../sql/catalyst/plans/logical/v2Commands.scala | 4 +-
.../spark/sql/catalyst/parser/DDLParserSuite.scala | 27 ++++
.../catalog/InMemoryRowLevelOperationTable.scala | 51 ++++++++
.../InMemoryRowLevelOperationTableCatalog.scala | 25 ++++
.../datasources/v2/DataSourceV2Strategy.scala | 8 +-
.../datasources/v2/DeleteFromTableExec.scala | 4 +-
.../v2/OptimizeMetadataOnlyDeleteFromTable.scala | 5 +-
.../datasources/v2/TruncateTableExec.scala | 6 +-
.../sql/connector/DeleteFromTableSuiteBase.scala | 142 ++++++++++++++++++++-
14 files changed, 297 insertions(+), 17 deletions(-)
diff --git
a/sql/api/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBaseParser.g4
b/sql/api/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBaseParser.g4
index cdeb34c6a0ca..c591dae38da5 100644
---
a/sql/api/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBaseParser.g4
+++
b/sql/api/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBaseParser.g4
@@ -747,7 +747,7 @@ resource
dmlStatementNoWith
: insertInto (query | LEFT_PAREN query RIGHT_PAREN queryAlias=tableAlias)
#singleInsertQuery
| fromClause multiInsertQueryBody+
#multiInsertQuery
- | DELETE FROM identifierReference tableAlias whereClause?
#deleteFromTable
+ | DELETE FROM identifierReference tableAlias optionsClause? whereClause?
#deleteFromTable
| UPDATE identifierReference tableAlias optionsClause? setClause
whereClause? #updateTable
| MERGE (WITH SCHEMA EVOLUTION)? INTO target=identifierReference
targetAlias=tableAlias
targetOptions=optionsClause?
diff --git
a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/SupportsDeleteV2.java
b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/SupportsDeleteV2.java
index dc5911ad9899..53795dc159e4 100644
---
a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/SupportsDeleteV2.java
+++
b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/SupportsDeleteV2.java
@@ -20,6 +20,7 @@ package org.apache.spark.sql.connector.catalog;
import org.apache.spark.annotation.Evolving;
import org.apache.spark.sql.connector.expressions.filter.AlwaysTrue;
import org.apache.spark.sql.connector.expressions.filter.Predicate;
+import org.apache.spark.sql.util.CaseInsensitiveStringMap;
/**
* A mix-in interface for {@link Table} delete support. Data sources can
implement this
@@ -72,6 +73,24 @@ public interface SupportsDeleteV2 extends TruncatableTable {
*/
void deleteWhere(Predicate[] predicates);
+ /**
+ * Delete data from a data source table that matches filter expressions,
forwarding the
+ * per-statement options from a {@code DELETE ... WITH (key=value)} clause.
+ * <p>
+ * The default implementation ignores {@code options} and delegates to
+ * {@link #deleteWhere(Predicate[])}, which preserves backward compatibility
for connectors
+ * that do not need per-statement options.
+ *
+ * @param predicates predicate expressions, used to select rows to delete
when all expressions
+ * match
+ * @param options per-statement options from the {@code WITH} clause; empty
if none were given
+ * @throws IllegalArgumentException If the delete is rejected due to
required effort
+ * @since 4.3.0
+ */
+ default void deleteWhere(Predicate[] predicates, CaseInsensitiveStringMap
options) {
+ deleteWhere(predicates);
+ }
+
@Override
default boolean truncateTable() {
Predicate[] predicates = new Predicate[] { new AlwaysTrue() };
diff --git
a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/TruncatableTable.java
b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/TruncatableTable.java
index 058e105758c7..eccd9809c5e6 100644
---
a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/TruncatableTable.java
+++
b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/TruncatableTable.java
@@ -20,6 +20,7 @@ package org.apache.spark.sql.connector.catalog;
import org.apache.spark.annotation.Evolving;
import org.apache.spark.sql.connector.metric.CustomMetric;
import org.apache.spark.sql.connector.metric.CustomTaskMetric;
+import org.apache.spark.sql.util.CaseInsensitiveStringMap;
/**
* Represents a table which can be atomically truncated.
@@ -37,6 +38,20 @@ public interface TruncatableTable extends Table {
*/
boolean truncateTable();
+ /**
+ * Truncate a table with per-statement options from a {@code DELETE ... WITH
(key=value)} clause.
+ * <p>
+ * The default implementation ignores {@code options} and delegates to
{@link #truncateTable()},
+ * which preserves backward compatibility for connectors that do not need
per-statement options.
+ *
+ * @param options per-statement options from the {@code WITH} clause; empty
if none were given
+ * @return true if the table was truncated successfully, false otherwise
+ * @since 4.3.0
+ */
+ default boolean truncateTable(CaseInsensitiveStringMap options) {
+ return truncateTable();
+ }
+
/**
* Returns an array of supported custom metrics with name and description.
* By default it returns empty array.
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/RewriteDeleteFromTable.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/RewriteDeleteFromTable.scala
index f8881e207710..c8795b8ad9b1 100644
---
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/RewriteDeleteFromTable.scala
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/RewriteDeleteFromTable.scala
@@ -25,7 +25,6 @@ import
org.apache.spark.sql.connector.catalog.{SupportsDeleteV2, SupportsRowLeve
import org.apache.spark.sql.connector.write.{RowLevelOperationTable,
SupportsDelta}
import org.apache.spark.sql.connector.write.RowLevelOperation.Command.DELETE
import org.apache.spark.sql.execution.datasources.v2.{DataSourceV2Relation,
ExtractV2Table}
-import org.apache.spark.sql.util.CaseInsensitiveStringMap
/**
* A rule that rewrites DELETE operations using plans that operate on
individual or groups of rows.
@@ -45,7 +44,7 @@ object RewriteDeleteFromTable extends RewriteRowLevelCommand {
d
case r @ ExtractV2Table(t: SupportsRowLevelOperations) =>
- val table = buildOperationTable(t, DELETE,
CaseInsensitiveStringMap.empty())
+ val table = buildOperationTable(t, DELETE, r.options)
table.operation match {
case _: SupportsDelta =>
buildWriteDeltaPlan(r, table, cond)
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala
index 8214c67cb8b4..2faf07b5379c 100644
---
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala
@@ -1244,7 +1244,8 @@ class AstBuilder extends DataTypeAstBuilder
override def visitDeleteFromTable(
ctx: DeleteFromTableContext): LogicalPlan = withOrigin(ctx) {
val table = createUnresolvedRelation(
- ctx.identifierReference, writePrivileges =
Set(TableWritePrivilege.DELETE))
+ ctx.identifierReference, Option(ctx.optionsClause()),
+ writePrivileges = Set(TableWritePrivilege.DELETE))
val tableAlias = getTableAliasWithoutColumnAlias(ctx.tableAlias(),
"DELETE")
val aliasedTable = tableAlias.map(SubqueryAlias(_, table)).getOrElse(table)
val predicate = if (ctx.whereClause() != null) {
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/v2Commands.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/v2Commands.scala
index e82a65c8f99e..85f3c726c7ec 100644
---
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/v2Commands.scala
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/v2Commands.scala
@@ -42,6 +42,7 @@ import org.apache.spark.sql.errors.QueryExecutionErrors
import org.apache.spark.sql.execution.datasources.v2.{DataSourceV2Relation,
ExtractV2Table}
import org.apache.spark.sql.internal.SQLConf
import org.apache.spark.sql.types.{BooleanType, DataType, IntegerType,
MapType, MetadataBuilder, StringType, StructType}
+import org.apache.spark.sql.util.CaseInsensitiveStringMap
import org.apache.spark.util.ArrayImplicits._
import org.apache.spark.util.Utils
import org.apache.spark.util.collection.BitSet
@@ -1107,7 +1108,8 @@ case class DeleteFromTable(
*/
case class DeleteFromTableWithFilters(
table: LogicalPlan,
- condition: Seq[Predicate]) extends LeafCommand
+ condition: Seq[Predicate],
+ options: CaseInsensitiveStringMap = CaseInsensitiveStringMap.empty())
extends LeafCommand
/**
* The logical plan of the UPDATE TABLE command.
diff --git
a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/DDLParserSuite.scala
b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/DDLParserSuite.scala
index 5d8ff7d2155b..d85cb0b93133 100644
---
a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/DDLParserSuite.scala
+++
b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/DDLParserSuite.scala
@@ -2327,6 +2327,33 @@ class DDLParserSuite extends AnalysisTest {
stop = 56))
}
+ test("SPARK-58008: delete from table: with options") {
+ parseCompare(
+ """
+ |DELETE FROM testcat.ns1.ns2.tbl WITH (`write.split-size` = 10)
+ |WHERE a = 1
+ """.stripMargin,
+ DeleteFromTable(
+ UnresolvedRelation(Seq("testcat", "ns1", "ns2", "tbl"),
+ new CaseInsensitiveStringMap(
+ java.util.Map.of("write.split-size", "10"))),
+ EqualTo(UnresolvedAttribute("a"), Literal(1))))
+ }
+
+ test("SPARK-58008: delete from table: with options and alias") {
+ parseCompare(
+ """
+ |DELETE FROM testcat.ns1.ns2.tbl AS t WITH (`k` = 'v')
+ |WHERE t.a = 2
+ """.stripMargin,
+ DeleteFromTable(
+ SubqueryAlias("t",
+ UnresolvedRelation(Seq("testcat", "ns1", "ns2", "tbl"),
+ new CaseInsensitiveStringMap(
+ java.util.Map.of("k", "v")))),
+ EqualTo(UnresolvedAttribute("t.a"), Literal(2))))
+ }
+
test("update table: basic") {
parseCompare(
"""
diff --git
a/sql/catalyst/src/test/scala/org/apache/spark/sql/connector/catalog/InMemoryRowLevelOperationTable.scala
b/sql/catalyst/src/test/scala/org/apache/spark/sql/connector/catalog/InMemoryRowLevelOperationTable.scala
index f62f1b0dce25..fd02f926e814 100644
---
a/sql/catalyst/src/test/scala/org/apache/spark/sql/connector/catalog/InMemoryRowLevelOperationTable.scala
+++
b/sql/catalyst/src/test/scala/org/apache/spark/sql/connector/catalog/InMemoryRowLevelOperationTable.scala
@@ -24,6 +24,7 @@ import
org.apache.spark.sql.catalyst.expressions.GenericInternalRow
import org.apache.spark.sql.connector.catalog.constraints.Constraint
import org.apache.spark.sql.connector.distributions.{Distribution,
Distributions}
import org.apache.spark.sql.connector.expressions.{FieldReference,
LogicalExpressions, NamedReference, SortDirection, SortOrder, Transform}
+import org.apache.spark.sql.connector.expressions.filter.Predicate
import org.apache.spark.sql.connector.read.{Scan, ScanBuilder}
import org.apache.spark.sql.connector.write.{BatchWrite, DeltaBatchWrite,
DeltaWrite, DeltaWriteBuilder, DeltaWriter, DeltaWriterFactory,
LogicalWriteInfo, PhysicalWriteInfo, RequiresDistributionAndOrdering,
RowLevelOperation, RowLevelOperationBuilder, RowLevelOperationInfo,
SupportsDelta, Write, WriteBuilder, WriterCommitMessage}
import org.apache.spark.sql.connector.write.RowLevelOperation.Command
@@ -86,6 +87,14 @@ class InMemoryRowLevelOperationTable private (
// used in row-level operation tests to verify passed records
// (operation, id, metadata, row)
var lastWriteLog: Seq[InternalRow] = Seq.empty
+ // used in delete-with-options tests to verify options reach the connector
on the deleteWhere path
+ var lastDeleteOptions: CaseInsensitiveStringMap =
CaseInsensitiveStringMap.empty()
+
+ override def deleteWhere(
+ predicates: Array[Predicate], options: CaseInsensitiveStringMap): Unit =
{
+ lastDeleteOptions = options
+ deleteWhere(predicates)
+ }
override def copy(): Table = {
val copied = InMemoryRowLevelOperationTable.withColumns(
@@ -323,3 +332,45 @@ object InMemoryRowLevelOperationTable {
name, columns, partitioning, properties, constraints, tableId)
}
}
+
+/**
+ * An in-memory table that implements only [[TruncatableTable]] (NOT
[[SupportsDeleteV2]]).
+ * Used to exercise the [[TruncateTableExec]] planning path for DELETE
statements with no WHERE.
+ */
+class InMemoryTruncatableOnlyTable(
+ name: String,
+ columns: Array[Column],
+ partitioning: Array[Transform],
+ properties: util.Map[String, String],
+ constraints: Array[Constraint])
+ extends InMemoryBaseTable(name, columns, partitioning, properties,
constraints)
+ with TruncatableTable {
+
+ var lastTruncateOptions: CaseInsensitiveStringMap =
CaseInsensitiveStringMap.empty()
+
+ override def newWriteBuilder(info: LogicalWriteInfo): WriteBuilder = {
+ InMemoryBaseTable.maybeSimulateFailedTableWrite(new
CaseInsensitiveStringMap(properties))
+ InMemoryBaseTable.maybeSimulateFailedTableWrite(info.options)
+ new InMemoryWriterBuilder(info) {
+ override def truncate(): WriteBuilder = {
+ writer = new TruncateAndAppend(this.info)
+ streamingWriter = new StreamingTruncateAndAppend(this.info)
+ this
+ }
+ }
+ }
+
+ override def truncateTable(): Boolean = {
+ lastTruncateOptions = CaseInsensitiveStringMap.empty()
+ dataMap.synchronized { dataMap.clear() }
+ increaseVersion()
+ true
+ }
+
+ override def truncateTable(options: CaseInsensitiveStringMap): Boolean = {
+ lastTruncateOptions = options
+ dataMap.synchronized { dataMap.clear() }
+ increaseVersion()
+ true
+ }
+}
diff --git
a/sql/catalyst/src/test/scala/org/apache/spark/sql/connector/catalog/InMemoryRowLevelOperationTableCatalog.scala
b/sql/catalyst/src/test/scala/org/apache/spark/sql/connector/catalog/InMemoryRowLevelOperationTableCatalog.scala
index a29f1ed29ee8..95e6659956ca 100644
---
a/sql/catalyst/src/test/scala/org/apache/spark/sql/connector/catalog/InMemoryRowLevelOperationTableCatalog.scala
+++
b/sql/catalyst/src/test/scala/org/apache/spark/sql/connector/catalog/InMemoryRowLevelOperationTableCatalog.scala
@@ -151,3 +151,28 @@ class PartialSchemaEvolutionCatalog extends
InMemoryRowLevelOperationTableCatalo
currentSchema: StructType,
changes: Seq[TableChange]): StructType = currentSchema
}
+
+/**
+ * A catalog that creates [[InMemoryTruncatableOnlyTable]] instances - tables
that implement
+ * [[TruncatableTable]] but NOT [[SupportsDeleteV2]]. Used to test the
[[TruncateTableExec]]
+ * planning path for DELETE statements with no WHERE clause.
+ */
+class InMemoryTruncatableOnlyTableCatalog extends InMemoryTableCatalog {
+ import CatalogV2Implicits._
+
+ override def loadTable(ident: Identifier): Table = liveTable(ident)
+
+ override def createTable(ident: Identifier, tableInfo: TableInfo): Table = {
+ if (tables.containsKey(ident)) {
+ throw new TableAlreadyExistsException(ident.asMultipartIdentifier)
+ }
+ InMemoryTableCatalog.maybeSimulateFailedTableCreation(tableInfo.properties)
+ val tableName = s"$name.${ident.quoted}"
+ val columns = tableInfo.columns
+ val table = new InMemoryTruncatableOnlyTable(
+ tableName, columns, tableInfo.partitions, tableInfo.properties,
tableInfo.constraints())
+ tables.put(ident, table)
+ namespaces.putIfAbsent(ident.namespace.toList, Map())
+ table
+ }
+}
diff --git
a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DataSourceV2Strategy.scala
b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DataSourceV2Strategy.scala
index 9c745461b170..bcdbf5fe6ee2 100644
---
a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DataSourceV2Strategy.scala
+++
b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DataSourceV2Strategy.scala
@@ -527,8 +527,8 @@ class DataSourceV2Strategy(session: SparkSession) extends
Strategy with Predicat
case OverwritePartitionsDynamic(r: DataSourceV2Relation, query, _, _, _,
Some(write)) =>
OverwritePartitionsDynamicExec(planLater(query), refreshCache(r), write,
r.name) :: Nil
- case DeleteFromTableWithFilters(r: DataSourceV2Relation, filters) =>
- DeleteFromTableExec(r.table.asDeletable, filters.toArray,
refreshCache(r)) :: Nil
+ case DeleteFromTableWithFilters(r: DataSourceV2Relation, filters, options)
=>
+ DeleteFromTableExec(r.table.asDeletable, filters.toArray,
refreshCache(r), options) :: Nil
case DeleteFromTable(relation, condition) =>
relation match {
@@ -547,11 +547,11 @@ class DataSourceV2Strategy(session: SparkSession) extends
Strategy with Predicat
table match {
case t: SupportsDeleteV2 if t.canDeleteWhere(filters) =>
- DeleteFromTableExec(t, filters, refreshCache(r)) :: Nil
+ DeleteFromTableExec(t, filters, refreshCache(r), r.options) ::
Nil
case t: SupportsDeleteV2 =>
throw
QueryCompilationErrors.cannotDeleteTableWhereFiltersError(t, filters)
case t: TruncatableTable if condition == TrueLiteral =>
- TruncateTableExec(t, refreshCache(r)) :: Nil
+ TruncateTableExec(t, refreshCache(r), r.options) :: Nil
case _ =>
throw
QueryCompilationErrors.tableDoesNotSupportDeletesError(table)
}
diff --git
a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DeleteFromTableExec.scala
b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DeleteFromTableExec.scala
index ea61bca26660..c13cf4e8d402 100644
---
a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DeleteFromTableExec.scala
+++
b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DeleteFromTableExec.scala
@@ -24,11 +24,13 @@ import
org.apache.spark.sql.connector.catalog.SupportsDeleteV2
import org.apache.spark.sql.connector.catalog.transactions.Transaction
import org.apache.spark.sql.connector.expressions.filter.Predicate
import org.apache.spark.sql.execution.metric.SQLMetric
+import org.apache.spark.sql.util.CaseInsensitiveStringMap
case class DeleteFromTableExec(
table: SupportsDeleteV2,
condition: Array[Predicate],
refreshCache: () => Unit,
+ options: CaseInsensitiveStringMap = CaseInsensitiveStringMap.empty(),
transaction: Option[Transaction] = None)
extends LeafV2CommandExec
with TransactionalExec
@@ -42,7 +44,7 @@ case class DeleteFromTableExec(
override protected def run(): Seq[InternalRow] = {
try {
- table.deleteWhere(condition)
+ table.deleteWhere(condition, options)
} finally {
postDriverMetrics(table.reportDriverMetrics())
}
diff --git
a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/OptimizeMetadataOnlyDeleteFromTable.scala
b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/OptimizeMetadataOnlyDeleteFromTable.scala
index c02fdec1fab6..c052edea53e0 100644
---
a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/OptimizeMetadataOnlyDeleteFromTable.scala
+++
b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/OptimizeMetadataOnlyDeleteFromTable.scala
@@ -47,7 +47,8 @@ object OptimizeMetadataOnlyDeleteFromTable extends
Rule[LogicalPlan] with Predic
if (filtersOpt.exists(table.canDeleteWhere)) {
logDebug(s"Switching to delete with filters: " +
s"${filtersOpt.get.mkString("[", ", ", "]")}")
- DeleteFromTableWithFilters(relation,
filtersOpt.get.toImmutableArraySeq)
+ DeleteFromTableWithFilters(
+ relation, filtersOpt.get.toImmutableArraySeq, relation.options)
} else {
tryDeleteWithPartitionPredicates(table, relation,
normalizedPredicates)
.getOrElse {
@@ -90,7 +91,7 @@ object OptimizeMetadataOnlyDeleteFromTable extends
Rule[LogicalPlan] with Predic
} yield {
logDebug(s"Switching to delete with PartitionPredicate filters: " +
s"${combined.mkString("[", ", ", "]")}")
- DeleteFromTableWithFilters(relation, combined.toImmutableArraySeq)
+ DeleteFromTableWithFilters(relation, combined.toImmutableArraySeq,
relation.options)
}
}
diff --git
a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/TruncateTableExec.scala
b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/TruncateTableExec.scala
index 0bcfc4f36418..6c6c5f5374a9 100644
---
a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/TruncateTableExec.scala
+++
b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/TruncateTableExec.scala
@@ -21,13 +21,15 @@ import org.apache.spark.sql.catalyst.InternalRow
import org.apache.spark.sql.catalyst.expressions.Attribute
import org.apache.spark.sql.connector.catalog.TruncatableTable
import org.apache.spark.sql.execution.metric.SQLMetric
+import org.apache.spark.sql.util.CaseInsensitiveStringMap
/**
* Physical plan node for table truncation.
*/
case class TruncateTableExec(
table: TruncatableTable,
- refreshCache: () => Unit)
+ refreshCache: () => Unit,
+ options: CaseInsensitiveStringMap = CaseInsensitiveStringMap.empty())
extends LeafV2CommandExec
with SupportsCustomDriverMetrics {
@@ -38,7 +40,7 @@ case class TruncateTableExec(
override protected def run(): Seq[InternalRow] = {
try {
- if (table.truncateTable()) refreshCache()
+ if (table.truncateTable(options)) refreshCache()
Seq.empty
} finally {
postDriverMetrics(table.reportDriverMetrics())
diff --git
a/sql/core/src/test/scala/org/apache/spark/sql/connector/DeleteFromTableSuiteBase.scala
b/sql/core/src/test/scala/org/apache/spark/sql/connector/DeleteFromTableSuiteBase.scala
index b894d5d75b3c..b75c0fc1c947 100644
---
a/sql/core/src/test/scala/org/apache/spark/sql/connector/DeleteFromTableSuiteBase.scala
+++
b/sql/core/src/test/scala/org/apache/spark/sql/connector/DeleteFromTableSuiteBase.scala
@@ -19,12 +19,12 @@ package org.apache.spark.sql.connector
import org.apache.spark.internal.config
import org.apache.spark.sql.{AnalysisException, Row}
+import org.apache.spark.sql.QueryTest.withQueryExecutionsCaptured
import org.apache.spark.sql.catalyst.expressions.CheckInvariant
import org.apache.spark.sql.catalyst.plans.logical.Filter
-import org.apache.spark.sql.connector.catalog.{Aborted, Committed}
-import org.apache.spark.sql.connector.catalog.InMemoryTable
+import org.apache.spark.sql.connector.catalog.{Aborted, Committed,
InMemoryRowLevelOperationTable, InMemoryTable, InMemoryTruncatableOnlyTable,
InMemoryTruncatableOnlyTableCatalog, TableCatalog}
import org.apache.spark.sql.connector.write.DeleteSummary
-import org.apache.spark.sql.execution.datasources.v2.{DeleteFromTableExec,
ReplaceDataExec, WriteDeltaExec}
+import org.apache.spark.sql.execution.datasources.v2.{DeleteFromTableExec,
ReplaceDataExec, TruncateTableExec, WriteDeltaExec}
import org.apache.spark.sql.internal.SQLConf
import org.apache.spark.sql.sources
@@ -1037,6 +1037,142 @@ abstract class DeleteFromTableSuiteBase extends
RowLevelOperationSuiteBase {
}
}
+ test("SPARK-58008: delete with dynamic options") {
+ createAndInitTable("pk INT NOT NULL, salary INT, dep STRING",
+ """{ "pk": 1, "salary": 100, "dep": "hr" }
+ |{ "pk": 2, "salary": 200, "dep": "software" }
+ |""".stripMargin)
+
+ // Use salary < 200 (LessThan) rather than pk = 1 (EqualTo) so the
optimizer does not
+ // replace the row-level plan with a filter-only deleteWhere call via
+ // OptimizeMetadataOnlyDeleteFromTable (canDeleteWhere returns false for
LessThan).
+ checkRowLevelOperationOptions(
+ sql(s"DELETE FROM $tableNameAsString WITH (`write.split-size` = 10)
WHERE salary < 200"),
+ "write.split-size" -> "10")
+
+ checkAnswer(
+ sql(s"SELECT * FROM $tableNameAsString"),
+ Row(2, 200, "software") :: Nil)
+ }
+
+ test("SPARK-58008: delete with dynamic options and a subquery on the same
table") {
+ createAndInitTable("pk INT NOT NULL, salary INT, dep STRING",
+ """{ "pk": 1, "salary": 100, "dep": "hr" }
+ |{ "pk": 2, "salary": 200, "dep": "software" }
+ |{ "pk": 3, "salary": 300, "dep": "hr" }
+ |""".stripMargin)
+
+ checkRowLevelOperationOptions(
+ sql(s"DELETE FROM $tableNameAsString WITH (`write.split-size` = 10) " +
+ s"WHERE pk IN (SELECT pk FROM $tableNameAsString WHERE dep = 'hr')"),
+ "write.split-size" -> "10")
+
+ checkAnswer(
+ sql(s"SELECT * FROM $tableNameAsString"),
+ Row(2, 200, "software") :: Nil)
+ }
+
+ test("SPARK-58008: delete with dynamic options and a CTE") {
+ createAndInitTable("pk INT NOT NULL, salary INT, dep STRING",
+ """{ "pk": 1, "salary": 100, "dep": "hr" }
+ |{ "pk": 2, "salary": 200, "dep": "software" }
+ |{ "pk": 3, "salary": 300, "dep": "hr" }
+ |""".stripMargin)
+
+ checkRowLevelOperationOptions(
+ sql(s"""WITH cte AS (
+ | SELECT pk FROM $tableNameAsString WHERE dep = 'hr'
+ |)
+ |DELETE FROM $tableNameAsString WITH (`write.split-size` = 10)
+ |WHERE pk IN (SELECT pk FROM cte)""".stripMargin),
+ "write.split-size" -> "10")
+
+ checkAnswer(
+ sql(s"SELECT * FROM $tableNameAsString"),
+ Row(2, 200, "software") :: Nil)
+ }
+
+ test("SPARK-58008: delete with dynamic options on the deleteWhere path") {
+ createAndInitTable("pk INT NOT NULL, salary INT, dep STRING",
+ """{ "pk": 1, "salary": 100, "dep": "hr" }
+ |{ "pk": 2, "salary": 200, "dep": "software" }
+ |""".stripMargin)
+
+ // dep = 'hr' is an EqualTo on the partition column.
OptimizeMetadataOnlyDeleteFromTable
+ // converts the row-level plan to a deleteWhere call. Verify options flow
into the exec.
+ val Seq(qe) = withQueryExecutionsCaptured(spark) {
+ sql(s"DELETE FROM $tableNameAsString WITH (`write.split-size` = 10)
WHERE dep = 'hr'")
+ }
+ val exec = qe.executedPlan.collectFirst {
+ case e: DeleteFromTableExec => e
+ }.getOrElse(fail("expected DeleteFromTableExec for the metadata-only
deleteWhere path"))
+ assert(exec.options.get("write.split-size") === "10",
+ "options must reach DeleteFromTableExec on the deleteWhere path")
+
assert(exec.table.asInstanceOf[InMemoryRowLevelOperationTable].lastDeleteOptions
+ .get("write.split-size") === "10",
+ "options must be forwarded to SupportsDeleteV2.deleteWhere(predicates,
options)")
+
+ checkAnswer(
+ sql(s"SELECT * FROM $tableNameAsString"),
+ Row(2, 200, "software") :: Nil)
+ }
+
+ test("SPARK-58008: delete all rows with dynamic options") {
+ createAndInitTable("pk INT NOT NULL, salary INT, dep STRING",
+ """{ "pk": 1, "salary": 100, "dep": "hr" }
+ |{ "pk": 2, "salary": 200, "dep": "software" }
+ |""".stripMargin)
+
+ // DELETE with no WHERE: RewriteDeleteFromTable leaves the plan
unrewritten (TruncatableTable
+ // short-circuit). DataSourceV2Strategy then matches
SupportsDeleteV2.canDeleteWhere(AlwaysTrue)
+ // and produces DeleteFromTableExec, carrying the WITH-clause options
through r.options.
+ val Seq(qe) = withQueryExecutionsCaptured(spark) {
+ sql(s"DELETE FROM $tableNameAsString WITH (`write.split-size` = 10)")
+ }
+ val exec = qe.executedPlan.collectFirst {
+ case e: DeleteFromTableExec => e
+ }.getOrElse(fail("expected DeleteFromTableExec for the delete-all path"))
+ assert(exec.options.get("write.split-size") === "10",
+ "options must reach DeleteFromTableExec on the delete-all path")
+
assert(exec.table.asInstanceOf[InMemoryRowLevelOperationTable].lastDeleteOptions
+ .get("write.split-size") === "10",
+ "options must be forwarded to SupportsDeleteV2.deleteWhere(predicates,
options)")
+
+ checkAnswer(sql(s"SELECT * FROM $tableNameAsString"), Nil)
+ }
+
+ test("SPARK-58008: delete all rows with dynamic options on the truncate
path") {
+ // Use a TruncatableTable-only fixture (no SupportsDeleteV2) so that
DELETE with no WHERE
+ // is planned as TruncateTableExec (not DeleteFromTableExec).
+ withSQLConf(
+ "spark.sql.catalog.trunccat" ->
+ classOf[InMemoryTruncatableOnlyTableCatalog].getName) {
+ withTable("trunccat.ns.tbl") {
+ sql("CREATE TABLE trunccat.ns.tbl (pk INT NOT NULL, dep STRING) USING
foo")
+ sql("INSERT INTO trunccat.ns.tbl VALUES (1, 'hr'), (2, 'software')")
+
+ val Seq(qe) = withQueryExecutionsCaptured(spark) {
+ sql("DELETE FROM trunccat.ns.tbl WITH (`write.split-size` = 10)")
+ }
+ val exec = qe.executedPlan.collectFirst {
+ case e: TruncateTableExec => e
+ }.getOrElse(fail("expected TruncateTableExec for the truncate path"))
+ assert(exec.options.get("write.split-size") === "10",
+ "options must reach TruncateTableExec")
+
+ val truncTable = spark.sessionState.catalogManager
+ .catalog("trunccat").asInstanceOf[TableCatalog]
+ .loadTable(org.apache.spark.sql.connector.catalog.Identifier.of(
+ Array("ns"), "tbl"))
+ .asInstanceOf[InMemoryTruncatableOnlyTable]
+ assert(truncTable.lastTruncateOptions.get("write.split-size") === "10",
+ "options must be forwarded to
TruncatableTable.truncateTable(options)")
+
+ checkAnswer(spark.table("trunccat.ns.tbl"), Nil)
+ }
+ }
+ }
+
private def executeDeleteWithRewrite(query: String): Unit = {
val executedPlan = executeAndKeepPlan {
sql(query)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]