This is an automated email from the ASF dual-hosted git repository.
dongjoon-hyun pushed a commit to branch branch-4.x
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-4.x by this push:
new 0328886deebe [SPARK-60824][SQL] Fix internal error when adding a DSv2
CHECK constraint that always holds
0328886deebe is described below
commit 0328886deebe33ec02d3fced7b08209d8b1d88c2
Author: Dongjoon Hyun <[email protected]>
AuthorDate: Tue Nov 3 14:57:17 2026 -0800
[SPARK-60824][SQL] Fix internal error when adding a DSv2 CHECK constraint
that always holds
### What changes were proposed in this pull request?
This PR fixes `ALTER TABLE ... ADD CONSTRAINT ... CHECK` on a Data Source
V2 table, which fails with `INTERNAL_ERROR` when the optimizer can prove that
no existing row violates the condition. Examples are `CHECK (true)`, `CHECK (1
= 1)`, `CHECK (id > 0 OR true)`, `CHECK (null)`, and `CHECK (id IS NOT NULL)`
on a `NOT NULL` column.
`AddCheckConstraint` validates the existing rows through its child,
`Filter(NOT condition, table)`. When the condition always holds, the optimizer
folds this filter to `false` and replaces the whole child with an empty
`LocalRelation`. `DataSourceV2Strategy` looked for the target table only in the
optimized child (`PhysicalOperation(_, _, DataSourceV2ScanRelation)`), so no
strategy could plan the command.
```
AddCheckConstraint checkconstraint(true, id IS NOT NULL, c1, tbl, ...)
+- LocalRelation <empty>, [id#48, data#49]
```
This PR:
- Makes `AddCheckConstraint` extend `KeepAnalyzedQuery`, so that it stores
its analyzed child in a non-child field at the end of analysis, as the V2 write
commands do.
- Makes `DataSourceV2Strategy` take the table from the optimized child as
before, and from the analyzed query when the optimizer removed the scan.
Validation still runs on the optimized child, which returns no rows, and then
the constraint is added.
### Why are the changes needed?
A condition that always holds cannot be violated by the existing rows, so
the statement should succeed and store the constraint. Instead, it fails as
follows.
```
[INTERNAL_ERROR] The Spark SQL phase planning failed with an internal
error. ...
Caused by: java.lang.AssertionError: assertion failed: No plan for
AddCheckConstraint ...
```
### Does this PR introduce _any_ user-facing change?
Yes. `ALTER TABLE ... ADD CONSTRAINT ... CHECK` with a condition that
always holds now adds the constraint instead of failing with `INTERNAL_ERROR`.
This bug exists since
[v4.1.0](https://github.com/apache/spark/releases/tag/v4.1.0) (2025-12-11),
which started validating existing rows when adding a CHECK constraint.
### How was this patch tested?
Pass the CIs with the newly added test case in `CheckConstraintSuite`. The
test fails without this fix with the `INTERNAL_ERROR` above.
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: Claude Opus 5.5
Closes #60042 from dongjoon-hyun/SPARK-60824.
Authored-by: Dongjoon Hyun <[email protected]>
Signed-off-by: Dongjoon Hyun <[email protected]>
(cherry picked from commit b7557f1b5df91a6942edacc2e1dfd8f965e2333b)
Signed-off-by: Dongjoon Hyun <[email protected]>
---
.../sql/catalyst/analysis/CheckAnalysis.scala | 2 +-
.../plans/logical/v2AlterTableCommands.scala | 7 ++++-
.../datasources/v2/DataSourceV2Strategy.scala | 30 +++++++++++++++++-----
.../command/v2/CheckConstraintSuite.scala | 13 ++++++++++
4 files changed, 43 insertions(+), 9 deletions(-)
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/CheckAnalysis.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/CheckAnalysis.scala
index 476e84127db6..d25b15fdf091 100644
---
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/CheckAnalysis.scala
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/CheckAnalysis.scala
@@ -420,7 +420,7 @@ trait CheckAnalysis extends LookupCatalog with
QueryErrorsBase with PlanToString
// The child `Filter` holds the same condition. Check it first so that
an error of the
// subquery in the `Filter` does not hide that the constraint cannot
have a subquery. A
// missing table is still reported first, by the bottom-up traversal.
- case AddCheckConstraint(Filter(_, relation), check) if relation.resolved
=>
+ case AddCheckConstraint(Filter(_, relation), check, _) if
relation.resolved =>
check.failIfContainsSubquery()
case _ =>
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/v2AlterTableCommands.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/v2AlterTableCommands.scala
index 66d1cd6f2df2..d254bdf5ced1 100644
---
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/v2AlterTableCommands.scala
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/v2AlterTableCommands.scala
@@ -382,7 +382,12 @@ case class AddConstraint(
*/
case class AddCheckConstraint(
child: LogicalPlan,
- checkConstraint: CheckConstraint) extends UnaryCommand {
+ checkConstraint: CheckConstraint,
+ analyzedQuery: Option[LogicalPlan] = None) extends UnaryCommand with
KeepAnalyzedQuery {
+
+ // The optimizer replaces the filtered scan with an empty relation if no row
can violate the
+ // check, so keep the analyzed child to find the table when planning.
+ override def storeAnalyzedQuery(): Command = copy(analyzedQuery =
Some(child))
override protected def withNewChildInternal(newChild: LogicalPlan):
LogicalPlan =
copy(child = newChild)
diff --git
a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DataSourceV2Strategy.scala
b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DataSourceV2Strategy.scala
index e452731d12dd..06ff20ca5e53 100644
---
a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DataSourceV2Strategy.scala
+++
b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DataSourceV2Strategy.scala
@@ -829,15 +829,15 @@ class DataSourceV2Strategy(session: SparkSession) extends
Strategy with Predicat
}
UncacheTableExec(r.table, cascade = !isTempView(r.table)) :: Nil
- case a @ AddCheckConstraint(PhysicalOperation(_, _, d:
DataSourceV2ScanRelation), check) =>
- assert(d.relation.catalog.isDefined, "Catalog should be defined after
analysis")
- assert(d.relation.identifier.isDefined, "Identifier should be defined
after analysis")
- val catalog = d.relation.catalog.get.asTableCatalog
- val ident = d.relation.identifier.get
+ case a @ AddCheckConstraintTarget(relation) =>
+ assert(relation.catalog.isDefined, "Catalog should be defined after
analysis")
+ assert(relation.identifier.isDefined, "Identifier should be defined
after analysis")
+ val catalog = relation.catalog.get.asTableCatalog
+ val ident = relation.identifier.get
val condition = a.checkConstraint.condition
val change = TableChange.addConstraint(
- check.toV2Constraint,
- d.relation.table.version)
+ a.checkConstraint.toV2Constraint,
+ relation.table.version)
ResolveTableConstraints.validateCatalogForTableChange(Seq(change),
catalog, ident)
AddCheckConstraintExec(catalog, ident, change, condition,
planLater(a.child)) :: Nil
@@ -902,6 +902,22 @@ private object AppendWrite {
}
}
+/**
+ * Pattern that matches an [[AddCheckConstraint]] on a v2 table and exposes
the table relation.
+ */
+private object AddCheckConstraintTarget {
+ def unapply(a: AddCheckConstraint): Option[DataSourceV2Relation] = a match {
+ case AddCheckConstraint(PhysicalOperation(_, _, d:
DataSourceV2ScanRelation), _, _) =>
+ Some(d.relation)
+ // The optimizer replaces the filtered scan with an empty relation if no
row can violate the
+ // check, so find the table in the analyzed query.
+ case AddCheckConstraint(_, _,
+ Some(Filter(_, SubqueryAlias(_, PhysicalOperation(_, _, r:
DataSourceV2Relation))))) =>
+ Some(r)
+ case _ => None
+ }
+}
+
private[sql] object DataSourceV2Strategy extends Logging {
private def translateLeafNodeFilterV2(predicate: Expression):
Option[Predicate] = {
diff --git
a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v2/CheckConstraintSuite.scala
b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v2/CheckConstraintSuite.scala
index 78e94cac1c50..dc65d35158ad 100644
---
a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v2/CheckConstraintSuite.scala
+++
b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v2/CheckConstraintSuite.scala
@@ -555,6 +555,19 @@ class CheckConstraintSuite extends QueryTest with
CommandSuiteBase with DDLComma
}
}
+ test("SPARK-60824: Alter table add check constraint that always holds") {
+ withNamespaceAndTable("ns", "tbl", nonPartitionCatalog) { t =>
+ sql(s"CREATE TABLE $t (id INT NOT NULL, data STRING) $defaultUsing")
+ sql(s"INSERT INTO $t VALUES (1, 'a')")
+ // The optimizer replaces the scan that validates the existing rows with
an empty relation.
+ sql(s"ALTER TABLE $t ADD CONSTRAINT c1 CHECK (id IS NOT NULL)")
+ val table = loadTable(nonPartitionCatalog, "ns", "tbl")
+ assert(table.validatedVersion() == "1")
+ assert(getCheckConstraint(table).toDDL ==
+ "CONSTRAINT c1 CHECK (id IS NOT NULL) ENFORCED NORELY")
+ }
+ }
+
test("Alter table add new check constraint with nested column") {
withNamespaceAndTable("ns", "tbl", nonPartitionCatalog) { t =>
sql(s"CREATE TABLE $t (id INT, s STRUCT<num INT, str STRING>)
$defaultUsing")
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]