This is an automated email from the ASF dual-hosted git repository.
dongjoon-hyun pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 7ffdda03a276 [SPARK-60856][SQL] Fix DSv2 CHECK constraint failing
every write when its condition subscripts a lambda variable
7ffdda03a276 is described below
commit 7ffdda03a2765a074e88cfd585ae8da5fad32ff3
Author: Dongjoon Hyun <[email protected]>
AuthorDate: Tue Nov 3 19:34:38 2026 -0800
[SPARK-60856][SQL] Fix DSv2 CHECK constraint failing every write when its
condition subscripts a lambda variable
### What changes were proposed in this pull request?
This PR fixes `ResolveTableConstraints.buildColumnExtractors` so that it no
longer uses an extraction that refers to a lambda variable, such as `x[0]` in
`forall(a, x -> x[0] > 0)`, as a column extractor of `CheckInvariant`. Such an
extraction now goes through the existing recursion into its children, like any
other expression.
Column extractors provide the values that a CHECK constraint violation
error shows. `buildColumnExtractors` took every `UnresolvedExtractValue` in the
condition as an extractor and placed it in the children of `CheckInvariant`,
outside the lambda function where it was written. There,
`ResolveLambdaVariables` turns the unbound lambda variable `x` into
`UnresolvedAttribute(x)`, and the analysis of the write fails.
A condition with a lambda function has no V2 predicate because
`V2ExpressionBuilder` does not convert lambda functions. So both `CREATE TABLE`
and `ALTER TABLE ... ADD CONSTRAINT` re-parse `predicateSql` and reach this
code path.
### Why are the changes needed?
A CHECK constraint whose condition subscripts a lambda variable makes every
write to a DSv2 table fail, including writes of rows that satisfy the
constraint.
```sql
CREATE TABLE t (id INT, a ARRAY<ARRAY<INT>>, CONSTRAINT c1 CHECK (forall(a,
x -> x[0] > 0)));
INSERT INTO t VALUES (1, array(array(1)));
```
```
[UNRESOLVED_COLUMN.WITH_SUGGESTION] A column, variable, or function
parameter with name `x` cannot be resolved. Did you mean one of the following?
[`a`, `id`]. SQLSTATE: 42703
```
The same failure happens with a map key such as `x['k']` and with a value
extracted from a function of a lambda variable such as `reverse(x)[0]`.
### Does this PR introduce _any_ user-facing change?
Yes. Writes to a DSv2 table with such a CHECK constraint now succeed for
rows that satisfy the constraint and fail with `CHECK_CONSTRAINT_VIOLATION` for
rows that violate it. Previously, every write failed with `UNRESOLVED_COLUMN`.
This bug has existed since Apache Spark 4.1.0. The violation errors of other
conditions do not change.
### How was this patch tested?
Pass the CIs with the newly added test case in `CheckConstraintSuite`.
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: Claude Opus 5.5
Closes #60072 from dongjoon-hyun/SPARK-60856.
Authored-by: Dongjoon Hyun <[email protected]>
Signed-off-by: Dongjoon Hyun <[email protected]>
---
.../sql/catalyst/analysis/ResolveTableConstraints.scala | 7 +++++--
.../sql/execution/command/v2/CheckConstraintSuite.scala | 13 +++++++++++++
2 files changed, 18 insertions(+), 2 deletions(-)
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveTableConstraints.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveTableConstraints.scala
index 303bc09125b9..d07a3e9122bd 100644
---
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveTableConstraints.scala
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveTableConstraints.scala
@@ -22,7 +22,7 @@ import org.apache.spark.sql.catalyst.FunctionIdentifier
import org.apache.spark.sql.catalyst.expressions.{And, CheckInvariant,
EqualNullSafe, Expression, V2ExpressionUtils}
import org.apache.spark.sql.catalyst.plans.logical.{Filter, LogicalPlan,
V2WriteCommand, WriteDelta}
import org.apache.spark.sql.catalyst.rules.Rule
-import org.apache.spark.sql.catalyst.trees.TreePattern.COMMAND
+import org.apache.spark.sql.catalyst.trees.TreePattern.{COMMAND,
LAMBDA_VARIABLE}
import org.apache.spark.sql.catalyst.util.GeneratedColumn
import org.apache.spark.sql.connector.catalog.{CatalogManager,
GenerationExpression}
import org.apache.spark.sql.connector.catalog.constraints.Check
@@ -146,7 +146,10 @@ class ResolveTableConstraints(val catalogManager:
CatalogManager) extends Rule[L
expr: Expression,
columnExtractors: mutable.Map[String, Expression]): Unit = {
expr match {
- case u: UnresolvedExtractValue =>
+ // An extraction that refers to a lambda variable, such as `x[0]` in
+ // `forall(a, x -> x[0] > 0)`, cannot be an extractor: the lambda
variable does not resolve
+ // outside its lambda function.
+ case u: UnresolvedExtractValue if !u.containsPattern(LAMBDA_VARIABLE) =>
// When extracting a value from a Map or Array type, we display only
the specific extracted
// value rather than the entire Map or Array structure for clarity and
readability.
columnExtractors(u.sql) = u
diff --git
a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v2/CheckConstraintSuite.scala
b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v2/CheckConstraintSuite.scala
index dc65d35158ad..8b63e33a2391 100644
---
a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v2/CheckConstraintSuite.scala
+++
b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v2/CheckConstraintSuite.scala
@@ -447,6 +447,19 @@ class CheckConstraintSuite extends QueryTest with
CommandSuiteBase with DDLComma
}
}
+ test("SPARK-60856: Check constraint with a subscript of a lambda variable") {
+ withNamespaceAndTable("ns", "tbl", nonPartitionCatalog) { t =>
+ sql(s"CREATE TABLE $t (id INT, a ARRAY<ARRAY<INT>>, " +
+ s"CONSTRAINT c1 CHECK (forall(a, x -> x[0] > 0))) $defaultUsing")
+ sql(s"INSERT INTO $t VALUES (1, array(array(1)))")
+ val error = intercept[SparkRuntimeException] {
+ sql(s"INSERT INTO $t VALUES (2, array(array(-1)))")
+ }
+ assert(error.getCondition == "CHECK_CONSTRAINT_VIOLATION")
+ checkAnswer(spark.table(t), Row(1, Seq(Seq(1))))
+ }
+ }
+
def getConstraintCharacteristics(): Seq[(String, String)] = {
Seq(
("", s"ENFORCED NORELY"),
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]