This is an automated email from the ASF dual-hosted git repository.

dongjoon-hyun pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git


The following commit(s) were added to refs/heads/master by this push:
     new 7ffdda03a276 [SPARK-60856][SQL] Fix DSv2 CHECK constraint failing 
every write when  its condition subscripts a lambda variable
7ffdda03a276 is described below

commit 7ffdda03a2765a074e88cfd585ae8da5fad32ff3
Author: Dongjoon Hyun <[email protected]>
AuthorDate: Tue Nov 3 19:34:38 2026 -0800

    [SPARK-60856][SQL] Fix DSv2 CHECK constraint failing every write when  its 
condition subscripts a lambda variable
    
    ### What changes were proposed in this pull request?
    
    This PR fixes `ResolveTableConstraints.buildColumnExtractors` so that it no 
longer uses an extraction that refers to a lambda variable, such as `x[0]` in 
`forall(a, x -> x[0] > 0)`, as a column extractor of `CheckInvariant`. Such an 
extraction now goes through the existing recursion into its children, like any 
other expression.
    
    Column extractors provide the values that a CHECK constraint violation 
error shows. `buildColumnExtractors` took every `UnresolvedExtractValue` in the 
condition as an extractor and placed it in the children of `CheckInvariant`, 
outside the lambda function where it was written. There, 
`ResolveLambdaVariables` turns the unbound lambda variable `x` into 
`UnresolvedAttribute(x)`, and the analysis of the write fails.
    
    A condition with a lambda function has no V2 predicate because 
`V2ExpressionBuilder` does not convert lambda functions. So both `CREATE TABLE` 
and `ALTER TABLE ... ADD CONSTRAINT` re-parse `predicateSql` and reach this 
code path.
    
    ### Why are the changes needed?
    
    A CHECK constraint whose condition subscripts a lambda variable makes every 
write to a DSv2 table fail, including writes of rows that satisfy the 
constraint.
    
    ```sql
    CREATE TABLE t (id INT, a ARRAY<ARRAY<INT>>, CONSTRAINT c1 CHECK (forall(a, 
x -> x[0] > 0)));
    INSERT INTO t VALUES (1, array(array(1)));
    ```
    
    ```
    [UNRESOLVED_COLUMN.WITH_SUGGESTION] A column, variable, or function 
parameter with name `x` cannot be resolved. Did you mean one of the following? 
[`a`, `id`]. SQLSTATE: 42703
    ```
    
    The same failure happens with a map key such as `x['k']` and with a value 
extracted from a function of a lambda variable such as `reverse(x)[0]`.
    
    ### Does this PR introduce _any_ user-facing change?
    
    Yes. Writes to a DSv2 table with such a CHECK constraint now succeed for 
rows that satisfy the constraint and fail with `CHECK_CONSTRAINT_VIOLATION` for 
rows that violate it. Previously, every write failed with `UNRESOLVED_COLUMN`. 
This bug has existed since Apache Spark 4.1.0. The violation errors of other 
conditions do not change.
    
    ### How was this patch tested?
    
    Pass the CIs with the newly added test case in `CheckConstraintSuite`.
    
    ### Was this patch authored or co-authored using generative AI tooling?
    
    Generated-by: Claude Opus 5.5
    
    Closes #60072 from dongjoon-hyun/SPARK-60856.
    
    Authored-by: Dongjoon Hyun <[email protected]>
    Signed-off-by: Dongjoon Hyun <[email protected]>
---
 .../sql/catalyst/analysis/ResolveTableConstraints.scala     |  7 +++++--
 .../sql/execution/command/v2/CheckConstraintSuite.scala     | 13 +++++++++++++
 2 files changed, 18 insertions(+), 2 deletions(-)

diff --git 
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveTableConstraints.scala
 
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveTableConstraints.scala
index 303bc09125b9..d07a3e9122bd 100644
--- 
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveTableConstraints.scala
+++ 
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveTableConstraints.scala
@@ -22,7 +22,7 @@ import org.apache.spark.sql.catalyst.FunctionIdentifier
 import org.apache.spark.sql.catalyst.expressions.{And, CheckInvariant, 
EqualNullSafe, Expression, V2ExpressionUtils}
 import org.apache.spark.sql.catalyst.plans.logical.{Filter, LogicalPlan, 
V2WriteCommand, WriteDelta}
 import org.apache.spark.sql.catalyst.rules.Rule
-import org.apache.spark.sql.catalyst.trees.TreePattern.COMMAND
+import org.apache.spark.sql.catalyst.trees.TreePattern.{COMMAND, 
LAMBDA_VARIABLE}
 import org.apache.spark.sql.catalyst.util.GeneratedColumn
 import org.apache.spark.sql.connector.catalog.{CatalogManager, 
GenerationExpression}
 import org.apache.spark.sql.connector.catalog.constraints.Check
@@ -146,7 +146,10 @@ class ResolveTableConstraints(val catalogManager: 
CatalogManager) extends Rule[L
       expr: Expression,
       columnExtractors: mutable.Map[String, Expression]): Unit = {
     expr match {
-      case u: UnresolvedExtractValue =>
+      // An extraction that refers to a lambda variable, such as `x[0]` in
+      // `forall(a, x -> x[0] > 0)`, cannot be an extractor: the lambda 
variable does not resolve
+      // outside its lambda function.
+      case u: UnresolvedExtractValue if !u.containsPattern(LAMBDA_VARIABLE) =>
         // When extracting a value from a Map or Array type, we display only 
the specific extracted
         // value rather than the entire Map or Array structure for clarity and 
readability.
         columnExtractors(u.sql) = u
diff --git 
a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v2/CheckConstraintSuite.scala
 
b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v2/CheckConstraintSuite.scala
index dc65d35158ad..8b63e33a2391 100644
--- 
a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v2/CheckConstraintSuite.scala
+++ 
b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v2/CheckConstraintSuite.scala
@@ -447,6 +447,19 @@ class CheckConstraintSuite extends QueryTest with 
CommandSuiteBase with DDLComma
     }
   }
 
+  test("SPARK-60856: Check constraint with a subscript of a lambda variable") {
+    withNamespaceAndTable("ns", "tbl", nonPartitionCatalog) { t =>
+      sql(s"CREATE TABLE $t (id INT, a ARRAY<ARRAY<INT>>, " +
+        s"CONSTRAINT c1 CHECK (forall(a, x -> x[0] > 0))) $defaultUsing")
+      sql(s"INSERT INTO $t VALUES (1, array(array(1)))")
+      val error = intercept[SparkRuntimeException] {
+        sql(s"INSERT INTO $t VALUES (2, array(array(-1)))")
+      }
+      assert(error.getCondition == "CHECK_CONSTRAINT_VIOLATION")
+      checkAnswer(spark.table(t), Row(1, Seq(Seq(1))))
+    }
+  }
+
   def getConstraintCharacteristics(): Seq[(String, String)] = {
     Seq(
       ("", s"ENFORCED NORELY"),


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to