This is an automated email from the ASF dual-hosted git repository.
dongjoon-hyun pushed a commit to branch branch-4.1
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-4.1 by this push:
new 49385ee8e10e [SPARK-60856][SQL] Fix DSv2 CHECK constraint failing
every write when its condition subscripts a lambda variable
49385ee8e10e is described below
commit 49385ee8e10e5efcc57a9b666e46cd21ebac95a6
Author: Dongjoon Hyun <[email protected]>
AuthorDate: Tue Nov 3 19:34:38 2026 -0800
[SPARK-60856][SQL] Fix DSv2 CHECK constraint failing every write when its
condition subscripts a lambda variable
This PR fixes `ResolveTableConstraints.buildColumnExtractors` so that it no
longer uses an extraction that refers to a lambda variable, such as `x[0]` in
`forall(a, x -> x[0] > 0)`, as a column extractor of `CheckInvariant`. Such an
extraction now goes through the existing recursion into its children, like any
other expression.
Column extractors provide the values that a CHECK constraint violation
error shows. `buildColumnExtractors` took every `UnresolvedExtractValue` in the
condition as an extractor and placed it in the children of `CheckInvariant`,
outside the lambda function where it was written. There,
`ResolveLambdaVariables` turns the unbound lambda variable `x` into
`UnresolvedAttribute(x)`, and the analysis of the write fails.
A condition with a lambda function has no V2 predicate because
`V2ExpressionBuilder` does not convert lambda functions. So both `CREATE TABLE`
and `ALTER TABLE ... ADD CONSTRAINT` re-parse `predicateSql` and reach this
code path.
A CHECK constraint whose condition subscripts a lambda variable makes every
write to a DSv2 table fail, including writes of rows that satisfy the
constraint.
```sql
CREATE TABLE t (id INT, a ARRAY<ARRAY<INT>>, CONSTRAINT c1 CHECK (forall(a,
x -> x[0] > 0)));
INSERT INTO t VALUES (1, array(array(1)));
```
```
[UNRESOLVED_COLUMN.WITH_SUGGESTION] A column, variable, or function
parameter with name `x` cannot be resolved. Did you mean one of the following?
[`a`, `id`]. SQLSTATE: 42703
```
The same failure happens with a map key such as `x['k']` and with a value
extracted from a function of a lambda variable such as `reverse(x)[0]`.
Yes. Writes to a DSv2 table with such a CHECK constraint now succeed for
rows that satisfy the constraint and fail with `CHECK_CONSTRAINT_VIOLATION` for
rows that violate it. Previously, every write failed with `UNRESOLVED_COLUMN`.
This bug has existed since Apache Spark 4.1.0. The violation errors of other
conditions do not change.
Pass the CIs with the newly added test case in `CheckConstraintSuite`.
Generated-by: Claude Opus 5.5
Closes #60072 from dongjoon-hyun/SPARK-60856.
Authored-by: Dongjoon Hyun <[email protected]>
Signed-off-by: Dongjoon Hyun <[email protected]>
(cherry picked from commit 7ffdda03a2765a074e88cfd585ae8da5fad32ff3)
Signed-off-by: Dongjoon Hyun <[email protected]>
(cherry picked from commit a401457f9dccf97ef5a4dea973638842464b3254)
(cherry picked from commit d142cf4267f30d1f6917a24787ea0085de81fcbe)
Signed-off-by: Dongjoon Hyun <[email protected]>
---
.../sql/catalyst/analysis/ResolveTableConstraints.scala | 7 +++++--
.../sql/execution/command/v2/CheckConstraintSuite.scala | 13 +++++++++++++
2 files changed, 18 insertions(+), 2 deletions(-)
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveTableConstraints.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveTableConstraints.scala
index 41631b24a83e..f64f1277a592 100644
---
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveTableConstraints.scala
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveTableConstraints.scala
@@ -21,7 +21,7 @@ import scala.collection.mutable
import org.apache.spark.sql.catalyst.expressions.{And, CheckInvariant,
Expression, V2ExpressionUtils}
import org.apache.spark.sql.catalyst.plans.logical.{Filter, LogicalPlan,
V2WriteCommand, WriteDelta}
import org.apache.spark.sql.catalyst.rules.Rule
-import org.apache.spark.sql.catalyst.trees.TreePattern.COMMAND
+import org.apache.spark.sql.catalyst.trees.TreePattern.{COMMAND,
LAMBDA_VARIABLE}
import org.apache.spark.sql.connector.catalog.CatalogManager
import org.apache.spark.sql.connector.catalog.constraints.Check
import org.apache.spark.sql.connector.write.RowLevelOperation
@@ -76,7 +76,10 @@ class ResolveTableConstraints(val catalogManager:
CatalogManager) extends Rule[L
expr: Expression,
columnExtractors: mutable.Map[String, Expression]): Unit = {
expr match {
- case u: UnresolvedExtractValue =>
+ // An extraction that refers to a lambda variable, such as `x[0]` in
+ // `forall(a, x -> x[0] > 0)`, cannot be an extractor: the lambda
variable does not resolve
+ // outside its lambda function.
+ case u: UnresolvedExtractValue if !u.containsPattern(LAMBDA_VARIABLE) =>
// When extracting a value from a Map or Array type, we display only
the specific extracted
// value rather than the entire Map or Array structure for clarity and
readability.
columnExtractors(u.sql) = u
diff --git
a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v2/CheckConstraintSuite.scala
b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v2/CheckConstraintSuite.scala
index 8e295ecc3d5d..0105404fb469 100644
---
a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v2/CheckConstraintSuite.scala
+++
b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v2/CheckConstraintSuite.scala
@@ -147,6 +147,19 @@ class CheckConstraintSuite extends QueryTest with
CommandSuiteBase with DDLComma
}
}
+ test("SPARK-60856: Check constraint with a subscript of a lambda variable") {
+ withNamespaceAndTable("ns", "tbl", nonPartitionCatalog) { t =>
+ sql(s"CREATE TABLE $t (id INT, a ARRAY<ARRAY<INT>>, " +
+ s"CONSTRAINT c1 CHECK (forall(a, x -> x[0] > 0))) $defaultUsing")
+ sql(s"INSERT INTO $t VALUES (1, array(array(1)))")
+ val error = intercept[SparkRuntimeException] {
+ sql(s"INSERT INTO $t VALUES (2, array(array(-1)))")
+ }
+ assert(error.getCondition == "CHECK_CONSTRAINT_VIOLATION")
+ checkAnswer(spark.table(t), Row(1, Seq(Seq(1))))
+ }
+ }
+
def getConstraintCharacteristics(): Seq[(String, String)] = {
Seq(
("", s"ENFORCED NORELY"),
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]