This is an automated email from the ASF dual-hosted git repository.
dongjoon-hyun pushed a commit to branch branch-4.3
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-4.3 by this push:
new cb0324a1c4b4 [SPARK-60764][SQL] Do not embed constraint text in the
generated code of DSv2 CheckInvariant
cb0324a1c4b4 is described below
commit cb0324a1c4b4cf9b4d0e99b3141e453504c810f9
Author: Dongjoon Hyun <[email protected]>
AuthorDate: Sat Oct 31 07:28:32 2026 -0700
[SPARK-60764][SQL] Do not embed constraint text in the generated code of
DSv2 CheckInvariant
### What changes were proposed in this pull request?
This PR makes `CheckInvariant`, which enforces CHECK constraints and
generated columns on DSv2 writes, pass the constraint name, the predicate SQL
and the column names to its generated code with `ctx.addReferenceObj`, instead
of embedding them in the generated Java source as string literals.
`AssertNotNull` and `PrintToStderr` already pass their texts in the same way.
### Why are the changes needed?
`CheckInvariant` embeds these texts in the generated Java source as string
literals without escaping.
- A double quote, a newline or an invalid escape sequence makes the
generated code fail to compile. An ordinary condition such as `s <> "x"` hits
this, and so does `ALTER TABLE cat.ns.t ADD CONSTRAINT c1 CHECK (s <> 'x"y')`.
With the default configuration Spark falls back to the interpreter, so the
check and the data stay correct, but every such write disables whole-stage
codegen and logs a compile failure for each partition.
- A backslash is decoded by the Java compiler, so the violation message
differs from the constraint. `s <> 'x\\y'` is shown as `s <> 'x\y'`.
Generated column checks go through the same expression, so a generation
expression hits the same failures.
### Does this PR introduce _any_ user-facing change?
Yes. This is a bug fix for DSv2 tables in
[v4.1.0](https://github.com/apache/spark/releases/tag/v4.1.0) (2025-12-11) and
later. Other tables are not affected because `CheckInvariant` is only added to
DSv2 writes.
- Writes to a DSv2 table whose constraint text has such a character keep
whole-stage codegen and no longer log compile failures.
- The violation message shows the constraint name, the condition and the
column names as they were written.
### How was this patch tested?
Pass the CIs with the newly added test cases. The new test cases fail
without this patch.
```
build/sbt 'catalyst/testOnly *ConstraintExpressionSuite'
build/sbt 'sql/testOnly *CheckConstraintSuite *GeneratedColumnWriteSuite'
```
I also ran the same writes against `InMemoryTableCatalog` in a JVM without
the test settings. Before this patch, compile failures and `Whole-stage codegen
disabled` warnings were logged. After this patch, neither appears.
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: Claude Opus 5.5
Closes #59971 from dongjoon-hyun/SPARK-60764.
Authored-by: Dongjoon Hyun <[email protected]>
Signed-off-by: Dongjoon Hyun <[email protected]>
(cherry picked from commit d781c67c7c0c00d6e8de28e3aeb8729984fb7278)
Signed-off-by: Dongjoon Hyun <[email protected]>
---
.../sql/catalyst/expressions/constraints.scala | 9 +++-
.../expressions/ConstraintExpressionSuite.scala | 27 +++++++++++
.../sql/connector/GeneratedColumnWriteSuite.scala | 23 +++++++++
.../command/v2/CheckConstraintSuite.scala | 56 ++++++++++++++++++++++
4 files changed, 113 insertions(+), 2 deletions(-)
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/constraints.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/constraints.scala
index 1a7e3b03c0e6..12cfab81f959 100644
---
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/constraints.scala
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/constraints.scala
@@ -338,8 +338,10 @@ case class CheckInvariant(
columnExtractors.map {
case (name, extractor) =>
val colValue = extractor.genCode(ctx)
+ // The name is not Java source, so pass it by reference instead of
embedding it.
+ val nameRef = ctx.addReferenceObj("columnName", name)
code"""
- |$colList.add("$name");
+ |$colList.add($nameRef);
|${colValue.code}
|if (${colValue.isNull}) {
| $valList.add(null);
@@ -354,12 +356,15 @@ case class CheckInvariant(
val elementValue = child.genCode(ctx)
val colListName = ctx.freshName("colList")
val valListName = ctx.freshName("valList")
+ // These texts are not Java source, so pass them by reference instead of
embedding them.
+ val nameRef = ctx.addReferenceObj("constraintName", constraintName)
+ val sqlRef = ctx.addReferenceObj("predicateSql", predicateSql)
val ret = code"""${elementValue.code}
|
|if (!${elementValue.isNull} && ${elementValue.value} == false) {
| ${generateColumnValuesCode(colListName, valListName, ctx)}
| throw
org.apache.spark.sql.errors.QueryExecutionErrors.checkViolationJava(
- | "$constraintName", "$predicateSql", $colListName,
$valListName);
+ | $nameRef, $sqlRef, $colListName, $valListName);
|}
""".stripMargin
ret
diff --git
a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ConstraintExpressionSuite.scala
b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ConstraintExpressionSuite.scala
index 3bc40374fc01..d9ad27aa60c1 100644
---
a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ConstraintExpressionSuite.scala
+++
b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ConstraintExpressionSuite.scala
@@ -18,6 +18,7 @@ package org.apache.spark.sql.catalyst.expressions
import org.apache.spark.{SparkFunSuite, SparkRuntimeException}
import org.apache.spark.sql.catalyst.InternalRow
+import org.apache.spark.sql.catalyst.expressions.codegen.CodegenContext
import org.apache.spark.sql.types.IntegerType
class ConstraintExpressionSuite extends SparkFunSuite with
ExpressionEvalHelper {
@@ -42,4 +43,30 @@ class ConstraintExpressionSuite extends SparkFunSuite with
ExpressionEvalHelper
checkExceptionInExpression[SparkRuntimeException](
expr, InternalRow(-1), expectedMessage("-1"))
}
+
+ test("SPARK-60764: CheckInvariant: the texts are not embedded in the
generated code") {
+ val ctx = new CodegenContext
+ val code = CheckInvariant(
+ GreaterThan(boundRef, Literal(0)),
+ Seq(("column_text", boundRef)),
+ "name_text",
+ "predicate_text").genCode(ctx).code.toString
+ Seq("column_text", "name_text", "predicate_text").foreach { text =>
+ assert(!code.contains(text))
+ assert(ctx.references.contains(text))
+ }
+ }
+
+ test("SPARK-60764: CheckInvariant: double quote, newline and backslash in
the texts") {
+ Seq("a\"b", "a\nb", "a\\b", "a\\qb").foreach { text =>
+ val invariant =
+ CheckInvariant(GreaterThan(boundRef, Literal(0)), Seq((text,
boundRef)), text, text)
+ checkEvaluation(invariant, true, InternalRow(1))
+ checkExceptionInExpression[SparkRuntimeException](
+ invariant,
+ InternalRow(-1),
+ s"[CHECK_CONSTRAINT_VIOLATION] CHECK constraint $text $text violated
by row with values:" +
+ s"\n - $text : -1\n SQLSTATE: 23001")
+ }
+ }
}
diff --git
a/sql/core/src/test/scala/org/apache/spark/sql/connector/GeneratedColumnWriteSuite.scala
b/sql/core/src/test/scala/org/apache/spark/sql/connector/GeneratedColumnWriteSuite.scala
index c963d91005f1..38ed391a0b36 100644
---
a/sql/core/src/test/scala/org/apache/spark/sql/connector/GeneratedColumnWriteSuite.scala
+++
b/sql/core/src/test/scala/org/apache/spark/sql/connector/GeneratedColumnWriteSuite.scala
@@ -362,6 +362,29 @@ class GeneratedColumnWriteSuite extends QueryTest with
DatasourceV2SQLBase {
}
}
+ test("SPARK-60764: INSERT by name with a double quote in the generation
expression") {
+ val tblName = "my_tab"
+ withTable(s"testcat.$tblName") {
+ sql(s"""CREATE TABLE testcat.$tblName(
+ | a STRING,
+ | b STRING GENERATED ALWAYS AS (concat(a, '"'))
+ |) USING foo""".stripMargin)
+ // Read from `range` so that the check runs in whole-stage codegen.
+ sql(s"""INSERT INTO testcat.$tblName(a, b) SELECT 'p', concat('p', '"')
FROM range(2)""")
+ val ex = intercept[SparkRuntimeException] {
+ sql(s"INSERT INTO testcat.$tblName(a, b) SELECT 'p', 'q' FROM
range(2)")
+ }
+ checkError(
+ exception = ex,
+ condition = "CHECK_CONSTRAINT_VIOLATION",
+ sqlState = "23001",
+ parameters = Map(
+ "constraintName" -> "Generated Column",
+ "expression" -> """b <=> concat(a, '"')""",
+ "values" -> " - b : q"))
+ }
+ }
+
test("INSERT by position does not auto-fill trailing generated column") {
val tblName = "my_tab"
withTable(s"testcat.$tblName") {
diff --git
a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v2/CheckConstraintSuite.scala
b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v2/CheckConstraintSuite.scala
index 8e295ecc3d5d..709f992d57d3 100644
---
a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v2/CheckConstraintSuite.scala
+++
b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v2/CheckConstraintSuite.scala
@@ -698,6 +698,62 @@ class CheckConstraintSuite extends QueryTest with
CommandSuiteBase with DDLComma
}
}
+ test("SPARK-60764: Check constraint violation on table insert - " +
+ "special characters in the condition") {
+ Seq(
+ """id > 0 OR s = "x"""",
+ """id > 0 OR s = 'x"y'""",
+ "id > 0 OR\n s = 'xy'",
+ """id > 0 OR s = 'x\\y'""",
+ """id > 0 OR s = 'x\qy'"""
+ ).foreach { condition =>
+ withNamespaceAndTable("ns", "tbl", nonPartitionCatalog) { t =>
+ sql(s"CREATE TABLE $t (id INT, s STRING) $defaultUsing")
+ sql(s"ALTER TABLE $t ADD CONSTRAINT c1 CHECK ($condition)")
+ // Read from `range` so that the check runs in whole-stage codegen.
+ sql(s"INSERT INTO $t SELECT id + 1, 'ok' FROM range(2)")
+ val error = intercept[SparkRuntimeException] {
+ sql(s"INSERT INTO $t SELECT id, 'ok' FROM range(2)")
+ }
+ checkError(
+ exception = error,
+ condition = "CHECK_CONSTRAINT_VIOLATION",
+ sqlState = "23001",
+ parameters = Map(
+ "constraintName" -> "c1",
+ "expression" -> condition,
+ "values" -> " - id : 0\n - s : ok"
+ )
+ )
+ }
+ }
+ }
+
+ test("SPARK-60764: Check constraint violation on table insert - " +
+ "special characters in the names") {
+ Seq("a\"b", "a\nb", "a\\b").foreach { name =>
+ withNamespaceAndTable("ns", "tbl", nonPartitionCatalog) { t =>
+ sql(s"CREATE TABLE $t (id INT, `$name` STRING) $defaultUsing")
+ sql(s"ALTER TABLE $t ADD CONSTRAINT `$name` CHECK (id > 0 OR `$name` =
'x')")
+ // Read from `range` so that the check runs in whole-stage codegen.
+ sql(s"INSERT INTO $t SELECT id + 1, 'ok' FROM range(2)")
+ val error = intercept[SparkRuntimeException] {
+ sql(s"INSERT INTO $t SELECT id, 'ok' FROM range(2)")
+ }
+ checkError(
+ exception = error,
+ condition = "CHECK_CONSTRAINT_VIOLATION",
+ sqlState = "23001",
+ parameters = Map(
+ "constraintName" -> name,
+ "expression" -> s"id > 0 OR `$name` = 'x'",
+ "values" -> s" - $name : ok\n - id : 0"
+ )
+ )
+ }
+ }
+ }
+
test("Check constraint violation on table update - top level column") {
withNamespaceAndTable("ns", "tbl", rowLevelOPCatalog) { t =>
sql(s"CREATE TABLE $t (id INT, value INT," +
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]