This is an automated email from the ASF dual-hosted git repository.

dongjoon-hyun pushed a commit to branch branch-4.x
in repository https://gitbox.apache.org/repos/asf/spark.git


The following commit(s) were added to refs/heads/branch-4.x by this push:
     new 03fcb4879493 [SPARK-60764][SQL] Do not embed constraint text in the 
generated code of DSv2 CheckInvariant
03fcb4879493 is described below

commit 03fcb4879493c8d12e07e320967d8a7c2049017c
Author: Dongjoon Hyun <[email protected]>
AuthorDate: Sat Oct 31 07:28:32 2026 -0700

    [SPARK-60764][SQL] Do not embed constraint text in the generated code of 
DSv2 CheckInvariant
    
    ### What changes were proposed in this pull request?
    
    This PR makes `CheckInvariant`, which enforces CHECK constraints and 
generated columns on DSv2 writes, pass the constraint name, the predicate SQL 
and the column names to its generated code with `ctx.addReferenceObj`, instead 
of embedding them in the generated Java source as string literals. 
`AssertNotNull` and `PrintToStderr` already pass their texts in the same way.
    
    ### Why are the changes needed?
    
    `CheckInvariant` embeds these texts in the generated Java source as string 
literals without escaping.
    
    - A double quote, a newline or an invalid escape sequence makes the 
generated code fail to compile. An ordinary condition such as `s <> "x"` hits 
this, and so does `ALTER TABLE cat.ns.t ADD CONSTRAINT c1 CHECK (s <> 'x"y')`. 
With the default configuration Spark falls back to the interpreter, so the 
check and the data stay correct, but every such write disables whole-stage 
codegen and logs a compile failure for each partition.
    - A backslash is decoded by the Java compiler, so the violation message 
differs from the constraint. `s <> 'x\\y'` is shown as `s <> 'x\y'`.
    
    Generated column checks go through the same expression, so a generation 
expression hits the same failures.
    
    ### Does this PR introduce _any_ user-facing change?
    
    Yes. This is a bug fix for DSv2 tables in 
[v4.1.0](https://github.com/apache/spark/releases/tag/v4.1.0) (2025-12-11) and 
later. Other tables are not affected because `CheckInvariant` is only added to 
DSv2 writes.
    
    - Writes to a DSv2 table whose constraint text has such a character keep 
whole-stage codegen and no longer log compile failures.
    - The violation message shows the constraint name, the condition and the 
column names as they were written.
    
    ### How was this patch tested?
    
    Pass the CIs with the newly added test cases. The new test cases fail 
without this patch.
    
    ```
    build/sbt 'catalyst/testOnly *ConstraintExpressionSuite'
    build/sbt 'sql/testOnly *CheckConstraintSuite *GeneratedColumnWriteSuite'
    ```
    
    I also ran the same writes against `InMemoryTableCatalog` in a JVM without 
the test settings. Before this patch, compile failures and `Whole-stage codegen 
disabled` warnings were logged. After this patch, neither appears.
    
    ### Was this patch authored or co-authored using generative AI tooling?
    
    Generated-by: Claude Opus 5.5
    
    Closes #59971 from dongjoon-hyun/SPARK-60764.
    
    Authored-by: Dongjoon Hyun <[email protected]>
    Signed-off-by: Dongjoon Hyun <[email protected]>
    (cherry picked from commit d781c67c7c0c00d6e8de28e3aeb8729984fb7278)
    Signed-off-by: Dongjoon Hyun <[email protected]>
---
 .../sql/catalyst/expressions/constraints.scala     |  9 +++-
 .../expressions/ConstraintExpressionSuite.scala    | 27 +++++++++++
 .../sql/connector/GeneratedColumnWriteSuite.scala  | 23 +++++++++
 .../command/v2/CheckConstraintSuite.scala          | 56 ++++++++++++++++++++++
 4 files changed, 113 insertions(+), 2 deletions(-)

diff --git 
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/constraints.scala
 
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/constraints.scala
index 1a7e3b03c0e6..12cfab81f959 100644
--- 
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/constraints.scala
+++ 
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/constraints.scala
@@ -338,8 +338,10 @@ case class CheckInvariant(
     columnExtractors.map {
       case (name, extractor) =>
         val colValue = extractor.genCode(ctx)
+        // The name is not Java source, so pass it by reference instead of 
embedding it.
+        val nameRef = ctx.addReferenceObj("columnName", name)
         code"""
-              |$colList.add("$name");
+              |$colList.add($nameRef);
               |${colValue.code}
               |if (${colValue.isNull}) {
               |  $valList.add(null);
@@ -354,12 +356,15 @@ case class CheckInvariant(
     val elementValue = child.genCode(ctx)
     val colListName = ctx.freshName("colList")
     val valListName = ctx.freshName("valList")
+    // These texts are not Java source, so pass them by reference instead of 
embedding them.
+    val nameRef = ctx.addReferenceObj("constraintName", constraintName)
+    val sqlRef = ctx.addReferenceObj("predicateSql", predicateSql)
     val ret = code"""${elementValue.code}
           |
           |if (!${elementValue.isNull} && ${elementValue.value} == false) {
           |  ${generateColumnValuesCode(colListName, valListName, ctx)}
           |  throw 
org.apache.spark.sql.errors.QueryExecutionErrors.checkViolationJava(
-          |     "$constraintName", "$predicateSql", $colListName, 
$valListName);
+          |     $nameRef, $sqlRef, $colListName, $valListName);
           |}
      """.stripMargin
     ret
diff --git 
a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ConstraintExpressionSuite.scala
 
b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ConstraintExpressionSuite.scala
index 3bc40374fc01..d9ad27aa60c1 100644
--- 
a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ConstraintExpressionSuite.scala
+++ 
b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ConstraintExpressionSuite.scala
@@ -18,6 +18,7 @@ package org.apache.spark.sql.catalyst.expressions
 
 import org.apache.spark.{SparkFunSuite, SparkRuntimeException}
 import org.apache.spark.sql.catalyst.InternalRow
+import org.apache.spark.sql.catalyst.expressions.codegen.CodegenContext
 import org.apache.spark.sql.types.IntegerType
 
 class ConstraintExpressionSuite extends SparkFunSuite with 
ExpressionEvalHelper {
@@ -42,4 +43,30 @@ class ConstraintExpressionSuite extends SparkFunSuite with 
ExpressionEvalHelper
     checkExceptionInExpression[SparkRuntimeException](
       expr, InternalRow(-1), expectedMessage("-1"))
   }
+
+  test("SPARK-60764: CheckInvariant: the texts are not embedded in the 
generated code") {
+    val ctx = new CodegenContext
+    val code = CheckInvariant(
+      GreaterThan(boundRef, Literal(0)),
+      Seq(("column_text", boundRef)),
+      "name_text",
+      "predicate_text").genCode(ctx).code.toString
+    Seq("column_text", "name_text", "predicate_text").foreach { text =>
+      assert(!code.contains(text))
+      assert(ctx.references.contains(text))
+    }
+  }
+
+  test("SPARK-60764: CheckInvariant: double quote, newline and backslash in 
the texts") {
+    Seq("a\"b", "a\nb", "a\\b", "a\\qb").foreach { text =>
+      val invariant =
+        CheckInvariant(GreaterThan(boundRef, Literal(0)), Seq((text, 
boundRef)), text, text)
+      checkEvaluation(invariant, true, InternalRow(1))
+      checkExceptionInExpression[SparkRuntimeException](
+        invariant,
+        InternalRow(-1),
+        s"[CHECK_CONSTRAINT_VIOLATION] CHECK constraint $text $text violated 
by row with values:" +
+          s"\n - $text : -1\n SQLSTATE: 23001")
+    }
+  }
 }
diff --git 
a/sql/core/src/test/scala/org/apache/spark/sql/connector/GeneratedColumnWriteSuite.scala
 
b/sql/core/src/test/scala/org/apache/spark/sql/connector/GeneratedColumnWriteSuite.scala
index 943d883799e1..274ec02f262d 100644
--- 
a/sql/core/src/test/scala/org/apache/spark/sql/connector/GeneratedColumnWriteSuite.scala
+++ 
b/sql/core/src/test/scala/org/apache/spark/sql/connector/GeneratedColumnWriteSuite.scala
@@ -362,6 +362,29 @@ class GeneratedColumnWriteSuite extends QueryTest with 
DatasourceV2SQLBase {
     }
   }
 
+  test("SPARK-60764: INSERT by name with a double quote in the generation 
expression") {
+    val tblName = "my_tab"
+    withTable(s"testcat.$tblName") {
+      sql(s"""CREATE TABLE testcat.$tblName(
+             |  a STRING,
+             |  b STRING GENERATED ALWAYS AS (concat(a, '"'))
+             |) USING foo""".stripMargin)
+      // Read from `range` so that the check runs in whole-stage codegen.
+      sql(s"""INSERT INTO testcat.$tblName(a, b) SELECT 'p', concat('p', '"') 
FROM range(2)""")
+      val ex = intercept[SparkRuntimeException] {
+        sql(s"INSERT INTO testcat.$tblName(a, b) SELECT 'p', 'q' FROM 
range(2)")
+      }
+      checkError(
+        exception = ex,
+        condition = "CHECK_CONSTRAINT_VIOLATION",
+        sqlState = "23001",
+        parameters = Map(
+          "constraintName" -> "Generated Column",
+          "expression" -> """b <=> concat(a, '"')""",
+          "values" -> " - b : q"))
+    }
+  }
+
   test("INSERT by position does not auto-fill trailing generated column") {
     val tblName = "my_tab"
     withTable(s"testcat.$tblName") {
diff --git 
a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v2/CheckConstraintSuite.scala
 
b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v2/CheckConstraintSuite.scala
index 631c7bfb37c9..52da4802ef14 100644
--- 
a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v2/CheckConstraintSuite.scala
+++ 
b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v2/CheckConstraintSuite.scala
@@ -721,6 +721,62 @@ class CheckConstraintSuite extends QueryTest with 
CommandSuiteBase with DDLComma
     }
   }
 
+  test("SPARK-60764: Check constraint violation on table insert - " +
+    "special characters in the condition") {
+    Seq(
+      """id > 0 OR s = "x"""",
+      """id > 0 OR s = 'x"y'""",
+      "id > 0 OR\n s = 'xy'",
+      """id > 0 OR s = 'x\\y'""",
+      """id > 0 OR s = 'x\qy'"""
+    ).foreach { condition =>
+      withNamespaceAndTable("ns", "tbl", nonPartitionCatalog) { t =>
+        sql(s"CREATE TABLE $t (id INT, s STRING) $defaultUsing")
+        sql(s"ALTER TABLE $t ADD CONSTRAINT c1 CHECK ($condition)")
+        // Read from `range` so that the check runs in whole-stage codegen.
+        sql(s"INSERT INTO $t SELECT id + 1, 'ok' FROM range(2)")
+        val error = intercept[SparkRuntimeException] {
+          sql(s"INSERT INTO $t SELECT id, 'ok' FROM range(2)")
+        }
+        checkError(
+          exception = error,
+          condition = "CHECK_CONSTRAINT_VIOLATION",
+          sqlState = "23001",
+          parameters = Map(
+            "constraintName" -> "c1",
+            "expression" -> condition,
+            "values" -> " - id : 0\n - s : ok"
+          )
+        )
+      }
+    }
+  }
+
+  test("SPARK-60764: Check constraint violation on table insert - " +
+    "special characters in the names") {
+    Seq("a\"b", "a\nb", "a\\b").foreach { name =>
+      withNamespaceAndTable("ns", "tbl", nonPartitionCatalog) { t =>
+        sql(s"CREATE TABLE $t (id INT, `$name` STRING) $defaultUsing")
+        sql(s"ALTER TABLE $t ADD CONSTRAINT `$name` CHECK (id > 0 OR `$name` = 
'x')")
+        // Read from `range` so that the check runs in whole-stage codegen.
+        sql(s"INSERT INTO $t SELECT id + 1, 'ok' FROM range(2)")
+        val error = intercept[SparkRuntimeException] {
+          sql(s"INSERT INTO $t SELECT id, 'ok' FROM range(2)")
+        }
+        checkError(
+          exception = error,
+          condition = "CHECK_CONSTRAINT_VIOLATION",
+          sqlState = "23001",
+          parameters = Map(
+            "constraintName" -> name,
+            "expression" -> s"id > 0 OR `$name` = 'x'",
+            "values" -> s" - $name : ok\n - id : 0"
+          )
+        )
+      }
+    }
+  }
+
   test("Check constraint violation on table update - top level column") {
     withNamespaceAndTable("ns", "tbl", rowLevelOPCatalog) { t =>
       sql(s"CREATE TABLE $t (id INT, value INT," +


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to