This is an automated email from the ASF dual-hosted git repository.
HyukjinKwon pushed a commit to branch branch-4.x
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-4.x by this push:
new c4ef618e9a6f [SPARK-60574][SQL] Quote collated string literals in
connector expressions
c4ef618e9a6f is described below
commit c4ef618e9a6f8fb0f7e6e1c7a54c7b3e2368435a
Author: Linhong Liu <[email protected]>
AuthorDate: Wed Oct 28 10:24:58 2026 -0700
[SPARK-60574][SQL] Quote collated string literals in connector expressions
### What changes were proposed in this pull request?
Quote every StringType literal in the connector SQL renderer, including
non-default collations and CHAR/VARCHAR string types. Add a comment explaining
why the type pattern is required.
### Why are the changes needed?
The `StringType` singleton pattern matches only UTF8_BINARY strings without
a length constraint: `StringType.equals` compares both the collation ID and the
constraint. It therefore misses non-default collations and CHAR/VARCHAR
literals, even when their collation is UTF8_BINARY, and renders them without
SQL quoting or escaping. A UTF8_LCASE literal containing `it's` currently
renders as `it's` without surrounding SQL quotes, instead of `'it''s'`. Pattern
predicates strip the assumed s [...]
### Does this PR introduce _any_ user-facing change?
Yes, a bug fix. Collated string literals and CHAR/VARCHAR literals receive
the same quoting and escaping as ordinary string literals. NULL continues to
render as NULL.
### How was this patch tested?
Added regression coverage for collated and constrained string types, NULL,
empty and quoted strings, and pattern-predicate descriptions. Four new literal
cases fail on unchanged Apache code. On JDK 17, all 10 LiteralValueSuite tests,
21 V2PredicateSuite tests, and the JDBC filter-compilation test passed.
Catalyst main/test and SQL test Scalastyle checks and git diff --check passed.
```bash
build/sbt 'catalyst/testOnly
org.apache.spark.sql.connector.expressions.LiteralValueSuite'
build/sbt 'sql/testOnly
org.apache.spark.sql.execution.datasources.v2.V2PredicateSuite'
build/sbt 'sql/testOnly org.apache.spark.sql.jdbc.JDBCSuite -- -z "compile
filters"'
```
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: OpenAI Codex (GPT-6)
Closes #59812 from linhongliu-db/fix-collated-literal-quoting.
Authored-by: Linhong Liu <[email protected]>
Signed-off-by: Hyukjin Kwon <[email protected]>
(cherry picked from commit a584b11749e817b60af14ce5cc91099d5af21b0a)
Signed-off-by: Hyukjin Kwon <[email protected]>
---
.../spark/sql/connector/expressions/expressions.scala | 6 +++++-
.../sql/connector/expressions/LiteralValueSuite.scala | 16 +++++++++++++++-
.../sql/execution/datasources/v2/V2PredicateSuite.scala | 16 ++++++++++++++++
3 files changed, 36 insertions(+), 2 deletions(-)
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/connector/expressions/expressions.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/connector/expressions/expressions.scala
index 0028f78d22f9..409681cb1156 100644
---
a/sql/catalyst/src/main/scala/org/apache/spark/sql/connector/expressions/expressions.scala
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/connector/expressions/expressions.scala
@@ -393,7 +393,11 @@ private[sql] final case class LiteralValue[T](value: T,
dataType: DataType) exte
"NULL"
} else {
dataType match {
- case StringType => s"'${s"$value".replace("'", "''")}'"
+ // The StringType value pattern matches only UTF8_BINARY strings
without a length
+ // constraint. Other collations and CHAR/VARCHAR are StringType
instances that need the
+ // same quoting and escaping.
+ // Pattern predicates also rely on these quotes when extracting the
literal contents.
+ case _: StringType => s"'${s"$value".replace("'", "''")}'"
case BinaryType =>
assert(value.isInstanceOf[Array[Byte]])
val bytes = value.asInstanceOf[Array[Byte]]
diff --git
a/sql/catalyst/src/test/scala/org/apache/spark/sql/connector/expressions/LiteralValueSuite.scala
b/sql/catalyst/src/test/scala/org/apache/spark/sql/connector/expressions/LiteralValueSuite.scala
index 9a21f14e022a..bc408db48317 100644
---
a/sql/catalyst/src/test/scala/org/apache/spark/sql/connector/expressions/LiteralValueSuite.scala
+++
b/sql/catalyst/src/test/scala/org/apache/spark/sql/connector/expressions/LiteralValueSuite.scala
@@ -18,7 +18,8 @@
package org.apache.spark.sql.connector.expressions
import org.apache.spark.SparkFunSuite
-import org.apache.spark.sql.types.{BinaryType, BooleanType, IntegerType,
StringType}
+import org.apache.spark.sql.types.{BinaryType, BooleanType, CharType,
IntegerType, StringType, VarcharType}
+import org.apache.spark.unsafe.types.UTF8String
class LiteralValueSuite extends SparkFunSuite {
@@ -35,6 +36,19 @@ class LiteralValueSuite extends SparkFunSuite {
assert(LiteralValue("it's", StringType).toString === "'it''s'")
}
+ for (dataType <- Seq(
+ StringType, StringType("UTF8_LCASE"), StringType("UNICODE_CI"),
+ CharType(8), VarcharType(8))) {
+ test(s"collated and constrained string literals should be quoted:
$dataType") {
+ for ((value, expected) <- Seq(
+ "" -> "''", "a" -> "'a'", "abc" -> "'abc'", "null" -> "'null'",
+ "it's" -> "'it''s'", "a%b_c" -> "'a%b_c'", "a\\b" -> "'a\\b'")) {
+ assert(LiteralValue(UTF8String.fromString(value), dataType).toString
=== expected)
+ }
+ assert(LiteralValue(null, dataType).toString === "NULL")
+ }
+ }
+
test("non-null numeric literals should not be quoted") {
assert(LiteralValue(42, IntegerType).toString === "42")
assert(LiteralValue(0, IntegerType).toString === "0")
diff --git
a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/v2/V2PredicateSuite.scala
b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/v2/V2PredicateSuite.scala
index b3a097180725..932a105dd392 100644
---
a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/v2/V2PredicateSuite.scala
+++
b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/v2/V2PredicateSuite.scala
@@ -308,6 +308,22 @@ class V2PredicateSuite extends SparkFunSuite {
assert(PredicateUtils.toV1(predicate4) == None)
}
+ Seq("STARTS_WITH", "ENDS_WITH", "CONTAINS").foreach { predicateName =>
+ test(s"$predicateName descriptions preserve collated string literals") {
+ for {
+ collation <- Seq("UTF8_LCASE", "UNICODE_CI")
+ value <- Seq("", "a", "abc", "it's", "a%b_c", "a\\b")
+ } {
+ val string = UTF8String.fromString(value)
+ val predicate = new Predicate(predicateName,
+ Array[Expression](ref("a"), LiteralValue(string,
StringType(collation))))
+ val binaryPredicate = new Predicate(predicateName,
+ Array[Expression](ref("a"), LiteralValue(string, StringType)))
+ assert(predicate.describe() == binaryPredicate.describe())
+ }
+ }
+ }
+
test("StringStartsWith") {
val literal = LiteralValue(UTF8String.fromString("str"), StringType)
val predicate1 = new Predicate("STARTS_WITH",
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
