This is an automated email from the ASF dual-hosted git repository.
dtenedor pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 1dd8ce005e37 [SPARK-60036][SQL] Preserve USER_SPECIFIED_CAST when
retargeting collation-only CAST
1dd8ce005e37 is described below
commit 1dd8ce005e379da6fc78f0c4b295002dafe2b829
Author: Serge Rielau <[email protected]>
AuthorDate: Thu Oct 8 22:46:35 2026 -0700
[SPARK-60036][SQL] Preserve USER_SPECIFIED_CAST when retargeting
collation-only CAST
### What changes were proposed in this pull request?
SPARK-60003 nested Cast only when a CHAR/VARCHAR constraint changes, and
retargeted collation-only LCT with `cast.copy(dataType = ...)`. `copy` drops
`USER_SPECIFIED_CAST`, which `truncateCharVarcharOnCast` uses. After a
collation-only retarget, a user CAST to VARCHAR(n) then runs the write-side
length check instead of truncating.
This PR copies tags onto the retargeted Cast. Nested Cast (constraint
change) is unchanged: the inner node still has the original tags.
### Why are the changes needed?
A struct CAST with a VARCHAR field and a STRING field can take the copy
branch when only the STRING field's collation changes. Without the tag,
`CAST(named_struct('v', 'abcdef', ...) AS STRUCT<v VARCHAR(2), ...>)` inside
`COALESCE` raises `EXCEED_LIMIT_LENGTH` instead of truncating to `"ab"`.
### Does this PR introduce _any_ user-facing change?
Yes. After SPARK-60003, collation-only LCT on a user CAST could turn
CHAR/VARCHAR truncation into `EXCEED_LIMIT_LENGTH`. Truncation is restored.
Compared with Spark 4.0 this is not a change: SPARK-60003 itself is unreleased.
### How was this patch tested?
- `BasicCharVarcharTestSuite`: struct `VARCHAR(n)` + `STRING` where only
the STRING collation changes (copy branch); asserts truncation to `"ab"`.
- Confirmed the new test fails with `EXCEED_LIMIT_LENGTH` when tags are not
copied.
```
JAVA_HOME=/usr/lib/jvm/java-17-openjdk-amd64 build/sbt
-Dsbt.override.build.repos=true \
'sql/testOnly org.apache.spark.sql.BasicCharVarcharTestSuite -- -z
"SPARK-60036"'
```
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: Cursor Grok 4.6
Closes #59270 from srielau/SPARK-60036.
Authored-by: Serge Rielau <[email protected]>
Signed-off-by: Daniel Tenedorio <[email protected]>
---
.../sql/catalyst/analysis/CollationTypeCoercion.scala | 12 +++++++++---
.../org/apache/spark/sql/CharVarcharTestSuite.scala | 19 +++++++++++++++++++
2 files changed, 28 insertions(+), 3 deletions(-)
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/CollationTypeCoercion.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/CollationTypeCoercion.scala
index 82160dec2a18..a16b00322b44 100644
---
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/CollationTypeCoercion.scala
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/CollationTypeCoercion.scala
@@ -113,8 +113,10 @@ object CollationTypeCoercion extends SQLConfHelper {
*
* Literal and Cast: wrap in a new Cast when the CHAR/VARCHAR constraint
* changes so truncation, overflow, and padding stay on the original node.
- * Collation-only changes retarget with `copy(dataType)`, which keeps a
- * non-string Cast child non-string and therefore Default strength.
+ * Collation-only changes retarget with `copy(dataType)` and copy
+ * `USER_SPECIFIED_CAST` (truncation). Do not copy all tags: a stale
+ * `COLLATION_CONTEXT_TAG` would reuse the pre-LCT collation. That keeps
+ * a non-string Cast child non-string and therefore Default strength.
*/
private def changeType(expr: Expression, newType: DataType): Expression = {
mergeTypes(expr.dataType, newType) match {
@@ -128,7 +130,11 @@ object CollationTypeCoercion extends SQLConfHelper {
case cast: Cast if stringConstraintChanged(cast.dataType,
newDataType) =>
Cast(cast, newDataType, timeZoneId =
Some(conf.sessionLocalTimeZone))
case cast: Cast =>
- cast.copy(dataType = newDataType)
+ val retargeted = cast.copy(dataType = newDataType)
+ if (cast.containsTag(Cast.USER_SPECIFIED_CAST)) {
+ retargeted.setTagValue(Cast.USER_SPECIFIED_CAST, ())
+ }
+ retargeted
case subquery: SubqueryExpression =>
changeTypeInSubquery(subquery, newType)
diff --git
a/sql/core/src/test/scala/org/apache/spark/sql/CharVarcharTestSuite.scala
b/sql/core/src/test/scala/org/apache/spark/sql/CharVarcharTestSuite.scala
index 58397079b8d7..c203fa1d8c34 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/CharVarcharTestSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/CharVarcharTestSuite.scala
@@ -1200,6 +1200,25 @@ class BasicCharVarcharTestSuite extends
SharedSparkSession {
}
}
+ test("SPARK-60036: collation-only LCT copy keeps USER_SPECIFIED_CAST") {
+ withSQLConf(SQLConf.CHAR_VARCHAR_STANDARD_SEMANTICS.key -> "true") {
+ // VARCHAR length is unchanged; only the STRING field collation changes,
so
+ // changeType uses Cast.copy. Without copying USER_SPECIFIED_CAST, the
+ // VARCHAR(2) field would raise EXCEED_LIMIT_LENGTH instead of
truncating.
+ val df = sql(
+ """SELECT coalesce(
+ | CAST(named_struct('v', 'abcdef', 's', 'foo')
+ | AS STRUCT<v VARCHAR(2), s STRING>),
+ | named_struct('v', CAST('xy' AS VARCHAR(2)),
+ | 's', collate('bar', 'UTF8_LCASE'))
+ |) AS c""".stripMargin)
+ assert(df.schema.head.dataType === StructType(Seq(
+ StructField("v", VarcharType(2)),
+ StructField("s", StringType("UTF8_LCASE")))))
+ checkAnswer(df, Row(Row("ab", "foo")))
+ }
+ }
+
test("SPARK-58799: transforming string functions return STRING") {
withSQLConf(SQLConf.CHAR_VARCHAR_STANDARD_SEMANTICS.key -> "true") {
assert(sql("SELECT upper(cast('ab' AS CHAR(2))) AS c")
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
