This is an automated email from the ASF dual-hosted git repository.
wenchen pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new e00d26fcf24f [SPARK-48600][SQL] Fix FrameLessOffsetWindowFunction
expressions implicit casting
e00d26fcf24f is described below
commit e00d26fcf24fc5f0ae2737864405e52750fc977a
Author: Mihailo Milosevic <[email protected]>
AuthorDate: Mon Jun 17 15:01:43 2024 +0800
[SPARK-48600][SQL] Fix FrameLessOffsetWindowFunction expressions implicit
casting
### What changes were proposed in this pull request?
Fix for implicit casting of `FrameLessOffsetWindowFunction`s.
### Why are the changes needed?
These functions require third parameter to have the same type as the first
one. For that reason `CollationTypeCasts` needs to cover this case.
### Does this PR introduce _any_ user-facing change?
Yes, Lag and Lead will preform implicit type check.
### How was this patch tested?
Tests added to relevant suite.
### Was this patch authored or co-authored using generative AI tooling?
No.
Closes #46987 from mihailom-db/SPARK-48600.
Authored-by: Mihailo Milosevic <[email protected]>
Signed-off-by: Wenchen Fan <[email protected]>
---
.../sql/catalyst/analysis/CollationTypeCasts.scala | 5 ++++
.../spark/sql/CollationSQLExpressionsSuite.scala | 34 ++++++++++++++++++++++
2 files changed, 39 insertions(+)
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/CollationTypeCasts.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/CollationTypeCasts.scala
index b832cd4416a9..b131b8b148b6 100644
---
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/CollationTypeCasts.scala
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/CollationTypeCasts.scala
@@ -78,6 +78,11 @@ object CollationTypeCasts extends TypeCoercionRule {
}
raiseError.withNewChildren(Seq(raiseError.errorClass, newErrorParams))
+ case framelessOffsetWindow @ (_: Lag | _: Lead) =>
+ val Seq(input, offset, default) = framelessOffsetWindow.children
+ val Seq(newInput, newDefault) = collateToSingleType(Seq(input, default))
+ framelessOffsetWindow.withNewChildren(Seq(newInput, offset, newDefault))
+
case otherExpr @ (
_: In | _: InSubquery | _: CreateArray | _: ArrayJoin | _: Concat | _:
Greatest | _: Least |
_: Coalesce | _: BinaryExpression | _: ConcatWs | _: Mask | _:
StringReplace |
diff --git
a/sql/core/src/test/scala/org/apache/spark/sql/CollationSQLExpressionsSuite.scala
b/sql/core/src/test/scala/org/apache/spark/sql/CollationSQLExpressionsSuite.scala
index d34fd554e7bd..5f3d33d8825a 100644
---
a/sql/core/src/test/scala/org/apache/spark/sql/CollationSQLExpressionsSuite.scala
+++
b/sql/core/src/test/scala/org/apache/spark/sql/CollationSQLExpressionsSuite.scala
@@ -1854,6 +1854,40 @@ class CollationSQLExpressionsSuite
})
}
+ test("Lag expression with collation") {
+ // Supported collations
+ testSuppCollations.foreach(collationName => {
+ val query =
+ s"""
+ |SELECT lag(a, -1, 'default' collate $collationName) OVER
(PARTITION BY b ORDER BY a)
+ |FROM VALUES ('A1', 2), ('A2', 1), ('A2', 3), ('A1', 1) tab(a, b);
+ |""".stripMargin
+ // Result & data type check
+ val testQuery = sql(query)
+ val dataType = StringType(collationName)
+ val expectedResult = Seq("A2", "default", "default", "default")
+ assert(testQuery.schema.fields.head.dataType.sameType(dataType))
+ checkAnswer(testQuery, expectedResult.map(Row(_)))
+ })
+ }
+
+ test("Lead expression with collation") {
+ // Supported collations
+ testSuppCollations.foreach(collationName => {
+ val query =
+ s"""
+ |SELECT lead(a, -1, 'default' collate $collationName) OVER
(PARTITION BY b ORDER BY a)
+ |FROM VALUES ('A1', 2), ('A2', 1), ('A2', 3), ('A1', 1) tab(a, b);
+ |""".stripMargin
+ // Result & data type check
+ val testQuery = sql(query)
+ val dataType = StringType(collationName)
+ val expectedResult = Seq("A1", "default", "default", "default")
+ assert(testQuery.schema.fields.head.dataType.sameType(dataType))
+ checkAnswer(testQuery, expectedResult.map(Row(_)))
+ })
+ }
+
test("DatePart expression with collation") {
// Supported collations
testSuppCollations.foreach(collationName => {
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
