stevomitric commented on code in PR #47372:
URL: https://github.com/apache/spark/pull/47372#discussion_r1686423780
##########
sql/core/src/test/scala/org/apache/spark/sql/CollationSQLExpressionsSuite.scala:
##########
@@ -2295,6 +2295,827 @@ class CollationSQLExpressionsSuite
assert(typeException.getErrorClass ===
"DATATYPE_MISMATCH.UNEXPECTED_STATIC_METHOD")
}
+ test("min_by supports collation") {
+ val collation = "UNICODE"
+ val query = s"SELECT min_by(x, y) FROM VALUES ('a', 10), ('b', 50), ('c',
20) AS tab(x, y);"
+ withSQLConf(SqlApiConf.DEFAULT_COLLATION -> collation) {
+ checkAnswer(
+ sql(query),
+ Seq(
+ Row("a")
+ )
+ )
+ // check result row data type
+ val dataType = StringType(collation)
+ assert(sql(query).schema.head.dataType == dataType)
+ }
+ }
+
+ test("max_by supports collation") {
+ val collation = "UNICODE"
+ val query = s"SELECT max_by(x, y) FROM VALUES ('a', 10), ('b', 50), ('c',
20) AS tab(x, y);"
+ withSQLConf(SqlApiConf.DEFAULT_COLLATION -> collation) {
+ checkAnswer(
+ sql(query),
+ Seq(
+ Row("b")
+ )
+ )
+ // check result row data type
+ val dataType = StringType(collation)
+ assert(sql(query).schema.head.dataType == dataType)
+ }
+ }
+
+ test("array supports collation") {
Review Comment:
Comparing two expressions that are cast as different collations explicitly
fails, ex:
```sql
SELECT 'a' COLLATE UTF8_LCASE == 'a' COLLATE UNICODE
> Error occurred due to the mismatch between explicit collations: `string
collate UTF8_LCASE`.`string collate UNICODE`. Decide on a single explicit
collation and remove others.
```
while comparing elements in the array that are cast explicitly passes:
```sql
select array('a' COLLATE UTF8_LCASE)[0] == 'A' COLLATE UTF8_BINARY
```
The resulting comparisons appears to be `UTF8_BINARY`. Does this mean that
accessing elements in the array removes the explicit cast? Adding @mihailom-db
for comment.
##########
sql/core/src/test/scala/org/apache/spark/sql/CollationSQLExpressionsSuite.scala:
##########
@@ -2295,6 +2295,827 @@ class CollationSQLExpressionsSuite
assert(typeException.getErrorClass ===
"DATATYPE_MISMATCH.UNEXPECTED_STATIC_METHOD")
}
+ test("min_by supports collation") {
+ val collation = "UNICODE"
+ val query = s"SELECT min_by(x, y) FROM VALUES ('a', 10), ('b', 50), ('c',
20) AS tab(x, y);"
+ withSQLConf(SqlApiConf.DEFAULT_COLLATION -> collation) {
+ checkAnswer(
+ sql(query),
+ Seq(
+ Row("a")
+ )
+ )
+ // check result row data type
+ val dataType = StringType(collation)
+ assert(sql(query).schema.head.dataType == dataType)
+ }
+ }
+
+ test("max_by supports collation") {
+ val collation = "UNICODE"
+ val query = s"SELECT max_by(x, y) FROM VALUES ('a', 10), ('b', 50), ('c',
20) AS tab(x, y);"
+ withSQLConf(SqlApiConf.DEFAULT_COLLATION -> collation) {
+ checkAnswer(
+ sql(query),
+ Seq(
+ Row("b")
+ )
+ )
+ // check result row data type
+ val dataType = StringType(collation)
+ assert(sql(query).schema.head.dataType == dataType)
+ }
+ }
+
+ test("array supports collation") {
+ val collation = "UNICODE"
+ val query = s"SELECT array('a', 'b', 'c');"
+ withSQLConf(SqlApiConf.DEFAULT_COLLATION -> collation) {
+ checkAnswer(
+ sql(query),
+ Seq(
+ Row(Seq("a", "b", "c"))
+ )
+ )
+ // check result row data type
+ val dataType = ArrayType(StringType(collation), false)
+ assert(sql(query).schema.head.dataType == dataType)
+ }
+ }
+
+ test("array_agg supports collation") {
+ val collation = "UNICODE"
+ val query = s"SELECT array_agg(col) FROM VALUES ('a'), ('b'), ('c') AS
tab(col);"
+ withSQLConf(SqlApiConf.DEFAULT_COLLATION -> collation) {
+ checkAnswer(
+ sql(query),
+ Seq(
+ Row(Seq("a", "b", "c"))
+ )
+ )
+ // check result row data type
+ val dataType = ArrayType(StringType(collation), false)
+ assert(sql(query).schema.head.dataType == dataType)
+ }
+ }
+
+ test("array_contains supports collation") {
+ val collation = "UNICODE"
+ val query = s"SELECT array_contains(array('a', 'b', 'c'), 'b');"
+ withSQLConf(SqlApiConf.DEFAULT_COLLATION -> collation) {
+ checkAnswer(
+ sql(query),
+ Seq(
+ Row(true)
+ )
+ )
+ // check result row data type
+ val dataType = BooleanType
+ assert(sql(query).schema.head.dataType == dataType)
+ }
+ }
+
+ test("arrays_overlap supports collation") {
+ val collation = "UNICODE"
+ val query = s"SELECT arrays_overlap(array('a', 'b', 'c'), array('c', 'd',
'e'));"
+ withSQLConf(SqlApiConf.DEFAULT_COLLATION -> collation) {
+ checkAnswer(
+ sql(query),
+ Seq(
+ Row(true)
+ )
+ )
+ // check result row data type
+ val dataType = BooleanType
+ assert(sql(query).schema.head.dataType == dataType)
+ }
+ }
+
+ test("array_insert supports collation") {
+ val collation = "UNICODE"
+ val query = s"SELECT array_insert(array('a', 'b', 'c', 'd'), 5, 'e');"
+ withSQLConf(SqlApiConf.DEFAULT_COLLATION -> collation) {
+ checkAnswer(
+ sql(query),
+ Seq(
+ Row(Seq("a", "b", "c", "d", "e"))
+ )
+ )
+ // check result row data type
+ val dataType = ArrayType(StringType(collation), true)
+ assert(sql(query).schema.head.dataType == dataType)
+ }
+ }
+
+ test("array_intersect supports collation") {
+ val collation = "UNICODE"
+ val query = s"SELECT array_intersect(array('a', 'b', 'c'), array('b', 'c',
'd'));"
+ withSQLConf(SqlApiConf.DEFAULT_COLLATION -> collation) {
+ checkAnswer(
+ sql(query),
+ Seq(
+ Row(Seq("b", "c"))
+ )
+ )
+ // check result row data type
+ val dataType = ArrayType(StringType(collation), false)
+ assert(sql(query).schema.head.dataType == dataType)
+ }
+ }
+
+ test("array_join supports collation") {
+ val collation = "UNICODE"
+ val query = s"SELECT array_join(array('hello', 'world'), ' ');"
+ withSQLConf(SqlApiConf.DEFAULT_COLLATION -> collation) {
+ checkAnswer(
+ sql(query),
+ Seq(
+ Row("hello world")
+ )
+ )
+ // check result row data type
+ val dataType = StringType(collation)
+ assert(sql(query).schema.head.dataType == dataType)
+ }
+ }
+
+ test("array_position supports collation") {
+ val collation = "UNICODE"
+ val query = s"SELECT array_position(array('a', 'b', 'c', 'c'), 'c');"
+ withSQLConf(SqlApiConf.DEFAULT_COLLATION -> collation) {
+ checkAnswer(
+ sql(query),
+ Seq(
+ Row(3)
+ )
+ )
+ // check result row data type
+ val dataType = LongType
+ assert(sql(query).schema.head.dataType == dataType)
+ }
+ }
+
+ test("array_size supports collation") {
+ val collation = "UNICODE"
+ val query = s"SELECT array_size(array('a', 'b', 'c', 'c'));"
+ withSQLConf(SqlApiConf.DEFAULT_COLLATION -> collation) {
+ checkAnswer(
+ sql(query),
+ Seq(
+ Row(4)
+ )
+ )
+ // check result row data type
+ val dataType = IntegerType
+ assert(sql(query).schema.head.dataType == dataType)
+ }
+ }
+
+ test("array_sort supports collation") {
+ val collation = "UNICODE"
+ val query = s"SELECT array_sort(array('b', null, 'a'));"
Review Comment:
Maybe change to capital `A` here?
##########
sql/core/src/test/scala/org/apache/spark/sql/CollationSQLExpressionsSuite.scala:
##########
@@ -2295,6 +2295,827 @@ class CollationSQLExpressionsSuite
assert(typeException.getErrorClass ===
"DATATYPE_MISMATCH.UNEXPECTED_STATIC_METHOD")
}
+ test("min_by supports collation") {
+ val collation = "UNICODE"
+ val query = s"SELECT min_by(x, y) FROM VALUES ('a', 10), ('b', 50), ('c',
20) AS tab(x, y);"
+ withSQLConf(SqlApiConf.DEFAULT_COLLATION -> collation) {
+ checkAnswer(
+ sql(query),
+ Seq(
+ Row("a")
+ )
+ )
+ // check result row data type
+ val dataType = StringType(collation)
+ assert(sql(query).schema.head.dataType == dataType)
+ }
+ }
+
+ test("max_by supports collation") {
+ val collation = "UNICODE"
+ val query = s"SELECT max_by(x, y) FROM VALUES ('a', 10), ('b', 50), ('c',
20) AS tab(x, y);"
+ withSQLConf(SqlApiConf.DEFAULT_COLLATION -> collation) {
+ checkAnswer(
+ sql(query),
+ Seq(
+ Row("b")
+ )
+ )
+ // check result row data type
+ val dataType = StringType(collation)
+ assert(sql(query).schema.head.dataType == dataType)
+ }
+ }
+
+ test("array supports collation") {
+ val collation = "UNICODE"
+ val query = s"SELECT array('a', 'b', 'c');"
+ withSQLConf(SqlApiConf.DEFAULT_COLLATION -> collation) {
+ checkAnswer(
+ sql(query),
+ Seq(
+ Row(Seq("a", "b", "c"))
+ )
+ )
+ // check result row data type
+ val dataType = ArrayType(StringType(collation), false)
+ assert(sql(query).schema.head.dataType == dataType)
+ }
+ }
+
+ test("array_agg supports collation") {
+ val collation = "UNICODE"
+ val query = s"SELECT array_agg(col) FROM VALUES ('a'), ('b'), ('c') AS
tab(col);"
+ withSQLConf(SqlApiConf.DEFAULT_COLLATION -> collation) {
+ checkAnswer(
+ sql(query),
+ Seq(
+ Row(Seq("a", "b", "c"))
+ )
+ )
+ // check result row data type
+ val dataType = ArrayType(StringType(collation), false)
+ assert(sql(query).schema.head.dataType == dataType)
+ }
+ }
+
+ test("array_contains supports collation") {
+ val collation = "UNICODE"
+ val query = s"SELECT array_contains(array('a', 'b', 'c'), 'b');"
+ withSQLConf(SqlApiConf.DEFAULT_COLLATION -> collation) {
+ checkAnswer(
+ sql(query),
+ Seq(
+ Row(true)
+ )
+ )
+ // check result row data type
+ val dataType = BooleanType
+ assert(sql(query).schema.head.dataType == dataType)
+ }
+ }
+
+ test("arrays_overlap supports collation") {
+ val collation = "UNICODE"
+ val query = s"SELECT arrays_overlap(array('a', 'b', 'c'), array('c', 'd',
'e'));"
+ withSQLConf(SqlApiConf.DEFAULT_COLLATION -> collation) {
+ checkAnswer(
+ sql(query),
+ Seq(
+ Row(true)
+ )
+ )
+ // check result row data type
+ val dataType = BooleanType
+ assert(sql(query).schema.head.dataType == dataType)
+ }
+ }
+
+ test("array_insert supports collation") {
+ val collation = "UNICODE"
+ val query = s"SELECT array_insert(array('a', 'b', 'c', 'd'), 5, 'e');"
+ withSQLConf(SqlApiConf.DEFAULT_COLLATION -> collation) {
+ checkAnswer(
+ sql(query),
+ Seq(
+ Row(Seq("a", "b", "c", "d", "e"))
+ )
+ )
+ // check result row data type
+ val dataType = ArrayType(StringType(collation), true)
+ assert(sql(query).schema.head.dataType == dataType)
+ }
+ }
+
+ test("array_intersect supports collation") {
+ val collation = "UNICODE"
+ val query = s"SELECT array_intersect(array('a', 'b', 'c'), array('b', 'c',
'd'));"
+ withSQLConf(SqlApiConf.DEFAULT_COLLATION -> collation) {
+ checkAnswer(
+ sql(query),
+ Seq(
+ Row(Seq("b", "c"))
+ )
+ )
+ // check result row data type
+ val dataType = ArrayType(StringType(collation), false)
+ assert(sql(query).schema.head.dataType == dataType)
+ }
+ }
+
+ test("array_join supports collation") {
+ val collation = "UNICODE"
+ val query = s"SELECT array_join(array('hello', 'world'), ' ');"
+ withSQLConf(SqlApiConf.DEFAULT_COLLATION -> collation) {
+ checkAnswer(
+ sql(query),
+ Seq(
+ Row("hello world")
+ )
+ )
+ // check result row data type
+ val dataType = StringType(collation)
+ assert(sql(query).schema.head.dataType == dataType)
+ }
+ }
+
+ test("array_position supports collation") {
+ val collation = "UNICODE"
+ val query = s"SELECT array_position(array('a', 'b', 'c', 'c'), 'c');"
+ withSQLConf(SqlApiConf.DEFAULT_COLLATION -> collation) {
+ checkAnswer(
+ sql(query),
+ Seq(
+ Row(3)
+ )
+ )
+ // check result row data type
+ val dataType = LongType
+ assert(sql(query).schema.head.dataType == dataType)
+ }
+ }
+
+ test("array_size supports collation") {
+ val collation = "UNICODE"
+ val query = s"SELECT array_size(array('a', 'b', 'c', 'c'));"
+ withSQLConf(SqlApiConf.DEFAULT_COLLATION -> collation) {
+ checkAnswer(
+ sql(query),
+ Seq(
+ Row(4)
+ )
+ )
+ // check result row data type
+ val dataType = IntegerType
+ assert(sql(query).schema.head.dataType == dataType)
+ }
+ }
+
+ test("array_sort supports collation") {
Review Comment:
NOTE(long output here, not related to collations):
```
scala> sql("select array_sort(array('a', 'B'))").show
warning: 1 deprecation (since 2.13.3); for details, enable `:setting
-deprecation` or `:replay -deprecation`
+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
|array_sort(array(a, B), lambdafunction((IF(((namedlambdavariable() IS NULL)
AND (namedlambdavariable() IS NULL)), 0, (IF((namedlambdavariable() IS NULL),
1, (IF((namedlambdavariable() IS NULL), -1, (IF((namedlambdavariable() <
namedlambdavariable()), -1, (IF((namedlambdavariable() >
namedlambdavariable()), 1, 0)))))))))), namedlambdavariable(),
namedlambdavariable()))|
+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
|
[B, a]|
+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
```
Should we consider removing the default lambda function from the output?
##########
sql/core/src/test/scala/org/apache/spark/sql/CollationSQLExpressionsSuite.scala:
##########
@@ -2295,6 +2295,827 @@ class CollationSQLExpressionsSuite
assert(typeException.getErrorClass ===
"DATATYPE_MISMATCH.UNEXPECTED_STATIC_METHOD")
}
+ test("min_by supports collation") {
+ val collation = "UNICODE"
+ val query = s"SELECT min_by(x, y) FROM VALUES ('a', 10), ('b', 50), ('c',
20) AS tab(x, y);"
+ withSQLConf(SqlApiConf.DEFAULT_COLLATION -> collation) {
+ checkAnswer(
+ sql(query),
+ Seq(
+ Row("a")
+ )
+ )
+ // check result row data type
+ val dataType = StringType(collation)
+ assert(sql(query).schema.head.dataType == dataType)
+ }
+ }
+
+ test("max_by supports collation") {
+ val collation = "UNICODE"
+ val query = s"SELECT max_by(x, y) FROM VALUES ('a', 10), ('b', 50), ('c',
20) AS tab(x, y);"
+ withSQLConf(SqlApiConf.DEFAULT_COLLATION -> collation) {
+ checkAnswer(
+ sql(query),
+ Seq(
+ Row("b")
+ )
+ )
+ // check result row data type
+ val dataType = StringType(collation)
+ assert(sql(query).schema.head.dataType == dataType)
+ }
+ }
+
+ test("array supports collation") {
+ val collation = "UNICODE"
+ val query = s"SELECT array('a', 'b', 'c');"
+ withSQLConf(SqlApiConf.DEFAULT_COLLATION -> collation) {
+ checkAnswer(
+ sql(query),
+ Seq(
+ Row(Seq("a", "b", "c"))
+ )
+ )
+ // check result row data type
+ val dataType = ArrayType(StringType(collation), false)
+ assert(sql(query).schema.head.dataType == dataType)
+ }
+ }
+
+ test("array_agg supports collation") {
+ val collation = "UNICODE"
+ val query = s"SELECT array_agg(col) FROM VALUES ('a'), ('b'), ('c') AS
tab(col);"
+ withSQLConf(SqlApiConf.DEFAULT_COLLATION -> collation) {
+ checkAnswer(
+ sql(query),
+ Seq(
+ Row(Seq("a", "b", "c"))
+ )
+ )
+ // check result row data type
+ val dataType = ArrayType(StringType(collation), false)
+ assert(sql(query).schema.head.dataType == dataType)
+ }
+ }
+
+ test("array_contains supports collation") {
+ val collation = "UNICODE"
+ val query = s"SELECT array_contains(array('a', 'b', 'c'), 'b');"
+ withSQLConf(SqlApiConf.DEFAULT_COLLATION -> collation) {
+ checkAnswer(
+ sql(query),
+ Seq(
+ Row(true)
+ )
+ )
+ // check result row data type
+ val dataType = BooleanType
+ assert(sql(query).schema.head.dataType == dataType)
+ }
+ }
+
+ test("arrays_overlap supports collation") {
+ val collation = "UNICODE"
+ val query = s"SELECT arrays_overlap(array('a', 'b', 'c'), array('c', 'd',
'e'));"
+ withSQLConf(SqlApiConf.DEFAULT_COLLATION -> collation) {
+ checkAnswer(
+ sql(query),
+ Seq(
+ Row(true)
+ )
+ )
+ // check result row data type
+ val dataType = BooleanType
+ assert(sql(query).schema.head.dataType == dataType)
+ }
+ }
+
+ test("array_insert supports collation") {
+ val collation = "UNICODE"
+ val query = s"SELECT array_insert(array('a', 'b', 'c', 'd'), 5, 'e');"
+ withSQLConf(SqlApiConf.DEFAULT_COLLATION -> collation) {
+ checkAnswer(
+ sql(query),
+ Seq(
+ Row(Seq("a", "b", "c", "d", "e"))
+ )
+ )
+ // check result row data type
+ val dataType = ArrayType(StringType(collation), true)
+ assert(sql(query).schema.head.dataType == dataType)
+ }
+ }
+
+ test("array_intersect supports collation") {
+ val collation = "UNICODE"
+ val query = s"SELECT array_intersect(array('a', 'b', 'c'), array('b', 'c',
'd'));"
+ withSQLConf(SqlApiConf.DEFAULT_COLLATION -> collation) {
+ checkAnswer(
+ sql(query),
+ Seq(
+ Row(Seq("b", "c"))
+ )
+ )
+ // check result row data type
+ val dataType = ArrayType(StringType(collation), false)
+ assert(sql(query).schema.head.dataType == dataType)
+ }
+ }
+
+ test("array_join supports collation") {
+ val collation = "UNICODE"
+ val query = s"SELECT array_join(array('hello', 'world'), ' ');"
+ withSQLConf(SqlApiConf.DEFAULT_COLLATION -> collation) {
+ checkAnswer(
+ sql(query),
+ Seq(
+ Row("hello world")
+ )
+ )
+ // check result row data type
+ val dataType = StringType(collation)
+ assert(sql(query).schema.head.dataType == dataType)
+ }
+ }
+
+ test("array_position supports collation") {
+ val collation = "UNICODE"
+ val query = s"SELECT array_position(array('a', 'b', 'c', 'c'), 'c');"
+ withSQLConf(SqlApiConf.DEFAULT_COLLATION -> collation) {
+ checkAnswer(
+ sql(query),
+ Seq(
+ Row(3)
+ )
+ )
+ // check result row data type
+ val dataType = LongType
+ assert(sql(query).schema.head.dataType == dataType)
+ }
+ }
+
+ test("array_size supports collation") {
+ val collation = "UNICODE"
+ val query = s"SELECT array_size(array('a', 'b', 'c', 'c'));"
+ withSQLConf(SqlApiConf.DEFAULT_COLLATION -> collation) {
+ checkAnswer(
+ sql(query),
+ Seq(
+ Row(4)
+ )
+ )
+ // check result row data type
+ val dataType = IntegerType
+ assert(sql(query).schema.head.dataType == dataType)
+ }
+ }
+
+ test("array_sort supports collation") {
+ val collation = "UNICODE"
+ val query = s"SELECT array_sort(array('b', null, 'a'));"
+ withSQLConf(SqlApiConf.DEFAULT_COLLATION -> collation) {
+ checkAnswer(
+ sql(query),
+ Seq(
+ Row(Seq("a", "b", null))
+ )
+ )
+ // check result row data type
+ val dataType = ArrayType(StringType(collation), true)
+ assert(sql(query).schema.head.dataType == dataType)
+ }
+ }
+
+ test("array_except supports collation") {
+ val collation = "UNICODE"
+ val query = s"SELECT array_except(array('a', 'b', 'c'), array('c', 'd',
'e'));"
+ withSQLConf(SqlApiConf.DEFAULT_COLLATION -> collation) {
+ checkAnswer(
+ sql(query),
+ Seq(
+ Row(Seq("a", "b"))
+ )
+ )
+ // check result row data type
+ val dataType = ArrayType(StringType(collation), false)
+ assert(sql(query).schema.head.dataType == dataType)
+ }
+ }
+
+ test("array_union supports collation") {
+ val collation = "UNICODE"
+ val query = s"SELECT array_union(array('a', 'b', 'c'), array('a', 'c',
'd'));"
+ withSQLConf(SqlApiConf.DEFAULT_COLLATION -> collation) {
+ checkAnswer(
+ sql(query),
+ Seq(
+ Row(Seq("a", "b", "c", "d"))
+ )
+ )
+ // check result row data type
+ val dataType = ArrayType(StringType(collation), false)
+ assert(sql(query).schema.head.dataType == dataType)
+ }
+ }
+
+ test("array_compact supports collation") {
+ val collation = "UNICODE"
+ val query = s"SELECT array_compact(array('a', 'b', null, 'c'));"
+ withSQLConf(SqlApiConf.DEFAULT_COLLATION -> collation) {
+ checkAnswer(
+ sql(query),
+ Seq(
+ Row(Seq("a", "b", "c"))
+ )
+ )
+ // check result row data type
+ val dataType = ArrayType(StringType(collation), true)
+ assert(sql(query).schema.head.dataType == dataType)
+ }
+ }
+
+ test("arrays_zip supports collation") {
+ val collation = "UNICODE"
+ val query = s"SELECT arrays_zip(array('a', 'b', 'c'), array(1, 2, 3));"
+ withSQLConf(SqlApiConf.DEFAULT_COLLATION -> collation) {
+ checkAnswer(
+ sql(query),
+ Seq(
+ Row(Seq(Row("a", 1), Row("b", 2), Row("c", 3)))
+ )
+ )
+ // check result row data type
+ val dataType = ArrayType(StructType(
+ StructField("0", StringType(collation), true) ::
+ StructField("1", IntegerType, true) :: Nil
+ ), false)
+ assert(sql(query).schema.head.dataType == dataType)
+ }
+ }
+
+ test("array_min supports collation") {
+ val collation = "UNICODE"
+ val query = s"SELECT array_min(array('a', 'b', null, 'c'));"
+ withSQLConf(SqlApiConf.DEFAULT_COLLATION -> collation) {
+ checkAnswer(
+ sql(query),
+ Seq(
+ Row("a")
+ )
+ )
+ // check result row data type
+ val dataType = StringType(collation)
+ assert(sql(query).schema.head.dataType == dataType)
+ }
+ }
+
+ test("array_max supports collation") {
+ val collation = "UNICODE"
+ val query = s"SELECT array_max(array('a', 'b', null, 'c'));"
+ withSQLConf(SqlApiConf.DEFAULT_COLLATION -> collation) {
+ checkAnswer(
+ sql(query),
+ Seq(
+ Row("c")
+ )
+ )
+ // check result row data type
+ val dataType = StringType(collation)
+ assert(sql(query).schema.head.dataType == dataType)
+ }
+ }
+
+ test("array_append supports collation") {
Review Comment:
Appending a string with different collation fails (default collation is
`UTF8_BINARY`):
```sql
SELECT array_append(array('b', 'd', 'c', 'a' COLLATE UTF8_LCASE), 'e')
```
while, `array_contains` passes:
```sql
SELECT array_contains(array('b', 'd', 'c', 'a' COLLATE UTF8_LCASE), 'e')
```
I'm a bit confused how implicit casting works here? @mihailom-db can you
provide some insights?
##########
sql/core/src/test/scala/org/apache/spark/sql/CollationSQLExpressionsSuite.scala:
##########
@@ -2295,6 +2295,827 @@ class CollationSQLExpressionsSuite
assert(typeException.getErrorClass ===
"DATATYPE_MISMATCH.UNEXPECTED_STATIC_METHOD")
}
+ test("min_by supports collation") {
+ val collation = "UNICODE"
+ val query = s"SELECT min_by(x, y) FROM VALUES ('a', 10), ('b', 50), ('c',
20) AS tab(x, y);"
+ withSQLConf(SqlApiConf.DEFAULT_COLLATION -> collation) {
+ checkAnswer(
+ sql(query),
+ Seq(
+ Row("a")
+ )
+ )
+ // check result row data type
+ val dataType = StringType(collation)
+ assert(sql(query).schema.head.dataType == dataType)
+ }
+ }
+
+ test("max_by supports collation") {
+ val collation = "UNICODE"
+ val query = s"SELECT max_by(x, y) FROM VALUES ('a', 10), ('b', 50), ('c',
20) AS tab(x, y);"
+ withSQLConf(SqlApiConf.DEFAULT_COLLATION -> collation) {
+ checkAnswer(
+ sql(query),
+ Seq(
+ Row("b")
+ )
+ )
+ // check result row data type
+ val dataType = StringType(collation)
+ assert(sql(query).schema.head.dataType == dataType)
+ }
+ }
+
+ test("array supports collation") {
+ val collation = "UNICODE"
+ val query = s"SELECT array('a', 'b', 'c');"
+ withSQLConf(SqlApiConf.DEFAULT_COLLATION -> collation) {
+ checkAnswer(
+ sql(query),
+ Seq(
+ Row(Seq("a", "b", "c"))
+ )
+ )
+ // check result row data type
+ val dataType = ArrayType(StringType(collation), false)
+ assert(sql(query).schema.head.dataType == dataType)
+ }
+ }
+
+ test("array_agg supports collation") {
+ val collation = "UNICODE"
+ val query = s"SELECT array_agg(col) FROM VALUES ('a'), ('b'), ('c') AS
tab(col);"
+ withSQLConf(SqlApiConf.DEFAULT_COLLATION -> collation) {
+ checkAnswer(
+ sql(query),
+ Seq(
+ Row(Seq("a", "b", "c"))
+ )
+ )
+ // check result row data type
+ val dataType = ArrayType(StringType(collation), false)
+ assert(sql(query).schema.head.dataType == dataType)
+ }
+ }
+
+ test("array_contains supports collation") {
+ val collation = "UNICODE"
+ val query = s"SELECT array_contains(array('a', 'b', 'c'), 'b');"
+ withSQLConf(SqlApiConf.DEFAULT_COLLATION -> collation) {
+ checkAnswer(
+ sql(query),
+ Seq(
+ Row(true)
+ )
+ )
+ // check result row data type
+ val dataType = BooleanType
+ assert(sql(query).schema.head.dataType == dataType)
+ }
+ }
+
+ test("arrays_overlap supports collation") {
+ val collation = "UNICODE"
+ val query = s"SELECT arrays_overlap(array('a', 'b', 'c'), array('c', 'd',
'e'));"
+ withSQLConf(SqlApiConf.DEFAULT_COLLATION -> collation) {
+ checkAnswer(
+ sql(query),
+ Seq(
+ Row(true)
+ )
+ )
+ // check result row data type
+ val dataType = BooleanType
+ assert(sql(query).schema.head.dataType == dataType)
+ }
+ }
+
+ test("array_insert supports collation") {
+ val collation = "UNICODE"
+ val query = s"SELECT array_insert(array('a', 'b', 'c', 'd'), 5, 'e');"
+ withSQLConf(SqlApiConf.DEFAULT_COLLATION -> collation) {
+ checkAnswer(
+ sql(query),
+ Seq(
+ Row(Seq("a", "b", "c", "d", "e"))
+ )
+ )
+ // check result row data type
+ val dataType = ArrayType(StringType(collation), true)
+ assert(sql(query).schema.head.dataType == dataType)
+ }
+ }
+
+ test("array_intersect supports collation") {
+ val collation = "UNICODE"
+ val query = s"SELECT array_intersect(array('a', 'b', 'c'), array('b', 'c',
'd'));"
+ withSQLConf(SqlApiConf.DEFAULT_COLLATION -> collation) {
+ checkAnswer(
+ sql(query),
+ Seq(
+ Row(Seq("b", "c"))
+ )
+ )
+ // check result row data type
+ val dataType = ArrayType(StringType(collation), false)
+ assert(sql(query).schema.head.dataType == dataType)
+ }
+ }
+
+ test("array_join supports collation") {
+ val collation = "UNICODE"
+ val query = s"SELECT array_join(array('hello', 'world'), ' ');"
+ withSQLConf(SqlApiConf.DEFAULT_COLLATION -> collation) {
+ checkAnswer(
+ sql(query),
+ Seq(
+ Row("hello world")
+ )
+ )
+ // check result row data type
+ val dataType = StringType(collation)
+ assert(sql(query).schema.head.dataType == dataType)
+ }
+ }
+
+ test("array_position supports collation") {
+ val collation = "UNICODE"
+ val query = s"SELECT array_position(array('a', 'b', 'c', 'c'), 'c');"
+ withSQLConf(SqlApiConf.DEFAULT_COLLATION -> collation) {
+ checkAnswer(
+ sql(query),
+ Seq(
+ Row(3)
+ )
+ )
+ // check result row data type
+ val dataType = LongType
+ assert(sql(query).schema.head.dataType == dataType)
+ }
+ }
+
+ test("array_size supports collation") {
+ val collation = "UNICODE"
+ val query = s"SELECT array_size(array('a', 'b', 'c', 'c'));"
+ withSQLConf(SqlApiConf.DEFAULT_COLLATION -> collation) {
+ checkAnswer(
+ sql(query),
+ Seq(
+ Row(4)
+ )
+ )
+ // check result row data type
+ val dataType = IntegerType
+ assert(sql(query).schema.head.dataType == dataType)
+ }
+ }
+
+ test("array_sort supports collation") {
+ val collation = "UNICODE"
+ val query = s"SELECT array_sort(array('b', null, 'a'));"
+ withSQLConf(SqlApiConf.DEFAULT_COLLATION -> collation) {
+ checkAnswer(
+ sql(query),
+ Seq(
+ Row(Seq("a", "b", null))
+ )
+ )
+ // check result row data type
+ val dataType = ArrayType(StringType(collation), true)
+ assert(sql(query).schema.head.dataType == dataType)
+ }
+ }
+
+ test("array_except supports collation") {
+ val collation = "UNICODE"
+ val query = s"SELECT array_except(array('a', 'b', 'c'), array('c', 'd',
'e'));"
+ withSQLConf(SqlApiConf.DEFAULT_COLLATION -> collation) {
+ checkAnswer(
+ sql(query),
+ Seq(
+ Row(Seq("a", "b"))
+ )
+ )
+ // check result row data type
+ val dataType = ArrayType(StringType(collation), false)
+ assert(sql(query).schema.head.dataType == dataType)
+ }
+ }
+
+ test("array_union supports collation") {
+ val collation = "UNICODE"
+ val query = s"SELECT array_union(array('a', 'b', 'c'), array('a', 'c',
'd'));"
+ withSQLConf(SqlApiConf.DEFAULT_COLLATION -> collation) {
+ checkAnswer(
+ sql(query),
+ Seq(
+ Row(Seq("a", "b", "c", "d"))
+ )
+ )
+ // check result row data type
+ val dataType = ArrayType(StringType(collation), false)
+ assert(sql(query).schema.head.dataType == dataType)
+ }
+ }
+
+ test("array_compact supports collation") {
+ val collation = "UNICODE"
+ val query = s"SELECT array_compact(array('a', 'b', null, 'c'));"
+ withSQLConf(SqlApiConf.DEFAULT_COLLATION -> collation) {
+ checkAnswer(
+ sql(query),
+ Seq(
+ Row(Seq("a", "b", "c"))
+ )
+ )
+ // check result row data type
+ val dataType = ArrayType(StringType(collation), true)
+ assert(sql(query).schema.head.dataType == dataType)
+ }
+ }
+
+ test("arrays_zip supports collation") {
+ val collation = "UNICODE"
+ val query = s"SELECT arrays_zip(array('a', 'b', 'c'), array(1, 2, 3));"
+ withSQLConf(SqlApiConf.DEFAULT_COLLATION -> collation) {
+ checkAnswer(
+ sql(query),
+ Seq(
+ Row(Seq(Row("a", 1), Row("b", 2), Row("c", 3)))
+ )
+ )
+ // check result row data type
+ val dataType = ArrayType(StructType(
+ StructField("0", StringType(collation), true) ::
+ StructField("1", IntegerType, true) :: Nil
+ ), false)
+ assert(sql(query).schema.head.dataType == dataType)
+ }
+ }
+
+ test("array_min supports collation") {
+ val collation = "UNICODE"
+ val query = s"SELECT array_min(array('a', 'b', null, 'c'));"
+ withSQLConf(SqlApiConf.DEFAULT_COLLATION -> collation) {
+ checkAnswer(
+ sql(query),
+ Seq(
+ Row("a")
+ )
+ )
+ // check result row data type
+ val dataType = StringType(collation)
+ assert(sql(query).schema.head.dataType == dataType)
+ }
+ }
+
+ test("array_max supports collation") {
+ val collation = "UNICODE"
+ val query = s"SELECT array_max(array('a', 'b', null, 'c'));"
+ withSQLConf(SqlApiConf.DEFAULT_COLLATION -> collation) {
+ checkAnswer(
+ sql(query),
+ Seq(
+ Row("c")
+ )
+ )
+ // check result row data type
+ val dataType = StringType(collation)
+ assert(sql(query).schema.head.dataType == dataType)
+ }
+ }
+
+ test("array_append supports collation") {
+ val collation = "UNICODE"
+ val query = s"SELECT array_append(array('b', 'd', 'c', 'a'), 'e');"
+ withSQLConf(SqlApiConf.DEFAULT_COLLATION -> collation) {
+ checkAnswer(
+ sql(query),
+ Seq(
+ Row(Seq("b", "d", "c", "a", "e"))
+ )
+ )
+ // check result row data type
+ val dataType = ArrayType(StringType(collation), true)
+ assert(sql(query).schema.head.dataType == dataType)
+ }
+ }
+
+ test("array_repeat supports collation") {
+ val collation = "UNICODE"
+ val query = s"SELECT array_repeat('abc', 2);"
+ withSQLConf(SqlApiConf.DEFAULT_COLLATION -> collation) {
+ checkAnswer(
+ sql(query),
+ Seq(
+ Row(Seq("abc", "abc"))
+ )
+ )
+ // check result row data type
+ val dataType = ArrayType(StringType(collation), false)
+ assert(sql(query).schema.head.dataType == dataType)
+ }
+ }
+
+ test("array_remove supports collation") {
+ val collation = "UNICODE"
+ val query = s"SELECT array_remove(array('a', 'b', null, 'c'), 'b');"
+ withSQLConf(SqlApiConf.DEFAULT_COLLATION -> collation) {
+ checkAnswer(
+ sql(query),
+ Seq(
+ Row(Seq("a", null, "c"))
+ )
+ )
+ // check result row data type
+ val dataType = ArrayType(StringType(collation), true)
+ assert(sql(query).schema.head.dataType == dataType)
+ }
+ }
+
+ test("array_prepend supports collation") {
+ val collation = "UNICODE"
+ val query = s"SELECT array_prepend(array('b', 'd', 'c', 'a'), 'd');"
+ withSQLConf(SqlApiConf.DEFAULT_COLLATION -> collation) {
+ checkAnswer(
+ sql(query),
+ Seq(
+ Row(Seq("d", "b", "d", "c", "a"))
+ )
+ )
+ // check result row data type
+ val dataType = ArrayType(StringType(collation), true)
+ assert(sql(query).schema.head.dataType == dataType)
+ }
+ }
+
+ test("array_distinct supports collation") {
+ val collation = "UNICODE"
+ val query = s"SELECT array_distinct(array('a', 'b', 'c', null, 'c'));"
+ withSQLConf(SqlApiConf.DEFAULT_COLLATION -> collation) {
+ checkAnswer(
+ sql(query),
+ Seq(
+ Row(Seq("a", "b", "c", null))
+ )
+ )
+ // check result row data type
+ val dataType = ArrayType(StringType(collation), true)
+ assert(sql(query).schema.head.dataType == dataType)
+ }
+ }
+
+ test("collect_list supports collation") {
+ val collation = "UNICODE"
+ val query = s"SELECT collect_list(col) FROM VALUES ('a'), ('b'), ('c') AS
tab(col);"
+ withSQLConf(SqlApiConf.DEFAULT_COLLATION -> collation) {
+ checkAnswer(
+ sql(query),
+ Seq(
+ Row(Seq("a", "b", "c"))
+ )
+ )
+ // check result row data type
+ val dataType = ArrayType(StringType(collation), false)
+ assert(sql(query).schema.head.dataType == dataType)
+ }
+ }
+
+ test("collect_set supports collation") {
+ val collation = "UNICODE"
+ val query = s"SELECT collect_set(col) FROM VALUES ('a'), ('b'), ('a') AS
tab(col);"
+ withSQLConf(SqlApiConf.DEFAULT_COLLATION -> collation) {
+ checkAnswer(
+ sql(query),
+ Seq(
+ Row(Seq("a", "b"))
+ )
+ )
+ // check result row data type
+ val dataType = ArrayType(StringType(collation), false)
+ assert(sql(query).schema.head.dataType == dataType)
+ }
+ }
+
+ test("element_at supports collation") {
+ val collation = "UNICODE"
+ val query = s"SELECT element_at(array('a', 'b', 'c'), 2);"
+ withSQLConf(SqlApiConf.DEFAULT_COLLATION -> collation) {
+ checkAnswer(
+ sql(query),
+ Seq(
+ Row("b")
+ )
+ )
+ // check result row data type
+ val dataType = StringType(collation)
+ assert(sql(query).schema.head.dataType == dataType)
+ }
+ }
+
+ test("aggregate supports collation") {
+ val collation = "UNICODE"
+ val query = s"SELECT aggregate(array('a', 'b', 'c'), '', (acc, x) ->
concat(acc, x));"
+ withSQLConf(SqlApiConf.DEFAULT_COLLATION -> collation) {
+ checkAnswer(
+ sql(query),
+ Seq(
+ Row("abc")
+ )
+ )
+ // check result row data type
+ val dataType = StringType(collation)
+ assert(sql(query).schema.head.dataType == dataType)
+ }
+ }
+
+ test("explode supports collation") {
+ val collation = "UNICODE"
+ val query = s"SELECT explode(array('a', 'b'));"
+ withSQLConf(SqlApiConf.DEFAULT_COLLATION -> collation) {
+ checkAnswer(
+ sql(query),
+ Seq(
+ Row("a"),
+ Row("b")
+ )
+ )
+ // check result row data type
+ val dataType = StringType(collation)
+ assert(sql(query).schema.head.dataType == dataType)
+ }
+ }
+
+ test("posexplode supports collation") {
+ val collation = "UNICODE"
+ val query = s"SELECT posexplode(array('a', 'b'));"
+ withSQLConf(SqlApiConf.DEFAULT_COLLATION -> collation) {
+ checkAnswer(
+ sql(query),
+ Seq(
+ Row(0, "a"),
+ Row(1, "b")
+ )
+ )
+ // check result row data type
+ val dataType = StringType(collation)
+ assert(sql(query).schema.head.dataType == IntegerType)
+ assert(sql(query).schema(1).dataType == dataType)
+ }
+ }
+
+ test("filter supports collation") {
+ val collation = "UNICODE"
+ val query = s"SELECT filter(array('a', 'b', 'c'), x -> x < 'b');"
+ withSQLConf(SqlApiConf.DEFAULT_COLLATION -> collation) {
+ checkAnswer(
+ sql(query),
+ Seq(
+ Row(Seq("a"))
+ )
+ )
+ // check result row data type
+ val dataType = ArrayType(StringType(collation), false)
+ assert(sql(query).schema.head.dataType == dataType)
+ }
+ }
+
+ test("flatten supports collation") {
+ val collation = "UNICODE"
+ val query = s"SELECT flatten(array(array('a', 'b'), array('c', 'd')));"
+ withSQLConf(SqlApiConf.DEFAULT_COLLATION -> collation) {
+ checkAnswer(
+ sql(query),
+ Seq(
+ Row(Seq("a", "b", "c", "d"))
+ )
+ )
+ // check result row data type
+ val dataType = ArrayType(StringType(collation), false)
+ assert(sql(query).schema.head.dataType == dataType)
+ }
+ }
+
+ test("inline supports collation") {
+ val collation = "UNICODE"
+ val query = s"SELECT inline(array(struct(1, 'a'), struct(2, 'b')));"
+ withSQLConf(SqlApiConf.DEFAULT_COLLATION -> collation) {
+ checkAnswer(
+ sql(query),
+ Seq(
+ Row(1, "a"),
+ Row(2, "b")
+ )
+ )
+ // check result row data type
+ val dataType = StringType(collation)
+ assert(sql(query).schema.head.dataType == IntegerType)
+ assert(sql(query).schema(1).dataType == dataType)
+ }
+ }
+
+ test("shuffle supports collation") {
+ val collation = "UNICODE"
+ val query = s"SELECT shuffle(array('a', 'b', 'c', 'd'));"
+ withSQLConf(SqlApiConf.DEFAULT_COLLATION -> collation) {
+ // check result row data type
+ val dataType = ArrayType(StringType(collation), false)
+ assert(sql(query).schema.head.dataType == dataType)
+ }
+ }
+
+ test("slice supports collation") {
+ val collation = "UNICODE"
+ val query = s"SELECT slice(array('a', 'b', 'c', 'd'), 2, 2);"
+ withSQLConf(SqlApiConf.DEFAULT_COLLATION -> collation) {
+ checkAnswer(
+ sql(query),
+ Seq(
+ Row(Seq("b", "c"))
+ )
+ )
+ // check result row data type
+ val dataType = ArrayType(StringType(collation), false)
+ assert(sql(query).schema.head.dataType == dataType)
+ }
+ }
+
+ test("sort_array supports collation") {
+ val collation = "UNICODE"
+ val query = s"SELECT sort_array(array('b', 'd', null, 'c', 'a'), true);"
+ withSQLConf(SqlApiConf.DEFAULT_COLLATION -> collation) {
+ checkAnswer(
+ sql(query),
+ Seq(
+ Row(Seq(null, "a", "b", "c", "d"))
+ )
+ )
+ // check result row data type
+ val dataType = ArrayType(StringType(collation), true)
+ assert(sql(query).schema.head.dataType == dataType)
+ }
+ }
+
+ test("zip_with supports collation") {
+ val collation = "UNICODE"
+ val query = s"SELECT zip_with(array('a', 'b'), array('x', 'y'), (x, y) ->
concat(x, y));"
+ withSQLConf(SqlApiConf.DEFAULT_COLLATION -> collation) {
+ checkAnswer(
+ sql(query),
+ Seq(
+ Row(Seq("ax", "by"))
+ )
+ )
+ // check result row data type
+ val dataType = ArrayType(
+ StringType(collation),
+ containsNull = true
+ )
+ assert(sql(query).schema.head.dataType == dataType)
+ }
+ }
+
+ test("map_contains_key supports collation") {
+ val collation = "UNICODE"
+ val query = s"SELECT map_contains_key(map('a', 1, 'b', 2), 'a')"
+ withSQLConf(SqlApiConf.DEFAULT_COLLATION -> collation) {
+ checkAnswer(
+ sql(query),
+ Seq(
+ Row(true)
+ )
+ )
+ // check result row data type
+ val dataType = BooleanType
+ assert(sql(query).schema.head.dataType == dataType)
+ }
+ }
+
+ test("map_from_arrays supports collation") {
+ val collation = "UNICODE"
+ val query = s"SELECT map_from_arrays(array('a','b','c'), array(1,2,3))"
+ withSQLConf(SqlApiConf.DEFAULT_COLLATION -> collation) {
+ checkAnswer(
+ sql(query),
+ Seq(
+ Row(Map("a" -> 1, "b" -> 2, "c" -> 3))
+ )
+ )
+ // check result row data type
+ val dataType = MapType(
+ StringType(collation),
+ IntegerType, false
+ )
+ assert(sql(query).schema.head.dataType == dataType)
+ }
+ }
+
+ test("map_keys supports collation") {
+ val collation = "UNICODE"
+ val query = s"SELECT map_keys(map('a', 1, 'b', 2))"
+ withSQLConf(SqlApiConf.DEFAULT_COLLATION -> collation) {
+ checkAnswer(
+ sql(query),
+ Seq(
+ Row(Seq("a", "b"))
+ )
+ )
+ // check result row data type
+ val dataType = ArrayType(
+ StringType(collation), true
+ )
+ assert(sql(query).schema.head.dataType == dataType)
+ }
+ }
+
+ test("map_values supports collation") {
+ val collation = "UNICODE"
+ val query = s"SELECT map_values(map(1, 'a', 2, 'b'))"
+ withSQLConf(SqlApiConf.DEFAULT_COLLATION -> collation) {
+ checkAnswer(
+ sql(query),
+ Seq(
+ Row(Seq("a", "b"))
+ )
+ )
+ // check result row data type
+ val dataType = ArrayType(
+ StringType(collation), true
+ )
+ assert(sql(query).schema.head.dataType == dataType)
+ }
+ }
+
+ test("map_entries supports collation") {
+ val collation = "UNICODE"
+ val query = s"SELECT map_entries(map('a', 1, 'b', 2))"
+ withSQLConf(SqlApiConf.DEFAULT_COLLATION -> collation) {
+ checkAnswer(
+ sql(query),
+ Seq(
+ Row(Seq(Row("a", 1), Row("b", 2)))
+ )
+ )
+ // check result row data type
+ val dataType = ArrayType(StructType(
+ StructField("key", StringType(collation), false) ::
+ StructField("value", IntegerType, false) :: Nil
+ ), false)
+ assert(sql(query).schema.head.dataType == dataType)
+ }
+ }
+
+ test("map_from_entries supports collation") {
+ val collation = "UNICODE"
+ val query = s"SELECT map_from_entries(array(struct(1, 'a'), struct(2,
'b')))"
+ withSQLConf(SqlApiConf.DEFAULT_COLLATION -> collation) {
+ checkAnswer(
+ sql(query),
+ Seq(
+ Row(Map(1 -> "a", 2 -> "b"))
+ )
+ )
+ // check result row data type
+ val dataType = MapType(
+ IntegerType,
+ StringType(collation),
+ valueContainsNull = false
+ )
+ assert(sql(query).schema.head.dataType == dataType)
+ }
+ }
+
+ test("map_concat supports collation") {
+ val collation = "UNICODE"
+ val query = s"SELECT map_concat(map(1, 'a'), map(2, 'b'))"
+ withSQLConf(SqlApiConf.DEFAULT_COLLATION -> collation) {
+ checkAnswer(
+ sql(query),
+ Seq(
+ Row(Map(1 -> "a", 2 -> "b"))
+ )
+ )
+ // check result row data type
+ val dataType = MapType(
+ IntegerType,
+ StringType(collation),
+ valueContainsNull = false
+ )
+ assert(sql(query).schema.head.dataType == dataType)
+ }
+ }
+
+ test("map_filter supports collation") {
+ val collation = "UNICODE"
+ val query = s"SELECT map_filter(map('a', 1, 'b', 2, 'c', 3), (k, v) -> k <
'c')"
+ withSQLConf(SqlApiConf.DEFAULT_COLLATION -> collation) {
+ checkAnswer(
+ sql(query),
+ Seq(
+ Row(Map("a" -> 1, "b" -> 2))
+ )
+ )
+ // check result row data type
+ val dataType = MapType(
+ StringType(collation),
+ IntegerType,
+ valueContainsNull = false
+ )
+ assert(sql(query).schema.head.dataType == dataType)
+ }
+ }
+
+ test("map_zip_with supports collation") {
+ val collation = "UNICODE"
+ val query = s"SELECT map_zip_with(map(1, 'a'), map(1, 'x'), (k, v1, v2) ->
concat(v1, v2))"
+ withSQLConf(SqlApiConf.DEFAULT_COLLATION -> collation) {
+ checkAnswer(
+ sql(query),
+ Seq(
+ Row(Map(1 -> "ax"))
+ )
+ )
+ // check result row data type
+ val dataType = MapType(
+ IntegerType,
+ StringType(collation),
+ valueContainsNull = true
+ )
+ assert(sql(query).schema.head.dataType == dataType)
+ }
+ }
+
+ test("transform supports collation") {
+ val collation = "UNICODE"
+ val query = s"SELECT transform(array('aa', 'bb', 'cc'), x -> substring(x,
2))"
+ withSQLConf(SqlApiConf.DEFAULT_COLLATION -> collation) {
+ checkAnswer(
+ sql(query),
+ Seq(
+ Row(Seq("a", "b", "c"))
+ )
+ )
+ // check result row data type
+ val dataType = ArrayType(StringType(collation), false)
+ assert(sql(query).schema.head.dataType == dataType)
+ }
+ }
+
+ test("transform_values supports collation") {
+ val collation = "UNICODE"
+ val query = s"SELECT transform_values(map_from_arrays(array(1, 2, 3)," +
+ s"array('aa', 'bb', 'cc')), (k, v) -> substring(v, 2))"
+ withSQLConf(SqlApiConf.DEFAULT_COLLATION -> collation) {
+ checkAnswer(
+ sql(query),
+ Seq(
+ Row(Map(1 -> "a", 2 -> "b", 3 -> "c"))
+ )
+ )
+ // check result row data type
+ val dataType = MapType(IntegerType,
+ StringType(collation), false)
+ assert(sql(query).schema.head.dataType == dataType)
+ }
+ }
+
+ test("transform_keys supports collation") {
+ val collation = "UNICODE"
+ val query = s"SELECT transform_keys(map_from_arrays(array('aa', 'bb',
'cc')," +
+ s"array(1, 2, 3)), (k, v) -> substring(k, 2))"
+ withSQLConf(SqlApiConf.DEFAULT_COLLATION -> collation) {
+ checkAnswer(
+ sql(query),
+ Seq(
+ Row(Map("a" -> 1, "b" -> 2, "c" -> 3))
+ )
+ )
+ // check result row data type
+ val dataType = MapType(
+ StringType(collation), IntegerType, false
+ )
+ assert(sql(query).schema.head.dataType == dataType)
+ }
+ }
+
+ test("stack supports collation") {
+ val query = s"SELECT stack(2, 'a', 'b', 'c')"
+ val collation = "UNICODE"
+ withSQLConf(SqlApiConf.DEFAULT_COLLATION -> collation) {
+ checkAnswer(
+ sql(query),
+ Seq(
+ Row("a", "b"),
+ Row("c", null)
+ )
+ )
+ // check result row data type
+ val dataType = StringType(collation)
+ assert(sql(query).schema.head.dataType == dataType)
+ }
+ }
+
Review Comment:
Creating a map with keys that have different collation passes (second key is
not implicitly `UTF8_LCASE` but rather what the default collation is):
```sql
select map('a' COLLATE UTF8_LCASE, 'b', 'c', 'c')
```
The resulting collation appears to be whatever the last key is (this
statement should be verified). In the example above, the collation for keys is
`UTF8_BINARY`.
q. Should we prevent creating a map with different collations as keys or is
this something that we are okay with?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
