This is an automated email from the ASF dual-hosted git repository.
wenchen pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 6a17b794338b [SPARK-48305][SQL] Add collation support for CurrentLike
expressions
6a17b794338b is described below
commit 6a17b794338b0473c11ae17e5c8f1450c0b3f358
Author: Uros Bojanic <157381213+uros...@users.noreply.github.com>
AuthorDate: Mon May 20 15:51:23 2024 +0800
[SPARK-48305][SQL] Add collation support for CurrentLike expressions
### What changes were proposed in this pull request?
Introduce collation awareness for CurrentLike expressions:
current_database/current_schema, current_catalog,
user/current_user/session_user.
### Why are the changes needed?
Add collation support for CurrentLike expressions in Spark.
### Does this PR introduce _any_ user-facing change?
Yes, users should now be able to use collated strings within arguments for
CurrentLike functions: current_database/current_schema, current_catalog,
user/current_user/session_user.
### How was this patch tested?
E2e sql tests.
### Was this patch authored or co-authored using generative AI tooling?
No.
Closes #46613 from uros-db/current-like-expressions.
Authored-by: Uros Bojanic <157381213+uros...@users.noreply.github.com>
Signed-off-by: Wenchen Fan <wenc...@databricks.com>
---
.../org/apache/spark/sql/catalyst/expressions/misc.scala | 6 +++---
.../spark/sql/catalyst/optimizer/finishAnalysis.scala | 7 ++++---
.../apache/spark/sql/CollationSQLExpressionsSuite.scala | 16 ++++++++++++++++
3 files changed, 23 insertions(+), 6 deletions(-)
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/misc.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/misc.scala
index eda65ae48f00..e9fa362de14c 100644
---
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/misc.scala
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/misc.scala
@@ -200,7 +200,7 @@ object AssertTrue {
since = "1.6.0",
group = "misc_funcs")
case class CurrentDatabase() extends LeafExpression with Unevaluable {
- override def dataType: DataType = StringType
+ override def dataType: DataType = SQLConf.get.defaultStringType
override def nullable: Boolean = false
override def prettyName: String = "current_schema"
final override val nodePatterns: Seq[TreePattern] = Seq(CURRENT_LIKE)
@@ -219,7 +219,7 @@ case class CurrentDatabase() extends LeafExpression with
Unevaluable {
since = "3.1.0",
group = "misc_funcs")
case class CurrentCatalog() extends LeafExpression with Unevaluable {
- override def dataType: DataType = StringType
+ override def dataType: DataType = SQLConf.get.defaultStringType
override def nullable: Boolean = false
override def prettyName: String = "current_catalog"
final override val nodePatterns: Seq[TreePattern] = Seq(CURRENT_LIKE)
@@ -335,7 +335,7 @@ case class TypeOf(child: Expression) extends
UnaryExpression {
// scalastyle:on line.size.limit
case class CurrentUser() extends LeafExpression with Unevaluable {
override def nullable: Boolean = false
- override def dataType: DataType = StringType
+ override def dataType: DataType = SQLConf.get.defaultStringType
override def prettyName: String =
getTagValue(FunctionRegistry.FUNC_ALIAS).getOrElse("current_user")
final override val nodePatterns: Seq[TreePattern] = Seq(CURRENT_LIKE)
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/finishAnalysis.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/finishAnalysis.scala
index 92ac7599a8ff..48753fbfe326 100644
---
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/finishAnalysis.scala
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/finishAnalysis.scala
@@ -33,6 +33,7 @@ import org.apache.spark.sql.catalyst.util.DateTimeUtils
import org.apache.spark.sql.catalyst.util.DateTimeUtils.{convertSpecialDate,
convertSpecialTimestamp, convertSpecialTimestampNTZ, instantToMicros,
localDateTimeToMicros}
import org.apache.spark.sql.catalyst.util.TypeUtils.toSQLExpr
import org.apache.spark.sql.connector.catalog.CatalogManager
+import org.apache.spark.sql.internal.SQLConf
import org.apache.spark.sql.types._
@@ -151,11 +152,11 @@ case class ReplaceCurrentLike(catalogManager:
CatalogManager) extends Rule[Logic
plan.transformAllExpressionsWithPruning(_.containsPattern(CURRENT_LIKE)) {
case CurrentDatabase() =>
- Literal.create(currentNamespace, StringType)
+ Literal.create(currentNamespace, SQLConf.get.defaultStringType)
case CurrentCatalog() =>
- Literal.create(currentCatalog, StringType)
+ Literal.create(currentCatalog, SQLConf.get.defaultStringType)
case CurrentUser() =>
- Literal.create(currentUser, StringType)
+ Literal.create(currentUser, SQLConf.get.defaultStringType)
}
}
}
diff --git
a/sql/core/src/test/scala/org/apache/spark/sql/CollationSQLExpressionsSuite.scala
b/sql/core/src/test/scala/org/apache/spark/sql/CollationSQLExpressionsSuite.scala
index 37dcdf9bd721..0d48f9f0a88d 100644
---
a/sql/core/src/test/scala/org/apache/spark/sql/CollationSQLExpressionsSuite.scala
+++
b/sql/core/src/test/scala/org/apache/spark/sql/CollationSQLExpressionsSuite.scala
@@ -997,6 +997,22 @@ class CollationSQLExpressionsSuite
})
}
+ test("Support CurrentDatabase/Catalog/User expressions with collation") {
+ // Supported collations
+ Seq("UTF8_BINARY_LCASE", "UNICODE", "UNICODE_CI").foreach(collationName =>
+ withSQLConf(SqlApiConf.DEFAULT_COLLATION -> collationName) {
+ val queryDatabase = sql("SELECT current_schema()")
+ val queryCatalog = sql("SELECT current_catalog()")
+ val queryUser = sql("SELECT current_user()")
+ // Data type
+ val dataType = StringType(collationName)
+ assert(queryDatabase.schema.fields.head.dataType.sameType(dataType))
+ assert(queryCatalog.schema.fields.head.dataType.sameType(dataType))
+ assert(queryUser.schema.fields.head.dataType.sameType(dataType))
+ }
+ )
+ }
+
test("Support Uuid misc expression with collation") {
// Supported collations
Seq("UTF8_BINARY_LCASE", "UNICODE", "UNICODE_CI").foreach(collationName =>
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org
