This is an automated email from the ASF dual-hosted git repository.
maxgekk pushed a commit to branch branch-3.4
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-3.4 by this push:
new 033eb37f12e [SPARK-42516][SQL] Always capture the session time zone
config while creating views
033eb37f12e is described below
commit 033eb37f12e3753ea22eb9a069f716981cc58be1
Author: Max Gekk <max.g...@gmail.com>
AuthorDate: Wed Feb 22 14:03:20 2023 +0300
[SPARK-42516][SQL] Always capture the session time zone config while
creating views
### What changes were proposed in this pull request?
In the PR, I propose to capture the session time zone config
(`spark.sql.session.timeZone`) as a view property, and use it while
re-parsing/analysing the view. If the SQL config is not set while creating a
view, use the default value of the config.
### Why are the changes needed?
To improve user experience with Spark SQL. The current behaviour might
confuse users because query results depends on whether or not the session time
zone was set explicitly while creating a view.
### Does this PR introduce _any_ user-facing change?
Yes. Before the changes, the current value of the session time zone is used
in view analysis but this behaviour can be restored via another SQL config
`spark.sql.legacy.useCurrentConfigsForView`.
### How was this patch tested?
By running the new test via:
```
$ build/sbt "test:testOnly *.PersistedViewTestSuite"
```
Closes #40103 from MaxGekk/view-tz-conf.
Authored-by: Max Gekk <max.g...@gmail.com>
Signed-off-by: Max Gekk <max.g...@gmail.com>
(cherry picked from commit 00e56905f77955f67e3809d724b33aebcc79cb5e)
Signed-off-by: Max Gekk <max.g...@gmail.com>
---
.../org/apache/spark/sql/execution/command/views.scala | 9 ++++++++-
.../apache/spark/sql/execution/SQLViewTestSuite.scala | 17 +++++++++++++++++
2 files changed, 25 insertions(+), 1 deletion(-)
diff --git
a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/views.scala
b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/views.scala
index 3ad98fa0d0c..f998e134a0a 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/views.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/views.scala
@@ -398,8 +398,15 @@ object ViewHelper extends SQLConfHelper with Logging {
val modifiedConfs = conf.getAllConfs.filter { case (k, _) =>
conf.isModifiable(k) && shouldCaptureConfig(k)
}
+ // Some configs have dynamic default values, such as
SESSION_LOCAL_TIMEZONE whose
+ // default value relies on the JVM system timezone. We need to always
capture them to
+ // to make sure we apply the same configs when reading the view.
+ val alwaysCaptured = Seq(SQLConf.SESSION_LOCAL_TIMEZONE)
+ .filter(c => !modifiedConfs.contains(c.key))
+ .map(c => (c.key, conf.getConf(c)))
+
val props = new mutable.HashMap[String, String]
- for ((key, value) <- modifiedConfs) {
+ for ((key, value) <- modifiedConfs ++ alwaysCaptured) {
props.put(s"$VIEW_SQL_CONFIG_PREFIX$key", value)
}
props.toMap
diff --git
a/sql/core/src/test/scala/org/apache/spark/sql/execution/SQLViewTestSuite.scala
b/sql/core/src/test/scala/org/apache/spark/sql/execution/SQLViewTestSuite.scala
index 1d4c52d3ae5..592f1c2607d 100644
---
a/sql/core/src/test/scala/org/apache/spark/sql/execution/SQLViewTestSuite.scala
+++
b/sql/core/src/test/scala/org/apache/spark/sql/execution/SQLViewTestSuite.scala
@@ -24,6 +24,7 @@ import org.apache.spark.sql.catalyst.{FunctionIdentifier,
TableIdentifier}
import org.apache.spark.sql.catalyst.catalog.CatalogFunction
import org.apache.spark.sql.catalyst.expressions.Expression
import org.apache.spark.sql.catalyst.plans.logical.Repartition
+import org.apache.spark.sql.catalyst.util.DateTimeTestUtils.withDefaultTimeZone
import org.apache.spark.sql.connector.catalog._
import
org.apache.spark.sql.connector.catalog.CatalogManager.SESSION_CATALOG_NAME
import org.apache.spark.sql.internal.SQLConf._
@@ -706,6 +707,22 @@ class PersistedViewTestSuite extends SQLViewTestSuite with
SharedSparkSession {
}
}
+ test("capture the session time zone config while creating a view") {
+ val viewName = "v1_capture_test"
+ withView(viewName) {
+ assert(get.sessionLocalTimeZone === "America/Los_Angeles")
+ createView(viewName,
+ """select hour(ts) as H from (
+ | select cast('2022-01-01T00:00:00.000 America/Los_Angeles' as
timestamp) as ts
+ |)""".stripMargin, Seq("H"))
+ withDefaultTimeZone(java.time.ZoneId.of("UTC-09:00")) {
+ withSQLConf(SESSION_LOCAL_TIMEZONE.key -> "UTC-10:00") {
+ checkAnswer(sql(s"select H from $viewName"), Row(0))
+ }
+ }
+ }
+ }
+
def getShowCreateDDL(view: String, serde: Boolean = false): String = {
val result = if (serde) {
sql(s"SHOW CREATE TABLE $view AS SERDE")
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org
