This is an automated email from the ASF dual-hosted git repository.
liuneng pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/main by this push:
new 7a0af446d [GLUTEN-5841][CH]Fix session timezone diff (#5892)
7a0af446d is described below
commit 7a0af446ddcf6e78b1ad5d7e58ba45729fc3e4a1
Author: KevinyhZou <[email protected]>
AuthorDate: Thu Jun 6 10:38:10 2024 +0800
[GLUTEN-5841][CH]Fix session timezone diff (#5892)
What changes were proposed in this pull request?
(Please fill in changes proposed in this fix)
(Fixes: #5841)
How was this patch tested?
TEST BY UT
---
.../execution/GlutenClickHouseTPCHSaltNullParquetSuite.scala | 6 ++++++
cpp-ch/local-engine/Common/CHUtil.cpp | 10 ++++++++++
cpp-ch/local-engine/Common/CHUtil.h | 1 +
.../common/src/main/scala/org/apache/gluten/GlutenConfig.scala | 4 ++++
4 files changed, 21 insertions(+)
diff --git
a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseTPCHSaltNullParquetSuite.scala
b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseTPCHSaltNullParquetSuite.scala
index d2752a073..ada980a20 100644
---
a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseTPCHSaltNullParquetSuite.scala
+++
b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseTPCHSaltNullParquetSuite.scala
@@ -719,6 +719,12 @@ class GlutenClickHouseTPCHSaltNullParquetSuite extends
GlutenClickHouseTPCHAbstr
"select unix_timestamp(concat(cast(l_shipdate as String), ' 00:00:00'))
" +
"from lineitem order by l_shipdate limit 10;")(
checkGlutenOperatorMatch[ProjectExecTransformer])
+ withSQLConf(SQLConf.SESSION_LOCAL_TIMEZONE.key -> "UTC") {
+ runQueryAndCompare(
+ "select to_unix_timestamp(concat(cast(l_shipdate as String), '
00:00:00')) " +
+ "from lineitem order by l_shipdate limit 10")(
+ checkGlutenOperatorMatch[ProjectExecTransformer])
+ }
}
test("test literals") {
diff --git a/cpp-ch/local-engine/Common/CHUtil.cpp
b/cpp-ch/local-engine/Common/CHUtil.cpp
index e37114756..62b42f981 100644
--- a/cpp-ch/local-engine/Common/CHUtil.cpp
+++ b/cpp-ch/local-engine/Common/CHUtil.cpp
@@ -657,6 +657,16 @@ void
BackendInitializerUtil::initSettings(std::map<std::string, std::string> & b
settings.set(k, toField(k, value));
LOG_DEBUG(&Poco::Logger::get("CHUtil"), "Set settings key:{}
value:{}", key, value);
}
+ else if (key == SPARK_SESSION_TIME_ZONE)
+ {
+ String time_zone_val = value;
+ /// Convert timezone ID like '+8:00' to GMT+8:00
+ if (value.starts_with("+") || value.starts_with("-"))
+ time_zone_val = "GMT" + value;
+ time_zone_val = DateLUT::mappingForJavaTimezone(time_zone_val);
+ settings.set("session_timezone", time_zone_val);
+ LOG_DEBUG(&Poco::Logger::get("CHUtil"), "Set settings key:{}
value:{}", "session_timezone", time_zone_val);
+ }
}
/// Finally apply some fixed kvs to settings.
diff --git a/cpp-ch/local-engine/Common/CHUtil.h
b/cpp-ch/local-engine/Common/CHUtil.h
index 458eec9d3..2ef3c6ef9 100644
--- a/cpp-ch/local-engine/Common/CHUtil.h
+++ b/cpp-ch/local-engine/Common/CHUtil.h
@@ -168,6 +168,7 @@ public:
inline static const std::string SPARK_HADOOP_PREFIX = "spark.hadoop.";
inline static const std::string S3A_PREFIX = "fs.s3a.";
inline static const std::string SPARK_DELTA_PREFIX =
"spark.databricks.delta.";
+ inline static const std::string SPARK_SESSION_TIME_ZONE =
"spark.sql.session.timeZone";
inline static const String GLUTEN_TASK_OFFHEAP =
"spark.gluten.memory.task.offHeap.size.in.bytes";
inline static const String CH_TASK_MEMORY = "off_heap_per_task";
diff --git a/shims/common/src/main/scala/org/apache/gluten/GlutenConfig.scala
b/shims/common/src/main/scala/org/apache/gluten/GlutenConfig.scala
index 6659a42c7..d76e698dc 100644
--- a/shims/common/src/main/scala/org/apache/gluten/GlutenConfig.scala
+++ b/shims/common/src/main/scala/org/apache/gluten/GlutenConfig.scala
@@ -709,6 +709,10 @@ object GlutenConfig {
.filter(_._1.startsWith(SPARK_ABFS_ACCOUNT_KEY))
.foreach(entry => nativeConfMap.put(entry._1, entry._2))
+ conf
+ .filter(_._1.startsWith(SQLConf.SESSION_LOCAL_TIMEZONE.key))
+ .foreach(entry => nativeConfMap.put(entry._1, entry._2))
+
// return
nativeConfMap
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
