This is an automated email from the ASF dual-hosted git repository.
changchen pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/main by this push:
new 5170be1207 [GLUTEN-8926][CH] MergeTree Parameter Configuration
Optimization to Prevent Multithreading Competition for activeSession Being None
(#8927)
5170be1207 is described below
commit 5170be1207e00b2ba3984ff6c58284506494467e
Author: Guoliang Sun <[email protected]>
AuthorDate: Fri Mar 7 13:49:09 2025 +0800
[GLUTEN-8926][CH] MergeTree Parameter Configuration Optimization to Prevent
Multithreading Competition for activeSession Being None (#8927)
---
.../scala/org/apache/spark/sql/delta/ClickhouseSnapshot.scala | 8 +++++---
1 file changed, 5 insertions(+), 3 deletions(-)
diff --git
a/backends-clickhouse/src/main/scala/org/apache/spark/sql/delta/ClickhouseSnapshot.scala
b/backends-clickhouse/src/main/scala/org/apache/spark/sql/delta/ClickhouseSnapshot.scala
index e3f6430466..3631055492 100644
---
a/backends-clickhouse/src/main/scala/org/apache/spark/sql/delta/ClickhouseSnapshot.scala
+++
b/backends-clickhouse/src/main/scala/org/apache/spark/sql/delta/ClickhouseSnapshot.scala
@@ -84,9 +84,11 @@ case class FilterExprsAsKey(
}
object ClickhouseSnapshot {
+ protected def sparkSession: SparkSession = SparkSession.active
+
val deltaScanCache: Cache[FilterExprsAsKey, DeltaScan] =
CacheBuilder.newBuilder
.maximumSize(
- SparkSession.getActiveSession.get.conf
+ sparkSession.conf
.get(CHBackendSettings.GLUTEN_CLICKHOUSE_DELTA_SCAN_CACHE_SIZE,
"10000")
.toLong)
.expireAfterAccess(7200L, TimeUnit.SECONDS)
@@ -95,7 +97,7 @@ object ClickhouseSnapshot {
val addFileToAddMTPCache: LoadingCache[AddFileAsKey, AddMergeTreeParts] =
CacheBuilder.newBuilder
.maximumSize(
- SparkSession.getActiveSession.get.conf
+ sparkSession.conf
.get(CHBackendSettings.GLUTEN_CLICKHOUSE_ADDFILES_TO_MTPS_CACHE_SIZE,
"1000000")
.toLong)
.expireAfterAccess(7200L, TimeUnit.SECONDS)
@@ -109,7 +111,7 @@ object ClickhouseSnapshot {
val pathToAddMTPCache: Cache[String, AddMergeTreeParts] =
CacheBuilder.newBuilder
.maximumSize(
- SparkSession.getActiveSession.get.conf
+ sparkSession.conf
.get(CHBackendSettings.GLUTEN_CLICKHOUSE_TABLE_PATH_TO_MTPS_CACHE_SIZE,
"1000000")
.toLong)
.expireAfterAccess(7200L, TimeUnit.SECONDS)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
