uros-b commented on code in PR #56793:
URL: https://github.com/apache/spark/pull/56793#discussion_r3479735959
##########
sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala:
##########
@@ -2900,13 +2900,27 @@ object SQLConf {
val NUM_STATE_STORE_MAINTENANCE_THREADS =
buildConf("spark.sql.streaming.stateStore.numStateStoreMaintenanceThreads")
.internal()
- .doc("Number of threads in the thread pool that perform clean up and
snapshotting tasks " +
- "for stateful streaming queries. The default value is the number of
cores * 0.25 " +
- "so that this thread pool doesn't take too many resources " +
- "away from the query and affect performance.")
+ .doc("Total number of threads split between the snapshot and cleanup " +
+ "maintenance pools for stateful streaming queries. Each pool needs at
least " +
+ "1 thread, so the minimum is 2. The default value is the number of " +
+ "cores * 0.25 so that the pools don't take too many resources away
from the " +
+ "query and affect performance. Use snapshotToCleanupThreadRatio to " +
+ "configure the split between snapshot and cleanup pools.")
.intConf
- .checkValue(_ > 0, "Must be greater than 0")
- .createWithDefault(Math.max(Runtime.getRuntime.availableProcessors() /
4, 1))
+ .checkValue(_ > 1, "Must be greater than 1")
+ .createWithDefault(Math.max(Runtime.getRuntime.availableProcessors() /
4, 2))
+
+ val STATE_STORE_MAINTENANCE_SNAPSHOT_THREAD_RATIO =
+ buildConf("spark.sql.streaming.stateStore.snapshotToCleanupThreadRatio")
+ .internal()
+ .doc("Ratio of total maintenance threads allocated to the snapshot " +
Review Comment:
Please add config .version
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
