anishshri-db commented on code in PR #42066:
URL: https://github.com/apache/spark/pull/42066#discussion_r1280124681
##########
sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/state/StateStore.scala:
##########
@@ -607,25 +650,42 @@ object StateStore extends Logging {
}
/** Start the periodic maintenance task if not already started and if Spark
active */
- private def startMaintenanceIfNeeded(storeConf: StateStoreConf): Unit =
+ private def startMaintenanceIfNeeded(storeConf: StateStoreConf): Unit = {
+ val numMaintenanceThreads = storeConf.numStateStoreMaintenanceThreads
loadedProviders.synchronized {
if (SparkEnv.get != null && !isMaintenanceRunning) {
maintenanceTask = new MaintenanceTask(
storeConf.maintenanceInterval,
- task = { doMaintenance() },
- onError = { loadedProviders.synchronized {
+ task = {
+ doMaintenance()
+ },
+ onError = {
+ loadedProviders.synchronized {
logInfo("Stopping maintenance task since an error was
encountered.")
stopMaintenanceTask()
- // SPARK-44504 - Unload explicitly to force closing underlying
DB instance
- // and releasing allocated resources, especially for
RocksDBStateStoreProvider.
loadedProviders.keySet.foreach { key => unload(key) }
loadedProviders.clear()
}
}
)
+ maintenanceThreadPool = new MaintenanceThreadPool(
+ numThreads = numMaintenanceThreads
Review Comment:
We can just pass `numMaintenanceThreads` right ? we only have 1 argument for
the class ?
```
maintenanceThreadPool = new MaintenanceThreadPool(numMaintenanceThreads)
``` ?
##########
sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/state/StateStore.scala:
##########
@@ -607,25 +650,42 @@ object StateStore extends Logging {
}
/** Start the periodic maintenance task if not already started and if Spark
active */
- private def startMaintenanceIfNeeded(storeConf: StateStoreConf): Unit =
+ private def startMaintenanceIfNeeded(storeConf: StateStoreConf): Unit = {
+ val numMaintenanceThreads = storeConf.numStateStoreMaintenanceThreads
loadedProviders.synchronized {
if (SparkEnv.get != null && !isMaintenanceRunning) {
maintenanceTask = new MaintenanceTask(
storeConf.maintenanceInterval,
- task = { doMaintenance() },
- onError = { loadedProviders.synchronized {
+ task = {
+ doMaintenance()
+ },
+ onError = {
+ loadedProviders.synchronized {
logInfo("Stopping maintenance task since an error was
encountered.")
stopMaintenanceTask()
- // SPARK-44504 - Unload explicitly to force closing underlying
DB instance
- // and releasing allocated resources, especially for
RocksDBStateStoreProvider.
loadedProviders.keySet.foreach { key => unload(key) }
loadedProviders.clear()
}
}
)
+ maintenanceThreadPool = new MaintenanceThreadPool(
+ numThreads = numMaintenanceThreads
Review Comment:
We can just pass `numMaintenanceThreads` right ? we only have 1 argument for
the class ?
```
maintenanceThreadPool = new MaintenanceThreadPool(numMaintenanceThreads)
```
?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
