mridulm commented on code in PR #38441:
URL: https://github.com/apache/spark/pull/38441#discussion_r1021139605
##########
core/src/main/scala/org/apache/spark/internal/config/package.scala:
##########
@@ -2024,6 +2024,15 @@ package object config {
.stringConf
.createOptional
+ private[spark] val SCHEDULER_MAX_RETAINED_REMOVED_EXECUTORS =
+ ConfigBuilder("spark.scheduler.maxRetainedRemovedExecutors")
+ .doc("Max number of removed executors by decommission to retain. This
affects " +
+ "whether fetch failure caused by removed decommissioned executors
could be ignored " +
+ "when spark.stage.ignoreDecommissionFetchFailure is enabled.")
+ .version("3.4.0")
+ .longConf
Review Comment:
`intConf`
##########
core/src/main/scala/org/apache/spark/internal/config/package.scala:
##########
@@ -2024,6 +2024,15 @@ package object config {
.stringConf
.createOptional
+ private[spark] val SCHEDULER_MAX_RETAINED_REMOVED_EXECUTORS =
+ ConfigBuilder("spark.scheduler.maxRetainedRemovedExecutors")
+ .doc("Max number of removed executors by decommission to retain. This
affects " +
+ "whether fetch failure caused by removed decommissioned executors
could be ignored " +
+ "when spark.stage.ignoreDecommissionFetchFailure is enabled.")
+ .version("3.4.0")
Review Comment:
Mark this `internal` ?
##########
core/src/main/scala/org/apache/spark/scheduler/TaskSchedulerImpl.scala:
##########
@@ -145,6 +147,16 @@ private[spark] class TaskSchedulerImpl(
// continue to run even after being asked to decommission, but they will
eventually exit.
val executorsPendingDecommission = new HashMap[String,
ExecutorDecommissionState]
+ // Keep removed executors due to decommission, so
getExecutorDecommissionState
+ // still return correct value even after executor is lost
+ val executorsRemovedByDecom =
+ CacheBuilder.newBuilder()
+ .maximumSize(conf.get(SCHEDULER_MAX_RETAINED_REMOVED_EXECUTORS))
+ .build[String, ExecutorDecommissionState]()
+
+ // Max size to keep removed executors due to decommission
+ val REMOVED_EXECUTORS_MAX_SIZE = 10000
Review Comment:
Remove this variable ?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
