q2w commented on a change in pull request #32766:
URL: https://github.com/apache/spark/pull/32766#discussion_r646276308
##########
File path:
core/src/main/scala/org/apache/spark/scheduler/cluster/CoarseGrainedSchedulerBackend.scala
##########
@@ -478,6 +478,45 @@ class CoarseGrainedSchedulerBackend(scheduler:
TaskSchedulerImpl, val rpcEnv: Rp
protected def minRegisteredRatio: Double = _minRegisteredRatio
+ @GuardedBy("CoarseGrainedSchedulerBackend.this")
+ private val executorsToDecommissionInBatches = new HashSet[String]
+
+ private val executorDecommissionEnabled =
conf.get(config.DECOMMISSION_ENABLED)
+ private var periodicBatchExecutorsDecommissionThread:
Option[ScheduledExecutorService] = None
+ private var periodicBatchExecutorsDecommissionTask:
Option[ScheduledFuture[_]] = None
+
+ if (executorDecommissionEnabled) {
+ /**
+ * Start a thread which runs a task periodically to decommission executors.
Review comment:
I have kept batch decommission interval as 3s and batch size to
Int.MaxValue for now. We won't regress with default settings.
However we can set these variables aggressively to try to migrate data from
all decommissioning executors.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
