Ngone51 commented on a change in pull request #29332:
URL: https://github.com/apache/spark/pull/29332#discussion_r464396088
##########
File path:
core/src/main/scala/org/apache/spark/scheduler/TaskSchedulerImpl.scala
##########
@@ -468,51 +468,6 @@ private[spark] class TaskSchedulerImpl(
Some(localTaskReqAssign.toMap)
}
- // Use the resource that the resourceProfile has as the limiting resource to
calculate the
- // total number of slots available based on the current offers.
- private def calculateAvailableSlots(
- resourceProfileIds: Array[Int],
- availableCpus: Array[Int],
- availableResources: Array[Map[String, Buffer[String]]],
- taskSet: TaskSetManager): Int = {
- val resourceProfile = sc.resourceProfileManager.resourceProfileFromId(
- taskSet.taskSet.resourceProfileId)
- val offersForResourceProfile = resourceProfileIds.zipWithIndex.filter {
case (id, _) =>
- (id == resourceProfile.id)
- }
- val coresKnown = resourceProfile.isCoresLimitKnown
- var limitingResource = resourceProfile.limitingResource(conf)
- val taskCpus =
ResourceProfile.getTaskCpusOrDefaultForProfile(resourceProfile, conf)
-
- offersForResourceProfile.map { case (o, index) =>
- val numTasksPerExecCores = availableCpus(index) / taskCpus
- // if limiting resource is empty then we have no other resources, so it
has to be CPU
- if (limitingResource == ResourceProfile.CPUS ||
limitingResource.isEmpty) {
- numTasksPerExecCores
- } else {
- val taskLimit =
resourceProfile.taskResources.get(limitingResource).map(_.amount)
- .getOrElse {
- val errorMsg = "limitingResource returns from ResourceProfile " +
- s"$resourceProfile doesn't actually contain that task resource!"
Review comment:
@tgravescs This actually should not happen, right? According to:
https://github.com/apache/spark/blob/998086c9a179692b2687bc9a104dbbb35f5a44e2/core/src/main/scala/org/apache/spark/resource/ResourceProfile.scala#L194-L197
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
