tgravescs commented on a change in pull request #30650:
URL: https://github.com/apache/spark/pull/30650#discussion_r578499068
##########
File path:
core/src/main/scala/org/apache/spark/scheduler/TaskSchedulerImpl.scala
##########
@@ -670,27 +670,27 @@ private[spark] class TaskSchedulerImpl(
val barrierPendingLaunchTasks =
taskSet.barrierPendingLaunchTasks.values.toArray
// Check whether the barrier tasks are partially launched.
if (barrierPendingLaunchTasks.length != taskSet.numTasks) {
- val msg = s"Fail resource offers for barrier stage
${taskSet.stageId} " +
- s"because only ${barrierPendingLaunchTasks.length} out of a
total number " +
- s"of ${taskSet.numTasks} tasks got resource offers."
if (legacyLocalityWaitReset) {
// Legacy delay scheduling always reset the timer when there's a
task that is able
// to be scheduled. Thus, whenever there's a timer reset could
happen during a single
// round resourceOffer, tasks that don't get or have the
preferred locations would
// always reject the offered resources. As a result, the barrier
taskset can't get
// launched. And if we retry the resourceOffer, we'd go through
the same path again
// and get into the endless loop in the end.
- val errorMsg = s"$msg We highly recommend you to use the
non-legacy" +
- s" delay scheduling by setting
${LEGACY_LOCALITY_WAIT_RESET.key} " +
- s"to false to get rid of this error."
+ val errorMsg = s"Fail resource offers for barrier stage
${taskSet.stageId} " +
+ s"because only ${barrierPendingLaunchTasks.length} out of a
total number " +
+ s"of ${taskSet.numTasks} tasks got resource offers. We highly
recommend " +
+ s"you to use the non-legacy delay scheduling by setting " +
Review comment:
nit s not required here
##########
File path:
core/src/main/scala/org/apache/spark/scheduler/TaskSchedulerImpl.scala
##########
@@ -670,27 +670,27 @@ private[spark] class TaskSchedulerImpl(
val barrierPendingLaunchTasks =
taskSet.barrierPendingLaunchTasks.values.toArray
// Check whether the barrier tasks are partially launched.
if (barrierPendingLaunchTasks.length != taskSet.numTasks) {
- val msg = s"Fail resource offers for barrier stage
${taskSet.stageId} " +
- s"because only ${barrierPendingLaunchTasks.length} out of a
total number " +
- s"of ${taskSet.numTasks} tasks got resource offers."
if (legacyLocalityWaitReset) {
// Legacy delay scheduling always reset the timer when there's a
task that is able
// to be scheduled. Thus, whenever there's a timer reset could
happen during a single
// round resourceOffer, tasks that don't get or have the
preferred locations would
// always reject the offered resources. As a result, the barrier
taskset can't get
// launched. And if we retry the resourceOffer, we'd go through
the same path again
// and get into the endless loop in the end.
- val errorMsg = s"$msg We highly recommend you to use the
non-legacy" +
- s" delay scheduling by setting
${LEGACY_LOCALITY_WAIT_RESET.key} " +
- s"to false to get rid of this error."
+ val errorMsg = s"Fail resource offers for barrier stage
${taskSet.stageId} " +
+ s"because only ${barrierPendingLaunchTasks.length} out of a
total number " +
+ s"of ${taskSet.numTasks} tasks got resource offers. We highly
recommend " +
+ s"you to use the non-legacy delay scheduling by setting " +
+ s"${LEGACY_LOCALITY_WAIT_RESET.key} to false to get rid of
this error."
logWarning(errorMsg)
taskSet.abort(errorMsg)
throw new SparkException(errorMsg)
} else {
val curTime = clock.getTimeMillis()
if (curTime - taskSet.lastResourceOfferFailLogTime >
TaskSetManager.BARRIER_LOGGING_INTERVAL) {
- logInfo(s"$msg Waiting for later round resource offers.")
+ logInfo(s"Releasing the assigned resource offers since only
partial tasks can " +
Review comment:
nit s for string interpolation is not required here or below.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
