Ngone51 commented on code in PR #43954:
URL: https://github.com/apache/spark/pull/43954#discussion_r1408622291
##########
core/src/main/scala/org/apache/spark/internal/config/package.scala:
##########
@@ -2603,4 +2603,13 @@ package object config {
.stringConf
.toSequence
.createWithDefault("org.apache.spark.sql.connect.client" :: Nil)
+
+ private[spark] val ABORT_STAGE_AFTER_CANCEL_TASKS =
+ ConfigBuilder("spark.scheduler.stage.abortStageAfterCancelTasks")
Review Comment:
Right, should probably add the "legacy" key word.
##########
core/src/test/scala/org/apache/spark/scheduler/TaskSchedulerImplSuite.scala:
##########
@@ -1671,37 +1671,6 @@ class TaskSchedulerImplSuite extends SparkFunSuite with
LocalSparkContext
assert(taskScheduler.taskSetManagerForAttempt(0, 0).isEmpty)
}
- test("killAllTaskAttempts shall kill all the running tasks and not fail the
stage") {
- val taskScheduler = setupScheduler()
-
- taskScheduler.initialize(new FakeSchedulerBackend {
- override def killTask(
- taskId: Long,
- executorId: String,
- interruptThread: Boolean,
- reason: String): Unit = {
- // Since we only submit one stage attempt, the following call is
sufficient to mark the
- // task as killed.
- taskScheduler.taskSetManagerForAttempt(0,
0).get.runningTasksSet.remove(taskId)
- }
- })
-
- val attempt1 = FakeTask.createTaskSet(10)
- taskScheduler.submitTasks(attempt1)
-
- val workerOffers = IndexedSeq(new WorkerOffer("executor0", "host0", 1),
- new WorkerOffer("executor1", "host1", 1))
- val taskDescriptions = taskScheduler.resourceOffers(workerOffers).flatten
- assert(2 === taskDescriptions.length)
- val tsm = taskScheduler.taskSetManagerForAttempt(0, 0).get
- assert(2 === tsm.runningTasks)
-
- taskScheduler.killAllTaskAttempts(0, false, "test")
Review Comment:
Sounds good.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
