xuanyuanking commented on a change in pull request #25850: [SPARK-29177] [Core]
fix zombie tasks after stage abort
URL: https://github.com/apache/spark/pull/25850#discussion_r326718804
##########
File path:
core/src/test/scala/org/apache/spark/scheduler/TaskResultGetterSuite.scala
##########
@@ -130,6 +141,30 @@ class TaskResultGetterSuite extends SparkFunSuite with
BeforeAndAfter with Local
"Expect result to be removed from the block manager.")
}
+ test("handling total size of results larger than maxResultSize") {
+ sc = new SparkContext("local", "test", conf)
+ val scheduler = new DummyTaskSchedulerImpl(sc)
+ val spyScheduler = spy(scheduler)
+ val resultGetter = new TaskResultGetter(sc.env, spyScheduler)
+ scheduler.taskResultGetter = resultGetter
+ val myTsm = new TaskSetManager(spyScheduler, FakeTask.createTaskSet(2), 1)
{
+ // always returns false
+ override def canFetchMoreResults(size: Long): Boolean = false
+ }
+ val indirectTaskResult = IndirectTaskResult(TaskResultBlockId(0), 0)
+ val directTaskResult = new DirectTaskResult(ByteBuffer.allocate(0), Nil,
Array())
+ val ser = sc.env.closureSerializer.newInstance()
+ val serializedIndirect = ser.serialize(indirectTaskResult)
+ val serializedDirect = ser.serialize(directTaskResult)
+ resultGetter.enqueueSuccessfulTask(myTsm, 0, serializedDirect)
+ resultGetter.enqueueSuccessfulTask(myTsm, 1, serializedIndirect)
+ Thread.sleep(1000L)
Review comment:
Explicit sleep will make the test flaky, condition wait is better
```
eventually(timeout(10.second)) {
verify(spyScheduler, times(1)).handleFailedTask(
myTsm, 0, TaskState.KILLED, TaskKilled("Tasks result size has exceeded
maxResultSize"))
verify(spyScheduler, times(1)).handleFailedTask(
myTsm, 1, TaskState.KILLED, TaskKilled("Tasks result size has exceeded
maxResultSize"))
}
```
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
