holdenk commented on a change in pull request #28331:
URL: https://github.com/apache/spark/pull/28331#discussion_r421812504
##########
File path:
core/src/test/scala/org/apache/spark/storage/BlockManagerDecommissionSuite.scala
##########
@@ -69,36 +84,64 @@ class BlockManagerDecommissionSuite extends SparkFunSuite
with LocalSparkContext
})
// Cache the RDD lazily
- sleepyRdd.persist()
+ if (persist) {
+ testRdd.persist()
+ }
// Start the computation of RDD - this step will also cache the RDD
- val asyncCount = sleepyRdd.countAsync()
+ val asyncCount = testRdd.countAsync()
// Wait for the job to have started
sem.acquire(1)
+ // Give Spark a tiny bit to start the tasks after the listener says hello
+ Thread.sleep(100)
+
// Decommission one of the executor
val sched = sc.schedulerBackend.asInstanceOf[StandaloneSchedulerBackend]
val execs = sched.getExecutorIds()
- assert(execs.size == 3, s"Expected 3 executors but found ${execs.size}")
+ assert(execs.size == numExecs, s"Expected ${numExecs} executors but found
${execs.size}")
val execToDecommission = execs.head
+ logDebug(s"Decommissioning executor ${execToDecommission}")
sched.decommissionExecutor(execToDecommission)
// Wait for job to finish
- val asyncCountResult = ThreadUtils.awaitResult(asyncCount, 3.seconds)
+ val asyncCountResult = ThreadUtils.awaitResult(asyncCount, 6.seconds)
assert(asyncCountResult === 10)
// All 10 tasks finished, so accum should have been increased 10 times
assert(accum.value === 10)
// All tasks should be successful, nothing should have failed
sc.listenerBus.waitUntilEmpty()
- assert(taskEndEvents.size === 10) // 10 mappers
+ if (shuffle) {
+ // 10 mappers & 10 reducers
+ assert(taskEndEvents.size === 20,
+ s"Expected 20 tasks got ${taskEndEvents.size} (${taskEndEvents})")
+ } else {
+ // 10 mappers
+ assert(taskEndEvents.size === 10,
+ s"Expected 10 tasks got ${taskEndEvents.size} (${taskEndEvents})")
Review comment:
Good catch!
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
