srowen commented on a change in pull request #23801: [SPARK-26891][YARN] Fixing
flaky test in YarnSchedulerBackendSuite
URL: https://github.com/apache/spark/pull/23801#discussion_r257517252
##########
File path:
resource-managers/yarn/src/test/scala/org/apache/spark/scheduler/cluster/YarnSchedulerBackendSuite.scala
##########
@@ -32,15 +33,33 @@ import org.apache.spark.ui.TestFilter
class YarnSchedulerBackendSuite extends SparkFunSuite with MockitoSugar with
LocalSparkContext {
+ private var yarnSchedulerBackend: YarnSchedulerBackend = _
+
+ override def afterEach() {
+ try {
+ yarnSchedulerBackend.stop()
+ } finally {
+ super.afterEach()
+ }
+ }
+
test("RequestExecutors reflects node blacklist and is serializable") {
sc = new SparkContext("local", "YarnSchedulerBackendSuite")
- val sched = mock[TaskSchedulerImpl]
- when(sched.sc).thenReturn(sc)
- val yarnSchedulerBackend = new YarnSchedulerBackend(sched, sc) {
+ // Subclassing the TaskSchedulerImpl here instead of using Mockito. For
details see SPARK-26891.
+ val sched = new TaskSchedulerImpl(sc) {
+ val blacklistedNodes = new AtomicReference[Set[String]]()
+
+ def setNodeBlacklist(nodeBlacklist: Set[String]): Unit =
blacklistedNodes.set(nodeBlacklist)
+
+ override def nodeBlacklist(): Set[String] = blacklistedNodes.get()
+ }
+
+ val yarnSchedulerBackendExtended = new YarnSchedulerBackend(sched, sc) {
Review comment:
Do you need the extra variable here, vs assigning to `yarnSchedulerBackend`?
I don't see that they are used separately.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
