mridulm commented on code in PR #37779:
URL: https://github.com/apache/spark/pull/37779#discussion_r984943244
##########
core/src/test/scala/org/apache/spark/executor/CoarseGrainedExecutorBackendSuite.scala:
##########
@@ -535,6 +538,40 @@ class CoarseGrainedExecutorBackendSuite extends
SparkFunSuite
}
}
+ /**
+ * A fatal error occurred when [[Executor]] was initialized, this should be
caught by
+ * [[SparkUncaughtExceptionHandler]] and [[Executor]] can exit by itself.
+ */
+ test("SPARK-40320 Executor should exit when initialization failed for fatal
error") {
+ new SparkUncaughtExceptionHandler
+ val conf = new SparkConf()
+ .setMaster("local-cluster[1, 1, 1024]")
+ .set(PLUGINS, Seq(classOf[FatalErrorPlugin].getName))
+ .setAppName("test")
+ sc = new SparkContext(conf)
+ val executorAddCounter = new AtomicInteger(0)
+ val executorRemovedCounter = new AtomicInteger(0)
+
+ val listener = new SparkListener() {
+ override def onExecutorAdded(executorAdded: SparkListenerExecutorAdded):
Unit = {
+ executorAddCounter.getAndIncrement()
+ }
+
+ override def onExecutorRemoved(executorRemoved:
SparkListenerExecutorRemoved): Unit = {
+ executorRemovedCounter.getAndIncrement()
+ }
+ }
+ try {
+ sc.addSparkListener(listener)
+ eventually(timeout(60.seconds)) {
Review Comment:
Do we want to run this for 60s ? Perhaps, reduce it to 10s ?
Thoughts @Ngone51 ?
##########
core/src/main/scala/org/apache/spark/rpc/netty/MessageLoop.scala:
##########
@@ -171,7 +171,11 @@ private class DedicatedMessageLoop(
}
(1 to endpoint.threadCount()).foreach { _ =>
- threadpool.submit(receiveLoopRunnable)
+ /**
+ * We need to be careful not to use [[ExecutorService#submit]].
+ * `submit` api will swallow errors in [[FutureTask#setException]].
Review Comment:
`swallow uncaught exceptions` might be better (since the handler is
`UncaughtExceptionHandler`).
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
