This is an automated email from the ASF dual-hosted git repository.
dongjoon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 0b0f4dd [SPARK-36383][CORE] Avoid NullPointerException during
executor shutdown
0b0f4dd is described below
commit 0b0f4dd1862db08636d5fa1913d42be5e9f0ba14
Author: yi.wu <[email protected]>
AuthorDate: Mon Aug 2 23:57:14 2021 -0700
[SPARK-36383][CORE] Avoid NullPointerException during executor shutdown
### What changes were proposed in this pull request?
Fix `NullPointerException` in `Executor.stop()`.
### Why are the changes needed?
Some initialization steps could fail before the initialization of
`metricsPoller`, `heartbeater`, `threadPool`, which results in the null of
`metricsPoller`, `heartbeater`, `threadPool`. For example, I encountered a
failure of:
https://github.com/apache/spark/blob/c20af535803a7250fef047c2bf0fe30be242369d/core/src/main/scala/org/apache/spark/executor/Executor.scala#L137
where the executor itself failed to register at the driver.
This PR helps to eliminate the error messages when the issue happens to not
confuse users:
<details>
<summary><mark><font color=darkred>[click to see the detailed error
message]</font></mark></summary>
<pre>
21/07/23 16:04:10 WARN Executor: Unable to stop executor metrics poller
java.lang.NullPointerException
at org.apache.spark.executor.Executor.stop(Executor.scala:318)
at
org.apache.spark.executor.Executor.$anonfun$stopHookReference$1(Executor.scala:76)
at
org.apache.spark.util.SparkShutdownHook.run(ShutdownHookManager.scala:214)
at
org.apache.spark.util.SparkShutdownHookManager.$anonfun$runAll$2(ShutdownHookManager.scala:188)
at
scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.java:23)
at
org.apache.spark.util.Utils$.logUncaughtExceptions(Utils.scala:2025)
at
org.apache.spark.util.SparkShutdownHookManager.$anonfun$runAll$1(ShutdownHookManager.scala:188)
at
scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.java:23)
at scala.util.Try$.apply(Try.scala:213)
at
org.apache.spark.util.SparkShutdownHookManager.runAll(ShutdownHookManager.scala:188)
at
org.apache.spark.util.SparkShutdownHookManager$$anon$2.run(ShutdownHookManager.scala:178)
at
java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
21/07/23 16:04:10 WARN Executor: Unable to stop heartbeater
java.lang.NullPointerException
at org.apache.spark.executor.Executor.stop(Executor.scala:324)
at
org.apache.spark.executor.Executor.$anonfun$stopHookReference$1(Executor.scala:76)
at
org.apache.spark.util.SparkShutdownHook.run(ShutdownHookManager.scala:214)
at
org.apache.spark.util.SparkShutdownHookManager.$anonfun$runAll$2(ShutdownHookManager.scala:188)
at
scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.java:23)
at
org.apache.spark.util.Utils$.logUncaughtExceptions(Utils.scala:2025)
at
org.apache.spark.util.SparkShutdownHookManager.$anonfun$runAll$1(ShutdownHookManager.scala:188)
at
scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.java:23)
at scala.util.Try$.apply(Try.scala:213)
at
org.apache.spark.util.SparkShutdownHookManager.runAll(ShutdownHookManager.scala:188)
at
org.apache.spark.util.SparkShutdownHookManager$$anon$2.run(ShutdownHookManager.scala:178)
at
java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
21/07/23 16:04:10 ERROR Utils: Uncaught exception in thread shutdown-hook-0
java.lang.NullPointerException
at
org.apache.spark.executor.Executor.$anonfun$stop$3(Executor.scala:334)
at
scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.java:23)
at
org.apache.spark.util.Utils$.withContextClassLoader(Utils.scala:231)
at org.apache.spark.executor.Executor.stop(Executor.scala:334)
at
org.apache.spark.executor.Executor.$anonfun$stopHookReference$1(Executor.scala:76)
at
org.apache.spark.util.SparkShutdownHook.run(ShutdownHookManager.scala:214)
at
org.apache.spark.util.SparkShutdownHookManager.$anonfun$runAll$2(ShutdownHookManager.scala:188)
at
scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.java:23)
at
org.apache.spark.util.Utils$.logUncaughtExceptions(Utils.scala:2025)
at
org.apache.spark.util.SparkShutdownHookManager.$anonfun$runAll$1(ShutdownHookManager.scala:188)
at
scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.java:23)
at scala.util.Try$.apply(Try.scala:213)
at
org.apache.spark.util.SparkShutdownHookManager.runAll(ShutdownHookManager.scala:188)
at
org.apache.spark.util.SparkShutdownHookManager$$anon$2.run(ShutdownHookManager.scala:178)
at
java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
</pre>
</details>
### Does this PR introduce _any_ user-facing change?
Yes, users won't see error messages of `NullPointerException` after this
fix.
### How was this patch tested?
Pass existing tests.
Closes #33612 from Ngone51/avoid-npe-during-executor-shutdown.
Authored-by: yi.wu <[email protected]>
Signed-off-by: Dongjoon Hyun <[email protected]>
---
.../scala/org/apache/spark/executor/Executor.scala | 21 ++++++++++++++-------
1 file changed, 14 insertions(+), 7 deletions(-)
diff --git a/core/src/main/scala/org/apache/spark/executor/Executor.scala
b/core/src/main/scala/org/apache/spark/executor/Executor.scala
index 9c257d8..5fb1d7f 100644
--- a/core/src/main/scala/org/apache/spark/executor/Executor.scala
+++ b/core/src/main/scala/org/apache/spark/executor/Executor.scala
@@ -315,23 +315,30 @@ private[spark] class Executor(
ShutdownHookManager.removeShutdownHook(stopHookReference)
env.metricsSystem.report()
try {
- metricsPoller.stop()
+ if (metricsPoller != null) {
+ metricsPoller.stop()
+ }
} catch {
case NonFatal(e) =>
logWarning("Unable to stop executor metrics poller", e)
}
try {
- heartbeater.stop()
+ if (heartbeater != null) {
+ heartbeater.stop()
+ }
} catch {
case NonFatal(e) =>
logWarning("Unable to stop heartbeater", e)
}
ShuffleBlockPusher.stop()
- threadPool.shutdown()
-
- // Notify plugins that executor is shutting down so they can terminate
cleanly
- Utils.withContextClassLoader(replClassLoader) {
- plugins.foreach(_.shutdown())
+ if (threadPool != null) {
+ threadPool.shutdown()
+ }
+ if (replClassLoader != null && plugins != null) {
+ // Notify plugins that executor is shutting down so they can terminate
cleanly
+ Utils.withContextClassLoader(replClassLoader) {
+ plugins.foreach(_.shutdown())
+ }
}
if (!isLocal) {
env.stop()
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
