This is an automated email from the ASF dual-hosted git repository.
pan3793 pushed a commit to branch branch-4.x
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-4.x by this push:
new 411add971a62 [SPARK-57387][YARN] Make executor JVM options
`-XX:OnOutOfMemoryError` configurable on YARN
411add971a62 is described below
commit 411add971a6205c8f2a3bbe05741d1209020ba35
Author: Cheng Pan <[email protected]>
AuthorDate: Fri Jun 12 11:25:38 2026 +0800
[SPARK-57387][YARN] Make executor JVM options `-XX:OnOutOfMemoryError`
configurable on YARN
### What changes were proposed in this pull request?
Currently, Spark always adds `-XX:OnOutOfMemoryError='kill %p'` to executor
JVM options when running on YARN. This PR makes the behavior configurable.
### Why are the changes needed?
There are many places where Spark's internal memory management handles
`OutOfMemoryError` and intends to recover from that, with
`-XX:OnOutOfMemoryError='kill %p'`, the recovery code does not work, and the
executor always gets killed when hit `OutOfMemoryError`.
Note: when `-XX:OnOutOfMemoryError='kill %p'` is configured, `kill %p` will
be performed no matter whether the code catches the `OutOfMemoryError` when it
is thrown by JVM.
### Does this PR introduce _any_ user-facing change?
No.
### How was this patch tested?
UTs are added.
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: Claude Opus 4.8.
Closes #56448 from pan3793/SPARK-57387.
Authored-by: Cheng Pan <[email protected]>
Signed-off-by: Cheng Pan <[email protected]>
(cherry picked from commit 913f105ed50aeafc95e31699acfb9a45e056e86b)
Signed-off-by: Cheng Pan <[email protected]>
---
docs/running-on-yarn.md | 9 ++++++++
.../spark/deploy/yarn/ExecutorRunnable.scala | 5 +++-
.../apache/spark/deploy/yarn/config/package.scala | 11 ++++++++-
.../spark/deploy/yarn/ExecutorRunnableSuite.scala | 27 ++++++++++++++++++++++
4 files changed, 50 insertions(+), 2 deletions(-)
diff --git a/docs/running-on-yarn.md b/docs/running-on-yarn.md
index d42e5eb4257e..020690b744c1 100644
--- a/docs/running-on-yarn.md
+++ b/docs/running-on-yarn.md
@@ -539,6 +539,15 @@ To use a custom metrics.properties for the application
master and executors, upd
</td>
<td>1.4.0</td>
</tr>
+<tr>
+ <td><code>spark.yarn.executor.oomKill.enabled</code></td>
+ <td>true</td>
+ <td>
+ Whether to add `-XX:OnOutOfMemoryError='kill %p'` (or its counterpart on
Windows) to executor
+ JVM options.
+ </td>
+ <td>4.3.0</td>
+</tr>
<tr>
<td><code>spark.yarn.tags</code></td>
<td>(none)</td>
diff --git
a/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/ExecutorRunnable.scala
b/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/ExecutorRunnable.scala
index f38f1053ef05..f14f9c26b657 100644
---
a/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/ExecutorRunnable.scala
+++
b/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/ExecutorRunnable.scala
@@ -38,6 +38,7 @@ import org.apache.hadoop.yarn.ipc.YarnRPC
import org.apache.hadoop.yarn.util.Records
import org.apache.spark.{SecurityManager, SparkConf, SparkException}
+import org.apache.spark.deploy.yarn.config.YARN_EXECUTOR_OOM_KILL_ENABLED
import org.apache.spark.internal.{Logging, MessageWithContext}
import org.apache.spark.internal.LogKeys.{EXECUTOR_ENVS,
EXECUTOR_LAUNCH_COMMANDS, EXECUTOR_RESOURCES}
import org.apache.spark.internal.config._
@@ -190,7 +191,9 @@ private[yarn] class ExecutorRunnable(
// For log4j configuration to reference
javaOpts += ("-Dspark.yarn.app.container.log.dir=" +
ApplicationConstants.LOG_DIR_EXPANSION_VAR)
- YarnSparkHadoopUtil.addOutOfMemoryErrorArgument(javaOpts)
+ if (sparkConf.get(YARN_EXECUTOR_OOM_KILL_ENABLED)) {
+ YarnSparkHadoopUtil.addOutOfMemoryErrorArgument(javaOpts)
+ }
val commands = prefixEnv ++
Seq(Environment.JAVA_HOME.$$() + "/bin/java", "-server") ++
javaOpts ++
diff --git
a/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/config/package.scala
b/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/config/package.scala
index 8b63d117e7fa..272f682bb9b8 100644
---
a/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/config/package.scala
+++
b/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/config/package.scala
@@ -21,7 +21,7 @@ import java.util.Properties
import java.util.concurrent.TimeUnit
import org.apache.spark.internal.Logging
-import org.apache.spark.internal.config.ConfigBuilder
+import org.apache.spark.internal.config.{ConfigBindingPolicy, ConfigBuilder}
import org.apache.spark.network.util.ByteUnit
package object config extends Logging {
@@ -348,6 +348,15 @@ package object config extends Logging {
.stringConf
.createOptional
+ private[spark] val YARN_EXECUTOR_OOM_KILL_ENABLED =
+ ConfigBuilder("spark.yarn.executor.oomKill.enabled")
+ .doc("Whether to add `-XX:OnOutOfMemoryError='kill %p'` (or its
counterpart on Windows) " +
+ "to executor JVM options.")
+ .version("4.3.0")
+ .withBindingPolicy(ConfigBindingPolicy.NOT_APPLICABLE)
+ .booleanConf
+ .createWithDefault(true)
+
/* Unmanaged AM configuration. */
private[spark] val YARN_UNMANAGED_AM =
ConfigBuilder("spark.yarn.unmanagedAM.enabled")
diff --git
a/resource-managers/yarn/src/test/scala/org/apache/spark/deploy/yarn/ExecutorRunnableSuite.scala
b/resource-managers/yarn/src/test/scala/org/apache/spark/deploy/yarn/ExecutorRunnableSuite.scala
index e06fadefcc0d..38f816731195 100644
---
a/resource-managers/yarn/src/test/scala/org/apache/spark/deploy/yarn/ExecutorRunnableSuite.scala
+++
b/resource-managers/yarn/src/test/scala/org/apache/spark/deploy/yarn/ExecutorRunnableSuite.scala
@@ -30,6 +30,7 @@ import org.scalatest.matchers.must.Matchers.{contain, not}
import org.scalatest.matchers.should.Matchers.convertToAnyShouldWrapper
import org.apache.spark.{SecurityManager, SparkConf, SparkFunSuite}
+import org.apache.spark.deploy.yarn.config.YARN_EXECUTOR_OOM_KILL_ENABLED
import org.apache.spark.internal.config._
import org.apache.spark.network.util.JavaUtils
@@ -138,4 +139,30 @@ class ExecutorRunnableSuite extends SparkFunSuite with
PrivateMethodTester {
commands should contain ("-XX:ActiveProcessorCount=7")
commands should contain inOrderElementsOf List("--cores", "7")
}
+
+ test("test executor OnOutOfMemoryError JVM options") {
+ Seq(true, false).foreach { oomKill =>
+ val sparkConf = new SparkConf()
+ .set(YARN_EXECUTOR_OOM_KILL_ENABLED, oomKill)
+ val execRunnable = createExecutorRunnable(sparkConf)
+
+ val commands = execRunnable invokePrivate _prepareCommand()
+ if (oomKill) {
+ assert(commands.exists(_.contains("-XX:OnOutOfMemoryError")))
+ } else {
+ assert(!commands.exists(_.contains("-XX:OnOutOfMemoryError")))
+ }
+ }
+ }
+
+ test("user provided OnOutOfMemoryError option takes precedence over
default") {
+ val sparkConf = new SparkConf()
+ .set(YARN_EXECUTOR_OOM_KILL_ENABLED, true)
+ .set(EXECUTOR_JAVA_OPTIONS, "-XX:OnOutOfMemoryError='kill -9 %p'")
+ val execRunnable = createExecutorRunnable(sparkConf)
+
+ val commands = execRunnable invokePrivate _prepareCommand()
+ assert(commands.count(_.contains("-XX:OnOutOfMemoryError")) === 1)
+ assert(commands.exists(_.contains("-XX:OnOutOfMemoryError=kill -9 %p")))
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
