This is an automated email from the ASF dual-hosted git repository.
dongjoon-hyun pushed a commit to branch branch-4.3
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-4.3 by this push:
new 36be47d663c7 [SPARK-59842][K8S] Fix
`spark.kubernetes.executor.resizeInterval` default value and `0` handling
36be47d663c7 is described below
commit 36be47d663c7ca116490a0698e6f1c19d1ee4c3b
Author: Dongjoon Hyun <[email protected]>
AuthorDate: Tue Sep 29 06:07:51 2026 -0700
[SPARK-59842][K8S] Fix `spark.kubernetes.executor.resizeInterval` default
value and `0` handling
This PR aims to fix `spark.kubernetes.executor.resizeInterval` handling in
`ExecutorResizePlugin`.
- Change the default value from `0s` to `1m` to match the existing runtime
behavior.
- Disable `ExecutorResizePlugin` when the interval is `0`, like
`ExecutorPVCResizePlugin` does.
Since Apache Spark 4.2.0, `spark.kubernetes.executor.resizeInterval` is
documented as `0s` (disabled) by default, but `ExecutorResizePlugin` actually
runs every minute when the config is unset. Moreover, setting `0` as documented
makes `SparkContext` fail to start due to `IllegalArgumentException` from
`scheduleAtFixedRate`.
Yes, but this is a bug fix where
`spark.kubernetes.executor.resizeInterval=0` disables `ExecutorResizePlugin`
instead of failing `SparkContext` initialization. The behavior of the unset
case is unchanged.
Pass the CIs with the newly added test cases.
Generated-by: Claude Opus 5.5
Closes #59114 from dongjoon-hyun/SPARK-59842.
Authored-by: Dongjoon Hyun <[email protected]>
Signed-off-by: Dongjoon Hyun <[email protected]>
(cherry picked from commit 8582186d9d52477029a41729d7e47a1da3c5b3f0)
Signed-off-by: Dongjoon Hyun <[email protected]>
---
docs/running-on-kubernetes.md | 4 +--
.../scala/org/apache/spark/deploy/k8s/Config.scala | 4 +--
.../cluster/k8s/ExecutorResizePlugin.scala | 9 ++++--
.../cluster/k8s/ExecutorResizePluginSuite.scala | 32 ++++++++++++++++++++++
4 files changed, 42 insertions(+), 7 deletions(-)
diff --git a/docs/running-on-kubernetes.md b/docs/running-on-kubernetes.md
index 8c3039e07eb1..a306fdb1d545 100644
--- a/docs/running-on-kubernetes.md
+++ b/docs/running-on-kubernetes.md
@@ -1977,9 +1977,9 @@ See the [configuration page](configuration.html) for
information on Spark config
</tr>
<tr>
<td><code>spark.kubernetes.executor.resizeInterval</code></td>
- <td><code>0s</code></td>
+ <td><code>1min</code></td>
<td>
- Interval between executor resize operations. To disable, set 0 (default).
+ Interval between executor resize operations. To disable, set 0.
Takes effect only when
<code>org.apache.spark.scheduler.cluster.k8s.ExecutorResizePlugin</code>
is registered via <code>spark.plugins</code>.
</td>
diff --git
a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/Config.scala
b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/Config.scala
index 160f64c77f04..cdc82ea4ac93 100644
---
a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/Config.scala
+++
b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/Config.scala
@@ -284,11 +284,11 @@ private[spark] object Config extends Logging {
val EXECUTOR_RESIZE_INTERVAL =
ConfigBuilder("spark.kubernetes.executor.resizeInterval")
- .doc("Interval between executor resize operations. To disable, set 0
(default)")
+ .doc("Interval between executor resize operations. To disable, set 0.")
.version("4.2.0")
.timeConf(TimeUnit.SECONDS)
.checkValue(_ >= 0, "Interval should be non-negative")
- .createWithDefault(0)
+ .createWithDefaultString("1m")
val EXECUTOR_RESIZE_THRESHOLD =
ConfigBuilder("spark.kubernetes.executor.resizeThreshold")
diff --git
a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorResizePlugin.scala
b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorResizePlugin.scala
index ad4611b52fc7..0b9e33672827 100644
---
a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorResizePlugin.scala
+++
b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorResizePlugin.scala
@@ -32,7 +32,7 @@ import org.apache.spark.deploy.k8s.Config._
import org.apache.spark.deploy.k8s.Constants._
import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys.{CLASS_NAME, CONFIG, CONFIG2,
EXECUTOR_ID, MEMORY_SIZE}
-import org.apache.spark.util.{ThreadUtils, Utils}
+import org.apache.spark.util.ThreadUtils
/**
* Spark plugin to monitor executor pod memory usage and increase the memory
limit
@@ -59,8 +59,11 @@ class ExecutorResizeDriverPlugin extends DriverPlugin with
Logging {
return Map.empty[String, String].asJava
}
- val interval = Utils.timeStringAsSeconds(
- sc.conf.get(EXECUTOR_RESIZE_INTERVAL.key, "1m"))
+ val interval = sc.conf.get(EXECUTOR_RESIZE_INTERVAL)
+ if (interval <= 0) {
+ logInfo("ExecutorResizePlugin disabled (interval <= 0).")
+ return Map.empty[String, String].asJava
+ }
val threshold = sc.conf.get(EXECUTOR_RESIZE_THRESHOLD)
val factor = sc.conf.get(EXECUTOR_RESIZE_FACTOR)
val namespace = sc.conf.get(KUBERNETES_NAMESPACE)
diff --git
a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorResizePluginSuite.scala
b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorResizePluginSuite.scala
index e07d93ca107c..cfa4b0fd780a 100644
---
a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorResizePluginSuite.scala
+++
b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorResizePluginSuite.scala
@@ -17,6 +17,7 @@
package org.apache.spark.scheduler.cluster.k8s
import java.util.Collections
+import java.util.concurrent.ScheduledThreadPoolExecutor
import scala.jdk.CollectionConverters._
@@ -334,4 +335,35 @@ class ExecutorResizePluginSuite
}
}
}
+
+ test("SPARK-59842: resizeInterval defaults to 1 minute") {
+ assert(new SparkConf(false).get(EXECUTOR_RESIZE_INTERVAL) === 60)
+ }
+
+ test("SPARK-59842: init returns early when resizeInterval is 0") {
+ val plugin = new ExecutorResizeDriverPlugin()
+ val sparkConf = new SparkConf().set(EXECUTOR_RESIZE_INTERVAL.key, "0")
+ val sc = mock(classOf[SparkContext])
+ when(sc.conf).thenReturn(sparkConf)
+ val pluginCtx = mock(classOf[PluginContext])
+
+ val result = plugin.init(sc, pluginCtx)
+
+ assert(result.isEmpty)
+ }
+
+ test("SPARK-59842: init schedules the resize task by default") {
+ val plugin = new ExecutorResizeDriverPlugin()
+ val sc = mock(classOf[SparkContext])
+ when(sc.conf).thenReturn(new SparkConf())
+ val pluginCtx = mock(classOf[PluginContext])
+ try {
+ assert(plugin.init(sc, pluginCtx).isEmpty)
+ val field = plugin.getClass.getDeclaredField("periodicService")
+ field.setAccessible(true)
+
assert(field.get(plugin).asInstanceOf[ScheduledThreadPoolExecutor].getQueue.size
=== 1)
+ } finally {
+ plugin.shutdown()
+ }
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
