This is an automated email from the ASF dual-hosted git repository.
dongjoon-hyun pushed a commit to branch branch-4.3
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-4.3 by this push:
new d0fe25bb9a90 [SPARK-59914][K8S] Make `ExecutorResizePlugin` skip
executor pods with the placeholder `EXECID` label
d0fe25bb9a90 is described below
commit d0fe25bb9a909e9eee7072fa41e5e81f42e84b0a
Author: Dongjoon Hyun <[email protected]>
AuthorDate: Wed Sep 30 17:55:42 2026 -0700
[SPARK-59914][K8S] Make `ExecutorResizePlugin` skip executor pods with the
placeholder `EXECID` label
---
.../scheduler/cluster/k8s/ExecutorResizePlugin.scala | 13 +++++++++++--
.../cluster/k8s/ExecutorResizePluginSuite.scala | 18 ++++++++++++++++++
2 files changed, 29 insertions(+), 2 deletions(-)
diff --git
a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorResizePlugin.scala
b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorResizePlugin.scala
index 0b9e33672827..7b29448c9b76 100644
---
a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorResizePlugin.scala
+++
b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorResizePlugin.scala
@@ -106,8 +106,17 @@ class ExecutorResizeDriverPlugin extends DriverPlugin with
Logging {
.list()
.getItems.asScala
- pods.filter(_.getMetadata.getLabels.get(SPARK_EXECUTOR_ID_LABEL) !=
null).foreach { pod =>
- val execId = pod.getMetadata.getLabels.get(SPARK_EXECUTOR_ID_LABEL)
+ val executorPods = pods.flatMap { pod =>
+ pod.getMetadata.getLabels.get(SPARK_EXECUTOR_ID_LABEL) match {
+ case "EXECID" | null =>
+ // The exec label has not yet been assigned
+ None
+ case id =>
+ Some((id, pod))
+ }
+ }
+
+ executorPods.foreach { case (execId, pod) =>
try {
val metrics = kubernetesClient.top().pods().metrics(namespace,
pod.getMetadata.getName)
diff --git
a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorResizePluginSuite.scala
b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorResizePluginSuite.scala
index cfa4b0fd780a..b921cc6854d4 100644
---
a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorResizePluginSuite.scala
+++
b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorResizePluginSuite.scala
@@ -149,6 +149,24 @@ class ExecutorResizePluginSuite
verify(podMetricOperations, never()).metrics(anyString(), anyString())
}
+ test("Pod with placeholder executor ID label should be skipped") {
+ val plugin = createPlugin()
+ val pod = new PodBuilder()
+ .withNewMetadata()
+ .withName("spark-executor-1")
+ .addToLabels(SPARK_APP_ID_LABEL, appId)
+ .addToLabels(SPARK_ROLE_LABEL, SPARK_POD_EXECUTOR_ROLE)
+ .addToLabels(SPARK_EXECUTOR_ID_LABEL, "EXECID")
+ .endMetadata()
+ .build()
+
+ when(podList.getItems).thenReturn(Collections.singletonList(pod))
+
+ plugin.invokePrivate(_checkAndIncreaseMemory(namespace, 0.9, 0.1,
kubernetesClient))
+
+ verify(podMetricOperations, never()).metrics(anyString(), anyString())
+ }
+
test("SPARK-59840: Inactive executor pods are excluded from the listing") {
val plugin = createPlugin()
when(podList.getItems).thenReturn(Collections.emptyList())
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
