This is an automated email from the ASF dual-hosted git repository.
pan3793 pushed a commit to branch branch-4.x
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-4.x by this push:
new 12b31f5440bc [SPARK-56793][K8S] Avoid cluster-wide LIST in executor
pods polling
12b31f5440bc is described below
commit 12b31f5440bc9234fc8376ee96fd59047c5806e5
Author: TongWei <[email protected]>
AuthorDate: Tue May 12 14:32:07 2026 +0800
[SPARK-56793][K8S] Avoid cluster-wide LIST in executor pods polling
### What changes were proposed in this pull request?
Scope the executor pod LIST issued by `ExecutorPodsPollingSnapshotSource`
to the configured Kubernetes namespace by inserting `.inNamespace(namespace)`
between `.pods()` and the label filters.
### Why are the changes needed?
Without `.inNamespace(...)` the fabric8 client issues a cluster-wide LIST
against the K8s API server. Other paths in the K8s scheduler module (e.g.
`KubernetesClusterSchedulerBackend.doKillExecutors`,
`ExecutorPodsLifecycleManager`) already scope their pod operations to the
configured namespace; the polling source was inconsistent.
A cluster-wide LIST:
- fails under the typical least-privilege deployment where the driver
ServiceAccount is bound to a namespaced Role rather than a ClusterRole;
- causes unnecessary load and broadens the visibility surface even when
ClusterRole permissions are granted.
### Does this PR introduce _any_ user-facing change?
No.
### How was this patch tested?
Updated `ExecutorPodsPollingSnapshotSourceSuite` to mock the new
`.inNamespace(...)` link in the chain. Ran the full
`resource-managers/kubernetes/core` test module: 344 tests across 42 suites,
all passing.
### Was this patch authored or co-authored using generative AI tooling?
Yes, Generated-by: Claude Code 4.7
Closes #55754 from TongWei1105/spark-k8s-scope-poller-namespace.
Authored-by: TongWei <[email protected]>
Signed-off-by: Cheng Pan <[email protected]>
(cherry picked from commit 73df934271edf8db5104810796594d505b353162)
Signed-off-by: Cheng Pan <[email protected]>
---
.../scheduler/cluster/k8s/ExecutorPodsPollingSnapshotSource.scala | 2 ++
.../cluster/k8s/ExecutorPodsPollingSnapshotSourceSuite.scala | 7 ++++++-
2 files changed, 8 insertions(+), 1 deletion(-)
diff --git
a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorPodsPollingSnapshotSource.scala
b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorPodsPollingSnapshotSource.scala
index 4ed34ec3e4c0..3d2822e5eb51 100644
---
a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorPodsPollingSnapshotSource.scala
+++
b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorPodsPollingSnapshotSource.scala
@@ -47,6 +47,7 @@ class ExecutorPodsPollingSnapshotSource(
private val pollingInterval =
conf.get(KUBERNETES_EXECUTOR_API_POLLING_INTERVAL)
private val pollingEnabled = conf.get(KUBERNETES_EXECUTOR_ENABLE_API_POLLING)
+ private val namespace = conf.get(KUBERNETES_NAMESPACE)
private var pollingFuture: Future[_] = _
@@ -76,6 +77,7 @@ class ExecutorPodsPollingSnapshotSource(
logDebug(s"Resynchronizing full executor pod state from Kubernetes.")
val pods = kubernetesClient
.pods()
+ .inNamespace(namespace)
.withLabel(SPARK_APP_ID_LABEL, applicationId)
.withLabel(SPARK_ROLE_LABEL, SPARK_POD_EXECUTOR_ROLE)
.withoutLabel(SPARK_EXECUTOR_INACTIVE_LABEL, "true")
diff --git
a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorPodsPollingSnapshotSourceSuite.scala
b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorPodsPollingSnapshotSourceSuite.scala
index e0016a2ae050..71c187a9caf8 100644
---
a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorPodsPollingSnapshotSourceSuite.scala
+++
b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorPodsPollingSnapshotSourceSuite.scala
@@ -43,6 +43,9 @@ class ExecutorPodsPollingSnapshotSourceSuite extends
SparkFunSuite with BeforeAn
@Mock
private var podOperations: PODS = _
+ @Mock
+ private var namespacedPodOperations: PODS_WITH_NAMESPACE = _
+
@Mock
private var appIdLabeledPods: LABELED_PODS = _
@@ -62,7 +65,9 @@ class ExecutorPodsPollingSnapshotSourceSuite extends
SparkFunSuite with BeforeAn
MockitoAnnotations.openMocks(this).close()
pollingExecutor = new DeterministicScheduler()
when(kubernetesClient.pods()).thenReturn(podOperations)
- when(podOperations.withLabel(SPARK_APP_ID_LABEL, TEST_SPARK_APP_ID))
+ when(podOperations.inNamespace(defaultConf.get(KUBERNETES_NAMESPACE)))
+ .thenReturn(namespacedPodOperations)
+ when(namespacedPodOperations.withLabel(SPARK_APP_ID_LABEL,
TEST_SPARK_APP_ID))
.thenReturn(appIdLabeledPods)
when(appIdLabeledPods.withLabel(SPARK_ROLE_LABEL, SPARK_POD_EXECUTOR_ROLE))
.thenReturn(executorRoleLabeledPods)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
