This is an automated email from the ASF dual-hosted git repository.
dongjoon-hyun pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/spark-kubernetes-operator.git
The following commit(s) were added to refs/heads/main by this push:
new 04b1b9f [SPARK-58284] Remove duplicated NetworkPolicy creation in
favor of `NetworkPolicyFeatureStep` of Spark 4.2
04b1b9f is described below
commit 04b1b9f5ecca98e17c559cc5f0dd13fdc0824ef9
Author: Dongjoon Hyun <[email protected]>
AuthorDate: Thu Jul 23 00:03:13 2026 -0700
[SPARK-58284] Remove duplicated NetworkPolicy creation in favor of
`NetworkPolicyFeatureStep` of Spark 4.2
### What changes were proposed in this pull request?
This PR aims to remove the operator-side `NetworkPolicy` creation from
`SparkAppResourceSpec` in favor of Apache Spark 4.2.0's built-in
`NetworkPolicyFeatureStep`.
- Remove `buildNetworkPolicy` and its invocation from
`SparkAppResourceSpec`.
- Remove the corresponding `testNetworkPolicy` test case and adjust the
pre-resource count assertion.
- Keep the `networkpolicies` RBAC rule in the Helm chart because the
operator still needs it to create the `NetworkPolicy` generated by Spark's
`NetworkPolicyFeatureStep`.
### Why are the changes needed?
The operator has created a `NetworkPolicy` per `SparkApplication` since
SPARK-55085. After SPARK-58194 upgraded the Spark dependency to 4.2.0, Spark's
`KubernetesDriverBuilder` also generates a functionally identical
`NetworkPolicy` via `NetworkPolicyFeatureStep` (SPARK-55653). As a result,
every `SparkApplication` ends up with two duplicated policies.
- https://github.com/apache/spark/pull/54442
```
$ kubectl get networkpolicy
NAME POD-SELECTOR AGE
pi-0-network-policy spark-app-selector=pi-0,spark-role=executor 41s
pi-0-policy spark-app-selector=pi-0,spark-role=executor 40s
```
Both policies have the same pod selector and ingress rule (allowing traffic
only from the pods of the same Spark application). Delegating to Spark's
`NetworkPolicyFeatureStep` removes the duplication and avoids maintaining the
same logic in two places.
### Does this PR introduce _any_ user-facing change?
Yes, only one `NetworkPolicy` (`<appId>-policy`, created by Spark) is
deployed per `SparkApplication` instead of two. The security behavior is
unchanged because the two policies were functionally identical.
### How was this patch tested?
Pass the CIs.
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: Claude Fable 5
Closes #764 from dongjoon-hyun/SPARK-58284.
Authored-by: Dongjoon Hyun <[email protected]>
Signed-off-by: Dongjoon Hyun <[email protected]>
---
.../spark/k8s/operator/SparkAppResourceSpec.java | 33 --------------
.../k8s/operator/SparkAppResourceSpecTest.java | 50 +---------------------
2 files changed, 1 insertion(+), 82 deletions(-)
diff --git
a/spark-submission-worker/src/main/java/org/apache/spark/k8s/operator/SparkAppResourceSpec.java
b/spark-submission-worker/src/main/java/org/apache/spark/k8s/operator/SparkAppResourceSpec.java
index 5b50d54..c47c482 100644
---
a/spark-submission-worker/src/main/java/org/apache/spark/k8s/operator/SparkAppResourceSpec.java
+++
b/spark-submission-worker/src/main/java/org/apache/spark/k8s/operator/SparkAppResourceSpec.java
@@ -27,8 +27,6 @@ import io.fabric8.kubernetes.api.model.ContainerBuilder;
import io.fabric8.kubernetes.api.model.HasMetadata;
import io.fabric8.kubernetes.api.model.Pod;
import io.fabric8.kubernetes.api.model.PodBuilder;
-import io.fabric8.kubernetes.api.model.networking.v1.NetworkPolicy;
-import io.fabric8.kubernetes.api.model.networking.v1.NetworkPolicyBuilder;
import lombok.Getter;
import org.apache.spark.deploy.k8s.Config;
@@ -103,7 +101,6 @@ public class SparkAppResourceSpec {
KubernetesClientUtils.buildConfigMapJava(
kubernetesDriverConf.configMapNameDriver(), confFilesMap,
Map.of()));
this.driverPreResources.addAll(ConfigMapSpecUtils.buildConfigMaps(configMapSpecs));
-
this.driverPreResources.add(buildNetworkPolicy(kubernetesDriverConf.appId(),
namespace));
this.driverResources.addAll(configureDriverServerIngress(sparkPod,
driverServiceIngressList));
this.driverResources.addAll(configureDriverHttpRoutes(sparkPod,
driverHttpRouteList));
this.driverResources.addAll(configureDriverGrpcRoutes(sparkPod,
driverGrpcRouteList));
@@ -211,34 +208,4 @@ public class SparkAppResourceSpec {
.flatMap(Collection::stream)
.collect(Collectors.toList());
}
-
- /**
- * Builds the NetworkPolicy for the SparkApplication.
- *
- * @param appId The application ID of the SparkApplication.
- * @param namespace The namespace of the SparkApplication.
- * @return A NetworkPolicy object.
- */
- private NetworkPolicy buildNetworkPolicy(String appId, String namespace) {
- return new NetworkPolicyBuilder()
- .withNewMetadata()
- .withName(appId + "-network-policy")
- .withNamespace(namespace)
-
.addToLabels(org.apache.spark.k8s.operator.Constants.LABEL_SPARK_APPLICATION_NAME,
appId)
- .endMetadata()
- .withNewSpec()
- .withNewPodSelector()
- .addToMatchLabels(Constants.SPARK_ROLE_LABEL(),
Constants.SPARK_POD_EXECUTOR_ROLE())
- .addToMatchLabels(Constants.SPARK_APP_ID_LABEL(), appId)
- .endPodSelector()
- .addNewIngress()
- .addNewFrom()
- .withNewPodSelector()
- .addToMatchLabels(Constants.SPARK_APP_ID_LABEL(), appId)
- .endPodSelector()
- .endFrom()
- .endIngress()
- .endSpec()
- .build();
- }
}
diff --git
a/spark-submission-worker/src/test/java/org/apache/spark/k8s/operator/SparkAppResourceSpecTest.java
b/spark-submission-worker/src/test/java/org/apache/spark/k8s/operator/SparkAppResourceSpecTest.java
index b241eaf..243dd1b 100644
---
a/spark-submission-worker/src/test/java/org/apache/spark/k8s/operator/SparkAppResourceSpecTest.java
+++
b/spark-submission-worker/src/test/java/org/apache/spark/k8s/operator/SparkAppResourceSpecTest.java
@@ -33,8 +33,6 @@ import io.fabric8.kubernetes.api.model.Pod;
import io.fabric8.kubernetes.api.model.PodBuilder;
import io.fabric8.kubernetes.api.model.Volume;
import io.fabric8.kubernetes.api.model.VolumeMount;
-import io.fabric8.kubernetes.api.model.networking.v1.NetworkPolicy;
-import io.fabric8.kubernetes.api.model.networking.v1.NetworkPolicyBuilder;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
@@ -66,7 +64,7 @@ class SparkAppResourceSpecTest {
new SparkAppResourceSpec(mockConf, spec, List.of(), List.of(),
List.of(), List.of());
Assertions.assertEquals(2, appResourceSpec.getDriverResources().size());
- Assertions.assertEquals(2, appResourceSpec.getDriverPreResources().size());
+ Assertions.assertEquals(1, appResourceSpec.getDriverPreResources().size());
Assertions.assertEquals(Pod.class,
appResourceSpec.getDriverResources().get(0).getClass());
Assertions.assertEquals(
ConfigMap.class,
appResourceSpec.getDriverResources().get(1).getClass());
@@ -104,52 +102,6 @@ class SparkAppResourceSpecTest {
Assertions.assertEquals(proposedConfigVolume.getName(),
proposedConfigVolumeMount.getName());
}
- @Test
- void testNetworkPolicy() {
- SparkAppDriverConf mockConf = mock(SparkAppDriverConf.class);
- when(mockConf.appId()).thenReturn("app1");
- when(mockConf.configMapNameDriver()).thenReturn("foo-network-policy");
- when(mockConf.sparkConf())
- .thenReturn(new SparkConf().set("spark.kubernetes.namespace",
"foo-namespace"));
-
- Pod driver = buildBasicPod("driver");
- SparkPod sparkPod = new SparkPod(driver, buildBasicContainer());
-
- KubernetesDriverSpec spec =
- KubernetesDriverSpec.create(sparkPod, List.of(), List.of(), Map.of());
-
- SparkAppResourceSpec appResourceSpec =
- new SparkAppResourceSpec(mockConf, spec, List.of(), List.of(),
List.of(), List.of());
-
- Assertions.assertEquals(1, appResourceSpec.getDriverPreResources().size());
- Assertions.assertEquals(NetworkPolicy.class,
- appResourceSpec.getDriverPreResources().get(0).getClass());
-
- NetworkPolicy expected = new NetworkPolicyBuilder()
- .withNewMetadata()
- .withName("app1-network-policy")
- .withNamespace("foo-namespace")
- .addToLabels(Constants.LABEL_SPARK_APPLICATION_NAME, "app1")
- .endMetadata()
- .withNewSpec()
- .withNewPodSelector()
- .addToMatchLabels(Constants.LABEL_SPARK_ROLE_NAME,
- Constants.LABEL_SPARK_ROLE_EXECUTOR_VALUE)
- .addToMatchLabels("spark-app-selector", "app1")
- .endPodSelector()
- .addNewIngress()
- .addNewFrom()
- .withNewPodSelector()
- .addToMatchLabels("spark-app-selector", "app1")
- .endPodSelector()
- .endFrom()
- .endIngress()
- .endSpec()
- .build();
- Assertions.assertEquals(1, appResourceSpec.getDriverPreResources().size());
- Assertions.assertEquals(expected,
appResourceSpec.getDriverPreResources().get(0));
- }
-
protected Container buildBasicContainer() {
return new ContainerBuilder()
.withName("foo-container")
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
