This is an automated email from the ASF dual-hosted git repository.
dongjoon-hyun pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/spark-kubernetes-operator.git
The following commit(s) were added to refs/heads/main by this push:
new e588fca [SPARK-58304] Simplify examples to show only feature-specific
configurations
e588fca is described below
commit e588fca22efdd104756086337e85fe9aacb0a503
Author: Dongjoon Hyun <[email protected]>
AuthorDate: Thu Jul 23 15:22:27 2026 -0700
[SPARK-58304] Simplify examples to show only feature-specific configurations
### What changes were proposed in this pull request?
This PR aims to simplify the example YAML files so that each file contains
only the
configurations relevant to the feature its name describes.
- Remove the `spark.dynamicAllocation.*` boilerplate from all examples
except `pi.yaml`.
- Remove the generic `applicationTolerations` boilerplate
(`resourceRetainPolicy`,
`ttlAfterStopMillis`) where it is not the subject of the example.
- Remove a few unrelated configurations (`spark.eventLog.*` from
`stream-word-count.yaml`,
`spark.logConf` from `dfs-read-write.yaml`).
The examples referenced by the e2e tests keep their
`applicationTolerations` unchanged.
### Why are the changes needed?
The repeated boilerplate obscured the actual point of each example. Now
each example shows only
the feature-specific configurations, making it easier to read, copy, and
maintain.
### Does this PR introduce _any_ user-facing change?
No.
### How was this patch tested?
Pass the CIs.
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: Claude Fable 5
Closes #772 from dongjoon-hyun/SPARK-58304.
Authored-by: Dongjoon Hyun <[email protected]>
Signed-off-by: Dongjoon Hyun <[email protected]>
---
examples/dfs-read-write.yaml | 6 ------
examples/pi-java17.yaml | 5 -----
examples/pi-java25-deployment.yaml | 3 ---
examples/pi-java25.yaml | 3 ---
examples/pi-on-volcano.yaml | 5 -----
examples/pi-on-yunikorn.yaml | 5 -----
examples/pi-python.yaml | 5 -----
examples/pi-statefulset.yaml | 2 --
examples/pi-v1beta1.yaml | 5 -----
examples/pi-with-comet.yaml | 3 ---
examples/pi-with-driver-timeout.yaml | 4 ----
examples/pi-with-eventlog.yaml | 6 ------
examples/pi-with-gluten.yaml | 3 ---
examples/pi-with-instanceconfig.yaml | 2 --
examples/pi-with-on-demand-pvc.yaml | 6 ------
examples/pi-with-spark-connect-plugin.yaml | 5 -----
examples/pi-with-template.yaml | 5 -----
examples/spark-connect-server-iceberg.yaml | 6 ------
examples/spark-connect-server-usw2-az2.yaml | 7 -------
examples/spark-connect-server.yaml | 6 ------
examples/spark-thrift-server.yaml | 4 ----
examples/sql.yaml | 3 ---
examples/stream-word-count.yaml | 6 ------
examples/word-count-celeborn.yaml | 3 ---
examples/word-count.yaml | 4 ----
25 files changed, 112 deletions(-)
diff --git a/examples/dfs-read-write.yaml b/examples/dfs-read-write.yaml
index 08f7d64..4c90c37 100644
--- a/examples/dfs-read-write.yaml
+++ b/examples/dfs-read-write.yaml
@@ -24,13 +24,9 @@ spec:
jars: "local:///opt/spark/examples/jars/spark-examples.jar"
driverArgs: [ "/opt/spark/RELEASE", "s3a://data/" ]
sparkConf:
- spark.logConf: "true"
spark.jars.packages: "org.apache.hadoop:hadoop-aws:3.4.2"
spark.jars.ivy: "/tmp/.ivy2.5.2"
spark.driver.memory: "2g"
- spark.dynamicAllocation.enabled: "true"
- spark.dynamicAllocation.shuffleTracking.enabled: "true"
- spark.dynamicAllocation.maxExecutors: "3"
spark.kubernetes.authenticate.driver.serviceAccountName: "spark"
spark.kubernetes.container.image: "apache/spark:{{SPARK_VERSION}}-scala"
spark.hadoop.fs.defaultFS: "s3a://data"
@@ -38,7 +34,5 @@ spec:
spark.hadoop.fs.s3a.path.style.access: "true"
spark.hadoop.fs.s3a.access.key: "test"
spark.hadoop.fs.s3a.secret.key: "test"
- applicationTolerations:
- resourceRetainPolicy: OnFailure
runtimeVersions:
sparkVersion: "4.2.0"
diff --git a/examples/pi-java17.yaml b/examples/pi-java17.yaml
index ed451a3..a8be134 100644
--- a/examples/pi-java17.yaml
+++ b/examples/pi-java17.yaml
@@ -20,12 +20,7 @@ spec:
mainClass: "org.apache.spark.examples.SparkPi"
jars: "local:///opt/spark/examples/jars/spark-examples.jar"
sparkConf:
- spark.dynamicAllocation.enabled: "true"
- spark.dynamicAllocation.shuffleTracking.enabled: "true"
- spark.dynamicAllocation.maxExecutors: "3"
spark.kubernetes.authenticate.driver.serviceAccountName: "spark"
spark.kubernetes.container.image:
"apache/spark:{{SPARK_VERSION}}-scala-java17"
- applicationTolerations:
- resourceRetainPolicy: OnFailure
runtimeVersions:
sparkVersion: "4.2.0"
diff --git a/examples/pi-java25-deployment.yaml
b/examples/pi-java25-deployment.yaml
index 5e55a11..5c32509 100644
--- a/examples/pi-java25-deployment.yaml
+++ b/examples/pi-java25-deployment.yaml
@@ -25,8 +25,5 @@ spec:
spark.kubernetes.allocation.pods.allocator: "deployment"
spark.kubernetes.authenticate.driver.serviceAccountName: "spark"
spark.kubernetes.container.image: "apache/spark:4.2.0-scala-java25"
- applicationTolerations:
- resourceRetainPolicy: OnFailure
- ttlAfterStopMillis: 10000
runtimeVersions:
sparkVersion: "4.2.0"
diff --git a/examples/pi-java25.yaml b/examples/pi-java25.yaml
index 8fe6723..072f7a1 100644
--- a/examples/pi-java25.yaml
+++ b/examples/pi-java25.yaml
@@ -20,9 +20,6 @@ spec:
mainClass: "org.apache.spark.examples.SparkPi"
jars: "local:///opt/spark/examples/jars/spark-examples.jar"
sparkConf:
- spark.dynamicAllocation.enabled: "true"
- spark.dynamicAllocation.shuffleTracking.enabled: "true"
- spark.dynamicAllocation.maxExecutors: "3"
spark.kubernetes.authenticate.driver.serviceAccountName: "spark"
spark.kubernetes.container.image: "apache/spark:4.2.0-scala-java25"
applicationTolerations:
diff --git a/examples/pi-on-volcano.yaml b/examples/pi-on-volcano.yaml
index 551a13b..c6b8673 100644
--- a/examples/pi-on-volcano.yaml
+++ b/examples/pi-on-volcano.yaml
@@ -21,16 +21,11 @@ spec:
jars: "local:///opt/spark/examples/jars/spark-examples.jar"
driverArgs: [ "20000" ]
sparkConf:
- spark.dynamicAllocation.enabled: "true"
- spark.dynamicAllocation.maxExecutors: "3"
- spark.dynamicAllocation.shuffleTracking.enabled: "true"
spark.kubernetes.authenticate.driver.serviceAccountName: "spark"
spark.kubernetes.container.image: "apache/spark:{{SPARK_VERSION}}-scala"
# This requires SPARK-54916 (or an image with `volcano`-enabled
distribution)
# spark.kubernetes.driver.pod.featureSteps:
"org.apache.spark.deploy.k8s.features.VolcanoFeatureStep"
# spark.kubernetes.executor.pod.featureSteps:
"org.apache.spark.deploy.k8s.features.VolcanoFeatureStep"
spark.kubernetes.scheduler.name: "volcano"
- applicationTolerations:
- resourceRetainPolicy: OnFailure
runtimeVersions:
sparkVersion: "4.2.0"
diff --git a/examples/pi-on-yunikorn.yaml b/examples/pi-on-yunikorn.yaml
index 103c712..494d870 100644
--- a/examples/pi-on-yunikorn.yaml
+++ b/examples/pi-on-yunikorn.yaml
@@ -21,9 +21,6 @@ spec:
jars: "local:///opt/spark/examples/jars/spark-examples.jar"
driverArgs: [ "20000" ]
sparkConf:
- spark.dynamicAllocation.enabled: "true"
- spark.dynamicAllocation.shuffleTracking.enabled: "true"
- spark.dynamicAllocation.maxExecutors: "3"
spark.kubernetes.authenticate.driver.serviceAccountName: "spark"
spark.kubernetes.container.image: "apache/spark:{{SPARK_VERSION}}-scala"
spark.kubernetes.scheduler.name: "yunikorn"
@@ -31,7 +28,5 @@ spec:
spark.kubernetes.executor.label.queue: "root.default"
spark.kubernetes.driver.annotation.yunikorn.apache.org/app-id: "{{APP_ID}}"
spark.kubernetes.executor.annotation.yunikorn.apache.org/app-id:
"{{APP_ID}}"
- applicationTolerations:
- resourceRetainPolicy: OnFailure
runtimeVersions:
sparkVersion: "4.2.0"
diff --git a/examples/pi-python.yaml b/examples/pi-python.yaml
index 16a6b49..dd93394 100644
--- a/examples/pi-python.yaml
+++ b/examples/pi-python.yaml
@@ -19,12 +19,7 @@ metadata:
spec:
pyFiles: "local:///opt/spark/examples/src/main/python/pi.py"
sparkConf:
- spark.dynamicAllocation.enabled: "true"
- spark.dynamicAllocation.shuffleTracking.enabled: "true"
- spark.dynamicAllocation.maxExecutors: "3"
spark.kubernetes.authenticate.driver.serviceAccountName: "spark"
spark.kubernetes.container.image: "apache/spark:{{SPARK_VERSION}}-python3"
- applicationTolerations:
- resourceRetainPolicy: OnFailure
runtimeVersions:
sparkVersion: "4.2.0"
diff --git a/examples/pi-statefulset.yaml b/examples/pi-statefulset.yaml
index 856c7f6..c065257 100644
--- a/examples/pi-statefulset.yaml
+++ b/examples/pi-statefulset.yaml
@@ -25,7 +25,5 @@ spec:
spark.kubernetes.allocation.pods.allocator: "statefulset"
spark.kubernetes.authenticate.driver.serviceAccountName: "spark"
spark.kubernetes.container.image: "apache/spark:{{SPARK_VERSION}}-scala"
- applicationTolerations:
- resourceRetainPolicy: OnFailure
runtimeVersions:
sparkVersion: "4.2.0"
diff --git a/examples/pi-v1beta1.yaml b/examples/pi-v1beta1.yaml
index 73d17e4..55c8b8a 100644
--- a/examples/pi-v1beta1.yaml
+++ b/examples/pi-v1beta1.yaml
@@ -20,12 +20,7 @@ spec:
mainClass: "org.apache.spark.examples.SparkPi"
jars: "local:///opt/spark/examples/jars/spark-examples.jar"
sparkConf:
- spark.dynamicAllocation.enabled: "true"
- spark.dynamicAllocation.shuffleTracking.enabled: "true"
- spark.dynamicAllocation.maxExecutors: "3"
spark.kubernetes.authenticate.driver.serviceAccountName: "spark"
spark.kubernetes.container.image: "apache/spark:{{SPARK_VERSION}}-scala"
- applicationTolerations:
- resourceRetainPolicy: OnFailure
runtimeVersions:
sparkVersion: "4.2.0"
diff --git a/examples/pi-with-comet.yaml b/examples/pi-with-comet.yaml
index e242710..f22ace1 100644
--- a/examples/pi-with-comet.yaml
+++ b/examples/pi-with-comet.yaml
@@ -25,9 +25,6 @@ spec:
spark.comet.exec.shuffle.enabled: "true"
spark.comet.exec.shuffle.mode: "auto"
spark.driver.extraClassPath: "local:///comet/comet.jar"
- spark.dynamicAllocation.enabled: "true"
- spark.dynamicAllocation.maxExecutors: "3"
- spark.dynamicAllocation.shuffleTracking.enabled: "true"
spark.executor.extraClassPath: "local:///comet/comet.jar"
spark.kubernetes.authenticate.driver.serviceAccountName: "spark"
spark.kubernetes.container.image: "apache/spark:4.1.2-scala"
diff --git a/examples/pi-with-driver-timeout.yaml
b/examples/pi-with-driver-timeout.yaml
index a9f02df..8db0646 100644
--- a/examples/pi-with-driver-timeout.yaml
+++ b/examples/pi-with-driver-timeout.yaml
@@ -23,13 +23,9 @@ spec:
sparkConf:
spark.driver.timeout: "1min" # Pod status will be 'Error'
spark.plugins: "org.apache.spark.deploy.DriverTimeoutPlugin"
- spark.dynamicAllocation.enabled: "true"
- spark.dynamicAllocation.shuffleTracking.enabled: "true"
- spark.dynamicAllocation.maxExecutors: "3"
spark.kubernetes.authenticate.driver.serviceAccountName: "spark"
spark.kubernetes.container.image: "apache/spark:{{SPARK_VERSION}}-scala"
applicationTolerations:
resourceRetainPolicy: OnFailure
- ttlAfterStopMillis: 10000
runtimeVersions:
sparkVersion: "4.2.0"
diff --git a/examples/pi-with-eventlog.yaml b/examples/pi-with-eventlog.yaml
index 9d3226c..a10df9f 100644
--- a/examples/pi-with-eventlog.yaml
+++ b/examples/pi-with-eventlog.yaml
@@ -23,9 +23,6 @@ spec:
sparkConf:
spark.jars.packages: "org.apache.hadoop:hadoop-aws:3.4.2"
spark.jars.ivy: "/tmp/.ivy2.5.2"
- spark.dynamicAllocation.enabled: "true"
- spark.dynamicAllocation.shuffleTracking.enabled: "true"
- spark.dynamicAllocation.maxExecutors: "3"
spark.kubernetes.authenticate.driver.serviceAccountName: "spark"
spark.kubernetes.container.image: "apache/spark:{{SPARK_VERSION}}-scala"
spark.eventLog.enabled: "true"
@@ -36,8 +33,5 @@ spec:
spark.hadoop.fs.s3a.access.key: "test"
spark.hadoop.fs.s3a.secret.key: "test"
spark.driver.memory: "2g"
- applicationTolerations:
- resourceRetainPolicy: OnFailure
- ttlAfterStopMillis: 10000
runtimeVersions:
sparkVersion: "4.2.0"
diff --git a/examples/pi-with-gluten.yaml b/examples/pi-with-gluten.yaml
index 14f37af..02f6243 100644
--- a/examples/pi-with-gluten.yaml
+++ b/examples/pi-with-gluten.yaml
@@ -21,9 +21,6 @@ spec:
jars: "local:///opt/spark/examples/jars/spark-examples.jar"
sparkConf:
spark.driver.extraClassPath: "local:///gluten/gluten.jar"
- spark.dynamicAllocation.enabled: "true"
- spark.dynamicAllocation.maxExecutors: "3"
- spark.dynamicAllocation.shuffleTracking.enabled: "true"
spark.executor.extraClassPath: "local:///gluten/gluten.jar"
spark.kubernetes.authenticate.driver.serviceAccountName: "spark"
spark.kubernetes.container.image: "apache/spark:4.0.3-scala"
diff --git a/examples/pi-with-instanceconfig.yaml
b/examples/pi-with-instanceconfig.yaml
index 5723921..dba8149 100644
--- a/examples/pi-with-instanceconfig.yaml
+++ b/examples/pi-with-instanceconfig.yaml
@@ -28,7 +28,5 @@ spec:
instanceConfig:
maxExecutors: 3
minExecutors: 3
- resourceRetainPolicy: OnFailure
- ttlAfterStopMillis: 10000
runtimeVersions:
sparkVersion: "4.2.0"
diff --git a/examples/pi-with-on-demand-pvc.yaml
b/examples/pi-with-on-demand-pvc.yaml
index a33e0b2..8080002 100644
--- a/examples/pi-with-on-demand-pvc.yaml
+++ b/examples/pi-with-on-demand-pvc.yaml
@@ -21,9 +21,6 @@ spec:
jars: "local:///opt/spark/examples/jars/spark-examples.jar"
driverArgs: [ "20000" ]
sparkConf:
- spark.dynamicAllocation.enabled: "true"
- spark.dynamicAllocation.shuffleTracking.enabled: "true"
- spark.dynamicAllocation.maxExecutors: "3"
spark.kubernetes.authenticate.driver.serviceAccountName: "spark"
spark.kubernetes.container.image: "apache/spark:{{SPARK_VERSION}}-scala"
spark.kubernetes.executor.volumes.persistentVolumeClaim.spark-local-dir-1.mount.path:
"/data"
@@ -31,8 +28,5 @@ spec:
spark.kubernetes.executor.volumes.persistentVolumeClaim.spark-local-dir-1.options.claimName:
"OnDemand"
spark.kubernetes.executor.volumes.persistentVolumeClaim.spark-local-dir-1.options.sizeLimit:
"1Gi"
spark.kubernetes.executor.volumes.persistentVolumeClaim.spark-local-dir-1.options.storageClass:
"local-path"
- applicationTolerations:
- resourceRetainPolicy: OnFailure
- ttlAfterStopMillis: 10000
runtimeVersions:
sparkVersion: "4.2.0"
diff --git a/examples/pi-with-spark-connect-plugin.yaml
b/examples/pi-with-spark-connect-plugin.yaml
index a39b1c2..cb7abfa 100644
--- a/examples/pi-with-spark-connect-plugin.yaml
+++ b/examples/pi-with-spark-connect-plugin.yaml
@@ -22,12 +22,7 @@ spec:
jars: "local:///opt/spark/examples/jars/spark-examples.jar"
sparkConf:
spark.plugins: "org.apache.spark.sql.connect.SparkConnectPlugin"
- spark.dynamicAllocation.enabled: "true"
- spark.dynamicAllocation.shuffleTracking.enabled: "true"
- spark.dynamicAllocation.maxExecutors: "3"
spark.kubernetes.authenticate.driver.serviceAccountName: "spark"
spark.kubernetes.container.image: "apache/spark:{{SPARK_VERSION}}-scala"
- applicationTolerations:
- resourceRetainPolicy: OnFailure
runtimeVersions:
sparkVersion: "4.2.0"
diff --git a/examples/pi-with-template.yaml b/examples/pi-with-template.yaml
index ef8d33a..79f8393 100644
--- a/examples/pi-with-template.yaml
+++ b/examples/pi-with-template.yaml
@@ -20,13 +20,8 @@ spec:
mainClass: "org.apache.spark.examples.SparkPi"
jars: "local:///opt/spark/examples/jars/spark-examples.jar"
sparkConf:
- spark.dynamicAllocation.enabled: "true"
- spark.dynamicAllocation.shuffleTracking.enabled: "true"
- spark.dynamicAllocation.maxExecutors: "3"
spark.kubernetes.authenticate.driver.serviceAccountName: "spark"
spark.kubernetes.container.image: "apache/spark:{{SPARK_VERSION}}-scala"
- applicationTolerations:
- resourceRetainPolicy: OnFailure
driverSpec:
podTemplateSpec:
spec:
diff --git a/examples/spark-connect-server-iceberg.yaml
b/examples/spark-connect-server-iceberg.yaml
index 76a325f..c82b32b 100644
--- a/examples/spark-connect-server-iceberg.yaml
+++ b/examples/spark-connect-server-iceberg.yaml
@@ -19,10 +19,6 @@ metadata:
spec:
mainClass: "org.apache.spark.sql.connect.service.SparkConnectServer"
sparkConf:
- spark.dynamicAllocation.enabled: "true"
- spark.dynamicAllocation.maxExecutors: "3"
- spark.dynamicAllocation.minExecutors: "3"
- spark.dynamicAllocation.shuffleTracking.enabled: "true"
spark.hadoop.fs.s3a.access.key: "test"
spark.hadoop.fs.s3a.endpoint: "http://localstack:4566";
spark.hadoop.fs.s3a.path.style.access: "true"
@@ -38,7 +34,5 @@ spec:
spark.sql.catalog.s3: "org.apache.iceberg.spark.SparkCatalog"
spark.sql.defaultCatalog: "s3"
spark.sql.extensions:
"org.apache.iceberg.spark.extensions.IcebergSparkSessionExtensions"
- applicationTolerations:
- resourceRetainPolicy: OnFailure
runtimeVersions:
sparkVersion: "4.1.2"
diff --git a/examples/spark-connect-server-usw2-az2.yaml
b/examples/spark-connect-server-usw2-az2.yaml
index 9716098..21c9997 100644
--- a/examples/spark-connect-server-usw2-az2.yaml
+++ b/examples/spark-connect-server-usw2-az2.yaml
@@ -19,10 +19,6 @@ metadata:
spec:
mainClass: "org.apache.spark.sql.connect.service.SparkConnectServer"
sparkConf:
- spark.dynamicAllocation.enabled: "true"
- spark.dynamicAllocation.shuffleTracking.enabled: "true"
- spark.dynamicAllocation.minExecutors: "3"
- spark.dynamicAllocation.maxExecutors: "3"
spark.kubernetes.authenticate.driver.serviceAccountName: "spark"
spark.kubernetes.container.image: "apache/spark:{{SPARK_VERSION}}"
spark.kubernetes.executor.podNamePrefix: "spark-connect-server"
@@ -37,8 +33,5 @@ spec:
spec:
nodeSelector:
topology.k8s.aws/zone-id: usw2-az2
- applicationTolerations:
- resourceRetainPolicy: OnFailure
- ttlAfterStopMillis: 10000
runtimeVersions:
sparkVersion: "4.2.0"
diff --git a/examples/spark-connect-server.yaml
b/examples/spark-connect-server.yaml
index 48da081..f6312f3 100644
--- a/examples/spark-connect-server.yaml
+++ b/examples/spark-connect-server.yaml
@@ -19,15 +19,9 @@ metadata:
spec:
mainClass: "org.apache.spark.sql.connect.service.SparkConnectServer"
sparkConf:
- spark.dynamicAllocation.enabled: "true"
- spark.dynamicAllocation.shuffleTracking.enabled: "true"
- spark.dynamicAllocation.minExecutors: "3"
- spark.dynamicAllocation.maxExecutors: "3"
spark.kubernetes.authenticate.driver.serviceAccountName: "spark"
spark.kubernetes.container.image: "apache/spark:{{SPARK_VERSION}}"
spark.kubernetes.executor.podNamePrefix: "spark-connect-server"
spark.scheduler.mode: "FAIR"
- applicationTolerations:
- resourceRetainPolicy: OnFailure
runtimeVersions:
sparkVersion: "4.2.0"
diff --git a/examples/spark-thrift-server.yaml
b/examples/spark-thrift-server.yaml
index 6fc05e0..2d494d8 100644
--- a/examples/spark-thrift-server.yaml
+++ b/examples/spark-thrift-server.yaml
@@ -19,10 +19,6 @@ metadata:
spec:
mainClass: "org.apache.spark.sql.hive.thriftserver.HiveThriftServer2"
sparkConf:
- spark.dynamicAllocation.enabled: "true"
- spark.dynamicAllocation.shuffleTracking.enabled: "true"
- spark.dynamicAllocation.minExecutors: "3"
- spark.dynamicAllocation.maxExecutors: "3"
spark.kubernetes.authenticate.driver.serviceAccountName: "spark"
spark.kubernetes.container.image: "apache/spark:{{SPARK_VERSION}}-scala"
spark.kubernetes.executor.podNamePrefix: "spark-thrift-server"
diff --git a/examples/sql.yaml b/examples/sql.yaml
index 0eb05ea..68761ae 100644
--- a/examples/sql.yaml
+++ b/examples/sql.yaml
@@ -21,9 +21,6 @@ spec:
jars: "local:///opt/spark/examples/jars/spark-examples.jar"
driverArgs: [ "SHOW DATABASES", "SHOW TABLES", "SELECT VERSION()" ]
sparkConf:
- spark.dynamicAllocation.enabled: "true"
- spark.dynamicAllocation.shuffleTracking.enabled: "true"
- spark.dynamicAllocation.maxExecutors: "3"
spark.kubernetes.authenticate.driver.serviceAccountName: "spark"
spark.kubernetes.container.image: "apache/spark:{{SPARK_VERSION}}-scala"
runtimeVersions:
diff --git a/examples/stream-word-count.yaml b/examples/stream-word-count.yaml
index 8e368b5..6b9799e 100644
--- a/examples/stream-word-count.yaml
+++ b/examples/stream-word-count.yaml
@@ -23,15 +23,9 @@ spec:
sparkConf:
spark.jars.packages: "org.apache.hadoop:hadoop-aws:3.4.2"
spark.jars.ivy: "/tmp/.ivy2.5.2"
- spark.dynamicAllocation.enabled: "true"
- spark.dynamicAllocation.shuffleTracking.enabled: "true"
- spark.dynamicAllocation.maxExecutors: "3"
spark.kubernetes.authenticate.driver.serviceAccountName: "spark"
spark.kubernetes.container.image: "apache/spark:{{SPARK_VERSION}}-scala"
spark.log.level: "WARN"
- spark.eventLog.enabled: "true"
- spark.eventLog.dir: "s3a://spark-events/"
- spark.eventLog.rolling.maxFileSize: "10m"
spark.hadoop.fs.s3a.endpoint: "http://localstack:4566";
spark.hadoop.fs.s3a.path.style.access: "true"
spark.hadoop.fs.s3a.access.key: "test"
diff --git a/examples/word-count-celeborn.yaml
b/examples/word-count-celeborn.yaml
index 94f38aa..0882276 100644
--- a/examples/word-count-celeborn.yaml
+++ b/examples/word-count-celeborn.yaml
@@ -33,8 +33,5 @@ spec:
spark.shuffle.service.enabled: "false"
spark.sql.adaptive.localShuffleReader.enabled: "false"
spark.serializer: "org.apache.spark.serializer.KryoSerializer"
- applicationTolerations:
- resourceRetainPolicy: OnFailure
- ttlAfterStopMillis: 10000
runtimeVersions:
sparkVersion: "4.0.3"
diff --git a/examples/word-count.yaml b/examples/word-count.yaml
index 8072bfe..cbbb580 100644
--- a/examples/word-count.yaml
+++ b/examples/word-count.yaml
@@ -21,11 +21,7 @@ spec:
driverArgs: [ "/opt/spark/RELEASE" ]
jars: "local:///opt/spark/examples/jars/spark-examples.jar"
sparkConf:
- spark.dynamicAllocation.enabled: "true"
- spark.dynamicAllocation.maxExecutors: "3"
spark.kubernetes.authenticate.driver.serviceAccountName: "spark"
spark.kubernetes.container.image: "apache/spark:{{SPARK_VERSION}}-scala"
- applicationTolerations:
- resourceRetainPolicy: OnFailure
runtimeVersions:
sparkVersion: "4.2.0"
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
