dongjoon-hyun commented on PR #44:
URL:
https://github.com/apache/spark-kubernetes-operator/pull/44#issuecomment-2292350377
I revised this PR, @jiangzho and @viirya .
- Use Apache Spark 4.0.0-preview1 as the example.
- Use `helm install` instead of `helm upgrade` to avoid the initial failure.
- Add Spark Job deletion example, `kubectl delete -f spark-pi.yaml`
- Add `--strict` on helm linting.
- Renaming during copy CRDs, `rename '(.+).yml', '$1.yaml'`.
- Minor code clean up to focus on the main feature.
```
diff --git a/.github/.licenserc.yaml b/.github/.licenserc.yaml
index 6cafe63..5507e4d 100644
--- a/.github/.licenserc.yaml
+++ b/.github/.licenserc.yaml
@@ -19,6 +19,5 @@ header:
- 'build/**'
- '**/*.json'
- '**/.helmignore'
- - '**/*.yml'
comment: on-failure
diff --git a/.github/workflows/build_and_test.yml
b/.github/workflows/build_and_test.yml
index 2ca7bd2..fd99dfc 100644
--- a/.github/workflows/build_and_test.yml
+++ b/.github/workflows/build_and_test.yml
@@ -46,7 +46,7 @@ jobs:
./gradlew build
- name: Validate helm chart linting
run: |
- helm lint build-tools/helm/spark-kubernetes-operator
+ helm lint --strict build-tools/helm/spark-kubernetes-operator
build-image:
name: "Build Operator Image CI"
runs-on: ubuntu-latest
diff --git a/.gitignore b/.gitignore
index d5afef7..b174a96 100644
--- a/.gitignore
+++ b/.gitignore
@@ -6,6 +6,7 @@
.vscode
/lib/
target/
+build-tools/helm/spark-kubernetes-operator/crds
# Gradle Files #
################
@@ -16,7 +17,3 @@ build
dependencies.lock
**/dependencies.lock
gradle/wrapper/gradle-wrapper.jar
-
-# Generated Files #
-###################
-build-tools/helm/spark-kubernetes-operator/crds
diff --git a/build-tools/helm/spark-kubernetes-operator/.helmignore
b/build-tools/helm/spark-kubernetes-operator/.helmignore
index 0e8a0eb..5920392 100644
--- a/build-tools/helm/spark-kubernetes-operator/.helmignore
+++ b/build-tools/helm/spark-kubernetes-operator/.helmignore
@@ -1,15 +1,4 @@
-# Patterns to ignore when building packages.
-# This supports shell glob matching, relative path matching, and
-# negation (prefixed with !). Only one pattern per line.
.DS_Store
-# Common VCS dirs
-.git/
-.gitignore
-.bzr/
-.bzrignore
-.hg/
-.hgignore
-.svn/
# Common backup files
*.swp
*.bak
diff --git a/build-tools/helm/spark-kubernetes-operator/Chart.yaml
b/build-tools/helm/spark-kubernetes-operator/Chart.yaml
index 46e612a..deed565 100644
--- a/build-tools/helm/spark-kubernetes-operator/Chart.yaml
+++ b/build-tools/helm/spark-kubernetes-operator/Chart.yaml
@@ -1,4 +1,3 @@
-################################################################################
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
@@ -13,9 +12,6 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-################################################################################
-
----
apiVersion: v2
name: spark-kubernetes-operator
description: A Helm chart for the Apache Spark Kubernetes Operator
diff --git
a/build-tools/helm/spark-kubernetes-operator/conf/log4j2.properties
b/build-tools/helm/spark-kubernetes-operator/conf/log4j2.properties
deleted file mode 100644
index 3a0256b..0000000
--- a/build-tools/helm/spark-kubernetes-operator/conf/log4j2.properties
+++ /dev/null
@@ -1,51 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-status=info
-strict=true
-dest=out
-name=PropertiesConfig
-property.filename=/opt/spark-operator/logs/spark-operator
-filter.threshold.type=ThresholdFilter
-filter.threshold.level=debug
-# console
-appender.console.type=Console
-appender.console.name=STDOUT
-appender.console.layout.type=PatternLayout
-appender.console.layout.pattern=%d %p %X %C{1.} [%t] %m%n
-appender.console.filter.threshold.type=ThresholdFilter
-appender.console.filter.threshold.level=info
-# rolling JSON
-appender.rolling.type=RollingFile
-appender.rolling.name=RollingFile
-appender.rolling.append=true
-appender.rolling.fileName=${filename}.log
-appender.rolling.filePattern=${filename}-%i.log.gz
-appender.rolling.layout.type=JsonTemplateLayout
-appender.rolling.layout.eventTemplateUri=classpath:EcsLayout.json
-appender.rolling.policies.type=Policies
-appender.rolling.policies.size.type=SizeBasedTriggeringPolicy
-appender.rolling.policies.size.size=100MB
-appender.rolling.strategy.type=DefaultRolloverStrategy
-appender.rolling.strategy.max=20
-appender.rolling.immediateFlush=true
-# chatty loggers
-rootLogger.level=all
-logger.netty.name=io.netty
-logger.netty.level=warn
-log4j2.contextSelector=org.apache.logging.log4j.core.async.AsyncLoggerContextSelector
-rootLogger.appenderRef.stdout.ref=STDOUT
-rootLogger.appenderRef.rolling.ref=RollingFile
diff --git
a/build-tools/helm/spark-kubernetes-operator/conf/spark-operator.properties
b/build-tools/helm/spark-kubernetes-operator/conf/spark-operator.properties
deleted file mode 100644
index e644970..0000000
---
a/build-tools/helm/spark-kubernetes-operator/conf/spark-operator.properties
+++ /dev/null
@@ -1,19 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-# Property Overrides. e.g.
-# spark.kubernetes.operator.reconciler.intervalSeconds=60
diff --git
a/build-tools/helm/spark-kubernetes-operator/templates/_helpers.tpl
b/build-tools/helm/spark-kubernetes-operator/templates/_helpers.tpl
index 526b778..e8ee901 100644
--- a/build-tools/helm/spark-kubernetes-operator/templates/_helpers.tpl
+++ b/build-tools/helm/spark-kubernetes-operator/templates/_helpers.tpl
@@ -1,4 +1,3 @@
-################################################################################
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
@@ -13,7 +12,6 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-################################################################################
{{/*
Expand the name of the chart.
diff --git
a/build-tools/helm/spark-kubernetes-operator/templates/sparkapps-resource.yaml
b/build-tools/helm/spark-kubernetes-operator/templates/app-rbac.yaml
similarity index 96%
rename from
build-tools/helm/spark-kubernetes-operator/templates/sparkapps-resource.yaml
rename to build-tools/helm/spark-kubernetes-operator/templates/app-rbac.yaml
index 613e489..f76d56b 100644
---
a/build-tools/helm/spark-kubernetes-operator/templates/sparkapps-resource.yaml
+++ b/build-tools/helm/spark-kubernetes-operator/templates/app-rbac.yaml
@@ -1,4 +1,3 @@
-################################################################################
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
@@ -13,7 +12,6 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-################################################################################
{{/*
RBAC rules used to create the app (cluster)role based on the scope
diff --git a/build-tools/helm/spark-kubernetes-operator/templates/rbac.yaml
b/build-tools/helm/spark-kubernetes-operator/templates/operator-rbac.yaml
similarity index 95%
rename from build-tools/helm/spark-kubernetes-operator/templates/rbac.yaml
rename to
build-tools/helm/spark-kubernetes-operator/templates/operator-rbac.yaml
index e03809e..f8364e6 100644
--- a/build-tools/helm/spark-kubernetes-operator/templates/rbac.yaml
+++ b/build-tools/helm/spark-kubernetes-operator/templates/operator-rbac.yaml
@@ -1,4 +1,3 @@
-################################################################################
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
@@ -13,7 +12,6 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-################################################################################
{{/*
RBAC rules used to create the operator (cluster)role
@@ -61,7 +59,7 @@ Labels and annotations to be applied on rbacResources
{{- end }}
---
-#Service account and rolebindings for operator
+# Service account and rolebindings for operator
apiVersion: v1
kind: ServiceAccount
metadata:
diff --git
a/build-tools/helm/spark-kubernetes-operator/templates/spark-operator.yaml
b/build-tools/helm/spark-kubernetes-operator/templates/spark-operator.yaml
index 555b239..5edd228 100644
---
a/build-tools/helm/spark-kubernetes-operator/templates/spark-operator.yaml
+++
b/build-tools/helm/spark-kubernetes-operator/templates/spark-operator.yaml
@@ -1,4 +1,3 @@
-################################################################################
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
@@ -13,7 +12,6 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-################################################################################
apiVersion: apps/v1
kind: Deployment
@@ -25,7 +23,6 @@ metadata:
{{- include "spark-operator.commonLabels" . | nindent 4 }}
spec:
replicas: {{ .Values.operatorDeployment.replicas }}
- revisionHistoryLimit: 2
strategy:
{{- toYaml .Values.operatorDeployment.strategy | nindent 4 }}
selector:
diff --git
a/build-tools/helm/spark-kubernetes-operator/templates/tests/test-rbac.yaml
b/build-tools/helm/spark-kubernetes-operator/templates/tests/test-rbac.yaml
index 03fc4b1..33d8fa3 100644
---
a/build-tools/helm/spark-kubernetes-operator/templates/tests/test-rbac.yaml
+++
b/build-tools/helm/spark-kubernetes-operator/templates/tests/test-rbac.yaml
@@ -1,4 +1,3 @@
-################################################################################
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
@@ -13,8 +12,6 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-################################################################################
-
apiVersion: v1
kind: Pod
metadata:
diff --git a/build-tools/helm/spark-kubernetes-operator/values.yaml
b/build-tools/helm/spark-kubernetes-operator/values.yaml
index 16ecc5e..87fda1e 100644
--- a/build-tools/helm/spark-kubernetes-operator/values.yaml
+++ b/build-tools/helm/spark-kubernetes-operator/values.yaml
@@ -1,4 +1,3 @@
-################################################################################
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
@@ -13,7 +12,6 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-################################################################################
image:
repository: spark-kubernetes-operator
@@ -43,7 +41,7 @@ operatorDeployment:
#
https://kubernetes.io/docs/concepts/scheduling-eviction/topology-spread-constraints/
topologySpreadConstraints: [ ]
operatorContainer:
- jvmArgs: "-XX:+UseG1GC -Xms3G -Xmx3G -Dfile.encoding=UTF8"
+ jvmArgs: "-Dfile.encoding=UTF8"
env:
envFrom:
volumeMounts: { }
@@ -76,15 +74,7 @@ operatorDeployment:
seccompProfile:
type: RuntimeDefault
additionalContainers: { }
- # additionalContainers:
- # - name: ""
- # image: ""
volumes: { }
- # volumes:
- # - name: spark-artifacts
- # hostPath:
- # path: /tmp/spark/artifacts
- # type: DirectoryOrCreate
securityContext: { }
dnsPolicy:
dnsConfig:
@@ -191,5 +181,5 @@ operatorConfiguration:
annotations:
"helm.sh/resource-policy": keep
data:
- # Spark Operator Config Runtime Properties Overrides. e.g.
+ # Spark Operator Config Runtime Properties Overrides. e.g.
spark.kubernetes.operator.reconciler.intervalSeconds: 60
diff --git a/dev/.rat-excludes b/dev/.rat-excludes
index a556cb7..4234634 100644
--- a/dev/.rat-excludes
+++ b/dev/.rat-excludes
@@ -16,4 +16,3 @@ build
.*gradle
.*json
.helmignore
-.*yml
diff --git a/spark-operator-api/build.gradle
b/spark-operator-api/build.gradle
index 13b80e4..8b1dfc3 100644
--- a/spark-operator-api/build.gradle
+++ b/spark-operator-api/build.gradle
@@ -38,4 +38,5 @@ tasks.register('relocateGeneratedCRD', Copy) {
dependsOn finalizeGeneratedCRD
from
"build/classes/java/main/META-INF/fabric8/sparkapplications.org.apache.spark-v1.yml"
into "../build-tools/helm/spark-kubernetes-operator/crds"
+ rename '(.+).yml', '$1.yaml'
}
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
