This is an automated email from the ASF dual-hosted git repository.
feiwang pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/celeborn.git
The following commit(s) were added to refs/heads/main by this push:
new c83d498d9 [CELEBORN-1528][HELM] Use volume claim template to support
various storage backend
c83d498d9 is described below
commit c83d498d99c27851b1c3aa84a4cb096697e12706
Author: Yi Chen <[email protected]>
AuthorDate: Tue May 27 16:32:46 2025 -0700
[CELEBORN-1528][HELM] Use volume claim template to support various storage
backend
### What changes were proposed in this pull request?
- Add support for configuring volumeClaimTemplates by adding new values
`master.volumeClaimTemplates` and `worker.volumeClaimTemplates`.
### Why are the changes needed?
Use volume claim template to support various storage backend.
### Does this PR introduce _any_ user-facing change?
Yes. New Helm values `master.volumeClaimTemplates` and
`worker.volumeClaimTemplates` are added.
### How was this patch tested?
```bash
helm unittest charts/celeborn --file "tests/**/*_test.yaml" --strict --debug
```
Closes #3277 from ChenYi015/helm/volume-claim-templates.
Authored-by: Yi Chen <[email protected]>
Signed-off-by: Wang, Fei <[email protected]>
---
charts/celeborn/ci/values.yaml | 26 -------
charts/celeborn/templates/master/statefulset.yaml | 32 +++++---
charts/celeborn/templates/worker/statefulset.yaml | 31 +++++---
charts/celeborn/tests/master/statefulset_test.yaml | 52 +++++++++++++
charts/celeborn/tests/worker/statefulset_test.yaml | 52 +++++++++++++
charts/celeborn/values.yaml | 90 ++++++++++++++++++----
6 files changed, 220 insertions(+), 63 deletions(-)
diff --git a/charts/celeborn/ci/values.yaml b/charts/celeborn/ci/values.yaml
index c6c77dbcc..1f15bf195 100644
--- a/charts/celeborn/ci/values.yaml
+++ b/charts/celeborn/ci/values.yaml
@@ -28,16 +28,6 @@ image:
# -- Image pull policy
pullPolicy: IfNotPresent
-service:
- # -- Specifies service type
- type: ClusterIP
- # -- Specifies service port
- port: 9097
-
-cluster:
- # -- Specifies Kubernetes cluster name
- name: cluster
-
celeborn:
celeborn.metrics.enabled: false
celeborn.worker.storage.dirs:
/mnt/disk1:disktype=SSD:capacity=1Gi,/mnt/disk2:disktype=SSD:capacity=1Gi
@@ -134,26 +124,10 @@ worker:
podMonitor:
# -- Specifies whether to enable creating pod monitors for Celeborn pods
enable: false
- # -- Specifies pod metrics endpoint
- podMetricsEndpoint:
- # Specifies scheme
- scheme: http
- # Specifies scrape interval
- interval: 5s
- # Specifies port name
- portName: metrics
serviceAccount:
# -- Specifies whether to create a service account for Celeborn
create: false
- # -- Specifies the name of the service account
- name: default
rbac:
create: false
- roleName: default
- roleBindingName: default
- rules:
- - apiGroups: [""]
- resources: ["pods"]
- verbs: ["create", "list", "delete"]
diff --git a/charts/celeborn/templates/master/statefulset.yaml
b/charts/celeborn/templates/master/statefulset.yaml
index 335fd2671..2ccf49e72 100644
--- a/charts/celeborn/templates/master/statefulset.yaml
+++ b/charts/celeborn/templates/master/statefulset.yaml
@@ -23,10 +23,14 @@ metadata:
{{- include "celeborn.master.labels" . | nindent 4 }}
spec:
replicas: {{ .Values.master.replicas }}
- serviceName: {{ include "celeborn.master.service.name" . }}
selector:
matchLabels:
{{- include "celeborn.master.selectorLabels" . | nindent 6 }}
+ {{- with .Values.master.volumeClaimTemplates }}
+ volumeClaimTemplates:
+ {{- toYaml . | nindent 2 }}
+ {{- end }}
+ serviceName: {{ include "celeborn.master.service.name" . }}
template:
metadata:
labels:
@@ -37,7 +41,18 @@ spec:
{{- toYaml . | nindent 8 }}
{{- end }}
spec:
- serviceAccountName: {{ include "celeborn.serviceAccountName" . }}
+ {{- /* Add an init container to chown mount paths of Celeborn master
volumes if necessary. */}}
+ {{- $paths := list }}
+ {{- range $volumeMount := .Values.master.volumeMounts }}
+ {{- range $volume := $.Values.master.volumes }}
+ {{- if eq $volume.name $volumeMount.name }}
+ {{- if or $volume.hostPath $volume.emptyDir }}
+ {{- $paths = append $paths $volumeMount.mountPath }}
+ {{- end }}
+ {{- end }}
+ {{- end }}
+ {{- end }}
+ {{- if $paths }}
initContainers:
- name: chown-celeborn-master-volume
image: {{ include "celeborn.image" . }}
@@ -48,14 +63,8 @@ spec:
- chown
- -R
- {{ .Values.master.podSecurityContext.runAsUser | default 10006 }}:{{
.Values.master.podSecurityContext.runAsGroup | default 10006 }}
- {{- range $volumeMount := .Values.master.volumeMounts }}
- {{- range $volume := $.Values.master.volumes }}
- {{- if eq $volume.name $volumeMount.name }}
- {{- if or $volume.hostPath $volume.emptyDir }}
- - {{ $volumeMount.mountPath }}
- {{- end }}
- {{- end }}
- {{- end }}
+ {{- range $path := $paths }}
+ - {{ $path }}
{{- end }}
{{- with .Values.master.volumeMounts }}
volumeMounts:
@@ -67,6 +76,7 @@ spec:
{{- end }}
securityContext:
runAsUser: 0
+ {{- end }}
containers:
- name: {{ .Chart.Name }}
image: {{ include "celeborn.image" . }}
@@ -161,8 +171,10 @@ spec:
{{- with .Values.master.hostNetwork }}
hostNetwork: {{ . }}
{{- end }}
+ serviceAccountName: {{ include "celeborn.serviceAccountName" . }}
{{- with .Values.master.podSecurityContext }}
securityContext:
{{- toYaml . | nindent 8 }}
{{- end }}
terminationGracePeriodSeconds: 30
+
diff --git a/charts/celeborn/templates/worker/statefulset.yaml
b/charts/celeborn/templates/worker/statefulset.yaml
index d3abc7009..b8441a3bd 100644
--- a/charts/celeborn/templates/worker/statefulset.yaml
+++ b/charts/celeborn/templates/worker/statefulset.yaml
@@ -23,10 +23,14 @@ metadata:
{{- include "celeborn.worker.labels" . | nindent 4 }}
spec:
replicas: {{ .Values.worker.replicas }}
- serviceName: {{ include "celeborn.worker.service.name" . }}
selector:
matchLabels:
{{- include "celeborn.worker.selectorLabels" . | nindent 6 }}
+ {{- with .Values.worker.volumeClaimTemplates }}
+ volumeClaimTemplates:
+ {{- toYaml . | nindent 2 }}
+ {{- end }}
+ serviceName: {{ include "celeborn.worker.service.name" . }}
template:
metadata:
labels:
@@ -37,7 +41,18 @@ spec:
{{- toYaml . | nindent 8 }}
{{- end }}
spec:
- serviceAccountName: {{ include "celeborn.serviceAccountName" . }}
+ {{- /* Add an init container to chown mount paths of Celeborn workers
volume if necessary. */}}
+ {{- $paths := list }}
+ {{- range $volumeMount := .Values.worker.volumeMounts }}
+ {{- range $volume := $.Values.worker.volumes }}
+ {{- if eq $volume.name $volumeMount.name }}
+ {{- if or $volume.hostPath $volume.emptyDir }}
+ {{- $paths = append $paths $volumeMount.mountPath }}
+ {{- end }}
+ {{- end }}
+ {{- end }}
+ {{- end }}
+ {{- if $paths }}
initContainers:
- name: chown-celeborn-worker-volume
image: {{ include "celeborn.image" . }}
@@ -48,14 +63,8 @@ spec:
- chown
- -R
- {{ .Values.worker.podSecurityContext.runAsUser | default 10006 }}:{{
.Values.worker.podSecurityContext.runAsGroup | default 10006 }}
- {{- range $volumeMount := .Values.worker.volumeMounts }}
- {{- range $volume := $.Values.worker.volumes }}
- {{- if eq $volume.name $volumeMount.name }}
- {{- if or $volume.hostPath $volume.emptyDir }}
- - {{ $volumeMount.mountPath }}
- {{- end }}
- {{- end }}
- {{- end }}
+ {{- range $path := $paths }}
+ - {{ $path }}
{{- end }}
{{- with .Values.worker.volumeMounts }}
volumeMounts:
@@ -67,6 +76,7 @@ spec:
{{- end }}
securityContext:
runAsUser: 0
+ {{- end }}
containers:
- name: {{ .Chart.Name }}
image: {{ include "celeborn.image" . }}
@@ -160,6 +170,7 @@ spec:
{{- with .Values.worker.hostNetwork }}
hostNetwork: {{ . }}
{{- end }}
+ serviceAccountName: {{ include "celeborn.serviceAccountName" . }}
{{- with .Values.worker.podSecurityContext }}
securityContext:
{{- toYaml . | nindent 8 }}
diff --git a/charts/celeborn/tests/master/statefulset_test.yaml
b/charts/celeborn/tests/master/statefulset_test.yaml
index 2fa9bc581..981a80d6c 100644
--- a/charts/celeborn/tests/master/statefulset_test.yaml
+++ b/charts/celeborn/tests/master/statefulset_test.yaml
@@ -376,3 +376,55 @@ tests:
- equal:
path: spec.template.spec.securityContext.fsGroup
value: 3000
+
+ - it: Should add volume claim templates if `master.volumeClaimTemplates` is
set
+ set:
+ master:
+ volumeClaimTemplates:
+ - metadata:
+ name: test-volume-claim-template-1
+ spec:
+ accessModes:
+ - ReadWriteMany
+ resources:
+ request:
+ storage: 100Gi
+ limits:
+ storage: 100Gi
+ - metadata:
+ name: test-volume-claim-template-2
+ spec:
+ accessModes:
+ - ReadWriteOnce
+ resources:
+ request:
+ storage: 200Gi
+ limits:
+ storage: 200Gi
+ asserts:
+ - contains:
+ path: spec.volumeClaimTemplates
+ content:
+ metadata:
+ name: test-volume-claim-template-1
+ spec:
+ accessModes:
+ - ReadWriteMany
+ resources:
+ request:
+ storage: 100Gi
+ limits:
+ storage: 100Gi
+ - contains:
+ path: spec.volumeClaimTemplates
+ content:
+ metadata:
+ name: test-volume-claim-template-2
+ spec:
+ accessModes:
+ - ReadWriteOnce
+ resources:
+ request:
+ storage: 200Gi
+ limits:
+ storage: 200Gi
diff --git a/charts/celeborn/tests/worker/statefulset_test.yaml
b/charts/celeborn/tests/worker/statefulset_test.yaml
index 8440310ed..6ec98a091 100644
--- a/charts/celeborn/tests/worker/statefulset_test.yaml
+++ b/charts/celeborn/tests/worker/statefulset_test.yaml
@@ -375,3 +375,55 @@ tests:
- equal:
path: spec.template.spec.securityContext.fsGroup
value: 3000
+
+ - it: Should add volume claim templates if `worker.volumeClaimTemplates` is
set
+ set:
+ worker:
+ volumeClaimTemplates:
+ - metadata:
+ name: test-volume-claim-template-1
+ spec:
+ accessModes:
+ - ReadWriteMany
+ resources:
+ request:
+ storage: 100Gi
+ limits:
+ storage: 100Gi
+ - metadata:
+ name: test-volume-claim-template-2
+ spec:
+ accessModes:
+ - ReadWriteOnce
+ resources:
+ request:
+ storage: 200Gi
+ limits:
+ storage: 200Gi
+ asserts:
+ - contains:
+ path: spec.volumeClaimTemplates
+ content:
+ metadata:
+ name: test-volume-claim-template-1
+ spec:
+ accessModes:
+ - ReadWriteMany
+ resources:
+ request:
+ storage: 100Gi
+ limits:
+ storage: 100Gi
+ - contains:
+ path: spec.volumeClaimTemplates
+ content:
+ metadata:
+ name: test-volume-claim-template-2
+ spec:
+ accessModes:
+ - ReadWriteOnce
+ resources:
+ request:
+ storage: 200Gi
+ limits:
+ storage: 200Gi
diff --git a/charts/celeborn/values.yaml b/charts/celeborn/values.yaml
index 5dcd85d5d..81c264479 100644
--- a/charts/celeborn/values.yaml
+++ b/charts/celeborn/values.yaml
@@ -195,13 +195,13 @@ master:
# -- Tolerations for Celeborn master pods.
tolerations:
- # - key: key1
- # operator: Equal
- # value: value1
- # effect: NoSchedule
- # - key: key2
- # operator: Exists
- # effect: NoSchedule
+ # - key: key1
+ # operator: Equal
+ # value: value1
+ # effect: NoSchedule
+ # - key: key2
+ # operator: Exists
+ # effect: NoSchedule
# Priority class for Celeborn master pods.
priorityClass:
@@ -227,6 +227,19 @@ master:
# The group ID to use when modifying the ownership and permissions of the
mounted volumes.
fsGroup: 10006
+ # -- Volume claim templates for Celeborn master statefulset.
+ volumeClaimTemplates:
+ # - metadata:
+ # name: celeborn-ratis
+ # spec:
+ # accessModes:
+ # - ReadWriteOnce
+ # resources:
+ # requests:
+ # storage: 100Gi
+ # limits:
+ # storage: 100Gi
+
worker:
# -- Number of Celeborn worker replicas to deploy, should less than node
number.
replicas: 5
@@ -328,13 +341,13 @@ worker:
# -- Tolerations for Celeborn worker pods.
tolerations:
- # - key: key1
- # operator: Equal
- # value: value1
- # effect: NoSchedule
- # - key: key2
- # operator: Exists
- # effect: NoSchedule
+ # - key: key1
+ # operator: Equal
+ # value: value1
+ # effect: NoSchedule
+ # - key: key2
+ # operator: Exists
+ # effect: NoSchedule
# Priority class for Celeborn worker pods.
priorityClass:
@@ -360,6 +373,49 @@ worker:
# The group ID to use when modifying the ownership and permissions of the
mounted volumes.
fsGroup: 10006
+ # -- Volume claim templates for Celeborn worker pods.
+ volumeClaimTemplates:
+ # - metadata:
+ # name: disk1
+ # spec:
+ # accessModes:
+ # - ReadWriteOnce
+ # resources:
+ # requests:
+ # storage: 100Gi
+ # limits:
+ # storage: 100Gi
+ # - metadata:
+ # name: disk2
+ # spec:
+ # accessModes:
+ # - ReadWriteOnce
+ # resources:
+ # requests:
+ # storage: 100Gi
+ # limits:
+ # storage: 100Gi
+ # - metadata:
+ # name: disk3
+ # spec:
+ # accessModes:
+ # - ReadWriteOnce
+ # resources:
+ # requests:
+ # storage: 100Gi
+ # limits:
+ # storage: 100Gi
+ # - metadata:
+ # name: disk4
+ # spec:
+ # accessModes:
+ # - ReadWriteOnce
+ # resources:
+ # requests:
+ # storage: 100Gi
+ # limits:
+ # storage: 100Gi
+
podMonitor:
# -- Specifies whether to enable creating pod monitors for Celeborn pods
enable: true
@@ -383,6 +439,6 @@ rbac:
roleName: default
roleBindingName: default
rules:
- - apiGroups: [""]
- resources: ["pods"]
- verbs: ["create", "list", "delete"]
+ - apiGroups: [""]
+ resources: ["pods"]
+ verbs: ["create", "list", "delete"]
