[incubator-celeborn] branch main updated: [CELEBORN-902][HELM][CAPACITY][CAPACITY] Associate Celeborn.storage.dir with volumes in Helm values.yml

[zhouky]

This is an automated email from the ASF dual-hosted git repository.

zhouky pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-celeborn.git


The following commit(s) were added to refs/heads/main by this push:
     new 8e69acc0f [CELEBORN-902][HELM][CAPACITY][CAPACITY] Associate 
Celeborn.storage.dir with volumes in Helm values.yml
8e69acc0f is described below

commit 8e69acc0f9aa22a53058c125cfce548d1614d57b
Author: zwangsheng <2213335...@qq.com>
AuthorDate: Wed Aug 23 16:31:09 2023 +0800

    [CELEBORN-902][HELM][CAPACITY][CAPACITY] Associate Celeborn.storage.dir 
with volumes in Helm values.yml
    
    ### What changes were proposed in this pull request?
    Build `celeborn.worker.storage.dirs` with volumes capacity in values.yml.
    
    ### Why are the changes needed?
    Associate the volumes size in the` values.yml` file with the capacity 
attribute in the `celeborn.worker.storage.dirs` to avoid user doubt.
    
    ### Does this PR introduce _any_ user-facing change?
    No, but users will find  volumes capacity can affect 
`celeborn.worker.storage.dirs`.
    
    ### How was this patch tested?
    Helm install in local dry-run
    
    Default helm will build following config:
    ```yaml
    
    
celeborn.worker.storage.dirs=/mnt/disk1:disktype=HDD:capacity=1Gi,/mnt/disk2:disktype=HDD:capacity=1Gi,/mnt/disk3:disktype=HDD:capacity=1Gi,/mnt/disk4:disktype=HDD:capacity=1Gi
    ```
    
    Closes #1826 from zwangsheng/CELEBORN-902.
    
    Authored-by: zwangsheng <2213335...@qq.com>
    Signed-off-by: zky.zhoukeyong <zky.zhoukey...@alibaba-inc.com>
---
 charts/celeborn/templates/configmap.yaml          |  2 +-
 charts/celeborn/templates/master-statefulset.yaml |  2 +-
 charts/celeborn/templates/worker-statefulset.yaml |  2 +-
 charts/celeborn/values.yaml                       | 20 ++++++++++----------
 4 files changed, 13 insertions(+), 13 deletions(-)

diff --git a/charts/celeborn/templates/configmap.yaml 
b/charts/celeborn/templates/configmap.yaml
index 546d41114..db695484a 100644
--- a/charts/celeborn/templates/configmap.yaml
+++ b/charts/celeborn/templates/configmap.yaml
@@ -37,7 +37,7 @@ data:
     celeborn.master.ha.ratis.raft.server.storage.dir={{ (index $dirs 
0).mountPath }}
     {{- $path := "" }}
     {{- range $worker := .Values.volumes.worker }}
-    {{- $info := (cat $worker.mountPath ":disktype=" (get $worker "diskType" | 
default "HDD") | nospace) }}
+    {{- $info := (cat $worker.mountPath ":disktype=" (get $worker "diskType" | 
default "HDD") ":capacity=" (get $worker "capacity" | default "1PB") | nospace) 
}}
     {{- if eq $path "" }}
     {{- $path = $info }}
     {{- else }}
diff --git a/charts/celeborn/templates/master-statefulset.yaml 
b/charts/celeborn/templates/master-statefulset.yaml
index e9c686393..37cfff96d 100644
--- a/charts/celeborn/templates/master-statefulset.yaml
+++ b/charts/celeborn/templates/master-statefulset.yaml
@@ -129,7 +129,7 @@ spec:
         - name: {{ $.Release.Name }}-master-vol-{{ $index }}
         {{- if eq "emptyDir" $volume.type }}
           emptyDir:
-            sizeLimit: {{ $volume.size }}
+            sizeLimit: {{ $volume.capacity }}
         {{- else if eq "hostPath" $volume.type }}
           hostPath:
             path: {{ $volume.hostPath | default $volume.mountPath }}/master
diff --git a/charts/celeborn/templates/worker-statefulset.yaml 
b/charts/celeborn/templates/worker-statefulset.yaml
index e5a6d0d70..05f649860 100644
--- a/charts/celeborn/templates/worker-statefulset.yaml
+++ b/charts/celeborn/templates/worker-statefulset.yaml
@@ -134,7 +134,7 @@ spec:
         - name: {{ $.Release.Name }}-worker-vol-{{ $index }}
         {{- if eq "emptyDir" $volume.type }}
           emptyDir:
-            sizeLimit: {{ $volume.size }}
+            sizeLimit: {{ $volume.capacity }}
         {{- else if eq "hostPath" $volume.type }}
           hostPath:
             path: {{ $volume.hostPath | default $volume.mountPath }}/worker
diff --git a/charts/celeborn/values.yaml b/charts/celeborn/values.yaml
index d0b884019..2d754f58a 100644
--- a/charts/celeborn/values.yaml
+++ b/charts/celeborn/values.yaml
@@ -44,7 +44,7 @@ securityContext:
 # Current Celeborn support followings volume type:
 # - emptyDir
 # - hostPath
-# Note: size only works in emptyDir type
+# Note:
 # hostPath only works in hostPath type using to set `volumes hostPath path`
 # Celeborn Master will pick first volumes for store raft log
 # diskType only works in Celeborn Worker with hostPath type to manifest local 
disk type
@@ -53,28 +53,28 @@ volumes:
     - mountPath: /mnt/celeborn_ratis
       hostPath: /mnt/celeborn_ratis
       type: hostPath
-      size: 1Gi
+      capacity: 100Gi
   worker:
     - mountPath: /mnt/disk1
       hostPath: /mnt/disk1
       type: hostPath
-      diskType: HDD
-      size: 1Gi
+      diskType: SSD
+      capacity: 100Gi
     - mountPath: /mnt/disk2
       hostPath: /mnt/disk2
       type: hostPath
-      diskType: HDD
-      size: 1Gi
+      diskType: SSD
+      capacity: 100Gi
     - mountPath: /mnt/disk3
       hostPath: /mnt/disk3
       type: hostPath
-      diskType: HDD
-      size: 1Gi
+      diskType: SSD
+      capacity: 100Gi
     - mountPath: /mnt/disk4
       hostPath: /mnt/disk4
       type: hostPath
-      diskType: HDD
-      size: 1Gi
+      diskType: SSD
+      capacity: 100Gi
 
 # celeborn configurations
 celeborn: