This is an automated email from the ASF dual-hosted git repository.
zhouky pushed a commit to branch branch-0.3
in repository https://gitbox.apache.org/repos/asf/incubator-celeborn.git
The following commit(s) were added to refs/heads/branch-0.3 by this push:
new 5d0cc36bb [CELEBORN-644][HELM] Support Helm Deploy Celeborn with
HostNetwork And DnsPolicy
5d0cc36bb is described below
commit 5d0cc36bb80a369723be6585baa173dba572db01
Author: zwangsheng <[email protected]>
AuthorDate: Fri Jun 16 17:59:39 2023 +0800
[CELEBORN-644][HELM] Support Helm Deploy Celeborn with HostNetwork And
DnsPolicy
---
charts/celeborn/templates/master-statefulset.yaml | 2 +
charts/celeborn/templates/worker-statefulset.yaml | 2 +
charts/celeborn/values.yaml | 3 +
tests/kubernetes-it/docker/helm/values.yaml | 118 ++++++++++++++++++++++
4 files changed, 125 insertions(+)
diff --git a/charts/celeborn/templates/master-statefulset.yaml
b/charts/celeborn/templates/master-statefulset.yaml
index 974771005..be25769fc 100644
--- a/charts/celeborn/templates/master-statefulset.yaml
+++ b/charts/celeborn/templates/master-statefulset.yaml
@@ -71,6 +71,8 @@ spec:
securityContext:
{{- toYaml . | nindent 8 }}
{{- end }}
+ hostNetwork: {{ .Values.hostNetwork | default false }}
+ dnsPolicy: {{ .Values.dnsPolicy | default "ClusterFirst" }}
initContainers:
{{- $dirs := .Values.volumes.master }}
{{- if eq "hostPath" (index $dirs 0).type }}
diff --git a/charts/celeborn/templates/worker-statefulset.yaml
b/charts/celeborn/templates/worker-statefulset.yaml
index 1d758fc8d..0a4ab5756 100644
--- a/charts/celeborn/templates/worker-statefulset.yaml
+++ b/charts/celeborn/templates/worker-statefulset.yaml
@@ -71,6 +71,8 @@ spec:
securityContext:
{{- toYaml . | nindent 8 }}
{{- end }}
+ hostNetwork: {{ .Values.hostNetwork | default false }}
+ dnsPolicy: {{ .Values.dnsPolicy | default "ClusterFirst" }}
initContainers:
{{- $dirs := .Values.volumes.worker }}
{{- if eq "hostPath" (index $dirs 0).type }}
diff --git a/charts/celeborn/values.yaml b/charts/celeborn/values.yaml
index 932fefaea..741efc3cc 100644
--- a/charts/celeborn/values.yaml
+++ b/charts/celeborn/values.yaml
@@ -33,6 +33,9 @@ masterReplicas: 3
# worker replicas set on demand, should less than node number
workerReplicas: 5
+hostNetwork: false
+dnsPolicy: ClusterFirst
+
securityContext:
runAsUser: 10006
runAsGroup: 10006
diff --git a/tests/kubernetes-it/docker/helm/values.yaml
b/tests/kubernetes-it/docker/helm/values.yaml
new file mode 100644
index 000000000..9d36ab62e
--- /dev/null
+++ b/tests/kubernetes-it/docker/helm/values.yaml
@@ -0,0 +1,118 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+image:
+ repository: apache/celeborn
+ pullPolicy: IfNotPresent
+ tag: latest
+
+imagePullSecrets: {}
+
+# For integration test, using single celeborn mater & worker
+masterReplicas: 1
+workerReplicas: 1
+
+hostNetwork: true
+dnsPolicy: ClusterFirstWithHostNet
+securityContext:
+ runAsUser: 10006
+ runAsGroup: 10006
+ fsGroup: 10006
+
+# Current Celeborn support followings volume type:
+# - emptyDir
+# - hostPath
+# Note: size only works in emptyDir type
+# Celeborn Master will pick first volumes for store raft log
+volumes:
+ master:
+ - mountPath: /mnt/rss_ratis
+ type: emptyDir
+ size: 1Gi
+ worker:
+ - mountPath: /mnt/disk1
+ type: emptyDir
+ size: 1Gi
+ - mountPath: /mnt/disk2
+ type: emptyDir
+ size: 1Gi
+
+# celeborn configurations
+celeborn:
+ celeborn.ha.enabled: false
+ celeborn.metrics.enabled: false
+ celeborn.master.metrics.prometheus.port: 9098
+ celeborn.worker.metrics.prometheus.port: 9096
+ celeborn.worker.monitor.disk.enabled: false
+ celeborn.shuffle.chunk.size: 8m
+ celeborn.rpc.io.serverThreads: 64
+ celeborn.rpc.io.numConnectionsPerPeer: 2
+ celeborn.rpc.io.clientThreads: 64
+ celeborn.rpc.dispatcher.numThreads: 4
+ celeborn.worker.flusher.buffer.size: 256K
+ celeborn.worker.fetch.io.threads: 32
+ celeborn.worker.push.io.threads: 32
+ celeborn.push.stageEnd.timeout: 120s
+ celeborn.application.heartbeat.timeout: 120s
+ celeborn.worker.heartbeat.timeout: 120s
+
+environments:
+ CELEBORN_MASTER_MEMORY: 100m
+ CELEBORN_WORKER_MEMORY: 100m
+ CELEBORN_WORKER_OFFHEAP_MEMORY: 100m
+ CELEBORN_NO_DAEMONIZE: 1
+ TZ: "Asia/Shanghai"
+
+podMonitor:
+ enable: false
+ podMetricsEndpoint:
+ scheme: http
+ interval: 5s
+ portName: metrics
+
+service:
+ type: ClusterIP
+ port: 9097
+
+cluster:
+ name: cluster
+
+configmap: celeborn-conf
+
+resources:
+ master:
+ limits:
+ cpu: 100m
+ memory: 800Mi
+ requests:
+ cpu: 100m
+ memory: 800Mi
+ worker:
+ limits:
+ cpu: 100m
+ memory: 1Gi
+ requests:
+ cpu: 100m
+ memory: 1Gi
+
+podAnnotations: {}
+
+affinity: {}
+
+tolerations: []
+
+nodeSelector: {}
