This is an automated email from the ASF dual-hosted git repository.
karthikz pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-heron.git
The following commit(s) were added to refs/heads/master by this push:
new ef50533 added S3 uploader configuration in Helm chart (#2939)
ef50533 is described below
commit ef50533d4ff9021d5cc3d3cbc2f57521245a8dd9
Author: Cristóbal Carnero Liñán <ccarneroli...@gmail.com>
AuthorDate: Sun Jul 1 16:56:41 2018 +0200
added S3 uploader configuration in Helm chart (#2939)
* added S3 uploader configuration in Helm chart
* fixed package path
* added 'packing' configuration variable (RoundRobin by default)
---
deploy/kubernetes/helm/templates/tools.yaml | 19 +++++++++++++++++++
deploy/kubernetes/helm/values.yaml.template | 12 ++++++++++++
.../deployment/schedulers/kubernetes-helm.md | 15 +++++++++++++++
3 files changed, 46 insertions(+)
diff --git a/deploy/kubernetes/helm/templates/tools.yaml
b/deploy/kubernetes/helm/templates/tools.yaml
index 65daefe..7ee348d 100644
--- a/deploy/kubernetes/helm/templates/tools.yaml
+++ b/deploy/kubernetes/helm/templates/tools.yaml
@@ -127,9 +127,28 @@ spec:
-D heron.kubernetes.scheduler.uri=http://localhost:8001
-D heron.kubernetes.scheduler.namespace={{ .Release.Namespace }}
-D heron.executor.docker.image={{ .Values.image }}
+ {{- if eq .Values.uploader.class "dlog" }}
-D heron.class.uploader=org.apache.heron.uploader.dlog.DLUploader
-D heron.uploader.dlog.topologies.num.replicas={{ $jobReplicas }}
-D
heron.uploader.dlog.topologies.namespace.uri=distributedlog://{{ .Release.Name
}}-zookeeper:2181/distributedlog
+ {{- else if eq .Values.uploader.class "s3" }}
+ -D heron.class.uploader=org.apache.heron.uploader.s3.S3Uploader
+ -D heron.uploader.s3.bucket={{ .Values.uploader.s3Bucket }}
+ -D heron.uploader.s3.path_prefix={{
.Values.uploader.s3PathPrefix }}
+ -D heron.uploader.s3.access_key={{ .Values.uploader.s3AccessKey
}}
+ -D heron.uploader.s3.secret_key={{ .Values.uploader.s3SecretKey
}}
+ -D heron.uploader.s3.region={{ .Values.uploader.s3Region }}
+ {{- end }}
+ {{- if eq .Values.packing "RoundRobin" }}
+ -D
heron.class.packing.algorithm=org.apache.heron.packing.roundrobin.RoundRobinPacking
+ -D
heron.class.repacking.algorithm=org.apache.heron.packing.roundrobin.RoundRobinPacking
+ {{- else if eq .Values.packing "ResourceCompliantRR" }}
+ -D
heron.class.packing.algorithm=org.apache.heron.packing.roundrobin.ResourceCompliantRRPacking
+ -D
heron.class.repacking.algorithm=org.apache.heron.packing.roundrobin.ResourceCompliantRRPacking
+ {{- else if eq .Values.packing "FirstFitDecreasing" }}
+ -D
heron.class.packing.algorithm=org.apache.heron.packing.binpacking.FirstFitDecreasingPacking
+ -D
heron.class.repacking.algorithm=org.apache.heron.packing.binpacking.FirstFitDecreasingPacking
+ {{- end }}
envFrom:
- configMapRef:
name: {{ .Release.Name }}-tools-config
diff --git a/deploy/kubernetes/helm/values.yaml.template
b/deploy/kubernetes/helm/values.yaml.template
index 114da7d..524eef7 100644
--- a/deploy/kubernetes/helm/values.yaml.template
+++ b/deploy/kubernetes/helm/values.yaml.template
@@ -22,6 +22,18 @@ jobReplicas: 1
# amount of memory to provide for API server
apiServerMemory: 512M
+# Topologies uploader
+uploader:
+ class: dlog # s3
+ s3Bucket: ""
+ s3PathPrefix: ""
+ s3AccessKey: ""
+ s3SecretKey: ""
+ s3Region: ""
+
+# Packing algorithms
+packing: RoundRobin # ResourceCompliantRR, FirstFitDecreasing
+
# Number of replicas for storage bookies, memory and storage requirements
bookieReplicas: 1
bookieCpuMin: 100m
diff --git
a/website/content/docs/operators/deployment/schedulers/kubernetes-helm.md
b/website/content/docs/operators/deployment/schedulers/kubernetes-helm.md
index b42602f..b38c14f 100644
--- a/website/content/docs/operators/deployment/schedulers/kubernetes-helm.md
+++ b/website/content/docs/operators/deployment/schedulers/kubernetes-helm.md
@@ -209,6 +209,21 @@ $ helm install heron-charts/heron \
--set platform=aws
```
+##### Using S3 uploader
+
+You can make Heron to use S3 to distribute the user topologies. First you need
to set up a S3 bucket and configure an IAM user with enough permissions over
it. Get access keys for the user. Then you can deploy Heron like this:
+
+```bash
+$ helm install heron-charts/heron \
+ --set platform=aws \
+ --set uploader.class=s3 \
+ --set uploader.s3Bucket=heron \
+ --set uploader.s3PathPrefix=topologies \
+ --set uploader.s3AccessKey=XXXXXXXXXXXXXXXXXXXX \
+ --set uploader.s3SecretKey=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX \
+ --set uploader.s3Region=us-west-1
+```
+
#### Bare metal
To run Heron on a bare metal Kubernetes cluster:
