This is an automated email from the ASF dual-hosted git repository.
mmerli pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pulsar.git
The following commit(s) were added to refs/heads/master by this push:
new f703926 Added config to deploy pulsar in aws using ebs storage (#1586)
f703926 is described below
commit f70392664c28f096cecee16d764be0e609f3bc2a
Author: Andrews <sahaya.andr...@gmail.com>
AuthorDate: Thu Apr 19 10:31:20 2018 -0700
Added config to deploy pulsar in aws using ebs storage (#1586)
---
deployment/kubernetes/aws/README.md | 26 ++++
deployment/kubernetes/aws/bookkeeper.yaml | 215 ++++++++++++++++++++++++++++++
deployment/kubernetes/aws/broker.yaml | 144 ++++++++++++++++++++
deployment/kubernetes/aws/zookeeper.yaml | 153 +++++++++++++++++++++
4 files changed, 538 insertions(+)
diff --git a/deployment/kubernetes/aws/README.md
b/deployment/kubernetes/aws/README.md
new file mode 100644
index 0000000..54ec6f7
--- /dev/null
+++ b/deployment/kubernetes/aws/README.md
@@ -0,0 +1,26 @@
+<!--
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+-->
+
+This directory contains the Kubernetes services definitions for all the
+components required to run Pulsar and use ebs volume as bookie storage.
+
+Refer to [Kubernetes.md](../site/docs/latest/deployment/Kubernetes.md)
document for instructions on
+how to deploy Pulsar on a Kubernetes cluster.
diff --git a/deployment/kubernetes/aws/bookkeeper.yaml
b/deployment/kubernetes/aws/bookkeeper.yaml
new file mode 100644
index 0000000..9f1418c
--- /dev/null
+++ b/deployment/kubernetes/aws/bookkeeper.yaml
@@ -0,0 +1,215 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+## Credits to Franck Cuny :
https://github.com/fcuny/distributedlog-on-k8s/blob/master/bookkeeper.statefulset.yaml
+
+# SSDs for bookie journal storage
+kind: StorageClass
+apiVersion: storage.k8s.io/v1beta1
+metadata:
+ name: bookie-ssd
+provisioner: kubernetes.io/aws-ebs
+parameters:
+ type: gp2
+ zones: us-east-2a
+ fsType: xfs
+# To create encrypted ebs volume using kms
+# encrypted: "true"
+# kmsKeyId: <enter the key id here>
+reclaimPolicy: Retain
+---
+# HDDs for bookie ledger storage
+kind: StorageClass
+apiVersion: storage.k8s.io/v1beta1
+metadata:
+ name: bookie-hdd
+provisioner: kubernetes.io/aws-ebs
+parameters:
+ type: st1
+ zones: us-east-2a
+ fsType: xfs
+# To create encrypted ebs volume using kms
+# encrypted: "true"
+# kmsKeyId: <enter the key id here>
+reclaimPolicy: Retain
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+ name: bookie-config
+data:
+ PULSAR_MEM: "\"-Dio.netty.leakDetectionLevel=disabled
-Dio.netty.recycler.linkCapacity=1024 -XX:+UseG1GC -XX:MaxGCPauseMillis=10
-XX:+ParallelRefProcEnabled -XX:+UnlockExperimentalVMOptions
-XX:+AggressiveOpts -XX:+DoEscapeAnalysis -XX:ParallelGCThreads=32
-XX:ConcGCThreads=32 -XX:G1NewSizePercent=50 -XX:+DisableExplicitGC
-XX:-ResizePLAB -XX:+ExitOnOutOfMemoryError -XX:+PerfDisableSharedMem
-XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintGCApplicationStoppedTime
-XX:+PrintHeapAtG [...]
+ dbStorage_writeCacheMaxSizeMb: "2048" # Write cache size (direct memory)
+ dbStorage_readAheadCacheMaxSizeMb: "2048" # Read cache size (direct memory)
+ dbStorage_rocksDB_blockCacheSize: "4294967296"
+ journalMaxSizeMB: "2048"
+ zkServers: zk-0.zookeeper,zk-1.zookeeper,zk-2.zookeeper
+ statsProviderClass: org.apache.bookkeeper.stats.PrometheusMetricsProvider
+ useHostNameAsBookieID: "true"
+---
+apiVersion: apps/v1beta1
+kind: StatefulSet
+metadata:
+ name: bookkeeper
+ labels:
+ app: pulsar
+ component: bookkeeper
+ role: bookie
+spec:
+ serviceName: bookkeeper
+ replicas: 2
+ template:
+ metadata:
+ labels:
+ app: pulsar
+ component: bookkeeper
+ # Specify cluster to allow aggregation by cluster in
+ # the metrics
+ cluster: us-east
+ role: bookie
+ annotations:
+ prometheus.io/scrape: "true"
+ prometheus.io/port: "8000"
+
+ spec:
+ containers:
+ - name: bookie
+ image: apachepulsar/pulsar:latest
+ command: ["sh", "-c"]
+ args:
+ - >
+ bin/apply-config-from-env.py conf/bookkeeper.conf &&
+ bin/apply-config-from-env.py conf/pulsar_env.sh &&
+ bin/pulsar bookie
+ ports:
+ - containerPort: 3181
+ name: client
+ envFrom:
+ - configMapRef:
+ name: bookie-config
+ env:
+
+ volumeMounts:
+ - name: journal-disk
+ mountPath: /pulsar/data/bookkeeper/journal
+ - name: ledgers-disk
+ mountPath: /pulsar/data/bookkeeper/ledgers
+
+ initContainers:
+ # The first time, initialize BK metadata in zookeeper
+ # Otherwise ignore error if it's already there
+ - name: bookie-metaformat
+ image: apachepulsar/pulsar:latest
+ command: ["sh", "-c"]
+ args:
+ - >
+ bin/apply-config-from-env.py conf/bookkeeper.conf &&
+ bin/bookkeeper shell metaformat --nonInteractive || true;
+ envFrom:
+ - configMapRef:
+ name: bookie-config
+
+# Uncomment the following if bookie should be deployed on a specific instance
type
+# nodeSelector:
+# beta.kubernetes.io/instance-type: m4.2xlarge
+
+ volumeClaimTemplates:
+ - metadata:
+ name: journal-disk
+ spec:
+ accessModes: [ "ReadWriteOnce" ]
+ resources:
+ requests:
+ storage: 200Gi
+ storageClassName: bookie-ssd
+ - metadata:
+ name: ledgers-disk
+ spec:
+ accessModes: [ "ReadWriteOnce" ]
+ resources:
+ requests:
+ storage: 1000Gi
+ storageClassName: bookie-hdd
+
+---
+##
+## Define the Bookie headless service
+## In practice, in this case, it is only useful to have a view of
+## all the bookie pods that are present
+##
+apiVersion: v1
+kind: Service
+metadata:
+ name: bookkeeper
+ labels:
+ app: pulsar
+ component: bookkeeper
+ role: bookie
+spec:
+ ports:
+ - port: 3181
+ name: server
+ clusterIP: None
+ selector:
+ app: pulsar
+ component: bookkeeper
+
+---
+
+##
+## Run BookKeeper auto-recovery from a different set of containers
+## Auto-Recovery makes sure to restore the replication factor when any bookie
+## crashes and it's not recovering on its own.
+##
+apiVersion: apps/v1beta1
+kind: Deployment
+metadata:
+ name: bookie-autorecovery
+spec:
+ replicas: 2
+ template:
+ metadata:
+ labels:
+ app: pulsar
+ component: bookkeeper-replication
+ role: bookie
+ spec:
+ containers:
+ - name: replication-worker
+ image: apachepulsar/pulsar:latest
+ command: ["sh", "-c"]
+ args:
+ - >
+ bin/apply-config-from-env.py conf/bookkeeper.conf &&
+ bin/bookkeeper autorecovery
+ envFrom:
+ - configMapRef:
+ name: bookie-config
+ env:
+ ## Configure for lower mem usage
+ - name: PULSAR_MEM
+ value: "\" -Xmx256m \""
+ - name: PULSAR_GC
+ value: "\" \""
+# Uncomment the following if bookie should be deployed on a specific instance
type
+# nodeSelector:
+# beta.kubernetes.io/instance-type: m4.2xlarge
+
+
+---
diff --git a/deployment/kubernetes/aws/broker.yaml
b/deployment/kubernetes/aws/broker.yaml
new file mode 100644
index 0000000..c01ff20
--- /dev/null
+++ b/deployment/kubernetes/aws/broker.yaml
@@ -0,0 +1,144 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+
+apiVersion: v1
+kind: ConfigMap
+metadata:
+ name: broker-config
+data:
+ # Tune for available memory. Increase the heap up to 24G to have
+ # better GC behavior at high throughput
+ PULSAR_MEM: "\" -Dio.netty.leakDetectionLevel=disabled
-Dio.netty.recycler.linkCapacity=1024 -XX:+ParallelRefProcEnabled
-XX:+UnlockExperimentalVMOptions -XX:+AggressiveOpts -XX:+DoEscapeAnalysis
-XX:ParallelGCThreads=32 -XX:ConcGCThreads=32 -XX:G1NewSizePercent=50
-XX:+DisableExplicitGC -XX:-ResizePLAB -XX:+ExitOnOutOfMemoryError
-XX:+PerfDisableSharedMem -Xms12g -Xmx12g -XX:MaxDirectMemorySize=14g
-Dpulsar.root.logger=DEBUG,FILE \""
+ PULSAR_GC: "\" -XX:+UseG1GC -XX:MaxGCPauseMillis=10\""
+ zookeeperServers: zk-0.zookeeper,zk-1.zookeeper,zk-2.zookeeper
+ globalZookeeperServers: zk-0.zookeeper,zk-1.zookeeper,zk-2.zookeeper
+ clusterName: us-east
+ managedLedgerDefaultEnsembleSize: "2"
+ managedLedgerDefaultWriteQuorum: "2"
+ managedLedgerDefaultAckQuorum: "2"
+ deduplicationEnabled: "false"
+---
+##
+## Broker deployment definition
+##
+apiVersion: apps/v1beta1
+kind: Deployment
+metadata:
+ name: broker
+ labels:
+ app: pulsar
+ component: broker
+ role: broker
+spec:
+ replicas: 1
+ template:
+ metadata:
+ labels:
+ app: pulsar
+ component: broker
+ role: broker
+ annotations:
+ prometheus.io/scrape: "true"
+ prometheus.io/port: "8080"
+ spec:
+# Enable hostNetwork only if broker pod should use the host ip
+# hostNetwork: true
+# dnsPolicy: ClusterFirstWithHostNet
+ containers:
+ - name: broker
+ image: apachepulsar/pulsar:1.20.0-incubating
+ command: ["sh", "-c"]
+ args:
+ - >
+ mkdir logs &&
+ bin/apply-config-from-env.py conf/broker.conf &&
+ bin/apply-config-from-env.py conf/pulsar_env.sh &&
+ bin/pulsar broker
+ ports:
+ - containerPort: 8080
+ # hostPort: 8080
+ - containerPort: 6650
+ # hostPort: 6650
+ envFrom:
+ - configMapRef:
+ name: broker-config
+ env:
+ - name: advertisedAddress
+ valueFrom:
+ fieldRef:
+ fieldPath: status.podIP
+# Uncomment the following if broker should be deployed on a specific instance
type
+# nodeSelector:
+# beta.kubernetes.io/instance-type: m4.2xlarge
+---
+
+##
+## Define the Broker headless service
+## In practice, in this case, it is only useful to have a view of
+## all the broker pods that are present
+##
+apiVersion: v1
+kind: Service
+metadata:
+ name: broker
+ labels:
+ app: pulsar
+ component: broker
+ role: broker
+spec:
+ ports:
+ - port: 8080
+ name: http
+ - port: 6650
+ name: pulsar
+ clusterIP: None
+ selector:
+ app: pulsar
+ component: broker
+
+---
+
+###
+
+apiVersion: v1
+kind: Pod
+metadata:
+ name: pulsar-admin
+ labels:
+ app: pulsar
+ role: admin
+spec:
+ containers:
+ - name: pulsar-admin
+ image: apachepulsar/pulsar:1.20.0-incubating
+ command: ["sh", "-c"]
+ args:
+ - >
+ bin/apply-config-from-env.py conf/client.conf &&
+ sleep 10000000000
+ envFrom:
+ - configMapRef:
+ name: broker-config
+ env:
+ - name: webServiceUrl
+ value: http://broker:8080/
+ - name: brokerServiceUrl
+ value: pulsar://broker:6650/
+
diff --git a/deployment/kubernetes/aws/zookeeper.yaml
b/deployment/kubernetes/aws/zookeeper.yaml
new file mode 100644
index 0000000..e4c5d46
--- /dev/null
+++ b/deployment/kubernetes/aws/zookeeper.yaml
@@ -0,0 +1,153 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+
+apiVersion: v1
+kind: ConfigMap
+metadata:
+ name: zookeeper-config
+data:
+ PULSAR_MEM: "\" -Dcom.sun.management.jmxremote -Djute.maxbuffer=10485760
-Xms18g -Xmx18g -XX:+ParallelRefProcEnabled -XX:+UnlockExperimentalVMOptions
-XX:+AggressiveOpts -XX:+DoEscapeAnalysis -XX:+DisableExplicitGC
-XX:+PerfDisableSharedMem -Dzookeeper.forceSync=no \""
+ PULSAR_GC: "\" -XX:+UseG1GC -XX:MaxGCPauseMillis=10\""
+---
+
+## Define a disruption budget to ensure there are at least
+## 2 ZK servers running all the time
+apiVersion: policy/v1beta1
+kind: PodDisruptionBudget
+metadata:
+ name: zk-budget
+spec:
+ selector:
+ matchLabels:
+ app: zk
+ minAvailable: 2
+---
+
+## Define a StatefulSet for ZK servers
+apiVersion: apps/v1beta1
+kind: StatefulSet
+metadata:
+ name: zk
+ labels:
+ app: pulsar
+ component: zookeeper
+spec:
+ serviceName: zookeeper
+ replicas: 3
+ template:
+ metadata:
+ labels:
+ app: pulsar
+ component: zookeeper
+ cluster: us-east
+ annotations:
+ pod.alpha.kubernetes.io/initialized: "true"
+ prometheus.io/scrape: "true"
+ prometheus.io/port: "8080"
+
+ spec:
+ # Make sure multiple pods of ZK don't get scheduled on the
+ # same node, unless there are no other available nodes
+ affinity:
+ podAntiAffinity:
+ preferredDuringSchedulingIgnoredDuringExecution:
+ - weight: 1
+ podAffinityTerm:
+ labelSelector:
+ matchExpressions:
+ - key: "app"
+ operator: In
+ values:
+ - zookeeper
+ topologyKey: "kubernetes.io/hostname"
+ containers:
+ - name: zookeeper
+ image: apachepulsar/pulsar:latest
+ command: ["sh", "-c"]
+ args:
+ - >
+ bin/apply-config-from-env.py conf/zookeeper.conf &&
+ bin/apply-config-from-env.py conf/pulsar_env.sh &&
+ bin/generate-zookeeper-config.sh conf/zookeeper.conf &&
+ bin/pulsar zookeeper
+ ports:
+ - containerPort: 2181
+ name: client
+ - containerPort: 2888
+ name: server
+ - containerPort: 3888
+ name: leader-election
+ env:
+ - name: ZOOKEEPER_SERVERS
+ value: zk-0,zk-1,zk-2
+ envFrom:
+ - configMapRef:
+ name: zookeeper-config
+
+ readinessProbe:
+ exec:
+ command:
+ - "bin/pulsar-zookeeper-ruok.sh"
+ initialDelaySeconds: 5
+ timeoutSeconds: 5
+
+ livenessProbe:
+ exec:
+ command:
+ - "bin/pulsar-zookeeper-ruok.sh"
+ initialDelaySeconds: 15
+ timeoutSeconds: 5
+
+ volumeMounts:
+ - name: datadir
+ mountPath: /pulsar/data
+ volumes:
+ - name: datadir
+ emptyDir: {}
+# Uncomment the following if zk should be deployed on a specific instance type
+# nodeSelector:
+# beta.kubernetes.io/instance-type: t2.large
+
+---
+
+##
+## Define the ZooKeeper headless service
+##
+apiVersion: v1
+kind: Service
+metadata:
+ annotations:
+ service.alpha.kubernetes.io/tolerate-unready-endpoints: "true"
+ name: zookeeper
+ labels:
+ app: pulsar
+ component: zookeeper
+spec:
+ ports:
+ - port: 2888
+ name: server
+ - port: 3888
+ name: leader-election
+ - port: 2181
+ name: stats
+ clusterIP: None
+ selector:
+ app: pulsar
+ component: zookeeper
--
To stop receiving notification emails like this one, please contact
mme...@apache.org.
