This is an automated email from the ASF dual-hosted git repository.
ywkim pushed a commit to branch cnb
in repository https://gitbox.apache.org/repos/asf/bigtop.git
The following commit(s) were added to refs/heads/cnb by this push:
new a46354f BIGTOP-3258: Deploy distributed Minio object storage
a46354f is described below
commit a46354f2b9a229de148838684acf5ba04ca500c8
Author: Youngwoo Kim <[email protected]>
AuthorDate: Fri Oct 18 16:12:35 2019 +0900
BIGTOP-3258: Deploy distributed Minio object storage
---
README.md | 43 +++++-
kubectl/plugin/kubectl-bigtop | 37 +++++
storage/minio/values.yaml | 278 +++++++++++++++++++++++++++++++++++
storage/rook/minio/object-store.yaml | 16 +-
4 files changed, 364 insertions(+), 10 deletions(-)
diff --git a/README.md b/README.md
index 02fba6e..79b9e3a 100755
--- a/README.md
+++ b/README.md
@@ -130,6 +130,8 @@ sudo apt-get install -y lvm2
```
Refer to https://rook.io/docs/rook/v1.1/k8s-pre-reqs.html for prerequisites on
Rook
+### Rook Ceph
+
Run ```download``` task to get Rook binary:
```
$ ./gradlew rook-clean rook-download && cd dl/ && tar xvfz rook-1.1.2.tar.gz
@@ -169,10 +171,12 @@ Refer to https://rook.io/docs/rook/v1.1/ceph-toolbox.html
for more details.
Create a StorageClass for Ceph RBD:
```
$ kubectl create -f
dl/rook-1.1.2/cluster/examples/kubernetes/ceph/csi/rbd/storageclass.yaml
-kubectl get storageclass
+
+$ kubectl get storageclass
rook-ceph-block
```
+### Rook Minio
Create Minio operator:
```
$ kubectl create -f
dl/rook-1.1.2/cluster/examples/kubernetes/minio/operator.yaml
@@ -180,9 +184,44 @@ $ kubectl create -f
dl/rook-1.1.2/cluster/examples/kubernetes/minio/operator.yam
#
$ kubectl -n rook-minio-system get pod
```
+
+Create object store:
```
$ kubectl create -f storage/rook/minio/object-store.yaml
-$ kubectl -n rook-minio get pod -l app=minio,objectstore=my-store
+$ kubectl -n rook-minio get objectstores.minio.rook.io
+$ kubectl -n rook-minio get pod -l app=minio,objectstore=bigtop-rook-minio
+```
+
+### Minio
+
+```
+$ cd $BIGTOP_HOME
+$ helm install --name bigtop-minio --namespace bigtop -f
storage/minio/values.yaml stable/minio
+
+```
+
+```
+Minio can be accessed via port 9000 on the following DNS name from within your
cluster:
+bigtop-minio.bigtop.svc.cluster.local
+
+To access Minio from localhost, run the below commands:
+
+ 1. export POD_NAME=$(kubectl get pods --namespace bigtop -l
"release=bigtop-minio" -o jsonpath="{.items[0].metadata.name}")
+
+ 2. kubectl port-forward $POD_NAME 9000 --namespace bigtop
+
+Read more about port forwarding here:
http://kubernetes.io/docs/user-guide/kubectl/kubectl_port-forward/
+
+You can now access Minio server on http://localhost:9000. Follow the below
steps to connect to Minio server with mc client:
+
+ 1. Download the Minio mc client -
https://docs.minio.io/docs/minio-client-quickstart-guide
+
+ 2. mc config host add bigtop-minio-local http://localhost:9000 minio
minio123 S3v4
+
+ 3. mc ls bigtop-minio-local
+
+Alternately, you can use your browser or the Minio SDK to access the server -
https://docs.minio.io/categories/17
+
```
## Zookeeper
diff --git a/kubectl/plugin/kubectl-bigtop b/kubectl/plugin/kubectl-bigtop
index 89abf3c..1291376 100755
--- a/kubectl/plugin/kubectl-bigtop
+++ b/kubectl/plugin/kubectl-bigtop
@@ -61,3 +61,40 @@ if [[ "$1" == "helm-install" ]]; then
helm version
exit 0
fi
+
+# Install Rook-Ceph
+if [[ "$1" == "rook-ceph-install" ]]; then
+
+ kubectl create -f
$BIGTOP_HOME/dl/rook-1.1.2/cluster/examples/kubernetes/ceph/common.yaml
+ kubectl create -f
$BIGTOP_HOME/dl/rook-1.1.2/cluster/examples/kubernetes/ceph/operator.yaml
+ kubectl -n rook-ceph get pod
+
+ # Ceph cluster
+ # test
+ kubectl create -f $BIGTOP_HOME/storage/rook/ceph/cluster-test.yaml
+ # production
+ # kubectl create -f $BIGTOP_HOME/storage/rook/ceph/cluster.yaml
+
+ sleep 10s
+
+ # Ceph toolbox
+ kubectl create -f
$BIGTOP_HOME/dl/rook-1.1.2/cluster/examples/kubernetes/ceph/toolbox.yaml
+
+ # StorageClass: Ceph RBD
+ kubectl create -f
$BIGTOP_HOME/dl/rook-1.1.2/cluster/examples/kubernetes/ceph/csi/rbd/storageclass.yaml
+ kubectl get storageclass
+ exit 0
+fi
+
+# Rook Ceph toolbox
+if [[ "$1" == "rook-ceph-toolbox" ]]; then
+ kubectl -n rook-ceph exec -it $(kubectl -n rook-ceph get pod -l
"app=rook-ceph-tools" -o jsonpath='{.items[0].metadata.name}') bash
+ exit 0
+fi
+
+# Minio
+if [[ "$1" == "minio-install" ]]; then
+ helm install --name bigtop-minio --namespace bigtop -f
storage/minio/values.yaml stable/minio
+ exit 0
+fi
+
diff --git a/storage/minio/values.yaml b/storage/minio/values.yaml
new file mode 100644
index 0000000..272e64a
--- /dev/null
+++ b/storage/minio/values.yaml
@@ -0,0 +1,278 @@
+## set kubernetes cluster domain where minio is running
+##
+clusterDomain: cluster.local
+
+## Set default image, imageTag, and imagePullPolicy. mode is used to indicate
the
+##
+image:
+ repository: minio/minio
+ tag: RELEASE.2019-08-07T01-59-21Z
+ pullPolicy: IfNotPresent
+
+## Set default image, imageTag, and imagePullPolicy for the `mc` (the minio
+## client used to create a default bucket).
+##
+mcImage:
+ repository: minio/mc
+ tag: RELEASE.2019-08-07T23-14-43Z
+ pullPolicy: IfNotPresent
+
+## minio server mode, i.e. standalone or distributed.
+## Distributed Minio ref:
https://docs.minio.io/docs/distributed-minio-quickstart-guide
+##
+#mode: standalone
+mode: distributed
+
+## Additional arguments to pass to minio binary
+extraArgs: []
+
+## Update strategy for Deployments
+DeploymentUpdate:
+ type: RollingUpdate
+ maxUnavailable: 0
+ maxSurge: 100%
+
+## Update strategy for StatefulSets
+StatefulSetUpdate:
+ updateStrategy: RollingUpdate
+
+## Pod priority settings
+## ref:
https://kubernetes.io/docs/concepts/configuration/pod-priority-preemption/
+##
+priorityClassName: ""
+
+## Set default accesskey, secretkey, Minio config file path, volume mount path
and
+## number of nodes (only used for Minio distributed mode)
+## AccessKey and secretKey is generated when not set
+## Distributed Minio ref:
https://docs.minio.io/docs/distributed-minio-quickstart-guide
+##
+existingSecret: ""
+accessKey: "minio"
+secretKey: "minio123"
+configPath: "/root/.minio/"
+configPathmc: "/root/.mc/"
+mountPath: "/export"
+replicas: 4
+
+## TLS Settings for Minio
+tls:
+ enabled: false
+ ## Create a secret with private.key and public.crt files and pass that here.
Ref:
https://github.com/minio/minio/tree/master/docs/tls/kubernetes#2-create-kubernetes-secret
+ certSecret: ""
+ publicCrt: public.crt
+ privateKey: private.key
+
+## Enable persistence using Persistent Volume Claims
+## ref: http://kubernetes.io/docs/user-guide/persistent-volumes/
+##
+persistence:
+ enabled: true
+
+ ## A manually managed Persistent Volume and Claim
+ ## Requires persistence.enabled: true
+ ## If defined, PVC must be created manually before volume will be bound
+ # existingClaim:
+
+ ## minio data Persistent Volume Storage Class
+ ## If defined, storageClassName: <storageClass>
+ ## If set to "-", storageClassName: "", which disables dynamic provisioning
+ ## If undefined (the default) or set to null, no storageClassName spec is
+ ## set, choosing the default provisioner. (gp2 on AWS, standard on
+ ## GKE, AWS & OpenStack)
+ ##
+ ## Storage class of PV to bind. By default it looks for standard storage
class.
+ ## If the PV uses a different storage class, specify that here.
+ # storageClass: standard
+ # VolumeName: ""
+ storageClass: "rook-ceph-block"
+
+ accessMode: ReadWriteOnce
+ size: 10Gi
+
+ ## If subPath is set mount a sub folder of a volume instead of the root of
the volume.
+ ## This is especially handy for volume plugins that don't natively support
sub mounting (like glusterfs).
+ ##
+ subPath: ""
+
+## Expose the Minio service to be accessed from outside the cluster
(LoadBalancer service).
+## or access it from within the cluster (ClusterIP service). Set the service
type and the port to serve it.
+## ref: http://kubernetes.io/docs/user-guide/services/
+##
+
+service:
+ type: ClusterIP
+ clusterIP: ~
+ port: 9000
+ nodePort: 31311
+ # externalIPs:
+ # - externalIp1
+ annotations: {}
+ # prometheus.io/scrape: 'true'
+ # prometheus.io/path: '/minio/prometheus/metrics'
+ # prometheus.io/port: '9000'
+
+## Configure Ingress based on the documentation here:
https://kubernetes.io/docs/concepts/services-networking/ingress/
+##
+
+ingress:
+ enabled: false
+ annotations: {}
+ # kubernetes.io/ingress.class: nginx
+ # kubernetes.io/tls-acme: "true"
+ # kubernetes.io/ingress.allow-http: "false"
+ # kubernetes.io/ingress.global-static-ip-name: ""
+ # nginx.ingress.kubernetes.io/secure-backends: "true"
+ # nginx.ingress.kubernetes.io/backend-protocol: "HTTPS"
+ # nginx.ingress.kubernetes.io/whitelist-source-range: 0.0.0.0/0
+ path: /
+ hosts:
+ - chart-example.local
+ tls: []
+ # - secretName: chart-example-tls
+ # hosts:
+ # - chart-example.local
+
+## Node labels for pod assignment
+## Ref: https://kubernetes.io/docs/user-guide/node-selection/
+##
+nodeSelector: {}
+tolerations: []
+affinity: {}
+
+# Additational pod annotations
+podAnnotations: {}
+
+# Additional pod labels
+podLabels: {}
+
+## Liveness and Readiness probe values.
+## ref:
https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-probes/
+livenessProbe:
+ initialDelaySeconds: 5
+ periodSeconds: 30
+ timeoutSeconds: 1
+ successThreshold: 1
+ failureThreshold: 3
+readinessProbe:
+ initialDelaySeconds: 5
+ periodSeconds: 15
+ timeoutSeconds: 1
+ successThreshold: 1
+ failureThreshold: 3
+
+## Configure resource requests and limits
+## ref: http://kubernetes.io/docs/user-guide/compute-resources/
+##
+resources:
+ requests:
+ memory: 256Mi
+ cpu: 250m
+
+## Create a bucket after minio install
+##
+defaultBucket:
+ enabled: true
+ ## If enabled, must be a string with length > 0
+ name: testbucket
+ ## Can be one of none|download|upload|public
+ policy: none
+ ## Purge if bucket exists already
+ purge: false
+
+## Create multiple buckets after minio install
+## Enabling `defaultBucket` will take priority over this list
+##
+buckets: []
+ # - name: bucket1
+ # policy: none
+ # purge: false
+ # - name: bucket2
+ # policy: none
+ # purge: false
+
+s3gateway:
+ enabled: false
+ replicas: 4
+ serviceEndpoint: ""
+
+## Use minio as an azure blob gateway, you should disable data persistence so
no volume claim are created.
+## https://docs.minio.io/docs/minio-gateway-for-azure
+azuregateway:
+ enabled: false
+ # Number of parallel instances
+ replicas: 4
+
+## Use minio as GCS (Google Cloud Storage) gateway, you should disable data
persistence so no volume claim are created.
+## https://docs.minio.io/docs/minio-gateway-for-gcs
+
+gcsgateway:
+ enabled: false
+ # Number of parallel instances
+ replicas: 4
+ # credential json file of service account key
+ gcsKeyJson: ""
+ # Google cloud project-id
+ projectId: ""
+
+ossgateway:
+ enabled: false
+ # Number of parallel instances
+ replicas: 4
+ endpointURL: ""
+
+## Use minio on NAS backend
+## https://docs.minio.io/docs/minio-gateway-for-nas
+
+nasgateway:
+ enabled: false
+ # Number of parallel instances
+ replicas: 4
+ # For NAS Gateway, you may want to bind the PVC to a specific PV. To ensure
that happens, PV to bind to should have
+ # a label like "pv: <value>", use value here.
+ pv: ~
+
+## Use this field to add environment variables relevant to Minio server. These
fields will be passed on to Minio container(s)
+## when Chart is deployed
+environment:
+ ## To disable Minio Browser, set this value to off
+ MINIO_BROWSER: "on"
+ ## To enable virtual-host-style requests, set this value to Minio host
domain name.
+ # MINIO_DOMAIN: ""
+ ## Minio Cache settings, refer:
https://docs.minio.io/docs/minio-disk-cache-guide.html
+ # MINIO_CACHE_DRIVES: ""
+ # MINIO_CACHE_EXCLUDE: ""
+ # MINIO_CACHE_EXPIRY: ""
+ # MINIO_CACHE_MAXUSE: ""
+ ## Minio WORM setting, refer:
https://docs.minio.io/docs/minio-server-configuration-guide.html
+ # MINIO_WORM: ""
+ ## Minio KMS settings, refer:
https://docs.minio.io/docs/minio-kms-quickstart-guide.html
+ # MINIO_SSE_VAULT_ENDPOINT: ""
+ # MINIO_SSE_VAULT_APPROLE_ID: ""
+ # MINIO_SSE_VAULT_APPROLE_SECRET: ""
+ # MINIO_SSE_VAULT_KEY_NAME: ""
+ ## Minio Federation settings, refer:
https://docs.minio.io/docs/minio-federation-quickstart-guide.html
+ # MINIO_ETCD_ENDPOINTS: ""
+ # MINIO_PUBLIC_IPS: ""
+ # MINIO_DOMAIN: ""
+ ## Add other environment variables relevant to Minio server here. These
values will be added to the container(s) as this Chart is deployed
+
+networkPolicy:
+ enabled: false
+ allowExternal: true
+
+## Specify the service account to use for the Minio pods. If 'create' is set
to 'false'
+## and 'name' is left unspecified, the account 'default' will be used.
+serviceAccount:
+ create: true
+ ## The name of the service account to use. If 'create' is 'true', a service
account with that name
+ ## will be created. Otherwise, a name will be auto-generated.
+ name:
+
+metrics:
+ # Metrics can not be disabled yet: https://github.com/minio/minio/issues/7493
+ serviceMonitor:
+ enabled: false
+ additionalLabels: {}
+ # namespace: monitoring
+ # interval: 30s
+ # scrapeTimeout: 10s
diff --git a/storage/rook/minio/object-store.yaml
b/storage/rook/minio/object-store.yaml
index 200aea3..e40cf24 100644
--- a/storage/rook/minio/object-store.yaml
+++ b/storage/rook/minio/object-store.yaml
@@ -6,19 +6,19 @@ metadata:
apiVersion: v1
kind: Secret
metadata:
- name: minio-my-store-access-keys
+ name: minio-object-store-access-keys
namespace: rook-minio
type: Opaque
data:
- # Base64 encoded string: "TEMP_DEMO_ACCESS_KEY"
- username: VEVNUF9ERU1PX0FDQ0VTU19LRVk=
- # Base64 encoded string: "TEMP_DEMO_SECRET_KEY"
- password: VEVNUF9ERU1PX1NFQ1JFVF9LRVk=
+ # Base64 encoded string: "minio"
+ username: bWluaW8=
+ # Base64 encoded string: "minio123"
+ password: bWluaW8xMjM=
---
apiVersion: minio.rook.io/v1alpha1
kind: ObjectStore
metadata:
- name: my-store
+ name: bigtop-rook-minio
namespace: rook-minio
spec:
scope:
@@ -55,14 +55,14 @@ spec:
podAffinity:
podAnyAffinity:
credentials:
- name: minio-my-store-access-keys
+ name: minio-object-store-access-keys
namespace: rook-minio
clusterDomain:
---
apiVersion: v1
kind: Service
metadata:
- name: minio-my-store
+ name: bigtop-rook-minio-svc
namespace: rook-minio
spec:
type: NodePort
