This is an automated email from the ASF dual-hosted git repository.
ywkim pushed a commit to branch cnb
in repository https://gitbox.apache.org/repos/asf/bigtop.git
The following commit(s) were added to refs/heads/cnb by this push:
new d35e545 BIGTOP-3263: Log aggregation for Kubernetes cluster
d35e545 is described below
commit d35e545088c2003cfd4d7de3d378ec240c026106
Author: Youngwoo Kim <[email protected]>
AuthorDate: Thu Oct 24 17:21:05 2019 +0900
BIGTOP-3263: Log aggregation for Kubernetes cluster
---
README.md | 2 +
kubectl/plugin/kubectl-bigtop | 11 +++
logs/loki/README.md | 22 +++++
logs/loki/loki-values.yaml | 208 +++++++++++++++++++++++++++++++++++++++++
logs/loki/promtail-values.yaml | 136 +++++++++++++++++++++++++++
5 files changed, 379 insertions(+)
diff --git a/README.md b/README.md
index a865f6f..6b980a4 100755
--- a/README.md
+++ b/README.md
@@ -348,6 +348,8 @@ See spark/README.md
See prometheus/README.md
+And also, You can aggregate logs from Kubernetes cluster (Pods) in using Loki
stack (Loki and promtail). See logs/loki/README.md for deploying Liki and
querying logs from Loki on Grafana.
+
# Getting Started
Below are some recipes for getting started with using Apache Bigtop. As Apache
Bigtop has different subprojects, these recipes will continue to evolve.
diff --git a/kubectl/plugin/kubectl-bigtop b/kubectl/plugin/kubectl-bigtop
index e2c921e..3387d11 100755
--- a/kubectl/plugin/kubectl-bigtop
+++ b/kubectl/plugin/kubectl-bigtop
@@ -123,3 +123,14 @@ if [[ "$1" == "minio-deploy" ]]; then
exit 0
fi
+# Loki
+if [[ "$1" == "loki-deploy" ]]; then
+ helm repo add loki https://grafana.github.io/loki/charts
+ helm repo update
+ helm upgrade --install --namespace default loki -f
logs/loki/loki-values.yaml loki/loki
+ sleep 10s
+ helm upgrade --install --namespace default promtail -f
logs/loki/proimtail-values.yaml loki/promtail --set "loki.serviceName=loki"
+ # http url for loki server: http://loki:3100
+ helm list && kubectl get po -A
+ exit 0
+fi
diff --git a/logs/loki/README.md b/logs/loki/README.md
new file mode 100644
index 0000000..e17933b
--- /dev/null
+++ b/logs/loki/README.md
@@ -0,0 +1,22 @@
+# Log aggregation for Kubernetes cluster
+
+Deploy Loki stack:
+```
+$ helm repo add loki https://grafana.github.io/loki/charts
+$ helm repo update
+$ helm upgrade --install --namespace default loki -f
logs/loki/loki-values.yaml loki/loki
+$ helm upgrade --install --namespace default promtail -f
logs/loki/promtail-values.yaml loki/promtail --set "loki.serviceName=loki"
+```
+
+## Loki in Grafana
+You should add the Loki data source to Grafana configuration.
+
+1. In Grafana, go to Configuration > Data Sources
+2. Go to ```Add data source```
+3. Choose Loki --> The http URL field ```http://loki:3100```
+
+## Querying logs from Loki on Grafana
+
+1. Explore > Choose 'Loki' data source
+2. Enter a Liki query. E.g., ```{app="zookeeper",namespace="bigtop"}```
+3. You can see the log streams from Zookeeper Pods on Kubernetes cluster
diff --git a/logs/loki/loki-values.yaml b/logs/loki/loki-values.yaml
new file mode 100644
index 0000000..623a11f
--- /dev/null
+++ b/logs/loki/loki-values.yaml
@@ -0,0 +1,208 @@
+## Affinity for pod assignment
+## ref:
https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity
+affinity: {}
+# podAntiAffinity:
+# requiredDuringSchedulingIgnoredDuringExecution:
+# - labelSelector:
+# matchExpressions:
+# - key: app
+# operator: In
+# values:
+# - loki
+# topologyKey: "kubernetes.io/hostname"
+
+## StatefulSet annotations
+annotations: {}
+
+# enable tracing for debug, need install jaeger and specify right
jaeger_agent_host
+tracing:
+ jaegerAgentHost:
+
+config:
+ auth_enabled: false
+ ingester:
+ chunk_idle_period: 3m
+ chunk_block_size: 262144
+ chunk_retain_period: 1m
+ lifecycler:
+ ring:
+ kvstore:
+ store: inmemory
+ replication_factor: 1
+
+ ## Different ring configs can be used. E.g. Consul
+ # ring:
+ # store: consul
+ # replication_factor: 1
+ # consul:
+ # host: "consul:8500"
+ # prefix: ""
+ # httpclienttimeout: "20s"
+ # consistentreads: true
+ limits_config:
+ enforce_metric_name: false
+ reject_old_samples: true
+ reject_old_samples_max_age: 168h
+ schema_config:
+ configs:
+ - from: 2018-04-15
+ store: boltdb
+ object_store: filesystem
+ schema: v9
+ index:
+ prefix: index_
+ period: 168h
+ server:
+ http_listen_port: 3100
+ storage_config:
+ boltdb:
+ directory: /data/loki/index
+ filesystem:
+ directory: /data/loki/chunks
+ chunk_store_config:
+ max_look_back_period: 0
+ table_manager:
+ retention_deletes_enabled: false
+ retention_period: 0
+
+image:
+ repository: grafana/loki
+ tag: v0.3.0
+ pullPolicy: IfNotPresent
+
+## Additional Loki container arguments, e.g. log level (debug, info, warn,
error)
+extraArgs: {}
+ # log.level: debug
+
+livenessProbe:
+ httpGet:
+ path: /ready
+ port: http-metrics
+ initialDelaySeconds: 45
+
+## Enable persistence using Persistent Volume Claims
+networkPolicy:
+ enabled: false
+
+## ref: https://kubernetes.io/docs/user-guide/node-selection/
+nodeSelector: {}
+
+## ref: http://kubernetes.io/docs/user-guide/persistent-volumes/
+## If you set enabled as "True", you need :
+## - create a pv which above 10Gi and has same namespace with loki
+## - keep storageClassName same with below setting
+persistence:
+ enabled: true
+ accessModes:
+ - ReadWriteOnce
+ size: 10Gi
+ annotations: {}
+ # subPath: ""
+ # existingClaim:
+
+## Pod Labels
+podLabels: {}
+
+## Pod Annotations
+podAnnotations:
+ prometheus.io/scrape: "true"
+ prometheus.io/port: "http-metrics"
+
+podManagementPolicy: OrderedReady
+
+## Assign a PriorityClassName to pods if set
+# priorityClassName:
+
+rbac:
+ create: true
+ pspEnabled: true
+
+readinessProbe:
+ httpGet:
+ path: /ready
+ port: http-metrics
+ initialDelaySeconds: 45
+
+replicas: 1
+
+resources: {}
+# limits:
+# cpu: 200m
+# memory: 256Mi
+# requests:
+# cpu: 100m
+# memory: 128Mi
+
+securityContext:
+ fsGroup: 10001
+ runAsGroup: 10001
+ runAsNonRoot: true
+ runAsUser: 10001
+
+service:
+ type: ClusterIP
+ nodePort:
+ port: 3100
+ annotations: {}
+ labels: {}
+
+serviceAccount:
+ create: true
+ name:
+
+terminationGracePeriodSeconds: 30
+
+## Tolerations for pod assignment
+## ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/
+tolerations: []
+
+# The values to set in the PodDisruptionBudget spec
+# If not set then a PodDisruptionBudget will not be created
+podDisruptionBudget: {}
+# minAvailable: 1
+# maxUnavailable: 1
+
+updateStrategy:
+ type: RollingUpdate
+
+serviceMonitor:
+ enabled: true
+ interval: ""
+ additionalLabels: {}
+ # scrapeTimeout: 10s
+
+extraContainers: []
+## Additional containers to be added to the loki pod.
+# - name: reverse-proxy
+# image: angelbarrera92/basic-auth-reverse-proxy:dev
+# args:
+# - "serve"
+# - "--upstream=http://localhost:3100";
+# - "--auth-config=/etc/reverse-proxy-conf/authn.yaml"
+# ports:
+# - name: http
+# containerPort: 11811
+# protocol: TCP
+# volumeMounts:
+# - name: reverse-proxy-auth-config
+# mountPath: /etc/reverse-proxy-conf
+
+
+extraVolumes: []
+## Additional volumes to the loki pod.
+# - name: reverse-proxy-auth-config
+# secret:
+# secretName: reverse-proxy-auth-config
+
+## Extra volume mounts that will be added to the loki container
+extraVolumeMounts: []
+
+extraPorts: []
+## Additional ports to the loki services. Useful to expose extra container
ports.
+# - port: 11811
+# protocol: TCP
+# name: http
+# targetPort: http
+
+# Extra env variables to pass to the loki container
+env: []
diff --git a/logs/loki/promtail-values.yaml b/logs/loki/promtail-values.yaml
new file mode 100644
index 0000000..959bf54
--- /dev/null
+++ b/logs/loki/promtail-values.yaml
@@ -0,0 +1,136 @@
+## Affinity for pod assignment
+## ref:
https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity
+affinity: {}
+
+annotations: {}
+
+deploymentStrategy: RollingUpdate
+
+image:
+ repository: grafana/promtail
+ tag: v0.3.0
+ pullPolicy: IfNotPresent
+
+livenessProbe: {}
+
+loki:
+ serviceName: "" # Defaults to "${RELEASE}-loki" if not set
+ servicePort: 3100
+ serviceScheme: http
+ # user: user
+ # password: pass
+
+nameOverride: promtail
+
+## Node labels for pod assignment
+## ref: https://kubernetes.io/docs/user-guide/node-selection/
+nodeSelector: {}
+
+pipelineStages:
+- docker: {}
+
+## Pod Labels
+podLabels: {}
+
+podAnnotations:
+ prometheus.io/scrape: "true"
+ prometheus.io/port: "http-metrics"
+
+## Assign a PriorityClassName to pods if set
+# priorityClassName:
+
+rbac:
+ create: true
+ pspEnabled: true
+
+readinessProbe:
+ failureThreshold: 5
+ httpGet:
+ path: /ready
+ port: http-metrics
+ initialDelaySeconds: 10
+ periodSeconds: 10
+ successThreshold: 1
+ timeoutSeconds: 1
+
+resources: {}
+# limits:
+# cpu: 200m
+# memory: 128Mi
+# requests:
+# cpu: 100m
+# memory: 128Mi
+
+# Custom scrape_configs to override the default ones in the configmap
+scrapeConfigs: []
+
+# Custom scrape_configs together with the default ones in the configmap
+extraScrapeConfigs: []
+
+securityContext:
+ readOnlyRootFilesystem: true
+ runAsGroup: 0
+ runAsUser: 0
+
+serviceAccount:
+ create: true
+ name:
+
+## Tolerations for pod assignment
+## ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/
+tolerations:
+- key: node-role.kubernetes.io/master
+ operator: Exists
+ effect: NoSchedule
+
+# Extra volumes to scrape logs from
+volumes:
+- name: docker
+ hostPath:
+ path: /var/lib/docker/containers
+- name: pods
+ hostPath:
+ path: /var/log/pods
+
+volumeMounts:
+- name: docker
+ mountPath: /var/lib/docker/containers
+ readOnly: true
+- name: pods
+ mountPath: /var/log/pods
+ readOnly: true
+
+config:
+ client:
+ # Maximum wait period before sending batch
+ batchwait: 1s
+ # Maximum batch size to accrue before sending, unit is byte
+ batchsize: 102400
+
+ # Maximum time to wait for server to respond to a request
+ timeout: 10s
+
+ backoff_config:
+ # Initial backoff time between retries
+ minbackoff: 100ms
+ # Maximum backoff time between retries
+ maxbackoff: 5s
+ # Maximum number of retries when sending batches, 0 means infinite
retries
+ maxretries: 20
+
+ # The labels to add to any time series or alerts when communicating with
loki
+ external_labels: {}
+
+ server:
+ http_listen_port: 3101
+ positions:
+ filename: /run/promtail/positions.yaml
+ target_config:
+ # Period to resync directories being watched and files being tailed
+ sync_period: 10s
+
+serviceMonitor:
+ enabled: false
+ interval: ""
+ additionalLabels: {}
+ # scrapeTimeout: 10s
