[bigtop] branch cnb updated: BIGTOP-3250: Upgrade Presto docker image
This is an automated email from the ASF dual-hosted git repository.
ywkim pushed a commit to branch cnb
in repository https://gitbox.apache.org/repos/asf/bigtop.git
The following commit(s) were added to refs/heads/cnb by this push:
new 9a40032 BIGTOP-3250: Upgrade Presto docker image
9a40032 is described below
commit 9a40032bb65443db6e0e696f67802386d75b8c1c
Author: Youngwoo Kim
AuthorDate: Wed Oct 16 17:04:29 2019 +0900
BIGTOP-3250: Upgrade Presto docker image
---
presto3-minio/coordinator/Dockerfile | 6 --
presto3-minio/coordinator/README.md | 8
presto3-minio/coordinator/config.properties | 8
presto3-minio/coordinator/json| 2 --
presto3-minio/test.sh | 19 ---
presto3-minio/worker/Dockerfile | 5 -
presto3-minio/worker/README.md| 6 --
presto3-minio/worker/json | 2 --
{presto3-minio => prestosql}/README.md| 0
prestosql/docker/Makefile | 8
prestosql/docker/README.md| 6 ++
prestosql/docker/coordinator/Dockerfile | 14 ++
prestosql/docker/coordinator/README.md| 7 +++
prestosql/docker/coordinator/config.properties| 5 +
prestosql/docker/coordinator/jvm.config | 12
.../docker/coordinator}/minio.properties | 0
prestosql/docker/coordinator/node.properties | 3 +++
prestosql/docker/worker/Dockerfile| 12
prestosql/docker/worker/README.md | 6 ++
.../docker}/worker/config.properties | 3 ---
prestosql/docker/worker/jvm.config| 12
.../docker/worker}/minio.properties | 0
prestosql/docker/worker/node.properties | 3 +++
.../presto-deployment-master.yaml | 2 +-
.../presto-deployment-worker.yaml | 4 ++--
{presto3-minio => prestosql}/presto-deployment.yaml | 2 +-
{presto3-minio => prestosql}/service.yaml | 0
27 files changed, 92 insertions(+), 63 deletions(-)
diff --git a/presto3-minio/coordinator/Dockerfile
b/presto3-minio/coordinator/Dockerfile
deleted file mode 100755
index ac9c71b..000
--- a/presto3-minio/coordinator/Dockerfile
+++ /dev/null
@@ -1,6 +0,0 @@
-FROM wlan0/presto-benchmark-environment:coordinator
-COPY ./json /resources/nyc_taxi_data_3M.json
-COPY ./minio.properties
/resources/presto-server-302-e.11/etc/catalog/minio.properties
-
-COPY config.properties /resources/presto-server-302-e.11/etc/config.properties
-CMD cat /resources/presto-server-302-e.11/etc/config.properties ;
/resources/presto-server-302-e.11/bin/launcher start && cat
/resources/presto-server-302-e.11/etc/config.properties && echo "Hardcoded to
s3://minio:9000 , using 'minio' and 'minio123' as the access credentials..." &&
sleep 60 && cat /resources/presto-server-302-e.11/etc/config.properties &&
tail -f ./resources/presto-server-302-e.11/var/log/server.log
diff --git a/presto3-minio/coordinator/README.md
b/presto3-minio/coordinator/README.md
deleted file mode 100755
index 3423f65..000
--- a/presto3-minio/coordinator/README.md
+++ /dev/null
@@ -1,8 +0,0 @@
-# Use squash !
-
-Building this image, make sure you use squash so you can gut the 12 GB file :)
-
-
-docker build --squash -t jayunit100/wlan0-presto:latest ./
-
-
diff --git a/presto3-minio/coordinator/config.properties
b/presto3-minio/coordinator/config.properties
deleted file mode 100755
index 6a214ae..000
--- a/presto3-minio/coordinator/config.properties
+++ /dev/null
@@ -1,8 +0,0 @@
-coordinator=true
-node-scheduler.include-coordinator=true
-http-server.http.port=8080
-query.max-memory=1GB
-query.max-memory-per-node=1GB
-query.max-total-memory-per-node=1GB
-discovery-server.enabled=true
-discovery.uri=http://presto:8080
diff --git a/presto3-minio/coordinator/json b/presto3-minio/coordinator/json
deleted file mode 100755
index 6906dcd..000
--- a/presto3-minio/coordinator/json
+++ /dev/null
@@ -1,2 +0,0 @@
-{"id":"3418178","vendor_id":"1","pickup_datetime":"2014-03-13
20:17:16","dropoff_datetime":"2014-03-13
20:22:29","store_and_fwd_flag":"N","rate_code_id":"1","pickup_longitude":"-73.9873046875","pickup_latitude":"40.691890716552734","dropoff_longitude":"-73.975677490234375","dropoff_latitude":"40.696006774902344","passenger_count":"1","trip_distance":"0.90","fare_amount":"5.5","extra":"0.5","mta_tax":"0.5","tip_amount":"0","tolls_amount":"0","ehail_fee":"","improvement_surcharge":"","tota
[...]
-
diff --git a/presto3-minio/test.sh b/presto3-minio/test.sh
deleted file mode 100755
index be3..000
--- a/presto3-minio/test.sh
+++ /dev/null
@@ -1,19 +0,0 @@
-echo "ok to cleanup?"
-read x
-
[bigtop] branch cnb updated: BIGTOP-3263: Log aggregation for Kubernetes cluster
This is an automated email from the ASF dual-hosted git repository.
ywkim pushed a commit to branch cnb
in repository https://gitbox.apache.org/repos/asf/bigtop.git
The following commit(s) were added to refs/heads/cnb by this push:
new d35e545 BIGTOP-3263: Log aggregation for Kubernetes cluster
d35e545 is described below
commit d35e545088c2003cfd4d7de3d378ec240c026106
Author: Youngwoo Kim
AuthorDate: Thu Oct 24 17:21:05 2019 +0900
BIGTOP-3263: Log aggregation for Kubernetes cluster
---
README.md | 2 +
kubectl/plugin/kubectl-bigtop | 11 +++
logs/loki/README.md| 22 +
logs/loki/loki-values.yaml | 208 +
logs/loki/promtail-values.yaml | 136 +++
5 files changed, 379 insertions(+)
diff --git a/README.md b/README.md
index a865f6f..6b980a4 100755
--- a/README.md
+++ b/README.md
@@ -348,6 +348,8 @@ See spark/README.md
See prometheus/README.md
+And also, You can aggregate logs from Kubernetes cluster (Pods) in using Loki
stack (Loki and promtail). See logs/loki/README.md for deploying Liki and
querying logs from Loki on Grafana.
+
# Getting Started
Below are some recipes for getting started with using Apache Bigtop. As Apache
Bigtop has different subprojects, these recipes will continue to evolve.
diff --git a/kubectl/plugin/kubectl-bigtop b/kubectl/plugin/kubectl-bigtop
index e2c921e..3387d11 100755
--- a/kubectl/plugin/kubectl-bigtop
+++ b/kubectl/plugin/kubectl-bigtop
@@ -123,3 +123,14 @@ if [[ "$1" == "minio-deploy" ]]; then
exit 0
fi
+# Loki
+if [[ "$1" == "loki-deploy" ]]; then
+helm repo add loki https://grafana.github.io/loki/charts
+helm repo update
+helm upgrade --install --namespace default loki -f
logs/loki/loki-values.yaml loki/loki
+sleep 10s
+helm upgrade --install --namespace default promtail -f
logs/loki/proimtail-values.yaml loki/promtail --set "loki.serviceName=loki"
+# http url for loki server: http://loki:3100
+helm list && kubectl get po -A
+exit 0
+fi
diff --git a/logs/loki/README.md b/logs/loki/README.md
new file mode 100644
index 000..e17933b
--- /dev/null
+++ b/logs/loki/README.md
@@ -0,0 +1,22 @@
+# Log aggregation for Kubernetes cluster
+
+Deploy Loki stack:
+```
+$ helm repo add loki https://grafana.github.io/loki/charts
+$ helm repo update
+$ helm upgrade --install --namespace default loki -f
logs/loki/loki-values.yaml loki/loki
+$ helm upgrade --install --namespace default promtail -f
logs/loki/promtail-values.yaml loki/promtail --set "loki.serviceName=loki"
+```
+
+## Loki in Grafana
+You should add the Loki data source to Grafana configuration.
+
+1. In Grafana, go to Configuration > Data Sources
+2. Go to ```Add data source```
+3. Choose Loki --> The http URL field ```http://loki:3100```
+
+## Querying logs from Loki on Grafana
+
+1. Explore > Choose 'Loki' data source
+2. Enter a Liki query. E.g., ```{app="zookeeper",namespace="bigtop"}```
+3. You can see the log streams from Zookeeper Pods on Kubernetes cluster
diff --git a/logs/loki/loki-values.yaml b/logs/loki/loki-values.yaml
new file mode 100644
index 000..623a11f
--- /dev/null
+++ b/logs/loki/loki-values.yaml
@@ -0,0 +1,208 @@
+## Affinity for pod assignment
+## ref:
https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity
+affinity: {}
+# podAntiAffinity:
+# requiredDuringSchedulingIgnoredDuringExecution:
+# - labelSelector:
+# matchExpressions:
+# - key: app
+# operator: In
+# values:
+# - loki
+# topologyKey: "kubernetes.io/hostname"
+
+## StatefulSet annotations
+annotations: {}
+
+# enable tracing for debug, need install jaeger and specify right
jaeger_agent_host
+tracing:
+ jaegerAgentHost:
+
+config:
+ auth_enabled: false
+ ingester:
+chunk_idle_period: 3m
+chunk_block_size: 262144
+chunk_retain_period: 1m
+lifecycler:
+ ring:
+kvstore:
+ store: inmemory
+replication_factor: 1
+
+ ## Different ring configs can be used. E.g. Consul
+ # ring:
+ # store: consul
+ # replication_factor: 1
+ # consul:
+ # host: "consul:8500"
+ # prefix: ""
+ # httpclienttimeout: "20s"
+ # consistentreads: true
+ limits_config:
+enforce_metric_name: false
+reject_old_samples: true
+reject_old_samples_max_age: 168h
+ schema_config:
+configs:
+- from: 2018-04-15
+ store: boltdb
+ object_store: filesystem
+ schema: v9
+ index:
+prefix: index_
+period: 168h
+ server:
+http_listen_port: 3100
+ storage_config:
+boltdb:
+ directory: /data/loki/index
+filesystem:
+ directory: /data/loki/chunks
+ chunk_store_config:
+max_look_back_period: 0
+ table_manager:
+retention_deletes_enabled: false
+retention_period: 0
+
+image:
+ repository: grafana/loki
+ tag: v0.3.0
+
