This is an automated email from the ASF dual-hosted git repository.
liuxun pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/gravitino-playground.git
The following commit(s) were added to refs/heads/main by this push:
new 7d49747 [Enhancement] Integrate Prometheus and Grafana for monitoring
in playground (#96)
7d49747 is described below
commit 7d497472e8949766a85e0413bcbf729494f90834
Author: JUN <[email protected]>
AuthorDate: Wed Nov 6 14:23:53 2024 +0800
[Enhancement] Integrate Prometheus and Grafana for monitoring in playground
(#96)
Closes #94
1. **Add monitoring information to the README**
2. **Add Prometheus and Grafana settings to docker-compose**
3. **Add a basic Prometheus configuration file**
4. **Add a basic Grafana configuration, provision data source to
Prometheus, and provision a dashboard to Gravitino Playground**
5. **Add port checks in `playground.sh`**
### Fixes
- Change `- 18888:8888` to `- "18888:8888"` in the docker-compose.yaml
for correct port mapping format.
- Change `docker-compose ps` to `docker-compose ps -a` to view all
containers (including stopped ones).
---
**Note**: Please ignore the high JVM Non-Heap Memory Usage and JVM
Direct Memory Usage caused by my WSL2 environment.
---------
Signed-off-by: jun <[email protected]>
---
README.md | 24 ++++++++++++++++-------
docker-compose.yaml | 20 ++++++++++++++++++-
init/grafana/dashboards/dashboards.yaml | 10 ++++++++++
init/grafana/dashboards/gravitino-playground.json | 1 +
init/grafana/datasources.yaml | 7 +++++++
init/grafana/grafana.ini | 9 +++++++++
init/prometheus/prometheus.yml | 8 ++++++++
playground.sh | 4 ++--
8 files changed, 73 insertions(+), 10 deletions(-)
diff --git a/README.md b/README.md
index a81d174..ef50544 100644
--- a/README.md
+++ b/README.md
@@ -35,14 +35,16 @@ Install Git, Docker, Docker Compose.
The playground runs a number of services. The TCP ports used may clash with
existing services you run, such as MySQL or Postgres.
-| Docker container | Ports used |
-|-----------------------|----------------------|
-| playground-gravitino | 8090 9001 |
+| Docker container | Ports used |
+|-----------------------|------------------------|
+| playground-gravitino | 8090 9001 |
| playground-hive | 3307 19000 19083 60070 |
-| playground-mysql | 13306 |
-| playground-postgresql | 15342 |
-| playground-trino | 18080 |
-| playground-jupyter | 18888 |
+| playground-mysql | 13306 |
+| playground-postgresql | 15342 |
+| playground-trino | 18080 |
+| playground-jupyter | 18888 |
+| playground-prometheus | 19090 |
+| playground-grafana | 13000 |
## Playground usage
@@ -101,6 +103,14 @@ docker exec -it playground-spark bash
spark@container_id:/$ cd /opt/spark && /bin/bash bin/spark-sql
```
+## Monitoring Gravitino
+
+1. Open the Grafana in the browser at
[http://localhost:13000](http://localhost:13000).
+
+2. In the navigation menu, click **Dashboards** -> **Gravitino Playground**.
+
+3. Experiment with the default template.
+
## Example
### Simple Trino queries
diff --git a/docker-compose.yaml b/docker-compose.yaml
index 1decede..6eee836 100644
--- a/docker-compose.yaml
+++ b/docker-compose.yaml
@@ -141,7 +141,7 @@ services:
image: jupyter/pyspark-notebook:spark-3.4.1
container_name: playground-jupyter
ports:
- - 18888:8888
+ - "18888:8888"
volumes:
- ./init/jupyter:/tmp/gravitino
entrypoint: /bin/bash /tmp/gravitino/init.sh
@@ -151,5 +151,23 @@ services:
gravitino :
condition: service_healthy
+ prometheus:
+ image: prom/prometheus:v2.55.0
+ container_name: playground-prometheus
+ ports:
+ - "19090:9090"
+ volumes:
+ - ./init/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml
+
+ grafana:
+ image: grafana/grafana:11.3.0-ubuntu
+ container_name: playground-grafana
+ ports:
+ - "13000:3000"
+ volumes:
+ - ./init/grafana/grafana.ini:/etc/grafana/grafana.ini
+ -
./init/grafana/datasources.yaml:/etc/grafana/provisioning/datasources/datasources.yaml
+ - ./init/grafana/dashboards:/etc/grafana/provisioning/dashboards
+
volumes:
spark_jars:
\ No newline at end of file
diff --git a/init/grafana/dashboards/dashboards.yaml
b/init/grafana/dashboards/dashboards.yaml
new file mode 100644
index 0000000..b34b703
--- /dev/null
+++ b/init/grafana/dashboards/dashboards.yaml
@@ -0,0 +1,10 @@
+apiVersion: 1
+providers:
+ - name: 'Playground Dashboards'
+ orgId: 1
+ folder: ''
+ type: file
+ disableDeletion: false
+ editable: true
+ options:
+ path: /etc/grafana/provisioning/dashboards
\ No newline at end of file
diff --git a/init/grafana/dashboards/gravitino-playground.json
b/init/grafana/dashboards/gravitino-playground.json
new file mode 100644
index 0000000..4786f41
--- /dev/null
+++ b/init/grafana/dashboards/gravitino-playground.json
@@ -0,0 +1 @@
+{"annotations":{"list":[{"builtIn":1,"datasource":{"type":"grafana","uid":"--
Grafana --"},"enable":true,"hide":true,"iconColor":"rgba(0, 211, 255,
1)","name":"Annotations &
Alerts","type":"dashboard"}]},"editable":true,"fiscalYearStartMonth":0,"graphTooltip":0,"id":6,"links":[],"panels":[{"collapsed":false,"gridPos":{"h":1,"w":24,"x":0,"y":0},"id":4,"panels":[],"title":"Quick
Facts","type":"row"},{"datasource":{"type":"prometheus","uid":"PBFA97CFB590B2093"},"fieldConfig":{"defaults":{"c
[...]
\ No newline at end of file
diff --git a/init/grafana/datasources.yaml b/init/grafana/datasources.yaml
new file mode 100644
index 0000000..8d9f9d8
--- /dev/null
+++ b/init/grafana/datasources.yaml
@@ -0,0 +1,7 @@
+apiVersion: 1
+datasources:
+ - name: Prometheus
+ type: prometheus
+ access: proxy
+ url: http://prometheus:9090
+ isDefault: true
\ No newline at end of file
diff --git a/init/grafana/grafana.ini b/init/grafana/grafana.ini
new file mode 100644
index 0000000..a13f34e
--- /dev/null
+++ b/init/grafana/grafana.ini
@@ -0,0 +1,9 @@
+[auth]
+disable_login_form = true
+
+[auth.anonymous]
+enabled = true
+org_role = Admin
+
+[dashboards]
+default_home_dashboard_path =
/etc/grafana/provisioning/dashboards/gravitino-playground.json
\ No newline at end of file
diff --git a/init/prometheus/prometheus.yml b/init/prometheus/prometheus.yml
new file mode 100644
index 0000000..602effc
--- /dev/null
+++ b/init/prometheus/prometheus.yml
@@ -0,0 +1,8 @@
+global:
+ scrape_interval: 15s
+
+scrape_configs:
+ - job_name: 'gravitino'
+ static_configs:
+ - targets: ['gravitino:8090']
+ metrics_path: '/prometheus/metrics'
\ No newline at end of file
diff --git a/playground.sh b/playground.sh
index abd4777..84b2c07 100755
--- a/playground.sh
+++ b/playground.sh
@@ -64,7 +64,7 @@ start() {
testDocker
checkCompose
- ports=(8090 9001 3307 19000 19083 60070 13306 15342 18080 18888)
+ ports=(8090 9001 3307 19000 19083 60070 13306 15342 18080 18888 19090 13000)
for port in "${ports[@]}"; do
checkPortInUse ${port}
done
@@ -82,7 +82,7 @@ start() {
}
status() {
- docker-compose ps
+ docker-compose ps -a
}
stop() {
