andytael commented on issue #10921:
URL: https://github.com/apache/apisix/issues/10921#issuecomment-1998501139
Changing the configuration to:
```yaml
prometheus:
# ref: https://apisix.apache.org/docs/apisix/plugins/prometheus/
enabled: true
# -- path of the metrics endpoint
path: /apisix/prometheus/metrics
# -- prefix of the metrics
metricPrefix: apisix_
# -- container port where the metrics are exposed
containerPort: 9091
metrics: # For http request related metrics
http_status:
extra_labels:
- upstream_addr: $upstream_addr
- upstream_status: $upstream_status # The label name does not
need to be the same as the variable name.
http_latency:
extra_labels:
- upstream_addr: $upstream_addr
bandwidth:
extra_labels:
- upstream_addr: $upstream_addr
```
and put some load on APISIX site doesn't give any metrics for
apisix_bandwidth, apisix_http_status and apisix_http_latency
```text
# HELP apisix_etcd_modify_indexes Etcd modify index for APISIX keys
# TYPE apisix_etcd_modify_indexes gauge
apisix_etcd_modify_indexes{key="consumers"} 0
apisix_etcd_modify_indexes{key="global_rules"} 0
apisix_etcd_modify_indexes{key="max_modify_index"} 20
apisix_etcd_modify_indexes{key="prev_index"} 14
apisix_etcd_modify_indexes{key="protos"} 0
apisix_etcd_modify_indexes{key="routes"} 20
apisix_etcd_modify_indexes{key="services"} 0
apisix_etcd_modify_indexes{key="ssls"} 0
apisix_etcd_modify_indexes{key="stream_routes"} 0
apisix_etcd_modify_indexes{key="upstreams"} 0
apisix_etcd_modify_indexes{key="x_etcd_index"} 20
# HELP apisix_etcd_reachable Config server etcd reachable from APISIX, 0 is
unreachable
# TYPE apisix_etcd_reachable gauge
apisix_etcd_reachable 1
# HELP apisix_http_requests_total The total number of client requests since
APISIX started
# TYPE apisix_http_requests_total gauge
apisix_http_requests_total 1225
# HELP apisix_nginx_http_current_connections Number of HTTP connections
# TYPE apisix_nginx_http_current_connections gauge
apisix_nginx_http_current_connections{state="accepted"} 133
apisix_nginx_http_current_connections{state="active"} 20
apisix_nginx_http_current_connections{state="handled"} 133
apisix_nginx_http_current_connections{state="reading"} 0
apisix_nginx_http_current_connections{state="waiting"} 14
apisix_nginx_http_current_connections{state="writing"} 6
# HELP apisix_nginx_metric_errors_total Number of nginx-lua-prometheus errors
# TYPE apisix_nginx_metric_errors_total counter
apisix_nginx_metric_errors_total 0
# HELP apisix_node_info Info of APISIX node
# TYPE apisix_node_info gauge
apisix_node_info{hostname="apisix-97ffdb47c-ngb6l"} 1
# HELP apisix_shared_dict_capacity_bytes The capacity of each nginx shared
DICT since APISIX start
# TYPE apisix_shared_dict_capacity_bytes gauge
apisix_shared_dict_capacity_bytes{name="access-tokens"} 1048576
apisix_shared_dict_capacity_bytes{name="balancer-ewma"} 10485760
apisix_shared_dict_capacity_bytes{name="balancer-ewma-last-touched-at"}
10485760
apisix_shared_dict_capacity_bytes{name="balancer-ewma-locks"} 10485760
apisix_shared_dict_capacity_bytes{name="discovery"} 1048576
apisix_shared_dict_capacity_bytes{name="etcd-cluster-health-check"} 10485760
apisix_shared_dict_capacity_bytes{name="ext-plugin"} 1048576
apisix_shared_dict_capacity_bytes{name="internal-status"} 10485760
apisix_shared_dict_capacity_bytes{name="introspection"} 10485760
apisix_shared_dict_capacity_bytes{name="jwks"} 1048576
apisix_shared_dict_capacity_bytes{name="kubernetes"} 1048576
apisix_shared_dict_capacity_bytes{name="lrucache-lock"} 10485760
apisix_shared_dict_capacity_bytes{name="plugin-api-breaker"} 10485760
apisix_shared_dict_capacity_bytes{name="plugin-limit-conn"} 10485760
apisix_shared_dict_capacity_bytes{name="plugin-limit-count"} 10485760
apisix_shared_dict_capacity_bytes{name="plugin-limit-count-redis-cluster-slot-lock"}
1048576
apisix_shared_dict_capacity_bytes{name="plugin-limit-count-reset-header"}
10485760
apisix_shared_dict_capacity_bytes{name="plugin-limit-req"} 10485760
apisix_shared_dict_capacity_bytes{name="prometheus-metrics"} 10485760
apisix_shared_dict_capacity_bytes{name="upstream-healthcheck"} 10485760
apisix_shared_dict_capacity_bytes{name="worker-events"} 10485760
# HELP apisix_shared_dict_free_space_bytes The free space of each nginx
shared DICT since APISIX start
# TYPE apisix_shared_dict_free_space_bytes gauge
apisix_shared_dict_free_space_bytes{name="access-tokens"} 1032192
apisix_shared_dict_free_space_bytes{name="balancer-ewma"} 10412032
apisix_shared_dict_free_space_bytes{name="balancer-ewma-last-touched-at"}
10412032
apisix_shared_dict_free_space_bytes{name="balancer-ewma-locks"} 10412032
apisix_shared_dict_free_space_bytes{name="discovery"} 1032192
apisix_shared_dict_free_space_bytes{name="etcd-cluster-health-check"}
10412032
apisix_shared_dict_free_space_bytes{name="ext-plugin"} 1032192
apisix_shared_dict_free_space_bytes{name="internal-status"} 10407936
apisix_shared_dict_free_space_bytes{name="introspection"} 10412032
apisix_shared_dict_free_space_bytes{name="jwks"} 1032192
apisix_shared_dict_free_space_bytes{name="kubernetes"} 999424
apisix_shared_dict_free_space_bytes{name="lrucache-lock"} 10412032
apisix_shared_dict_free_space_bytes{name="plugin-api-breaker"} 10412032
apisix_shared_dict_free_space_bytes{name="plugin-limit-conn"} 10412032
apisix_shared_dict_free_space_bytes{name="plugin-limit-count"} 10412032
apisix_shared_dict_free_space_bytes{name="plugin-limit-count-redis-cluster-slot-lock"}
1036288
apisix_shared_dict_free_space_bytes{name="plugin-limit-count-reset-header"}
10412032
apisix_shared_dict_free_space_bytes{name="plugin-limit-req"} 10412032
apisix_shared_dict_free_space_bytes{name="prometheus-metrics"} 10387456
apisix_shared_dict_free_space_bytes{name="upstream-healthcheck"} 10412032
apisix_shared_dict_free_space_bytes{name="worker-events"} 10412032
(14/03/2024 - 16:17)[atael@atael-mac:~/tmp/vault-monitoring]
% curl http://localhost:9091/apisix/prometheus/metrics
# HELP apisix_etcd_modify_indexes Etcd modify index for APISIX keys
# TYPE apisix_etcd_modify_indexes gauge
apisix_etcd_modify_indexes{key="consumers"} 0
apisix_etcd_modify_indexes{key="global_rules"} 0
apisix_etcd_modify_indexes{key="max_modify_index"} 20
apisix_etcd_modify_indexes{key="prev_index"} 14
apisix_etcd_modify_indexes{key="protos"} 0
apisix_etcd_modify_indexes{key="routes"} 20
apisix_etcd_modify_indexes{key="services"} 0
apisix_etcd_modify_indexes{key="ssls"} 0
apisix_etcd_modify_indexes{key="stream_routes"} 0
apisix_etcd_modify_indexes{key="upstreams"} 0
apisix_etcd_modify_indexes{key="x_etcd_index"} 20
# HELP apisix_etcd_reachable Config server etcd reachable from APISIX, 0 is
unreachable
# TYPE apisix_etcd_reachable gauge
apisix_etcd_reachable 1
# HELP apisix_http_requests_total The total number of client requests since
APISIX started
# TYPE apisix_http_requests_total gauge
apisix_http_requests_total 1266
# HELP apisix_nginx_http_current_connections Number of HTTP connections
# TYPE apisix_nginx_http_current_connections gauge
apisix_nginx_http_current_connections{state="accepted"} 137
apisix_nginx_http_current_connections{state="active"} 20
apisix_nginx_http_current_connections{state="handled"} 137
apisix_nginx_http_current_connections{state="reading"} 0
apisix_nginx_http_current_connections{state="waiting"} 14
apisix_nginx_http_current_connections{state="writing"} 6
# HELP apisix_nginx_metric_errors_total Number of nginx-lua-prometheus errors
# TYPE apisix_nginx_metric_errors_total counter
apisix_nginx_metric_errors_total 0
# HELP apisix_node_info Info of APISIX node
# TYPE apisix_node_info gauge
apisix_node_info{hostname="apisix-97ffdb47c-ngb6l"} 1
# HELP apisix_shared_dict_capacity_bytes The capacity of each nginx shared
DICT since APISIX start
# TYPE apisix_shared_dict_capacity_bytes gauge
apisix_shared_dict_capacity_bytes{name="access-tokens"} 1048576
apisix_shared_dict_capacity_bytes{name="balancer-ewma"} 10485760
apisix_shared_dict_capacity_bytes{name="balancer-ewma-last-touched-at"}
10485760
apisix_shared_dict_capacity_bytes{name="balancer-ewma-locks"} 10485760
apisix_shared_dict_capacity_bytes{name="discovery"} 1048576
apisix_shared_dict_capacity_bytes{name="etcd-cluster-health-check"} 10485760
apisix_shared_dict_capacity_bytes{name="ext-plugin"} 1048576
apisix_shared_dict_capacity_bytes{name="internal-status"} 10485760
apisix_shared_dict_capacity_bytes{name="introspection"} 10485760
apisix_shared_dict_capacity_bytes{name="jwks"} 1048576
apisix_shared_dict_capacity_bytes{name="kubernetes"} 1048576
apisix_shared_dict_capacity_bytes{name="lrucache-lock"} 10485760
apisix_shared_dict_capacity_bytes{name="plugin-api-breaker"} 10485760
apisix_shared_dict_capacity_bytes{name="plugin-limit-conn"} 10485760
apisix_shared_dict_capacity_bytes{name="plugin-limit-count"} 10485760
apisix_shared_dict_capacity_bytes{name="plugin-limit-count-redis-cluster-slot-lock"}
1048576
apisix_shared_dict_capacity_bytes{name="plugin-limit-count-reset-header"}
10485760
apisix_shared_dict_capacity_bytes{name="plugin-limit-req"} 10485760
apisix_shared_dict_capacity_bytes{name="prometheus-metrics"} 10485760
apisix_shared_dict_capacity_bytes{name="upstream-healthcheck"} 10485760
apisix_shared_dict_capacity_bytes{name="worker-events"} 10485760
# HELP apisix_shared_dict_free_space_bytes The free space of each nginx
shared DICT since APISIX start
# TYPE apisix_shared_dict_free_space_bytes gauge
apisix_shared_dict_free_space_bytes{name="access-tokens"} 1032192
apisix_shared_dict_free_space_bytes{name="balancer-ewma"} 10412032
apisix_shared_dict_free_space_bytes{name="balancer-ewma-last-touched-at"}
10412032
apisix_shared_dict_free_space_bytes{name="balancer-ewma-locks"} 10412032
apisix_shared_dict_free_space_bytes{name="discovery"} 1032192
apisix_shared_dict_free_space_bytes{name="etcd-cluster-health-check"}
10412032
apisix_shared_dict_free_space_bytes{name="ext-plugin"} 1032192
apisix_shared_dict_free_space_bytes{name="internal-status"} 10407936
apisix_shared_dict_free_space_bytes{name="introspection"} 10412032
apisix_shared_dict_free_space_bytes{name="jwks"} 1032192
apisix_shared_dict_free_space_bytes{name="kubernetes"} 999424
apisix_shared_dict_free_space_bytes{name="lrucache-lock"} 10412032
apisix_shared_dict_free_space_bytes{name="plugin-api-breaker"} 10412032
apisix_shared_dict_free_space_bytes{name="plugin-limit-conn"} 10412032
apisix_shared_dict_free_space_bytes{name="plugin-limit-count"} 10412032
apisix_shared_dict_free_space_bytes{name="plugin-limit-count-redis-cluster-slot-lock"}
1036288
apisix_shared_dict_free_space_bytes{name="plugin-limit-count-reset-header"}
10412032
apisix_shared_dict_free_space_bytes{name="plugin-limit-req"} 10412032
apisix_shared_dict_free_space_bytes{name="prometheus-metrics"} 10387456
apisix_shared_dict_free_space_bytes{name="upstream-healthcheck"} 10412032
apisix_shared_dict_free_space_bytes{name="worker-events"} 10412032
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
