This is an automated email from the ASF dual-hosted git repository.
spacewander pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/apisix.git
The following commit(s) were added to refs/heads/master by this push:
new 0802c95 feat: add batch process metrics (#3070)
0802c95 is described below
commit 0802c95faf4c5615c3b9fbe56ef913be7af08287
Author: guyang <[email protected]>
AuthorDate: Thu Dec 24 19:24:46 2020 +0800
feat: add batch process metrics (#3070)
---
apisix/plugins/http-logger.lua | 2 +
apisix/plugins/prometheus/exporter.lua | 6 +-
apisix/plugins/sls-logger.lua | 4 +-
apisix/plugins/syslog.lua | 2 +
apisix/plugins/tcp-logger.lua | 4 +-
apisix/plugins/udp-logger.lua | 4 +-
apisix/plugins/zipkin.lua | 1 +
apisix/plugins/zipkin/reporter.lua | 7 +-
apisix/utils/batch-processor.lua | 23 +-
doc/plugins/prometheus.md | 9 +
doc/zh-cn/plugins/prometheus.md | 10 +-
t/plugin/prometheus.t | 389 +++++++++++++++++++++++++++++++++
12 files changed, 452 insertions(+), 9 deletions(-)
diff --git a/apisix/plugins/http-logger.lua b/apisix/plugins/http-logger.lua
index 9dfab62..90d3a69 100644
--- a/apisix/plugins/http-logger.lua
+++ b/apisix/plugins/http-logger.lua
@@ -278,6 +278,8 @@ function _M.log(conf, ctx)
max_retry_count = conf.max_retry_count,
buffer_duration = conf.buffer_duration,
inactive_timeout = conf.inactive_timeout,
+ route_id = ctx.var.route_id,
+ server_addr = ctx.var.server_addr,
}
local err
diff --git a/apisix/plugins/prometheus/exporter.lua
b/apisix/plugins/prometheus/exporter.lua
index 1bfb039..13b17f4 100644
--- a/apisix/plugins/prometheus/exporter.lua
+++ b/apisix/plugins/prometheus/exporter.lua
@@ -43,11 +43,10 @@ local DEFAULT_BUCKETS = { 1, 2, 5, 7, 10, 15, 20, 25, 30,
40, 50, 60, 70,
local metrics = {}
+local inner_tab_arr = {}
- local inner_tab_arr = {}
local function gen_arr(...)
clear_tab(inner_tab_arr)
-
for i = 1, select('#', ...) do
inner_tab_arr[i] = select(i, ...)
end
@@ -328,5 +327,8 @@ function _M.metric_data()
return prometheus:metric_data()
end
+function _M.get_prometheus()
+ return prometheus
+end
return _M
diff --git a/apisix/plugins/sls-logger.lua b/apisix/plugins/sls-logger.lua
index 945548c..bfc17d6 100644
--- a/apisix/plugins/sls-logger.lua
+++ b/apisix/plugins/sls-logger.lua
@@ -188,7 +188,9 @@ function _M.log(conf, ctx)
batch_max_size = conf.batch_max_size,
max_retry_count = conf.max_retry_count,
buffer_duration = conf.buffer_duration,
- inactive_timeout = conf.inactive_timeout
+ inactive_timeout = conf.inactive_timeout,
+ route_id = ctx.var.route_id,
+ server_addr = ctx.var.server_addr,
}
log_buffer, err = batch_processor:new(handle_log, process_conf)
diff --git a/apisix/plugins/syslog.lua b/apisix/plugins/syslog.lua
index 904027e..a3ec822 100644
--- a/apisix/plugins/syslog.lua
+++ b/apisix/plugins/syslog.lua
@@ -174,6 +174,8 @@ function _M.log(conf, ctx)
max_retry_count = conf.max_retry_times,
buffer_duration = conf.buffer_duration,
inactive_timeout = conf.timeout,
+ route_id = ctx.var.route_id,
+ server_addr = ctx.var.server_addr,
}
local err
diff --git a/apisix/plugins/tcp-logger.lua b/apisix/plugins/tcp-logger.lua
index 78a20d0..66e8e2f 100644
--- a/apisix/plugins/tcp-logger.lua
+++ b/apisix/plugins/tcp-logger.lua
@@ -118,7 +118,7 @@ local function remove_stale_objects(premature)
end
-function _M.log(conf)
+function _M.log(conf, ctx)
local entry = log_util.get_full_log(ngx, conf)
if not stale_timer_running then
@@ -156,6 +156,8 @@ function _M.log(conf)
max_retry_count = conf.max_retry_count,
buffer_duration = conf.buffer_duration,
inactive_timeout = conf.inactive_timeout,
+ route_id = ctx.var.route_id,
+ server_addr = ctx.var.server_addr,
}
local err
diff --git a/apisix/plugins/udp-logger.lua b/apisix/plugins/udp-logger.lua
index ea81b5c..85a008b 100644
--- a/apisix/plugins/udp-logger.lua
+++ b/apisix/plugins/udp-logger.lua
@@ -102,7 +102,7 @@ local function remove_stale_objects(premature)
end
-function _M.log(conf)
+function _M.log(conf, ctx)
local entry = log_util.get_full_log(ngx, conf)
if not stale_timer_running then
@@ -140,6 +140,8 @@ function _M.log(conf)
max_retry_count = conf.max_retry_count,
buffer_duration = conf.buffer_duration,
inactive_timeout = conf.inactive_timeout,
+ route_id = ctx.var.route_id,
+ server_addr = ctx.var.server_addr,
}
local err
diff --git a/apisix/plugins/zipkin.lua b/apisix/plugins/zipkin.lua
index 0255193..01234ba 100644
--- a/apisix/plugins/zipkin.lua
+++ b/apisix/plugins/zipkin.lua
@@ -82,6 +82,7 @@ local function create_tracer(conf,ctx)
conf.sample_ratio = 1
end
+ conf.route_id = ctx.route_id
local reporter = new_reporter(conf)
reporter:init_processor()
local tracer = new_tracer(reporter, new_random_sampler(conf))
diff --git a/apisix/plugins/zipkin/reporter.lua
b/apisix/plugins/zipkin/reporter.lua
index 202c626..16234c7 100644
--- a/apisix/plugins/zipkin/reporter.lua
+++ b/apisix/plugins/zipkin/reporter.lua
@@ -49,7 +49,8 @@ function _M.new(conf)
service_name = service_name,
server_addr = server_addr,
server_port = server_port,
- pending_spans_n = 0
+ pending_spans_n = 0,
+ route_id = conf.route_id
}, mt)
end
@@ -144,7 +145,9 @@ function _M.init_processor(self)
batch_max_size = 1000,
max_retry_count = 0,
buffer_duration = 60,
- inactive_timeout = 5
+ inactive_timeout = 5,
+ route_id = self.route_id,
+ server_addr = self.server_addr,
}
local flush = function (entries, batch_max_size)
diff --git a/apisix/utils/batch-processor.lua b/apisix/utils/batch-processor.lua
index 2839a85..c779439 100644
--- a/apisix/utils/batch-processor.lua
+++ b/apisix/utils/batch-processor.lua
@@ -27,6 +27,8 @@ local batch_processor_mt = {
}
local execute_func
local create_buffer_timer
+local batch_metrics
+local prometheus = require("apisix.plugins.prometheus.exporter")
local schema = {
@@ -131,7 +133,9 @@ function batch_processor:new(func, config)
entry_buffer = { entries = {}, retry_count = 0},
is_timer_running = false,
first_entry_t = 0,
- last_entry_t = 0
+ last_entry_t = 0,
+ route_id = config.route_id,
+ server_addr = config.server_addr,
}
return setmetatable(processor, batch_processor_mt)
@@ -146,8 +150,20 @@ function batch_processor:push(entry)
return
end
+ if not batch_metrics and prometheus.get_prometheus() and self.name
+ and self.route_id and self.server_addr then
+ batch_metrics =
prometheus.get_prometheus():gauge("batch_process_entries",
+ "batch process
remaining entries",
+ {"name", "route_id",
"server_addr"})
+ end
+
local entries = self.entry_buffer.entries
table.insert(entries, entry)
+ -- add batch metric for every route
+ if batch_metrics then
+ self.label = {self.name, self.route_id, self.server_addr}
+ batch_metrics:set(#entries, self.label)
+ end
if #entries == 1 then
self.first_entry_t = now()
@@ -173,11 +189,16 @@ function batch_processor:process_buffer()
"buffercount[", #self.entry_buffer.entries ,"]")
self.batch_to_process[#self.batch_to_process + 1] = self.entry_buffer
self.entry_buffer = { entries = {}, retry_count = 0 }
+ if batch_metrics then
+ self.label = {self.name, self.route_id, self.server_addr}
+ batch_metrics:set(0, self.label)
+ end
end
for _, batch in ipairs(self.batch_to_process) do
schedule_func_exec(self, 0, batch)
end
+
self.batch_to_process = {}
end
diff --git a/doc/plugins/prometheus.md b/doc/plugins/prometheus.md
index 68e900c..85d0fb7 100644
--- a/doc/plugins/prometheus.md
+++ b/doc/plugins/prometheus.md
@@ -125,6 +125,7 @@ Or you can goto [Grafana
official](https://grafana.com/grafana/dashboards/11719)
* `Bandwidth`: Total Bandwidth (egress/ingress) flowing through apisix. This
metric is available per service and as a sum across all services.
* `etcd reachability`: A gauge type with a value of 0 or 1, representing if
etcd can be reached by a apisix or not.
* `Connections`: Various Nginx connection metrics like active, reading,
writing, and number of accepted connections.
+* `Batch process entries`: A gauge type, when we use plugins and the plugin
used batch process to send data, such as: sys logger, http logger, sls logger,
tcp logger, udp logger and zipkin, then the entries which hasn't been sent in
batch process will be counted in the metrics.
Here is the original metric data of apisix:
@@ -138,6 +139,14 @@ apisix_bandwidth{type="egress",service="foo.com"} 2379
apisix_bandwidth{type="ingress",service="127.0.0.2"} 83
apisix_bandwidth{type="ingress",service="bar.com"} 76
apisix_bandwidth{type="ingress",service="foo.com"} 988
+# HELP apisix_batch_process_entries batch process remaining entries
+# TYPE apisix_batch_process_entries gauge
+apisix_batch_process_entries{name="http-logger",route_id="9",server_addr="127.0.0.1"}
1
+apisix_batch_process_entries{name="sls-logger",route_id="9",server_addr="127.0.0.1"}
1
+apisix_batch_process_entries{name="tcp-logger",route_id="9",server_addr="127.0.0.1"}
1
+apisix_batch_process_entries{name="udp-logger",route_id="9",server_addr="127.0.0.1"}
1
+apisix_batch_process_entries{name="sys-logger",route_id="9",server_addr="127.0.0.1"}
1
+apisix_batch_process_entries{name="zipkin_report",route_id="9",server_addr="127.0.0.1"}
1
# HELP apisix_etcd_reachable Config server etcd reachable from Apisix, 0 is
unreachable
# TYPE apisix_etcd_reachable gauge
apisix_etcd_reachable 1
diff --git a/doc/zh-cn/plugins/prometheus.md b/doc/zh-cn/plugins/prometheus.md
index a1c7c90..457eb62 100644
--- a/doc/zh-cn/plugins/prometheus.md
+++ b/doc/zh-cn/plugins/prometheus.md
@@ -123,7 +123,7 @@ plugin_attr:
* `Bandwidth`: 流经apisix的总带宽(可分出口带宽和入口带宽). 每个服务指标或者是所有服务指标的总和都可以统计到。
* `etcd reachability`: apisix 连接 etcd 的可用性，用 0 和 1来表示。
* `Connections`: 各种的 Nginx 连接指标，如 active（正处理的活动连接数），reading（nginx 读取到客户端的
Header 信息数），writing（nginx 返回给客户端的 Header 信息数），已建立的连接数。.
-
+* `Batch process entries`: 批处理未发送数据计数器，当你使用了批处理发送插件，比如：sys logger, http
logger, sls logger, tcp logger, udp logger and zipkin,
那么你将会在此指标中看到批处理当前尚未发送的数据的数量。
这里是apisix的原始的指标数据集:
```
@@ -136,6 +136,14 @@ apisix_bandwidth{type="egress",service="foo.com"} 2379
apisix_bandwidth{type="ingress",service="127.0.0.2"} 83
apisix_bandwidth{type="ingress",service="bar.com"} 76
apisix_bandwidth{type="ingress",service="foo.com"} 988
+# HELP apisix_batch_process_entries batch process remaining entries
+# TYPE apisix_batch_process_entries gauge
+apisix_batch_process_entries{name="http-logger",route_id="9",server_addr="127.0.0.1"}
1
+apisix_batch_process_entries{name="sls-logger",route_id="9",server_addr="127.0.0.1"}
1
+apisix_batch_process_entries{name="tcp-logger",route_id="9",server_addr="127.0.0.1"}
1
+apisix_batch_process_entries{name="udp-logger",route_id="9",server_addr="127.0.0.1"}
1
+apisix_batch_process_entries{name="sys-logger",route_id="9",server_addr="127.0.0.1"}
1
+apisix_batch_process_entries{name="zipkin_report",route_id="9",server_addr="127.0.0.1"}
1
# HELP apisix_etcd_reachable Config server etcd reachable from Apisix, 0 is
unreachable
# TYPE apisix_etcd_reachable gauge
apisix_etcd_reachable 1
diff --git a/t/plugin/prometheus.t b/t/plugin/prometheus.t
index bcf57cf..5ef4527 100644
--- a/t/plugin/prometheus.t
+++ b/t/plugin/prometheus.t
@@ -1088,3 +1088,392 @@ GET /apisix/prometheus/metrics
--- error_code: 200
--- no_error_log
[error]
+
+
+
+=== TEST 62: set sys plugins
+--- config
+ location /t {
+ content_by_lua_block {
+ local t = require("lib.test_admin").test
+ local code, body = t('/apisix/admin/routes/9',
+ ngx.HTTP_PUT,
+ [[{
+ "methods": ["GET"],
+ "plugins": {
+ "prometheus": {},
+ "syslog": {
+ "host": "127.0.0.1",
+ "include_req_body": false,
+ "max_retry_times": 1,
+ "tls": false,
+ "retry_interval": 1,
+ "batch_max_size": 1000,
+ "buffer_duration": 60,
+ "port": 1000,
+ "name": "sys-logger",
+ "flush_limit": 4096,
+ "sock_type": "tcp",
+ "timeout": 3,
+ "drop_limit": 1048576,
+ "pool_size": 5
+ }
+ },
+ "upstream": {
+ "nodes": {
+ "127.0.0.1:1980": 1
+ },
+ "type": "roundrobin"
+ },
+ "uri": "/batch-process-metrics"
+ }]]
+ )
+
+ if code >= 300 then
+ ngx.status = code
+ end
+ ngx.say(body)
+ }
+ }
+--- request
+GET /t
+--- response_body
+passed
+--- no_error_log
+[error]
+
+
+
+=== TEST 63: hit batch-process-metrics
+--- request
+GET /batch-process-metrics
+--- error_code: 404
+
+
+
+=== TEST 64: check sys logger metrics
+--- request
+GET /apisix/prometheus/metrics
+--- error_code: 200
+--- response_body_like eval
+qr/apisix_batch_process_entries\{name="sys-logger",route_id="9",server_addr="127.0.0.1"\}
\d+/
+
+
+
+=== TEST 65: set zipkin plugins
+--- config
+ location /t {
+ content_by_lua_block {
+ local t = require("lib.test_admin").test
+ local code, body = t('/apisix/admin/routes/9',
+ ngx.HTTP_PUT,
+ [[{
+ "methods": ["GET"],
+ "plugins": {
+ "prometheus": {},
+ "zipkin": {
+ "endpoint": "http://127.0.0.1:9447";,
+ "service_name": "APISIX",
+ "sample_ratio": 1
+ }
+ },
+ "upstream": {
+ "nodes": {
+ "127.0.0.1:1980": 1
+ },
+ "type": "roundrobin"
+ },
+ "uri": "/batch-process-metrics"
+ }]]
+ )
+
+ if code >= 300 then
+ ngx.status = code
+ end
+ ngx.say(body)
+ }
+ }
+--- request
+GET /t
+--- response_body
+passed
+--- no_error_log
+[error]
+
+
+
+=== TEST 66: hit batch-process-metrics
+--- request
+GET /batch-process-metrics
+--- error_code: 404
+
+
+
+=== TEST 67: check zipkin log metrics
+--- request
+GET /apisix/prometheus/metrics
+--- error_code: 200
+--- response_body_like eval
+qr/apisix_batch_process_entries\{name="zipkin_report",route_id="9",server_addr="127.0.0.1"\}
\d+/
+
+
+
+=== TEST 68: set http plugins
+--- config
+ location /t {
+ content_by_lua_block {
+ local t = require("lib.test_admin").test
+ local code, body = t('/apisix/admin/routes/9',
+ ngx.HTTP_PUT,
+ [[{
+ "methods": ["GET"],
+ "plugins": {
+ "prometheus": {},
+ "http-logger": {
+ "inactive_timeout": 5,
+ "include_req_body": false,
+ "timeout": 3,
+ "name": "http-logger",
+ "retry_delay": 1,
+ "buffer_duration": 60,
+ "uri": "http://127.0.0.1:19080/report";,
+ "concat_method": "json",
+ "batch_max_size": 1000,
+ "max_retry_count": 0
+ }
+ },
+ "upstream": {
+ "nodes": {
+ "127.0.0.1:1980": 1
+ },
+ "type": "roundrobin"
+ },
+ "uri": "/batch-process-metrics"
+ }]]
+ )
+
+ if code >= 300 then
+ ngx.status = code
+ end
+ ngx.say(body)
+ }
+ }
+--- request
+GET /t
+--- response_body
+passed
+--- no_error_log
+[error]
+
+
+
+=== TEST 69: hit batch-process-metrics
+--- request
+GET /batch-process-metrics
+--- error_code: 404
+
+
+
+=== TEST 70: check http log metrics
+--- request
+GET /apisix/prometheus/metrics
+--- error_code: 200
+--- response_body_like eval
+qr/apisix_batch_process_entries\{name="http-logger",route_id="9",server_addr="127.0.0.1"\}
\d+/
+
+
+
+=== TEST 71: set tcp-logger plugins
+--- config
+ location /t {
+ content_by_lua_block {
+ local t = require("lib.test_admin").test
+ local code, body = t('/apisix/admin/routes/10',
+ ngx.HTTP_PUT,
+ [[{
+ "methods": ["GET"],
+ "plugins": {
+ "prometheus": {},
+ "tcp-logger": {
+ "host": "127.0.0.1",
+ "include_req_body": false,
+ "timeout": 1000,
+ "name": "tcp-logger",
+ "retry_delay": 1,
+ "buffer_duration": 60,
+ "port": 1000,
+ "batch_max_size": 1000,
+ "inactive_timeout": 5,
+ "tls": false,
+ "max_retry_count": 0
+ }
+ },
+ "upstream": {
+ "nodes": {
+ "127.0.0.1:1980": 1
+ },
+ "type": "roundrobin"
+ },
+ "uri": "/batch-process-metrics-10"
+ }]]
+ )
+
+ if code >= 300 then
+ ngx.status = code
+ end
+ ngx.say(body)
+ }
+ }
+--- request
+GET /t
+--- response_body
+passed
+--- no_error_log
+[error]
+
+
+
+=== TEST 72: tigger metircs batch-process-metrics
+--- request
+GET /batch-process-metrics-10
+--- error_code: 404
+
+
+
+=== TEST 73: check tcp log metrics
+--- request
+GET /apisix/prometheus/metrics
+--- error_code: 200
+--- response_body_like eval
+qr/apisix_batch_process_entries\{name="tcp-logger",route_id="10",server_addr="127.0.0.1"\}
\d+/
+
+
+
+=== TEST 74: set udp-logger plugins
+--- config
+ location /t {
+ content_by_lua_block {
+ local t = require("lib.test_admin").test
+ local code, body = t('/apisix/admin/routes/10',
+ ngx.HTTP_PUT,
+ [[{
+ "methods": ["GET"],
+ "plugins": {
+ "prometheus": {},
+ "udp-logger": {
+ "host": "127.0.0.1",
+ "port": 1000,
+ "include_req_body": false,
+ "timeout": 3,
+ "batch_max_size": 1000,
+ "name": "udp-logger",
+ "inactive_timeout": 5,
+ "buffer_duration": 60
+ }
+ },
+ "upstream": {
+ "nodes": {
+ "127.0.0.1:1980": 1
+ },
+ "type": "roundrobin"
+ },
+ "uri": "/batch-process-metrics-10"
+ }]]
+ )
+
+ if code >= 300 then
+ ngx.status = code
+ end
+ ngx.say(body)
+ }
+ }
+--- request
+GET /t
+--- response_body
+passed
+--- no_error_log
+[error]
+
+
+
+=== TEST 75: tigger metircs batch-process-metrics
+--- request
+GET /batch-process-metrics-10
+--- error_code: 404
+
+
+
+=== TEST 76: check udp log metrics
+--- request
+GET /apisix/prometheus/metrics
+--- error_code: 200
+--- response_body_like eval
+qr/apisix_batch_process_entries\{name="udp-logger",route_id="10",server_addr="127.0.0.1"\}
\d+/
+
+
+
+=== TEST 77: set sls-logger plugins
+--- config
+ location /t {
+ content_by_lua_block {
+ local t = require("lib.test_admin").test
+ local code, body = t('/apisix/admin/routes/10',
+ ngx.HTTP_PUT,
+ [[{
+ "methods": ["GET"],
+ "plugins": {
+ "prometheus": {},
+ "sls-logger": {
+ "host": "127.0.0.1",
+ "batch_max_size": 1000,
+ "name": "sls-logger",
+ "inactive_timeout": 5,
+ "logstore": "your_logstore",
+ "buffer_duration": 60,
+ "port": 10009,
+ "max_retry_count": 0,
+ "retry_delay": 1,
+ "access_key_id": "your_access_id",
+ "access_key_secret": "your_key_secret",
+ "timeout": 5000,
+ "project": "your_project"
+ }
+ },
+ "upstream": {
+ "nodes": {
+ "127.0.0.1:1980": 1
+ },
+ "type": "roundrobin"
+ },
+ "uri": "/batch-process-metrics-10"
+ }]]
+ )
+
+ if code >= 300 then
+ ngx.status = code
+ end
+ ngx.say(body)
+ }
+ }
+--- request
+GET /t
+--- response_body
+passed
+--- no_error_log
+[error]
+
+
+
+=== TEST 78: tigger metircs batch-process-metrics
+--- request
+GET /batch-process-metrics-10
+--- error_code: 404
+
+
+
+=== TEST 79: check sls-logger metrics
+--- request
+GET /apisix/prometheus/metrics
+--- error_code: 200
+--- response_body_like eval
+qr/apisix_batch_process_entries\{name="sls-logger",route_id="10",server_addr="127.0.0.1"\}
\d+/
