This is an automated email from the ASF dual-hosted git repository.
spacewander pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/apisix.git
The following commit(s) were added to refs/heads/master by this push:
new f0a835573 feat(clickhouse-logger): support multi clickhouse endpoints
(#7517)
f0a835573 is described below
commit f0a8355739db474dae5a150c7d195e0a4a5dbfa9
Author: Zhendong Qi <[email protected]>
AuthorDate: Thu Jul 28 13:43:38 2022 +0800
feat(clickhouse-logger): support multi clickhouse endpoints (#7517)
---
apisix/plugins/clickhouse-logger.lua | 18 +++++++--
docs/en/latest/plugins/clickhouse-logger.md | 6 ++-
docs/zh/latest/plugins/clickhouse-logger.md | 24 ++++++------
t/plugin/clickhouse-logger.t | 58 ++++++++++++++++++++++++++++-
4 files changed, 88 insertions(+), 18 deletions(-)
diff --git a/apisix/plugins/clickhouse-logger.lua
b/apisix/plugins/clickhouse-logger.lua
index f7b734645..026f0cfa9 100644
--- a/apisix/plugins/clickhouse-logger.lua
+++ b/apisix/plugins/clickhouse-logger.lua
@@ -21,6 +21,7 @@ local core = require("apisix.core")
local http = require("resty.http")
local url = require("net.url")
local plugin = require("apisix.plugin")
+local math_random = math.random
local ngx = ngx
local tostring = tostring
@@ -31,7 +32,9 @@ local batch_processor_manager =
bp_manager_mod.new(plugin_name)
local schema = {
type = "object",
properties = {
+ -- deprecated, use "endpoint_addrs" instead
endpoint_addr = core.schema.uri_def,
+ endpoint_addrs = {items = core.schema.uri_def, type = "array",
minItems = 1},
user = {type = "string", default = ""},
password = {type = "string", default = ""},
database = {type = "string", default = ""},
@@ -40,7 +43,10 @@ local schema = {
name = {type = "string", default = "clickhouse logger"},
ssl_verify = {type = "boolean", default = true},
},
- required = {"endpoint_addr", "user", "password", "database", "logtable"}
+ oneOf = {
+ {required = {"endpoint_addr", "user", "password", "database",
"logtable"}},
+ {required = {"endpoint_addrs", "user", "password", "database",
"logtable"}}
+ },
}
@@ -72,11 +78,17 @@ end
local function send_http_data(conf, log_message)
local err_msg
local res = true
- local url_decoded = url.parse(conf.endpoint_addr)
+ local selected_endpoint_addr
+ if conf.endpoint_addr then
+ selected_endpoint_addr = conf.endpoint_addr
+ else
+ selected_endpoint_addr =
conf.endpoint_addrs[math_random(#conf.endpoint_addrs)]
+ end
+ local url_decoded = url.parse(selected_endpoint_addr)
local host = url_decoded.host
local port = url_decoded.port
- core.log.info("sending a batch logs to ", conf.endpoint_addr)
+ core.log.info("sending a batch logs to ", selected_endpoint_addr)
if not port then
if url_decoded.scheme == "https" then
diff --git a/docs/en/latest/plugins/clickhouse-logger.md
b/docs/en/latest/plugins/clickhouse-logger.md
index 5eb8f77f9..f0808b19b 100644
--- a/docs/en/latest/plugins/clickhouse-logger.md
+++ b/docs/en/latest/plugins/clickhouse-logger.md
@@ -35,7 +35,8 @@ The `clickhouse-logger` Plugin is used to push logs to
[ClickHouse](https://clic
| Name | Type | Required | Default | Valid values |
Description |
|---------------|---------|----------|---------------------|--------------|----------------------------------------------------------------|
-| endpoint_addr | string | True | | |
ClickHouse endpoint. |
+| endpoint_addr | Deprecated | True | | |
Use `endpoint_addrs` instead. ClickHouse endpoints. |
+| endpoint_addrs | array | True | | |
ClickHouse endpoints. |
| database | string | True | | |
Name of the database to store the logs. |
| logtable | string | True | | |
Table name to store the logs. |
| user | string | True | | |
ClickHouse username. |
@@ -96,6 +97,7 @@ Now, if you run `select * from default.test;`, you will get
the following row:
## Enabling the Plugin
+If multiple endpoints are configured, they will be written randomly.
The example below shows how you can enable the Plugin on a specific Route:
```shell
@@ -107,7 +109,7 @@ curl http://127.0.0.1:9080/apisix/admin/routes/1 -H
'X-API-KEY: edd1c9f034335f13
"password": "a",
"database": "default",
"logtable": "test",
- "endpoint_addr": "http://127.0.0.1:8123";
+ "endpoint_addrs": ["http://127.0.0.1:8123";]
}
},
"upstream": {
diff --git a/docs/zh/latest/plugins/clickhouse-logger.md
b/docs/zh/latest/plugins/clickhouse-logger.md
index c704893b1..c967b3d6e 100644
--- a/docs/zh/latest/plugins/clickhouse-logger.md
+++ b/docs/zh/latest/plugins/clickhouse-logger.md
@@ -27,21 +27,23 @@ title: clickhouse-logger
## 属性
-| 名称 | 类型 | 必选项 | 默认值 | 有效值 | 描述
|
-| ---------------- | ------- | ------ | ------------- | ------- |
------------------------------------------------ |
-| endpoint_addr | string | 必须 | | | `clickhouse`
服务器的 endpoint。 |
-| database | string | 必须 | | | 使用的数据库。
|
-| logtable | string | 必须 | | | 写入的表名 。 |
-| user | string | 必须 | | | clickhouse
的用户。 |
-| password | string | 必须 | | | clickhouse 的密码
。 |
-| timeout | integer | 可选 | 3 | [1,...] |
发送请求后保持连接活动的时间。 |
-| name | string | 可选 | "clickhouse logger" | | 标识
logger 的唯一标识符。 |
-| ssl_verify | boolean | 可选 | true | [true,false] | 验证证书。
|
+| 名称 | 类型 | 必选项 | 默认值 | 有效值 | 描述
|
+| ---------------- | ------- | ------ | ------------- | ------- |
------------------------------------------------ |
+| endpoint_addr | 废弃 | 必须 | | | 推荐使用
`endpoint_addrs` 代替。`clickhouse` 服务器的 endpoints。|
+| endpoint_addrs | array | 必须 | | | `clickhouse`
服务器的 endpoints。|
+| database | string | 必须 | | | 使用的数据库。
|
+| logtable | string | 必须 | | | 写入的表名 。
|
+| user | string | 必须 | | | clickhouse
的用户。 |
+| password | string | 必须 | | | clickhouse 的密码
。 |
+| timeout | integer | 可选 | 3 | [1,...] |
发送请求后保持连接活动的时间。 |
+| name | string | 可选 | "clickhouse logger" | | 标识
logger 的唯一标识符。 |
+| ssl_verify | boolean | 可选 | true | [true,false] | 验证证书。
|
本插件支持使用批处理器来聚合并批量处理条目（日志/数据）。这样可以避免插件频繁地提交数据，默认设置情况下批处理器会每 `5` 秒钟或队列中的数据达到
`1000` 条时提交数据，如需了解或自定义批处理器相关参数设置，请参考
[Batch-Processor](../batch-processor.md#配置) 配置部分。
## 如何开启
+如果配置多个 endpoints 将随机写入。
这是有关如何为特定路由启用 `clickhouse-logger` 插件的示例。
```shell
@@ -53,7 +55,7 @@ curl http://127.0.0.1:9080/apisix/admin/routes/1 -H
'X-API-KEY: edd1c9f034335f13
"password": "a",
"database": "default",
"logtable": "test",
- "endpoint_addr": "http://127.0.0.1:8123";
+ "endpoint_addrs": ["http://127.0.0.1:8123";]
}
},
"upstream": {
diff --git a/t/plugin/clickhouse-logger.t b/t/plugin/clickhouse-logger.t
index 5426ce028..ccb0be11a 100644
--- a/t/plugin/clickhouse-logger.t
+++ b/t/plugin/clickhouse-logger.t
@@ -48,6 +48,18 @@ add_block_preprocessor(sub {
ngx.say("ok")
}
}
+ location /clickhouse-logger/test1 {
+ content_by_lua_block {
+ ngx.req.read_body()
+ local data = ngx.req.get_body_data()
+ local headers = ngx.req.get_headers()
+ ngx.log(ngx.WARN, "clickhouse body: ", data)
+ for k, v in pairs(headers) do
+ ngx.log(ngx.WARN, "clickhouse headers: " .. k .. ":" .. v)
+ end
+ ngx.say("ok")
+ }
+ }
}
_EOC_
@@ -131,7 +143,7 @@ passed
}
}
--- response_body
-property "endpoint_addr" is required
+value should match only one schema, but matches none
@@ -175,7 +187,49 @@ passed
-=== TEST 5: access local server
+=== TEST 5: add plugin on routes using multi clickhouse-logger
+--- config
+ location /t {
+ content_by_lua_block {
+ local t = require("lib.test_admin").test
+ local code, body = t('/apisix/admin/routes/1',
+ ngx.HTTP_PUT,
+ [[{
+ "plugins": {
+ "clickhouse-logger": {
+ "user": "default",
+ "password": "a",
+ "database": "default",
+ "logtable": "t",
+ "endpoint_addrs":
["http://127.0.0.1:10420/clickhouse-logger/test";,
+
"http://127.0.0.1:10420/clickhouse-logger/test1";],
+ "batch_max_size":1,
+ "inactive_timeout":1
+ }
+ },
+ "upstream": {
+ "nodes": {
+ "127.0.0.1:1982": 1
+ },
+ "type": "roundrobin"
+ },
+ "uri": "/opentracing"
+ }]]
+ )
+
+ if code >= 300 then
+ ngx.status = code
+ end
+ ngx.say(body)
+ }
+ }
+--- error_code: 200
+--- response_body
+passed
+
+
+
+=== TEST 6: access local server
--- request
GET /opentracing
--- response_body
