This is an automated email from the ASF dual-hosted git repository. membphis pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/apisix.git
The following commit(s) were added to refs/heads/master by this push: new 3f9685f feat(http-logger): support to concat multiple log with separator. (#2286) 3f9685f is described below commit 3f9685f823050f3c3324d20ffacafec4e3e70150 Author: YuanSheng Wang <membp...@gmail.com> AuthorDate: Thu Sep 24 10:58:44 2020 +0800 feat(http-logger): support to concat multiple log with separator. (#2286) --- apisix/plugins/http-logger.lua | 37 ++++--- doc/plugins/http-logger.md | 8 +- doc/zh-cn/plugins/http-logger.md | 3 +- t/lib/server.lua | 6 ++ t/plugin/http-logger-new-line.t | 223 +++++++++++++++++++++++++++++++++++++++ 5 files changed, 260 insertions(+), 17 deletions(-) diff --git a/apisix/plugins/http-logger.lua b/apisix/plugins/http-logger.lua index 44df6ae..4694b60 100644 --- a/apisix/plugins/http-logger.lua +++ b/apisix/plugins/http-logger.lua @@ -23,6 +23,7 @@ local tostring = tostring local http = require "resty.http" local url = require "net.url" local buffers = {} +local ipairs = ipairs local schema = { type = "object", @@ -36,7 +37,9 @@ local schema = { buffer_duration = {type = "integer", minimum = 1, default = 60}, inactive_timeout = {type = "integer", minimum = 1, default = 5}, batch_max_size = {type = "integer", minimum = 1, default = 1000}, - include_req_body = {type = "boolean", default = false} + include_req_body = {type = "boolean", default = false}, + concat_method = {type = "string", default = "json", + enum = {"json", "new_line"}} }, required = {"uri"} } @@ -110,13 +113,6 @@ local function send_http_data(conf, log_message) .. "body[" .. httpc_res:read_body() .. "]" end - -- keep the connection alive - ok, err = httpc:set_keepalive(conf.keepalive) - - if not ok then - core.log.debug("failed to keep the connection alive", err) - end - return res, err_msg end @@ -139,10 +135,26 @@ function _M.log(conf) -- Generate a function to be executed by the batch processor local func = function(entries, batch_max_size) local data, err - if batch_max_size == 1 then - data, err = core.json.encode(entries[1]) -- encode as single {} - else - data, err = core.json.encode(entries) -- encode as array [{}] + if conf.concat_method == "json" then + if batch_max_size == 1 then + data, err = core.json.encode(entries[1]) -- encode as single {} + else + data, err = core.json.encode(entries) -- encode as array [{}] + end + + elseif conf.concat_method == "new_line" then + if batch_max_size == 1 then + data, err = core.json.encode(entries[1]) -- encode as single {} + else + local t = core.table.new(#entries, 0) + for i, entrie in ipairs(entries) do + t[i], err = core.json.encode(entrie) + if err then + break + end + end + data = core.table.concat(t, "\n") -- encode as multiple string + end end if not data then @@ -173,4 +185,5 @@ function _M.log(conf) log_buffer:push(entry) end + return _M diff --git a/doc/plugins/http-logger.md b/doc/plugins/http-logger.md index f8c0941..3fd5b9e 100644 --- a/doc/plugins/http-logger.md +++ b/doc/plugins/http-logger.md @@ -20,6 +20,7 @@ - [中文](../zh-cn/plugins/http-logger.md) # Summary + - [**Name**](#name) - [**Attributes**](#attributes) - [**How To Enable**](#how-to-enable) @@ -35,7 +36,6 @@ This will provide the ability to send Log data requests as JSON objects to Monit ## Attributes - | Name | Type | Requirement | Default | Valid | Description | | ---------------- | ------- | ----------- | ------------- | ------- | ---------------------------------------------------------------------------------------- | | uri | string | required | | | URI of the server | @@ -48,6 +48,7 @@ This will provide the ability to send Log data requests as JSON objects to Monit | max_retry_count | integer | optional | 0 | [0,...] | Maximum number of retries before removing from the processing pipe line | | retry_delay | integer | optional | 1 | [0,...] | Number of seconds the process execution should be delayed if the execution fails | | include_req_body | boolean | optional | false | | Whether to include the request body | +| concat_method | string | optional | "json" | | Enum type, `json` and `new_line`. **json**: use `json.encode` for all pending logs. **new_line**: use `json.encode` for each pending log and concat them with "\n" line. | ## How To Enable @@ -58,7 +59,7 @@ curl http://127.0.0.1:9080/apisix/admin/routes/1 -H 'X-API-KEY: edd1c9f034335f13 { "plugins": { "http-logger": { - "uri": "127.0.0.1:80/postendpoint?param=1", + "uri": "127.0.0.1:80/postendpoint?param=1", } }, "upstream": { @@ -73,7 +74,7 @@ curl http://127.0.0.1:9080/apisix/admin/routes/1 -H 'X-API-KEY: edd1c9f034335f13 ## Test Plugin -* success: +> success: ```shell $ curl -i http://127.0.0.1:9080/hello @@ -90,7 +91,6 @@ APISIX plugins are hot-reloaded, therefore no need to restart APISIX. ```shell $ curl http://127.0.0.1:2379/apisix/admin/routes/1 -H 'X-API-KEY: edd1c9f034335f136f87ad84b625c8f1' -X PUT -d value=' { - "methods": ["GET"], "uri": "/hello", "plugins": {}, "upstream": { diff --git a/doc/zh-cn/plugins/http-logger.md b/doc/zh-cn/plugins/http-logger.md index 7ea7dc1..7791007 100644 --- a/doc/zh-cn/plugins/http-logger.md +++ b/doc/zh-cn/plugins/http-logger.md @@ -47,6 +47,7 @@ | max_retry_count | integer | 可选 | 0 | [0,...] | 从处理管道中移除之前的最大重试次数 | | retry_delay | integer | 可选 | 1 | [0,...] | 如果执行失败,则应延迟执行流程的秒数 | | include_req_body | boolean | 可选 | | | 是否包括请求 body | +| concat_method | string | 可选 | "json" | | 枚举类型,`json`、`new_line`。**json**: 对所有待发日志使用 `json.encode` 编码。**new_line**: 对每一条待发日志单独使用 `json.encode` 编码并使用 "\n" 连接起来。 | ## 如何开启 @@ -57,7 +58,7 @@ curl http://127.0.0.1:9080/apisix/admin/routes/1 -H 'X-API-KEY: edd1c9f034335f13 { "plugins": { "http-logger": { - "uri": "127.0.0.1:80/postendpoint?param=1" + "uri": "127.0.0.1:80/postendpoint?param=1" } }, "upstream": { diff --git a/t/lib/server.lua b/t/lib/server.lua index c9e89e9..3a0edae 100644 --- a/t/lib/server.lua +++ b/t/lib/server.lua @@ -282,4 +282,10 @@ function _M.headers() ngx.say("/headers") end +function _M.log() + ngx.req.read_body() + local body = ngx.req.get_body_data() + ngx.log(ngx.WARN, "request log: ", body or "nil") +end + return _M diff --git a/t/plugin/http-logger-new-line.t b/t/plugin/http-logger-new-line.t new file mode 100644 index 0000000..14c2daa --- /dev/null +++ b/t/plugin/http-logger-new-line.t @@ -0,0 +1,223 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +use t::APISIX 'no_plan'; + +log_level('info'); +repeat_each(1); +no_long_string(); +no_root_location(); +run_tests; + +__DATA__ + +=== TEST 1: sanity, batch_max_size=1 +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + local code, body = t('/apisix/admin/routes/1', + ngx.HTTP_PUT, + [[{ + "plugins": { + "http-logger": { + "uri": "http://127.0.0.1:1980/log", + "batch_max_size": 1, + "max_retry_count": 1, + "retry_delay": 2, + "buffer_duration": 2, + "inactive_timeout": 2, + "concat_method": "new_line" + } + }, + "upstream": { + "nodes": { + "127.0.0.1:1982": 1 + }, + "type": "roundrobin" + }, + "uri": "/hello" + }]] + ) + + if code >= 300 then + ngx.status = code + end + ngx.say(body) + } + } +--- request +GET /t +--- response_body +passed +--- no_error_log +[error] + + + +=== TEST 2: hit route and report http logger +--- request +GET /hello +--- response_body +hello world +--- wait: 0.5 +--- no_error_log +[error] +--- error_log +request log: {"upstream":"127.0.0.1:1982" + + + +=== TEST 3: sanity, batch_max_size=1 +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + local code, body = t('/apisix/admin/routes/1', + ngx.HTTP_PUT, + [[{ + "plugins": { + "http-logger": { + "uri": "http://127.0.0.1:1980/log", + "batch_max_size": 3, + "max_retry_count": 3, + "retry_delay": 2, + "buffer_duration": 2, + "inactive_timeout": 1, + "concat_method": "new_line" + } + }, + "upstream": { + "nodes": { + "127.0.0.1:1982": 1 + }, + "type": "roundrobin" + }, + "uri": "/hello" + }]] + ) + + if code >= 300 then + ngx.status = code + end + ngx.say(body) + } + } +--- request +GET /t +--- response_body +passed +--- no_error_log +[error] + + + +=== TEST 4: hit route, and no report log +--- request +GET /hello +--- response_body +hello world +--- no_error_log +[error] +request log: + + + +=== TEST 5: hit route, and report log +--- config +location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + + for i = 1, 6 do + t('/hello', ngx.HTTP_GET) + end + + ngx.sleep(3) + ngx.say("done") + } +} +--- request +GET /t +--- timeout: 10 +--- no_error_log +[error] +--- grep_error_log eval +qr/request log:/ +--- grep_error_log_out +request log: +request log: + + + +=== TEST 6: hit route, and report log +--- config +location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + + for i = 1, 6 do + t('/hello', ngx.HTTP_GET) + end + + ngx.sleep(3) + ngx.say("done") + } +} +--- request +GET /t +--- timeout: 10 +--- no_error_log +[error] +--- grep_error_log eval +qr/"upstream":"127.0.0.1:1982"/ +--- grep_error_log_out +"upstream":"127.0.0.1:1982" +"upstream":"127.0.0.1:1982" +"upstream":"127.0.0.1:1982" +"upstream":"127.0.0.1:1982" +"upstream":"127.0.0.1:1982" +"upstream":"127.0.0.1:1982" + + + +=== TEST 7: hit route, and report log +--- config +location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + + for i = 1, 5 do + t('/hello', ngx.HTTP_GET) + end + + ngx.sleep(3) + ngx.say("done") + } +} +--- request +GET /t +--- timeout: 10 +--- no_error_log +[error] +--- grep_error_log eval +qr/"upstream":"127.0.0.1:1982"/ +--- grep_error_log_out +"upstream":"127.0.0.1:1982" +"upstream":"127.0.0.1:1982" +"upstream":"127.0.0.1:1982" +"upstream":"127.0.0.1:1982" +"upstream":"127.0.0.1:1982"