gy09535 commented on a change in pull request #2177: URL: https://github.com/apache/apisix/pull/2177#discussion_r520256181
########## File path: apisix/plugins/sls-logger.lua ########## @@ -0,0 +1,206 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one or more +-- contributor license agreements. See the NOTICE file distributed with +-- this work for additional information regarding copyright ownership. +-- The ASF licenses this file to You under the Apache License, Version 2.0 +-- (the "License"); you may not use this file except in compliance with +-- the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- +local core = require("apisix.core") +local log_util = require("apisix.utils.log-util") +local batch_processor = require("apisix.utils.batch-processor") +local plugin_name = "sls-logger" +local ngx = ngx +local rf5424 = require("apisix.plugins.slslog.rfc5424") +local stale_timer_running = false; +local timer_at = ngx.timer.at +local tcp = ngx.socket.tcp +local buffers = {} +local tostring = tostring +local ipairs = ipairs +local schema = { + type = "object", + properties = { + include_req_body = {type = "boolean", default = false}, + name = {type = "string", default = "sls-logger"}, + timeout = {type = "integer", minimum = 1, default= 5000}, + max_retry_count = {type = "integer", minimum = 0, default = 0}, + retry_delay = {type = "integer", minimum = 0, default = 1}, + buffer_duration = {type = "integer", minimum = 1, default = 60}, + inactive_timeout = {type = "integer", minimum = 1, default = 5}, + batch_max_size = {type = "integer", minimum = 1, default = 1000}, + host = {type = "string"}, + port = {type = "integer"}, + project = {type = "string"}, + logstore = {type = "string"}, + access_key_id = {type = "string"}, + access_key_secret = {type ="string"} + }, + required = {"host", "port", "project", "logstore", "access_key_id", "access_key_secret"} +} + +local _M = { + version = 0.1, + priority = 406, + name = plugin_name, + schema = schema, +} + +function _M.check_schema(conf) + return core.schema.check(schema, conf) +end + +local function send_tcp_data(route_conf, log_message) + local err_msg + local res = true + local sock, soc_err = tcp() + local can_close + + if not sock then + return false, "failed to init the socket" .. soc_err + end + + sock:settimeout(route_conf.timeout) + local ok, err = sock:connect(route_conf.host, route_conf.port) + if not ok then + return false, "failed to connect to TCP server: host[" .. route_conf.host + .. "] port[" .. tostring(route_conf.port) .. "] err: " .. err + end + + ok, err = sock:sslhandshake(true, nil, false) + if not ok then + return false, "failed to to perform TLS handshake to TCP server: host[" + .. route_conf.host .. "] port[" .. tostring(route_conf.port) + .. "] err: " .. err + end + + core.log.debug("sls logger send data ", log_message) + ok, err = sock:send(log_message) + if not ok then + res = false + can_close = true + err_msg = "failed to send data to TCP server: host[" .. route_conf.host + .. "] port[" .. tostring(route_conf.port) .. "] err: " .. err + else + ok, err = sock:setkeepalive(120 * 1000, 20) + if not ok then + can_close = true + core.log.warn("failed to set socket keepalive: host[" .. route_conf.host + .. "] port[" .. tostring(route_conf.port) .. "] err: " .. err) + end + end + + if can_close then + ok, err = sock:close() + if not ok then + core.log.warn("failed to close the TCP connection, host[", + route_conf.host, "] port[", route_conf.port, "] ", err) + end + end + + return res, err_msg +end + +-- remove stale objects from the memory after timer expires +local function remove_stale_objects(premature) + if premature then + return + end + + for key, batch in ipairs(buffers) do + if #batch.entry_buffer.entries == 0 and #batch.batch_to_process == 0 then + core.log.warn("removing batch processor stale object, route id:", tostring(key)) + buffers[key] = nil + end + end + + stale_timer_running = false +end + +local function combine_syslog(entries) + local data + for _, entry in ipairs(entries) do + if not data then + data = entry.data + end + + data = data .. entry.data + core.log.info(entry.data) + end + + return data +end + +local function handle_log(entries) + local data = combine_syslog(entries) + if not data then + return true + end + + -- get the config from entries, replace of local value + return send_tcp_data(entries[1].route_conf, data) +end + +-- log phase in APISIX +function _M.log(conf, ctx) + local entry = log_util.get_full_log(ngx, conf) + if not entry.route_id then + core.log.error("failed to obtain the route id for sys logger") + return + end + + local json_str, err = core.json.encode(entry) + if not json_str then + core.log.error('error occurred while encoding the data: ', err) + return + end + + local rf5424_data = rf5424.encode("SYSLOG", "INFO", ctx.var.host,"apisix", + ngx.var.pid, conf.project, conf.logstore, Review comment: fix it. ########## File path: apisix/plugins/sls-logger.lua ########## @@ -0,0 +1,206 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one or more +-- contributor license agreements. See the NOTICE file distributed with +-- this work for additional information regarding copyright ownership. +-- The ASF licenses this file to You under the Apache License, Version 2.0 +-- (the "License"); you may not use this file except in compliance with +-- the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- +local core = require("apisix.core") +local log_util = require("apisix.utils.log-util") +local batch_processor = require("apisix.utils.batch-processor") +local plugin_name = "sls-logger" +local ngx = ngx +local rf5424 = require("apisix.plugins.slslog.rfc5424") +local stale_timer_running = false; +local timer_at = ngx.timer.at +local tcp = ngx.socket.tcp +local buffers = {} +local tostring = tostring +local ipairs = ipairs +local schema = { + type = "object", + properties = { + include_req_body = {type = "boolean", default = false}, + name = {type = "string", default = "sls-logger"}, + timeout = {type = "integer", minimum = 1, default= 5000}, + max_retry_count = {type = "integer", minimum = 0, default = 0}, + retry_delay = {type = "integer", minimum = 0, default = 1}, + buffer_duration = {type = "integer", minimum = 1, default = 60}, + inactive_timeout = {type = "integer", minimum = 1, default = 5}, + batch_max_size = {type = "integer", minimum = 1, default = 1000}, + host = {type = "string"}, + port = {type = "integer"}, + project = {type = "string"}, + logstore = {type = "string"}, + access_key_id = {type = "string"}, + access_key_secret = {type ="string"} + }, + required = {"host", "port", "project", "logstore", "access_key_id", "access_key_secret"} +} + +local _M = { + version = 0.1, + priority = 406, + name = plugin_name, + schema = schema, +} + +function _M.check_schema(conf) + return core.schema.check(schema, conf) +end + +local function send_tcp_data(route_conf, log_message) + local err_msg + local res = true + local sock, soc_err = tcp() + local can_close + + if not sock then + return false, "failed to init the socket" .. soc_err + end + + sock:settimeout(route_conf.timeout) + local ok, err = sock:connect(route_conf.host, route_conf.port) + if not ok then + return false, "failed to connect to TCP server: host[" .. route_conf.host + .. "] port[" .. tostring(route_conf.port) .. "] err: " .. err + end + + ok, err = sock:sslhandshake(true, nil, false) + if not ok then + return false, "failed to to perform TLS handshake to TCP server: host[" + .. route_conf.host .. "] port[" .. tostring(route_conf.port) + .. "] err: " .. err + end + + core.log.debug("sls logger send data ", log_message) + ok, err = sock:send(log_message) + if not ok then + res = false + can_close = true + err_msg = "failed to send data to TCP server: host[" .. route_conf.host + .. "] port[" .. tostring(route_conf.port) .. "] err: " .. err + else + ok, err = sock:setkeepalive(120 * 1000, 20) + if not ok then + can_close = true + core.log.warn("failed to set socket keepalive: host[" .. route_conf.host + .. "] port[" .. tostring(route_conf.port) .. "] err: " .. err) + end + end + + if can_close then + ok, err = sock:close() + if not ok then + core.log.warn("failed to close the TCP connection, host[", + route_conf.host, "] port[", route_conf.port, "] ", err) + end + end + + return res, err_msg +end + +-- remove stale objects from the memory after timer expires +local function remove_stale_objects(premature) + if premature then + return + end + + for key, batch in ipairs(buffers) do + if #batch.entry_buffer.entries == 0 and #batch.batch_to_process == 0 then + core.log.warn("removing batch processor stale object, route id:", tostring(key)) + buffers[key] = nil + end + end + + stale_timer_running = false +end + +local function combine_syslog(entries) + local data + for _, entry in ipairs(entries) do + if not data then + data = entry.data + end + + data = data .. entry.data + core.log.info(entry.data) Review comment: fix it. ########## File path: apisix/plugins/slslog/rfc5424.lua ########## @@ -0,0 +1,107 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one or more +-- contributor license agreements. See the NOTICE file distributed with +-- this work for additional information regarding copyright ownership. +-- The ASF licenses this file to You under the Apache License, Version 2.0 +-- (the "License"); you may not use this file except in compliance with +-- the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- +local LOG_EMERG = 0 -- system is unusable +local LOG_ALERT = 1 -- action must be taken immediately +local LOG_CRIT = 2 -- critical conditions +local LOG_ERR = 3 -- error conditions +local LOG_WARNING = 4 -- warning conditions +local LOG_NOTICE = 5 -- normal but significant condition +local LOG_INFO = 6 -- informational +local LOG_DEBUG = 7 -- debug-level messages + +local LOG_KERN = 0 -- kernel messages +local LOG_USER = 1 -- random user-level messages +local LOG_MAIL = 2 -- mail system +local LOG_DAEMON = 3 -- system daemons +local LOG_AUTH = 4 -- security/authorization messages +local LOG_SYSLOG = 5 -- messages generated internally by syslogd +local LOG_LPR = 6 -- line printer subsystem +local LOG_NEWS = 7 -- network news subsystem +local LOG_UUCP = 8 -- UUCP subsystem +local LOG_CRON = 9 -- clock daemon +local LOG_AUTHPRIV = 10 -- security/authorization messages (private) +local LOG_FTP = 11 -- FTP daemon +local LOG_LOCAL0 = 16 -- reserved for local use +local LOG_LOCAL1 = 17 -- reserved for local use +local LOG_LOCAL2 = 18 -- reserved for local use +local LOG_LOCAL3 = 19 -- reserved for local use +local LOG_LOCAL4 = 20 -- reserved for local use +local LOG_LOCAL5 = 21 -- reserved for local use +local LOG_LOCAL6 = 22 -- reserved for local use +local LOG_LOCAL7 = 23 -- reserved for local use + +local Facility = { + ["KERN"] = LOG_KERN, + ["USER"] = LOG_USER, + ["MAIL"] = LOG_MAIL, + ["DAEMON"] = LOG_DAEMON, + ["AUTH"] = LOG_AUTH, + ["SYSLOG"] = LOG_SYSLOG, + ["LPR"] = LOG_LPR, + ["NEWS"] = LOG_NEWS, + ["UUCP"] = LOG_UUCP, + ["CRON"] = LOG_CRON, + ["AUTHPRIV"] = LOG_AUTHPRIV, + ["FTP"] = LOG_FTP, + ["LOCAL0"] = LOG_LOCAL0, + ["LOCAL1"] = LOG_LOCAL1, + ["LOCAL2"] = LOG_LOCAL2, + ["LOCAL3"] = LOG_LOCAL3, + ["LOCAL4"] = LOG_LOCAL4, + ["LOCAL5"] = LOG_LOCAL5, + ["LOCAL6"] = LOG_LOCAL6, + ["LOCAL7"] = LOG_LOCAL7, +} + +local Severity = { + ["EMEGR"] = LOG_EMERG, + ["ALERT"] = LOG_ALERT, + ["CRIT"] = LOG_CRIT, + ["ERR"] = LOG_ERR, + ["WARNING"] = LOG_WARNING, + ["NOTICE"] = LOG_NOTICE, + ["INFO"] = LOG_INFO, + ["DEBUG"] = LOG_DEBUG, +} +local os_date = os.date +local ngx = ngx +local string_format = string.format +local rfc5424_timestamp_format = "!%Y-%m-%dT%H:%M:%S.000Z" +local rfc5424_format = "<%d>1 %s %s %s %d - [logservice project=\"%s\" logstore=\"%s\"" .. + " access-key-id=\"%s\" access-key-secret=\"%s\"] %s\n" Review comment: fix it. ########## File path: doc/plugins/sls-logger.md ########## @@ -0,0 +1,118 @@ +<!-- +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +--> + +- [中文](../zh-cn/plugins/sls-logger.md) + +# Summary + +- [**Name**](#name) +- [**Attributes**](#attributes) +- [**How To Enable**](#how-to-enable) +- [**Test Plugin**](#test-plugin) +- [**Disable Plugin**](#disable-plugin) + +## Name + +`sls-logger` is a plugin which push Log data requests to ali cloud [Log Server](https://help.aliyun.com/document_detail/112903.html?spm=a2c4g.11186623.6.763.21321b47wcwt1u) with [RF5424](https://tools.ietf.org/html/rfc5424). + +This plugin provides the ability to push Log data as a batch to ali cloud log service. In case if you did not recieve the log data don't worry give it some time it will automatically send the logs after the timer function expires in our Batch Processor. + +For more info on Batch-Processor in Apache APISIX please refer. +[Batch-Processor](../batch-processor.md) + +## Attributes + +|Name |Requirement |Description| +|--------- |-------- |-----------| +|host |required | IP address or the Hostname of the TCP server, please reference ali cloud log [Serve List](https://help.aliyun.com/document_detail/29008.html?spm=a2c4g.11186623.2.14.49301b4793uX0z#reference-wgx-pwq-zdb), use IP address insted of domain.| +|port |required |Target upstream port, default 10009.| +|timeout |optional |Timeout for the upstream to send data.| +| project |required|Ali cloud log service project name，please creat in sls before us this plugin.| +| logstore | required |Ali cloud log service logstore name，please creat in sls before us this plugin.| +| access_key_id | required | Ali cloud AccessKey ID, reference [Authorization](https://help.aliyun.com/document_detail/47664.html?spm=a2c4g.11186623.2.15.49301b47lfvxXP#task-xsk-ttc-ry)。| +| access_key_secret | required |Ali cloud AccessKey Secret, reference [Authorization](https://help.aliyun.com/document_detail/47664.html?spm=a2c4g.11186623.2.15.49301b47lfvxXP#task-xsk-ttc-ry)。| +| include_req_body | required| Boolean value. | +|name |optional |A unique identifier to identity the batch processor| +|batch_max_size |optional |Max size of each batch.| +|inactive_timeout|optional |maximum age in seconds when the buffer will be flushed if inactive.| +|buffer_duration|optional |Maximum age in seconds of the oldest entry in a batch before the batch must be processed.| +|max_retry_count|optional |Maximum number of retries before removing from the processing pipe line; default is zero| +|retry_delay |optional |Number of seconds the process execution should be delayed if the execution fails; default is 1| + +## How To Enable + +The following is an example on how to enable the sls-logger for a specific route. + +```shell +curl http://127.0.0.1:9080/apisix/admin/routes/5 -H 'X-API-KEY: edd1c9f034335f136f87ad84b625c8f1' -X PUT -d ' +{ + "plugins": { Review comment: fix it. ---------------------------------------------------------------- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: [email protected]
