This is an automated email from the ASF dual-hosted git repository.
spacewander pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/apisix.git
The following commit(s) were added to refs/heads/master by this push:
new db758902e feat(log-rorate): log-rotate plugin support max_size (#7749)
db758902e is described below
commit db758902e4cb5ad3fd9ad80fd76e22f15dc2f6f8
Author: HaiYan <[email protected]>
AuthorDate: Fri Aug 26 14:12:13 2022 +0800
feat(log-rorate): log-rotate plugin support max_size (#7749)
Co-authored-by: 罗泽轩 <[email protected]>
Co-authored-by: Fei Han <[email protected]>
Co-authored-by: qihaiyan <[email protected]>
---
apisix/plugins/log-rotate.lua | 121 +++++++++++++++---------------
conf/config-default.yaml | 1 +
docs/en/latest/plugins/log-rotate.md | 8 +-
docs/zh/latest/plugins/log-rotate.md | 2 +
t/plugin/log-rotate3.t | 141 +++++++++++++++++++++++++++++++++++
5 files changed, 211 insertions(+), 62 deletions(-)
diff --git a/apisix/plugins/log-rotate.lua b/apisix/plugins/log-rotate.lua
index 794593717..571fc9b52 100644
--- a/apisix/plugins/log-rotate.lua
+++ b/apisix/plugins/log-rotate.lua
@@ -21,6 +21,7 @@ local plugin = require("apisix.plugin")
local process = require("ngx.process")
local signal = require("resty.signal")
local shell = require("resty.shell")
+local ipairs = ipairs
local ngx = ngx
local ngx_time = ngx.time
local ngx_update_time = ngx.update_time
@@ -43,6 +44,7 @@ local local_conf
local plugin_name = "log-rotate"
local INTERVAL = 60 * 60 -- rotate interval (unit: second)
local MAX_KEPT = 24 * 7 -- max number of log files will be kept
+local MAX_SIZE = -1 -- max size of file will be rotated
local COMPRESSION_FILE_SUFFIX = ".tar.gz" -- compression file suffix
local rotate_time
local default_logs
@@ -123,34 +125,22 @@ local function tab_sort_comp(a, b)
end
-local function scan_log_folder()
- local t = {
- access = {},
- error = {},
- }
+local function scan_log_folder(log_file_name)
+ local t = {}
- local log_dir, access_name = get_log_path_info("access.log")
- local _, error_name = get_log_path_info("error.log")
-
- if enable_compression then
- access_name = access_name .. COMPRESSION_FILE_SUFFIX
- error_name = error_name .. COMPRESSION_FILE_SUFFIX
- end
+ local log_dir, _ = get_log_path_info(log_file_name)
for file in lfs.dir(log_dir) do
local n = get_last_index(file, "__")
if n ~= nil then
local log_type = file:sub(n + 2)
- if log_type == access_name then
- tab_insert(t.access, file)
- elseif log_type == error_name then
- tab_insert(t.error, file)
+ if log_type == log_file_name then
+ tab_insert(t, file)
end
end
end
- tab_sort(t.access, tab_sort_comp)
- tab_sort(t.error, tab_sort_comp)
+ tab_sort(t, tab_sort_comp)
return t, log_dir
end
@@ -219,18 +209,62 @@ local function init_default_logs(logs_info, log_type)
end
+local function file_size(file)
+ local attr = lfs.attributes(file)
+ if attr then
+ return attr.size
+ end
+ return 0
+end
+
+
+local function rotate_file(files, now_time, max_kept)
+ for _, file in ipairs(files) do
+ local now_date = os_date("%Y-%m-%d_%H-%M-%S", now_time)
+ local new_file = rename_file(default_logs[file], now_date)
+ if not new_file then
+ return
+ end
+
+ local pid = process.get_master_pid()
+ core.log.warn("send USR1 signal to master process [", pid, "] for
reopening log file")
+ local ok, err = signal.kill(pid, signal.signum("USR1"))
+ if not ok then
+ core.log.error("failed to send USR1 signal for reopening log file:
", err)
+ end
+
+ if enable_compression then
+ compression_file(new_file)
+ end
+
+ -- clean the oldest file
+ local log_list, log_dir = scan_log_folder(file)
+ for i = max_kept + 1, #log_list do
+ local path = log_dir .. log_list[i]
+ local ok, err = os_remove(path)
+ if err then
+ core.log.error("remove old log file: ", path, " err: ", err, "
res:", ok)
+ end
+ end
+ end
+end
+
+
local function rotate()
local interval = INTERVAL
local max_kept = MAX_KEPT
+ local max_size = MAX_SIZE
local attr = plugin.plugin_attr(plugin_name)
if attr then
interval = attr.interval or interval
max_kept = attr.max_kept or max_kept
+ max_size = attr.max_size or max_size
enable_compression = attr.enable_compression or enable_compression
end
core.log.info("rotate interval:", interval)
core.log.info("rotate max keep:", max_kept)
+ core.log.info("rotate max size:", max_size)
if not default_logs then
-- first init default log filepath and filename
@@ -248,53 +282,22 @@ local function rotate()
return
end
- if now_time < rotate_time then
- -- did not reach the rotate time
- core.log.info("rotate time: ", rotate_time, " now time: ", now_time)
- return
- end
+ if now_time >= rotate_time then
+ local files = {DEFAULT_ACCESS_LOG_FILENAME, DEFAULT_ERROR_LOG_FILENAME}
+ rotate_file(files, now_time, max_kept)
- local now_date = os_date("%Y-%m-%d_%H-%M-%S", now_time)
- local access_new_file =
rename_file(default_logs[DEFAULT_ACCESS_LOG_FILENAME], now_date)
- local error_new_file =
rename_file(default_logs[DEFAULT_ERROR_LOG_FILENAME], now_date)
- if not access_new_file and not error_new_file then
-- reset rotate time
rotate_time = rotate_time + interval
- return
- end
-
- core.log.warn("send USR1 signal to master process [",
- process.get_master_pid(), "] for reopening log file")
- local ok, err = signal.kill(process.get_master_pid(),
signal.signum("USR1"))
- if not ok then
- core.log.error("failed to send USR1 signal for reopening log file: ",
err)
- end
-
- if enable_compression then
- compression_file(access_new_file)
- compression_file(error_new_file)
- end
-
- -- clean the oldest file
- local log_list, log_dir = scan_log_folder()
- for i = max_kept + 1, #log_list.error do
- local path = log_dir .. log_list.error[i]
- ok, err = os_remove(path)
- if err then
- core.log.error("remove old error file: ", path, " err: ", err, "
res:", ok)
+ elseif max_size > 0 then
+ local access_log_file_size =
file_size(default_logs[DEFAULT_ACCESS_LOG_FILENAME].file)
+ local error_log_file_size =
file_size(default_logs[DEFAULT_ERROR_LOG_FILENAME].file)
+ if access_log_file_size >= max_size then
+ rotate_file({DEFAULT_ACCESS_LOG_FILENAME}, now_time, max_kept)
end
- end
-
- for i = max_kept + 1, #log_list.access do
- local path = log_dir .. log_list.access[i]
- ok, err = os_remove(path)
- if err then
- core.log.error("remove old error file: ", path, " err: ", err, "
res:", ok)
+ if error_log_file_size >= max_size then
+ rotate_file({DEFAULT_ERROR_LOG_FILENAME}, now_time, max_kept)
end
end
-
- -- reset rotate time
- rotate_time = rotate_time + interval
end
diff --git a/conf/config-default.yaml b/conf/config-default.yaml
index b4074cd31..e9ae1ffcd 100755
--- a/conf/config-default.yaml
+++ b/conf/config-default.yaml
@@ -495,6 +495,7 @@ plugin_attr:
log-rotate:
interval: 3600 # rotate interval (unit: second)
max_kept: 168 # max number of log files will be kept
+ max_size: -1 # max size bytes of log files to be rotated, size check
would be skipped with a value less than 0
enable_compression: false # enable log file compression(gzip) or not,
default false
skywalking:
service_name: APISIX
diff --git a/docs/en/latest/plugins/log-rotate.md
b/docs/en/latest/plugins/log-rotate.md
index 3c4cc7ecc..3aee947fe 100644
--- a/docs/en/latest/plugins/log-rotate.md
+++ b/docs/en/latest/plugins/log-rotate.md
@@ -39,6 +39,7 @@ You can configure how often the logs are rotated and how many
logs to keep. When
|--------------------|---------|----------|---------|------------------------------------------------------------------------------------------------|
| interval | integer | True | 60 * 60 | Time in seconds
specifying how often to rotate the logs. |
| max_kept | integer | True | 24 * 7 | Maximum number of
historical logs to keep. If this number is exceeded, older logs are deleted. |
+| max_size | integer | False | -1 | Max size(Bytes) of log
files to be rotated, size check would be skipped with a value less than 0 or
time is up specified by interval. |
| enable_compression | boolean | False | false | When set to `true`,
compresses the log file (gzip). Requires `tar` to be installed. |
## Enabling the Plugin
@@ -51,9 +52,10 @@ plugins:
plugin_attr:
log-rotate:
- interval: 3600
- max_kept: 168
- enable_compression: false
+ interval: 3600 # rotate interval (unit: second)
+ max_kept: 168 # max number of log files will be kept
+ max_size: -1 # max size of log files will be kept
+ enable_compression: false # enable log file compression(gzip) or
not, default false
```
## Example usage
diff --git a/docs/zh/latest/plugins/log-rotate.md
b/docs/zh/latest/plugins/log-rotate.md
index 58b675946..14f01c837 100644
--- a/docs/zh/latest/plugins/log-rotate.md
+++ b/docs/zh/latest/plugins/log-rotate.md
@@ -39,6 +39,7 @@ description: 云原生 API 网关 Apache APISIX log-rotate 插件用于定期切
| ------------------ | ------- | ------ | ------- | ------------- |
---------------------------------------------------------------------------- |
| interval | integer | 是 | 60 * 60 | |
每间隔多长时间切分一次日志，以秒为单位。 |
| max_kept | integer | 是 | 24 * 7 | |
最多保留多少份历史日志，超过指定数量后，自动删除老文件。 |
+| max_size | integer | 否 | -1 | |
日志文件超过指定大小时进行切分，单位为 Byte 。如果 `max_size` 小于 0 或者根据 `interval` 计算的时间到达时，将不会根据
`max_size` 切分日志。 |
| enable_compression | boolean | 否 | false | [false, true] | 当设置为 `true`
时，启用日志文件压缩。该功能需要在系统中安装 `tar` 。 |
开启该插件后，就会按照参数自动切分日志文件了。比如以下示例是根据 `interval: 10` 和 `max_kept: 10` 得到的样本。
@@ -92,6 +93,7 @@ plugin_attr:
log-rotate:
interval: 3600 # rotate interval (unit: second)
max_kept: 168 # max number of log files will be kept
+ max_size: -1 # max size of log files will be kept
enable_compression: false # enable log file compression(gzip) or
not, default false
```
diff --git a/t/plugin/log-rotate3.t b/t/plugin/log-rotate3.t
new file mode 100644
index 000000000..bfab0f9b6
--- /dev/null
+++ b/t/plugin/log-rotate3.t
@@ -0,0 +1,141 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+use t::APISIX 'no_plan';
+
+repeat_each(1);
+no_long_string();
+no_shuffle();
+no_root_location();
+
+add_block_preprocessor(sub {
+ my ($block) = @_;
+
+ if (!defined $block->yaml_config) {
+ my $yaml_config = <<_EOC_;
+apisix:
+ node_listen: 1984
+ admin_key: ~
+plugins:
+ - log-rotate
+plugin_attr:
+ log-rotate:
+ interval: 86400
+ max_size: 9
+ max_kept: 3
+ enable_compression: false
+_EOC_
+
+ $block->set_value("yaml_config", $yaml_config);
+ }
+
+ if ((!defined $block->error_log) && (!defined $block->no_error_log)) {
+ $block->set_value("no_error_log", "[error]");
+ }
+
+ if (!defined $block->request) {
+ $block->set_value("request", "GET /t");
+ }
+
+});
+
+run_tests;
+
+__DATA__
+
+=== TEST 1: log rotate by max_size
+--- config
+ location /t {
+ content_by_lua_block {
+ ngx.log(ngx.ERR, "start xxxxxx")
+ ngx.sleep(2)
+ local has_split_access_file = false
+ local has_split_error_file = false
+ local lfs = require("lfs")
+ for file_name in lfs.dir(ngx.config.prefix() .. "/logs/") do
+ if string.match(file_name, "__access.log$") then
+ has_split_access_file = true
+ end
+
+ if string.match(file_name, "__error.log$") then
+ has_split_error_file = true
+ end
+ end
+
+ if not has_split_access_file and has_split_error_file then
+ ngx.status = 200
+ else
+ ngx.status = 500
+ end
+ }
+ }
+
+
+
+=== TEST 2: in current log
+--- config
+ location /t {
+ content_by_lua_block {
+ ngx.sleep(0.1)
+ ngx.log(ngx.WARN, "start xxxxxx")
+ ngx.say("done")
+ }
+ }
+--- response_body
+done
+--- error_log
+start xxxxxx
+
+
+
+=== TEST 3: check file changes
+--- config
+ location /t {
+ content_by_lua_block {
+ ngx.sleep(1)
+
+ local default_logs = {}
+ for file_name in lfs.dir(ngx.config.prefix() .. "/logs/") do
+ if string.match(file_name, "__error.log$") or
string.match(file_name, "__access.log$") then
+ local filepath = ngx.config.prefix() .. "/logs/" ..
file_name
+ local attr = lfs.attributes(filepath)
+ if attr then
+ default_logs[filepath] = { change = attr.change, size
= attr.size }
+ end
+ end
+ end
+
+ ngx.sleep(1)
+
+ local passed = false
+ for filepath, origin_attr in pairs(default_logs) do
+ local check_attr = lfs.attributes(filepath)
+ if check_attr.change == origin_attr.change and check_attr.size
== origin_attr.size then
+ passed = true
+ else
+ passed = false
+ break
+ end
+ end
+
+ if passed then
+ ngx.say("passed")
+ end
+ }
+ }
+--- response_body
+passed
