This is an automated email from the ASF dual-hosted git repository.
spacewander pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/apisix.git
The following commit(s) were added to refs/heads/master by this push:
new 05fc230 feat(log-rotate): log file compression feature (#4795)
05fc230 is described below
commit 05fc2306b5f005799e908273cfe11bd5f830459f
Author: okaybase <[email protected]>
AuthorDate: Thu Aug 12 10:02:25 2021 +0800
feat(log-rotate): log file compression feature (#4795)
---
apisix/plugins/log-rotate.lua | 40 ++++++++++++---
conf/config-default.yaml | 1 +
docs/en/latest/plugins/log-rotate.md | 17 +++++++
docs/zh/latest/plugins/log-rotate.md | 17 +++++++
t/plugin/log-rotate2.t | 96 ++++++++++++++++++++++++++++++++++++
5 files changed, 165 insertions(+), 6 deletions(-)
diff --git a/apisix/plugins/log-rotate.lua b/apisix/plugins/log-rotate.lua
index c44abe2..54dbed0 100644
--- a/apisix/plugins/log-rotate.lua
+++ b/apisix/plugins/log-rotate.lua
@@ -20,6 +20,7 @@ local timers = require("apisix.timers")
local plugin = require("apisix.plugin")
local process = require("ngx.process")
local signal = require("resty.signal")
+local shell = require("resty.shell")
local ngx = ngx
local lfs = require("lfs")
local io = io
@@ -33,6 +34,9 @@ local local_conf
local plugin_name = "log-rotate"
local INTERVAL = 60 * 60 -- rotate interval (unit: second)
local MAX_KEPT = 24 * 7 -- max number of log files will be kept
+local COMPRESSION_FILE_SUFFIX = ".tar.gz" -- compression file suffix
+local enable_compression = false
+
local schema = {
type = "object",
properties = {},
@@ -106,7 +110,8 @@ local function rotate_file(date_str, file_type)
core.log.info("rotate log_dir:", log_dir)
core.log.info("rotate filename:", filename)
- local file_path = log_dir .. date_str .. "__" .. filename
+ local new_filename = date_str .. "__" .. filename
+ local file_path = log_dir .. new_filename
if file_exists(file_path) then
core.log.info("file exist: ", file_path)
return false
@@ -116,6 +121,25 @@ local function rotate_file(date_str, file_type)
local ok, msg = os.rename(file_path_org, file_path)
core.log.info("move file from ", file_path_org, " to ", file_path,
" res:", ok, " msg:", msg)
+
+ if ok and enable_compression then
+ local compression_filename = new_filename .. COMPRESSION_FILE_SUFFIX
+ local cmd = string.format("cd %s && tar -zcf %s %s",
+ log_dir, compression_filename, new_filename)
+ core.log.info("log file compress command: " .. cmd)
+ local ok, stdout, stderr, reason, status = shell.run(cmd)
+ core.log.info("compress log file from ", new_filename, " to ",
compression_filename,
+ " res:", ok)
+
+ if ok then
+ ok = os.remove(file_path)
+ core.log.warn("remove uncompressed log file: ", file_path, " ret:
", ok)
+ else
+ core.log.error("failed to compress log file: ", new_filename, "
ret: ", ok,
+ " stdout: ", stdout, " stderr: ", stderr, " reason: ", reason,
" status: ", status)
+ end
+ end
+
return true
end
@@ -133,6 +157,11 @@ local function scan_log_folder()
local log_dir, access_name = get_log_path_info("access.log")
local _, error_name = get_log_path_info("error.log")
+ if enable_compression then
+ access_name = access_name .. COMPRESSION_FILE_SUFFIX
+ error_name = error_name .. COMPRESSION_FILE_SUFFIX
+ end
+
for file in lfs.dir(log_dir) do
local n = get_last_index(file, "__")
if n ~= nil then
@@ -147,7 +176,7 @@ local function scan_log_folder()
table.sort(t.access, tab_sort)
table.sort(t.error, tab_sort)
- return t
+ return t, log_dir
end
@@ -158,6 +187,7 @@ local function rotate()
if attr then
interval = attr.interval or interval
max_kept = attr.max_kept or max_kept
+ enable_compression = attr.enable_compression or enable_compression
end
core.log.info("rotate interval:", interval)
@@ -182,13 +212,11 @@ local function rotate()
process.get_master_pid(), "] for reopening log file")
local ok, err = signal.kill(process.get_master_pid(),
signal.signum("USR1"))
if not ok then
- core.log.error("failed to send USER1 signal for reopening log file: ",
- err)
+ core.log.error("failed to send USER1 signal for reopening log file: ",
err)
end
-- clean the oldest file
- local log_list = scan_log_folder()
- local log_dir, _ = get_log_path_info("access.log")
+ local log_list, log_dir = scan_log_folder()
for i = max_kept + 1, #log_list.error do
local path = log_dir .. log_list.error[i]
local ok = os.remove(path)
diff --git a/conf/config-default.yaml b/conf/config-default.yaml
index c1f38b0..53930c3 100644
--- a/conf/config-default.yaml
+++ b/conf/config-default.yaml
@@ -334,6 +334,7 @@ plugin_attr:
log-rotate:
interval: 3600 # rotate interval (unit: second)
max_kept: 168 # max number of log files will be kept
+ enable_compression: false # enable log file compression(gzip) or not,
default false
skywalking:
service_name: APISIX
service_instance_name: APISIX Instance Name
diff --git a/docs/en/latest/plugins/log-rotate.md
b/docs/en/latest/plugins/log-rotate.md
index 41bfa65..bd1a56b 100644
--- a/docs/en/latest/plugins/log-rotate.md
+++ b/docs/en/latest/plugins/log-rotate.md
@@ -34,6 +34,7 @@ When the number of log files exceeds the remaining number,
the old files are aut
| -------- | ------- | ----------- | ------- | ----- |
--------------------------------------------------------------------------------------------------------------------
|
| interval | integer | required | 60 * 60 | | How often to rotate the
log in seconds
|
| max_kept | integer | required | 24 * 7 | | How many historical
logs can be kept at most. When this number is exceeded, old files will be
deleted automatically. |
+| enable_compression | boolean | optional | false | | Whether to
enable log file compression(gzip). This feature requires `tar` installed.
|
After this plug-in is enabled, the log file will be automatically rotated
according to the configuration.
For example, the following example is a sample based on `interval: 10` and
`max_kept: 10`.
@@ -65,6 +66,21 @@ total 44K
-rw-r--r--. 1 resty resty 1.5K Mar 20 21:31 error.log
```
+When enable log file compression, log file will be like below.
+
+```shell
+$ ll logs
+total 10.5K
+-rw-r--r--. 1 resty resty 1.5K Mar 20 20:33
2020-03-20_20-33-50_access.log.tar.gz
+-rw-r--r--. 1 resty resty 1.5K Mar 20 20:33
2020-03-20_20-33-50_error.log.tar.gz
+-rw-r--r--. 1 resty resty 1.5K Mar 20 20:33
2020-03-20_20-34-00_access.log.tar.gz
+-rw-r--r--. 1 resty resty 1.5K Mar 20 20:34
2020-03-20_20-34-00_error.log.tar.gz
+-rw-r--r--. 1 resty resty 1.5K Mar 20 20:34
2020-03-20_20-34-10_access.log.tar.gz
+-rw-r--r--. 1 resty resty 1.5K Mar 20 20:34
2020-03-20_20-34-10_error.log.tar.gz
+-rw-r--r--. 1 resty resty 0 Mar 20 20:34 access.log
+-rw-r--r--. 1 resty resty 1.5K Mar 20 21:31 error.log
+```
+
### Example
#### Enable plugin
@@ -83,6 +99,7 @@ plugin_attr:
log-rotate:
interval: 3600 # rotate interval (unit: second)
max_kept: 168 # max number of log files will be kept
+ enable_compression: false # enable log file compression(gzip) or
not, default false
```
#### Disable plugin
diff --git a/docs/zh/latest/plugins/log-rotate.md
b/docs/zh/latest/plugins/log-rotate.md
index ab84c63..0c60823 100644
--- a/docs/zh/latest/plugins/log-rotate.md
+++ b/docs/zh/latest/plugins/log-rotate.md
@@ -30,6 +30,7 @@ title: log-rotate
| -------- | ------- | ------ | ------- | ------ |
------------------------------------------------------ |
| interval | integer | 必须 | 60 * 60 | | 每间隔多长时间切分一次日志，秒为单位
|
| max_kept | integer | 必须 | 24 * 7 | | 最多保留多少份历史日志，超过指定数量后，自动删除老文件 |
+| enable_compression | boolean | 可选 | false | |
是否启用日志文件压缩（gzip）。该功能需要安装 `tar` 。 |
开启该插件后，就会按照参数自动切分日志文件了。比如下面的例子是根据 `interval: 10` 和 `max_kept: 10` 得到的样本。
@@ -60,6 +61,21 @@ total 44K
-rw-r--r--. 1 resty resty 1.5K Mar 20 21:31 error.log
```
+当开启日志文件压缩时，日志文件将如下所示。
+
+```shell
+$ ll logs
+total 10.5K
+-rw-r--r--. 1 resty resty 1.5K Mar 20 20:33
2020-03-20_20-33-50_access.log.tar.gz
+-rw-r--r--. 1 resty resty 1.5K Mar 20 20:33
2020-03-20_20-33-50_error.log.tar.gz
+-rw-r--r--. 1 resty resty 1.5K Mar 20 20:33
2020-03-20_20-34-00_access.log.tar.gz
+-rw-r--r--. 1 resty resty 1.5K Mar 20 20:34
2020-03-20_20-34-00_error.log.tar.gz
+-rw-r--r--. 1 resty resty 1.5K Mar 20 20:34
2020-03-20_20-34-10_access.log.tar.gz
+-rw-r--r--. 1 resty resty 1.5K Mar 20 20:34
2020-03-20_20-34-10_error.log.tar.gz
+-rw-r--r--. 1 resty resty 0 Mar 20 20:34 access.log
+-rw-r--r--. 1 resty resty 1.5K Mar 20 21:31 error.log
+```
+
### 示例
#### 开启插件
@@ -75,6 +91,7 @@ plugin_attr:
log-rotate:
interval: 3600 # rotate interval (unit: second)
max_kept: 168 # max number of log files will be kept
+ enable_compression: false # enable log file compression(gzip) or
not, default false
```
#### 禁用插件
diff --git a/t/plugin/log-rotate2.t b/t/plugin/log-rotate2.t
new file mode 100644
index 0000000..d611ffc
--- /dev/null
+++ b/t/plugin/log-rotate2.t
@@ -0,0 +1,96 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+use t::APISIX 'no_plan';
+
+repeat_each(1);
+no_long_string();
+no_shuffle();
+no_root_location();
+
+add_block_preprocessor(sub {
+ my ($block) = @_;
+
+ my $user_yaml_config = <<_EOC_;
+apisix:
+ node_listen: 1984
+ admin_key: null
+
+plugins: # plugin list
+ - log-rotate
+
+plugin_attr:
+ log-rotate:
+ interval: 1
+ max_kept: 3
+ enable_compression: true
+_EOC_
+
+ $block->set_value("yaml_config", $user_yaml_config);
+ $block->set_value("request", "GET /t");
+});
+
+run_tests;
+
+__DATA__
+
+=== TEST 1: log rotate, with enable log file compression
+--- config
+ location /t {
+ content_by_lua_block {
+ ngx.log(ngx.ERR, "start xxxxxx")
+ ngx.sleep(2.5)
+ local has_split_access_file = false
+ local has_split_error_file = false
+ local lfs = require("lfs")
+ for file_name in lfs.dir(ngx.config.prefix() .. "/logs/") do
+ if string.match(file_name, "__access.log.tar.gz$") then
+ has_split_access_file = true
+ end
+
+ if string.match(file_name, "__error.log.tar.gz$") then
+ has_split_error_file = true
+ end
+ end
+
+ if not has_split_access_file or not has_split_error_file then
+ ngx.status = 500
+ else
+ ngx.status = 200
+ end
+ }
+ }
+--- no_error_log
+[error]
+
+
+
+=== TEST 2: in current log, with enable log file compression
+--- config
+ location /t {
+ content_by_lua_block {
+ ngx.sleep(0.1)
+ ngx.log(ngx.WARN, "start xxxxxx")
+ ngx.say("done")
+ }
+ }
+--- response_body
+done
+--- no_error_log
+[error]
+--- error_log
+start xxxxxx
