Copilot commented on code in PR #13347: URL: https://github.com/apache/apisix/pull/13347#discussion_r3230790576
########## apisix/plugins/data-mask.lua: ########## @@ -0,0 +1,265 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one or more +-- contributor license agreements. See the NOTICE file distributed with +-- this work for additional information regarding copyright ownership. +-- The ASF licenses this file to You under the Apache License, Version 2.0 +-- (the "License"); you may not use this file except in compliance with +-- the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- +local ngx = ngx +local ipairs = ipairs +local next = next +local type = type +local re_sub = ngx.re.sub +local core = require("apisix.core") +local jp = require("jsonpath") + +local plugin_name = "data-mask" + +local schema = { + type = "object", + properties = { + request = { + type = "array", + items = { + type = "object", + properties = { + type = {type = "string", enum = {"query", "header", "body"}}, + body_format = {type = "string", enum = {"json", "urlencoded"}}, + name = {type = "string"}, + action = {type = "string", enum = {"regex", "replace", "remove"}}, + regex = {type = "string"}, + value = {type = "string"}, + }, + required = {"type", "name", "action"}, + allOf = { + { + ["if"] = { + properties = {type = {const = "body"}}, + }, + ["then"] = { + required = {"body_format"}, + }, + }, + { + ["if"] = { + properties = {action = {const = "regex"}}, + }, + ["then"] = { + required = {"regex", "value"}, + }, + }, + { + ["if"] = { + properties = {action = {const = "replace"}}, + }, + ["then"] = { + required = {"value"}, + }, + }, + }, + }, + }, + max_body_size = { + type = "integer", + exclusiveMinimum = 0, + default = 1024 * 1024, + }, + max_req_post_args = { + type = "integer", + default = 100, + minimum = 0, + } + }, +} + + +local _M = { + version = 0.1, + priority = 1500, + name = plugin_name, + schema = schema, +} + + +function _M.check_schema(conf) + local ok, err = core.schema.check(schema, conf) + if not ok then + return false, err + end + return true +end + + +local function regex_replace(origin, regex, new) + local res, _, err = re_sub(origin, regex, new, "jo") + if not res then + core.log.error("failed to replace (" .. origin .. ") by regex (".. regex .. + ") with new value (" .. new .. "): ", err) + end + return res Review Comment: `regex_replace()` logs the original value and replacement on regex errors. This can leak the very sensitive data this plugin is meant to redact into the error log. Consider logging only the rule metadata (field name/type/action) and the regex error, without including the original value (and ideally without the replacement string either). ########## docs/en/latest/plugins/data-mask.md: ########## @@ -0,0 +1,296 @@ +--- +title: data-mask +keywords: + - Apache APISIX + - API Gateway + - Plugin + - data-mask +description: This document contains information about the Apache APISIX data-mask Plugin. +--- + +<!-- +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +--> + +<head> + <link rel="canonical" href="https://docs.api7.ai/hub/data-mask"; /> +</head> + +## Description + +The `data-mask` Plugin masks or redacts sensitive fields in request data — query parameters, headers, and body — before they appear in access logs or logger plugins (such as `file-logger` or `http-logger`). + +This is useful for preventing credentials, tokens, payment card numbers, and other sensitive information from being written to logs. + +The plugin runs in the `log` phase and supports three masking actions: + +- `remove`: completely removes the field from the request data. +- `replace`: replaces the field value with a fixed string. +- `regex`: applies a regular expression substitution to the field value. Review Comment: The docs state the plugin masks data before it appears in access logs, but access logs only reflect the masked query string if the Nginx `access_log_format` uses `$request_line` (as used in the tests) rather than the default `$request`. Consider documenting this requirement and providing a config snippet showing how to update `nginx_config.http.access_log_format` accordingly. ########## apisix/plugins/data-mask.lua: ########## @@ -0,0 +1,265 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one or more +-- contributor license agreements. See the NOTICE file distributed with +-- this work for additional information regarding copyright ownership. +-- The ASF licenses this file to You under the Apache License, Version 2.0 +-- (the "License"); you may not use this file except in compliance with +-- the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- +local ngx = ngx +local ipairs = ipairs +local next = next +local type = type +local re_sub = ngx.re.sub +local core = require("apisix.core") +local jp = require("jsonpath") + +local plugin_name = "data-mask" + +local schema = { + type = "object", + properties = { + request = { + type = "array", + items = { + type = "object", + properties = { + type = {type = "string", enum = {"query", "header", "body"}}, + body_format = {type = "string", enum = {"json", "urlencoded"}}, + name = {type = "string"}, + action = {type = "string", enum = {"regex", "replace", "remove"}}, + regex = {type = "string"}, + value = {type = "string"}, + }, + required = {"type", "name", "action"}, + allOf = { + { + ["if"] = { + properties = {type = {const = "body"}}, + }, + ["then"] = { + required = {"body_format"}, + }, + }, + { + ["if"] = { + properties = {action = {const = "regex"}}, + }, + ["then"] = { + required = {"regex", "value"}, + }, + }, + { + ["if"] = { + properties = {action = {const = "replace"}}, + }, + ["then"] = { + required = {"value"}, + }, + }, + }, + }, + }, + max_body_size = { + type = "integer", + exclusiveMinimum = 0, + default = 1024 * 1024, + }, + max_req_post_args = { + type = "integer", + default = 100, + minimum = 0, + } + }, +} + + +local _M = { + version = 0.1, + priority = 1500, + name = plugin_name, + schema = schema, +} + + +function _M.check_schema(conf) + local ok, err = core.schema.check(schema, conf) + if not ok then + return false, err + end + return true +end + + +local function regex_replace(origin, regex, new) + local res, _, err = re_sub(origin, regex, new, "jo") + if not res then + core.log.error("failed to replace (" .. origin .. ") by regex (".. regex .. + ") with new value (" .. new .. "): ", err) + end + return res +end + + +local function mask_table(tab, conf) + if not tab[conf.name] then + return false + end + local masked = false + if conf.action == "remove" then + tab[conf.name] = nil + masked = true + elseif conf.action == "replace" then + tab[conf.name] = conf.value + masked = true + elseif conf.action == "regex" then + local new_arg = regex_replace(tab[conf.name], conf.regex, conf.value) + if new_arg then + tab[conf.name] = new_arg + masked = true + end + end + return masked Review Comment: `mask_table()` assumes `tab[name]` is a scalar string. For query/form args, OpenResty can return a table when the parameter appears multiple times; `ngx.re.sub` will then throw a type error. Please handle table values explicitly (e.g., apply remove/replace to all entries, and apply regex to each string entry) to avoid runtime failures and incomplete masking. ########## apisix/plugins/data-mask.lua: ########## @@ -0,0 +1,265 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one or more +-- contributor license agreements. See the NOTICE file distributed with +-- this work for additional information regarding copyright ownership. +-- The ASF licenses this file to You under the Apache License, Version 2.0 +-- (the "License"); you may not use this file except in compliance with +-- the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- +local ngx = ngx +local ipairs = ipairs +local next = next +local type = type +local re_sub = ngx.re.sub +local core = require("apisix.core") +local jp = require("jsonpath") + +local plugin_name = "data-mask" + +local schema = { + type = "object", + properties = { + request = { + type = "array", + items = { + type = "object", + properties = { + type = {type = "string", enum = {"query", "header", "body"}}, + body_format = {type = "string", enum = {"json", "urlencoded"}}, + name = {type = "string"}, + action = {type = "string", enum = {"regex", "replace", "remove"}}, + regex = {type = "string"}, + value = {type = "string"}, + }, + required = {"type", "name", "action"}, + allOf = { + { + ["if"] = { + properties = {type = {const = "body"}}, + }, + ["then"] = { + required = {"body_format"}, + }, + }, + { + ["if"] = { + properties = {action = {const = "regex"}}, + }, + ["then"] = { + required = {"regex", "value"}, + }, + }, + { + ["if"] = { + properties = {action = {const = "replace"}}, + }, + ["then"] = { + required = {"value"}, + }, + }, + }, + }, + }, + max_body_size = { + type = "integer", + exclusiveMinimum = 0, + default = 1024 * 1024, + }, + max_req_post_args = { + type = "integer", + default = 100, + minimum = 0, + } + }, +} + + +local _M = { + version = 0.1, + priority = 1500, + name = plugin_name, + schema = schema, +} + + +function _M.check_schema(conf) + local ok, err = core.schema.check(schema, conf) + if not ok then + return false, err + end + return true +end + + +local function regex_replace(origin, regex, new) + local res, _, err = re_sub(origin, regex, new, "jo") + if not res then + core.log.error("failed to replace (" .. origin .. ") by regex (".. regex .. + ") with new value (" .. new .. "): ", err) + end + return res +end + + +local function mask_table(tab, conf) + if not tab[conf.name] then + return false + end + local masked = false + if conf.action == "remove" then + tab[conf.name] = nil + masked = true + elseif conf.action == "replace" then + tab[conf.name] = conf.value + masked = true + elseif conf.action == "regex" then + local new_arg = regex_replace(tab[conf.name], conf.regex, conf.value) + if new_arg then + tab[conf.name] = new_arg + masked = true + end + end + return masked +end + + +-- jsonpath index of array starts from 0, lua table index starts from 1 +local function table_index(idx) + if type(idx) == "number" then + return idx + 1 + end + return idx +end + + +local function mask_json(obj, conf) + -- local nodes = jp.nodes(data, '$..author') + -- { + -- { path = {'$', 'store', 'book', 0, 'author'}, value = 'Nigel Rees' }, + -- { path = {'$', 'store', 'book', 1, 'author'}, value = 'Evelyn Waugh' }, + -- } + local nodes = jp.nodes(obj, conf.name) + if not nodes then + return false + end + + local masked = false + for _, node in ipairs(nodes) do + local nested = obj + -- first element is root($), last element is the field name + for i = 2, #node.path - 1 do + nested = nested[table_index(node.path[i])] + end + local index = table_index(node.path[#node.path]) + if conf.action == "remove" then + nested[index] = nil + elseif conf.action == "replace" then + nested[index] = conf.value + elseif conf.action == "regex" then + nested[index] = regex_replace(node.value, conf.regex, conf.value) Review Comment: In `mask_json()`, the `regex` action assigns `nested[index] = regex_replace(...)` even when `regex_replace()` fails and returns nil, which will effectively remove the field from the logged JSON. This differs from `mask_table()` (which only overwrites on success) and can cause unexpected data loss. Consider only updating the field when the regex substitution succeeds, and leave the original value intact otherwise. ########## t/plugin/data-mask.t: ########## @@ -0,0 +1,724 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +use t::APISIX 'no_plan'; + +no_long_string(); +no_root_location(); + +add_block_preprocessor(sub { + my ($block) = @_; + + if (! $block->request) { + $block->set_value("request", "GET /t"); + if (!$block->response_body) { + $block->set_value("response_body", "passed\n"); + } + } +}); + + +run_tests; + +__DATA__ + +=== TEST 1: mask query +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + local code, body = t('/apisix/admin/routes/1', + ngx.HTTP_PUT, + [[{ + "plugins": { + "data-mask": { + "request": [ + { + "action": "remove", + "name": "password", + "type": "query" + }, + { + "action": "replace", + "name": "token", + "type": "query", + "value": "*****" + }, + { + "action": "regex", + "name": "card", + "regex": "(\\d+)\\-\\d+\\-\\d+\\-(\\d+)", + "type": "query", + "value": "$1-****-****-$2" + } + ] + }, + "file-logger": { + "path": "mask-query.log.1" + } + }, + "upstream": { + "nodes": { + "127.0.0.1:1982": 1 + }, + "type": "roundrobin" + }, + "uri": "/hello" + }]] + ) + + if code >= 300 then + ngx.status = code + end + ngx.say(body) + } + } + + + +=== TEST 2: verify +--- config + location /t { + content_by_lua_block { + local core = require("apisix.core") + local t = require("lib.test_admin").test + + local code = t("/hello?password=abc&token=xyz&card=1234-1234-1234-1234", ngx.HTTP_GET) + local fd, err = io.open("mask-query.log.1", "r") + if not fd then + core.log.error("failed to open file: ", err) + return + end + local line = fd:read() + local log = core.json.decode(line) + + if log.request.querystring.password then + ngx.say("password arg mask failed: " .. log.request.querystring.password) + return + end + if log.request.querystring.token ~= "*****" then + ngx.say("token arg mask failed: " .. log.request.querystring.token) + return + end + if log.request.querystring.card ~= "1234-****-****-1234" then + ngx.say("card arg mask failed: " .. log.request.querystring.card) + return + end + if log.request.uri ~= "/hello?token=*****&card=1234-****-****-1234" and + log.request.uri ~= "/hello?card=1234-****-****-1234&token=*****" then + ngx.say("uri mask failed: " .. log.request.uri) + return + end + + os.remove("mask-query.log.1") + ngx.say("success") + } + } +--- response_body +success + + + +=== TEST 3: mask header +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + local code, body = t('/apisix/admin/routes/1', + ngx.HTTP_PUT, + [[{ + "plugins": { + "data-mask": { + "request": [ + { + "action": "remove", + "name": "password", + "type": "header" + }, + { + "action": "replace", + "name": "token", + "type": "header", + "value": "*****" + }, + { + "action": "regex", + "name": "card", + "regex": "(\\d+)\\-\\d+\\-\\d+\\-(\\d+)", + "type": "header", + "value": "$1-****-****-$2" + } + ] + }, + "file-logger": { + "path": "mask-header.log.2" + } + }, + "upstream": { + "nodes": { + "127.0.0.1:1982": 1 + }, + "type": "roundrobin" + }, + "uri": "/hello" + }]] + ) + + if code >= 300 then + ngx.status = code + end + ngx.say(body) + } + } + + + +=== TEST 4: verify +--- config + location /t { + content_by_lua_block { + local core = require("apisix.core") + local t = require("lib.test_admin").test + + local headers = {} + headers["password"] = "abc" + headers["token"] = "xyz" + headers["card"] = "1234-1234-1234-1234" + local code = t("/hello", ngx.HTTP_GET, "", nil, headers) + + local fd, err = io.open("mask-header.log.2", "r") + if not fd then + core.log.error("failed to open file: ", err) + return + end + local line = fd:read() + local log = core.json.decode(line) + + if log.request.headers.password then + ngx.say("password header mask failed: " .. log.request.headers.password) + return + end + if log.request.headers.token ~= "*****" then + ngx.say("token header mask failed: " .. log.request.headers.token) + return + end + if log.request.headers.card ~= "1234-****-****-1234" then + ngx.say("card header mask failed: " .. log.request.headers.card) + return + end + + os.remove("mask-header.log.2") + ngx.say("success") + } + } +--- response_body +success + + + +=== TEST 5: mask urlencoded body +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + local code, body = t('/apisix/admin/routes/1', + ngx.HTTP_PUT, + [[{ + "plugins": { + "data-mask": { + "request": [ + { + "action": "remove", + "body_format": "urlencoded", + "name": "password", + "type": "body" + }, + { + "action": "replace", + "body_format": "urlencoded", + "name": "token", + "type": "body", + "value": "*****" + }, + { + "action": "regex", + "body_format": "urlencoded", + "name": "card", + "regex": "(\\d+)\\-\\d+\\-\\d+\\-(\\d+)", + "type": "body", + "value": "$1-****-****-$2" + } + ] + }, + "file-logger": { + "include_req_body": true, + "path": "mask-urlencoded-body.log" + } + }, + "upstream": { + "nodes": { + "127.0.0.1:1982": 1 + }, + "type": "roundrobin" + }, + "uri": "/hello" + }]] + ) + + if code >= 300 then + ngx.status = code + end + ngx.say(body) + } + } + + + +=== TEST 6: verify +--- config + location /t { + content_by_lua_block { + local core = require("apisix.core") + local t = require("lib.test_admin").test + + local code = t("/hello", ngx.HTTP_POST, "password=abc&token=xyz&card=1234-1234-1234-1234") + + local fd, err = io.open("mask-urlencoded-body.log", "r") + if not fd then + core.log.error("failed to open file: ", err) + return + end + local line = fd:read() + local log = core.json.decode(line) + + if log.request.body ~= "token=*****&card=1234-****-****-1234" and + log.request.body ~= "card=1234-****-****-1234&token=*****" then + ngx.say("urlencoded body mask failed: " .. log.request.body) + return + end + + os.remove("mask-urlencoded-body.log") + ngx.say("success") + } + } +--- response_body +success + + + +=== TEST 7: mask json body +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + local code, body = t('/apisix/admin/routes/1', + ngx.HTTP_PUT, + [[{ + "plugins": { + "data-mask": { + "request": [ + { + "action": "remove", + "body_format": "json", + "name": "$.password", + "type": "body" + }, + { + "action": "replace", + "body_format": "json", + "name": "users[*].token", + "type": "body", + "value": "*****" + }, + { + "action": "regex", + "body_format": "json", + "name": "$.users[*].credit.card", + "regex": "(\\d+)\\-\\d+\\-\\d+\\-(\\d+)", + "type": "body", + "value": "$1-****-****-$2" + } + ] + }, + "file-logger": { + "include_req_body": true, + "path": "mask-json-body.log" + } + }, + "upstream": { + "nodes": { + "127.0.0.1:1982": 1 + }, + "type": "roundrobin" + }, + "uri": "/hello" + }]] + ) + + if code >= 300 then + ngx.status = code + end + ngx.say(body) + } + } + + + +=== TEST 8: verify +--- config + location /t { + content_by_lua_block { + local core = require("apisix.core") + local t = require("lib.test_admin").test + + local code = t("/hello", + ngx.HTTP_POST, + [[{ + "password": "abc", + "users": [ + { + "token": "xyz", + "credit": { + "card": "1234-1234-1234-1234" + } + }, + { + "token": "xyz", + "credit": { + "card": "1234-1234-1234-1234" + } + } + ] + }]] + ) + + local fd, err = io.open("mask-json-body.log", "r") + if not fd then + core.log.error("failed to open file: ", err) + return + end + local line = fd:read() + local log = core.json.decode(line) + + local body = core.json.decode(log.request.body) + if body.password then + ngx.say("$.password mask failed: " .. body.password) + return + end + for _, user in ipairs(body.users) do + if user.token ~= "*****" then + ngx.say("$.users[*].token mask failed: " .. user.token) + return + end + if user.credit.card ~= "1234-****-****-1234" then + ngx.say("$.users[*].credit.card mask failed: " .. user.credit.card) + return + end + end + + os.remove("mask-json-body.log") + ngx.say("success") + } + } +--- response_body +success + + + +=== TEST 9: plugin within global rule should not throw error for missing body. +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + local code, body = t('/apisix/admin/global_rules/1', + ngx.HTTP_PUT, + [[{ + "plugins": { + "data-mask": { + "request": [ + { + "action": "remove", + "name": "password", + "type": "query" + }, + { + "action": "replace", + "name": "token", + "type": "query", + "value": "*****" + }, + { + "action": "regex", + "name": "card", + "regex": "(\\d+)\\-\\d+\\-\\d+\\-(\\d+)", + "type": "query", + "value": "$1-****-****-$2" + } + ] + }, + "file-logger": { + "path": "mask-query.log.4" + } + } + }]] + ) + if code >= 300 then + ngx.status = code + end + ngx.say(body) + } + } + + + +=== TEST 10: verify +--- config + location /t { + content_by_lua_block { + local core = require("apisix.core") + local t = require("lib.test_admin").test + + local code = t("/random", ngx.HTTP_POST, "password=abc&token=xyz&card=1234-1234-1234-1234") + + ngx.say("code: ", code) + } + } +--- response_body +code: 404 +--- no_error_log +no request body found + + + +=== TEST 11: create plugin with default value for `max_req_post_args` +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + local code, body = t('/apisix/admin/routes/1', + ngx.HTTP_PUT, + [[{ + "plugins": { + "data-mask": { + "request": [ + { + "action": "regex", + "body_format": "urlencoded", + "name": "arg100", + "regex": "(\\d+)$", + "type": "body", + "value": "$1" + } + ] + }, + "file-logger": { + "include_req_body": true, + "path": "mask-urlencoded-body.log" + } + }, + "upstream": { + "nodes": { + "127.0.0.1:1982": 1 + }, + "type": "roundrobin" + }, + "uri": "/hello" + }]] + ) + + if code >= 300 then + ngx.status = code + end + ngx.say(body) + } + } + + + +=== TEST 12: verify default value for `max_req_post_args`` Review Comment: Test title has an extra trailing backtick: `verify default value for `max_req_post_args``. Consider removing the extra backtick for consistency with other test titles. ########## docs/zh/latest/plugins/data-mask.md: ########## @@ -0,0 +1,297 @@ +--- +title: data-mask +keywords: + - APISIX + - API 网关 + - Plugin + - data-mask +description: API 网关 Apache APISIX data-mask 插件可用于在请求数据写入访问日志或日志插件之前，对敏感字段进行掩码或脱敏处理。 +--- + +<!-- +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +--> + +<head> + <link rel="canonical" href="https://docs.api7.ai/hub/data-mask"; /> +</head> + +## 描述 + +`data-mask` 插件可在请求数据（查询参数、请求头、请求体）写入访问日志或日志插件（如 `file-logger`、`http-logger`）之前，对敏感字段进行掩码或脱敏处理。 + +该插件适用于防止凭证、令牌、支付卡号及其他敏感信息被写入日志的场景。 + +插件在 `log` 阶段运行，支持以下三种掩码动作： + +- `remove`：从请求数据中完全删除该字段。 +- `replace`：将字段值替换为固定字符串。 +- `regex`：对字段值执行正则表达式替换。 + Review Comment: 文档提到插件会在访问日志中生效，但要让访问日志中的请求行反映被掩码后的查询参数，需要将 Nginx `access_log_format` 从默认的 `$request` 改为使用 `$request_line`（测试里也是这么做的）。建议在文档中明确这一点，并给出 `nginx_config.http.access_log_format` 的配置示例。 ########## apisix/plugins/data-mask.lua: ########## @@ -0,0 +1,265 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one or more +-- contributor license agreements. See the NOTICE file distributed with +-- this work for additional information regarding copyright ownership. +-- The ASF licenses this file to You under the Apache License, Version 2.0 +-- (the "License"); you may not use this file except in compliance with +-- the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- +local ngx = ngx +local ipairs = ipairs +local next = next +local type = type +local re_sub = ngx.re.sub +local core = require("apisix.core") +local jp = require("jsonpath") + +local plugin_name = "data-mask" + +local schema = { + type = "object", + properties = { + request = { + type = "array", + items = { + type = "object", + properties = { + type = {type = "string", enum = {"query", "header", "body"}}, + body_format = {type = "string", enum = {"json", "urlencoded"}}, + name = {type = "string"}, + action = {type = "string", enum = {"regex", "replace", "remove"}}, + regex = {type = "string"}, + value = {type = "string"}, + }, + required = {"type", "name", "action"}, + allOf = { + { + ["if"] = { + properties = {type = {const = "body"}}, + }, + ["then"] = { + required = {"body_format"}, + }, + }, + { + ["if"] = { + properties = {action = {const = "regex"}}, + }, + ["then"] = { + required = {"regex", "value"}, + }, + }, + { + ["if"] = { + properties = {action = {const = "replace"}}, + }, + ["then"] = { + required = {"value"}, + }, + }, + }, + }, + }, + max_body_size = { + type = "integer", + exclusiveMinimum = 0, + default = 1024 * 1024, + }, + max_req_post_args = { + type = "integer", + default = 100, + minimum = 0, + } + }, +} + + +local _M = { + version = 0.1, + priority = 1500, + name = plugin_name, + schema = schema, +} + + +function _M.check_schema(conf) + local ok, err = core.schema.check(schema, conf) + if not ok then + return false, err + end + return true +end + + +local function regex_replace(origin, regex, new) + local res, _, err = re_sub(origin, regex, new, "jo") + if not res then + core.log.error("failed to replace (" .. origin .. ") by regex (".. regex .. + ") with new value (" .. new .. "): ", err) + end + return res +end + + +local function mask_table(tab, conf) + if not tab[conf.name] then + return false + end + local masked = false + if conf.action == "remove" then + tab[conf.name] = nil + masked = true + elseif conf.action == "replace" then + tab[conf.name] = conf.value + masked = true + elseif conf.action == "regex" then + local new_arg = regex_replace(tab[conf.name], conf.regex, conf.value) + if new_arg then + tab[conf.name] = new_arg + masked = true + end + end + return masked +end + + +-- jsonpath index of array starts from 0, lua table index starts from 1 +local function table_index(idx) + if type(idx) == "number" then + return idx + 1 + end + return idx +end + + +local function mask_json(obj, conf) + -- local nodes = jp.nodes(data, '$..author') + -- { + -- { path = {'$', 'store', 'book', 0, 'author'}, value = 'Nigel Rees' }, + -- { path = {'$', 'store', 'book', 1, 'author'}, value = 'Evelyn Waugh' }, + -- } + local nodes = jp.nodes(obj, conf.name) + if not nodes then + return false + end + + local masked = false + for _, node in ipairs(nodes) do + local nested = obj + -- first element is root($), last element is the field name + for i = 2, #node.path - 1 do + nested = nested[table_index(node.path[i])] + end + local index = table_index(node.path[#node.path]) + if conf.action == "remove" then + nested[index] = nil + elseif conf.action == "replace" then + nested[index] = conf.value + elseif conf.action == "regex" then + nested[index] = regex_replace(node.value, conf.regex, conf.value) + end + masked = true Review Comment: For JSON body masking with `action: regex`, `node.value` may be a number/boolean/object rather than a string. Passing a non-string to `ngx.re.sub` can throw an error, breaking the log phase. Please either skip regex masking for non-string values (with a warning) or convert primitives to strings in a well-defined way before applying the regex. ########## apisix/plugins/data-mask.lua: ########## @@ -0,0 +1,265 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one or more +-- contributor license agreements. See the NOTICE file distributed with +-- this work for additional information regarding copyright ownership. +-- The ASF licenses this file to You under the Apache License, Version 2.0 +-- (the "License"); you may not use this file except in compliance with +-- the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- +local ngx = ngx +local ipairs = ipairs +local next = next +local type = type +local re_sub = ngx.re.sub +local core = require("apisix.core") +local jp = require("jsonpath") + +local plugin_name = "data-mask" + +local schema = { + type = "object", + properties = { + request = { + type = "array", + items = { + type = "object", + properties = { + type = {type = "string", enum = {"query", "header", "body"}}, + body_format = {type = "string", enum = {"json", "urlencoded"}}, + name = {type = "string"}, + action = {type = "string", enum = {"regex", "replace", "remove"}}, + regex = {type = "string"}, + value = {type = "string"}, + }, + required = {"type", "name", "action"}, + allOf = { + { + ["if"] = { + properties = {type = {const = "body"}}, + }, + ["then"] = { + required = {"body_format"}, + }, + }, + { + ["if"] = { + properties = {action = {const = "regex"}}, + }, + ["then"] = { + required = {"regex", "value"}, + }, + }, + { + ["if"] = { + properties = {action = {const = "replace"}}, + }, + ["then"] = { + required = {"value"}, + }, + }, + }, + }, + }, + max_body_size = { + type = "integer", + exclusiveMinimum = 0, + default = 1024 * 1024, + }, + max_req_post_args = { + type = "integer", + default = 100, + minimum = 0, + } + }, +} + + +local _M = { + version = 0.1, + priority = 1500, + name = plugin_name, + schema = schema, +} + + +function _M.check_schema(conf) + local ok, err = core.schema.check(schema, conf) + if not ok then + return false, err + end + return true +end + + +local function regex_replace(origin, regex, new) + local res, _, err = re_sub(origin, regex, new, "jo") + if not res then + core.log.error("failed to replace (" .. origin .. ") by regex (".. regex .. + ") with new value (" .. new .. "): ", err) + end + return res +end + + +local function mask_table(tab, conf) + if not tab[conf.name] then + return false + end + local masked = false + if conf.action == "remove" then + tab[conf.name] = nil + masked = true + elseif conf.action == "replace" then + tab[conf.name] = conf.value + masked = true + elseif conf.action == "regex" then + local new_arg = regex_replace(tab[conf.name], conf.regex, conf.value) + if new_arg then + tab[conf.name] = new_arg + masked = true + end + end + return masked +end + + +-- jsonpath index of array starts from 0, lua table index starts from 1 +local function table_index(idx) + if type(idx) == "number" then + return idx + 1 + end + return idx +end + + +local function mask_json(obj, conf) + -- local nodes = jp.nodes(data, '$..author') + -- { + -- { path = {'$', 'store', 'book', 0, 'author'}, value = 'Nigel Rees' }, + -- { path = {'$', 'store', 'book', 1, 'author'}, value = 'Evelyn Waugh' }, + -- } + local nodes = jp.nodes(obj, conf.name) + if not nodes then + return false + end + + local masked = false + for _, node in ipairs(nodes) do + local nested = obj + -- first element is root($), last element is the field name + for i = 2, #node.path - 1 do + nested = nested[table_index(node.path[i])] + end + local index = table_index(node.path[#node.path]) + if conf.action == "remove" then + nested[index] = nil + elseif conf.action == "replace" then + nested[index] = conf.value + elseif conf.action == "regex" then + nested[index] = regex_replace(node.value, conf.regex, conf.value) + end + masked = true + end + return masked +end + + +function _M.log(conf, ctx) + local args = core.request.get_uri_args(ctx) + local query_masked = false + local post_args = {} + local post_args_masked = false + local body = ngx.req.get_body_data() + if body then + post_args = ngx.req.get_post_args(conf.max_req_post_args) + end + local json_body + local body_masked = false + + if conf.request then + for _, item in ipairs(conf.request) do + if item.type == "query" then + if mask_table(args, item) then + query_masked = true + end + end + + if item.type == "header" then + local header = core.request.header(ctx, item.name) + if header then + if item.action == "remove" then + core.request.set_header(ctx, item.name, nil) + elseif item.action == "replace" then + core.request.set_header(ctx, item.name, item.value) + elseif item.action == "regex" then + core.request.set_header(ctx, item.name, + regex_replace(header, item.regex, item.value)) + end + end + end + + if item.type == "body" then + if item.body_format == "urlencoded" then + if mask_table(post_args, item) then + post_args_masked = true + end + elseif item.body_format == "json" then + if body and #body <= conf.max_body_size then + if not json_body then + local js, err = core.json.decode(body) + if not js then + core.log.warn("failed to decode json body: ", err) + else + json_body = js + end + end + if json_body then + if mask_json(json_body, item) then + body_masked = true + end + end + elseif body and #body > conf.max_body_size then + core.log.warn("data-mask: skipping body masking for field '", + item.name, "' because body size (", #body, + ") exceeds max_body_size (", conf.max_body_size, ")") + end + + end + end Review Comment: Body masking currently depends on `ngx.req.get_body_data()` being non-nil; if the body was buffered to a temp file (or simply not read yet), `body` will be nil and both JSON and `urlencoded` masking are skipped, potentially leaving sensitive data in logger outputs. Use `core.request.get_body(conf.max_body_size, ctx)` (or at least `ngx.req.read_body()` + `ngx.req.get_body_data()/get_body_file()`) so masking also works when the body is stored in a file, and so it can actively read the body when needed. ########## apisix/plugins/data-mask.lua: ########## @@ -0,0 +1,265 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one or more +-- contributor license agreements. See the NOTICE file distributed with +-- this work for additional information regarding copyright ownership. +-- The ASF licenses this file to You under the Apache License, Version 2.0 +-- (the "License"); you may not use this file except in compliance with +-- the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- +local ngx = ngx +local ipairs = ipairs +local next = next +local type = type +local re_sub = ngx.re.sub +local core = require("apisix.core") +local jp = require("jsonpath") + +local plugin_name = "data-mask" + +local schema = { + type = "object", + properties = { + request = { + type = "array", + items = { + type = "object", + properties = { + type = {type = "string", enum = {"query", "header", "body"}}, + body_format = {type = "string", enum = {"json", "urlencoded"}}, + name = {type = "string"}, + action = {type = "string", enum = {"regex", "replace", "remove"}}, + regex = {type = "string"}, + value = {type = "string"}, + }, + required = {"type", "name", "action"}, + allOf = { + { + ["if"] = { + properties = {type = {const = "body"}}, + }, + ["then"] = { + required = {"body_format"}, + }, + }, + { + ["if"] = { + properties = {action = {const = "regex"}}, + }, + ["then"] = { + required = {"regex", "value"}, + }, + }, + { + ["if"] = { + properties = {action = {const = "replace"}}, + }, + ["then"] = { + required = {"value"}, + }, + }, + }, + }, + }, + max_body_size = { + type = "integer", + exclusiveMinimum = 0, + default = 1024 * 1024, + }, + max_req_post_args = { + type = "integer", + default = 100, + minimum = 0, + } + }, +} + + +local _M = { + version = 0.1, + priority = 1500, + name = plugin_name, + schema = schema, +} + + +function _M.check_schema(conf) + local ok, err = core.schema.check(schema, conf) + if not ok then + return false, err + end + return true +end + + +local function regex_replace(origin, regex, new) + local res, _, err = re_sub(origin, regex, new, "jo") + if not res then + core.log.error("failed to replace (" .. origin .. ") by regex (".. regex .. + ") with new value (" .. new .. "): ", err) + end + return res +end + + +local function mask_table(tab, conf) + if not tab[conf.name] then + return false + end + local masked = false + if conf.action == "remove" then + tab[conf.name] = nil + masked = true + elseif conf.action == "replace" then + tab[conf.name] = conf.value + masked = true + elseif conf.action == "regex" then + local new_arg = regex_replace(tab[conf.name], conf.regex, conf.value) + if new_arg then + tab[conf.name] = new_arg + masked = true + end + end + return masked +end + + +-- jsonpath index of array starts from 0, lua table index starts from 1 +local function table_index(idx) + if type(idx) == "number" then + return idx + 1 + end + return idx +end + + +local function mask_json(obj, conf) + -- local nodes = jp.nodes(data, '$..author') + -- { + -- { path = {'$', 'store', 'book', 0, 'author'}, value = 'Nigel Rees' }, + -- { path = {'$', 'store', 'book', 1, 'author'}, value = 'Evelyn Waugh' }, + -- } + local nodes = jp.nodes(obj, conf.name) + if not nodes then + return false + end + + local masked = false + for _, node in ipairs(nodes) do + local nested = obj + -- first element is root($), last element is the field name + for i = 2, #node.path - 1 do + nested = nested[table_index(node.path[i])] + end + local index = table_index(node.path[#node.path]) + if conf.action == "remove" then + nested[index] = nil + elseif conf.action == "replace" then + nested[index] = conf.value + elseif conf.action == "regex" then + nested[index] = regex_replace(node.value, conf.regex, conf.value) + end + masked = true + end + return masked +end + + +function _M.log(conf, ctx) + local args = core.request.get_uri_args(ctx) + local query_masked = false + local post_args = {} + local post_args_masked = false + local body = ngx.req.get_body_data() + if body then + post_args = ngx.req.get_post_args(conf.max_req_post_args) + end + local json_body + local body_masked = false + + if conf.request then + for _, item in ipairs(conf.request) do + if item.type == "query" then + if mask_table(args, item) then + query_masked = true + end + end + + if item.type == "header" then + local header = core.request.header(ctx, item.name) + if header then + if item.action == "remove" then + core.request.set_header(ctx, item.name, nil) + elseif item.action == "replace" then + core.request.set_header(ctx, item.name, item.value) + elseif item.action == "regex" then + core.request.set_header(ctx, item.name, + regex_replace(header, item.regex, item.value)) + end + end + end + + if item.type == "body" then + if item.body_format == "urlencoded" then + if mask_table(post_args, item) then + post_args_masked = true + end + elseif item.body_format == "json" then Review Comment: `ngx.req.get_post_args(conf.max_req_post_args)` is executed for any request with an in-memory body, even if there are no `type: body` + `body_format: urlencoded` rules (and regardless of `Content-Type`). This can add unnecessary parsing overhead in the log phase. Consider deferring `get_post_args` until you actually encounter a matching `urlencoded` rule, and optionally gating it on `Content-Type: application/x-www-form-urlencoded`. ########## t/plugin/data-mask.t: ########## @@ -0,0 +1,724 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +use t::APISIX 'no_plan'; + +no_long_string(); +no_root_location(); + +add_block_preprocessor(sub { + my ($block) = @_; + + if (! $block->request) { + $block->set_value("request", "GET /t"); + if (!$block->response_body) { + $block->set_value("response_body", "passed\n"); + } + } +}); + + +run_tests; + +__DATA__ + +=== TEST 1: mask query +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + local code, body = t('/apisix/admin/routes/1', + ngx.HTTP_PUT, + [[{ + "plugins": { + "data-mask": { + "request": [ + { + "action": "remove", + "name": "password", + "type": "query" + }, + { + "action": "replace", + "name": "token", + "type": "query", + "value": "*****" + }, + { + "action": "regex", + "name": "card", + "regex": "(\\d+)\\-\\d+\\-\\d+\\-(\\d+)", + "type": "query", + "value": "$1-****-****-$2" + } + ] + }, + "file-logger": { + "path": "mask-query.log.1" + } + }, + "upstream": { + "nodes": { + "127.0.0.1:1982": 1 + }, + "type": "roundrobin" + }, + "uri": "/hello" + }]] + ) + + if code >= 300 then + ngx.status = code + end + ngx.say(body) + } + } + + + +=== TEST 2: verify +--- config + location /t { + content_by_lua_block { + local core = require("apisix.core") + local t = require("lib.test_admin").test + + local code = t("/hello?password=abc&token=xyz&card=1234-1234-1234-1234", ngx.HTTP_GET) + local fd, err = io.open("mask-query.log.1", "r") + if not fd then + core.log.error("failed to open file: ", err) + return + end + local line = fd:read() + local log = core.json.decode(line) + + if log.request.querystring.password then + ngx.say("password arg mask failed: " .. log.request.querystring.password) + return + end + if log.request.querystring.token ~= "*****" then + ngx.say("token arg mask failed: " .. log.request.querystring.token) + return + end + if log.request.querystring.card ~= "1234-****-****-1234" then + ngx.say("card arg mask failed: " .. log.request.querystring.card) + return + end + if log.request.uri ~= "/hello?token=*****&card=1234-****-****-1234" and + log.request.uri ~= "/hello?card=1234-****-****-1234&token=*****" then + ngx.say("uri mask failed: " .. log.request.uri) + return + end + + os.remove("mask-query.log.1") + ngx.say("success") + } + } +--- response_body +success + + + +=== TEST 3: mask header +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + local code, body = t('/apisix/admin/routes/1', + ngx.HTTP_PUT, + [[{ + "plugins": { + "data-mask": { + "request": [ + { + "action": "remove", + "name": "password", + "type": "header" + }, + { + "action": "replace", + "name": "token", + "type": "header", + "value": "*****" + }, + { + "action": "regex", + "name": "card", + "regex": "(\\d+)\\-\\d+\\-\\d+\\-(\\d+)", + "type": "header", + "value": "$1-****-****-$2" + } + ] + }, + "file-logger": { + "path": "mask-header.log.2" + } + }, + "upstream": { + "nodes": { + "127.0.0.1:1982": 1 + }, + "type": "roundrobin" + }, + "uri": "/hello" + }]] + ) + + if code >= 300 then + ngx.status = code + end + ngx.say(body) + } + } + + + +=== TEST 4: verify +--- config + location /t { + content_by_lua_block { + local core = require("apisix.core") + local t = require("lib.test_admin").test + + local headers = {} + headers["password"] = "abc" + headers["token"] = "xyz" + headers["card"] = "1234-1234-1234-1234" + local code = t("/hello", ngx.HTTP_GET, "", nil, headers) + + local fd, err = io.open("mask-header.log.2", "r") + if not fd then + core.log.error("failed to open file: ", err) + return + end + local line = fd:read() + local log = core.json.decode(line) + + if log.request.headers.password then + ngx.say("password header mask failed: " .. log.request.headers.password) + return + end + if log.request.headers.token ~= "*****" then + ngx.say("token header mask failed: " .. log.request.headers.token) + return + end + if log.request.headers.card ~= "1234-****-****-1234" then + ngx.say("card header mask failed: " .. log.request.headers.card) + return + end + + os.remove("mask-header.log.2") + ngx.say("success") + } + } +--- response_body +success + + + +=== TEST 5: mask urlencoded body +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + local code, body = t('/apisix/admin/routes/1', + ngx.HTTP_PUT, + [[{ + "plugins": { + "data-mask": { + "request": [ + { + "action": "remove", + "body_format": "urlencoded", + "name": "password", + "type": "body" + }, + { + "action": "replace", + "body_format": "urlencoded", + "name": "token", + "type": "body", + "value": "*****" + }, + { + "action": "regex", + "body_format": "urlencoded", + "name": "card", + "regex": "(\\d+)\\-\\d+\\-\\d+\\-(\\d+)", + "type": "body", + "value": "$1-****-****-$2" + } + ] + }, + "file-logger": { + "include_req_body": true, + "path": "mask-urlencoded-body.log" + } + }, + "upstream": { + "nodes": { + "127.0.0.1:1982": 1 + }, + "type": "roundrobin" + }, + "uri": "/hello" + }]] + ) + + if code >= 300 then + ngx.status = code + end + ngx.say(body) + } + } + + + +=== TEST 6: verify +--- config + location /t { + content_by_lua_block { + local core = require("apisix.core") + local t = require("lib.test_admin").test + + local code = t("/hello", ngx.HTTP_POST, "password=abc&token=xyz&card=1234-1234-1234-1234") + + local fd, err = io.open("mask-urlencoded-body.log", "r") + if not fd then + core.log.error("failed to open file: ", err) + return + end + local line = fd:read() + local log = core.json.decode(line) + + if log.request.body ~= "token=*****&card=1234-****-****-1234" and + log.request.body ~= "card=1234-****-****-1234&token=*****" then + ngx.say("urlencoded body mask failed: " .. log.request.body) + return + end + + os.remove("mask-urlencoded-body.log") + ngx.say("success") + } + } +--- response_body +success + + + +=== TEST 7: mask json body +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + local code, body = t('/apisix/admin/routes/1', + ngx.HTTP_PUT, + [[{ + "plugins": { + "data-mask": { + "request": [ + { + "action": "remove", + "body_format": "json", + "name": "$.password", + "type": "body" + }, + { + "action": "replace", + "body_format": "json", + "name": "users[*].token", + "type": "body", + "value": "*****" + }, + { + "action": "regex", + "body_format": "json", + "name": "$.users[*].credit.card", + "regex": "(\\d+)\\-\\d+\\-\\d+\\-(\\d+)", + "type": "body", + "value": "$1-****-****-$2" + } + ] + }, + "file-logger": { + "include_req_body": true, + "path": "mask-json-body.log" + } + }, + "upstream": { + "nodes": { + "127.0.0.1:1982": 1 + }, + "type": "roundrobin" + }, + "uri": "/hello" + }]] + ) + + if code >= 300 then + ngx.status = code + end + ngx.say(body) + } + } + + + +=== TEST 8: verify +--- config + location /t { + content_by_lua_block { + local core = require("apisix.core") + local t = require("lib.test_admin").test + + local code = t("/hello", + ngx.HTTP_POST, + [[{ + "password": "abc", + "users": [ + { + "token": "xyz", + "credit": { + "card": "1234-1234-1234-1234" + } + }, + { + "token": "xyz", + "credit": { + "card": "1234-1234-1234-1234" + } + } + ] + }]] + ) + + local fd, err = io.open("mask-json-body.log", "r") + if not fd then + core.log.error("failed to open file: ", err) + return + end + local line = fd:read() + local log = core.json.decode(line) + + local body = core.json.decode(log.request.body) + if body.password then + ngx.say("$.password mask failed: " .. body.password) + return + end + for _, user in ipairs(body.users) do + if user.token ~= "*****" then + ngx.say("$.users[*].token mask failed: " .. user.token) + return + end + if user.credit.card ~= "1234-****-****-1234" then + ngx.say("$.users[*].credit.card mask failed: " .. user.credit.card) + return + end + end + + os.remove("mask-json-body.log") + ngx.say("success") + } + } +--- response_body +success + + + +=== TEST 9: plugin within global rule should not throw error for missing body. +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + local code, body = t('/apisix/admin/global_rules/1', + ngx.HTTP_PUT, + [[{ + "plugins": { + "data-mask": { + "request": [ + { + "action": "remove", + "name": "password", + "type": "query" + }, + { + "action": "replace", + "name": "token", + "type": "query", + "value": "*****" + }, + { + "action": "regex", + "name": "card", + "regex": "(\\d+)\\-\\d+\\-\\d+\\-(\\d+)", + "type": "query", + "value": "$1-****-****-$2" + } + ] + }, + "file-logger": { + "path": "mask-query.log.4" + } + } + }]] + ) + if code >= 300 then + ngx.status = code + end + ngx.say(body) + } + } + + + +=== TEST 10: verify +--- config + location /t { + content_by_lua_block { + local core = require("apisix.core") + local t = require("lib.test_admin").test + + local code = t("/random", ngx.HTTP_POST, "password=abc&token=xyz&card=1234-1234-1234-1234") + + ngx.say("code: ", code) + } + } +--- response_body +code: 404 +--- no_error_log +no request body found + + + +=== TEST 11: create plugin with default value for `max_req_post_args` +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + local code, body = t('/apisix/admin/routes/1', + ngx.HTTP_PUT, + [[{ + "plugins": { + "data-mask": { + "request": [ + { + "action": "regex", + "body_format": "urlencoded", + "name": "arg100", + "regex": "(\\d+)$", + "type": "body", + "value": "$1" + } + ] + }, + "file-logger": { + "include_req_body": true, + "path": "mask-urlencoded-body.log" + } + }, + "upstream": { + "nodes": { + "127.0.0.1:1982": 1 + }, + "type": "roundrobin" + }, + "uri": "/hello" + }]] + ) + + if code >= 300 then + ngx.status = code + end + ngx.say(body) + } + } + + + +=== TEST 12: verify default value for `max_req_post_args`` +--- config + location /t { + content_by_lua_block { + local core = require("apisix.core") + local t = require("lib.test_admin").test + + local url_encoded = "arg1=1" + for i = 2, 110, 1 do + url_encoded = url_encoded .. "&arg" .. i .. "=" .. i + end + + local code = t("/hello", ngx.HTTP_POST, url_encoded) + + local fd, err = io.open("mask-urlencoded-body.log", "r") + if not fd then + core.log.error("failed to open file: ", err) + return + end + local line = fd:read() + local log = core.json.decode(line) + local match100, err = ngx.re.match(log.request.body, "arg100=100") + local match101, err = ngx.re.match(log.request.body, "arg101=101") + os.remove("mask-urlencoded-body.log") + if match100 and not match101 then + ngx.say("success") + return + end + ngx.say(match) Review Comment: In the failure branch, this test prints `match` which is never defined (likely meant to print `match100/match101` or the request body). This makes debugging failures harder and can mask the real issue when the assertion fails. This issue also appears on line 655 of the same file. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
