This is an automated email from the ASF dual-hosted git repository.
shreemaan-abhishek pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/apisix.git
The following commit(s) were added to refs/heads/master by this push:
new a242282958 fix(ai-content-moderation): moderate the OpenAI `developer`
role (#13780)
a242282958 is described below
commit a242282958aadaaba6b02400b1314655872a0f4f
Author: Shreemaan Abhishek <[email protected]>
AuthorDate: Mon Aug 10 17:51:19 2026 +0800
fix(ai-content-moderation): moderate the OpenAI `developer` role (#13780)
---
apisix/plugins/ai-protocols/openai-chat.lua | 18 +-
apisix/plugins/ai-protocols/openai-responses.lua | 16 +-
.../latest/plugins/ai-aliyun-content-moderation.md | 2 +-
.../en/latest/plugins/ai-aws-content-moderation.md | 2 +-
.../latest/plugins/ai-aliyun-content-moderation.md | 2 +-
.../zh/latest/plugins/ai-aws-content-moderation.md | 2 +-
t/plugin/ai-aliyun-content-moderation.t | 245 +++++++++++++++++++++
t/plugin/ai-aws-content-moderation.t | 106 +++++++++
8 files changed, 382 insertions(+), 11 deletions(-)
diff --git a/apisix/plugins/ai-protocols/openai-chat.lua
b/apisix/plugins/ai-protocols/openai-chat.lua
index 827512e8b3..4ac1cf0c76 100644
--- a/apisix/plugins/ai-protocols/openai-chat.lua
+++ b/apisix/plugins/ai-protocols/openai-chat.lua
@@ -248,6 +248,15 @@ function _M.extract_request_content(body)
end
+-- Roles carrying the system prompt. `developer` is what OpenAI renamed
`system`
+-- to on o1 and later models; both land in the same prompt slot, so the two are
+-- extracted together and the `system` role selector covers both.
+local SYSTEM_ROLES = {
+ system = true,
+ developer = true,
+}
+
+
local function is_turn_role(message, roles)
return type(message) == "table" and message.role ~= nil and
roles[message.role]
end
@@ -289,14 +298,15 @@ function _M.extract_turn_content(body, mode, roles)
end
--- Extract system-role text for request moderation. Unlike turn content, the
--- system prompt is checked on every request (it can be poisoned by malicious
--- ToolCall arguments), so the last-turn rule does not apply here.
+-- Extract system-role text (see SYSTEM_ROLES) for request moderation. Unlike
+-- turn content, the system prompt is checked on every request (it can be
+-- poisoned by malicious ToolCall arguments), so the last-turn rule does not
+-- apply here.
function _M.extract_system_content(body)
local contents = {}
if type(body.messages) == "table" then
for _, message in ipairs(body.messages) do
- if type(message) == "table" and message.role == "system" then
+ if type(message) == "table" and SYSTEM_ROLES[message.role] then
append_message_text(contents, message)
end
end
diff --git a/apisix/plugins/ai-protocols/openai-responses.lua
b/apisix/plugins/ai-protocols/openai-responses.lua
index 32c6325def..cd38b22963 100644
--- a/apisix/plugins/ai-protocols/openai-responses.lua
+++ b/apisix/plugins/ai-protocols/openai-responses.lua
@@ -247,6 +247,15 @@ function _M.extract_request_content(body)
end
+-- Roles carrying the system prompt. `developer` is what OpenAI renamed
`system`
+-- to on o1 and later models; both land in the same prompt slot, so the two are
+-- extracted together and the `system` role selector covers both.
+local SYSTEM_ROLES = {
+ system = true,
+ developer = true,
+}
+
+
-- Whether an input item belongs to a selected turn role. A bare string is user
-- text; a role item matches when its role is in `roles`; a Responses-API tool
-- result (`function_call_output`, which has no role) matches when tool is
selected.
@@ -306,8 +315,9 @@ function _M.extract_turn_content(body, mode, roles)
end
--- Extract system-role text for request moderation. Responses API carries the
--- system prompt in `instructions`; an `input` array may also hold system
items.
+-- Extract system-role text (see SYSTEM_ROLES) for request moderation.
Responses
+-- API carries the system prompt in `instructions`; an `input` array may also
+-- hold system items.
function _M.extract_system_content(body)
local contents = {}
if type(body.instructions) == "string" then
@@ -315,7 +325,7 @@ function _M.extract_system_content(body)
end
if type(body.input) == "table" then
for _, item in ipairs(body.input) do
- if type(item) == "table" and item.role == "system" then
+ if type(item) == "table" and SYSTEM_ROLES[item.role] then
append_item_text(contents, item)
end
end
diff --git a/docs/en/latest/plugins/ai-aliyun-content-moderation.md
b/docs/en/latest/plugins/ai-aliyun-content-moderation.md
index 89c0f20c62..7019149edf 100644
--- a/docs/en/latest/plugins/ai-aliyun-content-moderation.md
+++ b/docs/en/latest/plugins/ai-aliyun-content-moderation.md
@@ -58,7 +58,7 @@ The `ai-aliyun-content-moderation` Plugin should be used with
either [`ai-proxy`
| stream_check_mode | string | False | `"final_packet"` | `realtime`,
`final_packet` | Streaming moderation mode. `realtime`: batched checks during
streaming. `final_packet`: append risk level at the end. |
| stream_check_cache_size | integer | False | `128` | >= 1 | Maximum bytes per
moderation batch in `realtime` mode. Length is measured using Lua string
length, so for UTF-8 text non-ASCII characters may consume multiple bytes. |
| stream_check_interval | number | False | `3` | >= 0.1 | Seconds between
batch checks in `realtime` mode. |
-| request_check_roles | array[string] | False | `["user"]` | items are `user`,
`tool`, `system` | Which message roles to moderate on the request side. `user`
and `tool` follow `request_check_mode`; `system` is checked on every request
(it can be poisoned by malicious ToolCall arguments overwriting the system
prompt). The default `["user"]` preserves the previous behavior. Note:
tool-result moderation applies to OpenAI-compatible formats where the tool
output is a distinct `tool` role/ite [...]
+| request_check_roles | array[string] | False | `["user"]` | items are `user`,
`tool`, `system` | Which message roles to moderate on the request side. `user`
and `tool` follow `request_check_mode`; `system` is checked on every request
(it can be poisoned by malicious ToolCall arguments overwriting the system
prompt) and also covers OpenAI's `developer` role, which replaces `system` on
newer models. The default `["user"]` preserves the previous behavior. Note:
tool-result moderation appli [...]
| request_check_mode | string | False | `"last"` | `last`, `all` | Which
user/tool messages to moderate. `last`: only the latest consecutive block of
selected-role messages (the newest turn). `all`: every selected-role message.
Does not apply to `system`, which is always moderated when enabled via
`request_check_roles`. |
| request_check_service | string | False | `"llm_query_moderation"` | | Aliyun
service for request moderation. |
| request_check_length_limit | number | False | `2000` | >= 1 | Request
content length limit. If exceeded, the content is sent to Aliyun in chunks. For
instance, if the request content is 250 characters and
`request_check_length_limit` is set to `100`, the content is sent in 3 requests
to Aliyun. |
diff --git a/docs/en/latest/plugins/ai-aws-content-moderation.md
b/docs/en/latest/plugins/ai-aws-content-moderation.md
index 8d830548bf..f44632c94c 100644
--- a/docs/en/latest/plugins/ai-aws-content-moderation.md
+++ b/docs/en/latest/plugins/ai-aws-content-moderation.md
@@ -62,7 +62,7 @@ The `ai-aws-content-moderation` Plugin should be used with
either [`ai-proxy`](.
| `moderation_threshold` | number | False | 0.5 | 0 - 1 | Overall toxicity
threshold. A higher value means more toxic content allowed. This option differs
from the individual category thresholds in `moderation_categories`. For
example, if `moderation_categories` is set with a `PROFANITY` threshold of
`0.5`, and a request has a `PROFANITY` score of `0.1`, the request will not
exceed the category threshold. However, if the request has other categories
like `SEXUAL` or `VIOLENCE_OR_THREAT` [...]
| `check_request` | boolean | False | `true` | | If `true`, moderate the
request content. |
| `check_response` | boolean | False | `false` | | If `true`, moderate the LLM
response content. |
-| `request_check_roles` | array[string] | False |
`["user","tool","system","assistant"]` | items are `user`, `tool`, `system`,
`assistant` | Which message roles to moderate on the request side. `user`,
`tool` and `assistant` follow `request_check_mode`; `system` is checked on
every request (it can be poisoned by malicious ToolCall arguments overwriting
the system prompt). `assistant` messages in a request are client-supplied
history rather than the model's own output, so they are moderat [...]
+| `request_check_roles` | array[string] | False |
`["user","tool","system","assistant"]` | items are `user`, `tool`, `system`,
`assistant` | Which message roles to moderate on the request side. `user`,
`tool` and `assistant` follow `request_check_mode`; `system` is checked on
every request (it can be poisoned by malicious ToolCall arguments overwriting
the system prompt) and also covers OpenAI's `developer` role, which replaces
`system` on newer models. `assistant` messages in a request [...]
| `request_check_mode` | string | False | `all` | `last`, `all` | Which
user/tool/assistant messages to moderate. `last`: only the latest consecutive
block of selected-role messages (the newest turn). `all`: every selected-role
message. Does not apply to `system`, which is always moderated when enabled via
`request_check_roles`. Note that `last` combined with `assistant` widens the
block rather than narrowing it, because assistant turns no longer end it — drop
`assistant` from `request_c [...]
| `request_check_length_limit` | integer | False | `1000` | [4, 1024] |
Maximum bytes of request content per Comprehend text segment. Longer content is
split on character boundaries into several segments, which are then batched
into as few Comprehend calls as possible. The upper bound is AWS Comprehend's 1
KB per-segment limit. |
| `response_check_length_limit` | integer | False | `1000` | [4, 1024] |
Maximum bytes of response content per Comprehend text segment. Longer content
is split on character boundaries into several segments, which are then batched
into as few Comprehend calls as possible. The upper bound is AWS Comprehend's 1
KB per-segment limit. |
diff --git a/docs/zh/latest/plugins/ai-aliyun-content-moderation.md
b/docs/zh/latest/plugins/ai-aliyun-content-moderation.md
index ae420dbc1f..5b68b94be6 100644
--- a/docs/zh/latest/plugins/ai-aliyun-content-moderation.md
+++ b/docs/zh/latest/plugins/ai-aliyun-content-moderation.md
@@ -58,7 +58,7 @@ import TabItem from '@theme/TabItem';
| stream_check_mode | string | 否 | `"final_packet"` |
`realtime`、`final_packet` |
流式审核模式。`realtime`：流式传输期间批量检查。`final_packet`：在最后附加风险等级。 |
| stream_check_cache_size | integer | 否 | `128` | >= 1 | `realtime`
模式下每次审核批次的最大字节数（按 UTF-8 编码后的字节长度计算）。 |
| stream_check_interval | number | 否 | `3` | >= 0.1 | `realtime`
模式下批次检查之间的间隔秒数。 |
-| request_check_roles | array[string] | 否 | `["user"]` | 取值为
`user`、`tool`、`system` | 请求侧审核哪些消息角色。`user` 与 `tool` 遵循
`request_check_mode`；`system` 每次请求都审核（其可能被恶意 ToolCall 参数覆盖篡改）。默认 `["user"]`
保持既有行为。注意：tool 结果审核适用于 OpenAI 兼容格式（tool 输出为独立的 `tool` 角色/项）；Anthropic、Bedrock 的
tool 结果以嵌套 block 形式存在于 user 消息中，其内容不会被抽取。 |
+| request_check_roles | array[string] | 否 | `["user"]` | 取值为
`user`、`tool`、`system` | 请求侧审核哪些消息角色。`user` 与 `tool` 遵循
`request_check_mode`；`system` 每次请求都审核（其可能被恶意 ToolCall 参数覆盖篡改），并且同时覆盖 OpenAI 的
`developer` 角色（新模型上用于替代 `system`）。默认 `["user"]` 保持既有行为。注意：tool 结果审核适用于 OpenAI
兼容格式（tool 输出为独立的 `tool` 角色/项）；Anthropic、Bedrock 的 tool 结果以嵌套 block 形式存在于 user
消息中，其内容不会被抽取。 |
| request_check_mode | string | 否 | `"last"` | `last`, `all` | 审核哪些 user/tool
消息。`last`：仅审核最后一段连续的所选角色消息（最新一轮）；`all`：审核所有所选角色消息。不作用于 `system`——只要通过
`request_check_roles` 启用，`system` 每次都审核。 |
| request_check_service | string | 否 | `"llm_query_moderation"` | |
用于请求审核的阿里云服务。 |
| request_check_length_limit | number | 否 | `2000` | >= 1 |
请求内容长度上限。如果超过该限制，内容将分块发送到阿里云。例如，如果请求内容为 250 个字符，且 `request_check_length_limit`
设置为 `100`，则内容将分 3 次请求发送到阿里云。 |
diff --git a/docs/zh/latest/plugins/ai-aws-content-moderation.md
b/docs/zh/latest/plugins/ai-aws-content-moderation.md
index 4c90914ee5..d85fc82012 100644
--- a/docs/zh/latest/plugins/ai-aws-content-moderation.md
+++ b/docs/zh/latest/plugins/ai-aws-content-moderation.md
@@ -57,7 +57,7 @@ import TabItem from '@theme/TabItem';
| `moderation_categories` | object | 否 | | | 审核类别及其对应阈值的键值对。在每个键值对中，键应为
`PROFANITY`、`HATE_SPEECH`、`INSULT`、`HARASSMENT_OR_ABUSE`、`SEXUAL` 或
`VIOLENCE_OR_THREAT` 之一；阈值应在 0 到 1 之间（包含）。 |
| `moderation_threshold` | number | 否 | 0.5 | 0 - 1 |
整体毒性阈值。值越高，允许的有害内容越多。此选项与 `moderation_categories` 中的单独类别阈值不同。例如，如果
`moderation_categories` 中设置了 `PROFANITY` 阈值为 `0.5`，而请求的 `PROFANITY` 分数为
`0.1`，则请求不会超过类别阈值。但如果请求的其他类别（如 `SEXUAL` 或 `VIOLENCE_OR_THREAT`）超过了
`moderation_threshold`，则请求将被拒绝。 |
| `check_request` | boolean | 否 | `true` | | 如果为 `true`，则审核请求内容。 |
-| `request_check_roles` | array[string] | 否 |
`["user","tool","system","assistant"]` | 取值为 `user`、`tool`、`system`、`assistant`
| 请求侧审核哪些消息角色。`user`、`tool` 与 `assistant` 遵循 `request_check_mode`；`system`
每次请求都审核（其可能被恶意 ToolCall 参数覆盖篡改）。请求中的 `assistant`
消息由客户端提供，而非模型自身的输出，因此默认也会被审核。注意：tool 结果审核适用于 OpenAI 兼容格式（tool 输出为独立的 `tool`
角色/项）；Anthropic、Bedrock 的 tool 结果以嵌套 block 形式存在于 user 消息中，其内容不会被抽取。 |
+| `request_check_roles` | array[string] | 否 |
`["user","tool","system","assistant"]` | 取值为 `user`、`tool`、`system`、`assistant`
| 请求侧审核哪些消息角色。`user`、`tool` 与 `assistant` 遵循 `request_check_mode`；`system`
每次请求都审核（其可能被恶意 ToolCall 参数覆盖篡改），并且同时覆盖 OpenAI 的 `developer` 角色（新模型上用于替代
`system`）。请求中的 `assistant` 消息由客户端提供，而非模型自身的输出，因此默认也会被审核。注意：tool 结果审核适用于 OpenAI
兼容格式（tool 输出为独立的 `tool` 角色/项）；Anthropic、Bedrock 的 tool 结果以嵌套 block 形式存在于 user
消息中，其内容不会被抽取。 |
| `request_check_mode` | string | 否 | `all` | `last`、`all` | 审核哪些
user/tool/assistant 消息。`last`：仅审核最后一段连续的所选角色消息（最新一轮）；`all`：审核所有所选角色消息。不作用于
`system`——只要通过 `request_check_roles` 启用，`system` 每次都审核。注意：`last` 与 `assistant`
同时使用会扩大而非缩小审核范围，因为 assistant 消息不再中断该连续块；若只想审核最新一轮，请从 `request_check_roles` 中移除
`assistant`。 |
| `deny_code` | integer | 否 | `200` | [200, 599] | 请求被拒绝时返回的 HTTP 状态码。默认为
`200`，使兼容 provider 的拒绝响应在客户端 SDK 中被解析为正常补全；设置为 4xx 可将拒绝暴露为 HTTP 错误。 |
| `deny_message` | string | 否 | | | 请求被拒绝时返回的消息。未设置时，返回审核原因（例如 `request body
exceeds toxicity threshold`）。 |
diff --git a/t/plugin/ai-aliyun-content-moderation.t
b/t/plugin/ai-aliyun-content-moderation.t
index e12ff01cac..604fe8861b 100644
--- a/t/plugin/ai-aliyun-content-moderation.t
+++ b/t/plugin/ai-aliyun-content-moderation.t
@@ -2462,3 +2462,248 @@ qr/event: message_stop/
qr/execute content moderation/
--- grep_error_log_out
execute content moderation
+
+
+
+=== TEST 77: openai-chat extract_system_content also collects the developer
role
+--- config
+ location /t {
+ content_by_lua_block {
+ local proto = require("apisix.plugins.ai-protocols.openai-chat")
+ local body = {
+ messages = {
+ { role = "developer", content = "dev" },
+ { role = "system", content = "sys" },
+ { role = "user", content = "u1" },
+ }
+ }
+ ngx.say("system:",
table.concat(proto.extract_system_content(body), ","))
+ ngx.say("turn:", table.concat(
+ proto.extract_turn_content(body, "all", { user = true,
tool = true }), ","))
+ }
+ }
+--- response_body
+system:dev,sys
+turn:u1
+
+
+
+=== TEST 78: openai-responses extract_system_content also collects the
developer role
+--- config
+ location /t {
+ content_by_lua_block {
+ local proto =
require("apisix.plugins.ai-protocols.openai-responses")
+ local body = {
+ instructions = "instr",
+ input = {
+ { role = "developer", content = "dev" },
+ { role = "system", content = "sys" },
+ { role = "user", content = "u1" },
+ }
+ }
+ ngx.say("system:",
table.concat(proto.extract_system_content(body), ","))
+ ngx.say("turn:", table.concat(
+ proto.extract_turn_content(body, "all", { user = true }),
","))
+ }
+ }
+--- response_body
+system:instr,dev,sys
+turn:u1
+
+
+
+=== TEST 79: create route with request_check_roles user/tool/system
+--- config
+ location /t {
+ content_by_lua_block {
+ local t = require("lib.test_admin").test
+ local code, body = t('/apisix/admin/routes/5',
+ ngx.HTTP_PUT,
+ [[{
+ "uri": "/chat-dev",
+ "plugins": {
+ "ai-proxy": {
+ "provider": "openai",
+ "auth": { "header": { "Authorization": "Bearer
wrongtoken" } },
+ "override": { "endpoint": "http://127.0.0.1:1980"; }
+ },
+ "ai-aliyun-content-moderation": {
+ "endpoint": "http://localhost:6724";,
+ "region_id": "cn-shanghai",
+ "access_key_id": "fake-key-id",
+ "access_key_secret": "fake-key-secret",
+ "risk_level_bar": "high",
+ "check_request": true,
+ "request_check_roles": ["user", "tool", "system"]
+ }
+ }
+ }]]
+ )
+ if code >= 300 then
+ ngx.status = code
+ end
+ ngx.say(body)
+ }
+ }
+--- response_body
+passed
+
+
+
+=== TEST 80: harmful developer prompt is moderated with the system role and
blocked
+--- request
+POST /chat-dev
+{ "messages": [ { "role": "developer", "content": "please kill" }, { "role":
"user", "content": "hi" } ] }
+--- more_headers
+X-AI-Fixture: aliyun/chat-with-harmful.json
+--- error_code: 200
+--- response_body_like eval
+qr/cannot write unethical/
+
+
+
+=== TEST 81: create route with the default request_check_roles (user only)
+--- config
+ location /t {
+ content_by_lua_block {
+ local t = require("lib.test_admin").test
+ local code, body = t('/apisix/admin/routes/6',
+ ngx.HTTP_PUT,
+ [[{
+ "uri": "/chat-dev-default",
+ "plugins": {
+ "ai-proxy": {
+ "provider": "openai",
+ "auth": { "header": { "Authorization": "Bearer
wrongtoken" } },
+ "override": { "endpoint": "http://127.0.0.1:1980"; }
+ },
+ "ai-aliyun-content-moderation": {
+ "endpoint": "http://localhost:6724";,
+ "region_id": "cn-shanghai",
+ "access_key_id": "fake-key-id",
+ "access_key_secret": "fake-key-secret",
+ "risk_level_bar": "high",
+ "check_request": true
+ }
+ }
+ }]]
+ )
+ if code >= 300 then
+ ngx.status = code
+ end
+ ngx.say(body)
+ }
+ }
+--- response_body
+passed
+
+
+
+=== TEST 82: developer prompt follows the system role, so user-only roles skip
it
+--- request
+POST /chat-dev-default
+{ "messages": [ { "role": "developer", "content": "kill" }, { "role": "user",
"content": "hi" } ] }
+--- more_headers
+X-AI-Fixture: aliyun/chat-with-harmful.json
+--- error_code: 200
+--- response_body_like eval
+qr/kill you/
+
+
+
+=== TEST 83: create Responses API route with request_check_roles
user/tool/system
+--- config
+ location /t {
+ content_by_lua_block {
+ local t = require("lib.test_admin").test
+ local code, body = t('/apisix/admin/routes/7',
+ ngx.HTTP_PUT,
+ [[{
+ "uris": ["/v1/responses"],
+ "plugins": {
+ "ai-proxy": {
+ "provider": "openai",
+ "auth": { "header": { "Authorization": "Bearer
wrongtoken" } },
+ "override": { "endpoint": "http://127.0.0.1:1980"; }
+ },
+ "ai-aliyun-content-moderation": {
+ "endpoint": "http://localhost:6724";,
+ "region_id": "cn-shanghai",
+ "access_key_id": "fake-key-id",
+ "access_key_secret": "fake-key-secret",
+ "risk_level_bar": "high",
+ "check_request": true,
+ "request_check_roles": ["user", "tool", "system"]
+ }
+ }
+ }]]
+ )
+ if code >= 300 then
+ ngx.status = code
+ end
+ ngx.say(body)
+ }
+ }
+--- response_body
+passed
+
+
+
+=== TEST 84: Responses API developer item is moderated with the system role
and blocked
+--- request
+POST /v1/responses
+{ "model": "gpt-4o", "input": [ { "role": "developer", "content": "please
kill" }, { "role": "user", "content": "hi" } ] }
+--- more_headers
+X-AI-Fixture: aliyun/chat-with-harmful.json
+--- error_code: 200
+--- response_body_like eval
+qr/cannot write unethical/
+
+
+
+=== TEST 85: create Responses API route with the default request_check_roles
(user only)
+--- config
+ location /t {
+ content_by_lua_block {
+ local t = require("lib.test_admin").test
+ local code, body = t('/apisix/admin/routes/7',
+ ngx.HTTP_PUT,
+ [[{
+ "uris": ["/v1/responses"],
+ "plugins": {
+ "ai-proxy": {
+ "provider": "openai",
+ "auth": { "header": { "Authorization": "Bearer
wrongtoken" } },
+ "override": { "endpoint": "http://127.0.0.1:1980"; }
+ },
+ "ai-aliyun-content-moderation": {
+ "endpoint": "http://localhost:6724";,
+ "region_id": "cn-shanghai",
+ "access_key_id": "fake-key-id",
+ "access_key_secret": "fake-key-secret",
+ "risk_level_bar": "high",
+ "check_request": true
+ }
+ }
+ }]]
+ )
+ if code >= 300 then
+ ngx.status = code
+ end
+ ngx.say(body)
+ }
+ }
+--- response_body
+passed
+
+
+
+=== TEST 86: Responses API developer item follows system, so user-only roles
skip it
+--- request
+POST /v1/responses
+{ "model": "gpt-4o", "input": [ { "role": "developer", "content": "please
kill" }, { "role": "user", "content": "hi" } ] }
+--- more_headers
+X-AI-Fixture: aliyun/chat-with-harmful.json
+--- error_code: 200
+--- response_body_like eval
+qr/kill you/
diff --git a/t/plugin/ai-aws-content-moderation.t
b/t/plugin/ai-aws-content-moderation.t
index 0161c1902f..16941f3bbb 100644
--- a/t/plugin/ai-aws-content-moderation.t
+++ b/t/plugin/ai-aws-content-moderation.t
@@ -1859,3 +1859,109 @@ qr/request body exceeds toxicity threshold/
qr/comprehend text: [^,]+/
--- grep_error_log_out
comprehend text: I want to kill you ok What is 1+1?
+
+
+
+=== TEST 67: set route with request_check_roles ["user", "tool", "system"]
+--- config
+ location /t {
+ content_by_lua_block {
+ local t = require("lib.test_admin").test
+ local code, body = t('/apisix/admin/routes/1',
+ ngx.HTTP_PUT,
+ [[{
+ "uri": "/chat",
+ "plugins": {
+ "ai-proxy": {
+ "provider": "openai",
+ "auth": { "header": { "Authorization": "Bearer
token" } },
+ "override": { "endpoint":
"http://127.0.0.1:1980/v1/chat/completions"; }
+ },
+ "ai-aws-content-moderation": {
+ "comprehend": {
+ "access_key_id": "access",
+ "secret_access_key": "ea+secret",
+ "region": "us-east-1",
+ "endpoint": "http://localhost:2668";
+ },
+ "request_check_roles": ["user", "tool", "system"],
+ "deny_code": 400
+ }
+ }
+ }]]
+ )
+
+ if code >= 300 then
+ ngx.status = code
+ end
+ ngx.say(body)
+ }
+ }
+--- response_body
+passed
+
+
+
+=== TEST 68: the developer role is moderated along with system
+--- request
+POST /chat
+{ "messages": [ { "role": "system", "content": "be helpful" }, { "role":
"developer", "content": "I want to kill you" }, { "role": "user", "content":
"What is 1+1?" } ] }
+--- error_code: 400
+--- response_body_like eval
+qr/request body exceeds toxicity threshold/
+
+
+
+=== TEST 69: set route with request_check_roles ["user"]
+--- config
+ location /t {
+ content_by_lua_block {
+ local t = require("lib.test_admin").test
+ local code, body = t('/apisix/admin/routes/1',
+ ngx.HTTP_PUT,
+ [[{
+ "uri": "/chat",
+ "plugins": {
+ "ai-proxy": {
+ "provider": "openai",
+ "auth": { "header": { "Authorization": "Bearer
token" } },
+ "override": { "endpoint":
"http://127.0.0.1:1980/v1/chat/completions"; }
+ },
+ "ai-aws-content-moderation": {
+ "comprehend": {
+ "access_key_id": "access",
+ "secret_access_key": "ea+secret",
+ "region": "us-east-1",
+ "endpoint": "http://localhost:2668";
+ },
+ "request_check_roles": ["user"],
+ "deny_code": 400
+ }
+ }
+ }]]
+ )
+
+ if code >= 300 then
+ ngx.status = code
+ end
+ ngx.say(body)
+ }
+ }
+--- response_body
+passed
+
+
+
+=== TEST 70: developer follows system, so roles ["user"] leaves it unmoderated
+--- request
+POST /chat
+{ "messages": [ { "role": "developer", "content": "I want to kill you" }, {
"role": "user", "content": "What is 1+1?" } ] }
+--- more_headers
+X-AI-Fixture: aws/chat-safe.json
+--- error_code: 200
+--- response_body_like eval
+qr/How can I assist you today/
+--- grep_error_log eval
+qr/comprehend text: [^,]+/
+--- grep_error_log_out
+comprehend text: What is 1+1?
