This is an automated email from the ASF dual-hosted git repository.
nic-6443 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/apisix.git
The following commit(s) were added to refs/heads/master by this push:
new aa4472cab feat(ai-proxy-multi): let configured HTTP statuses trigger a
fallback (#13852)
aa4472cab is described below
commit aa4472cab41b98636dde4aa4bb75418bcc1470d4
Author: Nic <[email protected]>
AuthorDate: Thu Aug 20 11:13:16 2026 +0800
feat(ai-proxy-multi): let configured HTTP statuses trigger a fallback
(#13852)
---
apisix/plugins/ai-proxy-multi.lua | 3 +-
apisix/plugins/ai-proxy/base.lua | 24 +++-
apisix/plugins/ai-proxy/schema.lua | 21 ++++
docs/en/latest/plugins/ai-proxy-multi.md | 9 +-
docs/zh/latest/plugins/ai-proxy-multi.md | 5 +-
t/plugin/ai-proxy-multi-retry.t | 193 +++++++++++++++++++++++++++++++
6 files changed, 247 insertions(+), 8 deletions(-)
diff --git a/apisix/plugins/ai-proxy-multi.lua
b/apisix/plugins/ai-proxy-multi.lua
index ca92b2ab8..6ce28e793 100644
--- a/apisix/plugins/ai-proxy-multi.lua
+++ b/apisix/plugins/ai-proxy-multi.lua
@@ -934,7 +934,8 @@ local function retry_on_error(ctx, conf, code, body)
ctx.server_picker.after_balance(ctx, true)
if (code == 429 and fallback_strategy_has(conf.fallback_strategy,
"http_429")) or
(code >= 500 and code < 600 and
- fallback_strategy_has(conf.fallback_strategy, "http_5xx")) then
+ fallback_strategy_has(conf.fallback_strategy, "http_5xx")) or
+ base.is_fallback_http_status(conf, code) then
-- Slow-failure guard: only retry when the failed attempt finished
within
-- retry_on_failure_within_ms. A slow failure (e.g. a 5xx returned
after
-- minutes) is given back to the client directly, so fallback never
doubles
diff --git a/apisix/plugins/ai-proxy/base.lua b/apisix/plugins/ai-proxy/base.lua
index 5c9e9a5b3..db94afcda 100644
--- a/apisix/plugins/ai-proxy/base.lua
+++ b/apisix/plugins/ai-proxy/base.lua
@@ -21,6 +21,7 @@ local core = require("apisix.core")
local require = require
local pcall = pcall
local pairs = pairs
+local ipairs = ipairs
local type = type
local table = table
local math_floor = math.floor
@@ -48,6 +49,26 @@ local function count_request_tools(body)
end
+-- Statuses the user opted into retrying through fallback_http_statuses.
+-- Both the error path below and ai-proxy-multi's retry decision consult this,
+-- and they have to agree: a status that is not diverted to the error path has
+-- its response streamed to the client and never reaches the retry callback.
+function _M.is_fallback_http_status(conf, status)
+ local statuses = conf.fallback_http_statuses
+ if not statuses then
+ return false
+ end
+
+ for _, s in ipairs(statuses) do
+ if s == status then
+ return true
+ end
+ end
+
+ return false
+end
+
+
local function resolve_cap(cap_entry, key, conf, ctx)
local val = cap_entry and cap_entry[key]
if type(val) == "function" then
@@ -387,7 +408,8 @@ function _M.before_proxy(conf, ctx, on_error)
-- Upstream responded — mark source before any early returns
core.response.set_response_source(ctx, "upstream")
- if res.status == 429 or (res.status >= 500 and res.status < 600)
then
+ if res.status == 429 or (res.status >= 500 and res.status < 600)
+ or _M.is_fallback_http_status(conf, res.status) then
-- Read the upstream error body before closing so the
provider's
-- error details survive: logged on fallback (see
retry_on_error)
-- and returned to the client when no retry happens.
diff --git a/apisix/plugins/ai-proxy/schema.lua
b/apisix/plugins/ai-proxy/schema.lua
index ada2ab425..78787a77e 100644
--- a/apisix/plugins/ai-proxy/schema.lua
+++ b/apisix/plugins/ai-proxy/schema.lua
@@ -453,6 +453,27 @@ _M.ai_proxy_multi_schema = {
}
}
},
+ fallback_http_statuses = {
+ type = "array",
+ minItems = 1,
+ uniqueItems = true,
+ items = {
+ type = "integer",
+ minimum = 400,
+ maximum = 599,
+ },
+ description = "Additional upstream HTTP status codes that make the
"
+ .. "request fall back to another instance, on top of the "
+ .. "http_429 / http_5xx entries of fallback_strategy. Use it "
+ .. "for statuses that mean the instance's credential is "
+ .. "unusable rather than the request being wrong, e.g. [401, "
+ .. "402] when an API key is expired or out of quota. It is "
+ .. "opt-in per status because most 4xx responses are caused by
"
+ .. "the request itself and retrying them on another instance "
+ .. "would only burn quota. max_retries and "
+ .. "retry_on_failure_within_ms bound these retries the same "
+ .. "way they bound the fallback_strategy ones.",
+ },
max_retries = {
type = "integer",
minimum = 0,
diff --git a/docs/en/latest/plugins/ai-proxy-multi.md
b/docs/en/latest/plugins/ai-proxy-multi.md
index 4c5d77482..e04b327b4 100644
--- a/docs/en/latest/plugins/ai-proxy-multi.md
+++ b/docs/en/latest/plugins/ai-proxy-multi.md
@@ -68,8 +68,9 @@ When an instance's `provider` is set to `bedrock`, the Plugin
expects requests i
| Name | Type | Required | Default
| Valid values | Description |
|------------------------------------|----------------|----------|-----------------------------------|--------------|-------------|
| fallback_strategy | string or array | False | |
string: "instance_health_and_rate_limiting", "http_429", "http_5xx"<br />array:
["rate_limiting", "http_429", "http_5xx"] | Fallback strategy. When set, the
Plugin will check whether the specified instance's token has been exhausted
when a request is forwarded. If so, forward the request to the next instance
regardless of the instance priority. When not set, the Plugin will not forward
the request to low prior [...]
-| max_retries | integer | False |
| greater or equal to 0 | Maximum number of fallback
retries after the initial request fails. Bounds how many additional instances a
single request tries, so it does not exhaust every configured instance. Only
takes effect together with `fallback_strategy`. When unset, the Plugin retries
until an instance succeeds or all are tried. |
-| retry_on_failure_within_ms | integer | False |
| greater or equal to 1 | Only fall back to another
instance when the upstream fails within this many milliseconds. Fast failures
(such as connection errors or quick `429`/`5xx`) are retried, while a slow
failure that takes longer than this is returned to the client directly to avoid
doubling the wait time. Only takes effect together with `fallback_strategy`.
When unset, the Plugin retrie [...]
+| fallback_http_statuses | array[integer] | False |
| between 400 and 599 | Additional upstream HTTP status
codes that make the request fall back to another instance, on top of the
`http_429` and `http_5xx` entries of `fallback_strategy`. Use it for statuses
that mean the instance's credential is unusable rather than the request being
wrong, such as `[401, 402]` when an API key is expired or out of quota. It is
opt-in per status because mo [...]
+| max_retries | integer | False |
| greater or equal to 0 | Maximum number of fallback
retries after the initial request fails. Bounds how many additional instances a
single request tries, so it does not exhaust every configured instance. Only
takes effect together with `fallback_strategy` or `fallback_http_statuses`.
When unset, the Plugin retries until an instance succeeds or all are tried. |
+| retry_on_failure_within_ms | integer | False |
| greater or equal to 1 | Only fall back to another
instance when the upstream fails within this many milliseconds. Fast failures
(such as connection errors or quick `429`/`5xx`) are retried, while a slow
failure that takes longer than this is returned to the client directly to avoid
doubling the wait time. Only takes effect together with `fallback_strategy` or
`fallback_http_statuses`. W [...]
| balancer | object | False |
| | Load balancing configurations. |
| balancer.algorithm | string | False | roundrobin
| [roundrobin, chash, semantic] | Load balancing algorithm.
When set to `roundrobin`, weighted round robin algorithm is used. When set to
`chash`, consistent hashing algorithm is used. When set to `semantic`, the
Plugin picks an instance by the semantic similarity between the request prompt
and each instance's `examples`, and its options are configured under
`semantic_opts`. Note that `seman [...]
| balancer.hash_on | string | False |
| [vars, headers, cookie, consumer, vars_combinations] |
Used when `type` is `chash`. Support hashing on [NGINX
variables](https://nginx.org/en/docs/varindex.html), headers, cookie, consumer,
or a combination of [NGINX variables](https://nginx.org/en/docs/varindex.html).
|
@@ -173,8 +174,8 @@ The setting covers `ai-proxy`, `ai-proxy-multi`, and
`ai-request-rewrite`, which
When the selected LLM upstream returns a `429` or `5xx` status,
`ai-proxy-multi` reads the upstream error body before deciding whether to fall
back:
-- If the request is retried on another instance (per `fallback_strategy`,
`max_retries`, and `retry_on_failure_within_ms`), the failed instance's error
body is recorded in the error log for diagnostics, since a later attempt's
response is sent to the client instead.
-- If the request is not retried (no matching `fallback_strategy`, retries
exhausted, or the failure took longer than `retry_on_failure_within_ms`), the
upstream status code and error body are returned to the client, preserving the
upstream `Content-Type`.
+- If the request is retried on another instance (per `fallback_strategy`,
`fallback_http_statuses`, `max_retries`, and `retry_on_failure_within_ms`), the
failed instance's error body is recorded in the error log for diagnostics,
since a later attempt's response is sent to the client instead.
+- If the request is not retried (no matching `fallback_strategy` or
`fallback_http_statuses`, retries exhausted, or the failure took longer than
`retry_on_failure_within_ms`), the upstream status code and error body are
returned to the client, preserving the upstream `Content-Type`.
## Examples
diff --git a/docs/zh/latest/plugins/ai-proxy-multi.md
b/docs/zh/latest/plugins/ai-proxy-multi.md
index d056497bb..e5141d016 100644
--- a/docs/zh/latest/plugins/ai-proxy-multi.md
+++ b/docs/zh/latest/plugins/ai-proxy-multi.md
@@ -68,8 +68,9 @@ import TabItem from '@theme/TabItem';
| 名称 | 类型 | 必选项 | 默认值
| 有效值 | 描述 |
|------------------------------------|----------------|----------|-----------------------------------|--------------|-------------|
| fallback_strategy | string 或 array | 否 | |
string: "instance_health_and_rate_limiting", "http_429", "http_5xx"<br />array:
["rate_limiting", "http_429", "http_5xx"] |
故障转移策略。设置后，插件将在转发请求时检查指定实例的令牌是否已耗尽。如果是，则无论实例优先级如何，都将请求转发到下一个实例。未设置时，当高优先级实例的令牌耗尽时，插件不会将请求转发到低优先级实例。
|
-| max_retries | integer | 否 |
| 大于或等于 0 |
初始请求失败后允许的最大故障转移重试次数。用于限制单个请求最多尝试多少个额外实例，避免穷举所有已配置的实例。仅在配置 `fallback_strategy`
时生效。未设置时，插件会持续重试直到某个实例成功或所有实例都已尝试。 |
-| retry_on_failure_within_ms | integer | 否 |
| 大于或等于 1 | 仅当上游在指定毫秒数内失败时才故障转移到其他实例。快速失败（如连接错误、快速返回的
`429`/`5xx`）会触发重试，而耗时超过该值的慢失败会直接将错误返回给客户端，避免客户端等待时间翻倍。仅在配置 `fallback_strategy`
时生效。未设置时，插件无论失败请求耗时多久都会重试。 |
+| fallback_http_statuses | array[integer] | 否 |
| 400 到 599 之间 | 除 `fallback_strategy` 中的 `http_429` 和
`http_5xx` 之外，额外触发故障转移到其他实例的上游 HTTP 状态码。适用于表示该实例凭证不可用（而非请求本身有问题）的状态码，例如 API Key
过期或额度耗尽时的 `[401, 402]`。该能力按状态码显式启用，因为大多数 `4xx`
响应由请求本身导致，在其他实例上重试只会白白消耗额度。`max_retries` 和 `retry_on_failure_within_ms`
对这些重试的约束方式与对 `fallback_strategy` 的重试一致。 |
+| max_retries | integer | 否 |
| 大于或等于 0 |
初始请求失败后允许的最大故障转移重试次数。用于限制单个请求最多尝试多少个额外实例，避免穷举所有已配置的实例。仅在配置 `fallback_strategy`
或 `fallback_http_statuses` 时生效。未设置时，插件会持续重试直到某个实例成功或所有实例都已尝试。 |
+| retry_on_failure_within_ms | integer | 否 |
| 大于或等于 1 | 仅当上游在指定毫秒数内失败时才故障转移到其他实例。快速失败（如连接错误、快速返回的
`429`/`5xx`）会触发重试，而耗时超过该值的慢失败会直接将错误返回给客户端，避免客户端等待时间翻倍。仅在配置 `fallback_strategy`
或 `fallback_http_statuses` 时生效。未设置时，插件无论失败请求耗时多久都会重试。 |
| balancer | object | 否 |
| | 负载均衡配置。 |
| balancer.algorithm | string | 否 | roundrobin
| [roundrobin, chash, semantic] | 负载均衡算法。设置为 `roundrobin`
时，使用加权轮询算法。设置为 `chash` 时，使用一致性哈希算法。设置为 `semantic` 时，插件根据请求提示词与各实例 `examples`
之间的语义相似度选择实例，其相关选项配置在 `semantic_opts` 下。注意：`semantic` 不参与健康检查，也不参与
`fallback_strategy` / 重试——被选中实例的上游失败会直接返回给客户端；只有当没有实例达到其阈值或嵌入请求失败时，才会回退（回退到
`semantic_opts.fallback` 实例，若未配置则回退到第一个实例）。 |
| balancer.hash_on | string | 否 |
| [vars, headers, cookie, consumer, vars_combinations] | 当
`type` 为 `chash` 时使用。支持基于 [NGINX
变量](https://nginx.org/en/docs/varindex.html)、标头、cookie、消费者或 [NGINX
变量](https://nginx.org/en/docs/varindex.html)组合进行哈希。 |
diff --git a/t/plugin/ai-proxy-multi-retry.t b/t/plugin/ai-proxy-multi-retry.t
index a5f8d7cb2..845337e29 100644
--- a/t/plugin/ai-proxy-multi-retry.t
+++ b/t/plugin/ai-proxy-multi-retry.t
@@ -64,6 +64,32 @@ _EOC_
}
}
}
+ # Instances whose API key is rejected or out of quota. Neither status
+ # takes the 429/5xx error path, so they exercise
fallback_http_statuses.
+ server {
+ server_name unauthorized_instance;
+ default_type 'application/json';
+ listen 6735;
+ location / {
+ content_by_lua_block {
+ ngx.status = 401
+ ngx.say([[{ "error": {"message":"invalid api key"}}]])
+ return
+ }
+ }
+ }
+ server {
+ server_name payment_required_instance;
+ default_type 'application/json';
+ listen 6736;
+ location / {
+ content_by_lua_block {
+ ngx.status = 402
+ ngx.say([[{ "error": {"message":"insufficient balance"}}]])
+ return
+ }
+ }
+ }
server {
server_name success_instance;
default_type 'application/json';
@@ -307,3 +333,170 @@ model=upstream-model-B
no temperature leak
--- error_log
FALLBACKVARS request_llm_model=client-model llm_model=upstream-model-B
+
+
+
+=== TEST 9: fallback_http_statuses makes 401 and 402 fall back to another
instance
+--- config
+ location /t {
+ content_by_lua_block {
+ local t = require("lib.test_admin").test
+ local code, body = t('/apisix/admin/routes/1',
+ ngx.HTTP_PUT,
+ [[{
+ "uri": "/anything",
+ "plugins": {
+ "ai-proxy-multi": {
+ "fallback_http_statuses": [401, 402],
+ "instances": [
+
{"name":"expired-key","provider":"openai-compatible","weight":1,"priority":20,"auth":{"header":{"Authorization":"Bearer
token"}},"options":{"model":"gpt-4"},"override":{"endpoint":"http://127.0.0.1:6735"}},
+
{"name":"drained-key","provider":"openai-compatible","weight":1,"priority":10,"auth":{"header":{"Authorization":"Bearer
token"}},"options":{"model":"gpt-4"},"override":{"endpoint":"http://127.0.0.1:6736"}},
+
{"name":"good-key","provider":"openai-compatible","weight":1,"priority":0,"auth":{"header":{"Authorization":"Bearer
token"}},"options":{"model":"gpt-4"},"override":{"endpoint":"http://127.0.0.1:6733"}}
+ ],
+ "ssl_verify": false
+ }
+ }
+ }]]
+ )
+ if code >= 300 then
+ ngx.status = code
+ end
+ ngx.say(body)
+ }
+ }
+--- response_body
+passed
+
+
+
+=== TEST 10: the request is served by the instance with a working key
+--- request
+POST /anything
+{ "messages": [ { "role": "user", "content": "What is 1+1?"} ] }
+--- response_body chomp
+success
+--- error_code: 200
+--- error_log
+returned status 401, falling back to
+returned status 402, falling back to
+
+
+
+=== TEST 11: max_retries also bounds the fallback_http_statuses retries
+--- config
+ location /t {
+ content_by_lua_block {
+ local t = require("lib.test_admin").test
+ local code, body = t('/apisix/admin/routes/1',
+ ngx.HTTP_PUT,
+ [[{
+ "uri": "/anything",
+ "plugins": {
+ "ai-proxy-multi": {
+ "fallback_http_statuses": [401, 402],
+ "max_retries": 0,
+ "instances": [
+
{"name":"expired-key","provider":"openai-compatible","weight":1,"priority":20,"auth":{"header":{"Authorization":"Bearer
token"}},"options":{"model":"gpt-4"},"override":{"endpoint":"http://127.0.0.1:6735"}},
+
{"name":"good-key","provider":"openai-compatible","weight":1,"priority":0,"auth":{"header":{"Authorization":"Bearer
token"}},"options":{"model":"gpt-4"},"override":{"endpoint":"http://127.0.0.1:6733"}}
+ ],
+ "ssl_verify": false
+ }
+ }
+ }]]
+ )
+ if code >= 300 then
+ ngx.status = code
+ end
+ ngx.say(body)
+ }
+ }
+--- response_body
+passed
+
+
+
+=== TEST 12: the upstream status and error body reach the client once retries
are capped
+--- request
+POST /anything
+{ "messages": [ { "role": "user", "content": "What is 1+1?"} ] }
+--- error_code: 401
+--- response_body eval
+qr/invalid api key/
+--- error_log
+reached max_retries 0
+
+
+
+=== TEST 13: without fallback_http_statuses a 401 is still returned as is
+--- config
+ location /t {
+ content_by_lua_block {
+ local t = require("lib.test_admin").test
+ local code, body = t('/apisix/admin/routes/1',
+ ngx.HTTP_PUT,
+ [[{
+ "uri": "/anything",
+ "plugins": {
+ "ai-proxy-multi": {
+ "fallback_strategy": ["http_429", "http_5xx"],
+ "instances": [
+
{"name":"expired-key","provider":"openai-compatible","weight":1,"priority":20,"auth":{"header":{"Authorization":"Bearer
token"}},"options":{"model":"gpt-4"},"override":{"endpoint":"http://127.0.0.1:6735"}},
+
{"name":"good-key","provider":"openai-compatible","weight":1,"priority":0,"auth":{"header":{"Authorization":"Bearer
token"}},"options":{"model":"gpt-4"},"override":{"endpoint":"http://127.0.0.1:6733"}}
+ ],
+ "ssl_verify": false
+ }
+ }
+ }]]
+ )
+ if code >= 300 then
+ ngx.status = code
+ end
+ ngx.say(body)
+ }
+ }
+--- response_body
+passed
+
+
+
+=== TEST 14: the 401 is not retried and reaches the client
+--- request
+POST /anything
+{ "messages": [ { "role": "user", "content": "What is 1+1?"} ] }
+--- error_code: 401
+--- response_body eval
+qr/invalid api key/
+--- no_error_log
+falling back to
+
+
+
+=== TEST 15: fallback_http_statuses only accepts error statuses
+--- config
+ location /t {
+ content_by_lua_block {
+ local t = require("lib.test_admin").test
+ local code, body = t('/apisix/admin/routes/1',
+ ngx.HTTP_PUT,
+ [[{
+ "uri": "/anything",
+ "plugins": {
+ "ai-proxy-multi": {
+ "fallback_http_statuses": [200],
+ "instances": [
+
{"name":"good-key","provider":"openai-compatible","weight":1,"auth":{"header":{"Authorization":"Bearer
token"}},"options":{"model":"gpt-4"},"override":{"endpoint":"http://127.0.0.1:6733"}}
+ ],
+ "ssl_verify": false
+ }
+ }
+ }]]
+ )
+ if code >= 300 then
+ ngx.status = code
+ end
+ ngx.say(body)
+ }
+ }
+--- error_code: 400
+--- response_body eval
+qr/expected 200 to be at least 400/
