This is an automated email from the ASF dual-hosted git repository.
nic-6443 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/apisix.git
The following commit(s) were added to refs/heads/master by this push:
new 5c5e1b2cb5 fix(ai-proxy-multi): keep the client request body intact
across fallback retries (#13793)
5c5e1b2cb5 is described below
commit 5c5e1b2cb54564d817cbd27f18b223678a792a39
Author: Mohammad Izzraff Janius
<[email protected]>
AuthorDate: Wed Aug 12 10:03:35 2026 +0800
fix(ai-proxy-multi): keep the client request body intact across fallback
retries (#13793)
---
apisix/plugins/ai-proxy/base.lua | 6 +--
t/plugin/ai-proxy-multi-retry.t | 90 ++++++++++++++++++++++++++++++++++++++++
2 files changed, 92 insertions(+), 4 deletions(-)
diff --git a/apisix/plugins/ai-proxy/base.lua b/apisix/plugins/ai-proxy/base.lua
index fa815f430f..5c9e9a5b31 100644
--- a/apisix/plugins/ai-proxy/base.lua
+++ b/apisix/plugins/ai-proxy/base.lua
@@ -187,6 +187,8 @@ function _M.before_proxy(conf, ctx, on_error)
return 400, err
end
+ request_body = core.table.deepcopy(request_body)
+
local extra_opts = {
name = ai_instance.name,
endpoint = ai_instance._resolved_endpoint
@@ -242,10 +244,6 @@ function _M.before_proxy(conf, ctx, on_error)
-- Step 2: Extract model from request
local request_model = request_body.model
-
- if request_model then
- ctx.var.request_llm_model = request_model
- end
local model = ai_instance.options and ai_instance.options.model or
request_model
if model then
ctx.var.llm_model = model
diff --git a/t/plugin/ai-proxy-multi-retry.t b/t/plugin/ai-proxy-multi-retry.t
index e09bfc3400..a5f8d7cb2f 100644
--- a/t/plugin/ai-proxy-multi-retry.t
+++ b/t/plugin/ai-proxy-multi-retry.t
@@ -34,6 +34,7 @@ add_block_preprocessor(sub {
plugins:
- ai-proxy-multi
- prometheus
+ - serverless-post-function
_EOC_
$block->set_value("extra_yaml_config", $user_yaml_config);
@@ -75,6 +76,33 @@ _EOC_
}
}
}
+ # Upstream that echoes the request body it receives inside a
well-formed
+ # chat completion so the test can assert exactly what was forwarded to
+ # the fallback instance.
+ server {
+ server_name echo_instance;
+ default_type 'application/json';
+ listen 6734;
+ location / {
+ content_by_lua_block {
+ local json = require("cjson.safe")
+ ngx.req.read_body()
+ local raw = ngx.req.get_body_data() or ""
+ ngx.status = 200
+ ngx.say(json.encode({
+ id = "chatcmpl-echo",
+ object = "chat.completion",
+ model = "echo",
+ choices = {{
+ index = 0,
+ message = { role = "assistant", content = raw },
+ finish_reason = "stop",
+ }},
+ usage = { prompt_tokens = 1, completion_tokens = 1,
total_tokens = 2 },
+ }))
+ }
+ }
+ }
_EOC_
$block->set_value("http_config", $http_config);
@@ -217,3 +245,65 @@ POST /anything
--- response_body_like: slow internal error
--- error_log
exceeding retry_on_failure_within_ms 200
+
+
+
+=== TEST 7: fallback preserves the client body: set up asymmetric instances
+--- config
+ location /t {
+ content_by_lua_block {
+ local t = require("lib.test_admin").test
+ local code, body = t('/apisix/admin/routes/1',
+ ngx.HTTP_PUT,
+ [[{
+ "uri": "/anything",
+ "plugins": {
+ "ai-proxy-multi": {
+ "fallback_strategy": ["http_5xx"],
+ "instances": [
+
{"name":"err-a","provider":"openai-compatible","weight":1,"priority":10,"auth":{"header":{"Authorization":"Bearer
token"}},"options":{"model":"upstream-model-A","temperature":0.9},"override":{"endpoint":"http://127.0.0.1:6731"}},
+
{"name":"echo-b","provider":"openai-compatible","weight":1,"priority":0,"auth":{"header":{"Authorization":"Bearer
token"}},"options":{"model":"upstream-model-B"},"override":{"endpoint":"http://127.0.0.1:6734"}}
+ ],
+ "ssl_verify": false
+ },
+ "serverless-post-function": {
+ "phase": "log",
+ "functions": ["return function(conf, ctx)
ngx.log(ngx.WARN, \"FALLBACKVARS request_llm_model=\",
tostring(ctx.var.request_llm_model), \" llm_model=\",
tostring(ctx.var.llm_model)) end"]
+ }
+ }
+ }]]
+ )
+ if code >= 300 then
+ ngx.status = code
+ end
+ ngx.say(body)
+ }
+ }
+--- response_body
+passed
+
+
+
+=== TEST 8: retry keeps the original model in vars and does not leak instance
A options into instance B
+--- config
+ location /t {
+ content_by_lua_block {
+ local http = require("resty.http").new()
+ local cjson = require("cjson.safe")
+ local res = assert(http:request_uri(
+ "http://127.0.0.1:"; .. ngx.var.server_port .. "/anything", {
+ method = "POST",
+ body = '{ "model": "client-model", "messages": [ { "role":
"user", "content": "What is 1+1?"} ] }',
+ headers = { ["Content-Type"] = "application/json" },
+ }))
+ local completion = cjson.decode(res.body)
+ local forwarded =
cjson.decode(completion.choices[1].message.content)
+ ngx.say("model=", forwarded.model)
+ ngx.say(forwarded.temperature == nil and "no temperature leak" or
"temperature leaked")
+ }
+ }
+--- response_body
+model=upstream-model-B
+no temperature leak
+--- error_log
+FALLBACKVARS request_llm_model=client-model llm_model=upstream-model-B
