This is an automated email from the ASF dual-hosted git repository.
AlinsRan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/apisix.git
The following commit(s) were added to refs/heads/master by this push:
new fbf78b020 feat(ai-proxy-multi): add semantic load-balancing algorithm
(#13676)
fbf78b020 is described below
commit fbf78b0203f39f30039c4083712f57103cfb2f4c
Author: AlinsRan <[email protected]>
AuthorDate: Tue Jul 28 08:54:08 2026 +0800
feat(ai-proxy-multi): add semantic load-balancing algorithm (#13676)
---
apisix/plugins/ai-proxy-multi.lua | 275 ++++++++-
apisix/plugins/ai-proxy/embedding.lua | 156 +++++
apisix/plugins/ai-proxy/schema.lua | 116 +++-
apisix/plugins/ai-proxy/semantic.lua | 83 +++
docs/en/latest/plugins/ai-proxy-multi.md | 157 ++++-
docs/zh/latest/plugins/ai-proxy-multi.md | 157 ++++-
t/plugin/ai-proxy-semantic-routing.t | 997 +++++++++++++++++++++++++++++++
t/plugin/ai-proxy-semantic-schema.t | 470 +++++++++++++++
t/plugin/ai-proxy-semantic.t | 67 +++
9 files changed, 2472 insertions(+), 6 deletions(-)
diff --git a/apisix/plugins/ai-proxy-multi.lua
b/apisix/plugins/ai-proxy-multi.lua
index 9b8f2eab5..2746a9357 100644
--- a/apisix/plugins/ai-proxy-multi.lua
+++ b/apisix/plugins/ai-proxy-multi.lua
@@ -33,12 +33,18 @@ local ngx_now = ngx.now
local require = require
local pcall = pcall
+local error = error
+local tostring = tostring
local ipairs = ipairs
local type = type
local string = string
+local sub = string.sub
local url = require("socket.url")
local priority_balancer = require("apisix.balancer.priority")
+local semantic = require("apisix.plugins.ai-proxy.semantic")
+local embedding = require("apisix.plugins.ai-proxy.embedding")
+local ai_protocols = require("apisix.plugins.ai-protocols")
local endpoint_regex = "^(https?)://([^:/]+):?(%d*)/?.*$"
local pickers = {}
@@ -48,6 +54,19 @@ local lrucache_server_picker = core.lrucache.new({
local lrucache_health_status = core.lrucache.new({
ttl = 300, count = 256
})
+-- Keyed by route + conf version, so config changes invalidate immediately;
+-- the long ttl just avoids re-embedding references on unchanged config.
+local lrucache_semantic_vectors = core.lrucache.new({
+ ttl = 3600, count = 256
+})
+-- The prompt is sent verbatim to a third-party embedding endpoint. Bound it: a
+-- request body may be up to max_req_body_size (64MB by default), and an
oversized
+-- input would blow the embedding model's token limit, 400, and silently push
every
+-- large prompt to the fallback. OpenAI-compatible embedding models (which
LiteLLM
+-- and others target) cap input at ~8192 tokens; we bound by bytes since
counting
+-- tokens needs a tokenizer, using ~2 bytes/token so the cap stays under the
token
+-- limit even for multi-byte scripts while keeping the whole routing prompt.
+local MAX_EMBED_PROMPT_BYTES = 16384
local plugin_name = "ai-proxy-multi"
local _M = {
@@ -156,7 +175,72 @@ function _M.check_schema(conf)
end
end
- return ok
+ if algo == "semantic" then
+ local semantic_opts = conf.semantic_opts
+ if not semantic_opts or not semantic_opts.embeddings then
+ return false, "must configure `semantic_opts.embeddings` when
balancer " ..
+ "algorithm is semantic"
+ end
+ local embeddings = semantic_opts.embeddings
+ -- Same scheme+host check the instance endpoints get: without it an
+ -- endpoint like "host/path" parses to a nil host and every request
would
+ -- silently fail open, i.e. the route would never work with no signal.
+ local eendpoint = embeddings.endpoint
+ if eendpoint then
+ local scheme, host = eendpoint:match(endpoint_regex)
+ if not scheme or not host then
+ return false, "invalid `semantic_opts.embeddings.endpoint`"
+ end
+ end
+ if embeddings.provider == "azure-openai" then
+ -- Azure carries the deployment in the URL and declares no default
host
+ -- or path, so the endpoint must be the full embeddings URL.
Without a
+ -- path every request would silently fail open to the fallback.
+ if not eendpoint then
+ return false, "must configure
`semantic_opts.embeddings.endpoint` " ..
+ "when embeddings provider is azure-openai"
+ end
+ local parsed = url.parse(eendpoint)
+ local epath = parsed and parsed.path
+ if not epath or epath == "" or epath == "/" then
+ return false, "`semantic_opts.embeddings.endpoint` for
azure-openai " ..
+ "must include the full deployment path, e.g. " ..
+ "https://{resource}.openai.azure.com"; ..
+
"/openai/deployments/{deployment}/embeddings?api-version=..."
+ end
+ end
+ -- Every instance must declare non-empty `examples`. The `fallback`
+ -- instance is a normal ranked instance too -- it competes on
similarity
+ -- like the rest and only additionally serves as the target when
nothing
+ -- clears its threshold -- so it is not exempt. A named fallback must
point
+ -- at an instance that actually exists.
+ local fallback = semantic_opts.fallback
+ local fallback_found = false
+ for _, instance in ipairs(conf.instances) do
+ if fallback and instance.name == fallback then
+ fallback_found = true
+ end
+ local has_example = false
+ if instance.examples then
+ for _, ex in ipairs(instance.examples) do
+ if type(ex) == "string" and ex ~= "" then
+ has_example = true
+ break
+ end
+ end
+ end
+ if not has_example then
+ return false, "instance '" .. (instance.name or "?") ..
+ "': must configure non-empty `examples` for the semantic
algorithm"
+ end
+ end
+ if fallback and not fallback_found then
+ return false, "`semantic_opts.fallback` names unknown instance '"
..
+ fallback .. "'"
+ end
+ end
+
+ return true
end
@@ -598,9 +682,196 @@ local function pick_target(ctx, conf, ups_tab)
end
+local function extract_last_user_message(ctx)
+ local body = core.request.get_json_request_body_table()
+ if not body then
+ return nil
+ end
+ -- Normalize through the protocol adapter instead of reading body.messages
+ -- directly, so the prompt is found for every supported client protocol:
+ -- OpenAI Responses carries it in body.input, Anthropic/Bedrock in
structured
+ -- content blocks. get_messages returns canonical {role, content} entries.
+ local messages = ai_protocols.get_messages(body, ctx)
+ for i = #messages, 1, -1 do
+ local m = messages[i]
+ if type(m) == "table" and m.role == "user" then
+ local content = m.content
+ if type(content) == "string" then
+ if content ~= "" then
+ return content
+ end
+ elseif type(content) == "table" then
+ -- Some adapters return multimodal content unflattened
+ -- ({type=text|image_url,...}); concatenate the text parts so
+ -- routing still works.
+ local parts = {}
+ for _, p in ipairs(content) do
+ if type(p) == "table" and p.type == "text"
+ and type(p.text) == "string" then
+ parts[#parts + 1] = p.text
+ end
+ end
+ if #parts > 0 then
+ return table_concat(parts, " ")
+ end
+ end
+ end
+ end
+ return nil
+end
+
+
+-- Embed every instance's examples in one batch and group the normalized
+-- reference vectors by instance name. Raises on embedding failure so the
+-- lrucache below does not cache a bad result.
+local function build_instance_vectors(conf)
+ local texts = {}
+ local owners = {}
+ for _, inst in ipairs(conf.instances) do
+ if inst.examples then
+ for _, ex in ipairs(inst.examples) do
+ texts[#texts + 1] = ex
+ owners[#texts] = inst.name
+ end
+ end
+ end
+
+ local vecs, err = embedding.fetch(conf.semantic_opts.embeddings, texts)
+ if not vecs then
+ error("failed to fetch reference embeddings: " .. tostring(err))
+ end
+
+ local by_instance = {}
+ for i, v in ipairs(vecs) do
+ local name = owners[i]
+ if name then
+ by_instance[name] = by_instance[name] or {}
+ core.table.insert(by_instance[name], semantic.normalize(v))
+ end
+ end
+ return by_instance
+end
+
+
+-- Guaranteed fallback: the instance named by semantic_opts.fallback if
+-- configured, else the first instance. Never fails, so a request always has a
+-- target.
+local function semantic_fallback(conf)
+ local fallback = conf.semantic_opts and conf.semantic_opts.fallback
+ if fallback then
+ for _, inst in ipairs(conf.instances) do
+ if inst.name == fallback then
+ return inst.name, inst
+ end
+ end
+ end
+ local inst = conf.instances[1]
+ return inst.name, inst
+end
+
+
+local function pick_semantic_instance(ctx, conf)
+ local version = plugin.conf_version(conf)
+ local ok, by_instance = pcall(lrucache_semantic_vectors,
+ ctx.matched_route.key .. "#semantic",
version,
+ build_instance_vectors, conf)
+ if not ok or not by_instance then
+ core.log.warn("semantic routing: ", by_instance, ", falling back")
+ return semantic_fallback(conf)
+ end
+
+ local prompt = extract_last_user_message(ctx)
+ if not prompt then
+ core.log.warn("semantic routing: no user message found, falling back")
+ return semantic_fallback(conf)
+ end
+
+ if #prompt > MAX_EMBED_PROMPT_BYTES then
+ prompt = sub(prompt, 1, MAX_EMBED_PROMPT_BYTES)
+ end
+
+ -- pcall, like the reference path above: the embedding response is
+ -- provider-controlled, so a raise here must fall back rather than 500.
+ local fetched, qvecs, err = pcall(embedding.fetch,
conf.semantic_opts.embeddings,
+ { prompt })
+ if not fetched then
+ core.log.warn("semantic routing: query embedding error: ", qvecs, ",
falling back")
+ return semantic_fallback(conf)
+ end
+ if not qvecs or not qvecs[1] then
+ core.log.warn("semantic routing: query embedding failed: ", err, ",
falling back")
+ return semantic_fallback(conf)
+ end
+ local qvec = semantic.normalize(qvecs[1])
+ local qdim = #qvec
+
+ local ranked = {}
+ for _, inst in ipairs(conf.instances) do
+ local refs = by_instance[inst.name]
+ if refs then
+ local scores = {}
+ for _, rv in ipairs(refs) do
+ -- guard against dimension drift (e.g. embedding model
changed):
+ -- mismatched vectors would make dot() error, so fail open
instead.
+ if #rv ~= qdim then
+ core.log.warn("semantic routing: embedding dimension
mismatch ",
+ "(query ", qdim, " vs reference ", #rv, "),
falling back")
+ return semantic_fallback(conf)
+ end
+ scores[#scores + 1] = semantic.dot(qvec, rv)
+ end
+ core.table.insert(ranked, {
+ name = inst.name,
+ score = semantic.max(scores),
+ })
+ end
+ end
+ core.table.sort(ranked, function(a, b) return a.score > b.score end)
+
+ local debugging = conf.semantic_opts.debugging
+ if debugging then
+ local parts = {}
+ for _, c in ipairs(ranked) do
+ parts[#parts + 1] = c.name .. ":" .. string.format("%.4f", c.score)
+ end
+ core.response.set_header("X-AI-Semantic-Scores", table_concat(parts,
","))
+ end
+
+ -- Highest score first; pick the first instance that clears its own
threshold
+ -- (per-instance override, else the global semantic_opts.threshold).
+ for _, cand in ipairs(ranked) do
+ local inst = get_instance_conf(conf.instances, cand.name)
+ local thr = inst.threshold or conf.semantic_opts.threshold or 0
+ if cand.score >= thr then
+ if debugging then
+ core.response.set_header("X-AI-Semantic-Picked-Instance",
cand.name)
+ end
+ core.log.info("semantic routing picked instance: ", cand.name,
+ ", score: ", cand.score)
+ return cand.name, inst
+ end
+ end
+
+ if debugging then
+ core.response.set_header("X-AI-Semantic-Picked-Instance", "fallback")
+ end
+ -- Only on the fallback path: surface why nothing matched, without
requiring
+ -- debugging. Cheap, because this runs once per unmatched request.
+ local unmatched = {}
+ for _, c in ipairs(ranked) do
+ unmatched[#unmatched + 1] = c.name .. ":" .. string.format("%.4f",
c.score)
+ end
+ core.log.warn("semantic routing: no instance cleared threshold (scores: ",
+ table_concat(unmatched, ","), "), falling back")
+ return semantic_fallback(conf)
+end
+
+
local function pick_ai_instance(ctx, conf, ups_tab)
local instance_name, instance_conf, err
- if #conf.instances == 1 then
+ if conf.balancer and conf.balancer.algorithm == "semantic" then
+ instance_name, instance_conf = pick_semantic_instance(ctx, conf)
+ elseif #conf.instances == 1 then
instance_name = conf.instances[1].name
instance_conf = conf.instances[1]
else
diff --git a/apisix/plugins/ai-proxy/embedding.lua
b/apisix/plugins/ai-proxy/embedding.lua
new file mode 100644
index 000000000..ca09336ae
--- /dev/null
+++ b/apisix/plugins/ai-proxy/embedding.lua
@@ -0,0 +1,156 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one or more
+-- contributor license agreements. See the NOTICE file distributed with
+-- this work for additional information regarding copyright ownership.
+-- The ASF licenses this file to You under the Apache License, Version 2.0
+-- (the "License"); you may not use this file except in compliance with
+-- the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
+-- Batch embedding client for the semantic balancer. Routes the request through
+-- the shared ai-provider layer (apisix.plugins.ai-providers.*) so endpoint
+-- resolution, authentication and HTTP transport match the chat path, instead
of
+-- re-implementing them here. One call embeds a list of texts
(OpenAI-compatible
+-- /embeddings: input is an array), returning a vector per text. Keeps semantic
+-- routing free of a vector database.
+local core = require("apisix.core")
+local ipairs = ipairs
+local type = type
+local tostring = tostring
+local require = require
+local pcall = pcall
+local INF = math.huge
+
+local _M = {}
+
+local EMBEDDINGS_PROTOCOL = "openai-embeddings"
+
+
+-- The provider's embeddings capability supplies a default path when it has one
+-- (openai: /v1/embeddings). Providers whose endpoint always carries its own
path
+-- (azure-openai) declare none, and build_request then requires `endpoint` to
give
+-- it. Which providers are allowed is gated by the `embeddings.provider` enum,
so
+-- a missing capability is not an error here.
+local function embeddings_target_path(ai_provider)
+ local cap = ai_provider.capabilities and
ai_provider.capabilities[EMBEDDINGS_PROTOCOL]
+ if not cap then
+ return nil
+ end
+ local path = cap.path
+ if type(path) == "function" then
+ path = path()
+ end
+ return path
+end
+
+
+-- Fetch embeddings for `texts` in a single batch request.
+-- Returns an array of vectors index-aligned with `texts`, or nil + err.
+function _M.fetch(conf, texts)
+ if not texts or #texts == 0 then
+ return {}
+ end
+
+ if not conf.model or conf.model == "" then
+ return nil, "embedding model is not configured"
+ end
+
+ local ok, ai_provider = pcall(require, "apisix.plugins.ai-providers." ..
conf.provider)
+ if not ok then
+ return nil, "failed to load ai-provider: " .. tostring(conf.provider)
+ end
+
+ local target_path = embeddings_target_path(ai_provider)
+
+ -- The embedding call is a self-contained sidecar request. It sets none of
the
+ -- opts.client_* fields, so build_request forwards none of the client's
headers
+ -- (Authorization, Cookie, ...) or query to the embedding provider, and
the pure
+ -- request client always sends our { model, input } body rather than the
+ -- client's request body.
+ local extra_opts = {
+ endpoint = conf.endpoint,
+ auth = conf.auth,
+ target_path = target_path,
+ }
+ local req_conf = {
+ ssl_verify = conf.ssl_verify ~= false,
+ timeout = conf.timeout or 3000,
+ keepalive = true,
+ }
+
+ local status, raw_body, err = ai_provider:request(req_conf,
+ { model = conf.model, input = texts }, extra_opts)
+ -- The upstream body never reaches a log line or a returned error. It is
+ -- untrusted third-party content that may echo the prompt back, be
arbitrarily
+ -- large, or forge log lines with newlines. Report the status and a
bounded,
+ -- sanitized summary instead.
+ if status ~= 200 then
+ if err then
+ return nil, "embedding request failed: " .. err
+ end
+ return nil, "embedding endpoint returned status " .. tostring(status)
+ end
+
+ -- A 200 body that decodes to a scalar, boolean or null must not be indexed
+ -- (`data.data` on a number raises). `derr` is cjson's own parse error,
which
+ -- is bounded and carries no body content.
+ local data, derr = core.json.decode(raw_body)
+ if type(data) ~= "table" or type(data.data) ~= "table" then
+ return nil, "invalid embedding response: " .. (derr or "unexpected
shape")
+ end
+
+ local vectors = {}
+ for i, item in ipairs(data.data) do
+ -- `data` may be an array of non-objects; indexing those would raise
and
+ -- turn a sidecar failure into a 500, so check before touching a field.
+ if type(item) ~= "table" then
+ return nil, "invalid embedding entry at index " .. (i - 1)
+ end
+ -- OpenAI returns an `index`; fall back to positional order. It must
be a
+ -- usable table key: cjson decodes JSON `NaN` to a number, and `t[NaN]`
+ -- raises. A float or out-of-range index is equally unusable.
+ local idx = i
+ local raw_idx = item.index
+ if type(raw_idx) == "number" then
+ if raw_idx ~= raw_idx or raw_idx < 0 or raw_idx >= #texts
+ or raw_idx % 1 ~= 0
+ then
+ return nil, "invalid embedding index at entry " .. (i - 1)
+ end
+ idx = raw_idx + 1
+ end
+ local emb = item.embedding
+ if type(emb) ~= "table" or #emb == 0 then
+ return nil, "invalid embedding vector at index " .. (idx - 1)
+ end
+ for _, n in ipairs(emb) do
+ -- reject non-finite components: cjson decodes `1e999` to inf,
which
+ -- would normalize to NaN and silently poison every similarity
score.
+ if type(n) ~= "number" or n ~= n or n == INF or n == -INF then
+ return nil, "non-numeric embedding component at index " ..
(idx - 1)
+ end
+ end
+ vectors[idx] = emb
+ end
+
+ -- Every input position must have produced a vector; a hole means a
+ -- dropped/duplicated index we must not silently route on. Checking each
+ -- slot is reliable where `#vectors` is not on a sparse table.
+ for i = 1, #texts do
+ if vectors[i] == nil then
+ return nil, "missing embedding for input index " .. (i - 1)
+ end
+ end
+ return vectors
+end
+
+
+return _M
diff --git a/apisix/plugins/ai-proxy/schema.lua
b/apisix/plugins/ai-proxy/schema.lua
index a7833f0c1..d5f13751d 100644
--- a/apisix/plugins/ai-proxy/schema.lua
+++ b/apisix/plugins/ai-proxy/schema.lua
@@ -202,12 +202,113 @@ local ai_instance_schema = {
active = schema_def.health_checker_active,
},
required = {"active"}
- }
+ },
+ examples = {
+ type = "array",
+ minItems = 1,
+ maxItems = 64,
+ items = { type = "string", minLength = 1 },
+ description = "Example utterances representing this instance's
"
+ .. "intent; each is embedded into its own reference vector
"
+ .. "for the semantic algorithm. Required for every
instance "
+ .. "when the balancer algorithm is semantic, including the
one "
+ .. "named by semantic_opts.fallback.",
+ },
+ threshold = {
+ type = "number",
+ minimum = -1,
+ maximum = 1,
+ description = "Per-instance minimum cosine similarity for the "
+ .. "semantic algorithm; overrides
semantic_opts.threshold.",
+ },
},
required = {"name", "provider", "auth", "weight"},
},
}
+local embeddings_schema = {
+ type = "object",
+ properties = {
+ provider = {
+ type = "string",
+ enum = { "openai", "azure-openai" },
+ description = "Embedding provider used by the semantic algorithm.",
+ },
+ model = {
+ type = "string",
+ minLength = 1,
+ description = "Embedding model name, e.g. text-embedding-3-small.",
+ },
+ endpoint = {
+ type = "string",
+ description = "Embedding API endpoint. Optional for openai
(defaults "
+ .. "to the public API). Required for azure-openai, where it
must "
+ .. "be the full URL, e.g. https://{resource}.openai.azure.com";
+ ..
"/openai/deployments/{deployment}/embeddings?api-version=...",
+ },
+ auth = {
+ type = "object",
+ properties = {
+ header = { type = "object", additionalProperties = { type =
"string" } },
+ query = { type = "object", additionalProperties = { type =
"string" } },
+ },
+ -- header/query only. The sidecar call runs on a throwaway ctx, so
the
+ -- ctx-dependent schemes (gcp, aws) cannot work here; reject them
at
+ -- config time instead of failing open on every request.
+ additionalProperties = false,
+ },
+ timeout = {
+ type = "integer",
+ minimum = 1,
+ default = 3000,
+ description = "Embedding request timeout in milliseconds. The
query "
+ .. "prompt is embedded synchronously on every request, so this
"
+ .. "bounds the latency added to each request when the
embedding "
+ .. "endpoint is slow or down (the request then fails open to
the "
+ .. "fallback).",
+ },
+ ssl_verify = { type = "boolean", default = true },
+ },
+ required = { "provider", "model", "auth" },
+}
+
+-- All options specific to the semantic balancer live here, so they are grouped
+-- in one place instead of being scattered across balancer/instances/top-level.
+local semantic_opts_schema = {
+ type = "object",
+ properties = {
+ embeddings = embeddings_schema,
+ threshold = {
+ type = "number",
+ minimum = -1,
+ maximum = 1,
+ default = 0,
+ description = "Global minimum cosine similarity for the semantic "
+ .. "algorithm. When no instance clears its threshold the
request "
+ .. "falls back to the `fallback` instance, or the first
instance "
+ .. "if none is named. The default of 0 admits essentially any "
+ .. "prompt, so the fallback only ever runs if a threshold
above 0 "
+ .. "is set.",
+ },
+ fallback = {
+ type = "string",
+ minLength = 1,
+ description = "Name of the instance to route to when no instance "
+ .. "clears its threshold or the embedding request fails. It is
"
+ .. "otherwise a normal ranked instance and needs `examples`
like "
+ .. "the rest. Defaults to the first instance when unset.",
+ },
+ debugging = {
+ type = "boolean",
+ default = false,
+ description = "When true, the semantic algorithm exposes
per-instance "
+ .. "scores and the routing decision via X-AI-Semantic-*
response "
+ .. "headers, for debugging.",
+ },
+ },
+ required = { "embeddings" },
+}
+
local logging_schema = {
type = "object",
properties = {
@@ -303,7 +404,15 @@ _M.ai_proxy_multi_schema = {
properties = {
algorithm = {
type = "string",
- enum = { "chash", "roundrobin" },
+ enum = { "chash", "roundrobin", "semantic" },
+ description = "Load-balancing algorithm. Note: 'semantic' "
+ .. "picks an instance by prompt similarity and does
not "
+ .. "participate in health checks or fallback_strategy
/ "
+ .. "retry — an upstream failure on the chosen instance
is "
+ .. "returned to the client. It only falls back (to the
"
+ .. "semantic_opts.fallback, else the first instance)
when "
+ .. "no instance clears its threshold or embedding
fails. "
+ .. "Configure it under `semantic_opts`.",
},
hash_on = {
type = "string",
@@ -324,6 +433,7 @@ _M.ai_proxy_multi_schema = {
default = { algorithm = "roundrobin" }
},
instances = ai_instance_schema,
+ semantic_opts = semantic_opts_schema,
logging = logging_schema,
fallback_strategy = {
anyOf = {
@@ -419,6 +529,8 @@ _M.ai_proxy_multi_schema = {
"instances.auth.gcp.service_account_json",
"instances.auth.aws.secret_access_key",
"instances.auth.aws.session_token",
+ "semantic_opts.embeddings.auth.header",
+ "semantic_opts.embeddings.auth.query",
},
}
diff --git a/apisix/plugins/ai-proxy/semantic.lua
b/apisix/plugins/ai-proxy/semantic.lua
new file mode 100644
index 000000000..c58c8629d
--- /dev/null
+++ b/apisix/plugins/ai-proxy/semantic.lua
@@ -0,0 +1,83 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one or more
+-- contributor license agreements. See the NOTICE file distributed with
+-- this work for additional information regarding copyright ownership.
+-- The ASF licenses this file to You under the Apache License, Version 2.0
+-- (the "License"); you may not use this file except in compliance with
+-- the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
+-- Pure-lua vector math for semantic routing: no apisix.core dependency so it
can
+-- be unit-tested standalone. Vectors are plain arrays of numbers.
+local sqrt = math.sqrt
+
+local _M = {}
+
+
+-- Return the unit vector of `vec`. A zero vector normalizes to zeros (no NaN).
+function _M.normalize(vec)
+ local sum = 0
+ for i = 1, #vec do
+ sum = sum + vec[i] * vec[i]
+ end
+ local norm = sqrt(sum)
+ local out = {}
+ if norm == 0 then
+ for i = 1, #vec do
+ out[i] = 0
+ end
+ return out
+ end
+ for i = 1, #vec do
+ out[i] = vec[i] / norm
+ end
+ return out
+end
+
+
+-- Dot product. On pre-normalized vectors this equals cosine similarity.
+function _M.dot(a, b)
+ local sum = 0
+ for i = 1, #a do
+ sum = sum + a[i] * b[i]
+ end
+ return sum
+end
+
+
+-- Cosine similarity in [-1, 1]. Normalizes both inputs; for hot paths prefer
+-- pre-normalizing once and calling dot() directly.
+function _M.cosine(a, b)
+ return _M.dot(_M.normalize(a), _M.normalize(b))
+end
+
+
+-- Score an instance from its per-example similarities.
+-- `examples` are alternative phrasings of one intent, so the question is
whether
+-- ANY of them matches: take the best. Averaging would dilute a dead-on example
+-- with the instance's broader ones, and would make adding an example lower the
+-- instance's score.
+function _M.max(scores)
+ local n = #scores
+ if n == 0 then
+ return nil
+ end
+ local m = scores[1]
+ for i = 2, n do
+ if scores[i] > m then
+ m = scores[i]
+ end
+ end
+ return m
+end
+
+
+return _M
diff --git a/docs/en/latest/plugins/ai-proxy-multi.md
b/docs/en/latest/plugins/ai-proxy-multi.md
index 88a1274ef..dc58ca565 100644
--- a/docs/en/latest/plugins/ai-proxy-multi.md
+++ b/docs/en/latest/plugins/ai-proxy-multi.md
@@ -71,7 +71,7 @@ When an instance's `provider` is set to `bedrock`, the Plugin
expects requests i
| max_retries | integer | False |
| greater or equal to 0 | Maximum number of fallback
retries after the initial request fails. Bounds how many additional instances a
single request tries, so it does not exhaust every configured instance. Only
takes effect together with `fallback_strategy`. When unset, the Plugin retries
until an instance succeeds or all are tried. |
| retry_on_failure_within_ms | integer | False |
| greater or equal to 1 | Only fall back to another
instance when the upstream fails within this many milliseconds. Fast failures
(such as connection errors or quick `429`/`5xx`) are retried, while a slow
failure that takes longer than this is returned to the client directly to avoid
doubling the wait time. Only takes effect together with `fallback_strategy`.
When unset, the Plugin retrie [...]
| balancer | object | False |
| | Load balancing configurations. |
-| balancer.algorithm | string | False | roundrobin
| [roundrobin, chash] | Load balancing algorithm. When set
to `roundrobin`, weighted round robin algorithm is used. When set to `chash`,
consistent hashing algorithm is used. |
+| balancer.algorithm | string | False | roundrobin
| [roundrobin, chash, semantic] | Load balancing algorithm.
When set to `roundrobin`, weighted round robin algorithm is used. When set to
`chash`, consistent hashing algorithm is used. When set to `semantic`, the
Plugin picks an instance by the semantic similarity between the request prompt
and each instance's `examples`, and its options are configured under
`semantic_opts`. Note that `seman [...]
| balancer.hash_on | string | False |
| [vars, headers, cookie, consumer, vars_combinations] |
Used when `type` is `chash`. Support hashing on [NGINX
variables](https://nginx.org/en/docs/varindex.html), headers, cookie, consumer,
or a combination of [NGINX variables](https://nginx.org/en/docs/varindex.html).
|
| balancer.key | string | False |
| | Used when `type` is `chash`. When
`hash_on` is set to `header` or `cookie`, `key` is required. When `hash_on` is
set to `consumer`, `key` is not required as the consumer name will be used as
the key automatically. |
| instances | array[object] | True |
| | LLM instance configurations. |
@@ -95,6 +95,8 @@ When an instance's `provider` is set to `bedrock`, the Plugin
expects requests i
| instances.auth.aws.session_token | string | False |
| minLength = 1 | AWS session token for temporary
credentials (e.g., from STS assume-role). Encrypted at rest. |
| instances.options | object | False |
| | Model configurations. In addition to
`model`, you can configure additional parameters and they will be forwarded to
the upstream LLM service in the request body. For instance, if you are working
with OpenAI, DeepSeek, or AIMLAPI, you can configure additional parameters such
as `max_tokens`, `temperature`, `top_p`, and `stream`. See your LLM provider's
API documentation f [...]
| instances.options.model | string | False |
| | Name of the LLM model, such as `gpt-4`
or `gpt-3.5`. See your LLM provider's API documentation for more available
models. For Bedrock, this can be a foundation model ID (e.g.,
`anthropic.claude-3-5-sonnet-20240620-v1:0`), a cross-region inference profile
ID (e.g., `us.anthropic.claude-3-5-sonnet-20240620-v1:0`), or an application
inference profile ARN (e.g., `arn:aws:bed [...]
+| instances.examples | array[string] | False |
| 1 to 64 items | Used when `algorithm` is `semantic`.
Example utterances representing this instance's intent; each is embedded into
its own reference vector. Required for every instance when `algorithm` is
`semantic`, including the one named by `semantic_opts.fallback`. |
+| instances.threshold | number | False |
| -1 to 1 | Used when `algorithm` is `semantic`.
Per-instance minimum cosine similarity; overrides `semantic_opts.threshold`. |
| logging | object | False |
| | Logging configurations. |
| logging.summaries | boolean | False | false
| | If true, log request LLM model, duration,
request, and response tokens. |
| logging.payloads | boolean | False | false
| | If true, log request and response
payload. |
@@ -122,6 +124,19 @@ When an instance's `provider` is set to `bedrock`, the
Plugin expects requests i
| instances.checks.active.unhealthy.http_statuses | array[integer] | False |
[429,404,500,501,502,503,504,505] | status code between 200 and 599 inclusive |
An array of HTTP status codes that defines an unhealthy node. |
| instances.checks.active.unhealthy.http_failures | integer | False |
5 | between 1 and 254 inclusive | Number of HTTP
failures to define an unhealthy node. |
| instances.checks.active.unhealthy.timeout | integer | False |
3 | between 1 and 254 inclusive | Number of probe
timeouts to define an unhealthy node. |
+| semantic_opts | object | False |
| | Options for the `semantic` balancer,
grouped in one place. Required when `balancer.algorithm` is `semantic`. |
+| semantic_opts.threshold | number | False | 0
| -1 to 1 | Global minimum cosine similarity an
instance must reach to be selected. **The default of 0 admits essentially any
prompt** (real embeddings are rarely dissimilar enough to score below 0), so
the fallback only ever runs if you raise this above 0. When no instance clears
its threshold, the request falls back to the `fallback` instance, or the first
instance if none is named. |
+| semantic_opts.fallback | string | False |
| | Name of the instance to route to when
no instance clears its threshold or the embedding request fails. It is
otherwise a normal ranked instance and needs `examples` like the rest. Defaults
to the first instance when unset. |
+| semantic_opts.debugging | boolean | False | false
| | If true, expose per-instance scores and
the routing decision via `X-AI-Semantic-Scores` and
`X-AI-Semantic-Picked-Instance` response headers, for debugging. |
+| semantic_opts.embeddings | object | True |
| | Embedding service configuration used to
score prompts against each instance's `examples`. |
+| semantic_opts.embeddings.provider | string | True |
| [openai, azure-openai] | Embedding provider used by
the semantic algorithm. |
+| semantic_opts.embeddings.model | string | True |
| | Embedding model name, e.g.
`text-embedding-3-small`. |
+| semantic_opts.embeddings.endpoint | string | False |
| | Embedding API endpoint. Optional for
`openai` (defaults to the public API). Required for `azure-openai`, where it
must be the **full** URL including the deployment path and API version, e.g.
`https://{resource}.openai.azure.com/openai/deployments/{deployment}/embeddings?api-version=2024-02-01`.
|
+| semantic_opts.embeddings.auth | object | True |
| | Authentication configurations for the
embedding service. |
+| semantic_opts.embeddings.auth.header | object | False |
| | Authentication headers. Encrypted at
rest. |
+| semantic_opts.embeddings.auth.query | object | False |
| | Authentication query parameters.
Encrypted at rest. |
+| semantic_opts.embeddings.timeout | integer | False | 3000
| minimum = 1 | Embedding request timeout in
milliseconds. The query prompt is embedded synchronously on every request, so
this bounds the latency added to each request when the embedding endpoint is
slow or down (the request then fails open to the fallback). |
+| semantic_opts.embeddings.ssl_verify | boolean | False | true
| | If true, verify the embedding service's
certificate. |
| timeout | integer | False | 30000
| greater than or equal to 1 | Request timeout in
milliseconds when requesting the LLM service. Applied per socket operation
(connect / send / read block); does not cap the total duration of a streaming
response. |
| max_req_body_size | integer | False | 67108864
| greater than or equal to 1 | Maximum request body size
in bytes that the plugin reads into memory. Requests whose body exceeds this
limit are rejected with `413`. Prevents unbounded memory buffering of large
request bodies. |
| max_stream_duration_ms | integer | False |
| greater than or equal to 1 | Maximum wall-clock duration
(in milliseconds) for a streaming AI response. If the upstream keeps sending
data past this deadline, the gateway closes the connection. Unset means no cap.
Use this to protect the gateway from upstream bugs that produce tokens
indefinitely. When the limit is hit mid-stream, the downstream SSE stream is
truncated (no protocol-speci [...]
@@ -2974,3 +2989,143 @@ You should receive a response similar to the following
if the request is forward
```
In the Kafka topic, you should also see a log entry corresponding to the
request with the LLM summary and request/response payload.
+
+### Route by Semantic Similarity
+
+The following example demonstrates how you can use the `semantic` algorithm to
pick an instance by the meaning of the request prompt, rather than by weight or
hash. Each ranked instance declares `examples` that describe the kind of prompt
it should handle; the Plugin embeds those examples and the incoming prompt with
the configured embedding model, and forwards the request to the instance whose
examples are most similar.
+
+Create a Route as such and update the LLM providers, embedding model, and API
keys accordingly. The `code` instance handles programming prompts, the
`translate` instance handles translation prompts, and the `default` instance
handles general prompts and is named as the `semantic_opts.fallback`, used when
no instance clears the similarity threshold:
+
+```shell
+curl "http://127.0.0.1:9180/apisix/admin/routes"; -X PUT \
+ -H "X-API-KEY: ${admin_key}" \
+ -d '{
+ "id": "ai-proxy-multi-route",
+ "uri": "/anything",
+ "methods": ["POST"],
+ "plugins": {
+ "ai-proxy-multi": {
+ "balancer": {
+ "algorithm": "semantic"
+ },
+ "semantic_opts": {
+ "embeddings": {
+ "provider": "openai",
+ "model": "text-embedding-3-small",
+ "auth": {
+ "header": {
+ "Authorization": "Bearer '"$YOUR_EMBEDDING_API_KEY"'"
+ }
+ }
+ },
+ "threshold": 0.5,
+ "fallback": "default"
+ },
+ "instances": [
+ {
+ "name": "code",
+ "provider": "openai",
+ "weight": 1,
+ "auth": {
+ "header": {
+ "Authorization": "Bearer '"$YOUR_LLM_API_KEY"'"
+ }
+ },
+ "options": {
+ "model": "gpt-4o"
+ },
+ "examples": [
+ "write a python function",
+ "debug this code"
+ ]
+ },
+ {
+ "name": "translate",
+ "provider": "openai",
+ "weight": 1,
+ "auth": {
+ "header": {
+ "Authorization": "Bearer '"$YOUR_LLM_API_KEY"'"
+ }
+ },
+ "options": {
+ "model": "gpt-4o-mini"
+ },
+ "examples": [
+ "translate this sentence into English"
+ ]
+ },
+ {
+ "name": "default",
+ "provider": "openai",
+ "weight": 1,
+ "auth": {
+ "header": {
+ "Authorization": "Bearer '"$YOUR_LLM_API_KEY"'"
+ }
+ },
+ "options": {
+ "model": "gpt-4o-mini"
+ },
+ "examples": [
+ "what is the weather today",
+ "tell me a joke"
+ ]
+ }
+ ]
+ }
+ }
+ }'
+```
+
+Send a request with a programming prompt:
+
+```shell
+curl "http://127.0.0.1:9080/anything"; -X POST \
+ -H "Content-Type: application/json" \
+ -d '{
+ "model": "auto",
+ "messages": [
+ { "role": "user", "content": "help me debug this python code" }
+ ]
+ }'
+```
+
+The request should be routed to the `code` instance and served by `gpt-4o`. A
general prompt such as `what is the weather today` instead matches the
`default` instance's own `examples`. The `default` instance is also named as
`semantic_opts.fallback`, so it additionally receives any prompt that is
ambiguous enough that no instance clears the threshold.
+
+#### Debugging the routing decision
+
+Set `semantic_opts.debugging` to `true` to have the Plugin report how each
instance scored and which one it picked:
+
+```shell
+curl -i "http://127.0.0.1:9080/anything"; -X POST \
+ -H "Content-Type: application/json" \
+ -d '{"model":"auto","messages":[{"role":"user","content":"help me debug this
python code"}]}'
+```
+
+```text
+HTTP/1.1 200 OK
+X-AI-Semantic-Picked-Instance: code
+X-AI-Semantic-Scores: code:0.8213,default:0.2451,translate:0.1904
+```
+
+`X-AI-Semantic-Scores` lists every instance, highest score first, so you can
see not only the winner but how close the runners-up came. The `default`
fallback instance is ranked alongside the others — it needs `examples` too —
and additionally receives the request whenever no instance clears its threshold.
+
+When no instance clears its threshold, the pick becomes `fallback` while the
scores still show how close each one got:
+
+```text
+X-AI-Semantic-Picked-Instance: fallback
+X-AI-Semantic-Scores: code:0.3057,translate:0.2884,default:0.2013
+```
+
+Use this to calibrate `threshold`: pick a value between the scores of the
prompts you want matched and the scores of the prompts you want to reach the
fallback.
+
+The same information is available without exposing it to clients: whenever no
instance clears its threshold, the Plugin logs the full score list at `warn`
level.
+
+```text
+semantic routing: no instance cleared threshold (scores:
code:0.3057,translate:0.2884), falling back
+```
+
+If the headers are missing entirely while `debugging` is on, the embedding
step itself failed and the request was served by the fallback before any score
was computed; the reason is logged at `warn`.
+
+Because `debugging` reveals instance names to the caller, keep it off in
production and rely on the log line instead.
diff --git a/docs/zh/latest/plugins/ai-proxy-multi.md
b/docs/zh/latest/plugins/ai-proxy-multi.md
index 2e18dd9b9..686ad7459 100644
--- a/docs/zh/latest/plugins/ai-proxy-multi.md
+++ b/docs/zh/latest/plugins/ai-proxy-multi.md
@@ -71,7 +71,7 @@ import TabItem from '@theme/TabItem';
| max_retries | integer | 否 |
| 大于或等于 0 |
初始请求失败后允许的最大故障转移重试次数。用于限制单个请求最多尝试多少个额外实例，避免穷举所有已配置的实例。仅在配置 `fallback_strategy`
时生效。未设置时，插件会持续重试直到某个实例成功或所有实例都已尝试。 |
| retry_on_failure_within_ms | integer | 否 |
| 大于或等于 1 | 仅当上游在指定毫秒数内失败时才故障转移到其他实例。快速失败（如连接错误、快速返回的
`429`/`5xx`）会触发重试，而耗时超过该值的慢失败会直接将错误返回给客户端，避免客户端等待时间翻倍。仅在配置 `fallback_strategy`
时生效。未设置时，插件无论失败请求耗时多久都会重试。 |
| balancer | object | 否 |
| | 负载均衡配置。 |
-| balancer.algorithm | string | 否 | roundrobin
| [roundrobin, chash] | 负载均衡算法。设置为 `roundrobin` 时，使用加权轮询算法。设置为
`chash` 时，使用一致性哈希算法。 |
+| balancer.algorithm | string | 否 | roundrobin
| [roundrobin, chash, semantic] | 负载均衡算法。设置为 `roundrobin`
时，使用加权轮询算法。设置为 `chash` 时，使用一致性哈希算法。设置为 `semantic` 时，插件根据请求提示词与各实例 `examples`
之间的语义相似度选择实例，其相关选项配置在 `semantic_opts` 下。注意：`semantic` 不参与健康检查，也不参与
`fallback_strategy` / 重试——被选中实例的上游失败会直接返回给客户端；只有当没有实例达到其阈值或嵌入请求失败时，才会回退（回退到
`semantic_opts.fallback` 实例，若未配置则回退到第一个实例）。 |
| balancer.hash_on | string | 否 |
| [vars, headers, cookie, consumer, vars_combinations] | 当
`type` 为 `chash` 时使用。支持基于 [NGINX
变量](https://nginx.org/en/docs/varindex.html)、标头、cookie、消费者或 [NGINX
变量](https://nginx.org/en/docs/varindex.html)组合进行哈希。 |
| balancer.key | string | 否 |
| | 当 `type` 为 `chash` 时使用。当 `hash_on` 设置为
`header` 或 `cookie` 时，需要 `key`。当 `hash_on` 设置为 `consumer` 时，不需要
`key`，因为消费者名称将自动用作键。 |
| instances | array[object] | 是 |
| | LLM 实例配置。 |
@@ -101,6 +101,8 @@ import TabItem from '@theme/TabItem';
| instances.override.llm_options.max_tokens | integer | 否 |
| ≥ 1 | 最大输出 token 数。APISIX
会自动将该值映射为各上游服务商对应的字段名。始终强制覆盖客户端值。 |
| instances.override.request_body | object | 否 |
| | 按目标协议的请求体覆盖配置。请参阅 `ai-proxy`
文档中的[按协议的请求体覆盖](./ai-proxy.md#per-protocol-request-body-override)。 |
| instances.override.request_body_force_override | boolean | 否 | false |
| 为
`false`（默认）时，客户端请求体中的字段优先，`instances.override.request_body` 仅补充缺失字段。为 `true`
时，`instances.override.request_body` 的值强制覆盖客户端请求体中的同名字段。不影响
`instances.override.llm_options`。 |
+| instances.examples | array[string] | 否 |
| 1 到 64 项 | 当 `algorithm` 为 `semantic`
时使用。代表该实例意图的示例语句；每一条都会被嵌入为独立的参考向量。当 `algorithm` 为 `semantic` 时每个实例都必填，包括被
`semantic_opts.fallback` 指定的实例。 |
+| instances.threshold | number | 否 |
| -1 到 1 | 当 `algorithm` 为 `semantic`
时使用。该实例的最小余弦相似度，覆盖 `semantic_opts.threshold`。 |
| logging | object | 否 |
| | 日志配置。不影响 `error.log`。 |
| logging.summaries | boolean | 否 | false
| | 如果为 true，记录请求 LLM 模型、持续时间、请求和响应令牌。 |
| logging.payloads | boolean | 否 | false
| | 如果为 true，记录请求和响应负载。 |
@@ -124,6 +126,19 @@ import TabItem from '@theme/TabItem';
| instances.checks.active.unhealthy.http_statuses | array[integer] | 否 |
[429,404,500,501,502,503,504,505] | 200 到 599 之间的状态码（包含） | 定义不健康节点的 HTTP 状态码数组。
|
| instances.checks.active.unhealthy.http_failures | integer | 否 | 5
| 1 到 254（包含） | 定义不健康节点的 HTTP 失败次数。 |
| instances.checks.active.unhealthy.timeout | integer | 否 | 3
| 1 到 254（包含） | 定义不健康节点的探测超时次数。 |
+| semantic_opts | object | 否 |
| | `semantic` 均衡器的选项，集中配置在一处。当
`balancer.algorithm` 为 `semantic` 时必填。 |
+| semantic_opts.threshold | number | 否 | 0
| -1 到 1 | 实例被选中所需达到的全局最小余弦相似度。**默认值 0
几乎会接纳任何提示词**（真实嵌入向量很少会不相似到得分低于 0），因此只有把它调到 0 以上，回退实例才会真正生效。当没有实例达到其阈值时，请求回退到
`fallback` 实例；若未指定，则回退到第一个实例。 |
+| semantic_opts.fallback | string | 否 |
| |
当没有实例达到其阈值或嵌入请求失败时，路由到的实例名称。它本身也是一个参与排名的普通实例，同样需要配置 `examples`。未设置时默认回退到第一个实例。 |
+| semantic_opts.debugging | boolean | 否 | false
| | 若为 true，通过 `X-AI-Semantic-Scores` 和
`X-AI-Semantic-Picked-Instance` 响应头暴露各实例的分数和路由决策，用于调试。 |
+| semantic_opts.embeddings | object | 是 |
| | 嵌入服务配置，用于将提示词与各实例的 `examples` 进行相似度打分。 |
+| semantic_opts.embeddings.provider | string | 是 |
| [openai, azure-openai] | 语义算法使用的嵌入服务提供商。 |
+| semantic_opts.embeddings.model | string | 是 |
| | 嵌入模型名称，例如 `text-embedding-3-small`。 |
+| semantic_opts.embeddings.endpoint | string | 否 |
| | 嵌入 API 端点。对于 `openai` 可选（默认使用公共 API）。对于
`azure-openai` 必填，且必须是**完整** URL，包含 deployment 路径和 API 版本，例如
`https://{resource}.openai.azure.com/openai/deployments/{deployment}/embeddings?api-version=2024-02-01`。
|
+| semantic_opts.embeddings.auth | object | 是 |
| | 嵌入服务的认证配置。 |
+| semantic_opts.embeddings.auth.header | object | 否 |
| | 认证请求头。加密存储。 |
+| semantic_opts.embeddings.auth.query | object | 否 |
| | 认证查询参数。加密存储。 |
+| semantic_opts.embeddings.timeout | integer | 否 | 3000
| 最小值为 1 |
嵌入请求的超时时间（毫秒）。每个请求都会同步嵌入查询提示词，因此该值限定了当嵌入端点变慢或不可用时每个请求增加的延迟上限（随后请求会 fail-open
到回退实例）。 |
+| semantic_opts.embeddings.ssl_verify | boolean | 否 | true
| | 如果为 true，验证嵌入服务的证书。 |
| timeout | integer | 否 | 30000
| 大于或等于 1 | 请求 LLM 服务时的请求超时时间（毫秒）。应用于单次 socket 操作（连接 / 发送 /
读取块），不限制流式响应的总时长。 |
| max_stream_duration_ms | integer | 否 |
| 大于或等于 1 | 流式 AI
响应的总墙钟时长上限（毫秒）。若上游在此时间后仍持续发送数据，网关将关闭连接。未设置时不限制。用于防护上游持续输出 token 导致网关 CPU
被打满的异常情况。中途触发上限时，下游 SSE 流会被截断（不再发送协议特定的终止标记，例如 `[DONE]`、`message_stop` 或
`response.completed`），客户端应将缺失的终止标记视为响应未完成。 |
| max_response_bytes | integer | 否 |
| 大于或等于 1 | 单次 AI 响应（流式或非流式）允许从上游读取的最大总字节数。超出时关闭连接。非流式响应若存在
`Content-Length`，在读取 body 之前预检；否则（chunked 传输）与流式响应一样在接收字节的过程中增量检查。未设置时不限制。 |
@@ -2779,3 +2794,143 @@ nginx_config:
```
访问日志条目显示请求类型为 `ai_chat`，Apisix 上游响应时间为 `5765` 毫秒，首次令牌时间为 `2858` 毫秒，请求的 LLM 模型为
`gpt-4`。LLM 模型为 `gpt-4`，提示令牌使用量为 `23`，完成令牌使用量为 `8`。
+
+### 按语义相似度路由
+
+以下示例演示了如何使用 `semantic` 算法，根据请求提示词的语义而非权重或哈希来选择实例。每个参与排名的实例通过 `examples`
声明它应当处理的提示词类型；插件使用配置的嵌入模型将这些示例和传入的提示词嵌入为向量，并将请求转发给示例与之最相似的实例。
+
+创建如下路由，并相应地更新 LLM 提供商、嵌入模型和 API 密钥。`code` 实例处理编程类提示词，`translate`
实例处理翻译类提示词，`default` 实例处理通用提示词并被指定为 `semantic_opts.fallback`，当没有实例达到相似度阈值时兜底：
+
+```shell
+curl "http://127.0.0.1:9180/apisix/admin/routes"; -X PUT \
+ -H "X-API-KEY: ${admin_key}" \
+ -d '{
+ "id": "ai-proxy-multi-route",
+ "uri": "/anything",
+ "methods": ["POST"],
+ "plugins": {
+ "ai-proxy-multi": {
+ "balancer": {
+ "algorithm": "semantic"
+ },
+ "semantic_opts": {
+ "embeddings": {
+ "provider": "openai",
+ "model": "text-embedding-3-small",
+ "auth": {
+ "header": {
+ "Authorization": "Bearer '"$YOUR_EMBEDDING_API_KEY"'"
+ }
+ }
+ },
+ "threshold": 0.5,
+ "fallback": "default"
+ },
+ "instances": [
+ {
+ "name": "code",
+ "provider": "openai",
+ "weight": 1,
+ "auth": {
+ "header": {
+ "Authorization": "Bearer '"$YOUR_LLM_API_KEY"'"
+ }
+ },
+ "options": {
+ "model": "gpt-4o"
+ },
+ "examples": [
+ "write a python function",
+ "debug this code"
+ ]
+ },
+ {
+ "name": "translate",
+ "provider": "openai",
+ "weight": 1,
+ "auth": {
+ "header": {
+ "Authorization": "Bearer '"$YOUR_LLM_API_KEY"'"
+ }
+ },
+ "options": {
+ "model": "gpt-4o-mini"
+ },
+ "examples": [
+ "translate this sentence into English"
+ ]
+ },
+ {
+ "name": "default",
+ "provider": "openai",
+ "weight": 1,
+ "auth": {
+ "header": {
+ "Authorization": "Bearer '"$YOUR_LLM_API_KEY"'"
+ }
+ },
+ "options": {
+ "model": "gpt-4o-mini"
+ },
+ "examples": [
+ "what is the weather today",
+ "tell me a joke"
+ ]
+ }
+ ]
+ }
+ }
+ }'
+```
+
+发送一个编程类提示词的请求：
+
+```shell
+curl "http://127.0.0.1:9080/anything"; -X POST \
+ -H "Content-Type: application/json" \
+ -d '{
+ "model": "auto",
+ "messages": [
+ { "role": "user", "content": "help me debug this python code" }
+ ]
+ }'
+```
+
+该请求应被路由到 `code` 实例，由 `gpt-4o` 处理。像 `what is the weather today` 这样的通用提示词则会匹配到
`default` 实例自己的 `examples`。`default` 同时被指定为
`semantic_opts.fallback`，因此当提示词过于模糊、没有任何实例达到阈值时，也会由它兜底接收。
+
+#### 调试路由决策
+
+将 `semantic_opts.debugging` 设置为 `true`，插件会在响应中报告各实例的得分以及最终选中的实例：
+
+```shell
+curl -i "http://127.0.0.1:9080/anything"; -X POST \
+ -H "Content-Type: application/json" \
+ -d '{"model":"auto","messages":[{"role":"user","content":"help me debug this
python code"}]}'
+```
+
+```text
+HTTP/1.1 200 OK
+X-AI-Semantic-Picked-Instance: code
+X-AI-Semantic-Scores: code:0.8213,default:0.2451,translate:0.1904
+```
+
+`X-AI-Semantic-Scores` 按得分从高到低列出**所有实例**，因此你不仅能看到胜出者，也能看到其他实例差了多少。`default`
回退实例与其他实例一起参与排名（它同样需要配置 `examples`），并在没有任何实例达到阈值时额外承担兜底。
+
+当没有任何实例达到阈值时，选中结果变为 `fallback`，而得分仍会显示各实例的接近程度：
+
+```text
+X-AI-Semantic-Picked-Instance: fallback
+X-AI-Semantic-Scores: code:0.3057,translate:0.2884,default:0.2013
+```
+
+据此可以校准 `threshold`：取一个介于「希望匹配的提示词得分」与「希望落到回退实例的提示词得分」之间的值。
+
+同样的信息也可以不暴露给客户端：每当没有实例达到阈值时，插件都会以 `warn` 级别记录完整的得分列表。
+
+```text
+semantic routing: no instance cleared threshold (scores:
code:0.3057,translate:0.2884), falling back
+```
+
+如果开启了 `debugging` 却完全没有这两个响应头，说明嵌入请求本身失败了，请求在计算任何得分之前就已回退到回退实例，失败原因会以 `warn`
级别记录。
+
+由于 `debugging` 会向调用方暴露实例名称，生产环境应保持关闭，改用日志排查。
diff --git a/t/plugin/ai-proxy-semantic-routing.t
b/t/plugin/ai-proxy-semantic-routing.t
new file mode 100644
index 000000000..47afb6d19
--- /dev/null
+++ b/t/plugin/ai-proxy-semantic-routing.t
@@ -0,0 +1,997 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+use t::APISIX 'no_plan';
+
+log_level("info");
+repeat_each(1);
+no_long_string();
+no_root_location();
+no_shuffle();
+
+add_block_preprocessor(sub {
+ my ($block) = @_;
+
+ if (!defined $block->request) {
+ $block->set_value("request", "GET /t");
+ }
+
+ my $http_config = $block->http_config // <<_EOC_;
+ # Mock embedding endpoint: returns a 2-D vector keyed on keywords so
+ # tests can steer which instance a prompt matches.
+ server {
+ server_name mock_embedding;
+ listen 6797;
+ default_type 'application/json';
+ location /v1/embeddings {
+ content_by_lua_block {
+ local json = require("cjson.safe")
+ ngx.req.read_body()
+ -- the embedding call must not carry the client's headers
+ ngx.log(ngx.WARN, "embed-recv-cookie:",
+ ngx.var.http_cookie or "none")
+ local body = json.decode(ngx.req.get_body_data()) or {}
+ for _, t in ipairs(body.input or {}) do
+ if string.lower(t):find("servererror") then
+ -- an error body that echoes the prompt back, as
some
+ -- providers do -- it must never reach our logs
+ ngx.status = 500
+ ngx.say('{"error":"bad input: ', t, '
SENSITIVE-ECHO"}')
+ return
+ elseif string.lower(t):find("scalarbody") then
+ -- 200 with a bare JSON null: decodes to a
non-table,
+ -- which must not be indexed
+ ngx.status = 200
+ ngx.say("null")
+ return
+ elseif string.lower(t):find("scalarentry") then
+ -- 200 whose data array holds non-objects: indexing
+ -- item.index on a number would raise
+ ngx.status = 200
+ ngx.say('{"data":[1,2,3]}')
+ return
+ elseif string.lower(t):find("naninf") then
+ -- 200 with a non-finite embedding component
+ -- (1e999 decodes to inf): must be rejected rather
+ -- than normalized to NaN and poisoning every score
+ ngx.status = 200
+
ngx.say('{"data":[{"index":0,"embedding":[1e999,0]}]}')
+ return
+ end
+ end
+ local function vec(text)
+ text = string.lower(text or "")
+ if text:find("code") or text:find("python") or
text:find("debug") then
+ return {1, 0, 0}
+ elseif text:find("translate") or text:find("summar")
then
+ return {0, 1, 0}
+ elseif text:find("weather") or text:find("joke") then
+ -- the `default` instance's own domain (general
chat),
+ -- so it competes on similarity like any other
instance
+ return {0, 0, 1}
+ end
+ -- A prompt matching no instance's domain: its
components
+ -- are distinct and none aligns with an axis, so it
clears
+ -- no high threshold and the request falls back.
Distinct
+ -- values also keep the reported score order
deterministic.
+ return {0.7, 0.5, 0.3}
+ end
+ local data = {}
+ for i, t in ipairs(body.input or {}) do
+ if string.lower(t):find("malformed") then
+ -- emit a JSON null embedding (decodes to
cjson.null)
+ -- to exercise fail-open on malformed data
+ data[i] = { index = i - 1, embedding = json.null }
+ elseif string.lower(t):find("dimmismatch") then
+ -- 2-D vector vs the 3-D reference vectors
+ data[i] = { index = i - 1, embedding = {1, 0} }
+ else
+ data[i] = { index = i - 1, embedding = vec(t) }
+ end
+ end
+ ngx.status = 200
+ ngx.say(json.encode({ data = data }))
+ }
+ }
+ }
+ # Mock Azure OpenAI embeddings: the deployment and api-version live in
+ # the URL, exactly as a real Azure endpoint requires.
+ server {
+ server_name mock_azure_embedding;
+ listen 6799;
+ default_type 'application/json';
+ location /openai/deployments/emb/embeddings {
+ content_by_lua_block {
+ local json = require("cjson.safe")
+ ngx.req.read_body()
+ -- prove the request reached the full Azure path with its
query
+ -- (an nginx arg variable cannot express the hyphen)
+ local args = ngx.req.get_uri_args()
+ ngx.log(ngx.WARN, "azure-embed-hit api-version=",
+ args["api-version"] or "none")
+ local body = json.decode(ngx.req.get_body_data()) or {}
+ local data = {}
+ for i, t in ipairs(body.input or {}) do
+ local v = {0.6, 0.6}
+ if string.lower(t):find("code") or
string.lower(t):find("python") then
+ v = {1, 0}
+ end
+ data[i] = { index = i - 1, embedding = v }
+ end
+ ngx.status = 200
+ ngx.say(json.encode({ data = data }))
+ }
+ }
+ }
+ # Mock LLM: echoes the model it received so tests can tell which
+ # instance was selected.
+ server {
+ server_name mock_llm;
+ listen 6798;
+ default_type 'application/json';
+ location /v1/chat/completions {
+ content_by_lua_block {
+ local json = require("cjson.safe")
+ ngx.req.read_body()
+ local body = json.decode(ngx.req.get_body_data()) or {}
+ ngx.status = 200
+ ngx.print(body.model or "unknown")
+ }
+ }
+ }
+_EOC_
+ $block->set_value("http_config", $http_config);
+
+ my $user_yaml_config = <<_EOC_;
+plugins:
+ - ai-proxy-multi
+_EOC_
+ $block->set_value("extra_yaml_config", $user_yaml_config);
+});
+
+run_tests();
+
+__DATA__
+
+=== TEST 1: configure a semantic route (code / translate / fallback)
+--- config
+ location /t {
+ content_by_lua_block {
+ -- Build the config as a Lua table and core.json.encode it. A long
+ -- inline JSON `[[ ]]` block that crosses nginx's config buffer
+ -- boundary is misparsed by ngx_lua ("missing the closing long
+ -- bracket"), so we keep the inlined Lua short instead.
+ local core = require("apisix.core")
+ local t = require("lib.test_admin").test
+ local function inst(name, model, examples)
+ local i = {
+ name = name, provider = "openai", weight = 1,
+ auth = { header = { Authorization = "Bearer token" } },
+ options = { model = model },
+ override = {
+ endpoint = "http://127.0.0.1:6798/v1/chat/completions";,
+ },
+ }
+ if examples then i.examples = examples end
+ return i
+ end
+ local conf = {
+ uri = "/anything",
+ plugins = {
+ ["ai-proxy-multi"] = {
+ balancer = { algorithm = "semantic" },
+ semantic_opts = {
+ embeddings = {
+ provider = "openai",
+ model = "text-embedding-3-small",
+ endpoint =
"http://127.0.0.1:6797/v1/embeddings";,
+ auth = { header = { Authorization = "Bearer
token" } },
+ ssl_verify = false,
+ },
+ threshold = 0.9,
+ fallback = "default",
+ },
+ instances = {
+ inst("code", "model-code", {"write python code"}),
+ inst("cheap", "model-cheap", {"translate this
text"}),
+ inst("default", "model-fallback",
+ {"what is the weather today", "tell me a
joke"}),
+ },
+ ssl_verify = false,
+ },
+ },
+ }
+ local code, body = t('/apisix/admin/routes/1', ngx.HTTP_PUT,
+ core.json.encode(conf))
+ if code >= 300 then
+ ngx.status = code
+ ngx.say(body)
+ return
+ end
+ ngx.say("passed")
+ }
+ }
+--- response_body
+passed
+--- no_error_log
+[error]
+
+
+
+=== TEST 2: coding prompt routes to the code model
+--- request
+POST /anything
+{"model":"auto","messages":[{"role":"user","content":"help me debug this
python code"}]}
+--- more_headers
+Content-Type: application/json
+--- response_body chomp
+model-code
+--- no_error_log
+[error]
+
+
+
+=== TEST 3: translation prompt routes to the cheap model
+--- request
+POST /anything
+{"model":"auto","messages":[{"role":"user","content":"please translate this to
english"}]}
+--- more_headers
+Content-Type: application/json
+--- response_body chomp
+model-cheap
+--- no_error_log
+[error]
+
+
+
+=== TEST 4: unrelated prompt clears no threshold, falls back to the fallback
instance
+--- request
+POST /anything
+{"model":"auto","messages":[{"role":"user","content":"recommend a good hotel
in paris"}]}
+--- more_headers
+Content-Type: application/json
+--- response_body chomp
+model-fallback
+--- error_log eval
+qr/no instance cleared threshold \(scores:
code:0\.\d+,cheap:0\.\d+,default:0\.\d+\)/
+--- no_error_log
+[error]
+
+
+
+=== TEST 5: malformed embedding fails open to the fallback, not the matching
instance
+--- request
+POST /anything
+{"model":"auto","messages":[{"role":"user","content":"debug this python code,
malformed"}]}
+--- more_headers
+Content-Type: application/json
+--- error_code: 200
+--- response_body chomp
+model-fallback
+--- error_log
+semantic routing: query embedding failed
+--- no_error_log
+[error]
+
+
+
+=== TEST 6: reconfigure the route with debugging enabled
+--- config
+ location /t {
+ content_by_lua_block {
+ local core = require("apisix.core")
+ local t = require("lib.test_admin").test
+ local function inst(name, model, examples)
+ local i = {
+ name = name, provider = "openai", weight = 1,
+ auth = { header = { Authorization = "Bearer token" } },
+ options = { model = model },
+ override = {
+ endpoint = "http://127.0.0.1:6798/v1/chat/completions";,
+ },
+ }
+ if examples then i.examples = examples end
+ return i
+ end
+ local conf = {
+ uri = "/anything",
+ plugins = {
+ ["ai-proxy-multi"] = {
+ balancer = { algorithm = "semantic" },
+ semantic_opts = {
+ embeddings = {
+ provider = "openai",
+ model = "text-embedding-3-small",
+ endpoint =
"http://127.0.0.1:6797/v1/embeddings";,
+ auth = { header = { Authorization = "Bearer
token" } },
+ ssl_verify = false,
+ },
+ threshold = 0.9,
+ fallback = "default",
+ debugging = true,
+ },
+ instances = {
+ inst("code", "model-code", {"write python code"}),
+ inst("cheap", "model-cheap", {"translate this
text"}),
+ inst("default", "model-fallback",
+ {"what is the weather today", "tell me a
joke"}),
+ },
+ ssl_verify = false,
+ },
+ },
+ }
+ local code, body = t('/apisix/admin/routes/1', ngx.HTTP_PUT,
+ core.json.encode(conf))
+ if code >= 300 then
+ ngx.status = code
+ ngx.say(body)
+ return
+ end
+ ngx.say("passed")
+ }
+ }
+--- response_body
+passed
+--- no_error_log
+[error]
+
+
+
+=== TEST 7: debug exposes per-instance scores and the pick via headers
+--- request
+POST /anything
+{"model":"auto","messages":[{"role":"user","content":"help me debug this
python code"}]}
+--- more_headers
+Content-Type: application/json
+--- response_body chomp
+model-code
+--- response_headers_like
+X-AI-Semantic-Picked-Instance: code
+X-AI-Semantic-Scores: code:1\.\d+(,\w+:0\.\d+){2}
+--- no_error_log
+[error]
+
+
+
+=== TEST 8: embedding dimension mismatch fails open to the fallback
+--- request
+POST /anything
+{"model":"auto","messages":[{"role":"user","content":"dimmismatch python
code"}]}
+--- more_headers
+Content-Type: application/json
+--- error_code: 200
+--- response_body chomp
+model-fallback
+--- error_log
+embedding dimension mismatch
+--- no_error_log
+[error]
+
+
+
+=== TEST 9: multimodal content array extracts text parts for routing
+--- request
+POST /anything
+{"model":"auto","messages":[{"role":"user","content":[
+{"type":"text","text":"write python code"},
+{"type":"image_url","image_url":{"url":"http://x/y.png"}}
+]}]}
+--- more_headers
+Content-Type: application/json
+--- response_body chomp
+model-code
+--- no_error_log
+[error]
+
+
+
+=== TEST 10: client headers are not forwarded to the embedding endpoint
+--- request
+POST /anything
+{"model":"auto","messages":[{"role":"user","content":"help me debug this
python code"}]}
+--- more_headers
+Content-Type: application/json
+Cookie: session=super-secret
+--- response_body chomp
+model-code
+--- error_log
+embed-recv-cookie:none
+--- no_error_log
+embed-recv-cookie:session=super-secret
+
+
+
+=== TEST 11: non-200 fails open to the fallback, without logging the upstream
body
+--- request
+POST /anything
+{"model":"auto","messages":[{"role":"user","content":"servererror python
code"}]}
+--- more_headers
+Content-Type: application/json
+--- error_code: 200
+--- response_body chomp
+model-fallback
+--- error_log
+embedding endpoint returned status 500
+--- no_error_log
+SENSITIVE-ECHO
+
+
+
+=== TEST 12: 200 with a non-table body fails open instead of raising
+--- request
+POST /anything
+{"model":"auto","messages":[{"role":"user","content":"scalarbody python
code"}]}
+--- more_headers
+Content-Type: application/json
+--- error_code: 200
+--- response_body chomp
+model-fallback
+--- error_log
+invalid embedding response
+--- no_error_log
+[error]
+
+
+
+=== TEST 13: 200 whose data array holds non-objects fails open instead of
raising
+--- request
+POST /anything
+{"model":"auto","messages":[{"role":"user","content":"scalarentry python
code"}]}
+--- more_headers
+Content-Type: application/json
+--- error_code: 200
+--- response_body chomp
+model-fallback
+--- error_log
+invalid embedding entry at index 0
+--- no_error_log
+[error]
+
+
+
+=== TEST 14: reference embedding failure fails open to the fallback
+--- config
+ location /t {
+ content_by_lua_block {
+ local core = require("apisix.core")
+ local t = require("lib.test_admin").test
+ local function inst(name, model, examples)
+ local i = {
+ name = name, provider = "openai", weight = 1,
+ auth = { header = { Authorization = "Bearer token" } },
+ options = { model = model },
+ override = {
+ endpoint = "http://127.0.0.1:6798/v1/chat/completions";,
+ },
+ }
+ if examples then i.examples = examples end
+ return i
+ end
+ local conf = {
+ uri = "/anything",
+ plugins = {
+ ["ai-proxy-multi"] = {
+ balancer = { algorithm = "semantic" },
+ semantic_opts = {
+ embeddings = {
+ provider = "openai",
+ model = "text-embedding-3-small",
+ endpoint =
"http://127.0.0.1:6797/v1/embeddings";,
+ auth = { header = { Authorization = "Bearer
token" } },
+ ssl_verify = false,
+ },
+ threshold = 0.9,
+ fallback = "default",
+ },
+ instances = {
+ -- embedding this instance's examples makes the
mock
+ -- fail the whole reference batch
+ inst("code", "model-code", {"servererror
example"}),
+ inst("default", "model-fallback",
+ {"what is the weather today", "tell me a
joke"}),
+ },
+ ssl_verify = false,
+ },
+ },
+ }
+ local code, body = t('/apisix/admin/routes/1', ngx.HTTP_PUT,
+ core.json.encode(conf))
+ if code >= 300 then
+ ngx.status = code
+ ngx.say(body)
+ return
+ end
+ ngx.say("passed")
+ }
+ }
+--- response_body
+passed
+--- no_error_log
+[error]
+
+
+
+=== TEST 15: request on the broken-reference route still routes to the fallback
+--- request
+POST /anything
+{"model":"auto","messages":[{"role":"user","content":"help me debug this
python code"}]}
+--- more_headers
+Content-Type: application/json
+--- error_code: 200
+--- response_body chomp
+model-fallback
+--- error_log
+failed to fetch reference embeddings
+--- no_error_log
+[error]
+
+
+
+=== TEST 16: azure-openai embeddings route via the full deployment URL
+--- config
+ location /t {
+ content_by_lua_block {
+ local core = require("apisix.core")
+ local t = require("lib.test_admin").test
+ local function inst(name, model, examples)
+ local i = {
+ name = name, provider = "openai", weight = 1,
+ auth = { header = { Authorization = "Bearer token" } },
+ options = { model = model },
+ override = {
+ endpoint = "http://127.0.0.1:6798/v1/chat/completions";,
+ },
+ }
+ if examples then i.examples = examples end
+ return i
+ end
+ local conf = {
+ uri = "/anything",
+ plugins = {
+ ["ai-proxy-multi"] = {
+ balancer = { algorithm = "semantic" },
+ semantic_opts = {
+ embeddings = {
+ provider = "azure-openai",
+ model = "text-embedding-3-small",
+ endpoint =
"http://127.0.0.1:6799/openai/deployments/emb";
+ ..
"/embeddings?api-version=2024-02-01",
+ auth = { header = { ["api-key"] = "azure-key"
} },
+ ssl_verify = false,
+ },
+ threshold = 0.9,
+ fallback = "default",
+ },
+ instances = {
+ inst("code", "model-code", {"write python code"}),
+ inst("default", "model-fallback",
+ {"what is the weather today", "tell me a
joke"}),
+ },
+ ssl_verify = false,
+ },
+ },
+ }
+ local code, body = t('/apisix/admin/routes/1', ngx.HTTP_PUT,
+ core.json.encode(conf))
+ if code >= 300 then
+ ngx.status = code
+ ngx.say(body)
+ return
+ end
+ ngx.say("passed")
+ }
+ }
+--- response_body
+passed
+--- no_error_log
+[error]
+
+
+
+=== TEST 17: azure embeddings request reaches the deployment path with
api-version
+--- request
+POST /anything
+{"model":"auto","messages":[{"role":"user","content":"help me debug this
python code"}]}
+--- more_headers
+Content-Type: application/json
+--- response_body chomp
+model-code
+--- error_log
+azure-embed-hit api-version=2024-02-01
+--- no_error_log
+[error]
+
+
+
+=== TEST 18: no fallback configured -- fallback is the first instance
+--- config
+ location /t {
+ content_by_lua_block {
+ local core = require("apisix.core")
+ local t = require("lib.test_admin").test
+ local function inst(name, model, examples)
+ return {
+ name = name, provider = "openai", weight = 1,
+ auth = { header = { Authorization = "Bearer token" } },
+ options = { model = model },
+ override = {
+ endpoint = "http://127.0.0.1:6798/v1/chat/completions";,
+ },
+ examples = examples,
+ }
+ end
+ local conf = {
+ uri = "/anything",
+ plugins = {
+ ["ai-proxy-multi"] = {
+ balancer = { algorithm = "semantic" },
+ semantic_opts = {
+ embeddings = {
+ provider = "openai",
+ model = "text-embedding-3-small",
+ endpoint =
"http://127.0.0.1:6797/v1/embeddings";,
+ auth = { header = { Authorization = "Bearer
token" } },
+ ssl_verify = false,
+ },
+ threshold = 0.9,
+ },
+ instances = {
+ inst("first", "model-first", {"write python
code"}),
+ inst("second", "model-second", {"translate this
text"}),
+ },
+ ssl_verify = false,
+ },
+ },
+ }
+ local code, body = t('/apisix/admin/routes/1', ngx.HTTP_PUT,
+ core.json.encode(conf))
+ if code >= 300 then
+ ngx.status = code
+ ngx.say(body)
+ return
+ end
+ ngx.say("passed")
+ }
+ }
+--- response_body
+passed
+--- no_error_log
+[error]
+
+
+
+=== TEST 19: unmatched prompt falls back to the first instance
+--- request
+POST /anything
+{"model":"auto","messages":[{"role":"user","content":"what is the weather
today"}]}
+--- more_headers
+Content-Type: application/json
+--- response_body chomp
+model-first
+--- error_log
+no instance cleared threshold
+--- no_error_log
+[error]
+
+
+
+=== TEST 20: per-instance threshold overrides the global one
+--- config
+ location /t {
+ content_by_lua_block {
+ local core = require("apisix.core")
+ local t = require("lib.test_admin").test
+ local conf = {
+ uri = "/anything",
+ plugins = {
+ ["ai-proxy-multi"] = {
+ -- the global threshold is permissive (0.1); the code
+ -- instance raises its own bar to 0.9, so a prompt that
+ -- partially matches it (~0.77) clears the global
threshold
+ -- but not the instance's, and the request goes to
default
+ balancer = { algorithm = "semantic" },
+ semantic_opts = {
+ embeddings = {
+ provider = "openai",
+ model = "text-embedding-3-small",
+ endpoint =
"http://127.0.0.1:6797/v1/embeddings";,
+ auth = { header = { Authorization = "Bearer
token" } },
+ ssl_verify = false,
+ },
+ threshold = 0.1,
+ fallback = "default",
+ },
+ instances = {
+ {
+ name = "code", provider = "openai", weight = 1,
+ auth = { header = { Authorization = "Bearer
token" } },
+ options = { model = "model-code" },
+ override = {
+ endpoint =
"http://127.0.0.1:6798/v1/chat/completions";,
+ },
+ examples = {"write python code"},
+ threshold = 0.9,
+ },
+ {
+ name = "default", provider = "openai", weight
= 1,
+ auth = { header = { Authorization = "Bearer
token" } },
+ options = { model = "model-fallback" },
+ override = {
+ endpoint =
"http://127.0.0.1:6798/v1/chat/completions";,
+ },
+ examples = {"what is the weather today", "tell
me a joke"},
+ },
+ },
+ ssl_verify = false,
+ },
+ },
+ }
+ local code, body = t('/apisix/admin/routes/1', ngx.HTTP_PUT,
+ core.json.encode(conf))
+ if code >= 300 then
+ ngx.status = code
+ ngx.say(body)
+ return
+ end
+ ngx.say("passed")
+ }
+ }
+--- response_body
+passed
+--- no_error_log
+[error]
+
+
+
+=== TEST 21: a prompt below the per-instance threshold reaches the fallback
+--- request
+POST /anything
+{"model":"auto","messages":[{"role":"user","content":"recommend a good hotel
in paris"}]}
+--- more_headers
+Content-Type: application/json
+--- response_body chomp
+model-fallback
+--- no_error_log
+[error]
+
+
+
+=== TEST 22: configure a semantic route on the Responses API path
+--- config
+ location /t {
+ content_by_lua_block {
+ local core = require("apisix.core")
+ local t = require("lib.test_admin").test
+ local function inst(name, model, examples)
+ local i = {
+ name = name, provider = "openai", weight = 1,
+ auth = { header = { Authorization = "Bearer token" } },
+ options = { model = model },
+ override = {
+ endpoint = "http://127.0.0.1:6798/v1/chat/completions";,
+ },
+ }
+ if examples then i.examples = examples end
+ return i
+ end
+ local conf = {
+ uri = "/llm/v1/responses",
+ plugins = {
+ ["ai-proxy-multi"] = {
+ balancer = { algorithm = "semantic" },
+ semantic_opts = {
+ embeddings = {
+ provider = "openai",
+ model = "text-embedding-3-small",
+ endpoint =
"http://127.0.0.1:6797/v1/embeddings";,
+ auth = { header = { Authorization = "Bearer
token" } },
+ ssl_verify = false,
+ },
+ threshold = 0.9,
+ fallback = "default",
+ debugging = true,
+ },
+ instances = {
+ inst("code", "model-code", {"write python code"}),
+ inst("cheap", "model-cheap", {"translate this
text"}),
+ inst("default", "model-fallback",
+ {"what is the weather today", "tell me a
joke"}),
+ },
+ ssl_verify = false,
+ },
+ },
+ }
+ local code, body = t('/apisix/admin/routes/2', ngx.HTTP_PUT,
+ core.json.encode(conf))
+ if code >= 300 then
+ ngx.status = code
+ ngx.say(body)
+ return
+ end
+ ngx.say("passed")
+ }
+ }
+--- response_body
+passed
+--- no_error_log
+[error]
+
+
+
+=== TEST 23: a Responses request (body.input) is routed, not fallen back
+--- request
+POST /llm/v1/responses
+{"model":"auto","input":"help me debug this python code"}
+--- more_headers
+Content-Type: application/json
+--- response_headers_like
+X-AI-Semantic-Picked-Instance: code
+X-AI-Semantic-Scores: code:1\.\d+(,\w+:0\.\d+){2}
+--- no_error_log
+[error]
+
+
+
+=== TEST 24: configure a semantic route on the Anthropic Messages path
+--- config
+ location /t {
+ content_by_lua_block {
+ local core = require("apisix.core")
+ local t = require("lib.test_admin").test
+ local function inst(name, model, examples)
+ local i = {
+ name = name, provider = "openai", weight = 1,
+ auth = { header = { Authorization = "Bearer token" } },
+ options = { model = model },
+ override = {
+ endpoint = "http://127.0.0.1:6798/v1/chat/completions";,
+ },
+ }
+ if examples then i.examples = examples end
+ return i
+ end
+ local conf = {
+ uri = "/llm/v1/messages",
+ plugins = {
+ ["ai-proxy-multi"] = {
+ balancer = { algorithm = "semantic" },
+ semantic_opts = {
+ embeddings = {
+ provider = "openai",
+ model = "text-embedding-3-small",
+ endpoint =
"http://127.0.0.1:6797/v1/embeddings";,
+ auth = { header = { Authorization = "Bearer
token" } },
+ ssl_verify = false,
+ },
+ threshold = 0.9,
+ fallback = "default",
+ debugging = true,
+ },
+ instances = {
+ inst("code", "model-code", {"write python code"}),
+ inst("cheap", "model-cheap", {"translate this
text"}),
+ inst("default", "model-fallback",
+ {"what is the weather today", "tell me a
joke"}),
+ },
+ ssl_verify = false,
+ },
+ },
+ }
+ local code, body = t('/apisix/admin/routes/3', ngx.HTTP_PUT,
+ core.json.encode(conf))
+ if code >= 300 then
+ ngx.status = code
+ ngx.say(body)
+ return
+ end
+ ngx.say("passed")
+ }
+ }
+--- response_body
+passed
+--- no_error_log
+[error]
+
+
+
+=== TEST 25: an Anthropic request (body.system + content) is routed by the
last user turn
+--- request
+POST /llm/v1/messages
+{"model":"auto","system":"you are a helpful
assistant","messages":[{"role":"user","content":"help me debug this python
code"}]}
+--- more_headers
+Content-Type: application/json
+--- response_headers_like
+X-AI-Semantic-Picked-Instance: code
+X-AI-Semantic-Scores: code:1\.\d+(,\w+:0\.\d+){2}
+
+
+
+=== TEST 26: configure a route to exercise the non-finite-component guard
+--- config
+ location /t {
+ content_by_lua_block {
+ local core = require("apisix.core")
+ local t = require("lib.test_admin").test
+ local conf = {
+ uri = "/anything",
+ plugins = {
+ ["ai-proxy-multi"] = {
+ balancer = { algorithm = "semantic" },
+ semantic_opts = {
+ embeddings = {
+ provider = "openai",
+ model = "text-embedding-3-small",
+ endpoint =
"http://127.0.0.1:6797/v1/embeddings";,
+ auth = { header = { Authorization = "Bearer
token" } },
+ ssl_verify = false,
+ },
+ threshold = 0.9,
+ fallback = "default",
+ },
+ instances = {
+ {
+ name = "code", provider = "openai", weight = 1,
+ auth = { header = { Authorization = "Bearer
token" } },
+ options = { model = "model-code" },
+ override = {
+ endpoint =
"http://127.0.0.1:6798/v1/chat/completions";,
+ },
+ examples = {"write python code"},
+ },
+ {
+ name = "default", provider = "openai", weight
= 1,
+ auth = { header = { Authorization = "Bearer
token" } },
+ options = { model = "model-fallback" },
+ override = {
+ endpoint =
"http://127.0.0.1:6798/v1/chat/completions";,
+ },
+ examples = {"what is the weather today", "tell
me a joke"},
+ },
+ },
+ ssl_verify = false,
+ },
+ },
+ }
+ local code, body = t('/apisix/admin/routes/1', ngx.HTTP_PUT,
+ core.json.encode(conf))
+ if code >= 300 then
+ ngx.status = code
+ ngx.say(body)
+ return
+ end
+ ngx.say("passed")
+ }
+ }
+--- response_body
+passed
+--- no_error_log
+[error]
+
+
+
+=== TEST 27: a non-finite embedding component fails open instead of poisoning
scores
+--- request
+POST /anything
+{"model":"auto","messages":[{"role":"user","content":"naninf please route
me"}]}
+--- more_headers
+Content-Type: application/json
+--- error_code: 200
+--- response_body chomp
+model-fallback
+--- error_log
+non-numeric embedding component
+--- no_error_log
+[error]
diff --git a/t/plugin/ai-proxy-semantic-schema.t
b/t/plugin/ai-proxy-semantic-schema.t
new file mode 100644
index 000000000..da756fe3b
--- /dev/null
+++ b/t/plugin/ai-proxy-semantic-schema.t
@@ -0,0 +1,470 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+use t::APISIX 'no_plan';
+
+log_level("info");
+repeat_each(1);
+no_long_string();
+no_root_location();
+no_shuffle();
+
+add_block_preprocessor(sub {
+ my ($block) = @_;
+ if (!defined $block->request) {
+ $block->set_value("request", "GET /t");
+ }
+});
+
+run_tests();
+
+__DATA__
+
+=== TEST 1: valid semantic config is accepted
+--- config
+ location /t {
+ content_by_lua_block {
+ local t = require("lib.test_admin").test
+ local code, body = t('/apisix/admin/routes/1', ngx.HTTP_PUT, [[{
+ "uri": "/anything",
+ "plugins": {
+ "ai-proxy-multi": {
+ "balancer": { "algorithm": "semantic" },
+ "semantic_opts": {
+ "embeddings": {
+ "provider": "openai",
+ "model": "text-embedding-3-small",
+ "auth": { "header": { "Authorization": "Bearer
sk-emb" } }
+ },
+ "threshold": 0.75,
+ "fallback": "default"
+ },
+ "instances": [
+ {
+ "name": "code", "provider": "openai",
"weight": 1,
+ "auth": { "header": { "Authorization": "Bearer
token" } },
+ "options": { "model": "gpt-4o" },
+ "examples": ["write a python function", "debug
this code"],
+ "threshold": 0.85
+ },
+ {
+ "name": "default", "provider": "openai",
"weight": 1,
+ "auth": { "header": { "Authorization": "Bearer
token" } },
+ "options": { "model": "gpt-4o-mini" },
+ "examples": ["what is the weather today",
"tell me a joke"]
+ }
+ ],
+ "ssl_verify": false
+ }
+ }
+ }]])
+ if code >= 300 then
+ ngx.status = code
+ ngx.say(body)
+ return
+ end
+ ngx.say("passed")
+ }
+ }
+--- response_body
+passed
+--- no_error_log
+[error]
+
+
+
+=== TEST 2: semantic requires examples on a non-fallback instance
+--- config
+ location /t {
+ content_by_lua_block {
+ local t = require("lib.test_admin").test
+ local code, body = t('/apisix/admin/routes/1', ngx.HTTP_PUT, [[{
+ "uri": "/anything",
+ "plugins": {
+ "ai-proxy-multi": {
+ "balancer": { "algorithm": "semantic" },
+ "semantic_opts": {
+ "embeddings": {
+ "provider": "openai", "model":
"text-embedding-3-small",
+ "auth": { "header": { "Authorization": "Bearer
sk-emb" } }
+ }
+ },
+ "instances": [
+ {
+ "name": "code", "provider": "openai",
"weight": 1,
+ "auth": { "header": { "Authorization": "Bearer
token" } },
+ "options": { "model": "gpt-4o" }
+ }
+ ],
+ "ssl_verify": false
+ }
+ }
+ }]])
+ ngx.status = code
+ ngx.say(body)
+ }
+ }
+--- error_code: 400
+--- response_body_like: examples
+
+
+
+=== TEST 3: semantic requires semantic_opts.embeddings config
+--- config
+ location /t {
+ content_by_lua_block {
+ local t = require("lib.test_admin").test
+ local code, body = t('/apisix/admin/routes/1', ngx.HTTP_PUT, [[{
+ "uri": "/anything",
+ "plugins": {
+ "ai-proxy-multi": {
+ "balancer": { "algorithm": "semantic" },
+ "instances": [
+ {
+ "name": "code", "provider": "openai",
"weight": 1,
+ "auth": { "header": { "Authorization": "Bearer
token" } },
+ "options": { "model": "gpt-4o" },
+ "examples": ["write code"]
+ }
+ ],
+ "ssl_verify": false
+ }
+ }
+ }]])
+ ngx.status = code
+ ngx.say(body)
+ }
+ }
+--- error_code: 400
+--- response_body_like: embeddings
+
+
+
+=== TEST 4: semantic_opts.fallback must name an existing instance
+--- config
+ location /t {
+ content_by_lua_block {
+ local t = require("lib.test_admin").test
+ local code, body = t('/apisix/admin/routes/1', ngx.HTTP_PUT, [[{
+ "uri": "/anything",
+ "plugins": {
+ "ai-proxy-multi": {
+ "balancer": { "algorithm": "semantic" },
+ "semantic_opts": {
+ "embeddings": {
+ "provider": "openai", "model":
"text-embedding-3-small",
+ "auth": { "header": { "Authorization": "Bearer
sk-emb" } }
+ },
+ "fallback": "nope"
+ },
+ "instances": [
+ {
+ "name": "a", "provider": "openai", "weight": 1,
+ "auth": { "header": { "Authorization": "Bearer
token" } },
+ "options": { "model": "gpt-4o" },
+ "examples": ["write code"]
+ }
+ ],
+ "ssl_verify": false
+ }
+ }
+ }]])
+ ngx.status = code
+ ngx.say(body)
+ }
+ }
+--- error_code: 400
+--- response_body_like: unknown instance
+
+
+
+=== TEST 5: azure-openai embeddings require an endpoint
+--- config
+ location /t {
+ content_by_lua_block {
+ local t = require("lib.test_admin").test
+ local code, body = t('/apisix/admin/routes/1', ngx.HTTP_PUT, [[{
+ "uri": "/anything",
+ "plugins": {
+ "ai-proxy-multi": {
+ "balancer": { "algorithm": "semantic" },
+ "semantic_opts": {
+ "embeddings": {
+ "provider": "azure-openai", "model":
"text-embedding-3-small",
+ "auth": { "header": { "api-key": "sk-emb" } }
+ }
+ },
+ "instances": [
+ {
+ "name": "a", "provider": "openai", "weight": 1,
+ "auth": { "header": { "Authorization": "Bearer
token" } },
+ "options": { "model": "gpt-4o" },
+ "examples": ["write code"]
+ }
+ ],
+ "ssl_verify": false
+ }
+ }
+ }]])
+ ngx.status = code
+ ngx.say(body)
+ }
+ }
+--- error_code: 400
+--- response_body_like: endpoint
+
+
+
+=== TEST 6: embeddings.model is required
+--- config
+ location /t {
+ content_by_lua_block {
+ local t = require("lib.test_admin").test
+ local code, body = t('/apisix/admin/routes/1', ngx.HTTP_PUT, [[{
+ "uri": "/anything",
+ "plugins": {
+ "ai-proxy-multi": {
+ "balancer": { "algorithm": "semantic" },
+ "semantic_opts": {
+ "embeddings": {
+ "provider": "openai",
+ "auth": { "header": { "Authorization": "Bearer
sk-emb" } }
+ }
+ },
+ "instances": [
+ {
+ "name": "a", "provider": "openai", "weight": 1,
+ "auth": { "header": { "Authorization": "Bearer
token" } },
+ "options": { "model": "gpt-4o" },
+ "examples": ["write code"]
+ }
+ ],
+ "ssl_verify": false
+ }
+ }
+ }]])
+ ngx.status = code
+ ngx.say(body)
+ }
+ }
+--- error_code: 400
+--- response_body_like: model
+
+
+
+=== TEST 7: the named fallback is not exempt from the examples requirement
+--- config
+ location /t {
+ content_by_lua_block {
+ local t = require("lib.test_admin").test
+ local code, body = t('/apisix/admin/routes/1', ngx.HTTP_PUT, [[{
+ "uri": "/anything",
+ "plugins": {
+ "ai-proxy-multi": {
+ "balancer": { "algorithm": "semantic" },
+ "semantic_opts": {
+ "embeddings": {
+ "provider": "openai", "model":
"text-embedding-3-small",
+ "auth": { "header": { "Authorization": "Bearer
sk-emb" } }
+ },
+ "fallback": "default"
+ },
+ "instances": [
+ {
+ "name": "a", "provider": "openai", "weight": 1,
+ "auth": { "header": { "Authorization": "Bearer
token" } },
+ "options": { "model": "gpt-4o" },
+ "examples": ["write code"]
+ },
+ {
+ "name": "default", "provider": "openai",
"weight": 1,
+ "auth": { "header": { "Authorization": "Bearer
token" } },
+ "options": { "model": "gpt-4o-mini" }
+ }
+ ],
+ "ssl_verify": false
+ }
+ }
+ }]])
+ ngx.status = code
+ ngx.say(body)
+ }
+ }
+--- error_code: 400
+--- response_body_like: examples
+
+
+
+=== TEST 8: azure-openai embeddings endpoint must include the deployment path
+--- config
+ location /t {
+ content_by_lua_block {
+ local t = require("lib.test_admin").test
+ local code, body = t('/apisix/admin/routes/1', ngx.HTTP_PUT, [[{
+ "uri": "/anything",
+ "plugins": {
+ "ai-proxy-multi": {
+ "balancer": { "algorithm": "semantic" },
+ "semantic_opts": {
+ "embeddings": {
+ "provider": "azure-openai", "model":
"text-embedding-3-small",
+ "endpoint": "https://my.openai.azure.com";,
+ "auth": { "header": { "api-key": "sk-emb" } }
+ }
+ },
+ "instances": [
+ {
+ "name": "a", "provider": "openai", "weight": 1,
+ "auth": { "header": { "Authorization": "Bearer
token" } },
+ "options": { "model": "gpt-4o" },
+ "examples": ["write code"]
+ }
+ ],
+ "ssl_verify": false
+ }
+ }
+ }]])
+ ngx.status = code
+ ngx.say(body)
+ }
+ }
+--- error_code: 400
+--- response_body_like: full deployment path
+
+
+
+=== TEST 9: embeddings.auth rejects ctx-dependent schemes
+--- config
+ location /t {
+ content_by_lua_block {
+ local t = require("lib.test_admin").test
+ local code, body = t('/apisix/admin/routes/1', ngx.HTTP_PUT, [[{
+ "uri": "/anything",
+ "plugins": {
+ "ai-proxy-multi": {
+ "balancer": { "algorithm": "semantic" },
+ "semantic_opts": {
+ "embeddings": {
+ "provider": "openai", "model":
"text-embedding-3-small",
+ "auth": { "gcp": { "service_account_json":
"{}" } }
+ }
+ },
+ "instances": [
+ {
+ "name": "a", "provider": "openai", "weight": 1,
+ "auth": { "header": { "Authorization": "Bearer
token" } },
+ "options": { "model": "gpt-4o" },
+ "examples": ["write code"]
+ }
+ ],
+ "ssl_verify": false
+ }
+ }
+ }]])
+ ngx.status = code
+ ngx.say(body)
+ }
+ }
+--- error_code: 400
+--- response_body_like: gcp
+
+
+
+=== TEST 10: embeddings.endpoint must carry a scheme and host
+--- config
+ location /t {
+ content_by_lua_block {
+ local t = require("lib.test_admin").test
+ local code, body = t('/apisix/admin/routes/1', ngx.HTTP_PUT, [[{
+ "uri": "/anything",
+ "plugins": {
+ "ai-proxy-multi": {
+ "balancer": { "algorithm": "semantic" },
+ "semantic_opts": {
+ "embeddings": {
+ "provider": "openai", "model":
"text-embedding-3-small",
+ "endpoint": "my.embeddings.host/v1/embeddings",
+ "auth": { "header": { "Authorization": "Bearer
sk" } }
+ }
+ },
+ "instances": [
+ {
+ "name": "a", "provider": "openai", "weight": 1,
+ "auth": { "header": { "Authorization": "Bearer
token" } },
+ "options": { "model": "gpt-4o" },
+ "examples": ["write code"]
+ }
+ ],
+ "ssl_verify": false
+ }
+ }
+ }]])
+ ngx.status = code
+ ngx.say(body)
+ }
+ }
+--- error_code: 400
+--- response_body_like: semantic_opts.embeddings.endpoint
+
+
+
+=== TEST 11: the fallback instance may also carry examples and rank
+--- config
+ location /t {
+ content_by_lua_block {
+ local t = require("lib.test_admin").test
+ local code, body = t('/apisix/admin/routes/1', ngx.HTTP_PUT, [[{
+ "uri": "/anything",
+ "plugins": {
+ "ai-proxy-multi": {
+ "balancer": { "algorithm": "semantic" },
+ "semantic_opts": {
+ "embeddings": {
+ "provider": "openai", "model":
"text-embedding-3-small",
+ "auth": { "header": { "Authorization": "Bearer
sk-emb" } }
+ },
+ "fallback": "generalist"
+ },
+ "instances": [
+ {
+ "name": "code", "provider": "openai",
"weight": 1,
+ "auth": { "header": { "Authorization": "Bearer
token" } },
+ "options": { "model": "gpt-4o" },
+ "examples": ["write a python function"]
+ },
+ {
+ "name": "generalist", "provider": "openai",
"weight": 1,
+ "auth": { "header": { "Authorization": "Bearer
token" } },
+ "options": { "model": "gpt-4o-mini" },
+ "examples": ["answer a general question"]
+ }
+ ],
+ "ssl_verify": false
+ }
+ }
+ }]])
+ if code >= 300 then
+ ngx.status = code
+ ngx.say(body)
+ return
+ end
+ ngx.say("passed")
+ }
+ }
+--- response_body
+passed
+--- no_error_log
+[error]
diff --git a/t/plugin/ai-proxy-semantic.t b/t/plugin/ai-proxy-semantic.t
new file mode 100644
index 000000000..5fe9e0fa6
--- /dev/null
+++ b/t/plugin/ai-proxy-semantic.t
@@ -0,0 +1,67 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+use t::APISIX 'no_plan';
+
+log_level("info");
+repeat_each(1);
+no_long_string();
+no_root_location();
+no_shuffle();
+
+run_tests();
+
+__DATA__
+
+=== TEST 1: semantic math helpers (normalize / dot / cosine / max)
+--- config
+ location /t {
+ content_by_lua_block {
+ local s = require("apisix.plugins.ai-proxy.semantic")
+ local function approx(a, b) return math.abs(a - b) < 1e-9 end
+
+ -- normalize
+ local n = s.normalize({3, 4})
+ assert(approx(n[1], 0.6) and approx(n[2], 0.8), "normalize unit
vector")
+ local z = s.normalize({0, 0})
+ assert(approx(z[1], 0) and approx(z[2], 0), "normalize zero
vector")
+
+ -- cosine
+ assert(approx(s.cosine({1, 0}, {1, 0}), 1.0), "cosine identical")
+ assert(approx(s.cosine({1, 0}, {0, 1}), 0.0), "cosine orthogonal")
+ assert(approx(s.cosine({1, 1}, {2, 2}), 1.0), "cosine same
direction")
+ assert(approx(s.cosine({1, 0}, {-1, 0}), -1.0), "cosine opposite")
+
+ -- dot on normalized == cosine
+ local a = s.normalize({1, 2, 3})
+ assert(approx(s.dot(a, a), 1.0), "dot of normalized identical")
+
+ -- max: an instance scores by its best-matching example, so one
+ -- dead-on example is not diluted by the instance's broader ones
+ assert(approx(s.max({0.7, 0.85}), 0.85), "max picks the best")
+ assert(approx(s.max({1.0, 0.5, 0.4}), 1.0), "max is not diluted")
+ assert(approx(s.max({0.5}), 0.5), "max of a single score")
+ assert(s.max({}) == nil, "max of empty is nil")
+
+ ngx.say("passed")
+ }
+ }
+--- request
+GET /t
+--- response_body
+passed
+--- no_error_log
+[error]
![http://x/y.png"}}](http://x/y.png"}}){kind=link}