This is an automated email from the ASF dual-hosted git repository.
nic-6443 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/apisix.git
The following commit(s) were added to refs/heads/master by this push:
new 7fd4515ee feat(ai-cache): add semantic (L2) cache layer (#13632)
7fd4515ee is described below
commit 7fd4515ee419de72b8877ef4e8c1c2027b59a2b4
Author: Mohammad Izzraff Janius
<[email protected]>
AuthorDate: Thu Jul 2 12:28:15 2026 +0800
feat(ai-cache): add semantic (L2) cache layer (#13632)
---
Makefile | 4 +
apisix/plugins/ai-cache.lua | 119 +-
.../plugins/ai-cache/embeddings/azure_openai.lua | 39 +
apisix/plugins/ai-cache/embeddings/base.lua | 59 +
apisix/plugins/ai-cache/embeddings/openai.lua | 41 +
apisix/plugins/ai-cache/key.lua | 62 +-
apisix/plugins/ai-cache/schema.lua | 85 +-
apisix/plugins/ai-cache/semantic.lua | 324 +++++
apisix/plugins/ai-cache/vector-search/redis.lua | 133 ++
docs/en/latest/plugins/ai-cache.md | 185 ++-
docs/zh/latest/plugins/ai-cache.md | 185 ++-
t/fixtures/openai/embeddings-capital-city.json | 80 +
t/fixtures/openai/embeddings-capital.json | 80 +
t/fixtures/openai/embeddings-largest-city.json | 80 +
t/fixtures/openai/embeddings-tire.json | 80 +
t/lib/ai_cache_mock.lua | 117 ++
t/plugin/ai-cache-semantic.t | 1527 ++++++++++++++++++++
17 files changed, 3167 insertions(+), 33 deletions(-)
diff --git a/Makefile b/Makefile
index 733f62d20..667a0774e 100644
--- a/Makefile
+++ b/Makefile
@@ -403,6 +403,10 @@ install: runtime
$(ENV_INSTALL) -d $(ENV_INST_LUADIR)/apisix/plugins/ai-cache
$(ENV_INSTALL) apisix/plugins/ai-cache/*.lua
$(ENV_INST_LUADIR)/apisix/plugins/ai-cache
+ $(ENV_INSTALL) -d $(ENV_INST_LUADIR)/apisix/plugins/ai-cache/embeddings
+ $(ENV_INSTALL) apisix/plugins/ai-cache/embeddings/*.lua
$(ENV_INST_LUADIR)/apisix/plugins/ai-cache/embeddings
+ $(ENV_INSTALL) -d
$(ENV_INST_LUADIR)/apisix/plugins/ai-cache/vector-search
+ $(ENV_INSTALL) apisix/plugins/ai-cache/vector-search/*.lua
$(ENV_INST_LUADIR)/apisix/plugins/ai-cache/vector-search
$(ENV_INSTALL) -d $(ENV_INST_LUADIR)/apisix/plugins/ai-lakera-guard
$(ENV_INSTALL) apisix/plugins/ai-lakera-guard/*.lua
$(ENV_INST_LUADIR)/apisix/plugins/ai-lakera-guard
diff --git a/apisix/plugins/ai-cache.lua b/apisix/plugins/ai-cache.lua
index ba6f38b43..895ccc7c5 100644
--- a/apisix/plugins/ai-cache.lua
+++ b/apisix/plugins/ai-cache.lua
@@ -20,16 +20,21 @@ local schema = require("apisix.plugins.ai-cache.schema")
local key_mod = require("apisix.plugins.ai-cache.key")
local binding = require("apisix.plugins.ai-protocols.binding")
local redis_util = require("apisix.utils.redis")
+local semantic = require("apisix.plugins.ai-cache.semantic")
local ngx = ngx
local ngx_null = ngx.null
local ipairs = ipairs
+local pcall = pcall
local concat = table.concat
+local str_format = string.format
-local CACHE_STATUS_HEADER = "X-AI-Cache-Status"
-local CACHE_AGE_HEADER = "X-AI-Cache-Age"
-local DEFAULT_TTL = 3600
-local DEFAULT_MAX_BODY = 1048576
+local CACHE_STATUS_HEADER = "X-AI-Cache-Status"
+local CACHE_AGE_HEADER = "X-AI-Cache-Age"
+local CACHE_SIMILARITY_HEADER = "X-AI-Cache-Similarity"
+local DEFAULT_TTL = 3600
+local DEFAULT_MAX_BODY = 1048576
+local DEFAULT_SEMANTIC_TTL = 86400
local _M = {
version = 0.1,
@@ -39,8 +44,31 @@ local _M = {
}
+local function has_layer(conf, name)
+ local layers = conf.layers or { "exact" }
+ for _, l in ipairs(layers) do
+ if l == name then
+ return true
+ end
+ end
+ return false
+end
+
+
function _M.check_schema(conf)
- return core.schema.check(schema, conf)
+ core.utils.check_https({
+ "semantic.embedding.openai.endpoint",
+ "semantic.embedding.azure_openai.endpoint",
+ }, conf, _M.name)
+ local ok, err = core.schema.check(schema, conf)
+ if not ok then
+ return false, err
+ end
+ if conf.semantic and not has_layer(conf, "semantic") then
+ core.log.warn("ai-cache: 'semantic' is configured but not listed in ",
+ "'layers'; the semantic (L2) cache is inactive")
+ end
+ return true
end
@@ -53,12 +81,17 @@ local function release(conf, red)
end
-local function serve_hit(conf, ctx, cached)
- ctx.ai_cache_status = "HIT"
+local function serve_hit(conf, ctx, cached, similarity)
+ local status = "HIT"
+ ctx.ai_cache_status = status
if conf.cache_headers ~= false then
- core.response.set_header(CACHE_STATUS_HEADER, "HIT")
+ core.response.set_header(CACHE_STATUS_HEADER, status)
local age = ngx.time() - (cached.created_at or ngx.time())
core.response.set_header(CACHE_AGE_HEADER, age < 0 and 0 or age)
+ if similarity then
+ core.response.set_header(CACHE_SIMILARITY_HEADER,
+ str_format("%.4f", similarity))
+ end
end
core.response.set_header("Content-Type", "application/json")
return core.response.exit(200, cached.body)
@@ -126,16 +159,56 @@ function _M.access(conf, ctx)
ctx.ai_cache_status = "MISS"
return
end
- release(conf, red)
-
if res ~= nil and res ~= ngx_null then
local cached = core.json.decode(res)
if cached and cached.body then
+ release(conf, red)
return serve_hit(conf, ctx, cached)
end
core.log.warn("ai-cache: discarding malformed cache entry for ",
ctx.ai_cache_key)
end
+ -- L1 miss -> L2 semantic lookup. Release the L1 connection before
+ -- embed_query()'s HTTP call so the pool isn't pinned across the embedding
+ -- round-trip; re-acquire for the vector search. pcall keeps throws
fail-open.
+ if has_layer(conf, "semantic") and conf.semantic then
+ release(conf, red)
+
+ local ok, vec = pcall(semantic.embed_query, conf, ctx, body)
+ if not ok then
+ core.log.warn("ai-cache: semantic embed error, fail-open as MISS:
", vec)
+ vec = nil
+ -- prevent log() from scheduling a write with partial/bad state
+ ctx.ai_cache_embedding = nil
+ end
+
+ if vec then
+ local sred
+ sred, err = redis_util.new(conf)
+ if not sred then
+ core.log.warn("ai-cache: redis unavailable for semantic
search, ",
+ "fail-open as MISS: ", err)
+ else
+ local sok, hit = pcall(semantic.search, sred, conf, ctx, vec)
+ if not sok then
+ sred:close()
+ core.log.warn("ai-cache: semantic search error, fail-open
as MISS: ", hit)
+ ctx.ai_cache_embedding = nil
+ else
+ release(conf, sred)
+ if hit then
+ return serve_hit(conf, ctx,
+ { body = hit.body, created_at = hit.created_at },
hit.similarity)
+ end
+ end
+ end
+ end
+
+ ctx.ai_cache_status = "MISS"
+ return
+ end
+
+ release(conf, red)
ctx.ai_cache_status = "MISS"
end
@@ -174,7 +247,8 @@ end
-- The response-capturing phases (body_filter / log) run in contexts where
-- cosockets are disabled, so the Redis write is deferred to a 0-delay timer
-- (timers run in a light thread where cosockets are allowed).
-local function write_to_cache(premature, conf, cache_key, response_body)
+-- l2 (optional) = { partition, embedding, dim, fingerprint, ttl } for L2
write.
+local function write_to_cache(premature, conf, cache_key, response_body, l2)
if premature then
return
end
@@ -192,6 +266,13 @@ local function write_to_cache(premature, conf, cache_key,
response_body)
core.log.warn("ai-cache: redis set failed: ", err)
return
end
+ if l2 then
+ l2.created_at = ngx.time()
+ local wok, werr = pcall(semantic.write, red, conf, l2, response_body)
+ if not wok then
+ core.log.warn("ai-cache: semantic write error: ", werr)
+ end
+ end
release(conf, red)
end
@@ -218,7 +299,21 @@ function _M.log(conf, ctx)
local response_body = concat(buf, "", 1, buf.n)
local cache_key = key_mod.build(conf, ctx, ctx.ai_cache_fingerprint)
- local ok, err = ngx.timer.at(0, write_to_cache, conf, cache_key,
response_body)
+
+ -- Build the L2 doc from ctx fields stashed by semantic.embed_query(); the
+ -- embedding is only set on a successful embed, so a nil check guards the
write.
+ local l2
+ if has_layer(conf, "semantic") and ctx.ai_cache_embedding then
+ l2 = {
+ partition = ctx.ai_cache_partition,
+ embedding = ctx.ai_cache_embedding,
+ dim = ctx.ai_cache_dim,
+ fingerprint = ctx.ai_cache_fingerprint,
+ ttl = (conf.semantic and conf.semantic.ttl) or
DEFAULT_SEMANTIC_TTL,
+ }
+ end
+
+ local ok, err = ngx.timer.at(0, write_to_cache, conf, cache_key,
response_body, l2)
if not ok then
core.log.warn("ai-cache: failed to schedule cache write: ", err)
end
diff --git a/apisix/plugins/ai-cache/embeddings/azure_openai.lua
b/apisix/plugins/ai-cache/embeddings/azure_openai.lua
new file mode 100644
index 000000000..7421a6a72
--- /dev/null
+++ b/apisix/plugins/ai-cache/embeddings/azure_openai.lua
@@ -0,0 +1,39 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one or more
+-- contributor license agreements. See the NOTICE file distributed with
+-- this work for additional information regarding copyright ownership.
+-- The ASF licenses this file to You under the Apache License, Version 2.0
+-- (the "License"); you may not use this file except in compliance with
+-- the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+local base = require("apisix.plugins.ai-cache.embeddings.base")
+
+local _M = {}
+
+-- get_embeddings(conf, text, httpc, ssl_verify) -> (vector_table, err)
+function _M.get_embeddings(conf, text, httpc, ssl_verify)
+ local req = { input = text }
+ if conf.dimensions then
+ req.dimensions = conf.dimensions
+ end
+ return base.fetch({
+ endpoint = conf.endpoint,
+ headers = {
+ ["Content-Type"] = "application/json",
+ ["api-key"] = conf.api_key,
+ },
+ request = req,
+ httpc = httpc,
+ ssl_verify = ssl_verify,
+ })
+end
+
+return _M
diff --git a/apisix/plugins/ai-cache/embeddings/base.lua
b/apisix/plugins/ai-cache/embeddings/base.lua
new file mode 100644
index 000000000..dda5f1ee2
--- /dev/null
+++ b/apisix/plugins/ai-cache/embeddings/base.lua
@@ -0,0 +1,59 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one or more
+-- contributor license agreements. See the NOTICE file distributed with
+-- this work for additional information regarding copyright ownership.
+-- The ASF licenses this file to You under the Apache License, Version 2.0
+-- (the "License"); you may not use this file except in compliance with
+-- the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
+-- Shared request/parse core for the embeddings drivers. Each provider differs
+-- only in its request shape, auth header, and default endpoint; the HTTP call,
+-- status check, and data[1].embedding extraction live here so a parsing fix
+-- lands in one place.
+local core = require("apisix.core")
+local type = type
+
+local HTTP_OK = ngx.HTTP_OK
+
+local _M = {}
+
+-- fetch(opts) where opts = { endpoint, headers, request, httpc, ssl_verify }
+-- -> (vector_table, err)
+function _M.fetch(opts)
+ local payload, err = core.json.encode(opts.request)
+ if not payload then
+ return nil, "encode embeddings request: " .. (err or "")
+ end
+
+ local res
+ res, err = opts.httpc:request_uri(opts.endpoint, {
+ method = "POST",
+ headers = opts.headers,
+ body = payload,
+ ssl_verify = opts.ssl_verify,
+ })
+ if not res or not res.body then
+ return nil, "embeddings request failed: " .. (err or "")
+ end
+ if res.status ~= HTTP_OK then
+ return nil, "embeddings endpoint returned " .. res.status
+ end
+
+ local decoded = core.json.decode(res.body)
+ if not decoded or type(decoded.data) ~= "table" or type(decoded.data[1])
~= "table"
+ or type(decoded.data[1].embedding) ~= "table" then
+ return nil, "malformed embeddings response"
+ end
+ return decoded.data[1].embedding
+end
+
+return _M
diff --git a/apisix/plugins/ai-cache/embeddings/openai.lua
b/apisix/plugins/ai-cache/embeddings/openai.lua
new file mode 100644
index 000000000..43e123aa5
--- /dev/null
+++ b/apisix/plugins/ai-cache/embeddings/openai.lua
@@ -0,0 +1,41 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one or more
+-- contributor license agreements. See the NOTICE file distributed with
+-- this work for additional information regarding copyright ownership.
+-- The ASF licenses this file to You under the Apache License, Version 2.0
+-- (the "License"); you may not use this file except in compliance with
+-- the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+local base = require("apisix.plugins.ai-cache.embeddings.base")
+
+local _M = {}
+
+_M.DEFAULT_ENDPOINT = "https://api.openai.com/v1/embeddings";
+
+-- get_embeddings(conf, text, httpc, ssl_verify) -> (vector_table, err)
+function _M.get_embeddings(conf, text, httpc, ssl_verify)
+ local req = { model = conf.model, input = text }
+ if conf.dimensions then
+ req.dimensions = conf.dimensions
+ end
+ return base.fetch({
+ endpoint = conf.endpoint or _M.DEFAULT_ENDPOINT,
+ headers = {
+ ["Content-Type"] = "application/json",
+ ["Authorization"] = "Bearer " .. conf.api_key,
+ },
+ request = req,
+ httpc = httpc,
+ ssl_verify = ssl_verify,
+ })
+end
+
+return _M
diff --git a/apisix/plugins/ai-cache/key.lua b/apisix/plugins/ai-cache/key.lua
index 3f5d408ef..bd09cc486 100644
--- a/apisix/plugins/ai-cache/key.lua
+++ b/apisix/plugins/ai-cache/key.lua
@@ -46,17 +46,17 @@ local function client_messages(ctx, body)
end
--- Identity of the EFFECTIVE upstream request, reconstructed from access-time
--- inputs only.
---
--- final_upstream_body = build_request(client_body, ai_client_protocol,
--- instance{provider, options, override})
---
--- is deterministic, and ai-proxy builds it later (in before_proxy) so it
cannot
--- be observed here. Hashing build_request's INPUTS therefore identifies its
--- output uniquely, without invoking the (side-effecting) builder. This is the
--- ONLY place request-determining data lives; scope() below is pure isolation.
-function _M.fingerprint(ctx, body)
+function _M.messages(ctx, body)
+ return client_messages(ctx, body)
+end
+
+
+-- Build the canonical representable struct. `messages` is the message list
+-- folded into the representation: the full client messages for the exact (L1)
+-- fingerprint, or only the response-determining context the embedding does not
+-- cover (system prompts, prior turns, RAG documents) for the semantic (L2)
+-- partition. nil omits message text entirely.
+local function build_repr(ctx, body, messages)
local inst = ctx.picked_ai_instance
local ov = inst.override or {}
@@ -67,14 +67,14 @@ function _M.fingerprint(ctx, body)
end
end
- local repr = core.json.canonical_encode({
+ return {
client = {
protocol = ctx.ai_client_protocol or "",
- messages = client_messages(ctx, body),
+ messages = messages,
params = params,
},
effective = {
- provider = inst.provider,
+ provider = inst.provider,
-- effective model precedence mirrors ai-proxy/base.lua exactly:
-- the instance's options.model wins over the client body model.
model = (inst.options and inst.options.model) or body.model
or "",
@@ -87,8 +87,32 @@ function _M.fingerprint(ctx, body)
-- ARN, vertex project/region/model), so it is
response-determining.
endpoint = ov.endpoint,
},
- })
- return hex_digest(repr)
+ }
+end
+
+
+-- Identity of the EFFECTIVE upstream request, reconstructed from access-time
+-- inputs only.
+--
+-- final_upstream_body = build_request(client_body, ai_client_protocol,
+-- instance{provider, options, override})
+--
+-- is deterministic, and ai-proxy builds it later (in before_proxy) so it
cannot
+-- be observed here. Hashing build_request's INPUTS therefore identifies its
+-- output uniquely, without invoking the (side-effecting) builder. This is the
+-- ONLY place request-determining data lives; scope() below is pure isolation.
+function _M.fingerprint(ctx, body)
+ local repr = build_repr(ctx, body, client_messages(ctx, body))
+ return hex_digest(core.json.canonical_encode(repr))
+end
+
+
+-- Returns the SHA-256 hex digest of the effective context with message text
+-- removed. Queries that differ only in phrasing (same model/params/instance)
+-- share one fingerprint, enabling semantic deduplication without storing the
+-- raw prompt.
+function _M.context_fingerprint(ctx, body)
+ return hex_digest(core.json.canonical_encode(build_repr(ctx, body, nil)))
end
@@ -122,6 +146,12 @@ local function scope(conf, ctx)
end
+function _M.partition(conf, ctx, body, context_messages)
+ local context_repr = core.json.canonical_encode(build_repr(ctx, body,
context_messages))
+ return hex_digest(scope(conf, ctx) .. "|" .. context_repr)
+end
+
+
function _M.build(conf, ctx, fingerprint)
return KEY_PREFIX .. scope(conf, ctx) .. ":" .. fingerprint
end
diff --git a/apisix/plugins/ai-cache/schema.lua
b/apisix/plugins/ai-cache/schema.lua
index 5eb62661c..9be74c97f 100644
--- a/apisix/plugins/ai-cache/schema.lua
+++ b/apisix/plugins/ai-cache/schema.lua
@@ -74,6 +74,78 @@ local _M = {
enum = { "redis" },
default = "redis",
},
+
+ layers = {
+ type = "array",
+ items = { enum = { "exact", "semantic" } },
+ minItems = 1,
+ uniqueItems = true,
+ contains = { const = "exact" },
+ default = { "exact" },
+ },
+
+ semantic = {
+ type = "object",
+ properties = {
+ similarity_threshold = {
+ type = "number", minimum = 0, maximum = 1, default = 0.95,
+ },
+ top_k = { type = "integer", minimum = 1, default = 1 },
+ distance_metric = { enum = { "cosine" }, default = "cosine" },
+ ttl = { type = "integer", minimum = 1, default = 86400 },
+ match = {
+ type = "object",
+ properties = {
+ message_countback = { type = "integer", minimum = 1,
default = 1 },
+ ignore_system_prompts = { type = "boolean", default =
true },
+ ignore_assistant_prompts = { type = "boolean", default
= true },
+ ignore_tool_prompts = { type = "boolean", default =
true },
+ },
+ default = {},
+ },
+ embedding = {
+ type = "object",
+ properties = {
+ openai = {
+ type = "object",
+ properties = {
+ endpoint = { type = "string" },
+ model = { type = "string" },
+ api_key = { type = "string" },
+ dimensions = { type = "integer", minimum = 1 },
+ ssl_verify = { type = "boolean", default =
true },
+ timeout = { type = "integer", minimum = 1,
default = 5000 },
+ },
+ required = { "model", "api_key" },
+ },
+ azure_openai = {
+ type = "object",
+ properties = {
+ endpoint = { type = "string" },
+ api_key = { type = "string" },
+ dimensions = { type = "integer", minimum = 1 },
+ ssl_verify = { type = "boolean", default =
true },
+ timeout = { type = "integer", minimum = 1,
default = 5000 },
+ },
+ required = { "endpoint", "api_key" },
+ },
+ },
+ oneOf = { { required = { "openai" } }, { required = {
"azure_openai" } } },
+ },
+ vector_search = {
+ type = "object",
+ properties = {
+ redis = {
+ type = "object",
+ properties = { index = { type = "string", default
= "ai-cache" } },
+ default = {},
+ },
+ },
+ required = { "redis" },
+ },
+ },
+ required = { "embedding", "vector_search" },
+ },
},
["if"] = {
properties = {
@@ -83,7 +155,18 @@ local _M = {
},
},
["then"] = policy_to_additional_properties.redis,
- encrypt_fields = { "redis_password" },
+ allOf = {
+ {
+ ["if"] = { properties = { layers = { contains = { const =
"semantic" } } },
+ required = { "layers" } },
+ ["then"] = { required = { "semantic" } },
+ },
+ },
+ encrypt_fields = {
+ "redis_password",
+ "semantic.embedding.openai.api_key",
+ "semantic.embedding.azure_openai.api_key",
+ },
}
return _M
diff --git a/apisix/plugins/ai-cache/semantic.lua
b/apisix/plugins/ai-cache/semantic.lua
new file mode 100644
index 000000000..4c603a9dc
--- /dev/null
+++ b/apisix/plugins/ai-cache/semantic.lua
@@ -0,0 +1,324 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one or more
+-- contributor license agreements. See the NOTICE file distributed with
+-- this work for additional information regarding copyright ownership.
+-- The ASF licenses this file to You under the Apache License, Version 2.0
+-- (the "License"); you may not use this file except in compliance with
+-- the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+local core = require("apisix.core")
+local http = require("resty.http")
+local sha256 = require("resty.sha256")
+local to_hex = require("resty.string").to_hex
+local key_mod = require("apisix.plugins.ai-cache.key")
+local vs = require("apisix.plugins.ai-cache.vector-search.redis")
+
+local ipairs = ipairs
+local type = type
+local concat = table.concat
+local tostring = tostring
+
+-- Pre-require both drivers so a misconfigured provider name cannot escape
+-- lookup()'s fail-open boundary via a request-time require() raise.
+local drivers = {
+ openai = require("apisix.plugins.ai-cache.embeddings.openai"),
+ azure_openai = require("apisix.plugins.ai-cache.embeddings.azure_openai"),
+}
+
+local _M = {}
+
+local DEFAULT_THRESHOLD = 0.95
+local DEFAULT_TOP_K = 1
+
+-- Semantic L2 reconstructs the prompt through the client protocol's
+-- get_messages(): the last turn becomes the embedded query and the rest
becomes
+-- the partition context. That is only sound when get_messages() is a faithful,
+-- lossless view of the prompt. Today only openai-chat qualifies -- it returns
+-- body.messages verbatim -- whereas the other protocols drop all non-text
+-- content (images, tool calls, structured input), so two distinct prompts
could
+-- canonicalise to the same messages and collide on one cache cell. L2
therefore
+-- engages only for these protocols and bypasses (exact L1 still applies) for
the
+-- rest, rather than risk a cross-prompt false hit.
+local SEMANTIC_PROTOCOLS = {
+ ["openai-chat"] = true,
+}
+
+
+local function text_of(content)
+ if type(content) == "string" then
+ return content
+ end
+ if type(content) == "table" then
+ local parts = {}
+ for _, block in ipairs(content) do
+ if type(block) == "table" and block.type == "text" and block.text
then
+ parts[#parts + 1] = block.text
+ end
+ end
+ return concat(parts, "\n")
+ end
+ return ""
+end
+
+
+-- Indices (into `messages`) of the messages whose text is embedded: the last
+-- `message_countback` messages surviving the role filters. Shared by
+-- extract_embed_text (which embeds them) and context_messages (which excludes
+-- them from the L2 partition) so the two can never disagree on the split.
+local function embed_window(messages, match)
+ local m = match or {}
+ local kept = {}
+ for i, msg in ipairs(messages) do
+ local role = msg.role
+ local skip = (role == "system" and m.ignore_system_prompts ~= false)
+ or (role == "assistant" and m.ignore_assistant_prompts ~=
false)
+ or (role == "tool" and m.ignore_tool_prompts ~= false)
+ if not skip then
+ kept[#kept + 1] = i
+ end
+ end
+ local countback = m.message_countback or 1
+ local start = #kept - countback + 1
+ if start < 1 then start = 1 end
+ local window = {}
+ for w = start, #kept do
+ window[#window + 1] = kept[w]
+ end
+ return window
+end
+
+
+function _M.extract_embed_text(messages, match)
+ local texts = {}
+ for _, i in ipairs(embed_window(messages, match)) do
+ local t = text_of(messages[i].content)
+ if t ~= "" then
+ texts[#texts + 1] = t
+ end
+ end
+ return concat(texts, "\n")
+end
+
+
+-- The messages NOT in the embed window: response-determining context (system
+-- prompts, prior turns, RAG documents) that the embedding ignores. The
semantic
+-- layer folds these into the L2 partition so a generic instruction over
+-- different context never collides on another context's cached response.
+function _M.context_messages(messages, match)
+ local in_window = {}
+ for _, i in ipairs(embed_window(messages, match)) do
+ in_window[i] = true
+ end
+ local context = {}
+ for i, msg in ipairs(messages) do
+ if not in_window[i] then
+ context[#context + 1] = msg
+ end
+ end
+ return context
+end
+
+
+function _M.window_has_nontext(messages, match)
+ for _, i in ipairs(embed_window(messages, match)) do
+ local content = messages[i].content
+ if type(content) == "table" then
+ for _, block in ipairs(content) do
+ if type(block) == "table" and block.type and block.type ~=
"text" then
+ return true
+ end
+ end
+ end
+ end
+ return false
+end
+
+
+-- Base name for this plugin instance's RediSearch index and L2 key prefix.
+-- The schema requires vector_search and defaults redis.index to "ai-cache",
+-- so the validated conf always carries the value.
+local function l2_base(conf)
+ return conf.semantic.vector_search.redis.index
+end
+
+
+-- Short, stable fingerprint of the EMBEDDING model space (provider + model +
+-- endpoint + dimensions). Cosine distance is only meaningful between vectors
+-- from the same model, so this is folded into the index name and key prefix:
+-- changing the embedding model (even to one of the same dimensionality) lands
+-- in a fresh, isolated index instead of being compared against stale vectors
+-- from the previous model.
+local function emb_identity(conf)
+ local emb = conf.semantic.embedding
+ local provider = emb.openai and "openai" or "azure_openai"
+ local c = emb[provider]
+ local endpoint = c.endpoint
+ if provider == "openai" then
+ endpoint = endpoint or drivers.openai.DEFAULT_ENDPOINT
+ end
+ local repr = provider .. "|" .. (c.model or "") .. "|" .. (endpoint or "")
+ .. "|" .. tostring(c.dimensions or "")
+ local h = sha256:new()
+ h:update(repr)
+ return to_hex(h:final()):sub(1, 16)
+end
+
+
+function _M.index_name(conf, dim)
+ return l2_base(conf) .. ":idx:" .. emb_identity(conf) .. ":" .. dim
+end
+
+
+-- HASH key prefix the index is built over. Scoped by embedding identity AND
+-- dimension so two indexes (different model or different dim) never share
docs.
+local function l2_prefix(conf, dim)
+ return l2_base(conf) .. ":l2:" .. emb_identity(conf) .. ":" .. dim .. ":"
+end
+
+
+-- host#port#db identity, folded into the FT.CREATE memo key so the same index
+-- name against different Redis targets is created on each (see redis.lua).
+local function redis_target(conf)
+ return (conf.redis_host or "") .. "#" .. (conf.redis_port or 6379)
+ .. "#" .. (conf.redis_database or 0)
+end
+
+
+-- conf.semantic.embedding is a one-key sub-object {openai=..|azure_openai=..}
+local function embed(conf, text)
+ local emb = conf.semantic.embedding
+ local provider = emb.openai and "openai" or "azure_openai"
+ local driver = drivers[provider]
+ local pconf = emb[provider]
+ local httpc = http.new()
+ -- Bound the synchronous embedding call so a slow/hung provider cannot
stall
+ -- the request for the resty default (~60s) before fail-open kicks in.
+ local t = pconf.timeout or 5000
+ httpc:set_timeouts(t, t, t)
+ return driver.get_embeddings(pconf, text, httpc, pconf.ssl_verify ~= false)
+end
+
+
+-- Phase 1 of the L2 lookup: gates + embedding. Does no Redis work -- embed()
is
+-- an HTTP call the caller must not pin a pooled connection across. Returns the
+-- query vector (stashing the write-back ctx fields) or nil to fail open as
MISS.
+function _M.embed_query(conf, ctx, body)
+ -- Bypass L2 (exact L1 still applies) for protocols whose canonical message
+ -- form cannot faithfully represent the prompt; never a cross-prompt hit.
+ if not SEMANTIC_PROTOCOLS[ctx.ai_client_protocol or ""] then
+ return nil
+ end
+ local sem = conf.semantic
+ local messages = key_mod.messages(ctx, body)
+ -- Bypass L2 when an embedded message carries non-text content (images,
etc.):
+ -- it is absent from both the vector and the partition, so a same-text
+ -- different-image prompt would otherwise collide on one L2 cell.
+ if _M.window_has_nontext(messages, sem.match) then
+ return nil
+ end
+ local text = _M.extract_embed_text(messages, sem.match)
+ if text == "" then
+ return nil
+ end
+
+ local vec, err = embed(conf, text)
+ if not vec then
+ core.log.warn("ai-cache: embedding failed, fail-open as MISS: ", err)
+ return nil
+ end
+ -- stash for the write-back in log() (only set when embedding succeeded).
+ -- context = the response-determining messages the embedding ignores;
folding
+ -- them into the partition isolates this prompt from other contexts. nil
for a
+ -- plain single-turn prompt, which keeps the partition identical to before.
+ local ctxmsgs = _M.context_messages(messages, sem.match)
+ local part_ctx = #ctxmsgs > 0 and ctxmsgs or nil
+ ctx.ai_cache_embedding = vec
+ ctx.ai_cache_dim = #vec
+ ctx.ai_cache_partition = key_mod.partition(conf, ctx, body, part_ctx)
+ return vec
+end
+
+
+-- Phase 2 of the L2 lookup: vector search over a caller-owned connection
+-- acquired AFTER embed_query() (so the pool isn't pinned across embedding).
+-- Returns a hit {body, created_at, similarity} on a >=threshold match, else
nil.
+function _M.search(red, conf, ctx, vec)
+ local sem = conf.semantic
+ local target = redis_target(conf)
+ local index = _M.index_name(conf, #vec)
+ local ok, err = vs.ensure_index(red, target, index, l2_prefix(conf, #vec),
#vec)
+ if not ok then
+ core.log.warn("ai-cache: ensure_index failed, fail-open as MISS: ",
err)
+ return nil
+ end
+
+ local hit
+ hit, err = vs.knn_search(red, target, index, ctx.ai_cache_partition, vec,
+ sem.top_k or DEFAULT_TOP_K)
+ if err then
+ core.log.warn("ai-cache: knn search failed, fail-open as MISS: ", err)
+ return nil
+ end
+ if not hit then
+ return nil
+ end
+
+ local similarity = 1 - hit.distance
+ if similarity < (sem.similarity_threshold or DEFAULT_THRESHOLD) then
+ return nil
+ end
+
+ -- L2 -> L1 backfill, carrying the L2 entry's original created_at so Age is
+ -- consistent whether the next hit is served from L1 or L2. A real
semantic
+ -- hit must be served regardless — only the backfill SET is skipped on
error.
+ local envelope = core.json.encode({ body = hit.response, created_at =
hit.created_at })
+ if not envelope then
+ core.log.warn("ai-cache: L1 backfill skipped: json.encode returned
nil")
+ else
+ local exact_ttl = (conf.exact and conf.exact.ttl) or 3600
+ local bok, berr = red:set(ctx.ai_cache_key, envelope, "EX", exact_ttl)
+ if not bok then
+ core.log.warn("ai-cache: L1 backfill SET failed: ", berr)
+ end
+ end
+
+ return { body = hit.response, created_at = hit.created_at, similarity =
similarity }
+end
+
+
+-- Called from the write-back timer (after the L1 SET) with a still-open `red`.
+-- l2 = { partition, embedding, dim, fingerprint, ttl, created_at }
+function _M.write(red, conf, l2, response_body)
+ if not l2 or not l2.embedding then
+ return
+ end
+ local target = redis_target(conf)
+ local index = _M.index_name(conf, l2.dim)
+ local prefix = l2_prefix(conf, l2.dim)
+ local ok, err = vs.ensure_index(red, target, index, prefix, l2.dim)
+ if not ok then
+ core.log.warn("ai-cache: ensure_index on write failed: ", err)
+ return
+ end
+ local doc_key = prefix .. l2.partition .. ":" .. l2.fingerprint
+ ok, err = vs.upsert(red, doc_key, {
+ partition = l2.partition,
+ embedding = vs.pack_float32(l2.embedding),
+ response = response_body,
+ created_at = l2.created_at,
+ }, l2.ttl)
+ if not ok then
+ core.log.warn("ai-cache: L2 upsert failed: ", err)
+ end
+end
+
+
+return _M
diff --git a/apisix/plugins/ai-cache/vector-search/redis.lua
b/apisix/plugins/ai-cache/vector-search/redis.lua
new file mode 100644
index 000000000..9b2adeaf7
--- /dev/null
+++ b/apisix/plugins/ai-cache/vector-search/redis.lua
@@ -0,0 +1,133 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one or more
+-- contributor license agreements. See the NOTICE file distributed with
+-- this work for additional information regarding copyright ownership.
+-- The ASF licenses this file to You under the Apache License, Version 2.0
+-- (the "License"); you may not use this file except in compliance with
+-- the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+local ffi = require("ffi")
+local ffi_new = ffi.new
+local ffi_str = ffi.string
+local tonumber = tonumber
+local ipairs = ipairs
+local type = type
+
+local _M = {}
+
+local ensured = {}
+
+
+-- Memo keyed by Redis target (host#port#db) + index, not index alone: the same
+-- index name against different Redis servers must be created on each.
+local function memo_key(target, index)
+ return target .. "|" .. index
+end
+
+
+-- little-endian FLOAT32 blob (RediSearch VECTOR PARAMS / HSET value)
+function _M.pack_float32(vec)
+ local n = #vec
+ local buf = ffi_new("float[?]", n)
+ for i = 1, n do
+ buf[i - 1] = vec[i]
+ end
+ return ffi_str(buf, n * 4)
+end
+
+
+function _M.ensure_index(red, target, index, prefix, dim)
+ local mk = memo_key(target, index)
+ if ensured[mk] then
+ return true
+ end
+ local ok, err = red[ "FT.CREATE" ](red, index,
+ "ON", "HASH", "PREFIX", 1, prefix,
+ "SCHEMA",
+ "partition", "TAG",
+ "embedding", "VECTOR", "HNSW", 6,
+ "TYPE", "FLOAT32", "DIM", dim, "DISTANCE_METRIC", "COSINE")
+ if not ok then
+ -- FT.CREATE on an existing index returns this error; treat as success.
+ if err and err:find("Index already exists", 1, true) then
+ ensured[mk] = true
+ return true
+ end
+ return nil, err
+ end
+ ensured[mk] = true
+ return true
+end
+
+
+function _M.upsert(red, doc_key, fields, ttl)
+ red:init_pipeline()
+ red[ "HSET" ](red, doc_key,
+ "partition", fields.partition,
+ "embedding", fields.embedding,
+ "response", fields.response,
+ "created_at", fields.created_at)
+ red:expire(doc_key, ttl)
+ local res, err = red:commit_pipeline()
+ if not res then
+ return nil, err
+ end
+ for _, reply in ipairs(res) do
+ if type(reply) == "table" and reply[1] == false then
+ return nil, reply[2]
+ end
+ end
+ return true
+end
+
+
+-- Returns the nearest hit { distance, response, created_at } or nil (no err)
on
+-- an empty result set.
+function _M.knn_search(red, target, index, partition, vec, top_k)
+ local query = "(@partition:{" .. partition .. "})=>[KNN " .. top_k ..
+ " @embedding $vec AS __score]"
+ local res, err = red[ "FT.SEARCH" ](red, index, query,
+ "PARAMS", 2, "vec", _M.pack_float32(vec),
+ "RETURN", 3, "__score", "response", "created_at",
+ "SORTBY", "__score",
+ "DIALECT", 2)
+ if not res then
+ -- self-heal: clear the memo on error so the next call re-runs
FT.CREATE.
+ ensured[memo_key(target, index)] = nil
+ return nil, err
+ end
+ -- RESP: { total, docKey1, {f, v, f, v, ...}, docKey2, {...}, ... }
+ if type(res) ~= "table" or (tonumber(res[1]) or 0) < 1 then
+ return nil
+ end
+ local fields = res[3]
+ if type(fields) ~= "table" then
+ return nil
+ end
+ local hit = {}
+ for i = 1, #fields, 2 do
+ local k, v = fields[i], fields[i + 1]
+ if k == "__score" then
+ hit.distance = tonumber(v)
+ elseif k == "response" then
+ hit.response = v
+ elseif k == "created_at" then
+ hit.created_at = tonumber(v)
+ end
+ end
+ if not hit.response or not hit.distance then
+ return nil
+ end
+ return hit
+end
+
+
+return _M
diff --git a/docs/en/latest/plugins/ai-cache.md
b/docs/en/latest/plugins/ai-cache.md
index bbeacdf73..d16a9a57f 100644
--- a/docs/en/latest/plugins/ai-cache.md
+++ b/docs/en/latest/plugins/ai-cache.md
@@ -40,7 +40,10 @@ import TabItem from '@theme/TabItem';
The `ai-cache` Plugin caches LLM responses and replays them for later requests
that resolve to the same prompt, cutting upstream token cost and latency for
repetitive workloads (FAQ bots, document Q&A, translation).
-This release implements the **exact** cache layer (L1); a semantic cache layer
(L2) is planned for a future release.
+This Plugin supports two cache layers:
+
+- **Exact (L1):** A SHA-256 fingerprint of the effective prompt is used as the
Redis key. An identical prompt always hits the same entry.
+- **Semantic (L2):** When L1 misses, the prompt is embedded into a vector and
a nearest-neighbour search retrieves a past response whose embedding is within
the configured similarity threshold. L2 is disabled by default; enable it by
adding `"semantic"` to `layers`.
The `ai-cache` Plugin must be used with the [`ai-proxy`](./ai-proxy.md) or
[`ai-proxy-multi`](./ai-proxy-multi.md) Plugin.
@@ -61,12 +64,13 @@ Even with `cache_key.share_across_routes` enabled, the
cache key identifies the
| cache_key.include_consumer | boolean | False | false | | If true, scope the
cache per consumer so entries are not shared across consumers. |
| cache_key.include_vars | array[string] | False | [] | | NGINX variables
added to the cache scope (for example `["http_x_tenant"]`), isolating entries
by their values. |
| max_cache_body_size | integer | False | 1048576 | >= 0 | Maximum response
body size, in bytes, to cache. Larger responses are not cached. |
-| cache_headers | boolean | False | true | | If true, add the
`X-AI-Cache-Status` response header (and `X-AI-Cache-Age`, the entry age in
seconds, on a hit). |
+| cache_headers | boolean | False | true | | If true, emit the following
response headers: `X-AI-Cache-Status` (always), one of `MISS`, `HIT` (exact or
semantic cache hit), or `BYPASS`; `X-AI-Cache-Age`, the entry age in seconds,
on any cache hit; `X-AI-Cache-Similarity`, the cosine similarity score (0–1)
between the incoming prompt and the matched entry, on a semantic cache hit
only. |
| fail_mode | string | False | `"skip"` | `skip`, `warn`, `error` | Behavior
when the request is not a recognized AI request that this Plugin can cache (for
example, a request that did not pass through `ai-proxy` or `ai-proxy-multi`).
`skip`: let the request pass through uncached; `warn`: pass through uncached
and log a warning; `error`: reject the request. |
| bypass_on | array[object] | False | | | Rules that skip the cache entirely
(no lookup, no write-back) when any rule matches. |
| bypass_on[].header | string | True | | | Request header name to match. |
| bypass_on[].equals | string | True | | | Bypass when the request header's
value exactly equals this string. |
| policy | string | False | redis | redis | Storage backend. Only single-node
`redis` is available in this release. |
+| layers | array[string] | False | ["exact"] | exact, semantic | Cache layers
to activate. `exact` performs an exact-match fingerprint lookup (L1) and is
always active; `"exact"` must always be present in this array. `semantic`
enables a vector-similarity lookup (L2) that is consulted only on an L1 miss.
At least one value is required; values must be unique. |
| redis_host | string | True | | | Address of the Redis node. |
| redis_port | integer | False | 6379 | >= 1 | Port of the Redis node. |
| redis_username | string | False | | | Username for Redis if Redis ACL is
used. For the legacy `requirepass` method, configure only `redis_password`. |
@@ -78,6 +82,54 @@ Even with `cache_key.share_across_routes` enabled, the cache
key identifies the
| redis_keepalive_timeout | integer | False | 10000 | >= 1000 | Keepalive
timeout, in milliseconds, for the Redis connection pool. |
| redis_keepalive_pool | integer | False | 100 | >= 1 | Maximum number of
connections in the Redis keepalive pool. |
+### Semantic (L2) Attributes
+
+:::caution Redis Stack required for semantic caching
+
+When `"semantic"` is included in `layers`, the configured Redis instance
**must** be [Redis Stack](https://redis.io/docs/stack/) (with the RediSearch
module). The L1 exact cache and the L2 semantic cache share the same Redis
connection configured by `redis_host` / `redis_port` etc.
+
+Vanilla Redis is sufficient when `layers` is omitted or contains only
`"exact"` (the default).
+
+:::
+
+The `semantic` object is required when `"semantic"` is present in `layers`. It
accepts the following attributes:
+
+| Name | Type | Required | Default | Valid values | Description |
+|------|------|----------|---------|--------------|-------------|
+| semantic.similarity_threshold | number | False | 0.95 | [0, 1] | Minimum
cosine similarity (equivalently, 1 − distance) required to consider a retrieved
vector a match. Requests below this threshold fall through to the upstream. |
+| semantic.top_k | integer | False | 1 | >= 1 | Number of nearest-neighbour
candidates to retrieve from the vector index. Only the highest-scoring result
is evaluated against `similarity_threshold`. |
+| semantic.distance_metric | string | False | `"cosine"` | `cosine` | Vector
distance metric. Only `cosine` is currently supported. |
+| semantic.ttl | integer | False | 86400 | >= 1 | Time-to-live, in seconds, of
a semantic (L2) cache entry. |
+| semantic.match.message_countback | integer | False | 1 | >= 1 | Number of
trailing `user`-role messages to include in the embedding input. |
+| semantic.match.ignore_system_prompts | boolean | False | true | | If true,
`system`-role messages are excluded from the embedding input. |
+| semantic.match.ignore_assistant_prompts | boolean | False | true | | If
true, `assistant`-role messages are excluded from the embedding input. |
+| semantic.match.ignore_tool_prompts | boolean | False | true | | If true,
`tool`-role messages are excluded from the embedding input. |
+| semantic.embedding | object | **True** | | | Embedding provider
configuration. Exactly one of `openai` or `azure_openai` must be specified. |
+| semantic.embedding.openai.endpoint | string | False | | | OpenAI-compatible
embedding API endpoint URL. Defaults to the public OpenAI API when omitted. |
+| semantic.embedding.openai.model | string | **True** | | | Embedding model
name (for example, `text-embedding-3-small`). |
+| semantic.embedding.openai.api_key | string | **True** | | | OpenAI API key.
Encrypted at rest in etcd. |
+| semantic.embedding.openai.dimensions | integer | False | | >= 1 | Override
the embedding output dimension for models that support it. |
+| semantic.embedding.openai.ssl_verify | boolean | False | true | | If true,
verifies the embedding service's certificate. |
+| semantic.embedding.openai.timeout | integer | False | 5000 | >= 1 | Request
timeout in milliseconds for the embedding service. |
+| semantic.embedding.azure_openai.endpoint | string | **True** | | | Azure
OpenAI deployment endpoint URL. |
+| semantic.embedding.azure_openai.api_key | string | **True** | | | Azure
OpenAI API key. Encrypted at rest in etcd. |
+| semantic.embedding.azure_openai.dimensions | integer | False | | >= 1 |
Override the embedding output dimension. |
+| semantic.embedding.azure_openai.ssl_verify | boolean | False | true | | If
true, verifies the embedding service's certificate. |
+| semantic.embedding.azure_openai.timeout | integer | False | 5000 | >= 1 |
Request timeout in milliseconds for the embedding service. |
+| semantic.vector_search | object | **True** | | | Vector index configuration.
|
+| semantic.vector_search.redis.index | string | False | `"ai-cache"` | |
RediSearch index name used as the vector store. |
+
+:::note Security: multi-tenant deployments
+
+Cache entries are scoped per route by default. In a multi-tenant deployment
where multiple consumers share a route, a cached response produced for one
consumer could be served to another. To prevent cross-tenant leakage:
+
+- Set `cache_key.include_consumer: true` to scope entries per consumer
identity.
+- Use `cache_key.include_vars` to add tenant-identifying NGINX variables (for
example `["http_x_tenant_id"]`) to the cache scope.
+
+Both L1 and L2 entries respect the same `cache_key` scoping rules.
+
+:::
+
## Example
The example below uses OpenAI as the Upstream LLM provider. Obtain an [OpenAI
API key](https://openai.com/blog/openai-api) and save it, along with your Admin
API key, to environment variables:
@@ -333,3 +385,132 @@ curl -i "http://127.0.0.1:9080/anything"; -X POST \
```
The cache is skipped entirely (no lookup and no write-back), and the response
carries the `X-AI-Cache-Status: BYPASS` header.
+
+### Cache LLM Responses with Semantic Matching
+
+This example adds the semantic (L2) cache layer so that near-duplicate prompts
are served from cache even when the wording differs slightly. In addition to a
running Redis Stack instance, an OpenAI API key is needed for the embedding
service.
+
+:::caution
+
+The Redis instance must be [Redis Stack](https://redis.io/docs/stack/)
(RediSearch module). Vanilla Redis is not supported for semantic caching.
+
+:::
+
+<Tabs
+groupId="api"
+defaultValue="admin-api"
+values={[
+{label: 'Admin API', value: 'admin-api'},
+{label: 'ADC', value: 'adc'}
+]}>
+
+<TabItem value="admin-api">
+
+```shell
+curl "http://127.0.0.1:9180/apisix/admin/routes"; -X PUT \
+ -H "X-API-KEY: ${admin_key}" \
+ -d '{
+ "id": "ai-cache-semantic-route",
+ "uri": "/anything",
+ "plugins": {
+ "ai-proxy": {
+ "provider": "openai",
+ "auth": { "header": { "Authorization": "Bearer '"$OPENAI_API_KEY"'" }
},
+ "options": { "model": "gpt-4o" }
+ },
+ "ai-cache": {
+ "redis_host": "127.0.0.1",
+ "layers": ["exact", "semantic"],
+ "semantic": {
+ "similarity_threshold": 0.92,
+ "embedding": {
+ "openai": {
+ "model": "text-embedding-3-small",
+ "api_key": "'"$OPENAI_API_KEY"'"
+ }
+ },
+ "vector_search": {
+ "redis": {
+ "index": "ai-cache"
+ }
+ }
+ }
+ }
+ }
+ }'
+```
+
+</TabItem>
+
+<TabItem value="adc">
+
+```yaml title="adc.yaml"
+services:
+ - name: ai-cache-semantic-service
+ routes:
+ - name: ai-cache-semantic-route
+ uris:
+ - /anything
+ methods:
+ - POST
+ plugins:
+ ai-proxy:
+ provider: openai
+ auth:
+ header:
+ Authorization: "Bearer ${OPENAI_API_KEY}"
+ options:
+ model: gpt-4o
+ ai-cache:
+ redis_host: 127.0.0.1
+ layers:
+ - exact
+ - semantic
+ semantic:
+ similarity_threshold: 0.92
+ embedding:
+ openai:
+ model: text-embedding-3-small
+ api_key: "${OPENAI_API_KEY}"
+ vector_search:
+ redis:
+ index: ai-cache
+```
+
+Synchronize the configuration to the gateway:
+
+```shell
+adc sync -f adc.yaml
+```
+
+</TabItem>
+
+</Tabs>
+
+Send an initial request:
+
+```shell
+curl -i "http://127.0.0.1:9080/anything"; -X POST \
+ -H "Content-Type: application/json" \
+ -d '{ "messages": [{ "role": "user", "content": "What is Apache APISIX?" }]
}'
+```
+
+The first request is an L1 and L2 miss; the Plugin proxies it to the LLM,
embeds the prompt, and stores both the exact entry and the vector in Redis. The
response carries `X-AI-Cache-Status: MISS`.
+
+Send a semantically similar but differently worded request:
+
+```shell
+curl -i "http://127.0.0.1:9080/anything"; -X POST \
+ -H "Content-Type: application/json" \
+ -d '{ "messages": [{ "role": "user", "content": "Can you explain what Apache
APISIX is?" }] }'
+```
+
+This request misses L1 (different fingerprint) but hits L2 (similar
embedding). The response is served from the semantic cache with:
+
+```text
+X-AI-Cache-Status: HIT
+X-AI-Cache-Age: 12
+X-AI-Cache-Similarity: 0.9487
+```
+
+The `X-AI-Cache-Similarity` header reports the cosine similarity (1 −
distance) between the incoming prompt and the matched cache entry.
diff --git a/docs/zh/latest/plugins/ai-cache.md
b/docs/zh/latest/plugins/ai-cache.md
index 3793317bd..16f3c2b7c 100644
--- a/docs/zh/latest/plugins/ai-cache.md
+++ b/docs/zh/latest/plugins/ai-cache.md
@@ -40,7 +40,10 @@ import TabItem from '@theme/TabItem';
`ai-cache` 插件缓存 LLM 响应，并在后续解析到相同提示词的请求中重放这些响应，从而为重复性工作负载（FAQ
机器人、文档问答、翻译等）降低上游的 Token 消耗与延迟。
-本次发布实现了**精确**缓存层（L1）；语义缓存层（L2）计划在未来的版本中提供。
+该插件支持两个缓存层：
+
+- **精确缓存（L1）：** 对有效提示词计算 SHA-256 指纹并用作 Redis 键。完全相同的提示词始终命中同一条缓存条目。
+- **语义缓存（L2）：** 当 L1 未命中时，将提示词向量化，并通过最近邻搜索检索相似度在阈值以上的历史响应。L2 默认关闭；在 `layers`
中加入 `"semantic"` 即可启用。
`ai-cache` 插件必须与 [`ai-proxy`](./ai-proxy.md) 或
[`ai-proxy-multi`](./ai-proxy-multi.md) 插件一起使用。
@@ -61,12 +64,13 @@ import TabItem from '@theme/TabItem';
| cache_key.include_consumer | boolean | 否 | false | | 如果为
true，则按消费者隔离缓存，使缓存条目不会在不同消费者之间共享。 |
| cache_key.include_vars | array[string] | 否 | [] | | 加入缓存作用域的 NGINX 变量（例如
`["http_x_tenant"]`），按其取值隔离缓存条目。 |
| max_cache_body_size | integer | 否 | 1048576 | >= 0 |
允许缓存的最大响应体大小，单位为字节。超过该大小的响应不会被缓存。 |
-| cache_headers | boolean | 否 | true | | 如果为 true，则添加 `X-AI-Cache-Status`
响应头（命中时还会添加 `X-AI-Cache-Age`，表示缓存条目的存在时长，单位为秒）。 |
+| cache_headers | boolean | 否 | true | | 如果为
true，则输出以下响应头：`X-AI-Cache-Status`（始终输出），取值为 `MISS`、`HIT`（精确或语义缓存命中）或
`BYPASS`；`X-AI-Cache-Age`，表示缓存条目的存在时长（秒），在任意缓存命中时输出；`X-AI-Cache-Similarity`，表示请求提示词与命中条目之间的余弦相似度（0–1），仅在语义缓存命中时输出。
|
| fail_mode | string | 否 | `"skip"` | `skip`、`warn`、`error` | 当请求不是该插件可缓存的 AI
请求时的处理行为（例如未经过 `ai-proxy` 或 `ai-proxy-multi`
的请求）。`skip`：放行请求且不缓存；`warn`：放行不缓存并记录 warning 日志；`error`：拒绝请求。 |
| bypass_on | array[object] | 否 | | | 当任一规则匹配时，完全跳过缓存（不查询、不回写）的规则列表。 |
| bypass_on[].header | string | 是 | | | 要匹配的请求头名称。 |
| bypass_on[].equals | string | 是 | | | 当该请求头的值与此字符串完全相等时，绕过缓存。 |
| policy | string | 否 | redis | redis | 存储后端。本次发布仅支持单节点 `redis`。 |
+| layers | array[string] | 否 | ["exact"] | exact, semantic | 要启用的缓存层。`exact`
执行精确指纹匹配（L1），始终处于激活状态，数组中必须包含 `"exact"`；`semantic` 启用向量相似度匹配（L2），仅在 L1
未命中时查询。至少需要一个值，且不可重复。 |
| redis_host | string | 是 | | | Redis 节点的地址。 |
| redis_port | integer | 否 | 6379 | >= 1 | Redis 节点的端口。 |
| redis_username | string | 否 | | | 使用 Redis ACL 时的用户名。如果使用传统的 `requirepass`
认证方式，则仅配置 `redis_password`。 |
@@ -78,6 +82,54 @@ import TabItem from '@theme/TabItem';
| redis_keepalive_timeout | integer | 否 | 10000 | >= 1000 | Redis
连接池的保活超时时间，单位为毫秒。 |
| redis_keepalive_pool | integer | 否 | 100 | >= 1 | Redis 保活连接池中的最大连接数。 |
+### 语义缓存（L2）属性
+
+:::caution 语义缓存需要 Redis Stack
+
+当 `layers` 中包含 `"semantic"` 时，所配置的 Redis 实例**必须**为 [Redis
Stack](https://redis.io/docs/stack/)（含 RediSearch 模块）。L1 精确缓存与 L2 语义缓存共用同一个由
`redis_host` / `redis_port` 等参数配置的 Redis 连接。
+
+若 `layers` 省略或仅包含 `"exact"`（默认值），则使用普通 Redis 即可。
+
+:::
+
+当 `layers` 中包含 `"semantic"` 时，`semantic` 对象为必填项，其属性如下：
+
+| 名称 | 类型 | 必选项 | 默认值 | 有效值 | 描述 |
+|------|------|--------|--------|--------|------|
+| semantic.similarity_threshold | number | 否 | 0.95 | [0, 1] |
将检索向量视为匹配所需的最小余弦相似度（即 1 − 距离）。低于该阈值的请求将透传至上游。 |
+| semantic.top_k | integer | 否 | 1 | >= 1 | 从向量索引中检索的最近邻候选数量。只有得分最高的结果会与
`similarity_threshold` 进行比较。 |
+| semantic.distance_metric | string | 否 | `"cosine"` | `cosine` |
向量距离度量方式。目前仅支持 `cosine`（余弦距离）。 |
+| semantic.ttl | integer | 否 | 86400 | >= 1 | 语义缓存（L2）条目的存活时间（TTL），单位为秒。 |
+| semantic.match.message_countback | integer | 否 | 1 | >= 1 | 纳入向量化输入的末尾
`user` 角色消息数量。 |
+| semantic.match.ignore_system_prompts | boolean | 否 | true | | 如果为 true，则
`system` 角色消息不纳入向量化输入。 |
+| semantic.match.ignore_assistant_prompts | boolean | 否 | true | | 如果为 true，则
`assistant` 角色消息不纳入向量化输入。 |
+| semantic.match.ignore_tool_prompts | boolean | 否 | true | | 如果为 true，则
`tool` 角色消息不纳入向量化输入。 |
+| semantic.embedding | object | **是** | | | 向量化服务配置。`openai` 与 `azure_openai`
二选一，必须且只能配置其中一个。 |
+| semantic.embedding.openai.endpoint | string | 否 | | | OpenAI 兼容的向量化 API 端点
URL。省略时默认使用 OpenAI 公共 API。 |
+| semantic.embedding.openai.model | string | **是** | | | 向量化模型名称（例如
`text-embedding-3-small`）。 |
+| semantic.embedding.openai.api_key | string | **是** | | | OpenAI API 密钥。存入
etcd 时使用 AES 加密。 |
+| semantic.embedding.openai.dimensions | integer | 否 | | >= 1 |
覆盖向量输出维度（仅对支持该参数的模型有效）。 |
+| semantic.embedding.openai.ssl_verify | boolean | 否 | true | | 如果为
true，验证向量化服务的证书。 |
+| semantic.embedding.openai.timeout | integer | 否 | 5000 | >= 1 |
向量化服务的请求超时时间（毫秒）。 |
+| semantic.embedding.azure_openai.endpoint | string | **是** | | | Azure OpenAI
部署端点 URL。 |
+| semantic.embedding.azure_openai.api_key | string | **是** | | | Azure OpenAI
API 密钥。存入 etcd 时使用 AES 加密。 |
+| semantic.embedding.azure_openai.dimensions | integer | 否 | | >= 1 |
覆盖向量输出维度。 |
+| semantic.embedding.azure_openai.ssl_verify | boolean | 否 | true | | 如果为
true，验证向量化服务的证书。 |
+| semantic.embedding.azure_openai.timeout | integer | 否 | 5000 | >= 1 |
向量化服务的请求超时时间（毫秒）。 |
+| semantic.vector_search | object | **是** | | | 向量索引配置。 |
+| semantic.vector_search.redis.index | string | 否 | `"ai-cache"` | | 作为向量存储使用的
RediSearch 索引名称。 |
+
+:::note 安全说明：多租户部署
+
+缓存条目默认按路由隔离。在多个消费者共用同一路由的多租户场景下，为某个消费者生成的缓存响应可能会被返回给其他消费者。为防止跨租户信息泄漏，请采取以下措施：
+
+- 将 `cache_key.include_consumer` 设为 `true`，按消费者身份隔离缓存条目。
+- 使用 `cache_key.include_vars` 添加标识租户的 NGINX 变量（例如
`["http_x_tenant_id"]`）到缓存作用域。
+
+L1 与 L2 缓存条目均遵循相同的 `cache_key` 作用域规则。
+
+:::
+
## 示例
以下示例使用 OpenAI 作为上游 LLM 服务提供商。请获取 [OpenAI API
key](https://openai.com/blog/openai-api)，并将其与 Admin API key 一起保存到环境变量中：
@@ -333,3 +385,132 @@ curl -i "http://127.0.0.1:9080/anything"; -X POST \
```
缓存被完全跳过（不查询、不回写），响应中携带 `X-AI-Cache-Status: BYPASS` 响应头。
+
+### 使用语义匹配缓存 LLM 响应
+
+以下示例启用语义缓存（L2）层，使措辞略有不同但语义相近的提示词也能命中缓存。除可用的 Redis Stack 实例外，还需要 OpenAI API
密钥用于向量化服务。
+
+:::caution
+
+Redis 实例必须为 [Redis Stack](https://redis.io/docs/stack/)（含 RediSearch
模块）。语义缓存不支持普通 Redis。
+
+:::
+
+<Tabs
+groupId="api"
+defaultValue="admin-api"
+values={[
+{label: 'Admin API', value: 'admin-api'},
+{label: 'ADC', value: 'adc'}
+]}>
+
+<TabItem value="admin-api">
+
+```shell
+curl "http://127.0.0.1:9180/apisix/admin/routes"; -X PUT \
+ -H "X-API-KEY: ${admin_key}" \
+ -d '{
+ "id": "ai-cache-semantic-route",
+ "uri": "/anything",
+ "plugins": {
+ "ai-proxy": {
+ "provider": "openai",
+ "auth": { "header": { "Authorization": "Bearer '"$OPENAI_API_KEY"'" }
},
+ "options": { "model": "gpt-4o" }
+ },
+ "ai-cache": {
+ "redis_host": "127.0.0.1",
+ "layers": ["exact", "semantic"],
+ "semantic": {
+ "similarity_threshold": 0.92,
+ "embedding": {
+ "openai": {
+ "model": "text-embedding-3-small",
+ "api_key": "'"$OPENAI_API_KEY"'"
+ }
+ },
+ "vector_search": {
+ "redis": {
+ "index": "ai-cache"
+ }
+ }
+ }
+ }
+ }
+ }'
+```
+
+</TabItem>
+
+<TabItem value="adc">
+
+```yaml title="adc.yaml"
+services:
+ - name: ai-cache-semantic-service
+ routes:
+ - name: ai-cache-semantic-route
+ uris:
+ - /anything
+ methods:
+ - POST
+ plugins:
+ ai-proxy:
+ provider: openai
+ auth:
+ header:
+ Authorization: "Bearer ${OPENAI_API_KEY}"
+ options:
+ model: gpt-4o
+ ai-cache:
+ redis_host: 127.0.0.1
+ layers:
+ - exact
+ - semantic
+ semantic:
+ similarity_threshold: 0.92
+ embedding:
+ openai:
+ model: text-embedding-3-small
+ api_key: "${OPENAI_API_KEY}"
+ vector_search:
+ redis:
+ index: ai-cache
+```
+
+将配置同步到网关：
+
+```shell
+adc sync -f adc.yaml
+```
+
+</TabItem>
+
+</Tabs>
+
+发送初始请求：
+
+```shell
+curl -i "http://127.0.0.1:9080/anything"; -X POST \
+ -H "Content-Type: application/json" \
+ -d '{ "messages": [{ "role": "user", "content": "What is Apache APISIX?" }]
}'
+```
+
+首次请求同时未命中 L1 和 L2；插件将其代理到 LLM，对提示词进行向量化，并将精确缓存条目和向量分别存入 Redis。响应携带
`X-AI-Cache-Status: MISS`。
+
+发送语义相近但措辞不同的请求：
+
+```shell
+curl -i "http://127.0.0.1:9080/anything"; -X POST \
+ -H "Content-Type: application/json" \
+ -d '{ "messages": [{ "role": "user", "content": "Can you explain what Apache
APISIX is?" }] }'
+```
+
+该请求未命中 L1（指纹不同），但命中了 L2（向量相似度超过阈值）。响应由语义缓存直接返回，并携带以下响应头：
+
+```text
+X-AI-Cache-Status: HIT
+X-AI-Cache-Age: 12
+X-AI-Cache-Similarity: 0.9487
+```
+
+`X-AI-Cache-Similarity` 响应头表示请求提示词与命中缓存条目之间的余弦相似度（1 − 距离）。
diff --git a/t/fixtures/openai/embeddings-capital-city.json
b/t/fixtures/openai/embeddings-capital-city.json
new file mode 100644
index 000000000..2095ad452
--- /dev/null
+++ b/t/fixtures/openai/embeddings-capital-city.json
@@ -0,0 +1,80 @@
+{
+ "object": "list",
+ "data": [
+ {
+ "object": "embedding",
+ "embedding": [
+ 0.125122,
+ 0.053131,
+ 0.143066,
+ 0.09198,
+ -0.188232,
+ 0.044159,
+ -0.028748,
+ -0.041748,
+ 0.072693,
+ 0.049744,
+ -0.080383,
+ -0.084106,
+ -0.29541,
+ -0.011398,
+ -0.174927,
+ 0.029709,
+ -0.146729,
+ 0.161987,
+ 0.328613,
+ -0.108459,
+ 0.089172,
+ -0.093079,
+ -0.162109,
+ 0.03894,
+ 0.291992,
+ 0.050995,
+ -0.160645,
+ 0.074646,
+ 0.029282,
+ 0.124512,
+ 0.195923,
+ -0.095764,
+ 0.014839,
+ 0.172974,
+ -0.12384,
+ -0.15686,
+ -0.129272,
+ -0.194946,
+ 0.105835,
+ 0.171021,
+ -0.03775,
+ -0.084412,
+ 0.047058,
+ 0.044617,
+ -0.116882,
+ -0.046753,
+ -0.076965,
+ -0.110168,
+ -0.113342,
+ 0.12323,
+ 0.137817,
+ 0.046082,
+ 0.135254,
+ 0.123291,
+ -0.072327,
+ 0.034637,
+ 0.066711,
+ 0.006908,
+ 0.206177,
+ 0.037903,
+ 0.093811,
+ 0.020889,
+ 0.115112,
+ 0.072815
+ ],
+ "index": 0
+ }
+ ],
+ "model": "text-embedding-3-small",
+ "usage": {
+ "prompt_tokens": 8,
+ "total_tokens": 8
+ }
+}
diff --git a/t/fixtures/openai/embeddings-capital.json
b/t/fixtures/openai/embeddings-capital.json
new file mode 100644
index 000000000..fd30fb5c2
--- /dev/null
+++ b/t/fixtures/openai/embeddings-capital.json
@@ -0,0 +1,80 @@
+{
+ "object": "list",
+ "data": [
+ {
+ "object": "embedding",
+ "embedding": [
+ 0.183716,
+ 0.069519,
+ 0.124023,
+ 0.107178,
+ -0.101868,
+ -0.012199,
+ -0.062805,
+ 0.063293,
+ 0.047943,
+ -0.045044,
+ 0.030533,
+ -0.106262,
+ -0.271729,
+ -0.066528,
+ -0.062561,
+ 0.102234,
+ -0.029175,
+ 0.085632,
+ 0.319336,
+ -0.107727,
+ 0.013527,
+ -0.044586,
+ -0.180786,
+ 0.052734,
+ 0.273926,
+ 0.031464,
+ -0.201172,
+ -0.032227,
+ 0.015778,
+ 0.17395,
+ 0.185791,
+ -0.111084,
+ -0.008842,
+ 0.190186,
+ -0.108459,
+ -0.17627,
+ -0.166016,
+ -0.173706,
+ 0.093994,
+ 0.131104,
+ -0.013832,
+ -0.057434,
+ 0.030212,
+ 0.05838,
+ -0.120728,
+ -0.13562,
+ -0.040771,
+ -0.176147,
+ -0.148438,
+ 0.121399,
+ 0.171265,
+ 0.017853,
+ 0.143311,
+ 0.143921,
+ -0.07428,
+ 0.018631,
+ 0.108704,
+ 0.042389,
+ 0.205933,
+ 0.068542,
+ 0.159424,
+ -0.045441,
+ 0.116211,
+ 0.019028
+ ],
+ "index": 0
+ }
+ ],
+ "model": "text-embedding-3-small",
+ "usage": {
+ "prompt_tokens": 7,
+ "total_tokens": 7
+ }
+}
diff --git a/t/fixtures/openai/embeddings-largest-city.json
b/t/fixtures/openai/embeddings-largest-city.json
new file mode 100644
index 000000000..a12eee4bc
--- /dev/null
+++ b/t/fixtures/openai/embeddings-largest-city.json
@@ -0,0 +1,80 @@
+{
+ "object": "list",
+ "data": [
+ {
+ "object": "embedding",
+ "embedding": [
+ 0.115112,
+ 0.029099,
+ 0.171021,
+ 0.131348,
+ -0.213135,
+ 0.01947,
+ -0.080261,
+ -0.019409,
+ -0.075928,
+ 0.186401,
+ -0.005394,
+ -0.125732,
+ -0.26416,
+ -0.128418,
+ -0.071533,
+ 0.053223,
+ -0.104248,
+ -0.003386,
+ 0.244629,
+ -0.121338,
+ 0.039551,
+ 0.001368,
+ -0.196899,
+ -0.075317,
+ 0.286377,
+ -0.040039,
+ -0.160889,
+ 0.210938,
+ 0.092285,
+ 0.185913,
+ 0.035889,
+ -0.078186,
+ 0.029709,
+ 0.090393,
+ -0.320068,
+ -0.235474,
+ -0.04541,
+ -0.006218,
+ 0.101807,
+ 0.282227,
+ 0.004814,
+ -0.023636,
+ 0.073242,
+ 0.009438,
+ 0.01104,
+ 0.08374,
+ -0.02684,
+ -0.235229,
+ -0.013588,
+ -0.0131,
+ 0.14856,
+ 0.007298,
+ -0.049011,
+ 0.01033,
+ 0.017471,
+ -0.06311,
+ 0.086182,
+ 0.091125,
+ 0.137817,
+ -0.002174,
+ 0.011612,
+ -0.019135,
+ 0.061584,
+ 0.062683
+ ],
+ "index": 0
+ }
+ ],
+ "model": "text-embedding-3-small",
+ "usage": {
+ "prompt_tokens": 8,
+ "total_tokens": 8
+ }
+}
diff --git a/t/fixtures/openai/embeddings-tire.json
b/t/fixtures/openai/embeddings-tire.json
new file mode 100644
index 000000000..adedaa79e
--- /dev/null
+++ b/t/fixtures/openai/embeddings-tire.json
@@ -0,0 +1,80 @@
+{
+ "object": "list",
+ "data": [
+ {
+ "object": "embedding",
+ "embedding": [
+ -0.03241,
+ -0.196411,
+ -0.089966,
+ -0.031738,
+ -0.146973,
+ -0.188354,
+ -0.072815,
+ -0.165894,
+ 0.094727,
+ 0.011772,
+ 0.13501,
+ 0.041718,
+ 0.044312,
+ 0.38208,
+ 0.186035,
+ -0.020386,
+ 0.19519,
+ 0.144165,
+ -0.170776,
+ -0.023315,
+ -0.055328,
+ -0.075562,
+ 0.078308,
+ -0.112854,
+ 0.082275,
+ -0.103516,
+ 0.25293,
+ -0.100952,
+ -0.160278,
+ 0.001236,
+ 0.010345,
+ -0.140747,
+ 0.024475,
+ -0.127319,
+ 0.074402,
+ -0.081299,
+ 0.033875,
+ 0.066711,
+ 0.186768,
+ -0.143066,
+ -0.139282,
+ 0.038696,
+ 0.072021,
+ 0.169067,
+ 0.012856,
+ 0.095337,
+ 0.107605,
+ 0.238037,
+ -0.070129,
+ -0.02948,
+ 0.114197,
+ -0.070679,
+ -0.001534,
+ 0.032501,
+ -0.118164,
+ 0.152954,
+ 0.058075,
+ -0.049316,
+ 0.222168,
+ 0.053436,
+ -0.173828,
+ -0.115051,
+ 0.098389,
+ -0.004353
+ ],
+ "index": 0
+ }
+ ],
+ "model": "text-embedding-3-small",
+ "usage": {
+ "prompt_tokens": 9,
+ "total_tokens": 9
+ }
+}
diff --git a/t/lib/ai_cache_mock.lua b/t/lib/ai_cache_mock.lua
new file mode 100644
index 000000000..733014bfc
--- /dev/null
+++ b/t/lib/ai_cache_mock.lua
@@ -0,0 +1,117 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one or more
+-- contributor license agreements. See the NOTICE file distributed with
+-- this work for additional information regarding copyright ownership.
+-- The ASF licenses this file to You under the Apache License, Version 2.0
+-- (the "License"); you may not use this file except in compliance with
+-- the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
+--- Mock embeddings upstream for the ai-cache semantic (L2) tests.
+-- Replays the real text-embedding-3-small responses captured (at
dimensions=64)
+-- under t/fixtures/openai/embeddings-*.json, selecting one by the request
prompt
+-- so the suite exercises genuine embedding geometry without calling OpenAI:
+-- "...capital city..." -> capital_city "...capital..." -> capital
+-- "...largest..." -> largest_city "...tire..." -> tire
+
+local fixture_loader = require("lib.fixture_loader")
+local cjson = require("cjson.safe")
+
+local ngx = ngx
+local type = type
+
+local _M = {}
+
+local FIXTURE = {
+ capital = "openai/embeddings-capital.json",
+ capital_city = "openai/embeddings-capital-city.json",
+ largest_city = "openai/embeddings-largest-city.json",
+ tire = "openai/embeddings-tire.json",
+}
+
+
+local function serve(name)
+ local content, err = fixture_loader.load(name)
+ if not content then
+ ngx.status = 500
+ ngx.say(err or "fixture not found")
+ return
+ end
+ ngx.print(content)
+end
+
+
+-- Pick the embedding fixture the request is asking about. "capital city" is
+-- checked before "capital" so the paraphrase stays distinct from the anchor.
+local function pick(input)
+ if input:find("largest", 1, true) then
+ return FIXTURE.largest_city
+ elseif input:find("tire", 1, true) then
+ return FIXTURE.tire
+ elseif input:find("capital city", 1, true) then
+ return FIXTURE.capital_city
+ elseif input:find("capital", 1, true) then
+ return FIXTURE.capital
+ end
+ return FIXTURE.tire -- default: an unrelated/orthogonal vector
+end
+
+
+-- Prompt-keyed embeddings endpoint used by the end-to-end semantic tests.
+function _M.embeddings()
+ ngx.req.read_body()
+ local body = cjson.decode(ngx.req.get_body_data() or "{}") or {}
+ local input = type(body.input) == "string" and body.input or ""
+ serve(pick(input))
+end
+
+
+-- openai driver unit mock: must carry Authorization: Bearer.
+function _M.embeddings_openai()
+ if ngx.req.get_headers()["authorization"] ~= "Bearer test-key" then
+ ngx.status = 401
+ ngx.say("bad authorization")
+ return
+ end
+ serve(FIXTURE.capital)
+end
+
+
+-- azure_openai driver unit mock: must carry the api-key header, never
Authorization.
+function _M.embeddings_azure()
+ if ngx.req.get_headers()["api-key"] ~= "test-key" then
+ ngx.status = 401
+ ngx.say("missing api-key header")
+ return
+ end
+ if ngx.req.get_headers()["authorization"] then
+ ngx.status = 400
+ ngx.say("azure driver must not send Authorization")
+ return
+ end
+ serve(FIXTURE.capital)
+end
+
+
+-- always 5xx: drives the embedding-provider fail-open path.
+function _M.broken()
+ ngx.status = 500
+ ngx.say("embedding upstream error")
+end
+
+
+-- well-formed HTTP 200 but a body the driver must reject.
+function _M.malformed()
+ ngx.say('{"data":[]}')
+end
+
+
+return _M
diff --git a/t/plugin/ai-cache-semantic.t b/t/plugin/ai-cache-semantic.t
new file mode 100644
index 000000000..e7b22a7c6
--- /dev/null
+++ b/t/plugin/ai-cache-semantic.t
@@ -0,0 +1,1527 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+BEGIN {
+ $ENV{TEST_ENABLE_CONTROL_API_V1} = "0";
+}
+
+use t::APISIX 'no_plan';
+
+log_level("info");
+repeat_each(1);
+no_long_string();
+no_root_location();
+
+# Real text-embedding-3-small responses (captured at dimensions=64) live as
+# fixtures under t/fixtures/openai/embeddings-*.json, and the mock that replays
+# them by prompt lives in t/lib/ai_cache_mock.lua -- so every
HIT/MISS/threshold
+# decision is driven by genuine embedding geometry, hermetically:
+# cos(capital, capital_city) = 0.922 (paraphrase -> HIT at threshold 0.9)
+# cos(capital, largest_city) = 0.706 (related -> the real threshold
knee)
+# cos(capital, tire) = -0.148 (unrelated -> always a MISS)
+
+add_block_preprocessor(sub {
+ my ($block) = @_;
+
+ if (!defined $block->request) {
+ $block->set_value("request", "GET /t");
+ }
+
+ my $user_yaml_config = <<_EOC_;
+plugins:
+ - ai-proxy
+ - ai-cache
+_EOC_
+ if (!defined $block->extra_yaml_config) {
+ $block->set_value("extra_yaml_config", $user_yaml_config);
+ }
+
+ # Only the embedding call is mocked (the chat completion uses the shared
+ # :1980 X-AI-Fixture upstream); the mock logic lives in
lib/ai_cache_mock.lua.
+ my $http_config = $block->http_config // <<_EOC_;
+ server {
+ listen 6724;
+ default_type 'application/json';
+
+ location /v1/embeddings {
+ content_by_lua_block { require("lib.ai_cache_mock").embeddings() }
+ }
+ location /v1/embeddings-broken {
+ content_by_lua_block { require("lib.ai_cache_mock").broken() }
+ }
+ location /v1/embeddings-malformed {
+ content_by_lua_block { require("lib.ai_cache_mock").malformed() }
+ }
+ location /v1/embeddings-openai {
+ content_by_lua_block {
require("lib.ai_cache_mock").embeddings_openai() }
+ }
+ location /v1/embeddings-azure {
+ content_by_lua_block {
require("lib.ai_cache_mock").embeddings_azure() }
+ }
+ }
+_EOC_
+ $block->set_value("http_config", $http_config);
+});
+
+run_tests();
+
+__DATA__
+
+=== TEST 1: layers defaults to ["exact"]; a minimal exact-only config is valid
+--- config
+ location /t {
+ content_by_lua_block {
+ local plugin = require("apisix.plugins.ai-cache")
+ local ok, err = plugin.check_schema({ redis_host = "127.0.0.1" })
+ ngx.say(ok and "passed" or err)
+ }
+ }
+--- response_body
+passed
+
+
+
+=== TEST 2: an explicit layers=["exact"] is valid (no semantic block required)
+--- config
+ location /t {
+ content_by_lua_block {
+ local plugin = require("apisix.plugins.ai-cache")
+ local ok, err = plugin.check_schema({
+ redis_host = "127.0.0.1",
+ layers = { "exact" },
+ })
+ ngx.say(ok and "passed" or err)
+ }
+ }
+--- response_body
+passed
+
+
+
+=== TEST 3: layers=["exact","semantic"] without a semantic block is rejected
+--- config
+ location /t {
+ content_by_lua_block {
+ local plugin = require("apisix.plugins.ai-cache")
+ local ok, err = plugin.check_schema({
+ redis_host = "127.0.0.1",
+ layers = { "exact", "semantic" },
+ })
+ ngx.say(ok and "passed" or err)
+ }
+ }
+--- response_body eval
+qr/allOf 1 failed: then clause did not match/
+
+
+
+=== TEST 4: a full openai semantic config is valid
+--- config
+ location /t {
+ content_by_lua_block {
+ local plugin = require("apisix.plugins.ai-cache")
+ local ok, err = plugin.check_schema({
+ redis_host = "127.0.0.1",
+ layers = { "exact", "semantic" },
+ semantic = {
+ similarity_threshold = 0.9,
+ embedding = { openai = {
+ endpoint = "https://api.openai.com/v1/embeddings";,
+ model = "text-embedding-3-small",
+ api_key = "test-key",
+ dimensions = 1536,
+ } },
+ vector_search = { redis = { index = "ai-cache" } },
+ },
+ })
+ ngx.say(ok and "passed" or err)
+ }
+ }
+--- response_body
+passed
+
+
+
+=== TEST 5: a full azure_openai semantic config is valid (endpoint+api_key
required)
+--- config
+ location /t {
+ content_by_lua_block {
+ local plugin = require("apisix.plugins.ai-cache")
+ local ok, err = plugin.check_schema({
+ redis_host = "127.0.0.1",
+ layers = { "exact", "semantic" },
+ semantic = {
+ embedding = { azure_openai = {
+ endpoint =
"https://my.openai.azure.com/.../embeddings?api-version=2024-02-01";,
+ api_key = "test-key",
+ } },
+ vector_search = { redis = {} },
+ },
+ })
+ ngx.say(ok and "passed" or err)
+ }
+ }
+--- response_body
+passed
+
+
+
+=== TEST 6: layers=["semantic"] without "exact" is rejected (exact is always
required)
+--- config
+ location /t {
+ content_by_lua_block {
+ local plugin = require("apisix.plugins.ai-cache")
+ local ok, err = plugin.check_schema({
+ redis_host = "127.0.0.1",
+ layers = { "semantic" },
+ semantic = {
+ embedding = { openai = { model = "text-embedding-3-small",
api_key = "test-key" } },
+ vector_search = { redis = {} },
+ },
+ })
+ ngx.say(ok and "passed" or err)
+ }
+ }
+--- response_body eval
+qr/property "layers" validation failed: failed to check contains/
+
+
+
+=== TEST 7: distance_metric "euclidean" is rejected (cosine-only in this layer)
+--- config
+ location /t {
+ content_by_lua_block {
+ local plugin = require("apisix.plugins.ai-cache")
+ local ok, err = plugin.check_schema({
+ redis_host = "127.0.0.1",
+ layers = { "exact", "semantic" },
+ semantic = {
+ distance_metric = "euclidean",
+ embedding = { openai = { model = "text-embedding-3-small",
api_key = "test-key" } },
+ vector_search = { redis = {} },
+ },
+ })
+ ngx.say(ok and "passed" or err)
+ }
+ }
+--- response_body eval
+qr/property "distance_metric" validation failed: matches none of the enum
values/
+
+
+
+=== TEST 8: similarity_threshold above 1 is rejected (must be within [0, 1])
+--- config
+ location /t {
+ content_by_lua_block {
+ local plugin = require("apisix.plugins.ai-cache")
+ local ok, err = plugin.check_schema({
+ redis_host = "127.0.0.1",
+ layers = { "exact", "semantic" },
+ semantic = {
+ similarity_threshold = 1.5,
+ embedding = { openai = { model = "text-embedding-3-small",
api_key = "test-key" } },
+ vector_search = { redis = {} },
+ },
+ })
+ ngx.say(ok and "passed" or err)
+ }
+ }
+--- response_body eval
+qr/property "similarity_threshold" validation failed: expected 1\.5 to be at
most 1/
+
+
+
+=== TEST 9: semantic.ttl below 1 is rejected
+--- config
+ location /t {
+ content_by_lua_block {
+ local plugin = require("apisix.plugins.ai-cache")
+ local ok, err = plugin.check_schema({
+ redis_host = "127.0.0.1",
+ layers = { "exact", "semantic" },
+ semantic = {
+ ttl = 0,
+ embedding = { openai = { model = "text-embedding-3-small",
api_key = "test-key" } },
+ vector_search = { redis = {} },
+ },
+ })
+ ngx.say(ok and "passed" or err)
+ }
+ }
+--- response_body eval
+qr/property "ttl" validation failed: expected 0 to be at least 1/
+
+
+
+=== TEST 10: embedding must name exactly one provider -- neither is rejected
+--- config
+ location /t {
+ content_by_lua_block {
+ local plugin = require("apisix.plugins.ai-cache")
+ local ok, err = plugin.check_schema({
+ redis_host = "127.0.0.1",
+ layers = { "exact", "semantic" },
+ semantic = {
+ embedding = {},
+ vector_search = { redis = {} },
+ },
+ })
+ ngx.say(ok and "passed" or err)
+ }
+ }
+--- response_body eval
+qr/property "embedding" validation failed: value should match only one schema,
but matches none/
+
+
+
+=== TEST 11: embedding must name exactly one provider -- both is rejected
(oneOf)
+--- config
+ location /t {
+ content_by_lua_block {
+ local plugin = require("apisix.plugins.ai-cache")
+ local ok, err = plugin.check_schema({
+ redis_host = "127.0.0.1",
+ layers = { "exact", "semantic" },
+ semantic = {
+ embedding = {
+ openai = { model = "text-embedding-3-small",
api_key = "test-key" },
+ azure_openai = { endpoint = "https://a.b/e";, api_key =
"test-key" },
+ },
+ vector_search = { redis = {} },
+ },
+ })
+ ngx.say(ok and "passed" or err)
+ }
+ }
+--- response_body eval
+qr/property "embedding" validation failed: value should match only one schema,
but matches both schemas 1 and 2/
+
+
+
+=== TEST 12: match.message_countback below 1 is rejected
+--- config
+ location /t {
+ content_by_lua_block {
+ local plugin = require("apisix.plugins.ai-cache")
+ local ok, err = plugin.check_schema({
+ redis_host = "127.0.0.1",
+ layers = { "exact", "semantic" },
+ semantic = {
+ match = { message_countback = 0 },
+ embedding = { openai = { model = "text-embedding-3-small",
api_key = "test-key" } },
+ vector_search = { redis = {} },
+ },
+ })
+ ngx.say(ok and "passed" or err)
+ }
+ }
+--- response_body eval
+qr/property "message_countback" validation failed: expected 0 to be at least 1/
+
+
+
+=== TEST 13: context_fingerprint ignores message TEXT but reacts to
model/params (key.lua unit)
+--- config
+ location /t {
+ content_by_lua_block {
+ local key = require("apisix.plugins.ai-cache.key")
+
+ -- the context fingerprint deduplicates by the EFFECTIVE request
context
+ -- (provider/model/params/instance) with the message wording
removed, so a
+ -- paraphrase collapses to one fingerprint while a parameter
change does not.
+ local function ctx()
+ return { ai_client_protocol = "openai-chat",
+ picked_ai_instance = { provider = "openai",
+ options = { model = "gpt-4o"
}, override = {} },
+ var = {} }
+ end
+ local a = key.context_fingerprint(ctx(), { model = "gpt-4o",
temperature = 0.2,
+ messages = {{ role = "user", content = "how do I return an
item?" }} })
+ local b = key.context_fingerprint(ctx(), { model = "gpt-4o",
temperature = 0.2,
+ messages = {{ role = "user", content = "what is the return
policy?" }} })
+ local c = key.context_fingerprint(ctx(), { model = "gpt-4o",
temperature = 0.9,
+ messages = {{ role = "user", content = "how do I return an
item?" }} })
+
+ assert(a == b, "differently-worded prompts must share one context
fingerprint")
+ assert(a ~= c, "a parameter change (temperature) must flip the
context fingerprint")
+ ngx.say("passed")
+ }
+ }
+--- response_body
+passed
+
+
+
+=== TEST 14: partition is stable and isolates by scope and effective model
(key.lua unit)
+--- config
+ location /t {
+ content_by_lua_block {
+ local key = require("apisix.plugins.ai-cache.key")
+
+ -- partition = sha256(scope | context_repr); it is what segregates
the L2
+ -- vector index so a paraphrase can only ever hit within its own
+ -- tenant + route + effective-model cell.
+ local function ctx(tenant, model)
+ return { ai_client_protocol = "openai-chat",
+ picked_ai_instance = { provider = "openai",
+ options = { model = model or
"gpt-4o" }, override = {} },
+ var = { route_id = "1", http_x_tenant = tenant } }
+ end
+ local body = { model = "gpt-4o", messages = {{ role = "user",
content = "hi" }} }
+ local conf = { cache_key = { include_vars = { "http_x_tenant" } } }
+
+ assert(key.partition(conf, ctx("acme"), body) ==
key.partition(conf, ctx("acme"), body),
+ "the same inputs must produce the same partition")
+ assert(key.partition(conf, ctx("acme"), body) ~=
key.partition(conf, ctx("globex"), body),
+ "an include_vars (tenant) change must change the partition")
+ assert(key.partition(conf, ctx("acme", "gpt-4o"), body)
+ ~= key.partition(conf, ctx("acme", "gpt-4o-mini"),
body),
+ "a different effective model must change the partition")
+ ngx.say("passed")
+ }
+ }
+--- response_body
+passed
+
+
+
+=== TEST 15: extract_embed_text keeps the last user message, skipping
system/assistant (semantic.lua unit)
+--- config
+ location /t {
+ content_by_lua_block {
+ local semantic = require("apisix.plugins.ai-cache.semantic")
+ local msgs = {
+ { role = "system", content = "you are helpful" },
+ { role = "user", content = "first question" },
+ { role = "assistant", content = "an answer" },
+ { role = "user", content = "the real question" },
+ }
+ -- countback 1 -> only the most recent user message
+ ngx.say(semantic.extract_embed_text(msgs, { message_countback = 1
}))
+ -- countback 2 -> the two most recent kept (user) messages,
newline-joined
+ ngx.say(semantic.extract_embed_text(msgs, { message_countback = 2
}))
+ }
+ }
+--- response_body
+the real question
+first question
+the real question
+
+
+
+=== TEST 16: extract_embed_text flattens multimodal text blocks, dropping
non-text (semantic.lua unit)
+--- config
+ location /t {
+ content_by_lua_block {
+ local semantic = require("apisix.plugins.ai-cache.semantic")
+ local msgs = {{ role = "user", content = {
+ { type = "text", text = "describe" },
+ { type = "image_url", image_url = { url = "http://x/y.png"; } },
+ { type = "text", text = "this image" },
+ }}}
+ ngx.say(semantic.extract_embed_text(msgs, {}))
+ }
+ }
+--- response_body
+describe
+this image
+
+
+
+=== TEST 17: extract_embed_text honours ignore flags (system kept when not
ignored) (semantic.lua unit)
+--- config
+ location /t {
+ content_by_lua_block {
+ local semantic = require("apisix.plugins.ai-cache.semantic")
+ local msgs = {
+ { role = "system", content = "system preamble" },
+ { role = "user", content = "the question" },
+ }
+ -- default: system prompts ignored -> only the user message
+ ngx.say(semantic.extract_embed_text(msgs, { message_countback = 2
}))
+ ngx.say("---")
+ -- opt-in: keep system prompts -> both, newline-joined
+ ngx.say(semantic.extract_embed_text(msgs,
+ { message_countback = 2, ignore_system_prompts = false }))
+ }
+ }
+--- response_body
+the question
+---
+system preamble
+the question
+
+
+
+=== TEST 18: openai embeddings driver extracts data[1].embedding (real
1536-dim vector) + sends Bearer
+--- config
+ location /t {
+ content_by_lua_block {
+ local http = require("resty.http")
+ local drv = require("apisix.plugins.ai-cache.embeddings.openai")
+ -- the mock returns 401 unless Authorization: Bearer test-key is
present,
+ -- so a successful vector also proves the driver sent the bearer
token.
+ local vec, err = drv.get_embeddings(
+ { endpoint = "http://127.0.0.1:6724/v1/embeddings-openai";,
+ model = "text-embedding-3-small", api_key = "test-key" },
+ "What is the capital of France?", http.new(), false)
+ if not vec then ngx.say("err:", err); return end
+ -- the real captured "capital" embedding (text-embedding-3-small)
+ ngx.say("dim=", #vec)
+ ngx.say(string.format("v1=%.6f v2=%.6f v3=%.6f", vec[1], vec[2],
vec[3]))
+ }
+ }
+--- response_body
+dim=64
+v1=0.183716 v2=0.069519 v3=0.124023
+
+
+
+=== TEST 19: openai embeddings driver fails closed on a non-2xx response
+--- config
+ location /t {
+ content_by_lua_block {
+ local http = require("resty.http")
+ local drv = require("apisix.plugins.ai-cache.embeddings.openai")
+ local vec = drv.get_embeddings(
+ { endpoint = "http://127.0.0.1:6724/v1/embeddings-broken";,
+ model = "text-embedding-3-small", api_key = "test-key" },
+ "hi", http.new(), false)
+ ngx.say(vec and "got-vec" or "nil-on-error")
+ }
+ }
+--- response_body
+nil-on-error
+
+
+
+=== TEST 20: openai embeddings driver rejects a malformed (well-formed-HTTP)
body
+--- config
+ location /t {
+ content_by_lua_block {
+ local http = require("resty.http")
+ local drv = require("apisix.plugins.ai-cache.embeddings.openai")
+ local vec, err = drv.get_embeddings(
+ { endpoint = "http://127.0.0.1:6724/v1/embeddings-malformed";,
+ model = "text-embedding-3-small", api_key = "test-key" },
+ "hi", http.new(), false)
+ ngx.say(vec and "got-vec" or "nil-on-error")
+ ngx.say(err)
+ }
+ }
+--- response_body
+nil-on-error
+malformed embeddings response
+
+
+
+=== TEST 21: azure_openai embeddings driver sends api-key (not Authorization),
returns the real vector
+--- config
+ location /t {
+ content_by_lua_block {
+ local http = require("resty.http")
+ local drv =
require("apisix.plugins.ai-cache.embeddings.azure_openai")
+ -- the mock 401s without api-key and 400s if Authorization is
present,
+ -- so a vector here proves the azure auth scheme is used
exclusively.
+ local vec, err = drv.get_embeddings(
+ { endpoint = "http://127.0.0.1:6724/v1/embeddings-azure";,
api_key = "test-key" },
+ "What is the capital of France?", http.new(), false)
+ if not vec then ngx.say("err:", err); return end
+ ngx.say("dim=", #vec)
+ ngx.say(string.format("v1=%.6f v2=%.6f v3=%.6f", vec[1], vec[2],
vec[3]))
+ }
+ }
+--- response_body
+dim=64
+v1=0.183716 v2=0.069519 v3=0.124023
+
+
+
+=== TEST 22: azure_openai embeddings driver fails closed on a non-2xx response
+--- config
+ location /t {
+ content_by_lua_block {
+ local http = require("resty.http")
+ local drv =
require("apisix.plugins.ai-cache.embeddings.azure_openai")
+ local vec = drv.get_embeddings(
+ { endpoint = "http://127.0.0.1:6724/v1/embeddings-broken";,
api_key = "test-key" },
+ "hi", http.new(), false)
+ ngx.say(vec and "got-vec" or "nil-on-error")
+ }
+ }
+--- response_body
+nil-on-error
+
+
+
+=== TEST 23: pack_float32 produces a 4-bytes-per-element little-endian blob
(vector-search unit)
+--- config
+ location /t {
+ content_by_lua_block {
+ local vs = require("apisix.plugins.ai-cache.vector-search.redis")
+ local blob = vs.pack_float32({ 1.0, 0.0, 0.0 })
+ ngx.say(type(blob), ":", #blob)
+ -- 1.0f little-endian = 00 00 80 3f
+ ngx.say(string.byte(blob, 3), ",", string.byte(blob, 4))
+ }
+ }
+--- response_body
+string:12
+128,63
+
+
+
+=== TEST 24: ensure_index creates the index and is idempotent on a second call
(vector-search unit)
+--- config
+ location /t {
+ content_by_lua_block {
+ local redis_util = require("apisix.utils.redis")
+ local vs = require("apisix.plugins.ai-cache.vector-search.redis")
+ local red = assert(redis_util.new({
+ redis_host = "127.0.0.1", redis_port = 6379, redis_database =
0 }))
+ red:flushdb()
+ local tgt = "127.0.0.1#6379#0"
+ assert(vs.ensure_index(red, tgt, "ut-create:idx:3",
"ut-create:l2:", 3))
+ assert(vs.ensure_index(red, tgt, "ut-create:idx:3",
"ut-create:l2:", 3)) -- idempotent
+ ngx.say("passed")
+ }
+ }
+--- response_body
+passed
+
+
+
+=== TEST 25: upsert + knn_search round-trip returns the nearest doc by cosine
distance (vector-search unit)
+--- config
+ location /t {
+ content_by_lua_block {
+ local redis_util = require("apisix.utils.redis")
+ local vs = require("apisix.plugins.ai-cache.vector-search.redis")
+ local red = assert(redis_util.new({
+ redis_host = "127.0.0.1", redis_port = 6379, redis_database =
0 }))
+ red:flushdb()
+ local tgt = "127.0.0.1#6379#0"
+ assert(vs.ensure_index(red, tgt, "ut-knn:idx:3", "ut-knn:l2:", 3))
+
+ assert(vs.upsert(red, "ut-knn:l2:p1:near",
+ { partition = "p1", embedding = vs.pack_float32({ 1.0, 0.0,
0.0 }),
+ response = [[{"answer":"NEAR"}]], created_at = 100 }, 600))
+ assert(vs.upsert(red, "ut-knn:l2:p1:far",
+ { partition = "p1", embedding = vs.pack_float32({ 0.0, 1.0,
0.0 }),
+ response = [[{"answer":"FAR"}]], created_at = 100 }, 600))
+
+ local hit, err = vs.knn_search(red, tgt, "ut-knn:idx:3", "p1", {
0.99, 0.01, 0.0 }, 1)
+ if not hit then ngx.say("no-hit:", err or ""); return end
+ ngx.say(hit.response)
+ ngx.say(hit.distance < 0.01 and "near-distance" or ("dist=" ..
hit.distance))
+ }
+ }
+--- response_body
+{"answer":"NEAR"}
+near-distance
+
+
+
+=== TEST 26: knn_search returns nil and no error when the partition holds no
docs (vector-search unit)
+--- config
+ location /t {
+ content_by_lua_block {
+ local redis_util = require("apisix.utils.redis")
+ local vs = require("apisix.plugins.ai-cache.vector-search.redis")
+ local red = assert(redis_util.new({
+ redis_host = "127.0.0.1", redis_port = 6379, redis_database =
0 }))
+ red:flushdb()
+ local tgt = "127.0.0.1#6379#0"
+ assert(vs.ensure_index(red, tgt, "ut-empty:idx:3", "ut-empty:l2:",
3))
+ -- partitions are sha256 hex in production; use a hex-shaped value
here
+ local hit, err = vs.knn_search(red, tgt, "ut-empty:idx:3",
"deadbeef", { 1, 0, 0 }, 1)
+ ngx.say(hit == nil and not err and "clean-miss" or "unexpected")
+ }
+ }
+--- response_body
+clean-miss
+
+
+
+=== TEST 27: knn_search is partition-scoped -- an identical vector in another
partition is invisible (vector-search unit)
+--- config
+ location /t {
+ content_by_lua_block {
+ local redis_util = require("apisix.utils.redis")
+ local vs = require("apisix.plugins.ai-cache.vector-search.redis")
+ local red = assert(redis_util.new({
+ redis_host = "127.0.0.1", redis_port = 6379, redis_database =
0 }))
+ red:flushdb()
+ local tgt = "127.0.0.1#6379#0"
+ assert(vs.ensure_index(red, tgt, "ut-part:idx:3", "ut-part:l2:",
3))
+
+ -- two docs with the SAME vector but different partition tags
+ assert(vs.upsert(red, "ut-part:l2:p1:d1",
+ { partition = "p1", embedding = vs.pack_float32({ 1.0, 0.0,
0.0 }),
+ response = [[{"answer":"P1"}]], created_at = 100 }, 600))
+ assert(vs.upsert(red, "ut-part:l2:p2:d2",
+ { partition = "p2", embedding = vs.pack_float32({ 1.0, 0.0,
0.0 }),
+ response = [[{"answer":"P2"}]], created_at = 100 }, 600))
+
+ local hit = vs.knn_search(red, tgt, "ut-part:idx:3", "p1", { 1, 0,
0 }, 1)
+ ngx.say(hit and hit.response or "no-hit")
+ -- querying a partition with no docs must not leak p1/p2's
identical vector
+ local other = vs.knn_search(red, tgt, "ut-part:idx:3", "p3", { 1,
0, 0 }, 1)
+ ngx.say(other == nil and "isolated" or "leaked")
+ }
+ }
+--- response_body
+{"answer":"P1"}
+isolated
+
+
+
+=== TEST 28: set a semantic route (exact+semantic, default "ai-cache" index,
threshold 0.9)
+--- config
+ location /t {
+ content_by_lua_block {
+ require("lib.test_redis").flush_port("127.0.0.1", 6379)
+
+ local t = require("lib.test_admin").test
+ local code, body = t('/apisix/admin/routes/1',
+ ngx.HTTP_PUT,
+ [[{
+ "uri": "/semantic",
+ "plugins": {
+ "ai-proxy": {
+ "provider": "openai",
+ "auth": { "header": { "Authorization": "Bearer
test-key" } },
+ "options": { "model": "gpt-4o" },
+ "override": { "endpoint": "http://127.0.0.1:1980"; }
+ },
+ "ai-cache": {
+ "redis_host": "127.0.0.1",
+ "redis_port": 6379,
+ "layers": ["exact", "semantic"],
+ "semantic": {
+ "similarity_threshold": 0.9,
+ "embedding": {
+ "openai": {
+ "endpoint":
"http://127.0.0.1:6724/v1/embeddings";,
+ "model": "text-embedding-3-small",
+ "api_key": "test-key"
+ }
+ },
+ "vector_search": { "redis": {} }
+ }
+ }
+ }
+ }]]
+ )
+ if code >= 300 then ngx.status = code end
+ ngx.say(body)
+ }
+ }
+--- response_body
+passed
+
+
+
+=== TEST 29: a cold prompt ("capital of France") is a semantic MISS and is
proxied upstream
+--- request
+POST /semantic
+{"model":"gpt-4o","messages":[{"role":"user","content":"What is the capital of
France?"}]}
+--- more_headers
+X-AI-Fixture: openai/chat-basic.json
+--- response_headers
+X-AI-Cache-Status: MISS
+--- response_body_like eval
+qr/1 \+ 1 = 2/
+--- wait: 0.5
+
+
+
+=== TEST 30: a real paraphrase ("capital city of France", cos 0.922) is a
semantic L2 HIT
+--- request
+POST /semantic
+{"model":"gpt-4o","messages":[{"role":"user","content":"What's the capital
city of France?"}]}
+--- response_headers_like
+X-AI-Cache-Status: HIT
+X-AI-Cache-Similarity: 0\.92\d\d
+X-AI-Cache-Age: \d+
+--- response_body_like eval
+qr/1 \+ 1 = 2/
+--- wait: 0.3
+
+
+
+=== TEST 31: repeating the paraphrase is now an exact L1 HIT (backfill -- no
similarity header)
+--- request
+POST /semantic
+{"model":"gpt-4o","messages":[{"role":"user","content":"What's the capital
city of France?"}]}
+--- response_headers
+X-AI-Cache-Status: HIT
+! X-AI-Cache-Similarity
+--- response_body_like eval
+qr/1 \+ 1 = 2/
+
+
+
+=== TEST 32: an unrelated prompt ("flat car tire", cos -0.148) is a MISS (far
below threshold)
+--- request
+POST /semantic
+{"model":"gpt-4o","messages":[{"role":"user","content":"How do I change a flat
car tire?"}]}
+--- more_headers
+X-AI-Fixture: openai/chat-basic.json
+--- response_headers
+X-AI-Cache-Status: MISS
+--- response_body_like eval
+qr/1 \+ 1 = 2/
+--- wait: 0.3
+
+
+
+=== TEST 33: the default "ai-cache:l2:" namespace was populated by the
semantic writes
+--- config
+ location /t {
+ content_by_lua_block {
+ local redis_util = require("apisix.utils.redis")
+ local red = assert(redis_util.new({
+ redis_host = "127.0.0.1", redis_port = 6379, redis_database =
0 }))
+ local keys = red:keys("ai-cache:l2:*")
+ red:close()
+ ngx.say("default-l2=", (keys and #keys > 0) and "present" or
"absent")
+ }
+ }
+--- response_body
+default-l2=present
+
+
+
+=== TEST 34: set a single semantic route with NO options.model (effective
model = body model)
+--- config
+ location /t {
+ content_by_lua_block {
+ require("lib.test_redis").flush_port("127.0.0.1", 6379)
+
+ local t = require("lib.test_admin").test
+ local code, body = t('/apisix/admin/routes/1',
+ ngx.HTTP_PUT,
+ [[{
+ "uri": "/semantic",
+ "plugins": {
+ "ai-proxy": {
+ "provider": "openai",
+ "auth": { "header": { "Authorization": "Bearer
test-key" } },
+ "override": { "endpoint": "http://127.0.0.1:1980"; }
+ },
+ "ai-cache": {
+ "redis_host": "127.0.0.1",
+ "redis_port": 6379,
+ "layers": ["exact", "semantic"],
+ "semantic": {
+ "similarity_threshold": 0.9,
+ "embedding": {
+ "openai": {
+ "endpoint":
"http://127.0.0.1:6724/v1/embeddings";,
+ "model": "text-embedding-3-small",
+ "api_key": "test-key"
+ }
+ },
+ "vector_search": { "redis": { "index":
"idx-model" } }
+ }
+ }
+ }
+ }]]
+ )
+ if code >= 300 then ngx.status = code end
+ ngx.say(body)
+ }
+ }
+--- response_body
+passed
+
+
+
+=== TEST 35: gpt-4o cold request is a MISS (warms the gpt-4o partition)
+--- request
+POST /semantic
+{"model":"gpt-4o","messages":[{"role":"user","content":"What is the capital of
France?"}]}
+--- more_headers
+X-AI-Fixture: openai/chat-basic.json
+--- response_headers
+X-AI-Cache-Status: MISS
+--- wait: 0.5
+
+
+
+=== TEST 36: gpt-4o-mini with the same prompt and same vector is still a MISS
(partition is model-scoped)
+--- request
+POST /semantic
+{"model":"gpt-4o-mini","messages":[{"role":"user","content":"What is the
capital of France?"}]}
+--- more_headers
+X-AI-Fixture: openai/chat-basic.json
+--- response_headers
+X-AI-Cache-Status: MISS
+--- wait: 0.5
+
+
+
+=== TEST 37: a gpt-4o paraphrase is a semantic HIT (gpt-4o's L2 partition is
intact and isolated)
+--- request
+POST /semantic
+{"model":"gpt-4o","messages":[{"role":"user","content":"What's the capital
city of France?"}]}
+--- response_headers_like
+X-AI-Cache-Status: HIT
+X-AI-Cache-Similarity: 0\.92\d\d
+
+
+
+=== TEST 38: set a semantic route isolated per-tenant via include_vars
+--- config
+ location /t {
+ content_by_lua_block {
+ require("lib.test_redis").flush_port("127.0.0.1", 6379)
+
+ local t = require("lib.test_admin").test
+ local code, body = t('/apisix/admin/routes/1',
+ ngx.HTTP_PUT,
+ [[{
+ "uri": "/semantic",
+ "plugins": {
+ "ai-proxy": {
+ "provider": "openai",
+ "auth": { "header": { "Authorization": "Bearer
test-key" } },
+ "options": { "model": "gpt-4o" },
+ "override": { "endpoint": "http://127.0.0.1:1980"; }
+ },
+ "ai-cache": {
+ "redis_host": "127.0.0.1",
+ "redis_port": 6379,
+ "layers": ["exact", "semantic"],
+ "cache_key": { "include_vars": ["http_x_tenant"] },
+ "semantic": {
+ "similarity_threshold": 0.9,
+ "embedding": {
+ "openai": {
+ "endpoint":
"http://127.0.0.1:6724/v1/embeddings";,
+ "model": "text-embedding-3-small",
+ "api_key": "test-key"
+ }
+ },
+ "vector_search": { "redis": { "index":
"idx-tenant" } }
+ }
+ }
+ }
+ }]]
+ )
+ if code >= 300 then ngx.status = code end
+ ngx.say(body)
+ }
+ }
+--- response_body
+passed
+
+
+
+=== TEST 39: tenant acme cold request is a MISS (warms the acme scope)
+--- request
+POST /semantic
+{"model":"gpt-4o","messages":[{"role":"user","content":"What is the capital of
France?"}]}
+--- more_headers
+X-AI-Fixture: openai/chat-basic.json
+X-Tenant: acme
+--- response_headers
+X-AI-Cache-Status: MISS
+--- wait: 0.5
+
+
+
+=== TEST 40: tenant globex with the same prompt and vector is a MISS (no
cross-tenant semantic leak)
+--- request
+POST /semantic
+{"model":"gpt-4o","messages":[{"role":"user","content":"What is the capital of
France?"}]}
+--- more_headers
+X-AI-Fixture: openai/chat-basic.json
+X-Tenant: globex
+--- response_headers
+X-AI-Cache-Status: MISS
+--- wait: 0.5
+
+
+
+=== TEST 41: a tenant acme paraphrase is a semantic HIT (acme's own scope
persisted)
+--- request
+POST /semantic
+{"model":"gpt-4o","messages":[{"role":"user","content":"What's the capital
city of France?"}]}
+--- more_headers
+X-Tenant: acme
+--- response_headers_like
+X-AI-Cache-Status: HIT
+X-AI-Cache-Similarity: 0\.92\d\d
+
+
+
+=== TEST 42: set a semantic route whose embedding endpoint always 5xxs
(fail-open)
+--- config
+ location /t {
+ content_by_lua_block {
+ require("lib.test_redis").flush_port("127.0.0.1", 6379)
+
+ local t = require("lib.test_admin").test
+ local code, body = t('/apisix/admin/routes/1',
+ ngx.HTTP_PUT,
+ [[{
+ "uri": "/semantic",
+ "plugins": {
+ "ai-proxy": {
+ "provider": "openai",
+ "auth": { "header": { "Authorization": "Bearer
test-key" } },
+ "options": { "model": "gpt-4o" },
+ "override": { "endpoint": "http://127.0.0.1:1980"; }
+ },
+ "ai-cache": {
+ "redis_host": "127.0.0.1",
+ "redis_port": 6379,
+ "layers": ["exact", "semantic"],
+ "semantic": {
+ "embedding": {
+ "openai": {
+ "endpoint":
"http://127.0.0.1:6724/v1/embeddings-broken";,
+ "model": "text-embedding-3-small",
+ "api_key": "test-key"
+ }
+ },
+ "vector_search": { "redis": { "index":
"idx-failopen" } }
+ }
+ }
+ }
+ }]]
+ )
+ if code >= 300 then ngx.status = code end
+ ngx.say(body)
+ }
+ }
+--- response_body
+passed
+
+
+
+=== TEST 43: a broken embedding provider fails open to a MISS; exact (L1)
still warms
+--- request
+POST /semantic
+{"model":"gpt-4o","messages":[{"role":"user","content":"What is the capital of
France?"}]}
+--- more_headers
+X-AI-Fixture: openai/chat-basic.json
+--- response_headers
+X-AI-Cache-Status: MISS
+--- response_body_like eval
+qr/1 \+ 1 = 2/
+--- error_log
+ai-cache: embedding failed, fail-open as MISS
+--- wait: 0.5
+
+
+
+=== TEST 44: the identical request is an exact L1 HIT (the broken embedder
never corrupted L1)
+--- request
+POST /semantic
+{"model":"gpt-4o","messages":[{"role":"user","content":"What is the capital of
France?"}]}
+--- response_headers
+X-AI-Cache-Status: HIT
+! X-AI-Cache-Similarity
+--- response_body_like eval
+qr/1 \+ 1 = 2/
+
+
+
+=== TEST 45: set a semantic route with a custom vector_search index ("myidx")
+--- config
+ location /t {
+ content_by_lua_block {
+ require("lib.test_redis").flush_port("127.0.0.1", 6379)
+
+ local t = require("lib.test_admin").test
+ local code, body = t('/apisix/admin/routes/1',
+ ngx.HTTP_PUT,
+ [[{
+ "uri": "/semantic",
+ "plugins": {
+ "ai-proxy": {
+ "provider": "openai",
+ "auth": { "header": { "Authorization": "Bearer
test-key" } },
+ "options": { "model": "gpt-4o" },
+ "override": { "endpoint": "http://127.0.0.1:1980"; }
+ },
+ "ai-cache": {
+ "redis_host": "127.0.0.1",
+ "redis_port": 6379,
+ "layers": ["exact", "semantic"],
+ "semantic": {
+ "similarity_threshold": 0.9,
+ "embedding": {
+ "openai": {
+ "endpoint":
"http://127.0.0.1:6724/v1/embeddings";,
+ "model": "text-embedding-3-small",
+ "api_key": "test-key"
+ }
+ },
+ "vector_search": { "redis": { "index": "myidx"
} }
+ }
+ }
+ }
+ }]]
+ )
+ if code >= 300 then ngx.status = code end
+ ngx.say(body)
+ }
+ }
+--- response_body
+passed
+
+
+
+=== TEST 46: custom-index cold request is a MISS (warms L2 under "myidx")
+--- request
+POST /semantic
+{"model":"gpt-4o","messages":[{"role":"user","content":"What is the capital of
France?"}]}
+--- more_headers
+X-AI-Fixture: openai/chat-basic.json
+--- response_headers
+X-AI-Cache-Status: MISS
+--- wait: 0.5
+
+
+
+=== TEST 47: a paraphrase is a semantic HIT served from the custom index
+--- request
+POST /semantic
+{"model":"gpt-4o","messages":[{"role":"user","content":"What's the capital
city of France?"}]}
+--- response_headers_like
+X-AI-Cache-Status: HIT
+X-AI-Cache-Similarity: 0\.92\d\d
+
+
+
+=== TEST 48: L2 docs live under the custom "myidx:l2:" prefix, never the
default "ai-cache:l2:"
+--- config
+ location /t {
+ content_by_lua_block {
+ local redis_util = require("apisix.utils.redis")
+ local red = assert(redis_util.new({
+ redis_host = "127.0.0.1", redis_port = 6379, redis_database =
0 }))
+ local myidx = red:keys("myidx:l2:*")
+ local default = red:keys("ai-cache:l2:*")
+ red:close()
+ ngx.say("myidx-l2=", (myidx and #myidx > 0) and "present" or
"absent")
+ ngx.say("default-l2=", (default and #default > 0) and "present" or
"none")
+ }
+ }
+--- response_body
+myidx-l2=present
+default-l2=none
+
+
+
+=== TEST 49: set a strict route -- similarity_threshold 0.8 (above the real
related-question score)
+--- config
+ location /t {
+ content_by_lua_block {
+ require("lib.test_redis").flush_port("127.0.0.1", 6379)
+
+ local t = require("lib.test_admin").test
+ local code, body = t('/apisix/admin/routes/1',
+ ngx.HTTP_PUT,
+ [[{
+ "uri": "/semantic",
+ "plugins": {
+ "ai-proxy": {
+ "provider": "openai",
+ "auth": { "header": { "Authorization": "Bearer
test-key" } },
+ "options": { "model": "gpt-4o" },
+ "override": { "endpoint": "http://127.0.0.1:1980"; }
+ },
+ "ai-cache": {
+ "redis_host": "127.0.0.1",
+ "redis_port": 6379,
+ "layers": ["exact", "semantic"],
+ "semantic": {
+ "similarity_threshold": 0.8,
+ "embedding": {
+ "openai": {
+ "endpoint":
"http://127.0.0.1:6724/v1/embeddings";,
+ "model": "text-embedding-3-small",
+ "api_key": "test-key"
+ }
+ },
+ "vector_search": { "redis": { "index":
"idx-thrhi" } }
+ }
+ }
+ }
+ }]]
+ )
+ if code >= 300 then ngx.status = code end
+ ngx.say(body)
+ }
+ }
+--- response_body
+passed
+
+
+
+=== TEST 50: anchor cold request is a MISS (stores the "capital" embedding)
+--- request
+POST /semantic
+{"model":"gpt-4o","messages":[{"role":"user","content":"What is the capital of
France?"}]}
+--- more_headers
+X-AI-Fixture: openai/chat-basic.json
+--- response_headers
+X-AI-Cache-Status: MISS
+--- wait: 0.5
+
+
+
+=== TEST 51: the related question ("largest city", cos 0.706) is a MISS under
the 0.8 threshold
+--- request
+POST /semantic
+{"model":"gpt-4o","messages":[{"role":"user","content":"What is the largest
city in France?"}]}
+--- more_headers
+X-AI-Fixture: openai/chat-basic.json
+--- response_headers
+X-AI-Cache-Status: MISS
+--- wait: 0.3
+
+
+
+=== TEST 52: set the same scenario with a lenient similarity_threshold of 0.6
+--- config
+ location /t {
+ content_by_lua_block {
+ require("lib.test_redis").flush_port("127.0.0.1", 6379)
+
+ local t = require("lib.test_admin").test
+ local code, body = t('/apisix/admin/routes/1',
+ ngx.HTTP_PUT,
+ [[{
+ "uri": "/semantic",
+ "plugins": {
+ "ai-proxy": {
+ "provider": "openai",
+ "auth": { "header": { "Authorization": "Bearer
test-key" } },
+ "options": { "model": "gpt-4o" },
+ "override": { "endpoint": "http://127.0.0.1:1980"; }
+ },
+ "ai-cache": {
+ "redis_host": "127.0.0.1",
+ "redis_port": 6379,
+ "layers": ["exact", "semantic"],
+ "semantic": {
+ "similarity_threshold": 0.6,
+ "embedding": {
+ "openai": {
+ "endpoint":
"http://127.0.0.1:6724/v1/embeddings";,
+ "model": "text-embedding-3-small",
+ "api_key": "test-key"
+ }
+ },
+ "vector_search": { "redis": { "index":
"idx-thrlo" } }
+ }
+ }
+ }
+ }]]
+ )
+ if code >= 300 then ngx.status = code end
+ ngx.say(body)
+ }
+ }
+--- response_body
+passed
+
+
+
+=== TEST 53: anchor cold request is a MISS (stores the "capital" embedding)
+--- request
+POST /semantic
+{"model":"gpt-4o","messages":[{"role":"user","content":"What is the capital of
France?"}]}
+--- more_headers
+X-AI-Fixture: openai/chat-basic.json
+--- response_headers
+X-AI-Cache-Status: MISS
+--- wait: 0.5
+
+
+
+=== TEST 54: the same related question is a HIT under the 0.6 threshold (real
similarity 0.706)
+--- request
+POST /semantic
+{"model":"gpt-4o","messages":[{"role":"user","content":"What is the largest
city in France?"}]}
+--- response_headers_like
+X-AI-Cache-Status: HIT
+X-AI-Cache-Similarity: 0\.70\d\d
+--- response_body_like eval
+qr/1 \+ 1 = 2/
+
+
+
+=== TEST 55: context_messages returns the messages the embedding does NOT
cover (semantic.lua unit)
+--- config
+ location /t {
+ content_by_lua_block {
+ local semantic = require("apisix.plugins.ai-cache.semantic")
+ local msgs = {
+ { role = "system", content = "a document" },
+ { role = "user", content = "first question" },
+ { role = "assistant", content = "an answer" },
+ { role = "user", content = "the real question" },
+ }
+ -- default match embeds only the last user message; everything else
+ -- is response-determining context that must isolate the partition
+ local ctx = semantic.context_messages(msgs, { message_countback =
1 })
+ local out = {}
+ for _, m in ipairs(ctx) do out[#out + 1] = m.role .. ":" ..
m.content end
+ ngx.say(table.concat(out, ","))
+ -- opting every message INTO the embedding leaves no separate
context
+ local all = semantic.context_messages(msgs,
+ { message_countback = 4, ignore_system_prompts = false,
+ ignore_assistant_prompts = false })
+ ngx.say(#all == 0 and "empty" or "non-empty")
+ }
+ }
+--- response_body
+system:a document,user:first question,assistant:an answer
+empty
+
+
+
+=== TEST 56: set a semantic route for the doc-Q&A regression (default match,
threshold 0.9)
+--- config
+ location /t {
+ content_by_lua_block {
+ require("lib.test_redis").flush_port("127.0.0.1", 6379)
+
+ local t = require("lib.test_admin").test
+ local code, body = t('/apisix/admin/routes/1',
+ ngx.HTTP_PUT,
+ [[{
+ "uri": "/semantic",
+ "plugins": {
+ "ai-proxy": {
+ "provider": "openai",
+ "auth": { "header": { "Authorization": "Bearer
test-key" } },
+ "options": { "model": "gpt-4o" },
+ "override": { "endpoint": "http://127.0.0.1:1980"; }
+ },
+ "ai-cache": {
+ "redis_host": "127.0.0.1",
+ "redis_port": 6379,
+ "layers": ["exact", "semantic"],
+ "semantic": {
+ "similarity_threshold": 0.9,
+ "embedding": {
+ "openai": {
+ "endpoint":
"http://127.0.0.1:6724/v1/embeddings";,
+ "model": "text-embedding-3-small",
+ "api_key": "test-key"
+ }
+ },
+ "vector_search": { "redis": {} }
+ }
+ }
+ }
+ }]]
+ )
+ if code >= 300 then ngx.status = code end
+ ngx.say(body)
+ }
+ }
+--- response_body
+passed
+
+
+
+=== TEST 57: doc A + a generic question is a cold MISS (warms doc A's
partition)
+--- request
+POST /semantic
+{"model":"gpt-4o","messages":[{"role":"system","content":"Document A: Paris is
the capital of France."},{"role":"user","content":"What is the capital of
France?"}]}
+--- more_headers
+X-AI-Fixture: openai/chat-basic.json
+--- response_headers
+X-AI-Cache-Status: MISS
+--- wait: 0.5
+
+
+
+=== TEST 58: the SAME question under a different document (doc B) is a MISS,
not a cross-context HIT
+--- request
+POST /semantic
+{"model":"gpt-4o","messages":[{"role":"system","content":"Document B: Berlin
is the capital of Germany."},{"role":"user","content":"What is the capital of
France?"}]}
+--- more_headers
+X-AI-Fixture: openai/chat-basic.json
+--- response_headers
+X-AI-Cache-Status: MISS
+--- wait: 0.5
+
+
+
+=== TEST 59: a paraphrase under the SAME document (doc A) still hits L2
(context preserved, wording fuzzy)
+--- request
+POST /semantic
+{"model":"gpt-4o","messages":[{"role":"system","content":"Document A: Paris is
the capital of France."},{"role":"user","content":"What's the capital city of
France?"}]}
+--- response_headers_like
+X-AI-Cache-Status: HIT
+X-AI-Cache-Similarity: 0\.92\d\d
+--- response_body_like eval
+qr/1 \+ 1 = 2/
+
+
+
+=== TEST 60: semantic L2 bypasses a protocol whose canonical form is lossy
(openai-responses)
+--- config
+ location /t {
+ content_by_lua_block {
+ local semantic = require("apisix.plugins.ai-cache.semantic")
+ -- the gate returns before any embedding work, so conf/body are
+ -- never touched; an unsupported protocol must fail open as no-L2
+ local ctx = { ai_client_protocol = "openai-responses" }
+ local vec = semantic.embed_query(nil, ctx, nil)
+ ngx.say(vec == nil and "bypassed" or "engaged")
+ ngx.say(ctx.ai_cache_embedding == nil and "no-embedding" or
"embedded")
+ }
+ }
+--- response_body
+bypassed
+no-embedding
+
+
+
+=== TEST 61: a semantic block without "semantic" in layers is accepted
(inactive), not rejected
+--- config
+ location /t {
+ content_by_lua_block {
+ local plugin = require("apisix.plugins.ai-cache")
+ -- a full, valid semantic block but layers left at the default
+ -- ["exact"]: allowed so the config can be staged/feature-flagged
+ local ok, err = plugin.check_schema({
+ redis_host = "127.0.0.1",
+ semantic = {
+ embedding = { openai = {
+ model = "text-embedding-3-small", api_key = "test-key"
} },
+ vector_search = { redis = {} },
+ },
+ })
+ ngx.say(ok and "passed" or err)
+ }
+ }
+--- response_body
+passed
+
+
+
+=== TEST 62: set a semantic route for the multimodal-bypass regression
(threshold 0.9)
+--- config
+ location /t {
+ content_by_lua_block {
+ require("lib.test_redis").flush_port("127.0.0.1", 6379)
+
+ local t = require("lib.test_admin").test
+ local code, body = t('/apisix/admin/routes/1',
+ ngx.HTTP_PUT,
+ [[{
+ "uri": "/semantic",
+ "plugins": {
+ "ai-proxy": {
+ "provider": "openai",
+ "auth": { "header": { "Authorization": "Bearer
test-key" } },
+ "options": { "model": "gpt-4o" },
+ "override": { "endpoint": "http://127.0.0.1:1980"; }
+ },
+ "ai-cache": {
+ "redis_host": "127.0.0.1",
+ "redis_port": 6379,
+ "layers": ["exact", "semantic"],
+ "semantic": {
+ "similarity_threshold": 0.9,
+ "embedding": {
+ "openai": {
+ "endpoint":
"http://127.0.0.1:6724/v1/embeddings";,
+ "model": "text-embedding-3-small",
+ "api_key": "test-key"
+ }
+ },
+ "vector_search": { "redis": {} }
+ }
+ }
+ }
+ }]]
+ )
+ if code >= 300 then ngx.status = code end
+ ngx.say(body)
+ }
+ }
+--- response_body
+passed
+
+
+
+=== TEST 63: a text-only prompt warms L2 with the "capital" vector (cold MISS)
+--- request
+POST /semantic
+{"model":"gpt-4o","messages":[{"role":"user","content":"What is the capital of
France?"}]}
+--- more_headers
+X-AI-Fixture: openai/chat-basic.json
+--- response_headers
+X-AI-Cache-Status: MISS
+--- wait: 0.5
+
+
+
+=== TEST 64: the SAME text carried alongside an image block bypasses L2 (a
MISS, not a cross-modal L2 hit)
+--- request
+POST /semantic
+{"model":"gpt-4o","messages":[{"role":"user","content":[{"type":"text","text":"What
is the capital of
France?"},{"type":"image_url","image_url":{"url":"https://example.com/paris.jpg"}}]}]}
+--- more_headers
+X-AI-Fixture: openai/chat-basic.json
+--- response_headers
+X-AI-Cache-Status: MISS
+--- wait: 0.5
+
+
+
+=== TEST 65: ensure_index memo is scoped per Redis target, not by index name
alone (vector-search unit)
+--- config
+ location /t {
+ content_by_lua_block {
+ -- Unit-level on purpose: the bug only shows across two Redis
servers
+ -- (RediSearch forbids FT.CREATE on db!=0) and self-heals
end-to-end.
+ -- Model a second target that lacks the index via FT.DROPINDEX.
+ local redis_util = require("apisix.utils.redis")
+ local vs = require("apisix.plugins.ai-cache.vector-search.redis")
+ local red = assert(redis_util.new({
+ redis_host = "127.0.0.1", redis_port = 6379, redis_database =
0 }))
+ red:flushdb()
+ local index = "tgt-scope:idx:3"
+ -- ensure against target A: the per-worker memo records (A|index)
+ assert(vs.ensure_index(red, "hostA#6379#0", index,
"tgt-scope:l2:", 3))
+ -- drop the index; a different target B has never had it created
+ red[ "FT.DROPINDEX" ](red, index)
+ -- target B must NOT be served by target A's memo entry -- it must
+ -- re-issue FT.CREATE (index-name-only keying would wrongly skip
here)
+ assert(vs.ensure_index(red, "hostB#6379#0", index,
"tgt-scope:l2:", 3))
+ -- proof B's create really happened: a search is a clean miss, not
the
+ -- "no such index" error a memo collision would leave behind
+ local hit, err = vs.knn_search(red, "hostB#6379#0", index,
"deadbeef", { 1, 0, 0 }, 1)
+ ngx.say(hit == nil and not err and "recreated-for-target-B"
+ or ("skipped:" .. (err or "unexpected-hit")))
+ }
+ }
+--- response_body
+recreated-for-target-B
![http://x/y.png"](http://x/y.png"){kind=link}
![https://example.com/paris.jpg"}}]}]}](https://example.com/paris.jpg"}}]}]}){kind=link}