This is an automated email from the ASF dual-hosted git repository.
nic-6443 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/apisix.git
The following commit(s) were added to refs/heads/master by this push:
new 30ed8d7d1 feat: add ai-cache plugin (#13578)
30ed8d7d1 is described below
commit 30ed8d7d152256532c3711d31f8335b4501d4966
Author: Mohammad Izzraff Janius
<[email protected]>
AuthorDate: Mon Jun 29 10:59:01 2026 +0900
feat: add ai-cache plugin (#13578)
---
Makefile | 3 +
apisix/cli/config.lua | 1 +
apisix/core/json.lua | 48 ++
apisix/plugins/ai-cache.lua | 228 ++++++
apisix/plugins/ai-cache/key.lua | 130 +++
apisix/plugins/ai-cache/schema.lua | 89 +++
apisix/plugins/ai-transport/http.lua | 36 +-
conf/config.yaml.example | 1 +
docs/en/latest/config.json | 1 +
docs/en/latest/plugins/ai-cache.md | 335 ++++++++
docs/zh/latest/config.json | 1 +
docs/zh/latest/plugins/ai-cache.md | 335 ++++++++
t/admin/plugins.t | 1 +
t/plugin/ai-cache.t | 1448 ++++++++++++++++++++++++++++++++++
14 files changed, 2622 insertions(+), 35 deletions(-)
diff --git a/Makefile b/Makefile
index 28a76a3e2..733f62d20 100644
--- a/Makefile
+++ b/Makefile
@@ -401,6 +401,9 @@ install: runtime
$(ENV_INSTALL) -d $(ENV_INST_LUADIR)/apisix/plugins/ai-rag/vector-search
$(ENV_INSTALL) apisix/plugins/ai-rag/vector-search/*.lua
$(ENV_INST_LUADIR)/apisix/plugins/ai-rag/vector-search
+ $(ENV_INSTALL) -d $(ENV_INST_LUADIR)/apisix/plugins/ai-cache
+ $(ENV_INSTALL) apisix/plugins/ai-cache/*.lua
$(ENV_INST_LUADIR)/apisix/plugins/ai-cache
+
$(ENV_INSTALL) -d $(ENV_INST_LUADIR)/apisix/plugins/ai-lakera-guard
$(ENV_INSTALL) apisix/plugins/ai-lakera-guard/*.lua
$(ENV_INST_LUADIR)/apisix/plugins/ai-lakera-guard
diff --git a/apisix/cli/config.lua b/apisix/cli/config.lua
index 5c9d60e27..bdc22f107 100644
--- a/apisix/cli/config.lua
+++ b/apisix/cli/config.lua
@@ -244,6 +244,7 @@ local _M = {
"ai-rate-limiting",
"ai-proxy-multi",
"ai-proxy",
+ "ai-cache",
"ai-aws-content-moderation",
"ai-aliyun-content-moderation",
"ai-lakera-guard",
diff --git a/apisix/core/json.lua b/apisix/core/json.lua
index 397b80191..9418917b3 100644
--- a/apisix/core/json.lua
+++ b/apisix/core/json.lua
@@ -24,12 +24,19 @@ local json_encode = cjson.encode
local json_decode = cjson.decode
local cjson_null = cjson.null
local clear_tab = require("table.clear")
+local require = require
local ngx = ngx
local tostring = tostring
local type = type
local pairs = pairs
+local ipairs = ipairs
+local getmetatable = getmetatable
local cached_tab = {}
+local rapidjson
+local rapidjson_null
+local rapidjson_encode_opts = { sort_keys = true }
+
cjson.encode_escape_forward_slash(false)
cjson.decode_array_with_array_mt(true)
@@ -122,6 +129,47 @@ local function encode(data, force)
end
_M.encode = encode
+
+local function to_rapidjson_value(data)
+ if data == cjson_null then
+ return rapidjson_null
+ end
+
+ if type(data) ~= "table" then
+ return data
+ end
+
+ if getmetatable(data) == cjson.array_mt then
+ local arr = {}
+ for i, v in ipairs(data) do
+ arr[i] = to_rapidjson_value(v)
+ end
+ return rapidjson.array(arr)
+ end
+
+ local obj = {}
+ for k, v in pairs(data) do
+ obj[k] = to_rapidjson_value(v)
+ end
+ return obj
+end
+
+
+--- Encode a Lua value to a canonical JSON string with sorted object keys.
+-- Unlike core.json.encode, object keys are emitted in a stable (sorted) order,
+-- so the same logical value always produces the same string -- suitable for
+-- hashing, cache keys and signatures. cjson null / array_mt markers are
+-- preserved. Backed by rapidjson, which is loaded on first use.
+-- @tparam table data The value to encode.
+-- @treturn string The canonically-encoded JSON string.
+function _M.canonical_encode(data)
+ if not rapidjson then
+ rapidjson = require("rapidjson")
+ rapidjson_null = rapidjson.null
+ end
+ return rapidjson.encode(to_rapidjson_value(data), rapidjson_encode_opts)
+end
+
local max_delay_encode_items = 16
local delay_tab_idx = 0
local delay_tab_arr = {}
diff --git a/apisix/plugins/ai-cache.lua b/apisix/plugins/ai-cache.lua
new file mode 100644
index 000000000..ba6f38b43
--- /dev/null
+++ b/apisix/plugins/ai-cache.lua
@@ -0,0 +1,228 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one or more
+-- contributor license agreements. See the NOTICE file distributed with
+-- this work for additional information regarding copyright ownership.
+-- The ASF licenses this file to You under the Apache License, Version 2.0
+-- (the "License"); you may not use this file except in compliance with
+-- the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
+local core = require("apisix.core")
+local schema = require("apisix.plugins.ai-cache.schema")
+local key_mod = require("apisix.plugins.ai-cache.key")
+local binding = require("apisix.plugins.ai-protocols.binding")
+local redis_util = require("apisix.utils.redis")
+
+local ngx = ngx
+local ngx_null = ngx.null
+local ipairs = ipairs
+local concat = table.concat
+
+local CACHE_STATUS_HEADER = "X-AI-Cache-Status"
+local CACHE_AGE_HEADER = "X-AI-Cache-Age"
+local DEFAULT_TTL = 3600
+local DEFAULT_MAX_BODY = 1048576
+
+local _M = {
+ version = 0.1,
+ priority = 1035,
+ name = "ai-cache",
+ schema = schema,
+}
+
+
+function _M.check_schema(conf)
+ return core.schema.check(schema, conf)
+end
+
+
+local function release(conf, red)
+ local ok, err = red:set_keepalive(conf.redis_keepalive_timeout or 10000,
+ conf.redis_keepalive_pool or 100)
+ if not ok then
+ core.log.warn("ai-cache: failed to set redis keepalive: ", err)
+ end
+end
+
+
+local function serve_hit(conf, ctx, cached)
+ ctx.ai_cache_status = "HIT"
+ if conf.cache_headers ~= false then
+ core.response.set_header(CACHE_STATUS_HEADER, "HIT")
+ local age = ngx.time() - (cached.created_at or ngx.time())
+ core.response.set_header(CACHE_AGE_HEADER, age < 0 and 0 or age)
+ end
+ core.response.set_header("Content-Type", "application/json")
+ return core.response.exit(200, cached.body)
+end
+
+
+function _M.access(conf, ctx)
+ if not ctx.picked_ai_instance then
+ local handled, code, body = binding.on_unsupported(
+ conf.fail_mode, _M.name, ctx,
+ "no ai instance picked (request did not pass through
ai-proxy/ai-proxy-multi)",
+ 500, "ai-cache must be used with the ai-proxy or ai-proxy-multi
plugin")
+ if handled then
+ return code, body
+ end
+ ctx.ai_cache_status = "BYPASS"
+ return
+ end
+
+ -- Streaming responses are not cached in PR-1 (SSE replay is a later
+ -- increment). ai-proxy (higher priority) has already classified the
+ -- request, so bypass before doing any work.
+ if ctx.var.request_type == "ai_stream" then
+ ctx.ai_cache_status = "BYPASS"
+ return
+ end
+
+ if conf.bypass_on then
+ for _, rule in ipairs(conf.bypass_on) do
+ if core.request.header(ctx, rule.header) == rule.equals then
+ ctx.ai_cache_status = "BYPASS"
+ return
+ end
+ end
+ end
+
+ local body, err = core.request.get_json_request_body_table()
+ if not body then
+ core.log.warn("ai-cache: cannot read request body, bypassing: ", err)
+ ctx.ai_cache_status = "BYPASS"
+ return
+ end
+
+ ctx.ai_cache_fingerprint = key_mod.fingerprint(ctx, body)
+ ctx.ai_cache_key = key_mod.build(conf, ctx, ctx.ai_cache_fingerprint)
+ -- Remember which instance the fingerprint was computed for. ai-proxy-multi
+ -- may fall back to a different instance in before_proxy; the log phase
uses
+ -- this to avoid writing that fallback response under the original key.
+ ctx.ai_cache_picked_at_access = ctx.picked_ai_instance
+
+ local red
+ red, err = redis_util.new(conf)
+ if not red then
+ -- fail-open: never let a cache-backend outage break the request.
+ core.log.warn("ai-cache: redis unavailable, fail-open as MISS: ", err)
+ ctx.ai_cache_status = "MISS"
+ return
+ end
+
+ local res
+ res, err = red:get(ctx.ai_cache_key)
+ if err then
+ red:close()
+ core.log.warn("ai-cache: redis get failed, fail-open as MISS: ", err)
+ ctx.ai_cache_status = "MISS"
+ return
+ end
+ release(conf, red)
+
+ if res ~= nil and res ~= ngx_null then
+ local cached = core.json.decode(res)
+ if cached and cached.body then
+ return serve_hit(conf, ctx, cached)
+ end
+ core.log.warn("ai-cache: discarding malformed cache entry for ",
ctx.ai_cache_key)
+ end
+
+ ctx.ai_cache_status = "MISS"
+end
+
+
+function _M.header_filter(conf, ctx)
+ if ctx.ai_cache_status and conf.cache_headers ~= false then
+ core.response.set_header(CACHE_STATUS_HEADER, ctx.ai_cache_status)
+ end
+end
+
+
+function _M.body_filter(conf, ctx)
+ -- only a MISS gets written back; HIT exited in access, BYPASS opts out.
+ if ctx.ai_cache_status ~= "MISS" or ctx.ai_cache_oversized then
+ return
+ end
+ local chunk = ngx.arg[1]
+ if chunk and #chunk > 0 then
+ local buf = ctx.ai_cache_buf
+ if not buf then
+ buf = { n = 0, bytes = 0 }
+ ctx.ai_cache_buf = buf
+ end
+ local n = buf.n + 1
+ buf.n = n
+ buf[n] = chunk
+ buf.bytes = buf.bytes + #chunk
+ if buf.bytes > (conf.max_cache_body_size or DEFAULT_MAX_BODY) then
+ ctx.ai_cache_buf = nil
+ ctx.ai_cache_oversized = true
+ end
+ end
+end
+
+
+-- The response-capturing phases (body_filter / log) run in contexts where
+-- cosockets are disabled, so the Redis write is deferred to a 0-delay timer
+-- (timers run in a light thread where cosockets are allowed).
+local function write_to_cache(premature, conf, cache_key, response_body)
+ if premature then
+ return
+ end
+ local red, err = redis_util.new(conf)
+ if not red then
+ core.log.warn("ai-cache: redis unavailable on write: ", err)
+ return
+ end
+ local envelope = core.json.encode({ body = response_body, created_at =
ngx.time() })
+ local ttl = (conf.exact and conf.exact.ttl) or DEFAULT_TTL
+ local ok
+ ok, err = red:set(cache_key, envelope, "EX", ttl)
+ if not ok then
+ red:close()
+ core.log.warn("ai-cache: redis set failed: ", err)
+ return
+ end
+ release(conf, red)
+end
+
+
+function _M.log(conf, ctx)
+ if ctx.ai_cache_status ~= "MISS" or not ctx.ai_cache_fingerprint then
+ return
+ end
+ -- ai-proxy-multi may reassign the picked instance on fallback/retry during
+ -- before_proxy. The frozen fingerprint identifies the ORIGINAL instance,
so a
+ -- response actually produced by a different (fallback) instance must not
be
+ -- written under it -- that would replay the wrong instance's response on a
+ -- later hit.
+ if ctx.picked_ai_instance ~= ctx.ai_cache_picked_at_access then
+ return
+ end
+ if ngx.status ~= 200 then
+ return
+ end
+ local buf = ctx.ai_cache_buf
+ if not buf or buf.bytes == 0 then
+ return
+ end
+ local response_body = concat(buf, "", 1, buf.n)
+
+ local cache_key = key_mod.build(conf, ctx, ctx.ai_cache_fingerprint)
+ local ok, err = ngx.timer.at(0, write_to_cache, conf, cache_key,
response_body)
+ if not ok then
+ core.log.warn("ai-cache: failed to schedule cache write: ", err)
+ end
+end
+
+
+return _M
diff --git a/apisix/plugins/ai-cache/key.lua b/apisix/plugins/ai-cache/key.lua
new file mode 100644
index 000000000..3f5d408ef
--- /dev/null
+++ b/apisix/plugins/ai-cache/key.lua
@@ -0,0 +1,130 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one or more
+-- contributor license agreements. See the NOTICE file distributed with
+-- this work for additional information regarding copyright ownership.
+-- The ASF licenses this file to You under the Apache License, Version 2.0
+-- (the "License"); you may not use this file except in compliance with
+-- the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
+local core = require("apisix.core")
+local protocols = require("apisix.plugins.ai-protocols")
+local sha256 = require("resty.sha256")
+local to_hex = require("resty.string").to_hex
+
+local ipairs = ipairs
+local pairs = pairs
+local concat = table.concat
+local tostring = tostring
+
+local KEY_PREFIX = "ai-cache:l1:"
+
+local _M = {}
+
+
+local function hex_digest(s)
+ local hash = sha256:new()
+ hash:update(s)
+ return to_hex(hash:final())
+end
+
+
+local function client_messages(ctx, body)
+ local proto = ctx.ai_client_protocol and
protocols.get(ctx.ai_client_protocol)
+ if proto and proto.get_messages then
+ return proto.get_messages(body) or {}
+ end
+ return {}
+end
+
+
+-- Identity of the EFFECTIVE upstream request, reconstructed from access-time
+-- inputs only.
+--
+-- final_upstream_body = build_request(client_body, ai_client_protocol,
+-- instance{provider, options, override})
+--
+-- is deterministic, and ai-proxy builds it later (in before_proxy) so it
cannot
+-- be observed here. Hashing build_request's INPUTS therefore identifies its
+-- output uniquely, without invoking the (side-effecting) builder. This is the
+-- ONLY place request-determining data lives; scope() below is pure isolation.
+function _M.fingerprint(ctx, body)
+ local inst = ctx.picked_ai_instance
+ local ov = inst.override or {}
+
+ local params = {}
+ for k, v in pairs(body) do
+ if k ~= "messages" and k ~= "model" and k ~= "stream" then
+ params[k] = v
+ end
+ end
+
+ local repr = core.json.canonical_encode({
+ client = {
+ protocol = ctx.ai_client_protocol or "",
+ messages = client_messages(ctx, body),
+ params = params,
+ },
+ effective = {
+ provider = inst.provider,
+ -- effective model precedence mirrors ai-proxy/base.lua exactly:
+ -- the instance's options.model wins over the client body model.
+ model = (inst.options and inst.options.model) or body.model
or "",
+ options = inst.options,
+ llm_options = ov.llm_options,
+ request_body = ov.request_body,
+ request_body_force_override = ov.request_body_force_override,
+ -- override.endpoint can carry a path/query that selects a
different
+ -- deployment or model (azure deployment, bedrock inference-profile
+ -- ARN, vertex project/region/model), so it is
response-determining.
+ endpoint = ov.endpoint,
+ },
+ })
+ return hex_digest(repr)
+end
+
+
+-- Percent-encode "%", ":" and "=" (in that order) in scope values so a
request-controlled
+-- include_vars value can't shift "name=value:" boundaries to forge another
scope.
+local function esc(v)
+ return (tostring(v or ""):gsub("%%", "%%25"):gsub(":", "%%3A"):gsub("=",
"%%3D"))
+end
+
+
+local function scope(conf, ctx)
+ local ck = conf.cache_key or {}
+
+ local parts = {}
+ if not ck.share_across_routes then
+ parts[#parts + 1] = "route=" .. esc(ctx.var.route_id)
+ end
+ if ck.include_consumer then
+ parts[#parts + 1] = "consumer=" .. esc(ctx.consumer_name)
+ end
+ if ck.include_vars then
+ for _, name in ipairs(ck.include_vars) do
+ parts[#parts + 1] = name .. "=" .. esc(ctx.var[name])
+ end
+ end
+
+ if #parts == 0 then
+ return "shared"
+ end
+ return concat(parts, ":")
+end
+
+
+function _M.build(conf, ctx, fingerprint)
+ return KEY_PREFIX .. scope(conf, ctx) .. ":" .. fingerprint
+end
+
+
+return _M
diff --git a/apisix/plugins/ai-cache/schema.lua
b/apisix/plugins/ai-cache/schema.lua
new file mode 100644
index 000000000..5eb62661c
--- /dev/null
+++ b/apisix/plugins/ai-cache/schema.lua
@@ -0,0 +1,89 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one or more
+-- contributor license agreements. See the NOTICE file distributed with
+-- this work for additional information regarding copyright ownership.
+-- The ASF licenses this file to You under the Apache License, Version 2.0
+-- (the "License"); you may not use this file except in compliance with
+-- the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
+local core = require("apisix.core")
+local redis_schema = require("apisix.utils.redis-schema")
+local binding = require("apisix.plugins.ai-protocols.binding")
+
+local policy_to_additional_properties =
core.table.deepcopy(redis_schema.schema)
+
+local _M = {
+ type = "object",
+ properties = {
+ exact = {
+ type = "object",
+ properties = {
+ ttl = { type = "integer", minimum = 1, default = 3600 },
+ },
+ default = {},
+ },
+
+ cache_key = {
+ type = "object",
+ properties = {
+ share_across_routes = { type = "boolean", default = false },
+ include_consumer = { type = "boolean", default = false },
+ include_vars = {
+ type = "array",
+ items = { type = "string" },
+ default = {},
+ },
+ },
+ default = {},
+ },
+
+ max_cache_body_size = {
+ type = "integer", minimum = 0, default = 1048576,
+ },
+
+ cache_headers = {
+ type = "boolean", default = true,
+ },
+
+ fail_mode = binding.schema_property("skip"),
+
+ bypass_on = {
+ type = "array",
+ minItems = 1,
+ items = {
+ type = "object",
+ properties = {
+ header = { type = "string", minLength = 1 },
+ equals = { type = "string" },
+ },
+ required = { "header", "equals" },
+ },
+ },
+
+ policy = {
+ type = "string",
+ enum = { "redis" },
+ default = "redis",
+ },
+ },
+ ["if"] = {
+ properties = {
+ policy = {
+ enum = { "redis" },
+ },
+ },
+ },
+ ["then"] = policy_to_additional_properties.redis,
+ encrypt_fields = { "redis_password" },
+}
+
+return _M
diff --git a/apisix/plugins/ai-transport/http.lua
b/apisix/plugins/ai-transport/http.lua
index eb7efc34b..5ea9d2194 100644
--- a/apisix/plugins/ai-transport/http.lua
+++ b/apisix/plugins/ai-transport/http.lua
@@ -20,8 +20,6 @@
local core = require("apisix.core")
local http = require("resty.http")
-local rapidjson = require("rapidjson")
-local getmetatable = getmetatable
local ngx_now = ngx.now
local pairs = pairs
local ipairs = ipairs
@@ -31,8 +29,6 @@ local str_lower = string.lower
local tostring = tostring
local _M = {}
-local rapidjson_encode_opts = {sort_keys = true}
-local rapidjson_null = rapidjson.null
--- Map network errors to HTTP status codes.
@@ -73,38 +69,8 @@ function _M.construct_forward_headers(ext_opts_headers, ctx)
end
-local function to_rapidjson_value(data)
- if data == core.json.null then
- return rapidjson_null
- end
-
- if type(data) ~= "table" then
- return data
- end
-
- if getmetatable(data) == core.json.array_mt then
- local arr = {}
- for i, v in ipairs(data) do
- arr[i] = to_rapidjson_value(v)
- end
- return rapidjson.array(arr)
- end
-
- local obj = {}
- for k, v in pairs(data) do
- obj[k] = to_rapidjson_value(v)
- end
- return obj
-end
-
-
-local function rapidjson_encode(body)
- return rapidjson.encode(to_rapidjson_value(body), rapidjson_encode_opts)
-end
-
-
local function encode_body(body)
- local ok, encoded = pcall(rapidjson_encode, body)
+ local ok, encoded = pcall(core.json.canonical_encode, body)
if ok and encoded then
return encoded
end
diff --git a/conf/config.yaml.example b/conf/config.yaml.example
index 0a129a5ac..efbf20d6b 100644
--- a/conf/config.yaml.example
+++ b/conf/config.yaml.example
@@ -538,6 +538,7 @@ plugins: # plugin list (sorted by
priority)
- ai-aws-content-moderation # priority: 1050
- ai-proxy-multi # priority: 1041
- ai-proxy # priority: 1040
+ - ai-cache # priority: 1035
- ai-rate-limiting # priority: 1030
- ai-aliyun-content-moderation # priority: 1029
- ai-lakera-guard # priority: 1028
diff --git a/docs/en/latest/config.json b/docs/en/latest/config.json
index a881707e7..e5cfeb512 100644
--- a/docs/en/latest/config.json
+++ b/docs/en/latest/config.json
@@ -73,6 +73,7 @@
"items": [
"plugins/ai-proxy",
"plugins/ai-proxy-multi",
+ "plugins/ai-cache",
"plugins/ai-rate-limiting",
"plugins/ai-prompt-guard",
"plugins/ai-aws-content-moderation",
diff --git a/docs/en/latest/plugins/ai-cache.md
b/docs/en/latest/plugins/ai-cache.md
new file mode 100644
index 000000000..bbeacdf73
--- /dev/null
+++ b/docs/en/latest/plugins/ai-cache.md
@@ -0,0 +1,335 @@
+---
+title: ai-cache
+keywords:
+ - Apache APISIX
+ - API Gateway
+ - Plugin
+ - ai-cache
+ - AI
+ - LLM
+description: The ai-cache Plugin caches LLM responses in Redis and replays
them for later requests that resolve to the same prompt, cutting upstream token
cost and latency.
+---
+
+<!--
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+-->
+
+<head>
+ <link rel="canonical" href="https://docs.api7.ai/hub/ai-cache"; />
+</head>
+
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+
+## Description
+
+The `ai-cache` Plugin caches LLM responses and replays them for later requests
that resolve to the same prompt, cutting upstream token cost and latency for
repetitive workloads (FAQ bots, document Q&A, translation).
+
+This release implements the **exact** cache layer (L1); a semantic cache layer
(L2) is planned for a future release.
+
+The `ai-cache` Plugin must be used with the [`ai-proxy`](./ai-proxy.md) or
[`ai-proxy-multi`](./ai-proxy-multi.md) Plugin.
+
+:::note
+
+By default the cache is isolated per route, so two routes never serve each
other's entries even when they see the same protocol, model and messages. Set
`cache_key.share_across_routes` to `true` to share one cache space across
routes.
+
+Even with `cache_key.share_across_routes` enabled, the cache key identifies
the *effective* upstream request — the request `ai-proxy` actually sends after
applying the AI instance's `provider`, `options` (model, temperature, and other
model parameters) and `override`. Routes that would call the model differently
therefore keep separate cache entries, so one route's response is never served
for another.
+
+:::
+
+## Attributes
+
+| Name | Type | Required | Default | Valid values | Description |
+|------|------|----------|---------|--------------|-------------|
+| exact.ttl | integer | False | 3600 | >= 1 | Time-to-live, in seconds, of an
exact-cache entry. |
+| cache_key.share_across_routes | boolean | False | false | | By default the
cache is isolated per route. If true, entries are shared across every route
that computes the same key. |
+| cache_key.include_consumer | boolean | False | false | | If true, scope the
cache per consumer so entries are not shared across consumers. |
+| cache_key.include_vars | array[string] | False | [] | | NGINX variables
added to the cache scope (for example `["http_x_tenant"]`), isolating entries
by their values. |
+| max_cache_body_size | integer | False | 1048576 | >= 0 | Maximum response
body size, in bytes, to cache. Larger responses are not cached. |
+| cache_headers | boolean | False | true | | If true, add the
`X-AI-Cache-Status` response header (and `X-AI-Cache-Age`, the entry age in
seconds, on a hit). |
+| fail_mode | string | False | `"skip"` | `skip`, `warn`, `error` | Behavior
when the request is not a recognized AI request that this Plugin can cache (for
example, a request that did not pass through `ai-proxy` or `ai-proxy-multi`).
`skip`: let the request pass through uncached; `warn`: pass through uncached
and log a warning; `error`: reject the request. |
+| bypass_on | array[object] | False | | | Rules that skip the cache entirely
(no lookup, no write-back) when any rule matches. |
+| bypass_on[].header | string | True | | | Request header name to match. |
+| bypass_on[].equals | string | True | | | Bypass when the request header's
value exactly equals this string. |
+| policy | string | False | redis | redis | Storage backend. Only single-node
`redis` is available in this release. |
+| redis_host | string | True | | | Address of the Redis node. |
+| redis_port | integer | False | 6379 | >= 1 | Port of the Redis node. |
+| redis_username | string | False | | | Username for Redis if Redis ACL is
used. For the legacy `requirepass` method, configure only `redis_password`. |
+| redis_password | string | False | | | Password of the Redis node. Encrypted
with AES before being stored in etcd. |
+| redis_database | integer | False | 0 | >= 0 | Database number in Redis. |
+| redis_timeout | integer | False | 1000 | >= 1 | Redis timeout value in
milliseconds. |
+| redis_ssl | boolean | False | false | | If true, use SSL to connect to
Redis. |
+| redis_ssl_verify | boolean | False | false | | If true, verify the Redis
server SSL certificate. |
+| redis_keepalive_timeout | integer | False | 10000 | >= 1000 | Keepalive
timeout, in milliseconds, for the Redis connection pool. |
+| redis_keepalive_pool | integer | False | 100 | >= 1 | Maximum number of
connections in the Redis keepalive pool. |
+
+## Example
+
+The example below uses OpenAI as the Upstream LLM provider. Obtain an [OpenAI
API key](https://openai.com/blog/openai-api) and save it, along with your Admin
API key, to environment variables:
+
+```shell
+export OPENAI_API_KEY=your-openai-api-key
+export admin_key=$(yq '.deployment.admin.admin_key[0].key' conf/config.yaml |
sed 's/"//g')
+```
+
+A Redis instance must be reachable at the configured `redis_host`.
+
+### Cache LLM Responses
+
+Create a Route to the LLM chat completion endpoint with the
[`ai-proxy`](./ai-proxy.md) and `ai-cache` Plugins.
+
+<Tabs
+groupId="api"
+defaultValue="admin-api"
+values={[
+{label: 'Admin API', value: 'admin-api'},
+{label: 'ADC', value: 'adc'},
+{label: 'Ingress Controller', value: 'aic'}
+]}>
+
+<TabItem value="admin-api">
+
+```shell
+curl "http://127.0.0.1:9180/apisix/admin/routes"; -X PUT \
+ -H "X-API-KEY: ${admin_key}" \
+ -d '{
+ "id": "ai-cache-route",
+ "uri": "/anything",
+ "plugins": {
+ "ai-proxy": {
+ "provider": "openai",
+ "auth": { "header": { "Authorization": "Bearer '"$OPENAI_API_KEY"'" }
},
+ "options": { "model": "gpt-4o" }
+ },
+ "ai-cache": {
+ "redis_host": "127.0.0.1"
+ }
+ }
+ }'
+```
+
+</TabItem>
+
+<TabItem value="adc">
+
+```yaml title="adc.yaml"
+services:
+ - name: ai-cache-service
+ routes:
+ - name: ai-cache-route
+ uris:
+ - /anything
+ methods:
+ - POST
+ plugins:
+ ai-proxy:
+ provider: openai
+ auth:
+ header:
+ Authorization: "Bearer ${OPENAI_API_KEY}"
+ options:
+ model: gpt-4o
+ ai-cache:
+ redis_host: 127.0.0.1
+```
+
+Synchronize the configuration to the gateway:
+
+```shell
+adc sync -f adc.yaml
+```
+
+</TabItem>
+
+<TabItem value="aic">
+
+<Tabs
+groupId="k8s-api"
+defaultValue="gateway-api"
+values={[
+{label: 'Gateway API', value: 'gateway-api'},
+{label: 'APISIX CRD', value: 'apisix-crd'}
+]}>
+
+<TabItem value="gateway-api">
+
+```yaml title="ai-cache-ic.yaml"
+apiVersion: apisix.apache.org/v1alpha1
+kind: PluginConfig
+metadata:
+ namespace: aic
+ name: ai-cache-plugin-config
+spec:
+ plugins:
+ - name: ai-cache
+ config:
+ redis_host: 127.0.0.1
+ - name: ai-proxy
+ config:
+ provider: openai
+ auth:
+ header:
+ Authorization: "Bearer your-openai-api-key"
+ options:
+ model: gpt-4o
+---
+apiVersion: gateway.networking.k8s.io/v1
+kind: HTTPRoute
+metadata:
+ namespace: aic
+ name: ai-cache-route
+spec:
+ parentRefs:
+ - name: apisix
+ rules:
+ - matches:
+ - path:
+ type: Exact
+ value: /anything
+ method: POST
+ filters:
+ - type: ExtensionRef
+ extensionRef:
+ group: apisix.apache.org
+ kind: PluginConfig
+ name: ai-cache-plugin-config
+```
+
+Apply the configuration to your cluster:
+
+```shell
+kubectl apply -f ai-cache-ic.yaml
+```
+
+</TabItem>
+
+<TabItem value="apisix-crd">
+
+```yaml title="ai-cache-ic.yaml"
+apiVersion: apisix.apache.org/v2
+kind: ApisixRoute
+metadata:
+ namespace: aic
+ name: ai-cache-route
+spec:
+ ingressClassName: apisix
+ http:
+ - name: ai-cache-route
+ match:
+ paths:
+ - /anything
+ methods:
+ - POST
+ plugins:
+ - name: ai-cache
+ enable: true
+ config:
+ redis_host: 127.0.0.1
+ - name: ai-proxy
+ enable: true
+ config:
+ provider: openai
+ auth:
+ header:
+ Authorization: "Bearer your-openai-api-key"
+ options:
+ model: gpt-4o
+```
+
+Apply the configuration to your cluster:
+
+```shell
+kubectl apply -f ai-cache-ic.yaml
+```
+
+</TabItem>
+
+</Tabs>
+
+</TabItem>
+
+</Tabs>
+
+Send a request to the Route:
+
+```shell
+curl -i "http://127.0.0.1:9080/anything"; -X POST \
+ -H "Content-Type: application/json" \
+ -d '{ "messages": [{ "role": "user", "content": "What is Apache APISIX?
Answer in one sentence." }] }'
+```
+
+The first request is a cache miss and is proxied to the LLM. The response
carries the `X-AI-Cache-Status: MISS` header and a body similar to the
following:
+
+```json
+{
+ "id": "chatcmpl-DtmdUDZeSZ0t62y6BvLkSk5qfH3zA",
+ "object": "chat.completion",
+ "created": 1782187368,
+ "model": "gpt-4o-2024-08-06",
+ "choices": [
+ {
+ "index": 0,
+ "message": {
+ "role": "assistant",
+ "content": "Apache APISIX is a dynamic, cloud-native API gateway that
provides high performance, scalability, and security for API management."
+ },
+ "finish_reason": "stop"
+ }
+ ],
+ "usage": {
+ "prompt_tokens": 19,
+ "completion_tokens": 25,
+ "total_tokens": 44
+ }
+}
+```
+
+Send the same request again. It is served from the cache without calling the
LLM, returning the identical body with the headers:
+
+```text
+X-AI-Cache-Status: HIT
+X-AI-Cache-Age: 8
+```
+
+### Bypass the Cache
+
+To skip the cache for selected requests, add a `bypass_on` rule and update the
Route:
+
+```shell
+curl "http://127.0.0.1:9180/apisix/admin/routes/ai-cache-route"; -X PATCH \
+ -H "X-API-KEY: ${admin_key}" \
+ -d '{
+ "plugins": {
+ "ai-cache": {
+ "redis_host": "127.0.0.1",
+ "bypass_on": [{ "header": "X-Cache-Bypass", "equals": "1" }]
+ }
+ }
+ }'
+```
+
+Send a request with the matching header:
+
+```shell
+curl -i "http://127.0.0.1:9080/anything"; -X POST \
+ -H "Content-Type: application/json" \
+ -H "X-Cache-Bypass: 1" \
+ -d '{ "messages": [{ "role": "user", "content": "What is Apache APISIX?
Answer in one sentence." }] }'
+```
+
+The cache is skipped entirely (no lookup and no write-back), and the response
carries the `X-AI-Cache-Status: BYPASS` header.
diff --git a/docs/zh/latest/config.json b/docs/zh/latest/config.json
index 4d3cdf1e6..b1cac9d24 100644
--- a/docs/zh/latest/config.json
+++ b/docs/zh/latest/config.json
@@ -64,6 +64,7 @@
"items": [
"plugins/ai-proxy",
"plugins/ai-proxy-multi",
+ "plugins/ai-cache",
"plugins/ai-rate-limiting",
"plugins/ai-prompt-guard",
"plugins/ai-aws-content-moderation",
diff --git a/docs/zh/latest/plugins/ai-cache.md
b/docs/zh/latest/plugins/ai-cache.md
new file mode 100644
index 000000000..3793317bd
--- /dev/null
+++ b/docs/zh/latest/plugins/ai-cache.md
@@ -0,0 +1,335 @@
+---
+title: ai-cache
+keywords:
+ - Apache APISIX
+ - API 网关
+ - 插件
+ - ai-cache
+ - AI
+ - LLM
+description: ai-cache 插件将 LLM 响应缓存在 Redis 中，并在后续解析到相同提示词的请求中重放这些响应，从而降低上游的
Token 消耗与延迟。
+---
+
+<!--
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+-->
+
+<head>
+ <link rel="canonical" href="https://docs.api7.ai/hub/ai-cache"; />
+</head>
+
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+
+## 描述
+
+`ai-cache` 插件缓存 LLM 响应，并在后续解析到相同提示词的请求中重放这些响应，从而为重复性工作负载（FAQ
机器人、文档问答、翻译等）降低上游的 Token 消耗与延迟。
+
+本次发布实现了**精确**缓存层（L1）；语义缓存层（L2）计划在未来的版本中提供。
+
+`ai-cache` 插件必须与 [`ai-proxy`](./ai-proxy.md) 或
[`ai-proxy-multi`](./ai-proxy-multi.md) 插件一起使用。
+
+:::note
+
+默认情况下缓存按路由隔离，因此即使两个路由看到相同的协议、模型与消息，也不会相互返回对方的缓存条目。将
`cache_key.share_across_routes` 设为 `true` 可让多个路由共享同一个缓存空间。
+
+即使开启 `cache_key.share_across_routes`，来自不同上游模型或 provider
的响应也会分别存储在各自的缓存条目中，因此某个模型的响应绝不会被返回给另一个模型。
+
+:::
+
+## 属性
+
+| 名称 | 类型 | 必选项 | 默认值 | 有效值 | 描述 |
+|------|------|--------|--------|--------|------|
+| exact.ttl | integer | 否 | 3600 | >= 1 | 精确缓存条目的存活时间（TTL），单位为秒。 |
+| cache_key.share_across_routes | boolean | 否 | false | | 默认情况下缓存按路由隔离。如果为
true，则计算出相同缓存键的所有路由之间共享缓存条目。 |
+| cache_key.include_consumer | boolean | 否 | false | | 如果为
true，则按消费者隔离缓存，使缓存条目不会在不同消费者之间共享。 |
+| cache_key.include_vars | array[string] | 否 | [] | | 加入缓存作用域的 NGINX 变量（例如
`["http_x_tenant"]`），按其取值隔离缓存条目。 |
+| max_cache_body_size | integer | 否 | 1048576 | >= 0 |
允许缓存的最大响应体大小，单位为字节。超过该大小的响应不会被缓存。 |
+| cache_headers | boolean | 否 | true | | 如果为 true，则添加 `X-AI-Cache-Status`
响应头（命中时还会添加 `X-AI-Cache-Age`，表示缓存条目的存在时长，单位为秒）。 |
+| fail_mode | string | 否 | `"skip"` | `skip`、`warn`、`error` | 当请求不是该插件可缓存的 AI
请求时的处理行为（例如未经过 `ai-proxy` 或 `ai-proxy-multi`
的请求）。`skip`：放行请求且不缓存；`warn`：放行不缓存并记录 warning 日志；`error`：拒绝请求。 |
+| bypass_on | array[object] | 否 | | | 当任一规则匹配时，完全跳过缓存（不查询、不回写）的规则列表。 |
+| bypass_on[].header | string | 是 | | | 要匹配的请求头名称。 |
+| bypass_on[].equals | string | 是 | | | 当该请求头的值与此字符串完全相等时，绕过缓存。 |
+| policy | string | 否 | redis | redis | 存储后端。本次发布仅支持单节点 `redis`。 |
+| redis_host | string | 是 | | | Redis 节点的地址。 |
+| redis_port | integer | 否 | 6379 | >= 1 | Redis 节点的端口。 |
+| redis_username | string | 否 | | | 使用 Redis ACL 时的用户名。如果使用传统的 `requirepass`
认证方式，则仅配置 `redis_password`。 |
+| redis_password | string | 否 | | | Redis 节点的密码。在存入 etcd 之前使用 AES 加密。 |
+| redis_database | integer | 否 | 0 | >= 0 | Redis 中使用的数据库编号。 |
+| redis_timeout | integer | 否 | 1000 | >= 1 | Redis 超时时间，单位为毫秒。 |
+| redis_ssl | boolean | 否 | false | | 如果为 true，则使用 SSL 连接 Redis。 |
+| redis_ssl_verify | boolean | 否 | false | | 如果为 true，则校验 Redis 服务器的 SSL 证书。 |
+| redis_keepalive_timeout | integer | 否 | 10000 | >= 1000 | Redis
连接池的保活超时时间，单位为毫秒。 |
+| redis_keepalive_pool | integer | 否 | 100 | >= 1 | Redis 保活连接池中的最大连接数。 |
+
+## 示例
+
+以下示例使用 OpenAI 作为上游 LLM 服务提供商。请获取 [OpenAI API
key](https://openai.com/blog/openai-api)，并将其与 Admin API key 一起保存到环境变量中：
+
+```shell
+export OPENAI_API_KEY=your-openai-api-key
+export admin_key=$(yq '.deployment.admin.admin_key[0].key' conf/config.yaml |
sed 's/"//g')
+```
+
+在配置的 `redis_host` 上必须有一个可访问的 Redis 实例。
+
+### 缓存 LLM 响应
+
+使用 [`ai-proxy`](./ai-proxy.md) 和 `ai-cache` 插件创建一个指向 LLM 聊天补全端点的路由。
+
+<Tabs
+groupId="api"
+defaultValue="admin-api"
+values={[
+{label: 'Admin API', value: 'admin-api'},
+{label: 'ADC', value: 'adc'},
+{label: 'Ingress Controller', value: 'aic'}
+]}>
+
+<TabItem value="admin-api">
+
+```shell
+curl "http://127.0.0.1:9180/apisix/admin/routes"; -X PUT \
+ -H "X-API-KEY: ${admin_key}" \
+ -d '{
+ "id": "ai-cache-route",
+ "uri": "/anything",
+ "plugins": {
+ "ai-proxy": {
+ "provider": "openai",
+ "auth": { "header": { "Authorization": "Bearer '"$OPENAI_API_KEY"'" }
},
+ "options": { "model": "gpt-4o" }
+ },
+ "ai-cache": {
+ "redis_host": "127.0.0.1"
+ }
+ }
+ }'
+```
+
+</TabItem>
+
+<TabItem value="adc">
+
+```yaml title="adc.yaml"
+services:
+ - name: ai-cache-service
+ routes:
+ - name: ai-cache-route
+ uris:
+ - /anything
+ methods:
+ - POST
+ plugins:
+ ai-proxy:
+ provider: openai
+ auth:
+ header:
+ Authorization: "Bearer ${OPENAI_API_KEY}"
+ options:
+ model: gpt-4o
+ ai-cache:
+ redis_host: 127.0.0.1
+```
+
+将配置同步到网关：
+
+```shell
+adc sync -f adc.yaml
+```
+
+</TabItem>
+
+<TabItem value="aic">
+
+<Tabs
+groupId="k8s-api"
+defaultValue="gateway-api"
+values={[
+{label: 'Gateway API', value: 'gateway-api'},
+{label: 'APISIX CRD', value: 'apisix-crd'}
+]}>
+
+<TabItem value="gateway-api">
+
+```yaml title="ai-cache-ic.yaml"
+apiVersion: apisix.apache.org/v1alpha1
+kind: PluginConfig
+metadata:
+ namespace: aic
+ name: ai-cache-plugin-config
+spec:
+ plugins:
+ - name: ai-cache
+ config:
+ redis_host: 127.0.0.1
+ - name: ai-proxy
+ config:
+ provider: openai
+ auth:
+ header:
+ Authorization: "Bearer your-openai-api-key"
+ options:
+ model: gpt-4o
+---
+apiVersion: gateway.networking.k8s.io/v1
+kind: HTTPRoute
+metadata:
+ namespace: aic
+ name: ai-cache-route
+spec:
+ parentRefs:
+ - name: apisix
+ rules:
+ - matches:
+ - path:
+ type: Exact
+ value: /anything
+ method: POST
+ filters:
+ - type: ExtensionRef
+ extensionRef:
+ group: apisix.apache.org
+ kind: PluginConfig
+ name: ai-cache-plugin-config
+```
+
+将配置应用到您的集群：
+
+```shell
+kubectl apply -f ai-cache-ic.yaml
+```
+
+</TabItem>
+
+<TabItem value="apisix-crd">
+
+```yaml title="ai-cache-ic.yaml"
+apiVersion: apisix.apache.org/v2
+kind: ApisixRoute
+metadata:
+ namespace: aic
+ name: ai-cache-route
+spec:
+ ingressClassName: apisix
+ http:
+ - name: ai-cache-route
+ match:
+ paths:
+ - /anything
+ methods:
+ - POST
+ plugins:
+ - name: ai-cache
+ enable: true
+ config:
+ redis_host: 127.0.0.1
+ - name: ai-proxy
+ enable: true
+ config:
+ provider: openai
+ auth:
+ header:
+ Authorization: "Bearer your-openai-api-key"
+ options:
+ model: gpt-4o
+```
+
+将配置应用到您的集群：
+
+```shell
+kubectl apply -f ai-cache-ic.yaml
+```
+
+</TabItem>
+
+</Tabs>
+
+</TabItem>
+
+</Tabs>
+
+向该路由发送请求：
+
+```shell
+curl -i "http://127.0.0.1:9080/anything"; -X POST \
+ -H "Content-Type: application/json" \
+ -d '{ "messages": [{ "role": "user", "content": "What is Apache APISIX?
Answer in one sentence." }] }'
+```
+
+第一次请求是缓存未命中（MISS），会被代理到 LLM。响应中携带 `X-AI-Cache-Status: MISS` 响应头，响应体类似如下：
+
+```json
+{
+ "id": "chatcmpl-DtmdUDZeSZ0t62y6BvLkSk5qfH3zA",
+ "object": "chat.completion",
+ "created": 1782187368,
+ "model": "gpt-4o-2024-08-06",
+ "choices": [
+ {
+ "index": 0,
+ "message": {
+ "role": "assistant",
+ "content": "Apache APISIX is a dynamic, cloud-native API gateway that
provides high performance, scalability, and security for API management."
+ },
+ "finish_reason": "stop"
+ }
+ ],
+ "usage": {
+ "prompt_tokens": 19,
+ "completion_tokens": 25,
+ "total_tokens": 44
+ }
+}
+```
+
+再次发送相同的请求。该请求将直接由缓存返回，而不会调用 LLM，返回完全相同的响应体，并携带以下响应头：
+
+```text
+X-AI-Cache-Status: HIT
+X-AI-Cache-Age: 8
+```
+
+### 绕过缓存
+
+如需为特定请求跳过缓存，可添加 `bypass_on` 规则并更新路由：
+
+```shell
+curl "http://127.0.0.1:9180/apisix/admin/routes/ai-cache-route"; -X PATCH \
+ -H "X-API-KEY: ${admin_key}" \
+ -d '{
+ "plugins": {
+ "ai-cache": {
+ "redis_host": "127.0.0.1",
+ "bypass_on": [{ "header": "X-Cache-Bypass", "equals": "1" }]
+ }
+ }
+ }'
+```
+
+发送带有匹配请求头的请求：
+
+```shell
+curl -i "http://127.0.0.1:9080/anything"; -X POST \
+ -H "Content-Type: application/json" \
+ -H "X-Cache-Bypass: 1" \
+ -d '{ "messages": [{ "role": "user", "content": "What is Apache APISIX?
Answer in one sentence." }] }'
+```
+
+缓存被完全跳过（不查询、不回写），响应中携带 `X-AI-Cache-Status: BYPASS` 响应头。
diff --git a/t/admin/plugins.t b/t/admin/plugins.t
index ab80a63ed..ec33d940a 100644
--- a/t/admin/plugins.t
+++ b/t/admin/plugins.t
@@ -108,6 +108,7 @@ ai-rag
ai-aws-content-moderation
ai-proxy-multi
ai-proxy
+ai-cache
ai-rate-limiting
ai-aliyun-content-moderation
ai-lakera-guard
diff --git a/t/plugin/ai-cache.t b/t/plugin/ai-cache.t
new file mode 100644
index 000000000..ddc72eb1c
--- /dev/null
+++ b/t/plugin/ai-cache.t
@@ -0,0 +1,1448 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+BEGIN {
+ $ENV{TEST_ENABLE_CONTROL_API_V1} = "0";
+}
+
+use t::APISIX 'no_plan';
+
+log_level("info");
+repeat_each(1);
+no_long_string();
+no_root_location();
+
+add_block_preprocessor(sub {
+ my ($block) = @_;
+
+ if (!defined $block->request) {
+ $block->set_value("request", "GET /t");
+ }
+
+ my $user_yaml_config = <<_EOC_;
+plugins:
+ - ai-proxy
+ - ai-cache
+_EOC_
+ if (!defined $block->extra_yaml_config) {
+ $block->set_value("extra_yaml_config", $user_yaml_config);
+ }
+
+ my $http_config = $block->http_config // <<_EOC_;
+ server {
+ listen 6731;
+ default_type 'application/json';
+ location / {
+ content_by_lua_block {
+ ngx.status = 500
+ ngx.say([[{"error":{"message":"primary down"}}]])
+ }
+ }
+ }
+_EOC_
+ $block->set_value("http_config", $http_config);
+});
+
+run_tests();
+
+__DATA__
+
+=== TEST 1: minimal valid exact-cache configuration
+--- config
+ location /t {
+ content_by_lua_block {
+ local plugin = require("apisix.plugins.ai-cache")
+ local ok, err = plugin.check_schema({
+ redis_host = "127.0.0.1",
+ redis_port = 6379,
+ })
+
+ if not ok then
+ ngx.say(err)
+ else
+ ngx.say("passed")
+ end
+ }
+ }
+--- response_body
+passed
+
+
+
+=== TEST 2: reject config missing required redis (policy=redis then-clause)
+--- config
+ location /t {
+ content_by_lua_block {
+ local plugin = require("apisix.plugins.ai-cache")
+ local ok, err = plugin.check_schema({})
+
+ if not ok then
+ ngx.say(err)
+ else
+ ngx.say("passed")
+ end
+ }
+ }
+--- response_body eval
+qr/then clause did not match/
+
+
+
+=== TEST 3: reject an out-of-range exact.ttl
+--- config
+ location /t {
+ content_by_lua_block {
+ local plugin = require("apisix.plugins.ai-cache")
+ local ok, err = plugin.check_schema({
+ redis_host = "127.0.0.1",
+ exact = { ttl = 0 },
+ })
+
+ if not ok then
+ ngx.say(err)
+ else
+ ngx.say("passed")
+ end
+ }
+ }
+--- response_body eval
+qr/ttl/
+
+
+
+=== TEST 4: flush redis, then set route with ai-proxy + ai-cache (mock
upstream)
+--- config
+ location /t {
+ content_by_lua_block {
+ require("lib.test_redis").flush_port("127.0.0.1", 6379)
+
+ local t = require("lib.test_admin").test
+ local code, body = t('/apisix/admin/routes/1',
+ ngx.HTTP_PUT,
+ [[{
+ "uri": "/anything",
+ "plugins": {
+ "ai-proxy": {
+ "provider": "openai",
+ "auth": {
+ "header": {
+ "Authorization": "Bearer test-key"
+ }
+ },
+ "options": {
+ "model": "gpt-4o"
+ },
+ "override": {
+ "endpoint": "http://127.0.0.1:1980";
+ }
+ },
+ "ai-cache": {
+ "redis_host": "127.0.0.1",
+ "redis_port": 6379
+ }
+ }
+ }]]
+ )
+
+ if code >= 300 then
+ ngx.status = code
+ end
+ ngx.say(body)
+ }
+ }
+--- response_body
+passed
+
+
+
+=== TEST 5: cold request is a cache MISS and is proxied upstream
+--- request
+POST /anything
+{"model":"gpt-4o","messages":[{"role":"user","content":"ai-cache miss
unique-prompt-5"}]}
+--- more_headers
+X-AI-Fixture: openai/chat-basic.json
+--- response_headers
+X-AI-Cache-Status: MISS
+--- response_body_like eval
+qr/1 \+ 1 = 2/
+--- wait: 0.3
+
+
+
+=== TEST 6: identical re-request is a HIT served from cache (upstream not
called)
+--- request
+POST /anything
+{"model":"gpt-4o","messages":[{"role":"user","content":"ai-cache miss
unique-prompt-5"}]}
+--- error_code: 200
+--- response_headers_like
+X-AI-Cache-Status: HIT
+X-AI-Cache-Age: \d+
+--- response_body_like eval
+qr/1 \+ 1 = 2/
+
+
+
+=== TEST 7: fingerprint covers the client request AND the effective instance
config (key.lua unit)
+--- config
+ location /t {
+ content_by_lua_block {
+ local core = require("apisix.core")
+ local key = require("apisix.plugins.ai-cache.key")
+
+ -- fingerprint() identifies the effective upstream request, so it
reads
+ -- both the client body and the picked instance
(provider/options/override).
+ local function fp(inst, body)
+ local ctx = { ai_client_protocol = "openai-chat", var = {},
+ picked_ai_instance = inst }
+ return key.fingerprint(ctx, body)
+ end
+
+ local inst = { provider = "openai", options = {}, override = {} }
+ local prompt = { model = "gpt-4o", messages = {{role="user",
content="hi"}}, temperature = 0.2 }
+ local base = fp(inst, prompt)
+
+ -- client fields the upstream would see: a change flips the
fingerprint
+ local other_messages = { model = "gpt-4o", messages =
{{role="user", content="bye"}} }
+ local other_model = { model = "gpt-4o-mini", messages =
{{role="user", content="hi"}} }
+ local other_temp = { model = "gpt-4o", messages =
{{role="user", content="hi"}}, temperature = 0.9 }
+ local with_stream = { model = "gpt-4o", messages =
{{role="user", content="hi"}}, temperature = 0.2, stream = true }
+
+ assert(fp(inst, prompt) == base, "identical request and
config must match")
+ assert(fp(inst, other_messages) ~= base, "different messages")
+ assert(fp(inst, other_model) ~= base, "different client model")
+ assert(fp(inst, other_temp) ~= base, "different temperature")
+ assert(fp(inst, with_stream) == base, "the stream flag is
stripped, so it must not matter")
+
+ -- instance config that ai-proxy applies upstream: a change flips
it too
+ local other_provider = { provider = "deepseek", options = {},
override = {} }
+ local forced_model = { provider = "openai", options = { model
= "gpt-4o-mini" }, override = {} }
+ local server_temp = { provider = "openai", options = {
temperature = 0.9 }, override = {} }
+ local llm_opts = { provider = "openai", options = {},
override = { llm_options = { max_tokens = 16 } } }
+ local body_override = { provider = "openai", options = {},
override = { request_body = { ["openai-chat"] = { foo = "bar" } } } }
+ local endpoint_a = { provider = "openai", options = {},
override = { endpoint = "http://host/a"; } }
+ local endpoint_b = { provider = "openai", options = {},
override = { endpoint = "http://host/b"; } }
+
+ assert(fp(other_provider, prompt) ~= base, "different provider")
+ assert(fp(forced_model, prompt) ~= base, "different effective
model")
+ assert(fp(server_temp, prompt) ~= base, "different server-side
options")
+ assert(fp(llm_opts, prompt) ~= base, "different
override.llm_options")
+ assert(fp(body_override, prompt) ~= base, "different
override.request_body")
+ assert(fp(endpoint_a, prompt) ~= fp(endpoint_b, prompt),
"different override.endpoint")
+
+ -- a forced options.model overrides the client model, so the
client model stops mattering
+ assert(fp(forced_model, { messages = prompt.messages, model =
"gpt-4o" })
+ == fp(forced_model, { messages = prompt.messages, model =
"zzz" }),
+ "forced options.model makes the client model irrelevant")
+
+ -- an explicit JSON null must encode stably (regression for the
rapidjson null path)
+ local with_null = core.json.decode(
+
'{"model":"gpt-4o","messages":[{"role":"user","content":"hi"}],"stop":null}')
+ assert(fp(inst, with_null) == fp(inst, with_null), "null-bearing
fingerprint must be stable")
+ assert(fp(inst, with_null) ~= base, "an explicit null must change
the fingerprint")
+
+ ngx.say("passed")
+ }
+ }
+--- response_body
+passed
+
+
+
+=== TEST 8: non-2xx upstream (no fixture -> 401) is a MISS
+--- request
+POST /anything
+{"model":"gpt-4o","messages":[{"role":"user","content":"non-2xx-test-prompt"}]}
+--- error_code: 401
+--- response_headers
+X-AI-Cache-Status: MISS
+--- wait: 0.3
+
+
+
+=== TEST 9: same prompt with a valid fixture is still a MISS (the 401 was not
cached)
+--- request
+POST /anything
+{"model":"gpt-4o","messages":[{"role":"user","content":"non-2xx-test-prompt"}]}
+--- more_headers
+X-AI-Fixture: openai/chat-basic.json
+--- error_code: 200
+--- response_headers
+X-AI-Cache-Status: MISS
+--- response_body_like eval
+qr/1 \+ 1 = 2/
+
+
+
+=== TEST 10: set route with a bypass_on header rule
+--- config
+ location /t {
+ content_by_lua_block {
+ local t = require("lib.test_admin").test
+ local code, body = t('/apisix/admin/routes/1',
+ ngx.HTTP_PUT,
+ [[{
+ "uri": "/anything",
+ "plugins": {
+ "ai-proxy": {
+ "provider": "openai",
+ "auth": { "header": { "Authorization": "Bearer
test-key" } },
+ "options": { "model": "gpt-4o" },
+ "override": { "endpoint": "http://127.0.0.1:1980"; }
+ },
+ "ai-cache": {
+ "redis_host": "127.0.0.1",
+ "redis_port": 6379,
+ "bypass_on": [{"header": "X-AI-Cache-Bypass",
"equals": "1"}]
+ }
+ }
+ }]]
+ )
+ if code >= 300 then
+ ngx.status = code
+ end
+ ngx.say(body)
+ }
+ }
+--- response_body
+passed
+
+
+
+=== TEST 11: a matching bypass_on header value is a BYPASS
+--- request
+POST /anything
+{"model":"gpt-4o","messages":[{"role":"user","content":"bypass rule test"}]}
+--- more_headers
+X-AI-Fixture: openai/chat-basic.json
+X-AI-Cache-Bypass: 1
+--- response_headers
+X-AI-Cache-Status: BYPASS
+
+
+
+=== TEST 12: a non-matching bypass_on header value does not bypass (normal
MISS)
+--- request
+POST /anything
+{"model":"gpt-4o","messages":[{"role":"user","content":"bypass-nonmatch-test"}]}
+--- more_headers
+X-AI-Fixture: openai/chat-basic.json
+X-AI-Cache-Bypass: 0
+--- response_headers
+X-AI-Cache-Status: MISS
+
+
+
+=== TEST 13: set route with multiple bypass_on rules
+--- config
+ location /t {
+ content_by_lua_block {
+ local t = require("lib.test_admin").test
+ local code, body = t('/apisix/admin/routes/1',
+ ngx.HTTP_PUT,
+ [[{
+ "uri": "/anything",
+ "plugins": {
+ "ai-proxy": {
+ "provider": "openai",
+ "auth": { "header": { "Authorization": "Bearer
test-key" } },
+ "options": { "model": "gpt-4o" },
+ "override": { "endpoint": "http://127.0.0.1:1980"; }
+ },
+ "ai-cache": {
+ "redis_host": "127.0.0.1",
+ "redis_port": 6379,
+ "bypass_on": [
+ {"header": "X-AI-Cache-Bypass", "equals": "1"},
+ {"header": "X-Debug", "equals": "on"}
+ ]
+ }
+ }
+ }]]
+ )
+ if code >= 300 then
+ ngx.status = code
+ end
+ ngx.say(body)
+ }
+ }
+--- response_body
+passed
+
+
+
+=== TEST 14: any matching bypass_on rule triggers a BYPASS (second rule
matches)
+--- request
+POST /anything
+{"model":"gpt-4o","messages":[{"role":"user","content":"any-rule-bypass-test"}]}
+--- more_headers
+X-AI-Fixture: openai/chat-basic.json
+X-Debug: on
+--- response_headers
+X-AI-Cache-Status: BYPASS
+
+
+
+=== TEST 15: set route with a tiny max_cache_body_size
+--- config
+ location /t {
+ content_by_lua_block {
+ local t = require("lib.test_admin").test
+ local code, body = t('/apisix/admin/routes/1',
+ ngx.HTTP_PUT,
+ [[{
+ "uri": "/anything",
+ "plugins": {
+ "ai-proxy": {
+ "provider": "openai",
+ "auth": { "header": { "Authorization": "Bearer
test-key" } },
+ "options": { "model": "gpt-4o" },
+ "override": { "endpoint": "http://127.0.0.1:1980"; }
+ },
+ "ai-cache": {
+ "redis_host": "127.0.0.1",
+ "redis_port": 6379,
+ "max_cache_body_size": 10
+ }
+ }
+ }]]
+ )
+ if code >= 300 then
+ ngx.status = code
+ end
+ ngx.say(body)
+ }
+ }
+--- response_body
+passed
+
+
+
+=== TEST 16: cold request (response exceeds max_cache_body_size) is a MISS
+--- request
+POST /anything
+{"model":"gpt-4o","messages":[{"role":"user","content":"body-size-test"}]}
+--- more_headers
+X-AI-Fixture: openai/chat-basic.json
+--- response_headers
+X-AI-Cache-Status: MISS
+--- wait: 0.3
+
+
+
+=== TEST 17: same prompt is still a MISS (oversized response was not cached)
+--- request
+POST /anything
+{"model":"gpt-4o","messages":[{"role":"user","content":"body-size-test"}]}
+--- more_headers
+X-AI-Fixture: openai/chat-basic.json
+--- response_headers
+X-AI-Cache-Status: MISS
+
+
+
+=== TEST 18: set route isolating the cache by a request variable
+--- config
+ location /t {
+ content_by_lua_block {
+ local t = require("lib.test_admin").test
+ local code, body = t('/apisix/admin/routes/1',
+ ngx.HTTP_PUT,
+ [[{
+ "uri": "/anything",
+ "plugins": {
+ "ai-proxy": {
+ "provider": "openai",
+ "auth": { "header": { "Authorization": "Bearer
test-key" } },
+ "options": { "model": "gpt-4o" },
+ "override": { "endpoint": "http://127.0.0.1:1980"; }
+ },
+ "ai-cache": {
+ "redis_host": "127.0.0.1",
+ "redis_port": 6379,
+ "cache_key": { "include_vars": ["http_x_tenant"] }
+ }
+ }
+ }]]
+ )
+ if code >= 300 then
+ ngx.status = code
+ end
+ ngx.say(body)
+ }
+ }
+--- response_body
+passed
+
+
+
+=== TEST 19: tenant alpha cold request is a MISS (warms scope=alpha)
+--- request
+POST /anything
+{"model":"gpt-4o","messages":[{"role":"user","content":"scope isolation
test"}]}
+--- more_headers
+X-AI-Fixture: openai/chat-basic.json
+X-Tenant: alpha
+--- response_headers
+X-AI-Cache-Status: MISS
+--- wait: 0.3
+
+
+
+=== TEST 20: same prompt, tenant beta is a MISS (not shared with alpha)
+--- request
+POST /anything
+{"model":"gpt-4o","messages":[{"role":"user","content":"scope isolation
test"}]}
+--- more_headers
+X-AI-Fixture: openai/chat-basic.json
+X-Tenant: beta
+--- response_headers
+X-AI-Cache-Status: MISS
+
+
+
+=== TEST 21: same prompt, tenant alpha is a HIT (its own scope persisted)
+--- request
+POST /anything
+{"model":"gpt-4o","messages":[{"role":"user","content":"scope isolation
test"}]}
+--- more_headers
+X-Tenant: alpha
+--- error_code: 200
+--- response_headers
+X-AI-Cache-Status: HIT
+
+
+
+=== TEST 22: set route with a 1-second exact ttl
+--- config
+ location /t {
+ content_by_lua_block {
+ local t = require("lib.test_admin").test
+ local code, body = t('/apisix/admin/routes/1',
+ ngx.HTTP_PUT,
+ [[{
+ "uri": "/anything",
+ "plugins": {
+ "ai-proxy": {
+ "provider": "openai",
+ "auth": { "header": { "Authorization": "Bearer
test-key" } },
+ "options": { "model": "gpt-4o" },
+ "override": { "endpoint": "http://127.0.0.1:1980"; }
+ },
+ "ai-cache": {
+ "redis_host": "127.0.0.1",
+ "redis_port": 6379,
+ "exact": { "ttl": 1 }
+ }
+ }
+ }]]
+ )
+ if code >= 300 then
+ ngx.status = code
+ end
+ ngx.say(body)
+ }
+ }
+--- response_body
+passed
+
+
+
+=== TEST 23: cold request is a MISS (cached with ttl=1), then wait past the ttl
+--- request
+POST /anything
+{"model":"gpt-4o","messages":[{"role":"user","content":"ttl-expiry-test"}]}
+--- more_headers
+X-AI-Fixture: openai/chat-basic.json
+--- response_headers
+X-AI-Cache-Status: MISS
+--- wait: 2
+
+
+
+=== TEST 24: same prompt is a MISS again (entry expired)
+--- request
+POST /anything
+{"model":"gpt-4o","messages":[{"role":"user","content":"ttl-expiry-test"}]}
+--- more_headers
+X-AI-Fixture: openai/chat-basic.json
+--- response_headers
+X-AI-Cache-Status: MISS
+
+
+
+=== TEST 25: set an anthropic-messages route (cross-protocol)
+--- config
+ location /t {
+ content_by_lua_block {
+ local t = require("lib.test_admin").test
+ local code, body = t('/apisix/admin/routes/2',
+ ngx.HTTP_PUT,
+ [[{
+ "uri": "/v1/messages",
+ "plugins": {
+ "ai-proxy": {
+ "provider": "anthropic",
+ "auth": { "header": { "x-api-key": "test-key" } },
+ "options": { "model": "claude-3-5-sonnet-20241022"
},
+ "override": { "endpoint": "http://127.0.0.1:1980"; }
+ },
+ "ai-cache": {
+ "redis_host": "127.0.0.1",
+ "redis_port": 6379
+ }
+ }
+ }]]
+ )
+ if code >= 300 then
+ ngx.status = code
+ end
+ ngx.say(body)
+ }
+ }
+--- response_body
+passed
+
+
+
+=== TEST 26: anthropic cold request is a MISS
+--- request
+POST /v1/messages
+{"model":"claude-3-5-sonnet-20241022","messages":[{"role":"user","content":"cross-protocol
test"}],"max_tokens":100}
+--- more_headers
+X-AI-Fixture: anthropic/messages-basic.json
+--- response_headers
+X-AI-Cache-Status: MISS
+--- wait: 0.3
+
+
+
+=== TEST 27: identical anthropic re-request is a HIT (upstream not called)
+--- request
+POST /v1/messages
+{"model":"claude-3-5-sonnet-20241022","messages":[{"role":"user","content":"cross-protocol
test"}],"max_tokens":100}
+--- error_code: 200
+--- response_headers
+X-AI-Cache-Status: HIT
+
+
+
+=== TEST 28: set route whose redis is unreachable
+--- config
+ location /t {
+ content_by_lua_block {
+ local t = require("lib.test_admin").test
+ local code, body = t('/apisix/admin/routes/1',
+ ngx.HTTP_PUT,
+ [[{
+ "uri": "/anything",
+ "plugins": {
+ "ai-proxy": {
+ "provider": "openai",
+ "auth": { "header": { "Authorization": "Bearer
test-key" } },
+ "options": { "model": "gpt-4o" },
+ "override": { "endpoint": "http://127.0.0.1:1980"; }
+ },
+ "ai-cache": {
+ "redis_host": "127.0.0.1",
+ "redis_port": 6390,
+ "redis_timeout": 200
+ }
+ }
+ }]]
+ )
+ if code >= 300 then
+ ngx.status = code
+ end
+ ngx.say(body)
+ }
+ }
+--- response_body
+passed
+
+
+
+=== TEST 29: redis unreachable fails open (request still proxied as MISS, no
5xx)
+--- request
+POST /anything
+{"model":"gpt-4o","messages":[{"role":"user","content":"redis-down failopen"}]}
+--- more_headers
+X-AI-Fixture: openai/chat-basic.json
+--- error_code: 200
+--- response_headers
+X-AI-Cache-Status: MISS
+--- response_body_like eval
+qr/1 \+ 1 = 2/
+--- error_log
+ai-cache: redis unavailable, fail-open as MISS
+
+
+
+=== TEST 30: set route with cache_headers disabled
+--- config
+ location /t {
+ content_by_lua_block {
+ local t = require("lib.test_admin").test
+ local code, body = t('/apisix/admin/routes/1',
+ ngx.HTTP_PUT,
+ [[{
+ "uri": "/anything",
+ "plugins": {
+ "ai-proxy": {
+ "provider": "openai",
+ "auth": { "header": { "Authorization": "Bearer
test-key" } },
+ "options": { "model": "gpt-4o" },
+ "override": { "endpoint": "http://127.0.0.1:1980"; }
+ },
+ "ai-cache": {
+ "redis_host": "127.0.0.1",
+ "redis_port": 6379,
+ "cache_headers": false
+ }
+ }
+ }]]
+ )
+ if code >= 300 then
+ ngx.status = code
+ end
+ ngx.say(body)
+ }
+ }
+--- response_body
+passed
+
+
+
+=== TEST 31: cache_headers=false suppresses the X-AI-Cache-* headers
+--- request
+POST /anything
+{"model":"gpt-4o","messages":[{"role":"user","content":"cache-headers-off-test"}]}
+--- more_headers
+X-AI-Fixture: openai/chat-basic.json
+--- error_code: 200
+--- response_headers
+! X-AI-Cache-Status
+! X-AI-Cache-Age
+--- response_body_like eval
+qr/1 \+ 1 = 2/
+
+
+
+=== TEST 32: set a default ai-proxy + ai-cache route (for status-code tests)
+--- config
+ location /t {
+ content_by_lua_block {
+ local t = require("lib.test_admin").test
+ local code, body = t('/apisix/admin/routes/1',
+ ngx.HTTP_PUT,
+ [[{
+ "uri": "/anything",
+ "plugins": {
+ "ai-proxy": {
+ "provider": "openai",
+ "auth": { "header": { "Authorization": "Bearer
test-key" } },
+ "options": { "model": "gpt-4o" },
+ "override": { "endpoint": "http://127.0.0.1:1980"; }
+ },
+ "ai-cache": {
+ "redis_host": "127.0.0.1",
+ "redis_port": 6379
+ }
+ }
+ }]]
+ )
+ if code >= 300 then
+ ngx.status = code
+ end
+ ngx.say(body)
+ }
+ }
+--- response_body
+passed
+
+
+
+=== TEST 33: a 2xx that is not 200 (201) is a MISS and is proxied through
+--- request
+POST /anything
+{"model":"gpt-4o","messages":[{"role":"user","content":"status-201-test-prompt"}]}
+--- more_headers
+X-AI-Fixture: openai/chat-basic.json
+X-AI-Fixture-Status: 201
+--- error_code: 201
+--- response_headers
+X-AI-Cache-Status: MISS
+--- wait: 0.3
+
+
+
+=== TEST 34: same prompt with a 200 fixture is still a MISS (the 201 was not
cached)
+--- request
+POST /anything
+{"model":"gpt-4o","messages":[{"role":"user","content":"status-201-test-prompt"}]}
+--- more_headers
+X-AI-Fixture: openai/chat-basic.json
+--- error_code: 200
+--- response_headers
+X-AI-Cache-Status: MISS
+--- response_body_like eval
+qr/1 \+ 1 = 2/
+
+
+
+=== TEST 35: set two openai routes (same model, default scope) sharing one
Redis
+--- config
+ location /t {
+ content_by_lua_block {
+ local t = require("lib.test_admin").test
+
+ local code, body = t('/apisix/admin/routes/1',
+ ngx.HTTP_PUT,
+ [[{
+ "uri": "/anything",
+ "plugins": {
+ "ai-proxy": {
+ "provider": "openai",
+ "auth": { "header": { "Authorization": "Bearer
test-key" } },
+ "options": { "model": "gpt-4o" },
+ "override": { "endpoint": "http://127.0.0.1:1980"; }
+ },
+ "ai-cache": {
+ "redis_host": "127.0.0.1",
+ "redis_port": 6379
+ }
+ }
+ }]]
+ )
+ if code >= 300 then
+ ngx.status = code
+ end
+ ngx.say(body)
+
+ code, body = t('/apisix/admin/routes/2',
+ ngx.HTTP_PUT,
+ [[{
+ "uri": "/cache-route-b",
+ "plugins": {
+ "ai-proxy": {
+ "provider": "openai",
+ "auth": { "header": { "Authorization": "Bearer
test-key" } },
+ "options": { "model": "gpt-4o" },
+ "override": { "endpoint": "http://127.0.0.1:1980"; }
+ },
+ "ai-cache": {
+ "redis_host": "127.0.0.1",
+ "redis_port": 6379
+ }
+ }
+ }]]
+ )
+ if code >= 300 then
+ ngx.status = code
+ end
+ ngx.say(body)
+ }
+ }
+--- response_body
+passed
+passed
+
+
+
+=== TEST 36: route 1 cold request is a MISS (warms scope=route=1)
+--- request
+POST /anything
+{"model":"gpt-4o","messages":[{"role":"user","content":"cross-route isolation
test"}]}
+--- more_headers
+X-AI-Fixture: openai/chat-basic.json
+--- response_headers
+X-AI-Cache-Status: MISS
+--- wait: 0.3
+
+
+
+=== TEST 37: same prompt on route 2 is a MISS (not shared with route 1 by
default)
+--- request
+POST /cache-route-b
+{"model":"gpt-4o","messages":[{"role":"user","content":"cross-route isolation
test"}]}
+--- more_headers
+X-AI-Fixture: openai/chat-basic.json
+--- response_headers
+X-AI-Cache-Status: MISS
+
+
+
+=== TEST 38: same prompt on route 1 is a HIT (its own per-route scope
persisted)
+--- request
+POST /anything
+{"model":"gpt-4o","messages":[{"role":"user","content":"cross-route isolation
test"}]}
+--- error_code: 200
+--- response_headers
+X-AI-Cache-Status: HIT
+
+
+
+=== TEST 39: set both routes with share_across_routes enabled
+--- config
+ location /t {
+ content_by_lua_block {
+ local t = require("lib.test_admin").test
+
+ local code, body = t('/apisix/admin/routes/1',
+ ngx.HTTP_PUT,
+ [[{
+ "uri": "/anything",
+ "plugins": {
+ "ai-proxy": {
+ "provider": "openai",
+ "auth": { "header": { "Authorization": "Bearer
test-key" } },
+ "options": { "model": "gpt-4o" },
+ "override": { "endpoint": "http://127.0.0.1:1980"; }
+ },
+ "ai-cache": {
+ "redis_host": "127.0.0.1",
+ "redis_port": 6379,
+ "cache_key": { "share_across_routes": true }
+ }
+ }
+ }]]
+ )
+ if code >= 300 then
+ ngx.status = code
+ end
+ ngx.say(body)
+
+ code, body = t('/apisix/admin/routes/2',
+ ngx.HTTP_PUT,
+ [[{
+ "uri": "/cache-route-b",
+ "plugins": {
+ "ai-proxy": {
+ "provider": "openai",
+ "auth": { "header": { "Authorization": "Bearer
test-key" } },
+ "options": { "model": "gpt-4o" },
+ "override": { "endpoint": "http://127.0.0.1:1980"; }
+ },
+ "ai-cache": {
+ "redis_host": "127.0.0.1",
+ "redis_port": 6379,
+ "cache_key": { "share_across_routes": true }
+ }
+ }
+ }]]
+ )
+ if code >= 300 then
+ ngx.status = code
+ end
+ ngx.say(body)
+ }
+ }
+--- response_body
+passed
+passed
+
+
+
+=== TEST 40: route 1 cold request is a MISS (warms the shared scope)
+--- request
+POST /anything
+{"model":"gpt-4o","messages":[{"role":"user","content":"cross-route share
test"}]}
+--- more_headers
+X-AI-Fixture: openai/chat-basic.json
+--- response_headers
+X-AI-Cache-Status: MISS
+--- wait: 0.3
+
+
+
+=== TEST 41: same prompt on route 2 is a HIT (cache shared across routes)
+--- request
+POST /cache-route-b
+{"model":"gpt-4o","messages":[{"role":"user","content":"cross-route share
test"}]}
+--- error_code: 200
+--- response_headers
+X-AI-Cache-Status: HIT
+
+
+
+=== TEST 42: route with ai-cache but NO ai-proxy in front
+--- config
+ location /t {
+ content_by_lua_block {
+ local t = require("lib.test_admin").test
+ local code, body = t('/apisix/admin/routes/1',
+ ngx.HTTP_PUT,
+ [[{
+ "uri": "/v1/chat/completions",
+ "upstream": {
+ "type": "roundrobin",
+ "nodes": { "127.0.0.1:1980": 1 }
+ },
+ "plugins": {
+ "ai-cache": {
+ "redis_host": "127.0.0.1",
+ "redis_port": 6379
+ }
+ }
+ }]]
+ )
+ if code >= 300 then
+ ngx.status = code
+ end
+ ngx.say(body)
+ }
+ }
+--- response_body
+passed
+
+
+
+=== TEST 43: a request that never passed through ai-proxy is bypassed, not
cached
+--- request
+POST /v1/chat/completions
+{"model":"gpt-4o","messages":[{"role":"user","content":"no ai-proxy guard
test"}]}
+--- more_headers
+X-AI-Fixture: openai/chat-basic.json
+--- error_code: 200
+--- response_headers
+X-AI-Cache-Status: BYPASS
+
+
+
+=== TEST 44: route with ai-cache fail_mode=error and NO ai-proxy
+--- config
+ location /t {
+ content_by_lua_block {
+ local t = require("lib.test_admin").test
+ local code, body = t('/apisix/admin/routes/1',
+ ngx.HTTP_PUT,
+ [[{
+ "uri": "/v1/chat/completions",
+ "upstream": {
+ "type": "roundrobin",
+ "nodes": { "127.0.0.1:1980": 1 }
+ },
+ "plugins": {
+ "ai-cache": {
+ "redis_host": "127.0.0.1",
+ "redis_port": 6379,
+ "fail_mode": "error"
+ }
+ }
+ }]]
+ )
+ if code >= 300 then
+ ngx.status = code
+ end
+ ngx.say(body)
+ }
+ }
+--- response_body
+passed
+
+
+
+=== TEST 45: fail_mode=error rejects a request that bypassed the AI proxy
+--- request
+POST /v1/chat/completions
+{"model":"gpt-4o","messages":[{"role":"user","content":"fail_mode error guard
test"}]}
+--- more_headers
+X-AI-Fixture: openai/chat-basic.json
+--- error_code: 500
+--- response_body_like eval
+qr/must be used with the ai-proxy/
+
+
+
+=== TEST 46: flush redis, then set one ai-proxy-multi route with two instances
+--- extra_yaml_config
+plugins:
+ - ai-proxy-multi
+ - ai-cache
+--- config
+ location /t {
+ content_by_lua_block {
+ require("lib.test_redis").flush_port("127.0.0.1", 6379)
+
+ local t = require("lib.test_admin").test
+ local code, body = t('/apisix/admin/routes/1',
+ ngx.HTTP_PUT,
+ [[{
+ "uri": "/multi",
+ "plugins": {
+ "ai-proxy-multi": {
+ "instances": [
+ {
+ "name": "instance-gpt4o",
+ "provider": "openai",
+ "weight": 1,
+ "auth": { "header": { "Authorization":
"Bearer test-key" } },
+ "options": { "model": "gpt-4o" },
+ "override": { "endpoint":
"http://127.0.0.1:1980"; }
+ },
+ {
+ "name": "instance-gpt4o-mini",
+ "provider": "openai",
+ "weight": 1,
+ "auth": { "header": { "Authorization":
"Bearer test-key" } },
+ "options": { "model": "gpt-4o-mini" },
+ "override": { "endpoint":
"http://127.0.0.1:1980"; }
+ }
+ ]
+ },
+ "ai-cache": {
+ "redis_host": "127.0.0.1",
+ "redis_port": 6379
+ }
+ }
+ }]]
+ )
+ if code >= 300 then
+ ngx.status = code
+ end
+ ngx.say(body)
+ }
+ }
+--- response_body
+passed
+
+
+
+=== TEST 47: round-robin alternates instances, so each one caches independently
+--- extra_yaml_config
+plugins:
+ - ai-proxy-multi
+ - ai-cache
+--- pipelined_requests eval
+[
+ "POST /multi\n" .
'{"model":"gpt-4o","messages":[{"role":"user","content":"multi-instance
isolation"}]}',
+ "POST /multi\n" .
'{"model":"gpt-4o","messages":[{"role":"user","content":"multi-instance
isolation"}]}',
+ "POST /multi\n" .
'{"model":"gpt-4o","messages":[{"role":"user","content":"multi-instance
isolation"}]}',
+ "POST /multi\n" .
'{"model":"gpt-4o","messages":[{"role":"user","content":"multi-instance
isolation"}]}',
+]
+--- more_headers
+X-AI-Fixture: openai/chat-basic.json
+--- response_headers eval
+[
+ "X-AI-Cache-Status: MISS",
+ "X-AI-Cache-Status: MISS",
+ "X-AI-Cache-Status: HIT",
+ "X-AI-Cache-Status: HIT",
+]
+
+
+
+=== TEST 48: flush redis, then two plain ai-proxy routes, same provider,
different
+options.model, shared cache
+--- config
+ location /t {
+ content_by_lua_block {
+ require("lib.test_redis").flush_port("127.0.0.1", 6379)
+
+ local t = require("lib.test_admin").test
+
+ local code, body = t('/apisix/admin/routes/1',
+ ngx.HTTP_PUT,
+ [[{
+ "uri": "/anything",
+ "plugins": {
+ "ai-proxy": {
+ "provider": "openai",
+ "auth": { "header": { "Authorization": "Bearer
test-key" } },
+ "options": { "model": "gpt-4o" },
+ "override": { "endpoint": "http://127.0.0.1:1980"; }
+ },
+ "ai-cache": {
+ "redis_host": "127.0.0.1",
+ "redis_port": 6379,
+ "cache_key": { "share_across_routes": true }
+ }
+ }
+ }]]
+ )
+ if code >= 300 then
+ ngx.status = code
+ end
+ ngx.say(body)
+
+ code, body = t('/apisix/admin/routes/2',
+ ngx.HTTP_PUT,
+ [[{
+ "uri": "/cache-route-b",
+ "plugins": {
+ "ai-proxy": {
+ "provider": "openai",
+ "auth": { "header": { "Authorization": "Bearer
test-key" } },
+ "options": { "model": "gpt-4o-mini" },
+ "override": { "endpoint": "http://127.0.0.1:1980"; }
+ },
+ "ai-cache": {
+ "redis_host": "127.0.0.1",
+ "redis_port": 6379,
+ "cache_key": { "share_across_routes": true }
+ }
+ }
+ }]]
+ )
+ if code >= 300 then
+ ngx.status = code
+ end
+ ngx.say(body)
+ }
+ }
+--- response_body
+passed
+passed
+
+
+
+=== TEST 49: route 1 cold request is a MISS (warms the shared scope under
gpt-4o)
+--- request
+POST /anything
+{"model":"gpt-4o","messages":[{"role":"user","content":"cross-model share
test"}]}
+--- more_headers
+X-AI-Fixture: openai/chat-basic.json
+--- response_headers
+X-AI-Cache-Status: MISS
+--- wait: 0.3
+
+
+
+=== TEST 50: identical request on route 2 is a MISS, not a HIT (different
effective
+model is not shared even with share_across_routes)
+--- request
+POST /cache-route-b
+{"model":"gpt-4o","messages":[{"role":"user","content":"cross-model share
test"}]}
+--- more_headers
+X-AI-Fixture: openai/chat-basic.json
+--- response_headers
+X-AI-Cache-Status: MISS
+
+
+
+=== TEST 51: flush redis, then two plain ai-proxy routes, same provider and
same
+options.model, but different server-side options (temperature), shared cache
+--- config
+ location /t {
+ content_by_lua_block {
+ require("lib.test_redis").flush_port("127.0.0.1", 6379)
+
+ local t = require("lib.test_admin").test
+
+ local code, body = t('/apisix/admin/routes/1',
+ ngx.HTTP_PUT,
+ [[{
+ "uri": "/anything",
+ "plugins": {
+ "ai-proxy": {
+ "provider": "openai",
+ "auth": { "header": { "Authorization": "Bearer
test-key" } },
+ "options": { "model": "gpt-4o", "temperature": 0.2
},
+ "override": { "endpoint": "http://127.0.0.1:1980"; }
+ },
+ "ai-cache": {
+ "redis_host": "127.0.0.1",
+ "redis_port": 6379,
+ "cache_key": { "share_across_routes": true }
+ }
+ }
+ }]]
+ )
+ if code >= 300 then
+ ngx.status = code
+ end
+ ngx.say(body)
+
+ code, body = t('/apisix/admin/routes/2',
+ ngx.HTTP_PUT,
+ [[{
+ "uri": "/cache-route-b",
+ "plugins": {
+ "ai-proxy": {
+ "provider": "openai",
+ "auth": { "header": { "Authorization": "Bearer
test-key" } },
+ "options": { "model": "gpt-4o", "temperature": 0.8
},
+ "override": { "endpoint": "http://127.0.0.1:1980"; }
+ },
+ "ai-cache": {
+ "redis_host": "127.0.0.1",
+ "redis_port": 6379,
+ "cache_key": { "share_across_routes": true }
+ }
+ }
+ }]]
+ )
+ if code >= 300 then
+ ngx.status = code
+ end
+ ngx.say(body)
+ }
+ }
+--- response_body
+passed
+passed
+
+
+
+=== TEST 52: route 1 cold request is a MISS (warms the shared scope under
temperature=0.2)
+--- request
+POST /anything
+{"model":"gpt-4o","messages":[{"role":"user","content":"cross-options share
test"}]}
+--- more_headers
+X-AI-Fixture: openai/chat-basic.json
+--- response_headers
+X-AI-Cache-Status: MISS
+--- wait: 0.3
+
+
+
+=== TEST 53: identical client body on route 2 is a MISS, not a HIT (different
+server-side options are not shared even with share_across_routes)
+--- request
+POST /cache-route-b
+{"model":"gpt-4o","messages":[{"role":"user","content":"cross-options share
test"}]}
+--- more_headers
+X-AI-Fixture: openai/chat-basic.json
+--- response_headers
+X-AI-Cache-Status: MISS
+
+
+
+=== TEST 54: ai-proxy-multi route whose higher-priority instance always 5xxs
and
+falls back to a healthy instance (different effective endpoint)
+--- extra_yaml_config
+plugins:
+ - ai-proxy-multi
+ - ai-cache
+--- config
+ location /t {
+ content_by_lua_block {
+ require("lib.test_redis").flush_port("127.0.0.1", 6379)
+
+ local t = require("lib.test_admin").test
+ local code, body = t('/apisix/admin/routes/1',
+ ngx.HTTP_PUT,
+ [[{
+ "uri": "/fallback",
+ "plugins": {
+ "ai-proxy-multi": {
+ "fallback_strategy": ["http_5xx"],
+ "balancer": { "algorithm": "roundrobin" },
+ "instances": [
+ {
+ "name": "primary-fail",
+ "provider": "openai",
+ "weight": 1,
+ "priority": 2,
+ "auth": { "header": { "Authorization":
"Bearer test-key" } },
+ "options": { "model": "gpt-4o" },
+ "override": { "endpoint":
"http://127.0.0.1:6731"; }
+ },
+ {
+ "name": "backup-ok",
+ "provider": "openai",
+ "weight": 1,
+ "priority": 1,
+ "auth": { "header": { "Authorization":
"Bearer test-key" } },
+ "options": { "model": "gpt-4o" },
+ "override": { "endpoint":
"http://127.0.0.1:1980"; }
+ }
+ ]
+ },
+ "ai-cache": {
+ "redis_host": "127.0.0.1",
+ "redis_port": 6379
+ }
+ }
+ }]]
+ )
+ if code >= 300 then
+ ngx.status = code
+ end
+ ngx.say(body)
+ }
+ }
+--- response_body
+passed
+
+
+
+=== TEST 55: first request falls back to the healthy instance and is a MISS
+--- extra_yaml_config
+plugins:
+ - ai-proxy-multi
+ - ai-cache
+--- request
+POST /fallback
+{"model":"gpt-4o","messages":[{"role":"user","content":"fallback poison
test"}]}
+--- more_headers
+X-AI-Fixture: openai/chat-basic.json
+--- error_code: 200
+--- response_headers
+X-AI-Cache-Status: MISS
+--- wait: 0.3
+
+
+
+=== TEST 56: identical request is STILL a MISS -- the fallback instance's
response
+must not be cached under the originally-picked instance's key
+--- extra_yaml_config
+plugins:
+ - ai-proxy-multi
+ - ai-cache
+--- request
+POST /fallback
+{"model":"gpt-4o","messages":[{"role":"user","content":"fallback poison
test"}]}
+--- more_headers
+X-AI-Fixture: openai/chat-basic.json
+--- error_code: 200
+--- response_headers
+X-AI-Cache-Status: MISS
+
+
+
+=== TEST 57: scope() escapes include_vars values so a tenant cannot forge
another's scope (key.lua unit)
+--- config
+ location /t {
+ content_by_lua_block {
+ local key = require("apisix.plugins.ai-cache.key")
+
+ -- include_vars values are request-controlled (e.g. headers), so
they
+ -- must not be able to inject the ":"/"=" scope separators and
shift
+ -- field boundaries into another tenant's scope.
+ local conf = { cache_key = { share_across_routes = true,
+ include_vars = { "http_x_a",
"http_x_b" } } }
+ local function key_for(a, b)
+ return key.build(conf, { var = { http_x_a = a, http_x_b = b }
}, "fp")
+ end
+
+ -- baseline: identical values share a key, and plain values stay
greppable
+ assert(key_for("acme", "us") == key_for("acme", "us"),
+ "identical scope values must produce the same key")
+ assert(key_for("acme", "us"):find("http_x_a=acme", 1, true),
+ "plain values must stay readable in the key")
+
+ -- two tenants whose raw "http_x_a=<a>:http_x_b=<b>" join both
collapse
+ -- to "http_x_a=x:http_x_b=yZ:http_x_b=" unless the values are
escaped
+ local tenant1 = key_for("x:http_x_b=yZ", "")
+ local tenant2 = key_for("x", "yZ:http_x_b=")
+ assert(tenant1 ~= tenant2,
+ "separator-injecting values must not collide into one
scope")
+ ngx.say("passed")
+ }
+ }
+--- response_body
+passed
