This is an automated email from the ASF dual-hosted git repository.
nic-6443 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/apisix.git
The following commit(s) were added to refs/heads/master by this push:
new 5dccf0e049 fix(oas-validator): release remote specs unused for
spec_url_ttl (#14061)
5dccf0e049 is described below
commit 5dccf0e04977a1a31d46eade07d4a5ca2fca4beb
Author: Nic <[email protected]>
AuthorDate: Mon Oct 26 11:15:52 2026 +0800
fix(oas-validator): release remote specs unused for spec_url_ttl (#14061)
---
apisix/core/lrucache.lua | 124 ++++++-
apisix/plugins/oas-validator.lua | 3 +
docs/en/latest/plugins/oas-validator.md | 2 +-
docs/zh/latest/plugins/oas-validator.md | 2 +-
t/core/lrucache.t | 249 +++++++++++++
t/plugin/oas-validator3.t | 601 ++++++++++++++++++++++++++++++++
6 files changed, 962 insertions(+), 19 deletions(-)
diff --git a/apisix/core/lrucache.lua b/apisix/core/lrucache.lua
index 6999685ab3..46f1d23c26 100644
--- a/apisix/core/lrucache.lua
+++ b/apisix/core/lrucache.lua
@@ -23,11 +23,14 @@ local lru_new = require("resty.lrucache").new
local resty_lock = require("resty.lock")
local log = require("apisix.core.log")
local pairs = pairs
+local ipairs = ipairs
local pcall = pcall
local unpack = unpack
local tostring = tostring
+local setmetatable = setmetatable
local ngx = ngx
local get_phase = ngx.get_phase
+local ngx_now = ngx.now
local timer_every = ngx.timer.every
local exiting = ngx.worker.exiting
@@ -47,7 +50,13 @@ local can_yield_phases = {
timer = true
}
-local stale_obj_pool = {}
+-- weak keys: an lrucache dropped by its owner (e.g. rebuilt with a new ttl)
+-- must not be kept alive by its pending refresh entries
+local stale_obj_pool = setmetatable({}, {__mode = "k"})
+
+-- lru_obj -> {inactive = seconds, access_times = {key -> last access time}}
+-- for the lrucaches created with `inactive`; weak keys for the same reason
+local inactive_lru_pool = setmetatable({}, {__mode = "k"})
local GLOBAL_ITEMS_COUNT = 1024
local GLOBAL_TTL = 60 * 60 -- 60 min
@@ -70,12 +79,22 @@ local function fetch_valid_cache(lru_obj, invalid_stale,
refresh_stale, item_ttl
end
if refresh_stale then
- stale_obj_pool[lru_obj][key] = {
- fn = create_obj_fun,
- args = {...},
- ver = version,
- ttl = item_ttl,
- }
+ -- update a pending refresh in place: refresh_stale_objs writes
+ -- back only while the pending entry is still the one it ran
+ local pending = stale_obj_pool[lru_obj][key]
+ if pending then
+ pending.fn = create_obj_fun
+ pending.args = {...}
+ pending.ver = version
+ pending.ttl = item_ttl
+ else
+ stale_obj_pool[lru_obj][key] = {
+ fn = create_obj_fun,
+ args = {...},
+ ver = version,
+ ttl = item_ttl,
+ }
+ end
return stale_obj
end
end
@@ -106,6 +125,17 @@ local function new_lru_fun(opts)
stale_obj_pool[lru_obj] = {}
+ -- `inactive`: drop an item not accessed for this many seconds, so an
+ -- expired item nobody asks for anymore is not kept until count eviction
+ local access_times
+ if opts and opts.inactive then
+ access_times = {}
+ inactive_lru_pool[lru_obj] = {
+ inactive = opts.inactive,
+ access_times = access_times,
+ }
+ end
+
return function (key, version, create_obj_fun, ...)
-- check negative cache first
if neg_lru_obj then
@@ -119,12 +149,18 @@ local function new_lru_fun(opts)
local cache_obj = fetch_valid_cache(lru_obj, invalid_stale,
refresh_stale,
item_ttl, key, version, create_obj_fun, ...)
if cache_obj then
+ if access_times then
+ access_times[key] = ngx_now()
+ end
return cache_obj.val
end
local obj, err = create_obj_fun(...)
if obj ~= nil then
lru_obj:set(key, {val = obj, ver = version}, item_ttl)
+ if access_times then
+ access_times[key] = ngx_now()
+ end
elseif neg_lru_obj then
-- cache the failure in negative cache
neg_lru_obj:set(key, {err = err, ver = version}, opts.neg_ttl)
@@ -136,6 +172,9 @@ local function new_lru_fun(opts)
local cache_obj = fetch_valid_cache(lru_obj, invalid_stale,
refresh_stale, item_ttl,
key, version, create_obj_fun, ...)
if cache_obj then
+ if access_times then
+ access_times[key] = ngx_now()
+ end
return cache_obj.val
end
@@ -157,12 +196,18 @@ local function new_lru_fun(opts)
if cache_obj then
lock:unlock()
log.info("unlock with key ", key_s)
+ if access_times then
+ access_times[key] = ngx_now()
+ end
return cache_obj.val
end
local obj, err = create_obj_fun(...)
if obj ~= nil then
lru_obj:set(key, {val = obj, ver = version}, item_ttl)
+ if access_times then
+ access_times[key] = ngx_now()
+ end
elseif neg_lru_obj then
-- cache the failure in negative cache
neg_lru_obj:set(key, {err = err, ver = version}, opts.neg_ttl)
@@ -218,34 +263,72 @@ end
local function refresh_stale_objs()
+ -- snapshot first: creating an object may yield, and the pool can change
+ -- meanwhile (new stale hits, purge_inactive_objs)
+ local pending = {}
for lru_obj, keys in pairs(stale_obj_pool) do
for key, new_obj in pairs(keys) do
- local obj, err = new_obj.fn(unpack(new_obj.args))
- if obj ~= nil then
- lru_obj:set(key, {val = obj, ver = new_obj.ver}, new_obj.ttl)
+ pending[#pending + 1] = {lru_obj, keys, key, new_obj}
+ end
+ end
+
+ for _, item in ipairs(pending) do
+ local lru_obj, keys, key, new_obj = item[1], item[2], item[3], item[4]
+ if keys[key] ~= new_obj then
+ goto continue
+ end
+
+ local ver, ttl = new_obj.ver, new_obj.ttl
+ local obj, err = new_obj.fn(unpack(new_obj.args))
+ if obj ~= nil then
+ -- the item may have been purged while refreshing; writing it back
+ -- then would resurrect it without an access record
+ if keys[key] == new_obj then
+ lru_obj:set(key, {val = obj, ver = ver}, ttl)
keys[key] = nil
log.info("successfully refresh stale obj for key ",
- tostring(key), " to ver ", new_obj.ver)
- else
- log.error("failed to refresh stale obj for key ", key, ": ",
err)
+ tostring(key), " to ver ", ver)
end
+ else
+ log.error("failed to refresh stale obj for key ", key, ": ", err)
end
+
+ ::continue::
end
end
-local function init_worker()
+local function purge_inactive_objs()
+ local now = ngx_now()
+ for lru_obj, inactive_lru in pairs(inactive_lru_pool) do
+ local access_times = inactive_lru.access_times
+ local inactive = inactive_lru.inactive
+ local pending = stale_obj_pool[lru_obj]
+ for key, access_time in pairs(access_times) do
+ if now - access_time > inactive then
+ lru_obj:delete(key)
+ access_times[key] = nil
+ if pending then
+ pending[key] = nil
+ end
+ end
+ end
+ end
+end
+
+
+local function run_every_second(name, fn)
local running = false
timer_every(1, function ()
if not exiting() then
if running then
- log.info("timer_refresh_stale is already running, skipping
this iteration")
+ log.info(name, " is already running, skipping this iteration")
return
end
running = true
- local ok, err = pcall(refresh_stale_objs)
+ local ok, err = pcall(fn)
if not ok then
- log.error("failed to run timer_refresh_stale: ", err)
+ log.error("failed to run ", name, ": ", err)
end
running = false
end
@@ -253,6 +336,13 @@ local function init_worker()
end
+local function init_worker()
+ run_every_second("timer_refresh_stale", refresh_stale_objs)
+ -- a separate timer, so a slow refresh does not hold up purging
+ run_every_second("timer_purge_inactive", purge_inactive_objs)
+end
+
+
local _M = {
version = 0.1,
init_worker = init_worker,
diff --git a/apisix/plugins/oas-validator.lua b/apisix/plugins/oas-validator.lua
index 602585d5d7..dd3ec5ecd9 100644
--- a/apisix/plugins/oas-validator.lua
+++ b/apisix/plugins/oas-validator.lua
@@ -141,6 +141,9 @@ local function get_spec_url_lrucache()
invalid_stale = true,
refresh_stale = true,
serial_creating = true,
+ -- release a spec no resource has asked for within a ttl, instead
+ -- of keeping it until 512 other specs push it out
+ inactive = ttl,
})
spec_url_lrucache_ttl = ttl
end
diff --git a/docs/en/latest/plugins/oas-validator.md
b/docs/en/latest/plugins/oas-validator.md
index d4292508e6..9cfe9dd5ac 100644
--- a/docs/en/latest/plugins/oas-validator.md
+++ b/docs/en/latest/plugins/oas-validator.md
@@ -124,7 +124,7 @@ You should receive a `400` response with a validation error
message.
### Validate Requests with a Remote Specification URL
-The following example demonstrates how to fetch the OpenAPI specification from
a remote URL. The spec is fetched once and cached for the duration specified by
`spec_url_ttl` in the plugin metadata.
+The following example demonstrates how to fetch the OpenAPI specification from
a remote URL. The spec is fetched once and cached for the duration specified by
`spec_url_ttl` in the plugin metadata. Resources that declare the same
`spec_url` with the same `ssl_verify` and `spec_url_request_headers` share one
cached copy. A cached spec that no request has used for `spec_url_ttl` seconds
is released, so a spec that is no longer in use, for example after its Route
changes `spec_url` or is de [...]
Configure the plugin metadata to set the cache TTL for the remote spec:
diff --git a/docs/zh/latest/plugins/oas-validator.md
b/docs/zh/latest/plugins/oas-validator.md
index 0dc3e95850..4a05f39853 100644
--- a/docs/zh/latest/plugins/oas-validator.md
+++ b/docs/zh/latest/plugins/oas-validator.md
@@ -123,7 +123,7 @@ curl -i "http://127.0.0.1:9080/api/v3/pet"; -X POST \
### 使用远程规范 URL 校验请求
-以下示例演示如何从远程 URL 获取 OpenAPI 规范。规范在首次获取后会被缓存，缓存时长由插件元数据的 `spec_url_ttl` 参数决定。
+以下示例演示如何从远程 URL 获取 OpenAPI 规范。规范在首次获取后会被缓存，缓存时长由插件元数据的 `spec_url_ttl`
参数决定。声明相同 `spec_url`、且 `ssl_verify` 与 `spec_url_request_headers`
也相同的资源共享同一份缓存。若某个缓存的规范在 `spec_url_ttl` 秒内没有被任何请求使用，它会被释放，因此不再使用的规范（例如路由修改了
`spec_url` 或被删除之后）不会一直占用内存。
配置插件元数据以设置远程规范的缓存时间：
diff --git a/t/core/lrucache.t b/t/core/lrucache.t
index 905c20fd0f..ab718de7dc 100644
--- a/t/core/lrucache.t
+++ b/t/core/lrucache.t
@@ -235,3 +235,252 @@ fetch obj: true
create obj 2 time
fetch obj: true
fetch obj: true
+
+
+
+=== TEST 7: a dropped lrucache releases its cached objects
+--- config
+ location /t {
+ content_by_lua_block {
+ local core = require("apisix.core")
+
+ local lru_get = core.lrucache.new({
+ ttl = 10, count = 256, invalid_stale = true, refresh_stale =
true,
+ })
+ local probe = setmetatable({}, {__mode = "v"})
+ lru_get("key", "ver", function ()
+ local obj = {}
+ probe[1] = obj
+ return obj
+ end)
+
+ lru_get = nil
+ collectgarbage()
+ collectgarbage()
+ ngx.say("released: ", probe[1] == nil)
+ }
+ }
+--- request
+GET /t
+--- response_body
+released: true
+
+
+
+=== TEST 8: inactive: an item not accessed within `inactive` is purged
+--- config
+ location /t {
+ content_by_lua_block {
+ local core = require("apisix.core")
+
+ local lru_get = core.lrucache.new({
+ ttl = 1, count = 256, invalid_stale = true, refresh_stale =
true,
+ inactive = 1,
+ })
+ local probe = setmetatable({}, {__mode = "v"})
+ local created = 0
+ local function create_obj()
+ created = created + 1
+ local obj = {}
+ probe[created] = obj
+ return obj
+ end
+
+ lru_get("key", "ver", create_obj)
+ -- purged once the 1s purge timer runs after `inactive`
+ local deadline = ngx.now() + 5
+ repeat
+ ngx.sleep(0.2)
+ collectgarbage()
+ collectgarbage()
+ until probe[1] == nil or ngx.now() > deadline
+ ngx.say("released: ", probe[1] == nil)
+
+ lru_get("key", "ver", create_obj)
+ ngx.say("created: ", created)
+ }
+ }
+--- request
+GET /t
+--- response_body
+released: true
+created: 2
+--- timeout: 8
+
+
+
+=== TEST 9: inactive: an item still in use is kept while its refresh keeps
failing
+--- config
+ location /t {
+ content_by_lua_block {
+ local core = require("apisix.core")
+
+ local lru_get = core.lrucache.new({
+ ttl = 0.5, count = 256, invalid_stale = true, refresh_stale =
true,
+ inactive = 1,
+ })
+ local created = 0
+ local function create_obj()
+ created = created + 1
+ if created > 1 then
+ return nil, "refresh failure"
+ end
+ return {gen = created}
+ end
+
+ local obj = lru_get("key", "ver", create_obj)
+ local same = true
+ for _ = 1, 10 do
+ ngx.sleep(0.3)
+ if lru_get("key", "ver", create_obj) ~= obj then
+ same = false
+ end
+ end
+ ngx.say("served the first object: ", same)
+ ngx.say("refreshed: ", created > 1)
+ }
+ }
+--- request
+GET /t
+--- response_body
+served the first object: true
+refreshed: true
+--- error_log
+refresh failure
+--- timeout: 6
+
+
+
+=== TEST 10: inactive: purging an item drops its pending refresh
+--- config
+ location /t {
+ content_by_lua_block {
+ local core = require("apisix.core")
+
+ local lru_get = core.lrucache.new({
+ ttl = 0.2, count = 256, invalid_stale = true, refresh_stale =
true,
+ inactive = 1,
+ })
+ local created = 0
+ local function create_obj()
+ created = created + 1
+ if created > 1 then
+ return nil, "refresh failure"
+ end
+ return {}
+ end
+
+ lru_get("key", "ver", create_obj)
+ ngx.sleep(0.3)
+ -- served stale, queues a refresh that keeps failing
+ lru_get("key", "ver", create_obj)
+ -- the purge itself is not observable here: leave it a wide margin
+ ngx.sleep(3.5)
+ local after_purge = created
+ ngx.sleep(2.5)
+ ngx.say("refresh retried after the purge: ", created > after_purge)
+ }
+ }
+--- request
+GET /t
+--- response_body
+refresh retried after the purge: false
+--- error_log
+refresh failure
+--- timeout: 10
+
+
+
+=== TEST 11: inactive: a refresh in flight does not bring a purged item back
+--- config
+ location /t {
+ content_by_lua_block {
+ local core = require("apisix.core")
+
+ local lru_get = core.lrucache.new({
+ ttl = 0.5, count = 256, invalid_stale = true, refresh_stale =
true,
+ inactive = 1,
+ })
+ local created = 0
+ local function create_obj()
+ created = created + 1
+ local gen = created
+ if ngx.get_phase() == "timer" then
+ -- the item is purged while this refresh is running
+ ngx.sleep(4)
+ end
+ return {gen = gen}
+ end
+
+ lru_get("key", "ver", create_obj)
+ ngx.sleep(0.6)
+ -- served stale, queues the slow refresh
+ lru_get("key", "ver", create_obj)
+ ngx.sleep(5.5)
+
+ ngx.say("gen: ", lru_get("key", "ver", create_obj).gen)
+ }
+ }
+--- request
+GET /t
+--- response_body
+gen: 3
+--- timeout: 10
+
+
+
+=== TEST 12: inactive: a dropped lrucache releases its cached objects
+--- config
+ location /t {
+ content_by_lua_block {
+ local core = require("apisix.core")
+
+ local lru_get = core.lrucache.new({
+ ttl = 10, count = 256, invalid_stale = true, refresh_stale =
true,
+ inactive = 10,
+ })
+ local probe = setmetatable({}, {__mode = "v"})
+ lru_get("key", "ver", function ()
+ local obj = {}
+ probe[1] = obj
+ return obj
+ end)
+
+ lru_get = nil
+ collectgarbage()
+ collectgarbage()
+ ngx.say("released: ", probe[1] == nil)
+ }
+ }
+--- request
+GET /t
+--- response_body
+released: true
+
+
+
+=== TEST 13: an lrucache without inactive keeps idle items
+--- config
+ location /t {
+ content_by_lua_block {
+ local core = require("apisix.core")
+
+ local lru_get = core.lrucache.new({ttl = 0.5, count = 256})
+ local created = 0
+ local function create_obj()
+ created = created + 1
+ return {}
+ end
+
+ local obj = lru_get("key", "ver", create_obj)
+ ngx.sleep(2.5)
+ ngx.say("same object: ", lru_get("key", "ver", create_obj) == obj)
+ ngx.say("created: ", created)
+ }
+ }
+--- request
+GET /t
+--- response_body
+same object: true
+created: 1
+--- timeout: 5
diff --git a/t/plugin/oas-validator3.t b/t/plugin/oas-validator3.t
index cdd2b90104..8f457e4e1c 100644
--- a/t/plugin/oas-validator3.t
+++ b/t/plugin/oas-validator3.t
@@ -25,8 +25,22 @@ add_block_preprocessor(sub {
my ($block) = @_;
my $http_config = $block->http_config // <<_EOC_;
+ lua_shared_dict oas_spec_fetches 1m;
+
server {
listen 1979;
+ location /counted/ {
+ content_by_lua_block {
+ ngx.shared.oas_spec_fetches:incr(ngx.var.uri, 1, 0)
+ if ngx.var.arg_delay then
+ ngx.sleep(tonumber(ngx.var.arg_delay))
+ end
+ local file = io.open("t/spec/spec.json", "r")
+ local content = file:read("*a")
+ file:close()
+ ngx.print(content)
+ }
+ }
location /spec.json {
content_by_lua_block {
local file = io.open("t/spec/spec.json", "r")
@@ -562,3 +576,590 @@ status: 200
}
--- response_body
passed
+
+
+
+=== TEST 24: a spec_url no resource uses any more is released after
spec_url_ttl
+--- config
+ location /t {
+ content_by_lua_block {
+ local t = require("lib.test_admin")
+ local http = require("resty.http")
+ local fetches = ngx.shared.oas_spec_fetches
+
+ -- track compiled specs that are still reachable
+ local ov = require("resty.openapi_validator")
+ local orig_compile = ov.compile
+ local compiled = setmetatable({}, {__mode = "v"})
+ local n_compiled = 0
+ ov.compile = function (...)
+ local v, err = orig_compile(...)
+ if v then
+ n_compiled = n_compiled + 1
+ compiled[n_compiled] = v
+ end
+ return v, err
+ end
+
+ local function live()
+ collectgarbage()
+ collectgarbage()
+ local n = 0
+ for _ in pairs(compiled) do
+ n = n + 1
+ end
+ return n
+ end
+
+ local function put_route(id, spec_url)
+ local code, body = t.test('/apisix/admin/routes/' .. id,
ngx.HTTP_PUT, {
+ uri = "/api/v3/pet",
+ host = "r" .. id .. ".test",
+ plugins = {["oas-validator"] = {spec_url = spec_url}},
+ upstream = {type = "roundrobin", nodes =
{["127.0.0.1:1970"] = 1}},
+ })
+ assert(code < 300, body)
+ ngx.sleep(0.5)
+ end
+
+ local function hit(id)
+ local res, err = http.new():request_uri(
+ "http://127.0.0.1:"; .. ngx.var.server_port ..
"/api/v3/pet", {
+ method = "POST",
+ body = '{"id": 10, "name": "doggie", "photoUrls":
["string"]}',
+ headers = {
+ ["Content-Type"] = "application/json",
+ ["Host"] = "r" .. id .. ".test",
+ },
+ })
+ assert(res and res.status == 200, err or (res and res.status))
+ end
+
+ local function set_ttl(ttl)
+ local code, body =
t.test('/apisix/admin/plugin_metadata/oas-validator',
+ ngx.HTTP_PUT, {spec_url_ttl = ttl})
+ assert(code < 300, body)
+ end
+
+ set_ttl(3)
+ put_route(1, "http://127.0.0.1:1979/counted/a.json";)
+ hit(1)
+ put_route(1, "http://127.0.0.1:1979/counted/b.json";)
+ put_route(2, "http://127.0.0.1:1979/counted/c.json";)
+ hit(2)
+ t.test('/apisix/admin/routes/2', ngx.HTTP_DELETE)
+ ngx.say("live before spec_url_ttl: ", live())
+
+ -- neither a.json nor c.json is asked for again
+ -- released once the 1s purge timer runs after spec_url_ttl idle
+ local function live_after_idle()
+ local deadline = ngx.now() + 10
+ while live() > 0 and ngx.now() < deadline do
+ ngx.sleep(0.2)
+ end
+ return live()
+ end
+ ngx.say("live after spec_url_ttl: ", live_after_idle())
+
+ t.test('/apisix/admin/plugin_metadata/oas-validator',
ngx.HTTP_DELETE)
+ for id = 1, 2 do
+ t.test('/apisix/admin/routes/' .. id, ngx.HTTP_DELETE)
+ end
+ }
+ }
+--- response_body
+live before spec_url_ttl: 2
+live after spec_url_ttl: 0
+--- timeout: 15
+
+
+
+=== TEST 25: routes declaring the same spec_url share one compiled spec
+--- config
+ location /t {
+ content_by_lua_block {
+ local t = require("lib.test_admin")
+ local http = require("resty.http")
+ local fetches = ngx.shared.oas_spec_fetches
+
+ local function put_route(id, spec_url)
+ local code, body = t.test('/apisix/admin/routes/' .. id,
ngx.HTTP_PUT, {
+ uri = "/api/v3/pet",
+ host = "r" .. id .. ".test",
+ plugins = {["oas-validator"] = {spec_url = spec_url}},
+ upstream = {type = "roundrobin", nodes =
{["127.0.0.1:1970"] = 1}},
+ })
+ assert(code < 300, body)
+ ngx.sleep(0.5)
+ end
+
+ local function hit(id)
+ local res, err = http.new():request_uri(
+ "http://127.0.0.1:"; .. ngx.var.server_port ..
"/api/v3/pet", {
+ method = "POST",
+ body = '{"id": 10, "name": "doggie", "photoUrls":
["string"]}',
+ headers = {
+ ["Content-Type"] = "application/json",
+ ["Host"] = "r" .. id .. ".test",
+ },
+ })
+ assert(res and res.status == 200, err or (res and res.status))
+ end
+
+ local function set_ttl(ttl)
+ local code, body =
t.test('/apisix/admin/plugin_metadata/oas-validator',
+ ngx.HTTP_PUT, {spec_url_ttl = ttl})
+ assert(code < 300, body)
+ end
+
+ put_route(1, "http://127.0.0.1:1979/counted/c.json";)
+ put_route(2, "http://127.0.0.1:1979/counted/c.json";)
+ hit(1)
+ hit(2)
+ hit(1)
+ hit(2)
+
+ ngx.say("c: ", fetches:get("/counted/c.json"))
+
+ t.test('/apisix/admin/plugin_metadata/oas-validator',
ngx.HTTP_DELETE)
+ for id = 1, 2 do
+ t.test('/apisix/admin/routes/' .. id, ngx.HTTP_DELETE)
+ end
+ }
+ }
+--- response_body
+c: 1
+
+
+
+=== TEST 26: changing spec_url_ttl releases the specs held by the previous
cache
+--- config
+ location /t {
+ content_by_lua_block {
+ local t = require("lib.test_admin")
+ local http = require("resty.http")
+ local fetches = ngx.shared.oas_spec_fetches
+
+ -- track compiled specs that are still reachable
+ local ov = require("resty.openapi_validator")
+ local orig_compile = ov.compile
+ local compiled = setmetatable({}, {__mode = "v"})
+ local n_compiled = 0
+ ov.compile = function (...)
+ local v, err = orig_compile(...)
+ if v then
+ n_compiled = n_compiled + 1
+ compiled[n_compiled] = v
+ end
+ return v, err
+ end
+
+ local function live()
+ collectgarbage()
+ collectgarbage()
+ local n = 0
+ for _ in pairs(compiled) do
+ n = n + 1
+ end
+ return n
+ end
+
+ local function put_route(id, spec_url)
+ local code, body = t.test('/apisix/admin/routes/' .. id,
ngx.HTTP_PUT, {
+ uri = "/api/v3/pet",
+ host = "r" .. id .. ".test",
+ plugins = {["oas-validator"] = {spec_url = spec_url}},
+ upstream = {type = "roundrobin", nodes =
{["127.0.0.1:1970"] = 1}},
+ })
+ assert(code < 300, body)
+ ngx.sleep(0.5)
+ end
+
+ local function hit(id)
+ local res, err = http.new():request_uri(
+ "http://127.0.0.1:"; .. ngx.var.server_port ..
"/api/v3/pet", {
+ method = "POST",
+ body = '{"id": 10, "name": "doggie", "photoUrls":
["string"]}',
+ headers = {
+ ["Content-Type"] = "application/json",
+ ["Host"] = "r" .. id .. ".test",
+ },
+ })
+ assert(res and res.status == 200, err or (res and res.status))
+ end
+
+ local function set_ttl(ttl)
+ local code, body =
t.test('/apisix/admin/plugin_metadata/oas-validator',
+ ngx.HTTP_PUT, {spec_url_ttl = ttl})
+ assert(code < 300, body)
+ end
+
+ put_route(1, "http://127.0.0.1:1979/counted/d.json";)
+ hit(1)
+
+ set_ttl(7200)
+ ngx.sleep(0.5)
+ -- the next request replaces the cache with one for the new ttl
+ put_route(2, "http://127.0.0.1:1979/counted/e.json";)
+ hit(2)
+ ngx.say("live: ", live())
+
+ t.test('/apisix/admin/plugin_metadata/oas-validator',
ngx.HTTP_DELETE)
+ for id = 1, 2 do
+ t.test('/apisix/admin/routes/' .. id, ngx.HTTP_DELETE)
+ end
+ }
+ }
+--- response_body
+live: 1
+
+
+
+=== TEST 27: idle resources do not pin old generations of a shared spec_url
+--- config
+ location /t {
+ content_by_lua_block {
+ local t = require("lib.test_admin")
+ local http = require("resty.http")
+ local fetches = ngx.shared.oas_spec_fetches
+
+ -- track compiled specs that are still reachable
+ local ov = require("resty.openapi_validator")
+ local orig_compile = ov.compile
+ local compiled = setmetatable({}, {__mode = "v"})
+ local n_compiled = 0
+ ov.compile = function (...)
+ local v, err = orig_compile(...)
+ if v then
+ n_compiled = n_compiled + 1
+ compiled[n_compiled] = v
+ end
+ return v, err
+ end
+
+ local function live()
+ collectgarbage()
+ collectgarbage()
+ local n = 0
+ for _ in pairs(compiled) do
+ n = n + 1
+ end
+ return n
+ end
+
+ local function put_route(id, spec_url)
+ local code, body = t.test('/apisix/admin/routes/' .. id,
ngx.HTTP_PUT, {
+ uri = "/api/v3/pet",
+ host = "r" .. id .. ".test",
+ plugins = {["oas-validator"] = {spec_url = spec_url}},
+ upstream = {type = "roundrobin", nodes =
{["127.0.0.1:1970"] = 1}},
+ })
+ assert(code < 300, body)
+ ngx.sleep(0.5)
+ end
+
+ local function hit(id)
+ local res, err = http.new():request_uri(
+ "http://127.0.0.1:"; .. ngx.var.server_port ..
"/api/v3/pet", {
+ method = "POST",
+ body = '{"id": 10, "name": "doggie", "photoUrls":
["string"]}',
+ headers = {
+ ["Content-Type"] = "application/json",
+ ["Host"] = "r" .. id .. ".test",
+ },
+ })
+ assert(res and res.status == 200, err or (res and res.status))
+ end
+
+ local function set_ttl(ttl)
+ local code, body =
t.test('/apisix/admin/plugin_metadata/oas-validator',
+ ngx.HTTP_PUT, {spec_url_ttl = ttl})
+ assert(code < 300, body)
+ end
+
+ set_ttl(2)
+ local url = "http://127.0.0.1:1979/counted/one.json";
+ for id = 0, 3 do
+ put_route(id, url)
+ end
+
+ -- route 0 keeps using the spec, so it is refreshed every ttl and
+ -- never idle; routes 1..3 each use one generation and then stay
idle
+ local function hit_0_until_generation(gen)
+ local deadline = ngx.now() + 10
+ while n_compiled < gen do
+ assert(ngx.now() < deadline, "no refresh")
+ ngx.sleep(0.5)
+ hit(0)
+ end
+ end
+
+ hit(0)
+ hit(1)
+ for gen = 2, 4 do
+ hit_0_until_generation(gen)
+ if gen <= 3 then
+ hit(gen)
+ end
+ end
+
+ ngx.say("generations: ", n_compiled)
+ ngx.say("live: ", live())
+
+ -- nobody asks for it any more
+ -- released once the 1s purge timer runs after spec_url_ttl idle
+ local function live_after_idle()
+ local deadline = ngx.now() + 10
+ while live() > 0 and ngx.now() < deadline do
+ ngx.sleep(0.2)
+ end
+ return live()
+ end
+ ngx.say("live after spec_url_ttl: ", live_after_idle())
+
+ t.test('/apisix/admin/plugin_metadata/oas-validator',
ngx.HTTP_DELETE)
+ for id = 0, 3 do
+ t.test('/apisix/admin/routes/' .. id, ngx.HTTP_DELETE)
+ end
+ }
+ }
+--- response_body
+generations: 4
+live: 1
+live after spec_url_ttl: 0
+--- timeout: 30
+
+
+
+=== TEST 28: resources hitting one cold spec_url concurrently fetch it once
+--- config
+ location /t {
+ content_by_lua_block {
+ local t = require("lib.test_admin")
+ local http = require("resty.http")
+ local fetches = ngx.shared.oas_spec_fetches
+
+ local function put_route(id, spec_url)
+ local code, body = t.test('/apisix/admin/routes/' .. id,
ngx.HTTP_PUT, {
+ uri = "/api/v3/pet",
+ host = "r" .. id .. ".test",
+ plugins = {["oas-validator"] = {spec_url = spec_url}},
+ upstream = {type = "roundrobin", nodes =
{["127.0.0.1:1970"] = 1}},
+ })
+ assert(code < 300, body)
+ ngx.sleep(0.5)
+ end
+
+ local function hit(id)
+ local res, err = http.new():request_uri(
+ "http://127.0.0.1:"; .. ngx.var.server_port ..
"/api/v3/pet", {
+ method = "POST",
+ body = '{"id": 10, "name": "doggie", "photoUrls":
["string"]}',
+ headers = {
+ ["Content-Type"] = "application/json",
+ ["Host"] = "r" .. id .. ".test",
+ },
+ })
+ assert(res and res.status == 200, err or (res and res.status))
+ end
+
+ local function set_ttl(ttl)
+ local code, body =
t.test('/apisix/admin/plugin_metadata/oas-validator',
+ ngx.HTTP_PUT, {spec_url_ttl = ttl})
+ assert(code < 300, body)
+ end
+
+ local spec_url = "http://127.0.0.1:1979/counted/g.json?delay=0.3";
+ put_route(1, spec_url)
+ put_route(2, spec_url)
+ put_route(3, spec_url)
+
+ local threads = {}
+ for id = 1, 3 do
+ threads[id] = ngx.thread.spawn(hit, id)
+ end
+ for id = 1, 3 do
+ assert(ngx.thread.wait(threads[id]))
+ end
+
+ ngx.say("g: ", fetches:get("/counted/g.json"))
+
+ t.test('/apisix/admin/plugin_metadata/oas-validator',
ngx.HTTP_DELETE)
+ for id = 1, 3 do
+ t.test('/apisix/admin/routes/' .. id, ngx.HTTP_DELETE)
+ end
+ }
+ }
+--- response_body
+g: 1
+
+
+
+=== TEST 29: the 512-spec limit counts URLs, not the resources declaring them
+--- config
+ location /t {
+ content_by_lua_block {
+ local t = require("lib.test_admin")
+ local http = require("resty.http")
+ local fetches = ngx.shared.oas_spec_fetches
+
+ -- track compiled specs that are still reachable
+ local ov = require("resty.openapi_validator")
+ local orig_compile = ov.compile
+ local compiled = setmetatable({}, {__mode = "v"})
+ local n_compiled = 0
+ ov.compile = function (...)
+ local v, err = orig_compile(...)
+ if v then
+ n_compiled = n_compiled + 1
+ compiled[n_compiled] = v
+ end
+ return v, err
+ end
+
+ local function live()
+ collectgarbage()
+ collectgarbage()
+ local n = 0
+ for _ in pairs(compiled) do
+ n = n + 1
+ end
+ return n
+ end
+
+ local function put_route(id, spec_url)
+ local code, body = t.test('/apisix/admin/routes/' .. id,
ngx.HTTP_PUT, {
+ uri = "/api/v3/pet",
+ host = "r" .. id .. ".test",
+ plugins = {["oas-validator"] = {spec_url = spec_url}},
+ upstream = {type = "roundrobin", nodes =
{["127.0.0.1:1970"] = 1}},
+ })
+ assert(code < 300, body)
+ end
+
+ local function hit(id)
+ local res, err = http.new():request_uri(
+ "http://127.0.0.1:"; .. ngx.var.server_port ..
"/api/v3/pet", {
+ method = "POST",
+ body = '{"id": 10, "name": "doggie", "photoUrls":
["string"]}',
+ headers = {
+ ["Content-Type"] = "application/json",
+ ["Host"] = "r" .. id .. ".test",
+ },
+ })
+ assert(res and res.status == 200, err or (res and res.status))
+ end
+
+ local function set_ttl(ttl)
+ local code, body =
t.test('/apisix/admin/plugin_metadata/oas-validator',
+ ngx.HTTP_PUT, {spec_url_ttl = ttl})
+ assert(code < 300, body)
+ end
+
+ -- 600 routes over 51 URLs: every spec stays cached
+ local N, DISTINCT = 600, 50
+ local function url_of(id)
+ if id <= DISTINCT then
+ return "http://127.0.0.1:1979/counted/u"; .. id .. ".json"
+ end
+ return "http://127.0.0.1:1979/counted/shared.json";
+ end
+ for id = 1, N do
+ put_route(id, url_of(id))
+ end
+ ngx.sleep(3)
+
+ local function total()
+ local s = 0
+ for _, k in ipairs(fetches:get_keys(0)) do
+ s = s + fetches:get(k)
+ end
+ return s
+ end
+
+ for id = 1, N do
+ hit(id)
+ end
+ local pass1, compiled1 = total(), n_compiled
+ collectgarbage()
+ for id = 1, N do
+ hit(id)
+ end
+ ngx.say("pass1 fetches: ", pass1, ", compiles: ", compiled1)
+ ngx.say("pass2 fetches: ", total() - pass1, ", compiles: ",
n_compiled - compiled1)
+ ngx.say("live: ", live())
+
+ t.test('/apisix/admin/plugin_metadata/oas-validator',
ngx.HTTP_DELETE)
+ for id = 1, N do
+ t.test('/apisix/admin/routes/' .. id, ngx.HTTP_DELETE)
+ end
+ }
+ }
+--- response_body
+pass1 fetches: 51, compiles: 51
+pass2 fetches: 0, compiles: 0
+live: 51
+--- timeout: 300
+
+
+
+=== TEST 30: a spec shared by routes is refreshed once it is older than
spec_url_ttl
+--- config
+ location /t {
+ content_by_lua_block {
+ local t = require("lib.test_admin")
+ local http = require("resty.http")
+ local fetches = ngx.shared.oas_spec_fetches
+
+ local function put_route(id, spec_url)
+ local code, body = t.test('/apisix/admin/routes/' .. id,
ngx.HTTP_PUT, {
+ uri = "/api/v3/pet",
+ host = "r" .. id .. ".test",
+ plugins = {["oas-validator"] = {spec_url = spec_url}},
+ upstream = {type = "roundrobin", nodes =
{["127.0.0.1:1970"] = 1}},
+ })
+ assert(code < 300, body)
+ ngx.sleep(0.5)
+ end
+
+ local function hit(id)
+ local res, err = http.new():request_uri(
+ "http://127.0.0.1:"; .. ngx.var.server_port ..
"/api/v3/pet", {
+ method = "POST",
+ body = '{"id": 10, "name": "doggie", "photoUrls":
["string"]}',
+ headers = {
+ ["Content-Type"] = "application/json",
+ ["Host"] = "r" .. id .. ".test",
+ },
+ })
+ assert(res and res.status == 200, err or (res and res.status))
+ end
+
+ local function set_ttl(ttl)
+ local code, body =
t.test('/apisix/admin/plugin_metadata/oas-validator',
+ ngx.HTTP_PUT, {spec_url_ttl = ttl})
+ assert(code < 300, body)
+ end
+
+ set_ttl(2)
+ local url = "http://127.0.0.1:1979/counted/fresh.json";
+ put_route(1, url)
+ put_route(2, url)
+
+ hit(1) -- fetch at t0
+ ngx.sleep(1.3)
+ hit(2) -- reuses the t0 spec
+ ngx.sleep(1.0) -- the spec is now older than the ttl
+ hit(2) -- served stale, queues a refresh
+ ngx.sleep(1.5)
+ ngx.say("fetches: ", fetches:get("/counted/fresh.json"))
+
+ t.test('/apisix/admin/plugin_metadata/oas-validator',
ngx.HTTP_DELETE)
+ for id = 1, 2 do
+ t.test('/apisix/admin/routes/' .. id, ngx.HTTP_DELETE)
+ end
+ }
+ }
+--- response_body
+fetches: 2
+--- timeout: 15
