This is an automated email from the ASF dual-hosted git repository. membphis pushed a commit to branch feat/router-min-rebuild-interval in repository https://gitbox.apache.org/repos/asf/apisix.git
commit f5dd1dde07b86449503f3a3f8574e57160c7b0e9 Author: Rain <[email protected]> AuthorDate: Thu Mar 5 13:32:48 2026 +0800 feat: add router_rebuild_min_interval to reduce CPU spikes from frequent route changes When routes change frequently (e.g., 20-50 changes/sec), the router is rebuilt on every change which causes CPU spikes under high-frequency updates due to full radixtree recompilation with all routes. This adds a configurable minimum interval between router rebuilds. When set to a positive value (e.g., 1 or 5 seconds), rebuilds are throttled so the router is not recompiled more frequently than the configured interval, significantly reducing CPU usage during bulk route updates. Default value is 0 (rebuild immediately on every change), preserving backward compatibility with existing behavior. --- apisix/cli/config.lua | 3 +- apisix/http/router/radixtree_host_uri.lua | 14 + apisix/http/router/radixtree_uri.lua | 14 + .../http/router/radixtree_uri_with_parameter.lua | 14 + apisix/router.lua | 12 + apisix/stream/router/ip_port.lua | 15 + conf/config.yaml.example | 6 + t/router/router-rebuild-min-interval.t | 363 +++++++++++++++++++++ 8 files changed, 440 insertions(+), 1 deletion(-) diff --git a/apisix/cli/config.lua b/apisix/cli/config.lua index 956eef30c..24b85d3d6 100644 --- a/apisix/cli/config.lua +++ b/apisix/cli/config.lua @@ -49,7 +49,8 @@ local _M = { normalize_uri_like_servlet = false, router = { http = "radixtree_host_uri", - ssl = "radixtree_sni" + ssl = "radixtree_sni", + router_rebuild_min_interval = 0, }, proxy_mode = "http", resolver_timeout = 5, diff --git a/apisix/http/router/radixtree_host_uri.lua b/apisix/http/router/radixtree_host_uri.lua index 680a04fbe..46b475fcf 100644 --- a/apisix/http/router/radixtree_host_uri.lua +++ b/apisix/http/router/radixtree_host_uri.lua @@ -20,6 +20,7 @@ local core = require("apisix.core") local event = require("apisix.core.event") local get_services = require("apisix.http.service").services local service_fetch = require("apisix.http.service").get +local ngx_now = ngx.now local ipairs = ipairs local type = type local tab_insert = table.insert @@ -27,6 +28,7 @@ local loadstring = loadstring local pairs = pairs local cached_router_version local cached_service_version +local last_router_rebuild_time local host_router local only_uri_router @@ -148,11 +150,23 @@ function _M.match(api_ctx) if not cached_router_version or cached_router_version ~= user_routes.conf_version or not cached_service_version or cached_service_version ~= service_version then + local min_interval = _M.router_rebuild_min_interval or 0 + if min_interval > 0 and last_router_rebuild_time then + local elapsed = ngx_now() - last_router_rebuild_time + if elapsed < min_interval then + core.log.info("skip router rebuild, elapsed: ", elapsed, + "s, min_interval: ", min_interval, "s") + goto MATCH + end + end + create_radixtree_router(user_routes.values) cached_router_version = user_routes.conf_version cached_service_version = service_version + last_router_rebuild_time = ngx_now() end + ::MATCH:: return _M.matching(api_ctx) end diff --git a/apisix/http/router/radixtree_uri.lua b/apisix/http/router/radixtree_uri.lua index 7c1b5c0c1..a3520053f 100644 --- a/apisix/http/router/radixtree_uri.lua +++ b/apisix/http/router/radixtree_uri.lua @@ -18,8 +18,10 @@ local require = require local core = require("apisix.core") local base_router = require("apisix.http.route") local get_services = require("apisix.http.service").services +local ngx_now = ngx.now local cached_router_version local cached_service_version +local last_router_rebuild_time local _M = {version = 0.2} @@ -33,12 +35,24 @@ function _M.match(api_ctx) if not cached_router_version or cached_router_version ~= user_routes.conf_version or not cached_service_version or cached_service_version ~= service_version then + local min_interval = _M.router_rebuild_min_interval or 0 + if min_interval > 0 and last_router_rebuild_time then + local elapsed = ngx_now() - last_router_rebuild_time + if elapsed < min_interval then + core.log.info("skip router rebuild, elapsed: ", elapsed, + "s, min_interval: ", min_interval, "s") + goto MATCH + end + end + uri_router = base_router.create_radixtree_uri_router(user_routes.values, uri_routes, false) cached_router_version = user_routes.conf_version cached_service_version = service_version + last_router_rebuild_time = ngx_now() end + ::MATCH:: if not uri_router then core.log.error("failed to fetch valid `uri` router: ") return true diff --git a/apisix/http/router/radixtree_uri_with_parameter.lua b/apisix/http/router/radixtree_uri_with_parameter.lua index 3f10f4fca..bafab2c22 100644 --- a/apisix/http/router/radixtree_uri_with_parameter.lua +++ b/apisix/http/router/radixtree_uri_with_parameter.lua @@ -18,8 +18,10 @@ local require = require local core = require("apisix.core") local base_router = require("apisix.http.route") local get_services = require("apisix.http.service").services +local ngx_now = ngx.now local cached_router_version local cached_service_version +local last_router_rebuild_time local _M = {} @@ -33,12 +35,24 @@ function _M.match(api_ctx) if not cached_router_version or cached_router_version ~= user_routes.conf_version or not cached_service_version or cached_service_version ~= service_version then + local min_interval = _M.router_rebuild_min_interval or 0 + if min_interval > 0 and last_router_rebuild_time then + local elapsed = ngx_now() - last_router_rebuild_time + if elapsed < min_interval then + core.log.info("skip router rebuild, elapsed: ", elapsed, + "s, min_interval: ", min_interval, "s") + goto MATCH + end + end + uri_router = base_router.create_radixtree_uri_router(user_routes.values, uri_routes, true) cached_router_version = user_routes.conf_version cached_service_version = service_version + last_router_rebuild_time = ngx_now() end + ::MATCH:: if not uri_router then core.log.error("failed to fetch valid `uri_with_parameter` router: ") return true diff --git a/apisix/router.lua b/apisix/router.lua index 6cdd07175..64fff5876 100644 --- a/apisix/router.lua +++ b/apisix/router.lua @@ -23,6 +23,7 @@ local str_lower = string.lower local ipairs = ipairs local _M = {version = 0.3} +local router_rebuild_min_interval = 0 local function filter(route) @@ -76,11 +77,14 @@ function _M.http_init_worker() if conf and conf.apisix and conf.apisix.router then router_http_name = conf.apisix.router.http or router_http_name router_ssl_name = conf.apisix.router.ssl or router_ssl_name + router_rebuild_min_interval = conf.apisix.router.router_rebuild_min_interval + or router_rebuild_min_interval end local router_http = require("apisix.http.router." .. router_http_name) attach_http_router_common_methods(router_http) router_http.init_worker(filter) + router_http.router_rebuild_min_interval = router_rebuild_min_interval _M.router_http = router_http local router_ssl = require("apisix.ssl.router." .. router_ssl_name) @@ -92,6 +96,7 @@ function _M.http_init_worker() if conf and conf.apisix and conf.apisix.stream_proxy then local router_stream = require("apisix.stream.router.ip_port") router_stream.stream_init_worker(filter) + router_stream.router_rebuild_min_interval = router_rebuild_min_interval _M.router_stream = router_stream end @@ -100,10 +105,17 @@ end function _M.stream_init_worker() + local conf = core.config.local_conf() local router_ssl_name = "radixtree_sni" + local min_interval = 0 + + if conf and conf.apisix and conf.apisix.router then + min_interval = conf.apisix.router.router_rebuild_min_interval or 0 + end local router_stream = require("apisix.stream.router.ip_port") router_stream.stream_init_worker(filter) + router_stream.router_rebuild_min_interval = min_interval _M.router_stream = router_stream local router_ssl = require("apisix.ssl.router." .. router_ssl_name) diff --git a/apisix/stream/router/ip_port.lua b/apisix/stream/router/ip_port.lua index 4d502cab0..14bf17ea9 100644 --- a/apisix/stream/router/ip_port.lua +++ b/apisix/stream/router/ip_port.lua @@ -21,12 +21,14 @@ local stream_plugin_checker = require("apisix.plugin").stream_plugin_checker local router_new = require("apisix.utils.router").new local apisix_ssl = require("apisix.ssl") local xrpc = require("apisix.stream.xrpc") +local ngx_now = ngx.now local error = error local tonumber = tonumber local ipairs = ipairs local user_routes local router_ver +local last_router_rebuild_time local tls_router local other_routes = {} local _M = {version = 0.1} @@ -144,14 +146,27 @@ do function _M.match(api_ctx) if router_ver ~= user_routes.conf_version then + local min_interval = _M.router_rebuild_min_interval or 0 + if min_interval > 0 and last_router_rebuild_time then + local elapsed = ngx_now() - last_router_rebuild_time + if elapsed < min_interval then + core.log.info("skip stream router rebuild, elapsed: ", elapsed, + "s, min_interval: ", min_interval, "s") + goto MATCH + end + end + local err = create_router(user_routes.values) if err then return false, "failed to create router: " .. err end router_ver = user_routes.conf_version + last_router_rebuild_time = ngx_now() end + ::MATCH:: + local sni = apisix_ssl.server_name() if sni and tls_router then local sni_rev = sni:reverse() diff --git a/conf/config.yaml.example b/conf/config.yaml.example index a32499710..49e378e7d 100644 --- a/conf/config.yaml.example +++ b/conf/config.yaml.example @@ -71,6 +71,12 @@ apisix: # radixtree_uri: match route by URI # radixtree_uri_with_parameter: similar to radixtree_uri but match URI with parameters. See https://github.com/api7/lua-resty-radixtree/#parameters-in-path for more details. ssl: radixtree_sni # radixtree_sni: match route by SNI + # router_rebuild_min_interval: 0 # Minimum interval (in seconds) between router rebuilds. + # When routes change frequently, the router is rebuilt on every + # change which can cause CPU spikes under high-frequency updates. + # Set this to a positive value (e.g. 1 or 5) to limit the + # rebuild frequency. Default 0 means rebuild immediately on + # every change (current behavior). # http is the default proxy mode. proxy_mode can be one of `http`, `stream`, or `http&stream` proxy_mode: "http" diff --git a/t/router/router-rebuild-min-interval.t b/t/router/router-rebuild-min-interval.t new file mode 100644 index 000000000..67dc6a1f1 --- /dev/null +++ b/t/router/router-rebuild-min-interval.t @@ -0,0 +1,363 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +use t::APISIX 'no_plan'; + +repeat_each(1); +log_level('info'); +worker_connections(256); +no_root_location(); +no_shuffle(); + +add_block_preprocessor(sub { + my ($block) = @_; + + if (!defined $block->yaml_config) { + my $yaml_config = <<_EOC_; +apisix: + node_listen: 1984 + router: + http: 'radixtree_uri' + router_rebuild_min_interval: 0 +_EOC_ + $block->set_value("yaml_config", $yaml_config); + } +}); + +run_tests(); + +__DATA__ + +=== TEST 1: default behavior (min_interval=0) - router rebuilds on every route change +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + local code, body = t('/apisix/admin/routes/1', + ngx.HTTP_PUT, + [[{ + "upstream": { + "nodes": { + "127.0.0.1:1980": 1 + }, + "type": "roundrobin" + }, + "uri": "/hello" + }]] + ) + + if code >= 300 then + ngx.status = code + end + ngx.say(body) + } + } +--- request +GET /t +--- response_body +passed + + + +=== TEST 2: hit route to trigger router rebuild +--- request +GET /hello +--- response_body +hello world +--- no_error_log +skip router rebuild + + + +=== TEST 3: update route to trigger another rebuild +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + local code, body = t('/apisix/admin/routes/1', + ngx.HTTP_PUT, + [[{ + "upstream": { + "nodes": { + "127.0.0.1:1981": 1 + }, + "type": "roundrobin" + }, + "uri": "/hello" + }]] + ) + + if code >= 300 then + ngx.status = code + end + ngx.say(body) + } + } +--- request +GET /t +--- response_body +passed + + + +=== TEST 4: hit route - should rebuild immediately (min_interval=0) +--- request +GET /hello +--- response_body_like eval +qr/hello world/ +--- no_error_log +skip router rebuild + + + +=== TEST 5: set router_rebuild_min_interval to 2 seconds and create initial route +--- yaml_config +apisix: + node_listen: 1984 + router: + http: 'radixtree_uri' + router_rebuild_min_interval: 2 +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + local code, body = t('/apisix/admin/routes/1', + ngx.HTTP_PUT, + [[{ + "upstream": { + "nodes": { + "127.0.0.1:1980": 1 + }, + "type": "roundrobin" + }, + "uri": "/hello" + }]] + ) + + if code >= 300 then + ngx.status = code + end + ngx.say(body) + } + } +--- request +GET /t +--- response_body +passed + + + +=== TEST 6: hit route to trigger initial router build +--- yaml_config +apisix: + node_listen: 1984 + router: + http: 'radixtree_uri' + router_rebuild_min_interval: 2 +--- request +GET /hello +--- response_body +hello world +--- no_error_log +skip router rebuild + + + +=== TEST 7: update route and request immediately - rebuild should be skipped +--- yaml_config +apisix: + node_listen: 1984 + router: + http: 'radixtree_uri' + router_rebuild_min_interval: 2 +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + local code, body = t('/apisix/admin/routes/1', + ngx.HTTP_PUT, + [[{ + "upstream": { + "nodes": { + "127.0.0.1:1981": 1 + }, + "type": "roundrobin" + }, + "uri": "/hello" + }]] + ) + + if code >= 300 then + ngx.status = code + ngx.say(body) + return + end + + -- request immediately after route change, within min_interval + local http = require("resty.http") + local httpc = http.new() + local res, err = httpc:request_uri("http://127.0.0.1:1984/hello";) + if not res then + ngx.say("request failed: ", err) + return + end + + ngx.say("status: ", res.status) + ngx.say("body: ", res.body) + } + } +--- request +GET /t +--- response_body +status: 200 +body: hello world +--- error_log +skip router rebuild + + + +=== TEST 8: wait for min_interval to pass, then rebuild should happen +--- yaml_config +apisix: + node_listen: 1984 + router: + http: 'radixtree_uri' + router_rebuild_min_interval: 2 +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + + -- update route + local code, body = t('/apisix/admin/routes/1', + ngx.HTTP_PUT, + [[{ + "upstream": { + "nodes": { + "127.0.0.1:1980": 1 + }, + "type": "roundrobin" + }, + "uri": "/hello" + }]] + ) + + if code >= 300 then + ngx.status = code + ngx.say(body) + return + end + + -- wait for min_interval to expire + ngx.sleep(2.1) + + -- this request should trigger a rebuild + local http = require("resty.http") + local httpc = http.new() + local res, err = httpc:request_uri("http://127.0.0.1:1984/hello";) + if not res then + ngx.say("request failed: ", err) + return + end + + ngx.say("status: ", res.status) + ngx.say("body: ", res.body) + } + } +--- request +GET /t +--- response_body +status: 200 +body: hello world +--- no_error_log +skip router rebuild +--- timeout: 5 + + + +=== TEST 9: rapid route updates within min_interval - only one rebuild +--- yaml_config +apisix: + node_listen: 1984 + router: + http: 'radixtree_uri' + router_rebuild_min_interval: 3 +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + local http = require("resty.http") + + -- create initial route and trigger first build + local code, body = t('/apisix/admin/routes/1', + ngx.HTTP_PUT, + [[{ + "upstream": { + "nodes": { + "127.0.0.1:1980": 1 + }, + "type": "roundrobin" + }, + "uri": "/hello" + }]] + ) + + if code >= 300 then + ngx.status = code + ngx.say(body) + return + end + + -- trigger initial build + local httpc = http.new() + local res, err = httpc:request_uri("http://127.0.0.1:1984/hello";) + if not res then + ngx.say("initial request failed: ", err) + return + end + + -- rapid updates (simulating high-frequency route changes) + for i = 1, 5 do + t('/apisix/admin/routes/1', + ngx.HTTP_PUT, + [[{ + "upstream": { + "nodes": { + "127.0.0.1:1980": 1 + }, + "type": "roundrobin" + }, + "uri": "/hello" + }]] + ) + -- request after each update + httpc = http.new() + res, err = httpc:request_uri("http://127.0.0.1:1984/hello";) + if not res then + ngx.say("request ", i, " failed: ", err) + return + end + end + + ngx.say("all requests succeeded") + } + } +--- request +GET /t +--- response_body +all requests succeeded +--- error_log +skip router rebuild
