This is an automated email from the ASF dual-hosted git repository. membphis pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/apisix.git
The following commit(s) were added to refs/heads/master by this push: new 6a7dfa6 feat: implement api breaker plugin. (#2455) 6a7dfa6 is described below commit 6a7dfa677562876538f3afe8b61f25ad2af39e20 Author: YuanSheng Wang <membp...@gmail.com> AuthorDate: Tue Oct 27 13:40:23 2020 +0800 feat: implement api breaker plugin. (#2455) Co-authored-by: liuheng <liuhenglove...@gmail.com> --- apisix/plugins/api-breaker.lua | 248 +++++++++++++++ bin/apisix | 1 + conf/config-default.yaml | 1 + doc/plugins/api-breaker.md | 117 +++++++ doc/zh-cn/plugins/api-breaker.md | 116 +++++++ t/APISIX.pm | 1 + t/admin/plugins.t | 2 +- t/debug/debug-mode.t | 1 + t/lib/server.lua | 9 + t/plugin/api-breaker.t | 651 +++++++++++++++++++++++++++++++++++++++ 10 files changed, 1146 insertions(+), 1 deletion(-) diff --git a/apisix/plugins/api-breaker.lua b/apisix/plugins/api-breaker.lua new file mode 100644 index 0000000..d97577d --- /dev/null +++ b/apisix/plugins/api-breaker.lua @@ -0,0 +1,248 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one or more +-- contributor license agreements. See the NOTICE file distributed with +-- this work for additional information regarding copyright ownership. +-- The ASF licenses this file to You under the Apache License, Version 2.0 +-- (the "License"); you may not use this file except in compliance with +-- the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- + +local core = require("apisix.core") +local plugin_name = "api-breaker" +local ngx = ngx +local math = math +local error = error +local ipairs = ipairs + +local shared_buffer = ngx.shared['plugin-'.. plugin_name] +if not shared_buffer then + error("failed to get ngx.shared dict when load plugin " .. plugin_name) +end + + +local schema = { + type = "object", + properties = { + break_response_code = { + type = "integer", + minimum = 200, + maximum = 599, + }, + max_breaker_sec = { + type = "integer", + minimum = 3, + default = 300, + }, + unhealthy = { + type = "object", + properties = { + http_statuses = { + type = "array", + minItems = 1, + items = { + type = "integer", + minimum = 500, + maximum = 599, + }, + uniqueItems = true, + default = {500} + }, + failures = { + type = "integer", + minimum = 1, + default = 3, + } + }, + default = {http_statuses = {500}, failures = 3} + }, + healthy = { + type = "object", + properties = { + http_statuses = { + type = "array", + minItems = 1, + items = { + type = "integer", + minimum = 200, + maximum = 499, + }, + uniqueItems = true, + default = {200} + }, + successes = { + type = "integer", + minimum = 1, + default = 3, + } + }, + default = {http_statuses = {200}, successes = 3} + } + }, + required = {"break_response_code"}, +} + + +-- todo: we can move this into `core.talbe` +local function array_find(array, val) + for i, v in ipairs(array) do + if v == val then + return i + end + end + + return nil +end + + +local function gen_healthy_key(ctx) + return "healthy-" .. core.request.get_host(ctx) .. ctx.var.uri +end + + +local function gen_unhealthy_key(ctx) + return "unhealthy-" .. core.request.get_host(ctx) .. ctx.var.uri +end + + +local function gen_lasttime_key(ctx) + return "unhealthy-lastime" .. core.request.get_host(ctx) .. ctx.var.uri +end + + +local _M = { + version = 0.1, + name = plugin_name, + priority = 1005, + schema = schema, +} + + +function _M.check_schema(conf) + return core.schema.check(schema, conf) +end + + +function _M.access(conf, ctx) + local unhealthy_key = gen_unhealthy_key(ctx) + -- unhealthy counts + local unhealthy_count, err = shared_buffer:get(unhealthy_key) + if err then + core.log.warn("failed to get unhealthy_key: ", + unhealthy_key, " err: ", err) + return + end + + if not unhealthy_count then + return + end + + -- timestamp of the last time a unhealthy state was triggered + local lasttime_key = gen_lasttime_key(ctx) + local lasttime, err = shared_buffer:get(lasttime_key) + if err then + core.log.warn("failed to get lasttime_key: ", + lasttime_key, " err: ", err) + return + end + + if not lasttime then + return + end + + local failure_times = math.ceil(unhealthy_count / conf.unhealthy.failures) + if failure_times < 1 then + failure_times = 1 + end + + -- cannot exceed the maximum value of the user configuration + local breaker_time = 2 ^ failure_times + if breaker_time > conf.max_breaker_sec then + breaker_time = conf.max_breaker_sec + end + core.log.info("breaker_time: ", breaker_time) + + -- breaker + if lasttime + breaker_time >= ngx.time() then + return conf.break_response_code + end + + return +end + + +function _M.log(conf, ctx) + local unhealthy_key = gen_unhealthy_key(ctx) + local healthy_key = gen_healthy_key(ctx) + local upstream_status = core.response.get_upstream_status(ctx) + + if not upstream_status then + return + end + + -- unhealth process + if array_find(conf.unhealthy.http_statuses, upstream_status) then + local unhealthy_count, err = shared_buffer:incr(unhealthy_key, 1, 0) + if err then + core.log.warn("failed to incr unhealthy_key: ", unhealthy_key, + " err: ", err) + end + core.log.info("unhealthy_key: ", unhealthy_key, " count: ", + unhealthy_count) + + shared_buffer:delete(healthy_key) + + -- whether the user-configured number of failures has been reached, + -- and if so, the timestamp for entering the unhealthy state. + if unhealthy_count % conf.unhealthy.failures == 0 then + shared_buffer:set(gen_lasttime_key(ctx), ngx.time(), + conf.max_breaker_sec) + core.log.info("update unhealthy_key: ", unhealthy_key, " to ", + unhealthy_count) + end + + return + end + + -- health process + if not array_find(conf.healthy.http_statuses, upstream_status) then + return + end + + local unhealthy_count, err = shared_buffer:get(unhealthy_key) + if err then + core.log.warn("failed to `get` unhealthy_key: ", unhealthy_key, + " err: ", err) + end + + if not unhealthy_count then + return + end + + local healthy_count, err = shared_buffer:incr(healthy_key, 1, 0) + if err then + core.log.warn("failed to `incr` healthy_key: ", healthy_key, + " err: ", err) + end + + -- clear related status + if healthy_count >= conf.healthy.successes then + -- stat change to normal + core.log.info("chagne to normal, ", healthy_key, " ", healthy_count) + shared_buffer:delete(gen_lasttime_key(ctx)) + shared_buffer:delete(unhealthy_key) + shared_buffer:delete(healthy_key) + end + + return +end + + +return _M diff --git a/bin/apisix b/bin/apisix index 5e24fac..efefcea 100755 --- a/bin/apisix +++ b/bin/apisix @@ -191,6 +191,7 @@ http { lua_shared_dict balancer_ewma_locks 10m; lua_shared_dict balancer_ewma_last_touched_at 10m; lua_shared_dict plugin-limit-count-redis-cluster-slot-lock 1m; + lua_shared_dict plugin-api-breaker 10m; # for openid-connect plugin lua_shared_dict discovery 1m; # cache for discovery metadata documents diff --git a/conf/config-default.yaml b/conf/config-default.yaml index 00e0cbc..688fa54 100644 --- a/conf/config-default.yaml +++ b/conf/config-default.yaml @@ -196,6 +196,7 @@ plugins: # plugin list - proxy-mirror - request-id - hmac-auth + - api-breaker stream_plugins: - mqtt-proxy diff --git a/doc/plugins/api-breaker.md b/doc/plugins/api-breaker.md new file mode 100644 index 0000000..5d94236 --- /dev/null +++ b/doc/plugins/api-breaker.md @@ -0,0 +1,117 @@ +<!-- +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +--> + +- [中文](../zh-cn/plugins/api-blocker.md) + +# Summary + +- [**Name**](#name) +- [**Attributes**](#attributes) +- [**How To Enable**](#how-to-enable) +- [**Test Plugin**](#test-plugin) +- [**Disable Plugin**](#disable-plugin) + +## Name + +The plugin implements API fuse functionality to help us protect our upstream business services. + +> About the breaker timeout logic + +the code logic automatically **triggers the unhealthy state** incrementation of the number of operations. + +Whenever the upstream service returns a status code from the `unhealthy.http_statuses` configuration (e.g., 500), up to `unhealthy.failures` (e.g., three times) and considers the upstream service to be in an unhealthy state. + +The first time unhealthy status is triggered, **breaken for 2 seconds**. + +Then, the request is forwarded to the upstream service again after 2 seconds, and if the `unhealthy.http_statuses` status code is returned, and the count reaches `unhealthy.failures` again, **broken for 4 seconds**. + +and so on, 2, 4, 8, 16, 32, 64, ..., 256, 300. `300` is the maximum value of `max_breaker_sec`, allow users to specify. + +In an unhealthy state, when a request is forwarded to an upstream service and the status code in the `healthy.http_statuses` configuration is returned (e.g., 200) that `healthy.successes` is reached (e.g., three times), and the upstream service is considered healthy again. + +## Attributes + +| Name | Type | Requirement | Default | Valid | Description | +| ------------- | ------------- | ----------- | ------- | ---------- | --------------------------------------------------------------------------- | +| break_response_code | integer | required | | [200, ..., 600] | return error code when unhealthy | +| max_breaker_sec | integer | optional | 300 | >=60 | Maximum breaker time(seconds) | +| unhealthy.http_statuses | array[integer] | optional | {500} | [500, ..., 599] | Status codes when unhealthy | +| unhealthy.failures | integer | optional | 1 | >=1 | Number of consecutive error requests that triggered an unhealthy state | +| healthy.http_statuses | array[integer] | optional | {200, 206} | [200, ..., 499] | Status codes when healthy | +| healthy.successes | integer | optional | 1 | >=1 | Number of consecutive normal requests that trigger health status | + +## How To Enable + +Here's an example, enable the `api-breaker` plugin on the specified route. + +Response 500 or 503 three times in a row to trigger a unhealthy. Response 200 once in a row to restore healthy. + +```shell +curl "http://127.0.0.1:9080/apisix/admin/routes/1" -H 'X-API-KEY: edd1c9f034335f136f87ad84b625c8f1' -X PUT -d ' +{ + "plugins": { + "api-breaker": { + "break_response_code": 502, + "unhealthy": { + "http_statuses": [500, 503], + "failures": 3 + }, + "healthy": { + "http_statuses": [200], + "successes": 1 + } + } + }, + "uri": "/hello", + "host": "127.0.0.1", +}' +``` + +## Test Plugin + +Then. Like the configuration above, if your upstream service returns 500. 3 times in a row. The client will receive a 502 (break_response_code) response. + +```shell +$ curl -i -X POST "http://127.0.0.1:9080/get" +HTTP/1.1 502 Bad Gateway +Content-Type: application/octet-stream +Connection: keep-alive +Server: APISIX/1.5 + +... ... +``` + +## Disable Plugin + +When you want to disable the `api-breader` plugin, it is very simple, you can delete the corresponding json configuration in the plugin configuration, no need to restart the service, it will take effect immediately: + +```shell +curl http://127.0.0.1:9080/apisix/admin/routes/1 -H 'X-API-KEY: edd1c9f034335f136f87ad84b625c8f1' -X PUT -d ' +{ + "uri": "/hello", + "upstream": { + "type": "roundrobin", + "nodes": { + "127.0.0.1:1980": 1 + } + } +}' +``` + +The `api-breaker` plugin has been disabled now. It works for other plugins. diff --git a/doc/zh-cn/plugins/api-breaker.md b/doc/zh-cn/plugins/api-breaker.md new file mode 100644 index 0000000..5b90bb6 --- /dev/null +++ b/doc/zh-cn/plugins/api-breaker.md @@ -0,0 +1,116 @@ +<!-- +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +--> + +- [English](../../plugins/api-blocker.md) + +# 目录 + +- [**定义**](#定义) +- [**属性列表**](#属性列表) +- [**启用方式**](#启用方式) +- [**测试插件**](#测试插件) +- [**禁用插件**](#禁用插件) + +## 定义 + +该插件实现 API 熔断功能,帮助我们保护上游业务服务。 + +> 关于熔断超时逻辑 + +由代码逻辑自动按**触发不健康状态**的次数递增运算: + +每当上游服务返回`unhealthy.http_statuses`配置中的状态码(比如:500),达到`unhealthy.failures`次时(比如:3 次),认为上游服务处于不健康状态。 + +第一次触发不健康状态,**熔断 2 秒**。 + +然后,2 秒过后重新开始转发请求到上游服务,如果继续返回`unhealthy.http_statuses`状态码,记数再次达到`unhealthy.failures`次时,**熔断 4 秒**(倍数方式)。 + +依次类推,2, 4, 8, 16, 32, 64, ..., 256, 最大到 300。 300 是 `max_breaker_sec` 的最大值,允许自定义修改。 + +在不健康状态时,当转发请求到上游服务并返回`healthy.http_statuses`配置中的状态码(比如:200),达到`healthy.successes`次时(比如:3 次),认为上游服务恢复健康状态。 + +## 属性列表 + +| 名称 | 类型 | 必选项 | 默认值 | 有效值 | 描述 | +| ----------------------- | -------------- | ------ | ---------- | --------------- | -------------------------------- | +| break_response_code | integer | 必须 | 无 | [200, ..., 600] | 不健康返回错误码 | +| max_breaker_sec | integer | 可选 | 300 | >=60 | 最大熔断持续时间 | +| unhealthy.http_statuses | array[integer] | 可选 | {500} | [500, ..., 599] | 不健康时候的状态码 | +| unhealthy.failures | integer | 可选 | 1 | >=1 | 触发不健康状态的连续错误请求次数 | +| healthy.http_statuses | array[integer] | 可选 | {200, 206} | [200, ..., 499] | 健康时候的状态码 | +| healthy.successes | integer | 可选 | 1 | >=1 | 触发健康状态的连续正常请求次数 | + +## 启用方式 + +这是一个示例,在指定的路由上启用`api-breaker`插件。 +应答 500 或 503 连续 3 次,触发熔断。应答 200 连续 1 次,恢复健康。 + +```shell +curl "http://127.0.0.1:9080/apisix/admin/routes/1" -H 'X-API-KEY: edd1c9f034335f136f87ad84b625c8f1' -X PUT -d ' +{ + "plugins": { + "api-breaker": { + "break_response_code": 502, + "unhealthy": { + "http_statuses": [500, 503], + "failures": 3 + }, + "healthy": { + "http_statuses": [200], + "successes": 1 + } + } + }, + "uri": "/hello", + "host": "127.0.0.1" +}' +``` + +## 测试插件 + +使用上游的配置,如果你的上流服务返回 500,连续 3 次。客户端将会收到 502(break_response_code)应答。 + +```shell +$ curl -i "http://127.0.0.1:9080/get" +HTTP/1.1 502 Bad Gateway +Content-Type: application/octet-stream +Connection: keep-alive +Server: APISIX/1.5 + +... ... +``` + +## 禁用插件 + +当想禁用`api-breaker`插件时,非常简单,只需要在插件配置中删除相应的 json 配置,无需重启服务,即可立即生效: + +```shell +curl http://127.0.0.1:9080/apisix/admin/routes/1 -H 'X-API-KEY: edd1c9f034335f136f87ad84b625c8f1' -X PUT -d ' +{ + "uri": "/hello", + "upstream": { + "type": "roundrobin", + "nodes": { + "127.0.0.1:1980": 1 + } + } +}' +``` + +`api-breaker` 插件现在已被禁用,它也适用于其他插件。 diff --git a/t/APISIX.pm b/t/APISIX.pm index 17448e7..66fd0cc 100644 --- a/t/APISIX.pm +++ b/t/APISIX.pm @@ -241,6 +241,7 @@ _EOC_ lua_shared_dict balancer_ewma_locks 1m; lua_shared_dict balancer_ewma_last_touched_at 1m; lua_shared_dict plugin-limit-count-redis-cluster-slot-lock 1m; + lua_shared_dict plugin-api-breaker 10m; resolver $dns_addrs_str; resolver_timeout 5; diff --git a/t/admin/plugins.t b/t/admin/plugins.t index 4caca83..5f3b3c6 100644 --- a/t/admin/plugins.t +++ b/t/admin/plugins.t @@ -30,7 +30,7 @@ __DATA__ --- request GET /apisix/admin/plugins/list --- response_body_like eval -qr/\["request-id","fault-injection","serverless-pre-function","batch-requests","cors","ip-restriction","referer-restriction","uri-blocker","request-validation","openid-connect","wolf-rbac","hmac-auth","basic-auth","jwt-auth","key-auth","consumer-restriction","authz-keycloak","proxy-mirror","proxy-cache","proxy-rewrite","limit-conn","limit-count","limit-req","node-status","redirect","response-rewrite","grpc-transcode","prometheus","echo","http-logger","tcp-logger","kafka-logger","syslog", [...] +qr/\["request-id","fault-injection","serverless-pre-function","batch-requests","cors","ip-restriction","referer-restriction","uri-blocker","request-validation","openid-connect","wolf-rbac","hmac-auth","basic-auth","jwt-auth","key-auth","consumer-restriction","authz-keycloak","proxy-mirror","proxy-cache","proxy-rewrite","api-breaker","limit-conn","limit-count","limit-req","node-status","redirect","response-rewrite","grpc-transcode","prometheus","echo","http-logger","tcp-logger","kafka-log [...] --- no_error_log [error] diff --git a/t/debug/debug-mode.t b/t/debug/debug-mode.t index 9ad669f..0ad80ce 100644 --- a/t/debug/debug-mode.t +++ b/t/debug/debug-mode.t @@ -66,6 +66,7 @@ loaded plugin and sort by priority: 2000 name: authz-keycloak loaded plugin and sort by priority: 1010 name: proxy-mirror loaded plugin and sort by priority: 1009 name: proxy-cache loaded plugin and sort by priority: 1008 name: proxy-rewrite +loaded plugin and sort by priority: 1005 name: api-breaker loaded plugin and sort by priority: 1003 name: limit-conn loaded plugin and sort by priority: 1002 name: limit-count loaded plugin and sort by priority: 1001 name: limit-req diff --git a/t/lib/server.lua b/t/lib/server.lua index afed16e..0919d2a 100644 --- a/t/lib/server.lua +++ b/t/lib/server.lua @@ -262,6 +262,15 @@ function _M.websocket_handshake() end _M.websocket_handshake_route = _M.websocket_handshake +function _M.api_breaker() + ngx.exit(tonumber(ngx.var.arg_code)) +end + +function _M.mysleep() + ngx.sleep(tonumber(ngx.var.arg_seconds)) + ngx.say(ngx.var.arg_seconds) +end + local function print_uri() ngx.say(ngx.var.uri) end diff --git a/t/plugin/api-breaker.t b/t/plugin/api-breaker.t new file mode 100644 index 0000000..9b5ff4e --- /dev/null +++ b/t/plugin/api-breaker.t @@ -0,0 +1,651 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +use t::APISIX 'no_plan'; + +repeat_each(1); +no_long_string(); +no_shuffle(); +no_root_location(); +log_level('info'); +run_tests; + +__DATA__ + +=== TEST 1: sanity +--- config + location /t { + content_by_lua_block { + local plugin = require("apisix.plugins.api-breaker") + local ok, err = plugin.check_schema({ + break_response_code = 502, + unhealthy = { + http_statuses = {500}, + failures = 1, + }, + healthy = { + http_statuses = {200}, + successes = 1, + }, + }) + if not ok then + ngx.say(err) + end + + ngx.say("done") + } + } +--- request +GET /t +--- response_body +done +--- no_error_log +[error] + + + +=== TEST 2: default configuration +--- config + location /t { + content_by_lua_block { + local plugin = require("apisix.plugins.api-breaker") + local conf = { + break_response_code = 502 + } + + local ok, err = plugin.check_schema(conf) + if not ok then + ngx.say(err) + end + + ngx.say(require("lib.json_sort").encode(conf)) + } + } +--- request +GET /t +--- response_body +{"break_response_code":502,"healthy":{"http_statuses":[200],"successes":3},"max_breaker_sec":300,"unhealthy":{"failures":3,"http_statuses":[500]}} +--- no_error_log +[error] + + + +=== TEST 3: default `healthy` +--- config + location /t { + content_by_lua_block { + local plugin = require("apisix.plugins.api-breaker") + local conf = { + break_response_code = 502, + healthy = {} + } + + local ok, err = plugin.check_schema(conf) + if not ok then + ngx.say(err) + end + + ngx.say(require("lib.json_sort").encode(conf)) + } + } +--- request +GET /t +--- response_body +{"break_response_code":502,"healthy":{"http_statuses":[200],"successes":3},"max_breaker_sec":300,"unhealthy":{"failures":3,"http_statuses":[500]}} +--- no_error_log +[error] + + + +=== TEST 4: default `unhealthy` +--- config + location /t { + content_by_lua_block { + local plugin = require("apisix.plugins.api-breaker") + local conf = { + break_response_code = 502, + unhealthy = {} + } + + local ok, err = plugin.check_schema(conf) + if not ok then + ngx.say(err) + end + + ngx.say(require("lib.json_sort").encode(conf)) + } + } +--- request +GET /t +--- response_body +{"break_response_code":502,"healthy":{"http_statuses":[200],"successes":3},"max_breaker_sec":300,"unhealthy":{"failures":3,"http_statuses":[500]}} +--- no_error_log +[error] + + + +=== TEST 5: bad break_response_code +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + local code, body = t('/apisix/admin/routes/1', + ngx.HTTP_PUT, + [[{ + "plugins": { + "api-breaker": { + "break_response_code": 199, + "unhealthy": { + "http_statuses": [500, 503], + "failures": 3 + }, + "healthy": { + "http_statuses": [200, 206], + "successes": 3 + } + } + }, + "upstream": { + "nodes": { + "127.0.0.1:1980": 1 + }, + "type": "roundrobin" + }, + "uri": "/api_breaker" + }]] + ) + + if code >= 300 then + ngx.status = code + end + ngx.print(body) + } + } +--- request +GET /t +--- error_code: 400 +--- response_body +{"error_msg":"failed to check the configuration of plugin api-breaker err: property \"break_response_code\" validation failed: expected 199 to be greater than 200"} +--- no_error_log +[error] + + + +=== TEST 6: bad max_breaker_sec +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + local code, body = t('/apisix/admin/routes/1', + ngx.HTTP_PUT, + [[{ + "plugins": { + "api-breaker": { + "break_response_code": 200, + "max_breaker_sec": -1 + } + }, + "upstream": { + "nodes": { + "127.0.0.1:1980": 1 + }, + "type": "roundrobin" + }, + "uri": "/api_breaker" + }]] + ) + + if code >= 300 then + ngx.status = code + end + ngx.say(body) + } + } +--- request +GET /t +--- error_code: 400 +--- no_error_log +[error] + + + +=== TEST 7: bad unhealthy.http_statuses +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + local code, body = t('/apisix/admin/routes/1', + ngx.HTTP_PUT, + [[{ + "plugins": { + "api-breaker": { + "break_response_code": 200, + "max_breaker_sec": 40, + "unhealthy": { + "http_statuses": [500, 603], + "failures": 3 + }, + "healthy": { + "http_statuses": [200, 206], + "successes": 3 + } + } + }, + "upstream": { + "nodes": { + "127.0.0.1:1980": 1 + }, + "type": "roundrobin" + }, + "uri": "/api_breaker" + }]] + ) + + if code >= 300 then + ngx.status = code + end + ngx.say(body) + } + } +--- request +GET /t +--- error_code: 400 +--- no_error_log +[error] + + + +=== TEST 8: same http_statuses in healthy +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + local code, body = t('/apisix/admin/routes/1', + ngx.HTTP_PUT, + [[{ + "plugins": { + "api-breaker": { + "break_response_code": 500, + "unhealthy": { + "http_statuses": [500, 503], + "failures": 3 + }, + "healthy": { + "http_statuses": [206, 206], + "successes": 3 + } + } + }, + "upstream": { + "nodes": { + "127.0.0.1:1980": 1 + }, + "type": "roundrobin" + }, + "uri": "/api_breaker" + }]] + ) + + if code >= 300 then + ngx.status = code + end + ngx.print(body) + } + } +--- request +GET /t +--- error_code: 400 +--- response_body +{"error_msg":"failed to check the configuration of plugin api-breaker err: property \"healthy\" validation failed: property \"http_statuses\" validation failed: expected unique items but items 2 and 1 are equal"} +--- no_error_log +[error] + + + +=== TEST 9: set route, http_statuses: [500, 503] +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + local code, body = t('/apisix/admin/routes/1', + ngx.HTTP_PUT, + [[{ + "plugins": { + "api-breaker": { + "break_response_code": 599, + "unhealthy": { + "http_statuses": [500, 503], + "failures": 3 + }, + "healthy": { + "http_statuses": [200, 206], + "successes": 3 + } + } + }, + "upstream": { + "nodes": { + "127.0.0.1:1980": 1 + }, + "type": "roundrobin" + }, + "uri": "/api_breaker" + }]] + ) + + if code >= 300 then + ngx.status = code + end + ngx.say(body) + } + } +--- request +GET /t +--- response_body +passed +--- no_error_log +[error] + + + +=== TEST 10: trigger breaker +--- request eval +[ + "GET /api_breaker?code=200", "GET /api_breaker?code=500", + "GET /api_breaker?code=503", "GET /api_breaker?code=500", + "GET /api_breaker?code=500", "GET /api_breaker?code=500" +] +--- error_code eval +[200, 500, 503, 500, 599, 599] +--- no_error_log +[error] + + + +=== TEST 11: trigger reset status +--- request eval +[ + "GET /api_breaker?code=500", "GET /api_breaker?code=500", + + "GET /api_breaker?code=200", "GET /api_breaker?code=200", + "GET /api_breaker?code=200", + + "GET /api_breaker?code=500", "GET /api_breaker?code=500" +] +--- error_code eval +[ + 500, 500, + 200, 200, 200, + 500, 500 +] +--- no_error_log +[error] + + + +=== TEST 12: trigger del healthy numeration +--- config +location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + local json = require("lib.json_sort") + + -- trigger to unhealth + for i = 1, 4 do + local code = t('/api_breaker?code=500', ngx.HTTP_GET) + ngx.say("code: ", code) + end + + -- break for 3 seconds + ngx.sleep(3) + + -- make a try + for i = 1, 4 do + local code = t('/api_breaker?code=200', ngx.HTTP_GET) + ngx.say("code: ", code) + end + + for i = 1, 4 do + local code = t('/api_breaker?code=500', ngx.HTTP_GET) + ngx.say("code: ", code) + end + } +} +--- request +GET /t +--- response_body +code: 500 +code: 500 +code: 500 +code: 599 +code: 200 +code: 200 +code: 200 +code: 200 +code: 500 +code: 500 +code: 500 +code: 599 +--- no_error_log +[error] +breaker_time: 4 +--- error_log +breaker_time: 2 + + + +=== TEST 13: add plugin with default config value +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + local code, body = t('/apisix/admin/routes/1', + ngx.HTTP_PUT, + [[{ + "plugins": { + "api-breaker": { + "break_response_code": 502, + "unhealthy": { + "failures": 3 + }, + "healthy": { + "successes": 3 + } + } + }, + "upstream": { + "nodes": { + "127.0.0.1:1980": 1 + }, + "type": "roundrobin" + }, + "uri": "/api_breaker" + }]] + ) + + if code >= 300 then + ngx.status = code + end + ngx.say(body) + } + } +--- request +GET /t +--- response_body +passed +--- no_error_log +[error] + + + +=== TEST 14: default value +--- request +GET /api_breaker?code=500 +--- error_code: 500 +--- no_error_log +[error] + + + +=== TEST 15: trigger default value of unhealthy.http_statuses breaker +--- request eval +[ + "GET /api_breaker?code=200", "GET /api_breaker?code=500", + "GET /api_breaker?code=503", "GET /api_breaker?code=500", + "GET /api_breaker?code=500", "GET /api_breaker?code=500" +] +--- error_code eval +[200, 500, 503, 500, 500, 502] +--- no_error_log +[error] + + + +=== TEST 16: unhealthy -> timeout -> normal +--- config + location /sleep1 { + proxy_pass "http://127.0.0.1:1980/sleep1"; + } +--- request eval +[ + "GET /api_breaker?code=500", + "GET /api_breaker?code=500", + "GET /api_breaker?code=500", + "GET /api_breaker?code=200", + + "GET /sleep1", + "GET /sleep1", + "GET /sleep1", + + "GET /api_breaker?code=200", + "GET /api_breaker?code=200", + "GET /api_breaker?code=200", + "GET /api_breaker?code=200", + "GET /api_breaker?code=200"] +--- error_code eval +[ + 500, 500, 500, 502, + 200, 200, 200, + 200, 200, 200, 200,200 +] +--- no_error_log +[error] + + + +=== TEST 17: unhealthy -> timeout -> unhealthy +--- config +location /sleep1 { + proxy_pass "http://127.0.0.1:1980/sleep1"; +} +--- request eval +[ + "GET /api_breaker?code=500", "GET /api_breaker?code=500", + "GET /api_breaker?code=500", "GET /api_breaker?code=200", + + "GET /sleep1", "GET /sleep1", "GET /sleep1", + + "GET /api_breaker?code=500","GET /api_breaker?code=500", + "GET /api_breaker?code=500","GET /api_breaker?code=500" + ] +--- error_code eval +[ + 500, 500, 500, 502, + 200, 200, 200, + 500,502,502,502 +] +--- no_error_log +[error] + + + +=== TEST 18: enable plugin, unhealthy.failures=1 +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + local code, body = t('/apisix/admin/routes/1', + ngx.HTTP_PUT, + [[{ + "plugins": { + "api-breaker": { + "break_response_code": 502, + "max_breaker_sec": 10, + "unhealthy": { + "http_statuses": [500, 503], + "failures": 1 + }, + "healthy": { + "successes": 3 + } + } + }, + "upstream": { + "nodes": { + "127.0.0.1:1980": 1 + }, + "type": "roundrobin" + }, + "uri": "/api_breaker" + }]] + ) + + if code >= 300 then + ngx.status = code + end + ngx.say(body) + } + } +--- request +GET /t +--- response_body +passed +--- no_error_log +[error] + + + +=== TEST 19: hit route 20 times, confirm the breaker time +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + local json = require("lib.json_sort") + + local status_count = {} + for i = 1, 20 do + local code = t('/api_breaker?code=500', ngx.HTTP_GET) + code = tostring(code) + status_count[code] = (status_count[code] or 0) + 1 + ngx.sleep(1) + end + + ngx.say(json.encode(status_count)) + } + } +--- request +GET /t +--- no_error_log +[error] +phase_func(): breaker_time: 16 +--- error_log +phase_func(): breaker_time: 2 +phase_func(): breaker_time: 4 +phase_func(): breaker_time: 8 +phase_func(): breaker_time: 10 +--- response_body +{"500":4,"502":16} +--- timeout: 25