This is an automated email from the ASF dual-hosted git repository.
spacewander pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/apisix.git
The following commit(s) were added to refs/heads/master by this push:
new 1896cb1 feat: add dump for consul_kv (#3848)
1896cb1 is described below
commit 1896cb1826255aefebe989c54a6dfd7e6feef53d
Author: yongboy <[email protected]>
AuthorDate: Tue Mar 23 10:07:32 2021 +0800
feat: add dump for consul_kv (#3848)
Co-authored-by: nieyong <[email protected]>
---
apisix/control/router.lua | 41 +++-
apisix/discovery/consul_kv.lua | 105 ++++++++-
docs/en/latest/discovery/consul_kv.md | 64 +++++-
t/discovery/consul_kv_dump.t | 386 ++++++++++++++++++++++++++++++++++
4 files changed, 591 insertions(+), 5 deletions(-)
diff --git a/apisix/control/router.lua b/apisix/control/router.lua
index 49fdc23..43fb7e2 100644
--- a/apisix/control/router.lua
+++ b/apisix/control/router.lua
@@ -31,6 +31,41 @@ local get_method = ngx.req.get_method
local _M = {}
+local function format_dismod_uri(mod_name, uri)
+ if core.string.has_prefix(uri, "/v1/") then
+ return uri
+ end
+
+ local tmp = {"/v1/discovery/", mod_name}
+ if not core.string.has_prefix(uri, "/") then
+ core.table.insert(tmp, "/")
+ end
+ core.table.insert(tmp, uri)
+
+ return core.table.concat(tmp, "")
+end
+
+-- we do not hardcode the discovery module's control api uri
+local function format_dismod_control_api_uris(mod_name, api_route)
+ if not api_route or #api_route == 0 then
+ return api_route
+ end
+
+ local clone_route = core.table.clone(api_route)
+ for _, v in ipairs(clone_route) do
+ local uris = v.uris
+ local target_uris = core.table.new(#uris, 0)
+ for _, uri in ipairs(uris) do
+ local target_uri = format_dismod_uri(mod_name, uri)
+ core.table.insert(target_uris, target_uri)
+ end
+ v.uris = target_uris
+ end
+
+ return clone_route
+end
+
+
local fetch_control_api_router
do
local function register_api_routes(routes, api_routes)
@@ -78,14 +113,16 @@ function fetch_control_api_router()
local api_fun = dis_mod.control_api
if api_fun then
local api_route = api_fun()
- register_api_routes(routes, api_route)
+ local format_route = format_dismod_control_api_uris(key,
api_route)
+ register_api_routes(routes, format_route)
end
local dump_data = dis_mod.dump_data
if dump_data then
+ local target_uri = format_dismod_uri(key, "/dump")
local item = {
methods = {"GET"},
- uris = {"/v1/discovery/" .. key .. "/dump"},
+ uris = {target_uri},
handler = function()
return 200, dump_data()
end
diff --git a/apisix/discovery/consul_kv.lua b/apisix/discovery/consul_kv.lua
index e94f043..bd61e17 100644
--- a/apisix/discovery/consul_kv.lua
+++ b/apisix/discovery/consul_kv.lua
@@ -20,11 +20,12 @@ local core = require("apisix.core")
local resty_consul = require('resty.consul')
local cjson = require('cjson')
local http = require('resty.http')
+local util = require("apisix.cli.util")
local ipairs = ipairs
local error = error
local ngx = ngx
local unpack = unpack
-local ngx_re_match = ngx.re.match
+local ngx_re_match = ngx.re.match
local tonumber = tonumber
local pairs = pairs
local ipairs = ipairs
@@ -33,13 +34,14 @@ local ngx_timer_every = ngx.timer.every
local log = core.log
local ngx_decode_base64 = ngx.decode_base64
local json_delay_encode = core.json.delay_encode
-local cjson_null = cjson.null
+local cjson_null = cjson.null
local applications = core.table.new(0, 5)
local default_service
local default_weight
local default_prefix_rule
local skip_keys_map = core.table.new(0, 1)
+local dump_params
local events
local events_list
@@ -82,6 +84,15 @@ local schema = {
type = "string",
}
},
+ dump = {
+ type = "object",
+ properties = {
+ path = {type = "string", minLength = 1},
+ load_on_init = {type = "boolean", default = true},
+ expire = {type = "integer", default = 0},
+ },
+ required = {"path"},
+ },
default_service = {
type = "object",
properties = {
@@ -230,6 +241,72 @@ local function update_application(server_name_prefix, data)
end
+local function read_dump_srvs()
+ local data, err = util.read_file(dump_params.path)
+ if not data then
+ log.notice("read dump file get error: ", err)
+ return
+ end
+
+ log.info("read dump file: ", data)
+ data = util.trim(data)
+ if #data == 0 then
+ log.error("dump file is empty")
+ return
+ end
+
+ local entity, err = core.json.decode(data)
+ if err then
+ log.error("decoded dump data got error: ", err, ", file content: ",
data)
+ return
+ end
+
+ if not entity.services or not entity.last_update then
+ log.warn("decoded dump data miss fields, file content: ", data)
+ return
+ end
+
+ local now_time = ngx.time()
+ log.info("dump file last_update: ", entity.last_update, ",
dump_params.expire: ",
+ dump_params.expire, ", now_time: ", now_time)
+ if dump_params.expire ~= 0 and (entity.last_update + dump_params.expire)
< now_time then
+ log.warn("dump file: ", dump_params.path, " had expired, ignored it")
+ return
+ end
+
+ applications = entity.services
+ log.info("load dump file into memory success")
+end
+
+
+local function write_dump_srvs()
+ local entity = {
+ services = applications,
+ last_update = ngx.time(),
+ expire = dump_params.expire, -- later need handle it
+ }
+ local data = core.json.encode(entity)
+ local succ, err = util.write_file(dump_params.path, data)
+ if not succ then
+ log.error("write dump into file got error: ", err)
+ end
+end
+
+
+local function show_dump_file()
+ if not dump_params then
+ return 503, "dump params is nil"
+ end
+
+ local data, err = util.read_file(dump_params.path)
+ if not data then
+ return 503, err
+ end
+
+ return 200, data
+end
+
+
function _M.connect(premature, consul_server)
if premature then
return
@@ -283,6 +360,10 @@ function _M.connect(premature, consul_server)
log.error("post_event failure with ", events_list._source,
", update application error: ", err)
end
+
+ if dump_params then
+ ngx_timer_at(0, write_dump_srvs)
+ end
end
end
@@ -356,6 +437,15 @@ function _M.init_worker()
return
end
+ if consul_conf.dump then
+ local dump = consul_conf.dump
+ dump_params = dump
+
+ if dump.load_on_init then
+ read_dump_srvs()
+ end
+ end
+
events = require("resty.worker.events")
events_list = events.event_list(
"discovery_consul_update_application",
@@ -411,4 +501,15 @@ function _M.dump_data()
end
+function _M.control_api()
+ return {
+ {
+ methods = {"GET"},
+ uris = {"/show_dump_file"},
+ handler = show_dump_file,
+ }
+ }
+end
+
+
return _M
diff --git a/docs/en/latest/discovery/consul_kv.md
b/docs/en/latest/discovery/consul_kv.md
index b5af505..f729154 100644
--- a/docs/en/latest/discovery/consul_kv.md
+++ b/docs/en/latest/discovery/consul_kv.md
@@ -57,6 +57,9 @@ discovery:
fail_timeout: 1 # default 1 ms
weight: 1 # default 1
max_fails: 1 # default 1
+ dump: # if you need, when registered nodes updated
can dump into file
+ path: "logs/consul_kv.dump"
+ expire: 2592000 # unit sec, here is 30 day
```
And you can config it in short by default value:
@@ -73,6 +76,31 @@ The `keepalive` has two optional values:
- `true`, default and recommend value, use the long pull way to query consul
servers
- `false`, not recommend, it would use the short pull way to query consul
servers, then you can set the `fetch_interval` for fetch interval
+#### Dump Data
+
+When we need reload `apisix` online, as the `consul_kv` module maybe loads
data from CONSUL slower than load routes from ETCD, and would get the log at
the moment before load successfully from consul:
+
+```
+ http_access_phase(): failed to set upstream: no valid upstream node
+```
+
+So, we import the `dump` function for `consul_kv` module. When reload, would
load the dump file before from consul; when the registered nodes in consul been
updated, would dump the upstream nodes into file automatically.
+
+The `dump` has three optional values now:
+
+- `path`, the dump file save path
+ - support relative path, eg: `logs/consul_kv.dump`
+ - support absolute path, eg: `/tmp/consul_kv.bin`
+ - make sure the dump file's parent path exist
+ - make sure the `apisix` has the dump file's read-write access
permission,eg: `chown www:root conf/upstream.d/`
+- `load_on_init`, default value is `true`
+ - if `true`, just try to load the data from the dump file before loading
data from consul when starting, does not care the dump file exists or not
+ - if `false`, ignore loading data from the dump file
+ - Whether `true` or `false`, we don't need to prepare a dump file for
apisix at anytime
+- `expire`, unit sec, avoiding load expired dump data when load
+ - default `0`, it is unexpired forever
+ - recommend 2592000, which is 30 days(equals 3600 \* 24 \* 30)
+
### Register Http API Services
Service register Key&Value template:
@@ -147,7 +175,9 @@ You could find more usage in the
`apisix/t/discovery/consul_kv.t` file.
## Debugging API
-It also offers control api for debugging:
+It also offers control api for debugging.
+
+### Memory Dump API
```shell
GET /v1/discovery/consul_kv/dump
@@ -220,3 +250,35 @@ For example:
}
}
```
+
+### Show Dump File API
+
+It offers another control api for dump file view now. Maybe would add more api
for debugging in future.
+
+```shell
+GET /v1/discovery/consul_kv/show_dump_file
+```
+
+For example:
+
+```shell
+curl http://127.0.0.1:9090/v1/discovery/consul_kv/show_dump_file | jq
+{
+ "services": {
+ "http://172.19.5.31:8500/v1/kv/upstreams/1614480/webpages/": [
+ {
+ "host": "172.19.5.12",
+ "port": 8000,
+ "weight": 120
+ },
+ {
+ "host": "172.19.5.13",
+ "port": 8000,
+ "weight": 120
+ }
+ ]
+ },
+ "expire": 0,
+ "last_update": 1615877468
+}
+```
diff --git a/t/discovery/consul_kv_dump.t b/t/discovery/consul_kv_dump.t
new file mode 100644
index 0000000..7d4e276
--- /dev/null
+++ b/t/discovery/consul_kv_dump.t
@@ -0,0 +1,386 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+use t::APISIX 'no_plan';
+
+repeat_each(1);
+no_long_string();
+no_root_location();
+no_shuffle();
+log_level("info");
+
+
+add_block_preprocessor(sub {
+ my ($block) = @_;
+
+ my $http_config = $block->http_config // <<_EOC_;
+
+ server {
+ listen 30511;
+
+ location /hello {
+ content_by_lua_block {
+ ngx.say("server 1")
+ }
+ }
+ }
+_EOC_
+
+ $block->set_value("http_config", $http_config);
+});
+
+our $yaml_config = <<_EOC_;
+apisix:
+ node_listen: 1984
+ config_center: yaml
+ enable_admin: false
+ enable_control: true
+
+discovery:
+ consul_kv:
+ servers:
+ - "http://127.0.0.1:8500";
+ dump:
+ path: "consul_kv.dump"
+ load_on_init: true
+_EOC_
+
+
+run_tests();
+
+__DATA__
+
+=== TEST 1: prepare nodes
+--- config
+location /v1/kv {
+ proxy_pass http://127.0.0.1:8500;
+}
+--- request eval
+[
+ "DELETE /v1/kv/upstreams/?recurse=true",
+ "PUT /v1/kv/upstreams/webpages/127.0.0.1:30511\n" . "{\"weight\": 1,
\"max_fails\": 1, \"fail_timeout\": 1}",
+]
+--- response_body eval
+[
+ 'true',
+ 'true',
+ 'true',
+]
+
+
+
+=== TEST 2: show dump services
+--- yaml_config eval: $::yaml_config
+--- config
+ location /t {
+ content_by_lua_block {
+ local json = require("toolkit.json")
+ local t = require("lib.test_admin")
+ ngx.sleep(2)
+
+ local code, body, res =
t.test('/v1/discovery/consul_kv/show_dump_file',
+ ngx.HTTP_GET)
+ local entity = json.decode(res)
+ ngx.say(json.encode(entity.services))
+ }
+ }
+--- timeout: 3
+--- request
+GET /t
+--- response_body
+{"http://127.0.0.1:8500/v1/kv/upstreams/webpages/":[{"host":"127.0.0.1","port":30511,"weight":1}]}
+
+
+
+=== TEST 3: prepare dump file for next test
+--- yaml_config
+apisix:
+ node_listen: 1984
+ config_center: yaml
+ enable_admin: false
+ enable_control: true
+
+discovery:
+ consul_kv:
+ servers:
+ - "http://127.0.0.1:8500";
+ dump:
+ path: "/tmp/consul_kv.dump"
+ load_on_init: true
+#END
+--- apisix_yaml
+routes:
+ -
+ uri: /*
+ upstream:
+ service_name: http://127.0.0.1:8500/v1/kv/upstreams/webpages/
+ discovery_type: consul_kv
+ type: roundrobin
+#END
+--- request
+GET /hello
+--- response_body
+server 1
+
+
+
+=== TEST 4: clean registered nodes
+--- config
+location /v1/kv {
+ proxy_pass http://127.0.0.1:8500;
+}
+--- request eval
+[
+ "DELETE /v1/kv/upstreams/?recurse=true",
+]
+--- response_body eval
+[
+ 'true'
+]
+
+
+
+=== TEST 5: test load dump on init
+Configure the invalid consul server addr, and loading the last test 3
generated /tmp/consul_kv.dump file into memory when initializing
+--- yaml_config
+apisix:
+ node_listen: 1984
+ config_center: yaml
+ enable_admin: false
+ enable_control: true
+
+discovery:
+ consul_kv:
+ servers:
+ - "http://127.0.0.1:38500";
+ dump:
+ path: "/tmp/consul_kv.dump"
+ load_on_init: true
+#END
+--- apisix_yaml
+routes:
+ -
+ uri: /*
+ upstream:
+ service_name: http://127.0.0.1:8500/v1/kv/upstreams/webpages/
+ discovery_type: consul_kv
+ type: roundrobin
+#END
+--- request
+GET /hello
+--- response_body
+server 1
+
+
+
+=== TEST 6: delete dump file
+--- config
+ location /t {
+ content_by_lua_block {
+ local util = require("apisix.cli.util")
+ local succ, err = util.execute_cmd("rm -f /tmp/consul_kv.dump")
+ ngx.say(succ and "success" or err)
+ }
+ }
+--- request
+GET /t
+--- response_body
+success
+
+
+
+=== TEST 7: miss load dump on init
+--- yaml_config
+apisix:
+ node_listen: 1984
+ config_center: yaml
+ enable_admin: false
+ enable_control: true
+
+discovery:
+ consul_kv:
+ servers:
+ - "http://127.0.0.1:38500";
+ dump:
+ path: "/tmp/consul_kv.dump"
+ load_on_init: true
+#END
+--- apisix_yaml
+routes:
+ -
+ uri: /*
+ upstream:
+ service_name: http://127.0.0.1:8500/v1/kv/upstreams/webpages/
+ discovery_type: consul_kv
+ type: roundrobin
+#END
+--- request
+GET /hello
+--- error_code: 503
+
+
+
+=== TEST 8: prepare expired dump file
+--- config
+ location /t {
+ content_by_lua_block {
+ local util = require("apisix.cli.util")
+ local json = require("toolkit.json")
+
+ local applications =
json.decode('{"http://127.0.0.1:8500/v1/kv/upstreams/webpages/":[{"host":"127.0.0.1","port":30511,"weight":1}]}')
+ local entity = {
+ services = applications,
+ last_update = ngx.time(),
+ expire = 10,
+ }
+ local succ, err = util.write_file("/tmp/consul_kv.dump",
json.encode(entity))
+
+ ngx.sleep(2)
+ ngx.say(succ and "success" or err)
+ }
+ }
+--- timeout: 3
+--- request
+GET /t
+--- response_body
+success
+
+
+
+=== TEST 9: unexpired dump
+test load unexpired /tmp/consul_kv.dump file generated by upper test when
initializing
+ when initializing
+--- yaml_config
+apisix:
+ node_listen: 1984
+ config_center: yaml
+ enable_admin: false
+ enable_control: true
+
+discovery:
+ consul_kv:
+ servers:
+ - "http://127.0.0.1:38500";
+ dump:
+ path: "/tmp/consul_kv.dump"
+ load_on_init: true
+ expire: 5
+#END
+--- apisix_yaml
+routes:
+ -
+ uri: /*
+ upstream:
+ service_name: http://127.0.0.1:8500/v1/kv/upstreams/webpages/
+ discovery_type: consul_kv
+ type: roundrobin
+#END
+--- request
+GET /hello
+--- response_body
+server 1
+
+
+
+=== TEST 10: expired dump
+test load expired ( by check: (dump_file.last_update + dump.expire) < ngx.time
) ) /tmp/consul_kv.dump file generated by upper test when initializing
+ when initializing
+--- yaml_config
+apisix:
+ node_listen: 1984
+ config_center: yaml
+ enable_admin: false
+ enable_control: true
+
+discovery:
+ consul_kv:
+ servers:
+ - "http://127.0.0.1:38500";
+ dump:
+ path: "/tmp/consul_kv.dump"
+ load_on_init: true
+ expire: 1
+#END
+--- apisix_yaml
+routes:
+ -
+ uri: /*
+ upstream:
+ service_name: http://127.0.0.1:8500/v1/kv/upstreams/webpages/
+ discovery_type: consul_kv
+ type: roundrobin
+#END
+--- request
+GET /hello
+--- error_code: 503
+--- error_log
+dump file: /tmp/consul_kv.dump had expired, ignored it
+
+
+
+=== TEST 11: delete dump file
+--- config
+ location /t {
+ content_by_lua_block {
+ local util = require("apisix.cli.util")
+ local succ, err = util.execute_cmd("rm -f /tmp/consul_kv.dump")
+ ngx.say(succ and "success" or err)
+ }
+ }
+--- request
+GET /t
+--- response_body
+success
+
+
+
+=== TEST 12: dump file inexistence
+--- yaml_config
+apisix:
+ node_listen: 1984
+ config_center: yaml
+ enable_admin: false
+ enable_control: true
+
+discovery:
+ consul_kv:
+ servers:
+ - "http://127.0.0.1:38500";
+ dump:
+ path: "/tmp/consul_kv.dump"
+#END
+--- request
+GET /v1/discovery/consul_kv/show_dump_file
+--- error_code: 503
+
+
+
+=== TEST 13: no dump config
+--- yaml_config
+apisix:
+ node_listen: 1984
+ config_center: yaml
+ enable_admin: false
+ enable_control: true
+
+discovery:
+ consul_kv:
+ servers:
+ - "http://127.0.0.1:38500";
+#END
+--- request
+GET /v1/discovery/consul_kv/show_dump_file
+--- error_code: 503
