This is an automated email from the ASF dual-hosted git repository.
spacewander pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/apisix.git
The following commit(s) were added to refs/heads/master by this push:
new 8e1a1fd10 feat(deployment): select backend & retry (#7309)
8e1a1fd10 is described below
commit 8e1a1fd10ab55891f63f15f0ded3df88ad6f44bd
Author: 罗泽轩 <[email protected]>
AuthorDate: Fri Jun 24 09:41:37 2022 +0800
feat(deployment): select backend & retry (#7309)
Signed-off-by: spacewander <[email protected]>
---
apisix/cli/snippet.lua | 21 +++--
apisix/conf_server.lua | 195 +++++++++++++++++++++++++++++++++++++++++++++
apisix/init.lua | 2 +
t/deployment/conf_server.t | 166 ++++++++++++++++++++++++++++++++++++++
4 files changed, 379 insertions(+), 5 deletions(-)
diff --git a/apisix/cli/snippet.lua b/apisix/cli/snippet.lua
index 191e3b0ed..2ce4a6627 100644
--- a/apisix/cli/snippet.lua
+++ b/apisix/cli/snippet.lua
@@ -46,18 +46,25 @@ function _M.generate_conf_server(env, conf)
if not to then
return nil, "bad etcd endpoint format"
end
- servers[i] = s:sub(to + 1)
end
local conf_render = template.compile([[
upstream apisix_conf_backend {
- {% for _, addr in ipairs(servers) do %}
- server {* addr *};
- {% end %}
+ server 0.0.0.0:80;
+ balancer_by_lua_block {
+ local conf_server = require("apisix.conf_server")
+ conf_server.balancer()
+ }
}
server {
listen unix:{* home *}/conf/config_listen.sock;
access_log off;
+
+ access_by_lua_block {
+ local conf_server = require("apisix.conf_server")
+ conf_server.access()
+ }
+
location / {
{% if enable_https then %}
proxy_pass https://apisix_conf_backend;
@@ -70,10 +77,14 @@ function _M.generate_conf_server(env, conf)
proxy_http_version 1.1;
proxy_set_header Connection "";
}
+
+ log_by_lua_block {
+ local conf_server = require("apisix.conf_server")
+ conf_server.log()
+ }
}
]])
return conf_render({
- servers = servers,
enable_https = enable_https,
home = env.apisix_home or ".",
})
diff --git a/apisix/conf_server.lua b/apisix/conf_server.lua
new file mode 100644
index 000000000..9c59b3795
--- /dev/null
+++ b/apisix/conf_server.lua
@@ -0,0 +1,195 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one or more
+-- contributor license agreements. See the NOTICE file distributed with
+-- this work for additional information regarding copyright ownership.
+-- The ASF licenses this file to You under the Apache License, Version 2.0
+-- (the "License"); you may not use this file except in compliance with
+-- the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+local core = require("apisix.core")
+local fetch_local_conf = require("apisix.core.config_local").local_conf
+local picker = require("apisix.balancer.least_conn")
+local balancer = require("ngx.balancer")
+local error = error
+local ipairs = ipairs
+local ngx = ngx
+
+
+local _M = {}
+local servers = {}
+local resolved_results = {}
+local server_picker
+local has_domain = false
+
+
+local function create_resolved_result(server)
+ local host, port = core.utils.parse_addr(server)
+ return {
+ host = host,
+ port = port,
+ }
+end
+
+
+function _M.init()
+ local conf = fetch_local_conf()
+ if not (conf.deployment and conf.deployment.etcd) then
+ return
+ end
+
+ local etcd = conf.deployment.etcd
+ for i, s in ipairs(etcd.host) do
+ local _, to = core.string.find(s, "://")
+ if not to then
+ error("bad etcd endpoint format")
+ end
+
+ local addr = s:sub(to + 1)
+ local host, _, err = core.utils.parse_addr(addr)
+ if err then
+ error("failed to parse host: ".. err)
+ end
+
+ resolved_results[i] = create_resolved_result(addr)
+ servers[i] = addr
+
+ if not core.utils.parse_ipv4(host) and not core.utils.parse_ipv6(host)
then
+ has_domain = true
+ resolved_results[i].domain = host
+ end
+ end
+
+ if #servers > 1 then
+ local nodes = {}
+ for _, s in ipairs(servers) do
+ nodes[s] = 1
+ end
+ server_picker = picker.new(nodes, {})
+ end
+end
+
+
+local function response_err(err)
+ ngx.log(ngx.ERR, "failure in conf server: ", err)
+ ngx.say(core.json.encode({error = err}))
+ ngx.exit(0)
+end
+
+
+local function resolve_servers(ctx)
+ if not has_domain then
+ return
+ end
+
+ local changed = false
+ for _, res in ipairs(resolved_results) do
+ local domain = res.domain
+ if not domain then
+ goto CONTINUE
+ end
+
+ local ip, err = core.resolver.parse_domain(domain)
+ if ip and res.host ~= ip then
+ res.host = ip
+ changed = true
+ core.log.info(domain, " is resolved to: ", ip)
+ end
+
+ if err then
+ core.log.error("dns resolver resolves domain: ", domain, " error:
", err)
+ end
+
+ ::CONTINUE::
+ end
+
+ if not changed then
+ return
+ end
+
+ if #servers > 1 then
+ local nodes = {}
+ for _, res in ipairs(resolved_results) do
+ local s = res.host .. ":" .. res.port
+ nodes[s] = 1
+ end
+ server_picker = picker.new(nodes, {})
+ end
+end
+
+
+local function pick_node(ctx)
+ local res
+ if server_picker then
+ local server, err = server_picker.get(ctx)
+ if not server then
+ err = err or "no valid upstream node"
+ return nil, "failed to find valid upstream server, " .. err
+ end
+
+ ctx.server_picker = server_picker
+ ctx.balancer_server = server
+ res = create_resolved_result(server)
+ else
+ res = resolved_results[1]
+ end
+
+ ctx.balancer_ip = res.host
+ ctx.balancer_port = res.port
+ return true
+end
+
+
+function _M.access()
+ local ctx = ngx.ctx
+ -- Nginx's DNS resolver doesn't support search option,
+ -- so we have to use our own resolver
+ resolve_servers(ctx)
+ local ok, err = pick_node(ctx)
+ if not ok then
+ return response_err(err)
+ end
+end
+
+
+function _M.balancer()
+ local ctx = ngx.ctx
+ if not ctx.balancer_run then
+ ctx.balancer_run = true
+ local retries = #servers - 1
+ local ok, err = balancer.set_more_tries(retries)
+ if not ok then
+ core.log.error("could not set upstream retries: ", err)
+ elseif err then
+ core.log.warn("could not set upstream retries: ", err)
+ end
+ else
+ local ok, err = pick_node(ctx)
+ if not ok then
+ return response_err(err)
+ end
+ end
+
+ local ok, err = balancer.set_current_peer(ctx.balancer_ip,
ctx.balancer_port)
+ if not ok then
+ return response_err(err)
+ end
+end
+
+
+function _M.log()
+ local ctx = ngx.ctx
+ if ctx.server_picker and ctx.server_picker.after_balance then
+ ctx.server_picker.after_balance(ctx, false)
+ end
+end
+
+
+return _M
diff --git a/apisix/init.lua b/apisix/init.lua
index 9cbe6d204..25d9d5aa2 100644
--- a/apisix/init.lua
+++ b/apisix/init.lua
@@ -27,6 +27,7 @@ require("jit.opt").start("minstitch=2", "maxtrace=4000",
require("apisix.patch").patch()
local core = require("apisix.core")
+local conf_server = require("apisix.conf_server")
local plugin = require("apisix.plugin")
local plugin_config = require("apisix.plugin_config")
local script = require("apisix.script")
@@ -95,6 +96,7 @@ function _M.http_init(args)
end
xrpc.init()
+ conf_server.init()
end
diff --git a/t/deployment/conf_server.t b/t/deployment/conf_server.t
index 84b045055..2e89ac2ae 100644
--- a/t/deployment/conf_server.t
+++ b/t/deployment/conf_server.t
@@ -98,3 +98,169 @@ deployment:
- https://127.0.0.1:12379
tls:
verify: false
+
+
+
+=== TEST 2: mix ip & domain
+--- config
+ location /t {
+ content_by_lua_block {
+ local etcd = require("apisix.core.etcd")
+ assert(etcd.set("/apisix/test", "foo"))
+ local res = assert(etcd.get("/apisix/test"))
+ ngx.say(res.body.node.value)
+ }
+ }
+--- extra_yaml_config
+deployment:
+ role: traditional
+ role_traditional:
+ config_provider: etcd
+ etcd:
+ prefix: "/apisix"
+ host:
+ - http://127.0.0.2:2379
+ - http://localhost:2379
+ - http://[::1]:2379
+--- error_log
+dns resolve localhost, result:
+--- no_error_log
+[error]
+--- response_body
+foo
+
+
+
+=== TEST 3: resolve domain, result changed
+--- extra_init_by_lua
+ local resolver = require("apisix.core.resolver")
+ local old_f = resolver.parse_domain
+ local counter = 0
+ resolver.parse_domain = function (domain)
+ if domain == "x.com" then
+ counter = counter + 1
+ if counter % 2 == 0 then
+ return "127.0.0.2"
+ else
+ return "127.0.0.3"
+ end
+ else
+ return old_f(domain)
+ end
+ end
+--- config
+ location /t {
+ content_by_lua_block {
+ local etcd = require("apisix.core.etcd")
+ assert(etcd.set("/apisix/test", "foo"))
+ local res = assert(etcd.get("/apisix/test"))
+ ngx.say(res.body.node.value)
+ }
+ }
+--- extra_yaml_config
+deployment:
+ role: traditional
+ role_traditional:
+ config_provider: etcd
+ etcd:
+ prefix: "/apisix"
+ host:
+ - http://x.com:2379
+--- response_body
+foo
+--- error_log
+x.com is resolved to: 127.0.0.3
+x.com is resolved to: 127.0.0.2
+--- no_error_log
+[error]
+
+
+
+=== TEST 4: update balancer if the DNS result changed
+--- extra_init_by_lua
+ local resolver = require("apisix.core.resolver")
+ local old_f = resolver.parse_domain
+ package.loaded.counter = 0
+ resolver.parse_domain = function (domain)
+ if domain == "x.com" then
+ local counter = package.loaded.counter
+ package.loaded.counter = counter + 1
+ if counter % 2 == 0 then
+ return "127.0.0.2"
+ else
+ return "127.0.0.3"
+ end
+ else
+ return old_f(domain)
+ end
+ end
+
+ local picker = require("apisix.balancer.least_conn")
+ package.loaded.n_picker = 0
+ local old_f = picker.new
+ picker.new = function (nodes, upstream)
+ package.loaded.n_picker = package.loaded.n_picker + 1
+ return old_f(nodes, upstream)
+ end
+--- config
+ location /t {
+ content_by_lua_block {
+ local etcd = require("apisix.core.etcd")
+ assert(etcd.set("/apisix/test", "foo"))
+ local res = assert(etcd.get("/apisix/test"))
+ ngx.say(res.body.node.value)
+ local counter = package.loaded.counter
+ local n_picker = package.loaded.n_picker
+ if counter == n_picker then
+ ngx.say("OK")
+ else
+ ngx.say(counter, " ", n_picker)
+ end
+ }
+ }
+--- extra_yaml_config
+deployment:
+ role: traditional
+ role_traditional:
+ config_provider: etcd
+ etcd:
+ prefix: "/apisix"
+ host:
+ - http://127.0.0.1:2379
+ - http://x.com:2379
+--- response_body
+foo
+OK
+--- error_log
+x.com is resolved to: 127.0.0.3
+x.com is resolved to: 127.0.0.2
+--- no_error_log
+[error]
+
+
+
+=== TEST 5: retry
+--- config
+ location /t {
+ content_by_lua_block {
+ local etcd = require("apisix.core.etcd")
+ assert(etcd.set("/apisix/test", "foo"))
+ local res = assert(etcd.get("/apisix/test"))
+ ngx.say(res.body.node.value)
+ }
+ }
+--- extra_yaml_config
+deployment:
+ role: traditional
+ role_traditional:
+ config_provider: etcd
+ etcd:
+ prefix: "/apisix"
+ host:
+ - http://127.0.0.1:1979
+ - http://[::1]:1979
+ - http://localhost:2379
+--- error_log
+connect() failed
+--- response_body
+foo
