This is an automated email from the ASF dual-hosted git repository. membphis pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/apisix.git
The following commit(s) were added to refs/heads/master by this push: new e295ebb feat: support upstream mTLS (#4005) e295ebb is described below commit e295ebb9e811e505f0c9f6b604dd6a7b25bb0a74 Author: 罗泽轩 <spacewander...@gmail.com> AuthorDate: Sat Apr 10 18:55:20 2021 +0800 feat: support upstream mTLS (#4005) --- apisix/admin/routes.lua | 4 +- apisix/admin/services.lua | 4 +- apisix/admin/upstreams.lua | 77 +---- apisix/http/service.lua | 41 +-- apisix/router.lua | 40 +-- apisix/schema_def.lua | 28 +- apisix/ssl.lua | 50 +++- apisix/ssl/router/radixtree_sni.lua | 49 +-- apisix/upstream.lua | 202 ++++++++++--- docs/en/latest/admin-api.md | 6 + docs/zh/latest/admin-api.md | 6 + t/APISIX.pm | 10 + t/config-center-yaml/route-upstream.t | 54 ++++ t/node/upstream-mtls.t | 547 ++++++++++++++++++++++++++++++++++ 14 files changed, 867 insertions(+), 251 deletions(-) diff --git a/apisix/admin/routes.lua b/apisix/admin/routes.lua index 28d0272..cad66fc 100644 --- a/apisix/admin/routes.lua +++ b/apisix/admin/routes.lua @@ -16,8 +16,8 @@ -- local expr = require("resty.expr.v1") local core = require("apisix.core") +local apisix_upstream = require("apisix.upstream") local schema_plugin = require("apisix.admin.plugins").check_schema -local upstreams = require("apisix.admin.upstreams") local utils = require("apisix.admin.utils") local tostring = tostring local type = type @@ -68,7 +68,7 @@ local function check_conf(id, conf, need_id) local upstream_conf = conf.upstream if upstream_conf then - local ok, err = upstreams.check_upstream_conf(upstream_conf) + local ok, err = apisix_upstream.check_upstream_conf(upstream_conf) if not ok then return nil, {error_msg = err} end diff --git a/apisix/admin/services.lua b/apisix/admin/services.lua index 9ac26aa..4b6e98e 100644 --- a/apisix/admin/services.lua +++ b/apisix/admin/services.lua @@ -16,8 +16,8 @@ -- local core = require("apisix.core") local get_routes = require("apisix.router").http_routes +local apisix_upstream = require("apisix.upstream") local schema_plugin = require("apisix.admin.plugins").check_schema -local upstreams = require("apisix.admin.upstreams") local utils = require("apisix.admin.utils") local tostring = tostring local ipairs = ipairs @@ -63,7 +63,7 @@ local function check_conf(id, conf, need_id) local upstream_conf = conf.upstream if upstream_conf then - local ok, err = upstreams.check_upstream_conf(upstream_conf) + local ok, err = apisix_upstream.check_upstream_conf(upstream_conf) if not ok then return nil, {error_msg = err} end diff --git a/apisix/admin/upstreams.lua b/apisix/admin/upstreams.lua index 312bf77..d367ec3 100644 --- a/apisix/admin/upstreams.lua +++ b/apisix/admin/upstreams.lua @@ -17,6 +17,7 @@ local core = require("apisix.core") local get_routes = require("apisix.router").http_routes local get_services = require("apisix.http.service").services +local apisix_upstream = require("apisix.upstream") local utils = require("apisix.admin.utils") local tostring = tostring local ipairs = ipairs @@ -28,77 +29,6 @@ local _M = { } -local function get_chash_key_schema(hash_on) - if not hash_on then - return nil, "hash_on is nil" - end - - if hash_on == "vars" then - return core.schema.upstream_hash_vars_schema - end - - if hash_on == "header" or hash_on == "cookie" then - return core.schema.upstream_hash_header_schema - end - - if hash_on == "consumer" then - return nil, nil - end - - if hash_on == "vars_combinations" then - return core.schema.upstream_hash_vars_combinations_schema - end - - return nil, "invalid hash_on type " .. hash_on -end - - -local function check_upstream_conf(conf) - local ok, err = core.schema.check(core.schema.upstream, conf) - if not ok then - return false, "invalid configuration: " .. err - end - - if conf.pass_host == "node" and conf.nodes and - core.table.nkeys(conf.nodes) ~= 1 - then - return false, "only support single node for `node` mode currently" - end - - if conf.pass_host == "rewrite" and - (conf.upstream_host == nil or conf.upstream_host == "") - then - return false, "`upstream_host` can't be empty when `pass_host` is `rewrite`" - end - - if conf.type ~= "chash" then - return true - end - - if not conf.hash_on then - conf.hash_on = "vars" - end - - if conf.hash_on ~= "consumer" and not conf.key then - return false, "missing key" - end - - local key_schema, err = get_chash_key_schema(conf.hash_on) - if err then - return false, "type is chash, err: " .. err - end - - if key_schema then - local ok, err = core.schema.check(key_schema, conf.key) - if not ok then - return false, "invalid configuration: " .. err - end - end - - return true -end - - local function check_conf(id, conf, need_id) if not conf then return nil, {error_msg = "missing configurations"} @@ -123,7 +53,7 @@ local function check_conf(id, conf, need_id) core.log.info("schema: ", core.json.delay_encode(core.schema.upstream)) core.log.info("conf : ", core.json.delay_encode(conf)) - local ok, err = check_upstream_conf(conf) + local ok, err = apisix_upstream.check_upstream_conf(conf) if not ok then return nil, {error_msg = err} end @@ -295,8 +225,5 @@ function _M.patch(id, conf, sub_path) return res.status, res.body end --- for routes and services check upstream conf -_M.check_upstream_conf = check_upstream_conf - return _M diff --git a/apisix/http/service.lua b/apisix/http/service.lua index 0e36f9a..83bcb9b 100644 --- a/apisix/http/service.lua +++ b/apisix/http/service.lua @@ -15,11 +15,10 @@ -- limitations under the License. -- local core = require("apisix.core") +local apisix_upstream = require("apisix.upstream") local plugin_checker = require("apisix.plugin").plugin_checker -local ipairs = ipairs local services local error = error -local pairs = pairs local _M = { @@ -47,43 +46,7 @@ local function filter(service) return end - if not service.value.upstream then - return - end - - service.value.upstream.parent = service - - if not service.value.upstream.nodes then - return - end - - local nodes = service.value.upstream.nodes - if core.table.isarray(nodes) then - for _, node in ipairs(nodes) do - local host = node.host - if not core.utils.parse_ipv4(host) and - not core.utils.parse_ipv6(host) then - service.has_domain = true - break - end - end - else - local new_nodes = core.table.new(core.table.nkeys(nodes), 0) - for addr, weight in pairs(nodes) do - local host, port = core.utils.parse_addr(addr) - if not core.utils.parse_ipv4(host) and - not core.utils.parse_ipv6(host) then - service.has_domain = true - end - local node = { - host = host, - port = port, - weight = weight, - } - core.table.insert(new_nodes, node) - end - service.value.upstream.nodes = new_nodes - end + apisix_upstream.filter_upstream(service.value.upstream, service) core.log.info("filter service: ", core.json.delay_encode(service, true)) end diff --git a/apisix/router.lua b/apisix/router.lua index 3afb3a4..840f0cd 100644 --- a/apisix/router.lua +++ b/apisix/router.lua @@ -16,11 +16,11 @@ -- local require = require local http_route = require("apisix.http.route") +local apisix_upstream = require("apisix.upstream") local core = require("apisix.core") local plugin_checker = require("apisix.plugin").plugin_checker local str_lower = string.lower local error = error -local pairs = pairs local ipairs = ipairs @@ -44,43 +44,7 @@ local function filter(route) end end - if not route.value.upstream then - return - end - - route.value.upstream.parent = route - - if not route.value.upstream.nodes then - return - end - - local nodes = route.value.upstream.nodes - if core.table.isarray(nodes) then - for _, node in ipairs(nodes) do - local host = node.host - if not core.utils.parse_ipv4(host) and - not core.utils.parse_ipv6(host) then - route.has_domain = true - break - end - end - else - local new_nodes = core.table.new(core.table.nkeys(nodes), 0) - for addr, weight in pairs(nodes) do - local host, port = core.utils.parse_addr(addr) - if not core.utils.parse_ipv4(host) and - not core.utils.parse_ipv6(host) then - route.has_domain = true - end - local node = { - host = host, - port = port, - weight = weight, - } - core.table.insert(new_nodes, node) - end - route.value.upstream.nodes = new_nodes - end + apisix_upstream.filter_upstream(route.value.upstream, route) core.log.info("filter route: ", core.json.delay_encode(route, true)) end diff --git a/apisix/schema_def.lua b/apisix/schema_def.lua index 6e171a7..b60faad 100644 --- a/apisix/schema_def.lua +++ b/apisix/schema_def.lua @@ -322,6 +322,16 @@ local nodes_schema = { } +local certificate_scheme = { + type = "string", minLength = 128, maxLength = 64*1024 +} + + +local private_key_schema = { + type = "string", minLength = 128, maxLength = 64*1024 +} + + local upstream_schema = { type = "object", properties = { @@ -341,6 +351,14 @@ local upstream_schema = { }, required = {"connect", "send", "read"}, }, + tls = { + type = "object", + properties = { + client_cert = certificate_scheme, + client_key = private_key_schema, + }, + required = {"client_cert", "client_key"}, + }, type = { description = "algorithms of load balancing", type = "string", @@ -598,16 +616,6 @@ _M.consumer = { _M.upstream = upstream_schema -local certificate_scheme = { - type = "string", minLength = 128, maxLength = 64*1024 -} - - -local private_key_schema = { - type = "string", minLength = 128, maxLength = 64*1024 -} - - _M.ssl = { type = "object", properties = { diff --git a/apisix/ssl.lua b/apisix/ssl.lua index 47a4e34..a3b9a96 100644 --- a/apisix/ssl.lua +++ b/apisix/ssl.lua @@ -23,6 +23,15 @@ local assert = assert local type = type +local cert_cache = core.lrucache.new { + ttl = 3600, count = 1024, +} + +local pkey_cache = core.lrucache.new { + ttl = 3600, count = 1024, +} + + local _M = {} @@ -60,13 +69,13 @@ end local function decrypt_priv_pkey(iv, key) local decoded_key = ngx_decode_base64(key) if not decoded_key then - core.log.error("base64 decode ssl key failed and skipped. key[", key, "] ") + core.log.error("base64 decode ssl key failed. key[", key, "] ") return nil end local decrypted = iv:decrypt(decoded_key) if not decrypted then - core.log.error("decrypt ssl key failed and skipped. key[", key, "] ") + core.log.error("decrypt ssl key failed. key[", key, "] ") end return decrypted @@ -84,7 +93,6 @@ local function aes_decrypt_pkey(origin) end return origin end -_M.aes_decrypt_pkey = aes_decrypt_pkey function _M.validate(cert, key) @@ -108,4 +116,40 @@ function _M.validate(cert, key) end +local function parse_pem_cert(sni, cert) + core.log.debug("parsing cert for sni: ", sni) + + local parsed, err = ngx_ssl.parse_pem_cert(cert) + return parsed, err +end + + +function _M.fetch_cert(sni, cert) + local parsed_cert, err = cert_cache(cert, nil, parse_pem_cert, sni, cert) + if not parsed_cert then + return false, err + end + + return parsed_cert +end + + +local function parse_pem_priv_key(sni, pkey) + core.log.debug("parsing priv key for sni: ", sni) + + local parsed, err = ngx_ssl.parse_pem_priv_key(aes_decrypt_pkey(pkey)) + return parsed, err +end + + +function _M.fetch_pkey(sni, pkey) + local parsed_pkey, err = pkey_cache(pkey, nil, parse_pem_priv_key, sni, pkey) + if not parsed_pkey then + return false, err + end + + return parsed_pkey +end + + return _M diff --git a/apisix/ssl/router/radixtree_sni.lua b/apisix/ssl/router/radixtree_sni.lua index 0790627..fe0bf35 100644 --- a/apisix/ssl/router/radixtree_sni.lua +++ b/apisix/ssl/router/radixtree_sni.lua @@ -29,37 +29,12 @@ local ssl_certificates local radixtree_router local radixtree_router_ver -local cert_cache = core.lrucache.new { - ttl = 3600, count = 512, -} - -local pkey_cache = core.lrucache.new { - ttl = 3600, count = 512, -} - - local _M = { version = 0.1, server_name = ngx_ssl.server_name, } -local function parse_pem_cert(sni, cert) - core.log.debug("parsing cert for sni: ", sni) - - local parsed, err = ngx_ssl.parse_pem_cert(cert) - return parsed, err -end - - -local function parse_pem_priv_key(sni, pkey) - core.log.debug("parsing priv key for sni: ", sni) - - local parsed, err = ngx_ssl.parse_pem_priv_key(pkey) - return parsed, err -end - - local function create_router(ssl_items) local ssl_items = ssl_items or {} @@ -82,23 +57,6 @@ local function create_router(ssl_items) sni = ssl.value.sni:reverse() end - -- decrypt private key - if ssl.value.key then - local decrypted = apisix_ssl.aes_decrypt_pkey(ssl.value.key) - if decrypted then - ssl.value.key = decrypted - end - end - - if ssl.value.keys then - for i = 1, #ssl.value.keys do - local decrypted = apisix_ssl.aes_decrypt_pkey(ssl.value.keys[i]) - if decrypted then - ssl.value.keys[i] = decrypted - end - end - end - idx = idx + 1 route_items[idx] = { paths = sni, @@ -133,7 +91,7 @@ local function set_pem_ssl_key(sni, cert, pkey) return false, "no request found" end - local parsed_cert, err = cert_cache(cert, nil, parse_pem_cert, sni, cert) + local parsed_cert, err = apisix_ssl.fetch_cert(sni, cert) if not parsed_cert then return false, "failed to parse PEM cert: " .. err end @@ -143,9 +101,8 @@ local function set_pem_ssl_key(sni, cert, pkey) return false, "failed to set PEM cert: " .. err end - local parsed_pkey, err = pkey_cache(pkey, nil, parse_pem_priv_key, sni, - pkey) - if not parsed_pkey then + local parsed_pkey, err = apisix_ssl.fetch_pkey(sni, pkey) + if not parsed_cert then return false, "failed to parse PEM priv key: " .. err end diff --git a/apisix/upstream.lua b/apisix/upstream.lua index 7cef761..5bcc679 100644 --- a/apisix/upstream.lua +++ b/apisix/upstream.lua @@ -18,6 +18,7 @@ local require = require local core = require("apisix.core") local discovery = require("apisix.discovery.init").discovery local upstream_util = require("apisix.utils.upstream") +local apisix_ssl = require("apisix.ssl") local error = error local tostring = tostring local ipairs = ipairs @@ -27,6 +28,17 @@ local upstreams local healthcheck +local set_upstream_tls_client_param +local ok, apisix_ngx_upstream = pcall(require, "resty.apisix.upstream") +if ok then + set_upstream_tls_client_param = apisix_ngx_upstream.set_cert_and_key +else + set_upstream_tls_client_param = function () + return nil, "need to build APISIX-Openresty to support upstream mTLS" + end +end + + local HTTP_CODE_UPSTREAM_UNAVAILABLE = 503 local _M = {} @@ -279,6 +291,25 @@ function _M.set_by_route(route, api_ctx) api_ctx.up_checker = checker end + if up_conf.scheme == "https" and up_conf.tls then + -- the sni here is just for logging + local sni = api_ctx.var.upstream_host + local cert, err = apisix_ssl.fetch_cert(sni, up_conf.tls.client_cert) + if not ok then + return 503, err + end + + local key, err = apisix_ssl.fetch_pkey(sni, up_conf.tls.client_key) + if not ok then + return 503, err + end + + local ok, err = set_upstream_tls_client_param(cert, key) + if not ok then + return 503, err + end + end + return end @@ -297,50 +328,149 @@ function _M.check_schema(conf) end +local function get_chash_key_schema(hash_on) + if not hash_on then + return nil, "hash_on is nil" + end + + if hash_on == "vars" then + return core.schema.upstream_hash_vars_schema + end + + if hash_on == "header" or hash_on == "cookie" then + return core.schema.upstream_hash_header_schema + end + + if hash_on == "consumer" then + return nil, nil + end + + if hash_on == "vars_combinations" then + return core.schema.upstream_hash_vars_combinations_schema + end + + return nil, "invalid hash_on type " .. hash_on +end + + +local function check_upstream_conf(in_dp, conf) + if not in_dp then + local ok, err = core.schema.check(core.schema.upstream, conf) + if not ok then + return false, "invalid configuration: " .. err + end + + -- encrypt the key in the admin + if conf.tls and conf.tls.client_key then + conf.tls.client_key = apisix_ssl.aes_encrypt_pkey(conf.tls.client_key) + end + end + + if conf.pass_host == "node" and conf.nodes and + core.table.nkeys(conf.nodes) ~= 1 + then + return false, "only support single node for `node` mode currently" + end + + if conf.pass_host == "rewrite" and + (conf.upstream_host == nil or conf.upstream_host == "") + then + return false, "`upstream_host` can't be empty when `pass_host` is `rewrite`" + end + + if conf.tls then + local cert = conf.tls.client_cert + local key = conf.tls.client_key + local ok, err = apisix_ssl.validate(cert, key) + if not ok then + return false, err + end + end + + if conf.type ~= "chash" then + return true + end + + if conf.hash_on ~= "consumer" and not conf.key then + return false, "missing key" + end + + local key_schema, err = get_chash_key_schema(conf.hash_on) + if err then + return false, "type is chash, err: " .. err + end + + if key_schema then + local ok, err = core.schema.check(key_schema, conf.key) + if not ok then + return false, "invalid configuration: " .. err + end + end + + return true +end + + +function _M.check_upstream_conf(conf) + return check_upstream_conf(false, conf) +end + + +local function filter_upstream(value, parent) + if not value then + return + end + + value.parent = parent + + if not value.nodes then + return + end + + local nodes = value.nodes + if core.table.isarray(nodes) then + for _, node in ipairs(nodes) do + local host = node.host + if not core.utils.parse_ipv4(host) and + not core.utils.parse_ipv6(host) then + parent.has_domain = true + break + end + end + else + local new_nodes = core.table.new(core.table.nkeys(nodes), 0) + for addr, weight in pairs(nodes) do + local host, port = core.utils.parse_addr(addr) + if not core.utils.parse_ipv4(host) and + not core.utils.parse_ipv6(host) then + parent.has_domain = true + end + local node = { + host = host, + port = port, + weight = weight, + } + core.table.insert(new_nodes, node) + end + value.nodes = new_nodes + end +end +_M.filter_upstream = filter_upstream + + function _M.init_worker() local err upstreams, err = core.config.new("/upstreams", { automatic = true, item_schema = core.schema.upstream, + -- also check extra fields in the DP side + checker = function (item, schema_type) + return check_upstream_conf(true, item) + end, filter = function(upstream) upstream.has_domain = false - if not upstream.value then - return - end - - upstream.value.parent = upstream - if not upstream.value.nodes then - return - end - - local nodes = upstream.value.nodes - if core.table.isarray(nodes) then - for _, node in ipairs(nodes) do - local host = node.host - if not core.utils.parse_ipv4(host) and - not core.utils.parse_ipv6(host) then - upstream.has_domain = true - break - end - end - else - local new_nodes = core.table.new(core.table.nkeys(nodes), 0) - for addr, weight in pairs(nodes) do - local host, port = core.utils.parse_addr(addr) - if not core.utils.parse_ipv4(host) and - not core.utils.parse_ipv6(host) then - upstream.has_domain = true - end - local node = { - host = host, - port = port, - weight = weight, - } - core.table.insert(new_nodes, node) - end - upstream.value.nodes = new_nodes - end + filter_upstream(upstream.value, upstream) core.log.info("filter upstream: ", core.json.delay_encode(upstream, true)) end, diff --git a/docs/en/latest/admin-api.md b/docs/en/latest/admin-api.md index 1634ef7..c0500c6 100644 --- a/docs/en/latest/admin-api.md +++ b/docs/en/latest/admin-api.md @@ -543,6 +543,8 @@ In addition to the basic complex equalization algorithm selection, APISIX's Upst |labels |optional |Key/value pairs to specify attributes|{"version":"v2","build":"16","env":"production"}| |create_time |optional| epoch timestamp in second, like `1602883670`, will be created automatically if missing| |update_time |optional| epoch timestamp in second, like `1602883670`, will be created automatically if missing| +|tls.client_cert |optional| Set the client certificate when connecting to TLS upstream, see below for more details| +|tls.client_key |optional| Set the client priviate key when connecting to TLS upstream, see below for more details| `type` can be one of: @@ -560,6 +562,10 @@ In addition to the basic complex equalization algorithm selection, APISIX's Upst 1. when it is `vars_combinations`, the `key` is required. The `key` can be any [Nginx builtin variables](http://nginx.org/en/docs/varindex.html) combinations, such as `$request_uri$remote_addr`. 1. If there is no value for either `hash_on` or `key`, `remote_addr` will be used as key. +`tls.client_cert/key` can be used to communicate with upstream via mTLS. +Their formats are the same as SSL's `cert` and `key` fields. +This feature requires APISIX to run on [APISIX-OpenResty](../how-to-build.md#6-build-openresty-for-apisix). + **Config Example:** ```shell diff --git a/docs/zh/latest/admin-api.md b/docs/zh/latest/admin-api.md index e3ca1c2..8bffa0f 100644 --- a/docs/zh/latest/admin-api.md +++ b/docs/zh/latest/admin-api.md @@ -546,6 +546,8 @@ APISIX 的 Upstream 除了基本的复杂均衡算法选择外,还支持对上 | labels | 可选 | 匹配规则 | 标识附加属性的键值对 | {"version":"v2","build":"16","env":"production"} | | create_time | 可选 | 辅助 | 单位为秒的 epoch 时间戳,如果不指定则自动创建 | 1602883670 | | update_time | 可选 | 辅助 | 单位为秒的 epoch 时间戳,如果不指定则自动创建 | 1602883670 | +| tls.client_cert | 可选 | https 证书 | 设置跟上游通信时的客户端证书,细节见下文 | | +| update_time | 可选 | https 证书私钥 | 设置跟上游通信时的客户端私钥,细节见下文 | | `type` 可以是以下的一种: @@ -562,6 +564,10 @@ APISIX 的 Upstream 除了基本的复杂均衡算法选择外,还支持对上 4. 设为 `consumer` 时,`key` 不需要设置。此时哈希算法采用的 `key` 为认证通过的 `consumer_name`。 5. 如果指定的 `hash_on` 和 `key` 获取不到值时,就是用默认值:`remote_addr`。 +`tls.client_cert/key` 可以用来跟上游进行 mTLS 通信。 +他们的格式和 SSL 对象的 `cert` 和 `key` 一样。 +这个特性需要 APISIX 运行于 [APISIX-OpenResty](../how-to-build.md#6-build-openresty-for-apisix)。 + **upstream 对象 json 配置内容:** ```shell diff --git a/t/APISIX.pm b/t/APISIX.pm index 2d72236..2b7324d 100644 --- a/t/APISIX.pm +++ b/t/APISIX.pm @@ -389,6 +389,10 @@ _EOC_ _EOC_ + if (defined $block->upstream_server_config) { + $http_config .= $block->upstream_server_config; + } + my $ipv6_fake_server = ""; if (defined $block->listen_ipv6) { $ipv6_fake_server = "listen \[::1\]:1980;"; @@ -426,7 +430,13 @@ _EOC_ ssl_certificate cert/apisix.crt; ssl_certificate_key cert/apisix.key; lua_ssl_trusted_certificate cert/apisix.crt; +_EOC_ + if (defined $block->upstream_server_config) { + $http_config .= $block->upstream_server_config; + } + + $http_config .= <<_EOC_; server_tokens off; ssl_certificate_by_lua_block { diff --git a/t/config-center-yaml/route-upstream.t b/t/config-center-yaml/route-upstream.t index 42908da..ff50ce2 100644 --- a/t/config-center-yaml/route-upstream.t +++ b/t/config-center-yaml/route-upstream.t @@ -158,3 +158,57 @@ GET /get --- error_code: 200 --- no_error_log [error] + + + +=== TEST 6: upstream hash_on (bad) +--- yaml_config eval: $::yaml_config +--- apisix_yaml +routes: + - + id: 1 + uri: /get + upstream_id: 1 +upstreams: + - + id: 1 + nodes: + "httpbin.org:80": 1 + type: chash + hash_on: header + key: "$aaa" +#END +--- request +GET /get +--- error_code: 502 +--- error_log +invalid configuration: failed to match pattern + + + +=== TEST 7: upstream hash_on (good) +--- yaml_config eval: $::yaml_config +--- apisix_yaml +routes: + - + id: 1 + uri: /hello + upstream_id: 1 +upstreams: + - + id: 1 + nodes: + "127.0.0.1:1980": 1 + "127.0.0.2:1980": 1 + type: chash + hash_on: header + key: "test" +#END +--- request +GET /hello +--- more_headers +test: one +--- error_log +proxy request to 127.0.0.1:1980 +--- no_error_log +[error] diff --git a/t/node/upstream-mtls.t b/t/node/upstream-mtls.t new file mode 100644 index 0000000..9c0a49d --- /dev/null +++ b/t/node/upstream-mtls.t @@ -0,0 +1,547 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +use t::APISIX; + +my $nginx_binary = $ENV{'TEST_NGINX_BINARY'} || 'nginx'; +my $version = eval { `$nginx_binary -V 2>&1` }; + +if ($version !~ m/\/apisix-nginx-module/) { + plan(skip_all => "apisix-nginx-module not installed"); +} else { + plan('no_plan'); +} + +repeat_each(1); +log_level('info'); +no_root_location(); +no_shuffle(); + +add_block_preprocessor(sub { + my ($block) = @_; + + if ((!defined $block->error_log) && (!defined $block->no_error_log)) { + $block->set_value("no_error_log", "[error]"); + } +}); + +run_tests(); + +__DATA__ + +=== TEST 1: tls without key +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin") + local json = require("toolkit.json") + local ssl_cert = t.read_file("t/certs/mtls_client.crt") + local data = { + upstream = { + scheme = "https", + type = "roundrobin", + nodes = { + ["127.0.0.1:1983"] = 1, + }, + tls = { + client_cert = ssl_cert, + } + }, + uri = "/hello" + } + local code, body = t.test('/apisix/admin/routes/1', + ngx.HTTP_PUT, + json.encode(data) + ) + + if code >= 300 then + ngx.status = code + end + ngx.print(body) + } + } +--- request +GET /t +--- error_code: 400 +--- response_body +{"error_msg":"invalid configuration: property \"upstream\" validation failed: property \"tls\" validation failed: property \"client_key\" is required"} + + + +=== TEST 2: tls with bad key +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin") + local json = require("toolkit.json") + local ssl_cert = t.read_file("t/certs/mtls_client.crt") + local data = { + upstream = { + scheme = "https", + type = "roundrobin", + nodes = { + ["127.0.0.1:1983"] = 1, + }, + tls = { + client_cert = ssl_cert, + client_key = ("AAA"):rep(128), + } + }, + uri = "/hello" + } + local code, body = t.test('/apisix/admin/routes/1', + ngx.HTTP_PUT, + json.encode(data) + ) + + if code >= 300 then + ngx.status = code + end + ngx.print(body) + } + } +--- request +GET /t +--- error_code: 400 +--- response_body +{"error_msg":"failed to decrypt previous encrypted key"} +--- error_log +decrypt ssl key failed + + + +=== TEST 3: encrypt key by default +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin") + local json = require("toolkit.json") + local ssl_cert = t.read_file("t/certs/mtls_client.crt") + local ssl_key = t.read_file("t/certs/mtls_client.key") + local data = { + upstream = { + scheme = "https", + type = "roundrobin", + nodes = { + ["127.0.0.1:1983"] = 1, + }, + tls = { + client_cert = ssl_cert, + client_key = ssl_key, + } + }, + uri = "/hello" + } + local code, body = t.test('/apisix/admin/routes/1', + ngx.HTTP_PUT, + json.encode(data) + ) + + if code >= 300 then + ngx.status = code + ngx.say(body) + return + end + + local code, body, res = t.test('/apisix/admin/routes/1', + ngx.HTTP_GET + ) + + if code >= 300 then + ngx.status = code + ngx.say(body) + return + end + + res = json.decode(res) + ngx.say(res.node.value.upstream.tls.client_key == ssl_key) + + -- upstream + local data = { + scheme = "https", + type = "roundrobin", + nodes = { + ["127.0.0.1:1983"] = 1, + }, + tls = { + client_cert = ssl_cert, + client_key = ssl_key, + } + } + local code, body = t.test('/apisix/admin/upstreams/1', + ngx.HTTP_PUT, + json.encode(data) + ) + + if code >= 300 then + ngx.status = code + ngx.say(body) + return + end + + local code, body, res = t.test('/apisix/admin/upstreams/1', + ngx.HTTP_GET + ) + + if code >= 300 then + ngx.status = code + ngx.say(body) + return + end + + res = json.decode(res) + ngx.say(res.node.value.tls.client_key == ssl_key) + + local data = { + upstream = { + scheme = "https", + type = "roundrobin", + nodes = { + ["127.0.0.1:1983"] = 1, + }, + tls = { + client_cert = ssl_cert, + client_key = ssl_key, + } + }, + } + local code, body = t.test('/apisix/admin/services/1', + ngx.HTTP_PUT, + json.encode(data) + ) + + if code >= 300 then + ngx.status = code + ngx.say(body) + return + end + + local code, body, res = t.test('/apisix/admin/services/1', + ngx.HTTP_GET + ) + + if code >= 300 then + ngx.status = code + ngx.say(body) + return + end + + res = json.decode(res) + ngx.say(res.node.value.upstream.tls.client_key == ssl_key) + } + } +--- request +GET /t +--- response_body +false +false +false + + + +=== TEST 4: hit +--- upstream_server_config + ssl_client_certificate ../../certs/mtls_ca.crt; + ssl_verify_client on; +--- request +GET /hello +--- response_body +hello world + + + +=== TEST 5: wrong cert +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin") + local json = require("toolkit.json") + local ssl_cert = t.read_file("t/certs/apisix.crt") + local ssl_key = t.read_file("t/certs/apisix.key") + local data = { + upstream = { + scheme = "https", + type = "roundrobin", + nodes = { + ["127.0.0.1:1983"] = 1, + }, + tls = { + client_cert = ssl_cert, + client_key = ssl_key, + } + }, + uri = "/hello" + } + local code, body = t.test('/apisix/admin/routes/1', + ngx.HTTP_PUT, + json.encode(data) + ) + + if code >= 300 then + ngx.status = code + end + + ngx.say(body) + } + } +--- request +GET /t +--- response_body +passed + + + +=== TEST 6: hit +--- upstream_server_config + ssl_client_certificate ../../certs/mtls_ca.crt; + ssl_verify_client on; +--- request +GET /hello +--- error_code: 400 +--- error_log +client SSL certificate verify error + + + +=== TEST 7: clean old data +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin") + assert(t.test('/apisix/admin/routes/1', + ngx.HTTP_DELETE + )) + assert(t.test('/apisix/admin/services/1', + ngx.HTTP_DELETE + )) + assert(t.test('/apisix/admin/upstreams/1', + ngx.HTTP_DELETE + )) + } + } +--- request +GET /t + + + +=== TEST 8: don't encrypt key +--- yaml_config +apisix: + node_listen: 1984 + admin_key: null + ssl: + key_encrypt_salt: null +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin") + local json = require("toolkit.json") + local ssl_cert = t.read_file("t/certs/mtls_client.crt") + local ssl_key = t.read_file("t/certs/mtls_client.key") + local data = { + upstream = { + scheme = "https", + type = "roundrobin", + nodes = { + ["127.0.0.1:1983"] = 1, + }, + tls = { + client_cert = ssl_cert, + client_key = ssl_key, + } + }, + uri = "/hello" + } + local code, body = t.test('/apisix/admin/routes/1', + ngx.HTTP_PUT, + json.encode(data) + ) + + if code >= 300 then + ngx.status = code + ngx.say(body) + return + end + + local code, body, res = t.test('/apisix/admin/routes/1', + ngx.HTTP_GET + ) + + if code >= 300 then + ngx.status = code + ngx.say(body) + return + end + + res = json.decode(res) + ngx.say(res.node.value.upstream.tls.client_key == ssl_key) + + -- upstream + local data = { + scheme = "https", + type = "roundrobin", + nodes = { + ["127.0.0.1:1983"] = 1, + }, + tls = { + client_cert = ssl_cert, + client_key = ssl_key, + } + } + local code, body = t.test('/apisix/admin/upstreams/1', + ngx.HTTP_PUT, + json.encode(data) + ) + + if code >= 300 then + ngx.status = code + ngx.say(body) + return + end + + local code, body, res = t.test('/apisix/admin/upstreams/1', + ngx.HTTP_GET + ) + + if code >= 300 then + ngx.status = code + ngx.say(body) + return + end + + res = json.decode(res) + ngx.say(res.node.value.tls.client_key == ssl_key) + + local data = { + upstream = { + scheme = "https", + type = "roundrobin", + nodes = { + ["127.0.0.1:1983"] = 1, + }, + tls = { + client_cert = ssl_cert, + client_key = ssl_key, + } + }, + } + local code, body = t.test('/apisix/admin/services/1', + ngx.HTTP_PUT, + json.encode(data) + ) + + if code >= 300 then + ngx.status = code + ngx.say(body) + return + end + + local code, body, res = t.test('/apisix/admin/services/1', + ngx.HTTP_GET + ) + + if code >= 300 then + ngx.status = code + ngx.say(body) + return + end + + res = json.decode(res) + ngx.say(res.node.value.upstream.tls.client_key == ssl_key) + } + } +--- request +GET /t +--- response_body +true +true +true + + + +=== TEST 9: bind upstream +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin") + local json = require("toolkit.json") + local data = { + upstream_id = 1, + uri = "/server_port" + } + local code, body = t.test('/apisix/admin/routes/1', + ngx.HTTP_PUT, + json.encode(data) + ) + + if code >= 300 then + ngx.status = code + ngx.say(body) + return + end + } + } +--- request +GET /t + + + +=== TEST 10: hit +--- upstream_server_config + ssl_client_certificate ../../certs/mtls_ca.crt; + ssl_verify_client on; +--- request +GET /server_port +--- response_body chomp +1983 + + + +=== TEST 11: bind service +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin") + local json = require("toolkit.json") + local data = { + service_id = 1, + uri = "/hello_chunked" + } + local code, body = t.test('/apisix/admin/routes/1', + ngx.HTTP_PUT, + json.encode(data) + ) + + if code >= 300 then + ngx.status = code + ngx.say(body) + return + end + } + } +--- request +GET /t + + + +=== TEST 12: hit +--- upstream_server_config + ssl_client_certificate ../../certs/mtls_ca.crt; + ssl_verify_client on; +--- request +GET /hello_chunked +--- response_body +hello world