This is an automated email from the ASF dual-hosted git repository.
AlinsRan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/apisix.git
The following commit(s) were added to refs/heads/master by this push:
new 57708db45 fix(stream): support upstream client certificate (mTLS) in
L4 proxy (#13596)
57708db45 is described below
commit 57708db45691a243bd3384c8013e7799b1a2b32a
Author: AlinsRan <[email protected]>
AuthorDate: Mon Jun 29 08:32:27 2026 +0800
fix(stream): support upstream client certificate (mTLS) in L4 proxy (#13596)
---
.requirements | 2 +-
apisix/init.lua | 56 +++++++----
apisix/upstream.lua | 65 ++++++++++++-
ci/linux-install-openresty.sh | 6 +-
docs/en/latest/mtls.md | 5 +
t/stream-node/upstream-mtls.t | 220 ++++++++++++++++++++++++++++++++++++++++++
6 files changed, 329 insertions(+), 25 deletions(-)
diff --git a/.requirements b/.requirements
index 75e381cbc..1e23078f8 100644
--- a/.requirements
+++ b/.requirements
@@ -17,5 +17,5 @@
APISIX_PACKAGE_NAME=apisix
-APISIX_RUNTIME=1.3.6
+APISIX_RUNTIME=1.3.8
APISIX_DASHBOARD_COMMIT=c8d3466d3c36386d3888efbc8250cd8183c77298
diff --git a/apisix/init.lua b/apisix/init.lua
index 01838da5b..f975e71dd 100644
--- a/apisix/init.lua
+++ b/apisix/init.lua
@@ -493,6 +493,32 @@ local function common_phase(phase_name)
end
+-- Resolve the upstream client certificate referenced by `tls.client_cert_id`
+-- into `api_ctx.upstream_ssl`. Shared by the http and stream subsystems.
+-- Returns false on error (invalid/missing referenced ssl object).
+local function resolve_upstream_client_cert(api_ctx)
+ if not (api_ctx.matched_upstream and api_ctx.matched_upstream.tls and
+ api_ctx.matched_upstream.tls.client_cert_id) then
+ return true
+ end
+
+ local cert_id = api_ctx.matched_upstream.tls.client_cert_id
+ local upstream_ssl = router.router_ssl.get_by_id(cert_id)
+ if not upstream_ssl or upstream_ssl.type ~= "client" then
+ local err = upstream_ssl and
+ "ssl type should be 'client'" or
+ "ssl id [" .. cert_id .. "] not exits"
+ core.log.error("failed to get ssl cert: ", err)
+ return false
+ end
+
+ core.log.info("matched upstream client ssl object, id: ", cert_id,
+ ", type: ", upstream_ssl.type)
+ api_ctx.upstream_ssl = upstream_ssl
+ return true
+end
+
+
function _M.handle_upstream(api_ctx, route, enable_websocket)
-- some plugins(ai-proxy...) request upstream by http client directly
if api_ctx.bypass_nginx_upstream then
@@ -537,27 +563,12 @@ function _M.handle_upstream(api_ctx, route,
enable_websocket)
api_ctx.matched_upstream = route_val.upstream
end
- if api_ctx.matched_upstream and api_ctx.matched_upstream.tls and
- api_ctx.matched_upstream.tls.client_cert_id then
-
- local cert_id = api_ctx.matched_upstream.tls.client_cert_id
- local upstream_ssl = router.router_ssl.get_by_id(cert_id)
- if not upstream_ssl or upstream_ssl.type ~= "client" then
- local err = upstream_ssl and
- "ssl type should be 'client'" or
- "ssl id [" .. cert_id .. "] not exits"
- core.log.error("failed to get ssl cert: ", err)
-
- if is_http then
- return core.response.exit(502)
- end
-
- return ngx_exit(1)
+ local ok = resolve_upstream_client_cert(api_ctx)
+ if not ok then
+ if is_http then
+ return core.response.exit(502)
end
-
- core.log.info("matched ssl: ",
- core.json.delay_encode(upstream_ssl, true))
- api_ctx.upstream_ssl = upstream_ssl
+ return ngx_exit(1)
end
if enable_websocket then
@@ -1385,6 +1396,11 @@ function _M.stream_preread_phase()
return
end
+ local ok = resolve_upstream_client_cert(api_ctx)
+ if not ok then
+ return ngx_exit(1)
+ end
+
local code, err = set_upstream(matched_route, api_ctx)
if code then
core.log.error("failed to set upstream: ", err)
diff --git a/apisix/upstream.lua b/apisix/upstream.lua
index 1c77543a9..ef89f2a22 100644
--- a/apisix/upstream.lua
+++ b/apisix/upstream.lua
@@ -42,15 +42,25 @@ else
end
local set_stream_upstream_tls
+local set_stream_upstream_cert_and_key
if not is_http then
local ok, apisix_ngx_stream_upstream = pcall(require,
"resty.apisix.stream.upstream")
if ok then
set_stream_upstream_tls = apisix_ngx_stream_upstream.set_tls
- else
+ set_stream_upstream_cert_and_key =
apisix_ngx_stream_upstream.set_cert_and_key
+ end
+ -- guard each function independently: an older runtime may expose the
module
+ -- (set_tls) without the newer mTLS C-API (set_cert_and_key)
+ if not set_stream_upstream_tls then
set_stream_upstream_tls = function ()
return nil, "need to build APISIX-Runtime to support TLS over TCP
upstream"
end
end
+ if not set_stream_upstream_cert_and_key then
+ set_stream_upstream_cert_and_key = function ()
+ return nil, "need to build APISIX-Runtime to support upstream mTLS
over TCP"
+ end
+ end
end
@@ -160,6 +170,54 @@ local function fill_node_info(up_conf, scheme, is_stream)
end
+-- Set upstream client certificate (mTLS) for the stream (L4) subsystem.
+-- Mirrors the http subsystem: the cert/key are parsed and cached once (the key
+-- is AES-decrypted at rest by fetch_pkey) and applied to the upstream SSL
+-- handshake through the apisix-nginx-module stream C API, so the plaintext key
+-- is never stringified into an nginx variable.
+local function set_stream_upstream_client_cert(api_ctx, up_conf)
+ local tls = up_conf.tls
+ if not (tls and (tls.client_cert or tls.client_cert_id)) then
+ return true
+ end
+
+ local client_cert, client_key
+ if tls.client_cert_id then
+ if not api_ctx.upstream_ssl then
+ return nil, "failed to find upstream ssl object for client_cert_id"
+ end
+ client_cert = api_ctx.upstream_ssl.cert
+ client_key = api_ctx.upstream_ssl.key
+ else
+ client_cert = tls.client_cert
+ client_key = tls.client_key
+ end
+
+ if not (client_cert and client_key) then
+ return nil, "missing client certificate or key for upstream mTLS"
+ end
+
+ -- the sni here is just for logging
+ local sni = api_ctx.var.upstream_host
+ local cert, err = apisix_ssl.fetch_cert(sni, client_cert)
+ if not cert then
+ return nil, err
+ end
+
+ local key, err = apisix_ssl.fetch_pkey(sni, client_key)
+ if not key then
+ return nil, err
+ end
+
+ local ok, err = set_stream_upstream_cert_and_key(cert, key)
+ if not ok then
+ return nil, err
+ end
+
+ return true
+end
+
+
function _M.set_by_route(route, api_ctx)
if api_ctx.upstream_conf then
-- upstream_conf has been set by traffic-split plugin
@@ -246,6 +304,11 @@ function _M.set_by_route(route, api_ctx)
if sni then
ngx_var.upstream_sni = sni
end
+
+ local ok, err = set_stream_upstream_client_cert(api_ctx, up_conf)
+ if not ok then
+ return 503, err
+ end
end
local node_ver = resource.get_nodes_ver(up_conf.resource_key)
local resource_version =
upstream_util.version(up_conf.resource_version,
diff --git a/ci/linux-install-openresty.sh b/ci/linux-install-openresty.sh
index 2c8fd81bb..7db860dbb 100755
--- a/ci/linux-install-openresty.sh
+++ b/ci/linux-install-openresty.sh
@@ -61,7 +61,7 @@ else
sudo apt-get -y update --fix-missing
sudo apt-get install -y build-essential gcc g++ cpanminus libxml2-dev
libxslt-dev
- if [ "$APISIX_RUNTIME" != "1.3.6" ]; then
+ if [ "$APISIX_RUNTIME" != "1.3.8" ]; then
echo "Please update the apisix-runtime-debug checksum for
APISIX_RUNTIME=$APISIX_RUNTIME" >&2
exit 1
fi
@@ -69,11 +69,11 @@ else
case "$ARCH" in
x86_64|amd64)
DEB_ARCH="amd64"
-
EXPECTED_SHA256="f3c3836270e4d71c7154bea3dd13005cacad5b489eacf9fab7b048907fa4d641"
+
EXPECTED_SHA256="d617eb9dbabdaa97c9722c7b48260aa26d121c280ecbb2c5e1bdeebc6fbeeb8e"
;;
arm64|aarch64)
DEB_ARCH="arm64"
-
EXPECTED_SHA256="6f5ba1e4dee34f9c2593687b3e97dad53cbc1f2b90283961fd87eba62e4c9bc4"
+
EXPECTED_SHA256="4e263650a6bfb773b53ebf5643fed791d21115e92b4b370d0cd6d43c58fd870c"
;;
*)
echo "Unsupported architecture: $ARCH" >&2
diff --git a/docs/en/latest/mtls.md b/docs/en/latest/mtls.md
index d6cfc0122..0119702ca 100644
--- a/docs/en/latest/mtls.md
+++ b/docs/en/latest/mtls.md
@@ -208,3 +208,8 @@ curl http://127.0.0.1:9180/apisix/admin/upstreams/1 \
}
}'
```
+
+This also works in the stream (L4) subsystem: when an upstream uses the `tls`
+scheme and configures `tls.client_cert`/`tls.client_key` (or
+`tls.client_cert_id`), APISIX presents the client certificate while
+establishing the TLS connection to the upstream.
diff --git a/t/stream-node/upstream-mtls.t b/t/stream-node/upstream-mtls.t
new file mode 100644
index 000000000..fd4c753f8
--- /dev/null
+++ b/t/stream-node/upstream-mtls.t
@@ -0,0 +1,220 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+use t::APISIX;
+
+repeat_each(1);
+no_long_string();
+no_shuffle();
+no_root_location();
+
+my $nginx_binary = $ENV{'TEST_NGINX_BINARY'} || 'nginx';
+my $version = eval { `$nginx_binary -V 2>&1` };
+
+if ($version !~ m/\/apisix-nginx-module/) {
+ plan(skip_all => "apisix-nginx-module not installed");
+} else {
+ plan('no_plan');
+}
+
+add_block_preprocessor(sub {
+ my ($block) = @_;
+
+ if (!$block->request) {
+ $block->set_value("stream_enable", 1);
+
+ # An mTLS-enabled upstream: it requires (and verifies) a client
+ # certificate signed by mtls_ca. A connection without a valid client
+ # certificate is rejected during the TLS handshake.
+ my $stream_config = $block->stream_config // '';
+ $stream_config .= <<_EOC_;
+ server {
+ listen 8765 ssl;
+ ssl_certificate ../../certs/mtls_server.crt;
+ ssl_certificate_key ../../certs/mtls_server.key;
+ ssl_client_certificate ../../certs/mtls_ca.crt;
+ ssl_verify_client on;
+
+ content_by_lua_block {
+ local sock = ngx.req.socket()
+ local data = sock:receive("1")
+ ngx.say("hello mtls upstream")
+ }
+ }
+_EOC_
+
+ $block->set_value("extra_stream_config", $stream_config);
+ }
+});
+
+run_tests();
+
+__DATA__
+
+=== TEST 1: set upstream (with client cert) & stream_route
+--- config
+ location /t {
+ content_by_lua_block {
+ local t = require("lib.test_admin")
+ local json = require("toolkit.json")
+ local ssl_cert = t.read_file("t/certs/mtls_client.crt")
+ local ssl_key = t.read_file("t/certs/mtls_client.key")
+ local data = {
+ scheme = "tls",
+ type = "roundrobin",
+ nodes = {
+ ["127.0.0.1:8765"] = 1,
+ },
+ tls = {
+ client_cert = ssl_cert,
+ client_key = ssl_key,
+ }
+ }
+ local code, body = t.test('/apisix/admin/upstreams/1',
+ ngx.HTTP_PUT,
+ json.encode(data)
+ )
+ if code >= 300 then
+ ngx.status = code
+ ngx.say(body)
+ return
+ end
+
+ local code, body = t.test('/apisix/admin/stream_routes/1',
+ ngx.HTTP_PUT,
+ [[{
+ "remote_addr": "127.0.0.1",
+ "upstream_id": "1"
+ }]]
+ )
+ if code >= 300 then
+ ngx.status = code
+ end
+ ngx.say(body)
+ }
+ }
+--- request
+GET /t
+--- response_body
+passed
+
+
+
+=== TEST 2: hit route, upstream mTLS succeeds with client cert
+--- stream_request
+mmm
+--- stream_response
+hello mtls upstream
+--- no_error_log
+[error]
+
+
+
+=== TEST 3: set upstream WITHOUT client cert
+--- config
+ location /t {
+ content_by_lua_block {
+ local t = require("lib.test_admin")
+ local code, body = t.test('/apisix/admin/upstreams/1',
+ ngx.HTTP_PUT,
+ [[{
+ "scheme": "tls",
+ "type": "roundrobin",
+ "nodes": {
+ "127.0.0.1:8765": 1
+ }
+ }]]
+ )
+ if code >= 300 then
+ ngx.status = code
+ end
+ ngx.say(body)
+ }
+ }
+--- request
+GET /t
+--- response_body
+passed
+
+
+
+=== TEST 4: hit route, upstream rejects connection without client cert
+--- stream_request
+mmm
+--- stream_response eval
+qr//
+--- error_log
+client sent no required SSL certificate while SSL handshaking
+
+
+
+=== TEST 5: set upstream client cert via client_cert_id (ssl object)
+--- config
+ location /t {
+ content_by_lua_block {
+ local t = require("lib.test_admin")
+ local json = require("toolkit.json")
+ local ssl_cert = t.read_file("t/certs/mtls_client.crt")
+ local ssl_key = t.read_file("t/certs/mtls_client.key")
+
+ local data = {
+ cert = ssl_cert,
+ key = ssl_key,
+ type = "client",
+ }
+ local code, body = t.test('/apisix/admin/ssls/1',
+ ngx.HTTP_PUT,
+ json.encode(data)
+ )
+ if code >= 300 then
+ ngx.status = code
+ ngx.say(body)
+ return
+ end
+
+ local code, body = t.test('/apisix/admin/upstreams/1',
+ ngx.HTTP_PUT,
+ [[{
+ "scheme": "tls",
+ "type": "roundrobin",
+ "nodes": {
+ "127.0.0.1:8765": 1
+ },
+ "tls": {
+ "client_cert_id": "1"
+ }
+ }]]
+ )
+ if code >= 300 then
+ ngx.status = code
+ end
+ ngx.say(body)
+ }
+ }
+--- request
+GET /t
+--- response_body
+passed
+
+
+
+=== TEST 6: hit route, upstream mTLS succeeds with client_cert_id
+--- stream_request
+mmm
+--- stream_response
+hello mtls upstream
+--- no_error_log
+[error]
