This is an automated email from the ASF dual-hosted git repository.
AlinsRan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/apisix.git
The following commit(s) were added to refs/heads/master by this push:
new 57b401dab feat(stream): support TLS passthrough on the stream proxy
(#13912)
57b401dab is described below
commit 57b401dab32351178404d693ea658989ce52fca5
Author: AlinsRan <[email protected]>
AuthorDate: Mon Sep 7 14:51:42 2026 +0800
feat(stream): support TLS passthrough on the stream proxy (#13912)
---
apisix/cli/ngx_tpl.lua | 91 ++++++-
apisix/cli/ops.lua | 143 +++++++++--
apisix/cli/schema.lua | 3 +
apisix/init.lua | 55 ++++-
apisix/schema_def.lua | 7 +
apisix/ssl.lua | 12 +-
apisix/stream/router/ip_port.lua | 2 +-
apisix/upstream.lua | 9 +
conf/config.yaml.example | 20 ++
docs/en/latest/stream-proxy.md | 85 +++++++
docs/zh/latest/stream-proxy.md | 85 +++++++
t/APISIX.pm | 9 +-
t/cli/test_stream_tls_passthrough.sh | 449 +++++++++++++++++++++++++++++++++++
t/stream-node/tls-passthrough.t | 302 +++++++++++++++++++++++
14 files changed, 1248 insertions(+), 24 deletions(-)
diff --git a/apisix/cli/ngx_tpl.lua b/apisix/cli/ngx_tpl.lua
index 2b4a75231..42833de45 100644
--- a/apisix/cli/ngx_tpl.lua
+++ b/apisix/cli/ngx_tpl.lua
@@ -234,14 +234,27 @@ stream {
}
{% for _, server_group in ipairs(stream_proxy.servers or {}) do %}
+ {% if server_group.tls_mixed then %}
+ upstream {* server_group.tls_terminate_up *} {
+ server unix:{* server_group.tls_terminate_sock *};
+ }
+
+ upstream {* server_group.tls_passthrough_up *} {
+ server unix:{* server_group.tls_passthrough_sock *};
+ }
+ {% end %}
server {
{% for _, item in ipairs(server_group.tcp) do %}
- listen {*item.addr*} {% if item.tls then %} ssl {% end %} {% if
enable_reuseport then %} reuseport {% end %} {% if item.proxy_protocol then %}
proxy_protocol {% end %};
+ listen {*item.addr*} {% if item.tls and not server_group.tls_mixed
then %} ssl {% end %} {% if enable_reuseport then %} reuseport {% end %} {% if
item.proxy_protocol then %} proxy_protocol {% end %};
{% end %}
{% for _, addr in ipairs(server_group.udp) do %}
listen {*addr*} udp {% if enable_reuseport then %} reuseport {% end %};
{% end %}
+ {% if server_group.tls_passthrough then %}
+ ssl_preread on;
+ {% end %}
+
{% if server_group.tcp_enable_ssl then %}
ssl_certificate {* ssl.ssl_cert *};
ssl_certificate_key {* ssl.ssl_cert_key *};
@@ -255,12 +268,43 @@ stream {
}
{% end %}
+ {% if server_group.tls_mixed then %}
+ # carries the client address across the internal hop
+ proxy_protocol on;
+ access_log off;
+
+ set $stream_tls_target "";
+
+ preread_by_lua_block {
+ apisix.stream_tls_route_phase("{* server_group.tls_terminate_up
*}",
+ "{* server_group.tls_passthrough_up
*}")
+ }
+
+ proxy_pass $stream_tls_target;
+ }
+
+ # internal: terminates the handshake
+ server {
+ listen unix:{* server_group.tls_terminate_sock *} ssl proxy_protocol;
+ set_real_ip_from unix:;
+
+ ssl_certificate {* ssl.ssl_cert *};
+ ssl_certificate_key {* ssl.ssl_cert_key *};
+
+ ssl_client_hello_by_lua_block {
+ apisix.ssl_client_hello_phase()
+ }
+
+ ssl_certificate_by_lua_block {
+ apisix.ssl_phase()
+ }
+
{% if server_group.proxy_protocol_to_upstream then %}
proxy_protocol on;
{% end %}
preread_by_lua_block {
- apisix.stream_preread_phase()
+ apisix.stream_preread_phase(nil, true)
}
proxy_pass apisix_backend;
@@ -275,6 +319,49 @@ stream {
apisix.stream_log_phase()
}
}
+
+ # internal: forwards the stream untouched, prereading the same ClientHello
again
+ server {
+ listen unix:{* server_group.tls_passthrough_sock *} proxy_protocol;
+ set_real_ip_from unix:;
+ ssl_preread on;
+
+ {% if server_group.proxy_protocol_to_upstream then %}
+ proxy_protocol on;
+ {% end %}
+
+ preread_by_lua_block {
+ apisix.stream_preread_phase(true, true)
+ }
+
+ proxy_pass apisix_backend;
+
+ log_by_lua_block {
+ apisix.stream_log_phase()
+ }
+ }
+ {% else %}
+ {% if server_group.proxy_protocol_to_upstream then %}
+ proxy_protocol on;
+ {% end %}
+
+ preread_by_lua_block {
+ apisix.stream_preread_phase({% if server_group.tls_passthrough
then %}true{% end %})
+ }
+
+ proxy_pass apisix_backend;
+
+ {% if use_apisix_base and not server_group.tls_passthrough then %}
+ set $upstream_sni "apisix_backend";
+ proxy_ssl_server_name on;
+ proxy_ssl_name $upstream_sni;
+ {% end %}
+
+ log_by_lua_block {
+ apisix.stream_log_phase()
+ }
+ }
+ {% end %}
{% end %}
}
{% end %}
diff --git a/apisix/cli/ops.lua b/apisix/cli/ops.lua
index 0f6a4fae6..6b0f6ee62 100644
--- a/apisix/cli/ops.lua
+++ b/apisix/cli/ops.lua
@@ -131,6 +131,50 @@ local function validate_port_or_range(port_entry)
end
+-- Split a stream_proxy.tcp `addr` into its address and port range. Returns
nil for
+-- the forms validate_port_or_range leaves to nginx.
+local function parse_listen_addr(addr)
+ if type(addr) == "number" then
+ return "", addr, addr
+ end
+
+ local ip, port_part
+ if str_find(addr, "[", 1, true) then
+ local bracket_end = str_find(addr, "]", 1, true)
+ if not (bracket_end and str_sub(addr, bracket_end + 1, bracket_end +
1) == ":") then
+ return nil
+ end
+ ip = str_sub(addr, 1, bracket_end)
+ port_part = str_sub(addr, bracket_end + 2)
+ else
+ local colon_pos = str_find(addr, ":", 1, true)
+ if colon_pos then
+ ip = str_sub(addr, 1, colon_pos - 1)
+ port_part = str_sub(addr, colon_pos + 1)
+ else
+ ip = ""
+ port_part = addr
+ end
+ end
+
+ if ip == "0.0.0.0" then
+ ip = ""
+ end
+
+ local start_str, end_str = port_part:match("^(%d+)%-(%d+)$")
+ if start_str then
+ return ip, tonumber(start_str), tonumber(end_str)
+ end
+
+ local port = tonumber(port_part)
+ if not port then
+ return nil
+ end
+
+ return ip, port, port
+end
+
+
local function help()
print([[
Usage: apisix [action] <argument>
@@ -637,23 +681,23 @@ Please modify "admin_key" in conf/config.yaml .
end
end
- -- Split stream listens into nginx server blocks.
`proxy_protocol_to_upstream`
- -- (sending the PROXY protocol toward the upstream) is a server-level
directive,
- -- so TCP listens that enable it need a dedicated server block. The
accept-side
- -- `proxy_protocol` and `ssl` are per-listen directives and coexist in one
block.
- -- Per-listen settings fall back to the global `proxy_protocol` options;
UDP never
- -- sends the PROXY protocol upstream and always joins the plain block.
+ -- Split stream listens into nginx server blocks, one per combination of
the
+ -- server-level directives a listen needs: `proxy_protocol on` (PROXY
protocol to
+ -- the upstream) and the TLS mode -- plain (`listen ... [ssl]`),
passthrough
+ -- (`ssl_preread on`) or mixed (`ssl_preread on` plus the two internal
servers the
+ -- route picks between). The accept-side `proxy_protocol` and `ssl` are
per-listen
+ -- and coexist in one block. Per-listen settings fall back to the global
+ -- `proxy_protocol` options; UDP always joins the plain block.
if enable_stream and yaml_conf.apisix.stream_proxy then
local stream_proxy = yaml_conf.apisix.stream_proxy
local pp = yaml_conf.apisix.proxy_protocol or {}
local plain = {
tcp = {}, udp = stream_proxy.udp or {},
proxy_protocol_to_upstream = false, tcp_enable_ssl = false,
+ tls_passthrough = false, tls_mixed = false,
}
- local to_upstream = {
- tcp = {}, udp = {},
- proxy_protocol_to_upstream = true, tcp_enable_ssl = false,
- }
+ local servers = {plain}
+ local group_by_key = {["false|plain"] = plain}
for _, item in ipairs(stream_proxy.tcp or {}) do
if item.proxy_protocol == nil then
item.proxy_protocol = pp.enable_tcp_pp
@@ -662,18 +706,83 @@ Please modify "admin_key" in conf/config.yaml .
if up == nil then
up = pp.enable_tcp_pp_to_upstream
end
- local group = up and to_upstream or plain
- if item.tls then
+ up = up and true or false
+ local mode = "plain"
+ if item.tls_passthrough then
+ mode = item.tls and "mixed" or "passthrough"
+ end
+ if mode == "mixed" and up then
+ -- The mixed block reaches its internal servers over a unix
socket,
+ -- and nginx builds the upstream PROXY protocol header from
that
+ -- socket: the destination comes out as `unix:/... 0`, which
is not
+ -- a valid TCP4/TCP6 address and strict backends reject it. A
+ -- dedicated tls or tls_passthrough listen has no internal hop
and
+ -- writes a correct header.
+ util.die("invalid stream_proxy.tcp entry: `",
tostring(item.addr),
+ "` can not combine proxy_protocol_to_upstream with a
mixed ",
+ "`tls` + `tls_passthrough` listen; use a dedicated
listen ",
+ "for either mode\n")
+ end
+
+ local key = tostring(up) .. "|" .. mode
+ local group = group_by_key[key]
+ if not group then
+ group = {
+ tcp = {}, udp = {},
+ proxy_protocol_to_upstream = up, tcp_enable_ssl = false,
+ tls_passthrough = mode ~= "plain", tls_mixed = mode ==
"mixed",
+ }
+ group_by_key[key] = group
+ table_insert(servers, group)
+ end
+ -- in mixed mode the internal server terminates, so the listen
stays plain
+ if item.tls and mode == "plain" then
group.tcp_enable_ssl = true
end
table_insert(group.tcp, item)
end
- local servers = {}
- if #plain.tcp > 0 or #plain.udp > 0 then
- table_insert(servers, plain)
+ if #plain.tcp == 0 and #plain.udp == 0 then
+ table_remove(servers, 1)
+ end
+ -- Two listens needing different server blocks can not share an
address: nginx
+ -- refuses to start with reuseport on, and silently keeps only the
first without.
+ local addr_owner = {}
+ for idx, group in ipairs(servers) do
+ for _, item in ipairs(group.tcp) do
+ local ip, first, last = parse_listen_addr(item.addr)
+ if ip then
+ for port = first, last do
+ local key = ip .. "|" .. port
+ if addr_owner[key] and addr_owner[key] ~= idx then
+ util.die("invalid stream_proxy.tcp entry: `",
tostring(item.addr),
+ "` collides with an earlier entry on the
same address ",
+ "that needs a different nginx server
block; one address ",
+ "can only have one TLS mode and one ",
+ "proxy_protocol_to_upstream setting\n")
+ end
+ addr_owner[key] = idx
+ end
+ end
+ end
end
- if #to_upstream.tcp > 0 then
- table_insert(servers, to_upstream)
+
+ -- Short names on purpose: the kernel caps a unix socket path at ~108
bytes and
+ -- it is rooted at the user-chosen apisix home.
+ for idx, group in ipairs(servers) do
+ if group.tls_mixed then
+ group.tls_terminate_sock = env.apisix_home .. "/logs/stls-t"
.. idx .. ".sock"
+ group.tls_passthrough_sock = env.apisix_home .. "/logs/stls-p"
.. idx .. ".sock"
+ group.tls_terminate_up = "apisix_stream_tls_terminate_" .. idx
+ group.tls_passthrough_up = "apisix_stream_tls_passthrough_" ..
idx
+ for _, path in ipairs({group.tls_terminate_sock,
group.tls_passthrough_sock}) do
+ if #path > 100 then
+ util.die("mixed TLS listens need internal unix sockets
under ",
+ env.apisix_home, "/logs, but `", path, "` is
", #path,
+ " bytes, over the ~108 byte limit; install
APISIX ",
+ "under a shorter path\n")
+ end
+ end
+ end
end
stream_proxy.servers = servers
end
diff --git a/apisix/cli/schema.lua b/apisix/cli/schema.lua
index 25c4cc507..e52453e56 100644
--- a/apisix/cli/schema.lua
+++ b/apisix/cli/schema.lua
@@ -178,6 +178,9 @@ local config_schema = {
tls = {
type = "boolean",
},
+ tls_passthrough = {
+ type = "boolean",
+ },
proxy_protocol = {
type = "boolean",
},
diff --git a/apisix/init.lua b/apisix/init.lua
index 049c15ad9..eb96f7a29 100644
--- a/apisix/init.lua
+++ b/apisix/init.lua
@@ -1389,17 +1389,68 @@ function _M.stream_init_worker()
end
-function _M.stream_preread_phase()
+-- Preread phase of a mixed TLS listen: pick which internal server gets the
+-- connection. Plugins and the log phase run there, not here.
+function _M.stream_tls_route_phase(terminate_upstream, passthrough_upstream)
local ngx_ctx = ngx.ctx
local api_ctx = core.tablepool.fetch("api_ctx", 0, 32)
ngx_ctx.api_ctx = api_ctx
+ -- nothing was terminated here, so the SNI can only come from the
ClientHello
+ api_ctx.tls_passthrough = true
- if not verify_tls_client(api_ctx) then
+ core.ctx.set_vars_meta(api_ctx)
+
+ local ok, err = router.router_stream.match(api_ctx)
+ if not ok then
+ core.log.error(err)
+ end
+
+ -- an unmatched connection goes to the terminating server, which reports
the miss
+ local matched_route = api_ctx.matched_route
+ local target = terminate_upstream
+ if matched_route and matched_route.value.tls_passthrough then
+ target = passthrough_upstream
+ end
+ ngx_var.stream_tls_target = target
+
+ core.log.info("stream tls route: sni: ",
api_ctx.var.ssl_preread_server_name,
+ ", target: ", target)
+
+ core.ctx.release_vars(api_ctx)
+ core.tablepool.release("api_ctx", api_ctx)
+ ngx_ctx.api_ctx = nil
+end
+
+
+-- `tls_passthrough` is set by the template on listens running `ssl_preread
on`:
+-- no local handshake, so no client certificate to verify and the SNI is
prereaded.
+function _M.stream_preread_phase(tls_passthrough, behind_mixed_hop)
+ local ngx_ctx = ngx.ctx
+ local api_ctx = core.tablepool.fetch("api_ctx", 0, 32)
+ ngx_ctx.api_ctx = api_ctx
+ api_ctx.tls_passthrough = tls_passthrough
+
+ if not tls_passthrough and not verify_tls_client(api_ctx) then
return ngx_exit(1)
end
core.ctx.set_vars_meta(api_ctx)
+ -- On the internal servers of a mixed listen the connection arrives over a
unix
+ -- socket, so $server_addr/$server_port describe that socket rather than
the port
+ -- the client reached. The real one is in the PROXY protocol header, and
routes
+ -- match on it.
+ if behind_mixed_hop then
+ local addr = ngx_var.proxy_protocol_server_addr
+ if addr and addr ~= "" then
+ api_ctx.var.server_addr = addr
+ end
+ local port = ngx_var.proxy_protocol_server_port
+ if port and port ~= "" then
+ api_ctx.var.server_port = port
+ end
+ end
+
local ok, err = router.router_stream.match(api_ctx)
if not ok then
core.log.error(err)
diff --git a/apisix/schema_def.lua b/apisix/schema_def.lua
index 3687cbcf5..85dd40905 100644
--- a/apisix/schema_def.lua
+++ b/apisix/schema_def.lua
@@ -1050,6 +1050,13 @@ _M.stream_route = {
type = "string",
pattern = host_def_pat,
},
+ tls_passthrough = {
+ description = "forward the TLS stream to the upstream untouched
instead of "
+ .. "terminating it here; only consulted on a mixed
listen, one "
+ .. "with both tls and tls_passthrough set",
+ type = "boolean",
+ default = false,
+ },
upstream = upstream_schema,
upstream_id = id_schema,
service_id = id_schema,
diff --git a/apisix/ssl.lua b/apisix/ssl.lua
index f8a1b2e8a..29d0e635e 100644
--- a/apisix/ssl.lua
+++ b/apisix/ssl.lua
@@ -23,6 +23,7 @@ local ngx_ssl_client = require("ngx.ssl.clienthello")
local str_lower = string.lower
local str_byte = string.byte
local ngx_sub = ngx.re.sub
+local ngx_var = ngx.var
local cert_cache = core.lrucache.new {
ttl = 3600, count = 1024,
@@ -36,10 +37,19 @@ local pkey_cache = core.lrucache.new {
local _M = {}
-function _M.server_name(clienthello)
+-- `preread` takes the SNI from ngx_stream_ssl_preread_module: under TLS
passthrough
+-- this worker never performs the handshake, so ngx_ssl.server_name() has
nothing.
+function _M.server_name(clienthello, preread)
local sni, err
if clienthello then
sni, err = ngx_ssl_client.get_client_hello_server_name()
+ elseif preread then
+ sni = ngx_var.ssl_preread_server_name
+ if sni == "" then
+ -- defined but empty when the ClientHello carried no SNI; "" is
truthy
+ -- and would skip the fallback below
+ sni = nil
+ end
else
sni, err = ngx_ssl.server_name()
end
diff --git a/apisix/stream/router/ip_port.lua b/apisix/stream/router/ip_port.lua
index 1757824f9..ce0a81910 100644
--- a/apisix/stream/router/ip_port.lua
+++ b/apisix/stream/router/ip_port.lua
@@ -166,7 +166,7 @@ do
service_ver = cur_svc_ver
end
- local sni = apisix_ssl.server_name()
+ local sni = apisix_ssl.server_name(nil, api_ctx.tls_passthrough)
if sni and tls_router then
local sni_rev = sni:reverse()
diff --git a/apisix/upstream.lua b/apisix/upstream.lua
index 7ed61faf9..99e1e857d 100644
--- a/apisix/upstream.lua
+++ b/apisix/upstream.lua
@@ -328,6 +328,15 @@ end
function _M.set_by_route(route, api_ctx)
+ -- Ahead of the traffic-split short circuit below so its inline upstream is
+ -- covered too. A second handshake would send the client's ClientHello as
payload.
+ if api_ctx.tls_passthrough then
+ local passthrough_up = api_ctx.upstream_conf or
api_ctx.matched_upstream
+ if passthrough_up and passthrough_up.scheme == "tls" then
+ return 503, "upstream scheme `tls` can not be used on a
tls_passthrough listen"
+ end
+ end
+
if api_ctx.upstream_conf then
-- upstream_conf has been set by traffic-split plugin
return
diff --git a/conf/config.yaml.example b/conf/config.yaml.example
index 6f6c791ea..b201c97fb 100644
--- a/conf/config.yaml.example
+++ b/conf/config.yaml.example
@@ -95,6 +95,26 @@ apisix:
# proxy_protocol_to_upstream: true # Send the PROXY protocol to the
upstream on this
# # port only. Both override the
global
# # proxy_protocol.enable_tcp_pp*
defaults.
+ # - addr: 9500
+ # tls_passthrough: true # Forward the TLS stream untouched,
picking the upstream from
+ # # the SNI in the prereaded ClientHello.
The backend terminates
+ # # the handshake, so payload-inspecting
stream plugins
+ # # (mqtt-proxy, xrpc, redis) and client
certificate verification
+ # # do not apply. Every connection on this
port is passed
+ # # through, whatever the stream_route says.
+ # - addr: 9501
+ # tls: true # Both flags open a mixed port: the
listen only prereads, and
+ # tls_passthrough: true # each connection follows
`tls_passthrough` on the stream_route
+ # # it matches (default false). Costs one
internal hop, so prefer
+ # # a dedicated tls_passthrough port when
the whole port passes
+ # # through. That hop runs over unix
sockets under logs/, and
+ # # announces the client in a PROXY
protocol header, so a local
+ # # user able to connect to them could
forge it. Keep logs/ off
+ # # limits to untrusted local users, as it
already has to be for
+ # # stream_worker_events.sock.
+ # # proxy_protocol_to_upstream is rejected
on a mixed port: the
+ # # upstream PROXY header would carry the
internal unix socket
+ # # as its destination. Use a dedicated
listen instead.
# - "2000-2100" # port range (nginx native support)
# - addr: "3000-3100" # port range in table form
# - addr: "127.0.0.1:4000-4100" # address with port range in table form
diff --git a/docs/en/latest/stream-proxy.md b/docs/en/latest/stream-proxy.md
index b87b42ddd..e7152058d 100644
--- a/docs/en/latest/stream-proxy.md
+++ b/docs/en/latest/stream-proxy.md
@@ -242,6 +242,91 @@ By setting the `scheme` to `tls`, APISIX will do TLS
handshake with the upstream
When the client is also speaking TLS over TCP, the SNI from the client will
pass through to the upstream. Otherwise, a dummy SNI `apisix_backend` will be
used.
+## TLS passthrough
+
+The two sections above both terminate the client's TLS handshake at APISIX.
With `tls_passthrough`, APISIX instead forwards the encrypted stream to the
upstream untouched, and still picks the upstream from the SNI, which it reads
out of the prereaded `ClientHello` (`ssl_preread on`) rather than out of a
handshake it performed itself:
+
+```yaml
+apisix:
+ proxy_mode: http&stream
+ stream_proxy:
+ tcp:
+ - addr: 9100
+ tls_passthrough: true
+```
+
+The upstream terminates the handshake, so on such a port:
+
+- payload-inspecting stream plugins (`mqtt-proxy`, `xrpc`, `redis`) have
nothing to read, and gateway mTLS does not apply — client certificate
verification moves to the upstream;
+- an upstream with `"scheme": "tls"` is rejected with a `503`, because a
second handshake would send the client's `ClientHello` to the upstream as
payload.
+
+Routing is otherwise unchanged, so a stream route matches by SNI exactly as it
does for a terminating port:
+
+```shell
+curl http://127.0.0.1:9180/apisix/admin/stream_routes/1 -H "X-API-KEY:
$admin_key" -X PUT -d '
+{
+ "sni": "a.test.com",
+ "upstream": {
+ "nodes": {
+ "127.0.0.1:5991": 1
+ },
+ "type": "roundrobin"
+ }
+}'
+```
+
+### Deciding per route on a mixed port
+
+Setting both flags on one listen opens a **mixed** port, where each connection
is terminated or passed through according to `tls_passthrough` on the stream
route it matches (a boolean, `false` by default):
+
+```yaml
+apisix:
+ proxy_mode: http&stream
+ stream_proxy:
+ tcp:
+ - addr: 9100
+ tls: true
+ tls_passthrough: true
+```
+
+```shell
+# passed through to the upstream, which terminates the handshake
+curl http://127.0.0.1:9180/apisix/admin/stream_routes/1 -H "X-API-KEY:
$admin_key" -X PUT -d '
+{
+ "sni": "a.test.com",
+ "tls_passthrough": true,
+ "upstream": {
+ "nodes": {
+ "127.0.0.1:5991": 1
+ },
+ "type": "roundrobin"
+ }
+}'
+
+# terminated by APISIX, using the certificate configured for this SNI
+curl http://127.0.0.1:9180/apisix/admin/stream_routes/2 -H "X-API-KEY:
$admin_key" -X PUT -d '
+{
+ "sni": "b.test.com",
+ "upstream": {
+ "nodes": {
+ "127.0.0.1:5992": 1
+ },
+ "type": "roundrobin"
+ }
+}'
+```
+
+The route flag comes from etcd, so moving a service between the two modes
needs no configuration change or restart.
+
+A port itself cannot be both: `ssl_preread on` and `listen ... ssl` are
configuration-time directives, and on one server the handshake consumes the
`ClientHello` before the preread phase can read it. So the two behaviours live
in internal servers reachable over unix sockets under `logs/`, and the preread
phase of the public listen picks between them. Note that:
+
+- only mixed ports pay that extra hop; a `tls: true` or `tls_passthrough:
true` port keeps its direct path;
+- the client address crosses the hop in a PROXY protocol header restored with
`set_real_ip_from unix:`, so route matching, stream plugins and logs see the
real peer. A local user able to connect to those sockets could forge that
header, so keep `logs/` off limits to untrusted local users — as it already has
to be for `stream_worker_events.sock`;
+- `proxy_protocol_to_upstream` cannot be used on a mixed port and is rejected
at startup: nginx builds the upstream PROXY protocol header from the socket the
connection arrived on, which for the internal servers is the unix socket,
producing an invalid `PROXY TCP4 <client> unix:/... <port> 0`. Use a dedicated
`tls` or `tls_passthrough` listen, which has no internal hop and writes a
correct header;
+- `apisix_stream_metrics_zone` counts nginx sessions and has no per-server
switch, so a mixed port counts each client connection twice.
+
+Two `stream_proxy.tcp` entries on the same address must agree on their TLS
mode and on `proxy_protocol_to_upstream`; otherwise they would need different
nginx `server` blocks on one address, and APISIX rejects the configuration at
startup.
+
## PROXY protocol
APISIX can accept the [PROXY
protocol](https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt) on TCP
stream ports and forward it to the upstream.
diff --git a/docs/zh/latest/stream-proxy.md b/docs/zh/latest/stream-proxy.md
index 55636184d..bf1cb3ec8 100644
--- a/docs/zh/latest/stream-proxy.md
+++ b/docs/zh/latest/stream-proxy.md
@@ -233,6 +233,91 @@ curl http://127.0.0.1:9180/apisix/admin/stream_routes/1 -H
"X-API-KEY: $admin_ke
当客户端也使用基于 TCP 的 TLS 上游时，客户端发送的 SNI 将传递给上游。否则，将使用一个假的 SNI `apisix_backend`。
+## TLS 透传
+
+上面两节都会在 APISIX 上终止客户端的 TLS 握手。设置 `tls_passthrough` 后，APISIX
会把加密的流原封不动地转发给上游，同时仍然根据 SNI 选择上游——该 SNI 来自预读到的 `ClientHello`（`ssl_preread
on`），而不是 APISIX 自己完成的握手：
+
+```yaml
+apisix:
+ proxy_mode: http&stream
+ stream_proxy:
+ tcp:
+ - addr: 9100
+ tls_passthrough: true
+```
+
+握手由上游完成，因此在这样的端口上：
+
+- 需要检查负载的 stream 插件（`mqtt-proxy`、`xrpc`、`redis`）读不到任何内容，网关侧的 mTLS
也不再适用——客户端证书校验转移到上游；
+- `"scheme": "tls"` 的上游会被拒绝并返回 `503`，因为再握手一次会把客户端的 `ClientHello` 当作负载发给上游。
+
+路由行为没有变化，stream route 依然按 SNI 匹配，与终止 TLS 的端口一致：
+
+```shell
+curl http://127.0.0.1:9180/apisix/admin/stream_routes/1 -H "X-API-KEY:
$admin_key" -X PUT -d '
+{
+ "sni": "a.test.com",
+ "upstream": {
+ "nodes": {
+ "127.0.0.1:5991": 1
+ },
+ "type": "roundrobin"
+ }
+}'
+```
+
+### 在混合端口上按路由决定
+
+在同一个 listen 上同时设置两个开关会开启**混合**端口：每条连接是终止还是透传，取决于它匹配到的 stream route 上的
`tls_passthrough`（布尔值，默认为 `false`）：
+
+```yaml
+apisix:
+ proxy_mode: http&stream
+ stream_proxy:
+ tcp:
+ - addr: 9100
+ tls: true
+ tls_passthrough: true
+```
+
+```shell
+# 透传给上游，由上游完成握手
+curl http://127.0.0.1:9180/apisix/admin/stream_routes/1 -H "X-API-KEY:
$admin_key" -X PUT -d '
+{
+ "sni": "a.test.com",
+ "tls_passthrough": true,
+ "upstream": {
+ "nodes": {
+ "127.0.0.1:5991": 1
+ },
+ "type": "roundrobin"
+ }
+}'
+
+# 由 APISIX 终止，使用为该 SNI 配置的证书
+curl http://127.0.0.1:9180/apisix/admin/stream_routes/2 -H "X-API-KEY:
$admin_key" -X PUT -d '
+{
+ "sni": "b.test.com",
+ "upstream": {
+ "nodes": {
+ "127.0.0.1:5992": 1
+ },
+ "type": "roundrobin"
+ }
+}'
+```
+
+路由上的该字段来自 etcd，因此在两种模式之间切换某个服务无需修改配置或重启。
+
+端口本身无法同时是两者：`ssl_preread on` 与 `listen ... ssl` 都是配置期指令，在同一个 server
上握手会在预读阶段读到 `ClientHello` 之前就将其消耗掉。因此这两种行为被放到通过 `logs/` 下的 unix socket 访问的内部
server 中，由公开 listen 的预读阶段在两者之间做选择。需要注意：
+
+- 只有混合端口会付出这一跳的代价；`tls: true` 或 `tls_passthrough: true` 的端口仍走直连路径；
+- 客户端地址通过 PROXY 协议头跨过这一跳，并由 `set_real_ip_from unix:` 还原，因此路由匹配、stream
插件和日志看到的都是真实对端。能够连接这些 socket 的本地用户可以伪造该头部，因此要像对待 `stream_worker_events.sock`
一样，不要让不受信任的本地用户访问 `logs/`；
+- 混合端口上不能使用 `proxy_protocol_to_upstream`，启动时会被拒绝：nginx 依据连接进入时的 socket 生成发往上游的
PROXY 协议头，而内部 server 上那是一个 unix socket，生成的是 `PROXY TCP4 <client> unix:/...
<port> 0` 这种非法内容。请改用专用的 `tls` 或 `tls_passthrough` 监听，它们没有内部跳转，生成的头是正确的；
+- `apisix_stream_metrics_zone` 统计的是 nginx session 且没有 per-server
开关，因此混合端口上每条客户端连接会被统计两次。
+
+同一地址上的两个 `stream_proxy.tcp` 条目必须使用相同的 TLS 模式和相同的
`proxy_protocol_to_upstream`；否则它们需要在同一地址上使用不同的 nginx `server` 块，APISIX
会在启动时拒绝该配置。
+
## PROXY 协议
APISIX 可以在 TCP stream 端口上接收 [PROXY
协议](https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt)，并将其转发给上游。
diff --git a/t/APISIX.pm b/t/APISIX.pm
index 91b6ef133..76dc94530 100644
--- a/t/APISIX.pm
+++ b/t/APISIX.pm
@@ -353,6 +353,9 @@ _EOC_
if ($block->stream_sni) {
$sni = '"' . $block->stream_sni . '"';
}
+
+ # a bare `--- stream_tls_verify` section has an empty, false value
+ my $tls_verify = defined $block->stream_tls_verify ? "true" : "false";
chomp $stream_tls_request;
my $repeat = "1";
@@ -372,7 +375,7 @@ _EOC_
return
end
- sess, err = sock:sslhandshake(sess, $sni, false)
+ sess, err = sock:sslhandshake(sess, $sni, $tls_verify)
if not sess then
ngx.say("failed to do SSL handshake: ", err)
return
@@ -591,6 +594,10 @@ $stream_config
}
}
_EOC_
+ # the stream block lives in the main config here, so drop the block
values
+ # or Test::Nginx renders a second, conflicting one
+ $block->set_value("stream_config");
+ $block->set_value("stream_server_config");
}
$block->set_value("main_config", $main_config);
diff --git a/t/cli/test_stream_tls_passthrough.sh
b/t/cli/test_stream_tls_passthrough.sh
new file mode 100755
index 000000000..cc819b4ec
--- /dev/null
+++ b/t/cli/test_stream_tls_passthrough.sh
@@ -0,0 +1,449 @@
+#!/usr/bin/env bash
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+. ./t/cli/common.sh
+
+# What lives here and not in t/stream-node/tls-passthrough.t: the Test::Nginx
+# framework builds its own nginx.conf and never runs apisix/cli/ops.lua or
+# ngx_tpl.lua, so the half of this feature that *is* config generation -- how
+# listens are grouped by TLS mode, what a passthrough or mixed block renders --
+# is unreachable from a .t. The mixed-mode run at the end is here for the same
+# reason: its internal unix-socket servers are generated by the template, and
+# hand-writing them in a .t would test the copy rather than the template.
+#
+# Everything a .t can reach -- the preread SNI selecting an upstream, the
+# mixed-mode phase picking its target, the `scheme: tls` refusal -- is asserted
+# there instead.
+
+# Print the stream server{} block that contains "listen <port>", isolating
+# it by brace-counting from each "server {".
+block_with_listen() {
+ awk -v port="$1" '
+ /server[ \t]*\{/ { depth = 0; block = ""; inblock = 1 }
+ inblock {
+ block = block $0 "\n"
+ depth += gsub(/\{/, "{")
+ depth -= gsub(/\}/, "}")
+ if (depth == 0) {
+ if (block ~ ("listen " port "[ ;]")) { printf "%s", block }
+ inblock = 0
+ }
+ }
+ ' conf/nginx.conf
+}
+
+# Print the stream server{} block whose listen line matches <regex>.
+block_with_listen_re() {
+ awk -v re="$1" '
+ /server[ \t]*\{/ { depth = 0; block = ""; inblock = 1 }
+ inblock {
+ block = block $0 "\n"
+ depth += gsub(/\{/, "{")
+ depth -= gsub(/\}/, "}")
+ if (depth == 0) {
+ if (block ~ re) { printf "%s", block }
+ inblock = 0
+ }
+ }
+ ' conf/nginx.conf
+}
+
+# Handshake against <port> with <sni>, validating the served chain against
+# <ca>. Prints the openssl transcript. Retried by the caller: the stream route
+# has to propagate from etcd to the stream worker first.
+tls_probe() {
+ local port="$1"
+ local sni="$2"
+ local ca="$3"
+ echo -e 'mmm' | timeout 3 openssl s_client -connect 127.0.0.1:"$port" \
+ -servername "$sni" -CAfile "$ca" -ign_eof 2>/dev/null
+}
+
+# Retry tls_probe under a 20s deadline until one transcript matches every
+# pattern given after the CA. All of them have to hold for the same handshake:
+# the route reaches the stream workers one at a time, so a probe answered by a
+# worker that has already synced says nothing about the next connection.
+tls_probe_until() {
+ local port="$1" sni="$2" ca="$3"
+ shift 3
+ local deadline=$(( $(date +%s) + 20 ))
+ { set +x; } 2>/dev/null
+ while [ "$(date +%s)" -lt "$deadline" ]; do
+ local out
+ out=$(tls_probe "$port" "$sni" "$ca")
+ local matched=1
+ local pattern
+ for pattern in "$@"; do
+ if ! echo "$out" | grep -q "$pattern"; then
+ matched=0
+ break
+ fi
+ done
+ if [ "$matched" -eq 1 ]; then
+ set -x
+ return 0
+ fi
+ sleep 0.3
+ done
+ set -x
+ return 1
+}
+
+# === `tls` + `tls_passthrough` on one listen is mixed mode ===
+# The port itself can not be both, so it is opened for preread only and the two
+# real behaviours move into internal servers the preread phase picks between.
+echo '
+apisix:
+ proxy_mode: "http&stream"
+ stream_proxy:
+ tcp:
+ - addr: 9103
+ tls: true
+ tls_passthrough: true
+' > conf/config.yaml
+make init
+
+if block_with_listen 9103 | grep -E "listen 9103[ ;].*ssl" > /dev/null; then
+ echo "failed: a mixed listen must stay plain so ssl_preread can see the
ClientHello"
+ exit 1
+fi
+if ! block_with_listen 9103 | grep -E "ssl_preread on;" > /dev/null; then
+ echo "failed: a mixed listen must enable ssl_preread"
+ exit 1
+fi
+if ! block_with_listen 9103 | grep -E "proxy_pass \\\$stream_tls_target;" >
/dev/null; then
+ echo "failed: a mixed listen must route to the target the preread phase
picked"
+ exit 1
+fi
+if ! block_with_listen 9103 | grep -E "^\s*proxy_protocol on;" > /dev/null;
then
+ echo "failed: a mixed listen must announce the client across the internal
hop"
+ exit 1
+fi
+
+term_block=$(block_with_listen_re "listen unix:.*stls-t")
+pass_block=$(block_with_listen_re "listen unix:.*stls-p")
+if ! echo "$term_block" | grep -E "listen unix:.*stls-t.*ssl proxy_protocol;"
> /dev/null; then
+ echo "failed: the internal terminating server should accept ssl over PROXY
protocol"
+ exit 1
+fi
+if ! echo "$term_block" | grep -E "ssl_certificate_by_lua_block" > /dev/null;
then
+ echo "failed: the internal terminating server should run the certificate
phase"
+ exit 1
+fi
+if ! echo "$term_block" | grep -E "set_real_ip_from unix:;" > /dev/null; then
+ echo "failed: the internal terminating server should restore the client
address"
+ exit 1
+fi
+if ! echo "$pass_block" | grep -E "ssl_preread on;" > /dev/null; then
+ echo "failed: the internal passthrough server should preread the
ClientHello again"
+ exit 1
+fi
+if echo "$pass_block" | grep -E "ssl_certificate_by_lua_block" > /dev/null;
then
+ echo "failed: the internal passthrough server must not terminate anything"
+ exit 1
+fi
+if ! echo "$pass_block" | grep -E "set_real_ip_from unix:;" > /dev/null; then
+ echo "failed: the internal passthrough server should restore the client
address"
+ exit 1
+fi
+if ! block_with_listen 9103 | grep -E "^\s*access_log off;" > /dev/null; then
+ echo "failed: the mixed preread block should not log the connection a
second time"
+ exit 1
+fi
+echo "passed: tls + tls_passthrough renders a mixed-mode port"
+
+# === Two listens on one address that need different server blocks are
rejected ===
+# nginx refuses to start over this with reuseport on, and silently keeps only
the
+# first server with it off.
+echo '
+apisix:
+ proxy_mode: "http&stream"
+ stream_proxy:
+ tcp:
+ - 9100
+ - addr: 9100
+ tls_passthrough: true
+' > conf/config.yaml
+
+out=$(make init 2>&1 || true)
+if ! echo "$out" | grep -q "collides with an earlier entry"; then
+ echo "failed: the same address in two different server blocks should be
rejected"
+ exit 1
+fi
+
+# The same holds for a port range overlapping a single port.
+echo '
+apisix:
+ proxy_mode: "http&stream"
+ stream_proxy:
+ tcp:
+ - "9100-9110"
+ - addr: 9105
+ tls_passthrough: true
+' > conf/config.yaml
+
+out=$(make init 2>&1 || true)
+if ! echo "$out" | grep -q "collides with an earlier entry"; then
+ echo "failed: a range overlapping another group should be rejected"
+ exit 1
+fi
+
+# Same address, same group is untouched.
+echo '
+apisix:
+ proxy_mode: "http&stream"
+ stream_proxy:
+ tcp:
+ - addr: 9100
+ tls_passthrough: true
+ - addr: 9101
+ tls_passthrough: true
+' > conf/config.yaml
+make init
+echo "passed: colliding listens across server blocks are rejected"
+
+# === A passthrough listen renders without ssl, with ssl_preread on ===
+echo '
+apisix:
+ proxy_mode: "http&stream"
+ stream_proxy:
+ tcp:
+ - 9100
+ - addr: 9101
+ tls: true
+ - addr: 9102
+ tls_passthrough: true
+ udp:
+ - 9200
+' > conf/config.yaml
+make init
+
+if block_with_listen 9102 | grep -E "listen 9102[ ;].*ssl" > /dev/null; then
+ echo "failed: a passthrough listen must not carry the ssl flag"
+ exit 1
+fi
+if ! block_with_listen 9102 | grep -E "ssl_preread on;" > /dev/null; then
+ echo "failed: a passthrough listen must enable ssl_preread"
+ exit 1
+fi
+if ! block_with_listen 9102 | grep -E "apisix.stream_preread_phase\(true\)" >
/dev/null; then
+ echo "failed: a passthrough listen must tell the preread phase it is
passthrough"
+ exit 1
+fi
+if block_with_listen 9102 | grep -E
"ssl_certificate_by_lua_block|proxy_ssl_server_name" > /dev/null; then
+ echo "failed: a passthrough listen must not terminate or re-encrypt TLS"
+ exit 1
+fi
+echo "passed: passthrough listen renders ssl_preread without termination"
+
+# === Terminating and plain listens are untouched, and keep their own block ===
+if ! block_with_listen 9101 | grep -E "listen 9101[ ;].*ssl" > /dev/null; then
+ echo "failed: a tls listen should still terminate"
+ exit 1
+fi
+if ! block_with_listen 9101 | grep -E "ssl_certificate_by_lua_block" >
/dev/null; then
+ echo "failed: a tls listen should still run the certificate phase"
+ exit 1
+fi
+if block_with_listen 9101 | grep -E "ssl_preread on;" > /dev/null; then
+ echo "failed: ssl_preread must not leak into the terminating block"
+ exit 1
+fi
+if ! block_with_listen 9101 | grep -E "listen 9100[ ;]" > /dev/null; then
+ echo "failed: plain and tls listens should share one block"
+ exit 1
+fi
+if block_with_listen 9102 | grep -E "listen 9100[ ;]|listen 9200 udp" >
/dev/null; then
+ echo "failed: the passthrough listen needs a dedicated server block"
+ exit 1
+fi
+if block_with_listen 9102 | grep -E "proxy_pass \\\$stream_tls_target;" >
/dev/null; then
+ echo "failed: a pure passthrough listen should not pay for the mixed-mode
hop"
+ exit 1
+fi
+if grep -E "listen unix:.*stls-" conf/nginx.conf > /dev/null; then
+ echo "failed: internal sockets should only exist for mixed listens"
+ exit 1
+fi
+echo "passed: passthrough grouping leaves the other listens alone"
+
+# === A passthrough listen still honours proxy_protocol_to_upstream ===
+echo '
+apisix:
+ proxy_mode: "http&stream"
+ stream_proxy:
+ tcp:
+ - addr: 9102
+ tls_passthrough: true
+ proxy_protocol_to_upstream: true
+' > conf/config.yaml
+make init
+
+if ! block_with_listen 9102 | grep -E "ssl_preread on;" > /dev/null; then
+ echo "failed: passthrough + proxy_protocol_to_upstream should still
preread"
+ exit 1
+fi
+if ! block_with_listen 9102 | grep -E "^\s*proxy_protocol on;" > /dev/null;
then
+ echo "failed: passthrough + proxy_protocol_to_upstream should still send
PROXY"
+ exit 1
+fi
+echo "passed: passthrough combines with proxy_protocol_to_upstream"
+
+# === proxy_protocol_to_upstream is refused on a mixed listen ===
+# The mixed block reaches its internal servers over a unix socket, and nginx
+# builds the upstream PROXY header from that socket, producing an invalid
+# destination like `unix:/... 0`.
+echo '
+apisix:
+ proxy_mode: "http&stream"
+ stream_proxy:
+ tcp:
+ - addr: 9103
+ tls: true
+ tls_passthrough: true
+ proxy_protocol_to_upstream: true
+' > conf/config.yaml
+
+out=$(make init 2>&1 || true)
+if ! echo "$out" | grep -q "can not combine proxy_protocol_to_upstream"; then
+ echo "failed: proxy_protocol_to_upstream on a mixed listen should be
rejected"
+ exit 1
+fi
+
+# The same flag set globally must be refused too, not silently applied.
+echo '
+apisix:
+ proxy_mode: "http&stream"
+ proxy_protocol:
+ enable_tcp_pp_to_upstream: true
+ stream_proxy:
+ tcp:
+ - addr: 9103
+ tls: true
+ tls_passthrough: true
+' > conf/config.yaml
+
+out=$(make init 2>&1 || true)
+if ! echo "$out" | grep -q "can not combine proxy_protocol_to_upstream"; then
+ echo "failed: the global proxy protocol default should be refused on a
mixed listen too"
+ exit 1
+fi
+echo "passed: proxy_protocol_to_upstream is refused on a mixed listen"
+
+# === Mixed mode end to end: one port, the route decides ===
+# 9101 terminates its own TLS and is reached untouched; 9102 is plaintext and
is
+# reached only after the gateway terminates. Same port, same handshake target.
+# Both routes are pinned to server_port 9100: the internal servers see a unix
+# socket, so these only match if the router reads the address the PROXY header
+# carried rather than the socket's own.
+echo '
+apisix:
+ proxy_mode: "http&stream"
+ stream_proxy:
+ tcp:
+ - addr: 9100
+ tls: true
+ tls_passthrough: true
+nginx_config:
+ stream_configuration_snippet: |
+ server {
+ listen 9101 ssl;
+ ssl_certificate ../t/certs/mtls_server.crt;
+ ssl_certificate_key ../t/certs/mtls_server.key;
+ return "OK FROM PASSTHROUGH BACKEND";
+ }
+ server {
+ listen 9102;
+ return "OK FROM TERMINATED BACKEND";
+ }
+' > conf/config.yaml
+
+make run
+wait_for_tcp 127.0.0.1 9180
+wait_for_tcp 127.0.0.1 9100
+
+admin_key=$(yq '.deployment.admin.admin_key[0].key' conf/config.yaml | sed
's/"//g')
+curl -k -i http://127.0.0.1:9180/apisix/admin/ssls/1 \
+ -H "X-API-KEY: $admin_key" -X PUT -d '
+{
+ "cert": "'"$(cat t/certs/apisix.crt)"'",
+ "key": "'"$(cat t/certs/apisix.key)"'",
+ "snis": ["test.com"]
+}'
+
+curl -k -i http://127.0.0.1:9180/apisix/admin/stream_routes/1 \
+ -H "X-API-KEY: $admin_key" -X PUT -d \
+
'{"sni":"admin.apisix.dev","server_port":9100,"tls_passthrough":true,"upstream":{"nodes":{"127.0.0.1:9101":1},"type":"roundrobin"}}'
+# The whitelist is the assertion that the client address survived the internal
+# unix-socket hop: without set_real_ip_from the plugin would see `unix:`.
+curl -k -i http://127.0.0.1:9180/apisix/admin/stream_routes/2 \
+ -H "X-API-KEY: $admin_key" -X PUT -d \
+
'{"sni":"test.com","server_port":9100,"plugins":{"ip-restriction":{"whitelist":["127.0.0.1"]}},"upstream":{"nodes":{"127.0.0.1:9102":1},"type":"roundrobin"}}'
+
+if ! tls_probe_until 9100 admin.apisix.dev t/certs/mtls_ca.crt \
+ "OK FROM PASSTHROUGH BACKEND" \
+ "CN *= *admin.apisix.dev" \
+ "Verify return code: 0 (ok)"; then
+ echo "failed: a passthrough route on a mixed port should reach its backend
untouched"
+ exit 1
+fi
+echo "passed: mixed port passes through the route that asked for it"
+
+if ! tls_probe_until 9100 test.com t/certs/apisix.crt \
+ "OK FROM TERMINATED BACKEND" \
+ "CN *= *test.com" \
+ "Verify return code: 0 (ok)"; then
+ echo "failed: a terminating route on the same port should be served by the
gateway"
+ exit 1
+fi
+echo "passed: mixed port terminates the other route, with the client address
intact"
+
+# And the client address is really being checked, not merely tolerated.
+curl -k -i http://127.0.0.1:9180/apisix/admin/stream_routes/2 \
+ -H "X-API-KEY: $admin_key" -X PUT -d \
+
'{"sni":"test.com","plugins":{"ip-restriction":{"blacklist":["127.0.0.1"]}},"upstream":{"nodes":{"127.0.0.1:9102":1},"type":"roundrobin"}}'
+
+deadline=$(( $(date +%s) + 20 ))
+denied=0
+{ set +x; } 2>/dev/null
+while [ "$(date +%s)" -lt "$deadline" ]; do
+ if ! tls_probe 9100 test.com t/certs/apisix.crt | grep -q "OK FROM
TERMINATED BACKEND"; then
+ denied=1
+ break
+ fi
+ sleep 0.3
+done
+set -x
+if [ "$denied" -ne 1 ]; then
+ echo "failed: blacklisting the real client address should deny the
terminating route"
+ exit 1
+fi
+echo "passed: the internal hop reports the real client address, not the socket"
+
+# Drop what this test created before handing the etcd instance to the next one.
+# The blacklist route above matches sni test.com on port 9100, which is exactly
+# what t/cli/test_tls_over_tcp.sh then uses, so leaving it behind fails that
test.
+for res in stream_routes/1 stream_routes/2 ssls/1; do
+ curl -k -s -o /dev/null http://127.0.0.1:9180/apisix/admin/$res \
+ -H "X-API-KEY: $admin_key" -X DELETE
+done
+
+make stop
+
+echo "All stream TLS passthrough tests passed."
diff --git a/t/stream-node/tls-passthrough.t b/t/stream-node/tls-passthrough.t
new file mode 100644
index 000000000..f0d138e86
--- /dev/null
+++ b/t/stream-node/tls-passthrough.t
@@ -0,0 +1,302 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+use t::APISIX 'no_plan';
+
+log_level('info');
+no_root_location();
+worker_connections(1024);
+
+# 1997 and 1998 terminate TLS themselves, so a client that completes a
handshake
+# with either of them proves the bytes were forwarded rather than answered. The
+# two upstreams below are cross-mapped against the routes' own upstreams, so in
+# the mixed tests only the target the preread phase picked can explain the
answer.
+my $stream_backends = <<'_EOC_';
+ upstream apisix_test_terminate { server 127.0.0.1:1997; }
+ upstream apisix_test_passthrough { server 127.0.0.1:1998; }
+
+ server {
+ listen 1997 ssl;
+ # not cert/apisix.crt: the gateway holds no copy of this one
+ ssl_certificate ../../certs/mtls_server.crt;
+ ssl_certificate_key ../../certs/mtls_server.key;
+ content_by_lua_block {
+ ngx.say("hello from backend A")
+ }
+ }
+
+ server {
+ listen 1998 ssl;
+ ssl_certificate cert/apisix.crt;
+ ssl_certificate_key cert/apisix.key;
+ content_by_lua_block {
+ ngx.say("hello from backend B")
+ }
+ }
+_EOC_
+
+# A passthrough listen: no `ssl`, so it holds no certificate and could not
+# terminate anything even if it wanted to.
+my $passthrough_server = <<'_EOC_';
+ listen 2005;
+ ssl_preread on;
+
+ preread_by_lua_block {
+ ngx.sleep(0.1)
+ apisix.stream_preread_phase(true)
+ }
+
+ proxy_pass apisix_backend;
+_EOC_
+
+add_block_preprocessor(sub {
+ my ($block) = @_;
+
+ return unless defined $block->stream_tls_request;
+
+ if (!defined $block->extra_stream_config) {
+ $block->set_value("extra_stream_config", $stream_backends);
+ }
+
+ if (!defined $block->stream_server_config) {
+ $block->set_value("stream_server_config", $passthrough_server);
+ }
+});
+
+run_tests();
+
+__DATA__
+
+=== TEST 1: set two stream routes, matched by SNI
+--- config
+ location /t {
+ content_by_lua_block {
+ local t = require("lib.test_admin")
+
+ local code, body = t.test('/apisix/admin/stream_routes/1',
+ ngx.HTTP_PUT,
+ [[{
+ "sni": "admin.apisix.dev",
+ "upstream": {
+ "nodes": {"127.0.0.1:1997": 1},
+ "type": "roundrobin"
+ }
+ }]]
+ )
+ if code >= 300 then
+ ngx.status = code
+ return
+ end
+
+ local code, body = t.test('/apisix/admin/stream_routes/2',
+ ngx.HTTP_PUT,
+ [[{
+ "sni": "b.test.com",
+ "upstream": {
+ "nodes": {"127.0.0.1:1998": 1},
+ "type": "roundrobin"
+ }
+ }]]
+ )
+ if code >= 300 then
+ ngx.status = code
+ return
+ end
+
+ ngx.say(body)
+ }
+ }
+--- request
+GET /t
+--- response_body
+passed
+
+
+
+=== TEST 2: the prereaded SNI selects the upstream, which terminates the
handshake
+--- custom_trusted_cert: ../../certs/mtls_ca.crt
+--- stream_tls_request
+mmm
+--- stream_sni: admin.apisix.dev
+--- stream_tls_verify
+--- response_body
+hello from backend A
+--- error_log
+proxy request to 127.0.0.1:1997
+
+
+
+=== TEST 3: a second SNI on the same port picks the other upstream
+--- stream_tls_request
+mmm
+--- stream_sni: b.test.com
+--- response_body
+hello from backend B
+--- error_log
+proxy request to 127.0.0.1:1998
+
+
+
+=== TEST 4: a ClientHello with no SNI falls back instead of matching on ""
+--- yaml_config
+apisix:
+ node_listen: 1984
+ ssl:
+ fallback_sni: b.test.com
+--- stream_tls_request
+mmm
+--- response_body
+hello from backend B
+--- error_log
+proxy request to 127.0.0.1:1998
+
+
+
+=== TEST 5: set a passthrough route and a terminating route for the mixed
listen
+--- config
+ location /t {
+ content_by_lua_block {
+ local t = require("lib.test_admin")
+
+ local code, body = t.test('/apisix/admin/stream_routes/1',
+ ngx.HTTP_PUT,
+ [[{
+ "sni": "admin.apisix.dev",
+ "tls_passthrough": true,
+ "upstream": {
+ "nodes": {"127.0.0.1:1997": 1},
+ "type": "roundrobin"
+ }
+ }]]
+ )
+ if code >= 300 then
+ ngx.status = code
+ return
+ end
+
+ local code, body = t.test('/apisix/admin/stream_routes/2',
+ ngx.HTTP_PUT,
+ [[{
+ "sni": "b.test.com",
+ "upstream": {
+ "nodes": {"127.0.0.1:1998": 1},
+ "type": "roundrobin"
+ }
+ }]]
+ )
+ if code >= 300 then
+ ngx.status = code
+ return
+ end
+
+ ngx.say(body)
+ }
+ }
+--- request
+GET /t
+--- response_body
+passed
+
+
+
+=== TEST 6: a route asking for passthrough goes to the passthrough target
+--- stream_server_config
+ listen 2005;
+ ssl_preread on;
+
+ set $stream_tls_target "";
+
+ preread_by_lua_block {
+ ngx.sleep(0.1)
+ apisix.stream_tls_route_phase("apisix_test_terminate",
+ "apisix_test_passthrough")
+ }
+
+ proxy_pass $stream_tls_target;
+--- stream_tls_request
+mmm
+--- stream_sni: admin.apisix.dev
+--- response_body
+hello from backend B
+--- error_log
+target: apisix_test_passthrough
+
+
+
+=== TEST 7: a route that did not ask for it goes to the terminating target
+--- stream_server_config
+ listen 2005;
+ ssl_preread on;
+
+ set $stream_tls_target "";
+
+ preread_by_lua_block {
+ ngx.sleep(0.1)
+ apisix.stream_tls_route_phase("apisix_test_terminate",
+ "apisix_test_passthrough")
+ }
+
+ proxy_pass $stream_tls_target;
+--- stream_tls_request
+mmm
+--- stream_sni: b.test.com
+--- response_body
+hello from backend A
+--- error_log
+target: apisix_test_terminate
+
+
+
+=== TEST 8: set a route with a tls upstream
+--- config
+ location /t {
+ content_by_lua_block {
+ local t = require("lib.test_admin")
+
+ local code, body = t.test('/apisix/admin/stream_routes/1',
+ ngx.HTTP_PUT,
+ [[{
+ "sni": "admin.apisix.dev",
+ "upstream": {
+ "scheme": "tls",
+ "nodes": {"127.0.0.1:1997": 1},
+ "type": "roundrobin"
+ }
+ }]]
+ )
+ if code >= 300 then
+ ngx.status = code
+ return
+ end
+
+ ngx.say(body)
+ }
+ }
+--- request
+GET /t
+--- response_body
+passed
+
+
+
+=== TEST 9: a tls upstream on a passthrough listen is refused
+--- stream_tls_request
+mmm
+--- stream_sni: admin.apisix.dev
+--- response_body_like eval
+qr/failed to (do SSL handshake|receive)/
+--- error_log
+upstream scheme `tls` can not be used on a tls_passthrough listen
