Hi,
I tested a simple configuration with connecting to a tls backend server
using kTLS.
haproxy.cfg
```
global
expose-experimental-directives
defaults
mode tcp
timeout connect 5s
timeout client 30s
timeout server 30s
frontend my_frontend
bind *:8080
use_backend my_backend
backend my_backend
server my_server 10.1.0.3:4043 ssl verify none ktls on force-tlsv12
```
I've noticed that when connecting to a backend server on loopback
(server my_server 127.0.0.1:4043 ...) HAProxy always enabled kTLS,
but when connecting to an external address it often failed to do so.
I did some digging and it turns out that after calling connect() on a
socket,
HAProxy immediately calls setsockopt(..., TCP_ULP, "tls", ...)
Linux tls module requires the socket to be in TCP_ESTABLISHED state:
https://elixir.bootlin.com/linux/v7.0.12/source/net/tls/tls_main.c#L1059-L1066
If a SYN+ACK packet from the server don't arrive in time when we call
setsockopt(), it will fail with ENOTCONN error.
I've tested on HAProxy 3.4.0 with OpenSSL 3.5.6 and AWS-LC 5.0.0
strace
```
3947 socket(AF_INET, SOCK_STREAM, IPPROTO_TCP) = 156
3947 fcntl(156, F_SETFL, O_RDONLY|O_NONBLOCK) = 0
3947 setsockopt(156, SOL_TCP, TCP_NODELAY, [1], 4) = 0
3947 connect(156, {sa_family=AF_INET, sin_port=htons(4043),
sin_addr=inet_addr("10.1.0.3")}, 16) = -1 EINPROGRESS (Operation now in
progress)
3947 setsockopt(156, SOL_TCP, TCP_ULP, [7564404], 4) = -1 ENOTCONN
(Transport endpoint is not connected)
3947 recvmsg(155, {msg_name=NULL, msg_namelen=0, msg_iov=[{iov_base="
make teardown-iface \342\200\224 prz"..., iov_len=1024}],
msg_iovlen=1, msg_controllen=0, msg_flags=0}, 0) = 1024
3947 epoll_ctl(148, EPOLL_CTL_ADD, 156,
{events=EPOLLIN|EPOLLOUT|EPOLLRDHUP, data={u32=156, u64=156}}) = 0
3947 clock_gettime(CLOCK_THREAD_CPUTIME_ID, {tv_sec=0,
tv_nsec=4370772}) = 0
3947 epoll_wait(148, [{events=EPOLLOUT, data={u32=156, u64=156}}], 200,
0) = 1
```
haproxy -vv (OpenSSL build):
```
HAProxy version 3.4.0-64a335366 2026/06/03 - https://haproxy.org/
Status: long-term supported branch - will stop receiving fixes around Q2
2031.
Known bugs: http://www.haproxy.org/bugs/bugs-3.4.0.html
Running on: Linux 6.8.0-107-generic #107-Ubuntu SMP PREEMPT_DYNAMIC Fri
Mar 13 19:51:50 UTC 2026 x86_64
Build options :
TARGET = linux-glibc
CC = cc
CFLAGS = -O2 -g -fwrapv -fvect-cost-model=very-cheap
OPTIONS = USE_OPENSSL=1 USE_KTLS=1
DEBUG =
Feature list : -51DEGREES +ACCEPT4 +ACME +BACKTRACE -CLOSEFROM
+CPU_AFFINITY +CRYPT_H -DEVICEATLAS +DL -ECH -ENGINE +EPOLL -EVPORTS
+GETADDRINFO +HAVE_TCP_MD5SIG -KQUEUE +KTLS -LIBATOMIC +LIBCRYPT
+LINUX_CAP +LINUX_SPLICE +LINUX_TPROXY -LUA -MATH -MEMORY_PROFILING
+NETFILTER +NS -OBSOLETE_LINKER +OPENSSL -OPENSSL_AWSLC -OPENSSL_WOLFSSL
-OT -PCRE -PCRE2 -PCRE2_JIT -PCRE_JIT +POLL +PRCTL -PROCCTL -PROMEX
-PTHREAD_EMULATION -QUIC -QUIC_OPENSSL_COMPAT +RT +SHM_OPEN +SLZ +SSL
-STATIC_PCRE -STATIC_PCRE2 +TFO +THREAD +THREAD_DUMP +TPROXY +TRACE
-WURFL -ZLIB
Detected feature list : +HAVE_WORKING_TCP_MD5SIG
Default settings :
bufsize = 16384, maxrewrite = 1024, maxpollevents = 200
Built with multi-threading support (MAX_TGROUPS=32, MAX_THREADS=1024,
default=48).
Built with SSL library version : OpenSSL 3.5.6 7 Apr 2026
Running on SSL library version : OpenSSL 3.5.6 7 Apr 2026
SSL library supports TLS extensions : yes
SSL library supports SNI : yes
SSL library default verify directory : /usr/lib/ssl/certs
SSL library supports : TLSv1.0 TLSv1.1 TLSv1.2 TLSv1.3
OpenSSL providers loaded : default
Built with network namespace support.
Built with libslz for stateless compression.
Compression algorithms supported : identity("identity"),
deflate("deflate"), raw-deflate("deflate"), gzip("gzip")
Built with transparent proxy support using: IP_TRANSPARENT
IPV6_TRANSPARENT IP_FREEBIND
Built without PCRE or PCRE2 support (using libc's regex instead)
Encrypted password support via crypt(3): yes
Built with gcc compiler version 13.3.0
Available polling systems :
epoll : pref=300, test result OK
poll : pref=200, test result OK
select : pref=150, test result OK
Total: 3 (3 usable), will use epoll.
Available multiplexer protocols :
(protocols marked as <default> cannot be specified using 'proto'
keyword)
h2 : mode=HTTP side=FE|BE mux=H2 flags=HTX|HOL_RISK|NO_UPG
<default> : mode=HTTP side=FE|BE mux=H1 flags=HTX
h1 : mode=HTTP side=FE|BE mux=H1 flags=HTX|NO_UPG
fcgi : mode=HTTP side=BE mux=FCGI flags=HTX|HOL_RISK|NO_UPG
<default> : mode=SPOP side=BE mux=SPOP flags=HOL_RISK|NO_UPG
spop : mode=SPOP side=BE mux=SPOP flags=HOL_RISK|NO_UPG
<default> : mode=TCP side=FE|BE mux=PASS flags=
none : mode=TCP side=FE|BE mux=PASS flags=NO_UPG
Available services : none
Available filters :
[BWLIM] bwlim-in
[BWLIM] bwlim-out
[CACHE] cache
[COMP] comp-req
[COMP] comp-res
[COMP] compression
[FCGI] fcgi-app
[SPOE] spoe
[TRACE] trace
```
haproxy -vv output (AWS-LC build):
```
HAProxy version 3.4.0-64a335366 2026/06/03 - https://haproxy.org/
Status: long-term supported branch - will stop receiving fixes around Q2
2031.
Known bugs: http://www.haproxy.org/bugs/bugs-3.4.0.html
Running on: Linux 6.8.0-107-generic #107-Ubuntu SMP PREEMPT_DYNAMIC Fri
Mar 13 19:51:50 UTC 2026 x86_64
Build options :
TARGET = linux-glibc
CC = cc
CFLAGS = -O2 -g -fwrapv -fvect-cost-model=very-cheap
OPTIONS = USE_OPENSSL_AWSLC=1 USE_KTLS=1
DEBUG =
Feature list : -51DEGREES +ACCEPT4 +ACME +BACKTRACE -CLOSEFROM
+CPU_AFFINITY +CRYPT_H -DEVICEATLAS +DL -ECH -ENGINE +EPOLL -EVPORTS
+GETADDRINFO +HAVE_TCP_MD5SIG -KQUEUE +KTLS -LIBATOMIC +LIBCRYPT
+LINUX_CAP +LINUX_SPLICE +LINUX_TPROXY -LUA -MATH -MEMORY_PROFILING
+NETFILTER +NS -OBSOLETE_LINKER +OPENSSL +OPENSSL_AWSLC -OPENSSL_WOLFSSL
-OT -PCRE -PCRE2 -PCRE2_JIT -PCRE_JIT +POLL +PRCTL -PROCCTL -PROMEX
-PTHREAD_EMULATION -QUIC -QUIC_OPENSSL_COMPAT +RT +SHM_OPEN +SLZ +SSL
-STATIC_PCRE -STATIC_PCRE2 +TFO +THREAD +THREAD_DUMP +TPROXY +TRACE
-WURFL -ZLIB
Detected feature list : +HAVE_WORKING_TCP_MD5SIG
Default settings :
bufsize = 16384, maxrewrite = 1024, maxpollevents = 200
Built with multi-threading support (MAX_TGROUPS=32, MAX_THREADS=1024,
default=48).
Built with SSL library version : OpenSSL 1.1.1 (compatible; AWS-LC
5.0.0)
Running on SSL library version : AWS-LC 5.0.0
SSL library supports TLS extensions : yes
SSL library supports SNI : yes
SSL library FIPS mode : no
SSL library default verify directory : /etc/ssl/certs
SSL library supports : TLSv1.0 TLSv1.1 TLSv1.2 TLSv1.3
Built with network namespace support.
Built with libslz for stateless compression.
Compression algorithms supported : identity("identity"),
deflate("deflate"), raw-deflate("deflate"), gzip("gzip")
Built with transparent proxy support using: IP_TRANSPARENT
IPV6_TRANSPARENT IP_FREEBIND
Built without PCRE or PCRE2 support (using libc's regex instead)
Encrypted password support via crypt(3): yes
Built with gcc compiler version 13.3.0
Available polling systems :
epoll : pref=300, test result OK
poll : pref=200, test result OK
select : pref=150, test result OK
Total: 3 (3 usable), will use epoll.
Available multiplexer protocols :
(protocols marked as <default> cannot be specified using 'proto'
keyword)
h2 : mode=HTTP side=FE|BE mux=H2 flags=HTX|HOL_RISK|NO_UPG
<default> : mode=HTTP side=FE|BE mux=H1 flags=HTX
h1 : mode=HTTP side=FE|BE mux=H1 flags=HTX|NO_UPG
fcgi : mode=HTTP side=BE mux=FCGI flags=HTX|HOL_RISK|NO_UPG
<default> : mode=SPOP side=BE mux=SPOP flags=HOL_RISK|NO_UPG
spop : mode=SPOP side=BE mux=SPOP flags=HOL_RISK|NO_UPG
<default> : mode=TCP side=FE|BE mux=PASS flags=
none : mode=TCP side=FE|BE mux=PASS flags=NO_UPG
Available services : none
Available filters :
[BWLIM] bwlim-in
[BWLIM] bwlim-out
[CACHE] cache
[COMP] comp-req
[COMP] comp-res
[COMP] compression
[FCGI] fcgi-app
[SPOE] spoe
[TRACE] trace
```
I've moved the setsockopt(..., TCP_ULP, "tls", ...) call just before
configuring the kTLS keys and it resolved the issue for me.
I've also tested the change on another system with kTLS disbled and
HAProxy
properly falls back to software tls.
From 5f1c0bf40ed9a8549ef7ae7127e39ee77ad0e2b8 Mon Sep 17 00:00:00 2001
From: Karol Kucharski <[email protected]>
Date: Wed, 10 Jun 2026 16:48:18 +0200
Subject: [PATCH] BUG: ktls: defer enabling TLS ULP on a socket until
just
before configuring kTLS keys
The Linux tls module requires a socket to be in TCP_ESTABLISHED state
before we can enable the TLS ULP on the socket, if the socket is in any
other state then setsockopt(..., TCP_ULP, "tls", ...) call will fail
with
ENOTCONN error.
We can enable kTLS on a socket only after the TCP handshake has
completed.
Signed-off-by: Karol Kucharski <[email protected]>
---
include/haproxy/ssl_sock-t.h | 1 +
src/ssl_sock.c | 34 ++++++++++++++++++++++++++--------
2 files changed, 27 insertions(+), 8 deletions(-)
diff --git a/include/haproxy/ssl_sock-t.h b/include/haproxy/ssl_sock-t.h
index 6af6dea9f..c40c55b33 100644
--- a/include/haproxy/ssl_sock-t.h
+++ b/include/haproxy/ssl_sock-t.h
@@ -255,6 +255,7 @@ struct ssl_keylog {
#define SSL_SOCK_F_KTLS_RECV (1 << 3) /* kTLS receive is
configure on that socket */
#define SSL_SOCK_F_CTRL_SEND (1 << 4) /* We want to send a
kTLS control message for that socket */
#define SSL_SOCK_F_HAS_ALPN (1 << 5) /* An ALPN has been
negotiated */
+#define SSL_SOCK_F_KTLS_ULP (1 << 6) /* TLS ULP is enabled
on that socket */
struct ssl_sock_ctx {
struct connection *conn;
diff --git a/src/ssl_sock.c b/src/ssl_sock.c
index 4c703fe5c..84066c20e 100644
--- a/src/ssl_sock.c
+++ b/src/ssl_sock.c
@@ -439,6 +439,20 @@ static int ha_ssl_read(BIO *h, char *buf, int size)
}
#ifdef HA_USE_KTLS
+/* Returns 0 on success, -1 on failure */
+static int ktls_enable_ulp(struct ssl_sock_ctx *ctx)
+{
+ int ret = 0;
+
+ if (!(ctx->flags & SSL_SOCK_F_KTLS_ULP)) {
+ ret = setsockopt(ctx->conn->handle.fd, SOL_TCP, TCP_ULP, "tls",
sizeof("tls"));
+ if (ret == 0)
+ ctx->flags |= SSL_SOCK_F_KTLS_ULP;
+ }
+
+ return ret;
+}
+
/* Returns 0 on success, -1 on failure */
static int ktls_set_key(struct ssl_sock_ctx *ctx, void *info, size_t
info_len, int is_tx)
{
@@ -485,6 +499,12 @@ static long ha_ssl_ctrl(BIO *h, int cmd, long arg1,
void *arg2)
if (!(ctx->flags & SSL_SOCK_F_KTLS_ENABLED))
return 0;
+
+ if (ktls_enable_ulp(ctx) != 0) {
+ ctx->flags &= ~SSL_SOCK_F_KTLS_ENABLED;
+ return 0;
+ }
+
/*
* As OpenSSL doesn't export struct tls_crypto_info_all,
* and it puts the size at the end of the struct,
@@ -5670,14 +5690,7 @@ static int ssl_sock_start(struct connection
*conn, void *xprt_ctx)
if (ret < 0)
return ret;
}
-#ifdef HA_USE_KTLS
- /*
- * Make the socket usable for kTLS. That does not mean that we will
- * use kTLS, though, just that the socket will be able to do it.
- */
- if ((ctx->flags & SSL_SOCK_F_KTLS_ENABLED) &&
setsockopt(conn->handle.fd, SOL_TCP, TCP_ULP, "tls", sizeof("tls")) !=
0)
- ctx->flags &= ~SSL_SOCK_F_KTLS_ENABLED;
-#endif
+
tasklet_wakeup(ctx->wait_event.tasklet);
return 0;
@@ -6617,6 +6630,11 @@ static void ssl_sock_setup_ktls(struct
ssl_sock_ctx *ctx)
if (!(ctx->flags & SSL_SOCK_F_KTLS_ENABLED))
return;
+ if (ktls_enable_ulp(ctx) != 0) {
+ ctx->flags &= ~SSL_SOCK_F_KTLS_ENABLED;
+ return;
+ }
+
switch (SSL_version(ctx->ssl)) {
case TLS_1_2_VERSION:
is_tls_12 = 1;
--
2.47.3
