details: http://freenginx.org/hg/nginx/rev/cb20978439c8
branches:
changeset: 9337:cb20978439c8
user: Maxim Dounin <mdou...@mdounin.ru>
date: Mon Apr 07 18:57:35 2025 +0300
description:
Support for Multipath TCP on Linux.
With this change, Multipath TCP on Linux can be activated with
"listen ... multipath". The "listen ... multipath" option is supported
in http, stream, and mail modules.
Note that on Linux to activate Multipath TCP one should create a socket
with the IPPROTO_MPTCP protocol explicitly specified, and it is not possible
to change an existing socket. To make transition possible with minimal
impact on client connections, if the multipath option is changed in the
configuration, SO_REUSEPORT is set on the old socket, and the new socket
is opened with IPPROTO_MPTCP.
Note that this creates a race window, and connection requests which are
assigned to the old socket will be lost. In particular, this might affect
binary upgrade when the WINCH signal is used to preserve the old master
process.
Requires Linux kernel 5.6 or newer. Note though that some of the socket
options might not be supported with Multipath TCP or only supported in
new kernels. Most notably, TCP_NODELAY is only supported with kernel
version 5.17 or newer.
Based on patches by Maxime Dourov and Anthony Doeraene.
diffstat:
auto/unix | 11 ++++++++
src/core/ngx_connection.c | 50 ++++++++++++++++++++++++++++++++++--
src/core/ngx_connection.h | 2 +
src/core/ngx_cycle.c | 6 ++++
src/http/ngx_http.c | 4 ++
src/http/ngx_http_core_module.c | 19 ++++++++++++++
src/http/ngx_http_core_module.h | 1 +
src/mail/ngx_mail.c | 4 ++
src/mail/ngx_mail.h | 1 +
src/mail/ngx_mail_core_module.c | 12 ++++++++
src/stream/ngx_stream.c | 4 ++
src/stream/ngx_stream.h | 1 +
src/stream/ngx_stream_core_module.c | 18 +++++++++++++
13 files changed, 129 insertions(+), 4 deletions(-)
diffs (326 lines):
diff --git a/auto/unix b/auto/unix
--- a/auto/unix
+++ b/auto/unix
@@ -532,6 +532,17 @@ ngx_feature_test="socklen_t optlen = siz
. auto/feature
+ngx_feature="IPPROTO_MPTCP"
+ngx_feature_name="NGX_HAVE_MULTIPATH"
+ngx_feature_run=no
+ngx_feature_incs="#include <sys/socket.h>
+ #include <netinet/in.h>"
+ngx_feature_path=
+ngx_feature_libs=
+ngx_feature_test="socket(0, 0, IPPROTO_MPTCP)"
+. auto/feature
+
+
ngx_feature="accept4()"
ngx_feature_name="NGX_HAVE_ACCEPT4"
ngx_feature_run=no
diff --git a/src/core/ngx_connection.c b/src/core/ngx_connection.c
--- a/src/core/ngx_connection.c
+++ b/src/core/ngx_connection.c
@@ -150,6 +150,9 @@ ngx_set_inherited_sockets(ngx_cycle_t *c
#if (NGX_HAVE_REUSEPORT)
int reuseport;
#endif
+#if (NGX_HAVE_MULTIPATH)
+ int protocol;
+#endif
ls = cycle->listening.elts;
for (i = 0; i < cycle->listening.nelts; i++) {
@@ -338,6 +341,25 @@ ngx_set_inherited_sockets(ngx_cycle_t *c
#endif
+#if (NGX_HAVE_MULTIPATH)
+
+ protocol = 0;
+ olen = sizeof(int);
+
+ if (getsockopt(ls[i].fd, SOL_SOCKET, SO_PROTOCOL,
+ (void *) &protocol, &olen)
+ == -1)
+ {
+ ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_socket_errno,
+ "getsockopt(SO_PROTOCOL) %V failed, ignored",
+ &ls[i].addr_text);
+
+ } else {
+ ls[i].multipath = (protocol == IPPROTO_MPTCP) ? 1 : 0;
+ }
+
+#endif
+
#if (NGX_HAVE_DEFERRED_ACCEPT && defined SO_ACCEPTFILTER)
ngx_memzero(&af, sizeof(struct accept_filter_arg));
@@ -406,7 +428,7 @@ ngx_set_inherited_sockets(ngx_cycle_t *c
ngx_int_t
ngx_open_listening_sockets(ngx_cycle_t *cycle)
{
- int reuseaddr;
+ int reuseaddr, proto;
ngx_uint_t i, tries, failed;
ngx_err_t err;
ngx_log_t *log;
@@ -436,7 +458,7 @@ ngx_open_listening_sockets(ngx_cycle_t *
#if (NGX_HAVE_REUSEPORT)
- if (ls[i].add_reuseport) {
+ if (ls[i].add_reuseport || ls[i].reopen) {
/*
* to allow transition from a socket without SO_REUSEPORT
@@ -472,6 +494,18 @@ ngx_open_listening_sockets(ngx_cycle_t *
ls[i].add_reuseport = 0;
}
+
+ if (ls[i].reopen) {
+
+ /*
+ * to allow transition to Multipath TCP we set SO_REUSEPORT
+ * on the old socket, and then open a new one
+ */
+
+ ls[i].fd = (ngx_socket_t) -1;
+ ls[i].inherited = 0;
+ ls[i].previous->remain = 0;
+ }
#endif
if (ls[i].fd != (ngx_socket_t) -1) {
@@ -487,7 +521,15 @@ ngx_open_listening_sockets(ngx_cycle_t *
continue;
}
- s = ngx_socket(ls[i].sockaddr->sa_family, ls[i].type, 0);
+ proto = 0;
+
+#if (NGX_HAVE_MULTIPATH)
+ if (ls[i].multipath) {
+ proto = IPPROTO_MPTCP;
+ }
+#endif
+
+ s = ngx_socket(ls[i].sockaddr->sa_family, ls[i].type, proto);
if (s == (ngx_socket_t) -1) {
ngx_log_error(NGX_LOG_EMERG, log, ngx_socket_errno,
@@ -517,7 +559,7 @@ ngx_open_listening_sockets(ngx_cycle_t *
#if (NGX_HAVE_REUSEPORT)
- if (ls[i].reuseport && !ngx_test_config) {
+ if ((ls[i].reuseport || ls[i].reopen) && !ngx_test_config) {
int reuseport;
reuseport = 1;
diff --git a/src/core/ngx_connection.h b/src/core/ngx_connection.h
--- a/src/core/ngx_connection.h
+++ b/src/core/ngx_connection.h
@@ -57,6 +57,7 @@ struct ngx_listening_s {
unsigned open:1;
unsigned remain:1;
unsigned ignore:1;
+ unsigned reopen:1;
unsigned bound:1; /* already bound */
unsigned inherited:1; /* inherited from previous process */
@@ -72,6 +73,7 @@ struct ngx_listening_s {
#endif
unsigned reuseport:1;
unsigned add_reuseport:1;
+ unsigned multipath:1;
unsigned keepalive:2;
unsigned quic:1;
diff --git a/src/core/ngx_cycle.c b/src/core/ngx_cycle.c
--- a/src/core/ngx_cycle.c
+++ b/src/core/ngx_cycle.c
@@ -581,6 +581,12 @@ ngx_init_cycle(ngx_cycle_t *old_cycle)
}
#endif
+#if (NGX_HAVE_MULTIPATH)
+ if (ls[i].multipath != nls[n].multipath) {
+ nls[n].reopen = 1;
+ }
+#endif
+
break;
}
}
diff --git a/src/http/ngx_http.c b/src/http/ngx_http.c
--- a/src/http/ngx_http.c
+++ b/src/http/ngx_http.c
@@ -1880,6 +1880,10 @@ ngx_http_add_listening(ngx_conf_t *cf, n
ls->reuseport = addr->opt.reuseport;
#endif
+#if (NGX_HAVE_MULTIPATH)
+ ls->multipath = addr->opt.multipath;
+#endif
+
ls->wildcard = addr->opt.wildcard;
#if (NGX_HTTP_V3)
diff --git a/src/http/ngx_http_core_module.c b/src/http/ngx_http_core_module.c
--- a/src/http/ngx_http_core_module.c
+++ b/src/http/ngx_http_core_module.c
@@ -4179,6 +4179,19 @@ ngx_http_core_listen(ngx_conf_t *cf, ngx
continue;
}
+ if (ngx_strcmp(value[n].data, "multipath") == 0) {
+#if (NGX_HAVE_MULTIPATH)
+ lsopt.multipath = 1;
+ lsopt.set = 1;
+ lsopt.bind = 1;
+#else
+ ngx_conf_log_error(NGX_LOG_EMERG, cf, 0,
+ "multipath is not supported "
+ "on this platform, ignored");
+#endif
+ continue;
+ }
+
if (ngx_strcmp(value[n].data, "ssl") == 0) {
#if (NGX_HTTP_SSL)
lsopt.ssl = 1;
@@ -4345,6 +4358,12 @@ ngx_http_core_listen(ngx_conf_t *cf, ngx
}
#endif
+#if (NGX_HAVE_MULTIPATH)
+ if (lsopt.multipath) {
+ return "\"multipath\" parameter is incompatible with \"quic\"";
+ }
+#endif
+
#if (NGX_HTTP_SSL)
if (lsopt.ssl) {
return "\"ssl\" parameter is incompatible with \"quic\"";
diff --git a/src/http/ngx_http_core_module.h b/src/http/ngx_http_core_module.h
--- a/src/http/ngx_http_core_module.h
+++ b/src/http/ngx_http_core_module.h
@@ -81,6 +81,7 @@ typedef struct {
#endif
unsigned deferred_accept:1;
unsigned reuseport:1;
+ unsigned multipath:1;
unsigned so_keepalive:2;
unsigned proxy_protocol:1;
diff --git a/src/mail/ngx_mail.c b/src/mail/ngx_mail.c
--- a/src/mail/ngx_mail.c
+++ b/src/mail/ngx_mail.c
@@ -347,6 +347,10 @@ ngx_mail_optimize_servers(ngx_conf_t *cf
ls->ipv6only = addr[i].opt.ipv6only;
#endif
+#if (NGX_HAVE_MULTIPATH)
+ ls->multipath = addr[i].opt.multipath;
+#endif
+
mport = ngx_palloc(cf->pool, sizeof(ngx_mail_port_t));
if (mport == NULL) {
return NGX_CONF_ERROR;
diff --git a/src/mail/ngx_mail.h b/src/mail/ngx_mail.h
--- a/src/mail/ngx_mail.h
+++ b/src/mail/ngx_mail.h
@@ -40,6 +40,7 @@ typedef struct {
#if (NGX_HAVE_INET6)
unsigned ipv6only:1;
#endif
+ unsigned multipath:1;
unsigned so_keepalive:2;
unsigned proxy_protocol:1;
#if (NGX_HAVE_KEEPALIVE_TUNABLE)
diff --git a/src/mail/ngx_mail_core_module.c b/src/mail/ngx_mail_core_module.c
--- a/src/mail/ngx_mail_core_module.c
+++ b/src/mail/ngx_mail_core_module.c
@@ -456,6 +456,18 @@ ngx_mail_core_listen(ngx_conf_t *cf, ngx
#endif
}
+ if (ngx_strcmp(value[i].data, "multipath") == 0) {
+#if (NGX_HAVE_MULTIPATH)
+ ls->multipath = 1;
+ ls->bind = 1;
+#else
+ ngx_conf_log_error(NGX_LOG_EMERG, cf, 0,
+ "multipath is not supported "
+ "on this platform, ignored");
+#endif
+ continue;
+ }
+
if (ngx_strcmp(value[i].data, "ssl") == 0) {
#if (NGX_MAIL_SSL)
ngx_mail_ssl_conf_t *sslcf;
diff --git a/src/stream/ngx_stream.c b/src/stream/ngx_stream.c
--- a/src/stream/ngx_stream.c
+++ b/src/stream/ngx_stream.c
@@ -518,6 +518,10 @@ ngx_stream_optimize_servers(ngx_conf_t *
ls->reuseport = addr[i].opt.reuseport;
#endif
+#if (NGX_HAVE_MULTIPATH)
+ ls->multipath = addr[i].opt.multipath;
+#endif
+
stport = ngx_palloc(cf->pool, sizeof(ngx_stream_port_t));
if (stport == NULL) {
return NGX_CONF_ERROR;
diff --git a/src/stream/ngx_stream.h b/src/stream/ngx_stream.h
--- a/src/stream/ngx_stream.h
+++ b/src/stream/ngx_stream.h
@@ -55,6 +55,7 @@ typedef struct {
unsigned ipv6only:1;
#endif
unsigned reuseport:1;
+ unsigned multipath:1;
unsigned so_keepalive:2;
unsigned proxy_protocol:1;
#if (NGX_HAVE_KEEPALIVE_TUNABLE)
diff --git a/src/stream/ngx_stream_core_module.c
b/src/stream/ngx_stream_core_module.c
--- a/src/stream/ngx_stream_core_module.c
+++ b/src/stream/ngx_stream_core_module.c
@@ -738,6 +738,18 @@ ngx_stream_core_listen(ngx_conf_t *cf, n
continue;
}
+ if (ngx_strcmp(value[i].data, "multipath") == 0) {
+#if (NGX_HAVE_MULTIPATH)
+ ls->multipath = 1;
+ ls->bind = 1;
+#else
+ ngx_conf_log_error(NGX_LOG_EMERG, cf, 0,
+ "multipath is not supported "
+ "on this platform, ignored");
+#endif
+ continue;
+ }
+
if (ngx_strcmp(value[i].data, "ssl") == 0) {
#if (NGX_STREAM_SSL)
ngx_stream_ssl_conf_t *sslcf;
@@ -884,6 +896,12 @@ ngx_stream_core_listen(ngx_conf_t *cf, n
return "\"fastopen\" parameter is incompatible with \"udp\"";
}
#endif
+
+#if (NGX_HAVE_MULTIPATH)
+ if (ls->multipath) {
+ return "\"multipath\" parameter is incompatible with \"udp\"";
+ }
+#endif
}
for (n = 0; n < u.naddrs; n++) {
