Hi Christopher, Willy,
I've created a patch that can be applied on top of your tcp-fallback
patch to allow for 'conditional' offloading.
It shows 'ability' to have both offloading and pass-trough for ssl
depending on a sni name or other acl criteria.
-I resorted rather heavily to changing flags and statuses to make it
step back to handshake negotiation. And although it seems to work, it
should be checked if any of this will possibly cause hangs / memory
leaks / other trouble..
-Task timer added to fix excessive looping while waiting for
tcp-fallback to happen in case of a smtp connection.
-'peek'-ing for filling the data buffer so the FD stream keeps the
handshake data it had before for the ssl object to read if the
ssl_upgrade() is performed later. I couldnt find how to fill the ssl
object otherwise.. I think SSL_set_rbio() would be the solution there,
but that isnt available in current openssl releases.
I hope you guys have the time and ability to check for problems
introduced by my 'hackery'.. Or just throw it in the bin and
re-implement it using a better understanding of how the flow is supposed
to go.
Any feedback is appreciated :)
Regards,
PiBa-NL
Op 9-2-2016 om 11:53 schreef Christopher Faulet:
Le 09/02/2016 09:04, Willy Tarreau a écrit :
thanks for this. It looks clean enough to be merged.
I'm a little bit concerned with the addition of conn->ssl_detection_exp
because we try to keep the connection struct as small as possible. But
in this case there's no other place to store it. Thus I would change it
to "exp" and ensure it can be used by whatever requires an expiration in
the connection. That may be used in the future to wait for the proxy
protocol header or for outgoing connection timeout for example.
That also makes me wonder how we currently deal with a timeout during
the
SSL handshake, I'll have to check this. Maybe something is redundant, I
don't remember.
There's something interesting in your approach. When we started to
implement
SSL, I wondered if we could have a "tcp-request connection" rule such as
"expect-ssl" or something like this so that we could decide whether
or not
we'll use SSL based on some ACLs. One of the difficulties I didn't solve
was the ability to specify the SSL parameters in the rule.
With your model we could easily imagine having a bind option to
disable the
SSL upgrade by default, and only enable it once the correct rule has
been
matched in tcp-request connection.
Hi Willy,
Thanks !
About the fallback timeout, maybe I missed something, but I didn't
found any parameter to set a timeout on the SSL handshake.
By the way, good point about the renaming of 'ssl_detection_exp' into
'exp'. It is always good to take a step back and think a minute on
possible extension :)
From 95d6069b544b71204cd8d16ba2eb1b61794adb04 Mon Sep 17 00:00:00 2001
From: Pieter Baauw <[email protected]>
Date: Sun, 28 Feb 2016 22:25:42 +0100
Subject: [PATCH] MAJOR: SSL, add option "tcp-request content offloadssl" to
allow for conditional offloading of ssl traffic
This allows for using the following configuration:
frontend sslupgrade
mode tcp
bind :1234 ssl crt /root/FEvhost1_default.pem tcp-fallback 5s
acl traffic_is_ssl req_ssl_ver gt 0
acl clienthello req_ssl_hello_type 1
acl ishttp req_proto_http
tcp-request inspect-delay 5s
tcp-request content offloadssl if { req_ssl_sni -i ssl.dev.tld }
tcp-request content accept if traffic_is_ssl
tcp-request content accept if HTTP
use_backend backend-ssl if clienthello
use_backend backend-offload if { ssl_fc }
use_backend backend-http if HTTP
default_backend backend-mail
---
include/proto/ssl_sock.h | 3 +-
include/types/action.h | 1 +
include/types/connection.h | 9 +-
src/proto_tcp.c | 18 +++
src/raw_sock.c | 20 ++-
src/session.c | 15 ++-
src/ssl_sock.c | 301 +++++++++++++++++++++++++++++----------------
7 files changed, 249 insertions(+), 118 deletions(-)
diff --git a/include/proto/ssl_sock.h b/include/proto/ssl_sock.h
index 45d3aeb..fa0c9ce 100644
--- a/include/proto/ssl_sock.h
+++ b/include/proto/ssl_sock.h
@@ -43,7 +43,8 @@ int ssl_sock_is_ssl(struct connection *conn)
}
int ssl_sock_handshake(struct connection *conn, unsigned int flag);
-int ssl_sock_detection(struct connection *conn, int flag);
+int ssl_sock_detection(struct connection *conn, int flag);
+int ssl_upgrade(struct stream *s, struct channel *req);
int ssl_sock_prepare_ctx(struct bind_conf *bind_conf, SSL_CTX *ctx, struct
proxy *proxy);
int ssl_sock_prepare_all_ctx(struct bind_conf *bind_conf, struct proxy *px);
int ssl_sock_prepare_srv_ctx(struct server *srv, struct proxy *px);
diff --git a/include/types/action.h b/include/types/action.h
index b97f9bf..0c0b082 100644
--- a/include/types/action.h
+++ b/include/types/action.h
@@ -86,6 +86,7 @@ enum act_name {
ACT_TCP_EXPECT_PX,
ACT_TCP_CLOSE, /* close at the sender's */
ACT_TCP_CAPTURE, /* capture a fetched sample */
+ ACT_TCP_OFFLOADSSL, /* conditional SSL offloading */
/* track stick counters */
ACT_ACTION_TRK_SC0,
diff --git a/include/types/connection.h b/include/types/connection.h
index 702e7f5..0a7c3d7 100644
--- a/include/types/connection.h
+++ b/include/types/connection.h
@@ -107,10 +107,10 @@ enum {
CO_FL_SEND_PROXY = 0x01000000, /* send a valid PROXY protocol
header */
CO_FL_SSL_WAIT_HS = 0x02000000, /* wait for an SSL handshake to
complete */
CO_FL_ACCEPT_PROXY = 0x04000000, /* receive a valid PROXY protocol
header */
- CO_FL_SSL_DETECTION = 0x08000000, /* try to detect ssl connection */
+ CO_FL_SSL_DETECTION = 0x08000000, /* try to detect ssl connection */
/* below we have all handshake flags grouped into one */
- CO_FL_HANDSHAKE = CO_FL_SEND_PROXY | CO_FL_SSL_WAIT_HS |
CO_FL_ACCEPT_PROXY | CO_FL_SSL_DETECTION,
+ CO_FL_HANDSHAKE = CO_FL_SEND_PROXY | CO_FL_SSL_WAIT_HS |
CO_FL_ACCEPT_PROXY | CO_FL_SSL_DETECTION,
/* when any of these flags is set, polling is defined by socket-layer
* operations, as opposed to data-layer. Transport is explicitly not
@@ -257,13 +257,16 @@ struct connection {
void *xprt_ctx; /* general purpose pointer, initialized
to NULL */
void *owner; /* pointer to upper layer's entity (eg:
stream interface) */
int xprt_st; /* transport layer state, initialized to
zero */
- int ssl_detection_exp; /* expiration date for the SSL detection
*/
+ int ssl_detection_exp; /* expiration date for the SSL detection
*/
union { /* definitions which depend on connection
type */
struct { /*** information used by socket-based
connections ***/
int fd; /* file descriptor for a stream driver
when known */
} sock;
} t;
+ int peekstep;
+ int peekcount;
+ void *ssldetecttask;
enum obj_type *target; /* the target to connect to (server,
proxy, applet, ...) */
struct list list; /* attach point to various connection
lists (idle, ...) */
const struct netns_entry *proxy_netns;
diff --git a/src/proto_tcp.c b/src/proto_tcp.c
index cce0acb..3fc2ddf 100644
--- a/src/proto_tcp.c
+++ b/src/proto_tcp.c
@@ -1123,6 +1123,16 @@ resume_execution:
s->flags |= SF_FINST_R;
return 0;
}
+ else if (rule->action == ACT_TCP_OFFLOADSSL) {
+ int fd = (unsigned
short)objt_conn(s->sess->origin)->t.sock.fd;
+ struct connection *conn = fdtab[fd].owner;
+ if (conn) {
+ if (ssl_upgrade(s, req)) {
+ goto missing_data;
+ }
+ continue;
+ }
+ }
else if (rule->action >= ACT_ACTION_TRK_SC0 &&
rule->action <= ACT_ACTION_TRK_SCMAX) {
/* Note: only the first valid tracking
parameter of each
* applies.
@@ -1512,6 +1522,10 @@ static int tcp_parse_response_rule(char **args, int arg,
int section_type,
arg++;
rule->action = ACT_ACTION_DENY;
}
+ else if (strcmp(args[arg], "offloadssl") == 0) {
+ arg++;
+ rule->action = ACT_TCP_OFFLOADSSL;
+ }
else if (strcmp(args[arg], "close") == 0) {
arg++;
rule->action = ACT_TCP_CLOSE;
@@ -1573,6 +1587,10 @@ static int tcp_parse_request_rule(char **args, int arg,
int section_type,
arg++;
rule->action = ACT_ACTION_DENY;
}
+ else if (!strcmp(args[arg], "offloadssl")) {
+ arg++;
+ rule->action = ACT_TCP_OFFLOADSSL;
+ }
else if (strcmp(args[arg], "capture") == 0) {
struct sample_expr *expr;
struct cap_hdr *hdr;
diff --git a/src/raw_sock.c b/src/raw_sock.c
index c093377..b872c8b 100644
--- a/src/raw_sock.c
+++ b/src/raw_sock.c
@@ -287,8 +287,24 @@ static int raw_sock_to_buf(struct connection *conn, struct
buffer *buf, int coun
}
if (try > count)
try = count;
-
- ret = recv(conn->t.sock.fd, bi_end(buf), try, 0);
+
+ if (conn->peekstep <= -2){
+ if (conn->peekcount > 0) {
+ // first read the previously peeked data from
the FD..
+ recv(conn->t.sock.fd, bi_end(buf),
conn->peekcount, 0);
+ conn->peekcount = 0;
+ }
+ ret = recv(conn->t.sock.fd, bi_end(buf), try, 0);
+ } else {
+ ret = recv(conn->t.sock.fd, bi_end(buf), try, MSG_PEEK);
+ if (conn->peekcount == ret) {
+ conn->peekstep = -2;
+ ret = -1;
+ errno = EAGAIN;
+ } else {
+ conn->peekcount = ret;
+ }
+ }
if (ret > 0) {
buf->i += ret;
diff --git a/src/session.c b/src/session.c
index 24b70f1..16a96da 100644
--- a/src/session.c
+++ b/src/session.c
@@ -133,15 +133,18 @@ int session_accept_fd(struct listener *l, int cfd, struct
sockaddr_storage *addr
cli_conn->flags |= CO_FL_ADDR_FROM_SET;
cli_conn->target = &l->obj_type;
cli_conn->proxy_netns = l->netns;
+ cli_conn->peekstep = 0;
+ cli_conn->peekcount = 0;
+ cli_conn->ssldetecttask = NULL;
conn_ctrl_init(cli_conn);
- /* try to detect ssl connection */
- if (l->options & LI_O_SSL_DETECTION) {
- cli_conn->flags |= CO_FL_SSL_DETECTION;
- conn_sock_want_recv(cli_conn);
- }
-
+ /* try to detect ssl connection */
+ if (l->options & LI_O_SSL_DETECTION) {
+ cli_conn->flags |= CO_FL_SSL_DETECTION;
+ conn_sock_want_recv(cli_conn);
+ }
+
/* wait for a PROXY protocol header */
if (l->options & LI_O_ACC_PROXY) {
cli_conn->flags |= CO_FL_ACCEPT_PROXY;
diff --git a/src/ssl_sock.c b/src/ssl_sock.c
index 79657dd..f33f9ba 100644
--- a/src/ssl_sock.c
+++ b/src/ssl_sock.c
@@ -83,12 +83,14 @@
#include <proto/server.h>
#include <proto/log.h>
#include <proto/proxy.h>
-#include <proto/raw_sock.h>
+#include <proto/raw_sock.h>
#include <proto/shctx.h>
#include <proto/ssl_sock.h>
#include <proto/stream.h>
#include <proto/task.h>
+#include "proto/proto_http.h"
+
/* Warning, these are bits, not integers! */
#define SSL_SOCK_ST_FL_VERIFY_DONE 0x00000001
#define SSL_SOCK_ST_FL_16K_WBFSIZE 0x00000002
@@ -3547,87 +3549,175 @@ reneg_ok:
return 0;
}
-
-int ssl_sock_detection(struct connection *conn, int flag)
-{
- struct bind_conf *bind;
-
- /* we might have been called just after an asynchronous shutr */
- if (conn->flags & CO_FL_SOCK_RD_SH)
- goto fail;
- if (!conn_ctrl_ready(conn))
- goto fail;
- if (ssl_sock_is_ssl(conn))
- goto done;
-
- conn->flags |= CO_FL_WAIT_L6_CONN;
- if (!fd_recv_ready(conn->t.sock.fd))
- goto missing_data;
- trash.len = recv(conn->t.sock.fd, trash.str, trash.size, MSG_PEEK);
- if (trash.len < 0) {
- if (errno == EAGAIN)
- goto missing_data;
- goto sock_error;
- }
- conn->flags &= ~CO_FL_WAIT_L6_CONN;
-
- if (trash.len) {
- unsigned char *b = (unsigned char *)trash.str;
- if ((b[0] & 0x80) == 0x80) {
- /* try to match SSLv2 */
- if (trash.len >= 3 && (((b[0] & 0x7F) << 8 | b[1]) > 9)
&& b[2] == 0x01)
- goto ssl_detected;
- }
- else if (b[0] == 0x16) {
- /* try to match SSLv3, TLSv1.0, TLSv1.1 or TLSv1.2 */
- if (trash.len >= 6 && b[1] == 0x03 && b[5] == 0x01)
- goto ssl_detected;
- }
- }
- goto done; /* TCP fallback */
-
- ssl_detected:
- conn->xprt = &ssl_sock;
- conn->flags &= ~CO_FL_XPRT_READY;
- conn_data_want_recv(conn);
- if (conn_xprt_init(conn) < 0)
- goto sock_error;
-
- if (!(objt_listener(conn->target)->options & LI_O_UNLIMITED)) {
- update_freq_ctr(&global.ssl_per_sec, 1);
- if (global.ssl_per_sec.curr_ctr > global.ssl_max)
- global.ssl_max = global.ssl_per_sec.curr_ctr;
- }
-
- done:
- conn->flags &= ~flag;
- return 1;
-
- missing_data:
- bind = objt_listener(conn->target)->bind_conf;
- if (bind->ssl_detection_delay) {
- if (!conn->ssl_detection_exp)
- conn->ssl_detection_exp =
- tick_add(now_ms, bind->ssl_detection_delay);
- else if (tick_is_expired(conn->ssl_detection_exp, now_ms)) {
- conn->flags &= ~CO_FL_WAIT_L6_CONN;
- goto done; /* TCP fallback */
- }
- }
- fd_may_recv(conn->t.sock.fd);
- return 0;
-
- sock_error:
- conn->err_code = CO_ER_SOCK_ERR;
- conn->flags |= CO_FL_SOCK_RD_SH | CO_FL_SOCK_WR_SH;
-
- fail:
- __conn_sock_stop_both(conn);
- conn->flags |= CO_FL_ERROR;
- conn->flags &= ~flag;
- return 0;
-}
-
+static struct task *ssl_sock_detection_expire(struct task *t)
+{
+ struct connection *conn = t->context;
+ conn->ssldetecttask = NULL;
+
+ conn->flags &= ~CO_FL_WAIT_L6_CONN;// TCP fallback..
+ conn->flags &= ~CO_FL_SSL_DETECTION;
+
+ task_delete(t);
+ task_free(t);
+ fd_may_recv(conn->t.sock.fd); // kick its butt.?
+ return NULL;
+}
+
+int ssl_sock_detection(struct connection *conn, int flag)
+{
+ struct bind_conf *bind;
+
+ /* we might have been called just after an asynchronous shutr */
+ if (conn->flags & CO_FL_SOCK_RD_SH)
+ goto fail;
+ if (!conn_ctrl_ready(conn))
+ goto fail;
+ if (ssl_sock_is_ssl(conn))
+ goto done;
+
+ conn->flags |= CO_FL_WAIT_L6_CONN;
+ if (!fd_recv_ready(conn->t.sock.fd))
+ goto missing_data;
+ trash.len = recv(conn->t.sock.fd, trash.str, trash.size, MSG_PEEK);
+ if (trash.len < 0) {
+ if (errno == EAGAIN)
+ goto missing_data;
+ goto sock_error;
+ }
+ conn->flags &= ~CO_FL_WAIT_L6_CONN;
+
+ if (trash.len) {
+ unsigned char *b = (unsigned char *)trash.str;
+ if ((b[0] & 0x80) == 0x80) {
+ /* try to match SSLv2 */
+ if (trash.len >= 3 && (((b[0] & 0x7F) << 8 | b[1]) > 9)
&& b[2] == 0x01) {
+ if (conn->peekstep <= -2)
+ goto ssl_detected;
+ else
+ goto done;
+ }
+ }
+ else if (b[0] == 0x16) {
+ /* try to match SSLv3, TLSv1.0, TLSv1.1 or TLSv1.2 */
+ if (trash.len >= 6 && b[1] == 0x03 && b[5] == 0x01){
+ if (conn->peekstep <= -2)
+ goto ssl_detected;
+ else
+ goto done;
+ }
+ }
+ }
+ goto done; /* TCP fallback */
+
+ ssl_detected:
+ if (conn->ssldetecttask) {
+ struct task *t = conn->ssldetecttask;
+ task_delete(t);
+ task_free(t);
+ conn->ssldetecttask = NULL;
+ }
+
+ conn->xprt = &ssl_sock;
+ conn->flags &= ~CO_FL_XPRT_READY;
+ conn_data_want_recv(conn);
+ if (conn_xprt_init(conn) < 0)
+ goto sock_error;
+
+ if (!(objt_listener(conn->target)->options & LI_O_UNLIMITED)) {
+ update_freq_ctr(&global.ssl_per_sec, 1);
+ if (global.ssl_per_sec.curr_ctr > global.ssl_max)
+ global.ssl_max = global.ssl_per_sec.curr_ctr;
+ }
+
+ done:
+ conn->flags &= ~flag;
+ return 1;
+
+ missing_data:
+ bind = objt_listener(conn->target)->bind_conf;
+ if (bind->ssl_detection_delay) {
+ if (!conn->ssl_detection_exp)
+ conn->ssl_detection_exp =
+ tick_add(now_ms, bind->ssl_detection_delay);
+ else if (tick_is_expired(conn->ssl_detection_exp, now_ms)) {
+ conn->flags &= ~CO_FL_WAIT_L6_CONN;
+ goto done; /* TCP fallback */
+ }
+ }
+ if (conn->flags & CO_FL_WAIT_L4_CONN)
+ conn->flags &= ~CO_FL_WAIT_L4_CONN;
+ __conn_sock_stop_send(conn);
+ __conn_sock_want_recv(conn);
+ fd_cant_recv(conn->t.sock.fd);
+
+ if (!conn->ssldetecttask) {
+ struct task *t;
+ if (unlikely((t = task_new()) == NULL))
+ goto fail;
+
+ conn->ssldetecttask = t;
+ t->context = conn;
+ t->process = ssl_sock_detection_expire;
+ t->expire = tick_add_ifset(now_ms, bind->ssl_detection_delay);
+ task_queue(t);
+ }
+ return 0;
+
+ sock_error:
+ conn->err_code = CO_ER_SOCK_ERR;
+ conn->flags |= CO_FL_SOCK_RD_SH | CO_FL_SOCK_WR_SH;
+
+ fail:
+ __conn_sock_stop_both(conn);
+ conn->flags |= CO_FL_ERROR;
+ conn->flags &= ~flag;
+ return 0;
+}
+
+int ssl_upgrade(struct stream *s, struct channel *req)
+{
+ int fd;
+ if (req->buf->i) {
+ unsigned char *b = (unsigned char *)req->buf->p;
+ if ((b[0] & 0x80) == 0x80) {
+ /* try to match SSLv2 */
+ if (req->buf->i >= 3 && (((b[0] & 0x7F) << 8 | b[1]) >
9) && b[2] == 0x01)
+ goto ssl_detected;
+ }
+ else if (b[0] == 0x16) {
+ /* try to match SSLv3, TLSv1.0, TLSv1.1 or TLSv1.2 */
+ if (req->buf->i >= 6 && b[1] == 0x03 && b[5] == 0x01){
+ goto ssl_detected;
+ }
+ }
+ }
+ goto sock_error;
+
+ ssl_detected:
+ fd = (unsigned short)objt_conn(s->sess->origin)->t.sock.fd;
+ struct connection *conn = fdtab[fd].owner;
+
+ conn->flags |= CO_FL_SSL_WAIT_HS | CO_FL_WAIT_L6_CONN |
CO_FL_SSL_DETECTION;
+ conn->flags |= CO_FL_SOCK_RD_ENA;
+ conn->flags &= ~CO_FL_CONNECTED;
+
+ fd_stop_recv(fd);
+ fd_may_recv(fd);
+ fd_want_recv(fd);
+
+ /* remove the events before leaving */
+ fdtab[fd].ev &= FD_POLL_STICKY;
+ /* commit polling changes */
+ conn_cond_update_polling(conn);
+ conn->xprt = &ssl_sock;
+ conn->peekstep = -3;
+ req->buf->i = 0; // reset buffer data index to 0 so it may be filled
with unencrypted data
+ return 1;
+sock_error:
+ return 0;
+}
+
+
/* Receive up to <count> bytes from connection <conn>'s socket and store them
* into buffer <buf>. Only one call to recv() is performed, unless the
* buffer wraps, in which case a second call may be performed. The connection's
@@ -5535,28 +5625,28 @@ static int bind_parse_verify(char **args, int cur_arg,
struct proxy *px, struct
return 0;
}
-static int bind_parse_tcp_fallback(char **args, int cur_arg, struct proxy *px,
struct bind_conf *conf, char **err)
-{
- struct listener *l;
-
- if (!*args[cur_arg + 1]) {
- if (err)
- memprintf(err, "'%s' : missing tcp fallback timeout",
args[cur_arg]);
- return ERR_ALERT | ERR_FATAL;
- }
- if (parse_time_err(args[cur_arg+1], &conf->ssl_detection_delay,
TIME_UNIT_MS)) {
- memprintf(err, "'%s' : expects a positive timeout in
milliseconds\n", args[cur_arg]);
- return ERR_ALERT | ERR_FATAL;
- }
-
- /* Use raw socket operations here to do the SSL detection */
- list_for_each_entry(l, &conf->listeners, by_bind) {
- l->xprt = &raw_sock;
- l->options |= LI_O_SSL_DETECTION;
- }
- return 0;
-}
-
+static int bind_parse_tcp_fallback(char **args, int cur_arg, struct proxy *px,
struct bind_conf *conf, char **err)
+{
+ struct listener *l;
+
+ if (!*args[cur_arg + 1]) {
+ if (err)
+ memprintf(err, "'%s' : missing tcp fallback timeout",
args[cur_arg]);
+ return ERR_ALERT | ERR_FATAL;
+ }
+ if (parse_time_err(args[cur_arg+1], &conf->ssl_detection_delay,
TIME_UNIT_MS)) {
+ memprintf(err, "'%s' : expects a positive timeout in
milliseconds\n", args[cur_arg]);
+ return ERR_ALERT | ERR_FATAL;
+ }
+
+ /* Use raw socket operations here to do the SSL detection */
+ list_for_each_entry(l, &conf->listeners, by_bind) {
+ l->xprt = &raw_sock;
+ l->options |= LI_O_SSL_DETECTION;
+ }
+ return 0;
+}
+
/************** "server" keywords ****************/
/* parse the "ca-file" server keyword */
@@ -6018,7 +6108,7 @@ static struct bind_kw_list bind_kws = { "SSL", { }, {
{ "tls-ticket-keys", bind_parse_tls_ticket_keys, 1 }, /* set file
to load TLS ticket keys from */
{ "verify", bind_parse_verify, 1 }, /* set SSL
verify method */
{ "npn", bind_parse_npn, 1 }, /* set NPN
supported protocols */
- { "tcp-fallback", bind_parse_tcp_fallback, 1 }, /* enable
tcp fallback*/
+ { "tcp-fallback", bind_parse_tcp_fallback, 1 }, /* enable
tcp fallback*/
{ NULL, NULL, 0 },
}};
@@ -6072,7 +6162,6 @@ struct xprt_ops ssl_sock = {
.init = ssl_sock_init,
};
-
#if (OPENSSL_VERSION_NUMBER >= 0x1000200fL && !defined OPENSSL_NO_TLSEXT &&
!defined OPENSSL_IS_BORINGSSL && !defined LIBRESSL_VERSION_NUMBER)
static void ssl_sock_sctl_free_func(void *parent, void *ptr, CRYPTO_EX_DATA
*ad, int idx, long argl, void *argp)
--
2.7.0.windows.1
