ssl_async is a global configuration parameter which enables asynchronous
processing in OPENSSL for all SSL connections haproxy handles. With
SSL_MODE_ASYNC mode set, TLS I/O operations may indicate a retry with
SSL_ERROR_WANT_ASYNC with this mode set if an asynchronous capable
engine is used to perform cryptographic operations.
---
include/common/epoll.h | 2 +
include/proto/fd.h | 8 ++++
include/types/connection.h | 2 +
include/types/fd.h | 1 +
include/types/global.h | 1 +
src/cfgparse.c | 10 +++++
src/ev_epoll.c | 11 +++++
src/fd.c | 13 ++++++
src/haproxy.c | 1 +
src/ssl_sock.c | 104 +++++++++++++++++++++++++++++++++++++++++++++
10 files changed, 153 insertions(+)
diff --git a/include/common/epoll.h b/include/common/epoll.h
index cc395cd..dd1c7d6 100644
--- a/include/common/epoll.h
+++ b/include/common/epoll.h
@@ -70,6 +70,8 @@ struct epoll_event {
} data;
};
+int remove_fd_from_epoll(int fd);
+
#if defined(CONFIG_HAP_LINUX_VSYSCALL) && defined(__linux__) &&
defined(__i386__)
/* Those are our self-defined functions */
extern int epoll_create(int size);
diff --git a/include/proto/fd.h b/include/proto/fd.h
index 87309bf..922fee5 100644
--- a/include/proto/fd.h
+++ b/include/proto/fd.h
@@ -40,6 +40,7 @@ extern int fd_nbupdt; // number of updates in
the list
* The file descriptor is also closed.
*/
void fd_delete(int fd);
+void fd_async_delete(int fd);
/* disable the specified poller */
void disable_poller(const char *poller_name);
@@ -339,6 +340,13 @@ static inline void fd_insert(int fd)
maxfd = fd + 1;
}
+/* Prepares async <fd> for being polled */
+static inline void fd_async_insert(int fd)
+{
+ fdtab[fd].state = 0;
+ fdtab[fd].async = 1;
+ fd_insert(fd);
+}
#endif /* _PROTO_FD_H */
diff --git a/include/types/connection.h b/include/types/connection.h
index 8b732ff..3233f25 100644
--- a/include/types/connection.h
+++ b/include/types/connection.h
@@ -288,6 +288,8 @@ struct connection {
struct sockaddr_storage from; /* client address, or address
to spoof when connecting to the server */
struct sockaddr_storage to; /* address reached by the
client, or address to connect to */
} addr; /* addresses of the remote side, client for producer and server
for consumer */
+
+ OSSL_ASYNC_FD async_fd;
};
/* proxy protocol v2 definitions */
diff --git a/include/types/fd.h b/include/types/fd.h
index 7f63093..f3d03f8 100644
--- a/include/types/fd.h
+++ b/include/types/fd.h
@@ -100,6 +100,7 @@ struct fdtab {
unsigned char updated:1; /* 1 if this fd is already in the
update list */
unsigned char linger_risk:1; /* 1 if we must kill lingering
before closing */
unsigned char cloned:1; /* 1 if a cloned socket, requires
EPOLL_CTL_DEL on close */
+ unsigned char async:1; /* 1 if this fd is async ssl fd */
};
/* less often used information */
diff --git a/include/types/global.h b/include/types/global.h
index 9a6e2c9..37727b9 100644
--- a/include/types/global.h
+++ b/include/types/global.h
@@ -85,6 +85,7 @@ struct global {
char *crt_base; /* base directory path for certificates */
char *ca_base; /* base directory path for CAs and CRLs */
char *ssl_engine; /* openssl engine to use */
+ int ssl_async; /* whether we use ssl async mode */
#endif
int uid;
int gid;
diff --git a/src/cfgparse.c b/src/cfgparse.c
index f8ad855..7eb0593 100644
--- a/src/cfgparse.c
+++ b/src/cfgparse.c
@@ -652,6 +652,16 @@ int cfg_parse_global(const char *file, int linenum, char
**args, int kwm)
goto out;
#endif
}
+ else if (!strcmp(args[0], "ssl_async")) {
+#ifdef USE_OPENSSL
+ global.ssl_async = 1;
+ qfprintf(stdout, "parsing [%s:%d] : turns on SSL async
mode.\n", file, linenum);
+#else
+ Alert("parsing [%s:%d] : ssl_async is not implemented.\n",
file, linenum);
+ err_code |= ERR_ALERT | ERR_FATAL;
+ goto out;
+#endif
+ }
else if (!strcmp(args[0], "daemon")) {
if (alertif_too_many_args(0, file, linenum, args, &err_code))
goto out;
diff --git a/src/ev_epoll.c b/src/ev_epoll.c
index ccb7c33..27d630e 100644
--- a/src/ev_epoll.c
+++ b/src/ev_epoll.c
@@ -41,6 +41,17 @@ static struct epoll_event ev;
#define EPOLLRDHUP 0x2000
#endif
+int remove_fd_from_epoll(int fd)
+{
+ struct epoll_event ee;
+ int ret;
+
+ ee.events = 0;
+ ee.data.ptr = NULL;
+ ret = epoll_ctl(epoll_fd, EPOLL_CTL_DEL, fd, &ee);
+ return ret;
+}
+
/*
* Immediately remove file descriptor from epoll set upon close.
* Since we forked, some fds share inodes with the other process, and epoll may
diff --git a/src/fd.c b/src/fd.c
index aeee602..483f3eb 100644
--- a/src/fd.c
+++ b/src/fd.c
@@ -198,6 +198,19 @@ void fd_delete(int fd)
maxfd--;
}
+void fd_async_delete(int fd)
+{
+ fd_release_cache_entry(fd);
+
+ fdtab[fd].state = 0;
+ fdtab[fd].async = 0;
+ fdtab[fd].owner = NULL;
+ fdtab[fd].new = 0;
+
+ while ((maxfd-1 >= 0) && !fdtab[maxfd-1].owner)
+ maxfd--;
+}
+
/* Scan and process the cached events. This should be called right after
* the poller. The loop may cause new entries to be created, for example
* if a listener causes an accept() to initiate a new incoming connection
diff --git a/src/haproxy.c b/src/haproxy.c
index 69a4551..8c316c3 100644
--- a/src/haproxy.c
+++ b/src/haproxy.c
@@ -183,6 +183,7 @@ struct global global = {
#endif
},
#ifdef USE_OPENSSL
+ .ssl_async = 0, /* SSL_MODE_ASYNC is off by default */
#ifdef DEFAULT_MAXSSLCONN
.maxsslconn = DEFAULT_MAXSSLCONN,
#endif
diff --git a/src/ssl_sock.c b/src/ssl_sock.c
index 0b3cee5..6b31432 100644
--- a/src/ssl_sock.c
+++ b/src/ssl_sock.c
@@ -57,6 +57,7 @@
#include <import/xxhash.h>
#include <openssl/engine.h>
+#include <openssl/async.h>
#include <common/buffer.h>
#include <common/compat.h>
@@ -68,6 +69,7 @@
#include <common/time.h>
#include <common/cfgparse.h>
#include <common/base64.h>
+#include <common/epoll.h>
#include <ebsttree.h>
@@ -267,6 +269,74 @@ void ssl_init_engine(const char *engine_id)
}
/*
+ * openssl async fd handler
+ */
+int ssl_async_process_fds(struct connection *conn)
+{
+ OSSL_ASYNC_FD *add_fds = NULL;
+ OSSL_ASYNC_FD *del_fds = NULL;
+ size_t num_add_fds = 0;
+ size_t num_del_fds = 0;
+ unsigned int loop = 0;
+
+ SSL *ssl = conn->xprt_ctx;
+
+ SSL_get_changed_async_fds(ssl, NULL, &num_add_fds, NULL, &num_del_fds);
+
+ if (num_add_fds == 0 && num_del_fds == 0)
+ return 0;
+
+ if (num_add_fds) {
+ add_fds = OPENSSL_malloc(num_add_fds * sizeof(OSSL_ASYNC_FD));
+ if (add_fds == NULL) {
+ Alert("Memory Allocation Error");
+ return 0;
+ }
+ }
+
+ if (num_del_fds) {
+ del_fds = OPENSSL_malloc(num_del_fds * sizeof(OSSL_ASYNC_FD));
+ if (del_fds == NULL) {
+ Alert("Memory Allocation Error");
+ if (add_fds)
+ OPENSSL_free(add_fds);
+ return 0;
+ }
+ }
+
+ SSL_get_changed_async_fds(ssl, add_fds, &num_add_fds, del_fds,
+ &num_del_fds);
+
+ if (num_del_fds) {
+ for (loop = 0; loop < num_del_fds; loop++) {
+ int async_fd = del_fds[loop];
+
+ fd_stop_recv(async_fd);
+ fd_async_delete(async_fd);
+ }
+ }
+
+ if (num_add_fds) {
+ for (loop = 0; loop < num_add_fds; loop++) {
+ int async_fd = add_fds[loop];
+
+ fd_async_insert(async_fd);
+ fdtab[async_fd].iocb = conn_fd_handler;
+ conn->async_fd = async_fd;
+ fdtab[async_fd].owner = conn;
+ fd_want_recv(async_fd);
+ }
+ }
+
+ if (add_fds)
+ OPENSSL_free(add_fds);
+ if (del_fds)
+ OPENSSL_free(del_fds);
+
+ return 1;
+}
+
+/*
* This function returns the number of seconds elapsed
* since the Epoch, 1970-01-01 00:00:00 +0000 (UTC) and the
* date presented un ASN1_GENERALIZEDTIME.
@@ -2743,6 +2813,9 @@ int ssl_sock_prepare_ctx(struct bind_conf *bind_conf,
SSL_CTX *ctx, struct proxy
cfgerr++;
}
+ if (global.ssl_async)
+ sslmode |= SSL_MODE_ASYNC;
+
if (bind_conf->ssl_options & BC_SSL_O_NO_SSLV3)
ssloptions |= SSL_OP_NO_SSLv3;
if (bind_conf->ssl_options & BC_SSL_O_NO_TLSV10)
@@ -3149,6 +3222,10 @@ int ssl_sock_prepare_srv_ctx(struct server *srv, struct
proxy *curproxy)
#endif
SSL_CTX_set_options(srv->ssl_ctx.ctx, options);
+
+ if (global.ssl_async)
+ mode |= SSL_MODE_ASYNC;
+
SSL_CTX_set_mode(srv->ssl_ctx.ctx, mode);
if (global.ssl_server_verify == SSL_SERVER_VERIFY_REQUIRED)
@@ -3533,6 +3610,12 @@ int ssl_sock_handshake(struct connection *conn, unsigned
int flag)
if (ret <= 0) {
/* handshake may have not been completed, let's find
why */
ret = SSL_get_error(conn->xprt_ctx, ret);
+
+ if (ret == SSL_ERROR_WANT_ASYNC) {
+ ssl_async_process_fds(conn);
+ return 0;
+ }
+
if (ret == SSL_ERROR_WANT_WRITE) {
/* SSL handshake needs to write, L4 connection
may not be ready */
__conn_sock_stop_recv(conn);
@@ -3615,6 +3698,11 @@ int ssl_sock_handshake(struct connection *conn, unsigned
int flag)
/* handshake did not complete, let's find why */
ret = SSL_get_error(conn->xprt_ctx, ret);
+ if (ret == SSL_ERROR_WANT_ASYNC) {
+ ssl_async_process_fds(conn);
+ return 0;
+ }
+
if (ret == SSL_ERROR_WANT_WRITE) {
/* SSL handshake needs to write, L4 connection may not
be ready */
__conn_sock_stop_recv(conn);
@@ -3796,6 +3884,11 @@ static int ssl_sock_to_buf(struct connection *conn,
struct buffer *buf, int coun
}
else {
ret = SSL_get_error(conn->xprt_ctx, ret);
+ if (ret == SSL_ERROR_WANT_ASYNC) {
+ ssl_async_process_fds(conn);
+ break;
+ }
+
if (ret == SSL_ERROR_WANT_WRITE) {
/* handshake is running, and it needs to enable
write */
conn->flags |= CO_FL_SSL_WAIT_HS;
@@ -3897,6 +3990,12 @@ static int ssl_sock_from_buf(struct connection *conn,
struct buffer *buf, int fl
}
else {
ret = SSL_get_error(conn->xprt_ctx, ret);
+
+ if (ret == SSL_ERROR_WANT_ASYNC) {
+ ssl_async_process_fds(conn);
+ break;
+ }
+
if (ret == SSL_ERROR_WANT_WRITE) {
if (SSL_renegotiate_pending(conn->xprt_ctx)) {
/* handshake is running, and it may
need to re-enable write */
@@ -3931,6 +4030,11 @@ static int ssl_sock_from_buf(struct connection *conn,
struct buffer *buf, int fl
static void ssl_sock_close(struct connection *conn) {
if (conn->xprt_ctx) {
+ if (global.ssl_async) {
+ remove_fd_from_epoll(conn->async_fd);
+ fd_stop_recv(conn->async_fd);
+ fd_async_delete(conn->async_fd);
+ }
SSL_free(conn->xprt_ctx);
conn->xprt_ctx = NULL;
sslconns--;
--
1.9.1
