experimental patch to make use of IO_URING to batch load certificates;
this drastically reduces the number of syscall and might benefit to
setup with large number of certificates.
it uses liburing in order to batch operation as follow:
for each certificate directory, we apply the same operations on each
file:
- statx
- openat
- read
- close
the results are stored in an ebtree. Then when we need to load them
with the SSL lib, instead of providing a path, we provide a buffer to
be consumed. The tree is freed after each directory.
for now it requires a quite large limit of file descriptors, as all
operations types are done one after another; so the limit of fd should
be set higher than the number of certificates you have to load. This
part is probably going to evolve very soon as IO_URING plans are to be
able to chain operations with a given pre-defined file descriptor.
on a setup with 25k certificates I was able to measure a minimum of 20%
gain on the init time when the filesystem cache is empty. My testing is
as follow; for each directory:
- put a timer before `ssl_sock_load_cert_list_file`, including
`ssl_load_certiodir` in the case of io_uring case.
- measure the time at the end of those operations, just after
`ssl_free_certiodir` in the case of io_uring.
some results with my old ssd laptop, the average certificate init time
in ms, using 24686 certs:
▒▒▒▒▒▒▒▒▒▒▒▒▒▒ 8554 io_uring_empty_cache
▒▒▒▒▒▒▒▒▒▒▒▒▒ 8100 io_uring_full_cache
▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒ 11395 syscall_empty_cache
▒▒▒▒▒▒▒▒▒▒▒▒▒ 8087 syscall_full_cache
The benefits are of course less obvious with following reloads on a
machine where you only have haproxy process managing its own memory, but
it can be interesting on a busier setup where the filesystem cache is
often invalidated. However a very quick overview of reload time on our
production seems to show that more than 50% of the time, the files are
not present in cache anymore while reloading (I hope I'm not too far
from the reality in my analysis); so I would expect this patch could
participate to flatten our reload time; I however did not tested it in a
production environment.
It remains linux-specific though; the operations used require a minimum
of kernel >= v5.6 even though some operations were supported in v5.4, I
did not change the code to adapt the behavior. All my tests were
performed on >= v5.7rc4 as the time of writing.
it introduces three build parameters:
- USE_IO_URING
- IO_URING_INC
- IO_URING_LIB
Signed-off-by: William Dauchy <w.dau...@criteo.com>
---
Makefile | 10 +-
doc/io_uring.txt | 18 +++
include/proto/ssl_load.h | 23 ++++
include/types/ssl_load.h | 38 ++++++
include/types/ssl_sock.h | 7 +
src/ssl_load.c | 274 +++++++++++++++++++++++++++++++++++++++
src/ssl_sock.c | 32 ++++-
7 files changed, 395 insertions(+), 7 deletions(-)
create mode 100644 doc/io_uring.txt
create mode 100644 include/proto/ssl_load.h
create mode 100644 include/types/ssl_load.h
create mode 100644 src/ssl_load.c
diff --git a/Makefile b/Makefile
index 1e4213989..afba381c0 100644
--- a/Makefile
+++ b/Makefile
@@ -52,6 +52,7 @@
# USE_SYSTEMD : enable sd_notify() support.
# USE_OBSOLETE_LINKER : use when the linker fails to emit
__start_init/__stop_init
# USE_THREAD_DUMP : use the more advanced thread state dump system.
Automatic.
+# USE_IO_URING : use IO_URING advanced async features
#
# Options can be forced by specifying "USE_xxx=1" or can be disabled by using
# "USE_xxx=" (empty string). The list of enabled and disabled options for a
@@ -293,7 +294,8 @@ use_opts = USE_EPOLL USE_KQUEUE USE_NETFILTER
\
USE_GETADDRINFO USE_OPENSSL USE_LUA USE_FUTEX USE_ACCEPT4 \
USE_ZLIB USE_SLZ USE_CPU_AFFINITY USE_TFO USE_NS \
USE_DL USE_RT USE_DEVICEATLAS USE_51DEGREES USE_WURFL USE_SYSTEMD \
- USE_OBSOLETE_LINKER USE_PRCTL USE_THREAD_DUMP USE_EVPORTS
+ USE_OBSOLETE_LINKER USE_PRCTL USE_THREAD_DUMP USE_EVPORTS \
+ USE_IO_URING
#### Target system options
# Depending on the target platform, some options are set, as well as some
@@ -531,6 +533,12 @@ ifneq ($(USE_BACKTRACE),)
OPTIONS_LDFLAGS += -Wl,$(if $(EXPORT_SYMBOL),$(EXPORT_SYMBOL),--export-dynamic)
endif
+ifneq ($(USE_IO_URING),)
+OPTIONS_OBJS += src/ssl_load.o
+OPTIONS_CFLAGS += $(if $(IO_URING_INC),-I$(IO_URING_INC))
+OPTIONS_LDFLAGS += $(if $(IO_URING_LIB),-L$(IO_URING_LIB)) -luring
+endif
+
ifneq ($(USE_OPENSSL),)
# OpenSSL is packaged in various forms and with various dependencies.
# In general -lssl is enough, but on some platforms, -lcrypto may be needed,
diff --git a/doc/io_uring.txt b/doc/io_uring.txt
new file mode 100644
index 000000000..aa290dd0d
--- /dev/null
+++ b/doc/io_uring.txt
@@ -0,0 +1,18 @@
+Linux IO_URING support for HAProxy
+==================================
+
+0. Build
+--------
+
+Add the following parameters to your make options:
+
+- USE_IO_URING=1
+- IO_URING_INC='/path/to/liburing/src/include/'
+- IO_URING_LIB='/path/to/liburing/src/'
+
+1. Certificates loading
+-----------------------
+
+HAProxy can make use of IO_URING in order to reduce the number of syscall used
+to load the certificates. It requires Linux kernel >= v5.6 according to the
+operations used (statx, open, read, close).
diff --git a/include/proto/ssl_load.h b/include/proto/ssl_load.h
new file mode 100644
index 000000000..b1b5ba07d
--- /dev/null
+++ b/include/proto/ssl_load.h
@@ -0,0 +1,23 @@
+/*
+ * load certificates in batch with liburing
+ *
+ * Copyright 2020 William Dauchy <wdau...@gmail.com>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version
+ * 2 of the License, or (at your option) any later version.
+ */
+
+#ifndef _PROTO_SSL_LOAD_H
+#define _PROTO_SSL_LOAD_H
+
+#ifdef USE_IO_URING
+
+#include <ebpttree.h>
+
+int ssl_load_certiodir(const char *filedir, struct eb_root *file_tree);
+void ssl_free_certiodir(const char *filedir, struct eb_root *file_tree);
+
+#endif /* USE_IO_URING */
+#endif /* _PROTO_SSL_LOAD_H */
diff --git a/include/types/ssl_load.h b/include/types/ssl_load.h
new file mode 100644
index 000000000..67384ee65
--- /dev/null
+++ b/include/types/ssl_load.h
@@ -0,0 +1,38 @@
+/*
+ * load certificates in batch with liburing
+ *
+ * Copyright 2020 William Dauchy <wdau...@gmail.com>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version
+ * 2 of the License, or (at your option) any later version.
+ */
+
+#ifndef _TYPES_SSL_LOAD_H
+#define _TYPES_SSL_LOAD_H
+
+#ifdef USE_IO_URING
+
+#include <liburing.h>
+#include <ebpttree.h>
+
+#define QD 4096
+
+struct cert_iobuf {
+ int fd;
+ char *buf;
+ off_t filesize;
+ struct statx stx;
+ struct ebmb_node node;
+ char filepath[0];
+};
+
+struct io_op {
+ char name[6]; /*
operation name for logging */
+ int (*queue)(struct io_uring *, struct cert_iobuf *, int);
+ void (*handle)(struct cert_iobuf *, struct io_uring_cqe *);
+};
+
+#endif /* USE_IO_URING */
+#endif /* _TYPES_SSL_LOAD_H */
diff --git a/include/types/ssl_sock.h b/include/types/ssl_sock.h
index dbfa3d726..1f38c470e 100644
--- a/include/types/ssl_sock.h
+++ b/include/types/ssl_sock.h
@@ -31,6 +31,13 @@
#include <common/mini-clist.h>
#include <common/openssl-compat.h>
+#if HA_OPENSSL_VERSION_NUMBER >= 0x1000200fL
+extern const char *SSL_SOCK_KEYTYPE_NAMES[];
+#define SSL_SOCK_NUM_KEYTYPES 3
+#else
+#define SSL_SOCK_NUM_KEYTYPES 1
+#endif
+
struct pkey_info {
uint8_t sig; /* TLSEXT_signature_[rsa,ecdsa,...] */
uint16_t bits; /* key size in bits */
diff --git a/src/ssl_load.c b/src/ssl_load.c
new file mode 100644
index 000000000..f3b4b9c19
--- /dev/null
+++ b/src/ssl_load.c
@@ -0,0 +1,274 @@
+/*
+ * load certificates in batch with liburing
+ *
+ * Copyright 2020 William Dauchy <wdau...@gmail.com>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version
+ * 2 of the License, or (at your option) any later version.
+ */
+
+#define _GNU_SOURCE
+
+#include <fcntl.h>
+#include <dirent.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <string.h>
+
+#include <liburing.h>
+
+#include <ebpttree.h>
+#include <ebsttree.h>
+
+#include <types/ssl_sock.h>
+#include <types/ssl_load.h>
+
+#include <proto/log.h>
+
+static int setup_context(unsigned entries, struct io_uring *ring)
+{
+ int ret;
+
+ ret = io_uring_queue_init(entries, ring, 0);
+ if (ret < 0) {
+ ha_alert("queue_init: %s\n", strerror(-ret));
+ return ERR_ALERT;
+ }
+ return 0;
+}
+
+static int do_io_op(struct io_uring *ring, struct eb_root *cert_iobuf_tree,
+ struct io_op *op, int dirfd)
+{
+ struct cert_iobuf *cert_io;
+ struct eb_node *node, *next;
+ struct io_uring_cqe *cqe;
+ unsigned int i;
+ int inqueue;
+ int pending;
+ int ret;
+
+ inqueue = 0;
+ node = eb_first(cert_iobuf_tree);
+ while (node) {
+ for (; node && inqueue < QD; inqueue++) {
+ next = eb_next(node);
+ cert_io = ebmb_entry(node, struct cert_iobuf, node);
+ ret = op->queue(ring, cert_io, dirfd);
+ if (ret)
+ ha_warning("ssl_load: queue error\n");
+ node = next;
+ }
+ ret = io_uring_submit(ring);
+ if (ret < 0) {
+ ha_warning("ssl_load, %s: io_uring_submit: %s\n",
+ op->name, strerror(-ret));
+ return ERR_ALERT;
+ }
+ pending = ret;
+ for (i = 0; i < pending; i++) {
+ ret = io_uring_wait_cqe(ring, &cqe);
+ if (ret < 0) {
+ ha_warning("ssl_load, %s: io_uring_wait_cqe:
%s\n",
+ op->name, strerror(-ret));
+ continue;
+ }
+ if (!cqe)
+ continue;
+ cert_io = io_uring_cqe_get_data(cqe);
+ if (cqe->res < 0)
+ ha_warning("ssl_load, %s: cqe failed: %s
(%s)\n",
+ op->name, strerror(-cqe->res),
cert_io->filepath);
+ else
+ op->handle(cert_io, cqe);
+ io_uring_cqe_seen(ring, cqe);
+ inqueue--;
+ }
+ }
+ if (inqueue > 0) {
+ ha_warning("ssl_load, %s: inqueue error\n", op->name);
+ return ERR_ALERT;
+ }
+ return 0;
+}
+
+static int queue_statx(struct io_uring *ring, struct cert_iobuf *cert_io, int
dirfd)
+{
+ struct io_uring_sqe *sqe;
+
+ sqe = io_uring_get_sqe(ring);
+ if (!sqe)
+ return ERR_ALERT;
+
+ io_uring_prep_statx(sqe, dirfd, cert_io->filepath, 0, STATX_SIZE,
&(cert_io->stx));
+ io_uring_sqe_set_data(sqe, cert_io);
+ return 0;
+}
+
+static void handle_statx(struct cert_iobuf *cert_io, struct io_uring_cqe *cqe)
+{
+ cert_io->filesize = cert_io->stx.stx_size;
+}
+
+static int queue_open(struct io_uring *ring, struct cert_iobuf *cert_io, int
dirfd)
+{
+ struct io_uring_sqe *sqe;
+
+ sqe = io_uring_get_sqe(ring);
+ if (!sqe)
+ return ERR_ALERT;
+
+ io_uring_prep_openat(sqe, dirfd, cert_io->filepath, O_RDONLY, 0);
+ io_uring_sqe_set_data(sqe, cert_io);
+ return 0;
+}
+
+static void handle_open(struct cert_iobuf *cert_io, struct io_uring_cqe *cqe)
+{
+ cert_io->fd = cqe->res;
+}
+
+static int queue_read(struct io_uring *ring, struct cert_iobuf *cert_io, int
dirfd)
+{
+ struct io_uring_sqe *sqe;
+
+ sqe = io_uring_get_sqe(ring);
+ if (!sqe)
+ return ERR_ALERT;
+
+ cert_io->buf = malloc(sizeof(*(cert_io->buf)) * cert_io->filesize + 1);
+ if (!cert_io->buf) {
+ ha_alert("ssl_load: out of memory in buffer allocation\n");
+ return ERR_ALERT;
+ }
+
+ io_uring_prep_read(sqe, cert_io->fd, cert_io->buf, cert_io->filesize,
0);
+ io_uring_sqe_set_data(sqe, cert_io);
+ return 0;
+}
+
+void handle_read(struct cert_iobuf *cert_io, struct io_uring_cqe *cqe)
+{
+ cert_io->buf[cert_io->filesize] = 0;
+}
+
+static int queue_close(struct io_uring *ring, struct cert_iobuf *cert_io, int
dirfd)
+{
+ struct io_uring_sqe *sqe;
+
+ sqe = io_uring_get_sqe(ring);
+ if (!sqe)
+ return ERR_ALERT;
+
+ io_uring_prep_close(sqe, cert_io->fd);
+ return 0;
+}
+
+static void handle_close(struct cert_iobuf *cert_io, struct io_uring_cqe *cqe)
+{
+}
+
+static int ssl_load_preparetree(const char *filedir, struct eb_root
*cert_iobuf_tree)
+{
+ struct cert_iobuf *cert_io;
+ struct dirent **de_list;
+ struct dirent *de;
+ int path_len;
+ int nb_file;
+ int i, j;
+ char *end;
+
+ nb_file = scandir(filedir, &de_list, 0, alphasort);
+ if (nb_file < 0) {
+ ha_warning("ssl_load: unable to scan directory\n");
+ return ERR_ALERT;
+ }
+ for (i = 0; i < nb_file; i++) {
+ de = de_list[i];
+ end = strrchr(de->d_name, '.');
+ for (j = 0; j < SSL_SOCK_NUM_KEYTYPES; j++)
+ if (!strcmp(end + 1, SSL_SOCK_KEYTYPE_NAMES[j]))
+ goto load_entry;
+ goto skip_entry;
+load_entry:
+ /* filedir + slash + name + \0 */
+ path_len = strlen(filedir) + strlen(de->d_name) + 2;
+ cert_io = malloc(sizeof(*cert_io) + path_len);
+ if (cert_io == NULL) {
+ ha_alert("ssl_load: out of memory in tree
allocation\n");
+ return ERR_ALERT;
+ }
+ snprintf((char *) cert_io->node.key, path_len, "%s/%s",
+ filedir, de->d_name);
+ cert_io->filesize = 0;
+ cert_io->buf = NULL;
+ ebst_insert(cert_iobuf_tree, &cert_io->node);
+skip_entry:
+ free(de);
+ }
+ free(de_list);
+ return 0;
+}
+
+int ssl_load_certiodir(const char *filedir, struct eb_root *cert_iobuf_tree)
+{
+ struct io_uring ring;
+ struct io_op op;
+ int dirfd;
+ int ret = 0;
+
+ ret = ssl_load_preparetree(filedir, cert_iobuf_tree);
+ if (ret)
+ return ret;
+
+ dirfd = open(filedir, 0);
+ if (dirfd < 0)
+ return ERR_ALERT;
+ if (setup_context(QD, &ring))
+ return ERR_ALERT;
+ memcpy(op.name, "statx", strlen("statx") + 1);
+ op.queue = queue_statx;
+ op.handle = handle_statx;
+ ret = do_io_op(&ring, cert_iobuf_tree, &op, dirfd);
+ if (ret)
+ goto exit;
+ memcpy(op.name, "open", strlen("open") + 1);
+ op.queue = queue_open;
+ op.handle = handle_open;
+ ret = do_io_op(&ring, cert_iobuf_tree, &op, dirfd);
+ if (ret)
+ goto exit;
+ memcpy(op.name, "read", strlen("open") + 1);
+ op.queue = queue_read;
+ op.handle = handle_read;
+ ret = do_io_op(&ring, cert_iobuf_tree, &op, dirfd);
+ if (ret)
+ goto exit;
+ memcpy(op.name, "close", strlen("open") + 1);
+ op.queue = queue_close;
+ op.handle = handle_close;
+ ret = do_io_op(&ring, cert_iobuf_tree, &op, dirfd);
+ if (ret)
+ goto exit;
+exit:
+ io_uring_queue_exit(&ring);
+ return ret;
+}
+
+void ssl_free_certiodir(const char *filedir, struct eb_root *cert_iobuf_tree)
+{
+ struct eb_node *node, *next;
+ struct cert_iobuf *cert_io;
+
+ node = eb_first(cert_iobuf_tree);
+ while (node) {
+ next = eb_next(node);
+ eb_delete(node);
+ cert_io = ebmb_entry(node, struct cert_iobuf, node);
+ free(cert_io->buf);
+ free(cert_io);
+ node = next;
+ }
+}
diff --git a/src/ssl_sock.c b/src/ssl_sock.c
index 78d6da303..1f4fa1599 100644
--- a/src/ssl_sock.c
+++ b/src/ssl_sock.c
@@ -64,6 +64,7 @@
#include <types/cli.h>
#include <types/global.h>
#include <types/ssl_sock.h>
+#include <types/ssl_load.h>
#include <types/stats.h>
#include <proto/acl.h>
@@ -85,6 +86,7 @@
#include <proto/proxy.h>
#include <proto/shctx.h>
#include <proto/ssl_sock.h>
+#include <proto/ssl_load.h>
#include <proto/stream.h>
#include <proto/task.h>
#include <proto/vars.h>
@@ -163,6 +165,10 @@ int nb_engines = 0;
static struct eb_root cert_issuer_tree = EB_ROOT; /* issuers tree from
"issuers-chain-path" */
static struct issuer_chain* ssl_get0_issuer_chain(X509 *cert);
+#ifdef USE_IO_URING
+static struct eb_root cert_iobuf_tree = EB_ROOT_UNIQUE; /* IO_URING
certificates buffer */
+#endif
+
static struct {
char *crt_base; /* base directory path for certificates */
char *ca_base; /* base directory path for CAs and CRLs */
@@ -613,9 +619,6 @@ const char *SSL_SOCK_KEYTYPE_NAMES[] = {
"ecdsa",
"rsa"
};
-#define SSL_SOCK_NUM_KEYTYPES 3
-#else
-#define SSL_SOCK_NUM_KEYTYPES 1
#endif
static struct shared_context *ssl_shctx = NULL; /* ssl shared session cache */
@@ -3680,12 +3683,22 @@ end:
*/
static int ssl_sock_load_files_into_ckch(const char *path, struct
cert_key_and_chain *ckch, char **err)
{
+ char *buf = NULL;
int ret = 1;
+#ifdef USE_IO_URING
+ struct ebmb_node *eb = NULL;
+ struct cert_iobuf *cert_io;
+
+ eb = ebst_lookup(&cert_iobuf_tree, path);
+ if (eb) {
+ cert_io = ebmb_entry(eb, struct cert_iobuf, node);
+ buf = cert_io->buf;
+ }
+#endif
/* try to load the PEM */
- if (ssl_sock_load_pem_into_ckch(path, NULL, ckch , err) != 0) {
+ if (ssl_sock_load_pem_into_ckch(path, buf, ckch , err) != 0)
goto end;
- }
/* try to load an external private key if it wasn't in the PEM */
if ((ckch->key == NULL) && (global_ssl.extra_files & SSL_GF_KEY)) {
@@ -5072,7 +5085,14 @@ int ssl_sock_load_cert(char *path, struct bind_conf
*bind_conf, char **err)
return ssl_sock_load_ckchs(path, ckchs, bind_conf,
NULL, NULL, 0, &ckch_inst, err);
} else {
- return ssl_sock_load_cert_list_file(path, 1, bind_conf,
bind_conf->frontend, err);
+#ifdef USE_IO_URING
+ cfgerr = ssl_load_certiodir(path, &cert_iobuf_tree);
+#endif
+ cfgerr |= ssl_sock_load_cert_list_file(path, 1,
bind_conf, bind_conf->frontend, err);
+#ifdef USE_IO_URING
+ ssl_free_certiodir(path, &cert_iobuf_tree);
+#endif
+ return cfgerr;
}
} else {
/* stat failed, could be a bundle */
--
2.26.2
