On Fri, Apr 11, 2014 at 08:15:27PM -0600, Bob Beck wrote: > On Fri, Apr 11, 2014 at 6:09 PM, Reyk Floeter <[email protected]> wrote: > > > > > > I did some testing with apache bench (ab) and it shows a negative > > performance impact when running with multiple preforked relays and > > concurrent requests. But this is expected because all processes have > > to wait for the single "ca" process to complete the RSA operations. > > > > Is this desirable? > > Ahh - as compared to what - relayd running with each process itself > doing the RSA ops (acting as an SSL accelerator) > in front of it? then that would be expected. I wouldn't call it > "desirable" - let relayd have more than one "CA" process.
This updated diff creates one ca process per relay and gives much better performance results (a little decrease is still there but that's because of the imsg passing. That can also be optimized later). Currently, relayd creates 5 relay processe by default unless you change it with the "prefork" option. I changed the default to 3 because I think 5 is a little bit too much for most "normal" use cases and I didn't measure big differences with normal load testing. But these 3 relays will turn into 6 processes now, each one with a "CA". Creating a more flexible N:M mapping (eg. 5 relays and 3 CAs) would make the code more complex and doesn't seem to work very well, so I want to stick on the "1 CA per relay". Any more testers? Thoughts? Reyk Index: Makefile =================================================================== RCS file: /cvs/src/usr.sbin/relayd/Makefile,v retrieving revision 1.25 diff -u -p -u -p -r1.25 Makefile --- Makefile 14 Apr 2014 12:58:04 -0000 1.25 +++ Makefile 17 Apr 2014 19:26:59 -0000 @@ -1,11 +1,12 @@ # $OpenBSD: Makefile,v 1.25 2014/04/14 12:58:04 blambert Exp $ PROG= relayd -SRCS= parse.y log.c control.c ssl.c ssl_privsep.c \ - relayd.c pfe.c pfe_filter.c pfe_route.c hce.c relay.c \ - relay_http.c relay_udp.c carp.c check_icmp.c check_tcp.c \ - check_script.c name2id.c snmp.c shuffle.c proc.c config.c \ - agentx.c +SRCS= parse.y +SRCS+= agentx.c ca.c carp.c check_icmp.c check_script.c \ + check_tcp.c config.c control.c hce.c log.c name2id.c \ + pfe.c pfe_filter.c pfe_route.c proc.c \ + relay.c relay_http.c relay_udp.c relayd.c \ + shuffle.c snmp.c ssl.c ssl_privsep.c MAN= relayd.8 relayd.conf.5 LDADD= -levent -lssl -lcrypto -lutil Index: ca.c =================================================================== RCS file: ca.c diff -N ca.c --- /dev/null 1 Jan 1970 00:00:00 -0000 +++ ca.c 17 Apr 2014 19:26:59 -0000 @@ -0,0 +1,431 @@ +/* $OpenBSD$ */ + +/* + * Copyright (c) 2014 Reyk Floeter <[email protected]> + * + * Permission to use, copy, modify, and distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + */ + +#include <sys/param.h> +#include <sys/queue.h> +#include <sys/socket.h> +#include <sys/uio.h> + +#include <net/if.h> +#include <netinet/in.h> + +#include <limits.h> +#include <event.h> +#include <fcntl.h> +#include <unistd.h> +#include <string.h> +#include <stdlib.h> +#include <errno.h> + +#include <openssl/pem.h> +#include <openssl/evp.h> +#include <openssl/rsa.h> +#include <openssl/engine.h> + +#include "relayd.h" + +void ca_init(struct privsep *, struct privsep_proc *p, void *); +void ca_launch(void); + +int ca_dispatch_parent(int, struct privsep_proc *, struct imsg *); +int ca_dispatch_relay(int, struct privsep_proc *, struct imsg *); + +int rsae_pub_enc(int, const u_char *, u_char *, RSA *, int); +int rsae_pub_dec(int,const u_char *, u_char *, RSA *, int); +int rsae_priv_enc(int, const u_char *, u_char *, RSA *, int); +int rsae_priv_dec(int, const u_char *, u_char *, RSA *, int); +int rsae_mod_exp(BIGNUM *, const BIGNUM *, RSA *, BN_CTX *); +int rsae_bn_mod_exp(BIGNUM *, const BIGNUM *, const BIGNUM *, + const BIGNUM *, BN_CTX *, BN_MONT_CTX *); +int rsae_init(RSA *); +int rsae_finish(RSA *); +int rsae_sign(int, const u_char *, u_int, + u_char *, u_int *, const RSA *); +int rsae_verify(int dtype, const u_char *m, u_int, + const u_char *, u_int, const RSA *); +int rsae_keygen(RSA *, int, BIGNUM *, BN_GENCB *); + +static struct relayd *env = NULL; +extern int proc_id; + +static struct privsep_proc procs[] = { + { "parent", PROC_PARENT, ca_dispatch_parent }, + { "relay", PROC_RELAY, ca_dispatch_relay }, +}; + +pid_t +ca(struct privsep *ps, struct privsep_proc *p) +{ + env = ps->ps_env; + + return (proc_run(ps, p, procs, nitems(procs), ca_init, NULL)); +} + +void +ca_init(struct privsep *ps, struct privsep_proc *p, void *arg) +{ + if (config_init(ps->ps_env) == -1) + fatal("failed to initialize configuration"); + + proc_id = p->p_instance; + env->sc_id = getpid() & 0xffff; +} + +void +ca_launch(void) +{ + BIO *in = NULL; + EVP_PKEY *pkey = NULL; + struct relay *rlay; + + TAILQ_FOREACH(rlay, env->sc_relays, rl_entry) { + if ((rlay->rl_conf.flags & (F_SSL|F_SSLCLIENT)) == 0) + continue; + + if ((in = BIO_new_mem_buf(rlay->rl_ssl_key, + rlay->rl_conf.ssl_key_len)) == NULL) + fatalx("ca_launch: key"); + + if ((pkey = PEM_read_bio_PrivateKey(in, + NULL, NULL, NULL)) == NULL) + fatalx("ca_launch: PEM"); + + purge_key(&rlay->rl_ssl_key, rlay->rl_conf.ssl_key_len); + purge_key(&rlay->rl_ssl_cert, rlay->rl_conf.ssl_cert_len); + + rlay->rl_ssl_pkey = pkey; + + BIO_free(in); + } +} + +int +ca_dispatch_parent(int fd, struct privsep_proc *p, struct imsg *imsg) +{ + switch (imsg->hdr.type) { + case IMSG_CFG_RELAY: + config_getrelay(env, imsg); + break; + case IMSG_CFG_DONE: + config_getcfg(env, imsg); + break; + case IMSG_CTL_START: + ca_launch(); + break; + case IMSG_CTL_RESET: + config_getreset(env, imsg); + break; + default: + return (-1); + } + + return (0); +} + +static EVP_PKEY * +ca_get_key(objid_t id) +{ + struct relay *rlay; + + if ((rlay = relay_find(env, id)) == NULL) + return (NULL); + + return (rlay->rl_ssl_pkey); +} + +int +ca_dispatch_relay(int fd, struct privsep_proc *p, struct imsg *imsg) +{ + struct ctl_keyop cko; + EVP_PKEY *pkey; + RSA *rsa; + u_char *from = NULL, *to = NULL; + struct iovec iov[2]; + int c = 0; + + switch (imsg->hdr.type) { + case IMSG_CA_PRIVENC: + case IMSG_CA_PRIVDEC: + IMSG_SIZE_CHECK(imsg, (&cko)); + bcopy(imsg->data, &cko, sizeof(cko)); + if (cko.cko_proc > env->sc_prefork_relay) + fatalx("ca_dispatch_relay: " + "invalid relay proc"); + if (IMSG_DATA_SIZE(imsg) != (sizeof(cko) + cko.cko_flen)) + fatalx("ca_dispatch_relay: " + "invalid key operation"); + if ((pkey = ca_get_key(cko.cko_id)) == NULL || + (rsa = EVP_PKEY_get1_RSA(pkey)) == NULL) + fatalx("ca_dispatch_relay: " + "invalid relay key or id"); + + from = (u_char *)imsg->data + sizeof(cko); + if ((to = calloc(1, cko.cko_tlen)) == NULL) + fatalx("ca_dispatch_relay: calloc"); + + switch (imsg->hdr.type) { + case IMSG_CA_PRIVENC: + cko.cko_tlen = RSA_private_encrypt(cko.cko_flen, + from, to, rsa, cko.cko_padding); + break; + case IMSG_CA_PRIVDEC: + cko.cko_tlen = RSA_private_decrypt(cko.cko_flen, + from, to, rsa, cko.cko_padding); + break; + } + + iov[c].iov_base = &cko; + iov[c++].iov_len = sizeof(cko); + if (cko.cko_tlen) { + iov[c].iov_base = to; + iov[c++].iov_len = cko.cko_tlen; + } + + proc_composev_imsg(env->sc_ps, PROC_RELAY, cko.cko_proc, + imsg->hdr.type, -1, iov, c); + + free(to); + RSA_free(rsa); + break; + default: + return (-1); + } + + return (0); +} + +/* + * RSA privsep engine (called from unprivileged processes) + */ + +const RSA_METHOD *rsa_default = NULL; + +static RSA_METHOD rsae_method = { + "RSA privsep engine", + rsae_pub_enc, + rsae_pub_dec, + rsae_priv_enc, + rsae_priv_dec, + rsae_mod_exp, + rsae_bn_mod_exp, + rsae_init, + rsae_finish, + 0, + NULL, + rsae_sign, + rsae_verify, + rsae_keygen +}; + +static int +rsae_send_imsg(int flen, const u_char *from, u_char *to, + RSA *rsa,int padding, u_int cmd) +{ + struct ctl_keyop cko; + int ret = 0; + u_int32_t *id; + struct iovec iov[2]; + struct imsgbuf *ibuf; + struct imsgev *iev; + struct imsg imsg; + int n, done = 0, cnt = 0; + u_char *toptr; + + if ((id = RSA_get_app_data(rsa)) == NULL) + return (0); + + iev = proc_iev(env->sc_ps, PROC_CA, proc_id); + ibuf = &iev->ibuf; + + /* + * XXX this could be nicer... + */ + + cko.cko_id = *id; + cko.cko_proc = proc_id; + cko.cko_flen = flen; + cko.cko_tlen = RSA_size(rsa); + cko.cko_padding = padding; + + iov[cnt].iov_base = &cko; + iov[cnt++].iov_len = sizeof(cko); + iov[cnt].iov_base = from; + iov[cnt++].iov_len = flen; + + /* + * Send a synchronous imsg because we cannot defer the RSA + * operation in OpenSSL's engine layer. + */ + imsg_composev(ibuf, cmd, 0, 0, -1, iov, cnt); + imsg_flush(ibuf); + + while (!done) { + if ((n = imsg_read(ibuf)) == -1) + fatalx("imsg_read"); + if (n == 0) + fatalx("pipe closed"); + + while (!done) { + if ((n = imsg_get(ibuf, &imsg)) == -1) + fatalx("imsg_get error"); + if (n == 0) + break; + if (imsg.hdr.type != cmd) + fatalx("invalid response"); + + IMSG_SIZE_CHECK(&imsg, (&cko)); + memcpy(&cko, imsg.data, sizeof(cko)); + if (IMSG_DATA_SIZE(&imsg) != + (sizeof(cko) + cko.cko_tlen)) + fatalx("data size"); + + ret = cko.cko_tlen; + if (ret) { + toptr = (u_char *)imsg.data + sizeof(cko); + memcpy(to, toptr, ret); + } + done = 1; + + imsg_free(&imsg); + } + } + imsg_event_add(iev); + + return (ret); +} + +int +rsae_pub_enc(int flen,const u_char *from, + u_char *to, RSA *rsa,int padding) +{ + DPRINTF("%s:%d", __func__, __LINE__); + return (rsa_default->rsa_pub_enc(flen, from, to, rsa, padding)); +} + +int +rsae_pub_dec(int flen,const u_char *from, u_char + *to, RSA *rsa,int padding) +{ + DPRINTF("%s:%d", __func__, __LINE__); + return (rsa_default->rsa_pub_dec(flen, from, to, rsa, padding)); +} + +int +rsae_priv_enc(int flen, const u_char *from, u_char *to, + RSA *rsa, int padding) +{ + DPRINTF("%s:%d", __func__, __LINE__); + return (rsae_send_imsg(flen, from, to, rsa, padding, + IMSG_CA_PRIVENC)); +} + +int +rsae_priv_dec(int flen, const u_char *from, u_char *to, + RSA *rsa, int padding) +{ + DPRINTF("%s:%d", __func__, __LINE__); + return (rsae_send_imsg(flen, from, to, rsa, padding, + IMSG_CA_PRIVDEC)); +} + +int +rsae_mod_exp(BIGNUM *r0, const BIGNUM *I, RSA *rsa, BN_CTX *ctx) +{ + DPRINTF("%s:%d", __func__, __LINE__); + return (rsa_default->rsa_mod_exp(r0, I, rsa, ctx)); +} + +int +rsae_bn_mod_exp(BIGNUM *r, const BIGNUM *a, const BIGNUM *p, const + BIGNUM *m, BN_CTX *ctx, BN_MONT_CTX *m_ctx) +{ + DPRINTF("%s:%d", __func__, __LINE__); + return (rsa_default->bn_mod_exp(r, a, p, m, ctx, m_ctx)); +} + +int +rsae_init(RSA *rsa) +{ + DPRINTF("%s:%d", __func__, __LINE__); + if (rsa_default->init == NULL) + return (1); + return (rsa_default->init(rsa)); +} + +int +rsae_finish(RSA *rsa) +{ + DPRINTF("%s:%d", __func__, __LINE__); + if (rsa_default->finish == NULL) + return (1); + return (rsa_default->finish(rsa)); +} + +int +rsae_sign(int type, const u_char *m, u_int + m_length, u_char *sigret, u_int *siglen, const RSA *rsa) +{ + DPRINTF("%s:%d", __func__, __LINE__); + return (rsa_default->rsa_sign(type, m, m_length, sigret, siglen, rsa)); +} + +int +rsae_verify(int dtype, const u_char *m, u_int m_length, + const u_char *sigbuf, u_int siglen, const RSA *rsa) +{ + DPRINTF("%s:%d", __func__, __LINE__); + return (rsa_default->rsa_verify(dtype, m, m_length, sigbuf, siglen, rsa)); +} + +int +rsae_keygen(RSA *rsa, int bits, BIGNUM *e, BN_GENCB *cb) +{ + DPRINTF("%s:%d", __func__, __LINE__); + return (rsa_default->rsa_keygen(rsa, bits, e, cb)); +} + +int +ca_engine_init(struct relayd *x_env) +{ + ENGINE *e; + + if (env == NULL) + env = x_env; + + if ((e = ENGINE_get_default_RSA()) == NULL || + (rsa_default = ENGINE_get_RSA(e)) == NULL) + return (-1); + + if (rsa_default->rsa_mod_exp == NULL) + rsae_method.rsa_mod_exp = NULL; + if (rsa_default->rsa_mod_exp == NULL) + rsae_method.rsa_mod_exp = NULL; + if (rsa_default->bn_mod_exp == NULL) + rsae_method.bn_mod_exp = NULL; + if (rsa_default->rsa_keygen == NULL) + rsae_method.rsa_keygen = NULL; + rsae_method.flags = rsa_default->flags | + RSA_METHOD_FLAG_NO_CHECK; + rsae_method.app_data = rsa_default->app_data; + + if (!ENGINE_set_RSA(e, &rsae_method) || + !ENGINE_set_default_RSA(e)) + return (-1); + + return (0); +} Index: config.c =================================================================== RCS file: /cvs/src/usr.sbin/relayd/config.c,v retrieving revision 1.11 diff -u -p -u -p -r1.11 config.c --- config.c 24 Feb 2014 06:55:11 -0000 1.11 +++ config.c 17 Apr 2014 19:26:59 -0000 @@ -1,7 +1,7 @@ /* $OpenBSD: config.c,v 1.11 2014/02/24 06:55:11 jsg Exp $ */ /* - * Copyright (c) 2011 Reyk Floeter <[email protected]> + * Copyright (c) 2011 - 2014 Reyk Floeter <[email protected]> * * Permission to use, copy, modify, and distribute this software for any * purpose with or without fee is hereby granted, provided that the above @@ -64,8 +64,9 @@ config_init(struct relayd *env) ps->ps_what[PROC_PARENT] = CONFIG_ALL; ps->ps_what[PROC_PFE] = CONFIG_ALL & ~CONFIG_PROTOS; ps->ps_what[PROC_HCE] = CONFIG_TABLES; - ps->ps_what[PROC_RELAY] = - CONFIG_TABLES|CONFIG_RELAYS|CONFIG_PROTOS; + ps->ps_what[PROC_CA] = CONFIG_RELAYS; + ps->ps_what[PROC_RELAY] = CONFIG_RELAYS| + CONFIG_TABLES|CONFIG_PROTOS|CONFIG_CA_ENGINE; } /* Other configuration */ @@ -245,6 +246,7 @@ config_getcfg(struct relayd *env, struct struct table *tb; struct host *h, *ph; struct ctl_flags cf; + u_int what; if (IMSG_DATA_SIZE(imsg) != sizeof(cf)) return (0); /* ignore */ @@ -254,7 +256,9 @@ config_getcfg(struct relayd *env, struct env->sc_opts = cf.cf_opts; env->sc_flags = cf.cf_flags; - if (ps->ps_what[privsep_process] & CONFIG_TABLES) { + what = ps->ps_what[privsep_process]; + + if (what & CONFIG_TABLES) { /* Update the tables */ TAILQ_FOREACH(tb, env->sc_tables, entry) { TAILQ_FOREACH(h, &tb->hosts, entry) { @@ -267,8 +271,12 @@ config_getcfg(struct relayd *env, struct } } - if (env->sc_flags & (F_SSL|F_SSLCLIENT)) + if (env->sc_flags & (F_SSL|F_SSLCLIENT)) { ssl_init(env); + if ((what & CONFIG_CA_ENGINE) && + (ca_engine_init(env)) == -1) + fatal("CA engine failed"); + } if (privsep_process != PROC_PARENT) proc_compose_imsg(env->sc_ps, PROC_PARENT, -1, @@ -802,45 +810,52 @@ config_setrelay(struct relayd *env, stru struct privsep *ps = env->sc_ps; struct ctl_relaytable crt; struct relay_table *rlt; + struct relay_config rl; int id; int fd, n, m; struct iovec iov[6]; size_t c; + u_int what; /* opens listening sockets etc. */ if (relay_privinit(rlay) == -1) return (-1); for (id = 0; id < PROC_MAX; id++) { - if ((ps->ps_what[id] & CONFIG_RELAYS) == 0 || - id == privsep_process) + what = ps->ps_what[id]; + + if ((what & CONFIG_RELAYS) == 0 || id == privsep_process) continue; DPRINTF("%s: sending relay %s to %s fd %d", __func__, rlay->rl_conf.name, ps->ps_title[id], rlay->rl_s); + memcpy(&rl, &rlay->rl_conf, sizeof(rl)); + c = 0; - iov[c].iov_base = &rlay->rl_conf; - iov[c++].iov_len = sizeof(rlay->rl_conf); - if (rlay->rl_conf.ssl_cert_len) { + iov[c].iov_base = &rl; + iov[c++].iov_len = sizeof(rl); + if (rl.ssl_cert_len) { iov[c].iov_base = rlay->rl_ssl_cert; - iov[c++].iov_len = rlay->rl_conf.ssl_cert_len; + iov[c++].iov_len = rl.ssl_cert_len; } - if (rlay->rl_conf.ssl_key_len) { + if ((what & CONFIG_CA_ENGINE) == 0 && + rl.ssl_key_len) { iov[c].iov_base = rlay->rl_ssl_key; - iov[c++].iov_len = rlay->rl_conf.ssl_key_len; - } - if (rlay->rl_conf.ssl_ca_len) { + iov[c++].iov_len = rl.ssl_key_len; + } else + rl.ssl_key_len = 0; + if (rl.ssl_ca_len) { iov[c].iov_base = rlay->rl_ssl_ca; - iov[c++].iov_len = rlay->rl_conf.ssl_ca_len; + iov[c++].iov_len = rl.ssl_ca_len; } - if (rlay->rl_conf.ssl_cacert_len) { + if (rl.ssl_cacert_len) { iov[c].iov_base = rlay->rl_ssl_cacert; - iov[c++].iov_len = rlay->rl_conf.ssl_cacert_len; + iov[c++].iov_len = rl.ssl_cacert_len; } - if (rlay->rl_conf.ssl_cakey_len) { + if (rl.ssl_cakey_len) { iov[c].iov_base = rlay->rl_ssl_cakey; - iov[c++].iov_len = rlay->rl_conf.ssl_cakey_len; + iov[c++].iov_len = rl.ssl_cakey_len; } if (id == PROC_RELAY) { @@ -857,6 +872,9 @@ config_setrelay(struct relayd *env, stru proc_composev_imsg(ps, id, -1, IMSG_CFG_RELAY, -1, iov, c); } + + if ((what & CONFIG_TABLES) == 0) + continue; /* Now send the tables associated to this relay */ TAILQ_FOREACH(rlt, &rlay->rl_tables, rlt_entry) { Index: proc.c =================================================================== RCS file: /cvs/src/usr.sbin/relayd/proc.c,v retrieving revision 1.8 diff -u -p -u -p -r1.8 proc.c --- proc.c 14 Apr 2014 07:18:05 -0000 1.8 +++ proc.c 17 Apr 2014 19:27:00 -0000 @@ -1,7 +1,7 @@ /* $OpenBSD: proc.c,v 1.8 2014/04/14 07:18:05 blambert Exp $ */ /* - * Copyright (c) 2010,2011 Reyk Floeter <[email protected]> + * Copyright (c) 2010 - 2014 Reyk Floeter <[email protected]> * Copyright (c) 2008 Pierre-Yves Ritschard <[email protected]> * * Permission to use, copy, modify, and distribute this software for any @@ -48,6 +48,7 @@ int proc_ispeer(struct privsep_proc *, void proc_shutdown(struct privsep_proc *); void proc_sig_handler(int, short, void *); void proc_range(struct privsep *, enum privsep_procid, int *, int *); +u_int proc_instances(struct privsep *, u_int, u_int); int proc_ispeer(struct privsep_proc *p, u_int nproc, enum privsep_procid type) @@ -116,19 +117,9 @@ proc_setup(struct privsep *ps) for (i = 0; i < PROC_MAX; i++) for (j = 0; j < PROC_MAX; j++) { - /* - * find out how many instances of this peer there are. - */ - if (i >= j || ps->ps_instances[i] == 0 || - ps->ps_instances[j] == 0) + if ((count = proc_instances(ps, i, j)) == 0) continue; - if (ps->ps_instances[i] > 1 && - ps->ps_instances[j] > 1) - fatalx("N:M peering not supported"); - - count = ps->ps_instances[i] * ps->ps_instances[j]; - if ((ps->ps_pipes[i][j] = calloc(count, sizeof(int))) == NULL || (ps->ps_pipes[j][i] = @@ -165,13 +156,9 @@ proc_config(struct privsep *ps, struct p */ for (i = 0; i < PROC_MAX; i++) { for (j = 0; j < PROC_MAX; j++) { - if (i == j || - ps->ps_instances[i] == 0 || - ps->ps_instances[j] == 0) + if ((count = proc_instances(ps, i, j)) == 0) continue; - count = ps->ps_instances[i] * ps->ps_instances[j]; - for (n = 0; n < count; n++) { instance = ps->ps_instances[i] > 1 ? n : 0; if (i == src && @@ -197,7 +184,8 @@ proc_config(struct privsep *ps, struct p if (src == dst) fatal("proc_config: cannot peer with oneself"); - count = ps->ps_instances[src] * ps->ps_instances[dst]; + if ((count = proc_instances(ps, src, dst)) == 0) + continue; if ((ps->ps_ievs[dst] = calloc(count, sizeof(struct imsgev))) == NULL) @@ -234,11 +222,10 @@ proc_clear(struct privsep *ps, int purge return; for (dst = 0; dst < PROC_MAX; dst++) { - if (src == dst || ps->ps_ievs[dst] == NULL) + if ((count = proc_instances(ps, src, dst)) == 0 || + ps->ps_ievs[dst] == NULL) continue; - count = ps->ps_instances[src] * ps->ps_instances[dst]; - for (n = 0; n < count; n++) { if (ps->ps_pipes[src][dst][n] == -1) continue; @@ -254,6 +241,26 @@ proc_clear(struct privsep *ps, int purge } } +u_int +proc_instances(struct privsep *ps, u_int src, u_int dst) +{ + u_int count; + + if (ps->ps_instances[src] > 1 && + ps->ps_instances[dst] > 1 && + ps->ps_instances[src] != ps->ps_instances[dst]) + fatalx("N:M peering not supported"); + + if (src == dst || + ps->ps_instances[src] == 0 || + ps->ps_instances[dst] == 0) + return (0); + + count = MAX(ps->ps_instances[src], ps->ps_instances[dst]); + + return (count); +} + void proc_shutdown(struct privsep_proc *p) { @@ -596,4 +603,13 @@ proc_ibuf(struct privsep *ps, enum privs proc_range(ps, id, &n, &m); return (&ps->ps_ievs[id][n].ibuf); +} + +struct imsgev * +proc_iev(struct privsep *ps, enum privsep_procid id, int n) +{ + int m; + + proc_range(ps, id, &n, &m); + return (&ps->ps_ievs[id][n]); } Index: relay.c =================================================================== RCS file: /cvs/src/usr.sbin/relayd/relay.c,v retrieving revision 1.167 diff -u -p -u -p -r1.167 relay.c --- relay.c 9 Sep 2013 17:57:44 -0000 1.167 +++ relay.c 17 Apr 2014 19:27:02 -0000 @@ -1,7 +1,7 @@ /* $OpenBSD: relay.c,v 1.167 2013/09/09 17:57:44 reyk Exp $ */ /* - * Copyright (c) 2006 - 2013 Reyk Floeter <[email protected]> + * Copyright (c) 2006 - 2014 Reyk Floeter <[email protected]> * * Permission to use, copy, modify, and distribute this software for any * purpose with or without fee is hereby granted, provided that the above @@ -52,6 +52,8 @@ int relay_dispatch_parent(int, struct struct imsg *); int relay_dispatch_pfe(int, struct privsep_proc *, struct imsg *); +int relay_dispatch_ca(int, struct privsep_proc *, + struct imsg *); void relay_shutdown(void); void relay_nodedebug(const char *, struct protonode *); @@ -96,6 +98,7 @@ int proc_id; static struct privsep_proc procs[] = { { "parent", PROC_PARENT, relay_dispatch_parent }, { "pfe", PROC_PFE, relay_dispatch_pfe }, + { "ca", PROC_CA, relay_dispatch_ca } }; pid_t @@ -1782,6 +1785,12 @@ relay_dispatch_pfe(int fd, struct privse } int +relay_dispatch_ca(int fd, struct privsep_proc *p, struct imsg *imsg) +{ + return (-1); +} + +int relay_dispatch_parent(int fd, struct privsep_proc *p, struct imsg *imsg) { struct rsession *con; @@ -1904,9 +1913,11 @@ relay_ssl_ctx_create(struct relay *rlay) goto err; log_debug("%s: loading private key", __func__); - if (!ssl_ctx_use_private_key(ctx, rlay->rl_ssl_key, - rlay->rl_conf.ssl_key_len)) + if (!ssl_ctx_fake_private_key(ctx, + &rlay->rl_conf.id, rlay->rl_ssl_cert, rlay->rl_conf.ssl_cert_len, + &rlay->rl_ssl_x509, &rlay->rl_ssl_pkey)) goto err; + if (!SSL_CTX_check_private_key(ctx)) goto err; @@ -1915,6 +1926,9 @@ relay_ssl_ctx_create(struct relay *rlay) strlen(rlay->rl_conf.name))) goto err; + /* The text versions of the keys/certs are not needed anymore */ + purge_key(&rlay->rl_ssl_cert, rlay->rl_conf.ssl_cert_len); + return (ctx); err: @@ -2087,7 +2101,7 @@ relay_ssl_connect(int fd, short event, v SSL_get_peer_certificate(con->se_out.ssl)) != NULL) { con->se_in.sslcert = ssl_update_certificate(servercert, - rlay->rl_ssl_key, rlay->rl_conf.ssl_key_len, + rlay->rl_ssl_pkey, rlay->rl_ssl_cakey, rlay->rl_conf.ssl_cakey_len, rlay->rl_ssl_cacert, rlay->rl_conf.ssl_cacert_len); } else Index: relayd.c =================================================================== RCS file: /cvs/src/usr.sbin/relayd/relayd.c,v retrieving revision 1.119 diff -u -p -u -p -r1.119 relayd.c --- relayd.c 16 Mar 2014 18:38:30 -0000 1.119 +++ relayd.c 17 Apr 2014 19:27:04 -0000 @@ -1,7 +1,7 @@ /* $OpenBSD: relayd.c,v 1.119 2014/03/16 18:38:30 guenther Exp $ */ /* - * Copyright (c) 2007, 2008 Reyk Floeter <[email protected]> + * Copyright (c) 2007 - 2014 Reyk Floeter <[email protected]> * Copyright (c) 2006 Pierre-Yves Ritschard <[email protected]> * * Permission to use, copy, modify, and distribute this software for any @@ -57,6 +57,8 @@ int parent_dispatch_pfe(int, struct pr int parent_dispatch_hce(int, struct privsep_proc *, struct imsg *); int parent_dispatch_relay(int, struct privsep_proc *, struct imsg *); +int parent_dispatch_ca(int, struct privsep_proc *, + struct imsg *); int bindany(struct ctl_bindany *); struct relayd *relayd_env; @@ -64,7 +66,8 @@ struct relayd *relayd_env; static struct privsep_proc procs[] = { { "pfe", PROC_PFE, parent_dispatch_pfe, pfe }, { "hce", PROC_HCE, parent_dispatch_hce, hce }, - { "relay", PROC_RELAY, parent_dispatch_relay, relay } + { "relay", PROC_RELAY, parent_dispatch_relay, relay }, + { "ca", PROC_CA, parent_dispatch_ca, ca } }; void @@ -222,6 +225,7 @@ main(int argc, char *argv[]) log_info("startup"); ps->ps_instances[PROC_RELAY] = env->sc_prefork_relay; + ps->ps_instances[PROC_CA] = env->sc_prefork_relay; proc_init(ps, procs, nitems(procs)); setproctitle("parent"); @@ -300,8 +304,8 @@ parent_configure(struct relayd *env) config_setrelay(env, rlay); } - /* HCE, PFE and the preforked relays need to reload their config. */ - env->sc_reload = 2 + env->sc_prefork_relay; + /* HCE, PFE, CA and the relays need to reload their config. */ + env->sc_reload = 2 + (2 * env->sc_prefork_relay); for (id = 0; id < PROC_MAX; id++) { if (id == privsep_process) @@ -326,7 +330,7 @@ parent_configure(struct relayd *env) ret = 0; done: - config_purge(env, CONFIG_ALL); + config_purge(env, CONFIG_ALL & ~CONFIG_RELAYS); return (ret); } @@ -513,6 +517,22 @@ parent_dispatch_relay(int fd, struct pri return (0); } +int +parent_dispatch_ca(int fd, struct privsep_proc *p, struct imsg *imsg) +{ + struct relayd *env = p->p_env; + + switch (imsg->hdr.type) { + case IMSG_CFG_DONE: + parent_configure_done(env); + break; + default: + return (-1); + } + + return (0); +} + void purge_tree(struct proto_tree *tree) { @@ -568,6 +588,20 @@ purge_table(struct tablelist *head, stru } void +purge_key(char **ptr, off_t len) +{ + char *key = *ptr; + + if (key == NULL || len == 0) + return; + + explicit_bzero(key, len); + free(key); + + *ptr = NULL; +} + +void purge_relay(struct relayd *env, struct relay *rlay) { struct rsession *con; @@ -590,14 +624,22 @@ purge_relay(struct relayd *env, struct r if (rlay->rl_dstbev != NULL) bufferevent_free(rlay->rl_dstbev); + purge_key(&rlay->rl_ssl_cert, rlay->rl_conf.ssl_cert_len); + purge_key(&rlay->rl_ssl_key, rlay->rl_conf.ssl_key_len); + purge_key(&rlay->rl_ssl_ca, rlay->rl_conf.ssl_ca_len); + purge_key(&rlay->rl_ssl_cakey, rlay->rl_conf.ssl_cakey_len); + + if (rlay->rl_ssl_x509 != NULL) { + X509_free(rlay->rl_ssl_x509); + rlay->rl_ssl_x509 = NULL; + } + if (rlay->rl_ssl_pkey != NULL) { + EVP_PKEY_free(rlay->rl_ssl_pkey); + rlay->rl_ssl_pkey = NULL; + } + if (rlay->rl_ssl_ctx != NULL) SSL_CTX_free(rlay->rl_ssl_ctx); - if (rlay->rl_ssl_cert != NULL) - free(rlay->rl_ssl_cert); - if (rlay->rl_ssl_key != NULL) - free(rlay->rl_ssl_key); - if (rlay->rl_ssl_ca != NULL) - free(rlay->rl_ssl_ca); while ((rlt = TAILQ_FIRST(&rlay->rl_tables))) { TAILQ_REMOVE(&rlay->rl_tables, rlt, rlt_entry); Index: relayd.conf.5 =================================================================== RCS file: /cvs/src/usr.sbin/relayd/relayd.conf.5,v retrieving revision 1.141 diff -u -p -u -p -r1.141 relayd.conf.5 --- relayd.conf.5 14 Apr 2014 15:24:25 -0000 1.141 +++ relayd.conf.5 17 Apr 2014 19:27:06 -0000 @@ -145,7 +145,7 @@ relayed connections. This increases the performance and prevents delays when connecting to a relay. .Xr relayd 8 -runs 5 relay processes by default and every process will handle +runs 3 relay processes by default and every process will handle all configured relays. .It Ic snmp Oo Ic trap Oc Op Qq Ar path Send an SNMP trap when the state of a host changes. Index: relayd.h =================================================================== RCS file: /cvs/src/usr.sbin/relayd/relayd.h,v retrieving revision 1.173 diff -u -p -u -p -r1.173 relayd.h --- relayd.h 14 Apr 2014 12:58:04 -0000 1.173 +++ relayd.h 17 Apr 2014 19:27:07 -0000 @@ -1,7 +1,7 @@ /* $OpenBSD: relayd.h,v 1.173 2014/04/14 12:58:04 blambert Exp $ */ /* - * Copyright (c) 2006 - 2012 Reyk Floeter <[email protected]> + * Copyright (c) 2006 - 2014 Reyk Floeter <[email protected]> * Copyright (c) 2006, 2007 Pierre-Yves Ritschard <[email protected]> * Copyright (c) 2003, 2004 Henning Brauer <[email protected]> * @@ -54,7 +54,7 @@ #define RELAY_MAX_SESSIONS 1024 #define RELAY_TIMEOUT 600 #define RELAY_CACHESIZE -1 /* use default size */ -#define RELAY_NUMPROC 5 +#define RELAY_NUMPROC 3 #define RELAY_MAXPROC 32 #define RELAY_MAXHOSTS 32 #define RELAY_STATINTERVAL 60 @@ -69,6 +69,7 @@ #define CONFIG_PROTOS 0x08 #define CONFIG_ROUTES 0x10 #define CONFIG_RTS 0x20 +#define CONFIG_CA_ENGINE 0x40 #define CONFIG_ALL 0xff #define SMALL_READ_BUF_SIZE 1024 @@ -229,6 +230,14 @@ struct ctl_bindany { int bnd_proto; }; +struct ctl_keyop { + objid_t cko_id; + int cko_proc; + int cko_flen; + int cko_tlen; + int cko_padding; +}; + struct ctl_stats { objid_t id; int proc; @@ -657,8 +666,13 @@ struct relay { struct event rl_evt; SSL_CTX *rl_ssl_ctx; + char *rl_ssl_cert; + X509 *rl_ssl_x509; + char *rl_ssl_key; + EVP_PKEY *rl_ssl_pkey; + char *rl_ssl_ca; char *rl_ssl_cacert; char *rl_ssl_cakey; @@ -828,7 +842,9 @@ enum imsg_type { IMSG_CFG_PROTONODE, IMSG_CFG_RELAY, IMSG_CFG_RELAY_TABLE, - IMSG_CFG_DONE + IMSG_CFG_DONE, + IMSG_CA_PRIVENC, + IMSG_CA_PRIVDEC }; enum privsep_procid { @@ -837,6 +853,7 @@ enum privsep_procid { PROC_HCE, PROC_RELAY, PROC_PFE, + PROC_CA, PROC_MAX } privsep_process; @@ -1068,14 +1085,19 @@ void ssl_transaction(struct ctl_tcp_eve SSL_CTX *ssl_ctx_create(struct relayd *); void ssl_error(const char *, const char *); char *ssl_load_key(struct relayd *, const char *, off_t *, char *); -X509 *ssl_update_certificate(X509 *, char *, off_t, +X509 *ssl_update_certificate(X509 *, EVP_PKEY *, char *, off_t, char *, off_t); +int ssl_ctx_fake_private_key(SSL_CTX *, void *, char *, off_t, + X509 **, EVP_PKEY **); /* ssl_privsep.c */ -int ssl_ctx_use_private_key(SSL_CTX *, char *, off_t); int ssl_ctx_use_certificate_chain(SSL_CTX *, char *, off_t); int ssl_ctx_load_verify_memory(SSL_CTX *, char *, off_t); +/* ca.c */ +pid_t ca(struct privsep *, struct privsep_proc *); +int ca_engine_init(struct relayd *); + /* relayd.c */ struct host *host_find(struct relayd *, objid_t); struct table *table_find(struct relayd *, objid_t); @@ -1096,6 +1118,7 @@ struct relay *relay_findbyname(struct re struct relay *relay_findbyaddr(struct relayd *, struct relay_config *); int expand_string(char *, size_t, const char *, const char *); void translate_string(char *); +void purge_key(char **, off_t); void purge_tree(struct proto_tree *); void purge_table(struct tablelist *, struct table *); void purge_relay(struct relayd *, struct relay *); @@ -1170,6 +1193,8 @@ int proc_forward_imsg(struct privsep *, enum privsep_procid, int); struct imsgbuf * proc_ibuf(struct privsep *, enum privsep_procid, int); +struct imsgev * + proc_iev(struct privsep *, enum privsep_procid, int); void imsg_event_add(struct imsgev *); int imsg_compose_event(struct imsgev *, u_int16_t, u_int32_t, pid_t, int, void *, u_int16_t); Index: ssl.c =================================================================== RCS file: /cvs/src/usr.sbin/relayd/ssl.c,v retrieving revision 1.19 diff -u -p -u -p -r1.19 ssl.c --- ssl.c 31 May 2013 20:23:37 -0000 1.19 +++ ssl.c 17 Apr 2014 19:27:07 -0000 @@ -1,7 +1,7 @@ /* $OpenBSD: ssl.c,v 1.19 2013/05/31 20:23:37 benno Exp $ */ /* - * Copyright (c) 2007-2013 Reyk Floeter <[email protected]> + * Copyright (c) 2007 - 2014 Reyk Floeter <[email protected]> * Copyright (c) 2006 Pierre-Yves Ritschard <[email protected]> * * Permission to use, copy, modify, and distribute this software for any @@ -359,12 +359,12 @@ ssl_load_key(struct relayd *env, const c } X509 * -ssl_update_certificate(X509 *oldcert, char *keystr, off_t keylen, +ssl_update_certificate(X509 *oldcert, EVP_PKEY *pkey, char *cakeystr, off_t cakeylen, char *cacertstr, off_t cacertlen) { char name[2][SSL_NAME_SIZE]; X509 *cert = NULL, *cacert = NULL; - EVP_PKEY *key = NULL, *cakey = NULL; + EVP_PKEY *cakey = NULL; BIO *bio = NULL; name[0][0] = name[1][0] = '\0'; @@ -374,13 +374,6 @@ ssl_update_certificate(X509 *oldcert, ch name[1], sizeof(name[1]))) goto done; - /* Get SSL key */ - if ((bio = BIO_new_mem_buf(keystr, keylen)) == NULL) - goto done; - if ((key = PEM_read_bio_PrivateKey(bio, &key, - ssl_password_cb, NULL)) == NULL) - goto done; - /* Get CA key */ BIO_free_all(bio); if ((bio = BIO_new_mem_buf(cakeystr, cakeylen)) == NULL) @@ -401,7 +394,7 @@ ssl_update_certificate(X509 *oldcert, ch goto done; /* Update certificate key and use our CA as the issuer */ - X509_set_pubkey(cert, key); + X509_set_pubkey(cert, pkey); X509_set_issuer_name(cert, X509_get_subject_name(cacert)); /* Sign with our CA */ @@ -423,12 +416,74 @@ ssl_update_certificate(X509 *oldcert, ch ssl_error(__func__, name[0]); if (bio != NULL) BIO_free_all(bio); - if (key != NULL) - EVP_PKEY_free(key); if (cacert != NULL) X509_free(cacert); if (cakey != NULL) EVP_PKEY_free(cakey); return (cert); +} + +int +ssl_ctx_fake_private_key(SSL_CTX *ctx, void *data, char *buf, off_t len, + X509 **x509ptr, EVP_PKEY **pkeyptr) +{ + int ret = 0; + BIO *in; + X509 *x509 = NULL; + EVP_PKEY *pkey = NULL; + RSA *rsa = NULL; + + if ((in = BIO_new_mem_buf(buf, len)) == NULL) { + SSLerr(SSL_F_SSL_CTX_USE_PRIVATEKEY, ERR_R_BUF_LIB); + return (0); + } + + if ((x509 = PEM_read_bio_X509(in, NULL, + ctx->default_passwd_callback, + ctx->default_passwd_callback_userdata)) == NULL) { + SSLerr(SSL_F_SSL_CTX_USE_PRIVATEKEY, ERR_R_PEM_LIB); + goto fail; + } + + if ((pkey = X509_get_pubkey(x509)) == NULL) { + SSLerr(SSL_F_SSL_CTX_USE_PRIVATEKEY, ERR_R_X509_LIB); + goto fail; + } + + if ((rsa = EVP_PKEY_get1_RSA(pkey)) == NULL) { + SSLerr(SSL_F_SSL_CTX_USE_PRIVATEKEY, ERR_R_EVP_LIB); + goto fail; + } + + RSA_set_app_data(rsa, data); + + /* + * Use the public key as the "private" key - the secret key + * parameters are hidden in an extra process that will be + * contacted by the RSA engine. The SSL/TLS library needs at + * least the public key parameters in the current process. + */ + if (!SSL_CTX_use_PrivateKey(ctx, pkey)) { + SSLerr(SSL_F_SSL_CTX_USE_PRIVATEKEY, ERR_R_SSL_LIB); + goto fail; + } + + *x509ptr = x509; + *pkeyptr = pkey; + ret = 1; + + goto done; + + fail: + if (pkey != NULL) + EVP_PKEY_free(pkey); + if (x509 != NULL) + X509_free(x509); + + done: + if (in != NULL) + BIO_free(in); + + return ret; } Index: ssl_privsep.c =================================================================== RCS file: /cvs/src/usr.sbin/relayd/ssl_privsep.c,v retrieving revision 1.9 diff -u -p -u -p -r1.9 ssl_privsep.c --- ssl_privsep.c 4 Oct 2012 20:53:30 -0000 1.9 +++ ssl_privsep.c 17 Apr 2014 19:27:07 -0000 @@ -76,7 +76,6 @@ #include <openssl/pem.h> #include <openssl/ssl.h> -int ssl_ctx_use_private_key(SSL_CTX *, char *, off_t); int ssl_ctx_use_certificate_chain(SSL_CTX *, char *, off_t); int ssl_ctx_load_verify_memory(SSL_CTX *, char *, off_t); int ssl_by_mem_ctrl(X509_LOOKUP *, int, const char *, long, char **); @@ -95,37 +94,6 @@ X509_LOOKUP_METHOD x509_mem_lookup = { }; #define X509_L_ADD_MEM 3 - -int -ssl_ctx_use_private_key(SSL_CTX *ctx, char *buf, off_t len) -{ - int ret; - BIO *in; - EVP_PKEY *pkey; - - ret = 0; - - if ((in = BIO_new_mem_buf(buf, len)) == NULL) { - SSLerr(SSL_F_SSL_CTX_USE_PRIVATEKEY_FILE, ERR_R_BUF_LIB); - return 0; - } - - pkey = PEM_read_bio_PrivateKey(in, NULL, - ctx->default_passwd_callback, - ctx->default_passwd_callback_userdata); - - if (pkey == NULL) { - SSLerr(SSL_F_SSL_CTX_USE_PRIVATEKEY_FILE, ERR_R_PEM_LIB); - goto end; - } - ret = SSL_CTX_use_PrivateKey(ctx, pkey); - EVP_PKEY_free(pkey); -end: - if (in != NULL) - BIO_free(in); - return ret; -} - int ssl_ctx_use_certificate_chain(SSL_CTX *ctx, char *buf, off_t len)
