[PATCH] PKCS#11 key support provider & engine (rev. 2)
Diff attached.
Regards
Richard
diff --git a/include/haproxy/ssl_ckch-t.h b/include/haproxy/ssl_ckch-t.h
index 0002b84d9..2470c45b2 100644
--- a/include/haproxy/ssl_ckch-t.h
+++ b/include/haproxy/ssl_ckch-t.h
@@ -56,6 +56,13 @@ struct ckch_data {
X509 *ocsp_issuer;
OCSP_CERTID *ocsp_cid;
int ocsp_update_mode;
+#if defined(USE_ENGINE) && !defined(OPENSSL_NO_ENGINE)
+ char *engine;
+ char *private_key;
+#endif
+ int len;
+ char *pem_data;
+ EVP_PKEY *worker_key;
};
/*
diff --git a/src/ssl_ckch.c b/src/ssl_ckch.c
index b69caacb3..0f6ab6a0a 100644
--- a/src/ssl_ckch.c
+++ b/src/ssl_ckch.c
@@ -516,11 +516,45 @@ int ssl_sock_load_files_into_ckch(const char *path, struct ckch_data *data, char
*
* Return 0 on success or != 0 on failure
*/
+
+/*
+ * This caches the BIO data in case the private key i
+ *
+ * to be instantiated in the child process (used for HSM keys)
+ *
+ * The key ckch_data* and the value is ckch_data_cache*
+ *
+ */
+static void ckch_data_put(struct ckch_data *cur, char *data, int len)
+{
+ cur->pem_data = malloc(len);
+ memcpy(cur->pem_data, data, len);
+ cur->len = len;
+}
+
+#if defined(USE_ENGINE) && !defined(OPENSSL_NO_ENGINE)
+static void engine_data_put(struct ckch_data *cur, char *engine, char *private_key)
+{
+ cur->engine = strdup(engine);
+ cur->private_key = strdup(private_key);
+ cur->len = -1;
+}
+#endif
+
int ssl_sock_load_key_into_ckch(const char *path, char *buf, struct ckch_data *data , char **err)
{
BIO *in = NULL;
int ret = 1;
EVP_PKEY *key = NULL;
+ static char src[16384];
+ char *src_temp;
+ int len;
+ int filetype = 0;
+#if defined(USE_ENGINE) && !defined(OPENSSL_NO_ENGINE)
+ CONF *conf = NULL;
+ BIO *conf_in = NULL;
+ ENGINE *eng;
+#endif
if (buf) {
/* reading from a buffer */
@@ -538,16 +572,67 @@ int ssl_sock_load_key_into_ckch(const char *path, char *buf, struct ckch_data *d
if (BIO_read_filename(in, path) <= 0)
goto end;
+ filetype = 1;
}
/* Read Private Key */
+ if(filetype) {
+ len = BIO_read(in, src, sizeof(src));
+ BIO_reset(in);
+ } else {
+ len = BIO_get_mem_data(in, _temp);
+ memcpy(src, src_temp, len);
+ }
+
key = PEM_read_bio_PrivateKey(in, NULL, NULL, NULL);
+#if OPENSSL_VERSION_NUMBER >= 0x03000UL
+ data->len = 0;
+ if(key && global.mode_MWORKER) {
+ ckch_data_put(data, src, len);
+ }
+#endif
+ if(key)
+ goto ok;
+#if defined(USE_ENGINE) && !defined(OPENSSL_NO_ENGINE)
+ /* ENGINE should be loaded in global config */
+ conf = NCONF_new(NCONF_default());
+ if (conf == NULL)
+ goto end;
+ conf_in = BIO_new_mem_buf(src, len);
+ if (conf_in == NULL)
+ goto end;
+ ret = NCONF_load_bio(conf, conf_in, NULL);
+ if (ret<=0)
+ goto end;
+ char *eng_id = NCONF_get_string(conf, NULL, "engine");
+ char *private_key = NCONF_get_string(conf, NULL, "private_key");
+
+ ha_notice("ENGINE private_key = %s\n", private_key);
+ ha_notice("ENGINE engine_id = %s\n", eng_id);
+
+ eng = ENGINE_by_id(eng_id);
+ if (eng == NULL)
+ goto end;
+ else
+ fprintf(stderr, "ENGINE %s is successfully loaded!\n", eng_id);
+ key = ENGINE_load_private_key(eng, private_key, NULL, NULL);
+ if (key)
+ fprintf(stderr, "private key %s is successfully loaded!\n", private_key);
+
+ data->len = 0;
+ if(key && global.mode_MWORKER) {
+ ha_notice("storing key = %p engine_id = %s private_key = %s\n",
+ data, eng_id, private_key);
+ engine_data_put(data, eng_id, private_key);
+ }
+#endif
if (key == NULL) {
memprintf(err, "%sunable to load private key from file '%s'.\n",
err && *err ? *err : "", path);
goto end;
}
+ok:
ret = 0;
SWAP(data->key, key);
@@ -560,6 +645,12 @@ int ssl_sock_load_key_into_ckch(const char *path, char *buf, struct ckch_data *d
if (key)
EVP_PKEY_free(key);
+#if defined(USE_ENGINE) && !defined(OPENSSL_NO_ENGINE)
+ if (conf)
+ NCONF_free(conf);
+ if (conf_in)
+ BIO_free(conf_in);
+#endif
return ret;
}
@@ -581,6 +672,10 @@ int ssl_sock_load_pem_into_ckch(const char *path, char *buf, struct ckch_data *d
EVP_PKEY *key = NULL;
HASSL_DH *dh = NULL;
STACK_OF(X509) *chain = NULL;
+ int filetype = 0;
+ int len;
+ static char src[16384];
+ char *src_temp;
if (buf) {
/* reading from a buffer */
@@ -603,11 +698,24 @@ int ssl_sock_load_pem_into_ckch(const char *path, char *buf, struct ckch_data *d
err && *err ? *err : "", path);
goto end;
}
+ filetype = 1;
}
/* Read Private Key */
+ if(filetype) {
+ len = BIO_read(in, src, sizeof(src));
+ BIO_reset(in);
+ } else {
+ len = BIO_get_mem_data(in, _temp);
+ memcpy(src, src_temp, len);
+ }
+
key = PEM_read_bio_PrivateKey(in, NULL, NULL, NULL);
/* no need to check for errors here, because the private key could be loaded later */
+ data->len = 0;
+ if(key && global.mode_MWORKER) {
+ ckch_data_put(data, src, len);
+ }
#ifndef OPENSSL_NO_DH
/* Seek back to beginning of file */
diff --git a/src/ssl_sock.c b/src/ssl_sock.c
index 824bdaa72..5034745c8 100644
--- a/src/ssl_sock.c
+++
[RFC] PKCS#11 key support provider & engine (rev. 2) - GH#71
Hello list, [RFC] PKCS#11 key support provider & engine (rev. 2) Addresses: https://github.com/haproxy/haproxy/issues/71 This RFC now supports provider(OpenSSL 3) and engine(OpenSSL 1.1.1) The patch will follow in the next email. Background Here is the revised version of my original RFC[1] to support PKCS#11 keys in master-worker mode by recreating private keys in the worker. First let me address some previous questions: from @wlallemand[2] "Did you identify why the fork was causing an issue? We should probably try to understand this before, it could be something stupid in haproxy's code or in the pkcs11 provider." This is a known PKCS#11 issue and not really a haproxy problem. Several integrations, such as provider/engine, do try to mitigate this but in general it is a tricky problem. After fork some of these integrations will attempt to restore handles but as GnuTLS says[3] "Note that, PKCS #11 modules behave in a peculiar way after a fork; they require a reinitialization of all the used PKCS #11 resources. While GnuTLS automates that process, there are corner cases where it is not possible to handle it correctly in an automated way. For that, it is recommended not to mix fork() and PKCS #11 module usage. It is recommended to initialize and use any PKCS #11 resources in a single process." PKCS#11 v3 does have support for PKCS#11 modules with "fork tolerant semantics"[4], but such modules are rare at the moment. Revised version Changes from [1] - now support both provider(OpenSSL 3) and engine(OpenSSL 1.1.1); the exemplars are pkcs11-provider[5] and opensc pkcs11 engine[6] - remove use of maps (ckch_data* -> PEM, SSL_CTX* -> ckch_data*) What does this RFC do? In master-worker mode it recreates the private key in the worker and installs this private key in the SSL_CTX*, overriding the original private key created in master. How does it do this? - extend struct ckch_data to store a copy of the PEM data used to create the cert / key - (engine specific) extend struct ckch_data to store the engine name and key name - thus - hash map (struct ckch_data -> PEM/engine data) is not needed - when creating SSL_CTX* store a pointer to struct ckch_data with SSL_CTX_set_app_data(); thus - hash map (SSL_CTX* -> struct ckch_data) is no longer needed - in the worker process, just before SSL_use_SSL_CTX(), we use the stored PEM data to recreate the provider private key or (engine specific) use the stored engine/key name to recreate the engine private key - the worker key is cached for future use How are engine keys referenced? - use crt "extra files" and store engine "key" in a specially formatted ".crt.key" file - this is not a private key PEM file, it is simply a text file containing two lines: engine = pkcs11 private_key = pkcs11:token=SomePKCS11Token;object=SomePrivateKeyLabel;type=private - repeat: an engine ".crt.key" file is not a PEM-"PRIVATE KEY" file that we are familiar with; it is a special text file with the name of the engine and private key - engine keys must live in a separate .key file as they are not PEM How are pkcs11-provider keys referenced? - like TPM2 pkcs11-provider can store keys in PEM objects with label "PKCS#11 PROVIDER URI" - a provider key can be merged into the .crt file - current glitch in pkcs11-provider PEM parser: a key in a merged file must be the first PEM object References 1. https://www.mail-archive.com/haproxy@formilux.org/msg44736.html 2. https://www.mail-archive.com/haproxy@formilux.org/msg44741.html 3. https://www.gnutls.org/manual/html_node/PKCS11-Initialization.html 4. https://docs.oasis-open.org/pkcs11/pkcs11-base/v3.0/csprd01/pkcs11-base-v3.0-csprd01.html 5. https://github.com/latchset/pkcs11-provider 6. https://github.com/OpenSC/libp11 Regards Richard
Re: RFC: PKCS#11 create private keys in worker process
"Did you identify why the fork was causing an issue? We should probably try to understand this before, it could be something stupid in haproxy's code or in the pkcs11 provider." - PKCS#11 drivers contain session objects and handles to private keys in the HSM; these session objects and handles don't always behave well after fork - this is a known problem of a lot of existing PKCS#11 drivers Indeed GnuTLS says: "Note that, PKCS #11 modules behave in a peculiar way after a fork; they require a reinitialization of all the used PKCS #11 resources. While GnuTLS automates that process, there are corner cases where it is not possible to handle it correctly in an automated way. For that, it is recommended not to mix fork() and PKCS #11 module usage. It is recommended to initialize and use any PKCS #11 resources in a single process" This is not a problem of HAProxy specifically - having said this new PKCS#11 v3 drivers may support "fork tolerant semantics": such drivers don't need this kind of RFC which is targeted to drivers without this feature. https://docs.oasis-open.org/pkcs11/pkcs11-base/v3.0/cs01/pkcs11-base-v3.0-cs01.html - the double caching(ssl_ckch.c/ssl_sock.c) is inelegant and messy just to be able to recall the PEM bytes. I think I have found a way to do it without cache. -- cache PEM data by extending (struct ckch_data*) -- store the (struct ckch_data*) in the SSL_CTX* using SSL_CTX_set_app_data -- let me think about this and come up with another RFC. Cheers Richard
RFC: PKCS#11 create private keys in worker process - take 3
Fix typo in patch formatting.
Richard
diff --git a/include/haproxy/ssl_ckch.h b/include/haproxy/ssl_ckch.h
index 94c53b301..00ba2bf18 100644
--- a/include/haproxy/ssl_ckch.h
+++ b/include/haproxy/ssl_ckch.h
@@ -72,5 +72,14 @@ int __ssl_store_load_locations_file(char *path, int create_if_none, enum cafile_
extern struct cert_exts cert_exts[];
extern int (*ssl_commit_crlfile_cb)(const char *path, X509_STORE *ctx, char **err);
+struct ckch_data_cache {
+ struct ckch_data_cache *next;
+ struct ckch_data *key;
+ uint8_t *data;
+ int len;
+};
+
+struct ckch_data_cache *ckch_data_get(void *key);
+
#endif /* USE_OPENSSL */
#endif /* _HAPROXY_SSL_CRTLIST_H */
diff --git a/src/ssl_ckch.c b/src/ssl_ckch.c
index add42b69e..9c256f193 100644
--- a/src/ssl_ckch.c
+++ b/src/ssl_ckch.c
@@ -516,11 +516,63 @@ int ssl_sock_load_files_into_ckch(const char *path, struct ckch_data *data, char
*
* Return 0 on success or != 0 on failure
*/
+
+struct ckch_data_cache *cache[31] = {0};
+
+/*
+ * This caches the BIO data in case the private key i
+ *
+ * to be instantiated in the child process (used for HSM keys)
+ *
+ * The key ckch_data* and the value is ckch_data_cache*
+ *
+ */
+static void ckch_data_put(void *key, uint8_t *data, int len)
+{
+ int idx = (unsigned long)key % 31;
+ struct ckch_data_cache *prev, *cur, *last = NULL;
+
+ ha_notice("caching ckch_data:%p len:%d\n", key, len);
+
+ for(prev = cache[idx]; prev != NULL;) {
+ last = prev;
+ prev = prev->next;
+ }
+
+ cur = malloc(sizeof(struct ckch_data_cache));
+ memset(cur, 0, sizeof(struct ckch_data_cache));
+ if (last == NULL)
+ cache[idx] = cur;
+ else
+ last->next = cur;
+ cur->key = key;
+ cur->data = malloc(len);
+ memcpy(cur->data, data, len);
+ cur->len = len;
+}
+
+struct ckch_data_cache *ckch_data_get(void *key)
+{
+ int idx = (unsigned long)key % 31;
+ struct ckch_data_cache *prev;
+
+ for(prev = cache[idx]; prev != NULL; prev = prev->next) {
+ if (prev->key == key)
+ return prev;
+ }
+
+ return NULL;
+}
+
int ssl_sock_load_key_into_ckch(const char *path, char *buf, struct ckch_data *data , char **err)
{
BIO *in = NULL;
int ret = 1;
EVP_PKEY *key = NULL;
+ static char src[16384];
+ char *src_temp;
+ int len;
+ int filetype = 0;
if (buf) {
/* reading from a buffer */
@@ -538,14 +590,25 @@ int ssl_sock_load_key_into_ckch(const char *path, char *buf, struct ckch_data *d
if (BIO_read_filename(in, path) <= 0)
goto end;
+ filetype = 1;
}
/* Read Private Key */
+ if(filetype) {
+ len = BIO_read(in, src, sizeof(src));
+ BIO_reset(in);
+ } else {
+ len = BIO_get_mem_data(in, _temp);
+ memcpy(src, src_temp, len);
+ }
+
key = PEM_read_bio_PrivateKey(in, NULL, NULL, NULL);
if (key == NULL) {
memprintf(err, "%sunable to load private key from file '%s'.\n",
err && *err ? *err : "", path);
goto end;
+ } else {
+ ckch_data_put(data, (uint8_t*)src, len);
}
ret = 0;
@@ -581,6 +644,10 @@ int ssl_sock_load_pem_into_ckch(const char *path, char *buf, struct ckch_data *d
EVP_PKEY *key = NULL;
HASSL_DH *dh = NULL;
STACK_OF(X509) *chain = NULL;
+ int filetype = 0;
+ int len;
+ static char src[16384];
+ char *src_temp;
if (buf) {
/* reading from a buffer */
@@ -603,11 +670,23 @@ int ssl_sock_load_pem_into_ckch(const char *path, char *buf, struct ckch_data *d
err && *err ? *err : "", path);
goto end;
}
+ filetype = 1;
}
/* Read Private Key */
+ if(filetype) {
+ len = BIO_read(in, src, sizeof(src));
+ BIO_reset(in);
+ } else {
+ len = BIO_get_mem_data(in, _temp);
+ memcpy(src, src_temp, len);
+ }
+
key = PEM_read_bio_PrivateKey(in, NULL, NULL, NULL);
/* no need to check for errors here, because the private key could be loaded later */
+ if(key && global.mode_MWORKER) {
+ ckch_data_put(data, (uint8_t *)src, len);
+ }
#ifndef OPENSSL_NO_DH
/* Seek back to beginning of file */
diff --git a/src/ssl_sock.c b/src/ssl_sock.c
index 824bdaa72..64bd7ac04 100644
--- a/src/ssl_sock.c
+++ b/src/ssl_sock.c
@@ -2009,10 +2009,77 @@ struct methodVersions methodVersions[] = {
{SSL_OP_NO_TLSv1_3, MC_SSL_O_NO_TLSV13, ctx_set_TLSv13_func, ssl_set_TLSv13_func, "TLSv1.3"}, /* CONF_TLSV13 */
};
+/* cache SSL_CTX* to ckch_data
+ * so we can recreate private key
+ */
+struct ssl_ctx_cache {
+ void *key;
+ struct ckch_data_cache *value;
+ struct ssl_ctx_cache *next;
+ EVP_PKEY *pkey;
+};
+static struct ssl_ctx_cache *ctx_cache[31];
+
+void ssl_ctx_put(void *key, void *data)
+{
+ int idx = (unsigned long)key % 31;
+ struct ssl_ctx_cache *prev, *cur, *last = NULL;
+
+ ha_notice("ssl_ctx_cache key = %p value = %p\n", key, data);
+
+ for(prev = ctx_cache[idx]; prev != NULL;) {
+ last = prev;
+ prev = prev->next;
+ }
+
+ cur = malloc(sizeof(struct ssl_ctx_cache));
+ memset(cur, 0, sizeof(struct ssl_ctx_cache));
+ if (last == NULL)
+ ctx_cache[idx] = cur;
+ else
+ last->next = cur;
+ cur->next = NULL;
+ cur->key = key;
+ cur->value = data;
+}
+
+struct ssl_ctx_cache
RFC: PKCS#11 create private keys in worker process - take 2
Apologies for the badly pasted diff
Richard
diff --git a/include/haproxy/ssl_ckch.h b/include/haproxy/ssl_ckch.h
index 94c53b301..00ba2bf18 100644
--- a/include/haproxy/ssl_ckch.h
+++ b/include/haproxy/ssl_ckch.h
@@ -72,5 +72,14 @@ int __ssl_store_load_locations_file(char *path, int create_if_none, enum cafile_
extern struct cert_exts cert_exts[];
extern int (*ssl_commit_crlfile_cb)(const char *path, X509_STORE *ctx, char **err);
+struct ckch_data_cache {
+ struct ckch_data_cache *next;
+ struct ckch_data *key;
+ uint8_t *data;
+ int len;
+};
+
+struct ckch_data_cache *ckch_data_get(void *key);
+
#endif /* USE_OPENSSL */
#endif /* _HAPROXY_SSL_CRTLIST_H */
diff --git a/src/ssl_ckch.c b/src/ssl_ckch.c
index add42b69e..f659014ba 100644
--- a/src/ssl_ckch.c
+++ b/src/ssl_ckch.c
@@ -516,11 +516,63 @@ int ssl_sock_load_files_into_ckch(const char *path, struct ckch_data *data, char
*
* Return 0 on success or != 0 on failure
*/
+
+struct ckch_data_cache *cache[31] = {0};
+
+/*
+ * This caches the BIO data in case the private key i
+ *
+ * to be instantiated in the child process (used for HSM keys)
+ *
+ * The key ckch_data* and the value is ckch_data_cache*
+ *
+ */
+static void ckch_data_put(void *key, uint8_t *data, int len)
+{
+ int idx = (unsigned long)key % 31;
+ struct ckch_data_cache *prev, *cur, *last = NULL;
+
+ ha_notice("caching ckch_data:%p len:%d\n", key, len);
+
+ for(prev = cache[idx]; prev != NULL;) {
+ last = prev;
+ prev = prev->next;
+ }
+
+ cur = malloc(sizeof(struct ckch_data_cache));
+ memset(cur, 0, sizeof(struct ckch_data_cache));
+ if (last == NULL)
+ cache[idx] = cur;
+ else
+ last = cur;
+ cur->key = key;
+ cur->data = malloc(len);
+ memcpy(cur->data, data, len);
+ cur->len = len;
+}
+
+struct ckch_data_cache *ckch_data_get(void *key)
+{
+ int idx = (unsigned long)key % 31;
+ struct ckch_data_cache *prev;
+
+ for(prev = cache[idx]; prev != NULL; prev = prev->next) {
+ if (prev->key == key)
+ return prev;
+ }
+
+ return NULL;
+}
+
int ssl_sock_load_key_into_ckch(const char *path, char *buf, struct ckch_data *data , char **err)
{
BIO *in = NULL;
int ret = 1;
EVP_PKEY *key = NULL;
+ static char src[16384];
+ char *src_temp;
+ int len;
+ int filetype = 0;
if (buf) {
/* reading from a buffer */
@@ -538,14 +590,25 @@ int ssl_sock_load_key_into_ckch(const char *path, char *buf, struct ckch_data *d
if (BIO_read_filename(in, path) <= 0)
goto end;
+ filetype = 1;
}
/* Read Private Key */
+ if(filetype) {
+ len = BIO_read(in, src, sizeof(src));
+ BIO_reset(in);
+ } else {
+ len = BIO_get_mem_data(in, _temp);
+ memcpy(src, src_temp, len);
+ }
+
key = PEM_read_bio_PrivateKey(in, NULL, NULL, NULL);
if (key == NULL) {
memprintf(err, "%sunable to load private key from file '%s'.\n",
err && *err ? *err : "", path);
goto end;
+ } else {
+ ckch_data_put(data, (uint8_t*)src, len);
}
ret = 0;
@@ -581,6 +644,10 @@ int ssl_sock_load_pem_into_ckch(const char *path, char *buf, struct ckch_data *d
EVP_PKEY *key = NULL;
HASSL_DH *dh = NULL;
STACK_OF(X509) *chain = NULL;
+ int filetype = 0;
+ int len;
+ static char src[16384];
+ char *src_temp;
if (buf) {
/* reading from a buffer */
@@ -603,11 +670,23 @@ int ssl_sock_load_pem_into_ckch(const char *path, char *buf, struct ckch_data *d
err && *err ? *err : "", path);
goto end;
}
+ filetype = 1;
}
/* Read Private Key */
+ if(filetype) {
+ len = BIO_read(in, src, sizeof(src));
+ BIO_reset(in);
+ } else {
+ len = BIO_get_mem_data(in, _temp);
+ memcpy(src, src_temp, len);
+ }
+
key = PEM_read_bio_PrivateKey(in, NULL, NULL, NULL);
/* no need to check for errors here, because the private key could be loaded later */
+ if(key && global.mode_MWORKER) {
+ ckch_data_put(data, (uint8_t *)src, len);
+ }
#ifndef OPENSSL_NO_DH
/* Seek back to beginning of file */
diff --git a/src/ssl_sock.c b/src/ssl_sock.c
index 824bdaa72..64bd7ac04 100644
--- a/src/ssl_sock.c
+++ b/src/ssl_sock.c
@@ -2009,10 +2009,77 @@ struct methodVersions methodVersions[] = {
{SSL_OP_NO_TLSv1_3, MC_SSL_O_NO_TLSV13, ctx_set_TLSv13_func, ssl_set_TLSv13_func, "TLSv1.3"}, /* CONF_TLSV13 */
};
+/* cache SSL_CTX* to ckch_data
+ * so we can recreate private key
+ */
+struct ssl_ctx_cache {
+ void *key;
+ struct ckch_data_cache *value;
+ struct ssl_ctx_cache *next;
+ EVP_PKEY *pkey;
+};
+static struct ssl_ctx_cache *ctx_cache[31];
+
+void ssl_ctx_put(void *key, void *data)
+{
+ int idx = (unsigned long)key % 31;
+ struct ssl_ctx_cache *prev, *cur, *last = NULL;
+
+ ha_notice("ssl_ctx_cache key = %p value = %p\n", key, data);
+
+ for(prev = ctx_cache[idx]; prev != NULL;) {
+ last = prev;
+ prev = prev->next;
+ }
+
+ cur = malloc(sizeof(struct ssl_ctx_cache));
+ memset(cur, 0, sizeof(struct ssl_ctx_cache));
+ if (last == NULL)
+ ctx_cache[idx] = cur;
+ else
+ last->next = cur;
+ cur->next = NULL;
+ cur->key = key;
+ cur->value = data;
+}
+
+struct ssl_ctx_cache
RFC: PKCS#11 create private keys in worker process diff
diff --git a/include/haproxy/ssl_ckch.h b/include/haproxy/ssl_ckch.h
index 94c53b301..00ba2bf18 100644
--- a/include/haproxy/ssl_ckch.h
+++ b/include/haproxy/ssl_ckch.h
@@ -72,5 +72,14 @@ int __ssl_store_load_locations_file(char *path, int
create_if_none, enum cafile_
extern struct cert_exts cert_exts[];
extern int (*ssl_commit_crlfile_cb)(const char *path, X509_STORE *ctx,
char **err);
+struct ckch_data_cache {
+ struct ckch_data_cache *next;
+ struct ckch_data *key;
+ uint8_t *data;
+ int len;
+};
+
+struct ckch_data_cache *ckch_data_get(void *key);
+
#endif /* USE_OPENSSL */
#endif /* _HAPROXY_SSL_CRTLIST_H */
diff --git a/src/ssl_ckch.c b/src/ssl_ckch.c
index add42b69e..f659014ba 100644
--- a/src/ssl_ckch.c
+++ b/src/ssl_ckch.c
@@ -516,11 +516,63 @@ int ssl_sock_load_files_into_ckch(const char *path,
struct ckch_data *data, char
*
* Return 0 on success or != 0 on failure
*/
+
+struct ckch_data_cache *cache[31] = {0};
+
+/*
+ * This caches the BIO data in case the private key i
+ *
+ * to be instantiated in the child process (used for HSM keys)
+ *
+ * The key ckch_data* and the value is ckch_data_cache*
+ *
+ */
+static void ckch_data_put(void *key, uint8_t *data, int len)
+{
+ int idx = (unsigned long)key % 31;
+ struct ckch_data_cache *prev, *cur, *last = NULL;
+
+ ha_notice("caching ckch_data:%p len:%d\n", key, len);
+
+ for(prev = cache[idx]; prev != NULL;) {
+ last = prev;
+ prev = prev->next;
+ }
+
+ cur = malloc(sizeof(struct ckch_data_cache));
+ memset(cur, 0, sizeof(struct ckch_data_cache));
+ if (last == NULL)
+ cache[idx] = cur;
+ else
+ last = cur;
+ cur->key = key;
+ cur->data = malloc(len);
+ memcpy(cur->data, data, len);
+ cur->len = len;
+}
+
+struct ckch_data_cache *ckch_data_get(void *key)
+{
+ int idx = (unsigned long)key % 31;
+ struct ckch_data_cache *prev;
+
+ for(prev = cache[idx]; prev != NULL; prev = prev->next) {
+ if (prev->key == key)
+ return prev;
+ }
+
+ return NULL;
+}
+
int ssl_sock_load_key_into_ckch(const char *path, char *buf, struct
ckch_data *data , char **err)
{
BIO *in = NULL;
int ret = 1;
EVP_PKEY *key = NULL;
+ static char src[16384];
+ char *src_temp;
+ int len;
+ int filetype = 0;
if (buf) {
/* reading from a buffer */
@@ -538,14 +590,25 @@ int ssl_sock_load_key_into_ckch(const char *path,
char *buf, struct ckch_data *d
if (BIO_read_filename(in, path) <= 0)
goto end;
+ filetype = 1;
}
/* Read Private Key */
+ if(filetype) {
+ len = BIO_read(in, src, sizeof(src));
+ BIO_reset(in);
+ } else {
+ len = BIO_get_mem_data(in, _temp);
+ memcpy(src, src_temp, len);
+ }
+
key = PEM_read_bio_PrivateKey(in, NULL, NULL, NULL);
if (key == NULL) {
memprintf(err, "%sunable to load private key from file '%s'.\n",
err && *err ? *err : "", path);
goto end;
+ } else {
+ ckch_data_put(data, (uint8_t*)src, len);
}
ret = 0;
@@ -581,6 +644,10 @@ int ssl_sock_load_pem_into_ckch(const char *path, char
*buf, struct ckch_data *d
EVP_PKEY *key = NULL;
HASSL_DH *dh = NULL;
STACK_OF(X509) *chain = NULL;
+ int filetype = 0;
+ int len;
+ static char src[16384];
+ char *src_temp;
if (buf) {
/* reading from a buffer */
@@ -603,11 +670,23 @@ int ssl_sock_load_pem_into_ckch(const char *path,
char *buf, struct ckch_data *d
err && *err ? *err : "", path);
goto end;
}
+ filetype = 1;
}
/* Read Private Key */
+ if(filetype) {
+ len = BIO_read(in, src, sizeof(src));
+ BIO_reset(in);
+ } else {
+ len = BIO_get_mem_data(in, _temp);
+ memcpy(src, src_temp, len);
+ }
+
key = PEM_read_bio_PrivateKey(in, NULL, NULL, NULL);
/* no need to check for errors here, because the private key could be
loaded later */
+ if(key && global.mode_MWORKER) {
+ ckch_data_put(data, (uint8_t *)src, len);
+ }
#ifndef OPENSSL_NO_DH
/* Seek back to beginning of file */
diff --git a/src/ssl_sock.c b/src/ssl_sock.c
index 824bdaa72..64bd7ac04 100644
--- a/src/ssl_sock.c
+++ b/src/ssl_sock.c
@@ -2009,10 +2009,77 @@ struct methodVersions methodVersions[] = {
{SSL_OP_NO_TLSv1_3, MC_SSL_O_NO_TLSV13, ctx_set_TLSv13_func,
ssl_set_TLSv13_func, "TLSv1.3"}, /* CONF_TLSV13 */
};
+/* cache SSL_CTX* to ckch_data
+ * so we can recreate private key
+ */
+struct ssl_ctx_cache {
+ void *key;
+ struct ckch_data_cache *value;
+ struct ssl_ctx_cache *next;
+ EVP_PKEY *pkey;
+};
+static struct ssl_ctx_cache *ctx_cache[31];
+
+void ssl_ctx_put(void *key, void *data)
+{
+ int idx = (unsigned long)key % 31;
+ struct ssl_ctx_cache *prev, *cur, *last = NULL;
+
+ ha_notice("ssl_ctx_cache key = %p value = %p\n", key, data);
+
+ for(prev = ctx_cache[idx]; prev != NULL;) {
+ last = prev;
+ prev = prev->next;
+ }
+
+ cur = malloc(sizeof(struct ssl_ctx_cache));
+ memset(cur, 0, sizeof(struct ssl_ctx_cache));
+ if (last == NULL)
+ ctx_cache[idx] = cur;
+ else
+ last->next = cur;
+ cur->next = NULL;
+ cur->key = key;
+ cur->value = data;
+}
+
+struct ssl_ctx_cache *ssl_ctx_get(void *key)
+{
+ int idx = (unsigned long)key % 31;
+ struct ssl_ctx_cache *prev;
RFC: PKCS#11 create private keys in worker process
Hello, This is an RFC to recreate private keys in the worker process for PKCS#11, so that HSM keys can be used in -W mode. - ssl_ckch.c: add map of ckch_data to PEM data - ssl_sock.c: add map of SSL_CTX* to ckch_data - maps are implemented using buckets of linked lists it is explicit and in the code for easier review instead of using more optimized hashmap implementations - when the SSL context is created and the correct SSL_CTX is assigned with SSL_use_SSL_CTX the private key data is retrieved just once once, cached, and installed into the SSL_CTX; this is done in the worker process - the PEM data has an arbitrary limit of 16384 bytes Regards Richard
Re: [PR] FEATURE: load private keys from PKCS#11 pkcs11-provider PEM files
Yes I would be happy to include HAProxy with pkcs11-provider examples. On Thu, 21 Mar 2024, 16:43 William Lallemand, wrote: > On Thu, Mar 21, 2024 at 10:39:58AM +0800, Richard Chan wrote: > > Subject: Re: [PR] FEATURE: load private keys from PKCS#11 > pkcs11-provider PEM files > > On Thu, 21 Mar 2024, 00:15 William Lallemand, > wrote > > > > > > > > We made test in the past with the TPM2 provider which also uses a URI > in > > > the privatekey: > > > > > > > https://github.com/haproxy/wiki/wiki/OpenSSL-Providers-in-HAProxy#tpm2-provider > > > > > > Further testing shows that this PR is not needed. Sorry for the noise. > > > > There is a glitch in pkcs11-provider that requires the private key to be > > the first PEM object. Apart from this HAProxy loads the private key with > > no issues. > > > > Okay that's good to read :-) > > Would you be interested in contributing on the OpenSSL providers wiki > page? We could have a pkcs11 section like the one we already have for > the TPM2 provider. > > Regards, > > -- > William Lallemand >
Re: [PR] FEATURE: load private keys from PKCS#11 pkcs11-provider PEM files
On Thu, 21 Mar 2024, 00:15 William Lallemand, wrote > > We made test in the past with the TPM2 provider which also uses a URI in > the privatekey: > > https://github.com/haproxy/wiki/wiki/OpenSSL-Providers-in-HAProxy#tpm2-provider Further testing shows that this PR is not needed. Sorry for the noise. There is a glitch in pkcs11-provider that requires the private key to be the first PEM object. Apart from this HAProxy loads the private key with no issues. > >
Re: [PR] FEATURE: load private keys from PKCS#11 pkcs11-provider PEM files
Interesting about the TPM2 stuff - it has implemented a store loader for "TSS2 PRIVATE KEY" stanza. Since PEM is new to pkcs11-provider it may not have implemented a store loader yet (i.e. not PEM_bio_read_PrivateKey ready) hence this PR uses store directly. I will check with pkcs11-provider whether they can implement a store loader. Re current MWORKER problem (before any solution in 3.1!) - [RFC] Delayed private key loading - check if we are in MWORKER mode then skip EVP_PKEY loading in master for all PKCS#11 keys Hmmm - how to identify such keys? Maybe .crt file has a first line "key-type = PKCS#11" - in child process: create a map based on SSL_CTX *pointer and just-in-time load private key when needed at at SSL_set_SSL_CTX; does the child_process have access to bind_conf/ssl_bind_conf so it can find the crt file name? Otherwise, cache the SSL_CTX* -> crt(or key) filename in master before fork() WDYT? Thanks S-P On Thu, 21 Mar 2024 at 00:15, William Lallemand wrote: > On Wed, Mar 20, 2024 at 06:23:03AM +, PR Bot wrote: > > Subject: [PR] FEATURE: load private keys from PKCS#11 pkcs11-provider > PEM files > > Dear list! > > > > Author: S-P Chan > > Number of patches: 1 > > > > This is an automated relay of the Github pull request: > >FEATURE: load private keys from PKCS#11 pkcs11-provider PEM files > > > > Patch title(s): > >FEATURE: load private keys from PKCS#11 pkcs11-provider PEM files > > > > Link: > >https://github.com/haproxy/haproxy/pull/2493 > > > > Edit locally: > >wget https://github.com/haproxy/haproxy/pull/2493.patch && vi > 2493.patch > > > > Apply locally: > >curl https://github.com/haproxy/haproxy/pull/2493.patch | git am - > > > > Description: > >With pkcs11-provider (https://github.com/latchset/pkcs11-provider) a > >specially formatted PEM stanza can be used > >to reference a PKCS#11 > >URI to locate the private key. > > > >This PEM stanza can be used > >inside the crt file so that there is no change to the HAProxy config > >language. > > > >This works with OpenSSL 3 and pkcs11-provider after > >https://github.com/latchset/pkcs11-provider/commit/0806c3665 which > >added support for PKCS#11 URI-in-PEM. > > > >TODO: This PR works > >without forking (i.e., not in master-worker mode) as PKCS#11 drivers > >are fragile after fork. > >To use PKCS#11 keys in master-worker mode, > >we need to defer key loading to the child process. > > > >Format of > >PEM stanza: > >``` > >-BEGIN PKCS#11 PROVIDER URI- > >MIHWGhlQS0NTIzExIFByb3ZpZGVyIFVSSSB2MS4wDIG4cGtjczExOm1vZGVsPU5T > >UyUyMDM7bWFudWZhY3R1cmVyPU1vemlsbGElMjBGb3VuZGF0aW9uO3NlcmlhbD0w > >MDAwMDAwMDAwMDAwMDAwO3Rva2VuPU5TUyUyMENlcnRpZmljYXRlJTIwREI7aWQ9 > >JTczJTQ5JTU1JTFBJTMyJUFFJThDJUIwJTQ1JTQ5JTAzJURDJUE4JTA0JTg0JTlF > >JUI0JTlGJTQxJUFFO3R5cGU9cHJpdmF0ZQ== > >-END PKCS#11 PROVIDER > >URI- > >``` > > > >Parsed ASN.1: > >``` > >0:d=0 hl=3 l= > >214 cons: SEQUENCE > >3:d=1 hl=2 l= 25 prim: VISIBLESTRING > >:PKCS#11 Provider URI v1.0 > > 30:d=1 hl=3 l= 184 prim: UTF8STRING > >:pkcs11:model=NSS%203;manufacturer=Mozilla%20Foundation;serial=000 > >0;token=NSS%20Certificate%20DB;id=%73%49%55%1A%32%AE%8C%B0%45% > >49%03%DC%A8%04%84%9E%B4%9F%41%AE;type=private > >``` > > > > Instructions: > >This github pull request will be closed automatically; patch should be > >reviewed on the haproxy mailing list (haproxy@formilux.org). > Everyone is > >invited to comment, even the patch's author. Please keep the author > and > >list CCed in replies. Please note that in absence of any response this > >pull request will be lost. > > > > Hello, > > Thank you for your contribution, I'm surprised this is not already > working with the current code. > > We made test in the past with the TPM2 provider which also uses a URI in > the privatekey: > > https://github.com/haproxy/wiki/wiki/OpenSSL-Providers-in-HAProxy#tpm2-provider > > Do you have any specific configuration for HAProxy? > > Regards, > > -- > William Lallemand >
