This is an automated email from the ASF dual-hosted git repository.
eze pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/trafficserver.git
The following commit(s) were added to refs/heads/master by this push:
new ffd8d8ba34 Parallell ssl cert load (#12998)
ffd8d8ba34 is described below
commit ffd8d8ba3409bdff2de4a80c8c476f05409af520
Author: Evan Zelkowitz <[email protected]>
AuthorDate: Mon Mar 23 10:07:33 2026 -0600
Parallell ssl cert load (#12998)
* Add parallel ssl cert loading
---
doc/admin-guide/files/records.yaml.en.rst | 7 ++
include/iocore/net/SSLMultiCertConfigLoader.h | 10 ++-
src/iocore/net/P_SSLConfig.h | 1 +
src/iocore/net/QUICMultiCertConfigLoader.cc | 2 +-
src/iocore/net/SSLConfig.cc | 9 ++-
src/iocore/net/SSLUtils.cc | 93 ++++++++++++++++++-----
src/records/RecordsConfig.cc | 4 +-
tests/gold_tests/tls/ssl_multicert_loader.test.py | 44 ++++++++++-
8 files changed, 148 insertions(+), 22 deletions(-)
diff --git a/doc/admin-guide/files/records.yaml.en.rst
b/doc/admin-guide/files/records.yaml.en.rst
index 492dc9849f..f64028b647 100644
--- a/doc/admin-guide/files/records.yaml.en.rst
+++ b/doc/admin-guide/files/records.yaml.en.rst
@@ -4089,6 +4089,13 @@ SSL Termination
:file:`ssl_multicert.yaml` file successfully load. If false (``0``), SSL
certificate
load failures will not prevent |TS| from starting.
+.. ts:cv:: CONFIG proxy.config.ssl.server.multicert.concurrency INT 1
+
+ Controls how many threads are used to load SSL certificates from
:file:`ssl_multicert.yaml`
+ during configuration reloads. On first startup, |TS| always uses all
available CPU cores
+ regardless of this setting. Set to ``0`` to automatically use the number
of hardware
+ threads. Default ``1`` (single-threaded reloads).
+
.. ts:cv:: CONFIG proxy.config.ssl.server.cert.path STRING /config
The location of the SSL certificates and chains used for accepting
diff --git a/include/iocore/net/SSLMultiCertConfigLoader.h
b/include/iocore/net/SSLMultiCertConfigLoader.h
index b2b7fc246e..d0f68469ce 100644
--- a/include/iocore/net/SSLMultiCertConfigLoader.h
+++ b/include/iocore/net/SSLMultiCertConfigLoader.h
@@ -25,10 +25,12 @@
#include "iocore/net/SSLTypes.h"
#include "tsutil/DbgCtl.h"
+#include "config/ssl_multicert.h"
#include <openssl/ssl.h>
#include <swoc/Errata.h>
+#include <mutex>
#include <string>
#include <set>
#include <vector>
@@ -51,7 +53,7 @@ public:
SSLMultiCertConfigLoader(const SSLConfigParams *p) : _params(p) {}
virtual ~SSLMultiCertConfigLoader(){};
- swoc::Errata load(SSLCertLookup *lookup);
+ swoc::Errata load(SSLCertLookup *lookup, bool firstLoad = false);
virtual SSL_CTX *default_server_ssl_ctx();
@@ -88,6 +90,12 @@ private:
virtual bool _store_ssl_ctx(SSLCertLookup *lookup, const
shared_SSLMultiCertConfigParams &ssl_multi_cert_params);
bool _prep_ssl_ctx(const shared_SSLMultiCertConfigParams
&sslMultCertSettings, SSLMultiCertConfigLoader::CertLoadData &data,
std::set<std::string> &common_names,
std::unordered_map<int, std::set<std::string>> &unique_names);
+
+ void _load_items(SSLCertLookup *lookup,
config::SSLMultiCertConfig::const_iterator begin,
+ config::SSLMultiCertConfig::const_iterator end, int
base_index, swoc::Errata &errata);
+
+ std::mutex _loader_mutex;
+
virtual void _set_handshake_callbacks(SSL_CTX *ctx);
virtual bool _setup_session_cache(SSL_CTX *ctx);
virtual bool _setup_dialog(SSL_CTX *ctx, const SSLMultiCertConfigParams
*sslMultCertSettings);
diff --git a/src/iocore/net/P_SSLConfig.h b/src/iocore/net/P_SSLConfig.h
index 135d25a5c9..0d6ee6b14e 100644
--- a/src/iocore/net/P_SSLConfig.h
+++ b/src/iocore/net/P_SSLConfig.h
@@ -66,6 +66,7 @@ struct SSLConfigParams : public ConfigInfo {
char *cipherSuite;
char *client_cipherSuite;
int configExitOnLoadError;
+ int configLoadConcurrency;
int clientCertLevel;
int verify_depth;
int ssl_origin_session_cache{0};
diff --git a/src/iocore/net/QUICMultiCertConfigLoader.cc
b/src/iocore/net/QUICMultiCertConfigLoader.cc
index 34a39115c4..8e5bbe1dca 100644
--- a/src/iocore/net/QUICMultiCertConfigLoader.cc
+++ b/src/iocore/net/QUICMultiCertConfigLoader.cc
@@ -45,7 +45,7 @@ QUICCertConfig::reconfigure(ConfigContext ctx)
SSLCertLookup *lookup = new SSLCertLookup();
QUICMultiCertConfigLoader loader(params);
- auto errata = loader.load(lookup);
+ auto errata = loader.load(lookup, _config_id == 0);
if (!lookup->is_valid || (errata.has_severity() && errata.severity() >=
ERRATA_ERROR)) {
retStatus = false;
}
diff --git a/src/iocore/net/SSLConfig.cc b/src/iocore/net/SSLConfig.cc
index 6aaf8c2374..936821b41b 100644
--- a/src/iocore/net/SSLConfig.cc
+++ b/src/iocore/net/SSLConfig.cc
@@ -46,9 +46,11 @@
#include "mgmt/config/ConfigRegistry.h"
#include <openssl/pem.h>
+#include <algorithm>
#include <array>
#include <cstring>
#include <cmath>
+#include <thread>
#include <unordered_map>
int SSLConfig::config_index = 0;
@@ -125,6 +127,7 @@ SSLConfigParams::reset()
ssl_ctx_options = SSL_OP_NO_SSLv2 |
SSL_OP_NO_SSLv3;
ssl_client_ctx_options = SSL_OP_NO_SSLv2 |
SSL_OP_NO_SSLv3;
configExitOnLoadError = 1;
+ configLoadConcurrency = 1;
}
void
@@ -431,6 +434,10 @@ SSLConfigParams::initialize()
configFilePath =
ats_stringdup(RecConfigReadConfigPath("proxy.config.ssl.server.multicert.filename"));
configExitOnLoadError =
RecGetRecordInt("proxy.config.ssl.server.multicert.exit_on_load_fail").value_or(0);
+ configLoadConcurrency =
RecGetRecordInt("proxy.config.ssl.server.multicert.concurrency").value_or(1);
+ if (configLoadConcurrency == 0) {
+ configLoadConcurrency =
std::clamp(static_cast<int>(std::thread::hardware_concurrency()), 1, 256);
+ }
{
auto
rec_str{RecGetRecordStringAlloc("proxy.config.ssl.server.private_key.path")};
@@ -671,7 +678,7 @@ SSLCertificateConfig::reconfigure(ConfigContext ctx)
ink_hrtime_sleep(HRTIME_SECONDS(secs));
}
- auto errata = SSLMultiCertConfigLoader(params).load(lookup);
+ auto errata = SSLMultiCertConfigLoader(params).load(lookup, configid == 0);
if (!lookup->is_valid || (errata.has_severity() && errata.severity() >=
ERRATA_ERROR)) {
retStatus = false;
}
diff --git a/src/iocore/net/SSLUtils.cc b/src/iocore/net/SSLUtils.cc
index 9bca4c3086..ee2231615a 100644
--- a/src/iocore/net/SSLUtils.cc
+++ b/src/iocore/net/SSLUtils.cc
@@ -69,6 +69,8 @@
#include <openssl/ts.h>
#endif
+#include <algorithm>
+#include <thread>
#include <utility>
#include <string>
#include <unistd.h>
@@ -1599,11 +1601,20 @@ SSLMultiCertConfigLoader::_store_ssl_ctx(SSLCertLookup
*lookup, const shared_SSL
SSLMultiCertConfigLoader::CertLoadData data;
if (!this->_prep_ssl_ctx(sslMultCertSettings, data, common_names,
unique_names)) {
- lookup->is_valid = false;
+ {
+ std::lock_guard<std::mutex> lock(_loader_mutex);
+ lookup->is_valid = false;
+ }
return false;
}
std::vector<SSLLoadingContext> ctxs = this->init_server_ssl_ctx(data,
sslMultCertSettings.get());
+
+ // Serialize all mutations to the shared SSLCertLookup.
+ // The expensive work above (_prep_ssl_ctx + init_server_ssl_ctx) runs
+ // without the lock, allowing parallel cert loading across threads.
+ std::lock_guard<std::mutex> lock(_loader_mutex);
+
for (const auto &loadingctx : ctxs) {
if (!sslMultCertSettings ||
!this->_store_single_ssl_ctx(lookup, sslMultCertSettings,
shared_SSL_CTX{loadingctx.ctx, SSL_CTX_free}, loadingctx.ctx_type,
@@ -1782,7 +1793,7 @@
SSLMultiCertConfigLoader::_store_single_ssl_ctx(SSLCertLookup *lookup, const sha
}
swoc::Errata
-SSLMultiCertConfigLoader::load(SSLCertLookup *lookup)
+SSLMultiCertConfigLoader::load(SSLCertLookup *lookup, bool firstLoad)
{
const SSLConfigParams *params = this->_params;
@@ -1806,10 +1817,69 @@ SSLMultiCertConfigLoader::load(SSLCertLookup *lookup)
}
swoc::Errata errata(ERRATA_NOTE);
- int item_num = 0;
- for (const auto &item : parse_result.value) {
+ static constexpr int MAX_LOAD_THREADS = 256;
+
+ int num_threads = params->configLoadConcurrency;
+ if (firstLoad) {
+ num_threads =
std::clamp(static_cast<int>(std::thread::hardware_concurrency()), 1,
MAX_LOAD_THREADS);
+ }
+ num_threads = std::min(num_threads,
static_cast<int>(parse_result.value.size()));
+
+ if (num_threads > 1 && parse_result.value.size() > 1) {
+ std::size_t bucket_size = parse_result.value.size() / num_threads;
+ std::size_t remainder = parse_result.value.size() % num_threads;
+ auto current = parse_result.value.cbegin();
+
+ std::vector<std::thread> threads;
+ Note("(%s) loading %zu certs with %d threads", this->_debug_tag(),
parse_result.value.size(), num_threads);
+
+ for (int t = 0; t < num_threads; ++t) {
+ std::size_t this_bucket = bucket_size + (static_cast<std::size_t>(t) <
remainder ? 1 : 0);
+ auto end = current + this_bucket;
+ int base_index =
static_cast<int>(std::distance(parse_result.value.cbegin(), current));
+ threads.emplace_back(&SSLMultiCertConfigLoader::_load_items, this,
lookup, current, end, base_index, std::ref(errata));
+ current = end;
+ }
+
+ for (auto &th : threads) {
+ th.join();
+ }
+
+ Note("(%s) loaded %zu certs in %d threads", this->_debug_tag(),
parse_result.value.size(), num_threads);
+ } else {
+ _load_items(lookup, parse_result.value.cbegin(),
parse_result.value.cend(), 0, errata);
+ Note("(%s) loaded %zu certs (single-threaded)", this->_debug_tag(),
parse_result.value.size());
+ }
+
+ // We *must* have a default context even if it can't possibly work. The
default context is used to
+ // bootstrap the SSL handshake so that we can subsequently do the SNI lookup
to switch to the real
+ // context.
+ if (lookup->ssl_default == nullptr) {
+ shared_SSLMultiCertConfigParams sslMultiCertSettings(new
SSLMultiCertConfigParams);
+ sslMultiCertSettings->addr = ats_strdup("*");
+ if (!this->_store_ssl_ctx(lookup, sslMultiCertSettings)) {
+ errata.note(ERRATA_ERROR, "failed set default context");
+ }
+ }
+
+ return errata;
+}
+
+void
+SSLMultiCertConfigLoader::_load_items(SSLCertLookup *lookup,
config::SSLMultiCertConfig::const_iterator begin,
+
config::SSLMultiCertConfig::const_iterator end, int base_index, swoc::Errata
&errata)
+{
+ // Each thread needs its own elevated privileges since POSIX capabilities
are per-thread
+ uint32_t elevate_setting = 0;
+ elevate_setting =
RecGetRecordInt("proxy.config.ssl.cert.load_elevated").value_or(0);
+ ElevateAccess elevate_access(elevate_setting ? ElevateAccess::FILE_PRIVILEGE
: 0);
+
+ int item_num = base_index;
+ for (auto it = begin; it != end; ++it) {
item_num++;
+ const auto &item = *it;
+
shared_SSLMultiCertConfigParams sslMultiCertSettings =
std::make_shared<SSLMultiCertConfigParams>();
if (!item.ssl_cert_name.empty()) {
@@ -1846,25 +1916,14 @@ SSLMultiCertConfigLoader::load(SSLCertLookup *lookup)
// There must be a certificate specified unless the tunnel action is set.
if (sslMultiCertSettings->cert || sslMultiCertSettings->opt ==
SSLCertContextOption::OPT_TUNNEL) {
if (!this->_store_ssl_ctx(lookup, sslMultiCertSettings)) {
+ std::lock_guard<std::mutex> lock(_loader_mutex);
errata.note(ERRATA_ERROR, "Failed to load certificate at item {}",
item_num);
}
} else {
+ std::lock_guard<std::mutex> lock(_loader_mutex);
errata.note(ERRATA_WARN, "No ssl_cert_name specified and no tunnel
action set at item {}", item_num);
}
}
-
- // We *must* have a default context even if it can't possibly work. The
default context is used to
- // bootstrap the SSL handshake so that we can subsequently do the SNI lookup
to switch to the real
- // context.
- if (lookup->ssl_default == nullptr) {
- shared_SSLMultiCertConfigParams sslMultiCertSettings(new
SSLMultiCertConfigParams);
- sslMultiCertSettings->addr = ats_strdup("*");
- if (!this->_store_ssl_ctx(lookup, sslMultiCertSettings)) {
- errata.note(ERRATA_ERROR, "failed set default context");
- }
- }
-
- return errata;
}
// Release SSL_CTX and the associated data. This works for both
diff --git a/src/records/RecordsConfig.cc b/src/records/RecordsConfig.cc
index f2c85649a7..68af49ba18 100644
--- a/src/records/RecordsConfig.cc
+++ b/src/records/RecordsConfig.cc
@@ -1182,7 +1182,9 @@ static constexpr RecordElement RecordsConfig[] =
{RECT_CONFIG, "proxy.config.ssl.server.multicert.filename", RECD_STRING,
ts::filename::SSL_MULTICERT, RECU_RESTART_TS, RR_NULL, RECC_NULL, nullptr,
RECA_NULL}
,
{RECT_CONFIG, "proxy.config.ssl.server.multicert.exit_on_load_fail",
RECD_INT, "1", RECU_RESTART_TS, RR_NULL, RECC_INT, "[0-1]", RECA_NULL}
-,
+ ,
+ {RECT_CONFIG, "proxy.config.ssl.server.multicert.concurrency", RECD_INT,
"1", RECU_RESTART_TS, RR_NULL, RECC_INT, "[0-256]", RECA_NULL}
+ ,
{RECT_CONFIG, "proxy.config.ssl.servername.filename", RECD_STRING,
ts::filename::SNI, RECU_RESTART_TS, RR_NULL, RECC_NULL, nullptr, RECA_NULL}
,
{RECT_CONFIG, "proxy.config.ssl.server.ticket_key.filename", RECD_STRING,
nullptr, RECU_DYNAMIC, RR_NULL, RECC_NULL, nullptr, RECA_NULL}
diff --git a/tests/gold_tests/tls/ssl_multicert_loader.test.py
b/tests/gold_tests/tls/ssl_multicert_loader.test.py
index 6b74da7c53..5eb95a7f53 100644
--- a/tests/gold_tests/tls/ssl_multicert_loader.test.py
+++ b/tests/gold_tests/tls/ssl_multicert_loader.test.py
@@ -22,7 +22,7 @@ sni_domain = 'example.com'
ts = Test.MakeATSProcess("ts", enable_tls=True)
server = Test.MakeOriginServer("server")
-server2 = Test.MakeOriginServer("server3")
+server2 = Test.MakeOriginServer("server2")
request_header = {"headers": f"GET / HTTP/1.1\r\nHost: {sni_domain}\r\n\r\n",
"timestamp": "1469733493.993", "body": ""}
response_header = {"headers": "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n",
"timestamp": "1469733493.993", "body": ""}
@@ -123,3 +123,45 @@ ts2.Ready = 0 # Need this to be 0 because we are testing
shutdown, this is to m
ts2.Disk.traffic_out.Content = Testers.ExcludesExpression(
'Traffic Server is fully initialized', 'process should fail when invalid
certificate specified')
ts2.Disk.diags_log.Content = Testers.IncludesExpression('EMERGENCY: failed to
load SSL certificate file', 'check diags.log"')
+
+##########################################################################
+# Verify parallel cert loading on startup (firstLoad uses hardware_concurrency,
+# not the configured concurrency value, so the thread count is host-dependent)
+
+ts3 = Test.MakeATSProcess("ts3", enable_tls=True)
+server3 = Test.MakeOriginServer("server3")
+server3.addResponse("sessionlog.json", request_header, response_header)
+
+ts3.Disk.records_config.update(
+ {
+ 'proxy.config.ssl.server.cert.path': f'{ts3.Variables.SSLDir}',
+ 'proxy.config.ssl.server.private_key.path': f'{ts3.Variables.SSLDir}',
+ })
+
+ts3.addDefaultSSLFiles()
+
+ts3.Disk.remap_config.AddLine(f'map /
http://127.0.0.1:{server3.Variables.Port}')
+
+# Need at least 2 certs for multi-threading to kick in
+ts3.Disk.ssl_multicert_yaml.AddLines(
+ """
+ssl_multicert:
+ - dest_ip: "*"
+ ssl_cert_name: server.pem
+ ssl_key_name: server.key
+ - ssl_cert_name: server.pem
+ ssl_key_name: server.key
+""".split("\n"))
+
+tr5 = Test.AddTestRun("Verify parallel cert loading")
+tr5.Processes.Default.StartBefore(ts3)
+tr5.Processes.Default.StartBefore(server3)
+tr5.StillRunningAfter = ts3
+tr5.StillRunningAfter = server3
+tr5.MakeCurlCommand(
+ f"-q -s -v -k --resolve '{sni_domain}:{ts3.Variables.ssl_port}:127.0.0.1'
https://{sni_domain}:{ts3.Variables.ssl_port}";,
+ ts=ts3)
+tr5.Processes.Default.ReturnCode = 0
+tr5.Processes.Default.Streams.stdout = Testers.ExcludesExpression("Could Not
Connect", "Check response")
+tr5.Processes.Default.Streams.stderr =
Testers.IncludesExpression(f"CN={sni_domain}", "Check response")
+ts3.Disk.diags_log.Content = Testers.IncludesExpression('loaded 2 certs',
'verify certs were loaded successfully')
