This is an automated email from the ASF dual-hosted git repository.
amc pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/trafficserver.git
The following commit(s) were added to refs/heads/master by this push:
new ea85a7d use schedule_spawn to start accept threads when set to 0
ea85a7d is described below
commit ea85a7d59a3cbfa666091594e01db4ba20ea79f8
Author: Fei Deng <duke8...@gmail.com>
AuthorDate: Tue Nov 14 15:51:25 2017 -0600
use schedule_spawn to start accept threads when set to 0
---
iocore/net/P_NetAccept.h | 4 ++++
iocore/net/SSLSNIConfig.cc | 2 +-
iocore/net/UnixNetAccept.cc | 4 ++++
iocore/net/UnixNetProcessor.cc | 8 ++++++--
proxy/InkAPI.cc | 8 +++++---
proxy/Main.cc | 32 ++++++++++++++++++++++----------
proxy/http/HttpProxyServerMain.cc | 39 ++++++++++++++++++++++++++++++++++++++-
proxy/http/HttpProxyServerMain.h | 17 +++++++++++++++--
8 files changed, 95 insertions(+), 19 deletions(-)
diff --git a/iocore/net/P_NetAccept.h b/iocore/net/P_NetAccept.h
index a9c98ca..d14c5a5 100644
--- a/iocore/net/P_NetAccept.h
+++ b/iocore/net/P_NetAccept.h
@@ -39,6 +39,7 @@
#ifndef __P_NETACCEPT_H__
#define __P_NETACCEPT_H__
+#include <vector>
#include "ts/ink_platform.h"
#include "P_Connection.h"
@@ -114,4 +115,7 @@ struct NetAccept : public Continuation {
virtual ~NetAccept() { action_ = nullptr; }
};
+extern Ptr<ProxyMutex> naVecMutex;
+extern std::vector<NetAccept *> naVec;
+
#endif
diff --git a/iocore/net/SSLSNIConfig.cc b/iocore/net/SSLSNIConfig.cc
index fa51bf9..8f14d5b 100644
--- a/iocore/net/SSLSNIConfig.cc
+++ b/iocore/net/SSLSNIConfig.cc
@@ -42,7 +42,6 @@
static ConfigUpdateHandler<SNIConfig> *sniConfigUpdate;
struct NetAccept;
-extern std::vector<NetAccept *> naVec;
Map<int, SSLNextProtocolSet *> snpsMap;
extern TunnelHashMap TunnelMap;
NextHopProperty::NextHopProperty()
@@ -226,6 +225,7 @@ SNIConfig::startup()
void
SNIConfig::cloneProtoSet()
{
+ SCOPED_MUTEX_LOCK(lock, naVecMutex, this_ethread());
for (auto na : naVec) {
if (na->snpa) {
auto snps = na->snpa->cloneProtoSet();
diff --git a/iocore/net/UnixNetAccept.cc b/iocore/net/UnixNetAccept.cc
index aac3bbd..29775f1 100644
--- a/iocore/net/UnixNetAccept.cc
+++ b/iocore/net/UnixNetAccept.cc
@@ -31,6 +31,9 @@
using NetAcceptHandler = int (NetAccept::*)(int, void *);
int accept_till_done = 1;
+// we need to protect naVec since it might be accessed
+// in different threads at the same time
+Ptr<ProxyMutex> naVecMutex;
std::vector<NetAccept *> naVec;
static void
safe_delay(int msec)
@@ -139,6 +142,7 @@ Ldone:
NetAccept *
getNetAccept(int ID)
{
+ SCOPED_MUTEX_LOCK(lock, naVecMutex, this_ethread());
return naVec.at(ID);
}
diff --git a/iocore/net/UnixNetProcessor.cc b/iocore/net/UnixNetProcessor.cc
index 13d37a9..f63eaa0 100644
--- a/iocore/net/UnixNetProcessor.cc
+++ b/iocore/net/UnixNetProcessor.cc
@@ -30,7 +30,6 @@
#include "StatPages.h"
int net_accept_number = 0;
-extern std::vector<NetAccept *> naVec;
NetProcessor::AcceptOptions const NetProcessor::DEFAULT_ACCEPT_OPTIONS;
NetProcessor::AcceptOptions &
@@ -174,7 +173,12 @@ UnixNetProcessor::accept_internal(Continuation *cont, int
fd, AcceptOptions cons
} else {
na->init_accept(nullptr);
}
- naVec.push_back(na);
+
+ {
+ SCOPED_MUTEX_LOCK(lock, naVecMutex, this_ethread());
+ naVec.push_back(na);
+ }
+
#ifdef TCP_DEFER_ACCEPT
// set tcp defer accept timeout if it is configured, this will not trigger
an accept until there is
// data on the socket ready to be read
diff --git a/proxy/InkAPI.cc b/proxy/InkAPI.cc
index a077b21..eb59342 100644
--- a/proxy/InkAPI.cc
+++ b/proxy/InkAPI.cc
@@ -9289,12 +9289,13 @@ TSAcceptorGet(TSVConn sslp)
return ssl_vc ? reinterpret_cast<TSAcceptor>(ssl_vc->accept_object) :
nullptr;
}
-extern std::vector<NetAccept *> naVec;
TSAcceptor
TSAcceptorGetbyID(int ID)
{
- Debug("ssl", "getNetAccept in INK API.cc %p", naVec.at(ID));
- return reinterpret_cast<TSAcceptor>(naVec.at(ID));
+ SCOPED_MUTEX_LOCK(lock, naVecMutex, this_ethread());
+ auto ret = naVec.at(ID);
+ Debug("ssl", "getNetAccept in INK API.cc %p", ret);
+ return reinterpret_cast<TSAcceptor>(ret);
}
int
@@ -9307,6 +9308,7 @@ TSAcceptorIDGet(TSAcceptor acceptor)
int
TSAcceptorCount()
{
+ SCOPED_MUTEX_LOCK(lock, naVecMutex, this_ethread());
return naVec.size();
}
diff --git a/proxy/Main.cc b/proxy/Main.cc
index 5117954..53a122a 100644
--- a/proxy/Main.cc
+++ b/proxy/Main.cc
@@ -128,9 +128,11 @@ static void *mgmt_lifecycle_msg_callback(void *x, char
*data, int len);
static void init_ssl_ctx_callback(void *ctx, bool server);
static void load_ssl_file_callback(const char *ssl_file, unsigned int options);
-static int num_of_net_threads = ink_number_of_processors();
+// We need these two to be accessible somewhere else now
+int num_of_net_threads = ink_number_of_processors();
+int num_accept_threads = 0;
+
static int num_of_udp_threads = 0;
-static int num_accept_threads = 0;
static int num_task_threads = 0;
static char *http_accept_port_descriptor;
@@ -1718,7 +1720,7 @@ main(int /* argc ATS_UNUSED */, const char **argv)
// We need to do this early so we can initialize the Machine
// singleton, which depends on configuration values loaded in this.
// We want to initialize Machine as early as possible because it
- // has other dependencies. Hopefully not in init_HttpProxyServer().
+ // has other dependencies. Hopefully not in prep_HttpProxyServer().
HttpConfig::startup();
/* Set up the machine with the outbound address if that's set,
or the inbound address if set, otherwise let it default.
@@ -1793,9 +1795,21 @@ main(int /* argc ATS_UNUSED */, const char **argv)
ink_dns_init(makeModuleVersion(HOSTDB_MODULE_MAJOR_VERSION,
HOSTDB_MODULE_MINOR_VERSION, PRIVATE_MODULE_HEADER));
ink_split_dns_init(makeModuleVersion(1, 0, PRIVATE_MODULE_HEADER));
+ naVecMutex = new_ProxyMutex();
+ started_et_net_threads = 0;
+
// Do the inits for NetProcessors that use ET_NET threads. MUST be before
starting those threads.
netProcessor.init();
- init_HttpProxyServer();
+ prep_HttpProxyServer();
+
+ if (num_accept_threads == 0) {
+ eventProcessor.schedule_spawn(&init_HttpProxyServer, ET_NET);
+ } else {
+ std::unique_lock<std::mutex> lock(proxyServerMutex);
+ et_net_threads_ready = true;
+ lock.unlock();
+ proxyServerCheck.notify_one();
+ }
// !! ET_NET threads start here !!
// This means any spawn scheduling must be done before this point.
@@ -1934,6 +1948,10 @@ main(int /* argc ATS_UNUSED */, const char **argv)
if (delay_p && ink_atomic_cas(&delay_listen_for_cache_p, 0, 1)) {
Debug("http_listen", "Delaying listen, waiting for cache
initialization");
} else {
+ // Use a condition variable to check if we are ready to call
+ // start_HttpProxyServer() when num_accept_threads is set to 0.
+ std::unique_lock<std::mutex> lock(proxyServerMutex);
+ proxyServerCheck.wait(lock, [] { return et_net_threads_ready; });
start_HttpProxyServer(); // PORTS_READY_HOOK called from in here
}
}
@@ -1946,12 +1964,6 @@ main(int /* argc ATS_UNUSED */, const char **argv)
// "Task" processor, possibly with its own set of task threads
tasksProcessor.start(num_task_threads, stacksize);
- int back_door_port = NO_FD;
- REC_ReadConfigInteger(back_door_port,
"proxy.config.process_manager.mgmt_port");
- if (back_door_port != NO_FD) {
- start_HttpProxyServerBackDoor(back_door_port, num_accept_threads > 0 ? 1
: 0); // One accept thread is enough
- }
-
if (netProcessor.socks_conf_stuff->accept_enabled) {
start_SocksProxy(netProcessor.socks_conf_stuff->accept_port);
}
diff --git a/proxy/http/HttpProxyServerMain.cc
b/proxy/http/HttpProxyServerMain.cc
index 21142ca..e4dce2d 100644
--- a/proxy/http/HttpProxyServerMain.cc
+++ b/proxy/http/HttpProxyServerMain.cc
@@ -39,6 +39,7 @@
#include "ProtocolProbeSessionAccept.h"
#include "http2/Http2SessionAccept.h"
#include "HttpConnectionCount.h"
+#include "HttpProxyServerMain.h"
#include <vector>
@@ -48,6 +49,15 @@ HttpSessionAccept *plugin_http_transparent_accept = nullptr;
static SLL<SSLNextProtocolAccept> ssl_plugin_acceptors;
static Ptr<ProxyMutex> ssl_plugin_mutex;
+// used to keep count of how many et_net threads we have started
+std::atomic<int> started_et_net_threads;
+std::mutex proxyServerMutex;
+std::condition_variable proxyServerCheck;
+bool et_net_threads_ready = false;
+
+extern int num_of_net_threads;
+extern int num_accept_threads;
+
bool
ssl_register_protocol(const char *protocol, Continuation *contp)
{
@@ -227,7 +237,7 @@ MakeHttpProxyAcceptor(HttpProxyAcceptor &acceptor,
HttpProxyPort &port, unsigned
/// Do all pre-thread initialization / setup.
void
-init_HttpProxyServer()
+prep_HttpProxyServer()
{
httpSessionManager.init();
}
@@ -274,6 +284,25 @@ init_accept_HttpProxyServer(int n_accept_threads)
}
}
+/** Increment the counter to keep track of how many et_net threads
+ * we have started. This function is scheduled at the start of each
+ * et_net thread using schedule_spawn(). We also check immediately
+ * after incrementing the counter to see whether all of the et_net
+ * threads have started such that we can notify main() to call
+ * start_HttpProxyServer().
+ */
+void
+init_HttpProxyServer(EThread *)
+{
+ auto check_et_net_num = ++started_et_net_threads;
+ if (check_et_net_num == num_of_net_threads) {
+ std::unique_lock<std::mutex> lock(proxyServerMutex);
+ et_net_threads_ready = true;
+ lock.unlock();
+ proxyServerCheck.notify_one();
+ }
+}
+
void
start_HttpProxyServer()
{
@@ -318,6 +347,14 @@ start_HttpProxyServer()
hook->invoke(TS_EVENT_LIFECYCLE_PORTS_READY, nullptr);
hook = hook->next();
}
+
+ // Start the back door, since it's just a special HttpProxyServer,
+ // the requirements to start it has been met if we got here.
+ int back_door_port = NO_FD;
+ REC_ReadConfigInteger(back_door_port,
"proxy.config.process_manager.mgmt_port");
+ if (back_door_port != NO_FD) {
+ start_HttpProxyServerBackDoor(back_door_port, !!num_accept_threads); //
One accept thread is enough
+ }
}
void
diff --git a/proxy/http/HttpProxyServerMain.h b/proxy/http/HttpProxyServerMain.h
index c640069..039c334 100644
--- a/proxy/http/HttpProxyServerMain.h
+++ b/proxy/http/HttpProxyServerMain.h
@@ -21,17 +21,25 @@
limitations under the License.
*/
+#include <atomic>
+#include <mutex>
+#include <condition_variable>
+
struct HttpProxyPort;
/// Perform any pre-thread start initialization.
-void init_HttpProxyServer();
+void prep_HttpProxyServer();
/** Initialize all HTTP proxy port data structures needed to accept
connections.
*/
void init_accept_HttpProxyServer(int n_accept_threads = 0);
+/** Checkes whether we can call start_HttpProxyServer().
+ */
+void init_HttpProxyServer(EThread *);
+
/** Start the proxy server.
- The port data should have been created by @c init_HttpProxyServer().
+ The port data should have been created by @c prep_HttpProxyServer().
*/
void start_HttpProxyServer();
@@ -40,3 +48,8 @@ void stop_HttpProxyServer();
void start_HttpProxyServerBackDoor(int port, int accept_threads = 0);
NetProcessor::AcceptOptions make_net_accept_options(const HttpProxyPort *port,
unsigned nthreads);
+
+extern std::atomic<int> started_et_net_threads;
+extern std::mutex proxyServerMutex;
+extern std::condition_variable proxyServerCheck;
+extern bool et_net_threads_ready;
--
To stop receiving notification emails like this one, please contact
a...@apache.org.
