This is an automated email from the ASF dual-hosted git repository.
gavinchou pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 310e302bcc9 [fix](be) Stop background workers before startup exit
(#68652)
310e302bcc9 is described below
commit 310e302bcc975b91265ac589a264d80fb25160e5
Author: Gavin Chou <[email protected]>
AuthorDate: Thu Oct 8 12:22:42 2026 +0800
[fix](be) Stop background workers before startup exit (#68652)
When a BE service fails to start with `enable_graceful_exit_check=true`,
the startup failure path calls `exit(1)` while StorageEngine background
workers are still running. `exit(1)` begins process-wide static
destruction before `ExecEnv` and `StorageEngine` are torn down.
A path GC worker can therefore keep matching tablet paths while the
function-local static RE2 object in
`TabletManager::get_tablet_id_and_schema_hash_from_path()` is being
destroyed. This was observed after the backend Thrift port failed to
bind: the path GC thread crashed at address `0x0` in
`re2::Prog::SearchOnePass`, and the main thread subsequently waited
indefinitely for that worker.
This change:
- stops and joins every created server starter on the graceful
startup-failure path;
- releases the backend service before destroying its dependencies;
- explicitly destroys `ExecEnv`, which stops and joins StorageEngine
workers, before calling `exit(1)`;
- preserves the existing immediate `_exit(1)` behavior when graceful
exit checking is disabled.
Failed starters are joined to release partially initialized resources,
while `stop()` is called only for starters that completed startup
successfully.
---
be/src/service/doris_main.cpp | 69 +++++++++++++++++++++++++++++++++----------
1 file changed, 54 insertions(+), 15 deletions(-)
diff --git a/be/src/service/doris_main.cpp b/be/src/service/doris_main.cpp
index fd7f0746e3d..374d698a2a5 100644
--- a/be/src/service/doris_main.cpp
+++ b/be/src/service/doris_main.cpp
@@ -353,10 +353,18 @@ struct Checker {
// running the leak check on this abnormal-exit path reports them as
false-positive
// leaks. enable_graceful_exit_check is honored so memleak-check mode still
runs LSAN.
[[noreturn]] static void exit_on_startup_failure() {
- google::FlushLogFiles(google::GLOG_INFO);
if (!doris::config::enable_graceful_exit_check) {
+ google::FlushLogFiles(google::GLOG_INFO);
_exit(1);
}
+
+ // exit() starts destroying function-local statics while background
threads are still
+ // running. Tear down ExecEnv first so StorageEngine workers are stopped
and joined before
+ // they can race with those destructors.
+ if (doris::ExecEnv::ready()) {
+ doris::ExecEnv::GetInstance()->destroy();
+ }
+ google::FlushLogFiles(google::GLOG_INFO);
exit(1);
}
@@ -662,11 +670,38 @@ int main(int argc, char** argv) {
doris::ThriftRpcHelper::setup(exec_env);
// 1. thrift server with be_port
std::shared_ptr<doris::BaseBackendService> service;
+ std::unique_ptr<doris::server::IServerStarter> backend_thrift_starter;
+ std::unique_ptr<doris::server::IServerStarter> brpc_starter;
+ std::unique_ptr<doris::server::IServerStarter> http_starter;
+ std::unique_ptr<doris::server::IServerStarter> heartbeat_thrift_starter;
+ std::unique_ptr<doris::server::IServerStarter> flight_starter;
+ bool backend_thrift_started = false;
+ bool brpc_started = false;
+ bool http_started = false;
+ bool heartbeat_thrift_started = false;
+ bool flight_started = false;
+ auto stop_and_join_server =
[](std::unique_ptr<doris::server::IServerStarter>& starter,
+ bool started) {
+ if (starter != nullptr) {
+ if (started) {
+ starter->stop();
+ }
+ starter->join();
+ }
+ };
std::function<void(Status&, std::string_view)> stop_work_if_error =
[&](Status& status,
std::string_view msg) {
if (!status.ok()) {
std::cerr << msg << '\n';
service->stop_works();
+ if (doris::config::enable_graceful_exit_check) {
+ stop_and_join_server(flight_starter, flight_started);
+ stop_and_join_server(heartbeat_thrift_starter,
heartbeat_thrift_started);
+ stop_and_join_server(http_starter, http_started);
+ stop_and_join_server(backend_thrift_starter,
backend_thrift_started);
+ stop_and_join_server(brpc_starter, brpc_started);
+ service.reset();
+ }
exit_on_startup_failure();
}
};
@@ -679,44 +714,48 @@ int main(int argc, char** argv) {
exec_env);
}
- std::unique_ptr<doris::server::IServerStarter> backend_thrift_starter;
- EXIT_IF_ERROR(doris::server::create_backend_thrift_starter(exec_env,
doris::config::be_port,
- service,
&backend_thrift_starter));
+ status = doris::server::create_backend_thrift_starter(exec_env,
doris::config::be_port, service,
+
&backend_thrift_starter);
+ stop_work_if_error(status, "Failed to create BE server, exiting");
status = backend_thrift_starter->start();
+ backend_thrift_started = status.ok();
stop_work_if_error(status, "Doris BE server did not start correctly,
exiting");
// 2. brpc service
- std::unique_ptr<doris::server::IServerStarter> brpc_starter;
- EXIT_IF_ERROR(doris::server::create_brpc_starter(
- exec_env, doris::config::brpc_port,
doris::config::brpc_num_threads, &brpc_starter));
+ status = doris::server::create_brpc_starter(exec_env,
doris::config::brpc_port,
+
doris::config::brpc_num_threads, &brpc_starter);
+ stop_work_if_error(status, "Failed to create BRPC service, exiting");
status = brpc_starter->start();
+ brpc_started = status.ok();
stop_work_if_error(status, "BRPC service did not start correctly,
exiting");
// 3. http service
- std::unique_ptr<doris::server::IServerStarter> http_starter;
- EXIT_IF_ERROR(doris::server::create_http_starter(exec_env,
doris::config::webserver_port,
-
doris::config::webserver_num_workers,
- &http_starter));
+ status =
+ doris::server::create_http_starter(exec_env,
doris::config::webserver_port,
+
doris::config::webserver_num_workers, &http_starter);
+ stop_work_if_error(status, "Failed to create BE HTTP service, exiting");
status = http_starter->start();
+ http_started = status.ok();
stop_work_if_error(status, "Doris Be http service did not start correctly,
exiting");
// 4. heart beat server
doris::ClusterInfo* cluster_info = exec_env->cluster_info();
- std::unique_ptr<doris::server::IServerStarter> heartbeat_thrift_starter;
status = doris::server::create_heartbeat_thrift_starter(
exec_env, doris::config::heartbeat_service_port,
doris::config::heartbeat_service_thread_count, cluster_info,
&heartbeat_thrift_starter);
stop_work_if_error(status, "Heartbeat services did not start correctly,
exiting");
status = heartbeat_thrift_starter->start();
+ heartbeat_thrift_started = status.ok();
stop_work_if_error(status, "Doris BE HeartBeat Service did not start
correctly, exiting: " +
status.to_string());
// 5. arrow flight service
- std::unique_ptr<doris::server::IServerStarter> flight_starter;
-
EXIT_IF_ERROR(doris::server::create_flight_starter(doris::config::arrow_flight_sql_port,
- &flight_starter));
+ status =
doris::server::create_flight_starter(doris::config::arrow_flight_sql_port,
+ &flight_starter);
+ stop_work_if_error(status, "Failed to create Arrow Flight service,
exiting");
status = flight_starter->start();
+ flight_started = status.ok();
stop_work_if_error(
status, "Arrow Flight Service did not start correctly, exiting, "
+ status.to_string());
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
