This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-4.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-4.1 by this push:
new 6217a2c6186 branch-4.1: [fix](be) Stop background workers before
startup exit #68652 (#68653)
6217a2c6186 is described below
commit 6217a2c6186046ddf2d103843c2f95fb3fb352ca
Author: Gavin Chou <[email protected]>
AuthorDate: Thu Oct 8 10:13:24 2026 +0800
branch-4.1: [fix](be) Stop background workers before startup exit #68652
(#68653)
### What problem does this PR solve?
Issue Number: None
Related PR: #68652
Problem Summary:
This PR backports the startup-failure lifecycle fix from #68652 to
`branch-4.1`.
When a BE service fails to start with `enable_graceful_exit_check=true`,
the startup failure path calls `exit(1)` while StorageEngine background
workers are still running. Process-wide static destruction can then race
with a path GC worker that is using the function-local static RE2
matcher in `TabletManager::get_tablet_id_and_schema_hash_from_path()`,
causing a null-address crash in `re2::Prog::SearchOnePass`.
The fix stops and joins created server starters, releases the backend
service, and explicitly destroys `ExecEnv` before calling `exit(1)`. The
immediate `_exit(1)` path remains unchanged when graceful exit checking
is disabled.
The backport has the same stable patch-id as #68652.
### Release note
None
### Check List (For Author)
- Test:
- [x] Manual test
- Targeted clang-format 16 check passed for
`be/src/service/doris_main.cpp` on `branch-4.1`.
- A local BE build could not start because the macOS environment does
not have Homebrew; `run buildall` is requested below for full build
validation.
- Behavior changed:
- [ ] Yes
- [x] No. Only abnormal BE startup cleanup ordering changes.
- Does this need documentation?
- [x] No
- [ ] Yes
---
be/src/service/doris_main.cpp | 69 +++++++++++++++++++++++++++++++++----------
1 file changed, 54 insertions(+), 15 deletions(-)
diff --git a/be/src/service/doris_main.cpp b/be/src/service/doris_main.cpp
index 346ab386b15..2f31e3055a0 100644
--- a/be/src/service/doris_main.cpp
+++ b/be/src/service/doris_main.cpp
@@ -326,10 +326,18 @@ struct Checker {
// running the leak check on this abnormal-exit path reports them as
false-positive
// leaks. enable_graceful_exit_check is honored so memleak-check mode still
runs LSAN.
[[noreturn]] static void exit_on_startup_failure() {
- google::FlushLogFiles(google::GLOG_INFO);
if (!doris::config::enable_graceful_exit_check) {
+ google::FlushLogFiles(google::GLOG_INFO);
_exit(1);
}
+
+ // exit() starts destroying function-local statics while background
threads are still
+ // running. Tear down ExecEnv first so StorageEngine workers are stopped
and joined before
+ // they can race with those destructors.
+ if (doris::ExecEnv::ready()) {
+ doris::ExecEnv::GetInstance()->destroy();
+ }
+ google::FlushLogFiles(google::GLOG_INFO);
exit(1);
}
@@ -635,11 +643,38 @@ int main(int argc, char** argv) {
doris::ThriftRpcHelper::setup(exec_env);
// 1. thrift server with be_port
std::shared_ptr<doris::BaseBackendService> service;
+ std::unique_ptr<doris::server::IServerStarter> backend_thrift_starter;
+ std::unique_ptr<doris::server::IServerStarter> brpc_starter;
+ std::unique_ptr<doris::server::IServerStarter> http_starter;
+ std::unique_ptr<doris::server::IServerStarter> heartbeat_thrift_starter;
+ std::unique_ptr<doris::server::IServerStarter> flight_starter;
+ bool backend_thrift_started = false;
+ bool brpc_started = false;
+ bool http_started = false;
+ bool heartbeat_thrift_started = false;
+ bool flight_started = false;
+ auto stop_and_join_server =
[](std::unique_ptr<doris::server::IServerStarter>& starter,
+ bool started) {
+ if (starter != nullptr) {
+ if (started) {
+ starter->stop();
+ }
+ starter->join();
+ }
+ };
std::function<void(Status&, std::string_view)> stop_work_if_error =
[&](Status& status,
std::string_view msg) {
if (!status.ok()) {
std::cerr << msg << '\n';
service->stop_works();
+ if (doris::config::enable_graceful_exit_check) {
+ stop_and_join_server(flight_starter, flight_started);
+ stop_and_join_server(heartbeat_thrift_starter,
heartbeat_thrift_started);
+ stop_and_join_server(http_starter, http_started);
+ stop_and_join_server(backend_thrift_starter,
backend_thrift_started);
+ stop_and_join_server(brpc_starter, brpc_started);
+ service.reset();
+ }
exit_on_startup_failure();
}
};
@@ -652,44 +687,48 @@ int main(int argc, char** argv) {
exec_env);
}
- std::unique_ptr<doris::server::IServerStarter> backend_thrift_starter;
- EXIT_IF_ERROR(doris::server::create_backend_thrift_starter(exec_env,
doris::config::be_port,
- service,
&backend_thrift_starter));
+ status = doris::server::create_backend_thrift_starter(exec_env,
doris::config::be_port, service,
+
&backend_thrift_starter);
+ stop_work_if_error(status, "Failed to create BE server, exiting");
status = backend_thrift_starter->start();
+ backend_thrift_started = status.ok();
stop_work_if_error(status, "Doris BE server did not start correctly,
exiting");
// 2. brpc service
- std::unique_ptr<doris::server::IServerStarter> brpc_starter;
- EXIT_IF_ERROR(doris::server::create_brpc_starter(
- exec_env, doris::config::brpc_port,
doris::config::brpc_num_threads, &brpc_starter));
+ status = doris::server::create_brpc_starter(exec_env,
doris::config::brpc_port,
+
doris::config::brpc_num_threads, &brpc_starter);
+ stop_work_if_error(status, "Failed to create BRPC service, exiting");
status = brpc_starter->start();
+ brpc_started = status.ok();
stop_work_if_error(status, "BRPC service did not start correctly,
exiting");
// 3. http service
- std::unique_ptr<doris::server::IServerStarter> http_starter;
- EXIT_IF_ERROR(doris::server::create_http_starter(exec_env,
doris::config::webserver_port,
-
doris::config::webserver_num_workers,
- &http_starter));
+ status =
+ doris::server::create_http_starter(exec_env,
doris::config::webserver_port,
+
doris::config::webserver_num_workers, &http_starter);
+ stop_work_if_error(status, "Failed to create BE HTTP service, exiting");
status = http_starter->start();
+ http_started = status.ok();
stop_work_if_error(status, "Doris Be http service did not start correctly,
exiting");
// 4. heart beat server
doris::ClusterInfo* cluster_info = exec_env->cluster_info();
- std::unique_ptr<doris::server::IServerStarter> heartbeat_thrift_starter;
status = doris::server::create_heartbeat_thrift_starter(
exec_env, doris::config::heartbeat_service_port,
doris::config::heartbeat_service_thread_count, cluster_info,
&heartbeat_thrift_starter);
stop_work_if_error(status, "Heartbeat services did not start correctly,
exiting");
status = heartbeat_thrift_starter->start();
+ heartbeat_thrift_started = status.ok();
stop_work_if_error(status, "Doris BE HeartBeat Service did not start
correctly, exiting: " +
status.to_string());
// 5. arrow flight service
- std::unique_ptr<doris::server::IServerStarter> flight_starter;
-
EXIT_IF_ERROR(doris::server::create_flight_starter(doris::config::arrow_flight_sql_port,
- &flight_starter));
+ status =
doris::server::create_flight_starter(doris::config::arrow_flight_sql_port,
+ &flight_starter);
+ stop_work_if_error(status, "Failed to create Arrow Flight service,
exiting");
status = flight_starter->start();
+ flight_started = status.ok();
stop_work_if_error(
status, "Arrow Flight Service did not start correctly, exiting, "
+ status.to_string());
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
