This is an automated email from the ASF dual-hosted git repository. zhangstar333 pushed a commit to branch doris_master_new in repository https://gitbox.apache.org/repos/asf/doris.git
commit 74b0031545452b3653d352d4f85cb9031d426ec1 Author: zhangstar333 <[email protected]> AuthorDate: Fri May 22 17:07:35 2026 +0800 be graceful shutdown --- be/src/agent/heartbeat_server.cpp | 5 +++++ be/src/runtime/exec_env.cpp | 28 ++++++++++++++++++++++++++++ be/src/runtime/exec_env.h | 9 +++++++++ be/src/service/doris_main.cpp | 4 ++++ be/src/service/internal_service.cpp | 7 +++++++ 5 files changed, 53 insertions(+) diff --git a/be/src/agent/heartbeat_server.cpp b/be/src/agent/heartbeat_server.cpp index 441218f469c..2253e096b17 100644 --- a/be/src/agent/heartbeat_server.cpp +++ b/be/src/agent/heartbeat_server.cpp @@ -90,6 +90,11 @@ void HeartbeatServer::heartbeat(THeartbeatResult& heartbeat_result, heartbeat_result.backend_info.__set_be_node_role(config::be_node_role); // If be is gracefully stop, then k_doris_exist is set to true heartbeat_result.backend_info.__set_is_shutdown(doris::k_doris_exit); + if (doris::k_doris_exit) { + // FE Master has pulled at least one heartbeat carrying + // is_shutdown=true. Unblock Phase A in graceful_shutdown. + doris::k_shutdown_fe_known = true; + } heartbeat_result.backend_info.__set_fragment_executing_count( get_fragment_executing_count()); heartbeat_result.backend_info.__set_fragment_last_active_time( diff --git a/be/src/runtime/exec_env.cpp b/be/src/runtime/exec_env.cpp index 0ce4544ddc8..5189eac6d55 100644 --- a/be/src/runtime/exec_env.cpp +++ b/be/src/runtime/exec_env.cpp @@ -170,6 +170,34 @@ std::map<TNetworkAddress, FrontendInfo> ExecEnv::get_running_frontends() { return res; } +void ExecEnv::wait_for_all_fe_known() { + // Phase A of graceful shutdown: + // Heartbeat callback (HeartbeatServer::heartbeat) sets + // k_shutdown_fe_known=true the next time Master pulls a heartbeat + // (which carries is_shutdown=true). Block here up to 30s for that to + // happen, then sleep an extra 10s so the resulting OP_HEARTBEAT EditLog + // can be replicated by BDBJE to all FE Followers before we start + // tearing down running tasks. The 30s deadline is a safety net for the + // case where Master is itself in drain / election and heartbeats are + // delayed. + constexpr int32_t kWaitLimitSeconds = 30; + constexpr int32_t kBufferSeconds = 10; + int32_t passed = 0; + while (!k_shutdown_fe_known && passed < kWaitLimitSeconds) { + sleep(1); + ++passed; + } + if (k_shutdown_fe_known) { + LOG(INFO) << "FE Master has acknowledged shutdown after " << passed + << "s, sleeping additional " << kBufferSeconds + << "s to let OP_HEARTBEAT EditLog replicate to all Followers."; + } else { + LOG(WARNING) << "FE Master did not acknowledge shutdown within " << kWaitLimitSeconds + << "s, proceeding anyway."; + } + sleep(kBufferSeconds); +} + void ExecEnv::wait_for_all_tasks_done() { // For graceful shutdown, need to wait for all running queries to stop int32_t wait_seconds_passed = 0; diff --git a/be/src/runtime/exec_env.h b/be/src/runtime/exec_env.h index 3198be77337..dfbd1eee0dd 100644 --- a/be/src/runtime/exec_env.h +++ b/be/src/runtime/exec_env.h @@ -140,6 +140,9 @@ class DeleteBitmapAggCache; // set to true when BE is shutting down inline bool k_doris_exit = false; +// set to true once FE Master has pulled a heartbeat carrying is_shutdown=true. +// Used by Phase A of graceful shutdown to know FE has been informed. +inline bool k_shutdown_fe_known = false; // set to true after BE start ready inline bool k_is_server_ready = false; @@ -383,6 +386,12 @@ public: void wait_for_all_tasks_done(); + // Phase A of graceful shutdown: block until FE Master has observed + // our is_shutdown=true (via heartbeat round-trip), then sleep an extra + // buffer so the OP_HEARTBEAT EditLog can propagate to all Followers. + // Bounded by config::grace_shutdown_fe_known_wait_seconds. + void wait_for_all_fe_known(); + void update_frontends(const std::vector<TFrontendInfo>& new_infos); std::vector<TFrontendInfo> get_frontends(); std::map<TNetworkAddress, FrontendInfo> get_running_frontends(); diff --git a/be/src/service/doris_main.cpp b/be/src/service/doris_main.cpp index 12f5af0e2a4..abd30278565 100644 --- a/be/src/service/doris_main.cpp +++ b/be/src/service/doris_main.cpp @@ -714,6 +714,10 @@ int main(int argc, char** argv) { LOG(INFO) << "Flush profile file."; #endif // For graceful shutdown, need to wait for all running queries to stop + // Phase A: wait for FE Master to learn we are shutting down (via heartbeat), + // then sleep an extra buffer so the OP_HEARTBEAT EditLog reaches all Followers. + exec_env->wait_for_all_fe_known(); + // Phase B: wait for in-flight queries to finish. exec_env->wait_for_all_tasks_done(); if (!doris::config::enable_graceful_exit_check) { diff --git a/be/src/service/internal_service.cpp b/be/src/service/internal_service.cpp index 67df019be26..c6ed81e8616 100644 --- a/be/src/service/internal_service.cpp +++ b/be/src/service/internal_service.cpp @@ -355,6 +355,9 @@ void PInternalService::exec_plan_fragment(google::protobuf::RpcController* contr timeval tv {}; gettimeofday(&tv, nullptr); response->set_received_time(tv.tv_sec * 1000LL + tv.tv_usec / 1000); + if (k_doris_exit) { + LOG(WARNING) << "BE is shutting down but still received exec_plan_fragment request."; + } bool ret = _light_work_pool.try_offer([this, controller, request, response, done]() { _exec_plan_fragment_in_pthread(controller, request, response, done); }); @@ -402,6 +405,10 @@ void PInternalService::exec_plan_fragment_prepare(google::protobuf::RpcControlle timeval tv {}; gettimeofday(&tv, nullptr); response->set_received_time(tv.tv_sec * 1000LL + tv.tv_usec / 1000); + if (k_doris_exit) { + LOG(WARNING) << "BE is shutting down but still received exec_plan_fragment_prepare " + "request."; + } bool ret = _light_work_pool.try_offer([this, controller, request, response, done]() { _exec_plan_fragment_in_pthread(controller, request, response, done); }); --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
