uchenily commented on PR #61240:
URL: https://github.com/apache/doris/pull/61240#issuecomment-4066242731
It can be reproduced by injecting an `IO_ERROR` and triggering the
`DataDir::health_check`. I made the following modifications in version 2.1:
```diff
diff --git a/be/src/agent/task_worker_pool.cpp
b/be/src/agent/task_worker_pool.cpp
index c36426b1954..b6af569687d 100644
--- a/be/src/agent/task_worker_pool.cpp
+++ b/be/src/agent/task_worker_pool.cpp
@@ -1387,6 +1387,13 @@ void create_tablet_callback(StorageEngine& engine,
const TAgentTaskRequest& req)
.error(status);
} else {
increase_report_version();
+ DBUG_EXECUTE_IF("create_tablet_callback.block", DBUG_BLOCK);
+ DBUG_EXECUTE_IF("create_tablet_callback.health_check", {
+ auto created_tablet =
engine.tablet_manager()->get_tablet(create_tablet_req.tablet_id);
+ if (created_tablet != nullptr) {
+ created_tablet->data_dir()->health_check();
+ }
+ });
// get path hash of the created tablet
TabletSharedPtr tablet;
{
diff --git a/be/src/olap/data_dir.cpp b/be/src/olap/data_dir.cpp
index 7cbc3543062..31ff9e733e2 100644
--- a/be/src/olap/data_dir.cpp
+++ b/be/src/olap/data_dir.cpp
@@ -63,6 +63,7 @@
#include "olap/txn_manager.h"
#include "olap/utils.h" // for check_dir_existed
#include "service/backend_options.h"
+#include "util/debug_points.h"
#include "util/doris_metrics.h"
#include "util/string_util.h"
#include "util/uid_util.h"
@@ -250,6 +251,9 @@ void DataDir::health_check() {
}
Status DataDir::_read_and_write_test_file() {
+ DBUG_EXECUTE_IF("_read_and_write_test_file.io_error", {
+ return Status::Error<IO_ERROR>("debug point injected io error");
+ });
auto test_file = fmt::format("{}/{}", _path, kTestFilePath);
return read_write_test_file(test_file);
}
```
1. enable block debug point
```bash
curl -XPOST localhost:8040/api/debug_point/add/create_tablet_callback.block
```
2. create a table
```sql
DROP TABLE IF EXISTS t1;
CREATE TABLE t1 (
id BIGINT,
v1 STRING
)
DUPLICATE KEY(id)
DISTRIBUTED BY HASH(id) BUCKETS 1
PROPERTIES (
"replication_num" = "1"
);
```
3. raise IO_ERROR and trigger the DataDir::health_check
```bash
curl -XPOST
localhost:8040/api/debug_point/add/_read_and_write_test_file.io_error
curl -XPOST
localhost:8040/api/debug_point/add/create_tablet_callback.health_check
curl -XPOST
localhost:8040/api/debug_point/remove/create_tablet_callback.block
```
then, we will get the following error:
```text
RuntimeLogger W20260316 17:14:58.979130 85806 status.h:415] meet error
status: [IO_ERROR]debug point injected io error
0# doris::Status doris::Status::Error<34,
true>(std::basic_string_view<char, std::char_traits<char> >) at
/root/work/doris-2.1/be/src/common/status.h:414
1# doris::DataDir::_read_and_write_test_file() at
/root/work/doris-2.1/be/src/olap/data_dir.cpp:254
2# doris::DataDir::health_check() at
/root/work/doris-2.1/be/src/olap/data_dir.cpp:242
3# doris::create_tablet_callback(doris::StorageEngine&,
doris::TAgentTaskRequest const&) at
/root/work/doris-2.1/be/src/agent/task_worker_pool.cpp:1391
4# auto
doris::AgentServer::start_workers(doris::ExecEnv*)::$_10::operator()<doris::TAgentTaskRequest
const&>(doris::TAgentTaskRequest const&) const at
/root/work/doris-2.1/be/src/agent/agent_server.cpp:123
5# void std::__invoke_impl<void,
doris::AgentServer::start_workers(doris::ExecEnv*)::$_10&,
doris::TAgentTaskRequest const&>(std::__invoke_other,
doris::AgentServer::start_workers(doris::ExecEnv*)::$_10&,
doris::TAgentTaskRequest const&) at
/opt/ldb_toolchain/bin/../lib/gcc/x86_64-pc-linux-gnu/15/include/g++-v15/bits/invoke.h:63
6# std::enable_if<is_invocable_r_v<void,
doris::AgentServer::start_workers(doris::ExecEnv*)::$_10&,
doris::TAgentTaskRequest const&>, void>::type std::__invoke_r<void,
doris::AgentServer::start_workers(doris::ExecEnv*)::$_10&,
doris::TAgentTaskRequest
const&>(doris::AgentServer::start_workers(doris::ExecEnv*)::$_10&,
doris::TAgentTaskRequest const&) at
/opt/ldb_toolchain/bin/../lib/gcc/x86_64-pc-linux-gnu/15/include/g++-v15/bits/invoke.h:119
7# std::_Function_handler<void (doris::TAgentTaskRequest const&),
doris::AgentServer::start_workers(doris::ExecEnv*)::$_10>::_M_invoke(std::_Any_data
const&, doris::TAgentTaskRequest const&) at
/opt/ldb_toolchain/bin/../lib/gcc/x86_64-pc-linux-gnu/15/include/g++-v15/bits/std_function.h:292
8# std::function<void (doris::TAgentTaskRequest
const&)>::operator()(doris::TAgentTaskRequest const&) const at
/opt/ldb_toolchain/bin/../lib/gcc/x86_64-pc-linux-gnu/15/include/g++-v15/bits/std_function.h:593
9# doris::TaskWorkerPool::submit_task(doris::TAgentTaskRequest
const&)::$_0::operator()<doris::TAgentTaskRequest
const&>(doris::TAgentTaskRequest const&) const::{lambda()#1}::operator()()
const at /root/work/doris-2.1/be/src/agent/task_worker_pool.cpp:444
10# void std::__invoke_impl<void,
doris::TaskWorkerPool::submit_task(doris::TAgentTaskRequest
const&)::$_0::operator()<doris::TAgentTaskRequest
const&>(doris::TAgentTaskRequest const&)
const::{lambda()#1}&>(std::__invoke_other,
doris::TaskWorkerPool::submit_task(doris::TAgentTaskRequest
const&)::$_0::operator()<doris::TAgentTaskRequest
const&>(doris::TAgentTaskRequest const&) const::{lambda()#1}&) at
/opt/ldb_toolchain/bin/../lib/gcc/x86_64-pc-linux-gnu/15/include/g++-v15/bits/invoke.h:63
11# std::enable_if<is_invocable_r_v<void,
doris::TaskWorkerPool::submit_task(doris::TAgentTaskRequest
const&)::$_0::operator()<doris::TAgentTaskRequest
const&>(doris::TAgentTaskRequest const&) const::{lambda()#1}&>, void>::type
std::__invoke_r<void,
doris::TaskWorkerPool::submit_task(doris::TAgentTaskRequest
const&)::$_0::operator()<doris::TAgentTaskRequest
const&>(doris::TAgentTaskRequest const&)
const::{lambda()#1}&>(doris::TaskWorkerPool::submit_task(doris::TAgentTaskRequest
const&)::$_0::operator()<doris::TAgentTaskRequest
const&>(doris::TAgentTaskRequest const&) const::{lambda()#1}&) at
/opt/ldb_toolchain/bin/../lib/gcc/x86_64-pc-linux-gnu/15/include/g++-v15/bits/invoke.h:119
12# std::_Function_handler<void (),
doris::TaskWorkerPool::submit_task(doris::TAgentTaskRequest
const&)::$_0::operator()<doris::TAgentTaskRequest
const&>(doris::TAgentTaskRequest const&)
const::{lambda()#1}>::_M_invoke(std::_Any_data const&) at
/opt/ldb_toolchain/bin/../lib/gcc/x86_64-pc-linux-gnu/15/include/g++-v15/bits/std_function.h:292
13# std::function<void ()>::operator()() const at
/opt/ldb_toolchain/bin/../lib/gcc/x86_64-pc-linux-gnu/15/include/g++-v15/bits/std_function.h:593
14# doris::FunctionRunnable::run() at
/root/work/doris-2.1/be/src/util/threadpool.cpp:48
15# doris::ThreadPool::dispatch_thread() at
/root/work/doris-2.1/be/src/util/threadpool.cpp:544
16# void std::__invoke_impl<void, void (doris::ThreadPool::*&)(),
doris::ThreadPool*&>(std::__invoke_memfun_deref, void
(doris::ThreadPool::*&)(), doris::ThreadPool*&) at
/opt/ldb_toolchain/bin/../lib/gcc/x86_64-pc-linux-gnu/15/include/g++-v15/bits/invoke.h:76
17# std::__invoke_result<void (doris::ThreadPool::*&)(),
doris::ThreadPool*&>::type std::__invoke<void (doris::ThreadPool::*&)(),
doris::ThreadPool*&>(void (doris::ThreadPool::*&)(), doris::ThreadPool*&) at
/opt/ldb_toolchain/bin/../lib/gcc/x86_64-pc-linux-gnu/15/include/g++-v15/bits/invoke.h:98
18# void std::_Bind<void
(doris::ThreadPool::*(doris::ThreadPool*))()>::__call<void, ,
0ul>(std::tuple<>&&, std::_Index_tuple<0ul>) at
/opt/ldb_toolchain/bin/../lib/gcc/x86_64-pc-linux-gnu/15/include/g++-v15/functional:515
19# void std::_Bind<void
(doris::ThreadPool::*(doris::ThreadPool*))()>::operator()<, void>() at
/opt/ldb_toolchain/bin/../lib/gcc/x86_64-pc-linux-gnu/15/include/g++-v15/functional:600
20# void std::__invoke_impl<void, std::_Bind<void
(doris::ThreadPool::*(doris::ThreadPool*))()>&>(std::__invoke_other,
std::_Bind<void (doris::ThreadPool::*(doris::ThreadPool*))()>&) at
/opt/ldb_toolchain/bin/../lib/gcc/x86_64-pc-linux-gnu/15/include/g++-v15/bits/invoke.h:63
21# std::enable_if<is_invocable_r_v<void, std::_Bind<void
(doris::ThreadPool::*(doris::ThreadPool*))()>&>, void>::type
std::__invoke_r<void, std::_Bind<void
(doris::ThreadPool::*(doris::ThreadPool*))()>&>(std::_Bind<void
(doris::ThreadPool::*(doris::ThreadPool*))()>&) at
/opt/ldb_toolchain/bin/../lib/gcc/x86_64-pc-linux-gnu/15/include/g++-v15/bits/invoke.h:119
22# std::_Function_handler<void (), std::_Bind<void
(doris::ThreadPool::*(doris::ThreadPool*))()> >::_M_invoke(std::_Any_data
const&) at
/opt/ldb_toolchain/bin/../lib/gcc/x86_64-pc-linux-gnu/15/include/g++-v15/bits/std_function.h:292
23# std::function<void ()>::operator()() const at
/opt/ldb_toolchain/bin/../lib/gcc/x86_64-pc-linux-gnu/15/include/g++-v15/bits/std_function.h:593
24# doris::Thread::supervise_thread(void*) at
/root/work/doris-2.1/be/src/util/thread.cpp:498
25# ?
26# ?
RuntimeLogger W20260316 17:14:58.979444 85806 data_dir.cpp:244] store
read/write test file occur IO Error. path=/root/work/doris-2.1/storage, err:
[IO_ERROR]debug point injected io error
0# doris::Status doris::Status::Error<34,
true>(std::basic_string_view<char, std::char_traits<char> >) at
/root/work/doris-2.1/be/src/common/status.h:414
1# doris::DataDir::_read_and_write_test_file() at
/root/work/doris-2.1/be/src/olap/data_dir.cpp:254
2# doris::DataDir::health_check() at
/root/work/doris-2.1/be/src/olap/data_dir.cpp:242
3# doris::create_tablet_callback(doris::StorageEngine&,
doris::TAgentTaskRequest const&) at
/root/work/doris-2.1/be/src/agent/task_worker_pool.cpp:1391
4# auto
doris::AgentServer::start_workers(doris::ExecEnv*)::$_10::operator()<doris::TAgentTaskRequest
const&>(doris::TAgentTaskRequest const&) const at
/root/work/doris-2.1/be/src/agent/agent_server.cpp:123
5# void std::__invoke_impl<void,
doris::AgentServer::start_workers(doris::ExecEnv*)::$_10&,
doris::TAgentTaskRequest const&>(std::__invoke_other,
doris::AgentServer::start_workers(doris::ExecEnv*)::$_10&,
doris::TAgentTaskRequest const&) at
/opt/ldb_toolchain/bin/../lib/gcc/x86_64-pc-linux-gnu/15/include/g++-v15/bits/invoke.h:63
6# std::enable_if<is_invocable_r_v<void,
doris::AgentServer::start_workers(doris::ExecEnv*)::$_10&,
doris::TAgentTaskRequest const&>, void>::type std::__invoke_r<void,
doris::AgentServer::start_workers(doris::ExecEnv*)::$_10&,
doris::TAgentTaskRequest
const&>(doris::AgentServer::start_workers(doris::ExecEnv*)::$_10&,
doris::TAgentTaskRequest const&) at
/opt/ldb_toolchain/bin/../lib/gcc/x86_64-pc-linux-gnu/15/include/g++-v15/bits/invoke.h:119
7# std::_Function_handler<void (doris::TAgentTaskRequest const&),
doris::AgentServer::start_workers(doris::ExecEnv*)::$_10>::_M_invoke(std::_Any_data
const&, doris::TAgentTaskRequest const&) at
/opt/ldb_toolchain/bin/../lib/gcc/x86_64-pc-linux-gnu/15/include/g++-v15/bits/std_function.h:292
8# std::function<void (doris::TAgentTaskRequest
const&)>::operator()(doris::TAgentTaskRequest const&) const at
/opt/ldb_toolchain/bin/../lib/gcc/x86_64-pc-linux-gnu/15/include/g++-v15/bits/std_function.h:593
9# doris::TaskWorkerPool::submit_task(doris::TAgentTaskRequest
const&)::$_0::operator()<doris::TAgentTaskRequest
const&>(doris::TAgentTaskRequest const&) const::{lambda()#1}::operator()()
const at /root/work/doris-2.1/be/src/agent/task_worker_pool.cpp:444
10# void std::__invoke_impl<void,
doris::TaskWorkerPool::submit_task(doris::TAgentTaskRequest
const&)::$_0::operator()<doris::TAgentTaskRequest
const&>(doris::TAgentTaskRequest const&)
const::{lambda()#1}&>(std::__invoke_other,
doris::TaskWorkerPool::submit_task(doris::TAgentTaskRequest
const&)::$_0::operator()<doris::TAgentTaskRequest
const&>(doris::TAgentTaskRequest const&) const::{lambda()#1}&) at
/opt/ldb_toolchain/bin/../lib/gcc/x86_64-pc-linux-gnu/15/include/g++-v15/bits/invoke.h:63
11# std::enable_if<is_invocable_r_v<void,
doris::TaskWorkerPool::submit_task(doris::TAgentTaskRequest
const&)::$_0::operator()<doris::TAgentTaskRequest
const&>(doris::TAgentTaskRequest const&) const::{lambda()#1}&>, void>::type
std::__invoke_r<void,
doris::TaskWorkerPool::submit_task(doris::TAgentTaskRequest
const&)::$_0::operator()<doris::TAgentTaskRequest
const&>(doris::TAgentTaskRequest const&)
const::{lambda()#1}&>(doris::TaskWorkerPool::submit_task(doris::TAgentTaskRequest
const&)::$_0::operator()<doris::TAgentTaskRequest
const&>(doris::TAgentTaskRequest const&) const::{lambda()#1}&) at
/opt/ldb_toolchain/bin/../lib/gcc/x86_64-pc-linux-gnu/15/include/g++-v15/bits/invoke.h:119
12# std::_Function_handler<void (),
doris::TaskWorkerPool::submit_task(doris::TAgentTaskRequest
const&)::$_0::operator()<doris::TAgentTaskRequest
const&>(doris::TAgentTaskRequest const&)
const::{lambda()#1}>::_M_invoke(std::_Any_data const&) at
/opt/ldb_toolchain/bin/../lib/gcc/x86_64-pc-linux-gnu/15/include/g++-v15/bits/std_function.h:292
13# std::function<void ()>::operator()() const at
/opt/ldb_toolchain/bin/../lib/gcc/x86_64-pc-linux-gnu/15/include/g++-v15/bits/std_function.h:593
14# doris::FunctionRunnable::run() at
/root/work/doris-2.1/be/src/util/threadpool.cpp:48
15# doris::ThreadPool::dispatch_thread() at
/root/work/doris-2.1/be/src/util/threadpool.cpp:544
16# void std::__invoke_impl<void, void (doris::ThreadPool::*&)(),
doris::ThreadPool*&>(std::__invoke_memfun_deref, void
(doris::ThreadPool::*&)(), doris::ThreadPool*&) at
/opt/ldb_toolchain/bin/../lib/gcc/x86_64-pc-linux-gnu/15/include/g++-v15/bits/invoke.h:76
17# std::__invoke_result<void (doris::ThreadPool::*&)(),
doris::ThreadPool*&>::type std::__invoke<void (doris::ThreadPool::*&)(),
doris::ThreadPool*&>(void (doris::ThreadPool::*&)(), doris::ThreadPool*&) at
/opt/ldb_toolchain/bin/../lib/gcc/x86_64-pc-linux-gnu/15/include/g++-v15/bits/invoke.h:98
18# void std::_Bind<void
(doris::ThreadPool::*(doris::ThreadPool*))()>::__call<void, ,
0ul>(std::tuple<>&&, std::_Index_tuple<0ul>) at
/opt/ldb_toolchain/bin/../lib/gcc/x86_64-pc-linux-gnu/15/include/g++-v15/functional:515
19# void std::_Bind<void
(doris::ThreadPool::*(doris::ThreadPool*))()>::operator()<, void>() at
/opt/ldb_toolchain/bin/../lib/gcc/x86_64-pc-linux-gnu/15/include/g++-v15/functional:600
20# void std::__invoke_impl<void, std::_Bind<void
(doris::ThreadPool::*(doris::ThreadPool*))()>&>(std::__invoke_other,
std::_Bind<void (doris::ThreadPool::*(doris::ThreadPool*))()>&) at
/opt/ldb_toolchain/bin/../lib/gcc/x86_64-pc-linux-gnu/15/include/g++-v15/bits/invoke.h:63
21# std::enable_if<is_invocable_r_v<void, std::_Bind<void
(doris::ThreadPool::*(doris::ThreadPool*))()>&>, void>::type
std::__invoke_r<void, std::_Bind<void
(doris::ThreadPool::*(doris::ThreadPool*))()>&>(std::_Bind<void
(doris::ThreadPool::*(doris::ThreadPool*))()>&) at
/opt/ldb_toolchain/bin/../lib/gcc/x86_64-pc-linux-gnu/15/include/g++-v15/bits/invoke.h:119
22# std::_Function_handler<void (), std::_Bind<void
(doris::ThreadPool::*(doris::ThreadPool*))()> >::_M_invoke(std::_Any_data
const&) at
/opt/ldb_toolchain/bin/../lib/gcc/x86_64-pc-linux-gnu/15/include/g++-v15/bits/std_function.h:292
23# std::function<void ()>::operator()() const at
/opt/ldb_toolchain/bin/../lib/gcc/x86_64-pc-linux-gnu/15/include/g++-v15/bits/std_function.h:593
24# doris::Thread::supervise_thread(void*) at
/root/work/doris-2.1/be/src/util/thread.cpp:498
25# ?
26# ?
RuntimeLogger I20260316 17:14:59.001428 85806 storage_engine.cpp:1518]
persist broken_storae_path /root/work/doris-2.1/storage;[OK]
RuntimeLogger W20260316 17:14:59.001538 85806 tablet_manager.cpp:653] tablet
cannot be used. tablet=21084
RuntimeLogger F20260316 17:14:59.001605 85806 task_worker_pool.cpp:1403]
Check failed: tablet != nullptr
*** Check failure stack trace: ***
@ 0x5ad4937d5766 google::LogMessage::SendToLog()
RuntimeLogger W20260316 17:14:59.025067 85694 status.h:415] meet error
status: [IO_ERROR]debug point injected io error
0# doris::Status doris::Status::Error<34,
true>(std::basic_string_view<char, std::char_traits<char> >) at
/root/work/doris-2.1/be/src/common/status.h:414
1# doris::DataDir::_read_and_write_test_file() at
/root/work/doris-2.1/be/src/olap/data_dir.cpp:254
2# doris::DataDir::health_check() at
/root/work/doris-2.1/be/src/olap/data_dir.cpp:242
3# doris::StorageEngine::_start_disk_stat_monitor() at
/root/work/doris-2.1/be/src/olap/storage_engine.cpp:394
4# doris::StorageEngine::_disk_stat_monitor_thread_callback() at
/root/work/doris-2.1/be/src/olap/olap_server.cpp:349
5#
doris::StorageEngine::start_bg_threads(std::shared_ptr<doris::WorkloadGroup>)::$_2::operator()()
const at /root/work/doris-2.1/be/src/olap/olap_server.cpp:142
6# void std::__invoke_impl<void,
doris::StorageEngine::start_bg_threads(std::shared_ptr<doris::WorkloadGroup>)::$_2&>(std::__invoke_other,
doris::StorageEngine::start_bg_threads(std::shared_ptr<doris::WorkloadGroup>)::$_2&)
at
/opt/ldb_toolchain/bin/../lib/gcc/x86_64-pc-linux-gnu/15/include/g++-v15/bits/invoke.h:63
7# std::enable_if<is_invocable_r_v<void,
doris::StorageEngine::start_bg_threads(std::shared_ptr<doris::WorkloadGroup>)::$_2&>,
void>::type std::__invoke_r<void,
doris::StorageEngine::start_bg_threads(std::shared_ptr<doris::WorkloadGroup>)::$_2&>(doris::StorageEngine::start_bg_threads(std::shared_ptr<doris::WorkloadGroup>)::$_2&)
at
/opt/ldb_toolchain/bin/../lib/gcc/x86_64-pc-linux-gnu/15/include/g++-v15/bits/invoke.h:119
8# std::_Function_handler<void (),
doris::StorageEngine::start_bg_threads(std::shared_ptr<doris::WorkloadGroup>)::$_2>::_M_invoke(std::_Any_data
const&) at
/opt/ldb_toolchain/bin/../lib/gcc/x86_64-pc-linux-gnu/15/include/g++-v15/bits/std_function.h:292
9# std::function<void ()>::operator()() const at
/opt/ldb_toolchain/bin/../lib/gcc/x86_64-pc-linux-gnu/15/include/g++-v15/bits/std_function.h:593
10# doris::Thread::supervise_thread(void*) at
/root/work/doris-2.1/be/src/util/thread.cpp:498
11# ?
12# ?
RuntimeLogger W20260316 17:14:59.025279 85694 data_dir.cpp:244] store
read/write test file occur IO Error. path=/root/work/doris-2.1/storage, err:
[IO_ERROR]debug point injected io error
0# doris::Status doris::Status::Error<34,
true>(std::basic_string_view<char, std::char_traits<char> >) at
/root/work/doris-2.1/be/src/common/status.h:414
1# doris::DataDir::_read_and_write_test_file() at
/root/work/doris-2.1/be/src/olap/data_dir.cpp:254
2# doris::DataDir::health_check() at
/root/work/doris-2.1/be/src/olap/data_dir.cpp:242
3# doris::StorageEngine::_start_disk_stat_monitor() at
/root/work/doris-2.1/be/src/olap/storage_engine.cpp:394
4# doris::StorageEngine::_disk_stat_monitor_thread_callback() at
/root/work/doris-2.1/be/src/olap/olap_server.cpp:349
5#
doris::StorageEngine::start_bg_threads(std::shared_ptr<doris::WorkloadGroup>)::$_2::operator()()
const at /root/work/doris-2.1/be/src/olap/olap_server.cpp:142
6# void std::__invoke_impl<void,
doris::StorageEngine::start_bg_threads(std::shared_ptr<doris::WorkloadGroup>)::$_2&>(std::__invoke_other,
doris::StorageEngine::start_bg_threads(std::shared_ptr<doris::WorkloadGroup>)::$_2&)
at
/opt/ldb_toolchain/bin/../lib/gcc/x86_64-pc-linux-gnu/15/include/g++-v15/bits/invoke.h:63
7# std::enable_if<is_invocable_r_v<void,
doris::StorageEngine::start_bg_threads(std::shared_ptr<doris::WorkloadGroup>)::$_2&>,
void>::type std::__invoke_r<void,
doris::StorageEngine::start_bg_threads(std::shared_ptr<doris::WorkloadGroup>)::$_2&>(doris::StorageEngine::start_bg_threads(std::shared_ptr<doris::WorkloadGroup>)::$_2&)
at
/opt/ldb_toolchain/bin/../lib/gcc/x86_64-pc-linux-gnu/15/include/g++-v15/bits/invoke.h:119
8# std::_Function_handler<void (),
doris::StorageEngine::start_bg_threads(std::shared_ptr<doris::WorkloadGroup>)::$_2>::_M_invoke(std::_Any_data
const&) at
/opt/ldb_toolchain/bin/../lib/gcc/x86_64-pc-linux-gnu/15/include/g++-v15/bits/std_function.h:292
9# std::function<void ()>::operator()() const at
/opt/ldb_toolchain/bin/../lib/gcc/x86_64-pc-linux-gnu/15/include/g++-v15/bits/std_function.h:593
10# doris::Thread::supervise_thread(void*) at
/root/work/doris-2.1/be/src/util/thread.cpp:498
11# ?
12# ?
@ 0x5ad4937d21b0 google::LogMessage::Flush()
@ 0x5ad4937d5fa9 google::LogMessageFatal::~LogMessageFatal()
@ 0x5ad483f4d0e7 doris::create_tablet_callback()
@ 0x5ad4845c3b10
_ZZN5doris11AgentServer13start_workersEPNS_7ExecEnvEENK4$_10clIRKNS_17TAgentTaskRequestEEEDaOT_
@ 0x5ad4845c3add std::__invoke_impl<>()
@ 0x5ad4845c3a7d
_ZSt10__invoke_rIvRZN5doris11AgentServer13start_workersEPNS0_7ExecEnvEE4$_10JRKNS0_17TAgentTaskRequestEEENSt9enable_ifIX16is_invocable_r_vIT_T0_DpT1_EESA_E4typeEOSB_DpOSC_
@ 0x5ad4845c39a5 std::_Function_handler<>::_M_invoke()
@ 0x5ad483f56bd6 std::function<>::operator()()
@ 0x5ad483f52a64
_ZZZN5doris14TaskWorkerPool11submit_taskERKNS_17TAgentTaskRequestEENK3$_0clIS3_EEDaOT_ENKUlvE_clEv
@ 0x5ad483f52a35
_ZSt13__invoke_implIvRZZN5doris14TaskWorkerPool11submit_taskERKNS0_17TAgentTaskRequestEENK3$_0clIS4_EEDaOT_EUlvE_JEES7_St14__invoke_otherOT0_DpOT1_
@ 0x5ad483f529f5
_ZSt10__invoke_rIvRZZN5doris14TaskWorkerPool11submit_taskERKNS0_17TAgentTaskRequestEENK3$_0clIS4_EEDaOT_EUlvE_JEENSt9enable_ifIX16is_invocable_r_vIS7_T0_DpT1_EES7_E4typeEOSC_DpOSD_
@ 0x5ad483f5287d
_ZNSt17_Function_handlerIFvvEZZN5doris14TaskWorkerPool11submit_taskERKNS1_17TAgentTaskRequestEENK3$_0clIS5_EEDaOT_EUlvE_E9_M_invokeERKSt9_Any_data
@ 0x5ad4830d271e std::function<>::operator()()
@ 0x5ad48476cab9 doris::FunctionRunnable::run()
@ 0x5ad484766544 doris::ThreadPool::dispatch_thread()
@ 0x5ad4847737f2 std::__invoke_impl<>()
@ 0x5ad48477373d std::__invoke<>()
@ 0x5ad48477370d
_ZNSt5_BindIFMN5doris10ThreadPoolEFvvEPS1_EE6__callIvJEJLm0EEEET_OSt5tupleIJDpT0_EESt12_Index_tupleIJXspT1_EEE
@ 0x5ad4847736c6 std::_Bind<>::operator()<>()
@ 0x5ad484773695 std::__invoke_impl<>()
@ 0x5ad484773655
_ZSt10__invoke_rIvRSt5_BindIFMN5doris10ThreadPoolEFvvEPS2_EEJEENSt9enable_ifIX16is_invocable_r_vIT_T0_DpT1_EESA_E4typeEOSB_DpOSC_
@ 0x5ad4847734bd std::_Function_handler<>::_M_invoke()
@ 0x5ad4830d271e std::function<>::operator()()
@ 0x5ad48475741a doris::Thread::supervise_thread()
@ 0x72379249caa4 (unknown)
@ 0x723792529c3c (unknown)
@ (nil) (unknown)
*** Query id: 0-0 ***
*** is nereids: 0 ***
*** tablet id: 0 ***
*** Aborted at 1773652499 (unix time) try "date -d @1773652499" if you are
using GNU date ***
*** Current BE git commitID: d9c5eaa58b6 ***
*** SIGABRT unknown detail explain (@0x14c3c) received by PID 85052 (TID
85806 OR 0x7235d67e46c0) from PID 85052; stack trace: ***
RuntimeLogger I20260316 17:15:00.240028 86983 daemon.cpp:221] os physical
memory 47.04 GB. process memory used 1.79 GB(= 1.89 GB[vm/rss] - 105.01
MB[tc/jemalloc_cache] + 0[reserved] + 0B[waiting_refresh]), limit 42.34 GB,
soft limit 38.10 GB. sys available memory 28.75 GB(= 28.75 GB[proc/available] -
0[reserved] - 0B[waiting_refresh]), low water mark 2.35 GB, warning water mark
4.70 GB.
0# doris::signal::(anonymous namespace)::FailureSignalHandler(int,
siginfo_t*, void*) at /root/work/doris-2.1/be/src/common/signal_handler.h:421
1# 0x0000723792445330 in /lib/x86_64-linux-gnu/libc.so.6
2# pthread_kill at ./nptl/pthread_kill.c:89
3# gsignal at ../sysdeps/posix/raise.c:27
4# abort at ./stdlib/abort.c:81
5# 0x00005AD4937E032D in /root/work/doris-2.1/be/build/src/service/doris_be
6# 0x00005AD4937D267A in /root/work/doris-2.1/be/build/src/service/doris_be
7# google::LogMessage::SendToLog() in
/root/work/doris-2.1/be/build/src/service/doris_be
8# google::LogMessage::Flush() in
/root/work/doris-2.1/be/build/src/service/doris_be
9# google::LogMessageFatal::~LogMessageFatal() in
/root/work/doris-2.1/be/build/src/service/doris_be
10# doris::create_tablet_callback(doris::StorageEngine&,
doris::TAgentTaskRequest const&) in
/root/work/doris-2.1/be/build/src/service/doris_be
11# auto
doris::AgentServer::start_workers(doris::ExecEnv*)::$_10::operator()<doris::TAgentTaskRequest
const&>(doris::TAgentTaskRequest const&) const at
/root/work/doris-2.1/be/src/agent/agent_server.cpp:123
12# void std::__invoke_impl<void,
doris::AgentServer::start_workers(doris::ExecEnv*)::$_10&,
doris::TAgentTaskRequest const&>(std::__invoke_other,
doris::AgentServer::start_workers(doris::ExecEnv*)::$_10&,
doris::TAgentTaskRequest const&) at
/opt/ldb_toolchain/bin/../lib/gcc/x86_64-pc-linux-gnu/15/include/g++-v15/bits/invoke.h:63
13# std::enable_if<is_invocable_r_v<void,
doris::AgentServer::start_workers(doris::ExecEnv*)::$_10&,
doris::TAgentTaskRequest const&>, void>::type std::__invoke_r<void,
doris::AgentServer::start_workers(doris::ExecEnv*)::$_10&,
doris::TAgentTaskRequest
const&>(doris::AgentServer::start_workers(doris::ExecEnv*)::$_10&,
doris::TAgentTaskRequest const&) at
/opt/ldb_toolchain/bin/../lib/gcc/x86_64-pc-linux-gnu/15/include/g++-v15/bits/invoke.h:119
14# std::_Function_handler<void (doris::TAgentTaskRequest const&),
doris::AgentServer::start_workers(doris::ExecEnv*)::$_10>::_M_invoke(std::_Any_data
const&, doris::TAgentTaskRequest const&) at
/opt/ldb_toolchain/bin/../lib/gcc/x86_64-pc-linux-gnu/15/include/g++-v15/bits/std_function.h:292
15# std::function<void (doris::TAgentTaskRequest
const&)>::operator()(doris::TAgentTaskRequest const&) const at
/opt/ldb_toolchain/bin/../lib/gcc/x86_64-pc-linux-gnu/15/include/g++-v15/bits/std_function.h:593
16# doris::TaskWorkerPool::submit_task(doris::TAgentTaskRequest
const&)::$_0::operator()<doris::TAgentTaskRequest
const&>(doris::TAgentTaskRequest const&) const::{lambda()#1}::operator()()
const at /root/work/doris-2.1/be/src/agent/task_worker_pool.cpp:444
17# void std::__invoke_impl<void,
doris::TaskWorkerPool::submit_task(doris::TAgentTaskRequest
const&)::$_0::operator()<doris::TAgentTaskRequest
const&>(doris::TAgentTaskRequest const&)
const::{lambda()#1}&>(std::__invoke_other,
doris::TaskWorkerPool::submit_task(doris::TAgentTaskRequest
const&)::$_0::operator()<doris::TAgentTaskRequest
const&>(doris::TAgentTaskRequest const&) const::{lambda()#1}&) at
/opt/ldb_toolchain/bin/../lib/gcc/x86_64-pc-linux-gnu/15/include/g++-v15/bits/invoke.h:63
18# std::enable_if<is_invocable_r_v<void,
doris::TaskWorkerPool::submit_task(doris::TAgentTaskRequest
const&)::$_0::operator()<doris::TAgentTaskRequest
const&>(doris::TAgentTaskRequest const&) const::{lambda()#1}&>, void>::type
std::__invoke_r<void,
doris::TaskWorkerPool::submit_task(doris::TAgentTaskRequest
const&)::$_0::operator()<doris::TAgentTaskRequest
const&>(doris::TAgentTaskRequest const&)
const::{lambda()#1}&>(doris::TaskWorkerPool::submit_task(doris::TAgentTaskRequest
const&)::$_0::operator()<doris::TAgentTaskRequest
const&>(doris::TAgentTaskRequest const&) const::{lambda()#1}&) at
/opt/ldb_toolchain/bin/../lib/gcc/x86_64-pc-linux-gnu/15/include/g++-v15/bits/invoke.h:119
19# std::_Function_handler<void (),
doris::TaskWorkerPool::submit_task(doris::TAgentTaskRequest
const&)::$_0::operator()<doris::TAgentTaskRequest
const&>(doris::TAgentTaskRequest const&)
const::{lambda()#1}>::_M_invoke(std::_Any_data const&) at
/opt/ldb_toolchain/bin/../lib/gcc/x86_64-pc-linux-gnu/15/include/g++-v15/bits/std_function.h:292
20# std::function<void ()>::operator()() const at
/opt/ldb_toolchain/bin/../lib/gcc/x86_64-pc-linux-gnu/15/include/g++-v15/bits/std_function.h:593
21# doris::FunctionRunnable::run() at
/root/work/doris-2.1/be/src/util/threadpool.cpp:48
22# doris::ThreadPool::dispatch_thread() at
/root/work/doris-2.1/be/src/util/threadpool.cpp:544
23# void std::__invoke_impl<void, void (doris::ThreadPool::*&)(),
doris::ThreadPool*&>(std::__invoke_memfun_deref, void
(doris::ThreadPool::*&)(), doris::ThreadPool*&) at
/opt/ldb_toolchain/bin/../lib/gcc/x86_64-pc-linux-gnu/15/include/g++-v15/bits/invoke.h:76
24# std::__invoke_result<void (doris::ThreadPool::*&)(),
doris::ThreadPool*&>::type std::__invoke<void (doris::ThreadPool::*&)(),
doris::ThreadPool*&>(void (doris::ThreadPool::*&)(), doris::ThreadPool*&) at
/opt/ldb_toolchain/bin/../lib/gcc/x86_64-pc-linux-gnu/15/include/g++-v15/bits/invoke.h:98
25# void std::_Bind<void
(doris::ThreadPool::*(doris::ThreadPool*))()>::__call<void, ,
0ul>(std::tuple<>&&, std::_Index_tuple<0ul>) at
/opt/ldb_toolchain/bin/../lib/gcc/x86_64-pc-linux-gnu/15/include/g++-v15/functional:515
26# void std::_Bind<void
(doris::ThreadPool::*(doris::ThreadPool*))()>::operator()<, void>() at
/opt/ldb_toolchain/bin/../lib/gcc/x86_64-pc-linux-gnu/15/include/g++-v15/functional:600
27# void std::__invoke_impl<void, std::_Bind<void
(doris::ThreadPool::*(doris::ThreadPool*))()>&>(std::__invoke_other,
std::_Bind<void (doris::ThreadPool::*(doris::ThreadPool*))()>&) at
/opt/ldb_toolchain/bin/../lib/gcc/x86_64-pc-linux-gnu/15/include/g++-v15/bits/invoke.h:63
28# std::enable_if<is_invocable_r_v<void, std::_Bind<void
(doris::ThreadPool::*(doris::ThreadPool*))()>&>, void>::type
std::__invoke_r<void, std::_Bind<void
(doris::ThreadPool::*(doris::ThreadPool*))()>&>(std::_Bind<void
(doris::ThreadPool::*(doris::ThreadPool*))()>&) at
/opt/ldb_toolchain/bin/../lib/gcc/x86_64-pc-linux-gnu/15/include/g++-v15/bits/invoke.h:119
29# std::_Function_handler<void (), std::_Bind<void
(doris::ThreadPool::*(doris::ThreadPool*))()> >::_M_invoke(std::_Any_data
const&) at
/opt/ldb_toolchain/bin/../lib/gcc/x86_64-pc-linux-gnu/15/include/g++-v15/bits/std_function.h:292
30# std::function<void ()>::operator()() const at
/opt/ldb_toolchain/bin/../lib/gcc/x86_64-pc-linux-gnu/15/include/g++-v15/bits/std_function.h:593
31# doris::Thread::supervise_thread(void*) at
/root/work/doris-2.1/be/src/util/thread.cpp:498
32# start_thread at ./nptl/pthread_create.c:447
33# clone3 at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:80
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
