[
https://issues.apache.org/jira/browse/KUDU-3397?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
shenxingwuying updated KUDU-3397:
---------------------------------
Description:
I have added a SchedulerThread function, as Jira
https://issues.apache.org/jira/browse/KUDU-3364 said,
some periodic tasks can be refactor using SchedulerThread.
There are 2 benefits:
# Use a thread pool(very few threads 1 or 2 is ok) with SchedulerThread to
replace all the independent, isolated Threads. less threads.
# By the refactor, some tasks can support manual trigger, more maintainable.
## auto-rebalancer
## leader-auto-rebalancer(doing at: )
## compaction jobs
The latent Thread can be refactor may exist below.
They should be analysed more details.
{code:java}
//代码占位符
ag Thread::Create src/kudu/ | grep -v test | grep -v src/kudu/util/threadpool.cc
src/kudu/clock/builtin_ntp.cc:651: Thread::Create("ntp", "ntp client",
[this]() { this->PollThread(); }, &thread_),
src/kudu/hms/hms_catalog.cc:132: RETURN_NOT_OK(Thread::Create("hms_catalog",
"fetch_uuid",
src/kudu/master/hms_notification_log_listener.cc:98: return
kudu::Thread::Create("catalog manager", "hms-notification-log-listener",
src/kudu/master/master.cc:452: return Thread::Create("master",
src/kudu/master/catalog_manager.cc:787:
RETURN_NOT_OK(kudu::Thread::Create("catalog manager", "bgtasks",
src/kudu/rpc/acceptor_pool.cc:96: Status s = kudu::Thread::Create("acceptor
pool", "acceptor",
src/kudu/rpc/reactor.cc:193: return kudu::Thread::Create("reactor", "rpc
reactor",
src/kudu/rpc/reactor.cc:660:Status ReactorThread::CreateClientSocket(int
family, Socket* sock) {
src/kudu/rpc/result_tracker.cc:462: CHECK_OK(Thread::Create("server",
"result-tracker",
src/kudu/rpc/service_pool.cc:93: CHECK_OK(kudu::Thread::Create(
src/kudu/security/init.cc:318: RETURN_NOT_OK(Thread::Create("kerberos",
"reacquire thread",
src/kudu/server/diagnostics_log.cc:144: Status s = Thread::Create("server",
"diag-logger",
src/kudu/server/server_base.cc:837: return Thread::Create("server",
"excess-log-deleter",
src/kudu/server/server_base.cc:885: return Thread::Create("server",
"tcmalloc-memory-gc",
src/kudu/subprocess/server.cc:130: RETURN_NOT_OK(Thread::Create("subprocess",
"start",
src/kudu/subprocess/server.cc:153:
RETURN_NOT_OK(Thread::Create("subprocess", "responder",
src/kudu/subprocess/server.cc:157: RETURN_NOT_OK(Thread::Create("subprocess",
"reader",
src/kudu/subprocess/server.cc:160: RETURN_NOT_OK(Thread::Create("subprocess",
"writer",
src/kudu/subprocess/server.cc:163: return Thread::Create("subprocess",
"deadline-checker",
src/kudu/tools/tool_action_local_replica.cc:265:
RETURN_NOT_OK(Thread::Create("tool-tablet-copy", "check-progress",
src/kudu/tserver/heartbeater.cc:669: return
kudu::Thread::Create("heartbeater", "heartbeat",
src/kudu/tserver/scanners.cc:118: RETURN_NOT_OK(Thread::Create("scanners",
"removal_thread",
src/kudu/tserver/tablet_copy_service.cc:114:
CHECK_OK(Thread::Create("tablet-copy", "tc-session-exp",
src/kudu/util/cloud/instance_detector.cc:76: RETURN_NOT_OK(Thread::Create(
src/kudu/util/debug/trace_event_impl.cc:1364: Status s =
Thread::CreateWithFlags(
src/kudu/util/file_cache.cc:504: return Thread::Create("cache",
Substitute("$0-evict", cache_name_),
src/kudu/util/kernel_stack_watchdog.cc:71: CHECK_OK(Thread::CreateWithFlags(
src/kudu/util/maintenance_manager.cc:209: return Thread::Create("maintenance",
"maintenance_scheduler",
src/kudu/util/minidump.cc:288: return Thread::Create("minidump",
"sigusr1-handler",
src/kudu/util/pstack_watcher.cc:53: CHECK_OK(Thread::Create("pstack_watcher",
"pstack_watcher",
src/kudu/util/thread.h:118: // Flags passed to Thread::CreateWithFlags().
src/kudu/util/ttl_cache.h:139: CHECK_OK(Thread::Create( {code}
The direct purpose, the threadpool can unified some seperator threads, eg:
master's 'auto-rebalancer' (the Thread), 'expired-reserved-tables-deleter',
'hms-notification-log-listener' ("catalog manager", "bgtasks"), and later's
leader-rebalancer's Thread, and some other threads. As tserver has some others
thread
The first one : [https://gerrit.cloudera.org/c/18679/]
was:
I have added a SchedulerThread function, as Jira
https://issues.apache.org/jira/browse/KUDU-3364 said,
some periodic tasks can be refactor using SchedulerThread.
There are 2 benefits:
# Use a thread pool(very few threads 1 or 2 is ok) with SchedulerThread to
replace all the independent, isolated Threads. less threads.
# By the refactor, some tasks can support manual trigger, more maintainable.
## auto-rebalancer
## leader-auto-rebalancer(doing at: )
The latent Thread can be refactor may exist below.
They should be analysed more details.
{code:java}
//代码占位符
ag Thread::Create src/kudu/ | grep -v test | grep -v src/kudu/util/threadpool.cc
src/kudu/clock/builtin_ntp.cc:651: Thread::Create("ntp", "ntp client",
[this]() { this->PollThread(); }, &thread_),
src/kudu/hms/hms_catalog.cc:132: RETURN_NOT_OK(Thread::Create("hms_catalog",
"fetch_uuid",
src/kudu/master/hms_notification_log_listener.cc:98: return
kudu::Thread::Create("catalog manager", "hms-notification-log-listener",
src/kudu/master/master.cc:452: return Thread::Create("master",
src/kudu/master/catalog_manager.cc:787:
RETURN_NOT_OK(kudu::Thread::Create("catalog manager", "bgtasks",
src/kudu/rpc/acceptor_pool.cc:96: Status s = kudu::Thread::Create("acceptor
pool", "acceptor",
src/kudu/rpc/reactor.cc:193: return kudu::Thread::Create("reactor", "rpc
reactor",
src/kudu/rpc/reactor.cc:660:Status ReactorThread::CreateClientSocket(int
family, Socket* sock) {
src/kudu/rpc/result_tracker.cc:462: CHECK_OK(Thread::Create("server",
"result-tracker",
src/kudu/rpc/service_pool.cc:93: CHECK_OK(kudu::Thread::Create(
src/kudu/security/init.cc:318: RETURN_NOT_OK(Thread::Create("kerberos",
"reacquire thread",
src/kudu/server/diagnostics_log.cc:144: Status s = Thread::Create("server",
"diag-logger",
src/kudu/server/server_base.cc:837: return Thread::Create("server",
"excess-log-deleter",
src/kudu/server/server_base.cc:885: return Thread::Create("server",
"tcmalloc-memory-gc",
src/kudu/subprocess/server.cc:130: RETURN_NOT_OK(Thread::Create("subprocess",
"start",
src/kudu/subprocess/server.cc:153:
RETURN_NOT_OK(Thread::Create("subprocess", "responder",
src/kudu/subprocess/server.cc:157: RETURN_NOT_OK(Thread::Create("subprocess",
"reader",
src/kudu/subprocess/server.cc:160: RETURN_NOT_OK(Thread::Create("subprocess",
"writer",
src/kudu/subprocess/server.cc:163: return Thread::Create("subprocess",
"deadline-checker",
src/kudu/tools/tool_action_local_replica.cc:265:
RETURN_NOT_OK(Thread::Create("tool-tablet-copy", "check-progress",
src/kudu/tserver/heartbeater.cc:669: return
kudu::Thread::Create("heartbeater", "heartbeat",
src/kudu/tserver/scanners.cc:118: RETURN_NOT_OK(Thread::Create("scanners",
"removal_thread",
src/kudu/tserver/tablet_copy_service.cc:114:
CHECK_OK(Thread::Create("tablet-copy", "tc-session-exp",
src/kudu/util/cloud/instance_detector.cc:76: RETURN_NOT_OK(Thread::Create(
src/kudu/util/debug/trace_event_impl.cc:1364: Status s =
Thread::CreateWithFlags(
src/kudu/util/file_cache.cc:504: return Thread::Create("cache",
Substitute("$0-evict", cache_name_),
src/kudu/util/kernel_stack_watchdog.cc:71: CHECK_OK(Thread::CreateWithFlags(
src/kudu/util/maintenance_manager.cc:209: return Thread::Create("maintenance",
"maintenance_scheduler",
src/kudu/util/minidump.cc:288: return Thread::Create("minidump",
"sigusr1-handler",
src/kudu/util/pstack_watcher.cc:53: CHECK_OK(Thread::Create("pstack_watcher",
"pstack_watcher",
src/kudu/util/thread.h:118: // Flags passed to Thread::CreateWithFlags().
src/kudu/util/ttl_cache.h:139: CHECK_OK(Thread::Create( {code}
The direct purpose, the threadpool can unified some seperator threads, eg:
master's 'auto-rebalancer' (the Thread), 'expired-reserved-tables-deleter',
'hms-notification-log-listener' ("catalog manager", "bgtasks"), and later's
leader-rebalancer's Thread, and some other threads. As tserver has some others
thread
The first one : [https://gerrit.cloudera.org/c/18679/]
> do refactor for some periodic tasks using SchedulerThread
> ---------------------------------------------------------
>
> Key: KUDU-3397
> URL: https://issues.apache.org/jira/browse/KUDU-3397
> Project: Kudu
> Issue Type: Improvement
> Components: master, tserver
> Reporter: shenxingwuying
> Priority: Minor
>
> I have added a SchedulerThread function, as Jira
> https://issues.apache.org/jira/browse/KUDU-3364 said,
> some periodic tasks can be refactor using SchedulerThread.
>
> There are 2 benefits:
> # Use a thread pool(very few threads 1 or 2 is ok) with SchedulerThread to
> replace all the independent, isolated Threads. less threads.
> # By the refactor, some tasks can support manual trigger, more maintainable.
> ## auto-rebalancer
> ## leader-auto-rebalancer(doing at: )
> ## compaction jobs
>
> The latent Thread can be refactor may exist below.
> They should be analysed more details.
> {code:java}
> //代码占位符
> ag Thread::Create src/kudu/ | grep -v test | grep -v
> src/kudu/util/threadpool.cc
> src/kudu/clock/builtin_ntp.cc:651: Thread::Create("ntp", "ntp client",
> [this]() { this->PollThread(); }, &thread_),
> src/kudu/hms/hms_catalog.cc:132: RETURN_NOT_OK(Thread::Create("hms_catalog",
> "fetch_uuid",
> src/kudu/master/hms_notification_log_listener.cc:98: return
> kudu::Thread::Create("catalog manager", "hms-notification-log-listener",
> src/kudu/master/master.cc:452: return Thread::Create("master",
> src/kudu/master/catalog_manager.cc:787:
> RETURN_NOT_OK(kudu::Thread::Create("catalog manager", "bgtasks",
> src/kudu/rpc/acceptor_pool.cc:96: Status s =
> kudu::Thread::Create("acceptor pool", "acceptor",
> src/kudu/rpc/reactor.cc:193: return kudu::Thread::Create("reactor", "rpc
> reactor",
> src/kudu/rpc/reactor.cc:660:Status ReactorThread::CreateClientSocket(int
> family, Socket* sock) {
> src/kudu/rpc/result_tracker.cc:462: CHECK_OK(Thread::Create("server",
> "result-tracker",
> src/kudu/rpc/service_pool.cc:93: CHECK_OK(kudu::Thread::Create(
> src/kudu/security/init.cc:318: RETURN_NOT_OK(Thread::Create("kerberos",
> "reacquire thread",
> src/kudu/server/diagnostics_log.cc:144: Status s = Thread::Create("server",
> "diag-logger",
> src/kudu/server/server_base.cc:837: return Thread::Create("server",
> "excess-log-deleter",
> src/kudu/server/server_base.cc:885: return Thread::Create("server",
> "tcmalloc-memory-gc",
> src/kudu/subprocess/server.cc:130:
> RETURN_NOT_OK(Thread::Create("subprocess", "start",
> src/kudu/subprocess/server.cc:153:
> RETURN_NOT_OK(Thread::Create("subprocess", "responder",
> src/kudu/subprocess/server.cc:157:
> RETURN_NOT_OK(Thread::Create("subprocess", "reader",
> src/kudu/subprocess/server.cc:160:
> RETURN_NOT_OK(Thread::Create("subprocess", "writer",
> src/kudu/subprocess/server.cc:163: return Thread::Create("subprocess",
> "deadline-checker",
> src/kudu/tools/tool_action_local_replica.cc:265:
> RETURN_NOT_OK(Thread::Create("tool-tablet-copy", "check-progress",
> src/kudu/tserver/heartbeater.cc:669: return
> kudu::Thread::Create("heartbeater", "heartbeat",
> src/kudu/tserver/scanners.cc:118: RETURN_NOT_OK(Thread::Create("scanners",
> "removal_thread",
> src/kudu/tserver/tablet_copy_service.cc:114:
> CHECK_OK(Thread::Create("tablet-copy", "tc-session-exp",
> src/kudu/util/cloud/instance_detector.cc:76: RETURN_NOT_OK(Thread::Create(
> src/kudu/util/debug/trace_event_impl.cc:1364: Status s =
> Thread::CreateWithFlags(
> src/kudu/util/file_cache.cc:504: return Thread::Create("cache",
> Substitute("$0-evict", cache_name_),
> src/kudu/util/kernel_stack_watchdog.cc:71: CHECK_OK(Thread::CreateWithFlags(
> src/kudu/util/maintenance_manager.cc:209: return
> Thread::Create("maintenance", "maintenance_scheduler",
> src/kudu/util/minidump.cc:288: return Thread::Create("minidump",
> "sigusr1-handler",
> src/kudu/util/pstack_watcher.cc:53:
> CHECK_OK(Thread::Create("pstack_watcher", "pstack_watcher",
> src/kudu/util/thread.h:118: // Flags passed to Thread::CreateWithFlags().
> src/kudu/util/ttl_cache.h:139: CHECK_OK(Thread::Create( {code}
>
> The direct purpose, the threadpool can unified some seperator threads, eg:
> master's 'auto-rebalancer' (the Thread), 'expired-reserved-tables-deleter',
> 'hms-notification-log-listener' ("catalog manager", "bgtasks"), and later's
> leader-rebalancer's Thread, and some other threads. As tserver has some
> others thread
>
>
> The first one : [https://gerrit.cloudera.org/c/18679/]
>
--
This message was sent by Atlassian Jira
(v8.20.10#820010)
