Repository: incubator-impala Updated Branches: refs/heads/master 3bdde74a7 -> dae5eb8d8
IMPALA-6030: Don't start coordinator specific thread pools if a node isn't a coordinator node Since we introduced the FLAGS_is_coordinator, we've forgotten to disable the coordinator specific thread pools on nodes that have only the executor role. This patch fixes that. Change-Id: I1cd046dbe5f10fa51fcc37338e48b94843891b62 Reviewed-on: http://gerrit.cloudera.org:8080/8242 Reviewed-by: Dan Hecht <[email protected]> Tested-by: Impala Public Jenkins Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/0290e92d Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/0290e92d Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/0290e92d Branch: refs/heads/master Commit: 0290e92da866ac92b4b2fbfa6054f1fa9ad13995 Parents: 3bdde74 Author: Sailesh Mukil <[email protected]> Authored: Mon Oct 9 16:41:54 2017 -0700 Committer: Impala Public Jenkins <[email protected]> Committed: Wed Oct 11 22:32:50 2017 +0000 ---------------------------------------------------------------------- be/src/runtime/exec-env.cc | 14 ++++++++------ be/src/runtime/exec-env.h | 8 ++++++++ 2 files changed, 16 insertions(+), 6 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/0290e92d/be/src/runtime/exec-env.cc ---------------------------------------------------------------------- diff --git a/be/src/runtime/exec-env.cc b/be/src/runtime/exec-env.cc index 94d2ca6..20e921b 100644 --- a/be/src/runtime/exec-env.cc +++ b/be/src/runtime/exec-env.cc @@ -160,12 +160,8 @@ ExecEnv::ExecEnv(const string& hostname, int backend_port, int krpc_port, webserver_(new Webserver(webserver_port)), pool_mem_trackers_(new PoolMemTrackerRegistry), thread_mgr_(new ThreadResourceMgr), - hdfs_op_thread_pool_( - CreateHdfsOpThreadPool("hdfs-worker-pool", FLAGS_num_hdfs_worker_threads, 1024)), tmp_file_mgr_(new TmpFileMgr), frontend_(new Frontend()), - exec_rpc_thread_pool_(new CallableThreadPool("exec-rpc-pool", "worker", - FLAGS_coordinator_rpc_threads, numeric_limits<int32_t>::max())), async_rpc_pool_(new CallableThreadPool("rpc-pool", "async-rpc-sender", 8, 10000)), query_exec_mgr_(new QueryExecMgr()), enable_webserver_(FLAGS_enable_webserver && webserver_port > 0), @@ -191,6 +187,10 @@ ExecEnv::ExecEnv(const string& hostname, int backend_port, int krpc_port, subscriber_address, statestore_address, metrics_.get())); if (FLAGS_is_coordinator) { + hdfs_op_thread_pool_.reset( + CreateHdfsOpThreadPool("hdfs-worker-pool", FLAGS_num_hdfs_worker_threads, 1024)); + exec_rpc_thread_pool_.reset(new CallableThreadPool("exec-rpc-pool", "worker", + FLAGS_coordinator_rpc_threads, numeric_limits<int32_t>::max())); scheduler_.reset(new Scheduler(statestore_subscriber_.get(), statestore_subscriber_->id(), metrics_.get(), webserver_.get(), request_pool_service_.get())); @@ -219,9 +219,11 @@ Status ExecEnv::InitForFeTests() { Status ExecEnv::Init() { // Initialize thread pools - RETURN_IF_ERROR(exec_rpc_thread_pool_->Init()); + if (FLAGS_is_coordinator) { + RETURN_IF_ERROR(exec_rpc_thread_pool_->Init()); + RETURN_IF_ERROR(hdfs_op_thread_pool_->Init()); + } RETURN_IF_ERROR(async_rpc_pool_->Init()); - RETURN_IF_ERROR(hdfs_op_thread_pool_->Init()); // Initialize global memory limit. // Depending on the system configuration, we will have to calculate the process http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/0290e92d/be/src/runtime/exec-env.h ---------------------------------------------------------------------- diff --git a/be/src/runtime/exec-env.h b/be/src/runtime/exec-env.h index 31532df..416f855 100644 --- a/be/src/runtime/exec-env.h +++ b/be/src/runtime/exec-env.h @@ -174,11 +174,19 @@ class ExecEnv { boost::scoped_ptr<MemTracker> mem_tracker_; boost::scoped_ptr<PoolMemTrackerRegistry> pool_mem_trackers_; boost::scoped_ptr<ThreadResourceMgr> thread_mgr_; + + // Thread pool for running HdfsOp operations. Only used by the coordinator, so it's + // only started if FLAGS_is_coordinator is 'true'. boost::scoped_ptr<HdfsOpThreadPool> hdfs_op_thread_pool_; + boost::scoped_ptr<TmpFileMgr> tmp_file_mgr_; boost::scoped_ptr<RequestPoolService> request_pool_service_; boost::scoped_ptr<Frontend> frontend_; + + // Thread pool for the ExecQueryFInstances RPC. Only used by the coordinator, so it's + // only started if FLAGS_is_coordinator is 'true'. boost::scoped_ptr<CallableThreadPool> exec_rpc_thread_pool_; + boost::scoped_ptr<CallableThreadPool> async_rpc_pool_; boost::scoped_ptr<QueryExecMgr> query_exec_mgr_; boost::scoped_ptr<RpcMgr> rpc_mgr_;
