This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-doris.git
The following commit(s) were added to refs/heads/master by this push:
new 5a27981 [Config] Add thrift_client_retry_interval_ms config in be for
thrift client to avoid avalanche disaster in fe thrift server (#4022)
5a27981 is described below
commit 5a27981e491705f42c703d89abc0707b8cb6c5ea
Author: caiconghui <[email protected]>
AuthorDate: Wed Jul 8 21:07:00 2020 +0800
[Config] Add thrift_client_retry_interval_ms config in be for thrift client
to avoid avalanche disaster in fe thrift server (#4022)
This PR is mainly to add `thrift_client_retry_interval_ms` config in be
for thrift client
to avoid avalanche disaster in fe thrift server and fix some typo and some
rpc
setting problems at the same time.
---
be/src/common/config.h | 2 ++
be/src/runtime/stream_load/stream_load_executor.cpp | 2 +-
be/src/util/thrift_rpc_helper.cpp | 11 +++++++----
docs/en/administrator-guide/config/be_config.md | 6 ++++++
docs/zh-CN/administrator-guide/config/be_config.md | 6 ++++++
fe/src/main/java/org/apache/doris/common/Config.java | 2 +-
fe/src/main/java/org/apache/doris/common/ThriftServer.java | 2 +-
7 files changed, 24 insertions(+), 7 deletions(-)
diff --git a/be/src/common/config.h b/be/src/common/config.h
index 92bd75f..3adf389 100644
--- a/be/src/common/config.h
+++ b/be/src/common/config.h
@@ -184,6 +184,8 @@ namespace config {
CONF_Int32(port, "20001");
// default thrift client connect timeout(in seconds)
CONF_Int32(thrift_connect_timeout_seconds, "3");
+ // default thrift client retry interval (in milliseconds)
+ CONF_mInt64(thrift_client_retry_interval_ms, "1000");
// max row count number for single scan range
CONF_mInt32(doris_scan_range_row_count, "524288");
// size of scanner queue between scanner thread and compute thread
diff --git a/be/src/runtime/stream_load/stream_load_executor.cpp
b/be/src/runtime/stream_load/stream_load_executor.cpp
index 5782243..1e5d4a1 100644
--- a/be/src/runtime/stream_load/stream_load_executor.cpp
+++ b/be/src/runtime/stream_load/stream_load_executor.cpp
@@ -172,7 +172,7 @@ Status StreamLoadExecutor::commit_txn(StreamLoadContext*
ctx) {
request.sync = true;
request.commitInfos = std::move(ctx->commit_infos);
request.__isset.commitInfos = true;
- request.__set_thrift_rpc_timeout_ms(config::thrift_rpc_timeout_ms);
+ request.__set_thrift_rpc_timeout_ms(config::txn_commit_rpc_timeout_ms);
// set attachment if has
TTxnCommitAttachment attachment;
diff --git a/be/src/util/thrift_rpc_helper.cpp
b/be/src/util/thrift_rpc_helper.cpp
index d47d0a4..6bd750b 100644
--- a/be/src/util/thrift_rpc_helper.cpp
+++ b/be/src/util/thrift_rpc_helper.cpp
@@ -34,6 +34,7 @@
#include "util/thrift_util.h"
#include "util/runtime_profile.h"
#include "runtime/client_cache.h"
+#include "monotime.h"
namespace doris {
@@ -68,8 +69,9 @@ Status ThriftRpcHelper::rpc(
try {
callback(client);
} catch (apache::thrift::transport::TTransportException& e) {
- LOG(WARNING) << "retrying call frontend service, address="
- << address << ", reason=" << e.what();
+ LOG(WARNING) << "retrying call frontend service after " <<
config::thrift_client_retry_interval_ms
+ << " ms, address=" << address << ", reason=" << e.what();
+
SleepFor(MonoDelta::FromMilliseconds(config::thrift_client_retry_interval_ms));
status = client.reopen(timeout_ms);
if (!status.ok()) {
LOG(WARNING) << "client repoen failed. address=" << address
@@ -79,10 +81,11 @@ Status ThriftRpcHelper::rpc(
callback(client);
}
} catch (apache::thrift::TException& e) {
+ LOG(WARNING) << "call frontend service failed, address=" << address
+ << ", reason=" << e.what();
+
SleepFor(MonoDelta::FromMilliseconds(config::thrift_client_retry_interval_ms *
2));
// just reopen to disable this connection
client.reopen(timeout_ms);
- LOG(WARNING) << "call frontend service failed, address=" << address
- << ", reason=" << e.what();
return Status::ThriftRpcError("failed to call frontend service");
}
return Status::OK();
diff --git a/docs/en/administrator-guide/config/be_config.md
b/docs/en/administrator-guide/config/be_config.md
index ae4cf45..0486efe 100644
--- a/docs/en/administrator-guide/config/be_config.md
+++ b/docs/en/administrator-guide/config/be_config.md
@@ -468,6 +468,12 @@ If the system is found to be in a high-stress scenario and
a large number of thr
### `tc_use_memory_min`
+### `thrift_client_retry_interval_ms`
+
+* Type: int64
+* Description: Used to set retry interval for thrift client in be to avoid
avalanche disaster in fe thrift server, the unit is ms.
+* Default: 1000
+
### `thrift_connect_timeout_seconds`
### `thrift_rpc_timeout_ms`
diff --git a/docs/zh-CN/administrator-guide/config/be_config.md
b/docs/zh-CN/administrator-guide/config/be_config.md
index de57375..fd75d88 100644
--- a/docs/zh-CN/administrator-guide/config/be_config.md
+++ b/docs/zh-CN/administrator-guide/config/be_config.md
@@ -415,6 +415,12 @@ under the License.
### `tc_use_memory_min`
+### `thrift_client_retry_interval_ms`
+
+* 类型：int64
+* 描述：用来为be的thrift客户端设置重试间隔, 避免fe的thrift server发生雪崩问题，单位为ms。
+* 默认值：1000
+
### `thrift_connect_timeout_seconds`
### `thrift_rpc_timeout_ms`
diff --git a/fe/src/main/java/org/apache/doris/common/Config.java
b/fe/src/main/java/org/apache/doris/common/Config.java
index 0f13d55..44dfe12 100644
--- a/fe/src/main/java/org/apache/doris/common/Config.java
+++ b/fe/src/main/java/org/apache/doris/common/Config.java
@@ -864,7 +864,7 @@ public class Config extends ConfigBase {
/**
* The number of query retries.
* A query may retry if we encounter RPC exception and no result has been
sent to user.
- * You may reduce this number to void Avalanche disaster.
+ * You may reduce this number to avoid Avalanche disaster.
*/
@ConfField(mutable = true)
public static int max_query_retry_time = 2;
diff --git a/fe/src/main/java/org/apache/doris/common/ThriftServer.java
b/fe/src/main/java/org/apache/doris/common/ThriftServer.java
index 3c293dd..9caab7b 100644
--- a/fe/src/main/java/org/apache/doris/common/ThriftServer.java
+++ b/fe/src/main/java/org/apache/doris/common/ThriftServer.java
@@ -98,7 +98,7 @@ public class ThriftServer {
private void createThreadedServer() throws TTransportException {
TThreadedSelectorServer.Args args =
- new TThreadedSelectorServer.Args(new
TNonblockingServerSocket(port)).protocolFactory(
+ new TThreadedSelectorServer.Args(new TNonblockingServerSocket(port,
Config.thrift_client_timeout_ms)).protocolFactory(
new TBinaryProtocol.Factory()).processor(processor);
ThreadPoolExecutor threadPoolExecutor =
ThreadPoolManager.newDaemonCacheThreadPool(Config.thrift_server_max_worker_threads,
"thrift-server-pool");
args.executorService(threadPoolExecutor);
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
