This is an automated email from the ASF dual-hosted git repository.
lide pushed a commit to branch branch-1.2-lts
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-1.2-lts by this push:
new c35d2aed467 [fix](thrift)limit be and fe thrift server max pkg
size,avoid accepting error or too large package causing OOM (#26179)
c35d2aed467 is described below
commit c35d2aed4675c71aa22820ff6e65690f5869a4f8
Author: ryanzryu <[email protected]>
AuthorDate: Tue Nov 7 16:41:38 2023 +0800
[fix](thrift)limit be and fe thrift server max pkg size,avoid accepting
error or too large package causing OOM (#26179)
---
be/src/common/config.h | 3 +++
be/src/util/thrift_server.cpp | 5 +++++
docs/zh-CN/docs/admin-manual/config/be-config.md | 6 ++++++
docs/zh-CN/docs/admin-manual/config/fe-config.md | 12 ++++++++++++
.../src/main/java/org/apache/doris/common/Config.java | 6 ++++++
.../src/main/java/org/apache/doris/common/ThriftServer.java | 6 +++---
6 files changed, 35 insertions(+), 3 deletions(-)
diff --git a/be/src/common/config.h b/be/src/common/config.h
index 2cb34fd6b08..660865c36e1 100644
--- a/be/src/common/config.h
+++ b/be/src/common/config.h
@@ -929,6 +929,9 @@ CONF_mBool(allow_invalid_decimalv2_literal, "false");
// data page size for primary key index.
CONF_Int32(primary_key_data_page_size, "32768");
+// the max package size be thrift server can receive,avoid accepting error or
too large package causing OOM,default 20M
+CONF_Int32(be_thrift_max_pkg_bytes, "20000000");
+
#ifdef BE_TEST
// test s3
CONF_String(test_s3_resource, "resource");
diff --git a/be/src/util/thrift_server.cpp b/be/src/util/thrift_server.cpp
index c3dc639eb87..835fc7eaf92 100644
--- a/be/src/util/thrift_server.cpp
+++ b/be/src/util/thrift_server.cpp
@@ -285,6 +285,11 @@ Status ThriftServer::start() {
DCHECK(!_started);
std::shared_ptr<apache::thrift::protocol::TProtocolFactory>
protocol_factory(
new apache::thrift::protocol::TBinaryProtocolFactory());
+ // binary_protocal_factory for setStringSizeLimit function
+ std::shared_ptr<apache::thrift::protocol::TBinaryProtocolFactory>
binary_protocal_factory =
+
std::dynamic_pointer_cast<apache::thrift::protocol::TBinaryProtocolFactory>(
+ protocol_factory);
+
binary_protocal_factory->setStringSizeLimit(config::be_thrift_max_pkg_bytes);
std::shared_ptr<apache::thrift::concurrency::ThreadManager> thread_mgr;
std::shared_ptr<apache::thrift::concurrency::ThreadFactory> thread_factory
=
std::make_shared<apache::thrift::concurrency::ThreadFactory>();
diff --git a/docs/zh-CN/docs/admin-manual/config/be-config.md
b/docs/zh-CN/docs/admin-manual/config/be-config.md
index 8156d2b2a05..fe147ca47c1 100644
--- a/docs/zh-CN/docs/admin-manual/config/be-config.md
+++ b/docs/zh-CN/docs/admin-manual/config/be-config.md
@@ -1446,4 +1446,10 @@ load tablets from header failed, failed tablets size:
xxx, path=xxx
* 描述: 是否在导入json数据时用simdjson来解析。
* 默认值: false
+
+#### `be_thrift_max_pkg_bytes`
+
+* 描述: be节点thrift端口最大接收包大小
+* 默认值: 20000000
+
</version>
diff --git a/docs/zh-CN/docs/admin-manual/config/fe-config.md
b/docs/zh-CN/docs/admin-manual/config/fe-config.md
index e54ff756229..d78ae3804c1 100644
--- a/docs/zh-CN/docs/admin-manual/config/fe-config.md
+++ b/docs/zh-CN/docs/admin-manual/config/fe-config.md
@@ -2734,3 +2734,15 @@ show data （其他用法：HELP SHOW DATA）
这个参数主要用于避免因 external catalog 无法访问、信息过多等原因导致的查询 `information_schema` 超时的问题。
+
+#### `fe_thrift_max_pkg_bytes`
+
+默认值：20000000
+
+是否可以动态配置：false
+
+是否为 Master FE 节点独有的配置项：false
+
+用于限制fe节点thrift端口可以接收的最大包长度，避免接收到过大或者错误的包导致OOM
+
+
diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
index edfd56c522c..2a66a609643 100644
--- a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
+++ b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
@@ -2053,5 +2053,11 @@ public class Config extends ConfigBase {
@ConfField(mutable = true)
public static boolean use_mysql_bigint_for_largeint = false;
+
+ /**
+ * the max package size fe thrift server can receive,avoid accepting error
or too large package causing OOM,default 20M
+ */
+ @ConfField
+ public static int fe_thrift_max_pkg_bytes = 20000000;
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/ThriftServer.java
b/fe/fe-core/src/main/java/org/apache/doris/common/ThriftServer.java
index d101f5d227b..eda2f2e4d6d 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/common/ThriftServer.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/common/ThriftServer.java
@@ -91,14 +91,14 @@ public class ThriftServer {
private void createSimpleServer() throws TTransportException {
TServer.Args args = new TServer.Args(new
TServerSocket(port)).protocolFactory(
- new TBinaryProtocol.Factory()).processor(processor);
+ new TBinaryProtocol.Factory(Config.fe_thrift_max_pkg_bytes,
-1)).processor(processor);
server = new TSimpleServer(args);
}
private void createThreadedServer() throws TTransportException {
TThreadedSelectorServer.Args args = new TThreadedSelectorServer.Args(
new TNonblockingServerSocket(port,
Config.thrift_client_timeout_ms)).protocolFactory(
- new TBinaryProtocol.Factory()).processor(processor);
+ new
TBinaryProtocol.Factory(Config.fe_thrift_max_pkg_bytes,
-1)).processor(processor);
ThreadPoolExecutor threadPoolExecutor =
ThreadPoolManager.newDaemonCacheThreadPool(
Config.thrift_server_max_worker_threads, "thrift-server-pool",
true);
args.executorService(threadPoolExecutor);
@@ -114,7 +114,7 @@ public class ThriftServer {
TThreadPoolServer.Args serverArgs =
new TThreadPoolServer.Args(new
TServerSocket(socketTransportArgs)).protocolFactory(
- new TBinaryProtocol.Factory()).processor(processor);
+ new
TBinaryProtocol.Factory(Config.fe_thrift_max_pkg_bytes,
-1)).processor(processor);
ThreadPoolExecutor threadPoolExecutor =
ThreadPoolManager.newDaemonCacheThreadPool(
Config.thrift_server_max_worker_threads, "thrift-server-pool",
true);
serverArgs.executorService(threadPoolExecutor);
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
