This is an automated email from the ASF dual-hosted git repository.
kxiao pushed a commit to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.0 by this push:
new 92581d3d15a [fix](thrift)limit be and fe thrift server max pkg
size,avoid accepting error or too large package causing OOM #26179 (#26805)
92581d3d15a is described below
commit 92581d3d15ae6224675aa1d5ff082c26b8260e99
Author: ryanzryu <[email protected]>
AuthorDate: Sat Nov 11 18:15:14 2023 +0800
[fix](thrift)limit be and fe thrift server max pkg size,avoid accepting
error or too large package causing OOM #26179 (#26805)
---
be/src/common/config.cpp | 4 ++++
be/src/common/config.h | 4 ++++
be/src/util/thrift_server.cpp | 5 +++++
docs/zh-CN/docs/admin-manual/config/be-config.md | 6 ++++++
docs/zh-CN/docs/admin-manual/config/fe-config.md | 10 ++++++++++
fe/fe-common/src/main/java/org/apache/doris/common/Config.java | 8 ++++++++
.../src/main/java/org/apache/doris/common/ThriftServer.java | 6 +++---
7 files changed, 40 insertions(+), 3 deletions(-)
diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp
index d3bcbb14be6..bd3e5f046ac 100644
--- a/be/src/common/config.cpp
+++ b/be/src/common/config.cpp
@@ -1091,6 +1091,10 @@ DEFINE_Bool(ignore_always_true_predicate_for_segment,
"true");
// Dir of default timezone files
DEFINE_String(default_tzfiles_path, "${DORIS_HOME}/zoneinfo");
+// the max package bytes be thrift server can receive
+// avoid accepting error or too large package causing OOM,default 20000000(20M)
+DEFINE_Int32(be_thrift_max_pkg_bytes, "20000000");
+
// clang-format off
#ifdef BE_TEST
// test s3
diff --git a/be/src/common/config.h b/be/src/common/config.h
index 9e168f638e7..b200ae2d6cf 100644
--- a/be/src/common/config.h
+++ b/be/src/common/config.h
@@ -1148,6 +1148,10 @@ DECLARE_Bool(ignore_always_true_predicate_for_segment);
// Dir of default timezone files
DECLARE_String(default_tzfiles_path);
+// the max package bytes be thrift server can receive
+// avoid accepting error or too large package causing OOM,default 20000000(20M)
+DECLARE_Int32(be_thrift_max_pkg_bytes);
+
#ifdef BE_TEST
// test s3
DECLARE_String(test_s3_resource);
diff --git a/be/src/util/thrift_server.cpp b/be/src/util/thrift_server.cpp
index 88e71d16c76..e4367db7c81 100644
--- a/be/src/util/thrift_server.cpp
+++ b/be/src/util/thrift_server.cpp
@@ -303,6 +303,11 @@ Status ThriftServer::start() {
DCHECK(!_started);
std::shared_ptr<apache::thrift::protocol::TProtocolFactory>
protocol_factory(
new apache::thrift::protocol::TBinaryProtocolFactory());
+ // add binary_protocal_factory to call TBinaryProtocolFactory's member
function:setStringSizeLimit
+ std::shared_ptr<apache::thrift::protocol::TBinaryProtocolFactory>
binary_protocal_factory =
+
std::dynamic_pointer_cast<apache::thrift::protocol::TBinaryProtocolFactory>(
+ protocol_factory);
+
binary_protocal_factory->setStringSizeLimit(config::be_thrift_max_pkg_bytes);
std::shared_ptr<apache::thrift::concurrency::ThreadManager> thread_mgr;
std::shared_ptr<apache::thrift::concurrency::ThreadFactory> thread_factory
=
std::make_shared<apache::thrift::concurrency::ThreadFactory>();
diff --git a/docs/zh-CN/docs/admin-manual/config/be-config.md
b/docs/zh-CN/docs/admin-manual/config/be-config.md
index 533b7db5f48..7e1259c7f48 100644
--- a/docs/zh-CN/docs/admin-manual/config/be-config.md
+++ b/docs/zh-CN/docs/admin-manual/config/be-config.md
@@ -1529,3 +1529,9 @@ load tablets from header failed, failed tablets size:
xxx, path=xxx
* Description: BE 是否开启使用java-jni，开启后允许 c++ 与 java
之间的相互调用。目前已经支持hudi、java-udf、jdbc、max-compute、paimon、preload、avro
* Default value: true
+
+
+#### `be_thrift_max_pkg_bytes`
+
+* 描述: be节点thrift端口最大接收包大小(字节)
+* 默认值: 20000000
diff --git a/docs/zh-CN/docs/admin-manual/config/fe-config.md
b/docs/zh-CN/docs/admin-manual/config/fe-config.md
index a69c13b2521..cee124b9980 100644
--- a/docs/zh-CN/docs/admin-manual/config/fe-config.md
+++ b/docs/zh-CN/docs/admin-manual/config/fe-config.md
@@ -2731,3 +2731,13 @@ Doris 为了兼用 mysql 周边工具生态，会内置一个名为 mysql 的数
默认值：2000
对于自动分区表，防止用户意外创建大量分区，每个OLAP表允许的分区数量为`max_auto_partition_num`。默认2000。
+
+#### `fe_thrift_max_pkg_bytes`
+
+默认值：20000000
+
+是否可以动态配置：false
+
+是否为 Master FE 节点独有的配置项：false
+
+用于限制fe节点thrift端口可以接收的最大包长度，避免接收到过大或者错误的包导致OOM
diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
index 053ab159521..35b8ab838df 100644
--- a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
+++ b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
@@ -2237,4 +2237,12 @@ public class Config extends ConfigBase {
})
public static int sync_image_timeout_second = 300;
+ @ConfField(mutable = true, masterOnly = true)
+ public static int publish_topic_info_interval_ms = 30000; // 30s
+
+ @ConfField(description = {
+ "限制fe节点thrift server可以接收的最大包大小,默认20M,设置为-1表示不限制",
+ "the max package size fe thrift server can receive,avoid accepting
error"
+ + "or too large package causing OOM,default 20000000(20M),set -1
for unlimited. "})
+ public static int fe_thrift_max_pkg_bytes = 20000000;
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/ThriftServer.java
b/fe/fe-core/src/main/java/org/apache/doris/common/ThriftServer.java
index 2396dc95074..45c4bcb64f0 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/common/ThriftServer.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/common/ThriftServer.java
@@ -92,14 +92,14 @@ public class ThriftServer {
private void createSimpleServer() throws TTransportException {
TServer.Args args = new TServer.Args(new
TServerSocket(port)).protocolFactory(
- new TBinaryProtocol.Factory()).processor(processor);
+ new TBinaryProtocol.Factory(Config.fe_thrift_max_pkg_bytes,
-1)).processor(processor);
server = new TSimpleServer(args);
}
private void createThreadedServer() throws TTransportException {
TThreadedSelectorServer.Args args = new TThreadedSelectorServer.Args(
new TNonblockingServerSocket(port,
Config.thrift_client_timeout_ms)).protocolFactory(
- new TBinaryProtocol.Factory()).processor(processor);
+ new
TBinaryProtocol.Factory(Config.fe_thrift_max_pkg_bytes,
-1)).processor(processor);
ThreadPoolExecutor threadPoolExecutor =
ThreadPoolManager.newDaemonCacheThreadPool(
Config.thrift_server_max_worker_threads, "thrift-server-pool",
true);
args.executorService(threadPoolExecutor);
@@ -123,7 +123,7 @@ public class ThriftServer {
TThreadPoolServer.Args serverArgs =
new TThreadPoolServer.Args(new
TServerSocket(socketTransportArgs)).protocolFactory(
- new TBinaryProtocol.Factory()).processor(processor);
+ new
TBinaryProtocol.Factory(Config.fe_thrift_max_pkg_bytes,
-1)).processor(processor);
ThreadPoolExecutor threadPoolExecutor =
ThreadPoolManager.newDaemonCacheThreadPool(
Config.thrift_server_max_worker_threads, "thrift-server-pool",
true);
serverArgs.executorService(threadPoolExecutor);
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
