This is an automated email from the ASF dual-hosted git repository.
dataroaring pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 7895a6b269f [improve](report) split agent batch tasks automaticlly
(#43257)
7895a6b269f is described below
commit 7895a6b269f6a3bf18551e58044077ad6ef71710
Author: walter <[email protected]>
AuthorDate: Wed Nov 6 09:55:33 2024 +0800
[improve](report) split agent batch tasks automaticlly (#43257)
to avoid the binary message size exceeds the thrift max message size
limitation
---
fe/fe-common/src/main/java/org/apache/doris/common/Config.java | 8 ++++++++
.../src/main/java/org/apache/doris/master/ReportHandler.java | 2 +-
2 files changed, 9 insertions(+), 1 deletion(-)
diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
index dd0aca5923e..719a7b0a4a1 100644
--- a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
+++ b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
@@ -1208,6 +1208,14 @@ public class Config extends ConfigBase {
@ConfField(mutable = true, masterOnly = true)
public static int report_queue_size = 100;
+ // if the number of report task in FE exceed max_report_task_num_per_rpc,
then split it to multiple rpc
+ @ConfField(mutable = true, masterOnly = true, description = {
+ "重新发送 agent task 时，单次 RPC 分配给每个be的任务最大个数，默认值为10000个。",
+ "The maximum number of batched tasks per RPC assigned to each BE
when resending agent tasks, "
+ + "the default value is 10000."
+ })
+ public static int report_resend_batch_task_num_per_rpc = 10000;
+
/**
* If set to true, metric collector will be run as a daemon timer to
collect metrics at fix interval
*/
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/master/ReportHandler.java
b/fe/fe-core/src/main/java/org/apache/doris/master/ReportHandler.java
index 1c8f51bd4eb..bc51590ccb5 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/master/ReportHandler.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/master/ReportHandler.java
@@ -619,7 +619,7 @@ public class ReportHandler extends Daemon {
List<AgentTask> diffTasks = AgentTaskQueue.getDiffTasks(backendId,
runningTasks);
- AgentBatchTask batchTask = new AgentBatchTask();
+ AgentBatchTask batchTask = new
AgentBatchTask(Config.report_resend_batch_task_num_per_rpc);
long taskReportTime = System.currentTimeMillis();
for (AgentTask task : diffTasks) {
// these tasks no need to do diff
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
