This is an automated email from the ASF dual-hosted git repository.
lide pushed a commit to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.0 by this push:
new a71f75ad565 [enhancement](backup-restore) add config for
upload/download task num per be (#27772) (#34019)
a71f75ad565 is described below
commit a71f75ad565ca2805cb455510e41704b2d45823f
Author: xueweizhang <[email protected]>
AuthorDate: Wed Apr 24 11:17:55 2024 +0800
[enhancement](backup-restore) add config for upload/download task num per
be (#27772) (#34019)
---
docs/en/docs/admin-manual/config/fe-config.md | 20 ++++++++++++++++++++
docs/zh-CN/docs/admin-manual/config/fe-config.md | 20 ++++++++++++++++++++
.../main/java/org/apache/doris/common/Config.java | 12 ++++++++++++
.../main/java/org/apache/doris/backup/BackupJob.java | 4 ++--
.../java/org/apache/doris/backup/RestoreJob.java | 7 +++----
5 files changed, 57 insertions(+), 6 deletions(-)
diff --git a/docs/en/docs/admin-manual/config/fe-config.md
b/docs/en/docs/admin-manual/config/fe-config.md
index be1928f5395..1c2ab9939a1 100644
--- a/docs/en/docs/admin-manual/config/fe-config.md
+++ b/docs/en/docs/admin-manual/config/fe-config.md
@@ -2581,6 +2581,26 @@ MasterOnly：true
default timeout of backup job
+#### `backup_upload_task_num_per_be`
+
+Default：3
+
+IsMutable：true
+
+MasterOnly：true
+
+The max number of upload tasks assigned to each be during the backup process,
the default value is 3.
+
+#### `restore_download_task_num_per_be`
+
+Default：3
+
+IsMutable：true
+
+MasterOnly：true
+
+The max number of download tasks assigned to each be during the restore
process, the default value is 3.
+
#### `max_backup_restore_job_num_per_db`
Default: 10
diff --git a/docs/zh-CN/docs/admin-manual/config/fe-config.md
b/docs/zh-CN/docs/admin-manual/config/fe-config.md
index cf3124e442b..c93b2d17e01 100644
--- a/docs/zh-CN/docs/admin-manual/config/fe-config.md
+++ b/docs/zh-CN/docs/admin-manual/config/fe-config.md
@@ -2583,6 +2583,26 @@ SmallFileMgr 中存储的最大文件数
备份作业的默认超时时间
+#### `backup_upload_task_num_per_be`
+
+默认值：3
+
+是否可以动态配置：true
+
+是否为 Master FE 节点独有的配置项：true
+
+备份过程中，分配给每个be的upload任务最大个数，默认值为3个。
+
+#### `restore_download_task_num_per_be`
+
+默认值：3
+
+是否可以动态配置：true
+
+是否为 Master FE 节点独有的配置项：true
+
+恢复过程中，分配给每个be的download任务最大个数，默认值为3个。
+
#### `max_backup_restore_job_num_per_db`
默认值：10
diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
index e099f4567b9..6e87e20960a 100644
--- a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
+++ b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
@@ -2306,6 +2306,18 @@ public class Config extends ConfigBase {
+ "or too large package causing OOM,default 20000000(20M),set -1
for unlimited. "})
public static int fe_thrift_max_pkg_bytes = 20000000;
+ @ConfField(mutable = true, masterOnly = true, description = {
+ "备份过程中，分配给每个be的upload任务最大个数，默认值为3个。",
+ "The max number of upload tasks assigned to each be during the
backup process, the default value is 3."
+ })
+ public static int backup_upload_task_num_per_be = 3;
+
+ @ConfField(mutable = true, masterOnly = true, description = {
+ "恢复过程中，分配给每个be的download任务最大个数，默认值为3个。",
+ "The max number of download tasks assigned to each be during the
restore process, the default value is 3."
+ })
+ public static int restore_download_task_num_per_be = 3;
+
@ConfField(description = {"是否开启通过http接口获取log文件的功能",
"Whether to enable the function of getting log files through http
interface"})
public static boolean enable_get_log_file_api = false;
diff --git a/fe/fe-core/src/main/java/org/apache/doris/backup/BackupJob.java
b/fe/fe-core/src/main/java/org/apache/doris/backup/BackupJob.java
index 56972e93716..f1fd703041b 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/backup/BackupJob.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/backup/BackupJob.java
@@ -34,6 +34,7 @@ import org.apache.doris.catalog.Resource;
import org.apache.doris.catalog.Table;
import org.apache.doris.catalog.Tablet;
import org.apache.doris.catalog.View;
+import org.apache.doris.common.Config;
import org.apache.doris.common.io.Text;
import org.apache.doris.common.util.TimeUtils;
import org.apache.doris.datasource.property.S3ClientBEProperties;
@@ -597,8 +598,7 @@ public class BackupJob extends AbstractJob {
for (Long beId : beToSnapshots.keySet()) {
List<SnapshotInfo> infos = beToSnapshots.get(beId);
int totalNum = infos.size();
- // each backend allot at most 3 tasks
- int batchNum = Math.min(totalNum, 3);
+ int batchNum = Math.min(totalNum,
Config.backup_upload_task_num_per_be);
// each task contains several upload sub tasks
int taskNumPerBatch = Math.max(totalNum / batchNum, 1);
LOG.info("backend {} has {} batch, total {} tasks, {}", beId,
batchNum, totalNum, this);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java
b/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java
index 0f1512e442b..123600f4705 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java
@@ -52,6 +52,7 @@ import org.apache.doris.catalog.Tablet;
import org.apache.doris.catalog.TabletMeta;
import org.apache.doris.catalog.View;
import org.apache.doris.clone.DynamicPartitionScheduler;
+import org.apache.doris.common.Config;
import org.apache.doris.common.DdlException;
import org.apache.doris.common.FeMetaVersion;
import org.apache.doris.common.MarkedCountDownLatch;
@@ -1347,8 +1348,7 @@ public class RestoreJob extends AbstractJob {
for (Long beId : beToSnapshots.keySet()) {
List<SnapshotInfo> beSnapshotInfos =
beToSnapshots.get(beId);
int totalNum = beSnapshotInfos.size();
- // each backend allot at most 3 tasks
- int batchNum = Math.min(totalNum, 3);
+ int batchNum = Math.min(totalNum,
Config.restore_download_task_num_per_be);
// each task contains several upload sub tasks
int taskNumPerBatch = Math.max(totalNum / batchNum, 1);
LOG.debug("backend {} has {} batch, total {} tasks, {}",
@@ -1500,8 +1500,7 @@ public class RestoreJob extends AbstractJob {
for (Long beId : beToSnapshots.keySet()) {
List<SnapshotInfo> beSnapshotInfos =
beToSnapshots.get(beId);
int totalNum = beSnapshotInfos.size();
- // each backend allot at most 3 tasks
- int batchNum = Math.min(totalNum, 3);
+ int batchNum = Math.min(totalNum,
Config.restore_download_task_num_per_be);
// each task contains several upload sub tasks
int taskNumPerBatch = Math.max(totalNum / batchNum, 1);
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
