This is an automated email from the ASF dual-hosted git repository.
lide pushed a commit to branch branch-1.2-lts
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-1.2-lts by this push:
new 47b2980954c [branch-1.2](backup-restore) add config for
upload/download task num per be (#27946)
47b2980954c is described below
commit 47b2980954c6046a3778d1c62354bd71d6aec63f
Author: xueweizhang <[email protected]>
AuthorDate: Tue Dec 5 10:48:13 2023 +0800
[branch-1.2](backup-restore) add config for upload/download task num per be
(#27946)
---
docs/en/docs/admin-manual/config/fe-config.md | 20 ++++++++++++++++++++
docs/zh-CN/docs/admin-manual/config/fe-config.md | 20 ++++++++++++++++++++
.../main/java/org/apache/doris/common/Config.java | 6 ++++++
.../main/java/org/apache/doris/backup/BackupJob.java | 4 ++--
.../java/org/apache/doris/backup/RestoreJob.java | 3 +--
5 files changed, 49 insertions(+), 4 deletions(-)
diff --git a/docs/en/docs/admin-manual/config/fe-config.md
b/docs/en/docs/admin-manual/config/fe-config.md
index 9d557772b64..91daa268db1 100644
--- a/docs/en/docs/admin-manual/config/fe-config.md
+++ b/docs/en/docs/admin-manual/config/fe-config.md
@@ -2586,6 +2586,26 @@ MasterOnly：true
default timeout of backup job
+#### `backup_upload_task_num_per_be`
+
+Default：3
+
+IsMutable：true
+
+MasterOnly：true
+
+The max number of upload tasks assigned to each be during the backup process,
the default value is 3.
+
+#### `restore_download_task_num_per_be`
+
+Default：3
+
+IsMutable：true
+
+MasterOnly：true
+
+The max number of download tasks assigned to each be during the restore
process, the default value is 3.
+
#### `max_backup_restore_job_num_per_db`
Default: 10
diff --git a/docs/zh-CN/docs/admin-manual/config/fe-config.md
b/docs/zh-CN/docs/admin-manual/config/fe-config.md
index ec4d759be35..c0f04e0231a 100644
--- a/docs/zh-CN/docs/admin-manual/config/fe-config.md
+++ b/docs/zh-CN/docs/admin-manual/config/fe-config.md
@@ -2586,6 +2586,26 @@ SmallFileMgr 中存储的最大文件数
备份作业的默认超时时间
+#### `backup_upload_task_num_per_be`
+
+默认值：3
+
+是否可以动态配置：true
+
+是否为 Master FE 节点独有的配置项：true
+
+备份过程中，分配给每个be的upload任务最大个数，默认值为3个。
+
+#### `restore_download_task_num_per_be`
+
+默认值：3
+
+是否可以动态配置：true
+
+是否为 Master FE 节点独有的配置项：true
+
+恢复过程中，分配给每个be的download任务最大个数，默认值为3个。
+
#### `max_backup_restore_job_num_per_db`
默认值：10
diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
index 722c2303636..0bf8da05c27 100644
--- a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
+++ b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
@@ -2045,5 +2045,11 @@ public class Config extends ConfigBase {
*/
@ConfField
public static int fe_thrift_max_pkg_bytes = 20000000;
+
+ @ConfField(mutable = true, masterOnly = true)
+ public static int backup_upload_task_num_per_be = 3;
+
+ @ConfField(mutable = true, masterOnly = true)
+ public static int restore_download_task_num_per_be = 3;
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/backup/BackupJob.java
b/fe/fe-core/src/main/java/org/apache/doris/backup/BackupJob.java
index dc0d8ec7018..b15af91cead 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/backup/BackupJob.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/backup/BackupJob.java
@@ -35,6 +35,7 @@ import org.apache.doris.catalog.Table;
import org.apache.doris.catalog.TableIf.TableType;
import org.apache.doris.catalog.Tablet;
import org.apache.doris.catalog.View;
+import org.apache.doris.common.Config;
import org.apache.doris.common.io.Text;
import org.apache.doris.common.util.TimeUtils;
import org.apache.doris.task.AgentBatchTask;
@@ -581,8 +582,7 @@ public class BackupJob extends AbstractJob {
for (Long beId : beToSnapshots.keySet()) {
List<SnapshotInfo> infos = beToSnapshots.get(beId);
int totalNum = infos.size();
- // each backend allot at most 3 tasks
- int batchNum = Math.min(totalNum, 3);
+ int batchNum = Math.min(totalNum,
Config.backup_upload_task_num_per_be);
// each task contains several upload sub tasks
int taskNumPerBatch = Math.max(totalNum / batchNum, 1);
LOG.info("backend {} has {} batch, total {} tasks, {}", beId,
batchNum, totalNum, this);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java
b/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java
index 35e3a0fc478..9720f4e3bd0 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java
@@ -1276,8 +1276,7 @@ public class RestoreJob extends AbstractJob {
for (Long beId : beToSnapshots.keySet()) {
List<SnapshotInfo> beSnapshotInfos =
beToSnapshots.get(beId);
int totalNum = beSnapshotInfos.size();
- // each backend allot at most 3 tasks
- int batchNum = Math.min(totalNum, 3);
+ int batchNum = Math.min(totalNum,
Config.restore_download_task_num_per_be);
// each task contains several upload sub tasks
int taskNumPerBatch = Math.max(totalNum / batchNum, 1);
LOG.debug("backend {} has {} batch, total {} tasks, {}",
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
