This is an automated email from the ASF dual-hosted git repository. morningman pushed a commit to branch branch-1.2-lts in repository https://gitbox.apache.org/repos/asf/doris.git
commit 6461b423cf6db5e083cd56006dae796b39d74b2c Author: qiye <[email protected]> AuthorDate: Thu May 25 13:39:42 2023 +0800 [opt](balance) add config balance_slot_num_per_path (#19869) Make balance_slot_num_per_path configurable. --- docs/en/docs/admin-manual/config/fe-config.md | 10 ++++++++++ docs/zh-CN/docs/admin-manual/config/fe-config.md | 11 +++++++++++ .../src/main/java/org/apache/doris/common/Config.java | 4 ++++ .../main/java/org/apache/doris/clone/BeLoadRebalancer.java | 2 +- .../src/main/java/org/apache/doris/clone/DiskRebalancer.java | 3 ++- .../src/main/java/org/apache/doris/clone/TabletScheduler.java | 9 +++------ 6 files changed, 31 insertions(+), 8 deletions(-) diff --git a/docs/en/docs/admin-manual/config/fe-config.md b/docs/en/docs/admin-manual/config/fe-config.md index bc3acae538..738377cce6 100644 --- a/docs/en/docs/admin-manual/config/fe-config.md +++ b/docs/en/docs/admin-manual/config/fe-config.md @@ -1965,6 +1965,16 @@ This configs can set to true to disable the automatic colocate tables's relocate 2. Because once the balance is turned off, the unstable colocate table may not be restored 3. Eventually the colocate plan cannot be used when querying. +#### `balance_slot_num_per_path` + +Default: 1 + +IsMutable：true + +MasterOnly：true + +Default number of slots per path during balance. + #### `disable_tablet_scheduler` Default:false diff --git a/docs/zh-CN/docs/admin-manual/config/fe-config.md b/docs/zh-CN/docs/admin-manual/config/fe-config.md index 99b97544bd..eb6a93df07 100644 --- a/docs/zh-CN/docs/admin-manual/config/fe-config.md +++ b/docs/zh-CN/docs/admin-manual/config/fe-config.md @@ -1969,6 +1969,16 @@ BE副本数的平衡阈值。 2. 因为一旦关闭平衡，不稳定的 colocate 表可能无法恢复 3. 最终查询时无法使用 colocate 计划。 +#### `balance_slot_num_per_path` + +默认值：1 + +是否可以动态配置：true + +是否为 Master FE 节点独有的配置项：true + +balance 时每个路径的默认 slot 数量 + #### `disable_tablet_scheduler` 默认值：false @@ -2703,3 +2713,4 @@ show data （其他用法：HELP SHOW DATA） 当设置为 false 时，查询 `information_schema` 中的表时，将不再返回 external catalog 中的表的信息。 这个参数主要用于避免因 external catalog 无法访问、信息过多等原因导致的查询 `information_schema` 超时的问题。 + diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java index c17ef2a51c..6d561df1ce 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java +++ b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java @@ -1172,6 +1172,10 @@ public class Config extends ConfigBase { @ConfField(mutable = true, masterOnly = true) public static int partition_rebalance_max_moves_num_per_selection = 10; + // 1 slot for reduce unnecessary balance task, provided a more accurate estimate of capacity + @ConfField(masterOnly = true, mutable = true) + public static int balance_slot_num_per_path = 1; + // This threshold is to avoid piling up too many report task in FE, which may cause OOM exception. // In some large Doris cluster, eg: 100 Backends with ten million replicas, a tablet report may cost // several seconds after some modification of metadata(drop partition, etc..). diff --git a/fe/fe-core/src/main/java/org/apache/doris/clone/BeLoadRebalancer.java b/fe/fe-core/src/main/java/org/apache/doris/clone/BeLoadRebalancer.java index 44c7bc967e..9307f13ca0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/clone/BeLoadRebalancer.java +++ b/fe/fe-core/src/main/java/org/apache/doris/clone/BeLoadRebalancer.java @@ -129,7 +129,7 @@ public class BeLoadRebalancer extends Rebalancer { // for each path, we try to select at most BALANCE_SLOT_NUM_FOR_PATH tablets Map<Long, Integer> remainingPaths = Maps.newHashMap(); for (Long pathHash : pathHigh) { - remainingPaths.put(pathHash, TabletScheduler.BALANCE_SLOT_NUM_FOR_PATH); + remainingPaths.put(pathHash, Config.balance_slot_num_per_path); } if (remainingPaths.isEmpty()) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/clone/DiskRebalancer.java b/fe/fe-core/src/main/java/org/apache/doris/clone/DiskRebalancer.java index 82913bc796..02a31608f6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/clone/DiskRebalancer.java +++ b/fe/fe-core/src/main/java/org/apache/doris/clone/DiskRebalancer.java @@ -24,6 +24,7 @@ import org.apache.doris.clone.SchedException.Status; import org.apache.doris.clone.TabletSchedCtx.BalanceType; import org.apache.doris.clone.TabletSchedCtx.Priority; import org.apache.doris.clone.TabletScheduler.PathSlot; +import org.apache.doris.common.Config; import org.apache.doris.system.SystemInfoService; import org.apache.doris.thrift.TStorageMedium; @@ -171,7 +172,7 @@ public class DiskRebalancer extends Rebalancer { // for each path, we try to select at most BALANCE_SLOT_NUM_FOR_PATH tablets Map<Long, Integer> remainingPaths = Maps.newHashMap(); for (Long pathHash : pathHigh) { - remainingPaths.put(pathHash, TabletScheduler.BALANCE_SLOT_NUM_FOR_PATH); + remainingPaths.put(pathHash, Config.balance_slot_num_per_path); } if (remainingPaths.isEmpty()) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/clone/TabletScheduler.java b/fe/fe-core/src/main/java/org/apache/doris/clone/TabletScheduler.java index 7d2e7cc3b9..b243980362 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/clone/TabletScheduler.java +++ b/fe/fe-core/src/main/java/org/apache/doris/clone/TabletScheduler.java @@ -104,9 +104,6 @@ public class TabletScheduler extends MasterDaemon { private static final long SCHEDULE_INTERVAL_MS = 1000; // 1s - // 1 slot for reduce unnecessary balance task, provided a more accurate estimate of capacity - public static final int BALANCE_SLOT_NUM_FOR_PATH = 1; - /* * Tablet is added to pendingTablets as well it's id in allTabletIds. * TabletScheduler will take tablet from pendingTablets but will not remove it's id from allTabletIds when @@ -1861,7 +1858,7 @@ public class TabletScheduler extends MasterDaemon { public Slot(int total) { this.total = total; this.available = total; - this.balanceSlot = BALANCE_SLOT_NUM_FOR_PATH; + this.balanceSlot = Config.balance_slot_num_per_path; } public void rectify() { @@ -1872,8 +1869,8 @@ public class TabletScheduler extends MasterDaemon { available = total; } - if (balanceSlot > BALANCE_SLOT_NUM_FOR_PATH) { - balanceSlot = BALANCE_SLOT_NUM_FOR_PATH; + if (balanceSlot > Config.balance_slot_num_per_path) { + balanceSlot = Config.balance_slot_num_per_path; } } --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
