This is an automated email from the ASF dual-hosted git repository.
liaoxin pushed a commit to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.0 by this push:
new 598c46d4c44 [opt](routine-load) optimize allocate task to be algorithm
for load balance (#32021) (#37434)
598c46d4c44 is described below
commit 598c46d4c4481ed816cc1e13d787fc79dcf74563
Author: hui lai <[email protected]>
AuthorDate: Mon Jul 8 16:57:45 2024 +0800
[opt](routine-load) optimize allocate task to be algorithm for load balance
(#32021) (#37434)
---
.../java/org/apache/doris/load/routineload/RoutineLoadManager.java | 4 ++--
.../org/apache/doris/load/routineload/RoutineLoadTaskScheduler.java | 3 ++-
2 files changed, 4 insertions(+), 3 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadManager.java
b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadManager.java
index 4c66c510126..371251322ea 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadManager.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadManager.java
@@ -480,7 +480,7 @@ public class RoutineLoadManager implements Writable {
try {
Map<Long, Integer> beIdToConcurrentTasks =
getBeCurrentTasksNumMap();
- // 1. Find if the given BE id has available slots
+ // 1. Find if the given BE id has more than half of available slots
if (previousBeId != -1L && availableBeIds.contains(previousBeId)) {
// get the previousBackend info
Backend previousBackend =
Env.getCurrentSystemInfo().getBackend(previousBeId);
@@ -495,7 +495,7 @@ public class RoutineLoadManager implements Writable {
} else {
idleTaskNum =
beIdToMaxConcurrentTasks.get(previousBeId);
}
- if (idleTaskNum > 0) {
+ if (idleTaskNum > (Config.max_routine_load_task_num_per_be
>> 1)) {
return previousBeId;
}
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadTaskScheduler.java
b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadTaskScheduler.java
index 5c01375e7dc..10eec92b5e3 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadTaskScheduler.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadTaskScheduler.java
@@ -284,7 +284,8 @@ public class RoutineLoadTaskScheduler extends MasterDaemon {
}
// try to allocate a task to BE which has idle slot.
- // 1. First is to check if the previous allocated BE is available. If yes,
allocate task to previous BE.
+ // 1. First is to check if the previous allocated BE has more than half of
available slots.
+ // If yes, allocate task to previous BE.
// 2. If not, try to find a better one with most idle slots.
// return true if allocate successfully. return false if failed.
// throw exception if unrecoverable errors happen.
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
