This is an automated email from the ASF dual-hosted git repository. morningman pushed a commit to branch branch-1.2-lts in repository https://gitbox.apache.org/repos/asf/doris.git
commit b5fb28ee81b3ca0295644eb5f6524757a159a960 Author: Zhengguo Yang <[email protected]> AuthorDate: Fri Feb 24 12:07:02 2023 +0800 [enhancement](memory) reduce memory usage for failed broker loads (#16974) Reduce more memory usage for failed broker load msg in fe after pr #15895 --- .../src/main/java/org/apache/doris/load/loadv2/LoadLoadingTask.java | 5 ++++- fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java | 5 ++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/LoadLoadingTask.java b/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/LoadLoadingTask.java index 3a2974e69c..5a6e9cc4a0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/LoadLoadingTask.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/LoadLoadingTask.java @@ -45,6 +45,7 @@ import org.apache.logging.log4j.Logger; import java.util.List; import java.util.UUID; +import java.util.stream.Collectors; public class LoadLoadingTask extends LoadTask { private static final Logger LOG = LogManager.getLogger(LoadLoadingTask.class); @@ -176,7 +177,9 @@ public class LoadLoadingTask extends LoadTask { curCoordinator.getLoadCounters(), curCoordinator.getTrackingUrl(), TabletCommitInfo.fromThrift(curCoordinator.getCommitInfos()), - ErrorTabletInfo.fromThrift(curCoordinator.getErrorTabletInfos())); + ErrorTabletInfo.fromThrift(curCoordinator.getErrorTabletInfos() + .stream().limit(Config.max_error_tablet_of_broker_load).collect(Collectors.toList()))); + curCoordinator.getErrorTabletInfos().clear(); // Create profile of this task and add to the job profile. createProfile(curCoordinator); } else { diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java b/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java index a646cb06bd..0542b96c70 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java @@ -843,7 +843,10 @@ public class Coordinator { private void updateErrorTabletInfos(List<TErrorTabletInfo> errorTabletInfos) { lock.lock(); try { - this.errorTabletInfos.addAll(errorTabletInfos); + if (this.errorTabletInfos.size() <= Config.max_error_tablet_of_broker_load) { + this.errorTabletInfos.addAll(errorTabletInfos.stream().limit(Config.max_error_tablet_of_broker_load + - this.errorTabletInfos.size()).collect(Collectors.toList())); + } } finally { lock.unlock(); } --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
