This is an automated email from the ASF dual-hosted git repository.
dataroaring pushed a commit to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.0 by this push:
new f95221e34e1 [fix](oom) avoid oom when a lot of tablets fail on load
(#36962)
f95221e34e1 is described below
commit f95221e34e167e5149dda9dfcc7ae5962b1582c0
Author: Yongqiang YANG <[email protected]>
AuthorDate: Fri Jun 28 21:07:08 2024 +0800
[fix](oom) avoid oom when a lot of tablets fail on load (#36962)
When a lot of tablets fail when loading, then detailed information would
cause oom.
## Proposed changes
Issue Number: close #xxx
<!--Describe your changes.-->
---
.../doris/transaction/DatabaseTransactionMgr.java | 59 ++++++++++++++++++----
1 file changed, 48 insertions(+), 11 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/transaction/DatabaseTransactionMgr.java
b/fe/fe-core/src/main/java/org/apache/doris/transaction/DatabaseTransactionMgr.java
index 36b32f01069..7ef043136ba 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/transaction/DatabaseTransactionMgr.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/transaction/DatabaseTransactionMgr.java
@@ -1028,6 +1028,45 @@ public class DatabaseTransactionMgr {
return true;
}
+ private class TabletsPublishResultLogs {
+ public List<String> quorumSuccLogs = Lists.newArrayList();
+ public List<String> timeoutSuccLogs = Lists.newArrayList();
+ public List<String> failedLogs = Lists.newArrayList();
+
+ public void addQuorumSuccLog(String log) {
+ if (quorumSuccLogs.size() < 16) {
+ quorumSuccLogs.add(log);
+ }
+ }
+
+ public void addTimeoutSuccLog(String log) {
+ if (timeoutSuccLogs.size() < 16) {
+ timeoutSuccLogs.add(log);
+ }
+ }
+
+ public void addFailedLog(String log) {
+ if (failedLogs.size() < 16) {
+ failedLogs.add(log);
+ }
+ }
+
+ public void log() {
+ // log failed logs
+ for (String log : failedLogs) {
+ LOG.info(log);
+ }
+ // log timeout succ logs
+ for (String log : timeoutSuccLogs) {
+ LOG.info(log);
+ }
+ // log quorum succ logs
+ for (String log : quorumSuccLogs) {
+ LOG.info(log);
+ }
+ }
+ }
+
private PublishResult finishCheckQuorumReplicas(TransactionState
transactionState,
List<Pair<OlapTable, Partition>> relatedTblPartitions,
Set<Long> errorReplicaIds) {
@@ -1042,7 +1081,7 @@ public class DatabaseTransactionMgr {
List<Replica> tabletSuccReplicas = Lists.newArrayList();
List<Replica> tabletWriteFailedReplicas = Lists.newArrayList();
List<Replica> tabletVersionFailedReplicas = Lists.newArrayList();
- List<String> logs = Lists.newArrayList();
+ TabletsPublishResultLogs logs = new TabletsPublishResultLogs();
Map<Long, PublishVersionTask> publishTasks =
transactionState.getPublishVersionTasks();
PublishResult publishResult = PublishResult.QUORUM_SUCC;
@@ -1092,9 +1131,9 @@ public class DatabaseTransactionMgr {
if (hasFailedReplica) {
String writeDetail =
getTabletWriteDetail(tabletSuccReplicas,
tabletWriteFailedReplicas,
tabletVersionFailedReplicas);
- logs.add(String.format("publish version quorum
succ for transaction %s on tablet %s"
- + " with version %s, and has failed
replicas, load require replica num %s. "
- + "table %s, partition %s, tablet detail:
%s",
+ logs.addQuorumSuccLog(String.format("publish
version quorum succ for transaction %s on "
+ + "tablet %s with version %s, and has
failed replicas, load require replica num "
+ + " %s. table %s, partition %s, tablet
detail: %s",
transactionState, tablet.getId(),
newVersion,
loadRequiredReplicaNum, tableId,
partitionId, writeDetail));
}
@@ -1116,8 +1155,8 @@ public class DatabaseTransactionMgr {
// that are being publised exists on a few replicas we
should go
// ahead, otherwise data may be lost and thre
// publish task hangs forever.
- logs.add(String.format("publish version timeout succ
for transaction %s on tablet %s "
- + "with version %s, and has failed replicas,
load require replica num %s. "
+ logs.addTimeoutSuccLog(String.format("publish version
timeout succ for transaction %s on "
+ + " tablet %s with version %s, and has failed
replicas, load require replica num %s. "
+ "table %s, partition %s, tablet detail: %s",
transactionState, tablet.getId(), newVersion,
loadRequiredReplicaNum, tableId, partitionId,
writeDetail));
@@ -1129,8 +1168,8 @@ public class DatabaseTransactionMgr {
tablet.getId(), healthReplicaNum,
loadRequiredReplicaNum, tableId,
partitionId, newVersion);
transactionState.setErrorMsg(errMsg);
- logs.add(String.format("publish version failed for
transaction %s on tablet %s with version"
- + " %s, and has failed replicas, load required
replica num %s. table %s, "
+ logs.addFailedLog(String.format("publish version
failed for transaction %s on tablet %s "
+ + "with version %s, and has failed replicas,
load required replica num %s. table %s, "
+ "partition %s, tablet detail: %s",
transactionState, tablet.getId(), newVersion,
loadRequiredReplicaNum, tableId, partitionId,
writeDetail));
@@ -1143,9 +1182,7 @@ public class DatabaseTransactionMgr {
|| now - transactionState.getLastPublishLogTime() >
Config.publish_fail_log_interval_second * 1000L;
if (needLog) {
transactionState.setLastPublishLogTime(now);
- for (String log : logs) {
- LOG.info("{}. publish times {}", log,
transactionState.getPublishCount());
- }
+ logs.log();
}
return publishResult;
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
