[
https://issues.apache.org/jira/browse/RATIS-2227?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=17908119#comment-17908119
]
Tsz-wo Sze commented on RATIS-2227:
-----------------------------------
We should release all offered entries in case of RaftLogIOException. Cc
[~jianghuazhu]
{code}
diff --git
a/ratis-server/src/main/java/org/apache/ratis/server/leader/LogAppenderBase.java
b/ratis-server/src/main/java/org/apache/ratis/server/leader/LogAppenderBase.java
index e6c92ce04..515b9ed1b 100644
---
a/ratis-server/src/main/java/org/apache/ratis/server/leader/LogAppenderBase.java
+++
b/ratis-server/src/main/java/org/apache/ratis/server/leader/LogAppenderBase.java
@@ -261,18 +261,20 @@ public abstract class LogAppenderBase implements
LogAppender {
final long halfMs = heartbeatWaitTimeMs/2;
final Map<Long, ReferenceCountedObject<EntryWithData>> offered = new
HashMap<>();
for (long next = followerNext; leaderNext > next &&
getHeartbeatWaitTimeMs() - halfMs > 0; next++) {
- ReferenceCountedObject<EntryWithData> entryWithData = null;
+ final ReferenceCountedObject<EntryWithData> entryWithData;
try {
entryWithData = getRaftLog().retainEntryWithData(next);
+ offered.put(next, entryWithData);
if (!buffer.offer(entryWithData.get())) {
entryWithData.release();
break;
}
- offered.put(next, entryWithData);
} catch (Exception e){
- if (entryWithData != null) {
- entryWithData.release();
+ for (ReferenceCountedObject<EntryWithData> ref : offered.values()) {
+ ref.release();
}
+ offered.clear();
+ throw e;
}
}
if (buffer.isEmpty()) {
{code}
> LogEntryProto leak in SegmentedRaftLog
> --------------------------------------
>
> Key: RATIS-2227
> URL: https://issues.apache.org/jira/browse/RATIS-2227
> Project: Ratis
> Issue Type: Sub-task
> Components: server
> Reporter: Attila Doroszlai
> Assignee: Tsz-wo Sze
> Priority: Major
> Attachments: org.apache.ratis.grpc.TestRaftWithGrpc-output.txt,
> org.apache.ratis.grpc.TestRaftWithGrpc.txt
>
>
> Advanced LeakDetector traces for {{TestRaftWithGrpc#testWithLoad}}:
> {code}
> [main] WARN util.ReferenceCountedLeakDetector
> (ReferenceCountedLeakDetector.java:logLeakMessage(167)) - LEAK: (class
> org.apache.ratis.proto.RaftProtos$LogEntryProto, count=1, value=term: 3
> index: 254
> stateMachineLogEntry {
> logData: "0-13"
> stateMachineEntry {
> logEntryProtoSerializedSize: 55
> }
> clientId: "E\324#\221\247\032A0\240U\220\242\264~:\032"
> callId: 150
> }
> ) #TraceInfos=6
> CREATION_0: previousRefCount=-1, refCount=0, retainCount=0, releaseCount=0,
> Thread_170:s1@group-0B2DB4D05D60->s4-GrpcLogAppender-LogAppenderDaemon
>
> org.apache.ratis.util.ReferenceCountedLeakDetector$AdvancedTracing.<init>(ReferenceCountedLeakDetector.java:310)
>
> org.apache.ratis.util.ReferenceCountedLeakDetector$Mode$3.create(ReferenceCountedLeakDetector.java:82)
>
> org.apache.ratis.util.ReferenceCountedObject.wrap(ReferenceCountedObject.java:178)
>
> org.apache.ratis.util.ReferenceCountedObject.wrap(ReferenceCountedObject.java:103)
>
> org.apache.ratis.server.raftlog.segmented.LogSegment.readSegmentFile(LogSegment.java:185)
>
> org.apache.ratis.server.raftlog.segmented.LogSegment$LogEntryLoader.load(LogSegment.java:283)
>
> org.apache.ratis.server.raftlog.segmented.LogSegment.loadCache(LogSegment.java:507)
>
> org.apache.ratis.server.raftlog.segmented.SegmentedRaftLog.retainLog(SegmentedRaftLog.java:322)
>
> org.apache.ratis.server.raftlog.segmented.SegmentedRaftLog.retainEntryWithData(SegmentedRaftLog.java:332)
>
> org.apache.ratis.server.leader.LogAppenderBase.nextAppendEntriesRequest(LogAppenderBase.java:266)
>
> org.apache.ratis.grpc.server.GrpcLogAppender.appendLog(GrpcLogAppender.java:396)
> org.apache.ratis.grpc.server.GrpcLogAppender.run(GrpcLogAppender.java:262)
>
> org.apache.ratis.server.leader.LogAppenderDaemon.run(LogAppenderDaemon.java:80)
> java.lang.Thread.run(Thread.java:750)
> RETAIN_1: previousRefCount=0, refCount=1, retainCount=1, releaseCount=0,
> Thread_170:s1@group-0B2DB4D05D60->s4-GrpcLogAppender-LogAppenderDaemon
>
> org.apache.ratis.util.ReferenceCountedLeakDetector$AdvancedTracing.retain(ReferenceCountedLeakDetector.java:325)
>
> org.apache.ratis.server.raftlog.segmented.LogSegment$LogEntryLoader.lambda$load$0(LogSegment.java:284)
>
> org.apache.ratis.server.raftlog.segmented.LogSegment.readSegmentFile(LogSegment.java:185)
> ...
> RETAIN_2: previousRefCount=1, refCount=2, retainCount=2, releaseCount=0,
> Thread_170:s1@group-0B2DB4D05D60->s4-GrpcLogAppender-LogAppenderDaemon
>
> org.apache.ratis.util.ReferenceCountedLeakDetector$AdvancedTracing.retain(ReferenceCountedLeakDetector.java:325)
>
> org.apache.ratis.server.raftlog.segmented.LogSegment$EntryCache.put(LogSegment.java:370)
>
> org.apache.ratis.server.raftlog.segmented.LogSegment.putEntryCache(LogSegment.java:589)
>
> org.apache.ratis.server.raftlog.segmented.LogSegment$LogEntryLoader.lambda$load$0(LogSegment.java:287)
>
> org.apache.ratis.server.raftlog.segmented.LogSegment.readSegmentFile(LogSegment.java:185)
> ...
> RETAIN_3: previousRefCount=2, refCount=3, retainCount=3, releaseCount=0,
> Thread_170:s1@group-0B2DB4D05D60->s4-GrpcLogAppender-LogAppenderDaemon
>
> org.apache.ratis.util.ReferenceCountedLeakDetector$AdvancedTracing.retain(ReferenceCountedLeakDetector.java:325)
>
> org.apache.ratis.server.raftlog.segmented.SegmentedRaftLog.retainLog(SegmentedRaftLog.java:310)
>
> org.apache.ratis.server.raftlog.segmented.SegmentedRaftLog.retainEntryWithData(SegmentedRaftLog.java:332)
> ...
> RELEASE_4: previousRefCount=3, refCount=2, retainCount=3, releaseCount=1,
> Thread_170:s1@group-0B2DB4D05D60->s4-GrpcLogAppender-LogAppenderDaemon
>
> org.apache.ratis.util.ReferenceCountedLeakDetector$AdvancedTracing.release(ReferenceCountedLeakDetector.java:334)
>
> org.apache.ratis.server.raftlog.RaftLogBase$1.release(RaftLogBase.java:443)
>
> org.apache.ratis.util.ReferenceCountedObject$2.release(ReferenceCountedObject.java:123)
>
> org.apache.ratis.grpc.server.GrpcLogAppender.appendLog(GrpcLogAppender.java:423)
> org.apache.ratis.grpc.server.GrpcLogAppender.run(GrpcLogAppender.java:262)
> ...
> RELEASE_5: previousRefCount=2, refCount=1, retainCount=3, releaseCount=2,
> Thread_46:s1@group-0B2DB4D05D60-cacheEviction-AwaitToRun
>
> org.apache.ratis.util.ReferenceCountedLeakDetector$AdvancedTracing.release(ReferenceCountedLeakDetector.java:334)
>
> org.apache.ratis.server.raftlog.segmented.LogSegment$Item.release(LogSegment.java:320)
>
> org.apache.ratis.server.raftlog.segmented.LogSegment$EntryCache.release(LogSegment.java:380)
>
> org.apache.ratis.server.raftlog.segmented.LogSegment$EntryCache.evict(LogSegment.java:362)
>
> org.apache.ratis.server.raftlog.segmented.LogSegment.evictCache(LogSegment.java:585)
>
> org.apache.ratis.server.raftlog.segmented.SegmentedRaftLogCache.evictCache(SegmentedRaftLogCache.java:475)
>
> org.apache.ratis.server.raftlog.segmented.SegmentedRaftLog.checkAndEvictCache(SegmentedRaftLog.java:365)
> org.apache.ratis.util.AwaitToRun$RunnableImpl.run(AwaitToRun.java:55)
> java.lang.Thread.run(Thread.java:750)
> CURRENT_6: previousRefCount=1, refCount=1, retainCount=3, releaseCount=2,
> Thread_1:main
>
> org.apache.ratis.util.ReferenceCountedLeakDetector$AdvancedTracing.getTraceString(ReferenceCountedLeakDetector.java:343)
>
> org.apache.ratis.util.ReferenceCountedLeakDetector$SimpleTracing.logLeakMessage(ReferenceCountedLeakDetector.java:166)
>
> org.apache.ratis.util.LeakDetector$LeakTracker.reportLeak(LeakDetector.java:199)
>
> org.apache.ratis.util.LeakDetector$LeakTrackerSet.getNumLeaks(LeakDetector.java:89)
> org.apache.ratis.util.LeakDetector.assertNoLeaks(LeakDetector.java:171)
>
> org.apache.ratis.server.impl.MiniRaftCluster.shutdown(MiniRaftCluster.java:892)
>
> org.apache.ratis.grpc.MiniRaftClusterWithGrpc.shutdown(MiniRaftClusterWithGrpc.java:97)
>
> org.apache.ratis.server.impl.MiniRaftCluster$Factory$Get.runWithNewCluster(MiniRaftCluster.java:152)
>
> org.apache.ratis.server.impl.MiniRaftCluster$Factory$Get.runWithNewCluster(MiniRaftCluster.java:121)
> org.apache.ratis.RaftBasicTests.testWithLoad(RaftBasicTests.java:338)
>
> org.apache.ratis.grpc.TestRaftWithGrpc.testWithLoad(TestRaftWithGrpc.java:62)
> {code}
--
This message was sent by Atlassian Jira
(v8.20.10#820010)
