[
https://issues.apache.org/jira/browse/RATIS-1008?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
Glen Geng updated RATIS-1008:
-----------------------------
Description:
If raft server use Grpc, when append entries blocked at follower side (e.g., by
calling SimpleStateMachine4Testing::blockWriteStateMachineData), leader should
still be able to send heartbeat (size of entries of append entries request be
0) to followers, and receive response of heartbeat from follower side, since we
have the code
{code:java}
@Override
public void onNext(REQUEST request) {
if (!replyInOrder(request)) {
try {
process(request).thenAccept(this::handleReply);
} catch (Throwable e) {
handleError(e, request);
}
return;
}
final PendingServerRequest<REQUEST> current = new
PendingServerRequest<>(request);
final PendingServerRequest<REQUEST> previous =
previousOnNext.getAndSet(current);
final CompletableFuture<Void> previousFuture = Optional.ofNullable(previous)
.map(PendingServerRequest::getFuture){code}
But test case "testUpdateViaHeartbeat" shows that after calling
SimpleStateMachine4Testing::blockWriteStateMachineData, leader do send
heartbeats, but ServerRequestStreamObserver::onNext at
GrpcServerProtocolService was not called, until follower finish handling the
append entries request that blocked by blockWriteStateMachineData.
*How to reproduce this issue ?*
checkout to master, add following debug info
{code:java}
gengbin@GLENGENG-MB0 incubator-ratis % git diff
diff --git
a/ratis-grpc/src/main/java/org/apache/ratis/grpc/server/GrpcLogAppender.java
b/ratis-grpc/src/main/java/org/apache/ratis/grpc/server/GrpcLogAppender.java
index 4eb5cb9d..97c4679e 100644
--- a/ratis-grpc/src/main/java/org/apache/ratis/grpc/server/GrpcLogAppender.java
+++ b/ratis-grpc/src/main/java/org/apache/ratis/grpc/server/GrpcLogAppender.java
@@ -207,6 +207,8 @@ public class GrpcLogAppender extends LogAppender {
getServer().getId(), null, proto);
request.startRequestTimer();
s.onNext(proto);
+ LOG.info("pending log size: {}, pending hb size: {}",
+ pendingRequests.logRequests.size(), pendingRequests.heartbeats.size());
scheduler.onTimeout(requestTimeoutDuration,
() -> timeoutAppendRequest(request.getCallId(), request.isHeartbeat()),
LOG, () -> "Timeout check failed for append entry request: " +
request);
diff --git
a/ratis-grpc/src/main/java/org/apache/ratis/grpc/server/GrpcServerProtocolService.java
b/ratis-grpc/src/main/java/org/apache/ratis/grpc/server/GrpcServerProtocolService.java
index 21a6c560..272d3c0b 100644
---
a/ratis-grpc/src/main/java/org/apache/ratis/grpc/server/GrpcServerProtocolService.java
+++
b/ratis-grpc/src/main/java/org/apache/ratis/grpc/server/GrpcServerProtocolService.java
@@ -106,6 +106,8 @@ class GrpcServerProtocolService extends
RaftServerProtocolServiceImplBase {
@Override
public void onNext(REQUEST request) {
+ LOG.info("receive request, in order or not: {}", replyInOrder(request));
+
if (!replyInOrder(request)) {
try {
process(request).thenAccept(this::handleReply);
diff --git
a/ratis-server/src/main/java/org/apache/ratis/server/impl/RaftServerImpl.java
b/ratis-server/src/main/java/org/apache/ratis/server/impl/RaftServerImpl.java
index 3c4a3d88..2bb3d7e0 100644
---
a/ratis-server/src/main/java/org/apache/ratis/server/impl/RaftServerImpl.java
+++
b/ratis-server/src/main/java/org/apache/ratis/server/impl/RaftServerImpl.java
@@ -933,13 +933,9 @@ public class RaftServerImpl implements RaftServerProtocol,
RaftServerAsynchronou
static void logAppendEntries(boolean isHeartbeat, Supplier<String> message) {
if (isHeartbeat) {
- if (LOG.isTraceEnabled()) {
- LOG.trace("HEARTBEAT: " + message.get());
- }
+ LOG.info("HEARTBEAT: " + message.get());
} else {
- if (LOG.isDebugEnabled()) {
LOG.debug(message.get());
- }
}
}
diff --git
a/ratis-test/src/test/java/org/apache/ratis/grpc/TestRaftWithGrpc.java
b/ratis-test/src/test/java/org/apache/ratis/grpc/TestRaftWithGrpc.java
index 0f1d71db..00cd4833 100644
--- a/ratis-test/src/test/java/org/apache/ratis/grpc/TestRaftWithGrpc.java
+++ b/ratis-test/src/test/java/org/apache/ratis/grpc/TestRaftWithGrpc.java
@@ -17,18 +17,23 @@
*/
package org.apache.ratis.grpc;
+import org.apache.log4j.Level;
import org.apache.ratis.MiniRaftCluster;
import org.apache.ratis.RaftBasicTests;
import org.apache.ratis.RaftTestUtil;
import org.apache.ratis.client.RaftClient;
+import org.apache.ratis.grpc.server.GrpcLogAppender;
import org.apache.ratis.protocol.RaftClientReply;
+import org.apache.ratis.server.RaftServerConfigKeys;
import org.apache.ratis.server.impl.BlockRequestHandlingInjection;
+import org.apache.ratis.server.impl.RaftServerImpl;
import org.apache.ratis.server.impl.RaftServerTestUtil;
import org.apache.ratis.server.protocol.TermIndex;
import org.apache.ratis.server.raftlog.RaftLog;
import org.apache.ratis.statemachine.SimpleStateMachine4Testing;
import org.apache.ratis.statemachine.StateMachine;
import org.apache.ratis.util.JavaUtils;
+import org.apache.ratis.util.Log4jUtils;
import org.apache.ratis.util.TimeDuration;
import org.junit.Assert;
import org.junit.Test;
@@ -47,6 +52,11 @@ public class TestRaftWithGrpc
SimpleStateMachine4Testing.class, StateMachine.class);
}
+ {
+ Log4jUtils.setLogLevel(RaftServerImpl.LOG, Level.DEBUG);
+ Log4jUtils.setLogLevel(GrpcLogAppender.LOG, Level.DEBUG);
+ }
+
@Override
@Test
public void testWithLoad() throws Exception {
@@ -67,7 +77,12 @@ public class TestRaftWithGrpc
@Test
public void testUpdateViaHeartbeat() throws Exception {
+ final TimeDuration oldTimeoutMax =
RaftServerConfigKeys.Rpc.timeoutMax(getProperties());
+ RaftServerConfigKeys.Rpc.setTimeoutMax(getProperties(),
TimeDuration.valueOf(5, TimeUnit.SECONDS));
+
runWithNewCluster(NUM_SERVERS, this::runTestUpdateViaHeartbeat);
+
+ RaftServerConfigKeys.Rpc.setTimeoutMax(getProperties(), oldTimeoutMax);
}
void runTestUpdateViaHeartbeat(MiniRaftClusterWithGrpc cluster) throws
Exception {
{code}
*How to prove this ?*
run test case testUpdateViaHeartbeat
{code:java}
2020-07-21 17:12:50,603 [Time-limited test] INFO
statemachine.SimpleStateMachine4Testing
(SimpleStateMachine4Testing.java:block(135)) - block WRITE_STATE_MACHINE_DATA
2020-07-21 17:12:50,603 [Time-limited test] INFO
statemachine.SimpleStateMachine4Testing
(SimpleStateMachine4Testing.java:block(135)) - block WRITE_STATE_MACHINE_DATA
2020-07-21 17:12:50,603 [Time-limited test] INFO
statemachine.SimpleStateMachine4Testing
(SimpleStateMachine4Testing.java:block(135)) - block WRITE_STATE_MACHINE_DATA
2020-07-21 17:12:50,604 [Time-limited test] INFO
statemachine.SimpleStateMachine4Testing
(SimpleStateMachine4Testing.java:block(135)) - block WRITE_STATE_MACHINE_DATA
2020-07-21 17:12:50,627
[org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@7d13f127]
INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) - pending
log size: 0, pending hb size: 1
2020-07-21 17:12:50,627
[org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@5d0b025a]
INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) - pending
log size: 0, pending hb size: 1
2020-07-21 17:12:50,627
[org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@427cb8b9]
INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) - pending
log size: 0, pending hb size: 1
2020-07-21 17:12:50,627
[org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@66633814]
INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) - pending
log size: 0, pending hb size: 1
2020-07-21 17:12:50,627 [grpc-default-executor-6] INFO
server.GrpcServerProtocolService (GrpcServerProtocolService.java:onNext(109)) -
receive request, in order or not: false
2020-07-21 17:12:50,627 [grpc-default-executor-2] INFO
server.GrpcServerProtocolService (GrpcServerProtocolService.java:onNext(109)) -
receive request, in order or not: false
2020-07-21 17:12:50,627 [grpc-default-executor-0] INFO
server.GrpcServerProtocolService (GrpcServerProtocolService.java:onNext(109)) -
receive request, in order or not: false
2020-07-21 17:12:50,627 [grpc-default-executor-7] INFO
server.GrpcServerProtocolService (GrpcServerProtocolService.java:onNext(109)) -
receive request, in order or not: false
2020-07-21 17:12:50,628 [grpc-default-executor-0] INFO impl.RaftServerImpl
(RaftServerImpl.java:logAppendEntries(936)) - HEARTBEAT: s4@group-6CFE864C49B6:
receive appendEntries(s1, 1, (t:1, i:0), 0, false, commits[s1:c0, s3:c0, s4:c0,
s0:c0, s2:c0], entries: []
2020-07-21 17:12:50,628 [grpc-default-executor-2] INFO impl.RaftServerImpl
(RaftServerImpl.java:logAppendEntries(936)) - HEARTBEAT: s0@group-6CFE864C49B6:
receive appendEntries(s1, 1, (t:1, i:0), 0, false, commits[s1:c0, s3:c0, s4:c0,
s0:c0, s2:c0], entries: []
2020-07-21 17:12:50,628 [grpc-default-executor-6] INFO impl.RaftServerImpl
(RaftServerImpl.java:logAppendEntries(936)) - HEARTBEAT: s3@group-6CFE864C49B6:
receive appendEntries(s1, 1, (t:1, i:0), 0, false, commits[s1:c0, s3:c0, s4:c0,
s0:c0, s2:c0], entries: []
2020-07-21 17:12:50,628 [ForkJoinPool.commonPool-worker-9] INFO
impl.RaftServerImpl (RaftServerImpl.java:logAppendEntries(936)) - HEARTBEAT:
s4@group-6CFE864C49B6: succeeded to handle AppendEntries. Reply:
s1<-s4#69:OK,SUCCESS,nextIndex:1,term:1,followerCommit:0
2020-07-21 17:12:50,628 [grpc-default-executor-7] INFO impl.RaftServerImpl
(RaftServerImpl.java:logAppendEntries(936)) - HEARTBEAT: s2@group-6CFE864C49B6:
receive appendEntries(s1, 1, (t:1, i:0), 0, false, commits[s1:c0, s3:c0, s4:c0,
s0:c0, s2:c0], entries: []
2020-07-21 17:12:50,628 [ForkJoinPool.commonPool-worker-11] INFO
impl.RaftServerImpl (RaftServerImpl.java:logAppendEntries(936)) - HEARTBEAT:
s3@group-6CFE864C49B6: succeeded to handle AppendEntries. Reply:
s1<-s3#69:OK,SUCCESS,nextIndex:1,term:1,followerCommit:0
2020-07-21 17:12:50,628 [ForkJoinPool.commonPool-worker-4] INFO
impl.RaftServerImpl (RaftServerImpl.java:logAppendEntries(936)) - HEARTBEAT:
s0@group-6CFE864C49B6: succeeded to handle AppendEntries. Reply:
s1<-s0#69:OK,SUCCESS,nextIndex:1,term:1,followerCommit:0
2020-07-21 17:12:50,628 [ForkJoinPool.commonPool-worker-9] INFO
impl.RaftServerImpl (RaftServerImpl.java:logAppendEntries(936)) - HEARTBEAT:
s2@group-6CFE864C49B6: succeeded to handle AppendEntries. Reply:
s1<-s2#69:OK,SUCCESS,nextIndex:1,term:1,followerCommit:0
2020-07-21 17:12:50,629 [grpc-default-executor-7] DEBUG server.GrpcLogAppender
(GrpcLogAppender.java:onNext(259)) -
s1@group-6CFE864C49B6->s4-AppendLogResponseHandler: received a reply
s1<-s4#69:OK,SUCCESS,nextIndex:1,term:1,followerCommit:0,
request=AppendEntriesRequest:cid=69,entriesCount=0,lastEntry=null
2020-07-21 17:12:50,629 [grpc-default-executor-6] DEBUG server.GrpcLogAppender
(GrpcLogAppender.java:onNext(259)) -
s1@group-6CFE864C49B6->s3-AppendLogResponseHandler: received a reply
s1<-s3#69:OK,SUCCESS,nextIndex:1,term:1,followerCommit:0,
request=AppendEntriesRequest:cid=69,entriesCount=0,lastEntry=null
2020-07-21 17:12:50,629 [grpc-default-executor-2] DEBUG server.GrpcLogAppender
(GrpcLogAppender.java:onNext(259)) -
s1@group-6CFE864C49B6->s0-AppendLogResponseHandler: received a reply
s1<-s0#69:OK,SUCCESS,nextIndex:1,term:1,followerCommit:0,
request=AppendEntriesRequest:cid=69,entriesCount=0,lastEntry=null
2020-07-21 17:12:50,629 [grpc-default-executor-6] DEBUG server.GrpcLogAppender
(GrpcLogAppender.java:onNext(259)) -
s1@group-6CFE864C49B6->s2-AppendLogResponseHandler: received a reply
s1<-s2#69:OK,SUCCESS,nextIndex:1,term:1,followerCommit:0,
request=AppendEntriesRequest:cid=69,entriesCount=0,lastEntry=null
2020-07-21 17:12:50,643 [grpc-default-executor-6] DEBUG impl.RaftServerImpl
(RaftServerImpl.java:submitClientRequestAsync(587)) - s3@group-6CFE864C49B6:
receive client
request(RaftClientRequest:client-DE8B7308F86F->s3@group-6CFE864C49B6, cid=0,
seq=1*, Watch(0), Message:<EMPTY>)
2020-07-21 17:12:50,668 [grpc-default-executor-6] DEBUG impl.RaftServerImpl
(RaftServerImpl.java:submitClientRequestAsync(587)) - s1@group-6CFE864C49B6:
receive client
request(RaftClientRequest:client-DE8B7308F86F->s1@group-6CFE864C49B6, cid=0,
seq=1*, Watch(0), Message:<EMPTY>)
2020-07-21 17:12:50,669 [grpc-default-executor-6] DEBUG impl.RaftServerImpl
(LeaderState.java:addWatchReqeust(358)) - s1@group-6CFE864C49B6-LeaderState:
addWatchRequest RaftClientRequest:client-DE8B7308F86F->s1@group-6CFE864C49B6,
cid=0, seq=1*, Watch(0), Message:<EMPTY>
2020-07-21 17:12:50,674 [grpc-default-executor-2] DEBUG impl.RaftServerImpl
(RaftServerImpl.java:submitClientRequestAsync(587)) - s1@group-6CFE864C49B6:
receive client
request(RaftClientRequest:client-DE8B7308F86F->s1@group-6CFE864C49B6, cid=1,
seq=2, RW, Message:616263)
2020-07-21 17:12:50,685 [grpc-default-executor-2] DEBUG impl.RaftServerImpl
(LeaderState.java:addPendingRequest(340)) - s1@group-6CFE864C49B6-LeaderState:
addPendingRequest at
RaftClientRequest:client-DE8B7308F86F->s1@group-6CFE864C49B6, cid=1, seq=2, RW,
Message:616263, entry=(t:1, i:1), STATEMACHINELOGENTRY, client-DE8B7308F86F,
cid=1
2020-07-21 17:12:50,686
[org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@5d0b025a]
INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) - pending
log size: 1, pending hb size: 0
2020-07-21 17:12:50,686
[org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@66633814]
INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) - pending
log size: 1, pending hb size: 0
2020-07-21 17:12:50,686
[org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@427cb8b9]
INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) - pending
log size: 1, pending hb size: 0
2020-07-21 17:12:50,686
[org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@7d13f127]
INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) - pending
log size: 1, pending hb size: 0
2020-07-21 17:12:50,687 [grpc-default-executor-6] INFO
server.GrpcServerProtocolService (GrpcServerProtocolService.java:onNext(109)) -
receive request, in order or not: true
2020-07-21 17:12:50,687 [grpc-default-executor-0] INFO
server.GrpcServerProtocolService (GrpcServerProtocolService.java:onNext(109)) -
receive request, in order or not: true
2020-07-21 17:12:50,687 [grpc-default-executor-2] INFO
server.GrpcServerProtocolService (GrpcServerProtocolService.java:onNext(109)) -
receive request, in order or not: true
2020-07-21 17:12:50,687 [grpc-default-executor-0] DEBUG impl.RaftServerImpl
(RaftServerImpl.java:logAppendEntries(938)) - s2@group-6CFE864C49B6: receive
appendEntries(s1, 1, (t:1, i:0), 0, false, commits[s1:c0, s3:c0, s4:c0, s0:c0,
s2:c0], entries: (t:1, i:1), STATEMACHINELOGENTRY, client-DE8B7308F86F, cid=1
2020-07-21 17:12:50,687 [grpc-default-executor-2] DEBUG impl.RaftServerImpl
(RaftServerImpl.java:logAppendEntries(938)) - s3@group-6CFE864C49B6: receive
appendEntries(s1, 1, (t:1, i:0), 0, false, commits[s1:c0, s3:c0, s4:c0, s0:c0,
s2:c0], entries: (t:1, i:1), STATEMACHINELOGENTRY, client-DE8B7308F86F, cid=1
2020-07-21 17:12:50,687 [grpc-default-executor-7] INFO
server.GrpcServerProtocolService (GrpcServerProtocolService.java:onNext(109)) -
receive request, in order or not: true
2020-07-21 17:12:50,687 [grpc-default-executor-6] DEBUG impl.RaftServerImpl
(RaftServerImpl.java:logAppendEntries(938)) - s0@group-6CFE864C49B6: receive
appendEntries(s1, 1, (t:1, i:0), 0, false, commits[s1:c0, s3:c0, s4:c0, s0:c0,
s2:c0], entries: (t:1, i:1), STATEMACHINELOGENTRY, client-DE8B7308F86F, cid=1
2020-07-21 17:12:50,688 [grpc-default-executor-7] DEBUG impl.RaftServerImpl
(RaftServerImpl.java:logAppendEntries(938)) - s4@group-6CFE864C49B6: receive
appendEntries(s1, 1, (t:1, i:0), 0, false, commits[s1:c0, s3:c0, s4:c0, s0:c0,
s2:c0], entries: (t:1, i:1), STATEMACHINELOGENTRY, client-DE8B7308F86F, cid=1
2020-07-21 17:12:50,765
[org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@66633814]
INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) - pending
log size: 1, pending hb size: 1
2020-07-21 17:12:50,765
[org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@7d13f127]
INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) - pending
log size: 1, pending hb size: 1
2020-07-21 17:12:50,765
[org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@5d0b025a]
INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) - pending
log size: 1, pending hb size: 1
2020-07-21 17:12:50,765
[org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@427cb8b9]
INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) - pending
log size: 1, pending hb size: 1
2020-07-21 17:12:50,843
[org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@66633814]
INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) - pending
log size: 1, pending hb size: 2
2020-07-21 17:12:50,843
[org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@5d0b025a]
INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) - pending
log size: 1, pending hb size: 2
2020-07-21 17:12:50,843
[org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@427cb8b9]
INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) - pending
log size: 1, pending hb size: 2
2020-07-21 17:12:50,843
[org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@7d13f127]
INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) - pending
log size: 1, pending hb size: 2
2020-07-21 17:12:50,922
[org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@66633814]
INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) - pending
log size: 1, pending hb size: 3
2020-07-21 17:12:50,922
[org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@427cb8b9]
INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) - pending
log size: 1, pending hb size: 3
2020-07-21 17:12:50,922
[org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@5d0b025a]
INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) - pending
log size: 1, pending hb size: 3
2020-07-21 17:12:50,922
[org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@7d13f127]
INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) - pending
log size: 1, pending hb size: 3
2020-07-21 17:12:51,003
[org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@5d0b025a]
INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) - pending
log size: 1, pending hb size: 4
2020-07-21 17:12:51,003
[org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@66633814]
INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) - pending
log size: 1, pending hb size: 4
2020-07-21 17:12:51,003
[org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@7d13f127]
INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) - pending
log size: 1, pending hb size: 4
2020-07-21 17:12:51,003
[org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@427cb8b9]
INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) - pending
log size: 1, pending hb size: 4
2020-07-21 17:12:51,083
[org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@427cb8b9]
INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) - pending
log size: 1, pending hb size: 5
2020-07-21 17:12:51,083
[org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@5d0b025a]
INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) - pending
log size: 1, pending hb size: 5
2020-07-21 17:12:51,083
[org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@66633814]
INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) - pending
log size: 1, pending hb size: 5
2020-07-21 17:12:51,083
[org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@7d13f127]
INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) - pending
log size: 1, pending hb size: 5
2020-07-21 17:12:51,159
[org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@7d13f127]
INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) - pending
log size: 1, pending hb size: 6
2020-07-21 17:12:51,159
[org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@427cb8b9]
INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) - pending
log size: 1, pending hb size: 6
2020-07-21 17:12:51,159
[org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@5d0b025a]
INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) - pending
log size: 1, pending hb size: 6
2020-07-21 17:12:51,159
[org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@66633814]
INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) - pending
log size: 1, pending hb size: 6
2020-07-21 17:12:51,236
[org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@5d0b025a]
INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) - pending
log size: 1, pending hb size: 7
2020-07-21 17:12:51,236
[org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@7d13f127]
INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) - pending
log size: 1, pending hb size: 7
2020-07-21 17:12:51,236
[org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@427cb8b9]
INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) - pending
log size: 1, pending hb size: 7
2020-07-21 17:12:51,236
[org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@66633814]
INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) - pending
log size: 1, pending hb size: 7
2020-07-21 17:12:51,317
[org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@427cb8b9]
INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) - pending
log size: 1, pending hb size: 8
2020-07-21 17:12:51,317
[org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@5d0b025a]
INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) - pending
log size: 1, pending hb size: 8
2020-07-21 17:12:51,317
[org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@66633814]
INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) - pending
log size: 1, pending hb size: 8
2020-07-21 17:12:51,317
[org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@7d13f127]
INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) - pending
log size: 1, pending hb size: 8
2020-07-21 17:12:51,397
[org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@66633814]
INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) - pending
log size: 1, pending hb size: 9
2020-07-21 17:12:51,397
[org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@427cb8b9]
INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) - pending
log size: 1, pending hb size: 9
2020-07-21 17:12:51,397
[org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@7d13f127]
INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) - pending
log size: 1, pending hb size: 9
2020-07-21 17:12:51,397
[org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@5d0b025a]
INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) - pending
log size: 1, pending hb size: 9
2020-07-21 17:12:51,477
[org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@5d0b025a]
INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) - pending
log size: 1, pending hb size: 10
2020-07-21 17:12:51,477
[org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@66633814]
INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) - pending
log size: 1, pending hb size: 10
2020-07-21 17:12:51,477
[org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@427cb8b9]
INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) - pending
log size: 1, pending hb size: 10
2020-07-21 17:12:51,477
[org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@7d13f127]
INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) - pending
log size: 1, pending hb size: 10
2020-07-21 17:12:51,554
[org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@5d0b025a]
INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) - pending
log size: 1, pending hb size: 11
2020-07-21 17:12:51,554
[org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@427cb8b9]
INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) - pending
log size: 1, pending hb size: 11
2020-07-21 17:12:51,554
[org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@66633814]
INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) - pending
log size: 1, pending hb size: 11
2020-07-21 17:12:51,554
[org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@7d13f127]
INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) - pending
log size: 1, pending hb size: 11
2020-07-21 17:12:51,634
[org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@7d13f127]
INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) - pending
log size: 1, pending hb size: 12
2020-07-21 17:12:51,634
[org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@427cb8b9]
INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) - pending
log size: 1, pending hb size: 12
2020-07-21 17:12:51,634
[org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@66633814]
INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) - pending
log size: 1, pending hb size: 12
2020-07-21 17:12:51,634
[org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@5d0b025a]
INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) - pending
log size: 1, pending hb size: 12
2020-07-21 17:12:51,714
[org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@427cb8b9]
INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) - pending
log size: 1, pending hb size: 13
2020-07-21 17:12:51,714
[org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@5d0b025a]
INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) - pending
log size: 1, pending hb size: 13
2020-07-21 17:12:51,714
[org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@7d13f127]
INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) - pending
log size: 1, pending hb size: 13
2020-07-21 17:12:51,714
[org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@66633814]
INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) - pending
log size: 1, pending hb size: 13
2020-07-21 17:12:51,793
[org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@5d0b025a]
INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) - pending
log size: 1, pending hb size: 14
2020-07-21 17:12:51,793
[org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@427cb8b9]
INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) - pending
log size: 1, pending hb size: 14
2020-07-21 17:12:51,794
[org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@66633814]
INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) - pending
log size: 1, pending hb size: 14
2020-07-21 17:12:51,794
[org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@7d13f127]
INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) - pending
log size: 1, pending hb size: 14
2020-07-21 17:12:51,871
[org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@7d13f127]
INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) - pending
log size: 1, pending hb size: 15
2020-07-21 17:12:51,871
[org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@5d0b025a]
INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) - pending
log size: 1, pending hb size: 15
2020-07-21 17:12:51,871
[org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@66633814]
INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) - pending
log size: 1, pending hb size: 15
2020-07-21 17:12:51,871
[org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@427cb8b9]
INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) - pending
log size: 1, pending hb size: 15
2020-07-21 17:12:51,946
[org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@7d13f127]
INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) - pending
log size: 1, pending hb size: 16
2020-07-21 17:12:51,946
[org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@5d0b025a]
INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) - pending
log size: 1, pending hb size: 16
2020-07-21 17:12:51,946
[org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@427cb8b9]
INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) - pending
log size: 1, pending hb size: 16
2020-07-21 17:12:51,946
[org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@66633814]
INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) - pending
log size: 1, pending hb size: 16
2020-07-21 17:12:52,026
[org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@66633814]
INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) - pending
log size: 1, pending hb size: 17
2020-07-21 17:12:52,026
[org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@427cb8b9]
INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) - pending
log size: 1, pending hb size: 17
2020-07-21 17:12:52,026
[org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@7d13f127]
INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) - pending
log size: 1, pending hb size: 17
2020-07-21 17:12:52,026
[org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@5d0b025a]
INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) - pending
log size: 1, pending hb size: 17
2020-07-21 17:12:52,104
[org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@427cb8b9]
INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) - pending
log size: 1, pending hb size: 18
2020-07-21 17:12:52,104
[org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@66633814]
INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) - pending
log size: 1, pending hb size: 18
2020-07-21 17:12:52,104
[org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@7d13f127]
INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) - pending
log size: 1, pending hb size: 18
2020-07-21 17:12:52,104
[org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@5d0b025a]
INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) - pending
log size: 1, pending hb size: 18
2020-07-21 17:12:52,183
[org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@427cb8b9]
INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) - pending
log size: 1, pending hb size: 19
2020-07-21 17:12:52,183
[org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@7d13f127]
INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) - pending
log size: 1, pending hb size: 19
2020-07-21 17:12:52,183
[org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@5d0b025a]
INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) - pending
log size: 1, pending hb size: 19
2020-07-21 17:12:52,183
[org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@66633814]
INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) - pending
log size: 1, pending hb size: 19
2020-07-21 17:12:52,263
[org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@66633814]
INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) - pending
log size: 1, pending hb size: 20
2020-07-21 17:12:52,263
[org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@427cb8b9]
INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) - pending
log size: 1, pending hb size: 20
2020-07-21 17:12:52,263
[org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@5d0b025a]
INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) - pending
log size: 1, pending hb size: 20
2020-07-21 17:12:52,263
[org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@7d13f127]
INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) - pending
log size: 1, pending hb size: 20
2020-07-21 17:12:52,339
[org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@66633814]
INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) - pending
log size: 1, pending hb size: 21
2020-07-21 17:12:52,339
[org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@427cb8b9]
INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) - pending
log size: 1, pending hb size: 21
2020-07-21 17:12:52,344
[org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@7d13f127]
INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) - pending
log size: 1, pending hb size: 21
2020-07-21 17:12:52,344
[org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@5d0b025a]
INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) - pending
log size: 1, pending hb size: 21
2020-07-21 17:12:52,415
[org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@427cb8b9]
INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) - pending
log size: 1, pending hb size: 22
2020-07-21 17:12:52,415
[org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@66633814]
INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) - pending
log size: 1, pending hb size: 22
2020-07-21 17:12:52,424
[org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@5d0b025a]
INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) - pending
log size: 1, pending hb size: 22
2020-07-21 17:12:52,424
[org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@7d13f127]
INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) - pending
log size: 1, pending hb size: 22
2020-07-21 17:12:52,493
[org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@427cb8b9]
INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) - pending
log size: 1, pending hb size: 23
2020-07-21 17:12:52,493
[org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@66633814]
INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) - pending
log size: 1, pending hb size: 23
2020-07-21 17:12:52,504
[org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@7d13f127]
INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) - pending
log size: 1, pending hb size: 23
2020-07-21 17:12:52,504
[org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@5d0b025a]
INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) - pending
log size: 1, pending hb size: 23
2020-07-21 17:12:52,572
[org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@427cb8b9]
INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) - pending
log size: 1, pending hb size: 24
2020-07-21 17:12:52,572
[org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@66633814]
INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) - pending
log size: 1, pending hb size: 24
2020-07-21 17:12:52,583
[org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@7d13f127]
INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) - pending
log size: 1, pending hb size: 24
2020-07-21 17:12:52,585
[org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@5d0b025a]
INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) - pending
log size: 1, pending hb size: 24
2020-07-21 17:12:52,650
[org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@427cb8b9]
INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) - pending
log size: 1, pending hb size: 25
2020-07-21 17:12:52,650
[org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@66633814]
INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) - pending
log size: 1, pending hb size: 25
2020-07-21 17:12:52,662
[org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@5d0b025a]
INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) - pending
log size: 1, pending hb size: 25
2020-07-21 17:12:52,662
[org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@7d13f127]
INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) - pending
log size: 1, pending hb size: 25
2020-07-21 17:12:52,728
[org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@427cb8b9]
INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) - pending
log size: 1, pending hb size: 26
2020-07-21 17:12:52,728
{code}
{code:java}
2020-07-21 17:12:55,015
[org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@7d13f127]
INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) - pending
log size: 1, pending hb size: 55
2020-07-21 17:12:55,065
[org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@66633814]
INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) - pending
log size: 1, pending hb size: 56
2020-07-21 17:12:55,070
[org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@427cb8b9]
INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) - pending
log size: 1, pending hb size: 56
2020-07-21 17:12:55,091
[org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@7d13f127]
INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) - pending
log size: 1, pending hb size: 56
2020-07-21 17:12:55,091
[org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@5d0b025a]
INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) - pending
log size: 1, pending hb size: 56
2020-07-21 17:12:55,145
[org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@66633814]
INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) - pending
log size: 1, pending hb size: 57
2020-07-21 17:12:55,147
[org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@427cb8b9]
INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) - pending
log size: 1, pending hb size: 57
2020-07-21 17:12:55,171
[org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@5d0b025a]
INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) - pending
log size: 1, pending hb size: 57
2020-07-21 17:12:55,172
[org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@7d13f127]
INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) - pending
log size: 1, pending hb size: 57
2020-07-21 17:12:55,222
[org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@66633814]
INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) - pending
log size: 1, pending hb size: 58
2020-07-21 17:12:55,226
[org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@427cb8b9]
INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) - pending
log size: 1, pending hb size: 58
2020-07-21 17:12:55,247
[org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@5d0b025a]
INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) - pending
log size: 1, pending hb size: 58
2020-07-21 17:12:55,252
[org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@7d13f127]
INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) - pending
log size: 1, pending hb size: 58
2020-07-21 17:12:55,299
[org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@66633814]
INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) - pending
log size: 1, pending hb size: 59
2020-07-21 17:12:55,306
[org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@427cb8b9]
INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) - pending
log size: 1, pending hb size: 59
2020-07-21 17:12:55,322
[org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@5d0b025a]
INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) - pending
log size: 1, pending hb size: 59
2020-07-21 17:12:55,327
[org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@7d13f127]
INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) - pending
log size: 1, pending hb size: 59
2020-07-21 17:12:55,379
[org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@66633814]
INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) - pending
log size: 1, pending hb size: 60
2020-07-21 17:12:55,386
[org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@427cb8b9]
INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) - pending
log size: 1, pending hb size: 60
2020-07-21 17:12:55,403
[org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@5d0b025a]
INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) - pending
log size: 1, pending hb size: 60
2020-07-21 17:12:55,404
[org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@7d13f127]
INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) - pending
log size: 1, pending hb size: 60
2020-07-21 17:12:55,458
[org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@66633814]
INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) - pending
log size: 1, pending hb size: 61
2020-07-21 17:12:55,461
[org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@427cb8b9]
INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) - pending
log size: 1, pending hb size: 61
2020-07-21 17:12:55,483
[org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@5d0b025a]
INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) - pending
log size: 1, pending hb size: 61
2020-07-21 17:12:55,484
[org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@7d13f127]
INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) - pending
log size: 1, pending hb size: 61
2020-07-21 17:12:55,536
[org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@66633814]
INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) - pending
log size: 1, pending hb size: 62
2020-07-21 17:12:55,538
[org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@427cb8b9]
INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) - pending
log size: 1, pending hb size: 62
2020-07-21 17:12:55,559
[org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@5d0b025a]
INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) - pending
log size: 1, pending hb size: 62
2020-07-21 17:12:55,563
[org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@7d13f127]
INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) - pending
log size: 1, pending hb size: 62
2020-07-21 17:12:55,614
[org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@66633814]
INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) - pending
log size: 1, pending hb size: 63
2020-07-21 17:12:55,617
[org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@427cb8b9]
INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) - pending
log size: 1, pending hb size: 63
2020-07-21 17:12:55,637
[org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@5d0b025a]
INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) - pending
log size: 1, pending hb size: 63
2020-07-21 17:12:55,640
[org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@7d13f127]
INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) - pending
log size: 1, pending hb size: 63
2020-07-21 17:12:55,641 [Time-limited test] INFO
statemachine.SimpleStateMachine4Testing
(SimpleStateMachine4Testing.java:unblock(142)) - unblock
WRITE_STATE_MACHINE_DATA
2020-07-21 17:12:55,641 [Time-limited test] INFO
statemachine.SimpleStateMachine4Testing
(SimpleStateMachine4Testing.java:unblock(142)) - unblock
WRITE_STATE_MACHINE_DATA
2020-07-21 17:12:55,641 [Time-limited test] INFO
statemachine.SimpleStateMachine4Testing
(SimpleStateMachine4Testing.java:unblock(142)) - unblock
WRITE_STATE_MACHINE_DATA
2020-07-21 17:12:55,642 [Time-limited test] INFO
statemachine.SimpleStateMachine4Testing
(SimpleStateMachine4Testing.java:unblock(142)) - unblock
WRITE_STATE_MACHINE_DATA
2020-07-21 17:12:55,642 [grpc-default-executor-2] INFO
server.GrpcServerProtocolService (GrpcServerProtocolService.java:onNext(109)) -
receive request, in order or not: false
2020-07-21 17:12:55,642 [grpc-default-executor-0] INFO
server.GrpcServerProtocolService (GrpcServerProtocolService.java:onNext(109)) -
receive request, in order or not: false
2020-07-21 17:12:55,642 [Timer-0] INFO ratis.MiniRaftCluster
(MiniRaftCluster.java:lambda$null$5(297)) - TIMED-PRINT: printing ALL groups
s3: RUNNING FOLLOWER s3@group-6CFE864C49B6:t1, leader=s1, voted=s1,
raftlog=s3@group-6CFE864C49B6-SegmentedRaftLog:OPENED:c0,f0,i1, conf=0:
[s3:0.0.0.0:62545, s4:0.0.0.0:62546, s0:0.0.0.0:62542, s1:0.0.0.0:62543,
s2:0.0.0.0:62544], old=null RUNNING
s4: RUNNING FOLLOWER s4@group-6CFE864C49B6:t1, leader=s1, voted=s1,
raftlog=s4@group-6CFE864C49B6-SegmentedRaftLog:OPENED:c0,f0,i1, conf=0:
[s3:0.0.0.0:62545, s4:0.0.0.0:62546, s0:0.0.0.0:62542, s1:0.0.0.0:62543,
s2:0.0.0.0:62544], old=null RUNNING
s0: RUNNING FOLLOWER s0@group-6CFE864C49B6:t1, leader=s1, voted=s1,
raftlog=s0@group-6CFE864C49B6-SegmentedRaftLog:OPENED:c0,f0,i1, conf=0:
[s3:0.0.0.0:62545, s4:0.0.0.0:62546, s0:0.0.0.0:62542, s1:0.0.0.0:62543,
s2:0.0.0.0:62544], old=null RUNNING
s1: RUNNING LEADER s1@group-6CFE864C49B6:t1, leader=s1, voted=s1,
raftlog=s1@group-6CFE864C49B6-SegmentedRaftLog:OPENED:c0,f1,i1, conf=0:
[s3:0.0.0.0:62545, s4:0.0.0.0:62546, s0:0.0.0.0:62542, s1:0.0.0.0:62543,
s2:0.0.0.0:62544], old=null RUNNING
s2: RUNNING FOLLOWER s2@group-6CFE864C49B6:t1, leader=s1, voted=s1,
raftlog=s2@group-6CFE864C49B6-SegmentedRaftLog:OPENED:c0,f0,i1, conf=0:
[s3:0.0.0.0:62545, s4:0.0.0.0:62546, s0:0.0.0.0:62542, s1:0.0.0.0:62543,
s2:0.0.0.0:62544], old=null RUNNING
2020-07-21 17:12:55,642 [grpc-default-executor-7] INFO
server.GrpcServerProtocolService (GrpcServerProtocolService.java:onNext(109)) -
receive request, in order or not: false
2020-07-21 17:12:55,642 [grpc-default-executor-6] INFO
server.GrpcServerProtocolService (GrpcServerProtocolService.java:onNext(109)) -
receive request, in order or not: false
2020-07-21 17:12:55,643 [grpc-default-executor-7] INFO impl.RaftServerImpl
(RaftServerImpl.java:logAppendEntries(936)) - HEARTBEAT: s4@group-6CFE864C49B6:
receive appendEntries(s1, 1, (t:1, i:1), 0, false, commits[s1:c0, s3:c0, s4:c0,
s0:c0, s2:c0], entries: []
2020-07-21 17:12:55,642 [ForkJoinPool.commonPool-worker-4] DEBUG
impl.RaftServerImpl (RaftServerImpl.java:logAppendEntries(938)) -
s3@group-6CFE864C49B6: succeeded to handle AppendEntries. Reply:
s1<-s3#70:OK,SUCCESS,nextIndex:2,term:1,followerCommit:0
2020-07-21 17:12:55,642 [ForkJoinPool.commonPool-worker-11] DEBUG
impl.RaftServerImpl (RaftServerImpl.java:logAppendEntries(938)) -
s2@group-6CFE864C49B6: succeeded to handle AppendEntries. Reply:
s1<-s2#70:OK,SUCCESS,nextIndex:2,term:1,followerCommit:0
2020-07-21 17:12:55,642 [ForkJoinPool.commonPool-worker-9] DEBUG
impl.RaftServerImpl (RaftServerImpl.java:logAppendEntries(938)) -
s0@group-6CFE864C49B6: succeeded to handle AppendEntries. Reply:
s1<-s0#70:OK,SUCCESS,nextIndex:2,term:1,followerCommit:0
2020-07-21 17:12:55,642 [ForkJoinPool.commonPool-worker-2] DEBUG
impl.RaftServerImpl (RaftServerImpl.java:logAppendEntries(938)) -
s4@group-6CFE864C49B6: succeeded to handle AppendEntries. Reply:
s1<-s4#70:OK,SUCCESS,nextIndex:2,term:1,followerCommit:0
2020-07-21 17:12:55,642 [grpc-default-executor-0] INFO impl.RaftServerImpl
(RaftServerImpl.java:logAppendEntries(936)) - HEARTBEAT: s2@group-6CFE864C49B6:
receive appendEntries(s1, 1, (t:1, i:1), 0, false, commits[s1:c0, s3:c0, s4:c0,
s0:c0, s2:c0], entries: []
2020-07-21 17:12:55,642 [grpc-default-executor-2] INFO impl.RaftServerImpl
(RaftServerImpl.java:logAppendEntries(936)) - HEARTBEAT: s0@group-6CFE864C49B6:
receive appendEntries(s1, 1, (t:1, i:1), 0, false, commits[s1:c0, s3:c0, s4:c0,
s0:c0, s2:c0], entries: []
2020-07-21 17:12:55,643 [ForkJoinPool.commonPool-worker-2] INFO
impl.RaftServerImpl (RaftServerImpl.java:logAppendEntries(936)) - HEARTBEAT:
s2@group-6CFE864C49B6: succeeded to handle AppendEntries. Reply:
s1<-s2#71:OK,SUCCESS,nextIndex:2,term:1,followerCommit:0
2020-07-21 17:12:55,643 [grpc-default-executor-1] DEBUG server.GrpcLogAppender
(GrpcLogAppender.java:onNext(259)) -
s1@group-6CFE864C49B6->s2-AppendLogResponseHandler: received a reply
s1<-s2#70:OK,SUCCESS,nextIndex:2,term:1,followerCommit:0,
request=AppendEntriesRequest:cid=70,entriesCount=1,lastEntry=(t:1, i:1)
2020-07-21 17:12:55,643 [grpc-default-executor-3] DEBUG server.GrpcLogAppender
(GrpcLogAppender.java:onNext(259)) -
s1@group-6CFE864C49B6->s3-AppendLogResponseHandler: received a reply
s1<-s3#70:OK,SUCCESS,nextIndex:2,term:1,followerCommit:0,
request=AppendEntriesRequest:cid=70,entriesCount=1,lastEntry=(t:1, i:1)
2020-07-21 17:12:55,643 [grpc-default-executor-7] INFO
server.GrpcServerProtocolService (GrpcServerProtocolService.java:onNext(109)) -
receive request, in order or not: false
{code}
was:
We should make sure that the stale leader steps down to the candidate state
before the next leader election.
Proposal:
In the heartbeat thread in the Leader node, we should check if the last
response time of the follower is less than the leader election timeout. If the
majority of the follower’s last response time is less than the leader election
timeout, the current leader is still the active leader. Majority of the
followers are heartbeating to the current leader, so there can’t be a new
leader.
If the majority of follower’s last response time is greater than the leader
election timeout, the current leader should step down and become a candidate.
With this check, we can be sure that the current leader will step down and
become a candidate before the new leader election starts in case of a network
partition.
> Heartbeats under grpc bi-direction stream does not work as expected.
> --------------------------------------------------------------------
>
> Key: RATIS-1008
> URL: https://issues.apache.org/jira/browse/RATIS-1008
> Project: Ratis
> Issue Type: Bug
> Reporter: Glen Geng
> Assignee: Glen Geng
> Priority: Major
>
> If raft server use Grpc, when append entries blocked at follower side (e.g.,
> by calling SimpleStateMachine4Testing::blockWriteStateMachineData), leader
> should still be able to send heartbeat (size of entries of append entries
> request be 0) to followers, and receive response of heartbeat from follower
> side, since we have the code
>
> {code:java}
> @Override
> public void onNext(REQUEST request) {
> if (!replyInOrder(request)) {
> try {
> process(request).thenAccept(this::handleReply);
> } catch (Throwable e) {
> handleError(e, request);
> }
> return;
> }
> final PendingServerRequest<REQUEST> current = new
> PendingServerRequest<>(request);
> final PendingServerRequest<REQUEST> previous =
> previousOnNext.getAndSet(current);
> final CompletableFuture<Void> previousFuture = Optional.ofNullable(previous)
> .map(PendingServerRequest::getFuture){code}
>
>
> But test case "testUpdateViaHeartbeat" shows that after calling
> SimpleStateMachine4Testing::blockWriteStateMachineData, leader do send
> heartbeats, but ServerRequestStreamObserver::onNext at
> GrpcServerProtocolService was not called, until follower finish handling the
> append entries request that blocked by blockWriteStateMachineData.
>
> *How to reproduce this issue ?*
> checkout to master, add following debug info
> {code:java}
> gengbin@GLENGENG-MB0 incubator-ratis % git diff
> diff --git
> a/ratis-grpc/src/main/java/org/apache/ratis/grpc/server/GrpcLogAppender.java
> b/ratis-grpc/src/main/java/org/apache/ratis/grpc/server/GrpcLogAppender.java
> index 4eb5cb9d..97c4679e 100644
> ---
> a/ratis-grpc/src/main/java/org/apache/ratis/grpc/server/GrpcLogAppender.java
> +++
> b/ratis-grpc/src/main/java/org/apache/ratis/grpc/server/GrpcLogAppender.java
> @@ -207,6 +207,8 @@ public class GrpcLogAppender extends LogAppender {
> getServer().getId(), null, proto);
> request.startRequestTimer();
> s.onNext(proto);
> + LOG.info("pending log size: {}, pending hb size: {}",
> + pendingRequests.logRequests.size(),
> pendingRequests.heartbeats.size());
> scheduler.onTimeout(requestTimeoutDuration,
> () -> timeoutAppendRequest(request.getCallId(),
> request.isHeartbeat()),
> LOG, () -> "Timeout check failed for append entry request: " +
> request);
> diff --git
> a/ratis-grpc/src/main/java/org/apache/ratis/grpc/server/GrpcServerProtocolService.java
>
> b/ratis-grpc/src/main/java/org/apache/ratis/grpc/server/GrpcServerProtocolService.java
> index 21a6c560..272d3c0b 100644
> ---
> a/ratis-grpc/src/main/java/org/apache/ratis/grpc/server/GrpcServerProtocolService.java
> +++
> b/ratis-grpc/src/main/java/org/apache/ratis/grpc/server/GrpcServerProtocolService.java
> @@ -106,6 +106,8 @@ class GrpcServerProtocolService extends
> RaftServerProtocolServiceImplBase {
>
> @Override
> public void onNext(REQUEST request) {
> + LOG.info("receive request, in order or not: {}",
> replyInOrder(request));
> +
> if (!replyInOrder(request)) {
> try {
> process(request).thenAccept(this::handleReply);
> diff --git
> a/ratis-server/src/main/java/org/apache/ratis/server/impl/RaftServerImpl.java
> b/ratis-server/src/main/java/org/apache/ratis/server/impl/RaftServerImpl.java
> index 3c4a3d88..2bb3d7e0 100644
> ---
> a/ratis-server/src/main/java/org/apache/ratis/server/impl/RaftServerImpl.java
> +++
> b/ratis-server/src/main/java/org/apache/ratis/server/impl/RaftServerImpl.java
> @@ -933,13 +933,9 @@ public class RaftServerImpl implements
> RaftServerProtocol, RaftServerAsynchronou
>
> static void logAppendEntries(boolean isHeartbeat, Supplier<String>
> message) {
> if (isHeartbeat) {
> - if (LOG.isTraceEnabled()) {
> - LOG.trace("HEARTBEAT: " + message.get());
> - }
> + LOG.info("HEARTBEAT: " + message.get());
> } else {
> - if (LOG.isDebugEnabled()) {
> LOG.debug(message.get());
> - }
> }
> }
>
> diff --git
> a/ratis-test/src/test/java/org/apache/ratis/grpc/TestRaftWithGrpc.java
> b/ratis-test/src/test/java/org/apache/ratis/grpc/TestRaftWithGrpc.java
> index 0f1d71db..00cd4833 100644
> --- a/ratis-test/src/test/java/org/apache/ratis/grpc/TestRaftWithGrpc.java
> +++ b/ratis-test/src/test/java/org/apache/ratis/grpc/TestRaftWithGrpc.java
> @@ -17,18 +17,23 @@
> */
> package org.apache.ratis.grpc;
>
> +import org.apache.log4j.Level;
> import org.apache.ratis.MiniRaftCluster;
> import org.apache.ratis.RaftBasicTests;
> import org.apache.ratis.RaftTestUtil;
> import org.apache.ratis.client.RaftClient;
> +import org.apache.ratis.grpc.server.GrpcLogAppender;
> import org.apache.ratis.protocol.RaftClientReply;
> +import org.apache.ratis.server.RaftServerConfigKeys;
> import org.apache.ratis.server.impl.BlockRequestHandlingInjection;
> +import org.apache.ratis.server.impl.RaftServerImpl;
> import org.apache.ratis.server.impl.RaftServerTestUtil;
> import org.apache.ratis.server.protocol.TermIndex;
> import org.apache.ratis.server.raftlog.RaftLog;
> import org.apache.ratis.statemachine.SimpleStateMachine4Testing;
> import org.apache.ratis.statemachine.StateMachine;
> import org.apache.ratis.util.JavaUtils;
> +import org.apache.ratis.util.Log4jUtils;
> import org.apache.ratis.util.TimeDuration;
> import org.junit.Assert;
> import org.junit.Test;
> @@ -47,6 +52,11 @@ public class TestRaftWithGrpc
> SimpleStateMachine4Testing.class, StateMachine.class);
> }
>
> + {
> + Log4jUtils.setLogLevel(RaftServerImpl.LOG, Level.DEBUG);
> + Log4jUtils.setLogLevel(GrpcLogAppender.LOG, Level.DEBUG);
> + }
> +
> @Override
> @Test
> public void testWithLoad() throws Exception {
> @@ -67,7 +77,12 @@ public class TestRaftWithGrpc
>
> @Test
> public void testUpdateViaHeartbeat() throws Exception {
> + final TimeDuration oldTimeoutMax =
> RaftServerConfigKeys.Rpc.timeoutMax(getProperties());
> + RaftServerConfigKeys.Rpc.setTimeoutMax(getProperties(),
> TimeDuration.valueOf(5, TimeUnit.SECONDS));
> +
> runWithNewCluster(NUM_SERVERS, this::runTestUpdateViaHeartbeat);
> +
> + RaftServerConfigKeys.Rpc.setTimeoutMax(getProperties(), oldTimeoutMax);
> }
>
> void runTestUpdateViaHeartbeat(MiniRaftClusterWithGrpc cluster) throws
> Exception {
> {code}
>
> *How to prove this ?*
> run test case testUpdateViaHeartbeat
>
> {code:java}
> 2020-07-21 17:12:50,603 [Time-limited test] INFO
> statemachine.SimpleStateMachine4Testing
> (SimpleStateMachine4Testing.java:block(135)) - block WRITE_STATE_MACHINE_DATA
> 2020-07-21 17:12:50,603 [Time-limited test] INFO
> statemachine.SimpleStateMachine4Testing
> (SimpleStateMachine4Testing.java:block(135)) - block WRITE_STATE_MACHINE_DATA
> 2020-07-21 17:12:50,603 [Time-limited test] INFO
> statemachine.SimpleStateMachine4Testing
> (SimpleStateMachine4Testing.java:block(135)) - block WRITE_STATE_MACHINE_DATA
> 2020-07-21 17:12:50,604 [Time-limited test] INFO
> statemachine.SimpleStateMachine4Testing
> (SimpleStateMachine4Testing.java:block(135)) - block WRITE_STATE_MACHINE_DATA
> 2020-07-21 17:12:50,627
> [org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@7d13f127]
> INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) -
> pending log size: 0, pending hb size: 1
> 2020-07-21 17:12:50,627
> [org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@5d0b025a]
> INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) -
> pending log size: 0, pending hb size: 1
> 2020-07-21 17:12:50,627
> [org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@427cb8b9]
> INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) -
> pending log size: 0, pending hb size: 1
> 2020-07-21 17:12:50,627
> [org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@66633814]
> INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) -
> pending log size: 0, pending hb size: 1
> 2020-07-21 17:12:50,627 [grpc-default-executor-6] INFO
> server.GrpcServerProtocolService (GrpcServerProtocolService.java:onNext(109))
> - receive request, in order or not: false
> 2020-07-21 17:12:50,627 [grpc-default-executor-2] INFO
> server.GrpcServerProtocolService (GrpcServerProtocolService.java:onNext(109))
> - receive request, in order or not: false
> 2020-07-21 17:12:50,627 [grpc-default-executor-0] INFO
> server.GrpcServerProtocolService (GrpcServerProtocolService.java:onNext(109))
> - receive request, in order or not: false
> 2020-07-21 17:12:50,627 [grpc-default-executor-7] INFO
> server.GrpcServerProtocolService (GrpcServerProtocolService.java:onNext(109))
> - receive request, in order or not: false
> 2020-07-21 17:12:50,628 [grpc-default-executor-0] INFO impl.RaftServerImpl
> (RaftServerImpl.java:logAppendEntries(936)) - HEARTBEAT:
> s4@group-6CFE864C49B6: receive appendEntries(s1, 1, (t:1, i:0), 0, false,
> commits[s1:c0, s3:c0, s4:c0, s0:c0, s2:c0], entries: []
> 2020-07-21 17:12:50,628 [grpc-default-executor-2] INFO impl.RaftServerImpl
> (RaftServerImpl.java:logAppendEntries(936)) - HEARTBEAT:
> s0@group-6CFE864C49B6: receive appendEntries(s1, 1, (t:1, i:0), 0, false,
> commits[s1:c0, s3:c0, s4:c0, s0:c0, s2:c0], entries: []
> 2020-07-21 17:12:50,628 [grpc-default-executor-6] INFO impl.RaftServerImpl
> (RaftServerImpl.java:logAppendEntries(936)) - HEARTBEAT:
> s3@group-6CFE864C49B6: receive appendEntries(s1, 1, (t:1, i:0), 0, false,
> commits[s1:c0, s3:c0, s4:c0, s0:c0, s2:c0], entries: []
> 2020-07-21 17:12:50,628 [ForkJoinPool.commonPool-worker-9] INFO
> impl.RaftServerImpl (RaftServerImpl.java:logAppendEntries(936)) - HEARTBEAT:
> s4@group-6CFE864C49B6: succeeded to handle AppendEntries. Reply:
> s1<-s4#69:OK,SUCCESS,nextIndex:1,term:1,followerCommit:0
> 2020-07-21 17:12:50,628 [grpc-default-executor-7] INFO impl.RaftServerImpl
> (RaftServerImpl.java:logAppendEntries(936)) - HEARTBEAT:
> s2@group-6CFE864C49B6: receive appendEntries(s1, 1, (t:1, i:0), 0, false,
> commits[s1:c0, s3:c0, s4:c0, s0:c0, s2:c0], entries: []
> 2020-07-21 17:12:50,628 [ForkJoinPool.commonPool-worker-11] INFO
> impl.RaftServerImpl (RaftServerImpl.java:logAppendEntries(936)) - HEARTBEAT:
> s3@group-6CFE864C49B6: succeeded to handle AppendEntries. Reply:
> s1<-s3#69:OK,SUCCESS,nextIndex:1,term:1,followerCommit:0
> 2020-07-21 17:12:50,628 [ForkJoinPool.commonPool-worker-4] INFO
> impl.RaftServerImpl (RaftServerImpl.java:logAppendEntries(936)) - HEARTBEAT:
> s0@group-6CFE864C49B6: succeeded to handle AppendEntries. Reply:
> s1<-s0#69:OK,SUCCESS,nextIndex:1,term:1,followerCommit:0
> 2020-07-21 17:12:50,628 [ForkJoinPool.commonPool-worker-9] INFO
> impl.RaftServerImpl (RaftServerImpl.java:logAppendEntries(936)) - HEARTBEAT:
> s2@group-6CFE864C49B6: succeeded to handle AppendEntries. Reply:
> s1<-s2#69:OK,SUCCESS,nextIndex:1,term:1,followerCommit:0
> 2020-07-21 17:12:50,629 [grpc-default-executor-7] DEBUG
> server.GrpcLogAppender (GrpcLogAppender.java:onNext(259)) -
> s1@group-6CFE864C49B6->s4-AppendLogResponseHandler: received a reply
> s1<-s4#69:OK,SUCCESS,nextIndex:1,term:1,followerCommit:0,
> request=AppendEntriesRequest:cid=69,entriesCount=0,lastEntry=null
> 2020-07-21 17:12:50,629 [grpc-default-executor-6] DEBUG
> server.GrpcLogAppender (GrpcLogAppender.java:onNext(259)) -
> s1@group-6CFE864C49B6->s3-AppendLogResponseHandler: received a reply
> s1<-s3#69:OK,SUCCESS,nextIndex:1,term:1,followerCommit:0,
> request=AppendEntriesRequest:cid=69,entriesCount=0,lastEntry=null
> 2020-07-21 17:12:50,629 [grpc-default-executor-2] DEBUG
> server.GrpcLogAppender (GrpcLogAppender.java:onNext(259)) -
> s1@group-6CFE864C49B6->s0-AppendLogResponseHandler: received a reply
> s1<-s0#69:OK,SUCCESS,nextIndex:1,term:1,followerCommit:0,
> request=AppendEntriesRequest:cid=69,entriesCount=0,lastEntry=null
> 2020-07-21 17:12:50,629 [grpc-default-executor-6] DEBUG
> server.GrpcLogAppender (GrpcLogAppender.java:onNext(259)) -
> s1@group-6CFE864C49B6->s2-AppendLogResponseHandler: received a reply
> s1<-s2#69:OK,SUCCESS,nextIndex:1,term:1,followerCommit:0,
> request=AppendEntriesRequest:cid=69,entriesCount=0,lastEntry=null
> 2020-07-21 17:12:50,643 [grpc-default-executor-6] DEBUG impl.RaftServerImpl
> (RaftServerImpl.java:submitClientRequestAsync(587)) - s3@group-6CFE864C49B6:
> receive client
> request(RaftClientRequest:client-DE8B7308F86F->s3@group-6CFE864C49B6, cid=0,
> seq=1*, Watch(0), Message:<EMPTY>)
> 2020-07-21 17:12:50,668 [grpc-default-executor-6] DEBUG impl.RaftServerImpl
> (RaftServerImpl.java:submitClientRequestAsync(587)) - s1@group-6CFE864C49B6:
> receive client
> request(RaftClientRequest:client-DE8B7308F86F->s1@group-6CFE864C49B6, cid=0,
> seq=1*, Watch(0), Message:<EMPTY>)
> 2020-07-21 17:12:50,669 [grpc-default-executor-6] DEBUG impl.RaftServerImpl
> (LeaderState.java:addWatchReqeust(358)) - s1@group-6CFE864C49B6-LeaderState:
> addWatchRequest RaftClientRequest:client-DE8B7308F86F->s1@group-6CFE864C49B6,
> cid=0, seq=1*, Watch(0), Message:<EMPTY>
> 2020-07-21 17:12:50,674 [grpc-default-executor-2] DEBUG impl.RaftServerImpl
> (RaftServerImpl.java:submitClientRequestAsync(587)) - s1@group-6CFE864C49B6:
> receive client
> request(RaftClientRequest:client-DE8B7308F86F->s1@group-6CFE864C49B6, cid=1,
> seq=2, RW, Message:616263)
> 2020-07-21 17:12:50,685 [grpc-default-executor-2] DEBUG impl.RaftServerImpl
> (LeaderState.java:addPendingRequest(340)) -
> s1@group-6CFE864C49B6-LeaderState: addPendingRequest at
> RaftClientRequest:client-DE8B7308F86F->s1@group-6CFE864C49B6, cid=1, seq=2,
> RW, Message:616263, entry=(t:1, i:1), STATEMACHINELOGENTRY,
> client-DE8B7308F86F, cid=1
> 2020-07-21 17:12:50,686
> [org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@5d0b025a]
> INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) -
> pending log size: 1, pending hb size: 0
> 2020-07-21 17:12:50,686
> [org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@66633814]
> INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) -
> pending log size: 1, pending hb size: 0
> 2020-07-21 17:12:50,686
> [org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@427cb8b9]
> INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) -
> pending log size: 1, pending hb size: 0
> 2020-07-21 17:12:50,686
> [org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@7d13f127]
> INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) -
> pending log size: 1, pending hb size: 0
> 2020-07-21 17:12:50,687 [grpc-default-executor-6] INFO
> server.GrpcServerProtocolService (GrpcServerProtocolService.java:onNext(109))
> - receive request, in order or not: true
> 2020-07-21 17:12:50,687 [grpc-default-executor-0] INFO
> server.GrpcServerProtocolService (GrpcServerProtocolService.java:onNext(109))
> - receive request, in order or not: true
> 2020-07-21 17:12:50,687 [grpc-default-executor-2] INFO
> server.GrpcServerProtocolService (GrpcServerProtocolService.java:onNext(109))
> - receive request, in order or not: true
> 2020-07-21 17:12:50,687 [grpc-default-executor-0] DEBUG impl.RaftServerImpl
> (RaftServerImpl.java:logAppendEntries(938)) - s2@group-6CFE864C49B6: receive
> appendEntries(s1, 1, (t:1, i:0), 0, false, commits[s1:c0, s3:c0, s4:c0,
> s0:c0, s2:c0], entries: (t:1, i:1), STATEMACHINELOGENTRY,
> client-DE8B7308F86F, cid=1
> 2020-07-21 17:12:50,687 [grpc-default-executor-2] DEBUG impl.RaftServerImpl
> (RaftServerImpl.java:logAppendEntries(938)) - s3@group-6CFE864C49B6: receive
> appendEntries(s1, 1, (t:1, i:0), 0, false, commits[s1:c0, s3:c0, s4:c0,
> s0:c0, s2:c0], entries: (t:1, i:1), STATEMACHINELOGENTRY,
> client-DE8B7308F86F, cid=1
> 2020-07-21 17:12:50,687 [grpc-default-executor-7] INFO
> server.GrpcServerProtocolService (GrpcServerProtocolService.java:onNext(109))
> - receive request, in order or not: true
> 2020-07-21 17:12:50,687 [grpc-default-executor-6] DEBUG impl.RaftServerImpl
> (RaftServerImpl.java:logAppendEntries(938)) - s0@group-6CFE864C49B6: receive
> appendEntries(s1, 1, (t:1, i:0), 0, false, commits[s1:c0, s3:c0, s4:c0,
> s0:c0, s2:c0], entries: (t:1, i:1), STATEMACHINELOGENTRY,
> client-DE8B7308F86F, cid=1
> 2020-07-21 17:12:50,688 [grpc-default-executor-7] DEBUG impl.RaftServerImpl
> (RaftServerImpl.java:logAppendEntries(938)) - s4@group-6CFE864C49B6: receive
> appendEntries(s1, 1, (t:1, i:0), 0, false, commits[s1:c0, s3:c0, s4:c0,
> s0:c0, s2:c0], entries: (t:1, i:1), STATEMACHINELOGENTRY,
> client-DE8B7308F86F, cid=1
> 2020-07-21 17:12:50,765
> [org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@66633814]
> INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) -
> pending log size: 1, pending hb size: 1
> 2020-07-21 17:12:50,765
> [org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@7d13f127]
> INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) -
> pending log size: 1, pending hb size: 1
> 2020-07-21 17:12:50,765
> [org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@5d0b025a]
> INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) -
> pending log size: 1, pending hb size: 1
> 2020-07-21 17:12:50,765
> [org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@427cb8b9]
> INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) -
> pending log size: 1, pending hb size: 1
> 2020-07-21 17:12:50,843
> [org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@66633814]
> INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) -
> pending log size: 1, pending hb size: 2
> 2020-07-21 17:12:50,843
> [org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@5d0b025a]
> INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) -
> pending log size: 1, pending hb size: 2
> 2020-07-21 17:12:50,843
> [org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@427cb8b9]
> INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) -
> pending log size: 1, pending hb size: 2
> 2020-07-21 17:12:50,843
> [org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@7d13f127]
> INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) -
> pending log size: 1, pending hb size: 2
> 2020-07-21 17:12:50,922
> [org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@66633814]
> INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) -
> pending log size: 1, pending hb size: 3
> 2020-07-21 17:12:50,922
> [org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@427cb8b9]
> INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) -
> pending log size: 1, pending hb size: 3
> 2020-07-21 17:12:50,922
> [org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@5d0b025a]
> INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) -
> pending log size: 1, pending hb size: 3
> 2020-07-21 17:12:50,922
> [org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@7d13f127]
> INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) -
> pending log size: 1, pending hb size: 3
> 2020-07-21 17:12:51,003
> [org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@5d0b025a]
> INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) -
> pending log size: 1, pending hb size: 4
> 2020-07-21 17:12:51,003
> [org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@66633814]
> INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) -
> pending log size: 1, pending hb size: 4
> 2020-07-21 17:12:51,003
> [org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@7d13f127]
> INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) -
> pending log size: 1, pending hb size: 4
> 2020-07-21 17:12:51,003
> [org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@427cb8b9]
> INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) -
> pending log size: 1, pending hb size: 4
> 2020-07-21 17:12:51,083
> [org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@427cb8b9]
> INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) -
> pending log size: 1, pending hb size: 5
> 2020-07-21 17:12:51,083
> [org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@5d0b025a]
> INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) -
> pending log size: 1, pending hb size: 5
> 2020-07-21 17:12:51,083
> [org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@66633814]
> INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) -
> pending log size: 1, pending hb size: 5
> 2020-07-21 17:12:51,083
> [org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@7d13f127]
> INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) -
> pending log size: 1, pending hb size: 5
> 2020-07-21 17:12:51,159
> [org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@7d13f127]
> INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) -
> pending log size: 1, pending hb size: 6
> 2020-07-21 17:12:51,159
> [org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@427cb8b9]
> INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) -
> pending log size: 1, pending hb size: 6
> 2020-07-21 17:12:51,159
> [org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@5d0b025a]
> INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) -
> pending log size: 1, pending hb size: 6
> 2020-07-21 17:12:51,159
> [org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@66633814]
> INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) -
> pending log size: 1, pending hb size: 6
> 2020-07-21 17:12:51,236
> [org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@5d0b025a]
> INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) -
> pending log size: 1, pending hb size: 7
> 2020-07-21 17:12:51,236
> [org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@7d13f127]
> INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) -
> pending log size: 1, pending hb size: 7
> 2020-07-21 17:12:51,236
> [org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@427cb8b9]
> INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) -
> pending log size: 1, pending hb size: 7
> 2020-07-21 17:12:51,236
> [org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@66633814]
> INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) -
> pending log size: 1, pending hb size: 7
> 2020-07-21 17:12:51,317
> [org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@427cb8b9]
> INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) -
> pending log size: 1, pending hb size: 8
> 2020-07-21 17:12:51,317
> [org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@5d0b025a]
> INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) -
> pending log size: 1, pending hb size: 8
> 2020-07-21 17:12:51,317
> [org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@66633814]
> INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) -
> pending log size: 1, pending hb size: 8
> 2020-07-21 17:12:51,317
> [org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@7d13f127]
> INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) -
> pending log size: 1, pending hb size: 8
> 2020-07-21 17:12:51,397
> [org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@66633814]
> INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) -
> pending log size: 1, pending hb size: 9
> 2020-07-21 17:12:51,397
> [org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@427cb8b9]
> INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) -
> pending log size: 1, pending hb size: 9
> 2020-07-21 17:12:51,397
> [org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@7d13f127]
> INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) -
> pending log size: 1, pending hb size: 9
> 2020-07-21 17:12:51,397
> [org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@5d0b025a]
> INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) -
> pending log size: 1, pending hb size: 9
> 2020-07-21 17:12:51,477
> [org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@5d0b025a]
> INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) -
> pending log size: 1, pending hb size: 10
> 2020-07-21 17:12:51,477
> [org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@66633814]
> INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) -
> pending log size: 1, pending hb size: 10
> 2020-07-21 17:12:51,477
> [org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@427cb8b9]
> INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) -
> pending log size: 1, pending hb size: 10
> 2020-07-21 17:12:51,477
> [org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@7d13f127]
> INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) -
> pending log size: 1, pending hb size: 10
> 2020-07-21 17:12:51,554
> [org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@5d0b025a]
> INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) -
> pending log size: 1, pending hb size: 11
> 2020-07-21 17:12:51,554
> [org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@427cb8b9]
> INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) -
> pending log size: 1, pending hb size: 11
> 2020-07-21 17:12:51,554
> [org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@66633814]
> INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) -
> pending log size: 1, pending hb size: 11
> 2020-07-21 17:12:51,554
> [org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@7d13f127]
> INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) -
> pending log size: 1, pending hb size: 11
> 2020-07-21 17:12:51,634
> [org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@7d13f127]
> INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) -
> pending log size: 1, pending hb size: 12
> 2020-07-21 17:12:51,634
> [org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@427cb8b9]
> INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) -
> pending log size: 1, pending hb size: 12
> 2020-07-21 17:12:51,634
> [org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@66633814]
> INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) -
> pending log size: 1, pending hb size: 12
> 2020-07-21 17:12:51,634
> [org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@5d0b025a]
> INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) -
> pending log size: 1, pending hb size: 12
> 2020-07-21 17:12:51,714
> [org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@427cb8b9]
> INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) -
> pending log size: 1, pending hb size: 13
> 2020-07-21 17:12:51,714
> [org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@5d0b025a]
> INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) -
> pending log size: 1, pending hb size: 13
> 2020-07-21 17:12:51,714
> [org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@7d13f127]
> INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) -
> pending log size: 1, pending hb size: 13
> 2020-07-21 17:12:51,714
> [org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@66633814]
> INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) -
> pending log size: 1, pending hb size: 13
> 2020-07-21 17:12:51,793
> [org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@5d0b025a]
> INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) -
> pending log size: 1, pending hb size: 14
> 2020-07-21 17:12:51,793
> [org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@427cb8b9]
> INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) -
> pending log size: 1, pending hb size: 14
> 2020-07-21 17:12:51,794
> [org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@66633814]
> INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) -
> pending log size: 1, pending hb size: 14
> 2020-07-21 17:12:51,794
> [org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@7d13f127]
> INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) -
> pending log size: 1, pending hb size: 14
> 2020-07-21 17:12:51,871
> [org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@7d13f127]
> INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) -
> pending log size: 1, pending hb size: 15
> 2020-07-21 17:12:51,871
> [org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@5d0b025a]
> INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) -
> pending log size: 1, pending hb size: 15
> 2020-07-21 17:12:51,871
> [org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@66633814]
> INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) -
> pending log size: 1, pending hb size: 15
> 2020-07-21 17:12:51,871
> [org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@427cb8b9]
> INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) -
> pending log size: 1, pending hb size: 15
> 2020-07-21 17:12:51,946
> [org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@7d13f127]
> INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) -
> pending log size: 1, pending hb size: 16
> 2020-07-21 17:12:51,946
> [org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@5d0b025a]
> INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) -
> pending log size: 1, pending hb size: 16
> 2020-07-21 17:12:51,946
> [org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@427cb8b9]
> INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) -
> pending log size: 1, pending hb size: 16
> 2020-07-21 17:12:51,946
> [org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@66633814]
> INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) -
> pending log size: 1, pending hb size: 16
> 2020-07-21 17:12:52,026
> [org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@66633814]
> INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) -
> pending log size: 1, pending hb size: 17
> 2020-07-21 17:12:52,026
> [org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@427cb8b9]
> INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) -
> pending log size: 1, pending hb size: 17
> 2020-07-21 17:12:52,026
> [org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@7d13f127]
> INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) -
> pending log size: 1, pending hb size: 17
> 2020-07-21 17:12:52,026
> [org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@5d0b025a]
> INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) -
> pending log size: 1, pending hb size: 17
> 2020-07-21 17:12:52,104
> [org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@427cb8b9]
> INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) -
> pending log size: 1, pending hb size: 18
> 2020-07-21 17:12:52,104
> [org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@66633814]
> INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) -
> pending log size: 1, pending hb size: 18
> 2020-07-21 17:12:52,104
> [org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@7d13f127]
> INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) -
> pending log size: 1, pending hb size: 18
> 2020-07-21 17:12:52,104
> [org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@5d0b025a]
> INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) -
> pending log size: 1, pending hb size: 18
> 2020-07-21 17:12:52,183
> [org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@427cb8b9]
> INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) -
> pending log size: 1, pending hb size: 19
> 2020-07-21 17:12:52,183
> [org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@7d13f127]
> INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) -
> pending log size: 1, pending hb size: 19
> 2020-07-21 17:12:52,183
> [org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@5d0b025a]
> INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) -
> pending log size: 1, pending hb size: 19
> 2020-07-21 17:12:52,183
> [org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@66633814]
> INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) -
> pending log size: 1, pending hb size: 19
> 2020-07-21 17:12:52,263
> [org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@66633814]
> INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) -
> pending log size: 1, pending hb size: 20
> 2020-07-21 17:12:52,263
> [org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@427cb8b9]
> INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) -
> pending log size: 1, pending hb size: 20
> 2020-07-21 17:12:52,263
> [org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@5d0b025a]
> INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) -
> pending log size: 1, pending hb size: 20
> 2020-07-21 17:12:52,263
> [org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@7d13f127]
> INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) -
> pending log size: 1, pending hb size: 20
> 2020-07-21 17:12:52,339
> [org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@66633814]
> INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) -
> pending log size: 1, pending hb size: 21
> 2020-07-21 17:12:52,339
> [org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@427cb8b9]
> INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) -
> pending log size: 1, pending hb size: 21
> 2020-07-21 17:12:52,344
> [org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@7d13f127]
> INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) -
> pending log size: 1, pending hb size: 21
> 2020-07-21 17:12:52,344
> [org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@5d0b025a]
> INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) -
> pending log size: 1, pending hb size: 21
> 2020-07-21 17:12:52,415
> [org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@427cb8b9]
> INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) -
> pending log size: 1, pending hb size: 22
> 2020-07-21 17:12:52,415
> [org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@66633814]
> INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) -
> pending log size: 1, pending hb size: 22
> 2020-07-21 17:12:52,424
> [org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@5d0b025a]
> INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) -
> pending log size: 1, pending hb size: 22
> 2020-07-21 17:12:52,424
> [org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@7d13f127]
> INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) -
> pending log size: 1, pending hb size: 22
> 2020-07-21 17:12:52,493
> [org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@427cb8b9]
> INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) -
> pending log size: 1, pending hb size: 23
> 2020-07-21 17:12:52,493
> [org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@66633814]
> INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) -
> pending log size: 1, pending hb size: 23
> 2020-07-21 17:12:52,504
> [org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@7d13f127]
> INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) -
> pending log size: 1, pending hb size: 23
> 2020-07-21 17:12:52,504
> [org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@5d0b025a]
> INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) -
> pending log size: 1, pending hb size: 23
> 2020-07-21 17:12:52,572
> [org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@427cb8b9]
> INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) -
> pending log size: 1, pending hb size: 24
> 2020-07-21 17:12:52,572
> [org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@66633814]
> INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) -
> pending log size: 1, pending hb size: 24
> 2020-07-21 17:12:52,583
> [org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@7d13f127]
> INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) -
> pending log size: 1, pending hb size: 24
> 2020-07-21 17:12:52,585
> [org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@5d0b025a]
> INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) -
> pending log size: 1, pending hb size: 24
> 2020-07-21 17:12:52,650
> [org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@427cb8b9]
> INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) -
> pending log size: 1, pending hb size: 25
> 2020-07-21 17:12:52,650
> [org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@66633814]
> INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) -
> pending log size: 1, pending hb size: 25
> 2020-07-21 17:12:52,662
> [org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@5d0b025a]
> INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) -
> pending log size: 1, pending hb size: 25
> 2020-07-21 17:12:52,662
> [org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@7d13f127]
> INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) -
> pending log size: 1, pending hb size: 25
> 2020-07-21 17:12:52,728
> [org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@427cb8b9]
> INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) -
> pending log size: 1, pending hb size: 26
> 2020-07-21 17:12:52,728
> {code}
> {code:java}
> 2020-07-21 17:12:55,015
> [org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@7d13f127]
> INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) -
> pending log size: 1, pending hb size: 55
> 2020-07-21 17:12:55,065
> [org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@66633814]
> INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) -
> pending log size: 1, pending hb size: 56
> 2020-07-21 17:12:55,070
> [org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@427cb8b9]
> INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) -
> pending log size: 1, pending hb size: 56
> 2020-07-21 17:12:55,091
> [org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@7d13f127]
> INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) -
> pending log size: 1, pending hb size: 56
> 2020-07-21 17:12:55,091
> [org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@5d0b025a]
> INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) -
> pending log size: 1, pending hb size: 56
> 2020-07-21 17:12:55,145
> [org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@66633814]
> INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) -
> pending log size: 1, pending hb size: 57
> 2020-07-21 17:12:55,147
> [org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@427cb8b9]
> INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) -
> pending log size: 1, pending hb size: 57
> 2020-07-21 17:12:55,171
> [org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@5d0b025a]
> INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) -
> pending log size: 1, pending hb size: 57
> 2020-07-21 17:12:55,172
> [org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@7d13f127]
> INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) -
> pending log size: 1, pending hb size: 57
> 2020-07-21 17:12:55,222
> [org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@66633814]
> INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) -
> pending log size: 1, pending hb size: 58
> 2020-07-21 17:12:55,226
> [org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@427cb8b9]
> INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) -
> pending log size: 1, pending hb size: 58
> 2020-07-21 17:12:55,247
> [org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@5d0b025a]
> INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) -
> pending log size: 1, pending hb size: 58
> 2020-07-21 17:12:55,252
> [org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@7d13f127]
> INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) -
> pending log size: 1, pending hb size: 58
> 2020-07-21 17:12:55,299
> [org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@66633814]
> INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) -
> pending log size: 1, pending hb size: 59
> 2020-07-21 17:12:55,306
> [org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@427cb8b9]
> INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) -
> pending log size: 1, pending hb size: 59
> 2020-07-21 17:12:55,322
> [org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@5d0b025a]
> INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) -
> pending log size: 1, pending hb size: 59
> 2020-07-21 17:12:55,327
> [org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@7d13f127]
> INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) -
> pending log size: 1, pending hb size: 59
> 2020-07-21 17:12:55,379
> [org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@66633814]
> INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) -
> pending log size: 1, pending hb size: 60
> 2020-07-21 17:12:55,386
> [org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@427cb8b9]
> INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) -
> pending log size: 1, pending hb size: 60
> 2020-07-21 17:12:55,403
> [org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@5d0b025a]
> INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) -
> pending log size: 1, pending hb size: 60
> 2020-07-21 17:12:55,404
> [org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@7d13f127]
> INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) -
> pending log size: 1, pending hb size: 60
> 2020-07-21 17:12:55,458
> [org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@66633814]
> INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) -
> pending log size: 1, pending hb size: 61
> 2020-07-21 17:12:55,461
> [org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@427cb8b9]
> INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) -
> pending log size: 1, pending hb size: 61
> 2020-07-21 17:12:55,483
> [org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@5d0b025a]
> INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) -
> pending log size: 1, pending hb size: 61
> 2020-07-21 17:12:55,484
> [org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@7d13f127]
> INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) -
> pending log size: 1, pending hb size: 61
> 2020-07-21 17:12:55,536
> [org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@66633814]
> INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) -
> pending log size: 1, pending hb size: 62
> 2020-07-21 17:12:55,538
> [org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@427cb8b9]
> INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) -
> pending log size: 1, pending hb size: 62
> 2020-07-21 17:12:55,559
> [org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@5d0b025a]
> INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) -
> pending log size: 1, pending hb size: 62
> 2020-07-21 17:12:55,563
> [org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@7d13f127]
> INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) -
> pending log size: 1, pending hb size: 62
> 2020-07-21 17:12:55,614
> [org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@66633814]
> INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) -
> pending log size: 1, pending hb size: 63
> 2020-07-21 17:12:55,617
> [org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@427cb8b9]
> INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) -
> pending log size: 1, pending hb size: 63
> 2020-07-21 17:12:55,637
> [org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@5d0b025a]
> INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) -
> pending log size: 1, pending hb size: 63
> 2020-07-21 17:12:55,640
> [org.apache.ratis.server.impl.LogAppender$AppenderDaemon$$Lambda$532/387080925@7d13f127]
> INFO server.GrpcLogAppender (GrpcLogAppender.java:sendRequest(210)) -
> pending log size: 1, pending hb size: 63
> 2020-07-21 17:12:55,641 [Time-limited test] INFO
> statemachine.SimpleStateMachine4Testing
> (SimpleStateMachine4Testing.java:unblock(142)) - unblock
> WRITE_STATE_MACHINE_DATA
> 2020-07-21 17:12:55,641 [Time-limited test] INFO
> statemachine.SimpleStateMachine4Testing
> (SimpleStateMachine4Testing.java:unblock(142)) - unblock
> WRITE_STATE_MACHINE_DATA
> 2020-07-21 17:12:55,641 [Time-limited test] INFO
> statemachine.SimpleStateMachine4Testing
> (SimpleStateMachine4Testing.java:unblock(142)) - unblock
> WRITE_STATE_MACHINE_DATA
> 2020-07-21 17:12:55,642 [Time-limited test] INFO
> statemachine.SimpleStateMachine4Testing
> (SimpleStateMachine4Testing.java:unblock(142)) - unblock
> WRITE_STATE_MACHINE_DATA
> 2020-07-21 17:12:55,642 [grpc-default-executor-2] INFO
> server.GrpcServerProtocolService (GrpcServerProtocolService.java:onNext(109))
> - receive request, in order or not: false
> 2020-07-21 17:12:55,642 [grpc-default-executor-0] INFO
> server.GrpcServerProtocolService (GrpcServerProtocolService.java:onNext(109))
> - receive request, in order or not: false
> 2020-07-21 17:12:55,642 [Timer-0] INFO ratis.MiniRaftCluster
> (MiniRaftCluster.java:lambda$null$5(297)) - TIMED-PRINT: printing ALL groups
> s3: RUNNING FOLLOWER s3@group-6CFE864C49B6:t1, leader=s1, voted=s1,
> raftlog=s3@group-6CFE864C49B6-SegmentedRaftLog:OPENED:c0,f0,i1, conf=0:
> [s3:0.0.0.0:62545, s4:0.0.0.0:62546, s0:0.0.0.0:62542, s1:0.0.0.0:62543,
> s2:0.0.0.0:62544], old=null RUNNING
> s4: RUNNING FOLLOWER s4@group-6CFE864C49B6:t1, leader=s1, voted=s1,
> raftlog=s4@group-6CFE864C49B6-SegmentedRaftLog:OPENED:c0,f0,i1, conf=0:
> [s3:0.0.0.0:62545, s4:0.0.0.0:62546, s0:0.0.0.0:62542, s1:0.0.0.0:62543,
> s2:0.0.0.0:62544], old=null RUNNING
> s0: RUNNING FOLLOWER s0@group-6CFE864C49B6:t1, leader=s1, voted=s1,
> raftlog=s0@group-6CFE864C49B6-SegmentedRaftLog:OPENED:c0,f0,i1, conf=0:
> [s3:0.0.0.0:62545, s4:0.0.0.0:62546, s0:0.0.0.0:62542, s1:0.0.0.0:62543,
> s2:0.0.0.0:62544], old=null RUNNING
> s1: RUNNING LEADER s1@group-6CFE864C49B6:t1, leader=s1, voted=s1,
> raftlog=s1@group-6CFE864C49B6-SegmentedRaftLog:OPENED:c0,f1,i1, conf=0:
> [s3:0.0.0.0:62545, s4:0.0.0.0:62546, s0:0.0.0.0:62542, s1:0.0.0.0:62543,
> s2:0.0.0.0:62544], old=null RUNNING
> s2: RUNNING FOLLOWER s2@group-6CFE864C49B6:t1, leader=s1, voted=s1,
> raftlog=s2@group-6CFE864C49B6-SegmentedRaftLog:OPENED:c0,f0,i1, conf=0:
> [s3:0.0.0.0:62545, s4:0.0.0.0:62546, s0:0.0.0.0:62542, s1:0.0.0.0:62543,
> s2:0.0.0.0:62544], old=null RUNNING
> 2020-07-21 17:12:55,642 [grpc-default-executor-7] INFO
> server.GrpcServerProtocolService (GrpcServerProtocolService.java:onNext(109))
> - receive request, in order or not: false
> 2020-07-21 17:12:55,642 [grpc-default-executor-6] INFO
> server.GrpcServerProtocolService (GrpcServerProtocolService.java:onNext(109))
> - receive request, in order or not: false
> 2020-07-21 17:12:55,643 [grpc-default-executor-7] INFO impl.RaftServerImpl
> (RaftServerImpl.java:logAppendEntries(936)) - HEARTBEAT:
> s4@group-6CFE864C49B6: receive appendEntries(s1, 1, (t:1, i:1), 0, false,
> commits[s1:c0, s3:c0, s4:c0, s0:c0, s2:c0], entries: []
> 2020-07-21 17:12:55,642 [ForkJoinPool.commonPool-worker-4] DEBUG
> impl.RaftServerImpl (RaftServerImpl.java:logAppendEntries(938)) -
> s3@group-6CFE864C49B6: succeeded to handle AppendEntries. Reply:
> s1<-s3#70:OK,SUCCESS,nextIndex:2,term:1,followerCommit:0
> 2020-07-21 17:12:55,642 [ForkJoinPool.commonPool-worker-11] DEBUG
> impl.RaftServerImpl (RaftServerImpl.java:logAppendEntries(938)) -
> s2@group-6CFE864C49B6: succeeded to handle AppendEntries. Reply:
> s1<-s2#70:OK,SUCCESS,nextIndex:2,term:1,followerCommit:0
> 2020-07-21 17:12:55,642 [ForkJoinPool.commonPool-worker-9] DEBUG
> impl.RaftServerImpl (RaftServerImpl.java:logAppendEntries(938)) -
> s0@group-6CFE864C49B6: succeeded to handle AppendEntries. Reply:
> s1<-s0#70:OK,SUCCESS,nextIndex:2,term:1,followerCommit:0
> 2020-07-21 17:12:55,642 [ForkJoinPool.commonPool-worker-2] DEBUG
> impl.RaftServerImpl (RaftServerImpl.java:logAppendEntries(938)) -
> s4@group-6CFE864C49B6: succeeded to handle AppendEntries. Reply:
> s1<-s4#70:OK,SUCCESS,nextIndex:2,term:1,followerCommit:0
> 2020-07-21 17:12:55,642 [grpc-default-executor-0] INFO impl.RaftServerImpl
> (RaftServerImpl.java:logAppendEntries(936)) - HEARTBEAT:
> s2@group-6CFE864C49B6: receive appendEntries(s1, 1, (t:1, i:1), 0, false,
> commits[s1:c0, s3:c0, s4:c0, s0:c0, s2:c0], entries: []
> 2020-07-21 17:12:55,642 [grpc-default-executor-2] INFO impl.RaftServerImpl
> (RaftServerImpl.java:logAppendEntries(936)) - HEARTBEAT:
> s0@group-6CFE864C49B6: receive appendEntries(s1, 1, (t:1, i:1), 0, false,
> commits[s1:c0, s3:c0, s4:c0, s0:c0, s2:c0], entries: []
> 2020-07-21 17:12:55,643 [ForkJoinPool.commonPool-worker-2] INFO
> impl.RaftServerImpl (RaftServerImpl.java:logAppendEntries(936)) - HEARTBEAT:
> s2@group-6CFE864C49B6: succeeded to handle AppendEntries. Reply:
> s1<-s2#71:OK,SUCCESS,nextIndex:2,term:1,followerCommit:0
> 2020-07-21 17:12:55,643 [grpc-default-executor-1] DEBUG
> server.GrpcLogAppender (GrpcLogAppender.java:onNext(259)) -
> s1@group-6CFE864C49B6->s2-AppendLogResponseHandler: received a reply
> s1<-s2#70:OK,SUCCESS,nextIndex:2,term:1,followerCommit:0,
> request=AppendEntriesRequest:cid=70,entriesCount=1,lastEntry=(t:1, i:1)
> 2020-07-21 17:12:55,643 [grpc-default-executor-3] DEBUG
> server.GrpcLogAppender (GrpcLogAppender.java:onNext(259)) -
> s1@group-6CFE864C49B6->s3-AppendLogResponseHandler: received a reply
> s1<-s3#70:OK,SUCCESS,nextIndex:2,term:1,followerCommit:0,
> request=AppendEntriesRequest:cid=70,entriesCount=1,lastEntry=(t:1, i:1)
> 2020-07-21 17:12:55,643 [grpc-default-executor-7] INFO
> server.GrpcServerProtocolService (GrpcServerProtocolService.java:onNext(109))
> - receive request, in order or not: false
> {code}
>
>
--
This message was sent by Atlassian Jira
(v8.3.4#803005)
