[iotdb] branch master updated: [IOTDB-3317] RatisConsensus fallback for StateMachine RuntimeException (#6101)

[tanxinyu]

This is an automated email from the ASF dual-hosted git repository.

tanxinyu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git


The following commit(s) were added to refs/heads/master by this push:
     new c2afca12d5 [IOTDB-3317] RatisConsensus fallback for StateMachine 
RuntimeException (#6101)
c2afca12d5 is described below

commit c2afca12d576b917028bfdb6e2bbade7ed3ac2fd
Author: William Song <48054931+szywill...@users.noreply.github.com>
AuthorDate: Mon Jun 6 13:12:30 2022 +0800

    [IOTDB-3317] RatisConsensus fallback for StateMachine RuntimeException 
(#6101)
    
    * ratisconsensus fallback
    
    * add exception message
---
 .../ratis/ApplicationStateMachineProxy.java        | 10 ++++-
 .../apache/iotdb/consensus/ratis/RatisClient.java  | 49 +++++++++++++++++++---
 2 files changed, 51 insertions(+), 8 deletions(-)

diff --git 
a/consensus/src/main/java/org/apache/iotdb/consensus/ratis/ApplicationStateMachineProxy.java
 
b/consensus/src/main/java/org/apache/iotdb/consensus/ratis/ApplicationStateMachineProxy.java
index e6467401a0..6588d5dbb4 100644
--- 
a/consensus/src/main/java/org/apache/iotdb/consensus/ratis/ApplicationStateMachineProxy.java
+++ 
b/consensus/src/main/java/org/apache/iotdb/consensus/ratis/ApplicationStateMachineProxy.java
@@ -116,8 +116,14 @@ public class ApplicationStateMachineProxy extends 
BaseStateMachine {
               
log.getStateMachineLogEntry().getLogData().asReadOnlyByteBuffer());
     }
 
-    TSStatus result = applicationStateMachine.write(applicationRequest);
-    Message ret = new ResponseMessage(result);
+    Message ret;
+    try {
+      TSStatus result = applicationStateMachine.write(applicationRequest);
+      ret = new ResponseMessage(result);
+    } catch (Exception rte) {
+      logger.error("application statemachine throws a runtime exception: ", 
rte);
+      ret = Message.valueOf("internal error. statemachine throws a runtime 
exception: " + rte);
+    }
 
     return CompletableFuture.completedFuture(ret);
   }
diff --git 
a/consensus/src/main/java/org/apache/iotdb/consensus/ratis/RatisClient.java 
b/consensus/src/main/java/org/apache/iotdb/consensus/ratis/RatisClient.java
index 1d8c659b62..1e6ea9e000 100644
--- a/consensus/src/main/java/org/apache/iotdb/consensus/ratis/RatisClient.java
+++ b/consensus/src/main/java/org/apache/iotdb/consensus/ratis/RatisClient.java
@@ -28,7 +28,9 @@ import org.apache.ratis.client.RaftClient;
 import org.apache.ratis.client.RaftClientRpc;
 import org.apache.ratis.conf.RaftProperties;
 import org.apache.ratis.protocol.RaftGroup;
+import org.apache.ratis.protocol.exceptions.RaftException;
 import org.apache.ratis.retry.ExponentialBackoffRetry;
+import org.apache.ratis.retry.RetryPolicy;
 import org.apache.ratis.util.TimeDuration;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -97,12 +99,7 @@ public class RatisClient {
               RaftClient.newBuilder()
                   .setProperties(raftProperties)
                   .setRaftGroup(group)
-                  .setRetryPolicy(
-                      ExponentialBackoffRetry.newBuilder()
-                          .setBaseSleepTime(TimeDuration.valueOf(100, 
TimeUnit.MILLISECONDS))
-                          .setMaxSleepTime(TimeDuration.valueOf(10, 
TimeUnit.SECONDS))
-                          .setMaxAttempts(10)
-                          .build())
+                  .setRetryPolicy(new RatisRetryPolicy())
                   .setClientRpc(clientRpc)
                   .build(),
               clientManager));
@@ -113,4 +110,44 @@ public class RatisClient {
       return true;
     }
   }
+
+  /**
+   * RatisRetryPolicy is similar to ExceptionDependentRetry 1. By default, use
+   * ExponentialBackoffRetry to handle request failure 2. If unexpected 
IOException is caught,
+   * immediately fail the request and let application choose retry action.
+   *
+   * <p>potential IOException can be categorized into expected and unexpected 
1. expected, instance
+   * of RaftException, like LeaderNotReady / GroupMisMatch etc. 2. unexpected, 
IOException which is
+   * not an instance of RaftException
+   */
+  private static class RatisRetryPolicy implements RetryPolicy {
+
+    private static final Logger logger = 
LoggerFactory.getLogger(RatisClient.class);
+    private static final int maxAttempts = 10;
+    RetryPolicy defaultPolicy;
+
+    public RatisRetryPolicy() {
+      defaultPolicy =
+          ExponentialBackoffRetry.newBuilder()
+              .setBaseSleepTime(TimeDuration.valueOf(100, 
TimeUnit.MILLISECONDS))
+              .setMaxSleepTime(TimeDuration.valueOf(10, TimeUnit.SECONDS))
+              .setMaxAttempts(maxAttempts)
+              .build();
+    }
+
+    @Override
+    public Action handleAttemptFailure(Event event) {
+
+      if (event.getCause() != null) {
+        if (event.getCause() instanceof IOException
+            && !(event.getCause() instanceof RaftException)) {
+          // unexpected. may be caused by statemachine.
+          logger.debug("raft client request failed and caught exception: ", 
event.getCause());
+          return NO_RETRY_ACTION;
+        }
+      }
+
+      return defaultPolicy.handleAttemptFailure(event);
+    }
+  }
 }