sumitagrawl commented on code in PR #12015:
URL: https://github.com/apache/ozone/pull/12015#discussion_r4566034478
##########
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMStateMachine.java:
##########
@@ -252,6 +273,68 @@ public void notifyLogFailed(Throwable ex,
ExitUtils.terminate(1, ex.getMessage(), ex, StateMachine.LOG);
}
+ @Override
+ public void notifyServerShutdown(RaftProtos.RoleInfoProto roleInfo, boolean
allServer) {
+ // If SCM is stopped, this shutdown originates from scm.stop(); otherwise
it originates from Ratis.
+ if (!allServer || !isInitialized || scm.isStopped()) {
+ return;
+ }
+ startRatisShutdownTimer("SCM Ratis server is shut down by Ratis");
+ }
+
+ /**
+ * Ratis calls {@link #close()} and {@link #notifyServerShutdown} only after
the state machine updater has applied
+ * every committed transaction, which can take arbitrarily long. Poll the
division so the timer starts as soon as
+ * Ratis begins closing.
+ */
+ void checkRatisDivisionClosing() {
+ if (scm.isStopped() || ratisShutdownTimerStarted.get()) {
+ return;
+ }
+ SCMHAManager haManager = scm.getScmHAManager();
+ SCMRatisServer ratisServer = haManager == null ? null :
haManager.getRatisServer();
+ if (ratisServer == null || ratisServer.isStopped()) {
+ return;
+ }
+ LifeCycle.State state =
ratisServer.getDivision().getInfo().getLifeCycleState();
+ if (state == LifeCycle.State.CLOSING || state == LifeCycle.State.CLOSED) {
+ startRatisShutdownTimer("SCM Ratis division is " + state);
+ }
+ }
+
+ private void startRatisShutdownTimer(String reason) {
+ if (!ratisShutdownTimerStarted.compareAndSet(false, true)) {
+ return;
+ }
+ Duration timeout =
scm.getConfiguration().getObject(ScmConfig.class).getRatisShutdownTimeout();
+ LOG.error("{}, terminating SCM in {}", reason, timeout);
+ // Run on a different thread, so that the Ratis server can finish closing.
+ CompletableFuture.runAsync(this::closeAfterRatisShutdown,
+ CompletableFuture.delayedExecutor(timeout.toMillis(),
TimeUnit.MILLISECONDS));
+ }
+
+ private void closeAfterRatisShutdown() {
+ if (scm.isStopped()) {
+ return;
+ }
+ final String message = "scm statemachine is closed by ratis, terminate
SCM";
Review Comment:
what if call close() as already present in this class?
##########
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMStateMachine.java:
##########
@@ -187,6 +198,16 @@ public void initialize(RaftServer server, RaftGroupId id,
storage.init(raftStorage);
LOG.info("{}: initialize {}", server.getId(), id);
});
+ if (isInitialized) {
Review Comment:
I think we need not start this executor service on initialized, but can be
started at point of shutdown trigger
##########
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMStateMachine.java:
##########
@@ -252,6 +273,68 @@ public void notifyLogFailed(Throwable ex,
ExitUtils.terminate(1, ex.getMessage(), ex, StateMachine.LOG);
}
+ @Override
+ public void notifyServerShutdown(RaftProtos.RoleInfoProto roleInfo, boolean
allServer) {
+ // If SCM is stopped, this shutdown originates from scm.stop(); otherwise
it originates from Ratis.
+ if (!allServer || !isInitialized || scm.isStopped()) {
+ return;
+ }
+ startRatisShutdownTimer("SCM Ratis server is shut down by Ratis");
+ }
+
+ /**
+ * Ratis calls {@link #close()} and {@link #notifyServerShutdown} only after
the state machine updater has applied
+ * every committed transaction, which can take arbitrarily long. Poll the
division so the timer starts as soon as
+ * Ratis begins closing.
+ */
+ void checkRatisDivisionClosing() {
+ if (scm.isStopped() || ratisShutdownTimerStarted.get()) {
+ return;
+ }
+ SCMHAManager haManager = scm.getScmHAManager();
+ SCMRatisServer ratisServer = haManager == null ? null :
haManager.getRatisServer();
+ if (ratisServer == null || ratisServer.isStopped()) {
+ return;
+ }
+ LifeCycle.State state =
ratisServer.getDivision().getInfo().getLifeCycleState();
+ if (state == LifeCycle.State.CLOSING || state == LifeCycle.State.CLOSED) {
+ startRatisShutdownTimer("SCM Ratis division is " + state);
+ }
+ }
+
+ private void startRatisShutdownTimer(String reason) {
+ if (!ratisShutdownTimerStarted.compareAndSet(false, true)) {
+ return;
+ }
+ Duration timeout =
scm.getConfiguration().getObject(ScmConfig.class).getRatisShutdownTimeout();
+ LOG.error("{}, terminating SCM in {}", reason, timeout);
+ // Run on a different thread, so that the Ratis server can finish closing.
+ CompletableFuture.runAsync(this::closeAfterRatisShutdown,
+ CompletableFuture.delayedExecutor(timeout.toMillis(),
TimeUnit.MILLISECONDS));
+ }
+
+ private void closeAfterRatisShutdown() {
+ if (scm.isStopped()) {
+ return;
+ }
+ final String message = "scm statemachine is closed by ratis, terminate
SCM";
+ // scm.stop() may block on the Ratis server that is still applying
transactions, so bound it and force the exit.
+ final long stopTimeoutSeconds =
Math.max(HddsUtils.getShutDownTimeOut(scm.getConfiguration()),
+ OzoneServiceConfig.OZONE_SHUTDOWN_TIMEOUT_MINIMUM);
+ final Thread shutdown = new Thread(() -> scm.shutDown(message),
scm.threadNamePrefix() + "SCMRatisShutdown");
Review Comment:
why do we need this? already this method is called in scheduler thread.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]