Configured, a Active Name node and Standby Name Node in my cluster. Tried
importing 2 lacks documents into HBase, and in between Standby Name Node
got Shutdown.
Looked at the logs, i can see below errors. What could cause this issue,
and how this can be avoided?
2013-10-13 20:35:17,806 WARN
org.apache.hadoop.hdfs.qjournal.client.QuorumJournalManager: Took 5932ms to
send a batch of 5 edits (599 bytes) to remote journal 192.168.1.21:8485
2013-10-13 20:35:28,399 FATAL
org.apache.hadoop.hdfs.server.namenode.FSEditLog: Error: finalize log
segment 11505, 12383 failed for required journal (JournalAndStream(mgr=QJM
to [192.168.1.22:8485, 192.168.1.21:8485, 192.168.1.24:8485], stream=null))
org.apache.hadoop.hdfs.qjournal.client.QuorumException: Got too many
exceptions to achieve quorum size 2/3. 1 successful responses:
192.168.1.21:8485: null [success]
2 exceptions thrown:
192.168.1.24:8485: IPC's epoch 8 is less than the last promised epoch 9
at
org.apache.hadoop.hdfs.qjournal.server.Journal.checkRequest(Journal.java:408)
at
org.apache.hadoop.hdfs.qjournal.server.Journal.finalizeLogSegment(Journal.java:549)
at
org.apache.hadoop.hdfs.qjournal.server.JournalNodeRpcServer.finalizeLogSegment(JournalNodeRpcServer.java:162)
at
org.apache.hadoop.hdfs.qjournal.protocolPB.QJournalProtocolServerSideTranslatorPB.finalizeLogSegment(QJournalProtocolServerSideTranslatorPB.java:171)
at
org.apache.hadoop.hdfs.qjournal.protocol.QJournalProtocolProtos$QJournalProtocolService$2.callBlockingMethod(QJournalProtocolProtos.java:14090)
at
org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:453)
at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1002)
at org.apache.hadoop.ipc.Server$Handler$1.run(Server.java:1751)
at org.apache.hadoop.ipc.Server$Handler$1.run(Server.java:1747)
at java.security.AccessController.doPrivileged(Native Method)
at javax.security.auth.Subject.doAs(Subject.java:396)
at
org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1408)
at org.apache.hadoop.ipc.Server$Handler.run(Server.java:1745)
192.168.1.22:8485: IPC's epoch 8 is less than the last promised epoch 9
at
org.apache.hadoop.hdfs.qjournal.server.Journal.checkRequest(Journal.java:408)
at
org.apache.hadoop.hdfs.qjournal.server.Journal.finalizeLogSegment(Journal.java:549)
at
org.apache.hadoop.hdfs.qjournal.server.JournalNodeRpcServer.finalizeLogSegment(JournalNodeRpcServer.java:162)
at
org.apache.hadoop.hdfs.qjournal.protocolPB.QJournalProtocolServerSideTranslatorPB.finalizeLogSegment(QJournalProtocolServerSideTranslatorPB.java:171)
at
org.apache.hadoop.hdfs.qjournal.protocol.QJournalProtocolProtos$QJournalProtocolService$2.callBlockingMethod(QJournalProtocolProtos.java:14090)
at
org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:453)
at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1002)
at org.apache.hadoop.ipc.Server$Handler$1.run(Server.java:1751)
at org.apache.hadoop.ipc.Server$Handler$1.run(Server.java:1747)
at java.security.AccessController.doPrivileged(Native Method)
at javax.security.auth.Subject.doAs(Subject.java:396)
at
org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1408)
at org.apache.hadoop.ipc.Server$Handler.run(Server.java:1745)
at
org.apache.hadoop.hdfs.qjournal.client.QuorumException.create(QuorumException.java:81)
at
org.apache.hadoop.hdfs.qjournal.client.QuorumCall.rethrowException(QuorumCall.java:223)
at
org.apache.hadoop.hdfs.qjournal.client.AsyncLoggerSet.waitForWriteQuorum(AsyncLoggerSet.java:142)
at
org.apache.hadoop.hdfs.qjournal.client.QuorumJournalManager.finalizeLogSegment(QuorumJournalManager.java:398)
at
org.apache.hadoop.hdfs.server.namenode.JournalSet$3.apply(JournalSet.java:213)
at
org.apache.hadoop.hdfs.server.namenode.JournalSet.mapJournalsAndReportErrors(JournalSet.java:353)
at
org.apache.hadoop.hdfs.server.namenode.JournalSet.finalizeLogSegment(JournalSet.java:208)
at
org.apache.hadoop.hdfs.server.namenode.FSEditLog.endCurrentLogSegment(FSEditLog.java:959)
at
org.apache.hadoop.hdfs.server.namenode.FSEditLog.endCurrentLogSegment(FSEditLog.java:959)
at
org.apache.hadoop.hdfs.server.namenode.FSEditLog.close(FSEditLog.java:303)
at
org.apache.hadoop.hdfs.server.namenode.FSNamesystem.stopActiveServices(FSNamesystem.java:807)
at
org.apache.hadoop.hdfs.server.namenode.NameNode$NameNodeHAContext.stopActiveServices(NameNode.java:1357)
at
org.apache.hadoop.hdfs.server.namenode.ha.ActiveState.exitState(ActiveState.java:70)
at
org.apache.hadoop.hdfs.server.namenode.ha.HAState.setStateInternal(HAState.java:61)
at
org.apache.hadoop.hdfs.server.namenode.ha.ActiveState.setState(ActiveState.java:52)
at
org.apache.hadoop.hdfs.server.namenode.NameNode.transitionToStandby(NameNode.java:1270)
at
org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.transitionToStandby(NameNodeRpcServer.java:1045)
at
org.apache.hadoop.ha.protocolPB.HAServiceProtocolServerSideTranslatorPB.transitionToStandby(HAServiceProtocolServerSideTranslatorPB.java:119)
at
org.apache.hadoop.ha.proto.HAServiceProtocolProtos$HAServiceProtocolService$2.callBlockingMethod(HAServiceProtocolProtos.java:3635)
at
org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:453)
at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1002)
at org.apache.hadoop.ipc.Server$Handler$1.run(Server.java:1751)
at org.apache.hadoop.ipc.Server$Handler$1.run(Server.java:1747)
at java.security.AccessController.doPrivileged(Native Method)
at javax.security.auth.Subject.doAs(Subject.java:396)
at
org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1408)
at org.apache.hadoop.ipc.Server$Handler.run(Server.java:1745)
2013-10-13 20:35:28,401 INFO org.apache.hadoop.util.ExitUtil: Exiting with
status 1
2013-10-13 20:35:28,406 INFO
org.apache.hadoop.hdfs.server.namenode.NameNode: SHUTDOWN_MSG:
/************************************************************
SHUTDOWN_MSG: Shutting down NameNode at ubn04xen1/192.168.1.24
************************************************************/
