[
https://issues.apache.org/jira/browse/HDDS-3022?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=17261394#comment-17261394
]
Shashikant Banerjee commented on HDDS-3022:
-------------------------------------------
Some Detailed logging:
{code:java}
2021-01-08 12:43:00,483 [ChunkWriter-1-0] INFO keyvalue.KeyValueHandler
(ContainerUtils.java:logAndReturnError(89)) - Operation: WriteChunk , Trace ID:
, Message: java.io.IOException: No space left on device , Result: IO_EXCEPTION
, StorageContainerException Occurred.2021-01-08 12:43:00,483 [ChunkWriter-1-0]
INFO keyvalue.KeyValueHandler (ContainerUtils.java:logAndReturnError(89)) -
Operation: WriteChunk , Trace ID: , Message: java.io.IOException: No space
left on device , Result: IO_EXCEPTION , StorageContainerException
Occurred.org.apache.hadoop.hdds.scm.container.common.helpers.StorageContainerException:
java.io.IOException: No space left on device at
org.apache.hadoop.ozone.container.keyvalue.helpers.ChunkUtils.wrapInStorageContainerException(ChunkUtils.java:331)
at
org.apache.hadoop.ozone.container.keyvalue.helpers.ChunkUtils.writeData(ChunkUtils.java:125)
at
org.apache.hadoop.ozone.container.keyvalue.helpers.ChunkUtils.writeData(ChunkUtils.java:110)
at
org.apache.hadoop.ozone.container.keyvalue.impl.FilePerBlockStrategy.writeChunk(FilePerBlockStrategy.java:120)
at
org.apache.hadoop.ozone.container.keyvalue.impl.ChunkManagerDispatcher.writeChunk(ChunkManagerDispatcher.java:70)
at
org.apache.hadoop.ozone.container.keyvalue.KeyValueHandler.handleWriteChunk(KeyValueHandler.java:722)
at
org.apache.hadoop.ozone.container.keyvalue.KeyValueHandler.dispatchRequest(KeyValueHandler.java:199)
at
org.apache.hadoop.ozone.container.keyvalue.KeyValueHandler.handle(KeyValueHandler.java:163)
at
org.apache.hadoop.ozone.container.common.impl.HddsDispatcher.dispatchRequest(HddsDispatcher.java:309)
at
org.apache.hadoop.ozone.container.common.impl.HddsDispatcher.lambda$dispatch$0(HddsDispatcher.java:171)
at
org.apache.hadoop.hdds.server.OzoneProtocolMessageDispatcher.processRequest(OzoneProtocolMessageDispatcher.java:87)
at
org.apache.hadoop.ozone.container.common.impl.HddsDispatcher.dispatch(HddsDispatcher.java:170)
at
org.apache.hadoop.ozone.container.common.transport.server.ratis.ContainerStateMachine.dispatchCommand(ContainerStateMachine.java:398)
at
org.apache.hadoop.ozone.container.common.transport.server.ratis.ContainerStateMachine.runCommand(ContainerStateMachine.java:408)
at
org.apache.hadoop.ozone.container.common.transport.server.ratis.ContainerStateMachine.lambda$handleWriteChunk$2(ContainerStateMachine.java:445)
at
java.util.concurrent.CompletableFuture$AsyncSupply.run(CompletableFuture.java:1604)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)Caused by: java.io.IOException: No
space left on device at sun.nio.ch.FileDispatcherImpl.writev0(Native Method) at
sun.nio.ch.FileDispatcherImpl.writev(FileDispatcherImpl.java:72) at
sun.nio.ch.IOUtil.write(IOUtil.java:148) at
sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:239) at
java.nio.channels.FileChannel.write(FileChannel.java:414) at
org.apache.hadoop.ozone.common.ChunkBufferImplWithByteBufferList.writeTo(ChunkBufferImplWithByteBufferList.java:186)
at
org.apache.hadoop.ozone.container.keyvalue.helpers.ChunkUtils.writeDataToChannel(ChunkUtils.java:163)
at
org.apache.hadoop.ozone.container.keyvalue.helpers.ChunkUtils.lambda$writeData$1(ChunkUtils.java:111)
at
org.apache.hadoop.ozone.container.keyvalue.helpers.ChunkUtils.writeData(ChunkUtils.java:123)
... 17 more12:43:00.516 [grpc-default-executor-2] ERROR DNAudit - user=null |
ip=null | op=PUT_BLOCK {blockData=[blockId=conID: 5 locID: 105520154573406224
bcsId: 0,size=512]} |
ret=FAILUREorg.apache.hadoop.hdds.scm.container.common.helpers.ContainerNotOpenException:
Container 5 in UNHEALTHY state at
org.apache.hadoop.ozone.container.common.impl.HddsDispatcher.validateContainerCommand(HddsDispatcher.java:484)
~[hadoop-hdds-container-service-1.1.0-SNAPSHOT.jar:?] at
org.apache.hadoop.ozone.container.common.transport.server.ratis.ContainerStateMachine.startTransaction(ContainerStateMachine.java:317)
~[hadoop-hdds-container-service-1.1.0-SNAPSHOT.jar:?] at
org.apache.ratis.server.impl.RaftServerImpl.submitClientRequestAsync(RaftServerImpl.java:775)
~[ratis-server-1.1.0-35f17fa-SNAPSHOT.jar:1.1.0-35f17fa-SNAPSHOT] at
org.apache.ratis.server.impl.RaftServerProxy.lambda$submitClientRequestAsync$9(RaftServerProxy.java:415)
~[ratis-server-1.1.0-35f17fa-SNAPSHOT.jar:1.1.0-35f17fa-SNAPSHOT] at
org.apache.ratis.server.impl.RaftServerProxy.lambda$null$7(RaftServerProxy.java:410)
~[ratis-server-1.1.0-35f17fa-SNAPSHOT.jar:1.1.0-35f17fa-SNAPSHOT] at
org.apache.ratis.util.JavaUtils.callAsUnchecked(JavaUtils.java:117)
~[ratis-common-1.1.0-35f17fa-SNAPSHOT.jar:1.1.0-35f17fa-SNAPSHOT] at
org.apache.ratis.server.impl.RaftServerProxy.lambda$submitRequest$8(RaftServerProxy.java:410)
~[ratis-server-1.1.0-35f17fa-SNAPSHOT.jar:1.1.0-35f17fa-SNAPSHOT] at
java.util.concurrent.CompletableFuture.uniComposeStage(CompletableFuture.java:995)
~[?:1.8.0_232] at
java.util.concurrent.CompletableFuture.thenCompose(CompletableFuture.java:2137)
~[?:1.8.0_232] at
org.apache.ratis.server.impl.RaftServerProxy.submitRequest(RaftServerProxy.java:409)
~[ratis-server-1.1.0-35f17fa-SNAPSHOT.jar:1.1.0-35f17fa-SNAPSHOT] at
org.apache.ratis.server.impl.RaftServerProxy.submitClientRequestAsync(RaftServerProxy.java:415)
~[ratis-server-1.1.0-35f17fa-SNAPSHOT.jar:1.1.0-35f17fa-SNAPSHOT] at
org.apache.ratis.grpc.client.GrpcClientProtocolService$RequestStreamObserver.processClientRequest(GrpcClientProtocolService.java:240)
~[ratis-grpc-1.1.0-35f17fa-SNAPSHOT.jar:1.1.0-35f17fa-SNAPSHOT] at
org.apache.ratis.grpc.client.GrpcClientProtocolService$OrderedRequestStreamObserver.processClientRequest(GrpcClientProtocolService.java:346)
~[ratis-grpc-1.1.0-35f17fa-SNAPSHOT.jar:1.1.0-35f17fa-SNAPSHOT] at
org.apache.ratis.util.SlidingWindow$Server.processRequestsFromHead(SlidingWindow.java:429)
[ratis-common-1.1.0-35f17fa-SNAPSHOT.jar:1.1.0-35f17fa-SNAPSHOT] at
org.apache.ratis.util.SlidingWindow$Server.receivedRequest(SlidingWindow.java:421)
[ratis-common-1.1.0-35f17fa-SNAPSHOT.jar:1.1.0-35f17fa-SNAPSHOT] at
org.apache.ratis.grpc.client.GrpcClientProtocolService$OrderedRequestStreamObserver.processClientRequest(GrpcClientProtocolService.java:370)
[ratis-grpc-1.1.0-35f17fa-SNAPSHOT.jar:1.1.0-35f17fa-SNAPSHOT] at
org.apache.ratis.grpc.client.GrpcClientProtocolService$RequestStreamObserver.onNext(GrpcClientProtocolService.java:260)
[ratis-grpc-1.1.0-35f17fa-SNAPSHOT.jar:1.1.0-35f17fa-SNAPSHOT] at
org.apache.ratis.grpc.client.GrpcClientProtocolService$RequestStreamObserver.onNext(GrpcClientProtocolService.java:183)
[ratis-grpc-1.1.0-35f17fa-SNAPSHOT.jar:1.1.0-35f17fa-SNAPSHOT] at
org.apache.ratis.thirdparty.io.grpc.stub.ServerCalls$StreamingServerCallHandler$StreamingServerCallListener.onMessage(ServerCalls.java:251)
[ratis-thirdparty-misc-0.6.0-SNAPSHOT.jar:0.6.0-SNAPSHOT] at
org.apache.ratis.thirdparty.io.grpc.ForwardingServerCallListener.onMessage(ForwardingServerCallListener.java:33)
[ratis-thirdparty-misc-0.6.0-SNAPSHOT.jar:0.6.0-SNAPSHOT] at
org.apache.ratis.thirdparty.io.grpc.internal.ServerCallImpl$ServerStreamListenerImpl.messagesAvailableInternal(ServerCallImpl.java:309)
[ratis-thirdparty-misc-0.6.0-SNAPSHOT.jar:0.6.0-SNAPSHOT] at
org.apache.ratis.thirdparty.io.grpc.internal.ServerCallImpl$ServerStreamListenerImpl.messagesAvailable(ServerCallImpl.java:292)
[ratis-thirdparty-misc-0.6.0-SNAPSHOT.jar:0.6.0-SNAPSHOT] at
org.apache.ratis.thirdparty.io.grpc.internal.ServerImpl$JumpToApplicationThreadServerStreamListener$1MessagesAvailable.runInContext(ServerImpl.java:783)
[ratis-thirdparty-misc-0.6.0-SNAPSHOT.jar:0.6.0-SNAPSHOT] at
org.apache.ratis.thirdparty.io.grpc.internal.ContextRunnable.run(ContextRunnable.java:37)
[ratis-thirdparty-misc-0.6.0-SNAPSHOT.jar:0.6.0-SNAPSHOT] at
org.apache.ratis.thirdparty.io.grpc.internal.SerializingExecutor.run(SerializingExecutor.java:123)
[ratis-thirdparty-misc-0.6.0-SNAPSHOT.jar:0.6.0-SNAPSHOT] at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
[?:1.8.0_232] at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
[?:1.8.0_232] at java.lang.Thread.run(Thread.java:748) [?:1.8.0_232]2021-01-08
12:43:00,522
[EventQueue-IncrementalContainerReportForIncrementalContainerReportHandler]
WARN container.IncrementalContainerReportHandler
(AbstractContainerReportHandler.java:updateContainerState(182)) - Container #5
is in OPEN state, but the datanode 80022841-2f08-45c1-9e96-edec3266b7eb{ip:
172.17.0.2, host: 89a91aef9e88, networkLocation: /default-rack, certSerialId:
null, persistedOpState: IN_SERVICE, persistedOpStateExpiryEpochSec: 0} reports
an UNHEALTHY replica.2021-01-08 12:43:00,537
[80022841-2f08-45c1-9e96-edec3266b7eb@group-6B3EB6F55FD9-SegmentedRaftLogWorker]
ERROR ratis.XceiverServerRatis
(XceiverServerRatis.java:triggerPipelineClose(686)) - pipeline Action CLOSE on
pipeline PipelineID=bd082ca9-fe4c-4b51-bdf6-6b3eb6f55fd9.Reason :
org.yaml.snakeyaml.error.YAMLException: java.io.IOException: No space left on
device
{code}
> Datanode unable to close Pipeline after disk out of space
> ---------------------------------------------------------
>
> Key: HDDS-3022
> URL: https://issues.apache.org/jira/browse/HDDS-3022
> Project: Hadoop Distributed Data Store
> Issue Type: Bug
> Components: Ozone Datanode
> Affects Versions: 0.5.0
> Reporter: Vivek Ratnavel Subramanian
> Assignee: Shashikant Banerjee
> Priority: Critical
> Labels: TriagePending
> Attachments: ozone_logs.zip
>
>
> Datanode gets into a loop and keeps throwing errors while trying to close
> pipeline
> {code:java}
> 2020-02-14 00:25:10,208 INFO org.apache.ratis.server.impl.RaftServerImpl:
> 285cac09-7622-45e6-be02-b3c68ebf8b10@group-701265EC9F07: changes role from
> FOLLOWER to CANDIDATE at term 6240 for changeToCandidate
> 2020-02-14 00:25:10,208 ERROR
> org.apache.hadoop.ozone.container.common.transport.server.ratis.XceiverServerRatis:
> pipeline Action CLOSE on pipeline
> PipelineID=02e7e10e-2d50-4ace-a18b-701265ec9f07.Reason :
> 285cac09-7622-45e6-be02-b3c68ebf8b10 is in candidate state for 31898494ms
> 2020-02-14 00:25:10,208 INFO org.apache.ratis.server.impl.RoleInfo:
> 285cac09-7622-45e6-be02-b3c68ebf8b10: start LeaderElection
> 2020-02-14 00:25:10,223 INFO org.apache.ratis.server.impl.LeaderElection:
> 285cac09-7622-45e6-be02-b3c68ebf8b10@group-701265EC9F07-LeaderElection37032:
> begin an election at term 6241 for 0:
> [d432c890-5ec4-4cf1-9078-28497a08ab85:10.65.6.227:9858,
> 285cac09-7622-45e6-be02-b3c68ebf8b10:10.65.24.80:9858,
> cabbdef8-ed6c-4fc7-b7b2-d1ddd07da47e:10.65.8.165:9858], old=null
> 2020-02-14 00:25:10,259 INFO org.apache.ratis.server.impl.LeaderElection:
> 285cac09-7622-45e6-be02-b3c68ebf8b10@group-701265EC9F07-LeaderElection37032
> got exception when requesting votes: java.util.concurrent.ExecutionException:
> org.apache.ratis.thirdparty.io.grpc.StatusRuntimeException: INTERNAL:
> d432c890-5ec4-4cf1-9078-28497a08ab85: group-701265EC9F07 not found.
> 2020-02-14 00:25:10,270 INFO org.apache.ratis.server.impl.LeaderElection:
> 285cac09-7622-45e6-be02-b3c68ebf8b10@group-701265EC9F07-LeaderElection37032
> got exception when requesting votes: java.util.concurrent.ExecutionException:
> org.apache.ratis.thirdparty.io.grpc.StatusRuntimeException: INTERNAL:
> cabbdef8-ed6c-4fc7-b7b2-d1ddd07da47e: group-701265EC9F07 not found.
> 2020-02-14 00:25:10,270 INFO org.apache.ratis.server.impl.LeaderElection:
> 285cac09-7622-45e6-be02-b3c68ebf8b10@group-701265EC9F07-LeaderElection37032:
> Election REJECTED; received 0 response(s) [] and 2 exception(s);
> 285cac09-7622-45e6-be02-b3c68ebf8b10@group-701265EC9F07:t6241, leader=null,
> voted=285cac09-7622-45e6-be02-b3c68ebf8b10,
> raftlog=285cac09-7622-45e6-be02-b3c68ebf8b10@group-701265EC9F07-SegmentedRaftLog:OPENED:c4,f4,i14,
> conf=0: [d432c890-5ec4-4cf1-9078-28497a08ab85:10.65.6.227:9858,
> 285cac09-7622-45e6-be02-b3c68ebf8b10:10.65.24.80:9858,
> cabbdef8-ed6c-4fc7-b7b2-d1ddd07da47e:10.65.8.165:9858], old=null
> 2020-02-14 00:25:10,270 INFO org.apache.ratis.server.impl.LeaderElection:
> Exception 0: java.util.concurrent.ExecutionException:
> org.apache.ratis.thirdparty.io.grpc.StatusRuntimeException: INTERNAL:
> d432c890-5ec4-4cf1-9078-28497a08ab85: group-701265EC9F07 not found.
> 2020-02-14 00:25:10,270 INFO org.apache.ratis.server.impl.LeaderElection:
> Exception 1: java.util.concurrent.ExecutionException:
> org.apache.ratis.thirdparty.io.grpc.StatusRuntimeException: INTERNAL:
> cabbdef8-ed6c-4fc7-b7b2-d1ddd07da47e: group-701265EC9F07 not found.
> 2020-02-14 00:25:10,270 INFO org.apache.ratis.server.impl.RaftServerImpl:
> 285cac09-7622-45e6-be02-b3c68ebf8b10@group-701265EC9F07: changes role from
> CANDIDATE to FOLLOWER at term 6241 for DISCOVERED_A_NEW_TERM
> 2020-02-14 00:25:10,270 INFO org.apache.ratis.server.impl.RoleInfo:
> 285cac09-7622-45e6-be02-b3c68ebf8b10: shutdown LeaderElection
> 2020-02-14 00:25:10,270 INFO org.apache.ratis.server.impl.RoleInfo:
> 285cac09-7622-45e6-be02-b3c68ebf8b10: start FollowerState
> 2020-02-14 00:25:10,680 WARN org.apache.ratis.grpc.server.GrpcLogAppender:
> 285cac09-7622-45e6-be02-b3c68ebf8b10@group-DD847EC75388->d432c890-5ec4-4cf1-9078-28497a08ab85-GrpcLogAppender:
> HEARTBEAT appendEntries Timeout,
> request=AppendEntriesRequest:cid=12669,entriesCount=0,lastEntry=null
> 2020-02-14 00:25:10,752 ERROR
> org.apache.hadoop.ozone.container.common.transport.server.ratis.XceiverServerRatis:
> pipeline Action CLOSE on pipeline
> PipelineID=7ad5ce51-d3fa-4e71-99f2-dd847ec75388.Reason :
> 285cac09-7622-45e6-be02-b3c68ebf8b10 has not seen follower/s
> d432c890-5ec4-4cf1-9078-28497a08ab85 for 31623987ms
> cabbdef8-ed6c-4fc7-b7b2-d1ddd07da47e for 31618878ms
> 2020-02-14 00:25:10,894 INFO org.apache.ratis.server.impl.FollowerState:
> 285cac09-7622-45e6-be02-b3c68ebf8b10@group-0068FD2EA2C9-FollowerState: change
> to CANDIDATE, lastRpcTime:5021ms, electionTimeout:5017ms
> 2020-02-14 00:25:10,894 INFO org.apache.ratis.server.impl.RoleInfo:
> 285cac09-7622-45e6-be02-b3c68ebf8b10: shutdown FollowerState
> 2020-02-14 00:25:10,894 INFO org.apache.ratis.server.impl.RaftServerImpl:
> 285cac09-7622-45e6-be02-b3c68ebf8b10@group-0068FD2EA2C9: changes role from
> FOLLOWER to CANDIDATE at term 6220 for changeToCandidate
> 2020-02-14 00:25:10,894 ERROR
> org.apache.hadoop.ozone.container.common.transport.server.ratis.XceiverServerRatis:
> pipeline Action CLOSE on pipeline
> PipelineID=179ac1d0-e5d5-4898-bef7-0068fd2ea2c9.Reason :
> 285cac09-7622-45e6-be02-b3c68ebf8b10 is in candidate state for 31805092ms
> 2020-02-14 00:25:10,894 INFO org.apache.ratis.server.impl.RoleInfo:
> 285cac09-7622-45e6-be02-b3c68ebf8b10: start LeaderElection
> 2020-02-14 00:25:10,917 INFO org.apache.ratis.server.impl.LeaderElection:
> 285cac09-7622-45e6-be02-b3c68ebf8b10@group-0068FD2EA2C9-LeaderElection37033:
> begin an election at term 6221 for 0:
> [d432c890-5ec4-4cf1-9078-28497a08ab85:10.65.6.227:9858,
> 285cac09-7622-45e6-be02-b3c68ebf8b10:10.65.24.80:9858,
> cabbdef8-ed6c-4fc7-b7b2-d1ddd07da47e:10.65.8.165:9858], old=null
> 2020-02-14 00:25:10,921 INFO org.apache.ratis.server.impl.LeaderElection:
> 285cac09-7622-45e6-be02-b3c68ebf8b10@group-0068FD2EA2C9-LeaderElection37033
> got exception when requesting votes: java.util.concurrent.ExecutionException:
> org.apache.ratis.thirdparty.io.grpc.StatusRuntimeException: INTERNAL:
> cabbdef8-ed6c-4fc7-b7b2-d1ddd07da47e: group-0068FD2EA2C9 not found.
> 2020-02-14 00:25:10,921 INFO org.apache.ratis.server.impl.LeaderElection:
> 285cac09-7622-45e6-be02-b3c68ebf8b10@group-0068FD2EA2C9-LeaderElection37033
> got exception when requesting votes: java.util.concurrent.ExecutionException:
> org.apache.ratis.thirdparty.io.grpc.StatusRuntimeException: INTERNAL:
> d432c890-5ec4-4cf1-9078-28497a08ab85: group-0068FD2EA2C9 not found.
> 2020-02-14 00:25:10,921 INFO org.apache.ratis.server.impl.LeaderElection:
> 285cac09-7622-45e6-be02-b3c68ebf8b10@group-0068FD2EA2C9-LeaderElection37033:
> Election REJECTED; received 0 response(s) [] and 2 exception(s);
> 285cac09-7622-45e6-be02-b3c68ebf8b10@group-0068FD2EA2C9:t6221, leader=null,
> voted=285cac09-7622-45e6-be02-b3c68ebf8b10,
> raftlog=285cac09-7622-45e6-be02-b3c68ebf8b10@group-0068FD2EA2C9-SegmentedRaftLog:OPENED:c0,f0,i8,
> conf=0: [d432c890-5ec4-4cf1-9078-28497a08ab85:10.65.6.227:9858,
> 285cac09-7622-45e6-be02-b3c68ebf8b10:10.65.24.80:9858,
> cabbdef8-ed6c-4fc7-b7b2-d1ddd07da47e:10.65.8.165:9858], old=null
> 2020-02-14 00:25:10,921 INFO org.apache.ratis.server.impl.LeaderElection:
> Exception 0: java.util.concurrent.ExecutionException:
> org.apache.ratis.thirdparty.io.grpc.StatusRuntimeException: INTERNAL:
> cabbdef8-ed6c-4fc7-b7b2-d1ddd07da47e: group-0068FD2EA2C9 not found.
> 2020-02-14 00:25:10,921 INFO org.apache.ratis.server.impl.LeaderElection:
> Exception 1: java.util.concurrent.ExecutionException:
> org.apache.ratis.thirdparty.io.grpc.StatusRuntimeException: INTERNAL:
> d432c890-5ec4-4cf1-9078-28497a08ab85: group-0068FD2EA2C9 not found.
> 2020-02-14 00:25:10,921 INFO org.apache.ratis.server.impl.RaftServerImpl:
> 285cac09-7622-45e6-be02-b3c68ebf8b10@group-0068FD2EA2C9: changes role from
> CANDIDATE to FOLLOWER at term 6221 for DISCOVERED_A_NEW_TERM
> 2020-02-14 00:25:10,921 INFO org.apache.ratis.server.impl.RoleInfo:
> 285cac09-7622-45e6-be02-b3c68ebf8b10: shutdown LeaderElection
> 2020-02-14 00:25:10,921 INFO org.apache.ratis.server.impl.RoleInfo:
> 285cac09-7622-45e6-be02-b3c68ebf8b10: start FollowerState
> 2020-02-14 00:25:11,134 WARN org.apache.ratis.grpc.server.GrpcLogAppender:
> 285cac09-7622-45e6-be02-b3c68ebf8b10@group-DD847EC75388->cabbdef8-ed6c-4fc7-b7b2-d1ddd07da47e-GrpcLogAppender:
> HEARTBEAT appendEntries Timeout,
> request=AppendEntriesRequest:cid=12669,entriesCount=0,lastEntry=null
> 2020-02-14 00:25:11,218 ERROR
> org.apache.hadoop.ozone.container.common.transport.server.ratis.XceiverServerRatis:
> pipeline Action CLOSE on pipeline
> PipelineID=7ad5ce51-d3fa-4e71-99f2-dd847ec75388.Reason :
> 285cac09-7622-45e6-be02-b3c68ebf8b10 has not seen follower/s
> d432c890-5ec4-4cf1-9078-28497a08ab85 for 31624453ms
> cabbdef8-ed6c-4fc7-b7b2-d1ddd07da47e for 31619344ms
> 2020-02-14 00:25:11,347 WARN org.apache.ratis.grpc.server.GrpcLogAppender:
> 285cac09-7622-45e6-be02-b3c68ebf8b10@group-2338B042C07B->d432c890-5ec4-4cf1-9078-28497a08ab85-GrpcLogAppender:
> HEARTBEAT appendEntries Timeout,
> request=AppendEntriesRequest:cid=12579,entriesCount=0,lastEntry=null
> 2020-02-14 00:25:11,361 WARN org.apache.ratis.grpc.server.GrpcLogAppender:
> 285cac09-7622-45e6-be02-b3c68ebf8b10@group-2338B042C07B->cabbdef8-ed6c-4fc7-b7b2-d1ddd07da47e-GrpcLogAppender:
> HEARTBEAT appendEntries Timeout,
> request=AppendEntriesRequest:cid=12577,entriesCount=0,lastEntry=null
> 2020-02-14 00:25:11,399 ERROR
> org.apache.hadoop.ozone.container.common.transport.server.ratis.XceiverServerRatis:
> pipeline Action CLOSE on pipeline
> PipelineID=6a851c59-0345-4ad8-ac31-2338b042c07b.Reason :
> 285cac09-7622-45e6-be02-b3c68ebf8b10 has not seen follower/s
> d432c890-5ec4-4cf1-9078-28497a08ab85 for 31396085ms
> cabbdef8-ed6c-4fc7-b7b2-d1ddd07da47e for 31391530ms
> 2020-02-14 00:25:11,406 ERROR
> org.apache.hadoop.ozone.container.common.transport.server.ratis.XceiverServerRatis:
> pipeline Action CLOSE on pipeline
> PipelineID=6a851c59-0345-4ad8-ac31-2338b042c07b.Reason :
> 285cac09-7622-45e6-be02-b3c68ebf8b10 has not seen follower/s
> d432c890-5ec4-4cf1-9078-28497a08ab85 for 31396092ms
> cabbdef8-ed6c-4fc7-b7b2-d1ddd07da47e for 31391537ms
> 2020-02-14 00:25:11,423 WARN org.apache.ratis.grpc.server.GrpcLogAppender:
> 285cac09-7622-45e6-be02-b3c68ebf8b10@group-BA1E8724EE74->d432c890-5ec4-4cf1-9078-28497a08ab85-GrpcLogAppender:
> HEARTBEAT appendEntries Timeout,
> request=AppendEntriesRequest:cid=12817,entriesCount=0,lastEntry=null
> 2020-02-14 00:25:11,490 ERROR
> org.apache.hadoop.ozone.container.common.transport.server.ratis.XceiverServerRatis:
> pipeline Action CLOSE on pipeline
> PipelineID=1ed1be53-b526-41af-bdf9-ba1e8724ee74.Reason :
> 285cac09-7622-45e6-be02-b3c68ebf8b10 has not seen follower/s
> d432c890-5ec4-4cf1-9078-28497a08ab85 for 31946345ms
> cabbdef8-ed6c-4fc7-b7b2-d1ddd07da47e for 31945978ms
> 2020-02-14 00:25:11,909 INFO org.apache.ratis.server.impl.FollowerState:
> 285cac09-7622-45e6-be02-b3c68ebf8b10@group-D506E1A1894E-FollowerState: change
> to CANDIDATE, lastRpcTime:5094ms, electionTimeout:5093ms
> 2020-02-14 00:25:11,909 INFO org.apache.ratis.server.impl.RoleInfo:
> 285cac09-7622-45e6-be02-b3c68ebf8b10: shutdown FollowerState
> {code}
--
This message was sent by Atlassian Jira
(v8.3.4#803005)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
