Nilotpal Nandi created HDDS-772: ----------------------------------- Summary: ratis retries infinitely and does not timeout when datanode goes down Key: HDDS-772 URL: https://issues.apache.org/jira/browse/HDDS-772 Project: Hadoop Distributed Data Store Issue Type: Bug Components: Ozone Client Affects Versions: 0.3.0 Reporter: Nilotpal Nandi
steps taken : ----------------- # Ran ozonefs client operations. # Some of the datanodes were down. # client operations did not fail and are in waiting/hung state. reason: RATIS retries infinitely. datanode.log ------------------------ {noformat} 2018-10-31 11:13:28,423 WARN org.apache.ratis.grpc.server.GrpcServerProtocolService: 046351fe-bb76-4f86-b296-c682746981c4: Failed requestVote 54026017-a738-45f5-92f9-c50a0fc24a9f->046351fe-bb76-4f86-b296-c682746981c4#0 org.apache.ratis.protocol.GroupMismatchException: 046351fe-bb76-4f86-b296-c682746981c4: group-FF58136AA1BA not found. at org.apache.ratis.server.impl.RaftServerProxy$ImplMap.get(RaftServerProxy.java:114) at org.apache.ratis.server.impl.RaftServerProxy.getImplFuture(RaftServerProxy.java:257) at org.apache.ratis.server.impl.RaftServerProxy.getImpl(RaftServerProxy.java:266) at org.apache.ratis.server.impl.RaftServerProxy.getImpl(RaftServerProxy.java:261) at org.apache.ratis.server.impl.RaftServerProxy.requestVote(RaftServerProxy.java:428) at org.apache.ratis.grpc.server.GrpcServerProtocolService.requestVote(GrpcServerProtocolService.java:54) at org.apache.ratis.proto.grpc.RaftServerProtocolServiceGrpc$MethodHandlers.invoke(RaftServerProtocolServiceGrpc.java:319) at org.apache.ratis.thirdparty.io.grpc.stub.ServerCalls$UnaryServerCallHandler$UnaryServerCallListener.onHalfClose(ServerCalls.java:171) at org.apache.ratis.thirdparty.io.grpc.internal.ServerCallImpl$ServerStreamListenerImpl.halfClosed(ServerCallImpl.java:283) at org.apache.ratis.thirdparty.io.grpc.internal.ServerImpl$JumpToApplicationThreadServerStreamListener$1HalfClosed.runInContext(ServerImpl.java:707) at org.apache.ratis.thirdparty.io.grpc.internal.ContextRunnable.run(ContextRunnable.java:37) at org.apache.ratis.thirdparty.io.grpc.internal.SerializingExecutor.run(SerializingExecutor.java:123) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 2018-10-31 11:13:29,574 WARN org.apache.ratis.grpc.server.GrpcServerProtocolService: 046351fe-bb76-4f86-b296-c682746981c4: Failed requestVote 54026017-a738-45f5-92f9-c50a0fc24a9f->046351fe-bb76-4f86-b296-c682746981c4#0 org.apache.ratis.protocol.GroupMismatchException: 046351fe-bb76-4f86-b296-c682746981c4: group-FF58136AA1BA not found. at org.apache.ratis.server.impl.RaftServerProxy$ImplMap.get(RaftServerProxy.java:114) at org.apache.ratis.server.impl.RaftServerProxy.getImplFuture(RaftServerProxy.java:257) at org.apache.ratis.server.impl.RaftServerProxy.getImpl(RaftServerProxy.java:266) at org.apache.ratis.server.impl.RaftServerProxy.getImpl(RaftServerProxy.java:261) at org.apache.ratis.server.impl.RaftServerProxy.requestVote(RaftServerProxy.java:428) at org.apache.ratis.grpc.server.GrpcServerProtocolService.requestVote(GrpcServerProtocolService.java:54) at org.apache.ratis.proto.grpc.RaftServerProtocolServiceGrpc$MethodHandlers.invoke(RaftServerProtocolServiceGrpc.java:319) at org.apache.ratis.thirdparty.io.grpc.stub.ServerCalls$UnaryServerCallHandler$UnaryServerCallListener.onHalfClose(ServerCalls.java:171) at org.apache.ratis.thirdparty.io.grpc.internal.ServerCallImpl$ServerStreamListenerImpl.halfClosed(ServerCallImpl.java:283) at org.apache.ratis.thirdparty.io.grpc.internal.ServerImpl$JumpToApplicationThreadServerStreamListener$1HalfClosed.runInContext(ServerImpl.java:707) at org.apache.ratis.thirdparty.io.grpc.internal.ContextRunnable.run(ContextRunnable.java:37) at org.apache.ratis.thirdparty.io.grpc.internal.SerializingExecutor.run(SerializingExecutor.java:123) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 2018-10-31 11:13:30,772 WARN org.apache.ratis.grpc.server.GrpcServerProtocolService: 046351fe-bb76-4f86-b296-c682746981c4: Failed requestVote 54026017-a738-45f5-92f9-c50a0fc24a9f->046351fe-bb76-4f86-b296-c682746981c4#0 org.apache.ratis.protocol.GroupMismatchException: 046351fe-bb76-4f86-b296-c682746981c4: group-FF58136AA1BA not found. at org.apache.ratis.server.impl.RaftServerProxy$ImplMap.get(RaftServerProxy.java:114) at org.apache.ratis.server.impl.RaftServerProxy.getImplFuture(RaftServerProxy.java:257) at org.apache.ratis.server.impl.RaftServerProxy.getImpl(RaftServerProxy.java:266) at org.apache.ratis.server.impl.RaftServerProxy.getImpl(RaftServerProxy.java:261) at org.apache.ratis.server.impl.RaftServerProxy.requestVote(RaftServerProxy.java:428) at org.apache.ratis.grpc.server.GrpcServerProtocolService.requestVote(GrpcServerProtocolService.java:54) at org.apache.ratis.proto.grpc.RaftServerProtocolServiceGrpc$MethodHandlers.invoke(RaftServerProtocolServiceGrpc.java:319) at org.apache.ratis.thirdparty.io.grpc.stub.ServerCalls$UnaryServerCallHandler$UnaryServerCallListener.onHalfClose(ServerCalls.java:171) at org.apache.ratis.thirdparty.io.grpc.internal.ServerCallImpl$ServerStreamListenerImpl.halfClosed(ServerCallImpl.java:283) at org.apache.ratis.thirdparty.io.grpc.internal.ServerImpl$JumpToApplicationThreadServerStreamListener$1HalfClosed.runInContext(ServerImpl.java:707) at org.apache.ratis.thirdparty.io.grpc.internal.ContextRunnable.run(ContextRunnable.java:37) at org.apache.ratis.thirdparty.io.grpc.internal.SerializingExecutor.run(SerializingExecutor.java:123) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) {noformat} -- This message was sent by Atlassian JIRA (v7.6.3#76005) --------------------------------------------------------------------- To unsubscribe, e-mail: hdfs-issues-unsubscr...@hadoop.apache.org For additional commands, e-mail: hdfs-issues-h...@hadoop.apache.org