Jyotirmoy Sinha created HDDS-8546:
-------------------------------------
Summary: Ozone debug chunkinfo command throwing
GroupMismatchException after container is closed
Key: HDDS-8546
URL: https://issues.apache.org/jira/browse/HDDS-8546
Project: Apache Ozone
Issue Type: Bug
Components: Ozone CLI
Reporter: Jyotirmoy Sinha
Steps :
# Create volume, bucket (ratis), key
# Close container of the above key
# Run 'ozone debug chunkinfo <containerid>' for above container.
Exception stacktrace -
{code:java}
[root@jspriv02-1 ~]# ozone debug chunkinfo vol3/buck1/key1
23/04/12 08:17:10 WARN impl.MetricsConfig: Cannot locate configuration: tried
hadoop-metrics2-xceiverclientmetrics.properties,hadoop-metrics2.properties
23/04/12 08:17:10 INFO impl.MetricsSystemImpl: Scheduled Metric snapshot period
at 10 second(s).
23/04/12 08:17:10 INFO impl.MetricsSystemImpl: XceiverClientMetrics metrics
system started
23/04/12 08:17:11 INFO metrics.MetricRegistries: Loaded MetricRegistries class
org.apache.ratis.metrics.impl.MetricRegistriesImpl
23/04/12 08:17:11 ERROR impl.OrderedAsync: Failed to send request, message=null
java.util.concurrent.CompletionException:
org.apache.ratis.protocol.exceptions.GroupMismatchException:
2abf8d6e-0c47-4763-829c-375b92fd3fa4: group-68BBA540DE5D not found.
at
java.util.concurrent.CompletableFuture.encodeThrowable(CompletableFuture.java:292)
at
java.util.concurrent.CompletableFuture.completeThrowable(CompletableFuture.java:308)
at
java.util.concurrent.CompletableFuture.uniApply(CompletableFuture.java:607)
at
java.util.concurrent.CompletableFuture$UniApply.tryFire(CompletableFuture.java:591)
at
java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:488)
at
java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:1990)
at
org.apache.ratis.client.impl.OrderedAsync$PendingOrderedRequest.fail(OrderedAsync.java:104)
at org.apache.ratis.util.SlidingWindow$Client.fail(SlidingWindow.java:392)
at
org.apache.ratis.client.impl.OrderedAsync.failAllAsyncRequests(OrderedAsync.java:148)
at
org.apache.ratis.client.impl.OrderedAsync.lambda$sendRequest$10(OrderedAsync.java:273)
at
java.util.concurrent.CompletableFuture.uniExceptionally(CompletableFuture.java:884)
at
java.util.concurrent.CompletableFuture$UniExceptionally.tryFire(CompletableFuture.java:866)
at
java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:488)
at
java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:1990)
at
org.apache.ratis.grpc.client.GrpcClientProtocolClient$AsyncStreamObservers.completeReplyExceptionally(GrpcClientProtocolClient.java:394)
at
org.apache.ratis.grpc.client.GrpcClientProtocolClient$AsyncStreamObservers.access$000(GrpcClientProtocolClient.java:300)
at
org.apache.ratis.grpc.client.GrpcClientProtocolClient$AsyncStreamObservers$1.onError(GrpcClientProtocolClient.java:331)
at
org.apache.ratis.thirdparty.io.grpc.stub.ClientCalls$StreamObserverToCallListenerAdapter.onClose(ClientCalls.java:487)
at
org.apache.ratis.thirdparty.io.grpc.PartialForwardingClientCallListener.onClose(PartialForwardingClientCallListener.java:39)
at
org.apache.ratis.thirdparty.io.grpc.ForwardingClientCallListener.onClose(ForwardingClientCallListener.java:23)
at
org.apache.ratis.grpc.metrics.intercept.client.MetricClientCallListener.onClose(MetricClientCallListener.java:47)
at
org.apache.ratis.thirdparty.io.grpc.internal.ClientCallImpl.closeObserver(ClientCallImpl.java:562)
at
org.apache.ratis.thirdparty.io.grpc.internal.ClientCallImpl.access$300(ClientCallImpl.java:70)
at
org.apache.ratis.thirdparty.io.grpc.internal.ClientCallImpl$ClientStreamListenerImpl$1StreamClosed.runInternal(ClientCallImpl.java:743)
at
org.apache.ratis.thirdparty.io.grpc.internal.ClientCallImpl$ClientStreamListenerImpl$1StreamClosed.runInContext(ClientCallImpl.java:722)
at
org.apache.ratis.thirdparty.io.grpc.internal.ContextRunnable.run(ContextRunnable.java:37)
at
org.apache.ratis.thirdparty.io.grpc.internal.SerializingExecutor.run(SerializingExecutor.java:133)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
Caused by: org.apache.ratis.protocol.exceptions.GroupMismatchException:
2abf8d6e-0c47-4763-829c-375b92fd3fa4: group-68BBA540DE5D not found.
at
org.apache.ratis.server.impl.RaftServerProxy$ImplMap.get(RaftServerProxy.java:150)
at
org.apache.ratis.server.impl.RaftServerProxy.getImplFuture(RaftServerProxy.java:351)
at
org.apache.ratis.server.impl.RaftServerProxy.submitClientRequestAsync(RaftServerProxy.java:435)
at
org.apache.ratis.grpc.server.GrpcClientProtocolService$RequestStreamObserver.processClientRequest(GrpcClientProtocolService.java:226)
at
org.apache.ratis.grpc.server.GrpcClientProtocolService$OrderedRequestStreamObserver.processClientRequest(GrpcClientProtocolService.java:332)
at
org.apache.ratis.util.SlidingWindow$Server.processRequestsFromHead(SlidingWindow.java:459)
at
org.apache.ratis.util.SlidingWindow$Server.receivedRequest(SlidingWindow.java:451)
at
org.apache.ratis.grpc.server.GrpcClientProtocolService$OrderedRequestStreamObserver.processClientRequest(GrpcClientProtocolService.java:356)
at
org.apache.ratis.grpc.server.GrpcClientProtocolService$RequestStreamObserver.onNext(GrpcClientProtocolService.java:246)
at
org.apache.ratis.grpc.server.GrpcClientProtocolService$RequestStreamObserver.onNext(GrpcClientProtocolService.java:169)
at
org.apache.ratis.thirdparty.io.grpc.stub.ServerCalls$StreamingServerCallHandler$StreamingServerCallListener.onMessage(ServerCalls.java:262)
at
org.apache.ratis.thirdparty.io.grpc.ForwardingServerCallListener.onMessage(ForwardingServerCallListener.java:33)
at
org.apache.ratis.thirdparty.io.grpc.internal.ServerCallImpl$ServerStreamListenerImpl.messagesAvailableInternal(ServerCallImpl.java:332)
at
org.apache.ratis.thirdparty.io.grpc.internal.ServerCallImpl$ServerStreamListenerImpl.messagesAvailable(ServerCallImpl.java:315)
at
org.apache.ratis.thirdparty.io.grpc.internal.ServerImpl$JumpToApplicationThreadServerStreamListener$1MessagesAvailable.runInContext(ServerImpl.java:834)
... 5 more
23/04/12 08:17:11 ERROR impl.OrderedAsync: Failed to send request,
message=cmdType: ReadContainer
traceID: ""
containerID: 10002
datanodeUuid: "2abf8d6e-0c47-4763-829c-375b92fd3fa4"
readContainer {
}
encodedToken:
"MgoRc2NtQFJPT1QuSFdYLlNJVEUSAwiSThiavcLN9zAiETgxOTcyNDM5OTMyNjY4MjEyjgEAMp4dNIZBQN_X9eUBhOkBrowU4HuU6fo7UHtifWCDIxgz0op8qDTv78aoV8les-04Mmjjcx-LvjqbUDiDYg7gr7skz5U9iAZq0oc9glVJVlQZGZTl0_uBfS9HVuAwpM_6QESPY_9wzvj2z6o8uepF3ebcOtv8y9jq31ImC98RymrJaAepU4S8wq2jP71xtUrh5gjo4aoQaxtctc9K6QK-VO1iDeBHfxuRBDPwuBs_D8TRF1CNNoKXNJ-BCWOaY5WlQc0XlYP4mYP8YNGX67zzG3KMyzlxfx4k-Ka_aFXVX5xkYueaOQYFy46FVsJDtm7jE1di21Anb67dlc9WeHM9gBRIRERTX0NPTlRBSU5FUl9UT0tFTgYjMTAwMDI"
, data.size=0
java.util.concurrent.CompletionException:
org.apache.ratis.protocol.exceptions.AlreadyClosedException:
SlidingWindow$Client:client-554B5E583B90->RAFT is closed.
at
java.util.concurrent.CompletableFuture.encodeThrowable(CompletableFuture.java:292)
at
java.util.concurrent.CompletableFuture.completeThrowable(CompletableFuture.java:308)
at
java.util.concurrent.CompletableFuture.uniApply(CompletableFuture.java:607)
at
java.util.concurrent.CompletableFuture$UniApply.tryFire(CompletableFuture.java:591)
at
java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:488)
at
java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:1990)
at
org.apache.ratis.client.impl.OrderedAsync$PendingOrderedRequest.fail(OrderedAsync.java:104)
at
org.apache.ratis.util.SlidingWindow$Client.alreadyClosed(SlidingWindow.java:406)
at org.apache.ratis.util.SlidingWindow$Client.fail(SlidingWindow.java:394)
at
org.apache.ratis.client.impl.OrderedAsync.failAllAsyncRequests(OrderedAsync.java:148)
at
org.apache.ratis.client.impl.OrderedAsync.lambda$sendRequest$10(OrderedAsync.java:273)
at
java.util.concurrent.CompletableFuture.uniExceptionally(CompletableFuture.java:884)
at
java.util.concurrent.CompletableFuture$UniExceptionally.tryFire(CompletableFuture.java:866)
at
java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:488)
at
java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:1990)
at
org.apache.ratis.grpc.client.GrpcClientProtocolClient$AsyncStreamObservers.completeReplyExceptionally(GrpcClientProtocolClient.java:394)
at
org.apache.ratis.grpc.client.GrpcClientProtocolClient$AsyncStreamObservers.access$000(GrpcClientProtocolClient.java:300)
at
org.apache.ratis.grpc.client.GrpcClientProtocolClient$AsyncStreamObservers$1.onError(GrpcClientProtocolClient.java:331)
at
org.apache.ratis.thirdparty.io.grpc.stub.ClientCalls$StreamObserverToCallListenerAdapter.onClose(ClientCalls.java:487)
at
org.apache.ratis.thirdparty.io.grpc.PartialForwardingClientCallListener.onClose(PartialForwardingClientCallListener.java:39)
at
org.apache.ratis.thirdparty.io.grpc.ForwardingClientCallListener.onClose(ForwardingClientCallListener.java:23)
at
org.apache.ratis.grpc.metrics.intercept.client.MetricClientCallListener.onClose(MetricClientCallListener.java:47)
at
org.apache.ratis.thirdparty.io.grpc.internal.ClientCallImpl.closeObserver(ClientCallImpl.java:562)
at
org.apache.ratis.thirdparty.io.grpc.internal.ClientCallImpl.access$300(ClientCallImpl.java:70)
at
org.apache.ratis.thirdparty.io.grpc.internal.ClientCallImpl$ClientStreamListenerImpl$1StreamClosed.runInternal(ClientCallImpl.java:743)
at
org.apache.ratis.thirdparty.io.grpc.internal.ClientCallImpl$ClientStreamListenerImpl$1StreamClosed.runInContext(ClientCallImpl.java:722)
at
org.apache.ratis.thirdparty.io.grpc.internal.ContextRunnable.run(ContextRunnable.java:37)
at
org.apache.ratis.thirdparty.io.grpc.internal.SerializingExecutor.run(SerializingExecutor.java:133)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
Caused by: org.apache.ratis.protocol.exceptions.AlreadyClosedException:
SlidingWindow$Client:client-554B5E583B90->RAFT is closed.
... 24 more
Caused by: org.apache.ratis.protocol.exceptions.GroupMismatchException:
2abf8d6e-0c47-4763-829c-375b92fd3fa4: group-68BBA540DE5D not found.
at
org.apache.ratis.server.impl.RaftServerProxy$ImplMap.get(RaftServerProxy.java:150)
at
org.apache.ratis.server.impl.RaftServerProxy.getImplFuture(RaftServerProxy.java:351)
at
org.apache.ratis.server.impl.RaftServerProxy.submitClientRequestAsync(RaftServerProxy.java:435)
at
org.apache.ratis.grpc.server.GrpcClientProtocolService$RequestStreamObserver.processClientRequest(GrpcClientProtocolService.java:226)
at
org.apache.ratis.grpc.server.GrpcClientProtocolService$OrderedRequestStreamObserver.processClientRequest(GrpcClientProtocolService.java:332)
at
org.apache.ratis.util.SlidingWindow$Server.processRequestsFromHead(SlidingWindow.java:459)
at
org.apache.ratis.util.SlidingWindow$Server.receivedRequest(SlidingWindow.java:451)
at
org.apache.ratis.grpc.server.GrpcClientProtocolService$OrderedRequestStreamObserver.processClientRequest(GrpcClientProtocolService.java:356)
at
org.apache.ratis.grpc.server.GrpcClientProtocolService$RequestStreamObserver.onNext(GrpcClientProtocolService.java:246)
at
org.apache.ratis.grpc.server.GrpcClientProtocolService$RequestStreamObserver.onNext(GrpcClientProtocolService.java:169)
at
org.apache.ratis.thirdparty.io.grpc.stub.ServerCalls$StreamingServerCallHandler$StreamingServerCallListener.onMessage(ServerCalls.java:262)
at
org.apache.ratis.thirdparty.io.grpc.ForwardingServerCallListener.onMessage(ForwardingServerCallListener.java:33)
at
org.apache.ratis.thirdparty.io.grpc.internal.ServerCallImpl$ServerStreamListenerImpl.messagesAvailableInternal(ServerCallImpl.java:332)
at
org.apache.ratis.thirdparty.io.grpc.internal.ServerCallImpl$ServerStreamListenerImpl.messagesAvailable(ServerCallImpl.java:315)
at
org.apache.ratis.thirdparty.io.grpc.internal.ServerImpl$JumpToApplicationThreadServerStreamListener$1MessagesAvailable.runInContext(ServerImpl.java:834)
... 5 more
Failed to execute command cmdType: ReadContainer {code}
--
This message was sent by Atlassian Jira
(v8.20.10#820010)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
