[
https://issues.apache.org/jira/browse/HDDS-5017?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
Attila Doroszlai resolved HDDS-5017.
------------------------------------
Resolution: Fixed
> Grpc channel is not closed when there is exception during read/write
> --------------------------------------------------------------------
>
> Key: HDDS-5017
> URL: https://issues.apache.org/jira/browse/HDDS-5017
> Project: Apache Ozone
> Issue Type: Bug
> Reporter: Sammi Chen
> Priority: Critical
>
> There are many following logs in s3g.out. The reason is channel is not
> corrently closed when there is exception throwed during grpc client
> read/write. Refer to https://github.com/grpc/grpc-java/issues/4032.
> Mar 12, 2021 10:20:38 PM
> org.apache.ratis.thirdparty.io.grpc.internal.ManagedChannelOrphanWrapper$ManagedChannelReference
> cleanQueue
> SEVERE: *~*~*~ Channel ManagedChannelImpl{logId=329, target=9.180.8.67:9859}
> was not shutdown properly!!! ~*~*~*
> Make sure to call shutdown()/shutdownNow() and wait until
> awaitTermination() returns true.
> java.lang.RuntimeException: ManagedChannel allocation site
> at
> org.apache.ratis.thirdparty.io.grpc.internal.ManagedChannelOrphanWrapper$ManagedChannelReference.<init>(ManagedChannelOrphanWrapper.java:93)
> at
> org.apache.ratis.thirdparty.io.grpc.internal.ManagedChannelOrphanWrapper.<init>(ManagedChannelOrphanWrapper.java:53)
> at
> org.apache.ratis.thirdparty.io.grpc.internal.ManagedChannelOrphanWrapper.<init>(ManagedChannelOrphanWrapper.java:44)
> at
> org.apache.ratis.thirdparty.io.grpc.internal.AbstractManagedChannelImplBuilder.build(AbstractManagedChannelImplBuilder.java:518)
> at
> org.apache.hadoop.hdds.scm.XceiverClientGrpc.connectToDatanode(XceiverClientGrpc.java:193)
> at
> org.apache.hadoop.hdds.scm.XceiverClientGrpc.reconnect(XceiverClientGrpc.java:543)
> at
> org.apache.hadoop.hdds.scm.XceiverClientGrpc.checkOpen(XceiverClientGrpc.java:534)
> at
> org.apache.hadoop.hdds.scm.XceiverClientGrpc.sendCommandAsync(XceiverClientGrpc.java:462)
> at
> org.apache.hadoop.hdds.scm.XceiverClientGrpc.sendCommandWithRetry(XceiverClientGrpc.java:371)
> at
> org.apache.hadoop.hdds.scm.XceiverClientGrpc.lambda$sendCommandWithTraceIDAndRetry$0(XceiverClientGrpc.java:311)
> at
> org.apache.hadoop.hdds.tracing.TracingUtil.executeInSpan(TracingUtil.java:174)
> at
> org.apache.hadoop.hdds.tracing.TracingUtil.executeInNewSpan(TracingUtil.java:148)
> at
> org.apache.hadoop.hdds.scm.XceiverClientGrpc.sendCommandWithTraceIDAndRetry(XceiverClientGrpc.java:305)
> at
> org.apache.hadoop.hdds.scm.XceiverClientGrpc.sendCommand(XceiverClientGrpc.java:286)
> at
> org.apache.hadoop.hdds.scm.storage.ContainerProtocolCalls.getBlock(ContainerProtocolCalls.java:106)
> at
> org.apache.hadoop.hdds.scm.storage.BlockInputStream.getChunkInfos(BlockInputStream.java:213)
> at
> org.apache.hadoop.hdds.scm.storage.BlockInputStream.initialize(BlockInputStream.java:142)
> at
> org.apache.hadoop.hdds.scm.storage.BlockInputStream.read(BlockInputStream.java:276)
> at
> org.apache.hadoop.ozone.client.io.KeyInputStream.read(KeyInputStream.java:199)
> at
> org.apache.hadoop.ozone.client.io.OzoneInputStream.read(OzoneInputStream.java:49)
> at java.io.InputStream.read(InputStream.java:101)
> at org.apache.commons.io.IOUtils.copyLarge(IOUtils.java:2146)
> at org.apache.commons.io.IOUtils.copy(IOUtils.java:2102)
> at org.apache.commons.io.IOUtils.copyLarge(IOUtils.java:2123)
> at org.apache.commons.io.IOUtils.copy(IOUtils.java:2078)
> One example read failure stack
> Caused by:
> org.apache.hadoop.hdds.scm.container.common.helpers.StorageContainerException:
> The container(404197) replica is unhealthy.
> at
> org.apache.hadoop.hdds.scm.storage.ContainerProtocolCalls.validateContainerResponse(ContainerProtocolCalls.java:530)
> at
> org.apache.hadoop.hdds.scm.storage.ContainerProtocolCalls.lambda$getValidatorList$0(ContainerProtocolCalls.java:537)
> at
> org.apache.hadoop.hdds.scm.XceiverClientGrpc.sendCommandWithRetry(XceiverClientGrpc.java:374)
> at
> org.apache.hadoop.hdds.scm.XceiverClientGrpc.lambda$sendCommandWithTraceIDAndRetry$0(XceiverClientGrpc.java:311)
> at
> org.apache.hadoop.hdds.tracing.TracingUtil.executeInSpan(TracingUtil.java:174)
> at
> org.apache.hadoop.hdds.tracing.TracingUtil.executeInNewSpan(TracingUtil.java:148)
> at
> org.apache.hadoop.hdds.scm.XceiverClientGrpc.sendCommandWithTraceIDAndRetry(XceiverClientGrpc.java:305)
> at
> org.apache.hadoop.hdds.scm.XceiverClientGrpc.sendCommand(XceiverClientGrpc.java:286)
> at
> org.apache.hadoop.hdds.scm.storage.ContainerProtocolCalls.getBlock(ContainerProtocolCalls.java:106)
> at
> org.apache.hadoop.hdds.scm.storage.BlockInputStream.getChunkInfos(BlockInputStream.java:213)
> at
> org.apache.hadoop.hdds.scm.storage.BlockInputStream.initialize(BlockInputStream.java:142)
> at
> org.apache.hadoop.hdds.scm.storage.BlockInputStream.read(BlockInputStream.java:276)
> at
> org.apache.hadoop.ozone.client.io.KeyInputStream.read(KeyInputStream.java:199)
> at
> org.apache.hadoop.ozone.client.io.OzoneInputStream.read(OzoneInputStream.java:49)
> at java.io.InputStream.read(InputStream.java:101)
> at org.apache.commons.io.IOUtils.copyLarge(IOUtils.java:2146)
> at org.apache.commons.io.IOUtils.copy(IOUtils.java:2102)
> at org.apache.commons.io.IOUtils.copyLarge(IOUtils.java:2123)
> at org.apache.commons.io.IOUtils.copy(IOUtils.java:2078)
> at
> org.apache.hadoop.ozone.s3.endpoint.ObjectEndpoint.lambda$get$0(ObjectEndpoint.java:278)
> at
> org.glassfish.jersey.message.internal.StreamingOutputProvider.writeTo(StreamingOutputProvider.java:79)
> at
> org.glassfish.jersey.message.internal.StreamingOutputProvider.writeTo(StreamingOutputProvider.java:61)
> at
> org.glassfish.jersey.message.internal.WriterInterceptorExecutor$TerminalWriterInterceptor.invokeWriteTo(WriterInterceptorExecutor.java:266)
> at
> org.glassfish.jersey.message.internal.WriterInterceptorExecutor$TerminalWriterInterceptor.aroundWriteTo(WriterInterceptorExecutor.java:251)
> at
> org.glassfish.jersey.message.internal.WriterInterceptorExecutor.proceed(WriterInterceptorExecutor.java:163)
> at
> org.glassfish.jersey.server.internal.JsonWithPaddingInterceptor.aroundWriteTo(JsonWithPaddingInterceptor.java:109)
> at
> org.glassfish.jersey.message.internal.WriterInterceptorExecutor.proceed(WriterInterceptorExecutor.java:163)
> at
> org.glassfish.jersey.server.internal.MappableExceptionWrapperInterceptor.aroundWriteTo(MappableExceptionWrapperInterceptor.java:85)
> at
> org.glassfish.jersey.message.internal.WriterInterceptorExecutor.proceed(WriterInterceptorExecutor.java:163)
> at
> org.glassfish.jersey.message.internal.MessageBodyFactory.writeTo(MessageBodyFactory.java:1135)
> at
> org.glassfish.jersey.server.ServerRuntime$Responder.writeResponse(ServerRuntime.java:662)
> at
> org.glassfish.jersey.server.ServerRuntime$Responder.processResponse(ServerRuntime.java:395)
--
This message was sent by Atlassian Jira
(v8.20.10#820010)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
