[
https://issues.apache.org/jira/browse/HDDS-8780?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
Duong updated HDDS-8780:
------------------------
Description:
This is caught from an [integration
test|https://github.com/apache/ozone/actions/runs/5140040821?pr=4747] with
*secure* MiniOzoneHAClusterImpl.
{code:java}
Jun 07, 2023 4:29:35 PM
org.apache.ratis.thirdparty.io.grpc.internal.ManagedChannelOrphanWrapper$ManagedChannelReference
cleanQueue
SEVERE: *~*~*~ Previous channel ManagedChannelImpl{logId=538,
target=localhost:65138} was not shutdown properly!!! ~*~*~*
Make sure to call shutdown()/shutdownNow() and wait until
awaitTermination() returns true.
java.lang.RuntimeException: ManagedChannel allocation site
at
org.apache.ratis.thirdparty.io.grpc.internal.ManagedChannelOrphanWrapper$ManagedChannelReference.<init>(ManagedChannelOrphanWrapper.java:100)
at
org.apache.ratis.thirdparty.io.grpc.internal.ManagedChannelOrphanWrapper.<init>(ManagedChannelOrphanWrapper.java:60)
at
org.apache.ratis.thirdparty.io.grpc.internal.ManagedChannelOrphanWrapper.<init>(ManagedChannelOrphanWrapper.java:51)
at
org.apache.ratis.thirdparty.io.grpc.internal.ManagedChannelImplBuilder.build(ManagedChannelImplBuilder.java:631)
at
org.apache.ratis.thirdparty.io.grpc.internal.AbstractManagedChannelImplBuilder.build(AbstractManagedChannelImplBuilder.java:297)
at
org.apache.ratis.grpc.client.GrpcClientProtocolClient.buildChannel(GrpcClientProtocolClient.java:151)
at
org.apache.ratis.grpc.client.GrpcClientProtocolClient.<init>(GrpcClientProtocolClient.java:113)
at
org.apache.ratis.grpc.client.GrpcClientRpc.lambda$new$0(GrpcClientRpc.java:59)
at
org.apache.ratis.util.PeerProxyMap$PeerAndProxy.lambda$getProxy$0(PeerProxyMap.java:64)
at org.apache.ratis.util.LifeCycle.startAndTransition(LifeCycle.java:270)
at
org.apache.ratis.util.PeerProxyMap$PeerAndProxy.getProxy(PeerProxyMap.java:63)
at org.apache.ratis.util.PeerProxyMap.getProxy(PeerProxyMap.java:115)
at
org.apache.ratis.grpc.client.GrpcClientRpc.sendRequestAsync(GrpcClientRpc.java:69)
at
org.apache.ratis.client.impl.OrderedAsync.sendRequest(OrderedAsync.java:245)
at
org.apache.ratis.client.impl.OrderedAsync.sendRequestWithRetry(OrderedAsync.java:198)
at
org.apache.ratis.util.SlidingWindow$Client.sendOrDelayRequest(SlidingWindow.java:312)
at
org.apache.ratis.util.SlidingWindow$Client.submitNewRequest(SlidingWindow.java:291)
at org.apache.ratis.client.impl.OrderedAsync.send(OrderedAsync.java:170)
at
org.apache.ratis.client.impl.OrderedAsync.newInstance(OrderedAsync.java:118)
at
org.apache.ratis.client.impl.RaftClientImpl.lambda$new$1(RaftClientImpl.java:166)
at org.apache.ratis.util.MemoizedSupplier.get(MemoizedSupplier.java:62)
at
org.apache.ratis.client.impl.RaftClientImpl.getOrderedAsync(RaftClientImpl.java:235)
at org.apache.ratis.client.impl.AsyncImpl.send(AsyncImpl.java:41)
at org.apache.ratis.client.impl.AsyncImpl.send(AsyncImpl.java:46)
at
org.apache.hadoop.hdds.scm.ha.HASecurityUtils.submitScmCertsToRatis(HASecurityUtils.java:357)
at
org.apache.hadoop.hdds.scm.ha.SCMHAInvocationHandler.invokeRatis(SCMHAInvocationHandler.java:109)
at
org.apache.hadoop.hdds.scm.ha.SCMHAInvocationHandler.invoke(SCMHAInvocationHandler.java:71)
at com.sun.proxy.$Proxy34.storeValidCertificate(Unknown Source)
at
org.apache.hadoop.hdds.security.x509.certificate.authority.DefaultCAServer.signAndStoreCertificate(DefaultCAServer.java:300)
at
org.apache.hadoop.hdds.security.x509.certificate.authority.DefaultCAServer.requestCertificate(DefaultCAServer.java:260)
at
org.apache.hadoop.hdds.security.x509.certificate.authority.DefaultCAServer.requestCertificate(DefaultCAServer.java:315)
at
org.apache.hadoop.hdds.scm.server.SCMSecurityProtocolServer.getEncodedCertToString(SCMSecurityProtocolServer.java:288)
at
org.apache.hadoop.hdds.scm.server.SCMSecurityProtocolServer.getSCMCertificate(SCMSecurityProtocolServer.java:274)
at
org.apache.hadoop.hdds.scm.protocol.SCMSecurityProtocolServerSideTranslatorPB.getSCMCertificate(SCMSecurityProtocolServerSideTranslatorPB.java:252)
at
org.apache.hadoop.hdds.scm.protocol.SCMSecurityProtocolServerSideTranslatorPB.processRequest(SCMSecurityProtocolServerSideTranslatorPB.java:124)
at
org.apache.hadoop.hdds.server.OzoneProtocolMessageDispatcher.processRequest(OzoneProtocolMessageDispatcher.java:87)
at
org.apache.hadoop.hdds.scm.protocol.SCMSecurityProtocolServerSideTranslatorPB.submitRequest(SCMSecurityProtocolServerSideTranslatorPB.java:94)
at
org.apache.hadoop.hdds.protocol.proto.SCMSecurityProtocolProtos$SCMSecurityProtocolService$2.callBlockingMethod(SCMSecurityProtocolProtos.java:16080)
at
org.apache.hadoop.ipc.ProtobufRpcEngine$Server.processCall(ProtobufRpcEngine.java:484)
at
org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:595)
at
org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1213)
at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1089)
at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1012)
at java.security.AccessController.doPrivileged(Native Method)
at javax.security.auth.Subject.doAs(Subject.java:422)
at
org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1899)
at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3026){code}
was:
This is caught from an [integration
test|https://github.com/apache/ozone/actions/runs/5140040821?pr=4747] with
*secure* MiniOzone*HA*ClusterImpl.
{code:java}
Jun 07, 2023 4:29:35 PM
org.apache.ratis.thirdparty.io.grpc.internal.ManagedChannelOrphanWrapper$ManagedChannelReference
cleanQueue
SEVERE: *~*~*~ Previous channel ManagedChannelImpl{logId=538,
target=localhost:65138} was not shutdown properly!!! ~*~*~*
Make sure to call shutdown()/shutdownNow() and wait until
awaitTermination() returns true.
java.lang.RuntimeException: ManagedChannel allocation site
at
org.apache.ratis.thirdparty.io.grpc.internal.ManagedChannelOrphanWrapper$ManagedChannelReference.<init>(ManagedChannelOrphanWrapper.java:100)
at
org.apache.ratis.thirdparty.io.grpc.internal.ManagedChannelOrphanWrapper.<init>(ManagedChannelOrphanWrapper.java:60)
at
org.apache.ratis.thirdparty.io.grpc.internal.ManagedChannelOrphanWrapper.<init>(ManagedChannelOrphanWrapper.java:51)
at
org.apache.ratis.thirdparty.io.grpc.internal.ManagedChannelImplBuilder.build(ManagedChannelImplBuilder.java:631)
at
org.apache.ratis.thirdparty.io.grpc.internal.AbstractManagedChannelImplBuilder.build(AbstractManagedChannelImplBuilder.java:297)
at
org.apache.ratis.grpc.client.GrpcClientProtocolClient.buildChannel(GrpcClientProtocolClient.java:151)
at
org.apache.ratis.grpc.client.GrpcClientProtocolClient.<init>(GrpcClientProtocolClient.java:113)
at
org.apache.ratis.grpc.client.GrpcClientRpc.lambda$new$0(GrpcClientRpc.java:59)
at
org.apache.ratis.util.PeerProxyMap$PeerAndProxy.lambda$getProxy$0(PeerProxyMap.java:64)
at org.apache.ratis.util.LifeCycle.startAndTransition(LifeCycle.java:270)
at
org.apache.ratis.util.PeerProxyMap$PeerAndProxy.getProxy(PeerProxyMap.java:63)
at org.apache.ratis.util.PeerProxyMap.getProxy(PeerProxyMap.java:115)
at
org.apache.ratis.grpc.client.GrpcClientRpc.sendRequestAsync(GrpcClientRpc.java:69)
at
org.apache.ratis.client.impl.OrderedAsync.sendRequest(OrderedAsync.java:245)
at
org.apache.ratis.client.impl.OrderedAsync.sendRequestWithRetry(OrderedAsync.java:198)
at
org.apache.ratis.util.SlidingWindow$Client.sendOrDelayRequest(SlidingWindow.java:312)
at
org.apache.ratis.util.SlidingWindow$Client.submitNewRequest(SlidingWindow.java:291)
at org.apache.ratis.client.impl.OrderedAsync.send(OrderedAsync.java:170)
at
org.apache.ratis.client.impl.OrderedAsync.newInstance(OrderedAsync.java:118)
at
org.apache.ratis.client.impl.RaftClientImpl.lambda$new$1(RaftClientImpl.java:166)
at org.apache.ratis.util.MemoizedSupplier.get(MemoizedSupplier.java:62)
at
org.apache.ratis.client.impl.RaftClientImpl.getOrderedAsync(RaftClientImpl.java:235)
at org.apache.ratis.client.impl.AsyncImpl.send(AsyncImpl.java:41)
at org.apache.ratis.client.impl.AsyncImpl.send(AsyncImpl.java:46)
at
org.apache.hadoop.hdds.scm.ha.HASecurityUtils.submitScmCertsToRatis(HASecurityUtils.java:357)
at
org.apache.hadoop.hdds.scm.ha.SCMHAInvocationHandler.invokeRatis(SCMHAInvocationHandler.java:109)
at
org.apache.hadoop.hdds.scm.ha.SCMHAInvocationHandler.invoke(SCMHAInvocationHandler.java:71)
at com.sun.proxy.$Proxy34.storeValidCertificate(Unknown Source)
at
org.apache.hadoop.hdds.security.x509.certificate.authority.DefaultCAServer.signAndStoreCertificate(DefaultCAServer.java:300)
at
org.apache.hadoop.hdds.security.x509.certificate.authority.DefaultCAServer.requestCertificate(DefaultCAServer.java:260)
at
org.apache.hadoop.hdds.security.x509.certificate.authority.DefaultCAServer.requestCertificate(DefaultCAServer.java:315)
at
org.apache.hadoop.hdds.scm.server.SCMSecurityProtocolServer.getEncodedCertToString(SCMSecurityProtocolServer.java:288)
at
org.apache.hadoop.hdds.scm.server.SCMSecurityProtocolServer.getSCMCertificate(SCMSecurityProtocolServer.java:274)
at
org.apache.hadoop.hdds.scm.protocol.SCMSecurityProtocolServerSideTranslatorPB.getSCMCertificate(SCMSecurityProtocolServerSideTranslatorPB.java:252)
at
org.apache.hadoop.hdds.scm.protocol.SCMSecurityProtocolServerSideTranslatorPB.processRequest(SCMSecurityProtocolServerSideTranslatorPB.java:124)
at
org.apache.hadoop.hdds.server.OzoneProtocolMessageDispatcher.processRequest(OzoneProtocolMessageDispatcher.java:87)
at
org.apache.hadoop.hdds.scm.protocol.SCMSecurityProtocolServerSideTranslatorPB.submitRequest(SCMSecurityProtocolServerSideTranslatorPB.java:94)
at
org.apache.hadoop.hdds.protocol.proto.SCMSecurityProtocolProtos$SCMSecurityProtocolService$2.callBlockingMethod(SCMSecurityProtocolProtos.java:16080)
at
org.apache.hadoop.ipc.ProtobufRpcEngine$Server.processCall(ProtobufRpcEngine.java:484)
at
org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:595)
at
org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1213)
at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1089)
at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1012)
at java.security.AccessController.doPrivileged(Native Method)
at javax.security.auth.Subject.doAs(Subject.java:422)
at
org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1899)
at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3026){code}
> Leak of ManagedChannel in HASecurityUtils
> -----------------------------------------
>
> Key: HDDS-8780
> URL: https://issues.apache.org/jira/browse/HDDS-8780
> Project: Apache Ozone
> Issue Type: Improvement
> Reporter: Duong
> Assignee: Duong
> Priority: Major
>
> This is caught from an [integration
> test|https://github.com/apache/ozone/actions/runs/5140040821?pr=4747] with
> *secure* MiniOzoneHAClusterImpl.
>
> {code:java}
> Jun 07, 2023 4:29:35 PM
> org.apache.ratis.thirdparty.io.grpc.internal.ManagedChannelOrphanWrapper$ManagedChannelReference
> cleanQueue
> SEVERE: *~*~*~ Previous channel ManagedChannelImpl{logId=538,
> target=localhost:65138} was not shutdown properly!!! ~*~*~*
> Make sure to call shutdown()/shutdownNow() and wait until
> awaitTermination() returns true.
> java.lang.RuntimeException: ManagedChannel allocation site
> at
> org.apache.ratis.thirdparty.io.grpc.internal.ManagedChannelOrphanWrapper$ManagedChannelReference.<init>(ManagedChannelOrphanWrapper.java:100)
> at
> org.apache.ratis.thirdparty.io.grpc.internal.ManagedChannelOrphanWrapper.<init>(ManagedChannelOrphanWrapper.java:60)
> at
> org.apache.ratis.thirdparty.io.grpc.internal.ManagedChannelOrphanWrapper.<init>(ManagedChannelOrphanWrapper.java:51)
> at
> org.apache.ratis.thirdparty.io.grpc.internal.ManagedChannelImplBuilder.build(ManagedChannelImplBuilder.java:631)
> at
> org.apache.ratis.thirdparty.io.grpc.internal.AbstractManagedChannelImplBuilder.build(AbstractManagedChannelImplBuilder.java:297)
> at
> org.apache.ratis.grpc.client.GrpcClientProtocolClient.buildChannel(GrpcClientProtocolClient.java:151)
> at
> org.apache.ratis.grpc.client.GrpcClientProtocolClient.<init>(GrpcClientProtocolClient.java:113)
> at
> org.apache.ratis.grpc.client.GrpcClientRpc.lambda$new$0(GrpcClientRpc.java:59)
> at
> org.apache.ratis.util.PeerProxyMap$PeerAndProxy.lambda$getProxy$0(PeerProxyMap.java:64)
> at org.apache.ratis.util.LifeCycle.startAndTransition(LifeCycle.java:270)
> at
> org.apache.ratis.util.PeerProxyMap$PeerAndProxy.getProxy(PeerProxyMap.java:63)
> at org.apache.ratis.util.PeerProxyMap.getProxy(PeerProxyMap.java:115)
> at
> org.apache.ratis.grpc.client.GrpcClientRpc.sendRequestAsync(GrpcClientRpc.java:69)
> at
> org.apache.ratis.client.impl.OrderedAsync.sendRequest(OrderedAsync.java:245)
> at
> org.apache.ratis.client.impl.OrderedAsync.sendRequestWithRetry(OrderedAsync.java:198)
> at
> org.apache.ratis.util.SlidingWindow$Client.sendOrDelayRequest(SlidingWindow.java:312)
> at
> org.apache.ratis.util.SlidingWindow$Client.submitNewRequest(SlidingWindow.java:291)
> at org.apache.ratis.client.impl.OrderedAsync.send(OrderedAsync.java:170)
> at
> org.apache.ratis.client.impl.OrderedAsync.newInstance(OrderedAsync.java:118)
> at
> org.apache.ratis.client.impl.RaftClientImpl.lambda$new$1(RaftClientImpl.java:166)
> at org.apache.ratis.util.MemoizedSupplier.get(MemoizedSupplier.java:62)
> at
> org.apache.ratis.client.impl.RaftClientImpl.getOrderedAsync(RaftClientImpl.java:235)
> at org.apache.ratis.client.impl.AsyncImpl.send(AsyncImpl.java:41)
> at org.apache.ratis.client.impl.AsyncImpl.send(AsyncImpl.java:46)
> at
> org.apache.hadoop.hdds.scm.ha.HASecurityUtils.submitScmCertsToRatis(HASecurityUtils.java:357)
> at
> org.apache.hadoop.hdds.scm.ha.SCMHAInvocationHandler.invokeRatis(SCMHAInvocationHandler.java:109)
> at
> org.apache.hadoop.hdds.scm.ha.SCMHAInvocationHandler.invoke(SCMHAInvocationHandler.java:71)
> at com.sun.proxy.$Proxy34.storeValidCertificate(Unknown Source)
> at
> org.apache.hadoop.hdds.security.x509.certificate.authority.DefaultCAServer.signAndStoreCertificate(DefaultCAServer.java:300)
> at
> org.apache.hadoop.hdds.security.x509.certificate.authority.DefaultCAServer.requestCertificate(DefaultCAServer.java:260)
> at
> org.apache.hadoop.hdds.security.x509.certificate.authority.DefaultCAServer.requestCertificate(DefaultCAServer.java:315)
> at
> org.apache.hadoop.hdds.scm.server.SCMSecurityProtocolServer.getEncodedCertToString(SCMSecurityProtocolServer.java:288)
> at
> org.apache.hadoop.hdds.scm.server.SCMSecurityProtocolServer.getSCMCertificate(SCMSecurityProtocolServer.java:274)
> at
> org.apache.hadoop.hdds.scm.protocol.SCMSecurityProtocolServerSideTranslatorPB.getSCMCertificate(SCMSecurityProtocolServerSideTranslatorPB.java:252)
> at
> org.apache.hadoop.hdds.scm.protocol.SCMSecurityProtocolServerSideTranslatorPB.processRequest(SCMSecurityProtocolServerSideTranslatorPB.java:124)
> at
> org.apache.hadoop.hdds.server.OzoneProtocolMessageDispatcher.processRequest(OzoneProtocolMessageDispatcher.java:87)
> at
> org.apache.hadoop.hdds.scm.protocol.SCMSecurityProtocolServerSideTranslatorPB.submitRequest(SCMSecurityProtocolServerSideTranslatorPB.java:94)
> at
> org.apache.hadoop.hdds.protocol.proto.SCMSecurityProtocolProtos$SCMSecurityProtocolService$2.callBlockingMethod(SCMSecurityProtocolProtos.java:16080)
> at
> org.apache.hadoop.ipc.ProtobufRpcEngine$Server.processCall(ProtobufRpcEngine.java:484)
> at
> org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:595)
> at
> org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
> at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1213)
> at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1089)
> at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1012)
> at java.security.AccessController.doPrivileged(Native Method)
> at javax.security.auth.Subject.doAs(Subject.java:422)
> at
> org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1899)
> at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3026){code}
--
This message was sent by Atlassian Jira
(v8.20.10#820010)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
