Sarveksha Yeshavantha Raju created HDDS-13405:
-------------------------------------------------
Summary: ozone admin container create runs forever without kinit
Key: HDDS-13405
URL: https://issues.apache.org/jira/browse/HDDS-13405
Project: Apache Ozone
Issue Type: Task
Reporter: Sarveksha Yeshavantha Raju
{{ozone admin container create}} command runs forever in a secure cluster when
number of scm nodes > 1.
{code:java}
bash-5.1$ OZONE_LOGLEVEL=INFO ozone admin container create
2025-07-08 10:44:48,181 [main] INFO
proxy.SCMContainerLocationFailoverProxyProvider: Created fail-over proxy for
protocol StorageContainerLocationProtocolPB with 3 nodes:
[nodeId=scm2,nodeAddress=scm2.org/172.25.0.117:9860,
nodeId=scm1,nodeAddress=scm1.org/172.25.0.116:9860,
nodeId=scm3,nodeAddress=scm3.org/172.25.0.118:9860]
2025-07-08 10:44:48,229 [main] INFO
proxy.SecretKeyProtocolFailoverProxyProvider: Created fail-over proxy for
protocol SecretKeyProtocolScmPB with 3 nodes:
[nodeId=scm2,nodeAddress=scm2.org/172.25.0.117:9961,
nodeId=scm1,nodeAddress=scm1.org/172.25.0.116:9961,
nodeId=scm3,nodeAddress=scm3.org/172.25.0.118:9961]
2025-07-08 10:44:48,402 [main] INFO
proxy.SCMSecurityProtocolFailoverProxyProvider: Created fail-over proxy for
protocol SCMSecurityProtocolPB with 3 nodes:
[nodeId=scm2,nodeAddress=scm2.org/172.25.0.117:9961,
nodeId=scm1,nodeAddress=scm1.org/172.25.0.116:9961,
nodeId=scm3,nodeAddress=scm3.org/172.25.0.118:9961]
2025-07-08 10:44:48,470 [main] WARN ipc.Client: Exception encountered while
connecting to the server scm1.org/172.25.0.116:9961
org.apache.hadoop.security.AccessControlException: Client cannot authenticate
via:[KERBEROS]
at
org.apache.hadoop.security.SaslRpcClient.selectSaslClient(SaslRpcClient.java:179)
at
org.apache.hadoop.security.SaslRpcClient.saslConnect(SaslRpcClient.java:399)
at
org.apache.hadoop.ipc.Client$Connection.setupSaslConnection(Client.java:578)
at org.apache.hadoop.ipc.Client$Connection.access$2100(Client.java:364)
at org.apache.hadoop.ipc.Client$Connection$2.run(Client.java:799)
at org.apache.hadoop.ipc.Client$Connection$2.run(Client.java:795)
at
java.base/java.security.AccessController.doPrivileged(AccessController.java:714)
at java.base/javax.security.auth.Subject.doAs(Subject.java:525)
at
org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
at
org.apache.hadoop.ipc.Client$Connection.setupIOstreams(Client.java:795)
at org.apache.hadoop.ipc.Client$Connection.access$3800(Client.java:364)
at org.apache.hadoop.ipc.Client.getConnection(Client.java:1649)
at org.apache.hadoop.ipc.Client.call(Client.java:1473)
at org.apache.hadoop.ipc.Client.call(Client.java:1426)
at
org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:250)
at
org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:132)
at jdk.proxy2/jdk.proxy2.$Proxy22.submitRequest(Unknown Source)
at
java.base/jdk.internal.reflect.DirectMethodHandleAccessor.invoke(DirectMethodHandleAccessor.java:103)
at java.base/java.lang.reflect.Method.invoke(Method.java:580)
at
org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437)
at
org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170)
at
org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162)
at
org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100)
at
org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366)
at jdk.proxy2/jdk.proxy2.$Proxy22.submitRequest(Unknown Source)
at
org.apache.hadoop.hdds.protocolPB.SCMSecurityProtocolClientSideTranslatorPB.submitRequest(SCMSecurityProtocolClientSideTranslatorPB.java:93)
at
org.apache.hadoop.hdds.protocolPB.SCMSecurityProtocolClientSideTranslatorPB.listCACertificate(SCMSecurityProtocolClientSideTranslatorPB.java:363)
at org.apache.hadoop.hdds.utils.HAUtils.waitForCACerts(HAUtils.java:374)
at
org.apache.hadoop.hdds.utils.HAUtils.lambda$buildCAX509List$3(HAUtils.java:401)
at
org.apache.hadoop.hdds.utils.RetriableTask.call(RetriableTask.java:55)
at
org.apache.hadoop.hdds.utils.HAUtils.getCAListWithRetry(HAUtils.java:360)
at
org.apache.hadoop.hdds.utils.HAUtils.buildCAX509List(HAUtils.java:401)
at
org.apache.hadoop.hdds.scm.cli.ContainerOperationClient.lambda$newXCeiverClientManager$0(ContainerOperationClient.java:123)
at
org.apache.hadoop.hdds.scm.client.ClientTrustManager.loadCerts(ClientTrustManager.java:148)
at
org.apache.hadoop.hdds.scm.client.ClientTrustManager.<init>(ClientTrustManager.java:110)
at
org.apache.hadoop.hdds.scm.cli.ContainerOperationClient.newXCeiverClientManager(ContainerOperationClient.java:125)
at
org.apache.hadoop.hdds.scm.cli.ContainerOperationClient.getXceiverClientManager(ContainerOperationClient.java:91)
at
org.apache.hadoop.hdds.scm.cli.ContainerOperationClient.createContainer(ContainerOperationClient.java:212)
at
org.apache.hadoop.hdds.scm.cli.container.CreateSubcommand.execute(CreateSubcommand.java:59)
at
org.apache.hadoop.hdds.scm.cli.ScmSubcommand.call(ScmSubcommand.java:39)
at
org.apache.hadoop.hdds.scm.cli.ScmSubcommand.call(ScmSubcommand.java:29)
at picocli.CommandLine.executeUserObject(CommandLine.java:2031)
at picocli.CommandLine.access$1500(CommandLine.java:148)
at
picocli.CommandLine$RunLast.executeUserObjectOfLastSubcommandWithSameParent(CommandLine.java:2469)
at picocli.CommandLine$RunLast.handle(CommandLine.java:2461)
at picocli.CommandLine$RunLast.handle(CommandLine.java:2423)
at
picocli.CommandLine$AbstractParseResultHandler.execute(CommandLine.java:2277)
at picocli.CommandLine$RunLast.execute(CommandLine.java:2425)
at org.apache.hadoop.ozone.shell.Shell.lambda$execute$0(Shell.java:95)
at
org.apache.hadoop.hdds.tracing.TracingUtil.executeInSpan(TracingUtil.java:167)
at
org.apache.hadoop.hdds.tracing.TracingUtil.executeInNewSpan(TracingUtil.java:157)
at org.apache.hadoop.ozone.shell.Shell.execute(Shell.java:95)
at picocli.CommandLine.execute(CommandLine.java:2174)
at org.apache.hadoop.hdds.cli.GenericCli.execute(GenericCli.java:89)
at org.apache.hadoop.hdds.cli.GenericCli.run(GenericCli.java:80)
at org.apache.hadoop.ozone.admin.OzoneAdmin.main(OzoneAdmin.java:36)
2025-07-08 10:44:48,478 [main] INFO utils.RetriableTask: Execution of task
getCAList failed, will be retried in 10000 ms
2025-07-08 10:44:58,484 [main] WARN ipc.Client: Exception encountered while
connecting to the server scm1.org/172.25.0.116:9961
org.apache.hadoop.security.AccessControlException: Client cannot authenticate
via:[KERBEROS]
at
org.apache.hadoop.security.SaslRpcClient.selectSaslClient(SaslRpcClient.java:179)
at
org.apache.hadoop.security.SaslRpcClient.saslConnect(SaslRpcClient.java:399)
at
org.apache.hadoop.ipc.Client$Connection.setupSaslConnection(Client.java:578)
at org.apache.hadoop.ipc.Client$Connection.access$2100(Client.java:364)
at org.apache.hadoop.ipc.Client$Connection$2.run(Client.java:799)
at org.apache.hadoop.ipc.Client$Connection$2.run(Client.java:795)
at
java.base/java.security.AccessController.doPrivileged(AccessController.java:714)
at java.base/javax.security.auth.Subject.doAs(Subject.java:525)
at
org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
at
org.apache.hadoop.ipc.Client$Connection.setupIOstreams(Client.java:795)
at org.apache.hadoop.ipc.Client$Connection.access$3800(Client.java:364)
at org.apache.hadoop.ipc.Client.getConnection(Client.java:1649)
at org.apache.hadoop.ipc.Client.call(Client.java:1473)
at org.apache.hadoop.ipc.Client.call(Client.java:1426)
at
org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:250)
at
org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:132)
at jdk.proxy2/jdk.proxy2.$Proxy22.submitRequest(Unknown Source)
at
java.base/jdk.internal.reflect.DirectMethodHandleAccessor.invoke(DirectMethodHandleAccessor.java:103)
at java.base/java.lang.reflect.Method.invoke(Method.java:580)
at
org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437)
at
org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170)
at
org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162)
at
org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100)
at
org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366)
at jdk.proxy2/jdk.proxy2.$Proxy22.submitRequest(Unknown Source)
at
org.apache.hadoop.hdds.protocolPB.SCMSecurityProtocolClientSideTranslatorPB.submitRequest(SCMSecurityProtocolClientSideTranslatorPB.java:93)
at
org.apache.hadoop.hdds.protocolPB.SCMSecurityProtocolClientSideTranslatorPB.listCACertificate(SCMSecurityProtocolClientSideTranslatorPB.java:363)
at org.apache.hadoop.hdds.utils.HAUtils.waitForCACerts(HAUtils.java:374)
at
org.apache.hadoop.hdds.utils.HAUtils.lambda$buildCAX509List$3(HAUtils.java:401)
at
org.apache.hadoop.hdds.utils.RetriableTask.call(RetriableTask.java:55)
at
org.apache.hadoop.hdds.utils.HAUtils.getCAListWithRetry(HAUtils.java:360)
at
org.apache.hadoop.hdds.utils.HAUtils.buildCAX509List(HAUtils.java:401)
at
org.apache.hadoop.hdds.scm.cli.ContainerOperationClient.lambda$newXCeiverClientManager$0(ContainerOperationClient.java:123)
at
org.apache.hadoop.hdds.scm.client.ClientTrustManager.loadCerts(ClientTrustManager.java:148)
at
org.apache.hadoop.hdds.scm.client.ClientTrustManager.<init>(ClientTrustManager.java:110)
at
org.apache.hadoop.hdds.scm.cli.ContainerOperationClient.newXCeiverClientManager(ContainerOperationClient.java:125)
at
org.apache.hadoop.hdds.scm.cli.ContainerOperationClient.getXceiverClientManager(ContainerOperationClient.java:91)
at
org.apache.hadoop.hdds.scm.cli.ContainerOperationClient.createContainer(ContainerOperationClient.java:212)
at
org.apache.hadoop.hdds.scm.cli.container.CreateSubcommand.execute(CreateSubcommand.java:59)
at
org.apache.hadoop.hdds.scm.cli.ScmSubcommand.call(ScmSubcommand.java:39)
at
org.apache.hadoop.hdds.scm.cli.ScmSubcommand.call(ScmSubcommand.java:29)
at picocli.CommandLine.executeUserObject(CommandLine.java:2031)
at picocli.CommandLine.access$1500(CommandLine.java:148)
at
picocli.CommandLine$RunLast.executeUserObjectOfLastSubcommandWithSameParent(CommandLine.java:2469)
at picocli.CommandLine$RunLast.handle(CommandLine.java:2461)
at picocli.CommandLine$RunLast.handle(CommandLine.java:2423)
at
picocli.CommandLine$AbstractParseResultHandler.execute(CommandLine.java:2277)
at picocli.CommandLine$RunLast.execute(CommandLine.java:2425)
at org.apache.hadoop.ozone.shell.Shell.lambda$execute$0(Shell.java:95)
at
org.apache.hadoop.hdds.tracing.TracingUtil.executeInSpan(TracingUtil.java:167)
at
org.apache.hadoop.hdds.tracing.TracingUtil.executeInNewSpan(TracingUtil.java:157)
at org.apache.hadoop.ozone.shell.Shell.execute(Shell.java:95)
at picocli.CommandLine.execute(CommandLine.java:2174)
at org.apache.hadoop.hdds.cli.GenericCli.execute(GenericCli.java:89)
at org.apache.hadoop.hdds.cli.GenericCli.run(GenericCli.java:80)
at org.apache.hadoop.ozone.admin.OzoneAdmin.main(OzoneAdmin.java:36)
2025-07-08 10:44:58,487 [main] INFO utils.RetriableTask: Execution of task
getCAList failed, will be retried in 10000 ms
2025-07-08 10:45:08,494 [main] WARN ipc.Client: Exception encountered while
connecting to the server scm1.org/172.25.0.116:9961
org.apache.hadoop.security.AccessControlException: Client cannot authenticate
via:[KERBEROS]
at
org.apache.hadoop.security.SaslRpcClient.selectSaslClient(SaslRpcClient.java:179)
at
org.apache.hadoop.security.SaslRpcClient.saslConnect(SaslRpcClient.java:399)
at
org.apache.hadoop.ipc.Client$Connection.setupSaslConnection(Client.java:578)
at org.apache.hadoop.ipc.Client$Connection.access$2100(Client.java:364)
at org.apache.hadoop.ipc.Client$Connection$2.run(Client.java:799)
at org.apache.hadoop.ipc.Client$Connection$2.run(Client.java:795)
at
java.base/java.security.AccessController.doPrivileged(AccessController.java:714)
at java.base/javax.security.auth.Subject.doAs(Subject.java:525)
at
org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
at
org.apache.hadoop.ipc.Client$Connection.setupIOstreams(Client.java:795)
at org.apache.hadoop.ipc.Client$Connection.access$3800(Client.java:364)
at org.apache.hadoop.ipc.Client.getConnection(Client.java:1649)
at org.apache.hadoop.ipc.Client.call(Client.java:1473)
at org.apache.hadoop.ipc.Client.call(Client.java:1426)
at
org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:250)
at
org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:132)
at jdk.proxy2/jdk.proxy2.$Proxy22.submitRequest(Unknown Source)
at
java.base/jdk.internal.reflect.DirectMethodHandleAccessor.invoke(DirectMethodHandleAccessor.java:103)
at java.base/java.lang.reflect.Method.invoke(Method.java:580)
at
org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437)
at
org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170)
at
org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162)
at
org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100)
at
org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366)
at jdk.proxy2/jdk.proxy2.$Proxy22.submitRequest(Unknown Source)
at
org.apache.hadoop.hdds.protocolPB.SCMSecurityProtocolClientSideTranslatorPB.submitRequest(SCMSecurityProtocolClientSideTranslatorPB.java:93)
at
org.apache.hadoop.hdds.protocolPB.SCMSecurityProtocolClientSideTranslatorPB.listCACertificate(SCMSecurityProtocolClientSideTranslatorPB.java:363)
at org.apache.hadoop.hdds.utils.HAUtils.waitForCACerts(HAUtils.java:374)
at
org.apache.hadoop.hdds.utils.HAUtils.lambda$buildCAX509List$3(HAUtils.java:401)
at
org.apache.hadoop.hdds.utils.RetriableTask.call(RetriableTask.java:55)
at
org.apache.hadoop.hdds.utils.HAUtils.getCAListWithRetry(HAUtils.java:360)
at
org.apache.hadoop.hdds.utils.HAUtils.buildCAX509List(HAUtils.java:401)
at
org.apache.hadoop.hdds.scm.cli.ContainerOperationClient.lambda$newXCeiverClientManager$0(ContainerOperationClient.java:123)
at
org.apache.hadoop.hdds.scm.client.ClientTrustManager.loadCerts(ClientTrustManager.java:148)
at
org.apache.hadoop.hdds.scm.client.ClientTrustManager.<init>(ClientTrustManager.java:110)
at
org.apache.hadoop.hdds.scm.cli.ContainerOperationClient.newXCeiverClientManager(ContainerOperationClient.java:125)
at
org.apache.hadoop.hdds.scm.cli.ContainerOperationClient.getXceiverClientManager(ContainerOperationClient.java:91)
at
org.apache.hadoop.hdds.scm.cli.ContainerOperationClient.createContainer(ContainerOperationClient.java:212)
at
org.apache.hadoop.hdds.scm.cli.container.CreateSubcommand.execute(CreateSubcommand.java:59)
at
org.apache.hadoop.hdds.scm.cli.ScmSubcommand.call(ScmSubcommand.java:39)
at
org.apache.hadoop.hdds.scm.cli.ScmSubcommand.call(ScmSubcommand.java:29)
at picocli.CommandLine.executeUserObject(CommandLine.java:2031)
at picocli.CommandLine.access$1500(CommandLine.java:148)
at
picocli.CommandLine$RunLast.executeUserObjectOfLastSubcommandWithSameParent(CommandLine.java:2469)
at picocli.CommandLine$RunLast.handle(CommandLine.java:2461)
at picocli.CommandLine$RunLast.handle(CommandLine.java:2423)
at
picocli.CommandLine$AbstractParseResultHandler.execute(CommandLine.java:2277)
at picocli.CommandLine$RunLast.execute(CommandLine.java:2425)
at org.apache.hadoop.ozone.shell.Shell.lambda$execute$0(Shell.java:95)
at
org.apache.hadoop.hdds.tracing.TracingUtil.executeInSpan(TracingUtil.java:167)
at
org.apache.hadoop.hdds.tracing.TracingUtil.executeInNewSpan(TracingUtil.java:157)
at org.apache.hadoop.ozone.shell.Shell.execute(Shell.java:95)
at picocli.CommandLine.execute(CommandLine.java:2174)
at org.apache.hadoop.hdds.cli.GenericCli.execute(GenericCli.java:89)
at org.apache.hadoop.hdds.cli.GenericCli.run(GenericCli.java:80)
at org.apache.hadoop.ozone.admin.OzoneAdmin.main(OzoneAdmin.java:36)
2025-07-08 10:45:08,497 [main] INFO utils.RetriableTask: Execution of task
getCAList failed, will be retried in 10000 ms {code}
--
This message was sent by Atlassian Jira
(v8.20.10#820010)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
