[
https://issues.apache.org/jira/browse/STORM-863?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=14581375#comment-14581375
]
xiajun commented on STORM-863:
------------------------------
When I use storm with kerberos authentication, UI sometimes report
TTransportException, such as:
```
[qtp1237749363-31] ERROR b.s.s.a.k.KerberosSaslTransportPlugin - Client failed
to open SaslClientTransport to interact with a server during session
initiation: org.apache.thrift7.transport.TTransportException: Peer indicated
failure: GSS initiate failed
org.apache.thrift7.transport.TTransportException: Peer indicated failure: GSS
initiate failed
at
org.apache.thrift7.transport.TSaslTransport.receiveSaslMessage(TSaslTransport.java:199)
~[storm-core-0.11.0-SNAPSHOT.jar:0.11.0-SNAPSHOT]
at
org.apache.thrift7.transport.TSaslTransport.open(TSaslTransport.java:277)
~[storm-core-0.11.0-SNAPSHOT.jar:0.11.0-SNAPSHOT]
at
org.apache.thrift7.transport.TSaslClientTransport.open(TSaslClientTransport.java:37)
~[storm-core-0.11.0-SNAPSHOT.jar:0.11.0-SNAPSHOT]
at
backtype.storm.security.auth.kerberos.KerberosSaslTransportPlugin$1.run(KerberosSaslTransportPlugin.java:146)
[storm-core-0.11.0-SNAPSHOT.jar:0.11.0-SNAPSHOT]
at
backtype.storm.security.auth.kerberos.KerberosSaslTransportPlugin$1.run(KerberosSaslTransportPlugin.java:142)
[storm-core-0.11.0-SNAPSHOT.jar:0.11.0-SNAPSHOT]
at java.security.AccessController.doPrivileged(Native Method)
~[?:1.6.0_37]
at javax.security.auth.Subject.doAs(Subject.java:396) [?:1.6.0_37]
at
backtype.storm.security.auth.kerberos.KerberosSaslTransportPlugin.connect(KerberosSaslTransportPlugin.java:141)
[storm-core-0.11.0-SNAPSHOT.jar:0.11.0-SNAPSHOT]
at
backtype.storm.security.auth.TBackoffConnect.doConnectWithRetry(TBackoffConnect.java:48)
[storm-core-0.11.0-SNAPSHOT.jar:0.11.0-SNAPSHOT]
at
backtype.storm.security.auth.ThriftClient.reconnect(ThriftClient.java:103)
[storm-core-0.11.0-SNAPSHOT.jar:0.11.0-SNAPSHOT]
at
backtype.storm.security.auth.ThriftClient.<init>(ThriftClient.java:72)
[storm-core-0.11.0-SNAPSHOT.jar:0.11.0-SNAPSHOT]
at backtype.storm.utils.NimbusClient.<init>(NimbusClient.java:69)
[storm-core-0.11.0-SNAPSHOT.jar:0.11.0-SNAPSHOT]
at backtype.storm.thrift$nimbus_client_and_conn.invoke(thrift.clj:75)
[storm-core-0.11.0-SNAPSHOT.jar:0.11.0-SNAPSHOT]
at backtype.storm.ui.core$cluster_summary.invoke(core.clj:546)
[storm-core-0.11.0-SNAPSHOT.jar:0.11.0-SNAPSHOT]
at backtype.storm.ui.core$fn__9894.invoke(core.clj:947)
[storm-core-0.11.0-SNAPSHOT.jar:0.11.0-SNAPSHOT]
at compojure.core$make_route$fn__4484.invoke(core.clj:93)
[storm-core-0.11.0-SNAPSHOT.jar:0.11.0-SNAPSHOT]
at compojure.core$if_route$fn__4472.invoke(core.clj:39)
[storm-core-0.11.0-SNAPSHOT.jar:0.11.0-SNAPSHOT]
at compojure.core$if_method$fn__4465.invoke(core.clj:24)
[storm-core-0.11.0-SNAPSHOT.jar:0.11.0-SNAPSHOT]
at compojure.core$routing$fn__4490.invoke(core.clj:106)
[storm-core-0.11.0-SNAPSHOT.jar:0.11.0-SNAPSHOT]
at clojure.core$some.invoke(core.clj:2515) [clojure-1.6.0.jar:?]
at compojure.core$routing.doInvoke(core.clj:106)
[storm-core-0.11.0-SNAPSHOT.jar:0.11.0-SNAPSHOT]
at clojure.lang.RestFn.applyTo(RestFn.java:139) [clojure-1.6.0.jar:?]
at clojure.core$apply.invoke(core.clj:626) [clojure-1.6.0.jar:?]
at compojure.core$routes$fn__4494.invoke(core.clj:111)
[storm-core-0.11.0-SNAPSHOT.jar:0.11.0-SNAPSHOT]
at ring.middleware.json$wrap_json_params$fn__9178.invoke(json.clj:56)
[storm-core-0.11.0-SNAPSHOT.jar:0.11.0-SNAPSHOT]
at
ring.middleware.multipart_params$wrap_multipart_params$fn__5544.invoke(multipart_params.clj:103)
[storm-core-0.11.0-SNAPSHOT.jar:0.11.0-SNAPSHOT]
at ring.middleware.reload$wrap_reload$fn__6271.invoke(reload.clj:22)
[storm-core-0.11.0-SNAPSHOT.jar:0.11.0-SNAPSHOT]
at backtype.storm.ui.core$catch_errors$fn__9971.invoke(core.clj:1055)
[storm-core-0.11.0-SNAPSHOT.jar:0.11.0-SNAPSHOT]
at
ring.middleware.keyword_params$wrap_keyword_params$fn__5477.invoke(keyword_params.clj:27)
[storm-core-0.11.0-SNAPSHOT.jar:0.11.0-SNAPSHOT]
at
ring.middleware.nested_params$wrap_nested_params$fn__5516.invoke(nested_params.clj:65)
[storm-core-0.11.0-SNAPSHOT.jar:0.11.0-SNAPSHOT]
at ring.middleware.params$wrap_params$fn__5449.invoke(params.clj:55)
[storm-core-0.11.0-SNAPSHOT.jar:0.11.0-SNAPSHOT]
at
ring.middleware.multipart_params$wrap_multipart_params$fn__5544.invoke(multipart_params.clj:103)
[storm-core-0.11.0-SNAPSHOT.jar:0.11.0-SNAPSHOT]
at ring.middleware.flash$wrap_flash$fn__5725.invoke(flash.clj:14)
[storm-core-0.11.0-SNAPSHOT.jar:0.11.0-SNAPSHOT]
at ring.middleware.session$wrap_session$fn__5714.invoke(session.clj:43)
[storm-core-0.11.0-SNAPSHOT.jar:0.11.0-SNAPSHOT]
at
ring.middleware.cookies$wrap_cookies$fn__5645.invoke(cookies.clj:160)
[storm-core-0.11.0-SNAPSHOT.jar:0.11.0-SNAPSHOT]
at
ring.util.servlet$make_service_method$fn__5362.invoke(servlet.clj:127)
[storm-core-0.11.0-SNAPSHOT.jar:0.11.0-SNAPSHOT]
at ring.util.servlet$servlet$fn__5366.invoke(servlet.clj:136)
[storm-core-0.11.0-SNAPSHOT.jar:0.11.0-SNAPSHOT]
at
ring.util.servlet.proxy$javax.servlet.http.HttpServlet$ff19274a.service(Unknown
Source) [storm-core-0.11.0-SNAPSHOT.jar:0.11.0-SNAPSHOT]
at
org.apache.storm.jetty.servlet.ServletHolder.handle(ServletHolder.java:654)
[storm-core-0.11.0-SNAPSHOT.jar:0.11.0-SNAPSHOT]
at
org.apache.storm.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1320)
[storm-core-0.11.0-SNAPSHOT.jar:0.11.0-SNAPSHOT]
at
org.apache.storm.jetty.servlets.CrossOriginFilter.handle(CrossOriginFilter.java:247)
[storm-core-0.11.0-SNAPSHOT.jar:0.11.0-SNAPSHOT]
at
org.apache.storm.jetty.servlets.CrossOriginFilter.doFilter(CrossOriginFilter.java:210)
[storm-core-0.11.0-SNAPSHOT.jar:0.11.0-SNAPSHOT]
at
org.apache.storm.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1291)
[storm-core-0.11.0-SNAPSHOT.jar:0.11.0-SNAPSHOT]
at
org.apache.storm.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:443)
[storm-core-0.11.0-SNAPSHOT.jar:0.11.0-SNAPSHOT]
at
org.apache.storm.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1044)
[storm-core-0.11.0-SNAPSHOT.jar:0.11.0-SNAPSHOT]
at
org.apache.storm.jetty.servlet.ServletHandler.doScope(ServletHandler.java:372)
[storm-core-0.11.0-SNAPSHOT.jar:0.11.0-SNAPSHOT]
at
org.apache.storm.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:978)
[storm-core-0.11.0-SNAPSHOT.jar:0.11.0-SNAPSHOT]
at
org.apache.storm.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:135)
[storm-core-0.11.0-SNAPSHOT.jar:0.11.0-SNAPSHOT]
at
org.apache.storm.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:116)
[storm-core-0.11.0-SNAPSHOT.jar:0.11.0-SNAPSHOT]
at org.apache.storm.jetty.server.Server.handle(Server.java:369)
[storm-core-0.11.0-SNAPSHOT.jar:0.11.0-SNAPSHOT]
at
org.apache.storm.jetty.server.AbstractHttpConnection.handleRequest(AbstractHttpConnection.java:486)
[storm-core-0.11.0-SNAPSHOT.jar:0.11.0-SNAPSHOT]
at
org.apache.storm.jetty.server.AbstractHttpConnection.headerComplete(AbstractHttpConnection.java:933)
[storm-core-0.11.0-SNAPSHOT.jar:0.11.0-SNAPSHOT]
at
org.apache.storm.jetty.server.AbstractHttpConnection$RequestHandler.headerComplete(AbstractHttpConnection.java:995)
[storm-core-0.11.0-SNAPSHOT.jar:0.11.0-SNAPSHOT]
at
org.apache.storm.jetty.http.HttpParser.parseNext(HttpParser.java:644)
[storm-core-0.11.0-SNAPSHOT.jar:0.11.0-SNAPSHOT]
at
org.apache.storm.jetty.http.HttpParser.parseAvailable(HttpParser.java:235)
[storm-core-0.11.0-SNAPSHOT.jar:0.11.0-SNAPSHOT]
at
org.apache.storm.jetty.server.AsyncHttpConnection.handle(AsyncHttpConnection.java:82)
[storm-core-0.11.0-SNAPSHOT.jar:0.11.0-SNAPSHOT]
at
org.apache.storm.jetty.io.nio.SelectChannelEndPoint.handle(SelectChannelEndPoint.java:668)
[storm-core-0.11.0-SNAPSHOT.jar:0.11.0-SNAPSHOT]
at
org.apache.storm.jetty.io.nio.SelectChannelEndPoint$1.run(SelectChannelEndPoint.java:52)
[storm-core-0.11.0-SNAPSHOT.jar:0.11.0-SNAPSHOT]
at
org.apache.storm.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)
[storm-core-0.11.0-SNAPSHOT.jar:0.11.0-SNAPSHOT]
at
org.apache.storm.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)
[storm-core-0.11.0-SNAPSHOT.jar:0.11.0-SNAPSHOT]
at java.lang.Thread.run(Thread.java:662) [?:1.6.0_37]
```
and nimbus report
```
1266878 [pool-7-thread-62] ERROR o.a.t.t.TSaslTransport - SASL negotiation
failure
javax.security.sasl.SaslException: GSS initiate failed
at
com.sun.security.sasl.gsskerb.GssKrb5Server.evaluateResponse(GssKrb5Server.java:159)
~[?:1.6.0_37]
at
org.apache.thrift7.transport.TSaslTransport$SaslParticipant.evaluateChallengeOrResponse(TSaslTransport.java:539)
~[storm-core-0.11.0-SNAPSHOT.jar:0.11.0-SNAPSHOT]
at
org.apache.thrift7.transport.TSaslTransport.open(TSaslTransport.java:283)
[storm-core-0.11.0-SNAPSHOT.jar:0.11.0-SNAPSHOT]
at
org.apache.thrift7.transport.TSaslServerTransport.open(TSaslServerTransport.java:41)
[storm-core-0.11.0-SNAPSHOT.jar:0.11.0-SNAPSHOT]
at
org.apache.thrift7.transport.TSaslServerTransport$Factory.getTransport(TSaslServerTransport.java:216)
[storm-core-0.11.0-SNAPSHOT.jar:0.11.0-SNAPSHOT]
at
backtype.storm.security.auth.kerberos.KerberosSaslTransportPlugin$TUGIAssumingTransportFactory$1.run(KerberosSaslTransportPlugin.java:196)
[storm-core-0.11.0-SNAPSHOT.jar:0.11.0-SNAPSHOT]
at
backtype.storm.security.auth.kerberos.KerberosSaslTransportPlugin$TUGIAssumingTransportFactory$1.run(KerberosSaslTransportPlugin.java:193)
[storm-core-0.11.0-SNAPSHOT.jar:0.11.0-SNAPSHOT]
at java.security.AccessController.doPrivileged(Native Method)
~[?:1.6.0_37]
at javax.security.auth.Subject.doAs(Subject.java:396) [?:1.6.0_37]
at
backtype.storm.security.auth.kerberos.KerberosSaslTransportPlugin$TUGIAssumingTransportFactory.getTransport(KerberosSaslTransportPlugin.java:192)
[storm-core-0.11.0-SNAPSHOT.jar:0.11.0-SNAPSHOT]
at
org.apache.thrift7.server.TThreadPoolServer$WorkerProcess.run(TThreadPoolServer.java:268)
[storm-core-0.11.0-SNAPSHOT.jar:0.11.0-SNAPSHOT]
at
java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
[?:1.6.0_37]
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
[?:1.6.0_37]
at java.lang.Thread.run(Thread.java:662) [?:1.6.0_37]
Caused by: org.ietf.jgss.GSSException: Failure unspecified at GSS-API level
(Mechanism level: Request is a replay (34))
at
sun.security.jgss.krb5.Krb5Context.acceptSecContext(Krb5Context.java:741)
~[?:1.6.0_37]
at
sun.security.jgss.GSSContextImpl.acceptSecContext(GSSContextImpl.java:323)
~[?:1.6.0_37]
at
sun.security.jgss.GSSContextImpl.acceptSecContext(GSSContextImpl.java:267)
~[?:1.6.0_37]
at
com.sun.security.sasl.gsskerb.GssKrb5Server.evaluateResponse(GssKrb5Server.java:137)
~[?:1.6.0_37]
... 13 more
Caused by: sun.security.krb5.internal.KrbApErrException: Request is a replay
(34)
at sun.security.krb5.KrbApReq.authenticate(KrbApReq.java:298)
~[?:1.6.0_37]
at sun.security.krb5.KrbApReq.<init>(KrbApReq.java:134) ~[?:1.6.0_37]
at
sun.security.jgss.krb5.InitSecContextToken.<init>(InitSecContextToken.java:79)
~[?:1.6.0_37]
at
sun.security.jgss.krb5.Krb5Context.acceptSecContext(Krb5Context.java:724)
~[?:1.6.0_37]
at
sun.security.jgss.GSSContextImpl.acceptSecContext(GSSContextImpl.java:323)
~[?:1.6.0_37]
at
sun.security.jgss.GSSContextImpl.acceptSecContext(GSSContextImpl.java:267)
~[?:1.6.0_37]
at
com.sun.security.sasl.gsskerb.GssKrb5Server.evaluateResponse(GssKrb5Server.java:137)
~[?:1.6.0_37]
... 13 more
default etypes for permitted_enctypes: 3 1 23 16 17 18.
1266879 [pool-7-thread-62] ERROR b.s.s.a.k.KerberosSaslTransportPlugin - Storm
server failed to open transport to interact with a client during session
initiation: java.lang.RuntimeException:
org.apache.thrift7.transport.TTransportException: GSS initiate failed
java.lang.RuntimeException: org.apache.thrift7.transport.TTransportException:
GSS initiate failed
at
org.apache.thrift7.transport.TSaslServerTransport$Factory.getTransport(TSaslServerTransport.java:219)
~[storm-core-0.11.0-SNAPSHOT.jar:0.11.0-SNAPSHOT]
at
backtype.storm.security.auth.kerberos.KerberosSaslTransportPlugin$TUGIAssumingTransportFactory$1.run(KerberosSaslTransportPlugin.java:196)
[storm-core-0.11.0-SNAPSHOT.jar:0.11.0-SNAPSHOT]
at
backtype.storm.security.auth.kerberos.KerberosSaslTransportPlugin$TUGIAssumingTransportFactory$1.run(KerberosSaslTransportPlugin.java:193)
[storm-core-0.11.0-SNAPSHOT.jar:0.11.0-SNAPSHOT]
at java.security.AccessController.doPrivileged(Native Method)
~[?:1.6.0_37]
at javax.security.auth.Subject.doAs(Subject.java:396) [?:1.6.0_37]
at
backtype.storm.security.auth.kerberos.KerberosSaslTransportPlugin$TUGIAssumingTransportFactory.getTransport(KerberosSaslTransportPlugin.java:192)
[storm-core-0.11.0-SNAPSHOT.jar:0.11.0-SNAPSHOT]
at
org.apache.thrift7.server.TThreadPoolServer$WorkerProcess.run(TThreadPoolServer.java:268)
[storm-core-0.11.0-SNAPSHOT.jar:0.11.0-SNAPSHOT]
at
java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
[?:1.6.0_37]
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
[?:1.6.0_37]
at java.lang.Thread.run(Thread.java:662) [?:1.6.0_37]
Caused by: org.apache.thrift7.transport.TTransportException: GSS initiate failed
at
org.apache.thrift7.transport.TSaslTransport.sendAndThrowMessage(TSaslTransport.java:232)
~[storm-core-0.11.0-SNAPSHOT.jar:0.11.0-SNAPSHOT]
at
org.apache.thrift7.transport.TSaslTransport.open(TSaslTransport.java:316)
~[storm-core-0.11.0-SNAPSHOT.jar:0.11.0-SNAPSHOT]
at
org.apache.thrift7.transport.TSaslServerTransport.open(TSaslServerTransport.java:41)
~[storm-core-0.11.0-SNAPSHOT.jar:0.11.0-SNAPSHOT]
at
org.apache.thrift7.transport.TSaslServerTransport$Factory.getTransport(TSaslServerTransport.java:216)
~[storm-core-0.11.0-SNAPSHOT.jar:0.11.0-SNAPSHOT]
... 9 more
>>> EType: sun.security.krb5.internal.crypto.ArcFourHmacEType
1266879 [pool-7-thread-62] ERROR b.s.s.a.k.KerberosSaslTransportPlugin - Storm
server failed to open transport to interact with a client during session
initiation: java.lang.NullPointerException
java.lang.NullPointerException
at org.apache.thrift7.transport.TSocket.open(TSocket.java:170)
~[storm-core-0.11.0-SNAPSHOT.jar:0.11.0-SNAPSHOT]
at
org.apache.thrift7.transport.TSaslTransport.open(TSaslTransport.java:266)
~[storm-core-0.11.0-SNAPSHOT.jar:0.11.0-SNAPSHOT]
at
org.apache.thrift7.transport.TSaslServerTransport.open(TSaslServerTransport.java:41)
~[storm-core-0.11.0-SNAPSHOT.jar:0.11.0-SNAPSHOT]
at
org.apache.thrift7.transport.TSaslServerTransport$Factory.getTransport(TSaslServerTransport.java:216)
~[storm-core-0.11.0-SNAPSHOT.jar:0.11.0-SNAPSHOT]
at
backtype.storm.security.auth.kerberos.KerberosSaslTransportPlugin$TUGIAssumingTransportFactory$1.run(KerberosSaslTransportPlugin.java:196)
[storm-core-0.11.0-SNAPSHOT.jar:0.11.0-SNAPSHOT]
at
backtype.storm.security.auth.kerberos.KerberosSaslTransportPlugin$TUGIAssumingTransportFactory$1.run(KerberosSaslTransportPlugin.java:193)
[storm-core-0.11.0-SNAPSHOT.jar:0.11.0-SNAPSHOT]
at java.security.AccessController.doPrivileged(Native Method)
~[?:1.6.0_37]
at javax.security.auth.Subject.doAs(Subject.java:396) [?:1.6.0_37]
at
backtype.storm.security.auth.kerberos.KerberosSaslTransportPlugin$TUGIAssumingTransportFactory.getTransport(KerberosSaslTransportPlugin.java:192)
[storm-core-0.11.0-SNAPSHOT.jar:0.11.0-SNAPSHOT]
at
org.apache.thrift7.server.TThreadPoolServer$WorkerProcess.run(TThreadPoolServer.java:269)
[storm-core-0.11.0-SNAPSHOT.jar:0.11.0-SNAPSHOT]
at
java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
[?:1.6.0_37]
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
[?:1.6.0_37]
at java.lang.Thread.run(Thread.java:662) [?:1.6.0_37]
1266879 [pool-7-thread-62] ERROR o.a.t.s.TThreadPoolServer - Error occurred
during processing of message.
java.lang.NullPointerException
at
backtype.storm.security.auth.SaslTransportPlugin$TUGIWrapProcessor.process(SaslTransportPlugin.java:125)
~[storm-core-0.11.0-SNAPSHOT.jar:0.11.0-SNAPSHOT]
at
org.apache.thrift7.server.TThreadPoolServer$WorkerProcess.run(TThreadPoolServer.java:285)
[storm-core-0.11.0-SNAPSHOT.jar:0.11.0-SNAPSHOT]
at
java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
[?:1.6.0_37]
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
[?:1.6.0_37]
at java.lang.Thread.run(Thread.java:662) [?:1.6.0_37]
```
This not happen every time, some one will give me some advice, thanks.
> UI report TTransportException
> -----------------------------
>
> Key: STORM-863
> URL: https://issues.apache.org/jira/browse/STORM-863
> Project: Apache Storm
> Issue Type: Bug
> Affects Versions: 0.11.0
> Reporter: xiajun
>
--
This message was sent by Atlassian JIRA
(v6.3.4#6332)
