[
https://issues.apache.org/jira/browse/AMBARI-9721?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
Manish Nema updated AMBARI-9721:
--------------------------------
Description:
While securing cluster through Ambari (Storm only cluster), SPNEGO principals
for logviewers are not added for other supervisor nodes by ambari in
spnego.service.keytab. It only adds principal for Nimbus nodes, this results in
spnego.service.keytab only for Nimbus node.
Logviewer service for other nodes (supervisor) are not started because of this.
Copying the generated spnego.service.keytab from nimbus nodes to other nodes
leads to following error
2015-02-20 12:49:11 o.a.h.s.a.s.AuthenticationFilter [WARN] Authentication
exception: GSSException: Failure unspecified at GSS-API level (Mechanism level:
Checksum failed)
org.apache.hadoop.security.authentication.client.AuthenticationException:
GSSException: Failure unspecified at GSS-API level (Mechanism level: Checksum
failed)
at
org.apache.hadoop.security.authentication.server.KerberosAuthenticationHandler.authenticate(KerberosAuthenticationHandler.java:360)
~[hadoop-auth-2.4.0.jar:na]
at
org.apache.hadoop.security.authentication.server.AuthenticationFilter.doFilter(AuthenticationFilter.java:357)
~[hadoop-auth-2.4.0.jar:na]
at
org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1291)
[jetty-servlet-7.6.13.v20130916.jar:7.6.13.v20130916]
at
org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:443)
[jetty-servlet-7.6.13.v20130916.jar:7.6.13.v20130916]
at
org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1044)
[jetty-server-7.6.13.v20130916.jar:7.6.13.v20130916]
at
org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:372)
[jetty-servlet-7.6.13.v20130916.jar:7.6.13.v20130916]
at
org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:978)
[jetty-server-7.6.13.v20130916.jar:7.6.13.v20130916]
at
org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:135)
[jetty-server-7.6.13.v20130916.jar:7.6.13.v20130916]
at
org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:116)
[jetty-server-7.6.13.v20130916.jar:7.6.13.v20130916]
at org.eclipse.jetty.server.Server.handle(Server.java:369)
[jetty-server-7.6.13.v20130916.jar:7.6.13.v20130916]
at
org.eclipse.jetty.server.AbstractHttpConnection.handleRequest(AbstractHttpConnection.java:486)
[jetty-server-7.6.13.v20130916.jar:7.6.13.v20130916]
at
org.eclipse.jetty.server.AbstractHttpConnection.headerComplete(AbstractHttpConnection.java:933)
[jetty-server-7.6.13.v20130916.jar:7.6.13.v20130916]
at
org.eclipse.jetty.server.AbstractHttpConnection$RequestHandler.headerComplete(AbstractHttpConnection.java:995)
[jetty-server-7.6.13.v20130916.jar:7.6.13.v20130916]
at org.eclipse.jetty.http.HttpParser.parseNext(HttpParser.java:644)
[jetty-http-7.6.13.v20130916.jar:7.6.13.v20130916]
at
org.eclipse.jetty.http.HttpParser.parseAvailable(HttpParser.java:235)
[jetty-http-7.6.13.v20130916.jar:7.6.13.v20130916]
at
org.eclipse.jetty.server.AsyncHttpConnection.handle(AsyncHttpConnection.java:82)
[jetty-server-7.6.13.v20130916.jar:7.6.13.v20130916]
at
org.eclipse.jetty.io.nio.SelectChannelEndPoint.handle(SelectChannelEndPoint.java:668)
[jetty-io-7.6.13.v20130916.jar:7.6.13.v20130916]
at
org.eclipse.jetty.io.nio.SelectChannelEndPoint$1.run(SelectChannelEndPoint.java:52)
[jetty-io-7.6.13.v20130916.jar:7.6.13.v20130916]
at
org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)
[jetty-util-7.6.13.v20130916.jar:7.6.13.v20130916]
at
org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)
[jetty-util-7.6.13.v20130916.jar:7.6.13.v20130916]
at java.lang.Thread.run(Thread.java:745) [na:1.7.0_67]
Caused by: org.ietf.jgss.GSSException: Failure unspecified at GSS-API level
(Mechanism level: Checksum failed)
at
sun.security.jgss.krb5.Krb5Context.acceptSecContext(Krb5Context.java:788)
~[na:1.7.0_67]
at
sun.security.jgss.GSSContextImpl.acceptSecContext(GSSContextImpl.java:342)
~[na:1.7.0_67]
at
sun.security.jgss.GSSContextImpl.acceptSecContext(GSSContextImpl.java:285)
~[na:1.7.0_67]
at
sun.security.jgss.spnego.SpNegoContext.GSS_acceptSecContext(SpNegoContext.java:875)
~[na:1.7.0_67]
at
sun.security.jgss.spnego.SpNegoContext.acceptSecContext(SpNegoContext.java:548)
~[na:1.7.0_67]
at
sun.security.jgss.GSSContextImpl.acceptSecContext(GSSContextImpl.java:342)
~[na:1.7.0_67]
at
sun.security.jgss.GSSContextImpl.acceptSecContext(GSSContextImpl.java:285)
~[na:1.7.0_67]
at
org.apache.hadoop.security.authentication.server.KerberosAuthenticationHandler$2.run(KerberosAuthenticationHandler.java:327)
~[hadoop-auth-2.4.0.jar:na]
at
org.apache.hadoop.security.authentication.server.KerberosAuthenticationHandler$2.run(KerberosAuthenticationHandler.java:309)
~[hadoop-auth-2.4.0.jar:na]
at java.security.AccessController.doPrivileged(Native Method)
~[na:1.7.0_67]
at javax.security.auth.Subject.doAs(Subject.java:415) ~[na:1.7.0_67]
at
org.apache.hadoop.security.authentication.server.KerberosAuthenticationHandler.authenticate(KerberosAuthenticationHandler.java:309)
~[hadoop-auth-2.4.0.jar:na]
... 20 common frames omitted
Caused by: sun.security.krb5.KrbCryptoException: Checksum failed
at
sun.security.krb5.internal.crypto.Aes256CtsHmacSha1EType.decrypt(Aes256CtsHmacSha1EType.java:102)
~[na:1.7.0_67]
at
sun.security.krb5.internal.crypto.Aes256CtsHmacSha1EType.decrypt(Aes256CtsHmacSha1EType.java:94)
~[na:1.7.0_67]
at sun.security.krb5.EncryptedData.decrypt(EncryptedData.java:177)
~[na:1.7.0_67]
at sun.security.krb5.KrbApReq.authenticate(KrbApReq.java:278)
~[na:1.7.0_67]
at sun.security.krb5.KrbApReq.<init>(KrbApReq.java:144) ~[na:1.7.0_67]
at
sun.security.jgss.krb5.InitSecContextToken.<init>(InitSecContextToken.java:108)
~[na:1.7.0_67]
at
sun.security.jgss.krb5.Krb5Context.acceptSecContext(Krb5Context.java:771)
~[na:1.7.0_67]
... 31 common frames omitted
Caused by: java.security.GeneralSecurityException: Checksum failed
at
sun.security.krb5.internal.crypto.dk.AesDkCrypto.decryptCTS(AesDkCrypto.java:451)
~[na:1.7.0_67]
at
sun.security.krb5.internal.crypto.dk.AesDkCrypto.decrypt(AesDkCrypto.java:272)
~[na:1.7.0_67]
at sun.security.krb5.internal.crypto.Aes256.decrypt(Aes256.java:76)
~[na:1.7.0_67]
at
sun.security.krb5.internal.crypto.Aes256CtsHmacSha1EType.decrypt(Aes256CtsHmacSha1EType.java:100)
~[na:1.7.0_67]
... 37 common frames omitted
Also Ambari generates storm.yaml file on restarts of supervisor nodes and this
presently generates "kerberos.principal": "HTTP/<nimbus.host>" only whereas it
should generate kerberos principal for appropriate logviewer/supervisor node.
ui.filter.params:
"type": "kerberos"
"kerberos.principal": "HTTP/two.cluster"
"kerberos.keytab": "/etc/security/keytabs/spnego.service.keytab"
"kerberos.name.rules": "DEFAULT"
This leads to logviewer process initialize only with nimbus principal and later
on generate error while browsing UI of logviewer process with following error
after generating correct keytab which contains HTTP principals for each host
and distributing it to all supervisor/logviewer nodes, logviewer starts
properly but that require manual changes to storm.yaml file to change
kerberos.principal for that node and manual restart to logviewer process.
was:
While securing cluster through Ambari (Storm only cluster), SPNEGO principals
for logviewers are not added for other supervisor nodes by ambari in
spnego.service.keytab. It only adds principal for Nimbus nodes, this results in
spnego.service.keytab only for Nimbus node.
Logviewer service for other nodes (supervisor) are not started because of this.
Copying the generated spnego.service.keytab from nimbus nodes to other nodes
leads to following error
(Mechanism level: Invalid argument (400) - Cannot find key of appropriate type
to decrypt AP REP - AES256 CTS mode with HMAC SHA1-96)
Also Ambari generates storm.yaml file on restarts of supervisor nodes and this
presently generates "kerberos.principal": "HTTP/<nimbus.host>" only whereas it
should generate kerberos principal for appropriate logviewer/supervisor node.
ui.filter.params:
"type": "kerberos"
"kerberos.principal": "HTTP/two.cluster"
"kerberos.keytab": "/etc/security/keytabs/spnego.service.keytab"
"kerberos.name.rules": "DEFAULT"
This leads to logviewer process initialize only with nimbus principal and later
on generate error while browsing UI of logviewer process with following error
after generating correct keytab which contains HTTP principals for each host
and distributing it to all supervisor/logviewer nodes, logviewer starts
properly but that require manual changes to storm.yaml file to change
kerberos.principal for that node and manual restart to logviewer process.
> SPNEGO principals are not added for logviewer for all supervisor nodes for
> secure storm cluster
> -----------------------------------------------------------------------------------------------
>
> Key: AMBARI-9721
> URL: https://issues.apache.org/jira/browse/AMBARI-9721
> Project: Ambari
> Issue Type: Bug
> Components: ambari-admin, ambari-server
> Affects Versions: 1.7.0
> Environment: CentOS 6.6 64bit
> Java jdk1.7.0_67
> Kerberos enabled
> Reporter: Manish Nema
> Labels: ambari-web, storm-security
>
> While securing cluster through Ambari (Storm only cluster), SPNEGO principals
> for logviewers are not added for other supervisor nodes by ambari in
> spnego.service.keytab. It only adds principal for Nimbus nodes, this results
> in spnego.service.keytab only for Nimbus node.
> Logviewer service for other nodes (supervisor) are not started because of
> this. Copying the generated spnego.service.keytab from nimbus nodes to other
> nodes leads to following error
> 2015-02-20 12:49:11 o.a.h.s.a.s.AuthenticationFilter [WARN] Authentication
> exception: GSSException: Failure unspecified at GSS-API level (Mechanism
> level: Checksum failed)
> org.apache.hadoop.security.authentication.client.AuthenticationException:
> GSSException: Failure unspecified at GSS-API level (Mechanism level: Checksum
> failed)
> at
> org.apache.hadoop.security.authentication.server.KerberosAuthenticationHandler.authenticate(KerberosAuthenticationHandler.java:360)
> ~[hadoop-auth-2.4.0.jar:na]
> at
> org.apache.hadoop.security.authentication.server.AuthenticationFilter.doFilter(AuthenticationFilter.java:357)
> ~[hadoop-auth-2.4.0.jar:na]
> at
> org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1291)
> [jetty-servlet-7.6.13.v20130916.jar:7.6.13.v20130916]
> at
> org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:443)
> [jetty-servlet-7.6.13.v20130916.jar:7.6.13.v20130916]
> at
> org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1044)
> [jetty-server-7.6.13.v20130916.jar:7.6.13.v20130916]
> at
> org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:372)
> [jetty-servlet-7.6.13.v20130916.jar:7.6.13.v20130916]
> at
> org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:978)
> [jetty-server-7.6.13.v20130916.jar:7.6.13.v20130916]
> at
> org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:135)
> [jetty-server-7.6.13.v20130916.jar:7.6.13.v20130916]
> at
> org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:116)
> [jetty-server-7.6.13.v20130916.jar:7.6.13.v20130916]
> at org.eclipse.jetty.server.Server.handle(Server.java:369)
> [jetty-server-7.6.13.v20130916.jar:7.6.13.v20130916]
> at
> org.eclipse.jetty.server.AbstractHttpConnection.handleRequest(AbstractHttpConnection.java:486)
> [jetty-server-7.6.13.v20130916.jar:7.6.13.v20130916]
> at
> org.eclipse.jetty.server.AbstractHttpConnection.headerComplete(AbstractHttpConnection.java:933)
> [jetty-server-7.6.13.v20130916.jar:7.6.13.v20130916]
> at
> org.eclipse.jetty.server.AbstractHttpConnection$RequestHandler.headerComplete(AbstractHttpConnection.java:995)
> [jetty-server-7.6.13.v20130916.jar:7.6.13.v20130916]
> at org.eclipse.jetty.http.HttpParser.parseNext(HttpParser.java:644)
> [jetty-http-7.6.13.v20130916.jar:7.6.13.v20130916]
> at
> org.eclipse.jetty.http.HttpParser.parseAvailable(HttpParser.java:235)
> [jetty-http-7.6.13.v20130916.jar:7.6.13.v20130916]
> at
> org.eclipse.jetty.server.AsyncHttpConnection.handle(AsyncHttpConnection.java:82)
> [jetty-server-7.6.13.v20130916.jar:7.6.13.v20130916]
> at
> org.eclipse.jetty.io.nio.SelectChannelEndPoint.handle(SelectChannelEndPoint.java:668)
> [jetty-io-7.6.13.v20130916.jar:7.6.13.v20130916]
> at
> org.eclipse.jetty.io.nio.SelectChannelEndPoint$1.run(SelectChannelEndPoint.java:52)
> [jetty-io-7.6.13.v20130916.jar:7.6.13.v20130916]
> at
> org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)
> [jetty-util-7.6.13.v20130916.jar:7.6.13.v20130916]
> at
> org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)
> [jetty-util-7.6.13.v20130916.jar:7.6.13.v20130916]
> at java.lang.Thread.run(Thread.java:745) [na:1.7.0_67]
> Caused by: org.ietf.jgss.GSSException: Failure unspecified at GSS-API level
> (Mechanism level: Checksum failed)
> at
> sun.security.jgss.krb5.Krb5Context.acceptSecContext(Krb5Context.java:788)
> ~[na:1.7.0_67]
> at
> sun.security.jgss.GSSContextImpl.acceptSecContext(GSSContextImpl.java:342)
> ~[na:1.7.0_67]
> at
> sun.security.jgss.GSSContextImpl.acceptSecContext(GSSContextImpl.java:285)
> ~[na:1.7.0_67]
> at
> sun.security.jgss.spnego.SpNegoContext.GSS_acceptSecContext(SpNegoContext.java:875)
> ~[na:1.7.0_67]
> at
> sun.security.jgss.spnego.SpNegoContext.acceptSecContext(SpNegoContext.java:548)
> ~[na:1.7.0_67]
> at
> sun.security.jgss.GSSContextImpl.acceptSecContext(GSSContextImpl.java:342)
> ~[na:1.7.0_67]
> at
> sun.security.jgss.GSSContextImpl.acceptSecContext(GSSContextImpl.java:285)
> ~[na:1.7.0_67]
> at
> org.apache.hadoop.security.authentication.server.KerberosAuthenticationHandler$2.run(KerberosAuthenticationHandler.java:327)
> ~[hadoop-auth-2.4.0.jar:na]
> at
> org.apache.hadoop.security.authentication.server.KerberosAuthenticationHandler$2.run(KerberosAuthenticationHandler.java:309)
> ~[hadoop-auth-2.4.0.jar:na]
> at java.security.AccessController.doPrivileged(Native Method)
> ~[na:1.7.0_67]
> at javax.security.auth.Subject.doAs(Subject.java:415) ~[na:1.7.0_67]
> at
> org.apache.hadoop.security.authentication.server.KerberosAuthenticationHandler.authenticate(KerberosAuthenticationHandler.java:309)
> ~[hadoop-auth-2.4.0.jar:na]
> ... 20 common frames omitted
> Caused by: sun.security.krb5.KrbCryptoException: Checksum failed
> at
> sun.security.krb5.internal.crypto.Aes256CtsHmacSha1EType.decrypt(Aes256CtsHmacSha1EType.java:102)
> ~[na:1.7.0_67]
> at
> sun.security.krb5.internal.crypto.Aes256CtsHmacSha1EType.decrypt(Aes256CtsHmacSha1EType.java:94)
> ~[na:1.7.0_67]
> at sun.security.krb5.EncryptedData.decrypt(EncryptedData.java:177)
> ~[na:1.7.0_67]
> at sun.security.krb5.KrbApReq.authenticate(KrbApReq.java:278)
> ~[na:1.7.0_67]
> at sun.security.krb5.KrbApReq.<init>(KrbApReq.java:144) ~[na:1.7.0_67]
> at
> sun.security.jgss.krb5.InitSecContextToken.<init>(InitSecContextToken.java:108)
> ~[na:1.7.0_67]
> at
> sun.security.jgss.krb5.Krb5Context.acceptSecContext(Krb5Context.java:771)
> ~[na:1.7.0_67]
> ... 31 common frames omitted
> Caused by: java.security.GeneralSecurityException: Checksum failed
> at
> sun.security.krb5.internal.crypto.dk.AesDkCrypto.decryptCTS(AesDkCrypto.java:451)
> ~[na:1.7.0_67]
> at
> sun.security.krb5.internal.crypto.dk.AesDkCrypto.decrypt(AesDkCrypto.java:272)
> ~[na:1.7.0_67]
> at sun.security.krb5.internal.crypto.Aes256.decrypt(Aes256.java:76)
> ~[na:1.7.0_67]
> at
> sun.security.krb5.internal.crypto.Aes256CtsHmacSha1EType.decrypt(Aes256CtsHmacSha1EType.java:100)
> ~[na:1.7.0_67]
> ... 37 common frames omitted
> Also Ambari generates storm.yaml file on restarts of supervisor nodes and
> this presently generates "kerberos.principal": "HTTP/<nimbus.host>" only
> whereas it should generate kerberos principal for appropriate
> logviewer/supervisor node.
> ui.filter.params:
> "type": "kerberos"
> "kerberos.principal": "HTTP/two.cluster"
> "kerberos.keytab": "/etc/security/keytabs/spnego.service.keytab"
> "kerberos.name.rules": "DEFAULT"
> This leads to logviewer process initialize only with nimbus principal and
> later on generate error while browsing UI of logviewer process with following
> error
> after generating correct keytab which contains HTTP principals for each host
> and distributing it to all supervisor/logviewer nodes, logviewer starts
> properly but that require manual changes to storm.yaml file to change
> kerberos.principal for that node and manual restart to logviewer process.
--
This message was sent by Atlassian JIRA
(v6.3.4#6332)
