Can someone please respond? I am kind of blocked and can't proceed further..
On Tuesday, October 2, 2018, 10:06:16 PM EDT, Bala <[email protected]>
wrote:
I am trying to stream data using hivebolt with kerberos security enabled.I did
set the keytab and principal of the hive user and did verify that I can do the
kinit. And based on the logs the storm is also succeeding with the kinit but
throwing the following exception.What am I doing wrong?
2018-10-02 19:34:53.222 o.a.t.t.TSaslTransport hive-bolt-0 [ERROR] SASL
negotiation failure
javax.security.sasl.SaslException: GSS initiate failed
at
com.sun.security.sasl.gsskerb.GssKrb5Client.evaluateChallenge(GssKrb5Client.java:211)
~[?:1.8.0_112]
at
org.apache.thrift.transport.TSaslClientTransport.handleSaslStartMessage(TSaslClientTransport.java:94)
~[stormjar.jar:?]
at
org.apache.thrift.transport.TSaslTransport.open(TSaslTransport.java:271)
[stormjar.jar:?]
at
org.apache.thrift.transport.TSaslClientTransport.open(TSaslClientTransport.java:37)
[stormjar.jar:?]
at
org.apache.hadoop.hive.thrift.client.TUGIAssumingTransport$1.run(TUGIAssumingTransport.java:52)
[stormjar.jar:?]
at
org.apache.hadoop.hive.thrift.client.TUGIAssumingTransport$1.run(TUGIAssumingTransport.java:49)
[stormjar.jar:?]
at java.security.AccessController.doPrivileged(Native Method)
~[?:1.8.0_112]
at javax.security.auth.Subject.doAs(Subject.java:422) [?:1.8.0_112]
at
org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1869)
[stormjar.jar:?]
at
org.apache.hadoop.hive.thrift.client.TUGIAssumingTransport.open(TUGIAssumingTransport.java:49)
[stormjar.jar:?]
at
org.apache.hadoop.hive.metastore.HiveMetaStoreClient.open(HiveMetaStoreClient.java:487)
[stormjar.jar:?]
at
org.apache.hadoop.hive.metastore.HiveMetaStoreClient.<init>(HiveMetaStoreClient.java:282)
[stormjar.jar:?]
at
org.apache.hadoop.hive.metastore.HiveMetaStoreClient.<init>(HiveMetaStoreClient.java:188)
[stormjar.jar:?]
at
org.apache.hive.hcatalog.common.HiveClientCache$CacheableHiveMetaStoreClient.<init>(HiveClientCache.java:406)
[stormjar.jar:?]
at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native
Method) ~[?:1.8.0_112]
at
sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62)
[?:1.8.0_112]
at
sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
[?:1.8.0_112]
at java.lang.reflect.Constructor.newInstance(Constructor.java:423)
[?:1.8.0_112]
at
org.apache.hadoop.hive.metastore.MetaStoreUtils.newInstance(MetaStoreUtils.java:1564)
[stormjar.jar:?]
at
org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.<init>(RetryingMetaStoreClient.java:92)
[stormjar.jar:?]
at
org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.getProxy(RetryingMetaStoreClient.java:138)
[stormjar.jar:?]
at
org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.getProxy(RetryingMetaStoreClient.java:124)
[stormjar.jar:?]
at
org.apache.hive.hcatalog.common.HiveClientCache$5.call(HiveClientCache.java:295)
[stormjar.jar:?]
at
org.apache.hive.hcatalog.common.HiveClientCache$5.call(HiveClientCache.java:291)
[stormjar.jar:?]
at
com.google.common.cache.LocalCache$LocalManualCache$1.load(LocalCache.java:4792)
[stormjar.jar:?]
at
com.google.common.cache.LocalCache$LoadingValueReference.loadFuture(LocalCache.java:3599)
[stormjar.jar:?]
at
com.google.common.cache.LocalCache$Segment.loadSync(LocalCache.java:2379)
[stormjar.jar:?]
at
com.google.common.cache.LocalCache$Segment.lockedGetOrLoad(LocalCache.java:2342)
[stormjar.jar:?]
at com.google.common.cache.LocalCache$Segment.get(LocalCache.java:2257)
[stormjar.jar:?]
at com.google.common.cache.LocalCache.get(LocalCache.java:4000)
[stormjar.jar:?]
at
com.google.common.cache.LocalCache$LocalManualCache.get(LocalCache.java:4789)
[stormjar.jar:?]
at
org.apache.hive.hcatalog.common.HiveClientCache.getOrCreate(HiveClientCache.java:291)
[stormjar.jar:?]
at
org.apache.hive.hcatalog.common.HiveClientCache.get(HiveClientCache.java:266)
[stormjar.jar:?]
at
org.apache.hive.hcatalog.common.HCatUtil.getHiveMetastoreClient(HCatUtil.java:558)
[stormjar.jar:?]
at
org.apache.hive.hcatalog.streaming.HiveEndPoint$ConnectionImpl.getMetaStoreClient(HiveEndPoint.java:544)
[stormjar.jar:?]
at
org.apache.hive.hcatalog.streaming.HiveEndPoint$ConnectionImpl.<init>(HiveEndPoint.java:312)
[stormjar.jar:?]
at
org.apache.hive.hcatalog.streaming.HiveEndPoint$ConnectionImpl.<init>(HiveEndPoint.java:278)
[stormjar.jar:?]
at
org.apache.hive.hcatalog.streaming.HiveEndPoint.newConnectionImpl(HiveEndPoint.java:215)
[stormjar.jar:?]
at
org.apache.hive.hcatalog.streaming.HiveEndPoint.access$000(HiveEndPoint.java:62)
[stormjar.jar:?]
at
org.apache.hive.hcatalog.streaming.HiveEndPoint$1.run(HiveEndPoint.java:202)
[stormjar.jar:?]
at
org.apache.hive.hcatalog.streaming.HiveEndPoint$1.run(HiveEndPoint.java:197)
[stormjar.jar:?]
at java.security.AccessController.doPrivileged(Native Method)
~[?:1.8.0_112]
at javax.security.auth.Subject.doAs(Subject.java:422) [?:1.8.0_112]
at
org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1869)
[stormjar.jar:?]
at
org.apache.hive.hcatalog.streaming.HiveEndPoint.newConnection(HiveEndPoint.java:196)
[stormjar.jar:?]
at org.apache.storm.hive.common.HiveWriter$6.call(HiveWriter.java:271)
[stormjar.jar:?]
at org.apache.storm.hive.common.HiveWriter$6.call(HiveWriter.java:267)
[stormjar.jar:?]
at org.apache.storm.hive.common.HiveWriter$11.call(HiveWriter.java:419)
[stormjar.jar:?]
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
[?:1.8.0_112]
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
[?:1.8.0_112]
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
[?:1.8.0_112]
at java.lang.Thread.run(Thread.java:745) [?:1.8.0_112]
Caused by: org.ietf.jgss.GSSException: No valid credentials provided (Mechanism
level: Server not found in Kerberos database (7) - LOOKING_UP_SERVER)
at
sun.security.jgss.krb5.Krb5Context.initSecContext(Krb5Context.java:770)
~[?:1.8.0_112]
at
sun.security.jgss.GSSContextImpl.initSecContext(GSSContextImpl.java:248)
~[?:1.8.0_112]
at
sun.security.jgss.GSSContextImpl.initSecContext(GSSContextImpl.java:179)
~[?:1.8.0_112]
at
com.sun.security.sasl.gsskerb.GssKrb5Client.evaluateChallenge(GssKrb5Client.java:192)
~[?:1.8.0_112]
... 51 more Caused by: sun.security.krb5.KrbException: Server not found
in Kerberos database (7) - LOOKING_UP_SERVER
at sun.security.krb5.KrbTgsRep.<init>(KrbTgsRep.java:73) ~[?:1.8.0_112]
at sun.security.krb5.KrbTgsReq.getReply(KrbTgsReq.java:251)
~[?:1.8.0_112]
at sun.security.krb5.KrbTgsReq.sendAndGetCreds(KrbTgsReq.java:262)
~[?:1.8.0_112]
at
sun.security.krb5.internal.CredentialsUtil.serviceCreds(CredentialsUtil.java:308)
~[?:1.8.0_112]
at
sun.security.krb5.internal.CredentialsUtil.acquireServiceCreds(CredentialsUtil.java:126)
~[?:1.8.0_112]
at
sun.security.krb5.Credentials.acquireServiceCreds(Credentials.java:458)
~[?:1.8.0_112]
at
sun.security.jgss.krb5.Krb5Context.initSecContext(Krb5Context.java:693)
~[?:1.8.0_112]
at
sun.security.jgss.GSSContextImpl.initSecContext(GSSContextImpl.java:248)
~[?:1.8.0_112]
at
sun.security.jgss.GSSContextImpl.initSecContext(GSSContextImpl.java:179)
~[?:1.8.0_112]
at
com.sun.security.sasl.gsskerb.GssKrb5Client.evaluateChallenge(GssKrb5Client.java:192)
~[?:1.8.0_112]
... 51 more
Caused by: sun.security.krb5.Asn1Exception: Identifier doesn't match expected
value (906)
at sun.security.krb5.internal.KDCRep.init(KDCRep.java:140)
~[?:1.8.0_112]
at sun.security.krb5.internal.TGSRep.init(TGSRep.java:65) ~[?:1.8.0_112]
at sun.security.krb5.internal.TGSRep.<init>(TGSRep.java:60)
~[?:1.8.0_112]
at sun.security.krb5.KrbTgsRep.<init>(KrbTgsRep.java:55) ~[?:1.8.0_112]
at sun.security.krb5.KrbTgsReq.getReply(KrbTgsReq.java:251)
~[?:1.8.0_112]
at sun.security.krb5.KrbTgsReq.sendAndGetCreds(KrbTgsReq.java:262)
~[?:1.8.0_112]
at
sun.security.krb5.internal.CredentialsUtil.serviceCreds(CredentialsUtil.java:308)
~[?:1.8.0_112]
at
sun.security.krb5.internal.CredentialsUtil.acquireServiceCreds(CredentialsUtil.java:126)
~[?:1.8.0_112]
at
sun.security.krb5.Credentials.acquireServiceCreds(Credentials.java:458)
~[?:1.8.0_112]
at
sun.security.jgss.krb5.Krb5Context.initSecContext(Krb5Context.java:693)
~[?:1.8.0_112]
at
sun.security.jgss.GSSContextImpl.initSecContext(GSSContextImpl.java:248)
~[?:1.8.0_112]
at
sun.security.jgss.GSSContextImpl.initSecContext(GSSContextImpl.java:179)
~[?:1.8.0_112]
at
com.sun.security.sasl.gsskerb.GssKrb5Client.evaluateChallenge(GssKrb5Client.java:192)
~[?:1.8.0_112]
... 51 more
2018-10-02 19:34:53.224 h.metastore hive-bolt-0 [WARN] Failed to connect to the
MetaStore Server...
2018-10-02 19:34:53.224 h.metastore hive-bolt-0 [INFO] Waiting 1 seconds before
next connection attempt.
