I have two hbase cluster and enable kerberos I want run saprk application at clusterA to read clusterB with kerberos
in my code I add initKerberos functin like this sparkSession.sparkContext.addFile("hdfs://clusterA/krb5ClusterB.conf") sparkSession.sparkContext.addFile("hdfs://clusterA/ClusterB.keytab") val krb5Path = SparkFiles.get("krb5ClusterB.conf") val principal = config.getJSONObject("auth").getString("principal") val keytab = SparkFiles.get("ClusterB.keytab") System.setProperty("java.security.krb5.conf", krb5Path) val conf = new Configuration conf.set("hadoop.security.authentication", "Kerberos") UserGroupInformation.setConfiguration(conf) UserGroupInformation.loginUserFromKeytab(principal, keytab) I want use this function to login ClusterB at clusterA I run spark-submit --master yarn --deploy-mode cluster --principal ClusterA/sp...@join.com --keytab /hadoop/app/ClusterA.keytab --class com.join.Synctool but I get a error diagnostics: User class threw exception: org.apache.hadoop.security.KerberosAuthException: failure to login: for principal: jztwk/had...@join.com from keytab /hadoop/yarn/nm3/usercache/jztwk/appcache/application_1627287887991_1323/spark-9ac70772-d48f-43c3-814d-be210a5b33ea/userFiles-cfd44a8b-b252-4154-8876-1967e1609ec5/jztwk.keytab javax.security.auth.login.LoginException: Cannot locate KDC at org.apache.hadoop.security.UserGroupInformation.doSubjectLogin(UserGroupInformation.java:1992) at org.apache.hadoop.security.UserGroupInformation.loginUserFromKeytabAndReturnUGI(UserGroupInformation.java:1360) at org.apache.hadoop.security.UserGroupInformation.loginUserFromKeytab(UserGroupInformation.java:1140) at com.join.hbase.reader.HbaseReader.initKerberos(HbaseReader.scala:203) at com.join.hbase.reader.HbaseReader.beforeDo(HbaseReader.scala:138) at com.join.Synctool$.main(Synctool.scala:327) at com.join.Synctool.main(Synctool.scala) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.apache.spark.deploy.yarn.ApplicationMaster$$anon$2.run(ApplicationMaster.scala:673) Caused by: javax.security.auth.login.LoginException: Cannot locate KDC at com.sun.security.auth.module.Krb5LoginModule.attemptAuthentication(Krb5LoginModule.java:808) at com.sun.security.auth.module.Krb5LoginModule.login(Krb5LoginModule.java:617) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at javax.security.auth.login.LoginContext.invoke(LoginContext.java:755) at javax.security.auth.login.LoginContext.access$000(LoginContext.java:195) at javax.security.auth.login.LoginContext$4.run(LoginContext.java:682) at javax.security.auth.login.LoginContext$4.run(LoginContext.java:680) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.login.LoginContext.invokePriv(LoginContext.java:680) at javax.security.auth.login.LoginContext.login(LoginContext.java:587) at org.apache.hadoop.security.UserGroupInformation$HadoopLoginContext.login(UserGroupInformation.java:2070) at org.apache.hadoop.security.UserGroupInformation.doSubjectLogin(UserGroupInformation.java:1982) ... 11 more Caused by: KrbException: Cannot locate KDC at sun.security.krb5.Config.getKDCList(Config.java:1121) at sun.security.krb5.KdcComm.send(KdcComm.java:218) at sun.security.krb5.KdcComm.send(KdcComm.java:200) at sun.security.krb5.KrbAsReqBuilder.send(KrbAsReqBuilder.java:335) at sun.security.krb5.KrbAsReqBuilder.action(KrbAsReqBuilder.java:488) at com.sun.security.auth.module.Krb5LoginModule.attemptAuthentication(Krb5LoginModule.java:780) ... 25 more Caused by: KrbException: Generic error (description in e-text) (60) - Unable to locate KDC for realm JOIN.COM at sun.security.krb5.Config.getKDCFromDNS(Config.java:1218) at sun.security.krb5.Config.getKDCList(Config.java:1094) ... 30 more ApplicationMaster host: bigdser5 ApplicationMaster RPC port: 34278 queue: root.users.jztwk start time: 1629690007498 final status: FAILED tracking URL: http://bigdser3:8088/proxy/application_1627287887991_1323/ user: jztwk 21/08/23 11:40:29 ERROR yarn.Client: Application diagnostics message: User class threw exception: org.apache.hadoop.security.KerberosAuthException: failure to login: for principal: jztwk/had...@join.com from keytab /hadoop/yarn/nm3/usercache/jztwk/appcache/application_1627287887991_1323/spark-9ac70772-d48f-43c3-814d-be210a5b33ea/userFiles-cfd44a8b-b252-4154-8876-1967e1609ec5/jztwk.keytab javax.security.auth.login.LoginException: Cannot locate KDC at org.apache.hadoop.security.UserGroupInformation.doSubjectLogin(UserGroupInformation.java:1992) at org.apache.hadoop.security.UserGroupInformation.loginUserFromKeytabAndReturnUGI(UserGroupInformation.java:1360) at org.apache.hadoop.security.UserGroupInformation.loginUserFromKeytab(UserGroupInformation.java:1140) at com.join.hbase.reader.HbaseReader.initKerberos(HbaseReader.scala:203) at com.join.hbase.reader.HbaseReader.beforeDo(HbaseReader.scala:138) at com.join.Synctool$.main(Synctool.scala:327) at com.join.Synctool.main(Synctool.scala) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.apache.spark.deploy.yarn.ApplicationMaster$$anon$2.run(ApplicationMaster.scala:673) Caused by: javax.security.auth.login.LoginException: Cannot locate KDC at com.sun.security.auth.module.Krb5LoginModule.attemptAuthentication(Krb5LoginModule.java:808) at com.sun.security.auth.module.Krb5LoginModule.login(Krb5LoginModule.java:617) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at javax.security.auth.login.LoginContext.invoke(LoginContext.java:755) at javax.security.auth.login.LoginContext.access$000(LoginContext.java:195) at javax.security.auth.login.LoginContext$4.run(LoginContext.java:682) at javax.security.auth.login.LoginContext$4.run(LoginContext.java:680) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.login.LoginContext.invokePriv(LoginContext.java:680) at javax.security.auth.login.LoginContext.login(LoginContext.java:587) at org.apache.hadoop.security.UserGroupInformation$HadoopLoginContext.login(UserGroupInformation.java:2070) at org.apache.hadoop.security.UserGroupInformation.doSubjectLogin(UserGroupInformation.java:1982) ... 11 more Caused by: KrbException: Cannot locate KDC at sun.security.krb5.Config.getKDCList(Config.java:1121) at sun.security.krb5.KdcComm.send(KdcComm.java:218) at sun.security.krb5.KdcComm.send(KdcComm.java:200) at sun.security.krb5.KrbAsReqBuilder.send(KrbAsReqBuilder.java:335) at sun.security.krb5.KrbAsReqBuilder.action(KrbAsReqBuilder.java:488) at com.sun.security.auth.module.Krb5LoginModule.attemptAuthentication(Krb5LoginModule.java:780) ... 25 more Caused by: KrbException: Generic error (description in e-text) (60) - Unable to locate KDC for realm JOIN.COM at sun.security.krb5.Config.getKDCFromDNS(Config.java:1218) at sun.security.krb5.Config.getKDCList(Config.java:1094) ... 30 more Exception in thread "main" org.apache.spark.SparkException: Application application_1627287887991_1323 finished with failed status at org.apache.spark.deploy.yarn.Client.run(Client.scala:1158) at org.apache.spark.deploy.yarn.YarnClusterApplication.start(Client.scala:1606) at org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:851) at org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:167) at org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:195) at org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:86) at org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:926) at org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:935) at org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala) 21/08/23 11:40:29 INFO util.ShutdownHookManager: Shutdown hook called igyu