zengqinchris commented on issue #3271: URL: https://github.com/apache/incubator-dolphinscheduler/issues/3271#issuecomment-662347276
> can't see the log clearly? could you paste the log info, not use screenshot ERROR StatusLogger No log4j2 configuration file found. Using default configuration: logging only errors to the console. Set system property 'log4j2.debug' to show Log4j2 internal initialization logging. Using Spark's default log4j profile: org/apache/spark/log4j-defaults.properties SLF4J: Class path contains multiple SLF4J bindings. SLF4J: Found binding in [jar:file:/data7/hadoop/yarn/local/filecache/1283/spark2-hdp-yarn-archive.tar.gz/slf4j-log4j12-1.7.16.jar!/org/slf4j/impl/StaticLoggerBinder.class] SLF4J: Found binding in [jar:file:/usr/hdp/3.1.0.0-78/hadoop/lib/slf4j-log4j12-1.7.25.jar!/org/slf4j/impl/StaticLoggerBinder.class] SLF4J: See http://www.slf4j.org/codes.html#multiple_bindings for an explanation. SLF4J: Actual binding is of type [org.slf4j.impl.Log4jLoggerFactory] Hive Session ID = b88c76b1-99c3-4b44-bcfd-56d66621edf0 org.apache.spark.SparkException: Job aborted. at org.apache.spark.internal.io.SparkHadoopWriter$.write(SparkHadoopWriter.scala:100) at org.apache.spark.rdd.PairRDDFunctions$$anonfun$saveAsNewAPIHadoopDataset$1.apply$mcV$sp(PairRDDFunctions.scala:1083) at org.apache.spark.rdd.PairRDDFunctions$$anonfun$saveAsNewAPIHadoopDataset$1.apply(PairRDDFunctions.scala:1081) at org.apache.spark.rdd.PairRDDFunctions$$anonfun$saveAsNewAPIHadoopDataset$1.apply(PairRDDFunctions.scala:1081) at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151) at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:112) at org.apache.spark.rdd.RDD.withScope(RDD.scala:363) at org.apache.spark.rdd.PairRDDFunctions.saveAsNewAPIHadoopDataset(PairRDDFunctions.scala:1081) at org.apache.spark.rdd.PairRDDFunctions$$anonfun$saveAsNewAPIHadoopFile$2.apply$mcV$sp(PairRDDFunctions.scala:1000) at org.apache.spark.rdd.PairRDDFunctions$$anonfun$saveAsNewAPIHadoopFile$2.apply(PairRDDFunctions.scala:991) at org.apache.spark.rdd.PairRDDFunctions$$anonfun$saveAsNewAPIHadoopFile$2.apply(PairRDDFunctions.scala:991) at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151) at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:112) at org.apache.spark.rdd.RDD.withScope(RDD.scala:363) at org.apache.spark.rdd.PairRDDFunctions.saveAsNewAPIHadoopFile(PairRDDFunctions.scala:991) at org.apache.phoenix.spark.DataFrameFunctions.saveToPhoenix(DataFrameFunctions.scala:59) at org.apache.phoenix.spark.DataFrameFunctions.saveToPhoenix(DataFrameFunctions.scala:28) at org.apache.phoenix.spark.DefaultSource.createRelation(DefaultSource.scala:47) at org.apache.spark.sql.execution.datasources.SaveIntoDataSourceCommand.run(SaveIntoDataSourceCommand.scala:45) at org.apache.spark.sql.execution.command.ExecutedCommandExec.sideEffectResult$lzycompute(commands.scala:70) at org.apache.spark.sql.execution.command.ExecutedCommandExec.sideEffectResult(commands.scala:68) at org.apache.spark.sql.execution.command.ExecutedCommandExec.doExecute(commands.scala:86) at org.apache.spark.sql.execution.SparkPlan$$anonfun$execute$1.apply(SparkPlan.scala:131) at org.apache.spark.sql.execution.SparkPlan$$anonfun$execute$1.apply(SparkPlan.scala:127) at org.apache.spark.sql.execution.SparkPlan$$anonfun$executeQuery$1.apply(SparkPlan.scala:155) at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151) at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:152) at org.apache.spark.sql.execution.SparkPlan.execute(SparkPlan.scala:127) at org.apache.spark.sql.execution.QueryExecution.toRdd$lzycompute(QueryExecution.scala:80) at org.apache.spark.sql.execution.QueryExecution.toRdd(QueryExecution.scala:80) at org.apache.spark.sql.DataFrameWriter$$anonfun$runCommand$1.apply(DataFrameWriter.scala:656) at org.apache.spark.sql.DataFrameWriter$$anonfun$runCommand$1.apply(DataFrameWriter.scala:656) at org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId(SQLExecution.scala:77) at org.apache.spark.sql.DataFrameWriter.runCommand(DataFrameWriter.scala:656) at org.apache.spark.sql.DataFrameWriter.saveToV1Source(DataFrameWriter.scala:273) at org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:267) at com.wumii.analysis.data.center.ods.client.ClientALLPointEventLog.finished(ClientALLPointEventLog.scala:236) at com.wumii.analysis.data.center.JobBase.run(JobBase.scala:45) at com.wumii.analysis.data.center.ClientBehaviorSparkJobLaunch$.runJob(ClientBehaviorSparkJobLaunch.scala:39) at com.wumii.analysis.data.center.ClientBehaviorSparkJobLaunch$.main(ClientBehaviorSparkJobLaunch.scala:28) at com.wumii.analysis.data.center.ClientBehaviorSparkJobLaunch.main(ClientBehaviorSparkJobLaunch.scala) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.apache.spark.deploy.yarn.ApplicationMaster$$anon$4.run(ApplicationMaster.scala:721) Caused by: org.apache.spark.SparkException: Job aborted due to stage failure: Task 15 in stage 11.0 failed 4 times, most recent failure: Lost task 15.3 in stage 11.0 (TID 4494, host, executor 2): org.apache.phoenix.exception.PhoenixIOException: callTimeout=1200000, callDuration=1593368: Failed after attempts=36, exceptions: Wed Jul 22 13:49:25 GMT+08:00 2020, RpcRetryingCaller{globalStartTime=1595396965709, pause=100, maxAttempts=36}, java.io.IOException: Call to xxxxxxxxx:16020 failed on local exception: java.io.IOException: Can not send request because relogin is in progress. Wed Jul 22 13:49:25 GMT+08:00 2020, RpcRetryingCaller{globalStartTime=1595396965709, pause=100, maxAttempts=36}, java.io.IOException: Call to xxxxxxxxx:16020 failed on local exception: java.io.IOException: Can not send request because relogin is in progress. Wed Jul 22 13:49:26 GMT+08:00 2020, RpcRetryingCaller{globalStartTime=1595396965709, pause=100, maxAttempts=36}, java.io.IOException: Call to xxxxxxxxx:16020 failed on local exception: java.io.IOException: Can not send request because relogin is in progress. Wed Jul 22 13:49:26 GMT+08:00 2020, RpcRetryingCaller{globalStartTime=1595396965709, pause=100, maxAttempts=36}, java.io.IOException: Call to xxxxxxxxx:16020 failed on local exception: java.io.IOException: Can not send request because relogin is in progress. Wed Jul 22 13:49:26 GMT+08:00 2020, RpcRetryingCaller{globalStartTime=1595396965709, pause=100, maxAttempts=36}, java.io.IOException: Call to xxxxxxxxx:16020 failed on local exception: java.io.IOException: Can not send request because relogin is in progress. Wed Jul 22 13:49:27 GMT+08:00 2020, RpcRetryingCaller{globalStartTime=1595396965709, pause=100, maxAttempts=36}, java.io.IOException: Call to xxxxxxxxx:16020 failed on local exception: java.io.IOException: Can not send request because relogin is in progress. Wed Jul 22 13:49:29 GMT+08:00 2020, RpcRetryingCaller{globalStartTime=1595396965709, pause=100, maxAttempts=36}, java.io.IOException: Call to xxxxxxxxx:16020 failed on local exception: javax.security.sasl.SaslException: GSS initiate failed [Caused by GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt)] Wed Jul 22 13:49:33 GMT+08:00 2020, RpcRetryingCaller{globalStartTime=1595396965709, pause=100, maxAttempts=36}, java.io.IOException: Call to xxxxxxxxx:16020 failed on local exception: javax.security.sasl.SaslException: GSS initiate failed [Caused by GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt)] Wed Jul 22 13:49:43 GMT+08:00 2020, RpcRetryingCaller{globalStartTime=1595396965709, pause=100, maxAttempts=36}, java.io.IOException: Call to xxxxxxxxx:16020 failed on local exception: javax.security.sasl.SaslException: GSS initiate failed [Caused by GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt)] Wed Jul 22 13:49:53 GMT+08:00 2020, RpcRetryingCaller{globalStartTime=1595396965709, pause=100, maxAttempts=36}, java.io.IOException: Call to xxxxxxxxx:16020 failed on local exception: javax.security.sasl.SaslException: GSS initiate failed [Caused by GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt)] Wed Jul 22 13:50:03 GMT+08:00 2020, RpcRetryingCaller{globalStartTime=1595396965709, pause=100, maxAttempts=36}, java.io.IOException: Call to xxxxxxxxx:16020 failed on local exception: javax.security.sasl.SaslException: GSS initiate failed [Caused by GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt)] Wed Jul 22 13:50:13 GMT+08:00 2020, RpcRetryingCaller{globalStartTime=1595396965709, pause=100, maxAttempts=36}, java.io.IOException: Call to xxxxxxxxx:16020 failed on local exception: javax.security.sasl.SaslException: GSS initiate failed [Caused by GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt)] Wed Jul 22 13:50:34 GMT+08:00 2020, RpcRetryingCaller{globalStartTime=1595396965709, pause=100, maxAttempts=36}, java.io.IOException: Call to xxxxxxxxx:16020 failed on local exception: javax.security.sasl.SaslException: GSS initiate failed [Caused by GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt)] Wed Jul 22 13:50:54 GMT+08:00 2020, RpcRetryingCaller{globalStartTime=1595396965709, pause=100, maxAttempts=36}, java.io.IOException: Call to xxxxxxxxx:16020 failed on local exception: javax.security.sasl.SaslException: GSS initiate failed [Caused by GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt)] Wed Jul 22 13:51:14 GMT+08:00 2020, RpcRetryingCaller{globalStartTime=1595396965709, pause=100, maxAttempts=36}, java.io.IOException: Call to xxxxxxxxx:16020 failed on local exception: javax.security.sasl.SaslException: GSS initiate failed [Caused by GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt)] Wed Jul 22 13:51:34 GMT+08:00 2020, RpcRetryingCaller{globalStartTime=1595396965709, pause=100, maxAttempts=36}, java.io.IOException: Call to xxxxxxxxx:16020 failed on local exception: javax.security.sasl.SaslException: GSS initiate failed [Caused by GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt)] Wed Jul 22 13:51:54 GMT+08:00 2020, RpcRetryingCaller{globalStartTime=1595396965709, pause=100, maxAttempts=36}, java.io.IOException: Call to xxxxxxxxx:16020 failed on local exception: javax.security.sasl.SaslException: GSS initiate failed [Caused by GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt)] Wed Jul 22 13:52:14 GMT+08:00 2020, RpcRetryingCaller{globalStartTime=1595396965709, pause=100, maxAttempts=36}, java.io.IOException: Call to xxxxxxxxx:16020 failed on local exception: javax.security.sasl.SaslException: GSS initiate failed [Caused by GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt)] Wed Jul 22 13:52:34 GMT+08:00 2020, RpcRetryingCaller{globalStartTime=1595396965709, pause=100, maxAttempts=36}, java.io.IOException: Call to xxxxxxxxx:16020 failed on local exception: javax.security.sasl.SaslException: GSS initiate failed [Caused by GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt)] Wed Jul 22 13:52:54 GMT+08:00 2020, RpcRetryingCaller{globalStartTime=1595396965709, pause=100, maxAttempts=36}, java.io.IOException: Call to xxxxxxxxx:16020 failed on local exception: javax.security.sasl.SaslException: GSS initiate failed [Caused by GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt)] Wed Jul 22 13:53:14 GMT+08:00 2020, RpcRetryingCaller{globalStartTime=1595396965709, pause=100, maxAttempts=36}, java.io.IOException: Call to xxxxxxxxx:16020 failed on local exception: javax.security.sasl.SaslException: GSS initiate failed [Caused by GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt)] Wed Jul 22 13:53:34 GMT+08:00 2020, RpcRetryingCaller{globalStartTime=1595396965709, pause=100, maxAttempts=36}, java.io.IOException: Call to xxxxxxxxx:16020 failed on local exception: javax.security.sasl.SaslException: GSS initiate failed [Caused by GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt)] Wed Jul 22 13:53:54 GMT+08:00 2020, RpcRetryingCaller{globalStartTime=1595396965709, pause=100, maxAttempts=36}, java.io.IOException: Call to xxxxxxxxx:16020 failed on local exception: javax.security.sasl.SaslException: GSS initiate failed [Caused by GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt)] Wed Jul 22 13:54:14 GMT+08:00 2020, RpcRetryingCaller{globalStartTime=1595396965709, pause=100, maxAttempts=36}, java.io.IOException: Call to xxxxxxxxx:16020 failed on local exception: javax.security.sasl.SaslException: GSS initiate failed [Caused by GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt)] Wed Jul 22 13:54:35 GMT+08:00 2020, RpcRetryingCaller{globalStartTime=1595396965709, pause=100, maxAttempts=36}, java.io.IOException: Call to xxxxxxxxx:16020 failed on local exception: javax.security.sasl.SaslException: GSS initiate failed [Caused by GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt)] Wed Jul 22 13:54:55 GMT+08:00 2020, RpcRetryingCaller{globalStartTime=1595396965709, pause=100, maxAttempts=36}, java.io.IOException: Call to xxxxxxxxx:16020 failed on local exception: javax.security.sasl.SaslException: GSS initiate failed [Caused by GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt)] Wed Jul 22 13:55:15 GMT+08:00 2020, RpcRetryingCaller{globalStartTime=1595396965709, pause=100, maxAttempts=36}, java.io.IOException: Call to xxxxxxxxx:16020 failed on local exception: javax.security.sasl.SaslException: GSS initiate failed [Caused by GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt)] Wed Jul 22 13:55:35 GMT+08:00 2020, RpcRetryingCaller{globalStartTime=1595396965709, pause=100, maxAttempts=36}, java.io.IOException: Call to xxxxxxxxx:16020 failed on local exception: javax.security.sasl.SaslException: GSS initiate failed [Caused by GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt)] Wed Jul 22 13:55:55 GMT+08:00 2020, RpcRetryingCaller{globalStartTime=1595396965709, pause=100, maxAttempts=36}, java.io.IOException: Call to xxxxxxxxx:16020 failed on local exception: javax.security.sasl.SaslException: GSS initiate failed [Caused by GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt)] Wed Jul 22 13:56:15 GMT+08:00 2020, RpcRetryingCaller{globalStartTime=1595396965709, pause=100, maxAttempts=36}, java.io.IOException: Call to xxxxxxxxx:16020 failed on local exception: javax.security.sasl.SaslException: GSS initiate failed [Caused by GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt)] Wed Jul 22 13:56:35 GMT+08:00 2020, RpcRetryingCaller{globalStartTime=1595396965709, pause=100, maxAttempts=36}, java.io.IOException: Call to xxxxxxxxx:16020 failed on local exception: javax.security.sasl.SaslException: GSS initiate failed [Caused by GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt)] Wed Jul 22 13:56:55 GMT+08:00 2020, RpcRetryingCaller{globalStartTime=1595396965709, pause=100, maxAttempts=36}, java.io.IOException: Call to xxxxxxxxx:16020 failed on local exception: javax.security.sasl.SaslException: GSS initiate failed [Caused by GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt)] Wed Jul 22 13:57:15 GMT+08:00 2020, RpcRetryingCaller{globalStartTime=1595396965709, pause=100, maxAttempts=36}, java.io.IOException: Call to xxxxxxxxx:16020 failed on local exception: javax.security.sasl.SaslException: GSS initiate failed [Caused by GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt)] Wed Jul 22 13:57:36 GMT+08:00 2020, RpcRetryingCaller{globalStartTime=1595396965709, pause=100, maxAttempts=36}, java.io.IOException: Call to xxxxxxxxx:16020 failed on local exception: javax.security.sasl.SaslException: GSS initiate failed [Caused by GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt)] Wed Jul 22 13:57:56 GMT+08:00 2020, RpcRetryingCaller{globalStartTime=1595396965709, pause=100, maxAttempts=36}, java.io.IOException: Call to xxxxxxxxx:16020 failed on local exception: javax.security.sasl.SaslException: GSS initiate failed [Caused by GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt)] Wed Jul 22 13:58:16 GMT+08:00 2020, RpcRetryingCaller{globalStartTime=1595396965709, pause=100, maxAttempts=36}, java.io.IOException: Call to xxxxxxxxx:16020 failed on local exception: javax.security.sasl.SaslException: GSS initiate failed [Caused by GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt)] at org.apache.phoenix.util.ServerUtil.parseServerException(ServerUtil.java:138) at org.apache.phoenix.query.ConnectionQueryServicesImpl.ensureTableCreated(ConnectionQueryServicesImpl.java:1205) at org.apache.phoenix.query.ConnectionQueryServicesImpl.createTable(ConnectionQueryServicesImpl.java:1502) at org.apache.phoenix.schema.MetaDataClient.createTableInternal(MetaDataClient.java:2740) at org.apache.phoenix.schema.MetaDataClient.createTable(MetaDataClient.java:1114) at org.apache.phoenix.compile.CreateTableCompiler$1.execute(CreateTableCompiler.java:192) at org.apache.phoenix.jdbc.PhoenixStatement$2.call(PhoenixStatement.java:408) at org.apache.phoenix.jdbc.PhoenixStatement$2.call(PhoenixStatement.java:391) at org.apache.phoenix.call.CallRunner.run(CallRunner.java:53) at org.apache.phoenix.jdbc.PhoenixStatement.executeMutation(PhoenixStatement.java:390) at org.apache.phoenix.jdbc.PhoenixStatement.executeMutation(PhoenixStatement.java:378) at org.apache.phoenix.jdbc.PhoenixStatement.executeUpdate(PhoenixStatement.java:1806) at org.apache.phoenix.query.ConnectionQueryServicesImpl$12.call(ConnectionQueryServicesImpl.java:2570) at org.apache.phoenix.query.ConnectionQueryServicesImpl$12.call(ConnectionQueryServicesImpl.java:2533) at org.apache.phoenix.util.PhoenixContextExecutor.call(PhoenixContextExecutor.java:76) at org.apache.phoenix.query.ConnectionQueryServicesImpl.init(ConnectionQueryServicesImpl.java:2533) at org.apache.phoenix.jdbc.PhoenixDriver.getConnectionQueryServices(PhoenixDriver.java:255) at org.apache.phoenix.jdbc.PhoenixEmbeddedDriver.createConnection(PhoenixEmbeddedDriver.java:150) at org.apache.phoenix.jdbc.PhoenixDriver.connect(PhoenixDriver.java:221) at java.sql.DriverManager.getConnection(DriverManager.java:664) at java.sql.DriverManager.getConnection(DriverManager.java:208) at org.apache.phoenix.mapreduce.util.ConnectionUtil.getConnection(ConnectionUtil.java:113) at org.apache.phoenix.mapreduce.util.ConnectionUtil.getOutputConnection(ConnectionUtil.java:97) at org.apache.phoenix.mapreduce.util.ConnectionUtil.getOutputConnection(ConnectionUtil.java:92) at org.apache.phoenix.mapreduce.util.ConnectionUtil.getOutputConnection(ConnectionUtil.java:71) at org.apache.phoenix.mapreduce.util.PhoenixConfigurationUtil.getUpsertColumnMetadataList(PhoenixConfigurationUtil.java:297) at org.apache.phoenix.spark.DataFrameFunctions$$anonfun$1.apply(DataFrameFunctions.scala:49) at org.apache.phoenix.spark.DataFrameFunctions$$anonfun$1.apply(DataFrameFunctions.scala:45) at org.apache.spark.rdd.RDD$$anonfun$mapPartitions$1$$anonfun$apply$23.apply(RDD.scala:801) at org.apache.spark.rdd.RDD$$anonfun$mapPartitions$1$$anonfun$apply$23.apply(RDD.scala:801) at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:49) at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:324) at org.apache.spark.rdd.RDD.iterator(RDD.scala:288) at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:87) at org.apache.spark.scheduler.Task.run(Task.scala:109) at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:345) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: java.net.SocketTimeoutException: callTimeout=1200000, callDuration=1593368: Failed after attempts=36, exceptions: Wed Jul 22 13:49:25 GMT+08:00 2020, RpcRetryingCaller{globalStartTime=1595396965709, pause=100, maxAttempts=36}, java.io.IOException: Call to xxxxxxxxx:16020 failed on local exception: java.io.IOException: Can not send request because relogin is in progress. Wed Jul 22 13:49:25 GMT+08:00 2020, RpcRetryingCaller{globalStartTime=1595396965709, pause=100, maxAttempts=36}, java.io.IOException: Call to xxxxxxxxx:16020 failed on local exception: java.io.IOException: Can not send request because relogin is in progress. Wed Jul 22 13:49:26 GMT+08:00 2020, RpcRetryingCaller{globalStartTime=1595396965709, pause=100, maxAttempts=36}, java.io.IOException: Call to xxxxxxxxx:16020 failed on local exception: java.io.IOException: Can not send request because relogin is in progress. Wed Jul 22 13:49:26 GMT+08:00 2020, RpcRetryingCaller{globalStartTime=1595396965709, pause=100, maxAttempts=36}, java.io.IOException: Call to xxxxxxxxx:16020 failed on local exception: java.io.IOException: Can not send request because relogin is in progress. Wed Jul 22 13:49:26 GMT+08:00 2020, RpcRetryingCaller{globalStartTime=1595396965709, pause=100, maxAttempts=36}, java.io.IOException: Call to xxxxxxxxx:16020 failed on local exception: java.io.IOException: Can not send request because relogin is in progress. Wed Jul 22 13:49:27 GMT+08:00 2020, RpcRetryingCaller{globalStartTime=1595396965709, pause=100, maxAttempts=36}, java.io.IOException: Call to xxxxxxxxx:16020 failed on local exception: java.io.IOException: Can not send request because relogin is in progress. Wed Jul 22 13:49:29 GMT+08:00 2020, RpcRetryingCaller{globalStartTime=1595396965709, pause=100, maxAttempts=36}, java.io.IOException: Call to xxxxxxxxx:16020 failed on local exception: javax.security.sasl.SaslException: GSS initiate failed [Caused by GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt)] Wed Jul 22 13:49:33 GMT+08:00 2020, RpcRetryingCaller{globalStartTime=1595396965709, pause=100, maxAttempts=36}, java.io.IOException: Call to xxxxxxxxx:16020 failed on local exception: javax.security.sasl.SaslException: GSS initiate failed [Caused by GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt)] Wed Jul 22 13:49:43 GMT+08:00 2020, RpcRetryingCaller{globalStartTime=1595396965709, pause=100, maxAttempts=36}, java.io.IOException: Call to xxxxxxxxx:16020 failed on local exception: javax.security.sasl.SaslException: GSS initiate failed [Caused by GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt)] Wed Jul 22 13:49:53 GMT+08:00 2020, RpcRetryingCaller{globalStartTime=1595396965709, pause=100, maxAttempts=36}, java.io.IOException: Call to xxxxxxxxx:16020 failed on local exception: javax.security.sasl.SaslException: GSS initiate failed [Caused by GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt)] Wed Jul 22 13:50:03 GMT+08:00 2020, RpcRetryingCaller{globalStartTime=1595396965709, pause=100, maxAttempts=36}, java.io.IOException: Call to xxxxxxxxx:16020 failed on local exception: javax.security.sasl.SaslException: GSS initiate failed [Caused by GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt)] Wed Jul 22 13:50:13 GMT+08:00 2020, RpcRetryingCaller{globalStartTime=1595396965709, pause=100, maxAttempts=36}, java.io.IOException: Call to xxxxxxxxx:16020 failed on local exception: javax.security.sasl.SaslException: GSS initiate failed [Caused by GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt)] Wed Jul 22 13:50:34 GMT+08:00 2020, RpcRetryingCaller{globalStartTime=1595396965709, pause=100, maxAttempts=36}, java.io.IOException: Call to xxxxxxxxx:16020 failed on local exception: javax.security.sasl.SaslException: GSS initiate failed [Caused by GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt)] Wed Jul 22 13:50:54 GMT+08:00 2020, RpcRetryingCaller{globalStartTime=1595396965709, pause=100, maxAttempts=36}, java.io.IOException: Call to xxxxxxxxx:16020 failed on local exception: javax.security.sasl.SaslException: GSS initiate failed [Caused by GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt)] Wed Jul 22 13:51:14 GMT+08:00 2020, RpcRetryingCaller{globalStartTime=1595396965709, pause=100, maxAttempts=36}, java.io.IOException: Call to xxxxxxxxx:16020 failed on local exception: javax.security.sasl.SaslException: GSS initiate failed [Caused by GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt)] Wed Jul 22 13:51:34 GMT+08:00 2020, RpcRetryingCaller{globalStartTime=1595396965709, pause=100, maxAttempts=36}, java.io.IOException: Call to xxxxxxxxx:16020 failed on local exception: javax.security.sasl.SaslException: GSS initiate failed [Caused by GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt)] Wed Jul 22 13:51:54 GMT+08:00 2020, RpcRetryingCaller{globalStartTime=1595396965709, pause=100, maxAttempts=36}, java.io.IOException: Call to xxxxxxxxx:16020 failed on local exception: javax.security.sasl.SaslException: GSS initiate failed [Caused by GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt)] Wed Jul 22 13:52:14 GMT+08:00 2020, RpcRetryingCaller{globalStartTime=1595396965709, pause=100, maxAttempts=36}, java.io.IOException: Call to xxxxxxxxx:16020 failed on local exception: javax.security.sasl.SaslException: GSS initiate failed [Caused by GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt)] Wed Jul 22 13:52:34 GMT+08:00 2020, RpcRetryingCaller{globalStartTime=1595396965709, pause=100, maxAttempts=36}, java.io.IOException: Call to xxxxxxxxx:16020 failed on local exception: javax.security.sasl.SaslException: GSS initiate failed [Caused by GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt)] Wed Jul 22 13:52:54 GMT+08:00 2020, RpcRetryingCaller{globalStartTime=1595396965709, pause=100, maxAttempts=36}, java.io.IOException: Call to xxxxxxxxx:16020 failed on local exception: javax.security.sasl.SaslException: GSS initiate failed [Caused by GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt)] Wed Jul 22 13:53:14 GMT+08:00 2020, RpcRetryingCaller{globalStartTime=1595396965709, pause=100, maxAttempts=36}, java.io.IOException: Call to xxxxxxxxx:16020 failed on local exception: javax.security.sasl.SaslException: GSS initiate failed [Caused by GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt)] Wed Jul 22 13:53:34 GMT+08:00 2020, RpcRetryingCaller{globalStartTime=1595396965709, pause=100, maxAttempts=36}, java.io.IOException: Call to xxxxxxxxx:16020 failed on local exception: javax.security.sasl.SaslException: GSS initiate failed [Caused by GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt)] Wed Jul 22 13:53:54 GMT+08:00 2020, RpcRetryingCaller{globalStartTime=1595396965709, pause=100, maxAttempts=36}, java.io.IOException: Call to xxxxxxxxx:16020 failed on local exception: javax.security.sasl.SaslException: GSS initiate failed [Caused by GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt)] Wed Jul 22 13:54:14 GMT+08:00 2020, RpcRetryingCaller{globalStartTime=1595396965709, pause=100, maxAttempts=36}, java.io.IOException: Call to xxxxxxxxx:16020 failed on local exception: javax.security.sasl.SaslException: GSS initiate failed [Caused by GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt)] Wed Jul 22 13:54:35 GMT+08:00 2020, RpcRetryingCaller{globalStartTime=1595396965709, pause=100, maxAttempts=36}, java.io.IOException: Call to xxxxxxxxx:16020 failed on local exception: javax.security.sasl.SaslException: GSS initiate failed [Caused by GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt)] Wed Jul 22 13:54:55 GMT+08:00 2020, RpcRetryingCaller{globalStartTime=1595396965709, pause=100, maxAttempts=36}, java.io.IOException: Call to xxxxxxxxx:16020 failed on local exception: javax.security.sasl.SaslException: GSS initiate failed [Caused by GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt)] Wed Jul 22 13:55:15 GMT+08:00 2020, RpcRetryingCaller{globalStartTime=1595396965709, pause=100, maxAttempts=36}, java.io.IOException: Call to xxxxxxxxx:16020 failed on local exception: javax.security.sasl.SaslException: GSS initiate failed [Caused by GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt)] Wed Jul 22 13:55:35 GMT+08:00 2020, RpcRetryingCaller{globalStartTime=1595396965709, pause=100, maxAttempts=36}, java.io.IOException: Call to xxxxxxxxx:16020 failed on local exception: javax.security.sasl.SaslException: GSS initiate failed [Caused by GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt)] Wed Jul 22 13:55:55 GMT+08:00 2020, RpcRetryingCaller{globalStartTime=1595396965709, pause=100, maxAttempts=36}, java.io.IOException: Call to xxxxxxxxx:16020 failed on local exception: javax.security.sasl.SaslException: GSS initiate failed [Caused by GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt)] Wed Jul 22 13:56:15 GMT+08:00 2020, RpcRetryingCaller{globalStartTime=1595396965709, pause=100, maxAttempts=36}, java.io.IOException: Call to xxxxxxxxx:16020 failed on local exception: javax.security.sasl.SaslException: GSS initiate failed [Caused by GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt)] Wed Jul 22 13:56:35 GMT+08:00 2020, RpcRetryingCaller{globalStartTime=1595396965709, pause=100, maxAttempts=36}, java.io.IOException: Call to xxxxxxxxx:16020 failed on local exception: javax.security.sasl.SaslException: GSS initiate failed [Caused by GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt)] Wed Jul 22 13:56:55 GMT+08:00 2020, RpcRetryingCaller{globalStartTime=1595396965709, pause=100, maxAttempts=36}, java.io.IOException: Call to xxxxxxxxx:16020 failed on local exception: javax.security.sasl.SaslException: GSS initiate failed [Caused by GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt)] Wed Jul 22 13:57:15 GMT+08:00 2020, RpcRetryingCaller{globalStartTime=1595396965709, pause=100, maxAttempts=36}, java.io.IOException: Call to xxxxxxxxx:16020 failed on local exception: javax.security.sasl.SaslException: GSS initiate failed [Caused by GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt)] Wed Jul 22 13:57:36 GMT+08:00 2020, RpcRetryingCaller{globalStartTime=1595396965709, pause=100, maxAttempts=36}, java.io.IOException: Call to xxxxxxxxx:16020 failed on local exception: javax.security.sasl.SaslException: GSS initiate failed [Caused by GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt)] Wed Jul 22 13:57:56 GMT+08:00 2020, RpcRetryingCaller{globalStartTime=1595396965709, pause=100, maxAttempts=36}, java.io.IOException: Call to xxxxxxxxx:16020 failed on local exception: javax.security.sasl.SaslException: GSS initiate failed [Caused by GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt)] Wed Jul 22 13:58:16 GMT+08:00 2020, RpcRetryingCaller{globalStartTime=1595396965709, pause=100, maxAttempts=36}, java.io.IOException: Call to xxxxxxxxx:16020 failed on local exception: javax.security.sasl.SaslException: GSS initiate failed [Caused by GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt)] at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:159) at org.apache.hadoop.hbase.client.HBaseAdmin.executeCallable(HBaseAdmin.java:3080) at org.apache.hadoop.hbase.client.HBaseAdmin.executeCallable(HBaseAdmin.java:3072) at org.apache.hadoop.hbase.client.HBaseAdmin.tableExists(HBaseAdmin.java:438) at org.apache.phoenix.query.ConnectionQueryServicesImpl.ensureTableCreated(ConnectionQueryServicesImpl.java:1093) ... 37 more Caused by: org.apache.hadoop.hbase.client.RetriesExhaustedException: Failed after attempts=36, exceptions: Wed Jul 22 13:49:25 GMT+08:00 2020, RpcRetryingCaller{globalStartTime=1595396965709, pause=100, maxAttempts=36}, java.io.IOException: Call to xxxxxxxxx:16020 failed on local exception: java.io.IOException: Can not send request because relogin is in progress. Wed Jul 22 13:49:25 GMT+08:00 2020, RpcRetryingCaller{globalStartTime=1595396965709, pause=100, maxAttempts=36}, java.io.IOException: Call to xxxxxxxxx:16020 failed on local exception: java.io.IOException: Can not send request because relogin is in progress. Wed Jul 22 13:49:26 GMT+08:00 2020, RpcRetryingCaller{globalStartTime=1595396965709, pause=100, maxAttempts=36}, java.io.IOException: Call to xxxxxxxxx:16020 failed on local exception: java.io.IOException: Can not send request because relogin is in progress. Wed Jul 22 13:49:26 GMT+08:00 2020, RpcRetryingCaller{globalStartTime=1595396965709, pause=100, maxAttempts=36}, java.io.IOException: Call to xxxxxxxxx:16020 failed on local exception: java.io.IOException: Can not send request because relogin is in progress. Wed Jul 22 13:49:26 GMT+08:00 2020, RpcRetryingCaller{globalStartTime=1595396965709, pause=100, maxAttempts=36}, java.io.IOException: Call to xxxxxxxxx:16020 failed on local exception: java.io.IOException: Can not send request because relogin is in progress. Wed Jul 22 13:49:27 GMT+08:00 2020, RpcRetryingCaller{globalStartTime=1595396965709, pause=100, maxAttempts=36}, java.io.IOException: Call to xxxxxxxxx:16020 failed on local exception: java.io.IOException: Can not send request because relogin is in progress. Wed Jul 22 13:49:29 GMT+08:00 2020, RpcRetryingCaller{globalStartTime=1595396965709, pause=100, maxAttempts=36}, java.io.IOException: Call to xxxxxxxxx:16020 failed on local exception: javax.security.sasl.SaslException: GSS initiate failed [Caused by GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt)] Wed Jul 22 13:49:33 GMT+08:00 2020, RpcRetryingCaller{globalStartTime=1595396965709, pause=100, maxAttempts=36}, java.io.IOException: Call to xxxxxxxxx:16020 failed on local exception: javax.security.sasl.SaslException: GSS initiate failed [Caused by GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt)] Wed Jul 22 13:49:43 GMT+08:00 2020, RpcRetryingCaller{globalStartTime=1595396965709, pause=100, maxAttempts=36}, java.io.IOException: Call to xxxxxxxxx:16020 failed on local exception: javax.security.sasl.SaslException: GSS initiate failed [Caused by GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt)] Wed Jul 22 13:49:53 GMT+08:00 2020, RpcRetryingCaller{globalStartTime=1595396965709, pause=100, maxAttempts=36}, java.io.IOException: Call to xxxxxxxxx:16020 failed on local exception: javax.security.sasl.SaslException: GSS initiate failed [Caused by GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt)] Wed Jul 22 13:50:03 GMT+08:00 2020, RpcRetryingCaller{globalStartTime=1595396965709, pause=100, maxAttempts=36}, java.io.IOException: Call to xxxxxxxxx:16020 failed on local exception: javax.security.sasl.SaslException: GSS initiate failed [Caused by GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt)] Wed Jul 22 13:50:13 GMT+08:00 2020, RpcRetryingCaller{globalStartTime=1595396965709, pause=100, maxAttempts=36}, java.io.IOException: Call to xxxxxxxxx:16020 failed on local exception: javax.security.sasl.SaslException: GSS initiate failed [Caused by GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt)] Wed Jul 22 13:50:34 GMT+08:00 2020, RpcRetryingCaller{globalStartTime=1595396965709, pause=100, maxAttempts=36}, java.io.IOException: Call to xxxxxxxxx:16020 failed on local exception: javax.security.sasl.SaslException: GSS initiate failed [Caused by GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt)] Wed Jul 22 13:50:54 GMT+08:00 2020, RpcRetryingCaller{globalStartTime=1595396965709, pause=100, maxAttempts=36}, java.io.IOException: Call to xxxxxxxxx:16020 failed on local exception: javax.security.sasl.SaslException: GSS initiate failed [Caused by GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt)] Wed Jul 22 13:51:14 GMT+08:00 2020, RpcRetryingCaller{globalStartTime=1595396965709, pause=100, maxAttempts=36}, java.io.IOException: Call to xxxxxxxxx:16020 failed on local exception: javax.security.sasl.SaslException: GSS initiate failed [Caused by GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt)] Wed Jul 22 13:51:34 GMT+08:00 2020, RpcRetryingCaller{globalStartTime=1595396965709, pause=100, maxAttempts=36}, java.io.IOException: Call to xxxxxxxxx:16020 failed on local exception: javax.security.sasl.SaslException: GSS initiate failed [Caused by GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt)] Wed Jul 22 13:51:54 GMT+08:00 2020, RpcRetryingCaller{globalStartTime=1595396965709, pause=100, maxAttempts=36}, java.io.IOException: Call to xxxxxxxxx:16020 failed on local exception: javax.security.sasl.SaslException: GSS initiate failed [Caused by GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt)] Wed Jul 22 13:52:14 GMT+08:00 2020, RpcRetryingCaller{globalStartTime=1595396965709, pause=100, maxAttempts=36}, java.io.IOException: Call to xxxxxxxxx:16020 failed on local exception: javax.security.sasl.SaslException: GSS initiate failed [Caused by GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt)] Wed Jul 22 13:52:34 GMT+08:00 2020, RpcRetryingCaller{globalStartTime=1595396965709, pause=100, maxAttempts=36}, java.io.IOException: Call to xxxxxxxxx:16020 failed on local exception: javax.security.sasl.SaslException: GSS initiate failed [Caused by GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt)] Wed Jul 22 13:52:54 GMT+08:00 2020, RpcRetryingCaller{globalStartTime=1595396965709, pause=100, maxAttempts=36}, java.io.IOException: Call to xxxxxxxxx:16020 failed on local exception: javax.security.sasl.SaslException: GSS initiate failed [Caused by GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt)] Wed Jul 22 13:53:14 GMT+08:00 2020, RpcRetryingCaller{globalStartTime=1595396965709, pause=100, maxAttempts=36}, java.io.IOException: Call to xxxxxxxxx:16020 failed on local exception: javax.security.sasl.SaslException: GSS initiate failed [Caused by GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt)] Wed Jul 22 13:53:34 GMT+08:00 2020, RpcRetryingCaller{globalStartTime=1595396965709, pause=100, maxAttempts=36}, java.io.IOException: Call to xxxxxxxxx:16020 failed on local exception: javax.security.sasl.SaslException: GSS initiate failed [Caused by GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt)] Wed Jul 22 13:53:54 GMT+08:00 2020, RpcRetryingCaller{globalStartTime=1595396965709, pause=100, maxAttempts=36}, java.io.IOException: Call to xxxxxxxxx:16020 failed on local exception: javax.security.sasl.SaslException: GSS initiate failed [Caused by GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt)] Wed Jul 22 13:54:14 GMT+08:00 2020, RpcRetryingCaller{globalStartTime=1595396965709, pause=100, maxAttempts=36}, java.io.IOException: Call to xxxxxxxxx:16020 failed on local exception: javax.security.sasl.SaslException: GSS initiate failed [Caused by GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt)] Wed Jul 22 13:54:35 GMT+08:00 2020, RpcRetryingCaller{globalStartTime=1595396965709, pause=100, maxAttempts=36}, java.io.IOException: Call to xxxxxxxxx:16020 failed on local exception: javax.security.sasl.SaslException: GSS initiate failed [Caused by GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt)] Wed Jul 22 13:54:55 GMT+08:00 2020, RpcRetryingCaller{globalStartTime=1595396965709, pause=100, maxAttempts=36}, java.io.IOException: Call to xxxxxxxxx:16020 failed on local exception: javax.security.sasl.SaslException: GSS initiate failed [Caused by GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt)] Wed Jul 22 13:55:15 GMT+08:00 2020, RpcRetryingCaller{globalStartTime=1595396965709, pause=100, maxAttempts=36}, java.io.IOException: Call to xxxxxxxxx:16020 failed on local exception: javax.security.sasl.SaslException: GSS initiate failed [Caused by GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt)] Wed Jul 22 13:55:35 GMT+08:00 2020, RpcRetryingCaller{globalStartTime=1595396965709, pause=100, maxAttempts=36}, java.io.IOException: Call to xxxxxxxxx:16020 failed on local exception: javax.security.sasl.SaslException: GSS initiate failed [Caused by GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt)] Wed Jul 22 13:55:55 GMT+08:00 2020, RpcRetryingCaller{globalStartTime=1595396965709, pause=100, maxAttempts=36}, java.io.IOException: Call to xxxxxxxxx:16020 failed on local exception: javax.security.sasl.SaslException: GSS initiate failed [Caused by GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt)] Wed Jul 22 13:56:15 GMT+08:00 2020, RpcRetryingCaller{globalStartTime=1595396965709, pause=100, maxAttempts=36}, java.io.IOException: Call to xxxxxxxxx:16020 failed on local exception: javax.security.sasl.SaslException: GSS initiate failed [Caused by GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt)] Wed Jul 22 13:56:35 GMT+08:00 2020, RpcRetryingCaller{globalStartTime=1595396965709, pause=100, maxAttempts=36}, java.io.IOException: Call to xxxxxxxxx:16020 failed on local exception: javax.security.sasl.SaslException: GSS initiate failed [Caused by GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt)] Wed Jul 22 13:56:55 GMT+08:00 2020, RpcRetryingCaller{globalStartTime=1595396965709, pause=100, maxAttempts=36}, java.io.IOException: Call to xxxxxxxxx:16020 failed on local exception: javax.security.sasl.SaslException: GSS initiate failed [Caused by GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt)] Wed Jul 22 13:57:15 GMT+08:00 2020, RpcRetryingCaller{globalStartTime=1595396965709, pause=100, maxAttempts=36}, java.io.IOException: Call to xxxxxxxxx:16020 failed on local exception: javax.security.sasl.SaslException: GSS initiate failed [Caused by GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt)] Wed Jul 22 13:57:36 GMT+08:00 2020, RpcRetryingCaller{globalStartTime=1595396965709, pause=100, maxAttempts=36}, java.io.IOException: Call to xxxxxxxxx:16020 failed on local exception: javax.security.sasl.SaslException: GSS initiate failed [Caused by GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt)] Wed Jul 22 13:57:56 GMT+08:00 2020, RpcRetryingCaller{globalStartTime=1595396965709, pause=100, maxAttempts=36}, java.io.IOException: Call to xxxxxxxxx:16020 failed on local exception: javax.security.sasl.SaslException: GSS initiate failed [Caused by GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt)] Wed Jul 22 13:58:16 GMT+08:00 2020, RpcRetryingCaller{globalStartTime=1595396965709, pause=100, maxAttempts=36}, java.io.IOException: Call to xxxxxxxxx:16020 failed on local exception: javax.security.sasl.SaslException: GSS initiate failed [Caused by GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt)] at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:145) at org.apache.hadoop.hbase.client.HTable.get(HTable.java:386) at org.apache.hadoop.hbase.client.HTable.get(HTable.java:360) at org.apache.hadoop.hbase.MetaTableAccessor.getTableState(MetaTableAccessor.java:1066) at org.apache.hadoop.hbase.MetaTableAccessor.tableExists(MetaTableAccessor.java:389) at org.apache.hadoop.hbase.client.HBaseAdmin$6.rpcCall(HBaseAdmin.java:441) at org.apache.hadoop.hbase.client.HBaseAdmin$6.rpcCall(HBaseAdmin.java:438) at org.apache.hadoop.hbase.client.RpcRetryingCallable.call(RpcRetryingCallable.java:58) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:107) ... 41 more Caused by: java.io.IOException: Call to xxxxxxxxx:16020 failed on local exception: javax.security.sasl.SaslException: GSS initiate failed [Caused by GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt)] at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:185) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:390) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:95) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:410) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:406) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:103) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:118) at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:92) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:329) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:315) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:307) at org.apache.hbase.thirdparty.io.netty.channel.ChannelInboundHandlerAdapter.userEventTriggered(ChannelInboundHandlerAdapter.java:108) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:329) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:315) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:307) at org.apache.hbase.thirdparty.io.netty.channel.ChannelInboundHandlerAdapter.userEventTriggered(ChannelInboundHandlerAdapter.java:108) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.userEventTriggered(ByteToMessageDecoder.java:353) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:329) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:315) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:307) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1377) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:329) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:315) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:929) at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:179) at org.apache.hadoop.hbase.ipc.NettyRpcConnection.access$500(NettyRpcConnection.java:71) at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:247) at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:507) at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:481) at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:420) at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.addListener(DefaultPromise.java:163) at org.apache.hadoop.hbase.ipc.NettyRpcConnection.saslNegotiate(NettyRpcConnection.java:201) at org.apache.hadoop.hbase.ipc.NettyRpcConnection.access$800(NettyRpcConnection.java:71) at org.apache.hadoop.hbase.ipc.NettyRpcConnection$3.operationComplete(NettyRpcConnection.java:273) at org.apache.hadoop.hbase.ipc.NettyRpcConnection$3.operationComplete(NettyRpcConnection.java:261) at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:507) at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:500) at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:479) at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:420) at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.trySuccess(DefaultPromise.java:104) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPromise.trySuccess(DefaultChannelPromise.java:82) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:306) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:341) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:633) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:580) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:497) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:459) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:858) at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultThreadFactory$DefaultRunnableDecorator.run(DefaultThreadFactory.java:138) ... 1 more Caused by: javax.security.sasl.SaslException: GSS initiate failed [Caused by GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt)] at com.sun.security.sasl.gsskerb.GssKrb5Client.evaluateChallenge(GssKrb5Client.java:211) at org.apache.hadoop.hbase.security.AbstractHBaseSaslRpcClient.getInitialResponse(AbstractHBaseSaslRpcClient.java:131) at org.apache.hadoop.hbase.security.NettyHBaseSaslRpcClientHandler$1.run(NettyHBaseSaslRpcClientHandler.java:109) at org.apache.hadoop.hbase.security.NettyHBaseSaslRpcClientHandler$1.run(NettyHBaseSaslRpcClientHandler.java:105) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1730) at org.apache.hadoop.hbase.security.NettyHBaseSaslRpcClientHandler.handlerAdded(NettyHBaseSaslRpcClientHandler.java:105) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.callHandlerAdded0(DefaultChannelPipeline.java:606) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.addFirst(DefaultChannelPipeline.java:187) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.addFirst(DefaultChannelPipeline.java:380) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.addFirst(DefaultChannelPipeline.java:359) at org.apache.hadoop.hbase.ipc.NettyRpcConnection.saslNegotiate(NettyRpcConnection.java:200) ... 18 more Caused by: GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt) at sun.security.jgss.krb5.Krb5InitCredential.getInstance(Krb5InitCredential.java:162) at sun.security.jgss.krb5.Krb5MechFactory.getCredentialElement(Krb5MechFactory.java:122) at sun.security.jgss.krb5.Krb5MechFactory.getMechanismContext(Krb5MechFactory.java:189) at sun.security.jgss.GSSManagerImpl.getMechanismContext(GSSManagerImpl.java:224) at sun.security.jgss.GSSContextImpl.initSecContext(GSSContextImpl.java:212) at sun.security.jgss.GSSContextImpl.initSecContext(GSSContextImpl.java:179) at com.sun.security.sasl.gsskerb.GssKrb5Client.evaluateChallenge(GssKrb5Client.java:192) ... 30 more Driver stacktrace: at org.apache.spark.scheduler.DAGScheduler.org$apache$spark$scheduler$DAGScheduler$$failJobAndIndependentStages(DAGScheduler.scala:1651) at org.apache.spark.scheduler.DAGScheduler$$anonfun$abortStage$1.apply(DAGScheduler.scala:1639) at org.apache.spark.scheduler.DAGScheduler$$anonfun$abortStage$1.apply(DAGScheduler.scala:1638) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at org.apache.spark.scheduler.DAGScheduler.abortStage(DAGScheduler.scala:1638) at org.apache.spark.scheduler.DAGScheduler$$anonfun$handleTaskSetFailed$1.apply(DAGScheduler.scala:831) at org.apache.spark.scheduler.DAGScheduler$$anonfun$handleTaskSetFailed$1.apply(DAGScheduler.scala:831) at scala.Option.foreach(Option.scala:257) at org.apache.spark.scheduler.DAGScheduler.handleTaskSetFailed(DAGScheduler.scala:831) at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.doOnReceive(DAGScheduler.scala:1872) at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:1821) at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:1810) at org.apache.spark.util.EventLoop$$anon$1.run(EventLoop.scala:48) at org.apache.spark.scheduler.DAGScheduler.runJob(DAGScheduler.scala:642) at org.apache.spark.SparkContext.runJob(SparkContext.scala:2034) at org.apache.spark.SparkContext.runJob(SparkContext.scala:2055) at org.apache.spark.SparkContext.runJob(SparkContext.scala:2087) at org.apache.spark.internal.io.SparkHadoopWriter$.write(SparkHadoopWriter.scala:78) ... 45 more Caused by: org.apache.phoenix.exception.PhoenixIOException: callTimeout=1200000, callDuration=1593368: Failed after attempts=36, exceptions: Wed Jul 22 13:49:25 GMT+08:00 2020, RpcRetryingCaller{globalStartTime=1595396965709, pause=100, maxAttempts=36}, java.io.IOException: Call to xxxxxxxxx:16020 failed on local exception: java.io.IOException: Can not send request because relogin is in progress. Wed Jul 22 13:49:25 GMT+08:00 2020, RpcRetryingCaller{globalStartTime=1595396965709, pause=100, maxAttempts=36}, java.io.IOException: Call to xxxxxxxxx:16020 failed on local exception: java.io.IOException: Can not send request because relogin is in progress. Wed Jul 22 13:49:26 GMT+08:00 2020, RpcRetryingCaller{globalStartTime=1595396965709, pause=100, maxAttempts=36}, java.io.IOException: Call to xxxxxxxxx:16020 failed on local exception: java.io.IOException: Can not send request because relogin is in progress. Wed Jul 22 13:49:26 GMT+08:00 2020, RpcRetryingCaller{globalStartTime=1595396965709, pause=100, maxAttempts=36}, java.io.IOException: Call to xxxxxxxxx:16020 failed on local exception: java.io.IOException: Can not send request because relogin is in progress. Wed Jul 22 13:49:26 GMT+08:00 2020, RpcRetryingCaller{globalStartTime=1595396965709, pause=100, maxAttempts=36}, java.io.IOException: Call to xxxxxxxxx:16020 failed on local exception: java.io.IOException: Can not send request because relogin is in progress. Wed Jul 22 13:49:27 GMT+08:00 2020, RpcRetryingCaller{globalStartTime=1595396965709, pause=100, maxAttempts=36}, java.io.IOException: Call to xxxxxxxxx:16020 failed on local exception: java.io.IOException: Can not send request because relogin is in progress. Wed Jul 22 13:49:29 GMT+08:00 2020, RpcRetryingCaller{globalStartTime=1595396965709, pause=100, maxAttempts=36}, java.io.IOException: Call to xxxxxxxxx:16020 failed on local exception: javax.security.sasl.SaslException: GSS initiate failed [Caused by GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt)] Wed Jul 22 13:49:33 GMT+08:00 2020, RpcRetryingCaller{globalStartTime=1595396965709, pause=100, maxAttempts=36}, java.io.IOException: Call to xxxxxxxxx:16020 failed on local exception: javax.security.sasl.SaslException: GSS initiate failed [Caused by GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt)] Wed Jul 22 13:49:43 GMT+08:00 2020, RpcRetryingCaller{globalStartTime=1595396965709, pause=100, maxAttempts=36}, java.io.IOException: Call to xxxxxxxxx:16020 failed on local exception: javax.security.sasl.SaslException: GSS initiate failed [Caused by GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt)] Wed Jul 22 13:49:53 GMT+08:00 2020, RpcRetryingCaller{globalStartTime=1595396965709, pause=100, maxAttempts=36}, java.io.IOException: Call to xxxxxxxxx:16020 failed on local exception: javax.security.sasl.SaslException: GSS initiate failed [Caused by GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt)] Wed Jul 22 13:50:03 GMT+08:00 2020, RpcRetryingCaller{globalStartTime=1595396965709, pause=100, maxAttempts=36}, java.io.IOException: Call to xxxxxxxxx:16020 failed on local exception: javax.security.sasl.SaslException: GSS initiate failed [Caused by GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt)] Wed Jul 22 13:50:13 GMT+08:00 2020, RpcRetryingCaller{globalStartTime=1595396965709, pause=100, maxAttempts=36}, java.io.IOException: Call to xxxxxxxxx:16020 failed on local exception: javax.security.sasl.SaslException: GSS initiate failed [Caused by GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt)] Wed Jul 22 13:50:34 GMT+08:00 2020, RpcRetryingCaller{globalStartTime=1595396965709, pause=100, maxAttempts=36}, java.io.IOException: Call to xxxxxxxxx:16020 failed on local exception: javax.security.sasl.SaslException: GSS initiate failed [Caused by GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt)] Wed Jul 22 13:50:54 GMT+08:00 2020, RpcRetryingCaller{globalStartTime=1595396965709, pause=100, maxAttempts=36}, java.io.IOException: Call to xxxxxxxxx:16020 failed on local exception: javax.security.sasl.SaslException: GSS initiate failed [Caused by GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt)] Wed Jul 22 13:51:14 GMT+08:00 2020, RpcRetryingCaller{globalStartTime=1595396965709, pause=100, maxAttempts=36}, java.io.IOException: Call to xxxxxxxxx:16020 failed on local exception: javax.security.sasl.SaslException: GSS initiate failed [Caused by GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt)] Wed Jul 22 13:51:34 GMT+08:00 2020, RpcRetryingCaller{globalStartTime=1595396965709, pause=100, maxAttempts=36}, java.io.IOException: Call to xxxxxxxxx:16020 failed on local exception: javax.security.sasl.SaslException: GSS initiate failed [Caused by GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt)] Wed Jul 22 13:51:54 GMT+08:00 2020, RpcRetryingCaller{globalStartTime=1595396965709, pause=100, maxAttempts=36}, java.io.IOException: Call to xxxxxxxxx:16020 failed on local exception: javax.security.sasl.SaslException: GSS initiate failed [Caused by GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt)] Wed Jul 22 13:52:14 GMT+08:00 2020, RpcRetryingCaller{globalStartTime=1595396965709, pause=100, maxAttempts=36}, java.io.IOException: Call to xxxxxxxxx:16020 failed on local exception: javax.security.sasl.SaslException: GSS initiate failed [Caused by GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt)] Wed Jul 22 13:52:34 GMT+08:00 2020, RpcRetryingCaller{globalStartTime=1595396965709, pause=100, maxAttempts=36}, java.io.IOException: Call to xxxxxxxxx:16020 failed on local exception: javax.security.sasl.SaslException: GSS initiate failed [Caused by GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt)] Wed Jul 22 13:52:54 GMT+08:00 2020, RpcRetryingCaller{globalStartTime=1595396965709, pause=100, maxAttempts=36}, java.io.IOException: Call to xxxxxxxxx:16020 failed on local exception: javax.security.sasl.SaslException: GSS initiate failed [Caused by GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt)] Wed Jul 22 13:53:14 GMT+08:00 2020, RpcRetryingCaller{globalStartTime=1595396965709, pause=100, maxAttempts=36}, java.io.IOException: Call to xxxxxxxxx:16020 failed on local exception: javax.security.sasl.SaslException: GSS initiate failed [Caused by GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt)] Wed Jul 22 13:53:34 GMT+08:00 2020, RpcRetryingCaller{globalStartTime=1595396965709, pause=100, maxAttempts=36}, java.io.IOException: Call to xxxxxxxxx:16020 failed on local exception: javax.security.sasl.SaslException: GSS initiate failed [Caused by GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt)] Wed Jul 22 13:53:54 GMT+08:00 2020, RpcRetryingCaller{globalStartTime=1595396965709, pause=100, maxAttempts=36}, java.io.IOException: Call to xxxxxxxxx:16020 failed on local exception: javax.security.sasl.SaslException: GSS initiate failed [Caused by GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt)] Wed Jul 22 13:54:14 GMT+08:00 2020, RpcRetryingCaller{globalStartTime=1595396965709, pause=100, maxAttempts=36}, java.io.IOException: Call to xxxxxxxxx:16020 failed on local exception: javax.security.sasl.SaslException: GSS initiate failed [Caused by GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt)] Wed Jul 22 13:54:35 GMT+08:00 2020, RpcRetryingCaller{globalStartTime=1595396965709, pause=100, maxAttempts=36}, java.io.IOException: Call to xxxxxxxxx:16020 failed on local exception: javax.security.sasl.SaslException: GSS initiate failed [Caused by GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt)] Wed Jul 22 13:54:55 GMT+08:00 2020, RpcRetryingCaller{globalStartTime=1595396965709, pause=100, maxAttempts=36}, java.io.IOException: Call to xxxxxxxxx:16020 failed on local exception: javax.security.sasl.SaslException: GSS initiate failed [Caused by GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt)] Wed Jul 22 13:55:15 GMT+08:00 2020, RpcRetryingCaller{globalStartTime=1595396965709, pause=100, maxAttempts=36}, java.io.IOException: Call to xxxxxxxxx:16020 failed on local exception: javax.security.sasl.SaslException: GSS initiate failed [Caused by GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt)] Wed Jul 22 13:55:35 GMT+08:00 2020, RpcRetryingCaller{globalStartTime=1595396965709, pause=100, maxAttempts=36}, java.io.IOException: Call to xxxxxxxxx:16020 failed on local exception: javax.security.sasl.SaslException: GSS initiate failed [Caused by GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt)] Wed Jul 22 13:55:55 GMT+08:00 2020, RpcRetryingCaller{globalStartTime=1595396965709, pause=100, maxAttempts=36}, java.io.IOException: Call to xxxxxxxxx:16020 failed on local exception: javax.security.sasl.SaslException: GSS initiate failed [Caused by GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt)] Wed Jul 22 13:56:15 GMT+08:00 2020, RpcRetryingCaller{globalStartTime=1595396965709, pause=100, maxAttempts=36}, java.io.IOException: Call to xxxxxxxxx:16020 failed on local exception: javax.security.sasl.SaslException: GSS initiate failed [Caused by GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt)] Wed Jul 22 13:56:35 GMT+08:00 2020, RpcRetryingCaller{globalStartTime=1595396965709, pause=100, maxAttempts=36}, java.io.IOException: Call to xxxxxxxxx:16020 failed on local exception: javax.security.sasl.SaslException: GSS initiate failed [Caused by GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt)] Wed Jul 22 13:56:55 GMT+08:00 2020, RpcRetryingCaller{globalStartTime=1595396965709, pause=100, maxAttempts=36}, java.io.IOException: Call to xxxxxxxxx:16020 failed on local exception: javax.security.sasl.SaslException: GSS initiate failed [Caused by GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt)] Wed Jul 22 13:57:15 GMT+08:00 2020, RpcRetryingCaller{globalStartTime=1595396965709, pause=100, maxAttempts=36}, java.io.IOException: Call to xxxxxxxxx:16020 failed on local exception: javax.security.sasl.SaslException: GSS initiate failed [Caused by GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt)] Wed Jul 22 13:57:36 GMT+08:00 2020, RpcRetryingCaller{globalStartTime=1595396965709, pause=100, maxAttempts=36}, java.io.IOException: Call to xxxxxxxxx:16020 failed on local exception: javax.security.sasl.SaslException: GSS initiate failed [Caused by GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt)] Wed Jul 22 13:57:56 GMT+08:00 2020, RpcRetryingCaller{globalStartTime=1595396965709, pause=100, maxAttempts=36}, java.io.IOException: Call to xxxxxxxxx:16020 failed on local exception: javax.security.sasl.SaslException: GSS initiate failed [Caused by GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt)] Wed Jul 22 13:58:16 GMT+08:00 2020, RpcRetryingCaller{globalStartTime=1595396965709, pause=100, maxAttempts=36}, java.io.IOException: Call to xxxxxxxxx:16020 failed on local exception: javax.security.sasl.SaslException: GSS initiate failed [Caused by GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt)] at org.apache.phoenix.util.ServerUtil.parseServerException(ServerUtil.java:138) at org.apache.phoenix.query.ConnectionQueryServicesImpl.ensureTableCreated(ConnectionQueryServicesImpl.java:1205) at org.apache.phoenix.query.ConnectionQueryServicesImpl.createTable(ConnectionQueryServicesImpl.java:1502) at org.apache.phoenix.schema.MetaDataClient.createTableInternal(MetaDataClient.java:2740) at org.apache.phoenix.schema.MetaDataClient.createTable(MetaDataClient.java:1114) at org.apache.phoenix.compile.CreateTableCompiler$1.execute(CreateTableCompiler.java:192) at org.apache.phoenix.jdbc.PhoenixStatement$2.call(PhoenixStatement.java:408) at org.apache.phoenix.jdbc.PhoenixStatement$2.call(PhoenixStatement.java:391) at org.apache.phoenix.call.CallRunner.run(CallRunner.java:53) at org.apache.phoenix.jdbc.PhoenixStatement.executeMutation(PhoenixStatement.java:390) at org.apache.phoenix.jdbc.PhoenixStatement.executeMutation(PhoenixStatement.java:378) at org.apache.phoenix.jdbc.PhoenixStatement.executeUpdate(PhoenixStatement.java:1806) at org.apache.phoenix.query.ConnectionQueryServicesImpl$12.call(ConnectionQueryServicesImpl.java:2570) at org.apache.phoenix.query.ConnectionQueryServicesImpl$12.call(ConnectionQueryServicesImpl.java:2533) at org.apache.phoenix.util.PhoenixContextExecutor.call(PhoenixContextExecutor.java:76) at org.apache.phoenix.query.ConnectionQueryServicesImpl.init(ConnectionQueryServicesImpl.java:2533) at org.apache.phoenix.jdbc.PhoenixDriver.getConnectionQueryServices(PhoenixDriver.java:255) at org.apache.phoenix.jdbc.PhoenixEmbeddedDriver.createConnection(PhoenixEmbeddedDriver.java:150) at org.apache.phoenix.jdbc.PhoenixDriver.connect(PhoenixDriver.java:221) at java.sql.DriverManager.getConnection(DriverManager.java:664) at java.sql.DriverManager.getConnection(DriverManager.java:208) at org.apache.phoenix.mapreduce.util.ConnectionUtil.getConnection(ConnectionUtil.java:113) at org.apache.phoenix.mapreduce.util.ConnectionUtil.getOutputConnection(ConnectionUtil.java:97) at org.apache.phoenix.mapreduce.util.ConnectionUtil.getOutputConnection(ConnectionUtil.java:92) at org.apache.phoenix.mapreduce.util.ConnectionUtil.getOutputConnection(ConnectionUtil.java:71) at org.apache.phoenix.mapreduce.util.PhoenixConfigurationUtil.getUpsertColumnMetadataList(PhoenixConfigurationUtil.java:297) at org.apache.phoenix.spark.DataFrameFunctions$$anonfun$1.apply(DataFrameFunctions.scala:49) at org.apache.phoenix.spark.DataFrameFunctions$$anonfun$1.apply(DataFrameFunctions.scala:45) at org.apache.spark.rdd.RDD$$anonfun$mapPartitions$1$$anonfun$apply$23.apply(RDD.scala:801) at org.apache.spark.rdd.RDD$$anonfun$mapPartitions$1$$anonfun$apply$23.apply(RDD.scala:801) at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:49) at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:324) at org.apache.spark.rdd.RDD.iterator(RDD.scala:288) at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:87) at org.apache.spark.scheduler.Task.run(Task.scala:109) at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:345) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: java.net.SocketTimeoutException: callTimeout=1200000, callDuration=1593368: Failed after attempts=36, exceptions: Wed Jul 22 13:49:25 GMT+08:00 2020, RpcRetryingCaller{globalStartTime=1595396965709, pause=100, maxAttempts=36}, java.io.IOException: Call to xxxxxxxxx:16020 failed on local exception: java.io.IOException: Can not send request because relogin is in progress. Wed Jul 22 13:49:25 GMT+08:00 2020, RpcRetryingCaller{globalStartTime=1595396965709, pause=100, maxAttempts=36}, java.io.IOException: Call to xxxxxxxxx:16020 failed on local exception: java.io.IOException: Can not send request because relogin is in progress. Wed Jul 22 13:49:26 GMT+08:00 2020, RpcRetryingCaller{globalStartTime=1595396965709, pause=100, maxAttempts=36}, java.io.IOException: Call to xxxxxxxxx:16020 failed on local exception: java.io.IOException: Can not send request because relogin is in progress. Wed Jul 22 13:49:26 GMT+08:00 2020, RpcRetryingCaller{globalStartTime=1595396965709, pause=100, maxAttempts=36}, java.io.IOException: Call to xxxxxxxxx:16020 failed on local exception: java.io.IOException: Can not send request because relogin is in progress. Wed Jul 22 13:49:26 GMT+08:00 2020, RpcRetryingCaller{globalStartTime=1595396965709, pause=100, maxAttempts=36}, java.io.IOException: Call to xxxxxxxxx:16020 failed on local exception: java.io.IOException: Can not send request because relogin is in progress. Wed Jul 22 13:49:27 GMT+08:00 2020, RpcRetryingCaller{globalStartTime=1595396965709, pause=100, maxAttempts=36}, java.io.IOException: Call to xxxxxxxxx:16020 failed on local exception: java.io.IOException: Can not send request because relogin is in progress. Wed Jul 22 13:49:29 GMT+08:00 2020, RpcRetryingCaller{globalStartTime=1595396965709, pause=100, maxAttempts=36}, java.io.IOException: Call to xxxxxxxxx:16020 failed on local exception: javax.security.sasl.SaslException: GSS initiate failed [Caused by GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt)] Wed Jul 22 13:49:33 GMT+08:00 2020, RpcRetryingCaller{globalStartTime=1595396965709, pause=100, maxAttempts=36}, java.io.IOException: Call to xxxxxxxxx:16020 failed on local exception: javax.security.sasl.SaslException: GSS initiate failed [Caused by GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt)] Wed Jul 22 13:49:43 GMT+08:00 2020, RpcRetryingCaller{globalStartTime=1595396965709, pause=100, maxAttempts=36}, java.io.IOException: Call to xxxxxxxxx:16020 failed on local exception: javax.security.sasl.SaslException: GSS initiate failed [Caused by GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt)] Wed Jul 22 13:49:53 GMT+08:00 2020, RpcRetryingCaller{globalStartTime=1595396965709, pause=100, maxAttempts=36}, java.io.IOException: Call to xxxxxxxxx:16020 failed on local exception: javax.security.sasl.SaslException: GSS initiate failed [Caused by GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt)] Wed Jul 22 13:50:03 GMT+08:00 2020, RpcRetryingCaller{globalStartTime=1595396965709, pause=100, maxAttempts=36}, java.io.IOException: Call to xxxxxxxxx:16020 failed on local exception: javax.security.sasl.SaslException: GSS initiate failed [Caused by GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt)] Wed Jul 22 13:50:13 GMT+08:00 2020, RpcRetryingCaller{globalStartTime=1595396965709, pause=100, maxAttempts=36}, java.io.IOException: Call to xxxxxxxxx:16020 failed on local exception: javax.security.sasl.SaslException: GSS initiate failed [Caused by GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt)] Wed Jul 22 13:50:34 GMT+08:00 2020, RpcRetryingCaller{globalStartTime=1595396965709, pause=100, maxAttempts=36}, java.io.IOException: Call to xxxxxxxxx:16020 failed on local exception: javax.security.sasl.SaslException: GSS initiate failed [Caused by GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt)] Wed Jul 22 13:50:54 GMT+08:00 2020, RpcRetryingCaller{globalStartTime=1595396965709, pause=100, maxAttempts=36}, java.io.IOException: Call to xxxxxxxxx:16020 failed on local exception: javax.security.sasl.SaslException: GSS initiate failed [Caused by GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt)] Wed Jul 22 13:51:14 GMT+08:00 2020, RpcRetryingCaller{globalStartTime=1595396965709, pause=100, maxAttempts=36}, java.io.IOException: Call to xxxxxxxxx:16020 failed on local exception: javax.security.sasl.SaslException: GSS initiate failed [Caused by GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt)] Wed Jul 22 13:51:34 GMT+08:00 2020, RpcRetryingCaller{globalStartTime=1595396965709, pause=100, maxAttempts=36}, java.io.IOException: Call to xxxxxxxxx:16020 failed on local exception: javax.security.sasl.SaslException: GSS initiate failed [Caused by GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt)] Wed Jul 22 13:51:54 GMT+08:00 2020, RpcRetryingCaller{globalStartTime=1595396965709, pause=100, maxAttempts=36}, java.io.IOException: Call to xxxxxxxxx:16020 failed on local exception: javax.security.sasl.SaslException: GSS initiate failed [Caused by GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt)] Wed Jul 22 13:52:14 GMT+08:00 2020, RpcRetryingCaller{globalStartTime=1595396965709, pause=100, maxAttempts=36}, java.io.IOException: Call to xxxxxxxxx:16020 failed on local exception: javax.security.sasl.SaslException: GSS initiate failed [Caused by GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt)] Wed Jul 22 13:52:34 GMT+08:00 2020, RpcRetryingCaller{globalStartTime=1595396965709, pause=100, maxAttempts=36}, java.io.IOException: Call to xxxxxxxxx:16020 failed on local exception: javax.security.sasl.SaslException: GSS initiate failed [Caused by GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt)] Wed Jul 22 13:52:54 GMT+08:00 2020, RpcRetryingCaller{globalStartTime=1595396965709, pause=100, maxAttempts=36}, java.io.IOException: Call to xxxxxxxxx:16020 failed on local exception: javax.security.sasl.SaslException: GSS initiate failed [Caused by GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt)] Wed Jul 22 13:53:14 GMT+08:00 2020, RpcRetryingCaller{globalStartTime=1595396965709, pause=100, maxAttempts=36}, java.io.IOException: Call to xxxxxxxxx:16020 failed on local exception: javax.security.sasl.SaslException: GSS initiate failed [Caused by GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt)] Wed Jul 22 13:53:34 GMT+08:00 2020, RpcRetryingCaller{globalStartTime=1595396965709, pause=100, maxAttempts=36}, java.io.IOException: Call to xxxxxxxxx:16020 failed on local exception: javax.security.sasl.SaslException: GSS initiate failed [Caused by GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt)] Wed Jul 22 13:53:54 GMT+08:00 2020, RpcRetryingCaller{globalStartTime=1595396965709, pause=100, maxAttempts=36}, java.io.IOException: Call to xxxxxxxxx:16020 failed on local exception: javax.security.sasl.SaslException: GSS initiate failed [Caused by GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt)] Wed Jul 22 13:54:14 GMT+08:00 2020, RpcRetryingCaller{globalStartTime=1595396965709, pause=100, maxAttempts=36}, java.io.IOException: Call to xxxxxxxxx:16020 failed on local exception: javax.security.sasl.SaslException: GSS initiate failed [Caused by GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt)] Wed Jul 22 13:54:35 GMT+08:00 2020, RpcRetryingCaller{globalStartTime=1595396965709, pause=100, maxAttempts=36}, java.io.IOException: Call to xxxxxxxxx:16020 failed on local exception: javax.security.sasl.SaslException: GSS initiate failed [Caused by GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt)] Wed Jul 22 13:54:55 GMT+08:00 2020, RpcRetryingCaller{globalStartTime=1595396965709, pause=100, maxAttempts=36}, java.io.IOException: Call to xxxxxxxxx:16020 failed on local exception: javax.security.sasl.SaslException: GSS initiate failed [Caused by GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt)] Wed Jul 22 13:55:15 GMT+08:00 2020, RpcRetryingCaller{globalStartTime=1595396965709, pause=100, maxAttempts=36}, java.io.IOException: Call to xxxxxxxxx:16020 failed on local exception: javax.security.sasl.SaslException: GSS initiate failed [Caused by GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt)] Wed Jul 22 13:55:35 GMT+08:00 2020, RpcRetryingCaller{globalStartTime=1595396965709, pause=100, maxAttempts=36}, java.io.IOException: Call to xxxxxxxxx:16020 failed on local exception: javax.security.sasl.SaslException: GSS initiate failed [Caused by GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt)] Wed Jul 22 13:55:55 GMT+08:00 2020, RpcRetryingCaller{globalStartTime=1595396965709, pause=100, maxAttempts=36}, java.io.IOException: Call to xxxxxxxxx:16020 failed on local exception: javax.security.sasl.SaslException: GSS initiate failed [Caused by GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt)] Wed Jul 22 13:56:15 GMT+08:00 2020, RpcRetryingCaller{globalStartTime=1595396965709, pause=100, maxAttempts=36}, java.io.IOException: Call to xxxxxxxxx:16020 failed on local exception: javax.security.sasl.SaslException: GSS initiate failed [Caused by GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt)] Wed Jul 22 13:56:35 GMT+08:00 2020, RpcRetryingCaller{globalStartTime=1595396965709, pause=100, maxAttempts=36}, java.io.IOException: Call to xxxxxxxxx:16020 failed on local exception: javax.security.sasl.SaslException: GSS initiate failed [Caused by GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt)] Wed Jul 22 13:56:55 GMT+08:00 2020, RpcRetryingCaller{globalStartTime=1595396965709, pause=100, maxAttempts=36}, java.io.IOException: Call to xxxxxxxxx:16020 failed on local exception: javax.security.sasl.SaslException: GSS initiate failed [Caused by GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt)] Wed Jul 22 13:57:15 GMT+08:00 2020, RpcRetryingCaller{globalStartTime=1595396965709, pause=100, maxAttempts=36}, java.io.IOException: Call to xxxxxxxxx:16020 failed on local exception: javax.security.sasl.SaslException: GSS initiate failed [Caused by GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt)] Wed Jul 22 13:57:36 GMT+08:00 2020, RpcRetryingCaller{globalStartTime=1595396965709, pause=100, maxAttempts=36}, java.io.IOException: Call to xxxxxxxxx:16020 failed on local exception: javax.security.sasl.SaslException: GSS initiate failed [Caused by GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt)] Wed Jul 22 13:57:56 GMT+08:00 2020, RpcRetryingCaller{globalStartTime=1595396965709, pause=100, maxAttempts=36}, java.io.IOException: Call to xxxxxxxxx:16020 failed on local exception: javax.security.sasl.SaslException: GSS initiate failed [Caused by GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt)] Wed Jul 22 13:58:16 GMT+08:00 2020, RpcRetryingCaller{globalStartTime=1595396965709, pause=100, maxAttempts=36}, java.io.IOException: Call to xxxxxxxxx:16020 failed on local exception: javax.security.sasl.SaslException: GSS initiate failed [Caused by GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt)] at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:159) at org.apache.hadoop.hbase.client.HBaseAdmin.executeCallable(HBaseAdmin.java:3080) at org.apache.hadoop.hbase.client.HBaseAdmin.executeCallable(HBaseAdmin.java:3072) at org.apache.hadoop.hbase.client.HBaseAdmin.tableExists(HBaseAdmin.java:438) at org.apache.phoenix.query.ConnectionQueryServicesImpl.ensureTableCreated(ConnectionQueryServicesImpl.java:1093) ... 37 more Caused by: org.apache.hadoop.hbase.client.RetriesExhaustedException: Failed after attempts=36, exceptions: Wed Jul 22 13:49:25 GMT+08:00 2020, RpcRetryingCaller{globalStartTime=1595396965709, pause=100, maxAttempts=36}, java.io.IOException: Call to xxxxxxxxx:16020 failed on local exception: java.io.IOException: Can not send request because relogin is in progress. Wed Jul 22 13:49:25 GMT+08:00 2020, RpcRetryingCaller{globalStartTime=1595396965709, pause=100, maxAttempts=36}, java.io.IOException: Call to xxxxxxxxx:16020 failed on local exception: java.io.IOException: Can not send request because relogin is in progress. Wed Jul 22 13:49:26 GMT+08:00 2020, RpcRetryingCaller{globalStartTime=1595396965709, pause=100, maxAttempts=36}, java.io.IOException: Call to xxxxxxxxx:16020 failed on local exception: java.io.IOException: Can not send request because relogin is in progress. Wed Jul 22 13:49:26 GMT+08:00 2020, RpcRetryingCaller{globalStartTime=1595396965709, pause=100, maxAttempts=36}, java.io.IOException: Call to xxxxxxxxx:16020 failed on local exception: java.io.IOException: Can not send request because relogin is in progress. Wed Jul 22 13:49:26 GMT+08:00 2020, RpcRetryingCaller{globalStartTime=1595396965709, pause=100, maxAttempts=36}, java.io.IOException: Call to xxxxxxxxx:16020 failed on local exception: java.io.IOException: Can not send request because relogin is in progress. Wed Jul 22 13:49:27 GMT+08:00 2020, RpcRetryingCaller{globalStartTime=1595396965709, pause=100, maxAttempts=36}, java.io.IOException: Call to xxxxxxxxx:16020 failed on local exception: java.io.IOException: Can not send request because relogin is in progress. Wed Jul 22 13:49:29 GMT+08:00 2020, RpcRetryingCaller{globalStartTime=1595396965709, pause=100, maxAttempts=36}, java.io.IOException: Call to xxxxxxxxx:16020 failed on local exception: javax.security.sasl.SaslException: GSS initiate failed [Caused by GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt)] Wed Jul 22 13:49:33 GMT+08:00 2020, RpcRetryingCaller{globalStartTime=1595396965709, pause=100, maxAttempts=36}, java.io.IOException: Call to xxxxxxxxx:16020 failed on local exception: javax.security.sasl.SaslException: GSS initiate failed [Caused by GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt)] Wed Jul 22 13:49:43 GMT+08:00 2020, RpcRetryingCaller{globalStartTime=1595396965709, pause=100, maxAttempts=36}, java.io.IOException: Call to xxxxxxxxx:16020 failed on local exception: javax.security.sasl.SaslException: GSS initiate failed [Caused by GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt)] Wed Jul 22 13:49:53 GMT+08:00 2020, RpcRetryingCaller{globalStartTime=1595396965709, pause=100, maxAttempts=36}, java.io.IOException: Call to xxxxxxxxx:16020 failed on local exception: javax.security.sasl.SaslException: GSS initiate failed [Caused by GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt)] Wed Jul 22 13:50:03 GMT+08:00 2020, RpcRetryingCaller{globalStartTime=1595396965709, pause=100, maxAttempts=36}, java.io.IOException: Call to xxxxxxxxx:16020 failed on local exception: javax.security.sasl.SaslException: GSS initiate failed [Caused by GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt)] Wed Jul 22 13:50:13 GMT+08:00 2020, RpcRetryingCaller{globalStartTime=1595396965709, pause=100, maxAttempts=36}, java.io.IOException: Call to xxxxxxxxx:16020 failed on local exception: javax.security.sasl.SaslException: GSS initiate failed [Caused by GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt)] Wed Jul 22 13:50:34 GMT+08:00 2020, RpcRetryingCaller{globalStartTime=1595396965709, pause=100, maxAttempts=36}, java.io.IOException: Call to xxxxxxxxx:16020 failed on local exception: javax.security.sasl.SaslException: GSS initiate failed [Caused by GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt)] Wed Jul 22 13:50:54 GMT+08:00 2020, RpcRetryingCaller{globalStartTime=1595396965709, pause=100, maxAttempts=36}, java.io.IOException: Call to xxxxxxxxx:16020 failed on local exception: javax.security.sasl.SaslException: GSS initiate failed [Caused by GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt)] Wed Jul 22 13:51:14 GMT+08:00 2020, RpcRetryingCaller{globalStartTime=1595396965709, pause=100, maxAttempts=36}, java.io.IOException: Call to xxxxxxxxx:16020 failed on local exception: javax.security.sasl.SaslException: GSS initiate failed [Caused by GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt)] Wed Jul 22 13:51:34 GMT+08:00 2020, RpcRetryingCaller{globalStartTime=1595396965709, pause=100, maxAttempts=36}, java.io.IOException: Call to xxxxxxxxx:16020 failed on local exception: javax.security.sasl.SaslException: GSS initiate failed [Caused by GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt)] Wed Jul 22 13:51:54 GMT+08:00 2020, RpcRetryingCaller{globalStartTime=1595396965709, pause=100, maxAttempts=36}, java.io.IOException: Call to xxxxxxxxx:16020 failed on local exception: javax.security.sasl.SaslException: GSS initiate failed [Caused by GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt)] Wed Jul 22 13:52:14 GMT+08:00 2020, RpcRetryingCaller{globalStartTime=1595396965709, pause=100, maxAttempts=36}, java.io.IOException: Call to xxxxxxxxx:16020 failed on local exception: javax.security.sasl.SaslException: GSS initiate failed [Caused by GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt)] Wed Jul 22 13:52:34 GMT+08:00 2020, RpcRetryingCaller{globalStartTime=1595396965709, pause=100, maxAttempts=36}, java.io.IOException: Call to xxxxxxxxx:16020 failed on local exception: javax.security.sasl.SaslException: GSS initiate failed [Caused by GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt)] Wed Jul 22 13:52:54 GMT+08:00 2020, RpcRetryingCaller{globalStartTime=1595396965709, pause=100, maxAttempts=36}, java.io.IOException: Call to xxxxxxxxx:16020 failed on local exception: javax.security.sasl.SaslException: GSS initiate failed [Caused by GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt)] Wed Jul 22 13:53:14 GMT+08:00 2020, RpcRetryingCaller{globalStartTime=1595396965709, pause=100, maxAttempts=36}, java.io.IOException: Call to xxxxxxxxx:16020 failed on local exception: javax.security.sasl.SaslException: GSS initiate failed [Caused by GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt)] Wed Jul 22 13:53:34 GMT+08:00 2020, RpcRetryingCaller{globalStartTime=1595396965709, pause=100, maxAttempts=36}, java.io.IOException: Call to xxxxxxxxx:16020 failed on local exception: javax.security.sasl.SaslException: GSS initiate failed [Caused by GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt)] Wed Jul 22 13:53:54 GMT+08:00 2020, RpcRetryingCaller{globalStartTime=1595396965709, pause=100, maxAttempts=36}, java.io.IOException: Call to xxxxxxxxx:16020 failed on local exception: javax.security.sasl.SaslException: GSS initiate failed [Caused by GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt)] Wed Jul 22 13:54:14 GMT+08:00 2020, RpcRetryingCaller{globalStartTime=1595396965709, pause=100, maxAttempts=36}, java.io.IOException: Call to xxxxxxxxx:16020 failed on local exception: javax.security.sasl.SaslException: GSS initiate failed [Caused by GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt)] Wed Jul 22 13:54:35 GMT+08:00 2020, RpcRetryingCaller{globalStartTime=1595396965709, pause=100, maxAttempts=36}, java.io.IOException: Call to xxxxxxxxx:16020 failed on local exception: javax.security.sasl.SaslException: GSS initiate failed [Caused by GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt)] Wed Jul 22 13:54:55 GMT+08:00 2020, RpcRetryingCaller{globalStartTime=1595396965709, pause=100, maxAttempts=36}, java.io.IOException: Call to xxxxxxxxx:16020 failed on local exception: javax.security.sasl.SaslException: GSS initiate failed [Caused by GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt)] Wed Jul 22 13:55:15 GMT+08:00 2020, RpcRetryingCaller{globalStartTime=1595396965709, pause=100, maxAttempts=36}, java.io.IOException: Call to xxxxxxxxx:16020 failed on local exception: javax.security.sasl.SaslException: GSS initiate failed [Caused by GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt)] Wed Jul 22 13:55:35 GMT+08:00 2020, RpcRetryingCaller{globalStartTime=1595396965709, pause=100, maxAttempts=36}, java.io.IOException: Call to xxxxxxxxx:16020 failed on local exception: javax.security.sasl.SaslException: GSS initiate failed [Caused by GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt)] Wed Jul 22 13:55:55 GMT+08:00 2020, RpcRetryingCaller{globalStartTime=1595396965709, pause=100, maxAttempts=36}, java.io.IOException: Call to xxxxxxxxx:16020 failed on local exception: javax.security.sasl.SaslException: GSS initiate failed [Caused by GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt)] Wed Jul 22 13:56:15 GMT+08:00 2020, RpcRetryingCaller{globalStartTime=1595396965709, pause=100, maxAttempts=36}, java.io.IOException: Call to xxxxxxxxx:16020 failed on local exception: javax.security.sasl.SaslException: GSS initiate failed [Caused by GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt)] Wed Jul 22 13:56:35 GMT+08:00 2020, RpcRetryingCaller{globalStartTime=1595396965709, pause=100, maxAttempts=36}, java.io.IOException: Call to xxxxxxxxx:16020 failed on local exception: javax.security.sasl.SaslException: GSS initiate failed [Caused by GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt)] Wed Jul 22 13:56:55 GMT+08:00 2020, RpcRetryingCaller{globalStartTime=1595396965709, pause=100, maxAttempts=36}, java.io.IOException: Call to xxxxxxxxx:16020 failed on local exception: javax.security.sasl.SaslException: GSS initiate failed [Caused by GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt)] Wed Jul 22 13:57:15 GMT+08:00 2020, RpcRetryingCaller{globalStartTime=1595396965709, pause=100, maxAttempts=36}, java.io.IOException: Call to xxxxxxxxx:16020 failed on local exception: javax.security.sasl.SaslException: GSS initiate failed [Caused by GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt)] Wed Jul 22 13:57:36 GMT+08:00 2020, RpcRetryingCaller{globalStartTime=1595396965709, pause=100, maxAttempts=36}, java.io.IOException: Call to xxxxxxxxx:16020 failed on local exception: javax.security.sasl.SaslException: GSS initiate failed [Caused by GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt)] Wed Jul 22 13:57:56 GMT+08:00 2020, RpcRetryingCaller{globalStartTime=1595396965709, pause=100, maxAttempts=36}, java.io.IOException: Call to xxxxxxxxx:16020 failed on local exception: javax.security.sasl.SaslException: GSS initiate failed [Caused by GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt)] Wed Jul 22 13:58:16 GMT+08:00 2020, RpcRetryingCaller{globalStartTime=1595396965709, pause=100, maxAttempts=36}, java.io.IOException: Call to xxxxxxxxx:16020 failed on local exception: javax.security.sasl.SaslException: GSS initiate failed [Caused by GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt)] at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:145) at org.apache.hadoop.hbase.client.HTable.get(HTable.java:386) at org.apache.hadoop.hbase.client.HTable.get(HTable.java:360) at org.apache.hadoop.hbase.MetaTableAccessor.getTableState(MetaTableAccessor.java:1066) at org.apache.hadoop.hbase.MetaTableAccessor.tableExists(MetaTableAccessor.java:389) at org.apache.hadoop.hbase.client.HBaseAdmin$6.rpcCall(HBaseAdmin.java:441) at org.apache.hadoop.hbase.client.HBaseAdmin$6.rpcCall(HBaseAdmin.java:438) at org.apache.hadoop.hbase.client.RpcRetryingCallable.call(RpcRetryingCallable.java:58) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:107) ... 41 more Caused by: java.io.IOException: Call to xxxxxxxxx:16020 failed on local exception: javax.security.sasl.SaslException: GSS initiate failed [Caused by GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt)] at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:185) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:390) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:95) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:410) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:406) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:103) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:118) at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:92) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:329) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:315) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:307) at org.apache.hbase.thirdparty.io.netty.channel.ChannelInboundHandlerAdapter.userEventTriggered(ChannelInboundHandlerAdapter.java:108) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:329) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:315) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:307) at org.apache.hbase.thirdparty.io.netty.channel.ChannelInboundHandlerAdapter.userEventTriggered(ChannelInboundHandlerAdapter.java:108) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.userEventTriggered(ByteToMessageDecoder.java:353) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:329) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:315) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:307) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1377) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:329) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:315) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:929) at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:179) at org.apache.hadoop.hbase.ipc.NettyRpcConnection.access$500(NettyRpcConnection.java:71) at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:247) at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:507) at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:481) at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:420) at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.addListener(DefaultPromise.java:163) at org.apache.hadoop.hbase.ipc.NettyRpcConnection.saslNegotiate(NettyRpcConnection.java:201) at org.apache.hadoop.hbase.ipc.NettyRpcConnection.access$800(NettyRpcConnection.java:71) at org.apache.hadoop.hbase.ipc.NettyRpcConnection$3.operationComplete(NettyRpcConnection.java:273) at org.apache.hadoop.hbase.ipc.NettyRpcConnection$3.operationComplete(NettyRpcConnection.java:261) at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:507) at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:500) at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:479) at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:420) at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.trySuccess(DefaultPromise.java:104) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPromise.trySuccess(DefaultChannelPromise.java:82) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:306) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:341) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:633) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:580) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:497) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:459) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:858) at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultThreadFactory$DefaultRunnableDecorator.run(DefaultThreadFactory.java:138) ... 1 more Caused by: javax.security.sasl.SaslException: GSS initiate failed [Caused by GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt)] at com.sun.security.sasl.gsskerb.GssKrb5Client.evaluateChallenge(GssKrb5Client.java:211) at org.apache.hadoop.hbase.security.AbstractHBaseSaslRpcClient.getInitialResponse(AbstractHBaseSaslRpcClient.java:131) at org.apache.hadoop.hbase.security.NettyHBaseSaslRpcClientHandler$1.run(NettyHBaseSaslRpcClientHandler.java:109) at org.apache.hadoop.hbase.security.NettyHBaseSaslRpcClientHandler$1.run(NettyHBaseSaslRpcClientHandler.java:105) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1730) at org.apache.hadoop.hbase.security.NettyHBaseSaslRpcClientHandler.handlerAdded(NettyHBaseSaslRpcClientHandler.java:105) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.callHandlerAdded0(DefaultChannelPipeline.java:606) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.addFirst(DefaultChannelPipeline.java:187) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.addFirst(DefaultChannelPipeline.java:380) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.addFirst(DefaultChannelPipeline.java:359) at org.apache.hadoop.hbase.ipc.NettyRpcConnection.saslNegotiate(NettyRpcConnection.java:200) ... 18 more Caused by: GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt) at sun.security.jgss.krb5.Krb5InitCredential.getInstance(Krb5InitCredential.java:162) at sun.security.jgss.krb5.Krb5MechFactory.getCredentialElement(Krb5MechFactory.java:122) at sun.security.jgss.krb5.Krb5MechFactory.getMechanismContext(Krb5MechFactory.java:189) at sun.security.jgss.GSSManagerImpl.getMechanismContext(GSSManagerImpl.java:224) at sun.security.jgss.GSSContextImpl.initSecContext(GSSContextImpl.java:212) at sun.security.jgss.GSSContextImpl.initSecContext(GSSContextImpl.java:179) at com.sun.security.sasl.gsskerb.GssKrb5Client.evaluateChallenge(GssKrb5Client.java:192) ... 30 more ---------------------------------------------------------------- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: [email protected]
