I see the following exception when running in secure mode using Kerberos. The exception appears to happen only under load and not consistently. It seems to be somewhat similar to the bug below but the exceptions are different. I am using 0.94.13.
Any advice is appreciated. https://issues.apache.org/jira/browse/HBASE-7684 2013-12-05 06:28:36.529 GMT DEBUG [23805:hbase-tablepool-1-thread-1] org.apache.hadoop.hbase.security.HBaseSaslRpcClient.saslConnect - SASL client context established. Negotiated QoP: auth-conf 2013-12-05 06:43:36.541 GMT WARN [23805:Thread-3] org.apache.hadoop.hbase. client.HConnectionManager$HConnectionImplementation.processBatchCallback - Failed all from region=appraisal,600,1386186300253. 2e7e692eddc816ee3ac8e405c3b949cb., hostname=plgrid-udn009.fanniemae.com, port=60020 java.util.concurrent.ExecutionException: java.net.SocketTimeoutException: Call to plgrid-udn009.fanniemae.com/10.198.99.9:60020 failed on socket timeout exception: java.net.SocketTimeoutException: 900000 millis timeout while waiting for channel to be ready for read. ch : java.nio.channels.SocketChannel[connected local=/10.198.99.13:59900 remote= plgrid-udn009.fanniemae.com/10.198.99.9:60020] at java.util.concurrent.FutureTask$Sync.innerGet( FutureTask.java:222) at java.util.concurrent.FutureTask.get(FutureTask.java:83) at org.apache.hadoop.hbase.client.HConnectionManager$ HConnectionImplementation.processBatchCallback(HConnectionManager.java:1709) at org.apache.hadoop.hbase.client.HConnectionManager$ HConnectionImplementation.processBatch(HConnectionManager.java:1561) at org.apache.hadoop.hbase.client.HTable.flushCommits( HTable.java:994) at org.apache.hadoop.hbase.client.HTable.doPut(HTable.java:850) at org.apache.hadoop.hbase.client.HTable.put(HTable.java:826) at org.apache.hadoop.hbase.mapreduce.TableOutputFormat$ TableRecordWriter.write(TableOutputFormat.java:123) at org.apache.hadoop.hbase.mapreduce.TableOutputFormat$ TableRecordWriter.write(TableOutputFormat.java:84) at com.platform.mapreduce.MRServiceJava.mapper. DirectMapOutputWriter.write(DirectMapOutputWriter.java:55) at org.apache.hadoop.mapreduce.TaskInputOutputContext.write( TaskInputOutputContext.java:80) at com.fanniemae.cram.hadoop.FileLoaderMapper.putDataInChunks( FileLoaderMapper.java:354) at com.fanniemae.cram.hadoop.FileLoaderMapper.map( FileLoaderMapper.java:466) at com.fanniemae.cram.hadoop.FileLoaderMapper.map( FileLoaderMapper.java:1) at org.apache.hadoop.mapreduce.Mapper.run_withoutCache( Mapper.java:206) at org.apache.hadoop.mapreduce.Mapper.run(Mapper.java:181) at com.platform.mapreduce.MRServiceJava.mapper.MapTask. runInternal(MapTask.java:262) at com.platform.mapreduce.MRServiceJava.mapper.MapTask. run(MapTask.java:109) at com.platform.mapreduce.MRServiceJava.MRService$2.run( MRService.java:579) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:396) at org.apache.hadoop.security.UserGroupInformation.doAs( UserGroupInformation.java:1198) at com.platform.mapreduce.work.adaptor.CredentialsAdaptor. doAs(CredentialsAdaptor.java:232) at com.platform.mapreduce.MRServiceJava.MRService. onInvoke(MRService.java:573) at com.platform.symphony.soam.ServiceContainer.invoke( ServiceContainer.java:363) at com.platform.symphony.soam.SoamCallbackWorkerThread. nativeProcessNextCallbackEvent(Native Method) at com.platform.symphony.soam.SoamCallbackWorkerThread.run( SoamCallbackWorkerThread.java:77) Caused by: java.net.SocketTimeoutException: Call to plgrid-udn009.fanniemae.com/10.198.99.9:60020 failed on socket timeout exception: java.net.SocketTimeoutException: 900000 millis timeout while waiting for channel to be ready for read. ch : java.nio.channels.SocketChannel[connected local=/10.198.99.13:59900 remote= plgrid-udn009.fanniemae.com/10.198.99.9:60020] at org.apache.hadoop.hbase.ipc.HBaseClient.wrapException( HBaseClient.java:1043) at org.apache.hadoop.hbase.ipc.HBaseClient.call(HBaseClient. java:1016) at org.apache.hadoop.hbase.ipc.SecureRpcEngine$Invoker. invoke(SecureRpcEngine.java:107) at $Proxy5.multi(Unknown Source) at org.apache.hadoop.hbase.client.HConnectionManager$ HConnectionImplementation$3$1.call(HConnectionManager.java:1538) at org.apache.hadoop.hbase.client.HConnectionManager$ HConnectionImplementation$3$1.call(HConnectionManager.java:1536) at org.apache.hadoop.hbase.client.ServerCallable. withoutRetries(ServerCallable.java:229) at org.apache.hadoop.hbase.client.HConnectionManager$ HConnectionImplementation$3.call(HConnectionManager.java:1545) at org.apache.hadoop.hbase.client.HConnectionManager$ HConnectionImplementation$3.call(HConnectionManager.java:1533) at java.util.concurrent.FutureTask$Sync.innerRun( FutureTask.java:303) at java.util.concurrent.FutureTask.run(FutureTask.java:138) at java.util.concurrent.ThreadPoolExecutor$Worker. runTask(ThreadPoolExecutor.java:886) at java.util.concurrent.ThreadPoolExecutor$Worker.run( ThreadPoolExecutor.java:908) at java.lang.Thread.run(Thread.java:662) Caused by: java.net.SocketTimeoutException: 900000 millis timeout while waiting for channel to be ready for read. ch : java.nio.channels.SocketChannel[connected local=/10.198.99.13:59900 remote= plgrid-udn009.fanniemae.com/10.198.99.9:60020] at org.apache.hadoop.net.SocketIOWithTimeout.doIO( SocketIOWithTimeout.java:164) at org.apache.hadoop.net.SocketInputStream.read( SocketInputStream.java:155) at org.apache.hadoop.net.SocketInputStream.read( SocketInputStream.java:128) at java.io.DataInputStream.readFully(DataInputStream.java:178) at java.io.DataInputStream.readFully(DataInputStream.java:152) at org.apache.hadoop.security.SaslInputStream.readMoreData( SaslInputStream.java:88) at org.apache.hadoop.security.SaslInputStream.read( SaslInputStream.java:245) at java.io.FilterInputStream.read(FilterInputStream.java:116) at org.apache.hadoop.hbase.ipc.HBaseClient$Connection$ PingInputStream.read(HBaseClient.java:374) at java.io.BufferedInputStream.fill(BufferedInputStream.java:218) at java.io.BufferedInputStream.read(BufferedInputStream.java:237) at java.io.DataInputStream.readInt(DataInputStream.java:370) at org.apache.hadoop.hbase.ipc.SecureClient$SecureConnection. receiveResponse(SecureClient.java:366) at org.apache.hadoop.hbase.ipc.HBaseClient$Connection.run( HBaseClient.java:588)
