Hi,
I am seeing thrift crashes which seem to be originating due to some issues
at the region servers. Here is the log - this looks quite concerning since
the server is failing to acquire a lock even when blocking - what is going
on here ?
12/11/06 21:53:29 WARN client.HConnectionManager$HConnectionImplementation:
Failed all from
region=user_likes_pins_ts_tall,0025c0300000074b7fffffffb0563d20010f4010000d6d6a,1351650405902.7d695b6f2b9e13e97c8bde70641c8aee.,
hostname=ip-10-31-190-107.ec2.internal, port=60020
java.util.concurrent.ExecutionException: java.io.IOException:
java.io.IOException: java.lang.AssertionError: Should never fail to get
lock when blocking
at
org.apache.hadoop.hbase.regionserver.HRegion.doMiniBatchPut(HRegion.java:1850)
at
org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:1755)
at
org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:3109)
at sun.reflect.GeneratedMethodAccessor51.invoke(Unknown Source)
at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
at java.lang.reflect.Method.invoke(Method.java:597)
at
org.apache.hadoop.hbase.ipc.WritableRpcEngine$Server.call(WritableRpcEngine.java:369)
at
org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1326)
at
java.util.concurrent.FutureTask$Sync.innerGet(FutureTask.java:222)
at java.util.concurrent.FutureTask.get(FutureTask.java:83)
at
org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation.processBatchCallback(HConnectionManager.java:1558)
at
org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation.processBatch(HConnectionManager.java:1410)
at
org.apache.hadoop.hbase.client.HTable.flushCommits(HTable.java:943)
at org.apache.hadoop.hbase.client.HTable.doPut(HTable.java:820)
at org.apache.hadoop.hbase.client.HTable.put(HTable.java:795)
at
org.apache.hadoop.hbase.thrift.ThriftServer$HBaseHandler.mutateRowTs(ThriftServer.java:613)
at
org.apache.hadoop.hbase.thrift.ThriftServer$HBaseHandler.mutateRow(ThriftServer.java:576)
at
org.apache.hadoop.hbase.thrift.generated.Hbase$Processor$mutateRow.getResult(Hbase.java:3630)
at
org.apache.hadoop.hbase.thrift.generated.Hbase$Processor$mutateRow.getResult(Hbase.java:3618)
at
org.apache.thrift.ProcessFunction.process(ProcessFunction.java:32)
at org.apache.thrift.TBaseProcessor.process(TBaseProcessor.java:34)
at
org.apache.thrift.server.TThreadPoolServer$WorkerProcess.run(TThreadPoolServer.java:176)
at
java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
at java.lang.Thread.run(Thread.java:662)
Caused by: java.io.IOException: java.io.IOException:
java.lang.AssertionError: Should never fail to get lock when blocking
at
org.apache.hadoop.hbase.regionserver.HRegion.doMiniBatchPut(HRegion.java:1850)
at
org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:1755)
at
org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:3109)
at sun.reflect.GeneratedMethodAccessor51.invoke(Unknown Source)
at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
at java.lang.reflect.Method.invoke(Method.java:597)
at
org.apache.hadoop.hbase.ipc.WritableRpcEngine$Server.call(WritableRpcEngine.java:369)
at
org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1326)
at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native
Method)
at
sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:39)
at
sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:27)
at java.lang.reflect.Constructor.newInstance(Constructor.java:513)
at
org.apache.hadoop.hbase.RemoteExceptionHandler.decodeRemoteException(RemoteExceptionHandler.java:96)
at
org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation.translateException(HConnectionManager.java:1653)
at
org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation.getRegionServerWithoutRetries(HConnectionManager.java:1368)
at
org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation$3.call(HConnectionManager.java:1384)
at
org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation$3.call(HConnectionManager.java:1382)
at
java.util.concurrent.FutureTask$Sync.innerRun(FutureTask.java:303)
at java.util.concurrent.FutureTask.run(FutureTask.java:138)
... 3 more
Thanks
Varun
