[
https://issues.apache.org/jira/browse/HBASE-17118?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=15672402#comment-15672402
]
binlijin commented on HBASE-17118:
----------------------------------
We encounter the problem, and we find the problem at 2016.11.10 through
unnormal gc, and dump the heap, and find the StoreScanner leak, through the
heap dump we find the StoreScanner create at 2016-11-02
{code}
2016-11-02 07:36:15,056 ERROR [B.defaultRpcServer.handler=5,queue=5,port=16020]
ipc.RpcServer: Unexpected throwable object
java.lang.IllegalArgumentException
at java.nio.Buffer.limit(Buffer.java:267)
at
org.apache.hadoop.hbase.nio.SingleByteBuff.limit(SingleByteBuff.java:91)
at
org.apache.hadoop.hbase.nio.SingleByteBuff.limit(SingleByteBuff.java:33)
at
org.apache.hadoop.hbase.io.hfile.HFileBlock.getBufferReadOnly(HFileBlock.java:393)
at
org.apache.hadoop.hbase.io.hfile.ChecksumUtil.validateBlockChecksum(ChecksumUtil.java:158)
at
org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.validateBlockChecksum(HFileBlock.java:1737)
at
org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockDataInternal(HFileBlock.java:1686)
at
org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockData(HFileBlock.java:1495)
at
org.apache.hadoop.hbase.io.hfile.HFileReaderImpl.readBlock(HFileReaderImpl.java:1440)
at
org.apache.hadoop.hbase.io.hfile.HFileBlockIndex$CellBasedKeyBlockIndexReader.loadDataBlockWithScanInfo(HFileBlockIndex.java:322)
at
org.apache.hadoop.hbase.io.hfile.HFileReaderImpl$HFileScannerImpl.seekTo(HFileReaderImpl.java:817)
at
org.apache.hadoop.hbase.io.hfile.HFileReaderImpl$HFileScannerImpl.reseekTo(HFileReaderImpl.java:798)
at
org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseekAtOrAfter(StoreFileScanner.java:263)
at
org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseek(StoreFileScanner.java:180)
at
org.apache.hadoop.hbase.regionserver.NonLazyKeyValueScanner.doRealSeek(NonLazyKeyValueScanner.java:55)
at
org.apache.hadoop.hbase.regionserver.KeyValueHeap.generalizedSeek(KeyValueHeap.java:323)
at
org.apache.hadoop.hbase.regionserver.KeyValueHeap.reseek(KeyValueHeap.java:267)
at
org.apache.hadoop.hbase.regionserver.StoreScanner.reseek(StoreScanner.java:824)
at
org.apache.hadoop.hbase.regionserver.NonLazyKeyValueScanner.doRealSeek(NonLazyKeyValueScanner.java:55)
at
org.apache.hadoop.hbase.regionserver.NonLazyKeyValueScanner.requestSeek(NonLazyKeyValueScanner.java:39)
at
org.apache.hadoop.hbase.regionserver.KeyValueHeap.generalizedSeek(KeyValueHeap.java:321)
at
org.apache.hadoop.hbase.regionserver.KeyValueHeap.requestSeek(KeyValueHeap.java:279)
at
org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.joinedHeapMayHaveData(HRegion.java:5904)
at
org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextInternal(HRegion.java:5845)
at
org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:5589)
at
org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:2644)
at
org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:32205)
at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:839)
at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:102)
at
org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:133)
at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:108)
at java.lang.Thread.run(Thread.java:756)
2016-11-02 07:36:15,056 DEBUG [B.defaultRpcServer.handler=5,queue=5,port=16020]
ipc.RpcServer: B.defaultRpcServer.handler=5,queue=5,port=16020: callId: 115
service: ClientService methodName: Scan size: 29 connection: 11.180.36.86:54872
java.io.IOException
at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:894)
at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:102)
at
org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:133)
at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:108)
at java.lang.Thread.run(Thread.java:756)
Caused by: java.lang.IllegalArgumentException
at java.nio.Buffer.limit(Buffer.java:267)
at
org.apache.hadoop.hbase.nio.SingleByteBuff.limit(SingleByteBuff.java:91)
at
org.apache.hadoop.hbase.nio.SingleByteBuff.limit(SingleByteBuff.java:33)
at
org.apache.hadoop.hbase.io.hfile.HFileBlock.getBufferReadOnly(HFileBlock.java:393)
at
org.apache.hadoop.hbase.io.hfile.ChecksumUtil.validateBlockChecksum(ChecksumUtil.java:158)
at
org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.validateBlockChecksum(HFileBlock.java:1737)
at
org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockDataInternal(HFileBlock.java:1686)
at
org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockData(HFileBlock.java:1495)
at
org.apache.hadoop.hbase.io.hfile.HFileReaderImpl.readBlock(HFileReaderImpl.java:1440)
at
org.apache.hadoop.hbase.io.hfile.HFileBlockIndex$CellBasedKeyBlockIndexReader.loadDataBlockWithScanInfo(HFileBlockIndex.java:322)
at
org.apache.hadoop.hbase.io.hfile.HFileReaderImpl$HFileScannerImpl.seekTo(HFileReaderImpl.java:817)
at
org.apache.hadoop.hbase.io.hfile.HFileReaderImpl$HFileScannerImpl.reseekTo(HFileReaderImpl.java:798)
at
org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseekAtOrAfter(StoreFileScanner.java:263)
at
org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseek(StoreFileScanner.java:180)
at
org.apache.hadoop.hbase.regionserver.NonLazyKeyValueScanner.doRealSeek(NonLazyKeyValueScanner.java:55)
at
org.apache.hadoop.hbase.regionserver.KeyValueHeap.generalizedSeek(KeyValueHeap.java:323)
at
org.apache.hadoop.hbase.regionserver.KeyValueHeap.reseek(KeyValueHeap.java:267)
at
org.apache.hadoop.hbase.regionserver.StoreScanner.reseek(StoreScanner.java:824)
at
org.apache.hadoop.hbase.regionserver.NonLazyKeyValueScanner.doRealSeek(NonLazyKeyValueScanner.java:55)
at
org.apache.hadoop.hbase.regionserver.NonLazyKeyValueScanner.requestSeek(NonLazyKeyValueScanner.java:39)
at
org.apache.hadoop.hbase.regionserver.KeyValueHeap.generalizedSeek(KeyValueHeap.java:321)
at
org.apache.hadoop.hbase.regionserver.KeyValueHeap.requestSeek(KeyValueHeap.java:279)
at
org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.joinedHeapMayHaveData(HRegion.java:5904)
at
org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextInternal(HRegion.java:5845)
at
org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:5589)
at
org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:2644)
at
org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:32205)
at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:839)
... 4 more
{code}
> StoreScanner leaked in KeyValueHeap
> -----------------------------------
>
> Key: HBASE-17118
> URL: https://issues.apache.org/jira/browse/HBASE-17118
> Project: HBase
> Issue Type: Bug
> Reporter: binlijin
> Attachments: StoreScanner.png, StoreScannerLeakHeap.png
>
>
> KeyValueHeap#generalizedSeek
> KeyValueScanner scanner = current;
> while (scanner != null) {
> Cell topKey = scanner.peek();
> ......
> boolean seekResult;
> if (isLazy && heap.size() > 0) {
> // If there is only one scanner left, we don't do lazy seek.
> seekResult = scanner.requestSeek(seekKey, forward, useBloom);
> } else {
> seekResult = NonLazyKeyValueScanner.doRealSeek(scanner, seekKey,
> forward);
> }
> ......
> scanner = heap.poll();
> }
> (1) scanner = heap.poll(); Retrieves and removes the head of this queue
> (2) scanner.requestSeek(seekKey, forward, useBloom); or
> NonLazyKeyValueScanner.doRealSeek(scanner, seekKey, forward);
> throw exception, and scanner will have no change to close, so will cause the
> scanner leak.
--
This message was sent by Atlassian JIRA
(v6.3.4#6332)
