All Regions or just the one? Do thread dumps lock thread reading against hdfs every time you take one?
Is it always inside in updateReaders? Is there a bad file or lots of files to add to the list? Yours, S On Thu, Jan 2, 2020 at 8:34 PM Kang Minwoo <[email protected]> wrote: > Hello Users, > > I met an issue that is flush times is too long. > > MemStoreFlusher is waiting for a lock. > ``` > "MemStoreFlusher.0" > java.lang.Thread.State: WAITING (parking) > at sun.misc.Unsafe.park(Native Method) > - parking to wait for <0x00007f0412bddcb8> (a > java.util.concurrent.locks.ReentrantLock$NonfairSync) > at > java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) > at > java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836) > at > java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireQueued(AbstractQueuedSynchronizer.java:870) > at > java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:1199) > at > java.util.concurrent.locks.ReentrantLock$NonfairSync.lock(ReentrantLock.java:209) > at > java.util.concurrent.locks.ReentrantLock.lock(ReentrantLock.java:285) > at > org.apache.hadoop.hbase.regionserver.StoreScanner.updateReaders(StoreScanner.java:692) > at > org.apache.hadoop.hbase.regionserver.HStore.notifyChangedReadersObservers(HStore.java:1100) > at > org.apache.hadoop.hbase.regionserver.HStore.updateStorefiles(HStore.java:1079) > at > org.apache.hadoop.hbase.regionserver.HStore.access$700(HStore.java:118) > at > org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.commit(HStore.java:2321) > at > org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:2430) > at > org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2153) > at > org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2115) > at > org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2005) > at > org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:1930) > at > org.apache.hadoop.hbase.regionserver.MemStoreFlusher.flushRegion(MemStoreFlusher.java:514) > at > org.apache.hadoop.hbase.regionserver.MemStoreFlusher.flushRegion(MemStoreFlusher.java:475) > at > org.apache.hadoop.hbase.regionserver.MemStoreFlusher.access$900(MemStoreFlusher.java:75) > at > org.apache.hadoop.hbase.regionserver.MemStoreFlusher$FlushHandler.run(MemStoreFlusher.java:263) > at java.lang.Thread.run(Thread.java:748) > Locked ownable synchronizers: > - None > ``` > > > RPC Handler had the lock. > ``` > "B.defaultRpcServer.handler" > java.lang.Thread.State: RUNNABLE > at org.apache.log4j.Category.getEffectiveLevel(Category.java:442) > at org.apache.log4j.Category.isEnabledFor(Category.java:751) > at > org.apache.commons.logging.impl.Log4JLogger.isTraceEnabled(Log4JLogger.java:303) > at > org.apache.hadoop.hdfs.BlockReaderLocal.skip(BlockReaderLocal.java:622) > - locked <0x00007f0271bcc1c0> (a > org.apache.hadoop.hdfs.BlockReaderLocal) > at > org.apache.hadoop.hdfs.DFSInputStream.seek(DFSInputStream.java:1534) > - locked <0x00007f038265f980> (a > org.apache.hadoop.hdfs.DFSInputStream) > at > org.apache.hadoop.fs.FSDataInputStream.seek(FSDataInputStream.java:62) > at org.apache.hadoop.hbase.io > .hfile.HFileUtil.seekOnMultipleSources(HFileUtil.java:38) > at org.apache.hadoop.hbase.io > .hfile.HFileBlock$AbstractFSReader.readAtOffset(HFileBlock.java:1431) > at org.apache.hadoop.hbase.io > .hfile.HFileBlock$FSReaderImpl.readBlockDataInternal(HFileBlock.java:1654) > at org.apache.hadoop.hbase.io > .hfile.HFileBlock$FSReaderImpl.readBlockData(HFileBlock.java:1538) > at org.apache.hadoop.hbase.io > .hfile.HFileReaderV2.readBlock(HFileReaderV2.java:452) > at org.apache.hadoop.hbase.io > .hfile.HFileReaderV2$AbstractScannerV2.readNextDataBlock(HFileReaderV2.java:729) > at org.apache.hadoop.hbase.io > .hfile.HFileReaderV2$ScannerV2.isNextBlock(HFileReaderV2.java:854) > at org.apache.hadoop.hbase.io > .hfile.HFileReaderV2$ScannerV2.positionForNextBlock(HFileReaderV2.java:849) > at org.apache.hadoop.hbase.io > .hfile.HFileReaderV2$ScannerV2._next(HFileReaderV2.java:866) > at org.apache.hadoop.hbase.io > .hfile.HFileReaderV2$ScannerV2.next(HFileReaderV2.java:886) > at > org.apache.hadoop.hbase.regionserver.StoreFileScanner.skipKVsNewerThanReadpoint(StoreFileScanner.java:247) > at > org.apache.hadoop.hbase.regionserver.StoreFileScanner.seek(StoreFileScanner.java:194) > at > org.apache.hadoop.hbase.regionserver.StoreScanner.seekScanners(StoreScanner.java:350) > at > org.apache.hadoop.hbase.regionserver.StoreScanner.resetScannerStack(StoreScanner.java:750) > at > org.apache.hadoop.hbase.regionserver.StoreScanner.checkReseek(StoreScanner.java:725) > at > org.apache.hadoop.hbase.regionserver.StoreScanner.next(StoreScanner.java:475) > at > org.apache.hadoop.hbase.regionserver.KeyValueHeap.next(KeyValueHeap.java:150) > at > org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.populateResult(HRegion.java:5824) > at > org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextInternal(HRegion.java:5994) > at > org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:5761) > at > org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:2622) > - locked <0x00007f035a4df0f8> (a > org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl) > at > org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:33770) > at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2216) > at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:112) > at > org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:133) > at > org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:108) > at java.lang.Thread.run(Thread.java:748) > Locked ownable synchronizers: > - <0x00007f02bfaf7a90> (a > java.util.concurrent.locks.ReentrantLock$NonfairSync) > - <0x00007f0412bddcb8> (a > java.util.concurrent.locks.ReentrantLock$NonfairSync) > ``` > > ``` > B.defaultRpcServer.handler > java.lang.Thread.State: RUNNABLE > at java.io.FilterInputStream.read(FilterInputStream.java:83) > at > org.apache.hadoop.io.compress.BlockDecompressorStream.rawReadInt(BlockDecompressorStream.java:145) > at > org.apache.hadoop.io.compress.BlockDecompressorStream.decompress(BlockDecompressorStream.java:74) > at > org.apache.hadoop.io.compress.DecompressorStream.read(DecompressorStream.java:85) > at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) > at java.io.BufferedInputStream.read(BufferedInputStream.java:345) > - locked <0x00007f024acf0bf8> (a java.io.BufferedInputStream) > at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:200) > at org.apache.hadoop.hbase.io > .compress.Compression.decompress(Compression.java:453) > at org.apache.hadoop.hbase.io > .encoding.HFileBlockDefaultDecodingContext.prepareDecoding(HFileBlockDefaultDecodingContext.java:90) > at org.apache.hadoop.hbase.io > .hfile.HFileBlock.unpack(HFileBlock.java:549) > at org.apache.hadoop.hbase.io > .hfile.HFileReaderV2.readBlock(HFileReaderV2.java:455) > at org.apache.hadoop.hbase.io > .hfile.HFileReaderV2$AbstractScannerV2.readNextDataBlock(HFileReaderV2.java:729) > at org.apache.hadoop.hbase.io > .hfile.HFileReaderV2$ScannerV2.isNextBlock(HFileReaderV2.java:854) > at org.apache.hadoop.hbase.io > .hfile.HFileReaderV2$ScannerV2.positionForNextBlock(HFileReaderV2.java:849) > at org.apache.hadoop.hbase.io > .hfile.HFileReaderV2$ScannerV2._next(HFileReaderV2.java:866) > at org.apache.hadoop.hbase.io > .hfile.HFileReaderV2$ScannerV2.next(HFileReaderV2.java:886) > at > org.apache.hadoop.hbase.regionserver.StoreFileScanner.skipKVsNewerThanReadpoint(StoreFileScanner.java:247) > at > org.apache.hadoop.hbase.regionserver.StoreFileScanner.seek(StoreFileScanner.java:194) > at > org.apache.hadoop.hbase.regionserver.StoreScanner.seekScanners(StoreScanner.java:350) > at > org.apache.hadoop.hbase.regionserver.StoreScanner.resetScannerStack(StoreScanner.java:750) > at > org.apache.hadoop.hbase.regionserver.StoreScanner.checkReseek(StoreScanner.java:725) > at > org.apache.hadoop.hbase.regionserver.StoreScanner.next(StoreScanner.java:475) > at > org.apache.hadoop.hbase.regionserver.KeyValueHeap.next(KeyValueHeap.java:150) > at > org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.populateResult(HRegion.java:5824) > at > org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextInternal(HRegion.java:5994) > at > org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:5761) > at > org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:2622) > - locked <0x00007f035a4df0f8> (a > org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl) > at > org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:33770) > at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2216) > at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:112) > at > org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:133) > at > org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:108) > at java.lang.Thread.run(Thread.java:748) > Locked ownable synchronizers: > - <0x00007f0412bddcb8> (a > java.util.concurrent.locks.ReentrantLock$NonfairSync) > ``` > > Versions: > - HBase: 1.2.9 > - HDFS: 2.7.7 > > Best regards, > Minwoo Kang >
