longping_jie created HBASE-27375:
------------------------------------
Summary: Suspected memory leak caused by major compaction
Key: HBASE-27375
URL: https://issues.apache.org/jira/browse/HBASE-27375
Project: HBase
Issue Type: Bug
Affects Versions: 2.2.6
Environment: hbase 2.2.6
Reporter: longping_jie
Attachments: image-2022-09-18-12-46-08-478.png
There are 49G Hfiles in a certain region, and major compact has been
unsuccessful, and the Old area in the heap memory continues to fill up, causing
frequent Full GCs and causing the RegionServer process to OOM. After analyzing
the dump file of heap memory using MAT, we found that:
!image-2022-09-18-12-46-08-478.png!
The full stack involved is as follows:
regionserver/node71:60020-longCompactions-3
at
org.apache.hadoop.util.NativeCrc32.nativeComputeChunkedSums(IILjava/nio/ByteBuffer;ILjava/nio/ByteBuffer;IILjava/lang/String;JZ)V
(Native Method)
at
org.apache.hadoop.util.NativeCrc32.verifyChunkedSums(IILjava/nio/ByteBuffer;Ljava/nio/ByteBuffer;Ljava/lang/String;J)V
(NativeCrc32.java:63)
at
org.apache.hadoop.util.DataChecksum.verifyChunkedSums(Ljava/nio/ByteBuffer;Ljava/nio/ByteBuffer;Ljava/lang/String;J)V
(DataChecksum.java:380)
at
org.apache.hadoop.hbase.io.hfile.ChecksumUtil.verifyChunkedSums(Lorg/apache/hadoop/util/DataChecksum;Lorg/apache/hadoop/hbase/nio/ByteBuff;Lorg/apache/hadoop/hbase/nio/ByteBuff;Ljava/lang/String;)Z
(ChecksumUtil.java:104)
at
org.apache.hadoop.hbase.io.hfile.ChecksumUtil.validateChecksum(Lorg/apache/hadoop/hbase/nio/ByteBuff;Ljava/lang/String;JI)Z
(ChecksumUtil.java:191)
at
org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.validateChecksum(JLorg/apache/hadoop/hbase/nio/ByteBuff;I)Z
(HFileBlock.java:1842)
at
org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockDataInternal(Lorg/apache/hadoop/fs/FSDataInputStream;JJZZZZ)Lorg/apache/hadoop/hbase/io/hfile/HFileBlock;
(HFileBlock.java:1777)
at
org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockData(JJZZZ)Lorg/apache/hadoop/hbase/io/hfile/HFileBlock;
(HFileBlock.java:1570)
at
org.apache.hadoop.hbase.io.hfile.HFileReaderImpl.readBlock(JJZZZZLorg/apache/hadoop/hbase/io/hfile/BlockType;Lorg/apache/hadoop/hbase/io/encoding/DataBlockEncoding;)Lorg/apache/hadoop/hbase/io/hfile/HFileBlock;
(HFileReaderImpl.java:1519)
at
org.apache.hadoop.hbase.io.hfile.HFileReaderImpl$HFileScannerImpl.readNextDataBlock()Lorg/apache/hadoop/hbase/io/hfile/HFileBlock;
(HFileReaderImpl.java:928)
at
org.apache.hadoop.hbase.io.hfile.HFileReaderImpl$HFileScannerImpl.isNextBlock()Z
(HFileReaderImpl.java:1061)
at
org.apache.hadoop.hbase.io.hfile.HFileReaderImpl$HFileScannerImpl.positionForNextBlock()Z
(HFileReaderImpl.java:1055)
at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl$HFileScannerImpl._next()Z
(HFileReaderImpl.java:1073)
at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl$HFileScannerImpl.next()Z
(HFileReaderImpl.java:1094)
at org.apache.hadoop.hbase.io.HalfStoreFileReader$1.next()Z
(HalfStoreFileReader.java:169)
at
org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseekAtOrAfter(Lorg/apache/hadoop/hbase/io/hfile/HFileScanner;Lorg/apache/hadoop/hbase/Cell;)Z
(StoreFileScanner.java:351)
at
org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseek(Lorg/apache/hadoop/hbase/Cell;)Z
(StoreFileScanner.java:244)
at
org.apache.hadoop.hbase.regionserver.NonLazyKeyValueScanner.doRealSeek(Lorg/apache/hadoop/hbase/regionserver/KeyValueScanner;Lorg/apache/hadoop/hbase/Cell;Z)Z
(NonLazyKeyValueScanner.java:55)
at
org.apache.hadoop.hbase.regionserver.KeyValueHeap.generalizedSeek(ZLorg/apache/hadoop/hbase/Cell;ZZ)Z
(KeyValueHeap.java:324)
at
org.apache.hadoop.hbase.regionserver.KeyValueHeap.reseek(Lorg/apache/hadoop/hbase/Cell;)Z
(KeyValueHeap.java:267)
at
org.apache.hadoop.hbase.regionserver.StoreScanner.reseek(Lorg/apache/hadoop/hbase/Cell;)Z
(StoreScanner.java:1099)
at
org.apache.hadoop.hbase.regionserver.StoreScanner.seekAsDirection(Lorg/apache/hadoop/hbase/Cell;)Z
(StoreScanner.java:1088)
at
org.apache.hadoop.hbase.regionserver.StoreScanner.seekOrSkipToNextColumn(Lorg/apache/hadoop/hbase/Cell;)V
(StoreScanner.java:823)
at
org.apache.hadoop.hbase.regionserver.StoreScanner.next(Ljava/util/List;Lorg/apache/hadoop/hbase/regionserver/ScannerContext;)Z
(StoreScanner.java:730)
at
org.apache.hadoop.hbase.regionserver.compactions.Compactor.performCompaction(Lorg/apache/hadoop/hbase/regionserver/compactions/Compactor$FileDetails;Lorg/apache/hadoop/hbase/regionserver/InternalScanner;Lorg/apache/hadoop/hbase/regionserver/CellSink;JZLorg/apache/hadoop/hbase/regionserver/throttle/ThroughputController;ZI)Z
(Compactor.java:387)
at
org.apache.hadoop.hbase.regionserver.compactions.Compactor.compact(Lorg/apache/hadoop/hbase/regionserver/compactions/CompactionRequestImpl;Lorg/apache/hadoop/hbase/regionserver/compactions/Compactor$InternalScannerFactory;Lorg/apache/hadoop/hbase/regionserver/compactions/Compactor$CellSinkFactory;Lorg/apache/hadoop/hbase/regionserver/throttle/ThroughputController;Lorg/apache/hadoop/hbase/security/User;)Ljava/util/List;
(Compactor.java:326)
at
org.apache.hadoop.hbase.regionserver.compactions.DefaultCompactor.compact(Lorg/apache/hadoop/hbase/regionserver/compactions/CompactionRequestImpl;Lorg/apache/hadoop/hbase/regionserver/throttle/ThroughputController;Lorg/apache/hadoop/hbase/security/User;)Ljava/util/List;
(DefaultCompactor.java:65)
at
org.apache.hadoop.hbase.regionserver.DefaultStoreEngine$DefaultCompactionContext.compact(Lorg/apache/hadoop/hbase/regionserver/throttle/ThroughputController;Lorg/apache/hadoop/hbase/security/User;)Ljava/util/List;
(DefaultStoreEngine.java:126)
at
org.apache.hadoop.hbase.regionserver.HStore.compact(Lorg/apache/hadoop/hbase/regionserver/compactions/CompactionContext;Lorg/apache/hadoop/hbase/regionserver/throttle/ThroughputController;Lorg/apache/hadoop/hbase/security/User;)Ljava/util/List;
(HStore.java:1469)
at
org.apache.hadoop.hbase.regionserver.HRegion.compact(Lorg/apache/hadoop/hbase/regionserver/compactions/CompactionContext;Lorg/apache/hadoop/hbase/regionserver/HStore;Lorg/apache/hadoop/hbase/regionserver/throttle/ThroughputController;Lorg/apache/hadoop/hbase/security/User;)Z
(HRegion.java:2264)
at
org.apache.hadoop.hbase.regionserver.CompactSplit$CompactionRunner.doCompaction(Lorg/apache/hadoop/hbase/security/User;)V
(CompactSplit.java:624)
at org.apache.hadoop.hbase.regionserver.CompactSplit$CompactionRunner.run()V
(CompactSplit.java:666)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(Ljava/util/concurrent/ThreadPoolExecutor$Worker;)V
(ThreadPoolExecutor.java:1142)
at java.util.concurrent.ThreadPoolExecutor$Worker.run()V
(ThreadPoolExecutor.java:617)
at java.lang.Thread.run()V (Thread.java:748)
--
This message was sent by Atlassian Jira
(v8.20.10#820010)
