kangTwang created HBASE-25661:
---------------------------------
Summary: Unable rename error occurred in AbstractFSWAL, causing
regionserver to crash
Key: HBASE-25661
URL: https://issues.apache.org/jira/browse/HBASE-25661
Project: HBase
Issue Type: Bug
Components: API
Affects Versions: 2.1.0
Reporter: kangTwang
Fix For: 2.1.0
The error is as follows：
[ERROR] -
org.apache.hadoop.hbase.regionserver.MemStoreFlusher.flushRegion(MemStoreFlusher.java:631)
- Cache flush failed for region
test_2,00000000000000000293601280,1614762174258.030beae347d51a5fb6782f6cb025f763.
[ERROR] -
org.apache.hadoop.hbase.regionserver.MemStoreFlusher.flushRegion(MemStoreFlusher.java:631)
- Cache flush failed for region
test_2,00000000000000000293601280,1614762174258.030beae347d51a5fb6782f6cb025f763.java.io.IOException:
WAL has been closed at
org.apache.hadoop.hbase.regionserver.wal.AsyncFSWAL.doShutdown(AsyncFSWAL.java:698)
~[hbase-server-2.1.0-cdh6.3.0.jar:?] at
org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.shutdown(AbstractFSWAL.java:817)
~[hbase-server-2.1.0-cdh6.3.0.jar:?] at
org.apache.hadoop.hbase.regionserver.LogRoller.abort(LogRoller.java:143)
~[hbase-server-2.1.0-cdh6.3.0.jar:?] at
org.apache.hadoop.hbase.regionserver.LogRoller.run(LogRoller.java:201)
~[hbase-server-2.1.0-cdh6.3.0.jar:?] at java.lang.Thread.run(Thread.java:834)
~[?:?][17:11:59:664] [INFO] -
org.apache.hadoop.hbase.regionserver.HRegion.logFatLineOnFlush(HRegion.java:2636)
- Flushing 1/1 column families, dataSize=127.76 MB heapSize=136.99
MB[17:11:59:665] [WARN] -
org.apache.hadoop.hbase.regionserver.HRegion.doAbortFlushToWAL(HRegion.java:2652)
- Received unexpected exception trying to write ABORT_FLUSH marker to
WAL:java.io.IOException: Cannot append; log is closed, regionName =
test_2,00000000000000000377487360,1614762174258.146bbdf3caa203124cd039e48dd3e344.
at
org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.stampSequenceIdAndPublishToRingBuffer(AbstractFSWAL.java:962)
at
org.apache.hadoop.hbase.regionserver.wal.AsyncFSWAL.append(AsyncFSWAL.java:563)
at
org.apache.hadoop.hbase.regionserver.wal.WALUtil.doFullAppendTransaction(WALUtil.java:156)
at
org.apache.hadoop.hbase.regionserver.wal.WALUtil.writeFlushMarker(WALUtil.java:85)
at
org.apache.hadoop.hbase.regionserver.HRegion.doAbortFlushToWAL(HRegion.java:2649)
at
org.apache.hadoop.hbase.regionserver.HRegion.internalPrepareFlushCache(HRegion.java:2599)
at
org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2465)
at
org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2439)
at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2329)
at
org.apache.hadoop.hbase.regionserver.MemStoreFlusher.flushRegion(MemStoreFlusher.java:612)
at
org.apache.hadoop.hbase.regionserver.MemStoreFlusher.flushRegion(MemStoreFlusher.java:581)
at
org.apache.hadoop.hbase.regionserver.MemStoreFlusher.access$1000(MemStoreFlusher.java:68)
at
org.apache.hadoop.hbase.regionserver.MemStoreFlusher$FlushHandler.run(MemStoreFlusher.java:361)
at java.base/java.lang.Thread.run(Thread.java:834)
[17:11:59:665] [ERROR] -
org.apache.hadoop.hbase.regionserver.MemStoreFlusher.flushRegion(MemStoreFlusher.java:631)
- Cache flush failed for region
test_2,00000000000000000377487360,1614762174258.146bbdf3caa203124cd039e48dd3e344.java.io.IOException:
Cannot append; log is closed, regionName =
test_2,00000000000000000377487360,1614762174258.146bbdf3caa203124cd039e48dd3e344.
at
org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.stampSequenceIdAndPublishToRingBuffer(AbstractFSWAL.java:962)
~[hbase-server-2.1.0-cdh6.3.0.jar:?] at
org.apache.hadoop.hbase.regionserver.wal.AsyncFSWAL.append(AsyncFSWAL.java:563)
~[hbase-server-2.1.0-cdh6.3.0.jar:?] at
org.apache.hadoop.hbase.regionserver.wal.WALUtil.doFullAppendTransaction(WALUtil.java:156)
~[hbase-server-2.1.0-cdh6.3.0.jar:?] at
org.apache.hadoop.hbase.regionserver.wal.WALUtil.writeFlushMarker(WALUtil.java:85)
~[hbase-server-2.1.0-cdh6.3.0.jar:?] at
org.apache.hadoop.hbase.regionserver.HRegion.internalPrepareFlushCache(HRegion.java:2588)
~[hbase-server-2.1.0-cdh6.3.0.jar:?] at
org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2465)
~[hbase-server-2.1.0-cdh6.3.0.jar:?] at
org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2439)
~[hbase-server-2.1.0-cdh6.3.0.jar:?] at
org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2329)
~[hbase-server-2.1.0-cdh6.3.0.jar:?] at
org.apache.hadoop.hbase.regionserver.MemStoreFlusher.flushRegion(MemStoreFlusher.java:612)
~[hbase-server-2.1.0-cdh6.3.0.jar:?] at
org.apache.hadoop.hbase.regionserver.MemStoreFlusher.flushRegion(MemStoreFlusher.java:581)
~[hbase-server-2.1.0-cdh6.3.0.jar:?] at
org.apache.hadoop.hbase.regionserver.MemStoreFlusher.access$1000(MemStoreFlusher.java:68)
~[hbase-server-2.1.0-cdh6.3.0.jar:?] at
org.apache.hadoop.hbase.regionserver.MemStoreFlusher$FlushHandler.run(MemStoreFlusher.java:361)
~[hbase-server-2.1.0-cdh6.3.0.jar:?] at java.lang.Thread.run(Thread.java:834)
~[?:?][17:11:59:665] [INFO] -
org.apache.hadoop.hbase.regionserver.HRegion.logFatLineOnFlush(HRegion.java:2636)
- Flushing 1/1 column families, dataSize=126.24 MB heapSize=135.36
MB[17:11:59:665] [WARN] -
org.apache.hadoop.hbase.regionserver.HRegion.doAbortFlushToWAL(HRegion.java:2652)
- Received unexpected exception trying to write ABORT_FLUSH marker to
WAL:java.io.IOException: Cannot append; log is closed, regionName =
test_2,00000000000000000587202560,1614762174258.c958379b0ac4a008cb4d75223683377a.
at
org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.stampSequenceIdAndPublishToRingBuffer(AbstractFSWAL.java:962)
at
org.apache.hadoop.hbase.regionserver.wal.AsyncFSWAL.append(AsyncFSWAL.java:563)
at
org.apache.hadoop.hbase.regionserver.wal.WALUtil.doFullAppendTransaction(WALUtil.java:156)
at
org.apache.hadoop.hbase.regionserver.wal.WALUtil.writeFlushMarker(WALUtil.java:85)
at
org.apache.hadoop.hbase.regionserver.HRegion.doAbortFlushToWAL(HRegion.java:2649)
at
org.apache.hadoop.hbase.regionserver.HRegion.internalPrepareFlushCache(HRegion.java:2599)
at
org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2465)
at
org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2439)
at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2329)
at
org.apache.hadoop.hbase.regionserver.MemStoreFlusher.flushRegion(MemStoreFlusher.java:612)
at
org.apache.hadoop.hbase.regionserver.MemStoreFlusher.flushRegion(MemStoreFlusher.java:581)
at
org.apache.hadoop.hbase.regionserver.MemStoreFlusher.access$1000(MemStoreFlusher.java:68)
at
org.apache.hadoop.hbase.regionserver.MemStoreFlusher$FlushHandler.run(MemStoreFlusher.java:361)
at java.base/java.lang.Thread.run(Thread.java:834)
[17:11:59:665] [ERROR] -
org.apache.hadoop.hbase.regionserver.MemStoreFlusher.flushRegion(MemStoreFlusher.java:631)
- Cache flush failed for region
test_2,00000000000000000587202560,1614762174258.c958379b0ac4a008cb4d75223683377a.java.io.IOException:
Cannot append; log is closed, regionName =
test_2,00000000000000000587202560,1614762174258.c958379b0ac4a008cb4d75223683377a.
at
org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.stampSequenceIdAndPublishToRingBuffer(AbstractFSWAL.java:962)
~[hbase-server-2.1.0-cdh6.3.0.jar:?] at
org.apache.hadoop.hbase.regionserver.wal.AsyncFSWAL.append(AsyncFSWAL.java:563)
~[hbase-server-2.1.0-cdh6.3.0.jar:?] at
org.apache.hadoop.hbase.regionserver.wal.WALUtil.doFullAppendTransaction(WALUtil.java:156)
~[hbase-server-2.1.0-cdh6.3.0.jar:?] at
org.apache.hadoop.hbase.regionserver.wal.WALUtil.writeFlushMarker(WALUtil.java:85)
~[hbase-server-2.1.0-cdh6.3.0.jar:?] at
org.apache.hadoop.hbase.regionserver.HRegion.internalPrepareFlushCache(HRegion.java:2588)
~[hbase-server-2.1.0-cdh6.3.0.jar:?] at
org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2465)
~[hbase-server-2.1.0-cdh6.3.0.jar:?] at
org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2439)
~[hbase-server-2.1.0-cdh6.3.0.jar:?] at
org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2329)
~[hbase-server-2.1.0-cdh6.3.0.jar:?] at
org.apache.hadoop.hbase.regionserver.MemStoreFlusher.flushRegion(MemStoreFlusher.java:612)
~[hbase-server-2.1.0-cdh6.3.0.jar:?] at
org.apache.hadoop.hbase.regionserver.MemStoreFlusher.flushRegion(MemStoreFlusher.java:581)
~[hbase-server-2.1.0-cdh6.3.0.jar:?] at
org.apache.hadoop.hbase.regionserver.MemStoreFlusher.access$1000(MemStoreFlusher.java:68)
~[hbase-server-2.1.0-cdh6.3.0.jar:?] at
org.apache.hadoop.hbase.regionserver.MemStoreFlusher$FlushHandler.run(MemStoreFlusher.java:361)
~[hbase-server-2.1.0-cdh6.3.0.jar:?] at java.lang.Thread.run(Thread.java:834)
~[?:?][17:11:59:666] [WARN] -
org.apache.hadoop.hbase.regionserver.HRegion.doAbortFlushToWAL(HRegion.java:2652)
- Received unexpected exception trying to write ABORT_FLUSH marker to
WAL:java.io.IOException: Cannot append; log is closed, regionName =
test_2,00000000000000000125829120,1614762174258.b14f5ec6e23e1390b2daf0f83be84ee0.
at
org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.stampSequenceIdAndPublishToRingBuffer(AbstractFSWAL.java:962)
at
org.apache.hadoop.hbase.regionserver.wal.AsyncFSWAL.append(AsyncFSWAL.java:563)
at
org.apache.hadoop.hbase.regionserver.wal.WALUtil.doFullAppendTransaction(WALUtil.java:156)
at
org.apache.hadoop.hbase.regionserver.wal.WALUtil.writeFlushMarker(WALUtil.java:85)
at
org.apache.hadoop.hbase.regionserver.HRegion.doAbortFlushToWAL(HRegion.java:2649)
at
org.apache.hadoop.hbase.regionserver.HRegion.internalPrepareFlushCache(HRegion.java:2599)
at
org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2465)
at
org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2439)
at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2329)
at
org.apache.hadoop.hbase.regionserver.MemStoreFlusher.flushRegion(MemStoreFlusher.java:612)
at
org.apache.hadoop.hbase.regionserver.MemStoreFlusher.flushRegion(MemStoreFlusher.java:581)
at
org.apache.hadoop.hbase.regionserver.MemStoreFlusher.access$1000(MemStoreFlusher.java:68)
at
org.apache.hadoop.hbase.regionserver.MemStoreFlusher$FlushHandler.run(MemStoreFlusher.java:361)
at java.base/java.lang.Thread.run(Thread.java:834)
[17:11:59:666] [ERROR] -
org.apache.hadoop.hbase.regionserver.MemStoreFlusher.flushRegion(MemStoreFlusher.java:631)
- Cache flush failed for region
test_2,00000000000000000125829120,1614762174258.b14f5ec6e23e1390b2daf0f83be84ee0.java.io.IOException:
Cannot append; log is closed, regionName =
test_2,00000000000000000125829120,1614762174258.b14f5ec6e23e1390b2daf0f83be84ee0.
at
org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.stampSequenceIdAndPublishToRingBuffer(AbstractFSWAL.java:962)
~[hbase-server-2.1.0-cdh6.3.0.jar:?] at
org.apache.hadoop.hbase.regionserver.wal.AsyncFSWAL.append(AsyncFSWAL.java:563)
~[hbase-server-2.1.0-cdh6.3.0.jar:?] at
org.apache.hadoop.hbase.regionserver.wal.WALUtil.doFullAppendTransaction(WALUtil.java:156)
~[hbase-server-2.1.0-cdh6.3.0.jar:?] at
org.apache.hadoop.hbase.regionserver.wal.WALUtil.writeFlushMarker(WALUtil.java:85)
~[hbase-server-2.1.0-cdh6.3.0.jar:?] at
org.apache.hadoop.hbase.regionserver.HRegion.internalPrepareFlushCache(HRegion.java:2588)
~[hbase-server-2.1.0-cdh6.3.0.jar:?] at
org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2465)
~[hbase-server-2.1.0-cdh6.3.0.jar:?] at
org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2439)
~[hbase-server-2.1.0-cdh6.3.0.jar:?] at
org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2329)
~[hbase-server-2.1.0-cdh6.3.0.jar:?] at
org.apache.hadoop.hbase.regionserver.MemStoreFlusher.flushRegion(MemStoreFlusher.java:612)
~[hbase-server-2.1.0-cdh6.3.0.jar:?] at
org.apache.hadoop.hbase.regionserver.MemStoreFlusher.flushRegion(MemStoreFlusher.java:581)
~[hbase-server-2.1.0-cdh6.3.0.jar:?] at
org.apache.hadoop.hbase.regionserver.MemStoreFlusher.access$1000(MemStoreFlusher.java:68)
~[hbase-server-2.1.0-cdh6.3.0.jar:?] at
org.apache.hadoop.hbase.regionserver.MemStoreFlusher$FlushHandler.run(MemStoreFlusher.java:361)
~[hbase-server-2.1.0-cdh6.3.0.jar:?] at java.lang.Thread.run(Thread.java:834)
~[?:?][17:11:59:813] [ERROR] -
org.apache.hadoop.hbase.regionserver.HRegionServer.abort(HRegionServer.java:2376)
- ***** ABORTING region server node017,16020,1614750431707: IOE in log roller
*****java.io.IOException: Unable to rename
hdfs://buc-001/hbase/WALs/node017,16020,1614750431707/node017%2C16020%2C1614750431707.1614762525711
to hdfs://buc-001/hbase/oldWALs/node017%2C16020%2C1614750431707.1614762525711
at
org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.archiveLogFile(AbstractFSWAL.java:651)
~[hbase-server-2.1.0-cdh6.3.0.jar:?] at
org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.cleanOldLogs(AbstractFSWAL.java:627)
~[hbase-server-2.1.0-cdh6.3.0.jar:?] at
org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.rollWriter(AbstractFSWAL.java:767)
~[hbase-server-2.1.0-cdh6.3.0.jar:?] at
org.apache.hadoop.hbase.regionserver.LogRoller.run(LogRoller.java:190)
~[hbase-server-2.1.0-cdh6.3.0.jar:?] at java.lang.Thread.run(Thread.java:834)
~[?:?]
Has anyone encountered a similar problem？？？
--
This message was sent by Atlassian Jira
(v8.3.4#803005)
