[
https://issues.apache.org/jira/browse/HBASE-25661?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=17300186#comment-17300186
]
Duo Zhang commented on HBASE-25661:
-----------------------------------
It seems that the region server is already down?
> Unable rename error occurred in AbstractFSWAL, causing regionserver to crash
> ----------------------------------------------------------------------------
>
> Key: HBASE-25661
> URL: https://issues.apache.org/jira/browse/HBASE-25661
> Project: HBase
> Issue Type: Bug
> Components: API
> Affects Versions: 2.1.0
> Reporter: kangTwang
> Priority: Major
> Fix For: 2.1.0
>
>
> The error is as follows：
> [ERROR] -
> org.apache.hadoop.hbase.regionserver.MemStoreFlusher.flushRegion(MemStoreFlusher.java:631)
> - Cache flush failed for region
> test_2,00000000000000000293601280,1614762174258.030beae347d51a5fb6782f6cb025f763.
> [ERROR] -
> org.apache.hadoop.hbase.regionserver.MemStoreFlusher.flushRegion(MemStoreFlusher.java:631)
> - Cache flush failed for region
> test_2,00000000000000000293601280,1614762174258.030beae347d51a5fb6782f6cb025f763.java.io.IOException:
> WAL has been closed at
> org.apache.hadoop.hbase.regionserver.wal.AsyncFSWAL.doShutdown(AsyncFSWAL.java:698)
> ~[hbase-server-2.1.0-cdh6.3.0.jar:?] at
> org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.shutdown(AbstractFSWAL.java:817)
> ~[hbase-server-2.1.0-cdh6.3.0.jar:?] at
> org.apache.hadoop.hbase.regionserver.LogRoller.abort(LogRoller.java:143)
> ~[hbase-server-2.1.0-cdh6.3.0.jar:?] at
> org.apache.hadoop.hbase.regionserver.LogRoller.run(LogRoller.java:201)
> ~[hbase-server-2.1.0-cdh6.3.0.jar:?] at java.lang.Thread.run(Thread.java:834)
> ~[?:?][17:11:59:664] [INFO] -
> org.apache.hadoop.hbase.regionserver.HRegion.logFatLineOnFlush(HRegion.java:2636)
> - Flushing 1/1 column families, dataSize=127.76 MB heapSize=136.99
> MB[17:11:59:665] [WARN] -
> org.apache.hadoop.hbase.regionserver.HRegion.doAbortFlushToWAL(HRegion.java:2652)
> - Received unexpected exception trying to write ABORT_FLUSH marker to
> WAL:java.io.IOException: Cannot append; log is closed, regionName =
> test_2,00000000000000000377487360,1614762174258.146bbdf3caa203124cd039e48dd3e344.
> at
> org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.stampSequenceIdAndPublishToRingBuffer(AbstractFSWAL.java:962)
> at
> org.apache.hadoop.hbase.regionserver.wal.AsyncFSWAL.append(AsyncFSWAL.java:563)
> at
> org.apache.hadoop.hbase.regionserver.wal.WALUtil.doFullAppendTransaction(WALUtil.java:156)
> at
> org.apache.hadoop.hbase.regionserver.wal.WALUtil.writeFlushMarker(WALUtil.java:85)
> at
> org.apache.hadoop.hbase.regionserver.HRegion.doAbortFlushToWAL(HRegion.java:2649)
> at
> org.apache.hadoop.hbase.regionserver.HRegion.internalPrepareFlushCache(HRegion.java:2599)
> at
> org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2465)
> at
> org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2439)
> at
> org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2329) at
> org.apache.hadoop.hbase.regionserver.MemStoreFlusher.flushRegion(MemStoreFlusher.java:612)
> at
> org.apache.hadoop.hbase.regionserver.MemStoreFlusher.flushRegion(MemStoreFlusher.java:581)
> at
> org.apache.hadoop.hbase.regionserver.MemStoreFlusher.access$1000(MemStoreFlusher.java:68)
> at
> org.apache.hadoop.hbase.regionserver.MemStoreFlusher$FlushHandler.run(MemStoreFlusher.java:361)
> at java.base/java.lang.Thread.run(Thread.java:834)
> [17:11:59:665] [ERROR] -
> org.apache.hadoop.hbase.regionserver.MemStoreFlusher.flushRegion(MemStoreFlusher.java:631)
> - Cache flush failed for region
> test_2,00000000000000000377487360,1614762174258.146bbdf3caa203124cd039e48dd3e344.java.io.IOException:
> Cannot append; log is closed, regionName =
> test_2,00000000000000000377487360,1614762174258.146bbdf3caa203124cd039e48dd3e344.
> at
> org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.stampSequenceIdAndPublishToRingBuffer(AbstractFSWAL.java:962)
> ~[hbase-server-2.1.0-cdh6.3.0.jar:?] at
> org.apache.hadoop.hbase.regionserver.wal.AsyncFSWAL.append(AsyncFSWAL.java:563)
> ~[hbase-server-2.1.0-cdh6.3.0.jar:?] at
> org.apache.hadoop.hbase.regionserver.wal.WALUtil.doFullAppendTransaction(WALUtil.java:156)
> ~[hbase-server-2.1.0-cdh6.3.0.jar:?] at
> org.apache.hadoop.hbase.regionserver.wal.WALUtil.writeFlushMarker(WALUtil.java:85)
> ~[hbase-server-2.1.0-cdh6.3.0.jar:?] at
> org.apache.hadoop.hbase.regionserver.HRegion.internalPrepareFlushCache(HRegion.java:2588)
> ~[hbase-server-2.1.0-cdh6.3.0.jar:?] at
> org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2465)
> ~[hbase-server-2.1.0-cdh6.3.0.jar:?] at
> org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2439)
> ~[hbase-server-2.1.0-cdh6.3.0.jar:?] at
> org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2329)
> ~[hbase-server-2.1.0-cdh6.3.0.jar:?] at
> org.apache.hadoop.hbase.regionserver.MemStoreFlusher.flushRegion(MemStoreFlusher.java:612)
> ~[hbase-server-2.1.0-cdh6.3.0.jar:?] at
> org.apache.hadoop.hbase.regionserver.MemStoreFlusher.flushRegion(MemStoreFlusher.java:581)
> ~[hbase-server-2.1.0-cdh6.3.0.jar:?] at
> org.apache.hadoop.hbase.regionserver.MemStoreFlusher.access$1000(MemStoreFlusher.java:68)
> ~[hbase-server-2.1.0-cdh6.3.0.jar:?] at
> org.apache.hadoop.hbase.regionserver.MemStoreFlusher$FlushHandler.run(MemStoreFlusher.java:361)
> ~[hbase-server-2.1.0-cdh6.3.0.jar:?] at
> java.lang.Thread.run(Thread.java:834) ~[?:?][17:11:59:665] [INFO] -
> org.apache.hadoop.hbase.regionserver.HRegion.logFatLineOnFlush(HRegion.java:2636)
> - Flushing 1/1 column families, dataSize=126.24 MB heapSize=135.36
> MB[17:11:59:665] [WARN] -
> org.apache.hadoop.hbase.regionserver.HRegion.doAbortFlushToWAL(HRegion.java:2652)
> - Received unexpected exception trying to write ABORT_FLUSH marker to
> WAL:java.io.IOException: Cannot append; log is closed, regionName =
> test_2,00000000000000000587202560,1614762174258.c958379b0ac4a008cb4d75223683377a.
> at
> org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.stampSequenceIdAndPublishToRingBuffer(AbstractFSWAL.java:962)
> at
> org.apache.hadoop.hbase.regionserver.wal.AsyncFSWAL.append(AsyncFSWAL.java:563)
> at
> org.apache.hadoop.hbase.regionserver.wal.WALUtil.doFullAppendTransaction(WALUtil.java:156)
> at
> org.apache.hadoop.hbase.regionserver.wal.WALUtil.writeFlushMarker(WALUtil.java:85)
> at
> org.apache.hadoop.hbase.regionserver.HRegion.doAbortFlushToWAL(HRegion.java:2649)
> at
> org.apache.hadoop.hbase.regionserver.HRegion.internalPrepareFlushCache(HRegion.java:2599)
> at
> org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2465)
> at
> org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2439)
> at
> org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2329) at
> org.apache.hadoop.hbase.regionserver.MemStoreFlusher.flushRegion(MemStoreFlusher.java:612)
> at
> org.apache.hadoop.hbase.regionserver.MemStoreFlusher.flushRegion(MemStoreFlusher.java:581)
> at
> org.apache.hadoop.hbase.regionserver.MemStoreFlusher.access$1000(MemStoreFlusher.java:68)
> at
> org.apache.hadoop.hbase.regionserver.MemStoreFlusher$FlushHandler.run(MemStoreFlusher.java:361)
> at java.base/java.lang.Thread.run(Thread.java:834)
> [17:11:59:665] [ERROR] -
> org.apache.hadoop.hbase.regionserver.MemStoreFlusher.flushRegion(MemStoreFlusher.java:631)
> - Cache flush failed for region
> test_2,00000000000000000587202560,1614762174258.c958379b0ac4a008cb4d75223683377a.java.io.IOException:
> Cannot append; log is closed, regionName =
> test_2,00000000000000000587202560,1614762174258.c958379b0ac4a008cb4d75223683377a.
> at
> org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.stampSequenceIdAndPublishToRingBuffer(AbstractFSWAL.java:962)
> ~[hbase-server-2.1.0-cdh6.3.0.jar:?] at
> org.apache.hadoop.hbase.regionserver.wal.AsyncFSWAL.append(AsyncFSWAL.java:563)
> ~[hbase-server-2.1.0-cdh6.3.0.jar:?] at
> org.apache.hadoop.hbase.regionserver.wal.WALUtil.doFullAppendTransaction(WALUtil.java:156)
> ~[hbase-server-2.1.0-cdh6.3.0.jar:?] at
> org.apache.hadoop.hbase.regionserver.wal.WALUtil.writeFlushMarker(WALUtil.java:85)
> ~[hbase-server-2.1.0-cdh6.3.0.jar:?] at
> org.apache.hadoop.hbase.regionserver.HRegion.internalPrepareFlushCache(HRegion.java:2588)
> ~[hbase-server-2.1.0-cdh6.3.0.jar:?] at
> org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2465)
> ~[hbase-server-2.1.0-cdh6.3.0.jar:?] at
> org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2439)
> ~[hbase-server-2.1.0-cdh6.3.0.jar:?] at
> org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2329)
> ~[hbase-server-2.1.0-cdh6.3.0.jar:?] at
> org.apache.hadoop.hbase.regionserver.MemStoreFlusher.flushRegion(MemStoreFlusher.java:612)
> ~[hbase-server-2.1.0-cdh6.3.0.jar:?] at
> org.apache.hadoop.hbase.regionserver.MemStoreFlusher.flushRegion(MemStoreFlusher.java:581)
> ~[hbase-server-2.1.0-cdh6.3.0.jar:?] at
> org.apache.hadoop.hbase.regionserver.MemStoreFlusher.access$1000(MemStoreFlusher.java:68)
> ~[hbase-server-2.1.0-cdh6.3.0.jar:?] at
> org.apache.hadoop.hbase.regionserver.MemStoreFlusher$FlushHandler.run(MemStoreFlusher.java:361)
> ~[hbase-server-2.1.0-cdh6.3.0.jar:?] at
> java.lang.Thread.run(Thread.java:834) ~[?:?][17:11:59:666] [WARN] -
> org.apache.hadoop.hbase.regionserver.HRegion.doAbortFlushToWAL(HRegion.java:2652)
> - Received unexpected exception trying to write ABORT_FLUSH marker to
> WAL:java.io.IOException: Cannot append; log is closed, regionName =
> test_2,00000000000000000125829120,1614762174258.b14f5ec6e23e1390b2daf0f83be84ee0.
> at
> org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.stampSequenceIdAndPublishToRingBuffer(AbstractFSWAL.java:962)
> at
> org.apache.hadoop.hbase.regionserver.wal.AsyncFSWAL.append(AsyncFSWAL.java:563)
> at
> org.apache.hadoop.hbase.regionserver.wal.WALUtil.doFullAppendTransaction(WALUtil.java:156)
> at
> org.apache.hadoop.hbase.regionserver.wal.WALUtil.writeFlushMarker(WALUtil.java:85)
> at
> org.apache.hadoop.hbase.regionserver.HRegion.doAbortFlushToWAL(HRegion.java:2649)
> at
> org.apache.hadoop.hbase.regionserver.HRegion.internalPrepareFlushCache(HRegion.java:2599)
> at
> org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2465)
> at
> org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2439)
> at
> org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2329) at
> org.apache.hadoop.hbase.regionserver.MemStoreFlusher.flushRegion(MemStoreFlusher.java:612)
> at
> org.apache.hadoop.hbase.regionserver.MemStoreFlusher.flushRegion(MemStoreFlusher.java:581)
> at
> org.apache.hadoop.hbase.regionserver.MemStoreFlusher.access$1000(MemStoreFlusher.java:68)
> at
> org.apache.hadoop.hbase.regionserver.MemStoreFlusher$FlushHandler.run(MemStoreFlusher.java:361)
> at java.base/java.lang.Thread.run(Thread.java:834)
> [17:11:59:666] [ERROR] -
> org.apache.hadoop.hbase.regionserver.MemStoreFlusher.flushRegion(MemStoreFlusher.java:631)
> - Cache flush failed for region
> test_2,00000000000000000125829120,1614762174258.b14f5ec6e23e1390b2daf0f83be84ee0.java.io.IOException:
> Cannot append; log is closed, regionName =
> test_2,00000000000000000125829120,1614762174258.b14f5ec6e23e1390b2daf0f83be84ee0.
> at
> org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.stampSequenceIdAndPublishToRingBuffer(AbstractFSWAL.java:962)
> ~[hbase-server-2.1.0-cdh6.3.0.jar:?] at
> org.apache.hadoop.hbase.regionserver.wal.AsyncFSWAL.append(AsyncFSWAL.java:563)
> ~[hbase-server-2.1.0-cdh6.3.0.jar:?] at
> org.apache.hadoop.hbase.regionserver.wal.WALUtil.doFullAppendTransaction(WALUtil.java:156)
> ~[hbase-server-2.1.0-cdh6.3.0.jar:?] at
> org.apache.hadoop.hbase.regionserver.wal.WALUtil.writeFlushMarker(WALUtil.java:85)
> ~[hbase-server-2.1.0-cdh6.3.0.jar:?] at
> org.apache.hadoop.hbase.regionserver.HRegion.internalPrepareFlushCache(HRegion.java:2588)
> ~[hbase-server-2.1.0-cdh6.3.0.jar:?] at
> org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2465)
> ~[hbase-server-2.1.0-cdh6.3.0.jar:?] at
> org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2439)
> ~[hbase-server-2.1.0-cdh6.3.0.jar:?] at
> org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2329)
> ~[hbase-server-2.1.0-cdh6.3.0.jar:?] at
> org.apache.hadoop.hbase.regionserver.MemStoreFlusher.flushRegion(MemStoreFlusher.java:612)
> ~[hbase-server-2.1.0-cdh6.3.0.jar:?] at
> org.apache.hadoop.hbase.regionserver.MemStoreFlusher.flushRegion(MemStoreFlusher.java:581)
> ~[hbase-server-2.1.0-cdh6.3.0.jar:?] at
> org.apache.hadoop.hbase.regionserver.MemStoreFlusher.access$1000(MemStoreFlusher.java:68)
> ~[hbase-server-2.1.0-cdh6.3.0.jar:?] at
> org.apache.hadoop.hbase.regionserver.MemStoreFlusher$FlushHandler.run(MemStoreFlusher.java:361)
> ~[hbase-server-2.1.0-cdh6.3.0.jar:?] at
> java.lang.Thread.run(Thread.java:834) ~[?:?][17:11:59:813] [ERROR] -
> org.apache.hadoop.hbase.regionserver.HRegionServer.abort(HRegionServer.java:2376)
> - ***** ABORTING region server node017,16020,1614750431707: IOE in log
> roller *****java.io.IOException: Unable to rename
> hdfs://buc-001/hbase/WALs/node017,16020,1614750431707/node017%2C16020%2C1614750431707.1614762525711
> to
> hdfs://buc-001/hbase/oldWALs/node017%2C16020%2C1614750431707.1614762525711 at
> org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.archiveLogFile(AbstractFSWAL.java:651)
> ~[hbase-server-2.1.0-cdh6.3.0.jar:?] at
> org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.cleanOldLogs(AbstractFSWAL.java:627)
> ~[hbase-server-2.1.0-cdh6.3.0.jar:?] at
> org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.rollWriter(AbstractFSWAL.java:767)
> ~[hbase-server-2.1.0-cdh6.3.0.jar:?] at
> org.apache.hadoop.hbase.regionserver.LogRoller.run(LogRoller.java:190)
> ~[hbase-server-2.1.0-cdh6.3.0.jar:?] at java.lang.Thread.run(Thread.java:834)
> ~[?:?]
> Has anyone encountered a similar problem？？？
--
This message was sent by Atlassian Jira
(v8.3.4#803005)
