[
https://issues.apache.org/jira/browse/HBASE-27886?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=17869891#comment-17869891
]
SunQiang commented on HBASE-27886:
----------------------------------
What happened to the HBase system that caused this issue?
> wal.AbstractProtobufLogWriter: Failed to write trailer, non-fatal, continuing
> -----------------------------------------------------------------------------
>
> Key: HBASE-27886
> URL: https://issues.apache.org/jira/browse/HBASE-27886
> Project: HBase
> Issue Type: Bug
> Components: wal
> Affects Versions: 2.2.2
> Reporter: Jepson
> Priority: Major
>
> 2023-05-25 03:35:58,546 INFO [qtp772147225-791] hfile.CacheConfig: Created
> cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false,
> cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false,
> cacheDataCompressed=false, prefetchOnOpen=false with blockCache=null
> 2023-05-25 03:36:20,083 INFO [regionserver/bdp04:16020.logRoller]
> wal.AbstractFSWAL: Rolled WAL
> /hbase/WALs/bdp04,16020,1684954879778/bdp04%2C16020%2C1684954879778.1684956800003
> with entries=11, filesize=64.23 KB; new WAL
> /hbase/WALs/bdp04,16020,1684954879778/bdp04%2C16020%2C1684954879778.1684956980051
> 2023-05-25 03:36:20,083 WARN [Close-WAL-Writer-9]{color:#de350b}
> *wal.AbstractProtobufLogWriter: Failed to write trailer, non-fatal,
> continuing...*{color}
> *{color:#de350b}java.io.IOException: stream already broken{color}*
> at
> org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.flush0(FanOutOneBlockAsyncDFSOutput.java:424)
> at
> org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.flush(FanOutOneBlockAsyncDFSOutput.java:513)
> at
> org.apache.hadoop.hbase.regionserver.wal.AsyncProtobufLogWriter.lambda$writeWALTrailerAndMagic$3(AsyncProtobufLogWriter.java:220)
> at
> org.apache.hadoop.hbase.regionserver.wal.AsyncProtobufLogWriter.write(AsyncProtobufLogWriter.java:176)
> at
> org.apache.hadoop.hbase.regionserver.wal.AsyncProtobufLogWriter.writeWALTrailerAndMagic(AsyncProtobufLogWriter.java:211)
> at
> org.apache.hadoop.hbase.regionserver.wal.AbstractProtobufLogWriter.writeWALTrailer(AbstractProtobufLogWriter.java:235)
> at
> org.apache.hadoop.hbase.regionserver.wal.AsyncProtobufLogWriter.close(AsyncProtobufLogWriter.java:153)
> at
> org.apache.hadoop.hbase.regionserver.wal.AsyncFSWAL.lambda$closeWriter$6(AsyncFSWAL.java:646)
> at
> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
> at
> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
> at java.lang.Thread.run(Thread.java:748)
> 2023-05-25 03:36:20,084 WARN [Close-WAL-Writer-9]
> {color:#de350b}*wal.AsyncProtobufLogWriter: normal close failed, try
> recover*{color}
> java.io.IOException: stream already broken
> at
> org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:521)
> at
> org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:569)
> at
> org.apache.hadoop.hbase.regionserver.wal.AsyncProtobufLogWriter.close(AsyncProtobufLogWriter.java:154)
> at
> org.apache.hadoop.hbase.regionserver.wal.AsyncFSWAL.lambda$closeWriter$6(AsyncFSWAL.java:646)
> at
> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
> at
> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
> at java.lang.Thread.run(Thread.java:748)
> 2023-05-25 03:36:20,084 INFO [Close-WAL-Writer-9]
> util.FSHDFSUtils:{color:#de350b} Recover lease on dfs file
> /hbase/WALs/bdp04,16020,1684954879778/bdp04%2C16020%2C1684954879778.1684956800003{color}
> 2023-05-25 03:36:20,087 INFO [Close-WAL-Writer-9]
> util.FSHDFSUtils:{color:#de350b} Failed to recover lease, attempt=0 on
> file=/hbase/WALs/bdp04,16020,1684954879778/bdp04%2C16020%2C1684954879778.1684956800003
> after 3ms{color}
> 2023-05-25 03:36:24,010 INFO [LruBlockCacheStatsExecutor]
> hfile.LruBlockCache: totalSize=3.45 MB, freeSize=3.16 GB, max=3.16 GB,
> blockCount=2, accesses=23184, hits=23184, hitRatio=100.00%, ,
> cachingAccesses=23172, cachingHits=23172, cachingHitsRatio=100.00%,
> evictions=209, evicted=0, evictedPerRun=0.0
> 2023-05-25 03:36:24,088 INFO [Close-WAL-Writer-9] util.FSHDFSUtils:
> Recovered lease, attempt=1 on
> file=/hbase/WALs/bdp04,16020,1684954879778/bdp04%2C16020%2C1684954879778.1684956800003
> after 4004ms
> 2023-05-25 03:36:24,623 INFO [BucketCacheStatsExecutor] bucket.BucketCache:
> failedBlockAdditions=0, totalSize=3.99 GB, freeSize=3.99 GB, usedSize=99 KB,
> cacheSize=58.92 KB, accesses=30276, hits=271, IOhitsPerSecond=1,
> IOTimePerHit=0.03, hitRatio=0.90%, cachingAccesses=298, cachingHits=259,
> cachingHitsRatio=86.91%, evictions=0, evicted=8, evictedPerRun=0.0
> 2023-05-25 03:38:08,248 WARN [AsyncFSWAL-0] wal.AsyncFSWAL: sync failed
> java.io.IOException: stream already broken
> at
> org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.flush0(FanOutOneBlockAsyncDFSOutput.java:424)
> at
> org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.flush(FanOutOneBlockAsyncDFSOutput.java:513)
> at
> org.apache.hadoop.hbase.regionserver.wal.AsyncProtobufLogWriter.sync(AsyncProtobufLogWriter.java:144)
> at
> org.apache.hadoop.hbase.regionserver.wal.AsyncFSWAL.sync(AsyncFSWAL.java:352)
> at
> org.apache.hadoop.hbase.regionserver.wal.AsyncFSWAL.consume(AsyncFSWAL.java:539)
> at
> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
> at
> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
> at java.lang.Thread.run(Thread.java:748)
> 2023-05-25 03:38:08,300 INFO [regionserver/bdp04:16020.logRoller]
> wal.AbstractFSWAL: Rolled WAL
> /hbase/WALs/bdp04,16020,1684954879778/bdp04%2C16020%2C1684954879778.1684956980051
> with entries=7, filesize=57.58 KB; new WAL
> /hbase/WALs/bdp04,16020,1684954879778/bdp04%2C16020%2C1684954879778.1684957088248
> 2023-05-25 03:38:08,301 WARN [Close-WAL-Writer-10]
> wal.AbstractProtobufLogWriter: Failed to write trailer, non-fatal,
> continuing...
> java.io.IOException: stream already broken
> at
> org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.flush0(FanOutOneBlockAsyncDFSOutput.java:424)
> at
> org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.flush(FanOutOneBlockAsyncDFSOutput.java:513)
> at
> org.apache.hadoop.hbase.regionserver.wal.AsyncProtobufLogWriter.lambda$writeWALTrailerAndMagic$3(AsyncProtobufLogWriter.java:220)
> at
> org.apache.hadoop.hbase.regionserver.wal.AsyncProtobufLogWriter.write(AsyncProtobufLogWriter.java:176)
> at
> org.apache.hadoop.hbase.regionserver.wal.AsyncProtobufLogWriter.writeWALTrailerAndMagic(AsyncProtobufLogWriter.java:211)
> at
> org.apache.hadoop.hbase.regionserver.wal.AbstractProtobufLogWriter.writeWALTrailer(AbstractProtobufLogWriter.java:235)
> at
> org.apache.hadoop.hbase.regionserver.wal.AsyncProtobufLogWriter.close(AsyncProtobufLogWriter.java:153)
> at
> org.apache.hadoop.hbase.regionserver.wal.AsyncFSWAL.lambda$closeWriter$6(AsyncFSWAL.java:646)
> at
> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
> at
> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
> at java.lang.Thread.run(Thread.java:748)
> 2023-05-25 03:38:08,301 WARN [Close-WAL-Writer-10]
> wal.AsyncProtobufLogWriter: normal close failed, try recover
> java.io.IOException: stream already broken
> at
> org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:521)
> at
> org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:569)
> at
> org.apache.hadoop.hbase.regionserver.wal.AsyncProtobufLogWriter.close(AsyncProtobufLogWriter.java:154)
> at
> org.apache.hadoop.hbase.regionserver.wal.AsyncFSWAL.lambda$closeWriter$6(AsyncFSWAL.java:646)
> at
> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
> at
> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
> at java.lang.Thread.run(Thread.java:748)
> 2023-05-25 03:38:08,302 INFO [Close-WAL-Writer-10] util.FSHDFSUtils: Recover
> lease on dfs file
> /hbase/WALs/bdp04,16020,1684954879778/bdp04%2C16020%2C1684954879778.1684956980051
> 2023-05-25 03:38:08,304 INFO [Close-WAL-Writer-10] util.FSHDFSUtils: Failed
> to recover lease, attempt=0 on
> file=/hbase/WALs/bdp04,16020,1684954879778/bdp04%2C16020%2C1684954879778.1684956980051
> after 2ms
> 2023-05-25 03:38:12,306 INFO [Close-WAL-Writer-10] util.FSHDFSUtils:
> Recovered lease, attempt=1 on
> file=/hbase/WALs/bdp04,16020,1684954879778/bdp04%2C16020%2C1684954879778.1684956980051
> after 4004ms
> 2023-05-25 03:41:24,010 INFO [LruBlockCacheStatsExecutor]
> hfile.LruBlockCache: totalSize=3.45 MB, freeSize=3.16 GB, max=3.16 GB,
> blockCount=2, accesses=23193, hits=23193, hitRatio=100.00%, ,
> cachingAccesses=23181, cachingHits=23181, cachingHitsRatio=100.00%,
> evictions=239, evicted=0, evictedPerRun=0.0
--
This message was sent by Atlassian Jira
(v8.20.10#820010)
