Size ~ 4 000M, Store File Size ~ 1TB (FAST_DIFF) 23 columns in 1 column family
> On 16 Aug 2019, at 04:14, Sean Kennedy <sck4...@gmail.com> wrote: > > Alex, > > How large is the table and how many columns? > > Thx > > Sean > > On Thursday, August 15, 2019, Alexander Batyrshin <0x62...@gmail.com> wrote: > >> >> Hello all, >> >> We observer error "Added a key not lexically larger than previous” that >> cause most of our region-servers to crash in our cluster. >> HBase-1.4.10 >> >> 2019-08-15 18:02:10,554 INFO [MemStoreFlusher.0] regionserver.HRegion: >> Flushing 1/1 column families, memstore=56.08 MB >> 2019-08-15 18:02:10,727 WARN [MemStoreFlusher.0] regionserver.HStore: >> Failed flushing store file, retrying num=0 >> java.io.IOException: Added a key not lexically larger than previous. >> Current cell = >> \x0901820448218>wGavb'/d:elr/1565881054828/DeleteColumn/vlen=0/seqid=44456567, >> lastCell = \x0901820448218>wGavb'/d:elr/1565881054828/Put/vlen=1/ >> seqid=44457770 >> at org.apache.hadoop.hbase.io.hfile.AbstractHFileWriter. >> checkKey(AbstractHFileWriter.java:204) >> at org.apache.hadoop.hbase.io.hfile.HFileWriterV2.append( >> HFileWriterV2.java:279) >> at org.apache.hadoop.hbase.io.hfile.HFileWriterV3.append( >> HFileWriterV3.java:87) >> at org.apache.hadoop.hbase.regionserver.StoreFile$Writer. >> append(StoreFile.java:1127) >> at org.apache.hadoop.hbase.regionserver.StoreFlusher. >> performFlush(StoreFlusher.java:139) >> at org.apache.hadoop.hbase.regionserver.DefaultStoreFlusher. >> flushSnapshot(DefaultStoreFlusher.java:75) >> at org.apache.hadoop.hbase.regionserver.HStore. >> flushCache(HStore.java:1003) >> at org.apache.hadoop.hbase.regionserver.HStore$ >> StoreFlusherImpl.flushCache(HStore.java:2523) >> at org.apache.hadoop.hbase.regionserver.HRegion. >> internalFlushCacheAndCommit(HRegion.java:2622) >> at org.apache.hadoop.hbase.regionserver.HRegion. >> internalFlushcache(HRegion.java:2352) >> at org.apache.hadoop.hbase.regionserver.HRegion. >> internalFlushcache(HRegion.java:2314) >> at org.apache.hadoop.hbase.regionserver.HRegion. >> flushcache(HRegion.java:2200) >> at org.apache.hadoop.hbase.regionserver.HRegion.flush( >> HRegion.java:2125) >> at org.apache.hadoop.hbase.regionserver.MemStoreFlusher. >> flushRegion(MemStoreFlusher.java:512) >> at org.apache.hadoop.hbase.regionserver.MemStoreFlusher. >> flushRegion(MemStoreFlusher.java:482) >> at org.apache.hadoop.hbase.regionserver.MemStoreFlusher. >> access$900(MemStoreFlusher.java:76) >> at org.apache.hadoop.hbase.regionserver.MemStoreFlusher$ >> FlushHandler.run(MemStoreFlusher.java:264) >> at java.lang.Thread.run(Thread.java:748) >> >> 2019-08-15 18:02:21,776 WARN [MemStoreFlusher.0] regionserver.HStore: >> Failed flushing store file, retrying num=9 >> java.io.IOException: Added a key not lexically larger than previous. >> Current cell = >> \x0901820448218>wGavb'/d:elr/1565881054828/DeleteColumn/vlen=0/seqid=44456567, >> lastCell = \x0901820448218>wGavb'/d:elr/1565881054828/Put/vlen=1/ >> seqid=44457770 >> at org.apache.hadoop.hbase.io.hfile.AbstractHFileWriter. >> checkKey(AbstractHFileWriter.java:204) >> at org.apache.hadoop.hbase.io.hfile.HFileWriterV2.append( >> HFileWriterV2.java:279) >> at org.apache.hadoop.hbase.io.hfile.HFileWriterV3.append( >> HFileWriterV3.java:87) >> at org.apache.hadoop.hbase.regionserver.StoreFile$Writer. >> append(StoreFile.java:1127) >> at org.apache.hadoop.hbase.regionserver.StoreFlusher. >> performFlush(StoreFlusher.java:139) >> at org.apache.hadoop.hbase.regionserver.DefaultStoreFlusher. >> flushSnapshot(DefaultStoreFlusher.java:75) >> at org.apache.hadoop.hbase.regionserver.HStore. >> flushCache(HStore.java:1003) >> at org.apache.hadoop.hbase.regionserver.HStore$ >> StoreFlusherImpl.flushCache(HStore.java:2523) >> at org.apache.hadoop.hbase.regionserver.HRegion. >> internalFlushCacheAndCommit(HRegion.java:2622) >> at org.apache.hadoop.hbase.regionserver.HRegion. >> internalFlushcache(HRegion.java:2352) >> at org.apache.hadoop.hbase.regionserver.HRegion. >> internalFlushcache(HRegion.java:2314) >> at org.apache.hadoop.hbase.regionserver.HRegion. >> flushcache(HRegion.java:2200) >> at org.apache.hadoop.hbase.regionserver.HRegion.flush( >> HRegion.java:2125) >> at org.apache.hadoop.hbase.regionserver.MemStoreFlusher. >> flushRegion(MemStoreFlusher.java:512) >> at org.apache.hadoop.hbase.regionserver.MemStoreFlusher. >> flushRegion(MemStoreFlusher.java:482) >> at org.apache.hadoop.hbase.regionserver.MemStoreFlusher. >> access$900(MemStoreFlusher.java:76) >> at org.apache.hadoop.hbase.regionserver.MemStoreFlusher$ >> FlushHandler.run(MemStoreFlusher.java:264) >> at java.lang.Thread.run(Thread.java:748) >> >> 2019-08-15 18:02:21,777 FATAL [MemStoreFlusher.0] >> regionserver.HRegionServer: ABORTING region server >> prod006,60020,1565873610692: Replay of WAL required. Forcing server shutdown >> org.apache.hadoop.hbase.DroppedSnapshotException: region: TBL_TABLE_CODE,\ >> x0904606203097821slG=sPD,1563070299676.5110b3395ca64a51cea99c6572a4c3d9. >> at org.apache.hadoop.hbase.regionserver.HRegion. >> internalFlushCacheAndCommit(HRegion.java:2675) >> at org.apache.hadoop.hbase.regionserver.HRegion. >> internalFlushcache(HRegion.java:2352) >> at org.apache.hadoop.hbase.regionserver.HRegion. >> internalFlushcache(HRegion.java:2314) >> at org.apache.hadoop.hbase.regionserver.HRegion. >> flushcache(HRegion.java:2200) >> at org.apache.hadoop.hbase.regionserver.HRegion.flush( >> HRegion.java:2125) >> at org.apache.hadoop.hbase.regionserver.MemStoreFlusher. >> flushRegion(MemStoreFlusher.java:512) >> at org.apache.hadoop.hbase.regionserver.MemStoreFlusher. >> flushRegion(MemStoreFlusher.java:482) >> at org.apache.hadoop.hbase.regionserver.MemStoreFlusher. >> access$900(MemStoreFlusher.java:76) >> at org.apache.hadoop.hbase.regionserver.MemStoreFlusher$ >> FlushHandler.run(MemStoreFlusher.java:264) >> at java.lang.Thread.run(Thread.java:748) >> Caused by: java.io.IOException: Added a key not lexically larger than >> previous. Current cell = \x0901820448218>wGavb'/d:elr/ >> 1565881054828/DeleteColumn/vlen=0/seqid=44456567, lastCell = >> \x0901820448218>wGavb'/d:elr/1565881054828/Put/vlen=1/seqid=44457770 >> at org.apache.hadoop.hbase.io.hfile.AbstractHFileWriter. >> checkKey(AbstractHFileWriter.java:204) >> at org.apache.hadoop.hbase.io.hfile.HFileWriterV2.append( >> HFileWriterV2.java:279) >> at org.apache.hadoop.hbase.io.hfile.HFileWriterV3.append( >> HFileWriterV3.java:87) >> at org.apache.hadoop.hbase.regionserver.StoreFile$Writer. >> append(StoreFile.java:1127) >> at org.apache.hadoop.hbase.regionserver.StoreFlusher. >> performFlush(StoreFlusher.java:139) >> at org.apache.hadoop.hbase.regionserver.DefaultStoreFlusher. >> flushSnapshot(DefaultStoreFlusher.java:75) >> at org.apache.hadoop.hbase.regionserver.HStore. >> flushCache(HStore.java:1003) >> at org.apache.hadoop.hbase.regionserver.HStore$ >> StoreFlusherImpl.flushCache(HStore.java:2523) >> at org.apache.hadoop.hbase.regionserver.HRegion. >> internalFlushCacheAndCommit(HRegion.java:2622) >> ... 9 more >> >>
