Hi, all,
I use hive to load some data to do some test. This morning after I
loaded some data I did a sum query which leading to a HBase regionserver
crashed. Then I restart HBase hoping to do some load balace, but just got
these errors in the HBase regionserve log.
The worse is that I have no other choice but to execute "initialize
trafodion, drop" because the trafci can only execute this command.
*"**2016-09-13 14:30:40,404 ERROR [RS_OPEN_REGION-hadoop2slave9:60020-0]
handler.OpenRegionHandler: Failed open of
region=TRAFODION._MD_.OBJECTS,,1473729031974.951e233c8656e18b05d5fc815bfcca4a.,
starting to roll back the global memstore size.*
*org.apache.hadoop.hbase.DroppedSnapshotException: region:
TRAFODION._MD_.OBJECTS,,1473729031974.951e233c8656e18b05d5fc815bfcca4a.*
* at
org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:2243)*
* at
org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:1972)*
* at
org.apache.hadoop.hbase.regionserver.HRegion.replayRecoveredEditsIfAny(HRegion.java:3826)*
* at
org.apache.hadoop.hbase.regionserver.HRegion.initializeRegionStores(HRegion.java:969)*
* at
org.apache.hadoop.hbase.regionserver.HRegion.initializeRegionInternals(HRegion.java:841)*
* at
org.apache.hadoop.hbase.regionserver.HRegion.initialize(HRegion.java:814)*
* at
org.apache.hadoop.hbase.regionserver.HRegion.openHRegion(HRegion.java:5828)*
* at
org.apache.hadoop.hbase.regionserver.transactional.TransactionalRegion.openHRegion(TransactionalRegion.java:101)*
* at
org.apache.hadoop.hbase.regionserver.HRegion.openHRegion(HRegion.java:5794)*
* at
org.apache.hadoop.hbase.regionserver.HRegion.openHRegion(HRegion.java:5765)*
* at
org.apache.hadoop.hbase.regionserver.HRegion.openHRegion(HRegion.java:5721)*
* at
org.apache.hadoop.hbase.regionserver.HRegion.openHRegion(HRegion.java:5672)*
* at
org.apache.hadoop.hbase.regionserver.handler.OpenRegionHandler.openRegion(OpenRegionHandler.java:356)*
* at
org.apache.hadoop.hbase.regionserver.handler.OpenRegionHandler.process(OpenRegionHandler.java:126)*
* at
org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:128)*
* at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)*
* at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)*
* at java.lang.Thread.run(Thread.java:745)*
*Caused by: java.lang.AssertionError: Key TRAFODION
SEABASE
SB_HISTOGRAMS
BT/#1:\x01/1473737105925/Put/vlen=9/seqid=51 followed by a smaller key
TRAFODION
SEABASE
SB_HISTOGRAMS
BT/#1:\x01/1473737105925/Put/vlen=9/seqid=243 in cf #1*
* at
org.apache.hadoop.hbase.regionserver.StoreScanner.checkScanOrder(StoreScanner.java:699)*
* at
org.apache.hadoop.hbase.regionserver.StoreScanner.next(StoreScanner.java:493)*
* at
org.apache.hadoop.hbase.regionserver.StoreFlusher.performFlush(StoreFlusher.java:115)*
*"*
*and *
*"**2016-09-13 14:30:40,288 ERROR [RS_OPEN_REGION-hadoop2slave9:60020-2]
handler.OpenRegionHandler: Failed open of
region=TRAFODION._MD_.TEXT,,1473729102671.427abe5d9d8d13de9a755ebd52e94c9c.,
starting to roll back the global memstore size.*
*org.apache.hadoop.hbase.DroppedSnapshotException: region:
TRAFODION._MD_.TEXT,,1473729102671.427abe5d9d8d13de9a755ebd52e94c9c.*
* at
org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:2243)*
* at
org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:1972)*
* at
org.apache.hadoop.hbase.regionserver.HRegion.replayRecoveredEditsIfAny(HRegion.java:3826)*
* at
org.apache.hadoop.hbase.regionserver.HRegion.initializeRegionStores(HRegion.java:969)*
* at
org.apache.hadoop.hbase.regionserver.HRegion.initializeRegionInternals(HRegion.java:841)*
* at
org.apache.hadoop.hbase.regionserver.HRegion.initialize(HRegion.java:814)*
* at
org.apache.hadoop.hbase.regionserver.HRegion.openHRegion(HRegion.java:5828)*
* at
org.apache.hadoop.hbase.regionserver.transactional.TransactionalRegion.openHRegion(TransactionalRegion.java:101)*
* at
org.apache.hadoop.hbase.regionserver.HRegion.openHRegion(HRegion.java:5794)*
* at
org.apache.hadoop.hbase.regionserver.HRegion.openHRegion(HRegion.java:5765)*
* at
org.apache.hadoop.hbase.regionserver.HRegion.openHRegion(HRegion.java:5721)*
* at
org.apache.hadoop.hbase.regionserver.HRegion.openHRegion(HRegion.java:5672)*
* at
org.apache.hadoop.hbase.regionserver.handler.OpenRegionHandler.openRegion(OpenRegionHandler.java:356)*
* at
org.apache.hadoop.hbase.regionserver.handler.OpenRegionHandler.process(OpenRegionHandler.java:126)*
* at
org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:128)*
* at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)*
* at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)*
* at java.lang.Thread.run(Thread.java:745)*
*Caused by: java.lang.AssertionError: Key
\xE4\xB4b\x8C\xC6\x82\xE6\x15\x80\x00\x00\x02\x80\x00\x00\x00\x80\x00\x00\x00/#1:/1473737244640/DeleteFamily/vlen=0/seqid=73
followed by a smaller key
\xE4\xB4b\x8C\xC6\x82\xE6\x15\x80\x00\x00\x02\x80\x00\x00\x00\x80\x00\x00\x00/#1:/1473737244640/DeleteFamily/vlen=0/seqid=103
in cf #1*
* at
org.apache.hadoop.hbase.regionserver.StoreScanner.checkScanOrder(StoreScanner.java:699)*
* at
org.apache.hadoop.hbase.regionserver.StoreScanner.next(StoreScanner.java:493)*
* at
org.apache.hadoop.hbase.regionserver.StoreFlusher.performFlush(StoreFlusher.java:115)*
* at
org.apache.hadoop.hbase.regionserver.DefaultStoreFlusher.flushSnapshot(DefaultStoreFlusher.java:71)*
*"*
*and *
*"**2016-09-13 14:30:40,084 ERROR [RS_OPEN_REGION-hadoop2slave9:60020-0]
handler.OpenRegionHandler: Failed open of
region=TRAFODION._MD_.OBJECTS,,1473729031974.951e233c8656e18b05d5fc815bfcca4a.,
starting to roll back the global memstore size.*
*org.apache.hadoop.hbase.DroppedSnapshotException: region:
TRAFODION._MD_.OBJECTS,,1473729031974.951e233c8656e18b05d5fc815bfcca4a.*
* at
org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:2243)*
* at
org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:1972)*
* at
org.apache.hadoop.hbase.regionserver.HRegion.replayRecoveredEditsIfAny(HRegion.java:3826)*
* at
org.apache.hadoop.hbase.regionserver.HRegion.initializeRegionStores(HRegion.java:969)*
* at
org.apache.hadoop.hbase.regionserver.HRegion.initializeRegionInternals(HRegion.java:841)*
* at
org.apache.hadoop.hbase.regionserver.HRegion.initialize(HRegion.java:814)*
* at
org.apache.hadoop.hbase.regionserver.HRegion.openHRegion(HRegion.java:5828)*
* at
org.apache.hadoop.hbase.regionserver.transactional.TransactionalRegion.openHRegion(TransactionalRegion.java:101)*
* at
org.apache.hadoop.hbase.regionserver.HRegion.openHRegion(HRegion.java:5794)*
* at
org.apache.hadoop.hbase.regionserver.HRegion.openHRegion(HRegion.java:5765)*
* at
org.apache.hadoop.hbase.regionserver.HRegion.openHRegion(HRegion.java:5721)*
* at
org.apache.hadoop.hbase.regionserver.HRegion.openHRegion(HRegion.java:5672)*
* at
org.apache.hadoop.hbase.regionserver.handler.OpenRegionHandler.openRegion(OpenRegionHandler.java:356)*
* at
org.apache.hadoop.hbase.regionserver.handler.OpenRegionHandler.process(OpenRegionHandler.java:126)*
* at
org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:128)*
* at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)*
* at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)*
* at java.lang.Thread.run(Thread.java:745)*
*Caused by: java.lang.AssertionError: Key TRAFODION
SEABASE
SB_HISTOGRAMS
BT/#1:\x01/1473737105925/Put/vlen=9/seqid=51 followed by a smaller key
TRAFODION
SEABASE
SB_HISTOGRAMS
BT/#1:\x01/1473737105925/Put/vlen=9/seqid=243 in cf #1*
* at
org.apache.hadoop.hbase.regionserver.StoreScanner.checkScanOrder(StoreScanner.java:699)*
*"*
some other Trafodion tables also have the problem.
Has some one ever encountered this problem?
Any reply is appreciated.
thanks
Qiao
