[
https://issues.apache.org/jira/browse/PHOENIX-2915?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
Ankit Singhal updated PHOENIX-2915:
-----------------------------------
Description:
It could be possible , that there are some index writes in WAL which are not
replayed and the index is dropped.
And, now the table is not there, these writes cannot be replayed which result
in data table regions also to not come up.
{code}
2016-05-16 07:30:29,215 ERROR [RS_OPEN_REGION-R2-DN8-188:16020-72]
handler.OpenRegionHandler: Failed open of region=<regioname>, starting to roll
back the global memstore size.
org.apache.phoenix.hbase.index.exception.MultiIndexWriteFailureException:
Failed to write to multiple index tables
at
org.apache.phoenix.hbase.index.write.recovery.TrackingParallelWriterIndexCommitter.write(TrackingParallelWriterIndexCommitter.java:235)
at
org.apache.phoenix.hbase.index.write.IndexWriter.write(IndexWriter.java:193)
at
org.apache.phoenix.hbase.index.write.IndexWriter.write(IndexWriter.java:172)
at
org.apache.phoenix.hbase.index.Indexer.preWALRestore(Indexer.java:564)
at
org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost$58.call(RegionCoprocessorHost.java:1432)
at
org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost$RegionOperation.call(RegionCoprocessorHost.java:1673)
at
org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost.execOperation(RegionCoprocessorHost.java:1748)
at
org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost.execOperation(RegionCoprocessorHost.java:1705)
at
org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost.preWALRestore(RegionCoprocessorHost.java:1423)
at
org.apache.hadoop.hbase.regionserver.HRegion.replayRecoveredEdits(HRegion.java:4029)
at
org.apache.hadoop.hbase.regionserver.HRegion.replayRecoveredEditsIfAny(HRegion.java:3885)
at
org.apache.hadoop.hbase.regionserver.HRegion.initializeRegionStores(HRegion.java:949)
at
org.apache.hadoop.hbase.regionserver.HRegion.initializeRegionInternals(HRegion.java:819)
at
org.apache.hadoop.hbase.regionserver.HRegion.initialize(HRegion.java:794)
at
org.apache.hadoop.hbase.regionserver.HRegion.openHRegion(HRegion.java:6243)
at
org.apache.hadoop.hbase.regionserver.HRegion.openHRegion(HRegion.java:6204)
at
org.apache.hadoop.hbase.regionserver.HRegion.openHRegion(HRegion.java:6175)
at
org.apache.hadoop.hbase.regionserver.HRegion.openHRegion(HRegion.java:6131)
at
org.apache.hadoop.hbase.regionserver.HRegion.openHRegion(HRegion.java:6082)
at
org.apache.hadoop.hbase.regionserver.handler.OpenRegionHandler.openRegion(OpenRegionHandler.java:362)
at
org.apache.hadoop.hbase.regionserver.handler.OpenRegionHandler.process(OpenRegionHandler.java:129)
at
org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:128)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:745)
2016-05-16 07:30:29,216 INFO [RS_OPEN_REGION-R2-DN8-188:16020-72]
coordination.ZkOpenRegionCoordination: Opening of region {ENCODED =>
cd82f0b9f06972b6b111c368076677be, NAME => <regionname>', STARTKEY => 'x',
ENDKEY => 'y'} failed, transitioning from OPENING to FAILED_OPEN in ZK,
expecting version 28
{code}
was:
It could be possible , that there are some index writes in WAL which are not
replayed and the index is dropped.
And, now the table is not there, these writes cannot be replayed which result
in data table regions also to not come up.
2016-05-16 07:30:29,215 ERROR [RS_OPEN_REGION-R2-DN8-188:16020-72]
handler.OpenRegionHandler: Failed open of region=<regioname>, starting to roll
back the global memstore size.
org.apache.phoenix.hbase.index.exception.MultiIndexWriteFailureException:
Failed to write to multiple index tables
at
org.apache.phoenix.hbase.index.write.recovery.TrackingParallelWriterIndexCommitter.write(TrackingParallelWriterIndexCommitter.java:235)
at
org.apache.phoenix.hbase.index.write.IndexWriter.write(IndexWriter.java:193)
at
org.apache.phoenix.hbase.index.write.IndexWriter.write(IndexWriter.java:172)
at
org.apache.phoenix.hbase.index.Indexer.preWALRestore(Indexer.java:564)
at
org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost$58.call(RegionCoprocessorHost.java:1432)
at
org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost$RegionOperation.call(RegionCoprocessorHost.java:1673)
at
org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost.execOperation(RegionCoprocessorHost.java:1748)
at
org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost.execOperation(RegionCoprocessorHost.java:1705)
at
org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost.preWALRestore(RegionCoprocessorHost.java:1423)
at
org.apache.hadoop.hbase.regionserver.HRegion.replayRecoveredEdits(HRegion.java:4029)
at
org.apache.hadoop.hbase.regionserver.HRegion.replayRecoveredEditsIfAny(HRegion.java:3885)
at
org.apache.hadoop.hbase.regionserver.HRegion.initializeRegionStores(HRegion.java:949)
at
org.apache.hadoop.hbase.regionserver.HRegion.initializeRegionInternals(HRegion.java:819)
at
org.apache.hadoop.hbase.regionserver.HRegion.initialize(HRegion.java:794)
at
org.apache.hadoop.hbase.regionserver.HRegion.openHRegion(HRegion.java:6243)
at
org.apache.hadoop.hbase.regionserver.HRegion.openHRegion(HRegion.java:6204)
at
org.apache.hadoop.hbase.regionserver.HRegion.openHRegion(HRegion.java:6175)
at
org.apache.hadoop.hbase.regionserver.HRegion.openHRegion(HRegion.java:6131)
at
org.apache.hadoop.hbase.regionserver.HRegion.openHRegion(HRegion.java:6082)
at
org.apache.hadoop.hbase.regionserver.handler.OpenRegionHandler.openRegion(OpenRegionHandler.java:362)
at
org.apache.hadoop.hbase.regionserver.handler.OpenRegionHandler.process(OpenRegionHandler.java:129)
at
org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:128)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:745)
2016-05-16 07:30:29,216 INFO [RS_OPEN_REGION-R2-DN8-188:16020-72]
coordination.ZkOpenRegionCoordination: Opening of region {ENCODED =>
cd82f0b9f06972b6b111c368076677be, NAME => <regionname>', STARTKEY => 'x',
ENDKEY => 'y'} failed, transitioning from OPENING to FAILED_OPEN in ZK,
expecting version 28
> Dropping of Index can still leave some non-replayed writes in WAL
> -----------------------------------------------------------------
>
> Key: PHOENIX-2915
> URL: https://issues.apache.org/jira/browse/PHOENIX-2915
> Project: Phoenix
> Issue Type: Bug
> Reporter: Ankit Singhal
>
> It could be possible , that there are some index writes in WAL which are not
> replayed and the index is dropped.
> And, now the table is not there, these writes cannot be replayed which result
> in data table regions also to not come up.
> {code}
> 2016-05-16 07:30:29,215 ERROR [RS_OPEN_REGION-R2-DN8-188:16020-72]
> handler.OpenRegionHandler: Failed open of region=<regioname>, starting to
> roll back the global memstore size.
> org.apache.phoenix.hbase.index.exception.MultiIndexWriteFailureException:
> Failed to write to multiple index tables
> at
> org.apache.phoenix.hbase.index.write.recovery.TrackingParallelWriterIndexCommitter.write(TrackingParallelWriterIndexCommitter.java:235)
> at
> org.apache.phoenix.hbase.index.write.IndexWriter.write(IndexWriter.java:193)
> at
> org.apache.phoenix.hbase.index.write.IndexWriter.write(IndexWriter.java:172)
> at
> org.apache.phoenix.hbase.index.Indexer.preWALRestore(Indexer.java:564)
> at
> org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost$58.call(RegionCoprocessorHost.java:1432)
> at
> org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost$RegionOperation.call(RegionCoprocessorHost.java:1673)
> at
> org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost.execOperation(RegionCoprocessorHost.java:1748)
> at
> org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost.execOperation(RegionCoprocessorHost.java:1705)
> at
> org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost.preWALRestore(RegionCoprocessorHost.java:1423)
> at
> org.apache.hadoop.hbase.regionserver.HRegion.replayRecoveredEdits(HRegion.java:4029)
> at
> org.apache.hadoop.hbase.regionserver.HRegion.replayRecoveredEditsIfAny(HRegion.java:3885)
> at
> org.apache.hadoop.hbase.regionserver.HRegion.initializeRegionStores(HRegion.java:949)
> at
> org.apache.hadoop.hbase.regionserver.HRegion.initializeRegionInternals(HRegion.java:819)
> at
> org.apache.hadoop.hbase.regionserver.HRegion.initialize(HRegion.java:794)
> at
> org.apache.hadoop.hbase.regionserver.HRegion.openHRegion(HRegion.java:6243)
> at
> org.apache.hadoop.hbase.regionserver.HRegion.openHRegion(HRegion.java:6204)
> at
> org.apache.hadoop.hbase.regionserver.HRegion.openHRegion(HRegion.java:6175)
> at
> org.apache.hadoop.hbase.regionserver.HRegion.openHRegion(HRegion.java:6131)
> at
> org.apache.hadoop.hbase.regionserver.HRegion.openHRegion(HRegion.java:6082)
> at
> org.apache.hadoop.hbase.regionserver.handler.OpenRegionHandler.openRegion(OpenRegionHandler.java:362)
> at
> org.apache.hadoop.hbase.regionserver.handler.OpenRegionHandler.process(OpenRegionHandler.java:129)
> at
> org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:128)
> at
> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
> at
> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
> at java.lang.Thread.run(Thread.java:745)
> 2016-05-16 07:30:29,216 INFO [RS_OPEN_REGION-R2-DN8-188:16020-72]
> coordination.ZkOpenRegionCoordination: Opening of region {ENCODED =>
> cd82f0b9f06972b6b111c368076677be, NAME => <regionname>', STARTKEY => 'x',
> ENDKEY => 'y'} failed, transitioning from OPENING to FAILED_OPEN in ZK,
> expecting version 28
> {code}
--
This message was sent by Atlassian JIRA
(v6.3.4#6332)
