[
https://issues.apache.org/jira/browse/CASSANDRA-10579?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=14974357#comment-14974357
]
Jeff Griffith edited comment on CASSANDRA-10579 at 10/26/15 3:13 PM:
---------------------------------------------------------------------
I re-enabled assertions on this node and here is the first:
{code}
WARN [SharedPool-Worker-7] 2015-10-26 15:10:44,777
AbstractTracingAwareExecutorService.java:169 - Uncaught exception on thread
Thread[SharedPool-Worker-7,5,main]: {}
java.lang.AssertionError: null
at
org.apache.cassandra.db.AbstractNativeCell.checkPosition(AbstractNativeCell.java:585)
~[apache-cassandra-2.1.10.jar:2.1.10-SNAPSHOT]
at
org.apache.cassandra.db.AbstractNativeCell.getByteBuffer(AbstractNativeCell.java:657)
~[apache-cassandra-2.1.10.jar:2.1.10-SNAPSHOT]
at
org.apache.cassandra.db.AbstractNativeCell.get(AbstractNativeCell.java:304)
~[apache-cassandra-2.1.10.jar:2.1.10-SNAPSHOT]
at
org.apache.cassandra.db.AbstractNativeCell.get(AbstractNativeCell.java:291)
~[apache-cassandra-2.1.10.jar:2.1.10-SNAPSHOT]
at
org.apache.cassandra.db.AbstractNativeCell.sizeOf(AbstractNativeCell.java:132)
~[apache-cassandra-2.1.10.jar:2.1.10-SNAPSHOT]
at
org.apache.cassandra.db.AbstractNativeCell.<init>(AbstractNativeCell.java:120)
~[apache-cassandra-2.1.10.jar:2.1.10-SNAPSHOT]
at org.apache.cassandra.db.NativeCell.<init>(NativeCell.java:40)
~[apache-cassandra-2.1.10.jar:2.1.10-SNAPSHOT]
at
org.apache.cassandra.utils.memory.NativeAllocator.clone(NativeAllocator.java:72)
~[apache-cassandra-2.1.10.jar:2.1.10-SNAPSHOT]
at org.apache.cassandra.db.NativeCell.localCopy(NativeCell.java:64)
~[apache-cassandra-2.1.10.jar:2.1.10-SNAPSHOT]
at
org.apache.cassandra.db.AtomicBTreeColumns$ColumnUpdater.apply(AtomicBTreeColumns.java:445)
~[apache-cassandra-2.1.10.jar:2.1.10-SNAPSHOT]
at
org.apache.cassandra.db.AtomicBTreeColumns$ColumnUpdater.apply(AtomicBTreeColumns.java:418)
~[apache-cassandra-2.1.10.jar:2.1.10-SNAPSHOT]
at
org.apache.cassandra.utils.btree.NodeBuilder.addNewKey(NodeBuilder.java:322)
~[apache-cassandra-2.1.10.jar:2.1.10-SNAPSHOT]
at
org.apache.cassandra.utils.btree.NodeBuilder.update(NodeBuilder.java:190)
~[apache-cassandra-2.1.10.jar:2.1.10-SNAPSHOT]
at org.apache.cassandra.utils.btree.Builder.update(Builder.java:74)
~[apache-cassandra-2.1.10.jar:2.1.10-SNAPSHOT]
at org.apache.cassandra.utils.btree.BTree.update(BTree.java:186)
~[apache-cassandra-2.1.10.jar:2.1.10-SNAPSHOT]
at
org.apache.cassandra.db.AtomicBTreeColumns.addAllWithSizeDelta(AtomicBTreeColumns.java:225)
~[apache-cassandra-2.1.10.jar:2.1.10-SNAPSHOT]
at org.apache.cassandra.db.Memtable.put(Memtable.java:210)
~[apache-cassandra-2.1.10.jar:2.1.10-SNAPSHOT]
at
org.apache.cassandra.db.ColumnFamilyStore.apply(ColumnFamilyStore.java:1225)
~[apache-cassandra-2.1.10.jar:2.1.10-SNAPSHOT]
at org.apache.cassandra.db.Keyspace.apply(Keyspace.java:396)
~[apache-cassandra-2.1.10.jar:2.1.10-SNAPSHOT]
at org.apache.cassandra.db.Keyspace.apply(Keyspace.java:359)
~[apache-cassandra-2.1.10.jar:2.1.10-SNAPSHOT]
at
org.apache.cassandra.db.commitlog.CommitLogReplayer$1.runMayThrow(CommitLogReplayer.java:455)
~[apache-cassandra-2.1.10.jar:2.1.10-SNAPSHOT]
at
org.apache.cassandra.utils.WrappedRunnable.run(WrappedRunnable.java:28)
~[apache-cassandra-2.1.10.jar:2.1.10-SNAPSHOT]
at
java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
~[na:1.8.0_31]
at
org.apache.cassandra.concurrent.AbstractTracingAwareExecutorService$FutureTask.run(AbstractTracingAwareExecutorService.java:164)
~[apache-cassandra-2.1.10.jar:2.1.10-SNAPSHOT]
at org.apache.cassandra.concurrent.SEPWorker.run(SEPWorker.java:105)
[apache-cassandra-2.1.10.jar:2.1.10-SNAPSHOT]
at java.lang.Thread.run(Thread.java:745) [na:1.8.0_31]
{code}
was (Author: jeffery.griffith):
I re-enabled assertions on this node and here is the first:
WARN [SharedPool-Worker-7] 2015-10-26 15:10:44,777
AbstractTracingAwareExecutorService.java:169 - Uncaught exception on thread
Thread[SharedPool-Worker-7,5,main]: {}
java.lang.AssertionError: null
at
org.apache.cassandra.db.AbstractNativeCell.checkPosition(AbstractNativeCell.java:585)
~[apache-cassandra-2.1.10.jar:2.1.10-SNAPSHOT]
at
org.apache.cassandra.db.AbstractNativeCell.getByteBuffer(AbstractNativeCell.java:657)
~[apache-cassandra-2.1.10.jar:2.1.10-SNAPSHOT]
at
org.apache.cassandra.db.AbstractNativeCell.get(AbstractNativeCell.java:304)
~[apache-cassandra-2.1.10.jar:2.1.10-SNAPSHOT]
at
org.apache.cassandra.db.AbstractNativeCell.get(AbstractNativeCell.java:291)
~[apache-cassandra-2.1.10.jar:2.1.10-SNAPSHOT]
at
org.apache.cassandra.db.AbstractNativeCell.sizeOf(AbstractNativeCell.java:132)
~[apache-cassandra-2.1.10.jar:2.1.10-SNAPSHOT]
at
org.apache.cassandra.db.AbstractNativeCell.<init>(AbstractNativeCell.java:120)
~[apache-cassandra-2.1.10.jar:2.1.10-SNAPSHOT]
at org.apache.cassandra.db.NativeCell.<init>(NativeCell.java:40)
~[apache-cassandra-2.1.10.jar:2.1.10-SNAPSHOT]
at
org.apache.cassandra.utils.memory.NativeAllocator.clone(NativeAllocator.java:72)
~[apache-cassandra-2.1.10.jar:2.1.10-SNAPSHOT]
at org.apache.cassandra.db.NativeCell.localCopy(NativeCell.java:64)
~[apache-cassandra-2.1.10.jar:2.1.10-SNAPSHOT]
at
org.apache.cassandra.db.AtomicBTreeColumns$ColumnUpdater.apply(AtomicBTreeColumns.java:445)
~[apache-cassandra-2.1.10.jar:2.1.10-SNAPSHOT]
at
org.apache.cassandra.db.AtomicBTreeColumns$ColumnUpdater.apply(AtomicBTreeColumns.java:418)
~[apache-cassandra-2.1.10.jar:2.1.10-SNAPSHOT]
at
org.apache.cassandra.utils.btree.NodeBuilder.addNewKey(NodeBuilder.java:322)
~[apache-cassandra-2.1.10.jar:2.1.10-SNAPSHOT]
at
org.apache.cassandra.utils.btree.NodeBuilder.update(NodeBuilder.java:190)
~[apache-cassandra-2.1.10.jar:2.1.10-SNAPSHOT]
at org.apache.cassandra.utils.btree.Builder.update(Builder.java:74)
~[apache-cassandra-2.1.10.jar:2.1.10-SNAPSHOT]
at org.apache.cassandra.utils.btree.BTree.update(BTree.java:186)
~[apache-cassandra-2.1.10.jar:2.1.10-SNAPSHOT]
at
org.apache.cassandra.db.AtomicBTreeColumns.addAllWithSizeDelta(AtomicBTreeColumns.java:225)
~[apache-cassandra-2.1.10.jar:2.1.10-SNAPSHOT]
at org.apache.cassandra.db.Memtable.put(Memtable.java:210)
~[apache-cassandra-2.1.10.jar:2.1.10-SNAPSHOT]
at
org.apache.cassandra.db.ColumnFamilyStore.apply(ColumnFamilyStore.java:1225)
~[apache-cassandra-2.1.10.jar:2.1.10-SNAPSHOT]
at org.apache.cassandra.db.Keyspace.apply(Keyspace.java:396)
~[apache-cassandra-2.1.10.jar:2.1.10-SNAPSHOT]
at org.apache.cassandra.db.Keyspace.apply(Keyspace.java:359)
~[apache-cassandra-2.1.10.jar:2.1.10-SNAPSHOT]
at
org.apache.cassandra.db.commitlog.CommitLogReplayer$1.runMayThrow(CommitLogReplayer.java:455)
~[apache-cassandra-2.1.10.jar:2.1.10-SNAPSHOT]
at
org.apache.cassandra.utils.WrappedRunnable.run(WrappedRunnable.java:28)
~[apache-cassandra-2.1.10.jar:2.1.10-SNAPSHOT]
at
java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
~[na:1.8.0_31]
at
org.apache.cassandra.concurrent.AbstractTracingAwareExecutorService$FutureTask.run(AbstractTracingAwareExecutorService.java:164)
~[apache-cassandra-2.1.10.jar:2.1.10-SNAPSHOT]
at org.apache.cassandra.concurrent.SEPWorker.run(SEPWorker.java:105)
[apache-cassandra-2.1.10.jar:2.1.10-SNAPSHOT]
at java.lang.Thread.run(Thread.java:745) [na:1.8.0_31]
> IndexOutOfBoundsException during memtable flushing at startup (with
> offheap_objects)
> ------------------------------------------------------------------------------------
>
> Key: CASSANDRA-10579
> URL: https://issues.apache.org/jira/browse/CASSANDRA-10579
> Project: Cassandra
> Issue Type: Bug
> Components: Core
> Environment: 2.1.10 on linux
> Reporter: Jeff Griffith
> Assignee: Benedict
> Fix For: 2.1.x
>
>
> Sometimes we have problems at startup where memtable flushes with an index
> out of bounds exception as seen below. Cassandra is then dead in the water
> until we track down the corresponding commit log via the segment ID and
> remove it:
> {code}
> INFO [main] 2015-10-23 14:43:36,440 CommitLogReplayer.java:267 - Replaying
> /home/y/var/cassandra/commitlog/CommitLog-4-1445474832692.log
> INFO [main] 2015-10-23 14:43:36,440 CommitLogReplayer.java:270 - Replaying
> /home/y/var/cassandra/commitlog/CommitLog-4-1445474832692.log (CL version 4,
> messaging version 8)
> INFO [main] 2015-10-23 14:43:36,594 CommitLogReplayer.java:478 - Finished
> reading /home/y/var/cassandra/commitlog/CommitLog-4-1445474832692.log
> INFO [main] 2015-10-23 14:43:36,594 CommitLogReplayer.java:267 - Replaying
> /home/y/var/cassandra/commitlog/CommitLog-4-1445474832693.log
> INFO [main] 2015-10-23 14:43:36,595 CommitLogReplayer.java:270 - Replaying
> /home/y/var/cassandra/commitlog/CommitLog-4-1445474832693.log (CL version 4,
> messaging version 8)
> INFO [main] 2015-10-23 14:43:36,699 CommitLogReplayer.java:478 - Finished
> reading /home/y/var/cassandra/commitlog/CommitLog-4-1445474832693.log
> INFO [main] 2015-10-23 14:43:36,699 CommitLogReplayer.java:267 - Replaying
> /home/y/var/cassandra/commitlog/CommitLog-4-1445474832694.log
> INFO [main] 2015-10-23 14:43:36,699 CommitLogReplayer.java:270 - Replaying
> /home/y/var/cassandra/commitlog/CommitLog-4-1445474832694.log (CL version 4,
> messaging version 8)
> WARN [SharedPool-Worker-5] 2015-10-23 14:43:36,747
> AbstractTracingAwareExecutorService.java:169 - Uncaught exception on thread
> Thread[SharedPool-Worker-5,5,main]: {}
> java.lang.ArrayIndexOutOfBoundsException: 6
> at
> org.apache.cassandra.db.AbstractNativeCell.nametype(AbstractNativeCell.java:204)
> ~[apache-cassandra-2.1.10.jar:2.1.10-SNAPSHOT]
> at
> org.apache.cassandra.db.AbstractNativeCell.isStatic(AbstractNativeCell.java:199)
> ~[apache-cassandra-2.1.10.jar:2.1.10-SNAPSHOT]
> at
> org.apache.cassandra.db.composites.AbstractCType.compare(AbstractCType.java:166)
> ~[apache-cassandra-2.1.10.jar:2.1.10-SNAPSHOT]
> at
> org.apache.cassandra.db.composites.AbstractCellNameType$1.compare(AbstractCellNameType.java:61)
> ~[apache-cassandra-2.1.10.jar:2.1.10-SNAPSHOT]
> at
> org.apache.cassandra.db.composites.AbstractCellNameType$1.compare(AbstractCellNameType.java:58)
> ~[apache-cassandra-2.1.10.jar:2.1.10-SNAPSHOT]
> at org.apache.cassandra.utils.btree.BTree.find(BTree.java:277)
> ~[apache-cassandra-2.1.10.jar:2.1.10-SNAPSHOT]
> at
> org.apache.cassandra.utils.btree.NodeBuilder.update(NodeBuilder.java:154)
> ~[apache-cassandra-2.1.10.jar:2.1.10-SNAPSHOT]
> at org.apache.cassandra.utils.btree.Builder.update(Builder.java:74)
> ~[apache-cassandra-2.1.10.jar:2.1.10-SNAPSHOT]
> at org.apache.cassandra.utils.btree.BTree.update(BTree.java:186)
> ~[apache-cassandra-2.1.10.jar:2.1.10-SNAPSHOT]
> at
> org.apache.cassandra.db.AtomicBTreeColumns.addAllWithSizeDelta(AtomicBTreeColumns.java:225)
> ~[apache-cassandra-2.1.10.jar:2.1.10-SNAPSHOT]
> at org.apache.cassandra.db.Memtable.put(Memtable.java:210)
> ~[apache-cassandra-2.1.10.jar:2.1.10-SNAPSHOT]
> at
> org.apache.cassandra.db.ColumnFamilyStore.apply(ColumnFamilyStore.java:1225)
> ~[apache-cassandra-2.1.10.jar:2.1.10-SNAPSHOT]
> at org.apache.cassandra.db.Keyspace.apply(Keyspace.java:396)
> ~[apache-cassandra-2.1.10.jar:2.1.10-SNAPSHOT]
> at org.apache.cassandra.db.Keyspace.apply(Keyspace.java:359)
> ~[apache-cassandra-2.1.10.jar:2.1.10-SNAPSHOT]
> at
> org.apache.cassandra.db.commitlog.CommitLogReplayer$1.runMayThrow(CommitLogReplayer.java:455)
> ~[apache-cassandra-2.1.10.jar:2.1.10-SNAPSHOT]
> at
> org.apache.cassandra.utils.WrappedRunnable.run(WrappedRunnable.java:28)
> ~[apache-cassandra-2.1.10.jar:2.1.10-SNAPSHOT]
> at
> java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
> ~[na:1.8.0_31]
> at
> org.apache.cassandra.concurrent.AbstractTracingAwareExecutorService$FutureTask.run(AbstractTracingAwareExecutorService.java:164)
> ~[apache-cassandra-2.1.10.jar:2.1.10-SNAPSHOT]
> at org.apache.cassandra.concurrent.SEPWorker.run(SEPWorker.java:105)
> [apache-cassandra-2.1.10.jar:2.1.10-SNAPSHOT]
> at java.lang.Thread.run(Thread.java:745) [na:1.8.0_31]
> {code}
--
This message was sent by Atlassian JIRA
(v6.3.4#6332)
