[ https://issues.apache.org/jira/browse/CASSANDRA-7775?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=14098361#comment-14098361 ]
Omri Bahumi commented on CASSANDRA-7775: ---------------------------------------- Here's another manifestation of this bug, this time with the startup exception: {code} INFO [OptionalTasks:1] 2014-08-15 04:31:56,372 ColumnFamilyStore.java (line 785) Enqueuing flush of Memtable-app_recs_best_in_expr_prefix2@1878214183(0/0 serialized/live bytes, 0 ops) INFO [FlushWriter:38] 2014-08-15 04:31:56,373 Memtable.java (line 331) Writing Memtable-app_recs_best_in_expr_prefix2@1878214183(0/0 serialized/live bytes, 0 ops) ERROR [FlushWriter:38] 2014-08-15 04:31:56,380 CassandraDaemon.java (line 196) Exception in thread Thread[FlushWriter:38,5,main] java.lang.RuntimeException: Cannot get comparator 1 in org.apache.cassandra.db.marshal.CompositeType(org.apache.cassandra.db.marshal.UTF8Type). This might due to a mismatch between the schema and the data read at org.apache.cassandra.db.marshal.CompositeType.getComparator(CompositeType.java:133) at org.apache.cassandra.db.marshal.CompositeType.getComparator(CompositeType.java:140) at org.apache.cassandra.db.marshal.AbstractCompositeType.compare(AbstractCompositeType.java:96) at org.apache.cassandra.db.marshal.AbstractCompositeType.compare(AbstractCompositeType.java:35) at org.apache.cassandra.db.RangeTombstone$Tracker$1.compare(RangeTombstone.java:125) at org.apache.cassandra.db.RangeTombstone$Tracker$1.compare(RangeTombstone.java:122) at java.util.TreeMap.compare(TreeMap.java:1188) at java.util.TreeMap$NavigableSubMap.<init>(TreeMap.java:1264) at java.util.TreeMap$AscendingSubMap.<init>(TreeMap.java:1699) at java.util.TreeMap.tailMap(TreeMap.java:905) at java.util.TreeSet.tailSet(TreeSet.java:350) at java.util.TreeSet.tailSet(TreeSet.java:383) at org.apache.cassandra.db.RangeTombstone$Tracker.update(RangeTombstone.java:203) at org.apache.cassandra.db.ColumnIndex$Builder.add(ColumnIndex.java:192) at org.apache.cassandra.db.ColumnIndex$Builder.build(ColumnIndex.java:138) at org.apache.cassandra.io.sstable.SSTableWriter.rawAppend(SSTableWriter.java:202) at org.apache.cassandra.io.sstable.SSTableWriter.append(SSTableWriter.java:187) at org.apache.cassandra.db.Memtable$FlushRunnable.writeSortedContents(Memtable.java:365) at org.apache.cassandra.db.Memtable$FlushRunnable.runWith(Memtable.java:318) at org.apache.cassandra.io.util.DiskAwareRunnable.runMayThrow(DiskAwareRunnable.java:48) at org.apache.cassandra.utils.WrappedRunnable.run(WrappedRunnable.java:28) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) at java.lang.Thread.run(Thread.java:744) Caused by: java.lang.IndexOutOfBoundsException: index (1) must be less than size (1) at com.google.common.base.Preconditions.checkElementIndex(Preconditions.java:306) at com.google.common.base.Preconditions.checkElementIndex(Preconditions.java:285) at com.google.common.collect.SingletonImmutableList.get(SingletonImmutableList.java:45) at org.apache.cassandra.db.marshal.CompositeType.getComparator(CompositeType.java:124) ... 23 more {code} Startup exception: {code} INFO [main] 2014-08-15 08:52:53,796 CommitLog.java (line 130) Replaying /var/lib/cassandra/commitlog/CommitLog-3-1408027485034.log, /var/lib/cassandra/commitlog/CommitLog-3-1408027485035.log, /var/lib/cassandra/commitlog/CommitLog-3-1408027485040.log, /var/lib/cassandra/commitlog/CommitLog-3-1408027485041.log, /var/lib/cassandra/commitlog/CommitLog-3-1408027485042.log, /var/lib/cassandra/commitlog/CommitLog-3-1408027485043.log, /var/lib/cassandra/commitlog/CommitLog-3-1408027485044.log, /var/lib/cassandra/commitlog/CommitLog-3-1408027485045.log, /var/lib/cassandra/commitlog/CommitLog-3-1408027485046.log, /var/lib/cassandra/commitlog/CommitLog-3-1408027485047.log, /var/lib/cassandra/commitlog/CommitLog-3-1408027485048.log, /var/lib/cassandra/commitlog/CommitLog-3-1408027485049.log, /var/lib/cassandra/commitlog/CommitLog-3-1408027485050.log, /var/lib/cassandra/commitlog/CommitLog-3-1408027485051.log, /var/lib/cassandra/commitlog/CommitLog-3-1408027485052.log, /var/lib/cassandra/commitlog/CommitLog-3-1408027485053.log, /var/lib/cassandra/commitlog/CommitLog-3-1408027485054.log, /var/lib/cassandra/commitlog/CommitLog-3-1408027485055.log, /var/lib/cassandra/commitlog/CommitLog-3-1408027485056.log, /var/lib/cassandra/commitlog/CommitLog-3-1408027485057.log, /var/lib/cassandra/commitlog/CommitLog-3-1408027485058.log, /var/lib/cassandra/commitlog/CommitLog-3-1408027485059.log, /var/lib/cassandra/commitlog/CommitLog-3-1408027485060.log, /var/lib/cassandra/commitlog/CommitLog-3-1408027485061.log, /var/lib/cassandra/commitlog/CommitLog-3-1408027485062.log, /var/lib/cassandra/commitlog/CommitLog-3-1408027485063.log, /var/lib/cassandra/commitlog/CommitLog-3-1408027485064.log, /var/lib/cassandra/commitlog/CommitLog-3-1408027485065.log, /var/lib/cassandra/commitlog/CommitLog-3-1408027485066.log, /var/lib/cassandra/commitlog/CommitLog-3-1408027485067.log, /var/lib/cassandra/commitlog/CommitLog-3-1408027485068.log, /var/lib/cassandra/commitlog/CommitLog-3-1408027485069.log, /var/lib/cassandra/commitlog/CommitLog-3-1408027485070.log, /var/lib/cassandra/commitlog/CommitLog-3-1408027485071.log, /var/lib/cassandra/commitlog/CommitLog-3-1408027485072.log, /var/lib/cassandra/commitlog/CommitLog-3-1408027485073.log, /var/lib/cassandra/commitlog/CommitLog-3-1408027485074.log, /var/lib/cassandra/commitlog/CommitLog-3-1408027485075.log, /var/lib/cassandra/commitlog/CommitLog-3-1408027485076.log, /var/lib/cassandra/commitlog/CommitLog-3-1408027485077.log, /var/lib/cassandra/commitlog/CommitLog-3-1408027485078.log, /var/lib/cassandra/commitlog/CommitLog-3-1408027485079.log INFO [main] 2014-08-15 08:52:53,822 CommitLogReplayer.java (line 184) Replaying /var/lib/cassandra/commitlog/CommitLog-3-1408027485034.log (CL version 3, messaging version 7) INFO [CompactionExecutor:1] 2014-08-15 08:52:53,931 CompactionTask.java (line 275) Compacted 4 sstables to [/var/lib/cassandra/data/system/local/system-local-jb-1152,]. 6,090 bytes to 5,796 (~95% of original) in 2,077ms = 0.002661MB/s. 4 total partitions merged to 1. Partition merge counts were {4:1, } INFO [main] 2014-08-15 08:52:55,246 CommitLogReplayer.java (line 368) Finished reading /var/lib/cassandra/commitlog/CommitLog-3-1408027485034.log ERROR [MutationStage:9] 2014-08-15 08:52:55,247 CassandraDaemon.java (line 196) Exception in thread Thread[MutationStage:9,5,main] java.lang.RuntimeException: Cannot get comparator 1 in org.apache.cassandra.db.marshal.CompositeType(org.apache.cassandra.db.marshal.UTF8Type). This might due to a mismatch between the schema and the data read at org.apache.cassandra.db.marshal.CompositeType.getComparator(CompositeType.java:133) at org.apache.cassandra.db.marshal.CompositeType.getComparator(CompositeType.java:140) at org.apache.cassandra.db.marshal.AbstractCompositeType.compare(AbstractCompositeType.java:96) at org.apache.cassandra.db.marshal.AbstractCompositeType.compare(AbstractCompositeType.java:35) at org.apache.cassandra.db.RangeTombstoneList.insertFrom(RangeTombstoneList.java:455) at org.apache.cassandra.db.RangeTombstoneList.addAll(RangeTombstoneList.java:190) at org.apache.cassandra.db.DeletionInfo.add(DeletionInfo.java:219) at org.apache.cassandra.db.AtomicSortedColumns.addAllWithSizeDelta(AtomicSortedColumns.java:184) at org.apache.cassandra.db.Memtable.resolve(Memtable.java:194) at org.apache.cassandra.db.Memtable.put(Memtable.java:158) at org.apache.cassandra.db.ColumnFamilyStore.apply(ColumnFamilyStore.java:891) at org.apache.cassandra.db.Keyspace.apply(Keyspace.java:368) at org.apache.cassandra.db.Keyspace.apply(Keyspace.java:333) at org.apache.cassandra.db.commitlog.CommitLogReplayer$1.runMayThrow(CommitLogReplayer.java:352) at org.apache.cassandra.utils.WrappedRunnable.run(WrappedRunnable.java:28) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471) at java.util.concurrent.FutureTask.run(FutureTask.java:262) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) at java.lang.Thread.run(Thread.java:744) Caused by: java.lang.IndexOutOfBoundsException: index (1) must be less than size (1) at com.google.common.base.Preconditions.checkElementIndex(Preconditions.java:306) at com.google.common.base.Preconditions.checkElementIndex(Preconditions.java:285) at com.google.common.collect.SingletonImmutableList.get(SingletonImmutableList.java:45) at org.apache.cassandra.db.marshal.CompositeType.getComparator(CompositeType.java:124) ... 19 more ERROR [main] 2014-08-15 08:52:55,251 CassandraDaemon.java (line 488) Exception encountered during startup java.lang.RuntimeException: java.util.concurrent.ExecutionException: java.lang.RuntimeException: Cannot get comparator 1 in org.apache.cassandra.db.marshal.CompositeType(org.apache.cassandra.db.marshal.UTF8Type). This might due to a mismatch between the schema and the data read at org.apache.cassandra.utils.FBUtilities.waitOnFuture(FBUtilities.java:413) at org.apache.cassandra.utils.FBUtilities.waitOnFutures(FBUtilities.java:402) at org.apache.cassandra.db.commitlog.CommitLogReplayer.recover(CommitLogReplayer.java:360) at org.apache.cassandra.db.commitlog.CommitLogReplayer.recover(CommitLogReplayer.java:97) at org.apache.cassandra.db.commitlog.CommitLog.recover(CommitLog.java:151) at org.apache.cassandra.db.commitlog.CommitLog.recover(CommitLog.java:131) at org.apache.cassandra.service.CassandraDaemon.setup(CassandraDaemon.java:312) at org.apache.cassandra.service.CassandraDaemon.activate(CassandraDaemon.java:471) at org.apache.cassandra.service.CassandraDaemon.main(CassandraDaemon.java:560) Caused by: java.util.concurrent.ExecutionException: java.lang.RuntimeException: Cannot get comparator 1 in org.apache.cassandra.db.marshal.CompositeType(org.apache.cassandra.db.marshal.UTF8Type). This might due to a mismatch between the schema and the data read at java.util.concurrent.FutureTask.report(FutureTask.java:122) at java.util.concurrent.FutureTask.get(FutureTask.java:188) at org.apache.cassandra.utils.FBUtilities.waitOnFuture(FBUtilities.java:409) ... 8 more Caused by: java.lang.RuntimeException: Cannot get comparator 1 in org.apache.cassandra.db.marshal.CompositeType(org.apache.cassandra.db.marshal.UTF8Type). This might due to a mismatch between the schema and the data read at org.apache.cassandra.db.marshal.CompositeType.getComparator(CompositeType.java:133) at org.apache.cassandra.db.marshal.CompositeType.getComparator(CompositeType.java:140) at org.apache.cassandra.db.marshal.AbstractCompositeType.compare(AbstractCompositeType.java:96) at org.apache.cassandra.db.marshal.AbstractCompositeType.compare(AbstractCompositeType.java:35) at org.apache.cassandra.db.RangeTombstoneList.insertFrom(RangeTombstoneList.java:455) at org.apache.cassandra.db.RangeTombstoneList.addAll(RangeTombstoneList.java:190) at org.apache.cassandra.db.DeletionInfo.add(DeletionInfo.java:219) at org.apache.cassandra.db.AtomicSortedColumns.addAllWithSizeDelta(AtomicSortedColumns.java:184) at org.apache.cassandra.db.Memtable.resolve(Memtable.java:194) at org.apache.cassandra.db.Memtable.put(Memtable.java:158) at org.apache.cassandra.db.ColumnFamilyStore.apply(ColumnFamilyStore.java:891) at org.apache.cassandra.db.Keyspace.apply(Keyspace.java:368) at org.apache.cassandra.db.Keyspace.apply(Keyspace.java:333) at org.apache.cassandra.db.commitlog.CommitLogReplayer$1.runMayThrow(CommitLogReplayer.java:352) at org.apache.cassandra.utils.WrappedRunnable.run(WrappedRunnable.java:28) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471) at java.util.concurrent.FutureTask.run(FutureTask.java:262) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) at java.lang.Thread.run(Thread.java:744) Caused by: java.lang.IndexOutOfBoundsException: index (1) must be less than size (1) at com.google.common.base.Preconditions.checkElementIndex(Preconditions.java:306) at com.google.common.base.Preconditions.checkElementIndex(Preconditions.java:285) at com.google.common.collect.SingletonImmutableList.get(SingletonImmutableList.java:45) at org.apache.cassandra.db.marshal.CompositeType.getComparator(CompositeType.java:124) ... 19 more {code} Startup exception when running in DEBUG logging level: {code} DEBUG [main] 2014-08-15 08:56:02,159 CommitLogReplayer.java (line 215) Reading mutation at 13646092 INFO [main] 2014-08-15 08:56:02,162 CommitLogReplayer.java (line 368) Finished reading /var/lib/cassandra/commitlog/CommitLog-3-1408027485034.log ERROR [main] 2014-08-15 08:56:02,162 CassandraDaemon.java (line 488) Exception encountered during startup org.apache.cassandra.serializers.MarshalException: String didn't validate. at org.apache.cassandra.serializers.UTF8Serializer.validate(UTF8Serializer.java:35) at org.apache.cassandra.db.marshal.AbstractType.getString(AbstractType.java:154) at org.apache.cassandra.db.marshal.AbstractCompositeType.getString(AbstractCompositeType.java:242) at org.apache.cassandra.db.DeletionInfo.rangesAsString(DeletionInfo.java:308) at org.apache.cassandra.db.DeletionInfo.toString(DeletionInfo.java:294) at java.lang.String.valueOf(String.java:2854) at java.lang.StringBuilder.append(StringBuilder.java:128) at org.apache.cassandra.db.ColumnFamily.toString(ColumnFamily.java:373) at org.apache.commons.lang3.ObjectUtils.toString(ObjectUtils.java:303) at org.apache.commons.lang3.StringUtils.join(StringUtils.java:3474) at org.apache.cassandra.db.commitlog.CommitLogReplayer.recover(CommitLogReplayer.java:311) at org.apache.cassandra.db.commitlog.CommitLogReplayer.recover(CommitLogReplayer.java:97) at org.apache.cassandra.db.commitlog.CommitLog.recover(CommitLog.java:151) at org.apache.cassandra.db.commitlog.CommitLog.recover(CommitLog.java:131) at org.apache.cassandra.service.CassandraDaemon.setup(CassandraDaemon.java:312) at org.apache.cassandra.service.CassandraDaemon.activate(CassandraDaemon.java:471) at org.apache.cassandra.service.CassandraDaemon.main(CassandraDaemon.java:560) {code} > Cassandra attempts to flush an empty memtable into disk and fails > ----------------------------------------------------------------- > > Key: CASSANDRA-7775 > URL: https://issues.apache.org/jira/browse/CASSANDRA-7775 > Project: Cassandra > Issue Type: Bug > Components: Core > Environment: $ nodetool version > ReleaseVersion: 2.0.6 > $ java -version > java version "1.7.0_51" > Java(TM) SE Runtime Environment (build 1.7.0_51-b13) > Java HotSpot(TM) 64-Bit Server VM (build 24.51-b03, mixed mode) > Reporter: Omri Bahumi > > I'm not sure what triggers this flush, but when it happens the following > appears in our logs: > {code} > INFO [OptionalTasks:1] 2014-08-15 02:24:20,115 ColumnFamilyStore.java (line > 785) Enqueuing flush of Memtable-app_recs_best_in_expr_prefix2@1219170646(0/0 > serialized/live bytes, 0 ops) > INFO [FlushWriter:34] 2014-08-15 02:24:20,116 Memtable.java (line 331) > Writing Memtable-app_recs_best_in_expr_prefix2@1219170646(0/0 serialized/live > bytes, 0 ops) > ERROR [FlushWriter:34] 2014-08-15 02:24:20,127 CassandraDaemon.java (line > 196) Exception in thread Thread[FlushWriter:34,5,main] > java.lang.RuntimeException: Cannot get comparator 1 in > org.apache.cassandra.db.marshal.CompositeType(org.apache.cassandra.db.marshal.UTF8Type). > This might due to a mismatch between the schema and the data read > at > org.apache.cassandra.db.marshal.CompositeType.getComparator(CompositeType.java:133) > at > org.apache.cassandra.db.marshal.CompositeType.getComparator(CompositeType.java:140) > at > org.apache.cassandra.db.marshal.AbstractCompositeType.compare(AbstractCompositeType.java:96) > at > org.apache.cassandra.db.marshal.AbstractCompositeType.compare(AbstractCompositeType.java:35) > at > org.apache.cassandra.db.RangeTombstone$Tracker$1.compare(RangeTombstone.java:125) > at > org.apache.cassandra.db.RangeTombstone$Tracker$1.compare(RangeTombstone.java:122) > at java.util.TreeMap.compare(TreeMap.java:1188) > at java.util.TreeMap$NavigableSubMap.<init>(TreeMap.java:1264) > at java.util.TreeMap$AscendingSubMap.<init>(TreeMap.java:1699) > at java.util.TreeMap.tailMap(TreeMap.java:905) > at java.util.TreeSet.tailSet(TreeSet.java:350) > at java.util.TreeSet.tailSet(TreeSet.java:383) > at > org.apache.cassandra.db.RangeTombstone$Tracker.update(RangeTombstone.java:203) > at > org.apache.cassandra.db.ColumnIndex$Builder.add(ColumnIndex.java:192) > at > org.apache.cassandra.db.ColumnIndex$Builder.build(ColumnIndex.java:138) > at > org.apache.cassandra.io.sstable.SSTableWriter.rawAppend(SSTableWriter.java:202) > at > org.apache.cassandra.io.sstable.SSTableWriter.append(SSTableWriter.java:187) > at > org.apache.cassandra.db.Memtable$FlushRunnable.writeSortedContents(Memtable.java:365) > at > org.apache.cassandra.db.Memtable$FlushRunnable.runWith(Memtable.java:318) > at > org.apache.cassandra.io.util.DiskAwareRunnable.runMayThrow(DiskAwareRunnable.java:48) > at > org.apache.cassandra.utils.WrappedRunnable.run(WrappedRunnable.java:28) > at > java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) > at > java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) > at java.lang.Thread.run(Thread.java:744) > Caused by: java.lang.IndexOutOfBoundsException: index (1) must be less than > size (1) > at > com.google.common.base.Preconditions.checkElementIndex(Preconditions.java:306) > at > com.google.common.base.Preconditions.checkElementIndex(Preconditions.java:285) > at > com.google.common.collect.SingletonImmutableList.get(SingletonImmutableList.java:45) > at > org.apache.cassandra.db.marshal.CompositeType.getComparator(CompositeType.java:124) > ... 23 more > {code} > After this happens, the MemtablePostFlusher thread pool starts piling up. > When trying to restart the cluster, a similar exception occurs when trying to > replay the commit log. > Our way of recovering from this is to delete all commit logs in the faulty > node, start it and issue a repair. -- This message was sent by Atlassian JIRA (v6.2#6252)