[
https://issues.apache.org/jira/browse/CASSANDRA-2973?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=13073237#comment-13073237
]
Wojciech Meler edited comment on CASSANDRA-2973 at 8/27/11 11:12 PM:
---------------------------------------------------------------------
Scrub log from 9th node
2011-07-30 21:39:55:386 CEST INFO
[CompactionExecutor:172][org.apache.cassandra.db.compaction.CompactionManager]
Scrubbing
SSTableReader(path='/var/lib/cassandra/data/mail/mta_logs-g-733-Data.db')
2011-07-30 21:39:55:469 CEST WARN
[CompactionExecutor:172][org.apache.cassandra.db.compaction.CompactionManager]
Non-fatal error reading row (stacktrace follows)
java.io.IOError: java.io.IOException: Impossible row size 7933455022436196
at
org.apache.cassandra.db.compaction.CompactionManager.scrubOne(CompactionManager.java:719)
at
org.apache.cassandra.db.compaction.CompactionManager.doScrub(CompactionManager.java:633)
at
org.apache.cassandra.db.compaction.CompactionManager.access$600(CompactionManager.java:65)
at
org.apache.cassandra.db.compaction.CompactionManager$3.call(CompactionManager.java:250)
at java.util.concurrent.FutureTask$Sync.innerRun(FutureTask.java:303)
at java.util.concurrent.FutureTask.run(FutureTask.java:138)
at
java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
at java.lang.Thread.run(Thread.java:662)
Caused by: java.io.IOException: Impossible row size 7933455022436196
... 9 more
2011-07-30 21:39:55:470 CEST INFO
[CompactionExecutor:172][org.apache.cassandra.db.compaction.CompactionManager]
Retrying from row index; data is -8 bytes starting at 547199
2011-07-30 21:39:55:470 CEST WARN
[CompactionExecutor:172][org.apache.cassandra.db.compaction.CompactionManager]
Retry failed too. Skipping to next row (retry's stacktrace follows)
java.io.IOError: java.io.EOFException: bloom filter claims to be 909194543
bytes, longer than entire row size -8
at
org.apache.cassandra.io.sstable.SSTableIdentityIterator.<init>(SSTableIdentityIterator.java:149)
at
org.apache.cassandra.io.sstable.SSTableIdentityIterator.<init>(SSTableIdentityIterator.java:90)
at
org.apache.cassandra.db.compaction.CompactionManager.scrubOne(CompactionManager.java:748)
at
org.apache.cassandra.db.compaction.CompactionManager.doScrub(CompactionManager.java:633)
at
org.apache.cassandra.db.compaction.CompactionManager.access$600(CompactionManager.java:65)
at
org.apache.cassandra.db.compaction.CompactionManager$3.call(CompactionManager.java:250)
at java.util.concurrent.FutureTask$Sync.innerRun(FutureTask.java:303)
at java.util.concurrent.FutureTask.run(FutureTask.java:138)
at
java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
at java.lang.Thread.run(Thread.java:662)
Caused by: java.io.EOFException: bloom filter claims to be 909194543 bytes,
longer than entire row size -8
at
org.apache.cassandra.io.sstable.IndexHelper.defreezeBloomFilter(IndexHelper.java:111)
at
org.apache.cassandra.io.sstable.SSTableIdentityIterator.<init>(SSTableIdentityIterator.java:119)
... 10 more
2011-07-30 21:39:55:482 CEST WARN
[CompactionExecutor:172][org.apache.cassandra.db.compaction.CompactionManager]
Non-fatal error reading row (stacktrace follows)
was (Author: wmeler):
Scrub log from 9th node
2011-07-30 21:39:55:386 CEST INFO
[CompactionExecutor:172][org.apache.cassandra.db.compaction.CompactionManager]
Scrubbing
SSTableReader(path='/var/lib/cassandra/data/mail/mta_logs-g-733-Data.db')
2011-07-30 21:39:55:469 CEST WARN
[CompactionExecutor:172][org.apache.cassandra.db.compaction.CompactionManager]
Non-fatal error reading row (stacktrace follows)
java.io.IOError: java.io.IOException: Impossible row size 7933455022436196
at
org.apache.cassandra.db.compaction.CompactionManager.scrubOne(CompactionManager.java:719)
at
org.apache.cassandra.db.compaction.CompactionManager.doScrub(CompactionManager.java:633)
at
org.apache.cassandra.db.compaction.CompactionManager.access$600(CompactionManager.java:65)
at
org.apache.cassandra.db.compaction.CompactionManager$3.call(CompactionManager.java:250)
at java.util.concurrent.FutureTask$Sync.innerRun(FutureTask.java:303)
at java.util.concurrent.FutureTask.run(FutureTask.java:138)
at
java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
at java.lang.Thread.run(Thread.java:662)
Caused by: java.io.IOException: Impossible row size 7933455022436196
... 9 more
2011-07-30 21:39:55:470 CEST INFO
[CompactionExecutor:172][org.apache.cassandra.db.compaction.CompactionManager]
Retrying from row index; data is -8 bytes starting at 547199
2011-07-30 21:39:55:470 CEST WARN
[CompactionExecutor:172][org.apache.cassandra.db.compaction.CompactionManager]
Retry failed too. Skipping to next row (retry's stacktrace follows)
java.io.IOError: java.io.EOFException: bloom filter claims to be 909194543
bytes, longer than entire row size -8
at
org.apache.cassandra.io.sstable.SSTableIdentityIterator.<init>(SSTableIdentityIterator.java:149)
at
org.apache.cassandra.io.sstable.SSTableIdentityIterator.<init>(SSTableIdentityIterator.java:90)
at
org.apache.cassandra.db.compaction.CompactionManager.scrubOne(CompactionManager.java:748)
at
org.apache.cassandra.db.compaction.CompactionManager.doScrub(CompactionManager.java:633)
at
org.apache.cassandra.db.compaction.CompactionManager.access$600(CompactionManager.java:65)
at
org.apache.cassandra.db.compaction.CompactionManager$3.call(CompactionManager.java:250)
at java.util.concurrent.FutureTask$Sync.innerRun(FutureTask.java:303)
at java.util.concurrent.FutureTask.run(FutureTask.java:138)
at
java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
at java.lang.Thread.run(Thread.java:662)
Caused by: java.io.EOFException: bloom filter claims to be 909194543 bytes,
longer than entire row size -8
at
org.apache.cassandra.io.sstable.IndexHelper.defreezeBloomFilter(IndexHelper.java:111)
at
org.apache.cassandra.io.sstable.SSTableIdentityIterator.<init>(SSTableIdentityIterator.java:119)
... 10 more
2011-07-30 21:39:55:482 CEST WARN
[CompactionExecutor:172][org.apache.cassandra.db.compaction.CompactionManager]
Non-fatal error reading row (stacktrace follows)
# ls -l /var/lib/cassandra/data/mail/mta_logs-g-733*
-rw-r--r-- 1 cassandra cassandra 0 Jul 30 21:42
/var/lib/cassandra/data/mail/mta_logs-g-733-Compacted
-rw-r--r-- 3 cassandra cassandra 8398108921 Jul 29 21:37
/var/lib/cassandra/data/mail/mta_logs-g-733-Data.db
-rw-r--r-- 3 cassandra cassandra 2547136 Jul 29 21:37
/var/lib/cassandra/data/mail/mta_logs-g-733-Filter.db
-rw-r--r-- 3 cassandra cassandra 50895929 Jul 29 21:37
/var/lib/cassandra/data/mail/mta_logs-g-733-Index.db
-rw-r--r-- 3 cassandra cassandra 4276 Jul 29 21:37
/var/lib/cassandra/data/mail/mta_logs-g-733-Statistics.db
# ls -l /var/lib/cassandra/data/mail/snapshots/1312053521562/mta_logs-g-733*
-rw-r--r-- 3 cassandra cassandra 8398108921 Jul 29 21:37
/var/lib/cassandra/data/mail/snapshots/1312053521562/mta_logs-g-733-Data.db
-rw-r--r-- 3 cassandra cassandra 2547136 Jul 29 21:37
/var/lib/cassandra/data/mail/snapshots/1312053521562/mta_logs-g-733-Filter.db
-rw-r--r-- 3 cassandra cassandra 50895929 Jul 29 21:37
/var/lib/cassandra/data/mail/snapshots/1312053521562/mta_logs-g-733-Index.db
-rw-r--r-- 3 cassandra cassandra 4276 Jul 29 21:37
/var/lib/cassandra/data/mail/snapshots/1312053521562/mta_logs-g-733-Statistics.db
# ls -ld /var/lib/cassandra/data/mail/snapshots/1312053521562/
drwxr-xr-x 2 cassandra cassandra 8192 Jul 30 21:18
/var/lib/cassandra/data/mail/snapshots/1312053521562/
Why files in snapshot changed??? isn't it another bug?
> fatal errrors after nodetool cleanup
> ------------------------------------
>
> Key: CASSANDRA-2973
> URL: https://issues.apache.org/jira/browse/CASSANDRA-2973
> Project: Cassandra
> Issue Type: Bug
> Affects Versions: 0.8.1
> Reporter: Wojciech Meler
> Assignee: Sylvain Lebresne
>
> after adding nodes to cluster & running cleanup I get scaring exceptions in
> log:
> 2011-07-30 00:00:05:506 CEST ERROR
> [ReadStage:2335][org.apache.cassandra.service.AbstractCassandraDaemon] Fatal
> exception in thread Thread[ReadStage:2335,5,main]
> java.io.IOError: java.io.IOException: mmap segment underflow; remaining is
> 4394 but 60165 requested
> at
> org.apache.cassandra.db.columniterator.IndexedSliceReader.<init>(IndexedSliceReader.java:80)
> at
> org.apache.cassandra.db.columniterator.SSTableSliceIterator.createReader(SSTableSliceIterator.java:91)
> at
> org.apache.cassandra.db.columniterator.SSTableSliceIterator.<init>(SSTableSliceIterator.java:67)
> at
> org.apache.cassandra.db.filter.SliceQueryFilter.getSSTableColumnIterator(SliceQueryFilter.java:66)
> at
> org.apache.cassandra.db.filter.QueryFilter.getSSTableColumnIterator(QueryFilter.java:80)
> at
> org.apache.cassandra.db.ColumnFamilyStore.getTopLevelColumns(ColumnFamilyStore.java:1292)
> at
> org.apache.cassandra.db.ColumnFamilyStore.getColumnFamily(ColumnFamilyStore.java:1189)
> at
> org.apache.cassandra.db.ColumnFamilyStore.getColumnFamily(ColumnFamilyStore.java:1146)
> at org.apache.cassandra.db.Table.getRow(Table.java:385)
> at
> org.apache.cassandra.db.SliceFromReadCommand.getRow(SliceFromReadCommand.java:61)
> at
> org.apache.cassandra.db.ReadVerbHandler.doVerb(ReadVerbHandler.java:69)
> at
> org.apache.cassandra.net.MessageDeliveryTask.run(MessageDeliveryTask.java:72)
> at
> java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
> at
> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
> at java.lang.Thread.run(Thread.java:662)
> Caused by: java.io.IOException: mmap segment underflow; remaining is 4394
> but 60165 requested
> at
> org.apache.cassandra.io.util.MappedFileDataInput.readBytes(MappedFileDataInput.java:117)
> at
> org.apache.cassandra.utils.ByteBufferUtil.read(ByteBufferUtil.java:389)
> at
> org.apache.cassandra.utils.ByteBufferUtil.readWithShortLength(ByteBufferUtil.java:368)
> at
> org.apache.cassandra.io.sstable.IndexHelper$IndexInfo.deserialize(IndexHelper.java:194)
> at
> org.apache.cassandra.io.sstable.IndexHelper.deserializeIndex(IndexHelper.java:83)
> at
> org.apache.cassandra.db.columniterator.IndexedSliceReader.<init>(IndexedSliceReader.java:73)
> ... 14 more
> exceptions disappeared after running scrub
--
This message is automatically generated by JIRA.
For more information on JIRA, see: http://www.atlassian.com/software/jira
