[
https://issues.apache.org/jira/browse/CASSANDRA-15799?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=17106424#comment-17106424
]
Sumanth Pasupuleti commented on CASSANDRA-15799:
------------------------------------------------
[~dcapwell] So I applied your POC along with CASSANDRA-15373 on top of 3.0.19,
and below is the behavior now
{code:java}
cqlsh> select *from turbo."VersionedCollectionContents";
ReadFailure: Error from server: code=1300 [Replica(s) failed to execute read]
message="Operation failed - received 0 responses and 1 failures"
info={'failures': 1, 'received_responses': 0, 'required_responses': 1,
'consistency': 'ONE'}
{code}
{code:java}
ERROR [SharedPool-Worker-2] 2020-05-13 16:00:19,090
AbstractLocalAwareExecutorService.java:169 - Uncaught exception on thread
Thread[SharedPool-Worker-2,5,main]
java.lang.RuntimeException: org.apache.cassandra.serializers.MarshalException:
Expected 8 or 0 byte long (14394)
at
org.apache.cassandra.service.StorageProxy$DroppableRunnable.run(StorageProxy.java:2470)
~[nf-cassandra-3.0.19.8.jar:3.0.19.8]
at
java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
~[na:1.8.0_231]
at
org.apache.cassandra.concurrent.AbstractLocalAwareExecutorService$FutureTask.run(AbstractLocalAwareExecutorService.java:165)
~[nf-cassandra-3.0.19.8.jar:3.0.19.8]
at
org.apache.cassandra.concurrent.AbstractLocalAwareExecutorService$LocalSessionFutureTask.run(AbstractLocalAwareExecutorService.java:137)
[nf-cassandra-3.0.19.8.jar:3.0.19.8]
at org.apache.cassandra.concurrent.SEPWorker.run(SEPWorker.java:109)
[nf-cassandra-3.0.19.8.jar:3.0.19.8]
at java.lang.Thread.run(Thread.java:748) [na:1.8.0_231]
Caused by: org.apache.cassandra.serializers.MarshalException: Expected 8 or 0
byte long (14394)
at
org.apache.cassandra.serializers.LongSerializer.validate(LongSerializer.java:42)
~[nf-cassandra-3.0.19.8.jar:3.0.19.8]
at
org.apache.cassandra.db.marshal.AbstractType.validate(AbstractType.java:159)
~[nf-cassandra-3.0.19.8.jar:3.0.19.8]
at
org.apache.cassandra.db.marshal.AbstractType.validateIfFixedSize(AbstractType.java:390)
~[nf-cassandra-3.0.19.8.jar:3.0.19.8]
at
org.apache.cassandra.db.LegacyLayout.decodeClustering(LegacyLayout.java:456)
~[nf-cassandra-3.0.19.8.jar:3.0.19.8]
at
org.apache.cassandra.db.LegacyLayout.decodeCellName(LegacyLayout.java:162)
~[nf-cassandra-3.0.19.8.jar:3.0.19.8]
at
org.apache.cassandra.db.LegacyLayout.readLegacyCellBody(LegacyLayout.java:1227)
~[nf-cassandra-3.0.19.8.jar:3.0.19.8]
at
org.apache.cassandra.db.LegacyLayout.readLegacyAtom(LegacyLayout.java:1187)
~[nf-cassandra-3.0.19.8.jar:3.0.19.8]
at
org.apache.cassandra.db.UnfilteredDeserializer$OldFormatDeserializer.readAtom(UnfilteredDeserializer.java:294)
~[nf-cassandra-3.0.19.8.jar:3.0.19.8]
at
org.apache.cassandra.db.UnfilteredDeserializer$OldFormatDeserializer$UnfilteredIterator$AtomIterator.hasNext(UnfilteredDeserializer.java:601)
~[nf-cassandra-3.0.19.8.jar:3.0.19.8]
at
org.apache.cassandra.db.UnfilteredDeserializer$OldFormatDeserializer$UnfilteredIterator.hasNext(UnfilteredDeserializer.java:500)
~[nf-cassandra-3.0.19.8.jar:3.0.19.8]
at
org.apache.cassandra.db.UnfilteredDeserializer$OldFormatDeserializer.hasNext(UnfilteredDeserializer.java:336)
~[nf-cassandra-3.0.19.8.jar:3.0.19.8]
at
org.apache.cassandra.db.columniterator.SSTableIterator$ForwardReader.computeNext(SSTableIterator.java:140)
~[nf-cassandra-3.0.19.8.jar:3.0.19.8]
at
org.apache.cassandra.db.columniterator.SSTableIterator$ForwardReader.hasNextInternal(SSTableIterator.java:172)
~[nf-cassandra-3.0.19.8.jar:3.0.19.8]
at
org.apache.cassandra.db.columniterator.AbstractSSTableIterator$Reader.hasNext(AbstractSSTableIterator.java:336)
~[nf-cassandra-3.0.19.8.jar:3.0.19.8]
at
org.apache.cassandra.db.columniterator.AbstractSSTableIterator.hasNext(AbstractSSTableIterator.java:220)
~[nf-cassandra-3.0.19.8.jar:3.0.19.8]
at
org.apache.cassandra.db.columniterator.SSTableIterator.hasNext(SSTableIterator.java:33)
~[nf-cassandra-3.0.19.8.jar:3.0.19.8]
at
org.apache.cassandra.db.rows.LazilyInitializedUnfilteredRowIterator.computeNext(LazilyInitializedUnfilteredRowIterator.java:95)
~[nf-cassandra-3.0.19.8.jar:3.0.19.8]
at
org.apache.cassandra.db.rows.LazilyInitializedUnfilteredRowIterator.computeNext(LazilyInitializedUnfilteredRowIterator.java:32)
~[nf-cassandra-3.0.19.8.jar:3.0.19.8]
at
org.apache.cassandra.utils.AbstractIterator.hasNext(AbstractIterator.java:47)
~[nf-cassandra-3.0.19.8.jar:3.0.19.8]
at
org.apache.cassandra.db.transform.BaseRows.hasNext(BaseRows.java:129)
~[nf-cassandra-3.0.19.8.jar:3.0.19.8]
at
org.apache.cassandra.db.rows.LazilyInitializedUnfilteredRowIterator.computeNext(LazilyInitializedUnfilteredRowIterator.java:95)
~[nf-cassandra-3.0.19.8.jar:3.0.19.8]
at
org.apache.cassandra.db.rows.LazilyInitializedUnfilteredRowIterator.computeNext(LazilyInitializedUnfilteredRowIterator.java:32)
~[nf-cassandra-3.0.19.8.jar:3.0.19.8]
at
org.apache.cassandra.utils.AbstractIterator.hasNext(AbstractIterator.java:47)
~[nf-cassandra-3.0.19.8.jar:3.0.19.8]
at
org.apache.cassandra.db.transform.BaseRows.hasNext(BaseRows.java:129)
~[nf-cassandra-3.0.19.8.jar:3.0.19.8]
at
org.apache.cassandra.db.transform.UnfilteredRows.isEmpty(UnfilteredRows.java:71)
~[nf-cassandra-3.0.19.8.jar:3.0.19.8]
at
org.apache.cassandra.db.partitions.PurgeFunction.applyToPartition(PurgeFunction.java:75)
~[nf-cassandra-3.0.19.8.jar:3.0.19.8]
at
org.apache.cassandra.db.partitions.PurgeFunction.applyToPartition(PurgeFunction.java:26)
~[nf-cassandra-3.0.19.8.jar:3.0.19.8]
at
org.apache.cassandra.db.transform.BasePartitions.hasNext(BasePartitions.java:96)
~[nf-cassandra-3.0.19.8.jar:3.0.19.8]
at
org.apache.cassandra.db.partitions.UnfilteredPartitionIterators$Serializer.serialize(UnfilteredPartitionIterators.java:289)
~[nf-cassandra-3.0.19.8.jar:3.0.19.8]
at
org.apache.cassandra.db.ReadResponse$LocalDataResponse.build(ReadResponse.java:187)
~[nf-cassandra-3.0.19.8.jar:3.0.19.8]
at
org.apache.cassandra.db.ReadResponse$LocalDataResponse.<init>(ReadResponse.java:180)
~[nf-cassandra-3.0.19.8.jar:3.0.19.8]
at
org.apache.cassandra.db.ReadResponse$LocalDataResponse.<init>(ReadResponse.java:176)
~[nf-cassandra-3.0.19.8.jar:3.0.19.8]
at
org.apache.cassandra.db.ReadResponse.createDataResponse(ReadResponse.java:76)
~[nf-cassandra-3.0.19.8.jar:3.0.19.8]
at
org.apache.cassandra.db.ReadCommand.createResponse(ReadCommand.java:341)
~[nf-cassandra-3.0.19.8.jar:3.0.19.8]
at
org.apache.cassandra.service.StorageProxy$LocalReadRunnable.runMayThrow(StorageProxy.java:1785)
~[nf-cassandra-3.0.19.8.jar:3.0.19.8]
at
org.apache.cassandra.service.StorageProxy$DroppableRunnable.run(StorageProxy.java:2466)
~[nf-cassandra-3.0.19.8.jar:3.0.19.8]
... 5 common frames omitted
{code}
> CorruptSSTableException when compacting a 3.0 format sstable that was
> originally created as an outcome of 2.1 sstable upgrade
> -----------------------------------------------------------------------------------------------------------------------------
>
> Key: CASSANDRA-15799
> URL: https://issues.apache.org/jira/browse/CASSANDRA-15799
> Project: Cassandra
> Issue Type: Bug
> Components: Local/Compaction, Local/SSTable
> Reporter: Sumanth Pasupuleti
> Assignee: David Capwell
> Priority: Normal
> Fix For: 3.0.x
>
>
> Below is the exception with stack trace. This issue is reproduce-able.
> {code:java}
> DEBUG [CompactionExecutor:10] 2020-05-07 19:33:34,268 CompactionTask.java:158
> - Compacting (a3ea9fc0-9099-11ea-933f-c5e852f71338)
> [/mnt/data/cassandra/data/ks/cf/md-10802-big-Data.db:level=0, ]
> ERROR [CompactionExecutor:10] 2020-05-07 19:33:34,275
> CassandraDaemon.java:208 - Exception in thread
> Thread[CompactionExecutor:10,1,RMI Runtime]
> org.apache.cassandra.io.sstable.CorruptSSTableException: Corrupted:
> /mnt/data/cassandra/data/ks/cf/md-10802-big-Data.db
> at
> org.apache.cassandra.io.sstable.SSTableIdentityIterator.computeNext(SSTableIdentityIterator.java:105)
> ~[nf-cassandra-3.0.19.8.jar:3.0.19.8]
> at
> org.apache.cassandra.io.sstable.SSTableIdentityIterator.computeNext(SSTableIdentityIterator.java:30)
> ~[nf-cassandra-3.0.19.8.jar:3.0.19.8]
> at
> org.apache.cassandra.utils.AbstractIterator.hasNext(AbstractIterator.java:47)
> ~[nf-cassandra-3.0.19.8.jar:3.0.19.8]
> at
> org.apache.cassandra.db.rows.LazilyInitializedUnfilteredRowIterator.computeNext(LazilyInitializedUnfilteredRowIterator.java:95)
> ~[nf-cassandra-3.0.19.8.jar:3.0.19.8]
> at
> org.apache.cassandra.db.rows.LazilyInitializedUnfilteredRowIterator.computeNext(LazilyInitializedUnfilteredRowIterator.java:32)
> ~[nf-cassandra-3.0.19.8.jar:3.0.19.8]
> at
> org.apache.cassandra.utils.AbstractIterator.hasNext(AbstractIterator.java:47)
> ~[nf-cassandra-3.0.19.8.jar:3.0.19.8]
> at
> org.apache.cassandra.utils.MergeIterator$TrivialOneToOne.computeNext(MergeIterator.java:460)
> ~[nf-cassandra-3.0.19.8.jar:3.0.19.8]
> at
> org.apache.cassandra.utils.AbstractIterator.hasNext(AbstractIterator.java:47)
> ~[nf-cassandra-3.0.19.8.jar:3.0.19.8]
> at
> org.apache.cassandra.db.rows.UnfilteredRowIterators$UnfilteredRowMergeIterator.computeNext(UnfilteredRowIterators.java:534)
> ~[nf-cassandra-3.0.19.8.jar:3.0.19.8]
> at
> org.apache.cassandra.db.rows.UnfilteredRowIterators$UnfilteredRowMergeIterator.computeNext(UnfilteredRowIterators.java:394)
> ~[nf-cassandra-3.0.19.8.jar:3.0.19.8]
> at
> org.apache.cassandra.utils.AbstractIterator.hasNext(AbstractIterator.java:47)
> ~[nf-cassandra-3.0.19.8.jar:3.0.19.8]
> at
> org.apache.cassandra.db.transform.BaseRows.hasNext(BaseRows.java:129)
> ~[nf-cassandra-3.0.19.8.jar:3.0.19.8]
> at
> org.apache.cassandra.db.ColumnIndex$Builder.build(ColumnIndex.java:111)
> ~[nf-cassandra-3.0.19.8.jar:3.0.19.8]
> at
> org.apache.cassandra.db.ColumnIndex.writeAndBuildIndex(ColumnIndex.java:52)
> ~[nf-cassandra-3.0.19.8.jar:3.0.19.8]
> at
> org.apache.cassandra.io.sstable.format.big.BigTableWriter.append(BigTableWriter.java:165)
> ~[nf-cassandra-3.0.19.8.jar:3.0.19.8]
> at
> org.apache.cassandra.io.sstable.SSTableRewriter.append(SSTableRewriter.java:126)
> ~[nf-cassandra-3.0.19.8.jar:3.0.19.8]
> at
> org.apache.cassandra.db.compaction.writers.DefaultCompactionWriter.realAppend(DefaultCompactionWriter.java:57)
> ~[nf-cassandra-3.0.19.8.jar:3.0.19.8]
> at
> org.apache.cassandra.db.compaction.writers.CompactionAwareWriter.append(CompactionAwareWriter.java:109)
> ~[nf-cassandra-3.0.19.8.jar:3.0.19.8]
> at
> org.apache.cassandra.db.compaction.CompactionTask.runMayThrow(CompactionTask.java:195)
> ~[nf-cassandra-3.0.19.8.jar:3.0.19.8]
> at
> org.apache.cassandra.utils.WrappedRunnable.run(WrappedRunnable.java:28)
> ~[nf-cassandra-3.0.19.8.jar:3.0.19.8]
> at
> org.apache.cassandra.db.compaction.CompactionTask.executeInternal(CompactionTask.java:89)
> ~[nf-cassandra-3.0.19.8.jar:3.0.19.8]
> at
> org.apache.cassandra.db.compaction.AbstractCompactionTask.execute(AbstractCompactionTask.java:61)
> ~[nf-cassandra-3.0.19.8.jar:3.0.19.8]
> at
> org.apache.cassandra.db.compaction.CompactionManager$8.runMayThrow(CompactionManager.java:675)
> ~[nf-cassandra-3.0.19.8.jar:3.0.19.8]
> at
> org.apache.cassandra.utils.WrappedRunnable.run(WrappedRunnable.java:28)
> ~[nf-cassandra-3.0.19.8.jar:3.0.19.8]
> at
> java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
> ~[na:1.8.0_231]
> at java.util.concurrent.FutureTask.run(FutureTask.java:266)
> ~[na:1.8.0_231]
> at
> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
> ~[na:1.8.0_231]
> at
> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
> [na:1.8.0_231]
> at
> org.apache.cassandra.concurrent.NamedThreadFactory.lambda$threadLocalDeallocator$0(NamedThreadFactory.java:83)
> [nf-cassandra-3.0.19.8.jar:3.0.19.8]
> at java.lang.Thread.run(Thread.java:748) ~[na:1.8.0_231]
> Caused by: java.lang.ArrayIndexOutOfBoundsException: 119
> at
> org.apache.cassandra.db.RangeTombstone$Bound$Serializer.deserialize(RangeTombstone.java:201)
> ~[nf-cassandra-3.0.19.8.jar:3.0.19.8]
> at
> org.apache.cassandra.db.rows.UnfilteredSerializer.deserializeOne(UnfilteredSerializer.java:407)
> ~[nf-cassandra-3.0.19.8.jar:3.0.19.8]
> at
> org.apache.cassandra.db.rows.UnfilteredSerializer.deserialize(UnfilteredSerializer.java:373)
> ~[nf-cassandra-3.0.19.8.jar:3.0.19.8]
> at
> org.apache.cassandra.io.sstable.SSTableSimpleIterator$CurrentFormatIterator.computeNext(SSTableSimpleIterator.java:87)
> ~[nf-cassandra-3.0.19.8.jar:3.0.19.8]
> at
> org.apache.cassandra.io.sstable.SSTableSimpleIterator$CurrentFormatIterator.computeNext(SSTableSimpleIterator.java:65)
> ~[nf-cassandra-3.0.19.8.jar:3.0.19.8]
> at
> org.apache.cassandra.utils.AbstractIterator.hasNext(AbstractIterator.java:47)
> ~[nf-cassandra-3.0.19.8.jar:3.0.19.8]
> at
> org.apache.cassandra.io.sstable.SSTableIdentityIterator.doCompute(SSTableIdentityIterator.java:123)
> ~[nf-cassandra-3.0.19.8.jar:3.0.19.8]
> at
> org.apache.cassandra.io.sstable.SSTableIdentityIterator.computeNext(SSTableIdentityIterator.java:100)
> ~[nf-cassandra-3.0.19.8.jar:3.0.19.8]
> ... 29 common frames omitted
> {code}
> Below are the logs that indicate the origin of this sstable
> {code:java}
> DEBUG [CompactionExecutor:7] 2020-05-07 18:39:59,235 CompactionTask.java:158
> - Compacting (279b8530-9092-11ea-933f-c5e852f71338)
> [/mnt/data/cassandra/data/ks/cf/ks-cf-ka-10799-Data.db:level=0,
> /mnt/data/cassandra/data/ks/cf/ks-cf-ka-10800-Data.db:level=0,
> /mnt/data/cassandra/data/ks/cf/ks-cf-ka-10801-Data.db:level=0, ]
> DEBUG [CompactionExecutor:7] 2020-05-07 18:40:01,053 CompactionTask.java:235
> - Compacted (279b8530-9092-11ea-933f-c5e852f71338) 3 sstables to
> [/mnt/data/cassandra/data/ks/cf/md-10802-big,] to level=0. 35,435,268 bytes
> to 35,317,747 (~99% of original) in 1,817ms = 18.536945MB/s. 789 total
> partitions merged to 772. Partition merge counts were {1:786, 3:1, }
> {code}
--
This message was sent by Atlassian Jira
(v8.3.4#803005)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
