[
https://issues.apache.org/jira/browse/CASSANDRA-10102?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=14700110#comment-14700110
]
Russ Hatch commented on CASSANDRA-10102:
----------------------------------------
[~iamaleksey] I'm still seeing some kind of issue post-upgrade on 3.0 HEAD, not
sure if it's the same problem or not:
one node shows:
{noformat}
ERROR [SharedPool-Worker-1] 2015-08-17 13:38:03,311 Message.java:611 -
Unexpected exception during request; channel = [id: 0x68fac00f,
/127.0.0.1:57115 => /127.0.0.1:9042]
java.lang.AssertionError: null
at
org.apache.cassandra.db.ReadCommand$Serializer.serializedSize(ReadCommand.java:520)
~[main/:na]
at
org.apache.cassandra.db.ReadCommand$Serializer.serializedSize(ReadCommand.java:461)
~[main/:na]
at org.apache.cassandra.net.MessageOut.payloadSize(MessageOut.java:166)
~[main/:na]
at
org.apache.cassandra.net.OutboundTcpConnectionPool.getConnection(OutboundTcpConnectionPool.java:72)
~[main/:na]
at
org.apache.cassandra.net.MessagingService.getConnection(MessagingService.java:583)
~[main/:na]
at
org.apache.cassandra.net.MessagingService.sendOneWay(MessagingService.java:733)
~[main/:na]
at
org.apache.cassandra.net.MessagingService.sendRR(MessagingService.java:676)
~[main/:na]
at
org.apache.cassandra.net.MessagingService.sendRRWithFailure(MessagingService.java:659)
~[main/:na]
at
org.apache.cassandra.service.AbstractReadExecutor.makeRequests(AbstractReadExecutor.java:103)
~[main/:na]
at
org.apache.cassandra.service.AbstractReadExecutor.makeDataRequests(AbstractReadExecutor.java:76)
~[main/:na]
at
org.apache.cassandra.service.AbstractReadExecutor$AlwaysSpeculatingReadExecutor.executeAsync(AbstractReadExecutor.java:323)
~[main/:na]
at
org.apache.cassandra.service.StorageProxy$SinglePartitionReadLifecycle.doInitialQueries(StorageProxy.java:1599)
~[main/:na]
at
org.apache.cassandra.service.StorageProxy.fetchRows(StorageProxy.java:1554)
~[main/:na]
at
org.apache.cassandra.service.StorageProxy.readRegular(StorageProxy.java:1501)
~[main/:na]
at
org.apache.cassandra.service.StorageProxy.read(StorageProxy.java:1420)
~[main/:na]
at
org.apache.cassandra.db.SinglePartitionReadCommand$Group.execute(SinglePartitionReadCommand.java:457)
~[main/:na]
at
org.apache.cassandra.cql3.statements.SelectStatement.execute(SelectStatement.java:232)
~[main/:na]
at
org.apache.cassandra.cql3.statements.SelectStatement.execute(SelectStatement.java:202)
~[main/:na]
at
org.apache.cassandra.cql3.statements.SelectStatement.execute(SelectStatement.java:72)
~[main/:na]
at
org.apache.cassandra.cql3.QueryProcessor.processStatement(QueryProcessor.java:204)
~[main/:na]
at
org.apache.cassandra.cql3.QueryProcessor.processPrepared(QueryProcessor.java:470)
~[main/:na]
at
org.apache.cassandra.cql3.QueryProcessor.processPrepared(QueryProcessor.java:447)
~[main/:na]
at
org.apache.cassandra.transport.messages.ExecuteMessage.execute(ExecuteMessage.java:139)
~[main/:na]
at
org.apache.cassandra.transport.Message$Dispatcher.channelRead0(Message.java:507)
[main/:na]
at
org.apache.cassandra.transport.Message$Dispatcher.channelRead0(Message.java:401)
[main/:na]
at
io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:105)
[netty-all-4.0.23.Final.jar:4.0.23.Final]
at
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:333)
[netty-all-4.0.23.Final.jar:4.0.23.Final]
at
io.netty.channel.AbstractChannelHandlerContext.access$700(AbstractChannelHandlerContext.java:32)
[netty-all-4.0.23.Final.jar:4.0.23.Final]
at
io.netty.channel.AbstractChannelHandlerContext$8.run(AbstractChannelHandlerContext.java:324)
[netty-all-4.0.23.Final.jar:4.0.23.Final]
at
java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
[na:1.8.0_45]
at
org.apache.cassandra.concurrent.AbstractTracingAwareExecutorService$FutureTask.run(AbstractTracingAwareExecutorService.java:164)
[main/:na]
at org.apache.cassandra.concurrent.SEPWorker.run(SEPWorker.java:105)
[main/:na]
at java.lang.Thread.run(Thread.java:745) [na:1.8.0_45]
{noformat}
and another node shows:
{noformat}
ERROR [HintedHandoff:2] 2015-08-17 13:38:07,612 CassandraDaemon.java:191 -
Exception in thread Thread[HintedHandoff:2,1,main]
java.lang.RuntimeException: java.util.concurrent.ExecutionException:
java.lang.UnsupportedOperationException
at
org.apache.cassandra.db.HintedHandOffManager.compact(HintedHandOffManager.java:281)
~[main/:na]
at
org.apache.cassandra.db.HintedHandOffManager$5.run(HintedHandOffManager.java:535)
~[main/:na]
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
~[na:1.8.0_45]
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
~[na:1.8.0_45]
at java.lang.Thread.run(Thread.java:745) ~[na:1.8.0_45]
Caused by: java.util.concurrent.ExecutionException:
java.lang.UnsupportedOperationException
at java.util.concurrent.FutureTask.report(FutureTask.java:122)
~[na:1.8.0_45]
at java.util.concurrent.FutureTask.get(FutureTask.java:192)
~[na:1.8.0_45]
at
org.apache.cassandra.db.HintedHandOffManager.compact(HintedHandOffManager.java:277)
~[main/:na]
... 4 common frames omitted
Caused by: java.lang.UnsupportedOperationException: null
at
org.apache.cassandra.db.Serializers.clusteringPrefixSerializer(Serializers.java:52)
~[main/:na]
at
org.apache.cassandra.io.sstable.IndexHelper$IndexInfo$Serializer.deserialize(IndexHelper.java:171)
~[main/:na]
at
org.apache.cassandra.db.RowIndexEntry$Serializer.deserialize(RowIndexEntry.java:150)
~[main/:na]
at
org.apache.cassandra.io.sstable.format.big.BigTableScanner$KeyScanningIterator.computeNext(BigTableScanner.java:286)
~[main/:na]
at
org.apache.cassandra.io.sstable.format.big.BigTableScanner$KeyScanningIterator.computeNext(BigTableScanner.java:260)
~[main/:na]
at
org.apache.cassandra.utils.AbstractIterator.hasNext(AbstractIterator.java:47)
~[main/:na]
at
org.apache.cassandra.io.sstable.format.big.BigTableScanner.hasNext(BigTableScanner.java:240)
~[main/:na]
at
org.apache.cassandra.utils.MergeIterator$OneToOne.computeNext(MergeIterator.java:439)
~[main/:na]
at
org.apache.cassandra.utils.AbstractIterator.hasNext(AbstractIterator.java:47)
~[main/:na]
at
org.apache.cassandra.db.partitions.UnfilteredPartitionIterators$5.hasNext(UnfilteredPartitionIterators.java:234)
~[main/:na]
at
org.apache.cassandra.db.partitions.WrappingUnfilteredPartitionIterator.prepareNext(WrappingUnfilteredPartitionIterator.java:71)
~[main/:na]
at
org.apache.cassandra.db.partitions.WrappingUnfilteredPartitionIterator.hasNext(WrappingUnfilteredPartitionIterator.java:55)
~[main/:na]
at
org.apache.cassandra.db.partitions.PurgingPartitionIterator.hasNext(PurgingPartitionIterator.java:66)
~[main/:na]
at
org.apache.cassandra.db.compaction.CompactionIterator.hasNext(CompactionIterator.java:212)
~[main/:na]
at
org.apache.cassandra.db.compaction.CompactionTask.runMayThrow(CompactionTask.java:179)
~[main/:na]
at
org.apache.cassandra.utils.WrappedRunnable.run(WrappedRunnable.java:28)
~[main/:na]
at
org.apache.cassandra.db.compaction.CompactionTask.executeInternal(CompactionTask.java:80)
~[main/:na]
at
org.apache.cassandra.db.compaction.AbstractCompactionTask.execute(AbstractCompactionTask.java:59)
~[main/:na]
at
org.apache.cassandra.db.compaction.CompactionManager$9.runMayThrow(CompactionManager.java:638)
~[main/:na]
at
org.apache.cassandra.utils.WrappedRunnable.run(WrappedRunnable.java:28)
~[main/:na]
at
java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
~[na:1.8.0_45]
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
~[na:1.8.0_45]
... 3 common frames omitted
ERROR [CompactionExecutor:1] 2015-08-17 13:38:07,613 CassandraDaemon.java:191 -
Exception in thread Thread[CompactionExecutor:1,1,main]
java.lang.UnsupportedOperationException: null
at
org.apache.cassandra.db.Serializers.clusteringPrefixSerializer(Serializers.java:52)
~[main/:na]
at
org.apache.cassandra.io.sstable.IndexHelper$IndexInfo$Serializer.deserialize(IndexHelper.java:171)
~[main/:na]
at
org.apache.cassandra.db.RowIndexEntry$Serializer.deserialize(RowIndexEntry.java:150)
~[main/:na]
at
org.apache.cassandra.io.sstable.format.big.BigTableScanner$KeyScanningIterator.computeNext(BigTableScanner.java:286)
~[main/:na]
at
org.apache.cassandra.io.sstable.format.big.BigTableScanner$KeyScanningIterator.computeNext(BigTableScanner.java:260)
~[main/:na]
at
org.apache.cassandra.utils.AbstractIterator.hasNext(AbstractIterator.java:47)
~[main/:na]
at
org.apache.cassandra.io.sstable.format.big.BigTableScanner.hasNext(BigTableScanner.java:240)
~[main/:na]
at
org.apache.cassandra.utils.MergeIterator$OneToOne.computeNext(MergeIterator.java:439)
~[main/:na]
at
org.apache.cassandra.utils.AbstractIterator.hasNext(AbstractIterator.java:47)
~[main/:na]
at
org.apache.cassandra.db.partitions.UnfilteredPartitionIterators$5.hasNext(UnfilteredPartitionIterators.java:234)
~[main/:na]
at
org.apache.cassandra.db.partitions.WrappingUnfilteredPartitionIterator.prepareNext(WrappingUnfilteredPartitionIterator.java:71)
~[main/:na]
at
org.apache.cassandra.db.partitions.WrappingUnfilteredPartitionIterator.hasNext(WrappingUnfilteredPartitionIterator.java:55)
~[main/:na]
at
org.apache.cassandra.db.partitions.PurgingPartitionIterator.hasNext(PurgingPartitionIterator.java:66)
~[main/:na]
at
org.apache.cassandra.db.compaction.CompactionIterator.hasNext(CompactionIterator.java:212)
~[main/:na]
at
org.apache.cassandra.db.compaction.CompactionTask.runMayThrow(CompactionTask.java:179)
~[main/:na]
at
org.apache.cassandra.utils.WrappedRunnable.run(WrappedRunnable.java:28)
~[main/:na]
at
org.apache.cassandra.db.compaction.CompactionTask.executeInternal(CompactionTask.java:80)
~[main/:na]
at
org.apache.cassandra.db.compaction.AbstractCompactionTask.execute(AbstractCompactionTask.java:59)
~[main/:na]
at
org.apache.cassandra.db.compaction.CompactionManager$9.runMayThrow(CompactionManager.java:638)
~[main/:na]
at
org.apache.cassandra.utils.WrappedRunnable.run(WrappedRunnable.java:28)
~[main/:na]
at
java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
~[na:1.8.0_45]
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
~[na:1.8.0_45]
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
~[na:1.8.0_45]
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
[na:1.8.0_45]
at java.lang.Thread.run(Thread.java:745) [na:1.8.0_45]
.........................snip.........................
{noformat}
> java.lang.UnsupportedOperationException after upgrade to 3.0 alpha1
> -------------------------------------------------------------------
>
> Key: CASSANDRA-10102
> URL: https://issues.apache.org/jira/browse/CASSANDRA-10102
> Project: Cassandra
> Issue Type: Bug
> Reporter: Russ Hatch
> Attachments: node1.log, node2.log, node3.log
>
>
> Upgrade tests are showing a potential issue. I'm seeing this during rolling
> upgrades to 3.0 alpha 1, after one node has been upgraded to the alpha.
> I will attach cassandra logs here, node1.log is where most of the failures
> are seen.
> {noformat}
> ERROR [MessagingService-Incoming-/127.0.0.1] 2015-08-17 12:22:06,888
> CassandraDaemon.java:189 - Exception in thread
> Thread[MessagingService-Incoming-/127.0.0.1,5,main]
> java.lang.UnsupportedOperationException: null
> at
> org.apache.cassandra.db.ReadCommand$Serializer.deserialize(ReadCommand.java:485)
> ~[main/:na]
> at
> org.apache.cassandra.db.ReadCommand$Serializer.deserialize(ReadCommand.java:444)
> ~[main/:na]
> at org.apache.cassandra.net.MessageIn.read(MessageIn.java:98)
> ~[main/:na]
> at
> org.apache.cassandra.net.IncomingTcpConnection.receiveMessage(IncomingTcpConnection.java:195)
> ~[main/:na]
> at
> org.apache.cassandra.net.IncomingTcpConnection.receiveMessages(IncomingTcpConnection.java:172)
> ~[main/:na]
> at
> org.apache.cassandra.net.IncomingTcpConnection.run(IncomingTcpConnection.java:90)
> ~[main/:na]
> INFO [GossipStage:1] 2015-08-17 12:22:06,914 StorageService.java:1886 - Node
> /127.0.0.2 state jump to normal
> ERROR [MessagingService-Incoming-/127.0.0.1] 2015-08-17 12:22:06,915
> CassandraDaemon.java:189 - Exception in thread
> Thread[MessagingService-Incoming-/127.0.0.1,5,main]
> java.lang.UnsupportedOperationException: null
> at
> org.apache.cassandra.db.ReadCommand$Serializer.deserialize(ReadCommand.java:485)
> ~[main/:na]
> at
> org.apache.cassandra.db.ReadCommand$Serializer.deserialize(ReadCommand.java:444)
> ~[main/:na]
> at org.apache.cassandra.net.MessageIn.read(MessageIn.java:98)
> ~[main/:na]
> at
> org.apache.cassandra.net.IncomingTcpConnection.receiveMessage(IncomingTcpConnection.java:195)
> ~[main/:na]
> at
> org.apache.cassandra.net.IncomingTcpConnection.receiveMessages(IncomingTcpConnection.java:172)
> ~[main/:na]
> at
> org.apache.cassandra.net.IncomingTcpConnection.run(IncomingTcpConnection.java:90)
> ~[main/:na]
> {noformat}
> Another exception showing in logs:
> {noformat}
> ERROR [SharedPool-Worker-1] 2015-08-17 12:22:19,358 ErrorMessage.java:336 -
> Unexpected exception during request
> java.lang.UnsupportedOperationException: Version is 9
> at
> org.apache.cassandra.db.partitions.PartitionUpdate$PartitionUpdateSerializer.serializedSize(PartitionUpdate.java:760)
> ~[main/:na]
> at
> org.apache.cassandra.db.Mutation$MutationSerializer.serializedSize(Mutation.java:334)
> ~[main/:na]
> at
> org.apache.cassandra.db.Mutation$MutationSerializer.serializedSize(Mutation.java:246)
> ~[main/:na]
> at
> org.apache.cassandra.net.MessageOut.payloadSize(MessageOut.java:166)
> ~[main/:na]
> at
> org.apache.cassandra.net.OutboundTcpConnectionPool.getConnection(OutboundTcpConnectionPool.java:67)
> ~[main/:na]
> at
> org.apache.cassandra.net.MessagingService.getConnection(MessagingService.java:587)
> ~[main/:na]
> at
> org.apache.cassandra.net.MessagingService.sendOneWay(MessagingService.java:737)
> ~[main/:na]
> at
> org.apache.cassandra.net.MessagingService.sendRR(MessagingService.java:702)
> ~[main/:na]
> at
> org.apache.cassandra.service.StorageProxy.sendToHintedEndpoints(StorageProxy.java:1084)
> ~[main/:na]
> at
> org.apache.cassandra.service.StorageProxy$2.apply(StorageProxy.java:125)
> ~[main/:na]
> at
> org.apache.cassandra.service.StorageProxy.performWrite(StorageProxy.java:942)
> ~[main/:na]
> at
> org.apache.cassandra.service.StorageProxy.mutate(StorageProxy.java:549)
> ~[main/:na]
> at
> org.apache.cassandra.service.StorageProxy.mutateWithTriggers(StorageProxy.java:720)
> ~[main/:na]
> at
> org.apache.cassandra.cql3.statements.ModificationStatement.executeWithoutCondition(ModificationStatement.java:613)
> ~[main/:na]
> at
> org.apache.cassandra.cql3.statements.ModificationStatement.execute(ModificationStatement.java:599)
> ~[main/:na]
> at
> org.apache.cassandra.cql3.QueryProcessor.processStatement(QueryProcessor.java:204)
> ~[main/:na]
> at
> org.apache.cassandra.cql3.QueryProcessor.processPrepared(QueryProcessor.java:470)
> ~[main/:na]
> at
> org.apache.cassandra.cql3.QueryProcessor.processPrepared(QueryProcessor.java:447)
> ~[main/:na]
> at
> org.apache.cassandra.transport.messages.ExecuteMessage.execute(ExecuteMessage.java:139)
> ~[main/:na]
> at
> org.apache.cassandra.transport.Message$Dispatcher.channelRead0(Message.java:507)
> [main/:na]
> at
> org.apache.cassandra.transport.Message$Dispatcher.channelRead0(Message.java:401)
> [main/:na]
> at
> io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:105)
> [netty-all-4.0.23.Final.jar:4.0.23.Final]
> at
> io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:333)
> [netty-all-4.0.23.Final.jar:4.0.23.Final]
> at
> io.netty.channel.AbstractChannelHandlerContext.access$700(AbstractChannelHandlerContext.java:32)
> [netty-all-4.0.23.Final.jar:4.0.23.Final]
> at
> io.netty.channel.AbstractChannelHandlerContext$8.run(AbstractChannelHandlerContext.java:324)
> [netty-all-4.0.23.Final.jar:4.0.23.Final]
> at
> java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
> [na:1.8.0_45]
> at
> org.apache.cassandra.concurrent.AbstractTracingAwareExecutorService$FutureTask.run(AbstractTracingAwareExecutorService.java:164)
> [main/:na]
> at org.apache.cassandra.concurrent.SEPWorker.run(SEPWorker.java:105)
> [main/:na]
> at java.lang.Thread.run(Thread.java:745) [na:1.8.0_45]
> {noformat}
> I'm seeing several occurences of both exceptions above.
--
This message was sent by Atlassian JIRA
(v6.3.4#6332)
