[
https://issues.apache.org/jira/browse/CASSANDRA-15595?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=17045735#comment-17045735
]
Yakir Gibraltar commented on CASSANDRA-15595:
---------------------------------------------
I see the errors only in C* logs, example of logs with TRACE:
{code}
TRACE [Native-Transport-Requests-106] 2020-02-26 12:19:47,836
DigestResolver.java:96 - resolve: 0 ms.
TRACE [Native-Transport-Requests-207] 2020-02-26 12:19:47,836 Message.java:685
- Received: QUERY INSERT INTO ud.i_s (user_id,data) VALUES (?,?) USING TTL
7776000;[pageSize = 1000], v=4/v4
TRACE [Native-Transport-Requests-210] 2020-02-26 12:19:47,833
MessagingService.java:942 - /1.2.3.5 sending READ to -946659607@/1.2.3.15
TRACE [Native-Transport-Requests-44] 2020-02-26 12:19:47,834
StorageProxy.java:1352 - Adding FWD message to -946659552@/1.2.3.22
TRACE [Native-Transport-Requests-28] 2020-02-26 12:19:47,836
ReadCallback.java:101 - Blockfor is 2; setting up requests to /1.2.3.3,/1.2.3.14
TRACE [Native-Transport-Requests-118] 2020-02-26 12:19:47,833
ReadCallback.java:101 - Blockfor is 2; setting up requests to
/1.2.3.24,/1.2.3.23
TRACE [Native-Transport-Requests-18] 2020-02-26 12:19:47,833
StorageProxy.java:1357 - Sending message to -946659650@/1.2.3.25
TRACE [Native-Transport-Requests-127] 2020-02-26 12:19:47,824
StorageProxy.java:1352 - Adding FWD message to -946659949@/1.2.3.26
TRACE [Native-Transport-Requests-44] 2020-02-26 12:19:47,836
StorageProxy.java:1352 - Adding FWD message to -946659505@/1.2.3.27
TRACE [Native-Transport-Requests-28] 2020-02-26 12:19:47,836
AbstractReadExecutor.java:108 - reading data from /1.2.3.3
TRACE [Native-Transport-Requests-29] 2020-02-26 12:19:47,833 Message.java:704 -
Responding: EMPTY RESULT, v=4/v4
TRACE [Native-Transport-Requests-18] 2020-02-26 12:19:47,836
StorageProxy.java:1352 - Adding FWD message to -946659504@/1.2.3.37
TRACE [Native-Transport-Requests-22] 2020-02-26 12:19:47,834
ReadCallback.java:101 - Blockfor is 2; setting up requests to
/1.2.3.24,/1.2.3.12
TRACE [Native-Transport-Requests-44] 2020-02-26 12:19:47,836
MessagingService.java:942 - /1.2.3.5 sending MUTATION to -946659502@/1.2.3.28
TRACE [Native-Transport-Requests-62] 2020-02-26 12:19:47,833
StorageProxy.java:1357 - Sending message to -946659698@/1.2.3.37
TRACE [MutationStage-87] 2020-02-26 12:19:47,834 MessagingService.java:942 -
/1.2.3.5 sending REQUEST_RESPONSE to -946659658@/1.2.3.5
TRACE [Native-Transport-Requests-118] 2020-02-26 12:19:47,836
AbstractReadExecutor.java:108 - reading data from /1.2.3.24
TRACE [Native-Transport-Requests-5] 2020-02-26 12:19:47,834
MessagingService.java:942 - /1.2.3.5 sending MUTATION to -946659548@/1.2.3.29
TRACE [Native-Transport-Requests-64] 2020-02-26 12:19:47,833
MessagingService.java:942 - /1.2.3.5 sending MUTATION to -946659602@/1.2.3.30
TRACE [Native-Transport-Requests-3] 2020-02-26 12:19:47,833
MessagingService.java:942 - /1.2.3.5 sending READ to -946659600@/1.2.3.17
ERROR [ReadStage-110] 2020-02-26 12:19:47,836
AbstractLocalAwareExecutorService.java:169 - Uncaught exception on thread
Thread[ReadStage-110,5,main]
java.lang.AssertionError: Illegal bounds [-1777439288..-1777439280); size:
2761628520
at org.apache.cassandra.io.util.Memory.checkBounds(Memory.java:345)
~[apache-cassandra-3.11.6.jar:3.11.6]
at org.apache.cassandra.io.util.Memory.getLong(Memory.java:254)
~[apache-cassandra-3.11.6.jar:3.11.6]
at
org.apache.cassandra.io.compress.CompressionMetadata.chunkFor(CompressionMetadata.java:234)
~[apache-cassandra-3.11.6.jar:3.11.6]
at
org.apache.cassandra.io.util.CompressedChunkReader$Standard.readChunk(CompressedChunkReader.java:114)
~[apache-cassandra-3.11.6.jar:3.11.6]
at org.apache.cassandra.cache.ChunkCache.load(ChunkCache.java:158)
~[apache-cassandra-3.11.6.jar:3.11.6]
at org.apache.cassandra.cache.ChunkCache.load(ChunkCache.java:39)
~[apache-cassandra-3.11.6.jar:3.11.6]
at
com.github.benmanes.caffeine.cache.BoundedLocalCache$BoundedLocalLoadingCache.lambda$new$0(BoundedLocalCache.java:2949)
~[caffeine-2.2.6.jar:na]
at
com.github.benmanes.caffeine.cache.BoundedLocalCache.lambda$doComputeIfAbsent$15(BoundedLocalCache.java:1807)
~[caffeine-2.2.6.jar:na]
at
java.util.concurrent.ConcurrentHashMap.compute(ConcurrentHashMap.java:1853)
~[na:1.8.0-zing_19.12.102.0]
at
com.github.benmanes.caffeine.cache.BoundedLocalCache.doComputeIfAbsent(BoundedLocalCache.java:1805)
~[caffeine-2.2.6.jar:na]
at
com.github.benmanes.caffeine.cache.BoundedLocalCache.computeIfAbsent(BoundedLocalCache.java:1788)
~[caffeine-2.2.6.jar:na]
at
com.github.benmanes.caffeine.cache.LocalCache.computeIfAbsent(LocalCache.java:97)
~[caffeine-2.2.6.jar:na]
at
com.github.benmanes.caffeine.cache.LocalLoadingCache.get(LocalLoadingCache.java:66)
~[caffeine-2.2.6.jar:na]
at
org.apache.cassandra.cache.ChunkCache$CachingRebufferer.rebuffer(ChunkCache.java:236)
~[apache-cassandra-3.11.6.jar:3.11.6]
at
org.apache.cassandra.cache.ChunkCache$CachingRebufferer.rebuffer(ChunkCache.java:214)
~[apache-cassandra-3.11.6.jar:3.11.6]
at
org.apache.cassandra.io.util.RandomAccessReader.reBufferAt(RandomAccessReader.java:65)
~[apache-cassandra-3.11.6.jar:3.11.6]
at
org.apache.cassandra.io.util.RandomAccessReader.seek(RandomAccessReader.java:207)
~[apache-cassandra-3.11.6.jar:3.11.6]
at
org.apache.cassandra.io.util.FileHandle.createReader(FileHandle.java:150)
~[apache-cassandra-3.11.6.jar:3.11.6]
at
org.apache.cassandra.io.sstable.format.big.BigTableReader.getPosition(BigTableReader.java:251)
~[apache-cassandra-3.11.6.jar:3.11.6]
at
org.apache.cassandra.io.sstable.format.SSTableReader.getPosition(SSTableReader.java:1586)
~[apache-cassandra-3.11.6.jar:3.11.6]
at
org.apache.cassandra.io.sstable.format.big.BigTableReader.iterator(BigTableReader.java:64)
~[apache-cassandra-3.11.6.jar:3.11.6]
at
org.apache.cassandra.db.StorageHook$1.makeRowIterator(StorageHook.java:100)
~[apache-cassandra-3.11.6.jar:3.11.6]
at
org.apache.cassandra.db.SinglePartitionReadCommand.queryMemtableAndSSTablesInTimestampOrder(SinglePartitionReadCommand.java:982)
~[apache-cassandra-3.11.6.jar:3.11.6]
at
org.apache.cassandra.db.SinglePartitionReadCommand.queryMemtableAndDiskInternal(SinglePartitionReadCommand.java:693)
~[apache-cassandra-3.11.6.jar:3.11.6]
at
org.apache.cassandra.db.SinglePartitionReadCommand.queryMemtableAndDisk(SinglePartitionReadCommand.java:670)
~[apache-cassandra-3.11.6.jar:3.11.6]
at
org.apache.cassandra.db.SinglePartitionReadCommand.queryStorage(SinglePartitionReadCommand.java:504)
~[apache-cassandra-3.11.6.jar:3.11.6]
at
org.apache.cassandra.db.ReadCommand.executeLocally(ReadCommand.java:423)
~[apache-cassandra-3.11.6.jar:3.11.6]
at
org.apache.cassandra.service.StorageProxy$LocalReadRunnable.runMayThrow(StorageProxy.java:1874)
~[apache-cassandra-3.11.6.jar:3.11.6]
at
org.apache.cassandra.service.StorageProxy$DroppableRunnable.run(StorageProxy.java:2584)
~[apache-cassandra-3.11.6.jar:3.11.6]
at
java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
~[na:1.8.0-zing_19.12.102.0]
at
org.apache.cassandra.concurrent.AbstractLocalAwareExecutorService$FutureTask.run(AbstractLocalAwareExecutorService.java:165)
~[apache-cassandra-3.11.6.jar:3.11.6]
at
org.apache.cassandra.concurrent.AbstractLocalAwareExecutorService$LocalSessionFutureTask.run(AbstractLocalAwareExecutorService.java:137)
[apache-cassandra-3.11.6.jar:3.11.6]
at org.apache.cassandra.concurrent.SEPWorker.run(SEPWorker.java:113)
[apache-cassandra-3.11.6.jar:3.11.6]
at java.lang.Thread.run(Thread.java:748) [na:1.8.0-zing_19.12.102.0]
TRACE [Native-Transport-Requests-118] 2020-02-26 12:19:47,804
DigestResolver.java:70 - resolving 2 responses
TRACE [Native-Transport-Requests-39] 2020-02-26 12:19:47,803
MessagingService.java:942 - /1.2.3.5 sending MUTATION to -946660789@/1.2.3.31
TRACE [Native-Transport-Requests-11] 2020-02-26 12:19:47,804
AbstractReadExecutor.java:108 - reading digest from /1.2.3.12
TRACE [Native-Transport-Requests-81] 2020-02-26 12:19:47,805 Message.java:685 -
Received: QUERY SELECT data FROM ud.i_g WHERE user_id=?;[pageSize = 1000],
v=4/v4
TRACE [Native-Transport-Requests-34] 2020-02-26 12:19:47,802
StorageProxy.java:1352 - Adding FWD message to -946660827@/1.2.3.8
TRACE [Native-Transport-Requests-53] 2020-02-26 12:19:47,805
AbstractReadExecutor.java:108 - reading data from /1.2.3.15
TRACE [Native-Transport-Requests-6] 2020-02-26 12:19:47,804
MessagingService.java:942 - /1.2.3.5 sending MUTATION to -946660724@/1.2.3.9
TRACE [Native-Transport-Requests-41] 2020-02-26 12:19:47,804
StorageProxy.java:1352 - Adding FWD message to -946660725@/1.2.3.10
TRACE [Native-Transport-Requests-39] 2020-02-26 12:19:47,806
StorageProxy.java:1357 - Sending message to -946660789@/1.2.3.31
TRACE [Native-Transport-Requests-20] 2020-02-26 12:19:47,805
StorageProxy.java:1357 - Sending message to -946660731@/1.2.3.32
TRACE [Native-Transport-Requests-118] 2020-02-26 12:19:47,805
DigestResolver.java:96 - resolve: 0 ms.
TRACE [Native-Transport-Requests-81] 2020-02-26 12:19:47,806
QueryProcessor.java:220 - Process
org.apache.cassandra.cql3.statements.SelectStatement@4a8290b3 @CL.LOCAL_QUORUM
TRACE [Native-Transport-Requests-128] 2020-02-26 12:19:47,803
AbstractReadExecutor.java:108 - reading digest from /1.2.3.14
TRACE [Native-Transport-Requests-132] 2020-02-26 12:19:47,798
StorageProxy.java:1352 - Adding FWD message to -946660976@/11.2.3.33
TRACE [Native-Transport-Requests-11] 2020-02-26 12:19:47,806
MessagingService.java:942 - /1.2.3.5 sending READ to -946660674@/1.2.3.12
TRACE [Native-Transport-Requests-59] 2020-02-26 12:19:47,804
StorageProxy.java:1352 - Adding FWD message to -946660723@/1.2.3.34
ERROR [ReadStage-97] 2020-02-26 12:19:47,804
AbstractLocalAwareExecutorService.java:169 - Uncaught exception on thread
Thread[ReadStage-97,5,main]
java.lang.AssertionError: Illegal bounds [-1642545424..-1642545416); size:
2761628520
at org.apache.cassandra.io.util.Memory.checkBounds(Memory.java:345)
~[apache-cassandra-3.11.6.jar:3.11.6]
at org.apache.cassandra.io.util.Memory.getLong(Memory.java:254)
~[apache-cassandra-3.11.6.jar:3.11.6]
at
org.apache.cassandra.io.compress.CompressionMetadata.chunkFor(CompressionMetadata.java:234)
~[apache-cassandra-3.11.6.jar:3.11.6]
at
org.apache.cassandra.io.util.CompressedChunkReader$Standard.readChunk(CompressedChunkReader.java:114)
~[apache-cassandra-3.11.6.jar:3.11.6]
at org.apache.cassandra.cache.ChunkCache.load(ChunkCache.java:158)
~[apache-cassandra-3.11.6.jar:3.11.6]
at org.apache.cassandra.cache.ChunkCache.load(ChunkCache.java:39)
~[apache-cassandra-3.11.6.jar:3.11.6]
at
com.github.benmanes.caffeine.cache.BoundedLocalCache$BoundedLocalLoadingCache.lambda$new$0(BoundedLocalCache.java:2949)
~[caffeine-2.2.6.jar:na]
at
com.github.benmanes.caffeine.cache.BoundedLocalCache.lambda$doComputeIfAbsent$15(BoundedLocalCache.java:1807)
~[caffeine-2.2.6.jar:na]
at
java.util.concurrent.ConcurrentHashMap.compute(ConcurrentHashMap.java:1853)
~[na:1.8.0-zing_19.12.102.0]
at
com.github.benmanes.caffeine.cache.BoundedLocalCache.doComputeIfAbsent(BoundedLocalCache.java:1805)
~[caffeine-2.2.6.jar:na]
at
com.github.benmanes.caffeine.cache.BoundedLocalCache.computeIfAbsent(BoundedLocalCache.java:1788)
~[caffeine-2.2.6.jar:na]
at
com.github.benmanes.caffeine.cache.LocalCache.computeIfAbsent(LocalCache.java:97)
~[caffeine-2.2.6.jar:na]
at
com.github.benmanes.caffeine.cache.LocalLoadingCache.get(LocalLoadingCache.java:66)
~[caffeine-2.2.6.jar:na]
at
org.apache.cassandra.cache.ChunkCache$CachingRebufferer.rebuffer(ChunkCache.java:236)
~[apache-cassandra-3.11.6.jar:3.11.6]
at
org.apache.cassandra.cache.ChunkCache$CachingRebufferer.rebuffer(ChunkCache.java:214)
~[apache-cassandra-3.11.6.jar:3.11.6]
at
org.apache.cassandra.io.util.RandomAccessReader.reBufferAt(RandomAccessReader.java:65)
~[apache-cassandra-3.11.6.jar:3.11.6]
at
org.apache.cassandra.io.util.RandomAccessReader.seek(RandomAccessReader.java:207)
~[apache-cassandra-3.11.6.jar:3.11.6]
at
org.apache.cassandra.io.util.FileHandle.createReader(FileHandle.java:150)
~[apache-cassandra-3.11.6.jar:3.11.6]
at
org.apache.cassandra.io.sstable.format.big.BigTableReader.getPosition(BigTableReader.java:251)
~[apache-cassandra-3.11.6.jar:3.11.6]
at
org.apache.cassandra.io.sstable.format.SSTableReader.getPosition(SSTableReader.java:1586)
~[apache-cassandra-3.11.6.jar:3.11.6]
at
org.apache.cassandra.io.sstable.format.big.BigTableReader.iterator(BigTableReader.java:64)
~[apache-cassandra-3.11.6.jar:3.11.6]
at
org.apache.cassandra.db.StorageHook$1.makeRowIterator(StorageHook.java:100)
~[apache-cassandra-3.11.6.jar:3.11.6]
at
org.apache.cassandra.db.SinglePartitionReadCommand.queryMemtableAndSSTablesInTimestampOrder(SinglePartitionReadCommand.java:982)
~[apache-cassandra-3.11.6.jar:3.11.6]
at
org.apache.cassandra.db.SinglePartitionReadCommand.queryMemtableAndDiskInternal(SinglePartitionReadCommand.java:693)
~[apache-cassandra-3.11.6.jar:3.11.6]
at
org.apache.cassandra.db.SinglePartitionReadCommand.queryMemtableAndDisk(SinglePartitionReadCommand.java:670)
~[apache-cassandra-3.11.6.jar:3.11.6]
at
org.apache.cassandra.db.SinglePartitionReadCommand.queryStorage(SinglePartitionReadCommand.java:504)
~[apache-cassandra-3.11.6.jar:3.11.6]
at
org.apache.cassandra.db.ReadCommand.executeLocally(ReadCommand.java:423)
~[apache-cassandra-3.11.6.jar:3.11.6]
at
org.apache.cassandra.service.StorageProxy$LocalReadRunnable.runMayThrow(StorageProxy.java:1874)
~[apache-cassandra-3.11.6.jar:3.11.6]
at
org.apache.cassandra.service.StorageProxy$DroppableRunnable.run(StorageProxy.java:2584)
~[apache-cassandra-3.11.6.jar:3.11.6]
at
java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
~[na:1.8.0-zing_19.12.102.0]
at
org.apache.cassandra.concurrent.AbstractLocalAwareExecutorService$FutureTask.run(AbstractLocalAwareExecutorService.java:165)
~[apache-cassandra-3.11.6.jar:3.11.6]
at
org.apache.cassandra.concurrent.AbstractLocalAwareExecutorService$LocalSessionFutureTask.run(AbstractLocalAwareExecutorService.java:137)
[apache-cassandra-3.11.6.jar:3.11.6]
at org.apache.cassandra.concurrent.SEPWorker.run(SEPWorker.java:113)
[apache-cassandra-3.11.6.jar:3.11.6]
at java.lang.Thread.run(Thread.java:748) [na:1.8.0-zing_19.12.102.0]
TRACE [Native-Transport-Requests-31] 2020-02-26 12:19:47,800 Message.java:685 -
Received: QUERY SELECT data FROM ud.i_s WHERE user_id=?;[pageSize = 1000],
v=4/v4
TRACE [Native-Transport-Requests-33] 2020-02-26 12:19:47,800
AbstractReadExecutor.java:108 - reading digest from /1.2.3.12
TRACE [Native-Transport-Requests-126] 2020-02-26 12:19:47,799
StorageProxy.java:1352 - Adding FWD message to -946660922@/1.2.3.13
TRACE [Native-Transport-Requests-135] 2020-02-26 12:19:47,799
ReadCallback.java:101 - Blockfor is 2; setting up requests to /1.2.3.5,/1.2.3.14
TRACE [Native-Transport-Requests-44] 2020-02-26 12:19:47,800
DigestResolver.java:70 - resolving 2 responses
TRACE [Native-Transport-Requests-89] 2020-02-26 12:19:47,799
DigestResolver.java:96 - resolve: 0 ms.
TRACE [Native-Transport-Requests-82] 2020-02-26 12:19:47,800 Tracing.java:188 -
request complete
TRACE [Native-Transport-Requests-126] 2020-02-26 12:19:47,801
MessagingService.java:942 - /1.2.3.5 sending MUTATION to -946660864@/1.2.3.10
TRACE [Native-Transport-Requests-76] 2020-02-26 12:19:47,800 Message.java:685 -
Received: QUERY SELECT data FROM ud.i_s WHERE user_id=?;[pageSize = 1000],
v=4/v4
TRACE [Native-Transport-Requests-80] 2020-02-26 12:19:47,799
QueryProcessor.java:220 - Process
org.apache.cassandra.cql3.statements.UpdateStatement@75665751 @CL.LOCAL_QUORUM
TRACE [ReadStage-112] 2020-02-26 12:19:47,801 BufferPool.java:124 - Requested
buffer size 8.000KiB has been allocated directly due to lack of capacity
TRACE [Native-Transport-Requests-22] 2020-02-26 12:19:47,799
DigestResolver.java:70 - resolving 2 responses
TRACE [Native-Transport-Requests-109] 2020-02-26 12:19:47,798
StorageProxy.java:1352 - Adding FWD message to -946660966@/1.2.3.11
TRACE [Native-Transport-Requests-76] 2020-02-26 12:19:47,801
QueryProcessor.java:220 - Process
org.apache.cassandra.cql3.statements.SelectStatement@23ab710e @CL.LOCAL_QUORUM
TRACE [Native-Transport-Requests-96] 2020-02-26 12:19:47,799
QueryProcessor.java:220 - Process
org.apache.cassandra.cql3.statements.SelectStatement@2ee6e329 @CL.LOCAL_QUORUM
TRACE [Native-Transport-Requests-22] 2020-02-26 12:19:47,801
DigestResolver.java:96 - resolve: 0 ms.
TRACE [Native-Transport-Requests-107] 2020-02-26 12:19:47,798
QueryProcessor.java:220 - Process
org.apache.cassandra.cql3.statements.SelectStatement@1442c147 @CL.LOCAL_QUORUM
TRACE [Native-Transport-Requests-13] 2020-02-26 12:19:47,797
ReadCallback.java:101 - Blockfor is 2; setting up requests to
/1.2.3.12,/1.2.3.17
TRACE [Native-Transport-Requests-16] 2020-02-26 12:19:47,798 Message.java:704 -
Responding: ROWS [data(ud, i_g), org.apache.cassandra.db.marshal.BytesType]
| 121083ed7470f3ede4e2703977c2fac20a7f
---, v=4/v4
ERROR [ReadStage-116] 2020-02-26 12:19:47,799
AbstractLocalAwareExecutorService.java:169 - Uncaught exception on thread
Thread[ReadStage-116,5,main]
java.lang.AssertionError: Illegal bounds [-1646501592..-1646501584); size:
2761628520
at org.apache.cassandra.io.util.Memory.checkBounds(Memory.java:345)
~[apache-cassandra-3.11.6.jar:3.11.6]
at org.apache.cassandra.io.util.Memory.getLong(Memory.java:254)
~[apache-cassandra-3.11.6.jar:3.11.6]
at
org.apache.cassandra.io.compress.CompressionMetadata.chunkFor(CompressionMetadata.java:234)
~[apache-cassandra-3.11.6.jar:3.11.6]
at
org.apache.cassandra.io.util.CompressedChunkReader$Standard.readChunk(CompressedChunkReader.java:114)
~[apache-cassandra-3.11.6.jar:3.11.6]
at org.apache.cassandra.cache.ChunkCache.load(ChunkCache.java:158)
~[apache-cassandra-3.11.6.jar:3.11.6]
at org.apache.cassandra.cache.ChunkCache.load(ChunkCache.java:39)
~[apache-cassandra-3.11.6.jar:3.11.6]
at
com.github.benmanes.caffeine.cache.BoundedLocalCache$BoundedLocalLoadingCache.lambda$new$0(BoundedLocalCache.java:2949)
~[caffeine-2.2.6.jar:na]
at
com.github.benmanes.caffeine.cache.BoundedLocalCache.lambda$doComputeIfAbsent$15(BoundedLocalCache.java:1807)
~[caffeine-2.2.6.jar:na]
at
java.util.concurrent.ConcurrentHashMap.compute(ConcurrentHashMap.java:1853)
~[na:1.8.0-zing_19.12.102.0]
at
com.github.benmanes.caffeine.cache.BoundedLocalCache.doComputeIfAbsent(BoundedLocalCache.java:1805)
~[caffeine-2.2.6.jar:na]
at
com.github.benmanes.caffeine.cache.BoundedLocalCache.computeIfAbsent(BoundedLocalCache.java:1788)
~[caffeine-2.2.6.jar:na]
at
com.github.benmanes.caffeine.cache.LocalCache.computeIfAbsent(LocalCache.java:97)
~[caffeine-2.2.6.jar:na]
at
com.github.benmanes.caffeine.cache.LocalLoadingCache.get(LocalLoadingCache.java:66)
~[caffeine-2.2.6.jar:na]
at
org.apache.cassandra.cache.ChunkCache$CachingRebufferer.rebuffer(ChunkCache.java:236)
~[apache-cassandra-3.11.6.jar:3.11.6]
at
org.apache.cassandra.cache.ChunkCache$CachingRebufferer.rebuffer(ChunkCache.java:214)
~[apache-cassandra-3.11.6.jar:3.11.6]
at
org.apache.cassandra.io.util.RandomAccessReader.reBufferAt(RandomAccessReader.java:65)
~[apache-cassandra-3.11.6.jar:3.11.6]
at
org.apache.cassandra.io.util.RandomAccessReader.seek(RandomAccessReader.java:207)
~[apache-cassandra-3.11.6.jar:3.11.6]
at
org.apache.cassandra.io.util.FileHandle.createReader(FileHandle.java:150)
~[apache-cassandra-3.11.6.jar:3.11.6]
at
org.apache.cassandra.io.sstable.format.big.BigTableReader.getPosition(BigTableReader.java:251)
~[apache-cassandra-3.11.6.jar:3.11.6]
at
org.apache.cassandra.io.sstable.format.SSTableReader.getPosition(SSTableReader.java:1586)
~[apache-cassandra-3.11.6.jar:3.11.6]
at
org.apache.cassandra.io.sstable.format.big.BigTableReader.iterator(BigTableReader.java:64)
~[apache-cassandra-3.11.6.jar:3.11.6]
at
org.apache.cassandra.db.StorageHook$1.makeRowIterator(StorageHook.java:100)
~[apache-cassandra-3.11.6.jar:3.11.6]
at
org.apache.cassandra.db.SinglePartitionReadCommand.queryMemtableAndSSTablesInTimestampOrder(SinglePartitionReadCommand.java:982)
~[apache-cassandra-3.11.6.jar:3.11.6]
at
org.apache.cassandra.db.SinglePartitionReadCommand.queryMemtableAndDiskInternal(SinglePartitionReadCommand.java:693)
~[apache-cassandra-3.11.6.jar:3.11.6]
at
org.apache.cassandra.db.SinglePartitionReadCommand.queryMemtableAndDisk(SinglePartitionReadCommand.java:670)
~[apache-cassandra-3.11.6.jar:3.11.6]
at
org.apache.cassandra.db.SinglePartitionReadCommand.queryStorage(SinglePartitionReadCommand.java:504)
~[apache-cassandra-3.11.6.jar:3.11.6]
at
org.apache.cassandra.db.ReadCommand.executeLocally(ReadCommand.java:423)
~[apache-cassandra-3.11.6.jar:3.11.6]
at
org.apache.cassandra.service.StorageProxy$LocalReadRunnable.runMayThrow(StorageProxy.java:1874)
~[apache-cassandra-3.11.6.jar:3.11.6]
at
org.apache.cassandra.service.StorageProxy$DroppableRunnable.run(StorageProxy.java:2584)
~[apache-cassandra-3.11.6.jar:3.11.6]
at
java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
~[na:1.8.0-zing_19.12.102.0]
at
org.apache.cassandra.concurrent.AbstractLocalAwareExecutorService$FutureTask.run(AbstractLocalAwareExecutorService.java:165)
~[apache-cassandra-3.11.6.jar:3.11.6]
at
org.apache.cassandra.concurrent.AbstractLocalAwareExecutorService$LocalSessionFutureTask.run(AbstractLocalAwareExecutorService.java:137)
[apache-cassandra-3.11.6.jar:3.11.6]
at org.apache.cassandra.concurrent.SEPWorker.run(SEPWorker.java:113)
[apache-cassandra-3.11.6.jar:3.11.6]
at java.lang.Thread.run(Thread.java:748) [na:1.8.0-zing_19.12.102.0]
TRACE [Native-Transport-Requests-22] 2020-02-26 12:19:47,801
ReadCallback.java:147 - Read: 13 ms.
TRACE [Native-Transport-Requests-57] 2020-02-26 12:19:47,795
MessagingService.java:942 - /1.2.3.5 sending MUTATION to -946661060@/1.2.3.4
TRACE [Native-Transport-Requests-22] 2020-02-26 12:19:47,801 Tracing.java:188 -
request complete
TRACE [Native-Transport-Requests-96] 2020-02-26 12:19:47,801
ReadCallback.java:101 - Blockfor is 2; setting up requests to /1.2.3.3,/1.2.3.6
TRACE [Native-Transport-Requests-23] 2020-02-26 12:19:47,793
AbstractReadExecutor.java:108 - reading digest from /1.2.3.17
TRACE [Native-Transport-Requests-23] 2020-02-26 12:19:47,793
AbstractReadExecutor.java:108 - reading digest from /1.2.3.17
TRACE [Native-Transport-Requests-85] 2020-02-26 12:19:47,793
MessagingService.java:942 - /1.2.3.5 sending MUTATION to -946661140@/1.2.3.7
TRACE [Native-Transport-Requests-4] 2020-02-26 12:19:47,792
DigestResolver.java:70 - resolving 2 responses
TRACE [Native-Transport-Requests-52] 2020-02-26 12:19:47,793
MessagingService.java:942 - /1.2.3.5 sending READ to -946661138@/1.2.3.17
TRACE [PerDiskMemtableFlushWriter_0:1100] 2020-02-26 12:19:47,793
BigTableWriter.java:473 - wrote index entry:
org.apache.cassandra.db.RowIndexEntry@18b05153 at 5162975
TRACE [Native-Transport-Requests-22] 2020-02-26 12:19:47,790
AbstractReadExecutor.java:108 - reading data from /1.2.3.15
TRACE [Native-Transport-Requests-64] 2020-02-26 12:19:47,786
AbstractReadExecutor.java:108 - reading data from /1.2.3.14
TRACE [Native-Transport-Requests-58] 2020-02-26 12:19:47,792
AbstractReadExecutor.java:116 - reading data locally
TRACE [Native-Transport-Requests-85] 2020-02-26 12:19:47,793
StorageProxy.java:1357 - Sending message to -946661140@/1.2.3.7
TRACE [Native-Transport-Requests-45] 2020-02-26 12:19:47,792
MessagingService.java:942 - /1.2.3.5 sending READ to -946661199@/1.2.3.3
TRACE [Native-Transport-Requests-49] 2020-02-26 12:19:47,793
StorageProxy.java:1352 - Adding FWD message to -946661142@/1.2.3.8
TRACE [Native-Transport-Requests-44] 2020-02-26 12:19:47,793 Message.java:685 -
Received: QUERY SELECT data FROM ud.i_g WHERE user_id=?;[pageSize = 1000],
v=4/v4
TRACE [Native-Transport-Requests-73] 2020-02-26 12:19:47,793
MessagingService.java:942 - /1.2.3.5 sending MUTATION to -946661143@/1.2.3.16
TRACE [Native-Transport-Requests-105] 2020-02-26 12:19:47,787
StorageProxy.java:1352 - Adding FWD message to -946661340@/1.2.3.35
TRACE [Native-Transport-Requests-22] 2020-02-26 12:19:47,793
MessagingService.java:942 - /1.2.3.5 sending READ to -946661135@/1.2.3.15
TRACE [Native-Transport-Requests-92] 2020-02-26 12:19:47,792
AbstractReadExecutor.java:108 - reading digest from /1.2.3.17
TRACE [Native-Transport-Requests-69] 2020-02-26 12:19:47,793
AbstractReadExecutor.java:108 - reading digest from /1.2.3.4
TRACE [Native-Transport-Requests-58] 2020-02-26 12:19:47,793
AbstractReadExecutor.java:108 - reading digest from /1.2.3.6
TRACE [Native-Transport-Requests-64] 2020-02-26 12:19:47,793
MessagingService.java:942 - /1.2.3.5 sending READ to -946661134@/1.2.3.14
TRACE [Native-Transport-Requests-48] 2020-02-26 12:19:47,793
StorageProxy.java:1357 - Sending message to -946661250@/1.2.3.18
TRACE [Native-Transport-Requests-73] 2020-02-26 12:19:47,793
StorageProxy.java:1357 - Sending message to -946661143@/1.2.3.16
TRACE [Native-Transport-Requests-1] 2020-02-26 12:19:47,792
StorageProxy.java:1352 - Adding FWD message to -946661189@/1.2.3.36
TRACE [Native-Transport-Requests-132] 2020-02-26 12:19:47,787
MessagingService.java:942 - /1.2.3.5 sending MUTATION to -946661332@/1.2.3.19
TRACE [Native-Transport-Requests-103] 2020-02-26 12:19:47,792
AbstractReadExecutor.java:108 - reading digest from /1.2.3.6
TRACE [Native-Transport-Requests-48] 2020-02-26 12:19:47,793 Tracing.java:188 -
request complete
TRACE [Native-Transport-Requests-114] 2020-02-26 12:19:47,789
StorageProxy.java:1352 - Adding FWD message to -946661292@/1.2.3.20
TRACE [ReadStage-9] 2020-02-26 12:19:47,784 BufferPool.java:124 - Requested
buffer size 4.000KiB has been allocated directly due to lack of capacity
TRACE [Native-Transport-Requests-73] 2020-02-26 12:19:47,793
StorageProxy.java:1352 - Adding FWD message to -946661127@/1.2.3.21
ERROR [ReadStage-67] 2020-02-26 12:19:47,790
AbstractLocalAwareExecutorService.java:169 - Uncaught exception on thread
Thread[ReadStage-67,10,main]
java.lang.AssertionError: Illegal bounds [-2109842656..-2109842648); size:
2761628520
at org.apache.cassandra.io.util.Memory.checkBounds(Memory.java:345)
~[apache-cassandra-3.11.6.jar:3.11.6]
at org.apache.cassandra.io.util.Memory.getLong(Memory.java:254)
~[apache-cassandra-3.11.6.jar:3.11.6]
at
org.apache.cassandra.io.compress.CompressionMetadata.chunkFor(CompressionMetadata.java:234)
~[apache-cassandra-3.11.6.jar:3.11.6]
at
org.apache.cassandra.io.util.CompressedChunkReader$Standard.readChunk(CompressedChunkReader.java:114)
~[apache-cassandra-3.11.6.jar:3.11.6]
at org.apache.cassandra.cache.ChunkCache.load(ChunkCache.java:158)
~[apache-cassandra-3.11.6.jar:3.11.6]
at org.apache.cassandra.cache.ChunkCache.load(ChunkCache.java:39)
~[apache-cassandra-3.11.6.jar:3.11.6]
at
com.github.benmanes.caffeine.cache.BoundedLocalCache$BoundedLocalLoadingCache.lambda$new$0(BoundedLocalCache.java:2949)
~[caffeine-2.2.6.jar:na]
at
com.github.benmanes.caffeine.cache.BoundedLocalCache.lambda$doComputeIfAbsent$15(BoundedLocalCache.java:1807)
~[caffeine-2.2.6.jar:na]
at
java.util.concurrent.ConcurrentHashMap.compute(ConcurrentHashMap.java:1853)
~[na:1.8.0-zing_19.12.102.0]
at
com.github.benmanes.caffeine.cache.BoundedLocalCache.doComputeIfAbsent(BoundedLocalCache.java:1805)
~[caffeine-2.2.6.jar:na]
at
com.github.benmanes.caffeine.cache.BoundedLocalCache.computeIfAbsent(BoundedLocalCache.java:1788)
~[caffeine-2.2.6.jar:na]
at
com.github.benmanes.caffeine.cache.LocalCache.computeIfAbsent(LocalCache.java:97)
~[caffeine-2.2.6.jar:na]
at
com.github.benmanes.caffeine.cache.LocalLoadingCache.get(LocalLoadingCache.java:66)
~[caffeine-2.2.6.jar:na]
at
org.apache.cassandra.cache.ChunkCache$CachingRebufferer.rebuffer(ChunkCache.java:236)
~[apache-cassandra-3.11.6.jar:3.11.6]
at
org.apache.cassandra.cache.ChunkCache$CachingRebufferer.rebuffer(ChunkCache.java:214)
~[apache-cassandra-3.11.6.jar:3.11.6]
at
org.apache.cassandra.io.util.RandomAccessReader.reBufferAt(RandomAccessReader.java:65)
~[apache-cassandra-3.11.6.jar:3.11.6]
at
org.apache.cassandra.io.util.RandomAccessReader.seek(RandomAccessReader.java:207)
~[apache-cassandra-3.11.6.jar:3.11.6]
at
org.apache.cassandra.io.util.FileHandle.createReader(FileHandle.java:150)
~[apache-cassandra-3.11.6.jar:3.11.6]
at
org.apache.cassandra.io.sstable.format.big.BigTableReader.getPosition(BigTableReader.java:251)
~[apache-cassandra-3.11.6.jar:3.11.6]
at
org.apache.cassandra.io.sstable.format.SSTableReader.getPosition(SSTableReader.java:1586)
~[apache-cassandra-3.11.6.jar:3.11.6]
at
org.apache.cassandra.io.sstable.format.big.BigTableReader.iterator(BigTableReader.java:64)
~[apache-cassandra-3.11.6.jar:3.11.6]
at
org.apache.cassandra.db.StorageHook$1.makeRowIterator(StorageHook.java:100)
~[apache-cassandra-3.11.6.jar:3.11.6]
at
org.apache.cassandra.db.SinglePartitionReadCommand.queryMemtableAndSSTablesInTimestampOrder(SinglePartitionReadCommand.java:982)
~[apache-cassandra-3.11.6.jar:3.11.6]
at
org.apache.cassandra.db.SinglePartitionReadCommand.queryMemtableAndDiskInternal(SinglePartitionReadCommand.java:693)
~[apache-cassandra-3.11.6.jar:3.11.6]
at
org.apache.cassandra.db.SinglePartitionReadCommand.queryMemtableAndDisk(SinglePartitionReadCommand.java:670)
~[apache-cassandra-3.11.6.jar:3.11.6]
at
org.apache.cassandra.db.SinglePartitionReadCommand.queryStorage(SinglePartitionReadCommand.java:504)
~[apache-cassandra-3.11.6.jar:3.11.6]
at
org.apache.cassandra.db.ReadCommand.executeLocally(ReadCommand.java:423)
~[apache-cassandra-3.11.6.jar:3.11.6]
at
org.apache.cassandra.service.StorageProxy$LocalReadRunnable.runMayThrow(StorageProxy.java:1874)
~[apache-cassandra-3.11.6.jar:3.11.6]
at
org.apache.cassandra.service.StorageProxy$DroppableRunnable.run(StorageProxy.java:2584)
~[apache-cassandra-3.11.6.jar:3.11.6]
at
java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
~[na:1.8.0-zing_19.12.102.0]
at
org.apache.cassandra.concurrent.AbstractLocalAwareExecutorService$FutureTask.run(AbstractLocalAwareExecutorService.java:165)
~[apache-cassandra-3.11.6.jar:3.11.6]
at
org.apache.cassandra.concurrent.AbstractLocalAwareExecutorService$LocalSessionFutureTask.run(AbstractLocalAwareExecutorService.java:137)
[apache-cassandra-3.11.6.jar:3.11.6]
at org.apache.cassandra.concurrent.SEPWorker.run(SEPWorker.java:113)
[apache-cassandra-3.11.6.jar:3.11.6]
at java.lang.Thread.run(Thread.java:748) [na:1.8.0-zing_19.12.102.0]
{code}
> Many errors of "java.lang.AssertionError: Illegal bounds"
> ---------------------------------------------------------
>
> Key: CASSANDRA-15595
> URL: https://issues.apache.org/jira/browse/CASSANDRA-15595
> Project: Cassandra
> Issue Type: Bug
> Components: Local/Caching
> Reporter: Yakir Gibraltar
> Priority: Normal
> Fix For: 3.11.7
>
>
> Hi, i'm running cassandra 3.11.6 and getting on all hosts many errors of:
> {code}
> ERROR [ReadStage-6] 2020-02-24 13:53:34,528
> AbstractLocalAwareExecutorService.java:169 - Uncaught exception on thread
> Thread[ReadStage-6,5,main]
> java.lang.AssertionError: Illegal bounds [-2102982480..-2102982472); size:
> 2761628520
> at org.apache.cassandra.io.util.Memory.checkBounds(Memory.java:345)
> ~[apache-cassandra-3.11.6.jar:3.11.6]
> at org.apache.cassandra.io.util.Memory.getLong(Memory.java:254)
> ~[apache-cassandra-3.11.6.jar:3.11.6]
> at
> org.apache.cassandra.io.compress.CompressionMetadata.chunkFor(CompressionMetadata.java:234)
> ~[apache-cassandra-3.11.6.jar:3.11.6]
> at
> org.apache.cassandra.io.util.CompressedChunkReader$Standard.readChunk(CompressedChunkReader.java:114)
> ~[apache-cassandra-3.11.6.ja
> r:3.11.6]
> at org.apache.cassandra.cache.ChunkCache.load(ChunkCache.java:158)
> ~[apache-cassandra-3.11.6.jar:3.11.6]
> at org.apache.cassandra.cache.ChunkCache.load(ChunkCache.java:39)
> ~[apache-cassandra-3.11.6.jar:3.11.6]
> at
> com.github.benmanes.caffeine.cache.BoundedLocalCache$BoundedLocalLoadingCache.lambda$new$0(BoundedLocalCache.java:2949)
> ~[caffeine-2.2.6.jar:na]
> at
> com.github.benmanes.caffeine.cache.BoundedLocalCache.lambda$doComputeIfAbsent$15(BoundedLocalCache.java:1807)
> ~[caffeine-2.2.6.jar:na]
> at
> java.util.concurrent.ConcurrentHashMap.compute(ConcurrentHashMap.java:1853)
> ~[na:1.8.0-zing_19.12.102.0]
> at
> com.github.benmanes.caffeine.cache.BoundedLocalCache.doComputeIfAbsent(BoundedLocalCache.java:1805)
> ~[caffeine-2.2.6.jar:na]
> at
> com.github.benmanes.caffeine.cache.BoundedLocalCache.computeIfAbsent(BoundedLocalCache.java:1788)
> ~[caffeine-2.2.6.jar:na]
> at
> com.github.benmanes.caffeine.cache.LocalCache.computeIfAbsent(LocalCache.java:97)
> ~[caffeine-2.2.6.jar:na]
> at
> com.github.benmanes.caffeine.cache.LocalLoadingCache.get(LocalLoadingCache.java:66)
> ~[caffeine-2.2.6.jar:na]
> at
> org.apache.cassandra.cache.ChunkCache$CachingRebufferer.rebuffer(ChunkCache.java:236)
> ~[apache-cassandra-3.11.6.jar:3.11.6]
> at
> org.apache.cassandra.cache.ChunkCache$CachingRebufferer.rebuffer(ChunkCache.java:214)
> ~[apache-cassandra-3.11.6.jar:3.11.6]
> at
> org.apache.cassandra.io.util.RandomAccessReader.reBufferAt(RandomAccessReader.java:65)
> ~[apache-cassandra-3.11.6.jar:3.11.6]
> at
> org.apache.cassandra.io.util.RandomAccessReader.seek(RandomAccessReader.java:207)
> ~[apache-cassandra-3.11.6.jar:3.11.6]
> at
> org.apache.cassandra.io.util.FileHandle.createReader(FileHandle.java:150)
> ~[apache-cassandra-3.11.6.jar:3.11.6]
> at
> org.apache.cassandra.io.sstable.format.SSTableReader.getFileDataInput(SSTableReader.java:1807)
> ~[apache-cassandra-3.11.6.jar:3.11.6]
> at
> org.apache.cassandra.db.columniterator.AbstractSSTableIterator.<init>(AbstractSSTableIterator.java:103)
> ~[apache-cassandra-3.11.6.jar:3.11.6]
> at
> org.apache.cassandra.db.columniterator.SSTableIterator.<init>(SSTableIterator.java:49)
> ~[apache-cassandra-3.11.6.jar:3.11.6]
> at
> org.apache.cassandra.io.sstable.format.big.BigTableReader.iterator(BigTableReader.java:72)
> ~[apache-cassandra-3.11.6.jar:3.11.6]
> at
> org.apache.cassandra.io.sstable.format.big.BigTableReader.iterator(BigTableReader.java:65)
> ~[apache-cassandra-3.11.6.jar:3.11.6]
> at
> org.apache.cassandra.db.StorageHook$1.makeRowIterator(StorageHook.java:100)
> ~[apache-cassandra-3.11.6.jar:3.11.6]
> at
> org.apache.cassandra.db.SinglePartitionReadCommand.queryMemtableAndSSTablesInTimestampOrder(SinglePartitionReadCommand.java:982)
> ~[apache-cassandra-3.11.6.jar:3.11.6]
> at
> org.apache.cassandra.db.SinglePartitionReadCommand.queryMemtableAndDiskInternal(SinglePartitionReadCommand.java:693)
> ~[apache-cassandra-3.11.6.jar:3.11.6]
> at
> org.apache.cassandra.db.SinglePartitionReadCommand.queryMemtableAndDisk(SinglePartitionReadCommand.java:670)
> ~[apache-cassandra-3.11.6.jar:3.11.6]
> at
> org.apache.cassandra.db.SinglePartitionReadCommand.queryStorage(SinglePartitionReadCommand.java:504)
> ~[apache-cassandra-3.11.6.jar:3.11.6]
> at
> org.apache.cassandra.db.ReadCommand.executeLocally(ReadCommand.java:423)
> ~[apache-cassandra-3.11.6.jar:3.11.6]
> at
> org.apache.cassandra.db.ReadCommandVerbHandler.doVerb(ReadCommandVerbHandler.java:48)
> ~[apache-cassandra-3.11.6.jar:3.11.6]
> at
> org.apache.cassandra.net.MessageDeliveryTask.run(MessageDeliveryTask.java:66)
> ~[apache-cassandra-3.11.6.jar:3.11.6]
> at
> java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
> ~[na:1.8.0-zing_19.12.102.0]
> at
> org.apache.cassandra.concurrent.AbstractLocalAwareExecutorService$FutureTask.run(AbstractLocalAwareExecutorService.java:165)
> ~[apache-cassandra-3.11.6.jar:3.11.6]
> at
> org.apache.cassandra.concurrent.AbstractLocalAwareExecutorService$LocalSessionFutureTask.run(AbstractLocalAwareExecutorService.java:137)
> [apache-cassandra-3.11.6.jar:3.11.6]
> at org.apache.cassandra.concurrent.SEPWorker.run(SEPWorker.java:113)
> [apache-cassandra-3.11.6.jar:3.11.6]
> at java.lang.Thread.run(Thread.java:748) [na:1.8.0-zing_19.12.102.0]
> {code}
> Someone familiar with that error?
--
This message was sent by Atlassian Jira
(v8.3.4#803005)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
