[
https://issues.apache.org/jira/browse/CASSANDRA-8689?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=14339748#comment-14339748
]
T Jake Luciani commented on CASSANDRA-8689:
-------------------------------------------
I'm now seeing many other failures related to index summaries
{code}
[echo] running test bucket 0 tests
[junit] WARNING: multiple versions of ant detected in path for junit
[junit]
jar:file:/usr/share/ant/lib/ant.jar!/org/apache/tools/ant/Project.class
[junit] and
jar:file:/home/jake/workspace/cassandra/build/lib/jars/ant-1.6.5.jar!/org/apache/tools/ant/Project.class
[junit] Testsuite: org.apache.cassandra.io.sstable.IndexSummaryManagerTest
[junit] Tests run: 6, Failures: 5, Errors: 0, Skipped: 0, Time elapsed:
10.138 sec
[junit]
[junit] ------------- Standard Output ---------------
[junit] ERROR 05:25:52 LEAK DETECTED: a reference
(org.apache.cassandra.utils.concurrent.Ref$State@9f7cd4c) to class
org.apache.cassandra.io.sstable.SSTableReader$InstanceTidier@1638131854:build/test/cassandra/data:0/Keyspace1/StandardLowIndexInterval-15bf0780be4111e4869db7d84572b531/Keyspace1-StandardLowIndexInterval-ka-2
was not released before the reference was garbage collected
[junit] ERROR 05:25:52 LEAK DETECTED: a reference
(org.apache.cassandra.utils.concurrent.Ref$State@9f7cd4c) to class
org.apache.cassandra.io.sstable.SSTableReader$InstanceTidier@1638131854:build/test/cassandra/data:0/Keyspace1/StandardLowIndexInterval-15bf0780be4111e4869db7d84572b531/Keyspace1-StandardLowIndexInterval-ka-2
was not released before the reference was garbage collected
[junit] ERROR 05:25:52 Allocate trace
org.apache.cassandra.utils.concurrent.Ref$State@9f7cd4c:
[junit] Thread[main,5,main]
[junit] at java.lang.Thread.getStackTrace(Thread.java:1589)
[junit] at
org.apache.cassandra.utils.concurrent.Ref$Debug.<init>(Ref.java:196)
[junit] at
org.apache.cassandra.utils.concurrent.Ref$State.<init>(Ref.java:133)
[junit] at org.apache.cassandra.utils.concurrent.Ref.<init>(Ref.java:60)
[junit] at
org.apache.cassandra.io.sstable.SSTableReader.<init>(SSTableReader.java:242)
[junit] at
org.apache.cassandra.io.sstable.SSTableReader.<init>(SSTableReader.java:605)
[junit] at
org.apache.cassandra.io.sstable.SSTableReader.cloneWithNewSummarySamplingLevel(SSTableReader.java:991)
[junit] at
org.apache.cassandra.io.sstable.IndexSummaryManager.adjustSamplingLevels(IndexSummaryManager.java:410)
[junit] at
org.apache.cassandra.io.sstable.IndexSummaryManager.redistributeSummaries(IndexSummaryManager.java:288)
[junit] at
org.apache.cassandra.io.sstable.IndexSummaryManagerTest.testChangeMaxIndexInterval(IndexSummaryManagerTest.java:319)
[junit] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
[junit] at
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
[junit] at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
[junit] at java.lang.reflect.Method.invoke(Method.java:606)
[junit] at
org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:44)
[junit] at
org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:15)
[junit] at
org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:41)
[junit] at
org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:20)
[junit] at
org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:28)
[junit] at
org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:31)
[junit] at
org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:70)
[junit] at
org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:44)
[junit] at
org.junit.runners.ParentRunner.runChildren(ParentRunner.java:180)
[junit] at
org.junit.runners.ParentRunner.access$000(ParentRunner.java:41)
[junit] at
org.junit.runners.ParentRunner$1.evaluate(ParentRunner.java:173)
[junit] at
org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:28)
[junit] at
org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:31)
[junit] at org.junit.runners.ParentRunner.run(ParentRunner.java:220)
[junit] at
junit.framework.JUnit4TestAdapter.run(JUnit4TestAdapter.java:39)
[junit] at
org.apache.tools.ant.taskdefs.optional.junit.JUnitTestRunner.run(JUnitTestRunner.java:532)
[junit] at
org.apache.tools.ant.taskdefs.optional.junit.JUnitTestRunner.launch(JUnitTestRunner.java:1179)
[junit] at
org.apache.tools.ant.taskdefs.optional.junit.JUnitTestRunner.main(JUnitTestRunner.java:1030)
[junit]
[junit] ERROR 05:25:52 Allocate trace
org.apache.cassandra.utils.concurrent.Ref$State@9f7cd4c:
[junit] Thread[main,5,main]
[junit] at java.lang.Thread.getStackTrace(Thread.java:1589)
[junit] at
org.apache.cassandra.utils.concurrent.Ref$Debug.<init>(Ref.java:196)
[junit] at
org.apache.cassandra.utils.concurrent.Ref$State.<init>(Ref.java:133)
[junit] at org.apache.cassandra.utils.concurrent.Ref.<init>(Ref.java:60)
[junit] at
org.apache.cassandra.io.sstable.SSTableReader.<init>(SSTableReader.java:242)
[junit] at
org.apache.cassandra.io.sstable.SSTableReader.<init>(SSTableReader.java:605)
[junit] at
org.apache.cassandra.io.sstable.SSTableReader.cloneWithNewSummarySamplingLevel(SSTableReader.java:991)
[junit] at
org.apache.cassandra.io.sstable.IndexSummaryManager.adjustSamplingLevels(IndexSummaryManager.java:410)
[junit] at
org.apache.cassandra.io.sstable.IndexSummaryManager.redistributeSummaries(IndexSummaryManager.java:288)
[junit] at
org.apache.cassandra.io.sstable.IndexSummaryManagerTest.testChangeMaxIndexInterval(IndexSummaryManagerTest.java:319)
[junit] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
[junit] at
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
[junit] at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
[junit] at java.lang.reflect.Method.invoke(Method.java:606)
[junit] at
org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:44)
[junit] at
org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:15)
[junit] at
org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:41)
[junit] at
org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:20)
[junit] at
org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:28)
[junit] at
org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:31)
[junit] at
org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:70)
[junit] at
org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:44)
[junit] at
org.junit.runners.ParentRunner.runChildren(ParentRunner.java:180)
[junit] at
org.junit.runners.ParentRunner.access$000(ParentRunner.java:41)
[junit] at
org.junit.runners.ParentRunner$1.evaluate(ParentRunner.java:173)
[junit] at
org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:28)
[junit] at
org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:31)
[junit] at org.junit.runners.ParentRunner.run(ParentRunner.java:220)
[junit] at
junit.framework.JUnit4TestAdapter.run(JUnit4TestAdapter.java:39)
[junit] at
org.apache.tools.ant.taskdefs.optional.junit.JUnitTestRunner.run(JUnitTestRunner.java:532)
[junit] at
org.apache.tools.ant.taskdefs.optional.junit.JUnitTestRunner.launch(JUnitTestRunner.java:1179)
[junit] at
org.apache.tools.ant.taskdefs.optional.junit.JUnitTestRunner.main(JUnitTestRunner.java:1030)
[junit]
[junit] WARN 05:25:52 Unable to cancel in-progress compactions for
StandardLowIndexInterval. Perhaps there is an unusually large row in progress
somewhere, or the system is simply overloaded.
[junit] WARN 05:25:52 Unable to cancel in-progress compactions for
StandardLowIndexInterval. Perhaps there is an unusually large row in progress
somewhere, or the system is simply overloaded.
[junit] ------------- ---------------- ---------------
[junit] Testcase:
testChangeMinIndexInterval(org.apache.cassandra.io.sstable.IndexSummaryManagerTest):
FAILED
[junit] expected:<16.0> but was:<8.0>
[junit] junit.framework.AssertionFailedError: expected:<16.0> but was:<8.0>
[junit] at
org.apache.cassandra.io.sstable.IndexSummaryManagerTest.testChangeMinIndexInterval(IndexSummaryManagerTest.java:248)
[junit]
[junit]
[junit] Testcase:
testChangeMaxIndexInterval(org.apache.cassandra.io.sstable.IndexSummaryManagerTest):
FAILED
[junit] null
[junit] junit.framework.AssertionFailedError
[junit] at
org.apache.cassandra.io.sstable.SSTableReader.setReplacedBy(SSTableReader.java:875)
[junit] at
org.apache.cassandra.io.sstable.SSTableReader.cloneWithNewSummarySamplingLevel(SSTableReader.java:996)
[junit] at
org.apache.cassandra.io.sstable.IndexSummaryManager.adjustSamplingLevels(IndexSummaryManager.java:410)
[junit] at
org.apache.cassandra.io.sstable.IndexSummaryManager.redistributeSummaries(IndexSummaryManager.java:288)
[junit] at
org.apache.cassandra.io.sstable.IndexSummaryManagerTest.testChangeMaxIndexInterval(IndexSummaryManagerTest.java:319)
[junit]
[junit]
[junit] Testcase:
testRedistributeSummaries(org.apache.cassandra.io.sstable.IndexSummaryManagerTest):
FAILED
[junit]
[SSTableReader(path='/home/jake/workspace/cassandra/build/test/cassandra/data:0/Keyspace1/StandardLowIndexInterval-15bf0780be4111e4869db7d84572b531/Keyspace1-StandardLowIndexInterval-ka-2-Data.db')]
[junit] junit.framework.AssertionFailedError:
[SSTableReader(path='/home/jake/workspace/cassandra/build/test/cassandra/data:0/Keyspace1/StandardLowIndexInterval-15bf0780be4111e4869db7d84572b531/Keyspace1-StandardLowIndexInterval-ka-2-Data.db')]
[junit] at
org.apache.cassandra.db.ColumnFamilyStore.discardSSTables(ColumnFamilyStore.java:2889)
[junit] at
org.apache.cassandra.db.ColumnFamilyStore$10.run(ColumnFamilyStore.java:2532)
[junit] at
java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471)
[junit] at
org.apache.cassandra.db.ColumnFamilyStore.runWithCompactionsDisabled(ColumnFamilyStore.java:2596)
[junit] at
org.apache.cassandra.db.ColumnFamilyStore.truncateBlocking(ColumnFamilyStore.java:2543)
[junit] at
org.apache.cassandra.io.sstable.IndexSummaryManagerTest.createSSTables(IndexSummaryManagerTest.java:128)
[junit] at
org.apache.cassandra.io.sstable.IndexSummaryManagerTest.testRedistributeSummaries(IndexSummaryManagerTest.java:346)
[junit] at java.util.concurrent.FutureTask.run(FutureTask.java:262)
[junit] at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
[junit] at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
[junit] at java.lang.Thread.run(Thread.java:745)
[junit]
[junit]
[junit] Testcase:
testRebuildAtSamplingLevel(org.apache.cassandra.io.sstable.IndexSummaryManagerTest):
FAILED
[junit] expected:<1> but was:<2>
[junit] junit.framework.AssertionFailedError: expected:<1> but was:<2>
[junit] at
org.apache.cassandra.io.sstable.IndexSummaryManagerTest.testRebuildAtSamplingLevel(IndexSummaryManagerTest.java:481)
[junit]
[junit]
[junit] Testcase:
testJMXFunctions(org.apache.cassandra.io.sstable.IndexSummaryManagerTest):
FAILED
[junit]
[SSTableReader(path='/home/jake/workspace/cassandra/build/test/cassandra/data:0/Keyspace1/StandardLowIndexInterval-15bf0780be4111e4869db7d84572b531/Keyspace1-StandardLowIndexInterval-ka-2-Data.db'),
SSTableReader(path='/home/jake/workspace/cassandra/build/test/cassandra/data:0/Keyspace1/StandardLowIndexInterval-15bf0780be4111e4869db7d84572b531/Keyspace1-StandardLowIndexInterval-ka-3-Data.db')]
[junit] junit.framework.AssertionFailedError:
[SSTableReader(path='/home/jake/workspace/cassandra/build/test/cassandra/data:0/Keyspace1/StandardLowIndexInterval-15bf0780be4111e4869db7d84572b531/Keyspace1-StandardLowIndexInterval-ka-2-Data.db'),
SSTableReader(path='/home/jake/workspace/cassandra/build/test/cassandra/data:0/Keyspace1/StandardLowIndexInterval-15bf0780be4111e4869db7d84572b531/Keyspace1-StandardLowIndexInterval-ka-3-Data.db')]
[junit] at
org.apache.cassandra.db.ColumnFamilyStore.discardSSTables(ColumnFamilyStore.java:2889)
[junit] at
org.apache.cassandra.db.ColumnFamilyStore$10.run(ColumnFamilyStore.java:2532)
[junit] at
java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471)
[junit] at
org.apache.cassandra.db.ColumnFamilyStore.runWithCompactionsDisabled(ColumnFamilyStore.java:2596)
[junit] at
org.apache.cassandra.db.ColumnFamilyStore.truncateBlocking(ColumnFamilyStore.java:2543)
[junit] at
org.apache.cassandra.io.sstable.IndexSummaryManagerTest.testJMXFunctions(IndexSummaryManagerTest.java:525)
[junit]
[junit]
[junit] Test org.apache.cassandra.io.sstable.IndexSummaryManagerTest FAILED
{code}
> Assertion error in 2.1.2: ERROR [IndexSummaryManager:1]
> -------------------------------------------------------
>
> Key: CASSANDRA-8689
> URL: https://issues.apache.org/jira/browse/CASSANDRA-8689
> Project: Cassandra
> Issue Type: Bug
> Reporter: Jeff Liu
> Assignee: Benedict
> Fix For: 2.1.4
>
> Attachments: 8689-test.txt
>
>
> After upgrading a 6 nodes cassandra from 2.1.0 to 2.1.2, start getting the
> following assertion error.
> {noformat}
> ERROR [IndexSummaryManager:1] 2015-01-26 20:55:40,451
> CassandraDaemon.java:153 - Exception in thread
> Thread[IndexSummaryManager:1,1,main]
> java.lang.AssertionError: null
> at org.apache.cassandra.io.util.Memory.size(Memory.java:307)
> ~[apache-cassandra-2.1.2.jar:2.1.2]
> at
> org.apache.cassandra.io.sstable.IndexSummary.getOffHeapSize(IndexSummary.java:192)
> ~[apache-cassandra-2.1.2.jar:2.1.2]
> at
> org.apache.cassandra.io.sstable.SSTableReader.getIndexSummaryOffHeapSize(SSTableReader.java:1070)
> ~[apache-cassandra-2.1.2.jar:2.1.2]
> at
> org.apache.cassandra.io.sstable.IndexSummaryManager.redistributeSummaries(IndexSummaryManager.java:292)
> ~[apache-cassandra-2.1.2.jar:2.1.2]
> at
> org.apache.cassandra.io.sstable.IndexSummaryManager.redistributeSummaries(IndexSummaryManager.java:238)
> ~[apache-cassandra-2.1.2.jar:2.1.2]
> at
> org.apache.cassandra.io.sstable.IndexSummaryManager$1.runMayThrow(IndexSummaryManager.java:139)
> ~[apache-cassandra-2.1.2.jar:2.1.2]
> at
> org.apache.cassandra.utils.WrappedRunnable.run(WrappedRunnable.java:28)
> ~[apache-cassandra-2.1.2.jar:2.1.2]
> at
> org.apache.cassandra.concurrent.DebuggableScheduledThreadPoolExecutor$UncomplainingRunnable.run(DebuggableScheduledThreadPoolExecutor.java:77)
> ~[apache-cassandra-2.1.2.jar:2.1.2]
> at
> java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471)
> [na:1.7.0_45]
> at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:304)
> [na:1.7.0_45]
> at
> java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:178)
> [na:1.7.0_45]
> at
> java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293)
> [na:1.7.0_45]
> at
> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
> [na:1.7.0_45]
> at
> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
> [na:1.7.0_45]
> at java.lang.Thread.run(Thread.java:744) [na:1.7.0_45]
> {noformat}
> cassandra service is still running despite the issue. Node has total 8G
> memory with 2G allocated to heap. We are basically running read queries to
> retrieve data out of cassandra.
--
This message was sent by Atlassian JIRA
(v6.3.4#6332)
