Blake Eggleston created CASSANDRA-11958:
-------------------------------------------
Summary: Strong self ref loop detected
Key: CASSANDRA-11958
URL: https://issues.apache.org/jira/browse/CASSANDRA-11958
Project: Cassandra
Issue Type: Bug
Reporter: Blake Eggleston
Saw this pop up in one of our tests:
{code}
09:34:53 ERROR [Strong-Reference-Leak-Detector:1] 2016-06-02 16:28:08,566
NoSpamLogger.java:97 - Strong self-ref loop detected
[/tmp/tier1/keyspace1/standard1-b56b52b128da11e6a2847ba8a8db3060/mb-97-big,
private org.apache.cassandra.utils.concurrent.Ref
org.apache.cassandra.io.sstable.format.SSTableReader$InstanceTidier.globalRef-org.apache.cassandra.utils.concurrent.Ref,
final org.apache.cassandra.utils.concurrent.Ref$State
org.apache.cassandra.utils.concurrent.Ref.state-org.apache.cassandra.utils.concurrent.Ref$State,
final org.apache.cassandra.utils.concurrent.Ref$GlobalState
org.apache.cassandra.utils.concurrent.Ref$State.globalState-org.apache.cassandra.utils.concurrent.Ref$GlobalState,
private final org.apache.cassandra.utils.concurrent.RefCounted$Tidy
org.apache.cassandra.utils.concurrent.Ref$GlobalState.tidy-org.apache.cassandra.io.sstable.format.SSTableReader$GlobalTidy,
private volatile java.lang.Runnable
org.apache.cassandra.io.sstable.format.SSTableReader$GlobalTidy.obsoletion-org.apache.cassandra.db.lifecycle.LogTransaction$SSTableTidier,
private final org.apache.cassandra.db.lifecycle.Tracker
org.apache.cassandra.db.lifecycle.LogTransaction$SSTableTidier.tracker-org.apache.cassandra.db.lifecycle.Tracker,
public final java.util.Collection
org.apache.cassandra.db.lifecycle.Tracker.subscribers-java.util.concurrent.CopyOnWriteArrayList,
public final java.util.Collection
org.apache.cassandra.db.lifecycle.Tracker.subscribers-org.apache.cassandra.db.compaction.CompactionStrategyManager,
private final org.apache.cassandra.db.ColumnFamilyStore
org.apache.cassandra.db.compaction.CompactionStrategyManager.cfs-org.apache.cassandra.db.ColumnFamilyStore,
public final org.apache.cassandra.db.Keyspace
org.apache.cassandra.db.ColumnFamilyStore.keyspace-org.apache.cassandra.db.Keyspace,
private final java.util.concurrent.ConcurrentMap
org.apache.cassandra.db.Keyspace.columnFamilyStores-java.util.concurrent.ConcurrentHashMap,
private final java.util.concurrent.ConcurrentMap
org.apache.cassandra.db.Keyspace.columnFamilyStores-org.apache.cassandra.db.ColumnFamilyStore,
private final org.apache.cassandra.db.lifecycle.Tracker
org.apache.cassandra.db.ColumnFamilyStore.data-org.apache.cassandra.db.lifecycle.Tracker,
final java.util.concurrent.atomic.AtomicReference
org.apache.cassandra.db.lifecycle.Tracker.view-java.util.concurrent.atomic.AtomicReference,
private volatile java.lang.Object
java.util.concurrent.atomic.AtomicReference.value-org.apache.cassandra.db.lifecycle.View,
public final java.util.List
org.apache.cassandra.db.lifecycle.View.liveMemtables-com.google.common.collect.SingletonImmutableList,
final transient java.lang.Object
com.google.common.collect.SingletonImmutableList.element-org.apache.cassandra.db.Memtable,
private final org.apache.cassandra.utils.memory.MemtableAllocator
org.apache.cassandra.db.Memtable.allocator-org.apache.cassandra.utils.memory.SlabAllocator,
private final org.apache.cassandra.utils.memory.MemtableAllocator$SubAllocator
org.apache.cassandra.utils.memory.MemtableAllocator.onHeap-org.apache.cassandra.utils.memory.MemtableAllocator$SubAllocator,
private final org.apache.cassandra.utils.memory.MemtablePool$SubPool
org.apache.cassandra.utils.memory.MemtableAllocator$SubAllocator.parent-org.apache.cassandra.utils.memory.MemtablePool$SubPool,
final org.apache.cassandra.utils.memory.MemtablePool
org.apache.cassandra.utils.memory.MemtablePool$SubPool.this$0-org.apache.cassandra.utils.memory.SlabPool,
final org.apache.cassandra.utils.memory.MemtableCleanerThread
org.apache.cassandra.utils.memory.MemtablePool.cleaner-org.apache.cassandra.utils.memory.MemtableCleanerThread,
private java.lang.ThreadGroup java.lang.Thread.group-java.lang.ThreadGroup,
private final java.lang.ThreadGroup
java.lang.ThreadGroup.parent-java.lang.ThreadGroup,
java.lang.Thread[] java.lang.ThreadGroup.threads-[Ljava.lang.Thread;,
java.lang.Thread[] java.lang.ThreadGroup.threads-java.lang.Thread,
private java.lang.Runnable
java.lang.Thread.target-java.util.concurrent.ThreadPoolExecutor$Worker,
final java.util.concurrent.ThreadPoolExecutor
java.util.concurrent.ThreadPoolExecutor$Worker.this$0-java.util.concurrent.ScheduledThreadPoolExecutor,
private final java.util.concurrent.BlockingQueue
java.util.concurrent.ThreadPoolExecutor.workQueue-java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue,
private final java.util.concurrent.BlockingQueue
java.util.concurrent.ThreadPoolExecutor.workQueue-java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask,
private java.util.concurrent.Callable
java.util.concurrent.FutureTask.callable-java.util.concurrent.Executors$RunnableAdapter,
final java.lang.Runnable
java.util.concurrent.Executors$RunnableAdapter.task-sun.rmi.transport.DGCImpl$1,
final sun.rmi.transport.DGCImpl
sun.rmi.transport.DGCImpl$1.this$0-sun.rmi.transport.DGCImpl,
private java.util.Map sun.rmi.transport.DGCImpl.leaseTable-java.util.HashMap,
transient java.util.HashMap$Node[]
java.util.HashMap.table-[Ljava.util.HashMap$Node;,
transient java.util.HashMap$Node[]
java.util.HashMap.table-java.util.HashMap$Node,
java.lang.Object
java.util.HashMap$Node.value-sun.rmi.transport.DGCImpl$LeaseInfo,
java.util.Set sun.rmi.transport.DGCImpl$LeaseInfo.notifySet-java.util.HashSet,
private transient java.util.HashMap java.util.HashSet.map-java.util.HashMap,
transient java.util.HashMap$Node[]
java.util.HashMap.table-[Ljava.util.HashMap$Node;,
transient java.util.HashMap$Node[]
java.util.HashMap.table-java.util.HashMap$Node,
final java.lang.Object java.util.HashMap$Node.key-sun.rmi.transport.Target,
private final sun.rmi.transport.WeakRef
sun.rmi.transport.Target.weakImpl-sun.rmi.transport.WeakRef,
private java.lang.Object
sun.rmi.transport.WeakRef.strongRef-javax.management.remote.rmi.RMIJRMPServerImpl,
private javax.management.MBeanServer
javax.management.remote.rmi.RMIServerImpl.mbeanServer-com.sun.jmx.mbeanserver.JmxMBeanServer,
private volatile javax.management.MBeanServer
com.sun.jmx.mbeanserver.JmxMBeanServer.mbsInterceptor-com.sun.jmx.interceptor.DefaultMBeanServerInterceptor,
private final transient com.sun.jmx.mbeanserver.Repository
com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.repository-com.sun.jmx.mbeanserver.Repository,
private final java.util.Map
com.sun.jmx.mbeanserver.Repository.domainTb-java.util.HashMap,
transient java.util.HashMap$Node[]
java.util.HashMap.table-[Ljava.util.HashMap$Node;,
transient java.util.HashMap$Node[]
java.util.HashMap.table-java.util.HashMap$Node,
java.lang.Object java.util.HashMap$Node.value-java.util.HashMap,
transient java.util.HashMap$Node[]
java.util.HashMap.table-[Ljava.util.HashMap$Node;,
transient java.util.HashMap$Node[]
java.util.HashMap.table-java.util.HashMap$Node,
java.lang.Object
java.util.HashMap$Node.value-com.sun.jmx.mbeanserver.NamedObject,
private final javax.management.DynamicMBean
com.sun.jmx.mbeanserver.NamedObject.object-com.sun.jmx.mbeanserver.StandardMBeanSupport,
private final java.lang.Object
com.sun.jmx.mbeanserver.MBeanSupport.resource-org.apache.cassandra.metrics.CassandraMetricsRegistry$JmxGauge,
private final com.codahale.metrics.Gauge
org.apache.cassandra.metrics.CassandraMetricsRegistry$JmxGauge.metric-org.apache.cassandra.metrics.ThreadPoolMetrics$4,
final java.util.concurrent.ThreadPoolExecutor
org.apache.cassandra.metrics.ThreadPoolMetrics$4.val$executor-org.apache.cassandra.concurrent.JMXConfigurableThreadPoolExecutor,
private final java.util.HashSet
java.util.concurrent.ThreadPoolExecutor.workers-java.util.HashSet,
private transient java.util.HashMap java.util.HashSet.map-java.util.HashMap,
transient java.util.HashMap$Node[]
java.util.HashMap.table-[Ljava.util.HashMap$Node;,
transient java.util.HashMap$Node[]
java.util.HashMap.table-java.util.HashMap$Node,
final java.lang.Object
java.util.HashMap$Node.key-java.util.concurrent.ThreadPoolExecutor$Worker,
final java.lang.Thread
java.util.concurrent.ThreadPoolExecutor$Worker.thread-java.lang.Thread,
private java.lang.ThreadGroup java.lang.Thread.group-java.lang.ThreadGroup,
java.lang.Thread[] java.lang.ThreadGroup.threads-[Ljava.lang.Thread;,
java.lang.Thread[] java.lang.ThreadGroup.threads-java.lang.Thread,
private java.lang.Runnable
java.lang.Thread.target-org.apache.cassandra.streaming.ConnectionHandler$IncomingMessageHandler,
protected final org.apache.cassandra.streaming.StreamSession
org.apache.cassandra.streaming.ConnectionHandler$MessageHandler.session-org.apache.cassandra.streaming.StreamSession,
private org.apache.cassandra.streaming.StreamResultFuture
org.apache.cassandra.streaming.StreamSession.streamResult-org.apache.cassandra.streaming.StreamResultFuture,
private final java.util.Collection
org.apache.cassandra.streaming.StreamResultFuture.eventListeners-java.util.concurrent.ConcurrentLinkedQueue,
private final java.util.Collection
org.apache.cassandra.streaming.StreamResultFuture.eventListeners-org.apache.cassandra.repair.LocalSyncTask,
private final com.google.common.util.concurrent.ExecutionList
com.google.common.util.concurrent.AbstractFuture.executionList-com.google.common.util.concurrent.ExecutionList,
private com.google.common.util.concurrent.ExecutionList$RunnableExecutorPair
com.google.common.util.concurrent.ExecutionList.runnables-com.google.common.util.concurrent.ExecutionList$RunnableExecutorPair,
final java.lang.Runnable
com.google.common.util.concurrent.ExecutionList$RunnableExecutorPair.runnable-com.google.common.util.concurrent.Futures$CombinedFuture$2,
final com.google.common.util.concurrent.Futures$CombinedFuture
com.google.common.util.concurrent.Futures$CombinedFuture$2.this$0-com.google.common.util.concurrent.Futures$CombinedFuture,
private final com.google.common.util.concurrent.ExecutionList
com.google.common.util.concurrent.AbstractFuture.executionList-com.google.common.util.concurrent.ExecutionList,
private com.google.common.util.concurrent.ExecutionList$RunnableExecutorPair
com.google.common.util.concurrent.ExecutionList.runnables-com.google.common.util.concurrent.ExecutionList$RunnableExecutorPair,
final java.lang.Runnable
com.google.common.util.concurrent.ExecutionList$RunnableExecutorPair.runnable-com.google.common.util.concurrent.Futures$ChainingListenableFuture$1,
final com.google.common.util.concurrent.Futures$ChainingListenableFuture
com.google.common.util.concurrent.Futures$ChainingListenableFuture$1.this$0-com.google.common.util.concurrent.Futures$ChainingListenableFuture,
private final com.google.common.util.concurrent.ExecutionList
com.google.common.util.concurrent.AbstractFuture.executionList-com.google.common.util.concurrent.ExecutionList,
private com.google.common.util.concurrent.ExecutionList$RunnableExecutorPair
com.google.common.util.concurrent.ExecutionList.runnables-com.google.common.util.concurrent.ExecutionList$RunnableExecutorPair,
final java.lang.Runnable
com.google.common.util.concurrent.ExecutionList$RunnableExecutorPair.runnable-com.google.common.util.concurrent.Futures$6,
final com.google.common.util.concurrent.FutureCallback
com.google.common.util.concurrent.Futures$6.val$callback-org.apache.cassandra.repair.RepairJob$3,
final org.apache.cassandra.repair.RepairJob
org.apache.cassandra.repair.RepairJob$3.this$0-org.apache.cassandra.repair.RepairJob,
private final org.apache.cassandra.repair.RepairSession
org.apache.cassandra.repair.RepairJob.session-org.apache.cassandra.repair.RepairSession,
private final com.google.common.util.concurrent.ExecutionList
com.google.common.util.concurrent.AbstractFuture.executionList-com.google.common.util.concurrent.ExecutionList,
private com.google.common.util.concurrent.ExecutionList$RunnableExecutorPair
com.google.common.util.concurrent.ExecutionList.runnables-com.google.common.util.concurrent.ExecutionList$RunnableExecutorPair,
com.google.common.util.concurrent.ExecutionList$RunnableExecutorPair
com.google.common.util.concurrent.ExecutionList$RunnableExecutorPair.next-com.google.common.util.concurrent.ExecutionList$RunnableExecutorPair,
com.google.common.util.concurrent.ExecutionList$RunnableExecutorPair
com.google.common.util.concurrent.ExecutionList$RunnableExecutorPair.next-com.google.common.util.concurrent.ExecutionList$RunnableExecutorPair,
final java.lang.Runnable
com.google.common.util.concurrent.ExecutionList$RunnableExecutorPair.runnable-org.apache.cassandra.service.ActiveRepairService$1,
final org.apache.cassandra.service.ActiveRepairService
org.apache.cassandra.service.ActiveRepairService$1.this$0-org.apache.cassandra.service.ActiveRepairService,
private final java.util.concurrent.ConcurrentMap
org.apache.cassandra.service.ActiveRepairService.parentRepairSessions-java.util.concurrent.ConcurrentHashMap,
private final java.util.concurrent.ConcurrentMap
org.apache.cassandra.service.ActiveRepairService.parentRepairSessions-org.apache.cassandra.service.ActiveRepairService$ParentRepairSession,
private final java.util.Map
org.apache.cassandra.service.ActiveRepairService$ParentRepairSession.sstableMap-java.util.HashMap,
transient java.util.HashMap$Node[]
java.util.HashMap.table-[Ljava.util.HashMap$Node;,
transient java.util.HashMap$Node[]
java.util.HashMap.table-java.util.HashMap$Node,
java.lang.Object java.util.HashMap$Node.value-java.util.HashSet,
private transient java.util.HashMap java.util.HashSet.map-java.util.HashMap,
transient java.util.HashMap$Node[]
java.util.HashMap.table-[Ljava.util.HashMap$Node;,
transient java.util.HashMap$Node[]
java.util.HashMap.table-java.util.HashMap$Node,
final java.lang.Object
java.util.HashMap$Node.key-org.apache.cassandra.io.sstable.format.big.BigTableReader,
private final org.apache.cassandra.utils.concurrent.Ref
org.apache.cassandra.io.sstable.format.SSTableReader.selfRef-org.apache.cassandra.utils.concurrent.Ref]
{code}
It looks like Ref.Visitor is run after an sstable has been obsoleted, and while
there's a repair task in a threadpool which is streaming (or waiting to stream)
the same sstable to another node. Strangely, the final few steps back to itself
happens through ParentRepairSession.sstableMap, which maps uuids to sets of
strings, so I'm not sure how that's possible.
--
This message was sent by Atlassian JIRA
(v6.3.4#6332)
