[
https://issues.apache.org/jira/browse/CASSANDRA-11958?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
Blake Eggleston updated CASSANDRA-11958:
----------------------------------------
Description:
Saw this pop up in one of our tests:
{code}
09:34:53 ERROR [Strong-Reference-Leak-Detector:1] 2016-06-02 16:28:08,566
NoSpamLogger.java:97 - Strong self-ref loop detected
[/tmp/tier1/keyspace1/standard1-b56b52b128da11e6a2847ba8a8db3060/mb-97-big,
private org.apache.cassandra.utils.concurrent.Ref
org.apache.cassandra.io.sstable.format.SSTableReader$InstanceTidier.globalRef-org.apache.cassandra.utils.concurrent.Ref,
final org.apache.cassandra.utils.concurrent.Ref$State
org.apache.cassandra.utils.concurrent.Ref.state-org.apache.cassandra.utils.concurrent.Ref$State,
final org.apache.cassandra.utils.concurrent.Ref$GlobalState
org.apache.cassandra.utils.concurrent.Ref$State.globalState-org.apache.cassandra.utils.concurrent.Ref$GlobalState,
private final org.apache.cassandra.utils.concurrent.RefCounted$Tidy
org.apache.cassandra.utils.concurrent.Ref$GlobalState.tidy-org.apache.cassandra.io.sstable.format.SSTableReader$GlobalTidy,
private volatile java.lang.Runnable
org.apache.cassandra.io.sstable.format.SSTableReader$GlobalTidy.obsoletion-org.apache.cassandra.db.lifecycle.LogTransaction$SSTableTidier,
private final org.apache.cassandra.db.lifecycle.Tracker
org.apache.cassandra.db.lifecycle.LogTransaction$SSTableTidier.tracker-org.apache.cassandra.db.lifecycle.Tracker,
public final java.util.Collection
org.apache.cassandra.db.lifecycle.Tracker.subscribers-java.util.concurrent.CopyOnWriteArrayList,
public final java.util.Collection
org.apache.cassandra.db.lifecycle.Tracker.subscribers-org.apache.cassandra.db.compaction.CompactionStrategyManager,
private final org.apache.cassandra.db.ColumnFamilyStore
org.apache.cassandra.db.compaction.CompactionStrategyManager.cfs-org.apache.cassandra.db.ColumnFamilyStore,
public final org.apache.cassandra.db.Keyspace
org.apache.cassandra.db.ColumnFamilyStore.keyspace-org.apache.cassandra.db.Keyspace,
private final java.util.concurrent.ConcurrentMap
org.apache.cassandra.db.Keyspace.columnFamilyStores-java.util.concurrent.ConcurrentHashMap,
private final java.util.concurrent.ConcurrentMap
org.apache.cassandra.db.Keyspace.columnFamilyStores-org.apache.cassandra.db.ColumnFamilyStore,
private final org.apache.cassandra.db.lifecycle.Tracker
org.apache.cassandra.db.ColumnFamilyStore.data-org.apache.cassandra.db.lifecycle.Tracker,
final java.util.concurrent.atomic.AtomicReference
org.apache.cassandra.db.lifecycle.Tracker.view-java.util.concurrent.atomic.AtomicReference,
private volatile java.lang.Object
java.util.concurrent.atomic.AtomicReference.value-org.apache.cassandra.db.lifecycle.View,
public final java.util.List
org.apache.cassandra.db.lifecycle.View.liveMemtables-com.google.common.collect.SingletonImmutableList,
final transient java.lang.Object
com.google.common.collect.SingletonImmutableList.element-org.apache.cassandra.db.Memtable,
private final org.apache.cassandra.utils.memory.MemtableAllocator
org.apache.cassandra.db.Memtable.allocator-org.apache.cassandra.utils.memory.SlabAllocator,
private final org.apache.cassandra.utils.memory.MemtableAllocator$SubAllocator
org.apache.cassandra.utils.memory.MemtableAllocator.onHeap-org.apache.cassandra.utils.memory.MemtableAllocator$SubAllocator,
private final org.apache.cassandra.utils.memory.MemtablePool$SubPool
org.apache.cassandra.utils.memory.MemtableAllocator$SubAllocator.parent-org.apache.cassandra.utils.memory.MemtablePool$SubPool,
final org.apache.cassandra.utils.memory.MemtablePool
org.apache.cassandra.utils.memory.MemtablePool$SubPool.this$0-org.apache.cassandra.utils.memory.SlabPool,
final org.apache.cassandra.utils.memory.MemtableCleanerThread
org.apache.cassandra.utils.memory.MemtablePool.cleaner-org.apache.cassandra.utils.memory.MemtableCleanerThread,
private java.lang.ThreadGroup java.lang.Thread.group-java.lang.ThreadGroup,
private final java.lang.ThreadGroup
java.lang.ThreadGroup.parent-java.lang.ThreadGroup,
java.lang.Thread[] java.lang.ThreadGroup.threads-[Ljava.lang.Thread;,
java.lang.Thread[] java.lang.ThreadGroup.threads-java.lang.Thread,
private java.lang.Runnable
java.lang.Thread.target-java.util.concurrent.ThreadPoolExecutor$Worker,
final java.util.concurrent.ThreadPoolExecutor
java.util.concurrent.ThreadPoolExecutor$Worker.this$0-java.util.concurrent.ScheduledThreadPoolExecutor,
private final java.util.concurrent.BlockingQueue
java.util.concurrent.ThreadPoolExecutor.workQueue-java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue,
private final java.util.concurrent.BlockingQueue
java.util.concurrent.ThreadPoolExecutor.workQueue-java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask,
private java.util.concurrent.Callable
java.util.concurrent.FutureTask.callable-java.util.concurrent.Executors$RunnableAdapter,
final java.lang.Runnable
java.util.concurrent.Executors$RunnableAdapter.task-sun.rmi.transport.DGCImpl$1,
final sun.rmi.transport.DGCImpl
sun.rmi.transport.DGCImpl$1.this$0-sun.rmi.transport.DGCImpl,
private java.util.Map sun.rmi.transport.DGCImpl.leaseTable-java.util.HashMap,
transient java.util.HashMap$Node[]
java.util.HashMap.table-[Ljava.util.HashMap$Node;,
transient java.util.HashMap$Node[]
java.util.HashMap.table-java.util.HashMap$Node,
java.lang.Object
java.util.HashMap$Node.value-sun.rmi.transport.DGCImpl$LeaseInfo,
java.util.Set sun.rmi.transport.DGCImpl$LeaseInfo.notifySet-java.util.HashSet,
private transient java.util.HashMap java.util.HashSet.map-java.util.HashMap,
transient java.util.HashMap$Node[]
java.util.HashMap.table-[Ljava.util.HashMap$Node;,
transient java.util.HashMap$Node[]
java.util.HashMap.table-java.util.HashMap$Node,
final java.lang.Object java.util.HashMap$Node.key-sun.rmi.transport.Target,
private final sun.rmi.transport.WeakRef
sun.rmi.transport.Target.weakImpl-sun.rmi.transport.WeakRef,
private java.lang.Object
sun.rmi.transport.WeakRef.strongRef-javax.management.remote.rmi.RMIJRMPServerImpl,
private javax.management.MBeanServer
javax.management.remote.rmi.RMIServerImpl.mbeanServer-com.sun.jmx.mbeanserver.JmxMBeanServer,
private volatile javax.management.MBeanServer
com.sun.jmx.mbeanserver.JmxMBeanServer.mbsInterceptor-com.sun.jmx.interceptor.DefaultMBeanServerInterceptor,
private final transient com.sun.jmx.mbeanserver.Repository
com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.repository-com.sun.jmx.mbeanserver.Repository,
private final java.util.Map
com.sun.jmx.mbeanserver.Repository.domainTb-java.util.HashMap,
transient java.util.HashMap$Node[]
java.util.HashMap.table-[Ljava.util.HashMap$Node;,
transient java.util.HashMap$Node[]
java.util.HashMap.table-java.util.HashMap$Node,
java.lang.Object java.util.HashMap$Node.value-java.util.HashMap,
transient java.util.HashMap$Node[]
java.util.HashMap.table-[Ljava.util.HashMap$Node;,
transient java.util.HashMap$Node[]
java.util.HashMap.table-java.util.HashMap$Node,
java.lang.Object
java.util.HashMap$Node.value-com.sun.jmx.mbeanserver.NamedObject,
private final javax.management.DynamicMBean
com.sun.jmx.mbeanserver.NamedObject.object-com.sun.jmx.mbeanserver.StandardMBeanSupport,
private final java.lang.Object
com.sun.jmx.mbeanserver.MBeanSupport.resource-org.apache.cassandra.metrics.CassandraMetricsRegistry$JmxGauge,
private final com.codahale.metrics.Gauge
org.apache.cassandra.metrics.CassandraMetricsRegistry$JmxGauge.metric-org.apache.cassandra.metrics.ThreadPoolMetrics$4,
final java.util.concurrent.ThreadPoolExecutor
org.apache.cassandra.metrics.ThreadPoolMetrics$4.val$executor-org.apache.cassandra.concurrent.JMXConfigurableThreadPoolExecutor,
private final java.util.HashSet
java.util.concurrent.ThreadPoolExecutor.workers-java.util.HashSet,
private transient java.util.HashMap java.util.HashSet.map-java.util.HashMap,
transient java.util.HashMap$Node[]
java.util.HashMap.table-[Ljava.util.HashMap$Node;,
transient java.util.HashMap$Node[]
java.util.HashMap.table-java.util.HashMap$Node,
final java.lang.Object
java.util.HashMap$Node.key-java.util.concurrent.ThreadPoolExecutor$Worker,
final java.lang.Thread
java.util.concurrent.ThreadPoolExecutor$Worker.thread-java.lang.Thread,
private java.lang.ThreadGroup java.lang.Thread.group-java.lang.ThreadGroup,
java.lang.Thread[] java.lang.ThreadGroup.threads-[Ljava.lang.Thread;,
java.lang.Thread[] java.lang.ThreadGroup.threads-java.lang.Thread,
private java.lang.Runnable
java.lang.Thread.target-org.apache.cassandra.streaming.ConnectionHandler$IncomingMessageHandler,
protected final org.apache.cassandra.streaming.StreamSession
org.apache.cassandra.streaming.ConnectionHandler$MessageHandler.session-org.apache.cassandra.streaming.StreamSession,
private org.apache.cassandra.streaming.StreamResultFuture
org.apache.cassandra.streaming.StreamSession.streamResult-org.apache.cassandra.streaming.StreamResultFuture,
private final java.util.Collection
org.apache.cassandra.streaming.StreamResultFuture.eventListeners-java.util.concurrent.ConcurrentLinkedQueue,
private final java.util.Collection
org.apache.cassandra.streaming.StreamResultFuture.eventListeners-org.apache.cassandra.repair.LocalSyncTask,
private final com.google.common.util.concurrent.ExecutionList
com.google.common.util.concurrent.AbstractFuture.executionList-com.google.common.util.concurrent.ExecutionList,
private com.google.common.util.concurrent.ExecutionList$RunnableExecutorPair
com.google.common.util.concurrent.ExecutionList.runnables-com.google.common.util.concurrent.ExecutionList$RunnableExecutorPair,
final java.lang.Runnable
com.google.common.util.concurrent.ExecutionList$RunnableExecutorPair.runnable-com.google.common.util.concurrent.Futures$CombinedFuture$2,
final com.google.common.util.concurrent.Futures$CombinedFuture
com.google.common.util.concurrent.Futures$CombinedFuture$2.this$0-com.google.common.util.concurrent.Futures$CombinedFuture,
private final com.google.common.util.concurrent.ExecutionList
com.google.common.util.concurrent.AbstractFuture.executionList-com.google.common.util.concurrent.ExecutionList,
private com.google.common.util.concurrent.ExecutionList$RunnableExecutorPair
com.google.common.util.concurrent.ExecutionList.runnables-com.google.common.util.concurrent.ExecutionList$RunnableExecutorPair,
final java.lang.Runnable
com.google.common.util.concurrent.ExecutionList$RunnableExecutorPair.runnable-com.google.common.util.concurrent.Futures$ChainingListenableFuture$1,
final com.google.common.util.concurrent.Futures$ChainingListenableFuture
com.google.common.util.concurrent.Futures$ChainingListenableFuture$1.this$0-com.google.common.util.concurrent.Futures$ChainingListenableFuture,
private final com.google.common.util.concurrent.ExecutionList
com.google.common.util.concurrent.AbstractFuture.executionList-com.google.common.util.concurrent.ExecutionList,
private com.google.common.util.concurrent.ExecutionList$RunnableExecutorPair
com.google.common.util.concurrent.ExecutionList.runnables-com.google.common.util.concurrent.ExecutionList$RunnableExecutorPair,
final java.lang.Runnable
com.google.common.util.concurrent.ExecutionList$RunnableExecutorPair.runnable-com.google.common.util.concurrent.Futures$6,
final com.google.common.util.concurrent.FutureCallback
com.google.common.util.concurrent.Futures$6.val$callback-org.apache.cassandra.repair.RepairJob$3,
final org.apache.cassandra.repair.RepairJob
org.apache.cassandra.repair.RepairJob$3.this$0-org.apache.cassandra.repair.RepairJob,
private final org.apache.cassandra.repair.RepairSession
org.apache.cassandra.repair.RepairJob.session-org.apache.cassandra.repair.RepairSession,
private final com.google.common.util.concurrent.ExecutionList
com.google.common.util.concurrent.AbstractFuture.executionList-com.google.common.util.concurrent.ExecutionList,
private com.google.common.util.concurrent.ExecutionList$RunnableExecutorPair
com.google.common.util.concurrent.ExecutionList.runnables-com.google.common.util.concurrent.ExecutionList$RunnableExecutorPair,
com.google.common.util.concurrent.ExecutionList$RunnableExecutorPair
com.google.common.util.concurrent.ExecutionList$RunnableExecutorPair.next-com.google.common.util.concurrent.ExecutionList$RunnableExecutorPair,
com.google.common.util.concurrent.ExecutionList$RunnableExecutorPair
com.google.common.util.concurrent.ExecutionList$RunnableExecutorPair.next-com.google.common.util.concurrent.ExecutionList$RunnableExecutorPair,
final java.lang.Runnable
com.google.common.util.concurrent.ExecutionList$RunnableExecutorPair.runnable-org.apache.cassandra.service.ActiveRepairService$1,
final org.apache.cassandra.service.ActiveRepairService
org.apache.cassandra.service.ActiveRepairService$1.this$0-org.apache.cassandra.service.ActiveRepairService,
private final java.util.concurrent.ConcurrentMap
org.apache.cassandra.service.ActiveRepairService.parentRepairSessions-java.util.concurrent.ConcurrentHashMap,
private final java.util.concurrent.ConcurrentMap
org.apache.cassandra.service.ActiveRepairService.parentRepairSessions-org.apache.cassandra.service.ActiveRepairService$ParentRepairSession,
private final java.util.Map
org.apache.cassandra.service.ActiveRepairService$ParentRepairSession.sstableMap-java.util.HashMap,
transient java.util.HashMap$Node[]
java.util.HashMap.table-[Ljava.util.HashMap$Node;,
transient java.util.HashMap$Node[]
java.util.HashMap.table-java.util.HashMap$Node,
java.lang.Object java.util.HashMap$Node.value-java.util.HashSet,
private transient java.util.HashMap java.util.HashSet.map-java.util.HashMap,
transient java.util.HashMap$Node[]
java.util.HashMap.table-[Ljava.util.HashMap$Node;,
transient java.util.HashMap$Node[]
java.util.HashMap.table-java.util.HashMap$Node,
final java.lang.Object
java.util.HashMap$Node.key-org.apache.cassandra.io.sstable.format.big.BigTableReader,
private final org.apache.cassandra.utils.concurrent.Ref
org.apache.cassandra.io.sstable.format.SSTableReader.selfRef-org.apache.cassandra.utils.concurrent.Ref]
{code}
It looks like Ref.Visitor is run after an sstable has been obsoleted, and while
there's a repair task in a threadpool which is streaming (or waiting to stream)
the same sstable to another node. Strangely, the final few steps back to itself
happens through ParentRepairSession.sstableMap, which maps uuids to sets of
strings, so I'm not sure how that's possible.
This happened on cassandra-3.0, 5a5d0a1
was:
Saw this pop up in one of our tests:
{code}
09:34:53 ERROR [Strong-Reference-Leak-Detector:1] 2016-06-02 16:28:08,566
NoSpamLogger.java:97 - Strong self-ref loop detected
[/tmp/tier1/keyspace1/standard1-b56b52b128da11e6a2847ba8a8db3060/mb-97-big,
private org.apache.cassandra.utils.concurrent.Ref
org.apache.cassandra.io.sstable.format.SSTableReader$InstanceTidier.globalRef-org.apache.cassandra.utils.concurrent.Ref,
final org.apache.cassandra.utils.concurrent.Ref$State
org.apache.cassandra.utils.concurrent.Ref.state-org.apache.cassandra.utils.concurrent.Ref$State,
final org.apache.cassandra.utils.concurrent.Ref$GlobalState
org.apache.cassandra.utils.concurrent.Ref$State.globalState-org.apache.cassandra.utils.concurrent.Ref$GlobalState,
private final org.apache.cassandra.utils.concurrent.RefCounted$Tidy
org.apache.cassandra.utils.concurrent.Ref$GlobalState.tidy-org.apache.cassandra.io.sstable.format.SSTableReader$GlobalTidy,
private volatile java.lang.Runnable
org.apache.cassandra.io.sstable.format.SSTableReader$GlobalTidy.obsoletion-org.apache.cassandra.db.lifecycle.LogTransaction$SSTableTidier,
private final org.apache.cassandra.db.lifecycle.Tracker
org.apache.cassandra.db.lifecycle.LogTransaction$SSTableTidier.tracker-org.apache.cassandra.db.lifecycle.Tracker,
public final java.util.Collection
org.apache.cassandra.db.lifecycle.Tracker.subscribers-java.util.concurrent.CopyOnWriteArrayList,
public final java.util.Collection
org.apache.cassandra.db.lifecycle.Tracker.subscribers-org.apache.cassandra.db.compaction.CompactionStrategyManager,
private final org.apache.cassandra.db.ColumnFamilyStore
org.apache.cassandra.db.compaction.CompactionStrategyManager.cfs-org.apache.cassandra.db.ColumnFamilyStore,
public final org.apache.cassandra.db.Keyspace
org.apache.cassandra.db.ColumnFamilyStore.keyspace-org.apache.cassandra.db.Keyspace,
private final java.util.concurrent.ConcurrentMap
org.apache.cassandra.db.Keyspace.columnFamilyStores-java.util.concurrent.ConcurrentHashMap,
private final java.util.concurrent.ConcurrentMap
org.apache.cassandra.db.Keyspace.columnFamilyStores-org.apache.cassandra.db.ColumnFamilyStore,
private final org.apache.cassandra.db.lifecycle.Tracker
org.apache.cassandra.db.ColumnFamilyStore.data-org.apache.cassandra.db.lifecycle.Tracker,
final java.util.concurrent.atomic.AtomicReference
org.apache.cassandra.db.lifecycle.Tracker.view-java.util.concurrent.atomic.AtomicReference,
private volatile java.lang.Object
java.util.concurrent.atomic.AtomicReference.value-org.apache.cassandra.db.lifecycle.View,
public final java.util.List
org.apache.cassandra.db.lifecycle.View.liveMemtables-com.google.common.collect.SingletonImmutableList,
final transient java.lang.Object
com.google.common.collect.SingletonImmutableList.element-org.apache.cassandra.db.Memtable,
private final org.apache.cassandra.utils.memory.MemtableAllocator
org.apache.cassandra.db.Memtable.allocator-org.apache.cassandra.utils.memory.SlabAllocator,
private final org.apache.cassandra.utils.memory.MemtableAllocator$SubAllocator
org.apache.cassandra.utils.memory.MemtableAllocator.onHeap-org.apache.cassandra.utils.memory.MemtableAllocator$SubAllocator,
private final org.apache.cassandra.utils.memory.MemtablePool$SubPool
org.apache.cassandra.utils.memory.MemtableAllocator$SubAllocator.parent-org.apache.cassandra.utils.memory.MemtablePool$SubPool,
final org.apache.cassandra.utils.memory.MemtablePool
org.apache.cassandra.utils.memory.MemtablePool$SubPool.this$0-org.apache.cassandra.utils.memory.SlabPool,
final org.apache.cassandra.utils.memory.MemtableCleanerThread
org.apache.cassandra.utils.memory.MemtablePool.cleaner-org.apache.cassandra.utils.memory.MemtableCleanerThread,
private java.lang.ThreadGroup java.lang.Thread.group-java.lang.ThreadGroup,
private final java.lang.ThreadGroup
java.lang.ThreadGroup.parent-java.lang.ThreadGroup,
java.lang.Thread[] java.lang.ThreadGroup.threads-[Ljava.lang.Thread;,
java.lang.Thread[] java.lang.ThreadGroup.threads-java.lang.Thread,
private java.lang.Runnable
java.lang.Thread.target-java.util.concurrent.ThreadPoolExecutor$Worker,
final java.util.concurrent.ThreadPoolExecutor
java.util.concurrent.ThreadPoolExecutor$Worker.this$0-java.util.concurrent.ScheduledThreadPoolExecutor,
private final java.util.concurrent.BlockingQueue
java.util.concurrent.ThreadPoolExecutor.workQueue-java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue,
private final java.util.concurrent.BlockingQueue
java.util.concurrent.ThreadPoolExecutor.workQueue-java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask,
private java.util.concurrent.Callable
java.util.concurrent.FutureTask.callable-java.util.concurrent.Executors$RunnableAdapter,
final java.lang.Runnable
java.util.concurrent.Executors$RunnableAdapter.task-sun.rmi.transport.DGCImpl$1,
final sun.rmi.transport.DGCImpl
sun.rmi.transport.DGCImpl$1.this$0-sun.rmi.transport.DGCImpl,
private java.util.Map sun.rmi.transport.DGCImpl.leaseTable-java.util.HashMap,
transient java.util.HashMap$Node[]
java.util.HashMap.table-[Ljava.util.HashMap$Node;,
transient java.util.HashMap$Node[]
java.util.HashMap.table-java.util.HashMap$Node,
java.lang.Object
java.util.HashMap$Node.value-sun.rmi.transport.DGCImpl$LeaseInfo,
java.util.Set sun.rmi.transport.DGCImpl$LeaseInfo.notifySet-java.util.HashSet,
private transient java.util.HashMap java.util.HashSet.map-java.util.HashMap,
transient java.util.HashMap$Node[]
java.util.HashMap.table-[Ljava.util.HashMap$Node;,
transient java.util.HashMap$Node[]
java.util.HashMap.table-java.util.HashMap$Node,
final java.lang.Object java.util.HashMap$Node.key-sun.rmi.transport.Target,
private final sun.rmi.transport.WeakRef
sun.rmi.transport.Target.weakImpl-sun.rmi.transport.WeakRef,
private java.lang.Object
sun.rmi.transport.WeakRef.strongRef-javax.management.remote.rmi.RMIJRMPServerImpl,
private javax.management.MBeanServer
javax.management.remote.rmi.RMIServerImpl.mbeanServer-com.sun.jmx.mbeanserver.JmxMBeanServer,
private volatile javax.management.MBeanServer
com.sun.jmx.mbeanserver.JmxMBeanServer.mbsInterceptor-com.sun.jmx.interceptor.DefaultMBeanServerInterceptor,
private final transient com.sun.jmx.mbeanserver.Repository
com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.repository-com.sun.jmx.mbeanserver.Repository,
private final java.util.Map
com.sun.jmx.mbeanserver.Repository.domainTb-java.util.HashMap,
transient java.util.HashMap$Node[]
java.util.HashMap.table-[Ljava.util.HashMap$Node;,
transient java.util.HashMap$Node[]
java.util.HashMap.table-java.util.HashMap$Node,
java.lang.Object java.util.HashMap$Node.value-java.util.HashMap,
transient java.util.HashMap$Node[]
java.util.HashMap.table-[Ljava.util.HashMap$Node;,
transient java.util.HashMap$Node[]
java.util.HashMap.table-java.util.HashMap$Node,
java.lang.Object
java.util.HashMap$Node.value-com.sun.jmx.mbeanserver.NamedObject,
private final javax.management.DynamicMBean
com.sun.jmx.mbeanserver.NamedObject.object-com.sun.jmx.mbeanserver.StandardMBeanSupport,
private final java.lang.Object
com.sun.jmx.mbeanserver.MBeanSupport.resource-org.apache.cassandra.metrics.CassandraMetricsRegistry$JmxGauge,
private final com.codahale.metrics.Gauge
org.apache.cassandra.metrics.CassandraMetricsRegistry$JmxGauge.metric-org.apache.cassandra.metrics.ThreadPoolMetrics$4,
final java.util.concurrent.ThreadPoolExecutor
org.apache.cassandra.metrics.ThreadPoolMetrics$4.val$executor-org.apache.cassandra.concurrent.JMXConfigurableThreadPoolExecutor,
private final java.util.HashSet
java.util.concurrent.ThreadPoolExecutor.workers-java.util.HashSet,
private transient java.util.HashMap java.util.HashSet.map-java.util.HashMap,
transient java.util.HashMap$Node[]
java.util.HashMap.table-[Ljava.util.HashMap$Node;,
transient java.util.HashMap$Node[]
java.util.HashMap.table-java.util.HashMap$Node,
final java.lang.Object
java.util.HashMap$Node.key-java.util.concurrent.ThreadPoolExecutor$Worker,
final java.lang.Thread
java.util.concurrent.ThreadPoolExecutor$Worker.thread-java.lang.Thread,
private java.lang.ThreadGroup java.lang.Thread.group-java.lang.ThreadGroup,
java.lang.Thread[] java.lang.ThreadGroup.threads-[Ljava.lang.Thread;,
java.lang.Thread[] java.lang.ThreadGroup.threads-java.lang.Thread,
private java.lang.Runnable
java.lang.Thread.target-org.apache.cassandra.streaming.ConnectionHandler$IncomingMessageHandler,
protected final org.apache.cassandra.streaming.StreamSession
org.apache.cassandra.streaming.ConnectionHandler$MessageHandler.session-org.apache.cassandra.streaming.StreamSession,
private org.apache.cassandra.streaming.StreamResultFuture
org.apache.cassandra.streaming.StreamSession.streamResult-org.apache.cassandra.streaming.StreamResultFuture,
private final java.util.Collection
org.apache.cassandra.streaming.StreamResultFuture.eventListeners-java.util.concurrent.ConcurrentLinkedQueue,
private final java.util.Collection
org.apache.cassandra.streaming.StreamResultFuture.eventListeners-org.apache.cassandra.repair.LocalSyncTask,
private final com.google.common.util.concurrent.ExecutionList
com.google.common.util.concurrent.AbstractFuture.executionList-com.google.common.util.concurrent.ExecutionList,
private com.google.common.util.concurrent.ExecutionList$RunnableExecutorPair
com.google.common.util.concurrent.ExecutionList.runnables-com.google.common.util.concurrent.ExecutionList$RunnableExecutorPair,
final java.lang.Runnable
com.google.common.util.concurrent.ExecutionList$RunnableExecutorPair.runnable-com.google.common.util.concurrent.Futures$CombinedFuture$2,
final com.google.common.util.concurrent.Futures$CombinedFuture
com.google.common.util.concurrent.Futures$CombinedFuture$2.this$0-com.google.common.util.concurrent.Futures$CombinedFuture,
private final com.google.common.util.concurrent.ExecutionList
com.google.common.util.concurrent.AbstractFuture.executionList-com.google.common.util.concurrent.ExecutionList,
private com.google.common.util.concurrent.ExecutionList$RunnableExecutorPair
com.google.common.util.concurrent.ExecutionList.runnables-com.google.common.util.concurrent.ExecutionList$RunnableExecutorPair,
final java.lang.Runnable
com.google.common.util.concurrent.ExecutionList$RunnableExecutorPair.runnable-com.google.common.util.concurrent.Futures$ChainingListenableFuture$1,
final com.google.common.util.concurrent.Futures$ChainingListenableFuture
com.google.common.util.concurrent.Futures$ChainingListenableFuture$1.this$0-com.google.common.util.concurrent.Futures$ChainingListenableFuture,
private final com.google.common.util.concurrent.ExecutionList
com.google.common.util.concurrent.AbstractFuture.executionList-com.google.common.util.concurrent.ExecutionList,
private com.google.common.util.concurrent.ExecutionList$RunnableExecutorPair
com.google.common.util.concurrent.ExecutionList.runnables-com.google.common.util.concurrent.ExecutionList$RunnableExecutorPair,
final java.lang.Runnable
com.google.common.util.concurrent.ExecutionList$RunnableExecutorPair.runnable-com.google.common.util.concurrent.Futures$6,
final com.google.common.util.concurrent.FutureCallback
com.google.common.util.concurrent.Futures$6.val$callback-org.apache.cassandra.repair.RepairJob$3,
final org.apache.cassandra.repair.RepairJob
org.apache.cassandra.repair.RepairJob$3.this$0-org.apache.cassandra.repair.RepairJob,
private final org.apache.cassandra.repair.RepairSession
org.apache.cassandra.repair.RepairJob.session-org.apache.cassandra.repair.RepairSession,
private final com.google.common.util.concurrent.ExecutionList
com.google.common.util.concurrent.AbstractFuture.executionList-com.google.common.util.concurrent.ExecutionList,
private com.google.common.util.concurrent.ExecutionList$RunnableExecutorPair
com.google.common.util.concurrent.ExecutionList.runnables-com.google.common.util.concurrent.ExecutionList$RunnableExecutorPair,
com.google.common.util.concurrent.ExecutionList$RunnableExecutorPair
com.google.common.util.concurrent.ExecutionList$RunnableExecutorPair.next-com.google.common.util.concurrent.ExecutionList$RunnableExecutorPair,
com.google.common.util.concurrent.ExecutionList$RunnableExecutorPair
com.google.common.util.concurrent.ExecutionList$RunnableExecutorPair.next-com.google.common.util.concurrent.ExecutionList$RunnableExecutorPair,
final java.lang.Runnable
com.google.common.util.concurrent.ExecutionList$RunnableExecutorPair.runnable-org.apache.cassandra.service.ActiveRepairService$1,
final org.apache.cassandra.service.ActiveRepairService
org.apache.cassandra.service.ActiveRepairService$1.this$0-org.apache.cassandra.service.ActiveRepairService,
private final java.util.concurrent.ConcurrentMap
org.apache.cassandra.service.ActiveRepairService.parentRepairSessions-java.util.concurrent.ConcurrentHashMap,
private final java.util.concurrent.ConcurrentMap
org.apache.cassandra.service.ActiveRepairService.parentRepairSessions-org.apache.cassandra.service.ActiveRepairService$ParentRepairSession,
private final java.util.Map
org.apache.cassandra.service.ActiveRepairService$ParentRepairSession.sstableMap-java.util.HashMap,
transient java.util.HashMap$Node[]
java.util.HashMap.table-[Ljava.util.HashMap$Node;,
transient java.util.HashMap$Node[]
java.util.HashMap.table-java.util.HashMap$Node,
java.lang.Object java.util.HashMap$Node.value-java.util.HashSet,
private transient java.util.HashMap java.util.HashSet.map-java.util.HashMap,
transient java.util.HashMap$Node[]
java.util.HashMap.table-[Ljava.util.HashMap$Node;,
transient java.util.HashMap$Node[]
java.util.HashMap.table-java.util.HashMap$Node,
final java.lang.Object
java.util.HashMap$Node.key-org.apache.cassandra.io.sstable.format.big.BigTableReader,
private final org.apache.cassandra.utils.concurrent.Ref
org.apache.cassandra.io.sstable.format.SSTableReader.selfRef-org.apache.cassandra.utils.concurrent.Ref]
{code}
It looks like Ref.Visitor is run after an sstable has been obsoleted, and while
there's a repair task in a threadpool which is streaming (or waiting to stream)
the same sstable to another node. Strangely, the final few steps back to itself
happens through ParentRepairSession.sstableMap, which maps uuids to sets of
strings, so I'm not sure how that's possible.
> Strong self ref loop detected
> -----------------------------
>
> Key: CASSANDRA-11958
> URL: https://issues.apache.org/jira/browse/CASSANDRA-11958
> Project: Cassandra
> Issue Type: Bug
> Reporter: Blake Eggleston
>
> Saw this pop up in one of our tests:
> {code}
> 09:34:53 ERROR [Strong-Reference-Leak-Detector:1] 2016-06-02 16:28:08,566
> NoSpamLogger.java:97 - Strong self-ref loop detected
> [/tmp/tier1/keyspace1/standard1-b56b52b128da11e6a2847ba8a8db3060/mb-97-big,
> private org.apache.cassandra.utils.concurrent.Ref
> org.apache.cassandra.io.sstable.format.SSTableReader$InstanceTidier.globalRef-org.apache.cassandra.utils.concurrent.Ref,
>
> final org.apache.cassandra.utils.concurrent.Ref$State
> org.apache.cassandra.utils.concurrent.Ref.state-org.apache.cassandra.utils.concurrent.Ref$State,
>
> final org.apache.cassandra.utils.concurrent.Ref$GlobalState
> org.apache.cassandra.utils.concurrent.Ref$State.globalState-org.apache.cassandra.utils.concurrent.Ref$GlobalState,
>
> private final org.apache.cassandra.utils.concurrent.RefCounted$Tidy
> org.apache.cassandra.utils.concurrent.Ref$GlobalState.tidy-org.apache.cassandra.io.sstable.format.SSTableReader$GlobalTidy,
> private volatile java.lang.Runnable
> org.apache.cassandra.io.sstable.format.SSTableReader$GlobalTidy.obsoletion-org.apache.cassandra.db.lifecycle.LogTransaction$SSTableTidier,
> private final org.apache.cassandra.db.lifecycle.Tracker
> org.apache.cassandra.db.lifecycle.LogTransaction$SSTableTidier.tracker-org.apache.cassandra.db.lifecycle.Tracker,
> public final java.util.Collection
> org.apache.cassandra.db.lifecycle.Tracker.subscribers-java.util.concurrent.CopyOnWriteArrayList,
> public final java.util.Collection
> org.apache.cassandra.db.lifecycle.Tracker.subscribers-org.apache.cassandra.db.compaction.CompactionStrategyManager,
> private final org.apache.cassandra.db.ColumnFamilyStore
> org.apache.cassandra.db.compaction.CompactionStrategyManager.cfs-org.apache.cassandra.db.ColumnFamilyStore,
> public final org.apache.cassandra.db.Keyspace
> org.apache.cassandra.db.ColumnFamilyStore.keyspace-org.apache.cassandra.db.Keyspace,
> private final java.util.concurrent.ConcurrentMap
> org.apache.cassandra.db.Keyspace.columnFamilyStores-java.util.concurrent.ConcurrentHashMap,
> private final java.util.concurrent.ConcurrentMap
> org.apache.cassandra.db.Keyspace.columnFamilyStores-org.apache.cassandra.db.ColumnFamilyStore,
> private final org.apache.cassandra.db.lifecycle.Tracker
> org.apache.cassandra.db.ColumnFamilyStore.data-org.apache.cassandra.db.lifecycle.Tracker,
> final java.util.concurrent.atomic.AtomicReference
> org.apache.cassandra.db.lifecycle.Tracker.view-java.util.concurrent.atomic.AtomicReference,
> private volatile java.lang.Object
> java.util.concurrent.atomic.AtomicReference.value-org.apache.cassandra.db.lifecycle.View,
> public final java.util.List
> org.apache.cassandra.db.lifecycle.View.liveMemtables-com.google.common.collect.SingletonImmutableList,
> final transient java.lang.Object
> com.google.common.collect.SingletonImmutableList.element-org.apache.cassandra.db.Memtable,
> private final org.apache.cassandra.utils.memory.MemtableAllocator
> org.apache.cassandra.db.Memtable.allocator-org.apache.cassandra.utils.memory.SlabAllocator,
> private final
> org.apache.cassandra.utils.memory.MemtableAllocator$SubAllocator
> org.apache.cassandra.utils.memory.MemtableAllocator.onHeap-org.apache.cassandra.utils.memory.MemtableAllocator$SubAllocator,
> private final org.apache.cassandra.utils.memory.MemtablePool$SubPool
> org.apache.cassandra.utils.memory.MemtableAllocator$SubAllocator.parent-org.apache.cassandra.utils.memory.MemtablePool$SubPool,
> final org.apache.cassandra.utils.memory.MemtablePool
> org.apache.cassandra.utils.memory.MemtablePool$SubPool.this$0-org.apache.cassandra.utils.memory.SlabPool,
> final org.apache.cassandra.utils.memory.MemtableCleanerThread
> org.apache.cassandra.utils.memory.MemtablePool.cleaner-org.apache.cassandra.utils.memory.MemtableCleanerThread,
> private java.lang.ThreadGroup java.lang.Thread.group-java.lang.ThreadGroup,
> private final java.lang.ThreadGroup
> java.lang.ThreadGroup.parent-java.lang.ThreadGroup,
> java.lang.Thread[] java.lang.ThreadGroup.threads-[Ljava.lang.Thread;,
> java.lang.Thread[] java.lang.ThreadGroup.threads-java.lang.Thread,
> private java.lang.Runnable
> java.lang.Thread.target-java.util.concurrent.ThreadPoolExecutor$Worker,
> final java.util.concurrent.ThreadPoolExecutor
> java.util.concurrent.ThreadPoolExecutor$Worker.this$0-java.util.concurrent.ScheduledThreadPoolExecutor,
> private final java.util.concurrent.BlockingQueue
> java.util.concurrent.ThreadPoolExecutor.workQueue-java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue,
> private final java.util.concurrent.BlockingQueue
> java.util.concurrent.ThreadPoolExecutor.workQueue-java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask,
> private java.util.concurrent.Callable
> java.util.concurrent.FutureTask.callable-java.util.concurrent.Executors$RunnableAdapter,
> final java.lang.Runnable
> java.util.concurrent.Executors$RunnableAdapter.task-sun.rmi.transport.DGCImpl$1,
> final sun.rmi.transport.DGCImpl
> sun.rmi.transport.DGCImpl$1.this$0-sun.rmi.transport.DGCImpl,
> private java.util.Map sun.rmi.transport.DGCImpl.leaseTable-java.util.HashMap,
> transient java.util.HashMap$Node[]
> java.util.HashMap.table-[Ljava.util.HashMap$Node;,
> transient java.util.HashMap$Node[]
> java.util.HashMap.table-java.util.HashMap$Node,
> java.lang.Object
> java.util.HashMap$Node.value-sun.rmi.transport.DGCImpl$LeaseInfo,
> java.util.Set sun.rmi.transport.DGCImpl$LeaseInfo.notifySet-java.util.HashSet,
> private transient java.util.HashMap java.util.HashSet.map-java.util.HashMap,
> transient java.util.HashMap$Node[]
> java.util.HashMap.table-[Ljava.util.HashMap$Node;,
> transient java.util.HashMap$Node[]
> java.util.HashMap.table-java.util.HashMap$Node,
> final java.lang.Object java.util.HashMap$Node.key-sun.rmi.transport.Target,
> private final sun.rmi.transport.WeakRef
> sun.rmi.transport.Target.weakImpl-sun.rmi.transport.WeakRef,
> private java.lang.Object
> sun.rmi.transport.WeakRef.strongRef-javax.management.remote.rmi.RMIJRMPServerImpl,
> private javax.management.MBeanServer
> javax.management.remote.rmi.RMIServerImpl.mbeanServer-com.sun.jmx.mbeanserver.JmxMBeanServer,
> private volatile javax.management.MBeanServer
> com.sun.jmx.mbeanserver.JmxMBeanServer.mbsInterceptor-com.sun.jmx.interceptor.DefaultMBeanServerInterceptor,
> private final transient com.sun.jmx.mbeanserver.Repository
> com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.repository-com.sun.jmx.mbeanserver.Repository,
> private final java.util.Map
> com.sun.jmx.mbeanserver.Repository.domainTb-java.util.HashMap,
> transient java.util.HashMap$Node[]
> java.util.HashMap.table-[Ljava.util.HashMap$Node;,
> transient java.util.HashMap$Node[]
> java.util.HashMap.table-java.util.HashMap$Node,
> java.lang.Object java.util.HashMap$Node.value-java.util.HashMap,
> transient java.util.HashMap$Node[]
> java.util.HashMap.table-[Ljava.util.HashMap$Node;,
> transient java.util.HashMap$Node[]
> java.util.HashMap.table-java.util.HashMap$Node,
> java.lang.Object
> java.util.HashMap$Node.value-com.sun.jmx.mbeanserver.NamedObject,
> private final javax.management.DynamicMBean
> com.sun.jmx.mbeanserver.NamedObject.object-com.sun.jmx.mbeanserver.StandardMBeanSupport,
> private final java.lang.Object
> com.sun.jmx.mbeanserver.MBeanSupport.resource-org.apache.cassandra.metrics.CassandraMetricsRegistry$JmxGauge,
> private final com.codahale.metrics.Gauge
> org.apache.cassandra.metrics.CassandraMetricsRegistry$JmxGauge.metric-org.apache.cassandra.metrics.ThreadPoolMetrics$4,
> final java.util.concurrent.ThreadPoolExecutor
> org.apache.cassandra.metrics.ThreadPoolMetrics$4.val$executor-org.apache.cassandra.concurrent.JMXConfigurableThreadPoolExecutor,
> private final java.util.HashSet
> java.util.concurrent.ThreadPoolExecutor.workers-java.util.HashSet,
> private transient java.util.HashMap java.util.HashSet.map-java.util.HashMap,
> transient java.util.HashMap$Node[]
> java.util.HashMap.table-[Ljava.util.HashMap$Node;,
> transient java.util.HashMap$Node[]
> java.util.HashMap.table-java.util.HashMap$Node,
> final java.lang.Object
> java.util.HashMap$Node.key-java.util.concurrent.ThreadPoolExecutor$Worker,
> final java.lang.Thread
> java.util.concurrent.ThreadPoolExecutor$Worker.thread-java.lang.Thread,
> private java.lang.ThreadGroup java.lang.Thread.group-java.lang.ThreadGroup,
> java.lang.Thread[] java.lang.ThreadGroup.threads-[Ljava.lang.Thread;,
> java.lang.Thread[] java.lang.ThreadGroup.threads-java.lang.Thread,
> private java.lang.Runnable
> java.lang.Thread.target-org.apache.cassandra.streaming.ConnectionHandler$IncomingMessageHandler,
> protected final org.apache.cassandra.streaming.StreamSession
> org.apache.cassandra.streaming.ConnectionHandler$MessageHandler.session-org.apache.cassandra.streaming.StreamSession,
> private org.apache.cassandra.streaming.StreamResultFuture
> org.apache.cassandra.streaming.StreamSession.streamResult-org.apache.cassandra.streaming.StreamResultFuture,
> private final java.util.Collection
> org.apache.cassandra.streaming.StreamResultFuture.eventListeners-java.util.concurrent.ConcurrentLinkedQueue,
> private final java.util.Collection
> org.apache.cassandra.streaming.StreamResultFuture.eventListeners-org.apache.cassandra.repair.LocalSyncTask,
> private final com.google.common.util.concurrent.ExecutionList
> com.google.common.util.concurrent.AbstractFuture.executionList-com.google.common.util.concurrent.ExecutionList,
> private com.google.common.util.concurrent.ExecutionList$RunnableExecutorPair
> com.google.common.util.concurrent.ExecutionList.runnables-com.google.common.util.concurrent.ExecutionList$RunnableExecutorPair,
> final java.lang.Runnable
> com.google.common.util.concurrent.ExecutionList$RunnableExecutorPair.runnable-com.google.common.util.concurrent.Futures$CombinedFuture$2,
> final com.google.common.util.concurrent.Futures$CombinedFuture
> com.google.common.util.concurrent.Futures$CombinedFuture$2.this$0-com.google.common.util.concurrent.Futures$CombinedFuture,
> private final com.google.common.util.concurrent.ExecutionList
> com.google.common.util.concurrent.AbstractFuture.executionList-com.google.common.util.concurrent.ExecutionList,
> private com.google.common.util.concurrent.ExecutionList$RunnableExecutorPair
> com.google.common.util.concurrent.ExecutionList.runnables-com.google.common.util.concurrent.ExecutionList$RunnableExecutorPair,
> final java.lang.Runnable
> com.google.common.util.concurrent.ExecutionList$RunnableExecutorPair.runnable-com.google.common.util.concurrent.Futures$ChainingListenableFuture$1,
> final com.google.common.util.concurrent.Futures$ChainingListenableFuture
> com.google.common.util.concurrent.Futures$ChainingListenableFuture$1.this$0-com.google.common.util.concurrent.Futures$ChainingListenableFuture,
> private final com.google.common.util.concurrent.ExecutionList
> com.google.common.util.concurrent.AbstractFuture.executionList-com.google.common.util.concurrent.ExecutionList,
> private com.google.common.util.concurrent.ExecutionList$RunnableExecutorPair
> com.google.common.util.concurrent.ExecutionList.runnables-com.google.common.util.concurrent.ExecutionList$RunnableExecutorPair,
> final java.lang.Runnable
> com.google.common.util.concurrent.ExecutionList$RunnableExecutorPair.runnable-com.google.common.util.concurrent.Futures$6,
> final com.google.common.util.concurrent.FutureCallback
> com.google.common.util.concurrent.Futures$6.val$callback-org.apache.cassandra.repair.RepairJob$3,
> final org.apache.cassandra.repair.RepairJob
> org.apache.cassandra.repair.RepairJob$3.this$0-org.apache.cassandra.repair.RepairJob,
> private final org.apache.cassandra.repair.RepairSession
> org.apache.cassandra.repair.RepairJob.session-org.apache.cassandra.repair.RepairSession,
> private final com.google.common.util.concurrent.ExecutionList
> com.google.common.util.concurrent.AbstractFuture.executionList-com.google.common.util.concurrent.ExecutionList,
> private com.google.common.util.concurrent.ExecutionList$RunnableExecutorPair
> com.google.common.util.concurrent.ExecutionList.runnables-com.google.common.util.concurrent.ExecutionList$RunnableExecutorPair,
> com.google.common.util.concurrent.ExecutionList$RunnableExecutorPair
> com.google.common.util.concurrent.ExecutionList$RunnableExecutorPair.next-com.google.common.util.concurrent.ExecutionList$RunnableExecutorPair,
> com.google.common.util.concurrent.ExecutionList$RunnableExecutorPair
> com.google.common.util.concurrent.ExecutionList$RunnableExecutorPair.next-com.google.common.util.concurrent.ExecutionList$RunnableExecutorPair,
> final java.lang.Runnable
> com.google.common.util.concurrent.ExecutionList$RunnableExecutorPair.runnable-org.apache.cassandra.service.ActiveRepairService$1,
> final org.apache.cassandra.service.ActiveRepairService
> org.apache.cassandra.service.ActiveRepairService$1.this$0-org.apache.cassandra.service.ActiveRepairService,
> private final java.util.concurrent.ConcurrentMap
> org.apache.cassandra.service.ActiveRepairService.parentRepairSessions-java.util.concurrent.ConcurrentHashMap,
> private final java.util.concurrent.ConcurrentMap
> org.apache.cassandra.service.ActiveRepairService.parentRepairSessions-org.apache.cassandra.service.ActiveRepairService$ParentRepairSession,
> private final java.util.Map
> org.apache.cassandra.service.ActiveRepairService$ParentRepairSession.sstableMap-java.util.HashMap,
> transient java.util.HashMap$Node[]
> java.util.HashMap.table-[Ljava.util.HashMap$Node;,
> transient java.util.HashMap$Node[]
> java.util.HashMap.table-java.util.HashMap$Node,
> java.lang.Object java.util.HashMap$Node.value-java.util.HashSet,
> private transient java.util.HashMap java.util.HashSet.map-java.util.HashMap,
> transient java.util.HashMap$Node[]
> java.util.HashMap.table-[Ljava.util.HashMap$Node;,
> transient java.util.HashMap$Node[]
> java.util.HashMap.table-java.util.HashMap$Node,
> final java.lang.Object
> java.util.HashMap$Node.key-org.apache.cassandra.io.sstable.format.big.BigTableReader,
> private final org.apache.cassandra.utils.concurrent.Ref
> org.apache.cassandra.io.sstable.format.SSTableReader.selfRef-org.apache.cassandra.utils.concurrent.Ref]
> {code}
> It looks like Ref.Visitor is run after an sstable has been obsoleted, and
> while there's a repair task in a threadpool which is streaming (or waiting to
> stream) the same sstable to another node. Strangely, the final few steps back
> to itself happens through ParentRepairSession.sstableMap, which maps uuids to
> sets of strings, so I'm not sure how that's possible.
> This happened on cassandra-3.0, 5a5d0a1
--
This message was sent by Atlassian JIRA
(v6.3.4#6332)
