[
https://issues.apache.org/jira/browse/DAEMON-333?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=14571270#comment-14571270
]
Manuel Dominguez Sarmiento commented on DAEMON-333:
---------------------------------------------------
Full thread dump Java HotSpot(TM) 64-Bit Server VM (25.45-b02 mixed mode):
"RMI TCP Connection(283)-127.0.0.1" #801195 daemon prio=10 os_prio=0
tid=0x000000000b677000 nid=0xb4d6 runnable [0x00007f81f7749000]
java.lang.Thread.State: RUNNABLE
at java.net.SocketInputStream.socketRead0(Native Method)
at java.net.SocketInputStream.socketRead(SocketInputStream.java:116)
at java.net.SocketInputStream.read(SocketInputStream.java:170)
at java.net.SocketInputStream.read(SocketInputStream.java:141)
at java.io.BufferedInputStream.fill(BufferedInputStream.java:246)
at java.io.BufferedInputStream.read(BufferedInputStream.java:265)
- locked <0x00007f8376a2cc08> (a java.io.BufferedInputStream)
at java.io.FilterInputStream.read(FilterInputStream.java:83)
at
sun.rmi.transport.tcp.TCPTransport.handleMessages(TCPTransport.java:550)
at
sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.run0(TCPTransport.java:826)
at
sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.lambda$run$241(TCPTransport.java:683)
at
sun.rmi.transport.tcp.TCPTransport$ConnectionHandler$$Lambda$3/1498966449.run(Unknown
Source)
at java.security.AccessController.doPrivileged(Native Method)
at
sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.run(TCPTransport.java:682)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:745)
"oracle.jdbc.driver.BlockSource.ThreadedCachingBlockSource.BlockReleaser" #189
daemon prio=4 os_prio=0 tid=0x0000000010eec800 nid=0x68a1 in Object.wait()
[0x00007f81f6bc3000]
java.lang.Thread.State: TIMED_WAITING (on object monitor)
at java.lang.Object.wait(Native Method)
at
oracle.jdbc.driver.BlockSource$ThreadedCachingBlockSource$BlockReleaser.run(BlockSource.java:327)
- locked <0x00007f829c9ff180> (a
oracle.jdbc.driver.BlockSource$ThreadedCachingBlockSource$BlockReleaser)
"RMI RenewClean-[koji.renxo.net:1099]" #185 daemon prio=10 os_prio=0
tid=0x0000000009aba800 nid=0x550a in Object.wait() [0x00007f81f75c7000]
java.lang.Thread.State: TIMED_WAITING (on object monitor)
at java.lang.Object.wait(Native Method)
at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:143)
- locked <0x00007f829cbf7b58> (a java.lang.ref.ReferenceQueue$Lock)
at
sun.rmi.transport.DGCClient$EndpointEntry$RenewCleanThread.run(DGCClient.java:536)
at java.lang.Thread.run(Thread.java:745)
"RMI Scheduler(0)" #184 daemon prio=10 os_prio=0 tid=0x000000000a133800
nid=0x5509 runnable [0x00007f81f7648000]
java.lang.Thread.State: RUNNABLE
at java.lang.Thread.isInterrupted(Native Method)
at java.lang.Thread.interrupted(Thread.java:944)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.checkInterruptWhileWaiting(AbstractQueuedSynchronizer.java:2002)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2079)
at
java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
at
java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
at
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:745)
"agentsc-2-thread-1" #33 daemon prio=5 os_prio=0 tid=0x0000000005ba8000
nid=0x5428 waiting on condition [0x00007f81fcfac000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00007f8298183628> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at
java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
at
java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
at
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:745)
"TrackerProcessor" #32 daemon prio=5 os_prio=0 tid=0x0000000005ba9800
nid=0x5427 waiting on condition [0x00007f81fd02d000]
java.lang.Thread.State: WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00007f82a99fe140> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at
java.util.concurrent.ArrayBlockingQueue.take(ArrayBlockingQueue.java:403)
at
com.manageengine.apminsight.agent.trackers.TrackerProcessor.doExtract(TrackerProcessor.java:91)
at
com.manageengine.apminsight.agent.trackers.TrackerProcessor.doWork(TrackerProcessor.java:72)
at
com.manageengine.apminsight.agent.trackers.TrackerProcessor.run(TrackerProcessor.java:58)
"agentsc-1-thread-1" #31 daemon prio=5 os_prio=0 tid=0x0000000004e8b000
nid=0x5426 runnable [0x00007f81fd2ae000]
java.lang.Thread.State: RUNNABLE
at java.lang.Long.valueOf(Long.java:840)
at
com.manageengine.apminsight.agent.metrics.data.transport.MetricDataDispatcher.run(MetricDataDispatcher.java:49)
at
java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:308)
at
java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:180)
at
java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:294)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:745)
"GC Daemon" #28 daemon prio=2 os_prio=0 tid=0x0000000005537800 nid=0x5423 in
Object.wait() [0x00007f81fd5b0000]
java.lang.Thread.State: TIMED_WAITING (on object monitor)
at java.lang.Object.wait(Native Method)
at sun.misc.GC$Daemon.run(GC.java:117)
- locked <0x00007f829cc36860> (a sun.misc.GC$LatencyLock)
"Java2D Disposer" #27 daemon prio=10 os_prio=0 tid=0x0000000004c0f000
nid=0x5422 in Object.wait() [0x00007f81fd631000]
java.lang.Thread.State: WAITING (on object monitor)
at java.lang.Object.wait(Native Method)
at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:143)
- locked <0x00007f829cc18fb0> (a java.lang.ref.ReferenceQueue$Lock)
at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:164)
at sun.java2d.Disposer.run(Disposer.java:148)
at java.lang.Thread.run(Thread.java:745)
"Service Thread" #26 daemon prio=9 os_prio=0 tid=0x000000000463c000 nid=0x5420
runnable [0x0000000000000000]
java.lang.Thread.State: RUNNABLE
"C1 CompilerThread14" #25 daemon prio=9 os_prio=0 tid=0x000000000448a000
nid=0x541f waiting on condition [0x0000000000000000]
java.lang.Thread.State: RUNNABLE
"C1 CompilerThread13" #24 daemon prio=9 os_prio=0 tid=0x0000000004486800
nid=0x541e waiting on condition [0x0000000000000000]
java.lang.Thread.State: RUNNABLE
"C1 CompilerThread12" #23 daemon prio=9 os_prio=0 tid=0x0000000004472000
nid=0x541d runnable [0x0000000000000000]
java.lang.Thread.State: RUNNABLE
"C1 CompilerThread11" #22 daemon prio=9 os_prio=0 tid=0x0000000004465800
nid=0x541c runnable [0x0000000000000000]
java.lang.Thread.State: RUNNABLE
"C1 CompilerThread10" #21 daemon prio=9 os_prio=0 tid=0x0000000004467800
nid=0x541b waiting on condition [0x0000000000000000]
java.lang.Thread.State: RUNNABLE
"C2 CompilerThread9" #20 daemon prio=9 os_prio=0 tid=0x0000000004462800
nid=0x541a waiting on condition [0x0000000000000000]
java.lang.Thread.State: RUNNABLE
"C2 CompilerThread8" #19 daemon prio=9 os_prio=0 tid=0x0000000004451000
nid=0x5419 runnable [0x0000000000000000]
java.lang.Thread.State: RUNNABLE
"C2 CompilerThread7" #18 daemon prio=9 os_prio=0 tid=0x000000000443e800
nid=0x5418 waiting on condition [0x0000000000000000]
java.lang.Thread.State: RUNNABLE
"C2 CompilerThread6" #17 daemon prio=9 os_prio=0 tid=0x0000000004431800
nid=0x5417 waiting on condition [0x0000000000000000]
java.lang.Thread.State: RUNNABLE
"C2 CompilerThread5" #16 daemon prio=9 os_prio=0 tid=0x000000000441f800
nid=0x5416 waiting on condition [0x0000000000000000]
java.lang.Thread.State: RUNNABLE
"C2 CompilerThread4" #15 daemon prio=9 os_prio=0 tid=0x000000000440a800
nid=0x5415 waiting on condition [0x0000000000000000]
java.lang.Thread.State: RUNNABLE
"C2 CompilerThread3" #14 daemon prio=9 os_prio=0 tid=0x00000000043d8000
nid=0x5414 waiting on condition [0x0000000000000000]
java.lang.Thread.State: RUNNABLE
"C2 CompilerThread2" #13 daemon prio=9 os_prio=0 tid=0x00000000043b4000
nid=0x5413 waiting on condition [0x0000000000000000]
java.lang.Thread.State: RUNNABLE
"C2 CompilerThread1" #12 daemon prio=9 os_prio=0 tid=0x00000000043b1800
nid=0x5412 waiting on condition [0x0000000000000000]
java.lang.Thread.State: RUNNABLE
"C2 CompilerThread0" #11 daemon prio=9 os_prio=0 tid=0x00000000043a8800
nid=0x5411 waiting on condition [0x0000000000000000]
java.lang.Thread.State: RUNNABLE
"AsyncFileHandlerWriter-258952499" #8 daemon prio=5 os_prio=0
tid=0x000000000427f000 nid=0x540f runnable [0x00007f9d9808c000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00007f82980abe10> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at
java.util.concurrent.LinkedBlockingDeque.pollFirst(LinkedBlockingDeque.java:522)
at
java.util.concurrent.LinkedBlockingDeque.poll(LinkedBlockingDeque.java:684)
at
org.apache.juli.AsyncFileHandler$LoggerThread.run(AsyncFileHandler.java:144)
"Signal Dispatcher" #5 daemon prio=9 os_prio=0 tid=0x000000000419d800
nid=0x540e waiting on condition [0x0000000000000000]
java.lang.Thread.State: RUNNABLE
"Surrogate Locker Thread (Concurrent GC)" #4 daemon prio=9 os_prio=0
tid=0x000000000419b800 nid=0x540d waiting on condition [0x0000000000000000]
java.lang.Thread.State: RUNNABLE
"Finalizer" #3 daemon prio=8 os_prio=0 tid=0x000000000416b000 nid=0x540c in
Object.wait() [0x00007f9d982e2000]
java.lang.Thread.State: WAITING (on object monitor)
at java.lang.Object.wait(Native Method)
at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:143)
- locked <0x00007f8298013c68> (a java.lang.ref.ReferenceQueue$Lock)
at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:164)
at java.lang.ref.Finalizer$FinalizerThread.run(Finalizer.java:209)
"Reference Handler" #2 daemon prio=10 os_prio=0 tid=0x0000000004163000
nid=0x540b runnable [0x00007f9d98363000]
java.lang.Thread.State: RUNNABLE
at sun.misc.Unsafe.freeMemory(Native Method)
at java.nio.DirectByteBuffer$Deallocator.run(DirectByteBuffer.java:94)
at sun.misc.Cleaner.clean(Cleaner.java:143)
at java.lang.ref.Reference$ReferenceHandler.run(Reference.java:166)
"main" #1 prio=5 os_prio=0 tid=0x0000000001a39800 nid=0x53d2 runnable
[0x00007fff754a5000]
java.lang.Thread.State: RUNNABLE
at
sun.reflect.ReflectionFactory.newMethodAccessor(ReflectionFactory.java:157)
at java.lang.reflect.Method.acquireMethodAccessor(Method.java:563)
at java.lang.reflect.Method.invoke(Method.java:495)
at
org.apache.commons.daemon.support.DaemonLoader.destroy(DaemonLoader.java:289)
"VM Thread" os_prio=0 tid=0x000000000415e000 nid=0x540a runnable
"Gang worker#0 (Parallel GC Threads)" os_prio=0 tid=0x0000000001a50800
nid=0x53d3 runnable
"Gang worker#1 (Parallel GC Threads)" os_prio=0 tid=0x0000000001a52000
nid=0x53d4 runnable
"Gang worker#2 (Parallel GC Threads)" os_prio=0 tid=0x0000000001a54000
nid=0x53d5 runnable
"Gang worker#3 (Parallel GC Threads)" os_prio=0 tid=0x0000000001a55800
nid=0x53d6 runnable
"Gang worker#4 (Parallel GC Threads)" os_prio=0 tid=0x0000000001a57800
nid=0x53d7 runnable
"Gang worker#5 (Parallel GC Threads)" os_prio=0 tid=0x0000000001a59000
nid=0x53d8 runnable
"Gang worker#6 (Parallel GC Threads)" os_prio=0 tid=0x0000000001a5b000
nid=0x53d9 runnable
"Gang worker#7 (Parallel GC Threads)" os_prio=0 tid=0x0000000001a5c800
nid=0x53da runnable
"Gang worker#8 (Parallel GC Threads)" os_prio=0 tid=0x0000000001a5e800
nid=0x53db runnable
"Gang worker#9 (Parallel GC Threads)" os_prio=0 tid=0x0000000001a60000
nid=0x53dc runnable
"Gang worker#10 (Parallel GC Threads)" os_prio=0 tid=0x0000000001a62000
nid=0x53dd runnable
"Gang worker#11 (Parallel GC Threads)" os_prio=0 tid=0x0000000001a63800
nid=0x53de runnable
"Gang worker#12 (Parallel GC Threads)" os_prio=0 tid=0x0000000001a65800
nid=0x53df runnable
"Gang worker#13 (Parallel GC Threads)" os_prio=0 tid=0x0000000001a67000
nid=0x53e0 runnable
"Gang worker#14 (Parallel GC Threads)" os_prio=0 tid=0x0000000001a69000
nid=0x53e1 runnable
"Gang worker#15 (Parallel GC Threads)" os_prio=0 tid=0x0000000001a6a800
nid=0x53e2 runnable
"Gang worker#16 (Parallel GC Threads)" os_prio=0 tid=0x0000000001a6c800
nid=0x53e3 runnable
"Gang worker#17 (Parallel GC Threads)" os_prio=0 tid=0x0000000001a6e000
nid=0x53e4 runnable
"Gang worker#18 (Parallel GC Threads)" os_prio=0 tid=0x0000000001a70000
nid=0x53e5 runnable
"Gang worker#19 (Parallel GC Threads)" os_prio=0 tid=0x0000000001a71800
nid=0x53e6 runnable
"Gang worker#20 (Parallel GC Threads)" os_prio=0 tid=0x0000000001a73000
nid=0x53e7 runnable
"Gang worker#21 (Parallel GC Threads)" os_prio=0 tid=0x0000000001a75000
nid=0x53e8 runnable
"Gang worker#22 (Parallel GC Threads)" os_prio=0 tid=0x0000000001a76800
nid=0x53e9 runnable
"G1 Main Concurrent Mark GC Thread" os_prio=0 tid=0x0000000001ae9000 nid=0x5402
runnable
"Gang worker#0 (G1 Parallel Marking Threads)" os_prio=0 tid=0x0000000001aeb000
nid=0x5403 runnable
"Gang worker#1 (G1 Parallel Marking Threads)" os_prio=0 tid=0x0000000001aec800
nid=0x5404 runnable
"Gang worker#2 (G1 Parallel Marking Threads)" os_prio=0 tid=0x0000000001aee800
nid=0x5405 runnable
"Gang worker#3 (G1 Parallel Marking Threads)" os_prio=0 tid=0x0000000001af0000
nid=0x5406 runnable
"Gang worker#4 (G1 Parallel Marking Threads)" os_prio=0 tid=0x0000000001af2000
nid=0x5407 runnable
"Gang worker#5 (G1 Parallel Marking Threads)" os_prio=0 tid=0x0000000001af3800
nid=0x5408 runnable
"G1 Concurrent Refinement Thread#0" os_prio=0 tid=0x0000000001aa5000 nid=0x5401
runnable
"G1 Concurrent Refinement Thread#1" os_prio=0 tid=0x0000000001aa3000 nid=0x5400
runnable
"G1 Concurrent Refinement Thread#2" os_prio=0 tid=0x0000000001aa1000 nid=0x53ff
runnable
"G1 Concurrent Refinement Thread#3" os_prio=0 tid=0x0000000001a9f000 nid=0x53fe
runnable
"G1 Concurrent Refinement Thread#4" os_prio=0 tid=0x0000000001a9d800 nid=0x53fd
runnable
"G1 Concurrent Refinement Thread#5" os_prio=0 tid=0x0000000001a9b800 nid=0x53fc
runnable
"G1 Concurrent Refinement Thread#6" os_prio=0 tid=0x0000000001a99800 nid=0x53fb
runnable
"G1 Concurrent Refinement Thread#7" os_prio=0 tid=0x0000000001a98000 nid=0x53fa
runnable
"G1 Concurrent Refinement Thread#8" os_prio=0 tid=0x0000000001a96000 nid=0x53f9
runnable
"G1 Concurrent Refinement Thread#9" os_prio=0 tid=0x0000000001a94000 nid=0x53f8
runnable
"G1 Concurrent Refinement Thread#10" os_prio=0 tid=0x0000000001a92800
nid=0x53f7 runnable
"G1 Concurrent Refinement Thread#11" os_prio=0 tid=0x0000000001a90800
nid=0x53f6 runnable
"G1 Concurrent Refinement Thread#12" os_prio=0 tid=0x0000000001a8e800
nid=0x53f5 runnable
"G1 Concurrent Refinement Thread#13" os_prio=0 tid=0x0000000001a8c800
nid=0x53f4 runnable
"G1 Concurrent Refinement Thread#14" os_prio=0 tid=0x0000000001a8b000
nid=0x53f3 runnable
"G1 Concurrent Refinement Thread#15" os_prio=0 tid=0x0000000001a89000
nid=0x53f2 runnable
"G1 Concurrent Refinement Thread#16" os_prio=0 tid=0x0000000001a87000
nid=0x53f1 runnable
"G1 Concurrent Refinement Thread#17" os_prio=0 tid=0x0000000001a85800
nid=0x53f0 runnable
"G1 Concurrent Refinement Thread#18" os_prio=0 tid=0x0000000001a83800
nid=0x53ef runnable
"G1 Concurrent Refinement Thread#19" os_prio=0 tid=0x0000000001a81800
nid=0x53ee runnable
"G1 Concurrent Refinement Thread#20" os_prio=0 tid=0x0000000001a7f800
nid=0x53ed runnable
"G1 Concurrent Refinement Thread#21" os_prio=0 tid=0x0000000001a7e000
nid=0x53ec runnable
"G1 Concurrent Refinement Thread#22" os_prio=0 tid=0x0000000001a7c000
nid=0x53eb runnable
"G1 Concurrent Refinement Thread#23" os_prio=0 tid=0x0000000001a7a800
nid=0x53ea runnable
"String Deduplication Thread" os_prio=0 tid=0x00000000040e4000 nid=0x5409
runnable
"VM Periodic Task Thread" os_prio=0 tid=0x000000000463f000 nid=0x5421 waiting
on condition
> Stop abusing System.gc() on Commons Daemon jsvc shutdown
> --------------------------------------------------------
>
> Key: DAEMON-333
> URL: https://issues.apache.org/jira/browse/DAEMON-333
> Project: Commons Daemon
> Issue Type: Bug
> Components: Jsvc
> Affects Versions: 1.0.15
> Reporter: Manuel Dominguez Sarmiento
>
> The shutdown sequence invokes System.gc() several times. I'm not sure why
> this would be necessary, especially considering that System.gc() is only a
> hint and the JVM will not necessarily do anything when this method is
> invoked. Especially if -XX:+DisableExplicitGC is used, which is fairly common
> practice in production environments.
> In our case, we use rather huge heaps (over 100 GB) so shutdown can take
> forever, depending on how much uncollected garbage there is in the JVM heap.
> We've always used -XX:+DisableExplicitGC so this was never apparent, however
> we decided to stop using -XX:+DisableExplicitGC after careful code review
> showed none of our apps or supporting libraries are using System.gc() and
> sometimes we want to trigger GC manually from monitoring tools (Jconsole,
> VisualVM, etc.) - but we found the hard way that Commons Daemon / jsvc is
> using System.gc() on shutdown, preventing the speedy shutdown we were used to.
--
This message was sent by Atlassian JIRA
(v6.3.4#6332)
