This is a dump of it working: for
ajp-nio-8009-exec, I do not see any - locked <0x00000000f0a3f6b8> (a org.apache.tomcat.util.net.NioEndpoint$NioSocketWrapper) from the not working dump. possibly a result of stuff higher up in the chain? Cheers Greg Full thread dump Java HotSpot(TM) 64-Bit Server VM (25.40-b25 mixed mode): "pool-4-thread-5" #54 prio=5 os_prio=0 tid=0x000000000a6e5800 nid=0x2dfc waiting on condition [0x00002aef09792000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000000d233fca8> (a java.util.concurrent.SynchronousQueue$TransferStack) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) "pool-4-thread-4" #53 prio=5 os_prio=0 tid=0x000000000a834000 nid=0x2dc5 waiting on condition [0x00002aef09814000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000000d233fca8> (a java.util.concurrent.SynchronousQueue$TransferStack) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) "QuartzScheduler_scheduler-NON_CLUSTERED_MisfireHandler" #52 prio=5 os_prio=0 tid=0x000000000a692800 nid=0x2d67 waiting on condition [0x00002aef0a1ed000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.quartz.impl.jdbcjobstore.JobStoreSupport$MisfireHandler.run(JobStoreSupport.java:3992) "pool-3-thread-1" #51 prio=5 os_prio=0 tid=0x000000000a49a000 nid=0x2d66 waiting on condition [0x00002aef09855000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000000d21d62c8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) "ajp-nio-8009-exec-10" #45 daemon prio=5 os_prio=0 tid=0x000000000b45c800 nid=0x2d40 waiting on condition [0x00002aef09ea8000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000000d2cdb7a8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.tomcat.util.threads.TaskQueue.take(TaskQueue.java:103) at org.apache.tomcat.util.threads.TaskQueue.take(TaskQueue.java:31) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at org.apache.tomcat.util.threads.TaskThread$WrappingRunnable.run(TaskThread.java:61) at java.lang.Thread.run(Thread.java:745) "ajp-nio-8009-exec-9" #44 daemon prio=5 os_prio=0 tid=0x000000000b45c000 nid=0x2d3f waiting on condition [0x00002aef09e67000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000000d2cdb7a8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.tomcat.util.threads.TaskQueue.take(TaskQueue.java:103) at org.apache.tomcat.util.threads.TaskQueue.take(TaskQueue.java:31) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at org.apache.tomcat.util.threads.TaskThread$WrappingRunnable.run(TaskThread.java:61) at java.lang.Thread.run(Thread.java:745) "ajp-nio-8009-exec-8" #43 daemon prio=5 os_prio=0 tid=0x000000000a1d3000 nid=0x2d3e waiting on condition [0x00002aef09e26000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000000d2cdb7a8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.tomcat.util.threads.TaskQueue.take(TaskQueue.java:103) at org.apache.tomcat.util.threads.TaskQueue.take(TaskQueue.java:31) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at org.apache.tomcat.util.threads.TaskThread$WrappingRunnable.run(TaskThread.java:61) at java.lang.Thread.run(Thread.java:745) "ajp-nio-8009-exec-7" #42 daemon prio=5 os_prio=0 tid=0x000000000a1d1800 nid=0x2d3d waiting on condition [0x00002aef09de5000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000000d2cdb7a8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.tomcat.util.threads.TaskQueue.take(TaskQueue.java:103) at org.apache.tomcat.util.threads.TaskQueue.take(TaskQueue.java:31) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at org.apache.tomcat.util.threads.TaskThread$WrappingRunnable.run(TaskThread.java:61) at java.lang.Thread.run(Thread.java:745) "ajp-nio-8009-exec-6" #41 daemon prio=5 os_prio=0 tid=0x000000000a6e1000 nid=0x2d3b waiting on condition [0x00002aef09da4000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000000d2cdb7a8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.tomcat.util.threads.TaskQueue.take(TaskQueue.java:103) at org.apache.tomcat.util.threads.TaskQueue.take(TaskQueue.java:31) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at org.apache.tomcat.util.threads.TaskThread$WrappingRunnable.run(TaskThread.java:61) at java.lang.Thread.run(Thread.java:745) "ajp-nio-8009-exec-5" #40 daemon prio=5 os_prio=0 tid=0x000000000aa30800 nid=0x2d3a waiting on condition [0x00002aef056f4000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000000d2cdb7a8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.tomcat.util.threads.TaskQueue.take(TaskQueue.java:103) at org.apache.tomcat.util.threads.TaskQueue.take(TaskQueue.java:31) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at org.apache.tomcat.util.threads.TaskThread$WrappingRunnable.run(TaskThread.java:61) at java.lang.Thread.run(Thread.java:745) "ajp-nio-8009-exec-4" #39 daemon prio=5 os_prio=0 tid=0x000000000a375800 nid=0x2d38 waiting on condition [0x00002aef051e8000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000000d2cdb7a8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.tomcat.util.threads.TaskQueue.take(TaskQueue.java:103) at org.apache.tomcat.util.threads.TaskQueue.take(TaskQueue.java:31) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at org.apache.tomcat.util.threads.TaskThread$WrappingRunnable.run(TaskThread.java:61) at java.lang.Thread.run(Thread.java:745) "ajp-nio-8009-exec-3" #38 daemon prio=5 os_prio=0 tid=0x000000000a440800 nid=0x2d34 waiting on condition [0x00002aeefffd2000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000000d2cdb7a8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.tomcat.util.threads.TaskQueue.take(TaskQueue.java:103) at org.apache.tomcat.util.threads.TaskQueue.take(TaskQueue.java:31) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at org.apache.tomcat.util.threads.TaskThread$WrappingRunnable.run(TaskThread.java:61) at java.lang.Thread.run(Thread.java:745) "Thread-10" #37 daemon prio=5 os_prio=0 tid=0x000000000a290000 nid=0x2d30 waiting on condition [0x00002aef09961000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.events.business.runnable.ContinuousWorkerThread.run(ContinuousWorkerThread.java:106) "Statistics Thread-localCacheManager-1" #35 daemon prio=5 os_prio=0 tid=0x000000000b462800 nid=0x2d2d waiting on condition [0x00002aef09920000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000000d300b2a8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) "localCacheManager" #34 daemon prio=5 os_prio=0 tid=0x000000000ab8e800 nid=0x2d2c in Object.wait() [0x00002aef098df000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:502) at java.util.TimerThread.mainLoop(Timer.java:526) - locked <0x00000000d301e848> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) "ajp-nio-8009-exec-2" #33 daemon prio=5 os_prio=0 tid=0x000000000a1fa000 nid=0x2d29 waiting on condition [0x00002aef056b3000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000000d2cdb7a8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.tomcat.util.threads.TaskQueue.take(TaskQueue.java:103) at org.apache.tomcat.util.threads.TaskQueue.take(TaskQueue.java:31) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at org.apache.tomcat.util.threads.TaskThread$WrappingRunnable.run(TaskThread.java:61) at java.lang.Thread.run(Thread.java:745) "ajp-nio-8009-exec-1" #32 daemon prio=5 os_prio=0 tid=0x000000000a961000 nid=0x2d28 waiting on condition [0x00002aef05672000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000000d2cdb7a8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.tomcat.util.threads.TaskQueue.take(TaskQueue.java:103) at org.apache.tomcat.util.threads.TaskQueue.take(TaskQueue.java:31) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at org.apache.tomcat.util.threads.TaskThread$WrappingRunnable.run(TaskThread.java:61) at java.lang.Thread.run(Thread.java:745) "ajp-nio-8009-AsyncTimeout" #30 daemon prio=5 os_prio=0 tid=0x00002aef004c6000 nid=0x2d27 waiting on condition [0x00002aef05631000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.coyote.AbstractProtocol$AsyncTimeout.run(AbstractProtocol.java:1124) at java.lang.Thread.run(Thread.java:745) "ajp-nio-8009-Acceptor-0" #29 daemon prio=5 os_prio=0 tid=0x00002aef0024b800 nid=0x2d26 runnable [0x00002aef055f0000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250) - locked <0x00000000d0256d70> (a java.lang.Object) at org.apache.tomcat.util.net.NioEndpoint$Acceptor.run(NioEndpoint.java:444) at java.lang.Thread.run(Thread.java:745) "ajp-nio-8009-ClientPoller-1" #28 daemon prio=5 os_prio=0 tid=0x00002aef00447800 nid=0x2d25 runnable [0x00002aef055af000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00000000d2ca5f60> (a sun.nio.ch.Util$2) - locked <0x00000000d2ca5f50> (a java.util.Collections$UnmodifiableSet) - locked <0x00000000d2ca5e38> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.tomcat.util.net.NioEndpoint$Poller.run(NioEndpoint.java:778) at java.lang.Thread.run(Thread.java:745) "ajp-nio-8009-ClientPoller-0" #27 daemon prio=5 os_prio=0 tid=0x00002aef0025f800 nid=0x2d24 runnable [0x00002aef0556e000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00000000d2c55568> (a sun.nio.ch.Util$2) - locked <0x00000000d2c55558> (a java.util.Collections$UnmodifiableSet) - locked <0x00000000d2c55430> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.tomcat.util.net.NioEndpoint$Poller.run(NioEndpoint.java:778) at java.lang.Thread.run(Thread.java:745) "http-nio-8080-AsyncTimeout" #26 daemon prio=5 os_prio=0 tid=0x00002aef00444800 nid=0x2d23 waiting on condition [0x00002aef0552d000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.coyote.AbstractProtocol$AsyncTimeout.run(AbstractProtocol.java:1124) at java.lang.Thread.run(Thread.java:745) "http-nio-8080-Acceptor-0" #25 daemon prio=5 os_prio=0 tid=0x00002aef001ef800 nid=0x2d22 runnable [0x00002aef054ec000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250) - locked <0x00000000d0257fe8> (a java.lang.Object) at org.apache.tomcat.util.net.NioEndpoint$Acceptor.run(NioEndpoint.java:444) at java.lang.Thread.run(Thread.java:745) "http-nio-8080-ClientPoller-1" #24 daemon prio=5 os_prio=0 tid=0x00002aef0045d000 nid=0x2d21 runnable [0x00002aef054ab000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00000000d2e42df0> (a sun.nio.ch.Util$2) - locked <0x00000000d2e42de0> (a java.util.Collections$UnmodifiableSet) - locked <0x00000000d2e42cb8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.tomcat.util.net.NioEndpoint$Poller.run(NioEndpoint.java:778) at java.lang.Thread.run(Thread.java:745) "http-nio-8080-ClientPoller-0" #23 daemon prio=5 os_prio=0 tid=0x00002aef00256000 nid=0x2d20 runnable [0x00002aef0546a000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00000000d2e43a38> (a sun.nio.ch.Util$2) - locked <0x00000000d2e43a28> (a java.util.Collections$UnmodifiableSet) - locked <0x00000000d2e43900> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.tomcat.util.net.NioEndpoint$Poller.run(NioEndpoint.java:778) at java.lang.Thread.run(Thread.java:745) "ContainerBackgroundProcessor[StandardEngine[Catalina]]" #22 daemon prio=5 os_prio=0 tid=0x00002aef00457000 nid=0x2d1f waiting on condition [0x00002aef05429000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.catalina.core.ContainerBase$ContainerBackgroundProcessor.run(ContainerBase.java:1355) at java.lang.Thread.run(Thread.java:745) "Events Task Scheduler" #21 daemon prio=7 os_prio=0 tid=0x00002aef01948800 nid=0x2d10 waiting on condition [0x00002aef07d33000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.events.business.runnable.TaskScheduler.run(TaskScheduler.java:107) at java.lang.Thread.run(Thread.java:745) "scheduler_QuartzSchedulerThread" #19 prio=5 os_prio=0 tid=0x00002aef011a1800 nid=0x2d0b in Object.wait() [0x00002aef078b1000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at org.quartz.core.QuartzSchedulerThread.run(QuartzSchedulerThread.java:410) - locked <0x00000000d20a1f30> (a java.lang.Object) "scheduler_Worker-1" #18 prio=5 os_prio=0 tid=0x00002aef01194000 nid=0x2d0a in Object.wait() [0x00002aef07270000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at org.quartz.simpl.SimpleThreadPool$WorkerThread.run(SimpleThreadPool.java:568) - locked <0x00000000d2106f88> (a java.lang.Object) "Statistics Thread-eventsCacheManager-1" #16 daemon prio=5 os_prio=0 tid=0x00002aef0198d800 nid=0x2d01 waiting on condition [0x00002aef0722f000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000000d12b3bc0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) "eventsCacheManager" #15 daemon prio=5 os_prio=0 tid=0x00002aef00e0b800 nid=0x2cfe in Object.wait() [0x00002aef071ee000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:502) at java.util.TimerThread.mainLoop(Timer.java:526) - locked <0x00000000d12b4050> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) "NioBlockingSelector.BlockPoller-2" #12 daemon prio=5 os_prio=0 tid=0x00002aef003f7800 nid=0x2cdc runnable [0x00002aef051a7000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00000000d0240088> (a sun.nio.ch.Util$2) - locked <0x00000000d0240078> (a java.util.Collections$UnmodifiableSet) - locked <0x00000000d023ff50> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.tomcat.util.net.NioBlockingSelector$BlockPoller.run(NioBlockingSelector.java:339) "NioBlockingSelector.BlockPoller-1" #11 daemon prio=5 os_prio=0 tid=0x00002aef003f1800 nid=0x2cdb runnable [0x00002aef05166000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00000000d0242c70> (a sun.nio.ch.Util$2) - locked <0x00000000d0242c60> (a java.util.Collections$UnmodifiableSet) - locked <0x00000000d0242b48> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.tomcat.util.net.NioBlockingSelector$BlockPoller.run(NioBlockingSelector.java:339) "GC Daemon" #10 daemon prio=2 os_prio=0 tid=0x00002aef00367000 nid=0x2cda in Object.wait() [0x00002aeefff87000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x00000000d0620bc8> (a sun.misc.GC$LatencyLock) at sun.misc.GC$Daemon.run(GC.java:117) - locked <0x00000000d0620bc8> (a sun.misc.GC$LatencyLock) "Service Thread" #7 daemon prio=9 os_prio=0 tid=0x0000000009e5e800 nid=0x2cd8 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "C1 CompilerThread1" #6 daemon prio=9 os_prio=0 tid=0x0000000009e34000 nid=0x2cd7 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE "C2 CompilerThread0" #5 daemon prio=9 os_prio=0 tid=0x0000000009dff800 nid=0x2cd6 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Signal Dispatcher" #4 daemon prio=9 os_prio=0 tid=0x0000000009dfe000 nid=0x2cd5 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Finalizer" #3 daemon prio=8 os_prio=0 tid=0x0000000009dbf800 nid=0x2cd4 in Object.wait() [0x00002aeefbcdd000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:143) - locked <0x00000000d0161f58> (a java.lang.ref.ReferenceQueue$Lock) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:164) at java.lang.ref.Finalizer$FinalizerThread.run(Finalizer.java:209) "Reference Handler" #2 daemon prio=10 os_prio=0 tid=0x0000000009dbd800 nid=0x2cd3 in Object.wait() [0x00002aeefbc9c000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:502) at java.lang.ref.Reference$ReferenceHandler.run(Reference.java:157) - locked <0x00000000d05f6990> (a java.lang.ref.Reference$Lock) "main" #1 prio=5 os_prio=0 tid=0x0000000009d4f000 nid=0x2ccf runnable [0x00002aeee8c78000] java.lang.Thread.State: RUNNABLE at java.net.PlainSocketImpl.socketAccept(Native Method) at java.net.AbstractPlainSocketImpl.accept(AbstractPlainSocketImpl.java:404) at java.net.ServerSocket.implAccept(ServerSocket.java:545) at java.net.ServerSocket.accept(ServerSocket.java:513) at org.apache.catalina.core.StandardServer.await(StandardServer.java:466) at org.apache.catalina.startup.Catalina.await(Catalina.java:743) at org.apache.catalina.startup.Catalina.start(Catalina.java:689) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:497) at org.apache.catalina.startup.Bootstrap.start(Bootstrap.java:355) at org.apache.catalina.startup.Bootstrap.main(Bootstrap.java:495) "VM Thread" os_prio=0 tid=0x0000000009db8800 nid=0x2cd2 runnable "GC task thread#0 (ParallelGC)" os_prio=0 tid=0x0000000009d64000 nid=0x2cd0 runnable "GC task thread#1 (ParallelGC)" os_prio=0 tid=0x0000000009d65800 nid=0x2cd1 runnable "VM Periodic Task Thread" os_prio=0 tid=0x0000000009e61000 nid=0x2cd9 waiting on condition JNI global references: 383 Heap PSYoungGen total 74240K, used 20126K [0x00000000f0000000, 0x00000000f5500000, 0x0000000100000000) eden space 61440K, 16% used [0x00000000f0000000,0x00000000f0a03450,0x00000000f3c00000) from space 12800K, 77% used [0x00000000f4880000,0x00000000f52244a0,0x00000000f5500000) to space 12800K, 0% used [0x00000000f3c00000,0x00000000f3c00000,0x00000000f4880000) ParOldGen total 175104K, used 74721K [0x00000000d0000000, 0x00000000dab00000, 0x00000000f0000000) object space 175104K, 42% used [0x00000000d0000000,0x00000000d48f8488,0x00000000dab00000) Metaspace used 92528K, capacity 96514K, committed 96728K, reserved 1134592K class space used 10339K, capacity 10998K, committed 11008K, reserved 1048576K On 22 December 2016 at 20:26, Mark Thomas <ma...@apache.org> wrote: > On 22/12/2016 17:50, Greg Huber wrote: > > Chris, > > > > Thanks, I have been running this setup for ages, 8 plus years and various > > tomcat versions. It has never failed on <= 8.0.x I had to stop/start a > > couple of times to get it to fail and had to run a parallel maven build > to > > slow things down. > > > > Do not read much on dumps, but what would this -locked be? > > It is perfectly normal. > > Your problem is with the connection pool. > > Note that when Tomcat switched form DBCP1 to DBCP2, some of the names of > the attributes changed. You might not be using the settings you think > you are using. Please post your resource config. > > Mark > > > > > at > > org.apache.tomcat.util.net.NioEndpoint$SocketProcessor. > doRun(NioEndpoint.java:1437) > > at > > org.apache.tomcat.util.net.SocketProcessorBase.run( > SocketProcessorBase.java:49) > > - locked <0x00000000f1c54ed8> (a > > org.apache.tomcat.util.net.NioEndpoint$NioSocketWrapper) > > at > > java.util.concurrent.ThreadPoolExecutor.runWorker( > ThreadPoolExecutor.java:1142) > > at > > java.util.concurrent.ThreadPoolExecutor$Worker.run( > ThreadPoolExecutor.java:617) > > at > > org.apache.tomcat.util.threads.TaskThread$WrappingRunnable.run( > TaskThread.java:61) > > at java.lang.Thread.run(Thread.java:745) > > > > Cheers Greg > > > > > > On 22 December 2016 at 16:24, Christopher Schultz < > > ch...@christopherschultz.net> wrote: > > > > Greg, > > > > On 12/22/16 5:18 AM, Greg Huber wrote: > >>>> Here is the first dump and I cannot send all three. Its my prod > >>>> server which is locked up at the moment. I need to get it back on > >>>> line so is this enough info? I can emile the whole file which has > >>>> the three dumps if needed. > >>>> > >>>> [snip] > >>>> > >>>> Full thread dump Java HotSpot(TM) 64-Bit Server VM (25.40-b25 mixed > >>>> mode): > >>>> > >>>> "ajp-nio-8009-exec-26" #61 daemon prio=5 os_prio=0 > >>>> tid=0x000000000cb10000 nid=0x1960 in Object.wait() > >>>> [0x00002ba658a4b000] java.lang.Thread.State: WAITING (on object > >>>> monitor) at java.lang.Object.wait(Native Method) at > >>>> java.lang.Object.wait(Object.java:502) at > >>>> org.apache.commons.pool.impl.GenericObjectPool. > borrowObject(GenericObj > > ectPool.java:1104) > >>>> > >>>> > > - locked <0x00000000f2879c10> (a > >>>> org.apache.commons.pool.impl.GenericObjectPool$Latch) > > > > Waiting on a database connection. > > > > > >>>> "ajp-nio-8009-exec-25" #60 daemon prio=5 os_prio=0 > >>>> tid=0x000000000c74f800 nid=0x195d in Object.wait() > >>>> [0x00002ba658a0a000] java.lang.Thread.State: WAITING (on object > >>>> monitor) at java.lang.Object.wait(Native Method) at > >>>> java.lang.Object.wait(Object.java:502) at > >>>> org.apache.commons.pool.impl.GenericObjectPool. > borrowObject(GenericObj > > ectPool.java:1104) > >>>> > >>>> > > - locked <0x00000000f2852128> (a > >>>> org.apache.commons.pool.impl.GenericObjectPool$Latch) > > > > Another thread waiting for a db connection. > > > >>>> "ajp-nio-8009-exec-24" #59 daemon prio=5 os_prio=0 > >>>> tid=0x000000000dd3b000 nid=0x1959 in Object.wait() > >>>> [0x00002ba6589c9000] java.lang.Thread.State: WAITING (on object > >>>> monitor) at java.lang.Object.wait(Native Method) at > >>>> java.lang.Object.wait(Object.java:502) at > >>>> org.apache.commons.pool.impl.GenericObjectPool. > borrowObject(GenericObj > > ectPool.java:1104) > >>>> > >>>> > > - locked <0x00000000f282a818> (a > >>>> org.apache.commons.pool.impl.GenericObjectPool$Latch) > > > > I'm starting to see a pattern, here. > > > >>>> "ajp-nio-8009-exec-23" #58 daemon prio=5 os_prio=0 > >>>> tid=0x000000000cbe6000 nid=0x1953 in Object.wait() > >>>> [0x00002ba658988000] java.lang.Thread.State: WAITING (on object > >>>> monitor) at java.lang.Object.wait(Native Method) at > >>>> java.lang.Object.wait(Object.java:502) at > >>>> org.apache.commons.pool.impl.GenericObjectPool. > borrowObject(GenericObj > > ectPool.java:1104) > >>>> > >>>> > > - locked <0x00000000f28030f8> (a > >>>> org.apache.commons.pool.impl.GenericObjectPool$Latch) > > > > Hmm. Could it be... > > > >>>> "ajp-nio-8009-exec-22" #57 daemon prio=5 os_prio=0 > >>>> tid=0x000000000e797800 nid=0x194f in Object.wait() > >>>> [0x00002ba659472000] java.lang.Thread.State: WAITING (on object > >>>> monitor) at java.lang.Object.wait(Native Method) at > >>>> java.lang.Object.wait(Object.java:502) at > >>>> org.apache.commons.pool.impl.GenericObjectPool. > borrowObject(GenericObj > > ectPool.java:1104) > >>>> > >>>> > > - locked <0x00000000f27daf28> (a > >>>> org.apache.commons.pool.impl.GenericObjectPool$Latch) > > > > ... just possibly ... > > > >>>> "ajp-nio-8009-exec-21" #56 daemon prio=5 os_prio=0 > >>>> tid=0x000000000e874000 nid=0x194d in Object.wait() > >>>> [0x00002ba656c67000] java.lang.Thread.State: WAITING (on object > >>>> monitor) at java.lang.Object.wait(Native Method) at > >>>> java.lang.Object.wait(Object.java:502) at > >>>> org.apache.commons.pool.impl.GenericObjectPool. > borrowObject(GenericObj > > ectPool.java:1104) > >>>> > >>>> > > - locked <0x00000000f27b3d20> (a > >>>> org.apache.commons.pool.impl.GenericObjectPool$Latch) > > > > ... that you have run out of ... > > > >>>> "QuartzScheduler_scheduler-NON_CLUSTERED_MisfireHandler" #55 > >>>> prio=5 os_prio=0 tid=0x000000000cbe5000 nid=0x194b waiting on > >>>> condition [0x00002ba658c50000] java.lang.Thread.State: > >>>> TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) > >>>> at > >>>> org.quartz.impl.jdbcjobstore.JobStoreSupport$ > MisfireHandler.run(JobSto > > reSupport.java:3992) > > > > (Quartz > >>>> > > waiting around for some event. Ignore this one in erms of my > > overarching narrative.) > > > >>>> "ajp-nio-8009-exec-20" #54 daemon prio=5 os_prio=0 > >>>> tid=0x000000000e44f000 nid=0x194a in Object.wait() > >>>> [0x00002ba658948000] java.lang.Thread.State: WAITING (on object > >>>> monitor) at java.lang.Object.wait(Native Method) at > >>>> java.lang.Object.wait(Object.java:502) at > >>>> org.apache.commons.pool.impl.GenericObjectPool. > borrowObject(GenericObj > > ectPool.java:1104) > >>>> > >>>> > > - locked <0x00000000f1f541b8> (a > >>>> org.apache.commons.pool.impl.GenericObjectPool$Latch) > > > > Database connections? > > > >>>> "ajp-nio-8009-exec-19" #53 daemon prio=5 os_prio=0 > >>>> tid=0x000000000cc2e800 nid=0x1940 in Object.wait() > >>>> [0x00002ba658908000] java.lang.Thread.State: RUNNABLE at > >>>> org.events.ui.core.filters.IPBanFilter.doFilter(IPBanFilter.java:36) > > > > Ooh! > >>>> > > This one is doing something different. I suspect any ms it will > > ask for another db connection. > > > >>>> "ajp-nio-8009-exec-18" #52 daemon prio=5 os_prio=0 > >>>> tid=0x000000000cc7b800 nid=0x1933 in Object.wait() > >>>> [0x00002ba6588c6000] java.lang.Thread.State: WAITING (on object > >>>> monitor) at java.lang.Object.wait(Native Method) at > >>>> java.lang.Object.wait(Object.java:502) at > >>>> org.apache.commons.pool.impl.GenericObjectPool. > borrowObject(GenericObj > > ectPool.java:1104) > >>>> > >>>> > > - locked <0x00000000f1d766c0> (a > >>>> org.apache.commons.pool.impl.GenericObjectPool$Latch) > > > > That's more like it. > > > >>>> "ajp-nio-8009-exec-17" #51 daemon prio=5 os_prio=0 > >>>> tid=0x000000000cb83000 nid=0x1930 in Object.wait() > >>>> [0x00002ba658885000] java.lang.Thread.State: WAITING (on object > >>>> monitor) at java.lang.Object.wait(Native Method) at > >>>> java.lang.Object.wait(Object.java:502) at > >>>> org.apache.commons.pool.impl.GenericObjectPool. > borrowObject(GenericObj > > ectPool.java:1104) > >>>> > >>>> > > - locked <0x00000000f1d4e918> (a > >>>> org.apache.commons.pool.impl.GenericObjectPool$Latch) > > > > I'm reminded of a Far Side cartoon. > > [http://grahammercer.com.au/humour/GodMakesTheSnake.jpg] > > > >>>> "ajp-nio-8009-exec-16" #50 daemon prio=5 os_prio=0 > >>>> tid=0x000000000ea8b000 nid=0x192e in Object.wait() > >>>> [0x00002ba658ed7000] java.lang.Thread.State: RUNNABLE at > >>>> org.events.ui.core.filters.IPBanFilter.doFilter(IPBanFilter.java:36) > > > > Another > >>>> > > thread caught in the filter. Filters are usually quick, so I > > think this is a transient state. Probably about to request a database > > connection. > > > >>>> "ajp-nio-8009-exec-15" #49 daemon prio=5 os_prio=0 > >>>> tid=0x000000000c92a000 nid=0x1928 in Object.wait() > >>>> [0x00002ba65452d000] java.lang.Thread.State: RUNNABLE at > >>>> org.events.ui.core.filters.IPBanFilter.doFilter(IPBanFilter.java:36) > > > > Here, > >>>> > > too. > > > > > > Okay, I'm done reading the thread dump. > > > > Have a look at your database pool's size and "active" counts[1]. You > > may find that your application is locked-up waiting on db connections. > > you may have a resource leak (db connections). > > > > -chris > > > > [1] > > http://people.apache.org/~schultz/ApacheCon%20NA%202016/Monitoring%20Apa > > che%20Tomcat%20with%20JMX.pdf, > > slide 16 > >> > >> --------------------------------------------------------------------- > >> To unsubscribe, e-mail: users-unsubscr...@tomcat.apache.org > >> For additional commands, e-mail: users-h...@tomcat.apache.org > >> > >> > > > > > --------------------------------------------------------------------- > To unsubscribe, e-mail: users-unsubscr...@tomcat.apache.org > For additional commands, e-mail: users-h...@tomcat.apache.org > >