hi, we are using tomcat 6, jdk1.5, CentOS 5, with 4gb of ram, data base is Oracle 11g. For the past one month we are getting this problem before that we don't.
Tomcat hangs after running for several hours. Means initially it is taking around 24hr now it is hanging every 3 to 4 hr we can not say the exact time. When I see the graph using visualgc there is plenty of space and data sources are also sufficient I configured it 200. And http threads maxActive are 150. I am unable to find the problem. give suggestions to solve this problem. I didn't understand the thread dump. And we are sending per day nearly 50000 mails to users using sendmail using java. The full thred dump as follows. Full thread dump Java HotSpot(TM) Server VM (1.5.0_19-b02 mixed mode): "http-80-150" daemon prio=1 tid=0x093c4870 nid=0x37c2 runnable [0x7fb29000..0x7fb29db0] at java.net.SocketInputStream.socketRead0(Native Method) at java.net.SocketInputStream.read(SocketInputStream.java:129) at org.apache.coyote.http11.InternalInputBuffer.fill(InternalInputBuffer.java:735) at org.apache.coyote.http11.InternalInputBuffer.parseRequestLine(InternalInputBuffer.java:366) at org.apache.coyote.http11.Http11Processor.process(Http11Processor.java:808) at org.apache.coyote.http11.Http11Protocol$Http11ConnectionHandler.process(Http11Protocol.java:583) at org.apache.tomcat.util.net.JIoEndpoint$Worker.run(JIoEndpoint.java:454) at java.lang.Thread.run(Thread.java:595) "http-80-149" daemon prio=1 tid=0x093c4160 nid=0x37c1 runnable [0x7fbaa000..0x7fbaaf30] at java.net.SocketInputStream.socketRead0(Native Method) at java.net.SocketInputStream.read(SocketInputStream.java:129) at org.apache.coyote.http11.InternalInputBuffer.fill(InternalInputBuffer.java:735) at org.apache.coyote.http11.InternalInputBuffer.parseRequestLine(InternalInputBuffer.java:366) at org.apache.coyote.http11.Http11Processor.process(Http11Processor.java:808) at org.apache.coyote.http11.Http11Protocol$Http11ConnectionHandler.process(Http11Protocol.java:583) at org.apache.tomcat.util.net.JIoEndpoint$Worker.run(JIoEndpoint.java:454) at java.lang.Thread.run(Thread.java:595) ........................................ ................................... all http-80-xx threads are in runnable like above ............................................. ..................................... "DefaultQuartzScheduler_QuartzSchedulerThread" prio=1 tid=0x86244118 nid=0x371f sleeping[0x821f6000..0x821f70b0] at java.lang.Thread.sleep(Native Method) at org.quartz.core.QuartzSchedulerThread.run(QuartzSchedulerThread.java:272) "DefaultQuartzScheduler_Worker-9" prio=1 tid=0x8620e088 nid=0x371e in Object.wait() [0x82277000..0x82277e30] at java.lang.Object.wait(Native Method) - waiting on <0x92767810> (a java.lang.Object) at org.quartz.simpl.SimpleThreadPool.getNextRunnable(SimpleThreadPool.java:428) - locked <0x92767810> (a java.lang.Object) at org.quartz.simpl.SimpleThreadPool.access$000(SimpleThreadPool.java:47) at org.quartz.simpl.SimpleThreadPool$WorkerThread.run(SimpleThreadPool.java:518) "DefaultQuartzScheduler_Worker-8" prio=1 tid=0x86254488 nid=0x371d in Object.wait() [0x822f8000..0x822f8db0] at java.lang.Object.wait(Native Method) - waiting on <0x92767810> (a java.lang.Object) at org.quartz.simpl.SimpleThreadPool.getNextRunnable(SimpleThreadPool.java:428) - locked <0x92767810> (a java.lang.Object) at org.quartz.simpl.SimpleThreadPool.access$000(SimpleThreadPool.java:47) at org.quartz.simpl.SimpleThreadPool$WorkerThread.run(SimpleThreadPool.java:518) "DefaultQuartzScheduler_Worker-7" prio=1 tid=0x862542e8 nid=0x371c in Object.wait() [0x82379000..0x82379f30] at java.lang.Object.wait(Native Method) - waiting on <0x92767810> (a java.lang.Object) at org.quartz.simpl.SimpleThreadPool.getNextRunnable(SimpleThreadPool.java:428) - locked <0x92767810> (a java.lang.Object) at org.quartz.simpl.SimpleThreadPool.access$000(SimpleThreadPool.java:47) at org.quartz.simpl.SimpleThreadPool$WorkerThread.run(SimpleThreadPool.java:518) "DefaultQuartzScheduler_Worker-6" prio=1 tid=0x864c8018 nid=0x371b in Object.wait() [0x823fa000..0x823faeb0] at java.lang.Object.wait(Native Method) - waiting on <0x92767810> (a java.lang.Object) at org.quartz.simpl.SimpleThreadPool.getNextRunnable(SimpleThreadPool.java:428) - locked <0x92767810> (a java.lang.Object) at org.quartz.simpl.SimpleThreadPool.access$000(SimpleThreadPool.java:47) at org.quartz.simpl.SimpleThreadPool$WorkerThread.run(SimpleThreadPool.java:518) "DefaultQuartzScheduler_Worker-5" prio=1 tid=0x864c7e90 nid=0x371a in Object.wait() [0x8247b000..0x8247c030] at java.lang.Object.wait(Native Method) - waiting on <0x92767810> (a java.lang.Object) at org.quartz.simpl.SimpleThreadPool.getNextRunnable(SimpleThreadPool.java:428) - locked <0x92767810> (a java.lang.Object) at org.quartz.simpl.SimpleThreadPool.access$000(SimpleThreadPool.java:47) at org.quartz.simpl.SimpleThreadPool$WorkerThread.run(SimpleThreadPool.java:518) "DefaultQuartzScheduler_Worker-4" prio=1 tid=0x86685dd0 nid=0x3719 in Object.wait() [0x824fc000..0x824fcfb0] at java.lang.Object.wait(Native Method) - waiting on <0x92767810> (a java.lang.Object) at org.quartz.simpl.SimpleThreadPool.getNextRunnable(SimpleThreadPool.java:428) - locked <0x92767810> (a java.lang.Object) at org.quartz.simpl.SimpleThreadPool.access$000(SimpleThreadPool.java:47) at org.quartz.simpl.SimpleThreadPool$WorkerThread.run(SimpleThreadPool.java:518) "DefaultQuartzScheduler_Worker-3" prio=1 tid=0x86246cd0 nid=0x3718 in Object.wait() [0x8257d000..0x8257e130] at java.lang.Object.wait(Native Method) - waiting on <0x92767810> (a java.lang.Object) at org.quartz.simpl.SimpleThreadPool.getNextRunnable(SimpleThreadPool.java:428) - locked <0x92767810> (a java.lang.Object) at org.quartz.simpl.SimpleThreadPool.access$000(SimpleThreadPool.java:47) at org.quartz.simpl.SimpleThreadPool$WorkerThread.run(SimpleThreadPool.java:518) "DefaultQuartzScheduler_Worker-2" prio=1 tid=0x87d9e000 nid=0x3717 in Object.wait() [0x8269c000..0x8269d0b0] at java.lang.Object.wait(Native Method) - waiting on <0x92767810> (a java.lang.Object) at org.quartz.simpl.SimpleThreadPool.getNextRunnable(SimpleThreadPool.java:428) - locked <0x92767810> (a java.lang.Object) at org.quartz.simpl.SimpleThreadPool.access$000(SimpleThreadPool.java:47) at org.quartz.simpl.SimpleThreadPool$WorkerThread.run(SimpleThreadPool.java:518) "DefaultQuartzScheduler_Worker-1" prio=1 tid=0x87df4470 nid=0x3716 in Object.wait() [0x8271d000..0x8271de30] at java.lang.Object.wait(Native Method) - waiting on <0x92767810> (a java.lang.Object) at org.quartz.simpl.SimpleThreadPool.getNextRunnable(SimpleThreadPool.java:428) - locked <0x92767810> (a java.lang.Object) at org.quartz.simpl.SimpleThreadPool.access$000(SimpleThreadPool.java:47) at org.quartz.simpl.SimpleThreadPool$WorkerThread.run(SimpleThreadPool.java:518) "DefaultQuartzScheduler_Worker-0" prio=1 tid=0x86213ae0 nid=0x3715 in Object.wait() [0x8279e000..0x8279edb0] at java.lang.Object.wait(Native Method) - waiting on <0x92767810> (a java.lang.Object) at org.quartz.simpl.SimpleThreadPool.getNextRunnable(SimpleThreadPool.java:428) - locked <0x92767810> (a java.lang.Object) at org.quartz.simpl.SimpleThreadPool.access$000(SimpleThreadPool.java:47) at org.quartz.simpl.SimpleThreadPool$WorkerThread.run(SimpleThreadPool.java:518) "Java2D Disposer" daemon prio=1 tid=0x8649c208 nid=0x369a in Object.wait() [0x85918000..0x85918eb0] at java.lang.Object.wait(Native Method) - waiting on <0x919ce508> (a java.lang.ref.ReferenceQueue$Lock) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:120) - locked <0x919ce508> (a java.lang.ref.ReferenceQueue$Lock) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:136) at sun.java2d.Disposer.run(Disposer.java:125) at java.lang.Thread.run(Thread.java:595) "TP-Monitor" daemon prio=1 tid=0x87dcb340 nid=0x3687 in Object.wait() [0x8617d000..0x8617de30] at java.lang.Object.wait(Native Method) - waiting on <0x91885178> (a org.apache.tomcat.util.threads.ThreadPool$MonitorRunnable) at org.apache.tomcat.util.threads.ThreadPool$MonitorRunnable.run(ThreadPool.java:565) - locked <0x91885178> (a org.apache.tomcat.util.threads.ThreadPool$MonitorRunnable) at java.lang.Thread.run(Thread.java:595) "TP-Processor4" daemon prio=1 tid=0x87d70cf0 nid=0x3686 runnable [0x861fe000..0x861fedb0] at java.net.PlainSocketImpl.socketAccept(Native Method) at java.net.PlainSocketImpl.accept(PlainSocketImpl.java:384) - locked <0x920d3788> (a java.net.SocksSocketImpl) at java.net.ServerSocket.implAccept(ServerSocket.java:450) at java.net.ServerSocket.accept(ServerSocket.java:421) at org.apache.jk.common.ChannelSocket.accept(ChannelSocket.java:307) at org.apache.jk.common.ChannelSocket.acceptConnections(ChannelSocket.java:661) at org.apache.jk.common.ChannelSocket$SocketAcceptor.runIt(ChannelSocket.java:872) at org.apache.tomcat.util.threads.ThreadPool$ControlRunnable.run(ThreadPool.java:690) at java.lang.Thread.run(Thread.java:595) "TP-Processor3" daemon prio=1 tid=0x866193b8 nid=0x3685 in Object.wait() [0x863fe000..0x863fef30] at java.lang.Object.wait(Native Method) - waiting on <0x918b9210> (a org.apache.tomcat.util.threads.ThreadPool$ControlRunnable) at java.lang.Object.wait(Object.java:474) at org.apache.tomcat.util.threads.ThreadPool$ControlRunnable.run(ThreadPool.java:662) - locked <0x918b9210> (a org.apache.tomcat.util.threads.ThreadPool$ControlRunnable) at java.lang.Thread.run(Thread.java:595) "TP-Processor2" daemon prio=1 tid=0x87d260b8 nid=0x3684 in Object.wait() [0x868bc000..0x868bceb0] at java.lang.Object.wait(Native Method) - waiting on <0x918b9370> (a org.apache.tomcat.util.threads.ThreadPool$ControlRunnable) at java.lang.Object.wait(Object.java:474) at org.apache.tomcat.util.threads.ThreadPool$ControlRunnable.run(ThreadPool.java:662) - locked <0x918b9370> (a org.apache.tomcat.util.threads.ThreadPool$ControlRunnable) at java.lang.Thread.run(Thread.java:595) "TP-Processor1" daemon prio=1 tid=0x87d57cd0 nid=0x3683 in Object.wait() [0x8693d000..0x8693e030] at java.lang.Object.wait(Native Method) - waiting on <0x918b9580> (a org.apache.tomcat.util.threads.ThreadPool$ControlRunnable) at java.lang.Object.wait(Object.java:474) at org.apache.tomcat.util.threads.ThreadPool$ControlRunnable.run(ThreadPool.java:662) - locked <0x918b9580> (a org.apache.tomcat.util.threads.ThreadPool$ControlRunnable) at java.lang.Thread.run(Thread.java:595) "http-80-Acceptor-0" daemon prio=1 tid=0x87d70ed8 nid=0x367e in Object.wait() [0x877ad000..0x877addb0] at java.lang.Object.wait(Native Method) - waiting on <0x91aa46b0> (a org.apache.tomcat.util.net.JIoEndpoint$WorkerStack) at java.lang.Object.wait(Object.java:474) at org.apache.tomcat.util.net.JIoEndpoint.getWorkerThread(JIoEndpoint.java:712) - locked <0x91aa46b0> (a org.apache.tomcat.util.net.JIoEndpoint$WorkerStack) at org.apache.tomcat.util.net.JIoEndpoint.processSocket(JIoEndpoint.java:743) at org.apache.tomcat.util.net.JIoEndpoint$Acceptor.run(JIoEndpoint.java:320) at java.lang.Thread.run(Thread.java:595) "ContainerBackgroundProcessor[StandardEngine[Catalina]]" daemon prio=1 tid=0x87def630 nid=0x367d waiting on condition [0x87bfe000..0x87bfef30] at java.lang.Thread.sleep(Native Method) at org.apache.catalina.core.ContainerBase$ContainerBackgroundProcessor.run(ContainerBase.java:1579) at java.lang.Thread.run(Thread.java:595) "RMI TCP Accept-0" daemon prio=1 tid=0x88903878 nid=0x3652 runnable [0x88876000..0x88877030] at java.net.PlainSocketImpl.socketAccept(Native Method) at java.net.PlainSocketImpl.accept(PlainSocketImpl.java:384) - locked <0x916dcec0> (a java.net.SocksSocketImpl) at java.net.ServerSocket.implAccept(ServerSocket.java:450) at java.net.ServerSocket.accept(ServerSocket.java:421) at sun.management.jmxremote.LocalRMIServerSocketFactory$1.accept(LocalRMIServerSocketFactory.java:31) at sun.rmi.transport.tcp.TCPTransport.run(TCPTransport.java:340) at java.lang.Thread.run(Thread.java:595) "Timer-0" daemon prio=1 tid=0x88907078 nid=0x3650 in Object.wait() [0x888f7000..0x888f7fb0] at java.lang.Object.wait(Native Method) - waiting on <0x91817878> (a java.util.TaskQueue) at java.lang.Object.wait(Object.java:474) at java.util.TimerThread.mainLoop(Timer.java:483) - locked <0x91817878> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:462) "Low Memory Detector" daemon prio=1 tid=0x08fad958 nid=0x364e runnable [0x00000000..0x00000000] "CompilerThread1" daemon prio=1 tid=0x08fac4d8 nid=0x364d waiting on condition [0x00000000..0x88c412d8] "CompilerThread0" daemon prio=1 tid=0x08fab480 nid=0x364c waiting on condition [0x00000000..0x88cc2058] "AdapterThread" daemon prio=1 tid=0x08faa358 nid=0x364b waiting on condition [0x00000000..0x00000000] "Signal Dispatcher" daemon prio=1 tid=0x08fa9520 nid=0x364a runnable [0x00000000..0x00000000] "Finalizer" daemon prio=1 tid=0x08f9f0a8 nid=0x3649 in Object.wait() [0x89045000..0x89045eb0] at java.lang.Object.wait(Native Method) - waiting on <0x91744448> (a java.lang.ref.ReferenceQueue$Lock) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:120) - locked <0x91744448> (a java.lang.ref.ReferenceQueue$Lock) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:136) at java.lang.ref.Finalizer$FinalizerThread.run(Finalizer.java:159) "Reference Handler" daemon prio=1 tid=0x08f9eb68 nid=0x3648 in Object.wait() [0x890c6000..0x890c7030] at java.lang.Object.wait(Native Method) - waiting on <0x91a9c6f8> (a java.lang.ref.Reference$Lock) at java.lang.Object.wait(Object.java:474) at java.lang.ref.Reference$ReferenceHandler.run(Reference.java:116) - locked <0x91a9c6f8> (a java.lang.ref.Reference$Lock) "main" prio=1 tid=0x08ee1248 nid=0x3641 runnable [0xbfce1000..0xbfce1d28] at java.net.PlainSocketImpl.socketAccept(Native Method) at java.net.PlainSocketImpl.accept(PlainSocketImpl.java:384) - locked <0x9175b308> (a java.net.SocksSocketImpl) at java.net.ServerSocket.implAccept(ServerSocket.java:450) at java.net.ServerSocket.accept(ServerSocket.java:421) at org.apache.catalina.core.StandardServer.await(StandardServer.java:389) at org.apache.catalina.startup.Catalina.await(Catalina.java:647) at org.apache.catalina.startup.Catalina.start(Catalina.java:607) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:585) at org.apache.catalina.startup.Bootstrap.start(Bootstrap.java:288) at org.apache.catalina.startup.Bootstrap.main(Bootstrap.java:413) "VM Thread" prio=1 tid=0x08f9c648 nid=0x3647 runnable "GC task thread#0 (ParallelGC)" prio=1 tid=0x08efb410 nid=0x3643 runnable "GC task thread#1 (ParallelGC)" prio=1 tid=0x08efc060 nid=0x3644 runnable "VM Periodic Task Thread" prio=1 tid=0x88927f28 nid=0x3653 waiting on condition
