Hello and apologies in advance for the HTML formatting (not sure how to turn it off in outlook web access).
Our tomcat was non-responsive, and catalina.out had several 'java.lang.OutOfMemory' errors. So this isn't totally unknown to us (our apps leak a bit of memory, plus we're running on a small JVM). I tried to shutdown tomcat and it wouldn't die. Doing a kill -3 (before the eventual kill -9) I got a thread dump. Almost all the TP threads look like this: --------------------------------------------------------------------- "TP-Processor44" daemon prio=1 tid=0x09667438 nid=0xd95 in Object.wait() [b2bfe000..b2bff8d8] at java.lang.Object.wait(Native Method) - waiting on <0x45251bb0> (a sun.misc.SoftCache) at java.lang.Object.wait(Object.java:429) at java.util.ResourceBundle.findBundle(ResourceBundle.java:862) - locked <0x45251bb0> (a sun.misc.SoftCache) at java.util.ResourceBundle.getBundleImpl(ResourceBundle.java:730) at java.util.ResourceBundle.getBundle(ResourceBundle.java:538) at sun.text.resources.LocaleData$1.run(LocaleData.java:113) at java.security.AccessController.doPrivileged(Native Method) at sun.text.resources.LocaleData.getBundle(LocaleData.java:111) at sun.text.resources.LocaleData.getDateFormatZoneData(LocaleData.java:107) at java.text.DateFormatSymbols.cacheLookup(DateFormatSymbols.java:430) at java.text.DateFormatSymbols.initializeData(DateFormatSymbols.java:468) at java.text.DateFormatSymbols.<init>(DateFormatSymbols.java:103) at java.text.SimpleDateFormat.<init>(SimpleDateFormat.java:442) at org.apache.coyote.tomcat5.CoyoteRequest.<init>(CoyoteRequest.java:151) at org.apache.coyote.tomcat5.CoyoteConnector.createRequest(CoyoteConnector.java:1230) at org.apache.coyote.tomcat5.CoyoteAdapter.service(CoyoteAdapter.java:131) at org.apache.jk.server.JkCoyoteHandler.invoke(JkCoyoteHandler.java:300) at org.apache.jk.common.HandlerRequest.invoke(HandlerRequest.java:374) at org.apache.jk.common.ChannelSocket.invoke(ChannelSocket.java:743) at org.apache.jk.common.ChannelSocket.processConnection(ChannelSocket.java:675) at org.apache.jk.common.SocketConnection.runIt(ChannelSocket.java:866) at org.apache.tomcat.util.threads.ThreadPool$ControlRunnable.run(ThreadPool.java:684) at java.lang.Thread.run(Thread.java:534) --------------------------------------------------------------------- So, Tomcat code is waiting for some other code (my code) to release a lock? - waiting on <0x45251bb0> (a sun.misc.SoftCache) Is that the correct interpretation? If so, I'd need to find a stacktrace that contains *my* code that has this object locked? Because I did find one, but I'm not sure what it's telling me. Or can anything be told, since the VM threw an OOME and now all bets are off? "www.foo.com <http://www.foo.com> - /prod/order.jsp?orderID=225" daemon prio=1 tid=0x080ac938 nid=0x5f1a in Object.wait() [b61fd000..b61ff8d8] at java.lang.Object.wait(Native Method) - waiting on <0x45251bb0> (a sun.misc.SoftCache) at java.lang.Object.wait(Object.java:429) at java.util.ResourceBundle.findBundle(ResourceBundle.java:862) - locked <0x45251bb0> (a sun.misc.SoftCache) at java.util.ResourceBundle.getBundleImpl(ResourceBundle.java:730) at java.util.ResourceBundle.getBundle(ResourceBundle.java:538) at sun.text.resources.LocaleData$1.run(LocaleData.java:113) at java.security.AccessController.doPrivileged(Native Method) at sun.text.resources.LocaleData.getBundle(LocaleData.java:111) at sun.text.resources.LocaleData.getDateFormatZoneData(LocaleData.java:107) at java.text.DateFormatSymbols.cacheLookup(DateFormatSymbols.java:430) at java.text.DateFormatSymbols.initializeData(DateFormatSymbols.java:468) at java.text.DateFormatSymbols.<init>(DateFormatSymbols.java:103) at java.text.SimpleDateFormat.<init>(SimpleDateFormat.java:442) at java.util.Date.toString(Date.java:981) at java.lang.String.valueOf(String.java:2131) at java.lang.StringBuffer.append(StringBuffer.java:370) - locked <0x4b94e2e8> (a java.lang.StringBuffer) at com.xx.LineItem.setProductionDate(LineItem.java:1181) at com.xx.ItemGroup.setProductionDate(ItemGroup.java:1468) at com.xx.ItemGroup.addLineItem(ItemGroup.java:126) at com.xx.OrderHistory.load(OrderHistory.java:445) at com.xx.OrderHistory.<init>(OrderHistory.java:201) at org.apache.jsp.prod.order_jsp._jspService(order_jsp.java:174) at org.apache.jasper.runtime.HttpJspBase.service(HttpJspBase.java:94) at javax.servlet.http.HttpServlet.service(HttpServlet.java:802) at org.apache.jasper.servlet.JspServletWrapper.service(JspServletWrapper.java:324) at org.apache.jasper.servlet.JspServlet.serviceJspFile(JspServlet.java:292) at org.apache.jasper.servlet.JspServlet.service(JspServlet.java:236) at javax.servlet.http.HttpServlet.service(HttpServlet.java:802) at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:237) at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:157) at com.opensymphony.module.sitemesh.filter.PageFilter.parsePage(PageFilter.java:118) at com.opensymphony.module.sitemesh.filter.PageFilter.doFilter(PageFilter.java:52) at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:186) at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:157) at com.gb.conductor.filters.CommentLogger.doFilter(CommentLogger.java:55) at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:186) at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:157) at com.gb.data.DataSourcePoolFilter.doFilter(DataSourcePoolFilter.java:86) at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:186) at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:157) at com.gb.security.BasicAuthUserFilter.doFilter(BasicAuthUserFilter.java:53) at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:186) at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:157) at com.gb.web.ThreadDebugFilter.doFilter(ThreadDebugFilter.java:44) at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:186) at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:157) at com.gb.web.FileUploadFilter.doFilter(FileUploadFilter.java:89) at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:186) at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:157) at org.apache.catalina.core.StandardWrapperValve.invoke(StandardWrapperValve.java:214) at org.apache.catalina.core.StandardValveContext.invokeNext(StandardValveContext.java:104) at org.apache.catalina.core.StandardPipeline.invoke(StandardPipeline.java:520) at org.apache.catalina.core.StandardContextValve.invokeInternal(StandardContextValve.java:198) at org.apache.catalina.core.StandardContextValve.invoke(StandardContextValve.java:144) at org.apache.catalina.core.StandardValveContext.invokeNext(StandardValveContext.java:104) at org.apache.catalina.authenticator.AuthenticatorBase.invoke(AuthenticatorBase.java:540) at org.apache.catalina.core.StandardValveContext.invokeNext(StandardValveContext.java:102) at org.apache.catalina.core.StandardPipeline.invoke(StandardPipeline.java:520) at org.apache.catalina.core.StandardHostValve.invoke(StandardHostValve.java:137) at org.apache.catalina.core.StandardValveContext.invokeNext(StandardValveContext.java:104) at org.apache.catalina.valves.ErrorReportValve.invoke(ErrorReportValve.java:118) at org.apache.catalina.core.StandardValveContext.invokeNext(StandardValveContext.java:102) at org.apache.catalina.core.StandardPipeline.invoke(StandardPipeline.java:520) at org.apache.catalina.core.StandardEngineValve.invoke(StandardEngineValve.java:109) at org.apache.catalina.core.StandardValveContext.invokeNext(StandardValveContext.java:104) at org.apache.catalina.core.StandardPipeline.invoke(StandardPipeline.java:520) at org.apache.catalina.core.ContainerBase.invoke(ContainerBase.java:929) at org.apache.coyote.tomcat5.CoyoteAdapter.service(CoyoteAdapter.java:160) at org.apache.jk.server.JkCoyoteHandler.invoke(JkCoyoteHandler.java:300) at org.apache.jk.common.HandlerRequest.invoke(HandlerRequest.java:374) at org.apache.jk.common.ChannelSocket.invoke(ChannelSocket.java:743) at org.apache.jk.common.ChannelSocket.processConnection(ChannelSocket.java:675) at org.apache.jk.common.SocketConnection.runIt(ChannelSocket.java:866) at org.apache.tomcat.util.threads.ThreadPool$ControlRunnable.run(ThreadPool.java:684) at java.lang.Thread.run(Thread.java:534)
