Hi,
We are currently seeing the following two types of stuck
threads in our server logs due to which our server is going to the Unknown
state. Since this is happening in the production environment it will be great
if you could provide us some insight –
1. <Jan 30, 2014 10:30:20 AM
EST> <Error> <WebLogicServer> <BEA-000337> <[STUCK]
ExecuteThread: '3' for queue: 'weblogic.kernel.Default (self-tuning)' has been
busy for "672" seconds working on the request "Http Request:
/esmeim/SubmitRemittanceReportByServiceCode.m", which is more than the
configured time (StuckThreadMaxTime) of "600" seconds. Stack trace:
Thread-95 "[STUCK]
ExecuteThread: '3' for queue: 'weblogic.kernel.Default (self-tuning)'"
<alive, in native, suspended, priority=1, DAEMON> {
java.lang.String.<init>(String.java:197)
org.apache.axis.message.SAX2EventRecorder.characters(SAX2EventRecorder.java:130)
org.apache.axis.encoding.DeserializationContextImpl.characters(DeserializationContextImpl.java:890)
org.apache.xerces.parsers.AbstractSAXParser.characters(AbstractSAXParser.java:456)
org.apache.xerces.impl.XMLNamespaceBinder.characters(XMLNamespaceBinder.java:608)
org.apache.xerces.impl.dtd.XMLDTDValidator.characters(XMLDTDValidator.java:835)
org.apache.xerces.impl.XMLDocumentFragmentScannerImpl.handleCharacter(XMLDocumentFragmentScannerImpl.java:1089)
org.apache.xerces.impl.XMLDocumentFragmentScannerImpl.scanEntityReference(XMLDocumentFragmentScannerImpl.java:1032)
org.apache.xerces.impl.XMLDocumentFragmentScannerImpl$FragmentContentDispatcher.dispatch(XMLDocumentFragmentScannerImpl.java:1375)
org.apache.xerces.impl.XMLDocumentFragmentScannerImpl.scanDocument(XMLDocumentFragmentScannerImpl.java:331)
org.apache.xerces.parsers.StandardParserConfiguration.parse(StandardParserConfiguration.java:499)
org.apache.xerces.parsers.StandardParserConfiguration.parse(StandardParserConfiguration.java:577)
org.apache.xerces.parsers.XMLParser.parse(XMLParser.java:146)
org.apache.xerces.parsers.AbstractSAXParser.parse(AbstractSAXParser.java:1141)
javax.xml.parsers.SAXParser.parse(SAXParser.java:364)
org.apache.axis.encoding.DeserializationContextImpl.parse(DeserializationContextImpl.java:238)
org.apache.axis.SOAPPart.getAsSOAPEnvelope(SOAPPart.java:508)
org.apache.axis.Message.getSOAPEnvelope(Message.java:376)
org.apache.axis.client.Call.invokeEngine(Call.java:2564)
org.apache.axis.client.Call.invoke(Call.java:2438)
org.apache.axis.client.Call.invoke(Call.java:2209)
org.apache.axis.client.Call.invoke(Call.java:2155)
org.apache.axis.client.Call.invoke(Call.java:1644)
com.cognos.developer.schemas.bibus._3.CognosReportNetBindingStub.getOutput(CognosReportNetBindingStub.java:7141)
com.steps.application.service.provider.ManageReportServiceBean.runReportWithStreamOutput(ManageReportServiceBean.java:1909)
com.steps.application.service.provider.ManageReportService_fegjtc_EOImpl.runReportWithStreamOutput(ManageReportService_fegjtc_EOImpl.java:44)
com.steps.application.presentation.controller.BaseReportController.runReportToGetByteStream(BaseReportController.java:1079)
com.steps.application.presentation.controller.RemittanceReportByServiceCodeSubmitController.processRequest(RemittanceReportByServiceCodeSubmitController.java:84)
com.steps.framework.presentation.StepsController.perform(StepsController.java:1744)
org.infohazard.maverick.ctl.FormBeanUser.go(FormBeanUser.java:54)
org.infohazard.maverick.flow.CommandBase.go(CommandBase.java:39)
org.infohazard.maverick.Dispatcher.service(Dispatcher.java:197)
javax.servlet.http.HttpServlet.service(HttpServlet.java:851)
weblogic.servlet.internal.StubSecurityHelper$ServletServiceAction.run(StubSecurityHelper.java:224)
weblogic.servlet.internal.StubSecurityHelper.invokeServlet(StubSecurityHelper.java:108)
weblogic.servlet.internal.ServletStubImpl.execute(ServletStubImpl.java:198)
weblogic.servlet.internal.TailFilter.doFilter(TailFilter.java:26)
weblogic.servlet.internal.FilterChainImpl.doFilter(FilterChainImpl.java:41)
com.steps.framework.presentation.DecryptFIlter.doFilter(DecryptFIlter.java:240)
weblogic.servlet.internal.FilterChainImpl.doFilter(FilterChainImpl.java:41)
com.opensymphony.module.sitemesh.filter.PageFilter.doFilter(PageFilter.java:60)
weblogic.servlet.internal.FilterChainImpl.doFilter(FilterChainImpl.java:41)
com.sun.identity.agents.filter.AmAgentBaseFilter.allowRequestToContinue(AmAgentBaseFilter.java:80)
com.sun.identity.agents.filter.AmAgentBaseFilter.doFilter(AmAgentBaseFilter.java:33)
weblogic.servlet.internal.FilterChainImpl.doFilter(FilterChainImpl.java:41)
weblogic.servlet.internal.RequestDispatcherImpl.invokeServlet(RequestDispatcherImpl.java:524)
weblogic.servlet.internal.RequestDispatcherImpl.forward(RequestDispatcherImpl.java:100)
org.infohazard.maverick.view.DispatchedViewFactory$DispatchedView.go(DispatchedViewFactory.java:82)
org.infohazard.maverick.view.DocumentView.go(DocumentView.java:47)
org.infohazard.maverick.flow.CommandBase.go(CommandBase.java:39)
org.infohazard.maverick.Dispatcher.service(Dispatcher.java:197)
javax.servlet.http.HttpServlet.service(HttpServlet.java:851)
weblogic.servlet.internal.StubSecurityHelper$ServletServiceAction.run(StubSecurityHelper.java:224)
weblogic.servlet.internal.StubSecurityHelper.invokeServlet(StubSecurityHelper.java:108)
weblogic.servlet.internal.ServletStubImpl.execute(ServletStubImpl.java:198)
weblogic.servlet.internal.TailFilter.doFilter(TailFilter.java:26)
weblogic.servlet.internal.FilterChainImpl.doFilter(FilterChainImpl.java:41)
com.steps.framework.presentation.DecryptFIlter.doFilter(DecryptFIlter.java:240)
weblogic.servlet.internal.FilterChainImpl.doFilter(FilterChainImpl.java:41)
com.opensymphony.module.sitemesh.filter.PageFilter.parsePage(PageFilter.java:482)
com.opensymphony.module.sitemesh.filter.PageFilter.doFilter(PageFilter.java:60)
weblogic.servlet.internal.FilterChainImpl.doFilter(FilterChainImpl.java:41)
com.sun.identity.agents.filter.AmAgentBaseFilter.allowRequestToContinue(AmAgentBaseFilter.java:80)
com.sun.identity.agents.filter.AmAgentBaseFilter.doFilter(AmAgentBaseFilter.java:33)
weblogic.servlet.internal.FilterChainImpl.doFilter(FilterChainImpl.java:41)
weblogic.servlet.internal.WebAppServletContext$ServletInvocationAction.run(WebAppServletContext.java:3214)
weblogic.security.acl.internal.AuthenticatedSubject.doAs(AuthenticatedSubject.java:308)
weblogic.security.service.SecurityManager.runAs(SecurityManager.java:117)
weblogic.servlet.internal.WebAppServletContext.securedExecute(WebAppServletContext.java:1946)
weblogic.servlet.internal.WebAppServletContext.execute(WebAppServletContext.java:1868)
weblogic.servlet.internal.ServletRequestImpl.run(ServletRequestImpl.java:1331)
weblogic.work.ExecuteThread.execute(ExecuteThread.java:206)
weblogic.work.ExecuteThread.run(ExecuteThread.java:173)
}
2. ####<Jan 16, 2014 9:33:41
AM EST> <Error> <WebLogicServer> <vgprapp3>
<ESMEIM-MS3> <[ACTIVE] ExecuteThread: '3' for queue:
'weblogic.kernel.Default (self-tuning)'> <<WLS Kernel>> <>
<> <1389882821941> <BEA-000337> <[STUCK] ExecuteThread:
'17' for queue: 'weblogic.kernel.Default (self-tuning)' has been busy for
"714" seconds working on the request "Http Request:
/esmeim/SubmitRemittanceReportByServiceCode.m", which is more than the
configured time (StuckThreadMaxTime) of "600" seconds. Stack trace:
Thread-10339 "[STUCK]
ExecuteThread: '17' for queue: 'weblogic.kernel.Default (self-tuning)'"
<alive, in native, suspended, priority=1, DAEMON> {
java.lang.String.toCharArray(String.java:2514)
org.apache.axis.message.SAX2EventRecorder.replay(SAX2EventRecorder.java:180)
org.apache.axis.message.MessageElement.publishToHandler(MessageElement.java:719)
org.apache.axis.message.RPCElement.deserialize(RPCElement.java:156)
org.apache.axis.message.RPCElement.getParams(RPCElement.java:346)
org.apache.axis.client.Call.invoke(Call.java:2209)
org.apache.axis.client.Call.invoke(Call.java:2155)
org.apache.axis.client.Call.invoke(Call.java:1644)
com.cognos.developer.schemas.bibus._3.CognosReportNetBindingStub.getOutput(CognosReportNetBindingStub.java:7141)
com.steps.application.service.provider.ManageReportServiceBean.runReportWithStreamOutput(ManageReportServiceBean.java:1909)
com.steps.application.service.provider.ManageReportService_fegjtc_EOImpl.runReportWithStreamOutput(ManageReportService_fegjtc_EOImpl.java:44)
com.steps.application.presentation.controller.BaseReportController.runReportToGetByteStream(BaseReportController.java:1079)
com.steps.application.presentation.controller.RemittanceReportByServiceCodeSubmitController.processRequest(RemittanceReportByServiceCodeSubmitController.java:84)
com.steps.framework.presentation.StepsController.perform(StepsController.java:1744)
org.infohazard.maverick.ctl.FormBeanUser.go(FormBeanUser.java:54)
org.infohazard.maverick.flow.CommandBase.go(CommandBase.java:39)
org.infohazard.maverick.Dispatcher.service(Dispatcher.java:197)
javax.servlet.http.HttpServlet.service(HttpServlet.java:851)
weblogic.servlet.internal.StubSecurityHelper$ServletServiceAction.run(StubSecurityHelper.java:224)
weblogic.servlet.internal.StubSecurityHelper.invokeServlet(StubSecurityHelper.java:108)
weblogic.servlet.internal.ServletStubImpl.execute(ServletStubImpl.java:198)
weblogic.servlet.internal.TailFilter.doFilter(TailFilter.java:26)
weblogic.servlet.internal.FilterChainImpl.doFilter(FilterChainImpl.java:41)
com.steps.framework.presentation.DecryptFIlter.doFilter(DecryptFIlter.java:240)
weblogic.servlet.internal.FilterChainImpl.doFilter(FilterChainImpl.java:41)
com.opensymphony.module.sitemesh.filter.PageFilter.doFilter(PageFilter.java:60)
weblogic.servlet.internal.FilterChainImpl.doFilter(FilterChainImpl.java:41)
com.sun.identity.agents.filter.AmAgentBaseFilter.allowRequestToContinue(AmAgentBaseFilter.java:80)
com.sun.identity.agents.filter.AmAgentBaseFilter.doFilter(AmAgentBaseFilter.java:33)
weblogic.servlet.internal.FilterChainImpl.doFilter(FilterChainImpl.java:41)
weblogic.servlet.internal.RequestDispatcherImpl.invokeServlet(RequestDispatcherImpl.java:524)
weblogic.servlet.internal.RequestDispatcherImpl.forward(RequestDispatcherImpl.java:100)
org.infohazard.maverick.view.DispatchedViewFactory$DispatchedView.go(DispatchedViewFactory.java:82)
org.infohazard.maverick.view.DocumentView.go(DocumentView.java:47)
org.infohazard.maverick.flow.CommandBase.go(CommandBase.java:39)
org.infohazard.maverick.Dispatcher.service(Dispatcher.java:197)
javax.servlet.http.HttpServlet.service(HttpServlet.java:851)
weblogic.servlet.internal.StubSecurityHelper$ServletServiceAction.run(StubSecurityHelper.java:224)
weblogic.servlet.internal.StubSecurityHelper.invokeServlet(StubSecurityHelper.java:108)
weblogic.servlet.internal.ServletStubImpl.execute(ServletStubImpl.java:198)
weblogic.servlet.internal.TailFilter.doFilter(TailFilter.java:26)
weblogic.servlet.internal.FilterChainImpl.doFilter(FilterChainImpl.java:41)
com.steps.framework.presentation.DecryptFIlter.doFilter(DecryptFIlter.java:240)
weblogic.servlet.internal.FilterChainImpl.doFilter(FilterChainImpl.java:41)
com.opensymphony.module.sitemesh.filter.PageFilter.parsePage(PageFilter.java:482)
com.opensymphony.module.sitemesh.filter.PageFilter.doFilter(PageFilter.java:60)
weblogic.servlet.internal.FilterChainImpl.doFilter(FilterChainImpl.java:41)
com.sun.identity.agents.filter.AmAgentBaseFilter.allowRequestToContinue(AmAgentBaseFilter.java:80)
com.sun.identity.agents.filter.AmAgentBaseFilter.doFilter(AmAgentBaseFilter.java:33)
weblogic.servlet.internal.FilterChainImpl.doFilter(FilterChainImpl.java:41)
weblogic.servlet.internal.WebAppServletContext$ServletInvocationAction.run(WebAppServletContext.java:3214)
weblogic.security.acl.internal.AuthenticatedSubject.doAs(AuthenticatedSubject.java:308)
weblogic.security.service.SecurityManager.runAs(SecurityManager.java:117)
weblogic.servlet.internal.WebAppServletContext.securedExecute(WebAppServletContext.java:1946)
weblogic.servlet.internal.WebAppServletContext.execute(WebAppServletContext.java:1868)
weblogic.servlet.internal.ServletRequestImpl.run(ServletRequestImpl.java:1331)
weblogic.work.ExecuteThread.execute(ExecuteThread.java:206)
weblogic.work.ExecuteThread.run(ExecuteThread.java:173)
}
Could you please provide us some details that will help
resolve this? Let me know in case you have any issues.
Thanks,
Gaurvi
