Hi, We have JBoss 4.2 3 server that is configured with JBoss MQ. We have some components which communicate by JBoss MQ.
Frequently this server either hangs or stops. Sometime we get outofmemoryexceptions, I am not sure what may be the reason for this. I have pasted the log below. 2009-10-16 12:56:08,231 ERROR [org.jboss.ejb.plugins.jms.JMSContainerInvoker] Exception in JMSCI message listener | javax.ejb.TransactionRolledbackLocalException: Unexpected Error | java.lang.OutOfMemoryError: unable to create new native thread | at java.lang.Thread.start0(Native Method) | at java.lang.Thread.start(Thread.java:574) | at sun.rmi.transport.DGCClient$EndpointEntry.<init>(DGCClient.java:223) | at sun.rmi.transport.DGCClient$EndpointEntry.lookup(DGCClient.java:193) | at sun.rmi.transport.DGCClient.registerRefs(DGCClient.java:111) | at sun.rmi.transport.LiveRef.read(LiveRef.java:277) | at sun.rmi.server.UnicastRef2.readExternal(UnicastRef2.java:54) | at java.rmi.server.RemoteObject.readObject(RemoteObject.java:438) | at sun.reflect.GeneratedMethodAccessor177.invoke(Unknown Source) | at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) | at java.lang.reflect.Method.invoke(Method.java:585) | at java.io.ObjectStreamClass.invokeReadObject(ObjectStreamClass.java:946) | at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1818) | at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1718) | at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1304) | at java.io.ObjectInputStream.readObject(ObjectInputStream.java:349) | at java.rmi.MarshalledObject.get(MarshalledObject.java:135) | at org.jnp.interfaces.NamingContext.getServer(NamingContext.java:257) | at org.jnp.interfaces.NamingContext.checkRef(NamingContext.java:1533) | at org.jnp.interfaces.NamingContext.lookup(NamingContext.java:634) | at org.jnp.interfaces.NamingContext.lookup(NamingContext.java:627) | at javax.naming.InitialContext.lookup(InitialContext.java:351) | at uk.co.firstchoice.cyrus.server.jmx.locator.RemoteJBossServerLocator.locate(Unknown Source) | at uk.co.firstchoice.cyrus.server.seasoncontroller.SeasonControlFacade.getRow(SeasonControlFacade.java:148) | at uk.co.firstchoice.cyrus.server.seasoncontroller.SeasonControlFacade.getOnSaleDate(SeasonControlFacade.java:396) | at uk.co.firstchoice.cyrus.server.consensus.InventoryItemFacade.getOnSaleDay(InventoryItemFacade.java:76) | at uk.co.firstchoice.cyrus.server.consensus.InventoryItemFacade.getOnSalePeriod(InventoryItemFacade.java:54) | at uk.co.firstchoice.cyrus.server.consensus.ConsensusBean.createConsensus(ConsensusBean.java:298) | at uk.co.firstchoice.cyrus.server.consensusengine.ConsensusEngineConsumerBean.process(ConsensusEngineConsumerBean.java:204) | at uk.co.firstchoice.cyrus.framework.engine.AbstractJmsEngine.onMessage(AbstractJmsEngine.java:251) | at sun.reflect.GeneratedMethodAccessor297.invoke(Unknown Source) | at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) | at java.lang.reflect.Method.invoke(Method.java:585) | at org.jboss.invocation.Invocation.performCall(Invocation.java:359) | at org.jboss.ejb.MessageDrivenContainer$ContainerInterceptor.invoke(MessageDrivenContainer.java:495) | at org.jboss.resource.connectionmanager.CachedConnectionInterceptor.invoke(CachedConnectionInterceptor.java:158) | at org.jboss.ejb.plugins.MessageDrivenInstanceInterceptor.invoke(MessageDrivenInstanceInterceptor.java:116) | at org.jboss.ejb.plugins.CallValidationInterceptor.invoke(CallValidationInterceptor.java:63) | at org.jboss.ejb.plugins.AbstractTxInterceptor.invokeNext(AbstractTxInterceptor.java:121) | at org.jboss.ejb.plugins.TxInterceptorCMT.runWithTransactions(TxInterceptorCMT.java:350) | at org.jboss.ejb.plugins.TxInterceptorCMT.invoke(TxInterceptorCMT.java:181) | at org.jboss.ejb.plugins.RunAsSecurityInterceptor.invoke(RunAsSecurityInterceptor.java:109) | at org.jboss.ejb.plugins.LogInterceptor.invoke(LogInterceptor.java:205) | at org.jboss.ejb.plugins.ProxyFactoryFinderInterceptor.invoke(ProxyFactoryFinderInterceptor.java:138) | at org.jboss.ejb.MessageDrivenContainer.internalInvoke(MessageDrivenContainer.java:402) | at org.jboss.ejb.Container.invoke(Container.java:960) | at org.jboss.ejb.plugins.jms.JMSContainerInvoker.invoke(JMSContainerInvoker.java:1092) | at org.jboss.ejb.plugins.jms.JMSContainerInvoker$MessageListenerImpl.onMessage(JMSContainerInvoker.java:1392) | at org.jboss.jms.asf.StdServerSession.onMessage(StdServerSession.java:266) | at org.jboss.mq.SpyMessageConsumer.sessionConsumerProcessMessage(SpyMessageConsumer.java:906) | at org.jboss.mq.SpyMessageConsumer.addMessage(SpyMessageConsumer.java:170) | at org.jboss.mq.SpySession.run(SpySession.java:323) | at org.jboss.jms.asf.StdServerSession.run(StdServerSession.java:194) | at EDU.oswego.cs.dl.util.concurrent.PooledExecutor$Worker.run(PooledExecutor.java:761) | at java.lang.Thread.run(Thread.java:595) | | at org.jboss.ejb.plugins.AbstractTxInterceptor.invokeNext(AbstractTxInterceptor.java:262) | at org.jboss.ejb.plugins.TxInterceptorCMT.runWithTransactions(TxInterceptorCMT.java:350) | at org.jboss.ejb.plugins.TxInterceptorCMT.invoke(TxInterceptorCMT.java:181) | at org.jboss.ejb.plugins.RunAsSecurityInterceptor.invoke(RunAsSecurityInterceptor.java:109) | at org.jboss.ejb.plugins.LogInterceptor.invoke(LogInterceptor.java:205) | at org.jboss.ejb.plugins.ProxyFactoryFinderInterceptor.invoke(ProxyFactoryFinderInterceptor.java:138) | at org.jboss.ejb.MessageDrivenContainer.internalInvoke(MessageDrivenContainer.java:402) | at org.jboss.ejb.Container.invoke(Container.java:960) | at org.jboss.ejb.plugins.jms.JMSContainerInvoker.invoke(JMSContainerInvoker.java:1092) | at org.jboss.ejb.plugins.jms.JMSContainerInvoker$MessageListenerImpl.onMessage(JMSContainerInvoker.java:1392) | at org.jboss.jms.asf.StdServerSession.onMessage(StdServerSession.java:266) | at org.jboss.mq.SpyMessageConsumer.sessionConsumerProcessMessage(SpyMessageConsumer.java:906) | at org.jboss.mq.SpyMessageConsumer.addMessage(SpyMessageConsumer.java:170) | at org.jboss.mq.SpySession.run(SpySession.java:323) | at org.jboss.jms.asf.StdServerSession.run(StdServerSession.java:194) | at EDU.oswego.cs.dl.util.concurrent.PooledExecutor$Worker.run(PooledExecutor.java:761) | at java.lang.Thread.run(Thread.java:595) | Caused by: javax.ejb.EJBException: Unexpected Error | java.lang.OutOfMemoryError: unable to create new native thread | at java.lang.Thread.start0(Native Method) | at java.lang.Thread.start(Thread.java:574) | at sun.rmi.transport.DGCClient$EndpointEntry.<init>(DGCClient.java:223) | at sun.rmi.transport.DGCClient$EndpointEntry.lookup(DGCClient.java:193) | at sun.rmi.transport.DGCClient.registerRefs(DGCClient.java:111) | at sun.rmi.transport.LiveRef.read(LiveRef.java:277) | at sun.rmi.server.UnicastRef2.readExternal(UnicastRef2.java:54) | at java.rmi.server.RemoteObject.readObject(RemoteObject.java:438) | at sun.reflect.GeneratedMethodAccessor177.invoke(Unknown Source) | at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) | at java.lang.reflect.Method.invoke(Method.java:585) | at java.io.ObjectStreamClass.invokeReadObject(ObjectStreamClass.java:946) | at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1818) | at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1718) | at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1304) | at java.io.ObjectInputStream.readObject(ObjectInputStream.java:349) | at java.rmi.MarshalledObject.get(MarshalledObject.java:135) | at org.jnp.interfaces.NamingContext.getServer(NamingContext.java:257) | at org.jnp.interfaces.NamingContext.checkRef(NamingContext.java:1533) | at org.jnp.interfaces.NamingContext.lookup(NamingContext.java:634) | at org.jnp.interfaces.NamingContext.lookup(NamingContext.java:627) | at javax.naming.InitialContext.lookup(InitialContext.java:351) | at uk.co.firstchoice.cyrus.server.jmx.locator.RemoteJBossServerLocator.locate(Unknown Source) | at uk.co.firstchoice.cyrus.server.seasoncontroller.SeasonControlFacade.getRow(SeasonControlFacade.java:148) | at uk.co.firstchoice.cyrus.server.seasoncontroller.SeasonControlFacade.getOnSaleDate(SeasonControlFacade.java:396) | at uk.co.firstchoice.cyrus.server.consensus.InventoryItemFacade.getOnSaleDay(InventoryItemFacade.java:76) | at uk.co.firstchoice.cyrus.server.consensus.InventoryItemFacade.getOnSalePeriod(InventoryItemFacade.java:54) | at uk.co.firstchoice.cyrus.server.consensus.ConsensusBean.createConsensus(ConsensusBean.java:298) | at uk.co.firstchoice.cyrus.server.consensusengine.ConsensusEngineConsumerBean.process(ConsensusEngineConsumerBean.java:204) | at uk.co.firstchoice.cyrus.framework.engine.AbstractJmsEngine.onMessage(AbstractJmsEngine.java:251) | at sun.reflect.GeneratedMethodAccessor297.invoke(Unknown Source) | at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) | at java.lang.reflect.Method.invoke(Method.java:585) | at org.jboss.invocation.Invocation.performCall(Invocation.java:359) | at org.jboss.ejb.MessageDrivenContainer$ContainerInterceptor.invoke(MessageDrivenContainer.java:495) | at org.jboss.resource.connectionmanager.CachedConnectionInterceptor.invoke(CachedConnectionInterceptor.java:158) | at org.jboss.ejb.plugins.MessageDrivenInstanceInterceptor.invoke(MessageDrivenInstanceInterceptor.java:116) | at org.jboss.ejb.plugins.CallValidationInterceptor.invoke(CallValidationInterceptor.java:63) | at org.jboss.ejb.plugins.AbstractTxInterceptor.invokeNext(AbstractTxInterceptor.java:121) | at org.jboss.ejb.plugins.TxInterceptorCMT.runWithTransactions(TxInterceptorCMT.java:350) | at org.jboss.ejb.plugins.TxInterceptorCMT.invoke(TxInterceptorCMT.java:181) | at org.jboss.ejb.plugins.RunAsSecurityInterceptor.invoke(RunAsSecurityInterceptor.java:109) | at org.jboss.ejb.plugins.LogInterceptor.invoke(LogInterceptor.java:205) | at org.jboss.ejb.plugins.ProxyFactoryFinderInterceptor.invoke(ProxyFactoryFinderInterceptor.java:138) | at org.jboss.ejb.MessageDrivenContainer.internalInvoke(MessageDrivenContainer.java:402) | at org.jboss.ejb.Container.invoke(Container.java:960) | at org.jboss.ejb.plugins.jms.JMSContainerInvoker.invoke(JMSContainerInvoker.java:1092) | at org.jboss.ejb.plugins.jms.JMSContainerInvoker$MessageListenerImpl.onMessage(JMSContainerInvoker.java:1392) | at org.jboss.jms.asf.StdServerSession.onMessage(StdServerSession.java:266) | at org.jboss.mq.SpyMessageConsumer.sessionConsumerProcessMessage(SpyMessageConsumer.java:906) | at org.jboss.mq.SpyMessageConsumer.addMessage(SpyMessageConsumer.java:170) | at org.jboss.mq.SpySession.run(SpySession.java:323) | at org.jboss.jms.asf.StdServerSession.run(StdServerSession.java:194) | at EDU.oswego.cs.dl.util.concurrent.PooledExecutor$Worker.run(PooledExecutor.java:761) | at java.lang.Thread.run(Thread.java:595) | | at org.jboss.ejb.plugins.AbstractTxInterceptor.invokeNext(AbstractTxInterceptor.java:237) | ... 16 more Also sometimes we get the below socket time out exception: | | 2009-10-16 12:57:08,365 ERROR [uk.co.firstchoice.cyrus.server.consensus.ConsensusBean] Couldn't look up consensus weights. | java.lang.reflect.UndeclaredThrowableException | at uk.co.firstchoice.cyrus.server.jmx.locator.RemoteJBossServerLocator.locate(Unknown Source) | at uk.co.firstchoice.cyrus.server.seasoncontroller.SeasonControlFacade.getRow(SeasonControlFacade.java:148) | at uk.co.firstchoice.cyrus.server.seasoncontroller.SeasonControlFacade.getOnSaleDate(SeasonControlFacade.java:396) | at uk.co.firstchoice.cyrus.server.consensus.InventoryItemFacade.getOnSaleDay(InventoryItemFacade.java:76) | at uk.co.firstchoice.cyrus.server.consensus.InventoryItemFacade.getOnSalePeriod(InventoryItemFacade.java:54) | at uk.co.firstchoice.cyrus.server.consensus.ConsensusBean.createConsensus(ConsensusBean.java:298) | at uk.co.firstchoice.cyrus.server.consensusengine.ConsensusEngineConsumerBean.process(ConsensusEngineConsumerBean.java:204) | at uk.co.firstchoice.cyrus.framework.engine.AbstractJmsEngine.onMessage(AbstractJmsEngine.java:251) | at sun.reflect.GeneratedMethodAccessor297.invoke(Unknown Source) | at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) | at java.lang.reflect.Method.invoke(Method.java:585) | at org.jboss.invocation.Invocation.performCall(Invocation.java:359) | at org.jboss.ejb.MessageDrivenContainer$ContainerInterceptor.invoke(MessageDrivenContainer.java:495) | at org.jboss.resource.connectionmanager.CachedConnectionInterceptor.invoke(CachedConnectionInterceptor.java:158) | at org.jboss.ejb.plugins.MessageDrivenInstanceInterceptor.invoke(MessageDrivenInstanceInterceptor.java:116) | at org.jboss.ejb.plugins.CallValidationInterceptor.invoke(CallValidationInterceptor.java:63) | at org.jboss.ejb.plugins.AbstractTxInterceptor.invokeNext(AbstractTxInterceptor.java:121) | at org.jboss.ejb.plugins.TxInterceptorCMT.runWithTransactions(TxInterceptorCMT.java:350) | at org.jboss.ejb.plugins.TxInterceptorCMT.invoke(TxInterceptorCMT.java:181) | at org.jboss.ejb.plugins.RunAsSecurityInterceptor.invoke(RunAsSecurityInterceptor.java:109) | at org.jboss.ejb.plugins.LogInterceptor.invoke(LogInterceptor.java:205) | at org.jboss.ejb.plugins.ProxyFactoryFinderInterceptor.invoke(ProxyFactoryFinderInterceptor.java:138) | at org.jboss.ejb.MessageDrivenContainer.internalInvoke(MessageDrivenContainer.java:402) | at org.jboss.ejb.Container.invoke(Container.java:960) | at org.jboss.ejb.plugins.jms.JMSContainerInvoker.invoke(JMSContainerInvoker.java:1092) | at org.jboss.ejb.plugins.jms.JMSContainerInvoker$MessageListenerImpl.onMessage(JMSContainerInvoker.java:1392) | at org.jboss.jms.asf.StdServerSession.onMessage(StdServerSession.java:266) | at org.jboss.mq.SpyMessageConsumer.sessionConsumerProcessMessage(SpyMessageConsumer.java:906) | at org.jboss.mq.SpyMessageConsumer.addMessage(SpyMessageConsumer.java:170) | at org.jboss.mq.SpySession.run(SpySession.java:323) | at org.jboss.jms.asf.StdServerSession.run(StdServerSession.java:194) | at EDU.oswego.cs.dl.util.concurrent.PooledExecutor$Worker.run(PooledExecutor.java:761) | at java.lang.Thread.run(Thread.java:595) | Caused by: javax.naming.CommunicationException [Root exception is java.rmi.ConnectIOException: error during JRMP connection establishment; nested exception is: | java.net.SocketTimeoutException: Read timed out] | at org.jnp.interfaces.NamingContext.lookup(NamingContext.java:780) | at org.jnp.interfaces.NamingContext.lookup(NamingContext.java:627) | at javax.naming.InitialContext.lookup(InitialContext.java:351) | ... 33 more | Caused by: java.rmi.ConnectIOException: error during JRMP connection establishment; nested exception is: | java.net.SocketTimeoutException: Read timed out | at sun.rmi.transport.tcp.TCPChannel.createConnection(TCPChannel.java:273) | at sun.rmi.transport.tcp.TCPChannel.newConnection(TCPChannel.java:171) | at sun.rmi.server.UnicastRef.invoke(UnicastRef.java:94) | at org.jnp.server.NamingServer_Stub.lookup(Unknown Source) | at org.jnp.interfaces.NamingContext.lookup(NamingContext.java:667) | ... 35 more | Caused by: java.net.SocketTimeoutException: Read timed out | at java.net.SocketInputStream.socketRead0(Native Method) | at java.net.SocketInputStream.read(SocketInputStream.java:129) | at java.io.BufferedInputStream.fill(BufferedInputStream.java:218) | at java.io.BufferedInputStream.read(BufferedInputStream.java:235) | at java.io.DataInputStream.readByte(DataInputStream.java:241) | at sun.rmi.transport.tcp.TCPChannel.createConnection(TCPChannel.java:215) | ... 39 more | | Please let me know what may the reason for this failure. I have also taken trace details when the server was hangling. I have used pstack and jstack commands. Based on the review of this I will attach those files in the next post. How to attach a file in this forum? Thanks, Regards, Prabhu View the original post : http://www.jboss.org/index.html?module=bb&op=viewtopic&p=4269232#4269232 Reply to the post : http://www.jboss.org/index.html?module=bb&op=posting&mode=reply&p=4269232 _______________________________________________ jboss-user mailing list [email protected] https://lists.jboss.org/mailman/listinfo/jboss-user
