[
https://issues.apache.org/jira/browse/AMQ-6763?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=17422541#comment-17422541
]
Marco Bettiol commented on AMQ-6763:
------------------------------------
Hi [~jbonofre] ,
we are experiencing this issue under heavy load: the effect is the consumer
completely stuck and unable to recover. After rebooting the consumer, it keeps
working for few seconds, then hangs again.
The only way we found to workaround the issue is using
_jms.prefetchPolicy.all=*1*_
We dedicated some time to replicate the problem and realized an application
that can reproduce the problem after a few minutes running.
Our replication setup:
* single node
* connection url
** for local tests :(tcp://localhost:61616?jms.rmIdFromConnectionId=true)
** production
(failover:(ssl://...)?jms.rmIdFromConnectionId=true&timeout=3000)
* persistence : kahadb
* Prefetch 1000 (but also lower values hit the problem)
* Wildfly+ XA resource adapter (5.15.15) + MDB
The stacktraces reports hangs when calling "setXid" or "prepare" o "commit"
{code:java}
"default-threads - 31" #293 prio=5 os_prio=0 tid=0x000000002f57f800 nid=0x3064
waiting on condition [0x000000003c77e000]
java.lang.Thread.State: WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000000f8d7cd80> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at
java.util.concurrent.ArrayBlockingQueue.take(ArrayBlockingQueue.java:403)
at
org.apache.activemq.transport.FutureResponse.getResult(FutureResponse.java:48)
at
org.apache.activemq.transport.ResponseCorrelator.request(ResponseCorrelator.java:87)
at
org.apache.activemq.ActiveMQConnection.syncSendPacket(ActiveMQConnection.java:1392)
at
org.apache.activemq.ActiveMQConnection.syncSendPacket(ActiveMQConnection.java:1436)
at
org.apache.activemq.TransactionContext.prepare(TransactionContext.java:469)
at
org.apache.activemq.ra.LocalAndXATransaction.prepare(LocalAndXATransaction.java:130)
at
com.arjuna.ats.internal.jta.resources.arjunacore.XAResourceRecord.topLevelPrepare(XAResourceRecord.java:218)
at
com.arjuna.ats.arjuna.coordinator.BasicAction.doPrepare(BasicAction.java:2662)
at
com.arjuna.ats.arjuna.coordinator.BasicAction.doPrepare(BasicAction.java:2612)
at
com.arjuna.ats.arjuna.coordinator.BasicAction.prepare(BasicAction.java:2155)
- locked <0x00000000f8c92770> (a
com.arjuna.ats.internal.jta.transaction.arjunacore.AtomicAction)
at
com.arjuna.ats.arjuna.coordinator.BasicAction.End(BasicAction.java:1501)
- locked <0x00000000f8c92770> (a
com.arjuna.ats.internal.jta.transaction.arjunacore.AtomicAction)
at
com.arjuna.ats.arjuna.coordinator.TwoPhaseCoordinator.end(TwoPhaseCoordinator.java:96)
at com.arjuna.ats.arjuna.AtomicAction.commit(AtomicAction.java:162)
at
com.arjuna.ats.internal.jta.transaction.arjunacore.TransactionImple.commitAndDisassociate(TransactionImple.java:1200)
at
com.arjuna.ats.internal.jta.transaction.arjunacore.BaseTransaction.commit(BaseTransaction.java:126)
at
com.arjuna.ats.jbossatx.BaseTransactionManagerDelegate.commit(BaseTransactionManagerDelegate.java:89)
at
org.jboss.as.ejb3.inflow.MessageEndpointInvocationHandler.afterDelivery(MessageEndpointInvocationHandler.java:71)
at sun.reflect.GeneratedMethodAccessor94.invoke(Unknown Source)
at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at
org.jboss.as.ejb3.inflow.AbstractInvocationHandler.handle(AbstractInvocationHandler.java:60)
at
org.jboss.as.ejb3.inflow.MessageEndpointInvocationHandler.doInvoke(MessageEndpointInvocationHandler.java:135)
at
org.jboss.as.ejb3.inflow.AbstractInvocationHandler.invoke(AbstractInvocationHandler.java:73)
at
issue.reproducer.app.TestMDB_1B_Generated$$$endpoint20.afterDelivery(Unknown
Source)
at
org.apache.activemq.ra.MessageEndpointProxy$MessageEndpointAlive.afterDelivery(MessageEndpointProxy.java:128)
at
org.apache.activemq.ra.MessageEndpointProxy.afterDelivery(MessageEndpointProxy.java:69)
at
org.apache.activemq.ra.ServerSessionImpl.afterDelivery(ServerSessionImpl.java:225)
at org.apache.activemq.ActiveMQSession.run(ActiveMQSession.java:1072)
- locked <0x0000000085522078> (a java.lang.Object)
at
org.apache.activemq.ra.ServerSessionImpl.run(ServerSessionImpl.java:169)
at org.jboss.jca.core.workmanager.WorkWrapper.run(WorkWrapper.java:223)
at
org.jboss.threads.SimpleDirectExecutor.execute(SimpleDirectExecutor.java:33)
at org.jboss.threads.QueueExecutor.runTask(QueueExecutor.java:808)
at org.jboss.threads.QueueExecutor.access$100(QueueExecutor.java:45)
at org.jboss.threads.QueueExecutor$Worker.run(QueueExecutor.java:849)
at java.lang.Thread.run(Thread.java:748)
at org.jboss.threads.JBossThread.run(JBossThread.java:320)
"default-threads - 27" #289 prio=5 os_prio=0 tid=0x000000002f57b800 nid=0x2bd4
waiting on condition [0x000000003c37e000]
java.lang.Thread.State: WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x0000000091815730> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at
java.util.concurrent.ArrayBlockingQueue.take(ArrayBlockingQueue.java:403)
at
org.apache.activemq.transport.FutureResponse.getResult(FutureResponse.java:48)
at
org.apache.activemq.transport.ResponseCorrelator.request(ResponseCorrelator.java:87)
at
org.apache.activemq.ActiveMQConnection.syncSendPacket(ActiveMQConnection.java:1392)
at
org.apache.activemq.ActiveMQConnection.syncSendPacket(ActiveMQConnection.java:1436)
at
org.apache.activemq.TransactionContext.commit(TransactionContext.java:585)
at
org.apache.activemq.ra.LocalAndXATransaction.commit(LocalAndXATransaction.java:92)
at
com.arjuna.ats.internal.jta.resources.arjunacore.XAResourceRecord.topLevelCommit(XAResourceRecord.java:477)
at
com.arjuna.ats.arjuna.coordinator.BasicAction.doCommit(BasicAction.java:2886)
at
com.arjuna.ats.arjuna.coordinator.BasicAction.doCommit(BasicAction.java:2802)
at
com.arjuna.ats.arjuna.coordinator.BasicAction.phase2Commit(BasicAction.java:1871)
- locked <0x00000000917ec758> (a
com.arjuna.ats.internal.jta.transaction.arjunacore.AtomicAction)
at
com.arjuna.ats.arjuna.coordinator.BasicAction.End(BasicAction.java:1527)
- locked <0x00000000917ec758> (a
com.arjuna.ats.internal.jta.transaction.arjunacore.AtomicAction)
at
com.arjuna.ats.arjuna.coordinator.TwoPhaseCoordinator.end(TwoPhaseCoordinator.java:96)
at com.arjuna.ats.arjuna.AtomicAction.commit(AtomicAction.java:162)
at
com.arjuna.ats.internal.jta.transaction.arjunacore.TransactionImple.commitAndDisassociate(TransactionImple.java:1200)
at
com.arjuna.ats.internal.jta.transaction.arjunacore.BaseTransaction.commit(BaseTransaction.java:126)
at
com.arjuna.ats.jbossatx.BaseTransactionManagerDelegate.commit(BaseTransactionManagerDelegate.java:89)
at
org.jboss.as.ejb3.inflow.MessageEndpointInvocationHandler.afterDelivery(MessageEndpointInvocationHandler.java:71)
{code}
Regards,
Marco
> Thread hangs on setXid
> ----------------------
>
> Key: AMQ-6763
> URL: https://issues.apache.org/jira/browse/AMQ-6763
> Project: ActiveMQ
> Issue Type: Bug
> Components: XA
> Affects Versions: 5.14.5
> Reporter: Jakub
> Assignee: Jean-Baptiste Onofré
> Priority: Minor
> Fix For: 5.17.0, 5.15.16, 5.16.4
>
> Time Spent: 1.5h
> Remaining Estimate: 0h
>
> I've noticed issues with distributed transactions (XA) on karaf when using
> ActiveMQ with JDBC storeage (postgres). After some time (it isn't
> deterministic) I've observed that on database side 'idle in transaction'
> appeared (it's other schema than used by ActiveMQ). After debugging it seams
> that the reason why transactions are hanging is ActiveMQ and
> org.apache.activemq.transport.FutureResponse.getResult method that waits
> forever for a response.
> {code}
> java.lang.Thread.State: WAITING (parking)
> at sun.misc.Unsafe.park(Native Method)
> - parking to wait for <0x0000000768585aa8> (a
> java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
> at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
> at
> java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
> at
> java.util.concurrent.ArrayBlockingQueue.take(ArrayBlockingQueue.java:403)
> at
> org.apache.activemq.transport.FutureResponse.getResult(FutureResponse.java:48)
> at
> org.apache.activemq.transport.ResponseCorrelator.request(ResponseCorrelator.java:87)
> at
> org.apache.activemq.ActiveMQConnection.syncSendPacket(ActiveMQConnection.java:1388)
> at
> org.apache.activemq.ActiveMQConnection.syncSendPacket(ActiveMQConnection.java:1428)
> at
> org.apache.activemq.TransactionContext.setXid(TransactionContext.java:751)
> at
> org.apache.activemq.TransactionContext.invokeBeforeEnd(TransactionContext.java:424)
> at
> org.apache.activemq.TransactionContext.end(TransactionContext.java:408)
> at
> org.apache.geronimo.transaction.manager.WrapperNamedXAResource.end(WrapperNamedXAResource.java:61)
> at
> org.apache.geronimo.transaction.manager.TransactionImpl.endResources(TransactionImpl.java:588)
> at
> org.apache.geronimo.transaction.manager.TransactionImpl.endResources(TransactionImpl.java:567)
> at
> org.apache.geronimo.transaction.manager.TransactionImpl.beforePrepare(TransactionImpl.java:414)
> at
> org.apache.geronimo.transaction.manager.TransactionImpl.commit(TransactionImpl.java:262)
> at
> org.apache.geronimo.transaction.manager.TransactionManagerImpl.commit(TransactionManagerImpl.java:252)
> at
> org.springframework.transaction.jta.JtaTransactionManager.doCommit(JtaTransactionManager.java:1020)
> at
> org.springframework.transaction.support.AbstractPlatformTransactionManager.processCommit(AbstractPlatformTransactionManager.java:761)
> at
> org.springframework.transaction.support.AbstractPlatformTransactionManager.commit(AbstractPlatformTransactionManager.java:730)
> at
> org.apache.aries.transaction.internal.AriesPlatformTransactionManager.commit(AriesPlatformTransactionManager.java:75)
> at
> org.springframework.transaction.interceptor.TransactionAspectSupport.commitTransactionAfterReturning(TransactionAspectSupport.java:484)
> at
> org.springframework.transaction.interceptor.TransactionAspectSupport.invokeWithinTransaction(TransactionAspectSupport.java:291)
> at
> org.springframework.transaction.interceptor.TransactionInterceptor.invoke(TransactionInterceptor.java:96)
> at
> org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:179)
> at
> org.springframework.aop.framework.CglibAopProxy$DynamicAdvisedInterceptor.intercept(CglibAopProxy.java:655)
> ..... custom service
> {code}
> {code}
> "DefaultMessageListenerContainer-3" #13199 prio=5 os_prio=0
> tid=0x00007fb8687e6800 nid=0x3954 waiting on condition [0x00007fb7b0b98000]
> java.lang.Thread.State: WAITING (parking)
> at sun.misc.Unsafe.park(Native Method)
> - parking to wait for <0x0000000765f532c0> (a
> java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
> at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
> at
> java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
> at
> java.util.concurrent.ArrayBlockingQueue.take(ArrayBlockingQueue.java:403)
> at
> org.apache.activemq.transport.FutureResponse.getResult(FutureResponse.java:48)
> at
> org.apache.activemq.transport.ResponseCorrelator.request(ResponseCorrelator.java:87)
> at
> org.apache.activemq.ActiveMQConnection.syncSendPacket(ActiveMQConnection.java:1388)
> at
> org.apache.activemq.ActiveMQConnection.syncSendPacket(ActiveMQConnection.java:1428)
> at
> org.apache.activemq.TransactionContext.setXid(TransactionContext.java:751)
> at
> org.apache.activemq.TransactionContext.invokeBeforeEnd(TransactionContext.java:424)
> at
> org.apache.activemq.TransactionContext.end(TransactionContext.java:408)
> at
> org.apache.geronimo.transaction.manager.WrapperNamedXAResource.end(WrapperNamedXAResource.java:61)
> at
> org.apache.geronimo.transaction.manager.TransactionImpl.endResources(TransactionImpl.java:588)
> at
> org.apache.geronimo.transaction.manager.TransactionImpl.endResources(TransactionImpl.java:567)
> at
> org.apache.geronimo.transaction.manager.TransactionImpl.beforePrepare(TransactionImpl.java:414)
> at
> org.apache.geronimo.transaction.manager.TransactionImpl.commit(TransactionImpl.java:262)
> at
> org.apache.geronimo.transaction.manager.TransactionManagerImpl.commit(TransactionManagerImpl.java:252)
> at
> org.springframework.transaction.jta.JtaTransactionManager.doCommit(JtaTransactionManager.java:1020)
> at
> org.springframework.transaction.support.AbstractPlatformTransactionManager.processCommit(AbstractPlatformTransactionManager.java:761)
> at
> org.springframework.transaction.support.AbstractPlatformTransactionManager.commit(AbstractPlatformTransactionManager.java:730)
> at
> org.apache.aries.transaction.internal.AriesPlatformTransactionManager.commit(AriesPlatformTransactionManager.java:75)
> at
> org.springframework.jms.listener.AbstractPollingMessageListenerContainer.receiveAndExecute(AbstractPollingMessageListenerContainer.java:245)
> at
> org.springframework.jms.listener.DefaultMessageListenerContainer$AsyncMessageListenerInvoker.invokeListener(DefaultMessageListenerContainer.java:1164)
> at
> org.springframework.jms.listener.DefaultMessageListenerContainer$AsyncMessageListenerInvoker.executeOngoingLoop(DefaultMessageListenerContainer.java:1156)
> at
> org.springframework.jms.listener.DefaultMessageListenerContainer$AsyncMessageListenerInvoker.run(DefaultMessageListenerContainer.java:1053)
> at java.lang.Thread.run(Thread.java:748)
> {code}
--
This message was sent by Atlassian Jira
(v8.3.4#803005)
