I have a camel route consuming from an ActiveMQ queue, using 8 concurrent
consumers, that appears to hang after 'X' number of exceptions are thrown from
the bean within the route. The application is running in tomcat, after the
application appears hung and before shutting tomcat down, I ran a jstack on the
tomcat pid and found the following:
"Camel (camel) thread #25 - JmsConsumer[mappingLoadable]" daemon prio=10
tid=0x00007fdd30433800 nid=0x5a64 in Object.wait() [0x00007fdd16eeb000]
java.lang.Thread.State: WAITING (on object monitor)
at java.lang.Object.wait(Native Method)
- waiting on <0x0000000786a0c838> (a
org.apache.commons.pool.impl.GenericKeyedObjectPool$Latch)
at java.lang.Object.wait(Object.java:485)
at
org.apache.commons.pool.impl.GenericKeyedObjectPool.borrowObject(GenericKeyedObjectPool.java:1151)
- locked <0x0000000786a0c838> (a
org.apache.commons.pool.impl.GenericKeyedObjectPool$Latch)
at
org.apache.activemq.pool.ConnectionPool.createSession(ConnectionPool.java:142)
at
org.apache.activemq.pool.PooledConnection.createSession(PooledConnection.java:174)
at
org.springframework.jms.support.JmsAccessor.createSession(JmsAccessor.java:196)
at
org.springframework.jms.core.JmsTemplate.execute(JmsTemplate.java:457)
at
org.apache.camel.component.jms.JmsConfiguration$CamelJmsTemplate.send(JmsConfiguration.java:171)
at
org.apache.camel.component.jms.JmsProducer.doSend(JmsProducer.java:402)
at
org.apache.camel.component.jms.JmsProducer.processInOnly(JmsProducer.java:356)
at
org.apache.camel.component.jms.JmsProducer.process(JmsProducer.java:132)
at
org.apache.camel.util.AsyncProcessorHelper.process(AsyncProcessorHelper.java:73)
at
org.apache.camel.processor.SendProcessor$2.doInAsyncProducer(SendProcessor.java:122)
at
org.apache.camel.impl.ProducerCache.doInAsyncProducer(ProducerCache.java:298)
at
org.apache.camel.processor.SendProcessor.process(SendProcessor.java:117)
at
org.apache.camel.util.AsyncProcessorHelper.process(AsyncProcessorHelper.java:73)
at
org.apache.camel.processor.DelegateAsyncProcessor.processNext(DelegateAsyncProcessor.java:99)
at
org.apache.camel.processor.DelegateAsyncProcessor.process(DelegateAsyncProcessor.java:90)
at
org.apache.camel.management.InstrumentationProcessor.process(InstrumentationProcessor.java:72)
at
org.apache.camel.util.AsyncProcessorHelper.process(AsyncProcessorHelper.java:73)
at
org.apache.camel.processor.DelegateAsyncProcessor.processNext(DelegateAsyncProcessor.java:99)
at
org.apache.camel.processor.DelegateAsyncProcessor.process(DelegateAsyncProcessor.java:90)
at
org.apache.camel.processor.interceptor.BacklogTracerInterceptor.process(BacklogTracerInterceptor.java:84)
at
org.apache.camel.util.AsyncProcessorHelper.process(AsyncProcessorHelper.java:73)
at
org.apache.camel.processor.DelegateAsyncProcessor.processNext(DelegateAsyncProcessor.java:99)
at
org.apache.camel.processor.DelegateAsyncProcessor.process(DelegateAsyncProcessor.java:90)
at
org.apache.camel.processor.interceptor.TraceInterceptor.process(TraceInterceptor.java:91)
at
org.apache.camel.processor.interceptor.HandleFaultInterceptor.process(HandleFaultInterceptor.java:41)
at
org.apache.camel.util.AsyncProcessorHelper.process(AsyncProcessorHelper.java:73)
at
org.apache.camel.processor.DelegateAsyncProcessor.processNext(DelegateAsyncProcessor.java:99)
at
org.apache.camel.processor.DelegateAsyncProcessor.process(DelegateAsyncProcessor.java:90)
at
org.apache.camel.processor.RouteContextProcessor.processNext(RouteContextProcessor.java:46)
at
org.apache.camel.processor.DelegateAsyncProcessor.process(DelegateAsyncProcessor.java:90)
at
org.apache.camel.processor.interceptor.DefaultChannel.process(DefaultChannel.java:335)
at
org.apache.camel.util.AsyncProcessorHelper.process(AsyncProcessorHelper.java:73)
at
org.apache.camel.processor.DelegateAsyncProcessor.processNext(DelegateAsyncProcessor.java:99)
at
org.apache.camel.processor.FatalFallbackErrorHandler.processNext(FatalFallbackErrorHandler.java:42)
at
org.apache.camel.processor.DelegateAsyncProcessor.process(DelegateAsyncProcessor.java:90)
at
org.apache.camel.util.AsyncProcessorHelper.process(AsyncProcessorHelper.java:73)
at
org.apache.camel.processor.RedeliveryErrorHandler.deliverToFailureProcessor(RedeliveryErrorHandler.java:825)
at
org.apache.camel.processor.RedeliveryErrorHandler.processErrorHandler(RedeliveryErrorHandler.java:329)
at
org.apache.camel.processor.RedeliveryErrorHandler.process(RedeliveryErrorHandler.java:273)
at
org.apache.camel.processor.RouteContextProcessor.processNext(RouteContextProcessor.java:46)
at
org.apache.camel.processor.DelegateAsyncProcessor.process(DelegateAsyncProcessor.java:90)
at
org.apache.camel.processor.interceptor.DefaultChannel.process(DefaultChannel.java:335)
at
org.apache.camel.processor.RouteContextProcessor.processNext(RouteContextProcessor.java:46)
at
org.apache.camel.processor.DelegateAsyncProcessor.process(DelegateAsyncProcessor.java:90)
at
org.apache.camel.processor.UnitOfWorkProcessor.processAsync(UnitOfWorkProcessor.java:150)
at
org.apache.camel.processor.UnitOfWorkProcessor.process(UnitOfWorkProcessor.java:117)
at
org.apache.camel.processor.RouteInflightRepositoryProcessor.processNext(RouteInflightRepositoryProcessor.java:48)
at
org.apache.camel.processor.DelegateAsyncProcessor.process(DelegateAsyncProcessor.java:90)
at
org.apache.camel.util.AsyncProcessorHelper.process(AsyncProcessorHelper.java:73)
at
org.apache.camel.processor.DelegateAsyncProcessor.processNext(DelegateAsyncProcessor.java:99)
at
org.apache.camel.processor.DelegateAsyncProcessor.process(DelegateAsyncProcessor.java:90)
at
org.apache.camel.management.InstrumentationProcessor.process(InstrumentationProcessor.java:72)
at
org.apache.camel.util.AsyncProcessorHelper.process(AsyncProcessorHelper.java:99)
at
org.apache.camel.processor.DelegateAsyncProcessor.process(DelegateAsyncProcessor.java:86)
at
org.apache.camel.component.jms.EndpointMessageListener.onMessage(EndpointMessageListener.java:104)
at
org.springframework.jms.listener.AbstractMessageListenerContainer.doInvokeListener(AbstractMessageListenerContainer.java:562)
at
org.springframework.jms.listener.AbstractMessageListenerContainer.invokeListener(AbstractMessageListenerContainer.java:500)
at
org.springframework.jms.listener.AbstractMessageListenerContainer.doExecuteListener(AbstractMessageListenerContainer.java:468)
at
org.springframework.jms.listener.AbstractPollingMessageListenerContainer.doReceiveAndExecute(AbstractPollingMessageListenerContainer.java:325)
at
org.springframework.jms.listener.AbstractPollingMessageListenerContainer.receiveAndExecute(AbstractPollingMessageListenerContainer.java:263)
at
org.springframework.jms.listener.DefaultMessageListenerContainer$AsyncMessageListenerInvoker.invokeListener(DefaultMessageListenerContainer.java:1096)
at
org.springframework.jms.listener.DefaultMessageListenerContainer$AsyncMessageListenerInvoker.executeOngoingLoop(DefaultMessageListenerContainer.java:1088)
at
org.springframework.jms.listener.DefaultMessageListenerContainer$AsyncMessageListenerInvoker.run(DefaultMessageListenerContainer.java:985)
at
java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
at java.lang.Thread.run(Thread.java:662)
This same stacktrace was found for all 8 consumer threads.
I am using Camel 2.11.0, ActiveMQ 5.8.0, Tomcat 7.0.42 and Spring 3.2.3 and
Commons Pool 1.6.
Application is running locally on Linux Mint 15 and Java 1.6.0_25.
I tried using Camel 2.12.1 and had the same issue. I'm sure I probably have
some configuration slightly off.
Here are the pertaining pieces of my spring configuration:
<!-- ActiveMQ config for loadable filter route -->
<bean id="ndl_jmsConnectionFactory"
class="org.apache.activemq.ActiveMQConnectionFactory">
<property name="brokerURL"
value="failover:(tcp://localhost:61616,tcp://relaydevmq02.tmsgf.trb:61616)?jms.prefetchPolicy.queuePrefetch=1&randomize=false&maxReconnectDelay=250"
/>
</bean>
<bean id="ndl_pooledConnectionFactory"
class="org.apache.activemq.pool.PooledConnectionFactory" destroy-method="stop">
<property name="maxConnections" value="16" />
<property name="maximumActive" value="8" />
<property name="connectionFactory" ref="ndl_jmsConnectionFactory" />
<property name="idleTimeout" value="120000" />
</bean>
<bean id="ndl_jmsConfig"
class="org.apache.camel.component.jms.JmsConfiguration">
<property name="connectionFactory"
ref="ndl_pooledConnectionFactory"/>
<property name="concurrentConsumers" value="8" />
</bean>
<bean id="activemq"
class="org.apache.activemq.camel.component.ActiveMQComponent">
<property name="configuration" ref="ndl_jmsConfig"/>
</bean>
<camelContext id="camel" handleFault="true"
xmlns="http://camel.apache.org/schema/spring";
xmlns:ldb="http://www.tribune.com/relay/loadable";
shutdownRunningTask="CompleteCurrentTaskOnly">
<onException>
<exception>java.lang.Exception</exception>
<redeliveryPolicy redeliveryDelay="250"
maximumRedeliveries="2" />
<handled><constant>false</constant></handled>
<to uri="activemq:queue:loadableFailure" />
</onException>
<route id="mappingLoaderRoute">
<from uri="activemq:queue:mappingLoadable" />
<to uri="bean:loaderServiceEndpoint?method=load" />
</route>
</camelContext>
Thanks in advance for your time and help.
