Hi,
We've two brokers running on shared data-store mounted through NFS.
Broker version is 5.3.0 and jdk is 1.6.0.
Everything works fine in non-production environment. In production
environment, after startup, we see exceptions
that I can't seem to understand or reproduce in non-prod. environment.
Any pointer towards this is really appreciated.
Exception:
java.lang.RuntimeException: Failed to index
at
org.apache.activemq.kaha.impl.index.DiskIndexLinkedList.refreshEntry(DiskIndexLinkedList.java:332)
at
org.apache.activemq.kaha.impl.container.ContainerValueCollectionIterator.<init>(ContainerValueCollectionIterator.java:39)
at
org.apache.activemq.kaha.impl.container.ContainerListIterator.<init>(ContainerListIterator.java:30)
at
org.apache.activemq.kaha.impl.container.ListContainerImpl.listIterator(ListContainerImpl.java:603)
at
org.apache.activemq.broker.region.cursors.FilePendingMessageCursor.reset(FilePendingMessageCursor.java:123)
at org.apache.activemq.broker.region.Queue.doPageIn(Queue.java:1363)
at org.apache.activemq.broker.region.Queue.pageInMessages(Queue.java:1503)
at org.apache.activemq.broker.region.Queue.doBrowse(Queue.java:759)
at org.apache.activemq.broker.region.Queue.expireMessages(Queue.java:588)
at org.apache.activemq.broker.region.Queue.access$000(Queue.java:85)
at org.apache.activemq.broker.region.Queue$2.run(Queue.java:116)
at
org.apache.activemq.thread.SchedulerTimerTask.run(SchedulerTimerTask.java:33)
at java.util.TimerThread.mainLoop(Timer.java:512)
at java.util.TimerThread.run(Timer.java:462)
Caused by: java.io.IOException: Stale NFS file handle
at java.io.RandomAccessFile.readBytes(Native Method)
at java.io.RandomAccessFile.read(RandomAccessFile.java:322)
at java.io.RandomAccessFile.readFully(RandomAccessFile.java:381)
at
org.apache.activemq.kaha.impl.index.StoreIndexReader.updateIndexes(StoreIndexReader.java:57)
at
org.apache.activemq.kaha.impl.index.IndexManager.refreshIndex(IndexManager.java:76)
at
org.apache.activemq.kaha.impl.index.DiskIndexLinkedList.refreshEntry(DiskIndexLinkedList.java:330)
... 13 more
Broker cconfiguration:
<amq:broker brokerName="broker" persistent="true" useShutdownHook="true"
deleteAllMessagesOnStartup="false" advisorySupport="false"
dataDirectory="/sharedPath" useJmx="true" >
<amq:managementContext>
<amq:managementContext createConnector="false" />
</amq:managementContext>
<amq:persistenceAdapter>
<amq:amqPersistenceAdapter directory="/sharedPath"
maxFileLength="32mb" />
</amq:persistenceAdapter>
<amq:transportConnectors>
<amq:transportConnector uri="tcp://
0.0.0.0:61616?wireFormat.maxInactivityDuration=0"/>
</amq:transportConnectors>
<amq:destinations>
<amq:queue id="queue1" physicalName="bqueue" />
</amq:destinations>
<amq:plugins>
<amq:loggingBrokerPlugin logAll="true" />
</amq:plugins>
<amq:destinationPolicy>
<amq:policyMap>
<amq:policyEntries>
<amq:policyEntry queue=">" producerFlowControl="true"
memoryLimit="1mb">
<amq:pendingQueuePolicy>
<amq:fileQueueCursor/>
</amq:pendingQueuePolicy>
</amq:policyEntry>
</amq:policyEntries>
</amq:policyMap>
</amq:destinationPolicy>
<amq:systemUsage>
<amq:systemUsage sendFailIfNoSpace="true">
<amq:memoryUsage>
<amq:memoryUsage id="memoryUsage" limit="4 gb"/>
</amq:memoryUsage>
<amq:storeUsage>
<amq:storeUsage id="storeUsage" limit="10 gb" />
</amq:storeUsage>
<amq:tempUsage>
<amq:tempUsage id="tempUsage" limit="10 gb" />
</amq:tempUsage>
</amq:systemUsage>
</amq:systemUsage>
</amq:broker>
