[
https://issues.apache.org/jira/browse/AMQ-5026?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=15018408#comment-15018408
]
Alex Paransky commented on AMQ-5026:
------------------------------------
I am having a similar problem with a very similar configuration:
3 servers, 1 master/2 slave configuration, seeing this exception with 5.10.0:
{code}
2015-11-20 15:57:34,832 | INFO | Stopping BrokerService[localhost] due to
exception, java.io.EOFException |
org.apache.activemq.util.DefaultIOExceptionHandler | LevelDB IOException
handler.
java.io.EOFException
at
org.apache.activemq.util.DataByteArrayInputStream.readOrIOException(DataByteArrayInputStream.java:127)[activemq-client-5.10.0.jar:5.10.0]
at
org.apache.activemq.util.DataByteArrayInputStream.readBoolean(DataByteArrayInputStream.java:189)[activemq-client-5.10.0.jar:5.10.0]
at
org.apache.activemq.openwire.v6.BaseDataStreamMarshaller.looseUnmarshalByteSequence(BaseDataStreamMarshaller.java:637)[activemq-openwire-legacy-5.10.0.jar:5.10.0]
at
org.apache.activemq.openwire.v6.MessageMarshaller.looseUnmarshal(MessageMarshaller.java:229)[activemq-openwire-legacy-5.10.0.jar:5.10.0]
at
org.apache.activemq.openwire.v6.ActiveMQMessageMarshaller.looseUnmarshal(ActiveMQMessageMarshaller.java:101)[activemq-openwire-legacy-5.10.0.jar:5.10.0]
at
org.apache.activemq.openwire.v6.ActiveMQTextMessageMarshaller.looseUnmarshal(ActiveMQTextMessageMarshaller.java:101)[activemq-openwire-legacy-5.10.0.jar:5.10.0]
at
org.apache.activemq.openwire.OpenWireFormat.doUnmarshal(OpenWireFormat.java:356)[activemq-client-5.10.0.jar:5.10.0]
at
org.apache.activemq.openwire.OpenWireFormat.unmarshal(OpenWireFormat.java:193)[activemq-client-5.10.0.jar:5.10.0]
at
org.apache.activemq.leveldb.LevelDBClient.decodeMessage(LevelDBClient.scala:1336)[activemq-leveldb-store-5.10.0.jar:5.10.0]
at
org.apache.activemq.leveldb.LevelDBClient$$anonfun$getMessage$1.apply(LevelDBClient.scala:1327)[activemq-leveldb-store-5.10.0.jar:5.10.0]
at
org.apache.activemq.leveldb.LevelDBClient$$anonfun$getMessage$1.apply(LevelDBClient.scala:1327)[activemq-leveldb-store-5.10.0.jar:5.10.0]
at scala.Option.map(Option.scala:145)[scala-library-2.11.0.jar:]
at
org.apache.activemq.leveldb.LevelDBClient.getMessage(LevelDBClient.scala:1327)[activemq-leveldb-store-5.10.0.jar:5.10.0]
at
org.apache.activemq.leveldb.LevelDBClient$$anonfun$queueCursor$1.apply(LevelDBClient.scala:1262)[activemq-leveldb-store-5.10.0.jar:5.10.0]
at
org.apache.activemq.leveldb.LevelDBClient$$anonfun$queueCursor$1.apply(LevelDBClient.scala:1259)[activemq-leveldb-store-5.10.0.jar:5.10.0]
at
org.apache.activemq.leveldb.LevelDBClient$$anonfun$collectionCursor$1$$anonfun$apply$mcV$sp$12.apply(LevelDBClient.scala:1347)[activemq-leveldb-store-5.10.0.jar:5.10.0]
at
org.apache.activemq.leveldb.LevelDBClient$$anonfun$collectionCursor$1$$anonfun$apply$mcV$sp$12.apply(LevelDBClient.scala:1346)[activemq-leveldb-store-5.10.0.jar:5.10.0]
at
org.apache.activemq.leveldb.LevelDBClient$RichDB.check$4(LevelDBClient.scala:323)[activemq-leveldb-store-5.10.0.jar:5.10.0]
at
org.apache.activemq.leveldb.LevelDBClient$RichDB.cursorRange(LevelDBClient.scala:325)[activemq-leveldb-store-5.10.0.jar:5.10.0]
at
org.apache.activemq.leveldb.LevelDBClient$$anonfun$collectionCursor$1.apply$mcV$sp(LevelDBClient.scala:1346)[activemq-leveldb-store-5.10.0.jar:5.10.0]
at
org.apache.activemq.leveldb.LevelDBClient$$anonfun$collectionCursor$1.apply(LevelDBClient.scala:1346)[activemq-leveldb-store-5.10.0.jar:5.10.0]
at
org.apache.activemq.leveldb.LevelDBClient$$anonfun$collectionCursor$1.apply(LevelDBClient.scala:1346)[activemq-leveldb-store-5.10.0.jar:5.10.0]
at
org.apache.activemq.leveldb.LevelDBClient.usingIndex(LevelDBClient.scala:1026)[activemq-leveldb-store-5.10.0.jar:5.10.0]
at
org.apache.activemq.leveldb.LevelDBClient$$anonfun$might_fail_using_index$1.apply(LevelDBClient.scala:1032)[activemq-leveldb-store-5.10.0.jar:5.10.0]
at
org.apache.activemq.leveldb.LevelDBClient.might_fail(LevelDBClient.scala:549)[activemq-leveldb-store-5.10.0.jar:5.10.0]
at
org.apache.activemq.leveldb.LevelDBClient.might_fail_using_index(LevelDBClient.scala:1032)[activemq-leveldb-store-5.10.0.jar:5.10.0]
at
org.apache.activemq.leveldb.LevelDBClient.collectionCursor(LevelDBClient.scala:1345)[activemq-leveldb-store-5.10.0.jar:5.10.0]
at
org.apache.activemq.leveldb.LevelDBClient.queueCursor(LevelDBClient.scala:1259)[activemq-leveldb-store-5.10.0.jar:5.10.0]
at
org.apache.activemq.leveldb.DBManager.cursorMessages(DBManager.scala:735)[activemq-leveldb-store-5.10.0.jar:5.10.0]
at
org.apache.activemq.leveldb.LevelDBStore$LevelDBMessageStore.recoverNextMessages(LevelDBStore.scala:762)[activemq-leveldb-store-5.10.0.jar:5.10.0]
at
org.apache.activemq.broker.region.cursors.QueueStorePrefetch.doFillBatch(QueueStorePrefetch.java:106)[activemq-broker-5.10.0.jar:5.10.0]
at
org.apache.activemq.broker.region.cursors.AbstractStoreCursor.fillBatch(AbstractStoreCursor.java:277)[activemq-broker-5.10.0.jar:5.10.0]
at
org.apache.activemq.broker.region.cursors.AbstractStoreCursor.reset(AbstractStoreCursor.java:125)[activemq-broker-5.10.0.jar:5.10.0]
at
org.apache.activemq.broker.region.cursors.StoreQueueCursor.reset(StoreQueueCursor.java:157)[activemq-broker-5.10.0.jar:5.10.0]
at
org.apache.activemq.broker.region.Queue.doPageInForDispatch(Queue.java:1956)[activemq-broker-5.10.0.jar:5.10.0]
at
org.apache.activemq.broker.region.Queue.pageInMessages(Queue.java:2177)[activemq-broker-5.10.0.jar:5.10.0]
at
org.apache.activemq.broker.region.Queue.iterate(Queue.java:1658)[activemq-broker-5.10.0.jar:5.10.0]
at
org.apache.activemq.thread.PooledTaskRunner.runTask(PooledTaskRunner.java:133)[activemq-client-5.10.0.jar:5.10.0]
at
org.apache.activemq.thread.PooledTaskRunner$1.run(PooledTaskRunner.java:48)[activemq-client-5.10.0.jar:5.10.0]
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1110)[:1.7.0_06]
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)[:1.7.0_06]
at java.lang.Thread.run(Thread.java:722)[:1.7.0_06]
{code}
Are there any solutions or anything to check. This feels like message
corruption in the leveldb which causes de-serialization errors...
> Replicated LevelDB Store getting EOF exception
> ----------------------------------------------
>
> Key: AMQ-5026
> URL: https://issues.apache.org/jira/browse/AMQ-5026
> Project: ActiveMQ
> Issue Type: Bug
> Components: activemq-leveldb-store
> Affects Versions: 5.10.0
> Reporter: kal patel
>
> 3 brokers setup for ReplicatedLevelDB master/slave conf.
> running producer and slow consumer and doing failover by stopping master
> broker.
> After few failovers, the leveldb throws EOF exception while reading record
> from the store.
> using the latest source code from activemq 5.10 branch.
> The file that has the issue:
> [root@vmx0568 data]# ls -l /solidstate/ldbkp/000000001f4022e4.log
> -rw-r--r--. 1 root root 91500224 Jan 17 14:44
> /solidstate/ldbkp/000000001f4022e4.log
> [root@vmx0568 data]#
> Client | main
> 2014-01-17 14:44:19,118 | INFO | locator read: DataLocator(24b41aac, 2283) |
> org.apache.activemq.leveldb.LevelDBClient | main
> 2014-01-17 14:44:19,118 | INFO | Seq read: 260217 |
> org.apache.activemq.leveldb.LevelDBClient | main
> 2014-01-17 14:44:19,118 | INFO | locator read: DataLocator(24b423ee, 2283) |
> org.apache.activemq.leveldb.LevelDBClient | main
> 2014-01-17 14:44:19,118 | INFO | Seq read: 260218 |
> org.apache.activemq.leveldb.LevelDBClient | main
> 2014-01-17 14:44:19,118 | INFO | locator read: DataLocator(24b42d02, 2283) |
> org.apache.activemq.leveldb.LevelDBClient | main
> 2014-01-17 14:44:19,118 | INFO | Seq read: 260219 |
> org.apache.activemq.leveldb.LevelDBClient | main
> 2014-01-17 14:44:19,118 | INFO | locator read: DataLocator(24b43628, 2283) |
> org.apache.activemq.leveldb.LevelDBClient | main
> 2014-01-17 14:44:19,118 | INFO | Seq read: 260220 |
> org.apache.activemq.leveldb.LevelDBClient | main
> 2014-01-17 14:44:19,118 | INFO | locator read: DataLocator(24b43f6a, 2283) |
> org.apache.activemq.leveldb.LevelDBClient | main
> 2014-01-17 14:44:19,118 | INFO | Seq read: 260221 |
> org.apache.activemq.leveldb.LevelDBClient | main
> 2014-01-17 14:44:19,118 | INFO | locator read: DataLocator(24b4487e, 2283) |
> org.apache.activemq.leveldb.LevelDBClient | main
> 2014-01-17 14:44:19,118 | INFO | Seq read: 260222 |
> org.apache.activemq.leveldb.LevelDBClient | main
> 2014-01-17 14:44:19,118 | INFO | locator read: DataLocator(24b451a4, 2283) |
> org.apache.activemq.leveldb.LevelDBClient | main
> 2014-01-17 14:44:19,124 | INFO | No IOExceptionHandler registered, ignoring
> IO exception | org.apache.activemq.broker.BrokerService | LevelDB IOException
> handler.
> java.io.EOFException: File '/solidstate/ldbkp/000000001f4022e4.log' offset:
> 91500234
> at
> org.apache.activemq.leveldb.RecordLog$LogReader.read(RecordLog.scala:299)
> at
> org.apache.activemq.leveldb.RecordLog$$anonfun$read$2.apply(RecordLog.scala:564)
>
> at
> org.apache.activemq.leveldb.RecordLog$$anonfun$read$2.apply(RecordLog.scala:564)
>
> at
> org.apache.activemq.leveldb.RecordLog.get_reader(RecordLog.scala:554)
> at org.apache.activemq.leveldb.RecordLog.read(RecordLog.scala:564)
> at
> org.apache.activemq.leveldb.LevelDBClient.getMessage(LevelDBClient.scala:1318)
>
> at
> org.apache.activemq.leveldb.LevelDBClient$$anonfun$queueCursor$1.apply(LevelDBClient.scala:1257)
>
> at
> org.apache.activemq.leveldb.LevelDBClient$$anonfun$queueCursor$1.apply(LevelDBClient.scala:1252)
>
> at
> org.apache.activemq.leveldb.LevelDBClient$$anonfun$collectionCursor$1$$anonfun$apply$mcV$sp$12.apply(LevelDBClient.scala:1342)
>
> at
> org.apache.activemq.leveldb.LevelDBClient$$anonfun$collectionCursor$1$$anonfun$apply$mcV$sp$12.apply(LevelDBClient.scala:1341)
>
> at
> org.apache.activemq.leveldb.LevelDBClient$RichDB.check$4(LevelDBClient.scala:323)
>
> at
> org.apache.activemq.leveldb.LevelDBClient$RichDB.cursorRange(LevelDBClient.scala:325)
>
> at
> org.apache.activemq.leveldb.LevelDBClient$$anonfun$collectionCursor$1.apply$mcV$sp(LevelDBClient.scala:1341)
>
> at
> org.apache.activemq.leveldb.LevelDBClient$$anonfun$collectionCursor$1.apply(LevelDBClient.scala:1341)
>
> at
> org.apache.activemq.leveldb.LevelDBClient$$anonfun$collectionCursor$1.apply(LevelDBClient.scala:1341)
>
> at
> org.apache.activemq.leveldb.LevelDBClient.usingIndex(LevelDBClient.scala:1019)
>
> at
> org.apache.activemq.leveldb.LevelDBClient$$anonfun$might_fail_using_index$1.apply(LevelDBClient.scala:1025)
>
> at
> org.apache.activemq.leveldb.LevelDBClient.might_fail(LevelDBClient.scala:549)
> at
> org.apache.activemq.leveldb.LevelDBClient.might_fail_using_index(LevelDBClient.scala:1025)
>
> at
> org.apache.activemq.leveldb.LevelDBClient.collectionCursor(LevelDBClient.scala:1340)
>
> at
> org.apache.activemq.leveldb.LevelDBClient.queueCursor(LevelDBClient.scala:1252)
>
> at
> org.apache.activemq.leveldb.DBManager.cursorMessages(DBManager.scala:713)
> at
> org.apache.activemq.leveldb.LevelDBStore$LevelDBMessageStore.recoverNextMessages(LevelDBStore.scala:758)
>
> at org.apache.activemq.broker.region.Queue.initialize(Queue.java:369)
> at
> org.apache.activemq.broker.region.DestinationFactoryImpl.createDestination(DestinationFactoryImpl.java:87)
>
> at
> org.apache.activemq.broker.region.AbstractRegion.createDestination(AbstractRegion.java:516)
>
> at
> org.apache.activemq.broker.jmx.ManagedQueueRegion.createDestination(ManagedQueueRegion.java:56)
>
> at
> org.apache.activemq.broker.region.AbstractRegion.addDestination(AbstractRegion.java:134)
>
> at
> org.apache.activemq.broker.region.RegionBroker.addDestination(RegionBroker.java:321)
>
> at
> org.apache.activemq.broker.BrokerFilter.addDestination(BrokerFilter.java:167)
> at
> org.apache.activemq.advisory.AdvisoryBroker.addDestination(AdvisoryBroker.java:169)
>
> at
> org.apache.activemq.broker.BrokerFilter.addDestination(BrokerFilter.java:167)
> at
> org.apache.activemq.broker.BrokerFilter.addDestination(BrokerFilter.java:167)
> at
> org.apache.activemq.broker.MutableBrokerFilter.addDestination(MutableBrokerFilter.java:172)
>
> at
> org.apache.activemq.broker.region.AbstractRegion.start(AbstractRegion.java:98)
>
> at
> org.apache.activemq.broker.region.RegionBroker.start(RegionBroker.java:190)
> at
> org.apache.activemq.broker.jmx.ManagedRegionBroker.start(ManagedRegionBroker.java:120)
>
> at
> org.apache.activemq.broker.BrokerFilter.start(BrokerFilter.java:182)
> at
> org.apache.activemq.broker.BrokerFilter.start(BrokerFilter.java:182)
> at
> org.apache.activemq.broker.TransactionBroker.start(TransactionBroker.java:120)
>
> at
> org.apache.activemq.broker.BrokerService$5.start(BrokerService.java:2150)
> at
> org.apache.activemq.broker.BrokerService.doStartBroker(BrokerService.java:648)
>
> at
> org.apache.activemq.broker.BrokerService.startBroker(BrokerService.java:632)
> at
> org.apache.activemq.broker.BrokerService.start(BrokerService.java:568)
> at
> org.apache.activemq.xbean.XBeanBrokerService.afterPropertiesSet(XBeanBrokerService.java:73)
>
> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
> at
> sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
> at
> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
>
> at java.lang.reflect.Method.invoke(Method.java:606)
> at
> org.springframework.beans.factory.support.AbstractAutowireCapableBeanFactory.invokeCustomInitMethod(AbstractAutowireCapableBeanFactory.java:1608)
>
> at
> org.springframework.beans.factory.support.AbstractAutowireCapableBeanFactory.invokeInitMethods(AbstractAutowireCapableBeanFactory.java:1549)
>
> at
> org.springframework.beans.factory.support.AbstractAutowireCapableBeanFactory.initializeBean(AbstractAutowireCapableBeanFactory.java:1479)
>
> at
> org.springframework.beans.factory.support.AbstractAutowireCapableBeanFactory.doCreateBean(AbstractAutowireCapableBeanFactory.java:521)
>
> at
> org.springframework.beans.factory.support.AbstractAutowireCapableBeanFactory.createBean(AbstractAutowireCapableBeanFactory.java:458)
>
> at
> org.springframework.beans.factory.support.AbstractBeanFactory$1.getObject(AbstractBeanFactory.java:295)
>
> at
> org.springframework.beans.factory.support.DefaultSingletonBeanRegistry.getSingleton(DefaultSingletonBeanRegistry.java:223)
>
> at
> org.springframework.beans.factory.support.AbstractBeanFactory.doGetBean(AbstractBeanFactory.java:292)
>
> at
> org.springframework.beans.factory.support.AbstractBeanFactory.getBean(AbstractBeanFactory.java:194)
>
> at
> org.springframework.beans.factory.support.DefaultListableBeanFactory.preInstantiateSingletons(DefaultListableBeanFactory.java:628)
>
> at
> org.springframework.context.support.AbstractApplicationContext.finishBeanFactoryInitialization(AbstractApplicationContext.java:932)
>
> at
> org.springframework.context.support.AbstractApplicationContext.refresh(AbstractApplicationContext.java:479)
>
> at
> org.apache.xbean.spring.context.ResourceXmlApplicationContext.<init>(ResourceXmlApplicationContext.java:64)
>
> at
> org.apache.xbean.spring.context.ResourceXmlApplicationContext.<init>(ResourceXmlApplicationContext.java:52)
>
> at
> org.apache.activemq.xbean.XBeanBrokerFactory$1.<init>(XBeanBrokerFactory.java:104)
>
> at
> org.apache.activemq.xbean.XBeanBrokerFactory.createApplicationContext(XBeanBrokerFactory.java:104)
>
> at
> org.apache.activemq.xbean.XBeanBrokerFactory.createBroker(XBeanBrokerFactory.java:67)
>
> at
> org.apache.activemq.broker.BrokerFactory.createBroker(BrokerFactory.java:71)
> at
> org.apache.activemq.broker.BrokerFactory.createBroker(BrokerFactory.java:54)
> at
> org.apache.activemq.console.command.StartCommand.runTask(StartCommand.java:87)
>
> at
> org.apache.activemq.console.command.AbstractCommand.execute(AbstractCommand.java:57)
>
> at
> org.apache.activemq.console.command.ShellCommand.runTask(ShellCommand.java:150)
>
> at
> org.apache.activemq.console.command.AbstractCommand.execute(AbstractCommand.java:57)
>
> at
> org.apache.activemq.console.command.ShellCommand.main(ShellCommand.java:104)
> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
> at
> sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
> at
> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
>
> at java.lang.reflect.Method.invoke(Method.java:606)
> at org.apache.activemq.console.Main.runTaskClass(Main.java:262)
> at org.apache.activemq.console.Main.main(Main.java:115)
> 2014-01-17 14:44:19,156 | INFO | Stopped LevelDB[/solidstate/ldbkp] |
> org.apache.activemq.leveldb.LevelDBStore | LevelDB IOException handler.
--
This message was sent by Atlassian JIRA
(v6.3.4#6332)
