[ https://issues.apache.org/jira/browse/AMQ-6174?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=15583154#comment-15583154 ]
Vijay Govindarajulu edited comment on AMQ-6174 at 10/17/16 7:16 PM: -------------------------------------------------------------------- I am faced with the same problem reported by [~sunilasfvish], my cluster config is a little different Affects Version/s: 5.11.1 ActiveMQ 5.11.1 with replicated LevelDB (3 node cluster). Zookeeper 3.4.6 (3 node cluster). File system: ext4 {code} 2016-09-28 10:54:40,849 | INFO | Stopping BrokerService[localhost] due to exception, java.io.EOFException: File '/opt/activemq/activemq-batch/data/leveldb/000000000c8002d7.log' offset: 3012576207 | org.apache.activemq.u til.DefaultIOExceptionHandler | LevelDB IOException handler. java.io.EOFException: File '/opt/activemq/activemq-batch/data/leveldb/000000000c8002d7.log' offset: 3012576207 at org.apache.activemq.leveldb.RecordLog$LogReader.read(RecordLog.scala:389)[activemq-leveldb-store-5.11.1.jar:5.11.1] at org.apache.activemq.leveldb.RecordLog$$anonfun$read$2.apply(RecordLog.scala:654)[activemq-leveldb-store-5.11.1.jar:5.11.1] at org.apache.activemq.leveldb.RecordLog$$anonfun$read$2.apply(RecordLog.scala:654)[activemq-leveldb-store-5.11.1.jar:5.11.1] at org.apache.activemq.leveldb.RecordLog.get_reader(RecordLog.scala:644)[activemq-leveldb-store-5.11.1.jar:5.11.1] at org.apache.activemq.leveldb.RecordLog.read(RecordLog.scala:654)[activemq-leveldb-store-5.11.1.jar:5.11.1] at org.apache.activemq.leveldb.LevelDBClient.getMessage(LevelDBClient.scala:1335)[activemq-leveldb-store-5.11.1.jar:5.11.1] at org.apache.activemq.leveldb.LevelDBClient$$anonfun$queueCursor$1.apply(LevelDBClient.scala:1274)[activemq-leveldb-store-5.11.1.jar:5.11.1] at org.apache.activemq.leveldb.LevelDBClient$$anonfun$queueCursor$1.apply(LevelDBClient.scala:1271)[activemq-leveldb-store-5.11.1.jar:5.11.1] at org.apache.activemq.leveldb.LevelDBClient$$anonfun$collectionCursor$1$$anonfun$apply$mcV$sp$12.apply(LevelDBClient.scala:1359)[activemq-leveldb-store-5.11.1.jar:5.11.1] at org.apache.activemq.leveldb.LevelDBClient$$anonfun$collectionCursor$1$$anonfun$apply$mcV$sp$12.apply(LevelDBClient.scala:1358)[activemq-leveldb-store-5.11.1.jar:5.11.1] at org.apache.activemq.leveldb.LevelDBClient$RichDB.check$4(LevelDBClient.scala:323)[activemq-leveldb-store-5.11.1.jar:5.11.1] at org.apache.activemq.leveldb.LevelDBClient$RichDB.cursorRange(LevelDBClient.scala:325)[activemq-leveldb-store-5.11.1.jar:5.11.1] at org.apache.activemq.leveldb.LevelDBClient$$anonfun$collectionCursor$1.apply$mcV$sp(LevelDBClient.scala:1358)[activemq-leveldb-store-5.11.1.jar:5.11.1] at org.apache.activemq.leveldb.LevelDBClient$$anonfun$collectionCursor$1.apply(LevelDBClient.scala:1358)[activemq-leveldb-store-5.11.1.jar:5.11.1] at org.apache.activemq.leveldb.LevelDBClient$$anonfun$collectionCursor$1.apply(LevelDBClient.scala:1358)[activemq-leveldb-store-5.11.1.jar:5.11.1] at org.apache.activemq.leveldb.LevelDBClient.usingIndex(LevelDBClient.scala:1038)[activemq-leveldb-store-5.11.1.jar:5.11.1] at org.apache.activemq.leveldb.LevelDBClient$$anonfun$might_fail_using_index$1.apply(LevelDBClient.scala:1044)[activemq-leveldb-store-5.11.1.jar:5.11.1] at org.apache.activemq.leveldb.LevelDBClient.might_fail(LevelDBClient.scala:549)[activemq-leveldb-store-5.11.1.jar:5.11.1] at org.apache.activemq.leveldb.LevelDBClient.might_fail_using_index(LevelDBClient.scala:1044)[activemq-leveldb-store-5.11.1.jar:5.11.1] at org.apache.activemq.leveldb.LevelDBClient.collectionCursor(LevelDBClient.scala:1357)[activemq-leveldb-store-5.11.1.jar:5.11.1] at org.apache.activemq.leveldb.LevelDBClient.queueCursor(LevelDBClient.scala:1271)[activemq-leveldb-store-5.11.1.jar:5.11.1] at org.apache.activemq.leveldb.DBManager.cursorMessages(DBManager.scala:735)[activemq-leveldb-store-5.11.1.jar:5.11.1] at org.apache.activemq.leveldb.LevelDBStore$LevelDBMessageStore.recoverNextMessages(LevelDBStore.scala:860)[activemq-leveldb-store-5.11.1.jar:5.11.1] at org.apache.activemq.broker.region.cursors.QueueStorePrefetch.doFillBatch(QueueStorePrefetch.java:109)[activemq-broker-5.11.1.jar:5.11.1] at org.apache.activemq.broker.region.cursors.AbstractStoreCursor.fillBatch(AbstractStoreCursor.java:381)[activemq-broker-5.11.1.jar:5.11.1] at org.apache.activemq.broker.region.cursors.AbstractStoreCursor.reset(AbstractStoreCursor.java:142)[activemq-broker-5.11.1.jar:5.11.1] at org.apache.activemq.broker.region.cursors.StoreQueueCursor.reset(StoreQueueCursor.java:159)[activemq-broker-5.11.1.jar:5.11.1] at org.apache.activemq.broker.region.Queue.doPageInForDispatch(Queue.java:1897)[activemq-broker-5.11.1.jar:5.11.1] at org.apache.activemq.broker.region.Queue.pageInMessages(Queue.java:2119)[activemq-broker-5.11.1.jar:5.11.1] at org.apache.activemq.broker.region.Queue.doBrowse(Queue.java:1116)[activemq-broker-5.11.1.jar:5.11.1] at org.apache.activemq.broker.region.Queue.expireMessages(Queue.java:909)[activemq-broker-5.11.1.jar:5.11.1] at org.apache.activemq.broker.region.Queue.access$100(Queue.java:100)[activemq-broker-5.11.1.jar:5.11.1] at org.apache.activemq.broker.region.Queue$2.run(Queue.java:144)[activemq-broker-5.11.1.jar:5.11.1] at org.apache.activemq.thread.SchedulerTimerTask.run(SchedulerTimerTask.java:33)[activemq-client-5.11.1.jar:5.11.1] at java.util.TimerThread.mainLoop(Timer.java:555)[:1.8.0_92] at java.util.TimerThread.run(Timer.java:505)[:1.8.0_92]" {code} activemq.xml snip {code:xml} <persistenceAdapter> <replicatedLevelDB directory="${activemq.data}/leveldb" replicas="2" bind="tcp://0.0.0.0:61613" zkAddress="zk-1:2181,zk-2:2181,zk-3:2181" zkPassword="" zkPath="${activemq.data}/batch/leveldb" sync="quorum_disk" /> </persistenceAdapter> {code} I was able to resolve the issue by removing the data folder on all the cluster nodes and restarting the Brokers. *This error causes the activemq cluster to failover continuously* was (Author: vgovindarajulu): I am faced with the same problem reported by [~sunilasfvish], my cluster config is a little different Affects Version/s: 5.11.1 ActiveMQ 5.11.1 with replicated LevelDB (3 node cluster). Zookeeper 3.4.6 (3 node cluster). File system: ext4 {code} 2016-09-28 10:54:40,849 | INFO | Stopping BrokerService[localhost] due to exception, java.io.EOFException: File '/opt/activemq/activemq-batch/data/leveldb/000000000c8002d7.log' offset: 3012576207 | org.apache.activemq.u til.DefaultIOExceptionHandler | LevelDB IOException handler. java.io.EOFException: File '/opt/activemq/activemq-batch/data/leveldb/000000000c8002d7.log' offset: 3012576207 at org.apache.activemq.leveldb.RecordLog$LogReader.read(RecordLog.scala:389)[activemq-leveldb-store-5.11.1.jar:5.11.1] at org.apache.activemq.leveldb.RecordLog$$anonfun$read$2.apply(RecordLog.scala:654)[activemq-leveldb-store-5.11.1.jar:5.11.1] at org.apache.activemq.leveldb.RecordLog$$anonfun$read$2.apply(RecordLog.scala:654)[activemq-leveldb-store-5.11.1.jar:5.11.1] at org.apache.activemq.leveldb.RecordLog.get_reader(RecordLog.scala:644)[activemq-leveldb-store-5.11.1.jar:5.11.1] at org.apache.activemq.leveldb.RecordLog.read(RecordLog.scala:654)[activemq-leveldb-store-5.11.1.jar:5.11.1] at org.apache.activemq.leveldb.LevelDBClient.getMessage(LevelDBClient.scala:1335)[activemq-leveldb-store-5.11.1.jar:5.11.1] at org.apache.activemq.leveldb.LevelDBClient$$anonfun$queueCursor$1.apply(LevelDBClient.scala:1274)[activemq-leveldb-store-5.11.1.jar:5.11.1] at org.apache.activemq.leveldb.LevelDBClient$$anonfun$queueCursor$1.apply(LevelDBClient.scala:1271)[activemq-leveldb-store-5.11.1.jar:5.11.1] at org.apache.activemq.leveldb.LevelDBClient$$anonfun$collectionCursor$1$$anonfun$apply$mcV$sp$12.apply(LevelDBClient.scala:1359)[activemq-leveldb-store-5.11.1.jar:5.11.1] at org.apache.activemq.leveldb.LevelDBClient$$anonfun$collectionCursor$1$$anonfun$apply$mcV$sp$12.apply(LevelDBClient.scala:1358)[activemq-leveldb-store-5.11.1.jar:5.11.1] at org.apache.activemq.leveldb.LevelDBClient$RichDB.check$4(LevelDBClient.scala:323)[activemq-leveldb-store-5.11.1.jar:5.11.1] at org.apache.activemq.leveldb.LevelDBClient$RichDB.cursorRange(LevelDBClient.scala:325)[activemq-leveldb-store-5.11.1.jar:5.11.1] at org.apache.activemq.leveldb.LevelDBClient$$anonfun$collectionCursor$1.apply$mcV$sp(LevelDBClient.scala:1358)[activemq-leveldb-store-5.11.1.jar:5.11.1] at org.apache.activemq.leveldb.LevelDBClient$$anonfun$collectionCursor$1.apply(LevelDBClient.scala:1358)[activemq-leveldb-store-5.11.1.jar:5.11.1] at org.apache.activemq.leveldb.LevelDBClient$$anonfun$collectionCursor$1.apply(LevelDBClient.scala:1358)[activemq-leveldb-store-5.11.1.jar:5.11.1] at org.apache.activemq.leveldb.LevelDBClient.usingIndex(LevelDBClient.scala:1038)[activemq-leveldb-store-5.11.1.jar:5.11.1] at org.apache.activemq.leveldb.LevelDBClient$$anonfun$might_fail_using_index$1.apply(LevelDBClient.scala:1044)[activemq-leveldb-store-5.11.1.jar:5.11.1] at org.apache.activemq.leveldb.LevelDBClient.might_fail(LevelDBClient.scala:549)[activemq-leveldb-store-5.11.1.jar:5.11.1] at org.apache.activemq.leveldb.LevelDBClient.might_fail_using_index(LevelDBClient.scala:1044)[activemq-leveldb-store-5.11.1.jar:5.11.1] at org.apache.activemq.leveldb.LevelDBClient.collectionCursor(LevelDBClient.scala:1357)[activemq-leveldb-store-5.11.1.jar:5.11.1] at org.apache.activemq.leveldb.LevelDBClient.queueCursor(LevelDBClient.scala:1271)[activemq-leveldb-store-5.11.1.jar:5.11.1] at org.apache.activemq.leveldb.DBManager.cursorMessages(DBManager.scala:735)[activemq-leveldb-store-5.11.1.jar:5.11.1] at org.apache.activemq.leveldb.LevelDBStore$LevelDBMessageStore.recoverNextMessages(LevelDBStore.scala:860)[activemq-leveldb-store-5.11.1.jar:5.11.1] at org.apache.activemq.broker.region.cursors.QueueStorePrefetch.doFillBatch(QueueStorePrefetch.java:109)[activemq-broker-5.11.1.jar:5.11.1] at org.apache.activemq.broker.region.cursors.AbstractStoreCursor.fillBatch(AbstractStoreCursor.java:381)[activemq-broker-5.11.1.jar:5.11.1] at org.apache.activemq.broker.region.cursors.AbstractStoreCursor.reset(AbstractStoreCursor.java:142)[activemq-broker-5.11.1.jar:5.11.1] at org.apache.activemq.broker.region.cursors.StoreQueueCursor.reset(StoreQueueCursor.java:159)[activemq-broker-5.11.1.jar:5.11.1] at org.apache.activemq.broker.region.Queue.doPageInForDispatch(Queue.java:1897)[activemq-broker-5.11.1.jar:5.11.1] at org.apache.activemq.broker.region.Queue.pageInMessages(Queue.java:2119)[activemq-broker-5.11.1.jar:5.11.1] at org.apache.activemq.broker.region.Queue.doBrowse(Queue.java:1116)[activemq-broker-5.11.1.jar:5.11.1] at org.apache.activemq.broker.region.Queue.expireMessages(Queue.java:909)[activemq-broker-5.11.1.jar:5.11.1] at org.apache.activemq.broker.region.Queue.access$100(Queue.java:100)[activemq-broker-5.11.1.jar:5.11.1] at org.apache.activemq.broker.region.Queue$2.run(Queue.java:144)[activemq-broker-5.11.1.jar:5.11.1] at org.apache.activemq.thread.SchedulerTimerTask.run(SchedulerTimerTask.java:33)[activemq-client-5.11.1.jar:5.11.1] at java.util.TimerThread.mainLoop(Timer.java:555)[:1.8.0_92] at java.util.TimerThread.run(Timer.java:505)[:1.8.0_92]" {code} activemq.xml snip {code:xml} <persistenceAdapter> <replicatedLevelDB directory="${activemq.data}/leveldb" replicas="2" bind="tcp://0.0.0.0:61613" zkAddress="zk-1:2181,zk-2:2181,zk-3:2181" zkPassword="" zkPath="${activemq.data}/batch/leveldb" sync="quorum_disk" /> </persistenceAdapter> {code} I was able to resolve the issue by removing the data folder on all the cluster nodes and restarting the Brokers. This error causes the activemq cluster to failover repeatedly. > LevelDB gets corrupted when Primary ActiveMQ server is shutdown while > messages are queued to it > ----------------------------------------------------------------------------------------------- > > Key: AMQ-6174 > URL: https://issues.apache.org/jira/browse/AMQ-6174 > Project: ActiveMQ > Issue Type: Bug > Components: activemq-leveldb-store > Affects Versions: 5.13.0 > Environment: Virtual type detected as xen-para. > Last rubix: Mon Feb 1 11:02:37 2016 release: 74867 version: 2.0.7 > Installed kernel: 2.6.18-308.0.0.0.1.el5xen x86_64 > Reporter: Sunil Vishwanath > Priority: Critical > > Currently I am testing the following setup: > ActiveMQ 5.13.0 with LevelDB (3 node cluster). > Zookeeper 3.4.6 (3 node cluster). > File system: NFSv3 > Started up all 3 Zookeeper nodes. (aamqzk1, aamqzk2 and aamqzk3) > Started up all 5 ActiveMQ nodes. (aamql1, aamql2, aamql3, aamql4 and aamql5) > I started aamql1 first and all others in order. aamql1 is the master and I am > able to see all the queue statistics for aamql1 via ActiveMQ Web Console. > I am also watching all 5 AMQ's "application.log" file using "tail -f > application.log” command. > The message producer starts sending messages (about 120,000 of them). While > the messages are being queued and also being consumed, I stopped the master > instance (aamql1). Now aamql2 becomes the master. About 10 seconds later > after all the slave aamq reports to the new master, aamql2 throws the > following exception and the instance dies. This keeps repeating as it > failover to the next instance. > 2016-02-17T15:43:48.358885-08:00 aamql2.bus.jetqa1.syseng.tmcs severity=INFO > datetime=2016-02-17 15:43:48,354 thread=LevelDB IOException handler. > category=org.apache.activemq.util.DefaultIOExceptionHandler Stopping > BrokerService[localhost] due to exception, java.io.EOFException: File > '/aamql/local/activemq/data/leveldb/0000000038409848.log' offset: 477491777 > 2016-02-17T15:43:48.370881-08:00 aamql2.bus.jetqa1.syseng.tmcs > java.io.EOFException: File > '/aamql/local/activemq/data/leveldb/0000000038409848.log' offset: 477491777 > 2016-02-17T15:43:48.371003-08:00 aamql2.bus.jetqa1.syseng.tmcs at > org.apache.activemq.leveldb.RecordLog$LogReader.read(RecordLog.scala:389) > 2016-02-17T15:43:48.371082-08:00 aamql2.bus.jetqa1.syseng.tmcs at > org.apache.activemq.leveldb.RecordLog$$anonfun$read$2.apply(RecordLog.scala:654) > 2016-02-17T15:43:48.371148-08:00 aamql2.bus.jetqa1.syseng.tmcs at > org.apache.activemq.leveldb.RecordLog$$anonfun$read$2.apply(RecordLog.scala:654) > 2016-02-17T15:43:48.371219-08:00 aamql2.bus.jetqa1.syseng.tmcs at > org.apache.activemq.leveldb.RecordLog.get_reader(RecordLog.scala:644) > 2016-02-17T15:43:48.371380-08:00 aamql2.bus.jetqa1.syseng.tmcs at > org.apache.activemq.leveldb.RecordLog.read(RecordLog.scala:654) > 2016-02-17T15:43:48.371454-08:00 aamql2.bus.jetqa1.syseng.tmcs at > org.apache.activemq.leveldb.LevelDBClient.getMessage(LevelDBClient.scala:1335) > 2016-02-17T15:43:48.371526-08:00 aamql2.bus.jetqa1.syseng.tmcs at > org.apache.activemq.leveldb.LevelDBClient$$anonfun$queueCursor$1.apply(LevelDBClient.scala:1274) > 2016-02-17T15:43:48.371604-08:00 aamql2.bus.jetqa1.syseng.tmcs at > org.apache.activemq.leveldb.LevelDBClient$$anonfun$queueCursor$1.apply(LevelDBClient.scala:1271) > 2016-02-17T15:43:48.371675-08:00 aamql2.bus.jetqa1.syseng.tmcs at > org.apache.activemq.leveldb.LevelDBClient$$anonfun$collectionCursor$1$$anonfun$apply$mcV$sp$12.apply(LevelDBClient.scala:1359) > 2016-02-17T15:43:48.371746-08:00 aamql2.bus.jetqa1.syseng.tmcs at > org.apache.activemq.leveldb.LevelDBClient$$anonfun$collectionCursor$1$$anonfun$apply$mcV$sp$12.apply(LevelDBClient.scala:1358) > 2016-02-17T15:43:48.371818-08:00 aamql2.bus.jetqa1.syseng.tmcs at > org.apache.activemq.leveldb.LevelDBClient$RichDB.check$4(LevelDBClient.scala:323) > 2016-02-17T15:43:48.371888-08:00 aamql2.bus.jetqa1.syseng.tmcs at > org.apache.activemq.leveldb.LevelDBClient$RichDB.cursorRange(LevelDBClient.scala:325) > 2016-02-17T15:43:48.371960-08:00 aamql2.bus.jetqa1.syseng.tmcs at > org.apache.activemq.leveldb.LevelDBClient$$anonfun$collectionCursor$1.apply$mcV$sp(LevelDBClient.scala:1358) > 2016-02-17T15:43:48.372034-08:00 aamql2.bus.jetqa1.syseng.tmcs at > org.apache.activemq.leveldb.LevelDBClient$$anonfun$collectionCursor$1.apply(LevelDBClient.scala:1358) > 2016-02-17T15:43:48.372104-08:00 aamql2.bus.jetqa1.syseng.tmcs at > org.apache.activemq.leveldb.LevelDBClient$$anonfun$collectionCursor$1.apply(LevelDBClient.scala:1358) > 2016-02-17T15:43:48.372175-08:00 aamql2.bus.jetqa1.syseng.tmcs at > org.apache.activemq.leveldb.LevelDBClient.usingIndex(LevelDBClient.scala:1038) > 2016-02-17T15:43:48.372259-08:00 aamql2.bus.jetqa1.syseng.tmcs at > org.apache.activemq.leveldb.LevelDBClient$$anonfun$might_fail_using_index$1.apply(LevelDBClient.scala:1044) > 2016-02-17T15:43:48.372334-08:00 aamql2.bus.jetqa1.syseng.tmcs at > org.apache.activemq.leveldb.LevelDBClient.might_fail(LevelDBClient.scala:549) > 2016-02-17T15:43:48.372406-08:00 aamql2.bus.jetqa1.syseng.tmcs at > org.apache.activemq.leveldb.LevelDBClient.might_fail_using_index(LevelDBClient.scala:1044) > 2016-02-17T15:43:48.372488-08:00 aamql2.bus.jetqa1.syseng.tmcs at > org.apache.activemq.leveldb.LevelDBClient.collectionCursor(LevelDBClient.scala:1357) > 2016-02-17T15:43:48.372573-08:00 aamql2.bus.jetqa1.syseng.tmcs at > org.apache.activemq.leveldb.LevelDBClient.queueCursor(LevelDBClient.scala:1271) > 2016-02-17T15:43:48.372635-08:00 aamql2.bus.jetqa1.syseng.tmcs at > org.apache.activemq.leveldb.DBManager.cursorMessages(DBManager.scala:735) > 2016-02-17T15:43:48.372705-08:00 aamql2.bus.jetqa1.syseng.tmcs at > org.apache.activemq.leveldb.LevelDBStore$LevelDBMessageStore.recoverNextMessages(LevelDBStore.scala:860) > 2016-02-17T15:43:48.372774-08:00 aamql2.bus.jetqa1.syseng.tmcs at > org.apache.activemq.broker.region.cursors.QueueStorePrefetch.doFillBatch(QueueStorePrefetch.java:120) > 2016-02-17T15:43:48.372844-08:00 aamql2.bus.jetqa1.syseng.tmcs at > org.apache.activemq.broker.region.cursors.AbstractStoreCursor.fillBatch(AbstractStoreCursor.java:406) > 2016-02-17T15:43:48.372913-08:00 aamql2.bus.jetqa1.syseng.tmcs at > org.apache.activemq.broker.region.cursors.AbstractStoreCursor.reset(AbstractStoreCursor.java:149) > 2016-02-17T15:43:48.372990-08:00 aamql2.bus.jetqa1.syseng.tmcs at > org.apache.activemq.broker.region.cursors.StoreQueueCursor.reset(StoreQueueCursor.java:169) > 2016-02-17T15:43:48.373060-08:00 aamql2.bus.jetqa1.syseng.tmcs at > org.apache.activemq.broker.region.Queue.doPageInForDispatch(Queue.java:1938) > 2016-02-17T15:43:48.373130-08:00 aamql2.bus.jetqa1.syseng.tmcs at > org.apache.activemq.broker.region.Queue.pageInMessages(Queue.java:2149) > 2016-02-17T15:43:48.373201-08:00 aamql2.bus.jetqa1.syseng.tmcs at > org.apache.activemq.broker.region.Queue.doBrowse(Queue.java:1137) > 2016-02-17T15:43:48.373275-08:00 aamql2.bus.jetqa1.syseng.tmcs at > org.apache.activemq.broker.region.Queue.expireMessages(Queue.java:917) > 2016-02-17T15:43:48.373344-08:00 aamql2.bus.jetqa1.syseng.tmcs at > org.apache.activemq.broker.region.Queue.access$100(Queue.java:103) > 2016-02-17T15:43:48.373413-08:00 aamql2.bus.jetqa1.syseng.tmcs at > org.apache.activemq.broker.region.Queue$2.run(Queue.java:146) > 2016-02-17T15:43:48.373484-08:00 aamql2.bus.jetqa1.syseng.tmcs at > org.apache.activemq.thread.SchedulerTimerTask.run(SchedulerTimerTask.java:33) > 2016-02-17T15:43:48.373553-08:00 aamql2.bus.jetqa1.syseng.tmcs at > java.util.TimerThread.mainLoop(Timer.java:555) > 2016-02-17T15:43:48.373622-08:00 aamql2.bus.jetqa1.syseng.tmcs at > java.util.TimerThread.run(Timer.java:505) -- This message was sent by Atlassian JIRA (v6.3.4#6332)