Hi, Did you upgrade OrientDB before the second start? Lvc@
On 29 January 2015 at 23:08, Keith Freeman <[email protected]> wrote: > The database has been built over 4 months ingesting sensor data, and is > now over 400G in size (330M+ records). > > We got an exception in the server log earlier today: > > com.sat.bwu.db.exceptions.GRunException: GFactory.runConnected run >>> exception! >>> [OrientIF.findOrMakeVertexCluster-add-key:TDCarrier,tdv-cr-2015-01-29] >>> com.orientechnologies.orient.core.exception.OStorageException: Error in >>> creation of new cluster >> >> 'tdv-cr-2015-01-29 >> >> at com.sat.bwu.db.GFactory.runConnected(GFactory.java:620) >>> [bwu-tdrunprocessor-j8.jar:na] >> >> at com.sat.bwu.db.GFactory.runGraph(GFactory.java:800) >>> [bwu-tdrunprocessor-j8.jar:na] >> >> at com.sat.bwu.db.GFactory.runNoTx(GFactory.java:719) >>> [bwu-tdrunprocessor-j8.jar:na] >> >> at >>> com.sat.bwu.db.GFactory.runNoTxRuntimeExceptions(GFactory.java:839) >>> [bwu-tdrunprocessor-j8.jar:na] >> >> at >>> com.sat.bwu.common.OrientIF.findOrMakeVertexCluster(OrientIF.java:86) >>> [bwu-tdrunprocessor-j8.jar:na] >> >> at >>> com.sat.bwu.td.data.TDRecord.calculateClusterName(TDRecord.java:82) >>> [bwu-tdrunprocessor-j8.jar:na] >> >> at com.sat.bwu.td.parts.TDStore.doStore(TDStore.java:113) >>> [bwu-tdrunprocessor-j8.jar:na] >> >> at com.sat.qos.parts.Store.store(Store.java:101) >>> [bwu-tdrunprocessor-j8.jar:na] >> >> at sun.reflect.GeneratedMethodAccessor5.invoke(Unknown Source) >>> ~[na:na] >> >> at >>> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) >>> ~[na:1.8.0_31] >> >> at java.lang.reflect.Method.invoke(Method.java:483) ~[na:1.8.0_31] >> >> at >>> org.apache.camel.component.bean.MethodInfo.invoke(MethodInfo.java:409) >>> [bwu-tdrunprocessor-j8.jar:na] >> >> at >>> org.apache.camel.component.bean.MethodInfo$1.doProceed(MethodInfo.java:278) >>> [bwu-tdrunprocessor-j8.jar:na] >> >> at >>> org.apache.camel.component.bean.MethodInfo$1.proceed(MethodInfo.java:251) >>> [bwu-tdrunprocessor-j8.jar:na] >> >> at >>> org.apache.camel.component.bean.BeanProcessor.process(BeanProcessor.java:166) >>> [bwu-tdrunprocessor-j8.jar:na] >> >> at >>> org.apache.camel.management.InstrumentationProcessor.process(InstrumentationProcessor.java:72) >>> [bwu-tdrunprocessor-j8.jar:na] >> >> at >>> org.apache.camel.processor.RedeliveryErrorHandler.process(RedeliveryErrorHandler.java:398) >>> [bwu-tdrunprocessor-j8.jar:na] >> >> at >>> org.apache.camel.processor.CamelInternalProcessor.process(CamelInternalProcessor.java:191) >>> [bwu-tdrunprocessor-j8.jar:na] >> >> at org.apache.camel.processor.Pipeline.process(Pipeline.java:118) >>> [bwu-tdrunprocessor-j8.jar:na] >> >> at org.apache.camel.processor.Pipeline.process(Pipeline.java:80) >>> [bwu-tdrunprocessor-j8.jar:na] >> >> at >>> org.apache.camel.processor.CamelInternalProcessor.process(CamelInternalProcessor.java:191) >>> [bwu-tdrunprocessor-j8.jar:na] >> >> at >>> org.apache.camel.component.timer.TimerConsumer.sendTimerExchange(TimerConsumer.java:139) >>> [bwu-tdrunprocessor-j8.jar:na] >> >> at >>> org.apache.camel.component.timer.TimerConsumer$1.run(TimerConsumer.java:64) >>> [bwu-tdrunprocessor-j8.jar:na] >> >> at java.util.TimerThread.mainLoop(Timer.java:555) [na:1.8.0_31] >> >> at java.util.TimerThread.run(Timer.java:505) [na:1.8.0_31] >> >> com.orientechnologies.orient.core.exception.OStorageException: Error in >>> creation of new cluster 'tdv-cr-2015-01-29 >> >> at >>> com.orientechnologies.orient.core.storage.impl.local.OAbstractPaginatedStorage.addCluster(OAbstractPaginatedStorage.java:411) >>> ~[bwu-tdrunprocessor-j8.jar:na] >> >> at >>> com.orientechnologies.orient.core.db.document.ODatabaseDocumentTx.addCluster(ODatabaseDocumentTx.java:1166) >>> ~[bwu-tdrunprocessor-j8.jar:na] >> >> at >>> com.orientechnologies.orient.server.network.protocol.binary.ONetworkProtocolBinary.addCluster(ONetworkProtocolBinary.java:563) >>> ~[bwu-tdrunprocessor-j8.jar:na] >> >> at >>> com.orientechnologies.orient.server.network.protocol.binary.ONetworkProtocolBinary.executeRequest(ONetworkProtocolBinary.java:334) >>> ~[bwu-tdrunprocessor-j8.jar:na] >> >> at >>> com.orientechnologies.orient.server.network.protocol.binary.OBinaryNetworkProtocolAbstract.execute(OBinaryNetworkProtocolAbstract.java:216) >>> ~[bwu-tdrunprocessor-j8.jar:na] >> >> at >>> com.orientechnologies.common.thread.OSoftThread.run(OSoftThread.java:65) >>> ~[bwu-tdrunprocessor-j8.jar:na] >> >> Caused by: com.orientechnologies.common.exception.OException: File flush >>> was abnormally terminated >> >> at >>> com.orientechnologies.orient.core.index.hashindex.local.cache.OWOWCache.flush(OWOWCache.java:447) >>> ~[bwu-tdrunprocessor-j8.jar:na] >> >> at >>> com.orientechnologies.orient.core.index.hashindex.local.cache.OWOWCache.flush(OWOWCache.java:453) >>> ~[bwu-tdrunprocessor-j8.jar:na] >> >> at >>> com.orientechnologies.orient.core.index.hashindex.local.cache.OReadWriteDiskCache.flushBuffer(OReadWriteDiskCache.java:539) >>> ~[bwu-tdrunprocessor-j8.jar:na] >> >> at >>> com.orientechnologies.orient.core.storage.impl.local.OAbstractPaginatedStorage.makeFullCheckpoint(OAbstractPaginatedStorage.java:1336) >>> ~[bwu-tdrunprocessor-j8.jar:na] >> >> at >>> com.orientechnologies.orient.core.storage.impl.local.OAbstractPaginatedStorage.addClusterInternal(OAbstractPaginatedStorage.java:1915) >>> ~[bwu-tdrunprocessor-j8.jar:na] >> >> at >>> com.orientechnologies.orient.core.storage.impl.local.OAbstractPaginatedStorage.doAddCluster(OAbstractPaginatedStorage.java:1886) >>> ~[bwu-tdrunprocessor-j8.jar:na] >> >> at >>> com.orientechnologies.orient.core.storage.impl.local.OAbstractPaginatedStorage.addCluster(OAbstractPaginatedStorage.java:408) >>> ~[bwu-tdrunprocessor-j8.jar:na] >> >> ... 5 common frames omitted >> >> Caused by: java.util.concurrent.ExecutionException: >>> java.lang.IllegalArgumentException: Negative position >> >> at java.util.concurrent.FutureTask.report(FutureTask.java:122) >>> ~[na:1.8.0_31] >> >> at java.util.concurrent.FutureTask.get(FutureTask.java:192) >>> ~[na:1.8.0_31] >> >> at >>> com.orientechnologies.orient.core.index.hashindex.local.cache.OWOWCache.flush(OWOWCache.java:442) >>> ~[bwu-tdrunprocessor-j8.jar:na] >> >> ... 11 common frames omitted >> >> java.lang.IllegalArgumentException: Negative position >> >> at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:722) >>> ~[na:1.8.0_31] >> >> at >>> com.orientechnologies.orient.core.storage.fs.OFileClassic.writeInternal(OFileClassic.java:95) >>> ~[bwu-tdrunprocessor-j8.jar:na] >> >> at >>> com.orientechnologies.orient.core.storage.fs.OFileClassic.write(OFileClassic.java:214) >>> ~[bwu-tdrunprocessor-j8.jar:na] >> >> at >>> com.orientechnologies.orient.core.index.hashindex.local.cache.OWOWCache.flushPage(OWOWCache.java:941) >>> ~[bwu-tdrunprocessor-j8.jar:na] >> >> at >>> com.orientechnologies.orient.core.index.hashindex.local.cache.OWOWCache.access$2000(OWOWCache.java:66) >>> ~[bwu-tdrunprocessor-j8.jar:na] >> >> at >>> com.orientechnologies.orient.core.index.hashindex.local.cache.OWOWCache$FileFlushTask.call(OWOWCache.java:1240) >>> ~[bwu-tdrunprocessor-j8.jar:na] >> >> at >>> com.orientechnologies.orient.core.index.hashindex.local.cache.OWOWCache$FileFlushTask.call(OWOWCache.java:1208) >>> ~[bwu-tdrunprocessor-j8.jar:na] >> >> at java.util.concurrent.FutureTask.run(FutureTask.java:266) >>> ~[na:1.8.0_31] >> >> at >>> java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:180) >>> ~[na:1.8.0_31] >> >> at >>> java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293) >>> ~[na:1.8.0_31] >> >> at >>> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) >>> ~[na:1.8.0_31] >> >> at >>> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) >>> ~[na:1.8.0_31] >> >> at java.lang.Thread.run(Thread.java:745) ~[na:1.8.0_31] >> >> > > > So we stopped our ingester process, shutdown the server and restarted it. > Now we can't open the database in Studio or in our code, due to this error: > > 2015-01-29 21:58:43:280 INFO Databases directory: >>> /opt/orientdb/orientdb-community-2.0/databases [OServer] >> >> 2015-01-29 21:58:43:298 INFO Listening binary connections on >>> 0.0.0.0:2424 (protocol v.28, socket=default) [OServerNetworkListener] >> >> 2015-01-29 21:58:43:299 INFO Listening http connections on 0.0.0.0:2480 >>> (protocol v.10, socket=default) [OServerNetworkListener] >> >> 2015-01-29 21:58:43:307 INFO Installing dynamic plugin >>> 'orientdb-lucene-2.0-dist.jar'... [OServerPluginManager] >> >> 2015-01-29 21:58:43:357 INFO Lucene index plugin installed and active. >>> Lucene version: LUCENE_47 [OLuceneIndexPlugin] >> >> 2015-01-29 21:58:43:358 INFO Installing dynamic plugin >>> 'studio-2.0.zip'... [OServerPluginManager] >> >> 2015-01-29 21:58:43:366 INFO Installing GREMLIN language v.2.6.0 - >>> graph.pool.max=50 [OGraphServerHandler] >> >> 2015-01-29 21:58:43:366 INFO [OVariableParser.resolveVariables] Error on >>> resolving property: distributed [orientechnologies] >> >> 2015-01-29 21:58:43:367 INFO Installing Script interpreter. WARN: >>> authenticated clients can execute any kind of code into the server by using >>> the following allowed languages: [sql] [OServerSideScriptInterpreter] >> >> 2015-01-29 21:58:43:367 INFO OrientDB Server v2.0 is active. [OServer] >> >> 2015-01-29 21:58:50:950 WARNING {db=smsrb} segment file 'database.ocf' >>> was not closed correctly last time [OSingleFileSegment] >> >> 2015-01-29 21:58:51:292 WARNING {db=smsrb} Storage smsrb was not closed >>> properly. Will try to restore from write ahead log. [OLocalPaginatedStorage] >> >> 2015-01-29 21:58:51:293 INFO {db=smsrb} Looking for last checkpoint... >>> [OLocalPaginatedStorage] >> >> 2015-01-29 21:58:51:336 INFO {db=smsrb} FULL checkpoint found. >>> [OLocalPaginatedStorage] >> >> 2015-01-29 21:58:51:336 WARNING {db=smsrb} FULL checkpoint has not >>> completed. [OLocalPaginatedStorage] >> >> 2015-01-29 21:58:51:337 WARNING {db=smsrb} Restore will start from the >>> previous checkpoint. [OLocalPaginatedStorage]{db=smsrb} Exception during >>> storage data restore. >> >> com.orientechnologies.orient.core.exception.OStorageException: Unknown >>> checkpoint record type >>> com.orientechnologies.orient.core.storage.impl.local.paginated.wal.OFuzzyCheckpointStartRecord >> >> at >>> com.orientechnologies.orient.core.storage.impl.local.OAbstractPaginatedStorage.restoreFromWAL(OAbstractPaginatedStorage.java:2239) >> >> at >>> com.orientechnologies.orient.core.storage.impl.local.OAbstractPaginatedStorage.restoreIfNeeded(OAbstractPaginatedStorage.java:1561) >> >> at >>> com.orientechnologies.orient.core.storage.impl.local.OAbstractPaginatedStorage.open(OAbstractPaginatedStorage.java:205) >> >> at >>> com.orientechnologies.orient.core.db.document.ODatabaseDocumentTx.open(ODatabaseDocumentTx.java:243) >> >> at >>> com.orientechnologies.orient.server.OServer.openDatabase(OServer.java:631) >> >> at >>> com.orientechnologies.orient.server.OServer.openDatabase(OServer.java:613) >> >> at >>> com.orientechnologies.orient.server.network.protocol.http.command.OServerCommandAuthenticatedDbAbstract.authenticate(OServerCommandAuthenticatedDbAbstract.java:167) >> >> at >>> com.orientechnologies.orient.server.network.protocol.http.command.OServerCommandAuthenticatedDbAbstract.beforeExecute(OServerCommandAuthenticatedDbAbstract.java:127) >> >> at >>> com.orientechnologies.orient.server.network.protocol.http.command.get.OServerCommandGetConnect.beforeExecute(OServerCommandGetConnect.java:50) >> >> at >>> com.orientechnologies.orient.server.network.protocol.http.ONetworkProtocolHttpAbstract.service(ONetworkProtocolHttpAbstract.java:197) >> >> at >>> com.orientechnologies.orient.server.network.protocol.http.ONetworkProtocolHttpAbstract.execute(ONetworkProtocolHttpAbstract.java:600) >> >> at >>> com.orientechnologies.common.thread.OSoftThread.run(OSoftThread.java:65) >> >> >>> 2015-01-29 21:58:51:339 SEVERE {db=smsrb} Internal server error: >> >> com.orientechnologies.orient.core.exception.OStorageException: Cannot >>> open local storage '/opt/orientdb/orientdb-community-2.0/databases/smsrb' >>> with mode=rw >> >> --> com.orientechnologies.orient.core.exception.OStorageException: >>> Unknown checkpoint record type >>> com.orientechnologies.orient.core.storage.impl.local.paginated.wal.OFuzzyCheckpointStartRecord >>> [ONetworkProtocolHttpDb] >> >> >> > Is there any way to recover this? And any idea why the cluster creation > failed in the first place? The machine is otherwise working correctly. > > > -- > > --- > You received this message because you are subscribed to the Google Groups > "OrientDB" group. > To unsubscribe from this group and stop receiving emails from it, send an > email to [email protected]. > For more options, visit https://groups.google.com/d/optout. > -- --- You received this message because you are subscribed to the Google Groups "OrientDB" group. To unsubscribe from this group and stop receiving emails from it, send an email to [email protected]. For more options, visit https://groups.google.com/d/optout.
