Hi, Let's say there's a 30GB dataset [3] containing most threads/posts from [1]. After importing all of it, when I try to run /dba/db-optimize/ on it (which must have some corresponding command) I get the OOM error in the stacktrace attached. I am using -Xmx2g so BaseX is limited to 2GB of memory (the machine I'm running this on doesn't have a lot of memory). I was looking at [2] for some estimates of peak memory usage for this "db-optimize" operation, but couldn't find any. Actually it would be nice to know peak memory usage because.. of course, for any database (including BaseX) a common operation is to do server sizing, to know what kind of server would be needed. In this case, it seems like 2GB memory is enough to import 340k documents, weighing in at 30GB total, but it's not enough to run "dba-optimize". Is there any info about peak memory usage on [2] ? And are there guidelines for large-scale collection imports like I'm trying to do?
Thanks, Stefan [1] https://www.linuxquestions.org/ [2] http://docs.basex.org/wiki/Statistics [3] https://drive.google.com/open?id=1lTEGA4JqlhVf1JsMQbloNGC-tfNkeQt2
java.io.FileNotFoundException:
/share/CACHEDEV1_DATA/Public/builds/basex/data/linuxquestions.org_938223018/inf.basex
(No such file or directory)
at java.io.FileOutputStream.open0(Native Method)
at java.io.FileOutputStream.open(FileOutputStream.java:270)
at java.io.FileOutputStream.<init>(FileOutputStream.java:213)
at java.io.FileOutputStream.<init>(FileOutputStream.java:162)
at org.basex.io.IOFile.outputStream(IOFile.java:158)
at org.basex.io.out.DataOutput.<init>(DataOutput.java:47)
at org.basex.io.out.DataOutput.<init>(DataOutput.java:36)
at org.basex.data.DiskData.write(DiskData.java:137)
at org.basex.data.DiskData.close(DiskData.java:160)
at org.basex.core.cmd.OptimizeAll.optimizeAll(OptimizeAll.java:145)
at
org.basex.query.up.primitives.db.DBOptimize.apply(DBOptimize.java:124)
at org.basex.query.up.DataUpdates.apply(DataUpdates.java:175)
at org.basex.query.up.ContextModifier.apply(ContextModifier.java:120)
at org.basex.query.up.Updates.apply(Updates.java:178)
at org.basex.query.QueryContext.update(QueryContext.java:701)
at org.basex.query.QueryContext.iter(QueryContext.java:332)
at
org.basex.http.restxq.RestXqResponse.serialize(RestXqResponse.java:73)
at org.basex.http.web.WebResponse.create(WebResponse.java:63)
at org.basex.http.restxq.RestXqServlet.run(RestXqServlet.java:53)
at org.basex.http.BaseXServlet.service(BaseXServlet.java:65)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:790)
at
org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:865)
at
org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:542)
at
org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:146)
at
org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:548)
at
org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132)
at
org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:257)
at
org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1700)
at
org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255)
at
org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1345)
at
org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:203)
at
org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:480)
at
org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1667)
at
org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:201)
at
org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1247)
at
org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:144)
at
org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132)
at org.eclipse.jetty.server.Server.handle(Server.java:505)
at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:370)
at
org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:267)
at
org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:305)
at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:103)
at org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:117)
at
org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.runTask(EatWhatYouKill.java:333)
at
org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:310)
at
org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:126)
at
org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:366)
at
org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:698)
at
org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:804)
at java.lang.Thread.run(Thread.java:748)
[qtp940553268-16] WARN org.eclipse.jetty.server.HttpChannel - /dba/db-optimize
java.lang.OutOfMemoryError: Java heap space
at org.basex.index.ft.FTList.next(FTList.java:93)
at org.basex.index.ft.FTBuilder.merge(FTBuilder.java:239)
at org.basex.index.ft.FTBuilder.write(FTBuilder.java:147)
at org.basex.index.ft.FTBuilder.build(FTBuilder.java:86)
at org.basex.index.ft.FTBuilder.build(FTBuilder.java:1)
at org.basex.data.DiskData.createIndex(DiskData.java:198)
at org.basex.core.cmd.CreateIndex.create(CreateIndex.java:100)
at org.basex.core.cmd.CreateIndex.create(CreateIndex.java:88)
at org.basex.core.cmd.OptimizeAll.optimizeAll(OptimizeAll.java:139)
at
org.basex.query.up.primitives.db.DBOptimize.apply(DBOptimize.java:124)
at org.basex.query.up.DataUpdates.apply(DataUpdates.java:175)
at org.basex.query.up.ContextModifier.apply(ContextModifier.java:120)
at org.basex.query.up.Updates.apply(Updates.java:178)
at org.basex.query.QueryContext.update(QueryContext.java:701)
at org.basex.query.QueryContext.iter(QueryContext.java:332)
at
org.basex.http.restxq.RestXqResponse.serialize(RestXqResponse.java:73)
at org.basex.http.web.WebResponse.create(WebResponse.java:63)
at org.basex.http.restxq.RestXqServlet.run(RestXqServlet.java:53)
at org.basex.http.BaseXServlet.service(BaseXServlet.java:65)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:790)
at
org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:865)
at
org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:542)
at
org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:146)
at
org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:548)
at
org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132)
at
org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:257)
at
org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1700)
at
org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255)
at
org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1345)
at
org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:203)
at
org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:480)
at
org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1667)
