Author: toad
Date: 2008-08-25 20:41:10 +0000 (Mon, 25 Aug 2008)
New Revision: 22148
Modified:
branches/db4o/freenet/src/freenet/client/async/DatastoreChecker.java
Log:
In profiling, I was getting OOMs, because the database thread could not keep up
with the output of the datastore checker, hence the amount of memory dedicated
to KeyBlock's kept on increasing until the node OOMed.
Modified: branches/db4o/freenet/src/freenet/client/async/DatastoreChecker.java
===================================================================
--- branches/db4o/freenet/src/freenet/client/async/DatastoreChecker.java
2008-08-25 20:31:15 UTC (rev 22147)
+++ branches/db4o/freenet/src/freenet/client/async/DatastoreChecker.java
2008-08-25 20:41:10 UTC (rev 22148)
@@ -290,6 +290,28 @@
DatastoreCheckerItem item = null;
BlockSet blocks = null;
short priority = -1;
+ // If the queue is too large, don't check any more blocks. It
is possible
+ // that we can check the datastore faster than we can handle
the resulting
+ // blocks, this will cause OOM.
+ int queueSize =
context.jobRunner.getQueueSize(ClientRequestScheduler.TRIP_PENDING_PRIORITY);
+ if(queueSize > 500) {
+ // If the queue is over 500, don't run the datastore
checker at all.
+ // It's entirely possible that looking up blocks in the
store will
+ // make the situation first, because a key which is
queued for a
+ // non-persistent request may also be used by a
persistent one.
+
+ // FIXME consider setting a flag to not only only check
transient
+ // requests, but also check whether the keys are in the
persistent
+ // bloom filters first, and if they are not check them.
+ try {
+ Thread.sleep(10*1000);
+ } catch (InterruptedException e) {
+ // Ignore
+ }
+ return;
+ }
+ // If it's over 100, don't check blocks from persistent
requests.
+ boolean notPersistent = queueSize > 100;
synchronized(this) {
while(true) {
for(short prio =
0;prio<transientKeys.length;prio++) {
@@ -301,7 +323,7 @@
blocks =
transientBlockSets[prio].remove(0);
priority = prio;
break;
- } else
if(!persistentGetters[prio].isEmpty()) {
+ } else if((!notPersistent) &&
(!persistentGetters[prio].isEmpty())) {
keys =
persistentKeys[prio].remove(0);
getter =
persistentGetters[prio].remove(0);
persistent = true;
