Author: toad
Date: 2008-03-22 15:20:02 +0000 (Sat, 22 Mar 2008)
New Revision: 18719
Modified:
trunk/freenet/src/freenet/client/async/BaseSingleFileFetcher.java
trunk/freenet/src/freenet/client/async/ClientRequestScheduler.java
trunk/freenet/src/freenet/client/async/SplitFileFetcherSubSegment.java
trunk/freenet/src/freenet/node/BaseSendableGet.java
trunk/freenet/src/freenet/node/RequestStarter.java
trunk/freenet/src/freenet/support/RandomGrabArray.java
trunk/freenet/src/freenet/support/RemoveRandom.java
trunk/freenet/src/freenet/support/SectoredRandomGrabArray.java
Log:
Simple running-local-request coalescing: don't start a request for a key if
we're already running one for the same key.
This means we have to pass the list of currently running keys into the request
selection code, so there are some complications.
Logging changes.
Modified: trunk/freenet/src/freenet/client/async/BaseSingleFileFetcher.java
===================================================================
--- trunk/freenet/src/freenet/client/async/BaseSingleFileFetcher.java
2008-03-22 12:37:31 UTC (rev 18718)
+++ trunk/freenet/src/freenet/client/async/BaseSingleFileFetcher.java
2008-03-22 15:20:02 UTC (rev 18719)
@@ -48,6 +48,10 @@
return keys[0];
}
+ public boolean hasValidKeys(KeysFetchingLocally fetching) {
+ return !fetching.hasKey(key.getNodeKey());
+ }
+
public ClientKey getKey(Object token) {
return key;
}
Modified: trunk/freenet/src/freenet/client/async/ClientRequestScheduler.java
===================================================================
--- trunk/freenet/src/freenet/client/async/ClientRequestScheduler.java
2008-03-22 12:37:31 UTC (rev 18718)
+++ trunk/freenet/src/freenet/client/async/ClientRequestScheduler.java
2008-03-22 15:20:02 UTC (rev 18719)
@@ -411,7 +411,7 @@
}
if(logMINOR)
Logger.minor(this, "Got retry count
tracker "+rga);
- SendableRequest req = (SendableRequest)
rga.removeRandom();
+ SendableRequest req = (SendableRequest)
rga.removeRandom(starter);
if(rga.isEmpty()) {
if(logMINOR) Logger.minor(this,
"Removing retrycount "+rga.getNumber()+" : "+rga);
s.remove(rga.getNumber());
@@ -440,7 +440,7 @@
}
}
if(altRGA != null) {
- SendableRequest altReq =
(SendableRequest) (altRGA.removeRandom());
+ SendableRequest altReq =
(SendableRequest) (altRGA.removeRandom(starter));
if(altReq != null &&
altReq.getPriorityClass() <= choosenPriorityClass &&
fixRetryCount(altReq.getRetryCount()) <= rga.getNumber()) {
// Use the recent one instead
Modified: trunk/freenet/src/freenet/client/async/SplitFileFetcherSubSegment.java
===================================================================
--- trunk/freenet/src/freenet/client/async/SplitFileFetcherSubSegment.java
2008-03-22 12:37:31 UTC (rev 18718)
+++ trunk/freenet/src/freenet/client/async/SplitFileFetcherSubSegment.java
2008-03-22 15:20:02 UTC (rev 18719)
@@ -78,7 +78,7 @@
} else if(segment.isFinishing()) {
Logger.error(this, "Segment finishing but
didn't tell us! "+this);
} else {
- Logger.error(this, "Segment not finishing yet
still returns null for getKey()!: "+token+" for "+this);
+ Logger.error(this, "Segment not finishing yet
still returns null for getKey()!: "+token+" for "+this, new Exception("debug"));
}
}
return key;
@@ -117,6 +117,11 @@
}
// LOCKING: keys is safe to check, but segment isn't.
Key key =
segment.getBlockNodeKey(((Integer)ret).intValue());
+ if(key == null) {
+ if(segment.isFinishing() ||
segment.isFinished()) return null;
+ Logger.error(this, "Key is null for block
"+ret);
+ continue;
+ }
if(keys.hasKey(key)) {
synchronized(this) {
blockNums.add(ret);
@@ -130,6 +135,30 @@
return null;
}
+ public boolean hasValidKeys(KeysFetchingLocally keys) {
+ for(int i=0;i<10;i++) {
+ Object ret;
+ int x;
+ synchronized(this) {
+ if(blockNums.isEmpty()) return false;
+ x = ctx.random.nextInt(blockNums.size());
+ ret = (Integer) blockNums.get(x);
+ }
+ // LOCKING: keys is safe to check, but segment isn't.
+ Key key =
segment.getBlockNodeKey(((Integer)ret).intValue());
+ if(keys.hasKey(key)) {
+ synchronized(this) {
+ blockNums.add(ret);
+ }
+ continue;
+ }
+ if(logMINOR)
+ Logger.minor(this, "Removing block "+x+" of
"+(blockNums.size()+1)+ " : "+ret+ " on "+this);
+ return true;
+ }
+ return false;
+ }
+
public boolean ignoreStore() {
return ctx.ignoreStore;
}
Modified: trunk/freenet/src/freenet/node/BaseSendableGet.java
===================================================================
--- trunk/freenet/src/freenet/node/BaseSendableGet.java 2008-03-22 12:37:31 UTC
(rev 18718)
+++ trunk/freenet/src/freenet/node/BaseSendableGet.java 2008-03-22 15:20:02 UTC
(rev 18719)
@@ -7,4 +7,6 @@
/** Get a numbered key to fetch. */
public abstract Key getNodeKey(Object token);
+ public abstract boolean hasValidKeys(KeysFetchingLocally fetching);
+
}
Modified: trunk/freenet/src/freenet/node/RequestStarter.java
===================================================================
--- trunk/freenet/src/freenet/node/RequestStarter.java 2008-03-22 12:37:31 UTC
(rev 18718)
+++ trunk/freenet/src/freenet/node/RequestStarter.java 2008-03-22 15:20:02 UTC
(rev 18719)
@@ -8,6 +8,8 @@
import freenet.keys.Key;
import freenet.support.Logger;
import freenet.support.OOMHandler;
+import freenet.support.RandomGrabArrayItem;
+import freenet.support.RandomGrabArrayItemExclusionList;
import freenet.support.TokenBucket;
import freenet.support.math.RunningAverage;
@@ -17,7 +19,7 @@
* And you have to provide a RequestStarterClient. We do round robin between
* clients on the same priority level.
*/
-public class RequestStarter implements Runnable, KeysFetchingLocally {
+public class RequestStarter implements Runnable, KeysFetchingLocally,
RandomGrabArrayItemExclusionList {
/*
* Priority classes
@@ -211,8 +213,8 @@
} catch (Throwable t) {
if(keyNum != null) {
// Re-queue
+ Logger.error(this, "Caught "+t+" while
trying to start request");
req.internalError(keyNum, t, sched);
- Logger.error(this, "Caught "+t+" while
trying to start request");
return true; // Sort of ... maybe it
will clear
}
if(key != null) keysFetching.remove(key);
@@ -251,8 +253,12 @@
public void run() {
try {
freenet.support.Logger.OSThread.logPID(this);
- if(!req.send(core, sched, keyNum))
- Logger.error(this, "run() not able to send a
request on "+req);
+ if(!req.send(core, sched, keyNum)) {
+ if(!req.isCancelled())
+ Logger.error(this, "run() not able to
send a request on "+req);
+ else
+ Logger.normal(this, "run() not able to
send a request on "+req+" - request was cancelled");
+ }
if(Logger.shouldLog(Logger.MINOR, this))
Logger.minor(this, "Finished "+req);
} finally {
@@ -278,4 +284,13 @@
}
}
+ public boolean exclude(RandomGrabArrayItem item) {
+ if(isInsert) return false;
+ BaseSendableGet get = (BaseSendableGet) item;
+ if(get.hasValidKeys(this))
+ return false;
+ Logger.normal(this, "Excluding (no valid keys): "+get);
+ return true;
+ }
+
}
Modified: trunk/freenet/src/freenet/support/RandomGrabArray.java
===================================================================
--- trunk/freenet/src/freenet/support/RandomGrabArray.java 2008-03-22
12:37:31 UTC (rev 18718)
+++ trunk/freenet/src/freenet/support/RandomGrabArray.java 2008-03-22
15:20:02 UTC (rev 18719)
@@ -52,10 +52,12 @@
}
}
- public RandomGrabArrayItem removeRandom() {
+ public RandomGrabArrayItem
removeRandom(RandomGrabArrayItemExclusionList excluding) {
RandomGrabArrayItem ret, oret;
boolean logMINOR = Logger.shouldLog(Logger.MINOR, this);
synchronized(this) {
+ final int MAX_EXCLUDED = 10;
+ int excluded = 0;
while(true) {
if(index == 0) {
if(logMINOR) Logger.minor(this, "All
null on "+this);
@@ -68,6 +70,14 @@
if(logMINOR) Logger.minor(this, "Not
returning because cancelled: "+ret);
ret = null;
}
+ if(ret != null && excluding.exclude(ret)) {
+ excluded++;
+ if(excluded > MAX_EXCLUDED) {
+ Logger.error(this, "Remove
random returning null because "+excluded+" excluded items", new
Exception("error"));
+ return null;
+ }
+ continue;
+ }
if(ret != null && !ret.canRemove()) {
if(logMINOR) Logger.minor(this,
"Returning (cannot remove): "+ret);
return ret;
Modified: trunk/freenet/src/freenet/support/RemoveRandom.java
===================================================================
--- trunk/freenet/src/freenet/support/RemoveRandom.java 2008-03-22 12:37:31 UTC
(rev 18718)
+++ trunk/freenet/src/freenet/support/RemoveRandom.java 2008-03-22 15:20:02 UTC
(rev 18719)
@@ -3,6 +3,6 @@
public interface RemoveRandom {
/** Remove and return a random RandomGrabArrayItem. Should be fast. */
- public RandomGrabArrayItem removeRandom();
+ public RandomGrabArrayItem
removeRandom(RandomGrabArrayItemExclusionList excluding);
}
Modified: trunk/freenet/src/freenet/support/SectoredRandomGrabArray.java
===================================================================
--- trunk/freenet/src/freenet/support/SectoredRandomGrabArray.java
2008-03-22 12:37:31 UTC (rev 18718)
+++ trunk/freenet/src/freenet/support/SectoredRandomGrabArray.java
2008-03-22 15:20:02 UTC (rev 18719)
@@ -64,22 +64,25 @@
grabArrays = newArrays;
}
- public synchronized RandomGrabArrayItem removeRandom() {
+ public synchronized RandomGrabArrayItem
removeRandom(RandomGrabArrayItemExclusionList excluding) {
boolean logMINOR = Logger.shouldLog(Logger.MINOR, this);
+ /** Count of arrays that have items but didn't return anything
because of exclusions */
+ int excluded = 0;
+ final int MAX_EXCLUDED = 0;
while(true) {
if(grabArrays.length == 0) return null;
int x = rand.nextInt(grabArrays.length);
RemoveRandomWithObject rga = grabArrays[x];
if(logMINOR)
Logger.minor(this, "Picked "+x+" of
"+grabArrays.length+" : "+rga+" : "+rga.getObject()+" on "+this);
- RandomGrabArrayItem item = rga.removeRandom();
+ RandomGrabArrayItem item = rga.removeRandom(excluding);
if(logMINOR)
Logger.minor(this, "RGA has picked
"+x+"/"+grabArrays.length+": "+item+
(item==null ? "" : ("
cancelled="+item.isCancelled()+")"))+" rga.isEmpty="+rga.isEmpty());
// Just because the item is cancelled does not
necessarily mean the whole client is.
// E.g. a segment may return cancelled because it is
decoding, that doesn't mean
// other segments are cancelled. So just go around the
loop in that case.
- if(rga.isEmpty() || (item == null)) {
+ if(rga.isEmpty()) {
if(logMINOR)
Logger.minor(this, "Removing grab array
"+x+" : "+rga+" for "+rga.getObject()+" (is empty)");
Object client = rga.getObject();
@@ -91,7 +94,17 @@
System.arraycopy(grabArrays, x+1,
newArray, x, grabArrays.length - (x+1));
grabArrays = newArray;
}
- if(item == null) continue;
+ if(item == null) {
+ if(!rga.isEmpty()) {
+ // Hmmm...
+ excluded++;
+ if(excluded > MAX_EXCLUDED) {
+ Logger.error(this, "Too many
sub-arrays are entirely excluded on "+this, new Exception("error"));
+ return null;
+ }
+ }
+ continue;
+ }
if(item.isCancelled()) continue;
return item;
}
