Author: toad
Date: 2008-07-01 12:16:03 +0000 (Tue, 01 Jul 2008)
New Revision: 20902
Modified:
branches/db4o/freenet/src/freenet/client/async/BaseSingleFileFetcher.java
branches/db4o/freenet/src/freenet/client/async/ClientGetState.java
branches/db4o/freenet/src/freenet/client/async/ClientGetter.java
branches/db4o/freenet/src/freenet/client/async/ClientRequestScheduler.java
branches/db4o/freenet/src/freenet/client/async/ClientRequestSchedulerCore.java
branches/db4o/freenet/src/freenet/client/async/SingleBlockInserter.java
branches/db4o/freenet/src/freenet/client/async/SingleFileFetcher.java
branches/db4o/freenet/src/freenet/client/async/SplitFileFetcher.java
branches/db4o/freenet/src/freenet/client/async/SplitFileFetcherSegment.java
branches/db4o/freenet/src/freenet/client/async/SplitFileFetcherSubSegment.java
branches/db4o/freenet/src/freenet/client/async/USKFetcher.java
branches/db4o/freenet/src/freenet/client/async/USKFetcherTag.java
branches/db4o/freenet/src/freenet/client/async/USKInserter.java
branches/db4o/freenet/src/freenet/client/async/USKManager.java
branches/db4o/freenet/src/freenet/client/async/USKRetriever.java
branches/db4o/freenet/src/freenet/node/SendableGet.java
branches/db4o/freenet/src/freenet/node/SimpleSendableInsert.java
Log:
When registering a big (multi-segment) splitfile, add the RegisterMe and
schedule each segment as a separate job.
On a large splitfile (ISO size), we were getting *very* long stalls while every
block in the splitfile was addPendingKeys()'ed.
Modified:
branches/db4o/freenet/src/freenet/client/async/BaseSingleFileFetcher.java
===================================================================
--- branches/db4o/freenet/src/freenet/client/async/BaseSingleFileFetcher.java
2008-07-01 11:24:59 UTC (rev 20901)
+++ branches/db4o/freenet/src/freenet/client/async/BaseSingleFileFetcher.java
2008-07-01 12:16:03 UTC (rev 20902)
@@ -95,7 +95,7 @@
}
return true; // We will retry, just not yet.
See requeueAfterCooldown(Key).
} else {
- schedule(container, context, true);
+ schedule(container, context, false, true);
}
return true;
}
@@ -199,7 +199,7 @@
}
if(Logger.shouldLog(Logger.MINOR, this))
Logger.minor(this, "Requeueing after cooldown "+key+"
for "+this);
- schedule(container, context, true);
+ schedule(container, context, false, true);
}
}
Modified: branches/db4o/freenet/src/freenet/client/async/ClientGetState.java
===================================================================
--- branches/db4o/freenet/src/freenet/client/async/ClientGetState.java
2008-07-01 11:24:59 UTC (rev 20901)
+++ branches/db4o/freenet/src/freenet/client/async/ClientGetState.java
2008-07-01 12:16:03 UTC (rev 20902)
@@ -11,7 +11,7 @@
*/
public interface ClientGetState {
- public void schedule(ObjectContainer container, ClientContext context,
boolean probablyNotInStore);
+ public void schedule(ObjectContainer container, ClientContext context,
boolean delayedRegister, boolean probablyNotInStore);
public void cancel(ObjectContainer container, ClientContext context);
Modified: branches/db4o/freenet/src/freenet/client/async/ClientGetter.java
===================================================================
--- branches/db4o/freenet/src/freenet/client/async/ClientGetter.java
2008-07-01 11:24:59 UTC (rev 20901)
+++ branches/db4o/freenet/src/freenet/client/async/ClientGetter.java
2008-07-01 12:16:03 UTC (rev 20902)
@@ -113,7 +113,7 @@
return false;
}
}
- currentState.schedule(container, context,
false);
+ currentState.schedule(container, context,
false, false);
}
if(cancelled) cancel();
} catch (MalformedURLException e) {
Modified:
branches/db4o/freenet/src/freenet/client/async/ClientRequestScheduler.java
===================================================================
--- branches/db4o/freenet/src/freenet/client/async/ClientRequestScheduler.java
2008-07-01 11:24:59 UTC (rev 20901)
+++ branches/db4o/freenet/src/freenet/client/async/ClientRequestScheduler.java
2008-07-01 12:16:03 UTC (rev 20902)
@@ -155,14 +155,14 @@
choosenPriorityScheduler = val;
}
- public void register(final SendableRequest req, boolean
probablyNotInStore) {
- register(req, databaseExecutor.onThread(), null,
probablyNotInStore);
+ public void register(final SendableRequest req, boolean regmeOnly,
boolean probablyNotInStore) {
+ register(req, databaseExecutor.onThread(), regmeOnly, null,
probablyNotInStore);
}
/**
* Register and then delete the RegisterMe which is passed in to avoid
querying.
*/
- public void register(final SendableRequest req, boolean
onDatabaseThread, RegisterMe reg, final boolean probablyNotInStore) {
+ public void register(final SendableRequest req, boolean
onDatabaseThread, final boolean regmeOnly, RegisterMe reg, final boolean
probablyNotInStore) {
logMINOR = Logger.shouldLog(Logger.MINOR, this);
if(logMINOR) Logger.minor(this, "Registering "+req, new
Exception("debug"));
final boolean persistent = req.persistent();
@@ -172,6 +172,19 @@
final SendableGet getter = (SendableGet)req;
if(persistent && onDatabaseThread) {
+ if(regmeOnly) {
+ assert(reg == null);
+ reg = schedCore.queueRegister(getter,
databaseExecutor, selectorContainer);
+ final RegisterMe regme = reg;
+ clientContext.jobRunner.queue(new
DBJob() {
+
+ public void run(ObjectContainer
container, ClientContext context) {
+ register(req, true,
false, regme, probablyNotInStore);
+ }
+
+ }, NativeThread.NORM_PRIORITY+1, false);
+ return;
+ }
schedCore.addPendingKeys(getter,
selectorContainer);
final Object[] keyTokens =
getter.sendableKeys(selectorContainer);
final ClientKey[] keys = new
ClientKey[keyTokens.length];
Modified:
branches/db4o/freenet/src/freenet/client/async/ClientRequestSchedulerCore.java
===================================================================
---
branches/db4o/freenet/src/freenet/client/async/ClientRequestSchedulerCore.java
2008-07-01 11:24:59 UTC (rev 20901)
+++
branches/db4o/freenet/src/freenet/client/async/ClientRequestSchedulerCore.java
2008-07-01 12:16:03 UTC (rev 20902)
@@ -530,7 +530,7 @@
Logger.minor(this, "Running RegisterMe
for "+reg.getter+" : "+reg.key.addedTime+" : "+reg.key.priority);
// Don't need to activate, fields should exist?
FIXME
try {
- sched.register(reg.getter, true, reg,
false);
+ sched.register(reg.getter, true, false,
reg, false);
} catch (Throwable t) {
Logger.error(this, "Caught "+t+"
running RegisterMeRunner", t);
// Cancel the request, and commit so it
isn't tried again.
Modified:
branches/db4o/freenet/src/freenet/client/async/SingleBlockInserter.java
===================================================================
--- branches/db4o/freenet/src/freenet/client/async/SingleBlockInserter.java
2008-07-01 11:24:59 UTC (rev 20901)
+++ branches/db4o/freenet/src/freenet/client/async/SingleBlockInserter.java
2008-07-01 12:16:03 UTC (rev 20902)
@@ -199,7 +199,7 @@
}
if(persistent)
container.set(this);
- getScheduler(context).register(this, false);
+ getScheduler(context).register(this, false, false);
}
private void fail(InsertException e, ObjectContainer container,
ClientContext context) {
@@ -263,7 +263,7 @@
if(persistent)
container.set(this);
} else {
- getScheduler(context).register(this, false);
+ getScheduler(context).register(this, false, false);
}
}
Modified: branches/db4o/freenet/src/freenet/client/async/SingleFileFetcher.java
===================================================================
--- branches/db4o/freenet/src/freenet/client/async/SingleFileFetcher.java
2008-07-01 11:24:59 UTC (rev 20901)
+++ branches/db4o/freenet/src/freenet/client/async/SingleFileFetcher.java
2008-07-01 12:16:03 UTC (rev 20902)
@@ -522,7 +522,7 @@
f.addDecompressor(codec);
}
parent.onTransition(this, f, container);
- f.schedule(container, context, false);
+ f.schedule(container, context, false, false);
if(persistent) {
container.set(metaStrings);
container.set(this);
@@ -599,7 +599,7 @@
SplitFileFetcher sf = new
SplitFileFetcher(metadata, rcb, parent, ctx,
decompressors, clientMetadata,
actx, recursionLevel, returnBucket, token, container);
parent.onTransition(this, sf, container);
- sf.schedule(container, context, false);
+ sf.schedule(container, context, false, false);
rcb.onBlockSetFinished(this, container,
context);
// Clear our own metadata, we won't need it any
more.
// For multi-level metadata etc see above.
@@ -925,7 +925,7 @@
if(l == usk.suggestedEdition) {
SingleFileFetcher sf = new
SingleFileFetcher(parent, cb, clientMetadata, key, metaStrings,
key.getURI().addMetaStrings(metaStrings),
0, ctx, actx, null,
null, maxRetries, recursionLevel+1, dontTellClientGet, token, false,
returnBucket, true, container, context);
- sf.schedule(container, context, false);
+ sf.schedule(container, context, false,
false);
} else {
cb.onFailure(new
FetchException(FetchException.PERMANENT_REDIRECT,
newUSK.getURI().addMetaStrings(metaStrings)), null, container, context);
}
Modified: branches/db4o/freenet/src/freenet/client/async/SplitFileFetcher.java
===================================================================
--- branches/db4o/freenet/src/freenet/client/async/SplitFileFetcher.java
2008-07-01 11:24:59 UTC (rev 20901)
+++ branches/db4o/freenet/src/freenet/client/async/SplitFileFetcher.java
2008-07-01 12:16:03 UTC (rev 20902)
@@ -326,12 +326,14 @@
}
}
- public void schedule(ObjectContainer container, ClientContext context,
boolean probablyNotInStore) {
+ public void schedule(ObjectContainer container, ClientContext context,
boolean regmeOnly, boolean probablyNotInStore) {
if(persistent)
container.activate(this, 1);
+ if(segments.length > 1)
+ regmeOnly = true;
if(Logger.shouldLog(Logger.MINOR, this)) Logger.minor(this,
"Scheduling "+this);
for(int i=0;i<segments.length;i++) {
- segments[i].schedule(container, context,
probablyNotInStore);
+ segments[i].schedule(container, context, true,
probablyNotInStore);
}
}
Modified:
branches/db4o/freenet/src/freenet/client/async/SplitFileFetcherSegment.java
===================================================================
--- branches/db4o/freenet/src/freenet/client/async/SplitFileFetcherSegment.java
2008-07-01 11:24:59 UTC (rev 20901)
+++ branches/db4o/freenet/src/freenet/client/async/SplitFileFetcherSegment.java
2008-07-01 12:16:03 UTC (rev 20902)
@@ -621,7 +621,7 @@
parentFetcher.segmentFinished(this, container, context);
}
- public void schedule(ObjectContainer container, ClientContext context,
boolean probablyNotInStore) {
+ public void schedule(ObjectContainer container, ClientContext context,
boolean regmeOnly, boolean probablyNotInStore) {
if(persistent) {
container.activate(this, 1);
container.activate(parentFetcher, 1);
@@ -634,7 +634,7 @@
for(int
i=0;i<dataRetries.length+checkRetries.length;i++)
seg.add(i, true, container, context, false);
- seg.schedule(container, context, probablyNotInStore);
+ seg.schedule(container, context, regmeOnly,
probablyNotInStore);
synchronized(this) {
scheduled = true;
}
@@ -811,13 +811,13 @@
SplitFileFetcherSubSegment sub =
(SplitFileFetcherSubSegment) v.get(i);
RandomGrabArray rga = sub.getParentGrabArray();
if(sub.getParentGrabArray() == null) {
- sub.schedule(container, context, true);
+ sub.schedule(container, context, false,
true);
} else {
// if(logMINOR) {
container.activate(rga, 1);
if(!rga.contains(sub,
container)) {
Logger.error(this,
"Sub-segment has RGA but isn't registered to it!!: "+sub+" for "+rga);
- sub.schedule(container,
context, true);
+ sub.schedule(container,
context, false, true);
}
container.deactivate(rga, 1);
// }
Modified:
branches/db4o/freenet/src/freenet/client/async/SplitFileFetcherSubSegment.java
===================================================================
---
branches/db4o/freenet/src/freenet/client/async/SplitFileFetcherSubSegment.java
2008-07-01 11:24:59 UTC (rev 20901)
+++
branches/db4o/freenet/src/freenet/client/async/SplitFileFetcherSubSegment.java
2008-07-01 12:16:03 UTC (rev 20902)
@@ -456,7 +456,7 @@
}
if(persistent)
container.set(blockNums);
- if(schedule) schedule(container, context, true); // Retrying so
not in store
+ if(schedule) schedule(container, context, false, true); //
Retrying so not in store
else if(!dontSchedule)
// Already scheduled, however this key may not be
registered.
getScheduler(context).addPendingKey(segment.getBlockKey(blockNo, container),
this);
Modified: branches/db4o/freenet/src/freenet/client/async/USKFetcher.java
===================================================================
--- branches/db4o/freenet/src/freenet/client/async/USKFetcher.java
2008-07-01 11:24:59 UTC (rev 20901)
+++ branches/db4o/freenet/src/freenet/client/async/USKFetcher.java
2008-07-01 12:16:03 UTC (rev 20902)
@@ -159,7 +159,7 @@
if(logMINOR)
Logger.minor(this, "Checker == null in
schedule() for "+this, new Exception("debug"));
} else
- checker.schedule(container, context, false);
+ checker.schedule(container, context, false,
false);
}
public String toString() {
@@ -469,17 +469,17 @@
public void schedule(long delay, ObjectContainer container, final
ClientContext context) {
assert(container == null);
if (delay<=0) {
- schedule(container, context, false);
+ schedule(container, context, false, false);
} else {
uskManager.ticker.queueTimedJob(new Runnable() {
public void run() {
- USKFetcher.this.schedule(null, context,
false);
+ USKFetcher.this.schedule(null, context,
false, false);
}
}, delay);
}
}
- public void schedule(ObjectContainer container, ClientContext context,
boolean probablyNotInStore) {
+ public void schedule(ObjectContainer container, ClientContext context,
boolean regmeOnly, boolean probablyNotInStore) {
USKAttempt[] attempts;
long lookedUp = uskManager.lookup(origUSK);
synchronized(this) {
Modified: branches/db4o/freenet/src/freenet/client/async/USKFetcherTag.java
===================================================================
--- branches/db4o/freenet/src/freenet/client/async/USKFetcherTag.java
2008-07-01 11:24:59 UTC (rev 20901)
+++ branches/db4o/freenet/src/freenet/client/async/USKFetcherTag.java
2008-07-01 12:16:03 UTC (rev 20902)
@@ -91,7 +91,7 @@
usk = usk.copy(edition);
fetcher = manager.getFetcher(usk, ctx, new
USKFetcherWrapper(usk, priority, client), keepLastData);
fetcher.addCallback(this);
- fetcher.schedule(null, context, false); // non-persistent
+ fetcher.schedule(null, context, false, false); // non-persistent
}
public void cancel(ObjectContainer container, ClientContext context) {
@@ -118,7 +118,7 @@
return token;
}
- public void schedule(ObjectContainer container, ClientContext context,
boolean probablyNotInStore) {
+ public void schedule(ObjectContainer container, ClientContext context,
boolean regmeOnly, boolean probablyNotInStore) {
start(context.uskManager, context);
}
Modified: branches/db4o/freenet/src/freenet/client/async/USKInserter.java
===================================================================
--- branches/db4o/freenet/src/freenet/client/async/USKInserter.java
2008-07-01 11:24:59 UTC (rev 20901)
+++ branches/db4o/freenet/src/freenet/client/async/USKInserter.java
2008-07-01 12:16:03 UTC (rev 20902)
@@ -78,7 +78,7 @@
if(finished) return;
fetcher =
context.uskManager.getFetcherForInsertDontSchedule(pubUSK,
parent.priorityClass, this, parent.getClient(), container, context);
}
- fetcher.schedule(container, context, false);
+ fetcher.schedule(container, context, false, false);
}
public void onFoundEdition(long l, USK key, ObjectContainer container,
ClientContext context, boolean lastContentWasMetadata, short codec, byte[]
hisData) {
Modified: branches/db4o/freenet/src/freenet/client/async/USKManager.java
===================================================================
--- branches/db4o/freenet/src/freenet/client/async/USKManager.java
2008-07-01 11:24:59 UTC (rev 20901)
+++ branches/db4o/freenet/src/freenet/client/async/USKManager.java
2008-07-01 12:16:03 UTC (rev 20902)
@@ -140,7 +140,7 @@
fetcher.cancel(null, context);
}
}
- if(sched != null) sched.schedule(null, context, false);
+ if(sched != null) sched.schedule(null, context, false, false);
}
void update(final USK origUSK, final long number, final ClientContext
context) {
@@ -216,7 +216,7 @@
if(fetcher != null) {
ticker.queueTimedJob(new Runnable() {
public void run() {
- fetcher.schedule(null, context, false);
+ fetcher.schedule(null, context, false,
false);
}
}, 0);
}
Modified: branches/db4o/freenet/src/freenet/client/async/USKRetriever.java
===================================================================
--- branches/db4o/freenet/src/freenet/client/async/USKRetriever.java
2008-07-01 11:24:59 UTC (rev 20901)
+++ branches/db4o/freenet/src/freenet/client/async/USKRetriever.java
2008-07-01 12:16:03 UTC (rev 20902)
@@ -48,7 +48,7 @@
SingleFileFetcher getter =
(SingleFileFetcher)
SingleFileFetcher.create(this, this, new ClientMetadata(), uri, ctx, new
ArchiveContext(ctx.maxArchiveLevels),
ctx.maxNonSplitfileRetries, 0,
true, l, true, null, false, null, context);
- getter.schedule(null, context, false);
+ getter.schedule(null, context, false, false);
} catch (MalformedURLException e) {
Logger.error(this, "Impossible: "+e, e);
} catch (FetchException e) {
Modified: branches/db4o/freenet/src/freenet/node/SendableGet.java
===================================================================
--- branches/db4o/freenet/src/freenet/node/SendableGet.java 2008-07-01
11:24:59 UTC (rev 20901)
+++ branches/db4o/freenet/src/freenet/node/SendableGet.java 2008-07-01
12:16:03 UTC (rev 20902)
@@ -99,10 +99,10 @@
return true;
}
- public void schedule(ObjectContainer container, ClientContext context,
boolean assumeNotInStore) {
+ public void schedule(ObjectContainer container, ClientContext context,
boolean regmeOnly, boolean assumeNotInStore) {
if(Logger.shouldLog(Logger.MINOR, this))
Logger.minor(this, "Scheduling "+this);
- getScheduler(context).register(this, assumeNotInStore);
+ getScheduler(context).register(this, regmeOnly,
assumeNotInStore);
}
public ClientRequestScheduler getScheduler(ClientContext context) {
Modified: branches/db4o/freenet/src/freenet/node/SimpleSendableInsert.java
===================================================================
--- branches/db4o/freenet/src/freenet/node/SimpleSendableInsert.java
2008-07-01 11:24:59 UTC (rev 20901)
+++ branches/db4o/freenet/src/freenet/node/SimpleSendableInsert.java
2008-07-01 12:16:03 UTC (rev 20902)
@@ -110,7 +110,7 @@
public void schedule() {
finished = false; // can reschedule
- scheduler.register(this, false);
+ scheduler.register(this, false, false);
}
public void cancel(ObjectContainer container, ClientContext context) {
