Author: toad
Date: 2007-02-15 19:40:47 +0000 (Thu, 15 Feb 2007)
New Revision: 11803
Added:
trunk/freenet/src/freenet/client/async/SimpleSingleFileFetcher.java
Modified:
trunk/freenet/src/freenet/client/async/SingleFileFetcher.java
trunk/freenet/src/freenet/client/async/SplitFileFetcherSegment.java
trunk/freenet/src/freenet/keys/FreenetURI.java
trunk/freenet/src/freenet/node/SendableRequest.java
Log:
SimpleSingleFileFetcher: Fetch a single block. Much simpler and smaller than
SingleFileFetcher, which is now its subclass. Should save significant memory
when lots of requests queued.
Added: trunk/freenet/src/freenet/client/async/SimpleSingleFileFetcher.java
===================================================================
--- trunk/freenet/src/freenet/client/async/SimpleSingleFileFetcher.java
(rev 0)
+++ trunk/freenet/src/freenet/client/async/SimpleSingleFileFetcher.java
2007-02-15 19:40:47 UTC (rev 11803)
@@ -0,0 +1,139 @@
+/* This code is part of Freenet. It is distributed under the GNU General
+ * Public License, version 2 (or at your option any later version). See
+ * http://www.gnu.org/ for further details of the GPL. */
+package freenet.client.async;
+
+import java.io.IOException;
+
+import freenet.client.ClientMetadata;
+import freenet.client.FetchException;
+import freenet.client.FetchResult;
+import freenet.client.FetcherContext;
+import freenet.keys.ClientKey;
+import freenet.keys.ClientKeyBlock;
+import freenet.keys.KeyDecodeException;
+import freenet.node.LowLevelGetException;
+import freenet.support.Logger;
+import freenet.support.api.Bucket;
+
+/**
+ * Fetch a single block file.
+ * Used by SplitFileFetcherSegment.
+ */
+public class SimpleSingleFileFetcher extends BaseSingleFileFetcher implements
ClientGetState {
+
+ SimpleSingleFileFetcher(ClientKey key, int maxRetries, FetcherContext
ctx, BaseClientGetter parent, GetCompletionCallback rcb, boolean isEssential) {
+ super(key, maxRetries, ctx, parent);
+ this.rcb = rcb;
+ if(isEssential)
+ parent.addMustSucceedBlocks(1);
+ }
+
+ final GetCompletionCallback rcb;
+
+ // Translate it, then call the real onFailure
+ public void onFailure(LowLevelGetException e) {
+ switch(e.code) {
+ case LowLevelGetException.DATA_NOT_FOUND:
+ onFailure(new
FetchException(FetchException.DATA_NOT_FOUND));
+ return;
+ case LowLevelGetException.DATA_NOT_FOUND_IN_STORE:
+ onFailure(new
FetchException(FetchException.DATA_NOT_FOUND));
+ return;
+ case LowLevelGetException.DECODE_FAILED:
+ onFailure(new
FetchException(FetchException.BLOCK_DECODE_ERROR));
+ return;
+ case LowLevelGetException.INTERNAL_ERROR:
+ onFailure(new
FetchException(FetchException.INTERNAL_ERROR));
+ return;
+ case LowLevelGetException.REJECTED_OVERLOAD:
+ onFailure(new
FetchException(FetchException.REJECTED_OVERLOAD));
+ return;
+ case LowLevelGetException.ROUTE_NOT_FOUND:
+ onFailure(new
FetchException(FetchException.ROUTE_NOT_FOUND));
+ return;
+ case LowLevelGetException.TRANSFER_FAILED:
+ onFailure(new
FetchException(FetchException.TRANSFER_FAILED));
+ return;
+ case LowLevelGetException.VERIFY_FAILED:
+ onFailure(new
FetchException(FetchException.BLOCK_DECODE_ERROR));
+ return;
+ case LowLevelGetException.CANCELLED:
+ onFailure(new FetchException(FetchException.CANCELLED));
+ return;
+ default:
+ Logger.error(this, "Unknown LowLevelGetException code:
"+e.code);
+ onFailure(new
FetchException(FetchException.INTERNAL_ERROR));
+ return;
+ }
+ }
+
+ final void onFailure(FetchException e) {
+ onFailure(e, false);
+ }
+
+ // Real onFailure
+ protected void onFailure(FetchException e, boolean forceFatal) {
+ if(parent.isCancelled() || cancelled) {
+ if(Logger.shouldLog(Logger.MINOR, this))
+ Logger.minor(this, "Failing: cancelled");
+ e = new FetchException(FetchException.CANCELLED);
+ forceFatal = true;
+ }
+ if(!(e.isFatal() || forceFatal) ) {
+ if(retry()) return;
+ }
+ // :(
+ if(e.isFatal() || forceFatal)
+ parent.fatallyFailedBlock();
+ else
+ parent.failedBlock();
+ rcb.onFailure(e, this);
+ }
+
+ /** Will be overridden by SingleFileFetcher */
+ protected void onSuccess(FetchResult data) {
+ if(parent.isCancelled()) {
+ data.asBucket().free();
+ onFailure(new FetchException(FetchException.CANCELLED));
+ return;
+ }
+ rcb.onSuccess(data, this);
+ }
+
+ public void onSuccess(ClientKeyBlock block, boolean fromStore) {
+ Bucket data = extract(block);
+ if(data == null) return; // failed
+ if(!block.isMetadata()) {
+ onSuccess(new FetchResult((ClientMetadata)null, data));
+ } else {
+ onFailure(new
FetchException(FetchException.INVALID_METADATA, "Metadata where expected
data"));
+ }
+ }
+
+ /** Convert a ClientKeyBlock to a Bucket. If an error occurs, report it
via onFailure
+ * and return null.
+ */
+ protected Bucket extract(ClientKeyBlock block) {
+ Bucket data;
+ try {
+ data = block.decode(ctx.bucketFactory,
(int)(Math.min(ctx.maxOutputLength, Integer.MAX_VALUE)), false);
+ } catch (KeyDecodeException e1) {
+ if(Logger.shouldLog(Logger.MINOR, this))
+ Logger.minor(this, "Decode failure: "+e1, e1);
+ onFailure(new
FetchException(FetchException.BLOCK_DECODE_ERROR, e1.getMessage()));
+ return null;
+ } catch (IOException e) {
+ Logger.error(this, "Could not capture data - disk
full?: "+e, e);
+ onFailure(new
FetchException(FetchException.BUCKET_ERROR, e));
+ return null;
+ }
+ return data;
+ }
+
+ /** getToken() is not supported */
+ public Object getToken() {
+ throw new UnsupportedOperationException();
+ }
+
+}
Modified: trunk/freenet/src/freenet/client/async/SingleFileFetcher.java
===================================================================
--- trunk/freenet/src/freenet/client/async/SingleFileFetcher.java
2007-02-15 17:52:36 UTC (rev 11802)
+++ trunk/freenet/src/freenet/client/async/SingleFileFetcher.java
2007-02-15 19:40:47 UTC (rev 11803)
@@ -23,16 +23,14 @@
import freenet.keys.ClientKeyBlock;
import freenet.keys.ClientSSK;
import freenet.keys.FreenetURI;
-import freenet.keys.KeyDecodeException;
import freenet.keys.USK;
-import freenet.node.LowLevelGetException;
import freenet.support.Logger;
import freenet.support.api.Bucket;
import freenet.support.compress.CompressionOutputSizeException;
import freenet.support.compress.Compressor;
import freenet.support.io.BucketTools;
-public class SingleFileFetcher extends BaseSingleFileFetcher implements
ClientGetState {
+public class SingleFileFetcher extends SimpleSingleFileFetcher implements
ClientGetState {
private static boolean logMINOR;
/** Original URI */
@@ -42,7 +40,6 @@
/** Number of metaStrings which were added by redirects etc. They are
added to the start, so this is decremented
* when we consume one. */
private int addedMetaStrings;
- final GetCompletionCallback rcb;
final ClientMetadata clientMetadata;
private Metadata metadata;
private Metadata archiveMetadata;
@@ -68,7 +65,7 @@
ArchiveContext actx, int maxRetries, int recursionLevel,
boolean dontTellClientGet, Object token, boolean
isEssential,
Bucket returnBucket, boolean isFinal) throws
FetchException {
- super(key, maxRetries, ctx, get);
+ super(key, maxRetries, ctx, get, cb, isEssential);
logMINOR = Logger.shouldLog(Logger.MINOR, this);
if(logMINOR) Logger.minor(this, "Creating SingleFileFetcher for
"+key+" from "+origURI+" meta="+metaStrings.toString(), new Exception("debug"));
this.isFinal = isFinal;
@@ -81,7 +78,6 @@
//metaStrings = uri.listMetaStrings();
this.metaStrings = metaStrings;
this.addedMetaStrings = addedMetaStrings;
- this.rcb = cb;
this.clientMetadata = metadata;
thisKey = key.getURI();
this.uri = origURI;
@@ -91,15 +87,13 @@
throw new
FetchException(FetchException.TOO_MUCH_RECURSION, "Too much recursion:
"+recursionLevel+" > "+ctx.maxRecursionLevel);
this.decompressors = new LinkedList();
parent.addBlock();
- if(isEssential)
- parent.addMustSucceedBlocks(1);
}
/** Copy constructor, modifies a few given fields, don't call
schedule().
* Used for things like slave fetchers for MultiLevelMetadata,
therefore does not remember returnBucket,
* metaStrings etc. */
public SingleFileFetcher(SingleFileFetcher fetcher, Metadata newMeta,
GetCompletionCallback callback, FetcherContext ctx2) throws FetchException {
- super(fetcher.key, fetcher.maxRetries, ctx2, fetcher.parent);
+ super(fetcher.key, fetcher.maxRetries, ctx2, fetcher.parent,
callback, false);
logMINOR = Logger.shouldLog(Logger.MINOR, this);
if(logMINOR) Logger.minor(this, "Creating SingleFileFetcher for
"+fetcher.key+" meta="+fetcher.metaStrings.toString(), new Exception("debug"));
this.token = fetcher.token;
@@ -113,7 +107,6 @@
this.metadata = newMeta;
this.metaStrings = new LinkedList();
this.addedMetaStrings = 0;
- this.rcb = callback;
this.recursionLevel = fetcher.recursionLevel + 1;
if(recursionLevel > ctx.maxRecursionLevel)
throw new
FetchException(FetchException.TOO_MUCH_RECURSION);
@@ -127,19 +120,8 @@
public void onSuccess(ClientKeyBlock block, boolean fromStore) {
parent.completedBlock(fromStore);
// Extract data
- Bucket data;
- try {
- data = block.decode(ctx.bucketFactory,
(int)(Math.min(ctx.maxOutputLength, Integer.MAX_VALUE)), false);
- } catch (KeyDecodeException e1) {
- if(logMINOR)
- Logger.minor(this, "Decode failure: "+e1, e1);
- onFailure(new
FetchException(FetchException.BLOCK_DECODE_ERROR, e1.getMessage()));
- return;
- } catch (IOException e) {
- Logger.error(this, "Could not capture data - disk
full?: "+e, e);
- onFailure(new
FetchException(FetchException.BUCKET_ERROR, e));
- return;
- }
+ Bucket data = extract(block);
+ if(data == null) return; // failed
if(!block.isMetadata()) {
onSuccess(new FetchResult(clientMetadata, data));
} else {
@@ -182,7 +164,7 @@
}
}
- private void onSuccess(FetchResult result) {
+ protected void onSuccess(FetchResult result) {
if(parent.isCancelled()) {
result.asBucket().free();
onFailure(new FetchException(FetchException.CANCELLED));
@@ -544,69 +526,6 @@
}
- final void onFailure(FetchException e) {
- onFailure(e, false);
- }
-
- // Real onFailure
- protected void onFailure(FetchException e, boolean forceFatal) {
- if(parent.isCancelled() || cancelled) {
- if(logMINOR) Logger.minor(this, "Failing: cancelled");
- e = new FetchException(FetchException.CANCELLED);
- forceFatal = true;
- }
- if(!(e.isFatal() || forceFatal) ) {
- if(retry()) return;
- }
- // :(
- if(e.isFatal() || forceFatal)
- parent.fatallyFailedBlock();
- else
- parent.failedBlock();
- rcb.onFailure(e, this);
- }
-
- // Translate it, then call the real onFailure
- public void onFailure(LowLevelGetException e) {
- switch(e.code) {
- case LowLevelGetException.DATA_NOT_FOUND:
- onFailure(new
FetchException(FetchException.DATA_NOT_FOUND));
- return;
- case LowLevelGetException.DATA_NOT_FOUND_IN_STORE:
- onFailure(new
FetchException(FetchException.DATA_NOT_FOUND));
- return;
- case LowLevelGetException.DECODE_FAILED:
- onFailure(new
FetchException(FetchException.BLOCK_DECODE_ERROR));
- return;
- case LowLevelGetException.INTERNAL_ERROR:
- onFailure(new
FetchException(FetchException.INTERNAL_ERROR));
- return;
- case LowLevelGetException.REJECTED_OVERLOAD:
- onFailure(new
FetchException(FetchException.REJECTED_OVERLOAD));
- return;
- case LowLevelGetException.ROUTE_NOT_FOUND:
- onFailure(new
FetchException(FetchException.ROUTE_NOT_FOUND));
- return;
- case LowLevelGetException.TRANSFER_FAILED:
- onFailure(new
FetchException(FetchException.TRANSFER_FAILED));
- return;
- case LowLevelGetException.VERIFY_FAILED:
- onFailure(new
FetchException(FetchException.BLOCK_DECODE_ERROR));
- return;
- case LowLevelGetException.CANCELLED:
- onFailure(new FetchException(FetchException.CANCELLED));
- return;
- default:
- Logger.error(this, "Unknown LowLevelGetException code:
"+e.code);
- onFailure(new
FetchException(FetchException.INTERNAL_ERROR));
- return;
- }
- }
-
- public void schedule() {
- super.schedule();
- }
-
public Object getToken() {
return token;
}
@@ -615,8 +534,18 @@
return ctx.ignoreStore;
}
- public static ClientGetState create(BaseClientGetter parent,
GetCompletionCallback cb, ClientMetadata clientMetadata, FreenetURI uri,
FetcherContext ctx, ArchiveContext actx, int maxRetries, int recursionLevel,
boolean dontTellClientGet, Object token, boolean isEssential, Bucket
returnBucket, boolean isFinal) throws MalformedURLException, FetchException {
+ /**
+ * Create a fetcher for a key.
+ */
+ public static ClientGetState create(BaseClientGetter parent,
GetCompletionCallback cb,
+ ClientMetadata clientMetadata, FreenetURI uri,
FetcherContext ctx, ArchiveContext actx,
+ int maxRetries, int recursionLevel, boolean
dontTellClientGet, Object token, boolean isEssential,
+ Bucket returnBucket, boolean isFinal) throws
MalformedURLException, FetchException {
BaseClientKey key = BaseClientKey.getBaseKey(uri);
+ if((clientMetadata == null || clientMetadata.isTrivial()) &&
(!uri.hasMetaStrings()) &&
+ ctx.allowSplitfiles == false &&
ctx.followRedirects == true && token == null &&
+ returnBucket == null && key instanceof
ClientKey)
+ return new SimpleSingleFileFetcher((ClientKey)key,
maxRetries, ctx, parent, cb, isEssential);
if(key instanceof ClientKey)
return new SingleFileFetcher(parent, cb,
clientMetadata, (ClientKey)key, uri.listMetaStrings(), uri, 0, ctx, actx,
maxRetries, recursionLevel, dontTellClientGet, token, isEssential,
returnBucket, isFinal);
else {
Modified: trunk/freenet/src/freenet/client/async/SplitFileFetcherSegment.java
===================================================================
--- trunk/freenet/src/freenet/client/async/SplitFileFetcherSegment.java
2007-02-15 17:52:36 UTC (rev 11802)
+++ trunk/freenet/src/freenet/client/async/SplitFileFetcherSegment.java
2007-02-15 19:40:47 UTC (rev 11803)
@@ -16,6 +16,7 @@
import freenet.client.Metadata;
import freenet.client.MetadataParseException;
import freenet.client.SplitfileBlock;
+import freenet.keys.CHKBlock;
import freenet.keys.FreenetURI;
import freenet.support.Logger;
import freenet.support.api.Bucket;
@@ -31,8 +32,8 @@
final short splitfileType;
final FreenetURI[] dataBlocks;
final FreenetURI[] checkBlocks;
- final SingleFileFetcher[] dataBlockStatus;
- final SingleFileFetcher[] checkBlockStatus;
+ final BaseSingleFileFetcher[] dataBlockStatus;
+ final BaseSingleFileFetcher[] checkBlockStatus;
final MinimalSplitfileBlock[] dataBuckets;
final MinimalSplitfileBlock[] checkBuckets;
final int minFetched;
@@ -175,11 +176,11 @@
startedDecode = true;
}
for(int i=0;i<dataBlockStatus.length;i++) {
- SingleFileFetcher f = dataBlockStatus[i];
+ BaseSingleFileFetcher f = dataBlockStatus[i];
if(f != null) f.cancel();
}
for(int i=0;i<checkBlockStatus.length;i++) {
- SingleFileFetcher f = checkBlockStatus[i];
+ BaseSingleFileFetcher f = checkBlockStatus[i];
if(f != null) f.cancel();
}
Runnable r = new Decoder();
@@ -205,8 +206,7 @@
FECCodec codec = FECCodec.getCodec(splitfileType,
dataBlocks.length, checkBlocks.length);
try {
if(splitfileType !=
Metadata.SPLITFILE_NONREDUNDANT) {
- // FIXME hardcoded block size below.
- codec.decode(dataBuckets, checkBuckets,
32768, fetcherContext.bucketFactory);
+ codec.decode(dataBuckets, checkBuckets,
CHKBlock.DATA_LENGTH, fetcherContext.bucketFactory);
// Now have all the data blocks (not
necessarily all the check blocks)
}
@@ -252,7 +252,7 @@
for(int i=0;i<dataBlockStatus.length;i++) {
boolean heal = false;
if(!dataBlocksSucceeded[i]) {
- SingleFileFetcher fetcher =
dataBlockStatus[i];
+ BaseSingleFileFetcher fetcher =
dataBlockStatus[i];
if(fetcher.getRetryCount() > 0)
heal = true;
}
@@ -269,7 +269,7 @@
for(int i=0;i<checkBlockStatus.length;i++) {
boolean heal = false;
if(!checkBlocksSucceeded[i]) {
- SingleFileFetcher fetcher =
checkBlockStatus[i];
+ BaseSingleFileFetcher fetcher =
checkBlockStatus[i];
if(fetcher.getRetryCount() > 0)
heal = true;
}
@@ -337,7 +337,7 @@
return;
}
for(int i=0;i<dataBlockStatus.length;i++) {
- SingleFileFetcher f = dataBlockStatus[i];
+ BaseSingleFileFetcher f = dataBlockStatus[i];
if(f != null)
f.cancel();
MinimalSplitfileBlock b = dataBuckets[i];
@@ -348,7 +348,7 @@
dataBuckets[i] = null;
}
for(int i=0;i<checkBlockStatus.length;i++) {
- SingleFileFetcher f = checkBlockStatus[i];
+ BaseSingleFileFetcher f = checkBlockStatus[i];
if(f != null)
f.cancel();
MinimalSplitfileBlock b = checkBuckets[i];
Modified: trunk/freenet/src/freenet/keys/FreenetURI.java
===================================================================
--- trunk/freenet/src/freenet/keys/FreenetURI.java 2007-02-15 17:52:36 UTC
(rev 11802)
+++ trunk/freenet/src/freenet/keys/FreenetURI.java 2007-02-15 19:40:47 UTC
(rev 11803)
@@ -399,6 +399,10 @@
public String[] getAllMetaStrings() {
return metaStr;
}
+
+ public boolean hasMetaStrings() {
+ return metaStr == null || metaStr.length == 0;
+ }
public byte[] getRoutingKey() {
return routingKey;
Modified: trunk/freenet/src/freenet/node/SendableRequest.java
===================================================================
--- trunk/freenet/src/freenet/node/SendableRequest.java 2007-02-15 17:52:36 UTC
(rev 11802)
+++ trunk/freenet/src/freenet/node/SendableRequest.java 2007-02-15 19:40:47 UTC
(rev 11803)
@@ -9,11 +9,13 @@
*/
public interface SendableRequest extends RandomGrabArrayItem {
+ /** Get the priority class of the request. */
public short getPriorityClass();
public int getRetryCount();
- /** ONLY called by RequestStarter */
+ /** ONLY called by RequestStarter. Start the actual request using the
NodeClientCore
+ * provided. The request has been removed from the structure already. */
public void send(NodeClientCore node);
/** Get client context object */
