Author: toad
Date: 2006-08-22 19:08:34 +0000 (Tue, 22 Aug 2006)
New Revision: 10239
Modified:
trunk/freenet/src/freenet/client/HighLevelSimpleClientImpl.java
trunk/freenet/src/freenet/client/async/SingleFileFetcher.java
trunk/freenet/src/freenet/client/async/SplitFileFetcher.java
trunk/freenet/src/freenet/clients/http/FProxyToadlet.java
trunk/freenet/src/freenet/clients/http/QueueToadlet.java
trunk/freenet/src/freenet/keys/Key.java
trunk/freenet/src/freenet/support/compress/CompressionOutputSizeException.java
trunk/freenet/src/freenet/support/compress/Compressor.java
trunk/freenet/src/freenet/support/compress/GzipCompressor.java
Log:
Large file related fproxy fixes.
Modified: trunk/freenet/src/freenet/client/HighLevelSimpleClientImpl.java
===================================================================
--- trunk/freenet/src/freenet/client/HighLevelSimpleClientImpl.java
2006-08-22 17:47:54 UTC (rev 10238)
+++ trunk/freenet/src/freenet/client/HighLevelSimpleClientImpl.java
2006-08-22 19:08:34 UTC (rev 10239)
@@ -169,7 +169,7 @@
if(overrideMaxSize >= 0) {
maxLength = overrideMaxSize;
maxTempLength = overrideMaxSize;
- }
+ }
return
new FetcherContext(maxLength, maxTempLength,
curMaxMetadataLength,
MAX_RECURSION, MAX_ARCHIVE_RESTARTS,
MAX_ARCHIVE_LEVELS, DONT_ENTER_IMPLICIT_ARCHIVES,
Modified: trunk/freenet/src/freenet/client/async/SingleFileFetcher.java
===================================================================
--- trunk/freenet/src/freenet/client/async/SingleFileFetcher.java
2006-08-22 17:47:54 UTC (rev 10238)
+++ trunk/freenet/src/freenet/client/async/SingleFileFetcher.java
2006-08-22 19:08:34 UTC (rev 10239)
@@ -166,12 +166,13 @@
while(!decompressors.isEmpty()) {
Compressor c = (Compressor)
decompressors.removeLast();
try {
- data = c.decompress(data,
ctx.bucketFactory, Math.max(ctx.maxTempLength, ctx.maxOutputLength),
decompressors.isEmpty() ? returnBucket : null);
+ long maxLen =
Math.max(ctx.maxTempLength, ctx.maxOutputLength);
+ data = c.decompress(data,
ctx.bucketFactory, maxLen, maxLen * 4, decompressors.isEmpty() ? returnBucket :
null);
} catch (IOException e) {
onFailure(new
FetchException(FetchException.BUCKET_ERROR, e));
return;
} catch (CompressionOutputSizeException e) {
- onFailure(new
FetchException(FetchException.TOO_BIG, e));
+ onFailure(new
FetchException(FetchException.TOO_BIG, e.estimatedSize, (rcb == parent),
result.getMimeType()));
return;
}
}
Modified: trunk/freenet/src/freenet/client/async/SplitFileFetcher.java
===================================================================
--- trunk/freenet/src/freenet/client/async/SplitFileFetcher.java
2006-08-22 17:47:54 UTC (rev 10238)
+++ trunk/freenet/src/freenet/client/async/SplitFileFetcher.java
2006-08-22 19:08:34 UTC (rev 10239)
@@ -11,6 +11,7 @@
import freenet.client.FetcherContext;
import freenet.client.Metadata;
import freenet.client.MetadataParseException;
+import freenet.keys.CHKBlock;
import freenet.keys.FreenetURI;
import freenet.support.Fields;
import freenet.support.Logger;
@@ -77,6 +78,13 @@
this.splitfileType = metadata.getSplitfileType();
splitfileDataBlocks = metadata.getSplitfileDataKeys();
splitfileCheckBlocks = metadata.getSplitfileCheckKeys();
+ long finalLength = splitfileDataBlocks.length *
CHKBlock.DATA_LENGTH;
+ if(finalLength > overrideLength) {
+ if(finalLength - overrideLength > CHKBlock.DATA_LENGTH)
+ throw new
FetchException(FetchException.INVALID_METADATA, "Splitfile is "+finalLength+"
but length is "+finalLength);
+ finalLength = overrideLength;
+ }
+
splitUseLengths = metadata.splitUseLengths();
if(splitfileType == Metadata.SPLITFILE_NONREDUNDANT) {
// Don't need to do much - just fetch everything and
piece it together.
@@ -136,8 +144,11 @@
finalLength += s.decodedLength();
// Healing is done by Segment
}
- if(finalLength > overrideLength)
+ if(finalLength > overrideLength) {
+ if(finalLength - overrideLength > CHKBlock.DATA_LENGTH)
+ throw new
FetchException(FetchException.INVALID_METADATA, "Splitfile is "+finalLength+"
but length is "+finalLength);
finalLength = overrideLength;
+ }
long bytesWritten = 0;
OutputStream os = null;
@@ -204,15 +215,16 @@
// Decompress
while(!decompressors.isEmpty()) {
Compressor c = (Compressor)
decompressors.removeLast();
+ long maxLen =
Math.max(fetchContext.maxTempLength, fetchContext.maxOutputLength);
try {
Bucket out = returnBucket;
if(!decompressors.isEmpty()) out = null;
- data = c.decompress(data,
fetchContext.bucketFactory, Math.max(fetchContext.maxTempLength,
fetchContext.maxOutputLength), out);
+ data = c.decompress(data,
fetchContext.bucketFactory, maxLen, maxLen * 4, out);
} catch (IOException e) {
cb.onFailure(new
FetchException(FetchException.BUCKET_ERROR, e), this);
return;
} catch (CompressionOutputSizeException e) {
- cb.onFailure(new
FetchException(FetchException.TOO_BIG, e), this);
+ cb.onFailure(new
FetchException(FetchException.TOO_BIG, e.estimatedSize, false /* FIXME */,
clientMetadata.getMIMEType()), this);
return;
}
}
Modified: trunk/freenet/src/freenet/clients/http/FProxyToadlet.java
===================================================================
--- trunk/freenet/src/freenet/clients/http/FProxyToadlet.java 2006-08-22
17:47:54 UTC (rev 10238)
+++ trunk/freenet/src/freenet/clients/http/FProxyToadlet.java 2006-08-22
19:08:34 UTC (rev 10239)
@@ -70,8 +70,8 @@
}
}
-
- public static void handleDownload(ToadletContext context, Bucket data,
BucketFactory bucketFactory, String mimeType, String requestedMimeType, String
forceString, boolean forceDownload, String basePath, FreenetURI key) throws
ToadletContextClosedException, IOException {
+
+ public static void handleDownload(ToadletContext context, Bucket data,
BucketFactory bucketFactory, String mimeType, String requestedMimeType, String
forceString, boolean forceDownload, String basePath, FreenetURI key, String
extras) throws ToadletContextClosedException, IOException {
if(requestedMimeType != null)
mimeType = requestedMimeType;
@@ -115,14 +115,14 @@
infoboxContent.addChild("p", "Your options are:");
HTMLNode optionList = infoboxContent.addChild("ul");
HTMLNode option = optionList.addChild("li");
- option.addChild("a", "href", basePath +
key.toString(false) + "?type=text/plain", "Click here");
+ option.addChild("a", "href", basePath +
key.toString(false) + "?type=text/plain"+extras, "Click here");
option.addChild("#", " to open the file as plain text
(this should not be dangerous but it may be garbled).");
// FIXME: is this safe? See bug #131
option = optionList.addChild("li");
- option.addChild("a", "href", basePath +
key.toString(false) + "?forcedownload", "Click here");
+ option.addChild("a", "href", basePath +
key.toString(false) + "?forcedownload"+extras, "Click here");
option.addChild("#", " to force your browser to
download the file to disk.");
option = optionList.addChild("li");
- option.addChild("a", "href", basePath +
key.toString(false) + "?force=" + getForceValue(key, now), "Click here");
+ option.addChild("a", "href", basePath +
key.toString(false) + "?force=" + getForceValue(key, now)+extras, "Click here");
option.addChild("#", " to open the file as " + mimeType
+ ".");
option = optionList.addChild("li");
option.addChild("a", "href", "/", "Click here");
@@ -210,7 +210,7 @@
return;
}
try {
- Logger.minor(this, "FProxy fetching "+key);
+ Logger.minor(this, "FProxy fetching "+key+"
("+maxSize+")");
FetchResult result = fetch(key, maxSize);
// Now, is it safe?
@@ -219,7 +219,7 @@
String mimeType = result.getMimeType();
String requestedMimeType = httprequest.getParam("type",
null);
- handleDownload(ctx, data, ctx.getBucketFactory(),
mimeType, requestedMimeType, httprequest.getParam("force", null),
httprequest.isParameterSet("forcedownload"), "/", key);
+ handleDownload(ctx, data, ctx.getBucketFactory(),
mimeType, requestedMimeType, httprequest.getParam("force", null),
httprequest.isParameterSet("forcedownload"), "/", key, maxSize != MAX_LENGTH ?
"&max-size="+maxSize : "");
} catch (FetchException e) {
String msg = e.getMessage();
@@ -268,7 +268,7 @@
HTMLNode optionList =
infoboxContent.addChild("ul");
option = optionList.addChild("li");
HTMLNode optionForm = option.addChild("form",
new String[] { "action", "method" }, new String[] { "/" + key.toString(false),
"get" });
- optionForm.addChild("input", new String[] {
"type", "name", "value" }, new String[] { "hidden", "max-size",
String.valueOf(e.expectedSize) });
+ optionForm.addChild("input", new String[] {
"type", "name", "value" }, new String[] { "hidden", "max-size",
String.valueOf(e.expectedSize == -1 ? Long.MAX_VALUE : e.expectedSize*2) });
optionForm.addChild("input", new String[] {
"type", "name", "value" }, new String[] { "submit", "fetch", "Fetch anyway and
display file in browser" });
option = optionList.addChild("li");
optionForm = option.addChild("form", new
String[] { "action", "method" }, new String[] { "/queue/", "post" });
Modified: trunk/freenet/src/freenet/clients/http/QueueToadlet.java
===================================================================
--- trunk/freenet/src/freenet/clients/http/QueueToadlet.java 2006-08-22
17:47:54 UTC (rev 10238)
+++ trunk/freenet/src/freenet/clients/http/QueueToadlet.java 2006-08-22
19:08:34 UTC (rev 10239)
@@ -286,7 +286,7 @@
String mimeType =
clientGet.getMIMEType();
String
requestedMimeType = request.getParam("type", null);
String forceString =
request.getParam("force");
-
FProxyToadlet.handleDownload(ctx, data, ctx.getBucketFactory(), mimeType,
requestedMimeType, forceString, request.isParameterSet("forcedownload"),
"/queue/", key);
+
FProxyToadlet.handleDownload(ctx, data, ctx.getBucketFactory(), mimeType,
requestedMimeType, forceString, request.isParameterSet("forcedownload"),
"/queue/", key, "");
return;
}
}
Modified: trunk/freenet/src/freenet/keys/Key.java
===================================================================
--- trunk/freenet/src/freenet/keys/Key.java 2006-08-22 17:47:54 UTC (rev
10238)
+++ trunk/freenet/src/freenet/keys/Key.java 2006-08-22 19:08:34 UTC (rev
10239)
@@ -116,7 +116,7 @@
Compressor decompressor =
Compressor.getCompressionAlgorithmByMetadataID(compressionAlgorithm);
Bucket inputBucket = new SimpleReadOnlyArrayBucket(output,
shortLength?2:4, output.length-(shortLength?2:4));
try {
- return decompressor.decompress(inputBucket, bf,
maxLength, null);
+ return decompressor.decompress(inputBucket, bf,
maxLength, -1, null);
} catch (CompressionOutputSizeException e) {
throw new CHKDecodeException("Too big");
}
Modified:
trunk/freenet/src/freenet/support/compress/CompressionOutputSizeException.java
===================================================================
---
trunk/freenet/src/freenet/support/compress/CompressionOutputSizeException.java
2006-08-22 17:47:54 UTC (rev 10238)
+++
trunk/freenet/src/freenet/support/compress/CompressionOutputSizeException.java
2006-08-22 19:08:34 UTC (rev 10239)
@@ -5,4 +5,13 @@
*/
public class CompressionOutputSizeException extends Exception {
private static final long serialVersionUID = -1;
+ public final long estimatedSize;
+
+ CompressionOutputSizeException() {
+ estimatedSize = -1;
+ }
+
+ CompressionOutputSizeException(long sz) {
+ estimatedSize = sz;
+ }
}
Modified: trunk/freenet/src/freenet/support/compress/Compressor.java
===================================================================
--- trunk/freenet/src/freenet/support/compress/Compressor.java 2006-08-22
17:47:54 UTC (rev 10238)
+++ trunk/freenet/src/freenet/support/compress/Compressor.java 2006-08-22
19:08:34 UTC (rev 10239)
@@ -21,12 +21,13 @@
* @param data The data to decompress.
* @param bucketFactory A BucketFactory to create a new Bucket with if
necessary.
* @param maxLength The maximum length to decompress (we throw if more
is present).
+ * @param maxEstimateSizeLength If the data is too big, and this is >0,
read up to this many bytes in order to try to get the data size.
* @param preferred A Bucket to use instead. If null, we allocate one
from the BucketFactory.
* @return
* @throws IOException
* @throws CompressionOutputSizeException
*/
- public abstract Bucket decompress(Bucket data, BucketFactory
bucketFactory, long maxLength, Bucket preferred) throws IOException,
CompressionOutputSizeException;
+ public abstract Bucket decompress(Bucket data, BucketFactory
bucketFactory, long maxLength, long maxEstimateSizeLength, Bucket preferred)
throws IOException, CompressionOutputSizeException;
public short codecNumberForMetadata() {
return Metadata.COMPRESS_GZIP;
Modified: trunk/freenet/src/freenet/support/compress/GzipCompressor.java
===================================================================
--- trunk/freenet/src/freenet/support/compress/GzipCompressor.java
2006-08-22 17:47:54 UTC (rev 10238)
+++ trunk/freenet/src/freenet/support/compress/GzipCompressor.java
2006-08-22 19:08:34 UTC (rev 10239)
@@ -8,6 +8,7 @@
import java.util.zip.GZIPInputStream;
import java.util.zip.GZIPOutputStream;
+import freenet.support.Logger;
import freenet.support.io.Bucket;
import freenet.support.io.BucketFactory;
@@ -44,7 +45,7 @@
return output;
}
- public Bucket decompress(Bucket data, BucketFactory bf, long maxLength,
Bucket preferred) throws IOException, CompressionOutputSizeException {
+ public Bucket decompress(Bucket data, BucketFactory bf, long maxLength,
long maxCheckSizeLength, Bucket preferred) throws IOException,
CompressionOutputSizeException {
Bucket output;
if(preferred != null)
output = preferred;
@@ -52,13 +53,13 @@
output = bf.makeBucket(-1);
InputStream is = data.getInputStream();
OutputStream os = output.getOutputStream();
- decompress(is, os, maxLength);
+ decompress(is, os, maxLength, maxCheckSizeLength);
os.close();
is.close();
return output;
}
- private long decompress(InputStream is, OutputStream os, long
maxLength) throws IOException, CompressionOutputSizeException {
+ private long decompress(InputStream is, OutputStream os, long
maxLength, long maxCheckSizeBytes) throws IOException,
CompressionOutputSizeException {
GZIPInputStream gis = new GZIPInputStream(is);
long written = 0;
byte[] buffer = new byte[4096];
@@ -66,6 +67,17 @@
int l = (int) Math.min(buffer.length, maxLength -
written);
int x = gis.read(buffer, 0, 4096);
if(l < x) {
+ Logger.error(this, "l="+l+", x="+x+",
written="+written+", maxLength="+maxLength+" throwing a
CompressionOutputSizeException");
+ if(maxCheckSizeBytes > 0) {
+ written += x;
+ while(true) {
+ l = (int)
Math.min(buffer.length, maxLength + maxCheckSizeBytes - written);
+ x = gis.read(buffer, 0, 4096);
+ if(x <= -1) throw new
CompressionOutputSizeException(written);
+ if(x == 0) throw new
IOException("Returned zero from read()");
+ written += x;
+ }
+ }
throw new CompressionOutputSizeException();
}
if(x <= -1) return written;
@@ -82,7 +94,7 @@
ByteArrayOutputStream baos = new
ByteArrayOutputStream(output.length);
int bytes = 0;
try {
- bytes = (int)decompress(bais, baos, output.length);
+ bytes = (int)decompress(bais, baos, output.length, -1);
} catch (IOException e) {
throw new Error("Got IOException: "+e.getMessage());
}
