Author: toad
Date: 2008-07-29 15:27:22 +0000 (Tue, 29 Jul 2008)
New Revision: 21478
Modified:
branches/db4o/freenet/src/freenet/clients/http/QueueToadlet.java
branches/db4o/freenet/src/freenet/node/fcp/FCPServer.java
branches/db4o/freenet/src/freenet/support/SimpleReadOnlyArrayBucket.java
branches/db4o/freenet/src/freenet/support/api/Bucket.java
branches/db4o/freenet/src/freenet/support/io/ArrayBucket.java
branches/db4o/freenet/src/freenet/support/io/BucketChainBucket.java
branches/db4o/freenet/src/freenet/support/io/BucketTools.java
branches/db4o/freenet/src/freenet/support/io/DelayedFreeBucket.java
branches/db4o/freenet/src/freenet/support/io/FileBucket.java
branches/db4o/freenet/src/freenet/support/io/MultiReaderBucket.java
branches/db4o/freenet/src/freenet/support/io/NullBucket.java
branches/db4o/freenet/src/freenet/support/io/PaddedEphemerallyEncryptedBucket.java
branches/db4o/freenet/src/freenet/support/io/ReadOnlyFileSliceBucket.java
branches/db4o/freenet/src/freenet/support/io/TempFileBucket.java
Log:
Shadow buckets: A solution to the activation problems on the queue page which
doesn't (usually) involve copying to a transient bucket just to send to the
user.
Modified: branches/db4o/freenet/src/freenet/clients/http/QueueToadlet.java
===================================================================
--- branches/db4o/freenet/src/freenet/clients/http/QueueToadlet.java
2008-07-29 14:44:46 UTC (rev 21477)
+++ branches/db4o/freenet/src/freenet/clients/http/QueueToadlet.java
2008-07-29 15:27:22 UTC (rev 21478)
@@ -32,6 +32,7 @@
import freenet.client.FetchResult;
import freenet.client.HighLevelSimpleClient;
import freenet.client.MetadataUnresolvedException;
+import freenet.client.TempFetchResult;
import freenet.client.async.ClientContext;
import freenet.client.async.DBJob;
import freenet.keys.FreenetURI;
@@ -418,13 +419,15 @@
FreenetURI key = new FreenetURI(requestPath);
/* locate request */
- FetchResult result =
fcp.getCompletedRequestBlocking(key);
+ TempFetchResult result =
fcp.getCompletedRequestBlocking(key);
if(result != null) {
Bucket data = result.asBucket();
String mimeType = result.getMimeType();
String requestedMimeType =
request.getParam("type", null);
String forceString =
request.getParam("force");
FProxyToadlet.handleDownload(ctx, data,
ctx.getBucketFactory(), mimeType, requestedMimeType, forceString,
request.isParameterSet("forcedownload"), "/queue/", key, "", "/queue/", false,
ctx);
+ if(result.freeWhenDone)
+ data.free();
return;
}
} catch (MalformedURLException mue1) {
Modified: branches/db4o/freenet/src/freenet/node/fcp/FCPServer.java
===================================================================
--- branches/db4o/freenet/src/freenet/node/fcp/FCPServer.java 2008-07-29
14:44:46 UTC (rev 21477)
+++ branches/db4o/freenet/src/freenet/node/fcp/FCPServer.java 2008-07-29
15:27:22 UTC (rev 21478)
@@ -25,6 +25,7 @@
import freenet.client.FetchResult;
import freenet.client.HighLevelSimpleClient;
import freenet.client.InsertContext;
+import freenet.client.TempFetchResult;
import freenet.client.async.ClientContext;
import freenet.client.async.DBJob;
import freenet.config.Config;
@@ -44,8 +45,10 @@
import freenet.support.MutableBoolean;
import freenet.support.OOMHandler;
import freenet.support.api.BooleanCallback;
+import freenet.support.api.Bucket;
import freenet.support.api.IntCallback;
import freenet.support.api.StringCallback;
+import freenet.support.io.BucketTools;
import freenet.support.io.Closer;
import freenet.support.io.NativeThread;
@@ -1028,15 +1031,15 @@
- public FetchResult getCompletedRequestBlocking(final FreenetURI key) {
+ public TempFetchResult getCompletedRequestBlocking(final FreenetURI
key) {
ClientGet get = globalRebootClient.getCompletedRequest(key,
null);
if(get != null) {
// FIXME race condition with free() - arrange
refcounting for the data to prevent this
- return new FetchResult(new
ClientMetadata(get.getMIMEType(null)), get.getBucket());
+ return new TempFetchResult(new
ClientMetadata(get.getMIMEType(null)), get.getBucket(), false);
}
class OutputWrapper {
- FetchResult result;
+ TempFetchResult result;
boolean done;
}
@@ -1045,12 +1048,33 @@
core.clientContext.jobRunner.queue(new DBJob() {
public void run(ObjectContainer container,
ClientContext context) {
- FetchResult result = null;
+ TempFetchResult result = null;
try {
ClientGet get =
globalForeverClient.getCompletedRequest(key, container);
container.activate(get, 1);
if(get != null) {
- result = new FetchResult(new
ClientMetadata(get.getMIMEType(container)), get.getBucket());
+ Bucket origData =
get.getBucket();
+ container.activate(origData, 5);
+ boolean copied = false;
+ Bucket newData;
+ try {
+ newData =
origData.createShadow();
+ } catch (IOException e) {
+ Logger.error(this,
"Caught error "+e+" trying to create shallow copy, copying data...", e);
+ newData = null;
+ }
+ if(newData == null) {
+ try {
+ newData =
core.tempBucketFactory.makeBucket(origData.size());
+
BucketTools.copy(origData, newData);
+ } catch (IOException e)
{
+
Logger.error(this, "Unable to copy data: "+e, e);
+ result = null;
+ return;
+ }
+ copied = true;
+ }
+ result = new
TempFetchResult(new ClientMetadata(get.getMIMEType(container)), newData,
copied);
}
container.deactivate(get, 1);
} finally {
Modified:
branches/db4o/freenet/src/freenet/support/SimpleReadOnlyArrayBucket.java
===================================================================
--- branches/db4o/freenet/src/freenet/support/SimpleReadOnlyArrayBucket.java
2008-07-29 14:44:46 UTC (rev 21477)
+++ branches/db4o/freenet/src/freenet/support/SimpleReadOnlyArrayBucket.java
2008-07-29 15:27:22 UTC (rev 21478)
@@ -67,4 +67,13 @@
container.delete(this);
}
+ public Bucket createShadow() throws IOException {
+ if(buf.length < 256*1024) {
+ byte[] newBuf = new byte[length];
+ System.arraycopy(buf, offset, newBuf, 0, length);
+ return new SimpleReadOnlyArrayBucket(newBuf);
+ }
+ return null;
+ }
+
}
Modified: branches/db4o/freenet/src/freenet/support/api/Bucket.java
===================================================================
--- branches/db4o/freenet/src/freenet/support/api/Bucket.java 2008-07-29
14:44:46 UTC (rev 21477)
+++ branches/db4o/freenet/src/freenet/support/api/Bucket.java 2008-07-29
15:27:22 UTC (rev 21478)
@@ -64,4 +64,14 @@
*/
public void removeFrom(ObjectContainer container);
+ /**
+ * Create a shallow read-only copy of this bucket, using different
+ * objects but using the same external storage. If this is not
possible,
+ * return null. Note that if the underlying bucket is deleted, the copy
+ * will become invalid and probably throw an IOException on read, or
+ * possibly return too-short data etc. In some use cases e.g. on
fproxy,
+ * this is acceptable.
+ */
+ public Bucket createShadow() throws IOException;
+
}
Modified: branches/db4o/freenet/src/freenet/support/io/ArrayBucket.java
===================================================================
--- branches/db4o/freenet/src/freenet/support/io/ArrayBucket.java
2008-07-29 14:44:46 UTC (rev 21477)
+++ branches/db4o/freenet/src/freenet/support/io/ArrayBucket.java
2008-07-29 15:27:22 UTC (rev 21478)
@@ -197,4 +197,8 @@
public void removeFrom(ObjectContainer container) {
container.delete(data);
}
+
+ public Bucket createShadow() throws IOException {
+ return null;
+ }
}
Modified: branches/db4o/freenet/src/freenet/support/io/BucketChainBucket.java
===================================================================
--- branches/db4o/freenet/src/freenet/support/io/BucketChainBucket.java
2008-07-29 14:44:46 UTC (rev 21477)
+++ branches/db4o/freenet/src/freenet/support/io/BucketChainBucket.java
2008-07-29 15:27:22 UTC (rev 21478)
@@ -32,6 +32,14 @@
readOnly = false;
}
+ private BucketChainBucket(Vector newBuckets, long bucketSize2, long
size2, boolean readOnly, BucketFactory bf2) {
+ this.buckets = newBuckets;
+ this.bucketSize = bucketSize2;
+ this.size = size2;
+ this.readOnly = readOnly;
+ this.bf = bf2;
+ }
+
public void free() {
Bucket[] list;
synchronized(this) {
@@ -292,4 +300,18 @@
container.delete(this);
}
+ public Bucket createShadow() throws IOException {
+ Vector newBuckets = new Vector();
+ for(int i=0;i<buckets.size();i++) {
+ Bucket data = (Bucket) buckets.get(i);
+ Bucket shadow = data.createShadow();
+ if(shadow == null) {
+ // Shadow buckets don't need to be freed.
+ return null;
+ }
+ newBuckets.add(shadow);
+ }
+ return new BucketChainBucket(newBuckets, bucketSize, size,
true, bf);
+ }
+
}
Modified: branches/db4o/freenet/src/freenet/support/io/BucketTools.java
===================================================================
--- branches/db4o/freenet/src/freenet/support/io/BucketTools.java
2008-07-29 14:44:46 UTC (rev 21477)
+++ branches/db4o/freenet/src/freenet/support/io/BucketTools.java
2008-07-29 15:27:22 UTC (rev 21478)
@@ -424,4 +424,5 @@
return b;
} finally { os.close(); }
}
+
}
Modified: branches/db4o/freenet/src/freenet/support/io/DelayedFreeBucket.java
===================================================================
--- branches/db4o/freenet/src/freenet/support/io/DelayedFreeBucket.java
2008-07-29 14:44:46 UTC (rev 21477)
+++ branches/db4o/freenet/src/freenet/support/io/DelayedFreeBucket.java
2008-07-29 15:27:22 UTC (rev 21478)
@@ -129,6 +129,10 @@
// Cascading activation of dependancies
container.activate(bucket, 1);
}
+
+ public Bucket createShadow() throws IOException {
+ return bucket.createShadow();
+ }
// public void objectOnDeactivate(ObjectContainer container) {
// if(Logger.shouldLog(Logger.MINOR, this)) Logger.minor(this,
"Deactivating "+super.toString()+" : "+bucket, new Exception("debug"));
Modified: branches/db4o/freenet/src/freenet/support/io/FileBucket.java
===================================================================
--- branches/db4o/freenet/src/freenet/support/io/FileBucket.java
2008-07-29 14:44:46 UTC (rev 21477)
+++ branches/db4o/freenet/src/freenet/support/io/FileBucket.java
2008-07-29 15:27:22 UTC (rev 21478)
@@ -4,6 +4,7 @@
package freenet.support.io;
import java.io.File;
+import java.io.IOException;
import com.db4o.ObjectContainer;
@@ -115,4 +116,10 @@
public void objectOnActivate(ObjectContainer container) {
container.activate(file, 5);
}
+
+ public Bucket createShadow() throws IOException {
+ String fnam = new String(file.getPath());
+ File newFile = new File(fnam);
+ return new FileBucket(newFile, true, false, false, false,
false);
+ }
}
Modified: branches/db4o/freenet/src/freenet/support/io/MultiReaderBucket.java
===================================================================
--- branches/db4o/freenet/src/freenet/support/io/MultiReaderBucket.java
2008-07-29 14:44:46 UTC (rev 21477)
+++ branches/db4o/freenet/src/freenet/support/io/MultiReaderBucket.java
2008-07-29 15:27:22 UTC (rev 21478)
@@ -139,6 +139,10 @@
container.delete(readers);
container.delete(MultiReaderBucket.this);
}
+
+ public Bucket createShadow() throws IOException {
+ return null;
+ }
}
Modified: branches/db4o/freenet/src/freenet/support/io/NullBucket.java
===================================================================
--- branches/db4o/freenet/src/freenet/support/io/NullBucket.java
2008-07-29 14:44:46 UTC (rev 21477)
+++ branches/db4o/freenet/src/freenet/support/io/NullBucket.java
2008-07-29 15:27:22 UTC (rev 21478)
@@ -2,6 +2,7 @@
* Public License, version 2 (or at your option any later version). See
* http://www.gnu.org/ for further details of the GPL. */
package freenet.support.io;
+import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
@@ -80,5 +81,9 @@
public void removeFrom(ObjectContainer container) {
container.delete(this);
}
+
+ public Bucket createShadow() throws IOException {
+ return new NullBucket();
+ }
}
Modified:
branches/db4o/freenet/src/freenet/support/io/PaddedEphemerallyEncryptedBucket.java
===================================================================
---
branches/db4o/freenet/src/freenet/support/io/PaddedEphemerallyEncryptedBucket.java
2008-07-29 14:44:46 UTC (rev 21477)
+++
branches/db4o/freenet/src/freenet/support/io/PaddedEphemerallyEncryptedBucket.java
2008-07-29 15:27:22 UTC (rev 21478)
@@ -123,6 +123,16 @@
origRandom.nextBytes(randomSeed);
}
+ public
PaddedEphemerallyEncryptedBucket(PaddedEphemerallyEncryptedBucket orig, Bucket
newBucket) {
+ this.dataLength = orig.dataLength;
+ this.key = new byte[orig.key.length];
+ System.arraycopy(orig.key, 0, key, 0, orig.key.length);
+ this.randomSeed = null; // Will be read-only
+ setReadOnly();
+ this.bucket = newBucket;
+ this.minPaddedSize = orig.minPaddedSize;
+ }
+
public OutputStream getOutputStream() throws IOException {
if(readOnly) throw new IOException("Read only");
OutputStream os = bucket.getOutputStream();
@@ -392,4 +402,10 @@
container.activate(bucket, 1);
}
+ public Bucket createShadow() throws IOException {
+ Bucket newUnderlying = bucket.createShadow();
+ if(newUnderlying == null) return null;
+ return new PaddedEphemerallyEncryptedBucket(this,
newUnderlying);
+ }
+
}
Modified:
branches/db4o/freenet/src/freenet/support/io/ReadOnlyFileSliceBucket.java
===================================================================
--- branches/db4o/freenet/src/freenet/support/io/ReadOnlyFileSliceBucket.java
2008-07-29 14:44:46 UTC (rev 21477)
+++ branches/db4o/freenet/src/freenet/support/io/ReadOnlyFileSliceBucket.java
2008-07-29 15:27:22 UTC (rev 21478)
@@ -161,4 +161,10 @@
container.activate(file, 5);
}
+ public Bucket createShadow() throws IOException {
+ String fnam = new String(file.getPath());
+ File newFile = new File(fnam);
+ return new ReadOnlyFileSliceBucket(newFile, startAt, length);
+ }
+
}
Modified: branches/db4o/freenet/src/freenet/support/io/TempFileBucket.java
===================================================================
--- branches/db4o/freenet/src/freenet/support/io/TempFileBucket.java
2008-07-29 14:44:46 UTC (rev 21477)
+++ branches/db4o/freenet/src/freenet/support/io/TempFileBucket.java
2008-07-29 15:27:22 UTC (rev 21478)
@@ -1,6 +1,7 @@
package freenet.support.io;
import java.io.File;
+import java.io.IOException;
import com.db4o.ObjectContainer;
@@ -96,4 +97,10 @@
// filenameGenerator is a global, we don't need to worry about
it.
container.delete(this);
}
+
+ public Bucket createShadow() throws IOException {
+ TempFileBucket ret = new TempFileBucket(filenameID, generator);
+ ret.setReadOnly();
+ return ret;
+ }
}
