Author: toad
Date: 2006-09-29 16:42:10 +0000 (Fri, 29 Sep 2006)
New Revision: 10570
Added:
trunk/freenet/src/freenet/client/async/BackgroundBlockEncoder.java
Modified:
trunk/freenet/src/freenet/client/HighLevelSimpleClientImpl.java
trunk/freenet/src/freenet/client/InserterContext.java
trunk/freenet/src/freenet/client/async/SingleBlockInserter.java
trunk/freenet/src/freenet/client/async/SingleFileInserter.java
trunk/freenet/src/freenet/client/async/SplitFileInserter.java
trunk/freenet/src/freenet/client/async/SplitFileInserterSegment.java
trunk/freenet/src/freenet/node/NodeClientCore.java
Log:
Once we pass the two thirds of an insert needed to fetch it, encode all its
remaining blocks in the background, so that we can make the metadata ASAP (and
then the URI for the metadata, and insert it)
Modified: trunk/freenet/src/freenet/client/HighLevelSimpleClientImpl.java
===================================================================
--- trunk/freenet/src/freenet/client/HighLevelSimpleClientImpl.java
2006-09-29 16:15:28 UTC (rev 10569)
+++ trunk/freenet/src/freenet/client/HighLevelSimpleClientImpl.java
2006-09-29 16:42:10 UTC (rev 10570)
@@ -6,6 +6,7 @@
import java.io.IOException;
import java.util.HashMap;
+import freenet.client.async.BackgroundBlockEncoder;
import freenet.client.async.ClientGetter;
import freenet.client.async.ClientPutter;
import freenet.client.async.HealingQueue;
@@ -32,6 +33,7 @@
private final BucketFactory persistentBucketFactory;
private final PersistentFileTracker persistentFileTracker;
private final NodeClientCore core;
+ private final BackgroundBlockEncoder blockEncoder;
/** One CEP for all requests and inserts */
private final ClientEventProducer globalEventProducer;
private long curMaxLength;
@@ -89,6 +91,7 @@
this.cacheLocalRequests = cacheLocalRequests;
this.persistentBucketFactory =
node.persistentEncryptedTempBucketFactory;
this.healingQueue = node.getHealingQueue();
+ this.blockEncoder = node.backgroundBlockEncoder;
}
public void setMaxLength(long maxLength) {
@@ -188,6 +191,6 @@
forceNonPersistent ? new
NullPersistentFileTracker() : persistentFileTracker,
random, INSERT_RETRIES,
CONSECUTIVE_RNFS_ASSUME_SUCCESS,
SPLITFILE_INSERT_THREADS,
SPLITFILE_BLOCKS_PER_SEGMENT, SPLITFILE_CHECK_BLOCKS_PER_SEGMENT,
- globalEventProducer, cacheLocalRequests,
core.uskManager);
+ globalEventProducer, cacheLocalRequests,
core.uskManager, blockEncoder);
}
}
Modified: trunk/freenet/src/freenet/client/InserterContext.java
===================================================================
--- trunk/freenet/src/freenet/client/InserterContext.java 2006-09-29
16:15:28 UTC (rev 10569)
+++ trunk/freenet/src/freenet/client/InserterContext.java 2006-09-29
16:42:10 UTC (rev 10570)
@@ -1,5 +1,6 @@
package freenet.client;
+import freenet.client.async.BackgroundBlockEncoder;
import freenet.client.async.USKManager;
import freenet.client.events.ClientEventProducer;
import freenet.client.events.SimpleEventProducer;
@@ -27,10 +28,11 @@
/** Interesting tradeoff, see comments at top of Node.java. */
public final boolean cacheLocalRequests;
public final USKManager uskManager;
+ public final BackgroundBlockEncoder backgroundBlockEncoder;
public InserterContext(BucketFactory bf, BucketFactory persistentBF,
PersistentFileTracker tracker, RandomSource random,
int maxRetries, int rnfsToSuccess, int maxThreads, int
splitfileSegmentDataBlocks, int splitfileSegmentCheckBlocks,
- ClientEventProducer eventProducer, boolean
cacheLocalRequests, USKManager uskManager) {
+ ClientEventProducer eventProducer, boolean
cacheLocalRequests, USKManager uskManager, BackgroundBlockEncoder blockEncoder)
{
this.bf = bf;
this.persistentFileTracker = tracker;
this.persistentBucketFactory = persistentBF;
@@ -45,6 +47,7 @@
this.splitfileSegmentDataBlocks = splitfileSegmentDataBlocks;
this.splitfileSegmentCheckBlocks = splitfileSegmentCheckBlocks;
this.cacheLocalRequests = cacheLocalRequests;
+ this.backgroundBlockEncoder = blockEncoder;
}
public InserterContext(InserterContext ctx, SimpleEventProducer
producer, boolean forceNonPersistent) {
@@ -62,6 +65,7 @@
this.splitfileSegmentDataBlocks =
ctx.splitfileSegmentDataBlocks;
this.splitfileSegmentCheckBlocks =
ctx.splitfileSegmentCheckBlocks;
this.cacheLocalRequests = ctx.cacheLocalRequests;
+ this.backgroundBlockEncoder = ctx.backgroundBlockEncoder;
}
public InserterContext(InserterContext ctx, SimpleEventProducer
producer) {
@@ -79,6 +83,7 @@
this.splitfileSegmentDataBlocks =
ctx.splitfileSegmentDataBlocks;
this.splitfileSegmentCheckBlocks =
ctx.splitfileSegmentCheckBlocks;
this.cacheLocalRequests = ctx.cacheLocalRequests;
+ this.backgroundBlockEncoder = ctx.backgroundBlockEncoder;
}
}
Added: trunk/freenet/src/freenet/client/async/BackgroundBlockEncoder.java
===================================================================
--- trunk/freenet/src/freenet/client/async/BackgroundBlockEncoder.java
2006-09-29 16:15:28 UTC (rev 10569)
+++ trunk/freenet/src/freenet/client/async/BackgroundBlockEncoder.java
2006-09-29 16:42:10 UTC (rev 10570)
@@ -0,0 +1,63 @@
+package freenet.client.async;
+
+import java.lang.ref.WeakReference;
+import java.util.ArrayList;
+
+/**
+ * Keeps a queue of SingleBlockInserter's to encode.
+ * Encodes them.
+ */
+public class BackgroundBlockEncoder implements Runnable {
+
+ // Minimize memory usage at the cost of having to encode from the end
+ private final ArrayList queue;
+
+ public BackgroundBlockEncoder() {
+ queue = new ArrayList();
+ }
+
+ public void queue(SingleBlockInserter sbi) {
+ if(sbi.isFinished()) return;
+ if(sbi.resultingURI != null) return;
+ WeakReference ref = new WeakReference(sbi);
+ synchronized(this) {
+ queue.add(ref);
+ notifyAll();
+ }
+ }
+
+ public void queue(SingleBlockInserter[] sbis) {
+ synchronized(this) {
+ for(int i=0;i<sbis.length;i++) {
+ if(sbis[i].isFinished()) continue;
+ if(sbis[i].resultingURI != null) continue;
+ WeakReference ref = new WeakReference(sbis[i]);
+ queue.add(ref);
+ }
+ notifyAll();
+ }
+ }
+
+ public void run() {
+ while(true) {
+ SingleBlockInserter sbi = null;
+ synchronized(this) {
+ try {
+ wait(100*1000);
+ } catch (InterruptedException e) {
+ // Ignore
+ }
+ while(true) {
+ if(queue.isEmpty()) break;
+ WeakReference ref = (WeakReference)
queue.remove(queue.size()-1);
+ sbi = (SingleBlockInserter) ref.get();
+ if(sbi == null) continue;
+ }
+ }
+ if(sbi.isFinished()) continue;
+ if(sbi.resultingURI != null) continue;
+ sbi.tryEncode();
+ }
+ }
+
+}
Modified: trunk/freenet/src/freenet/client/async/SingleBlockInserter.java
===================================================================
--- trunk/freenet/src/freenet/client/async/SingleBlockInserter.java
2006-09-29 16:15:28 UTC (rev 10569)
+++ trunk/freenet/src/freenet/client/async/SingleBlockInserter.java
2006-09-29 16:42:10 UTC (rev 10570)
@@ -306,4 +306,17 @@
return null;
}
+ /** Attempt to encode the block, if necessary */
+ public void tryEncode() {
+ try {
+ encode();
+ } catch (InserterException e) {
+ fail(e);
+ } catch (Throwable t) {
+ Logger.error(this, "Caught "+t, t);
+ // Don't requeue on BackgroundBlockEncoder.
+ // Not necessary to do so (we'll ask again when we need
it), and it'll probably just break again.
+ }
+ }
+
}
Modified: trunk/freenet/src/freenet/client/async/SingleFileInserter.java
===================================================================
--- trunk/freenet/src/freenet/client/async/SingleFileInserter.java
2006-09-29 16:15:28 UTC (rev 10569)
+++ trunk/freenet/src/freenet/client/async/SingleFileInserter.java
2006-09-29 16:42:10 UTC (rev 10570)
@@ -584,16 +584,22 @@
private void startMetadata() {
try {
ClientPutState putter;
+ ClientPutState splitInserter;
synchronized(this) {
if(metadataPutter == null) {
if(logMINOR) Logger.minor(this,
"Cannot start metadata yet: no metadataPutter");
- return;
}
putter = metadataPutter;
+ splitInserter = sfi;
}
- if(logMINOR) Logger.minor(this, "Starting
metadata inserter: "+putter+" for "+this);
- putter.schedule();
- if(logMINOR) Logger.minor(this, "Started
metadata inserter: "+putter+" for "+this);
+ if(putter != null) {
+ if(logMINOR) Logger.minor(this,
"Starting metadata inserter: "+putter+" for "+this);
+ putter.schedule();
+ if(logMINOR) Logger.minor(this,
"Started metadata inserter: "+putter+" for "+this);
+ } else {
+ // Get all the URIs ASAP so we can
start to insert the metadata.
+
((SplitFileInserter)splitInserter).forceEncode();
+ }
} catch (InserterException e1) {
Logger.error(this, "Failing "+this+" : "+e1,
e1);
fail(e1);
Modified: trunk/freenet/src/freenet/client/async/SplitFileInserter.java
===================================================================
--- trunk/freenet/src/freenet/client/async/SplitFileInserter.java
2006-09-29 16:15:28 UTC (rev 10569)
+++ trunk/freenet/src/freenet/client/async/SplitFileInserter.java
2006-09-29 16:42:10 UTC (rev 10570)
@@ -436,4 +436,11 @@
return dataLength;
}
+ /** Force the remaining blocks which haven't been encoded so far to be
encoded ASAP. */
+ public void forceEncode() {
+ for(int i=0;i<segments.length;i++) {
+ segments[i].forceEncode();
+ }
+ }
+
}
Modified: trunk/freenet/src/freenet/client/async/SplitFileInserterSegment.java
===================================================================
--- trunk/freenet/src/freenet/client/async/SplitFileInserterSegment.java
2006-09-29 16:15:28 UTC (rev 10569)
+++ trunk/freenet/src/freenet/client/async/SplitFileInserterSegment.java
2006-09-29 16:42:10 UTC (rev 10570)
@@ -616,4 +616,10 @@
public void onFetchable(ClientPutState state) {
// Ignore
}
+
+ /** Force the remaining blocks which haven't been encoded so far to be
encoded ASAP. */
+ public void forceEncode() {
+ // TODO Auto-generated method stub
+
+ }
}
Modified: trunk/freenet/src/freenet/node/NodeClientCore.java
===================================================================
--- trunk/freenet/src/freenet/node/NodeClientCore.java 2006-09-29 16:15:28 UTC
(rev 10569)
+++ trunk/freenet/src/freenet/node/NodeClientCore.java 2006-09-29 16:42:10 UTC
(rev 10570)
@@ -7,6 +7,7 @@
import freenet.client.HighLevelSimpleClient;
import freenet.client.HighLevelSimpleClientImpl;
import freenet.client.InserterContext;
+import freenet.client.async.BackgroundBlockEncoder;
import freenet.client.async.HealingQueue;
import freenet.client.async.SimpleHealingQueue;
import freenet.client.async.USKManager;
@@ -79,6 +80,7 @@
SimpleToadletServer toadletContainer;
// FIXME why isn't this just in fproxy?
public BookmarkManager bookmarkManager;
+ public final BackgroundBlockEncoder backgroundBlockEncoder;
// Client stuff that needs to be configged - FIXME
static final int MAX_ARCHIVE_HANDLERS = 200; // don't take up much
RAM... FIXME
@@ -90,6 +92,10 @@
NodeClientCore(Node node, Config config, SubConfig nodeConfig, File
nodeDir, int portNumber, int sortOrder, SimpleFieldSet throttleFS) throws
NodeInitException {
this.node = node;
this.random = node.random;
+ this.backgroundBlockEncoder = new BackgroundBlockEncoder();
+ Thread t = new Thread(backgroundBlockEncoder, "Background block
encoder");
+ t.setDaemon(true);
+ t.start();
byte[] pwdBuf = new byte[16];
random.nextBytes(pwdBuf);
this.formPassword = Base64.encode(pwdBuf);
@@ -183,7 +189,7 @@
healingQueue = new
SimpleHealingQueue(requestStarters.chkPutScheduler,
new InserterContext(tempBucketFactory,
tempBucketFactory, persistentTempBucketFactory,
random, 0, 2, 1, 0, 0, new
SimpleEventProducer(),
-
!Node.DONT_CACHE_LOCAL_REQUESTS, uskManager),
RequestStarter.PREFETCH_PRIORITY_CLASS, 512 /* FIXME make configurable */);
+
!Node.DONT_CACHE_LOCAL_REQUESTS, uskManager, backgroundBlockEncoder),
RequestStarter.PREFETCH_PRIORITY_CLASS, 512 /* FIXME make configurable */);
}
