Author: toad
Date: 2006-01-25 16:44:41 +0000 (Wed, 25 Jan 2006)
New Revision: 7920
Modified:
branches/async-client/src/freenet/client/PutWaiter.java
branches/async-client/src/freenet/client/async/ClientPutter.java
branches/async-client/src/freenet/client/async/ClientRequestScheduler.java
branches/async-client/src/freenet/client/async/MultiPutCompletionCallback.java
branches/async-client/src/freenet/client/async/PutCompletionCallback.java
branches/async-client/src/freenet/client/async/SingleBlockInserter.java
branches/async-client/src/freenet/client/async/SingleFileFetcher.java
branches/async-client/src/freenet/client/async/SingleFileInserter.java
branches/async-client/src/freenet/client/async/SplitFileFetcherSegment.java
branches/async-client/src/freenet/client/async/SplitFileInserter.java
branches/async-client/src/freenet/client/async/SplitFileInserterSegment.java
branches/async-client/src/freenet/keys/Key.java
branches/async-client/src/freenet/node/TextModeClientInterface.java
Log:
splitfiles, compressed splitfiles work (with CHK@ via TMCI anyway).
Modified: branches/async-client/src/freenet/client/PutWaiter.java
===================================================================
--- branches/async-client/src/freenet/client/PutWaiter.java 2006-01-25
14:55:11 UTC (rev 7919)
+++ branches/async-client/src/freenet/client/PutWaiter.java 2006-01-25
16:44:41 UTC (rev 7920)
@@ -34,6 +34,7 @@
}
public synchronized void onGeneratedURI(FreenetURI uri, ClientPutter
state) {
+ Logger.minor(this, "URI: "+uri);
if(this.uri == null)
this.uri = uri;
if(uri.equals(this.uri)) return;
Modified: branches/async-client/src/freenet/client/async/ClientPutter.java
===================================================================
--- branches/async-client/src/freenet/client/async/ClientPutter.java
2006-01-25 14:55:11 UTC (rev 7919)
+++ branches/async-client/src/freenet/client/async/ClientPutter.java
2006-01-25 16:44:41 UTC (rev 7920)
@@ -41,6 +41,7 @@
try {
currentState =
new SingleFileInserter(this, this, new
InsertBlock(data, cm, targetURI), isMetadata, ctx, false, false, getCHKOnly);
+ ((SingleFileInserter)currentState).start();
} catch (InserterException e) {
finished = true;
currentState = null;
@@ -83,5 +84,9 @@
public FreenetURI getURI() {
return uri;
}
+
+ public void onTransition(ClientPutState oldState, ClientPutState
newState) {
+ // Ignore
+ }
}
Modified:
branches/async-client/src/freenet/client/async/ClientRequestScheduler.java
===================================================================
--- branches/async-client/src/freenet/client/async/ClientRequestScheduler.java
2006-01-25 14:55:11 UTC (rev 7919)
+++ branches/async-client/src/freenet/client/async/ClientRequestScheduler.java
2006-01-25 16:44:41 UTC (rev 7920)
@@ -35,7 +35,7 @@
}
public void register(SendableRequest req) {
- Logger.minor(this, "Registering "+req);
+ Logger.minor(this, "Registering "+req, new Exception("debug"));
synchronized(this) {
if((!isInsertScheduler) && req instanceof ClientPutter)
throw new IllegalArgumentException("Expected a
ClientPut: "+req);
Modified:
branches/async-client/src/freenet/client/async/MultiPutCompletionCallback.java
===================================================================
---
branches/async-client/src/freenet/client/async/MultiPutCompletionCallback.java
2006-01-25 14:55:11 UTC (rev 7919)
+++
branches/async-client/src/freenet/client/async/MultiPutCompletionCallback.java
2006-01-25 16:44:41 UTC (rev 7920)
@@ -1,6 +1,8 @@
package freenet.client.async;
+import java.util.Iterator;
import java.util.LinkedList;
+import java.util.ListIterator;
import freenet.client.InserterException;
import freenet.keys.ClientKey;
@@ -81,4 +83,16 @@
states[i].cancel();
}
+ public synchronized void onTransition(ClientPutState oldState,
ClientPutState newState) {
+ if(generator == oldState)
+ generator = newState;
+ if(oldState == newState) return;
+ for(ListIterator i = waitingFor.listIterator(0);i.hasNext();) {
+ if(i.next() == oldState) {
+ i.remove();
+ i.add(newState);
+ }
+ }
+ }
+
}
Modified:
branches/async-client/src/freenet/client/async/PutCompletionCallback.java
===================================================================
--- branches/async-client/src/freenet/client/async/PutCompletionCallback.java
2006-01-25 14:55:11 UTC (rev 7919)
+++ branches/async-client/src/freenet/client/async/PutCompletionCallback.java
2006-01-25 16:44:41 UTC (rev 7920)
@@ -14,4 +14,6 @@
public void onEncode(ClientKey key, ClientPutState state);
+ public void onTransition(ClientPutState oldState, ClientPutState
newState);
+
}
Modified:
branches/async-client/src/freenet/client/async/SingleBlockInserter.java
===================================================================
--- branches/async-client/src/freenet/client/async/SingleBlockInserter.java
2006-01-25 14:55:11 UTC (rev 7919)
+++ branches/async-client/src/freenet/client/async/SingleBlockInserter.java
2006-01-25 16:44:41 UTC (rev 7920)
@@ -65,7 +65,7 @@
String uriType = uri.getKeyType().toUpperCase();
if(uriType.equals("CHK")) {
try {
- return ClientCHKBlock.encode(sourceData,
isMetadata, true, compressionCodec, sourceLength);
+ return ClientCHKBlock.encode(sourceData,
isMetadata, compressionCodec == -1, compressionCodec, sourceLength);
} catch (CHKEncodeException e) {
Logger.error(this, "Caught "+e, e);
throw new
InserterException(InserterException.INTERNAL_ERROR, e, null);
@@ -76,7 +76,7 @@
} else if(uriType.equals("SSK")) {
try {
InsertableClientSSK ik =
InsertableClientSSK.create(uri);
- return ik.encode(sourceData, isMetadata, true,
compressionCodec, sourceLength, ctx.random);
+ return ik.encode(sourceData, isMetadata,
compressionCodec == -1, compressionCodec, sourceLength, ctx.random);
} catch (MalformedURLException e) {
throw new
InserterException(InserterException.INVALID_URI, e, null);
} catch (SSKEncodeException e) {
@@ -138,6 +138,7 @@
Logger.error(this, "Unknown LowLevelPutException code:
"+e.code);
errors.inc(InserterException.INTERNAL_ERROR);
}
+ Logger.minor(this, "Failed: "+e);
if(retries > ctx.maxInsertRetries) {
if(errors.isOneCodeOnly())
fail(new
InserterException(errors.getFirstCode()));
@@ -162,6 +163,7 @@
cb.onFailure(e, this);
return null;
} catch (Throwable t) {
+ Logger.error(this, "Caught "+t, t);
cb.onFailure(new
InserterException(InserterException.INTERNAL_ERROR, t, null), this);
return null;
}
@@ -179,6 +181,7 @@
}
public void onSuccess() {
+ Logger.minor(this, "Succeeded ("+this+"): "+token);
synchronized(this) {
finished = true;
}
@@ -203,11 +206,14 @@
public void send(Node node) {
try {
+ Logger.minor(this, "Starting request: "+this);
node.realPut(getBlock(), true);
} catch (LowLevelPutException e) {
onFailure(e);
+ Logger.minor(this, "Request failed: "+this+" for "+e);
return;
}
+ Logger.minor(this, "Request succeeded: "+this);
onSuccess();
}
Modified: branches/async-client/src/freenet/client/async/SingleFileFetcher.java
===================================================================
--- branches/async-client/src/freenet/client/async/SingleFileFetcher.java
2006-01-25 14:55:11 UTC (rev 7919)
+++ branches/async-client/src/freenet/client/async/SingleFileFetcher.java
2006-01-25 16:44:41 UTC (rev 7920)
@@ -56,6 +56,7 @@
* @param dontTellClientGet
*/
public SingleFileFetcher(ClientGetter get, GetCompletionCallback cb,
ClientMetadata metadata, ClientKey key, LinkedList metaStrings, FetcherContext
ctx, ArchiveContext actx, int maxRetries, int recursionLevel, boolean
dontTellClientGet, Object token) throws FetchException {
+ Logger.minor(this, "Creating SingleFileFetcher for "+key);
this.cancelled = false;
this.dontTellClientGet = dontTellClientGet;
this.token = token;
@@ -85,6 +86,7 @@
/** Copy constructor, modifies a few given fields, don't call
schedule() */
public SingleFileFetcher(SingleFileFetcher fetcher, Metadata newMeta,
GetCompletionCallback callback, FetcherContext ctx2) throws FetchException {
+ Logger.minor(this, "Creating SingleFileFetcher for
"+fetcher.key);
this.token = fetcher.token;
this.dontTellClientGet = fetcher.dontTellClientGet;
this.actx = fetcher.actx;
@@ -285,6 +287,7 @@
// Which will then fetch the target URI, and
call the rcd.success
// Hopefully!
FreenetURI uri = metadata.getSingleTarget();
+ Logger.minor(this, "Redirecting to "+uri);
ClientKey key;
try {
key = ClientKey.getBaseKey(uri);
@@ -308,6 +311,7 @@
// All done! No longer our problem!
return;
} else if(metadata.isSplitfile()) {
+ Logger.minor(this, "Fetching splitfile");
// FIXME implicit archive support
clientMetadata.mergeNoOverwrite(metadata.getClientMetadata()); // even
splitfiles can have mime types!
@@ -321,6 +325,7 @@
SplitFileFetcher sf = new
SplitFileFetcher(metadata, rcb, parent, ctx,
decompressors, clientMetadata,
actx, recursionLevel);
+ sf.schedule();
// SplitFile will now run.
// Then it will return data to rcd.
// We are now out of the loop. Yay!
Modified: branches/async-client/src/freenet/client/async/SingleFileInserter.java
===================================================================
--- branches/async-client/src/freenet/client/async/SingleFileInserter.java
2006-01-25 14:55:11 UTC (rev 7919)
+++ branches/async-client/src/freenet/client/async/SingleFileInserter.java
2006-01-25 16:44:41 UTC (rev 7920)
@@ -13,6 +13,7 @@
import freenet.keys.SSKBlock;
import freenet.support.Bucket;
import freenet.support.BucketTools;
+import freenet.support.Logger;
import freenet.support.compress.CompressionOutputSizeException;
import freenet.support.compress.Compressor;
@@ -51,10 +52,9 @@
this.getCHKOnly = getCHKOnly;
if(!dontTellParent)
parent.setCurrentState(this);
- start();
}
- private void start() throws InserterException {
+ public void start() throws InserterException {
if((!ctx.dontCompress) && block.getData().size() >
COMPRESS_OFF_THREAD_LIMIT) {
// Run off thread
OffThreadCompressor otc = new OffThreadCompressor();
@@ -146,6 +146,7 @@
// Just insert it
SingleBlockInserter bi = new
SingleBlockInserter(parent, data, codecNumber, block.desiredURI, ctx, cb,
metadata, (int)block.getData().size(), -1, getCHKOnly);
bi.schedule();
+ cb.onTransition(this, bi);
return;
}
}
@@ -162,8 +163,9 @@
throw new
InserterException(InserterException.BUCKET_ERROR, e, null);
}
SingleBlockInserter metaPutter = new
SingleBlockInserter(parent, metadataBucket, (short) -1, block.desiredURI, ctx,
mcb, true, (int)origSize, -1, getCHKOnly);
- mcb.add(metaPutter);
+ mcb.addURIGenerator(metaPutter);
mcb.add(dataPutter);
+ cb.onTransition(this, mcb);
mcb.arm();
dataPutter.schedule();
metaPutter.schedule();
@@ -179,6 +181,7 @@
sh.sfi = sfi;
if(!dontTellParent)
parent.setCurrentState(sh);
+ cb.onTransition(this, sh);
sfi.start();
return;
}
@@ -190,20 +193,39 @@
*/
class SplitHandler implements SplitPutCompletionCallback,
ClientPutState {
- SplitFileInserter sfi;
- SingleFileInserter metadataPutter;
+ ClientPutState sfi;
+ ClientPutState metadataPutter;
boolean finished = false;
boolean splitInsertSuccess = false;
boolean metaInsertSuccess = false;
+
+ public synchronized void onTransition(ClientPutState oldState,
ClientPutState newState) {
+ if(oldState == sfi)
+ sfi = newState;
+ if(oldState == metadataPutter)
+ metadataPutter = newState;
+ }
- public synchronized void onSuccess(ClientPutState state) {
- if(finished) return;
- if(state == sfi)
- splitInsertSuccess = true;
- else if(state == metadataPutter)
- metaInsertSuccess = true;
- if(splitInsertSuccess && metaInsertSuccess)
- cb.onSuccess(this);
+ public void onSuccess(ClientPutState state) {
+ Logger.minor(this, "onSuccess("+state+")");
+ synchronized(this) {
+ if(finished) return;
+ if(state == sfi) {
+ Logger.minor(this, "Splitfile insert
succeeded");
+ splitInsertSuccess = true;
+ } else if(state == metadataPutter) {
+ Logger.minor(this, "Metadata insert
succeeded");
+ metaInsertSuccess = true;
+ } else {
+ Logger.error(this, "Unknown: "+state);
+ }
+ if(splitInsertSuccess && metaInsertSuccess) {
+ Logger.minor(this, "Both succeeded");
+ finished = true;
+ }
+ else return;
+ }
+ cb.onSuccess(this);
}
public synchronized void onFailure(InserterException e,
ClientPutState state) {
@@ -224,14 +246,15 @@
}
InsertBlock newBlock = new
InsertBlock(metadataBucket, null, block.desiredURI);
try {
- metadataPutter = new
SingleFileInserter(parent, this, newBlock, true, ctx, false, getCHKOnly, true);
+ metadataPutter = new
SingleFileInserter(parent, this, newBlock, true, ctx, false, getCHKOnly, false);
+ Logger.minor(this, "Putting metadata on
"+metadataPutter);
} catch (InserterException e) {
cb.onFailure(e, this);
return;
}
}
try {
- metadataPutter.start();
+ ((SingleFileInserter)metadataPutter).start();
} catch (InserterException e) {
fail(e);
return;
@@ -239,6 +262,7 @@
}
private synchronized void fail(InserterException e) {
+ Logger.minor(this, "Failing: "+e, e);
if(finished) return;
finished = true;
cb.onFailure(e, this);
@@ -249,8 +273,8 @@
}
public void onEncode(ClientKey key, ClientPutState state) {
- // Ignore
-
+ if(state == metadataPutter)
+ cb.onEncode(key, this);
}
public void cancel() {
Modified:
branches/async-client/src/freenet/client/async/SplitFileFetcherSegment.java
===================================================================
--- branches/async-client/src/freenet/client/async/SplitFileFetcherSegment.java
2006-01-25 14:55:11 UTC (rev 7919)
+++ branches/async-client/src/freenet/client/async/SplitFileFetcherSegment.java
2006-01-25 16:44:41 UTC (rev 7920)
@@ -39,6 +39,7 @@
final boolean nonFullBlocksAllowed;
/** Has the segment finished processing? Irreversible. */
private boolean finished;
+ private boolean decoded;
/** Bucket to store the data retrieved, after it has been decoded */
private Bucket decodedData;
/** Fetch context for block fetches */
@@ -155,6 +156,10 @@
}
private void startDecode() {
+ synchronized(this) {
+ if(decoded) return;
+ decoded = true;
+ }
Runnable r = new Decoder();
Thread t = new Thread(r, "Decoder for "+this);
t.setDaemon(true);
Modified: branches/async-client/src/freenet/client/async/SplitFileInserter.java
===================================================================
--- branches/async-client/src/freenet/client/async/SplitFileInserter.java
2006-01-25 14:55:11 UTC (rev 7919)
+++ branches/async-client/src/freenet/client/async/SplitFileInserter.java
2006-01-25 16:44:41 UTC (rev 7920)
@@ -130,6 +130,7 @@
}
if(allHaveURIs) {
+ Logger.minor(this, "Have URIs from all segments");
boolean missingURIs;
Metadata m = null;
synchronized(this) {
@@ -148,6 +149,7 @@
haveSentMetadata = true;
}
if(missingURIs) {
+ Logger.minor(this, "Missing URIs");
// Error
fail(new
InserterException(InserterException.INTERNAL_ERROR, "Missing URIs after
encoding", null));
return;
@@ -219,7 +221,9 @@
if(finished) return;
for(int i=0;i<segments.length;i++)
if(!segments[i].isFinished()) allGone = false;
- if(segment.getException().isFatal()) {
+
+ InserterException e = segment.getException();
+ if(e != null && e.isFatal()) {
cancel();
} else {
if(!allGone) return;
Modified:
branches/async-client/src/freenet/client/async/SplitFileInserterSegment.java
===================================================================
---
branches/async-client/src/freenet/client/async/SplitFileInserterSegment.java
2006-01-25 14:55:11 UTC (rev 7919)
+++
branches/async-client/src/freenet/client/async/SplitFileInserterSegment.java
2006-01-25 16:44:41 UTC (rev 7920)
@@ -48,9 +48,11 @@
}
public void start() throws InserterException {
- for(int i=0;i<dataBlockInserters.length;i++)
+ for(int i=0;i<dataBlockInserters.length;i++) {
dataBlockInserters[i] =
new SingleBlockInserter(parent.parent,
dataBlocks[i], (short)-1, FreenetURI.EMPTY_CHK_URI, blockInsertContext, this,
false, ClientCHKBlock.DATA_LENGTH, i, getCHKOnly);
+ dataBlockInserters[i].schedule();
+ }
if(splitfileAlgo == null) {
// Don't need to encode blocks
} else {
@@ -75,9 +77,11 @@
encoded = true;
parent.encodedSegment(this);
// Start the inserts
- for(int i=0;i<checkBlockInserters.length;i++)
+ for(int i=0;i<checkBlockInserters.length;i++) {
checkBlockInserters[i] =
new SingleBlockInserter(parent.parent,
checkBlocks[i], (short)-1, FreenetURI.EMPTY_CHK_URI, blockInsertContext, this,
false, ClientCHKBlock.DATA_LENGTH, i + dataBlocks.length, getCHKOnly);
+ checkBlockInserters[i].schedule();
+ }
} catch (IOException e) {
InserterException ex =
new
InserterException(InserterException.BUCKET_ERROR, e, null);
@@ -111,7 +115,7 @@
SingleBlockInserter sbi = (SingleBlockInserter)state;
int x = sbi.token;
synchronized(this) {
- if(x > dataBlocks.length) {
+ if(x >= dataBlocks.length) {
if(checkURIs[x-dataBlocks.length] != null) {
Logger.normal(this, "Got uri twice for
check block "+x+" on "+this);
return;
@@ -159,7 +163,7 @@
private boolean completed(int x) {
synchronized(this) {
- if(x > dataBlocks.length) {
+ if(x >= dataBlocks.length) {
if(checkBlockInserters[x-dataBlocks.length] ==
null) {
Logger.error(this, "Completed twice:
check block "+x+" on "+this);
return true;
@@ -174,8 +178,6 @@
}
blocksCompleted++;
if(blocksCompleted != dataBlockInserters.length +
checkBlockInserters.length) return true;
- if(finished) return true;
- finished = true;
return false;
}
}
@@ -223,4 +225,8 @@
}
parent.segmentFinished(this);
}
+
+ public void onTransition(ClientPutState oldState, ClientPutState
newState) {
+ Logger.error(this, "Illegal transition in
SplitFileInserterSegment: "+oldState+" -> "+newState);
+ }
}
Modified: branches/async-client/src/freenet/keys/Key.java
===================================================================
--- branches/async-client/src/freenet/keys/Key.java 2006-01-25 14:55:11 UTC
(rev 7919)
+++ branches/async-client/src/freenet/keys/Key.java 2006-01-25 16:44:41 UTC
(rev 7920)
@@ -136,7 +136,7 @@
// because compressing it improves its entropy.
if(sourceData.size() > MAX_LENGTH_BEFORE_COMPRESSION)
throw new KeyEncodeException("Too big");
- if(!dontCompress) {
+ if((!dontCompress) || alreadyCompressedCodec >= 0) {
byte[] cbuf = null;
if(alreadyCompressedCodec >= 0) {
if(sourceData.size() > MAX_COMPRESSED_DATA_LENGTH)
Modified: branches/async-client/src/freenet/node/TextModeClientInterface.java
===================================================================
--- branches/async-client/src/freenet/node/TextModeClientInterface.java
2006-01-25 14:55:11 UTC (rev 7919)
+++ branches/async-client/src/freenet/node/TextModeClientInterface.java
2006-01-25 16:44:41 UTC (rev 7920)
@@ -357,6 +357,8 @@
// Guess MIME type
String mimeType = DefaultMIMETypes.guessMIMEType(line);
System.out.println("Using MIME type: "+mimeType);
+ if(mimeType.equals(DefaultMIMETypes.DEFAULT_MIME_TYPE))
+ mimeType = ""; // don't need to override it
FileBucket fb = new FileBucket(f, true, false, false);
InsertBlock block = new InsertBlock(fb, new
ClientMetadata(mimeType), FreenetURI.EMPTY_CHK_URI);
