Author: toad
Date: 2005-11-17 23:56:37 +0000 (Thu, 17 Nov 2005)
New Revision: 7551
Modified:
trunk/freenet/build.xml
trunk/freenet/src/freenet/client/ArchiveStoreContext.java
trunk/freenet/src/freenet/client/BlockFetcher.java
trunk/freenet/src/freenet/client/Fetcher.java
trunk/freenet/src/freenet/client/HighLevelSimpleClientImpl.java
trunk/freenet/src/freenet/client/Segment.java
trunk/freenet/src/freenet/client/SplitFetcher.java
trunk/freenet/src/freenet/client/StandardOnionFECCodec.java
trunk/freenet/src/freenet/client/StdSplitfileBlock.java
trunk/freenet/src/freenet/node/TextModeClientInterface.java
trunk/freenet/src/freenet/node/Version.java
trunk/freenet/src/freenet/support/BucketTools.java
Log:
187: (mandatory)
Multi-segment redundant multi-level splitfiles work now!
Fix build.xml (we weren't including the mersenne twister code).
Show help on unknown command in TMCI.
Modified: trunk/freenet/build.xml
===================================================================
--- trunk/freenet/build.xml 2005-11-17 22:35:26 UTC (rev 7550)
+++ trunk/freenet/build.xml 2005-11-17 23:56:37 UTC (rev 7551)
@@ -21,7 +21,7 @@
<!-- FIXME: remove the debug and replace with optimize -->
<javac srcdir="${src}" destdir="${build}" debug="on" optimize="on"
source="1.4">
<!-- following a very temporary list of files to be build -->
- <include name="org/**.java"/>
+ <include name="org/**/*.java"/>
<include name="freenet/**/*.java"/>
<include name="net/i2p/util/*.java"/>
<exclude name="org/**/MersenneTwisterTest.java"/>
Modified: trunk/freenet/src/freenet/client/ArchiveStoreContext.java
===================================================================
--- trunk/freenet/src/freenet/client/ArchiveStoreContext.java 2005-11-17
22:35:26 UTC (rev 7550)
+++ trunk/freenet/src/freenet/client/ArchiveStoreContext.java 2005-11-17
23:56:37 UTC (rev 7551)
@@ -66,7 +66,7 @@
if(fetchContext == null) return null;
Fetcher fetcher = new Fetcher(key, fetchContext,
archiveContext);
- FetchResult result = fetcher.realRun(dm,
recursionLevel, key, dontEnterImplicitArchives);
+ FetchResult result = fetcher.realRun(dm,
recursionLevel, key, dontEnterImplicitArchives, fetchContext.localRequestOnly);
manager.extractToCache(key, archiveType, result.data,
archiveContext, this);
return manager.getCached(key, internalName);
}
Modified: trunk/freenet/src/freenet/client/BlockFetcher.java
===================================================================
--- trunk/freenet/src/freenet/client/BlockFetcher.java 2005-11-17 22:35:26 UTC
(rev 7550)
+++ trunk/freenet/src/freenet/client/BlockFetcher.java 2005-11-17 23:56:37 UTC
(rev 7551)
@@ -47,7 +47,7 @@
Fetcher f = new Fetcher(uri, this.segment.blockFetchContext);
try {
FetchResult fr = f.realRun(new ClientMetadata(),
segment.recursionLevel, uri,
- (!this.segment.nonFullBlocksAllowed) ||
dontEnterImplicitArchives);
+ (!this.segment.nonFullBlocksAllowed) ||
dontEnterImplicitArchives, segment.blockFetchContext.localRequestOnly ||
completedTries == 0);
actuallyFetched = true;
fetchedData = fr.data;
Logger.minor(this, "Fetched "+fetchedData.size()+"
bytes on "+this);
Modified: trunk/freenet/src/freenet/client/Fetcher.java
===================================================================
--- trunk/freenet/src/freenet/client/Fetcher.java 2005-11-17 22:35:26 UTC
(rev 7550)
+++ trunk/freenet/src/freenet/client/Fetcher.java 2005-11-17 23:56:37 UTC
(rev 7551)
@@ -63,7 +63,7 @@
for(int i=0;i<ctx.maxArchiveRestarts;i++) {
try {
ClientMetadata dm = new ClientMetadata();
- return realRun(dm, 0, origURI,
ctx.dontEnterImplicitArchives);
+ return realRun(dm, 0, origURI,
ctx.dontEnterImplicitArchives, ctx.localRequestOnly);
} catch (ArchiveRestartException e) {
archiveContext = new ArchiveContext();
continue;
@@ -91,7 +91,7 @@
* @throws ArchiveFailureException If we could not extract data from an
archive.
* @throws ArchiveRestartException
*/
- FetchResult realRun(ClientMetadata dm, int recursionLevel, FreenetURI
uri, boolean dontEnterImplicitArchives)
+ FetchResult realRun(ClientMetadata dm, int recursionLevel, FreenetURI
uri, boolean dontEnterImplicitArchives, boolean localOnly)
throws FetchException, MetadataParseException, ArchiveFailureException,
ArchiveRestartException {
Logger.minor(this, "Running fetch for: "+uri);
ClientKey key;
@@ -109,7 +109,7 @@
// Do the fetch
KeyBlock block;
try {
- block = ctx.client.getKey(key, ctx.localRequestOnly,
ctx.starterClient);
+ block = ctx.client.getKey(key, localOnly,
ctx.starterClient);
} catch (LowLevelGetException e) {
switch(e.code) {
case LowLevelGetException.DATA_NOT_FOUND:
@@ -166,7 +166,7 @@
ctx.eventProducer.produceEvent(new FetchedMetadataEvent());
- FetchResult result = runMetadata(dm, recursionLevel, key,
metaStrings, metadata, null, key.getURI(), dontEnterImplicitArchives);
+ FetchResult result = runMetadata(dm, recursionLevel, key,
metaStrings, metadata, null, key.getURI(), dontEnterImplicitArchives,
localOnly);
if(metaStrings.isEmpty()) return result;
// Still got some meta-strings
throw new FetchException(FetchException.HAS_MORE_METASTRINGS);
@@ -188,7 +188,7 @@
* @throws ArchiveRestartException
*/
private FetchResult runMetadata(ClientMetadata dm, int recursionLevel,
ClientKey key, LinkedList metaStrings,
- Metadata metadata, ArchiveHandler container, FreenetURI
thisKey, boolean dontEnterImplicitArchives)
+ Metadata metadata, ArchiveHandler container, FreenetURI
thisKey, boolean dontEnterImplicitArchives, boolean localOnly)
throws MetadataParseException, FetchException, ArchiveFailureException,
ArchiveRestartException {
if(metadata.isSimpleManifest()) {
@@ -200,7 +200,7 @@
metadata = metadata.getDocument(name);
thisKey = thisKey.pushMetaString(name);
}
- return runMetadata(dm, recursionLevel, key,
metaStrings, metadata, container, thisKey, dontEnterImplicitArchives);
+ return runMetadata(dm, recursionLevel, key,
metaStrings, metadata, container, thisKey, dontEnterImplicitArchives,
localOnly);
} else if(metadata.isArchiveManifest()) {
container = ctx.archiveManager.makeHandler(thisKey,
metadata.getArchiveType());
Bucket metadataBucket =
container.getMetadata(archiveContext, ctx, dm, recursionLevel, true);
@@ -209,7 +209,7 @@
} catch (IOException e) {
throw new
FetchException(FetchException.BUCKET_ERROR);
}
- return runMetadata(dm, recursionLevel+1, key,
metaStrings, metadata, container, thisKey, dontEnterImplicitArchives);
+ return runMetadata(dm, recursionLevel+1, key,
metaStrings, metadata, container, thisKey, dontEnterImplicitArchives,
localOnly);
} else if(metadata.isArchiveInternalRedirect()) {
if(container == null)
throw new
FetchException(FetchException.NOT_IN_ARCHIVE);
@@ -230,7 +230,7 @@
} catch (IOException e) {
throw new
FetchException(FetchException.BUCKET_ERROR);
}
- return runMetadata(dm,
recursionLevel+1, key, metaStrings, metadata, container, thisKey,
dontEnterImplicitArchives);
+ return runMetadata(dm,
recursionLevel+1, key, metaStrings, metadata, container, thisKey,
dontEnterImplicitArchives, localOnly);
}
Bucket result =
container.get(metadata.getZIPInternalName(), archiveContext, ctx, dm,
recursionLevel, true);
dm.mergeNoOverwrite(metadata.getClientMetadata());
@@ -239,13 +239,13 @@
} else if(metadata.isMultiLevelMetadata()) {
// Doesn't have to be a splitfile; could be from a ZIP
or a plain file.
metadata.setSimpleRedirect();
- FetchResult res = runMetadata(dm, recursionLevel, key,
metaStrings, metadata, container, thisKey, true);
+ FetchResult res = runMetadata(dm, recursionLevel, key,
metaStrings, metadata, container, thisKey, true, localOnly);
try {
metadata = Metadata.construct(res.data);
} catch (IOException e) {
throw new
FetchException(FetchException.BUCKET_ERROR);
}
- return runMetadata(dm, recursionLevel, key,
metaStrings, metadata, container, thisKey, dontEnterImplicitArchives);
+ return runMetadata(dm, recursionLevel, key,
metaStrings, metadata, container, thisKey, dontEnterImplicitArchives,
localOnly);
} else if(metadata.isSingleFileRedirect()) {
FreenetURI uri = metadata.getSingleTarget();
dm.mergeNoOverwrite(metadata.getClientMetadata());
@@ -267,10 +267,10 @@
} catch (IOException e) {
throw new
FetchException(FetchException.BUCKET_ERROR);
}
- return runMetadata(dm,
recursionLevel+1, key, metaStrings, metadata, container, thisKey,
dontEnterImplicitArchives);
+ return runMetadata(dm,
recursionLevel+1, key, metaStrings, metadata, container, thisKey,
dontEnterImplicitArchives, localOnly);
}
}
- FetchResult fr = realRun(dm, recursionLevel, uri,
dontEnterImplicitArchives);
+ FetchResult fr = realRun(dm, recursionLevel, uri,
dontEnterImplicitArchives, localOnly);
if(metadata.compressed) {
Compressor codec =
Compressor.getCompressionAlgorithmByMetadataID(metadata.compressionCodec);
Bucket data = fr.data;
@@ -300,7 +300,7 @@
} catch (IOException e) {
throw new
FetchException(FetchException.BUCKET_ERROR, e);
}
- return runMetadata(dm, recursionLevel+1, key,
metaStrings, metadata, container, thisKey, dontEnterImplicitArchives);
+ return runMetadata(dm, recursionLevel+1, key,
metaStrings, metadata, container, thisKey, dontEnterImplicitArchives,
localOnly);
}
FetcherContext newCtx;
Modified: trunk/freenet/src/freenet/client/HighLevelSimpleClientImpl.java
===================================================================
--- trunk/freenet/src/freenet/client/HighLevelSimpleClientImpl.java
2005-11-17 22:35:26 UTC (rev 7550)
+++ trunk/freenet/src/freenet/client/HighLevelSimpleClientImpl.java
2005-11-17 23:56:37 UTC (rev 7551)
@@ -50,7 +50,7 @@
static final int MAX_SPLITFILE_BLOCKS_PER_SEGMENT = 1024;
static final int MAX_SPLITFILE_CHECK_BLOCKS_PER_SEGMENT = 1536;
static final int SPLITFILE_BLOCKS_PER_SEGMENT = 128;
- static final int SPLITFILE_CHECK_BLOCKS_PER_SEGMENT = 192;
+ static final int SPLITFILE_CHECK_BLOCKS_PER_SEGMENT = 64;
public HighLevelSimpleClientImpl(SimpleLowLevelClient client,
ArchiveManager mgr, BucketFactory bf, RandomSource r, RequestStarterClient
requestStarterClient, RequestStarterClient insertStarterClient) {
Modified: trunk/freenet/src/freenet/client/Segment.java
===================================================================
--- trunk/freenet/src/freenet/client/Segment.java 2005-11-17 22:35:26 UTC
(rev 7550)
+++ trunk/freenet/src/freenet/client/Segment.java 2005-11-17 23:56:37 UTC
(rev 7551)
@@ -97,6 +97,7 @@
blockFetchContext = new FetcherContext(fetcherContext,
FetcherContext.SPLITFILE_DEFAULT_BLOCK_MASK);
this.recursionLevel = 0;
}
+ Logger.minor(this, "Created segment: data blocks:
"+dataBlocks.length+", check blocks: "+checkBlocks.length+" "+this);
}
/**
@@ -133,7 +134,7 @@
long len = decodedData.size();
if(truncateLength >= 0 && truncateLength < len)
len = truncateLength;
- BucketTools.copyTo(decodedData, os, truncateLength);
+ BucketTools.copyTo(decodedData, os, Math.min(truncateLength,
decodedData.size()));
return len;
}
@@ -175,6 +176,7 @@
*/
public void finished(SplitfileBlock[] succeeded, SplitfileBlock[]
failed, SplitfileBlock[] fatalErrors) {
+ parentFetcher.gotBlocks(this);
if(succeeded.length >= minFetched)
// Not finished yet, need to decode
successfulFetch();
@@ -189,7 +191,6 @@
* Successful fetch, do the decode, tell the parent, etc.
*/
private void successfulFetch() {
- parentFetcher.gotBlocks(this);
// Now decode
@@ -202,18 +203,21 @@
}
decodedData =
fetcherContext.bucketFactory.makeBucket(-1);
+ Logger.minor(this, "Copying data from data blocks");
OutputStream os = decodedData.getOutputStream();
for(int i=0;i<dataBlockStatus.length;i++) {
BlockFetcher status = dataBlockStatus[i];
Bucket data = status.fetchedData;
BucketTools.copyTo(data, os, Long.MAX_VALUE);
}
+ Logger.minor(this, "Copied data");
os.close();
// Must set finished BEFORE calling parentFetcher.
// Otherwise a race is possible that might result in it
not seeing our finishing.
finished = true;
parentFetcher.segmentFinished(this);
} catch (IOException e) {
+ Logger.minor(this, "Caught bucket error?: "+e, e);
finished = true;
failureException = new
FetchException(FetchException.BUCKET_ERROR);
parentFetcher.segmentFinished(this);
Modified: trunk/freenet/src/freenet/client/SplitFetcher.java
===================================================================
--- trunk/freenet/src/freenet/client/SplitFetcher.java 2005-11-17 22:35:26 UTC
(rev 7550)
+++ trunk/freenet/src/freenet/client/SplitFetcher.java 2005-11-17 23:56:37 UTC
(rev 7551)
@@ -211,16 +211,22 @@
}
public void gotBlocks(Segment segment) {
+ Logger.minor(this, "Got blocks for segment: "+segment);
synchronized(this) {
fetchingSegment = null;
+ notifyAll();
}
}
public void segmentFinished(Segment segment) {
+ Logger.minor(this, "Finished segment: "+segment);
synchronized(this) {
boolean allDone = true;
for(int i=0;i<segments.length;i++)
- if(!segments[i].isFinished()) allDone = false;
+ if(!segments[i].isFinished()) {
+ Logger.minor(this, "Segment
"+segments[i]+" is not finished");
+ allDone = false;
+ }
if(allDone) allSegmentsFinished = true;
notifyAll();
}
Modified: trunk/freenet/src/freenet/client/StandardOnionFECCodec.java
===================================================================
--- trunk/freenet/src/freenet/client/StandardOnionFECCodec.java 2005-11-17
22:35:26 UTC (rev 7550)
+++ trunk/freenet/src/freenet/client/StandardOnionFECCodec.java 2005-11-17
23:56:37 UTC (rev 7551)
@@ -129,7 +129,9 @@
encoder = DefaultFECCodeFactory.getDefault().createFECCode(k,n);
// revert to below if above causes JVM crashes
// Worst performance, but decode crashes
- decoder = new PureCode(k,n);
+ //decoder = new PureCode(k,n);
+ // Crashes are caused by bugs which cause to use 320/128 etc. -
n > 256, k < 256.
+ decoder = encoder;
}
private static Object runningDecodesSync = new Object();
@@ -263,7 +265,7 @@
}
} finally {
-
+
for (int i = 0; i < k; i++) {
if (writers[i] != null)
writers[i].close();
@@ -333,8 +335,6 @@
// Runtime.getRuntime().runFinalization();
long memUsedAtStart = Runtime.getRuntime().totalMemory() -
Runtime.getRuntime().freeMemory();
Logger.minor(this, "Memory in use at start: "+memUsedAtStart+"
max="+Runtime.getRuntime().maxMemory());
- System.err.println("************* Encoding " +
dataBlockStatus.length
- + " -> " + checkBlockStatus.length + "
*************");
Logger.minor(this, "Doing encode: " + dataBlockStatus.length
+ " data blocks, " + checkBlockStatus.length
+ " check blocks, block length " + blockLength
+ " with "
@@ -391,6 +391,8 @@
}
if (numberToEncode > 0) {
+ System.err.println("************* Encoding " +
dataBlockStatus.length
+ + " -> " + numberToEncode + "
*************");
// Do the (striped) encode
for (int offset = 0; offset < blockLength;
offset += STRIPE_SIZE) {
// Read the data in first
@@ -432,6 +434,8 @@
STRIPE_SIZE);
}
}
+ System.err.println("************* Encoded " +
dataBlockStatus.length
+ + " -> " + numberToEncode + "
*************");
}
} finally {
@@ -451,8 +455,6 @@
throw new NullPointerException();
checkBlockStatus[i].setData(data);
}
- System.err.println("************* Encoded " +
dataBlockStatus.length
- + " -> " + checkBlockStatus.length + "
*************");
}
private Bucket pad(Bucket oldData, int blockLength, BucketFactory bf,
int l) throws IOException {
Modified: trunk/freenet/src/freenet/client/StdSplitfileBlock.java
===================================================================
--- trunk/freenet/src/freenet/client/StdSplitfileBlock.java 2005-11-17
22:35:26 UTC (rev 7550)
+++ trunk/freenet/src/freenet/client/StdSplitfileBlock.java 2005-11-17
23:56:37 UTC (rev 7551)
@@ -30,6 +30,7 @@
}
public void setData(Bucket data) {
+ if(data == fetchedData) return;
fetchedData = data;
Logger.minor(this, "Set data: "+(data == null ? "(null)" :
(""+data.size())+ " on "+this), new Exception("debug"));
}
Modified: trunk/freenet/src/freenet/node/TextModeClientInterface.java
===================================================================
--- trunk/freenet/src/freenet/node/TextModeClientInterface.java 2005-11-17
22:35:26 UTC (rev 7550)
+++ trunk/freenet/src/freenet/node/TextModeClientInterface.java 2005-11-17
23:56:37 UTC (rev 7551)
@@ -64,6 +64,21 @@
* Read commands, run them
*/
public void run() {
+ printHeader();
+ // Read command, and data
+ BufferedReader reader = new BufferedReader(new
InputStreamReader(System.in));
+ while(true) {
+ try {
+ processLine(reader);
+ } catch (Throwable t) {
+ Logger.error(this, "Caught "+t, t);
+ System.out.println("Caught: "+t);
+ t.printStackTrace();
+ }
+ }
+ }
+
+ private void printHeader() {
System.out.println("Freenet 0.7 Trivial Node Test Interface");
System.out.println("---------------------------------------");
System.out.println();
@@ -87,20 +102,9 @@
// System.out.println("SAY:<text> - send text to the last
created/pushed stream");
System.out.println("STATUS - display some status information on the
node including its reference and connections.");
System.out.println("QUIT - exit the program");
- // Read command, and data
- BufferedReader reader = new BufferedReader(new
InputStreamReader(System.in));
- while(true) {
- try {
- processLine(reader);
- } catch (Throwable t) {
- Logger.error(this, "Caught "+t, t);
- System.out.println("Caught: "+t);
- t.printStackTrace();
- }
- }
- }
+ }
- /**
+ /**
* Process a single command.
* @throws IOException If we could not write the data to stdout.
*/
@@ -310,7 +314,7 @@
System.out.println("New name: "+key);
n.setName(key);
} else {
-
+ printHeader();
}
}
Modified: trunk/freenet/src/freenet/node/Version.java
===================================================================
--- trunk/freenet/src/freenet/node/Version.java 2005-11-17 22:35:26 UTC (rev
7550)
+++ trunk/freenet/src/freenet/node/Version.java 2005-11-17 23:56:37 UTC (rev
7551)
@@ -20,10 +20,10 @@
public static final String protocolVersion = "1.0";
/** The build number of the current revision */
- public static final int buildNumber = 186;
+ public static final int buildNumber = 187;
/** Oldest build of Fred we will talk to */
- public static final int lastGoodBuild = 185;
+ public static final int lastGoodBuild = 187;
/** The highest reported build of fred */
public static int highestSeenBuild = buildNumber;
Modified: trunk/freenet/src/freenet/support/BucketTools.java
===================================================================
--- trunk/freenet/src/freenet/support/BucketTools.java 2005-11-17 22:35:26 UTC
(rev 7550)
+++ trunk/freenet/src/freenet/support/BucketTools.java 2005-11-17 23:56:37 UTC
(rev 7551)
@@ -389,7 +389,7 @@
if(bytes <= 0) {
if(truncateLength == Long.MAX_VALUE)
break;
- throw new IOException("Could not move
required quantity of data: "+bytes);
+ throw new IOException("Could not move
required quantity of data: "+bytes+" (moved "+moved+" of "+truncateLength+")");
}
os.write(buf, 0, bytes);
moved += bytes;
