Author: toad
Date: 2005-11-11 21:55:47 +0000 (Fri, 11 Nov 2005)
New Revision: 7529
Modified:
trunk/freenet/src/freenet/client/BlockFetcher.java
trunk/freenet/src/freenet/client/BlockInserter.java
trunk/freenet/src/freenet/client/DefaultMIMETypes.java
trunk/freenet/src/freenet/client/FECCodec.java
trunk/freenet/src/freenet/client/FetchException.java
trunk/freenet/src/freenet/client/FetcherContext.java
trunk/freenet/src/freenet/client/FileInserter.java
trunk/freenet/src/freenet/client/HighLevelSimpleClientImpl.java
trunk/freenet/src/freenet/client/InsertBlock.java
trunk/freenet/src/freenet/client/InserterContext.java
trunk/freenet/src/freenet/client/InserterException.java
trunk/freenet/src/freenet/client/Metadata.java
trunk/freenet/src/freenet/client/RetryTracker.java
trunk/freenet/src/freenet/client/SplitFetcher.java
trunk/freenet/src/freenet/client/SplitInserter.java
trunk/freenet/src/freenet/client/StandardOnionFECCodec.java
trunk/freenet/src/freenet/client/StdSplitfileBlock.java
trunk/freenet/src/freenet/node/Version.java
trunk/freenet/src/freenet/support/Fields.java
Log:
166 (mandatory):
Fixed FEC splitfiles. I think. :)
Modified: trunk/freenet/src/freenet/client/BlockFetcher.java
===================================================================
--- trunk/freenet/src/freenet/client/BlockFetcher.java 2005-11-11 18:57:07 UTC
(rev 7528)
+++ trunk/freenet/src/freenet/client/BlockFetcher.java 2005-11-11 21:55:47 UTC
(rev 7529)
@@ -50,6 +50,7 @@
(!this.segment.nonFullBlocksAllowed) ||
dontEnterImplicitArchives);
actuallyFetched = true;
fetchedData = fr.data;
+ Logger.minor(this, "Fetched "+fetchedData.size()+"
bytes on "+this);
tracker.success(this);
} catch (MetadataParseException e) {
fatalError(e, FetchException.INVALID_METADATA);
@@ -95,7 +96,7 @@
}
private void fatalError(Throwable e, int code) {
- Logger.normal(this, "Giving up on block: "+this+": "+e);
+ Logger.error(this, "Giving up on block: "+this+": "+e, e);
tracker.fatalError(this, code);
}
Modified: trunk/freenet/src/freenet/client/BlockInserter.java
===================================================================
--- trunk/freenet/src/freenet/client/BlockInserter.java 2005-11-11 18:57:07 UTC
(rev 7528)
+++ trunk/freenet/src/freenet/client/BlockInserter.java 2005-11-11 21:55:47 UTC
(rev 7529)
@@ -28,10 +28,13 @@
this.ctx = ctx;
block = new InsertBlock(bucket, null, FreenetURI.EMPTY_CHK_URI);
this.getCHKOnly = getCHKOnly;
+ Logger.minor(this, "Created "+this);
}
public synchronized void setData(Bucket data) {
if(this.fetchedData != null) throw new
IllegalArgumentException("Cannot set data when already have data");
+ block.data = data;
+ super.setData(data);
}
public void kill() {
@@ -48,9 +51,11 @@
public void run() {
try {
+ if(fetchedData == null)
+ throw new NullPointerException();
realRun();
} catch (Throwable t) {
- Logger.error(this, "Caught "+t, t);
+ Logger.error(this, "Caught "+t+" on "+this, t);
fatalError(t, InserterException.INTERNAL_ERROR);
} finally {
completedTries++;
Modified: trunk/freenet/src/freenet/client/DefaultMIMETypes.java
===================================================================
--- trunk/freenet/src/freenet/client/DefaultMIMETypes.java 2005-11-11
18:57:07 UTC (rev 7528)
+++ trunk/freenet/src/freenet/client/DefaultMIMETypes.java 2005-11-11
21:55:47 UTC (rev 7529)
@@ -57,7 +57,6 @@
addMIMEType(number, type);
Short t = new Short(number);
if(extensions != null) {
- boolean first = true;
for(int i=0;i<extensions.length;i++) {
String ext = extensions[i].toLowerCase();
if(mimeTypesByExtension.containsKey(ext)) {
Modified: trunk/freenet/src/freenet/client/FECCodec.java
===================================================================
--- trunk/freenet/src/freenet/client/FECCodec.java 2005-11-11 18:57:07 UTC
(rev 7528)
+++ trunk/freenet/src/freenet/client/FECCodec.java 2005-11-11 21:55:47 UTC
(rev 7529)
@@ -13,22 +13,6 @@
*/
abstract class FECCodec {
- public static int getCodecMaxSegmentDataBlocks(short splitfileType) {
- if(splitfileType == Metadata.SPLITFILE_NONREDUNDANT)
- return -1;
- if(splitfileType == Metadata.SPLITFILE_ONION_STANDARD)
- return 128;
- throw new IllegalArgumentException();
- }
-
- public static int getCodecMaxSegmentCheckBlocks(short splitfileType) {
- if(splitfileType == Metadata.SPLITFILE_NONREDUNDANT)
- return -1;
- if(splitfileType == Metadata.SPLITFILE_ONION_STANDARD)
- return 64;
- throw new IllegalArgumentException();
- }
-
/**
* Get a codec where we know both the number of data blocks and the
number
* of check blocks, and the codec type. Normally for decoding.
Modified: trunk/freenet/src/freenet/client/FetchException.java
===================================================================
--- trunk/freenet/src/freenet/client/FetchException.java 2005-11-11
18:57:07 UTC (rev 7528)
+++ trunk/freenet/src/freenet/client/FetchException.java 2005-11-11
21:55:47 UTC (rev 7529)
@@ -96,6 +96,10 @@
return "Invalid URI";
case TOO_BIG:
return "Too big";
+ case TOO_BIG_METADATA:
+ return "Metadata too big";
+ case TOO_MANY_BLOCKS_PER_SEGMENT:
+ return "Too many blocks per segment";
default:
return "Unknown fetch error code: "+mode;
}
@@ -147,4 +151,6 @@
public static final int TOO_BIG = 21;
/** Metadata too big */
public static final int TOO_BIG_METADATA = 22;
+ /** Splitfile has too big segments */
+ public static final int TOO_MANY_BLOCKS_PER_SEGMENT = 23;
}
Modified: trunk/freenet/src/freenet/client/FetcherContext.java
===================================================================
--- trunk/freenet/src/freenet/client/FetcherContext.java 2005-11-11
18:57:07 UTC (rev 7528)
+++ trunk/freenet/src/freenet/client/FetcherContext.java 2005-11-11
21:55:47 UTC (rev 7529)
@@ -32,12 +32,16 @@
* Set by the splitfile metadata and the mask constructor, so we don't
need to pass it in. */
final boolean splitfileUseLengths;
final int maxMetadataSize;
+ final int maxDataBlocksPerSegment;
+ final int maxCheckBlocksPerSegment;
+
public FetcherContext(SimpleLowLevelClient client, long curMaxLength,
long curMaxTempLength, int maxMetadataSize, int
maxRecursionLevel, int maxArchiveRestarts,
boolean dontEnterImplicitArchives, int
maxSplitfileThreads,
int maxSplitfileBlockRetries, int
maxNonSplitfileRetries,
boolean allowSplitfiles, boolean followRedirects,
boolean localRequestOnly,
+ int maxDataBlocksPerSegment, int
maxCheckBlocksPerSegment,
RandomSource random, ArchiveManager archiveManager,
BucketFactory bucketFactory,
ClientEventProducer producer) {
this.client = client;
@@ -58,6 +62,8 @@
this.localRequestOnly = localRequestOnly;
this.splitfileUseLengths = false;
this.eventProducer = producer;
+ this.maxDataBlocksPerSegment = maxDataBlocksPerSegment;
+ this.maxCheckBlocksPerSegment = maxCheckBlocksPerSegment;
}
public FetcherContext(FetcherContext ctx, int maskID) {
@@ -80,6 +86,8 @@
this.localRequestOnly = ctx.localRequestOnly;
this.splitfileUseLengths = false;
this.eventProducer = ctx.eventProducer;
+ this.maxDataBlocksPerSegment = 0;
+ this.maxCheckBlocksPerSegment = 0;
} else if(maskID == SPLITFILE_DEFAULT_MASK) {
this.client = ctx.client;
this.maxOutputLength = ctx.maxOutputLength;
@@ -99,6 +107,8 @@
this.localRequestOnly = ctx.localRequestOnly;
this.splitfileUseLengths = false;
this.eventProducer = ctx.eventProducer;
+ this.maxDataBlocksPerSegment =
ctx.maxDataBlocksPerSegment;
+ this.maxCheckBlocksPerSegment =
ctx.maxCheckBlocksPerSegment;
} else if(maskID == SPLITFILE_USE_LENGTHS_MASK) {
this.client = ctx.client;
this.maxOutputLength = ctx.maxOutputLength;
@@ -118,6 +128,8 @@
this.localRequestOnly = ctx.localRequestOnly;
this.splitfileUseLengths = true;
this.eventProducer = ctx.eventProducer;
+ this.maxDataBlocksPerSegment =
ctx.maxDataBlocksPerSegment;
+ this.maxCheckBlocksPerSegment =
ctx.maxCheckBlocksPerSegment;
} else throw new IllegalArgumentException();
}
Modified: trunk/freenet/src/freenet/client/FileInserter.java
===================================================================
--- trunk/freenet/src/freenet/client/FileInserter.java 2005-11-11 18:57:07 UTC
(rev 7528)
+++ trunk/freenet/src/freenet/client/FileInserter.java 2005-11-11 21:55:47 UTC
(rev 7529)
@@ -33,6 +33,8 @@
* @throws InserterException
*/
public FreenetURI run(InsertBlock block, boolean metadata, boolean
getCHKOnly) throws InserterException {
+ if(block.data == null)
+ throw new NullPointerException();
if(!block.desiredURI.toString(false).equals("CHK@"))
throw new
InserterException(InserterException.INVALID_URI);
Modified: trunk/freenet/src/freenet/client/HighLevelSimpleClientImpl.java
===================================================================
--- trunk/freenet/src/freenet/client/HighLevelSimpleClientImpl.java
2005-11-11 18:57:07 UTC (rev 7528)
+++ trunk/freenet/src/freenet/client/HighLevelSimpleClientImpl.java
2005-11-11 21:55:47 UTC (rev 7529)
@@ -43,6 +43,12 @@
static final boolean LOCAL_REQUESTS_ONLY = false;
static final int SPLITFILE_INSERT_THREADS = 10;
static final int SPLITFILE_INSERT_RETRIES = 0;
+ // going by memory usage only; 4kB per stripe
+ static final int MAX_SPLITFILE_BLOCKS_PER_SEGMENT = 1024;
+ static final int MAX_SPLITFILE_CHECK_BLOCKS_PER_SEGMENT = 1536;
+ // ~ 70kB/sec encode, 16MB segments
+ static final int SPLITFILE_BLOCKS_PER_SEGMENT = 512;
+ static final int SPLITFILE_CHECK_BLOCKS_PER_SEGMENT = 768;
public HighLevelSimpleClientImpl(SimpleLowLevelClient client,
ArchiveManager mgr, BucketFactory bf, RandomSource r) {
@@ -74,13 +80,15 @@
MAX_RECURSION, MAX_ARCHIVE_RESTARTS,
DONT_ENTER_IMPLICIT_ARCHIVES,
SPLITFILE_THREADS, SPLITFILE_BLOCK_RETRIES,
NON_SPLITFILE_RETRIES,
FETCH_SPLITFILES, FOLLOW_REDIRECTS,
LOCAL_REQUESTS_ONLY,
+ MAX_SPLITFILE_BLOCKS_PER_SEGMENT,
MAX_SPLITFILE_CHECK_BLOCKS_PER_SEGMENT,
random, archiveManager, bucketFactory,
globalEventProducer);
Fetcher f = new Fetcher(uri, context);
return f.run();
}
public FreenetURI insert(InsertBlock insert, boolean getCHKOnly) throws
InserterException {
- InserterContext context = new InserterContext(client,
bucketFactory, random, SPLITFILE_INSERT_RETRIES, SPLITFILE_INSERT_THREADS,
globalEventProducer);
+ InserterContext context = new InserterContext(client,
bucketFactory, random, SPLITFILE_INSERT_RETRIES,
+ SPLITFILE_INSERT_THREADS,
SPLITFILE_BLOCKS_PER_SEGMENT, SPLITFILE_CHECK_BLOCKS_PER_SEGMENT,
globalEventProducer);
FileInserter i = new FileInserter(context);
return i.run(insert, false, getCHKOnly);
}
Modified: trunk/freenet/src/freenet/client/InsertBlock.java
===================================================================
--- trunk/freenet/src/freenet/client/InsertBlock.java 2005-11-11 18:57:07 UTC
(rev 7528)
+++ trunk/freenet/src/freenet/client/InsertBlock.java 2005-11-11 21:55:47 UTC
(rev 7529)
@@ -8,7 +8,7 @@
*/
public class InsertBlock {
- final Bucket data;
+ Bucket data;
final FreenetURI desiredURI;
final ClientMetadata clientMetadata;
Modified: trunk/freenet/src/freenet/client/InserterContext.java
===================================================================
--- trunk/freenet/src/freenet/client/InserterContext.java 2005-11-11
18:57:07 UTC (rev 7528)
+++ trunk/freenet/src/freenet/client/InserterContext.java 2005-11-11
21:55:47 UTC (rev 7529)
@@ -16,10 +16,13 @@
final short splitfileAlgorithm;
final int maxInsertBlockRetries;
final int maxSplitInsertThreads;
+ final int splitfileSegmentDataBlocks;
+ final int splitfileSegmentCheckBlocks;
final ClientEventProducer eventProducer;
public InserterContext(SimpleLowLevelClient client, BucketFactory bf,
RandomSource random,
- int maxRetries, int maxThreads, ClientEventProducer
eventProducer) {
+ int maxRetries, int maxThreads, int
splitfileSegmentDataBlocks, int splitfileSegmentCheckBlocks,
+ ClientEventProducer eventProducer) {
this.client = client;
this.bf = bf;
this.random = random;
@@ -28,6 +31,8 @@
this.maxInsertBlockRetries = maxRetries;
this.maxSplitInsertThreads = maxThreads;
this.eventProducer = eventProducer;
+ this.splitfileSegmentDataBlocks = splitfileSegmentDataBlocks;
+ this.splitfileSegmentCheckBlocks = splitfileSegmentCheckBlocks;
}
}
Modified: trunk/freenet/src/freenet/client/InserterException.java
===================================================================
--- trunk/freenet/src/freenet/client/InserterException.java 2005-11-11
18:57:07 UTC (rev 7528)
+++ trunk/freenet/src/freenet/client/InserterException.java 2005-11-11
21:55:47 UTC (rev 7529)
@@ -19,14 +19,14 @@
public InserterException(int m, String msg) {
super(getMessage(m)+": "+msg);
mode = m;
- Logger.minor(this, "Creating InserterException:
"+getMessage(mode)+": "+msg);
+ Logger.minor(this, "Creating InserterException:
"+getMessage(mode)+": "+msg, this);
errorCodes = null;
}
public InserterException(int m) {
super(getMessage(m));
mode = m;
- Logger.minor(this, "Creating InserterException:
"+getMessage(mode));
+ Logger.minor(this, "Creating InserterException:
"+getMessage(mode), this);
errorCodes = null;
}
@@ -41,7 +41,7 @@
public InserterException(int mode, FailureCodeTracker errorCodes) {
super(getMessage(mode));
this.mode = mode;
- Logger.minor(this, "Creating InserterException:
"+getMessage(mode));
+ Logger.minor(this, "Creating InserterException:
"+getMessage(mode), this);
this.errorCodes = errorCodes;
}
Modified: trunk/freenet/src/freenet/client/Metadata.java
===================================================================
--- trunk/freenet/src/freenet/client/Metadata.java 2005-11-11 18:57:07 UTC
(rev 7528)
+++ trunk/freenet/src/freenet/client/Metadata.java 2005-11-11 21:55:47 UTC
(rev 7529)
@@ -13,6 +13,7 @@
import freenet.keys.ClientKey;
import freenet.keys.FreenetURI;
import freenet.support.Bucket;
+import freenet.support.Fields;
import freenet.support.Logger;
@@ -300,7 +301,7 @@
throw new IllegalArgumentException();
}
- public Metadata(short algo, FreenetURI[] dataURIs, FreenetURI[]
checkURIs, ClientMetadata cm, long dataLength, short compressionAlgo) {
+ public Metadata(short algo, FreenetURI[] dataURIs, FreenetURI[]
checkURIs, int segmentSize, int checkSegmentSize, ClientMetadata cm, long
dataLength, short compressionAlgo) {
documentType = SIMPLE_REDIRECT;
splitfile = true;
splitfileAlgorithm = algo;
@@ -315,6 +316,7 @@
setMIMEType(cm.getMIMEType());
else
setMIMEType(DefaultMIMETypes.DEFAULT_MIME_TYPE);
+ splitfileParams = Fields.intsToBytes(new int[] { segmentSize,
checkSegmentSize } );
}
/**
Modified: trunk/freenet/src/freenet/client/RetryTracker.java
===================================================================
--- trunk/freenet/src/freenet/client/RetryTracker.java 2005-11-11 18:57:07 UTC
(rev 7528)
+++ trunk/freenet/src/freenet/client/RetryTracker.java 2005-11-11 21:55:47 UTC
(rev 7529)
@@ -173,7 +173,6 @@
* we have run out of retries.
*/
public synchronized void nonfatalError(SplitfileBlock block, int
reasonCode) {
- if(killed) return;
nonfatalErrors.inc(reasonCode);
runningBlocks.remove(block);
int levelNumber = block.getRetryCount();
@@ -194,7 +193,6 @@
* @param reasonCode A client-specific code indicating the type of
failure.
*/
public synchronized void fatalError(SplitfileBlock block, int
reasonCode) {
- if(killed) return;
fatalErrors.inc(reasonCode);
runningBlocks.remove(block);
failedBlocksFatalErrors.add(block);
@@ -213,6 +211,7 @@
Logger.minor(this, "Only block running:
"+runningBlocks.toArray()[0]);
if((succeededBlocks.size() >= targetSuccesses)
|| (runningBlocks.isEmpty() && levels.isEmpty()
&& finishOnEmpty)) {
+ killed = true;
Logger.minor(this, "Finishing");
SplitfileBlock[] running = runningBlocks();
for(int i=0;i<running.length;i++) {
Modified: trunk/freenet/src/freenet/client/SplitFetcher.java
===================================================================
--- trunk/freenet/src/freenet/client/SplitFetcher.java 2005-11-11 18:57:07 UTC
(rev 7528)
+++ trunk/freenet/src/freenet/client/SplitFetcher.java 2005-11-11 21:55:47 UTC
(rev 7529)
@@ -10,6 +10,8 @@
import freenet.keys.FreenetURI;
import freenet.keys.NodeCHK;
import freenet.support.Bucket;
+import freenet.support.Fields;
+import freenet.support.Logger;
/**
* Class to fetch a splitfile.
@@ -55,7 +57,7 @@
/** Accept non-full splitfile chunks? */
private boolean splitUseLengths;
- public SplitFetcher(Metadata metadata, ArchiveContext archiveContext,
FetcherContext ctx, int recursionLevel) throws MetadataParseException {
+ public SplitFetcher(Metadata metadata, ArchiveContext archiveContext,
FetcherContext ctx, int recursionLevel) throws MetadataParseException,
FetchException {
actx = archiveContext;
fctx = ctx;
overrideLength = metadata.dataLength;
@@ -71,13 +73,20 @@
checkBlocksPerSegment = -1;
segmentCount = 1;
} else if(splitfileType == Metadata.SPLITFILE_ONION_STANDARD) {
- blocksPerSegment = 128;
- checkBlocksPerSegment = 64;
+ byte[] params = metadata.splitfileParams;
+ if(params == null || params.length < 8)
+ throw new MetadataParseException("No splitfile
params");
+ blocksPerSegment = Fields.bytesToInt(params, 0);
+ checkBlocksPerSegment = Fields.bytesToInt(params, 4);
+ if(blocksPerSegment > ctx.maxDataBlocksPerSegment
+ || checkBlocksPerSegment >
ctx.maxCheckBlocksPerSegment)
+ throw new
FetchException(FetchException.TOO_MANY_BLOCKS_PER_SEGMENT, "Too many blocks per
segment: "+blocksPerSegment+" data, "+checkBlocksPerSegment+" check");
segmentCount = (splitfileDataBlocks.length /
blocksPerSegment) +
(splitfileDataBlocks.length % blocksPerSegment
== 0 ? 0 : 1);
// Onion, 128/192.
// Will be segmented.
} else throw new MetadataParseException("Unknown splitfile
format: "+splitfileType);
+ Logger.minor(this, "Algorithm: "+splitfileType+", blocks per
segment: "+blocksPerSegment+", check blocks per segment:
"+checkBlocksPerSegment+", segments: "+segmentCount);
segments = new Segment[segmentCount]; // initially null on all
entries
if(segmentCount == 1) {
segments[0] = new Segment(splitfileType,
splitfileDataBlocks, splitfileCheckBlocks, this, archiveContext, ctx,
maxTempLength, splitUseLengths, recursionLevel+1);
@@ -102,6 +111,7 @@
unstartedSegments = new Vector();
for(int i=0;i<segments.length;i++)
unstartedSegments.add(segments[i]);
+ Logger.minor(this, "Segments: "+unstartedSegments.size());
}
/**
@@ -151,8 +161,8 @@
synchronized(unstartedSegments) {
if(unstartedSegments.isEmpty()) return null;
int x = fctx.random.nextInt(unstartedSegments.size());
- Segment s = (Segment) unstartedSegments.get(x);
- unstartedSegments.remove(x);
+ Logger.minor(this, "Starting segment "+x+" of
"+unstartedSegments.size());
+ Segment s = (Segment) unstartedSegments.remove(x);
return s;
}
}
Modified: trunk/freenet/src/freenet/client/SplitInserter.java
===================================================================
--- trunk/freenet/src/freenet/client/SplitInserter.java 2005-11-11 18:57:07 UTC
(rev 7528)
+++ trunk/freenet/src/freenet/client/SplitInserter.java 2005-11-11 21:55:47 UTC
(rev 7529)
@@ -49,8 +49,8 @@
this.splitfileAlgorithm = splitfileAlgorithm;
this.ctx = ctx;
this.dataLength = data.size();
- segmentSize =
FECCodec.getCodecMaxSegmentDataBlocks(splitfileAlgorithm);
- checkSegmentSize =
FECCodec.getCodecMaxSegmentCheckBlocks(splitfileAlgorithm);
+ segmentSize = ctx.splitfileSegmentDataBlocks;
+ checkSegmentSize = splitfileAlgorithm ==
Metadata.SPLITFILE_NONREDUNDANT ? 0 : ctx.splitfileSegmentCheckBlocks;
tracker = new RetryTracker(ctx.maxInsertBlockRetries,
Integer.MAX_VALUE, ctx.random, ctx.maxSplitInsertThreads, true, this);
try {
splitIntoBlocks();
@@ -83,6 +83,7 @@
} catch (Throwable t) {
Logger.error(this, "Caught "+t, t);
tracker.kill();
+ if(t instanceof InserterException) throw
(InserterException)t;
throw new
InserterException(InserterException.INTERNAL_ERROR, t.toString());
}
}
@@ -113,7 +114,7 @@
if(!missingURIs) {
- Metadata metadata = new Metadata(splitfileAlgorithm,
dataURIs, checkURIs, clientMetadata, dataLength, compressionCodec);
+ Metadata metadata = new Metadata(splitfileAlgorithm,
dataURIs, checkURIs, segmentSize, checkSegmentSize, clientMetadata, dataLength,
compressionCodec);
Bucket mbucket;
try {
Modified: trunk/freenet/src/freenet/client/StandardOnionFECCodec.java
===================================================================
--- trunk/freenet/src/freenet/client/StandardOnionFECCodec.java 2005-11-11
18:57:07 UTC (rev 7528)
+++ trunk/freenet/src/freenet/client/StandardOnionFECCodec.java 2005-11-11
21:55:47 UTC (rev 7529)
@@ -108,94 +108,103 @@
}
public void realDecode(SplitfileBlock[] dataBlockStatus,
SplitfileBlock[] checkBlockStatus, int blockLength, BucketFactory bf) throws
IOException {
- Logger.minor(this, "Doing decode: "+dataBlockStatus.length+"
data blocks, "+checkBlockStatus.length+" check blocks, block length
"+blockLength+" with "+this);
+ Logger.minor(this, "Doing decode: "+dataBlockStatus.length+"
data blocks, "+checkBlockStatus.length+" check blocks, block length
"+blockLength+" with "+this, new Exception("debug"));
if(dataBlockStatus.length + checkBlockStatus.length != n)
throw new IllegalArgumentException();
if(dataBlockStatus.length != k)
throw new IllegalArgumentException();
Buffer[] packets = new Buffer[k];
Bucket[] buckets = new Bucket[n];
- DataInputStream[] readers = new DataInputStream[k];
+ DataInputStream[] readers = new DataInputStream[n];
OutputStream[] writers = new OutputStream[k];
- int[] toDecode = new int[n-k];
int numberToDecode = 0; // can be less than n-k
- byte[] realBuffer = new byte[n * STRIPE_SIZE];
+ byte[] realBuffer = new byte[k * STRIPE_SIZE];
- for(int i=0;i<n;i++)
+ int[] packetIndexes = new int[k];
+ for(int i=0;i<packetIndexes.length;i++)
+ packetIndexes[i] = -1;
+
+ int idx = 0;
+
+ for(int i=0;i<k;i++)
packets[i] = new Buffer(realBuffer, i*STRIPE_SIZE,
STRIPE_SIZE);
for(int i=0;i<dataBlockStatus.length;i++) {
buckets[i] = dataBlockStatus[i].getData();
- long sz = buckets[i].size();
- if(sz < blockLength) {
- if(i != dataBlockStatus.length-1)
- throw new IllegalArgumentException("All
buckets except the last must be the full size");
- if(sz < blockLength)
- buckets[i] = pad(buckets[i],
blockLength, bf, (int) sz);
- else throw new IllegalArgumentException("Too
big: "+sz+" bigger than "+blockLength);
- }
if(buckets[i] == null) {
buckets[i] = bf.makeBucket(blockLength);
writers[i] = buckets[i].getOutputStream();
+ Logger.minor(this, "writers["+i+"] != null");
readers[i] = null;
- toDecode[numberToDecode++] = i;
+ numberToDecode++;
} else {
+ long sz = buckets[i].size();
+ if(sz < blockLength) {
+ if(i != dataBlockStatus.length-1)
+ throw new
IllegalArgumentException("All buckets except the last must be the full size but
data bucket "+i+" of "+dataBlockStatus.length+" ("+dataBlockStatus[i]+") is
"+sz+" not "+blockLength);
+ if(sz < blockLength)
+ buckets[i] = pad(buckets[i],
blockLength, bf, (int) sz);
+ else throw new
IllegalArgumentException("Too big: "+sz+" bigger than "+blockLength);
+ }
+ Logger.minor(this, "writers["+i+"] = null
(already filled)");
writers[i] = null;
readers[i] = new
DataInputStream(buckets[i].getInputStream());
+ packetIndexes[idx++] = i;
}
}
for(int i=0;i<checkBlockStatus.length;i++) {
buckets[i+k] = checkBlockStatus[i].getData();
if(buckets[i+k] == null) {
- buckets[i+k] = bf.makeBucket(blockLength);
- writers[i+k] = buckets[i+k].getOutputStream();
readers[i+k] = null;
- toDecode[numberToDecode++] = i+k;
} else {
- writers[i+k] = null;
readers[i+k] = new
DataInputStream(buckets[i+k].getInputStream());
+ if(idx < k)
+ packetIndexes[idx++] = i+k;
}
}
- if(numberToDecode != toDecode.length) {
- int[] newToDecode = new int[numberToDecode];
- System.arraycopy(toDecode, 0, newToDecode, 0,
numberToDecode);
- toDecode = newToDecode;
- }
-
+ if(idx < k)
+ throw new IllegalArgumentException("Must have at least
k packets");
+
+ for(int i=0;i<packetIndexes.length;i++)
+ Logger.minor(this, "["+i+"] = "+packetIndexes[i]);
+
if(numberToDecode > 0) {
// Do the (striped) decode
for(int
offset=0;offset<blockLength;offset+=STRIPE_SIZE) {
// Read the data in first
- for(int i=0;i<n;i++) {
- if(readers[i] != null) {
-
readers[i].readFully(realBuffer, i*STRIPE_SIZE, STRIPE_SIZE);
- }
+ for(int i=0;i<k;i++) {
+ int x = packetIndexes[i];
+ readers[x].readFully(realBuffer,
i*STRIPE_SIZE, STRIPE_SIZE);
}
// Do the decode
// Not shuffled
- code.decode(packets, toDecode);
+ int[] disposableIndexes = new
int[packetIndexes.length];
+ System.arraycopy(packetIndexes, 0,
disposableIndexes, 0, packetIndexes.length);
+ code.decode(packets, disposableIndexes);
// packets now contains an array of decoded
blocks, in order
// Write the data out
- for(int i=0;i<n;i++) {
+ for(int i=0;i<k;i++) {
if(writers[i] != null)
writers[i].write(realBuffer,
i*STRIPE_SIZE, STRIPE_SIZE);
}
}
}
+ for(int i=0;i<k;i++) {
+ if(writers[i] != null) writers[i].close();
+ }
for(int i=0;i<n;i++) {
- if(writers[i] != null) writers[i].close();
if(readers[i] != null) readers[i].close();
}
// Set new buckets only after have a successful decode.
// Note that the last data bucket will be overwritten padded.
for(int i=0;i<dataBlockStatus.length;i++) {
- dataBlockStatus[i].setData(buckets[i]);
+ Bucket data = buckets[i];
+ if(data.size() != blockLength)
+ throw new IllegalStateException("Block "+i+":
"+data+" : "+dataBlockStatus[i]+" length "+data.size());
+ dataBlockStatus[i].setData(data);
}
- for(int i=0;i<checkBlockStatus.length;i++) {
- checkBlockStatus[i].setData(buckets[i+k]);
- }
}
public void encode(SplitfileBlock[] dataBlockStatus, SplitfileBlock[]
checkBlockStatus, int blockLength, BucketFactory bf) throws IOException {
@@ -226,7 +235,7 @@
* Do the actual encode.
*/
private void realEncode(SplitfileBlock[] dataBlockStatus,
SplitfileBlock[] checkBlockStatus, int blockLength, BucketFactory bf) throws
IOException {
- System.err.println("************* Encoding
"+dataBlockStatus.length+" -> "+checkBlockStatus.length+"... *************");
+ System.err.println("************* Encoding
"+dataBlockStatus.length+" -> "+checkBlockStatus.length+" *************");
Logger.minor(this, "Doing encode: "+dataBlockStatus.length+"
data blocks, "+checkBlockStatus.length+" check blocks, block length
"+blockLength+" with "+this);
if(dataBlockStatus.length + checkBlockStatus.length != n)
throw new IllegalArgumentException();
@@ -298,9 +307,11 @@
if(writers[i] != null) writers[i].close();
// Set new buckets only after have a successful decode.
for(int i=0;i<checkBlockStatus.length;i++) {
- checkBlockStatus[i].setData(buckets[i+k]);
+ Bucket data = buckets[i+k];
+ if(data == null) throw new NullPointerException();
+ checkBlockStatus[i].setData(data);
}
- System.err.println("************* Encoded
"+dataBlockStatus.length+" -> "+checkBlockStatus.length+"... *************");
+ System.err.println("************* Encoded
"+dataBlockStatus.length+" -> "+checkBlockStatus.length+" *************");
}
private Bucket pad(Bucket oldData, int blockLength, BucketFactory bf,
int l) throws IOException {
Modified: trunk/freenet/src/freenet/client/StdSplitfileBlock.java
===================================================================
--- trunk/freenet/src/freenet/client/StdSplitfileBlock.java 2005-11-11
18:57:07 UTC (rev 7528)
+++ trunk/freenet/src/freenet/client/StdSplitfileBlock.java 2005-11-11
21:55:47 UTC (rev 7529)
@@ -31,6 +31,7 @@
public void setData(Bucket data) {
fetchedData = data;
+ Logger.minor(this, "Set data: "+(data == null ? "(null)" :
(""+data.size())+ " on "+this), new Exception("debug"));
}
public void start() {
Modified: trunk/freenet/src/freenet/node/Version.java
===================================================================
--- trunk/freenet/src/freenet/node/Version.java 2005-11-11 18:57:07 UTC (rev
7528)
+++ trunk/freenet/src/freenet/node/Version.java 2005-11-11 21:55:47 UTC (rev
7529)
@@ -20,10 +20,10 @@
public static final String protocolVersion = "1.0";
/** The build number of the current revision */
- public static final int buildNumber = 164;
+ public static final int buildNumber = 166;
/** Oldest build of Fred we will talk to */
- public static final int lastGoodBuild = 164;
+ public static final int lastGoodBuild = 166;
/** The highest reported build of fred */
public static int highestSeenBuild = buildNumber;
Modified: trunk/freenet/src/freenet/support/Fields.java
===================================================================
--- trunk/freenet/src/freenet/support/Fields.java 2005-11-11 18:57:07 UTC
(rev 7528)
+++ trunk/freenet/src/freenet/support/Fields.java 2005-11-11 21:55:47 UTC
(rev 7529)
@@ -510,4 +510,16 @@
return buf;
}
+ public static byte[] intsToBytes(int[] ints) {
+ byte[] buf = new byte[ints.length * 8];
+ for(int i=0;i<ints.length;i++) {
+ long x = ints[i];
+ for(int j=0;j<4;j++) {
+ buf[i*4+j] = (byte)x;
+ x >>= 8;
+ }
+ }
+ return buf;
+ }
+
}
