Author: toad
Date: 2007-02-16 19:54:01 +0000 (Fri, 16 Feb 2007)
New Revision: 11816
Modified:
trunk/freenet/src/freenet/client/Metadata.java
trunk/freenet/src/freenet/client/async/SplitFileFetcher.java
trunk/freenet/src/freenet/client/async/SplitFileFetcherSegment.java
trunk/freenet/src/freenet/client/async/SplitFileInserter.java
trunk/freenet/src/freenet/client/async/SplitFileInserterSegment.java
Log:
Drop support for irregular splitfiles.
Use ClientCHK instead of FreenetURI all over client/async/ splitfile code, and
in Metadata.
(Should save some memory and CPU)
Modified: trunk/freenet/src/freenet/client/Metadata.java
===================================================================
--- trunk/freenet/src/freenet/client/Metadata.java 2007-02-16 16:56:27 UTC
(rev 11815)
+++ trunk/freenet/src/freenet/client/Metadata.java 2007-02-16 19:54:01 UTC
(rev 11816)
@@ -59,15 +59,13 @@
boolean extraMetadata;
/** Keys stored in full (otherwise assumed to be CHKs) */
boolean fullKeys;
- /** Non-final splitfile chunks can be non-full */
- boolean splitUseLengths;
static final short FLAGS_SPLITFILE = 1;
static final short FLAGS_DBR = 2;
static final short FLAGS_NO_MIME = 4;
static final short FLAGS_COMPRESSED_MIME = 8;
static final short FLAGS_EXTRA_METADATA = 16;
static final short FLAGS_FULL_KEYS = 32;
- static final short FLAGS_SPLIT_USE_LENGTHS = 64;
+// static final short FLAGS_SPLIT_USE_LENGTHS = 64; FIXME not supported,
reassign to something else if we need a new flag
static final short FLAGS_COMPRESSED = 128;
/** Container archive type */
@@ -106,8 +104,8 @@
byte[] splitfileParams;
int splitfileBlocks;
int splitfileCheckBlocks;
- FreenetURI[] splitfileDataKeys;
- FreenetURI[] splitfileCheckKeys;
+ ClientCHK[] splitfileDataKeys;
+ ClientCHK[] splitfileCheckKeys;
// Manifests
/** Manifest entries by name */
@@ -189,7 +187,6 @@
compressedMIME = (flags & FLAGS_COMPRESSED_MIME) ==
FLAGS_COMPRESSED_MIME;
extraMetadata = (flags & FLAGS_EXTRA_METADATA) ==
FLAGS_EXTRA_METADATA;
fullKeys = (flags & FLAGS_FULL_KEYS) == FLAGS_FULL_KEYS;
- splitUseLengths = (flags & FLAGS_SPLIT_USE_LENGTHS) ==
FLAGS_SPLIT_USE_LENGTHS;
compressed = (flags & FLAGS_COMPRESSED) ==
FLAGS_COMPRESSED;
}
@@ -207,7 +204,7 @@
throw new MetadataParseException("Invalid real
content length "+dataLength);
if(dataLength == -1) {
- if(splitfile && !splitUseLengths)
+ if(splitfile)
throw new
MetadataParseException("Splitfile must have a real-length");
}
}
@@ -276,8 +273,8 @@
throw new MetadataParseException("Unknown
splitfile algorithm "+splitfileAlgorithm);
if((splitfileAlgorithm == SPLITFILE_NONREDUNDANT) &&
- !(fullKeys || splitUseLengths))
- throw new MetadataParseException("Non-redundant
splitfile invalid unless whacky");
+ !fullKeys)
+ throw new MetadataParseException("Non-redundant
splitfile invalid");
int paramsLength = dis.readInt();
if(paramsLength > MAX_SPLITFILE_PARAMS_LENGTH)
@@ -301,13 +298,13 @@
if(splitfileCheckBlocks > MAX_SPLITFILE_BLOCKS)
throw new MetadataParseException("Too many
splitfile check-blocks (soft limit to prevent memory DoS):
"+splitfileCheckBlocks);
- splitfileDataKeys = new FreenetURI[splitfileBlocks];
- splitfileCheckKeys = new
FreenetURI[splitfileCheckBlocks];
+ splitfileDataKeys = new ClientCHK[splitfileBlocks];
+ splitfileCheckKeys = new
ClientCHK[splitfileCheckBlocks];
for(int i=0;i<splitfileDataKeys.length;i++)
- if((splitfileDataKeys[i] = readKey(dis)) ==
null)
+ if((splitfileDataKeys[i] = readCHK(dis)) ==
null)
throw new MetadataParseException("Null
data key "+i);
for(int i=0;i<splitfileCheckKeys.length;i++)
- if((splitfileCheckKeys[i] = readKey(dis)) ==
null)
+ if((splitfileCheckKeys[i] = readCHK(dis)) ==
null)
throw new MetadataParseException("Null
check key: "+i);
}
@@ -542,13 +539,14 @@
noMIME = true;
}
simpleRedirectKey = uri;
- if(!(uri.getKeyType().equals("CHK") &&
((uri.getAllMetaStrings() == null) || (uri.getAllMetaStrings().length == 0))))
+ if(!(uri.getKeyType().equals("CHK") &&
!uri.hasMetaStrings()))
fullKeys = true;
} else
throw new IllegalArgumentException();
}
- public Metadata(short algo, FreenetURI[] dataURIs, FreenetURI[]
checkURIs, int segmentSize, int checkSegmentSize, ClientMetadata cm, long
dataLength, short compressionAlgo, boolean isMetadata, boolean
insertAsArchiveManifest) {
+ public Metadata(short algo, ClientCHK[] dataURIs, ClientCHK[]
checkURIs, int segmentSize, int checkSegmentSize,
+ ClientMetadata cm, long dataLength, short
compressionAlgo, boolean isMetadata, boolean insertAsArchiveManifest) {
if(isMetadata)
documentType = MULTI_LEVEL_METADATA;
else {
@@ -602,6 +600,13 @@
return baos.toByteArray();
}
+ private ClientCHK readCHK(DataInputStream dis) throws IOException,
MetadataParseException {
+ if(fullKeys) {
+ throw new MetadataParseException("fullKeys not
supported on a splitfile");
+ }
+ return ClientCHK.readRawBinaryKey(dis);
+ }
+
/**
* Read a key using the current settings.
* @throws IOException
@@ -636,6 +641,14 @@
}
}
+ private void writeCHK(DataOutputStream dos, ClientCHK chk) throws
IOException {
+ if(fullKeys) {
+ throw new UnsupportedOperationException("Full keys not
supported on splitfiles");
+ } else {
+ chk.writeRawBinaryKey(dos);
+ }
+ }
+
/** Is a manifest? */
public boolean isSimpleManifest() {
return documentType == SIMPLE_MANIFEST;
@@ -746,7 +759,6 @@
if(compressedMIME) flags |= FLAGS_COMPRESSED_MIME;
if(extraMetadata) flags |= FLAGS_EXTRA_METADATA;
if(fullKeys) flags |= FLAGS_FULL_KEYS;
- if(splitUseLengths) flags |= FLAGS_SPLIT_USE_LENGTHS;
if(compressionCodec >= 0) flags |= FLAGS_COMPRESSED;
dos.writeShort(flags);
}
@@ -798,9 +810,9 @@
dos.writeInt(splitfileBlocks);
dos.writeInt(splitfileCheckBlocks);
for(int i=0;i<splitfileBlocks;i++)
- writeKey(dos, splitfileDataKeys[i]);
+ writeCHK(dos, splitfileDataKeys[i]);
for(int i=0;i<splitfileCheckBlocks;i++)
- writeKey(dos, splitfileCheckKeys[i]);
+ writeCHK(dos, splitfileCheckKeys[i]);
}
if(documentType == SIMPLE_MANIFEST) {
@@ -860,11 +872,11 @@
return splitfileAlgorithm;
}
- public FreenetURI[] getSplitfileDataKeys() {
+ public ClientCHK[] getSplitfileDataKeys() {
return splitfileDataKeys;
}
- public FreenetURI[] getSplitfileCheckKeys() {
+ public ClientCHK[] getSplitfileCheckKeys() {
return splitfileCheckKeys;
}
@@ -872,10 +884,6 @@
return compressionCodec >= 0;
}
- public boolean splitUseLengths() {
- return splitUseLengths;
- }
-
public short getCompressionCodec() {
return compressionCodec;
}
Modified: trunk/freenet/src/freenet/client/async/SplitFileFetcher.java
===================================================================
--- trunk/freenet/src/freenet/client/async/SplitFileFetcher.java
2007-02-16 16:56:27 UTC (rev 11815)
+++ trunk/freenet/src/freenet/client/async/SplitFileFetcher.java
2007-02-16 19:54:01 UTC (rev 11816)
@@ -15,6 +15,7 @@
import freenet.client.Metadata;
import freenet.client.MetadataParseException;
import freenet.keys.CHKBlock;
+import freenet.keys.ClientCHK;
import freenet.keys.FreenetURI;
import freenet.support.Fields;
import freenet.support.Logger;
@@ -46,17 +47,15 @@
/** The detailed information on each segment */
final SplitFileFetcherSegment[] segments;
/** The splitfile data blocks. */
- final FreenetURI[] splitfileDataBlocks;
+ final ClientCHK[] splitfileDataBlocks;
/** The splitfile check blocks. */
- final FreenetURI[] splitfileCheckBlocks;
+ final ClientCHK[] splitfileCheckBlocks;
/** Maximum temporary length */
final long maxTempLength;
/** Have all segments finished? Access synchronized. */
private boolean allSegmentsFinished;
/** Override length. If this is positive, truncate the splitfile to
this length. */
private final long overrideLength;
- /** Accept non-full splitfile chunks? */
- private final boolean splitUseLengths;
/** Preferred bucket to return data in */
private final Bucket returnBucket;
private boolean finished;
@@ -91,7 +90,6 @@
finalLength = overrideLength;
}
- splitUseLengths = metadata.splitUseLengths();
if(splitfileType == Metadata.SPLITFILE_NONREDUNDANT) {
// Don't need to do much - just fetch everything and
piece it together.
blocksPerSegment = -1;
@@ -124,7 +122,7 @@
if(splitfileCheckBlocks.length > 0)
System.arraycopy(splitfileCheckBlocks, 0,
newSplitfileCheckBlocks, 0, splitfileCheckBlocks.length);
segments[0] = new
SplitFileFetcherSegment(splitfileType, newSplitfileDataBlocks,
newSplitfileCheckBlocks,
- this, archiveContext, fetchContext,
maxTempLength, splitUseLengths, recursionLevel);
+ this, archiveContext, fetchContext,
maxTempLength, recursionLevel);
} else {
int dataBlocksPtr = 0;
int checkBlocksPtr = 0;
@@ -140,7 +138,8 @@
System.arraycopy(splitfileCheckBlocks,
checkBlocksPtr, checkBlocks, 0, copyCheckBlocks);
dataBlocksPtr += copyDataBlocks;
checkBlocksPtr += copyCheckBlocks;
- segments[i] = new
SplitFileFetcherSegment(splitfileType, dataBlocks, checkBlocks, this,
archiveContext, fetchContext, maxTempLength, splitUseLengths, recursionLevel+1);
+ segments[i] = new
SplitFileFetcherSegment(splitfileType, dataBlocks, checkBlocks, this,
archiveContext,
+ fetchContext, maxTempLength,
recursionLevel+1);
}
}
this.token = token2;
Modified: trunk/freenet/src/freenet/client/async/SplitFileFetcherSegment.java
===================================================================
--- trunk/freenet/src/freenet/client/async/SplitFileFetcherSegment.java
2007-02-16 16:56:27 UTC (rev 11815)
+++ trunk/freenet/src/freenet/client/async/SplitFileFetcherSegment.java
2007-02-16 19:54:01 UTC (rev 11816)
@@ -41,7 +41,6 @@
final ArchiveContext archiveContext;
final FetcherContext fetcherContext;
final long maxBlockLength;
- final boolean nonFullBlocksAllowed;
/** Has the segment finished processing? Irreversible. */
private boolean finished;
private boolean startedDecode;
@@ -57,7 +56,7 @@
private int fetchedBlocks;
private final FailureCodeTracker errors;
- public SplitFileFetcherSegment(short splitfileType, FreenetURI[]
splitfileDataBlocks, FreenetURI[] splitfileCheckBlocks, SplitFileFetcher
fetcher, ArchiveContext archiveContext, FetcherContext fetchContext, long
maxTempLength, boolean splitUseLengths, int recursionLevel) throws
MetadataParseException, FetchException {
+ public SplitFileFetcherSegment(short splitfileType, FreenetURI[]
splitfileDataBlocks, FreenetURI[] splitfileCheckBlocks, SplitFileFetcher
fetcher, ArchiveContext archiveContext, FetcherContext fetchContext, long
maxTempLength, int recursionLevel) throws MetadataParseException,
FetchException {
logMINOR = Logger.shouldLog(Logger.MINOR, this);
this.parentFetcher = fetcher;
this.errors = new FailureCodeTracker(false);
@@ -81,16 +80,10 @@
}
for(int i=0;i<checkBuckets.length;i++)
checkBuckets[i] = new
MinimalSplitfileBlock(i+dataBuckets.length);
- nonFullBlocksAllowed = splitUseLengths;
this.fetcherContext = fetchContext;
maxBlockLength = maxTempLength;
- if(splitUseLengths) {
- blockFetchContext = new FetcherContext(fetcherContext,
FetcherContext.SPLITFILE_USE_LENGTHS_MASK, true);
- this.recursionLevel = recursionLevel + 1;
- } else {
- blockFetchContext = new FetcherContext(fetcherContext,
FetcherContext.SPLITFILE_DEFAULT_BLOCK_MASK, true);
- this.recursionLevel = 0;
- }
+ blockFetchContext = new FetcherContext(fetcherContext,
FetcherContext.SPLITFILE_DEFAULT_BLOCK_MASK, true);
+ this.recursionLevel = 0;
if(logMINOR) Logger.minor(this, "Created "+this+" for
"+parentFetcher);
for(int i=0;i<dataBlocks.length;i++)
if(dataBlocks[i] == null) throw new
NullPointerException("Null: data block "+i);
Modified: trunk/freenet/src/freenet/client/async/SplitFileInserter.java
===================================================================
--- trunk/freenet/src/freenet/client/async/SplitFileInserter.java
2007-02-16 16:56:27 UTC (rev 11815)
+++ trunk/freenet/src/freenet/client/async/SplitFileInserter.java
2007-02-16 19:54:01 UTC (rev 11816)
@@ -13,6 +13,7 @@
import freenet.client.InserterException;
import freenet.client.Metadata;
import freenet.keys.CHKBlock;
+import freenet.keys.ClientCHK;
import freenet.keys.FreenetURI;
import freenet.support.Logger;
import freenet.support.SimpleFieldSet;
@@ -271,8 +272,8 @@
Metadata m = null;
synchronized(this) {
// Create metadata
- FreenetURI[] dataURIs = getDataURIs();
- FreenetURI[] checkURIs = getCheckURIs();
+ ClientCHK[] dataURIs = getDataCHKs();
+ ClientCHK[] checkURIs = getCheckCHKs();
if(logMINOR) Logger.minor(this, "Data URIs:
"+dataURIs.length+", check URIs: "+checkURIs.length);
@@ -308,13 +309,13 @@
return false;
}
- private FreenetURI[] getCheckURIs() {
+ private ClientCHK[] getCheckCHKs() {
// Copy check blocks from each segment into a FreenetURI[].
- FreenetURI[] uris = new FreenetURI[countCheckBlocks];
+ ClientCHK[] uris = new ClientCHK[countCheckBlocks];
int x = 0;
for(int i=0;i<segments.length;i++) {
- FreenetURI[] segURIs = segments[i].getCheckURIs();
- if(x + segURIs.length > countCheckBlocks)
+ ClientCHK[] segURIs = segments[i].getCheckCHKs();
+ if(x + segURIs.length > countCheckBlocks)
throw new IllegalStateException("x="+x+",
segURIs="+segURIs.length+", countCheckBlocks="+countCheckBlocks);
System.arraycopy(segURIs, 0, uris, x, segURIs.length);
x += segURIs.length;
@@ -326,12 +327,12 @@
return uris;
}
- private FreenetURI[] getDataURIs() {
+ private ClientCHK[] getDataCHKs() {
// Copy check blocks from each segment into a FreenetURI[].
- FreenetURI[] uris = new FreenetURI[countDataBlocks];
+ ClientCHK[] uris = new ClientCHK[countDataBlocks];
int x = 0;
for(int i=0;i<segments.length;i++) {
- FreenetURI[] segURIs = segments[i].getDataURIs();
+ ClientCHK[] segURIs = segments[i].getDataCHKs();
if(x + segURIs.length > countDataBlocks)
throw new IllegalStateException("x="+x+",
segURIs="+segURIs.length+", countDataBlocks="+countDataBlocks);
System.arraycopy(segURIs, 0, uris, x, segURIs.length);
Modified: trunk/freenet/src/freenet/client/async/SplitFileInserterSegment.java
===================================================================
--- trunk/freenet/src/freenet/client/async/SplitFileInserterSegment.java
2007-02-16 16:56:27 UTC (rev 11815)
+++ trunk/freenet/src/freenet/client/async/SplitFileInserterSegment.java
2007-02-16 19:54:01 UTC (rev 11816)
@@ -10,6 +10,8 @@
import freenet.client.Metadata;
import freenet.keys.BaseClientKey;
import freenet.keys.CHKBlock;
+import freenet.keys.ClientCHK;
+import freenet.keys.ClientKey;
import freenet.keys.FreenetURI;
import freenet.support.Fields;
import freenet.support.Logger;
@@ -26,8 +28,8 @@
final FECCodec splitfileAlgo;
final Bucket[] dataBlocks;
final Bucket[] checkBlocks;
- final FreenetURI[] dataURIs;
- final FreenetURI[] checkURIs;
+ final ClientCHK[] dataURIs;
+ final ClientCHK[] checkURIs;
final SingleBlockInserter[] dataBlockInserters;
final SingleBlockInserter[] checkBlockInserters;
final InserterContext blockInsertContext;
@@ -52,8 +54,8 @@
this.dataBlocks = origDataBlocks;
int checkBlockCount = splitfileAlgo == null ? 0 :
splitfileAlgo.countCheckBlocks();
checkBlocks = new Bucket[checkBlockCount];
- checkURIs = new FreenetURI[checkBlockCount];
- dataURIs = new FreenetURI[origDataBlocks.length];
+ checkURIs = new ClientCHK[checkBlockCount];
+ dataURIs = new ClientCHK[origDataBlocks.length];
dataBlockInserters = new SingleBlockInserter[dataBlocks.length];
checkBlockInserters = new
SingleBlockInserter[checkBlocks.length];
parent.parent.addBlocks(dataURIs.length+checkURIs.length);
@@ -97,7 +99,7 @@
hasURIs = true;
dataBlocks = new Bucket[dataBlockCount];
- dataURIs = new FreenetURI[dataBlockCount];
+ dataURIs = new ClientCHK[dataBlockCount];
dataBlockInserters = new SingleBlockInserter[dataBlockCount];
// Check blocks first, because if there are missing check
blocks, we need
@@ -114,7 +116,7 @@
throw new ResumeException("Corrupt check blocks
count: "+e+" : "+tmp);
}
checkBlocks = new Bucket[checkBlockCount];
- checkURIs = new FreenetURI[checkBlockCount];
+ checkURIs = new ClientCHK[checkBlockCount];
checkBlockInserters = new
SingleBlockInserter[checkBlockCount];
for(int i=0;i<checkBlockCount;i++) {
String index = Integer.toString(i);
@@ -128,7 +130,7 @@
tmp = blockFS.get("URI");
if(tmp != null) {
try {
- checkURIs[i] = new
FreenetURI(tmp);
+ checkURIs[i] = (ClientCHK)
ClientKey.getBaseKey(new FreenetURI(tmp));
blocksGotURI++;
} catch (MalformedURLException e) {
throw new
ResumeException("Corrupt URI: "+e+" : "+tmp);
@@ -170,7 +172,7 @@
encoded = false;
splitfileAlgo = FECCodec.getCodec(splitfileAlgorithm,
dataBlockCount);
int checkBlocksCount = splitfileAlgo.countCheckBlocks();
- this.checkURIs = new FreenetURI[checkBlocksCount];
+ this.checkURIs = new ClientCHK[checkBlocksCount];
this.checkBlocks = new Bucket[checkBlocksCount];
this.checkBlockInserters = new
SingleBlockInserter[checkBlocksCount];
hasURIs = false;
@@ -183,7 +185,7 @@
tmp = blockFS.get("URI");
if(tmp != null) {
try {
- dataURIs[i] = new FreenetURI(tmp);
+ dataURIs[i] = (ClientCHK)
ClientKey.getBaseKey(new FreenetURI(tmp));
blocksGotURI++;
} catch (MalformedURLException e) {
throw new ResumeException("Corrupt URI:
"+e+" : "+tmp);
@@ -242,7 +244,7 @@
for(int i=0;i<dataBlocks.length;i++) {
SimpleFieldSet block = new SimpleFieldSet(false);
if(dataURIs[i] != null)
- block.putSingle("URI", dataURIs[i].toString());
+ block.putSingle("URI",
dataURIs[i].getURI().toString());
SingleBlockInserter sbi =
dataBlockInserters[i];
// If started, then sbi = null => block finished.
@@ -274,7 +276,7 @@
for(int i=0;i<checkBlocks.length;i++) {
SimpleFieldSet block = new SimpleFieldSet(false);
if(checkURIs[i] != null)
- block.putSingle("URI", checkURIs[i].toString());
+ block.putSingle("URI",
checkURIs[i].getURI().toString());
SingleBlockInserter sbi =
checkBlockInserters[i];
// If encoded, then sbi == null => block finished
@@ -425,22 +427,22 @@
parent.segmentFinished(this);
}
- public void onEncode(BaseClientKey key, ClientPutState state) {
+ public void onEncode(BaseClientKey k, ClientPutState state) {
+ ClientCHK key = (ClientCHK)k;
SingleBlockInserter sbi = (SingleBlockInserter)state;
int x = sbi.token;
- FreenetURI uri = key.getURI();
synchronized(this) {
if(finished) return;
if(x >= dataBlocks.length) {
if(checkURIs[x-dataBlocks.length] != null) {
return;
}
- checkURIs[x-dataBlocks.length] = uri;
+ checkURIs[x-dataBlocks.length] = key;
} else {
if(dataURIs[x] != null) {
return;
}
- dataURIs[x] = uri;
+ dataURIs[x] = key;
}
blocksGotURI++;
if(blocksGotURI != dataBlocks.length +
checkBlocks.length) return;
@@ -544,11 +546,11 @@
return dataBlocks.length;
}
- public FreenetURI[] getCheckURIs() {
+ public ClientCHK[] getCheckCHKs() {
return checkURIs;
}
- public FreenetURI[] getDataURIs() {
+ public ClientCHK[] getDataCHKs() {
return dataURIs;
}
