Author: toad
Date: 2005-11-07 23:51:07 +0000 (Mon, 07 Nov 2005)
New Revision: 7495
Added:
trunk/freenet/src/freenet/client/Compressor.java
trunk/freenet/src/freenet/client/FileInserter.java
trunk/freenet/src/freenet/client/GzipCompressor.java
trunk/freenet/src/freenet/client/InserterContext.java
trunk/freenet/src/freenet/client/InserterException.java
Modified:
trunk/freenet/src/freenet/client/ClientMetadata.java
trunk/freenet/src/freenet/client/FetchException.java
trunk/freenet/src/freenet/client/HighLevelSimpleClientImpl.java
trunk/freenet/src/freenet/client/InsertBlock.java
trunk/freenet/src/freenet/client/Metadata.java
trunk/freenet/src/freenet/keys/CHKBlock.java
trunk/freenet/src/freenet/keys/CHKEncodeException.java
trunk/freenet/src/freenet/keys/ClientCHKBlock.java
trunk/freenet/src/freenet/keys/FreenetURI.java
Log:
Lots of work on inserts inc. splitfiles & compression.
Modified: trunk/freenet/src/freenet/client/ClientMetadata.java
===================================================================
--- trunk/freenet/src/freenet/client/ClientMetadata.java 2005-11-07
20:17:23 UTC (rev 7494)
+++ trunk/freenet/src/freenet/client/ClientMetadata.java 2005-11-07
23:51:07 UTC (rev 7495)
@@ -33,4 +33,8 @@
if(mimeType == null || mimeType.equals(""))
mimeType = clientMetadata.mimeType;
}
+
+ public boolean isTrivial() {
+ return (mimeType == null ||
mimeType.equals(DefaultMIMETypes.DEFAULT_MIME_TYPE));
+ }
}
Added: trunk/freenet/src/freenet/client/Compressor.java
===================================================================
--- trunk/freenet/src/freenet/client/Compressor.java 2005-11-07 20:17:23 UTC
(rev 7494)
+++ trunk/freenet/src/freenet/client/Compressor.java 2005-11-07 23:51:07 UTC
(rev 7495)
@@ -0,0 +1,22 @@
+package freenet.client;
+
+import java.io.IOException;
+
+import freenet.support.Bucket;
+import freenet.support.BucketFactory;
+
+/**
+ * A data compressor. Contains methods to get all data compressors.
+ * This is for single-file compression (gzip, bzip2) as opposed to archives.
+ */
+public abstract class Compressor {
+
+ public static Compressor gzip = new GzipCompressor();
+
+ public abstract Bucket compress(Bucket data, BucketFactory bf) throws
IOException;
+
+ public int codecNumberForMetadata() {
+ return Metadata.COMPRESS_GZIP;
+ }
+
+}
Modified: trunk/freenet/src/freenet/client/FetchException.java
===================================================================
--- trunk/freenet/src/freenet/client/FetchException.java 2005-11-07
20:17:23 UTC (rev 7494)
+++ trunk/freenet/src/freenet/client/FetchException.java 2005-11-07
23:51:07 UTC (rev 7495)
@@ -60,7 +60,7 @@
public static final int NOT_IN_ARCHIVE = 10;
/** Has more metastrings, can't fulfill them */
public static final int HAS_MORE_METASTRINGS = 11;
- /** Internal error, probably failed to read from a bucket */
+ /** Failed to read from or write to a bucket; a kind of internal error
*/
public static final int BUCKET_ERROR = 12;
/** Data not found */
public static final int DATA_NOT_FOUND = 13;
Added: trunk/freenet/src/freenet/client/FileInserter.java
===================================================================
--- trunk/freenet/src/freenet/client/FileInserter.java 2005-11-07 20:17:23 UTC
(rev 7494)
+++ trunk/freenet/src/freenet/client/FileInserter.java 2005-11-07 23:51:07 UTC
(rev 7495)
@@ -0,0 +1,142 @@
+package freenet.client;
+
+import java.io.IOException;
+
+import freenet.keys.CHKEncodeException;
+import freenet.keys.ClientCHKBlock;
+import freenet.keys.FreenetURI;
+import freenet.keys.NodeCHK;
+import freenet.node.LowLevelPutException;
+import freenet.support.Bucket;
+import freenet.support.BucketTools;
+
+/**
+ * Class that does high-level inserts.
+ */
+public class FileInserter {
+
+ InserterContext ctx;
+
+ public FileInserter(InserterContext context) {
+ this.ctx = context;
+ }
+
+ /**
+ * Do an insert.
+ * @param block The data to insert.
+ * @return The URI of the inserted data.
+ * @throws InserterException
+ */
+ public FreenetURI run(InsertBlock block, boolean metadata) throws
InserterException {
+ if(!block.desiredURI.toString(false).equals("CHK@"))
+ throw new
InserterException(InserterException.INVALID_URI);
+
+ // Insert the content.
+ // If we have reason to create a metadata document, include the
client metadata.
+ // Otherwise only create one (a redirect) with the client
metadata, if there is any.
+
+ // First, can it fit into a single block?
+
+ Bucket data = block.data;
+ ClientCHKBlock chk;
+
+ int compressionCodec = -1; // no compression
+
+ int bestCompressionCodec = -1; // no compression
+ Bucket bestCompressedData;
+
+ if(data.size() > NodeCHK.BLOCK_SIZE && (!ctx.dontCompress)) {
+ // Try to compress the data.
+ // Try each algorithm, starting with the fastest and
weakest.
+ // Stop when run out of algorithms, or the compressed
data fits in a single block.
+ int algos = Metadata.countCompressAlgorithms();
+ for(int i=0;i<algos;i++) {
+ Compressor comp =
Metadata.getCompressionAlgorithmByDifficulty(i);
+ Bucket result = comp.compress(data, ctx.bf);
+ if(result.size() < NodeCHK.BLOCK_SIZE) {
+ compressionCodec = -1;
+ data = result;
+ if(bestCompressedData != null)
+
ctx.bf.freeBucket(bestCompressedData);
+ break;
+ }
+ if(bestCompressedData != null && result.size()
< bestCompressedData.size()) {
+ ctx.bf.freeBucket(bestCompressedData);
+ bestCompressedData = result;
+ bestCompressionCodec =
comp.codecNumberForMetadata();
+ } else if(bestCompressedData == null &&
result.size() < data.size()) {
+ bestCompressedData = result;
+ bestCompressionCodec =
comp.codecNumberForMetadata();
+ }
+ }
+ if(compressionCodec == -1) {
+ compressionCodec = bestCompressionCodec;
+ if(compressionCodec != -1) {
+ data = bestCompressedData;
+ }
+ }
+ }
+
+ if(data.size() <= NodeCHK.BLOCK_SIZE) {
+ if(compressionCodec == -1) {
+ chk =
ClientCHKBlock.encode(BucketTools.toByteArray(data), metadata, true);
+ }
+ }
+
+ if(data.size() <= NodeCHK.BLOCK_SIZE ||
+ data.size() <=
ClientCHKBlock.MAX_LENGTH_BEFORE_COMPRESSION) {
+ try {
+ chk =
ClientCHKBlock.encode(BucketTools.toByteArray(data), metadata);
+ return simplePutCHK(chk, block.clientMetadata);
+ } catch (CHKEncodeException e) {
+ // Too big! Encode to a splitfile, below.
+ } catch (IOException e) {
+ throw new
InserterException(InserterException.BUCKET_ERROR);
+ }
+ }
+
+ // Too big, encode to a splitfile
+ SplitInserter splitInsert = new SplitInserter(data,
block.clientMetadata);
+ splitInsert.run();
+ }
+
+ /**
+ * Simple insert. Only complication is that it might have some client
metadata.
+ * @param chk The data encoded into a single CHK.
+ * @param clientMetadata The client metadata. If this is non-trivial,
we will have to
+ * create a redirect document just to put the metadata in.
+ * @return The URI of the resulting CHK.
+ */
+ private FreenetURI simplePutCHK(ClientCHKBlock chk, ClientMetadata
clientMetadata) {
+ try {
+ ctx.client.putCHK(chk);
+ } catch (LowLevelPutException e) {
+ translateException(e);
+ }
+
+ if(clientMetadata == null || clientMetadata.isTrivial())
+ // Don't need a redirect for the metadata
+ return chk.getClientKey().getURI();
+ else {
+ // Do need a redirect for the metadata
+ Metadata metadata = new
Metadata(Metadata.SIMPLE_REDIRECT, chk.getClientKey().getURI(), clientMetadata);
+ return putMetadataCHK(metadata);
+ }
+ }
+
+ /** Put a metadata CHK
+ * @throws InserterException If the insert fails.
+ */
+ private FreenetURI putMetadataCHK(Metadata metadata) throws
InserterException {
+ byte[] data = metadata.writeToByteArray();
+ Bucket bucket;
+ try {
+ bucket = BucketTools.makeImmutableBucket(ctx.bf, data);
+ } catch (IOException e) {
+ throw new
InserterException(InserterException.BUCKET_ERROR);
+ }
+ InsertBlock block = new InsertBlock(bucket, null,
FreenetURI.EMPTY_CHK_URI);
+ return run(block, true);
+ }
+
+}
Added: trunk/freenet/src/freenet/client/GzipCompressor.java
===================================================================
--- trunk/freenet/src/freenet/client/GzipCompressor.java 2005-11-07
20:17:23 UTC (rev 7494)
+++ trunk/freenet/src/freenet/client/GzipCompressor.java 2005-11-07
23:51:07 UTC (rev 7495)
@@ -0,0 +1,29 @@
+package freenet.client;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.util.zip.GZIPOutputStream;
+
+import freenet.support.Bucket;
+import freenet.support.BucketFactory;
+
+public class GzipCompressor extends Compressor {
+
+ public Bucket compress(Bucket data, BucketFactory bf) throws
IOException {
+ Bucket output = bf.makeBucket(-1);
+ InputStream is = data.getInputStream();
+ OutputStream os = output.getOutputStream();
+ GZIPOutputStream gos = new GZIPOutputStream(os);
+ byte[] buffer = new byte[4096];
+ while(true) {
+ int x = is.read(buffer);
+ if(x <= -1) break;
+ if(x == 0) throw new IOException("Returned zero from
read()");
+ gos.write(buffer, 0, x);
+ }
+ gos.close();
+ return output;
+ }
+
+}
Modified: trunk/freenet/src/freenet/client/HighLevelSimpleClientImpl.java
===================================================================
--- trunk/freenet/src/freenet/client/HighLevelSimpleClientImpl.java
2005-11-07 20:17:23 UTC (rev 7494)
+++ trunk/freenet/src/freenet/client/HighLevelSimpleClientImpl.java
2005-11-07 23:51:07 UTC (rev 7495)
@@ -65,8 +65,9 @@
}
public FreenetURI insert(InsertBlock insert) {
- // TODO Auto-generated method stub
- return null;
+ InserterContext context = new InserterContext(client);
+ FileInserter i = new FileInserter(context);
+ return i.run(insert);
}
}
Modified: trunk/freenet/src/freenet/client/InsertBlock.java
===================================================================
--- trunk/freenet/src/freenet/client/InsertBlock.java 2005-11-07 20:17:23 UTC
(rev 7494)
+++ trunk/freenet/src/freenet/client/InsertBlock.java 2005-11-07 23:51:07 UTC
(rev 7495)
@@ -1,5 +1,6 @@
package freenet.client;
+import freenet.keys.FreenetURI;
import freenet.support.Bucket;
/**
@@ -7,12 +8,14 @@
*/
public class InsertBlock {
- private Bucket data;
- private String mimeType;
+ final Bucket data;
+ final FreenetURI desiredURI;
+ final ClientMetadata clientMetadata;
- public InsertBlock(Bucket data, String mimeType) {
+ public InsertBlock(Bucket data, ClientMetadata metadata, FreenetURI
desiredURI) {
this.data = data;
- this.mimeType = mimeType;
+ clientMetadata = metadata;
+ this.desiredURI = desiredURI;
}
}
Added: trunk/freenet/src/freenet/client/InserterContext.java
===================================================================
--- trunk/freenet/src/freenet/client/InserterContext.java 2005-11-07
20:17:23 UTC (rev 7494)
+++ trunk/freenet/src/freenet/client/InserterContext.java 2005-11-07
23:51:07 UTC (rev 7495)
@@ -0,0 +1,20 @@
+package freenet.client;
+
+import freenet.node.SimpleLowLevelClient;
+import freenet.support.BucketFactory;
+
+/** Context object for an insert operation, including both simple and
multi-file inserts */
+public class InserterContext {
+
+ final SimpleLowLevelClient client;
+ final BucketFactory bf;
+ /** If true, don't try to compress the data */
+ final boolean dontCompress;
+
+ public InserterContext(SimpleLowLevelClient client, BucketFactory bf) {
+ this.client = client;
+ this.bf = bf;
+ dontCompress = false;
+ }
+
+}
Added: trunk/freenet/src/freenet/client/InserterException.java
===================================================================
--- trunk/freenet/src/freenet/client/InserterException.java 2005-11-07
20:17:23 UTC (rev 7494)
+++ trunk/freenet/src/freenet/client/InserterException.java 2005-11-07
23:51:07 UTC (rev 7495)
@@ -0,0 +1,29 @@
+package freenet.client;
+
+import java.io.IOException;
+
+public class InserterException extends Exception {
+ private static final long serialVersionUID = -1106716067841151962L;
+
+ final int mode;
+
+ /** Get the failure mode. */
+ public int getMode() {
+ return mode;
+ }
+
+ public InserterException(int m) {
+ mode = m;
+ }
+
+ public InserterException(int mode, IOException e) {
+ this.mode = mode;
+ initCause(e);
+ }
+
+ /** Caller supplied a URI we cannot use */
+ public static final int INVALID_URI = 1;
+ /** Failed to read from or write to a bucket; a kind of internal error
*/
+ public static final int BUCKET_ERROR = 2;
+
+}
Modified: trunk/freenet/src/freenet/client/Metadata.java
===================================================================
--- trunk/freenet/src/freenet/client/Metadata.java 2005-11-07 20:17:23 UTC
(rev 7494)
+++ trunk/freenet/src/freenet/client/Metadata.java 2005-11-07 23:51:07 UTC
(rev 7495)
@@ -277,6 +277,22 @@
}
/**
+ * Create another kind of simple Metadata object (a redirect or similar
object).
+ * @param docType The document type.
+ * @param uri The URI pointed to.
+ * @param cm The client metadata, if any.
+ */
+ public Metadata(byte docType, FreenetURI uri, ClientMetadata cm) {
+ if(docType == SIMPLE_REDIRECT) {
+ documentType = docType;
+ clientMetadata = cm;
+ setMIMEType(cm.getMIMEType());
+ simpleRedirectKey = uri;
+ } else
+ throw new IllegalArgumentException();
+ }
+
+ /**
* Set the MIME type to a string. Compresses it if possible for transit.
*/
private void setMIMEType(String type) {
@@ -626,4 +642,17 @@
public FreenetURI[] getSplitfileCheckKeys() {
return splitfileCheckKeys;
}
+
+ /** Count the number of distinct compression algorithms currently
supported. */
+ public static int countCompressAlgorithms() {
+ // FIXME we presently only support gzip. This should change in
future.
+ return 1;
+ }
+
+ public static Compressor getCompressionAlgorithmByDifficulty(int i) {
+ if(i == 0)
+ return Compressor.gzip;
+ // FIXME when we get more compression algos, put them here.
+ return null;
+ }
}
Modified: trunk/freenet/src/freenet/keys/CHKBlock.java
===================================================================
--- trunk/freenet/src/freenet/keys/CHKBlock.java 2005-11-07 20:17:23 UTC
(rev 7494)
+++ trunk/freenet/src/freenet/keys/CHKBlock.java 2005-11-07 23:51:07 UTC
(rev 7495)
@@ -24,7 +24,7 @@
final byte[] header;
final short hashIdentifier;
final NodeCHK chk;
- protected static final int MAX_LENGTH_BEFORE_COMPRESSION = 1024 * 1024;
+ public static final int MAX_LENGTH_BEFORE_COMPRESSION = 1024 * 1024;
final static int HASH_SHA1 = 1;
public String toString() {
Modified: trunk/freenet/src/freenet/keys/CHKEncodeException.java
===================================================================
--- trunk/freenet/src/freenet/keys/CHKEncodeException.java 2005-11-07
20:17:23 UTC (rev 7494)
+++ trunk/freenet/src/freenet/keys/CHKEncodeException.java 2005-11-07
23:51:07 UTC (rev 7495)
@@ -4,6 +4,7 @@
* @author amphibian
*
* Exception thrown when a CHK encoding fails.
+ * Specifically, it is thrown when the data is too big to encode.
*/
public class CHKEncodeException extends Exception {
static final long serialVersionUID = -1;
Modified: trunk/freenet/src/freenet/keys/ClientCHKBlock.java
===================================================================
--- trunk/freenet/src/freenet/keys/ClientCHKBlock.java 2005-11-07 20:17:23 UTC
(rev 7494)
+++ trunk/freenet/src/freenet/keys/ClientCHKBlock.java 2005-11-07 23:51:07 UTC
(rev 7495)
@@ -47,9 +47,12 @@
/**
* Encode a block of data to a CHKBlock.
+ * @param sourceData The data to encode.
+ * @param asMetadata Is this a metadata key?
+ * @param dontCompress If set, don't even try to compress.
*/
- static public ClientCHKBlock encode(byte[] sourceData) throws
CHKEncodeException {
+ static public ClientCHKBlock encode(byte[] sourceData, boolean asMetadata,
boolean dontCompress) throws CHKEncodeException {
byte[] data;
byte[] header;
ClientCHK key;
@@ -58,7 +61,7 @@
boolean compressed = false;
if(sourceData.length > MAX_LENGTH_BEFORE_COMPRESSION)
throw new CHKEncodeException("Too big");
- if(sourceData.length != 0) {
+ if(sourceData.length > NodeCHK.BLOCK_SIZE && !dontCompress) {
int sourceLength = sourceData.length;
byte[] cbuf = new byte[32768+1024];
Deflater compressor = new Deflater();
@@ -149,7 +152,7 @@
byte[] finalHash = md160.digest(data);
// Now convert it into a ClientCHK
- key = new ClientCHK(finalHash, encKey, compressed, false,
ClientCHK.ALGO_AES_PCFB_256);
+ key = new ClientCHK(finalHash, encKey, compressed, asMetadata,
ClientCHK.ALGO_AES_PCFB_256);
try {
return new ClientCHKBlock(data, header, key, false);
Modified: trunk/freenet/src/freenet/keys/FreenetURI.java
===================================================================
--- trunk/freenet/src/freenet/keys/FreenetURI.java 2005-11-07 20:17:23 UTC
(rev 7494)
+++ trunk/freenet/src/freenet/keys/FreenetURI.java 2005-11-07 23:51:07 UTC
(rev 7495)
@@ -86,6 +86,8 @@
this(keyType, docName, (String[]) null, null, null, null);
}
+ public static final FreenetURI EMPTY_CHK_URI = new FreenetURI("CHK",
null, null, null, null, null);
+
public FreenetURI(
String keyType,
String docName,
