Author: toad
Date: 2007-05-25 00:40:17 +0000 (Fri, 25 May 2007)
New Revision: 13369
Added:
trunk/freenet/src/freenet/client/async/BinaryBlobFormatException.java
trunk/freenet/src/freenet/client/async/BinaryBlobInserter.java
Modified:
trunk/freenet/src/freenet/client/async/BinaryBlob.java
trunk/freenet/src/freenet/crypt/DSAPublicKey.java
trunk/freenet/src/freenet/keys/Key.java
trunk/freenet/src/freenet/keys/NodeSSK.java
trunk/freenet/src/freenet/node/NodeClientCore.java
trunk/freenet/src/freenet/node/SimpleSendableInsert.java
Log:
BinaryBlobInserter - not wired in yet, and various issues e.g. it keeps all the
keys in RAM (will file bugs; this one for example depends on making
SendableInsert work for multiple keys per object)
Modified: trunk/freenet/src/freenet/client/async/BinaryBlob.java
===================================================================
--- trunk/freenet/src/freenet/client/async/BinaryBlob.java 2007-05-24
23:07:19 UTC (rev 13368)
+++ trunk/freenet/src/freenet/client/async/BinaryBlob.java 2007-05-25
00:40:17 UTC (rev 13369)
@@ -15,7 +15,7 @@
binaryBlobStream.writeShort(BinaryBlob.BINARY_BLOB_OVERALL_VERSION);
}
public static void writeKey(DataOutputStream binaryBlobStream, KeyBlock
block, Key key) throws IOException {
- byte[] keyData = key.getRoutingKey();
+ byte[] keyData = key.getKeyBytes();
byte[] headers = block.getRawHeaders();
byte[] data = block.getRawData();
byte[] pubkey = block.getPubkeyBytes();
Added: trunk/freenet/src/freenet/client/async/BinaryBlobFormatException.java
===================================================================
--- trunk/freenet/src/freenet/client/async/BinaryBlobFormatException.java
(rev 0)
+++ trunk/freenet/src/freenet/client/async/BinaryBlobFormatException.java
2007-05-25 00:40:17 UTC (rev 13369)
@@ -0,0 +1,15 @@
+package freenet.client.async;
+
+import freenet.keys.KeyVerifyException;
+
+public class BinaryBlobFormatException extends Exception {
+
+ public BinaryBlobFormatException(String message) {
+ super(message);
+ }
+
+ public BinaryBlobFormatException(String message, KeyVerifyException e) {
+ super(message, e);
+ }
+
+}
Added: trunk/freenet/src/freenet/client/async/BinaryBlobInserter.java
===================================================================
--- trunk/freenet/src/freenet/client/async/BinaryBlobInserter.java
(rev 0)
+++ trunk/freenet/src/freenet/client/async/BinaryBlobInserter.java
2007-05-25 00:40:17 UTC (rev 13369)
@@ -0,0 +1,247 @@
+package freenet.client.async;
+
+import java.io.DataInputStream;
+import java.io.EOFException;
+import java.io.IOException;
+import java.util.Vector;
+
+import com.onionnetworks.util.FileUtil;
+
+import freenet.client.FailureCodeTracker;
+import freenet.client.InsertException;
+import freenet.keys.Key;
+import freenet.keys.KeyBlock;
+import freenet.keys.KeyVerifyException;
+import freenet.node.LowLevelPutException;
+import freenet.node.NodeClientCore;
+import freenet.node.SimpleSendableInsert;
+import freenet.support.Logger;
+import freenet.support.SimpleFieldSet;
+import freenet.support.api.Bucket;
+import freenet.support.api.BucketFactory;
+
+public class BinaryBlobInserter implements ClientPutState {
+
+ final ClientPutter parent;
+ final Object clientContext;
+ final MySendableInsert[] inserters;
+ final FailureCodeTracker errors;
+ final short maxRetries;
+ final short consecutiveRNFsCountAsSuccess;
+ private boolean logMINOR;
+ private int completedBlocks;
+ private int succeededBlocks;
+ private boolean fatal;
+
+ BinaryBlobInserter(Bucket blob, ClientPutter parent, BucketFactory bf,
Object clientContext, boolean tolerant, NodeClientCore core,
+ short prioClass, short maxRetries, short
consecutiveRNFsCountAsSuccess)
+ throws IOException, BinaryBlobFormatException {
+ logMINOR = Logger.shouldLog(Logger.MINOR, this);
+ this.maxRetries = maxRetries;
+ this.consecutiveRNFsCountAsSuccess =
consecutiveRNFsCountAsSuccess;
+ this.parent = parent;
+ this.clientContext = clientContext;
+ this.errors = new FailureCodeTracker(true);
+ Vector myInserters = new Vector();
+ DataInputStream dis = new
DataInputStream(blob.getInputStream());
+ long magic = dis.readLong();
+ if(magic != BinaryBlob.BINARY_BLOB_MAGIC)
+ throw new BinaryBlobFormatException("Bad magic");
+ short version = dis.readShort();
+ if(version != BinaryBlob.BINARY_BLOB_OVERALL_VERSION)
+ throw new BinaryBlobFormatException("Unknown overall
version");
+
+ int i=0;
+ while(true) {
+ long blobLength;
+ try {
+ blobLength = dis.readInt() & 0xFFFFFFFFL;
+ } catch (EOFException e) {
+ // End of file
+ dis.close();
+ break;
+ }
+ short blobType = dis.readShort();
+ short blobVer = dis.readShort();
+
+ if(blobType == BinaryBlob.BLOB_END) {
+ dis.close();
+ break;
+ } else if(blobType == BinaryBlob.BLOB_BLOCK) {
+ if(blobVer != BinaryBlob.BLOB_BLOCK_VERSION)
+ // Even if tolerant, if we can't read a
blob there probably isn't much we can do.
+ throw new
BinaryBlobFormatException("Unknown block blob version");
+ if(blobLength < 9)
+ throw new
BinaryBlobFormatException("Block blob too short");
+ short keyType = dis.readShort();
+ int keyLen = dis.readUnsignedByte();
+ short headersLen = dis.readShort();
+ short dataLen = dis.readShort();
+ short pubkeyLen = dis.readShort();
+ if(blobLength != 9 + keyLen + headersLen +
dataLen + pubkeyLen)
+ throw new
BinaryBlobFormatException("Binary blob too short for data lengths");
+ byte[] keyBytes = new byte[keyLen];
+ byte[] headersBytes = new byte[headersLen];
+ byte[] dataBytes = new byte[dataLen];
+ byte[] pubkeyBytes = new byte[pubkeyLen];
+ dis.readFully(keyBytes);
+ dis.readFully(headersBytes);
+ dis.readFully(dataBytes);
+ dis.readFully(pubkeyBytes);
+ KeyBlock block;
+ try {
+ block = Key.createBlock(keyType,
keyBytes, headersBytes, dataBytes, pubkeyBytes);
+ } catch (KeyVerifyException e) {
+ throw new
BinaryBlobFormatException("Invalid key: "+e.getMessage(), e);
+ }
+
+ MySendableInsert inserter =
+ new MySendableInsert(i, core, block,
prioClass);
+
+ myInserters.add(inserter);
+
+ } else {
+ if(tolerant) {
+ FileUtil.skipFully(dis, blobLength);
+ } else {
+ throw new
BinaryBlobFormatException("Unknown blob type: "+blobType);
+ }
+ }
+ i++;
+ }
+ inserters = (MySendableInsert[]) myInserters.toArray(new
MySendableInsert[myInserters.size()]);
+ parent.addMustSucceedBlocks(inserters.length);
+ }
+
+ public void cancel() {
+ for(int i=0;i<inserters.length;i++) {
+ if(inserters[i] != null)
+ inserters[i].cancel();
+ }
+ parent.onFailure(new
InsertException(InsertException.CANCELLED), this);
+ }
+
+ public BaseClientPutter getParent() {
+ return parent;
+ }
+
+ public SimpleFieldSet getProgressFieldset() {
+ // FIXME not supported
+ return null;
+ }
+
+ public Object getToken() {
+ return clientContext;
+ }
+
+ public void schedule() throws InsertException {
+ for(int i=0;i<inserters.length;i++) {
+ inserters[i].schedule();
+ }
+ }
+
+ class MySendableInsert extends SimpleSendableInsert {
+
+ final int blockNum;
+ private int consecutiveRNFs;
+ private int retries;
+
+ public MySendableInsert(int i, NodeClientCore core, KeyBlock
block, short prioClass) {
+ super(core, block, prioClass);
+ this.blockNum = i;
+ }
+
+ public void onSuccess() {
+ synchronized(this) {
+ if(inserters[blockNum] == null) return;
+ inserters[blockNum] = null;
+ completedBlocks++;
+ succeededBlocks++;
+ }
+ parent.completedBlock(false);
+ maybeFinish();
+ }
+
+ // FIXME duplicated code from SingleBlockInserter
+ // FIXME combine it somehow
+ public void onFailure(LowLevelPutException e) {
+ synchronized(BinaryBlobInserter.this) {
+ if(inserters[blockNum] == null) return;
+ }
+ if(parent.isCancelled()) {
+ fail(new
InsertException(InsertException.CANCELLED), true);
+ return;
+ }
+ logMINOR = Logger.shouldLog(Logger.MINOR,
BinaryBlobInserter.this);
+ switch(e.code) {
+ case LowLevelPutException.COLLISION:
+ fail(new
InsertException(InsertException.COLLISION), false);
+ break;
+ case LowLevelPutException.INTERNAL_ERROR:
+ errors.inc(InsertException.INTERNAL_ERROR);
+ break;
+ case LowLevelPutException.REJECTED_OVERLOAD:
+ errors.inc(InsertException.REJECTED_OVERLOAD);
+ break;
+ case LowLevelPutException.ROUTE_NOT_FOUND:
+ errors.inc(InsertException.ROUTE_NOT_FOUND);
+ break;
+ case LowLevelPutException.ROUTE_REALLY_NOT_FOUND:
+
errors.inc(InsertException.ROUTE_REALLY_NOT_FOUND);
+ break;
+ default:
+ Logger.error(this, "Unknown
LowLevelPutException code: "+e.code);
+ errors.inc(InsertException.INTERNAL_ERROR);
+ }
+ if(e.code == LowLevelPutException.ROUTE_NOT_FOUND) {
+ consecutiveRNFs++;
+ if(logMINOR) Logger.minor(this, "Consecutive
RNFs: "+consecutiveRNFs+" / "+consecutiveRNFsCountAsSuccess);
+ if(consecutiveRNFs ==
consecutiveRNFsCountAsSuccess) {
+ if(logMINOR) Logger.minor(this,
"Consecutive RNFs: "+consecutiveRNFs+" - counting as success");
+ onSuccess();
+ return;
+ }
+ } else
+ consecutiveRNFs = 0;
+ if(logMINOR) Logger.minor(this, "Failed: "+e);
+ retries++;
+ if((retries > maxRetries) && (maxRetries != -1)) {
+ fail(InsertException.construct(errors), false);
+ return;
+ }
+ schedule();
+ }
+
+ private void fail(InsertException e, boolean fatal) {
+ synchronized(BinaryBlobInserter.this) {
+ if(inserters[blockNum] == null) return;
+ inserters[blockNum] = null;
+ completedBlocks++;
+ if(fatal) BinaryBlobInserter.this.fatal = true;
+ }
+ if(fatal)
+ parent.fatallyFailedBlock();
+ else
+ parent.failedBlock();
+ maybeFinish();
+ }
+ }
+
+ public void maybeFinish() {
+ boolean success;
+ boolean wasFatal;
+ synchronized(this) {
+ if(completedBlocks != inserters.length)
+ return;
+ success = completedBlocks == succeededBlocks;
+ wasFatal = fatal;
+ }
+ if(success) {
+ parent.onSuccess(this);
+ } else if(wasFatal)
+ parent.onFailure(new
InsertException(InsertException.FATAL_ERRORS_IN_BLOCKS, errors, null), this);
+ else
+ parent.onFailure(new
InsertException(InsertException.TOO_MANY_RETRIES_IN_BLOCKS, errors, null),
this);
+ }
+
+}
Modified: trunk/freenet/src/freenet/crypt/DSAPublicKey.java
===================================================================
--- trunk/freenet/src/freenet/crypt/DSAPublicKey.java 2007-05-24 23:07:19 UTC
(rev 13368)
+++ trunk/freenet/src/freenet/crypt/DSAPublicKey.java 2007-05-25 00:40:17 UTC
(rev 13369)
@@ -54,7 +54,11 @@
throw new IllegalArgumentException("y must be < p but
y="+y+" p="+group.getP());
}
- public static DSAPublicKey create(byte[] pubkeyAsBytes) throws
CryptFormatException {
+ public DSAPublicKey(byte[] pubkeyBytes) throws IOException,
CryptFormatException {
+ this(new ByteArrayInputStream(pubkeyBytes));
+ }
+
+ public static DSAPublicKey create(byte[] pubkeyAsBytes) throws
CryptFormatException {
try {
return new DSAPublicKey(new
ByteArrayInputStream(pubkeyAsBytes));
} catch (IOException e) {
Modified: trunk/freenet/src/freenet/keys/Key.java
===================================================================
--- trunk/freenet/src/freenet/keys/Key.java 2007-05-24 23:07:19 UTC (rev
13368)
+++ trunk/freenet/src/freenet/keys/Key.java 2007-05-25 00:40:17 UTC (rev
13369)
@@ -9,6 +9,8 @@
import java.security.MessageDigest;
import java.util.Arrays;
+import freenet.crypt.CryptFormatException;
+import freenet.crypt.DSAPublicKey;
import freenet.crypt.SHA256;
import freenet.io.WritableToDataOutputStream;
import freenet.support.Fields;
@@ -72,6 +74,26 @@
throw new IOException("Unrecognized format: "+type);
}
+ public static KeyBlock createBlock(short keyType, byte[] keyBytes,
byte[] headersBytes, byte[] dataBytes, byte[] pubkeyBytes) throws
KeyVerifyException {
+ byte type = (byte)(keyType >> 8);
+ byte subtype = (byte)(keyType & 0xFF);
+ if(type == NodeCHK.BASE_TYPE) {
+ return CHKBlock.construct(headersBytes, dataBytes);
+ } else if(type == NodeSSK.BASE_TYPE) {
+ DSAPublicKey pubKey;
+ try {
+ pubKey = new DSAPublicKey(pubkeyBytes);
+ } catch (IOException e) {
+ throw new KeyVerifyException("Failed to
construct pubkey: "+e, e);
+ } catch (CryptFormatException e) {
+ throw new KeyVerifyException("Failed to
construct pubkey: "+e, e);
+ }
+ NodeSSK key = new NodeSSK(pubKey.asBytesHash(),
keyBytes, pubKey, subtype);
+ return new SSKBlock(dataBytes, headersBytes, key,
false);
+ } else {
+ throw new KeyVerifyException("No such key type
"+Integer.toHexString(type));
+ }
+ }
/**
* Convert the key to a double between 0.0 and 1.0.
@@ -222,5 +244,9 @@
public byte[] getRoutingKey() {
return routingKey;
}
-
+
+ // Not just the routing key, enough data to reconstruct the key (excluding
any pubkey needed)
+ public byte[] getKeyBytes() {
+ return routingKey;
+ }
}
Modified: trunk/freenet/src/freenet/keys/NodeSSK.java
===================================================================
--- trunk/freenet/src/freenet/keys/NodeSSK.java 2007-05-24 23:07:19 UTC (rev
13368)
+++ trunk/freenet/src/freenet/keys/NodeSSK.java 2007-05-25 00:40:17 UTC (rev
13369)
@@ -157,4 +157,9 @@
return hashCode;
}
+ // Not just the routing key, enough data to reconstruct the key (excluding
any pubkey needed)
+ public byte[] getKeyBytes() {
+ return encryptedHashedDocname;
+ }
+
}
Modified: trunk/freenet/src/freenet/node/NodeClientCore.java
===================================================================
--- trunk/freenet/src/freenet/node/NodeClientCore.java 2007-05-24 23:07:19 UTC
(rev 13368)
+++ trunk/freenet/src/freenet/node/NodeClientCore.java 2007-05-25 00:40:17 UTC
(rev 13369)
@@ -995,12 +995,7 @@
public void queueRandomReinsert(KeyBlock block) {
SimpleSendableInsert ssi = new SimpleSendableInsert(this,
block, RequestStarter.MAXIMUM_PRIORITY_CLASS);
if(logMINOR) Logger.minor(this, "Queueing random reinsert for
"+block+" : "+ssi);
- if(block instanceof CHKBlock)
- requestStarters.chkPutScheduler.register(ssi);
- else if(block instanceof SSKBlock)
- requestStarters.sskPutScheduler.register(ssi);
- else
- Logger.error(this, "Don't know what to do with
"+block+" should be queued for reinsert");
+ ssi.schedule();
}
public void storeConfig() {
Modified: trunk/freenet/src/freenet/node/SimpleSendableInsert.java
===================================================================
--- trunk/freenet/src/freenet/node/SimpleSendableInsert.java 2007-05-24
23:07:19 UTC (rev 13368)
+++ trunk/freenet/src/freenet/node/SimpleSendableInsert.java 2007-05-25
00:40:17 UTC (rev 13369)
@@ -3,9 +3,13 @@
* http://www.gnu.org/ for further details of the GPL. */
package freenet.node;
+import freenet.client.InsertException;
import freenet.client.async.ClientRequester;
+import freenet.keys.CHKBlock;
import freenet.keys.KeyBlock;
+import freenet.keys.SSKBlock;
import freenet.support.Logger;
+import freenet.support.RandomGrabArray;
/**
* Simple SendableInsert implementation. No feedback, no retries, just insert
the
@@ -78,4 +82,23 @@
return true;
}
+ public void schedule() {
+ finished = false; // can reschedule
+ if(block instanceof CHKBlock)
+ node.requestStarters.chkPutScheduler.register(this);
+ else if(block instanceof SSKBlock)
+ node.requestStarters.sskPutScheduler.register(this);
+ else
+ Logger.error(this, "Don't know what to do with "+block,
new Exception());
+ }
+
+ public void cancel() {
+ synchronized(this) {
+ if(finished) return;
+ finished = true;
+ }
+ RandomGrabArray arr = getParentGrabArray();
+ if(arr != null) arr.remove(this);
+ }
+
}
