Author: toad
Date: 2006-07-01 14:46:32 +0000 (Sat, 01 Jul 2006)
New Revision: 9414
Modified:
trunk/freenet/src/freenet/keys/CHKBlock.java
trunk/freenet/src/freenet/keys/SSKBlock.java
trunk/freenet/src/freenet/node/Node.java
trunk/freenet/src/freenet/node/Version.java
trunk/freenet/src/freenet/store/BerkeleyDBFreenetStore.java
Log:
846: Attempt to auto-fix broken datastores.
Modified: trunk/freenet/src/freenet/keys/CHKBlock.java
===================================================================
--- trunk/freenet/src/freenet/keys/CHKBlock.java 2006-07-01 13:55:22 UTC
(rev 9413)
+++ trunk/freenet/src/freenet/keys/CHKBlock.java 2006-07-01 14:46:32 UTC
(rev 9414)
@@ -47,9 +47,11 @@
if(headers.length != TOTAL_HEADERS_LENGTH)
throw new IllegalArgumentException("Wrong length:
"+headers.length+" should be "+TOTAL_HEADERS_LENGTH);
hashIdentifier = (short)(((headers[0] & 0xff) << 8) + (headers[1] &
0xff));
- this.chk = key;
// Logger.debug(CHKBlock.class, "Data length: "+data.length+", header
length: "+header.length);
- if(!verify) return;
+ if(key != null && !verify) {
+ this.chk = key;
+ return;
+ }
// Minimal verification
// Check the hash
@@ -65,11 +67,16 @@
md.update(headers);
md.update(data);
byte[] hash = md.digest();
- byte[] check = chk.routingKey;
- if(!java.util.Arrays.equals(hash, check)) {
- throw new CHKVerifyException("Hash does not verify");
+ if(key == null) {
+ chk = new NodeCHK(hash);
+ } else {
+ chk = key;
+ byte[] check = chk.routingKey;
+ if(!java.util.Arrays.equals(hash, check)) {
+ throw new CHKVerifyException("Hash does not verify");
+ }
+ // Otherwise it checks out
}
- // Otherwise it checks out
}
public Key getKey() {
Modified: trunk/freenet/src/freenet/keys/SSKBlock.java
===================================================================
--- trunk/freenet/src/freenet/keys/SSKBlock.java 2006-07-01 13:55:22 UTC
(rev 9413)
+++ trunk/freenet/src/freenet/keys/SSKBlock.java 2006-07-01 14:46:32 UTC
(rev 9414)
@@ -85,9 +85,9 @@
this.data = data;
this.headers = headers;
this.nodeKey = nodeKey;
- this.pubKey = nodeKey.getPubKey();
if(data.length != DATA_LENGTH)
throw new SSKVerifyException("Data length wrong:
"+data.length+" should be "+DATA_LENGTH);
+ this.pubKey = nodeKey.getPubKey();
if(pubKey == null)
throw new SSKVerifyException("PubKey was null from
"+nodeKey);
MessageDigest md;
Modified: trunk/freenet/src/freenet/node/Node.java
===================================================================
--- trunk/freenet/src/freenet/node/Node.java 2006-07-01 13:55:22 UTC (rev
9413)
+++ trunk/freenet/src/freenet/node/Node.java 2006-07-01 14:46:32 UTC (rev
9414)
@@ -29,6 +29,8 @@
import org.tanukisoftware.wrapper.WrapperManager;
+import com.sleepycat.je.DatabaseException;
+
import freenet.client.ArchiveManager;
import freenet.client.ClientMetadata;
import freenet.client.FetchException;
@@ -1370,13 +1372,25 @@
try {
Logger.normal(this, "Initializing CHK Datastore");
System.out.println("Initializing CHK Datastore");
- chkDatastore = new
BerkeleyDBFreenetStore(storeDir.getPath()+File.separator+"store-"+portNumber,
maxStoreKeys, 32768, CHKBlock.TOTAL_HEADERS_LENGTH);
+ BerkeleyDBFreenetStore tmp;
+ try {
+ tmp = new
BerkeleyDBFreenetStore(storeDir.getPath()+File.separator+"store-"+portNumber,
maxStoreKeys, 32768, CHKBlock.TOTAL_HEADERS_LENGTH);
+ } catch (DatabaseException e) {
+ tmp = new
BerkeleyDBFreenetStore(storeDir.getPath()+File.separator+"store-"+portNumber,
maxStoreKeys, 32768, CHKBlock.TOTAL_HEADERS_LENGTH,
BerkeleyDBFreenetStore.TYPE_CHK);
+ }
+ chkDatastore = tmp;
+ Logger.normal(this, "Initializing pubKey Datastore");
+ System.out.println("Initializing pubKey Datastore");
+ try {
+ tmp = new
BerkeleyDBFreenetStore(storeDir.getPath()+File.separator+"pubkeystore-"+portNumber,
maxStoreKeys, DSAPublicKey.PADDED_SIZE, 0);
+ } catch (DatabaseException e) {
+ tmp = new
BerkeleyDBFreenetStore(storeDir.getPath()+File.separator+"pubkeystore-"+portNumber,
maxStoreKeys, DSAPublicKey.PADDED_SIZE, 0, BerkeleyDBFreenetStore.TYPE_PUBKEY);
+ }
+ this.pubKeyDatastore = tmp;
+ // FIXME can't auto-fix SSK stores.
Logger.normal(this, "Initializing SSK Datastore");
System.out.println("Initializing SSK Datastore");
sskDatastore = new
BerkeleyDBFreenetStore(storeDir.getPath()+File.separator+"sskstore-"+portNumber,
maxStoreKeys, 1024, SSKBlock.TOTAL_HEADERS_LENGTH);
- Logger.normal(this, "Initializing pubKey Datastore");
- System.out.println("Initializing pubKey Datastore");
- pubKeyDatastore = new
BerkeleyDBFreenetStore(storeDir.getPath()+File.separator+"pubkeystore-"+portNumber,
maxStoreKeys, DSAPublicKey.PADDED_SIZE, 0);
} catch (FileNotFoundException e1) {
String msg = "Could not open datastore: "+e1;
Logger.error(this, msg, e1);
Modified: trunk/freenet/src/freenet/node/Version.java
===================================================================
--- trunk/freenet/src/freenet/node/Version.java 2006-07-01 13:55:22 UTC (rev
9413)
+++ trunk/freenet/src/freenet/node/Version.java 2006-07-01 14:46:32 UTC (rev
9414)
@@ -18,7 +18,7 @@
public static final String protocolVersion = "1.0";
/** The build number of the current revision */
- private static final int buildNumber = 845;
+ private static final int buildNumber = 846;
/** Oldest build of Fred we will talk to */
private static final int oldLastGoodBuild = 839;
Modified: trunk/freenet/src/freenet/store/BerkeleyDBFreenetStore.java
===================================================================
--- trunk/freenet/src/freenet/store/BerkeleyDBFreenetStore.java 2006-07-01
13:55:22 UTC (rev 9413)
+++ trunk/freenet/src/freenet/store/BerkeleyDBFreenetStore.java 2006-07-01
14:46:32 UTC (rev 9414)
@@ -1,5 +1,6 @@
package freenet.store;
+import java.io.EOFException;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
@@ -182,8 +183,173 @@
// Add shutdownhook
Runtime.getRuntime().addShutdownHook(new ShutdownHook());
}
+
+ public static final short TYPE_CHK = 0;
+ public static final short TYPE_PUBKEY = 1;
+ public static final short TYPE_SSK = 2;
/**
+ * Recreate the index from the data file. Call this when the index has
been corrupted.
+ * @param the directory where the store is located
+ * @throws FileNotFoundException if the dir does not exist and could not
be created
+ */
+ public BerkeleyDBFreenetStore(String storeDir, long maxChkBlocks, int
blockSize, int headerSize, short type) throws Exception {
+ this.dataBlockSize = blockSize;
+ this.headerBlockSize = headerSize;
+ // Percentage of the database that must contain usefull data
+ // decrease to increase performance, increase to save disk space
+ System.setProperty("je.cleaner.minUtilization","98");
+
+ // Delete empty log files
+ System.setProperty("je.cleaner.expunge","true");
+
+ // Percentage of the maximum heap size used as a cache
+ System.setProperty("je.maxMemoryPercent","30");
+
+ this.maxChkBlocks=maxChkBlocks;
+
+ // Delete old database.
+
+ File dir = new File(storeDir);
+ if(!dir.exists())
+ dir.mkdir();
+ File dbDir = new File(dir,"database");
+ if(dbDir.exists()) {
+ File[] files = dbDir.listFiles();
+ for(int i=0;i<files.length;i++)
+ files[i].delete();
+ } else
+ dbDir.mkdir();
+
+ // Now create a new one.
+
+ // Initialize environment
+ EnvironmentConfig envConfig = new EnvironmentConfig();
+ envConfig.setAllowCreate(true);
+ envConfig.setTransactional(true);
+ envConfig.setTxnWriteNoSync(true);
+
+ environment = new Environment(dbDir, envConfig);
+
+ // Initialize CHK database
+ DatabaseConfig dbConfig = new DatabaseConfig();
+ dbConfig.setAllowCreate(true);
+ dbConfig.setTransactional(true);
+ chkDB = environment.openDatabase(null,"CHK",dbConfig);
+
+ fixSecondaryFile = new File(storeDir, "recreate_secondary_db");
+ fixSecondaryFile.delete();
+
+ // Initialize secondary CHK database sorted on accesstime
+ SecondaryConfig secDbConfig = new SecondaryConfig();
+ secDbConfig.setAllowCreate(true);
+ secDbConfig.setSortedDuplicates(true);
+ secDbConfig.setTransactional(true);
+ secDbConfig.setAllowPopulate(true);
+ storeBlockTupleBinding = new StoreBlockTupleBinding();
+ longTupleBinding = TupleBinding.getPrimitiveBinding(Long.class);
+ AccessTimeKeyCreator accessTimeKeyCreator =
+ new AccessTimeKeyCreator(storeBlockTupleBinding);
+ secDbConfig.setKeyCreator(accessTimeKeyCreator);
+ chkDB_accessTime = environment.openSecondaryDatabase
+ (null,
"CHK_accessTime", chkDB, secDbConfig);
+
+ // Initialize other secondary database sorted on block number
+// try {
+// environment.removeDatabase(null, "CHK_blockNum");
+// } catch (DatabaseNotFoundException e) { };
+ SecondaryConfig blockNoDbConfig = new SecondaryConfig();
+ blockNoDbConfig.setAllowCreate(true);
+ blockNoDbConfig.setSortedDuplicates(false);
+ blockNoDbConfig.setAllowPopulate(true);
+ blockNoDbConfig.setTransactional(true);
+
+ BlockNumberKeyCreator bnkc =
+ new BlockNumberKeyCreator(storeBlockTupleBinding);
+ blockNoDbConfig.setKeyCreator(bnkc);
+ SecondaryDatabase blockNums;
+ System.err.println("Creating block db index");
+ chkDB_blockNum = environment.openSecondaryDatabase
+ (null, "CHK_blockNum", chkDB, blockNoDbConfig);
+
+ // Initialize the store file
+ File storeFile = new File(dir,"store");
+ if(!storeFile.exists())
+ storeFile.createNewFile();
+ chkStore = new RandomAccessFile(storeFile,"rw");
+
+ chkBlocksInStore = 0;
+
+ lastRecentlyUsed = 0;
+
+ reconstruct(type, storeDir);
+
+ chkBlocksInStore = countCHKBlocks();
+ lastRecentlyUsed = getMaxRecentlyUsed();
+
+// Add shutdownhook
+ Runtime.getRuntime().addShutdownHook(new ShutdownHook());
+ }
+
+ private void reconstruct(short type, String storeDir) throws
DatabaseException {
+ if(type == TYPE_SSK) {
+ System.err.println("Reconstruction of SSK store not
supported at present.");
+ throw new UnsupportedOperationException("Reconstruction
of SSK store not supported at present.");
+ // FIXME we would need to pass in a means to fetch the
pubkeys (an already-working BDBFS maybe).
+ // This could be via an interface. It might be
implemented by the node so we can use the in-RAM cache.
+ }
+ System.err.println("Reconstructing store index from store file:
type="+type);
+ Logger.error(this, "Reconstructing store index from store file:
type="+type);
+ byte[] header = new byte[headerBlockSize];
+ byte[] data = new byte[dataBlockSize];
+ try {
+ chkStore.seek(0);
+ long l = 0;
+ while(true) {
+ Transaction t = null;
+ try {
+ chkStore.readFully(header);
+ chkStore.readFully(data);
+ byte[] routingkey = null;
+ if(type == TYPE_CHK) {
+ try {
+ CHKBlock chk = new
CHKBlock(header, data, null);
+ routingkey =
chk.getKey().getRoutingKey();
+ } catch (CHKVerifyException e) {
+ Logger.error(this,
"Bogus key at slot "+l+" : "+e, e);
+ }
+ } else if(type == TYPE_PUBKEY) {
+ DSAPublicKey key = new
DSAPublicKey(data);
+ routingkey = key.asBytesHash();
+ } else {
+ l++;
+ continue;
+ }
+ t =
environment.beginTransaction(null,null);
+ long blockNum = chkBlocksInStore++;
+ StoreBlock storeBlock = new
StoreBlock(blockNum);
+ DatabaseEntry routingkeyDBE = new
DatabaseEntry(routingkey);
+ DatabaseEntry blockDBE = new
DatabaseEntry();
+
storeBlockTupleBinding.objectToEntry(storeBlock, blockDBE);
+ chkDB.put(t,routingkeyDBE,blockDBE);
+ t.commit();
+ t = null;
+ } finally {
+ l++;
+ if(t != null) t.abort();
+ }
+ }
+ } catch (EOFException e) {
+ migrate(storeDir);
+ return;
+ } catch (IOException e) {
+ Logger.error(this, "Caught "+e, e);
+ throw new Error(e);
+ // What else can we do? FIXME
+ }
+ }
+
+ /**
* Migrate from a store which didn't have a unique index on blockNum,
to one which does.
* How do we do this? We scan through all entries (slow), we fetch each
key, delete all data's
* under it, and then insert the one we are looking at.
