Author: toad
Date: 2006-06-07 14:25:22 +0000 (Wed, 07 Jun 2006)
New Revision: 9070
Modified:
trunk/freenet/src/freenet/node/Node.java
trunk/freenet/src/freenet/node/PacketSender.java
trunk/freenet/src/freenet/node/Version.java
trunk/freenet/src/freenet/store/BerkeleyDBFreenetStore.java
trunk/freenet/src/freenet/support/DoublyLinkedListImpl.java
Log:
783:
Datastore fixes:
- Check whether a pubkey exists in the pubkey store before adding it. This was
causing dupes in the pubkey store.
- Add a unique index on block number to the datastore. Migrate automatically on
startup, deleting any dupes.
- Get the next block number more accurately; again, avoid making dupes on the
same block number.
Don't start the watchdog until we start the PacketSender thread it watches;
long startup will no longer cause a restart.
Fix an exception in DLList.remove() that was happening when fetching stuff.
Modified: trunk/freenet/src/freenet/node/Node.java
===================================================================
--- trunk/freenet/src/freenet/node/Node.java 2006-06-07 11:42:20 UTC (rev
9069)
+++ trunk/freenet/src/freenet/node/Node.java 2006-06-07 14:25:22 UTC (rev
9070)
@@ -598,8 +598,12 @@
// USK inserter.
private final MyARKInserter arkPutter;
+ // The node starter
private static NodeStarter nodeStarter;
+ // Debugging stuff
+ private static final boolean USE_RAM_PUBKEYS_CACHE = true;
+
/**
* Read all storable settings (identity etc) from the node file.
* @param filename The name of the file to read from.
@@ -854,9 +858,8 @@
}
}
- Node(Config config, RandomSource random, LoggingConfigHandler lc,
NodeStarter ns) throws NodeInitException{
- this(config, random, lc);
- nodeStarter=ns;
+ Node(Config config, RandomSource random, LoggingConfigHandler lc) throws
NodeInitException{
+ this(config, random, lc, null);
}
public boolean isUsingWrapper(){
@@ -879,11 +882,11 @@
* @param the loggingHandler
* @throws NodeInitException If the node initialization fails.
*/
- Node(Config config, RandomSource random, LoggingConfigHandler lc) throws
NodeInitException {
+ Node(Config config, RandomSource random, LoggingConfigHandler lc,
NodeStarter ns) throws NodeInitException {
// Easy stuff
- nodeStarter=null;
+ nodeStarter=ns;
if(logConfigHandler != lc)
- this.logConfigHandler=lc;
+ logConfigHandler=lc;
arkPutter = new MyARKInserter();
startupTime = System.currentTimeMillis();
throttleWindow = new ThrottleWindowManager(2.0);
@@ -2624,12 +2627,14 @@
public DSAPublicKey getKey(byte[] hash) {
ImmutableByteArrayWrapper w = new
ImmutableByteArrayWrapper(hash);
Logger.minor(this, "Getting pubkey: "+HexUtil.bytesToHex(hash));
- synchronized(cachedPubKeys) {
- DSAPublicKey key = (DSAPublicKey) cachedPubKeys.get(w);
- if(key != null) {
- cachedPubKeys.push(w, key);
- Logger.minor(this, "Got
"+HexUtil.bytesToHex(hash)+" from cache");
- return key;
+ if(USE_RAM_PUBKEYS_CACHE) {
+ synchronized(cachedPubKeys) {
+ DSAPublicKey key = (DSAPublicKey)
cachedPubKeys.get(w);
+ if(key != null) {
+ cachedPubKeys.push(w, key);
+ Logger.minor(this, "Got
"+HexUtil.bytesToHex(hash)+" from cache");
+ return key;
+ }
}
}
try {
@@ -2650,6 +2655,7 @@
* Cache a public key
*/
public void cacheKey(byte[] hash, DSAPublicKey key) {
+ Logger.minor(this, "Cache key: "+HexUtil.bytesToHex(hash)+" :
"+key);
ImmutableByteArrayWrapper w = new
ImmutableByteArrayWrapper(hash);
synchronized(cachedPubKeys) {
DSAPublicKey key2 = (DSAPublicKey) cachedPubKeys.get(w);
@@ -2684,6 +2690,7 @@
}
try {
pubKeyDatastore.put(hash, key);
+ pubKeyDatastore.fetchPubKey(hash, true);
} catch (IOException e) {
// FIXME deal with disk full, access perms etc; tell
user about it.
Logger.error(this, "Error accessing pubkey store: "+e,
e);
Modified: trunk/freenet/src/freenet/node/PacketSender.java
===================================================================
--- trunk/freenet/src/freenet/node/PacketSender.java 2006-06-07 11:42:20 UTC
(rev 9069)
+++ trunk/freenet/src/freenet/node/PacketSender.java 2006-06-07 14:25:22 UTC
(rev 9070)
@@ -39,10 +39,6 @@
myThread = new Thread(this, "PacketSender thread for
"+node.portNumber);
myThread.setDaemon(true);
lastTimeInSeconds = (int) (System.currentTimeMillis() / 1000);
- // Necessary because of sun JVM bugs when NPTL is enabled. Write once,
debug everywhere!
- Thread t1 = new Thread(new Watchdog(), "PacketSender watchdog");
- t1.setDaemon(true);
- t1.start();
}
/**
@@ -76,6 +72,9 @@
WrapperManager.requestThreadDump();
WrapperManager.restart();
}else{
+
if(!Node.logConfigHandler.getFileLoggerHook().hasRedirectedStdOutErrNoLock())
+
System.err.println("Exiting on deadlock, but not running in the wrapper! Please
restart the node manually.");
+
// No wrapper : we
don't want to let it harm the network!
node.exit();
}
@@ -95,6 +94,10 @@
void start() {
Logger.normal(this,"Starting the PacketSender thread");
+ // Necessary because of sun JVM bugs when NPTL is enabled. Write once,
debug everywhere!
+ Thread t1 = new Thread(new Watchdog(), "PacketSender watchdog");
+ t1.setDaemon(true);
+ t1.start();
myThread.start();
}
Modified: trunk/freenet/src/freenet/node/Version.java
===================================================================
--- trunk/freenet/src/freenet/node/Version.java 2006-06-07 11:42:20 UTC (rev
9069)
+++ trunk/freenet/src/freenet/node/Version.java 2006-06-07 14:25:22 UTC (rev
9070)
@@ -18,7 +18,7 @@
public static final String protocolVersion = "1.0";
/** The build number of the current revision */
- private static final int buildNumber = 782;
+ private static final int buildNumber = 783;
/** Oldest build of Fred we will talk to */
private static final int lastGoodBuild = 765;
Modified: trunk/freenet/src/freenet/store/BerkeleyDBFreenetStore.java
===================================================================
--- trunk/freenet/src/freenet/store/BerkeleyDBFreenetStore.java 2006-06-07
11:42:20 UTC (rev 9069)
+++ trunk/freenet/src/freenet/store/BerkeleyDBFreenetStore.java 2006-06-07
14:25:22 UTC (rev 9070)
@@ -5,16 +5,17 @@
import java.io.IOException;
import java.io.RandomAccessFile;
import java.util.Arrays;
+import java.util.HashSet;
import com.sleepycat.bind.tuple.TupleBinding;
import com.sleepycat.bind.tuple.TupleInput;
import com.sleepycat.bind.tuple.TupleOutput;
-import com.sleepycat.je.BtreeStats;
import com.sleepycat.je.Cursor;
import com.sleepycat.je.Database;
import com.sleepycat.je.DatabaseConfig;
import com.sleepycat.je.DatabaseEntry;
import com.sleepycat.je.DatabaseException;
+import com.sleepycat.je.DatabaseNotFoundException;
import com.sleepycat.je.Environment;
import com.sleepycat.je.EnvironmentConfig;
import com.sleepycat.je.LockMode;
@@ -58,6 +59,7 @@
private long maxChkBlocks;
private final Database chkDB;
private final Database chkDB_accessTime;
+ private final Database chkDB_blockNum;
private final RandomAccessFile chkStore;
private long lastRecentlyUsed;
@@ -129,6 +131,38 @@
chkDB_accessTime = environment.openSecondaryDatabase
(null,
"CHK_accessTime", chkDB, secDbConfig);
+ // Initialize other secondary database sorted on block number
+// try {
+// environment.removeDatabase(null, "CHK_blockNum");
+// } catch (DatabaseNotFoundException e) { };
+ SecondaryConfig blockNoDbConfig = new SecondaryConfig();
+ blockNoDbConfig.setAllowCreate(false);
+ blockNoDbConfig.setSortedDuplicates(false);
+ blockNoDbConfig.setAllowPopulate(true);
+ blockNoDbConfig.setTransactional(true);
+
+ BlockNumberKeyCreator bnkc =
+ new BlockNumberKeyCreator(storeBlockTupleBinding);
+ blockNoDbConfig.setKeyCreator(bnkc);
+ SecondaryDatabase blockNums;
+ try {
+ System.err.println("Opening block db index");
+ blockNums = environment.openSecondaryDatabase
+ (null, "CHK_blockNum", chkDB, blockNoDbConfig);
+ } catch (DatabaseNotFoundException e) {
+ System.err.println("Migrating block db index");
+ // De-dupe on keys and block numbers.
+ migrate(storeDir);
+ System.err.println("De-duped, creating new index...");
+ blockNoDbConfig.setSortedDuplicates(false);
+ blockNoDbConfig.setAllowCreate(true);
+ blockNoDbConfig.setAllowPopulate(true);
+ blockNums = environment.openSecondaryDatabase
+ (null, "CHK_blockNum", chkDB, blockNoDbConfig);
+ }
+
+ chkDB_blockNum = blockNums;
+
// Initialize the store file
File storeFile = new File(dir,"store");
if(!storeFile.exists())
@@ -143,6 +177,60 @@
}
/**
+ * Migrate from a store which didn't have a unique index on blockNum,
to one which does.
+ * How do we do this? We scan through all entries (slow), we fetch each
key, delete all data's
+ * under it, and then insert the one we are looking at.
+ *
+ * FIXME: Create a list of reusable block numbers?
+ */
+ private void migrate(String storeDir) throws DatabaseException {
+
+ System.err.println("Migrating database "+storeDir+": Creating
unique index on block number");
+ HashSet s = new HashSet();
+
+ Cursor c = null;
+ Transaction t = null;
+ t = environment.beginTransaction(null,null);
+ c = chkDB.openCursor(t,null);
+ DatabaseEntry keyDBE = new DatabaseEntry();
+ DatabaseEntry blockDBE = new DatabaseEntry();
+ OperationStatus opStat;
+ opStat = c.getLast(keyDBE, blockDBE, LockMode.RMW);
+ if(opStat == OperationStatus.NOTFOUND) {
+ System.err.println("Database is empty.");
+ return;
+ }
+ Logger.minor(this, "Found first key");
+ try {
+ int x = 0;
+ while(true) {
+ StoreBlock storeBlock = (StoreBlock)
storeBlockTupleBinding.entryToObject(blockDBE);
+ Logger.minor(this, "Found another key
("+(x++)+") ("+storeBlock.offset+")");
+ Long l = new Long(storeBlock.offset);
+ if(s.contains(l)) {
+ Logger.minor(this, "Deleting (block
number conflict).");
+ chkDB.delete(t, keyDBE);
+ }
+ s.add(l);
+ opStat = c.getPrev(keyDBE, blockDBE,
LockMode.RMW);
+ if(opStat == OperationStatus.NOTFOUND) {
+ return;
+ }
+ }
+ } catch (DatabaseException e) {
+ System.err.println("Caught: "+e);
+ e.printStackTrace();
+ Logger.error(this, "Caught "+e, e);
+ t.abort();
+ t = null;
+ } finally {
+ c.close();
+ if(t != null)
+ t.commit();
+ }
+ }
+
+ /**
* Retrieve a block.
* @param dontPromote If true, don't promote data if fetched.
* @return null if there is no such block stored, otherwise the block.
@@ -218,7 +306,7 @@
Logger.minor(this, "Data: "+data.length+" bytes, hash
"+data);
}catch(CHKVerifyException ex){
- Logger.normal(this, "CHKBlock: Does not verify
("+ex+"), setting accessTime to 0 for : "+chk);
+ Logger.error(this, "CHKBlock: Does not verify ("+ex+"),
setting accessTime to 0 for : "+chk);
storeBlock.setRecentlyUsedToZero();
DatabaseEntry updateDBE = new DatabaseEntry();
storeBlockTupleBinding.objectToEntry(storeBlock,
updateDBE);
@@ -302,7 +390,7 @@
Logger.minor(this, "Data: "+data.length+" bytes, hash
"+data);
}catch(SSKVerifyException ex){
- Logger.normal(this, "SSHBlock: Does not verify
("+ex+"), setting accessTime to 0 for : "+chk, ex);
+ Logger.normal(this, "SSKBlock: Does not verify
("+ex+"), setting accessTime to 0 for : "+chk, ex);
storeBlock.setRecentlyUsedToZero();
DatabaseEntry updateDBE = new DatabaseEntry();
storeBlockTupleBinding.objectToEntry(storeBlock,
updateDBE);
@@ -377,7 +465,7 @@
}
if(!Arrays.equals(block.asBytesHash(), hash)) {
- Logger.normal(this, "DSAPublicKey: Does not
verify (unequal hashes), setting accessTime to 0 for :
"+HexUtil.bytesToHex(hash));
+ Logger.error(this, "DSAPublicKey: Does not
verify (unequal hashes), setting accessTime to 0 for :
"+HexUtil.bytesToHex(hash));
storeBlock.setRecentlyUsedToZero();
DatabaseEntry updateDBE = new DatabaseEntry();
storeBlockTupleBinding.objectToEntry(storeBlock, updateDBE);
@@ -592,10 +680,19 @@
}
}
+ /**
+ * Store a pubkey.
+ */
+ public void put(byte[] hash, DSAPublicKey key) throws IOException {
+ DSAPublicKey k = fetchPubKey(hash, true);
+ if(k == null)
+ innerPut(hash, key);
+ }
+
/**
* Store a block.
*/
- public void put(byte[] hash, DSAPublicKey key) throws IOException
+ public void innerPut(byte[] hash, DSAPublicKey key) throws IOException
{
if(closed)
return;
@@ -603,6 +700,11 @@
byte[] routingkey = hash;
byte[] data = key.asPaddedBytes();
+ if(!(Arrays.equals(hash, key.asBytesHash()))) {
+ Logger.error(this, "Invalid hash!: "+HexUtil.bytesToHex(hash)+"
: "+key.asBytesHash());
+ }
+
+
if(data.length!=dataBlockSize) {
Logger.error(this, "This data is "+data.length+" bytes. Should
be "+dataBlockSize);
return;
@@ -741,6 +843,26 @@
}
}
+ private class BlockNumberKeyCreator implements SecondaryKeyCreator {
+ private TupleBinding theBinding;
+
+ public BlockNumberKeyCreator(TupleBinding theBinding1) {
+ theBinding = theBinding1;
+ }
+
+ public boolean createSecondaryKey(SecondaryDatabase secDb,
+ DatabaseEntry keyEntry,
+ DatabaseEntry dataEntry,
+ DatabaseEntry resultEntry) {
+
+ StoreBlock storeblock = (StoreBlock)
theBinding.entryToObject(dataEntry);
+ Long blockNo = new Long(storeblock.offset);
+ longTupleBinding.objectToEntry(blockNo, resultEntry);
+ return true;
+ }
+
+ }
+
private class ShutdownHook extends Thread
{
public void run() {
@@ -760,6 +882,7 @@
Thread.sleep(5000);
chkStore.close();
chkDB_accessTime.close();
+ chkDB_blockNum.close();
chkDB.close();
environment.close();
Logger.minor(this, "Closing database finished.");
@@ -769,17 +892,30 @@
}
}
- private long countCHKBlocks() {
- long count =0;
+ private long countCHKBlocks() throws IOException {
+ long count = 0;
+
try{
- Logger.minor(this, "Started counting items in database");
-
- count = ((BtreeStats)chkDB.getStats(null)).getLeafNodeCount();
-
- Logger.minor(this, "Counted "+count+" items in database");
- }catch(DatabaseException ex){
- Logger.minor(this, "Exception while counting items in
database",ex);
+ Cursor c = chkDB_blockNum.openCursor(null,null);
+ DatabaseEntry keyDBE = new DatabaseEntry();
+ DatabaseEntry dataDBE = new DatabaseEntry();
+
if(c.getLast(keyDBE,dataDBE,null)==OperationStatus.SUCCESS) {
+ StoreBlock storeBlock = (StoreBlock)
storeBlockTupleBinding.entryToObject(dataDBE);
+ count = storeBlock.offset;
+ }
+ c.close();
+ }catch(DatabaseException ex){ex.printStackTrace();}
+
+ count++;
+ System.out.println("Count from database: "+count);
+
+ long oCount = chkStore.length() / (dataBlockSize + headerBlockSize);
+
+ if(oCount > count) {
+ System.err.println("Count from file length: "+oCount);
+ return oCount;
}
+
return count;
}
Modified: trunk/freenet/src/freenet/support/DoublyLinkedListImpl.java
===================================================================
--- trunk/freenet/src/freenet/support/DoublyLinkedListImpl.java 2006-06-07
11:42:20 UTC (rev 9069)
+++ trunk/freenet/src/freenet/support/DoublyLinkedListImpl.java 2006-06-07
14:25:22 UTC (rev 9070)
@@ -238,11 +238,11 @@
* @return this item, or null if the item was not in the list
*/
public DoublyLinkedList.Item remove(DoublyLinkedList.Item i) {
+ if (i.getParent() == null) return null; // not in list
if(isEmpty()) {
Logger.error(this, "Illegal ERROR: Removing from an empty
list!!");
throw new IllegalStateException("Illegal ERROR: Removing from
an empty list!!");
}
- if (i.getParent() == null) return null; // not in list
if (i.getParent() != this)
throw new PromiscuousItemException(i, i.getParent());
DoublyLinkedList.Item next = i.getNext(), prev = i.getPrev();
