Author: toad
Date: 2006-07-03 15:16:37 +0000 (Mon, 03 Jul 2006)
New Revision: 9438
Added:
trunk/freenet/src/freenet/node/DSAPublicKeyDatabase.java
trunk/freenet/src/freenet/support/SortedLongSet.java
Modified:
trunk/freenet/src/freenet/keys/SSKBlock.java
trunk/freenet/src/freenet/node/Node.java
trunk/freenet/src/freenet/node/Version.java
trunk/freenet/src/freenet/store/BerkeleyDBFreenetStore.java
trunk/freenet/src/freenet/store/FreenetStore.java
trunk/freenet/src/freenet/support/SortedVectorByNumber.java
Log:
852: Lots of work on the datastore.
- Online and offline shrinking.
- Better handling of errors.
- Automatic recovery from corrupted index (for CHK and pubkey stores).
- Free blocks list (not saved to disk, but used).
- Refactoring
Modified: trunk/freenet/src/freenet/keys/SSKBlock.java
===================================================================
--- trunk/freenet/src/freenet/keys/SSKBlock.java 2006-07-03 02:11:25 UTC
(rev 9437)
+++ trunk/freenet/src/freenet/keys/SSKBlock.java 2006-07-03 15:16:37 UTC
(rev 9438)
@@ -8,6 +8,7 @@
import freenet.crypt.DSA;
import freenet.crypt.DSAPublicKey;
import freenet.crypt.DSASignature;
+import freenet.node.DSAPublicKeyDatabase;
import freenet.support.HexUtil;
/**
@@ -69,7 +70,7 @@
// only compare some of the headers (see top)
for (int i = 0; i < HEADER_COMPARE_TO; i++) {
if (block.headers[i] != headers[i]) return false;
- }
+ }
//if(!Arrays.equals(block.headers, headers)) return false;
if(!Arrays.equals(block.data, data)) return false;
return true;
Added: trunk/freenet/src/freenet/node/DSAPublicKeyDatabase.java
===================================================================
--- trunk/freenet/src/freenet/node/DSAPublicKeyDatabase.java 2006-07-03
02:11:25 UTC (rev 9437)
+++ trunk/freenet/src/freenet/node/DSAPublicKeyDatabase.java 2006-07-03
15:16:37 UTC (rev 9438)
@@ -0,0 +1,17 @@
+package freenet.node;
+
+import freenet.crypt.DSAPublicKey;
+
+/**
+ * Interface for a DSA public key lookup service.
+ */
+public interface DSAPublicKeyDatabase {
+
+ /**
+ * Lookup a key by its hash.
+ * @param hash
+ * @return The key, or null.
+ */
+ public DSAPublicKey lookupKey(byte[] hash);
+
+}
Modified: trunk/freenet/src/freenet/node/Node.java
===================================================================
--- trunk/freenet/src/freenet/node/Node.java 2006-07-03 02:11:25 UTC (rev
9437)
+++ trunk/freenet/src/freenet/node/Node.java 2006-07-03 15:16:37 UTC (rev
9438)
@@ -609,6 +609,9 @@
FProxyToadlet fproxyServlet;
SimpleToadletServer toadletContainer;
+ // The version we were before we restarted.
+ public int lastVersion;
+
/** NodeUpdater **/
public NodeUpdater nodeUpdater;
@@ -704,6 +707,14 @@
if(myName == null) {
myName = newName();
}
+
+ String verString = fs.get("version");
+ if(verString == null) {
+ Logger.error(this, "No version!");
+ System.err.println("No version!");
+ } else {
+ lastVersion =
Version.getArbitraryBuildNumber(verString);
+ }
// FIXME: Back compatibility; REMOVE !!
try {
@@ -1341,9 +1352,20 @@
if(newMaxStoreKeys ==
maxStoreKeys) return;
// Update each datastore
maxStoreKeys = newMaxStoreKeys;
-
chkDatastore.setMaxKeys(maxStoreKeys);
-
sskDatastore.setMaxKeys(maxStoreKeys);
-
pubKeyDatastore.setMaxKeys(maxStoreKeys);
+ try {
+
chkDatastore.setMaxKeys(maxStoreKeys);
+
sskDatastore.setMaxKeys(maxStoreKeys);
+
pubKeyDatastore.setMaxKeys(maxStoreKeys);
+ } catch (IOException e) {
+ // FIXME we need to be
able to tell the user.
+ Logger.error(this,
"Caught "+e+" resizing the datastore", e);
+
System.err.println("Caught "+e+" resizing the datastore");
+ e.printStackTrace();
+ } catch (DatabaseException e) {
+ Logger.error(this,
"Caught "+e+" resizing the datastore", e);
+
System.err.println("Caught "+e+" resizing the datastore");
+ e.printStackTrace();
+ }
}
});
@@ -1378,23 +1400,35 @@
System.out.println("Initializing CHK Datastore");
BerkeleyDBFreenetStore tmp;
try {
- tmp = new
BerkeleyDBFreenetStore(storeDir.getPath()+File.separator+"store-"+portNumber,
maxStoreKeys, 32768, CHKBlock.TOTAL_HEADERS_LENGTH);
+ if(lastVersion > 0 && lastVersion < 852) {
+ throw new
DatabaseException("Reconstructing store because started from old version");
+ }
+ tmp = new
BerkeleyDBFreenetStore(storeDir.getPath()+File.separator+"store-"+portNumber,
maxStoreKeys, 32768, CHKBlock.TOTAL_HEADERS_LENGTH, true);
} catch (DatabaseException e) {
+ System.err.println("Could not open store: "+e);
+ e.printStackTrace();
+ System.err.println("Attempting to
reconstruct...");
tmp = new
BerkeleyDBFreenetStore(storeDir.getPath()+File.separator+"store-"+portNumber,
maxStoreKeys, 32768, CHKBlock.TOTAL_HEADERS_LENGTH,
BerkeleyDBFreenetStore.TYPE_CHK);
}
chkDatastore = tmp;
Logger.normal(this, "Initializing pubKey Datastore");
System.out.println("Initializing pubKey Datastore");
try {
- tmp = new
BerkeleyDBFreenetStore(storeDir.getPath()+File.separator+"pubkeystore-"+portNumber,
maxStoreKeys, DSAPublicKey.PADDED_SIZE, 0);
+ if(lastVersion > 0 && lastVersion < 852) {
+ throw new
DatabaseException("Reconstructing store because started from old version");
+ }
+ tmp = new
BerkeleyDBFreenetStore(storeDir.getPath()+File.separator+"pubkeystore-"+portNumber,
maxStoreKeys, DSAPublicKey.PADDED_SIZE, 0, true);
} catch (DatabaseException e) {
+ System.err.println("Could not open store: "+e);
+ e.printStackTrace();
+ System.err.println("Attempting to
reconstruct...");
tmp = new
BerkeleyDBFreenetStore(storeDir.getPath()+File.separator+"pubkeystore-"+portNumber,
maxStoreKeys, DSAPublicKey.PADDED_SIZE, 0, BerkeleyDBFreenetStore.TYPE_PUBKEY);
}
this.pubKeyDatastore = tmp;
// FIXME can't auto-fix SSK stores.
Logger.normal(this, "Initializing SSK Datastore");
System.out.println("Initializing SSK Datastore");
- sskDatastore = new
BerkeleyDBFreenetStore(storeDir.getPath()+File.separator+"sskstore-"+portNumber,
maxStoreKeys, 1024, SSKBlock.TOTAL_HEADERS_LENGTH);
+ sskDatastore = new
BerkeleyDBFreenetStore(storeDir.getPath()+File.separator+"sskstore-"+portNumber,
maxStoreKeys, 1024, SSKBlock.TOTAL_HEADERS_LENGTH, false);
} catch (FileNotFoundException e1) {
String msg = "Could not open datastore: "+e1;
Logger.error(this, msg, e1);
@@ -1409,6 +1443,7 @@
String msg = "Could not open datastore: "+e1;
Logger.error(this, msg, e1);
System.err.println(msg);
+ e1.printStackTrace();
throw new NodeInitException(EXIT_STORE_OTHER, msg);
}
Modified: trunk/freenet/src/freenet/node/Version.java
===================================================================
--- trunk/freenet/src/freenet/node/Version.java 2006-07-03 02:11:25 UTC (rev
9437)
+++ trunk/freenet/src/freenet/node/Version.java 2006-07-03 15:16:37 UTC (rev
9438)
@@ -18,7 +18,7 @@
public static final String protocolVersion = "1.0";
/** The build number of the current revision */
- private static final int buildNumber = 851;
+ private static final int buildNumber = 852;
/** Oldest build of Fred we will talk to */
private static final int oldLastGoodBuild = 839;
Modified: trunk/freenet/src/freenet/store/BerkeleyDBFreenetStore.java
===================================================================
--- trunk/freenet/src/freenet/store/BerkeleyDBFreenetStore.java 2006-07-03
02:11:25 UTC (rev 9437)
+++ trunk/freenet/src/freenet/store/BerkeleyDBFreenetStore.java 2006-07-03
15:16:37 UTC (rev 9438)
@@ -37,6 +37,7 @@
import freenet.support.Fields;
import freenet.support.HexUtil;
import freenet.support.Logger;
+import freenet.support.SortedLongSet;
/**
* Freenet datastore based on BerkelyDB Java Edition by sleepycat software
@@ -68,20 +69,23 @@
private final Database chkDB_accessTime;
private final Database chkDB_blockNum;
private final RandomAccessFile chkStore;
+ private final SortedLongSet freeBlocks;
private long lastRecentlyUsed;
private final Object lastRecentlyUsedSync = new Object();
private boolean closed = false;
+ private final static byte[] dummy = new byte[0];
/**
* Initializes database
* @param the directory where the store is located
* @throws FileNotFoundException if the dir does not exist and could not
be created
*/
- public BerkeleyDBFreenetStore(String storeDir, long maxChkBlocks, int
blockSize, int headerSize) throws Exception {
+ public BerkeleyDBFreenetStore(String storeDir, long maxChkBlocks, int
blockSize, int headerSize, boolean throwOnTooFewKeys) throws Exception {
this.dataBlockSize = blockSize;
this.headerBlockSize = headerSize;
+ this.freeBlocks = new SortedLongSet();
// Percentage of the database that must contain usefull data
// decrease to increase performance, increase to save disk space
System.setProperty("je.cleaner.minUtilization","98");
@@ -177,13 +181,102 @@
storeFile.createNewFile();
chkStore = new RandomAccessFile(storeFile,"rw");
- chkBlocksInStore = countCHKBlocks();
+ chkBlocksInStore = countCHKBlocksFromDatabase();
+ long chkBlocksFromFile = countCHKBlocksFromFile();
lastRecentlyUsed = getMaxRecentlyUsed();
+ if((chkBlocksInStore == 0 && chkBlocksFromFile != 0) ||
+ ((chkBlocksInStore + 10) * 1.1) <
chkBlocksFromFile) {
+ if(throwOnTooFewKeys) {
+ try {
+ close();
+ } catch (Throwable t) {
+ Logger.error(this, "Failed to close:
"+t, t);
+ System.err.println("Failed to close:
"+t);
+ t.printStackTrace();
+ }
+ throw new DatabaseException("Keys in database:
"+chkBlocksInStore+" but keys in file: "+chkBlocksFromFile);
+ } else
+ checkForHoles(chkBlocksFromFile);
+ }
+
+ chkBlocksInStore = Math.max(chkBlocksInStore,
chkBlocksFromFile);
+ Logger.minor(this, "Keys in store: "+chkBlocksInStore);
+ System.out.println("Keys in store: "+chkBlocksInStore+" /
"+maxChkBlocks);
+
+ maybeShrink(true);
+
// Add shutdownhook
Runtime.getRuntime().addShutdownHook(new ShutdownHook());
}
+ private void checkForHoles(long blocksInFile) throws DatabaseException {
+ System.err.println("Checking for holes in database...");
+ long holes = 0;
+ for(long i=0;i<blocksInFile;i++) {
+ Long blockNo = new Long(i);
+ DatabaseEntry blockNumEntry = new DatabaseEntry();
+ DatabaseEntry found = new DatabaseEntry();
+ longTupleBinding.objectToEntry(blockNo, blockNumEntry);
+
+ OperationStatus success =
+ chkDB_blockNum.get(null, blockNumEntry, found,
LockMode.DEFAULT);
+
+ if(success.equals(OperationStatus.NOTFOUND)) {
+ addFreeBlock(i);
+ holes++;
+ }
+ if(i % 1024 == 0)
+ System.err.println("Checked "+i+" blocks, found
"+holes+" holes");
+ }
+ System.err.println("Checked database, found "+holes+" holes");
+ }
+
+ private void maybeShrink(boolean dontCheck) throws DatabaseException,
IOException {
+ Transaction t = null;
+ try {
+ if(maxChkBlocks >= chkBlocksInStore)
+ return;
+ System.err.println("Shrinking store:
"+chkBlocksInStore+" -> "+maxChkBlocks);
+ Logger.error(this, "Shrinking store:
"+chkBlocksInStore+" -> "+maxChkBlocks);
+ while(true) {
+ t = environment.beginTransaction(null,null);
+ long deleted = 0;
+ for(long
i=chkBlocksInStore-1;i>=maxChkBlocks;i--) {
+
+ // Delete the block with this blocknum.
+
+ Long blockNo = new Long(i);
+ DatabaseEntry blockNumEntry = new
DatabaseEntry();
+ longTupleBinding.objectToEntry(blockNo,
blockNumEntry);
+
+ OperationStatus result =
+ chkDB_blockNum.delete(t,
blockNumEntry);
+
if(result.equals(OperationStatus.SUCCESS))
+ deleted++;
+ }
+
+ t.commit();
+
+ System.err.println("Deleted "+deleted+" keys");
+
+ t = null;
+
+ if(deleted == 0 || dontCheck) break;
+ else
+ System.err.println("Checking...");
+ }
+
+ chkStore.setLength(maxChkBlocks * (dataBlockSize +
headerBlockSize));
+
+ chkBlocksInStore = maxChkBlocks;
+ System.err.println("Successfully shrunk store to
"+chkBlocksInStore);
+
+ } finally {
+ if(t != null) t.abort();
+ }
+ }
+
public static final short TYPE_CHK = 0;
public static final short TYPE_PUBKEY = 1;
public static final short TYPE_SSK = 2;
@@ -196,6 +289,7 @@
public BerkeleyDBFreenetStore(String storeDir, long maxChkBlocks, int
blockSize, int headerSize, short type) throws Exception {
this.dataBlockSize = blockSize;
this.headerBlockSize = headerSize;
+ this.freeBlocks = new SortedLongSet();
// Percentage of the database that must contain usefull data
// decrease to increase performance, increase to save disk space
System.setProperty("je.cleaner.minUtilization","98");
@@ -267,7 +361,6 @@
BlockNumberKeyCreator bnkc =
new BlockNumberKeyCreator(storeBlockTupleBinding);
blockNoDbConfig.setKeyCreator(bnkc);
- SecondaryDatabase blockNums;
System.err.println("Creating block db index");
chkDB_blockNum = environment.openSecondaryDatabase
(null, "CHK_blockNum", chkDB, blockNoDbConfig);
@@ -283,10 +376,12 @@
lastRecentlyUsed = 0;
reconstruct(type, storeDir);
-
- chkBlocksInStore = countCHKBlocks();
+
+ chkBlocksInStore = countCHKBlocksFromFile();
lastRecentlyUsed = getMaxRecentlyUsed();
+ maybeShrink(true);
+
// Add shutdownhook
Runtime.getRuntime().addShutdownHook(new ShutdownHook());
}
@@ -313,10 +408,16 @@
byte[] routingkey = null;
if(type == TYPE_CHK) {
try {
- CHKBlock chk = new
CHKBlock(header, data, null);
+ CHKBlock chk = new
CHKBlock(data, header, null);
routingkey =
chk.getKey().getRoutingKey();
} catch (CHKVerifyException e) {
- Logger.error(this,
"Bogus key at slot "+l+" : "+e, e);
+ String err = "Bogus key
at slot "+l+" : "+e+" - lost block "+l;
+ Logger.error(this, err,
e);
+ System.err.println(err);
+ e.printStackTrace();
+ addFreeBlock(l);
+ routingkey = null;
+ continue;
}
} else if(type == TYPE_PUBKEY) {
DSAPublicKey key = new
DSAPublicKey(data);
@@ -333,6 +434,8 @@
storeBlockTupleBinding.objectToEntry(storeBlock, blockDBE);
chkDB.put(t,routingkeyDBE,blockDBE);
t.commit();
+ if(l % 1024 == 0)
+ System.out.println("Key
"+l+"/"+(chkStore.length()/(dataBlockSize+headerBlockSize))+" OK");
t = null;
} finally {
l++;
@@ -363,18 +466,22 @@
Cursor c = null;
Transaction t = null;
- t = environment.beginTransaction(null,null);
- c = chkDB.openCursor(t,null);
- DatabaseEntry keyDBE = new DatabaseEntry();
- DatabaseEntry blockDBE = new DatabaseEntry();
- OperationStatus opStat;
- opStat = c.getLast(keyDBE, blockDBE, LockMode.RMW);
- if(opStat == OperationStatus.NOTFOUND) {
- System.err.println("Database is empty.");
- return;
- }
- Logger.minor(this, "Found first key");
try {
+ t = environment.beginTransaction(null,null);
+ c = chkDB.openCursor(t,null);
+ DatabaseEntry keyDBE = new DatabaseEntry();
+ DatabaseEntry blockDBE = new DatabaseEntry();
+ OperationStatus opStat;
+ opStat = c.getLast(keyDBE, blockDBE, LockMode.RMW);
+ if(opStat == OperationStatus.NOTFOUND) {
+ System.err.println("Database is empty.");
+ c.close();
+ c = null;
+ t.abort();
+ t = null;
+ return;
+ }
+ Logger.minor(this, "Found first key");
int x = 0;
while(true) {
StoreBlock storeBlock = (StoreBlock)
storeBlockTupleBinding.entryToObject(blockDBE);
@@ -382,6 +489,7 @@
Long l = new Long(storeBlock.offset);
if(s.contains(l)) {
Logger.minor(this, "Deleting (block
number conflict).");
+ addFreeBlock(storeBlock.offset);
chkDB.delete(t, keyDBE);
}
s.add(l);
@@ -397,7 +505,8 @@
t.abort();
t = null;
} finally {
- c.close();
+ if(c != null)
+ c.close();
if(t != null)
t.commit();
}
@@ -432,9 +541,9 @@
if(c.getSearchKey(routingkeyDBE,blockDBE,LockMode.RMW)
!=OperationStatus.SUCCESS) {
c.close();
+ c = null;
t.abort();
t = null;
- c = null;
return null;
}
@@ -470,13 +579,13 @@
storeBlockTupleBinding.objectToEntry(storeBlock, updateDBE);
c.putCurrent(updateDBE);
c.close();
+ c = null;
t.commit();
- c = null;
t = null;
}else{
c.close();
+ c = null;
t.abort();
- c = null;
t = null;
}
@@ -486,12 +595,13 @@
}catch(CHKVerifyException ex){
Logger.error(this, "CHKBlock: Does not verify ("+ex+"),
setting accessTime to 0 for : "+chk);
- storeBlock.setRecentlyUsedToZero();
- DatabaseEntry updateDBE = new DatabaseEntry();
- storeBlockTupleBinding.objectToEntry(storeBlock,
updateDBE);
- c.putCurrent(updateDBE);
+ System.err.println("Does not verify (CHK block
"+storeBlock.offset+")");
c.close();
+ c = null;
+ chkDB.delete(t, routingkeyDBE);
t.commit();
+ t = null;
+ addFreeBlock(storeBlock.offset);
return null;
}
return block;
@@ -531,8 +641,8 @@
if(c.getSearchKey(routingkeyDBE,blockDBE,LockMode.RMW)
!=OperationStatus.SUCCESS) {
c.close();
+ c = null;
t.abort();
- c = null;
t = null;
return null;
}
@@ -558,13 +668,13 @@
storeBlockTupleBinding.objectToEntry(storeBlock, updateDBE);
c.putCurrent(updateDBE);
c.close();
+ c = null;
t.commit();
- c = null;
t = null;
}else{
c.close();
+ c = null;
t.abort();
- c = null;
t = null;
}
@@ -574,14 +684,12 @@
}catch(SSKVerifyException ex){
Logger.normal(this, "SSKBlock: Does not verify
("+ex+"), setting accessTime to 0 for : "+chk, ex);
- storeBlock.setRecentlyUsedToZero();
- DatabaseEntry updateDBE = new DatabaseEntry();
- storeBlockTupleBinding.objectToEntry(storeBlock,
updateDBE);
- c.putCurrent(updateDBE);
c.close();
+ c = null;
+ chkDB.delete(t, routingkeyDBE);
t.commit();
- c = null;
t = null;
+ addFreeBlock(storeBlock.offset);
return null;
}
return block;
@@ -629,8 +737,8 @@
if(c.getSearchKey(routingkeyDBE,blockDBE,LockMode.RMW)
!=OperationStatus.SUCCESS) {
c.close();
+ c = null;
t.abort();
- c = null;
t = null;
return null;
}
@@ -662,8 +770,8 @@
} catch (IOException e) {
Logger.error(this, "Could not read key");
c.close();
+ c = null;
t.abort();
- c = null;
t = null;
return null;
}
@@ -679,22 +787,20 @@
}
} else {
Logger.error(this, "DSAPublicKey: Does
not verify (unequal hashes), setting accessTime to 0 for :
"+HexUtil.bytesToHex(hash));
- storeBlock.setRecentlyUsedToZero();
- DatabaseEntry updateDBE = new
DatabaseEntry();
-
storeBlockTupleBinding.objectToEntry(storeBlock, updateDBE);
- c.putCurrent(updateDBE);
c.close();
+ c = null;
+ chkDB.delete(t, routingkeyDBE);
t.commit();
t = null;
- c = null;
+ addFreeBlock(storeBlock.offset);
return null;
}
}
// Finished, commit.
c.close();
+ c = null;
t.commit();
- c = null;
t = null;
Logger.minor(this, "Get key:
"+HexUtil.bytesToHex(hash));
@@ -717,7 +823,15 @@
// return null;
}
- public synchronized void put(CHKBlock b) throws IOException {
+ private void addFreeBlock(long offset) {
+ if(freeBlocks.push(offset)) {
+ Logger.normal(this, "Freed block "+offset);
+ } else {
+ Logger.minor(this, "Already freed block "+offset);
+ }
+ }
+
+ public synchronized void put(CHKBlock b) throws IOException {
NodeCHK chk = (NodeCHK) b.getKey();
CHKBlock oldBlock = fetch(chk, false);
if(oldBlock != null)
@@ -756,7 +870,9 @@
if(c.getSearchKey(routingkeyDBE,blockDBE,LockMode.RMW)
!=OperationStatus.SUCCESS) {
c.close();
+ c = null;
t.abort();
+ t = null;
return false;
}
@@ -821,57 +937,23 @@
t = environment.beginTransaction(null,null);
DatabaseEntry routingkeyDBE = new DatabaseEntry(routingkey);
- if(chkBlocksInStore<maxChkBlocks) {
+ // FIXME use the free blocks list!
+
+ long blockNum;
+ if((blockNum = grabFreeBlock()) >= 0) {
+ writeNewBlock(blockNum, header, data, t, routingkeyDBE);
+ } else if(chkBlocksInStore<maxChkBlocks) {
// Expand the store file
- long blockNum;
synchronized(chkBlocksInStoreLock) {
blockNum = chkBlocksInStore;
chkBlocksInStore++;
}
- long byteOffset =
blockNum*(dataBlockSize+headerBlockSize);
- StoreBlock storeBlock = new StoreBlock(blockNum);
- DatabaseEntry blockDBE = new DatabaseEntry();
- storeBlockTupleBinding.objectToEntry(storeBlock,
blockDBE);
- chkDB.put(t,routingkeyDBE,blockDBE);
- synchronized(chkStore) {
- try {
- chkStore.seek(byteOffset);
- } catch (IOException ioe) {
- if(byteOffset > (2*1024*1024*1024)) {
- Logger.error(this, "Environment
does not support files bigger than 2 GB?");
- System.out.println("Environment
does not support files bigger than 2 GB? (exception to follow)");
- }
- Logger.error(this, "Caught IOException
on chkStore.seek("+byteOffset+")");
- throw ioe;
- }
- chkStore.write(header);
- chkStore.write(data);
- }
- t.commit();
- t = null;
+ writeNewBlock(blockNum, header, data, t, routingkeyDBE);
}else{
- // Overwrite an other block
- Cursor c = chkDB_accessTime.openCursor(t,null);
- DatabaseEntry keyDBE = new DatabaseEntry();
- DatabaseEntry dataDBE = new DatabaseEntry();
- c.getFirst(keyDBE,dataDBE,LockMode.RMW);
- StoreBlock oldStoreBlock = (StoreBlock)
storeBlockTupleBinding.entryToObject(dataDBE);
- c.delete();
- c.close();
- // Deleted, so we can now reuse it.
- // Because we acquired a write lock, nobody else has
taken it.
- StoreBlock storeBlock = new
StoreBlock(oldStoreBlock.getOffset());
- DatabaseEntry blockDBE = new DatabaseEntry();
- storeBlockTupleBinding.objectToEntry(storeBlock,
blockDBE);
- chkDB.put(t,routingkeyDBE,blockDBE);
- synchronized(chkStore) {
-
chkStore.seek(storeBlock.getOffset()*(long)(dataBlockSize+headerBlockSize));
- chkStore.write(header);
- chkStore.write(data);
- }
- t.commit();
- t = null;
+ overwriteLRUBlock(header, data, t, routingkeyDBE);
}
+ t.commit();
+ t = null;
Logger.minor(this, "Put key: "+block.getKey());
Logger.minor(this, "Headers: "+header.length+" bytes, hash
"+Fields.hashCode(header));
@@ -889,7 +971,51 @@
}
}
- private synchronized void checkSecondaryDatabaseError(Throwable ex) {
+ private void overwriteLRUBlock(byte[] header, byte[] data, Transaction t,
DatabaseEntry routingkeyDBE) throws DatabaseException, IOException {
+ // Overwrite an other block
+ Cursor c = chkDB_accessTime.openCursor(t,null);
+ DatabaseEntry keyDBE = new DatabaseEntry();
+ DatabaseEntry dataDBE = new DatabaseEntry();
+ c.getFirst(keyDBE,dataDBE,LockMode.RMW);
+ StoreBlock oldStoreBlock = (StoreBlock)
storeBlockTupleBinding.entryToObject(dataDBE);
+ c.delete();
+ c.close();
+ // Deleted, so we can now reuse it.
+ // Because we acquired a write lock, nobody else has taken it.
+ StoreBlock storeBlock = new
StoreBlock(oldStoreBlock.getOffset());
+ DatabaseEntry blockDBE = new DatabaseEntry();
+ storeBlockTupleBinding.objectToEntry(storeBlock, blockDBE);
+ chkDB.put(t,routingkeyDBE,blockDBE);
+ synchronized(chkStore) {
+
chkStore.seek(storeBlock.getOffset()*(long)(dataBlockSize+headerBlockSize));
+ chkStore.write(header);
+ chkStore.write(data);
+ }
+ }
+
+ private void writeNewBlock(long blockNum, byte[] header, byte[] data,
Transaction t, DatabaseEntry routingkeyDBE) throws DatabaseException,
IOException {
+ long byteOffset = blockNum*(dataBlockSize+headerBlockSize);
+ StoreBlock storeBlock = new StoreBlock(blockNum);
+ DatabaseEntry blockDBE = new DatabaseEntry();
+ storeBlockTupleBinding.objectToEntry(storeBlock, blockDBE);
+ chkDB.put(t,routingkeyDBE,blockDBE);
+ synchronized(chkStore) {
+ try {
+ chkStore.seek(byteOffset);
+ } catch (IOException ioe) {
+ if(byteOffset > (2*1024*1024*1024)) {
+ Logger.error(this, "Environment does
not support files bigger than 2 GB?");
+ System.out.println("Environment does
not support files bigger than 2 GB? (exception to follow)");
+ }
+ Logger.error(this, "Caught IOException on
chkStore.seek("+byteOffset+")");
+ throw ioe;
+ }
+ chkStore.write(header);
+ chkStore.write(data);
+ }
+ }
+
+ private synchronized void checkSecondaryDatabaseError(Throwable ex) {
if(ex instanceof DatabaseException && ex.getMessage().indexOf("missing
key in the primary database") > -1) {
try {
fixSecondaryFile.createNewFile();
@@ -940,65 +1066,48 @@
DatabaseEntry routingkeyDBE = new DatabaseEntry(routingkey);
synchronized(chkStore) {
- if(chkBlocksInStore<maxChkBlocks) {
- // Expand the store file
- long blockNum;
- synchronized(chkBlocksInStoreLock) {
- blockNum = chkBlocksInStore;
- chkBlocksInStore++;
- }
-
- long byteOffset =
blockNum*(dataBlockSize+headerBlockSize);
- StoreBlock storeBlock = new
StoreBlock(blockNum);
- DatabaseEntry blockDBE = new DatabaseEntry();
- storeBlockTupleBinding.objectToEntry(storeBlock,
blockDBE);
- chkDB.put(t,routingkeyDBE,blockDBE);
- synchronized(chkStore) {
- chkStore.seek(byteOffset);
- chkStore.write(data);
- }
- t.commit();
- t = null;
+ long blockNum;
+ if((blockNum = grabFreeBlock()) >= 0) {
+ writeNewBlock(blockNum, dummy, data, t, routingkeyDBE);
+ } else if(chkBlocksInStore<maxChkBlocks) {
+ // Expand the store file
+ synchronized(chkBlocksInStoreLock) {
+ blockNum = chkBlocksInStore;
+ chkBlocksInStore++;
+ }
+ writeNewBlock(blockNum, dummy, data, t, routingkeyDBE);
}else{
- // Overwrite an other block
- Cursor c = chkDB_accessTime.openCursor(t,null);
- DatabaseEntry keyDBE = new DatabaseEntry();
- DatabaseEntry dataDBE = new DatabaseEntry();
- c.getFirst(keyDBE,dataDBE,LockMode.RMW);
- StoreBlock oldStoreBlock = (StoreBlock)
storeBlockTupleBinding.entryToObject(dataDBE);
- c.delete();
- c.close();
- // Deleted, so we can now reuse it.
- // Because we acquired a write lock, nobody
else has taken it.
- StoreBlock storeBlock = new
StoreBlock(oldStoreBlock.getOffset());
- DatabaseEntry blockDBE = new DatabaseEntry();
-
storeBlockTupleBinding.objectToEntry(storeBlock, blockDBE);
- chkDB.put(t,routingkeyDBE,blockDBE);
- synchronized(chkStore) {
-
chkStore.seek(storeBlock.getOffset()*(long)(dataBlockSize+headerBlockSize));
- chkStore.write(data);
- }
- t.commit();
- t = null;
+ overwriteLRUBlock(dummy, data, t,
routingkeyDBE);
}
}
+ t.commit();
+ t = null;
Logger.minor(this, "Put key: "+HexUtil.bytesToHex(hash));
Logger.minor(this, "Data: "+data.length+" bytes, hash
"+Fields.hashCode(data));
- }catch(Throwable ex) { // FIXME: ugly
+ } catch(Throwable ex) { // FIXME: ugly
+ Logger.error(this, "Caught "+ex, ex);
+ System.err.println("Caught: "+ex);
+ ex.printStackTrace();
if(t!=null){
try{t.abort();}catch(DatabaseException ex2){};
}
checkSecondaryDatabaseError(ex);
- Logger.error(this, "Caught "+ex, ex);
- ex.printStackTrace();
if(ex instanceof IOException) throw (IOException) ex;
else throw new IOException(ex.getMessage());
}
}
- private class StoreBlock {
+ private long grabFreeBlock() {
+ while(!freeBlocks.isEmpty()) {
+ long blockNum = freeBlocks.removeFirst();
+ if(blockNum < maxChkBlocks) return blockNum;
+ }
+ return -1;
+ }
+
+ private class StoreBlock {
private long recentlyUsed;
private long offset;
@@ -1116,44 +1225,49 @@
chkDB.close();
environment.close();
Logger.minor(this, "Closing database finished.");
+ System.err.println("Closed database");
}catch(Exception ex){
Logger.error(this,"Error while closing database.",ex);
ex.printStackTrace();
}
}
- private long countCHKBlocks() throws IOException {
- long count = 0;
-
- try{
- Cursor c = chkDB_blockNum.openCursor(null,null);
- DatabaseEntry keyDBE = new DatabaseEntry();
- DatabaseEntry dataDBE = new DatabaseEntry();
-
if(c.getLast(keyDBE,dataDBE,null)==OperationStatus.SUCCESS) {
- StoreBlock storeBlock = (StoreBlock)
storeBlockTupleBinding.entryToObject(dataDBE);
- count = storeBlock.offset;
- }
- c.close();
- }catch(DatabaseException ex){ex.printStackTrace();}
-
- count++;
- System.out.println("Count from database: "+count);
-
- long oCount = chkStore.length() / (dataBlockSize + headerBlockSize);
-
- if(oCount > count) {
- System.err.println("Count from file length: "+oCount);
- return oCount;
+ private long countCHKBlocksFromDatabase() throws DatabaseException {
+ Cursor c = null;
+ try {
+ c = chkDB_blockNum.openCursor(null,null);
+ DatabaseEntry keyDBE = new DatabaseEntry();
+ DatabaseEntry dataDBE = new DatabaseEntry();
+ if(c.getLast(keyDBE,dataDBE,null)==OperationStatus.SUCCESS) {
+ StoreBlock storeBlock = (StoreBlock)
storeBlockTupleBinding.entryToObject(dataDBE);
+ return storeBlock.offset + 1;
+ }
+ c.close();
+ c = null;
+ } finally {
+ if(c != null) {
+ try {
+ c.close();
+ } catch (DatabaseException e) {
+ Logger.error(this, "Caught "+e, e);
+ }
+ }
}
-
- return count;
+ return 0;
}
+ private long countCHKBlocksFromFile() throws IOException {
+ int keySize = headerBlockSize + dataBlockSize;
+ long fileSize = chkStore.length();
+ return fileSize / keySize;
+ }
+
private long getMaxRecentlyUsed() {
long maxRecentlyUsed = 0;
+ Cursor c = null;
try{
- Cursor c = chkDB_accessTime.openCursor(null,null);
+ c = chkDB_accessTime.openCursor(null,null);
DatabaseEntry keyDBE = new DatabaseEntry();
DatabaseEntry dataDBE = new DatabaseEntry();
if(c.getLast(keyDBE,dataDBE,null)==OperationStatus.SUCCESS) {
@@ -1161,7 +1275,18 @@
maxRecentlyUsed = storeBlock.getRecentlyUsed();
}
c.close();
- }catch(DatabaseException ex){ex.printStackTrace();}
+ c = null;
+ } catch(DatabaseException ex) {
+ ex.printStackTrace();
+ } finally {
+ if(c != null) {
+ try {
+ c.close();
+ } catch (DatabaseException e) {
+ Logger.error(this, "Caught "+e, e);
+ }
+ }
+ }
return maxRecentlyUsed;
}
@@ -1173,7 +1298,8 @@
}
}
- public void setMaxKeys(long maxStoreKeys) {
+ public void setMaxKeys(long maxStoreKeys) throws DatabaseException,
IOException {
maxChkBlocks = maxStoreKeys;
+ maybeShrink(false);
}
}
Modified: trunk/freenet/src/freenet/store/FreenetStore.java
===================================================================
--- trunk/freenet/src/freenet/store/FreenetStore.java 2006-07-03 02:11:25 UTC
(rev 9437)
+++ trunk/freenet/src/freenet/store/FreenetStore.java 2006-07-03 15:16:37 UTC
(rev 9438)
@@ -2,6 +2,8 @@
import java.io.IOException;
+import com.sleepycat.je.DatabaseException;
+
import freenet.crypt.DSAPublicKey;
import freenet.keys.CHKBlock;
import freenet.keys.NodeCHK;
@@ -52,6 +54,8 @@
/**
* Change the store size.
* @param maxStoreKeys The maximum number of keys to be cached.
+ * @throws IOException
+ * @throws DatabaseException
*/
- public void setMaxKeys(long maxStoreKeys);
+ public void setMaxKeys(long maxStoreKeys) throws DatabaseException,
IOException;
}
Added: trunk/freenet/src/freenet/support/SortedLongSet.java
===================================================================
--- trunk/freenet/src/freenet/support/SortedLongSet.java 2006-07-03
02:11:25 UTC (rev 9437)
+++ trunk/freenet/src/freenet/support/SortedLongSet.java 2006-07-03
15:16:37 UTC (rev 9438)
@@ -0,0 +1,115 @@
+package freenet.support;
+
+import java.util.Arrays;
+
+/**
+ * Sorted array of long's.
+ */
+public class SortedLongSet {
+
+ private long[] data;
+ private int length;
+ private static final int MIN_SIZE = 32;
+
+ public SortedLongSet() {
+ this.data = new long[MIN_SIZE];
+ for(int i=0;i<data.length;i++)
+ data[i] = Long.MAX_VALUE;
+ length = 0;
+ }
+
+ public synchronized long getFirst() {
+ if(length == 0) return -1;
+ return data[0];
+ }
+
+ public synchronized boolean isEmpty() {
+ return length == 0;
+ }
+
+ public synchronized boolean contains(long num) {
+ int x = Arrays.binarySearch(data, num);
+ if(x >= 0)
+ return true;
+ else
+ return false;
+ }
+
+ public synchronized void remove(long item) {
+ int x = Arrays.binarySearch(data, item);
+ if(x >= 0) {
+ if(x < length-1)
+ System.arraycopy(data, x+1, data, x,
length-x-1);
+ data[--length] = Long.MAX_VALUE;
+ }
+ if(length*4 < data.length && length > MIN_SIZE) {
+ long[] newData = new long[Math.max(length/2, MIN_SIZE)];
+ System.arraycopy(data, 0, newData, 0, length);
+ for(int i=length;i<newData.length;i++)
+ newData[i] = Long.MAX_VALUE;
+ data = newData;
+ }
+ verify();
+ }
+
+ private synchronized void verify() {
+ long lastItem = -1;
+ for(int i=0;i<length;i++) {
+ long item = data[i];
+ if(i>0) {
+ if(item <= lastItem)
+ throw new IllegalStateException("Verify
failed!");
+ }
+ lastItem = item;
+ }
+ for(int i=length;i<data.length;i++)
+ if(data[i] != Long.MAX_VALUE)
+ throw new
IllegalStateException("length="+length+", data.length="+data.length+" but
["+i+"] != Long.MAX_VALUE");
+ }
+
+ /**
+ * Add the item, if it (or an item of the same number) is not already
present.
+ * @return True if we added the item.
+ */
+ public synchronized boolean push(long num) {
+ int x = Arrays.binarySearch(data, num);
+ if(x >= 0) return false;
+ // insertion point
+ x = -x-1;
+ push(num, x);
+ return true;
+ }
+
+ public synchronized void add(long num) {
+ int x = Arrays.binarySearch(data, num);
+ if(x >= 0) throw new IllegalArgumentException(); // already
exists
+ // insertion point
+ x = -x-1;
+ push(num, x);
+ }
+
+ private void push(long num, int x) {
+ Logger.minor(this, "Insertion point: "+x+" length "+length+"
data.length "+data.length);
+ // Move the data
+ if(length == data.length) {
+ Logger.minor(this, "Expanding from "+length+" to
"+length*2);
+ long[] newData = new long[length*2];
+ System.arraycopy(data, 0, newData, 0, data.length);
+ for(int i=length;i<newData.length;i++)
+ newData[i] = Long.MAX_VALUE;
+ data = newData;
+ }
+ if(x < length)
+ System.arraycopy(data, x, data, x+1, length-x);
+ data[x] = num;
+ length++;
+ verify();
+ }
+
+ public long removeFirst() {
+ long val = getFirst();
+ remove(val);
+ return val;
+ }
+
+}
Modified: trunk/freenet/src/freenet/support/SortedVectorByNumber.java
===================================================================
--- trunk/freenet/src/freenet/support/SortedVectorByNumber.java 2006-07-03
02:11:25 UTC (rev 9437)
+++ trunk/freenet/src/freenet/support/SortedVectorByNumber.java 2006-07-03
15:16:37 UTC (rev 9438)
@@ -3,8 +3,6 @@
import java.util.Arrays;
import java.util.Comparator;
-import freenet.client.async.ClientRequester;
-
/**
* Map of an integer to an element, based on a sorted Vector.
* Note that we have to shuffle data around, so this is slowish if it gets big.
@@ -67,11 +65,28 @@
throw new
IllegalStateException("length="+length+", data.length="+data.length+" but
["+i+"] != null");
}
+ /**
+ * Add the item, if it (or an item of the same number) is not already
present.
+ * @return True if we added the item.
+ */
+ public synchronized boolean push(IntNumberedItem grabber) {
+ int x = Arrays.binarySearch(data, new
Integer(grabber.getNumber()), comparator);
+ if(x >= 0) return false;
+ // insertion point
+ x = -x-1;
+ push(grabber, x);
+ return true;
+ }
+
public synchronized void add(IntNumberedItem grabber) {
int x = Arrays.binarySearch(data, new
Integer(grabber.getNumber()), comparator);
if(x >= 0) throw new IllegalArgumentException(); // already
exists
// insertion point
x = -x-1;
+ push(grabber, x);
+ }
+
+ private void push(IntNumberedItem grabber, int x) {
Logger.minor(this, "Insertion point: "+x);
// Move the data
if(length == data.length) {
