Author: toad
Date: 2006-11-11 16:30:23 +0000 (Sat, 11 Nov 2006)
New Revision: 10886
Modified:
trunk/freenet/src/freenet/node/Node.java
trunk/freenet/src/freenet/store/BerkeleyDBFreenetStore.java
Log:
Datastore refactoring.
Modified: trunk/freenet/src/freenet/node/Node.java
===================================================================
--- trunk/freenet/src/freenet/node/Node.java 2006-11-11 16:23:22 UTC (rev
10885)
+++ trunk/freenet/src/freenet/node/Node.java 2006-11-11 16:30:23 UTC (rev
10886)
@@ -1172,77 +1172,25 @@
envMutableConfig.setCacheSize(nodeConfig.getLong("databaseMaxMemory"));
try {
- BerkeleyDBFreenetStore tmp;
Logger.normal(this, "Initializing CHK Datastore");
System.out.println("Initializing CHK Datastore
("+maxStoreKeys+" keys)");
- try {
- if((lastVersion > 0) && (lastVersion < 852)) {
- throw new
DatabaseException("Reconstructing store because started from old version");
- }
- tmp = new BerkeleyDBFreenetStore(chkStorePath,
maxStoreKeys, 32768, CHKBlock.TOTAL_HEADERS_LENGTH, true);
- } catch (DatabaseException e) {
- System.err.println("Could not open store: "+e);
- e.printStackTrace();
- System.err.println("Attempting to
reconstruct...");
- WrapperManager.signalStarting(5*60*60*1000);
- tmp = new BerkeleyDBFreenetStore(chkStorePath,
maxStoreKeys, 32768, CHKBlock.TOTAL_HEADERS_LENGTH,
BerkeleyDBFreenetStore.TYPE_CHK);
- }
- chkDatastore = tmp;
+ chkDatastore =
BerkeleyDBFreenetStore.construct(lastVersion, "", chkStorePath, maxStoreKeys,
CHKBlock.DATA_LENGTH, CHKBlock.TOTAL_HEADERS_LENGTH, true,
BerkeleyDBFreenetStore.TYPE_CHK);
Logger.normal(this, "Initializing CHK Datacache");
System.out.println("Initializing CHK Datacache
("+maxCacheKeys+":"+maxCacheKeys+" keys)");
- try {
- if((lastVersion > 0) && (lastVersion < 852)) {
- throw new
DatabaseException("Reconstructing store because started from old version");
- }
- tmp = new BerkeleyDBFreenetStore(chkCachePath,
maxCacheKeys, 32768, CHKBlock.TOTAL_HEADERS_LENGTH, true);
- } catch (DatabaseException e) {
- System.err.println("Could not open store: "+e);
- e.printStackTrace();
- System.err.println("Attempting to
reconstruct...");
- WrapperManager.signalStarting(5*60*60*1000);
- tmp = new BerkeleyDBFreenetStore(chkCachePath,
maxCacheKeys, 32768, CHKBlock.TOTAL_HEADERS_LENGTH,
BerkeleyDBFreenetStore.TYPE_CHK);
- }
- chkDatacache = tmp;
- chkDatacache.setMaxKeys(maxCacheKeys, false);
- // Shrink pubkey store immediately; it's tiny anyway.
+ chkDatacache =
BerkeleyDBFreenetStore.construct(lastVersion, "", chkCachePath, maxCacheKeys,
CHKBlock.DATA_LENGTH, CHKBlock.TOTAL_HEADERS_LENGTH, true,
BerkeleyDBFreenetStore.TYPE_CHK);
Logger.normal(this, "Initializing pubKey Datastore");
System.out.println("Initializing pubKey Datastore");
- try {
- if((lastVersion > 0) && (lastVersion < 852)) {
- throw new
DatabaseException("Reconstructing store because started from old version");
- }
- tmp = new BerkeleyDBFreenetStore(pkStorePath,
maxStoreKeys, DSAPublicKey.PADDED_SIZE, 0, true);
- } catch (DatabaseException e) {
- System.err.println("Could not open store: "+e);
- e.printStackTrace();
- System.err.println("Attempting to
reconstruct...");
- WrapperManager.signalStarting(5*60*60*1000);
- tmp = new BerkeleyDBFreenetStore(pkStorePath,
maxStoreKeys, DSAPublicKey.PADDED_SIZE, 0, BerkeleyDBFreenetStore.TYPE_PUBKEY);
- }
- this.pubKeyDatastore = tmp;
+ pubKeyDatastore =
BerkeleyDBFreenetStore.construct(lastVersion, "", pkStorePath, maxStoreKeys,
DSAPublicKey.PADDED_SIZE, 0, true, BerkeleyDBFreenetStore.TYPE_PUBKEY);
Logger.normal(this, "Initializing pubKey Datacache");
System.out.println("Initializing pubKey Datacache
("+maxCacheKeys+" keys)");
- try {
- if((lastVersion > 0) && (lastVersion < 852)) {
- throw new
DatabaseException("Reconstructing store because started from old version");
- }
- tmp = new BerkeleyDBFreenetStore(pkCachePath,
maxCacheKeys, DSAPublicKey.PADDED_SIZE, 0, true);
- } catch (DatabaseException e) {
- System.err.println("Could not open store: "+e);
- e.printStackTrace();
- System.err.println("Attempting to
reconstruct...");
- WrapperManager.signalStarting(5*60*60*1000);
- tmp = new BerkeleyDBFreenetStore(pkCachePath,
maxCacheKeys, DSAPublicKey.PADDED_SIZE, 0, BerkeleyDBFreenetStore.TYPE_PUBKEY);
- }
- this.pubKeyDatacache = tmp;
+ pubKeyDatacache =
BerkeleyDBFreenetStore.construct(lastVersion, "", pkCachePath, maxCacheKeys,
DSAPublicKey.PADDED_SIZE, 0, true, BerkeleyDBFreenetStore.TYPE_PUBKEY);
// FIXME can't auto-fix SSK stores.
Logger.normal(this, "Initializing SSK Datastore");
System.out.println("Initializing SSK Datastore");
- sskDatastore = new BerkeleyDBFreenetStore(sskStorePath,
maxStoreKeys, 1024, SSKBlock.TOTAL_HEADERS_LENGTH, false);
+ sskDatastore =
BerkeleyDBFreenetStore.construct(lastVersion, "", sskStorePath, maxStoreKeys,
SSKBlock.DATA_LENGTH, SSKBlock.TOTAL_HEADERS_LENGTH, false,
BerkeleyDBFreenetStore.TYPE_SSK);
Logger.normal(this, "Initializing SSK Datacache");
System.out.println("Initializing SSK Datacache
("+maxCacheKeys+" keys)");
- sskDatacache = new BerkeleyDBFreenetStore(sskCachePath,
maxCacheKeys, 1024, SSKBlock.TOTAL_HEADERS_LENGTH, false);
- sskDatacache.setMaxKeys(maxCacheKeys, false);
+ sskDatacache =
BerkeleyDBFreenetStore.construct(lastVersion, "", sskCachePath, maxStoreKeys,
SSKBlock.DATA_LENGTH, SSKBlock.TOTAL_HEADERS_LENGTH, false,
BerkeleyDBFreenetStore.TYPE_SSK);
} catch (FileNotFoundException e1) {
String msg = "Could not open datastore: "+e1;
Logger.error(this, msg, e1);
Modified: trunk/freenet/src/freenet/store/BerkeleyDBFreenetStore.java
===================================================================
--- trunk/freenet/src/freenet/store/BerkeleyDBFreenetStore.java 2006-11-11
16:23:22 UTC (rev 10885)
+++ trunk/freenet/src/freenet/store/BerkeleyDBFreenetStore.java 2006-11-11
16:30:23 UTC (rev 10886)
@@ -82,25 +82,92 @@
private boolean closed;
private final static byte[] dummy = new byte[0];
-
- /**
- * Initializes database
- * @param the directory where the store is located
- * @throws IOException
- * @throws DatabaseException
- * @throws FileNotFoundException if the dir does not exist and could not
be created
- */
- public BerkeleyDBFreenetStore(String storeDir, long maxChkBlocks, int
blockSize, int headerSize, boolean throwOnTooFewKeys) throws IOException,
DatabaseException {
- logMINOR = Logger.shouldLog(Logger.MINOR, this);
- this.dataBlockSize = blockSize;
- this.headerBlockSize = headerSize;
- this.freeBlocks = new SortedLongSet();
+
+ public static BerkeleyDBFreenetStore construct(int lastVersion, String
prefix, String storeDir, long maxStoreKeys, int blockSize, int headerSize,
boolean throwOnTooFewKeys, short type) throws Exception {
+
+ File dir = new File(storeDir);
+ if(!dir.exists())
+ dir.mkdir();
- this.maxChkBlocks=maxChkBlocks;
+ File dbDir = new File(dir,"database");
+ if(!dbDir.exists())
+ dbDir.mkdir();
+ Environment env = null;
+ // Initialize environment
+ try {
+ EnvironmentConfig envConfig = new EnvironmentConfig();
+ envConfig.setAllowCreate(true);
+ envConfig.setTransactional(true);
+ envConfig.setTxnWriteNoSync(true);
+ env = new Environment(dbDir, envConfig);
+ } catch (DatabaseException e) {
+ if(env != null)
+ env.close();
+ throw e;
+ }
+
+ BerkeleyDBFreenetStore tmp;
+ try {
+ if((lastVersion > 0) && (lastVersion < 852)) {
+ throw new DatabaseException("Reconstructing
store because started from old version");
+ }
+ tmp = new BerkeleyDBFreenetStore(env, prefix, dir,
maxStoreKeys, blockSize, headerSize, throwOnTooFewKeys);
+ } catch (DatabaseException e) {
+
+ System.err.println("Could not open store: "+e);
+ e.printStackTrace();
+
+ if(type == TYPE_SSK) {
+ System.err.println("Cannot reconstruct SSK
store/cache. Move the old store/cache out of the way, and report to
developers.");
+ throw e;
+ }
+
+ System.err.println("Attempting to reconstruct...");
+ WrapperManager.signalStarting(5*60*60*1000);
+
+ // Reconstruct
+
+ // First, close it.
+ try {
+ env.close();
+ } catch (Throwable t) {
+ // Ignore, probably double-closing
+ // FIXME shouldn't be necessary
+ }
+
+ // Now delete the old database
+
+ if(dbDir.exists()) {
+ File[] files = dbDir.listFiles();
+ for(int i=0;i<files.length;i++) {
+ if(!files[i].delete())
+ System.err.println("Failed to
delete "+files[i]);
+ }
+ } else
+ dbDir.mkdir();
+
+ // Now create a new one.
+
+ // Initialize environment
+ EnvironmentConfig envConfig = new EnvironmentConfig();
+ envConfig.setAllowCreate(true);
+ envConfig.setTransactional(true);
+ envConfig.setTxnWriteNoSync(true);
+
+ env = new Environment(dbDir, envConfig);
+
+ tmp = new BerkeleyDBFreenetStore(env, dir, dbDir,
storeDir, maxStoreKeys, blockSize, headerSize, type);
+ }
+ return tmp;
+ }
+
+ public static BerkeleyDBFreenetStore construct(String prefix, String
storeDir, long maxChkBlocks, int blockSize, int headerSize, boolean
throwOnTooFewKeys) throws IOException, DatabaseException {
+
File dir = new File(storeDir);
if(!dir.exists())
dir.mkdir();
+
File dbDir = new File(dir,"database");
if(!dbDir.exists())
dbDir.mkdir();
@@ -118,13 +185,32 @@
env.close();
throw e;
}
+
+ return new BerkeleyDBFreenetStore(env, prefix, dir,
maxChkBlocks, blockSize, headerSize, throwOnTooFewKeys);
+ }
+
+ /**
+ * Initializes database
+ * @param the directory where the store is located
+ * @throws IOException
+ * @throws DatabaseException
+ * @throws FileNotFoundException if the dir does not exist and could not
be created
+ */
+ public BerkeleyDBFreenetStore(Environment env, String prefix, File
storeDir, long maxChkBlocks, int blockSize, int headerSize, boolean
throwOnTooFewKeys) throws IOException, DatabaseException {
+ logMINOR = Logger.shouldLog(Logger.MINOR, this);
+ this.dataBlockSize = blockSize;
+ this.headerBlockSize = headerSize;
+ this.freeBlocks = new SortedLongSet();
+
+ this.maxChkBlocks=maxChkBlocks;
+
environment = env;
// Initialize CHK database
DatabaseConfig dbConfig = new DatabaseConfig();
dbConfig.setAllowCreate(true);
dbConfig.setTransactional(true);
- chkDB = environment.openDatabase(null,"CHK",dbConfig);
+ chkDB = environment.openDatabase(null,prefix+"CHK",dbConfig);
fixSecondaryFile = new File(storeDir, "recreate_secondary_db");
@@ -135,7 +221,7 @@
System.err.println("Recreating secondary database for
"+storeDir);
System.err.println("This may take some time...");
try {
- environment.truncateDatabase(null,
"CHK_accessTime", false);
+ environment.truncateDatabase(null,
prefix+"CHK_accessTime", false);
} catch (DatabaseException e) {
close(false);
throw e;
@@ -155,7 +241,7 @@
secDbConfig.setKeyCreator(accessTimeKeyCreator);
try {
chkDB_accessTime = environment.openSecondaryDatabase
- (null,
"CHK_accessTime", chkDB, secDbConfig);
+ (null,
prefix+"CHK_accessTime", chkDB, secDbConfig);
} catch (DatabaseException e1) {
close(false);
throw e1;
@@ -178,7 +264,7 @@
try {
System.err.println("Opening block db index");
blockNums = environment.openSecondaryDatabase
- (null, "CHK_blockNum", chkDB, blockNoDbConfig);
+ (null, prefix+"CHK_blockNum", chkDB,
blockNoDbConfig);
} catch (DatabaseNotFoundException e) {
System.err.println("Migrating block db index");
// De-dupe on keys and block numbers.
@@ -188,7 +274,7 @@
blockNoDbConfig.setAllowCreate(true);
blockNoDbConfig.setAllowPopulate(true);
blockNums = environment.openSecondaryDatabase
- (null, "CHK_blockNum", chkDB, blockNoDbConfig);
+ (null, prefix+"CHK_blockNum", chkDB,
blockNoDbConfig);
} catch (DatabaseException e) {
close(false);
throw e;
@@ -197,7 +283,7 @@
chkDB_blockNum = blockNums;
// Initialize the store file
- File storeFile = new File(dir,"store");
+ File storeFile = new File(storeDir,"store");
try {
if(!storeFile.exists())
storeFile.createNewFile();
@@ -543,48 +629,14 @@
* @param the directory where the store is located
* @throws FileNotFoundException if the dir does not exist and could not
be created
*/
- public BerkeleyDBFreenetStore(String storeDir, long maxChkBlocks, int
blockSize, int headerSize, short type) throws Exception {
+ public BerkeleyDBFreenetStore(Environment env, File dir, File dbDir,
String storeDir, long maxChkBlocks, int blockSize, int headerSize, short type)
throws Exception {
logMINOR = Logger.shouldLog(Logger.MINOR, this);
this.dataBlockSize = blockSize;
this.headerBlockSize = headerSize;
this.freeBlocks = new SortedLongSet();
- // Percentage of the database that must contain usefull data
- // decrease to increase performance, increase to save disk space
- System.setProperty("je.cleaner.minUtilization","98");
-
- // Delete empty log files
- System.setProperty("je.cleaner.expunge","true");
-
- // Percentage of the maximum heap size used as a cache
- System.setProperty("je.maxMemoryPercent","30");
-
this.maxChkBlocks=maxChkBlocks;
+ this.environment = env;
- // Delete old database.
-
- File dir = new File(storeDir);
- if(!dir.exists())
- dir.mkdir();
- File dbDir = new File(dir,"database");
- if(dbDir.exists()) {
- File[] files = dbDir.listFiles();
- for(int i=0;i<files.length;i++) {
- if(!files[i].delete())
- System.err.println("Failed to delete
"+files[i]);
- }
- } else
- dbDir.mkdir();
-
- // Now create a new one.
-
- // Initialize environment
- EnvironmentConfig envConfig = new EnvironmentConfig();
- envConfig.setAllowCreate(true);
- envConfig.setTransactional(true);
- envConfig.setTxnWriteNoSync(true);
-
- environment = new Environment(dbDir, envConfig);
-
// Initialize CHK database
DatabaseConfig dbConfig = new DatabaseConfig();
dbConfig.setAllowCreate(true);
@@ -635,7 +687,7 @@
lastRecentlyUsed = 0;
- reconstruct(type, storeDir);
+ reconstruct(type, dir);
chkBlocksInStore = countCHKBlocksFromFile();
lastRecentlyUsed = getMaxRecentlyUsed();
@@ -646,7 +698,7 @@
Runtime.getRuntime().addShutdownHook(new ShutdownHook());
}
- private void reconstruct(short type, String storeDir) throws
DatabaseException {
+ private void reconstruct(short type, File storeDir) throws
DatabaseException {
if(type == TYPE_SSK) {
System.err.println("Reconstruction of SSK store not
supported at present.");
throw new UnsupportedOperationException("Reconstruction
of SSK store not supported at present.");
@@ -719,7 +771,7 @@
*
* FIXME: Create a list of reusable block numbers?
*/
- private void migrate(String storeDir) throws DatabaseException {
+ private void migrate(File storeDir) throws DatabaseException {
System.err.println("Migrating database "+storeDir+": Creating
unique index on block number");
HashSet s = new HashSet();
