Author: toad
Date: 2007-05-18 17:05:28 +0000 (Fri, 18 May 2007)
New Revision: 13261
Modified:
trunk/freenet/src/freenet/node/Node.java
trunk/freenet/src/freenet/store/BerkeleyDBFreenetStore.java
Log:
Attempt to reconstruct via DbDump -r / DbLoad. Oddly enough this doesn't work
at the moment. There are more urgent matters to attend to, I'll get back to it
later.
Anyway create a file called "reconstruct" in the database dir to test.
Modified: trunk/freenet/src/freenet/node/Node.java
===================================================================
--- trunk/freenet/src/freenet/node/Node.java 2007-05-18 16:38:58 UTC (rev
13260)
+++ trunk/freenet/src/freenet/node/Node.java 2007-05-18 17:05:28 UTC (rev
13261)
@@ -38,6 +38,7 @@
import com.sleepycat.je.EnvironmentConfig;
import com.sleepycat.je.EnvironmentMutableConfig;
import com.sleepycat.je.StatsConfig;
+import com.sleepycat.je.util.DbDump;
import freenet.client.FetchContext;
import freenet.config.EnumerableOptionCallback;
@@ -1152,16 +1153,27 @@
File dbDir = new File(storeDir, "database-"+portNumber);
dbDir.mkdirs();
+ File reconstructFile = new File(dbDir, "reconstruct");
+
Environment env = null;
EnvironmentMutableConfig mutableConfig;
+
+ boolean tryDbLoad = false;
+
+ String suffix = "-" + portNumber;
+
// This can take some time
System.out.println("Starting database...");
try {
+ if(reconstructFile.exists()) {
+ reconstructFile.delete();
+ throw new DatabaseException();
+ }
env = new Environment(dbDir, envConfig);
mutableConfig = env.getConfig();
} catch (DatabaseException e) {
- // The database is broken
- // We will have to recover from scratch
+
+ // Close the database
if(env != null) {
try {
env.close();
@@ -1170,13 +1182,71 @@
t.printStackTrace();
}
}
+
+ // First try DbDump
+
+ boolean failed = false;
+
+ System.err.println("Attempting DbDump-level
recovery...");
+
+ boolean[] isStores = new boolean[] { true, false, true,
false, true, false };
+ short[] types = new short[] {
+ BerkeleyDBFreenetStore.TYPE_CHK,
+ BerkeleyDBFreenetStore.TYPE_CHK,
+ BerkeleyDBFreenetStore.TYPE_PUBKEY,
+ BerkeleyDBFreenetStore.TYPE_PUBKEY,
+ BerkeleyDBFreenetStore.TYPE_SSK,
+ BerkeleyDBFreenetStore.TYPE_SSK
+ };
+ int[] lengths = new int[] {
+ CHKBlock.TOTAL_HEADERS_LENGTH +
CHKBlock.DATA_LENGTH,
+ CHKBlock.TOTAL_HEADERS_LENGTH +
CHKBlock.DATA_LENGTH,
+ DSAPublicKey.PADDED_SIZE,
+ DSAPublicKey.PADDED_SIZE,
+ SSKBlock.TOTAL_HEADERS_LENGTH +
SSKBlock.DATA_LENGTH,
+ SSKBlock.TOTAL_HEADERS_LENGTH +
SSKBlock.DATA_LENGTH
+ };
+
+ for(int i=0;i<types.length;i++) {
+ boolean isStore = isStores[i];
+ short type = types[i];
+ String dbName =
BerkeleyDBFreenetStore.getName(isStore, type);
+ File dbFile =
BerkeleyDBFreenetStore.getFile(isStore, type, storeDir, suffix);
+ long keyCount = dbFile.length() / lengths[i];
+ // This is *slow* :(
+ int millis = (int)Math.min(24*60*60*1000 /*
horrible hack, because of the wrapper's braindead timeout additions */,
+ 5*60*1000 + (Math.max(keyCount,
1) * 10000));
+ WrapperManager.signalStarting(millis);
+ try {
+ File target = new File(storeDir,
dbName+".dump");
+ System.err.println("Dumping "+dbName+"
to "+target+" ("+keyCount+" keys from file, allowing "+millis+"ms)");
+ DbDump.main(new String[] { "-r", "-h",
dbDir.toString(),
+ "-s", dbName, "-f",
target.toString() });
+ } catch (DatabaseException e2) {
+ System.err.println("DbDump recovery
failed for "+dbName+" : "+e2);
+ e2.printStackTrace();
+ } catch (IOException e2) {
+ System.err.println("DbDump recovery
failed for "+dbName+" : "+e2);
+ e2.printStackTrace();
+ }
+ tryDbLoad = true;
+ }
+
+ // Delete the database logs
+
+ System.err.println("Deleting old database log
files...");
+
File[] files = dbDir.listFiles();
for(int i=0;i<files.length;i++) {
String name = files[i].getName().toLowerCase();
if(name.endsWith(".jdb") ||
name.equals("je.lck"))
- files[i].delete();
+ if(!files[i].delete())
+ System.err.println("Failed to
delete old database log file "+files[i]);
}
- dbDir.delete();
+
+ System.err.println("Recovering...");
+ // The database is broken
+ // We will have to recover from scratch
try {
env = new Environment(dbDir, envConfig);
mutableConfig = env.getConfig();
@@ -1253,34 +1323,32 @@
throw new NodeInitException(EXIT_STORE_OTHER,
e.getMessage());
}
- String suffix = "-" + portNumber;
-
try {
Logger.normal(this, "Initializing CHK Datastore");
System.out.println("Initializing CHK Datastore
("+maxStoreKeys+" keys)");
chkDatastore =
BerkeleyDBFreenetStore.construct(lastVersion, storeDir, true, suffix,
maxStoreKeys,
- CHKBlock.DATA_LENGTH,
CHKBlock.TOTAL_HEADERS_LENGTH, true, BerkeleyDBFreenetStore.TYPE_CHK,
storeEnvironment, random, storeShutdownHook);
+ CHKBlock.DATA_LENGTH,
CHKBlock.TOTAL_HEADERS_LENGTH, true, BerkeleyDBFreenetStore.TYPE_CHK,
storeEnvironment, random, storeShutdownHook, tryDbLoad, reconstructFile);
Logger.normal(this, "Initializing CHK Datacache");
System.out.println("Initializing CHK Datacache
("+maxCacheKeys+ ':' +maxCacheKeys+" keys)");
chkDatacache =
BerkeleyDBFreenetStore.construct(lastVersion, storeDir, false, suffix,
maxCacheKeys,
- CHKBlock.DATA_LENGTH,
CHKBlock.TOTAL_HEADERS_LENGTH, true, BerkeleyDBFreenetStore.TYPE_CHK,
storeEnvironment, random, storeShutdownHook);
+ CHKBlock.DATA_LENGTH,
CHKBlock.TOTAL_HEADERS_LENGTH, true, BerkeleyDBFreenetStore.TYPE_CHK,
storeEnvironment, random, storeShutdownHook, tryDbLoad, reconstructFile);
Logger.normal(this, "Initializing pubKey Datastore");
System.out.println("Initializing pubKey Datastore");
pubKeyDatastore =
BerkeleyDBFreenetStore.construct(lastVersion, storeDir, true, suffix,
maxStoreKeys,
- DSAPublicKey.PADDED_SIZE, 0, true,
BerkeleyDBFreenetStore.TYPE_PUBKEY, storeEnvironment, random,
storeShutdownHook);
+ DSAPublicKey.PADDED_SIZE, 0, true,
BerkeleyDBFreenetStore.TYPE_PUBKEY, storeEnvironment, random,
storeShutdownHook, tryDbLoad, reconstructFile);
Logger.normal(this, "Initializing pubKey Datacache");
System.out.println("Initializing pubKey Datacache
("+maxCacheKeys+" keys)");
pubKeyDatacache =
BerkeleyDBFreenetStore.construct(lastVersion, storeDir, false, suffix,
maxCacheKeys,
- DSAPublicKey.PADDED_SIZE, 0, true,
BerkeleyDBFreenetStore.TYPE_PUBKEY, storeEnvironment, random,
storeShutdownHook);
+ DSAPublicKey.PADDED_SIZE, 0, true,
BerkeleyDBFreenetStore.TYPE_PUBKEY, storeEnvironment, random,
storeShutdownHook, tryDbLoad, reconstructFile);
// FIXME can't auto-fix SSK stores.
Logger.normal(this, "Initializing SSK Datastore");
System.out.println("Initializing SSK Datastore");
sskDatastore =
BerkeleyDBFreenetStore.construct(lastVersion, storeDir, true, suffix,
maxStoreKeys,
- SSKBlock.DATA_LENGTH,
SSKBlock.TOTAL_HEADERS_LENGTH, false, BerkeleyDBFreenetStore.TYPE_SSK,
storeEnvironment, random, storeShutdownHook);
+ SSKBlock.DATA_LENGTH,
SSKBlock.TOTAL_HEADERS_LENGTH, false, BerkeleyDBFreenetStore.TYPE_SSK,
storeEnvironment, random, storeShutdownHook, tryDbLoad, reconstructFile);
Logger.normal(this, "Initializing SSK Datacache");
System.out.println("Initializing SSK Datacache
("+maxCacheKeys+" keys)");
sskDatacache =
BerkeleyDBFreenetStore.construct(lastVersion, storeDir, false, suffix,
maxStoreKeys,
- SSKBlock.DATA_LENGTH,
SSKBlock.TOTAL_HEADERS_LENGTH, false, BerkeleyDBFreenetStore.TYPE_SSK,
storeEnvironment, random, storeShutdownHook);
+ SSKBlock.DATA_LENGTH,
SSKBlock.TOTAL_HEADERS_LENGTH, false, BerkeleyDBFreenetStore.TYPE_SSK,
storeEnvironment, random, storeShutdownHook, tryDbLoad, reconstructFile);
} catch (FileNotFoundException e1) {
String msg = "Could not open datastore: "+e1;
Logger.error(this, msg, e1);
Modified: trunk/freenet/src/freenet/store/BerkeleyDBFreenetStore.java
===================================================================
--- trunk/freenet/src/freenet/store/BerkeleyDBFreenetStore.java 2007-05-18
16:38:58 UTC (rev 13260)
+++ trunk/freenet/src/freenet/store/BerkeleyDBFreenetStore.java 2007-05-18
17:05:28 UTC (rev 13261)
@@ -1,9 +1,12 @@
package freenet.store;
+import java.io.BufferedReader;
import java.io.EOFException;
import java.io.File;
+import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
+import java.io.InputStreamReader;
import java.io.RandomAccessFile;
import java.util.Arrays;
import java.util.Vector;
@@ -28,6 +31,7 @@
import com.sleepycat.je.SecondaryDatabase;
import com.sleepycat.je.SecondaryKeyCreator;
import com.sleepycat.je.Transaction;
+import com.sleepycat.je.util.DbLoad;
import freenet.crypt.CryptFormatException;
import freenet.crypt.DSAPublicKey;
@@ -59,6 +63,8 @@
private static boolean logMINOR;
+ // If we get a DbChecksumException, create this file.
+ final File reconstructFile;
final int dataBlockSize;
final int headerBlockSize;
@@ -85,8 +91,20 @@
private boolean reallyClosed;
private final static byte[] dummy = new byte[0];
+ public static String getName(boolean isStore, short type) {
+ String newDBPrefix = newTypeName(type)+ '-' +(isStore ? "store"
: "cache")+ '-';
+ return newDBPrefix + "CHK";
+ }
+
+ public static File getFile(boolean isStore, short type, File
baseStoreDir, String suffix) {
+ String newStoreFileName = newTypeName(type) + suffix + '.' +
(isStore ? "store" : "cache");
+ return new File(baseStoreDir, newStoreFileName);
+ }
+
public static BerkeleyDBFreenetStore construct(int lastVersion, File
baseStoreDir, boolean isStore,
- String suffix, long maxStoreKeys, int blockSize, int
headerSize, boolean throwOnTooFewKeys, short type, Environment
storeEnvironment, RandomSource random, SemiOrderedShutdownHook
storeShutdownHook) throws DatabaseException, IOException {
+ String suffix, long maxStoreKeys, int blockSize, int
headerSize, boolean throwOnTooFewKeys,
+ short type, Environment storeEnvironment, RandomSource
random,
+ SemiOrderedShutdownHook storeShutdownHook, boolean
tryDbLoad, File reconstructFile) throws DatabaseException, IOException {
/**
* Migration strategy:
@@ -129,8 +147,8 @@
// Try to load new database, reconstruct it if
necessary.
// Don't need to create a new Environment, since we can
use the old one.
- tmp = openStore(storeEnvironment, newDBPrefix,
newStoreFile, newFixSecondaryFile, maxStoreKeys,
- blockSize, headerSize,
throwOnTooFewKeys, false, lastVersion, type, false, storeShutdownHook);
+ tmp = openStore(storeEnvironment, baseStoreDir,
newDBPrefix, newStoreFile, newFixSecondaryFile, maxStoreKeys,
+ blockSize, headerSize,
throwOnTooFewKeys, false, lastVersion, type, false, storeShutdownHook,
tryDbLoad, reconstructFile);
} else if(oldDir.exists() && oldStoreFile.exists()) {
@@ -180,8 +198,9 @@
// Open the new store
// Don't reconstruct yet
- tmp = openStore(storeEnvironment, newDBPrefix,
storeFile, newFixSecondaryFile, maxStoreKeys,
- blockSize, headerSize,
false, true, lastVersion, type, true, storeShutdownHook);
+ tmp = openStore(storeEnvironment, baseStoreDir,
newDBPrefix, storeFile, newFixSecondaryFile,
+ maxStoreKeys, blockSize,
headerSize, false, true, lastVersion, type, true,
+ storeShutdownHook, tryDbLoad,
reconstructFile);
// Migrate all tuples from old database to new
database.
migrateTuples(oldEnv, oldChkDB, tmp);
@@ -198,7 +217,9 @@
// Reconstruct
- tmp = new
BerkeleyDBFreenetStore(storeEnvironment, newDBPrefix, newStoreFile,
newFixSecondaryFile, maxStoreKeys, blockSize, headerSize, type, true,
storeShutdownHook);
+ tmp = new
BerkeleyDBFreenetStore(storeEnvironment, newDBPrefix, newStoreFile,
+ newFixSecondaryFile,
maxStoreKeys, blockSize, headerSize, type, true,
+ storeShutdownHook,
reconstructFile);
}
tmp.checkForHoles(tmp.countCHKBlocksFromFile(),
false);
@@ -210,8 +231,9 @@
// No old database to worry about.
// Reconstruct the new database from the store
file which is now in the right place.
- tmp = openStore(storeEnvironment, newDBPrefix,
storeFile, newFixSecondaryFile, maxStoreKeys,
- blockSize, headerSize,
throwOnTooFewKeys, false, lastVersion, type, false, storeShutdownHook);
+ tmp = openStore(storeEnvironment, baseStoreDir,
newDBPrefix, storeFile, newFixSecondaryFile,
+ maxStoreKeys, blockSize,
headerSize, throwOnTooFewKeys, false, lastVersion, type,
+ false, storeShutdownHook,
tryDbLoad, reconstructFile);
}
@@ -220,8 +242,9 @@
// No new store file, no new database.
// Start from scratch, with new store.
- tmp = openStore(storeEnvironment, newDBPrefix,
newStoreFile, newFixSecondaryFile, maxStoreKeys,
- blockSize, headerSize,
throwOnTooFewKeys, false, lastVersion, type, false, storeShutdownHook);
+ tmp = openStore(storeEnvironment, baseStoreDir,
newDBPrefix, newStoreFile, newFixSecondaryFile,
+ maxStoreKeys, blockSize, headerSize,
throwOnTooFewKeys, false, lastVersion, type,
+ false, storeShutdownHook, tryDbLoad,
reconstructFile);
}
@@ -328,13 +351,38 @@
}
- private static BerkeleyDBFreenetStore openStore(Environment
storeEnvironment, String newDBPrefix, File newStoreFile,
+ private static BerkeleyDBFreenetStore openStore(Environment
storeEnvironment, File baseDir, String newDBPrefix, File newStoreFile,
File newFixSecondaryFile, long maxStoreKeys, int
blockSize, int headerSize, boolean throwOnTooFewKeys,
- boolean noCheck, int lastVersion, short type, boolean
wipe, SemiOrderedShutdownHook storeShutdownHook) throws DatabaseException,
IOException {
+ boolean noCheck, int lastVersion, short type, boolean
wipe, SemiOrderedShutdownHook storeShutdownHook,
+ boolean tryDbLoad, File reconstructFile) throws
DatabaseException, IOException {
+ if(tryDbLoad) {
+ String dbName = newDBPrefix+"CHK";
+ File dumpFilename = new File(baseDir, dbName+".dump");
+ System.err.println("Trying to restore from
"+dumpFilename);
+ try {
+ FileInputStream fis = new
FileInputStream(dumpFilename);
+ // DbDump used the default charset, so will
this.
+ BufferedReader br = new BufferedReader(new
InputStreamReader(fis));
+ DbLoad loader = new DbLoad();
+ loader.setEnv(storeEnvironment);
+ loader.setDbName(dbName);
+ loader.setInputReader(br);
+ loader.setNoOverwrite(false);
+ loader.setTextFileMode(false);
+ loader.load();
+ fis.close();
+ newFixSecondaryFile.createNewFile(); // force
reconstruct of secondary indexes
+ } catch (IOException e) {
+ System.err.println("Failed to reload database
"+dbName+": "+e);
+ e.printStackTrace();
+ }
+ }
+
try {
return new BerkeleyDBFreenetStore(storeEnvironment,
newDBPrefix, newStoreFile, newFixSecondaryFile,
- maxStoreKeys, blockSize, headerSize,
throwOnTooFewKeys, noCheck, wipe, storeShutdownHook);
+ maxStoreKeys, blockSize, headerSize,
throwOnTooFewKeys, noCheck, wipe, storeShutdownHook,
+ reconstructFile);
} catch (DatabaseException e) {
System.err.println("Could not open store: "+e);
@@ -350,7 +398,8 @@
// Reconstruct
- return new BerkeleyDBFreenetStore(storeEnvironment,
newDBPrefix, newStoreFile, newFixSecondaryFile, maxStoreKeys, blockSize,
headerSize, type, noCheck, storeShutdownHook);
+ return new BerkeleyDBFreenetStore(storeEnvironment,
newDBPrefix, newStoreFile, newFixSecondaryFile,
+ maxStoreKeys, blockSize, headerSize,
type, noCheck, storeShutdownHook, reconstructFile);
}
}
@@ -423,12 +472,13 @@
* Initializes database
* @param noCheck If true, don't check for holes etc.
* @param wipe If true, wipe the database first.
+ * @param reconstructFile
* @param the directory where the store is located
* @throws IOException
* @throws DatabaseException
* @throws FileNotFoundException if the dir does not exist and could not
be created
*/
- public BerkeleyDBFreenetStore(Environment env, String prefix, File
storeFile, File fixSecondaryFile, long maxChkBlocks, int blockSize, int
headerSize, boolean throwOnTooFewKeys, boolean noCheck, boolean wipe,
SemiOrderedShutdownHook storeShutdownHook) throws IOException,
DatabaseException {
+ public BerkeleyDBFreenetStore(Environment env, String prefix, File
storeFile, File fixSecondaryFile, long maxChkBlocks, int blockSize, int
headerSize, boolean throwOnTooFewKeys, boolean noCheck, boolean wipe,
SemiOrderedShutdownHook storeShutdownHook, File reconstructFile) throws
IOException, DatabaseException {
logMINOR = Logger.shouldLog(Logger.MINOR, this);
this.dataBlockSize = blockSize;
this.headerBlockSize = headerSize;
@@ -436,6 +486,7 @@
name = prefix;
this.maxChkBlocks=maxChkBlocks;
+ this.reconstructFile = reconstructFile;
environment = env;
@@ -549,11 +600,12 @@
new BlockNumberKeyCreator(storeBlockTupleBinding);
blockNoDbConfig.setKeyCreator(bnkc);
SecondaryDatabase blockNums = null;
+ String blockDBName = prefix+"CHK_blockNum";
try {
try {
System.err.println("Opening block db index");
blockNums = environment.openSecondaryDatabase
- (null, prefix+"CHK_blockNum", chkDB,
blockNoDbConfig);
+ (null, blockDBName, chkDB, blockNoDbConfig);
// The below is too slow to be useful, because
SecondaryDatabase.count() isn't optimised.
// long blockNumsCount = blockNums.count();
// long chkDBCount = chkDB.count();
@@ -568,20 +620,24 @@
// Either we find out what the maximum value is and we
do a static method somewhere ensuring
// it won't overflow ... or we debug the wrapper.
// NB: it might be a wrapper-version-missmatch problem
(nextgens)
- if(blockNums != null) blockNums.close();
+ if(blockNums != null) {
+ Logger.normal(this, "Closing database
"+blockDBName);
+ blockNums.close();
+ }
try {
- environment.removeDatabase(null,
prefix+"CHK_blockNum");
+ environment.removeDatabase(null, blockDBName);
} catch (DatabaseNotFoundException e1) {
// Ignore
+ System.err.println("Database "+blockDBName+"
does not exist deleting it");
}
- System.err.println("Reconstructing block numbers
index...");
- Logger.error(this, "Reconstructing block numbers
index...");
+ System.err.println("Reconstructing block numbers
index... ("+e+")");
+ Logger.error(this, "Reconstructing block numbers
index...", e);
System.err.println("Creating new block DB index");
- blockNoDbConfig.setSortedDuplicates(false);
+ blockNoDbConfig.setSortedDuplicates(true);
blockNoDbConfig.setAllowCreate(true);
blockNoDbConfig.setAllowPopulate(true);
blockNums = environment.openSecondaryDatabase
- (null, prefix+"CHK_blockNum", chkDB,
blockNoDbConfig);
+ (null, blockDBName, chkDB, blockNoDbConfig);
}
} catch (DatabaseException e1) {
// Log this now because close() will probably throw too
@@ -1131,18 +1187,20 @@
/**
* Recreate the index from the data file. Call this when the index has
been corrupted.
+ * @param reconstructFile
* @param the directory where the store is located
* @throws DatabaseException If the store cannot be opened because of a
database problem.
* @throws IOException If the store cannot be opened because of a
filesystem problem.
* @throws FileNotFoundException if the dir does not exist and could not
be created
*/
- public BerkeleyDBFreenetStore(Environment env, String prefix, File
storeFile, File fixSecondaryFile, long maxChkBlocks, int blockSize, int
headerSize, short type, boolean noCheck, SemiOrderedShutdownHook
storeShutdownHook) throws DatabaseException, IOException {
+ public BerkeleyDBFreenetStore(Environment env, String prefix, File
storeFile, File fixSecondaryFile, long maxChkBlocks, int blockSize, int
headerSize, short type, boolean noCheck, SemiOrderedShutdownHook
storeShutdownHook, File reconstructFile) throws DatabaseException, IOException {
logMINOR = Logger.shouldLog(Logger.MINOR, this);
this.dataBlockSize = blockSize;
this.headerBlockSize = headerSize;
this.freeBlocks = new SortedLongSet();
this.maxChkBlocks=maxChkBlocks;
this.environment = env;
+ this.reconstructFile = reconstructFile;
name = prefix;
wipeOldDatabases(prefix);
