Author: j16sdiz
Date: 2008-09-24 04:51:56 +0000 (Wed, 24 Sep 2008)
New Revision: 22789
Modified:
trunk/freenet/src/freenet/l10n/freenet.l10n.en.properties
trunk/freenet/src/freenet/node/Node.java
trunk/freenet/src/freenet/store/saltedhash/SaltedHashFreenetStore.java
Log:
preallocate space for SaltedHashFreenetStore
Modified: trunk/freenet/src/freenet/l10n/freenet.l10n.en.properties
===================================================================
--- trunk/freenet/src/freenet/l10n/freenet.l10n.en.properties 2008-09-23
23:46:32 UTC (rev 22788)
+++ trunk/freenet/src/freenet/l10n/freenet.l10n.en.properties 2008-09-24
04:51:56 UTC (rev 22789)
@@ -710,6 +710,8 @@
Node.storeDirectory=Store directory
Node.storeDirectoryLong=Name of directory to put store files in
Node.storeMaxMemTooHigh=Giving more than 80% of your ram to BDB is probably
not what you want to do!
+Node.storePreallocate=Preallocate space for datastore
+Node.storePreallocateLong=Preallocate space for datastore
Node.storeSize=Store size in bytes
Node.storeSizeLong=Store size in bytes
Node.storeType=Store type (LEAVE THIS ALONE)
Modified: trunk/freenet/src/freenet/node/Node.java
===================================================================
--- trunk/freenet/src/freenet/node/Node.java 2008-09-23 23:46:32 UTC (rev
22788)
+++ trunk/freenet/src/freenet/node/Node.java 2008-09-24 04:51:56 UTC (rev
22789)
@@ -505,6 +505,8 @@
private static final int MIN_UPTIME_STORE_KEY = 40;
private volatile boolean isPRNGReady = false;
+
+ private boolean storePreallocate;
/**
* Read all storable settings (identity etc) from the node file.
@@ -1604,6 +1606,28 @@
maxStoreKeys = maxTotalKeys / 2;
maxCacheKeys = maxTotalKeys - maxStoreKeys;
+ nodeConfig.register("storePreallocate", true, sortOrder++,
true, false, "Node.storePreallocate", "Node.storePreallocateLong",
+ new BooleanCallback() {
+ @Override
+ public Boolean get() {
+ return storePreallocate;
+ }
+
+ @Override
+ public void set(Boolean val) throws
InvalidConfigValueException, NodeNeedRestartException {
+ storePreallocate = val;
+ if
(storeType.equals("salt-hash")) {
+
((SaltedHashFreenetStore) chkDatastore.getStore()).setPreallocate(val);
+
((SaltedHashFreenetStore) chkDatacache.getStore()).setPreallocate(val);
+
((SaltedHashFreenetStore) pubKeyDatastore.getStore()).setPreallocate(val);
+
((SaltedHashFreenetStore) pubKeyDatacache.getStore()).setPreallocate(val);
+
((SaltedHashFreenetStore) sskDatastore.getStore()).setPreallocate(val);
+
((SaltedHashFreenetStore) sskDatacache.getStore()).setPreallocate(val);
+ }
+ }}
+ );
+ storePreallocate = nodeConfig.getBoolean("storePreallocate");
+
if (storeType.equals("salt-hash")) {
initSaltHashFS(suffix);
} else if (storeType.equals("bdb-index")) {
@@ -1744,32 +1768,32 @@
System.out.println("Initializing CHK Datastore (" +
maxStoreKeys + " keys)");
chkDatastore = new CHKStore();
SaltedHashFreenetStore chkDataFS =
SaltedHashFreenetStore.construct(storeDir, "CHK-store", chkDatastore,
- random, maxStoreKeys, bloomFilterSizeInM,
storeBloomFilterCounting, shutdownHook);
+ random, maxStoreKeys, bloomFilterSizeInM,
storeBloomFilterCounting, shutdownHook, storePreallocate);
Logger.normal(this, "Initializing CHK Datacache");
System.out.println("Initializing CHK Datacache (" +
maxCacheKeys + ':' + maxCacheKeys + " keys)");
chkDatacache = new CHKStore();
SaltedHashFreenetStore chkCacheFS =
SaltedHashFreenetStore.construct(storeDir, "CHK-cache", chkDatacache,
- random, maxCacheKeys, bloomFilterSizeInM,
storeBloomFilterCounting, shutdownHook);
+ random, maxCacheKeys, bloomFilterSizeInM,
storeBloomFilterCounting, shutdownHook, storePreallocate);
Logger.normal(this, "Initializing pubKey Datastore");
System.out.println("Initializing pubKey Datastore");
pubKeyDatastore = new PubkeyStore();
SaltedHashFreenetStore pubkeyDataFS =
SaltedHashFreenetStore.construct(storeDir, "PUBKEY-store",
- pubKeyDatastore, random, maxStoreKeys,
bloomFilterSizeInM, storeBloomFilterCounting, shutdownHook);
+ pubKeyDatastore, random, maxStoreKeys,
bloomFilterSizeInM, storeBloomFilterCounting, shutdownHook, storePreallocate);
Logger.normal(this, "Initializing pubKey Datacache");
System.out.println("Initializing pubKey Datacache (" +
maxCacheKeys + " keys)");
pubKeyDatacache = new PubkeyStore();
SaltedHashFreenetStore pubkeyCacheFS =
SaltedHashFreenetStore.construct(storeDir, "PUBKEY-cache",
- pubKeyDatacache, random, maxCacheKeys,
bloomFilterSizeInM, storeBloomFilterCounting, shutdownHook);
+ pubKeyDatacache, random, maxCacheKeys,
bloomFilterSizeInM, storeBloomFilterCounting, shutdownHook, storePreallocate);
Logger.normal(this, "Initializing SSK Datastore");
System.out.println("Initializing SSK Datastore");
sskDatastore = new SSKStore(this);
SaltedHashFreenetStore sskDataFS =
SaltedHashFreenetStore.construct(storeDir, "SSK-store", sskDatastore,
- random, maxStoreKeys, bloomFilterSizeInM,
storeBloomFilterCounting, shutdownHook);
+ random, maxStoreKeys, bloomFilterSizeInM,
storeBloomFilterCounting, shutdownHook, storePreallocate);
Logger.normal(this, "Initializing SSK Datacache");
System.out.println("Initializing SSK Datacache (" +
maxCacheKeys + " keys)");
sskDatacache = new SSKStore(this);
SaltedHashFreenetStore sskCacheFS =
SaltedHashFreenetStore.construct(storeDir, "SSK-cache", sskDatacache,
- random, maxCacheKeys, bloomFilterSizeInM,
storeBloomFilterCounting, shutdownHook);
+ random, maxCacheKeys, bloomFilterSizeInM,
storeBloomFilterCounting, shutdownHook, storePreallocate);
File migrationFile = new File(storeDir, "migrated");
if (!migrationFile.exists()) {
Modified: trunk/freenet/src/freenet/store/saltedhash/SaltedHashFreenetStore.java
===================================================================
--- trunk/freenet/src/freenet/store/saltedhash/SaltedHashFreenetStore.java
2008-09-23 23:46:32 UTC (rev 22788)
+++ trunk/freenet/src/freenet/store/saltedhash/SaltedHashFreenetStore.java
2008-09-24 04:51:56 UTC (rev 22789)
@@ -79,16 +79,18 @@
private long storeSize;
private int generation;
private int flags;
+
+ private boolean preallocate = true;
public static SaltedHashFreenetStore construct(File baseDir, String
name, StoreCallback callback, Random random,
- long maxKeys, int bloomFilterSize, boolean bloomCounting,
SemiOrderedShutdownHook shutdownHook)
+ long maxKeys, int bloomFilterSize, boolean bloomCounting,
SemiOrderedShutdownHook shutdownHook, boolean preallocate)
throws IOException {
return new SaltedHashFreenetStore(baseDir, name, callback,
random, maxKeys, bloomFilterSize, bloomCounting,
- shutdownHook);
+ shutdownHook, preallocate);
}
private SaltedHashFreenetStore(File baseDir, String name, StoreCallback
callback, Random random, long maxKeys,
- int bloomFilterSize, boolean bloomCounting,
SemiOrderedShutdownHook shutdownHook) throws IOException {
+ int bloomFilterSize, boolean bloomCounting,
SemiOrderedShutdownHook shutdownHook, boolean preallocate) throws IOException {
logMINOR = Logger.shouldLog(Logger.MINOR, this);
logDEBUG = Logger.shouldLog(Logger.DEBUG, this);
@@ -107,6 +109,7 @@
this.random = random;
storeSize = maxKeys;
this.bloomFilterSize = bloomFilterSize;
+ this.preallocate = preallocate;
lockManager = new LockManager();
@@ -746,14 +749,54 @@
}
/**
+ * Set preallocate storage space
+ * @param preallocate
+ */
+ public void setPreallocate(boolean preallocate) {
+ this.preallocate = preallocate;
+ }
+
+ /**
* Change on disk store file size
*
* @param storeFileSize
*/
private void setStoreFileSize(long storeFileSize) {
try {
- metaRAF.setLength(Entry.METADATA_LENGTH *
storeFileSize);
- hdRAF.setLength((headerBlockLength + dataBlockLength +
hdPadding) * storeFileSize);
+ long oldMetaLen = metaRAF.length();
+ long oldHdLen = hdRAF.length();
+
+ final long newMetaLen = Entry.METADATA_LENGTH *
storeFileSize;
+ final long newHdLen = (headerBlockLength +
dataBlockLength + hdPadding) * storeFileSize;
+
+ if (preallocate) {
+ byte[] b = new byte[4096];
+ ByteBuffer bf = ByteBuffer.wrap(b);
+
+ // start from next 4KB boundary => align to x86
page size
+ if (oldMetaLen % 4096 != 0)
+ oldMetaLen += 4096 - (oldMetaLen %
4096);
+ if (oldHdLen % 4096 != 0)
+ oldHdLen += 4096 - (oldHdLen % 4096);
+
+ // this may write excess the size, the
setLength() would fix it
+ while (oldMetaLen < newMetaLen) {
+ // never write random byte to meta data!
+ // this would screw up the isFree()
function
+ bf.rewind();
+ metaFC.write(bf, oldMetaLen);
+ oldMetaLen += 4096;
+ }
+ while (oldHdLen < newHdLen) {
+ random.nextBytes(b);
+ bf.rewind();
+ hdFC.write(bf, oldHdLen);
+ oldHdLen += 4096;
+ }
+ }
+
+ metaRAF.setLength(newMetaLen);
+ hdRAF.setLength(newHdLen);
} catch (IOException e) {
Logger.error(this, "error resizing store file", e);
}
@@ -1686,5 +1729,5 @@
} finally {
configLock.readLock().unlock();
}
- }
+ }
}
