On Wednesday 24 September 2008 05:51, j16sdiz at freenetproject.org wrote:
> Author: j16sdiz
> Date: 2008-09-24 04:51:56 +0000 (Wed, 24 Sep 2008)
> New Revision: 22789
>
> Modified:
> trunk/freenet/src/freenet/l10n/freenet.l10n.en.properties
> trunk/freenet/src/freenet/node/Node.java
> trunk/freenet/src/freenet/store/saltedhash/SaltedHashFreenetStore.java
> Log:
> preallocate space for SaltedHashFreenetStore
Is this the right place to do it? When exactly will it happen? During
migration (offline), during startup (offline), when the size is increased
from config (online) ??
>
> Modified: trunk/freenet/src/freenet/l10n/freenet.l10n.en.properties
> ===================================================================
> --- trunk/freenet/src/freenet/l10n/freenet.l10n.en.properties 2008-09-23
23:46:32 UTC (rev 22788)
> +++ trunk/freenet/src/freenet/l10n/freenet.l10n.en.properties 2008-09-24
04:51:56 UTC (rev 22789)
> @@ -710,6 +710,8 @@
> Node.storeDirectory=Store directory
> Node.storeDirectoryLong=Name of directory to put store files in
> Node.storeMaxMemTooHigh=Giving more than 80% of your ram to BDB is probably
not what you want to do!
> +Node.storePreallocate=Preallocate space for datastore
> +Node.storePreallocateLong=Preallocate space for datastore
> Node.storeSize=Store size in bytes
> Node.storeSizeLong=Store size in bytes
> Node.storeType=Store type (LEAVE THIS ALONE)
>
> Modified: trunk/freenet/src/freenet/node/Node.java
> ===================================================================
> --- trunk/freenet/src/freenet/node/Node.java 2008-09-23 23:46:32 UTC (rev
22788)
> +++ trunk/freenet/src/freenet/node/Node.java 2008-09-24 04:51:56 UTC (rev
22789)
> @@ -505,6 +505,8 @@
> private static final int MIN_UPTIME_STORE_KEY = 40;
>
> private volatile boolean isPRNGReady = false;
> +
> + private boolean storePreallocate;
>
> /**
> * Read all storable settings (identity etc) from the node file.
> @@ -1604,6 +1606,28 @@
> maxStoreKeys = maxTotalKeys / 2;
> maxCacheKeys = maxTotalKeys - maxStoreKeys;
>
> + nodeConfig.register("storePreallocate", true, sortOrder++,
> true,
false, "Node.storePreallocate", "Node.storePreallocateLong",
> + new BooleanCallback() {
> + @Override
> + public Boolean get() {
> + return storePreallocate;
> + }
> +
> + @Override
> + public void set(Boolean val) throws
InvalidConfigValueException, NodeNeedRestartException {
> + storePreallocate = val;
> + if
> (storeType.equals("salt-hash")) {
> +
> ((SaltedHashFreenetStore)
chkDatastore.getStore()).setPreallocate(val);
> +
> ((SaltedHashFreenetStore)
chkDatacache.getStore()).setPreallocate(val);
> +
> ((SaltedHashFreenetStore)
pubKeyDatastore.getStore()).setPreallocate(val);
> +
> ((SaltedHashFreenetStore)
pubKeyDatacache.getStore()).setPreallocate(val);
> +
> ((SaltedHashFreenetStore)
sskDatastore.getStore()).setPreallocate(val);
> +
> ((SaltedHashFreenetStore)
sskDatacache.getStore()).setPreallocate(val);
> + }
> + }}
> + );
> + storePreallocate = nodeConfig.getBoolean("storePreallocate");
> +
> if (storeType.equals("salt-hash")) {
> initSaltHashFS(suffix);
> } else if (storeType.equals("bdb-index")) {
> @@ -1744,32 +1768,32 @@
> System.out.println("Initializing CHK Datastore (" +
> maxStoreKeys + "
keys)");
> chkDatastore = new CHKStore();
> SaltedHashFreenetStore chkDataFS =
SaltedHashFreenetStore.construct(storeDir, "CHK-store", chkDatastore,
> - random, maxStoreKeys, bloomFilterSizeInM,
storeBloomFilterCounting, shutdownHook);
> + random, maxStoreKeys, bloomFilterSizeInM,
storeBloomFilterCounting, shutdownHook, storePreallocate);
> Logger.normal(this, "Initializing CHK Datacache");
> System.out.println("Initializing CHK Datacache (" +
> maxCacheKeys + ':' +
maxCacheKeys + " keys)");
> chkDatacache = new CHKStore();
> SaltedHashFreenetStore chkCacheFS =
SaltedHashFreenetStore.construct(storeDir, "CHK-cache", chkDatacache,
> - random, maxCacheKeys, bloomFilterSizeInM,
storeBloomFilterCounting, shutdownHook);
> + random, maxCacheKeys, bloomFilterSizeInM,
storeBloomFilterCounting, shutdownHook, storePreallocate);
> Logger.normal(this, "Initializing pubKey Datastore");
> System.out.println("Initializing pubKey Datastore");
> pubKeyDatastore = new PubkeyStore();
> SaltedHashFreenetStore pubkeyDataFS =
SaltedHashFreenetStore.construct(storeDir, "PUBKEY-store",
> - pubKeyDatastore, random, maxStoreKeys,
> bloomFilterSizeInM,
storeBloomFilterCounting, shutdownHook);
> + pubKeyDatastore, random, maxStoreKeys,
> bloomFilterSizeInM,
storeBloomFilterCounting, shutdownHook, storePreallocate);
> Logger.normal(this, "Initializing pubKey Datacache");
> System.out.println("Initializing pubKey Datacache (" +
> maxCacheKeys + "
keys)");
> pubKeyDatacache = new PubkeyStore();
> SaltedHashFreenetStore pubkeyCacheFS =
SaltedHashFreenetStore.construct(storeDir, "PUBKEY-cache",
> - pubKeyDatacache, random, maxCacheKeys,
> bloomFilterSizeInM,
storeBloomFilterCounting, shutdownHook);
> + pubKeyDatacache, random, maxCacheKeys,
> bloomFilterSizeInM,
storeBloomFilterCounting, shutdownHook, storePreallocate);
> Logger.normal(this, "Initializing SSK Datastore");
> System.out.println("Initializing SSK Datastore");
> sskDatastore = new SSKStore(this);
> SaltedHashFreenetStore sskDataFS =
SaltedHashFreenetStore.construct(storeDir, "SSK-store", sskDatastore,
> - random, maxStoreKeys, bloomFilterSizeInM,
storeBloomFilterCounting, shutdownHook);
> + random, maxStoreKeys, bloomFilterSizeInM,
storeBloomFilterCounting, shutdownHook, storePreallocate);
> Logger.normal(this, "Initializing SSK Datacache");
> System.out.println("Initializing SSK Datacache (" +
> maxCacheKeys + "
keys)");
> sskDatacache = new SSKStore(this);
> SaltedHashFreenetStore sskCacheFS =
SaltedHashFreenetStore.construct(storeDir, "SSK-cache", sskDatacache,
> - random, maxCacheKeys, bloomFilterSizeInM,
storeBloomFilterCounting, shutdownHook);
> + random, maxCacheKeys, bloomFilterSizeInM,
storeBloomFilterCounting, shutdownHook, storePreallocate);
>
> File migrationFile = new File(storeDir, "migrated");
> if (!migrationFile.exists()) {
>
> Modified:
trunk/freenet/src/freenet/store/saltedhash/SaltedHashFreenetStore.java
> ===================================================================
> --- trunk/freenet/src/freenet/store/saltedhash/SaltedHashFreenetStore.java
2008-09-23 23:46:32 UTC (rev 22788)
> +++ trunk/freenet/src/freenet/store/saltedhash/SaltedHashFreenetStore.java
2008-09-24 04:51:56 UTC (rev 22789)
> @@ -79,16 +79,18 @@
> private long storeSize;
> private int generation;
> private int flags;
> +
> + private boolean preallocate = true;
>
> public static SaltedHashFreenetStore construct(File baseDir, String
> name,
StoreCallback callback, Random random,
> - long maxKeys, int bloomFilterSize, boolean bloomCounting,
SemiOrderedShutdownHook shutdownHook)
> + long maxKeys, int bloomFilterSize, boolean bloomCounting,
SemiOrderedShutdownHook shutdownHook, boolean preallocate)
> throws IOException {
> return new SaltedHashFreenetStore(baseDir, name, callback,
> random,
maxKeys, bloomFilterSize, bloomCounting,
> - shutdownHook);
> + shutdownHook, preallocate);
> }
>
> private SaltedHashFreenetStore(File baseDir, String name, StoreCallback
callback, Random random, long maxKeys,
> - int bloomFilterSize, boolean bloomCounting,
SemiOrderedShutdownHook shutdownHook) throws IOException {
> + int bloomFilterSize, boolean bloomCounting,
SemiOrderedShutdownHook shutdownHook, boolean preallocate) throws IOException
{
> logMINOR = Logger.shouldLog(Logger.MINOR, this);
> logDEBUG = Logger.shouldLog(Logger.DEBUG, this);
>
> @@ -107,6 +109,7 @@
> this.random = random;
> storeSize = maxKeys;
> this.bloomFilterSize = bloomFilterSize;
> + this.preallocate = preallocate;
>
> lockManager = new LockManager();
>
> @@ -746,14 +749,54 @@
> }
>
> /**
> + * Set preallocate storage space
> + * @param preallocate
> + */
> + public void setPreallocate(boolean preallocate) {
> + this.preallocate = preallocate;
> + }
> +
> + /**
> * Change on disk store file size
> *
> * @param storeFileSize
> */
> private void setStoreFileSize(long storeFileSize) {
> try {
> - metaRAF.setLength(Entry.METADATA_LENGTH *
> storeFileSize);
> - hdRAF.setLength((headerBlockLength + dataBlockLength +
> hdPadding) *
storeFileSize);
> + long oldMetaLen = metaRAF.length();
> + long oldHdLen = hdRAF.length();
> +
> + final long newMetaLen = Entry.METADATA_LENGTH *
> storeFileSize;
> + final long newHdLen = (headerBlockLength +
> dataBlockLength + hdPadding)
* storeFileSize;
> +
> + if (preallocate) {
> + byte[] b = new byte[4096];
> + ByteBuffer bf = ByteBuffer.wrap(b);
> +
> + // start from next 4KB boundary => align to x86
> page size
> + if (oldMetaLen % 4096 != 0)
> + oldMetaLen += 4096 - (oldMetaLen %
> 4096);
> + if (oldHdLen % 4096 != 0)
> + oldHdLen += 4096 - (oldHdLen % 4096);
> +
> + // this may write excess the size, the
> setLength() would fix it
> + while (oldMetaLen < newMetaLen) {
> + // never write random byte to meta data!
> + // this would screw up the isFree()
> function
> + bf.rewind();
> + metaFC.write(bf, oldMetaLen);
> + oldMetaLen += 4096;
> + }
> + while (oldHdLen < newHdLen) {
> + random.nextBytes(b);
> + bf.rewind();
> + hdFC.write(bf, oldHdLen);
> + oldHdLen += 4096;
> + }
> + }
> +
> + metaRAF.setLength(newMetaLen);
> + hdRAF.setLength(newHdLen);
> } catch (IOException e) {
> Logger.error(this, "error resizing store file", e);
> }
> @@ -1686,5 +1729,5 @@
> } finally {
> configLock.readLock().unlock();
> }
> - }
> + }
> }
>
> _______________________________________________
> cvs mailing list
> cvs at freenetproject.org
> http://emu.freenetproject.org/cgi-bin/mailman/listinfo/cvs
>
>
-------------- next part --------------
A non-text attachment was scrubbed...
Name: not available
Type: application/pgp-signature
Size: 827 bytes
Desc: not available
URL:
<https://emu.freenetproject.org/pipermail/devl/attachments/20080924/02391acd/attachment.pgp>
