On Thu, Sep 25, 2008 at 3:03 AM, Matthew Toseland
<toad at amphibian.dyndns.org> wrote:
> On Wednesday 24 September 2008 05:51, j16sdiz at freenetproject.org wrote:
>> Author: j16sdiz
>> Date: 2008-09-24 04:51:56 +0000 (Wed, 24 Sep 2008)
>> New Revision: 22789
>>
>> Modified:
>> trunk/freenet/src/freenet/l10n/freenet.l10n.en.properties
>> trunk/freenet/src/freenet/node/Node.java
>> trunk/freenet/src/freenet/store/saltedhash/SaltedHashFreenetStore.java
>> Log:
>> preallocate space for SaltedHashFreenetStore
>
> Is this the right place to do it? When exactly will it happen? During
> migration (offline), during startup (offline), when the size is increased
> from config (online) ??
it preallocate space for all size increase cases.
>>
>> Modified: trunk/freenet/src/freenet/l10n/freenet.l10n.en.properties
>> ===================================================================
>> --- trunk/freenet/src/freenet/l10n/freenet.l10n.en.properties 2008-09-23
> 23:46:32 UTC (rev 22788)
>> +++ trunk/freenet/src/freenet/l10n/freenet.l10n.en.properties 2008-09-24
> 04:51:56 UTC (rev 22789)
>> @@ -710,6 +710,8 @@
>> Node.storeDirectory=Store directory
>> Node.storeDirectoryLong=Name of directory to put store files in
>> Node.storeMaxMemTooHigh=Giving more than 80% of your ram to BDB is probably
> not what you want to do!
>> +Node.storePreallocate=Preallocate space for datastore
>> +Node.storePreallocateLong=Preallocate space for datastore
>> Node.storeSize=Store size in bytes
>> Node.storeSizeLong=Store size in bytes
>> Node.storeType=Store type (LEAVE THIS ALONE)
>>
>> Modified: trunk/freenet/src/freenet/node/Node.java
>> ===================================================================
>> --- trunk/freenet/src/freenet/node/Node.java 2008-09-23 23:46:32 UTC (rev
> 22788)
>> +++ trunk/freenet/src/freenet/node/Node.java 2008-09-24 04:51:56 UTC (rev
> 22789)
>> @@ -505,6 +505,8 @@
>> private static final int MIN_UPTIME_STORE_KEY = 40;
>>
>> private volatile boolean isPRNGReady = false;
>> +
>> + private boolean storePreallocate;
>>
>> /**
>> * Read all storable settings (identity etc) from the node file.
>> @@ -1604,6 +1606,28 @@
>> maxStoreKeys = maxTotalKeys / 2;
>> maxCacheKeys = maxTotalKeys - maxStoreKeys;
>>
>> + nodeConfig.register("storePreallocate", true, sortOrder++,
>> true,
> false, "Node.storePreallocate", "Node.storePreallocateLong",
>> + new BooleanCallback() {
>> + @Override
>> + public Boolean get() {
>> + return storePreallocate;
>> + }
>> +
>> + @Override
>> + public void set(Boolean val) throws
> InvalidConfigValueException, NodeNeedRestartException {
>> + storePreallocate = val;
>> + if
>> (storeType.equals("salt-hash")) {
>> +
>> ((SaltedHashFreenetStore)
> chkDatastore.getStore()).setPreallocate(val);
>> +
>> ((SaltedHashFreenetStore)
> chkDatacache.getStore()).setPreallocate(val);
>> +
>> ((SaltedHashFreenetStore)
> pubKeyDatastore.getStore()).setPreallocate(val);
>> +
>> ((SaltedHashFreenetStore)
> pubKeyDatacache.getStore()).setPreallocate(val);
>> +
>> ((SaltedHashFreenetStore)
> sskDatastore.getStore()).setPreallocate(val);
>> +
>> ((SaltedHashFreenetStore)
> sskDatacache.getStore()).setPreallocate(val);
>> + }
>> + }}
>> + );
>> + storePreallocate = nodeConfig.getBoolean("storePreallocate");
>> +
>> if (storeType.equals("salt-hash")) {
>> initSaltHashFS(suffix);
>> } else if (storeType.equals("bdb-index")) {
>> @@ -1744,32 +1768,32 @@
>> System.out.println("Initializing CHK Datastore (" +
>> maxStoreKeys + "
> keys)");
>> chkDatastore = new CHKStore();
>> SaltedHashFreenetStore chkDataFS =
> SaltedHashFreenetStore.construct(storeDir, "CHK-store", chkDatastore,
>> - random, maxStoreKeys, bloomFilterSizeInM,
> storeBloomFilterCounting, shutdownHook);
>> + random, maxStoreKeys, bloomFilterSizeInM,
> storeBloomFilterCounting, shutdownHook, storePreallocate);
>> Logger.normal(this, "Initializing CHK Datacache");
>> System.out.println("Initializing CHK Datacache (" +
>> maxCacheKeys + ':' +
> maxCacheKeys + " keys)");
>> chkDatacache = new CHKStore();
>> SaltedHashFreenetStore chkCacheFS =
> SaltedHashFreenetStore.construct(storeDir, "CHK-cache", chkDatacache,
>> - random, maxCacheKeys, bloomFilterSizeInM,
> storeBloomFilterCounting, shutdownHook);
>> + random, maxCacheKeys, bloomFilterSizeInM,
> storeBloomFilterCounting, shutdownHook, storePreallocate);
>> Logger.normal(this, "Initializing pubKey Datastore");
>> System.out.println("Initializing pubKey Datastore");
>> pubKeyDatastore = new PubkeyStore();
>> SaltedHashFreenetStore pubkeyDataFS =
> SaltedHashFreenetStore.construct(storeDir, "PUBKEY-store",
>> - pubKeyDatastore, random, maxStoreKeys,
>> bloomFilterSizeInM,
> storeBloomFilterCounting, shutdownHook);
>> + pubKeyDatastore, random, maxStoreKeys,
>> bloomFilterSizeInM,
> storeBloomFilterCounting, shutdownHook, storePreallocate);
>> Logger.normal(this, "Initializing pubKey Datacache");
>> System.out.println("Initializing pubKey Datacache (" +
>> maxCacheKeys + "
> keys)");
>> pubKeyDatacache = new PubkeyStore();
>> SaltedHashFreenetStore pubkeyCacheFS =
> SaltedHashFreenetStore.construct(storeDir, "PUBKEY-cache",
>> - pubKeyDatacache, random, maxCacheKeys,
>> bloomFilterSizeInM,
> storeBloomFilterCounting, shutdownHook);
>> + pubKeyDatacache, random, maxCacheKeys,
>> bloomFilterSizeInM,
> storeBloomFilterCounting, shutdownHook, storePreallocate);
>> Logger.normal(this, "Initializing SSK Datastore");
>> System.out.println("Initializing SSK Datastore");
>> sskDatastore = new SSKStore(this);
>> SaltedHashFreenetStore sskDataFS =
> SaltedHashFreenetStore.construct(storeDir, "SSK-store", sskDatastore,
>> - random, maxStoreKeys, bloomFilterSizeInM,
> storeBloomFilterCounting, shutdownHook);
>> + random, maxStoreKeys, bloomFilterSizeInM,
> storeBloomFilterCounting, shutdownHook, storePreallocate);
>> Logger.normal(this, "Initializing SSK Datacache");
>> System.out.println("Initializing SSK Datacache (" +
>> maxCacheKeys + "
> keys)");
>> sskDatacache = new SSKStore(this);
>> SaltedHashFreenetStore sskCacheFS =
> SaltedHashFreenetStore.construct(storeDir, "SSK-cache", sskDatacache,
>> - random, maxCacheKeys, bloomFilterSizeInM,
> storeBloomFilterCounting, shutdownHook);
>> + random, maxCacheKeys, bloomFilterSizeInM,
> storeBloomFilterCounting, shutdownHook, storePreallocate);
>>
>> File migrationFile = new File(storeDir, "migrated");
>> if (!migrationFile.exists()) {
>>
>> Modified:
> trunk/freenet/src/freenet/store/saltedhash/SaltedHashFreenetStore.java
>> ===================================================================
>> --- trunk/freenet/src/freenet/store/saltedhash/SaltedHashFreenetStore.java
> 2008-09-23 23:46:32 UTC (rev 22788)
>> +++ trunk/freenet/src/freenet/store/saltedhash/SaltedHashFreenetStore.java
> 2008-09-24 04:51:56 UTC (rev 22789)
>> @@ -79,16 +79,18 @@
>> private long storeSize;
>> private int generation;
>> private int flags;
>> +
>> + private boolean preallocate = true;
>>
>> public static SaltedHashFreenetStore construct(File baseDir, String
>> name,
> StoreCallback callback, Random random,
>> - long maxKeys, int bloomFilterSize, boolean bloomCounting,
> SemiOrderedShutdownHook shutdownHook)
>> + long maxKeys, int bloomFilterSize, boolean bloomCounting,
> SemiOrderedShutdownHook shutdownHook, boolean preallocate)
>> throws IOException {
>> return new SaltedHashFreenetStore(baseDir, name, callback,
>> random,
> maxKeys, bloomFilterSize, bloomCounting,
>> - shutdownHook);
>> + shutdownHook, preallocate);
>> }
>>
>> private SaltedHashFreenetStore(File baseDir, String name, StoreCallback
> callback, Random random, long maxKeys,
>> - int bloomFilterSize, boolean bloomCounting,
> SemiOrderedShutdownHook shutdownHook) throws IOException {
>> + int bloomFilterSize, boolean bloomCounting,
> SemiOrderedShutdownHook shutdownHook, boolean preallocate) throws IOException
> {
>> logMINOR = Logger.shouldLog(Logger.MINOR, this);
>> logDEBUG = Logger.shouldLog(Logger.DEBUG, this);
>>
>> @@ -107,6 +109,7 @@
>> this.random = random;
>> storeSize = maxKeys;
>> this.bloomFilterSize = bloomFilterSize;
>> + this.preallocate = preallocate;
>>
>> lockManager = new LockManager();
>>
>> @@ -746,14 +749,54 @@
>> }
>>
>> /**
>> + * Set preallocate storage space
>> + * @param preallocate
>> + */
>> + public void setPreallocate(boolean preallocate) {
>> + this.preallocate = preallocate;
>> + }
>> +
>> + /**
>> * Change on disk store file size
>> *
>> * @param storeFileSize
>> */
>> private void setStoreFileSize(long storeFileSize) {
>> try {
>> - metaRAF.setLength(Entry.METADATA_LENGTH *
>> storeFileSize);
>> - hdRAF.setLength((headerBlockLength + dataBlockLength +
>> hdPadding) *
> storeFileSize);
>> + long oldMetaLen = metaRAF.length();
>> + long oldHdLen = hdRAF.length();
>> +
>> + final long newMetaLen = Entry.METADATA_LENGTH *
>> storeFileSize;
>> + final long newHdLen = (headerBlockLength +
>> dataBlockLength + hdPadding)
> * storeFileSize;
>> +
>> + if (preallocate) {
>> + byte[] b = new byte[4096];
>> + ByteBuffer bf = ByteBuffer.wrap(b);
>> +
>> + // start from next 4KB boundary => align to
>> x86 page size
>> + if (oldMetaLen % 4096 != 0)
>> + oldMetaLen += 4096 - (oldMetaLen %
>> 4096);
>> + if (oldHdLen % 4096 != 0)
>> + oldHdLen += 4096 - (oldHdLen % 4096);
>> +
>> + // this may write excess the size, the
>> setLength() would fix it
>> + while (oldMetaLen < newMetaLen) {
>> + // never write random byte to meta
>> data!
>> + // this would screw up the isFree()
>> function
>> + bf.rewind();
>> + metaFC.write(bf, oldMetaLen);
>> + oldMetaLen += 4096;
>> + }
>> + while (oldHdLen < newHdLen) {
>> + random.nextBytes(b);
>> + bf.rewind();
>> + hdFC.write(bf, oldHdLen);
>> + oldHdLen += 4096;
>> + }
>> + }
>> +
>> + metaRAF.setLength(newMetaLen);
>> + hdRAF.setLength(newHdLen);
>> } catch (IOException e) {
>> Logger.error(this, "error resizing store file", e);
>> }
>> @@ -1686,5 +1729,5 @@
>> } finally {
>> configLock.readLock().unlock();
>> }
>> - }
>> + }
>> }
>>
>> _______________________________________________
>> cvs mailing list
>> cvs at freenetproject.org
>> http://emu.freenetproject.org/cgi-bin/mailman/listinfo/cvs
>>
>>
>
> _______________________________________________
> Devl mailing list
> Devl at freenetproject.org
> http://emu.freenetproject.org/cgi-bin/mailman/listinfo/devl
>
